Spaces:
Runtime error
Runtime error
| # ------------------------------------------------------------------------ | |
| # HOTR official code : hotr/data/evaluators/hico_eval.py | |
| # Copyright (c) Kakao Brain, Inc. and its affiliates. All Rights Reserved | |
| # ------------------------------------------------------------------------ | |
| # Modified from QPIC (https://github.com/hitachi-rd-cv/qpic) | |
| # Copyright (c) Hitachi, Ltd. All Rights Reserved. | |
| # Licensed under the Apache License, Version 2.0 [see LICENSE for details] | |
| # ------------------------------------------------------------------------ | |
| import numpy as np | |
| from collections import defaultdict | |
| class HICOEvaluator(): | |
| def __init__(self, preds, gts, rare_triplets, non_rare_triplets, correct_mat): | |
| self.overlap_iou = 0.5 | |
| self.max_hois = 100 | |
| self.rare_triplets = rare_triplets | |
| self.non_rare_triplets = non_rare_triplets | |
| self.fp = defaultdict(list) | |
| self.tp = defaultdict(list) | |
| self.score = defaultdict(list) | |
| self.sum_gts = defaultdict(lambda: 0) | |
| self.gt_triplets = [] | |
| self.preds = [] | |
| for img_preds in preds: | |
| img_preds = {k: v.to('cpu').numpy() for k, v in img_preds.items() if k != 'hoi_recognition_time'} | |
| bboxes = [{'bbox': bbox, 'category_id': label} for bbox, label in zip(img_preds['boxes'], img_preds['labels'])] | |
| hoi_scores = img_preds['verb_scores'] | |
| verb_labels = np.tile(np.arange(hoi_scores.shape[1]), (hoi_scores.shape[0], 1)) | |
| subject_ids = np.tile(img_preds['sub_ids'], (hoi_scores.shape[1], 1)).T | |
| object_ids = np.tile(img_preds['obj_ids'], (hoi_scores.shape[1], 1)).T | |
| hoi_scores = hoi_scores.ravel() | |
| verb_labels = verb_labels.ravel() | |
| subject_ids = subject_ids.ravel() | |
| object_ids = object_ids.ravel() | |
| if len(subject_ids) > 0: | |
| object_labels = np.array([bboxes[object_id]['category_id'] for object_id in object_ids]) | |
| masks = correct_mat[verb_labels, object_labels] | |
| hoi_scores *= masks | |
| hois = [{'subject_id': subject_id, 'object_id': object_id, 'category_id': category_id, 'score': score} for | |
| subject_id, object_id, category_id, score in zip(subject_ids, object_ids, verb_labels, hoi_scores)] | |
| hois.sort(key=lambda k: (k.get('score', 0)), reverse=True) | |
| hois = hois[:self.max_hois] | |
| else: | |
| hois = [] | |
| self.preds.append({ | |
| 'predictions': bboxes, | |
| 'hoi_prediction': hois | |
| }) | |
| self.gts = [] | |
| for img_gts in gts: | |
| img_gts = {k: v.to('cpu').numpy() for k, v in img_gts.items() if k != 'id'} | |
| self.gts.append({ | |
| 'annotations': [{'bbox': bbox, 'category_id': label} for bbox, label in zip(img_gts['boxes'], img_gts['labels'])], | |
| 'hoi_annotation': [{'subject_id': hoi[0], 'object_id': hoi[1], 'category_id': hoi[2]} for hoi in img_gts['hois']] | |
| }) | |
| for hoi in self.gts[-1]['hoi_annotation']: | |
| triplet = (self.gts[-1]['annotations'][hoi['subject_id']]['category_id'], | |
| self.gts[-1]['annotations'][hoi['object_id']]['category_id'], | |
| hoi['category_id']) | |
| if triplet not in self.gt_triplets: | |
| self.gt_triplets.append(triplet) | |
| self.sum_gts[triplet] += 1 | |
| def evaluate(self): | |
| for img_id, (img_preds, img_gts) in enumerate(zip(self.preds, self.gts)): | |
| print(f"Evaluating Score Matrix... : [{(img_id+1):>4}/{len(self.gts):<4}]" ,flush=True, end="\r") | |
| pred_bboxes = img_preds['predictions'] | |
| gt_bboxes = img_gts['annotations'] | |
| pred_hois = img_preds['hoi_prediction'] | |
| gt_hois = img_gts['hoi_annotation'] | |
| if len(gt_bboxes) != 0: | |
| bbox_pairs, bbox_overlaps = self.compute_iou_mat(gt_bboxes, pred_bboxes) | |
| self.compute_fptp(pred_hois, gt_hois, bbox_pairs, pred_bboxes, bbox_overlaps) | |
| else: | |
| for pred_hoi in pred_hois: | |
| triplet = [pred_bboxes[pred_hoi['subject_id']]['category_id'], | |
| pred_bboxes[pred_hoi['object_id']]['category_id'], pred_hoi['category_id']] | |
| if triplet not in self.gt_triplets: | |
| continue | |
| self.tp[triplet].append(0) | |
| self.fp[triplet].append(1) | |
| self.score[triplet].append(pred_hoi['score']) | |
| print(f"[stats] Score Matrix Generation completed!! ") | |
| map = self.compute_map() | |
| return map | |
| def compute_map(self): | |
| ap = defaultdict(lambda: 0) | |
| rare_ap = defaultdict(lambda: 0) | |
| non_rare_ap = defaultdict(lambda: 0) | |
| max_recall = defaultdict(lambda: 0) | |
| for triplet in self.gt_triplets: | |
| sum_gts = self.sum_gts[triplet] | |
| if sum_gts == 0: | |
| continue | |
| tp = np.array((self.tp[triplet])) | |
| fp = np.array((self.fp[triplet])) | |
| if len(tp) == 0: | |
| ap[triplet] = 0 | |
| max_recall[triplet] = 0 | |
| if triplet in self.rare_triplets: | |
| rare_ap[triplet] = 0 | |
| elif triplet in self.non_rare_triplets: | |
| non_rare_ap[triplet] = 0 | |
| else: | |
| print('Warning: triplet {} is neither in rare triplets nor in non-rare triplets'.format(triplet)) | |
| continue | |
| score = np.array(self.score[triplet]) | |
| sort_inds = np.argsort(-score) | |
| fp = fp[sort_inds] | |
| tp = tp[sort_inds] | |
| fp = np.cumsum(fp) | |
| tp = np.cumsum(tp) | |
| rec = tp / sum_gts | |
| prec = tp / (fp + tp) | |
| ap[triplet] = self.voc_ap(rec, prec) | |
| max_recall[triplet] = np.amax(rec) | |
| if triplet in self.rare_triplets: | |
| rare_ap[triplet] = ap[triplet] | |
| elif triplet in self.non_rare_triplets: | |
| non_rare_ap[triplet] = ap[triplet] | |
| else: | |
| print('Warning: triplet {} is neither in rare triplets nor in non-rare triplets'.format(triplet)) | |
| m_ap = np.mean(list(ap.values())) * 100 # percentage | |
| m_ap_rare = np.mean(list(rare_ap.values())) * 100 # percentage | |
| m_ap_non_rare = np.mean(list(non_rare_ap.values())) * 100 # percentage | |
| m_max_recall = np.mean(list(max_recall.values())) | |
| return {'mAP': m_ap, 'mAP rare': m_ap_rare, 'mAP non-rare': m_ap_non_rare, 'mean max recall': m_max_recall} | |
| def voc_ap(self, rec, prec): | |
| ap = 0. | |
| for t in np.arange(0., 1.1, 0.1): | |
| if np.sum(rec >= t) == 0: | |
| p = 0 | |
| else: | |
| p = np.max(prec[rec >= t]) | |
| ap = ap + p / 11. | |
| return ap | |
| def compute_fptp(self, pred_hois, gt_hois, match_pairs, pred_bboxes, bbox_overlaps): | |
| pos_pred_ids = match_pairs.keys() | |
| vis_tag = np.zeros(len(gt_hois)) | |
| pred_hois.sort(key=lambda k: (k.get('score', 0)), reverse=True) | |
| if len(pred_hois) != 0: | |
| for pred_hoi in pred_hois: | |
| is_match = 0 | |
| if len(match_pairs) != 0 and pred_hoi['subject_id'] in pos_pred_ids and pred_hoi['object_id'] in pos_pred_ids: | |
| pred_sub_ids = match_pairs[pred_hoi['subject_id']] | |
| pred_obj_ids = match_pairs[pred_hoi['object_id']] | |
| pred_sub_overlaps = bbox_overlaps[pred_hoi['subject_id']] | |
| pred_obj_overlaps = bbox_overlaps[pred_hoi['object_id']] | |
| pred_category_id = pred_hoi['category_id'] | |
| max_overlap = 0 | |
| max_gt_hoi = 0 | |
| for gt_hoi in gt_hois: | |
| if gt_hoi['subject_id'] in pred_sub_ids and gt_hoi['object_id'] in pred_obj_ids \ | |
| and pred_category_id == gt_hoi['category_id']: | |
| is_match = 1 | |
| min_overlap_gt = min(pred_sub_overlaps[pred_sub_ids.index(gt_hoi['subject_id'])], | |
| pred_obj_overlaps[pred_obj_ids.index(gt_hoi['object_id'])]) | |
| if min_overlap_gt > max_overlap: | |
| max_overlap = min_overlap_gt | |
| max_gt_hoi = gt_hoi | |
| triplet = (pred_bboxes[pred_hoi['subject_id']]['category_id'], pred_bboxes[pred_hoi['object_id']]['category_id'], | |
| pred_hoi['category_id']) | |
| if triplet not in self.gt_triplets: | |
| continue | |
| if is_match == 1 and vis_tag[gt_hois.index(max_gt_hoi)] == 0: | |
| self.fp[triplet].append(0) | |
| self.tp[triplet].append(1) | |
| vis_tag[gt_hois.index(max_gt_hoi)] =1 | |
| else: | |
| self.fp[triplet].append(1) | |
| self.tp[triplet].append(0) | |
| self.score[triplet].append(pred_hoi['score']) | |
| def compute_iou_mat(self, bbox_list1, bbox_list2): | |
| iou_mat = np.zeros((len(bbox_list1), len(bbox_list2))) | |
| if len(bbox_list1) == 0 or len(bbox_list2) == 0: | |
| return {} | |
| for i, bbox1 in enumerate(bbox_list1): | |
| for j, bbox2 in enumerate(bbox_list2): | |
| iou_i = self.compute_IOU(bbox1, bbox2) | |
| iou_mat[i, j] = iou_i | |
| iou_mat_ov=iou_mat.copy() | |
| iou_mat[iou_mat>=self.overlap_iou] = 1 | |
| iou_mat[iou_mat<self.overlap_iou] = 0 | |
| match_pairs = np.nonzero(iou_mat) | |
| match_pairs_dict = {} | |
| match_pair_overlaps = {} | |
| if iou_mat.max() > 0: | |
| for i, pred_id in enumerate(match_pairs[1]): | |
| if pred_id not in match_pairs_dict.keys(): | |
| match_pairs_dict[pred_id] = [] | |
| match_pair_overlaps[pred_id]=[] | |
| match_pairs_dict[pred_id].append(match_pairs[0][i]) | |
| match_pair_overlaps[pred_id].append(iou_mat_ov[match_pairs[0][i],pred_id]) | |
| return match_pairs_dict, match_pair_overlaps | |
| def compute_IOU(self, bbox1, bbox2): | |
| if isinstance(bbox1['category_id'], str): | |
| bbox1['category_id'] = int(bbox1['category_id'].replace('\n', '')) | |
| if isinstance(bbox2['category_id'], str): | |
| bbox2['category_id'] = int(bbox2['category_id'].replace('\n', '')) | |
| if bbox1['category_id'] == bbox2['category_id']: | |
| rec1 = bbox1['bbox'] | |
| rec2 = bbox2['bbox'] | |
| # computing area of each rectangles | |
| S_rec1 = (rec1[2] - rec1[0]+1) * (rec1[3] - rec1[1]+1) | |
| S_rec2 = (rec2[2] - rec2[0]+1) * (rec2[3] - rec2[1]+1) | |
| # computing the sum_area | |
| sum_area = S_rec1 + S_rec2 | |
| # find the each edge of intersect rectangle | |
| left_line = max(rec1[1], rec2[1]) | |
| right_line = min(rec1[3], rec2[3]) | |
| top_line = max(rec1[0], rec2[0]) | |
| bottom_line = min(rec1[2], rec2[2]) | |
| # judge if there is an intersect | |
| if left_line >= right_line or top_line >= bottom_line: | |
| return 0 | |
| else: | |
| intersect = (right_line - left_line+1) * (bottom_line - top_line+1) | |
| return intersect / (sum_area - intersect) | |
| else: | |
| return 0 |