23

.gitattributes

@@ -0,0 +1,35 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -137,3 +137,6 @@ Makefile
 #datasets 
 hico_20160224_det
 v-coco
+
+# *.ipynb
+vis_res

hotr/models/hotr.py

@@ -182,7 +182,7 @@ class HOTR(nn.Module):
 
         H_Pointer_reprs_bag=torch.cat(H_Pointer_reprs_bag,1)
         O_Pointer_reprs_bag=torch.cat(O_Pointer_reprs_bag,1)
-        
+        # import pdb;pdb.set_trace()
         outputs_hidx = [(torch.bmm(H_Pointer_repr, inst_repr_all)) / self.tau for H_Pointer_repr in H_Pointer_reprs_bag] #(dec_layer,(1+len(aug))*bs,dec_q,hidden_dim)
         outputs_oidx = [(torch.bmm(O_Pointer_repr, inst_repr_all)) / self.tau for O_Pointer_repr in O_Pointer_reprs_bag]
 

hotr/util/misc.py

@@ -22,7 +22,7 @@ from torch import Tensor
 
 # needed due to empty tensor bug in pytorch and torchvision 0.5
 import torchvision
-if float(torchvision.__version__[:3]) < 0.7:
+if float(torchvision.__version__.split('.',2)[1]) < 5:
     from torchvision.ops import _new_empty_tensor
     from torchvision.ops.misc import _output_size
 
@@ -388,7 +388,7 @@ def interpolate(input, size=None, scale_factor=None, mode="nearest", align_corne
     This will eventually be supported natively by PyTorch, and this
     class can go away.
     """
-    if float(torchvision.__version__[:3]) < 0.7:
+    if float(torchvision.__version__.split('.',2)[1]) < 5:
         if input.numel() > 0:
             return torch.nn.functional.interpolate(
                 input, size, scale_factor, mode, align_corners

requirements.txt

@@ -0,0 +1,5 @@
+pycocotools
+opencv-python
+wandb
+imageio
+scipy

tools/vis_tool.py

@@ -0,0 +1,96 @@
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import numpy as np
+import cv2
+
+
+vcoco_action_string = {2: 'hold', 3: 'stand', 4: 'sit', 5: 'ride', 6: 'walk',\
+                       7: 'look', 8: 'hit_inst', 9: 'hit_obj', 10: 'eat_obj', \
+                       11: 'eat_inst', 12: 'jump', 13: 'lay', 14: 'talk', 15: \
+                       'carry', 16: 'throw', 17: 'catch', 18: 'cut_inst', 19:'cut_obj', \
+                       20: 'run', 21: 'work_on_comp', 22: 'ski', 23: 'surf', 24: 'skateboard', \
+                       25: 'smile', 26: 'drink', 27: 'kick', 28: 'point', 29: 'read', 30: 'snowboard'}
+def draw_box_on_img(box, img,color=None):
+
+    vis_img = img.copy()
+    box = [int(x) for x in box]
+    cv2.rectangle(vis_img, (box[0], box[1]), (box[2], box[3]), color, 2)
+    draw_point=[int((box[0]+box[2])*1.0/2),int((box[1]+box[3])*1.0/2)]
+    
+    return vis_img,color
+
+
+def draw_line_on_img_vcoco(box,line, img, class_index,color):
+    
+    vis_img = img.copy()
+    font=cv2.FONT_HERSHEY_SIMPLEX
+    x=int(box[0])+2
+    y=int(box[1])+2
+    f=int(box[1])+2
+    for i in range(len(class_index)):
+     
+        font_scale=1
+        font_thickness=2
+    
+        text_size, _ = cv2.getTextSize(vcoco_action_string[class_index[i]] , font, font_scale, font_thickness)
+        vis_img=cv2.rectangle(vis_img,(x,y),(x+text_size[0],y+text_size[1]+5),color[1],-1)
+        
+
+        vis_img=cv2.putText(vis_img, vcoco_action_string[class_index[i]] ,(x,y + text_size[1] ),font,font_scale,[51,255,153],font_thickness)
+        y=y+text_size[1]+5
+   
+    return vis_img,y
+
+
+def draw_img_vcoco(img, output_i, top_k,threshold,color):
+    list_action = []
+    for action in output_i['hoi_prediction']:
+        subject_id = action['subject_id']
+        object_id = action['object_id']
+        category_id = action['category_id']
+        score = action['score']
+        single_out = [subject_id,object_id,category_id,score]
+        list_action.append(single_out)
+    list_action = sorted(list_action, key=lambda x:x[-1], reverse=True)
+    action_dict = []
+    action_cate = []
+    action_color=[]
+    subj_box=[]
+    sb={}
+    sbj=[]
+    for action in list_action[:top_k]:
+
+        subject_id,object_id,category_id,score = action
+        if score<threshold:
+            break
+        subject_obj = output_i['predictions'][subject_id]
+        subject_box = subject_obj['bbox']
+        object_obj = output_i['predictions'][object_id]
+        object_box = object_obj['bbox']
+
+        point_1 = [int((subject_box[0]+subject_box[2])*1.0/2),int((subject_box[1]+subject_box[3])*1.0/2)]
+        point_2 = [int((object_box[0]+object_box[2])*1.0/2),int((object_box[1]+object_box[3])*1.0/2)]
+  
+        if [point_1,point_2] not in action_dict:
+            
+            img,color_hum = draw_box_on_img(subject_box, img, color[subject_obj['category_id']]['color'])
+            
+            img,color_obj = draw_box_on_img(object_box, img, color[object_obj['category_id']]['color'])
+            
+            action_dict.append([point_1,point_2])
+            action_color.append([color_hum,color_obj])
+            subj_box.append([int(subject_box[0]),int(subject_box[1])])    
+           
+        action_cate.append([])
+        action_cate[action_dict.index([point_1,point_2])].append(category_id)
+
+    for i,(action_item,clr) in enumerate(zip(action_dict,action_color)):
+        
+        img,offset = draw_line_on_img_vcoco(subj_box[i],action_item,img,action_cate[action_dict.index(action_item)],clr)
+
+        for p in range(i+1,len(subj_box)):
+            if subj_box[p]==subj_box[i]:
+                subj_box[p][1]=offset
+    return img

upload_checkpoint_hugginface.ipynb

@@ -0,0 +1,123 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "a19d0964-4c83-4bc9-b59f-f04a57ca020f",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "5dba33c11d7a4f708b5d6a03869ccb30",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "VBox(children=(HTML(value='<center> <img\\nsrc=https://huggingface.co/front/assets/huggingface_logo-noborder.sv…"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "from huggingface_hub import notebook_login\n",
+    " \n",
+    "notebook_login()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "4b0ba848-4799-476d-b1d6-16a7cde1f4ad",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Cloning https://huggingface.co/jhp/hoi into local empty directory.\n"
+     ]
+    }
+   ],
+   "source": [
+    "from huggingface_hub import Repository\n",
+    " \n",
+    "repo = Repository('CPC_HOTR', clone_from='jhp/hoi')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "id": "5d092b5a-99b3-4a4c-9ef4-90724eb665ef",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "b51f83592a414a1ea646717c434e2a23",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "checkpoint.pth:   0%|          | 0.00/301M [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/plain": [
+       "'https://huggingface.co/jhp/hoi/tree/main/./'"
+      ]
+     },
+     "execution_count": 9,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "from huggingface_hub import HfApi\n",
+    "api = HfApi()\n",
+    "api.upload_folder(\n",
+    "    folder_path=\"./checkpoints/\",\n",
+    "    repo_id=\"jhp/hoi\",\n",
+    "    path_in_repo=\"./\",\n",
+    "    # allow_patterns=\"*.txt\", # Upload all local text files\n",
+    "    # delete_patterns=\"*.txt\", # Delete all remote text files before\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "82eedf6b-21a2-4dad-bc7d-6f3631525ff2",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.9.17"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

visualization.py

@@ -0,0 +1,227 @@
+import argparse
+import datetime
+import json
+import random
+import time
+import multiprocessing
+from pathlib import Path
+import os
+import cv2
+import numpy as np
+import torch
+from torch.utils.data import DataLoader, DistributedSampler
+import hotr.data.datasets as datasets
+import hotr.util.misc as utils
+from hotr.engine.arg_parser import get_args_parser
+from hotr.data.datasets import build_dataset, get_coco_api_from_dataset
+from hotr.data.datasets.vcoco import make_hoi_transforms
+from PIL import Image
+from hotr.util.logger import print_params, print_args
+
+import copy
+from hotr.data.datasets import builtin_meta
+from PIL import Image
+import requests
+# import mmcv
+from matplotlib import pyplot as plt
+import imageio
+
+from tools.vis_tool import *
+from hotr.models.detr import build
+
+def change_format(results,valid_ids):
+    
+    boxes,labels,pair_score =\
+                    list(map(lambda x: x.cpu().numpy(), [results['boxes'], results['labels'], results['pair_score']]))
+    output_i={}
+    output_i['predictions']=[]
+    output_i['hoi_prediction']=[]
+
+    h_idx=np.where(labels==1)[0]
+    for box,label in zip(boxes,labels):
+        
+        output_i['predictions'].append({'bbox':box.tolist(),'category_id':label})
+    
+    for i,verb in enumerate(pair_score):
+        if i in [1,4,10,23,26,5,18]:
+            continue
+        for j,hum in enumerate(h_idx):
+            for k in range(len(boxes)):
+                if verb[j][k]>0:
+                    output_i['hoi_prediction'].append({'subject_id':hum,'object_id':k,'category_id':i+2,'score':verb[j][k]})
+            
+    return output_i
+def vis(args,id=294):
+
+    if args.frozen_weights is not None:
+        print("Freeze weights for detector")
+
+    device = torch.device(args.device)
+
+    # fix the seed for reproducibility
+    seed = args.seed + utils.get_rank()
+    torch.manual_seed(seed)
+    np.random.seed(seed)
+    random.seed(seed)
+
+    # Data Setup
+    dataset_train = build_dataset(image_set='train', args=args)
+    args.num_classes = dataset_train.num_category()
+    args.num_actions = dataset_train.num_action()
+    args.action_names = dataset_train.get_actions()
+    if args.share_enc: args.hoi_enc_layers = args.enc_layers
+    if args.pretrained_dec: args.hoi_dec_layers = args.dec_layers
+    if args.dataset_file == 'vcoco':
+        # Save V-COCO dataset statistics
+        args.valid_ids = np.array(dataset_train.get_object_label_idx()).nonzero()[0]
+        args.invalid_ids = np.argwhere(np.array(dataset_train.get_object_label_idx()) == 0).squeeze(1)
+        args.human_actions = dataset_train.get_human_action()
+        args.object_actions = dataset_train.get_object_action()
+        args.num_human_act = dataset_train.num_human_act()
+    elif args.dataset_file == 'hico-det':
+        args.valid_obj_ids = dataset_train.get_valid_obj_ids()
+    print_args(args)
+
+    args.HOIDet=True
+    args.eval=True
+    args.pretrained_dec=True
+    args.share_enc=True
+    args.share_dec_param = True
+    if args.dataset_file=='hico-det':
+        args.valid_ids=args.valid_obj_ids
+ 
+    # Model Setup
+    model, criterion, postprocessors = build(args)
+    model.to(device)
+
+    model_without_ddp = model
+
+    n_parameters = print_params(model)
+
+    param_dicts = [
+        {"params": [p for n, p in model_without_ddp.named_parameters() if "backbone" not in n and p.requires_grad]},
+        {
+            "params": [p for n, p in model_without_ddp.named_parameters() if "backbone" in n and p.requires_grad],
+            "lr": args.lr_backbone,
+        },
+    ]
+
+    output_dir = Path(args.output_dir)
+    
+    checkpoint = torch.load(args.resume, map_location='cpu')
+    #수정
+    module_name=list(checkpoint['model'].keys())
+    model_without_ddp.load_state_dict(checkpoint['model'], strict=False)
+    
+    # if not args.video_vis:
+    # url='http://images.cocodataset.org/val2014/COCO_val2014_{}.jpg'.format(str(id).zfill(12))
+    # req = requests.get(url, stream=True, timeout=1, verify=False).raw
+    req = args.image_dir
+    img = Image.open(req).convert('RGB')
+
+    w,h=img.size
+    orig_size = torch.as_tensor([int(h), int(w)]).unsqueeze(0).to(device)
+
+    transform=make_hoi_transforms('val')
+    sample=img.copy()
+    sample,_=transform(sample,None)
+    sample = sample.unsqueeze(0).to(device)
+    with torch.no_grad():
+        model.eval()
+        out=model(sample)
+        results = postprocessors['hoi'](out, orig_size,dataset=args.dataset_file,args=args)
+        output_i=change_format(results[0],args.valid_ids)
+
+    out_dir = './vis'
+    image = np.asarray(img, dtype=np.uint8)[:,:,::-1]
+    # image = cv2.imdecode(image_nparray, cv2.IMREAD_COLOR)
+
+    vis_img=draw_img_vcoco(image,output_i,top_k=args.topk,threshold=args.threshold,color=builtin_meta.COCO_CATEGORIES)        
+    plt.imshow(cv2.cvtColor(vis_img,cv2.COLOR_BGR2RGB))
+    cv2.imwrite('./vis_res/vis1.jpg',vis_img)
+        
+    # else:
+    #     frames=[]
+    #     video_file=id
+    
+    #     video_reader = mmcv.VideoReader('./vid/'+video_file+'.mp4')
+    #     fourcc = cv2.VideoWriter_fourcc(*'mp4v')
+    #     video_writer = cv2.VideoWriter(
+    #             './vid/'+video_file+'_vis.mp4', fourcc, video_reader.fps,
+    #             (video_reader.width, video_reader.height))
+
+    #     orig_size = torch.as_tensor([int(video_reader.height), int(video_reader.width)]).unsqueeze(0).to(device)
+    #     transform=make_hoi_transforms('val')
+
+    #     for frame in mmcv.track_iter_progress(video_reader):
+
+    #         frame=mmcv.imread(frame)
+    #         frame=frame.copy()
+  
+    #         frame=Image.fromarray(frame,'RGB')
+
+    #         sample,_=transform(frame,None)
+    #         sample=sample.unsqueeze(0).to(device)
+
+    #         with torch.no_grad():
+    #             model.eval()
+    #             out=model(sample)
+    #             results = postprocessors['hoi'](out, orig_size,dataset='vcoco',args=args)
+    #             output_i=change_format(results[0],args.valid_ids)
+
+    #         vis_img=draw_img_vcoco(np.array(frame),output_i,top_k=args.topk,threshold=args.threshold,color=builtin_meta.COCO_CATEGORIES)
+    #         frames.append(vis_img)
+    #         video_writer.write(vis_img)
+
+    #     with imageio.get_writer("smiling.gif", mode="I") as writer:
+    #         for idx, frame in enumerate(frames):
+    #             # print("Adding frame to GIF file: ", idx + 1)
+    #             writer.append_data(frame)
+    #     if video_writer:
+    #         video_writer.release()
+    #     cv2.destroyAllWindows()
+ 
+
+# def visualization(id, video_vis=False, dataset_file='vcoco', path_id = 0 ,data_path='v-coco', threshold=0.4, topk=10,aug_path = '[]'):
+
+#     parser = argparse.ArgumentParser('DETR training and evaluation script', parents=[get_args_parser()])
+#     checkpoint_dir= './checkpoints/vcoco/checkpoint.pth' if dataset_file=='vcoco' else './checkpoints/hico-det/hico_ft_q16.pth'
+#     with open('./v-coco/data/vcoco_test.ids') as file:
+#       test_idxs = [line.rstrip('\n') for line in file]
+#     if not video_vis:
+#       id = test_idxs[id]
+#     args = parser.parse_args(args=['--dataset_file',dataset_file,'--data_path',data_path,'--resume',checkpoint_dir,'--num_hoi_queries' ,'16','--temperature' ,'0.05', '--augpath_name',aug_path ,'--path_id','{}'.format(path_id)])
+#     args.video_vis=video_vis
+#     args.threshold=threshold
+#     args.topk=topk
+    
+#     if args.output_dir:
+#         Path(args.output_dir).mkdir(parents=True, exist_ok=True)
+#     vis(args,id)
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser('DETR training and evaluation script', parents=[get_args_parser()])
+    parser.add_argument('--threshold',help='score threshold for visualization', default=0.4, type=float)
+    # parser.add_argument('--path_id',help='index of inference path', default=1, type=int)
+    parser.add_argument('--topk',help='topk prediction', default=5, type=int)
+    parser.add_argument('--video_vis', action='store_true')
+    parser.add_argument('--image_dir', default='', type=str)
+    args = parser.parse_args()
+    # checkpoint_dir= './checkpoints/vcoco/checkpoint.pth' if dataset_file=='vcoco' else './checkpoints/hico-det/hico_ft_q16.pth'
+    args.resume= './checkpoints/vcoco/checkpoint.pth'
+    with open('./v-coco/data/splits/vcoco_test.ids') as file:
+      test_idxs = [line.rstrip('\n') for line in file]
+    # if not video_vis:
+    id = test_idxs[309]
+    # args = parser.parse_args()
+    # args.dataset_file = 'vcoco'
+    # args.data_path = 'v-coco'
+    # args.resume = checkpoint_dir
+    # args.num_hoi_queries = 16
+    # args.temperature = 0.05
+    args.augpath_name = ['p2','p3','p4']
+    # args.path_id = 1
+    
+    if args.output_dir:
+        Path(args.output_dir).mkdir(parents=True, exist_ok=True)
+    vis(args,id)