initial app

.gitignore

@@ -0,0 +1,3 @@
+*.ipynb_checkpoints
+__pycache__
+*.ipynb

app.py

@@ -0,0 +1,65 @@
+import streamlit as st
+from src.st_image_tools import ImageTool
+
+def call_in_image_tool(cfg_path):
+    image_tool = ImageTool(cfg_path)
+    return image_tool
+
+def main(cfg_path="cfg/cfg.yml"):
+    """_summary_
+
+    Args:
+        cfg_path (str, optional): _description_. Defaults to "cfg/cfg.yml".
+
+    Returns:
+        _type_: _description_
+    """
+    st.set_page_config(layout="wide")
+
+    st.markdown(
+        """ <style>
+                #MainMenu {visibility: hidden;}
+                footer {visibility: hidden;}
+                </style> """,
+        unsafe_allow_html=True,
+    )
+
+    image_tool = call_in_image_tool(cfg_path)
+
+    # Select Plot Option
+    # st.sidebar.markdown("Checkboxes")
+    # checkbox_one = st.sidebar.checkbox("Show Image", value=True)  # rename as necessary
+    checkbox_two = st.sidebar.checkbox("Show Inference", value=True)
+    checkbox_three = st.sidebar.checkbox("Show Ground Truth", value=True)
+    checkbox_four = st.sidebar.checkbox("Show Side by Side (GT and Pred)", value=False)
+
+    option = st.sidebar.selectbox("Select Image", image_tool.all_img)
+
+    if checkbox_two:
+
+        if checkbox_three:
+            if checkbox_four:
+                image_tool.plot_with_preds_gt(option=option, side_by_side=True)
+            else:
+                image_tool.plot_with_preds_gt(option=option, plot_type="all")
+
+        else:
+            image_tool.plot_with_preds_gt(option=option, plot_type="pred")
+
+    elif checkbox_three:
+
+        if checkbox_two:
+            if checkbox_four:
+                image_tool.plot_with_preds_gt(option=option, side_by_side=True)
+            else:
+                image_tool.plot_with_preds_gt(option=option, plot_type="all")
+
+        else:
+            image_tool.plot_with_preds_gt(option=option, plot_type="gt")
+
+    else:
+        image_tool.plot_with_preds_gt(option=option)
+
+
+if __name__ == "__main__":
+    main()

cfg/cfg.yml

@@ -0,0 +1,32 @@
+error_analysis:
+  # detection_classes: ["person", "bicycle"]
+  labels_dict: {"person": 1, "bicycle": 2}      # GT's index for the classes should be zero-index but COCO wants to be special.
+  inference_labels_dict: {"person": 0, "bicycle": 1} # model's index for the classes. Should be zero-index, but sometimes not
+  conf_thresholds: [0.2, 0.35, 0.5, 0.65, 0.8]  # some call it score threshold
+  iou_thresholds: [0.2, 0.35, 0.5, 0.65, 0.8]   # back in my day we call it NMS threshold *shakes fist*
+  # nms_thresholds: [0.2, 0.5, 0.8] 
+  bbox_format: "pascal_voc"                     # yolo / coco / pascal_voc (WIP feature)
+  peekingduck: True                             # False if using your own model for inference without peekingduck wrapper, else True
+  ground_truth_format: "coco"                   # yolo / coco / pascal_voc (WIP feature)
+  idx_base : 1                                  # to indicate whether the class index is zero or one based. Applies to both GT and pred class
+  task: seg                                     # either "det" or "seg" 
+
+pkd:
+  model: "yolact_edge"                          # either "yolo" or "yolact_edge"
+  yolo_ver: "v4tiny"
+  yolact_ver: "r50-fpn"
+
+dataset:
+  classes: ["person", "bicycle"]                # same as ['error_analysis']['detection_classes'] field above
+  img_folder_path: 'data/annotations_trainval2017/coco_small/' # relative path from root for saving the coco dataset images
+  annotations_folder_path: 'data/annotations_trainval2017/annotations/' # relative path from root to the annotations file
+  annotations_fname: "instances_val2017.json"    # what is the name of your json file?
+
+visual_tool:
+  bbox_thickness: 2                             # how thicc you want the bbox to be
+  font_scale: 1                                 # how big you want the fonts to be 
+  font_thickness: 2                             # how thicc you want the fonts to be  
+  pred_colour: [255, 0, 0]                      # prediction colour, [B,G,R]
+  gt_colour: [0, 255, 0]                        # Ground truth colour, [B,G,R]
+  conf_threshold: 0.2                           # Confidence Threshold for use in the Visual Tool. [0, 1]
+  iou_threshold: 0.2                            # IOU/NMS Threshold for use in the Visual Tool. [0, 1]

data/annotations_trainval2017/annotations/instances_val2017.json

@@ -0,0 +1 @@
+{"annotations": [{"segmentation": [[406.42,181.47,410.12,183.82,423.56,175.42,423.56,179.12,419.53,183.82,425.92,193.91,426.92,209.03,424.91,214.41,418.86,223.49,411.8,224.5,406.76,222.14,404.07,206.01,400.03,207.69,395.66,189.2,403.39,181.81],[400.86,150.77,418.55,149.9,417.68,146.88,400.43,146.02]],"area": 1052.5614000000007,"iscrowd": 0,"image_id": 441586,"bbox": [395.66,146.02,31.26,78.48],"category_id": 2,"id": 128928},{"segmentation": [[550.06,162.05,552.14,159.37,558.69,158.48,562.25,158.48,561.36,162.94,565.82,166.51,579.8,171.57,583.07,167.4,590.8,165.02,592.58,169.78,597.34,174.54,603.29,175.43,605.37,182.87,604.77,196.84,591.1,202.79,578.01,200.41,574.15,194.46,569.98,196.25,571.17,191.79,563.44,179.0,560.17,175.43,557.5,174.54,558.09,176.62,558.98,186.73,545.9,195.35,536.98,191.19,536.38,181.97,541.44,176.03,547.09,171.86,551.55,170.67,554.52,168.3,555.41,165.02,553.63,161.75,552.44,161.75]],"area": 1679.4667500000005,"iscrowd": 0,"image_id": 441586,"bbox": [536.38,158.48,68.99,44.31],"category_id": 2,"id": 131214},{"segmentation": [[350.34,272.33,348.42,263.71,347.47,257.01,346.51,239.78,342.68,218.72,336.94,193.83,334.07,175.65,335.98,151.72,336.94,136.4,336.94,122.04,336.94,104.81,336.94,103.86,334.07,96.2,334.07,96.2,325.45,113.43,323.54,124.92,319.71,137.36,313.01,141.19,309.18,140.23,306.31,135.44,309.18,108.64,311.09,98.11,312.05,88.54,316.84,79.93,326.41,70.35,330.24,66.53,339.81,56.95,337.89,54.08,336.94,43.55,335.98,34.94,338.85,24.41,355.12,17.71,371.4,19.62,376.18,27.28,382.88,38.77,387.67,52.17,397.24,69.4,403.94,81.84,407.77,96.2,407.77,103.86,405.86,123.96,397.24,141.19,388.63,161.29,387.67,198.62,386.71,225.42,383.84,236.91,380.97,244.57,380.01,250.31,378.1,257.01,377.14,262.75,371.4,265.63,366.61,263.71,366.61,246.48,366.61,233.08,367.57,223.51,367.57,200.53,367.57,192.88,365.65,175.65,358.95,194.79,358.95,210.11,361.82,223.51,358.95,235.95,355.12,253.18,356.08,264.67,356.08,268.5]],"area": 13597.403899999998,"iscrowd": 0,"image_id": 441586,"bbox": [306.31,17.71,101.46,254.62],"category_id": 1,"id": 224536},{"segmentation": [[252.78,131.62,255.52,126.13,256.35,124.21,257.44,118.45,258.27,115.71,263.75,119.28,265.67,123.94,269.51,130.25,269.51,134.09,268.14,138.75,266.22,151.91,265.95,157.95,265.4,163.16,259.36,167.0,258.54,165.9,257.17,158.22,257.17,151.09,256.35,148.07,255.25,145.61,255.52,140.12,252.78,135.73,249.77,132.44]],"area": 529.5225499999995,"iscrowd": 0,"image_id": 441586,"bbox": [249.77,115.71,19.74,51.29],"category_id": 1,"id": 236500},{"segmentation": [[405.9,403.99,459.38,394.44,459.38,394.44,482.3,372.47,488.99,331.4,470.84,282.7,455.56,263.6,452.7,262.64,427.87,250.22,387.75,248.31,387.75,248.31,365.79,265.51,357.19,276.97,341.91,276.97,347.64,260.73,340.96,209.16,340.96,209.16,326.63,203.43,333.31,166.18,333.31,161.4,319.94,135.62,317.08,145.17,318.03,154.72,322.81,169.04,322.81,169.04,325.67,173.82,316.12,177.64,290.34,185.28,286.52,197.7,286.52,198.65,276.97,215.84,276.01,258.82,286.52,285.56,326.63,289.38,345.73,324.72,349.55,332.36,351.46,335.22,364.83,375.34,364.83,375.34,387.75,401.12]],"area": 24496.91995,"iscrowd": 0,"image_id": 441586,"bbox": [276.01,135.62,212.98,268.37],"category_id": 4,"id": 245321},{"segmentation": [[100.49,132.54,99.78,129.85,99.78,127.97,100.13,125.56,101.12,124.58,102.73,123.15,104.34,123.15,104.43,122.34,104.52,121.45,103.89,120.91,103.35,120.82,103.44,118.94,104.07,118.14,104.7,117.78,106.22,117.15,108.81,117.24,109.52,118.14,109.52,121.18,109.88,122.16,113.55,124.04,114.71,125.29,115.43,127.26,111.13,126.99,106.57,126.63,105.5,126.9,104.07,127.08,103.98,128.24,103.71,130.84,103.71,131.28,103.8,133.52,103.71,133.79,103.71,134.06,101.39,134.15],[105.59,134.06,105.23,136.74,112.21,136.74,113.91,135.67,115.78,134.95,116.14,133.61,116.14,133.61,116.14,133.61,105.32,133.34]],"area": 140.68389999999982,"iscrowd": 0,"image_id": 441586,"bbox": [99.78,117.15,16.36,19.59],"category_id": 1,"id": 1237637},{"segmentation": [[115.82,126.99,128.23,127.43,129.88,132.7,130.98,132.04,130.98,128.52,127.24,123.47,124.6,121.82,123.07,116.11,118.23,115.45,117.57,118.64,118.45,122.26,114.72,122.81],[116.58,133.8,118.23,136.76,127.13,136.21,127.35,133.91]],"area": 136.59869999999984,"iscrowd": 0,"image_id": 441586,"bbox": [114.72,115.45,16.26,21.31],"category_id": 1,"id": 1254020},{"segmentation": [[525.33,82.16,533.15,74.5,543.26,74.19,549.23,80.63,548.31,88.9,540.65,96.72,529.93,96.56,525.18,91.66]],"area": 432.5689499999999,"iscrowd": 0,"image_id": 441586,"bbox": [525.18,74.19,24.05,22.53],"category_id": 13,"id": 1389230},{"segmentation": [[399.9,147.78,400.2,156.12,401.09,157.91,410.03,154.04,413.01,152.25,416.59,152.25,418.68,150.76,415.7,147.48,417.49,140.63,421.06,137.05,426.13,136.75,430.0,140.03,429.71,145.1,427.32,149.86,427.02,154.93,431.2,158.81,427.92,164.77,422.25,175.8,416.29,179.67,411.82,180.56,407.35,182.95,402.28,183.25,406.76,176.99,413.91,164.77,410.63,162.68,407.05,161.79,397.22,163.58,396.03,152.85]],"area": 743.4173000000001,"iscrowd": 0,"image_id": 441586,"bbox": [396.03,136.75,35.17,46.5],"category_id": 1,"id": 1727927},{"segmentation": [[103.28,272.37,73.12,280.59,48.44,274.19,35.65,249.52,30.16,248.6,31.08,233.98,51.18,210.22,94.14,207.47,102.37,214.78,103.28,198.33,93.23,194.68,85.0,188.28,90.48,181.88,105.11,182.8,116.08,186.45,121.56,191.02,113.33,192.85,108.76,192.85,105.11,197.42,172.74,197.42,171.83,191.02,160.86,190.11,162.69,180.05,163.6,170.91,148.98,174.57,150.81,169.09,159.03,163.6,167.26,166.34,166.34,160.86,157.2,158.12,166.34,151.72,172.74,169.09,166.34,184.62,175.48,189.19,182.8,208.39,181.88,215.7,200.16,212.96,221.18,223.01,233.06,244.95,226.67,266.88,216.61,282.42,195.59,286.99,171.83,281.51,161.77,273.28,155.38,259.57,150.81,243.12,159.95,227.58,173.66,219.35,174.57,215.7,171.83,204.73,163.6,206.56,157.2,220.27,148.06,214.78,141.67,214.78,129.78,223.92,120.65,212.96,115.16,208.39,109.68,210.22,111.51,217.53,116.08,223.01,122.47,243.12,132.53,246.77,130.7,254.09,125.22,260.48,123.39,260.48,118.82,260.48,116.08,262.31,112.42,263.23]],"area": 11828.584050000001,"iscrowd": 0,"image_id": 441586,"bbox": [30.16,151.72,202.9,135.27],"category_id": 2,"id": 126632},{"segmentation": [[222.16,121.37,204.15,131.43,198.32,148.39,198.32,171.17,205.21,201.89,210.51,210.37,211.03,222.56,214.74,222.56,220.57,211.96,246.53,222.03,267.19,205.6,276.73,216.2,294.21,211.96,298.45,193.42,345.6,158.98,345.6,166.93,340.3,174.35,330.24,198.72,324.41,224.68,332.35,227.32,350.9,262.29,372.62,271.83,403.35,272.36,422.95,242.16,416.59,203.48,404.93,176.47,389.57,161.63,395.93,156.33,402.82,113.42,371.56,116.6,342.42,112.36,357.78,106.0,345.6,103.36,320.17,103.36,296.86,98.06,294.21,99.65,293.15,107.06,320.17,107.06,333.94,110.24,338.71,112.36,339.77,136.73,275.14,119.25,268.25,108.65,282.56,107.59,286.26,100.18,283.09,99.12,275.14,98.06,272.49,98.06,268.78,94.88,266.13,92.76,258.19,91.17,252.36,95.41,256.6,102.83,261.36,104.94,263.48,106.53,266.66,108.12,266.66,113.42,264.01,118.19,260.83,118.72,252.89,113.95,249.18,111.3,241.76,109.71,238.05,108.65,228.52,108.12,224.81,108.12,216.33,110.24,216.33,120.31]],"area": 23082.81575,"iscrowd": 0,"image_id": 11149,"bbox": [198.32,91.17,224.63,181.19],"category_id": 2,"id": 126643},{"segmentation": [[0.0,133.87,0.0,265.63,14.36,280.83,21.96,279.14,26.18,269.85,30.41,261.4,38.85,263.09,50.68,263.09,54.05,271.54,55.74,286.74,61.66,308.7,68.41,323.06,72.64,333.19,83.61,345.86,96.28,350.93,120.78,349.24,130.91,339.95,135.14,324.75,136.82,310.39,136.82,295.19,135.98,280.83,132.6,259.71,126.69,247.89,119.93,235.22,113.18,223.4,108.95,212.42,103.89,208.19,99.66,202.28,95.44,192.99,92.91,182.01,90.37,171.88,89.53,163.43,89.53,154.98,90.37,151.6,108.11,138.94,103.04,130.49,95.44,130.49,90.37,132.18,88.68,133.02,81.93,133.02,77.7,132.18,79.39,126.27,81.08,122.04,85.3,119.51,88.68,111.91,88.68,103.46,92.91,93.33,97.13,89.95,103.04,84.88,111.49,76.44,111.49,69.68,106.42,68.83,90.37,68.83,86.15,68.83,85.3,75.59,86.15,77.28,84.46,102.62,75.17,105.15,71.79,106.84,67.57,110.22,61.66,106.84,53.21,97.55,47.3,97.55,39.7,103.46,36.32,106.0,36.32,110.22,33.78,115.29,29.56,118.67,27.03,121.2,21.11,121.2,16.05,115.29,8.45,109.38,0.84,104.31]],"area": 22842.389549999996,"iscrowd": 0,"image_id": 11149,"bbox": [0.0,68.83,136.82,282.1],"category_id": 4,"id": 151281}, {"segmentation": [[21.11,120.35,30.41,103.46,34.63,95.02,39.7,99.24,58.28,99.24,67.57,103.46,75.17,103.46,73.48,97.55,51.52,76.44,13.51,51.94,22.8,40.12,22.8,23.23,19.43,13.09,17.74,7.18,10.14,5.49,3.38,3.8,0.0,46.03,1.69,108.53,1.69,108.53,19.43,121.2]],"area": 3560.2923000000005,"iscrowd": 0,"image_id": 11149,"bbox": [0.0,3.8,75.17,117.4],"category_id": 1,"id": 197935},{"segmentation": [[355.4,44.07,359.52,125.03,386.96,119.54,400.68,120.91,400.68,134.64,380.1,145.61,369.12,146.99,375.98,174.43,393.82,200.5,389.7,284.2,391.08,326.74,403.43,341.84,426.75,341.84,434.99,341.84,448.71,334.98,452.83,321.25,441.85,318.51,437.73,317.14,419.89,285.58,421.26,273.23,421.26,260.88,426.75,219.71,424.01,193.64,422.64,173.06,430.87,174.43,441.85,163.45,437.73,152.47,437.73,126.4,445.96,75.63,448.71,49.56,448.71,34.47,445.96,27.6,419.89,24.86,374.61,24.86,367.75,30.35,366.38,35.84,359.52,42.7,359.52,45.44]],"area": 17385.67285,"iscrowd": 0,"image_id": 492937,"bbox": [355.4,24.86,97.43,316.98],"category_id": 1,"id": 198827},{"segmentation": [[417.13,288.9,436.52,268.68,439.04,228.23,434.83,143.12,445.79,111.1,445.79,98.46,458.43,79.92,450.0,57.16,453.37,34.41,471.07,1.54,498.88,1.54,500.0,245.08,488.76,260.25,486.24,281.32,485.39,310.81,474.44,315.03,467.7,315.87,460.96,313.34,471.91,233.29,470.22,222.33,469.38,222.33,460.11,259.41,453.37,293.12,423.88,297.33]],"area": 15959.356599999996,"iscrowd": 0,"image_id": 11149,"bbox": [417.13,1.54,82.87,314.33],"category_id": 1,"id": 199247}, {"segmentation": [[94.17,98.47,95.01,90.91,98.37,89.22,109.3,89.22,113.51,94.27,114.35,97.63,114.35,106.88,116.03,117.81,118.55,140.51,131.17,112.77,131.17,91.75,125.28,82.5,119.39,76.61,108.46,71.57,108.46,69.89,110.99,63.16,116.03,60.64,125.28,58.95,143.78,72.41,153.87,82.5,146.3,97.63,148.82,124.54,176.57,131.26,193.39,153.97,199.27,212.82,161.43,230.48,134.53,228.8,110.99,206.1,103.42,178.35,97.53,175.83,88.28,171.62,91.65,148.08,105.1,146.4,110.99,133.79,108.46,127.06,111.83,107.72,98.37,97.63]],"area": 10372.88825,"iscrowd": 0,"image_id": 11149,"bbox": [88.28,58.95,110.99,171.53],"category_id": 2,"id": 240899}, {"segmentation": [[282.61,469.21,290.16,422.83,299.87,354.88,285.84,340.85,253.48,330.07,231.91,309.57,224.36,296.63,239.46,258.88,237.3,239.46,240.54,207.1,258.88,172.58,283.69,147.78,299.87,134.83,303.1,114.34,310.65,87.37,319.28,78.74,339.78,84.13,351.64,102.47,362.43,122.97,359.19,142.38,352.72,151.01,362.43,173.66,362.43,195.24,361.35,221.12,376.45,244.85,403.42,273.98,382.92,285.84,366.74,288.0,354.88,306.34,341.93,318.2,344.09,347.33,347.33,374.29,347.33,393.71,345.17,408.81,362.43,422.83,369.98,432.54,351.64,441.17,337.62,440.09,333.3,459.51]],"area": 33491.37034999999,"iscrowd": 0,"image_id": 576031,"bbox": [224.36,78.74,179.06,390.47],"category_id": 1,"id": 445741}, {"segmentation": [[269.13,162.23,265.79,160.38,265.79,158.02,272.35,149.85,272.1,138.09,273.59,129.68,276.19,123.24,279.78,121.26,285.6,121.63,289.56,124.11,290.67,127.2,295.13,137.6,295.87,137.85,287.08,148.0]],"area": 613.1277999999993,"iscrowd": 0,"image_id": 576031,"bbox": [265.79,121.26,30.08,40.97],"category_id": 1,"id": 473571}, {"segmentation": [[188.78,186.73,182.84,195.79,182.84,205.17,182.84,221.42,177.84,224.55,168.78,224.55,164.72,223.92,166.59,219.55,169.72,218.92,170.03,211.73,172.84,201.73,172.84,190.17,174.09,183.29,171.28,175.16,170.97,165.79,170.03,154.22,161.59,157.03,149.09,155.47,145.02,158.28,144.09,153.28,143.77,148.91,147.84,147.66,155.03,150.78,165.97,142.97,170.34,140.47,172.53,134.22,168.78,130.15,167.53,122.96,170.34,115.15,176.59,112.02,183.47,116.4,183.78,124.21,184.1,128.28,186.6,131.4,192.22,134.53,195.35,137.65,195.35,145.78,203.79,167.66,205.35,172.66,205.35,178.29,202.54,185.48,200.97,193.29,207.85,210.17,212.23,218.92,211.6,221.74,206.29,223.92,200.97,225.49,197.54,225.49,196.6,222.67,200.35,219.24,197.85,214.55,192.53,206.73,190.97,201.42,188.78,189.23]],"area": 3038.4778499999998,"iscrowd": 0,"image_id": 576031,"bbox": [143.77,112.02,68.46,113.47],"category_id": 1,"id": 498483}, {"segmentation": [[169.58,141.88,170.12,139.69,170.67,136.95,169.58,133.94,167.66,130.1,167.66,125.71,167.38,125.17,163.27,126.54,157.52,133.39,155.6,136.95,155.33,139.96,155.33,141.88,152.86,145.45,152.59,147.64,152.59,149.28,152.59,149.83,155.87,149.28,159.16,146.27,163.27,143.53],[152.04,157.23,152.31,163.26,153.68,166.82,154.78,170.38,156.15,174.77,154.78,181.89,147.65,192.58,144.09,198.34,141.9,203.27,143.27,204.91,146.01,206.01,146.83,207.38,149.57,208.2,151.22,209.57,153.68,209.85,156.15,209.85,154.5,208.2,152.86,202.72,153.96,200.53,157.79,193.13,161.9,188.47,163.82,187.65,166.56,197.79,167.66,203.54,168.21,207.65,168.21,210.67,170.67,208.75,173.14,194.77,172.86,187.1,172.59,181.62,171.77,176.96,169.03,170.66,168.75,167.37,168.75,163.26,169.3,156.96,169.85,155.04]],"area": 1103.7693,"iscrowd": 0,"image_id": 576031,"bbox": [141.9,125.17,31.24,85.5],"category_id": 1,"id": 515669}, {"segmentation": [[201.13,438.05,288.11,443.26,283.65,434.34,202.62,433.59],[320.07,446.23,545.32,461.1,570.59,451.43,549.04,454.41,323.05,436.57],[208.57,455.89,300.75,480.0,314.87,478.2,259.86,466.3,210.8,453.66]],"area": 2864.165400000003,"iscrowd": 0,"image_id": 576031,"bbox": [201.13,433.59,369.46,46.41],"category_id": 35,"id": 1453611}, {"segmentation": [[431.59,362.72,438.02,334.25,435.26,320.48,433.43,304.87,430.67,290.18,422.41,271.81,424.25,260.79,425.16,247.94,404.96,247.94,405.88,222.22,410.47,191.0,403.12,186.41,414.14,169.88,402.21,158.86,405.88,147.84,419.65,149.68,448.12,144.17,451.79,161.62,448.12,170.8,463.73,183.66,481.18,217.63,492.2,234.16,492.2,248.85,496.79,247.94,491.28,255.28,493.12,275.48,495.87,292.01,486.69,296.6,487.61,314.05,490.36,328.74,477.51,331.5,473.83,319.56,479.34,297.52,479.34,278.24,479.34,270.89,472.0,309.46,472.91,328.74,470.16,333.34,470.16,352.62,470.16,358.13,477.51,370.98,473.83,382.92,455.47,378.33,452.71,370.07,446.28,372.82,431.59,373.74,428.84,361.8]],"area": 13001.949300000002,"iscrowd": 0,"image_id": 576052,"bbox": [402.21,144.17,94.58,238.75],"category_id": 19,"id": 53914}, {"segmentation": [[428.25,158.67,427.93,145.36,433.45,129.77,445.14,121.0,439.94,113.86,437.99,109.96,438.32,101.52,444.49,92.1,449.68,91.13,457.15,90.8,459.42,93.4,465.92,102.17,467.87,107.69,464.94,115.16,468.52,120.68,478.91,126.85,483.78,151.2,474.69,165.49,499.04,190.49,507.16,238.2,515.27,244.7,516.25,248.27,508.78,252.16,494.17,245.02,484.1,199.6,467.54,178.82,453.9,170.38,438.32,162.58]],"area": 4388.146849999999,"iscrowd": 0,"image_id": 576052,"bbox": [427.93,90.8,88.32,161.36],"category_id": 1,"id": 196260}]}

environment.yml

@@ -0,0 +1,16 @@
+dependencies:
+  - python=3.8
+  - pip
+  - pip:
+    - peekingduck # only necessary for pkd
+    - typeguard == 2.13.3 # only necessary for pkd
+    - beautifulsoup4==4.11.2
+    - opencv-python==4.7.0.68
+    - pandas==1.5.3
+    - numpy==1.24.2
+    - cython
+    - pycocotools-windows # change to pycocotools if non-Windows
+    - jupyter==1.0.0
+    - pyyaml
+    - streamlit==1.20.0
+    - plotly==5.13.1

requirements.txt

@@ -0,0 +1,14 @@
+pandas
+plotly
+datasets
+peekingduck
+beautifulsoup4==4.11.2
+opencv-python==4.7.0.68
+pandas==1.5.3
+numpy==1.24.2
+cython
+pycocotools
+jupyter==1.0.0
+pyyaml
+# streamlit==1.20.0
+plotly==5.13.1

src/confusion_matrix.py

@@ -0,0 +1,194 @@
+import numpy as np
+import pandas as pd
+
+def box_iou_calc(boxes1, boxes2):
+    # https://github.com/pytorch/vision/blob/master/torchvision/ops/boxes.py
+    """
+    Return intersection-over-union (Jaccard index) of boxes.
+    Both sets of boxes are expected to be in (x1, y1, x2, y2) format.
+    Arguments:
+        boxes1 (Array[N, 4])
+        boxes2 (Array[M, 4])
+    Returns:
+        iou (Array[N, M]): the NxM matrix containing the pairwise
+            IoU values for every element in boxes1 and boxes2
+
+    This implementation is taken from the above link and changed so that it only uses numpy.
+    """
+
+    def box_area(box):
+        # box = 4xn
+        return (box[2] - box[0]) * (box[3] - box[1])
+    
+
+    area1 = box_area(boxes1.T)
+    area2 = box_area(boxes2.T)
+
+    lt = np.maximum(boxes1[:, None, :2], boxes2[:, :2])  # [N,M,2]
+    rb = np.minimum(boxes1[:, None, 2:], boxes2[:, 2:])  # [N,M,2]
+
+    inter = np.prod(np.clip(rb - lt, a_min = 0, a_max = None), 2)
+    return inter / (area1[:, None] + area2 - inter)  # iou = inter / (area1 + area2 - inter)
+
+def mask_iou_calc(pred_masks, gt_masks):
+    """Helper function calculate the IOU of masks
+
+    Args:
+        pred_masks (_type_): N x H x W, array of N masks 
+        gt_masks (_type_): M x H x W, an array of M masks 
+
+    Returns:
+        iou: an array of NxM of IOU ([0,1]) 
+            N rows - number of actual labels
+            M columns - number of preds
+    """
+    if pred_masks.size == 0:
+        return np.array([])
+
+    # build function to take in two masks, compare them and see what their iou is.
+    # similar to above but in mask.
+    tp = np.sum(np.multiply(pred_masks[:, None], gt_masks), axis = (2,3))
+    fp = np.sum(np.where(pred_masks[:, None] > gt_masks, 1, 0), axis = (2,3))
+    fn = np.sum(np.where(pred_masks[:, None] < gt_masks, 1, 0), axis = (2,3))
+
+    # print (f"tp: {tp}")
+    # print (f"fp: {fp}")
+    # print (f"fn: {fn}")
+    iou = tp / (tp + fn + fp)
+
+    return iou.T
+
+class ConfusionMatrix:
+
+    def __init__(self, num_classes, CONF_THRESHOLD = 0.2, IOU_THRESHOLD = 0.5):
+        self.matrix = np.zeros((num_classes + 1, num_classes + 1))
+        self.num_classes = num_classes
+        self.CONF_THRESHOLD = CONF_THRESHOLD
+        self.IOU_THRESHOLD = IOU_THRESHOLD
+        self.got_tpfpfn = False
+    
+    def process_batch(self, detections, labels, return_matches=False, task = "det"):
+        '''
+        Return intersection-over-union (Jaccard index) of boxes.
+        Both sets of boxes are expected to be in (x1, y1, x2, y2) format.
+        Arguments:
+            detections (Array[N, 6]), x1, y1, x2, y2, conf, class
+            labels (Array[M, 5]), class, x1, y1, x2, y2
+        Returns:
+            None, updates confusion matrix accordingly
+        '''
+
+        if task == 'det':
+            detections = detections[detections[:, 4] > self.CONF_THRESHOLD]
+            gt_classes = labels[:, 0].astype(np.int16)
+            detection_classes = detections[:, 5].astype(np.int16)
+            all_ious = box_iou_calc(labels[:, 1:], detections[:, :4])
+            want_idx = np.where(all_ious > self.IOU_THRESHOLD)
+            
+        elif task == 'seg':
+            detections = [detection for detection in detections if detection[1] > self.CONF_THRESHOLD]
+            gt_classes = np.array([label[0]for label in labels], dtype = np.int16) 
+            detection_classes = np.array([detection[2] for detection in detections], dtype = np.int16)
+            all_ious = mask_iou_calc(np.array([detection[0] for detection in detections]), np.array([label[1] for label in labels]))
+            want_idx = np.where(all_ious > self.IOU_THRESHOLD)
+        
+        all_matches = []
+        for i in range(want_idx[0].shape[0]):
+            all_matches.append([want_idx[0][i], want_idx[1][i], all_ious[want_idx[0][i], want_idx[1][i]]])
+        
+        all_matches = np.array(all_matches)
+        if all_matches.shape[0] > 0: # if there is match
+            all_matches = all_matches[all_matches[:, 2].argsort()[::-1]]
+
+            all_matches = all_matches[np.unique(all_matches[:, 1], return_index = True)[1]]
+
+            all_matches = all_matches[all_matches[:, 2].argsort()[::-1]]
+
+            all_matches = all_matches[np.unique(all_matches[:, 0], return_index = True)[1]]
+        
+        for i, label in enumerate(labels):
+            if all_matches.shape[0] > 0 and all_matches[all_matches[:, 0] == i].shape[0] == 1:
+                gt_class = gt_classes[i]
+                detection_class = detection_classes[int(all_matches[all_matches[:, 0] == i, 1][0])]
+                self.matrix[(gt_class), detection_class] += 1
+            else:
+                gt_class = gt_classes[i]
+                self.matrix[(gt_class), self.num_classes] += 1
+        
+        for i, detection in enumerate(detections):
+            if all_matches.shape[0] and all_matches[all_matches[:, 1] == i].shape[0] == 0:
+                detection_class = detection_classes[i]
+                self.matrix[self.num_classes ,detection_class] += 1
+    
+        if return_matches:
+            return all_matches
+        
+    def get_tpfpfn(self):
+        self.tp = np.diag(self.matrix).sum()
+        fp = self.matrix.copy()
+        np.fill_diagonal(fp, 0)
+        self.fp = fp[:,:-1].sum()
+        self.fn = self.matrix[:-1, -1].sum()
+        self.got_tpfpfn = True
+
+    def get_PR(self):
+        if not self.got_tpfpfn:
+            self.get_tpfpfn()
+        # print (tp, fp, fn)
+        self.precision = self.tp / (self.tp+self.fp)
+        self.recall = self.tp/(self.tp+self.fn)
+
+    def return_matrix(self):
+        return self.matrix
+
+    def process_full_matrix(self):
+        """method to process matrix to something more readable
+        """
+        for idx, i in enumerate(self.matrix):
+            i[0] = idx
+        self.matrix = np.delete(self.matrix, 0, 0)
+            
+    def print_matrix_as_df(self):
+        """method to print out processed matrix
+        """
+        df = pd.DataFrame(self.matrix)
+        print (df.to_string(index=False))
+
+    # def print_matrix(self):
+    #     for i in range(self.num_classes + 1):
+    #         print(' '.join(map(str, self.matrix[i])))
+
+    def return_as_csv(self, csv_file_path):
+        """method to print out processed matrix
+        """
+        df = pd.DataFrame(self.matrix)
+        df.to_csv(csv_file_path, index = False)
+        print (f"saved to: {csv_file_path}")
+
+    def return_as_df(self):
+        """method to print out processed matrix
+        """
+        df = pd.DataFrame(self.matrix)
+        # df = df.set_index(0)
+        # df.set_index(0)
+        # print(df.columns)
+        return df
+    
+if __name__ == '__main__':
+    # # test IOU for segmentation masks
+    gtMasks = np.array([[[1, 1, 0], 
+                        [0, 1, 0], 
+                        [0, 0, 0]],
+                        [[1, 1, 0], 
+                        [0, 1, 1], 
+                        [0, 0, 0]]])
+    predMasks = np.array([[[1, 1, 0], 
+                        [0, 1, 1], 
+                        [0, 0, 0]],
+                        [[1, 1, 0], 
+                        [0, 1, 0], 
+                        [0, 0, 0]]])
+    
+    # IOU is 0.75 
+    IOU = mask_iou_calc(predMasks, gtMasks)
+    print (IOU.shape)

src/data_ingestion/data_ingestion.py

@@ -0,0 +1,119 @@
+import cv2
+import json
+import os
+import numpy as np
+from PIL import Image, ImageDraw
+
+def convert_seg_coord_to_mask(img_size, coords):
+    """Converts the segmentation coords found in COCO dataset to mask
+
+    Args:
+        img_size (_type_): _description_
+        coords (_type_): _description_
+
+    Returns:
+        _type_: _description_
+    """
+
+    img = Image.new('L', img_size, 0)
+    for polygon in coords:
+        ImageDraw.Draw(img).polygon(polygon, outline=1, fill=1)
+    mask = np.array(img)    
+
+    return mask
+
+class AnnotsGTGetter:
+    
+    def __init__(self, cfg_obj):
+
+        self.cfg_obj = cfg_obj
+        
+        self.img_folder_path = cfg_obj['dataset']['img_folder_path']
+        self.json_folder_path = cfg_obj['dataset']['annotations_folder_path']
+        self.annot_json_fname = cfg_obj['dataset']['annotations_fname']
+        self.labels_dict = cfg_obj['error_analysis']['labels_dict']
+        self.task = cfg_obj['error_analysis']['task']
+
+        json_file = open(self.json_folder_path + self.annot_json_fname)
+        self.annot_data = json.load(json_file)
+
+        self.img_ids_in_json = [annot['image_id'] for annot in self.annot_data['annotations']]
+        self.all_imgs = os.listdir(self.img_folder_path)
+
+        return
+
+    def get_imgs(self):
+        """method to get the mutually -inclusive- images between the img_ids in json and those in the folder path
+
+        not needed because all images in folder were accounted for in the json...
+        """
+        all_img_ids_in_folder = [int(i[:-4]) for i in self.all_imgs]
+
+        all_imgs_found = [i for i in all_img_ids_in_folder if i in self.img_ids_in_json]
+
+        print (len(all_imgs_found))
+
+
+    def get_annots(self, img_fname = '000000576052.jpg'):
+        """retrieve annotation given a filename
+
+        Args:
+            img_fname (_type_): image file name
+        
+        Returns:
+            np array: all annotations of an image
+        """
+
+        # change img_fname for extraction purpose
+        # assumes jpg, png, but not jpeg...
+        # TODO - what if jpeg?
+        annots = []
+        img_id = int(img_fname[:-4])
+        img = Image.open(self.img_folder_path + img_fname)
+        
+        for annot in self.annot_data['annotations']:
+            if img_id == annot['image_id']:
+                if annot['category_id'] in list(self.labels_dict.values()):
+                    if self.task == "det":
+                        annots.append([annot['category_id'],annot['bbox'][0],annot['bbox'][1],annot['bbox'][2],annot['bbox'][3]])
+                    elif self.task == "seg":
+                        # call convert_seg_coord_to_mask to convert segmentations [x1,y1,x2,y2,...,xn,yn] to binary mask
+                        mask = convert_seg_coord_to_mask(img.size, annot['segmentation'])
+                        annots.append([annot['category_id'], mask])
+
+        if self.task == "det":
+            return np.array(annots)
+        elif self.task == "seg":
+            return annots
+
+    def get_gt_annots(self):
+        """goes into the image folder, calls get_annots to extract image annotation
+
+        Returns:
+            dict: all annotations 
+        """
+
+        # create dictionary of gt annots
+        # for img in os.listdir(self.img_folder_path):
+        #     self.get_annots(img)
+        all_gt_annots = {img: self.get_annots(img) for img in os.listdir(self.img_folder_path)}
+
+        return all_gt_annots
+
+if __name__ == '__main__':
+    import yaml
+    # get_annots()
+    cfg_file = open("cfg/cfg.yml")
+    cfg_obj = yaml.load(cfg_file, Loader=yaml.FullLoader)
+    annots_obj = AnnotsGTGetter(cfg_obj)
+    gt_dict = annots_obj.get_gt_annots()
+    # print (gt_dict)
+    # annots_obj.get_imgs()
+    
+    # # to output mask
+    # img_annots = gt_dict['000000576031.jpg']
+
+    # import matplotlib.pyplot as plt
+    # plt.imshow(img_annots[0][1])
+    # plt.show()
+

src/error_analysis.py

@@ -0,0 +1,182 @@
+from peekingduck.pipeline.nodes.model import yolo as pkd_yolo
+from peekingduck.pipeline.nodes.model import yolact_edge as pkd_yolact
+from src.data_ingestion.data_ingestion import AnnotsGTGetter
+from src.inference import Inference
+from src.confusion_matrix import ConfusionMatrix
+import yaml
+from itertools import product
+import pandas as pd
+import numpy as np
+
+def transform_gt_bbox_format(ground_truth, img_size, format = "coco"):
+    """transforms ground truth bbox format to pascal voc for confusion matrix
+
+    Args:
+        ground_truth (_type_): nx5 numpy array, if coco - n x [class, x, y, w, h], if yolo - n x [class, x-mid, y-mid, w, h]
+        img_size (_type_): [Height * Weight * Dimension] values vector
+        format (str, optional): . Defaults to "coco".
+
+    Returns:
+        _type_: ground_truth. Transformed ground truth to pascal voc format
+    """
+    if format == "coco":
+        ground_truth[:, 3] = (ground_truth[:, 1] + ground_truth[:, 3])/img_size[1]
+        ground_truth[:, 1] = (ground_truth[:, 1]) /img_size[1]
+        ground_truth[:, 4] = (ground_truth[:, 2] + ground_truth[:, 4])/img_size[0]
+        ground_truth[:, 2] = (ground_truth[:, 2]) /img_size[0]
+    
+    return ground_truth
+
+def load_model(cfg_obj, iou_threshold, score_threshold):
+
+    pkd = cfg_obj['error_analysis']['peekingduck']
+    task = cfg_obj['error_analysis']['task']
+
+    if pkd:
+
+        pkd_model = cfg_obj['pkd']['model']
+        # assert task == "seg" and pkd_model == "yolact_edge", "For segmentation tasks, make sure task is seg and pkd_model is yolact_edge"
+        # assert task == "det" and pkd_model == "yolo", "For detection tasks, make sure task is det and pkd_model is yolo"
+        # only instantiates the v4tiny model, but you are free to change this to other pkd model 
+        if pkd_model == "yolo":
+            yolo_ver = cfg_obj['pkd']['yolo_ver']
+            model = pkd_yolo.Node(model_type = yolo_ver, 
+                                  detect= list(cfg_obj['error_analysis']['inference_labels_dict'].keys()),
+                                  iou_threshold = iou_threshold,
+                                  score_threshold = score_threshold)
+
+        if pkd_model == "yolact_edge":
+            yolact_ver = cfg_obj['pkd']['yolact_ver']
+            model = pkd_yolact.Node(model_type = yolact_ver, 
+                                    detect= list(cfg_obj['error_analysis']['inference_labels_dict'].values()), 
+                                    iou_threshold = iou_threshold,
+                                    score_threshold = score_threshold)
+
+    else:
+        # call in your own model
+        # self.model = <your model import here>
+        # make sure that your model has iou_threshold and score_threshold attributes 
+        # you can easily set those attributes in this else block
+        pass
+
+    return model
+
+class ErrorAnalysis:
+
+    def __init__(self,  cfg_path = 'cfg/cfg.yml'):
+        
+        cfg_file = open(cfg_path)
+        self.cfg_obj = yaml.load(cfg_file, Loader=yaml.FullLoader)
+        # self.nms_thresh = self.cfg_obj['error_analysis']['nms_thresholds']
+        self.iou_thresh = self.cfg_obj['error_analysis']['iou_thresholds']
+        self.conf_thresh = self.cfg_obj['error_analysis']['conf_thresholds']
+        self.inference_folder = self.cfg_obj['dataset']['img_folder_path']
+        self.task = self.cfg_obj['error_analysis']['task']
+        base_iou_threshold = self.cfg_obj['visual_tool']['iou_threshold']
+        base_score_threshold = self.cfg_obj['visual_tool']['conf_threshold']
+        
+        self.cm_results = []
+        
+        # instantiate a "base" model with configs already 
+        self.model = load_model(self.cfg_obj, base_iou_threshold, base_score_threshold) 
+
+    def generate_inference(self, img_fname = "000000576052.jpg"):
+        """Run inference on img based on the image file name. Path to the folder is determined by cfg 
+
+        Args:
+            img_fname (str, optional): _description_. Defaults to "000000576052.jpg".
+
+        Returns:
+            ndarray, tuple: if task is 'det': ndarray - n x [x1, y1, x2, y2, score, class], (H, W, D)
+            ndarray, tuple: if task is 'seg': list - n x [[array of binary mask], score, class], (H, W, D)
+        """
+        
+        inference_obj = Inference(self.model, self.cfg_obj)
+        img_path = f"{self.inference_folder}{img_fname}"
+        inference_outputs = inference_obj.run_inference_path(img_path)
+
+        return inference_outputs
+
+    def get_annots(self):
+        """get GT annotations from dataset 
+        """
+
+        annots_obj = AnnotsGTGetter(cfg_obj = self.cfg_obj)
+        self.gt_dict = annots_obj.get_gt_annots()
+
+    def generate_conf_matrix(self,iou_threshold = 0.5, conf_threshold = 0.2):
+        """generate the confusion matrix by running inference on each image
+        """
+
+        num_classes = len(list(self.cfg_obj['error_analysis']['labels_dict'].keys()))
+        ground_truth_format = self.cfg_obj["error_analysis"]["ground_truth_format"]
+        idx_base = self.cfg_obj["error_analysis"]["idx_base"]
+
+        # TODO - currently, Conf Matrix is 0 indexed but all my classes are one-based index. 
+        # need to find a better to resolve this 
+        # Infuriating.
+        cm = ConfusionMatrix(num_classes=num_classes, CONF_THRESHOLD = conf_threshold, IOU_THRESHOLD=iou_threshold)
+        
+        for fname in list(self.gt_dict.keys()):
+
+            inference_output, img_size = self.generate_inference(fname)
+            ground_truth = self.gt_dict[fname].copy()
+                        
+            if self.task == "det":
+
+                # deduct index_base from each inference's class index 
+                inference_output[:, -1] -= idx_base
+                # deduct index_base from each groundtruth's class index 
+                ground_truth[:, 0] -= idx_base
+                # inference is in x1, y1, x2, y2, scores, class, so OK
+                # coco gt is in x, y, width, height - need to change to suit conf matrix
+                # img shape is (H, W, D) so plug in accordingly to normalise 
+                ground_truth = transform_gt_bbox_format(ground_truth=ground_truth, img_size=img_size, format = ground_truth_format)
+
+            else: 
+                # deduct index_base from each groundtruth's class index
+                ground_truth = [[gt[0] - idx_base, gt[1]] for gt in ground_truth]
+
+            cm.process_batch(inference_output, ground_truth, task = self.task)
+        
+        cm.get_PR() 
+
+        return cm.matrix, cm.precision, cm.recall
+    
+    def generate_conf_matrices(self, print_matrix = True):
+        """generates the confidence matrices
+        """
+
+        # get all combinations of the threshold values:
+        combinations = list(product(self.iou_thresh, self.conf_thresh))
+        # print (combinations)
+        comb_cms = {}
+        for comb in combinations:
+            # print (f"IOU: {comb[0]}, Conf: {comb[1]}")  
+            self.model = load_model(self.cfg_obj, iou_threshold=comb[0], score_threshold=comb[1])
+            returned_matrix, precision, recall = self.generate_conf_matrix(iou_threshold = comb[0], conf_threshold = comb[1])
+            # print (returned_matrix)
+            # print (f"precision: {precision}")
+            # print (f"recall: {recall}")
+            comb_cms[f"IOU: {comb[0]}, Conf: {comb[1]}"] = returned_matrix
+            self.cm_results.append([comb[0], comb[1], precision, recall])
+        
+        if print_matrix:
+            for k, v in comb_cms.items():
+                print (k)
+                print (v)
+
+    def proc_pr_table(self):
+        
+        self.cm_table = pd.DataFrame(self.cm_results, columns = ['IOU_Threshold', 'Score Threshold', 'Precision', 'Recall'])
+
+        print (self.cm_table)
+
+
+if __name__ == "__main__":
+    ea_games = ErrorAnalysis()
+    # print (ea_games.generate_inference())
+    ea_games.get_annots()
+    ea_games.generate_conf_matrices()
+    # print (ea_games.generate_conf_matrix())
+    # print (ea_games.gt_dict)

src/get_data_coco/get_img.py

@@ -0,0 +1,33 @@
+from pycocotools.coco import COCO
+import requests
+import yaml
+
+def get_images(cfg_path = 'cfg/cfg.yml'):
+    """To query and get coco dataset by a certain subset e.g. 'person'
+
+    Args:
+        path (str, optional): _description_. Defaults to 'data/annotations_trainval2017/annotations/instances_val2017.json'.
+        catNms (list, optional): _description_. Defaults to ['person'].
+    """
+
+    cfg_file = open(cfg_path)
+    cfg = yaml.load(cfg_file, Loader=yaml.FullLoader)
+
+    # instantiate COCO specifying the annotations json path
+    coco = COCO(cfg['dataset']['annotations_folder_path'] + cfg['dataset']['annotations_fname'])
+    # Specify a list of category names of interest
+    catIds = coco.getCatIds(catNms=cfg['dataset']['classes'])
+    # Get the corresponding image ids and images using loadImgs
+    imgIds = coco.getImgIds(catIds=catIds)
+    images = coco.loadImgs(imgIds)
+
+    # Save the images into a local folder
+    for im in images:
+        img_data = requests.get(im['coco_url']).content
+        with open(cfg['dataset']['img_folder_path'] + im['file_name'], 'wb') as handler:
+            handler.write(img_data)
+
+    return
+
+if __name__ == '__main__':
+    get_images(cfg_path = 'cfg/cfg.yml') 

src/inference.py

@@ -0,0 +1,124 @@
+from peekingduck.pipeline.nodes.model import yolo as pkd_yolo
+import cv2
+from collections import defaultdict
+import numpy as np
+import warnings
+warnings.simplefilter(action='ignore', category=FutureWarning)
+
+def convert_labels(inference_labels_dict, bbox_labels):
+    for k, v in inference_labels_dict.items():
+        bbox_labels[bbox_labels == k] = v
+
+    # FutureWarning: elementwise comparison failed; returning scalar, but in the future will perform elementwise comparison
+    # throws up this warning because making a change string to int is something that numpy disagrees with (???). 
+
+    return bbox_labels
+
+def process_masks(inference_outputs, inference_labels_dict):
+
+    mask_labels = convert_labels(inference_labels_dict, inference_outputs["bbox_labels"])
+    masks  = inference_outputs["masks"]
+    scores = inference_outputs['bbox_scores']
+    
+    processed_output = [[masks[i], scores[i], int(mask_labels[i])] for i in range(len(scores))] 
+
+    return processed_output
+
+def process_bboxes(inference_outputs, inference_labels_dict):
+
+    bbox_labels = inference_outputs["bbox_labels"]
+    bbox_labels = convert_labels(inference_labels_dict, bbox_labels)
+    bboxes = inference_outputs["bboxes"]
+    bbox_scores = inference_outputs["bbox_scores"]
+    
+    # stack the bbox_scores and bbox_labels
+    # hence, array(['score', 'score','score']) and array(['class','class','class'])
+    # becomes array([['score','class'], ['score','class'],['score','class']])
+    stacked = np.stack((bbox_scores, bbox_labels), axis = 1)
+
+    # concatenate the values of the bbox wih the stacked values above
+    # use concatenate here because it is 1xnxm with 1xnxl dimension so it works
+    # it's just maths, people!
+    concated = np.concatenate((bboxes, stacked), axis = 1)
+
+    return concated.astype(np.float32)
+
+def run_inference(img_matrix, model, inference_labels_dict = {'person': 1, 'bicycle': 2}, task = "det"):
+    """Helper function to run per image inference, get bbox, labels and scores and stack them for confusion matrix output
+
+    Args:
+        img_matrix (np.array): _description_
+        model: _description_
+        labels_dict (dict, optional): _description_. Defaults to {'person': 0, 'bicycle': 1}.
+
+    Returns:
+        concated (np.array): concatenated inference of n x (bbox (default is x1, y1, x2, y2), score, class)
+        img_matrix.shape (np vector): vector with [Height * Weight * Dimension] values
+    """
+    # print(img_matrix.shape)
+    # for img_matrix, it's HxWxD. Need to resize it for the confusion matrix
+
+    # modify this to change the run to your model's inference method eg model(img) in pytorch
+    inference_inputs = {"img": img_matrix}
+    inference_outputs = model.run(inference_inputs)
+
+    # pkd outputs for segmentation - {'bboxes': [[],[]..,[]], 'bbox_labels':[], 'bbox_scores':[], 'masks':[[[],[],[]]]}
+
+    if task == "seg":
+        processed_output = process_masks(inference_outputs, inference_labels_dict)
+    
+    elif task == "det":
+        processed_output = process_bboxes(inference_outputs, inference_labels_dict)
+
+    return processed_output, img_matrix.shape
+
+class Inference:
+
+    def __init__(self, model, cfg_obj):
+        
+        self.model = model
+        self.labels_dict = cfg_obj['error_analysis']['labels_dict']
+        self.inference_labels_dict = cfg_obj['error_analysis']['inference_labels_dict']
+        self.task = cfg_obj['error_analysis']['task']
+        
+    def run_inference_path(self, img_path):
+        """use if img_path is specified 
+
+        Args:
+            img_path (_type_): _description_
+
+        Returns:
+            _type_: _description_
+        """
+        image_orig = cv2.imread(img_path)
+        image_orig = cv2.cvtColor(image_orig, cv2.COLOR_BGR2RGB)
+
+        output = run_inference(image_orig, self.model, inference_labels_dict = self.inference_labels_dict, task = self.task)
+
+        return output
+    
+    def run_inference_byte(self, img_bytes):
+        """use if the img_bytes is passed in instead of path
+
+        Args:
+            img_bytes (_type_): _description_
+
+        Returns:
+            _type_: _description_
+        """
+        img_decoded = cv2.imdecode(np.frombuffer(img_bytes, np.uint8), -1)
+        img_decoded = cv2.cvtColor(img_decoded, cv2.COLOR_BGR2RGB)
+
+        output = run_inference(img_decoded, self.model, labels_dict = self.inference_labels_dict, task = self.task)
+
+        return output
+
+if __name__ == "__main__":
+    import yaml
+    from src.error_analysis import load_model
+    cfg_file = open("cfg/cfg.yml")
+    cfg_obj = yaml.load(cfg_file, Loader=yaml.FullLoader)
+    img_path = "./data/annotations_trainval2017/coco_person/000000576052.jpg"
+    inference_obj = Inference(model = load_model(cfg_obj), cfg_obj = cfg_obj)
+    output = inference_obj.run_inference_path(img_path)
+    print (output)

src/pred_analysis_STEE.py

@@ -0,0 +1,595 @@
+import os
+import re
+import cv2
+import time
+
+import numpy as np
+import pandas as pd
+import xml.etree.ElementTree as ET
+
+from pathlib import Path
+from torchvision import transforms
+from configparser import ConfigParser, ExtendedInterpolation
+from ast import literal_eval
+
+from src.models.model import Model
+from src.models.eval.confusion_matrix import ConfusionMatrix
+
+
+def generate_inference_from_img_folder(csv_file, model_cfg, img_folder, ckpt_file, 
+nms_thresh, conf_thresh, device="cuda" ,csv_path=None):
+    """[Retrieve the inference information of the test images given a model checkpoint trained]
+
+    Parameters
+    ----------
+    csv_file : [str]
+        [path of the csv file containing the information of the test images]
+    model_cfg : [str]
+        [path of the model config file to use, specific to the checkpoint file]
+    img_folder : [str]
+        [folder containing the images]
+    ckpt_file : [str]
+        [path of the model checkpoint file to use for model inference]
+    nms_thresh : [float]
+        [Non-maximum suppression threshold to use for the model inference, values between 0 to 1]
+    conf_thresh : [float]
+        [Confidence threshold to use for the model inference, values between 0 to 1]
+    device : str, optional
+        [device to use for inference, option: "cuda" or "cpu"], by default "cuda"
+    csv_path : [str], optional
+        [path to save the pandas.DataFrame output as a csv], by default None i.e. csv not generated 
+
+    Returns
+    -------
+    df : [pandas.DataFrame]
+        [dataframe containing the inference information of the test images]
+    """
+
+    pl_config = ConfigParser(interpolation=ExtendedInterpolation())
+    pl_config.read(model_cfg)
+    
+    model_selected = Model(pl_config)
+
+    df_original = pd.read_csv(csv_file)
+    # Only perform inference on test images with at least 1 ground truth.
+    df_test = df_original[df_original['remarks_xml'] == 'Available xml file'].reset_index()
+    df_test = df_test[df_test['set_type'] == 'Test'].reset_index()
+
+    img_number = 0
+    prediction_info_list = []
+    for _,rows in df_test.iterrows():
+        img_file = rows["image_file_name"]
+        img_number += 1
+        inference_start_time = time.time()
+        img_file_path = os.path.join(img_folder,img_file)
+
+        # Perform inference on image with ckpt file with device either "cuda" or "cpu"
+        # img_inference = model_selected.inference(device='cpu', img_path=img_file_path, ckpt_path=ckpt_file)
+        img_inference = model_selected.inference(
+            device=device, img_path=img_file_path, ckpt_path=ckpt_file, nms_thresh=nms_thresh, conf_thresh=conf_thresh)
+
+        # Sieve out inference
+        predicted_boxes_unsorted = img_inference[0].tolist()
+        predicted_labels_unsorted = img_inference[1].tolist()
+        predicted_confidence_unsorted = img_inference[2].tolist()
+        
+        # print(f"Pre Boxes: {predicted_boxes}")
+        # print(f"Pre Labels: {predicted_labels}")
+        # print(f"Pre Labels: {predicted_confidence}")
+
+        # Sorting input
+        predicted_boxes = [x for _,x in sorted(zip(predicted_confidence_unsorted,predicted_boxes_unsorted), reverse=True)]
+        predicted_labels = [x for _,x in sorted(zip(predicted_confidence_unsorted,predicted_labels_unsorted), reverse=True)]
+        predicted_confidence = sorted(predicted_confidence_unsorted, reverse=True)
+
+        # print(f"Post Boxes: {predicted_boxes}")
+        # print(f"Post Labels: {predicted_labels}")
+        # print(f"Post Labels: {predicted_confidence}")
+
+        predicted_boxes_int = []
+        for box in predicted_boxes:
+            box_int = [round(x) for x in box]
+            predicted_boxes_int.append(box_int)
+        
+        # Prepare inputs for confusion matrix
+        cm_detections_list = []
+        for prediction in range(len(predicted_boxes)):
+            detection_list = predicted_boxes[prediction]
+            detection_list.append(predicted_confidence[prediction])
+            detection_list.append(predicted_labels[prediction])
+            cm_detections_list.append(detection_list)
+
+        # Re generate predicted boxes
+        predicted_boxes = [x for _,x in sorted(zip(predicted_confidence_unsorted,predicted_boxes_unsorted), reverse=True)]
+        
+        inference_time_per_image = round(time.time() - inference_start_time, 2)
+        if img_number%100 == 0: 
+            print(f'Performing inference on Image {img_number}: {img_file_path}')
+            print(f'Time taken for image: {inference_time_per_image}')
+
+        prediction_info = {
+            "image_file_path": img_file_path,
+            "image_file_name": img_file,
+            "number_of_predictions": len(predicted_boxes),
+            "predicted_boxes": predicted_boxes,
+            "predicted_boxes_int": predicted_boxes_int,
+            "predicted_labels": predicted_labels,
+            "predicted_confidence": predicted_confidence,
+            "cm_detections_list": cm_detections_list,
+            "inference_time": inference_time_per_image
+        }
+        prediction_info_list.append(prediction_info)
+
+    df = pd.DataFrame(prediction_info_list)
+    
+    if csv_path is not None:
+        df.to_csv(csv_path, index=False)
+        print ("Dataframe saved as csv to " + csv_path)
+
+    return df
+
+def get_gt_from_img_folder(csv_file, img_folder, xml_folder, names_file, map_start_index=1, csv_path=None):
+    """[Retrieve the ground truth information of the test images]
+
+    Parameters
+    ----------
+    csv_file : [str]
+        [path of the csv file containing the information of the test images]
+    img_folder : [str]
+        [folder containing the images]
+    xml_folder : [str]
+        [folder containing the xml files associated with the images]
+    names_file : [str]
+        [names file containing the class labels of interest]
+    map_start_index : int, optional
+        [attach a number to each class label listed in names file, starting from number given by map_start_index], by default 1
+    csv_path : [str], optional
+        [path to save the pandas.DataFrame output as a csv], by default None i.e. csv not generated
+
+    Returns
+    -------
+    df : [pandas.DataFrame]
+        [dataframe containing the ground truth information of the test images]
+    """
+
+    df_original = pd.read_csv(csv_file)
+
+    # Only perform inference on test images with at least 1 ground truth.
+    df_test = df_original[df_original['remarks_xml'] == 'Available xml file'].reset_index()
+    df_test = df_test[df_test['set_type'] == 'Test'].reset_index()
+    
+    # Create a dictionary to map numeric class as class labels
+    class_labels_dict = {}
+    with open(names_file) as f:
+        for index,line in enumerate(f):
+            idx = index + map_start_index
+            class_labels = line.splitlines()[0]
+            class_labels_dict[class_labels] = idx
+
+    gt_info_list = []
+    # for img_file in os.listdir(img_folder):
+    #     if re.search(".jpg", img_file):    
+    for _,rows in df_test.iterrows():
+        img_file = rows["image_file_name"]
+        # file_stem = Path(img_file_path).stem
+
+        # Get img tensor            
+        img_file_path = os.path.join(img_folder,img_file)
+        img = cv2.imread(filename = img_file_path)
+        img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
+
+        # Get associated xml file 
+        file_stem = Path(img_file_path).stem
+        xml_file_path =  xml_folder + file_stem + ".xml"
+
+        tree = ET.parse(xml_file_path)
+        root = tree.getroot()
+
+        for image_detail in root.findall('size'):
+            image_width = float(image_detail.find('width').text)
+            image_height = float(image_detail.find('height').text)
+            
+        class_index_list = []
+        bb_list = []
+        truncated_list = []
+        occluded_list = []
+        for item in root.findall('object'):                
+            if item.find('truncated') is not None:
+                truncated = int(item.find('truncated').text)
+            else:
+                truncated = 0
+
+            if item.find('occluded').text is not None:
+                occluded = int(item.find('occluded').text)
+            else:
+                occluded = 0
+
+            for bb_details in item.findall('bndbox'):
+                class_label = item.find('name').text
+                class_index = class_labels_dict[class_label]
+                xmin = float(bb_details.find('xmin').text)
+                ymin = float(bb_details.find('ymin').text)
+                xmax = float(bb_details.find('xmax').text)
+                ymax = float(bb_details.find('ymax').text)
+
+                class_index_list.append(class_index)
+                bb_list.append([xmin,ymin,xmax,ymax])
+                truncated_list.append(truncated)
+                occluded_list.append(occluded)
+
+        transform = A.Compose([
+            A.Resize(608,608),
+            ToTensor()
+            ],
+            bbox_params=A.BboxParams(format='pascal_voc',
+            label_fields=['class_labels']),
+            )
+
+        augmented = transform(image=img, bboxes = bb_list, class_labels = class_index_list)
+        # img comes out as int, need to change to float.
+        img = augmented['image'].float()
+        gt_boxes = augmented['bboxes']
+        gt_boxes_list = [list(box) for box in gt_boxes]
+        gt_labels = augmented['class_labels'] 
+        
+        gt_boxes_int = []
+        for box in gt_boxes:
+            box_int = [round(x) for x in box]
+            gt_boxes_int.append(box_int)       
+
+        cm_gt_list = []     
+        for gt in range(len(gt_boxes)):
+            gt_list = [gt_labels[gt]]
+            gt_list.extend(gt_boxes[gt])
+            cm_gt_list.append(gt_list)
+
+        # Calculate and Group by Size of Ground Truth 
+        gt_area_list = []
+        gt_area_type = []
+        for gt_box in gt_boxes:
+            gt_area = (gt_box[3] - gt_box[1]) * (gt_box[2] - gt_box[0])
+            gt_area_list.append(gt_area)
+
+            if gt_area < 32*32:
+                area_type = "S"
+                gt_area_type.append(area_type)       
+            elif gt_area < 96*96:
+                area_type = "M"
+                gt_area_type.append(area_type)  
+            else:
+                area_type = "L"
+                gt_area_type.append(area_type)
+        
+        gt_info = {
+            "image_file_path": img_file_path,
+            "image_file_name": img_file,
+            "image_width": image_width,
+            "image_height": image_height,
+            "number_of_gt": len(gt_boxes_list),
+            "gt_labels": gt_labels,
+            "gt_boxes": gt_boxes_list,
+            "gt_boxes_int": gt_boxes_int,
+            "cm_gt_list": cm_gt_list,
+            "gt_area_list": gt_area_list,
+            "gt_area_type": gt_area_type,
+            "truncated_list": truncated_list,
+            "occluded_list": occluded_list
+        }
+        gt_info_list.append(gt_info)
+
+    df = pd.DataFrame(gt_info_list)
+    
+    if csv_path is not None:
+        df.to_csv(csv_path, index=False)
+        print ("Dataframe saved as csv to " + csv_path)
+
+    return df
+
+def combine_gt_predictions(csv_file, img_folder, xml_folder, names_file, model_cfg, ckpt_file, csv_save_folder, 
+device="cuda", nms_threshold=0.1, confidence_threshold=0.7, iou_threshold=0.4, gt_statistics=True):
+    """[Retrieve the combined inference and ground truth information of the test images]
+
+    Parameters
+    ----------
+    csv_file : [str]
+        [path of the csv file containing the information of the test images]
+    img_folder : [str]
+        [folder containing the images]
+    xml_folder : [str]
+        [folder containing the xml files associated with the images]
+    names_file : [str]
+        [names file containing the class labels of interest]
+    model_cfg : [str]
+        [path of the model config file to use, specific to the checkpoint file]
+    ckpt_file : [str]
+        [path of the model checkpoint file to use for model inference]
+    csv_save_folder : [str]
+        [folder to save the generated csv files]
+    device : str, optional
+        [device to use for inference, option: "cuda" or "cpu"], by default "cuda"
+    nms_threshold : float, optional
+        [Non-maximum suppression threshold to use for the model inference, values between 0 to 1], by default 0.1
+    confidence_threshold : float, optional
+        [Confidence threshold to use for the model inference, values between 0 to 1], by default 0.7
+    iou_threshold : float, optional
+        [IOU threshold to use for identifying true positives from the predictions and ground truth], by default 0.4
+    gt_statistics : bool, optional
+        [option to generate the df_gt_analysis], by default True
+
+    Returns
+    -------
+    df_full : [pandas.DataFrame]
+        [dataframe containing the combined inference and ground truth information of the test images by image]
+    df_gt_analysis : pandas.DataFrame, optional
+        [dataframe containing the combined inference and ground truth information of the test images by ground truth]
+    """
+
+    print(f"NMS Threshold: {nms_threshold}")
+    print(f"Confidence Threshold: {confidence_threshold}")
+    print(f"IOU Threshold: {iou_threshold}")
+        
+    df_gt = get_gt_from_img_folder(
+        csv_file, img_folder, xml_folder, names_file)
+    print("Successful Generation of Ground Truth Information")
+    df_predictions = generate_inference_from_img_folder(
+        csv_file, model_cfg, img_folder, ckpt_file, 
+        nms_thresh=nms_threshold, conf_thresh=confidence_threshold, device=device)
+    print("Successful Generation of Inference")
+
+    df_all = pd.merge(df_gt, df_predictions, how='left', on=["image_file_path", "image_file_name"])
+    print("Successful Merging")
+    
+    class_labels_list = []
+    with open(names_file) as f:
+        for index,line in enumerate(f):
+            class_labels = line.splitlines()[0]
+            class_labels_list.append(class_labels)
+
+    combined_info_list = []
+    for _,rows in df_all.iterrows():
+        img_file = rows["image_file_name"]
+        predicted_boxes = rows["predicted_boxes"]
+        predicted_labels = rows["predicted_labels"]
+        predicted_confidence = rows["predicted_confidence"]
+        gt_boxes = rows["gt_boxes"]
+        gt_labels = rows["gt_labels"]
+        cm_gt_list = rows["cm_gt_list"]
+        cm_detections_list = rows["cm_detections_list"]
+        
+        if rows["number_of_predictions"] == 0:
+            # Ground Truth Analysis
+            gt_summary_list = []
+            gt_match_list = []            
+            gt_match_idx_list = []
+            gt_match_idx_conf_list = []
+            gt_match_idx_bb_list = []
+            for idx in range(len(gt_labels)):
+                gt_summary = "NO"
+                match = ["GT", idx, "-"]
+                match_idx = "-"
+                match_bb = "-"
+                gt_summary_list.append(gt_summary)
+                gt_match_list.append(tuple(match))
+                gt_match_idx_list.append(match_idx)
+                gt_match_idx_conf_list.append(match_idx)
+                gt_match_idx_bb_list.append(match_bb)
+
+            combined_info = {
+                "image_file_name": img_file,
+                "number_of_predictions_conf": [],
+                "predicted_labels_conf": [],
+                "predicted_confidence_conf": [],
+                "num_matches": [],
+                "num_mismatch": [],
+                "labels_hit": [],
+                "pairs_mislabel_gt_prediction": [],
+                "gt_match_idx_list": gt_match_idx_list,
+                "gt_match_idx_conf_list": gt_match_idx_conf_list,
+                "gt_match_idx_bb_list": gt_match_idx_bb_list,
+                "prediction_match": [],
+                "gt_analysis": gt_summary_list,
+                "prediction_analysis": [],
+                "gt_match": gt_match_list
+            }
+        
+        else:
+
+            # Generate Confusion Matrix with their corresponding matches
+            CM = ConfusionMatrix(
+                num_classes=len(class_labels_list)+1, 
+                CONF_THRESHOLD = confidence_threshold, 
+                IOU_THRESHOLD = iou_threshold)
+
+            matching_boxes = CM.process_batch(
+                detections=np.asarray(cm_detections_list),
+                labels=np.asarray(cm_gt_list),
+                return_matches=True)
+            
+            predicted_confidence_count = len([confidence for confidence in predicted_confidence if confidence > confidence_threshold]) 
+            predicted_confidence_round = [round(confidence, 4) for confidence in predicted_confidence]
+
+            predicted_confidence_conf = predicted_confidence_round[:predicted_confidence_count]
+            predicted_labels_conf = predicted_labels[:predicted_confidence_count]
+            predicted_boxes_conf = predicted_boxes[:predicted_confidence_count]
+
+            number_of_predictions_conf = len(predicted_labels_conf)
+
+            match_correct_list = []
+            match_wrong_list = []
+            gt_matched_idx_dict = {}
+            predicted_matched_idx_dict = {}
+            gt_mismatch_idx_dict = {}
+            predicted_mismatch_idx_dict = {}
+            labels_hit = []
+            pairs_mislabel_gt_prediction = []
+
+            for match in matching_boxes:            
+                gt_idx = int(match[0])
+                predicted_idx = int(match[1])
+                iou = round(match[2], 4)
+                match = [gt_idx, predicted_idx, iou]        
+
+                if gt_labels[gt_idx] == predicted_labels_conf[predicted_idx]:
+                    match_correct_list.append(match)
+                    gt_matched_idx_dict[gt_idx] = match
+                    predicted_matched_idx_dict[predicted_idx] = match
+                    labels_hit.append(gt_labels[gt_idx])
+                else:
+                    match_wrong_list.append(match)
+                    gt_mismatch_idx_dict[gt_idx] = match
+                    predicted_mismatch_idx_dict[predicted_idx] = match
+                    pairs_mislabel_gt_prediction.append(
+                        [gt_labels[gt_idx],predicted_labels_conf[predicted_idx]])
+
+            # Ground Truth Analysis
+            gt_summary_list = []
+            gt_match_list = []
+            gt_match_idx_list = []
+            gt_match_idx_conf_list = []
+            gt_match_idx_bb_list = []
+            for idx in range(len(gt_labels)):
+                if idx in gt_matched_idx_dict.keys():
+                    gt_summary = "MATCH"
+                    match =  gt_matched_idx_dict[idx]
+                    match_idx = predicted_labels_conf[match[1]]
+                    match_conf = predicted_confidence_conf[match[1]]
+                    match_bb = predicted_boxes_conf[match[1]]
+                elif idx in gt_mismatch_idx_dict.keys():
+                    gt_summary = "MISMATCH"
+                    match =  gt_mismatch_idx_dict[idx]
+                    match_idx = predicted_labels_conf[match[1]]
+                    match_conf = predicted_confidence_conf[match[1]]
+                    match_bb = predicted_boxes_conf[match[1]]
+                else:
+                    gt_summary = "NO"
+                    match = ["GT", idx, "-"]
+                    match_idx = "-"
+                    match_conf = "-"
+                    match_bb = "-"
+                gt_summary_list.append(gt_summary)
+                gt_match_list.append(tuple(match))
+                gt_match_idx_list.append(match_idx)
+                gt_match_idx_conf_list.append(match_conf)
+                gt_match_idx_bb_list.append(match_bb)
+
+            # Prediction Analysis
+            prediction_summary_list = []
+            prediction_match_list = []
+            for idx in range(len(predicted_labels_conf)):
+                if idx in predicted_matched_idx_dict.keys():
+                    prediction_summary = "MATCH"
+                    match = predicted_matched_idx_dict[idx]
+                elif idx in predicted_mismatch_idx_dict.keys():
+                    prediction_summary = "MISMATCH"
+                    match = predicted_mismatch_idx_dict[idx]
+                else:
+                    prediction_summary = "NO"
+                    match = [idx, "P", "-"]
+                prediction_summary_list.append(prediction_summary)
+                prediction_match_list.append(tuple(match))
+
+            combined_info = {
+                    "image_file_name": img_file,
+                    "number_of_predictions_conf": number_of_predictions_conf,
+                    "predicted_labels_conf": predicted_labels_conf,
+                    "predicted_confidence_conf": predicted_confidence_conf,
+                    "num_matches": len(match_correct_list),
+                    "num_mismatch": len(match_wrong_list),
+                    "labels_hit": labels_hit,
+                    "pairs_mislabel_gt_prediction": pairs_mislabel_gt_prediction,
+                    "gt_match_idx_list": gt_match_idx_list,
+                    "gt_match_idx_conf_list": gt_match_idx_conf_list,
+                    "gt_match_idx_bb_list": gt_match_idx_bb_list,
+                    "gt_match": gt_match_list,
+                    "prediction_match": prediction_match_list,
+                    "gt_analysis": gt_summary_list,
+                    "prediction_analysis": prediction_summary_list        
+                }
+
+        combined_info_list.append(combined_info)
+    
+    df_combined = pd.DataFrame(combined_info_list)
+
+    df_full = pd.merge(df_all, df_combined , how='left', on=["image_file_name"])
+
+    csv_path_combined = f"{csv_save_folder}df_inference_details_nms_{nms_threshold}_conf_{confidence_threshold}_iou_{iou_threshold}.csv"
+
+    df_full.to_csv(csv_path_combined, index=False)
+    print ("Dataframe saved as csv to " + csv_path_combined)
+
+    if gt_statistics:
+        print("Generating Statistics for Single Ground Truth")        
+        csv_path_gt = f"{csv_save_folder}df_gt_details_nms_{nms_threshold}_conf_{confidence_threshold}_iou_{iou_threshold}.csv"
+        df_gt_analysis = __get_single_gt_analysis(csv_output=csv_path_gt, df_input=df_full)
+
+        return df_full, df_gt_analysis
+    
+    else:
+        return df_full
+
+def __get_single_gt_analysis(csv_output, df_input=None,csv_input=None):
+
+    if df_input is None:
+        df_gt = pd.read_csv(csv_input)
+
+        # Apply literal eval of columns containing information on Ground Truth
+        df_gt.gt_labels = df_gt.gt_labels.apply(literal_eval)
+        df_gt.gt_boxes = df_gt.gt_boxes.apply(literal_eval)
+        df_gt.gt_boxes_int = df_gt.gt_boxes_int.apply(literal_eval)
+        df_gt.gt_area_list = df_gt.gt_area_list.apply(literal_eval)
+        df_gt.gt_area_type = df_gt.gt_area_type.apply(literal_eval)
+        df_gt.truncated_list = df_gt.truncated_list.apply(literal_eval)
+        df_gt.occluded_list = df_gt.occluded_list.apply(literal_eval)
+        df_gt.gt_match_idx_list = df_gt.gt_match_idx_list.apply(literal_eval)
+        df_gt.gt_match_idx_conf_list = df_gt.gt_match_idx_conf_list.apply(literal_eval)
+        df_gt.gt_match_idx_bb_list = df_gt.gt_match_idx_bb_list.apply(literal_eval)
+        df_gt.gt_match = df_gt.gt_match.apply(literal_eval)
+        df_gt.gt_analysis = df_gt.gt_analysis.apply(literal_eval)
+    
+    else:
+        df_gt = df_input
+    
+    gt_info_list = []
+    for _,rows in df_gt.iterrows():
+        # print(rows["image_file_name"])
+        for idx in range(rows["number_of_gt"]):
+            df_gt_image_dict = {
+                "GT_Image": rows["image_file_name"],
+                "GT_Label": rows["gt_labels"][idx],
+                "GT_Boxes": rows["gt_boxes"][idx],
+                "GT_Boxes_Int": rows["gt_boxes_int"][idx],
+                "GT_Area": rows["gt_area_list"][idx],
+                "GT_Area_Type": rows["gt_area_type"][idx],
+                "Truncated": rows["truncated_list"][idx],
+                "Occluded": rows["occluded_list"][idx],
+                "GT_Match": rows["gt_match"][idx],
+                "IOU": rows["gt_match"][idx][2],
+                "GT_Match_IDX": rows["gt_match_idx_list"][idx],
+                "GT_Confidence_IDX": rows["gt_match_idx_conf_list"][idx],
+                "GT_Predicted_Boxes_IDX": rows["gt_match_idx_bb_list"][idx],
+                "GT_Analysis": rows["gt_analysis"][idx]
+                }
+            gt_info_list.append(df_gt_image_dict)
+
+    df_final = pd.DataFrame(gt_info_list)
+    df_final = df_final.reset_index(drop=True)
+
+    df_final.to_csv(csv_output, index=False)
+    print ("Dataframe saved as csv to " + csv_output)
+
+    return df_final
+
+if __name__ == '__main__':
+    
+    combine_gt_predictions(
+        csv_file="/polyaxon-data/workspace/stee/voc_image_annotations_batch123.csv",
+        img_folder="/polyaxon-data/workspace/stee/data_batch123",
+        xml_folder="/polyaxon-data/workspace/stee/data_batch123/Annotations/",
+        names_file="/polyaxon-data/workspace/stee/data_batch123/obj.names",
+        model_cfg="cfg/cfg_frcn.ini",
+        ckpt_file="/polyaxon-data/workspace/stee/andy/epoch=99-step=61899.ckpt",
+        csv_save_folder="/polyaxon-data/workspace/stee/andy/generation/",
+        nms_threshold=0.9, 
+        confidence_threshold=0.3, 
+        iou_threshold=0.4,
+        gt_statistics=False)  

src/st_image_tools.py

@@ -0,0 +1,441 @@
+import streamlit as st
+import numpy as np
+import plotly.express as px
+import cv2
+from src.error_analysis import ErrorAnalysis, transform_gt_bbox_format
+import yaml
+import os
+from src.confusion_matrix import ConfusionMatrix
+from plotly.subplots import make_subplots
+import plotly.graph_objects as go
+import pandas as pd
+
+
+def amend_cm_df(cm_df, labels_dict):
+    """Helper function to amend the index and column name for readability
+    Example - index currently is 0, 1 ... -> GT - person
+    Likewise in Column - 0, 1 ... -> Pred - person etc
+
+    Args:
+        cm_df (_type_): confusion matrix dataframe.
+        labels_dict (_type_): dictionary of the class labels
+
+    Returns:
+        cm_df: confusion matrix dataframe with index and column names filled
+    """
+
+    index_list = list(labels_dict.values())
+    index_list.append("background")
+
+    cm_df = cm_df.set_axis([f"GT - {elem}" for elem in index_list])
+    cm_df = cm_df.set_axis([f"Pred - {elem}" for elem in index_list], axis=1)
+    cm_df = cm_df.astype(int)
+
+    return cm_df
+
+def find_top_left_pos(mask):
+    """gets the top left position of the mask 
+
+    Args:
+        mask (_type_): _description_
+
+    Returns:
+        _type_: _description_
+    """
+
+    return np.unravel_index(np.argmax(mask, axis=None), mask.shape)
+
+
+class ImageTool:
+    
+    def __init__(self, cfg_path="cfg/cfg.yml"):
+
+        # getting the config object                
+        cfg_file = open(cfg_path)
+        self.cfg_obj = yaml.load(cfg_file, Loader=yaml.FullLoader)
+        
+        # initialising the model and getting the annotations
+        self.ea_obj = ErrorAnalysis(cfg_path)
+        self.inference_folder = self.ea_obj.inference_folder
+        self.ea_obj.get_annots()
+        self.gt_annots = self.ea_obj.gt_dict
+        self.all_img = os.listdir(self.inference_folder)
+        self.ea_obj.model.score_threshold = self.cfg_obj["visual_tool"]["conf_threshold"]
+        self.ea_obj.model.iou_threshold = self.cfg_obj["visual_tool"]["iou_threshold"]
+
+        # for labels
+        self.labels_dict = self.cfg_obj["error_analysis"]["labels_dict"]
+        self.labels_dict = {v: k for k, v in self.labels_dict.items()}
+        self.inference_labels_dict = self.cfg_obj["error_analysis"]["inference_labels_dict"]
+        self.inference_labels_dict = {v: k for k, v in self.inference_labels_dict.items()}
+        self.idx_base = self.cfg_obj["error_analysis"]["idx_base"]
+
+        # for visualisation
+        self.bbox_thickness = self.cfg_obj["visual_tool"]["bbox_thickness"]
+        self.font_scale = self.cfg_obj["visual_tool"]["font_scale"]
+        self.font_thickness = self.cfg_obj["visual_tool"]["font_thickness"]
+        self.pred_colour = tuple(self.cfg_obj["visual_tool"]["pred_colour"])
+        self.gt_colour = tuple(self.cfg_obj["visual_tool"]["gt_colour"])
+
+    def show_img(self, img_fname="000000011149.jpg", show_preds=False, show_gt=False):
+        """generate img with option to overlay with GT and/or preds
+
+        Args:
+            img_fname (str, optional): Filename of the image. Defaults to "000000011149.jpg".
+            show_preds (bool, optional): Toggle True to run model to get the preds. Defaults to False.
+            show_gt (bool, optional): Toggle True to get the GT labels/boxes. Defaults to False.
+
+        Returns:
+            fig (Plotly Figure): image with overlays if toggled True
+            cm_df (pd.DataFrame): confusion matrix of the pred versus GT
+            cm_tpfpfn_dict (Dict): confusion matrix dictionary of tp/fp/fn
+        """
+
+        # get the image's file path. Concatenates with the folder in question
+        img = cv2.imread(f"{self.inference_folder}{img_fname}")
+
+        labels = {"x": "X", "y": "Y", "color": "Colour"}
+
+        if show_preds:
+
+            preds = self.get_preds(img_fname)
+            if self.ea_obj.task == "det":
+                img = self.draw_pred_bboxes(img, preds)
+            elif self.ea_obj.task == "seg":
+                img = self.draw_pred_masks(img, preds)
+
+        if show_gt:
+
+            gt_annots = self.get_gt_annot(img_fname)
+
+            if self.ea_obj.task == "det":
+                img = self.draw_gt_bboxes(img, preds)
+            elif self.ea_obj.task == "seg":
+                img = self.draw_gt_masks(img, gt_annots)
+
+        fig = px.imshow(img[..., ::-1], aspect="equal", labels=labels)
+
+        if show_gt and show_preds:
+
+            cm_df, cm_tpfpfn_dict = self.generate_cm_one_image(preds, gt_annots)
+            return [fig, cm_df, cm_tpfpfn_dict]
+
+        return fig
+
+    def show_img_sbs(self, img_fname="000000011149.jpg"):
+        """generate two imageso with confusion matrix and tp/fp/fn. fig1 is image with GT overlay, while fig2 is the image witih pred overlay.
+
+        Args:
+            img_fname (str, optional): Filename of the image. Defaults to "000000011149.jpg".
+
+        Returns:
+            list: fig1 - imshow of image with GT overlay
+                  fig2 - imshow of image with pred overlay
+                  cm_df - confusion matrix dataframe
+                  cm_tpfpfn_df - confusion matrix dictionary of tp/fp/fn
+        """
+
+        # shows the image side by side
+        img = cv2.imread(f"{self.inference_folder}{img_fname}")
+        labels = {"x": "X", "y": "Y", "color": "Colour"}
+
+        img_pred = img.copy()
+        img_gt = img.copy()
+
+        preds = self.get_preds(img_fname)
+
+        gt_annots = self.get_gt_annot(img_fname)
+
+        if self.ea_obj.task == 'det':
+            img_pred = self.draw_pred_bboxes(img_pred, preds)
+            img_gt = self.draw_gt_bboxes(img_gt, gt_annots)
+
+        elif self.ea_obj.task == 'seg':
+            img_pred = self.draw_pred_masks(img_pred, preds)
+            img_gt = self.draw_gt_masks(img_gt, gt_annots)
+
+
+        fig1 = px.imshow(img_gt[..., ::-1], aspect="equal", labels=labels)
+        fig2 = px.imshow(img_pred[..., ::-1], aspect="equal", labels=labels)
+        fig2.update_yaxes(visible=False)
+
+        cm_df, cm_tpfpfn_df = self.generate_cm_one_image(preds, gt_annots)
+
+        return [fig1, fig2, cm_df, cm_tpfpfn_df]
+
+    def generate_cm_one_image(self, preds, gt_annots):
+        """Generates confusion matrix between the inference and the Ground Truth of an image
+
+        Args:
+            preds (array): inference output of the model on the image
+            gt_annots (array): Ground Truth labels of the image
+
+        Returns:
+            cm_df (DataFrame): Confusion matrix dataframe. 
+            cm_tpfpfn_df (DataFrame): TP/FP/FN dataframe
+        """
+
+        num_classes = len(list(self.cfg_obj["error_analysis"]["labels_dict"].keys()))
+        idx_base = self.cfg_obj["error_analysis"]["idx_base"]
+
+        conf_threshold, iou_threshold = (
+            self.ea_obj.model.score_threshold,
+            self.ea_obj.model.iou_threshold,
+        )
+        cm = ConfusionMatrix(
+            num_classes=num_classes,
+            CONF_THRESHOLD=conf_threshold,
+            IOU_THRESHOLD=iou_threshold,
+        )
+        if self.ea_obj.task == 'det':
+            gt_annots[:, 0] -= idx_base
+            preds[:, -1] -= idx_base
+        elif self.ea_obj.task == 'seg':
+            gt_annots = [[gt[0] - idx_base, gt[1]] for gt in gt_annots]
+
+        cm.process_batch(preds, gt_annots, task = self.ea_obj.task)
+
+        confusion_matrix_df = cm.return_as_df()
+        cm.get_tpfpfn()
+        
+        cm_tpfpfn_dict = {
+            "True Positive": cm.tp,
+            "False Positive": cm.fp,
+            "False Negative": cm.fn,
+        }
+
+        cm_tpfpfn_df = pd.DataFrame(cm_tpfpfn_dict, index=[0])
+        cm_tpfpfn_df = cm_tpfpfn_df.set_axis(["Values"], axis=0)
+        cm_tpfpfn_df = cm_tpfpfn_df.astype(int)
+        # amend df
+
+        confusion_matrix_df = amend_cm_df(confusion_matrix_df, self.labels_dict)
+        # print (cm.matrix)
+
+        return confusion_matrix_df, cm_tpfpfn_df
+
+    def get_preds(self, img_fname="000000011149.jpg"):
+        """Using the model in the Error Analysis object, run inference to get outputs
+
+        Args:
+            img_fname (str): Image filename. Defaults to "000000011149.jpg".
+
+        Returns:
+            outputs (array): Inference output of the model on the image
+        """
+
+        # run inference using the error analysis object per image
+        outputs, img_shape = self.ea_obj.generate_inference(img_fname)
+
+        if self.ea_obj.task == 'det':
+            # converts image coordinates from normalised to integer values
+            # image shape is [Y, X, C] (because Rows are Y)
+            # So don't get confused!
+            outputs[:, 0] *= img_shape[1]
+            outputs[:, 1] *= img_shape[0]
+            outputs[:, 2] *= img_shape[1]
+            outputs[:, 3] *= img_shape[0]
+    
+        return outputs
+
+    def get_gt_annot(self, img_fname):
+        """Retrieve the Ground Truth annotations of the image. 
+
+        Args:
+            img_fname (_type_): Image filename
+
+        Returns:
+            grount_truth (array): GT labels of the image
+        """
+        ground_truth = self.gt_annots[img_fname].copy()
+        img = cv2.imread(f"{self.inference_folder}{img_fname}")
+        
+        # converts image coordinates from normalised to integer values
+        # image shape is [Y, X, C] (because Rows are Y)
+        # So don't get confused!
+        if self.ea_obj.task == 'det':
+            img_shape = img.shape
+            ground_truth = transform_gt_bbox_format(ground_truth, img_shape, format="coco")
+            ground_truth[:, 1] *= img_shape[1]
+            ground_truth[:, 2] *= img_shape[0]
+            ground_truth[:, 3] *= img_shape[1]
+            ground_truth[:, 4] *= img_shape[0]
+
+        return ground_truth
+    
+    def draw_pred_masks(self, img_pred, inference_outputs):
+        """Overlay mask onto img_pred
+
+        Args:
+            img_pred (_type_): _description_
+            preds (_type_): _description_
+        """
+
+        pred_mask = sum([output[0] for output in inference_outputs])
+        pred_mask = np.where(pred_mask > 1, 1, pred_mask)
+        # mask_3d = np.stack((mask,mask,mask),axis=0)
+        # mask_3d = mask_3d.reshape(mask.shape[0], mask.shape[1], 3)
+        colour = np.array(self.pred_colour, dtype='uint8')
+        masked_img = np.where(pred_mask[...,None], colour, img_pred)
+        masked_img = masked_img.astype(np.uint8)
+
+        img_pred = cv2.addWeighted(img_pred, 0.8, masked_img, 0.2, 0)
+        
+        def put_text_ina_mask(output, img):
+            
+            coords = find_top_left_pos(output[0])
+
+            img = cv2.putText(img, self.inference_labels_dict[output[2]], (coords[1], coords[0] + 5), fontFace = cv2.FONT_HERSHEY_SIMPLEX, fontScale = self.font_scale, 
+                        color = self.pred_colour, thickness = self.font_thickness)
+
+            return img
+        
+        for output in inference_outputs:
+            img_pred = put_text_ina_mask(output, img_pred)
+
+        return img_pred
+    
+    def draw_gt_masks(self, img_gt, gt_outputs):
+        """Overlay mask onto img_pred
+
+        Args:
+            img_pred (_type_): _description_
+            preds (_type_): _description_
+        """
+
+        gt_mask = sum([output[1] for output in gt_outputs])
+        gt_mask = np.where(gt_mask > 1, 1, gt_mask)
+        # mask_3d = np.stack((mask,mask,mask),axis=0)
+        # mask_3d = mask_3d.reshape(mask.shape[0], mask.shape[1], 3)
+        colour = np.array(self.gt_colour, dtype='uint8')
+        masked_img = np.where(gt_mask[...,None], colour, img_gt)
+                
+        def put_text_ina_mask(output, img):
+
+            coords = find_top_left_pos(output[1])
+
+            img = cv2.putText(img, self.labels_dict[output[0]], (coords[1], coords[0] + 5), fontFace = cv2.FONT_HERSHEY_SIMPLEX, fontScale = self.font_scale, 
+                        color = self.gt_colour, thickness = self.font_thickness)
+
+            return img
+        
+        img_gt = cv2.addWeighted(img_gt, 0.8, masked_img, 0.2,0)
+
+        for output in gt_outputs:
+            img_gt = put_text_ina_mask(output, img_gt)
+
+        return img_gt
+
+    def draw_pred_bboxes(self, img_pred, preds):
+        """Draws the preds onto the image
+
+        Args:
+            img_pred (array): image 
+            preds (array): model inference outputs 
+
+        Returns:
+            img_pred (array): image with outputs on overlay
+        """
+        for pred in preds:
+            pred = pred.astype(int)
+            img_pred = cv2.rectangle(
+                img_pred,
+                (pred[0], pred[1]),
+                (pred[2], pred[3]),
+                color=self.pred_colour,
+                thickness=self.bbox_thickness,
+            )
+            img_pred = cv2.putText(
+                img_pred,
+                self.labels_dict[pred[5]],
+                (pred[0] + 5, pred[1] + 25),
+                color=self.pred_colour,
+                fontFace=cv2.FONT_HERSHEY_SIMPLEX,
+                fontScale=self.font_scale,
+                thickness=self.font_thickness,
+            )
+        return img_pred
+
+    def draw_gt_bboxes(self, img_gt, gt_annots, **kwargs):
+        """Draws the GT onto the image
+
+        Args:
+            img_gt (array): image
+            gt_annots (array): GT labels
+
+        Returns:
+            img_gt (array): image with GT overlay
+        """
+        for annot in gt_annots:
+            annot = annot.astype(int)
+            # print (annot)
+            img_gt = cv2.rectangle(
+                img_gt,
+                (annot[1], annot[2]),
+                (annot[3], annot[4]),
+                color=self.gt_colour,
+                thickness=self.bbox_thickness,
+            )
+            img_gt = cv2.putText(
+                img_gt,
+                self.labels_dict[annot[0]],
+                (annot[1] + 5, annot[2] + 25),
+                color=(0, 255, 0),
+                fontFace=cv2.FONT_HERSHEY_SIMPLEX,
+                fontScale=self.font_scale,
+                thickness=self.font_thickness,
+            )
+        return img_gt
+
+    def plot_with_preds_gt(self, option, side_by_side=False, plot_type=None):
+        """Rules on what plot to generate
+
+        Args:
+            option (_string_): image filename. Toggled on the app itself. See app.py
+            side_by_side (bool, optional): Whether to have two plots side by side.
+                                            Defaults to False.
+            plot_type (_type_, optional): "all" - both GT and pred will be plotted,
+                                "pred" - only preds,
+                                "GT" - only ground truth
+                                None - only image generated
+                                Will be overridden if side_by_side = True
+                                Defaults to None.
+        """
+
+        if plot_type == "all":
+            plot, df, cm_tpfpfn_df = self.show_img(
+                option, show_preds=True, show_gt=True
+            )
+            st.plotly_chart(plot, use_container_width=True)
+            st.caption("Blue: Model BBox, Green: GT BBox")
+
+            st.table(df)
+            st.table(cm_tpfpfn_df)
+
+        elif plot_type == "pred":
+            st.plotly_chart(
+                self.show_img(option, show_preds=True), use_container_width=True
+            )
+
+        elif plot_type == "gt":
+            st.plotly_chart(
+                self.show_img(option, show_gt=True), use_container_width=True
+            )
+
+        elif side_by_side:
+
+            plot1, plot2, df, cm_tpfpfn_df = self.show_img_sbs(option)
+            col1, col2 = st.columns(2)
+
+            with col1:
+                col1.subheader("Ground Truth")
+                st.plotly_chart(plot1, use_container_width=True)
+            with col2:
+                col2.subheader("Prediction")
+                st.plotly_chart(plot2, use_container_width=True)
+
+            st.table(df)
+            st.table(cm_tpfpfn_df)
+
+        else:
+            st.plotly_chart(self.show_img(option), use_container_width=True)