changed to work with apb files, plotting seperated from decision

app.py

@@ -1,16 +1,31 @@
 import logging
+import os
+import json
+import matplotlib.pyplot as plt
 
 import gradio as gr
+from faiss import read_index_binary, write_index_binary
 
 from config import *
 from videomatch import index_hashes_for_video, get_decent_distance, \
-    get_video_index, compare_videos, get_change_points, get_videomatch_df
+    get_video_index, compare_videos, get_change_points, get_videomatch_df, \
+    get_target_urls
 from plot import plot_comparison, plot_multi_comparison, plot_segment_comparison
 
 logging.basicConfig()
 logging.getLogger().setLevel(logging.INFO)
                       
-
+def transfer_data_indices_to_temp(temp_path = VIDEO_DIRECTORY, data_path='./data'):
+    """ The binary indices created from the .json file are not stored in the temporary directory
+    This function will load these indices and write them to the temporary directory.
+    Doing it this way reserves the way to link dynamically downloaded files and the static
+    files are the same """
+    index_files = os.listdir(data_path)
+    for index_file in index_files:
+        # Read from static location and write to temp storage
+        binary_index = read_index_binary(os.path.join(data_path, index_file))
+        write_index_binary(binary_index, f'{temp_path}/{index_file}')
+        
 def get_comparison(url, target, MIN_DISTANCE = 4):  
     """ Function for Gradio to combine all helper functions"""
     video_index, hash_vectors = get_video_index(url)
@@ -19,25 +34,53 @@ def get_comparison(url, target, MIN_DISTANCE = 4):
     fig = plot_comparison(lims, D, I, hash_vectors, MIN_DISTANCE = MIN_DISTANCE)
     return fig
 
-def get_auto_comparison(url, target, smoothing_window_size=10, metric="OFFSET_LIP"):
-    """ Function for Gradio to combine all helper functions"""
+def compare(url, target):
+    # Get source and target indices 
     source_index, source_hash_vectors = get_video_index(url)
     target_index, _ = get_video_index(target)
+
+    # Get decent distance by comparing url index with the target hash vectors + target index
     distance = get_decent_distance(source_index, source_hash_vectors, target_index, MIN_DISTANCE, MAX_DISTANCE)
     if distance == None:
-        return _, []
-        # raise gr.Error("No matches found!")
-
-    # For each video do...
-    for i in range(0, 1):
+        logging.info(f"No matches found between {url} and {target}!")
+        return plt.figure(), []   
+    else:
+        # Compare videos with heuristic distance
         lims, D, I, hash_vectors = compare_videos(source_hash_vectors, target_index, MIN_DISTANCE = distance)
+
+        # Get dataframe holding all information
         df = get_videomatch_df(lims, D, I, hash_vectors, distance)
-        change_points = get_change_points(df, smoothing_window_size=smoothing_window_size, metric=metric, method="ROBUST")
-        fig, segment_decision = plot_segment_comparison(df, change_points, video_id="Placeholder_Video_ID")
-    return fig, segment_decision
+
+        # Determine change point using ROBUST method based on column ROLL_OFFSET_MODE
+        change_points = get_change_points(df, metric="ROLL_OFFSET_MODE", method="ROBUST")
+
+        # Plot and get figure and .json-style segment decision
+        fig, segment_decision = plot_segment_comparison(df, change_points, video_id=target)
+        return fig, segment_decision
+
+def multiple_comparison(url, return_figure=False):
+    targets = get_target_urls()
+
+    # Figure and decision (list of dicts) storage 
+    figures, decisions = [], []
+    for target in targets:
+        # Make comparison
+        fig, segment_decision = compare(url, target)
+
+        # Add decisions to global decision list
+        decisions.extend(segment_decision)
+        figures.append(fig)
     
+    if return_figure:
+        return figures
+    return decisions
 
-video_urls = ["https://www.dropbox.com/s/8c89a9aba0w8gjg/Ploumen.mp4?dl=1",
+def plot_multiple_comparison(url):
+    return multiple_comparison(url, return_figure=True)
+    
+transfer_data_indices_to_temp() # NOTE: Only works after doing 'git lfs pull' to actually obtain the .index files 
+example_video_urls = ["https://drive.google.com/uc?id=1Y1-ypXOvLrp1x0cjAe_hMobCEdA0UbEo&export=download",
+              "https://www.dropbox.com/s/8c89a9aba0w8gjg/Ploumen.mp4?dl=1",
               "https://www.dropbox.com/s/rzmicviu1fe740t/Bram%20van%20Ojik%20krijgt%20reprimande.mp4?dl=1",
               "https://www.dropbox.com/s/wcot34ldmb84071/Baudet%20ontmaskert%20Omtzigt_%20u%20bent%20door%20de%20mand%20gevallen%21.mp4?dl=1",
               "https://drive.google.com/uc?id=1XW0niHR1k09vPNv1cp6NvdGXe7FHJc1D&export=download",
@@ -46,22 +89,24 @@ video_urls = ["https://www.dropbox.com/s/8c89a9aba0w8gjg/Ploumen.mp4?dl=1",
 index_iface = gr.Interface(fn=lambda url: index_hashes_for_video(url).ntotal, 
                      inputs="text", 
                      outputs="text", 
-                     examples=video_urls, cache_examples=True)
-
-compare_iface = gr.Interface(fn=get_comparison,
-                     inputs=["text", "text", gr.Slider(2, 30, 4, step=2)], 
-                     outputs="plot", 
-                     examples=[[x, video_urls[-1]] for x in video_urls[:-1]])
-
-auto_compare_iface = gr.Interface(fn=get_auto_comparison,
-                     inputs=["text", 
-                            "text", 
-                            gr.Slider(2, 50, 10, step=1), 
-                            gr.Dropdown(choices=["OFFSET_LIP", "ROLL_OFFSET_MODE"], value="OFFSET_LIP")], 
-                     outputs=["plot", "json"], 
-                     examples=[[x, video_urls[-1]] for x in video_urls[:-1]])
-
-iface = gr.TabbedInterface([auto_compare_iface, compare_iface, index_iface,], ["AutoCompare", "Compare", "Index"])
+                     examples=example_video_urls, cache_examples=True)
+
+# compare_iface = gr.Interface(fn=get_comparison,
+#                      inputs=["text", "text", gr.Slider(2, 30, 4, step=2)], 
+#                      outputs="plot", 
+#                      examples=[[x, example_video_urls[-1]] for x in example_video_urls[:-1]])
+
+plot_compare_iface = gr.Interface(fn=plot_multiple_comparison,
+                     inputs=["text"], 
+                     outputs=[gr.Plot() for _ in range(len(get_target_urls()))], 
+                     examples=example_video_urls)
+
+auto_compare_iface = gr.Interface(fn=multiple_comparison,
+                     inputs=["text"], 
+                     outputs=["json"], 
+                     examples=example_video_urls)
+
+iface = gr.TabbedInterface([auto_compare_iface, plot_compare_iface, index_iface], ["AutoCompare", "PlotAutoCompare", "Index"])
 
 if __name__ == "__main__":
     import matplotlib

clip_data.ipynb

@@ -395,7 +395,7 @@
  ],
  "metadata": {
   "kernelspec": {
-   "display_name": "Python 3.9.7 ('base')",
+   "display_name": "Python 3.9.13 64-bit",
    "language": "python",
    "name": "python3"
   },
@@ -409,12 +409,12 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.9.7"
+   "version": "3.9.13"
   },
   "orig_nbformat": 4,
   "vscode": {
    "interpreter": {
-    "hash": "35ac539f20c4edc7c4b10c8a5969be22a35cbd7bf12b66c83932160e8a573333"
+    "hash": "397704579725e15f5c7cb49fe5f0341eb7531c82d19f2c29d197e8b64ab5776b"
    }
   }
  },

config.py

@@ -1,8 +1,9 @@
 import tempfile
 
 VIDEO_DIRECTORY = tempfile.gettempdir()
+# VIDEO_DIRECTORY = './data/'
 
 FPS = 5
 MIN_DISTANCE = 4
-MAX_DISTANCE = 30
+MAX_DISTANCE = 30 # Used to be 30
 ROLLING_WINDOW_SIZE = 10

plot.py

@@ -69,33 +69,40 @@ def add_seconds_to_datetime64(datetime64, seconds, subtract=False):
         return datetime64 - np.timedelta64(int(s), 's') - np.timedelta64(int(m * 1000), 'ms')
     return datetime64 + np.timedelta64(int(s), 's') + np.timedelta64(int(m * 1000), 'ms')
 
-def plot_segment_comparison(df, change_points, video_id="Placeholder_Video_ID"):
-    """ From the dataframe plot the current set of plots, where the bottom right is most indicative """
-    fig, ax_arr = plt.subplots(3, 1, figsize=(16, 6), dpi=100, sharex=True)
+def plot_segment_comparison(df, change_points, video_id="Placeholder_Video_ID", threshold_diff = 1.5):
+    """ Based on the dataframe and detected change points do two things:
+    1. Make a decision on where each segment belongs in time and return that info as a list of dicts
+    2. Plot how this decision got made as an informative plot
     
-    # Plot original datapoints without linear interpolation, offset by target video time 
-    sns.scatterplot(data = df, x='time', y='OFFSET', ax=ax_arr[0], label="OFFSET", alpha=0.5)
+    args:
+    - df: dataframe 
+    - change_points: detected points in time where the average metric value changes
+    - video_id: the unique identifier for the video currently being compared
+    - threshold_diff: to plot which segments are likely bad matches
+    """
+    fig, ax_arr = plt.subplots(4, 1, figsize=(16, 6), dpi=300, sharex=True)
+    ax_arr[0].set_title(video_id)
+    sns.scatterplot(data = df, x='time', y='SOURCE_S', ax=ax_arr[0], label="SOURCE_S", color='blue', alpha=1.0)
 
-    # Plot linearly interpolated values
-    sns.lineplot(data = df, x='time', y='OFFSET_LIP', ax=ax_arr[1], label="OFFSET_LIP", color='orange')
+    # Plot original datapoints without linear interpolation, offset by target video time 
+    sns.scatterplot(data = df, x='time', y='OFFSET', ax=ax_arr[1], label="OFFSET", color='orange', alpha=1.0)
 
-    # Plot our target metric wherer
+    # Plot linearly interpolated values next to metric vales
     metric = 'ROLL_OFFSET_MODE' # 'OFFSET'
-    sns.scatterplot(data = df, x='time', y=metric, ax=ax_arr[1], label=metric, alpha=0.5)
+    sns.lineplot(data = df, x='time', y='OFFSET_LIP', ax=ax_arr[2], label="OFFSET_LIP", color='orange')
+    sns.scatterplot(data = df, x='time', y=metric, ax=ax_arr[2], label=metric, alpha=0.5)
 
-    # Plot deteected change points as lines which will indicate the segments
-    sns.scatterplot(data = df, x='time', y=metric, ax=ax_arr[2], label=metric, s=20)
+    # Plot detected change points as lines which will indicate the segments
+    sns.scatterplot(data = df, x='time', y=metric, ax=ax_arr[3], label=metric, s=20)
     timestamps = change_points_to_segments(df, change_points) 
-
+    for x in timestamps:
+        plt.vlines(x=x, ymin=np.min(df[metric]), ymax=np.max(df[metric]), colors='black', lw=2, alpha=0.5)
+    
     # To store "decisions" about segments  
     segment_decisions = []
     seg_i = 0
 
-    # To plot the detected segment lines 
-    for x in timestamps:
-        plt.vlines(x=x, ymin=np.min(df[metric]), ymax=np.max(df[metric]), colors='black', lw=2, alpha=0.5)
-
-    threshold_diff = 1.5 # Average segment difference threshold for plotting
+    # Average segment difference threshold for plotting
     for start_time, end_time in zip(timestamps[:-1], timestamps[1:]):
 
         # Time to add to each origin time to get the correct time back since it is offset by add_offset
@@ -149,4 +156,6 @@ def plot_segment_comparison(df, change_points, video_id="Placeholder_Video_ID"):
 
     # Return figure
     plt.xticks(rotation=90)
-    return fig, segment_decisions
+    return fig, segment_decisions
+
+    

videomatch.py

@@ -1,6 +1,6 @@
 import os
 import logging
-
+import json
 import faiss
 
 from kats.detectors.cusum_detection import CUSUMDetector
@@ -15,6 +15,12 @@ import pandas as pd
 from videohash import compute_hashes, filepath_from_url
 from config import FPS, MIN_DISTANCE, MAX_DISTANCE, ROLLING_WINDOW_SIZE
 
+def get_target_urls(json_file='apb2022.json'):
+    """ Obtain target urls for the target videos of a json file containing .mp4 files """
+    with open('apb2022.json', "r") as json_file:
+        target_videos = json.load(json_file)
+        return [video['mp4'] for video in target_videos]
+
 def index_hashes_for_video(url: str) -> faiss.IndexBinaryIVF:
     """ Compute hashes of a video and index the video using faiss indices and return the index. """
     filepath = filepath_from_url(url)