init

.gitattributes

@@ -33,3 +33,24 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+videos/foleycrafter/04-typewriter-keyboard.mp4 filter=lfs diff=lfs merge=lfs -text
+videos/foleycrafter/04-typewriter-piano.mp4 filter=lfs diff=lfs merge=lfs -text
+videos/foleycrafter/04-typewriter.mp4 filter=lfs diff=lfs merge=lfs -text
+videos/foleycrafter/07-lion-cat.mp4 filter=lfs diff=lfs merge=lfs -text
+videos/foleycrafter/07-lion-elephant.mp4 filter=lfs diff=lfs merge=lfs -text
+videos/foleycrafter/08-squeeze-bone.mp4 filter=lfs diff=lfs merge=lfs -text
+videos/foleycrafter/08-squeeze-scream.mp4 filter=lfs diff=lfs merge=lfs -text
+videos/foleycrafter/08-squeeze.mp4 filter=lfs diff=lfs merge=lfs -text
+videos/foleycrafter/10-golf-explosion.mp4 filter=lfs diff=lfs merge=lfs -text
+videos/foleycrafter/10-golf-scream.mp4 filter=lfs diff=lfs merge=lfs -text
+videos/foleycrafter/10-golf.mp4 filter=lfs diff=lfs merge=lfs -text
+videos/ours/04-typewriter-keyboard.mp4 filter=lfs diff=lfs merge=lfs -text
+videos/ours/04-typewriter-piano.mp4 filter=lfs diff=lfs merge=lfs -text
+videos/ours/04-typewriter.mp4 filter=lfs diff=lfs merge=lfs -text
+videos/ours/07-lion-cat.mp4 filter=lfs diff=lfs merge=lfs -text
+videos/ours/07-lion-elephant.mp4 filter=lfs diff=lfs merge=lfs -text
+videos/ours/08-squeeze-bone.mp4 filter=lfs diff=lfs merge=lfs -text
+videos/ours/08-squeeze-scream.mp4 filter=lfs diff=lfs merge=lfs -text
+videos/ours/10-golf-explosion.mp4 filter=lfs diff=lfs merge=lfs -text
+videos/ours/10-golf-scream.mp4 filter=lfs diff=lfs merge=lfs -text
+videos/ours/10-golf.mp4 filter=lfs diff=lfs merge=lfs -text

app.py

@@ -0,0 +1,175 @@
+import gradio as gr
+import json
+from datetime import datetime
+import random
+import os
+from huggingface_hub import Repository
+random.seed(1234)
+
+import subprocess
+# Set Git user information
+subprocess.run(["git", "config", "--global", "user.email", "[email protected]"])
+subprocess.run(["git", "config", "--global", "user.name", "czyang"])
+
+
+hf_token = os.getenv("HF_TOKEN")
+print("HF Token is none?", hf_token is None)
+
+# Initialize the repository
+DATASET_REPO_URL = "https://huggingface.co/datasets/czyang/Foley-User-Study-Response-V2"
+repo = Repository(
+    local_dir="user_responses",
+    clone_from=DATASET_REPO_URL,
+    use_auth_token=hf_token
+)
+
+def prepare_test_cases():
+    json_path = "videos/videos.json"
+    with open(json_path, "r") as f:
+        video_dict = json.load(f)
+    
+    video_ids = list(video_dict.keys())
+    for video_id in video_ids:
+        if random.random() > 0.5:
+            video_list = [video_dict[video_id]['ours'], video_dict[video_id]['foleycrafter']]
+        else: 
+            video_list = [video_dict[video_id]['foleycrafter'], video_dict[video_id]['ours']]
+
+        random.shuffle(video_list)
+
+        video_dict[video_id]['Video 1'] = video_list[0]
+        video_dict[video_id]['Video 2'] = video_list[1]
+
+    return video_dict
+
+video_dict = prepare_test_cases()
+video_ids = list(video_dict.keys())  
+random.shuffle(video_ids)
+
+
+questions = [
+    "Which video's audio best matches the sound of {}?",
+    "In which video is the timing of the audio best synchronized with what you can see in the video?",
+    "Assuming the video is meant to sound like {}, which video has the best audio overall?"
+]
+submissions_file = "user_responses/response.jsonl"
+
+def has_already_submitted(user_id):
+    if os.path.exists(submissions_file):
+        with open(submissions_file, "r") as f:
+            for line in f:
+                submission = json.loads(line)
+                if submission.get("u_id") == user_id:
+                    return True
+    return False
+
+# Save responses
+def save_responses(unique_submission, *responses):
+    timestamp = datetime.now().isoformat()
+    info = responses[-1]
+    responses = responses[:-1]
+    unique_id = info["session_id"]
+    user_id = f"{unique_id}"
+
+    # Check for unique submission
+    if unique_submission and has_already_submitted(user_id):
+        return "You have already submitted responses. Thank you for participating!"
+
+    # Initialize the result dictionary
+    result = {
+        "u_id": user_id,
+        "timestamp": timestamp,
+        "responses": []
+    }
+
+    for index in range(len(video_ids)):
+        start_idx = index * len(questions)
+        end_idx = start_idx + len(questions)
+
+        response = responses[start_idx:end_idx]
+        if any(r is None for r in response):
+            return "Please answer all questions before submitting."
+
+        video_id = video_ids[index]
+        pair_response = {
+            video_id: {
+                'semantic': video_dict[video_id][response[0]],
+                'sync': video_dict[video_id][response[1]],
+                'overall': video_dict[video_id][response[2]],
+            }
+        }
+        result["responses"].append(pair_response)
+
+    result["responses"] = sorted(result["responses"], key=lambda x: x.keys())
+    
+    # Save response locally and push to Hugging Face Hub
+    with open(submissions_file, "a") as f:
+        f.write(json.dumps(result) + "\n")
+    
+    # Push changes to the Hugging Face dataset repo
+    repo.push_to_hub()
+
+    return "All responses saved! Thank you for participating!"
+
+
+
+
+def create_interface(unique_submission=False):
+    with gr.Blocks() as demo:
+        gr.Markdown("# Human Preference Study: Video Comparison")
+        gr.Markdown("""
+            In this study, you will watch (and listen to) pairs of videos side by side. 
+            
+            Please watch and **listen** to each pair of videos carefully and answer the three associated questions.   
+            
+            **Headphones are recommended!**
+        """)
+
+        # Display video pairs and questions
+        responses = []
+        for index, video_id in enumerate(video_ids):
+            video1 = video_dict[video_id]['Video 1']
+            video2 = video_dict[video_id]['Video 2']
+            audio_prompt = video_dict[video_id]['audio prompt']
+
+            gr.Markdown(f"### Video Pair {index + 1}")
+            with gr.Row():
+                gr.Video(video1, label="Video 1")
+                gr.Video(video2, label="Video 2")
+                with gr.Column():
+                    responses.append(gr.Radio(["Video 1", "Video 2"], label=questions[0].format(audio_prompt), value=None))
+                    responses.append(gr.Radio(["Video 1", "Video 2"], label=questions[1], value=None))
+                    responses.append(gr.Radio(["Video 1", "Video 2"], label=questions[2].format(audio_prompt), value=None))
+
+            gr.Markdown("---")
+
+        info = gr.JSON(visible=False)
+        demo.load(predict, None, info)
+
+        submit_btn = gr.Button("Submit")
+        result_message = gr.Textbox(label="Message (please only submit once)", interactive=False)
+
+        submit_btn.click(
+            fn=lambda *args: save_responses(unique_submission, *args),
+            inputs=responses+[info],
+            outputs=result_message
+        )
+
+    return demo
+
+def predict(request: gr.Request):
+    headers = request.headers
+    host = request.client.host
+    user_agent = request.headers["user-agent"]
+    session_id = request.session_hash
+    return {
+        "ip": host,
+        "user_agent": user_agent,
+        "headers": headers,
+        "session_id": session_id
+    }
+
+if __name__ == "__main__":    
+    # Launch with unique_submission set based on `--unique` flag
+    demo = create_interface(unique_submission=True)
+    demo.launch(share=True)

videos/foleycrafter/04-typewriter-keyboard.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:73148b48b5344edafc7d303b67ae60d0c37cd8c22ded0ec6b2c1068b340ca933
+size 1140333

videos/foleycrafter/04-typewriter-piano.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:31ad57b9dfa33055f27db3d2c65cfe6227c4cb5214ea5b96e0bd3d40b136e6ad
+size 1140333

videos/foleycrafter/04-typewriter.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9dfd68a98bc39e32f72f1cef3d3824e6a9e0af5f02010c73448188f47244d17f
+size 1123257

videos/foleycrafter/07-lion-cat.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:70d56dbc76ad0a1d2de64675a1d7768b8776d3f354adc39558ab4368c144a20c
+size 1081399

videos/foleycrafter/07-lion-elephant.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:58ef42ac6a63bdde04ef2e6f394d26e7e7ec0e30c48e14afbaa64b3c72583067
+size 1081399

videos/foleycrafter/08-squeeze-bone.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0d07fb408a95a281ccbedad309fa56380162ebb591d38f659356a34e9add5afa
+size 1106678

videos/foleycrafter/08-squeeze-scream.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9c203955ea99c2ffd3330bcd71bf714097743bf0db36801aa38673fa8514ab4f
+size 1106678

videos/foleycrafter/08-squeeze.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b705822176ec978c3b7eaf48b8e9dcf41802545158ccd0cd16de1320d67ae242
+size 1044540

videos/foleycrafter/10-golf-explosion.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:228141950e3da917815b70b1feede266f29303f4137b4b8986877fc5345530db
+size 2707753

videos/foleycrafter/10-golf-scream.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4316ceb0fee0390a13277fafeb3f02b2bd0ffcbe5cf437b6dc000faabd77e19f
+size 2707753

videos/foleycrafter/10-golf.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:60136d9a2d9b24cf6546c09867b30c3dd82d329edfdeb3793afc77532ac1efaa
+size 2583278

videos/ours/04-typewriter-keyboard.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7eeef131f1fcdbf5dde11169f86e67f8be8722037fe05048e88099ba34d65d8f
+size 1082586

videos/ours/04-typewriter-piano.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bbab300707e375fe0347465ba445b785cfa6e7783d17ed60a2fe7ad43664f18f
+size 1082454

videos/ours/04-typewriter.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:916a1d8cbbe426a952849ca9babeb8fb29934ec9b754122679b08b60ba1e0e92
+size 1063941