Create app.py

app.py

@@ -0,0 +1,100 @@
+from transformers import BitsAndBytesConfig, LlavaNextVideoForConditionalGeneration, LlavaNextVideoProcessor
+import torch
+import numpy as np
+import av
+import spaces
+import gradio as gr
+import os
+import json
+
+# Model Configuration
+quantization_config = BitsAndBytesConfig(
+    load_in_4bit=True,
+    bnb_4bit_compute_dtype=torch.float16
+)
+
+model_name = 'llava-hf/LLaVA-NeXT-Video-7B-DPO-hf'  
+
+# Load Model and Processor
+processor = LlavaNextVideoProcessor.from_pretrained(model_name)
+model = LlavaNextVideoForConditionalGeneration.from_pretrained(
+    model_name,
+    quantization_config=quantization_config,
+    device_map='auto'
+)
+
+@spaces.GPU
+def read_video_pyav(container, indices):
+    '''
+    Decode the video with PyAV decoder.
+    '''
+    frames = []
+    container.seek(0)
+    start_index = indices[0]
+    end_index = indices[-1]
+    for i, frame in enumerate(container.decode(video=0)):
+        if i > end_index:
+            break
+        if i >= start_index and i in indices:
+            frames.append(frame)
+    return np.stack([x.to_ndarray(format="rgb24") for x in frames])
+
+@spaces.GPU
+def process_video(video_file, question):
+    '''
+    Processes a single video and returns the answer to the given question.
+    '''
+    with av.open(video_file.name) as container:
+        total_frames = container.streams.video[0].frames
+        indices = np.arange(0, total_frames, total_frames / 8).astype(int)
+        video_clip = read_video_pyav(container, indices)
+
+    conversation = [
+        {
+            "role": "user",
+            "content": [
+                {"type": "text", "text": f"{question}"},
+                {"type": "video"},
+            ],
+        },
+    ]
+    prompt = processor.apply_chat_template(conversation, add_generation_prompt=True)
+    input = processor([prompt], videos=[video_clip], padding=True, return_tensors="pt").to(model.device)
+    generate_kwargs = {"max_new_tokens": 100, "do_sample": True, "top_p": 0.9}
+    output = model.generate(**input, **generate_kwargs)
+    generated_text = processor.batch_decode(output, skip_special_tokens=True)[0]
+    return generated_text.split("ASSISTANT: ", 1)[-1].strip()
+
+@spaces.GPU
+def analyze_videos(video_files, selected_question):
+    """Analyzes all videos with the selected question."""
+    all_results = {}
+
+    questions = {
+        "hands_free": "Examine the subject’s right and left hands in the video to check if they are holding anything like a microphone, book, paper(White color), object, or any electronic device, try segmentations and decide if the hands are free or not.",
+        "standing/sitting": "Evaluate the subject’s body posture and movement within the video. Are they standing upright with both feet planted firmly on the ground? If so, they are standing. If they seem to be seated, they are seated.",
+        "interaction_with_background": "Assess the surroundings behind the subject in the video. Do they seem to interact with any visible screens, such as laptops, TVs, or digital billboards? If yes, then they are interacting with a screen. If not, they are not interacting with a screen.",
+        "indoors/outdoors": "Consider the broader environmental context shown in the video’s background. Are there signs of an open-air space, like greenery, structures, or people passing by? If so, it’s an outdoor setting. If the setting looks confined with furniture, walls, or home decorations, it’s an indoor environment."
+    }
+
+    for video_file in video_files:
+        video_name = os.path.basename(video_file.name)
+        answer = process_video(video_file, questions[selected_question])
+        all_results[video_name] = {selected_question: answer}
+    return json.dumps(all_results, indent=4)
+
+# Define Gradio interface 
+iface = gr.Interface(
+    fn=analyze_videos,
+    inputs=[
+        gr.File(label="Upload Videos", file_count="multiple"),
+        gr.Dropdown(["hands_free", "standing/sitting", "interaction_with_background", "indoors/outdoors"], 
+                    label="Select Question to Apply")
+    ],
+    outputs=gr.JSON(label="Analysis Results"), 
+    title="Video Analysis",
+    description="Upload videos and select a question to analyze."
+)
+
+if __name__ == "__main__":
+    iface.launch(debug=True)