Spaces:
Running
on
L40S
Running
on
L40S
fix: typo in small checkpoint name
#1
by
mikhail-shevtsov
- opened
app.py
ADDED
@@ -0,0 +1,660 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
import subprocess
|
2 |
+
import re
|
3 |
+
from typing import List, Tuple, Optional
|
4 |
+
|
5 |
+
# Define the command to be executed
|
6 |
+
command = ["python", "setup.py", "build_ext", "--inplace"]
|
7 |
+
|
8 |
+
# Execute the command
|
9 |
+
result = subprocess.run(command, capture_output=True, text=True)
|
10 |
+
|
11 |
+
# Print the output and error (if any)
|
12 |
+
print("Output:\n", result.stdout)
|
13 |
+
print("Errors:\n", result.stderr)
|
14 |
+
|
15 |
+
# Check if the command was successful
|
16 |
+
if result.returncode == 0:
|
17 |
+
print("Command executed successfully.")
|
18 |
+
else:
|
19 |
+
print("Command failed with return code:", result.returncode)
|
20 |
+
|
21 |
+
import gradio as gr
|
22 |
+
from datetime import datetime
|
23 |
+
import os
|
24 |
+
os.environ["TORCH_CUDNN_SDPA_ENABLED"] = "1"
|
25 |
+
import torch
|
26 |
+
import numpy as np
|
27 |
+
import cv2
|
28 |
+
import matplotlib.pyplot as plt
|
29 |
+
from PIL import Image, ImageFilter
|
30 |
+
from sam2.build_sam import build_sam2_video_predictor
|
31 |
+
|
32 |
+
from moviepy.editor import ImageSequenceClip
|
33 |
+
|
34 |
+
def get_video_properties(video_path):
|
35 |
+
# Open the video file
|
36 |
+
cap = cv2.VideoCapture(video_path)
|
37 |
+
|
38 |
+
if not cap.isOpened():
|
39 |
+
print("Error: Could not open video.")
|
40 |
+
return None
|
41 |
+
|
42 |
+
# Get the FPS of the video
|
43 |
+
fps = cap.get(cv2.CAP_PROP_FPS)
|
44 |
+
|
45 |
+
# Get the width and height of the video
|
46 |
+
width = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH))
|
47 |
+
height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))
|
48 |
+
|
49 |
+
# Release the video capture object
|
50 |
+
cap.release()
|
51 |
+
|
52 |
+
return fps, width, height
|
53 |
+
|
54 |
+
def clear_points(image):
|
55 |
+
# we clean all
|
56 |
+
return [
|
57 |
+
image, # first_frame_path
|
58 |
+
gr.State([]), # tracking_points
|
59 |
+
gr.State([]), # trackings_input_label
|
60 |
+
image, # points_map
|
61 |
+
#gr.State() # stored_inference_state
|
62 |
+
]
|
63 |
+
|
64 |
+
def preprocess_video_in(video_path):
|
65 |
+
|
66 |
+
# Generate a unique ID based on the current date and time
|
67 |
+
unique_id = datetime.now().strftime('%Y%m%d%H%M%S')
|
68 |
+
|
69 |
+
# Set directory with this ID to store video frames
|
70 |
+
extracted_frames_output_dir = f'frames_{unique_id}'
|
71 |
+
|
72 |
+
# Create the output directory
|
73 |
+
os.makedirs(extracted_frames_output_dir, exist_ok=True)
|
74 |
+
|
75 |
+
### Process video frames ###
|
76 |
+
# Open the video file
|
77 |
+
cap = cv2.VideoCapture(video_path)
|
78 |
+
|
79 |
+
if not cap.isOpened():
|
80 |
+
print("Error: Could not open video.")
|
81 |
+
return None
|
82 |
+
|
83 |
+
# Get the frames per second (FPS) of the video
|
84 |
+
fps = cap.get(cv2.CAP_PROP_FPS)
|
85 |
+
|
86 |
+
# Calculate the number of frames to process (10 seconds of video)
|
87 |
+
max_frames = int(fps * 10)
|
88 |
+
|
89 |
+
frame_number = 0
|
90 |
+
first_frame = None
|
91 |
+
|
92 |
+
while True:
|
93 |
+
ret, frame = cap.read()
|
94 |
+
if not ret or frame_number >= max_frames:
|
95 |
+
break
|
96 |
+
|
97 |
+
# Format the frame filename as '00000.jpg'
|
98 |
+
frame_filename = os.path.join(extracted_frames_output_dir, f'{frame_number:05d}.jpg')
|
99 |
+
|
100 |
+
# Save the frame as a JPEG file
|
101 |
+
cv2.imwrite(frame_filename, frame)
|
102 |
+
|
103 |
+
# Store the first frame
|
104 |
+
if frame_number == 0:
|
105 |
+
first_frame = frame_filename
|
106 |
+
|
107 |
+
frame_number += 1
|
108 |
+
|
109 |
+
# Release the video capture object
|
110 |
+
cap.release()
|
111 |
+
|
112 |
+
# scan all the JPEG frame names in this directory
|
113 |
+
scanned_frames = [
|
114 |
+
p for p in os.listdir(extracted_frames_output_dir)
|
115 |
+
if os.path.splitext(p)[-1] in [".jpg", ".jpeg", ".JPG", ".JPEG"]
|
116 |
+
]
|
117 |
+
scanned_frames.sort(key=lambda p: int(os.path.splitext(p)[0]))
|
118 |
+
# print(f"SCANNED_FRAMES: {scanned_frames}")
|
119 |
+
|
120 |
+
return [
|
121 |
+
first_frame, # first_frame_path
|
122 |
+
gr.State([]), # tracking_points
|
123 |
+
gr.State([]), # trackings_input_label
|
124 |
+
first_frame, # input_first_frame_image
|
125 |
+
first_frame, # points_map
|
126 |
+
extracted_frames_output_dir, # video_frames_dir
|
127 |
+
scanned_frames, # scanned_frames
|
128 |
+
None, # stored_inference_state
|
129 |
+
None, # stored_frame_names
|
130 |
+
gr.update(open=False) # video_in_drawer
|
131 |
+
]
|
132 |
+
|
133 |
+
def get_point(point_type, tracking_points, trackings_input_label, input_first_frame_image, evt: gr.SelectData):
|
134 |
+
print(f"You selected {evt.value} at {evt.index} from {evt.target}")
|
135 |
+
|
136 |
+
tracking_points.value.append(evt.index)
|
137 |
+
print(f"TRACKING POINT: {tracking_points.value}")
|
138 |
+
|
139 |
+
if point_type == "include":
|
140 |
+
trackings_input_label.value.append(1)
|
141 |
+
elif point_type == "exclude":
|
142 |
+
trackings_input_label.value.append(0)
|
143 |
+
print(f"TRACKING INPUT LABEL: {trackings_input_label.value}")
|
144 |
+
|
145 |
+
# Open the image and get its dimensions
|
146 |
+
transparent_background = Image.open(input_first_frame_image).convert('RGBA')
|
147 |
+
w, h = transparent_background.size
|
148 |
+
|
149 |
+
# Define the circle radius as a fraction of the smaller dimension
|
150 |
+
fraction = 0.02 # You can adjust this value as needed
|
151 |
+
radius = int(fraction * min(w, h))
|
152 |
+
|
153 |
+
# Create a transparent layer to draw on
|
154 |
+
transparent_layer = np.zeros((h, w, 4), dtype=np.uint8)
|
155 |
+
|
156 |
+
for index, track in enumerate(tracking_points.value):
|
157 |
+
if trackings_input_label.value[index] == 1:
|
158 |
+
cv2.circle(transparent_layer, track, radius, (0, 255, 0, 255), -1)
|
159 |
+
else:
|
160 |
+
cv2.circle(transparent_layer, track, radius, (255, 0, 0, 255), -1)
|
161 |
+
|
162 |
+
# Convert the transparent layer back to an image
|
163 |
+
transparent_layer = Image.fromarray(transparent_layer, 'RGBA')
|
164 |
+
selected_point_map = Image.alpha_composite(transparent_background, transparent_layer)
|
165 |
+
|
166 |
+
return tracking_points, trackings_input_label, selected_point_map
|
167 |
+
|
168 |
+
# use bfloat16 for the entire notebook
|
169 |
+
torch.autocast(device_type="cuda", dtype=torch.bfloat16).__enter__()
|
170 |
+
|
171 |
+
if torch.cuda.get_device_properties(0).major >= 8:
|
172 |
+
# turn on tfloat32 for Ampere GPUs (https://pytorch.org/docs/stable/notes/cuda.html#tensorfloat-32-tf32-on-ampere-devices)
|
173 |
+
torch.backends.cuda.matmul.allow_tf32 = True
|
174 |
+
torch.backends.cudnn.allow_tf32 = True
|
175 |
+
|
176 |
+
def show_mask(mask, ax, obj_id=None, random_color=False):
|
177 |
+
if random_color:
|
178 |
+
color = np.concatenate([np.random.random(3), np.array([0.6])], axis=0)
|
179 |
+
else:
|
180 |
+
cmap = plt.get_cmap("tab10")
|
181 |
+
cmap_idx = 0 if obj_id is None else obj_id
|
182 |
+
color = np.array([*cmap(cmap_idx)[:3], 0.6])
|
183 |
+
h, w = mask.shape[-2:]
|
184 |
+
mask_image = mask.reshape(h, w, 1) * color.reshape(1, 1, -1)
|
185 |
+
ax.imshow(mask_image)
|
186 |
+
|
187 |
+
def show_white_mask(mask, ax):
|
188 |
+
# Ensure mask is binary (values 0 or 1)
|
189 |
+
mask = (mask > 0).astype(float) # Convert to binary mask
|
190 |
+
h, w = mask.shape[-2:]
|
191 |
+
|
192 |
+
# Create a white mask (RGBA: [1, 1, 1, alpha])
|
193 |
+
alpha = 1.0 # Fully opaque
|
194 |
+
color = np.array([1, 1, 1, alpha])
|
195 |
+
mask_image = mask.reshape(h, w, 1) * color.reshape(1, 1, -1)
|
196 |
+
|
197 |
+
# Display black background
|
198 |
+
ax.imshow(np.zeros((h, w, 3), dtype=float)) # Black background
|
199 |
+
ax.imshow(mask_image) # Overlay white mask
|
200 |
+
|
201 |
+
def show_points(coords, labels, ax, marker_size=200):
|
202 |
+
pos_points = coords[labels==1]
|
203 |
+
neg_points = coords[labels==0]
|
204 |
+
ax.scatter(pos_points[:, 0], pos_points[:, 1], color='green', marker='*', s=marker_size, edgecolor='white', linewidth=1.25)
|
205 |
+
ax.scatter(neg_points[:, 0], neg_points[:, 1], color='red', marker='*', s=marker_size, edgecolor='white', linewidth=1.25)
|
206 |
+
|
207 |
+
def show_box(box, ax):
|
208 |
+
x0, y0 = box[0], box[1]
|
209 |
+
w, h = box[2] - box[0], box[3] - box[1]
|
210 |
+
ax.add_patch(plt.Rectangle((x0, y0), w, h, edgecolor='green', facecolor=(0, 0, 0, 0), lw=2))
|
211 |
+
|
212 |
+
|
213 |
+
def load_model(checkpoint):
|
214 |
+
# Load model accordingly to user's choice
|
215 |
+
if checkpoint == "tiny":
|
216 |
+
sam2_checkpoint = "./checkpoints/sam2_hiera_tiny.pt"
|
217 |
+
model_cfg = "sam2_hiera_t.yaml"
|
218 |
+
return [sam2_checkpoint, model_cfg]
|
219 |
+
elif checkpoint == "small":
|
220 |
+
sam2_checkpoint = "./checkpoints/sam2_hiera_small.pt"
|
221 |
+
model_cfg = "sam2_hiera_s.yaml"
|
222 |
+
return [sam2_checkpoint, model_cfg]
|
223 |
+
elif checkpoint == "base-plus":
|
224 |
+
sam2_checkpoint = "./checkpoints/sam2_hiera_base_plus.pt"
|
225 |
+
model_cfg = "sam2_hiera_b+.yaml"
|
226 |
+
return [sam2_checkpoint, model_cfg]
|
227 |
+
elif checkpoint == "large":
|
228 |
+
sam2_checkpoint = "./checkpoints/sam2_hiera_large.pt"
|
229 |
+
model_cfg = "sam2_hiera_l.yaml"
|
230 |
+
return [sam2_checkpoint, model_cfg]
|
231 |
+
|
232 |
+
|
233 |
+
|
234 |
+
def get_mask_sam_process(
|
235 |
+
stored_inference_state,
|
236 |
+
input_first_frame_image,
|
237 |
+
checkpoint,
|
238 |
+
tracking_points,
|
239 |
+
trackings_input_label,
|
240 |
+
video_frames_dir, # extracted_frames_output_dir defined in 'preprocess_video_in' function
|
241 |
+
scanned_frames,
|
242 |
+
working_frame: str = None, # current frame being added points
|
243 |
+
available_frames_to_check: List[str] = [],
|
244 |
+
# progress=gr.Progress(track_tqdm=True)
|
245 |
+
):
|
246 |
+
|
247 |
+
# get model and model config paths
|
248 |
+
print(f"USER CHOSEN CHECKPOINT: {checkpoint}")
|
249 |
+
sam2_checkpoint, model_cfg = load_model(checkpoint)
|
250 |
+
print("MODEL LOADED")
|
251 |
+
|
252 |
+
# set predictor
|
253 |
+
predictor = build_sam2_video_predictor(model_cfg, sam2_checkpoint)
|
254 |
+
print("PREDICTOR READY")
|
255 |
+
|
256 |
+
# `video_dir` a directory of JPEG frames with filenames like `<frame_index>.jpg`
|
257 |
+
# print(f"STATE FRAME OUTPUT DIRECTORY: {video_frames_dir}")
|
258 |
+
video_dir = video_frames_dir
|
259 |
+
|
260 |
+
# scan all the JPEG frame names in this directory
|
261 |
+
frame_names = scanned_frames
|
262 |
+
|
263 |
+
# print(f"STORED INFERENCE STEP: {stored_inference_state}")
|
264 |
+
if stored_inference_state is None:
|
265 |
+
# Init SAM2 inference_state
|
266 |
+
inference_state = predictor.init_state(video_path=video_dir)
|
267 |
+
print("NEW INFERENCE_STATE INITIATED")
|
268 |
+
else:
|
269 |
+
inference_state = stored_inference_state
|
270 |
+
|
271 |
+
# segment and track one object
|
272 |
+
# predictor.reset_state(inference_state) # if any previous tracking, reset
|
273 |
+
|
274 |
+
### HANDLING WORKING FRAME
|
275 |
+
# new_working_frame = None
|
276 |
+
# Add new point
|
277 |
+
if working_frame is None:
|
278 |
+
ann_frame_idx = 0 # the frame index we interact with, 0 if it is the first frame
|
279 |
+
working_frame = "frame_0.jpg"
|
280 |
+
else:
|
281 |
+
# Use a regular expression to find the integer
|
282 |
+
match = re.search(r'frame_(\d+)', working_frame)
|
283 |
+
if match:
|
284 |
+
# Extract the integer from the match
|
285 |
+
frame_number = int(match.group(1))
|
286 |
+
ann_frame_idx = frame_number
|
287 |
+
|
288 |
+
print(f"NEW_WORKING_FRAME PATH: {working_frame}")
|
289 |
+
|
290 |
+
ann_obj_id = 1 # give a unique id to each object we interact with (it can be any integers)
|
291 |
+
|
292 |
+
# Let's add a positive click at (x, y) = (210, 350) to get started
|
293 |
+
points = np.array(tracking_points.value, dtype=np.float32)
|
294 |
+
# for labels, `1` means positive click and `0` means negative click
|
295 |
+
labels = np.array(trackings_input_label.value, np.int32)
|
296 |
+
_, out_obj_ids, out_mask_logits = predictor.add_new_points(
|
297 |
+
inference_state=inference_state,
|
298 |
+
frame_idx=ann_frame_idx,
|
299 |
+
obj_id=ann_obj_id,
|
300 |
+
points=points,
|
301 |
+
labels=labels,
|
302 |
+
)
|
303 |
+
|
304 |
+
# Create the plot
|
305 |
+
plt.figure(figsize=(12, 8))
|
306 |
+
plt.title(f"frame {ann_frame_idx}")
|
307 |
+
plt.imshow(Image.open(os.path.join(video_dir, frame_names[ann_frame_idx])))
|
308 |
+
show_points(points, labels, plt.gca())
|
309 |
+
show_mask((out_mask_logits[0] > 0.0).cpu().numpy(), plt.gca(), obj_id=out_obj_ids[0])
|
310 |
+
|
311 |
+
# Save the plot as a JPG file
|
312 |
+
first_frame_output_filename = "output_first_frame.jpg"
|
313 |
+
plt.savefig(first_frame_output_filename, format='jpg')
|
314 |
+
plt.close()
|
315 |
+
torch.cuda.empty_cache()
|
316 |
+
|
317 |
+
# Assuming available_frames_to_check.value is a list
|
318 |
+
if working_frame not in available_frames_to_check:
|
319 |
+
available_frames_to_check.append(working_frame)
|
320 |
+
print(available_frames_to_check)
|
321 |
+
|
322 |
+
return gr.update(visible=True), "output_first_frame.jpg", frame_names, predictor, inference_state, gr.update(choices=available_frames_to_check, value=working_frame, visible=True)
|
323 |
+
|
324 |
+
def propagate_to_all(video_in, checkpoint, stored_inference_state, stored_frame_names, video_frames_dir, vis_frame_type, available_frames_to_check, working_frame, progress=gr.Progress(track_tqdm=True)):
|
325 |
+
#### PROPAGATION ####
|
326 |
+
sam2_checkpoint, model_cfg = load_model(checkpoint)
|
327 |
+
predictor = build_sam2_video_predictor(model_cfg, sam2_checkpoint)
|
328 |
+
|
329 |
+
inference_state = stored_inference_state
|
330 |
+
frame_names = stored_frame_names
|
331 |
+
video_dir = video_frames_dir
|
332 |
+
|
333 |
+
# Define a directory to save the JPEG images
|
334 |
+
frames_output_dir = "frames_output_images"
|
335 |
+
os.makedirs(frames_output_dir, exist_ok=True)
|
336 |
+
|
337 |
+
mask_frames_output_dir = "mask_frames_output_images"
|
338 |
+
os.makedirs(mask_frames_output_dir, exist_ok=True)
|
339 |
+
|
340 |
+
# Initialize a list to store file paths of saved images
|
341 |
+
jpeg_images = []
|
342 |
+
masks_images = []
|
343 |
+
|
344 |
+
# run propagation throughout the video and collect the results in a dict
|
345 |
+
video_segments = {} # video_segments contains the per-frame segmentation results
|
346 |
+
for out_frame_idx, out_obj_ids, out_mask_logits in predictor.propagate_in_video(inference_state):
|
347 |
+
video_segments[out_frame_idx] = {
|
348 |
+
out_obj_id: (out_mask_logits[i] > 0.0).cpu().numpy()
|
349 |
+
for i, out_obj_id in enumerate(out_obj_ids)
|
350 |
+
}
|
351 |
+
|
352 |
+
# render the segmentation results every few frames
|
353 |
+
if vis_frame_type == "check":
|
354 |
+
vis_frame_stride = 15
|
355 |
+
elif vis_frame_type == "render":
|
356 |
+
vis_frame_stride = 1
|
357 |
+
|
358 |
+
plt.close("all")
|
359 |
+
for out_frame_idx in range(0, len(frame_names), vis_frame_stride):
|
360 |
+
plt.figure(figsize=(6, 4))
|
361 |
+
plt.title(f"frame {out_frame_idx}")
|
362 |
+
plt.imshow(Image.open(os.path.join(video_dir, frame_names[out_frame_idx])))
|
363 |
+
for out_obj_id, out_mask in video_segments[out_frame_idx].items():
|
364 |
+
show_mask(out_mask, plt.gca(), obj_id=out_obj_id)
|
365 |
+
|
366 |
+
# Define the output filename and save the figure as a JPEG file
|
367 |
+
output_filename = os.path.join(frames_output_dir, f"frame_{out_frame_idx}.jpg")
|
368 |
+
plt.savefig(output_filename, format='jpg')
|
369 |
+
|
370 |
+
# Close the plot
|
371 |
+
plt.close()
|
372 |
+
|
373 |
+
# Append the file path to the list
|
374 |
+
jpeg_images.append(output_filename)
|
375 |
+
|
376 |
+
if f"frame_{out_frame_idx}.jpg" not in available_frames_to_check:
|
377 |
+
available_frames_to_check.append(f"frame_{out_frame_idx}.jpg")
|
378 |
+
|
379 |
+
# Step 2: Create and store a black-and-white mask image using show_white_mask
|
380 |
+
# Create a figure without displaying it for the white mask
|
381 |
+
fig, ax = plt.subplots(figsize=(6, 4))
|
382 |
+
ax.axis("off") # Remove axes for a clean mask
|
383 |
+
|
384 |
+
# Overlay each mask as white on a black background
|
385 |
+
for out_mask in video_segments[out_frame_idx].values():
|
386 |
+
show_white_mask(out_mask, ax)
|
387 |
+
|
388 |
+
# Save the white mask figure to an image in memory
|
389 |
+
mask_filename = os.path.join(mask_frames_output_dir, f"mask_{out_frame_idx}.jpg")
|
390 |
+
fig.savefig(mask_filename, format='jpg', bbox_inches="tight", pad_inches=0)
|
391 |
+
plt.close(fig)
|
392 |
+
|
393 |
+
# Add the saved mask image to the masks_images array
|
394 |
+
masks_images.append(mask_filename)
|
395 |
+
|
396 |
+
|
397 |
+
|
398 |
+
torch.cuda.empty_cache()
|
399 |
+
print(f"JPEG_IMAGES: {jpeg_images}")
|
400 |
+
|
401 |
+
if vis_frame_type == "check":
|
402 |
+
return gr.update(value=jpeg_images), gr.update(value=None), gr.update(choices=available_frames_to_check, value=working_frame, visible=True), available_frames_to_check, gr.update(visible=True), None
|
403 |
+
elif vis_frame_type == "render":
|
404 |
+
# Create a video clip from the image sequence
|
405 |
+
original_fps, o_width, o_height = get_video_properties(video_in)
|
406 |
+
fps = original_fps # Frames per second
|
407 |
+
total_frames = len(jpeg_images)
|
408 |
+
clip = ImageSequenceClip(jpeg_images, fps=fps)
|
409 |
+
# Write the result to a file
|
410 |
+
final_vid_output_path = "output_video.mp4"
|
411 |
+
|
412 |
+
# Write the result to a file
|
413 |
+
clip.write_videofile(
|
414 |
+
final_vid_output_path,
|
415 |
+
codec='libx264'
|
416 |
+
)
|
417 |
+
|
418 |
+
print("MAKING MASK VIDEO ...")
|
419 |
+
|
420 |
+
# Create a video from the masks_images array
|
421 |
+
mask_video_filename = "final_masks_video.mp4"
|
422 |
+
|
423 |
+
# Define the video writer with the original video's dimensions
|
424 |
+
fourcc = cv2.VideoWriter_fourcc(*"mp4v")
|
425 |
+
video_writer = cv2.VideoWriter(mask_video_filename, fourcc, fps, (o_width, o_height))
|
426 |
+
|
427 |
+
# Write each mask image to the video
|
428 |
+
for mask_path in masks_images:
|
429 |
+
# Read the mask frame
|
430 |
+
frame = cv2.imread(mask_path)
|
431 |
+
|
432 |
+
# Resize the mask frame to the original video's dimensions
|
433 |
+
frame_resized = cv2.resize(frame, (o_width, o_height), interpolation=cv2.INTER_NEAREST)
|
434 |
+
|
435 |
+
# Write the resized frame to the video
|
436 |
+
video_writer.write(frame_resized)
|
437 |
+
|
438 |
+
# Release the video writer
|
439 |
+
video_writer.release()
|
440 |
+
print(f"Mask Video saved at {mask_video_filename}")
|
441 |
+
|
442 |
+
|
443 |
+
return gr.update(value=None), gr.update(value=final_vid_output_path), working_frame, available_frames_to_check, gr.update(visible=True), mask_video_filename
|
444 |
+
|
445 |
+
def update_ui(vis_frame_type):
|
446 |
+
if vis_frame_type == "check":
|
447 |
+
return gr.update(visible=True), gr.update(visible=False)
|
448 |
+
elif vis_frame_type == "render":
|
449 |
+
return gr.update(visible=False), gr.update(visible=True)
|
450 |
+
|
451 |
+
def switch_working_frame(working_frame, scanned_frames, video_frames_dir):
|
452 |
+
new_working_frame = None
|
453 |
+
if working_frame == None:
|
454 |
+
new_working_frame = os.path.join(video_frames_dir, scanned_frames[0])
|
455 |
+
|
456 |
+
else:
|
457 |
+
# Use a regular expression to find the integer
|
458 |
+
match = re.search(r'frame_(\d+)', working_frame)
|
459 |
+
if match:
|
460 |
+
# Extract the integer from the match
|
461 |
+
frame_number = int(match.group(1))
|
462 |
+
ann_frame_idx = frame_number
|
463 |
+
new_working_frame = os.path.join(video_frames_dir, scanned_frames[ann_frame_idx])
|
464 |
+
return gr.State([]), gr.State([]), new_working_frame, new_working_frame
|
465 |
+
|
466 |
+
def reset_propagation(first_frame_path, predictor, stored_inference_state):
|
467 |
+
|
468 |
+
predictor.reset_state(stored_inference_state)
|
469 |
+
# print(f"RESET State: {stored_inference_state} ")
|
470 |
+
return first_frame_path, gr.State([]), gr.State([]), gr.update(value=None, visible=False), stored_inference_state, None, ["frame_0.jpg"], first_frame_path, "frame_0.jpg", gr.update(visible=False)
|
471 |
+
|
472 |
+
css="""
|
473 |
+
div#component-18, div#component-25, div#component-35, div#component-41{
|
474 |
+
align-items: stretch!important;
|
475 |
+
}
|
476 |
+
"""
|
477 |
+
|
478 |
+
with gr.Blocks(css=css) as demo:
|
479 |
+
first_frame_path = gr.State()
|
480 |
+
tracking_points = gr.State([])
|
481 |
+
trackings_input_label = gr.State([])
|
482 |
+
video_frames_dir = gr.State()
|
483 |
+
scanned_frames = gr.State()
|
484 |
+
loaded_predictor = gr.State()
|
485 |
+
stored_inference_state = gr.State()
|
486 |
+
stored_frame_names = gr.State()
|
487 |
+
available_frames_to_check = gr.State([])
|
488 |
+
with gr.Column():
|
489 |
+
gr.Markdown("# SAM2 Video Predictor")
|
490 |
+
gr.Markdown("This is a simple demo for video segmentation with SAM2.")
|
491 |
+
|
492 |
+
with gr.Row():
|
493 |
+
|
494 |
+
with gr.Column():
|
495 |
+
with gr.Group():
|
496 |
+
with gr.Group():
|
497 |
+
with gr.Row():
|
498 |
+
point_type = gr.Radio(label="point type", choices=["include", "exclude"], value="include", scale=2)
|
499 |
+
clear_points_btn = gr.Button("Clear Points", scale=1)
|
500 |
+
|
501 |
+
input_first_frame_image = gr.Image(label="input image", interactive=False, type="filepath", visible=False)
|
502 |
+
|
503 |
+
points_map = gr.Image(
|
504 |
+
label="Point n Click map",
|
505 |
+
type="filepath",
|
506 |
+
interactive=False
|
507 |
+
)
|
508 |
+
|
509 |
+
with gr.Group():
|
510 |
+
with gr.Row():
|
511 |
+
checkpoint = gr.Dropdown(label="Checkpoint", choices=["tiny", "small", "base-plus", "large"], value="tiny")
|
512 |
+
submit_btn = gr.Button("Get Mask", size="lg")
|
513 |
+
|
514 |
+
with gr.Accordion("Your video IN", open=True) as video_in_drawer:
|
515 |
+
video_in = gr.Video(label="Video IN", format="mp4")
|
516 |
+
|
517 |
+
with gr.Group():
|
518 |
+
with gr.Group():
|
519 |
+
with gr.Row():
|
520 |
+
working_frame = gr.Dropdown(label="working frame ID", choices=[""], value=None, visible=False, allow_custom_value=False, interactive=True)
|
521 |
+
change_current = gr.Button("change current", visible=False)
|
522 |
+
output_result = gr.Image(label="current working mask ref")
|
523 |
+
with gr.Group():
|
524 |
+
with gr.Row():
|
525 |
+
vis_frame_type = gr.Radio(label="Propagation level", choices=["check", "render"], value="check", scale=2)
|
526 |
+
propagate_btn = gr.Button("Propagate", scale=1)
|
527 |
+
reset_prpgt_brn = gr.Button("Reset", visible=False)
|
528 |
+
|
529 |
+
gr.HTML("""
|
530 |
+
|
531 |
+
<a href="https://huggingface.co/spaces/{os.environ['SPACE_ID']}?duplicate=true">
|
532 |
+
<img src="https://huggingface.co/datasets/huggingface/badges/resolve/main/duplicate-this-space-lg-dark.svg" alt="Duplicate this Space" />
|
533 |
+
</a> to skip queue and avoid OOM errors from heavy public load
|
534 |
+
""")
|
535 |
+
|
536 |
+
with gr.Column():
|
537 |
+
|
538 |
+
gr.Markdown("""Instructions: (read the instructions)
|
539 |
+
|
540 |
+
1. Upload your video [MP4-24fps]
|
541 |
+
2. With 'include' point type selected, Click on the object to mask on first frame
|
542 |
+
3. Switch to 'exclude' point type if you want to specify an area to avoid
|
543 |
+
4. Get Mask !
|
544 |
+
5. Check Propagation every 15 frames
|
545 |
+
6. Add point on corresponding frame number if any mask needs to be refined
|
546 |
+
7. If propagation seems ok on every 15 frames, propagate with "render" to render final masked video !
|
547 |
+
8. Hit Reset button if you want to refresh and start again.
|
548 |
+
* Input video will be processed over 10 seconds only for demo purpose :)
|
549 |
+
""")
|
550 |
+
|
551 |
+
output_propagated = gr.Gallery(label="Propagated Mask samples gallery", columns=4, visible=False)
|
552 |
+
output_video = gr.Video(visible=False)
|
553 |
+
mask_final_output = gr.Video(label="Mask Video")
|
554 |
+
# output_result_mask = gr.Image()
|
555 |
+
|
556 |
+
|
557 |
+
|
558 |
+
# When new video is uploaded
|
559 |
+
video_in.upload(
|
560 |
+
fn = preprocess_video_in,
|
561 |
+
inputs = [video_in],
|
562 |
+
outputs = [
|
563 |
+
first_frame_path,
|
564 |
+
tracking_points, # update Tracking Points in the gr.State([]) object
|
565 |
+
trackings_input_label, # update Tracking Labels in the gr.State([]) object
|
566 |
+
input_first_frame_image, # hidden component used as ref when clearing points
|
567 |
+
points_map, # Image component where we add new tracking points
|
568 |
+
video_frames_dir, # Array where frames from video_in are deep stored
|
569 |
+
scanned_frames, # Scanned frames by SAM2
|
570 |
+
stored_inference_state, # Sam2 inference state
|
571 |
+
stored_frame_names, #
|
572 |
+
video_in_drawer, # Accordion to hide uploaded video player
|
573 |
+
],
|
574 |
+
queue = False
|
575 |
+
)
|
576 |
+
|
577 |
+
|
578 |
+
# triggered when we click on image to add new points
|
579 |
+
points_map.select(
|
580 |
+
fn = get_point,
|
581 |
+
inputs = [
|
582 |
+
point_type, # "include" or "exclude"
|
583 |
+
tracking_points, # get tracking_points values
|
584 |
+
trackings_input_label, # get tracking label values
|
585 |
+
input_first_frame_image, # gr.State() first frame path
|
586 |
+
],
|
587 |
+
outputs = [
|
588 |
+
tracking_points, # updated with new points
|
589 |
+
trackings_input_label, # updated with corresponding labels
|
590 |
+
points_map, # updated image with points
|
591 |
+
],
|
592 |
+
queue = False
|
593 |
+
)
|
594 |
+
|
595 |
+
# Clear every points clicked and added to the map
|
596 |
+
clear_points_btn.click(
|
597 |
+
fn = clear_points,
|
598 |
+
inputs = input_first_frame_image, # we get the untouched hidden image
|
599 |
+
outputs = [
|
600 |
+
first_frame_path,
|
601 |
+
tracking_points,
|
602 |
+
trackings_input_label,
|
603 |
+
points_map,
|
604 |
+
#stored_inference_state,
|
605 |
+
],
|
606 |
+
queue=False
|
607 |
+
)
|
608 |
+
|
609 |
+
|
610 |
+
change_current.click(
|
611 |
+
fn = switch_working_frame,
|
612 |
+
inputs = [working_frame, scanned_frames, video_frames_dir],
|
613 |
+
outputs = [tracking_points, trackings_input_label, input_first_frame_image, points_map],
|
614 |
+
queue=False
|
615 |
+
)
|
616 |
+
|
617 |
+
|
618 |
+
submit_btn.click(
|
619 |
+
fn = get_mask_sam_process,
|
620 |
+
inputs = [
|
621 |
+
stored_inference_state,
|
622 |
+
input_first_frame_image,
|
623 |
+
checkpoint,
|
624 |
+
tracking_points,
|
625 |
+
trackings_input_label,
|
626 |
+
video_frames_dir,
|
627 |
+
scanned_frames,
|
628 |
+
working_frame,
|
629 |
+
available_frames_to_check,
|
630 |
+
],
|
631 |
+
outputs = [
|
632 |
+
change_current,
|
633 |
+
output_result,
|
634 |
+
stored_frame_names,
|
635 |
+
loaded_predictor,
|
636 |
+
stored_inference_state,
|
637 |
+
working_frame,
|
638 |
+
],
|
639 |
+
queue=False
|
640 |
+
)
|
641 |
+
|
642 |
+
reset_prpgt_brn.click(
|
643 |
+
fn = reset_propagation,
|
644 |
+
inputs = [first_frame_path, loaded_predictor, stored_inference_state],
|
645 |
+
outputs = [points_map, tracking_points, trackings_input_label, output_propagated, stored_inference_state, output_result, available_frames_to_check, input_first_frame_image, working_frame, reset_prpgt_brn],
|
646 |
+
queue=False
|
647 |
+
)
|
648 |
+
|
649 |
+
propagate_btn.click(
|
650 |
+
fn = update_ui,
|
651 |
+
inputs = [vis_frame_type],
|
652 |
+
outputs = [output_propagated, output_video],
|
653 |
+
queue=False
|
654 |
+
).then(
|
655 |
+
fn = propagate_to_all,
|
656 |
+
inputs = [video_in, checkpoint, stored_inference_state, stored_frame_names, video_frames_dir, vis_frame_type, available_frames_to_check, working_frame],
|
657 |
+
outputs = [output_propagated, output_video, working_frame, available_frames_to_check, reset_prpgt_brn, mask_final_output]
|
658 |
+
)
|
659 |
+
|
660 |
+
demo.launch(show_api=False, show_error=True)
|