fix: typo in small checkpoint name

#1
Files changed (1) hide show
  1. app.py +660 -0
app.py ADDED
@@ -0,0 +1,660 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import subprocess
2
+ import re
3
+ from typing import List, Tuple, Optional
4
+
5
+ # Define the command to be executed
6
+ command = ["python", "setup.py", "build_ext", "--inplace"]
7
+
8
+ # Execute the command
9
+ result = subprocess.run(command, capture_output=True, text=True)
10
+
11
+ # Print the output and error (if any)
12
+ print("Output:\n", result.stdout)
13
+ print("Errors:\n", result.stderr)
14
+
15
+ # Check if the command was successful
16
+ if result.returncode == 0:
17
+ print("Command executed successfully.")
18
+ else:
19
+ print("Command failed with return code:", result.returncode)
20
+
21
+ import gradio as gr
22
+ from datetime import datetime
23
+ import os
24
+ os.environ["TORCH_CUDNN_SDPA_ENABLED"] = "1"
25
+ import torch
26
+ import numpy as np
27
+ import cv2
28
+ import matplotlib.pyplot as plt
29
+ from PIL import Image, ImageFilter
30
+ from sam2.build_sam import build_sam2_video_predictor
31
+
32
+ from moviepy.editor import ImageSequenceClip
33
+
34
+ def get_video_properties(video_path):
35
+ # Open the video file
36
+ cap = cv2.VideoCapture(video_path)
37
+
38
+ if not cap.isOpened():
39
+ print("Error: Could not open video.")
40
+ return None
41
+
42
+ # Get the FPS of the video
43
+ fps = cap.get(cv2.CAP_PROP_FPS)
44
+
45
+ # Get the width and height of the video
46
+ width = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH))
47
+ height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))
48
+
49
+ # Release the video capture object
50
+ cap.release()
51
+
52
+ return fps, width, height
53
+
54
+ def clear_points(image):
55
+ # we clean all
56
+ return [
57
+ image, # first_frame_path
58
+ gr.State([]), # tracking_points
59
+ gr.State([]), # trackings_input_label
60
+ image, # points_map
61
+ #gr.State() # stored_inference_state
62
+ ]
63
+
64
+ def preprocess_video_in(video_path):
65
+
66
+ # Generate a unique ID based on the current date and time
67
+ unique_id = datetime.now().strftime('%Y%m%d%H%M%S')
68
+
69
+ # Set directory with this ID to store video frames
70
+ extracted_frames_output_dir = f'frames_{unique_id}'
71
+
72
+ # Create the output directory
73
+ os.makedirs(extracted_frames_output_dir, exist_ok=True)
74
+
75
+ ### Process video frames ###
76
+ # Open the video file
77
+ cap = cv2.VideoCapture(video_path)
78
+
79
+ if not cap.isOpened():
80
+ print("Error: Could not open video.")
81
+ return None
82
+
83
+ # Get the frames per second (FPS) of the video
84
+ fps = cap.get(cv2.CAP_PROP_FPS)
85
+
86
+ # Calculate the number of frames to process (10 seconds of video)
87
+ max_frames = int(fps * 10)
88
+
89
+ frame_number = 0
90
+ first_frame = None
91
+
92
+ while True:
93
+ ret, frame = cap.read()
94
+ if not ret or frame_number >= max_frames:
95
+ break
96
+
97
+ # Format the frame filename as '00000.jpg'
98
+ frame_filename = os.path.join(extracted_frames_output_dir, f'{frame_number:05d}.jpg')
99
+
100
+ # Save the frame as a JPEG file
101
+ cv2.imwrite(frame_filename, frame)
102
+
103
+ # Store the first frame
104
+ if frame_number == 0:
105
+ first_frame = frame_filename
106
+
107
+ frame_number += 1
108
+
109
+ # Release the video capture object
110
+ cap.release()
111
+
112
+ # scan all the JPEG frame names in this directory
113
+ scanned_frames = [
114
+ p for p in os.listdir(extracted_frames_output_dir)
115
+ if os.path.splitext(p)[-1] in [".jpg", ".jpeg", ".JPG", ".JPEG"]
116
+ ]
117
+ scanned_frames.sort(key=lambda p: int(os.path.splitext(p)[0]))
118
+ # print(f"SCANNED_FRAMES: {scanned_frames}")
119
+
120
+ return [
121
+ first_frame, # first_frame_path
122
+ gr.State([]), # tracking_points
123
+ gr.State([]), # trackings_input_label
124
+ first_frame, # input_first_frame_image
125
+ first_frame, # points_map
126
+ extracted_frames_output_dir, # video_frames_dir
127
+ scanned_frames, # scanned_frames
128
+ None, # stored_inference_state
129
+ None, # stored_frame_names
130
+ gr.update(open=False) # video_in_drawer
131
+ ]
132
+
133
+ def get_point(point_type, tracking_points, trackings_input_label, input_first_frame_image, evt: gr.SelectData):
134
+ print(f"You selected {evt.value} at {evt.index} from {evt.target}")
135
+
136
+ tracking_points.value.append(evt.index)
137
+ print(f"TRACKING POINT: {tracking_points.value}")
138
+
139
+ if point_type == "include":
140
+ trackings_input_label.value.append(1)
141
+ elif point_type == "exclude":
142
+ trackings_input_label.value.append(0)
143
+ print(f"TRACKING INPUT LABEL: {trackings_input_label.value}")
144
+
145
+ # Open the image and get its dimensions
146
+ transparent_background = Image.open(input_first_frame_image).convert('RGBA')
147
+ w, h = transparent_background.size
148
+
149
+ # Define the circle radius as a fraction of the smaller dimension
150
+ fraction = 0.02 # You can adjust this value as needed
151
+ radius = int(fraction * min(w, h))
152
+
153
+ # Create a transparent layer to draw on
154
+ transparent_layer = np.zeros((h, w, 4), dtype=np.uint8)
155
+
156
+ for index, track in enumerate(tracking_points.value):
157
+ if trackings_input_label.value[index] == 1:
158
+ cv2.circle(transparent_layer, track, radius, (0, 255, 0, 255), -1)
159
+ else:
160
+ cv2.circle(transparent_layer, track, radius, (255, 0, 0, 255), -1)
161
+
162
+ # Convert the transparent layer back to an image
163
+ transparent_layer = Image.fromarray(transparent_layer, 'RGBA')
164
+ selected_point_map = Image.alpha_composite(transparent_background, transparent_layer)
165
+
166
+ return tracking_points, trackings_input_label, selected_point_map
167
+
168
+ # use bfloat16 for the entire notebook
169
+ torch.autocast(device_type="cuda", dtype=torch.bfloat16).__enter__()
170
+
171
+ if torch.cuda.get_device_properties(0).major >= 8:
172
+ # turn on tfloat32 for Ampere GPUs (https://pytorch.org/docs/stable/notes/cuda.html#tensorfloat-32-tf32-on-ampere-devices)
173
+ torch.backends.cuda.matmul.allow_tf32 = True
174
+ torch.backends.cudnn.allow_tf32 = True
175
+
176
+ def show_mask(mask, ax, obj_id=None, random_color=False):
177
+ if random_color:
178
+ color = np.concatenate([np.random.random(3), np.array([0.6])], axis=0)
179
+ else:
180
+ cmap = plt.get_cmap("tab10")
181
+ cmap_idx = 0 if obj_id is None else obj_id
182
+ color = np.array([*cmap(cmap_idx)[:3], 0.6])
183
+ h, w = mask.shape[-2:]
184
+ mask_image = mask.reshape(h, w, 1) * color.reshape(1, 1, -1)
185
+ ax.imshow(mask_image)
186
+
187
+ def show_white_mask(mask, ax):
188
+ # Ensure mask is binary (values 0 or 1)
189
+ mask = (mask > 0).astype(float) # Convert to binary mask
190
+ h, w = mask.shape[-2:]
191
+
192
+ # Create a white mask (RGBA: [1, 1, 1, alpha])
193
+ alpha = 1.0 # Fully opaque
194
+ color = np.array([1, 1, 1, alpha])
195
+ mask_image = mask.reshape(h, w, 1) * color.reshape(1, 1, -1)
196
+
197
+ # Display black background
198
+ ax.imshow(np.zeros((h, w, 3), dtype=float)) # Black background
199
+ ax.imshow(mask_image) # Overlay white mask
200
+
201
+ def show_points(coords, labels, ax, marker_size=200):
202
+ pos_points = coords[labels==1]
203
+ neg_points = coords[labels==0]
204
+ ax.scatter(pos_points[:, 0], pos_points[:, 1], color='green', marker='*', s=marker_size, edgecolor='white', linewidth=1.25)
205
+ ax.scatter(neg_points[:, 0], neg_points[:, 1], color='red', marker='*', s=marker_size, edgecolor='white', linewidth=1.25)
206
+
207
+ def show_box(box, ax):
208
+ x0, y0 = box[0], box[1]
209
+ w, h = box[2] - box[0], box[3] - box[1]
210
+ ax.add_patch(plt.Rectangle((x0, y0), w, h, edgecolor='green', facecolor=(0, 0, 0, 0), lw=2))
211
+
212
+
213
+ def load_model(checkpoint):
214
+ # Load model accordingly to user's choice
215
+ if checkpoint == "tiny":
216
+ sam2_checkpoint = "./checkpoints/sam2_hiera_tiny.pt"
217
+ model_cfg = "sam2_hiera_t.yaml"
218
+ return [sam2_checkpoint, model_cfg]
219
+ elif checkpoint == "small":
220
+ sam2_checkpoint = "./checkpoints/sam2_hiera_small.pt"
221
+ model_cfg = "sam2_hiera_s.yaml"
222
+ return [sam2_checkpoint, model_cfg]
223
+ elif checkpoint == "base-plus":
224
+ sam2_checkpoint = "./checkpoints/sam2_hiera_base_plus.pt"
225
+ model_cfg = "sam2_hiera_b+.yaml"
226
+ return [sam2_checkpoint, model_cfg]
227
+ elif checkpoint == "large":
228
+ sam2_checkpoint = "./checkpoints/sam2_hiera_large.pt"
229
+ model_cfg = "sam2_hiera_l.yaml"
230
+ return [sam2_checkpoint, model_cfg]
231
+
232
+
233
+
234
+ def get_mask_sam_process(
235
+ stored_inference_state,
236
+ input_first_frame_image,
237
+ checkpoint,
238
+ tracking_points,
239
+ trackings_input_label,
240
+ video_frames_dir, # extracted_frames_output_dir defined in 'preprocess_video_in' function
241
+ scanned_frames,
242
+ working_frame: str = None, # current frame being added points
243
+ available_frames_to_check: List[str] = [],
244
+ # progress=gr.Progress(track_tqdm=True)
245
+ ):
246
+
247
+ # get model and model config paths
248
+ print(f"USER CHOSEN CHECKPOINT: {checkpoint}")
249
+ sam2_checkpoint, model_cfg = load_model(checkpoint)
250
+ print("MODEL LOADED")
251
+
252
+ # set predictor
253
+ predictor = build_sam2_video_predictor(model_cfg, sam2_checkpoint)
254
+ print("PREDICTOR READY")
255
+
256
+ # `video_dir` a directory of JPEG frames with filenames like `<frame_index>.jpg`
257
+ # print(f"STATE FRAME OUTPUT DIRECTORY: {video_frames_dir}")
258
+ video_dir = video_frames_dir
259
+
260
+ # scan all the JPEG frame names in this directory
261
+ frame_names = scanned_frames
262
+
263
+ # print(f"STORED INFERENCE STEP: {stored_inference_state}")
264
+ if stored_inference_state is None:
265
+ # Init SAM2 inference_state
266
+ inference_state = predictor.init_state(video_path=video_dir)
267
+ print("NEW INFERENCE_STATE INITIATED")
268
+ else:
269
+ inference_state = stored_inference_state
270
+
271
+ # segment and track one object
272
+ # predictor.reset_state(inference_state) # if any previous tracking, reset
273
+
274
+ ### HANDLING WORKING FRAME
275
+ # new_working_frame = None
276
+ # Add new point
277
+ if working_frame is None:
278
+ ann_frame_idx = 0 # the frame index we interact with, 0 if it is the first frame
279
+ working_frame = "frame_0.jpg"
280
+ else:
281
+ # Use a regular expression to find the integer
282
+ match = re.search(r'frame_(\d+)', working_frame)
283
+ if match:
284
+ # Extract the integer from the match
285
+ frame_number = int(match.group(1))
286
+ ann_frame_idx = frame_number
287
+
288
+ print(f"NEW_WORKING_FRAME PATH: {working_frame}")
289
+
290
+ ann_obj_id = 1 # give a unique id to each object we interact with (it can be any integers)
291
+
292
+ # Let's add a positive click at (x, y) = (210, 350) to get started
293
+ points = np.array(tracking_points.value, dtype=np.float32)
294
+ # for labels, `1` means positive click and `0` means negative click
295
+ labels = np.array(trackings_input_label.value, np.int32)
296
+ _, out_obj_ids, out_mask_logits = predictor.add_new_points(
297
+ inference_state=inference_state,
298
+ frame_idx=ann_frame_idx,
299
+ obj_id=ann_obj_id,
300
+ points=points,
301
+ labels=labels,
302
+ )
303
+
304
+ # Create the plot
305
+ plt.figure(figsize=(12, 8))
306
+ plt.title(f"frame {ann_frame_idx}")
307
+ plt.imshow(Image.open(os.path.join(video_dir, frame_names[ann_frame_idx])))
308
+ show_points(points, labels, plt.gca())
309
+ show_mask((out_mask_logits[0] > 0.0).cpu().numpy(), plt.gca(), obj_id=out_obj_ids[0])
310
+
311
+ # Save the plot as a JPG file
312
+ first_frame_output_filename = "output_first_frame.jpg"
313
+ plt.savefig(first_frame_output_filename, format='jpg')
314
+ plt.close()
315
+ torch.cuda.empty_cache()
316
+
317
+ # Assuming available_frames_to_check.value is a list
318
+ if working_frame not in available_frames_to_check:
319
+ available_frames_to_check.append(working_frame)
320
+ print(available_frames_to_check)
321
+
322
+ return gr.update(visible=True), "output_first_frame.jpg", frame_names, predictor, inference_state, gr.update(choices=available_frames_to_check, value=working_frame, visible=True)
323
+
324
+ def propagate_to_all(video_in, checkpoint, stored_inference_state, stored_frame_names, video_frames_dir, vis_frame_type, available_frames_to_check, working_frame, progress=gr.Progress(track_tqdm=True)):
325
+ #### PROPAGATION ####
326
+ sam2_checkpoint, model_cfg = load_model(checkpoint)
327
+ predictor = build_sam2_video_predictor(model_cfg, sam2_checkpoint)
328
+
329
+ inference_state = stored_inference_state
330
+ frame_names = stored_frame_names
331
+ video_dir = video_frames_dir
332
+
333
+ # Define a directory to save the JPEG images
334
+ frames_output_dir = "frames_output_images"
335
+ os.makedirs(frames_output_dir, exist_ok=True)
336
+
337
+ mask_frames_output_dir = "mask_frames_output_images"
338
+ os.makedirs(mask_frames_output_dir, exist_ok=True)
339
+
340
+ # Initialize a list to store file paths of saved images
341
+ jpeg_images = []
342
+ masks_images = []
343
+
344
+ # run propagation throughout the video and collect the results in a dict
345
+ video_segments = {} # video_segments contains the per-frame segmentation results
346
+ for out_frame_idx, out_obj_ids, out_mask_logits in predictor.propagate_in_video(inference_state):
347
+ video_segments[out_frame_idx] = {
348
+ out_obj_id: (out_mask_logits[i] > 0.0).cpu().numpy()
349
+ for i, out_obj_id in enumerate(out_obj_ids)
350
+ }
351
+
352
+ # render the segmentation results every few frames
353
+ if vis_frame_type == "check":
354
+ vis_frame_stride = 15
355
+ elif vis_frame_type == "render":
356
+ vis_frame_stride = 1
357
+
358
+ plt.close("all")
359
+ for out_frame_idx in range(0, len(frame_names), vis_frame_stride):
360
+ plt.figure(figsize=(6, 4))
361
+ plt.title(f"frame {out_frame_idx}")
362
+ plt.imshow(Image.open(os.path.join(video_dir, frame_names[out_frame_idx])))
363
+ for out_obj_id, out_mask in video_segments[out_frame_idx].items():
364
+ show_mask(out_mask, plt.gca(), obj_id=out_obj_id)
365
+
366
+ # Define the output filename and save the figure as a JPEG file
367
+ output_filename = os.path.join(frames_output_dir, f"frame_{out_frame_idx}.jpg")
368
+ plt.savefig(output_filename, format='jpg')
369
+
370
+ # Close the plot
371
+ plt.close()
372
+
373
+ # Append the file path to the list
374
+ jpeg_images.append(output_filename)
375
+
376
+ if f"frame_{out_frame_idx}.jpg" not in available_frames_to_check:
377
+ available_frames_to_check.append(f"frame_{out_frame_idx}.jpg")
378
+
379
+ # Step 2: Create and store a black-and-white mask image using show_white_mask
380
+ # Create a figure without displaying it for the white mask
381
+ fig, ax = plt.subplots(figsize=(6, 4))
382
+ ax.axis("off") # Remove axes for a clean mask
383
+
384
+ # Overlay each mask as white on a black background
385
+ for out_mask in video_segments[out_frame_idx].values():
386
+ show_white_mask(out_mask, ax)
387
+
388
+ # Save the white mask figure to an image in memory
389
+ mask_filename = os.path.join(mask_frames_output_dir, f"mask_{out_frame_idx}.jpg")
390
+ fig.savefig(mask_filename, format='jpg', bbox_inches="tight", pad_inches=0)
391
+ plt.close(fig)
392
+
393
+ # Add the saved mask image to the masks_images array
394
+ masks_images.append(mask_filename)
395
+
396
+
397
+
398
+ torch.cuda.empty_cache()
399
+ print(f"JPEG_IMAGES: {jpeg_images}")
400
+
401
+ if vis_frame_type == "check":
402
+ return gr.update(value=jpeg_images), gr.update(value=None), gr.update(choices=available_frames_to_check, value=working_frame, visible=True), available_frames_to_check, gr.update(visible=True), None
403
+ elif vis_frame_type == "render":
404
+ # Create a video clip from the image sequence
405
+ original_fps, o_width, o_height = get_video_properties(video_in)
406
+ fps = original_fps # Frames per second
407
+ total_frames = len(jpeg_images)
408
+ clip = ImageSequenceClip(jpeg_images, fps=fps)
409
+ # Write the result to a file
410
+ final_vid_output_path = "output_video.mp4"
411
+
412
+ # Write the result to a file
413
+ clip.write_videofile(
414
+ final_vid_output_path,
415
+ codec='libx264'
416
+ )
417
+
418
+ print("MAKING MASK VIDEO ...")
419
+
420
+ # Create a video from the masks_images array
421
+ mask_video_filename = "final_masks_video.mp4"
422
+
423
+ # Define the video writer with the original video's dimensions
424
+ fourcc = cv2.VideoWriter_fourcc(*"mp4v")
425
+ video_writer = cv2.VideoWriter(mask_video_filename, fourcc, fps, (o_width, o_height))
426
+
427
+ # Write each mask image to the video
428
+ for mask_path in masks_images:
429
+ # Read the mask frame
430
+ frame = cv2.imread(mask_path)
431
+
432
+ # Resize the mask frame to the original video's dimensions
433
+ frame_resized = cv2.resize(frame, (o_width, o_height), interpolation=cv2.INTER_NEAREST)
434
+
435
+ # Write the resized frame to the video
436
+ video_writer.write(frame_resized)
437
+
438
+ # Release the video writer
439
+ video_writer.release()
440
+ print(f"Mask Video saved at {mask_video_filename}")
441
+
442
+
443
+ return gr.update(value=None), gr.update(value=final_vid_output_path), working_frame, available_frames_to_check, gr.update(visible=True), mask_video_filename
444
+
445
+ def update_ui(vis_frame_type):
446
+ if vis_frame_type == "check":
447
+ return gr.update(visible=True), gr.update(visible=False)
448
+ elif vis_frame_type == "render":
449
+ return gr.update(visible=False), gr.update(visible=True)
450
+
451
+ def switch_working_frame(working_frame, scanned_frames, video_frames_dir):
452
+ new_working_frame = None
453
+ if working_frame == None:
454
+ new_working_frame = os.path.join(video_frames_dir, scanned_frames[0])
455
+
456
+ else:
457
+ # Use a regular expression to find the integer
458
+ match = re.search(r'frame_(\d+)', working_frame)
459
+ if match:
460
+ # Extract the integer from the match
461
+ frame_number = int(match.group(1))
462
+ ann_frame_idx = frame_number
463
+ new_working_frame = os.path.join(video_frames_dir, scanned_frames[ann_frame_idx])
464
+ return gr.State([]), gr.State([]), new_working_frame, new_working_frame
465
+
466
+ def reset_propagation(first_frame_path, predictor, stored_inference_state):
467
+
468
+ predictor.reset_state(stored_inference_state)
469
+ # print(f"RESET State: {stored_inference_state} ")
470
+ return first_frame_path, gr.State([]), gr.State([]), gr.update(value=None, visible=False), stored_inference_state, None, ["frame_0.jpg"], first_frame_path, "frame_0.jpg", gr.update(visible=False)
471
+
472
+ css="""
473
+ div#component-18, div#component-25, div#component-35, div#component-41{
474
+ align-items: stretch!important;
475
+ }
476
+ """
477
+
478
+ with gr.Blocks(css=css) as demo:
479
+ first_frame_path = gr.State()
480
+ tracking_points = gr.State([])
481
+ trackings_input_label = gr.State([])
482
+ video_frames_dir = gr.State()
483
+ scanned_frames = gr.State()
484
+ loaded_predictor = gr.State()
485
+ stored_inference_state = gr.State()
486
+ stored_frame_names = gr.State()
487
+ available_frames_to_check = gr.State([])
488
+ with gr.Column():
489
+ gr.Markdown("# SAM2 Video Predictor")
490
+ gr.Markdown("This is a simple demo for video segmentation with SAM2.")
491
+
492
+ with gr.Row():
493
+
494
+ with gr.Column():
495
+ with gr.Group():
496
+ with gr.Group():
497
+ with gr.Row():
498
+ point_type = gr.Radio(label="point type", choices=["include", "exclude"], value="include", scale=2)
499
+ clear_points_btn = gr.Button("Clear Points", scale=1)
500
+
501
+ input_first_frame_image = gr.Image(label="input image", interactive=False, type="filepath", visible=False)
502
+
503
+ points_map = gr.Image(
504
+ label="Point n Click map",
505
+ type="filepath",
506
+ interactive=False
507
+ )
508
+
509
+ with gr.Group():
510
+ with gr.Row():
511
+ checkpoint = gr.Dropdown(label="Checkpoint", choices=["tiny", "small", "base-plus", "large"], value="tiny")
512
+ submit_btn = gr.Button("Get Mask", size="lg")
513
+
514
+ with gr.Accordion("Your video IN", open=True) as video_in_drawer:
515
+ video_in = gr.Video(label="Video IN", format="mp4")
516
+
517
+ with gr.Group():
518
+ with gr.Group():
519
+ with gr.Row():
520
+ working_frame = gr.Dropdown(label="working frame ID", choices=[""], value=None, visible=False, allow_custom_value=False, interactive=True)
521
+ change_current = gr.Button("change current", visible=False)
522
+ output_result = gr.Image(label="current working mask ref")
523
+ with gr.Group():
524
+ with gr.Row():
525
+ vis_frame_type = gr.Radio(label="Propagation level", choices=["check", "render"], value="check", scale=2)
526
+ propagate_btn = gr.Button("Propagate", scale=1)
527
+ reset_prpgt_brn = gr.Button("Reset", visible=False)
528
+
529
+ gr.HTML("""
530
+
531
+ <a href="https://huggingface.co/spaces/{os.environ['SPACE_ID']}?duplicate=true">
532
+ <img src="https://huggingface.co/datasets/huggingface/badges/resolve/main/duplicate-this-space-lg-dark.svg" alt="Duplicate this Space" />
533
+ </a> to skip queue and avoid OOM errors from heavy public load
534
+ """)
535
+
536
+ with gr.Column():
537
+
538
+ gr.Markdown("""Instructions: (read the instructions)
539
+
540
+ 1. Upload your video [MP4-24fps]
541
+ 2. With 'include' point type selected, Click on the object to mask on first frame
542
+ 3. Switch to 'exclude' point type if you want to specify an area to avoid
543
+ 4. Get Mask !
544
+ 5. Check Propagation every 15 frames
545
+ 6. Add point on corresponding frame number if any mask needs to be refined
546
+ 7. If propagation seems ok on every 15 frames, propagate with "render" to render final masked video !
547
+ 8. Hit Reset button if you want to refresh and start again.
548
+ * Input video will be processed over 10 seconds only for demo purpose :)
549
+ """)
550
+
551
+ output_propagated = gr.Gallery(label="Propagated Mask samples gallery", columns=4, visible=False)
552
+ output_video = gr.Video(visible=False)
553
+ mask_final_output = gr.Video(label="Mask Video")
554
+ # output_result_mask = gr.Image()
555
+
556
+
557
+
558
+ # When new video is uploaded
559
+ video_in.upload(
560
+ fn = preprocess_video_in,
561
+ inputs = [video_in],
562
+ outputs = [
563
+ first_frame_path,
564
+ tracking_points, # update Tracking Points in the gr.State([]) object
565
+ trackings_input_label, # update Tracking Labels in the gr.State([]) object
566
+ input_first_frame_image, # hidden component used as ref when clearing points
567
+ points_map, # Image component where we add new tracking points
568
+ video_frames_dir, # Array where frames from video_in are deep stored
569
+ scanned_frames, # Scanned frames by SAM2
570
+ stored_inference_state, # Sam2 inference state
571
+ stored_frame_names, #
572
+ video_in_drawer, # Accordion to hide uploaded video player
573
+ ],
574
+ queue = False
575
+ )
576
+
577
+
578
+ # triggered when we click on image to add new points
579
+ points_map.select(
580
+ fn = get_point,
581
+ inputs = [
582
+ point_type, # "include" or "exclude"
583
+ tracking_points, # get tracking_points values
584
+ trackings_input_label, # get tracking label values
585
+ input_first_frame_image, # gr.State() first frame path
586
+ ],
587
+ outputs = [
588
+ tracking_points, # updated with new points
589
+ trackings_input_label, # updated with corresponding labels
590
+ points_map, # updated image with points
591
+ ],
592
+ queue = False
593
+ )
594
+
595
+ # Clear every points clicked and added to the map
596
+ clear_points_btn.click(
597
+ fn = clear_points,
598
+ inputs = input_first_frame_image, # we get the untouched hidden image
599
+ outputs = [
600
+ first_frame_path,
601
+ tracking_points,
602
+ trackings_input_label,
603
+ points_map,
604
+ #stored_inference_state,
605
+ ],
606
+ queue=False
607
+ )
608
+
609
+
610
+ change_current.click(
611
+ fn = switch_working_frame,
612
+ inputs = [working_frame, scanned_frames, video_frames_dir],
613
+ outputs = [tracking_points, trackings_input_label, input_first_frame_image, points_map],
614
+ queue=False
615
+ )
616
+
617
+
618
+ submit_btn.click(
619
+ fn = get_mask_sam_process,
620
+ inputs = [
621
+ stored_inference_state,
622
+ input_first_frame_image,
623
+ checkpoint,
624
+ tracking_points,
625
+ trackings_input_label,
626
+ video_frames_dir,
627
+ scanned_frames,
628
+ working_frame,
629
+ available_frames_to_check,
630
+ ],
631
+ outputs = [
632
+ change_current,
633
+ output_result,
634
+ stored_frame_names,
635
+ loaded_predictor,
636
+ stored_inference_state,
637
+ working_frame,
638
+ ],
639
+ queue=False
640
+ )
641
+
642
+ reset_prpgt_brn.click(
643
+ fn = reset_propagation,
644
+ inputs = [first_frame_path, loaded_predictor, stored_inference_state],
645
+ outputs = [points_map, tracking_points, trackings_input_label, output_propagated, stored_inference_state, output_result, available_frames_to_check, input_first_frame_image, working_frame, reset_prpgt_brn],
646
+ queue=False
647
+ )
648
+
649
+ propagate_btn.click(
650
+ fn = update_ui,
651
+ inputs = [vis_frame_type],
652
+ outputs = [output_propagated, output_video],
653
+ queue=False
654
+ ).then(
655
+ fn = propagate_to_all,
656
+ inputs = [video_in, checkpoint, stored_inference_state, stored_frame_names, video_frames_dir, vis_frame_type, available_frames_to_check, working_frame],
657
+ outputs = [output_propagated, output_video, working_frame, available_frames_to_check, reset_prpgt_brn, mask_final_output]
658
+ )
659
+
660
+ demo.launch(show_api=False, show_error=True)