app.py

import sys
import random
import gradio as gr
import matplotlib.pyplot as plt

import os
import argparse
import random
from omegaconf import OmegaConf
import torch
import torchvision
from pytorch_lightning import seed_everything
from huggingface_hub import hf_hub_download
import spaces

sys.path.insert(0, "scripts/evaluation")
from funcs import (
    batch_ddim_sampling,
    batch_ddim_sampling_freetraj,
    load_model_checkpoint,
)
from utils.utils import instantiate_from_config
from utils.utils_freetraj import plan_path

video_length = 16
width = 512 
height = 320
MAX_KEYS = 5

ckpt_dir_512 = "checkpoints/base_512_v2"
ckpt_path_512 = "checkpoints/base_512_v2/model.ckpt"
if not os.path.exists(ckpt_path_512):
    os.makedirs(ckpt_dir_512, exist_ok=True)
    hf_hub_download(repo_id="VideoCrafter/VideoCrafter2", filename="model.ckpt", local_dir=ckpt_dir_512, force_download=True)
    print('Model Loaded.')
    
def check_move(trajectory, video_length=16):
    traj_len = len(trajectory)
    if traj_len < 2:
        return False
    prev_pos = trajectory[0]
    for i in range(1, traj_len):
        cur_pos = trajectory[i]
        if cur_pos[0] > video_length - 1:
            return False
        if (cur_pos[0] - prev_pos[0]) * ((cur_pos[1] - prev_pos[1]) ** 2 + (cur_pos[2] - prev_pos[2]) ** 2) ** 0.5 < 0.02:
            print("Too small movement, please use ori mode.")
            return False
        prev_pos = cur_pos

    return True

def check(radio_mode):
    if radio_mode == 'ori':
        video_path = "output.mp4"
        video_bbox_path = "output.mp4"
    else:
        video_path = "output_freetraj.mp4"
        video_bbox_path = "output_freetraj_bbox.mp4"
    return video_path, video_bbox_path
    

def infer(*user_args):
    prompt_in = user_args[0]
    target_indices = user_args[1]
    ddim_edit = user_args[2]
    seed = user_args[3]
    ddim_steps = user_args[4]
    unconditional_guidance_scale = user_args[5]
    video_fps = user_args[6]
    save_fps = user_args[7]
    height_ratio = user_args[8]
    width_ratio = user_args[9]
    radio_mode = user_args[10]
    dropdown_diy = user_args[11]
    frame_indices = user_args[-3 * MAX_KEYS: -2 * MAX_KEYS]
    h_positions = user_args[-2 * MAX_KEYS: -MAX_KEYS]
    w_positions = user_args[-MAX_KEYS:]
    print(user_args)

    if radio_mode == 'ori':
        config_512 = "configs/inference_t2v_512_v2.0.yaml"
    else:
        config_512 = "configs/inference_t2v_freetraj_512_v2.0.yaml"

    trajectory = []
    for i in range(dropdown_diy):
        trajectory.append([int(frame_indices[i]), h_positions[i], w_positions[i]])
    trajectory.sort()
    print(trajectory)

    if not check_move(trajectory):
        print("Error trajectory.")

    input_traj = []
    h_remain = 1 - height_ratio
    w_remain = 1 - width_ratio
    for i in trajectory:
        h_relative = i[1] * h_remain
        w_relative = i[2] * w_remain
        input_traj.append([i[0], h_relative, h_relative+height_ratio, w_relative, w_relative+width_ratio])

    if len(target_indices) < 1:
        indices_list = [1, 2]
    else:
        indices_list = target_indices.split(',')
    idx_list = []
    for i in indices_list:
        idx_list.append(int(i))

    config_512 = OmegaConf.load(config_512)
    model_config_512 = config_512.pop("model", OmegaConf.create())

    args = argparse.Namespace(
        mode="base",
        savefps=save_fps,
        n_samples=1,
        ddim_steps=ddim_steps,
        ddim_eta=0.0,
        bs=1,
        fps=video_fps,
        unconditional_guidance_scale=unconditional_guidance_scale,
        unconditional_guidance_scale_temporal=None,
        cond_input=None,
        prompt_in = prompt_in,
        seed = seed,
        ddim_edit = ddim_edit,
        model_config_512 = model_config_512,
        idx_list = idx_list,
        input_traj = input_traj,
    )

    print('GPU starts')
    video = infer_gpu_part(args)
    print('GPU ends')
    
    video = torch.clamp(video.float(), -1.0, 1.0)
    video = video.permute(2, 0, 1, 3, 4)  # t,n,c,h,w

    if radio_mode == 'ori':
        video_path = "output.mp4"
        video_bbox_path = "output.mp4"
        frame_grids = [
            torchvision.utils.make_grid(framesheet, nrow=int(args.n_samples))
            for framesheet in video
        ]  # [3, 1*h, n*w]
        grid = torch.stack(frame_grids, dim=0)  # stack in temporal dim [t, 3, n*h, w]
        grid = (grid + 1.0) / 2.0
        grid = (grid * 255).to(torch.uint8).permute(0, 2, 3, 1)

        torchvision.io.write_video(
            video_path,
            grid,
            fps=args.savefps,
            video_codec="h264",
            options={"crf": "10"},
        )
    else:
        video_path = "output_freetraj.mp4"
        video_bbox_path = "output_freetraj_bbox.mp4"
        frame_grids = [
            torchvision.utils.make_grid(framesheet, nrow=int(args.n_samples))
            for framesheet in video
        ]  # [3, 1*h, n*w]
        grid = torch.stack(frame_grids, dim=0)  # stack in temporal dim [t, 3, n*h, w]
        grid = (grid + 1.0) / 2.0
        grid = (grid * 255).to(torch.uint8).permute(0, 2, 3, 1)

        torchvision.io.write_video(
            video_path,
            grid,
            fps=args.savefps,
            video_codec="h264",
            options={"crf": "10"},
        )

        BOX_SIZE_H = input_traj[0][2] - input_traj[0][1]
        BOX_SIZE_W = input_traj[0][4] - input_traj[0][3]
        PATHS = plan_path(input_traj)
        h_len = grid.shape[1]
        w_len = grid.shape[2]
        sub_h = int(BOX_SIZE_H * h_len) 
        sub_w = int(BOX_SIZE_W * w_len)
        for j in range(grid.shape[0]):
            h_start = int(PATHS[j][0] * h_len)
            h_end = h_start + sub_h
            w_start = int(PATHS[j][2] * w_len)
            w_end = w_start + sub_w

            h_start = max(1, h_start)
            h_end = min(h_len-1, h_end)
            w_start = max(1, w_start)
            w_end = min(w_len-1, w_end)

            grid[j, h_start-1:h_end+1, w_start-1:w_start+2, :] = torch.ones_like(grid[j, h_start-1:h_end+1, w_start-1:w_start+2, :]) * torch.Tensor([127, 255, 127]).view(1, 1, 3)
            grid[j, h_start-1:h_end+1, w_end-2:w_end+1, :] = torch.ones_like(grid[j, h_start-1:h_end+1, w_end-2:w_end+1, :]) * torch.Tensor([127, 255, 127]).view(1, 1, 3)
            grid[j, h_start-1:h_start+2, w_start-1:w_end+1, :] = torch.ones_like(grid[j, h_start-1:h_start+2, w_start-1:w_end+1, :]) * torch.Tensor([127, 255, 127]).view(1, 1, 3)
            grid[j, h_end-2:h_end+1, w_start-1:w_end+1, :] = torch.ones_like(grid[j, h_end-2:h_end+1, w_start-1:w_end+1, :]) * torch.Tensor([127, 255, 127]).view(1, 1, 3)

        torchvision.io.write_video(
            video_bbox_path,
            grid,
            fps=args.savefps,
            video_codec="h264",
            options={"crf": "10"},
        )

    return video_path, video_bbox_path

    
@spaces.GPU(duration=250)
def infer_gpu_part(args):

    model = instantiate_from_config(args.model_config_512)
    model = model.cuda()
    model = load_model_checkpoint(model, ckpt_path_512)
    model.eval()

    if args.seed is None:
        seed = int.from_bytes(os.urandom(2), "big")
    else:
        seed = args.seed
    print(f"Using seed: {seed}")
    seed_everything(seed)

    ## latent noise shape
    h, w = height // 8, width // 8
    frames = video_length
    channels = model.channels

    batch_size = 1
    noise_shape = [batch_size, channels, frames, h, w]
    fps = torch.tensor([args.fps] * batch_size).to(model.device).long()
    prompts = [args.prompt_in]
    text_emb = model.get_learned_conditioning(prompts)

    cond = {"c_crossattn": [text_emb], "fps": fps}

    ## inference
    if radio_mode == 'ori':
        batch_samples = batch_ddim_sampling(
            model,
            cond,
            noise_shape,
            args.n_samples,
            args.ddim_steps,
            args.ddim_eta,
            args.unconditional_guidance_scale,
            args=args,
        )
    else:
        batch_samples = batch_ddim_sampling_freetraj(
            model,
            cond,
            noise_shape,
            args.n_samples,
            args.ddim_steps,
            args.ddim_eta,
            args.unconditional_guidance_scale,
            idx_list = args.idx_list,
            input_traj = args.input_traj,
            args=args,
        )

    vid_tensor = batch_samples[0]
    video = vid_tensor.detach().cpu()

    return video


examples = [
    ["A squirrel jumping from one tree to another.",],
    ["A bear climbing down a tree after spotting a threat.",],
    ["A corgi running on the grassland on the grassland.",],
    ["A barrel floating in a river.",],
    ["A horse galloping on a street.",],
    ["A majestic eagle soaring high above the treetops, surveying its territory.",],
]

css = """
#col-container {max-width: 1024px; margin-left: auto; margin-right: auto;}
a {text-decoration-line: underline; font-weight: 600;}
.animate-spin {
  animation: spin 1s linear infinite;
}
#share-btn-container {
  display: flex; 
  padding-left: 0.5rem !important; 
  padding-right: 0.5rem !important; 
  background-color: #000000; 
  justify-content: center; 
  align-items: center; 
  border-radius: 9999px !important; 
  max-width: 15rem;
  height: 36px;
}
div#share-btn-container > div {
    flex-direction: row;
    background: black;
    align-items: center;
}
#share-btn-container:hover {
  background-color: #060606;
}
#share-btn {
  all: initial; 
  color: #ffffff;
  font-weight: 600; 
  cursor:pointer; 
  font-family: 'IBM Plex Sans', sans-serif; 
  margin-left: 0.5rem !important; 
  padding-top: 0.5rem !important; 
  padding-bottom: 0.5rem !important;
  right:0;
}
#share-btn * {
  all: unset;
}
#share-btn-container div:nth-child(-n+2){
  width: auto !important;
  min-height: 0px !important;
}
#share-btn-container .wrap {
  display: none !important;
}
#share-btn-container.hidden {
  display: none!important;
}
img[src*='#center'] { 
    display: inline-block;
    margin: unset;
}
.footer {
        margin-bottom: 45px;
        margin-top: 10px;
        text-align: center;
        border-bottom: 1px solid #e5e5e5;
    }
    .footer>p {
        font-size: .8rem;
        display: inline-block;
        padding: 0 10px;
        transform: translateY(10px);
        background: white;
    }
    .dark .footer {
        border-color: #303030;
    }
    .dark .footer>p {
        background: #0b0f19;
    }
"""

def mode_update(mode):
    if mode == 'demo':
        trajectories_mode = [gr.Row(visible=True), gr.Row(visible=False)]
    elif mode == 'diy':
        trajectories_mode = [gr.Row(visible=False), gr.Row(visible=True)]
    else:
        trajectories_mode = [gr.Row(visible=False), gr.Row(visible=False)]
    return trajectories_mode

def keyframe_update(num):
    keyframes = []
    if type(num) != int:
        num = 0

    for i in range(num):
        keyframes.append(gr.Row(visible=True))
    for i in range(MAX_KEYS - num):
        keyframes.append(gr.Row(visible=False))
    return keyframes

def demo_update(mode):
    if mode == 'topleft->bottomright':
        num = 2
    elif mode == 'bottomleft->topright':
        num = 2
    elif mode == 'topleft->bottomleft->bottomright':
        num = 3
    elif mode == 'bottomright->topright->topleft':
        num = 3
    elif mode == '"V"':
        num = 4
    elif mode == '"^"':
        num = 4
    elif mode == 'left->right->left->right':
        num = 4
    elif mode == 'triangle':
        num = 4
    else:
        num = 0

    return num

def demo_update_frame(mode):
    frame_indices = []
    if mode == 'topleft->bottomright':
        num = 2
        frame_indices.append(gr.Text(value=0))
        frame_indices.append(gr.Text(value=15))
    elif mode == 'bottomleft->topright':
        num = 2
        frame_indices.append(gr.Text(value=0))
        frame_indices.append(gr.Text(value=15))
    elif mode == 'topleft->bottomleft->bottomright':
        num = 3
        frame_indices.append(gr.Text(value=0))
        frame_indices.append(gr.Text(value=9))
        frame_indices.append(gr.Text(value=15))
    elif mode == 'bottomright->topright->topleft':
        num = 3
        frame_indices.append(gr.Text(value=0))
        frame_indices.append(gr.Text(value=6))
        frame_indices.append(gr.Text(value=15))
    elif mode == '"V"':
        num = 4
        frame_indices.append(gr.Text(value=0))
        frame_indices.append(gr.Text(value=7))
        frame_indices.append(gr.Text(value=8))
        frame_indices.append(gr.Text(value=15))
    elif mode == '"^"':
        num = 4
        frame_indices.append(gr.Text(value=0))
        frame_indices.append(gr.Text(value=7))
        frame_indices.append(gr.Text(value=8))
        frame_indices.append(gr.Text(value=15))
    elif mode == 'left->right->left->right':
        num = 4
        frame_indices.append(gr.Text(value=0))
        frame_indices.append(gr.Text(value=5))
        frame_indices.append(gr.Text(value=10))
        frame_indices.append(gr.Text(value=15))
    elif mode == 'triangle':
        num = 4
        frame_indices.append(gr.Text(value=0))
        frame_indices.append(gr.Text(value=5))
        frame_indices.append(gr.Text(value=10))
        frame_indices.append(gr.Text(value=15))
    else:
        num = 0

    for i in range(MAX_KEYS - num):
        frame_indices.append(gr.Text())
    return frame_indices

def demo_update_h(mode):
    h_positions = []
    if mode == 'topleft->bottomright':
        num = 2
        h_positions.append(gr.Slider(value=0.1))
        h_positions.append(gr.Slider(value=0.9))
    elif mode == 'bottomleft->topright':
        num = 2
        h_positions.append(gr.Slider(value=0.9))
        h_positions.append(gr.Slider(value=0.1))
    elif mode == 'topleft->bottomleft->bottomright':
        num = 3
        h_positions.append(gr.Slider(value=0.1))
        h_positions.append(gr.Slider(value=0.9))
        h_positions.append(gr.Slider(value=0.9))
    elif mode == 'bottomright->topright->topleft':
        num = 3
        h_positions.append(gr.Slider(value=0.9))
        h_positions.append(gr.Slider(value=0.1))
        h_positions.append(gr.Slider(value=0.1))
    elif mode == '"V"':
        num = 4
        h_positions.append(gr.Slider(value=0.1))
        h_positions.append(gr.Slider(value=0.9))
        h_positions.append(gr.Slider(value=0.9))
        h_positions.append(gr.Slider(value=0.1))
    elif mode == '"^"':
        num = 4
        h_positions.append(gr.Slider(value=0.9))
        h_positions.append(gr.Slider(value=0.1))
        h_positions.append(gr.Slider(value=0.1))
        h_positions.append(gr.Slider(value=0.9))
    elif mode == 'left->right->left->right':
        num = 4
        h_positions.append(gr.Slider(value=0.5))
        h_positions.append(gr.Slider(value=0.5))
        h_positions.append(gr.Slider(value=0.5))
        h_positions.append(gr.Slider(value=0.5))
    elif mode == 'triangle':
        num = 4
        h_positions.append(gr.Slider(value=0.1))
        h_positions.append(gr.Slider(value=0.9))
        h_positions.append(gr.Slider(value=0.9))
        h_positions.append(gr.Slider(value=0.1))
    else:
        num = 0

    for i in range(MAX_KEYS - num):
        h_positions.append(gr.Slider())
    return h_positions

def demo_update_w(mode):
    w_positions = []
    if mode == 'topleft->bottomright':
        num = 2
        w_positions.append(gr.Slider(value=0.1))
        w_positions.append(gr.Slider(value=0.9))
    elif mode == 'bottomleft->topright':
        num = 2
        w_positions.append(gr.Slider(value=0.1))
        w_positions.append(gr.Slider(value=0.9))
    elif mode == 'topleft->bottomleft->bottomright':
        num = 3
        w_positions.append(gr.Slider(value=0.1))
        w_positions.append(gr.Slider(value=0.1))
        w_positions.append(gr.Slider(value=0.9))
    elif mode == 'bottomright->topright->topleft':
        num = 3
        w_positions.append(gr.Slider(value=0.9))
        w_positions.append(gr.Slider(value=0.9))
        w_positions.append(gr.Slider(value=0.1))
    elif mode == '"V"':
        num = 4
        w_positions.append(gr.Slider(value=0.1))
        w_positions.append(gr.Slider(value=0.8/15*7 + 0.1))
        w_positions.append(gr.Slider(value=0.8/15*8 + 0.1))
        w_positions.append(gr.Slider(value=0.9))
    elif mode == '"^"':
        num = 4
        w_positions.append(gr.Slider(value=0.9))
        w_positions.append(gr.Slider(value=0.8/15*8 + 0.1))
        w_positions.append(gr.Slider(value=0.8/15*7 + 0.1))
        w_positions.append(gr.Slider(value=0.1))
    elif mode == 'left->right->left->right':
        num = 4
        w_positions.append(gr.Slider(value=0.1))
        w_positions.append(gr.Slider(value=0.9))
        w_positions.append(gr.Slider(value=0.1))
        w_positions.append(gr.Slider(value=0.9))
    elif mode == 'triangle':
        num = 4
        w_positions.append(gr.Slider(value=0.5))
        w_positions.append(gr.Slider(value=0.9))
        w_positions.append(gr.Slider(value=0.1))
        w_positions.append(gr.Slider(value=0.5))
    else:
        num = 0

    for i in range(MAX_KEYS - num):
        w_positions.append(gr.Slider())
    return w_positions

def plot_update(*positions):
    key_length = positions[-1]
    frame_indices = positions[:key_length]
    if type(key_length) != int or len(frame_indices) < 2:
        traj_plot = gr.Plot(
            label="Trajectory"
        )
        return traj_plot
    frame_indices = [int(i) for i in frame_indices]
    h_positions = positions[MAX_KEYS:MAX_KEYS+key_length]
    w_positions = positions[2*MAX_KEYS:2*MAX_KEYS+key_length]
    frame_indices, h_positions, w_positions = zip(*sorted(zip(frame_indices, h_positions, w_positions)))
    plt.cla()
    plt.xlim(0, 1)
    plt.ylim(0, 1)
    plt.gca().invert_yaxis()
    plt.gca().xaxis.tick_top()
    plt.plot(w_positions, h_positions, linestyle='-', marker = 'o', markerfacecolor='r')
    traj_plot = gr.Plot(
        label="Trajectory",
        value = plt
    )
    return traj_plot


with gr.Blocks(css=css) as demo:
    with gr.Column(elem_id="col-container"):
        gr.Markdown(
            """
            <h1 style="text-align: center;">FreeTraj</h1>
            <p style="text-align: center;">
            Tuning-Free Trajectory Control in Video Diffusion Models
            </p>
            <p style="text-align: center;">
            <a href="https://arxiv.org/abs/2406.16863" target="_blank"><b>[arXiv]</b></a> &nbsp;&nbsp;&nbsp;&nbsp;
            <a href="http://haonanqiu.com/projects/FreeTraj.html" target="_blank"><b>[Project Page]</b></a> &nbsp;&nbsp;&nbsp;&nbsp;
            <a href="https://github.com/arthur-qiu/FreeTraj" target="_blank"><b>[Code]</b></a>
            </p>         
            """
        )

        keyframes = []
        frame_indices = []
        h_positions = []
        w_positions = []

        with gr.Row():
            video_result = gr.Video(label="Video Output")
            video_result_bbox = gr.Video(label="Video Output with BBox")

        with gr.Group():
            with gr.Row():
                prompt_in = gr.Textbox(label="Prompt", placeholder="A corgi running on the grassland on the grassland.", scale = 5)
                target_indices = gr.Textbox(label="Target Indices (1 for the first word, necessary!)", placeholder="1,2", scale = 2)

            with gr.Row():
                radio_mode = gr.Radio(label='Trajectory Mode', choices = ['demo', 'diy', 'ori'], scale = 1)
                height_ratio = gr.Slider(label='Height Ratio of BBox',
                                minimum=0.2,
                                maximum=0.4,
                                step=0.01,
                                value=0.3,
                                scale = 1)
                width_ratio = gr.Slider(label='Width Ratio of BBox',
                                minimum=0.2,
                                maximum=0.4,
                                step=0.01,
                                value=0.3, 
                                scale = 1)
            
            with gr.Row(visible=False) as row_demo:
                dropdown_demo = gr.Dropdown(
                    label="Demo Trajectory",
                    choices= ['topleft->bottomright', 'bottomleft->topright', 'topleft->bottomleft->bottomright', 'bottomright->topright->topleft', '"V"', '"^"', 'left->right->left->right', 'triangle']
                )
                
            with gr.Row(visible=False) as row_diy:
                dropdown_diy = gr.Dropdown(
                    label="Number of Keyframes",
                    choices=range(2, MAX_KEYS+1),
                )
                
            for i in range(MAX_KEYS):
                with gr.Row(visible=False) as row:
                    text = gr.Textbox(
                        value=f"Keyframe #{i}",
                        interactive=False,
                        container = False,
                        lines = 3,
                        scale=1
                    )
                    frame_ids = gr.Textbox(
                        None,
                        label=f"Frame Indices #{i}",
                        interactive=True,
                        scale=2
                    )
                    h_position = gr.Slider(label='Position in Height',
                        minimum=0.0,
                        maximum=1.0,
                        step=0.01,
                        scale=2)
                    w_position = gr.Slider(label='Position in Width',
                        minimum=0.0,
                        maximum=1.0,
                        step=0.01,
                        scale=2)
                    
                frame_indices.append(frame_ids)
                h_positions.append(h_position)
                w_positions.append(w_position)
                keyframes.append(row)


            dropdown_demo.change(demo_update, dropdown_demo, dropdown_diy)
            dropdown_diy.change(keyframe_update, dropdown_diy, keyframes)
            dropdown_demo.change(demo_update_frame, dropdown_demo, frame_indices)
            dropdown_demo.change(demo_update_h, dropdown_demo, h_positions)
            dropdown_demo.change(demo_update_w, dropdown_demo, w_positions)
            radio_mode.change(mode_update, radio_mode, [row_demo, row_diy])

            traj_plot = gr.Plot(
                label="Trajectory"
            )

            h_positions[0].change(plot_update, frame_indices + h_positions + w_positions + [dropdown_diy], traj_plot)
            h_positions[1].change(plot_update, frame_indices + h_positions + w_positions + [dropdown_diy], traj_plot)
            h_positions[2].change(plot_update, frame_indices + h_positions + w_positions + [dropdown_diy], traj_plot)
            h_positions[3].change(plot_update, frame_indices + h_positions + w_positions + [dropdown_diy], traj_plot)
            h_positions[4].change(plot_update, frame_indices + h_positions + w_positions + [dropdown_diy], traj_plot)
            w_positions[0].change(plot_update, frame_indices + h_positions + w_positions + [dropdown_diy], traj_plot)
            w_positions[1].change(plot_update, frame_indices + h_positions + w_positions + [dropdown_diy], traj_plot)
            w_positions[2].change(plot_update, frame_indices + h_positions + w_positions + [dropdown_diy], traj_plot)
            w_positions[3].change(plot_update, frame_indices + h_positions + w_positions + [dropdown_diy], traj_plot)
            w_positions[4].change(plot_update, frame_indices + h_positions + w_positions + [dropdown_diy], traj_plot)

            with gr.Row():
                with gr.Accordion('Useful FreeTraj Parameters (feel free to adjust these parameters based on your prompt): ', open=True):
                    with gr.Row():
                        ddim_edit = gr.Slider(label='Editing Steps (larger for better control while losing some quality)',
                                minimum=0,
                                maximum=12,
                                step=1,
                                value=6)
                        seed = gr.Slider(label='Random Seed',
                                minimum=0,
                                maximum=10000,
                                step=1,
                                value=123)
                        
            with gr.Row():
                with gr.Accordion('Useless FreeTraj Parameters (mostly no need to adjust): ', open=False):
                    with gr.Row():
                        ddim_steps = gr.Slider(label='DDIM Steps',
                                minimum=5,
                                maximum=50,
                                step=1,
                                value=50)
                        unconditional_guidance_scale = gr.Slider(label='Unconditional Guidance Scale',
                                minimum=1.0,
                                maximum=20.0,
                                step=0.1,
                                value=12.0)
                    with gr.Row():
                        video_fps = gr.Slider(label='Video FPS (larger for quicker motion)',
                                minimum=8,
                                maximum=36,
                                step=4,
                                value=16)
                        save_fps = gr.Slider(label='Save FPS',
                                minimum=1,
                                maximum=30,
                                step=1,
                                value=10)
                
        with gr.Row():
            submit_btn = gr.Button("Generate", variant='primary')

        with gr.Row():
            check_btn = gr.Button("Check Existing Results (in case of the connection lost)", variant='secondary')

        with gr.Row():
            gr.Examples(label='Sample Prompts', examples=examples, inputs=[prompt_in, target_indices, ddim_edit, seed, ddim_steps, unconditional_guidance_scale, video_fps, save_fps, height_ratio, width_ratio, radio_mode, dropdown_diy, *frame_indices, *h_positions, *w_positions])

        demo_list = ['0026_0_0.4_0.4.gif', '0047_1_0.4_0.3.gif', '0051_1_0.4_0.4.gif']
        demo_pick = random.randint(0, len(demo_list) - 1)
        with gr.Row():
            for i in range(len(demo_list)):
                gr.Image(show_label = False, show_download_button = False, value='assets/' + demo_list[i])

        with gr.Row():
            gr.Markdown(
                """
                <h2 style="text-align: center;">Hints</h2>
                <p style="text-align: center;">
                1. Choose trajectory mode <b>"ori"</b> to see whether the prompt works on the pre-trained model. 
                </p>    
                <p style="text-align: center;">
                2. Adjust the prompt or random seed to get a qualified video.
                </p>  
                <p style="text-align: center;">
                3. Choose trajectory mode <b>"demo"</b> to see whether <b>FreeTraj</b> works or not.
                </p>  
                <p style="text-align: center;">
                4. Choose trajectory mode <b>"diy"</b> to plan new trajectory. It may fail in some extreme cases.
                </p>       
                """
            )
            

    submit_btn.click(fn=infer,
            inputs=[prompt_in, target_indices, ddim_edit, seed, ddim_steps, unconditional_guidance_scale, video_fps, save_fps, height_ratio, width_ratio, radio_mode, dropdown_diy, *frame_indices, *h_positions, *w_positions],
            outputs=[video_result, video_result_bbox],
            api_name="generate")
    
    check_btn.click(fn=check,
            inputs=[radio_mode],
            outputs=[video_result, video_result_bbox],
            api_name="check")

demo.queue(max_size=8).launch(show_api=True)