| { "framework": "pytorch", | |
| "task": "text-to-video-synthesis", | |
| "model": { | |
| "type": "latent-text-to-video-synthesis", | |
| "model_args": { | |
| "ckpt_clip": "open_clip_pytorch_model.bin", | |
| "ckpt_unet": "text2video_pytorch_model.pth", | |
| "ckpt_autoencoder": "../VQGAN_autoencoder.pth", | |
| "max_frames": 16, | |
| "tiny_gpu": 1 | |
| }, | |
| "model_cfg": { | |
| "unet_in_dim": 4, | |
| "unet_dim": 320, | |
| "unet_y_dim": 768, | |
| "unet_context_dim": 1024, | |
| "unet_out_dim": 4, | |
| "unet_dim_mult": [1, 2, 4, 4], | |
| "unet_num_heads": 8, | |
| "unet_head_dim": 64, | |
| "unet_res_blocks": 2, | |
| "unet_attn_scales": [1, 0.5, 0.25], | |
| "unet_dropout": 0.1, | |
| "temporal_attention": "True", | |
| "num_timesteps": 1000, | |
| "mean_type": "eps", | |
| "var_type": "fixed_small", | |
| "loss_type": "mse" | |
| } | |
| }, | |
| "pipeline": { | |
| "type": "latent-text-to-video-synthesis" | |
| } | |
| } |