| dataset: | |
| align_stage_components: | |
| - download/videollava/valley_llavaimage.json | |
| - download/videollava | |
| dataset_id: videollava | |
| dataset_root_dir: data | |
| finetune_stage_components: | |
| - download/videollava/videochatgpt_llavaimage_tune.json | |
| - download/videollava | |
| type: videollava | |
| dataset_class: FinetuneVideoDataset | |
| hf_token: .hf_token | |
| model: | |
| align_epochs: 1 | |
| align_global_batch_size: 256 | |
| align_learning_rate: 0.001 | |
| align_lr_scheduler_type: linear-warmup+cosine-decay | |
| align_max_grad_norm: 1.0 | |
| align_max_steps: null | |
| align_per_device_batch_size: 16 | |
| align_train_strategy: fsdp-full-shard | |
| align_warmup_ratio: 0.03 | |
| align_weight_decay: 0.0 | |
| arch_specifier: 3davg+linear | |
| enable_gradient_checkpointing: true | |
| enable_mixed_precision_training: true | |
| feature_fusion: cross_attention_avg_lq | |
| finetune_epochs: 1 | |
| finetune_global_batch_size: 128 | |
| finetune_learning_rate: 2.0e-05 | |
| finetune_lr_scheduler_type: linear-warmup+cosine-decay | |
| finetune_max_grad_norm: 1.0 | |
| finetune_max_steps: null | |
| finetune_per_device_batch_size: 8 | |
| finetune_train_strategy: fsdp-full-shard | |
| finetune_warmup_ratio: 0.03 | |
| finetune_weight_decay: 0.1 | |
| image_resize_strategy: resize-naive | |
| llm_backbone_id: llama2-7b-pure | |
| llm_max_length: 2048 | |
| model_id: prism-siglip-controlled+7b | |
| num_frames: | |
| - 16 | |
| - 16 | |
| - 32 | |
| - 16 | |
| projector_token_length: 64 | |
| reduce_in_full_precision: false | |
| type: merv-full | |
| video_backbone_ids: | |
| - languagebind-video-noclass | |
| - dinov2-video-all-tokens | |
| - vivit-google-b-all-no-cls-16frames | |
| - siglip-vit-b16-224px-all-no-cls | |
| visual_feature_length: 1024 | |
| pretrained_checkpoint: runs/final-4e-stage1-lr1e-4/checkpoints/latest-checkpoint.pt | |
| run_id: final-4e-stage2-(lr1e-4) | |
| run_root_dir: runs | |
| seed: 7 | |
| slurm_id: '751580' | |
| stage: second_finetune | |
| trackers: | |
| - jsonl | |
| - wandb | |