Upload folder using huggingface_hub

merv-full/checkpoints/latest-checkpoint.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f4fb83072ab1b80a96943f9f1ee269739ee05b28a0263f90bedc2e814d1b6d2c
+size 27190883468

merv-full/config.json

@@ -0,0 +1,73 @@
+{
+  "dataset": {
+    "align_stage_components": [
+      "download/videollava/valley_llavaimage.json",
+      "download/videollava"
+    ],
+    "dataset_id": "videollava",
+    "dataset_root_dir": "data",
+    "finetune_stage_components": [
+      "download/videollava/videochatgpt_llavaimage_tune.json",
+      "download/videollava"
+    ],
+    "type": "videollava"
+  },
+  "dataset_class": "FinetuneVideoDataset",
+  "hf_token": ".hf_token",
+  "model": {
+    "align_epochs": 1,
+    "align_global_batch_size": 256,
+    "align_learning_rate": 0.001,
+    "align_lr_scheduler_type": "linear-warmup+cosine-decay",
+    "align_max_grad_norm": 1.0,
+    "align_max_steps": null,
+    "align_per_device_batch_size": 16,
+    "align_train_strategy": "fsdp-full-shard",
+    "align_warmup_ratio": 0.03,
+    "align_weight_decay": 0.0,
+    "arch_specifier": "3davg+linear",
+    "enable_gradient_checkpointing": true,
+    "enable_mixed_precision_training": true,
+    "feature_fusion": "cross_attention_avg_lq",
+    "finetune_epochs": 1,
+    "finetune_global_batch_size": 128,
+    "finetune_learning_rate": 2e-05,
+    "finetune_lr_scheduler_type": "linear-warmup+cosine-decay",
+    "finetune_max_grad_norm": 1.0,
+    "finetune_max_steps": null,
+    "finetune_per_device_batch_size": 8,
+    "finetune_train_strategy": "fsdp-full-shard",
+    "finetune_warmup_ratio": 0.03,
+    "finetune_weight_decay": 0.1,
+    "image_resize_strategy": "resize-naive",
+    "llm_backbone_id": "llama2-7b-pure",
+    "llm_max_length": 2048,
+    "model_id": "prism-siglip-controlled+7b",
+    "num_frames": [
+      16,
+      16,
+      32,
+      16
+    ],
+    "projector_token_length": 64,
+    "reduce_in_full_precision": false,
+    "type": "merv-full",
+    "video_backbone_ids": [
+      "languagebind-video-noclass",
+      "dinov2-video-all-tokens",
+      "vivit-google-b-all-no-cls-16frames",
+      "siglip-vit-b16-224px-all-no-cls"
+    ],
+    "visual_feature_length": 1024
+  },
+  "pretrained_checkpoint": "runs/final-4e-stage1-lr1e-4/checkpoints/latest-checkpoint.pt",
+  "run_id": "final-4e-stage2-(lr1e-4)",
+  "run_root_dir": "runs",
+  "seed": 7,
+  "slurm_id": "751580",
+  "stage": "second_finetune",
+  "trackers": [
+    "jsonl",
+    "wandb"
+  ]
+}

merv-full/config.yaml

@@ -0,0 +1,64 @@
+dataset:
+  align_stage_components:
+  - download/videollava/valley_llavaimage.json
+  - download/videollava
+  dataset_id: videollava
+  dataset_root_dir: data
+  finetune_stage_components:
+  - download/videollava/videochatgpt_llavaimage_tune.json
+  - download/videollava
+  type: videollava
+dataset_class: FinetuneVideoDataset
+hf_token: .hf_token
+model:
+  align_epochs: 1
+  align_global_batch_size: 256
+  align_learning_rate: 0.001
+  align_lr_scheduler_type: linear-warmup+cosine-decay
+  align_max_grad_norm: 1.0
+  align_max_steps: null
+  align_per_device_batch_size: 16
+  align_train_strategy: fsdp-full-shard
+  align_warmup_ratio: 0.03
+  align_weight_decay: 0.0
+  arch_specifier: 3davg+linear
+  enable_gradient_checkpointing: true
+  enable_mixed_precision_training: true
+  feature_fusion: cross_attention_avg_lq
+  finetune_epochs: 1
+  finetune_global_batch_size: 128
+  finetune_learning_rate: 2.0e-05
+  finetune_lr_scheduler_type: linear-warmup+cosine-decay
+  finetune_max_grad_norm: 1.0
+  finetune_max_steps: null
+  finetune_per_device_batch_size: 8
+  finetune_train_strategy: fsdp-full-shard
+  finetune_warmup_ratio: 0.03
+  finetune_weight_decay: 0.1
+  image_resize_strategy: resize-naive
+  llm_backbone_id: llama2-7b-pure
+  llm_max_length: 2048
+  model_id: prism-siglip-controlled+7b
+  num_frames:
+  - 16
+  - 16
+  - 32
+  - 16
+  projector_token_length: 64
+  reduce_in_full_precision: false
+  type: merv-full
+  video_backbone_ids:
+  - languagebind-video-noclass
+  - dinov2-video-all-tokens
+  - vivit-google-b-all-no-cls-16frames
+  - siglip-vit-b16-224px-all-no-cls
+  visual_feature_length: 1024
+pretrained_checkpoint: runs/final-4e-stage1-lr1e-4/checkpoints/latest-checkpoint.pt
+run_id: final-4e-stage2-(lr1e-4)
+run_root_dir: runs
+seed: 7
+slurm_id: '751580'
+stage: second_finetune
+trackers:
+- jsonl
+- wandb

merv-full/run-metrics.jsonl

@@ -0,0 +1 @@
+{"hparams": {"dataset": {"align_stage_components": ["download/videollava/valley_llavaimage.json", "download/videollava"], "dataset_id": "videoonlyllava", "dataset_root_dir": "data", "finetune_stage_components": ["download/videollava/videochatgpt_llavaimage_tune.json", "download/videollava"], "type": "videoonlyllava"}, "dataset_class": "FinetuneVideoOnlyDataset", "hf_token": ".hf_token", "model": {"adapter": "cross_attention_avg_lq", "align_epochs": 1, "align_global_batch_size": 256, "align_learning_rate": 0.001, "align_lr_scheduler_type": "linear-warmup+cosine-decay", "align_max_grad_norm": 1.0, "align_max_steps": null, "align_per_device_batch_size": 16, "align_train_strategy": "fsdp-full-shard", "align_warmup_ratio": 0.03, "align_weight_decay": 0.0, "arch_specifier": "3dagv+linear", "enable_gradient_checkpointing": true, "enable_mixed_precision_training": true, "finetune_epochs": 1, "finetune_global_batch_size": 128, "finetune_learning_rate": 2e-05, "finetune_lr_scheduler_type": "linear-warmup+cosine-decay", "finetune_max_grad_norm": 1.0, "finetune_max_steps": null, "finetune_per_device_batch_size": 8, "finetune_train_strategy": "fsdp-full-shard", "finetune_warmup_ratio": 0.03, "finetune_weight_decay": 0.1, "image_resize_strategy": "resize-naive", "llm_backbone_id": "llama2-7b-pure", "llm_max_length": 2048, "model_id": "prism-siglip-controlled+7b", "normalize_post_proj": false, "num_frames": [16, 16, 32, 16], "pre_proj_layernorm": false, "projector_token_length": 64, "reduce_in_full_precision": false, "text_embedding_loc": "openai", "type": "prism-siglip-controlled+7b", "video_backbone_id": null, "video_backbone_ids": ["languagebind-video-noclass", "dinov2-video-all-tokens", "vivit-google-b-all-no-cls-16frames", "siglip-vit-b16-224px-all-no-cls"], "vision_backbone_id": "siglip-vit-so400m-384px", "visual_feature_length": 1024}, "pretrained_checkpoint": "runs/final-4e-stage1-lr1e-4/checkpoints/latest-checkpoint.pt", "run_id": "final-4e-stage2-(lr1e-4)", "run_root_dir": "runs", "seed": 7, "slurm_id": "751580", "stage": "second_finetune", "trackers": ["jsonl", "wandb"], "wandb_entity": "tz-ego4d", "wandb_project": "vidlm"}, "run_id": "final-4e-stage2-(lr1e-4)"}