Training in progress, step 3400, checkpoint

last-checkpoint/adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1b41c1be8b974e693981390b4f8ea3183b990f4c6cbeb85e09c310968df2c797
+oid sha256:d712f05560ef9d87500810ebe4e96f57f0c9005ed8838b0dd018fe564e447831
 size 29034840

last-checkpoint/global_step3400/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:90042a29ea832b455e0b0febb703a80c27669d31f9276f94b31408d71131b4eb
+size 43429616

last-checkpoint/global_step3400/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:aeb05ce0de6c9cc9511842de2c347ed48abb1f8974e7d760fcd5aa31191bad35
+size 43429616

last-checkpoint/global_step3400/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bba280d3f32e7dc1f7458fecad62f8a761d5d50be21d68ad1b200a64ca75a15e
+size 43429616

last-checkpoint/global_step3400/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bee7623add4bb2ee37b178a4b5c0be5536d5ad2b1dc326c8ce3238e32b4cca1b
+size 43429616

last-checkpoint/global_step3400/zero_pp_rank_0_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e3f0a5054eba9f5f2ca369e0534352b8e94fdf765e97f88e64f72db4ba475921
+size 637299

last-checkpoint/global_step3400/zero_pp_rank_1_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0f4d45ab2afd18d672717de6056f7b8589be6a43ccd2a95722949876973300f7
+size 637171

last-checkpoint/global_step3400/zero_pp_rank_2_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ea3071cbf7dfa8e2d404c25c04d958ee3c7a04c22c8522cafd06b4dff7601dd9
+size 637171

last-checkpoint/global_step3400/zero_pp_rank_3_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4167c44ffe9ec068048477eaf74c83fdb2ca9cf033c480fee6e89e70506c8e98
+size 637171

last-checkpoint/latest

@@ -1 +1 @@
-global_step3350
+global_step3400

last-checkpoint/rng_state_0.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:26d08ced4d0f6490515a22c9e9401cc7f71de8b6e2c1525e9c8dac221d4b80ab
+oid sha256:abe9163f042a56ab41ea5c2436dff084d8a4a6358e7f4cb1f18e04cb69810300
 size 15024

last-checkpoint/rng_state_1.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:182d3e4bee7983edebdece2816547628c5ac3e14f5b53eac54c08ad9cbfbfac7
+oid sha256:3c621ead8d06a0f1d00c5217cc2cfdc90c8c62fa1cb0da0986461ec51fd1766b
 size 15024

last-checkpoint/rng_state_2.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:674c98433de6d8eccea8d9f711e54aa6a237220bb981e22133b7735c14dc835f
+oid sha256:eff8dbefa4ff395a5376144d756cbe824baaab98a892f200d30b7916c24d27cf
 size 15024

last-checkpoint/rng_state_3.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4e6ca2e2b0a88474561faf5e462ebee1de20672630f977a22533f633eb059d6d
+oid sha256:dd67eb847a256b4f0de5857c5e8a43697485d1a0f6032004d0bc19149d77879c
 size 15024

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:df3be552cf2524f9ece2b6a286f0ce246d18d14d42f9b8c771a555e051bcee33
+oid sha256:e5fc2897164e26dbf189cf39613143884cb612b3f808a6a18c481ece64d73bc7
 size 1064

last-checkpoint/trainer_state.json

@@ -1,9 +1,9 @@
 {
   "best_metric": 0.4145541489124298,
   "best_model_checkpoint": "saves/CADICA_qwenvl_stenosis_detect_scale4/lora/sft/checkpoint-2350",
-  "epoch": 0.8627349987123358,
+  "epoch": 0.8756116404841617,
   "eval_steps": 50,
-  "global_step": 3350,
+  "global_step": 3400,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -5970,11 +5970,100 @@
       "eval_steps_per_second": 0.798,
       "num_input_tokens_seen": 35179104,
       "step": 3350
+    },
+    {
+      "epoch": 0.8640226628895185,
+      "grad_norm": 5.553235621230406,
+      "learning_rate": 4.788399817602929e-08,
+      "loss": 0.3179,
+      "num_input_tokens_seen": 35231608,
+      "step": 3355
+    },
+    {
+      "epoch": 0.865310327066701,
+      "grad_norm": 4.202242288396885,
+      "learning_rate": 3.7835537837338506e-08,
+      "loss": 0.2829,
+      "num_input_tokens_seen": 35284448,
+      "step": 3360
+    },
+    {
+      "epoch": 0.8665979912438836,
+      "grad_norm": 3.6414795103608255,
+      "learning_rate": 2.8968690057051828e-08,
+      "loss": 0.2579,
+      "num_input_tokens_seen": 35336520,
+      "step": 3365
+    },
+    {
+      "epoch": 0.8678856554210662,
+      "grad_norm": 4.481511308866539,
+      "learning_rate": 2.128366453743591e-08,
+      "loss": 0.2862,
+      "num_input_tokens_seen": 35388728,
+      "step": 3370
+    },
+    {
+      "epoch": 0.8691733195982487,
+      "grad_norm": 4.3032070874799,
+      "learning_rate": 1.4780643030476438e-08,
+      "loss": 0.2812,
+      "num_input_tokens_seen": 35441824,
+      "step": 3375
+    },
+    {
+      "epoch": 0.8704609837754314,
+      "grad_norm": 5.8821140210764336,
+      "learning_rate": 9.459779333587104e-09,
+      "loss": 0.3174,
+      "num_input_tokens_seen": 35495128,
+      "step": 3380
+    },
+    {
+      "epoch": 0.871748647952614,
+      "grad_norm": 7.380544386822247,
+      "learning_rate": 5.3211992859791835e-09,
+      "loss": 0.3049,
+      "num_input_tokens_seen": 35548144,
+      "step": 3385
+    },
+    {
+      "epoch": 0.8730363121297966,
+      "grad_norm": 1.5375610441333851,
+      "learning_rate": 2.3650007656805806e-09,
+      "loss": 0.2882,
+      "num_input_tokens_seen": 35600936,
+      "step": 3390
+    },
+    {
+      "epoch": 0.8743239763069791,
+      "grad_norm": 6.170480848656164,
+      "learning_rate": 5.912536872321184e-10,
+      "loss": 0.2789,
+      "num_input_tokens_seen": 35653896,
+      "step": 3395
+    },
+    {
+      "epoch": 0.8756116404841617,
+      "grad_norm": 5.211578123351505,
+      "learning_rate": 0.0,
+      "loss": 0.3187,
+      "num_input_tokens_seen": 35706848,
+      "step": 3400
+    },
+    {
+      "epoch": 0.8756116404841617,
+      "eval_loss": 0.4701705873012543,
+      "eval_runtime": 37.5324,
+      "eval_samples_per_second": 3.197,
+      "eval_steps_per_second": 0.799,
+      "num_input_tokens_seen": 35706848,
+      "step": 3400
     }
   ],
   "logging_steps": 5,
   "max_steps": 3400,
-  "num_input_tokens_seen": 35179104,
+  "num_input_tokens_seen": 35706848,
   "num_train_epochs": 1,
   "save_steps": 50,
   "stateful_callbacks": {
@@ -5984,12 +6073,12 @@
         "should_evaluate": false,
         "should_log": false,
         "should_save": true,
-        "should_training_stop": false
+        "should_training_stop": true
       },
       "attributes": {}
     }
   },
-  "total_flos": 2321016547704832.0,
+  "total_flos": 2355853440057344.0,
   "train_batch_size": 1,
   "trial_name": null,
   "trial_params": null