Training in progress, step 1250, checkpoint

last-checkpoint/adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0c1933263a4147535e3edcd506e0269c5b74a0e48619e4b7de41e058b06d730b
+oid sha256:3bb18bc55e51868d1cfa188776cb8ea35d99e69f7889e54da75d1f08cd00c5fb
 size 29034840

last-checkpoint/global_step1250/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e4c18395628d692f85cdcc38b110c35b2ca6c6d509c6e30ed648c342ea83d216
+size 43429616

last-checkpoint/global_step1250/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:333339f16be8f91f3e0298ed7e91379420f1064e4c7d6380168d839e5acf739e
+size 43429616

last-checkpoint/global_step1250/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:aff86ed75cb315468dc04f8257764cbe9ed1353be3c48ce33cd8ce11f8e4cffa
+size 43429616

last-checkpoint/global_step1250/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:948fe1de83dd38e83dc0b548852c28d1b92a0369ee16abae2e25b19e0c322ede
+size 43429616

last-checkpoint/global_step1250/zero_pp_rank_0_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3ec0f2cde302545248de950463724f6d392acd94dddf06b2baba0a3d8f91b0e0
+size 637299

last-checkpoint/global_step1250/zero_pp_rank_1_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e30a5b6d29417f61fce028606e2a43dda69feda25ab401bf98a939ad68d0a488
+size 637171

last-checkpoint/global_step1250/zero_pp_rank_2_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:55d5732bdcd7f622a6122c509cb64b5a03edeaa513ee805d437e1045732cd1d7
+size 637171

last-checkpoint/global_step1250/zero_pp_rank_3_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ac58ba9b33d13c7004129934ac2b499707908f73809a09276a3945da3fd62d36
+size 637171

last-checkpoint/latest

@@ -1 +1 @@
-global_step1200
+global_step1250

last-checkpoint/rng_state_0.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a209a0c0025f9ce8e2beeba50c1f0828d5c34a2482310fcd0bf5fc24c2c67be2
+oid sha256:15bbef9a4c878a05b4f189e9f77701a153dc9faf093499714094cb36ac0ca030
 size 15024

last-checkpoint/rng_state_1.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a67fb929b8c51f9b1c6ff9f11366e57e55128a1d36df85a9d37a008b49017a75
+oid sha256:7a7122760cfda3f3e13eac4f7e56d09b0fd3beac4bd49b2f0e58da9519469ff9
 size 15024

last-checkpoint/rng_state_2.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1b9ef3b0c0978d0b611f4257c939f1c2c6f07e6227bfea6675532d285b0b64a7
+oid sha256:71edf044d4ea96e62ffecab515d42b58ae56cd64abd0092a897468e0e0d7c10f
 size 15024

last-checkpoint/rng_state_3.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0183d14c8ed52ee533139532e9bcf7bc34ec297a064845b35741cb501d92675f
+oid sha256:ef5ca30351deca9d83675a183b2186ef346093f56e0d5a6dacdf62ef921f12c2
 size 15024

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:16e0cffc6b063574ed312ee2198c86d3dddf2450d0400f042fdc08bd22dcbc7f
+oid sha256:e821d0d4ad1c6c22d1f5fd3b6821d2a85d44ee21d649d544e34dfcb627d9b580
 size 1064

last-checkpoint/trainer_state.json

@@ -1,9 +1,9 @@
 {
-  "best_metric": 0.45381438732147217,
-  "best_model_checkpoint": "saves/CADICA_qwenvl_stenosis_detect_scale4/lora/sft/checkpoint-1150",
-  "epoch": 0.3090394025238218,
+  "best_metric": 0.4339977502822876,
+  "best_model_checkpoint": "saves/CADICA_qwenvl_stenosis_detect_scale4/lora/sft/checkpoint-1250",
+  "epoch": 0.3219160442956477,
   "eval_steps": 50,
-  "global_step": 1200,
+  "global_step": 1250,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -2143,11 +2143,100 @@
       "eval_steps_per_second": 0.775,
       "num_input_tokens_seen": 12596208,
       "step": 1200
+    },
+    {
+      "epoch": 0.31032706670100435,
+      "grad_norm": 4.0964966925406365,
+      "learning_rate": 7.673382966299163e-05,
+      "loss": 0.5226,
+      "num_input_tokens_seen": 12648936,
+      "step": 1205
+    },
+    {
+      "epoch": 0.311614730878187,
+      "grad_norm": 7.87992303723905,
+      "learning_rate": 7.65280325993715e-05,
+      "loss": 0.4757,
+      "num_input_tokens_seen": 12702432,
+      "step": 1210
+    },
+    {
+      "epoch": 0.31290239505536954,
+      "grad_norm": 6.822793875901239,
+      "learning_rate": 7.63216081438678e-05,
+      "loss": 0.451,
+      "num_input_tokens_seen": 12755128,
+      "step": 1215
+    },
+    {
+      "epoch": 0.31419005923255217,
+      "grad_norm": 8.804840574778536,
+      "learning_rate": 7.611456117844934e-05,
+      "loss": 0.4155,
+      "num_input_tokens_seen": 12808152,
+      "step": 1220
+    },
+    {
+      "epoch": 0.31547772340973473,
+      "grad_norm": 12.832933509895003,
+      "learning_rate": 7.59068965998074e-05,
+      "loss": 0.4094,
+      "num_input_tokens_seen": 12861592,
+      "step": 1225
+    },
+    {
+      "epoch": 0.31676538758691736,
+      "grad_norm": 3.769639586972444,
+      "learning_rate": 7.569861931923989e-05,
+      "loss": 0.4663,
+      "num_input_tokens_seen": 12914240,
+      "step": 1230
+    },
+    {
+      "epoch": 0.3180530517640999,
+      "grad_norm": 5.011688667303979,
+      "learning_rate": 7.548973426253521e-05,
+      "loss": 0.468,
+      "num_input_tokens_seen": 12967472,
+      "step": 1235
+    },
+    {
+      "epoch": 0.3193407159412825,
+      "grad_norm": 5.925703481508644,
+      "learning_rate": 7.528024636985575e-05,
+      "loss": 0.4744,
+      "num_input_tokens_seen": 13020232,
+      "step": 1240
+    },
+    {
+      "epoch": 0.3206283801184651,
+      "grad_norm": 3.511846132089351,
+      "learning_rate": 7.507016059562107e-05,
+      "loss": 0.4269,
+      "num_input_tokens_seen": 13073032,
+      "step": 1245
+    },
+    {
+      "epoch": 0.3219160442956477,
+      "grad_norm": 6.878508053492975,
+      "learning_rate": 7.485948190839077e-05,
+      "loss": 0.4725,
+      "num_input_tokens_seen": 13125624,
+      "step": 1250
+    },
+    {
+      "epoch": 0.3219160442956477,
+      "eval_loss": 0.4339977502822876,
+      "eval_runtime": 39.1132,
+      "eval_samples_per_second": 3.068,
+      "eval_steps_per_second": 0.767,
+      "num_input_tokens_seen": 13125624,
+      "step": 1250
     }
   ],
   "logging_steps": 5,
   "max_steps": 3400,
-  "num_input_tokens_seen": 12596208,
+  "num_input_tokens_seen": 13125624,
   "num_train_epochs": 1,
   "save_steps": 50,
   "stateful_callbacks": {
@@ -2162,7 +2251,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 831025715609600.0,
+  "total_flos": 865954009186304.0,
   "train_batch_size": 1,
   "trial_name": null,
   "trial_params": null