Training in progress, step 1400, checkpoint

last-checkpoint/adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b3ce6daf3c20e1ee39c6a4135462748b3119e8e74d43e226b9b4b7810797d2c9
+oid sha256:a1c7ec080c49b4ded7d9add721716bdf06187fb00c39ff223f5982466d466c72
 size 29034840

last-checkpoint/global_step1400/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c8ee78757c87123bc8871369b24b5db68971f91924c7db6fa0b8f2846bfddb2f
+size 43429616

last-checkpoint/global_step1400/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b6b09c73d734710dcb2492433d7692deb088dde6c5eb1352333f8bec424636bc
+size 43429616

last-checkpoint/global_step1400/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:366e924c74d9b076360c87c0401c98bc225d173c1a2e270cc5336c3ebfa88f9d
+size 43429616

last-checkpoint/global_step1400/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6c02fbd817bfe5eb184c64d4b1dc62d9d79c1f4f6c5c418ab69f62ce35cc287b
+size 43429616

last-checkpoint/global_step1400/zero_pp_rank_0_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1906adf1b95058095138f9b57795e6e0826a899c9c7e5a525c8eae36ecaa15f4
+size 637299

last-checkpoint/global_step1400/zero_pp_rank_1_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1a0bb562939d112dd834c43097bc2d1ef141d083d45e4138023cb41209e8c874
+size 637171

last-checkpoint/global_step1400/zero_pp_rank_2_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3eb2bc0e5fa21a633543ff99a5a9ea2924b55031a219c0558ebd60e5ccef7d0e
+size 637171

last-checkpoint/global_step1400/zero_pp_rank_3_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:336e83dddf7ea2bb6e2ee90e33a484acff5b821f73e635c4aaaa44a24086519e
+size 637171

last-checkpoint/latest

@@ -1 +1 @@
-global_step1350
+global_step1400

last-checkpoint/rng_state_0.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9747fe881253e52a47314f48068ef9649032bec4cb284b1b4becbb8787f37faa
+oid sha256:fe66a68e61de2221b30fd9749bc68b45a1474bb2cc95901bca9557ac87909355
 size 15024

last-checkpoint/rng_state_1.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3ac02a5554a5ef9e3473dcd2926626ae41f4777354859c7d2bf0a0c1188c0583
+oid sha256:0cd4f3162e46c3bb0f1fc4d3c52c7c33e60f56764458e0c8a73c3810b0a25f8c
 size 15024

last-checkpoint/rng_state_2.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9405d230cc78dac3f3b2ab887674631c15f66fedab0042ab7bc1bd83b8575344
+oid sha256:185cc99aaa81b1b49b3ddc74aa6f97aa3036330983a7b69d52bd191057f9a5d5
 size 15024

last-checkpoint/rng_state_3.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b15237547030ac62d49d70a5465b2e29515e6334f62416eb16c0c6d073f7c6bf
+oid sha256:0e37403c30cb4309e54e5defdde1906486716fc859274035d44aaac5d48a97ba
 size 15024

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ca5b091f7e9fe97f38f540a5a26edd08efe79a114c6bc8ff4d4c0d9acb5b20f2
+oid sha256:ba18a87eab7efca964506a3a18bdf9452eae09db2b3c77bd3a82db3283b5abc0
 size 1064

last-checkpoint/trainer_state.json

@@ -1,9 +1,9 @@
 {
   "best_metric": 0.4339977502822876,
   "best_model_checkpoint": "saves/CADICA_qwenvl_stenosis_detect_scale4/lora/sft/checkpoint-1250",
-  "epoch": 0.3476693278392995,
+  "epoch": 0.3605459696111254,
   "eval_steps": 50,
-  "global_step": 1350,
+  "global_step": 1400,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -2410,11 +2410,100 @@
       "eval_steps_per_second": 0.782,
       "num_input_tokens_seen": 14173240,
       "step": 1350
+    },
+    {
+      "epoch": 0.3489569920164821,
+      "grad_norm": 6.100571377010892,
+      "learning_rate": 7.030720984447279e-05,
+      "loss": 0.41,
+      "num_input_tokens_seen": 14226032,
+      "step": 1355
+    },
+    {
+      "epoch": 0.3502446561936647,
+      "grad_norm": 3.531812694789996,
+      "learning_rate": 7.008477123264848e-05,
+      "loss": 0.3751,
+      "num_input_tokens_seen": 14278128,
+      "step": 1360
+    },
+    {
+      "epoch": 0.3515323203708473,
+      "grad_norm": 13.528736327050117,
+      "learning_rate": 6.986185761302224e-05,
+      "loss": 0.4814,
+      "num_input_tokens_seen": 14330624,
+      "step": 1365
+    },
+    {
+      "epoch": 0.3528199845480299,
+      "grad_norm": 6.2453361475565305,
+      "learning_rate": 6.963847425753403e-05,
+      "loss": 0.5007,
+      "num_input_tokens_seen": 14382416,
+      "step": 1370
+    },
+    {
+      "epoch": 0.35410764872521244,
+      "grad_norm": 3.5868157849734925,
+      "learning_rate": 6.941462644923318e-05,
+      "loss": 0.4335,
+      "num_input_tokens_seen": 14434896,
+      "step": 1375
+    },
+    {
+      "epoch": 0.35539531290239507,
+      "grad_norm": 7.0930284762784925,
+      "learning_rate": 6.919031948215335e-05,
+      "loss": 0.4427,
+      "num_input_tokens_seen": 14487152,
+      "step": 1380
+    },
+    {
+      "epoch": 0.35668297707957763,
+      "grad_norm": 1.8673746248959853,
+      "learning_rate": 6.896555866118741e-05,
+      "loss": 0.42,
+      "num_input_tokens_seen": 14539608,
+      "step": 1385
+    },
+    {
+      "epoch": 0.35797064125676026,
+      "grad_norm": 3.29378340171418,
+      "learning_rate": 6.87403493019619e-05,
+      "loss": 0.4573,
+      "num_input_tokens_seen": 14592168,
+      "step": 1390
+    },
+    {
+      "epoch": 0.3592583054339428,
+      "grad_norm": 4.710051493913417,
+      "learning_rate": 6.851469673071143e-05,
+      "loss": 0.4341,
+      "num_input_tokens_seen": 14643920,
+      "step": 1395
+    },
+    {
+      "epoch": 0.3605459696111254,
+      "grad_norm": 5.46737560287727,
+      "learning_rate": 6.828860628415253e-05,
+      "loss": 0.437,
+      "num_input_tokens_seen": 14697136,
+      "step": 1400
+    },
+    {
+      "epoch": 0.3605459696111254,
+      "eval_loss": 0.46620962023735046,
+      "eval_runtime": 38.4197,
+      "eval_samples_per_second": 3.123,
+      "eval_steps_per_second": 0.781,
+      "num_input_tokens_seen": 14697136,
+      "step": 1400
     }
   ],
   "logging_steps": 5,
   "max_steps": 3400,
-  "num_input_tokens_seen": 14173240,
+  "num_input_tokens_seen": 14697136,
   "num_train_epochs": 1,
   "save_steps": 50,
   "stateful_callbacks": {
@@ -2429,7 +2518,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 935103008079872.0,
+  "total_flos": 969656994627584.0,
   "train_batch_size": 1,
   "trial_name": null,
   "trial_params": null