Training in progress, step 2400, checkpoint

last-checkpoint/adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8464937776286fdba4fee3aa59e0e0cefe29cc3b83812b6842ff0a56228048f2
+oid sha256:2874dfb474adef2e86193b1ab18660860287a5e9abf08e0b2d06a88e2eefcb7a
 size 29034840

last-checkpoint/global_step2400/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c9317d695d9f53cd93973509664a22e899f24b338919af23a6bf6915df7d3a35
+size 43429616

last-checkpoint/global_step2400/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1eaa3db7e6d09a6adb220216d1b11d80f65308138095cae32a7906d8071e772d
+size 43429616

last-checkpoint/global_step2400/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d0998d8084f4151b2c8ead5dcaf9e1293b892ae5907f8819b94aa41a84519964
+size 43429616

last-checkpoint/global_step2400/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5bee44e2d0ba29219b9a7e0f0192ec3f7c2375000da164d8a81a247f9a00b087
+size 43429616

last-checkpoint/global_step2400/zero_pp_rank_0_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bb28cb80645e4e5b08d929feb6cb284b80b9374d0abb5cac7a034e04e045bf67
+size 637299

last-checkpoint/global_step2400/zero_pp_rank_1_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8e18a2b0b8ad83bd9cd5698849f6aafc46f23435824e7200c2e85a9eb4e492a1
+size 637171

last-checkpoint/global_step2400/zero_pp_rank_2_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cc63c2624e3a26f46cb4e76250e1daf40e4de23f6d0694e319d1c3d09e6246cc
+size 637171

last-checkpoint/global_step2400/zero_pp_rank_3_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a9469a2da3dcd4ac7538c85ff2efc3e7edb38b7bb9ddacc4ebbcca32aabfa5fe
+size 637171

last-checkpoint/latest

@@ -1 +1 @@
-global_step2350
+global_step2400

last-checkpoint/rng_state_0.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f12bf3da75454e5aae4644f2a1d46fdf90f68e680dbf5bdaa86861f825d32d80
+oid sha256:681faefe4cf303ca7f9bc3073b09b166da4f558d55bee3d5eee90ba5d83159bb
 size 15024

last-checkpoint/rng_state_1.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2faef1b8798e7516fd96ee7b3363866a8f97ca2d0ec5a8dd27bbfe70b0c6a733
+oid sha256:c0fb250f1d137fb55cef85743a342508178b4fe3a20c6793c82e279730ea280b
 size 15024

last-checkpoint/rng_state_2.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7f35f6d27fbd414dd4285d91816b37fb6b97ff10fbba4c074d56ad2a7f723033
+oid sha256:34a0d7cfaa34f7e3738b4ef4989d693ed7864fed3b2a44ef1b6892fdcf026bb9
 size 15024

last-checkpoint/rng_state_3.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ea5c64dfc7e3b2729065483dfef8e4bb0af0d9bae32df888d258ee3c2859d676
+oid sha256:2a82aabe23bc62e289ef7d075c79f353bbc81286ec0f8964eabda4209d630e10
 size 15024

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d71a7ba4d3f18f2d8b75b4d8773a5d2a13dce7aa34d7060fa3e4c5c241599a29
+oid sha256:33f198cd3e0b21f350098b16849fc1ced7c1d5261e89a1c503f4a0d3ce688f30
 size 1064

last-checkpoint/trainer_state.json

@@ -1,9 +1,9 @@
 {
   "best_metric": 0.4145541489124298,
   "best_model_checkpoint": "saves/CADICA_qwenvl_stenosis_detect_scale4/lora/sft/checkpoint-2350",
-  "epoch": 0.6052021632758177,
+  "epoch": 0.6180788050476436,
   "eval_steps": 50,
-  "global_step": 2350,
+  "global_step": 2400,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -4190,11 +4190,100 @@
       "eval_steps_per_second": 0.786,
       "num_input_tokens_seen": 24678768,
       "step": 2350
+    },
+    {
+      "epoch": 0.6064898274530003,
+      "grad_norm": 2.7740413001573057,
+      "learning_rate": 2.3678391856132204e-05,
+      "loss": 0.3229,
+      "num_input_tokens_seen": 24730528,
+      "step": 2355
+    },
+    {
+      "epoch": 0.6077774916301828,
+      "grad_norm": 6.766132750655314,
+      "learning_rate": 2.3471967400628513e-05,
+      "loss": 0.3308,
+      "num_input_tokens_seen": 24784472,
+      "step": 2360
+    },
+    {
+      "epoch": 0.6090651558073654,
+      "grad_norm": 6.8653733639649515,
+      "learning_rate": 2.3266170337008398e-05,
+      "loss": 0.356,
+      "num_input_tokens_seen": 24838168,
+      "step": 2365
+    },
+    {
+      "epoch": 0.610352819984548,
+      "grad_norm": 2.3990294356922615,
+      "learning_rate": 2.306100553240274e-05,
+      "loss": 0.2784,
+      "num_input_tokens_seen": 24890552,
+      "step": 2370
+    },
+    {
+      "epoch": 0.6116404841617307,
+      "grad_norm": 2.929256632803373,
+      "learning_rate": 2.2856477838989456e-05,
+      "loss": 0.2859,
+      "num_input_tokens_seen": 24942904,
+      "step": 2375
+    },
+    {
+      "epoch": 0.6129281483389132,
+      "grad_norm": 3.932586185965905,
+      "learning_rate": 2.2652592093878666e-05,
+      "loss": 0.3107,
+      "num_input_tokens_seen": 24995776,
+      "step": 2380
+    },
+    {
+      "epoch": 0.6142158125160958,
+      "grad_norm": 3.999914127947348,
+      "learning_rate": 2.244935311899829e-05,
+      "loss": 0.3131,
+      "num_input_tokens_seen": 25047848,
+      "step": 2385
+    },
+    {
+      "epoch": 0.6155034766932784,
+      "grad_norm": 3.803358403729212,
+      "learning_rate": 2.224676572098007e-05,
+      "loss": 0.3175,
+      "num_input_tokens_seen": 25100896,
+      "step": 2390
+    },
+    {
+      "epoch": 0.6167911408704609,
+      "grad_norm": 10.600664919848047,
+      "learning_rate": 2.2044834691045873e-05,
+      "loss": 0.3482,
+      "num_input_tokens_seen": 25153912,
+      "step": 2395
+    },
+    {
+      "epoch": 0.6180788050476436,
+      "grad_norm": 5.122783317200166,
+      "learning_rate": 2.184356480489432e-05,
+      "loss": 0.3445,
+      "num_input_tokens_seen": 25206168,
+      "step": 2400
+    },
+    {
+      "epoch": 0.6180788050476436,
+      "eval_loss": 0.42807063460350037,
+      "eval_runtime": 38.2326,
+      "eval_samples_per_second": 3.139,
+      "eval_steps_per_second": 0.785,
+      "num_input_tokens_seen": 25206168,
+      "step": 2400
     }
   ],
   "logging_steps": 5,
   "max_steps": 3400,
-  "num_input_tokens_seen": 24678768,
+  "num_input_tokens_seen": 25206168,
   "num_train_epochs": 1,
   "save_steps": 50,
   "stateful_callbacks": {
@@ -4209,7 +4298,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 1628169367453696.0,
+  "total_flos": 1663008045268992.0,
   "train_batch_size": 1,
   "trial_name": null,
   "trial_params": null