Training in progress, step 3150, checkpoint

last-checkpoint/adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:006fbddb28ccd3f24fce77178564cf32b9a0d6002aa0e149d2a632f6bc384cad
+oid sha256:d53665e5e9c8528df47b7ef76a4152d56ed4d96010a44cb39a4f16dca91d64bb
 size 29034840

last-checkpoint/global_step3150/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b2e0fb046093a81ad8e43afc7a08b41d46414aa1e33cf97ec353ec7163a1bb5f
+size 43429616

last-checkpoint/global_step3150/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fe875ee161babcf03d0770eb26eb263041dadae0cd2cbd75dacff805321898f4
+size 43429616

last-checkpoint/global_step3150/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:71b97fddeea106f58b8d7fffd5293f8bf4b53641c0c710ff7c3dd2cc88f6bf73
+size 43429616

last-checkpoint/global_step3150/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0148271fb37af87f1bee086b9aff46e286b003549b2331daa512affb62c41b9a
+size 43429616

last-checkpoint/global_step3150/zero_pp_rank_0_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:281a26e358d9b17c197c48c2ae0a968e2da59326496f249cb7f83e8232ef30a0
+size 637299

last-checkpoint/global_step3150/zero_pp_rank_1_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:65df32237bfebf0fa592beec8e1e4c267976dbf7907532a46eb21a9345b10fac
+size 637171

last-checkpoint/global_step3150/zero_pp_rank_2_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:039e6c13e3b222cdbb597c188657b75ea26ce47df30b29d835301e61700e88b1
+size 637171

last-checkpoint/global_step3150/zero_pp_rank_3_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a0a9da893289b58af4423adf1a183e2d4cf555b333a8fc3cb9ac838f1b86a164
+size 637171

last-checkpoint/latest

@@ -1 +1 @@
-global_step3100
+global_step3150

last-checkpoint/rng_state_0.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:381f090b12cbb3fcce976bd2e72d07a7786154c2848cc881d75715648e7c4dc2
+oid sha256:f819cf2c7fe8719d427eaa6ec9775983252517a512a5eb46040621049bb676c6
 size 15024

last-checkpoint/rng_state_1.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4321dc1beb6c245d4a16a8fb2bed2ce2a40d89e5a9c611c4572f63a09523846c
+oid sha256:2a78180f5e29cd98437c35591ab402d42bd98c8a12a813c23dfdfe59b5a850b5
 size 15024

last-checkpoint/rng_state_2.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5b79020117e301cd96518c9d1e3eb43a609bc85799f14bac63d9c572fc04cb89
+oid sha256:5f43520b5104ee75a59749a66f5367557a7f191243b389309b6acad668af81d2
 size 15024

last-checkpoint/rng_state_3.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5429623c7323b4a820ea3d76194bc456c2affeeb1f3af8978aec5aee11b2d1ef
+oid sha256:2c502c4157180ff49243cd3eeb13c063be673f3bf094a73b62e8315ca6f65442
 size 15024

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d44cc93262eaa36be09e5c05cb4a41bcbfdd9c2c86797c6c680d23dd8657e6ca
+oid sha256:7d8ecaac6ed58c567c8bab5c2469f31d2d7dbb24618ff65b711fa863465976b7
 size 1064

last-checkpoint/trainer_state.json

@@ -1,9 +1,9 @@
 {
   "best_metric": 0.4145541489124298,
   "best_model_checkpoint": "saves/CADICA_qwenvl_stenosis_detect_scale4/lora/sft/checkpoint-2350",
-  "epoch": 0.7983517898532063,
+  "epoch": 0.8112284316250322,
   "eval_steps": 50,
-  "global_step": 3100,
+  "global_step": 3150,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -5525,11 +5525,100 @@
       "eval_steps_per_second": 0.799,
       "num_input_tokens_seen": 32553288,
       "step": 3100
+    },
+    {
+      "epoch": 0.7996394540303888,
+      "grad_norm": 8.05961476917595,
+      "learning_rate": 2.0440767832595574e-06,
+      "loss": 0.3052,
+      "num_input_tokens_seen": 32606096,
+      "step": 3105
+    },
+    {
+      "epoch": 0.8009271182075715,
+      "grad_norm": 3.1428977500375326,
+      "learning_rate": 1.975829422950709e-06,
+      "loss": 0.2125,
+      "num_input_tokens_seen": 32659376,
+      "step": 3110
+    },
+    {
+      "epoch": 0.8022147823847541,
+      "grad_norm": 2.8855336999591295,
+      "learning_rate": 1.908717841359048e-06,
+      "loss": 0.3122,
+      "num_input_tokens_seen": 32712168,
+      "step": 3115
+    },
+    {
+      "epoch": 0.8035024465619367,
+      "grad_norm": 12.674047700213576,
+      "learning_rate": 1.8427436256833852e-06,
+      "loss": 0.3006,
+      "num_input_tokens_seen": 32764296,
+      "step": 3120
+    },
+    {
+      "epoch": 0.8047901107391192,
+      "grad_norm": 1.5292819995856641,
+      "learning_rate": 1.7779083362236547e-06,
+      "loss": 0.3077,
+      "num_input_tokens_seen": 32815296,
+      "step": 3125
+    },
+    {
+      "epoch": 0.8060777749163018,
+      "grad_norm": 12.068461011216378,
+      "learning_rate": 1.7142135063440035e-06,
+      "loss": 0.29,
+      "num_input_tokens_seen": 32867288,
+      "step": 3130
+    },
+    {
+      "epoch": 0.8073654390934845,
+      "grad_norm": 9.37062799812982,
+      "learning_rate": 1.6516606424365643e-06,
+      "loss": 0.3574,
+      "num_input_tokens_seen": 32919584,
+      "step": 3135
+    },
+    {
+      "epoch": 0.808653103270667,
+      "grad_norm": 5.777474878278418,
+      "learning_rate": 1.5902512238857858e-06,
+      "loss": 0.2414,
+      "num_input_tokens_seen": 32972736,
+      "step": 3140
+    },
+    {
+      "epoch": 0.8099407674478496,
+      "grad_norm": 3.0096174763729864,
+      "learning_rate": 1.5299867030334814e-06,
+      "loss": 0.2521,
+      "num_input_tokens_seen": 33026320,
+      "step": 3145
+    },
+    {
+      "epoch": 0.8112284316250322,
+      "grad_norm": 5.573236169553209,
+      "learning_rate": 1.4708685051444515e-06,
+      "loss": 0.2669,
+      "num_input_tokens_seen": 33078960,
+      "step": 3150
+    },
+    {
+      "epoch": 0.8112284316250322,
+      "eval_loss": 0.4687062203884125,
+      "eval_runtime": 37.4391,
+      "eval_samples_per_second": 3.205,
+      "eval_steps_per_second": 0.801,
+      "num_input_tokens_seen": 33078960,
+      "step": 3150
     }
   ],
   "logging_steps": 5,
   "max_steps": 3400,
-  "num_input_tokens_seen": 32553288,
+  "num_input_tokens_seen": 33078960,
   "num_train_epochs": 1,
   "save_steps": 50,
   "stateful_callbacks": {
@@ -5544,7 +5633,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 2147751457718272.0,
+  "total_flos": 2182476112592896.0,
   "train_batch_size": 1,
   "trial_name": null,
   "trial_params": null