Training in progress, step 1550, checkpoint

last-checkpoint/adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b0263e90dbe014ac70c526fa71d2fd7a1bebb2c4fe954d285088a83f154c10e0
+oid sha256:f1764f403b39c14f233fd2d6443f410cab81d66e3cc9f1d476e6e55a6642ff67
 size 29034840

last-checkpoint/global_step1550/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:692ec3b98c0932e8a11070876eb1fa1d8a671e4eac1d43ece644fabc90eec0ca
+size 43429616

last-checkpoint/global_step1550/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4b61c26876bdfdb016a6b966ea1cfbd5f94dd5ee6d99f15a124b9bfe78ce213f
+size 43429616

last-checkpoint/global_step1550/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:feeff9860ed00084fe5e75ca93276989e3037881afe67c9162b613db6e1c1951
+size 43429616

last-checkpoint/global_step1550/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dd2fb29dda5fe5770a9a3b31c2dddcba2c2c0c1c4abe476dc6b8b1bafde6c8c5
+size 43429616

last-checkpoint/global_step1550/zero_pp_rank_0_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9a43dd54cbbc70405ab57738adb3ec4581722d997b526b922cea67d12d7a3154
+size 637299

last-checkpoint/global_step1550/zero_pp_rank_1_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2160593af1ce08c3f12b9b90e1c3cbef9f94fa69d7b34c91af296952a4c32634
+size 637171

last-checkpoint/global_step1550/zero_pp_rank_2_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a26b6fdb593b50a1684d8681f4281e3d55be3afd1e2403c04c5de137e4315716
+size 637171

last-checkpoint/global_step1550/zero_pp_rank_3_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4062e6de2aea9cf48856d37c0a6068948c993876856dc4d6c5e69e86ba09a28e
+size 637171

last-checkpoint/latest

@@ -1 +1 @@
-global_step1500
+global_step1550

last-checkpoint/rng_state_0.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:dd3566049ad1f65d2f434d990deb65584d2b2dcb1aac8e89c68ea37dc533eab7
+oid sha256:7f1e48a120d69830576f7b582aa6cc46f0ca41d30015a7a674eaec3dcdfc0f09
 size 15024

last-checkpoint/rng_state_1.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2f96a47dc4176412bc893ccb49c004c8fa1cc8c306d67689d87ed20944233c62
+oid sha256:4dbabb9273d3983e52a4a981b5f60f8c2e19da375765d05bb9f2caad284b9652
 size 15024

last-checkpoint/rng_state_2.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ebd9fa3db1079ccf750b71f4eeedbf1f04422fc748026a4b866afe133f9fbfd1
+oid sha256:554ac925bb9c9ea292b7a41caac1cf75285511cf8aa440f37090891ee457a178
 size 15024

last-checkpoint/rng_state_3.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:383e9e252cc8292eef0120c964bdc8033972e800c085c97c42af97379e6b4b5c
+oid sha256:5be5e00123fc0a321e41599b50e07be02f4c165504c601192e5c73f5f5437c30
 size 15024

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7c2260420369b34b5fd5cfacd3f600b6daef0af14020a30634db4ed5d6637b6b
+oid sha256:8eb069683e7f84aa36296476346fc663361d9b05ad7b09b71f22f44afdb0ea48
 size 1064

last-checkpoint/trainer_state.json

@@ -1,9 +1,9 @@
 {
   "best_metric": 0.4339977502822876,
   "best_model_checkpoint": "saves/CADICA_qwenvl_stenosis_detect_scale4/lora/sft/checkpoint-1250",
-  "epoch": 0.38629925315477726,
+  "epoch": 0.39917589492660316,
   "eval_steps": 50,
-  "global_step": 1500,
+  "global_step": 1550,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -2677,11 +2677,100 @@
       "eval_steps_per_second": 0.781,
       "num_input_tokens_seen": 15744848,
       "step": 1500
+    },
+    {
+      "epoch": 0.3875869173319598,
+      "grad_norm": 5.097132399629058,
+      "learning_rate": 6.344911366961934e-05,
+      "loss": 0.4558,
+      "num_input_tokens_seen": 15797632,
+      "step": 1505
+    },
+    {
+      "epoch": 0.3888745815091424,
+      "grad_norm": 4.502325593575991,
+      "learning_rate": 6.321475976211266e-05,
+      "loss": 0.4518,
+      "num_input_tokens_seen": 15850040,
+      "step": 1510
+    },
+    {
+      "epoch": 0.390162245686325,
+      "grad_norm": 6.425152572566654,
+      "learning_rate": 6.298009332358856e-05,
+      "loss": 0.4092,
+      "num_input_tokens_seen": 15902496,
+      "step": 1515
+    },
+    {
+      "epoch": 0.3914499098635076,
+      "grad_norm": 3.968135032555422,
+      "learning_rate": 6.274511990394294e-05,
+      "loss": 0.478,
+      "num_input_tokens_seen": 15954936,
+      "step": 1520
+    },
+    {
+      "epoch": 0.3927375740406902,
+      "grad_norm": 4.636757769906518,
+      "learning_rate": 6.250984506033183e-05,
+      "loss": 0.4294,
+      "num_input_tokens_seen": 16007624,
+      "step": 1525
+    },
+    {
+      "epoch": 0.3940252382178728,
+      "grad_norm": 2.7967900169696347,
+      "learning_rate": 6.227427435703997e-05,
+      "loss": 0.3846,
+      "num_input_tokens_seen": 16059440,
+      "step": 1530
+    },
+    {
+      "epoch": 0.39531290239505534,
+      "grad_norm": 2.983520749639549,
+      "learning_rate": 6.203841336534924e-05,
+      "loss": 0.4372,
+      "num_input_tokens_seen": 16111136,
+      "step": 1535
+    },
+    {
+      "epoch": 0.39660056657223797,
+      "grad_norm": 8.364510466670477,
+      "learning_rate": 6.180226766340688e-05,
+      "loss": 0.484,
+      "num_input_tokens_seen": 16163976,
+      "step": 1540
+    },
+    {
+      "epoch": 0.39788823074942054,
+      "grad_norm": 4.45878743373729,
+      "learning_rate": 6.156584283609359e-05,
+      "loss": 0.3965,
+      "num_input_tokens_seen": 16217192,
+      "step": 1545
+    },
+    {
+      "epoch": 0.39917589492660316,
+      "grad_norm": 2.6831990995391717,
+      "learning_rate": 6.132914447489137e-05,
+      "loss": 0.3872,
+      "num_input_tokens_seen": 16269896,
+      "step": 1550
+    },
+    {
+      "epoch": 0.39917589492660316,
+      "eval_loss": 0.4416767656803131,
+      "eval_runtime": 38.4671,
+      "eval_samples_per_second": 3.12,
+      "eval_steps_per_second": 0.78,
+      "num_input_tokens_seen": 16269896,
+      "step": 1550
     }
   ],
   "logging_steps": 5,
   "max_steps": 3400,
-  "num_input_tokens_seen": 15744848,
+  "num_input_tokens_seen": 16269896,
   "num_train_epochs": 1,
   "save_steps": 50,
   "stateful_callbacks": {
@@ -2696,7 +2785,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 1038788106387456.0,
+  "total_flos": 1073418423500800.0,
   "train_batch_size": 1,
   "trial_name": null,
   "trial_params": null