Training in progress, step 2150, checkpoint

last-checkpoint/adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7f75c51d9ca969a9d3cbb26206d8c3751152111b430ee8f78c774b9d99426948
+oid sha256:023a38accdab689b6f8ae25c0137c8832b5274377f50198dd88dfa1030f6f9c7
 size 29034840

last-checkpoint/global_step2150/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d40d3bb1f6583d95896af685dc5228c8bd83fd49b36fbf1c17e0c3fc49e62e1f
+size 43429616

last-checkpoint/global_step2150/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2fd2c00c48889adf4e5546f739a5b0b524cf9df71d1a3435ca52b98ff0dd69da
+size 43429616

last-checkpoint/global_step2150/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e33a6bea860860e919efb1298c892f524ceb7ce017c75d7a1053df2b9e4f6968
+size 43429616

last-checkpoint/global_step2150/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:05f5afcc420acf2bc32c65c32c8cc37f106c4f89935f5e4c445a926a1f9de435
+size 43429616

last-checkpoint/global_step2150/zero_pp_rank_0_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c5a104173fae309cddd5cec659a0d42963dd55180042b7dffd3c0a9f5330d16c
+size 637299

last-checkpoint/global_step2150/zero_pp_rank_1_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c382bb29b9b9b93ee74ce79ebdc7427abdd4946c08810e6ab54148107b238730
+size 637171

last-checkpoint/global_step2150/zero_pp_rank_2_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1d55d5f44769829dccf11857c354f886ec28168ac4145aad5146200fcbf91acf
+size 637171

last-checkpoint/global_step2150/zero_pp_rank_3_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5959cc89357511b968c257c43a7386704691106ffbfe1db32582fe21f5f1e6b3
+size 637171

last-checkpoint/latest

@@ -1 +1 @@
-global_step2100
+global_step2150

last-checkpoint/rng_state_0.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5d1ddd7e7b4dc44903837b0414e4659f8383cd8f16b41dd396d4eaf5b9829f79
+oid sha256:b105708e2c99c8661b46698b8ccc5799ac83c1f0fc6a30c2d41c9fbfb349d480
 size 15024

last-checkpoint/rng_state_1.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:eeddfb77fe4d3b495c4e08307767e08df90e96ef241c3eb80d5f75adec393e80
+oid sha256:bc44cd2015d8c8fc2f109f07c797876873a52f478c57b0350b8a2cf5dcb17f25
 size 15024

last-checkpoint/rng_state_2.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9e6260196fabb00061b1f1c8de6288382570dc14d02d2aa308050ca858880a97
+oid sha256:9843dec201b5a542ebd69abfc596f99ad5a000cf81dab52c6a2c52a5b9224ea7
 size 15024

last-checkpoint/rng_state_3.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ea023bc5b1def54e0c49389175c0fae812f5f764c502525ce775d993d5ab2c03
+oid sha256:2296f0efda653dd4c7e861f5a867baa09d6d8bb50e57bc69af930268b40de9ef
 size 15024

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:41ca6ff868655333e43bb2ded3dfe8c66b7ed82fa8a98bace2e730c42bb9959b
+oid sha256:ca5770eed90150126cfd6d4c180a03d91fe366663610f38ec72b635b0b8cfd11
 size 1064

last-checkpoint/trainer_state.json

@@ -1,9 +1,9 @@
 {
   "best_metric": 0.4194311797618866,
   "best_model_checkpoint": "saves/CADICA_qwenvl_stenosis_detect_scale4/lora/sft/checkpoint-1800",
-  "epoch": 0.5408189544166881,
+  "epoch": 0.553695596188514,
   "eval_steps": 50,
-  "global_step": 2100,
+  "global_step": 2150,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -3745,11 +3745,100 @@
       "eval_steps_per_second": 0.79,
       "num_input_tokens_seen": 22050376,
       "step": 2100
+    },
+    {
+      "epoch": 0.5421066185938708,
+      "grad_norm": 4.2067016944481965,
+      "learning_rate": 3.4687969674819906e-05,
+      "loss": 0.4067,
+      "num_input_tokens_seen": 22102848,
+      "step": 2105
+    },
+    {
+      "epoch": 0.5433942827710533,
+      "grad_norm": 3.273955419211119,
+      "learning_rate": 3.445667702902969e-05,
+      "loss": 0.36,
+      "num_input_tokens_seen": 22155432,
+      "step": 2110
+    },
+    {
+      "epoch": 0.5446819469482359,
+      "grad_norm": 3.815876908682745,
+      "learning_rate": 3.4225751985120215e-05,
+      "loss": 0.3569,
+      "num_input_tokens_seen": 22207528,
+      "step": 2115
+    },
+    {
+      "epoch": 0.5459696111254185,
+      "grad_norm": 3.5864972836865845,
+      "learning_rate": 3.3995200004502816e-05,
+      "loss": 0.3503,
+      "num_input_tokens_seen": 22260016,
+      "step": 2120
+    },
+    {
+      "epoch": 0.547257275302601,
+      "grad_norm": 10.259154585756033,
+      "learning_rate": 3.3765026539765834e-05,
+      "loss": 0.342,
+      "num_input_tokens_seen": 22312616,
+      "step": 2125
+    },
+    {
+      "epoch": 0.5485449394797837,
+      "grad_norm": 10.27121418750564,
+      "learning_rate": 3.3535237034545675e-05,
+      "loss": 0.4113,
+      "num_input_tokens_seen": 22364776,
+      "step": 2130
+    },
+    {
+      "epoch": 0.5498326036569663,
+      "grad_norm": 7.798195914668443,
+      "learning_rate": 3.330583692339802e-05,
+      "loss": 0.311,
+      "num_input_tokens_seen": 22416944,
+      "step": 2135
+    },
+    {
+      "epoch": 0.5511202678341488,
+      "grad_norm": 4.484699705421769,
+      "learning_rate": 3.307683163166934e-05,
+      "loss": 0.358,
+      "num_input_tokens_seen": 22468960,
+      "step": 2140
+    },
+    {
+      "epoch": 0.5524079320113314,
+      "grad_norm": 4.1926914172544665,
+      "learning_rate": 3.284822657536856e-05,
+      "loss": 0.378,
+      "num_input_tokens_seen": 22521624,
+      "step": 2145
+    },
+    {
+      "epoch": 0.553695596188514,
+      "grad_norm": 4.378365479925035,
+      "learning_rate": 3.262002716103897e-05,
+      "loss": 0.3687,
+      "num_input_tokens_seen": 22574104,
+      "step": 2150
+    },
+    {
+      "epoch": 0.553695596188514,
+      "eval_loss": 0.4504742920398712,
+      "eval_runtime": 38.0971,
+      "eval_samples_per_second": 3.15,
+      "eval_steps_per_second": 0.787,
+      "num_input_tokens_seen": 22574104,
+      "step": 2150
     }
   ],
   "logging_steps": 5,
   "max_steps": 3400,
-  "num_input_tokens_seen": 22050376,
+  "num_input_tokens_seen": 22574104,
   "num_train_epochs": 1,
   "save_steps": 50,
   "stateful_callbacks": {
@@ -3764,7 +3853,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 1454769949900800.0,
+  "total_flos": 1489272114511872.0,
   "train_batch_size": 1,
   "trial_name": null,
   "trial_params": null