Training in progress, step 500, checkpoint

last-checkpoint/adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d705dd99814267821cd8d09503bcdb39775b52c9a5778b863885f85c749a48a4
+oid sha256:87c5c5bb13b2f9891961a1af048bd7469b196705458eff48651ae470ca78403c
 size 29034840

last-checkpoint/global_step500/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c3d611797fddaab35aecccce717e36f16bef903dc6991f1c12d4621f8edc39b1
+size 43429616

last-checkpoint/global_step500/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4b342fc4e308c158a39aa2386ee028dc276f33f77de37daa0c394d8a7958dc8a
+size 43429616

last-checkpoint/global_step500/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:27508e9cf51921a7343015a5254d03d5b55075372c8463336050fe1d528785f0
+size 43429616

last-checkpoint/global_step500/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:de0e0ed46deafb3abf780a43718b649e384f7b72bc5468aac237adab81bebaa8
+size 43429616

last-checkpoint/global_step500/zero_pp_rank_0_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:982b6b8112b1a68d2e38cae94a225f18bd9ea392e1654dbc7cc130776bff950b
+size 637299

last-checkpoint/global_step500/zero_pp_rank_1_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:854e2832a5dae385b495dad0465c766c553cf722b470925f5286a7230114f652
+size 637171

last-checkpoint/global_step500/zero_pp_rank_2_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:da17084af120106739c69dcfe393381f051b6adbcd329aae7c5a39f5b06e3345
+size 637171

last-checkpoint/global_step500/zero_pp_rank_3_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b7066b428556bc0b3f73235845964ecb2a136bcb07c44570c346b66a0f2f5992
+size 637171

last-checkpoint/latest

@@ -1 +1 @@
-global_step450
+global_step500

last-checkpoint/rng_state_0.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7dbc6521b0b64cb12d818506108fcf257a4089ca8a9b1e453776ed3e032e7176
+oid sha256:bdd1f02cb20d3f4f7e0dd26fea62af57e5e71316163f926a28ed6cf89a9f3777
 size 15024

last-checkpoint/rng_state_1.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2b13e3da1b0679cab1bab94f893e385a9a224d3335b5a6f62602f33c2be88d03
+oid sha256:fc6d54ba2aa85e2f895439a1b787ec947b848a1c34ea5a3a28821572bf2b9fec
 size 15024

last-checkpoint/rng_state_2.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6a24f0e0f117b5a8236e0d12594c0c358f41ef00068d4460002e95ad1cc3cb1c
+oid sha256:8b6927d26551cddd8e35b34b43e79bd58f8b6027b6a481bb6a563a3652addeb4
 size 15024

last-checkpoint/rng_state_3.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e46e4eab6c4a25d84ad36ddf1357401788adeeb6388c03cefa35a63b52ee7610
+oid sha256:f8295b1be8e66b4b30cb905dc48cfc717c027e427937b8142d00ae9de8106c6a
 size 15024

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:192829f095e6c906a86011515c9b1d243ad2d8f891793c9e500d2d433b84fc78
+oid sha256:ab7567aeded1bd7ef9f3ba115e57865dd25bb569f9711f33170eb2a51540c216
 size 1064

last-checkpoint/trainer_state.json

@@ -1,9 +1,9 @@
 {
-  "best_metric": 0.6526497006416321,
-  "best_model_checkpoint": "saves/CADICA_qwenvl_stenosis_detect_scale4/lora/sft/checkpoint-450",
-  "epoch": 0.11588977594643317,
+  "best_metric": 0.6272165775299072,
+  "best_model_checkpoint": "saves/CADICA_qwenvl_stenosis_detect_scale4/lora/sft/checkpoint-500",
+  "epoch": 0.12876641771825909,
   "eval_steps": 50,
-  "global_step": 450,
+  "global_step": 500,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -808,11 +808,100 @@
       "eval_steps_per_second": 0.768,
       "num_input_tokens_seen": 4722528,
       "step": 450
+    },
+    {
+      "epoch": 0.11717744012361576,
+      "grad_norm": 2.1823349329218074,
+      "learning_rate": 9.809128215864097e-05,
+      "loss": 0.6544,
+      "num_input_tokens_seen": 4774400,
+      "step": 455
+    },
+    {
+      "epoch": 0.11846510430079835,
+      "grad_norm": 1.434521593914191,
+      "learning_rate": 9.802417057704931e-05,
+      "loss": 0.652,
+      "num_input_tokens_seen": 4826704,
+      "step": 460
+    },
+    {
+      "epoch": 0.11975276847798094,
+      "grad_norm": 2.399754385687283,
+      "learning_rate": 9.795592321674045e-05,
+      "loss": 0.6582,
+      "num_input_tokens_seen": 4880072,
+      "step": 465
+    },
+    {
+      "epoch": 0.12104043265516354,
+      "grad_norm": 3.9235176077985536,
+      "learning_rate": 9.788654169177453e-05,
+      "loss": 0.6506,
+      "num_input_tokens_seen": 4931968,
+      "step": 470
+    },
+    {
+      "epoch": 0.12232809683234612,
+      "grad_norm": 3.659330745777227,
+      "learning_rate": 9.781602764303487e-05,
+      "loss": 0.6551,
+      "num_input_tokens_seen": 4983656,
+      "step": 475
+    },
+    {
+      "epoch": 0.12361576100952872,
+      "grad_norm": 1.9670601503398757,
+      "learning_rate": 9.774438273818911e-05,
+      "loss": 0.6978,
+      "num_input_tokens_seen": 5036528,
+      "step": 480
+    },
+    {
+      "epoch": 0.12490342518671131,
+      "grad_norm": 1.308580869419328,
+      "learning_rate": 9.767160867164979e-05,
+      "loss": 0.6407,
+      "num_input_tokens_seen": 5088768,
+      "step": 485
+    },
+    {
+      "epoch": 0.1261910893638939,
+      "grad_norm": 1.7349486072682865,
+      "learning_rate": 9.759770716453436e-05,
+      "loss": 0.6641,
+      "num_input_tokens_seen": 5142080,
+      "step": 490
+    },
+    {
+      "epoch": 0.1274787535410765,
+      "grad_norm": 2.993327939872198,
+      "learning_rate": 9.752267996462434e-05,
+      "loss": 0.6588,
+      "num_input_tokens_seen": 5194432,
+      "step": 495
+    },
+    {
+      "epoch": 0.12876641771825909,
+      "grad_norm": 2.6430988002320976,
+      "learning_rate": 9.744652884632406e-05,
+      "loss": 0.6304,
+      "num_input_tokens_seen": 5246640,
+      "step": 500
+    },
+    {
+      "epoch": 0.12876641771825909,
+      "eval_loss": 0.6272165775299072,
+      "eval_runtime": 39.4177,
+      "eval_samples_per_second": 3.044,
+      "eval_steps_per_second": 0.761,
+      "num_input_tokens_seen": 5246640,
+      "step": 500
     }
   ],
   "logging_steps": 5,
   "max_steps": 3400,
-  "num_input_tokens_seen": 4722528,
+  "num_input_tokens_seen": 5246640,
   "num_train_epochs": 1,
   "save_steps": 50,
   "stateful_callbacks": {
@@ -827,7 +916,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 311514461896704.0,
+  "total_flos": 346057693724672.0,
   "train_batch_size": 1,
   "trial_name": null,
   "trial_params": null