Training in progress, step 350, checkpoint

last-checkpoint/adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a4106b45ceccceae95e8b528b101fd891982dc12a562b267303c0a5c15c56e31
+oid sha256:7517520116a90df96714c96a11fa862e42a0dd35dffeff98e7454aeee1e7c1a9
 size 29034840

last-checkpoint/global_step350/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6e63bf19beb302cde525edcedb9b147ea29b65940d2369f8e7c0ea7f6821f078
+size 43429616

last-checkpoint/global_step350/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:20baf4ebf816bea31e633b3ca528715be18827cf26802b0c058c3c3c4292c432
+size 43429616

last-checkpoint/global_step350/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9e53c0e06083477716cb8a8a9e51d57e75f83feb0e7c8230713a628c80936c02
+size 43429616

last-checkpoint/global_step350/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:88fef3cb8d0931a45336b704ef5a8f8a0016263662fce5865f7f980858cc590c
+size 43429616

last-checkpoint/global_step350/zero_pp_rank_0_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:81931d0b9384ae31c6dabf80c0501d96a39631493fc01d9768303da174b201d0
+size 637299

last-checkpoint/global_step350/zero_pp_rank_1_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f499324fd68027eacf299cbbe7bc3fdb8132535402762a6d42a52093afb2e644
+size 637171

last-checkpoint/global_step350/zero_pp_rank_2_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bc84fd8164b463b5d2b20670849f7afe073adacfef5d35bdc708fd108503c2e5
+size 637171

last-checkpoint/global_step350/zero_pp_rank_3_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:af47b1cce71f0f2679a390a0d88f58018746837e2d03a9ea7b758abb44feb7a6
+size 637171

last-checkpoint/latest

@@ -1 +1 @@
-global_step300
+global_step350

last-checkpoint/rng_state_0.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ef002048764051a71fb00f8f978e9ec32b780dc850bdb059af362cc56494234b
+oid sha256:ee97cd82dba4d425fdd8dfdb88d4a43d0d4b1979b5c81ab4a24914fb00d4f332
 size 15024

last-checkpoint/rng_state_1.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:37194a6d48612e1a46a2d5d317ead97c70d9fc4569b0118fcd5f84c3dc9daa5a
+oid sha256:91dad95440fb85dc4a31745642117165c1a72173b2e389679ea8c0b2b6fcd7e2
 size 15024

last-checkpoint/rng_state_2.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:17c179483659a784aa1ace2427daff48c556a6bcc3c330e6f3274e4dc95e4b49
+oid sha256:98698326b023c2af02c94f18726ce52c7f7a6fe290734dd7edbe99bc807fcfa0
 size 15024

last-checkpoint/rng_state_3.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b56857c9b117629f35af2c3d64f522d33a9d8aa94faa81ec6956380a895118c4
+oid sha256:708e7c6b5bf8a327e688779ebc08830ce249928bcb1ff5c82b1b1d0bf6d2660b
 size 15024

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:75979a52de119ed6c9cd6063ded2b7d6cb6c4870db4a05a6cc6a997d1fa9b653
+oid sha256:ff02966f5e9ff1944d3a06acaf420bd5bf127f85a91f6b1b2b1649b32fc1fd58
 size 1064

last-checkpoint/trainer_state.json

@@ -1,9 +1,9 @@
 {
-  "best_metric": 0.7073924541473389,
-  "best_model_checkpoint": "saves/CADICA_qwenvl_stenosis_detect_scale4/lora/sft/checkpoint-300",
-  "epoch": 0.07725985063095545,
+  "best_metric": 0.7044599056243896,
+  "best_model_checkpoint": "saves/CADICA_qwenvl_stenosis_detect_scale4/lora/sft/checkpoint-350",
+  "epoch": 0.09013649240278135,
   "eval_steps": 50,
-  "global_step": 300,
+  "global_step": 350,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -541,11 +541,100 @@
       "eval_steps_per_second": 0.786,
       "num_input_tokens_seen": 3149032,
       "step": 300
+    },
+    {
+      "epoch": 0.07854751480813804,
+      "grad_norm": 0.4479276285203507,
+      "learning_rate": 9.956959413129585e-05,
+      "loss": 0.7208,
+      "num_input_tokens_seen": 3201560,
+      "step": 305
+    },
+    {
+      "epoch": 0.07983517898532062,
+      "grad_norm": 0.368457437106614,
+      "learning_rate": 9.953717199596598e-05,
+      "loss": 0.7144,
+      "num_input_tokens_seen": 3254632,
+      "step": 310
+    },
+    {
+      "epoch": 0.08112284316250322,
+      "grad_norm": 0.5531413254856732,
+      "learning_rate": 9.95035782992122e-05,
+      "loss": 0.6861,
+      "num_input_tokens_seen": 3306432,
+      "step": 315
+    },
+    {
+      "epoch": 0.08241050733968582,
+      "grad_norm": 0.41513991799613037,
+      "learning_rate": 9.94688138355304e-05,
+      "loss": 0.6836,
+      "num_input_tokens_seen": 3358392,
+      "step": 320
+    },
+    {
+      "epoch": 0.0836981715168684,
+      "grad_norm": 0.47052274706452957,
+      "learning_rate": 9.943287942710527e-05,
+      "loss": 0.7353,
+      "num_input_tokens_seen": 3411424,
+      "step": 325
+    },
+    {
+      "epoch": 0.08498583569405099,
+      "grad_norm": 0.6322586593511644,
+      "learning_rate": 9.939577592379088e-05,
+      "loss": 0.6774,
+      "num_input_tokens_seen": 3462992,
+      "step": 330
+    },
+    {
+      "epoch": 0.08627349987123359,
+      "grad_norm": 0.4129597798905344,
+      "learning_rate": 9.935750420309055e-05,
+      "loss": 0.7331,
+      "num_input_tokens_seen": 3516136,
+      "step": 335
+    },
+    {
+      "epoch": 0.08756116404841617,
+      "grad_norm": 0.4031509882699161,
+      "learning_rate": 9.931806517013612e-05,
+      "loss": 0.6939,
+      "num_input_tokens_seen": 3568360,
+      "step": 340
+    },
+    {
+      "epoch": 0.08884882822559877,
+      "grad_norm": 0.4444358747076587,
+      "learning_rate": 9.927745975766654e-05,
+      "loss": 0.7158,
+      "num_input_tokens_seen": 3620696,
+      "step": 345
+    },
+    {
+      "epoch": 0.09013649240278135,
+      "grad_norm": 0.5290547365449167,
+      "learning_rate": 9.923568892600578e-05,
+      "loss": 0.6932,
+      "num_input_tokens_seen": 3673152,
+      "step": 350
+    },
+    {
+      "epoch": 0.09013649240278135,
+      "eval_loss": 0.7044599056243896,
+      "eval_runtime": 38.2709,
+      "eval_samples_per_second": 3.136,
+      "eval_steps_per_second": 0.784,
+      "num_input_tokens_seen": 3673152,
+      "step": 350
     }
   ],
   "logging_steps": 5,
   "max_steps": 3400,
-  "num_input_tokens_seen": 3149032,
+  "num_input_tokens_seen": 3673152,
   "num_train_epochs": 1,
   "save_steps": 50,
   "stateful_callbacks": {
@@ -560,7 +649,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 207694926184448.0,
+  "total_flos": 242291271794688.0,
   "train_batch_size": 1,
   "trial_name": null,
   "trial_params": null