Training in progress, step 2450, checkpoint

last-checkpoint/adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2874dfb474adef2e86193b1ab18660860287a5e9abf08e0b2d06a88e2eefcb7a
+oid sha256:76ea3377531bb34d473bfe1ac7fb7435074a39aec464fce193b5f47faf92de11
 size 29034840

last-checkpoint/global_step2450/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:86c991cb2b0c6fce97f4086578fe94d5c23c70f076fb52a48baa7b11bb09d498
+size 43429616

last-checkpoint/global_step2450/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3984a1bec2c2e9e3da8d70da197414e9b2122f690c41781d08adee936459d118
+size 43429616

last-checkpoint/global_step2450/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:405fb66f00ad893a4539afd425a0d8c524d25fea660dbac42f31168c44fe86e2
+size 43429616

last-checkpoint/global_step2450/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:17fd8d9f5eaf834a0f5c3b85d6f12abeaa7aca49b42993cb71a3233cec52c024
+size 43429616

last-checkpoint/global_step2450/zero_pp_rank_0_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e7bb58eb71a4b6e22de5fbcac0e3524600ccd3b9e748afdf98ae5444c4762c21
+size 637299

last-checkpoint/global_step2450/zero_pp_rank_1_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c1b730284eeffe8a55bc9b46bf9d83671b158f9632e1a667dce54c8c675dc34c
+size 637171

last-checkpoint/global_step2450/zero_pp_rank_2_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c3d5d72e19d9b42ecf282357f58153c490d57412b040b45b0c8c2357329e8de7
+size 637171

last-checkpoint/global_step2450/zero_pp_rank_3_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a2a68efce44c4f1c6e156c0a70c7e22e0768504f6d042427b7d15db49da0f9fb
+size 637171

last-checkpoint/latest

@@ -1 +1 @@
-global_step2400
+global_step2450

last-checkpoint/rng_state_0.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:681faefe4cf303ca7f9bc3073b09b166da4f558d55bee3d5eee90ba5d83159bb
+oid sha256:6b8aff7a1897a7eaf48c78ea1f8115c061edfa2b6fa42280e2c1c58fe66b1f8a
 size 15024

last-checkpoint/rng_state_1.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c0fb250f1d137fb55cef85743a342508178b4fe3a20c6793c82e279730ea280b
+oid sha256:1d73a7524f07999ef35d5d9b107dcc1678eae2ada841644e1bd00ec0734368c2
 size 15024

last-checkpoint/rng_state_2.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:34a0d7cfaa34f7e3738b4ef4989d693ed7864fed3b2a44ef1b6892fdcf026bb9
+oid sha256:149a2ed30e88bf94d622f8d7693f382286a49ac536a3f63efc50cab63f6b9f39
 size 15024

last-checkpoint/rng_state_3.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2a82aabe23bc62e289ef7d075c79f353bbc81286ec0f8964eabda4209d630e10
+oid sha256:18f12c1b5aae2b7d4bb968649839fc7ff1ce6131508baad4b633693b04cee910
 size 15024

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:33f198cd3e0b21f350098b16849fc1ced7c1d5261e89a1c503f4a0d3ce688f30
+oid sha256:bf7df6dd05dd94a7aa9222f36c7d07fc54cb5fa02909dfb44c3de4cef2e774f9
 size 1064

last-checkpoint/trainer_state.json

@@ -1,9 +1,9 @@
 {
   "best_metric": 0.4145541489124298,
   "best_model_checkpoint": "saves/CADICA_qwenvl_stenosis_detect_scale4/lora/sft/checkpoint-2350",
-  "epoch": 0.6180788050476436,
+  "epoch": 0.6309554468194695,
   "eval_steps": 50,
-  "global_step": 2400,
+  "global_step": 2450,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -4279,11 +4279,100 @@
       "eval_steps_per_second": 0.785,
       "num_input_tokens_seen": 25206168,
       "step": 2400
+    },
+    {
+      "epoch": 0.6193664692248262,
+      "grad_norm": 9.751299586219911,
+      "learning_rate": 2.1642960822587878e-05,
+      "loss": 0.3147,
+      "num_input_tokens_seen": 25258880,
+      "step": 2405
+    },
+    {
+      "epoch": 0.6206541334020087,
+      "grad_norm": 1.3518511785939038,
+      "learning_rate": 2.1443027488440338e-05,
+      "loss": 0.3467,
+      "num_input_tokens_seen": 25310976,
+      "step": 2410
+    },
+    {
+      "epoch": 0.6219417975791913,
+      "grad_norm": 4.735151631850019,
+      "learning_rate": 2.124376953090456e-05,
+      "loss": 0.3085,
+      "num_input_tokens_seen": 25363520,
+      "step": 2415
+    },
+    {
+      "epoch": 0.623229461756374,
+      "grad_norm": 6.109775613794598,
+      "learning_rate": 2.104519166246059e-05,
+      "loss": 0.3376,
+      "num_input_tokens_seen": 25415400,
+      "step": 2420
+    },
+    {
+      "epoch": 0.6245171259335566,
+      "grad_norm": 1.896185856776787,
+      "learning_rate": 2.0847298579504344e-05,
+      "loss": 0.3312,
+      "num_input_tokens_seen": 25468296,
+      "step": 2425
+    },
+    {
+      "epoch": 0.6258047901107391,
+      "grad_norm": 3.9069124431889932,
+      "learning_rate": 2.065009496223638e-05,
+      "loss": 0.3282,
+      "num_input_tokens_seen": 25520816,
+      "step": 2430
+    },
+    {
+      "epoch": 0.6270924542879217,
+      "grad_norm": 4.998056844440976,
+      "learning_rate": 2.045358547455138e-05,
+      "loss": 0.321,
+      "num_input_tokens_seen": 25573416,
+      "step": 2435
+    },
+    {
+      "epoch": 0.6283801184651043,
+      "grad_norm": 4.15352407282998,
+      "learning_rate": 2.0257774763927655e-05,
+      "loss": 0.33,
+      "num_input_tokens_seen": 25626536,
+      "step": 2440
+    },
+    {
+      "epoch": 0.6296677826422868,
+      "grad_norm": 7.962840580433044,
+      "learning_rate": 2.0062667461317426e-05,
+      "loss": 0.2833,
+      "num_input_tokens_seen": 25679208,
+      "step": 2445
+    },
+    {
+      "epoch": 0.6309554468194695,
+      "grad_norm": 3.645397800601146,
+      "learning_rate": 1.9868268181037185e-05,
+      "loss": 0.3413,
+      "num_input_tokens_seen": 25730432,
+      "step": 2450
+    },
+    {
+      "epoch": 0.6309554468194695,
+      "eval_loss": 0.46914541721343994,
+      "eval_runtime": 38.1221,
+      "eval_samples_per_second": 3.148,
+      "eval_steps_per_second": 0.787,
+      "num_input_tokens_seen": 25730432,
+      "step": 2450
     }
   ],
   "logging_steps": 5,
   "max_steps": 3400,
-  "num_input_tokens_seen": 25206168,
+  "num_input_tokens_seen": 25730432,
   "num_train_epochs": 1,
   "save_steps": 50,
   "stateful_callbacks": {
@@ -4298,7 +4387,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 1663008045268992.0,
+  "total_flos": 1697604769677312.0,
   "train_batch_size": 1,
   "trial_name": null,
   "trial_params": null