Training in progress, step 1000, checkpoint

last-checkpoint/adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:75f1851db882a056a02f5437327881cf6ebb42ca6898a9f9c061a473b54ce23c
+oid sha256:c6e45eb274c1273932226b11445e2fd07b7c7ebefc12bc54da671bc0cebf939d
 size 29034840

last-checkpoint/global_step1000/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:330eb06c9db21349ddc0e66a8debf20a48792c47ea58a16a47fd40af17274126
+size 43429616

last-checkpoint/global_step1000/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:68a41236a0e7809eed8239767c0c2f900b61faf2e68412fb5bda170bf393dad0
+size 43429616

last-checkpoint/global_step1000/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7d861b816eaa9d54fe5cd1c19cf17691f7335d8ec708639db00653b8cdc39ef7
+size 43429616

last-checkpoint/global_step1000/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:61864cd72776620321c44f1a8ffc804645f013becee885e61b4d278bc0916514
+size 43429616

last-checkpoint/global_step1000/zero_pp_rank_0_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:acd77f7915b0d991c0851d79075b241bd64c36eeb0d1e0e2f985a499c290a497
+size 637299

last-checkpoint/global_step1000/zero_pp_rank_1_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3b3f26bda9e2a2460f4f3cc5008eaa37999cec73f83483980476af2f03541f21
+size 637171

last-checkpoint/global_step1000/zero_pp_rank_2_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d7f0d6d4da9d606901f6d8359ce5b27f2560634e6ffb9fc47cc3a2d7bea3f975
+size 637171

last-checkpoint/global_step1000/zero_pp_rank_3_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5c1c7eda36ab2045cf9de55c87c4f12c530a7dab4d03922e901e1a447629a96e
+size 637171

last-checkpoint/latest

@@ -1 +1 @@
-global_step950
+global_step1000

last-checkpoint/rng_state_0.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7f2439da621f14c22b4f733e91bfc9de6b506d28d7b8d6f3eaca2e0b4f24c078
+oid sha256:7d73dfcc09cf3d6f08149535e03920234febc15f7e9a166987f3bc01ee871abf
 size 15024

last-checkpoint/rng_state_1.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c9e3fb386557f376b8946af5b8c91f9418f374dddb2ad9da4868b1ef16778c32
+oid sha256:4396a64b6da4868d060d1e3c7c9ccb12c39d63bd0f7b146d2512400aff4c769c
 size 15024

last-checkpoint/rng_state_2.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:dc7774d06045635bece9e960378fdc6913bf7bbbc903444cc570d1ca6ac25645
+oid sha256:95877efc8fb5eb302819ee7effca4222569cdcfdebb9fa5d9846e68ed9e833fe
 size 15024

last-checkpoint/rng_state_3.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d98c54a80a914fecf43d06ea81432499f46e70664f1d04651bf339163e30fa9e
+oid sha256:e9fa4f23377f00fdde731da68a8690098617a1fdd912e03cdaa8bde87c493179
 size 15024

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b9f224baf5bd2044314606c1d88f84cce32f1b37c43c15835b14e72f6a72a4fc
+oid sha256:bce7739c5bb5cf50e8f1c942e662e33e6aa589036d55e6fddd63bdf3171c1cae
 size 1064

last-checkpoint/trainer_state.json

@@ -1,9 +1,9 @@
 {
   "best_metric": 0.49604204297065735,
   "best_model_checkpoint": "saves/CADICA_qwenvl_stenosis_detect_scale4/lora/sft/checkpoint-900",
-  "epoch": 0.24465619366469224,
+  "epoch": 0.25753283543651817,
   "eval_steps": 50,
-  "global_step": 950,
+  "global_step": 1000,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -1698,11 +1698,100 @@
       "eval_steps_per_second": 0.768,
       "num_input_tokens_seen": 9970600,
       "step": 950
+    },
+    {
+      "epoch": 0.24594385784187484,
+      "grad_norm": 2.8827978223065363,
+      "learning_rate": 8.612053338817581e-05,
+      "loss": 0.4549,
+      "num_input_tokens_seen": 10022248,
+      "step": 955
+    },
+    {
+      "epoch": 0.24723152201905743,
+      "grad_norm": 6.662877258417003,
+      "learning_rate": 8.595197232161824e-05,
+      "loss": 0.4791,
+      "num_input_tokens_seen": 10075280,
+      "step": 960
+    },
+    {
+      "epoch": 0.24851918619624003,
+      "grad_norm": 8.140970355143077,
+      "learning_rate": 8.578256098561275e-05,
+      "loss": 0.4833,
+      "num_input_tokens_seen": 10128392,
+      "step": 965
+    },
+    {
+      "epoch": 0.24980685037342262,
+      "grad_norm": 3.243184767888501,
+      "learning_rate": 8.561230338676239e-05,
+      "loss": 0.4672,
+      "num_input_tokens_seen": 10180720,
+      "step": 970
+    },
+    {
+      "epoch": 0.2510945145506052,
+      "grad_norm": 6.588760068173114,
+      "learning_rate": 8.544120355168451e-05,
+      "loss": 0.5205,
+      "num_input_tokens_seen": 10233256,
+      "step": 975
+    },
+    {
+      "epoch": 0.2523821787277878,
+      "grad_norm": 2.6240987196110837,
+      "learning_rate": 8.526926552691544e-05,
+      "loss": 0.5124,
+      "num_input_tokens_seen": 10284928,
+      "step": 980
+    },
+    {
+      "epoch": 0.2536698429049704,
+      "grad_norm": 8.242761558538728,
+      "learning_rate": 8.509649337881483e-05,
+      "loss": 0.5034,
+      "num_input_tokens_seen": 10338208,
+      "step": 985
+    },
+    {
+      "epoch": 0.254957507082153,
+      "grad_norm": 8.922137566500533,
+      "learning_rate": 8.492289119346943e-05,
+      "loss": 0.5226,
+      "num_input_tokens_seen": 10390224,
+      "step": 990
+    },
+    {
+      "epoch": 0.25624517125933555,
+      "grad_norm": 4.922275874717211,
+      "learning_rate": 8.474846307659658e-05,
+      "loss": 0.5399,
+      "num_input_tokens_seen": 10443080,
+      "step": 995
+    },
+    {
+      "epoch": 0.25753283543651817,
+      "grad_norm": 6.866585614783304,
+      "learning_rate": 8.457321315344694e-05,
+      "loss": 0.483,
+      "num_input_tokens_seen": 10495592,
+      "step": 1000
+    },
+    {
+      "epoch": 0.25753283543651817,
+      "eval_loss": 0.5305114388465881,
+      "eval_runtime": 38.9297,
+      "eval_samples_per_second": 3.082,
+      "eval_steps_per_second": 0.771,
+      "num_input_tokens_seen": 10495592,
+      "step": 1000
     }
   ],
   "logging_steps": 5,
   "max_steps": 3400,
-  "num_input_tokens_seen": 9970600,
+  "num_input_tokens_seen": 10495592,
   "num_train_epochs": 1,
   "save_steps": 50,
   "stateful_callbacks": {
@@ -1717,7 +1806,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 657761558069248.0,
+  "total_flos": 692447581306880.0,
   "train_batch_size": 1,
   "trial_name": null,
   "trial_params": null