Training in progress, step 2550, checkpoint

last-checkpoint/adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0cd56f5dfc9655c1cce72c58e7171b43d809ae6db173fb5bf3e8fc8c8fe2e604
+oid sha256:171dfcb8bb8ac5a7df90ac9ee419e7a8a301f24e9f99f3e8664ab7c1ada5f55f
 size 29034840

last-checkpoint/global_step2550/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7306fbf7c90e0959db1277b3c0b7e5b92ea93823605298953ad51fa7fbd9a197
+size 43429616

last-checkpoint/global_step2550/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1df1034db16dd3aa7480d586822b2c0fff91111cb4b6dfdae59f54b5cbe915f2
+size 43429616

last-checkpoint/global_step2550/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:62b83d05c2cfecfdfd49127decbdcbc0520d46ee34a6e406adc9e6bdfb17fc8d
+size 43429616

last-checkpoint/global_step2550/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9ecefff3e20c3e6e020ef8f7fd37175a04b52fb99b6ec3604862d40100759a7c
+size 43429616

last-checkpoint/global_step2550/zero_pp_rank_0_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:459c71b1156c55fcd73acb65442792d22282e3260b89f66cdf8ad0815d56f457
+size 637299

last-checkpoint/global_step2550/zero_pp_rank_1_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c42effbfd8441de135408f95c55462d6c1c4c8dd96b6417b67de471d2e95afcb
+size 637171

last-checkpoint/global_step2550/zero_pp_rank_2_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:304c5ba32c89fec95d937699d77819f8e533af674d93476e22a507dac625cebf
+size 637171

last-checkpoint/global_step2550/zero_pp_rank_3_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fcaea5da1d1730a97458c88513e37f66edcf7420f14ca71eb82cab77a1de4424
+size 637171

last-checkpoint/latest

@@ -1 +1 @@
-global_step2500
+global_step2550

last-checkpoint/rng_state_0.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a97c73c15a2a5b2de7dc426a700b2053aee43809425431c513cc5e3aab6c2107
+oid sha256:0362dfd92e8da01e4a0deedcbd1c493b8162d5d1d84d5a4c1cd210c556f2cf9b
 size 15024

last-checkpoint/rng_state_1.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1296b339c1b16ab7e14352a269004d20ede428aef748283fb0a6650d62f58129
+oid sha256:e65c5adee1a22c5343e38495a6905880496fb22d5e3ec5b16b87aadb731969d2
 size 15024

last-checkpoint/rng_state_2.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:000b1637f5e73170f2337500a6a083df3a43d967d642b6c3a68f60deb6c3b960
+oid sha256:f8d8858483b0c6944d55621cc2633469e3e0d04c48b6671eee92d4abab2352c2
 size 15024

last-checkpoint/rng_state_3.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b8bcb6e7802f6d888bc099642911087298cfb1adf7053a2d43a67192a53404ef
+oid sha256:ca42ef4f7a2f8c2285c4cf6cef585dcc0b132b21e8bb33d96d53b6db837f5e54
 size 15024

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:be1e9cd300c4f4c1fc9be3848ef7e995abd4a81c17c7a3b103813aaad4725565
+oid sha256:eb561d82386bf4b227a10b6e6e08effab17d0e684e1cd302e30cfc0a843bd1df
 size 1064

last-checkpoint/trainer_state.json

@@ -1,9 +1,9 @@
 {
   "best_metric": 0.4145541489124298,
   "best_model_checkpoint": "saves/CADICA_qwenvl_stenosis_detect_scale4/lora/sft/checkpoint-2350",
-  "epoch": 0.6438320885912954,
+  "epoch": 0.6567087303631213,
   "eval_steps": 50,
-  "global_step": 2500,
+  "global_step": 2550,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -4457,11 +4457,100 @@
       "eval_steps_per_second": 0.785,
       "num_input_tokens_seen": 26252584,
       "step": 2500
+    },
+    {
+      "epoch": 0.645119752768478,
+      "grad_norm": 12.715005532803087,
+      "learning_rate": 1.7777908288191176e-05,
+      "loss": 0.3113,
+      "num_input_tokens_seen": 26304800,
+      "step": 2505
+    },
+    {
+      "epoch": 0.6464074169456606,
+      "grad_norm": 4.389623559119695,
+      "learning_rate": 1.7592358989400883e-05,
+      "loss": 0.3581,
+      "num_input_tokens_seen": 26357680,
+      "step": 2510
+    },
+    {
+      "epoch": 0.6476950811228431,
+      "grad_norm": 4.708341940810254,
+      "learning_rate": 1.740757613610028e-05,
+      "loss": 0.3353,
+      "num_input_tokens_seen": 26410432,
+      "step": 2515
+    },
+    {
+      "epoch": 0.6489827453000258,
+      "grad_norm": 2.698266437964572,
+      "learning_rate": 1.7223564098431067e-05,
+      "loss": 0.2796,
+      "num_input_tokens_seen": 26463016,
+      "step": 2520
+    },
+    {
+      "epoch": 0.6502704094772084,
+      "grad_norm": 2.4430847474817843,
+      "learning_rate": 1.704032722830512e-05,
+      "loss": 0.3197,
+      "num_input_tokens_seen": 26515408,
+      "step": 2525
+    },
+    {
+      "epoch": 0.6515580736543909,
+      "grad_norm": 2.729151807047382,
+      "learning_rate": 1.68578698593014e-05,
+      "loss": 0.3182,
+      "num_input_tokens_seen": 26567024,
+      "step": 2530
+    },
+    {
+      "epoch": 0.6528457378315735,
+      "grad_norm": 12.016926866019531,
+      "learning_rate": 1.6676196306563613e-05,
+      "loss": 0.3822,
+      "num_input_tokens_seen": 26619744,
+      "step": 2535
+    },
+    {
+      "epoch": 0.6541334020087561,
+      "grad_norm": 3.7284612790252294,
+      "learning_rate": 1.6495310866698093e-05,
+      "loss": 0.2853,
+      "num_input_tokens_seen": 26672408,
+      "step": 2540
+    },
+    {
+      "epoch": 0.6554210661859388,
+      "grad_norm": 4.562253048250174,
+      "learning_rate": 1.631521781767214e-05,
+      "loss": 0.3622,
+      "num_input_tokens_seen": 26724488,
+      "step": 2545
+    },
+    {
+      "epoch": 0.6567087303631213,
+      "grad_norm": 9.803435725573266,
+      "learning_rate": 1.6135921418712956e-05,
+      "loss": 0.3195,
+      "num_input_tokens_seen": 26776816,
+      "step": 2550
+    },
+    {
+      "epoch": 0.6567087303631213,
+      "eval_loss": 0.43731561303138733,
+      "eval_runtime": 38.3205,
+      "eval_samples_per_second": 3.131,
+      "eval_steps_per_second": 0.783,
+      "num_input_tokens_seen": 26776816,
+      "step": 2550
     }
   ],
   "logging_steps": 5,
   "max_steps": 3400,
-  "num_input_tokens_seen": 26252584,
+  "num_input_tokens_seen": 26776816,
   "num_train_epochs": 1,
   "save_steps": 50,
   "stateful_callbacks": {
@@ -4476,7 +4565,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 1732044831850496.0,
+  "total_flos": 1766674781700096.0,
   "train_batch_size": 1,
   "trial_name": null,
   "trial_params": null