Training in progress, step 1600, checkpoint

last-checkpoint/adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f1764f403b39c14f233fd2d6443f410cab81d66e3cc9f1d476e6e55a6642ff67
+oid sha256:af1a6642ef8e6fc99c76eecc34b9716928e3cb77cbad5191e812f07407a40f6e
 size 29034840

last-checkpoint/global_step1600/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8a1e0a99f40993045f70a7d1f668c31acb92eab13df1356fdd6a4e2f8531aee8
+size 43429616

last-checkpoint/global_step1600/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b8c5f93b6cb2ecb145e33480d04275c956be11e05c85b5f894fa05e8e1d26794
+size 43429616

last-checkpoint/global_step1600/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:923bcefce8398e6986fbcdc09e8c8428d3b1795008063f23d6f71e3bcc41b61c
+size 43429616

last-checkpoint/global_step1600/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:90c591777dd0823c282439ceb0015389baa53ac2730917ec7f0a8e22fc101365
+size 43429616

last-checkpoint/global_step1600/zero_pp_rank_0_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2d4afb4aaf2fe09b00f009636090ac8140a92a92061b0beb56860dbb1563a322
+size 637299

last-checkpoint/global_step1600/zero_pp_rank_1_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:99752500f7623dfbbb0b804e1c22e1cbd188635e5967f6ff430fde200bbb865f
+size 637171

last-checkpoint/global_step1600/zero_pp_rank_2_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bf3c8ffb4a388a7c685af82e29e62c38dac41b19f00d659db4227bfe73f04b08
+size 637171

last-checkpoint/global_step1600/zero_pp_rank_3_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bab2075b924174a0c53aaeb6926f645d6f84f031f1c7a5a84a1e25330a267f11
+size 637171

last-checkpoint/latest

@@ -1 +1 @@
-global_step1550
+global_step1600

last-checkpoint/rng_state_0.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7f1e48a120d69830576f7b582aa6cc46f0ca41d30015a7a674eaec3dcdfc0f09
+oid sha256:9279ed4b01716237e789d2631c1f29bc5d43c5633c014d4401de21b672c1b355
 size 15024

last-checkpoint/rng_state_1.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4dbabb9273d3983e52a4a981b5f60f8c2e19da375765d05bb9f2caad284b9652
+oid sha256:ca1990d68e57c70df5c56d395dd3f3befbe07b380521f4144677c20f6fe2a3eb
 size 15024

last-checkpoint/rng_state_2.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:554ac925bb9c9ea292b7a41caac1cf75285511cf8aa440f37090891ee457a178
+oid sha256:e0790066885525e1b9a9390a40ae27abd57abb47f031abface27890732f9e684
 size 15024

last-checkpoint/rng_state_3.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5be5e00123fc0a321e41599b50e07be02f4c165504c601192e5c73f5f5437c30
+oid sha256:1325a2034fe48ebad4f00ac8a2b32ab5c4c43c2497712169a8e3b1112363d916
 size 15024

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8eb069683e7f84aa36296476346fc663361d9b05ad7b09b71f22f44afdb0ea48
+oid sha256:f2487a6c511ed8055eb0842d87966b09ae8b62c1b4514727282ca413d6e9c4e2
 size 1064

last-checkpoint/trainer_state.json

@@ -1,9 +1,9 @@
 {
   "best_metric": 0.4339977502822876,
   "best_model_checkpoint": "saves/CADICA_qwenvl_stenosis_detect_scale4/lora/sft/checkpoint-1250",
-  "epoch": 0.39917589492660316,
+  "epoch": 0.41205253669842906,
   "eval_steps": 50,
-  "global_step": 1550,
+  "global_step": 1600,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -2766,11 +2766,100 @@
       "eval_steps_per_second": 0.78,
       "num_input_tokens_seen": 16269896,
       "step": 1550
+    },
+    {
+      "epoch": 0.4004635591037857,
+      "grad_norm": 4.920079251827062,
+      "learning_rate": 6.109217817775139e-05,
+      "loss": 0.4593,
+      "num_input_tokens_seen": 16322496,
+      "step": 1555
+    },
+    {
+      "epoch": 0.40175122328096835,
+      "grad_norm": 9.068094163618136,
+      "learning_rate": 6.085494954896156e-05,
+      "loss": 0.4865,
+      "num_input_tokens_seen": 16375320,
+      "step": 1560
+    },
+    {
+      "epoch": 0.4030388874581509,
+      "grad_norm": 9.316944070527988,
+      "learning_rate": 6.061746419901388e-05,
+      "loss": 0.4422,
+      "num_input_tokens_seen": 16428096,
+      "step": 1565
+    },
+    {
+      "epoch": 0.4043265516353335,
+      "grad_norm": 2.4617418860122213,
+      "learning_rate": 6.0379727744471936e-05,
+      "loss": 0.3538,
+      "num_input_tokens_seen": 16480832,
+      "step": 1570
+    },
+    {
+      "epoch": 0.4056142158125161,
+      "grad_norm": 5.028400110331736,
+      "learning_rate": 6.014174580783794e-05,
+      "loss": 0.3923,
+      "num_input_tokens_seen": 16534016,
+      "step": 1575
+    },
+    {
+      "epoch": 0.4069018799896987,
+      "grad_norm": 6.638266454273257,
+      "learning_rate": 5.990352401741981e-05,
+      "loss": 0.3967,
+      "num_input_tokens_seen": 16586216,
+      "step": 1580
+    },
+    {
+      "epoch": 0.4081895441668813,
+      "grad_norm": 6.928848680437489,
+      "learning_rate": 5.9665068007197976e-05,
+      "loss": 0.4212,
+      "num_input_tokens_seen": 16639312,
+      "step": 1585
+    },
+    {
+      "epoch": 0.40947720834406387,
+      "grad_norm": 4.2324092477507005,
+      "learning_rate": 5.94263834166923e-05,
+      "loss": 0.3489,
+      "num_input_tokens_seen": 16692328,
+      "step": 1590
+    },
+    {
+      "epoch": 0.41076487252124644,
+      "grad_norm": 5.607976113391715,
+      "learning_rate": 5.918747589082853e-05,
+      "loss": 0.4105,
+      "num_input_tokens_seen": 16745088,
+      "step": 1595
+    },
+    {
+      "epoch": 0.41205253669842906,
+      "grad_norm": 5.155332109104381,
+      "learning_rate": 5.8948351079804875e-05,
+      "loss": 0.3914,
+      "num_input_tokens_seen": 16798768,
+      "step": 1600
+    },
+    {
+      "epoch": 0.41205253669842906,
+      "eval_loss": 0.4657597243785858,
+      "eval_runtime": 38.2951,
+      "eval_samples_per_second": 3.134,
+      "eval_steps_per_second": 0.783,
+      "num_input_tokens_seen": 16798768,
+      "step": 1600
     }
   ],
   "logging_steps": 5,
   "max_steps": 3400,
-  "num_input_tokens_seen": 16269896,
+  "num_input_tokens_seen": 16798768,
   "num_train_epochs": 1,
   "save_steps": 50,
   "stateful_callbacks": {
@@ -2785,7 +2874,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 1073418423500800.0,
+  "total_flos": 1108323298967552.0,
   "train_batch_size": 1,
   "trial_name": null,
   "trial_params": null