Training in progress, epoch 20, checkpoint

last-checkpoint/global_step3941/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:94b4dcfd71cab30d4f84d893b6fd0d1a6e2aa4b3a61d6c79e5479ce39b939d94
+size 2506176112

last-checkpoint/global_step3941/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f954c542f98537a9ed834facf341309119d8d1fba9b2ce996edb91caa0185864
+size 2506176112

last-checkpoint/global_step3941/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5b2c3b968bcad4a7393ddf5c5f119cdaac63191903276c1eb81d6c451c5d6752
+size 2506176112

last-checkpoint/global_step3941/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0bee82ebc9e3eee25d3b0df80e977d29d2fb8d31a2c15e34a592ca2e3b23b894
+size 2506176112

last-checkpoint/global_step3941/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cb059eb071e2c280b3205e611e8d35da6f4b0074daa865e2753febf692019be7
+size 2506176112

last-checkpoint/global_step3941/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:88b041f26328228bde99957a111e977af41d76464988508861df2fe943631d98
+size 2506176112

last-checkpoint/global_step3941/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3d86002b3f908ebc201f5fbcb089f747f46dc062a490b815c3292beeddbb2eba
+size 2506176112

last-checkpoint/global_step3941/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:eff104ff6a3804962fc8e81a4a25a68e1ddd23153106604a90d90db75ed7cc23
+size 2506176112

last-checkpoint/global_step3941/zero_pp_rank_0_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cac99f40ef6a29819b6bfee413e07a661faee0937b8d1c61c805da871e0945af
+size 85570

last-checkpoint/global_step3941/zero_pp_rank_1_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a5701cbcfc2c6647d84113e3d13ee7cc172b6847210fec719fae584f61e7c3a0
+size 85506

last-checkpoint/global_step3941/zero_pp_rank_2_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:24bf30e27f127c8845ba54ae9788e2ca7ed1ccbcbba555d599dab83769417d07
+size 85506

last-checkpoint/global_step3941/zero_pp_rank_3_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:45c29e366a2f7623d367aac2611b78199725dcfee1510c37b39fb5eff91c9287
+size 85506

last-checkpoint/global_step3941/zero_pp_rank_4_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:14f446b32558ac815c52ce36a2ece940f8c9860c1b6666d3cf16bcfedebda918
+size 85506

last-checkpoint/global_step3941/zero_pp_rank_5_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f63d5c8898ac42433f7add352d994d40bab319ea7c6840b8756db83a707506f3
+size 85506

last-checkpoint/global_step3941/zero_pp_rank_6_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1c9e1729073f017d91bafd979b84d7c37c6570c15926abbd20aeeb138871a251
+size 85506

last-checkpoint/global_step3941/zero_pp_rank_7_mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d9d37159e66416476690c422874e2523ac0312d3e34aa742482bd47edb58aebc
+size 85506

last-checkpoint/latest

@@ -1 +1 @@
-global_step3357
+global_step3941

last-checkpoint/model-00001-of-00002.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3578035697ae915bf8ed319e400346be0b8f4d900849a6d07f0ff9b4c3b1711c
+oid sha256:acdc933f849726cf9f6626987c11cea6a7678ccfd80dc2d0483bec3e83e0c3a7
 size 4945242264

last-checkpoint/model-00002-of-00002.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a652aa698a378ecc5fb4aaee9480c493c9a62c60f4f96b74c9d6698fa2aa8d33
+oid sha256:4b099839b0f1c5f0d5fb18759ba02999fd787c83dda2d02d2ba36941c44a246b
 size 67121608

last-checkpoint/rng_state_0.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1a0c9979566a5d89cb3c766336548670ec6f2291deba1b7ab1764c12d3187b24
+oid sha256:0f8f81f81a21cdf7d6a9d642f67427b821e87802eb5a4ca4a5c038480e9a673f
 size 15984

last-checkpoint/rng_state_1.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:03e36a570d6158fc25d1cf5d9f8f450fc64c5a7683330277f89ff76d5f2fc6cd
+oid sha256:c967add958d82a4ef6285ec19e0e5a560f82b1ea3488260d2539d200cb5f199b
 size 15984

last-checkpoint/rng_state_2.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b4f619cbef4b74f1680d667c8788285a602392e63bdf3760ef3a59ec8864d483
+oid sha256:1a5e30f320a5f3e4a4760af275c4a2bb9fdcfc6c661a3757da3b20b4c29a87fc
 size 15984

last-checkpoint/rng_state_3.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1fc037fba93ace1bf7ce01b1a5f7d785698d47b4cc2cedf2300bbf7a41ebf05c
+oid sha256:3411b849d13fdd9e668c8cc5fafa8402de13fd4a2d5761eaf33f2791780721e4
 size 15984

last-checkpoint/rng_state_4.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9ab728c2461d6d1c64f04d7cbfdfcbfa7bd7ad0ef6e19d52458501ee81b27128
+oid sha256:1765109deada93bbc4f9e38b5c3c6bbd9afe9a12839bbbd72227ca7ff46467b1
 size 15984

last-checkpoint/rng_state_5.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:27530e653ebf5997ae3159cdcde264607e6a6f86b7e3c7a1b3a1e8301cd43d03
+oid sha256:6147f79a889f2906a81eea5c06f72acf722f674546fd6be8432d3d70a04392a8
 size 15984

last-checkpoint/rng_state_6.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c1fddaeb1257697bd7c0101abf1ab23f2925d0d9165cd8bddfbd22f8444db2b7
+oid sha256:a9f5815a19b60dc1bb303c451b515a4ec523fa0faf1c87c69b03ad4032c4baa2
 size 15984

last-checkpoint/rng_state_7.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:942af3734a320fe12a3205a47ca1cdc7d1f0996bfde86c020a35545ccd2fd418
+oid sha256:395da160bbc37f8049700e0d82d501cbd1effc732719599c0cd143db7892910e
 size 15984

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:605054ed043f8ba321ca13100ae25afc2296eb67de83d5027f6f7f6d891a4130
+oid sha256:9dd8e459c5f044a7182784611ab9373a71c50f4a912290b4cfdcb531e69b1b46
 size 1064

last-checkpoint/trainer_state.json

@@ -1,9 +1,9 @@
 {
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 16.99746835443038,
+  "epoch": 20.0,
   "eval_steps": 100,
-  "global_step": 3357,
+  "global_step": 3941,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -5047,12 +5047,897 @@
       "rewards/margins": 0.5159622430801392,
       "rewards/rejected": -0.33039581775665283,
       "step": 3350
+    },
+    {
+      "epoch": 17.0126582278481,
+      "grad_norm": 218651.56901322177,
+      "learning_rate": 2.9144468818552176e-08,
+      "logits/chosen": 0.41573429107666016,
+      "logits/rejected": 1.103547215461731,
+      "logps/chosen": -37.6799201965332,
+      "logps/rejected": -569.5391235351562,
+      "loss": 14029.3563,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": 0.18778078258037567,
+      "rewards/margins": 0.5316546559333801,
+      "rewards/rejected": -0.34387388825416565,
+      "step": 3360
+    },
+    {
+      "epoch": 17.063291139240505,
+      "grad_norm": 236719.0916690887,
+      "learning_rate": 2.7577561892823564e-08,
+      "logits/chosen": -0.09267449378967285,
+      "logits/rejected": 0.3535307049751282,
+      "logps/chosen": -43.02147674560547,
+      "logps/rejected": -571.3306884765625,
+      "loss": 14216.225,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": 0.18815621733665466,
+      "rewards/margins": 0.52230304479599,
+      "rewards/rejected": -0.33414679765701294,
+      "step": 3370
+    },
+    {
+      "epoch": 17.11392405063291,
+      "grad_norm": 151995.97062770248,
+      "learning_rate": 2.6010654967094953e-08,
+      "logits/chosen": 1.3600900173187256,
+      "logits/rejected": 0.45606088638305664,
+      "logps/chosen": -33.07421112060547,
+      "logps/rejected": -574.2869262695312,
+      "loss": 14569.3875,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.1843741536140442,
+      "rewards/margins": 0.533474862575531,
+      "rewards/rejected": -0.3491007089614868,
+      "step": 3380
+    },
+    {
+      "epoch": 17.164556962025316,
+      "grad_norm": 229039.39535517112,
+      "learning_rate": 2.4443748041366342e-08,
+      "logits/chosen": 0.012326288037002087,
+      "logits/rejected": -0.24337856471538544,
+      "logps/chosen": -48.85834503173828,
+      "logps/rejected": -591.9976806640625,
+      "loss": 15141.0031,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.19536466896533966,
+      "rewards/margins": 0.5443064570426941,
+      "rewards/rejected": -0.34894177317619324,
+      "step": 3390
+    },
+    {
+      "epoch": 17.21518987341772,
+      "grad_norm": 224579.22425486994,
+      "learning_rate": 2.2876841115637728e-08,
+      "logits/chosen": -0.07731113582849503,
+      "logits/rejected": 0.8038260340690613,
+      "logps/chosen": -42.96089172363281,
+      "logps/rejected": -587.9930419921875,
+      "loss": 13962.9406,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": 0.18969421088695526,
+      "rewards/margins": 0.5471119284629822,
+      "rewards/rejected": -0.3574177622795105,
+      "step": 3400
+    },
+    {
+      "epoch": 17.265822784810126,
+      "grad_norm": 194108.45632178357,
+      "learning_rate": 2.1309934189909117e-08,
+      "logits/chosen": -1.735790491104126,
+      "logits/rejected": -0.8417277336120605,
+      "logps/chosen": -40.28795623779297,
+      "logps/rejected": -577.9163208007812,
+      "loss": 14457.5328,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": 0.19212636351585388,
+      "rewards/margins": 0.54021155834198,
+      "rewards/rejected": -0.3480851650238037,
+      "step": 3410
+    },
+    {
+      "epoch": 17.31645569620253,
+      "grad_norm": 323871.0912725565,
+      "learning_rate": 1.9743027264180506e-08,
+      "logits/chosen": 1.0423898696899414,
+      "logits/rejected": 1.1823880672454834,
+      "logps/chosen": -50.077327728271484,
+      "logps/rejected": -565.8704223632812,
+      "loss": 14191.8531,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": 0.18371161818504333,
+      "rewards/margins": 0.5181502103805542,
+      "rewards/rejected": -0.3344385623931885,
+      "step": 3420
+    },
+    {
+      "epoch": 17.367088607594937,
+      "grad_norm": 207973.13380554292,
+      "learning_rate": 1.8176120338451895e-08,
+      "logits/chosen": -0.8037737011909485,
+      "logits/rejected": -0.8005819320678711,
+      "logps/chosen": -45.626670837402344,
+      "logps/rejected": -544.7116088867188,
+      "loss": 14114.0906,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": 0.1760983169078827,
+      "rewards/margins": 0.5017568469047546,
+      "rewards/rejected": -0.32565850019454956,
+      "step": 3430
+    },
+    {
+      "epoch": 17.417721518987342,
+      "grad_norm": 191156.31750064602,
+      "learning_rate": 1.6609213412723284e-08,
+      "logits/chosen": 1.2277637720108032,
+      "logits/rejected": 0.573845386505127,
+      "logps/chosen": -50.492279052734375,
+      "logps/rejected": -586.3282470703125,
+      "loss": 13957.7594,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": 0.19206462800502777,
+      "rewards/margins": 0.5328875184059143,
+      "rewards/rejected": -0.34082287549972534,
+      "step": 3440
+    },
+    {
+      "epoch": 17.468354430379748,
+      "grad_norm": 262791.84599779843,
+      "learning_rate": 1.5042306486994673e-08,
+      "logits/chosen": 0.29228338599205017,
+      "logits/rejected": 0.9747223854064941,
+      "logps/chosen": -37.640201568603516,
+      "logps/rejected": -557.47119140625,
+      "loss": 14478.4906,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.18818514049053192,
+      "rewards/margins": 0.5214470624923706,
+      "rewards/rejected": -0.3332619369029999,
+      "step": 3450
+    },
+    {
+      "epoch": 17.518987341772153,
+      "grad_norm": 227441.5548714142,
+      "learning_rate": 1.347539956126606e-08,
+      "logits/chosen": -0.060483645647764206,
+      "logits/rejected": 0.41309136152267456,
+      "logps/chosen": -46.32054138183594,
+      "logps/rejected": -588.6563720703125,
+      "loss": 14804.9047,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": 0.19712397456169128,
+      "rewards/margins": 0.5437620878219604,
+      "rewards/rejected": -0.34663814306259155,
+      "step": 3460
+    },
+    {
+      "epoch": 17.569620253164558,
+      "grad_norm": 378558.8588589865,
+      "learning_rate": 1.1908492635537449e-08,
+      "logits/chosen": 2.0075535774230957,
+      "logits/rejected": 2.772726058959961,
+      "logps/chosen": -46.09113693237305,
+      "logps/rejected": -582.0597534179688,
+      "loss": 14645.9562,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": 0.18593838810920715,
+      "rewards/margins": 0.5331605076789856,
+      "rewards/rejected": -0.34722214937210083,
+      "step": 3470
+    },
+    {
+      "epoch": 17.620253164556964,
+      "grad_norm": 263891.6462573049,
+      "learning_rate": 1.0341585709808836e-08,
+      "logits/chosen": 0.273967444896698,
+      "logits/rejected": 1.9021276235580444,
+      "logps/chosen": -34.29851531982422,
+      "logps/rejected": -567.2389526367188,
+      "loss": 15085.2313,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.18956169486045837,
+      "rewards/margins": 0.5355597734451294,
+      "rewards/rejected": -0.3459981083869934,
+      "step": 3480
+    },
+    {
+      "epoch": 17.67088607594937,
+      "grad_norm": 276267.9285814808,
+      "learning_rate": 8.774678784080225e-09,
+      "logits/chosen": -0.02632077969610691,
+      "logits/rejected": 0.4594387114048004,
+      "logps/chosen": -45.098960876464844,
+      "logps/rejected": -568.720947265625,
+      "loss": 13750.4469,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": 0.19129987061023712,
+      "rewards/margins": 0.5272942781448364,
+      "rewards/rejected": -0.33599433302879333,
+      "step": 3490
+    },
+    {
+      "epoch": 17.72151898734177,
+      "grad_norm": 156087.69298121333,
+      "learning_rate": 7.207771858351613e-09,
+      "logits/chosen": 0.04748225212097168,
+      "logits/rejected": 0.4610685408115387,
+      "logps/chosen": -49.872169494628906,
+      "logps/rejected": -603.3367919921875,
+      "loss": 13778.1469,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": 0.19287212193012238,
+      "rewards/margins": 0.5504390001296997,
+      "rewards/rejected": -0.3575669229030609,
+      "step": 3500
+    },
+    {
+      "epoch": 17.772151898734176,
+      "grad_norm": 209667.9634643516,
+      "learning_rate": 5.6408649326230014e-09,
+      "logits/chosen": 1.4883615970611572,
+      "logits/rejected": 2.2038960456848145,
+      "logps/chosen": -46.18961715698242,
+      "logps/rejected": -575.4703369140625,
+      "loss": 13653.9672,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.18806029856204987,
+      "rewards/margins": 0.5310976505279541,
+      "rewards/rejected": -0.34303733706474304,
+      "step": 3510
+    },
+    {
+      "epoch": 17.82278481012658,
+      "grad_norm": 222056.5820151951,
+      "learning_rate": 4.07395800689439e-09,
+      "logits/chosen": -0.582931637763977,
+      "logits/rejected": -0.23906604945659637,
+      "logps/chosen": -60.795921325683594,
+      "logps/rejected": -590.2525024414062,
+      "loss": 14149.5938,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": 0.19504059851169586,
+      "rewards/margins": 0.5360020399093628,
+      "rewards/rejected": -0.34096142649650574,
+      "step": 3520
+    },
+    {
+      "epoch": 17.873417721518987,
+      "grad_norm": 213324.38139465638,
+      "learning_rate": 2.5070510811657785e-09,
+      "logits/chosen": -0.3791787028312683,
+      "logits/rejected": 0.26259681582450867,
+      "logps/chosen": -48.315147399902344,
+      "logps/rejected": -579.376220703125,
+      "loss": 14028.4,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.19789119064807892,
+      "rewards/margins": 0.5311988592147827,
+      "rewards/rejected": -0.333307683467865,
+      "step": 3530
+    },
+    {
+      "epoch": 17.924050632911392,
+      "grad_norm": 207695.40556695752,
+      "learning_rate": 9.40144155437167e-10,
+      "logits/chosen": 2.0871522426605225,
+      "logits/rejected": 2.378633975982666,
+      "logps/chosen": -36.07915115356445,
+      "logps/rejected": -560.524169921875,
+      "loss": 13942.7234,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": 0.18208447098731995,
+      "rewards/margins": 0.5249064564704895,
+      "rewards/rejected": -0.34282201528549194,
+      "step": 3540
+    },
+    {
+      "epoch": 18.020253164556962,
+      "grad_norm": 633377.3531549113,
+      "learning_rate": 2.774992165465371e-07,
+      "logits/chosen": 0.778042197227478,
+      "logits/rejected": 0.4570779800415039,
+      "logps/chosen": -30.87795639038086,
+      "logps/rejected": -562.3123168945312,
+      "loss": 14823.5117,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.19000156223773956,
+      "rewards/margins": 0.5307614803314209,
+      "rewards/rejected": -0.34075987339019775,
+      "step": 3550
+    },
+    {
+      "epoch": 18.070886075949367,
+      "grad_norm": 536501.4949459385,
+      "learning_rate": 2.767157630836728e-07,
+      "logits/chosen": -1.453107476234436,
+      "logits/rejected": -1.1603299379348755,
+      "logps/chosen": -48.59767532348633,
+      "logps/rejected": -607.5595703125,
+      "loss": 14268.2156,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.19779345393180847,
+      "rewards/margins": 0.5535213351249695,
+      "rewards/rejected": -0.3557279109954834,
+      "step": 3560
+    },
+    {
+      "epoch": 18.121518987341773,
+      "grad_norm": 613929.4964505757,
+      "learning_rate": 2.7593230962080847e-07,
+      "logits/chosen": -1.0204923152923584,
+      "logits/rejected": -1.006306529045105,
+      "logps/chosen": -40.379703521728516,
+      "logps/rejected": -586.8853759765625,
+      "loss": 14124.1406,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.18828515708446503,
+      "rewards/margins": 0.5416163206100464,
+      "rewards/rejected": -0.35333114862442017,
+      "step": 3570
+    },
+    {
+      "epoch": 18.172151898734178,
+      "grad_norm": 453188.0208924516,
+      "learning_rate": 2.751488561579442e-07,
+      "logits/chosen": 0.978573203086853,
+      "logits/rejected": 1.6422239542007446,
+      "logps/chosen": -40.75902557373047,
+      "logps/rejected": -571.6940307617188,
+      "loss": 14028.8266,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.19008655846118927,
+      "rewards/margins": 0.5349593758583069,
+      "rewards/rejected": -0.34487277269363403,
+      "step": 3580
+    },
+    {
+      "epoch": 18.222784810126583,
+      "grad_norm": 470617.1864493106,
+      "learning_rate": 2.743654026950799e-07,
+      "logits/chosen": 0.612755298614502,
+      "logits/rejected": 1.586531639099121,
+      "logps/chosen": -47.43413162231445,
+      "logps/rejected": -567.2514038085938,
+      "loss": 14305.0953,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": 0.18671520054340363,
+      "rewards/margins": 0.5188931226730347,
+      "rewards/rejected": -0.33217787742614746,
+      "step": 3590
+    },
+    {
+      "epoch": 18.27341772151899,
+      "grad_norm": 568328.2123455897,
+      "learning_rate": 2.7358194923221564e-07,
+      "logits/chosen": 2.5831315517425537,
+      "logits/rejected": 2.3743977546691895,
+      "logps/chosen": -36.72047805786133,
+      "logps/rejected": -561.4580688476562,
+      "loss": 14931.7812,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": 0.18578791618347168,
+      "rewards/margins": 0.5219975113868713,
+      "rewards/rejected": -0.33620959520339966,
+      "step": 3600
+    },
+    {
+      "epoch": 18.324050632911394,
+      "grad_norm": 258649.85824251673,
+      "learning_rate": 2.727984957693513e-07,
+      "logits/chosen": -0.6456964612007141,
+      "logits/rejected": 0.10119187831878662,
+      "logps/chosen": -45.66813659667969,
+      "logps/rejected": -584.33984375,
+      "loss": 13962.2891,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.19075247645378113,
+      "rewards/margins": 0.5430020093917847,
+      "rewards/rejected": -0.35224950313568115,
+      "step": 3610
+    },
+    {
+      "epoch": 18.374683544303796,
+      "grad_norm": 523823.39531677734,
+      "learning_rate": 2.72015042306487e-07,
+      "logits/chosen": -0.1337634027004242,
+      "logits/rejected": 0.3194190561771393,
+      "logps/chosen": -43.2452278137207,
+      "logps/rejected": -576.6324462890625,
+      "loss": 14478.6656,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": 0.19283099472522736,
+      "rewards/margins": 0.5422399640083313,
+      "rewards/rejected": -0.34940892457962036,
+      "step": 3620
+    },
+    {
+      "epoch": 18.4253164556962,
+      "grad_norm": 369527.7483340646,
+      "learning_rate": 2.712315888436227e-07,
+      "logits/chosen": -0.5704905390739441,
+      "logits/rejected": -0.24132680892944336,
+      "logps/chosen": -39.81604766845703,
+      "logps/rejected": -579.3060302734375,
+      "loss": 14853.9188,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.1893097311258316,
+      "rewards/margins": 0.5385677218437195,
+      "rewards/rejected": -0.3492580056190491,
+      "step": 3630
+    },
+    {
+      "epoch": 18.475949367088607,
+      "grad_norm": 487722.91173438437,
+      "learning_rate": 2.704481353807584e-07,
+      "logits/chosen": 0.30203062295913696,
+      "logits/rejected": 1.367623209953308,
+      "logps/chosen": -43.79780578613281,
+      "logps/rejected": -575.3096313476562,
+      "loss": 14337.7125,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": 0.18831488490104675,
+      "rewards/margins": 0.5326144099235535,
+      "rewards/rejected": -0.3442995548248291,
+      "step": 3640
+    },
+    {
+      "epoch": 18.526582278481012,
+      "grad_norm": 769147.1132735502,
+      "learning_rate": 2.6966468191789406e-07,
+      "logits/chosen": 0.5818338990211487,
+      "logits/rejected": 0.8189504742622375,
+      "logps/chosen": -40.80295944213867,
+      "logps/rejected": -569.6201171875,
+      "loss": 14414.5,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.19092252850532532,
+      "rewards/margins": 0.5284001231193542,
+      "rewards/rejected": -0.3374776244163513,
+      "step": 3650
+    },
+    {
+      "epoch": 18.577215189873417,
+      "grad_norm": 423741.6615039136,
+      "learning_rate": 2.6888122845502977e-07,
+      "logits/chosen": -2.1757419109344482,
+      "logits/rejected": -1.7465986013412476,
+      "logps/chosen": -33.543739318847656,
+      "logps/rejected": -566.9044189453125,
+      "loss": 13990.4406,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": 0.18767623603343964,
+      "rewards/margins": 0.5356841683387756,
+      "rewards/rejected": -0.3480078876018524,
+      "step": 3660
+    },
+    {
+      "epoch": 18.627848101265823,
+      "grad_norm": 405282.2937016151,
+      "learning_rate": 2.680977749921655e-07,
+      "logits/chosen": -0.054244786500930786,
+      "logits/rejected": 0.9029023051261902,
+      "logps/chosen": -49.31962966918945,
+      "logps/rejected": -585.48779296875,
+      "loss": 14779.0125,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.19979842007160187,
+      "rewards/margins": 0.5441454648971558,
+      "rewards/rejected": -0.3443470597267151,
+      "step": 3670
+    },
+    {
+      "epoch": 18.678481012658228,
+      "grad_norm": 468937.7683958159,
+      "learning_rate": 2.673143215293012e-07,
+      "logits/chosen": -0.046643782407045364,
+      "logits/rejected": -0.1421128809452057,
+      "logps/chosen": -40.85643768310547,
+      "logps/rejected": -577.6583862304688,
+      "loss": 14531.35,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": 0.195206418633461,
+      "rewards/margins": 0.5361508131027222,
+      "rewards/rejected": -0.3409443199634552,
+      "step": 3680
+    },
+    {
+      "epoch": 18.729113924050633,
+      "grad_norm": 627917.5959141933,
+      "learning_rate": 2.6653086806643683e-07,
+      "logits/chosen": 1.5284700393676758,
+      "logits/rejected": 1.2886362075805664,
+      "logps/chosen": -48.694664001464844,
+      "logps/rejected": -579.7990112304688,
+      "loss": 15195.4844,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": 0.1851346641778946,
+      "rewards/margins": 0.531388521194458,
+      "rewards/rejected": -0.346253901720047,
+      "step": 3690
+    },
+    {
+      "epoch": 18.77974683544304,
+      "grad_norm": 511207.857422736,
+      "learning_rate": 2.6574741460357254e-07,
+      "logits/chosen": 0.03928997367620468,
+      "logits/rejected": 0.5418666005134583,
+      "logps/chosen": -50.31745529174805,
+      "logps/rejected": -593.0169677734375,
+      "loss": 14929.1469,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": 0.19712677597999573,
+      "rewards/margins": 0.5447811484336853,
+      "rewards/rejected": -0.3476543724536896,
+      "step": 3700
+    },
+    {
+      "epoch": 18.830379746835444,
+      "grad_norm": 568133.4282182837,
+      "learning_rate": 2.6496396114070825e-07,
+      "logits/chosen": -0.7848063707351685,
+      "logits/rejected": -0.8312255144119263,
+      "logps/chosen": -39.726234436035156,
+      "logps/rejected": -566.0286254882812,
+      "loss": 14112.2844,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": 0.18878208100795746,
+      "rewards/margins": 0.5250921249389648,
+      "rewards/rejected": -0.3363099992275238,
+      "step": 3710
+    },
+    {
+      "epoch": 18.88101265822785,
+      "grad_norm": 293062.3175283677,
+      "learning_rate": 2.6418050767784395e-07,
+      "logits/chosen": -0.22776488959789276,
+      "logits/rejected": -0.043119143694639206,
+      "logps/chosen": -47.83971405029297,
+      "logps/rejected": -575.6166381835938,
+      "loss": 14345.3813,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": 0.19344884157180786,
+      "rewards/margins": 0.5261351466178894,
+      "rewards/rejected": -0.33268633484840393,
+      "step": 3720
+    },
+    {
+      "epoch": 18.931645569620255,
+      "grad_norm": 369584.46121245134,
+      "learning_rate": 2.633970542149796e-07,
+      "logits/chosen": 0.6460098028182983,
+      "logits/rejected": 0.6165057420730591,
+      "logps/chosen": -53.0880126953125,
+      "logps/rejected": -602.0147705078125,
+      "loss": 14143.9609,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.19915179908275604,
+      "rewards/margins": 0.5483053922653198,
+      "rewards/rejected": -0.3491537272930145,
+      "step": 3730
+    },
+    {
+      "epoch": 18.98227848101266,
+      "grad_norm": 328959.5337312854,
+      "learning_rate": 2.626136007521153e-07,
+      "logits/chosen": 0.25958794355392456,
+      "logits/rejected": 0.5823850631713867,
+      "logps/chosen": -49.16436004638672,
+      "logps/rejected": -584.5070190429688,
+      "loss": 14187.4844,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": 0.19965310394763947,
+      "rewards/margins": 0.5432143211364746,
+      "rewards/rejected": -0.34356123208999634,
+      "step": 3740
+    },
+    {
+      "epoch": 19.03291139240506,
+      "grad_norm": 1626740.8696455131,
+      "learning_rate": 2.61830147289251e-07,
+      "logits/chosen": -0.6601130366325378,
+      "logits/rejected": -0.8405634164810181,
+      "logps/chosen": -46.10778045654297,
+      "logps/rejected": -587.3377685546875,
+      "loss": 14051.6469,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": 0.1946493685245514,
+      "rewards/margins": 0.54271399974823,
+      "rewards/rejected": -0.34806469082832336,
+      "step": 3750
+    },
+    {
+      "epoch": 19.083544303797467,
+      "grad_norm": 786920.4959477714,
+      "learning_rate": 2.610466938263867e-07,
+      "logits/chosen": 0.280475914478302,
+      "logits/rejected": 1.5355632305145264,
+      "logps/chosen": -40.83550262451172,
+      "logps/rejected": -576.2233276367188,
+      "loss": 14524.6172,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": 0.19496676325798035,
+      "rewards/margins": 0.5368971228599548,
+      "rewards/rejected": -0.3419303297996521,
+      "step": 3760
+    },
+    {
+      "epoch": 19.134177215189872,
+      "grad_norm": 670222.9584254185,
+      "learning_rate": 2.602632403635224e-07,
+      "logits/chosen": 1.6073856353759766,
+      "logits/rejected": 2.1679255962371826,
+      "logps/chosen": -48.07741928100586,
+      "logps/rejected": -568.386962890625,
+      "loss": 16064.1922,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": 0.19028018414974213,
+      "rewards/margins": 0.5232519507408142,
+      "rewards/rejected": -0.3329717516899109,
+      "step": 3770
+    },
+    {
+      "epoch": 19.184810126582278,
+      "grad_norm": 779401.4265683588,
+      "learning_rate": 2.594797869006581e-07,
+      "logits/chosen": -1.2690767049789429,
+      "logits/rejected": -0.7741214036941528,
+      "logps/chosen": -35.147666931152344,
+      "logps/rejected": -588.05810546875,
+      "loss": 14594.675,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.20034465193748474,
+      "rewards/margins": 0.5537833571434021,
+      "rewards/rejected": -0.3534386456012726,
+      "step": 3780
+    },
+    {
+      "epoch": 19.235443037974683,
+      "grad_norm": 677896.0436831466,
+      "learning_rate": 2.586963334377938e-07,
+      "logits/chosen": 0.381600558757782,
+      "logits/rejected": 0.3627360761165619,
+      "logps/chosen": -47.129329681396484,
+      "logps/rejected": -583.4297485351562,
+      "loss": 14673.1125,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.19634023308753967,
+      "rewards/margins": 0.5409786105155945,
+      "rewards/rejected": -0.3446383774280548,
+      "step": 3790
+    },
+    {
+      "epoch": 19.28607594936709,
+      "grad_norm": 1708590.8406628803,
+      "learning_rate": 2.579128799749295e-07,
+      "logits/chosen": -0.6463128924369812,
+      "logits/rejected": -0.1966671198606491,
+      "logps/chosen": -51.58148193359375,
+      "logps/rejected": -571.8802490234375,
+      "loss": 14855.8094,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": 0.1947019398212433,
+      "rewards/margins": 0.5256696343421936,
+      "rewards/rejected": -0.3309677243232727,
+      "step": 3800
+    },
+    {
+      "epoch": 19.336708860759494,
+      "grad_norm": 906394.5199246205,
+      "learning_rate": 2.5712942651206515e-07,
+      "logits/chosen": 0.6537224054336548,
+      "logits/rejected": 1.356911301612854,
+      "logps/chosen": -37.791786193847656,
+      "logps/rejected": -541.84912109375,
+      "loss": 14494.675,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": 0.18946874141693115,
+      "rewards/margins": 0.5098165273666382,
+      "rewards/rejected": -0.3203478455543518,
+      "step": 3810
+    },
+    {
+      "epoch": 19.3873417721519,
+      "grad_norm": 1248788.3894635146,
+      "learning_rate": 2.5634597304920085e-07,
+      "logits/chosen": -1.4148962497711182,
+      "logits/rejected": -0.616938591003418,
+      "logps/chosen": -39.15003204345703,
+      "logps/rejected": -567.9779052734375,
+      "loss": 14511.9828,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.19468382000923157,
+      "rewards/margins": 0.5306459665298462,
+      "rewards/rejected": -0.33596211671829224,
+      "step": 3820
+    },
+    {
+      "epoch": 19.437974683544304,
+      "grad_norm": 699507.4776687805,
+      "learning_rate": 2.5556251958633656e-07,
+      "logits/chosen": -0.786666214466095,
+      "logits/rejected": -0.8524150848388672,
+      "logps/chosen": -37.31165313720703,
+      "logps/rejected": -559.5811767578125,
+      "loss": 15226.8953,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": 0.18610945343971252,
+      "rewards/margins": 0.5259476900100708,
+      "rewards/rejected": -0.3398382067680359,
+      "step": 3830
+    },
+    {
+      "epoch": 19.48860759493671,
+      "grad_norm": 750946.845865734,
+      "learning_rate": 2.5477906612347227e-07,
+      "logits/chosen": -0.5914249420166016,
+      "logits/rejected": -0.1790940761566162,
+      "logps/chosen": -41.875919342041016,
+      "logps/rejected": -580.2433471679688,
+      "loss": 15077.0078,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.1914350688457489,
+      "rewards/margins": 0.5402361154556274,
+      "rewards/rejected": -0.34880098700523376,
+      "step": 3840
+    },
+    {
+      "epoch": 19.539240506329115,
+      "grad_norm": 1438213.362152031,
+      "learning_rate": 2.539956126606079e-07,
+      "logits/chosen": -1.4764426946640015,
+      "logits/rejected": -1.0852867364883423,
+      "logps/chosen": -45.64619064331055,
+      "logps/rejected": -574.2334594726562,
+      "loss": 15001.6922,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": 0.19227740168571472,
+      "rewards/margins": 0.527544379234314,
+      "rewards/rejected": -0.33526697754859924,
+      "step": 3850
+    },
+    {
+      "epoch": 19.58987341772152,
+      "grad_norm": 1015656.6585732017,
+      "learning_rate": 2.532121591977436e-07,
+      "logits/chosen": 0.0265532024204731,
+      "logits/rejected": 0.4305901527404785,
+      "logps/chosen": -40.221275329589844,
+      "logps/rejected": -580.7332763671875,
+      "loss": 15005.4062,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": 0.1900371015071869,
+      "rewards/margins": 0.5353468656539917,
+      "rewards/rejected": -0.3453097939491272,
+      "step": 3860
+    },
+    {
+      "epoch": 19.640506329113926,
+      "grad_norm": 1480021.6334817603,
+      "learning_rate": 2.5242870573487933e-07,
+      "logits/chosen": -2.3115265369415283,
+      "logits/rejected": -1.9450628757476807,
+      "logps/chosen": -43.31880187988281,
+      "logps/rejected": -592.5906372070312,
+      "loss": 14681.3906,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.20574085414409637,
+      "rewards/margins": 0.5489095449447632,
+      "rewards/rejected": -0.3431686758995056,
+      "step": 3870
+    },
+    {
+      "epoch": 19.691139240506327,
+      "grad_norm": 652464.7916313735,
+      "learning_rate": 2.5164525227201504e-07,
+      "logits/chosen": 0.63951176404953,
+      "logits/rejected": 1.3804535865783691,
+      "logps/chosen": -33.52408981323242,
+      "logps/rejected": -556.9231567382812,
+      "loss": 15124.1328,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.1876874566078186,
+      "rewards/margins": 0.5311328172683716,
+      "rewards/rejected": -0.343445360660553,
+      "step": 3880
+    },
+    {
+      "epoch": 19.741772151898733,
+      "grad_norm": 697435.0328174214,
+      "learning_rate": 2.508617988091507e-07,
+      "logits/chosen": -1.7422069311141968,
+      "logits/rejected": -1.3413903713226318,
+      "logps/chosen": -42.224788665771484,
+      "logps/rejected": -584.3877563476562,
+      "loss": 15307.875,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.2013184279203415,
+      "rewards/margins": 0.5388418436050415,
+      "rewards/rejected": -0.3375234305858612,
+      "step": 3890
+    },
+    {
+      "epoch": 19.792405063291138,
+      "grad_norm": 680395.3386900029,
+      "learning_rate": 2.500783453462864e-07,
+      "logits/chosen": 0.6034026741981506,
+      "logits/rejected": 1.1066893339157104,
+      "logps/chosen": -39.37774658203125,
+      "logps/rejected": -585.9308471679688,
+      "loss": 15434.6031,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": 0.20041151344776154,
+      "rewards/margins": 0.545585036277771,
+      "rewards/rejected": -0.34517353773117065,
+      "step": 3900
+    },
+    {
+      "epoch": 19.843037974683543,
+      "grad_norm": 1036480.9072027011,
+      "learning_rate": 2.492948918834221e-07,
+      "logits/chosen": -0.37202078104019165,
+      "logits/rejected": -0.6633853316307068,
+      "logps/chosen": -50.845218658447266,
+      "logps/rejected": -565.1788330078125,
+      "loss": 14732.9813,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": 0.1860923022031784,
+      "rewards/margins": 0.5141801834106445,
+      "rewards/rejected": -0.32808783650398254,
+      "step": 3910
+    },
+    {
+      "epoch": 19.89367088607595,
+      "grad_norm": 960769.0916438915,
+      "learning_rate": 2.485114384205578e-07,
+      "logits/chosen": -2.4451048374176025,
+      "logits/rejected": -1.8602224588394165,
+      "logps/chosen": -49.44445037841797,
+      "logps/rejected": -588.972900390625,
+      "loss": 14954.0281,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": 0.20443923771381378,
+      "rewards/margins": 0.5393208265304565,
+      "rewards/rejected": -0.33488157391548157,
+      "step": 3920
+    },
+    {
+      "epoch": 19.944303797468354,
+      "grad_norm": 637831.6626185304,
+      "learning_rate": 2.477279849576935e-07,
+      "logits/chosen": -1.1525195837020874,
+      "logits/rejected": -0.6883751153945923,
+      "logps/chosen": -37.11662673950195,
+      "logps/rejected": -576.5172729492188,
+      "loss": 14910.0938,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": 0.20432814955711365,
+      "rewards/margins": 0.5393826961517334,
+      "rewards/rejected": -0.33505457639694214,
+      "step": 3930
+    },
+    {
+      "epoch": 19.99493670886076,
+      "grad_norm": 926025.3002487151,
+      "learning_rate": 2.4694453149482917e-07,
+      "logits/chosen": 0.10631950944662094,
+      "logits/rejected": 0.8977824449539185,
+      "logps/chosen": -43.12347412109375,
+      "logps/rejected": -561.4703369140625,
+      "loss": 15041.1219,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": 0.18227019906044006,
+      "rewards/margins": 0.5218333005905151,
+      "rewards/rejected": -0.3395631015300751,
+      "step": 3940
     }
   ],
   "logging_steps": 10,
-  "max_steps": 3546,
+  "max_steps": 7092,
   "num_input_tokens_seen": 0,
-  "num_train_epochs": 18,
+  "num_train_epochs": 36,
   "save_steps": 500,
   "stateful_callbacks": {
     "TrainerControl": {

last-checkpoint/training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a753652c11ab0ea9bd892fb4f6a4c90dd38862fe7afe3d497f10eddbd97dad2f
+oid sha256:32798bab9af49bccdac87f37019ce9013d05d2970814716fe899a3a90e4f5fcb
 size 6584