| { | |
| "best_metric": 0.6289177536964417, | |
| "best_model_checkpoint": "models/llama-3.2-3b-sft-dpo/checkpoint-500", | |
| "epoch": 3.0, | |
| "eval_steps": 100, | |
| "global_step": 633, | |
| "is_hyper_param_search": false, | |
| "is_local_process_zero": true, | |
| "is_world_process_zero": true, | |
| "log_history": [ | |
| { | |
| "epoch": 0.004739336492890996, | |
| "grad_norm": 18.306584799400138, | |
| "learning_rate": 5.2631578947368416e-08, | |
| "logits/chosen": 1.1032867431640625, | |
| "logits/rejected": 1.1176480054855347, | |
| "logps/chosen": -175.54205322265625, | |
| "logps/rejected": -196.64266967773438, | |
| "loss": 1.0, | |
| "rewards/accuracies": 0.0, | |
| "rewards/chosen": 0.0, | |
| "rewards/margins": 0.0, | |
| "rewards/rejected": 0.0, | |
| "step": 1 | |
| }, | |
| { | |
| "epoch": 0.04739336492890995, | |
| "grad_norm": 18.19518017806804, | |
| "learning_rate": 5.263157894736842e-07, | |
| "logits/chosen": 0.6209686994552612, | |
| "logits/rejected": 0.7449740171432495, | |
| "logps/chosen": -350.8912658691406, | |
| "logps/rejected": -307.96142578125, | |
| "loss": 0.9979, | |
| "rewards/accuracies": 0.4861111044883728, | |
| "rewards/chosen": 0.00011829059076262638, | |
| "rewards/margins": 0.016186419874429703, | |
| "rewards/rejected": -0.016068127006292343, | |
| "step": 10 | |
| }, | |
| { | |
| "epoch": 0.0947867298578199, | |
| "grad_norm": 15.415652807377189, | |
| "learning_rate": 9.99993455114332e-07, | |
| "logits/chosen": 0.9229280352592468, | |
| "logits/rejected": 0.8609384298324585, | |
| "logps/chosen": -252.894775390625, | |
| "logps/rejected": -263.6702575683594, | |
| "loss": 0.9588, | |
| "rewards/accuracies": 0.7124999761581421, | |
| "rewards/chosen": 0.0033816881477832794, | |
| "rewards/margins": 0.16803663969039917, | |
| "rewards/rejected": -0.164654940366745, | |
| "step": 20 | |
| }, | |
| { | |
| "epoch": 0.14218009478672985, | |
| "grad_norm": 12.850588595957225, | |
| "learning_rate": 9.992082761369566e-07, | |
| "logits/chosen": 0.8715411424636841, | |
| "logits/rejected": 0.8170267343521118, | |
| "logps/chosen": -296.8494567871094, | |
| "logps/rejected": -305.7926025390625, | |
| "loss": 0.8133, | |
| "rewards/accuracies": 0.793749988079071, | |
| "rewards/chosen": 0.6128842830657959, | |
| "rewards/margins": 1.1374889612197876, | |
| "rewards/rejected": -0.5246046781539917, | |
| "step": 30 | |
| }, | |
| { | |
| "epoch": 0.1895734597156398, | |
| "grad_norm": 14.501186311778227, | |
| "learning_rate": 9.971164749660148e-07, | |
| "logits/chosen": 0.9155582189559937, | |
| "logits/rejected": 0.9567469358444214, | |
| "logps/chosen": -313.08514404296875, | |
| "logps/rejected": -309.0679626464844, | |
| "loss": 0.7405, | |
| "rewards/accuracies": 0.737500011920929, | |
| "rewards/chosen": 0.23792271316051483, | |
| "rewards/margins": 2.1163926124572754, | |
| "rewards/rejected": -1.878469467163086, | |
| "step": 40 | |
| }, | |
| { | |
| "epoch": 0.23696682464454977, | |
| "grad_norm": 11.740811645701724, | |
| "learning_rate": 9.937235266586424e-07, | |
| "logits/chosen": 0.6986435651779175, | |
| "logits/rejected": 0.8309999704360962, | |
| "logps/chosen": -319.8310852050781, | |
| "logps/rejected": -317.59918212890625, | |
| "loss": 0.6552, | |
| "rewards/accuracies": 0.793749988079071, | |
| "rewards/chosen": 0.6028285622596741, | |
| "rewards/margins": 3.663621425628662, | |
| "rewards/rejected": -3.060793161392212, | |
| "step": 50 | |
| }, | |
| { | |
| "epoch": 0.2843601895734597, | |
| "grad_norm": 14.434952077378005, | |
| "learning_rate": 9.890383118800284e-07, | |
| "logits/chosen": 0.7444020509719849, | |
| "logits/rejected": 0.7484663724899292, | |
| "logps/chosen": -327.59576416015625, | |
| "logps/rejected": -349.929931640625, | |
| "loss": 0.6285, | |
| "rewards/accuracies": 0.706250011920929, | |
| "rewards/chosen": 0.3002261221408844, | |
| "rewards/margins": 3.5275771617889404, | |
| "rewards/rejected": -3.227351427078247, | |
| "step": 60 | |
| }, | |
| { | |
| "epoch": 0.33175355450236965, | |
| "grad_norm": 10.030890442911925, | |
| "learning_rate": 9.830730936592615e-07, | |
| "logits/chosen": 0.7815200090408325, | |
| "logits/rejected": 0.7069059610366821, | |
| "logps/chosen": -252.94921875, | |
| "logps/rejected": -323.2224426269531, | |
| "loss": 0.6106, | |
| "rewards/accuracies": 0.7875000238418579, | |
| "rewards/chosen": 1.3401187658309937, | |
| "rewards/margins": 5.26017427444458, | |
| "rewards/rejected": -3.920055866241455, | |
| "step": 70 | |
| }, | |
| { | |
| "epoch": 0.3791469194312796, | |
| "grad_norm": 12.131364583934603, | |
| "learning_rate": 9.758434852922123e-07, | |
| "logits/chosen": 0.7100412249565125, | |
| "logits/rejected": 0.6621907353401184, | |
| "logps/chosen": -271.33331298828125, | |
| "logps/rejected": -328.0660705566406, | |
| "loss": 0.59, | |
| "rewards/accuracies": 0.71875, | |
| "rewards/chosen": 0.908360481262207, | |
| "rewards/margins": 4.926724910736084, | |
| "rewards/rejected": -4.018364429473877, | |
| "step": 80 | |
| }, | |
| { | |
| "epoch": 0.4265402843601896, | |
| "grad_norm": 11.822232959802975, | |
| "learning_rate": 9.673684094754685e-07, | |
| "logits/chosen": 0.6003296375274658, | |
| "logits/rejected": 0.6765642762184143, | |
| "logps/chosen": -293.85015869140625, | |
| "logps/rejected": -305.929443359375, | |
| "loss": 0.586, | |
| "rewards/accuracies": 0.768750011920929, | |
| "rewards/chosen": 1.231705904006958, | |
| "rewards/margins": 4.982685089111328, | |
| "rewards/rejected": -3.750978946685791, | |
| "step": 90 | |
| }, | |
| { | |
| "epoch": 0.47393364928909953, | |
| "grad_norm": 9.616291876594419, | |
| "learning_rate": 9.576700487782773e-07, | |
| "logits/chosen": 0.6642001867294312, | |
| "logits/rejected": 0.6596721410751343, | |
| "logps/chosen": -326.2373046875, | |
| "logps/rejected": -381.3326110839844, | |
| "loss": 0.5801, | |
| "rewards/accuracies": 0.824999988079071, | |
| "rewards/chosen": 1.7316535711288452, | |
| "rewards/margins": 6.260350704193115, | |
| "rewards/rejected": -4.5286970138549805, | |
| "step": 100 | |
| }, | |
| { | |
| "epoch": 0.47393364928909953, | |
| "eval_logits/chosen": 0.610289990901947, | |
| "eval_logits/rejected": 0.6783497929573059, | |
| "eval_logps/chosen": -339.33251953125, | |
| "eval_logps/rejected": -361.24346923828125, | |
| "eval_loss": 0.6839759349822998, | |
| "eval_rewards/accuracies": 0.6898733973503113, | |
| "eval_rewards/chosen": 0.6485355496406555, | |
| "eval_rewards/margins": 3.587477684020996, | |
| "eval_rewards/rejected": -2.9389421939849854, | |
| "eval_runtime": 76.922, | |
| "eval_samples_per_second": 32.5, | |
| "eval_steps_per_second": 1.027, | |
| "step": 100 | |
| }, | |
| { | |
| "epoch": 0.5213270142180095, | |
| "grad_norm": 11.519611398516883, | |
| "learning_rate": 9.467737875821367e-07, | |
| "logits/chosen": 0.659843385219574, | |
| "logits/rejected": 0.6010033488273621, | |
| "logps/chosen": -293.62200927734375, | |
| "logps/rejected": -334.9098205566406, | |
| "loss": 0.5742, | |
| "rewards/accuracies": 0.8062499761581421, | |
| "rewards/chosen": 1.1434353590011597, | |
| "rewards/margins": 5.331825256347656, | |
| "rewards/rejected": -4.188389301300049, | |
| "step": 110 | |
| }, | |
| { | |
| "epoch": 0.5687203791469194, | |
| "grad_norm": 10.75922014108817, | |
| "learning_rate": 9.347081456399957e-07, | |
| "logits/chosen": 0.6637296676635742, | |
| "logits/rejected": 0.5958945155143738, | |
| "logps/chosen": -272.2585144042969, | |
| "logps/rejected": -393.41949462890625, | |
| "loss": 0.5821, | |
| "rewards/accuracies": 0.800000011920929, | |
| "rewards/chosen": 0.9803568124771118, | |
| "rewards/margins": 6.413501739501953, | |
| "rewards/rejected": -5.433144569396973, | |
| "step": 120 | |
| }, | |
| { | |
| "epoch": 0.6161137440758294, | |
| "grad_norm": 11.497074098204886, | |
| "learning_rate": 9.215047034289715e-07, | |
| "logits/chosen": 0.6836856603622437, | |
| "logits/rejected": 0.6638469696044922, | |
| "logps/chosen": -275.0943603515625, | |
| "logps/rejected": -332.6889343261719, | |
| "loss": 0.5752, | |
| "rewards/accuracies": 0.800000011920929, | |
| "rewards/chosen": 1.4476346969604492, | |
| "rewards/margins": 6.094024658203125, | |
| "rewards/rejected": -4.646389961242676, | |
| "step": 130 | |
| }, | |
| { | |
| "epoch": 0.6635071090047393, | |
| "grad_norm": 9.658859904375, | |
| "learning_rate": 9.07198019491959e-07, | |
| "logits/chosen": 0.61662757396698, | |
| "logits/rejected": 0.5779851675033569, | |
| "logps/chosen": -272.382080078125, | |
| "logps/rejected": -355.6089172363281, | |
| "loss": 0.5468, | |
| "rewards/accuracies": 0.793749988079071, | |
| "rewards/chosen": 0.8889511227607727, | |
| "rewards/margins": 5.594452857971191, | |
| "rewards/rejected": -4.705502510070801, | |
| "step": 140 | |
| }, | |
| { | |
| "epoch": 0.7109004739336493, | |
| "grad_norm": 10.07652231167762, | |
| "learning_rate": 8.918255399844853e-07, | |
| "logits/chosen": 0.5373108983039856, | |
| "logits/rejected": 0.654308021068573, | |
| "logps/chosen": -330.0559997558594, | |
| "logps/rejected": -349.55224609375, | |
| "loss": 0.5738, | |
| "rewards/accuracies": 0.7562500238418579, | |
| "rewards/chosen": 0.3335852324962616, | |
| "rewards/margins": 4.550914287567139, | |
| "rewards/rejected": -4.217329502105713, | |
| "step": 150 | |
| }, | |
| { | |
| "epoch": 0.7582938388625592, | |
| "grad_norm": 8.965490487953566, | |
| "learning_rate": 8.754275006635572e-07, | |
| "logits/chosen": 0.565764844417572, | |
| "logits/rejected": 0.539226233959198, | |
| "logps/chosen": -269.29742431640625, | |
| "logps/rejected": -355.60589599609375, | |
| "loss": 0.5997, | |
| "rewards/accuracies": 0.731249988079071, | |
| "rewards/chosen": 0.5406277179718018, | |
| "rewards/margins": 5.479567527770996, | |
| "rewards/rejected": -4.938939571380615, | |
| "step": 160 | |
| }, | |
| { | |
| "epoch": 0.8056872037914692, | |
| "grad_norm": 9.437674903727038, | |
| "learning_rate": 8.580468215750391e-07, | |
| "logits/chosen": 0.6932438611984253, | |
| "logits/rejected": 0.636594831943512, | |
| "logps/chosen": -296.7684631347656, | |
| "logps/rejected": -367.45318603515625, | |
| "loss": 0.5783, | |
| "rewards/accuracies": 0.768750011920929, | |
| "rewards/chosen": 1.147369146347046, | |
| "rewards/margins": 5.5389909744262695, | |
| "rewards/rejected": -4.391622066497803, | |
| "step": 170 | |
| }, | |
| { | |
| "epoch": 0.8530805687203792, | |
| "grad_norm": 8.5658002946873, | |
| "learning_rate": 8.39728994715202e-07, | |
| "logits/chosen": 0.6020892858505249, | |
| "logits/rejected": 0.5168766379356384, | |
| "logps/chosen": -288.558349609375, | |
| "logps/rejected": -348.62640380859375, | |
| "loss": 0.5531, | |
| "rewards/accuracies": 0.731249988079071, | |
| "rewards/chosen": 0.6757786870002747, | |
| "rewards/margins": 5.149857997894287, | |
| "rewards/rejected": -4.474079132080078, | |
| "step": 180 | |
| }, | |
| { | |
| "epoch": 0.9004739336492891, | |
| "grad_norm": 11.065263225689659, | |
| "learning_rate": 8.20521964960477e-07, | |
| "logits/chosen": 0.6599653363227844, | |
| "logits/rejected": 0.6458830237388611, | |
| "logps/chosen": -289.4867858886719, | |
| "logps/rejected": -342.56243896484375, | |
| "loss": 0.5439, | |
| "rewards/accuracies": 0.7250000238418579, | |
| "rewards/chosen": 1.274778962135315, | |
| "rewards/margins": 6.3435516357421875, | |
| "rewards/rejected": -5.068772792816162, | |
| "step": 190 | |
| }, | |
| { | |
| "epoch": 0.9478672985781991, | |
| "grad_norm": 8.426424572195439, | |
| "learning_rate": 8.0047600457707e-07, | |
| "logits/chosen": 0.6277160048484802, | |
| "logits/rejected": 0.6192003488540649, | |
| "logps/chosen": -318.033447265625, | |
| "logps/rejected": -377.3500061035156, | |
| "loss": 0.537, | |
| "rewards/accuracies": 0.7749999761581421, | |
| "rewards/chosen": 1.3354051113128662, | |
| "rewards/margins": 6.755140781402588, | |
| "rewards/rejected": -5.419735431671143, | |
| "step": 200 | |
| }, | |
| { | |
| "epoch": 0.9478672985781991, | |
| "eval_logits/chosen": 0.494819700717926, | |
| "eval_logits/rejected": 0.5648438930511475, | |
| "eval_logps/chosen": -343.7730712890625, | |
| "eval_logps/rejected": -372.1695861816406, | |
| "eval_loss": 0.6514427661895752, | |
| "eval_rewards/accuracies": 0.7278481125831604, | |
| "eval_rewards/chosen": 0.20448331534862518, | |
| "eval_rewards/margins": 4.236032485961914, | |
| "eval_rewards/rejected": -4.031548976898193, | |
| "eval_runtime": 74.0508, | |
| "eval_samples_per_second": 33.761, | |
| "eval_steps_per_second": 1.067, | |
| "step": 200 | |
| }, | |
| { | |
| "epoch": 0.995260663507109, | |
| "grad_norm": 9.878709661135902, | |
| "learning_rate": 7.796435816388898e-07, | |
| "logits/chosen": 0.6760674118995667, | |
| "logits/rejected": 0.6518660187721252, | |
| "logps/chosen": -284.24749755859375, | |
| "logps/rejected": -363.0601501464844, | |
| "loss": 0.554, | |
| "rewards/accuracies": 0.7749999761581421, | |
| "rewards/chosen": 0.6821473836898804, | |
| "rewards/margins": 6.51880407333374, | |
| "rewards/rejected": -5.8366570472717285, | |
| "step": 210 | |
| }, | |
| { | |
| "epoch": 1.042654028436019, | |
| "grad_norm": 10.875728154843127, | |
| "learning_rate": 7.580792226981954e-07, | |
| "logits/chosen": 0.5221652984619141, | |
| "logits/rejected": 0.44479990005493164, | |
| "logps/chosen": -281.39190673828125, | |
| "logps/rejected": -370.33941650390625, | |
| "loss": 0.4911, | |
| "rewards/accuracies": 0.800000011920929, | |
| "rewards/chosen": 2.0442254543304443, | |
| "rewards/margins": 7.068573951721191, | |
| "rewards/rejected": -5.024348258972168, | |
| "step": 220 | |
| }, | |
| { | |
| "epoch": 1.0900473933649288, | |
| "grad_norm": 10.04148994728917, | |
| "learning_rate": 7.358393700684032e-07, | |
| "logits/chosen": 0.5540430545806885, | |
| "logits/rejected": 0.5128260850906372, | |
| "logps/chosen": -279.4583435058594, | |
| "logps/rejected": -350.32684326171875, | |
| "loss": 0.5022, | |
| "rewards/accuracies": 0.793749988079071, | |
| "rewards/chosen": 0.9357398152351379, | |
| "rewards/margins": 5.9159369468688965, | |
| "rewards/rejected": -4.980198383331299, | |
| "step": 230 | |
| }, | |
| { | |
| "epoch": 1.1374407582938388, | |
| "grad_norm": 11.466420945945197, | |
| "learning_rate": 7.129822340926043e-07, | |
| "logits/chosen": 0.5252267122268677, | |
| "logits/rejected": 0.6392233371734619, | |
| "logps/chosen": -300.5268859863281, | |
| "logps/rejected": -328.5356750488281, | |
| "loss": 0.4908, | |
| "rewards/accuracies": 0.8187500238418579, | |
| "rewards/chosen": 1.1534405946731567, | |
| "rewards/margins": 6.1857991218566895, | |
| "rewards/rejected": -5.032358169555664, | |
| "step": 240 | |
| }, | |
| { | |
| "epoch": 1.1848341232227488, | |
| "grad_norm": 9.714339627017372, | |
| "learning_rate": 6.895676407844586e-07, | |
| "logits/chosen": 0.5342652797698975, | |
| "logits/rejected": 0.5475658178329468, | |
| "logps/chosen": -275.02972412109375, | |
| "logps/rejected": -325.74993896484375, | |
| "loss": 0.4508, | |
| "rewards/accuracies": 0.8374999761581421, | |
| "rewards/chosen": 2.0915255546569824, | |
| "rewards/margins": 6.8750715255737305, | |
| "rewards/rejected": -4.783546447753906, | |
| "step": 250 | |
| }, | |
| { | |
| "epoch": 1.2322274881516588, | |
| "grad_norm": 8.702659887264469, | |
| "learning_rate": 6.656568752402521e-07, | |
| "logits/chosen": 0.4584909975528717, | |
| "logits/rejected": 0.5478152632713318, | |
| "logps/chosen": -314.6927185058594, | |
| "logps/rejected": -357.88226318359375, | |
| "loss": 0.4621, | |
| "rewards/accuracies": 0.8062499761581421, | |
| "rewards/chosen": 1.3858213424682617, | |
| "rewards/margins": 6.8659563064575195, | |
| "rewards/rejected": -5.480134963989258, | |
| "step": 260 | |
| }, | |
| { | |
| "epoch": 1.2796208530805688, | |
| "grad_norm": 10.924278197277149, | |
| "learning_rate": 6.413125212319663e-07, | |
| "logits/chosen": 0.6362992525100708, | |
| "logits/rejected": 0.6484791040420532, | |
| "logps/chosen": -285.7840270996094, | |
| "logps/rejected": -360.7676086425781, | |
| "loss": 0.4712, | |
| "rewards/accuracies": 0.8687499761581421, | |
| "rewards/chosen": 2.0224599838256836, | |
| "rewards/margins": 7.362783908843994, | |
| "rewards/rejected": -5.3403239250183105, | |
| "step": 270 | |
| }, | |
| { | |
| "epoch": 1.3270142180094786, | |
| "grad_norm": 9.286266066829205, | |
| "learning_rate": 6.165982974012104e-07, | |
| "logits/chosen": 0.48062658309936523, | |
| "logits/rejected": 0.4873732626438141, | |
| "logps/chosen": -345.07586669921875, | |
| "logps/rejected": -393.88165283203125, | |
| "loss": 0.4628, | |
| "rewards/accuracies": 0.793749988079071, | |
| "rewards/chosen": 1.449973225593567, | |
| "rewards/margins": 7.039644718170166, | |
| "rewards/rejected": -5.589670658111572, | |
| "step": 280 | |
| }, | |
| { | |
| "epoch": 1.3744075829383886, | |
| "grad_norm": 9.83819564198541, | |
| "learning_rate": 5.915788904827553e-07, | |
| "logits/chosen": 0.43026304244995117, | |
| "logits/rejected": 0.459343820810318, | |
| "logps/chosen": -294.733154296875, | |
| "logps/rejected": -363.80340576171875, | |
| "loss": 0.4507, | |
| "rewards/accuracies": 0.831250011920929, | |
| "rewards/chosen": 1.6585981845855713, | |
| "rewards/margins": 6.437933444976807, | |
| "rewards/rejected": -4.779335021972656, | |
| "step": 290 | |
| }, | |
| { | |
| "epoch": 1.4218009478672986, | |
| "grad_norm": 8.577071743246128, | |
| "learning_rate": 5.663197859941938e-07, | |
| "logits/chosen": 0.6086027026176453, | |
| "logits/rejected": 0.6251193881034851, | |
| "logps/chosen": -262.66644287109375, | |
| "logps/rejected": -320.42974853515625, | |
| "loss": 0.4787, | |
| "rewards/accuracies": 0.7875000238418579, | |
| "rewards/chosen": 1.298060655593872, | |
| "rewards/margins": 6.941515922546387, | |
| "rewards/rejected": -5.643455505371094, | |
| "step": 300 | |
| }, | |
| { | |
| "epoch": 1.4218009478672986, | |
| "eval_logits/chosen": 0.45885032415390015, | |
| "eval_logits/rejected": 0.5325651168823242, | |
| "eval_logps/chosen": -341.7187194824219, | |
| "eval_logps/rejected": -371.7361145019531, | |
| "eval_loss": 0.6386769413948059, | |
| "eval_rewards/accuracies": 0.7215189933776855, | |
| "eval_rewards/chosen": 0.40991881489753723, | |
| "eval_rewards/margins": 4.398120880126953, | |
| "eval_rewards/rejected": -3.98820161819458, | |
| "eval_runtime": 72.3153, | |
| "eval_samples_per_second": 34.571, | |
| "eval_steps_per_second": 1.092, | |
| "step": 300 | |
| }, | |
| { | |
| "epoch": 1.4691943127962086, | |
| "grad_norm": 12.642599504555136, | |
| "learning_rate": 5.408870968348749e-07, | |
| "logits/chosen": 0.46862930059432983, | |
| "logits/rejected": 0.45317015051841736, | |
| "logps/chosen": -269.1434631347656, | |
| "logps/rejected": -348.3428955078125, | |
| "loss": 0.4684, | |
| "rewards/accuracies": 0.8187500238418579, | |
| "rewards/chosen": 1.3798935413360596, | |
| "rewards/margins": 6.562399864196777, | |
| "rewards/rejected": -5.182506561279297, | |
| "step": 310 | |
| }, | |
| { | |
| "epoch": 1.5165876777251186, | |
| "grad_norm": 9.79584839845262, | |
| "learning_rate": 5.153473902427354e-07, | |
| "logits/chosen": 0.47858723998069763, | |
| "logits/rejected": 0.5644794702529907, | |
| "logps/chosen": -321.48345947265625, | |
| "logps/rejected": -343.6278991699219, | |
| "loss": 0.4803, | |
| "rewards/accuracies": 0.793749988079071, | |
| "rewards/chosen": 1.1607013940811157, | |
| "rewards/margins": 5.799595832824707, | |
| "rewards/rejected": -4.638894557952881, | |
| "step": 320 | |
| }, | |
| { | |
| "epoch": 1.5639810426540284, | |
| "grad_norm": 8.875212778872154, | |
| "learning_rate": 4.897675135619516e-07, | |
| "logits/chosen": 0.47927242517471313, | |
| "logits/rejected": 0.605729341506958, | |
| "logps/chosen": -296.8520812988281, | |
| "logps/rejected": -339.26220703125, | |
| "loss": 0.48, | |
| "rewards/accuracies": 0.793749988079071, | |
| "rewards/chosen": 1.206688404083252, | |
| "rewards/margins": 6.4211745262146, | |
| "rewards/rejected": -5.214486598968506, | |
| "step": 330 | |
| }, | |
| { | |
| "epoch": 1.6113744075829384, | |
| "grad_norm": 9.788751062324735, | |
| "learning_rate": 4.642144192774429e-07, | |
| "logits/chosen": 0.6517030000686646, | |
| "logits/rejected": 0.6343492269515991, | |
| "logps/chosen": -256.8311767578125, | |
| "logps/rejected": -318.10504150390625, | |
| "loss": 0.4687, | |
| "rewards/accuracies": 0.8374999761581421, | |
| "rewards/chosen": 1.4574129581451416, | |
| "rewards/margins": 7.180891513824463, | |
| "rewards/rejected": -5.723478317260742, | |
| "step": 340 | |
| }, | |
| { | |
| "epoch": 1.6587677725118484, | |
| "grad_norm": 8.123068784558978, | |
| "learning_rate": 4.387549897741825e-07, | |
| "logits/chosen": 0.43539008498191833, | |
| "logits/rejected": 0.4823547303676605, | |
| "logps/chosen": -322.7386474609375, | |
| "logps/rejected": -349.6393127441406, | |
| "loss": 0.4903, | |
| "rewards/accuracies": 0.8187500238418579, | |
| "rewards/chosen": 1.6534090042114258, | |
| "rewards/margins": 6.494222164154053, | |
| "rewards/rejected": -4.840813159942627, | |
| "step": 350 | |
| }, | |
| { | |
| "epoch": 1.7061611374407581, | |
| "grad_norm": 10.106462346167355, | |
| "learning_rate": 4.1345586227998634e-07, | |
| "logits/chosen": 0.4860106110572815, | |
| "logits/rejected": 0.48908883333206177, | |
| "logps/chosen": -289.710693359375, | |
| "logps/rejected": -384.22686767578125, | |
| "loss": 0.446, | |
| "rewards/accuracies": 0.887499988079071, | |
| "rewards/chosen": 1.587738275527954, | |
| "rewards/margins": 7.2089128494262695, | |
| "rewards/rejected": -5.6211748123168945, | |
| "step": 360 | |
| }, | |
| { | |
| "epoch": 1.7535545023696684, | |
| "grad_norm": 10.81635763601606, | |
| "learning_rate": 3.883832544499735e-07, | |
| "logits/chosen": 0.5913195013999939, | |
| "logits/rejected": 0.5606914758682251, | |
| "logps/chosen": -292.9503173828125, | |
| "logps/rejected": -390.93878173828125, | |
| "loss": 0.4592, | |
| "rewards/accuracies": 0.7749999761581421, | |
| "rewards/chosen": 1.614689588546753, | |
| "rewards/margins": 6.656731605529785, | |
| "rewards/rejected": -5.042041301727295, | |
| "step": 370 | |
| }, | |
| { | |
| "epoch": 1.8009478672985781, | |
| "grad_norm": 10.495084061438284, | |
| "learning_rate": 3.636027910492114e-07, | |
| "logits/chosen": 0.4658740162849426, | |
| "logits/rejected": 0.5308722257614136, | |
| "logps/chosen": -305.28753662109375, | |
| "logps/rejected": -352.7513122558594, | |
| "loss": 0.4648, | |
| "rewards/accuracies": 0.8062499761581421, | |
| "rewards/chosen": 1.0712064504623413, | |
| "rewards/margins": 6.167966365814209, | |
| "rewards/rejected": -5.096759796142578, | |
| "step": 380 | |
| }, | |
| { | |
| "epoch": 1.8483412322274881, | |
| "grad_norm": 11.413974134819627, | |
| "learning_rate": 3.3917933218718566e-07, | |
| "logits/chosen": 0.6185089349746704, | |
| "logits/rejected": 0.6838531494140625, | |
| "logps/chosen": -284.1628112792969, | |
| "logps/rejected": -333.17657470703125, | |
| "loss": 0.4426, | |
| "rewards/accuracies": 0.8125, | |
| "rewards/chosen": 1.4776874780654907, | |
| "rewards/margins": 6.398137092590332, | |
| "rewards/rejected": -4.920449733734131, | |
| "step": 390 | |
| }, | |
| { | |
| "epoch": 1.8957345971563981, | |
| "grad_norm": 9.664147195442332, | |
| "learning_rate": 3.151768035536698e-07, | |
| "logits/chosen": 0.6407091617584229, | |
| "logits/rejected": 0.6542560458183289, | |
| "logps/chosen": -284.20037841796875, | |
| "logps/rejected": -345.27880859375, | |
| "loss": 0.4559, | |
| "rewards/accuracies": 0.824999988079071, | |
| "rewards/chosen": 2.0247559547424316, | |
| "rewards/margins": 7.09304141998291, | |
| "rewards/rejected": -5.0682854652404785, | |
| "step": 400 | |
| }, | |
| { | |
| "epoch": 1.8957345971563981, | |
| "eval_logits/chosen": 0.41101595759391785, | |
| "eval_logits/rejected": 0.4840773642063141, | |
| "eval_logps/chosen": -338.1277160644531, | |
| "eval_logps/rejected": -368.54248046875, | |
| "eval_loss": 0.6332134008407593, | |
| "eval_rewards/accuracies": 0.7341772317886353, | |
| "eval_rewards/chosen": 0.7690173983573914, | |
| "eval_rewards/margins": 4.437857151031494, | |
| "eval_rewards/rejected": -3.668839931488037, | |
| "eval_runtime": 72.5998, | |
| "eval_samples_per_second": 34.435, | |
| "eval_steps_per_second": 1.088, | |
| "step": 400 | |
| }, | |
| { | |
| "epoch": 1.943127962085308, | |
| "grad_norm": 10.263641095491934, | |
| "learning_rate": 2.9165802910033603e-07, | |
| "logits/chosen": 0.5565508604049683, | |
| "logits/rejected": 0.5877315402030945, | |
| "logps/chosen": -328.7551574707031, | |
| "logps/rejected": -364.5121154785156, | |
| "loss": 0.4644, | |
| "rewards/accuracies": 0.78125, | |
| "rewards/chosen": 1.852020502090454, | |
| "rewards/margins": 6.0383710861206055, | |
| "rewards/rejected": -4.186350345611572, | |
| "step": 410 | |
| }, | |
| { | |
| "epoch": 1.9905213270142181, | |
| "grad_norm": 8.889403142715599, | |
| "learning_rate": 2.686845666060415e-07, | |
| "logits/chosen": 0.5102426409721375, | |
| "logits/rejected": 0.43454083800315857, | |
| "logps/chosen": -271.08160400390625, | |
| "logps/rejected": -369.26458740234375, | |
| "loss": 0.461, | |
| "rewards/accuracies": 0.793749988079071, | |
| "rewards/chosen": 1.6376615762710571, | |
| "rewards/margins": 7.588493347167969, | |
| "rewards/rejected": -5.950831413269043, | |
| "step": 420 | |
| }, | |
| { | |
| "epoch": 2.037914691943128, | |
| "grad_norm": 7.4495856256114195, | |
| "learning_rate": 2.4631654655618287e-07, | |
| "logits/chosen": 0.37354058027267456, | |
| "logits/rejected": 0.4436867833137512, | |
| "logps/chosen": -310.15802001953125, | |
| "logps/rejected": -382.03253173828125, | |
| "loss": 0.3945, | |
| "rewards/accuracies": 0.8125, | |
| "rewards/chosen": 1.8288238048553467, | |
| "rewards/margins": 7.114483833312988, | |
| "rewards/rejected": -5.2856597900390625, | |
| "step": 430 | |
| }, | |
| { | |
| "epoch": 2.085308056872038, | |
| "grad_norm": 8.829254132221473, | |
| "learning_rate": 2.2461251475783155e-07, | |
| "logits/chosen": 0.5162326693534851, | |
| "logits/rejected": 0.4021889567375183, | |
| "logps/chosen": -288.923095703125, | |
| "logps/rejected": -389.34979248046875, | |
| "loss": 0.3748, | |
| "rewards/accuracies": 0.8500000238418579, | |
| "rewards/chosen": 1.8741111755371094, | |
| "rewards/margins": 7.6665802001953125, | |
| "rewards/rejected": -5.792468547821045, | |
| "step": 440 | |
| }, | |
| { | |
| "epoch": 2.132701421800948, | |
| "grad_norm": 8.156529944948277, | |
| "learning_rate": 2.0362927910258986e-07, | |
| "logits/chosen": 0.45688456296920776, | |
| "logits/rejected": 0.4526469111442566, | |
| "logps/chosen": -253.50131225585938, | |
| "logps/rejected": -349.1957702636719, | |
| "loss": 0.4147, | |
| "rewards/accuracies": 0.875, | |
| "rewards/chosen": 2.0875327587127686, | |
| "rewards/margins": 8.09435749053955, | |
| "rewards/rejected": -6.006823539733887, | |
| "step": 450 | |
| }, | |
| { | |
| "epoch": 2.1800947867298577, | |
| "grad_norm": 7.824692642426332, | |
| "learning_rate": 1.8342176087824573e-07, | |
| "logits/chosen": 0.4325633645057678, | |
| "logits/rejected": 0.3565566837787628, | |
| "logps/chosen": -284.46624755859375, | |
| "logps/rejected": -372.12091064453125, | |
| "loss": 0.3992, | |
| "rewards/accuracies": 0.8500000238418579, | |
| "rewards/chosen": 1.8221031427383423, | |
| "rewards/margins": 7.619426727294922, | |
| "rewards/rejected": -5.797322750091553, | |
| "step": 460 | |
| }, | |
| { | |
| "epoch": 2.227488151658768, | |
| "grad_norm": 13.407256371457692, | |
| "learning_rate": 1.6404285101840565e-07, | |
| "logits/chosen": 0.3386808931827545, | |
| "logits/rejected": 0.47734910249710083, | |
| "logps/chosen": -331.7251892089844, | |
| "logps/rejected": -367.4866638183594, | |
| "loss": 0.3822, | |
| "rewards/accuracies": 0.8374999761581421, | |
| "rewards/chosen": 1.9130542278289795, | |
| "rewards/margins": 7.692631721496582, | |
| "rewards/rejected": -5.779577732086182, | |
| "step": 470 | |
| }, | |
| { | |
| "epoch": 2.2748815165876777, | |
| "grad_norm": 10.86707059625683, | |
| "learning_rate": 1.455432716663517e-07, | |
| "logits/chosen": 0.36686116456985474, | |
| "logits/rejected": 0.48829737305641174, | |
| "logps/chosen": -285.77008056640625, | |
| "logps/rejected": -328.3174743652344, | |
| "loss": 0.4089, | |
| "rewards/accuracies": 0.856249988079071, | |
| "rewards/chosen": 1.7794748544692993, | |
| "rewards/margins": 6.214818477630615, | |
| "rewards/rejected": -4.435343265533447, | |
| "step": 480 | |
| }, | |
| { | |
| "epoch": 2.322274881516588, | |
| "grad_norm": 9.830177502454013, | |
| "learning_rate": 1.2797144341546883e-07, | |
| "logits/chosen": 0.3986554741859436, | |
| "logits/rejected": 0.44396382570266724, | |
| "logps/chosen": -321.13818359375, | |
| "logps/rejected": -390.934326171875, | |
| "loss": 0.4219, | |
| "rewards/accuracies": 0.8187500238418579, | |
| "rewards/chosen": 1.6029850244522095, | |
| "rewards/margins": 7.5643768310546875, | |
| "rewards/rejected": -5.961391448974609, | |
| "step": 490 | |
| }, | |
| { | |
| "epoch": 2.3696682464454977, | |
| "grad_norm": 9.42905977432162, | |
| "learning_rate": 1.1137335857372043e-07, | |
| "logits/chosen": 0.4437794089317322, | |
| "logits/rejected": 0.42870789766311646, | |
| "logps/chosen": -287.81451416015625, | |
| "logps/rejected": -374.01873779296875, | |
| "loss": 0.4028, | |
| "rewards/accuracies": 0.856249988079071, | |
| "rewards/chosen": 2.1330111026763916, | |
| "rewards/margins": 7.767390251159668, | |
| "rewards/rejected": -5.6343793869018555, | |
| "step": 500 | |
| }, | |
| { | |
| "epoch": 2.3696682464454977, | |
| "eval_logits/chosen": 0.3730663061141968, | |
| "eval_logits/rejected": 0.4475269019603729, | |
| "eval_logps/chosen": -338.3392028808594, | |
| "eval_logps/rejected": -370.232666015625, | |
| "eval_loss": 0.6289177536964417, | |
| "eval_rewards/accuracies": 0.7405063509941101, | |
| "eval_rewards/chosen": 0.7478683590888977, | |
| "eval_rewards/margins": 4.585729122161865, | |
| "eval_rewards/rejected": -3.8378612995147705, | |
| "eval_runtime": 73.3012, | |
| "eval_samples_per_second": 34.106, | |
| "eval_steps_per_second": 1.078, | |
| "step": 500 | |
| }, | |
| { | |
| "epoch": 2.4170616113744074, | |
| "grad_norm": 10.06462647313331, | |
| "learning_rate": 9.579246078389403e-08, | |
| "logits/chosen": 0.5295278429985046, | |
| "logits/rejected": 0.43623122572898865, | |
| "logps/chosen": -258.68963623046875, | |
| "logps/rejected": -339.7721252441406, | |
| "loss": 0.3858, | |
| "rewards/accuracies": 0.84375, | |
| "rewards/chosen": 1.592254400253296, | |
| "rewards/margins": 7.2217698097229, | |
| "rewards/rejected": -5.629514694213867, | |
| "step": 510 | |
| }, | |
| { | |
| "epoch": 2.4644549763033177, | |
| "grad_norm": 9.022052721765009, | |
| "learning_rate": 8.126953131469228e-08, | |
| "logits/chosen": 0.44106584787368774, | |
| "logits/rejected": 0.39466392993927, | |
| "logps/chosen": -303.3637390136719, | |
| "logps/rejected": -370.74114990234375, | |
| "loss": 0.4143, | |
| "rewards/accuracies": 0.875, | |
| "rewards/chosen": 1.8263496160507202, | |
| "rewards/margins": 7.823184013366699, | |
| "rewards/rejected": -5.996834754943848, | |
| "step": 520 | |
| }, | |
| { | |
| "epoch": 2.5118483412322274, | |
| "grad_norm": 8.021054640921763, | |
| "learning_rate": 6.784258232029472e-08, | |
| "logits/chosen": 0.3634105622768402, | |
| "logits/rejected": 0.3859165608882904, | |
| "logps/chosen": -307.2467041015625, | |
| "logps/rejected": -376.1995849609375, | |
| "loss": 0.3822, | |
| "rewards/accuracies": 0.8687499761581421, | |
| "rewards/chosen": 2.497091770172119, | |
| "rewards/margins": 7.9943437576293945, | |
| "rewards/rejected": -5.497252464294434, | |
| "step": 530 | |
| }, | |
| { | |
| "epoch": 2.5592417061611377, | |
| "grad_norm": 10.013425700067337, | |
| "learning_rate": 5.554675734776665e-08, | |
| "logits/chosen": 0.5024563074111938, | |
| "logits/rejected": 0.5056658387184143, | |
| "logps/chosen": -276.1619567871094, | |
| "logps/rejected": -368.4447021484375, | |
| "loss": 0.4035, | |
| "rewards/accuracies": 0.831250011920929, | |
| "rewards/chosen": 1.820339560508728, | |
| "rewards/margins": 8.141976356506348, | |
| "rewards/rejected": -6.321636199951172, | |
| "step": 540 | |
| }, | |
| { | |
| "epoch": 2.6066350710900474, | |
| "grad_norm": 9.209955480260117, | |
| "learning_rate": 4.4414239352730867e-08, | |
| "logits/chosen": 0.42310771346092224, | |
| "logits/rejected": 0.48689502477645874, | |
| "logps/chosen": -313.3210754394531, | |
| "logps/rejected": -351.4210205078125, | |
| "loss": 0.406, | |
| "rewards/accuracies": 0.893750011920929, | |
| "rewards/chosen": 2.1306679248809814, | |
| "rewards/margins": 7.7195258140563965, | |
| "rewards/rejected": -5.588858127593994, | |
| "step": 550 | |
| }, | |
| { | |
| "epoch": 2.654028436018957, | |
| "grad_norm": 9.959818332708023, | |
| "learning_rate": 3.447416646405632e-08, | |
| "logits/chosen": 0.5685544610023499, | |
| "logits/rejected": 0.5256290435791016, | |
| "logps/chosen": -287.7798156738281, | |
| "logps/rejected": -380.33685302734375, | |
| "loss": 0.4009, | |
| "rewards/accuracies": 0.893750011920929, | |
| "rewards/chosen": 1.8459497690200806, | |
| "rewards/margins": 7.295513153076172, | |
| "rewards/rejected": -5.449563503265381, | |
| "step": 560 | |
| }, | |
| { | |
| "epoch": 2.7014218009478674, | |
| "grad_norm": 8.593809820816018, | |
| "learning_rate": 2.575255571804391e-08, | |
| "logits/chosen": 0.41258078813552856, | |
| "logits/rejected": 0.4132450222969055, | |
| "logps/chosen": -287.94476318359375, | |
| "logps/rejected": -369.03656005859375, | |
| "loss": 0.4, | |
| "rewards/accuracies": 0.8125, | |
| "rewards/chosen": 1.5231783390045166, | |
| "rewards/margins": 7.392594814300537, | |
| "rewards/rejected": -5.8694167137146, | |
| "step": 570 | |
| }, | |
| { | |
| "epoch": 2.748815165876777, | |
| "grad_norm": 9.646946039027634, | |
| "learning_rate": 1.8272234961725084e-08, | |
| "logits/chosen": 0.48128992319107056, | |
| "logits/rejected": 0.4887717366218567, | |
| "logps/chosen": -303.7729797363281, | |
| "logps/rejected": -359.5372314453125, | |
| "loss": 0.3912, | |
| "rewards/accuracies": 0.90625, | |
| "rewards/chosen": 2.173060655593872, | |
| "rewards/margins": 8.012847900390625, | |
| "rewards/rejected": -5.839787006378174, | |
| "step": 580 | |
| }, | |
| { | |
| "epoch": 2.7962085308056874, | |
| "grad_norm": 11.09612482230785, | |
| "learning_rate": 1.2052783103508102e-08, | |
| "logits/chosen": 0.5081132650375366, | |
| "logits/rejected": 0.5602059364318848, | |
| "logps/chosen": -270.61737060546875, | |
| "logps/rejected": -335.85577392578125, | |
| "loss": 0.3991, | |
| "rewards/accuracies": 0.8125, | |
| "rewards/chosen": 1.619431495666504, | |
| "rewards/margins": 6.8268561363220215, | |
| "rewards/rejected": -5.207424163818359, | |
| "step": 590 | |
| }, | |
| { | |
| "epoch": 2.843601895734597, | |
| "grad_norm": 8.273064520857158, | |
| "learning_rate": 7.1104788675613315e-09, | |
| "logits/chosen": 0.32943224906921387, | |
| "logits/rejected": 0.4085375666618347, | |
| "logps/chosen": -288.88995361328125, | |
| "logps/rejected": -364.12860107421875, | |
| "loss": 0.4029, | |
| "rewards/accuracies": 0.893750011920929, | |
| "rewards/chosen": 2.0637223720550537, | |
| "rewards/margins": 7.937726020812988, | |
| "rewards/rejected": -5.874002933502197, | |
| "step": 600 | |
| }, | |
| { | |
| "epoch": 2.843601895734597, | |
| "eval_logits/chosen": 0.38198891282081604, | |
| "eval_logits/rejected": 0.45711585879325867, | |
| "eval_logps/chosen": -337.3143310546875, | |
| "eval_logps/rejected": -368.9125061035156, | |
| "eval_loss": 0.6283919215202332, | |
| "eval_rewards/accuracies": 0.7436708807945251, | |
| "eval_rewards/chosen": 0.8503568768501282, | |
| "eval_rewards/margins": 4.556199073791504, | |
| "eval_rewards/rejected": -3.7058422565460205, | |
| "eval_runtime": 73.7958, | |
| "eval_samples_per_second": 33.877, | |
| "eval_steps_per_second": 1.071, | |
| "step": 600 | |
| }, | |
| { | |
| "epoch": 2.890995260663507, | |
| "grad_norm": 9.238913123295514, | |
| "learning_rate": 3.4582581860612137e-09, | |
| "logits/chosen": 0.43385523557662964, | |
| "logits/rejected": 0.43230634927749634, | |
| "logps/chosen": -292.0911865234375, | |
| "logps/rejected": -353.61590576171875, | |
| "loss": 0.3884, | |
| "rewards/accuracies": 0.831250011920929, | |
| "rewards/chosen": 1.989989995956421, | |
| "rewards/margins": 6.724064826965332, | |
| "rewards/rejected": -4.734074115753174, | |
| "step": 610 | |
| }, | |
| { | |
| "epoch": 2.938388625592417, | |
| "grad_norm": 9.407237089972764, | |
| "learning_rate": 1.1056803408273085e-09, | |
| "logits/chosen": 0.48387131094932556, | |
| "logits/rejected": 0.4587581753730774, | |
| "logps/chosen": -282.6869201660156, | |
| "logps/rejected": -344.5205078125, | |
| "loss": 0.4089, | |
| "rewards/accuracies": 0.8374999761581421, | |
| "rewards/chosen": 1.806133508682251, | |
| "rewards/margins": 7.467283725738525, | |
| "rewards/rejected": -5.661149978637695, | |
| "step": 620 | |
| }, | |
| { | |
| "epoch": 2.985781990521327, | |
| "grad_norm": 8.481488205996529, | |
| "learning_rate": 5.890294296428955e-11, | |
| "logits/chosen": 0.44664233922958374, | |
| "logits/rejected": 0.5504810810089111, | |
| "logps/chosen": -319.47119140625, | |
| "logps/rejected": -348.36090087890625, | |
| "loss": 0.3848, | |
| "rewards/accuracies": 0.862500011920929, | |
| "rewards/chosen": 2.1828243732452393, | |
| "rewards/margins": 6.884246826171875, | |
| "rewards/rejected": -4.701422214508057, | |
| "step": 630 | |
| }, | |
| { | |
| "epoch": 3.0, | |
| "step": 633, | |
| "total_flos": 0.0, | |
| "train_loss": 0.5009220597491634, | |
| "train_runtime": 6227.6413, | |
| "train_samples_per_second": 13.002, | |
| "train_steps_per_second": 0.102 | |
| } | |
| ], | |
| "logging_steps": 10, | |
| "max_steps": 633, | |
| "num_input_tokens_seen": 0, | |
| "num_train_epochs": 3, | |
| "save_steps": 500, | |
| "stateful_callbacks": { | |
| "TrainerControl": { | |
| "args": { | |
| "should_epoch_stop": false, | |
| "should_evaluate": false, | |
| "should_log": false, | |
| "should_save": true, | |
| "should_training_stop": true | |
| }, | |
| "attributes": {} | |
| } | |
| }, | |
| "total_flos": 0.0, | |
| "train_batch_size": 4, | |
| "trial_name": null, | |
| "trial_params": null | |
| } | |