| { | |
| "best_metric": 1.3293771743774414, | |
| "best_model_checkpoint": "saves/Gemma-2B-It/lora/orpo-salt/checkpoint-1500", | |
| "epoch": 2.9969690846635686, | |
| "eval_steps": 500, | |
| "global_step": 1854, | |
| "is_hyper_param_search": false, | |
| "is_local_process_zero": true, | |
| "is_world_process_zero": true, | |
| "log_history": [ | |
| { | |
| "epoch": 0.01616488179430188, | |
| "grad_norm": 3.7809898853302, | |
| "learning_rate": 4.999648198770648e-06, | |
| "logits/chosen": -21.99495506286621, | |
| "logits/rejected": -22.17684555053711, | |
| "logps/chosen": -2.1064062118530273, | |
| "logps/rejected": -2.4076905250549316, | |
| "loss": 2.18, | |
| "odds_ratio_loss": 0.7356098294258118, | |
| "rewards/accuracies": 0.581250011920929, | |
| "rewards/chosen": -0.21064062416553497, | |
| "rewards/margins": 0.03012842871248722, | |
| "rewards/rejected": -0.24076907336711884, | |
| "sft_loss": 2.1064062118530273, | |
| "step": 10 | |
| }, | |
| { | |
| "epoch": 0.03232976358860376, | |
| "grad_norm": 2.3069310188293457, | |
| "learning_rate": 4.998578646361359e-06, | |
| "logits/chosen": -21.845182418823242, | |
| "logits/rejected": -21.866207122802734, | |
| "logps/chosen": -2.080873489379883, | |
| "logps/rejected": -2.124648332595825, | |
| "loss": 2.1663, | |
| "odds_ratio_loss": 0.8539472818374634, | |
| "rewards/accuracies": 0.5687500238418579, | |
| "rewards/chosen": -0.20808735489845276, | |
| "rewards/margins": 0.004377496428787708, | |
| "rewards/rejected": -0.2124648541212082, | |
| "sft_loss": 2.080873489379883, | |
| "step": 20 | |
| }, | |
| { | |
| "epoch": 0.04849464538290564, | |
| "grad_norm": 5.5587544441223145, | |
| "learning_rate": 4.996791614004449e-06, | |
| "logits/chosen": -21.90616226196289, | |
| "logits/rejected": -21.923837661743164, | |
| "logps/chosen": -2.2512574195861816, | |
| "logps/rejected": -2.225501537322998, | |
| "loss": 2.3454, | |
| "odds_ratio_loss": 0.9413110017776489, | |
| "rewards/accuracies": 0.46875, | |
| "rewards/chosen": -0.22512571513652802, | |
| "rewards/margins": -0.0025755497626960278, | |
| "rewards/rejected": -0.2225501835346222, | |
| "sft_loss": 2.2512574195861816, | |
| "step": 30 | |
| }, | |
| { | |
| "epoch": 0.06465952717720752, | |
| "grad_norm": 3.6638951301574707, | |
| "learning_rate": 4.994287614855618e-06, | |
| "logits/chosen": -21.935110092163086, | |
| "logits/rejected": -22.104324340820312, | |
| "logps/chosen": -2.171135425567627, | |
| "logps/rejected": -2.2752678394317627, | |
| "loss": 2.267, | |
| "odds_ratio_loss": 0.9586257934570312, | |
| "rewards/accuracies": 0.48750001192092896, | |
| "rewards/chosen": -0.21711353957653046, | |
| "rewards/margins": 0.010413245297968388, | |
| "rewards/rejected": -0.22752678394317627, | |
| "sft_loss": 2.171135425567627, | |
| "step": 40 | |
| }, | |
| { | |
| "epoch": 0.0808244089715094, | |
| "grad_norm": 6.454894065856934, | |
| "learning_rate": 4.991067367951343e-06, | |
| "logits/chosen": -22.541332244873047, | |
| "logits/rejected": -22.472043991088867, | |
| "logps/chosen": -2.1005358695983887, | |
| "logps/rejected": -2.1757073402404785, | |
| "loss": 2.1825, | |
| "odds_ratio_loss": 0.8198438882827759, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": -0.21005356311798096, | |
| "rewards/margins": 0.007517138961702585, | |
| "rewards/rejected": -0.2175707072019577, | |
| "sft_loss": 2.1005358695983887, | |
| "step": 50 | |
| }, | |
| { | |
| "epoch": 0.09698929076581128, | |
| "grad_norm": 2.4648330211639404, | |
| "learning_rate": 4.987131798002389e-06, | |
| "logits/chosen": -22.33713722229004, | |
| "logits/rejected": -22.142881393432617, | |
| "logps/chosen": -2.0364699363708496, | |
| "logps/rejected": -2.22188138961792, | |
| "loss": 2.1252, | |
| "odds_ratio_loss": 0.887610912322998, | |
| "rewards/accuracies": 0.53125, | |
| "rewards/chosen": -0.20364701747894287, | |
| "rewards/margins": 0.018541133031249046, | |
| "rewards/rejected": -0.22218814492225647, | |
| "sft_loss": 2.0364699363708496, | |
| "step": 60 | |
| }, | |
| { | |
| "epoch": 0.11315417256011315, | |
| "grad_norm": 4.808262825012207, | |
| "learning_rate": 4.982482035128285e-06, | |
| "logits/chosen": -22.240699768066406, | |
| "logits/rejected": -22.348970413208008, | |
| "logps/chosen": -2.144533634185791, | |
| "logps/rejected": -2.3089094161987305, | |
| "loss": 2.2393, | |
| "odds_ratio_loss": 0.9479582905769348, | |
| "rewards/accuracies": 0.512499988079071, | |
| "rewards/chosen": -0.21445336937904358, | |
| "rewards/margins": 0.01643756404519081, | |
| "rewards/rejected": -0.23089094460010529, | |
| "sft_loss": 2.144533634185791, | |
| "step": 70 | |
| }, | |
| { | |
| "epoch": 0.12931905435441504, | |
| "grad_norm": 5.7843122482299805, | |
| "learning_rate": 4.9771194145328e-06, | |
| "logits/chosen": -22.19432258605957, | |
| "logits/rejected": -22.165550231933594, | |
| "logps/chosen": -1.7035343647003174, | |
| "logps/rejected": -2.021710157394409, | |
| "loss": 1.7739, | |
| "odds_ratio_loss": 0.7040209174156189, | |
| "rewards/accuracies": 0.5687500238418579, | |
| "rewards/chosen": -0.17035342752933502, | |
| "rewards/margins": 0.03181760013103485, | |
| "rewards/rejected": -0.20217104256153107, | |
| "sft_loss": 1.7035343647003174, | |
| "step": 80 | |
| }, | |
| { | |
| "epoch": 0.1454839361487169, | |
| "grad_norm": 10.499777793884277, | |
| "learning_rate": 4.971045476120532e-06, | |
| "logits/chosen": -22.63768196105957, | |
| "logits/rejected": -22.5053768157959, | |
| "logps/chosen": -1.7764383554458618, | |
| "logps/rejected": -1.8638198375701904, | |
| "loss": 1.8559, | |
| "odds_ratio_loss": 0.7946864366531372, | |
| "rewards/accuracies": 0.46875, | |
| "rewards/chosen": -0.17764383554458618, | |
| "rewards/margins": 0.008738175965845585, | |
| "rewards/rejected": -0.1863820105791092, | |
| "sft_loss": 1.7764383554458618, | |
| "step": 90 | |
| }, | |
| { | |
| "epoch": 0.1616488179430188, | |
| "grad_norm": 2.4865188598632812, | |
| "learning_rate": 4.964261964054713e-06, | |
| "logits/chosen": -22.539587020874023, | |
| "logits/rejected": -22.56546401977539, | |
| "logps/chosen": -1.846840262413025, | |
| "logps/rejected": -2.0120983123779297, | |
| "loss": 1.9298, | |
| "odds_ratio_loss": 0.829124927520752, | |
| "rewards/accuracies": 0.48124998807907104, | |
| "rewards/chosen": -0.18468405306339264, | |
| "rewards/margins": 0.016525788232684135, | |
| "rewards/rejected": -0.20120985805988312, | |
| "sft_loss": 1.846840262413025, | |
| "step": 100 | |
| }, | |
| { | |
| "epoch": 0.17781369973732067, | |
| "grad_norm": 8.431103706359863, | |
| "learning_rate": 4.956770826256372e-06, | |
| "logits/chosen": -22.575443267822266, | |
| "logits/rejected": -22.463134765625, | |
| "logps/chosen": -1.6286426782608032, | |
| "logps/rejected": -1.7305223941802979, | |
| "loss": 1.7064, | |
| "odds_ratio_loss": 0.7774398922920227, | |
| "rewards/accuracies": 0.512499988079071, | |
| "rewards/chosen": -0.16286428272724152, | |
| "rewards/margins": 0.010187968611717224, | |
| "rewards/rejected": -0.17305223643779755, | |
| "sft_loss": 1.6286426782608032, | |
| "step": 110 | |
| }, | |
| { | |
| "epoch": 0.19397858153162256, | |
| "grad_norm": 2.32936429977417, | |
| "learning_rate": 4.94857421384497e-06, | |
| "logits/chosen": -22.672468185424805, | |
| "logits/rejected": -22.749752044677734, | |
| "logps/chosen": -1.6404567956924438, | |
| "logps/rejected": -1.8715522289276123, | |
| "loss": 1.7191, | |
| "odds_ratio_loss": 0.7863814830780029, | |
| "rewards/accuracies": 0.5625, | |
| "rewards/chosen": -0.16404569149017334, | |
| "rewards/margins": 0.023109545931220055, | |
| "rewards/rejected": -0.18715524673461914, | |
| "sft_loss": 1.6404567956924438, | |
| "step": 120 | |
| }, | |
| { | |
| "epoch": 0.21014346332592443, | |
| "grad_norm": 2.062532663345337, | |
| "learning_rate": 4.939674480520701e-06, | |
| "logits/chosen": -22.727245330810547, | |
| "logits/rejected": -22.860414505004883, | |
| "logps/chosen": -1.5943523645401, | |
| "logps/rejected": -1.7242279052734375, | |
| "loss": 1.6786, | |
| "odds_ratio_loss": 0.8422672152519226, | |
| "rewards/accuracies": 0.4937500059604645, | |
| "rewards/chosen": -0.1594352275133133, | |
| "rewards/margins": 0.012987568974494934, | |
| "rewards/rejected": -0.17242279648780823, | |
| "sft_loss": 1.5943523645401, | |
| "step": 130 | |
| }, | |
| { | |
| "epoch": 0.2263083451202263, | |
| "grad_norm": 1.7461190223693848, | |
| "learning_rate": 4.930074181888613e-06, | |
| "logits/chosen": -22.773456573486328, | |
| "logits/rejected": -22.877899169921875, | |
| "logps/chosen": -1.680092453956604, | |
| "logps/rejected": -1.8227007389068604, | |
| "loss": 1.7527, | |
| "odds_ratio_loss": 0.7260502576828003, | |
| "rewards/accuracies": 0.543749988079071, | |
| "rewards/chosen": -0.16800925135612488, | |
| "rewards/margins": 0.014260831288993359, | |
| "rewards/rejected": -0.1822700798511505, | |
| "sft_loss": 1.680092453956604, | |
| "step": 140 | |
| }, | |
| { | |
| "epoch": 0.2424732269145282, | |
| "grad_norm": 3.3798775672912598, | |
| "learning_rate": 4.91977607472475e-06, | |
| "logits/chosen": -23.01150131225586, | |
| "logits/rejected": -22.988527297973633, | |
| "logps/chosen": -1.5780761241912842, | |
| "logps/rejected": -1.6528291702270508, | |
| "loss": 1.6563, | |
| "odds_ratio_loss": 0.7826321721076965, | |
| "rewards/accuracies": 0.48124998807907104, | |
| "rewards/chosen": -0.1578076183795929, | |
| "rewards/margins": 0.007475291378796101, | |
| "rewards/rejected": -0.16528290510177612, | |
| "sft_loss": 1.5780761241912842, | |
| "step": 150 | |
| }, | |
| { | |
| "epoch": 0.2586381087088301, | |
| "grad_norm": 1.8430626392364502, | |
| "learning_rate": 4.908783116184534e-06, | |
| "logits/chosen": -22.70990562438965, | |
| "logits/rejected": -22.700252532958984, | |
| "logps/chosen": -1.5454621315002441, | |
| "logps/rejected": -1.7210956811904907, | |
| "loss": 1.614, | |
| "odds_ratio_loss": 0.6857101917266846, | |
| "rewards/accuracies": 0.53125, | |
| "rewards/chosen": -0.15454623103141785, | |
| "rewards/margins": 0.017563361674547195, | |
| "rewards/rejected": -0.17210958898067474, | |
| "sft_loss": 1.5454621315002441, | |
| "step": 160 | |
| }, | |
| { | |
| "epoch": 0.27480299050313195, | |
| "grad_norm": 1.635880470275879, | |
| "learning_rate": 4.897098462953598e-06, | |
| "logits/chosen": -23.044336318969727, | |
| "logits/rejected": -23.024044036865234, | |
| "logps/chosen": -1.457194447517395, | |
| "logps/rejected": -1.7030904293060303, | |
| "loss": 1.5297, | |
| "odds_ratio_loss": 0.7254581451416016, | |
| "rewards/accuracies": 0.581250011920929, | |
| "rewards/chosen": -0.14571943879127502, | |
| "rewards/margins": 0.02458963170647621, | |
| "rewards/rejected": -0.17030906677246094, | |
| "sft_loss": 1.457194447517395, | |
| "step": 170 | |
| }, | |
| { | |
| "epoch": 0.2909678722974338, | |
| "grad_norm": 1.3249585628509521, | |
| "learning_rate": 4.884725470341331e-06, | |
| "logits/chosen": -23.00472640991211, | |
| "logits/rejected": -23.162250518798828, | |
| "logps/chosen": -1.3314763307571411, | |
| "logps/rejected": -1.5617011785507202, | |
| "loss": 1.3961, | |
| "odds_ratio_loss": 0.6465403437614441, | |
| "rewards/accuracies": 0.6187499761581421, | |
| "rewards/chosen": -0.13314762711524963, | |
| "rewards/margins": 0.023022485896945, | |
| "rewards/rejected": -0.15617012977600098, | |
| "sft_loss": 1.3314763307571411, | |
| "step": 180 | |
| }, | |
| { | |
| "epoch": 0.3071327540917357, | |
| "grad_norm": 4.37128210067749, | |
| "learning_rate": 4.871667691317377e-06, | |
| "logits/chosen": -23.25027847290039, | |
| "logits/rejected": -23.159801483154297, | |
| "logps/chosen": -1.5023282766342163, | |
| "logps/rejected": -1.5170981884002686, | |
| "loss": 1.5883, | |
| "odds_ratio_loss": 0.8601799011230469, | |
| "rewards/accuracies": 0.46875, | |
| "rewards/chosen": -0.15023282170295715, | |
| "rewards/margins": 0.0014769884292036295, | |
| "rewards/rejected": -0.15170980989933014, | |
| "sft_loss": 1.5023282766342163, | |
| "step": 190 | |
| }, | |
| { | |
| "epoch": 0.3232976358860376, | |
| "grad_norm": 12.613110542297363, | |
| "learning_rate": 4.857928875491392e-06, | |
| "logits/chosen": -23.007986068725586, | |
| "logits/rejected": -22.91147232055664, | |
| "logps/chosen": -1.360762596130371, | |
| "logps/rejected": -1.479021668434143, | |
| "loss": 1.4383, | |
| "odds_ratio_loss": 0.7751696705818176, | |
| "rewards/accuracies": 0.5249999761581421, | |
| "rewards/chosen": -0.13607625663280487, | |
| "rewards/margins": 0.01182589866220951, | |
| "rewards/rejected": -0.14790216088294983, | |
| "sft_loss": 1.360762596130371, | |
| "step": 200 | |
| }, | |
| { | |
| "epoch": 0.33946251768033947, | |
| "grad_norm": 2.2189383506774902, | |
| "learning_rate": 4.843512968036314e-06, | |
| "logits/chosen": -22.693614959716797, | |
| "logits/rejected": -22.657732009887695, | |
| "logps/chosen": -1.3642535209655762, | |
| "logps/rejected": -1.4226882457733154, | |
| "loss": 1.4427, | |
| "odds_ratio_loss": 0.7848214507102966, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": -0.13642534613609314, | |
| "rewards/margins": 0.0058434633538126945, | |
| "rewards/rejected": -0.1422688066959381, | |
| "sft_loss": 1.3642535209655762, | |
| "step": 210 | |
| }, | |
| { | |
| "epoch": 0.35562739947464134, | |
| "grad_norm": 4.814913749694824, | |
| "learning_rate": 4.828424108555486e-06, | |
| "logits/chosen": -23.161609649658203, | |
| "logits/rejected": -23.108890533447266, | |
| "logps/chosen": -1.5788774490356445, | |
| "logps/rejected": -1.7273142337799072, | |
| "loss": 1.6537, | |
| "odds_ratio_loss": 0.748489499092102, | |
| "rewards/accuracies": 0.543749988079071, | |
| "rewards/chosen": -0.15788774192333221, | |
| "rewards/margins": 0.014843670651316643, | |
| "rewards/rejected": -0.1727314293384552, | |
| "sft_loss": 1.5788774490356445, | |
| "step": 220 | |
| }, | |
| { | |
| "epoch": 0.3717922812689432, | |
| "grad_norm": 0.8914692997932434, | |
| "learning_rate": 4.812666629893957e-06, | |
| "logits/chosen": -23.069910049438477, | |
| "logits/rejected": -23.09903907775879, | |
| "logps/chosen": -1.3772187232971191, | |
| "logps/rejected": -1.4173983335494995, | |
| "loss": 1.4523, | |
| "odds_ratio_loss": 0.7508037686347961, | |
| "rewards/accuracies": 0.512499988079071, | |
| "rewards/chosen": -0.13772186636924744, | |
| "rewards/margins": 0.004017965402454138, | |
| "rewards/rejected": -0.1417398303747177, | |
| "sft_loss": 1.3772187232971191, | |
| "step": 230 | |
| }, | |
| { | |
| "epoch": 0.3879571630632451, | |
| "grad_norm": 1.202868103981018, | |
| "learning_rate": 4.796245056894273e-06, | |
| "logits/chosen": -22.570789337158203, | |
| "logits/rejected": -22.600833892822266, | |
| "logps/chosen": -1.453567385673523, | |
| "logps/rejected": -1.5844666957855225, | |
| "loss": 1.5321, | |
| "odds_ratio_loss": 0.7851130962371826, | |
| "rewards/accuracies": 0.4625000059604645, | |
| "rewards/chosen": -0.14535674452781677, | |
| "rewards/margins": 0.013089925050735474, | |
| "rewards/rejected": -0.15844666957855225, | |
| "sft_loss": 1.453567385673523, | |
| "step": 240 | |
| }, | |
| { | |
| "epoch": 0.404122044857547, | |
| "grad_norm": 1.3130539655685425, | |
| "learning_rate": 4.779164105097148e-06, | |
| "logits/chosen": -23.004568099975586, | |
| "logits/rejected": -23.11286163330078, | |
| "logps/chosen": -1.325037956237793, | |
| "logps/rejected": -1.563007116317749, | |
| "loss": 1.391, | |
| "odds_ratio_loss": 0.6598650813102722, | |
| "rewards/accuracies": 0.5562499761581421, | |
| "rewards/chosen": -0.13250377774238586, | |
| "rewards/margins": 0.023796914145350456, | |
| "rewards/rejected": -0.15630069375038147, | |
| "sft_loss": 1.325037956237793, | |
| "step": 250 | |
| }, | |
| { | |
| "epoch": 0.42028692665184886, | |
| "grad_norm": 1.867422342300415, | |
| "learning_rate": 4.761428679387373e-06, | |
| "logits/chosen": -22.629825592041016, | |
| "logits/rejected": -22.818918228149414, | |
| "logps/chosen": -1.317091941833496, | |
| "logps/rejected": -1.4807102680206299, | |
| "loss": 1.3842, | |
| "odds_ratio_loss": 0.6714745759963989, | |
| "rewards/accuracies": 0.543749988079071, | |
| "rewards/chosen": -0.13170920312404633, | |
| "rewards/margins": 0.016361821442842484, | |
| "rewards/rejected": -0.1480710208415985, | |
| "sft_loss": 1.317091941833496, | |
| "step": 260 | |
| }, | |
| { | |
| "epoch": 0.4364518084461507, | |
| "grad_norm": 1.9516927003860474, | |
| "learning_rate": 4.7430438725853515e-06, | |
| "logits/chosen": -22.51862907409668, | |
| "logits/rejected": -22.682025909423828, | |
| "logps/chosen": -1.4145148992538452, | |
| "logps/rejected": -1.653055191040039, | |
| "loss": 1.4844, | |
| "odds_ratio_loss": 0.6991928815841675, | |
| "rewards/accuracies": 0.581250011920929, | |
| "rewards/chosen": -0.14145150780677795, | |
| "rewards/margins": 0.023854022845625877, | |
| "rewards/rejected": -0.16530552506446838, | |
| "sft_loss": 1.4145148992538452, | |
| "step": 270 | |
| }, | |
| { | |
| "epoch": 0.4526166902404526, | |
| "grad_norm": 1.7087641954421997, | |
| "learning_rate": 4.724014963984669e-06, | |
| "logits/chosen": -22.849842071533203, | |
| "logits/rejected": -22.771404266357422, | |
| "logps/chosen": -1.387978196144104, | |
| "logps/rejected": -1.5646307468414307, | |
| "loss": 1.4589, | |
| "odds_ratio_loss": 0.7090586423873901, | |
| "rewards/accuracies": 0.5562499761581421, | |
| "rewards/chosen": -0.1387978196144104, | |
| "rewards/margins": 0.01766524836421013, | |
| "rewards/rejected": -0.15646305680274963, | |
| "sft_loss": 1.387978196144104, | |
| "step": 280 | |
| }, | |
| { | |
| "epoch": 0.4687815720347545, | |
| "grad_norm": 3.348365068435669, | |
| "learning_rate": 4.704347417836116e-06, | |
| "logits/chosen": -22.53603172302246, | |
| "logits/rejected": -22.64337730407715, | |
| "logps/chosen": -1.2867505550384521, | |
| "logps/rejected": -1.4700194597244263, | |
| "loss": 1.3571, | |
| "odds_ratio_loss": 0.7038428783416748, | |
| "rewards/accuracies": 0.5562499761581421, | |
| "rewards/chosen": -0.12867505848407745, | |
| "rewards/margins": 0.018326906487345695, | |
| "rewards/rejected": -0.1470019519329071, | |
| "sft_loss": 1.2867505550384521, | |
| "step": 290 | |
| }, | |
| { | |
| "epoch": 0.4849464538290564, | |
| "grad_norm": 2.8705925941467285, | |
| "learning_rate": 4.684046881778603e-06, | |
| "logits/chosen": -22.37331771850586, | |
| "logits/rejected": -22.327777862548828, | |
| "logps/chosen": -1.3365970849990845, | |
| "logps/rejected": -1.4279487133026123, | |
| "loss": 1.4075, | |
| "odds_ratio_loss": 0.7086235880851746, | |
| "rewards/accuracies": 0.5249999761581421, | |
| "rewards/chosen": -0.13365969061851501, | |
| "rewards/margins": 0.009135158732533455, | |
| "rewards/rejected": -0.14279486238956451, | |
| "sft_loss": 1.3365970849990845, | |
| "step": 300 | |
| }, | |
| { | |
| "epoch": 0.5011113356233583, | |
| "grad_norm": 1.1998392343521118, | |
| "learning_rate": 4.663119185217409e-06, | |
| "logits/chosen": -22.524127960205078, | |
| "logits/rejected": -22.61183738708496, | |
| "logps/chosen": -1.3241046667099, | |
| "logps/rejected": -1.48367440700531, | |
| "loss": 1.3928, | |
| "odds_ratio_loss": 0.6864575147628784, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": -0.13241048157215118, | |
| "rewards/margins": 0.015956982970237732, | |
| "rewards/rejected": -0.14836746454238892, | |
| "sft_loss": 1.3241046667099, | |
| "step": 310 | |
| }, | |
| { | |
| "epoch": 0.5172762174176602, | |
| "grad_norm": 1.1742535829544067, | |
| "learning_rate": 4.641570337650232e-06, | |
| "logits/chosen": -22.668704986572266, | |
| "logits/rejected": -22.804088592529297, | |
| "logps/chosen": -1.2428923845291138, | |
| "logps/rejected": -1.3876885175704956, | |
| "loss": 1.3138, | |
| "odds_ratio_loss": 0.7085933685302734, | |
| "rewards/accuracies": 0.543749988079071, | |
| "rewards/chosen": -0.12428925186395645, | |
| "rewards/margins": 0.014479607343673706, | |
| "rewards/rejected": -0.13876885175704956, | |
| "sft_loss": 1.2428923845291138, | |
| "step": 320 | |
| }, | |
| { | |
| "epoch": 0.533441099211962, | |
| "grad_norm": 3.9174962043762207, | |
| "learning_rate": 4.61940652694154e-06, | |
| "logits/chosen": -22.036165237426758, | |
| "logits/rejected": -22.13457679748535, | |
| "logps/chosen": -1.3345186710357666, | |
| "logps/rejected": -1.4239206314086914, | |
| "loss": 1.4082, | |
| "odds_ratio_loss": 0.7365013360977173, | |
| "rewards/accuracies": 0.4625000059604645, | |
| "rewards/chosen": -0.13345186412334442, | |
| "rewards/margins": 0.008940198458731174, | |
| "rewards/rejected": -0.14239206910133362, | |
| "sft_loss": 1.3345186710357666, | |
| "step": 330 | |
| }, | |
| { | |
| "epoch": 0.5496059810062639, | |
| "grad_norm": 3.5989551544189453, | |
| "learning_rate": 4.596634117545689e-06, | |
| "logits/chosen": -22.659748077392578, | |
| "logits/rejected": -22.487016677856445, | |
| "logps/chosen": -1.4412176609039307, | |
| "logps/rejected": -1.6016347408294678, | |
| "loss": 1.5117, | |
| "odds_ratio_loss": 0.7047333121299744, | |
| "rewards/accuracies": 0.543749988079071, | |
| "rewards/chosen": -0.14412176609039307, | |
| "rewards/margins": 0.016041725873947144, | |
| "rewards/rejected": -0.1601634919643402, | |
| "sft_loss": 1.4412176609039307, | |
| "step": 340 | |
| }, | |
| { | |
| "epoch": 0.5657708628005658, | |
| "grad_norm": 1.72525954246521, | |
| "learning_rate": 4.573259648679335e-06, | |
| "logits/chosen": -22.422473907470703, | |
| "logits/rejected": -22.158262252807617, | |
| "logps/chosen": -1.3891839981079102, | |
| "logps/rejected": -1.5595767498016357, | |
| "loss": 1.4614, | |
| "odds_ratio_loss": 0.7222477197647095, | |
| "rewards/accuracies": 0.550000011920929, | |
| "rewards/chosen": -0.13891839981079102, | |
| "rewards/margins": 0.017039284110069275, | |
| "rewards/rejected": -0.1559576839208603, | |
| "sft_loss": 1.3891839981079102, | |
| "step": 350 | |
| }, | |
| { | |
| "epoch": 0.5819357445948676, | |
| "grad_norm": 6.434234619140625, | |
| "learning_rate": 4.549289832443663e-06, | |
| "logits/chosen": -22.174190521240234, | |
| "logits/rejected": -22.23097038269043, | |
| "logps/chosen": -1.315002202987671, | |
| "logps/rejected": -1.472235918045044, | |
| "loss": 1.3873, | |
| "odds_ratio_loss": 0.7233031988143921, | |
| "rewards/accuracies": 0.53125, | |
| "rewards/chosen": -0.1315002143383026, | |
| "rewards/margins": 0.015723396092653275, | |
| "rewards/rejected": -0.14722362160682678, | |
| "sft_loss": 1.315002202987671, | |
| "step": 360 | |
| }, | |
| { | |
| "epoch": 0.5981006263891695, | |
| "grad_norm": 1.595171332359314, | |
| "learning_rate": 4.524731551896978e-06, | |
| "logits/chosen": -21.987152099609375, | |
| "logits/rejected": -21.909650802612305, | |
| "logps/chosen": -1.2452141046524048, | |
| "logps/rejected": -1.389278769493103, | |
| "loss": 1.3152, | |
| "odds_ratio_loss": 0.7003225088119507, | |
| "rewards/accuracies": 0.48124998807907104, | |
| "rewards/chosen": -0.1245214194059372, | |
| "rewards/margins": 0.014406457543373108, | |
| "rewards/rejected": -0.1389278769493103, | |
| "sft_loss": 1.2452141046524048, | |
| "step": 370 | |
| }, | |
| { | |
| "epoch": 0.6142655081834714, | |
| "grad_norm": 2.505262851715088, | |
| "learning_rate": 4.4995918590781925e-06, | |
| "logits/chosen": -22.261516571044922, | |
| "logits/rejected": -22.25895118713379, | |
| "logps/chosen": -1.3019020557403564, | |
| "logps/rejected": -1.4299681186676025, | |
| "loss": 1.3751, | |
| "odds_ratio_loss": 0.7319445610046387, | |
| "rewards/accuracies": 0.53125, | |
| "rewards/chosen": -0.13019022345542908, | |
| "rewards/margins": 0.012806607410311699, | |
| "rewards/rejected": -0.14299681782722473, | |
| "sft_loss": 1.3019020557403564, | |
| "step": 380 | |
| }, | |
| { | |
| "epoch": 0.6304303899777733, | |
| "grad_norm": 1.0513848066329956, | |
| "learning_rate": 4.473877972981797e-06, | |
| "logits/chosen": -21.957128524780273, | |
| "logits/rejected": -21.8829288482666, | |
| "logps/chosen": -1.343338131904602, | |
| "logps/rejected": -1.5255149602890015, | |
| "loss": 1.4114, | |
| "odds_ratio_loss": 0.6804416179656982, | |
| "rewards/accuracies": 0.59375, | |
| "rewards/chosen": -0.13433381915092468, | |
| "rewards/margins": 0.01821768842637539, | |
| "rewards/rejected": -0.15255150198936462, | |
| "sft_loss": 1.343338131904602, | |
| "step": 390 | |
| }, | |
| { | |
| "epoch": 0.6465952717720752, | |
| "grad_norm": 2.7681405544281006, | |
| "learning_rate": 4.447597277484894e-06, | |
| "logits/chosen": -21.926578521728516, | |
| "logits/rejected": -21.960134506225586, | |
| "logps/chosen": -1.2327673435211182, | |
| "logps/rejected": -1.3560773134231567, | |
| "loss": 1.305, | |
| "odds_ratio_loss": 0.7222930192947388, | |
| "rewards/accuracies": 0.5562499761581421, | |
| "rewards/chosen": -0.1232767328619957, | |
| "rewards/margins": 0.012331007048487663, | |
| "rewards/rejected": -0.1356077492237091, | |
| "sft_loss": 1.2327673435211182, | |
| "step": 400 | |
| }, | |
| { | |
| "epoch": 0.6627601535663771, | |
| "grad_norm": 1.4995496273040771, | |
| "learning_rate": 4.42075731922687e-06, | |
| "logits/chosen": -22.4599552154541, | |
| "logits/rejected": -22.27519989013672, | |
| "logps/chosen": -1.383239507675171, | |
| "logps/rejected": -1.4447576999664307, | |
| "loss": 1.4576, | |
| "odds_ratio_loss": 0.7433907389640808, | |
| "rewards/accuracies": 0.5249999761581421, | |
| "rewards/chosen": -0.13832393288612366, | |
| "rewards/margins": 0.006151811219751835, | |
| "rewards/rejected": -0.1444757580757141, | |
| "sft_loss": 1.383239507675171, | |
| "step": 410 | |
| }, | |
| { | |
| "epoch": 0.6789250353606789, | |
| "grad_norm": 3.4521114826202393, | |
| "learning_rate": 4.3933658054423465e-06, | |
| "logits/chosen": -22.025806427001953, | |
| "logits/rejected": -21.983091354370117, | |
| "logps/chosen": -1.2636308670043945, | |
| "logps/rejected": -1.3890178203582764, | |
| "loss": 1.3327, | |
| "odds_ratio_loss": 0.6908599138259888, | |
| "rewards/accuracies": 0.543749988079071, | |
| "rewards/chosen": -0.12636308372020721, | |
| "rewards/margins": 0.012538706883788109, | |
| "rewards/rejected": -0.13890179991722107, | |
| "sft_loss": 1.2636308670043945, | |
| "step": 420 | |
| }, | |
| { | |
| "epoch": 0.6950899171549808, | |
| "grad_norm": 2.2511396408081055, | |
| "learning_rate": 4.365430601748003e-06, | |
| "logits/chosen": -21.880239486694336, | |
| "logits/rejected": -22.0391845703125, | |
| "logps/chosen": -1.4012739658355713, | |
| "logps/rejected": -1.4651644229888916, | |
| "loss": 1.4754, | |
| "odds_ratio_loss": 0.7415169477462769, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": -0.14012740552425385, | |
| "rewards/margins": 0.006389034446328878, | |
| "rewards/rejected": -0.14651645720005035, | |
| "sft_loss": 1.4012739658355713, | |
| "step": 430 | |
| }, | |
| { | |
| "epoch": 0.7112547989492827, | |
| "grad_norm": 6.550912857055664, | |
| "learning_rate": 4.336959729883925e-06, | |
| "logits/chosen": -22.025110244750977, | |
| "logits/rejected": -21.944046020507812, | |
| "logps/chosen": -1.2905068397521973, | |
| "logps/rejected": -1.3223841190338135, | |
| "loss": 1.3664, | |
| "odds_ratio_loss": 0.7585982084274292, | |
| "rewards/accuracies": 0.48124998807907104, | |
| "rewards/chosen": -0.12905070185661316, | |
| "rewards/margins": 0.003187725320458412, | |
| "rewards/rejected": -0.13223841786384583, | |
| "sft_loss": 1.2905068397521973, | |
| "step": 440 | |
| }, | |
| { | |
| "epoch": 0.7274196807435845, | |
| "grad_norm": 1.741584062576294, | |
| "learning_rate": 4.307961365410118e-06, | |
| "logits/chosen": -21.6479434967041, | |
| "logits/rejected": -21.64910888671875, | |
| "logps/chosen": -1.3282034397125244, | |
| "logps/rejected": -1.3957862854003906, | |
| "loss": 1.4023, | |
| "odds_ratio_loss": 0.7411014437675476, | |
| "rewards/accuracies": 0.4749999940395355, | |
| "rewards/chosen": -0.13282035291194916, | |
| "rewards/margins": 0.006758278701454401, | |
| "rewards/rejected": -0.13957861065864563, | |
| "sft_loss": 1.3282034397125244, | |
| "step": 450 | |
| }, | |
| { | |
| "epoch": 0.7435845625378864, | |
| "grad_norm": 2.630305528640747, | |
| "learning_rate": 4.278443835358854e-06, | |
| "logits/chosen": -22.009098052978516, | |
| "logits/rejected": -21.944019317626953, | |
| "logps/chosen": -1.2518932819366455, | |
| "logps/rejected": -1.4686853885650635, | |
| "loss": 1.3164, | |
| "odds_ratio_loss": 0.6453157067298889, | |
| "rewards/accuracies": 0.59375, | |
| "rewards/chosen": -0.12518933415412903, | |
| "rewards/margins": 0.0216792281717062, | |
| "rewards/rejected": -0.14686855673789978, | |
| "sft_loss": 1.2518932819366455, | |
| "step": 460 | |
| }, | |
| { | |
| "epoch": 0.7597494443321883, | |
| "grad_norm": 1.7374227046966553, | |
| "learning_rate": 4.248415615843523e-06, | |
| "logits/chosen": -22.067617416381836, | |
| "logits/rejected": -21.912487030029297, | |
| "logps/chosen": -1.278611421585083, | |
| "logps/rejected": -1.367891550064087, | |
| "loss": 1.3508, | |
| "odds_ratio_loss": 0.7223629951477051, | |
| "rewards/accuracies": 0.5249999761581421, | |
| "rewards/chosen": -0.1278611272573471, | |
| "rewards/margins": 0.008928016759455204, | |
| "rewards/rejected": -0.13678915798664093, | |
| "sft_loss": 1.278611421585083, | |
| "step": 470 | |
| }, | |
| { | |
| "epoch": 0.7759143261264903, | |
| "grad_norm": 5.568856716156006, | |
| "learning_rate": 4.217885329624666e-06, | |
| "logits/chosen": -22.169946670532227, | |
| "logits/rejected": -22.093523025512695, | |
| "logps/chosen": -1.207793951034546, | |
| "logps/rejected": -1.4276247024536133, | |
| "loss": 1.2738, | |
| "odds_ratio_loss": 0.660278856754303, | |
| "rewards/accuracies": 0.5874999761581421, | |
| "rewards/chosen": -0.1207793727517128, | |
| "rewards/margins": 0.021983083337545395, | |
| "rewards/rejected": -0.1427624672651291, | |
| "sft_loss": 1.207793951034546, | |
| "step": 480 | |
| }, | |
| { | |
| "epoch": 0.7920792079207921, | |
| "grad_norm": 1.7587084770202637, | |
| "learning_rate": 4.186861743633911e-06, | |
| "logits/chosen": -21.444040298461914, | |
| "logits/rejected": -21.620454788208008, | |
| "logps/chosen": -1.2591201066970825, | |
| "logps/rejected": -1.494167685508728, | |
| "loss": 1.3277, | |
| "odds_ratio_loss": 0.6855591535568237, | |
| "rewards/accuracies": 0.48750001192092896, | |
| "rewards/chosen": -0.12591201066970825, | |
| "rewards/margins": 0.023504754528403282, | |
| "rewards/rejected": -0.14941677451133728, | |
| "sft_loss": 1.2591201066970825, | |
| "step": 490 | |
| }, | |
| { | |
| "epoch": 0.808244089715094, | |
| "grad_norm": 1.509691596031189, | |
| "learning_rate": 4.155353766456497e-06, | |
| "logits/chosen": -22.305217742919922, | |
| "logits/rejected": -22.103668212890625, | |
| "logps/chosen": -1.335773229598999, | |
| "logps/rejected": -1.4481773376464844, | |
| "loss": 1.4057, | |
| "odds_ratio_loss": 0.6996302604675293, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": -0.13357731699943542, | |
| "rewards/margins": 0.011240402236580849, | |
| "rewards/rejected": -0.14481773972511292, | |
| "sft_loss": 1.335773229598999, | |
| "step": 500 | |
| }, | |
| { | |
| "epoch": 0.808244089715094, | |
| "eval_logits/chosen": -21.914384841918945, | |
| "eval_logits/rejected": -21.94023323059082, | |
| "eval_logps/chosen": -1.31483793258667, | |
| "eval_logps/rejected": -1.4522359371185303, | |
| "eval_loss": 1.38706636428833, | |
| "eval_odds_ratio_loss": 0.7222831845283508, | |
| "eval_rewards/accuracies": 0.5318182110786438, | |
| "eval_rewards/chosen": -0.131483793258667, | |
| "eval_rewards/margins": 0.01373977866023779, | |
| "eval_rewards/rejected": -0.14522357285022736, | |
| "eval_runtime": 82.3592, | |
| "eval_samples_per_second": 13.356, | |
| "eval_sft_loss": 1.31483793258667, | |
| "eval_steps_per_second": 6.678, | |
| "step": 500 | |
| }, | |
| { | |
| "epoch": 0.8244089715093958, | |
| "grad_norm": 1.657311201095581, | |
| "learning_rate": 4.123370445773134e-06, | |
| "logits/chosen": -22.07869529724121, | |
| "logits/rejected": -22.091659545898438, | |
| "logps/chosen": -1.2787725925445557, | |
| "logps/rejected": -1.3090568780899048, | |
| "loss": 1.3519, | |
| "odds_ratio_loss": 0.731525719165802, | |
| "rewards/accuracies": 0.53125, | |
| "rewards/chosen": -0.12787726521492004, | |
| "rewards/margins": 0.003028427716344595, | |
| "rewards/rejected": -0.13090567290782928, | |
| "sft_loss": 1.2787725925445557, | |
| "step": 510 | |
| }, | |
| { | |
| "epoch": 0.8405738533036977, | |
| "grad_norm": 4.675573825836182, | |
| "learning_rate": 4.090920965761906e-06, | |
| "logits/chosen": -21.979219436645508, | |
| "logits/rejected": -21.770206451416016, | |
| "logps/chosen": -1.304009199142456, | |
| "logps/rejected": -1.4551854133605957, | |
| "loss": 1.3752, | |
| "odds_ratio_loss": 0.7114746570587158, | |
| "rewards/accuracies": 0.5687500238418579, | |
| "rewards/chosen": -0.13040092587471008, | |
| "rewards/margins": 0.015117612667381763, | |
| "rewards/rejected": -0.14551854133605957, | |
| "sft_loss": 1.304009199142456, | |
| "step": 520 | |
| }, | |
| { | |
| "epoch": 0.8567387350979996, | |
| "grad_norm": 4.995480060577393, | |
| "learning_rate": 4.058014644460991e-06, | |
| "logits/chosen": -21.77149772644043, | |
| "logits/rejected": -21.79526710510254, | |
| "logps/chosen": -1.3110116720199585, | |
| "logps/rejected": -1.4032762050628662, | |
| "loss": 1.3819, | |
| "odds_ratio_loss": 0.708578884601593, | |
| "rewards/accuracies": 0.5562499761581421, | |
| "rewards/chosen": -0.13110119104385376, | |
| "rewards/margins": 0.009226455353200436, | |
| "rewards/rejected": -0.14032763242721558, | |
| "sft_loss": 1.3110116720199585, | |
| "step": 530 | |
| }, | |
| { | |
| "epoch": 0.8729036168923014, | |
| "grad_norm": 1.891120433807373, | |
| "learning_rate": 4.024660931092939e-06, | |
| "logits/chosen": -21.345439910888672, | |
| "logits/rejected": -21.40594482421875, | |
| "logps/chosen": -1.3298126459121704, | |
| "logps/rejected": -1.4730676412582397, | |
| "loss": 1.4012, | |
| "odds_ratio_loss": 0.713805079460144, | |
| "rewards/accuracies": 0.512499988079071, | |
| "rewards/chosen": -0.13298127055168152, | |
| "rewards/margins": 0.014325493946671486, | |
| "rewards/rejected": -0.14730675518512726, | |
| "sft_loss": 1.3298126459121704, | |
| "step": 540 | |
| }, | |
| { | |
| "epoch": 0.8890684986866033, | |
| "grad_norm": 7.624050617218018, | |
| "learning_rate": 3.990869403351272e-06, | |
| "logits/chosen": -21.680253982543945, | |
| "logits/rejected": -21.578588485717773, | |
| "logps/chosen": -1.3242005109786987, | |
| "logps/rejected": -1.496756911277771, | |
| "loss": 1.3887, | |
| "odds_ratio_loss": 0.644872784614563, | |
| "rewards/accuracies": 0.59375, | |
| "rewards/chosen": -0.13242004811763763, | |
| "rewards/margins": 0.017255645245313644, | |
| "rewards/rejected": -0.14967571198940277, | |
| "sft_loss": 1.3242005109786987, | |
| "step": 550 | |
| }, | |
| { | |
| "epoch": 0.9052333804809052, | |
| "grad_norm": 2.1821932792663574, | |
| "learning_rate": 3.956649764650206e-06, | |
| "logits/chosen": -22.087711334228516, | |
| "logits/rejected": -22.087757110595703, | |
| "logps/chosen": -1.2911579608917236, | |
| "logps/rejected": -1.4688221216201782, | |
| "loss": 1.3633, | |
| "odds_ratio_loss": 0.7218787670135498, | |
| "rewards/accuracies": 0.45625001192092896, | |
| "rewards/chosen": -0.12911580502986908, | |
| "rewards/margins": 0.017766421660780907, | |
| "rewards/rejected": -0.14688222110271454, | |
| "sft_loss": 1.2911579608917236, | |
| "step": 560 | |
| }, | |
| { | |
| "epoch": 0.9213982622752072, | |
| "grad_norm": 5.107442855834961, | |
| "learning_rate": 3.92201184133826e-06, | |
| "logits/chosen": -21.773061752319336, | |
| "logits/rejected": -21.753795623779297, | |
| "logps/chosen": -1.2945088148117065, | |
| "logps/rejected": -1.4896302223205566, | |
| "loss": 1.3613, | |
| "odds_ratio_loss": 0.6681925654411316, | |
| "rewards/accuracies": 0.581250011920929, | |
| "rewards/chosen": -0.12945088744163513, | |
| "rewards/margins": 0.019512150436639786, | |
| "rewards/rejected": -0.14896303415298462, | |
| "sft_loss": 1.2945088148117065, | |
| "step": 570 | |
| }, | |
| { | |
| "epoch": 0.937563144069509, | |
| "grad_norm": 1.4870102405548096, | |
| "learning_rate": 3.886965579876572e-06, | |
| "logits/chosen": -21.675262451171875, | |
| "logits/rejected": -21.543224334716797, | |
| "logps/chosen": -1.3047420978546143, | |
| "logps/rejected": -1.3921304941177368, | |
| "loss": 1.3741, | |
| "odds_ratio_loss": 0.694009006023407, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": -0.1304742395877838, | |
| "rewards/margins": 0.008738822303712368, | |
| "rewards/rejected": -0.13921304047107697, | |
| "sft_loss": 1.3047420978546143, | |
| "step": 580 | |
| }, | |
| { | |
| "epoch": 0.9537280258638109, | |
| "grad_norm": 2.04402494430542, | |
| "learning_rate": 3.851521043982716e-06, | |
| "logits/chosen": -21.711593627929688, | |
| "logits/rejected": -21.78032875061035, | |
| "logps/chosen": -1.311596155166626, | |
| "logps/rejected": -1.351053237915039, | |
| "loss": 1.3862, | |
| "odds_ratio_loss": 0.7462160587310791, | |
| "rewards/accuracies": 0.48750001192092896, | |
| "rewards/chosen": -0.13115961849689484, | |
| "rewards/margins": 0.00394571665674448, | |
| "rewards/rejected": -0.13510534167289734, | |
| "sft_loss": 1.311596155166626, | |
| "step": 590 | |
| }, | |
| { | |
| "epoch": 0.9698929076581128, | |
| "grad_norm": 8.317533493041992, | |
| "learning_rate": 3.81568841174086e-06, | |
| "logits/chosen": -21.358070373535156, | |
| "logits/rejected": -21.403278350830078, | |
| "logps/chosen": -1.361669898033142, | |
| "logps/rejected": -1.494240403175354, | |
| "loss": 1.4338, | |
| "odds_ratio_loss": 0.7209025025367737, | |
| "rewards/accuracies": 0.574999988079071, | |
| "rewards/chosen": -0.1361669898033142, | |
| "rewards/margins": 0.013257039710879326, | |
| "rewards/rejected": -0.14942404627799988, | |
| "sft_loss": 1.361669898033142, | |
| "step": 600 | |
| }, | |
| { | |
| "epoch": 0.9860577894524146, | |
| "grad_norm": 2.044271469116211, | |
| "learning_rate": 3.7794779726790664e-06, | |
| "logits/chosen": -21.353389739990234, | |
| "logits/rejected": -21.419639587402344, | |
| "logps/chosen": -1.222917914390564, | |
| "logps/rejected": -1.3896757364273071, | |
| "loss": 1.2907, | |
| "odds_ratio_loss": 0.6782225370407104, | |
| "rewards/accuracies": 0.518750011920929, | |
| "rewards/chosen": -0.12229180335998535, | |
| "rewards/margins": 0.016675781458616257, | |
| "rewards/rejected": -0.1389675885438919, | |
| "sft_loss": 1.222917914390564, | |
| "step": 610 | |
| }, | |
| { | |
| "epoch": 1.0022226712467166, | |
| "grad_norm": 4.229960918426514, | |
| "learning_rate": 3.7429001248146096e-06, | |
| "logits/chosen": -21.391468048095703, | |
| "logits/rejected": -21.433670043945312, | |
| "logps/chosen": -1.3207805156707764, | |
| "logps/rejected": -1.5086777210235596, | |
| "loss": 1.3864, | |
| "odds_ratio_loss": 0.655738115310669, | |
| "rewards/accuracies": 0.6000000238418579, | |
| "rewards/chosen": -0.13207808136940002, | |
| "rewards/margins": 0.018789704889059067, | |
| "rewards/rejected": -0.1508677750825882, | |
| "sft_loss": 1.3207805156707764, | |
| "step": 620 | |
| }, | |
| { | |
| "epoch": 1.0183875530410185, | |
| "grad_norm": 1.3560584783554077, | |
| "learning_rate": 3.7059653716681227e-06, | |
| "logits/chosen": -21.297557830810547, | |
| "logits/rejected": -21.24813461303711, | |
| "logps/chosen": -1.3175418376922607, | |
| "logps/rejected": -1.4784363508224487, | |
| "loss": 1.3899, | |
| "odds_ratio_loss": 0.7236071825027466, | |
| "rewards/accuracies": 0.5249999761581421, | |
| "rewards/chosen": -0.13175417482852936, | |
| "rewards/margins": 0.016089454293251038, | |
| "rewards/rejected": -0.1478436291217804, | |
| "sft_loss": 1.3175418376922607, | |
| "step": 630 | |
| }, | |
| { | |
| "epoch": 1.0345524348353203, | |
| "grad_norm": 4.676194190979004, | |
| "learning_rate": 3.668684319247463e-06, | |
| "logits/chosen": -21.50934410095215, | |
| "logits/rejected": -21.4769229888916, | |
| "logps/chosen": -1.2353088855743408, | |
| "logps/rejected": -1.5056722164154053, | |
| "loss": 1.298, | |
| "odds_ratio_loss": 0.6273224949836731, | |
| "rewards/accuracies": 0.5874999761581421, | |
| "rewards/chosen": -0.12353090196847916, | |
| "rewards/margins": 0.02703634463250637, | |
| "rewards/rejected": -0.15056723356246948, | |
| "sft_loss": 1.2353088855743408, | |
| "step": 640 | |
| }, | |
| { | |
| "epoch": 1.0507173166296222, | |
| "grad_norm": 1.456129550933838, | |
| "learning_rate": 3.6310676730021373e-06, | |
| "logits/chosen": -21.462495803833008, | |
| "logits/rejected": -21.57852554321289, | |
| "logps/chosen": -1.2601442337036133, | |
| "logps/rejected": -1.3218932151794434, | |
| "loss": 1.3324, | |
| "odds_ratio_loss": 0.7224392890930176, | |
| "rewards/accuracies": 0.518750011920929, | |
| "rewards/chosen": -0.12601442635059357, | |
| "rewards/margins": 0.006174898706376553, | |
| "rewards/rejected": -0.1321893036365509, | |
| "sft_loss": 1.2601442337036133, | |
| "step": 650 | |
| }, | |
| { | |
| "epoch": 1.066882198423924, | |
| "grad_norm": 3.662935733795166, | |
| "learning_rate": 3.593126234749178e-06, | |
| "logits/chosen": -21.538293838500977, | |
| "logits/rejected": -21.53152847290039, | |
| "logps/chosen": -1.3035653829574585, | |
| "logps/rejected": -1.4283910989761353, | |
| "loss": 1.375, | |
| "odds_ratio_loss": 0.7146205902099609, | |
| "rewards/accuracies": 0.518750011920929, | |
| "rewards/chosen": -0.1303565502166748, | |
| "rewards/margins": 0.012482564896345139, | |
| "rewards/rejected": -0.14283910393714905, | |
| "sft_loss": 1.3035653829574585, | |
| "step": 660 | |
| }, | |
| { | |
| "epoch": 1.083047080218226, | |
| "grad_norm": 2.5778450965881348, | |
| "learning_rate": 3.554870899571343e-06, | |
| "logits/chosen": -21.25736427307129, | |
| "logits/rejected": -21.203954696655273, | |
| "logps/chosen": -1.2814052104949951, | |
| "logps/rejected": -1.4217122793197632, | |
| "loss": 1.3498, | |
| "odds_ratio_loss": 0.6842816472053528, | |
| "rewards/accuracies": 0.5625, | |
| "rewards/chosen": -0.12814053893089294, | |
| "rewards/margins": 0.0140307005494833, | |
| "rewards/rejected": -0.1421712338924408, | |
| "sft_loss": 1.2814052104949951, | |
| "step": 670 | |
| }, | |
| { | |
| "epoch": 1.0992119620125278, | |
| "grad_norm": 3.7894856929779053, | |
| "learning_rate": 3.5163126526885373e-06, | |
| "logits/chosen": -21.061431884765625, | |
| "logits/rejected": -21.307607650756836, | |
| "logps/chosen": -1.2376093864440918, | |
| "logps/rejected": -1.3728429079055786, | |
| "loss": 1.3095, | |
| "odds_ratio_loss": 0.7185710668563843, | |
| "rewards/accuracies": 0.5249999761581421, | |
| "rewards/chosen": -0.12376093864440918, | |
| "rewards/margins": 0.013523365370929241, | |
| "rewards/rejected": -0.1372843086719513, | |
| "sft_loss": 1.2376093864440918, | |
| "step": 680 | |
| }, | |
| { | |
| "epoch": 1.1153768438068297, | |
| "grad_norm": 2.3537509441375732, | |
| "learning_rate": 3.4774625663033484e-06, | |
| "logits/chosen": -21.21095085144043, | |
| "logits/rejected": -21.2464542388916, | |
| "logps/chosen": -1.232604742050171, | |
| "logps/rejected": -1.3756181001663208, | |
| "loss": 1.3002, | |
| "odds_ratio_loss": 0.6756417155265808, | |
| "rewards/accuracies": 0.5562499761581421, | |
| "rewards/chosen": -0.1232604831457138, | |
| "rewards/margins": 0.014301331713795662, | |
| "rewards/rejected": -0.1375618278980255, | |
| "sft_loss": 1.232604742050171, | |
| "step": 690 | |
| }, | |
| { | |
| "epoch": 1.1315417256011315, | |
| "grad_norm": 1.4719160795211792, | |
| "learning_rate": 3.4383317964216067e-06, | |
| "logits/chosen": -21.036582946777344, | |
| "logits/rejected": -21.25110626220703, | |
| "logps/chosen": -1.2365304231643677, | |
| "logps/rejected": -1.3071203231811523, | |
| "loss": 1.3116, | |
| "odds_ratio_loss": 0.7511636018753052, | |
| "rewards/accuracies": 0.48124998807907104, | |
| "rewards/chosen": -0.12365303188562393, | |
| "rewards/margins": 0.007059005089104176, | |
| "rewards/rejected": -0.13071204721927643, | |
| "sft_loss": 1.2365304231643677, | |
| "step": 700 | |
| }, | |
| { | |
| "epoch": 1.1477066073954334, | |
| "grad_norm": 2.8253660202026367, | |
| "learning_rate": 3.398931579648877e-06, | |
| "logits/chosen": -21.2774658203125, | |
| "logits/rejected": -21.474933624267578, | |
| "logps/chosen": -1.3233165740966797, | |
| "logps/rejected": -1.49609375, | |
| "loss": 1.3932, | |
| "odds_ratio_loss": 0.6990328431129456, | |
| "rewards/accuracies": 0.5562499761581421, | |
| "rewards/chosen": -0.13233166933059692, | |
| "rewards/margins": 0.017277712002396584, | |
| "rewards/rejected": -0.14960938692092896, | |
| "sft_loss": 1.3233165740966797, | |
| "step": 710 | |
| }, | |
| { | |
| "epoch": 1.1638714891897353, | |
| "grad_norm": 2.0079872608184814, | |
| "learning_rate": 3.359273229963813e-06, | |
| "logits/chosen": -21.300012588500977, | |
| "logits/rejected": -21.20311737060547, | |
| "logps/chosen": -1.2444874048233032, | |
| "logps/rejected": -1.3516159057617188, | |
| "loss": 1.315, | |
| "odds_ratio_loss": 0.7051676511764526, | |
| "rewards/accuracies": 0.5249999761581421, | |
| "rewards/chosen": -0.1244487538933754, | |
| "rewards/margins": 0.010712863877415657, | |
| "rewards/rejected": -0.1351616084575653, | |
| "sft_loss": 1.2444874048233032, | |
| "step": 720 | |
| }, | |
| { | |
| "epoch": 1.1800363709840371, | |
| "grad_norm": 1.6793773174285889, | |
| "learning_rate": 3.319368135469285e-06, | |
| "logits/chosen": -21.23295021057129, | |
| "logits/rejected": -21.303585052490234, | |
| "logps/chosen": -1.294540524482727, | |
| "logps/rejected": -1.422060251235962, | |
| "loss": 1.3682, | |
| "odds_ratio_loss": 0.7366825938224792, | |
| "rewards/accuracies": 0.5249999761581421, | |
| "rewards/chosen": -0.12945404648780823, | |
| "rewards/margins": 0.012751981616020203, | |
| "rewards/rejected": -0.14220602810382843, | |
| "sft_loss": 1.294540524482727, | |
| "step": 730 | |
| }, | |
| { | |
| "epoch": 1.196201252778339, | |
| "grad_norm": 2.9670250415802, | |
| "learning_rate": 3.279227755122228e-06, | |
| "logits/chosen": -21.109752655029297, | |
| "logits/rejected": -21.158111572265625, | |
| "logps/chosen": -1.2494882345199585, | |
| "logps/rejected": -1.5186522006988525, | |
| "loss": 1.3163, | |
| "odds_ratio_loss": 0.6685449481010437, | |
| "rewards/accuracies": 0.612500011920929, | |
| "rewards/chosen": -0.12494882196187973, | |
| "rewards/margins": 0.02691638469696045, | |
| "rewards/rejected": -0.15186521410942078, | |
| "sft_loss": 1.2494882345199585, | |
| "step": 740 | |
| }, | |
| { | |
| "epoch": 1.2123661345726409, | |
| "grad_norm": 2.090475082397461, | |
| "learning_rate": 3.2388636154431417e-06, | |
| "logits/chosen": -21.39287757873535, | |
| "logits/rejected": -21.400285720825195, | |
| "logps/chosen": -1.3509362936019897, | |
| "logps/rejected": -1.5079971551895142, | |
| "loss": 1.4216, | |
| "odds_ratio_loss": 0.7062050104141235, | |
| "rewards/accuracies": 0.5625, | |
| "rewards/chosen": -0.13509364426136017, | |
| "rewards/margins": 0.015706082805991173, | |
| "rewards/rejected": -0.1507997065782547, | |
| "sft_loss": 1.3509362936019897, | |
| "step": 750 | |
| }, | |
| { | |
| "epoch": 1.2285310163669427, | |
| "grad_norm": 1.8322948217391968, | |
| "learning_rate": 3.198287307206192e-06, | |
| "logits/chosen": -20.998327255249023, | |
| "logits/rejected": -21.087013244628906, | |
| "logps/chosen": -1.254172921180725, | |
| "logps/rejected": -1.3801274299621582, | |
| "loss": 1.3241, | |
| "odds_ratio_loss": 0.6988333463668823, | |
| "rewards/accuracies": 0.5375000238418579, | |
| "rewards/chosen": -0.1254173070192337, | |
| "rewards/margins": 0.012595455162227154, | |
| "rewards/rejected": -0.13801275193691254, | |
| "sft_loss": 1.254172921180725, | |
| "step": 760 | |
| }, | |
| { | |
| "epoch": 1.2446958981612446, | |
| "grad_norm": 1.5089768171310425, | |
| "learning_rate": 3.157510482110856e-06, | |
| "logits/chosen": -21.451335906982422, | |
| "logits/rejected": -21.22077751159668, | |
| "logps/chosen": -1.2650538682937622, | |
| "logps/rejected": -1.3639296293258667, | |
| "loss": 1.337, | |
| "odds_ratio_loss": 0.7199094891548157, | |
| "rewards/accuracies": 0.53125, | |
| "rewards/chosen": -0.12650538980960846, | |
| "rewards/margins": 0.009887583553791046, | |
| "rewards/rejected": -0.1363929808139801, | |
| "sft_loss": 1.2650538682937622, | |
| "step": 770 | |
| }, | |
| { | |
| "epoch": 1.2608607799555465, | |
| "grad_norm": 1.1987851858139038, | |
| "learning_rate": 3.116544849436077e-06, | |
| "logits/chosen": -21.068490982055664, | |
| "logits/rejected": -20.816608428955078, | |
| "logps/chosen": -1.3564980030059814, | |
| "logps/rejected": -1.5898200273513794, | |
| "loss": 1.4243, | |
| "odds_ratio_loss": 0.6784577369689941, | |
| "rewards/accuracies": 0.512499988079071, | |
| "rewards/chosen": -0.13564978539943695, | |
| "rewards/margins": 0.023332208395004272, | |
| "rewards/rejected": -0.15898200869560242, | |
| "sft_loss": 1.3564980030059814, | |
| "step": 780 | |
| }, | |
| { | |
| "epoch": 1.2770256617498483, | |
| "grad_norm": 1.9816875457763672, | |
| "learning_rate": 3.0754021726778848e-06, | |
| "logits/chosen": -21.14816665649414, | |
| "logits/rejected": -21.171310424804688, | |
| "logps/chosen": -1.1857916116714478, | |
| "logps/rejected": -1.4323453903198242, | |
| "loss": 1.2508, | |
| "odds_ratio_loss": 0.6496400833129883, | |
| "rewards/accuracies": 0.5625, | |
| "rewards/chosen": -0.11857914924621582, | |
| "rewards/margins": 0.02465539053082466, | |
| "rewards/rejected": -0.14323453605175018, | |
| "sft_loss": 1.1857916116714478, | |
| "step": 790 | |
| }, | |
| { | |
| "epoch": 1.2931905435441502, | |
| "grad_norm": 0.9774518609046936, | |
| "learning_rate": 3.0340942661714463e-06, | |
| "logits/chosen": -21.33401870727539, | |
| "logits/rejected": -21.100515365600586, | |
| "logps/chosen": -1.3348338603973389, | |
| "logps/rejected": -1.4366466999053955, | |
| "loss": 1.4051, | |
| "odds_ratio_loss": 0.7027276754379272, | |
| "rewards/accuracies": 0.53125, | |
| "rewards/chosen": -0.1334833949804306, | |
| "rewards/margins": 0.010181264951825142, | |
| "rewards/rejected": -0.1436646580696106, | |
| "sft_loss": 1.3348338603973389, | |
| "step": 800 | |
| }, | |
| { | |
| "epoch": 1.3093554253384523, | |
| "grad_norm": 2.8201749324798584, | |
| "learning_rate": 2.992632991698512e-06, | |
| "logits/chosen": -20.885223388671875, | |
| "logits/rejected": -21.0069522857666, | |
| "logps/chosen": -1.28475022315979, | |
| "logps/rejected": -1.434466004371643, | |
| "loss": 1.3541, | |
| "odds_ratio_loss": 0.6936103105545044, | |
| "rewards/accuracies": 0.6000000238418579, | |
| "rewards/chosen": -0.12847502529621124, | |
| "rewards/margins": 0.014971581287682056, | |
| "rewards/rejected": -0.14344659447669983, | |
| "sft_loss": 1.28475022315979, | |
| "step": 810 | |
| }, | |
| { | |
| "epoch": 1.3255203071327541, | |
| "grad_norm": 1.557600498199463, | |
| "learning_rate": 2.9510302550812537e-06, | |
| "logits/chosen": -21.287960052490234, | |
| "logits/rejected": -21.395370483398438, | |
| "logps/chosen": -1.2278238534927368, | |
| "logps/rejected": -1.4308351278305054, | |
| "loss": 1.2925, | |
| "odds_ratio_loss": 0.646266520023346, | |
| "rewards/accuracies": 0.543749988079071, | |
| "rewards/chosen": -0.1227823942899704, | |
| "rewards/margins": 0.02030111663043499, | |
| "rewards/rejected": -0.14308352768421173, | |
| "sft_loss": 1.2278238534927368, | |
| "step": 820 | |
| }, | |
| { | |
| "epoch": 1.341685188927056, | |
| "grad_norm": 2.6895642280578613, | |
| "learning_rate": 2.9092980027634325e-06, | |
| "logits/chosen": -21.14548110961914, | |
| "logits/rejected": -21.261184692382812, | |
| "logps/chosen": -1.1719629764556885, | |
| "logps/rejected": -1.3467178344726562, | |
| "loss": 1.2381, | |
| "odds_ratio_loss": 0.6613830327987671, | |
| "rewards/accuracies": 0.5687500238418579, | |
| "rewards/chosen": -0.11719629913568497, | |
| "rewards/margins": 0.017475496977567673, | |
| "rewards/rejected": -0.13467177748680115, | |
| "sft_loss": 1.1719629764556885, | |
| "step": 830 | |
| }, | |
| { | |
| "epoch": 1.3578500707213579, | |
| "grad_norm": 1.8690439462661743, | |
| "learning_rate": 2.867448218379927e-06, | |
| "logits/chosen": -21.13509750366211, | |
| "logits/rejected": -21.14713478088379, | |
| "logps/chosen": -1.2881317138671875, | |
| "logps/rejected": -1.419306993484497, | |
| "loss": 1.359, | |
| "odds_ratio_loss": 0.7090221643447876, | |
| "rewards/accuracies": 0.550000011920929, | |
| "rewards/chosen": -0.12881317734718323, | |
| "rewards/margins": 0.013117517344653606, | |
| "rewards/rejected": -0.1419306844472885, | |
| "sft_loss": 1.2881317138671875, | |
| "step": 840 | |
| }, | |
| { | |
| "epoch": 1.3740149525156597, | |
| "grad_norm": 4.924416542053223, | |
| "learning_rate": 2.825492919315559e-06, | |
| "logits/chosen": -20.981931686401367, | |
| "logits/rejected": -21.142297744750977, | |
| "logps/chosen": -1.3466829061508179, | |
| "logps/rejected": -1.3898065090179443, | |
| "loss": 1.4234, | |
| "odds_ratio_loss": 0.7671664953231812, | |
| "rewards/accuracies": 0.518750011920929, | |
| "rewards/chosen": -0.1346682757139206, | |
| "rewards/margins": 0.004312378820031881, | |
| "rewards/rejected": -0.1389806568622589, | |
| "sft_loss": 1.3466829061508179, | |
| "step": 850 | |
| }, | |
| { | |
| "epoch": 1.3901798343099616, | |
| "grad_norm": 1.9793908596038818, | |
| "learning_rate": 2.7834441532542482e-06, | |
| "logits/chosen": -21.344064712524414, | |
| "logits/rejected": -21.355037689208984, | |
| "logps/chosen": -1.2326418161392212, | |
| "logps/rejected": -1.3959559202194214, | |
| "loss": 1.3028, | |
| "odds_ratio_loss": 0.7015843987464905, | |
| "rewards/accuracies": 0.581250011920929, | |
| "rewards/chosen": -0.12326417118310928, | |
| "rewards/margins": 0.016331423074007034, | |
| "rewards/rejected": -0.13959559798240662, | |
| "sft_loss": 1.2326418161392212, | |
| "step": 860 | |
| }, | |
| { | |
| "epoch": 1.4063447161042635, | |
| "grad_norm": 1.5754936933517456, | |
| "learning_rate": 2.74131399471945e-06, | |
| "logits/chosen": -21.10004234313965, | |
| "logits/rejected": -21.218639373779297, | |
| "logps/chosen": -1.2864024639129639, | |
| "logps/rejected": -1.3692989349365234, | |
| "loss": 1.3586, | |
| "odds_ratio_loss": 0.7215597033500671, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": -0.12864024937152863, | |
| "rewards/margins": 0.008289644494652748, | |
| "rewards/rejected": -0.13692989945411682, | |
| "sft_loss": 1.2864024639129639, | |
| "step": 870 | |
| }, | |
| { | |
| "epoch": 1.4225095978985653, | |
| "grad_norm": 2.591798782348633, | |
| "learning_rate": 2.6991145416068947e-06, | |
| "logits/chosen": -20.910709381103516, | |
| "logits/rejected": -21.049091339111328, | |
| "logps/chosen": -1.2924576997756958, | |
| "logps/rejected": -1.3584909439086914, | |
| "loss": 1.3658, | |
| "odds_ratio_loss": 0.7329753637313843, | |
| "rewards/accuracies": 0.53125, | |
| "rewards/chosen": -0.12924575805664062, | |
| "rewards/margins": 0.006603321526199579, | |
| "rewards/rejected": -0.13584908843040466, | |
| "sft_loss": 1.2924576997756958, | |
| "step": 880 | |
| }, | |
| { | |
| "epoch": 1.4386744796928672, | |
| "grad_norm": 1.6928390264511108, | |
| "learning_rate": 2.6568579117106143e-06, | |
| "logits/chosen": -20.872098922729492, | |
| "logits/rejected": -20.832103729248047, | |
| "logps/chosen": -1.2083466053009033, | |
| "logps/rejected": -1.406185507774353, | |
| "loss": 1.2749, | |
| "odds_ratio_loss": 0.6651225090026855, | |
| "rewards/accuracies": 0.6000000238418579, | |
| "rewards/chosen": -0.12083464860916138, | |
| "rewards/margins": 0.019783901050686836, | |
| "rewards/rejected": -0.14061854779720306, | |
| "sft_loss": 1.2083466053009033, | |
| "step": 890 | |
| }, | |
| { | |
| "epoch": 1.454839361487169, | |
| "grad_norm": 0.9949257373809814, | |
| "learning_rate": 2.6145562392432544e-06, | |
| "logits/chosen": -20.952428817749023, | |
| "logits/rejected": -20.857725143432617, | |
| "logps/chosen": -1.2628931999206543, | |
| "logps/rejected": -1.3288319110870361, | |
| "loss": 1.3382, | |
| "odds_ratio_loss": 0.75352543592453, | |
| "rewards/accuracies": 0.5062500238418579, | |
| "rewards/chosen": -0.12628932297229767, | |
| "rewards/margins": 0.006593865808099508, | |
| "rewards/rejected": -0.1328831911087036, | |
| "sft_loss": 1.2628931999206543, | |
| "step": 900 | |
| }, | |
| { | |
| "epoch": 1.471004243281471, | |
| "grad_norm": 2.0959157943725586, | |
| "learning_rate": 2.5722216713516682e-06, | |
| "logits/chosen": -20.460887908935547, | |
| "logits/rejected": -20.67595100402832, | |
| "logps/chosen": -1.2154316902160645, | |
| "logps/rejected": -1.3711063861846924, | |
| "loss": 1.284, | |
| "odds_ratio_loss": 0.6853248476982117, | |
| "rewards/accuracies": 0.59375, | |
| "rewards/chosen": -0.12154315412044525, | |
| "rewards/margins": 0.015567478723824024, | |
| "rewards/rejected": -0.1371106207370758, | |
| "sft_loss": 1.2154316902160645, | |
| "step": 910 | |
| }, | |
| { | |
| "epoch": 1.4871691250757728, | |
| "grad_norm": 2.9631924629211426, | |
| "learning_rate": 2.5298663646288064e-06, | |
| "logits/chosen": -20.95013999938965, | |
| "logits/rejected": -21.05425453186035, | |
| "logps/chosen": -1.213805079460144, | |
| "logps/rejected": -1.382340669631958, | |
| "loss": 1.2822, | |
| "odds_ratio_loss": 0.6836367845535278, | |
| "rewards/accuracies": 0.5249999761581421, | |
| "rewards/chosen": -0.1213805228471756, | |
| "rewards/margins": 0.01685354672372341, | |
| "rewards/rejected": -0.13823407888412476, | |
| "sft_loss": 1.213805079460144, | |
| "step": 920 | |
| }, | |
| { | |
| "epoch": 1.503334006870075, | |
| "grad_norm": 2.637305498123169, | |
| "learning_rate": 2.487502481622879e-06, | |
| "logits/chosen": -20.81560516357422, | |
| "logits/rejected": -20.701404571533203, | |
| "logps/chosen": -1.3175880908966064, | |
| "logps/rejected": -1.4211034774780273, | |
| "loss": 1.3892, | |
| "odds_ratio_loss": 0.7159314155578613, | |
| "rewards/accuracies": 0.5249999761581421, | |
| "rewards/chosen": -0.13175880908966064, | |
| "rewards/margins": 0.010351541452109814, | |
| "rewards/rejected": -0.14211036264896393, | |
| "sft_loss": 1.3175880908966064, | |
| "step": 930 | |
| }, | |
| { | |
| "epoch": 1.5194988886643768, | |
| "grad_norm": 1.9374908208847046, | |
| "learning_rate": 2.4451421873448253e-06, | |
| "logits/chosen": -20.670528411865234, | |
| "logits/rejected": -20.75124168395996, | |
| "logps/chosen": -1.2572494745254517, | |
| "logps/rejected": -1.367071509361267, | |
| "loss": 1.3282, | |
| "odds_ratio_loss": 0.7093643546104431, | |
| "rewards/accuracies": 0.4937500059604645, | |
| "rewards/chosen": -0.12572495639324188, | |
| "rewards/margins": 0.010982205159962177, | |
| "rewards/rejected": -0.13670717179775238, | |
| "sft_loss": 1.2572494745254517, | |
| "step": 940 | |
| }, | |
| { | |
| "epoch": 1.5356637704586786, | |
| "grad_norm": 3.949268102645874, | |
| "learning_rate": 2.40279764577506e-06, | |
| "logits/chosen": -21.262102127075195, | |
| "logits/rejected": -21.309324264526367, | |
| "logps/chosen": -1.3575810194015503, | |
| "logps/rejected": -1.4113634824752808, | |
| "loss": 1.4328, | |
| "odds_ratio_loss": 0.7524511218070984, | |
| "rewards/accuracies": 0.5062500238418579, | |
| "rewards/chosen": -0.13575811684131622, | |
| "rewards/margins": 0.005378241650760174, | |
| "rewards/rejected": -0.14113636314868927, | |
| "sft_loss": 1.3575810194015503, | |
| "step": 950 | |
| }, | |
| { | |
| "epoch": 1.5518286522529805, | |
| "grad_norm": 1.4890868663787842, | |
| "learning_rate": 2.3604810163705242e-06, | |
| "logits/chosen": -20.816661834716797, | |
| "logits/rejected": -20.944917678833008, | |
| "logps/chosen": -1.2137186527252197, | |
| "logps/rejected": -1.348968505859375, | |
| "loss": 1.2808, | |
| "odds_ratio_loss": 0.6708163022994995, | |
| "rewards/accuracies": 0.5625, | |
| "rewards/chosen": -0.12137186527252197, | |
| "rewards/margins": 0.013524973765015602, | |
| "rewards/rejected": -0.13489684462547302, | |
| "sft_loss": 1.2137186527252197, | |
| "step": 960 | |
| }, | |
| { | |
| "epoch": 1.5679935340472824, | |
| "grad_norm": 2.5744829177856445, | |
| "learning_rate": 2.3182044505730364e-06, | |
| "logits/chosen": -20.806903839111328, | |
| "logits/rejected": -20.799480438232422, | |
| "logps/chosen": -1.170987606048584, | |
| "logps/rejected": -1.3193377256393433, | |
| "loss": 1.2392, | |
| "odds_ratio_loss": 0.6825143098831177, | |
| "rewards/accuracies": 0.512499988079071, | |
| "rewards/chosen": -0.11709876358509064, | |
| "rewards/margins": 0.014834997244179249, | |
| "rewards/rejected": -0.1319337785243988, | |
| "sft_loss": 1.170987606048584, | |
| "step": 970 | |
| }, | |
| { | |
| "epoch": 1.5841584158415842, | |
| "grad_norm": 3.971120834350586, | |
| "learning_rate": 2.275980088319941e-06, | |
| "logits/chosen": -21.002077102661133, | |
| "logits/rejected": -20.75215721130371, | |
| "logps/chosen": -1.2036049365997314, | |
| "logps/rejected": -1.2877798080444336, | |
| "loss": 1.2758, | |
| "odds_ratio_loss": 0.7220691442489624, | |
| "rewards/accuracies": 0.5375000238418579, | |
| "rewards/chosen": -0.12036049365997314, | |
| "rewards/margins": 0.008417483419179916, | |
| "rewards/rejected": -0.12877798080444336, | |
| "sft_loss": 1.2036049365997314, | |
| "step": 980 | |
| }, | |
| { | |
| "epoch": 1.600323297635886, | |
| "grad_norm": 2.0432345867156982, | |
| "learning_rate": 2.2338200545580577e-06, | |
| "logits/chosen": -20.597959518432617, | |
| "logits/rejected": -20.82900619506836, | |
| "logps/chosen": -1.182715654373169, | |
| "logps/rejected": -1.4004007577896118, | |
| "loss": 1.2503, | |
| "odds_ratio_loss": 0.6758050322532654, | |
| "rewards/accuracies": 0.5375000238418579, | |
| "rewards/chosen": -0.11827157437801361, | |
| "rewards/margins": 0.021768497303128242, | |
| "rewards/rejected": -0.1400400698184967, | |
| "sft_loss": 1.182715654373169, | |
| "step": 990 | |
| }, | |
| { | |
| "epoch": 1.616488179430188, | |
| "grad_norm": 2.0639455318450928, | |
| "learning_rate": 2.191736455761947e-06, | |
| "logits/chosen": -20.99373435974121, | |
| "logits/rejected": -20.91073989868164, | |
| "logps/chosen": -1.153194785118103, | |
| "logps/rejected": -1.2575992345809937, | |
| "loss": 1.2196, | |
| "odds_ratio_loss": 0.663902223110199, | |
| "rewards/accuracies": 0.5562499761581421, | |
| "rewards/chosen": -0.11531949043273926, | |
| "rewards/margins": 0.010440433397889137, | |
| "rewards/rejected": -0.12575991451740265, | |
| "sft_loss": 1.153194785118103, | |
| "step": 1000 | |
| }, | |
| { | |
| "epoch": 1.616488179430188, | |
| "eval_logits/chosen": -20.882339477539062, | |
| "eval_logits/rejected": -20.911054611206055, | |
| "eval_logps/chosen": -1.2678194046020508, | |
| "eval_logps/rejected": -1.4078764915466309, | |
| "eval_loss": 1.3396929502487183, | |
| "eval_odds_ratio_loss": 0.7187345623970032, | |
| "eval_rewards/accuracies": 0.5354545712471008, | |
| "eval_rewards/chosen": -0.12678195536136627, | |
| "eval_rewards/margins": 0.01400570198893547, | |
| "eval_rewards/rejected": -0.14078764617443085, | |
| "eval_runtime": 158.823, | |
| "eval_samples_per_second": 6.926, | |
| "eval_sft_loss": 1.2678194046020508, | |
| "eval_steps_per_second": 3.463, | |
| "step": 1000 | |
| }, | |
| { | |
| "epoch": 1.6326530612244898, | |
| "grad_norm": 1.447840929031372, | |
| "learning_rate": 2.1497413764574673e-06, | |
| "logits/chosen": -21.098125457763672, | |
| "logits/rejected": -21.02555274963379, | |
| "logps/chosen": -1.2790714502334595, | |
| "logps/rejected": -1.4720830917358398, | |
| "loss": 1.3456, | |
| "odds_ratio_loss": 0.6656414270401001, | |
| "rewards/accuracies": 0.59375, | |
| "rewards/chosen": -0.1279071420431137, | |
| "rewards/margins": 0.01930117979645729, | |
| "rewards/rejected": -0.1472083032131195, | |
| "sft_loss": 1.2790714502334595, | |
| "step": 1010 | |
| }, | |
| { | |
| "epoch": 1.6488179430187917, | |
| "grad_norm": 2.2929062843322754, | |
| "learning_rate": 2.1078468757516395e-06, | |
| "logits/chosen": -21.029102325439453, | |
| "logits/rejected": -20.96735382080078, | |
| "logps/chosen": -1.1978142261505127, | |
| "logps/rejected": -1.2682490348815918, | |
| "loss": 1.2676, | |
| "odds_ratio_loss": 0.6979144811630249, | |
| "rewards/accuracies": 0.512499988079071, | |
| "rewards/chosen": -0.11978141963481903, | |
| "rewards/margins": 0.007043488323688507, | |
| "rewards/rejected": -0.12682488560676575, | |
| "sft_loss": 1.1978142261505127, | |
| "step": 1020 | |
| }, | |
| { | |
| "epoch": 1.6649828248130936, | |
| "grad_norm": 1.679506778717041, | |
| "learning_rate": 2.0660649838698145e-06, | |
| "logits/chosen": -21.126895904541016, | |
| "logits/rejected": -21.121044158935547, | |
| "logps/chosen": -1.2263692617416382, | |
| "logps/rejected": -1.3581560850143433, | |
| "loss": 1.2969, | |
| "odds_ratio_loss": 0.7051782011985779, | |
| "rewards/accuracies": 0.512499988079071, | |
| "rewards/chosen": -0.12263691425323486, | |
| "rewards/margins": 0.013178691267967224, | |
| "rewards/rejected": -0.1358156055212021, | |
| "sft_loss": 1.2263692617416382, | |
| "step": 1030 | |
| }, | |
| { | |
| "epoch": 1.6811477066073954, | |
| "grad_norm": 1.5033296346664429, | |
| "learning_rate": 2.0244076987011284e-06, | |
| "logits/chosen": -21.097610473632812, | |
| "logits/rejected": -21.023983001708984, | |
| "logps/chosen": -1.263479232788086, | |
| "logps/rejected": -1.430101990699768, | |
| "loss": 1.3286, | |
| "odds_ratio_loss": 0.6507799029350281, | |
| "rewards/accuracies": 0.581250011920929, | |
| "rewards/chosen": -0.12634792923927307, | |
| "rewards/margins": 0.01666227914392948, | |
| "rewards/rejected": -0.1430101990699768, | |
| "sft_loss": 1.263479232788086, | |
| "step": 1040 | |
| }, | |
| { | |
| "epoch": 1.6973125884016973, | |
| "grad_norm": 1.782820463180542, | |
| "learning_rate": 1.982886982353251e-06, | |
| "logits/chosen": -20.983699798583984, | |
| "logits/rejected": -20.792827606201172, | |
| "logps/chosen": -1.2545843124389648, | |
| "logps/rejected": -1.4638617038726807, | |
| "loss": 1.3236, | |
| "odds_ratio_loss": 0.6896650195121765, | |
| "rewards/accuracies": 0.512499988079071, | |
| "rewards/chosen": -0.12545843422412872, | |
| "rewards/margins": 0.020927732810378075, | |
| "rewards/rejected": -0.14638619124889374, | |
| "sft_loss": 1.2545843124389648, | |
| "step": 1050 | |
| }, | |
| { | |
| "epoch": 1.7134774701959992, | |
| "grad_norm": 2.0116512775421143, | |
| "learning_rate": 1.941514757717392e-06, | |
| "logits/chosen": -20.63968276977539, | |
| "logits/rejected": -20.679515838623047, | |
| "logps/chosen": -1.2405683994293213, | |
| "logps/rejected": -1.4277904033660889, | |
| "loss": 1.3065, | |
| "odds_ratio_loss": 0.6590927839279175, | |
| "rewards/accuracies": 0.5687500238418579, | |
| "rewards/chosen": -0.12405683845281601, | |
| "rewards/margins": 0.018722202628850937, | |
| "rewards/rejected": -0.14277902245521545, | |
| "sft_loss": 1.2405683994293213, | |
| "step": 1060 | |
| }, | |
| { | |
| "epoch": 1.729642351990301, | |
| "grad_norm": 2.4715521335601807, | |
| "learning_rate": 1.9003029050445953e-06, | |
| "logits/chosen": -20.77858543395996, | |
| "logits/rejected": -20.901081085205078, | |
| "logps/chosen": -1.2767174243927002, | |
| "logps/rejected": -1.3766672611236572, | |
| "loss": 1.3478, | |
| "odds_ratio_loss": 0.7110922336578369, | |
| "rewards/accuracies": 0.5062500238418579, | |
| "rewards/chosen": -0.1276717483997345, | |
| "rewards/margins": 0.00999497715383768, | |
| "rewards/rejected": -0.137666717171669, | |
| "sft_loss": 1.2767174243927002, | |
| "step": 1070 | |
| }, | |
| { | |
| "epoch": 1.745807233784603, | |
| "grad_norm": 1.7757654190063477, | |
| "learning_rate": 1.8592632585342523e-06, | |
| "logits/chosen": -20.718509674072266, | |
| "logits/rejected": -20.882915496826172, | |
| "logps/chosen": -1.2459039688110352, | |
| "logps/rejected": -1.3744847774505615, | |
| "loss": 1.3167, | |
| "odds_ratio_loss": 0.7076380848884583, | |
| "rewards/accuracies": 0.512499988079071, | |
| "rewards/chosen": -0.12459039688110352, | |
| "rewards/margins": 0.012858082540333271, | |
| "rewards/rejected": -0.13744845986366272, | |
| "sft_loss": 1.2459039688110352, | |
| "step": 1080 | |
| }, | |
| { | |
| "epoch": 1.7619721155789048, | |
| "grad_norm": 8.900938034057617, | |
| "learning_rate": 1.8184076029358527e-06, | |
| "logits/chosen": -20.89524269104004, | |
| "logits/rejected": -20.46439552307129, | |
| "logps/chosen": -1.1931703090667725, | |
| "logps/rejected": -1.2244675159454346, | |
| "loss": 1.2657, | |
| "odds_ratio_loss": 0.725264310836792, | |
| "rewards/accuracies": 0.5249999761581421, | |
| "rewards/chosen": -0.11931703239679337, | |
| "rewards/margins": 0.003129716496914625, | |
| "rewards/rejected": -0.12244673818349838, | |
| "sft_loss": 1.1931703090667725, | |
| "step": 1090 | |
| }, | |
| { | |
| "epoch": 1.7781369973732066, | |
| "grad_norm": 1.4426664113998413, | |
| "learning_rate": 1.7777476701649318e-06, | |
| "logits/chosen": -20.599040985107422, | |
| "logits/rejected": -20.622554779052734, | |
| "logps/chosen": -1.2419286966323853, | |
| "logps/rejected": -1.3642092943191528, | |
| "loss": 1.3125, | |
| "odds_ratio_loss": 0.7057370543479919, | |
| "rewards/accuracies": 0.512499988079071, | |
| "rewards/chosen": -0.12419287115335464, | |
| "rewards/margins": 0.012228056788444519, | |
| "rewards/rejected": -0.13642092049121857, | |
| "sft_loss": 1.2419286966323853, | |
| "step": 1100 | |
| }, | |
| { | |
| "epoch": 1.7943018791675085, | |
| "grad_norm": 2.3391785621643066, | |
| "learning_rate": 1.7372951359341925e-06, | |
| "logits/chosen": -20.814512252807617, | |
| "logits/rejected": -20.616971969604492, | |
| "logps/chosen": -1.183668851852417, | |
| "logps/rejected": -1.3042553663253784, | |
| "loss": 1.2526, | |
| "odds_ratio_loss": 0.6895106434822083, | |
| "rewards/accuracies": 0.518750011920929, | |
| "rewards/chosen": -0.11836691200733185, | |
| "rewards/margins": 0.012058648280799389, | |
| "rewards/rejected": -0.13042554259300232, | |
| "sft_loss": 1.183668851852417, | |
| "step": 1110 | |
| }, | |
| { | |
| "epoch": 1.8104667609618104, | |
| "grad_norm": 3.773312568664551, | |
| "learning_rate": 1.6970616164007547e-06, | |
| "logits/chosen": -20.64287567138672, | |
| "logits/rejected": -20.485708236694336, | |
| "logps/chosen": -1.1766704320907593, | |
| "logps/rejected": -1.3071858882904053, | |
| "loss": 1.2473, | |
| "odds_ratio_loss": 0.7061667442321777, | |
| "rewards/accuracies": 0.5687500238418579, | |
| "rewards/chosen": -0.11766704171895981, | |
| "rewards/margins": 0.013051542453467846, | |
| "rewards/rejected": -0.13071858882904053, | |
| "sft_loss": 1.1766704320907593, | |
| "step": 1120 | |
| }, | |
| { | |
| "epoch": 1.8266316427561122, | |
| "grad_norm": 6.383511543273926, | |
| "learning_rate": 1.6570586648305276e-06, | |
| "logits/chosen": -20.542699813842773, | |
| "logits/rejected": -20.710830688476562, | |
| "logps/chosen": -1.2573511600494385, | |
| "logps/rejected": -1.453174114227295, | |
| "loss": 1.3268, | |
| "odds_ratio_loss": 0.6940013766288757, | |
| "rewards/accuracies": 0.550000011920929, | |
| "rewards/chosen": -0.1257351189851761, | |
| "rewards/margins": 0.019582299515604973, | |
| "rewards/rejected": -0.1453174203634262, | |
| "sft_loss": 1.2573511600494385, | |
| "step": 1130 | |
| }, | |
| { | |
| "epoch": 1.842796524550414, | |
| "grad_norm": 2.227006196975708, | |
| "learning_rate": 1.6172977682806151e-06, | |
| "logits/chosen": -20.67087173461914, | |
| "logits/rejected": -20.963016510009766, | |
| "logps/chosen": -1.254607915878296, | |
| "logps/rejected": -1.4263418912887573, | |
| "loss": 1.32, | |
| "odds_ratio_loss": 0.6536819338798523, | |
| "rewards/accuracies": 0.5687500238418579, | |
| "rewards/chosen": -0.12546080350875854, | |
| "rewards/margins": 0.017173394560813904, | |
| "rewards/rejected": -0.14263419806957245, | |
| "sft_loss": 1.254607915878296, | |
| "step": 1140 | |
| }, | |
| { | |
| "epoch": 1.858961406344716, | |
| "grad_norm": 2.0494322776794434, | |
| "learning_rate": 1.5777903443007586e-06, | |
| "logits/chosen": -20.967281341552734, | |
| "logits/rejected": -20.45404624938965, | |
| "logps/chosen": -1.291761040687561, | |
| "logps/rejected": -1.4253360033035278, | |
| "loss": 1.3634, | |
| "odds_ratio_loss": 0.7167172431945801, | |
| "rewards/accuracies": 0.5625, | |
| "rewards/chosen": -0.12917611002922058, | |
| "rewards/margins": 0.013357502408325672, | |
| "rewards/rejected": -0.14253361523151398, | |
| "sft_loss": 1.291761040687561, | |
| "step": 1150 | |
| }, | |
| { | |
| "epoch": 1.8751262881390178, | |
| "grad_norm": 2.5553345680236816, | |
| "learning_rate": 1.5385477376547226e-06, | |
| "logits/chosen": -20.862293243408203, | |
| "logits/rejected": -20.802623748779297, | |
| "logps/chosen": -1.3029327392578125, | |
| "logps/rejected": -1.4472944736480713, | |
| "loss": 1.3688, | |
| "odds_ratio_loss": 0.6585663557052612, | |
| "rewards/accuracies": 0.5562499761581421, | |
| "rewards/chosen": -0.13029327988624573, | |
| "rewards/margins": 0.014436167664825916, | |
| "rewards/rejected": -0.14472945034503937, | |
| "sft_loss": 1.3029327392578125, | |
| "step": 1160 | |
| }, | |
| { | |
| "epoch": 1.89129116993332, | |
| "grad_norm": 1.7979766130447388, | |
| "learning_rate": 1.4995812170625845e-06, | |
| "logits/chosen": -20.667551040649414, | |
| "logits/rejected": -20.655712127685547, | |
| "logps/chosen": -1.317637324333191, | |
| "logps/rejected": -1.5292212963104248, | |
| "loss": 1.3849, | |
| "odds_ratio_loss": 0.6725908517837524, | |
| "rewards/accuracies": 0.5375000238418579, | |
| "rewards/chosen": -0.1317637413740158, | |
| "rewards/margins": 0.021158380433917046, | |
| "rewards/rejected": -0.152922123670578, | |
| "sft_loss": 1.317637324333191, | |
| "step": 1170 | |
| }, | |
| { | |
| "epoch": 1.9074560517276218, | |
| "grad_norm": 7.151007652282715, | |
| "learning_rate": 1.4609019719648666e-06, | |
| "logits/chosen": -21.043447494506836, | |
| "logits/rejected": -20.968263626098633, | |
| "logps/chosen": -1.2828052043914795, | |
| "logps/rejected": -1.4710747003555298, | |
| "loss": 1.3489, | |
| "odds_ratio_loss": 0.6608654856681824, | |
| "rewards/accuracies": 0.59375, | |
| "rewards/chosen": -0.12828053534030914, | |
| "rewards/margins": 0.018826942890882492, | |
| "rewards/rejected": -0.14710748195648193, | |
| "sft_loss": 1.2828052043914795, | |
| "step": 1180 | |
| }, | |
| { | |
| "epoch": 1.9236209335219236, | |
| "grad_norm": 3.7433314323425293, | |
| "learning_rate": 1.42252110930943e-06, | |
| "logits/chosen": -20.565446853637695, | |
| "logits/rejected": -20.41248893737793, | |
| "logps/chosen": -1.1272156238555908, | |
| "logps/rejected": -1.213004469871521, | |
| "loss": 1.2001, | |
| "odds_ratio_loss": 0.7286427021026611, | |
| "rewards/accuracies": 0.5062500238418579, | |
| "rewards/chosen": -0.11272156238555908, | |
| "rewards/margins": 0.008578887209296227, | |
| "rewards/rejected": -0.12130045890808105, | |
| "sft_loss": 1.1272156238555908, | |
| "step": 1190 | |
| }, | |
| { | |
| "epoch": 1.9397858153162255, | |
| "grad_norm": 2.740086793899536, | |
| "learning_rate": 1.3844496503620493e-06, | |
| "logits/chosen": -20.73773765563965, | |
| "logits/rejected": -20.874481201171875, | |
| "logps/chosen": -1.2707687616348267, | |
| "logps/rejected": -1.3804893493652344, | |
| "loss": 1.3383, | |
| "odds_ratio_loss": 0.6750306487083435, | |
| "rewards/accuracies": 0.5062500238418579, | |
| "rewards/chosen": -0.1270768642425537, | |
| "rewards/margins": 0.010972070507705212, | |
| "rewards/rejected": -0.1380489468574524, | |
| "sft_loss": 1.2707687616348267, | |
| "step": 1200 | |
| }, | |
| { | |
| "epoch": 1.9559506971105274, | |
| "grad_norm": 3.2234959602355957, | |
| "learning_rate": 1.3466985275416081e-06, | |
| "logits/chosen": -20.732952117919922, | |
| "logits/rejected": -20.362863540649414, | |
| "logps/chosen": -1.3259879350662231, | |
| "logps/rejected": -1.436372995376587, | |
| "loss": 1.4005, | |
| "odds_ratio_loss": 0.7446285486221313, | |
| "rewards/accuracies": 0.543749988079071, | |
| "rewards/chosen": -0.13259878754615784, | |
| "rewards/margins": 0.011038513854146004, | |
| "rewards/rejected": -0.14363731443881989, | |
| "sft_loss": 1.3259879350662231, | |
| "step": 1210 | |
| }, | |
| { | |
| "epoch": 1.9721155789048292, | |
| "grad_norm": 2.2601141929626465, | |
| "learning_rate": 1.309278581280791e-06, | |
| "logits/chosen": -20.75266456604004, | |
| "logits/rejected": -20.531200408935547, | |
| "logps/chosen": -1.2139203548431396, | |
| "logps/rejected": -1.3907318115234375, | |
| "loss": 1.2807, | |
| "odds_ratio_loss": 0.6679332852363586, | |
| "rewards/accuracies": 0.59375, | |
| "rewards/chosen": -0.12139203399419785, | |
| "rewards/margins": 0.017681140452623367, | |
| "rewards/rejected": -0.1390731781721115, | |
| "sft_loss": 1.2139203548431396, | |
| "step": 1220 | |
| }, | |
| { | |
| "epoch": 1.9882804606991311, | |
| "grad_norm": 1.351995825767517, | |
| "learning_rate": 1.272200556913199e-06, | |
| "logits/chosen": -20.665508270263672, | |
| "logits/rejected": -20.715679168701172, | |
| "logps/chosen": -1.2477220296859741, | |
| "logps/rejected": -1.4040950536727905, | |
| "loss": 1.3204, | |
| "odds_ratio_loss": 0.7266369462013245, | |
| "rewards/accuracies": 0.550000011920929, | |
| "rewards/chosen": -0.12477221339941025, | |
| "rewards/margins": 0.015637310221791267, | |
| "rewards/rejected": -0.14040951430797577, | |
| "sft_loss": 1.2477220296859741, | |
| "step": 1230 | |
| }, | |
| { | |
| "epoch": 2.004445342493433, | |
| "grad_norm": 3.1234052181243896, | |
| "learning_rate": 1.2354751015877698e-06, | |
| "logits/chosen": -20.652090072631836, | |
| "logits/rejected": -20.866174697875977, | |
| "logps/chosen": -1.1843925714492798, | |
| "logps/rejected": -1.4374680519104004, | |
| "loss": 1.2509, | |
| "odds_ratio_loss": 0.6650271415710449, | |
| "rewards/accuracies": 0.5249999761581421, | |
| "rewards/chosen": -0.11843927204608917, | |
| "rewards/margins": 0.02530754543840885, | |
| "rewards/rejected": -0.14374680817127228, | |
| "sft_loss": 1.1843925714492798, | |
| "step": 1240 | |
| }, | |
| { | |
| "epoch": 2.020610224287735, | |
| "grad_norm": 2.8965847492218018, | |
| "learning_rate": 1.1991127612113945e-06, | |
| "logits/chosen": -20.925878524780273, | |
| "logits/rejected": -21.110370635986328, | |
| "logps/chosen": -1.2498103380203247, | |
| "logps/rejected": -1.4370229244232178, | |
| "loss": 1.3151, | |
| "odds_ratio_loss": 0.6532084345817566, | |
| "rewards/accuracies": 0.574999988079071, | |
| "rewards/chosen": -0.12498103082180023, | |
| "rewards/margins": 0.018721258267760277, | |
| "rewards/rejected": -0.14370229840278625, | |
| "sft_loss": 1.2498103380203247, | |
| "step": 1250 | |
| }, | |
| { | |
| "epoch": 2.036775106082037, | |
| "grad_norm": 2.9547386169433594, | |
| "learning_rate": 1.1631239774206035e-06, | |
| "logits/chosen": -20.5147705078125, | |
| "logits/rejected": -20.376428604125977, | |
| "logps/chosen": -1.236487627029419, | |
| "logps/rejected": -1.4057966470718384, | |
| "loss": 1.3074, | |
| "odds_ratio_loss": 0.7092265486717224, | |
| "rewards/accuracies": 0.518750011920929, | |
| "rewards/chosen": -0.12364877760410309, | |
| "rewards/margins": 0.01693090796470642, | |
| "rewards/rejected": -0.14057967066764832, | |
| "sft_loss": 1.236487627029419, | |
| "step": 1260 | |
| }, | |
| { | |
| "epoch": 2.052939987876339, | |
| "grad_norm": 3.2073123455047607, | |
| "learning_rate": 1.1275190845831978e-06, | |
| "logits/chosen": -20.78199005126953, | |
| "logits/rejected": -20.76191520690918, | |
| "logps/chosen": -1.211749792098999, | |
| "logps/rejected": -1.4196996688842773, | |
| "loss": 1.2743, | |
| "odds_ratio_loss": 0.6256077289581299, | |
| "rewards/accuracies": 0.625, | |
| "rewards/chosen": -0.12117497622966766, | |
| "rewards/margins": 0.020794976502656937, | |
| "rewards/rejected": -0.1419699490070343, | |
| "sft_loss": 1.211749792098999, | |
| "step": 1270 | |
| }, | |
| { | |
| "epoch": 2.0691048696706407, | |
| "grad_norm": 1.841145634651184, | |
| "learning_rate": 1.0923083068306778e-06, | |
| "logits/chosen": -20.74366569519043, | |
| "logits/rejected": -20.469417572021484, | |
| "logps/chosen": -1.1939375400543213, | |
| "logps/rejected": -1.4483907222747803, | |
| "loss": 1.2578, | |
| "odds_ratio_loss": 0.6386052966117859, | |
| "rewards/accuracies": 0.5687500238418579, | |
| "rewards/chosen": -0.11939376592636108, | |
| "rewards/margins": 0.025445321574807167, | |
| "rewards/rejected": -0.1448390930891037, | |
| "sft_loss": 1.1939375400543213, | |
| "step": 1280 | |
| }, | |
| { | |
| "epoch": 2.0852697514649425, | |
| "grad_norm": 1.7583280801773071, | |
| "learning_rate": 1.0575017551223348e-06, | |
| "logits/chosen": -20.574234008789062, | |
| "logits/rejected": -20.410676956176758, | |
| "logps/chosen": -1.1411545276641846, | |
| "logps/rejected": -1.3073662519454956, | |
| "loss": 1.2096, | |
| "odds_ratio_loss": 0.6839541792869568, | |
| "rewards/accuracies": 0.5625, | |
| "rewards/chosen": -0.11411545425653458, | |
| "rewards/margins": 0.016621166840195656, | |
| "rewards/rejected": -0.13073661923408508, | |
| "sft_loss": 1.1411545276641846, | |
| "step": 1290 | |
| }, | |
| { | |
| "epoch": 2.1014346332592444, | |
| "grad_norm": 1.8893145322799683, | |
| "learning_rate": 1.023109424341833e-06, | |
| "logits/chosen": -20.674358367919922, | |
| "logits/rejected": -20.841875076293945, | |
| "logps/chosen": -1.318074107170105, | |
| "logps/rejected": -1.4704669713974, | |
| "loss": 1.3891, | |
| "odds_ratio_loss": 0.7104289531707764, | |
| "rewards/accuracies": 0.574999988079071, | |
| "rewards/chosen": -0.13180741667747498, | |
| "rewards/margins": 0.01523929089307785, | |
| "rewards/rejected": -0.14704671502113342, | |
| "sft_loss": 1.318074107170105, | |
| "step": 1300 | |
| }, | |
| { | |
| "epoch": 2.1175995150535463, | |
| "grad_norm": 1.844677209854126, | |
| "learning_rate": 9.891411904271273e-07, | |
| "logits/chosen": -20.51424217224121, | |
| "logits/rejected": -20.641857147216797, | |
| "logps/chosen": -1.168316125869751, | |
| "logps/rejected": -1.3018604516983032, | |
| "loss": 1.2364, | |
| "odds_ratio_loss": 0.6810965538024902, | |
| "rewards/accuracies": 0.5687500238418579, | |
| "rewards/chosen": -0.11683161556720734, | |
| "rewards/margins": 0.013354445807635784, | |
| "rewards/rejected": -0.1301860511302948, | |
| "sft_loss": 1.168316125869751, | |
| "step": 1310 | |
| }, | |
| { | |
| "epoch": 2.133764396847848, | |
| "grad_norm": 1.6055060625076294, | |
| "learning_rate": 9.556068075345363e-07, | |
| "logits/chosen": -20.898950576782227, | |
| "logits/rejected": -20.616153717041016, | |
| "logps/chosen": -1.2051537036895752, | |
| "logps/rejected": -1.2989599704742432, | |
| "loss": 1.2733, | |
| "odds_ratio_loss": 0.6817353367805481, | |
| "rewards/accuracies": 0.574999988079071, | |
| "rewards/chosen": -0.1205153614282608, | |
| "rewards/margins": 0.009380649775266647, | |
| "rewards/rejected": -0.12989601492881775, | |
| "sft_loss": 1.2051537036895752, | |
| "step": 1320 | |
| }, | |
| { | |
| "epoch": 2.14992927864215, | |
| "grad_norm": 1.451566219329834, | |
| "learning_rate": 9.225159052377838e-07, | |
| "logits/chosen": -20.857812881469727, | |
| "logits/rejected": -20.955089569091797, | |
| "logps/chosen": -1.242044448852539, | |
| "logps/rejected": -1.4105781316757202, | |
| "loss": 1.311, | |
| "odds_ratio_loss": 0.6894499063491821, | |
| "rewards/accuracies": 0.5062500238418579, | |
| "rewards/chosen": -0.12420444190502167, | |
| "rewards/margins": 0.016853367909789085, | |
| "rewards/rejected": -0.1410578191280365, | |
| "sft_loss": 1.242044448852539, | |
| "step": 1330 | |
| }, | |
| { | |
| "epoch": 2.166094160436452, | |
| "grad_norm": 3.742938995361328, | |
| "learning_rate": 8.898779857628184e-07, | |
| "logits/chosen": -20.516098022460938, | |
| "logits/rejected": -20.78036117553711, | |
| "logps/chosen": -1.1591819524765015, | |
| "logps/rejected": -1.2938919067382812, | |
| "loss": 1.2271, | |
| "odds_ratio_loss": 0.6796503663063049, | |
| "rewards/accuracies": 0.5687500238418579, | |
| "rewards/chosen": -0.11591820418834686, | |
| "rewards/margins": 0.013470989651978016, | |
| "rewards/rejected": -0.1293891966342926, | |
| "sft_loss": 1.1591819524765015, | |
| "step": 1340 | |
| }, | |
| { | |
| "epoch": 2.1822590422307537, | |
| "grad_norm": 1.6822633743286133, | |
| "learning_rate": 8.577024212591975e-07, | |
| "logits/chosen": -21.10406494140625, | |
| "logits/rejected": -20.88094711303711, | |
| "logps/chosen": -1.2675734758377075, | |
| "logps/rejected": -1.3737549781799316, | |
| "loss": 1.3373, | |
| "odds_ratio_loss": 0.6970787048339844, | |
| "rewards/accuracies": 0.5249999761581421, | |
| "rewards/chosen": -0.12675733864307404, | |
| "rewards/margins": 0.010618140920996666, | |
| "rewards/rejected": -0.13737548887729645, | |
| "sft_loss": 1.2675734758377075, | |
| "step": 1350 | |
| }, | |
| { | |
| "epoch": 2.1984239240250556, | |
| "grad_norm": 2.0817370414733887, | |
| "learning_rate": 8.259984511088276e-07, | |
| "logits/chosen": -20.762319564819336, | |
| "logits/rejected": -20.748157501220703, | |
| "logps/chosen": -1.2708746194839478, | |
| "logps/rejected": -1.3757803440093994, | |
| "loss": 1.3444, | |
| "odds_ratio_loss": 0.7355143427848816, | |
| "rewards/accuracies": 0.5062500238418579, | |
| "rewards/chosen": -0.12708747386932373, | |
| "rewards/margins": 0.010490546002984047, | |
| "rewards/rejected": -0.13757804036140442, | |
| "sft_loss": 1.2708746194839478, | |
| "step": 1360 | |
| }, | |
| { | |
| "epoch": 2.2145888058193575, | |
| "grad_norm": 1.3843287229537964, | |
| "learning_rate": 7.947751792728237e-07, | |
| "logits/chosen": -20.849414825439453, | |
| "logits/rejected": -20.718679428100586, | |
| "logps/chosen": -1.1914303302764893, | |
| "logps/rejected": -1.3572627305984497, | |
| "loss": 1.257, | |
| "odds_ratio_loss": 0.6554330587387085, | |
| "rewards/accuracies": 0.5874999761581421, | |
| "rewards/chosen": -0.119143046438694, | |
| "rewards/margins": 0.016583234071731567, | |
| "rewards/rejected": -0.13572627305984497, | |
| "sft_loss": 1.1914303302764893, | |
| "step": 1370 | |
| }, | |
| { | |
| "epoch": 2.2307536876136593, | |
| "grad_norm": 10.062128067016602, | |
| "learning_rate": 7.640415716772626e-07, | |
| "logits/chosen": -20.725990295410156, | |
| "logits/rejected": -20.761856079101562, | |
| "logps/chosen": -1.3113725185394287, | |
| "logps/rejected": -1.4960176944732666, | |
| "loss": 1.381, | |
| "odds_ratio_loss": 0.6963750123977661, | |
| "rewards/accuracies": 0.5874999761581421, | |
| "rewards/chosen": -0.13113726675510406, | |
| "rewards/margins": 0.01846454106271267, | |
| "rewards/rejected": -0.14960181713104248, | |
| "sft_loss": 1.3113725185394287, | |
| "step": 1380 | |
| }, | |
| { | |
| "epoch": 2.246918569407961, | |
| "grad_norm": 2.1556107997894287, | |
| "learning_rate": 7.338064536385722e-07, | |
| "logits/chosen": -20.7343692779541, | |
| "logits/rejected": -20.6729793548584, | |
| "logps/chosen": -1.2534866333007812, | |
| "logps/rejected": -1.4300428628921509, | |
| "loss": 1.3205, | |
| "odds_ratio_loss": 0.6699201464653015, | |
| "rewards/accuracies": 0.581250011920929, | |
| "rewards/chosen": -0.12534865736961365, | |
| "rewards/margins": 0.017655618488788605, | |
| "rewards/rejected": -0.14300426840782166, | |
| "sft_loss": 1.2534866333007812, | |
| "step": 1390 | |
| }, | |
| { | |
| "epoch": 2.263083451202263, | |
| "grad_norm": 2.0938661098480225, | |
| "learning_rate": 7.040785073292883e-07, | |
| "logits/chosen": -20.498939514160156, | |
| "logits/rejected": -20.509479522705078, | |
| "logps/chosen": -1.3050354719161987, | |
| "logps/rejected": -1.4407126903533936, | |
| "loss": 1.3786, | |
| "odds_ratio_loss": 0.7356770038604736, | |
| "rewards/accuracies": 0.543749988079071, | |
| "rewards/chosen": -0.1305035650730133, | |
| "rewards/margins": 0.013567718677222729, | |
| "rewards/rejected": -0.14407126605510712, | |
| "sft_loss": 1.3050354719161987, | |
| "step": 1400 | |
| }, | |
| { | |
| "epoch": 2.279248332996565, | |
| "grad_norm": 2.8200390338897705, | |
| "learning_rate": 6.748662692849297e-07, | |
| "logits/chosen": -20.943716049194336, | |
| "logits/rejected": -20.97843360900879, | |
| "logps/chosen": -1.2242540121078491, | |
| "logps/rejected": -1.4535945653915405, | |
| "loss": 1.2894, | |
| "odds_ratio_loss": 0.6512330770492554, | |
| "rewards/accuracies": 0.543749988079071, | |
| "rewards/chosen": -0.12242539972066879, | |
| "rewards/margins": 0.02293405309319496, | |
| "rewards/rejected": -0.14535947144031525, | |
| "sft_loss": 1.2242540121078491, | |
| "step": 1410 | |
| }, | |
| { | |
| "epoch": 2.295413214790867, | |
| "grad_norm": 3.1267454624176025, | |
| "learning_rate": 6.46178127952686e-07, | |
| "logits/chosen": -20.665185928344727, | |
| "logits/rejected": -20.615243911743164, | |
| "logps/chosen": -1.2219477891921997, | |
| "logps/rejected": -1.3721575736999512, | |
| "loss": 1.2881, | |
| "odds_ratio_loss": 0.6616476774215698, | |
| "rewards/accuracies": 0.5249999761581421, | |
| "rewards/chosen": -0.12219478189945221, | |
| "rewards/margins": 0.015020984224975109, | |
| "rewards/rejected": -0.1372157633304596, | |
| "sft_loss": 1.2219477891921997, | |
| "step": 1420 | |
| }, | |
| { | |
| "epoch": 2.3115780965851687, | |
| "grad_norm": 2.4645252227783203, | |
| "learning_rate": 6.180223212826289e-07, | |
| "logits/chosen": -20.52808380126953, | |
| "logits/rejected": -20.359468460083008, | |
| "logps/chosen": -1.2114827632904053, | |
| "logps/rejected": -1.3234493732452393, | |
| "loss": 1.2822, | |
| "odds_ratio_loss": 0.7067694664001465, | |
| "rewards/accuracies": 0.543749988079071, | |
| "rewards/chosen": -0.12114828824996948, | |
| "rewards/margins": 0.011196643114089966, | |
| "rewards/rejected": -0.13234491646289825, | |
| "sft_loss": 1.2114827632904053, | |
| "step": 1430 | |
| }, | |
| { | |
| "epoch": 2.3277429783794705, | |
| "grad_norm": 1.1290780305862427, | |
| "learning_rate": 5.904069343621443e-07, | |
| "logits/chosen": -20.84099006652832, | |
| "logits/rejected": -20.68367576599121, | |
| "logps/chosen": -1.1913143396377563, | |
| "logps/rejected": -1.3442729711532593, | |
| "loss": 1.2568, | |
| "odds_ratio_loss": 0.6552363634109497, | |
| "rewards/accuracies": 0.5687500238418579, | |
| "rewards/chosen": -0.1191314235329628, | |
| "rewards/margins": 0.015295890159904957, | |
| "rewards/rejected": -0.13442730903625488, | |
| "sft_loss": 1.1913143396377563, | |
| "step": 1440 | |
| }, | |
| { | |
| "epoch": 2.3439078601737724, | |
| "grad_norm": 4.884983539581299, | |
| "learning_rate": 5.633398970942544e-07, | |
| "logits/chosen": -20.582796096801758, | |
| "logits/rejected": -20.532047271728516, | |
| "logps/chosen": -1.2106900215148926, | |
| "logps/rejected": -1.3333368301391602, | |
| "loss": 1.2794, | |
| "odds_ratio_loss": 0.6871799826622009, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": -0.12106901407241821, | |
| "rewards/margins": 0.012264668941497803, | |
| "rewards/rejected": -0.13333368301391602, | |
| "sft_loss": 1.2106900215148926, | |
| "step": 1450 | |
| }, | |
| { | |
| "epoch": 2.3600727419680743, | |
| "grad_norm": 2.4621684551239014, | |
| "learning_rate": 5.368289819205069e-07, | |
| "logits/chosen": -20.69870376586914, | |
| "logits/rejected": -20.650863647460938, | |
| "logps/chosen": -1.1822137832641602, | |
| "logps/rejected": -1.2959617376327515, | |
| "loss": 1.2522, | |
| "odds_ratio_loss": 0.6997024416923523, | |
| "rewards/accuracies": 0.512499988079071, | |
| "rewards/chosen": -0.11822137981653214, | |
| "rewards/margins": 0.011374806985259056, | |
| "rewards/rejected": -0.12959617376327515, | |
| "sft_loss": 1.1822137832641602, | |
| "step": 1460 | |
| }, | |
| { | |
| "epoch": 2.376237623762376, | |
| "grad_norm": 2.461702585220337, | |
| "learning_rate": 5.108818015890785e-07, | |
| "logits/chosen": -20.82188606262207, | |
| "logits/rejected": -20.84579086303711, | |
| "logps/chosen": -1.279842734336853, | |
| "logps/rejected": -1.449312448501587, | |
| "loss": 1.348, | |
| "odds_ratio_loss": 0.6815730333328247, | |
| "rewards/accuracies": 0.4937500059604645, | |
| "rewards/chosen": -0.12798430025577545, | |
| "rewards/margins": 0.016946956515312195, | |
| "rewards/rejected": -0.14493125677108765, | |
| "sft_loss": 1.279842734336853, | |
| "step": 1470 | |
| }, | |
| { | |
| "epoch": 2.392402505556678, | |
| "grad_norm": 2.5109152793884277, | |
| "learning_rate": 4.855058069687291e-07, | |
| "logits/chosen": -20.60469627380371, | |
| "logits/rejected": -20.50313949584961, | |
| "logps/chosen": -1.1870944499969482, | |
| "logps/rejected": -1.382088541984558, | |
| "loss": 1.254, | |
| "odds_ratio_loss": 0.6693797707557678, | |
| "rewards/accuracies": 0.5375000238418579, | |
| "rewards/chosen": -0.11870943009853363, | |
| "rewards/margins": 0.01949942298233509, | |
| "rewards/rejected": -0.13820883631706238, | |
| "sft_loss": 1.1870944499969482, | |
| "step": 1480 | |
| }, | |
| { | |
| "epoch": 2.40856738735098, | |
| "grad_norm": 2.1160638332366943, | |
| "learning_rate": 4.607082849092523e-07, | |
| "logits/chosen": -20.56966781616211, | |
| "logits/rejected": -20.43010902404785, | |
| "logps/chosen": -1.324727177619934, | |
| "logps/rejected": -1.4259936809539795, | |
| "loss": 1.3968, | |
| "odds_ratio_loss": 0.7211604118347168, | |
| "rewards/accuracies": 0.543749988079071, | |
| "rewards/chosen": -0.1324726939201355, | |
| "rewards/margins": 0.010126668028533459, | |
| "rewards/rejected": -0.14259937405586243, | |
| "sft_loss": 1.324727177619934, | |
| "step": 1490 | |
| }, | |
| { | |
| "epoch": 2.4247322691452817, | |
| "grad_norm": 4.8063836097717285, | |
| "learning_rate": 4.3649635614901405e-07, | |
| "logits/chosen": -20.565994262695312, | |
| "logits/rejected": -20.88812255859375, | |
| "logps/chosen": -1.2172143459320068, | |
| "logps/rejected": -1.277565836906433, | |
| "loss": 1.2883, | |
| "odds_ratio_loss": 0.711229145526886, | |
| "rewards/accuracies": 0.5249999761581421, | |
| "rewards/chosen": -0.12172143161296844, | |
| "rewards/margins": 0.006035161670297384, | |
| "rewards/rejected": -0.12775658071041107, | |
| "sft_loss": 1.2172143459320068, | |
| "step": 1500 | |
| }, | |
| { | |
| "epoch": 2.4247322691452817, | |
| "eval_logits/chosen": -20.66438865661621, | |
| "eval_logits/rejected": -20.695892333984375, | |
| "eval_logps/chosen": -1.2573115825653076, | |
| "eval_logps/rejected": -1.3996058702468872, | |
| "eval_loss": 1.3293771743774414, | |
| "eval_odds_ratio_loss": 0.7206552624702454, | |
| "eval_rewards/accuracies": 0.5345454812049866, | |
| "eval_rewards/chosen": -0.12573117017745972, | |
| "eval_rewards/margins": 0.014229409396648407, | |
| "eval_rewards/rejected": -0.13996057212352753, | |
| "eval_runtime": 82.3734, | |
| "eval_samples_per_second": 13.354, | |
| "eval_sft_loss": 1.2573115825653076, | |
| "eval_steps_per_second": 6.677, | |
| "step": 1500 | |
| }, | |
| { | |
| "epoch": 2.4408971509395836, | |
| "grad_norm": 2.092519998550415, | |
| "learning_rate": 4.128769732701973e-07, | |
| "logits/chosen": -20.61777114868164, | |
| "logits/rejected": -20.48794174194336, | |
| "logps/chosen": -1.2248725891113281, | |
| "logps/rejected": -1.3724851608276367, | |
| "loss": 1.2928, | |
| "odds_ratio_loss": 0.6797080039978027, | |
| "rewards/accuracies": 0.5375000238418579, | |
| "rewards/chosen": -0.12248726189136505, | |
| "rewards/margins": 0.014761263504624367, | |
| "rewards/rejected": -0.13724853098392487, | |
| "sft_loss": 1.2248725891113281, | |
| "step": 1510 | |
| }, | |
| { | |
| "epoch": 2.4570620327338855, | |
| "grad_norm": 2.445197582244873, | |
| "learning_rate": 3.8985691870233046e-07, | |
| "logits/chosen": -20.743959426879883, | |
| "logits/rejected": -20.582164764404297, | |
| "logps/chosen": -1.2659637928009033, | |
| "logps/rejected": -1.4508241415023804, | |
| "loss": 1.3339, | |
| "odds_ratio_loss": 0.678980827331543, | |
| "rewards/accuracies": 0.5062500238418579, | |
| "rewards/chosen": -0.1265963912010193, | |
| "rewards/margins": 0.01848602667450905, | |
| "rewards/rejected": -0.14508241415023804, | |
| "sft_loss": 1.2659637928009033, | |
| "step": 1520 | |
| }, | |
| { | |
| "epoch": 2.4732269145281873, | |
| "grad_norm": 6.312464714050293, | |
| "learning_rate": 3.6744280277467904e-07, | |
| "logits/chosen": -20.964641571044922, | |
| "logits/rejected": -21.01685333251953, | |
| "logps/chosen": -1.2571816444396973, | |
| "logps/rejected": -1.413922905921936, | |
| "loss": 1.3289, | |
| "odds_ratio_loss": 0.7169492244720459, | |
| "rewards/accuracies": 0.518750011920929, | |
| "rewards/chosen": -0.12571817636489868, | |
| "rewards/margins": 0.015674125403165817, | |
| "rewards/rejected": -0.1413923054933548, | |
| "sft_loss": 1.2571816444396973, | |
| "step": 1530 | |
| }, | |
| { | |
| "epoch": 2.489391796322489, | |
| "grad_norm": 2.195627450942993, | |
| "learning_rate": 3.456410618180503e-07, | |
| "logits/chosen": -19.93832778930664, | |
| "logits/rejected": -20.330801010131836, | |
| "logps/chosen": -1.1545674800872803, | |
| "logps/rejected": -1.4308401346206665, | |
| "loss": 1.2195, | |
| "odds_ratio_loss": 0.649765133857727, | |
| "rewards/accuracies": 0.5874999761581421, | |
| "rewards/chosen": -0.11545674502849579, | |
| "rewards/margins": 0.027627255767583847, | |
| "rewards/rejected": -0.14308401942253113, | |
| "sft_loss": 1.1545674800872803, | |
| "step": 1540 | |
| }, | |
| { | |
| "epoch": 2.5055566781167915, | |
| "grad_norm": 1.8385835886001587, | |
| "learning_rate": 3.244579563165753e-07, | |
| "logits/chosen": -20.591102600097656, | |
| "logits/rejected": -20.71021270751953, | |
| "logps/chosen": -1.2204065322875977, | |
| "logps/rejected": -1.452178955078125, | |
| "loss": 1.2861, | |
| "odds_ratio_loss": 0.65655916929245, | |
| "rewards/accuracies": 0.574999988079071, | |
| "rewards/chosen": -0.12204064428806305, | |
| "rewards/margins": 0.02317725121974945, | |
| "rewards/rejected": -0.1452178955078125, | |
| "sft_loss": 1.2204065322875977, | |
| "step": 1550 | |
| }, | |
| { | |
| "epoch": 2.521721559911093, | |
| "grad_norm": 1.5258744955062866, | |
| "learning_rate": 3.038995691099697e-07, | |
| "logits/chosen": -20.90871238708496, | |
| "logits/rejected": -20.73806381225586, | |
| "logps/chosen": -1.288413166999817, | |
| "logps/rejected": -1.4578988552093506, | |
| "loss": 1.36, | |
| "odds_ratio_loss": 0.7158418297767639, | |
| "rewards/accuracies": 0.5249999761581421, | |
| "rewards/chosen": -0.1288413256406784, | |
| "rewards/margins": 0.016948552802205086, | |
| "rewards/rejected": -0.14578989148139954, | |
| "sft_loss": 1.288413166999817, | |
| "step": 1560 | |
| }, | |
| { | |
| "epoch": 2.5378864417053952, | |
| "grad_norm": 2.4840152263641357, | |
| "learning_rate": 2.839718036468192e-07, | |
| "logits/chosen": -20.923898696899414, | |
| "logits/rejected": -20.989917755126953, | |
| "logps/chosen": -1.3150464296340942, | |
| "logps/rejected": -1.4090582132339478, | |
| "loss": 1.3879, | |
| "odds_ratio_loss": 0.7283130288124084, | |
| "rewards/accuracies": 0.5562499761581421, | |
| "rewards/chosen": -0.13150465488433838, | |
| "rewards/margins": 0.009401168674230576, | |
| "rewards/rejected": -0.14090582728385925, | |
| "sft_loss": 1.3150464296340942, | |
| "step": 1570 | |
| }, | |
| { | |
| "epoch": 2.5540513234996967, | |
| "grad_norm": 4.045677185058594, | |
| "learning_rate": 2.646803822893723e-07, | |
| "logits/chosen": -20.986186981201172, | |
| "logits/rejected": -20.914697647094727, | |
| "logps/chosen": -1.2622355222702026, | |
| "logps/rejected": -1.387225866317749, | |
| "loss": 1.3346, | |
| "odds_ratio_loss": 0.7240074872970581, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": -0.12622354924678802, | |
| "rewards/margins": 0.012499034404754639, | |
| "rewards/rejected": -0.13872259855270386, | |
| "sft_loss": 1.2622355222702026, | |
| "step": 1580 | |
| }, | |
| { | |
| "epoch": 2.570216205293999, | |
| "grad_norm": 1.6737409830093384, | |
| "learning_rate": 2.460308446703341e-07, | |
| "logits/chosen": -20.74363136291504, | |
| "logits/rejected": -20.45053482055664, | |
| "logps/chosen": -1.2218042612075806, | |
| "logps/rejected": -1.310227394104004, | |
| "loss": 1.293, | |
| "odds_ratio_loss": 0.7117012739181519, | |
| "rewards/accuracies": 0.574999988079071, | |
| "rewards/chosen": -0.12218043953180313, | |
| "rewards/margins": 0.008842318318784237, | |
| "rewards/rejected": -0.13102273643016815, | |
| "sft_loss": 1.2218042612075806, | |
| "step": 1590 | |
| }, | |
| { | |
| "epoch": 2.5863810870883004, | |
| "grad_norm": 2.487335443496704, | |
| "learning_rate": 2.2802854610213143e-07, | |
| "logits/chosen": -20.622318267822266, | |
| "logits/rejected": -20.454730987548828, | |
| "logps/chosen": -1.184971570968628, | |
| "logps/rejected": -1.4719953536987305, | |
| "loss": 1.2481, | |
| "odds_ratio_loss": 0.6310954689979553, | |
| "rewards/accuracies": 0.612500011920929, | |
| "rewards/chosen": -0.11849715560674667, | |
| "rewards/margins": 0.02870238944888115, | |
| "rewards/rejected": -0.14719954133033752, | |
| "sft_loss": 1.184971570968628, | |
| "step": 1600 | |
| }, | |
| { | |
| "epoch": 2.6025459688826027, | |
| "grad_norm": 7.610108375549316, | |
| "learning_rate": 2.106786560391072e-07, | |
| "logits/chosen": -20.3998966217041, | |
| "logits/rejected": -20.51192855834961, | |
| "logps/chosen": -1.2781306505203247, | |
| "logps/rejected": -1.3927931785583496, | |
| "loss": 1.3503, | |
| "odds_ratio_loss": 0.7214857339859009, | |
| "rewards/accuracies": 0.581250011920929, | |
| "rewards/chosen": -0.12781307101249695, | |
| "rewards/margins": 0.011466264724731445, | |
| "rewards/rejected": -0.139279305934906, | |
| "sft_loss": 1.2781306505203247, | |
| "step": 1610 | |
| }, | |
| { | |
| "epoch": 2.6187108506769046, | |
| "grad_norm": 2.2551307678222656, | |
| "learning_rate": 1.9398615659308255e-07, | |
| "logits/chosen": -20.576255798339844, | |
| "logits/rejected": -20.769161224365234, | |
| "logps/chosen": -1.2140244245529175, | |
| "logps/rejected": -1.2931668758392334, | |
| "loss": 1.2827, | |
| "odds_ratio_loss": 0.6866000294685364, | |
| "rewards/accuracies": 0.53125, | |
| "rewards/chosen": -0.12140244245529175, | |
| "rewards/margins": 0.007914242334663868, | |
| "rewards/rejected": -0.12931668758392334, | |
| "sft_loss": 1.2140244245529175, | |
| "step": 1620 | |
| }, | |
| { | |
| "epoch": 2.6348757324712064, | |
| "grad_norm": 2.147873640060425, | |
| "learning_rate": 1.7795584110272184e-07, | |
| "logits/chosen": -20.9218807220459, | |
| "logits/rejected": -20.841930389404297, | |
| "logps/chosen": -1.261558175086975, | |
| "logps/rejected": -1.3861316442489624, | |
| "loss": 1.3318, | |
| "odds_ratio_loss": 0.7022627592086792, | |
| "rewards/accuracies": 0.543749988079071, | |
| "rewards/chosen": -0.1261558085680008, | |
| "rewards/margins": 0.01245735865086317, | |
| "rewards/rejected": -0.13861316442489624, | |
| "sft_loss": 1.261558175086975, | |
| "step": 1630 | |
| }, | |
| { | |
| "epoch": 2.6510406142655083, | |
| "grad_norm": 6.040190696716309, | |
| "learning_rate": 1.6259231275709636e-07, | |
| "logits/chosen": -20.989246368408203, | |
| "logits/rejected": -20.884021759033203, | |
| "logps/chosen": -1.2266334295272827, | |
| "logps/rejected": -1.2976648807525635, | |
| "loss": 1.3008, | |
| "odds_ratio_loss": 0.7414904832839966, | |
| "rewards/accuracies": 0.5249999761581421, | |
| "rewards/chosen": -0.12266335636377335, | |
| "rewards/margins": 0.007103143725544214, | |
| "rewards/rejected": -0.12976649403572083, | |
| "sft_loss": 1.2266334295272827, | |
| "step": 1640 | |
| }, | |
| { | |
| "epoch": 2.66720549605981, | |
| "grad_norm": 1.8157051801681519, | |
| "learning_rate": 1.478999832738548e-07, | |
| "logits/chosen": -20.997966766357422, | |
| "logits/rejected": -20.761661529541016, | |
| "logps/chosen": -1.2066887617111206, | |
| "logps/rejected": -1.385434865951538, | |
| "loss": 1.276, | |
| "odds_ratio_loss": 0.6930679082870483, | |
| "rewards/accuracies": 0.5687500238418579, | |
| "rewards/chosen": -0.12066888809204102, | |
| "rewards/margins": 0.017874618992209435, | |
| "rewards/rejected": -0.138543501496315, | |
| "sft_loss": 1.2066887617111206, | |
| "step": 1650 | |
| }, | |
| { | |
| "epoch": 2.683370377854112, | |
| "grad_norm": 2.9221081733703613, | |
| "learning_rate": 1.338830716323769e-07, | |
| "logits/chosen": -20.6229248046875, | |
| "logits/rejected": -20.6382999420166, | |
| "logps/chosen": -1.2255455255508423, | |
| "logps/rejected": -1.3001086711883545, | |
| "loss": 1.2979, | |
| "odds_ratio_loss": 0.7236055135726929, | |
| "rewards/accuracies": 0.5062500238418579, | |
| "rewards/chosen": -0.12255455553531647, | |
| "rewards/margins": 0.007456316612660885, | |
| "rewards/rejected": -0.13001087307929993, | |
| "sft_loss": 1.2255455255508423, | |
| "step": 1660 | |
| }, | |
| { | |
| "epoch": 2.699535259648414, | |
| "grad_norm": 3.0684990882873535, | |
| "learning_rate": 1.205456028622723e-07, | |
| "logits/chosen": -20.87864112854004, | |
| "logits/rejected": -20.77214241027832, | |
| "logps/chosen": -1.1658565998077393, | |
| "logps/rejected": -1.406781554222107, | |
| "loss": 1.2305, | |
| "odds_ratio_loss": 0.6469210386276245, | |
| "rewards/accuracies": 0.543749988079071, | |
| "rewards/chosen": -0.11658565700054169, | |
| "rewards/margins": 0.024092501029372215, | |
| "rewards/rejected": -0.14067815244197845, | |
| "sft_loss": 1.1658565998077393, | |
| "step": 1670 | |
| }, | |
| { | |
| "epoch": 2.7157001414427158, | |
| "grad_norm": 1.2988094091415405, | |
| "learning_rate": 1.0789140688756805e-07, | |
| "logits/chosen": -21.001827239990234, | |
| "logits/rejected": -20.79971694946289, | |
| "logps/chosen": -1.2181146144866943, | |
| "logps/rejected": -1.434340238571167, | |
| "loss": 1.2805, | |
| "odds_ratio_loss": 0.6233752369880676, | |
| "rewards/accuracies": 0.625, | |
| "rewards/chosen": -0.12181148678064346, | |
| "rewards/margins": 0.0216225478798151, | |
| "rewards/rejected": -0.14343401789665222, | |
| "sft_loss": 1.2181146144866943, | |
| "step": 1680 | |
| }, | |
| { | |
| "epoch": 2.7318650232370176, | |
| "grad_norm": 4.973554611206055, | |
| "learning_rate": 9.592411742693098e-08, | |
| "logits/chosen": -20.57479476928711, | |
| "logits/rejected": -20.399307250976562, | |
| "logps/chosen": -1.2814916372299194, | |
| "logps/rejected": -1.3428285121917725, | |
| "loss": 1.3583, | |
| "odds_ratio_loss": 0.7676435112953186, | |
| "rewards/accuracies": 0.4937500059604645, | |
| "rewards/chosen": -0.12814916670322418, | |
| "rewards/margins": 0.006133680697530508, | |
| "rewards/rejected": -0.13428285717964172, | |
| "sft_loss": 1.2814916372299194, | |
| "step": 1690 | |
| }, | |
| { | |
| "epoch": 2.7480299050313195, | |
| "grad_norm": 1.3830405473709106, | |
| "learning_rate": 8.464717095022168e-08, | |
| "logits/chosen": -20.695613861083984, | |
| "logits/rejected": -20.391788482666016, | |
| "logps/chosen": -1.1915602684020996, | |
| "logps/rejected": -1.3796567916870117, | |
| "loss": 1.2587, | |
| "odds_ratio_loss": 0.671554684638977, | |
| "rewards/accuracies": 0.606249988079071, | |
| "rewards/chosen": -0.11915602535009384, | |
| "rewards/margins": 0.01880965754389763, | |
| "rewards/rejected": -0.13796567916870117, | |
| "sft_loss": 1.1915602684020996, | |
| "step": 1700 | |
| }, | |
| { | |
| "epoch": 2.7641947868256214, | |
| "grad_norm": 2.5777227878570557, | |
| "learning_rate": 7.406380569169841e-08, | |
| "logits/chosen": -20.552873611450195, | |
| "logits/rejected": -20.59903335571289, | |
| "logps/chosen": -1.2640081644058228, | |
| "logps/rejected": -1.334334373474121, | |
| "loss": 1.3366, | |
| "odds_ratio_loss": 0.7264189720153809, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": -0.12640079855918884, | |
| "rewards/margins": 0.0070326318964362144, | |
| "rewards/rejected": -0.13343344628810883, | |
| "sft_loss": 1.2640081644058228, | |
| "step": 1710 | |
| }, | |
| { | |
| "epoch": 2.7803596686199232, | |
| "grad_norm": 10.019956588745117, | |
| "learning_rate": 6.417706072013808e-08, | |
| "logits/chosen": -20.83199119567871, | |
| "logits/rejected": -21.08270263671875, | |
| "logps/chosen": -1.1969940662384033, | |
| "logps/rejected": -1.326841115951538, | |
| "loss": 1.2649, | |
| "odds_ratio_loss": 0.6789032816886902, | |
| "rewards/accuracies": 0.550000011920929, | |
| "rewards/chosen": -0.11969941854476929, | |
| "rewards/margins": 0.012984703294932842, | |
| "rewards/rejected": -0.1326841115951538, | |
| "sft_loss": 1.1969940662384033, | |
| "step": 1720 | |
| }, | |
| { | |
| "epoch": 2.796524550414225, | |
| "grad_norm": 6.8160624504089355, | |
| "learning_rate": 5.498977506615294e-08, | |
| "logits/chosen": -20.777956008911133, | |
| "logits/rejected": -20.86117172241211, | |
| "logps/chosen": -1.2800042629241943, | |
| "logps/rejected": -1.3593509197235107, | |
| "loss": 1.3517, | |
| "odds_ratio_loss": 0.7168198227882385, | |
| "rewards/accuracies": 0.5687500238418579, | |
| "rewards/chosen": -0.1280004382133484, | |
| "rewards/margins": 0.007934670895338058, | |
| "rewards/rejected": -0.13593511283397675, | |
| "sft_loss": 1.2800042629241943, | |
| "step": 1730 | |
| }, | |
| { | |
| "epoch": 2.812689432208527, | |
| "grad_norm": 1.6938509941101074, | |
| "learning_rate": 4.6504586906947756e-08, | |
| "logits/chosen": -21.01276206970215, | |
| "logits/rejected": -21.02105140686035, | |
| "logps/chosen": -1.2667747735977173, | |
| "logps/rejected": -1.3524465560913086, | |
| "loss": 1.3367, | |
| "odds_ratio_loss": 0.6990799903869629, | |
| "rewards/accuracies": 0.5625, | |
| "rewards/chosen": -0.126677468419075, | |
| "rewards/margins": 0.008567174896597862, | |
| "rewards/rejected": -0.13524465262889862, | |
| "sft_loss": 1.2667747735977173, | |
| "step": 1740 | |
| }, | |
| { | |
| "epoch": 2.828854314002829, | |
| "grad_norm": 6.788274765014648, | |
| "learning_rate": 3.8723932808754914e-08, | |
| "logits/chosen": -21.133237838745117, | |
| "logits/rejected": -21.116573333740234, | |
| "logps/chosen": -1.3402550220489502, | |
| "logps/rejected": -1.3987007141113281, | |
| "loss": 1.4126, | |
| "odds_ratio_loss": 0.7239315509796143, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": -0.13402549922466278, | |
| "rewards/margins": 0.005844591651111841, | |
| "rewards/rejected": -0.1398700773715973, | |
| "sft_loss": 1.3402550220489502, | |
| "step": 1750 | |
| }, | |
| { | |
| "epoch": 2.8450191957971307, | |
| "grad_norm": 4.773237228393555, | |
| "learning_rate": 3.1650047027158014e-08, | |
| "logits/chosen": -20.84109115600586, | |
| "logits/rejected": -21.033550262451172, | |
| "logps/chosen": -1.2243554592132568, | |
| "logps/rejected": -1.3495168685913086, | |
| "loss": 1.2905, | |
| "odds_ratio_loss": 0.6615921258926392, | |
| "rewards/accuracies": 0.5375000238418579, | |
| "rewards/chosen": -0.12243553251028061, | |
| "rewards/margins": 0.012516150251030922, | |
| "rewards/rejected": -0.13495168089866638, | |
| "sft_loss": 1.2243554592132568, | |
| "step": 1760 | |
| }, | |
| { | |
| "epoch": 2.8611840775914326, | |
| "grad_norm": 1.8300281763076782, | |
| "learning_rate": 2.5284960865517848e-08, | |
| "logits/chosen": -20.63343048095703, | |
| "logits/rejected": -20.709888458251953, | |
| "logps/chosen": -1.1276973485946655, | |
| "logps/rejected": -1.3769396543502808, | |
| "loss": 1.1906, | |
| "odds_ratio_loss": 0.6287329196929932, | |
| "rewards/accuracies": 0.574999988079071, | |
| "rewards/chosen": -0.11276974529027939, | |
| "rewards/margins": 0.024924224242568016, | |
| "rewards/rejected": -0.13769397139549255, | |
| "sft_loss": 1.1276973485946655, | |
| "step": 1770 | |
| }, | |
| { | |
| "epoch": 2.8773489593857344, | |
| "grad_norm": 1.8408336639404297, | |
| "learning_rate": 1.9630502091670388e-08, | |
| "logits/chosen": -20.521255493164062, | |
| "logits/rejected": -20.47439956665039, | |
| "logps/chosen": -1.2427160739898682, | |
| "logps/rejected": -1.4044514894485474, | |
| "loss": 1.3062, | |
| "odds_ratio_loss": 0.6351091265678406, | |
| "rewards/accuracies": 0.6000000238418579, | |
| "rewards/chosen": -0.12427163124084473, | |
| "rewards/margins": 0.016173530369997025, | |
| "rewards/rejected": -0.14044514298439026, | |
| "sft_loss": 1.2427160739898682, | |
| "step": 1780 | |
| }, | |
| { | |
| "epoch": 2.8935138411800363, | |
| "grad_norm": 5.715972423553467, | |
| "learning_rate": 1.4688294413074677e-08, | |
| "logits/chosen": -20.392711639404297, | |
| "logits/rejected": -20.409814834594727, | |
| "logps/chosen": -1.12639582157135, | |
| "logps/rejected": -1.3582656383514404, | |
| "loss": 1.193, | |
| "odds_ratio_loss": 0.6661497354507446, | |
| "rewards/accuracies": 0.5249999761581421, | |
| "rewards/chosen": -0.11263958364725113, | |
| "rewards/margins": 0.023186981678009033, | |
| "rewards/rejected": -0.13582655787467957, | |
| "sft_loss": 1.12639582157135, | |
| "step": 1790 | |
| }, | |
| { | |
| "epoch": 2.909678722974338, | |
| "grad_norm": 1.500075340270996, | |
| "learning_rate": 1.0459757010556626e-08, | |
| "logits/chosen": -20.747310638427734, | |
| "logits/rejected": -20.764245986938477, | |
| "logps/chosen": -1.218435287475586, | |
| "logps/rejected": -1.2865406274795532, | |
| "loss": 1.2909, | |
| "odds_ratio_loss": 0.724606990814209, | |
| "rewards/accuracies": 0.4749999940395355, | |
| "rewards/chosen": -0.12184351682662964, | |
| "rewards/margins": 0.0068105412647128105, | |
| "rewards/rejected": -0.12865407764911652, | |
| "sft_loss": 1.218435287475586, | |
| "step": 1800 | |
| }, | |
| { | |
| "epoch": 2.92584360476864, | |
| "grad_norm": 4.831416606903076, | |
| "learning_rate": 6.94610413078306e-09, | |
| "logits/chosen": -20.301036834716797, | |
| "logits/rejected": -20.699962615966797, | |
| "logps/chosen": -1.2750685214996338, | |
| "logps/rejected": -1.4494614601135254, | |
| "loss": 1.3451, | |
| "odds_ratio_loss": 0.7007848024368286, | |
| "rewards/accuracies": 0.512499988079071, | |
| "rewards/chosen": -0.12750686705112457, | |
| "rewards/margins": 0.017439283430576324, | |
| "rewards/rejected": -0.1449461281299591, | |
| "sft_loss": 1.2750685214996338, | |
| "step": 1810 | |
| }, | |
| { | |
| "epoch": 2.942008486562942, | |
| "grad_norm": 3.175490617752075, | |
| "learning_rate": 4.14834473758563e-09, | |
| "logits/chosen": -20.20795249938965, | |
| "logits/rejected": -20.349489212036133, | |
| "logps/chosen": -1.168878197669983, | |
| "logps/rejected": -1.3752342462539673, | |
| "loss": 1.2352, | |
| "odds_ratio_loss": 0.6636675596237183, | |
| "rewards/accuracies": 0.59375, | |
| "rewards/chosen": -0.11688783019781113, | |
| "rewards/margins": 0.020635612308979034, | |
| "rewards/rejected": -0.13752342760562897, | |
| "sft_loss": 1.168878197669983, | |
| "step": 1820 | |
| }, | |
| { | |
| "epoch": 2.9581733683572438, | |
| "grad_norm": 1.267177700996399, | |
| "learning_rate": 2.067282222230349e-09, | |
| "logits/chosen": -20.883831024169922, | |
| "logits/rejected": -21.049150466918945, | |
| "logps/chosen": -1.19767427444458, | |
| "logps/rejected": -1.4529333114624023, | |
| "loss": 1.2618, | |
| "odds_ratio_loss": 0.6415713429450989, | |
| "rewards/accuracies": 0.606249988079071, | |
| "rewards/chosen": -0.1197674423456192, | |
| "rewards/margins": 0.02552589774131775, | |
| "rewards/rejected": -0.14529332518577576, | |
| "sft_loss": 1.19767427444458, | |
| "step": 1830 | |
| }, | |
| { | |
| "epoch": 2.9743382501515456, | |
| "grad_norm": 6.299037456512451, | |
| "learning_rate": 7.035141727212979e-10, | |
| "logits/chosen": -20.69939422607422, | |
| "logits/rejected": -20.93973159790039, | |
| "logps/chosen": -1.1146414279937744, | |
| "logps/rejected": -1.2947094440460205, | |
| "loss": 1.1819, | |
| "odds_ratio_loss": 0.6721066236495972, | |
| "rewards/accuracies": 0.550000011920929, | |
| "rewards/chosen": -0.11146414279937744, | |
| "rewards/margins": 0.018006790429353714, | |
| "rewards/rejected": -0.12947094440460205, | |
| "sft_loss": 1.1146414279937744, | |
| "step": 1840 | |
| }, | |
| { | |
| "epoch": 2.9905031319458475, | |
| "grad_norm": 7.900111675262451, | |
| "learning_rate": 5.743220219761592e-11, | |
| "logits/chosen": -20.688940048217773, | |
| "logits/rejected": -20.652080535888672, | |
| "logps/chosen": -1.2903720140457153, | |
| "logps/rejected": -1.3897130489349365, | |
| "loss": 1.3642, | |
| "odds_ratio_loss": 0.7377926111221313, | |
| "rewards/accuracies": 0.543749988079071, | |
| "rewards/chosen": -0.12903721630573273, | |
| "rewards/margins": 0.00993410311639309, | |
| "rewards/rejected": -0.13897129893302917, | |
| "sft_loss": 1.2903720140457153, | |
| "step": 1850 | |
| }, | |
| { | |
| "epoch": 2.9969690846635686, | |
| "step": 1854, | |
| "total_flos": 5.618252880760013e+17, | |
| "train_loss": 1.3892074464594277, | |
| "train_runtime": 8722.5011, | |
| "train_samples_per_second": 3.404, | |
| "train_steps_per_second": 0.213 | |
| } | |
| ], | |
| "logging_steps": 10, | |
| "max_steps": 1854, | |
| "num_input_tokens_seen": 0, | |
| "num_train_epochs": 3, | |
| "save_steps": 500, | |
| "total_flos": 5.618252880760013e+17, | |
| "train_batch_size": 2, | |
| "trial_name": null, | |
| "trial_params": null | |
| } | |