| { | |
| "best_metric": 0.15150243043899536, | |
| "best_model_checkpoint": "saves/Llama-3.1-8B-Instruct/lora/saa-900/checkpoint-200", | |
| "epoch": 9.876543209876543, | |
| "eval_steps": 50, | |
| "global_step": 500, | |
| "is_hyper_param_search": false, | |
| "is_local_process_zero": true, | |
| "is_world_process_zero": true, | |
| "log_history": [ | |
| { | |
| "epoch": 0.19753086419753085, | |
| "grad_norm": 5.683405876159668, | |
| "learning_rate": 1.0000000000000002e-06, | |
| "logits/chosen": -0.4294208884239197, | |
| "logits/rejected": -0.5036857724189758, | |
| "logps/chosen": -1.7164901494979858, | |
| "logps/rejected": -2.1387689113616943, | |
| "loss": 1.77, | |
| "odds_ratio_loss": 15.52961540222168, | |
| "rewards/accuracies": 0.793749988079071, | |
| "rewards/chosen": -0.1716490387916565, | |
| "rewards/margins": 0.0422278493642807, | |
| "rewards/rejected": -0.2138768881559372, | |
| "sft_loss": 0.21705082058906555, | |
| "step": 10 | |
| }, | |
| { | |
| "epoch": 0.3950617283950617, | |
| "grad_norm": 7.8601579666137695, | |
| "learning_rate": 2.0000000000000003e-06, | |
| "logits/chosen": -0.41924962401390076, | |
| "logits/rejected": -0.4839371144771576, | |
| "logps/chosen": -1.7584301233291626, | |
| "logps/rejected": -2.1139349937438965, | |
| "loss": 1.8145, | |
| "odds_ratio_loss": 15.842885971069336, | |
| "rewards/accuracies": 0.7562500238418579, | |
| "rewards/chosen": -0.1758430302143097, | |
| "rewards/margins": 0.03555048629641533, | |
| "rewards/rejected": -0.21139350533485413, | |
| "sft_loss": 0.23018595576286316, | |
| "step": 20 | |
| }, | |
| { | |
| "epoch": 0.5925925925925926, | |
| "grad_norm": 8.10688591003418, | |
| "learning_rate": 3e-06, | |
| "logits/chosen": -0.4261442720890045, | |
| "logits/rejected": -0.5029150247573853, | |
| "logps/chosen": -1.730046033859253, | |
| "logps/rejected": -2.217639684677124, | |
| "loss": 1.781, | |
| "odds_ratio_loss": 15.661776542663574, | |
| "rewards/accuracies": 0.800000011920929, | |
| "rewards/chosen": -0.173004612326622, | |
| "rewards/margins": 0.048759374767541885, | |
| "rewards/rejected": -0.2217639982700348, | |
| "sft_loss": 0.21483230590820312, | |
| "step": 30 | |
| }, | |
| { | |
| "epoch": 0.7901234567901234, | |
| "grad_norm": 6.699493885040283, | |
| "learning_rate": 4.000000000000001e-06, | |
| "logits/chosen": -0.41258639097213745, | |
| "logits/rejected": -0.4953138828277588, | |
| "logps/chosen": -1.566969633102417, | |
| "logps/rejected": -2.103066921234131, | |
| "loss": 1.6158, | |
| "odds_ratio_loss": 14.229395866394043, | |
| "rewards/accuracies": 0.7875000238418579, | |
| "rewards/chosen": -0.15669694542884827, | |
| "rewards/margins": 0.053609758615493774, | |
| "rewards/rejected": -0.21030668914318085, | |
| "sft_loss": 0.19286662340164185, | |
| "step": 40 | |
| }, | |
| { | |
| "epoch": 0.9876543209876543, | |
| "grad_norm": 9.212828636169434, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -0.4587825834751129, | |
| "logits/rejected": -0.5460661053657532, | |
| "logps/chosen": -1.524173378944397, | |
| "logps/rejected": -1.9827760457992554, | |
| "loss": 1.5773, | |
| "odds_ratio_loss": 13.699198722839355, | |
| "rewards/accuracies": 0.762499988079071, | |
| "rewards/chosen": -0.15241733193397522, | |
| "rewards/margins": 0.04586028307676315, | |
| "rewards/rejected": -0.19827762246131897, | |
| "sft_loss": 0.20735812187194824, | |
| "step": 50 | |
| }, | |
| { | |
| "epoch": 0.9876543209876543, | |
| "eval_logits/chosen": -0.4034404754638672, | |
| "eval_logits/rejected": -0.46630287170410156, | |
| "eval_logps/chosen": -1.3146759271621704, | |
| "eval_logps/rejected": -1.754432201385498, | |
| "eval_loss": 1.3696272373199463, | |
| "eval_odds_ratio_loss": 11.865678787231445, | |
| "eval_rewards/accuracies": 0.7666666507720947, | |
| "eval_rewards/chosen": -0.13146759569644928, | |
| "eval_rewards/margins": 0.04397563263773918, | |
| "eval_rewards/rejected": -0.17544323205947876, | |
| "eval_runtime": 3.7458, | |
| "eval_samples_per_second": 24.027, | |
| "eval_sft_loss": 0.18305927515029907, | |
| "eval_steps_per_second": 12.013, | |
| "step": 50 | |
| }, | |
| { | |
| "epoch": 1.1851851851851851, | |
| "grad_norm": 4.2851243019104, | |
| "learning_rate": 4.993910125649561e-06, | |
| "logits/chosen": -0.42754077911376953, | |
| "logits/rejected": -0.49272727966308594, | |
| "logps/chosen": -1.14907968044281, | |
| "logps/rejected": -1.5600894689559937, | |
| "loss": 1.2014, | |
| "odds_ratio_loss": 10.684017181396484, | |
| "rewards/accuracies": 0.762499988079071, | |
| "rewards/chosen": -0.11490797996520996, | |
| "rewards/margins": 0.041100967675447464, | |
| "rewards/rejected": -0.15600894391536713, | |
| "sft_loss": 0.13294857740402222, | |
| "step": 60 | |
| }, | |
| { | |
| "epoch": 1.382716049382716, | |
| "grad_norm": 2.6753902435302734, | |
| "learning_rate": 4.975670171853926e-06, | |
| "logits/chosen": -0.42295628786087036, | |
| "logits/rejected": -0.5050143003463745, | |
| "logps/chosen": -0.8449110984802246, | |
| "logps/rejected": -1.3062922954559326, | |
| "loss": 0.8939, | |
| "odds_ratio_loss": 7.972494602203369, | |
| "rewards/accuracies": 0.7875000238418579, | |
| "rewards/chosen": -0.08449110388755798, | |
| "rewards/margins": 0.04613811895251274, | |
| "rewards/rejected": -0.13062922656536102, | |
| "sft_loss": 0.09666236490011215, | |
| "step": 70 | |
| }, | |
| { | |
| "epoch": 1.5802469135802468, | |
| "grad_norm": 2.68009090423584, | |
| "learning_rate": 4.9453690018345144e-06, | |
| "logits/chosen": -0.41366061568260193, | |
| "logits/rejected": -0.5027936100959778, | |
| "logps/chosen": -0.5736119151115417, | |
| "logps/rejected": -1.0685169696807861, | |
| "loss": 0.6186, | |
| "odds_ratio_loss": 5.562970161437988, | |
| "rewards/accuracies": 0.8062499761581421, | |
| "rewards/chosen": -0.057361193001270294, | |
| "rewards/margins": 0.04949050396680832, | |
| "rewards/rejected": -0.10685168206691742, | |
| "sft_loss": 0.06228654831647873, | |
| "step": 80 | |
| }, | |
| { | |
| "epoch": 1.7777777777777777, | |
| "grad_norm": 2.270496129989624, | |
| "learning_rate": 4.903154239845798e-06, | |
| "logits/chosen": -0.4178565442562103, | |
| "logits/rejected": -0.48920226097106934, | |
| "logps/chosen": -0.3355949819087982, | |
| "logps/rejected": -0.902151882648468, | |
| "loss": 0.3717, | |
| "odds_ratio_loss": 3.370701551437378, | |
| "rewards/accuracies": 0.8500000238418579, | |
| "rewards/chosen": -0.03355949744582176, | |
| "rewards/margins": 0.05665569379925728, | |
| "rewards/rejected": -0.09021519124507904, | |
| "sft_loss": 0.03466098755598068, | |
| "step": 90 | |
| }, | |
| { | |
| "epoch": 1.9753086419753085, | |
| "grad_norm": 2.1452736854553223, | |
| "learning_rate": 4.849231551964771e-06, | |
| "logits/chosen": -0.39813241362571716, | |
| "logits/rejected": -0.4744625985622406, | |
| "logps/chosen": -0.22052116692066193, | |
| "logps/rejected": -0.8172726631164551, | |
| "loss": 0.2518, | |
| "odds_ratio_loss": 2.2916011810302734, | |
| "rewards/accuracies": 0.856249988079071, | |
| "rewards/chosen": -0.02205212041735649, | |
| "rewards/margins": 0.05967513844370842, | |
| "rewards/rejected": -0.08172726631164551, | |
| "sft_loss": 0.022620398551225662, | |
| "step": 100 | |
| }, | |
| { | |
| "epoch": 1.9753086419753085, | |
| "eval_logits/chosen": -0.3780798316001892, | |
| "eval_logits/rejected": -0.448334664106369, | |
| "eval_logps/chosen": -0.18976576626300812, | |
| "eval_logps/rejected": -0.7320783734321594, | |
| "eval_loss": 0.23485496640205383, | |
| "eval_odds_ratio_loss": 2.1322858333587646, | |
| "eval_rewards/accuracies": 0.8111110925674438, | |
| "eval_rewards/chosen": -0.018976576626300812, | |
| "eval_rewards/margins": 0.05423126742243767, | |
| "eval_rewards/rejected": -0.07320784032344818, | |
| "eval_runtime": 3.7439, | |
| "eval_samples_per_second": 24.039, | |
| "eval_sft_loss": 0.02162640169262886, | |
| "eval_steps_per_second": 12.02, | |
| "step": 100 | |
| }, | |
| { | |
| "epoch": 2.1728395061728394, | |
| "grad_norm": 2.2345616817474365, | |
| "learning_rate": 4.783863644106502e-06, | |
| "logits/chosen": -0.43218794465065, | |
| "logits/rejected": -0.4891538619995117, | |
| "logps/chosen": -0.18616971373558044, | |
| "logps/rejected": -0.6744140386581421, | |
| "loss": 0.2306, | |
| "odds_ratio_loss": 2.0546927452087402, | |
| "rewards/accuracies": 0.793749988079071, | |
| "rewards/chosen": -0.018616970628499985, | |
| "rewards/margins": 0.048824433237314224, | |
| "rewards/rejected": -0.06744140386581421, | |
| "sft_loss": 0.025166219100356102, | |
| "step": 110 | |
| }, | |
| { | |
| "epoch": 2.3703703703703702, | |
| "grad_norm": 1.977693796157837, | |
| "learning_rate": 4.707368982147318e-06, | |
| "logits/chosen": -0.4379637837409973, | |
| "logits/rejected": -0.5003925561904907, | |
| "logps/chosen": -0.11895088851451874, | |
| "logps/rejected": -0.6111856698989868, | |
| "loss": 0.1586, | |
| "odds_ratio_loss": 1.449549674987793, | |
| "rewards/accuracies": 0.831250011920929, | |
| "rewards/chosen": -0.011895090341567993, | |
| "rewards/margins": 0.04922347515821457, | |
| "rewards/rejected": -0.06111856549978256, | |
| "sft_loss": 0.01368994265794754, | |
| "step": 120 | |
| }, | |
| { | |
| "epoch": 2.567901234567901, | |
| "grad_norm": 0.721815288066864, | |
| "learning_rate": 4.620120240391065e-06, | |
| "logits/chosen": -0.3538777828216553, | |
| "logits/rejected": -0.4279879927635193, | |
| "logps/chosen": -0.09486258774995804, | |
| "logps/rejected": -0.7161315679550171, | |
| "loss": 0.1321, | |
| "odds_ratio_loss": 1.2123935222625732, | |
| "rewards/accuracies": 0.84375, | |
| "rewards/chosen": -0.009486258029937744, | |
| "rewards/margins": 0.062126897275447845, | |
| "rewards/rejected": -0.07161315530538559, | |
| "sft_loss": 0.010894850827753544, | |
| "step": 130 | |
| }, | |
| { | |
| "epoch": 2.765432098765432, | |
| "grad_norm": 1.1547597646713257, | |
| "learning_rate": 4.522542485937369e-06, | |
| "logits/chosen": -0.38979417085647583, | |
| "logits/rejected": -0.45057815313339233, | |
| "logps/chosen": -0.0999944657087326, | |
| "logps/rejected": -0.7246894836425781, | |
| "loss": 0.1326, | |
| "odds_ratio_loss": 1.1918091773986816, | |
| "rewards/accuracies": 0.84375, | |
| "rewards/chosen": -0.009999445639550686, | |
| "rewards/margins": 0.06246950477361679, | |
| "rewards/rejected": -0.07246895134449005, | |
| "sft_loss": 0.013394972309470177, | |
| "step": 140 | |
| }, | |
| { | |
| "epoch": 2.962962962962963, | |
| "grad_norm": 1.2962913513183594, | |
| "learning_rate": 4.415111107797445e-06, | |
| "logits/chosen": -0.3816613554954529, | |
| "logits/rejected": -0.43519797921180725, | |
| "logps/chosen": -0.09587013721466064, | |
| "logps/rejected": -0.5622029304504395, | |
| "loss": 0.1304, | |
| "odds_ratio_loss": 1.182822823524475, | |
| "rewards/accuracies": 0.824999988079071, | |
| "rewards/chosen": -0.009587015025317669, | |
| "rewards/margins": 0.0466332733631134, | |
| "rewards/rejected": -0.05622028559446335, | |
| "sft_loss": 0.012101124040782452, | |
| "step": 150 | |
| }, | |
| { | |
| "epoch": 2.962962962962963, | |
| "eval_logits/chosen": -0.345350980758667, | |
| "eval_logits/rejected": -0.4031663239002228, | |
| "eval_logps/chosen": -0.10941138863563538, | |
| "eval_logps/rejected": -0.6116734147071838, | |
| "eval_loss": 0.15299223363399506, | |
| "eval_odds_ratio_loss": 1.3987985849380493, | |
| "eval_rewards/accuracies": 0.8111110925674438, | |
| "eval_rewards/chosen": -0.010941138491034508, | |
| "eval_rewards/margins": 0.050226207822561264, | |
| "eval_rewards/rejected": -0.061167340725660324, | |
| "eval_runtime": 3.741, | |
| "eval_samples_per_second": 24.058, | |
| "eval_sft_loss": 0.013112363405525684, | |
| "eval_steps_per_second": 12.029, | |
| "step": 150 | |
| }, | |
| { | |
| "epoch": 3.1604938271604937, | |
| "grad_norm": 1.6140797138214111, | |
| "learning_rate": 4.2983495008466285e-06, | |
| "logits/chosen": -0.3475257158279419, | |
| "logits/rejected": -0.4172907769680023, | |
| "logps/chosen": -0.0872744545340538, | |
| "logps/rejected": -0.547534167766571, | |
| "loss": 0.1283, | |
| "odds_ratio_loss": 1.1653860807418823, | |
| "rewards/accuracies": 0.7875000238418579, | |
| "rewards/chosen": -0.008727445267140865, | |
| "rewards/margins": 0.046025972813367844, | |
| "rewards/rejected": -0.05475341156125069, | |
| "sft_loss": 0.011779388412833214, | |
| "step": 160 | |
| }, | |
| { | |
| "epoch": 3.3580246913580245, | |
| "grad_norm": 1.6337134838104248, | |
| "learning_rate": 4.172826515897146e-06, | |
| "logits/chosen": -0.3779260814189911, | |
| "logits/rejected": -0.44266828894615173, | |
| "logps/chosen": -0.08930256962776184, | |
| "logps/rejected": -0.6787833571434021, | |
| "loss": 0.1222, | |
| "odds_ratio_loss": 1.113948106765747, | |
| "rewards/accuracies": 0.856249988079071, | |
| "rewards/chosen": -0.008930256590247154, | |
| "rewards/margins": 0.058948077261447906, | |
| "rewards/rejected": -0.06787833571434021, | |
| "sft_loss": 0.01077614352107048, | |
| "step": 170 | |
| }, | |
| { | |
| "epoch": 3.5555555555555554, | |
| "grad_norm": 1.329077959060669, | |
| "learning_rate": 4.039153688314146e-06, | |
| "logits/chosen": -0.3644777834415436, | |
| "logits/rejected": -0.4096454977989197, | |
| "logps/chosen": -0.07825501263141632, | |
| "logps/rejected": -0.5899345874786377, | |
| "loss": 0.1135, | |
| "odds_ratio_loss": 1.0353591442108154, | |
| "rewards/accuracies": 0.8374999761581421, | |
| "rewards/chosen": -0.007825501263141632, | |
| "rewards/margins": 0.05116795748472214, | |
| "rewards/rejected": -0.05899345874786377, | |
| "sft_loss": 0.009976235218346119, | |
| "step": 180 | |
| }, | |
| { | |
| "epoch": 3.753086419753086, | |
| "grad_norm": 2.1350457668304443, | |
| "learning_rate": 3.897982258676867e-06, | |
| "logits/chosen": -0.342695415019989, | |
| "logits/rejected": -0.41270384192466736, | |
| "logps/chosen": -0.09099350869655609, | |
| "logps/rejected": -0.6971117854118347, | |
| "loss": 0.1184, | |
| "odds_ratio_loss": 1.0633059740066528, | |
| "rewards/accuracies": 0.856249988079071, | |
| "rewards/chosen": -0.009099351242184639, | |
| "rewards/margins": 0.06061183288693428, | |
| "rewards/rejected": -0.06971118599176407, | |
| "sft_loss": 0.012052880600094795, | |
| "step": 190 | |
| }, | |
| { | |
| "epoch": 3.950617283950617, | |
| "grad_norm": 1.1367535591125488, | |
| "learning_rate": 3.7500000000000005e-06, | |
| "logits/chosen": -0.3970497250556946, | |
| "logits/rejected": -0.4613776206970215, | |
| "logps/chosen": -0.07991655170917511, | |
| "logps/rejected": -0.6025404334068298, | |
| "loss": 0.1129, | |
| "odds_ratio_loss": 1.0367439985275269, | |
| "rewards/accuracies": 0.824999988079071, | |
| "rewards/chosen": -0.007991654798388481, | |
| "rewards/margins": 0.05226239562034607, | |
| "rewards/rejected": -0.0602540448307991, | |
| "sft_loss": 0.009189181961119175, | |
| "step": 200 | |
| }, | |
| { | |
| "epoch": 3.950617283950617, | |
| "eval_logits/chosen": -0.34801948070526123, | |
| "eval_logits/rejected": -0.4030517041683197, | |
| "eval_logps/chosen": -0.10840340703725815, | |
| "eval_logps/rejected": -0.5819076299667358, | |
| "eval_loss": 0.15150243043899536, | |
| "eval_odds_ratio_loss": 1.3828411102294922, | |
| "eval_rewards/accuracies": 0.8222222328186035, | |
| "eval_rewards/chosen": -0.010840341448783875, | |
| "eval_rewards/margins": 0.04735042154788971, | |
| "eval_rewards/rejected": -0.058190762996673584, | |
| "eval_runtime": 3.7401, | |
| "eval_samples_per_second": 24.064, | |
| "eval_sft_loss": 0.013218320906162262, | |
| "eval_steps_per_second": 12.032, | |
| "step": 200 | |
| }, | |
| { | |
| "epoch": 4.148148148148148, | |
| "grad_norm": 1.6960663795471191, | |
| "learning_rate": 3.595927866972694e-06, | |
| "logits/chosen": -0.3732069432735443, | |
| "logits/rejected": -0.4440130293369293, | |
| "logps/chosen": -0.06717614084482193, | |
| "logps/rejected": -0.516498327255249, | |
| "loss": 0.0942, | |
| "odds_ratio_loss": 0.8479182124137878, | |
| "rewards/accuracies": 0.8374999761581421, | |
| "rewards/chosen": -0.006717613898217678, | |
| "rewards/margins": 0.04493222385644913, | |
| "rewards/rejected": -0.05164983123540878, | |
| "sft_loss": 0.009402485564351082, | |
| "step": 210 | |
| }, | |
| { | |
| "epoch": 4.345679012345679, | |
| "grad_norm": 0.9651530385017395, | |
| "learning_rate": 3.436516483539781e-06, | |
| "logits/chosen": -0.3677094578742981, | |
| "logits/rejected": -0.4239886403083801, | |
| "logps/chosen": -0.09318893402814865, | |
| "logps/rejected": -0.6319376230239868, | |
| "loss": 0.13, | |
| "odds_ratio_loss": 1.189330816268921, | |
| "rewards/accuracies": 0.831250011920929, | |
| "rewards/chosen": -0.009318893775343895, | |
| "rewards/margins": 0.053874872624874115, | |
| "rewards/rejected": -0.06319376826286316, | |
| "sft_loss": 0.011091579683125019, | |
| "step": 220 | |
| }, | |
| { | |
| "epoch": 4.54320987654321, | |
| "grad_norm": 1.0926239490509033, | |
| "learning_rate": 3.272542485937369e-06, | |
| "logits/chosen": -0.41018205881118774, | |
| "logits/rejected": -0.48241281509399414, | |
| "logps/chosen": -0.06124984472990036, | |
| "logps/rejected": -0.6271657347679138, | |
| "loss": 0.0887, | |
| "odds_ratio_loss": 0.8028218150138855, | |
| "rewards/accuracies": 0.8500000238418579, | |
| "rewards/chosen": -0.006124985404312611, | |
| "rewards/margins": 0.056591589003801346, | |
| "rewards/rejected": -0.06271658092737198, | |
| "sft_loss": 0.008464094251394272, | |
| "step": 230 | |
| }, | |
| { | |
| "epoch": 4.7407407407407405, | |
| "grad_norm": 1.5143941640853882, | |
| "learning_rate": 3.1048047389991693e-06, | |
| "logits/chosen": -0.36270713806152344, | |
| "logits/rejected": -0.4348696172237396, | |
| "logps/chosen": -0.05753596872091293, | |
| "logps/rejected": -0.6398877501487732, | |
| "loss": 0.0825, | |
| "odds_ratio_loss": 0.7441199421882629, | |
| "rewards/accuracies": 0.887499988079071, | |
| "rewards/chosen": -0.005753597244620323, | |
| "rewards/margins": 0.058235179632902145, | |
| "rewards/rejected": -0.06398877501487732, | |
| "sft_loss": 0.008074344135820866, | |
| "step": 240 | |
| }, | |
| { | |
| "epoch": 4.938271604938271, | |
| "grad_norm": 1.6572201251983643, | |
| "learning_rate": 2.9341204441673267e-06, | |
| "logits/chosen": -0.3572860658168793, | |
| "logits/rejected": -0.41670694947242737, | |
| "logps/chosen": -0.09173234552145004, | |
| "logps/rejected": -0.7026180028915405, | |
| "loss": 0.1194, | |
| "odds_ratio_loss": 1.0817253589630127, | |
| "rewards/accuracies": 0.84375, | |
| "rewards/chosen": -0.009173234924674034, | |
| "rewards/margins": 0.061088573187589645, | |
| "rewards/rejected": -0.07026181370019913, | |
| "sft_loss": 0.011259505525231361, | |
| "step": 250 | |
| }, | |
| { | |
| "epoch": 4.938271604938271, | |
| "eval_logits/chosen": -0.3416781425476074, | |
| "eval_logits/rejected": -0.3981509208679199, | |
| "eval_logps/chosen": -0.10881069302558899, | |
| "eval_logps/rejected": -0.641668975353241, | |
| "eval_loss": 0.15224343538284302, | |
| "eval_odds_ratio_loss": 1.3890637159347534, | |
| "eval_rewards/accuracies": 0.8222222328186035, | |
| "eval_rewards/chosen": -0.01088106818497181, | |
| "eval_rewards/margins": 0.053285837173461914, | |
| "eval_rewards/rejected": -0.06416690349578857, | |
| "eval_runtime": 3.7429, | |
| "eval_samples_per_second": 24.045, | |
| "eval_sft_loss": 0.01333706360310316, | |
| "eval_steps_per_second": 12.023, | |
| "step": 250 | |
| }, | |
| { | |
| "epoch": 5.135802469135802, | |
| "grad_norm": 0.6316437721252441, | |
| "learning_rate": 2.761321158169134e-06, | |
| "logits/chosen": -0.38472387194633484, | |
| "logits/rejected": -0.43845945596694946, | |
| "logps/chosen": -0.08237903565168381, | |
| "logps/rejected": -0.6897364854812622, | |
| "loss": 0.1098, | |
| "odds_ratio_loss": 1.019060730934143, | |
| "rewards/accuracies": 0.8812500238418579, | |
| "rewards/chosen": -0.00823790393769741, | |
| "rewards/margins": 0.06073574349284172, | |
| "rewards/rejected": -0.06897365301847458, | |
| "sft_loss": 0.007918295450508595, | |
| "step": 260 | |
| }, | |
| { | |
| "epoch": 5.333333333333333, | |
| "grad_norm": 2.247709035873413, | |
| "learning_rate": 2.587248741756253e-06, | |
| "logits/chosen": -0.33236485719680786, | |
| "logits/rejected": -0.41399604082107544, | |
| "logps/chosen": -0.06683196127414703, | |
| "logps/rejected": -0.745326578617096, | |
| "loss": 0.0981, | |
| "odds_ratio_loss": 0.897613525390625, | |
| "rewards/accuracies": 0.856249988079071, | |
| "rewards/chosen": -0.0066831959411501884, | |
| "rewards/margins": 0.06784946471452713, | |
| "rewards/rejected": -0.07453266531229019, | |
| "sft_loss": 0.008373981341719627, | |
| "step": 270 | |
| }, | |
| { | |
| "epoch": 5.530864197530864, | |
| "grad_norm": 3.0054218769073486, | |
| "learning_rate": 2.4127512582437486e-06, | |
| "logits/chosen": -0.38011887669563293, | |
| "logits/rejected": -0.4398444592952728, | |
| "logps/chosen": -0.07033652812242508, | |
| "logps/rejected": -0.6755497455596924, | |
| "loss": 0.0967, | |
| "odds_ratio_loss": 0.8789058923721313, | |
| "rewards/accuracies": 0.8812500238418579, | |
| "rewards/chosen": -0.007033653557300568, | |
| "rewards/margins": 0.06052132323384285, | |
| "rewards/rejected": -0.06755497306585312, | |
| "sft_loss": 0.008822395466268063, | |
| "step": 280 | |
| }, | |
| { | |
| "epoch": 5.728395061728395, | |
| "grad_norm": 1.2473162412643433, | |
| "learning_rate": 2.238678841830867e-06, | |
| "logits/chosen": -0.35760438442230225, | |
| "logits/rejected": -0.432979017496109, | |
| "logps/chosen": -0.05806203559041023, | |
| "logps/rejected": -0.6580389142036438, | |
| "loss": 0.0822, | |
| "odds_ratio_loss": 0.7390257716178894, | |
| "rewards/accuracies": 0.8687499761581421, | |
| "rewards/chosen": -0.005806203465908766, | |
| "rewards/margins": 0.05999768525362015, | |
| "rewards/rejected": -0.0658038854598999, | |
| "sft_loss": 0.008302577771246433, | |
| "step": 290 | |
| }, | |
| { | |
| "epoch": 5.925925925925926, | |
| "grad_norm": 1.7050279378890991, | |
| "learning_rate": 2.0658795558326745e-06, | |
| "logits/chosen": -0.39829307794570923, | |
| "logits/rejected": -0.4691620469093323, | |
| "logps/chosen": -0.06409647315740585, | |
| "logps/rejected": -0.6852135062217712, | |
| "loss": 0.0898, | |
| "odds_ratio_loss": 0.8166918754577637, | |
| "rewards/accuracies": 0.862500011920929, | |
| "rewards/chosen": -0.00640964787453413, | |
| "rewards/margins": 0.062111712992191315, | |
| "rewards/rejected": -0.06852135807275772, | |
| "sft_loss": 0.00808151438832283, | |
| "step": 300 | |
| }, | |
| { | |
| "epoch": 5.925925925925926, | |
| "eval_logits/chosen": -0.3401709198951721, | |
| "eval_logits/rejected": -0.39598190784454346, | |
| "eval_logps/chosen": -0.11005988717079163, | |
| "eval_logps/rejected": -0.6839298605918884, | |
| "eval_loss": 0.153494730591774, | |
| "eval_odds_ratio_loss": 1.3989084959030151, | |
| "eval_rewards/accuracies": 0.8111110925674438, | |
| "eval_rewards/chosen": -0.011005990207195282, | |
| "eval_rewards/margins": 0.05738700181245804, | |
| "eval_rewards/rejected": -0.06839299947023392, | |
| "eval_runtime": 3.7412, | |
| "eval_samples_per_second": 24.056, | |
| "eval_sft_loss": 0.0136038763448596, | |
| "eval_steps_per_second": 12.028, | |
| "step": 300 | |
| }, | |
| { | |
| "epoch": 6.1234567901234565, | |
| "grad_norm": 0.6541029214859009, | |
| "learning_rate": 1.895195261000831e-06, | |
| "logits/chosen": -0.3678016662597656, | |
| "logits/rejected": -0.4242876470088959, | |
| "logps/chosen": -0.07081776112318039, | |
| "logps/rejected": -0.746510922908783, | |
| "loss": 0.0894, | |
| "odds_ratio_loss": 0.7998540997505188, | |
| "rewards/accuracies": 0.9125000238418579, | |
| "rewards/chosen": -0.007081775926053524, | |
| "rewards/margins": 0.0675693154335022, | |
| "rewards/rejected": -0.0746510922908783, | |
| "sft_loss": 0.009410968981683254, | |
| "step": 310 | |
| }, | |
| { | |
| "epoch": 6.320987654320987, | |
| "grad_norm": 1.3137905597686768, | |
| "learning_rate": 1.7274575140626318e-06, | |
| "logits/chosen": -0.38049232959747314, | |
| "logits/rejected": -0.4344969391822815, | |
| "logps/chosen": -0.06233459711074829, | |
| "logps/rejected": -0.6675801277160645, | |
| "loss": 0.0862, | |
| "odds_ratio_loss": 0.7855814695358276, | |
| "rewards/accuracies": 0.887499988079071, | |
| "rewards/chosen": -0.006233459338545799, | |
| "rewards/margins": 0.060524553060531616, | |
| "rewards/rejected": -0.06675801426172256, | |
| "sft_loss": 0.007671413477510214, | |
| "step": 320 | |
| }, | |
| { | |
| "epoch": 6.518518518518518, | |
| "grad_norm": 1.7016428709030151, | |
| "learning_rate": 1.56348351646022e-06, | |
| "logits/chosen": -0.35415560007095337, | |
| "logits/rejected": -0.4285738468170166, | |
| "logps/chosen": -0.06706033647060394, | |
| "logps/rejected": -0.7278292775154114, | |
| "loss": 0.0952, | |
| "odds_ratio_loss": 0.8702393770217896, | |
| "rewards/accuracies": 0.8687499761581421, | |
| "rewards/chosen": -0.0067060342989861965, | |
| "rewards/margins": 0.06607688963413239, | |
| "rewards/rejected": -0.07278293371200562, | |
| "sft_loss": 0.008213361725211143, | |
| "step": 330 | |
| }, | |
| { | |
| "epoch": 6.716049382716049, | |
| "grad_norm": 1.7326641082763672, | |
| "learning_rate": 1.4040721330273063e-06, | |
| "logits/chosen": -0.355478435754776, | |
| "logits/rejected": -0.42061376571655273, | |
| "logps/chosen": -0.057823676615953445, | |
| "logps/rejected": -0.6569600105285645, | |
| "loss": 0.0825, | |
| "odds_ratio_loss": 0.7589547634124756, | |
| "rewards/accuracies": 0.8812500238418579, | |
| "rewards/chosen": -0.005782367195934057, | |
| "rewards/margins": 0.05991363525390625, | |
| "rewards/rejected": -0.06569599360227585, | |
| "sft_loss": 0.006640643812716007, | |
| "step": 340 | |
| }, | |
| { | |
| "epoch": 6.91358024691358, | |
| "grad_norm": 2.150254011154175, | |
| "learning_rate": 1.2500000000000007e-06, | |
| "logits/chosen": -0.3611661493778229, | |
| "logits/rejected": -0.43402963876724243, | |
| "logps/chosen": -0.06616374105215073, | |
| "logps/rejected": -0.7352410554885864, | |
| "loss": 0.0928, | |
| "odds_ratio_loss": 0.8426556587219238, | |
| "rewards/accuracies": 0.856249988079071, | |
| "rewards/chosen": -0.00661637494340539, | |
| "rewards/margins": 0.06690772622823715, | |
| "rewards/rejected": -0.0735241025686264, | |
| "sft_loss": 0.00857949536293745, | |
| "step": 350 | |
| }, | |
| { | |
| "epoch": 6.91358024691358, | |
| "eval_logits/chosen": -0.33941933512687683, | |
| "eval_logits/rejected": -0.39487287402153015, | |
| "eval_logps/chosen": -0.11251324415206909, | |
| "eval_logps/rejected": -0.679356038570404, | |
| "eval_loss": 0.1572149693965912, | |
| "eval_odds_ratio_loss": 1.431775450706482, | |
| "eval_rewards/accuracies": 0.7888888716697693, | |
| "eval_rewards/chosen": -0.011251323856413364, | |
| "eval_rewards/margins": 0.05668427795171738, | |
| "eval_rewards/rejected": -0.06793560087680817, | |
| "eval_runtime": 3.7421, | |
| "eval_samples_per_second": 24.05, | |
| "eval_sft_loss": 0.014037418179214, | |
| "eval_steps_per_second": 12.025, | |
| "step": 350 | |
| }, | |
| { | |
| "epoch": 7.111111111111111, | |
| "grad_norm": 2.0509181022644043, | |
| "learning_rate": 1.1020177413231334e-06, | |
| "logits/chosen": -0.3650658130645752, | |
| "logits/rejected": -0.4295966625213623, | |
| "logps/chosen": -0.057928264141082764, | |
| "logps/rejected": -0.6719712615013123, | |
| "loss": 0.084, | |
| "odds_ratio_loss": 0.771135687828064, | |
| "rewards/accuracies": 0.875, | |
| "rewards/chosen": -0.005792826414108276, | |
| "rewards/margins": 0.061404310166835785, | |
| "rewards/rejected": -0.06719712913036346, | |
| "sft_loss": 0.006877593696117401, | |
| "step": 360 | |
| }, | |
| { | |
| "epoch": 7.308641975308642, | |
| "grad_norm": 3.04136323928833, | |
| "learning_rate": 9.608463116858544e-07, | |
| "logits/chosen": -0.4004891812801361, | |
| "logits/rejected": -0.45741385221481323, | |
| "logps/chosen": -0.06307969242334366, | |
| "logps/rejected": -0.67022705078125, | |
| "loss": 0.0872, | |
| "odds_ratio_loss": 0.7940786480903625, | |
| "rewards/accuracies": 0.8999999761581421, | |
| "rewards/chosen": -0.006307969335466623, | |
| "rewards/margins": 0.060714732855558395, | |
| "rewards/rejected": -0.06702270358800888, | |
| "sft_loss": 0.00781711470335722, | |
| "step": 370 | |
| }, | |
| { | |
| "epoch": 7.506172839506172, | |
| "grad_norm": 1.3740649223327637, | |
| "learning_rate": 8.271734841028553e-07, | |
| "logits/chosen": -0.3516360819339752, | |
| "logits/rejected": -0.4145506024360657, | |
| "logps/chosen": -0.06131374090909958, | |
| "logps/rejected": -0.8121838569641113, | |
| "loss": 0.084, | |
| "odds_ratio_loss": 0.779072642326355, | |
| "rewards/accuracies": 0.893750011920929, | |
| "rewards/chosen": -0.006131374277174473, | |
| "rewards/margins": 0.07508701831102371, | |
| "rewards/rejected": -0.08121839910745621, | |
| "sft_loss": 0.006131037604063749, | |
| "step": 380 | |
| }, | |
| { | |
| "epoch": 7.703703703703704, | |
| "grad_norm": 1.9797685146331787, | |
| "learning_rate": 7.016504991533727e-07, | |
| "logits/chosen": -0.37762802839279175, | |
| "logits/rejected": -0.4341215491294861, | |
| "logps/chosen": -0.06214147061109543, | |
| "logps/rejected": -0.7485532164573669, | |
| "loss": 0.0882, | |
| "odds_ratio_loss": 0.8045924305915833, | |
| "rewards/accuracies": 0.8687499761581421, | |
| "rewards/chosen": -0.006214146967977285, | |
| "rewards/margins": 0.06864118576049805, | |
| "rewards/rejected": -0.07485532760620117, | |
| "sft_loss": 0.007714971899986267, | |
| "step": 390 | |
| }, | |
| { | |
| "epoch": 7.901234567901234, | |
| "grad_norm": 1.4376161098480225, | |
| "learning_rate": 5.848888922025553e-07, | |
| "logits/chosen": -0.3702755868434906, | |
| "logits/rejected": -0.43015074729919434, | |
| "logps/chosen": -0.06369648873806, | |
| "logps/rejected": -0.6736232042312622, | |
| "loss": 0.0855, | |
| "odds_ratio_loss": 0.7752313613891602, | |
| "rewards/accuracies": 0.893750011920929, | |
| "rewards/chosen": -0.0063696494325995445, | |
| "rewards/margins": 0.06099267676472664, | |
| "rewards/rejected": -0.06736232340335846, | |
| "sft_loss": 0.007961621508002281, | |
| "step": 400 | |
| }, | |
| { | |
| "epoch": 7.901234567901234, | |
| "eval_logits/chosen": -0.3374575674533844, | |
| "eval_logits/rejected": -0.39353418350219727, | |
| "eval_logps/chosen": -0.11246351897716522, | |
| "eval_logps/rejected": -0.7215471267700195, | |
| "eval_loss": 0.1577637493610382, | |
| "eval_odds_ratio_loss": 1.4393588304519653, | |
| "eval_rewards/accuracies": 0.800000011920929, | |
| "eval_rewards/chosen": -0.011246351525187492, | |
| "eval_rewards/margins": 0.060908351093530655, | |
| "eval_rewards/rejected": -0.072154700756073, | |
| "eval_runtime": 3.7403, | |
| "eval_samples_per_second": 24.062, | |
| "eval_sft_loss": 0.013827885501086712, | |
| "eval_steps_per_second": 12.031, | |
| "step": 400 | |
| }, | |
| { | |
| "epoch": 8.098765432098766, | |
| "grad_norm": 0.5824891328811646, | |
| "learning_rate": 4.774575140626317e-07, | |
| "logits/chosen": -0.3890485167503357, | |
| "logits/rejected": -0.4562603533267975, | |
| "logps/chosen": -0.052860356867313385, | |
| "logps/rejected": -0.8017857670783997, | |
| "loss": 0.0769, | |
| "odds_ratio_loss": 0.7033621072769165, | |
| "rewards/accuracies": 0.8812500238418579, | |
| "rewards/chosen": -0.005286035593599081, | |
| "rewards/margins": 0.07489253580570221, | |
| "rewards/rejected": -0.08017858117818832, | |
| "sft_loss": 0.00657792529091239, | |
| "step": 410 | |
| }, | |
| { | |
| "epoch": 8.296296296296296, | |
| "grad_norm": 2.3090314865112305, | |
| "learning_rate": 3.798797596089351e-07, | |
| "logits/chosen": -0.36521631479263306, | |
| "logits/rejected": -0.43496161699295044, | |
| "logps/chosen": -0.052145786583423615, | |
| "logps/rejected": -0.6766245365142822, | |
| "loss": 0.0738, | |
| "odds_ratio_loss": 0.6751989722251892, | |
| "rewards/accuracies": 0.8812500238418579, | |
| "rewards/chosen": -0.0052145784720778465, | |
| "rewards/margins": 0.062447868287563324, | |
| "rewards/rejected": -0.06766244769096375, | |
| "sft_loss": 0.0063272216357290745, | |
| "step": 420 | |
| }, | |
| { | |
| "epoch": 8.493827160493828, | |
| "grad_norm": 3.1166369915008545, | |
| "learning_rate": 2.9263101785268253e-07, | |
| "logits/chosen": -0.3338077664375305, | |
| "logits/rejected": -0.4001466631889343, | |
| "logps/chosen": -0.05268881469964981, | |
| "logps/rejected": -0.8022252321243286, | |
| "loss": 0.0766, | |
| "odds_ratio_loss": 0.6915222406387329, | |
| "rewards/accuracies": 0.8812500238418579, | |
| "rewards/chosen": -0.005268882028758526, | |
| "rewards/margins": 0.07495363801717758, | |
| "rewards/rejected": -0.08022252470254898, | |
| "sft_loss": 0.007409657351672649, | |
| "step": 430 | |
| }, | |
| { | |
| "epoch": 8.691358024691358, | |
| "grad_norm": 2.2428250312805176, | |
| "learning_rate": 2.1613635589349756e-07, | |
| "logits/chosen": -0.3932591676712036, | |
| "logits/rejected": -0.4431510865688324, | |
| "logps/chosen": -0.05874771624803543, | |
| "logps/rejected": -0.8061612844467163, | |
| "loss": 0.0783, | |
| "odds_ratio_loss": 0.7208496928215027, | |
| "rewards/accuracies": 0.925000011920929, | |
| "rewards/chosen": -0.005874771624803543, | |
| "rewards/margins": 0.07474134862422943, | |
| "rewards/rejected": -0.08061611652374268, | |
| "sft_loss": 0.006212843116372824, | |
| "step": 440 | |
| }, | |
| { | |
| "epoch": 8.88888888888889, | |
| "grad_norm": 2.0012757778167725, | |
| "learning_rate": 1.507684480352292e-07, | |
| "logits/chosen": -0.3309887647628784, | |
| "logits/rejected": -0.3963066637516022, | |
| "logps/chosen": -0.07408946752548218, | |
| "logps/rejected": -0.7123581171035767, | |
| "loss": 0.0985, | |
| "odds_ratio_loss": 0.9039106369018555, | |
| "rewards/accuracies": 0.862500011920929, | |
| "rewards/chosen": -0.007408946752548218, | |
| "rewards/margins": 0.06382687389850616, | |
| "rewards/rejected": -0.07123581320047379, | |
| "sft_loss": 0.008154405280947685, | |
| "step": 450 | |
| }, | |
| { | |
| "epoch": 8.88888888888889, | |
| "eval_logits/chosen": -0.33720430731773376, | |
| "eval_logits/rejected": -0.3934166729450226, | |
| "eval_logps/chosen": -0.11222676187753677, | |
| "eval_logps/rejected": -0.7204592227935791, | |
| "eval_loss": 0.1574162095785141, | |
| "eval_odds_ratio_loss": 1.4357547760009766, | |
| "eval_rewards/accuracies": 0.800000011920929, | |
| "eval_rewards/chosen": -0.011222676374018192, | |
| "eval_rewards/margins": 0.06082325428724289, | |
| "eval_rewards/rejected": -0.07204592972993851, | |
| "eval_runtime": 3.7588, | |
| "eval_samples_per_second": 23.944, | |
| "eval_sft_loss": 0.013840759173035622, | |
| "eval_steps_per_second": 11.972, | |
| "step": 450 | |
| }, | |
| { | |
| "epoch": 9.08641975308642, | |
| "grad_norm": 1.9788808822631836, | |
| "learning_rate": 9.684576015420277e-08, | |
| "logits/chosen": -0.3609127700328827, | |
| "logits/rejected": -0.41957467794418335, | |
| "logps/chosen": -0.05120778828859329, | |
| "logps/rejected": -0.6972073316574097, | |
| "loss": 0.0681, | |
| "odds_ratio_loss": 0.6181837320327759, | |
| "rewards/accuracies": 0.9312499761581421, | |
| "rewards/chosen": -0.005120779387652874, | |
| "rewards/margins": 0.06459995359182358, | |
| "rewards/rejected": -0.06972073018550873, | |
| "sft_loss": 0.006232709623873234, | |
| "step": 460 | |
| }, | |
| { | |
| "epoch": 9.283950617283951, | |
| "grad_norm": 0.5987529158592224, | |
| "learning_rate": 5.463099816548578e-08, | |
| "logits/chosen": -0.3785928189754486, | |
| "logits/rejected": -0.44030579924583435, | |
| "logps/chosen": -0.06789538264274597, | |
| "logps/rejected": -0.718836784362793, | |
| "loss": 0.0919, | |
| "odds_ratio_loss": 0.8372634649276733, | |
| "rewards/accuracies": 0.8812500238418579, | |
| "rewards/chosen": -0.006789539009332657, | |
| "rewards/margins": 0.06509413570165634, | |
| "rewards/rejected": -0.0718836709856987, | |
| "sft_loss": 0.008134867064654827, | |
| "step": 470 | |
| }, | |
| { | |
| "epoch": 9.481481481481481, | |
| "grad_norm": 2.4453930854797363, | |
| "learning_rate": 2.4329828146074096e-08, | |
| "logits/chosen": -0.3719639182090759, | |
| "logits/rejected": -0.44119659066200256, | |
| "logps/chosen": -0.06199900060892105, | |
| "logps/rejected": -0.9121496081352234, | |
| "loss": 0.0796, | |
| "odds_ratio_loss": 0.7194961905479431, | |
| "rewards/accuracies": 0.90625, | |
| "rewards/chosen": -0.006199900526553392, | |
| "rewards/margins": 0.08501505851745605, | |
| "rewards/rejected": -0.09121496230363846, | |
| "sft_loss": 0.0076431455090641975, | |
| "step": 480 | |
| }, | |
| { | |
| "epoch": 9.679012345679013, | |
| "grad_norm": 2.6841046810150146, | |
| "learning_rate": 6.089874350439507e-09, | |
| "logits/chosen": -0.3508923053741455, | |
| "logits/rejected": -0.410422146320343, | |
| "logps/chosen": -0.050847820937633514, | |
| "logps/rejected": -0.717828631401062, | |
| "loss": 0.0729, | |
| "odds_ratio_loss": 0.6642852425575256, | |
| "rewards/accuracies": 0.90625, | |
| "rewards/chosen": -0.005084782373160124, | |
| "rewards/margins": 0.06669807434082031, | |
| "rewards/rejected": -0.07178286463022232, | |
| "sft_loss": 0.006511971354484558, | |
| "step": 490 | |
| }, | |
| { | |
| "epoch": 9.876543209876543, | |
| "grad_norm": 1.9767255783081055, | |
| "learning_rate": 0.0, | |
| "logits/chosen": -0.3572847545146942, | |
| "logits/rejected": -0.43199190497398376, | |
| "logps/chosen": -0.0603826642036438, | |
| "logps/rejected": -0.6694537401199341, | |
| "loss": 0.0859, | |
| "odds_ratio_loss": 0.7898350954055786, | |
| "rewards/accuracies": 0.887499988079071, | |
| "rewards/chosen": -0.006038266699761152, | |
| "rewards/margins": 0.060907114297151566, | |
| "rewards/rejected": -0.066945381462574, | |
| "sft_loss": 0.006888846401125193, | |
| "step": 500 | |
| }, | |
| { | |
| "epoch": 9.876543209876543, | |
| "eval_logits/chosen": -0.33729952573776245, | |
| "eval_logits/rejected": -0.39373528957366943, | |
| "eval_logps/chosen": -0.11289726942777634, | |
| "eval_logps/rejected": -0.7238631248474121, | |
| "eval_loss": 0.15817420184612274, | |
| "eval_odds_ratio_loss": 1.4418776035308838, | |
| "eval_rewards/accuracies": 0.7888888716697693, | |
| "eval_rewards/chosen": -0.011289726011455059, | |
| "eval_rewards/margins": 0.061096593737602234, | |
| "eval_rewards/rejected": -0.07238632440567017, | |
| "eval_runtime": 3.7425, | |
| "eval_samples_per_second": 24.048, | |
| "eval_sft_loss": 0.013986458070576191, | |
| "eval_steps_per_second": 12.024, | |
| "step": 500 | |
| }, | |
| { | |
| "epoch": 9.876543209876543, | |
| "step": 500, | |
| "total_flos": 8.823432944104243e+16, | |
| "train_loss": 0.3188942909240723, | |
| "train_runtime": 1116.7498, | |
| "train_samples_per_second": 7.253, | |
| "train_steps_per_second": 0.448 | |
| } | |
| ], | |
| "logging_steps": 10, | |
| "max_steps": 500, | |
| "num_input_tokens_seen": 0, | |
| "num_train_epochs": 10, | |
| "save_steps": 50, | |
| "stateful_callbacks": { | |
| "TrainerControl": { | |
| "args": { | |
| "should_epoch_stop": false, | |
| "should_evaluate": false, | |
| "should_log": false, | |
| "should_save": true, | |
| "should_training_stop": true | |
| }, | |
| "attributes": {} | |
| } | |
| }, | |
| "total_flos": 8.823432944104243e+16, | |
| "train_batch_size": 2, | |
| "trial_name": null, | |
| "trial_params": null | |
| } | |