{ "best_metric": null, "best_model_checkpoint": null, "epoch": 1.0, "eval_steps": 100, "global_step": 382, "is_hyper_param_search": false, "is_local_process_zero": true, "is_world_process_zero": true, "log_history": [ { "epoch": 0.002617801047120419, "grad_norm": 14.828922271728516, "learning_rate": 1.282051282051282e-08, "logits/chosen": -0.15906427800655365, "logits/rejected": -0.04599519073963165, "logps/chosen": -408.2510986328125, "logps/rejected": -356.72650146484375, "loss": 1.0, "rewards/accuracies": 0.0, "rewards/chosen": 0.0, "rewards/margins": 0.0, "rewards/rejected": 0.0, "step": 1 }, { "epoch": 0.02617801047120419, "grad_norm": 15.856895446777344, "learning_rate": 1.2820512820512818e-07, "logits/chosen": -0.08276861160993576, "logits/rejected": -0.17049165070056915, "logps/chosen": -341.0987548828125, "logps/rejected": -424.1305847167969, "loss": 0.9991, "rewards/accuracies": 0.5555555820465088, "rewards/chosen": -0.005943425931036472, "rewards/margins": 0.0022654342465102673, "rewards/rejected": -0.008208859711885452, "step": 10 }, { "epoch": 0.05235602094240838, "grad_norm": 16.610092163085938, "learning_rate": 2.5641025641025636e-07, "logits/chosen": 0.06369045376777649, "logits/rejected": -0.07681196928024292, "logps/chosen": -361.9441223144531, "logps/rejected": -487.20599365234375, "loss": 0.9757, "rewards/accuracies": 0.675000011920929, "rewards/chosen": -0.08942320197820663, "rewards/margins": 0.009710567072033882, "rewards/rejected": -0.09913377463817596, "step": 20 }, { "epoch": 0.07853403141361257, "grad_norm": 12.981971740722656, "learning_rate": 3.8461538461538463e-07, "logits/chosen": -0.18306311964988708, "logits/rejected": -0.23825812339782715, "logps/chosen": -380.07354736328125, "logps/rejected": -465.6441955566406, "loss": 0.8446, "rewards/accuracies": 0.824999988079071, "rewards/chosen": -0.5708522796630859, "rewards/margins": 0.034991003572940826, "rewards/rejected": -0.6058432459831238, "step": 30 }, { "epoch": 0.10471204188481675, "grad_norm": 6.047976493835449, "learning_rate": 4.999895137916554e-07, "logits/chosen": 0.013444331474602222, "logits/rejected": -0.1891317516565323, "logps/chosen": -594.7208251953125, "logps/rejected": -738.9493408203125, "loss": 0.5694, "rewards/accuracies": 0.875, "rewards/chosen": -2.218451499938965, "rewards/margins": 0.33880680799484253, "rewards/rejected": -2.5572586059570312, "step": 40 }, { "epoch": 0.13089005235602094, "grad_norm": 6.041112899780273, "learning_rate": 4.987322328603843e-07, "logits/chosen": -0.16668561100959778, "logits/rejected": -0.44701632857322693, "logps/chosen": -848.8206176757812, "logps/rejected": -1040.04345703125, "loss": 0.347, "rewards/accuracies": 0.925000011920929, "rewards/chosen": -4.788802623748779, "rewards/margins": 0.8872026205062866, "rewards/rejected": -5.6760053634643555, "step": 50 }, { "epoch": 0.15706806282722513, "grad_norm": 4.1997599601745605, "learning_rate": 4.953897892477663e-07, "logits/chosen": -0.2997971177101135, "logits/rejected": -0.4985489845275879, "logps/chosen": -1093.91552734375, "logps/rejected": -1338.5384521484375, "loss": 0.1998, "rewards/accuracies": 0.925000011920929, "rewards/chosen": -7.2311601638793945, "rewards/margins": 1.946942925453186, "rewards/rejected": -9.178102493286133, "step": 60 }, { "epoch": 0.18324607329842932, "grad_norm": 4.049875736236572, "learning_rate": 4.899902032011388e-07, "logits/chosen": -0.3460548520088196, "logits/rejected": -0.7166688442230225, "logps/chosen": -1084.109619140625, "logps/rejected": -1593.41357421875, "loss": 0.0898, "rewards/accuracies": 1.0, "rewards/chosen": -7.536008358001709, "rewards/margins": 4.000422954559326, "rewards/rejected": -11.536432266235352, "step": 70 }, { "epoch": 0.2094240837696335, "grad_norm": 1.4245041608810425, "learning_rate": 4.825787403267712e-07, "logits/chosen": -0.2860843539237976, "logits/rejected": -0.6564221382141113, "logps/chosen": -1109.26953125, "logps/rejected": -1771.4761962890625, "loss": 0.0397, "rewards/accuracies": 1.0, "rewards/chosen": -7.528865814208984, "rewards/margins": 5.633866310119629, "rewards/rejected": -13.16273307800293, "step": 80 }, { "epoch": 0.2356020942408377, "grad_norm": 2.5607237815856934, "learning_rate": 4.732175321209399e-07, "logits/chosen": -0.2063232958316803, "logits/rejected": -0.4774075150489807, "logps/chosen": -1142.3992919921875, "logps/rejected": -1939.1448974609375, "loss": 0.0213, "rewards/accuracies": 0.9750000238418579, "rewards/chosen": -8.23434829711914, "rewards/margins": 7.064591884613037, "rewards/rejected": -15.298940658569336, "step": 90 }, { "epoch": 0.2617801047120419, "grad_norm": 2.854067087173462, "learning_rate": 4.619850551115105e-07, "logits/chosen": -0.2572333812713623, "logits/rejected": -0.5195943117141724, "logps/chosen": -1152.82666015625, "logps/rejected": -2074.894287109375, "loss": 0.0103, "rewards/accuracies": 1.0, "rewards/chosen": -8.202000617980957, "rewards/margins": 8.496599197387695, "rewards/rejected": -16.698598861694336, "step": 100 }, { "epoch": 0.2617801047120419, "eval_logits/chosen": -0.2481275349855423, "eval_logits/rejected": -0.5485113859176636, "eval_logps/chosen": -1208.2967529296875, "eval_logps/rejected": -2315.333251953125, "eval_loss": 0.005963556934148073, "eval_rewards/accuracies": 1.0, "eval_rewards/chosen": -8.515900611877441, "eval_rewards/margins": 10.257165908813477, "eval_rewards/rejected": -18.7730655670166, "eval_runtime": 108.0217, "eval_samples_per_second": 5.953, "eval_steps_per_second": 0.38, "step": 100 }, { "epoch": 0.2879581151832461, "grad_norm": 0.14180447161197662, "learning_rate": 4.4897547297646633e-07, "logits/chosen": -0.16943633556365967, "logits/rejected": -0.5389738082885742, "logps/chosen": -1219.428466796875, "logps/rejected": -2504.55322265625, "loss": 0.0065, "rewards/accuracies": 1.0, "rewards/chosen": -8.39463996887207, "rewards/margins": 11.927027702331543, "rewards/rejected": -20.321666717529297, "step": 110 }, { "epoch": 0.31413612565445026, "grad_norm": 0.12577557563781738, "learning_rate": 4.3429784715451177e-07, "logits/chosen": -0.18040022253990173, "logits/rejected": -0.4654816687107086, "logps/chosen": -1176.073974609375, "logps/rejected": -2363.3984375, "loss": 0.0066, "rewards/accuracies": 1.0, "rewards/chosen": -8.234708786010742, "rewards/margins": 11.230855941772461, "rewards/rejected": -19.46556282043457, "step": 120 }, { "epoch": 0.3403141361256545, "grad_norm": 0.21279406547546387, "learning_rate": 4.180752225653292e-07, "logits/chosen": -0.13908787071704865, "logits/rejected": -0.4511902928352356, "logps/chosen": -1190.518310546875, "logps/rejected": -2524.310791015625, "loss": 0.0025, "rewards/accuracies": 1.0, "rewards/chosen": -8.432780265808105, "rewards/margins": 12.333601951599121, "rewards/rejected": -20.766382217407227, "step": 130 }, { "epoch": 0.36649214659685864, "grad_norm": 0.023909807205200195, "learning_rate": 4.00443596104045e-07, "logits/chosen": -0.050680726766586304, "logits/rejected": -0.36426204442977905, "logps/chosen": -1247.6243896484375, "logps/rejected": -2628.51611328125, "loss": 0.0012, "rewards/accuracies": 1.0, "rewards/chosen": -8.81135368347168, "rewards/margins": 12.927001953125, "rewards/rejected": -21.738351821899414, "step": 140 }, { "epoch": 0.39267015706806285, "grad_norm": 0.15804319083690643, "learning_rate": 3.815507765571832e-07, "logits/chosen": -0.13396403193473816, "logits/rejected": -0.4902087152004242, "logps/chosen": -1324.420654296875, "logps/rejected": -2804.68017578125, "loss": 0.0016, "rewards/accuracies": 1.0, "rewards/chosen": -8.971705436706543, "rewards/margins": 14.409167289733887, "rewards/rejected": -23.380870819091797, "step": 150 }, { "epoch": 0.418848167539267, "grad_norm": 0.023948727175593376, "learning_rate": 3.615551454976164e-07, "logits/chosen": -0.07858485728502274, "logits/rejected": -0.3286251127719879, "logps/chosen": -1267.703857421875, "logps/rejected": -2703.435302734375, "loss": 0.0019, "rewards/accuracies": 1.0, "rewards/chosen": -9.188163757324219, "rewards/margins": 13.80443000793457, "rewards/rejected": -22.99259376525879, "step": 160 }, { "epoch": 0.44502617801047123, "grad_norm": 0.017125073820352554, "learning_rate": 3.406243295461325e-07, "logits/chosen": -0.1788567751646042, "logits/rejected": -0.4664693772792816, "logps/chosen": -1347.940185546875, "logps/rejected": -3000.385009765625, "loss": 0.0007, "rewards/accuracies": 1.0, "rewards/chosen": -9.661752700805664, "rewards/margins": 15.739896774291992, "rewards/rejected": -25.40165138244629, "step": 170 }, { "epoch": 0.4712041884816754, "grad_norm": 0.041922952979803085, "learning_rate": 3.189337951302639e-07, "logits/chosen": -0.03258290886878967, "logits/rejected": -0.35061293840408325, "logps/chosen": -1287.1939697265625, "logps/rejected": -2967.044921875, "loss": 0.002, "rewards/accuracies": 1.0, "rewards/chosen": -9.461482048034668, "rewards/margins": 15.719799995422363, "rewards/rejected": -25.18128204345703, "step": 180 }, { "epoch": 0.4973821989528796, "grad_norm": 0.01813839003443718, "learning_rate": 2.966653775207444e-07, "logits/chosen": -0.07453624904155731, "logits/rejected": -0.3937017023563385, "logps/chosen": -1329.836669921875, "logps/rejected": -2857.601806640625, "loss": 0.0007, "rewards/accuracies": 1.0, "rewards/chosen": -9.457625389099121, "rewards/margins": 14.640398025512695, "rewards/rejected": -24.0980224609375, "step": 190 }, { "epoch": 0.5235602094240838, "grad_norm": 0.07226074486970901, "learning_rate": 2.7400575647692043e-07, "logits/chosen": -0.10540028661489487, "logits/rejected": -0.44681453704833984, "logps/chosen": -1375.4234619140625, "logps/rejected": -2861.271728515625, "loss": 0.0005, "rewards/accuracies": 1.0, "rewards/chosen": -9.861254692077637, "rewards/margins": 14.127073287963867, "rewards/rejected": -23.988330841064453, "step": 200 }, { "epoch": 0.5235602094240838, "eval_logits/chosen": -0.06605865061283112, "eval_logits/rejected": -0.37234431505203247, "eval_logps/chosen": -1349.2587890625, "eval_logps/rejected": -2929.19482421875, "eval_loss": 0.0004758847935590893, "eval_rewards/accuracies": 1.0, "eval_rewards/chosen": -9.925518989562988, "eval_rewards/margins": 14.986161231994629, "eval_rewards/rejected": -24.911680221557617, "eval_runtime": 107.9417, "eval_samples_per_second": 5.957, "eval_steps_per_second": 0.38, "step": 200 }, { "epoch": 0.5497382198952879, "grad_norm": 0.02703760750591755, "learning_rate": 2.511448912800265e-07, "logits/chosen": -0.15184970200061798, "logits/rejected": -0.2841305136680603, "logps/chosen": -1298.2142333984375, "logps/rejected": -2762.650146484375, "loss": 0.0006, "rewards/accuracies": 1.0, "rewards/chosen": -9.895256042480469, "rewards/margins": 13.653951644897461, "rewards/rejected": -23.549209594726562, "step": 210 }, { "epoch": 0.5759162303664922, "grad_norm": 0.1174333319067955, "learning_rate": 2.2827442827369772e-07, "logits/chosen": -0.026270607486367226, "logits/rejected": -0.35377901792526245, "logps/chosen": -1360.6993408203125, "logps/rejected": -3060.089111328125, "loss": 0.0005, "rewards/accuracies": 1.0, "rewards/chosen": -9.930294036865234, "rewards/margins": 15.97572135925293, "rewards/rejected": -25.906017303466797, "step": 220 }, { "epoch": 0.6020942408376964, "grad_norm": 0.011358899995684624, "learning_rate": 2.0558609426156343e-07, "logits/chosen": -0.11066827923059464, "logits/rejected": -0.36893755197525024, "logps/chosen": -1335.5362548828125, "logps/rejected": -2940.074951171875, "loss": 0.0005, "rewards/accuracies": 1.0, "rewards/chosen": -9.88429069519043, "rewards/margins": 15.1347074508667, "rewards/rejected": -25.018997192382812, "step": 230 }, { "epoch": 0.6282722513089005, "grad_norm": 0.044269297271966934, "learning_rate": 1.8327008923033222e-07, "logits/chosen": -0.0352923646569252, "logits/rejected": -0.3661387860774994, "logps/chosen": -1327.6500244140625, "logps/rejected": -3067.07568359375, "loss": 0.0008, "rewards/accuracies": 1.0, "rewards/chosen": -9.962610244750977, "rewards/margins": 16.210254669189453, "rewards/rejected": -26.172870635986328, "step": 240 }, { "epoch": 0.6544502617801047, "grad_norm": 0.019647102802991867, "learning_rate": 1.6151349187243063e-07, "logits/chosen": 0.07167226076126099, "logits/rejected": -0.2115941047668457, "logps/chosen": -1318.0687255859375, "logps/rejected": -2981.04296875, "loss": 0.0004, "rewards/accuracies": 1.0, "rewards/chosen": -9.883614540100098, "rewards/margins": 15.260563850402832, "rewards/rejected": -25.144176483154297, "step": 250 }, { "epoch": 0.680628272251309, "grad_norm": 0.09396984428167343, "learning_rate": 1.4049869127495884e-07, "logits/chosen": -0.15649020671844482, "logits/rejected": -0.3796946406364441, "logps/chosen": -1347.2823486328125, "logps/rejected": -2941.29541015625, "loss": 0.0003, "rewards/accuracies": 1.0, "rewards/chosen": -9.877740859985352, "rewards/margins": 15.382551193237305, "rewards/rejected": -25.260290145874023, "step": 260 }, { "epoch": 0.7068062827225131, "grad_norm": 0.1975828856229782, "learning_rate": 1.2040185792236873e-07, "logits/chosen": -0.11410210281610489, "logits/rejected": -0.18086448311805725, "logps/chosen": -1348.89697265625, "logps/rejected": -2811.34423828125, "loss": 0.0004, "rewards/accuracies": 1.0, "rewards/chosen": -9.77053165435791, "rewards/margins": 14.368619918823242, "rewards/rejected": -24.13915252685547, "step": 270 }, { "epoch": 0.7329842931937173, "grad_norm": 0.019497638568282127, "learning_rate": 1.0139146683069728e-07, "logits/chosen": 0.10111899673938751, "logits/rejected": -0.11041772365570068, "logps/chosen": -1279.238037109375, "logps/rejected": -2960.041748046875, "loss": 0.0003, "rewards/accuracies": 1.0, "rewards/chosen": -9.907655715942383, "rewards/margins": 15.42730712890625, "rewards/rejected": -25.334964752197266, "step": 280 }, { "epoch": 0.7591623036649214, "grad_norm": 0.033979929983615875, "learning_rate": 8.362688519416084e-08, "logits/chosen": -0.05218152329325676, "logits/rejected": -0.26625046133995056, "logps/chosen": -1348.1783447265625, "logps/rejected": -2767.76025390625, "loss": 0.0003, "rewards/accuracies": 1.0, "rewards/chosen": -10.007095336914062, "rewards/margins": 13.48937702178955, "rewards/rejected": -23.496471405029297, "step": 290 }, { "epoch": 0.7853403141361257, "grad_norm": 0.06925738602876663, "learning_rate": 6.725703638409861e-08, "logits/chosen": -0.1774607002735138, "logits/rejected": -0.43066462874412537, "logps/chosen": -1356.197265625, "logps/rejected": -3014.517578125, "loss": 0.0005, "rewards/accuracies": 1.0, "rewards/chosen": -10.209028244018555, "rewards/margins": 15.68299674987793, "rewards/rejected": -25.892024993896484, "step": 300 }, { "epoch": 0.7853403141361257, "eval_logits/chosen": -0.001429112977348268, "eval_logits/rejected": -0.28819936513900757, "eval_logps/chosen": -1365.4342041015625, "eval_logps/rejected": -3031.217529296875, "eval_loss": 0.0003506154753267765, "eval_rewards/accuracies": 1.0, "eval_rewards/chosen": -10.087275505065918, "eval_rewards/margins": 15.844633102416992, "eval_rewards/rejected": -25.931907653808594, "eval_runtime": 107.8271, "eval_samples_per_second": 5.963, "eval_steps_per_second": 0.38, "step": 300 }, { "epoch": 0.8115183246073299, "grad_norm": 0.02467566728591919, "learning_rate": 5.2419151500177395e-08, "logits/chosen": -0.025030899792909622, "logits/rejected": -0.2692425847053528, "logps/chosen": -1327.07763671875, "logps/rejected": -2993.914306640625, "loss": 0.0003, "rewards/accuracies": 1.0, "rewards/chosen": -10.048177719116211, "rewards/margins": 15.598210334777832, "rewards/rejected": -25.64638900756836, "step": 310 }, { "epoch": 0.837696335078534, "grad_norm": 0.028052976354956627, "learning_rate": 3.9237618939808646e-08, "logits/chosen": -0.056387972086668015, "logits/rejected": -0.31664109230041504, "logps/chosen": -1336.064208984375, "logps/rejected": -2948.28662109375, "loss": 0.0005, "rewards/accuracies": 1.0, "rewards/chosen": -10.119478225708008, "rewards/margins": 15.11084270477295, "rewards/rejected": -25.23032569885254, "step": 320 }, { "epoch": 0.8638743455497382, "grad_norm": 0.7250455617904663, "learning_rate": 2.7822941630022605e-08, "logits/chosen": -0.13823747634887695, "logits/rejected": -0.41743606328964233, "logps/chosen": -1375.549560546875, "logps/rejected": -3119.734619140625, "loss": 0.0005, "rewards/accuracies": 1.0, "rewards/chosen": -10.060212135314941, "rewards/margins": 16.325824737548828, "rewards/rejected": -26.386037826538086, "step": 330 }, { "epoch": 0.8900523560209425, "grad_norm": 0.012951872311532497, "learning_rate": 1.827081066349459e-08, "logits/chosen": -0.04458152875304222, "logits/rejected": -0.298855721950531, "logps/chosen": -1335.496826171875, "logps/rejected": -3045.818359375, "loss": 0.0002, "rewards/accuracies": 1.0, "rewards/chosen": -10.128870010375977, "rewards/margins": 15.832036972045898, "rewards/rejected": -25.960906982421875, "step": 340 }, { "epoch": 0.9162303664921466, "grad_norm": 0.06806211918592453, "learning_rate": 1.0661303104582881e-08, "logits/chosen": 0.04766862466931343, "logits/rejected": -0.2507403492927551, "logps/chosen": -1373.188232421875, "logps/rejected": -3208.185546875, "loss": 0.0002, "rewards/accuracies": 1.0, "rewards/chosen": -10.056095123291016, "rewards/margins": 17.54215431213379, "rewards/rejected": -27.59824562072754, "step": 350 }, { "epoch": 0.9424083769633508, "grad_norm": 0.013503313064575195, "learning_rate": 5.058210690300463e-09, "logits/chosen": 0.012905213050544262, "logits/rejected": -0.32594579458236694, "logps/chosen": -1348.8189697265625, "logps/rejected": -3084.17626953125, "loss": 0.0003, "rewards/accuracies": 1.0, "rewards/chosen": -10.090983390808105, "rewards/margins": 16.455230712890625, "rewards/rejected": -26.546215057373047, "step": 360 }, { "epoch": 0.9685863874345549, "grad_norm": 0.01822359301149845, "learning_rate": 1.5085050538290456e-09, "logits/chosen": 0.03194817900657654, "logits/rejected": -0.26868775486946106, "logps/chosen": -1382.170654296875, "logps/rejected": -3104.619873046875, "loss": 0.0004, "rewards/accuracies": 1.0, "rewards/chosen": -9.998517990112305, "rewards/margins": 16.327945709228516, "rewards/rejected": -26.326465606689453, "step": 370 }, { "epoch": 0.9947643979057592, "grad_norm": 0.021796291694045067, "learning_rate": 4.19439536940569e-11, "logits/chosen": -0.09669488668441772, "logits/rejected": -0.31547075510025024, "logps/chosen": -1308.7725830078125, "logps/rejected": -3017.67529296875, "loss": 0.0002, "rewards/accuracies": 1.0, "rewards/chosen": -10.053609848022461, "rewards/margins": 15.717524528503418, "rewards/rejected": -25.771133422851562, "step": 380 }, { "epoch": 1.0, "step": 382, "total_flos": 0.0, "train_loss": 0.10807222205026247, "train_runtime": 19238.6363, "train_samples_per_second": 0.635, "train_steps_per_second": 0.02 } ], "logging_steps": 10, "max_steps": 382, "num_input_tokens_seen": 0, "num_train_epochs": 1, "save_steps": 500, "stateful_callbacks": { "TrainerControl": { "args": { "should_epoch_stop": false, "should_evaluate": false, "should_log": false, "should_save": false, "should_training_stop": false }, "attributes": {} } }, "total_flos": 0.0, "train_batch_size": 2, "trial_name": null, "trial_params": null }