| { | |
| "best_metric": null, | |
| "best_model_checkpoint": null, | |
| "epoch": 1.0, | |
| "eval_steps": 100, | |
| "global_step": 355, | |
| "is_hyper_param_search": false, | |
| "is_local_process_zero": true, | |
| "is_world_process_zero": true, | |
| "log_history": [ | |
| { | |
| "dpo_losses": 0.6931471824645996, | |
| "epoch": 0.0, | |
| "grad_norm": 1.6018567815095135, | |
| "learning_rate": 1.3888888888888888e-07, | |
| "logits/chosen": -2.861618995666504, | |
| "logits/rejected": -2.8205904960632324, | |
| "logps/chosen": -271.06011962890625, | |
| "logps/rejected": -211.1704559326172, | |
| "loss": 0.6931, | |
| "positive_losses": 0.0, | |
| "rewards/accuracies": 0.0, | |
| "rewards/chosen": 0.0, | |
| "rewards/margins": 0.0, | |
| "rewards/margins_max": 0.0, | |
| "rewards/margins_min": 0.0, | |
| "rewards/margins_std": 0.0, | |
| "rewards/rejected": 0.0, | |
| "step": 1 | |
| }, | |
| { | |
| "dpo_losses": 0.6928361654281616, | |
| "epoch": 0.03, | |
| "grad_norm": 14.098492351037597, | |
| "learning_rate": 1.3888888888888892e-06, | |
| "logits/chosen": -2.8340628147125244, | |
| "logits/rejected": -2.7916715145111084, | |
| "logps/chosen": -324.87408447265625, | |
| "logps/rejected": -274.8518371582031, | |
| "loss": 0.6969, | |
| "positive_losses": 0.03656284138560295, | |
| "rewards/accuracies": 0.5138888955116272, | |
| "rewards/chosen": 0.001762597355991602, | |
| "rewards/margins": 0.0006246823468245566, | |
| "rewards/margins_max": 0.0034460597671568394, | |
| "rewards/margins_min": -0.002478615380823612, | |
| "rewards/margins_std": 0.002669532783329487, | |
| "rewards/rejected": 0.0011379148345440626, | |
| "step": 10 | |
| }, | |
| { | |
| "dpo_losses": 0.6901537775993347, | |
| "epoch": 0.06, | |
| "grad_norm": 1.829780676576113, | |
| "learning_rate": 2.7777777777777783e-06, | |
| "logits/chosen": -2.7248008251190186, | |
| "logits/rejected": -2.7065372467041016, | |
| "logps/chosen": -291.9751892089844, | |
| "logps/rejected": -214.52914428710938, | |
| "loss": 0.69, | |
| "positive_losses": 0.00235748291015625, | |
| "rewards/accuracies": 0.7875000238418579, | |
| "rewards/chosen": 0.01850745640695095, | |
| "rewards/margins": 0.006009287666529417, | |
| "rewards/margins_max": 0.013369890861213207, | |
| "rewards/margins_min": -0.0006899007130414248, | |
| "rewards/margins_std": 0.006301888730376959, | |
| "rewards/rejected": 0.01249817106872797, | |
| "step": 20 | |
| }, | |
| { | |
| "dpo_losses": 0.6790497303009033, | |
| "epoch": 0.08, | |
| "grad_norm": 2.096661038575657, | |
| "learning_rate": 4.166666666666667e-06, | |
| "logits/chosen": -2.8153939247131348, | |
| "logits/rejected": -2.7460672855377197, | |
| "logps/chosen": -298.10052490234375, | |
| "logps/rejected": -229.7678680419922, | |
| "loss": 0.677, | |
| "positive_losses": 0.0, | |
| "rewards/accuracies": 0.887499988079071, | |
| "rewards/chosen": 0.05605363845825195, | |
| "rewards/margins": 0.02858993411064148, | |
| "rewards/margins_max": 0.058357615023851395, | |
| "rewards/margins_min": 0.004640273749828339, | |
| "rewards/margins_std": 0.02467900700867176, | |
| "rewards/rejected": 0.027463700622320175, | |
| "step": 30 | |
| }, | |
| { | |
| "dpo_losses": 0.6675597429275513, | |
| "epoch": 0.11, | |
| "grad_norm": 1.7320035926217752, | |
| "learning_rate": 4.998060489154965e-06, | |
| "logits/chosen": -2.8310070037841797, | |
| "logits/rejected": -2.751425266265869, | |
| "logps/chosen": -268.48809814453125, | |
| "logps/rejected": -222.01107788085938, | |
| "loss": 0.6662, | |
| "positive_losses": 0.054492950439453125, | |
| "rewards/accuracies": 0.862500011920929, | |
| "rewards/chosen": 0.08996561169624329, | |
| "rewards/margins": 0.05272960662841797, | |
| "rewards/margins_max": 0.1101265698671341, | |
| "rewards/margins_min": 0.003616312053054571, | |
| "rewards/margins_std": 0.048521898686885834, | |
| "rewards/rejected": 0.03723599761724472, | |
| "step": 40 | |
| }, | |
| { | |
| "dpo_losses": 0.6397972106933594, | |
| "epoch": 0.14, | |
| "grad_norm": 9.583890638870626, | |
| "learning_rate": 4.976275538042932e-06, | |
| "logits/chosen": -2.7891061305999756, | |
| "logits/rejected": -2.7175135612487793, | |
| "logps/chosen": -262.20794677734375, | |
| "logps/rejected": -231.79653930664062, | |
| "loss": 0.6446, | |
| "positive_losses": 0.0, | |
| "rewards/accuracies": 0.9375, | |
| "rewards/chosen": 0.13362163305282593, | |
| "rewards/margins": 0.11281381547451019, | |
| "rewards/margins_max": 0.23626498878002167, | |
| "rewards/margins_min": 0.022470083087682724, | |
| "rewards/margins_std": 0.0988926962018013, | |
| "rewards/rejected": 0.02080780453979969, | |
| "step": 50 | |
| }, | |
| { | |
| "dpo_losses": 0.6110584139823914, | |
| "epoch": 0.17, | |
| "grad_norm": 2.0747443213986694, | |
| "learning_rate": 4.93049306999712e-06, | |
| "logits/chosen": -2.7118520736694336, | |
| "logits/rejected": -2.6753315925598145, | |
| "logps/chosen": -296.9767150878906, | |
| "logps/rejected": -263.8233947753906, | |
| "loss": 0.628, | |
| "positive_losses": 0.011554336175322533, | |
| "rewards/accuracies": 0.987500011920929, | |
| "rewards/chosen": 0.16662926971912384, | |
| "rewards/margins": 0.17714819312095642, | |
| "rewards/margins_max": 0.30765318870544434, | |
| "rewards/margins_min": 0.05318903177976608, | |
| "rewards/margins_std": 0.11578011512756348, | |
| "rewards/rejected": -0.010518952272832394, | |
| "step": 60 | |
| }, | |
| { | |
| "dpo_losses": 0.6022371053695679, | |
| "epoch": 0.2, | |
| "grad_norm": 1.5871888283763238, | |
| "learning_rate": 4.861156761634014e-06, | |
| "logits/chosen": -2.7271430492401123, | |
| "logits/rejected": -2.6688759326934814, | |
| "logps/chosen": -303.47613525390625, | |
| "logps/rejected": -236.2406463623047, | |
| "loss": 0.6175, | |
| "positive_losses": 0.19450588524341583, | |
| "rewards/accuracies": 0.949999988079071, | |
| "rewards/chosen": 0.19385087490081787, | |
| "rewards/margins": 0.19984133541584015, | |
| "rewards/margins_max": 0.4134605824947357, | |
| "rewards/margins_min": 0.04761160537600517, | |
| "rewards/margins_std": 0.16880682110786438, | |
| "rewards/rejected": -0.00599044980481267, | |
| "step": 70 | |
| }, | |
| { | |
| "dpo_losses": 0.5768495798110962, | |
| "epoch": 0.23, | |
| "grad_norm": 1.804849988880195, | |
| "learning_rate": 4.7689385491773934e-06, | |
| "logits/chosen": -2.738285779953003, | |
| "logits/rejected": -2.684203863143921, | |
| "logps/chosen": -300.8853454589844, | |
| "logps/rejected": -292.05633544921875, | |
| "loss": 0.6017, | |
| "positive_losses": 0.328561395406723, | |
| "rewards/accuracies": 0.987500011920929, | |
| "rewards/chosen": 0.20385125279426575, | |
| "rewards/margins": 0.26062771677970886, | |
| "rewards/margins_max": 0.4970013201236725, | |
| "rewards/margins_min": 0.05170217156410217, | |
| "rewards/margins_std": 0.2058703452348709, | |
| "rewards/rejected": -0.056776486337184906, | |
| "step": 80 | |
| }, | |
| { | |
| "dpo_losses": 0.5672236084938049, | |
| "epoch": 0.25, | |
| "grad_norm": 2.184742961229221, | |
| "learning_rate": 4.654732116743193e-06, | |
| "logits/chosen": -2.6370556354522705, | |
| "logits/rejected": -2.601066827774048, | |
| "logps/chosen": -252.70535278320312, | |
| "logps/rejected": -203.89418029785156, | |
| "loss": 0.5769, | |
| "positive_losses": 0.07196970283985138, | |
| "rewards/accuracies": 0.9750000238418579, | |
| "rewards/chosen": 0.2328944206237793, | |
| "rewards/margins": 0.2819642424583435, | |
| "rewards/margins_max": 0.514846682548523, | |
| "rewards/margins_min": 0.09985215216875076, | |
| "rewards/margins_std": 0.19404996931552887, | |
| "rewards/rejected": -0.049069829285144806, | |
| "step": 90 | |
| }, | |
| { | |
| "dpo_losses": 0.5702880620956421, | |
| "epoch": 0.28, | |
| "grad_norm": 2.550586173059517, | |
| "learning_rate": 4.5196442356717526e-06, | |
| "logits/chosen": -2.6703598499298096, | |
| "logits/rejected": -2.6374478340148926, | |
| "logps/chosen": -264.9583740234375, | |
| "logps/rejected": -273.49615478515625, | |
| "loss": 0.6232, | |
| "positive_losses": 1.2302151918411255, | |
| "rewards/accuracies": 0.949999988079071, | |
| "rewards/chosen": 0.16453364491462708, | |
| "rewards/margins": 0.27762115001678467, | |
| "rewards/margins_max": 0.5491287708282471, | |
| "rewards/margins_min": 0.05581303685903549, | |
| "rewards/margins_std": 0.22483690083026886, | |
| "rewards/rejected": -0.113087497651577, | |
| "step": 100 | |
| }, | |
| { | |
| "epoch": 0.28, | |
| "eval_dpo_losses": 0.6656126976013184, | |
| "eval_logits/chosen": -2.67258620262146, | |
| "eval_logits/rejected": -2.6360833644866943, | |
| "eval_logps/chosen": -280.30804443359375, | |
| "eval_logps/rejected": -261.0971984863281, | |
| "eval_loss": 1.1412982940673828, | |
| "eval_positive_losses": 4.261031627655029, | |
| "eval_rewards/accuracies": 0.6230000257492065, | |
| "eval_rewards/chosen": 0.04285382851958275, | |
| "eval_rewards/margins": 0.06803657114505768, | |
| "eval_rewards/margins_max": 0.40864306688308716, | |
| "eval_rewards/margins_min": -0.22808942198753357, | |
| "eval_rewards/margins_std": 0.2094314992427826, | |
| "eval_rewards/rejected": -0.02518274076282978, | |
| "eval_runtime": 429.2755, | |
| "eval_samples_per_second": 4.659, | |
| "eval_steps_per_second": 0.291, | |
| "step": 100 | |
| }, | |
| { | |
| "dpo_losses": 0.5097740888595581, | |
| "epoch": 0.31, | |
| "grad_norm": 6.336382416368574, | |
| "learning_rate": 4.364984038837727e-06, | |
| "logits/chosen": -2.742903709411621, | |
| "logits/rejected": -2.654869318008423, | |
| "logps/chosen": -349.24517822265625, | |
| "logps/rejected": -304.54730224609375, | |
| "loss": 0.543, | |
| "positive_losses": 0.44344156980514526, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.25360527634620667, | |
| "rewards/margins": 0.43474069237709045, | |
| "rewards/margins_max": 0.7704421281814575, | |
| "rewards/margins_min": 0.1366521120071411, | |
| "rewards/margins_std": 0.2834155559539795, | |
| "rewards/rejected": -0.1811354160308838, | |
| "step": 110 | |
| }, | |
| { | |
| "dpo_losses": 0.518837571144104, | |
| "epoch": 0.34, | |
| "grad_norm": 2.194144050007341, | |
| "learning_rate": 4.192250333880045e-06, | |
| "logits/chosen": -2.7281386852264404, | |
| "logits/rejected": -2.670868396759033, | |
| "logps/chosen": -321.75982666015625, | |
| "logps/rejected": -280.87091064453125, | |
| "loss": 0.5524, | |
| "positive_losses": 0.46012669801712036, | |
| "rewards/accuracies": 0.9750000238418579, | |
| "rewards/chosen": 0.26626402139663696, | |
| "rewards/margins": 0.4130307137966156, | |
| "rewards/margins_max": 0.7945607900619507, | |
| "rewards/margins_min": 0.14706461131572723, | |
| "rewards/margins_std": 0.2963123917579651, | |
| "rewards/rejected": -0.14676669239997864, | |
| "step": 120 | |
| }, | |
| { | |
| "dpo_losses": 0.4917011260986328, | |
| "epoch": 0.37, | |
| "grad_norm": 1.7534787479023215, | |
| "learning_rate": 4.0031170782990214e-06, | |
| "logits/chosen": -2.711912155151367, | |
| "logits/rejected": -2.634033441543579, | |
| "logps/chosen": -353.554443359375, | |
| "logps/rejected": -320.6388244628906, | |
| "loss": 0.5518, | |
| "positive_losses": 0.8977662920951843, | |
| "rewards/accuracies": 0.987500011920929, | |
| "rewards/chosen": 0.2880980372428894, | |
| "rewards/margins": 0.4901772439479828, | |
| "rewards/margins_max": 0.8924927711486816, | |
| "rewards/margins_min": 0.1499636471271515, | |
| "rewards/margins_std": 0.3346417546272278, | |
| "rewards/rejected": -0.20207922160625458, | |
| "step": 130 | |
| }, | |
| { | |
| "dpo_losses": 0.4866393208503723, | |
| "epoch": 0.39, | |
| "grad_norm": 21.27134583914694, | |
| "learning_rate": 3.7994171571810756e-06, | |
| "logits/chosen": -2.6895060539245605, | |
| "logits/rejected": -2.6512811183929443, | |
| "logps/chosen": -291.05548095703125, | |
| "logps/rejected": -294.4687805175781, | |
| "loss": 0.5718, | |
| "positive_losses": 0.2207096517086029, | |
| "rewards/accuracies": 0.987500011920929, | |
| "rewards/chosen": 0.2735855281352997, | |
| "rewards/margins": 0.5197780132293701, | |
| "rewards/margins_max": 1.003483772277832, | |
| "rewards/margins_min": 0.1269286870956421, | |
| "rewards/margins_std": 0.3979441523551941, | |
| "rewards/rejected": -0.24619252979755402, | |
| "step": 140 | |
| }, | |
| { | |
| "dpo_losses": 0.5046078562736511, | |
| "epoch": 0.42, | |
| "grad_norm": 3.3011186957688583, | |
| "learning_rate": 3.5831246207606597e-06, | |
| "logits/chosen": -2.6959190368652344, | |
| "logits/rejected": -2.658679962158203, | |
| "logps/chosen": -264.2646179199219, | |
| "logps/rejected": -234.5491180419922, | |
| "loss": 0.5366, | |
| "positive_losses": 0.490040123462677, | |
| "rewards/accuracies": 0.949999988079071, | |
| "rewards/chosen": 0.24420371651649475, | |
| "rewards/margins": 0.45886701345443726, | |
| "rewards/margins_max": 0.8680801391601562, | |
| "rewards/margins_min": 0.1154303103685379, | |
| "rewards/margins_std": 0.34930768609046936, | |
| "rewards/rejected": -0.2146632969379425, | |
| "step": 150 | |
| }, | |
| { | |
| "dpo_losses": 0.48088502883911133, | |
| "epoch": 0.45, | |
| "grad_norm": 2.135658014816511, | |
| "learning_rate": 3.3563355539546795e-06, | |
| "logits/chosen": -2.665548801422119, | |
| "logits/rejected": -2.6138901710510254, | |
| "logps/chosen": -274.263427734375, | |
| "logps/rejected": -260.50518798828125, | |
| "loss": 0.5724, | |
| "positive_losses": 0.9731669425964355, | |
| "rewards/accuracies": 0.987500011920929, | |
| "rewards/chosen": 0.262834370136261, | |
| "rewards/margins": 0.5239533185958862, | |
| "rewards/margins_max": 0.9884392023086548, | |
| "rewards/margins_min": 0.15575796365737915, | |
| "rewards/margins_std": 0.3754872977733612, | |
| "rewards/rejected": -0.26111894845962524, | |
| "step": 160 | |
| }, | |
| { | |
| "dpo_losses": 0.4504636824131012, | |
| "epoch": 0.48, | |
| "grad_norm": 3.940043763048366, | |
| "learning_rate": 3.121247763262235e-06, | |
| "logits/chosen": -2.708754777908325, | |
| "logits/rejected": -2.657917022705078, | |
| "logps/chosen": -297.7489013671875, | |
| "logps/rejected": -327.0563049316406, | |
| "loss": 0.4813, | |
| "positive_losses": 0.03098602220416069, | |
| "rewards/accuracies": 0.9624999761581421, | |
| "rewards/chosen": 0.3187271058559418, | |
| "rewards/margins": 0.6266334652900696, | |
| "rewards/margins_max": 1.0517089366912842, | |
| "rewards/margins_min": 0.17500966787338257, | |
| "rewards/margins_std": 0.3909396231174469, | |
| "rewards/rejected": -0.3079063296318054, | |
| "step": 170 | |
| }, | |
| { | |
| "dpo_losses": 0.4588772654533386, | |
| "epoch": 0.51, | |
| "grad_norm": 8.823245159881209, | |
| "learning_rate": 2.8801394778833475e-06, | |
| "logits/chosen": -2.6968963146209717, | |
| "logits/rejected": -2.6140356063842773, | |
| "logps/chosen": -305.4325866699219, | |
| "logps/rejected": -326.99798583984375, | |
| "loss": 0.5468, | |
| "positive_losses": 0.8232825994491577, | |
| "rewards/accuracies": 0.987500011920929, | |
| "rewards/chosen": 0.2646820843219757, | |
| "rewards/margins": 0.5928131937980652, | |
| "rewards/margins_max": 1.0361554622650146, | |
| "rewards/margins_min": 0.28750157356262207, | |
| "rewards/margins_std": 0.33570224046707153, | |
| "rewards/rejected": -0.32813113927841187, | |
| "step": 180 | |
| }, | |
| { | |
| "dpo_losses": 0.45539379119873047, | |
| "epoch": 0.54, | |
| "grad_norm": 3.517893013000186, | |
| "learning_rate": 2.6353472714635443e-06, | |
| "logits/chosen": -2.6537580490112305, | |
| "logits/rejected": -2.5634191036224365, | |
| "logps/chosen": -287.6109619140625, | |
| "logps/rejected": -265.6959228515625, | |
| "loss": 0.5435, | |
| "positive_losses": 0.9886103868484497, | |
| "rewards/accuracies": 0.949999988079071, | |
| "rewards/chosen": 0.34349915385246277, | |
| "rewards/margins": 0.6255816221237183, | |
| "rewards/margins_max": 1.1905597448349, | |
| "rewards/margins_min": 0.168921560049057, | |
| "rewards/margins_std": 0.453277051448822, | |
| "rewards/rejected": -0.2820824980735779, | |
| "step": 190 | |
| }, | |
| { | |
| "dpo_losses": 0.44315657019615173, | |
| "epoch": 0.56, | |
| "grad_norm": 27.976402148032502, | |
| "learning_rate": 2.3892434184240536e-06, | |
| "logits/chosen": -2.7400636672973633, | |
| "logits/rejected": -2.662397623062134, | |
| "logps/chosen": -309.39691162109375, | |
| "logps/rejected": -299.7530212402344, | |
| "loss": 0.5625, | |
| "positive_losses": 0.9616166353225708, | |
| "rewards/accuracies": 0.9624999761581421, | |
| "rewards/chosen": 0.30135902762413025, | |
| "rewards/margins": 0.6429153084754944, | |
| "rewards/margins_max": 1.131412148475647, | |
| "rewards/margins_min": 0.17879006266593933, | |
| "rewards/margins_std": 0.4260264039039612, | |
| "rewards/rejected": -0.34155628085136414, | |
| "step": 200 | |
| }, | |
| { | |
| "epoch": 0.56, | |
| "eval_dpo_losses": 0.6469283699989319, | |
| "eval_logits/chosen": -2.678022623062134, | |
| "eval_logits/rejected": -2.6380200386047363, | |
| "eval_logps/chosen": -286.4236145019531, | |
| "eval_logps/rejected": -272.83990478515625, | |
| "eval_loss": 1.7185667753219604, | |
| "eval_positive_losses": 9.667731285095215, | |
| "eval_rewards/accuracies": 0.6420000195503235, | |
| "eval_rewards/chosen": -0.018302178010344505, | |
| "eval_rewards/margins": 0.1243075579404831, | |
| "eval_rewards/margins_max": 0.6361650228500366, | |
| "eval_rewards/margins_min": -0.3433184325695038, | |
| "eval_rewards/margins_std": 0.32774004340171814, | |
| "eval_rewards/rejected": -0.14260973036289215, | |
| "eval_runtime": 428.2243, | |
| "eval_samples_per_second": 4.67, | |
| "eval_steps_per_second": 0.292, | |
| "step": 200 | |
| }, | |
| { | |
| "dpo_losses": 0.4354400634765625, | |
| "epoch": 0.59, | |
| "grad_norm": 23.522369776083625, | |
| "learning_rate": 2.1442129043167877e-06, | |
| "logits/chosen": -2.6434009075164795, | |
| "logits/rejected": -2.6138339042663574, | |
| "logps/chosen": -286.7272033691406, | |
| "logps/rejected": -291.8896789550781, | |
| "loss": 0.513, | |
| "positive_losses": 0.665066123008728, | |
| "rewards/accuracies": 0.9624999761581421, | |
| "rewards/chosen": 0.33457106351852417, | |
| "rewards/margins": 0.6892200708389282, | |
| "rewards/margins_max": 1.2269551753997803, | |
| "rewards/margins_min": 0.18099449574947357, | |
| "rewards/margins_std": 0.46556130051612854, | |
| "rewards/rejected": -0.35464900732040405, | |
| "step": 210 | |
| }, | |
| { | |
| "dpo_losses": 0.4387238025665283, | |
| "epoch": 0.62, | |
| "grad_norm": 11.92404423048434, | |
| "learning_rate": 1.9026303129961049e-06, | |
| "logits/chosen": -2.7612462043762207, | |
| "logits/rejected": -2.664234161376953, | |
| "logps/chosen": -319.7461853027344, | |
| "logps/rejected": -306.0053405761719, | |
| "loss": 0.5894, | |
| "positive_losses": 1.1452913284301758, | |
| "rewards/accuracies": 0.9750000238418579, | |
| "rewards/chosen": 0.33710065484046936, | |
| "rewards/margins": 0.6538791656494141, | |
| "rewards/margins_max": 1.1509373188018799, | |
| "rewards/margins_min": 0.19225715100765228, | |
| "rewards/margins_std": 0.4403897225856781, | |
| "rewards/rejected": -0.3167785704135895, | |
| "step": 220 | |
| }, | |
| { | |
| "dpo_losses": 0.44511428475379944, | |
| "epoch": 0.65, | |
| "grad_norm": 2.419282473127918, | |
| "learning_rate": 1.66683681459314e-06, | |
| "logits/chosen": -2.773876428604126, | |
| "logits/rejected": -2.67607045173645, | |
| "logps/chosen": -339.04718017578125, | |
| "logps/rejected": -293.1225891113281, | |
| "loss": 0.4763, | |
| "positive_losses": 0.6133368611335754, | |
| "rewards/accuracies": 0.987500011920929, | |
| "rewards/chosen": 0.32628515362739563, | |
| "rewards/margins": 0.6365767121315002, | |
| "rewards/margins_max": 1.125410795211792, | |
| "rewards/margins_min": 0.21782192587852478, | |
| "rewards/margins_std": 0.4051855504512787, | |
| "rewards/rejected": -0.3102915287017822, | |
| "step": 230 | |
| }, | |
| { | |
| "dpo_losses": 0.4544529318809509, | |
| "epoch": 0.68, | |
| "grad_norm": 13.447116267552904, | |
| "learning_rate": 1.4391174773015836e-06, | |
| "logits/chosen": -2.7197587490081787, | |
| "logits/rejected": -2.649749279022217, | |
| "logps/chosen": -302.6105041503906, | |
| "logps/rejected": -321.8402404785156, | |
| "loss": 0.692, | |
| "positive_losses": 2.48455810546875, | |
| "rewards/accuracies": 0.9375, | |
| "rewards/chosen": 0.22186538577079773, | |
| "rewards/margins": 0.6085190773010254, | |
| "rewards/margins_max": 1.1415433883666992, | |
| "rewards/margins_min": 0.23370866477489471, | |
| "rewards/margins_std": 0.41311854124069214, | |
| "rewards/rejected": -0.38665369153022766, | |
| "step": 240 | |
| }, | |
| { | |
| "dpo_losses": 0.45861634612083435, | |
| "epoch": 0.7, | |
| "grad_norm": 5.111403689556549, | |
| "learning_rate": 1.2216791228457778e-06, | |
| "logits/chosen": -2.716823101043701, | |
| "logits/rejected": -2.640800952911377, | |
| "logps/chosen": -280.11114501953125, | |
| "logps/rejected": -281.67138671875, | |
| "loss": 0.4992, | |
| "positive_losses": 0.6084854006767273, | |
| "rewards/accuracies": 0.987500011920929, | |
| "rewards/chosen": 0.31169968843460083, | |
| "rewards/margins": 0.6179708242416382, | |
| "rewards/margins_max": 1.2185614109039307, | |
| "rewards/margins_min": 0.1615341305732727, | |
| "rewards/margins_std": 0.4740964472293854, | |
| "rewards/rejected": -0.30627113580703735, | |
| "step": 250 | |
| }, | |
| { | |
| "dpo_losses": 0.4628082811832428, | |
| "epoch": 0.73, | |
| "grad_norm": 2.699692592075128, | |
| "learning_rate": 1.0166289402331391e-06, | |
| "logits/chosen": -2.7728962898254395, | |
| "logits/rejected": -2.684753894805908, | |
| "logps/chosen": -263.36126708984375, | |
| "logps/rejected": -289.21661376953125, | |
| "loss": 0.5624, | |
| "positive_losses": 0.9304378628730774, | |
| "rewards/accuracies": 0.9624999761581421, | |
| "rewards/chosen": 0.28732261061668396, | |
| "rewards/margins": 0.5901791453361511, | |
| "rewards/margins_max": 1.102694034576416, | |
| "rewards/margins_min": 0.17682021856307983, | |
| "rewards/margins_std": 0.4229150712490082, | |
| "rewards/rejected": -0.30285659432411194, | |
| "step": 260 | |
| }, | |
| { | |
| "dpo_losses": 0.4588424265384674, | |
| "epoch": 0.76, | |
| "grad_norm": 2.4735784513371377, | |
| "learning_rate": 8.259540650444736e-07, | |
| "logits/chosen": -2.717153787612915, | |
| "logits/rejected": -2.662932872772217, | |
| "logps/chosen": -278.75482177734375, | |
| "logps/rejected": -291.56866455078125, | |
| "loss": 0.5853, | |
| "positive_losses": 0.9098857641220093, | |
| "rewards/accuracies": 0.949999988079071, | |
| "rewards/chosen": 0.30360764265060425, | |
| "rewards/margins": 0.5942984223365784, | |
| "rewards/margins_max": 1.0322821140289307, | |
| "rewards/margins_min": 0.21275146305561066, | |
| "rewards/margins_std": 0.36198341846466064, | |
| "rewards/rejected": -0.2906908392906189, | |
| "step": 270 | |
| }, | |
| { | |
| "dpo_losses": 0.4629085958003998, | |
| "epoch": 0.79, | |
| "grad_norm": 13.451546074592132, | |
| "learning_rate": 6.515023221586722e-07, | |
| "logits/chosen": -2.6962451934814453, | |
| "logits/rejected": -2.6575076580047607, | |
| "logps/chosen": -274.9664001464844, | |
| "logps/rejected": -304.9722595214844, | |
| "loss": 0.5625, | |
| "positive_losses": 1.4465850591659546, | |
| "rewards/accuracies": 0.9750000238418579, | |
| "rewards/chosen": 0.2849060893058777, | |
| "rewards/margins": 0.60865718126297, | |
| "rewards/margins_max": 1.1329301595687866, | |
| "rewards/margins_min": 0.1755952537059784, | |
| "rewards/margins_std": 0.4414794445037842, | |
| "rewards/rejected": -0.3237510919570923, | |
| "step": 280 | |
| }, | |
| { | |
| "dpo_losses": 0.47258663177490234, | |
| "epoch": 0.82, | |
| "grad_norm": 2.654477953260434, | |
| "learning_rate": 4.949643185335288e-07, | |
| "logits/chosen": -2.707307815551758, | |
| "logits/rejected": -2.652792453765869, | |
| "logps/chosen": -259.1030578613281, | |
| "logps/rejected": -292.6324462890625, | |
| "loss": 0.6149, | |
| "positive_losses": 1.7202523946762085, | |
| "rewards/accuracies": 0.949999988079071, | |
| "rewards/chosen": 0.27813172340393066, | |
| "rewards/margins": 0.5642735958099365, | |
| "rewards/margins_max": 1.0385398864746094, | |
| "rewards/margins_min": 0.12702254951000214, | |
| "rewards/margins_std": 0.4158683717250824, | |
| "rewards/rejected": -0.28614187240600586, | |
| "step": 290 | |
| }, | |
| { | |
| "dpo_losses": 0.4324049949645996, | |
| "epoch": 0.85, | |
| "grad_norm": 11.591501845708454, | |
| "learning_rate": 3.578570595810274e-07, | |
| "logits/chosen": -2.7821717262268066, | |
| "logits/rejected": -2.6995315551757812, | |
| "logps/chosen": -309.7518310546875, | |
| "logps/rejected": -320.70916748046875, | |
| "loss": 0.4748, | |
| "positive_losses": 0.8444260358810425, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.3676120638847351, | |
| "rewards/margins": 0.6803697943687439, | |
| "rewards/margins_max": 1.199285864830017, | |
| "rewards/margins_min": 0.21351738274097443, | |
| "rewards/margins_std": 0.4415613114833832, | |
| "rewards/rejected": -0.312757670879364, | |
| "step": 300 | |
| }, | |
| { | |
| "epoch": 0.85, | |
| "eval_dpo_losses": 0.6448404788970947, | |
| "eval_logits/chosen": -2.715327739715576, | |
| "eval_logits/rejected": -2.6732916831970215, | |
| "eval_logps/chosen": -284.4527587890625, | |
| "eval_logps/rejected": -271.32244873046875, | |
| "eval_loss": 1.6048117876052856, | |
| "eval_positive_losses": 8.706162452697754, | |
| "eval_rewards/accuracies": 0.6470000147819519, | |
| "eval_rewards/chosen": 0.0014067561132833362, | |
| "eval_rewards/margins": 0.12884218990802765, | |
| "eval_rewards/margins_max": 0.6374967098236084, | |
| "eval_rewards/margins_min": -0.34605804085731506, | |
| "eval_rewards/margins_std": 0.3295030891895294, | |
| "eval_rewards/rejected": -0.12743544578552246, | |
| "eval_runtime": 428.2498, | |
| "eval_samples_per_second": 4.67, | |
| "eval_steps_per_second": 0.292, | |
| "step": 300 | |
| }, | |
| { | |
| "dpo_losses": 0.45941466093063354, | |
| "epoch": 0.87, | |
| "grad_norm": 2.6085680781835205, | |
| "learning_rate": 2.4150924791035037e-07, | |
| "logits/chosen": -2.774445056915283, | |
| "logits/rejected": -2.673360824584961, | |
| "logps/chosen": -267.74237060546875, | |
| "logps/rejected": -243.88473510742188, | |
| "loss": 0.5697, | |
| "positive_losses": 1.3653801679611206, | |
| "rewards/accuracies": 0.987500011920929, | |
| "rewards/chosen": 0.30073457956314087, | |
| "rewards/margins": 0.5973426699638367, | |
| "rewards/margins_max": 1.1060882806777954, | |
| "rewards/margins_min": 0.18351522088050842, | |
| "rewards/margins_std": 0.4086340069770813, | |
| "rewards/rejected": -0.2966081500053406, | |
| "step": 310 | |
| }, | |
| { | |
| "dpo_losses": 0.45310109853744507, | |
| "epoch": 0.9, | |
| "grad_norm": 10.060071948421735, | |
| "learning_rate": 1.4704840690808658e-07, | |
| "logits/chosen": -2.738978385925293, | |
| "logits/rejected": -2.680860757827759, | |
| "logps/chosen": -279.5138854980469, | |
| "logps/rejected": -293.9893493652344, | |
| "loss": 0.5692, | |
| "positive_losses": 1.6892318725585938, | |
| "rewards/accuracies": 0.9624999761581421, | |
| "rewards/chosen": 0.2875928282737732, | |
| "rewards/margins": 0.6207860708236694, | |
| "rewards/margins_max": 1.124011754989624, | |
| "rewards/margins_min": 0.14557920396327972, | |
| "rewards/margins_std": 0.44626301527023315, | |
| "rewards/rejected": -0.33319321274757385, | |
| "step": 320 | |
| }, | |
| { | |
| "dpo_losses": 0.42673492431640625, | |
| "epoch": 0.93, | |
| "grad_norm": 9.476085880429812, | |
| "learning_rate": 7.538995394063996e-08, | |
| "logits/chosen": -2.8187005519866943, | |
| "logits/rejected": -2.7311813831329346, | |
| "logps/chosen": -318.88360595703125, | |
| "logps/rejected": -302.66058349609375, | |
| "loss": 0.5314, | |
| "positive_losses": 0.5069873929023743, | |
| "rewards/accuracies": 0.987500011920929, | |
| "rewards/chosen": 0.35436224937438965, | |
| "rewards/margins": 0.7115713953971863, | |
| "rewards/margins_max": 1.23550546169281, | |
| "rewards/margins_min": 0.2139424830675125, | |
| "rewards/margins_std": 0.4558965563774109, | |
| "rewards/rejected": -0.35720914602279663, | |
| "step": 330 | |
| }, | |
| { | |
| "dpo_losses": 0.4437997341156006, | |
| "epoch": 0.96, | |
| "grad_norm": 2.682118994824555, | |
| "learning_rate": 2.722832907015971e-08, | |
| "logits/chosen": -2.6981847286224365, | |
| "logits/rejected": -2.6440398693084717, | |
| "logps/chosen": -266.6497802734375, | |
| "logps/rejected": -282.98199462890625, | |
| "loss": 0.5024, | |
| "positive_losses": 0.9627658724784851, | |
| "rewards/accuracies": 0.9750000238418579, | |
| "rewards/chosen": 0.3319571018218994, | |
| "rewards/margins": 0.6500804424285889, | |
| "rewards/margins_max": 1.2494922876358032, | |
| "rewards/margins_min": 0.25120097398757935, | |
| "rewards/margins_std": 0.4507668614387512, | |
| "rewards/rejected": -0.31812337040901184, | |
| "step": 340 | |
| }, | |
| { | |
| "dpo_losses": 0.4518283009529114, | |
| "epoch": 0.99, | |
| "grad_norm": 5.762126574549782, | |
| "learning_rate": 3.030265255329623e-09, | |
| "logits/chosen": -2.6820361614227295, | |
| "logits/rejected": -2.6376953125, | |
| "logps/chosen": -285.1527404785156, | |
| "logps/rejected": -317.6675720214844, | |
| "loss": 0.5059, | |
| "positive_losses": 0.9290813207626343, | |
| "rewards/accuracies": 0.987500011920929, | |
| "rewards/chosen": 0.2980636656284332, | |
| "rewards/margins": 0.6102195978164673, | |
| "rewards/margins_max": 1.0686355829238892, | |
| "rewards/margins_min": 0.20541608333587646, | |
| "rewards/margins_std": 0.38572338223457336, | |
| "rewards/rejected": -0.31215590238571167, | |
| "step": 350 | |
| }, | |
| { | |
| "epoch": 1.0, | |
| "step": 355, | |
| "total_flos": 0.0, | |
| "train_loss": 0.5743894765074824, | |
| "train_runtime": 4311.1014, | |
| "train_samples_per_second": 1.317, | |
| "train_steps_per_second": 0.082 | |
| } | |
| ], | |
| "logging_steps": 10, | |
| "max_steps": 355, | |
| "num_input_tokens_seen": 0, | |
| "num_train_epochs": 1, | |
| "save_steps": 100, | |
| "total_flos": 0.0, | |
| "train_batch_size": 4, | |
| "trial_name": null, | |
| "trial_params": null | |
| } | |