{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 4.0,
  "eval_steps": 100,
  "global_step": 1540,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.0,
      "learning_rate": 1.282051282051282e-07,
      "logits/chosen": -1.7278180122375488,
      "logits/rejected": -1.7377450466156006,
      "logps/chosen": -29.553977966308594,
      "logps/rejected": -42.813133239746094,
      "loss": 0.5,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 1
    },
    {
      "epoch": 0.03,
      "learning_rate": 1.282051282051282e-06,
      "logits/chosen": -1.8667322397232056,
      "logits/rejected": -1.8710299730300903,
      "logps/chosen": -36.98433303833008,
      "logps/rejected": -33.65640640258789,
      "loss": 0.4948,
      "rewards/accuracies": 0.5277777910232544,
      "rewards/chosen": 0.015585740096867085,
      "rewards/margins": 0.034019216895103455,
      "rewards/rejected": -0.018433474004268646,
      "step": 10
    },
    {
      "epoch": 0.05,
      "learning_rate": 2.564102564102564e-06,
      "logits/chosen": -1.9972655773162842,
      "logits/rejected": -1.9999065399169922,
      "logps/chosen": -29.6416072845459,
      "logps/rejected": -29.046981811523438,
      "loss": 0.5013,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.00041136163054034114,
      "rewards/margins": -0.011113673448562622,
      "rewards/rejected": 0.011525033973157406,
      "step": 20
    },
    {
      "epoch": 0.08,
      "learning_rate": 3.846153846153847e-06,
      "logits/chosen": -1.9205191135406494,
      "logits/rejected": -1.9178133010864258,
      "logps/chosen": -31.385412216186523,
      "logps/rejected": -33.23365020751953,
      "loss": 0.4942,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": 0.02147640287876129,
      "rewards/margins": 0.03211076930165291,
      "rewards/rejected": -0.010634368285536766,
      "step": 30
    },
    {
      "epoch": 0.1,
      "learning_rate": 4.999896948438434e-06,
      "logits/chosen": -2.017784833908081,
      "logits/rejected": -2.009059429168701,
      "logps/chosen": -32.547088623046875,
      "logps/rejected": -32.473243713378906,
      "loss": 0.5024,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": 0.020800117403268814,
      "rewards/margins": -0.006797379348427057,
      "rewards/rejected": 0.027597496286034584,
      "step": 40
    },
    {
      "epoch": 0.13,
      "learning_rate": 4.987541037542187e-06,
      "logits/chosen": -1.863996148109436,
      "logits/rejected": -1.8532311916351318,
      "logps/chosen": -33.490657806396484,
      "logps/rejected": -35.415367126464844,
      "loss": 0.4957,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.046776313334703445,
      "rewards/margins": 0.02089320495724678,
      "rewards/rejected": 0.025883108377456665,
      "step": 50
    },
    {
      "epoch": 0.16,
      "learning_rate": 4.954691471941119e-06,
      "logits/chosen": -1.9439821243286133,
      "logits/rejected": -1.9459091424942017,
      "logps/chosen": -32.50756072998047,
      "logps/rejected": -33.15720748901367,
      "loss": 0.489,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.06483479589223862,
      "rewards/margins": 0.05491489917039871,
      "rewards/rejected": 0.009919902309775352,
      "step": 60
    },
    {
      "epoch": 0.18,
      "learning_rate": 4.901618883413549e-06,
      "logits/chosen": -2.075462579727173,
      "logits/rejected": -2.0804481506347656,
      "logps/chosen": -33.86588668823242,
      "logps/rejected": -36.501365661621094,
      "loss": 0.491,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": 0.08581922948360443,
      "rewards/margins": 0.038563162088394165,
      "rewards/rejected": 0.04725607484579086,
      "step": 70
    },
    {
      "epoch": 0.21,
      "learning_rate": 4.828760511501322e-06,
      "logits/chosen": -1.9374949932098389,
      "logits/rejected": -1.9405946731567383,
      "logps/chosen": -34.200557708740234,
      "logps/rejected": -34.543209075927734,
      "loss": 0.4719,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.15907494723796844,
      "rewards/margins": 0.1246359720826149,
      "rewards/rejected": 0.03443896025419235,
      "step": 80
    },
    {
      "epoch": 0.23,
      "learning_rate": 4.7367166013034295e-06,
      "logits/chosen": -1.9453551769256592,
      "logits/rejected": -1.9498746395111084,
      "logps/chosen": -32.2825927734375,
      "logps/rejected": -32.25676727294922,
      "loss": 0.4858,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": 0.12937995791435242,
      "rewards/margins": 0.05788033455610275,
      "rewards/rejected": 0.07149962335824966,
      "step": 90
    },
    {
      "epoch": 0.26,
      "learning_rate": 4.626245458345211e-06,
      "logits/chosen": -2.04329776763916,
      "logits/rejected": -2.0412919521331787,
      "logps/chosen": -32.015525817871094,
      "logps/rejected": -31.169103622436523,
      "loss": 0.4799,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": 0.15771625936031342,
      "rewards/margins": 0.08974156528711319,
      "rewards/rejected": 0.06797470152378082,
      "step": 100
    },
    {
      "epoch": 0.26,
      "eval_logits/chosen": -2.238145351409912,
      "eval_logits/rejected": -2.2333030700683594,
      "eval_logps/chosen": -33.86069107055664,
      "eval_logps/rejected": -37.356990814208984,
      "eval_loss": 0.49784979224205017,
      "eval_rewards/accuracies": 0.5340532064437866,
      "eval_rewards/chosen": 0.12170296907424927,
      "eval_rewards/margins": 0.009964452125132084,
      "eval_rewards/rejected": 0.11173851788043976,
      "eval_runtime": 145.9039,
      "eval_samples_per_second": 2.351,
      "eval_steps_per_second": 0.295,
      "step": 100
    },
    {
      "epoch": 0.29,
      "learning_rate": 4.498257201263691e-06,
      "logits/chosen": -1.9996016025543213,
      "logits/rejected": -1.9972642660140991,
      "logps/chosen": -32.95330810546875,
      "logps/rejected": -33.88172912597656,
      "loss": 0.4742,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": 0.20335713028907776,
      "rewards/margins": 0.10000836849212646,
      "rewards/rejected": 0.10334876924753189,
      "step": 110
    },
    {
      "epoch": 0.31,
      "learning_rate": 4.353806263777678e-06,
      "logits/chosen": -2.0096044540405273,
      "logits/rejected": -2.0012660026550293,
      "logps/chosen": -32.153419494628906,
      "logps/rejected": -31.961782455444336,
      "loss": 0.4848,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.20403961837291718,
      "rewards/margins": 0.06423385441303253,
      "rewards/rejected": 0.13980577886104584,
      "step": 120
    },
    {
      "epoch": 0.34,
      "learning_rate": 4.1940827077152755e-06,
      "logits/chosen": -2.038412570953369,
      "logits/rejected": -2.0304617881774902,
      "logps/chosen": -30.165945053100586,
      "logps/rejected": -31.880346298217773,
      "loss": 0.4787,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.21683910489082336,
      "rewards/margins": 0.0959777683019638,
      "rewards/rejected": 0.12086131423711777,
      "step": 130
    },
    {
      "epoch": 0.36,
      "learning_rate": 4.0204024186666215e-06,
      "logits/chosen": -1.9683482646942139,
      "logits/rejected": -1.9784959554672241,
      "logps/chosen": -31.041671752929688,
      "logps/rejected": -32.388893127441406,
      "loss": 0.4621,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": 0.264592707157135,
      "rewards/margins": 0.16240297257900238,
      "rewards/rejected": 0.10218973457813263,
      "step": 140
    },
    {
      "epoch": 0.39,
      "learning_rate": 3.834196265035119e-06,
      "logits/chosen": -1.8799880743026733,
      "logits/rejected": -1.8811382055282593,
      "logps/chosen": -33.686256408691406,
      "logps/rejected": -34.564491271972656,
      "loss": 0.4503,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.3545047640800476,
      "rewards/margins": 0.21957096457481384,
      "rewards/rejected": 0.134933739900589,
      "step": 150
    },
    {
      "epoch": 0.42,
      "learning_rate": 3.636998309800573e-06,
      "logits/chosen": -1.931627869606018,
      "logits/rejected": -1.9282808303833008,
      "logps/chosen": -35.77287292480469,
      "logps/rejected": -32.49742889404297,
      "loss": 0.4707,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": 0.27916350960731506,
      "rewards/margins": 0.12143567949533463,
      "rewards/rejected": 0.15772780776023865,
      "step": 160
    },
    {
      "epoch": 0.44,
      "learning_rate": 3.4304331721118078e-06,
      "logits/chosen": -2.0332443714141846,
      "logits/rejected": -2.0259475708007812,
      "logps/chosen": -33.251670837402344,
      "logps/rejected": -31.17826271057129,
      "loss": 0.439,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": 0.382335364818573,
      "rewards/margins": 0.2608257234096527,
      "rewards/rejected": 0.1215096265077591,
      "step": 170
    },
    {
      "epoch": 0.47,
      "learning_rate": 3.2162026428305436e-06,
      "logits/chosen": -2.040196657180786,
      "logits/rejected": -2.045389413833618,
      "logps/chosen": -31.95781898498535,
      "logps/rejected": -32.18687057495117,
      "loss": 0.4516,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.4108788073062897,
      "rewards/margins": 0.20208871364593506,
      "rewards/rejected": 0.20879006385803223,
      "step": 180
    },
    {
      "epoch": 0.49,
      "learning_rate": 2.996071664294641e-06,
      "logits/chosen": -2.040775775909424,
      "logits/rejected": -2.038043737411499,
      "logps/chosen": -31.056228637695312,
      "logps/rejected": -31.10147476196289,
      "loss": 0.4647,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": 0.30290743708610535,
      "rewards/margins": 0.1511872559785843,
      "rewards/rejected": 0.15172019600868225,
      "step": 190
    },
    {
      "epoch": 0.52,
      "learning_rate": 2.7718537898066833e-06,
      "logits/chosen": -1.9112863540649414,
      "logits/rejected": -1.9159523248672485,
      "logps/chosen": -31.08163070678711,
      "logps/rejected": -32.598487854003906,
      "loss": 0.4453,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": 0.3672560155391693,
      "rewards/margins": 0.2325470894575119,
      "rewards/rejected": 0.1347089260816574,
      "step": 200
    },
    {
      "epoch": 0.52,
      "eval_logits/chosen": -2.234708309173584,
      "eval_logits/rejected": -2.229902505874634,
      "eval_logps/chosen": -33.76850891113281,
      "eval_logps/rejected": -37.29365921020508,
      "eval_loss": 0.49275800585746765,
      "eval_rewards/accuracies": 0.5369601845741272,
      "eval_rewards/chosen": 0.18622951209545135,
      "eval_rewards/margins": 0.030155813321471214,
      "eval_rewards/rejected": 0.1560736745595932,
      "eval_runtime": 145.5287,
      "eval_samples_per_second": 2.357,
      "eval_steps_per_second": 0.295,
      "step": 200
    },
    {
      "epoch": 0.55,
      "learning_rate": 2.5453962426402006e-06,
      "logits/chosen": -2.022939443588257,
      "logits/rejected": -2.033546209335327,
      "logps/chosen": -31.5457706451416,
      "logps/rejected": -33.67595672607422,
      "loss": 0.4628,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.2978750169277191,
      "rewards/margins": 0.16715146601200104,
      "rewards/rejected": 0.13072356581687927,
      "step": 210
    },
    {
      "epoch": 0.57,
      "learning_rate": 2.3185646976551794e-06,
      "logits/chosen": -1.9158331155776978,
      "logits/rejected": -1.9305419921875,
      "logps/chosen": -29.57406997680664,
      "logps/rejected": -31.42013168334961,
      "loss": 0.4363,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.37622833251953125,
      "rewards/margins": 0.2795966863632202,
      "rewards/rejected": 0.09663165360689163,
      "step": 220
    },
    {
      "epoch": 0.6,
      "learning_rate": 2.0932279108998323e-06,
      "logits/chosen": -1.9733517169952393,
      "logits/rejected": -1.9773433208465576,
      "logps/chosen": -32.83649444580078,
      "logps/rejected": -31.40326499938965,
      "loss": 0.4311,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.40668267011642456,
      "rewards/margins": 0.304456502199173,
      "rewards/rejected": 0.10222617536783218,
      "step": 230
    },
    {
      "epoch": 0.62,
      "learning_rate": 1.8712423238279358e-06,
      "logits/chosen": -1.9713729619979858,
      "logits/rejected": -1.9496545791625977,
      "logps/chosen": -33.5602912902832,
      "logps/rejected": -34.85026550292969,
      "loss": 0.4227,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.41541916131973267,
      "rewards/margins": 0.3450462222099304,
      "rewards/rejected": 0.07037289440631866,
      "step": 240
    },
    {
      "epoch": 0.65,
      "learning_rate": 1.6544367689701824e-06,
      "logits/chosen": -2.012058734893799,
      "logits/rejected": -2.0087602138519287,
      "logps/chosen": -32.46921157836914,
      "logps/rejected": -36.012245178222656,
      "loss": 0.4613,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": 0.31910061836242676,
      "rewards/margins": 0.17022350430488586,
      "rewards/rejected": 0.1488770991563797,
      "step": 250
    },
    {
      "epoch": 0.68,
      "learning_rate": 1.4445974030621963e-06,
      "logits/chosen": -1.878584861755371,
      "logits/rejected": -1.876153588294983,
      "logps/chosen": -33.71690368652344,
      "logps/rejected": -35.26210403442383,
      "loss": 0.4631,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.33004888892173767,
      "rewards/margins": 0.1602855622768402,
      "rewards/rejected": 0.16976332664489746,
      "step": 260
    },
    {
      "epoch": 0.7,
      "learning_rate": 1.243452991757889e-06,
      "logits/chosen": -1.863705039024353,
      "logits/rejected": -1.8611875772476196,
      "logps/chosen": -33.94118881225586,
      "logps/rejected": -31.579448699951172,
      "loss": 0.4582,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": 0.3194878101348877,
      "rewards/margins": 0.18517015874385834,
      "rewards/rejected": 0.13431766629219055,
      "step": 270
    },
    {
      "epoch": 0.73,
      "learning_rate": 1.0526606671603523e-06,
      "logits/chosen": -1.96773362159729,
      "logits/rejected": -1.957313895225525,
      "logps/chosen": -34.72475814819336,
      "logps/rejected": -31.634963989257812,
      "loss": 0.4297,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.4423200190067291,
      "rewards/margins": 0.298100084066391,
      "rewards/rejected": 0.14421990513801575,
      "step": 280
    },
    {
      "epoch": 0.75,
      "learning_rate": 8.737922755071455e-07,
      "logits/chosen": -2.0632526874542236,
      "logits/rejected": -2.0484511852264404,
      "logps/chosen": -30.412399291992188,
      "logps/rejected": -32.339237213134766,
      "loss": 0.4682,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.3582096993923187,
      "rewards/margins": 0.13939058780670166,
      "rewards/rejected": 0.21881911158561707,
      "step": 290
    },
    {
      "epoch": 0.78,
      "learning_rate": 7.08321427484816e-07,
      "logits/chosen": -1.9345197677612305,
      "logits/rejected": -1.9320701360702515,
      "logps/chosen": -32.11776351928711,
      "logps/rejected": -30.692516326904297,
      "loss": 0.3947,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": 0.5665432810783386,
      "rewards/margins": 0.4795381426811218,
      "rewards/rejected": 0.08700509369373322,
      "step": 300
    },
    {
      "epoch": 0.78,
      "eval_logits/chosen": -2.2321932315826416,
      "eval_logits/rejected": -2.227381944656372,
      "eval_logps/chosen": -33.75507736206055,
      "eval_logps/rejected": -37.28935623168945,
      "eval_loss": 0.49099090695381165,
      "eval_rewards/accuracies": 0.5564784407615662,
      "eval_rewards/chosen": 0.1956319361925125,
      "eval_rewards/margins": 0.03654861077666283,
      "eval_rewards/rejected": 0.15908333659172058,
      "eval_runtime": 145.678,
      "eval_samples_per_second": 2.355,
      "eval_steps_per_second": 0.295,
      "step": 300
    },
    {
      "epoch": 0.81,
      "grad_norm": 5.65625,
      "learning_rate": 4.84533120650964e-06,
      "logits/chosen": -2.069279909133911,
      "logits/rejected": -2.0565598011016846,
      "logps/chosen": -31.766155242919922,
      "logps/rejected": -32.60447311401367,
      "loss": 0.3879,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.5335645079612732,
      "rewards/margins": 0.5079580545425415,
      "rewards/rejected": 0.025606412440538406,
      "step": 310
    },
    {
      "epoch": 0.83,
      "grad_norm": 6.9375,
      "learning_rate": 4.825108134172131e-06,
      "logits/chosen": -1.9792858362197876,
      "logits/rejected": -1.9706569910049438,
      "logps/chosen": -31.41892433166504,
      "logps/rejected": -30.155033111572266,
      "loss": 0.3743,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": 0.6552442312240601,
      "rewards/margins": 0.5947929620742798,
      "rewards/rejected": 0.060451291501522064,
      "step": 320
    },
    {
      "epoch": 0.86,
      "grad_norm": 8.1875,
      "learning_rate": 4.80369052967602e-06,
      "logits/chosen": -1.9177625179290771,
      "logits/rejected": -1.9297670125961304,
      "logps/chosen": -29.455638885498047,
      "logps/rejected": -33.40266036987305,
      "loss": 0.3501,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": 0.7165408730506897,
      "rewards/margins": 0.7168257832527161,
      "rewards/rejected": -0.00028486884548328817,
      "step": 330
    },
    {
      "epoch": 0.88,
      "grad_norm": 9.25,
      "learning_rate": 4.781089396387968e-06,
      "logits/chosen": -1.880617380142212,
      "logits/rejected": -1.8715527057647705,
      "logps/chosen": -33.60810089111328,
      "logps/rejected": -35.90662384033203,
      "loss": 0.3405,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 0.7635797262191772,
      "rewards/margins": 0.7985055446624756,
      "rewards/rejected": -0.03492575138807297,
      "step": 340
    },
    {
      "epoch": 0.91,
      "grad_norm": 5.1875,
      "learning_rate": 4.757316345716554e-06,
      "logits/chosen": -1.9337537288665771,
      "logits/rejected": -1.9343976974487305,
      "logps/chosen": -33.176937103271484,
      "logps/rejected": -33.779075622558594,
      "loss": 0.3349,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": 0.8598405718803406,
      "rewards/margins": 0.8109108805656433,
      "rewards/rejected": 0.048929743468761444,
      "step": 350
    },
    {
      "epoch": 0.94,
      "grad_norm": 6.3125,
      "learning_rate": 4.73238359114687e-06,
      "logits/chosen": -2.060123920440674,
      "logits/rejected": -2.066301107406616,
      "logps/chosen": -30.665380477905273,
      "logps/rejected": -32.581119537353516,
      "loss": 0.3792,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": 0.6355056762695312,
      "rewards/margins": 0.5744917988777161,
      "rewards/rejected": 0.061013948172330856,
      "step": 360
    },
    {
      "epoch": 0.96,
      "grad_norm": 7.78125,
      "learning_rate": 4.706303941965804e-06,
      "logits/chosen": -1.9897676706314087,
      "logits/rejected": -1.989344835281372,
      "logps/chosen": -32.362205505371094,
      "logps/rejected": -35.91802215576172,
      "loss": 0.3617,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": 0.7579163908958435,
      "rewards/margins": 0.6656854748725891,
      "rewards/rejected": 0.09223100543022156,
      "step": 370
    },
    {
      "epoch": 0.99,
      "grad_norm": 5.71875,
      "learning_rate": 4.679090796681225e-06,
      "logits/chosen": -2.022163152694702,
      "logits/rejected": -2.017575740814209,
      "logps/chosen": -29.644603729248047,
      "logps/rejected": -29.16855239868164,
      "loss": 0.3514,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.746194064617157,
      "rewards/margins": 0.714457094669342,
      "rewards/rejected": 0.03173698112368584,
      "step": 380
    },
    {
      "epoch": 1.01,
      "grad_norm": 8.0625,
      "learning_rate": 4.650758136138454e-06,
      "logits/chosen": -1.7921892404556274,
      "logits/rejected": -1.7986253499984741,
      "logps/chosen": -31.069808959960938,
      "logps/rejected": -36.217376708984375,
      "loss": 0.3052,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": 0.9211344718933105,
      "rewards/margins": 0.9904948472976685,
      "rewards/rejected": -0.06936033815145493,
      "step": 390
    },
    {
      "epoch": 1.04,
      "grad_norm": 5.84375,
      "learning_rate": 4.621320516337559e-06,
      "logits/chosen": -1.946427583694458,
      "logits/rejected": -1.9403005838394165,
      "logps/chosen": -32.449764251708984,
      "logps/rejected": -32.27871322631836,
      "loss": 0.3136,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": 0.968052864074707,
      "rewards/margins": 0.9469360113143921,
      "rewards/rejected": 0.021116863936185837,
      "step": 400
    },
    {
      "epoch": 1.04,
      "eval_logits/chosen": -2.2079687118530273,
      "eval_logits/rejected": -2.2031569480895996,
      "eval_logps/chosen": -33.62800598144531,
      "eval_logps/rejected": -37.196067810058594,
      "eval_loss": 0.48570817708969116,
      "eval_rewards/accuracies": 0.579734206199646,
      "eval_rewards/chosen": 0.2845819294452667,
      "eval_rewards/margins": 0.06019504740834236,
      "eval_rewards/rejected": 0.22438688576221466,
      "eval_runtime": 145.4274,
      "eval_samples_per_second": 2.359,
      "eval_steps_per_second": 0.296,
      "step": 400
    },
    {
      "epoch": 1.06,
      "grad_norm": 5.96875,
      "learning_rate": 4.590793060955158e-06,
      "logits/chosen": -1.9444068670272827,
      "logits/rejected": -1.951677918434143,
      "logps/chosen": -28.08418846130371,
      "logps/rejected": -29.256505966186523,
      "loss": 0.3391,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 0.7253325581550598,
      "rewards/margins": 0.836132824420929,
      "rewards/rejected": -0.11080024391412735,
      "step": 410
    },
    {
      "epoch": 1.09,
      "grad_norm": 5.4375,
      "learning_rate": 4.559191453574582e-06,
      "logits/chosen": -1.966160774230957,
      "logits/rejected": -1.965192437171936,
      "logps/chosen": -32.974876403808594,
      "logps/rejected": -30.77309226989746,
      "loss": 0.3464,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 0.8930822610855103,
      "rewards/margins": 0.774114727973938,
      "rewards/rejected": 0.11896754801273346,
      "step": 420
    },
    {
      "epoch": 1.12,
      "grad_norm": 6.09375,
      "learning_rate": 4.52653192962838e-06,
      "logits/chosen": -1.9592077732086182,
      "logits/rejected": -1.9421688318252563,
      "logps/chosen": -29.91006851196289,
      "logps/rejected": -33.13286590576172,
      "loss": 0.3285,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 0.7469960451126099,
      "rewards/margins": 0.8916017413139343,
      "rewards/rejected": -0.14460588991641998,
      "step": 430
    },
    {
      "epoch": 1.14,
      "grad_norm": 5.3125,
      "learning_rate": 4.492831268057307e-06,
      "logits/chosen": -1.989438772201538,
      "logits/rejected": -1.9913489818572998,
      "logps/chosen": -35.05497741699219,
      "logps/rejected": -34.920108795166016,
      "loss": 0.2832,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 1.0901695489883423,
      "rewards/margins": 1.1322667598724365,
      "rewards/rejected": -0.0420973002910614,
      "step": 440
    },
    {
      "epoch": 1.17,
      "grad_norm": 6.5,
      "learning_rate": 4.458106782690094e-06,
      "logits/chosen": -2.0655436515808105,
      "logits/rejected": -2.0654349327087402,
      "logps/chosen": -31.32794761657715,
      "logps/rejected": -33.56629180908203,
      "loss": 0.3287,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 0.9137799143791199,
      "rewards/margins": 0.9181026220321655,
      "rewards/rejected": -0.004322645254433155,
      "step": 450
    },
    {
      "epoch": 1.19,
      "grad_norm": 5.28125,
      "learning_rate": 4.422376313348405e-06,
      "logits/chosen": -2.0107359886169434,
      "logits/rejected": -2.0033280849456787,
      "logps/chosen": -30.83009910583496,
      "logps/rejected": -36.22407150268555,
      "loss": 0.2765,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 1.1031007766723633,
      "rewards/margins": 1.2221224308013916,
      "rewards/rejected": -0.11902163922786713,
      "step": 460
    },
    {
      "epoch": 1.22,
      "grad_norm": 6.28125,
      "learning_rate": 4.3856582166815696e-06,
      "logits/chosen": -1.9149672985076904,
      "logits/rejected": -1.9115540981292725,
      "logps/chosen": -32.367496490478516,
      "logps/rejected": -32.911216735839844,
      "loss": 0.2962,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": 1.0953609943389893,
      "rewards/margins": 1.0864982604980469,
      "rewards/rejected": 0.008862635120749474,
      "step": 470
    },
    {
      "epoch": 1.25,
      "grad_norm": 5.5,
      "learning_rate": 4.347971356735789e-06,
      "logits/chosen": -2.040391445159912,
      "logits/rejected": -2.033470630645752,
      "logps/chosen": -29.63046646118164,
      "logps/rejected": -32.03357696533203,
      "loss": 0.3336,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.8386493921279907,
      "rewards/margins": 0.8667130470275879,
      "rewards/rejected": -0.028063708916306496,
      "step": 480
    },
    {
      "epoch": 1.27,
      "grad_norm": 5.59375,
      "learning_rate": 4.309335095262675e-06,
      "logits/chosen": -1.9858951568603516,
      "logits/rejected": -1.987535834312439,
      "logps/chosen": -33.76915740966797,
      "logps/rejected": -33.62469482421875,
      "loss": 0.2834,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 1.108822226524353,
      "rewards/margins": 1.0970509052276611,
      "rewards/rejected": 0.011771360412240028,
      "step": 490
    },
    {
      "epoch": 1.3,
      "grad_norm": 6.375,
      "learning_rate": 4.269769281772082e-06,
      "logits/chosen": -1.8744853734970093,
      "logits/rejected": -1.8722003698349,
      "logps/chosen": -31.54471206665039,
      "logps/rejected": -36.603973388671875,
      "loss": 0.2784,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 1.1393038034439087,
      "rewards/margins": 1.2370599508285522,
      "rewards/rejected": -0.09775615483522415,
      "step": 500
    },
    {
      "epoch": 1.3,
      "eval_logits/chosen": -2.2098300457000732,
      "eval_logits/rejected": -2.205024480819702,
      "eval_logps/chosen": -33.61191177368164,
      "eval_logps/rejected": -37.15669250488281,
      "eval_loss": 0.48908427357673645,
      "eval_rewards/accuracies": 0.5220099687576294,
      "eval_rewards/chosen": 0.29585084319114685,
      "eval_rewards/margins": 0.04390101507306099,
      "eval_rewards/rejected": 0.25194987654685974,
      "eval_runtime": 145.3523,
      "eval_samples_per_second": 2.36,
      "eval_steps_per_second": 0.296,
      "step": 500
    },
    {
      "epoch": 1.32,
      "grad_norm": 4.75,
      "learning_rate": 4.22929424333435e-06,
      "logits/chosen": -1.978362798690796,
      "logits/rejected": -1.98308527469635,
      "logps/chosen": -31.69674301147461,
      "logps/rejected": -31.60154151916504,
      "loss": 0.2762,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": 1.2324289083480835,
      "rewards/margins": 1.2012327909469604,
      "rewards/rejected": 0.03119627758860588,
      "step": 510
    },
    {
      "epoch": 1.35,
      "grad_norm": 5.5625,
      "learning_rate": 4.1879307741372085e-06,
      "logits/chosen": -2.0070841312408447,
      "logits/rejected": -2.01804256439209,
      "logps/chosen": -29.7496280670166,
      "logps/rejected": -31.527536392211914,
      "loss": 0.2821,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 1.148550271987915,
      "rewards/margins": 1.16244375705719,
      "rewards/rejected": -0.013893580064177513,
      "step": 520
    },
    {
      "epoch": 1.38,
      "grad_norm": 4.09375,
      "learning_rate": 4.145700124802693e-06,
      "logits/chosen": -1.9333133697509766,
      "logits/rejected": -1.9299663305282593,
      "logps/chosen": -30.964946746826172,
      "logps/rejected": -32.423377990722656,
      "loss": 0.2962,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 1.0188950300216675,
      "rewards/margins": 1.061314344406128,
      "rewards/rejected": -0.0424191839993,
      "step": 530
    },
    {
      "epoch": 1.4,
      "grad_norm": 6.15625,
      "learning_rate": 4.102623991469562e-06,
      "logits/chosen": -1.7982631921768188,
      "logits/rejected": -1.8074582815170288,
      "logps/chosen": -30.847604751586914,
      "logps/rejected": -31.964679718017578,
      "loss": 0.2773,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 1.205248475074768,
      "rewards/margins": 1.2813327312469482,
      "rewards/rejected": -0.07608427852392197,
      "step": 540
    },
    {
      "epoch": 1.43,
      "grad_norm": 5.40625,
      "learning_rate": 4.058724504646834e-06,
      "logits/chosen": -1.8933544158935547,
      "logits/rejected": -1.8870652914047241,
      "logps/chosen": -31.835384368896484,
      "logps/rejected": -30.871362686157227,
      "loss": 0.2807,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 1.3110452890396118,
      "rewards/margins": 1.2270557880401611,
      "rewards/rejected": 0.08398932963609695,
      "step": 550
    },
    {
      "epoch": 1.45,
      "grad_norm": 5.15625,
      "learning_rate": 4.014024217844167e-06,
      "logits/chosen": -1.9839521646499634,
      "logits/rejected": -1.9819648265838623,
      "logps/chosen": -32.755332946777344,
      "logps/rejected": -31.447025299072266,
      "loss": 0.2809,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": 1.271106481552124,
      "rewards/margins": 1.2358547449111938,
      "rewards/rejected": 0.035251684486866,
      "step": 560
    },
    {
      "epoch": 1.48,
      "grad_norm": 6.34375,
      "learning_rate": 3.968546095984911e-06,
      "logits/chosen": -1.8162040710449219,
      "logits/rejected": -1.814014196395874,
      "logps/chosen": -31.14450454711914,
      "logps/rejected": -30.988576889038086,
      "loss": 0.2898,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 1.1563327312469482,
      "rewards/margins": 1.146722435951233,
      "rewards/rejected": 0.009610283188521862,
      "step": 570
    },
    {
      "epoch": 1.51,
      "grad_norm": 5.5625,
      "learning_rate": 3.922313503607806e-06,
      "logits/chosen": -1.95193612575531,
      "logits/rejected": -1.948587417602539,
      "logps/chosen": -29.390304565429688,
      "logps/rejected": -34.86743927001953,
      "loss": 0.2599,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 1.2000386714935303,
      "rewards/margins": 1.3689311742782593,
      "rewards/rejected": -0.16889244318008423,
      "step": 580
    },
    {
      "epoch": 1.53,
      "grad_norm": 5.1875,
      "learning_rate": 3.875350192863368e-06,
      "logits/chosen": -1.8900426626205444,
      "logits/rejected": -1.8936386108398438,
      "logps/chosen": -28.26003646850586,
      "logps/rejected": -30.714879989624023,
      "loss": 0.3068,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": 0.9484975934028625,
      "rewards/margins": 0.9877050518989563,
      "rewards/rejected": -0.03920745849609375,
      "step": 590
    },
    {
      "epoch": 1.56,
      "grad_norm": 6.03125,
      "learning_rate": 3.8276802913111436e-06,
      "logits/chosen": -1.9284902811050415,
      "logits/rejected": -1.9283053874969482,
      "logps/chosen": -30.269977569580078,
      "logps/rejected": -31.168771743774414,
      "loss": 0.2593,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 1.3660885095596313,
      "rewards/margins": 1.3491170406341553,
      "rewards/rejected": 0.016971405595541,
      "step": 600
    },
    {
      "epoch": 1.56,
      "eval_logits/chosen": -2.1913671493530273,
      "eval_logits/rejected": -2.186584949493408,
      "eval_logps/chosen": -33.55670166015625,
      "eval_logps/rejected": -37.16816711425781,
      "eval_loss": 0.4795128405094147,
      "eval_rewards/accuracies": 0.574335515499115,
      "eval_rewards/chosen": 0.3344981372356415,
      "eval_rewards/margins": 0.09058075398206711,
      "eval_rewards/rejected": 0.24391737580299377,
      "eval_runtime": 145.1862,
      "eval_samples_per_second": 2.362,
      "eval_steps_per_second": 0.296,
      "step": 600
    },
    {
      "epoch": 1.58,
      "grad_norm": 5.625,
      "learning_rate": 3.7793282895240927e-06,
      "logits/chosen": -1.9897689819335938,
      "logits/rejected": -1.9904531240463257,
      "logps/chosen": -32.8941650390625,
      "logps/rejected": -33.11201477050781,
      "loss": 0.2422,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 1.4951863288879395,
      "rewards/margins": 1.5146669149398804,
      "rewards/rejected": -0.019480543211102486,
      "step": 610
    },
    {
      "epoch": 1.61,
      "grad_norm": 3.875,
      "learning_rate": 3.730319028506478e-06,
      "logits/chosen": -1.953918695449829,
      "logits/rejected": -1.9515819549560547,
      "logps/chosen": -31.27206039428711,
      "logps/rejected": -32.174869537353516,
      "loss": 0.2593,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 1.297335147857666,
      "rewards/margins": 1.3708014488220215,
      "rewards/rejected": -0.07346634566783905,
      "step": 620
    },
    {
      "epoch": 1.64,
      "grad_norm": 6.9375,
      "learning_rate": 3.6806776869317074e-06,
      "logits/chosen": -1.967087745666504,
      "logits/rejected": -1.9582030773162842,
      "logps/chosen": -30.80006980895996,
      "logps/rejected": -31.055767059326172,
      "loss": 0.2573,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": 1.3832311630249023,
      "rewards/margins": 1.4675111770629883,
      "rewards/rejected": -0.08427979052066803,
      "step": 630
    },
    {
      "epoch": 1.66,
      "grad_norm": 6.21875,
      "learning_rate": 3.6304297682067146e-06,
      "logits/chosen": -1.9678598642349243,
      "logits/rejected": -1.9646453857421875,
      "logps/chosen": -30.486230850219727,
      "logps/rejected": -32.526588439941406,
      "loss": 0.2662,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 1.1985342502593994,
      "rewards/margins": 1.2735540866851807,
      "rewards/rejected": -0.07501962780952454,
      "step": 640
    },
    {
      "epoch": 1.69,
      "grad_norm": 5.25,
      "learning_rate": 3.579601087369492e-06,
      "logits/chosen": -1.9753246307373047,
      "logits/rejected": -1.9776771068572998,
      "logps/chosen": -31.804119110107422,
      "logps/rejected": -33.76448440551758,
      "loss": 0.2474,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 1.2947121858596802,
      "rewards/margins": 1.3514697551727295,
      "rewards/rejected": -0.056757647544145584,
      "step": 650
    },
    {
      "epoch": 1.71,
      "grad_norm": 7.84375,
      "learning_rate": 3.5282177578265295e-06,
      "logits/chosen": -1.8884271383285522,
      "logits/rejected": -1.8889974355697632,
      "logps/chosen": -31.999134063720703,
      "logps/rejected": -31.639989852905273,
      "loss": 0.2419,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 1.363389253616333,
      "rewards/margins": 1.3863054513931274,
      "rewards/rejected": -0.022916005924344063,
      "step": 660
    },
    {
      "epoch": 1.74,
      "grad_norm": 6.15625,
      "learning_rate": 3.476306177936961e-06,
      "logits/chosen": -1.9295670986175537,
      "logits/rejected": -1.920019507408142,
      "logps/chosen": -32.041465759277344,
      "logps/rejected": -32.48603057861328,
      "loss": 0.2352,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 1.3718717098236084,
      "rewards/margins": 1.4966375827789307,
      "rewards/rejected": -0.1247657760977745,
      "step": 670
    },
    {
      "epoch": 1.77,
      "grad_norm": 3.453125,
      "learning_rate": 3.423893017450324e-06,
      "logits/chosen": -1.827956199645996,
      "logits/rejected": -1.8249555826187134,
      "logps/chosen": -29.395336151123047,
      "logps/rejected": -34.349639892578125,
      "loss": 0.2442,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 1.445934534072876,
      "rewards/margins": 1.4676737785339355,
      "rewards/rejected": -0.021739140152931213,
      "step": 680
    },
    {
      "epoch": 1.79,
      "grad_norm": 4.75,
      "learning_rate": 3.3710052038048794e-06,
      "logits/chosen": -1.8876937627792358,
      "logits/rejected": -1.8869459629058838,
      "logps/chosen": -32.98918151855469,
      "logps/rejected": -35.476680755615234,
      "loss": 0.2165,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 1.5967752933502197,
      "rewards/margins": 1.6827762126922607,
      "rewards/rejected": -0.08600117266178131,
      "step": 690
    },
    {
      "epoch": 1.82,
      "grad_norm": 4.15625,
      "learning_rate": 3.3176699082935546e-06,
      "logits/chosen": -1.862781286239624,
      "logits/rejected": -1.8658040761947632,
      "logps/chosen": -30.54743003845215,
      "logps/rejected": -35.61197280883789,
      "loss": 0.2606,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 1.4387942552566528,
      "rewards/margins": 1.418005108833313,
      "rewards/rejected": 0.020789090543985367,
      "step": 700
    },
    {
      "epoch": 1.82,
      "eval_logits/chosen": -2.1835741996765137,
      "eval_logits/rejected": -2.1788299083709717,
      "eval_logps/chosen": -33.57905578613281,
      "eval_logps/rejected": -37.20835876464844,
      "eval_loss": 0.47641801834106445,
      "eval_rewards/accuracies": 0.6063122749328613,
      "eval_rewards/chosen": 0.3188474476337433,
      "eval_rewards/margins": 0.10306696593761444,
      "eval_rewards/rejected": 0.21578048169612885,
      "eval_runtime": 145.3074,
      "eval_samples_per_second": 2.361,
      "eval_steps_per_second": 0.296,
      "step": 700
    },
    {
      "epoch": 1.84,
      "grad_norm": 5.8125,
      "learning_rate": 3.2639145321045933e-06,
      "logits/chosen": -1.970207929611206,
      "logits/rejected": -1.972961187362671,
      "logps/chosen": -32.8425407409668,
      "logps/rejected": -34.31882858276367,
      "loss": 0.2562,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 1.4371341466903687,
      "rewards/margins": 1.471206784248352,
      "rewards/rejected": -0.034072600305080414,
      "step": 710
    },
    {
      "epoch": 1.87,
      "grad_norm": 6.8125,
      "learning_rate": 3.2097666922441107e-06,
      "logits/chosen": -1.8238359689712524,
      "logits/rejected": -1.8179490566253662,
      "logps/chosen": -32.64445877075195,
      "logps/rejected": -32.34913635253906,
      "loss": 0.2397,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 1.5321400165557861,
      "rewards/margins": 1.530247449874878,
      "rewards/rejected": 0.0018926591146737337,
      "step": 720
    },
    {
      "epoch": 1.9,
      "grad_norm": 3.765625,
      "learning_rate": 3.1552542073477554e-06,
      "logits/chosen": -1.996241569519043,
      "logits/rejected": -1.9933366775512695,
      "logps/chosen": -28.67278480529785,
      "logps/rejected": -31.668869018554688,
      "loss": 0.2416,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 1.3861420154571533,
      "rewards/margins": 1.5406121015548706,
      "rewards/rejected": -0.1544700562953949,
      "step": 730
    },
    {
      "epoch": 1.92,
      "grad_norm": 3.875,
      "learning_rate": 3.100405083388799e-06,
      "logits/chosen": -1.8369905948638916,
      "logits/rejected": -1.8370345830917358,
      "logps/chosen": -31.223901748657227,
      "logps/rejected": -37.633975982666016,
      "loss": 0.2245,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 1.5674915313720703,
      "rewards/margins": 1.571808099746704,
      "rewards/rejected": -0.004316648934036493,
      "step": 740
    },
    {
      "epoch": 1.95,
      "grad_norm": 3.1875,
      "learning_rate": 3.0452474992899645e-06,
      "logits/chosen": -1.7186479568481445,
      "logits/rejected": -1.7238624095916748,
      "logps/chosen": -34.5988655090332,
      "logps/rejected": -33.955909729003906,
      "loss": 0.2323,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 1.6335166692733765,
      "rewards/margins": 1.6220157146453857,
      "rewards/rejected": 0.011500892229378223,
      "step": 750
    },
    {
      "epoch": 1.97,
      "grad_norm": 5.625,
      "learning_rate": 2.989809792446417e-06,
      "logits/chosen": -1.9235740900039673,
      "logits/rejected": -1.9250673055648804,
      "logps/chosen": -30.627086639404297,
      "logps/rejected": -32.82661437988281,
      "loss": 0.2421,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 1.385018229484558,
      "rewards/margins": 1.4181984663009644,
      "rewards/rejected": -0.03318030759692192,
      "step": 760
    },
    {
      "epoch": 2.0,
      "grad_norm": 5.34375,
      "learning_rate": 2.9341204441673267e-06,
      "logits/chosen": -1.894831895828247,
      "logits/rejected": -1.8939815759658813,
      "logps/chosen": -30.067148208618164,
      "logps/rejected": -34.84111785888672,
      "loss": 0.2551,
      "rewards/accuracies": 0.85833340883255,
      "rewards/chosen": 1.4759252071380615,
      "rewards/margins": 1.3449987173080444,
      "rewards/rejected": 0.13092657923698425,
      "step": 770
    },
    {
      "epoch": 2.03,
      "grad_norm": 3.671875,
      "learning_rate": 2.878208065043501e-06,
      "logits/chosen": -1.895990014076233,
      "logits/rejected": -1.8953578472137451,
      "logps/chosen": -32.30732727050781,
      "logps/rejected": -31.775619506835938,
      "loss": 0.1909,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 1.6387954950332642,
      "rewards/margins": 1.8880647420883179,
      "rewards/rejected": -0.24926936626434326,
      "step": 780
    },
    {
      "epoch": 2.05,
      "grad_norm": 4.90625,
      "learning_rate": 2.8221013802485974e-06,
      "logits/chosen": -1.9268324375152588,
      "logits/rejected": -1.9253270626068115,
      "logps/chosen": -27.4288387298584,
      "logps/rejected": -33.15345764160156,
      "loss": 0.1931,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": 1.4940404891967773,
      "rewards/margins": 1.8560407161712646,
      "rewards/rejected": -0.36200040578842163,
      "step": 790
    },
    {
      "epoch": 2.08,
      "grad_norm": 3.78125,
      "learning_rate": 2.76582921478147e-06,
      "logits/chosen": -1.9773414134979248,
      "logits/rejected": -1.974020004272461,
      "logps/chosen": -29.959461212158203,
      "logps/rejected": -34.62144470214844,
      "loss": 0.1758,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 1.7274243831634521,
      "rewards/margins": 2.023111581802368,
      "rewards/rejected": -0.2956870496273041,
      "step": 800
    },
    {
      "epoch": 2.08,
      "eval_logits/chosen": -2.172740936279297,
      "eval_logits/rejected": -2.1680266857147217,
      "eval_logps/chosen": -33.62887191772461,
      "eval_logps/rejected": -37.266761779785156,
      "eval_loss": 0.4767366945743561,
      "eval_rewards/accuracies": 0.5859634280204773,
      "eval_rewards/chosen": 0.28397703170776367,
      "eval_rewards/margins": 0.10907654464244843,
      "eval_rewards/rejected": 0.17490047216415405,
      "eval_runtime": 145.1754,
      "eval_samples_per_second": 2.363,
      "eval_steps_per_second": 0.296,
      "step": 800
    },
    {
      "epoch": 2.1,
      "grad_norm": 2.46875,
      "learning_rate": 2.7094204786572254e-06,
      "logits/chosen": -1.8107726573944092,
      "logits/rejected": -1.8030484914779663,
      "logps/chosen": -31.506906509399414,
      "logps/rejected": -35.04960250854492,
      "loss": 0.1452,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 2.0645337104797363,
      "rewards/margins": 2.407174587249756,
      "rewards/rejected": -0.3426409363746643,
      "step": 810
    },
    {
      "epoch": 2.13,
      "grad_norm": 4.40625,
      "learning_rate": 2.6529041520546072e-06,
      "logits/chosen": -1.887786865234375,
      "logits/rejected": -1.8982326984405518,
      "logps/chosen": -33.01645278930664,
      "logps/rejected": -32.74128341674805,
      "loss": 0.1899,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 1.8754699230194092,
      "rewards/margins": 2.0923609733581543,
      "rewards/rejected": -0.2168913334608078,
      "step": 820
    },
    {
      "epoch": 2.16,
      "grad_norm": 3.796875,
      "learning_rate": 2.5963092704273302e-06,
      "logits/chosen": -1.9369271993637085,
      "logits/rejected": -1.9413642883300781,
      "logps/chosen": -32.51754379272461,
      "logps/rejected": -29.556041717529297,
      "loss": 0.1869,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 1.6696659326553345,
      "rewards/margins": 1.8711254596710205,
      "rewards/rejected": -0.20145969092845917,
      "step": 830
    },
    {
      "epoch": 2.18,
      "grad_norm": 4.46875,
      "learning_rate": 2.53966490958702e-06,
      "logits/chosen": -1.9355465173721313,
      "logits/rejected": -1.9433647394180298,
      "logps/chosen": -32.05677795410156,
      "logps/rejected": -30.385608673095703,
      "loss": 0.1604,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": 1.911877989768982,
      "rewards/margins": 2.2884128093719482,
      "rewards/rejected": -0.3765348792076111,
      "step": 840
    },
    {
      "epoch": 2.21,
      "grad_norm": 4.625,
      "learning_rate": 2.4830001707654135e-06,
      "logits/chosen": -1.866355538368225,
      "logits/rejected": -1.8570215702056885,
      "logps/chosen": -29.2946720123291,
      "logps/rejected": -32.352394104003906,
      "loss": 0.1771,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": 1.7489668130874634,
      "rewards/margins": 1.9859797954559326,
      "rewards/rejected": -0.23701317608356476,
      "step": 850
    },
    {
      "epoch": 2.23,
      "grad_norm": 3.6875,
      "learning_rate": 2.4263441656635054e-06,
      "logits/chosen": -2.0053553581237793,
      "logits/rejected": -1.9958158731460571,
      "logps/chosen": -23.93155288696289,
      "logps/rejected": -30.337451934814453,
      "loss": 0.1938,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 1.604762077331543,
      "rewards/margins": 1.9303429126739502,
      "rewards/rejected": -0.3255811333656311,
      "step": 860
    },
    {
      "epoch": 2.26,
      "grad_norm": 3.4375,
      "learning_rate": 2.3697260014953107e-06,
      "logits/chosen": -1.8573694229125977,
      "logits/rejected": -1.8587795495986938,
      "logps/chosen": -31.69219398498535,
      "logps/rejected": -30.3853816986084,
      "loss": 0.1718,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": 1.8485662937164307,
      "rewards/margins": 2.130580186843872,
      "rewards/rejected": -0.28201407194137573,
      "step": 870
    },
    {
      "epoch": 2.29,
      "grad_norm": 3.1875,
      "learning_rate": 2.3131747660339396e-06,
      "logits/chosen": -1.8693485260009766,
      "logits/rejected": -1.8701515197753906,
      "logps/chosen": -29.962982177734375,
      "logps/rejected": -33.51511764526367,
      "loss": 0.1879,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 1.6983861923217773,
      "rewards/margins": 2.0233235359191895,
      "rewards/rejected": -0.3249376118183136,
      "step": 880
    },
    {
      "epoch": 2.31,
      "grad_norm": 3.25,
      "learning_rate": 2.256719512667651e-06,
      "logits/chosen": -1.7684412002563477,
      "logits/rejected": -1.7669271230697632,
      "logps/chosen": -32.97483825683594,
      "logps/rejected": -36.623138427734375,
      "loss": 0.1579,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 1.8830540180206299,
      "rewards/margins": 2.405186176300049,
      "rewards/rejected": -0.522132396697998,
      "step": 890
    },
    {
      "epoch": 2.34,
      "grad_norm": 3.296875,
      "learning_rate": 2.2003892454735786e-06,
      "logits/chosen": -1.9180320501327515,
      "logits/rejected": -1.9112794399261475,
      "logps/chosen": -29.328502655029297,
      "logps/rejected": -33.22010803222656,
      "loss": 0.1687,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": 1.8718944787979126,
      "rewards/margins": 2.1953208446502686,
      "rewards/rejected": -0.3234265446662903,
      "step": 900
    },
    {
      "epoch": 2.34,
      "eval_logits/chosen": -2.1673672199249268,
      "eval_logits/rejected": -2.1626341342926025,
      "eval_logps/chosen": -33.620479583740234,
      "eval_logps/rejected": -37.25470733642578,
      "eval_loss": 0.4770047068595886,
      "eval_rewards/accuracies": 0.5485880374908447,
      "eval_rewards/chosen": 0.2898494601249695,
      "eval_rewards/margins": 0.10651160031557083,
      "eval_rewards/rejected": 0.18333788216114044,
      "eval_runtime": 145.3634,
      "eval_samples_per_second": 2.36,
      "eval_steps_per_second": 0.296,
      "step": 900
    },
    {
      "epoch": 2.36,
      "grad_norm": 4.0625,
      "learning_rate": 2.1442129043167877e-06,
      "logits/chosen": -1.9585449695587158,
      "logits/rejected": -1.953966498374939,
      "logps/chosen": -31.010263442993164,
      "logps/rejected": -35.009552001953125,
      "loss": 0.164,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 1.783191442489624,
      "rewards/margins": 2.119842052459717,
      "rewards/rejected": -0.3366504907608032,
      "step": 910
    },
    {
      "epoch": 2.39,
      "grad_norm": 3.453125,
      "learning_rate": 2.088219349982323e-06,
      "logits/chosen": -1.8792098760604858,
      "logits/rejected": -1.8842833042144775,
      "logps/chosen": -32.50172805786133,
      "logps/rejected": -32.84755325317383,
      "loss": 0.1644,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": 1.9927990436553955,
      "rewards/margins": 2.236151933670044,
      "rewards/rejected": -0.24335305392742157,
      "step": 920
    },
    {
      "epoch": 2.42,
      "grad_norm": 4.15625,
      "learning_rate": 2.0324373493478803e-06,
      "logits/chosen": -1.9770915508270264,
      "logits/rejected": -1.9679561853408813,
      "logps/chosen": -29.701152801513672,
      "logps/rejected": -34.7978630065918,
      "loss": 0.1669,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.8086459636688232,
      "rewards/margins": 2.0571932792663574,
      "rewards/rejected": -0.2485475093126297,
      "step": 930
    },
    {
      "epoch": 2.44,
      "grad_norm": 3.78125,
      "learning_rate": 1.976895560604729e-06,
      "logits/chosen": -1.903071641921997,
      "logits/rejected": -1.8999197483062744,
      "logps/chosen": -29.023662567138672,
      "logps/rejected": -32.71151351928711,
      "loss": 0.2019,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 1.585354208946228,
      "rewards/margins": 1.757653832435608,
      "rewards/rejected": -0.17229948937892914,
      "step": 940
    },
    {
      "epoch": 2.47,
      "grad_norm": 3.4375,
      "learning_rate": 1.921622518534466e-06,
      "logits/chosen": -1.842889428138733,
      "logits/rejected": -1.8503259420394897,
      "logps/chosen": -30.36769676208496,
      "logps/rejected": -35.7653923034668,
      "loss": 0.2098,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 1.7523466348648071,
      "rewards/margins": 1.903167963027954,
      "rewards/rejected": -0.15082135796546936,
      "step": 950
    },
    {
      "epoch": 2.49,
      "grad_norm": 4.15625,
      "learning_rate": 1.8666466198491794e-06,
      "logits/chosen": -1.8521137237548828,
      "logits/rejected": -1.8454768657684326,
      "logps/chosen": -31.145715713500977,
      "logps/rejected": -37.1761589050293,
      "loss": 0.1613,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": 1.877960205078125,
      "rewards/margins": 2.282109260559082,
      "rewards/rejected": -0.4041489064693451,
      "step": 960
    },
    {
      "epoch": 2.52,
      "grad_norm": 4.1875,
      "learning_rate": 1.8119961086025376e-06,
      "logits/chosen": -1.8534562587738037,
      "logits/rejected": -1.8540384769439697,
      "logps/chosen": -28.50092124938965,
      "logps/rejected": -33.28984832763672,
      "loss": 0.1593,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 1.9148372411727905,
      "rewards/margins": 2.1308465003967285,
      "rewards/rejected": -0.2160094678401947,
      "step": 970
    },
    {
      "epoch": 2.55,
      "grad_norm": 4.375,
      "learning_rate": 1.7576990616793139e-06,
      "logits/chosen": -1.8584295511245728,
      "logits/rejected": -1.8696168661117554,
      "logps/chosen": -30.254512786865234,
      "logps/rejected": -34.45421600341797,
      "loss": 0.1653,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 1.97552490234375,
      "rewards/margins": 2.201627731323242,
      "rewards/rejected": -0.22610275447368622,
      "step": 980
    },
    {
      "epoch": 2.57,
      "grad_norm": 2.828125,
      "learning_rate": 1.7037833743707892e-06,
      "logits/chosen": -1.926839828491211,
      "logits/rejected": -1.924412488937378,
      "logps/chosen": -33.426719665527344,
      "logps/rejected": -32.59941101074219,
      "loss": 0.1853,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 1.8860301971435547,
      "rewards/margins": 2.0261569023132324,
      "rewards/rejected": -0.14012634754180908,
      "step": 990
    },
    {
      "epoch": 2.6,
      "grad_norm": 4.125,
      "learning_rate": 1.6502767460434588e-06,
      "logits/chosen": -1.8978465795516968,
      "logits/rejected": -1.9018335342407227,
      "logps/chosen": -31.86910629272461,
      "logps/rejected": -34.74235153198242,
      "loss": 0.1826,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 1.8159281015396118,
      "rewards/margins": 1.9953454732894897,
      "rewards/rejected": -0.17941740155220032,
      "step": 1000
    },
    {
      "epoch": 2.6,
      "eval_logits/chosen": -2.162538528442383,
      "eval_logits/rejected": -2.157820701599121,
      "eval_logps/chosen": -33.64886474609375,
      "eval_logps/rejected": -37.291744232177734,
      "eval_loss": 0.4763668477535248,
      "eval_rewards/accuracies": 0.5830564498901367,
      "eval_rewards/chosen": 0.26998022198677063,
      "eval_rewards/margins": 0.11256679147481918,
      "eval_rewards/rejected": 0.15741342306137085,
      "eval_runtime": 145.331,
      "eval_samples_per_second": 2.36,
      "eval_steps_per_second": 0.296,
      "step": 1000
    },
    {
      "epoch": 2.62,
      "grad_norm": 5.9375,
      "learning_rate": 1.5972066659083796e-06,
      "logits/chosen": -1.9077873229980469,
      "logits/rejected": -1.9126176834106445,
      "logps/chosen": -30.2926082611084,
      "logps/rejected": -32.68244171142578,
      "loss": 0.2024,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 1.7554700374603271,
      "rewards/margins": 1.8682985305786133,
      "rewards/rejected": -0.11282841116189957,
      "step": 1010
    },
    {
      "epoch": 2.65,
      "grad_norm": 3.734375,
      "learning_rate": 1.5446003988985041e-06,
      "logits/chosen": -1.8675119876861572,
      "logits/rejected": -1.8702566623687744,
      "logps/chosen": -27.929931640625,
      "logps/rejected": -32.15419006347656,
      "loss": 0.1718,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 1.7609975337982178,
      "rewards/margins": 2.079514503479004,
      "rewards/rejected": -0.31851673126220703,
      "step": 1020
    },
    {
      "epoch": 2.68,
      "grad_norm": 2.671875,
      "learning_rate": 1.4924849716612211e-06,
      "logits/chosen": -1.9224603176116943,
      "logits/rejected": -1.916998267173767,
      "logps/chosen": -31.501073837280273,
      "logps/rejected": -33.88199996948242,
      "loss": 0.1823,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 1.7416280508041382,
      "rewards/margins": 1.9964303970336914,
      "rewards/rejected": -0.2548021972179413,
      "step": 1030
    },
    {
      "epoch": 2.7,
      "grad_norm": 4.71875,
      "learning_rate": 1.440887158673332e-06,
      "logits/chosen": -1.8555800914764404,
      "logits/rejected": -1.8593838214874268,
      "logps/chosen": -33.17572021484375,
      "logps/rejected": -34.55617141723633,
      "loss": 0.19,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 1.8046057224273682,
      "rewards/margins": 1.9002208709716797,
      "rewards/rejected": -0.09561540186405182,
      "step": 1040
    },
    {
      "epoch": 2.73,
      "grad_norm": 3.5,
      "learning_rate": 1.3898334684855647e-06,
      "logits/chosen": -1.8540366888046265,
      "logits/rejected": -1.866837501525879,
      "logps/chosen": -29.41763687133789,
      "logps/rejected": -33.34986114501953,
      "loss": 0.1906,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 1.711678147315979,
      "rewards/margins": 1.9425528049468994,
      "rewards/rejected": -0.23087480664253235,
      "step": 1050
    },
    {
      "epoch": 2.75,
      "grad_norm": 4.0625,
      "learning_rate": 1.3393501301037245e-06,
      "logits/chosen": -1.8432916402816772,
      "logits/rejected": -1.8369777202606201,
      "logps/chosen": -29.536724090576172,
      "logps/rejected": -33.66853713989258,
      "loss": 0.1732,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": 1.835822343826294,
      "rewards/margins": 2.0812220573425293,
      "rewards/rejected": -0.24539966881275177,
      "step": 1060
    },
    {
      "epoch": 2.78,
      "grad_norm": 3.59375,
      "learning_rate": 1.2894630795134454e-06,
      "logits/chosen": -1.9685659408569336,
      "logits/rejected": -1.9693739414215088,
      "logps/chosen": -30.985530853271484,
      "logps/rejected": -33.102413177490234,
      "loss": 0.1709,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 1.7930816411972046,
      "rewards/margins": 1.9734855890274048,
      "rewards/rejected": -0.180403932929039,
      "step": 1070
    },
    {
      "epoch": 2.81,
      "grad_norm": 5.28125,
      "learning_rate": 1.2401979463554984e-06,
      "logits/chosen": -1.9013326168060303,
      "logits/rejected": -1.9002430438995361,
      "logps/chosen": -31.509572982788086,
      "logps/rejected": -33.25670623779297,
      "loss": 0.2033,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 1.8061821460723877,
      "rewards/margins": 1.8515968322753906,
      "rewards/rejected": -0.045414675027132034,
      "step": 1080
    },
    {
      "epoch": 2.83,
      "grad_norm": 3.234375,
      "learning_rate": 1.1915800407584705e-06,
      "logits/chosen": -1.9260213375091553,
      "logits/rejected": -1.918617606163025,
      "logps/chosen": -31.691287994384766,
      "logps/rejected": -31.300344467163086,
      "loss": 0.1849,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 1.774212121963501,
      "rewards/margins": 1.955381155014038,
      "rewards/rejected": -0.1811690330505371,
      "step": 1090
    },
    {
      "epoch": 2.86,
      "grad_norm": 2.796875,
      "learning_rate": 1.1436343403356019e-06,
      "logits/chosen": -1.896405577659607,
      "logits/rejected": -1.8956992626190186,
      "logps/chosen": -32.56309509277344,
      "logps/rejected": -36.86756134033203,
      "loss": 0.1541,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": 1.9987242221832275,
      "rewards/margins": 2.359121322631836,
      "rewards/rejected": -0.3603973388671875,
      "step": 1100
    },
    {
      "epoch": 2.86,
      "eval_logits/chosen": -2.1608357429504395,
      "eval_logits/rejected": -2.1561272144317627,
      "eval_logps/chosen": -33.625431060791016,
      "eval_logps/rejected": -37.27484893798828,
      "eval_loss": 0.47510504722595215,
      "eval_rewards/accuracies": 0.5776578187942505,
      "eval_rewards/chosen": 0.28638461232185364,
      "eval_rewards/margins": 0.11714447289705276,
      "eval_rewards/rejected": 0.16924014687538147,
      "eval_runtime": 145.2146,
      "eval_samples_per_second": 2.362,
      "eval_steps_per_second": 0.296,
      "step": 1100
    },
    {
      "epoch": 2.88,
      "grad_norm": 3.734375,
      "learning_rate": 1.0963854773524548e-06,
      "logits/chosen": -1.9410547018051147,
      "logits/rejected": -1.946345567703247,
      "logps/chosen": -32.88351821899414,
      "logps/rejected": -35.84196472167969,
      "loss": 0.1916,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 1.8085190057754517,
      "rewards/margins": 2.0173850059509277,
      "rewards/rejected": -0.2088661640882492,
      "step": 1110
    },
    {
      "epoch": 2.91,
      "grad_norm": 2.40625,
      "learning_rate": 1.049857726072005e-06,
      "logits/chosen": -1.914781928062439,
      "logits/rejected": -1.912883996963501,
      "logps/chosen": -30.12225341796875,
      "logps/rejected": -33.4071044921875,
      "loss": 0.1829,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 1.74576735496521,
      "rewards/margins": 2.0428128242492676,
      "rewards/rejected": -0.29704537987709045,
      "step": 1120
    },
    {
      "epoch": 2.94,
      "grad_norm": 2.828125,
      "learning_rate": 1.0040749902836508e-06,
      "logits/chosen": -1.8179181814193726,
      "logits/rejected": -1.8206218481063843,
      "logps/chosen": -27.161823272705078,
      "logps/rejected": -30.888957977294922,
      "loss": 0.1529,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": 1.8107696771621704,
      "rewards/margins": 2.2615151405334473,
      "rewards/rejected": -0.4507456421852112,
      "step": 1130
    },
    {
      "epoch": 2.96,
      "grad_norm": 3.171875,
      "learning_rate": 9.59060791022566e-07,
      "logits/chosen": -1.916400671005249,
      "logits/rejected": -1.9134107828140259,
      "logps/chosen": -30.4929141998291,
      "logps/rejected": -32.6131477355957,
      "loss": 0.178,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 1.897585153579712,
      "rewards/margins": 1.957972764968872,
      "rewards/rejected": -0.060387589037418365,
      "step": 1140
    },
    {
      "epoch": 2.99,
      "grad_norm": 4.5,
      "learning_rate": 9.148382544856885e-07,
      "logits/chosen": -1.8358739614486694,
      "logits/rejected": -1.8296668529510498,
      "logps/chosen": -25.861547470092773,
      "logps/rejected": -32.6159553527832,
      "loss": 0.2002,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 1.5935077667236328,
      "rewards/margins": 1.9208663702011108,
      "rewards/rejected": -0.32735878229141235,
      "step": 1150
    },
    {
      "epoch": 3.01,
      "grad_norm": 2.78125,
      "learning_rate": 8.714301001505568e-07,
      "logits/chosen": -1.9491764307022095,
      "logits/rejected": -1.9467414617538452,
      "logps/chosen": -30.44895362854004,
      "logps/rejected": -35.62247848510742,
      "loss": 0.1676,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": 1.852980375289917,
      "rewards/margins": 2.224640369415283,
      "rewards/rejected": -0.3716599941253662,
      "step": 1160
    },
    {
      "epoch": 3.04,
      "grad_norm": 2.578125,
      "learning_rate": 8.288586291031025e-07,
      "logits/chosen": -1.932654619216919,
      "logits/rejected": -1.930672287940979,
      "logps/chosen": -28.86700439453125,
      "logps/rejected": -32.90315246582031,
      "loss": 0.1532,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": 1.9231812953948975,
      "rewards/margins": 2.300574779510498,
      "rewards/rejected": -0.37739327549934387,
      "step": 1170
    },
    {
      "epoch": 3.06,
      "grad_norm": 3.125,
      "learning_rate": 7.871457125803897e-07,
      "logits/chosen": -1.9274743795394897,
      "logits/rejected": -1.9154332876205444,
      "logps/chosen": -33.33545684814453,
      "logps/rejected": -34.530860900878906,
      "loss": 0.1634,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 1.8968908786773682,
      "rewards/margins": 2.218459367752075,
      "rewards/rejected": -0.3215683400630951,
      "step": 1180
    },
    {
      "epoch": 3.09,
      "grad_norm": 4.5,
      "learning_rate": 7.463127807341966e-07,
      "logits/chosen": -1.819190263748169,
      "logits/rejected": -1.8140865564346313,
      "logps/chosen": -32.35272979736328,
      "logps/rejected": -34.45198440551758,
      "loss": 0.1541,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 2.097872734069824,
      "rewards/margins": 2.247788906097412,
      "rewards/rejected": -0.14991624653339386,
      "step": 1190
    },
    {
      "epoch": 3.12,
      "grad_norm": 3.046875,
      "learning_rate": 7.063808116212021e-07,
      "logits/chosen": -1.8649425506591797,
      "logits/rejected": -1.8679530620574951,
      "logps/chosen": -30.119836807250977,
      "logps/rejected": -31.720102310180664,
      "loss": 0.194,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 1.746681809425354,
      "rewards/margins": 1.91042959690094,
      "rewards/rejected": -0.163747638463974,
      "step": 1200
    },
    {
      "epoch": 3.12,
      "eval_logits/chosen": -2.1612026691436768,
      "eval_logits/rejected": -2.156489610671997,
      "eval_logps/chosen": -33.626529693603516,
      "eval_logps/rejected": -37.28034210205078,
      "eval_loss": 0.47478199005126953,
      "eval_rewards/accuracies": 0.5801494717597961,
      "eval_rewards/chosen": 0.28561660647392273,
      "eval_rewards/margins": 0.12022235989570618,
      "eval_rewards/rejected": 0.16539424657821655,
      "eval_runtime": 145.3188,
      "eval_samples_per_second": 2.36,
      "eval_steps_per_second": 0.296,
      "step": 1200
    },
    {
      "epoch": 3.14,
      "grad_norm": 3.8125,
      "learning_rate": 6.673703204254348e-07,
      "logits/chosen": -1.8913599252700806,
      "logits/rejected": -1.8870538473129272,
      "logps/chosen": -28.591426849365234,
      "logps/rejected": -31.342187881469727,
      "loss": 0.1557,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 1.8775701522827148,
      "rewards/margins": 2.181436538696289,
      "rewards/rejected": -0.30386656522750854,
      "step": 1210
    },
    {
      "epoch": 3.17,
      "grad_norm": 3.296875,
      "learning_rate": 6.293013489185315e-07,
      "logits/chosen": -1.9030640125274658,
      "logits/rejected": -1.897794485092163,
      "logps/chosen": -32.43680953979492,
      "logps/rejected": -35.05671691894531,
      "loss": 0.1566,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 1.9859931468963623,
      "rewards/margins": 2.3267762660980225,
      "rewards/rejected": -0.3407830595970154,
      "step": 1220
    },
    {
      "epoch": 3.19,
      "grad_norm": 2.828125,
      "learning_rate": 5.921934551632086e-07,
      "logits/chosen": -1.9055864810943604,
      "logits/rejected": -1.892482042312622,
      "logps/chosen": -30.573678970336914,
      "logps/rejected": -34.583587646484375,
      "loss": 0.1759,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 1.7642362117767334,
      "rewards/margins": 2.0888543128967285,
      "rewards/rejected": -0.32461827993392944,
      "step": 1230
    },
    {
      "epoch": 3.22,
      "grad_norm": 2.5,
      "learning_rate": 5.560657034652405e-07,
      "logits/chosen": -1.9504817724227905,
      "logits/rejected": -1.9479200839996338,
      "logps/chosen": -32.358009338378906,
      "logps/rejected": -32.21631622314453,
      "loss": 0.168,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 1.8724260330200195,
      "rewards/margins": 2.13594126701355,
      "rewards/rejected": -0.2635149657726288,
      "step": 1240
    },
    {
      "epoch": 3.25,
      "grad_norm": 2.421875,
      "learning_rate": 5.2093665457911e-07,
      "logits/chosen": -1.8791261911392212,
      "logits/rejected": -1.8764369487762451,
      "logps/chosen": -31.556161880493164,
      "logps/rejected": -35.48642349243164,
      "loss": 0.1379,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 1.987993597984314,
      "rewards/margins": 2.4705002307891846,
      "rewards/rejected": -0.48250651359558105,
      "step": 1250
    },
    {
      "epoch": 3.27,
      "grad_norm": 2.640625,
      "learning_rate": 4.868243561723535e-07,
      "logits/chosen": -1.9446487426757812,
      "logits/rejected": -1.9393678903579712,
      "logps/chosen": -28.063180923461914,
      "logps/rejected": -32.76237487792969,
      "loss": 0.1586,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": 1.8883317708969116,
      "rewards/margins": 2.1542108058929443,
      "rewards/rejected": -0.2658790349960327,
      "step": 1260
    },
    {
      "epoch": 3.3,
      "grad_norm": 3.375,
      "learning_rate": 4.537463335535161e-07,
      "logits/chosen": -1.9708101749420166,
      "logits/rejected": -1.975584626197815,
      "logps/chosen": -29.93453025817871,
      "logps/rejected": -32.06245422363281,
      "loss": 0.1628,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": 1.9435287714004517,
      "rewards/margins": 2.210292339324951,
      "rewards/rejected": -0.2667638957500458,
      "step": 1270
    },
    {
      "epoch": 3.32,
      "grad_norm": 4.78125,
      "learning_rate": 4.217195806684629e-07,
      "logits/chosen": -1.8651002645492554,
      "logits/rejected": -1.872433066368103,
      "logps/chosen": -32.172630310058594,
      "logps/rejected": -33.01711654663086,
      "loss": 0.1714,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": 1.914229154586792,
      "rewards/margins": 2.2074227333068848,
      "rewards/rejected": -0.29319363832473755,
      "step": 1280
    },
    {
      "epoch": 3.35,
      "grad_norm": 3.84375,
      "learning_rate": 3.907605513696808e-07,
      "logits/chosen": -1.7700433731079102,
      "logits/rejected": -1.7724153995513916,
      "logps/chosen": -30.311264038085938,
      "logps/rejected": -36.89826202392578,
      "loss": 0.1748,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 1.9744415283203125,
      "rewards/margins": 2.2152342796325684,
      "rewards/rejected": -0.2407928705215454,
      "step": 1290
    },
    {
      "epoch": 3.38,
      "grad_norm": 1.546875,
      "learning_rate": 3.6088515096305675e-07,
      "logits/chosen": -1.8248789310455322,
      "logits/rejected": -1.82831609249115,
      "logps/chosen": -30.20694923400879,
      "logps/rejected": -33.00934600830078,
      "loss": 0.1414,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": 2.1068058013916016,
      "rewards/margins": 2.488433361053467,
      "rewards/rejected": -0.38162797689437866,
      "step": 1300
    },
    {
      "epoch": 3.38,
      "eval_logits/chosen": -2.160539150238037,
      "eval_logits/rejected": -2.1558327674865723,
      "eval_logps/chosen": -33.62611389160156,
      "eval_logps/rejected": -37.27513122558594,
      "eval_loss": 0.47534918785095215,
      "eval_rewards/accuracies": 0.5830564498901367,
      "eval_rewards/chosen": 0.2859070897102356,
      "eval_rewards/margins": 0.11686603724956512,
      "eval_rewards/rejected": 0.16904108226299286,
      "eval_runtime": 145.3117,
      "eval_samples_per_second": 2.36,
      "eval_steps_per_second": 0.296,
      "step": 1300
    },
    {
      "epoch": 3.4,
      "grad_norm": 2.8125,
      "learning_rate": 3.321087280364757e-07,
      "logits/chosen": -1.9337997436523438,
      "logits/rejected": -1.9138942956924438,
      "logps/chosen": -28.2788143157959,
      "logps/rejected": -35.673282623291016,
      "loss": 0.1617,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 1.984209418296814,
      "rewards/margins": 2.339022159576416,
      "rewards/rejected": -0.3548130691051483,
      "step": 1310
    },
    {
      "epoch": 3.43,
      "grad_norm": 2.890625,
      "learning_rate": 3.044460665744284e-07,
      "logits/chosen": -1.9120073318481445,
      "logits/rejected": -1.917673110961914,
      "logps/chosen": -29.9759578704834,
      "logps/rejected": -33.02309036254883,
      "loss": 0.1289,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": 2.1404826641082764,
      "rewards/margins": 2.5131189823150635,
      "rewards/rejected": -0.3726363182067871,
      "step": 1320
    },
    {
      "epoch": 3.45,
      "grad_norm": 3.6875,
      "learning_rate": 2.779113783626916e-07,
      "logits/chosen": -1.884010910987854,
      "logits/rejected": -1.879150390625,
      "logps/chosen": -30.3497257232666,
      "logps/rejected": -35.16407012939453,
      "loss": 0.154,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 1.8709485530853271,
      "rewards/margins": 2.2851948738098145,
      "rewards/rejected": -0.41424623131752014,
      "step": 1330
    },
    {
      "epoch": 3.48,
      "grad_norm": 2.046875,
      "learning_rate": 2.5251829568697204e-07,
      "logits/chosen": -1.7457748651504517,
      "logits/rejected": -1.755414366722107,
      "logps/chosen": -30.9390811920166,
      "logps/rejected": -31.315013885498047,
      "loss": 0.1642,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 2.0023577213287354,
      "rewards/margins": 2.198988676071167,
      "rewards/rejected": -0.19663105905056,
      "step": 1340
    },
    {
      "epoch": 3.51,
      "grad_norm": 3.21875,
      "learning_rate": 2.2827986432927774e-07,
      "logits/chosen": -1.8068130016326904,
      "logits/rejected": -1.8115451335906982,
      "logps/chosen": -30.966552734375,
      "logps/rejected": -33.47523880004883,
      "loss": 0.1355,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": 2.0890681743621826,
      "rewards/margins": 2.2783665657043457,
      "rewards/rejected": -0.18929778039455414,
      "step": 1350
    },
    {
      "epoch": 3.53,
      "grad_norm": 3.96875,
      "learning_rate": 2.0520853686560177e-07,
      "logits/chosen": -1.8347194194793701,
      "logits/rejected": -1.840275764465332,
      "logps/chosen": -31.000539779663086,
      "logps/rejected": -35.11589813232422,
      "loss": 0.1568,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 1.981787085533142,
      "rewards/margins": 2.36877179145813,
      "rewards/rejected": -0.38698458671569824,
      "step": 1360
    },
    {
      "epoch": 3.56,
      "grad_norm": 2.125,
      "learning_rate": 1.833161662683672e-07,
      "logits/chosen": -1.9270436763763428,
      "logits/rejected": -1.9225692749023438,
      "logps/chosen": -29.990549087524414,
      "logps/rejected": -31.350738525390625,
      "loss": 0.169,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": 1.8515956401824951,
      "rewards/margins": 2.158782482147217,
      "rewards/rejected": -0.30718696117401123,
      "step": 1370
    },
    {
      "epoch": 3.58,
      "grad_norm": 3.859375,
      "learning_rate": 1.626139998169246e-07,
      "logits/chosen": -1.8426074981689453,
      "logits/rejected": -1.8444961309432983,
      "logps/chosen": -27.056955337524414,
      "logps/rejected": -31.148794174194336,
      "loss": 0.1754,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 1.8216432332992554,
      "rewards/margins": 2.104875087738037,
      "rewards/rejected": -0.2832319140434265,
      "step": 1380
    },
    {
      "epoch": 3.61,
      "grad_norm": 4.9375,
      "learning_rate": 1.4311267331922535e-07,
      "logits/chosen": -1.836014986038208,
      "logits/rejected": -1.8406941890716553,
      "logps/chosen": -29.474157333374023,
      "logps/rejected": -33.005123138427734,
      "loss": 0.185,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": 1.7817341089248657,
      "rewards/margins": 1.7936760187149048,
      "rewards/rejected": -0.011941825971007347,
      "step": 1390
    },
    {
      "epoch": 3.64,
      "grad_norm": 2.03125,
      "learning_rate": 1.2482220564763669e-07,
      "logits/chosen": -1.9031331539154053,
      "logits/rejected": -1.9063663482666016,
      "logps/chosen": -32.08364486694336,
      "logps/rejected": -33.883567810058594,
      "loss": 0.1492,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": 1.9928478002548218,
      "rewards/margins": 2.3148512840270996,
      "rewards/rejected": -0.32200372219085693,
      "step": 1400
    },
    {
      "epoch": 3.64,
      "eval_logits/chosen": -2.1602745056152344,
      "eval_logits/rejected": -2.155562400817871,
      "eval_logps/chosen": -33.62794876098633,
      "eval_logps/rejected": -37.28424072265625,
      "eval_loss": 0.4743622839450836,
      "eval_rewards/accuracies": 0.5917773842811584,
      "eval_rewards/chosen": 0.2846195697784424,
      "eval_rewards/margins": 0.12195253372192383,
      "eval_rewards/rejected": 0.16266703605651855,
      "eval_runtime": 145.0405,
      "eval_samples_per_second": 2.365,
      "eval_steps_per_second": 0.296,
      "step": 1400
    },
    {
      "epoch": 3.66,
      "grad_norm": 3.1875,
      "learning_rate": 1.0775199359171346e-07,
      "logits/chosen": -1.9723188877105713,
      "logits/rejected": -1.9656269550323486,
      "logps/chosen": -30.971176147460938,
      "logps/rejected": -34.55162811279297,
      "loss": 0.1679,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": 1.9819583892822266,
      "rewards/margins": 2.196903705596924,
      "rewards/rejected": -0.2149454653263092,
      "step": 1410
    },
    {
      "epoch": 3.69,
      "grad_norm": 4.28125,
      "learning_rate": 9.191080703056604e-08,
      "logits/chosen": -1.8590764999389648,
      "logits/rejected": -1.8698148727416992,
      "logps/chosen": -31.296035766601562,
      "logps/rejected": -34.09710693359375,
      "loss": 0.1478,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": 1.9972867965698242,
      "rewards/margins": 2.3515076637268066,
      "rewards/rejected": -0.35422074794769287,
      "step": 1420
    },
    {
      "epoch": 3.71,
      "grad_norm": 3.78125,
      "learning_rate": 7.730678442730539e-08,
      "logits/chosen": -1.9422740936279297,
      "logits/rejected": -1.9541749954223633,
      "logps/chosen": -32.07694625854492,
      "logps/rejected": -34.164581298828125,
      "loss": 0.1459,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 2.0887014865875244,
      "rewards/margins": 2.4570722579956055,
      "rewards/rejected": -0.3683708906173706,
      "step": 1430
    },
    {
      "epoch": 3.74,
      "grad_norm": 3.015625,
      "learning_rate": 6.394742864787806e-08,
      "logits/chosen": -1.9317998886108398,
      "logits/rejected": -1.933950424194336,
      "logps/chosen": -30.444339752197266,
      "logps/rejected": -34.485233306884766,
      "loss": 0.1473,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": 1.9348328113555908,
      "rewards/margins": 2.3725178241729736,
      "rewards/rejected": -0.4376850128173828,
      "step": 1440
    },
    {
      "epoch": 3.77,
      "grad_norm": 3.859375,
      "learning_rate": 5.183960310644748e-08,
      "logits/chosen": -1.9155280590057373,
      "logits/rejected": -1.9085607528686523,
      "logps/chosen": -32.7889289855957,
      "logps/rejected": -34.09833526611328,
      "loss": 0.172,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 1.9455804824829102,
      "rewards/margins": 2.1751322746276855,
      "rewards/rejected": -0.22955195605754852,
      "step": 1450
    },
    {
      "epoch": 3.79,
      "grad_norm": 2.875,
      "learning_rate": 4.098952823928693e-08,
      "logits/chosen": -1.884874701499939,
      "logits/rejected": -1.890824556350708,
      "logps/chosen": -28.527862548828125,
      "logps/rejected": -34.241851806640625,
      "loss": 0.159,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": 1.879720687866211,
      "rewards/margins": 2.2800161838531494,
      "rewards/rejected": -0.40029558539390564,
      "step": 1460
    },
    {
      "epoch": 3.82,
      "grad_norm": 3.25,
      "learning_rate": 3.1402778309014284e-08,
      "logits/chosen": -1.8300120830535889,
      "logits/rejected": -1.8280490636825562,
      "logps/chosen": -28.143136978149414,
      "logps/rejected": -30.871578216552734,
      "loss": 0.1654,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 1.8916912078857422,
      "rewards/margins": 2.0858335494995117,
      "rewards/rejected": -0.1941421777009964,
      "step": 1470
    },
    {
      "epoch": 3.84,
      "grad_norm": 3.6875,
      "learning_rate": 2.3084278540791427e-08,
      "logits/chosen": -2.004854440689087,
      "logits/rejected": -1.999352216720581,
      "logps/chosen": -32.60303497314453,
      "logps/rejected": -32.43815994262695,
      "loss": 0.16,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": 1.8775575160980225,
      "rewards/margins": 2.14457368850708,
      "rewards/rejected": -0.26701638102531433,
      "step": 1480
    },
    {
      "epoch": 3.87,
      "grad_norm": 3.140625,
      "learning_rate": 1.6038302591975807e-08,
      "logits/chosen": -1.8964626789093018,
      "logits/rejected": -1.8985786437988281,
      "logps/chosen": -26.187755584716797,
      "logps/rejected": -28.33608627319336,
      "loss": 0.1956,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 1.6020386219024658,
      "rewards/margins": 1.821703314781189,
      "rewards/rejected": -0.21966469287872314,
      "step": 1490
    },
    {
      "epoch": 3.9,
      "grad_norm": 4.375,
      "learning_rate": 1.0268470356514237e-08,
      "logits/chosen": -1.905769944190979,
      "logits/rejected": -1.9003044366836548,
      "logps/chosen": -30.289718627929688,
      "logps/rejected": -32.631587982177734,
      "loss": 0.1694,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": 1.7577499151229858,
      "rewards/margins": 2.221593141555786,
      "rewards/rejected": -0.46384334564208984,
      "step": 1500
    },
    {
      "epoch": 3.9,
      "eval_logits/chosen": -2.1607134342193604,
      "eval_logits/rejected": -2.1560044288635254,
      "eval_logps/chosen": -33.631370544433594,
      "eval_logps/rejected": -37.28603744506836,
      "eval_loss": 0.47470974922180176,
      "eval_rewards/accuracies": 0.5568937063217163,
      "eval_rewards/chosen": 0.2822282910346985,
      "eval_rewards/margins": 0.12082076817750931,
      "eval_rewards/rejected": 0.16140751540660858,
      "eval_runtime": 145.1244,
      "eval_samples_per_second": 2.363,
      "eval_steps_per_second": 0.296,
      "step": 1500
    },
    {
      "epoch": 3.92,
      "grad_norm": 5.78125,
      "learning_rate": 5.777746105209147e-09,
      "logits/chosen": -1.8295776844024658,
      "logits/rejected": -1.8337701559066772,
      "logps/chosen": -31.576879501342773,
      "logps/rejected": -34.6661262512207,
      "loss": 0.1827,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 1.8158810138702393,
      "rewards/margins": 1.994899034500122,
      "rewards/rejected": -0.17901787161827087,
      "step": 1510
    },
    {
      "epoch": 3.95,
      "grad_norm": 2.1875,
      "learning_rate": 2.5684369628148352e-09,
      "logits/chosen": -1.8878551721572876,
      "logits/rejected": -1.8862745761871338,
      "logps/chosen": -28.058696746826172,
      "logps/rejected": -33.05814743041992,
      "loss": 0.1623,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 1.872835397720337,
      "rewards/margins": 2.200737953186035,
      "rewards/rejected": -0.3279026448726654,
      "step": 1520
    },
    {
      "epoch": 3.97,
      "grad_norm": 4.0625,
      "learning_rate": 6.421917227455999e-10,
      "logits/chosen": -1.9854892492294312,
      "logits/rejected": -1.9778953790664673,
      "logps/chosen": -25.75176429748535,
      "logps/rejected": -29.129873275756836,
      "loss": 0.1804,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 1.6178255081176758,
      "rewards/margins": 1.938612699508667,
      "rewards/rejected": -0.3207874298095703,
      "step": 1530
    },
    {
      "epoch": 4.0,
      "grad_norm": 2.921875,
      "learning_rate": 0.0,
      "logits/chosen": -1.8875421285629272,
      "logits/rejected": -1.877659797668457,
      "logps/chosen": -30.428552627563477,
      "logps/rejected": -36.047637939453125,
      "loss": 0.1484,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": 1.8463512659072876,
      "rewards/margins": 2.291874408721924,
      "rewards/rejected": -0.445523202419281,
      "step": 1540
    },
    {
      "epoch": 4.0,
      "step": 1540,
      "total_flos": 0.0,
      "train_loss": 0.17319794591371115,
      "train_runtime": 10768.0107,
      "train_samples_per_second": 1.144,
      "train_steps_per_second": 0.143
    }
  ],
  "logging_steps": 10,
  "max_steps": 1540,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 4,
  "save_steps": 100,
  "total_flos": 0.0,
  "train_batch_size": 4,
  "trial_name": null,
  "trial_params": null
}