{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 4.0,
  "eval_steps": 100,
  "global_step": 1540,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.0,
      "learning_rate": 1.282051282051282e-07,
      "logits/chosen": -1.7278180122375488,
      "logits/rejected": -1.7377450466156006,
      "logps/chosen": -29.553977966308594,
      "logps/rejected": -42.813133239746094,
      "loss": 0.6931,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 1
    },
    {
      "epoch": 0.03,
      "learning_rate": 1.282051282051282e-06,
      "logits/chosen": -1.8663169145584106,
      "logits/rejected": -1.870638370513916,
      "logps/chosen": -36.98221206665039,
      "logps/rejected": -33.6473503112793,
      "loss": 0.6788,
      "rewards/accuracies": 0.5416666865348816,
      "rewards/chosen": 0.01950961910188198,
      "rewards/margins": 0.03332838416099548,
      "rewards/rejected": -0.013818766921758652,
      "step": 10
    },
    {
      "epoch": 0.05,
      "learning_rate": 2.564102564102564e-06,
      "logits/chosen": -1.9978923797607422,
      "logits/rejected": -2.0005345344543457,
      "logps/chosen": -29.642324447631836,
      "logps/rejected": -29.048343658447266,
      "loss": 0.7013,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": -0.00010492801811778918,
      "rewards/margins": -0.012185259722173214,
      "rewards/rejected": 0.012080332264304161,
      "step": 20
    },
    {
      "epoch": 0.08,
      "learning_rate": 3.846153846153847e-06,
      "logits/chosen": -1.9209339618682861,
      "logits/rejected": -1.9182507991790771,
      "logps/chosen": -31.401519775390625,
      "logps/rejected": -33.22309875488281,
      "loss": 0.6891,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": 0.011660982854664326,
      "rewards/margins": 0.015369392931461334,
      "rewards/rejected": -0.003708411008119583,
      "step": 30
    },
    {
      "epoch": 0.1,
      "learning_rate": 4.999896948438434e-06,
      "logits/chosen": -2.0180399417877197,
      "logits/rejected": -2.009289264678955,
      "logps/chosen": -32.559410095214844,
      "logps/rejected": -32.52582550048828,
      "loss": 0.6849,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.013918718323111534,
      "rewards/margins": 0.02444135770201683,
      "rewards/rejected": -0.010522643104195595,
      "step": 40
    },
    {
      "epoch": 0.13,
      "learning_rate": 4.987541037542187e-06,
      "logits/chosen": -1.8629716634750366,
      "logits/rejected": -1.8522107601165771,
      "logps/chosen": -33.554229736328125,
      "logps/rejected": -35.44757080078125,
      "loss": 0.6987,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": 0.002600290346890688,
      "rewards/margins": -0.0012185067171230912,
      "rewards/rejected": 0.003818795783445239,
      "step": 50
    },
    {
      "epoch": 0.16,
      "learning_rate": 4.954691471941119e-06,
      "logits/chosen": -1.9408414363861084,
      "logits/rejected": -1.9427950382232666,
      "logps/chosen": -32.56097412109375,
      "logps/rejected": -33.213417053222656,
      "loss": 0.6725,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": 0.03136637061834335,
      "rewards/margins": 0.06499636918306351,
      "rewards/rejected": -0.03362999111413956,
      "step": 60
    },
    {
      "epoch": 0.18,
      "learning_rate": 4.901618883413549e-06,
      "logits/chosen": -2.072221517562866,
      "logits/rejected": -2.077198028564453,
      "logps/chosen": -33.974578857421875,
      "logps/rejected": -36.629173278808594,
      "loss": 0.6793,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": 0.011124782264232635,
      "rewards/margins": 0.059367585927248,
      "rewards/rejected": -0.04824279993772507,
      "step": 70
    },
    {
      "epoch": 0.21,
      "learning_rate": 4.828760511501322e-06,
      "logits/chosen": -1.9338233470916748,
      "logits/rejected": -1.9369605779647827,
      "logps/chosen": -34.30416488647461,
      "logps/rejected": -34.634437561035156,
      "loss": 0.6423,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": 0.09891629219055176,
      "rewards/margins": 0.132537841796875,
      "rewards/rejected": -0.03362155705690384,
      "step": 80
    },
    {
      "epoch": 0.23,
      "learning_rate": 4.7367166013034295e-06,
      "logits/chosen": -1.9418385028839111,
      "logits/rejected": -1.9463545083999634,
      "logps/chosen": -32.39947509765625,
      "logps/rejected": -32.35419464111328,
      "loss": 0.6785,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.05435952544212341,
      "rewards/margins": 0.05058818310499191,
      "rewards/rejected": 0.0037713423371315002,
      "step": 90
    },
    {
      "epoch": 0.26,
      "learning_rate": 4.626245458345211e-06,
      "logits/chosen": -2.039783000946045,
      "logits/rejected": -2.037789821624756,
      "logps/chosen": -32.164188385009766,
      "logps/rejected": -31.309520721435547,
      "loss": 0.6538,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.061319977045059204,
      "rewards/margins": 0.09596750140190125,
      "rewards/rejected": -0.03464752808213234,
      "step": 100
    },
    {
      "epoch": 0.26,
      "eval_logits/chosen": -2.2338831424713135,
      "eval_logits/rejected": -2.229041814804077,
      "eval_logps/chosen": -34.03020095825195,
      "eval_logps/rejected": -37.52727127075195,
      "eval_loss": 0.6954607963562012,
      "eval_rewards/accuracies": 0.510797381401062,
      "eval_rewards/chosen": 0.0034842013847082853,
      "eval_rewards/margins": 0.012004716321825981,
      "eval_rewards/rejected": -0.008520514704287052,
      "eval_runtime": 146.115,
      "eval_samples_per_second": 2.347,
      "eval_steps_per_second": 0.294,
      "step": 100
    },
    {
      "epoch": 0.29,
      "learning_rate": 4.498257201263691e-06,
      "logits/chosen": -1.9946181774139404,
      "logits/rejected": -1.992236852645874,
      "logps/chosen": -33.117286682128906,
      "logps/rejected": -34.00868225097656,
      "loss": 0.6852,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": 0.10122231394052505,
      "rewards/margins": 0.0846698135137558,
      "rewards/rejected": 0.016552483662962914,
      "step": 110
    },
    {
      "epoch": 0.31,
      "learning_rate": 4.353806263777678e-06,
      "logits/chosen": -2.006875514984131,
      "logits/rejected": -1.9985148906707764,
      "logps/chosen": -32.336421966552734,
      "logps/rejected": -32.137081146240234,
      "loss": 0.675,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": 0.08678452670574188,
      "rewards/margins": 0.06724556535482407,
      "rewards/rejected": 0.019538963213562965,
      "step": 120
    },
    {
      "epoch": 0.34,
      "learning_rate": 4.1940827077152755e-06,
      "logits/chosen": -2.0351502895355225,
      "logits/rejected": -2.02717661857605,
      "logps/chosen": -30.30923843383789,
      "logps/rejected": -32.08501434326172,
      "loss": 0.6398,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": 0.13318516314029694,
      "rewards/margins": 0.15878939628601074,
      "rewards/rejected": -0.025604233145713806,
      "step": 130
    },
    {
      "epoch": 0.36,
      "learning_rate": 4.0204024186666215e-06,
      "logits/chosen": -1.966170072555542,
      "logits/rejected": -1.9764087200164795,
      "logps/chosen": -31.215194702148438,
      "logps/rejected": -32.55674743652344,
      "loss": 0.627,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.16357474029064178,
      "rewards/margins": 0.18106886744499207,
      "rewards/rejected": -0.017494117841124535,
      "step": 140
    },
    {
      "epoch": 0.39,
      "learning_rate": 3.834196265035119e-06,
      "logits/chosen": -1.8775428533554077,
      "logits/rejected": -1.8786998987197876,
      "logps/chosen": -33.92055130004883,
      "logps/rejected": -34.77721405029297,
      "loss": 0.6168,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": 0.21771302819252014,
      "rewards/margins": 0.23367898166179657,
      "rewards/rejected": -0.015965968370437622,
      "step": 150
    },
    {
      "epoch": 0.42,
      "learning_rate": 3.636998309800573e-06,
      "logits/chosen": -1.9291296005249023,
      "logits/rejected": -1.9257177114486694,
      "logps/chosen": -36.01557922363281,
      "logps/rejected": -32.72490692138672,
      "loss": 0.6444,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": 0.12488000094890594,
      "rewards/margins": 0.12660440802574158,
      "rewards/rejected": -0.0017244067275896668,
      "step": 160
    },
    {
      "epoch": 0.44,
      "learning_rate": 3.4304331721118078e-06,
      "logits/chosen": -2.0295331478118896,
      "logits/rejected": -2.0221762657165527,
      "logps/chosen": -33.48248291015625,
      "logps/rejected": -31.408077239990234,
      "loss": 0.5816,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": 0.25230517983436584,
      "rewards/margins": 0.2972865700721741,
      "rewards/rejected": -0.04498137906193733,
      "step": 170
    },
    {
      "epoch": 0.47,
      "learning_rate": 3.2162026428305436e-06,
      "logits/chosen": -2.036190986633301,
      "logits/rejected": -2.0414373874664307,
      "logps/chosen": -32.22594451904297,
      "logps/rejected": -32.46149444580078,
      "loss": 0.5993,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": 0.2550733685493469,
      "rewards/margins": 0.2361568957567215,
      "rewards/rejected": 0.018916476517915726,
      "step": 180
    },
    {
      "epoch": 0.49,
      "learning_rate": 2.996071664294641e-06,
      "logits/chosen": -2.0371222496032715,
      "logits/rejected": -2.0343565940856934,
      "logps/chosen": -31.28468894958496,
      "logps/rejected": -31.336734771728516,
      "loss": 0.6307,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.16341081261634827,
      "rewards/margins": 0.17822694778442383,
      "rewards/rejected": -0.014816122129559517,
      "step": 190
    },
    {
      "epoch": 0.52,
      "learning_rate": 2.7718537898066833e-06,
      "logits/chosen": -1.9067039489746094,
      "logits/rejected": -1.91135573387146,
      "logps/chosen": -31.312374114990234,
      "logps/rejected": -32.82074737548828,
      "loss": 0.6015,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": 0.2351258099079132,
      "rewards/margins": 0.25897616147994995,
      "rewards/rejected": -0.0238503310829401,
      "step": 200
    },
    {
      "epoch": 0.52,
      "eval_logits/chosen": -2.232191324234009,
      "eval_logits/rejected": -2.22735333442688,
      "eval_logps/chosen": -34.060691833496094,
      "eval_logps/rejected": -37.56568908691406,
      "eval_loss": 0.695566713809967,
      "eval_rewards/accuracies": 0.52491694688797,
      "eval_rewards/chosen": -0.020911961793899536,
      "eval_rewards/margins": 0.01834380254149437,
      "eval_rewards/rejected": -0.039255764335393906,
      "eval_runtime": 145.8849,
      "eval_samples_per_second": 2.351,
      "eval_steps_per_second": 0.295,
      "step": 200
    },
    {
      "epoch": 0.55,
      "learning_rate": 2.5453962426402006e-06,
      "logits/chosen": -2.0197176933288574,
      "logits/rejected": -2.0303761959075928,
      "logps/chosen": -31.751026153564453,
      "logps/rejected": -33.96234893798828,
      "loss": 0.5969,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": 0.17622438073158264,
      "rewards/margins": 0.25594404339790344,
      "rewards/rejected": -0.0797196701169014,
      "step": 210
    },
    {
      "epoch": 0.57,
      "learning_rate": 2.3185646976551794e-06,
      "logits/chosen": -1.9122480154037476,
      "logits/rejected": -1.9270412921905518,
      "logps/chosen": -29.86123275756836,
      "logps/rejected": -31.612594604492188,
      "loss": 0.6025,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": 0.20024582743644714,
      "rewards/margins": 0.24377915263175964,
      "rewards/rejected": -0.04353334754705429,
      "step": 220
    },
    {
      "epoch": 0.6,
      "learning_rate": 2.0932279108998323e-06,
      "logits/chosen": -1.9689687490463257,
      "logits/rejected": -1.9729585647583008,
      "logps/chosen": -33.11440658569336,
      "logps/rejected": -31.650421142578125,
      "loss": 0.5809,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.2424498349428177,
      "rewards/margins": 0.3233444094657898,
      "rewards/rejected": -0.08089461922645569,
      "step": 230
    },
    {
      "epoch": 0.62,
      "learning_rate": 1.8712423238279358e-06,
      "logits/chosen": -1.9670231342315674,
      "logits/rejected": -1.9451711177825928,
      "logps/chosen": -33.830162048339844,
      "logps/rejected": -35.1173095703125,
      "loss": 0.558,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": 0.2588713765144348,
      "rewards/margins": 0.3920826315879822,
      "rewards/rejected": -0.13321125507354736,
      "step": 240
    },
    {
      "epoch": 0.65,
      "learning_rate": 1.6544367689701824e-06,
      "logits/chosen": -2.0084152221679688,
      "logits/rejected": -2.005080223083496,
      "logps/chosen": -32.70518493652344,
      "logps/rejected": -36.280517578125,
      "loss": 0.6104,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.1759072244167328,
      "rewards/margins": 0.22038432955741882,
      "rewards/rejected": -0.044477105140686035,
      "step": 250
    },
    {
      "epoch": 0.68,
      "learning_rate": 1.4445974030621963e-06,
      "logits/chosen": -1.8755052089691162,
      "logits/rejected": -1.8730967044830322,
      "logps/chosen": -33.984092712402344,
      "logps/rejected": -35.538455963134766,
      "loss": 0.622,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.16344432532787323,
      "rewards/margins": 0.19051328301429749,
      "rewards/rejected": -0.02706894651055336,
      "step": 260
    },
    {
      "epoch": 0.7,
      "learning_rate": 1.243452991757889e-06,
      "logits/chosen": -1.8608484268188477,
      "logits/rejected": -1.8584181070327759,
      "logps/chosen": -34.17797088623047,
      "logps/rejected": -31.830347061157227,
      "loss": 0.6124,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": 0.1757020801305771,
      "rewards/margins": 0.22291450202465057,
      "rewards/rejected": -0.047212425619363785,
      "step": 270
    },
    {
      "epoch": 0.73,
      "learning_rate": 1.0526606671603523e-06,
      "logits/chosen": -1.9641139507293701,
      "logits/rejected": -1.9535942077636719,
      "logps/chosen": -35.01939010620117,
      "logps/rejected": -31.871440887451172,
      "loss": 0.5852,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.26980119943618774,
      "rewards/margins": 0.29416200518608093,
      "rewards/rejected": -0.024360809475183487,
      "step": 280
    },
    {
      "epoch": 0.75,
      "learning_rate": 8.737922755071455e-07,
      "logits/chosen": -2.0593204498291016,
      "logits/rejected": -2.0443997383117676,
      "logps/chosen": -30.722980499267578,
      "logps/rejected": -32.61235809326172,
      "loss": 0.6599,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.16091887652873993,
      "rewards/margins": 0.12933868169784546,
      "rewards/rejected": 0.031580209732055664,
      "step": 290
    },
    {
      "epoch": 0.78,
      "learning_rate": 7.08321427484816e-07,
      "logits/chosen": -1.930450201034546,
      "logits/rejected": -1.9279005527496338,
      "logps/chosen": -32.415870666503906,
      "logps/rejected": -30.882410049438477,
      "loss": 0.5385,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": 0.4089924395084381,
      "rewards/margins": 0.4614754319190979,
      "rewards/rejected": -0.05248301103711128,
      "step": 300
    },
    {
      "epoch": 0.78,
      "eval_logits/chosen": -2.229433536529541,
      "eval_logits/rejected": -2.22458553314209,
      "eval_logps/chosen": -34.090904235839844,
      "eval_logps/rejected": -37.59726333618164,
      "eval_loss": 0.69569993019104,
      "eval_rewards/accuracies": 0.5398671627044678,
      "eval_rewards/chosen": -0.04508008435368538,
      "eval_rewards/margins": 0.019436603412032127,
      "eval_rewards/rejected": -0.06451668590307236,
      "eval_runtime": 145.8403,
      "eval_samples_per_second": 2.352,
      "eval_steps_per_second": 0.295,
      "step": 300
    },
    {
      "epoch": 0.81,
      "learning_rate": 4.84533120650964e-06,
      "logits/chosen": -1.9146509170532227,
      "logits/rejected": -1.9113744497299194,
      "logps/chosen": -31.32379150390625,
      "logps/rejected": -33.805755615234375,
      "loss": 0.5978,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": 0.2236466407775879,
      "rewards/margins": 0.27452850341796875,
      "rewards/rejected": -0.050881873816251755,
      "step": 310
    },
    {
      "epoch": 0.83,
      "learning_rate": 4.825108134172131e-06,
      "logits/chosen": -1.9630565643310547,
      "logits/rejected": -1.9508411884307861,
      "logps/chosen": -34.32460403442383,
      "logps/rejected": -33.67837905883789,
      "loss": 0.5713,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": 0.217172771692276,
      "rewards/margins": 0.3288002014160156,
      "rewards/rejected": -0.1116274744272232,
      "step": 320
    },
    {
      "epoch": 0.86,
      "learning_rate": 4.80369052967602e-06,
      "logits/chosen": -1.9971084594726562,
      "logits/rejected": -1.995650053024292,
      "logps/chosen": -33.14687728881836,
      "logps/rejected": -32.54724884033203,
      "loss": 0.5766,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": 0.2711312174797058,
      "rewards/margins": 0.32526224851608276,
      "rewards/rejected": -0.05413101240992546,
      "step": 330
    },
    {
      "epoch": 0.88,
      "learning_rate": 4.781089396387968e-06,
      "logits/chosen": -2.0822432041168213,
      "logits/rejected": -2.066549301147461,
      "logps/chosen": -33.73598098754883,
      "logps/rejected": -33.096824645996094,
      "loss": 0.572,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.35473155975341797,
      "rewards/margins": 0.33776286244392395,
      "rewards/rejected": 0.016968706622719765,
      "step": 340
    },
    {
      "epoch": 0.91,
      "learning_rate": 4.757316345716554e-06,
      "logits/chosen": -1.9550272226333618,
      "logits/rejected": -1.9541677236557007,
      "logps/chosen": -32.79063415527344,
      "logps/rejected": -32.5118293762207,
      "loss": 0.5561,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.3931701183319092,
      "rewards/margins": 0.43051546812057495,
      "rewards/rejected": -0.03734535723924637,
      "step": 350
    },
    {
      "epoch": 0.94,
      "learning_rate": 4.73238359114687e-06,
      "logits/chosen": -1.9071505069732666,
      "logits/rejected": -1.917394995689392,
      "logps/chosen": -31.765522003173828,
      "logps/rejected": -35.38612365722656,
      "loss": 0.5497,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.35133686661720276,
      "rewards/margins": 0.4292295575141907,
      "rewards/rejected": -0.07789268344640732,
      "step": 360
    },
    {
      "epoch": 0.96,
      "learning_rate": 4.706303941965804e-06,
      "logits/chosen": -2.042468547821045,
      "logits/rejected": -2.0360734462738037,
      "logps/chosen": -33.295223236083984,
      "logps/rejected": -29.314971923828125,
      "loss": 0.5615,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": 0.28539514541625977,
      "rewards/margins": 0.35623133182525635,
      "rewards/rejected": -0.07083616405725479,
      "step": 370
    },
    {
      "epoch": 0.99,
      "learning_rate": 4.679090796681225e-06,
      "logits/chosen": -1.9002101421356201,
      "logits/rejected": -1.9024240970611572,
      "logps/chosen": -33.61799240112305,
      "logps/rejected": -30.948089599609375,
      "loss": 0.5044,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": 0.49684303998947144,
      "rewards/margins": 0.5572139024734497,
      "rewards/rejected": -0.06037076562643051,
      "step": 380
    },
    {
      "epoch": 1.01,
      "learning_rate": 4.650758136138454e-06,
      "logits/chosen": -1.9271265268325806,
      "logits/rejected": -1.9258983135223389,
      "logps/chosen": -33.7630500793457,
      "logps/rejected": -35.974002838134766,
      "loss": 0.4531,
      "rewards/accuracies": 0.7958333492279053,
      "rewards/chosen": 0.4973909258842468,
      "rewards/margins": 0.7517890930175781,
      "rewards/rejected": -0.2543981671333313,
      "step": 390
    },
    {
      "epoch": 1.04,
      "learning_rate": 4.621320516337559e-06,
      "logits/chosen": -1.8622478246688843,
      "logits/rejected": -1.8538604974746704,
      "logps/chosen": -31.049022674560547,
      "logps/rejected": -36.37419509887695,
      "loss": 0.4017,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 0.5799586772918701,
      "rewards/margins": 0.9268311262130737,
      "rewards/rejected": -0.34687256813049316,
      "step": 400
    },
    {
      "epoch": 1.04,
      "eval_logits/chosen": -2.2100114822387695,
      "eval_logits/rejected": -2.205153226852417,
      "eval_logps/chosen": -34.06473159790039,
      "eval_logps/rejected": -37.634727478027344,
      "eval_loss": 0.6775211691856384,
      "eval_rewards/accuracies": 0.6004983186721802,
      "eval_rewards/chosen": -0.024144915863871574,
      "eval_rewards/margins": 0.07034175843000412,
      "eval_rewards/rejected": -0.09448666870594025,
      "eval_runtime": 146.4012,
      "eval_samples_per_second": 2.343,
      "eval_steps_per_second": 0.294,
      "step": 400
    },
    {
      "epoch": 1.06,
      "learning_rate": 4.590793060955158e-06,
      "logits/chosen": -2.0343263149261475,
      "logits/rejected": -2.037248134613037,
      "logps/chosen": -32.27507019042969,
      "logps/rejected": -35.283382415771484,
      "loss": 0.3721,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 0.6230968236923218,
      "rewards/margins": 1.0155293941497803,
      "rewards/rejected": -0.3924326002597809,
      "step": 410
    },
    {
      "epoch": 1.09,
      "learning_rate": 4.559191453574582e-06,
      "logits/chosen": -1.8720476627349854,
      "logits/rejected": -1.8706938028335571,
      "logps/chosen": -28.4278507232666,
      "logps/rejected": -32.78199005126953,
      "loss": 0.3947,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 0.5117626786231995,
      "rewards/margins": 0.8832231760025024,
      "rewards/rejected": -0.371460497379303,
      "step": 420
    },
    {
      "epoch": 1.12,
      "learning_rate": 4.52653192962838e-06,
      "logits/chosen": -1.8284943103790283,
      "logits/rejected": -1.821554183959961,
      "logps/chosen": -33.15713119506836,
      "logps/rejected": -34.481746673583984,
      "loss": 0.3744,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.733561635017395,
      "rewards/margins": 0.9227407574653625,
      "rewards/rejected": -0.18917915225028992,
      "step": 430
    },
    {
      "epoch": 1.14,
      "learning_rate": 4.492831268057307e-06,
      "logits/chosen": -1.998443841934204,
      "logits/rejected": -1.993316888809204,
      "logps/chosen": -30.946176528930664,
      "logps/rejected": -32.4330940246582,
      "loss": 0.3658,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 0.669696033000946,
      "rewards/margins": 1.0420799255371094,
      "rewards/rejected": -0.3723840117454529,
      "step": 440
    },
    {
      "epoch": 1.17,
      "learning_rate": 4.458106782690094e-06,
      "logits/chosen": -1.8814995288848877,
      "logits/rejected": -1.8857187032699585,
      "logps/chosen": -33.60875701904297,
      "logps/rejected": -33.16690444946289,
      "loss": 0.3134,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.7256715297698975,
      "rewards/margins": 1.1882824897766113,
      "rewards/rejected": -0.4626109004020691,
      "step": 450
    },
    {
      "epoch": 1.19,
      "learning_rate": 4.422376313348405e-06,
      "logits/chosen": -1.8870223760604858,
      "logits/rejected": -1.8813492059707642,
      "logps/chosen": -34.445613861083984,
      "logps/rejected": -35.65070343017578,
      "loss": 0.3011,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.779128909111023,
      "rewards/margins": 1.3404181003570557,
      "rewards/rejected": -0.5612891316413879,
      "step": 460
    },
    {
      "epoch": 1.22,
      "learning_rate": 4.3856582166815696e-06,
      "logits/chosen": -1.9110435247421265,
      "logits/rejected": -1.910775899887085,
      "logps/chosen": -33.23371887207031,
      "logps/rejected": -34.685794830322266,
      "loss": 0.3634,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 0.747688353061676,
      "rewards/margins": 1.1479377746582031,
      "rewards/rejected": -0.40024933218955994,
      "step": 470
    },
    {
      "epoch": 1.25,
      "learning_rate": 4.347971356735789e-06,
      "logits/chosen": -1.9573456048965454,
      "logits/rejected": -1.938490867614746,
      "logps/chosen": -33.21842575073242,
      "logps/rejected": -33.71406555175781,
      "loss": 0.3363,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 0.753872275352478,
      "rewards/margins": 1.2691444158554077,
      "rewards/rejected": -0.5152722001075745,
      "step": 480
    },
    {
      "epoch": 1.27,
      "learning_rate": 4.309335095262675e-06,
      "logits/chosen": -1.9228092432022095,
      "logits/rejected": -1.9221388101577759,
      "logps/chosen": -30.777042388916016,
      "logps/rejected": -31.67429542541504,
      "loss": 0.3684,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 0.6674107313156128,
      "rewards/margins": 1.0100107192993164,
      "rewards/rejected": -0.34259992837905884,
      "step": 490
    },
    {
      "epoch": 1.3,
      "learning_rate": 4.269769281772082e-06,
      "logits/chosen": -1.8837568759918213,
      "logits/rejected": -1.8767935037612915,
      "logps/chosen": -31.731576919555664,
      "logps/rejected": -35.3281364440918,
      "loss": 0.3216,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 0.7508381605148315,
      "rewards/margins": 1.3119536638259888,
      "rewards/rejected": -0.5611153841018677,
      "step": 500
    },
    {
      "epoch": 1.3,
      "eval_logits/chosen": -2.170705556869507,
      "eval_logits/rejected": -2.165867567062378,
      "eval_logps/chosen": -34.18019104003906,
      "eval_logps/rejected": -37.78302764892578,
      "eval_loss": 0.6819708347320557,
      "eval_rewards/accuracies": 0.5768272280693054,
      "eval_rewards/chosen": -0.11651215702295303,
      "eval_rewards/margins": 0.09661299735307693,
      "eval_rewards/rejected": -0.21312516927719116,
      "eval_runtime": 145.937,
      "eval_samples_per_second": 2.35,
      "eval_steps_per_second": 0.295,
      "step": 500
    },
    {
      "epoch": 1.32,
      "learning_rate": 4.22929424333435e-06,
      "logits/chosen": -1.8785709142684937,
      "logits/rejected": -1.8822866678237915,
      "logps/chosen": -28.374109268188477,
      "logps/rejected": -33.53699493408203,
      "loss": 0.3449,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.6405723094940186,
      "rewards/margins": 1.1827360391616821,
      "rewards/rejected": -0.5421636700630188,
      "step": 510
    },
    {
      "epoch": 1.35,
      "learning_rate": 4.1879307741372085e-06,
      "logits/chosen": -1.8827375173568726,
      "logits/rejected": -1.89346444606781,
      "logps/chosen": -32.36341857910156,
      "logps/rejected": -31.319293975830078,
      "loss": 0.3592,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.7191308736801147,
      "rewards/margins": 1.2532163858413696,
      "rewards/rejected": -0.5340853929519653,
      "step": 520
    },
    {
      "epoch": 1.38,
      "learning_rate": 4.145700124802693e-06,
      "logits/chosen": -1.829416036605835,
      "logits/rejected": -1.8270241022109985,
      "logps/chosen": -30.80521011352539,
      "logps/rejected": -30.752178192138672,
      "loss": 0.3535,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 0.6894973516464233,
      "rewards/margins": 1.1498301029205322,
      "rewards/rejected": -0.46033263206481934,
      "step": 530
    },
    {
      "epoch": 1.4,
      "learning_rate": 4.102623991469562e-06,
      "logits/chosen": -1.9103914499282837,
      "logits/rejected": -1.9033727645874023,
      "logps/chosen": -33.29057312011719,
      "logps/rejected": -33.72592544555664,
      "loss": 0.33,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 0.7538042664527893,
      "rewards/margins": 1.3205549716949463,
      "rewards/rejected": -0.5667508244514465,
      "step": 540
    },
    {
      "epoch": 1.43,
      "learning_rate": 4.058724504646834e-06,
      "logits/chosen": -1.8793199062347412,
      "logits/rejected": -1.8857247829437256,
      "logps/chosen": -30.955270767211914,
      "logps/rejected": -33.297035217285156,
      "loss": 0.3587,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 0.6383341550827026,
      "rewards/margins": 1.0689740180969238,
      "rewards/rejected": -0.43063968420028687,
      "step": 550
    },
    {
      "epoch": 1.45,
      "learning_rate": 4.014024217844167e-06,
      "logits/chosen": -1.949784278869629,
      "logits/rejected": -1.9266754388809204,
      "logps/chosen": -30.55877685546875,
      "logps/rejected": -33.56010818481445,
      "loss": 0.3523,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 0.7030326128005981,
      "rewards/margins": 1.1542710065841675,
      "rewards/rejected": -0.45123839378356934,
      "step": 560
    },
    {
      "epoch": 1.48,
      "learning_rate": 3.968546095984911e-06,
      "logits/chosen": -1.883561134338379,
      "logits/rejected": -1.8786439895629883,
      "logps/chosen": -31.494693756103516,
      "logps/rejected": -32.63154220581055,
      "loss": 0.3697,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 0.7376305460929871,
      "rewards/margins": 1.1461317539215088,
      "rewards/rejected": -0.4085012972354889,
      "step": 570
    },
    {
      "epoch": 1.51,
      "learning_rate": 3.922313503607806e-06,
      "logits/chosen": -1.9235492944717407,
      "logits/rejected": -1.9254448413848877,
      "logps/chosen": -33.56511688232422,
      "logps/rejected": -35.665016174316406,
      "loss": 0.3133,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.6485405564308167,
      "rewards/margins": 1.339817762374878,
      "rewards/rejected": -0.6912771463394165,
      "step": 580
    },
    {
      "epoch": 1.53,
      "learning_rate": 3.875350192863368e-06,
      "logits/chosen": -1.9060910940170288,
      "logits/rejected": -1.9054912328720093,
      "logps/chosen": -29.636911392211914,
      "logps/rejected": -32.14686965942383,
      "loss": 0.332,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 0.7641160488128662,
      "rewards/margins": 1.312302827835083,
      "rewards/rejected": -0.5481868386268616,
      "step": 590
    },
    {
      "epoch": 1.56,
      "learning_rate": 3.8276802913111436e-06,
      "logits/chosen": -1.919725775718689,
      "logits/rejected": -1.9174749851226807,
      "logps/chosen": -31.991186141967773,
      "logps/rejected": -32.90464782714844,
      "loss": 0.336,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 0.8056806325912476,
      "rewards/margins": 1.2894303798675537,
      "rewards/rejected": -0.4837496280670166,
      "step": 600
    },
    {
      "epoch": 1.56,
      "eval_logits/chosen": -2.160193920135498,
      "eval_logits/rejected": -2.1554007530212402,
      "eval_logps/chosen": -34.264404296875,
      "eval_logps/rejected": -37.94013214111328,
      "eval_loss": 0.6617584228515625,
      "eval_rewards/accuracies": 0.6241694688796997,
      "eval_rewards/chosen": -0.18387840688228607,
      "eval_rewards/margins": 0.15492965281009674,
      "eval_rewards/rejected": -0.3388080298900604,
      "eval_runtime": 145.9206,
      "eval_samples_per_second": 2.351,
      "eval_steps_per_second": 0.295,
      "step": 600
    },
    {
      "epoch": 1.58,
      "learning_rate": 3.7793282895240927e-06,
      "logits/chosen": -1.9637705087661743,
      "logits/rejected": -1.9703562259674072,
      "logps/chosen": -31.264089584350586,
      "logps/rejected": -32.738319396972656,
      "loss": 0.3311,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 0.7786344289779663,
      "rewards/margins": 1.3187401294708252,
      "rewards/rejected": -0.5401058197021484,
      "step": 610
    },
    {
      "epoch": 1.61,
      "learning_rate": 3.730319028506478e-06,
      "logits/chosen": -1.9196746349334717,
      "logits/rejected": -1.9172741174697876,
      "logps/chosen": -33.669281005859375,
      "logps/rejected": -31.596511840820312,
      "loss": 0.3157,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 0.869280993938446,
      "rewards/margins": 1.3580834865570068,
      "rewards/rejected": -0.48880258202552795,
      "step": 620
    },
    {
      "epoch": 1.64,
      "learning_rate": 3.6806776869317074e-06,
      "logits/chosen": -1.8648065328598022,
      "logits/rejected": -1.8580557107925415,
      "logps/chosen": -34.281700134277344,
      "logps/rejected": -33.09290313720703,
      "loss": 0.3058,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 0.9460186958312988,
      "rewards/margins": 1.5345759391784668,
      "rewards/rejected": -0.5885573029518127,
      "step": 630
    },
    {
      "epoch": 1.66,
      "learning_rate": 3.6304297682067146e-06,
      "logits/chosen": -1.8759320974349976,
      "logits/rejected": -1.8822145462036133,
      "logps/chosen": -32.987449645996094,
      "logps/rejected": -33.8790283203125,
      "loss": 0.3408,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 0.7934818863868713,
      "rewards/margins": 1.3044978380203247,
      "rewards/rejected": -0.5110160708427429,
      "step": 640
    },
    {
      "epoch": 1.69,
      "learning_rate": 3.579601087369492e-06,
      "logits/chosen": -1.9594967365264893,
      "logits/rejected": -1.973659873008728,
      "logps/chosen": -31.017492294311523,
      "logps/rejected": -32.631839752197266,
      "loss": 0.3573,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 0.6679788827896118,
      "rewards/margins": 1.1753003597259521,
      "rewards/rejected": -0.5073216557502747,
      "step": 650
    },
    {
      "epoch": 1.71,
      "learning_rate": 3.5282177578265295e-06,
      "logits/chosen": -1.8336044549942017,
      "logits/rejected": -1.8305423259735107,
      "logps/chosen": -32.8193473815918,
      "logps/rejected": -35.608089447021484,
      "loss": 0.2779,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 0.8828850984573364,
      "rewards/margins": 1.6091547012329102,
      "rewards/rejected": -0.726269543170929,
      "step": 660
    },
    {
      "epoch": 1.74,
      "learning_rate": 3.476306177936961e-06,
      "logits/chosen": -1.9243332147598267,
      "logits/rejected": -1.9244455099105835,
      "logps/chosen": -30.38960838317871,
      "logps/rejected": -34.67184066772461,
      "loss": 0.3114,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 0.7167855501174927,
      "rewards/margins": 1.3931505680084229,
      "rewards/rejected": -0.676365077495575,
      "step": 670
    },
    {
      "epoch": 1.77,
      "learning_rate": 3.423893017450324e-06,
      "logits/chosen": -1.8737255334854126,
      "logits/rejected": -1.8705682754516602,
      "logps/chosen": -29.965240478515625,
      "logps/rejected": -33.58513259887695,
      "loss": 0.3412,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 0.7256845235824585,
      "rewards/margins": 1.339548945426941,
      "rewards/rejected": -0.613864541053772,
      "step": 680
    },
    {
      "epoch": 1.79,
      "learning_rate": 3.3710052038048794e-06,
      "logits/chosen": -1.908151626586914,
      "logits/rejected": -1.908117651939392,
      "logps/chosen": -28.949966430664062,
      "logps/rejected": -31.339040756225586,
      "loss": 0.3023,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 0.9705016016960144,
      "rewards/margins": 1.5053366422653198,
      "rewards/rejected": -0.5348349809646606,
      "step": 690
    },
    {
      "epoch": 1.82,
      "learning_rate": 3.3176699082935546e-06,
      "logits/chosen": -1.8213777542114258,
      "logits/rejected": -1.824627161026001,
      "logps/chosen": -33.35689163208008,
      "logps/rejected": -32.15576934814453,
      "loss": 0.3559,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 0.8972657322883606,
      "rewards/margins": 1.4501286745071411,
      "rewards/rejected": -0.5528628826141357,
      "step": 700
    },
    {
      "epoch": 1.82,
      "eval_logits/chosen": -2.158259153366089,
      "eval_logits/rejected": -2.1534817218780518,
      "eval_logps/chosen": -34.35598373413086,
      "eval_logps/rejected": -37.98072814941406,
      "eval_loss": 0.6946965456008911,
      "eval_rewards/accuracies": 0.5340532064437866,
      "eval_rewards/chosen": -0.2571484446525574,
      "eval_rewards/margins": 0.1141389012336731,
      "eval_rewards/rejected": -0.37128737568855286,
      "eval_runtime": 145.9197,
      "eval_samples_per_second": 2.351,
      "eval_steps_per_second": 0.295,
      "step": 700
    },
    {
      "epoch": 1.84,
      "learning_rate": 3.2639145321045933e-06,
      "logits/chosen": -1.9044933319091797,
      "logits/rejected": -1.8955074548721313,
      "logps/chosen": -35.51199722290039,
      "logps/rejected": -32.60811233520508,
      "loss": 0.3093,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.8000162839889526,
      "rewards/margins": 1.3618541955947876,
      "rewards/rejected": -0.5618377923965454,
      "step": 710
    },
    {
      "epoch": 1.87,
      "learning_rate": 3.2097666922441107e-06,
      "logits/chosen": -1.9182393550872803,
      "logits/rejected": -1.9193944931030273,
      "logps/chosen": -35.33127212524414,
      "logps/rejected": -33.76591110229492,
      "loss": 0.3149,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 0.8832913637161255,
      "rewards/margins": 1.435203194618225,
      "rewards/rejected": -0.5519118309020996,
      "step": 720
    },
    {
      "epoch": 1.9,
      "learning_rate": 3.1552542073477554e-06,
      "logits/chosen": -1.9362226724624634,
      "logits/rejected": -1.9337679147720337,
      "logps/chosen": -31.4454345703125,
      "logps/rejected": -33.67931365966797,
      "loss": 0.3081,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 0.8943157196044922,
      "rewards/margins": 1.4402475357055664,
      "rewards/rejected": -0.5459317564964294,
      "step": 730
    },
    {
      "epoch": 1.92,
      "learning_rate": 3.100405083388799e-06,
      "logits/chosen": -1.9176380634307861,
      "logits/rejected": -1.922844648361206,
      "logps/chosen": -30.585041046142578,
      "logps/rejected": -33.932350158691406,
      "loss": 0.2741,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 0.9605496525764465,
      "rewards/margins": 1.5825778245925903,
      "rewards/rejected": -0.6220281720161438,
      "step": 740
    },
    {
      "epoch": 1.95,
      "learning_rate": 3.0452474992899645e-06,
      "logits/chosen": -1.864728569984436,
      "logits/rejected": -1.8635470867156982,
      "logps/chosen": -32.103065490722656,
      "logps/rejected": -35.512298583984375,
      "loss": 0.3075,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 0.8175305128097534,
      "rewards/margins": 1.478601098060608,
      "rewards/rejected": -0.6610704660415649,
      "step": 750
    },
    {
      "epoch": 1.97,
      "learning_rate": 2.989809792446417e-06,
      "logits/chosen": -1.7420361042022705,
      "logits/rejected": -1.7370550632476807,
      "logps/chosen": -34.88098907470703,
      "logps/rejected": -35.97365951538086,
      "loss": 0.3032,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.8939619064331055,
      "rewards/margins": 1.506162166595459,
      "rewards/rejected": -0.6122000813484192,
      "step": 760
    },
    {
      "epoch": 2.0,
      "learning_rate": 2.9341204441673267e-06,
      "logits/chosen": -1.8814598321914673,
      "logits/rejected": -1.8852901458740234,
      "logps/chosen": -34.113304138183594,
      "logps/rejected": -34.245601654052734,
      "loss": 0.3271,
      "rewards/accuracies": 0.85833340883255,
      "rewards/chosen": 0.8802832365036011,
      "rewards/margins": 1.4008936882019043,
      "rewards/rejected": -0.5206103324890137,
      "step": 770
    },
    {
      "epoch": 2.03,
      "learning_rate": 2.878208065043501e-06,
      "logits/chosen": -1.8257869482040405,
      "logits/rejected": -1.8239891529083252,
      "logps/chosen": -32.38594436645508,
      "logps/rejected": -35.74359893798828,
      "loss": 0.154,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 1.2125210762023926,
      "rewards/margins": 2.2836754322052,
      "rewards/rejected": -1.071154236793518,
      "step": 780
    },
    {
      "epoch": 2.05,
      "learning_rate": 2.8221013802485974e-06,
      "logits/chosen": -1.8724597692489624,
      "logits/rejected": -1.8708446025848389,
      "logps/chosen": -31.859329223632812,
      "logps/rejected": -34.107154846191406,
      "loss": 0.1817,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 1.2131247520446777,
      "rewards/margins": 2.029592752456665,
      "rewards/rejected": -0.8164681196212769,
      "step": 790
    },
    {
      "epoch": 2.08,
      "learning_rate": 2.76582921478147e-06,
      "logits/chosen": -1.8005281686782837,
      "logits/rejected": -1.7946115732192993,
      "logps/chosen": -32.934303283691406,
      "logps/rejected": -32.45271301269531,
      "loss": 0.1978,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 1.1590179204940796,
      "rewards/margins": 1.9497814178466797,
      "rewards/rejected": -0.7907635569572449,
      "step": 800
    },
    {
      "epoch": 2.08,
      "eval_logits/chosen": -2.154881000518799,
      "eval_logits/rejected": -2.1501035690307617,
      "eval_logps/chosen": -34.32504653930664,
      "eval_logps/rejected": -37.97528076171875,
      "eval_loss": 0.6837867498397827,
      "eval_rewards/accuracies": 0.5834717750549316,
      "eval_rewards/chosen": -0.2323925942182541,
      "eval_rewards/margins": 0.1345335692167282,
      "eval_rewards/rejected": -0.3669261634349823,
      "eval_runtime": 145.9433,
      "eval_samples_per_second": 2.35,
      "eval_steps_per_second": 0.295,
      "step": 800
    },
    {
      "epoch": 2.1,
      "learning_rate": 2.7094204786572254e-06,
      "logits/chosen": -1.898033857345581,
      "logits/rejected": -1.905503511428833,
      "logps/chosen": -30.634164810180664,
      "logps/rejected": -35.38495635986328,
      "loss": 0.189,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 1.1952745914459229,
      "rewards/margins": 2.0985639095306396,
      "rewards/rejected": -0.9032894372940063,
      "step": 810
    },
    {
      "epoch": 2.13,
      "learning_rate": 2.6529041520546072e-06,
      "logits/chosen": -1.872579574584961,
      "logits/rejected": -1.875286340713501,
      "logps/chosen": -31.190942764282227,
      "logps/rejected": -34.01335906982422,
      "loss": 0.2433,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 1.104109525680542,
      "rewards/margins": 1.7309566736221313,
      "rewards/rejected": -0.6268469095230103,
      "step": 820
    },
    {
      "epoch": 2.16,
      "learning_rate": 2.5963092704273302e-06,
      "logits/chosen": -1.7723095417022705,
      "logits/rejected": -1.7764465808868408,
      "logps/chosen": -31.214824676513672,
      "logps/rejected": -36.142818450927734,
      "loss": 0.1808,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": 1.1013058423995972,
      "rewards/margins": 2.0874154567718506,
      "rewards/rejected": -0.9861096143722534,
      "step": 830
    },
    {
      "epoch": 2.18,
      "learning_rate": 2.53966490958702e-06,
      "logits/chosen": -1.8381569385528564,
      "logits/rejected": -1.8343050479888916,
      "logps/chosen": -31.591388702392578,
      "logps/rejected": -33.85132598876953,
      "loss": 0.2508,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 1.1018110513687134,
      "rewards/margins": 1.7871520519256592,
      "rewards/rejected": -0.6853411793708801,
      "step": 840
    },
    {
      "epoch": 2.21,
      "learning_rate": 2.4830001707654135e-06,
      "logits/chosen": -1.9189765453338623,
      "logits/rejected": -1.9210529327392578,
      "logps/chosen": -31.255529403686523,
      "logps/rejected": -36.854835510253906,
      "loss": 0.1574,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.2158453464508057,
      "rewards/margins": 2.1701653003692627,
      "rewards/rejected": -0.9543198347091675,
      "step": 850
    },
    {
      "epoch": 2.23,
      "learning_rate": 2.4263441656635054e-06,
      "logits/chosen": -1.7254867553710938,
      "logits/rejected": -1.7197097539901733,
      "logps/chosen": -34.86113357543945,
      "logps/rejected": -33.97612762451172,
      "loss": 0.2044,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 1.1364357471466064,
      "rewards/margins": 2.003814697265625,
      "rewards/rejected": -0.8673788905143738,
      "step": 860
    },
    {
      "epoch": 2.26,
      "learning_rate": 2.3697260014953107e-06,
      "logits/chosen": -1.7829675674438477,
      "logits/rejected": -1.7830712795257568,
      "logps/chosen": -34.304718017578125,
      "logps/rejected": -35.99662780761719,
      "loss": 0.181,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 1.2304695844650269,
      "rewards/margins": 2.150757312774658,
      "rewards/rejected": -0.9202877283096313,
      "step": 870
    },
    {
      "epoch": 2.29,
      "learning_rate": 2.3131747660339396e-06,
      "logits/chosen": -1.8317804336547852,
      "logits/rejected": -1.8203016519546509,
      "logps/chosen": -32.700950622558594,
      "logps/rejected": -34.08301544189453,
      "loss": 0.1692,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": 1.1245036125183105,
      "rewards/margins": 2.0532915592193604,
      "rewards/rejected": -0.9287878274917603,
      "step": 880
    },
    {
      "epoch": 2.31,
      "learning_rate": 2.256719512667651e-06,
      "logits/chosen": -1.9331659078598022,
      "logits/rejected": -1.9377342462539673,
      "logps/chosen": -31.932796478271484,
      "logps/rejected": -33.6646728515625,
      "loss": 0.1905,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": 1.1177093982696533,
      "rewards/margins": 2.100264310836792,
      "rewards/rejected": -0.9825549125671387,
      "step": 890
    },
    {
      "epoch": 2.34,
      "learning_rate": 2.2003892454735786e-06,
      "logits/chosen": -1.8520333766937256,
      "logits/rejected": -1.844834327697754,
      "logps/chosen": -33.29985427856445,
      "logps/rejected": -33.635623931884766,
      "loss": 0.1619,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": 1.243377923965454,
      "rewards/margins": 2.2301688194274902,
      "rewards/rejected": -0.9867912530899048,
      "step": 900
    },
    {
      "epoch": 2.34,
      "eval_logits/chosen": -2.1431386470794678,
      "eval_logits/rejected": -2.138375997543335,
      "eval_logps/chosen": -34.34238052368164,
      "eval_logps/rejected": -38.03608703613281,
      "eval_loss": 0.6788281202316284,
      "eval_rewards/accuracies": 0.5859634280204773,
      "eval_rewards/chosen": -0.24626301229000092,
      "eval_rewards/margins": 0.16930998861789703,
      "eval_rewards/rejected": -0.41557300090789795,
      "eval_runtime": 145.9104,
      "eval_samples_per_second": 2.351,
      "eval_steps_per_second": 0.295,
      "step": 900
    },
    {
      "epoch": 2.36,
      "learning_rate": 2.1442129043167877e-06,
      "logits/chosen": -1.8457978963851929,
      "logits/rejected": -1.8462024927139282,
      "logps/chosen": -29.82485008239746,
      "logps/rejected": -36.06700897216797,
      "loss": 0.1786,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": 1.0738495588302612,
      "rewards/margins": 2.1757030487060547,
      "rewards/rejected": -1.1018531322479248,
      "step": 910
    },
    {
      "epoch": 2.39,
      "learning_rate": 2.088219349982323e-06,
      "logits/chosen": -1.8021042346954346,
      "logits/rejected": -1.7940893173217773,
      "logps/chosen": -30.521142959594727,
      "logps/rejected": -34.798255920410156,
      "loss": 0.1883,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": 1.0542433261871338,
      "rewards/margins": 2.0357508659362793,
      "rewards/rejected": -0.9815075993537903,
      "step": 920
    },
    {
      "epoch": 2.42,
      "learning_rate": 2.0324373493478803e-06,
      "logits/chosen": -1.9680095911026,
      "logits/rejected": -1.968182921409607,
      "logps/chosen": -28.589441299438477,
      "logps/rejected": -33.940185546875,
      "loss": 0.2295,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 1.1260759830474854,
      "rewards/margins": 1.9863367080688477,
      "rewards/rejected": -0.8602607846260071,
      "step": 930
    },
    {
      "epoch": 2.44,
      "learning_rate": 1.976895560604729e-06,
      "logits/chosen": -1.847324013710022,
      "logits/rejected": -1.857102394104004,
      "logps/chosen": -32.901668548583984,
      "logps/rejected": -34.3242073059082,
      "loss": 0.1675,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 1.2187172174453735,
      "rewards/margins": 2.2863640785217285,
      "rewards/rejected": -1.0676472187042236,
      "step": 940
    },
    {
      "epoch": 2.47,
      "learning_rate": 1.921622518534466e-06,
      "logits/chosen": -1.8901779651641846,
      "logits/rejected": -1.8939307928085327,
      "logps/chosen": -29.412405014038086,
      "logps/rejected": -32.94640350341797,
      "loss": 0.2048,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": 0.9882534146308899,
      "rewards/margins": 1.9565131664276123,
      "rewards/rejected": -0.9682596921920776,
      "step": 950
    },
    {
      "epoch": 2.49,
      "learning_rate": 1.8666466198491794e-06,
      "logits/chosen": -1.8851401805877686,
      "logits/rejected": -1.8811264038085938,
      "logps/chosen": -32.44501495361328,
      "logps/rejected": -34.933692932128906,
      "loss": 0.2061,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 1.3041638135910034,
      "rewards/margins": 2.126009225845337,
      "rewards/rejected": -0.8218452334403992,
      "step": 960
    },
    {
      "epoch": 2.52,
      "learning_rate": 1.8119961086025376e-06,
      "logits/chosen": -1.8027584552764893,
      "logits/rejected": -1.805301308631897,
      "logps/chosen": -31.277942657470703,
      "logps/rejected": -36.393043518066406,
      "loss": 0.1663,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": 1.2278459072113037,
      "rewards/margins": 2.197005033493042,
      "rewards/rejected": -0.9691593050956726,
      "step": 970
    },
    {
      "epoch": 2.55,
      "learning_rate": 1.7576990616793139e-06,
      "logits/chosen": -1.835371971130371,
      "logits/rejected": -1.8288919925689697,
      "logps/chosen": -34.11837387084961,
      "logps/rejected": -37.60203552246094,
      "loss": 0.2066,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 1.0293549299240112,
      "rewards/margins": 2.101419687271118,
      "rewards/rejected": -1.0720646381378174,
      "step": 980
    },
    {
      "epoch": 2.57,
      "learning_rate": 1.7037833743707892e-06,
      "logits/chosen": -1.813190221786499,
      "logits/rejected": -1.807956337928772,
      "logps/chosen": -29.670574188232422,
      "logps/rejected": -37.203330993652344,
      "loss": 0.2099,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 1.06589937210083,
      "rewards/margins": 2.0366604328155518,
      "rewards/rejected": -0.970760703086853,
      "step": 990
    },
    {
      "epoch": 2.6,
      "learning_rate": 1.6502767460434588e-06,
      "logits/chosen": -1.7904030084609985,
      "logits/rejected": -1.779764175415039,
      "logps/chosen": -30.418493270874023,
      "logps/rejected": -30.5950984954834,
      "loss": 0.209,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 1.0641578435897827,
      "rewards/margins": 1.8604481220245361,
      "rewards/rejected": -0.796290397644043,
      "step": 1000
    },
    {
      "epoch": 2.6,
      "eval_logits/chosen": -2.1356685161590576,
      "eval_logits/rejected": -2.130932331085205,
      "eval_logps/chosen": -34.380455017089844,
      "eval_logps/rejected": -38.083457946777344,
      "eval_loss": 0.6776629090309143,
      "eval_rewards/accuracies": 0.5917773842811584,
      "eval_rewards/chosen": -0.2767248749732971,
      "eval_rewards/margins": 0.17674396932125092,
      "eval_rewards/rejected": -0.45346885919570923,
      "eval_runtime": 145.9185,
      "eval_samples_per_second": 2.351,
      "eval_steps_per_second": 0.295,
      "step": 1000
    },
    {
      "epoch": 2.62,
      "learning_rate": 1.5972066659083796e-06,
      "logits/chosen": -1.896877646446228,
      "logits/rejected": -1.896228551864624,
      "logps/chosen": -30.421009063720703,
      "logps/rejected": -31.305410385131836,
      "loss": 0.22,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 1.1225507259368896,
      "rewards/margins": 1.9600980281829834,
      "rewards/rejected": -0.8375473022460938,
      "step": 1010
    },
    {
      "epoch": 2.65,
      "learning_rate": 1.5446003988985041e-06,
      "logits/chosen": -1.9368255138397217,
      "logits/rejected": -1.9375327825546265,
      "logps/chosen": -30.484455108642578,
      "logps/rejected": -32.34059524536133,
      "loss": 0.177,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": 1.167297601699829,
      "rewards/margins": 2.1071255207061768,
      "rewards/rejected": -0.9398279190063477,
      "step": 1020
    },
    {
      "epoch": 2.68,
      "learning_rate": 1.4924849716612211e-06,
      "logits/chosen": -1.9036754369735718,
      "logits/rejected": -1.9070818424224854,
      "logps/chosen": -30.951061248779297,
      "logps/rejected": -28.893712997436523,
      "loss": 0.227,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 1.0632069110870361,
      "rewards/margins": 1.7912060022354126,
      "rewards/rejected": -0.7279989719390869,
      "step": 1030
    },
    {
      "epoch": 2.7,
      "learning_rate": 1.440887158673332e-06,
      "logits/chosen": -1.91388738155365,
      "logits/rejected": -1.906005859375,
      "logps/chosen": -29.750720977783203,
      "logps/rejected": -34.767372131347656,
      "loss": 0.1882,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": 1.1417419910430908,
      "rewards/margins": 2.1554553508758545,
      "rewards/rejected": -1.0137132406234741,
      "step": 1040
    },
    {
      "epoch": 2.73,
      "learning_rate": 1.3898334684855647e-06,
      "logits/chosen": -1.8500654697418213,
      "logits/rejected": -1.8609867095947266,
      "logps/chosen": -32.1203727722168,
      "logps/rejected": -33.35521697998047,
      "loss": 0.1865,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": 1.1461875438690186,
      "rewards/margins": 2.034356117248535,
      "rewards/rejected": -0.888168454170227,
      "step": 1050
    },
    {
      "epoch": 2.75,
      "learning_rate": 1.3393501301037245e-06,
      "logits/chosen": -1.9280004501342773,
      "logits/rejected": -1.9186464548110962,
      "logps/chosen": -32.13129425048828,
      "logps/rejected": -37.68938446044922,
      "loss": 0.2127,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 1.0955873727798462,
      "rewards/margins": 2.181983470916748,
      "rewards/rejected": -1.0863959789276123,
      "step": 1060
    },
    {
      "epoch": 2.78,
      "learning_rate": 1.2894630795134454e-06,
      "logits/chosen": -1.835701584815979,
      "logits/rejected": -1.837551474571228,
      "logps/chosen": -34.348480224609375,
      "logps/rejected": -33.834922790527344,
      "loss": 0.1815,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 1.3126420974731445,
      "rewards/margins": 2.2222840785980225,
      "rewards/rejected": -0.9096420407295227,
      "step": 1070
    },
    {
      "epoch": 2.81,
      "learning_rate": 1.2401979463554984e-06,
      "logits/chosen": -1.9658441543579102,
      "logits/rejected": -1.9667274951934814,
      "logps/chosen": -31.538421630859375,
      "logps/rejected": -35.54855728149414,
      "loss": 0.1617,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": 1.2153749465942383,
      "rewards/margins": 2.3616721630096436,
      "rewards/rejected": -1.1462972164154053,
      "step": 1080
    },
    {
      "epoch": 2.83,
      "learning_rate": 1.1915800407584705e-06,
      "logits/chosen": -1.9372714757919312,
      "logits/rejected": -1.9413518905639648,
      "logps/chosen": -29.44711685180664,
      "logps/rejected": -34.69606399536133,
      "loss": 0.2085,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 1.156960129737854,
      "rewards/margins": 2.0755460262298584,
      "rewards/rejected": -0.9185860753059387,
      "step": 1090
    },
    {
      "epoch": 2.86,
      "learning_rate": 1.1436343403356019e-06,
      "logits/chosen": -1.9265114068984985,
      "logits/rejected": -1.9317893981933594,
      "logps/chosen": -32.267982482910156,
      "logps/rejected": -31.293163299560547,
      "loss": 0.2513,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 1.0408172607421875,
      "rewards/margins": 1.7540076971054077,
      "rewards/rejected": -0.7131900191307068,
      "step": 1100
    },
    {
      "epoch": 2.86,
      "eval_logits/chosen": -2.1317365169525146,
      "eval_logits/rejected": -2.126985549926758,
      "eval_logps/chosen": -34.40774917602539,
      "eval_logps/rejected": -38.09048080444336,
      "eval_loss": 0.6897340416908264,
      "eval_rewards/accuracies": 0.5830564498901367,
      "eval_rewards/chosen": -0.2985561192035675,
      "eval_rewards/margins": 0.1605302393436432,
      "eval_rewards/rejected": -0.4590863585472107,
      "eval_runtime": 145.9092,
      "eval_samples_per_second": 2.351,
      "eval_steps_per_second": 0.295,
      "step": 1100
    },
    {
      "epoch": 2.88,
      "learning_rate": 1.0963854773524548e-06,
      "logits/chosen": -1.918264389038086,
      "logits/rejected": -1.9183082580566406,
      "logps/chosen": -31.37972068786621,
      "logps/rejected": -31.99541664123535,
      "loss": 0.1933,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": 1.248342514038086,
      "rewards/margins": 2.031942844390869,
      "rewards/rejected": -0.783600389957428,
      "step": 1110
    },
    {
      "epoch": 2.91,
      "learning_rate": 1.049857726072005e-06,
      "logits/chosen": -1.7625865936279297,
      "logits/rejected": -1.7643556594848633,
      "logps/chosen": -33.15352249145508,
      "logps/rejected": -34.140689849853516,
      "loss": 0.2375,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 1.1291639804840088,
      "rewards/margins": 2.008704900741577,
      "rewards/rejected": -0.8795410394668579,
      "step": 1120
    },
    {
      "epoch": 2.94,
      "learning_rate": 1.0040749902836508e-06,
      "logits/chosen": -1.7994060516357422,
      "logits/rejected": -1.7968616485595703,
      "logps/chosen": -29.835479736328125,
      "logps/rejected": -32.159576416015625,
      "loss": 0.2885,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 0.940991222858429,
      "rewards/margins": 1.736061692237854,
      "rewards/rejected": -0.7950705289840698,
      "step": 1130
    },
    {
      "epoch": 2.96,
      "learning_rate": 9.59060791022566e-07,
      "logits/chosen": -1.9358510971069336,
      "logits/rejected": -1.9303522109985352,
      "logps/chosen": -31.38555335998535,
      "logps/rejected": -34.0118408203125,
      "loss": 0.1779,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 1.3233940601348877,
      "rewards/margins": 2.1696605682373047,
      "rewards/rejected": -0.8462664484977722,
      "step": 1140
    },
    {
      "epoch": 2.99,
      "learning_rate": 9.148382544856885e-07,
      "logits/chosen": -1.7935917377471924,
      "logits/rejected": -1.783705472946167,
      "logps/chosen": -32.11836624145508,
      "logps/rejected": -32.378910064697266,
      "loss": 0.2083,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 1.066697120666504,
      "rewards/margins": 1.9359056949615479,
      "rewards/rejected": -0.8692083358764648,
      "step": 1150
    },
    {
      "epoch": 3.01,
      "learning_rate": 8.714301001505568e-07,
      "logits/chosen": -1.8640037775039673,
      "logits/rejected": -1.8639949560165405,
      "logps/chosen": -32.08525848388672,
      "logps/rejected": -32.31430435180664,
      "loss": 0.1695,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": 1.2771949768066406,
      "rewards/margins": 2.167479991912842,
      "rewards/rejected": -0.8902848958969116,
      "step": 1160
    },
    {
      "epoch": 3.04,
      "learning_rate": 8.288586291031025e-07,
      "logits/chosen": -1.9403139352798462,
      "logits/rejected": -1.9347970485687256,
      "logps/chosen": -32.035804748535156,
      "logps/rejected": -34.015647888183594,
      "loss": 0.186,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": 1.1921296119689941,
      "rewards/margins": 2.0516738891601562,
      "rewards/rejected": -0.8595443964004517,
      "step": 1170
    },
    {
      "epoch": 3.06,
      "learning_rate": 7.871457125803897e-07,
      "logits/chosen": -1.7985414266586304,
      "logits/rejected": -1.806396484375,
      "logps/chosen": -31.918773651123047,
      "logps/rejected": -33.53784942626953,
      "loss": 0.175,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": 1.1313796043395996,
      "rewards/margins": 2.0725083351135254,
      "rewards/rejected": -0.9411287307739258,
      "step": 1180
    },
    {
      "epoch": 3.09,
      "learning_rate": 7.463127807341966e-07,
      "logits/chosen": -1.86725652217865,
      "logits/rejected": -1.8614327907562256,
      "logps/chosen": -30.524410247802734,
      "logps/rejected": -34.46692657470703,
      "loss": 0.1668,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 1.319364309310913,
      "rewards/margins": 2.266146659851074,
      "rewards/rejected": -0.9467819929122925,
      "step": 1190
    },
    {
      "epoch": 3.12,
      "learning_rate": 7.063808116212021e-07,
      "logits/chosen": -1.8097419738769531,
      "logits/rejected": -1.8118005990982056,
      "logps/chosen": -32.011409759521484,
      "logps/rejected": -34.247711181640625,
      "loss": 0.1713,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 1.1832258701324463,
      "rewards/margins": 2.373357057571411,
      "rewards/rejected": -1.1901309490203857,
      "step": 1200
    },
    {
      "epoch": 3.12,
      "eval_logits/chosen": -2.131728410720825,
      "eval_logits/rejected": -2.1269805431365967,
      "eval_logps/chosen": -34.38140869140625,
      "eval_logps/rejected": -38.09336471557617,
      "eval_loss": 0.6780257821083069,
      "eval_rewards/accuracies": 0.594684362411499,
      "eval_rewards/chosen": -0.27748534083366394,
      "eval_rewards/margins": 0.18390974402427673,
      "eval_rewards/rejected": -0.4613950550556183,
      "eval_runtime": 145.9474,
      "eval_samples_per_second": 2.35,
      "eval_steps_per_second": 0.295,
      "step": 1200
    },
    {
      "epoch": 3.14,
      "learning_rate": 6.673703204254348e-07,
      "logits/chosen": -1.7440783977508545,
      "logits/rejected": -1.7430063486099243,
      "logps/chosen": -34.408180236816406,
      "logps/rejected": -34.03881072998047,
      "loss": 0.1517,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": 1.3154469728469849,
      "rewards/margins": 2.3790054321289062,
      "rewards/rejected": -1.0635584592819214,
      "step": 1210
    },
    {
      "epoch": 3.17,
      "learning_rate": 6.293013489185315e-07,
      "logits/chosen": -1.9152580499649048,
      "logits/rejected": -1.9091880321502686,
      "logps/chosen": -30.193584442138672,
      "logps/rejected": -34.31517791748047,
      "loss": 0.1516,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": 1.2080823183059692,
      "rewards/margins": 2.357390880584717,
      "rewards/rejected": -1.149308443069458,
      "step": 1220
    },
    {
      "epoch": 3.19,
      "learning_rate": 5.921934551632086e-07,
      "logits/chosen": -1.7672933340072632,
      "logits/rejected": -1.7558486461639404,
      "logps/chosen": -32.647369384765625,
      "logps/rejected": -34.192893981933594,
      "loss": 0.1462,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 1.3906490802764893,
      "rewards/margins": 2.429901599884033,
      "rewards/rejected": -1.0392528772354126,
      "step": 1230
    },
    {
      "epoch": 3.22,
      "learning_rate": 5.560657034652405e-07,
      "logits/chosen": -1.856805443763733,
      "logits/rejected": -1.850097894668579,
      "logps/chosen": -29.289770126342773,
      "logps/rejected": -30.042938232421875,
      "loss": 0.1928,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": 1.1283700466156006,
      "rewards/margins": 2.0686745643615723,
      "rewards/rejected": -0.9403042793273926,
      "step": 1240
    },
    {
      "epoch": 3.25,
      "learning_rate": 5.2093665457911e-07,
      "logits/chosen": -1.8852243423461914,
      "logits/rejected": -1.8920814990997314,
      "logps/chosen": -33.63641357421875,
      "logps/rejected": -32.55451202392578,
      "loss": 0.1642,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 1.4071805477142334,
      "rewards/margins": 2.2361385822296143,
      "rewards/rejected": -0.8289580345153809,
      "step": 1250
    },
    {
      "epoch": 3.27,
      "learning_rate": 4.868243561723535e-07,
      "logits/chosen": -1.8602968454360962,
      "logits/rejected": -1.8607813119888306,
      "logps/chosen": -31.587940216064453,
      "logps/rejected": -34.207176208496094,
      "loss": 0.1465,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": 1.1524927616119385,
      "rewards/margins": 2.2663419246673584,
      "rewards/rejected": -1.1138492822647095,
      "step": 1260
    },
    {
      "epoch": 3.3,
      "learning_rate": 4.537463335535161e-07,
      "logits/chosen": -1.800450086593628,
      "logits/rejected": -1.7998168468475342,
      "logps/chosen": -31.380176544189453,
      "logps/rejected": -34.821746826171875,
      "loss": 0.151,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 1.3685301542282104,
      "rewards/margins": 2.4094412326812744,
      "rewards/rejected": -1.0409109592437744,
      "step": 1270
    },
    {
      "epoch": 3.32,
      "learning_rate": 4.217195806684629e-07,
      "logits/chosen": -1.686762809753418,
      "logits/rejected": -1.6821815967559814,
      "logps/chosen": -33.52080535888672,
      "logps/rejected": -32.1031494140625,
      "loss": 0.1522,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 1.3733112812042236,
      "rewards/margins": 2.314640760421753,
      "rewards/rejected": -0.9413297772407532,
      "step": 1280
    },
    {
      "epoch": 3.35,
      "learning_rate": 3.907605513696808e-07,
      "logits/chosen": -1.8902908563613892,
      "logits/rejected": -1.8748180866241455,
      "logps/chosen": -32.840858459472656,
      "logps/rejected": -36.358482360839844,
      "loss": 0.1254,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": 1.2053766250610352,
      "rewards/margins": 2.4799063205718994,
      "rewards/rejected": -1.274530053138733,
      "step": 1290
    },
    {
      "epoch": 3.38,
      "learning_rate": 3.6088515096305675e-07,
      "logits/chosen": -1.83368718624115,
      "logits/rejected": -1.8386014699935913,
      "logps/chosen": -31.894006729125977,
      "logps/rejected": -37.62579345703125,
      "loss": 0.1199,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": 1.298893690109253,
      "rewards/margins": 2.5169498920440674,
      "rewards/rejected": -1.2180559635162354,
      "step": 1300
    },
    {
      "epoch": 3.38,
      "eval_logits/chosen": -2.131683349609375,
      "eval_logits/rejected": -2.1269302368164062,
      "eval_logps/chosen": -34.37530517578125,
      "eval_logps/rejected": -38.09721374511719,
      "eval_loss": 0.6739928126335144,
      "eval_rewards/accuracies": 0.5980066657066345,
      "eval_rewards/chosen": -0.2726014256477356,
      "eval_rewards/margins": 0.19187170267105103,
      "eval_rewards/rejected": -0.464473158121109,
      "eval_runtime": 145.964,
      "eval_samples_per_second": 2.35,
      "eval_steps_per_second": 0.295,
      "step": 1300
    },
    {
      "epoch": 3.4,
      "learning_rate": 3.321087280364757e-07,
      "logits/chosen": -1.8072227239608765,
      "logits/rejected": -1.8072770833969116,
      "logps/chosen": -34.529273986816406,
      "logps/rejected": -38.355064392089844,
      "loss": 0.165,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 1.37179696559906,
      "rewards/margins": 2.388556957244873,
      "rewards/rejected": -1.0167598724365234,
      "step": 1310
    },
    {
      "epoch": 3.43,
      "learning_rate": 3.044460665744284e-07,
      "logits/chosen": -1.8941234350204468,
      "logits/rejected": -1.8927080631256104,
      "logps/chosen": -30.5273494720459,
      "logps/rejected": -32.267601013183594,
      "loss": 0.1563,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": 1.2532024383544922,
      "rewards/margins": 2.304598093032837,
      "rewards/rejected": -1.0513951778411865,
      "step": 1320
    },
    {
      "epoch": 3.45,
      "learning_rate": 2.779113783626916e-07,
      "logits/chosen": -1.8009631633758545,
      "logits/rejected": -1.8024797439575195,
      "logps/chosen": -32.74066925048828,
      "logps/rejected": -34.96391296386719,
      "loss": 0.1456,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": 1.306864619255066,
      "rewards/margins": 2.3590540885925293,
      "rewards/rejected": -1.052189588546753,
      "step": 1330
    },
    {
      "epoch": 3.48,
      "learning_rate": 2.5251829568697204e-07,
      "logits/chosen": -1.8623323440551758,
      "logits/rejected": -1.8611522912979126,
      "logps/chosen": -29.76908302307129,
      "logps/rejected": -33.332550048828125,
      "loss": 0.1572,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": 1.1436678171157837,
      "rewards/margins": 2.21612811088562,
      "rewards/rejected": -1.072460412979126,
      "step": 1340
    },
    {
      "epoch": 3.51,
      "learning_rate": 2.2827986432927774e-07,
      "logits/chosen": -1.877587914466858,
      "logits/rejected": -1.862841248512268,
      "logps/chosen": -32.96019744873047,
      "logps/rejected": -37.72450637817383,
      "loss": 0.153,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": 1.0851805210113525,
      "rewards/margins": 2.253241539001465,
      "rewards/rejected": -1.1680610179901123,
      "step": 1350
    },
    {
      "epoch": 3.53,
      "learning_rate": 2.0520853686560177e-07,
      "logits/chosen": -1.8752319812774658,
      "logits/rejected": -1.8875735998153687,
      "logps/chosen": -30.424901962280273,
      "logps/rejected": -33.42066192626953,
      "loss": 0.1772,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 1.3148540258407593,
      "rewards/margins": 2.2595856189727783,
      "rewards/rejected": -0.9447315335273743,
      "step": 1360
    },
    {
      "epoch": 3.56,
      "learning_rate": 1.833161662683672e-07,
      "logits/chosen": -1.9719635248184204,
      "logits/rejected": -1.971278429031372,
      "logps/chosen": -30.404891967773438,
      "logps/rejected": -37.444129943847656,
      "loss": 0.1268,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": 1.300431489944458,
      "rewards/margins": 2.5910286903381348,
      "rewards/rejected": -1.2905975580215454,
      "step": 1370
    },
    {
      "epoch": 3.58,
      "learning_rate": 1.626139998169246e-07,
      "logits/chosen": -1.8381322622299194,
      "logits/rejected": -1.845894455909729,
      "logps/chosen": -32.695735931396484,
      "logps/rejected": -38.82695388793945,
      "loss": 0.1845,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 1.1877927780151367,
      "rewards/margins": 2.2896358966827393,
      "rewards/rejected": -1.101843237876892,
      "step": 1380
    },
    {
      "epoch": 3.61,
      "learning_rate": 1.4311267331922535e-07,
      "logits/chosen": -1.7921359539031982,
      "logits/rejected": -1.7879730463027954,
      "logps/chosen": -32.99811553955078,
      "logps/rejected": -32.61560821533203,
      "loss": 0.1972,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 1.4152759313583374,
      "rewards/margins": 2.229846715927124,
      "rewards/rejected": -0.8145707845687866,
      "step": 1390
    },
    {
      "epoch": 3.64,
      "learning_rate": 1.2482220564763669e-07,
      "logits/chosen": -1.9499454498291016,
      "logits/rejected": -1.9474738836288452,
      "logps/chosen": -29.77544593811035,
      "logps/rejected": -33.36935806274414,
      "loss": 0.1578,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": 1.2168843746185303,
      "rewards/margins": 2.1967949867248535,
      "rewards/rejected": -0.979910671710968,
      "step": 1400
    },
    {
      "epoch": 3.64,
      "eval_logits/chosen": -2.131394863128662,
      "eval_logits/rejected": -2.126645803451538,
      "eval_logps/chosen": -34.39291763305664,
      "eval_logps/rejected": -38.091670989990234,
      "eval_loss": 0.6838791370391846,
      "eval_rewards/accuracies": 0.5859634280204773,
      "eval_rewards/chosen": -0.28668972849845886,
      "eval_rewards/margins": 0.17335131764411926,
      "eval_rewards/rejected": -0.4600410461425781,
      "eval_runtime": 145.9578,
      "eval_samples_per_second": 2.35,
      "eval_steps_per_second": 0.295,
      "step": 1400
    },
    {
      "epoch": 3.66,
      "learning_rate": 1.0775199359171346e-07,
      "logits/chosen": -1.8938592672348022,
      "logits/rejected": -1.8896372318267822,
      "logps/chosen": -32.033363342285156,
      "logps/rejected": -30.647994995117188,
      "loss": 0.1724,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": 1.2873361110687256,
      "rewards/margins": 2.162166118621826,
      "rewards/rejected": -0.8748300671577454,
      "step": 1410
    },
    {
      "epoch": 3.69,
      "learning_rate": 9.191080703056604e-08,
      "logits/chosen": -1.8500086069107056,
      "logits/rejected": -1.8511066436767578,
      "logps/chosen": -31.751754760742188,
      "logps/rejected": -35.97992706298828,
      "loss": 0.1799,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": 1.29049813747406,
      "rewards/margins": 2.1352338790893555,
      "rewards/rejected": -0.8447355031967163,
      "step": 1420
    },
    {
      "epoch": 3.71,
      "learning_rate": 7.730678442730539e-08,
      "logits/chosen": -1.8036447763442993,
      "logits/rejected": -1.7969812154769897,
      "logps/chosen": -32.511940002441406,
      "logps/rejected": -38.00764846801758,
      "loss": 0.1569,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": 1.303862452507019,
      "rewards/margins": 2.358455181121826,
      "rewards/rejected": -1.0545929670333862,
      "step": 1430
    },
    {
      "epoch": 3.74,
      "learning_rate": 6.394742864787806e-08,
      "logits/chosen": -1.8134548664093018,
      "logits/rejected": -1.8079309463500977,
      "logps/chosen": -27.75140380859375,
      "logps/rejected": -32.93821334838867,
      "loss": 0.1916,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 1.2494693994522095,
      "rewards/margins": 2.2155184745788574,
      "rewards/rejected": -0.9660491943359375,
      "step": 1440
    },
    {
      "epoch": 3.77,
      "learning_rate": 5.183960310644748e-08,
      "logits/chosen": -1.8413394689559937,
      "logits/rejected": -1.8308031558990479,
      "logps/chosen": -31.208871841430664,
      "logps/rejected": -36.86098861694336,
      "loss": 0.1533,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": 1.0181248188018799,
      "rewards/margins": 2.2786247730255127,
      "rewards/rejected": -1.260500192642212,
      "step": 1450
    },
    {
      "epoch": 3.79,
      "learning_rate": 4.098952823928693e-08,
      "logits/chosen": -1.8131767511367798,
      "logits/rejected": -1.8103145360946655,
      "logps/chosen": -31.82034683227539,
      "logps/rejected": -31.843425750732422,
      "loss": 0.2074,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 1.1915438175201416,
      "rewards/margins": 1.9536281824111938,
      "rewards/rejected": -0.7620843052864075,
      "step": 1460
    },
    {
      "epoch": 3.82,
      "learning_rate": 3.1402778309014284e-08,
      "logits/chosen": -1.8745816946029663,
      "logits/rejected": -1.8810014724731445,
      "logps/chosen": -30.35885238647461,
      "logps/rejected": -34.228118896484375,
      "loss": 0.1567,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": 1.3682581186294556,
      "rewards/margins": 2.380812883377075,
      "rewards/rejected": -1.01255464553833,
      "step": 1470
    },
    {
      "epoch": 3.84,
      "learning_rate": 2.3084278540791427e-08,
      "logits/chosen": -1.8742589950561523,
      "logits/rejected": -1.8845386505126953,
      "logps/chosen": -30.218097686767578,
      "logps/rejected": -30.740955352783203,
      "loss": 0.168,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": 1.1817275285720825,
      "rewards/margins": 2.0716397762298584,
      "rewards/rejected": -0.889912486076355,
      "step": 1480
    },
    {
      "epoch": 3.87,
      "learning_rate": 1.6038302591975807e-08,
      "logits/chosen": -1.8073488473892212,
      "logits/rejected": -1.800286054611206,
      "logps/chosen": -32.302825927734375,
      "logps/rejected": -33.47874069213867,
      "loss": 0.1842,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": 1.082926869392395,
      "rewards/margins": 2.097539186477661,
      "rewards/rejected": -1.0146124362945557,
      "step": 1490
    },
    {
      "epoch": 3.9,
      "learning_rate": 1.0268470356514237e-08,
      "logits/chosen": -1.8606147766113281,
      "logits/rejected": -1.8575241565704346,
      "logps/chosen": -32.205726623535156,
      "logps/rejected": -34.68817138671875,
      "loss": 0.1614,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": 1.1358753442764282,
      "rewards/margins": 2.2782340049743652,
      "rewards/rejected": -1.142358660697937,
      "step": 1500
    },
    {
      "epoch": 3.9,
      "eval_logits/chosen": -2.1312177181243896,
      "eval_logits/rejected": -2.1264684200286865,
      "eval_logps/chosen": -34.386234283447266,
      "eval_logps/rejected": -38.08888244628906,
      "eval_loss": 0.6819679737091064,
      "eval_rewards/accuracies": 0.574335515499115,
      "eval_rewards/chosen": -0.2813448905944824,
      "eval_rewards/margins": 0.17646357417106628,
      "eval_rewards/rejected": -0.4578084647655487,
      "eval_runtime": 145.9567,
      "eval_samples_per_second": 2.35,
      "eval_steps_per_second": 0.295,
      "step": 1500
    },
    {
      "epoch": 3.92,
      "learning_rate": 5.777746105209147e-09,
      "logits/chosen": -1.9368705749511719,
      "logits/rejected": -1.9374831914901733,
      "logps/chosen": -28.38448715209961,
      "logps/rejected": -34.06378173828125,
      "loss": 0.2003,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 1.145431399345398,
      "rewards/margins": 2.204097032546997,
      "rewards/rejected": -1.0586655139923096,
      "step": 1510
    },
    {
      "epoch": 3.95,
      "learning_rate": 2.5684369628148352e-09,
      "logits/chosen": -1.7932342290878296,
      "logits/rejected": -1.7930355072021484,
      "logps/chosen": -31.60140037536621,
      "logps/rejected": -35.09097671508789,
      "loss": 0.2089,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 1.1523739099502563,
      "rewards/margins": 2.1261672973632812,
      "rewards/rejected": -0.9737933874130249,
      "step": 1520
    },
    {
      "epoch": 3.97,
      "learning_rate": 6.421917227455999e-10,
      "logits/chosen": -1.943830132484436,
      "logits/rejected": -1.9411506652832031,
      "logps/chosen": -30.156597137451172,
      "logps/rejected": -32.75658416748047,
      "loss": 0.1839,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": 1.053972840309143,
      "rewards/margins": 2.0723206996917725,
      "rewards/rejected": -1.018348217010498,
      "step": 1530
    },
    {
      "epoch": 4.0,
      "learning_rate": 0.0,
      "logits/chosen": -1.9267715215682983,
      "logits/rejected": -1.9278953075408936,
      "logps/chosen": -29.27459716796875,
      "logps/rejected": -30.95477294921875,
      "loss": 0.1802,
      "rewards/accuracies": 0.98333340883255,
      "rewards/chosen": 1.0762757062911987,
      "rewards/margins": 2.007948398590088,
      "rewards/rejected": -0.9316726922988892,
      "step": 1540
    },
    {
      "epoch": 4.0,
      "step": 1540,
      "total_flos": 0.0,
      "train_loss": 0.20605388523696305,
      "train_runtime": 10804.1609,
      "train_samples_per_second": 1.14,
      "train_steps_per_second": 0.143
    }
  ],
  "logging_steps": 10,
  "max_steps": 1540,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 4,
  "save_steps": 100,
  "total_flos": 0.0,
  "train_batch_size": 4,
  "trial_name": null,
  "trial_params": null
}