{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 1.0,
  "eval_steps": 500,
  "global_step": 805,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.0,
      "learning_rate": 6.172839506172839e-09,
      "logits/chosen": -2.8421168327331543,
      "logits/rejected": -2.6747336387634277,
      "logps/chosen": -92.33953094482422,
      "logps/rejected": -44.262760162353516,
      "loss": 0.6931,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 1
    },
    {
      "epoch": 0.01,
      "learning_rate": 6.172839506172839e-08,
      "logits/chosen": -2.9818074703216553,
      "logits/rejected": -2.9741735458374023,
      "logps/chosen": -197.26390075683594,
      "logps/rejected": -149.23976135253906,
      "loss": 0.6794,
      "rewards/accuracies": 0.5277777910232544,
      "rewards/chosen": 0.020052114501595497,
      "rewards/margins": 0.0660661831498146,
      "rewards/rejected": -0.04601407051086426,
      "step": 10
    },
    {
      "epoch": 0.02,
      "learning_rate": 1.2345679012345677e-07,
      "logits/chosen": -2.9329466819763184,
      "logits/rejected": -2.8775200843811035,
      "logps/chosen": -172.8914337158203,
      "logps/rejected": -133.22866821289062,
      "loss": 0.5158,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.48435258865356445,
      "rewards/margins": 0.6018184423446655,
      "rewards/rejected": -0.11746580898761749,
      "step": 20
    },
    {
      "epoch": 0.04,
      "learning_rate": 1.8518518518518516e-07,
      "logits/chosen": -3.0486252307891846,
      "logits/rejected": -3.003458023071289,
      "logps/chosen": -178.92950439453125,
      "logps/rejected": -143.7101287841797,
      "loss": 0.3374,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 0.6527904272079468,
      "rewards/margins": 1.396834135055542,
      "rewards/rejected": -0.7440436482429504,
      "step": 30
    },
    {
      "epoch": 0.05,
      "learning_rate": 2.4691358024691354e-07,
      "logits/chosen": -2.970651388168335,
      "logits/rejected": -2.966358184814453,
      "logps/chosen": -174.35009765625,
      "logps/rejected": -139.04750061035156,
      "loss": 0.2663,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": 0.44792431592941284,
      "rewards/margins": 2.088245391845703,
      "rewards/rejected": -1.6403213739395142,
      "step": 40
    },
    {
      "epoch": 0.06,
      "learning_rate": 3.086419753086419e-07,
      "logits/chosen": -2.9904656410217285,
      "logits/rejected": -2.920414686203003,
      "logps/chosen": -173.70179748535156,
      "logps/rejected": -138.68955993652344,
      "loss": 0.1817,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 0.6215466856956482,
      "rewards/margins": 2.8885910511016846,
      "rewards/rejected": -2.2670440673828125,
      "step": 50
    },
    {
      "epoch": 0.07,
      "learning_rate": 3.703703703703703e-07,
      "logits/chosen": -3.000448703765869,
      "logits/rejected": -2.8979151248931885,
      "logps/chosen": -145.14614868164062,
      "logps/rejected": -112.61439514160156,
      "loss": 0.1912,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.05986971780657768,
      "rewards/margins": 3.269381046295166,
      "rewards/rejected": -3.2095108032226562,
      "step": 60
    },
    {
      "epoch": 0.09,
      "learning_rate": 4.320987654320987e-07,
      "logits/chosen": -2.9225759506225586,
      "logits/rejected": -2.8518130779266357,
      "logps/chosen": -157.19439697265625,
      "logps/rejected": -119.10911560058594,
      "loss": 0.1713,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 0.39277735352516174,
      "rewards/margins": 4.21900749206543,
      "rewards/rejected": -3.82623028755188,
      "step": 70
    },
    {
      "epoch": 0.1,
      "learning_rate": 4.938271604938271e-07,
      "logits/chosen": -2.842589855194092,
      "logits/rejected": -2.805922031402588,
      "logps/chosen": -168.32815551757812,
      "logps/rejected": -141.83216857910156,
      "loss": 0.1762,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.5003746151924133,
      "rewards/margins": 5.050185203552246,
      "rewards/rejected": -5.550559043884277,
      "step": 80
    },
    {
      "epoch": 0.11,
      "learning_rate": 4.937845303867402e-07,
      "logits/chosen": -2.9922776222229004,
      "logits/rejected": -2.970557928085327,
      "logps/chosen": -172.38787841796875,
      "logps/rejected": -144.65328979492188,
      "loss": 0.144,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -2.053427219390869,
      "rewards/margins": 4.2831034660339355,
      "rewards/rejected": -6.336531639099121,
      "step": 90
    },
    {
      "epoch": 0.12,
      "learning_rate": 4.868784530386741e-07,
      "logits/chosen": -2.897087574005127,
      "logits/rejected": -2.9255499839782715,
      "logps/chosen": -183.1656951904297,
      "logps/rejected": -155.22946166992188,
      "loss": 0.1584,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 0.06980714946985245,
      "rewards/margins": 5.842454433441162,
      "rewards/rejected": -5.772646903991699,
      "step": 100
    },
    {
      "epoch": 0.14,
      "learning_rate": 4.799723756906077e-07,
      "logits/chosen": -3.016488790512085,
      "logits/rejected": -2.9143893718719482,
      "logps/chosen": -182.10484313964844,
      "logps/rejected": -151.207275390625,
      "loss": 0.185,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.8662151098251343,
      "rewards/margins": 4.957070350646973,
      "rewards/rejected": -6.823284149169922,
      "step": 110
    },
    {
      "epoch": 0.15,
      "learning_rate": 4.730662983425414e-07,
      "logits/chosen": -2.906358242034912,
      "logits/rejected": -2.8886070251464844,
      "logps/chosen": -162.4820556640625,
      "logps/rejected": -144.28274536132812,
      "loss": 0.2087,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.8441675901412964,
      "rewards/margins": 5.8418402671813965,
      "rewards/rejected": -6.686008453369141,
      "step": 120
    },
    {
      "epoch": 0.16,
      "learning_rate": 4.661602209944751e-07,
      "logits/chosen": -3.044167995452881,
      "logits/rejected": -2.989940881729126,
      "logps/chosen": -192.1724090576172,
      "logps/rejected": -160.28482055664062,
      "loss": 0.1281,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -1.2753368616104126,
      "rewards/margins": 5.963367462158203,
      "rewards/rejected": -7.238703727722168,
      "step": 130
    },
    {
      "epoch": 0.17,
      "learning_rate": 4.592541436464088e-07,
      "logits/chosen": -2.9158730506896973,
      "logits/rejected": -2.8665575981140137,
      "logps/chosen": -176.7301788330078,
      "logps/rejected": -157.2458038330078,
      "loss": 0.121,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 0.29454001784324646,
      "rewards/margins": 7.677084445953369,
      "rewards/rejected": -7.38254451751709,
      "step": 140
    },
    {
      "epoch": 0.19,
      "learning_rate": 4.5234806629834255e-07,
      "logits/chosen": -2.906966209411621,
      "logits/rejected": -2.878993034362793,
      "logps/chosen": -194.67013549804688,
      "logps/rejected": -177.0902557373047,
      "loss": 0.1565,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -2.4568727016448975,
      "rewards/margins": 5.8625688552856445,
      "rewards/rejected": -8.319440841674805,
      "step": 150
    },
    {
      "epoch": 0.2,
      "learning_rate": 4.454419889502762e-07,
      "logits/chosen": -3.035737991333008,
      "logits/rejected": -2.9880356788635254,
      "logps/chosen": -185.5491943359375,
      "logps/rejected": -158.82350158691406,
      "loss": 0.1474,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -1.9501594305038452,
      "rewards/margins": 6.810189247131348,
      "rewards/rejected": -8.76034927368164,
      "step": 160
    },
    {
      "epoch": 0.21,
      "learning_rate": 4.3853591160220993e-07,
      "logits/chosen": -3.051605463027954,
      "logits/rejected": -2.9627249240875244,
      "logps/chosen": -188.94488525390625,
      "logps/rejected": -159.45440673828125,
      "loss": 0.1317,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -3.886648178100586,
      "rewards/margins": 6.363480567932129,
      "rewards/rejected": -10.250128746032715,
      "step": 170
    },
    {
      "epoch": 0.22,
      "learning_rate": 4.3162983425414365e-07,
      "logits/chosen": -2.9437613487243652,
      "logits/rejected": -2.923678398132324,
      "logps/chosen": -197.8041229248047,
      "logps/rejected": -179.19410705566406,
      "loss": 0.1784,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.9531164169311523,
      "rewards/margins": 5.519814968109131,
      "rewards/rejected": -8.472932815551758,
      "step": 180
    },
    {
      "epoch": 0.24,
      "learning_rate": 4.247237569060773e-07,
      "logits/chosen": -2.9907093048095703,
      "logits/rejected": -2.91261625289917,
      "logps/chosen": -183.89828491210938,
      "logps/rejected": -151.09255981445312,
      "loss": 0.1128,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.34959203004837036,
      "rewards/margins": 6.793131351470947,
      "rewards/rejected": -7.142723083496094,
      "step": 190
    },
    {
      "epoch": 0.25,
      "learning_rate": 4.1781767955801103e-07,
      "logits/chosen": -3.0003714561462402,
      "logits/rejected": -2.9367072582244873,
      "logps/chosen": -161.82725524902344,
      "logps/rejected": -141.76551818847656,
      "loss": 0.1327,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -1.736950159072876,
      "rewards/margins": 6.6674089431762695,
      "rewards/rejected": -8.404359817504883,
      "step": 200
    },
    {
      "epoch": 0.26,
      "learning_rate": 4.1091160220994475e-07,
      "logits/chosen": -2.9085640907287598,
      "logits/rejected": -2.866804361343384,
      "logps/chosen": -186.4488983154297,
      "logps/rejected": -154.49505615234375,
      "loss": 0.0995,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -2.524721622467041,
      "rewards/margins": 6.8670477867126465,
      "rewards/rejected": -9.391769409179688,
      "step": 210
    },
    {
      "epoch": 0.27,
      "learning_rate": 4.0400552486187846e-07,
      "logits/chosen": -2.9839043617248535,
      "logits/rejected": -2.905033588409424,
      "logps/chosen": -196.44332885742188,
      "logps/rejected": -162.53839111328125,
      "loss": 0.1511,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -1.4525755643844604,
      "rewards/margins": 6.9121527671813965,
      "rewards/rejected": -8.364728927612305,
      "step": 220
    },
    {
      "epoch": 0.29,
      "learning_rate": 3.970994475138121e-07,
      "logits/chosen": -2.9412007331848145,
      "logits/rejected": -2.897495746612549,
      "logps/chosen": -169.09982299804688,
      "logps/rejected": -140.54649353027344,
      "loss": 0.1487,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.6759728193283081,
      "rewards/margins": 5.637655258178711,
      "rewards/rejected": -6.313628196716309,
      "step": 230
    },
    {
      "epoch": 0.3,
      "learning_rate": 3.9019337016574584e-07,
      "logits/chosen": -2.9135284423828125,
      "logits/rejected": -2.9134836196899414,
      "logps/chosen": -182.82675170898438,
      "logps/rejected": -172.9473419189453,
      "loss": 0.1548,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -2.683964490890503,
      "rewards/margins": 7.280943393707275,
      "rewards/rejected": -9.964908599853516,
      "step": 240
    },
    {
      "epoch": 0.31,
      "learning_rate": 3.832872928176795e-07,
      "logits/chosen": -2.9248108863830566,
      "logits/rejected": -2.929591655731201,
      "logps/chosen": -170.30747985839844,
      "logps/rejected": -158.3248291015625,
      "loss": 0.0943,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.5063244104385376,
      "rewards/margins": 8.612382888793945,
      "rewards/rejected": -9.118706703186035,
      "step": 250
    },
    {
      "epoch": 0.32,
      "learning_rate": 3.763812154696133e-07,
      "logits/chosen": -2.9988207817077637,
      "logits/rejected": -2.945227861404419,
      "logps/chosen": -176.06637573242188,
      "logps/rejected": -152.9449005126953,
      "loss": 0.1231,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.20524081587791443,
      "rewards/margins": 7.8652777671813965,
      "rewards/rejected": -7.660037994384766,
      "step": 260
    },
    {
      "epoch": 0.34,
      "learning_rate": 3.6947513812154694e-07,
      "logits/chosen": -3.014315366744995,
      "logits/rejected": -2.928999185562134,
      "logps/chosen": -171.6766357421875,
      "logps/rejected": -147.11154174804688,
      "loss": 0.1415,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": 1.9650657176971436,
      "rewards/margins": 6.9074506759643555,
      "rewards/rejected": -4.942384243011475,
      "step": 270
    },
    {
      "epoch": 0.35,
      "learning_rate": 3.6256906077348066e-07,
      "logits/chosen": -2.907224178314209,
      "logits/rejected": -2.906038761138916,
      "logps/chosen": -182.75013732910156,
      "logps/rejected": -167.35330200195312,
      "loss": 0.1677,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.3685176968574524,
      "rewards/margins": 7.3049635887146,
      "rewards/rejected": -6.936445713043213,
      "step": 280
    },
    {
      "epoch": 0.36,
      "learning_rate": 3.556629834254143e-07,
      "logits/chosen": -2.899120330810547,
      "logits/rejected": -2.84541654586792,
      "logps/chosen": -148.26084899902344,
      "logps/rejected": -133.62619018554688,
      "loss": 0.1569,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": 0.6295042037963867,
      "rewards/margins": 6.114689350128174,
      "rewards/rejected": -5.485184669494629,
      "step": 290
    },
    {
      "epoch": 0.37,
      "learning_rate": 3.4875690607734804e-07,
      "logits/chosen": -2.965310573577881,
      "logits/rejected": -2.8868935108184814,
      "logps/chosen": -181.91659545898438,
      "logps/rejected": -153.7183380126953,
      "loss": 0.1737,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": 0.35049858689308167,
      "rewards/margins": 7.459444999694824,
      "rewards/rejected": -7.108946323394775,
      "step": 300
    },
    {
      "epoch": 0.39,
      "learning_rate": 3.418508287292817e-07,
      "logits/chosen": -2.978998899459839,
      "logits/rejected": -2.9149255752563477,
      "logps/chosen": -195.01406860351562,
      "logps/rejected": -172.47183227539062,
      "loss": 0.1305,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -8.093232154846191,
      "rewards/margins": 7.233319282531738,
      "rewards/rejected": -15.326550483703613,
      "step": 310
    },
    {
      "epoch": 0.4,
      "learning_rate": 3.3494475138121547e-07,
      "logits/chosen": -2.976937770843506,
      "logits/rejected": -2.8859190940856934,
      "logps/chosen": -199.7006072998047,
      "logps/rejected": -168.48184204101562,
      "loss": 0.094,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -6.615779876708984,
      "rewards/margins": 7.833823204040527,
      "rewards/rejected": -14.449602127075195,
      "step": 320
    },
    {
      "epoch": 0.41,
      "learning_rate": 3.280386740331492e-07,
      "logits/chosen": -2.887190103530884,
      "logits/rejected": -2.877593517303467,
      "logps/chosen": -167.93466186523438,
      "logps/rejected": -152.63137817382812,
      "loss": 0.1647,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -7.1772050857543945,
      "rewards/margins": 6.209255218505859,
      "rewards/rejected": -13.386459350585938,
      "step": 330
    },
    {
      "epoch": 0.42,
      "learning_rate": 3.2113259668508285e-07,
      "logits/chosen": -3.028519868850708,
      "logits/rejected": -2.9872825145721436,
      "logps/chosen": -209.1421661376953,
      "logps/rejected": -189.716796875,
      "loss": 0.1479,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -8.37966251373291,
      "rewards/margins": 6.489211082458496,
      "rewards/rejected": -14.868873596191406,
      "step": 340
    },
    {
      "epoch": 0.43,
      "learning_rate": 3.1422651933701657e-07,
      "logits/chosen": -2.91877818107605,
      "logits/rejected": -2.86418080329895,
      "logps/chosen": -188.1709747314453,
      "logps/rejected": -169.12794494628906,
      "loss": 0.12,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -8.10500431060791,
      "rewards/margins": 7.924190521240234,
      "rewards/rejected": -16.029193878173828,
      "step": 350
    },
    {
      "epoch": 0.45,
      "learning_rate": 3.0732044198895023e-07,
      "logits/chosen": -2.9679653644561768,
      "logits/rejected": -2.9573211669921875,
      "logps/chosen": -191.16941833496094,
      "logps/rejected": -175.7834014892578,
      "loss": 0.1287,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -6.326230525970459,
      "rewards/margins": 7.559299468994141,
      "rewards/rejected": -13.885531425476074,
      "step": 360
    },
    {
      "epoch": 0.46,
      "learning_rate": 3.00414364640884e-07,
      "logits/chosen": -2.9169762134552,
      "logits/rejected": -2.834723949432373,
      "logps/chosen": -178.90940856933594,
      "logps/rejected": -150.42605590820312,
      "loss": 0.0809,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -4.185718059539795,
      "rewards/margins": 8.697784423828125,
      "rewards/rejected": -12.883502006530762,
      "step": 370
    },
    {
      "epoch": 0.47,
      "learning_rate": 2.9350828729281767e-07,
      "logits/chosen": -3.0018362998962402,
      "logits/rejected": -2.9604392051696777,
      "logps/chosen": -170.78225708007812,
      "logps/rejected": -154.3968505859375,
      "loss": 0.1239,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -2.6663613319396973,
      "rewards/margins": 8.932355880737305,
      "rewards/rejected": -11.598716735839844,
      "step": 380
    },
    {
      "epoch": 0.48,
      "learning_rate": 2.866022099447514e-07,
      "logits/chosen": -2.994075298309326,
      "logits/rejected": -2.9975550174713135,
      "logps/chosen": -215.20736694335938,
      "logps/rejected": -194.09291076660156,
      "loss": 0.1248,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -3.196017265319824,
      "rewards/margins": 10.618438720703125,
      "rewards/rejected": -13.81445598602295,
      "step": 390
    },
    {
      "epoch": 0.5,
      "learning_rate": 2.7969613259668505e-07,
      "logits/chosen": -3.0385982990264893,
      "logits/rejected": -2.966886043548584,
      "logps/chosen": -178.92239379882812,
      "logps/rejected": -152.96282958984375,
      "loss": 0.1334,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -1.9255645275115967,
      "rewards/margins": 9.632636070251465,
      "rewards/rejected": -11.55820083618164,
      "step": 400
    },
    {
      "epoch": 0.51,
      "learning_rate": 2.7279005524861877e-07,
      "logits/chosen": -3.092007637023926,
      "logits/rejected": -2.971647024154663,
      "logps/chosen": -212.3997039794922,
      "logps/rejected": -171.35873413085938,
      "loss": 0.1333,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -2.5890955924987793,
      "rewards/margins": 8.974533081054688,
      "rewards/rejected": -11.563628196716309,
      "step": 410
    },
    {
      "epoch": 0.52,
      "learning_rate": 2.6588397790055243e-07,
      "logits/chosen": -2.9924330711364746,
      "logits/rejected": -2.9680898189544678,
      "logps/chosen": -187.8248291015625,
      "logps/rejected": -179.09518432617188,
      "loss": 0.1164,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -2.1534762382507324,
      "rewards/margins": 8.748167991638184,
      "rewards/rejected": -10.901644706726074,
      "step": 420
    },
    {
      "epoch": 0.53,
      "learning_rate": 2.589779005524862e-07,
      "logits/chosen": -3.084371566772461,
      "logits/rejected": -3.0335347652435303,
      "logps/chosen": -200.3054962158203,
      "logps/rejected": -174.8027801513672,
      "loss": 0.1358,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.3474791646003723,
      "rewards/margins": 8.830670356750488,
      "rewards/rejected": -9.17814826965332,
      "step": 430
    },
    {
      "epoch": 0.55,
      "learning_rate": 2.5207182320441986e-07,
      "logits/chosen": -3.0466129779815674,
      "logits/rejected": -2.9966206550598145,
      "logps/chosen": -204.1211395263672,
      "logps/rejected": -184.7081298828125,
      "loss": 0.106,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.979719340801239,
      "rewards/margins": 8.62094783782959,
      "rewards/rejected": -9.600667953491211,
      "step": 440
    },
    {
      "epoch": 0.56,
      "learning_rate": 2.451657458563536e-07,
      "logits/chosen": -3.0778112411499023,
      "logits/rejected": -3.046935558319092,
      "logps/chosen": -193.56973266601562,
      "logps/rejected": -168.67868041992188,
      "loss": 0.0973,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -1.372416615486145,
      "rewards/margins": 8.402765274047852,
      "rewards/rejected": -9.775181770324707,
      "step": 450
    },
    {
      "epoch": 0.57,
      "learning_rate": 2.3825966850828727e-07,
      "logits/chosen": -2.967545986175537,
      "logits/rejected": -2.895456314086914,
      "logps/chosen": -157.90182495117188,
      "logps/rejected": -134.09254455566406,
      "loss": 0.1293,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": 0.49174195528030396,
      "rewards/margins": 9.59979248046875,
      "rewards/rejected": -9.108049392700195,
      "step": 460
    },
    {
      "epoch": 0.58,
      "learning_rate": 2.31353591160221e-07,
      "logits/chosen": -3.0117809772491455,
      "logits/rejected": -2.9402668476104736,
      "logps/chosen": -190.03530883789062,
      "logps/rejected": -168.36068725585938,
      "loss": 0.0976,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.2254009246826172,
      "rewards/margins": 9.530010223388672,
      "rewards/rejected": -9.755411148071289,
      "step": 470
    },
    {
      "epoch": 0.6,
      "learning_rate": 2.2444751381215468e-07,
      "logits/chosen": -3.088365077972412,
      "logits/rejected": -2.996295928955078,
      "logps/chosen": -197.9468994140625,
      "logps/rejected": -172.51962280273438,
      "loss": 0.0996,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -3.3391284942626953,
      "rewards/margins": 8.23055648803711,
      "rewards/rejected": -11.569684982299805,
      "step": 480
    },
    {
      "epoch": 0.61,
      "learning_rate": 2.175414364640884e-07,
      "logits/chosen": -2.968456983566284,
      "logits/rejected": -2.961946964263916,
      "logps/chosen": -186.1309051513672,
      "logps/rejected": -169.68812561035156,
      "loss": 0.1217,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -0.4508451521396637,
      "rewards/margins": 11.474364280700684,
      "rewards/rejected": -11.925207138061523,
      "step": 490
    },
    {
      "epoch": 0.62,
      "learning_rate": 2.1063535911602208e-07,
      "logits/chosen": -2.978132724761963,
      "logits/rejected": -2.919520616531372,
      "logps/chosen": -196.43170166015625,
      "logps/rejected": -174.66552734375,
      "loss": 0.1262,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -5.012697696685791,
      "rewards/margins": 8.436281204223633,
      "rewards/rejected": -13.448979377746582,
      "step": 500
    },
    {
      "epoch": 0.62,
      "eval_logits/chosen": -2.9003899097442627,
      "eval_logits/rejected": -2.85654616355896,
      "eval_logps/chosen": -188.17869567871094,
      "eval_logps/rejected": -171.17919921875,
      "eval_loss": 0.118459552526474,
      "eval_rewards/accuracies": 0.8670212626457214,
      "eval_rewards/chosen": -3.0113587379455566,
      "eval_rewards/margins": 9.170669555664062,
      "eval_rewards/rejected": -12.182028770446777,
      "eval_runtime": 242.1612,
      "eval_samples_per_second": 6.174,
      "eval_steps_per_second": 0.194,
      "step": 500
    },
    {
      "epoch": 0.63,
      "learning_rate": 2.0372928176795578e-07,
      "logits/chosen": -2.944871425628662,
      "logits/rejected": -2.9157793521881104,
      "logps/chosen": -180.36338806152344,
      "logps/rejected": -159.6240692138672,
      "loss": 0.1055,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -2.646656036376953,
      "rewards/margins": 9.775663375854492,
      "rewards/rejected": -12.422318458557129,
      "step": 510
    },
    {
      "epoch": 0.65,
      "learning_rate": 1.968232044198895e-07,
      "logits/chosen": -3.010129690170288,
      "logits/rejected": -2.9667623043060303,
      "logps/chosen": -171.2137908935547,
      "logps/rejected": -152.80848693847656,
      "loss": 0.1331,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -3.5951690673828125,
      "rewards/margins": 7.591134548187256,
      "rewards/rejected": -11.186304092407227,
      "step": 520
    },
    {
      "epoch": 0.66,
      "learning_rate": 1.899171270718232e-07,
      "logits/chosen": -3.0541915893554688,
      "logits/rejected": -2.9794061183929443,
      "logps/chosen": -179.92555236816406,
      "logps/rejected": -154.8466339111328,
      "loss": 0.1356,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -2.0221731662750244,
      "rewards/margins": 8.44990062713623,
      "rewards/rejected": -10.472073554992676,
      "step": 530
    },
    {
      "epoch": 0.67,
      "learning_rate": 1.830110497237569e-07,
      "logits/chosen": -2.930157423019409,
      "logits/rejected": -2.9100921154022217,
      "logps/chosen": -208.7513885498047,
      "logps/rejected": -196.99765014648438,
      "loss": 0.1238,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -4.048346519470215,
      "rewards/margins": 8.839378356933594,
      "rewards/rejected": -12.887723922729492,
      "step": 540
    },
    {
      "epoch": 0.68,
      "learning_rate": 1.7610497237569062e-07,
      "logits/chosen": -2.9872794151306152,
      "logits/rejected": -2.9665157794952393,
      "logps/chosen": -181.928955078125,
      "logps/rejected": -173.3523712158203,
      "loss": 0.1339,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -2.8346304893493652,
      "rewards/margins": 8.800179481506348,
      "rewards/rejected": -11.634809494018555,
      "step": 550
    },
    {
      "epoch": 0.7,
      "learning_rate": 1.691988950276243e-07,
      "logits/chosen": -3.0336689949035645,
      "logits/rejected": -3.022378444671631,
      "logps/chosen": -197.01950073242188,
      "logps/rejected": -182.48306274414062,
      "loss": 0.1319,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -2.3179187774658203,
      "rewards/margins": 9.856887817382812,
      "rewards/rejected": -12.174806594848633,
      "step": 560
    },
    {
      "epoch": 0.71,
      "learning_rate": 1.62292817679558e-07,
      "logits/chosen": -2.9352331161499023,
      "logits/rejected": -2.8333239555358887,
      "logps/chosen": -183.40594482421875,
      "logps/rejected": -157.8167266845703,
      "loss": 0.0838,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -3.595484495162964,
      "rewards/margins": 9.1626615524292,
      "rewards/rejected": -12.758146286010742,
      "step": 570
    },
    {
      "epoch": 0.72,
      "learning_rate": 1.5538674033149171e-07,
      "logits/chosen": -3.005654811859131,
      "logits/rejected": -2.9100375175476074,
      "logps/chosen": -184.67526245117188,
      "logps/rejected": -154.762451171875,
      "loss": 0.1174,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -1.7798635959625244,
      "rewards/margins": 10.110002517700195,
      "rewards/rejected": -11.889866828918457,
      "step": 580
    },
    {
      "epoch": 0.73,
      "learning_rate": 1.484806629834254e-07,
      "logits/chosen": -3.069862127304077,
      "logits/rejected": -2.978449583053589,
      "logps/chosen": -209.2169647216797,
      "logps/rejected": -170.54354858398438,
      "loss": 0.122,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -2.5865066051483154,
      "rewards/margins": 8.170557022094727,
      "rewards/rejected": -10.757063865661621,
      "step": 590
    },
    {
      "epoch": 0.75,
      "learning_rate": 1.4157458563535912e-07,
      "logits/chosen": -3.0297770500183105,
      "logits/rejected": -2.956226110458374,
      "logps/chosen": -179.11427307128906,
      "logps/rejected": -155.3955535888672,
      "loss": 0.1675,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -2.389403820037842,
      "rewards/margins": 7.711805820465088,
      "rewards/rejected": -10.10120964050293,
      "step": 600
    },
    {
      "epoch": 0.76,
      "learning_rate": 1.346685082872928e-07,
      "logits/chosen": -2.928321599960327,
      "logits/rejected": -2.89027738571167,
      "logps/chosen": -214.89675903320312,
      "logps/rejected": -190.57569885253906,
      "loss": 0.0856,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -1.7383911609649658,
      "rewards/margins": 10.676383018493652,
      "rewards/rejected": -12.414773941040039,
      "step": 610
    },
    {
      "epoch": 0.77,
      "learning_rate": 1.277624309392265e-07,
      "logits/chosen": -3.0483570098876953,
      "logits/rejected": -2.9306087493896484,
      "logps/chosen": -180.202880859375,
      "logps/rejected": -148.0863037109375,
      "loss": 0.1345,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -3.625720977783203,
      "rewards/margins": 7.224545955657959,
      "rewards/rejected": -10.85026741027832,
      "step": 620
    },
    {
      "epoch": 0.78,
      "learning_rate": 1.2085635359116022e-07,
      "logits/chosen": -3.087991237640381,
      "logits/rejected": -3.0041158199310303,
      "logps/chosen": -198.82481384277344,
      "logps/rejected": -169.78836059570312,
      "loss": 0.1162,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -2.9254348278045654,
      "rewards/margins": 8.505046844482422,
      "rewards/rejected": -11.430481910705566,
      "step": 630
    },
    {
      "epoch": 0.8,
      "learning_rate": 1.1395027624309392e-07,
      "logits/chosen": -2.9629359245300293,
      "logits/rejected": -2.980262517929077,
      "logps/chosen": -201.67306518554688,
      "logps/rejected": -190.01260375976562,
      "loss": 0.1416,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -4.385073661804199,
      "rewards/margins": 8.24610710144043,
      "rewards/rejected": -12.631181716918945,
      "step": 640
    },
    {
      "epoch": 0.81,
      "learning_rate": 1.0704419889502763e-07,
      "logits/chosen": -3.059577465057373,
      "logits/rejected": -2.9308218955993652,
      "logps/chosen": -221.16049194335938,
      "logps/rejected": -187.49386596679688,
      "loss": 0.103,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -3.075377941131592,
      "rewards/margins": 10.026552200317383,
      "rewards/rejected": -13.101930618286133,
      "step": 650
    },
    {
      "epoch": 0.82,
      "learning_rate": 1.0013812154696132e-07,
      "logits/chosen": -3.097217082977295,
      "logits/rejected": -2.9983904361724854,
      "logps/chosen": -198.12832641601562,
      "logps/rejected": -173.28384399414062,
      "loss": 0.122,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -3.524822950363159,
      "rewards/margins": 8.267110824584961,
      "rewards/rejected": -11.791935920715332,
      "step": 660
    },
    {
      "epoch": 0.83,
      "learning_rate": 9.323204419889502e-08,
      "logits/chosen": -3.0255918502807617,
      "logits/rejected": -2.9517340660095215,
      "logps/chosen": -174.54376220703125,
      "logps/rejected": -154.30923461914062,
      "loss": 0.1028,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -0.3350832462310791,
      "rewards/margins": 10.506619453430176,
      "rewards/rejected": -10.841703414916992,
      "step": 670
    },
    {
      "epoch": 0.84,
      "learning_rate": 8.632596685082872e-08,
      "logits/chosen": -2.9522705078125,
      "logits/rejected": -2.905046224594116,
      "logps/chosen": -184.12936401367188,
      "logps/rejected": -156.78311157226562,
      "loss": 0.1092,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -1.2172771692276,
      "rewards/margins": 10.139422416687012,
      "rewards/rejected": -11.35669994354248,
      "step": 680
    },
    {
      "epoch": 0.86,
      "learning_rate": 7.941988950276243e-08,
      "logits/chosen": -2.9998116493225098,
      "logits/rejected": -2.93977689743042,
      "logps/chosen": -159.04739379882812,
      "logps/rejected": -137.8501434326172,
      "loss": 0.1154,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -2.821601390838623,
      "rewards/margins": 8.011626243591309,
      "rewards/rejected": -10.833227157592773,
      "step": 690
    },
    {
      "epoch": 0.87,
      "learning_rate": 7.251381215469612e-08,
      "logits/chosen": -3.064391613006592,
      "logits/rejected": -3.0105395317077637,
      "logps/chosen": -216.6563720703125,
      "logps/rejected": -190.38059997558594,
      "loss": 0.143,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -3.8847603797912598,
      "rewards/margins": 9.102531433105469,
      "rewards/rejected": -12.987292289733887,
      "step": 700
    },
    {
      "epoch": 0.88,
      "learning_rate": 6.560773480662984e-08,
      "logits/chosen": -2.8895156383514404,
      "logits/rejected": -2.873819589614868,
      "logps/chosen": -165.34201049804688,
      "logps/rejected": -158.01522827148438,
      "loss": 0.1274,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -2.2765231132507324,
      "rewards/margins": 10.176279067993164,
      "rewards/rejected": -12.452800750732422,
      "step": 710
    },
    {
      "epoch": 0.89,
      "learning_rate": 5.870165745856354e-08,
      "logits/chosen": -2.992870569229126,
      "logits/rejected": -2.9288392066955566,
      "logps/chosen": -166.22451782226562,
      "logps/rejected": -149.21884155273438,
      "loss": 0.0849,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.1316981315612793,
      "rewards/margins": 11.313945770263672,
      "rewards/rejected": -11.44564437866211,
      "step": 720
    },
    {
      "epoch": 0.91,
      "learning_rate": 5.1795580110497236e-08,
      "logits/chosen": -3.0519726276397705,
      "logits/rejected": -2.9871127605438232,
      "logps/chosen": -198.3760528564453,
      "logps/rejected": -173.3822021484375,
      "loss": 0.1439,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -4.240297317504883,
      "rewards/margins": 9.328596115112305,
      "rewards/rejected": -13.568893432617188,
      "step": 730
    },
    {
      "epoch": 0.92,
      "learning_rate": 4.488950276243094e-08,
      "logits/chosen": -2.905094623565674,
      "logits/rejected": -2.898705005645752,
      "logps/chosen": -166.13040161132812,
      "logps/rejected": -166.46963500976562,
      "loss": 0.0909,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -2.6403870582580566,
      "rewards/margins": 10.037248611450195,
      "rewards/rejected": -12.677635192871094,
      "step": 740
    },
    {
      "epoch": 0.93,
      "learning_rate": 3.7983425414364637e-08,
      "logits/chosen": -3.0315964221954346,
      "logits/rejected": -2.945363998413086,
      "logps/chosen": -186.29241943359375,
      "logps/rejected": -159.93798828125,
      "loss": 0.1194,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -3.548847198486328,
      "rewards/margins": 9.53381061553955,
      "rewards/rejected": -13.082656860351562,
      "step": 750
    },
    {
      "epoch": 0.94,
      "learning_rate": 3.107734806629834e-08,
      "logits/chosen": -2.901912212371826,
      "logits/rejected": -2.8841569423675537,
      "logps/chosen": -165.88009643554688,
      "logps/rejected": -153.7960662841797,
      "loss": 0.1276,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -3.673292875289917,
      "rewards/margins": 10.183616638183594,
      "rewards/rejected": -13.856908798217773,
      "step": 760
    },
    {
      "epoch": 0.96,
      "learning_rate": 2.4171270718232044e-08,
      "logits/chosen": -3.0154058933258057,
      "logits/rejected": -2.9658362865448,
      "logps/chosen": -207.61538696289062,
      "logps/rejected": -184.4959259033203,
      "loss": 0.1581,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -3.740403652191162,
      "rewards/margins": 10.590685844421387,
      "rewards/rejected": -14.331089973449707,
      "step": 770
    },
    {
      "epoch": 0.97,
      "learning_rate": 1.7265193370165747e-08,
      "logits/chosen": -3.099858045578003,
      "logits/rejected": -2.996018171310425,
      "logps/chosen": -209.8662109375,
      "logps/rejected": -179.3186492919922,
      "loss": 0.0908,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.9476644396781921,
      "rewards/margins": 12.100104331970215,
      "rewards/rejected": -13.047770500183105,
      "step": 780
    },
    {
      "epoch": 0.98,
      "learning_rate": 1.0359116022099446e-08,
      "logits/chosen": -3.0576415061950684,
      "logits/rejected": -2.9471096992492676,
      "logps/chosen": -175.6331787109375,
      "logps/rejected": -154.61416625976562,
      "loss": 0.0915,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -3.3139986991882324,
      "rewards/margins": 9.281652450561523,
      "rewards/rejected": -12.595651626586914,
      "step": 790
    },
    {
      "epoch": 0.99,
      "learning_rate": 3.453038674033149e-09,
      "logits/chosen": -2.9590699672698975,
      "logits/rejected": -2.9163191318511963,
      "logps/chosen": -206.9409942626953,
      "logps/rejected": -183.0679931640625,
      "loss": 0.1379,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -2.519080638885498,
      "rewards/margins": 11.627119064331055,
      "rewards/rejected": -14.146200180053711,
      "step": 800
    },
    {
      "epoch": 1.0,
      "step": 805,
      "total_flos": 0.0,
      "train_loss": 0.14663515942437308,
      "train_runtime": 4275.5798,
      "train_samples_per_second": 3.012,
      "train_steps_per_second": 0.188
    }
  ],
  "logging_steps": 10,
  "max_steps": 805,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 500,
  "total_flos": 0.0,
  "train_batch_size": 4,
  "trial_name": null,
  "trial_params": null
}