{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 4.0,
  "eval_steps": 100,
  "global_step": 1540,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.0,
      "learning_rate": 1.282051282051282e-07,
      "logits/chosen": -1.7278180122375488,
      "logits/rejected": -1.7377450466156006,
      "logps/chosen": -29.553977966308594,
      "logps/rejected": -42.813133239746094,
      "loss": 0.5,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 1
    },
    {
      "epoch": 0.03,
      "learning_rate": 1.282051282051282e-06,
      "logits/chosen": -1.866452693939209,
      "logits/rejected": -1.8707809448242188,
      "logps/chosen": -36.98366165161133,
      "logps/rejected": -33.666290283203125,
      "loss": 0.4914,
      "rewards/accuracies": 0.5555555820465088,
      "rewards/chosen": 0.01834939420223236,
      "rewards/margins": 0.04732292518019676,
      "rewards/rejected": -0.02897353284060955,
      "step": 10
    },
    {
      "epoch": 0.05,
      "learning_rate": 2.564102564102564e-06,
      "logits/chosen": -1.9971332550048828,
      "logits/rejected": -1.999770164489746,
      "logps/chosen": -29.6362247467041,
      "logps/rejected": -29.044448852539062,
      "loss": 0.5009,
      "rewards/accuracies": 0.48750001192092896,
      "rewards/chosen": 0.004776014480739832,
      "rewards/margins": -0.01042011845856905,
      "rewards/rejected": 0.015196132473647594,
      "step": 20
    },
    {
      "epoch": 0.08,
      "learning_rate": 3.846153846153847e-06,
      "logits/chosen": -1.9208379983901978,
      "logits/rejected": -1.9181442260742188,
      "logps/chosen": -31.425155639648438,
      "logps/rejected": -33.237945556640625,
      "loss": 0.4997,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -0.007249836809933186,
      "rewards/margins": 0.008341209962964058,
      "rewards/rejected": -0.015591045841574669,
      "step": 30
    },
    {
      "epoch": 0.1,
      "learning_rate": 4.999896948438434e-06,
      "logits/chosen": -2.0180625915527344,
      "logits/rejected": -2.009308338165283,
      "logps/chosen": -32.559749603271484,
      "logps/rejected": -32.524681091308594,
      "loss": 0.4962,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.013646525330841541,
      "rewards/margins": 0.023250887170433998,
      "rewards/rejected": -0.009604359045624733,
      "step": 40
    },
    {
      "epoch": 0.13,
      "learning_rate": 4.987541037542187e-06,
      "logits/chosen": -1.8639278411865234,
      "logits/rejected": -1.8531490564346313,
      "logps/chosen": -33.53788375854492,
      "logps/rejected": -35.42852020263672,
      "loss": 0.5015,
      "rewards/accuracies": 0.48750001192092896,
      "rewards/chosen": 0.015677569434046745,
      "rewards/margins": -0.0033827773295342922,
      "rewards/rejected": 0.019060343503952026,
      "step": 50
    },
    {
      "epoch": 0.16,
      "learning_rate": 4.954691471941119e-06,
      "logits/chosen": -1.9435577392578125,
      "logits/rejected": -1.9454940557479858,
      "logps/chosen": -32.49355697631836,
      "logps/rejected": -33.1856575012207,
      "loss": 0.4817,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": 0.08530057966709137,
      "rewards/margins": 0.09672373533248901,
      "rewards/rejected": -0.011423162184655666,
      "step": 60
    },
    {
      "epoch": 0.18,
      "learning_rate": 4.901618883413549e-06,
      "logits/chosen": -2.075368881225586,
      "logits/rejected": -2.0803427696228027,
      "logps/chosen": -33.89010238647461,
      "logps/rejected": -36.549957275390625,
      "loss": 0.4868,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": 0.07870620489120483,
      "rewards/margins": 0.06357409805059433,
      "rewards/rejected": 0.015132094733417034,
      "step": 70
    },
    {
      "epoch": 0.21,
      "learning_rate": 4.828760511501322e-06,
      "logits/chosen": -1.9372116327285767,
      "logits/rejected": -1.9403302669525146,
      "logps/chosen": -34.205604553222656,
      "logps/rejected": -34.49314880371094,
      "loss": 0.4788,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": 0.17776472866535187,
      "rewards/margins": 0.09835849702358246,
      "rewards/rejected": 0.0794062465429306,
      "step": 80
    },
    {
      "epoch": 0.23,
      "learning_rate": 4.7367166013034295e-06,
      "logits/chosen": -1.945416808128357,
      "logits/rejected": -1.9499365091323853,
      "logps/chosen": -32.26136016845703,
      "logps/rejected": -32.26020050048828,
      "loss": 0.4788,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": 0.16484788060188293,
      "rewards/margins": 0.08588258177042007,
      "rewards/rejected": 0.07896529138088226,
      "step": 90
    },
    {
      "epoch": 0.26,
      "learning_rate": 4.626245458345211e-06,
      "logits/chosen": -2.043586254119873,
      "logits/rejected": -2.0416104793548584,
      "logps/chosen": -31.968154907226562,
      "logps/rejected": -31.14252281188965,
      "loss": 0.4749,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.21814338862895966,
      "rewards/margins": 0.11919467151165009,
      "rewards/rejected": 0.09894871711730957,
      "step": 100
    },
    {
      "epoch": 0.26,
      "eval_logits/chosen": -2.2375268936157227,
      "eval_logits/rejected": -2.2326817512512207,
      "eval_logps/chosen": -33.851219177246094,
      "eval_logps/rejected": -37.3537483215332,
      "eval_loss": 0.49629145860671997,
      "eval_rewards/accuracies": 0.5336378812789917,
      "eval_rewards/chosen": 0.14666710793972015,
      "eval_rewards/margins": 0.016369162127375603,
      "eval_rewards/rejected": 0.1302979290485382,
      "eval_runtime": 145.7938,
      "eval_samples_per_second": 2.353,
      "eval_steps_per_second": 0.295,
      "step": 100
    },
    {
      "epoch": 0.29,
      "learning_rate": 4.498257201263691e-06,
      "logits/chosen": -1.998853087425232,
      "logits/rejected": -1.9964749813079834,
      "logps/chosen": -32.937965393066406,
      "logps/rejected": -33.846004486083984,
      "loss": 0.474,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": 0.2446821928024292,
      "rewards/margins": 0.09798828512430191,
      "rewards/rejected": 0.1466939002275467,
      "step": 110
    },
    {
      "epoch": 0.31,
      "learning_rate": 4.353806263777678e-06,
      "logits/chosen": -2.0093610286712646,
      "logits/rejected": -2.0010437965393066,
      "logps/chosen": -32.1511116027832,
      "logps/rejected": -31.939533233642578,
      "loss": 0.4873,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": 0.2350343018770218,
      "rewards/margins": 0.057456426322460175,
      "rewards/rejected": 0.17757786810398102,
      "step": 120
    },
    {
      "epoch": 0.34,
      "learning_rate": 4.1940827077152755e-06,
      "logits/chosen": -2.0374419689178467,
      "logits/rejected": -2.029500961303711,
      "logps/chosen": -30.157424926757812,
      "logps/rejected": -31.90500831604004,
      "loss": 0.4716,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.2546332776546478,
      "rewards/margins": 0.1362351030111313,
      "rewards/rejected": 0.11839816719293594,
      "step": 130
    },
    {
      "epoch": 0.36,
      "learning_rate": 4.0204024186666215e-06,
      "logits/chosen": -1.9675471782684326,
      "logits/rejected": -1.9777450561523438,
      "logps/chosen": -31.059162139892578,
      "logps/rejected": -32.384300231933594,
      "loss": 0.4616,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": 0.28839823603630066,
      "rewards/margins": 0.16793587803840637,
      "rewards/rejected": 0.12046238034963608,
      "step": 140
    },
    {
      "epoch": 0.39,
      "learning_rate": 3.834196265035119e-06,
      "logits/chosen": -1.8793624639511108,
      "logits/rejected": -1.8805418014526367,
      "logps/chosen": -33.646915435791016,
      "logps/rejected": -34.56272888183594,
      "loss": 0.4385,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.4366222321987152,
      "rewards/margins": 0.2809991240501404,
      "rewards/rejected": 0.15562310814857483,
      "step": 150
    },
    {
      "epoch": 0.42,
      "learning_rate": 3.636998309800573e-06,
      "logits/chosen": -1.9307949542999268,
      "logits/rejected": -1.9274256229400635,
      "logps/chosen": -35.765926361083984,
      "logps/rejected": -32.48193359375,
      "loss": 0.4685,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": 0.32460084557533264,
      "rewards/margins": 0.13194666802883148,
      "rewards/rejected": 0.19265416264533997,
      "step": 160
    },
    {
      "epoch": 0.44,
      "learning_rate": 3.4304331721118078e-06,
      "logits/chosen": -2.0318870544433594,
      "logits/rejected": -2.0245723724365234,
      "logps/chosen": -33.25593566894531,
      "logps/rejected": -31.189266204833984,
      "loss": 0.4315,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.4335424304008484,
      "rewards/margins": 0.3034728467464447,
      "rewards/rejected": 0.1300695389509201,
      "step": 170
    },
    {
      "epoch": 0.47,
      "learning_rate": 3.2162026428305436e-06,
      "logits/chosen": -2.0387840270996094,
      "logits/rejected": -2.0439517498016357,
      "logps/chosen": -31.977941513061523,
      "logps/rejected": -32.183441162109375,
      "loss": 0.4499,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.4534761905670166,
      "rewards/margins": 0.2121172845363617,
      "rewards/rejected": 0.2413589060306549,
      "step": 180
    },
    {
      "epoch": 0.49,
      "learning_rate": 2.996071664294641e-06,
      "logits/chosen": -2.039411783218384,
      "logits/rejected": -2.036684513092041,
      "logps/chosen": -31.094594955444336,
      "logps/rejected": -31.10666275024414,
      "loss": 0.4653,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.3154846429824829,
      "rewards/margins": 0.1462438404560089,
      "rewards/rejected": 0.169240802526474,
      "step": 190
    },
    {
      "epoch": 0.52,
      "learning_rate": 2.7718537898066833e-06,
      "logits/chosen": -1.909691572189331,
      "logits/rejected": -1.914358139038086,
      "logps/chosen": -31.08693504333496,
      "logps/rejected": -32.61079406738281,
      "loss": 0.4376,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.41547876596450806,
      "rewards/margins": 0.27136775851249695,
      "rewards/rejected": 0.14411096274852753,
      "step": 200
    },
    {
      "epoch": 0.52,
      "eval_logits/chosen": -2.233895778656006,
      "eval_logits/rejected": -2.2290823459625244,
      "eval_logps/chosen": -33.78962326049805,
      "eval_logps/rejected": -37.29551696777344,
      "eval_loss": 0.4955826997756958,
      "eval_rewards/accuracies": 0.5485880374908447,
      "eval_rewards/chosen": 0.19594170153141022,
      "eval_rewards/margins": 0.019058095291256905,
      "eval_rewards/rejected": 0.17688362300395966,
      "eval_runtime": 145.7262,
      "eval_samples_per_second": 2.354,
      "eval_steps_per_second": 0.295,
      "step": 200
    },
    {
      "epoch": 0.55,
      "learning_rate": 2.5453962426402006e-06,
      "logits/chosen": -2.0220179557800293,
      "logits/rejected": -2.0326101779937744,
      "logps/chosen": -31.51509666442871,
      "logps/rejected": -33.655662536621094,
      "loss": 0.4554,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": 0.36496439576148987,
      "rewards/margins": 0.19933317601680756,
      "rewards/rejected": 0.16563120484352112,
      "step": 210
    },
    {
      "epoch": 0.57,
      "learning_rate": 2.3185646976551794e-06,
      "logits/chosen": -1.9142780303955078,
      "logits/rejected": -1.9289791584014893,
      "logps/chosen": -29.601001739501953,
      "logps/rejected": -31.418704986572266,
      "loss": 0.4339,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": 0.4084321856498718,
      "rewards/margins": 0.2968555688858032,
      "rewards/rejected": 0.11157669872045517,
      "step": 220
    },
    {
      "epoch": 0.6,
      "learning_rate": 2.0932279108998323e-06,
      "logits/chosen": -1.9722967147827148,
      "logits/rejected": -1.9762967824935913,
      "logps/chosen": -32.825157165527344,
      "logps/rejected": -31.445331573486328,
      "loss": 0.4142,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.47385087609291077,
      "rewards/margins": 0.39067354798316956,
      "rewards/rejected": 0.0831773579120636,
      "step": 230
    },
    {
      "epoch": 0.62,
      "learning_rate": 1.8712423238279358e-06,
      "logits/chosen": -1.9708006381988525,
      "logits/rejected": -1.9490963220596313,
      "logps/chosen": -33.599205017089844,
      "logps/rejected": -34.856910705566406,
      "loss": 0.4183,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.4436335563659668,
      "rewards/margins": 0.36852845549583435,
      "rewards/rejected": 0.07510510087013245,
      "step": 240
    },
    {
      "epoch": 0.65,
      "learning_rate": 1.6544367689701824e-06,
      "logits/chosen": -2.011307954788208,
      "logits/rejected": -2.008012533187866,
      "logps/chosen": -32.494956970214844,
      "logps/rejected": -35.98812484741211,
      "loss": 0.4641,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.3440879285335541,
      "rewards/margins": 0.15464913845062256,
      "rewards/rejected": 0.18943879008293152,
      "step": 250
    },
    {
      "epoch": 0.68,
      "learning_rate": 1.4445974030621963e-06,
      "logits/chosen": -1.8780018091201782,
      "logits/rejected": -1.8755817413330078,
      "logps/chosen": -33.72126388549805,
      "logps/rejected": -35.290069580078125,
      "loss": 0.4544,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": 0.37370795011520386,
      "rewards/margins": 0.20206721127033234,
      "rewards/rejected": 0.17164072394371033,
      "step": 260
    },
    {
      "epoch": 0.7,
      "learning_rate": 1.243452991757889e-06,
      "logits/chosen": -1.8628206253051758,
      "logits/rejected": -1.8603187799453735,
      "logps/chosen": -33.929344177246094,
      "logps/rejected": -31.61127281188965,
      "loss": 0.4451,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": 0.3746057152748108,
      "rewards/margins": 0.246560737490654,
      "rewards/rejected": 0.128044992685318,
      "step": 270
    },
    {
      "epoch": 0.73,
      "learning_rate": 1.0526606671603523e-06,
      "logits/chosen": -1.9666433334350586,
      "logits/rejected": -1.9561984539031982,
      "logps/chosen": -34.74022674560547,
      "logps/rejected": -31.632495880126953,
      "loss": 0.4239,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": 0.4931296706199646,
      "rewards/margins": 0.3263343572616577,
      "rewards/rejected": 0.16679534316062927,
      "step": 280
    },
    {
      "epoch": 0.75,
      "learning_rate": 8.737922755071455e-07,
      "logits/chosen": -2.0619094371795654,
      "logits/rejected": -2.0470869541168213,
      "logps/chosen": -30.43317222595215,
      "logps/rejected": -32.3420524597168,
      "loss": 0.4679,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": 0.39276057481765747,
      "rewards/margins": 0.1449340283870697,
      "rewards/rejected": 0.24782654643058777,
      "step": 290
    },
    {
      "epoch": 0.78,
      "learning_rate": 7.08321427484816e-07,
      "logits/chosen": -1.9332126379013062,
      "logits/rejected": -1.9307750463485718,
      "logps/chosen": -32.100502014160156,
      "logps/rejected": -30.6780948638916,
      "loss": 0.3835,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.6612862348556519,
      "rewards/margins": 0.5503143072128296,
      "rewards/rejected": 0.1109718531370163,
      "step": 300
    },
    {
      "epoch": 0.78,
      "eval_logits/chosen": -2.231203556060791,
      "eval_logits/rejected": -2.226395606994629,
      "eval_logps/chosen": -33.778892517089844,
      "eval_logps/rejected": -37.28717803955078,
      "eval_loss": 0.4949820339679718,
      "eval_rewards/accuracies": 0.5245016813278198,
      "eval_rewards/chosen": 0.204526886343956,
      "eval_rewards/margins": 0.020971858873963356,
      "eval_rewards/rejected": 0.1835550218820572,
      "eval_runtime": 145.6189,
      "eval_samples_per_second": 2.355,
      "eval_steps_per_second": 0.295,
      "step": 300
    },
    {
      "epoch": 0.81,
      "learning_rate": 4.84533120650964e-06,
      "logits/chosen": -1.9174331426620483,
      "logits/rejected": -1.9142847061157227,
      "logps/chosen": -31.049779891967773,
      "logps/rejected": -33.54362487792969,
      "loss": 0.4366,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": 0.4428572654724121,
      "rewards/margins": 0.28403452038764954,
      "rewards/rejected": 0.15882274508476257,
      "step": 310
    },
    {
      "epoch": 0.83,
      "learning_rate": 4.825108134172131e-06,
      "logits/chosen": -1.9670381546020508,
      "logits/rejected": -1.9549226760864258,
      "logps/chosen": -34.00459671020508,
      "logps/rejected": -33.429378509521484,
      "loss": 0.4145,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": 0.4731821119785309,
      "rewards/margins": 0.38561564683914185,
      "rewards/rejected": 0.08756652474403381,
      "step": 320
    },
    {
      "epoch": 0.86,
      "learning_rate": 4.80369052967602e-06,
      "logits/chosen": -2.001861095428467,
      "logits/rejected": -2.0005335807800293,
      "logps/chosen": -32.78104019165039,
      "logps/rejected": -32.248443603515625,
      "loss": 0.4183,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": 0.5638058185577393,
      "rewards/margins": 0.37889450788497925,
      "rewards/rejected": 0.18491129577159882,
      "step": 330
    },
    {
      "epoch": 0.88,
      "learning_rate": 4.781089396387968e-06,
      "logits/chosen": -2.0892767906188965,
      "logits/rejected": -2.073693037033081,
      "logps/chosen": -33.377323150634766,
      "logps/rejected": -32.779090881347656,
      "loss": 0.4167,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.641654908657074,
      "rewards/margins": 0.37049978971481323,
      "rewards/rejected": 0.2711551785469055,
      "step": 340
    },
    {
      "epoch": 0.91,
      "learning_rate": 4.757316345716554e-06,
      "logits/chosen": -1.9618675708770752,
      "logits/rejected": -1.9610551595687866,
      "logps/chosen": -32.51572799682617,
      "logps/rejected": -32.19974899291992,
      "loss": 0.4137,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.6130937933921814,
      "rewards/margins": 0.4007722735404968,
      "rewards/rejected": 0.21232159435749054,
      "step": 350
    },
    {
      "epoch": 0.94,
      "learning_rate": 4.73238359114687e-06,
      "logits/chosen": -1.916259765625,
      "logits/rejected": -1.9264857769012451,
      "logps/chosen": -31.4000244140625,
      "logps/rejected": -35.018882751464844,
      "loss": 0.4065,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": 0.6437316536903381,
      "rewards/margins": 0.427828311920166,
      "rewards/rejected": 0.21590332686901093,
      "step": 360
    },
    {
      "epoch": 0.96,
      "learning_rate": 4.706303941965804e-06,
      "logits/chosen": -2.050283908843994,
      "logits/rejected": -2.043849468231201,
      "logps/chosen": -32.88298797607422,
      "logps/rejected": -28.950063705444336,
      "loss": 0.4093,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.6151864528656006,
      "rewards/margins": 0.3940979838371277,
      "rewards/rejected": 0.2210884839296341,
      "step": 370
    },
    {
      "epoch": 0.99,
      "learning_rate": 4.679090796681225e-06,
      "logits/chosen": -1.9082043170928955,
      "logits/rejected": -1.9103626012802124,
      "logps/chosen": -33.2851676940918,
      "logps/rejected": -30.665771484375,
      "loss": 0.3759,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": 0.7631045579910278,
      "rewards/margins": 0.5976192355155945,
      "rewards/rejected": 0.1654852330684662,
      "step": 380
    },
    {
      "epoch": 1.01,
      "learning_rate": 4.650758136138454e-06,
      "logits/chosen": -1.9357115030288696,
      "logits/rejected": -1.9344127178192139,
      "logps/chosen": -33.36685562133789,
      "logps/rejected": -35.62567138671875,
      "loss": 0.3409,
      "rewards/accuracies": 0.7583333253860474,
      "rewards/chosen": 0.8143496513366699,
      "rewards/margins": 0.790077805519104,
      "rewards/rejected": 0.02427184209227562,
      "step": 390
    },
    {
      "epoch": 1.04,
      "learning_rate": 4.621320516337559e-06,
      "logits/chosen": -1.8698232173919678,
      "logits/rejected": -1.8616269826889038,
      "logps/chosen": -30.623443603515625,
      "logps/rejected": -36.03126907348633,
      "loss": 0.3117,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 0.920426070690155,
      "rewards/margins": 0.9929585456848145,
      "rewards/rejected": -0.07253243774175644,
      "step": 400
    },
    {
      "epoch": 1.04,
      "eval_logits/chosen": -2.2160162925720215,
      "eval_logits/rejected": -2.2111997604370117,
      "eval_logps/chosen": -33.652793884277344,
      "eval_logps/rejected": -37.193359375,
      "eval_loss": 0.4891091287136078,
      "eval_rewards/accuracies": 0.5651993155479431,
      "eval_rewards/chosen": 0.30540817975997925,
      "eval_rewards/margins": 0.04680241644382477,
      "eval_rewards/rejected": 0.2586057484149933,
      "eval_runtime": 146.2599,
      "eval_samples_per_second": 2.345,
      "eval_steps_per_second": 0.294,
      "step": 400
    },
    {
      "epoch": 1.06,
      "learning_rate": 4.590793060955158e-06,
      "logits/chosen": -2.044127941131592,
      "logits/rejected": -2.046945571899414,
      "logps/chosen": -31.79901695251465,
      "logps/rejected": -34.87114334106445,
      "loss": 0.299,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 1.003937005996704,
      "rewards/margins": 1.0665733814239502,
      "rewards/rejected": -0.0626363754272461,
      "step": 410
    },
    {
      "epoch": 1.09,
      "learning_rate": 4.559191453574582e-06,
      "logits/chosen": -1.8841445446014404,
      "logits/rejected": -1.8827756643295288,
      "logps/chosen": -27.981430053710938,
      "logps/rejected": -32.39961624145508,
      "loss": 0.3151,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": 0.868902325630188,
      "rewards/margins": 0.9344655871391296,
      "rewards/rejected": -0.06556323915719986,
      "step": 420
    },
    {
      "epoch": 1.12,
      "learning_rate": 4.52653192962838e-06,
      "logits/chosen": -1.8422701358795166,
      "logits/rejected": -1.835268259048462,
      "logps/chosen": -32.6194953918457,
      "logps/rejected": -34.0600700378418,
      "loss": 0.2904,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 1.1636759042739868,
      "rewards/margins": 1.0155143737792969,
      "rewards/rejected": 0.14816154539585114,
      "step": 430
    },
    {
      "epoch": 1.14,
      "learning_rate": 4.492831268057307e-06,
      "logits/chosen": -2.0123181343078613,
      "logits/rejected": -2.0072364807128906,
      "logps/chosen": -30.41460609436035,
      "logps/rejected": -32.033443450927734,
      "loss": 0.284,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 1.0949517488479614,
      "rewards/margins": 1.1476168632507324,
      "rewards/rejected": -0.05266512185335159,
      "step": 440
    },
    {
      "epoch": 1.17,
      "learning_rate": 4.458106782690094e-06,
      "logits/chosen": -1.8972448110580444,
      "logits/rejected": -1.9013574123382568,
      "logps/chosen": -33.069793701171875,
      "logps/rejected": -32.7395133972168,
      "loss": 0.263,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 1.156842589378357,
      "rewards/margins": 1.2775455713272095,
      "rewards/rejected": -0.12070278078317642,
      "step": 450
    },
    {
      "epoch": 1.19,
      "learning_rate": 4.422376313348405e-06,
      "logits/chosen": -1.9023644924163818,
      "logits/rejected": -1.8966976404190063,
      "logps/chosen": -33.858001708984375,
      "logps/rejected": -35.19942092895508,
      "loss": 0.2537,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 1.249218225479126,
      "rewards/margins": 1.4494789838790894,
      "rewards/rejected": -0.20026080310344696,
      "step": 460
    },
    {
      "epoch": 1.22,
      "learning_rate": 4.3856582166815696e-06,
      "logits/chosen": -1.9303083419799805,
      "logits/rejected": -1.9299156665802002,
      "logps/chosen": -32.68412399291992,
      "logps/rejected": -34.166542053222656,
      "loss": 0.2809,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 1.1873619556427002,
      "rewards/margins": 1.1722075939178467,
      "rewards/rejected": 0.01515440084040165,
      "step": 470
    },
    {
      "epoch": 1.25,
      "learning_rate": 4.347971356735789e-06,
      "logits/chosen": -1.97918701171875,
      "logits/rejected": -1.960694670677185,
      "logps/chosen": -32.545894622802734,
      "logps/rejected": -33.19091033935547,
      "loss": 0.2501,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 1.291896104812622,
      "rewards/margins": 1.3886420726776123,
      "rewards/rejected": -0.09674612432718277,
      "step": 480
    },
    {
      "epoch": 1.27,
      "learning_rate": 4.309335095262675e-06,
      "logits/chosen": -1.9456336498260498,
      "logits/rejected": -1.9451408386230469,
      "logps/chosen": -30.120983123779297,
      "logps/rejected": -31.22079849243164,
      "loss": 0.2773,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 1.1922577619552612,
      "rewards/margins": 1.1720596551895142,
      "rewards/rejected": 0.020198041573166847,
      "step": 490
    },
    {
      "epoch": 1.3,
      "learning_rate": 4.269769281772082e-06,
      "logits/chosen": -1.9087337255477905,
      "logits/rejected": -1.9019416570663452,
      "logps/chosen": -30.988162994384766,
      "logps/rejected": -34.79794692993164,
      "loss": 0.2459,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 1.345572829246521,
      "rewards/margins": 1.4825364351272583,
      "rewards/rejected": -0.13696344196796417,
      "step": 500
    },
    {
      "epoch": 1.3,
      "eval_logits/chosen": -2.1905555725097656,
      "eval_logits/rejected": -2.1857681274414062,
      "eval_logps/chosen": -33.6363525390625,
      "eval_logps/rejected": -37.182701110839844,
      "eval_loss": 0.4884692430496216,
      "eval_rewards/accuracies": 0.5622923374176025,
      "eval_rewards/chosen": 0.3185593783855438,
      "eval_rewards/margins": 0.05142458155751228,
      "eval_rewards/rejected": 0.26713478565216064,
      "eval_runtime": 145.9464,
      "eval_samples_per_second": 2.35,
      "eval_steps_per_second": 0.295,
      "step": 500
    },
    {
      "epoch": 1.32,
      "learning_rate": 4.22929424333435e-06,
      "logits/chosen": -1.903969168663025,
      "logits/rejected": -1.9078088998794556,
      "logps/chosen": -27.756145477294922,
      "logps/rejected": -33.003501892089844,
      "loss": 0.2721,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 1.1349422931671143,
      "rewards/margins": 1.2503130435943604,
      "rewards/rejected": -0.11537061631679535,
      "step": 510
    },
    {
      "epoch": 1.35,
      "learning_rate": 4.1879307741372085e-06,
      "logits/chosen": -1.9105024337768555,
      "logits/rejected": -1.9210312366485596,
      "logps/chosen": -31.711151123046875,
      "logps/rejected": -30.852153778076172,
      "loss": 0.2636,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 1.240944743156433,
      "rewards/margins": 1.4013227224349976,
      "rewards/rejected": -0.1603778600692749,
      "step": 520
    },
    {
      "epoch": 1.38,
      "learning_rate": 4.145700124802693e-06,
      "logits/chosen": -1.8562660217285156,
      "logits/rejected": -1.8539501428604126,
      "logps/chosen": -30.094646453857422,
      "logps/rejected": -30.31314468383789,
      "loss": 0.2542,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 1.2579472064971924,
      "rewards/margins": 1.3670498132705688,
      "rewards/rejected": -0.10910268872976303,
      "step": 530
    },
    {
      "epoch": 1.4,
      "learning_rate": 4.102623991469562e-06,
      "logits/chosen": -1.9388301372528076,
      "logits/rejected": -1.931828260421753,
      "logps/chosen": -32.52373504638672,
      "logps/rejected": -33.19480514526367,
      "loss": 0.234,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 1.3672724962234497,
      "rewards/margins": 1.5091286897659302,
      "rewards/rejected": -0.14185625314712524,
      "step": 540
    },
    {
      "epoch": 1.43,
      "learning_rate": 4.058724504646834e-06,
      "logits/chosen": -1.9054105281829834,
      "logits/rejected": -1.9118045568466187,
      "logps/chosen": -30.23921775817871,
      "logps/rejected": -32.79788589477539,
      "loss": 0.2692,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 1.211173176765442,
      "rewards/margins": 1.2424962520599365,
      "rewards/rejected": -0.03132311627268791,
      "step": 550
    },
    {
      "epoch": 1.45,
      "learning_rate": 4.014024217844167e-06,
      "logits/chosen": -1.9779608249664307,
      "logits/rejected": -1.9552392959594727,
      "logps/chosen": -29.897476196289062,
      "logps/rejected": -33.047569274902344,
      "loss": 0.2672,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 1.2320712804794312,
      "rewards/margins": 1.273280143737793,
      "rewards/rejected": -0.041208960115909576,
      "step": 560
    },
    {
      "epoch": 1.48,
      "learning_rate": 3.968546095984911e-06,
      "logits/chosen": -1.9123926162719727,
      "logits/rejected": -1.9074643850326538,
      "logps/chosen": -30.69790267944336,
      "logps/rejected": -32.02743911743164,
      "loss": 0.2612,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 1.3750633001327515,
      "rewards/margins": 1.3002793788909912,
      "rewards/rejected": 0.07478378713130951,
      "step": 570
    },
    {
      "epoch": 1.51,
      "learning_rate": 3.922313503607806e-06,
      "logits/chosen": -1.9546325206756592,
      "logits/rejected": -1.956498384475708,
      "logps/chosen": -32.70453643798828,
      "logps/rejected": -35.04417037963867,
      "loss": 0.2395,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 1.337005853652954,
      "rewards/margins": 1.5316107273101807,
      "rewards/rejected": -0.19460472464561462,
      "step": 580
    },
    {
      "epoch": 1.53,
      "learning_rate": 3.875350192863368e-06,
      "logits/chosen": -1.9385898113250732,
      "logits/rejected": -1.9379469156265259,
      "logps/chosen": -28.891321182250977,
      "logps/rejected": -31.504505157470703,
      "loss": 0.2493,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 1.3605870008468628,
      "rewards/margins": 1.3948794603347778,
      "rewards/rejected": -0.034292541444301605,
      "step": 590
    },
    {
      "epoch": 1.56,
      "learning_rate": 3.8276802913111436e-06,
      "logits/chosen": -1.9496517181396484,
      "logits/rejected": -1.9475892782211304,
      "logps/chosen": -31.307537078857422,
      "logps/rejected": -32.269371032714844,
      "loss": 0.2639,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 1.3525952100753784,
      "rewards/margins": 1.3281266689300537,
      "rewards/rejected": 0.024468522518873215,
      "step": 600
    },
    {
      "epoch": 1.56,
      "eval_logits/chosen": -2.183182716369629,
      "eval_logits/rejected": -2.178420066833496,
      "eval_logps/chosen": -33.58167266845703,
      "eval_logps/rejected": -37.20375442504883,
      "eval_loss": 0.47496160864830017,
      "eval_rewards/accuracies": 0.5855481624603271,
      "eval_rewards/chosen": 0.3623029887676239,
      "eval_rewards/margins": 0.11201038211584091,
      "eval_rewards/rejected": 0.2502925992012024,
      "eval_runtime": 145.9651,
      "eval_samples_per_second": 2.35,
      "eval_steps_per_second": 0.295,
      "step": 600
    },
    {
      "epoch": 1.58,
      "learning_rate": 3.7793282895240927e-06,
      "logits/chosen": -1.9928886890411377,
      "logits/rejected": -1.9994275569915771,
      "logps/chosen": -30.57297706604004,
      "logps/rejected": -32.0395622253418,
      "loss": 0.2611,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 1.3315242528915405,
      "rewards/margins": 1.3126256465911865,
      "rewards/rejected": 0.01889852061867714,
      "step": 610
    },
    {
      "epoch": 1.61,
      "learning_rate": 3.730319028506478e-06,
      "logits/chosen": -1.9484421014785767,
      "logits/rejected": -1.946126937866211,
      "logps/chosen": -32.853248596191406,
      "logps/rejected": -30.977664947509766,
      "loss": 0.2377,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 1.5221078395843506,
      "rewards/margins": 1.5158333778381348,
      "rewards/rejected": 0.0062743364833295345,
      "step": 620
    },
    {
      "epoch": 1.64,
      "learning_rate": 3.6806776869317074e-06,
      "logits/chosen": -1.8969027996063232,
      "logits/rejected": -1.8902422189712524,
      "logps/chosen": -33.44438171386719,
      "logps/rejected": -32.37852478027344,
      "loss": 0.2252,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 1.6158740520477295,
      "rewards/margins": 1.6329265832901,
      "rewards/rejected": -0.017052406445145607,
      "step": 630
    },
    {
      "epoch": 1.66,
      "learning_rate": 3.6304297682067146e-06,
      "logits/chosen": -1.9018688201904297,
      "logits/rejected": -1.9080841541290283,
      "logps/chosen": -32.16619110107422,
      "logps/rejected": -33.13113021850586,
      "loss": 0.2565,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 1.450488567352295,
      "rewards/margins": 1.3631832599639893,
      "rewards/rejected": 0.08730525523424149,
      "step": 640
    },
    {
      "epoch": 1.69,
      "learning_rate": 3.579601087369492e-06,
      "logits/chosen": -1.98688542842865,
      "logits/rejected": -2.0009474754333496,
      "logps/chosen": -30.202587127685547,
      "logps/rejected": -31.93996238708496,
      "loss": 0.2613,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 1.3199015855789185,
      "rewards/margins": 1.2737209796905518,
      "rewards/rejected": 0.046180594712495804,
      "step": 650
    },
    {
      "epoch": 1.71,
      "learning_rate": 3.5282177578265295e-06,
      "logits/chosen": -1.861903429031372,
      "logits/rejected": -1.8589446544647217,
      "logps/chosen": -31.801767349243164,
      "logps/rejected": -34.892234802246094,
      "loss": 0.1937,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 1.6969530582427979,
      "rewards/margins": 1.8505403995513916,
      "rewards/rejected": -0.15358731150627136,
      "step": 660
    },
    {
      "epoch": 1.74,
      "learning_rate": 3.476306177936961e-06,
      "logits/chosen": -1.9529634714126587,
      "logits/rejected": -1.9532558917999268,
      "logps/chosen": -29.594532012939453,
      "logps/rejected": -33.946571350097656,
      "loss": 0.2422,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 1.3528481721878052,
      "rewards/margins": 1.449000597000122,
      "rewards/rejected": -0.09615238755941391,
      "step": 670
    },
    {
      "epoch": 1.77,
      "learning_rate": 3.423893017450324e-06,
      "logits/chosen": -1.9031528234481812,
      "logits/rejected": -1.9001739025115967,
      "logps/chosen": -29.16634178161621,
      "logps/rejected": -32.86220169067383,
      "loss": 0.2577,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 1.3648045063018799,
      "rewards/margins": 1.40032160282135,
      "rewards/rejected": -0.03551710769534111,
      "step": 680
    },
    {
      "epoch": 1.79,
      "learning_rate": 3.3710052038048794e-06,
      "logits/chosen": -1.9385061264038086,
      "logits/rejected": -1.9385082721710205,
      "logps/chosen": -27.996692657470703,
      "logps/rejected": -30.74009132385254,
      "loss": 0.2085,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 1.7331215143203735,
      "rewards/margins": 1.7887967824935913,
      "rewards/rejected": -0.05567514896392822,
      "step": 690
    },
    {
      "epoch": 1.82,
      "learning_rate": 3.3176699082935546e-06,
      "logits/chosen": -1.8473536968231201,
      "logits/rejected": -1.850669503211975,
      "logps/chosen": -32.4169921875,
      "logps/rejected": -31.4881649017334,
      "loss": 0.2437,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 1.6491864919662476,
      "rewards/margins": 1.6679694652557373,
      "rewards/rejected": -0.01878293789923191,
      "step": 700
    },
    {
      "epoch": 1.82,
      "eval_logits/chosen": -2.178642988204956,
      "eval_logits/rejected": -2.173909902572632,
      "eval_logps/chosen": -33.59923553466797,
      "eval_logps/rejected": -37.22935485839844,
      "eval_loss": 0.4741733968257904,
      "eval_rewards/accuracies": 0.5747508406639099,
      "eval_rewards/chosen": 0.34825241565704346,
      "eval_rewards/margins": 0.1184391975402832,
      "eval_rewards/rejected": 0.22981324791908264,
      "eval_runtime": 145.9455,
      "eval_samples_per_second": 2.35,
      "eval_steps_per_second": 0.295,
      "step": 700
    },
    {
      "epoch": 1.84,
      "learning_rate": 3.2639145321045933e-06,
      "logits/chosen": -1.9303743839263916,
      "logits/rejected": -1.9213898181915283,
      "logps/chosen": -34.50909423828125,
      "logps/rejected": -31.8797550201416,
      "loss": 0.2249,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 1.602341890335083,
      "rewards/margins": 1.5814943313598633,
      "rewards/rejected": 0.02084733545780182,
      "step": 710
    },
    {
      "epoch": 1.87,
      "learning_rate": 3.2097666922441107e-06,
      "logits/chosen": -1.9459857940673828,
      "logits/rejected": -1.9470113515853882,
      "logps/chosen": -34.359092712402344,
      "logps/rejected": -33.046634674072266,
      "loss": 0.2173,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 1.6610389947891235,
      "rewards/margins": 1.6375271081924438,
      "rewards/rejected": 0.02351185865700245,
      "step": 720
    },
    {
      "epoch": 1.9,
      "learning_rate": 3.1552542073477554e-06,
      "logits/chosen": -1.962262511253357,
      "logits/rejected": -1.9598932266235352,
      "logps/chosen": -30.564239501953125,
      "logps/rejected": -32.997047424316406,
      "loss": 0.23,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 1.5992707014083862,
      "rewards/margins": 1.5993889570236206,
      "rewards/rejected": -0.00011831596202682704,
      "step": 730
    },
    {
      "epoch": 1.92,
      "learning_rate": 3.100405083388799e-06,
      "logits/chosen": -1.9474010467529297,
      "logits/rejected": -1.952619194984436,
      "logps/chosen": -29.694049835205078,
      "logps/rejected": -33.282554626464844,
      "loss": 0.2069,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 1.6733427047729492,
      "rewards/margins": 1.7755340337753296,
      "rewards/rejected": -0.10219136625528336,
      "step": 740
    },
    {
      "epoch": 1.95,
      "learning_rate": 3.0452474992899645e-06,
      "logits/chosen": -1.892179250717163,
      "logits/rejected": -1.8910681009292603,
      "logps/chosen": -31.141448974609375,
      "logps/rejected": -34.69000244140625,
      "loss": 0.24,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 1.5868273973464966,
      "rewards/margins": 1.5900630950927734,
      "rewards/rejected": -0.0032357454765588045,
      "step": 750
    },
    {
      "epoch": 1.97,
      "learning_rate": 2.989809792446417e-06,
      "logits/chosen": -1.7668031454086304,
      "logits/rejected": -1.7617915868759155,
      "logps/chosen": -33.82783889770508,
      "logps/rejected": -35.33191680908203,
      "loss": 0.2193,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 1.7364822626113892,
      "rewards/margins": 1.8352861404418945,
      "rewards/rejected": -0.0988037958741188,
      "step": 760
    },
    {
      "epoch": 2.0,
      "learning_rate": 2.9341204441673267e-06,
      "logits/chosen": -1.9073164463043213,
      "logits/rejected": -1.9110107421875,
      "logps/chosen": -33.157711029052734,
      "logps/rejected": -33.576969146728516,
      "loss": 0.2221,
      "rewards/accuracies": 0.8708333969116211,
      "rewards/chosen": 1.6447594165802002,
      "rewards/margins": 1.6304632425308228,
      "rewards/rejected": 0.014296052046120167,
      "step": 770
    },
    {
      "epoch": 2.03,
      "learning_rate": 2.878208065043501e-06,
      "logits/chosen": -1.8515024185180664,
      "logits/rejected": -1.8497060537338257,
      "logps/chosen": -31.297021865844727,
      "logps/rejected": -35.01959991455078,
      "loss": 0.1312,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 2.0836594104766846,
      "rewards/margins": 2.575618028640747,
      "rewards/rejected": -0.4919588565826416,
      "step": 780
    },
    {
      "epoch": 2.05,
      "learning_rate": 2.8221013802485974e-06,
      "logits/chosen": -1.899592638015747,
      "logits/rejected": -1.8983027935028076,
      "logps/chosen": -30.751636505126953,
      "logps/rejected": -33.40520477294922,
      "loss": 0.1431,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 2.0992822647094727,
      "rewards/margins": 2.354191541671753,
      "rewards/rejected": -0.25490933656692505,
      "step": 790
    },
    {
      "epoch": 2.08,
      "learning_rate": 2.76582921478147e-06,
      "logits/chosen": -1.83371901512146,
      "logits/rejected": -1.8279550075531006,
      "logps/chosen": -31.871252059936523,
      "logps/rejected": -31.73343849182129,
      "loss": 0.1567,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 2.0094635486602783,
      "rewards/margins": 2.224806547164917,
      "rewards/rejected": -0.21534299850463867,
      "step": 800
    },
    {
      "epoch": 2.08,
      "eval_logits/chosen": -2.1802878379821777,
      "eval_logits/rejected": -2.1755456924438477,
      "eval_logps/chosen": -33.54963302612305,
      "eval_logps/rejected": -37.20660400390625,
      "eval_loss": 0.4695200026035309,
      "eval_rewards/accuracies": 0.5826411843299866,
      "eval_rewards/chosen": 0.3879339396953583,
      "eval_rewards/margins": 0.1399237960577011,
      "eval_rewards/rejected": 0.24801018834114075,
      "eval_runtime": 145.8879,
      "eval_samples_per_second": 2.351,
      "eval_steps_per_second": 0.295,
      "step": 800
    },
    {
      "epoch": 2.1,
      "learning_rate": 2.7094204786572254e-06,
      "logits/chosen": -1.9300823211669922,
      "logits/rejected": -1.9375168085098267,
      "logps/chosen": -29.62691307067871,
      "logps/rejected": -34.63849639892578,
      "loss": 0.1613,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 2.0010743141174316,
      "rewards/margins": 2.3072001934051514,
      "rewards/rejected": -0.3061259686946869,
      "step": 810
    },
    {
      "epoch": 2.13,
      "learning_rate": 2.6529041520546072e-06,
      "logits/chosen": -1.9036821126937866,
      "logits/rejected": -1.9064573049545288,
      "logps/chosen": -30.34005355834961,
      "logps/rejected": -33.30119705200195,
      "loss": 0.1999,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 1.784820556640625,
      "rewards/margins": 1.8419349193572998,
      "rewards/rejected": -0.05711423233151436,
      "step": 820
    },
    {
      "epoch": 2.16,
      "learning_rate": 2.5963092704273302e-06,
      "logits/chosen": -1.805069923400879,
      "logits/rejected": -1.8091821670532227,
      "logps/chosen": -30.127094268798828,
      "logps/rejected": -35.30133056640625,
      "loss": 0.1557,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": 1.9714915752410889,
      "rewards/margins": 2.2844126224517822,
      "rewards/rejected": -0.31292054057121277,
      "step": 830
    },
    {
      "epoch": 2.18,
      "learning_rate": 2.53966490958702e-06,
      "logits/chosen": -1.8699356317520142,
      "logits/rejected": -1.865962028503418,
      "logps/chosen": -30.581180572509766,
      "logps/rejected": -33.02351379394531,
      "loss": 0.1821,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 1.9099775552749634,
      "rewards/margins": 1.9330689907073975,
      "rewards/rejected": -0.02309143915772438,
      "step": 840
    },
    {
      "epoch": 2.21,
      "learning_rate": 2.4830001707654135e-06,
      "logits/chosen": -1.9513721466064453,
      "logits/rejected": -1.9533140659332275,
      "logps/chosen": -30.263479232788086,
      "logps/rejected": -36.080474853515625,
      "loss": 0.1402,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": 2.0094857215881348,
      "rewards/margins": 2.344318389892578,
      "rewards/rejected": -0.3348326086997986,
      "step": 850
    },
    {
      "epoch": 2.23,
      "learning_rate": 2.4263441656635054e-06,
      "logits/chosen": -1.7577623128890991,
      "logits/rejected": -1.7520900964736938,
      "logps/chosen": -33.64806365966797,
      "logps/rejected": -33.127098083496094,
      "loss": 0.157,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 2.106886625289917,
      "rewards/margins": 2.2950429916381836,
      "rewards/rejected": -0.18815645575523376,
      "step": 860
    },
    {
      "epoch": 2.26,
      "learning_rate": 2.3697260014953107e-06,
      "logits/chosen": -1.8190982341766357,
      "logits/rejected": -1.8191255331039429,
      "logps/chosen": -33.16907501220703,
      "logps/rejected": -35.244049072265625,
      "loss": 0.1428,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 2.138986110687256,
      "rewards/margins": 2.457213878631592,
      "rewards/rejected": -0.31822749972343445,
      "step": 870
    },
    {
      "epoch": 2.29,
      "learning_rate": 2.3131747660339396e-06,
      "logits/chosen": -1.867693543434143,
      "logits/rejected": -1.8564653396606445,
      "logps/chosen": -31.62196922302246,
      "logps/rejected": -33.34123992919922,
      "loss": 0.1484,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": 1.9876899719238281,
      "rewards/margins": 2.3230578899383545,
      "rewards/rejected": -0.3353681266307831,
      "step": 880
    },
    {
      "epoch": 2.31,
      "learning_rate": 2.256719512667651e-06,
      "logits/chosen": -1.9699180126190186,
      "logits/rejected": -1.9743999242782593,
      "logps/chosen": -30.770462036132812,
      "logps/rejected": -32.943992614746094,
      "loss": 0.1449,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 2.0475802421569824,
      "rewards/margins": 2.453592300415039,
      "rewards/rejected": -0.40601205825805664,
      "step": 890
    },
    {
      "epoch": 2.34,
      "learning_rate": 2.2003892454735786e-06,
      "logits/chosen": -1.8890403509140015,
      "logits/rejected": -1.881980538368225,
      "logps/chosen": -32.13774490356445,
      "logps/rejected": -32.89883041381836,
      "loss": 0.131,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": 2.173067808151245,
      "rewards/margins": 2.5704257488250732,
      "rewards/rejected": -0.397358238697052,
      "step": 900
    },
    {
      "epoch": 2.34,
      "eval_logits/chosen": -2.1704888343811035,
      "eval_logits/rejected": -2.165771961212158,
      "eval_logps/chosen": -33.59296798706055,
      "eval_logps/rejected": -37.2408447265625,
      "eval_loss": 0.4716451168060303,
      "eval_rewards/accuracies": 0.5859634280204773,
      "eval_rewards/chosen": 0.3532681465148926,
      "eval_rewards/margins": 0.1326470971107483,
      "eval_rewards/rejected": 0.22062106430530548,
      "eval_runtime": 145.952,
      "eval_samples_per_second": 2.35,
      "eval_steps_per_second": 0.295,
      "step": 900
    },
    {
      "epoch": 2.36,
      "learning_rate": 2.1442129043167877e-06,
      "logits/chosen": -1.8803346157073975,
      "logits/rejected": -1.8807852268218994,
      "logps/chosen": -28.69057846069336,
      "logps/rejected": -35.19638442993164,
      "loss": 0.1531,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 1.9812660217285156,
      "rewards/margins": 2.3866193294525146,
      "rewards/rejected": -0.4053533673286438,
      "step": 910
    },
    {
      "epoch": 2.39,
      "learning_rate": 2.088219349982323e-06,
      "logits/chosen": -1.8385648727416992,
      "logits/rejected": -1.8306169509887695,
      "logps/chosen": -29.56528091430664,
      "logps/rejected": -34.04883575439453,
      "loss": 0.1693,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 1.8189325332641602,
      "rewards/margins": 2.2009072303771973,
      "rewards/rejected": -0.3819747567176819,
      "step": 920
    },
    {
      "epoch": 2.42,
      "learning_rate": 2.0324373493478803e-06,
      "logits/chosen": -2.004300117492676,
      "logits/rejected": -2.004765033721924,
      "logps/chosen": -27.645904541015625,
      "logps/rejected": -33.17176818847656,
      "loss": 0.1901,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 1.8809070587158203,
      "rewards/margins": 2.126433849334717,
      "rewards/rejected": -0.24552695453166962,
      "step": 930
    },
    {
      "epoch": 2.44,
      "learning_rate": 1.976895560604729e-06,
      "logits/chosen": -1.883927583694458,
      "logits/rejected": -1.893393874168396,
      "logps/chosen": -31.66170883178711,
      "logps/rejected": -33.47935104370117,
      "loss": 0.1308,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": 2.2106852531433105,
      "rewards/margins": 2.602447509765625,
      "rewards/rejected": -0.3917620778083801,
      "step": 940
    },
    {
      "epoch": 2.47,
      "learning_rate": 1.921622518534466e-06,
      "logits/chosen": -1.9267995357513428,
      "logits/rejected": -1.9306179285049438,
      "logps/chosen": -28.294620513916016,
      "logps/rejected": -32.00605010986328,
      "loss": 0.1628,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": 1.882483720779419,
      "rewards/margins": 2.0984606742858887,
      "rewards/rejected": -0.21597695350646973,
      "step": 950
    },
    {
      "epoch": 2.49,
      "learning_rate": 1.8666466198491794e-06,
      "logits/chosen": -1.9250777959823608,
      "logits/rejected": -1.921263337135315,
      "logps/chosen": -31.393346786499023,
      "logps/rejected": -34.266326904296875,
      "loss": 0.1531,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 2.145495891571045,
      "rewards/margins": 2.4334449768066406,
      "rewards/rejected": -0.28794899582862854,
      "step": 960
    },
    {
      "epoch": 2.52,
      "learning_rate": 1.8119961086025376e-06,
      "logits/chosen": -1.8407471179962158,
      "logits/rejected": -1.84335458278656,
      "logps/chosen": -30.177082061767578,
      "logps/rejected": -35.5606803894043,
      "loss": 0.15,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": 2.1085331439971924,
      "rewards/margins": 2.411801815032959,
      "rewards/rejected": -0.3032683730125427,
      "step": 970
    },
    {
      "epoch": 2.55,
      "learning_rate": 1.7576990616793139e-06,
      "logits/chosen": -1.8759149312973022,
      "logits/rejected": -1.8695703744888306,
      "logps/chosen": -32.879905700683594,
      "logps/rejected": -36.688812255859375,
      "loss": 0.1591,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 2.020131826400757,
      "rewards/margins": 2.3616156578063965,
      "rewards/rejected": -0.34148353338241577,
      "step": 980
    },
    {
      "epoch": 2.57,
      "learning_rate": 1.7037833743707892e-06,
      "logits/chosen": -1.8502181768417358,
      "logits/rejected": -1.845171332359314,
      "logps/chosen": -28.556514739990234,
      "logps/rejected": -36.38850784301758,
      "loss": 0.167,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 1.9571492671966553,
      "rewards/margins": 2.2760462760925293,
      "rewards/rejected": -0.31889697909355164,
      "step": 990
    },
    {
      "epoch": 2.6,
      "learning_rate": 1.6502767460434588e-06,
      "logits/chosen": -1.8279926776885986,
      "logits/rejected": -1.8174269199371338,
      "logps/chosen": -29.500579833984375,
      "logps/rejected": -29.74460220336914,
      "loss": 0.1784,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 1.798485517501831,
      "rewards/margins": 1.9143791198730469,
      "rewards/rejected": -0.11589355766773224,
      "step": 1000
    },
    {
      "epoch": 2.6,
      "eval_logits/chosen": -2.169346570968628,
      "eval_logits/rejected": -2.1646294593811035,
      "eval_logps/chosen": -33.57487106323242,
      "eval_logps/rejected": -37.250404357910156,
      "eval_loss": 0.4672938883304596,
      "eval_rewards/accuracies": 0.5859634280204773,
      "eval_rewards/chosen": 0.3677443265914917,
      "eval_rewards/margins": 0.15477292239665985,
      "eval_rewards/rejected": 0.21297141909599304,
      "eval_runtime": 145.9529,
      "eval_samples_per_second": 2.35,
      "eval_steps_per_second": 0.295,
      "step": 1000
    },
    {
      "epoch": 2.62,
      "learning_rate": 1.5972066659083796e-06,
      "logits/chosen": -1.938071846961975,
      "logits/rejected": -1.9374967813491821,
      "logps/chosen": -29.38279151916504,
      "logps/rejected": -30.51906394958496,
      "loss": 0.1659,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 1.953122854232788,
      "rewards/margins": 2.1615920066833496,
      "rewards/rejected": -0.20846888422966003,
      "step": 1010
    },
    {
      "epoch": 2.65,
      "learning_rate": 1.5446003988985041e-06,
      "logits/chosen": -1.9777361154556274,
      "logits/rejected": -1.9785385131835938,
      "logps/chosen": -29.455398559570312,
      "logps/rejected": -31.60211753845215,
      "loss": 0.1457,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": 1.9905421733856201,
      "rewards/margins": 2.3395895957946777,
      "rewards/rejected": -0.3490474820137024,
      "step": 1020
    },
    {
      "epoch": 2.68,
      "learning_rate": 1.4924849716612211e-06,
      "logits/chosen": -1.948656439781189,
      "logits/rejected": -1.951843500137329,
      "logps/chosen": -29.8292236328125,
      "logps/rejected": -28.163440704345703,
      "loss": 0.1712,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 1.960677146911621,
      "rewards/margins": 2.1044540405273438,
      "rewards/rejected": -0.14377683401107788,
      "step": 1030
    },
    {
      "epoch": 2.7,
      "learning_rate": 1.440887158673332e-06,
      "logits/chosen": -1.9582103490829468,
      "logits/rejected": -1.9505630731582642,
      "logps/chosen": -28.690776824951172,
      "logps/rejected": -33.91931915283203,
      "loss": 0.1486,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 1.9896939992904663,
      "rewards/margins": 2.3249640464782715,
      "rewards/rejected": -0.33527034521102905,
      "step": 1040
    },
    {
      "epoch": 2.73,
      "learning_rate": 1.3898334684855647e-06,
      "logits/chosen": -1.8943002223968506,
      "logits/rejected": -1.905225157737732,
      "logps/chosen": -30.873615264892578,
      "logps/rejected": -32.47500228881836,
      "loss": 0.1427,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 2.143594264984131,
      "rewards/margins": 2.3275885581970215,
      "rewards/rejected": -0.18399406969547272,
      "step": 1050
    },
    {
      "epoch": 2.75,
      "learning_rate": 1.3393501301037245e-06,
      "logits/chosen": -1.9732818603515625,
      "logits/rejected": -1.9640705585479736,
      "logps/chosen": -30.988611221313477,
      "logps/rejected": -36.728416442871094,
      "loss": 0.171,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 2.009733200073242,
      "rewards/margins": 2.327359199523926,
      "rewards/rejected": -0.31762614846229553,
      "step": 1060
    },
    {
      "epoch": 2.78,
      "learning_rate": 1.2894630795134454e-06,
      "logits/chosen": -1.8804638385772705,
      "logits/rejected": -1.8824098110198975,
      "logps/chosen": -33.24834442138672,
      "logps/rejected": -33.089820861816406,
      "loss": 0.1391,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 2.1927542686462402,
      "rewards/margins": 2.506317615509033,
      "rewards/rejected": -0.31356340646743774,
      "step": 1070
    },
    {
      "epoch": 2.81,
      "learning_rate": 1.2401979463554984e-06,
      "logits/chosen": -2.0114994049072266,
      "logits/rejected": -2.012619972229004,
      "logps/chosen": -30.447467803955078,
      "logps/rejected": -34.64774703979492,
      "loss": 0.1376,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": 2.088136911392212,
      "rewards/margins": 2.513786792755127,
      "rewards/rejected": -0.42564988136291504,
      "step": 1080
    },
    {
      "epoch": 2.83,
      "learning_rate": 1.1915800407584705e-06,
      "logits/chosen": -1.9806941747665405,
      "logits/rejected": -1.9849655628204346,
      "logps/chosen": -28.488088607788086,
      "logps/rejected": -33.84292984008789,
      "loss": 0.1691,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 1.9241825342178345,
      "rewards/margins": 2.160259485244751,
      "rewards/rejected": -0.23607663810253143,
      "step": 1090
    },
    {
      "epoch": 2.86,
      "learning_rate": 1.1436343403356019e-06,
      "logits/chosen": -1.9700733423233032,
      "logits/rejected": -1.9752953052520752,
      "logps/chosen": -31.235626220703125,
      "logps/rejected": -30.409204483032227,
      "loss": 0.1956,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 1.8667011260986328,
      "rewards/margins": 1.8727245330810547,
      "rewards/rejected": -0.0060233683325350285,
      "step": 1100
    },
    {
      "epoch": 2.86,
      "eval_logits/chosen": -2.1669461727142334,
      "eval_logits/rejected": -2.162247896194458,
      "eval_logps/chosen": -33.58707809448242,
      "eval_logps/rejected": -37.24416732788086,
      "eval_loss": 0.47056397795677185,
      "eval_rewards/accuracies": 0.5859634280204773,
      "eval_rewards/chosen": 0.3579804301261902,
      "eval_rewards/margins": 0.14001740515232086,
      "eval_rewards/rejected": 0.21796302497386932,
      "eval_runtime": 145.9563,
      "eval_samples_per_second": 2.35,
      "eval_steps_per_second": 0.295,
      "step": 1100
    },
    {
      "epoch": 2.88,
      "learning_rate": 1.0963854773524548e-06,
      "logits/chosen": -1.9610811471939087,
      "logits/rejected": -1.9610563516616821,
      "logps/chosen": -30.302623748779297,
      "logps/rejected": -31.24424171447754,
      "loss": 0.1495,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 2.1100215911865234,
      "rewards/margins": 2.2926793098449707,
      "rewards/rejected": -0.18265783786773682,
      "step": 1110
    },
    {
      "epoch": 2.91,
      "learning_rate": 1.049857726072005e-06,
      "logits/chosen": -1.8055435419082642,
      "logits/rejected": -1.8071558475494385,
      "logps/chosen": -31.964313507080078,
      "logps/rejected": -33.263916015625,
      "loss": 0.1722,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 2.08052921295166,
      "rewards/margins": 2.258647918701172,
      "rewards/rejected": -0.17811879515647888,
      "step": 1120
    },
    {
      "epoch": 2.94,
      "learning_rate": 1.0040749902836508e-06,
      "logits/chosen": -1.8441641330718994,
      "logits/rejected": -1.8414533138275146,
      "logps/chosen": -28.680904388427734,
      "logps/rejected": -31.291534423828125,
      "loss": 0.1972,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 1.8646504878997803,
      "rewards/margins": 1.9652849435806274,
      "rewards/rejected": -0.10063423216342926,
      "step": 1130
    },
    {
      "epoch": 2.96,
      "learning_rate": 9.59060791022566e-07,
      "logits/chosen": -1.979723334312439,
      "logits/rejected": -1.9741865396499634,
      "logps/chosen": -30.334341049194336,
      "logps/rejected": -33.13202667236328,
      "loss": 0.1522,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 2.1643624305725098,
      "rewards/margins": 2.3067753314971924,
      "rewards/rejected": -0.14241299033164978,
      "step": 1140
    },
    {
      "epoch": 2.99,
      "learning_rate": 9.148382544856885e-07,
      "logits/chosen": -1.8336684703826904,
      "logits/rejected": -1.823862075805664,
      "logps/chosen": -30.90520668029785,
      "logps/rejected": -31.456531524658203,
      "loss": 0.1763,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 2.037224292755127,
      "rewards/margins": 2.1685280799865723,
      "rewards/rejected": -0.13130377233028412,
      "step": 1150
    },
    {
      "epoch": 3.01,
      "learning_rate": 8.714301001505568e-07,
      "logits/chosen": -1.9085521697998047,
      "logits/rejected": -1.908399224281311,
      "logps/chosen": -31.05959701538086,
      "logps/rejected": -31.56801414489746,
      "loss": 0.1469,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": 2.09771990776062,
      "rewards/margins": 2.3909716606140137,
      "rewards/rejected": -0.2932516634464264,
      "step": 1160
    },
    {
      "epoch": 3.04,
      "learning_rate": 8.288586291031025e-07,
      "logits/chosen": -1.9817787408828735,
      "logits/rejected": -1.9764562845230103,
      "logps/chosen": -30.92386817932129,
      "logps/rejected": -33.19076919555664,
      "loss": 0.1496,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 2.0816802978515625,
      "rewards/margins": 2.2813220024108887,
      "rewards/rejected": -0.19964155554771423,
      "step": 1170
    },
    {
      "epoch": 3.06,
      "learning_rate": 7.871457125803897e-07,
      "logits/chosen": -1.8389742374420166,
      "logits/rejected": -1.8466594219207764,
      "logps/chosen": -30.842212677001953,
      "logps/rejected": -32.71555709838867,
      "loss": 0.1498,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": 1.9926265478134155,
      "rewards/margins": 2.2759203910827637,
      "rewards/rejected": -0.28329357504844666,
      "step": 1180
    },
    {
      "epoch": 3.09,
      "learning_rate": 7.463127807341966e-07,
      "logits/chosen": -1.9125378131866455,
      "logits/rejected": -1.9067026376724243,
      "logps/chosen": -29.305688858032227,
      "logps/rejected": -33.628910064697266,
      "loss": 0.1274,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 2.294342279434204,
      "rewards/margins": 2.570711612701416,
      "rewards/rejected": -0.27636927366256714,
      "step": 1190
    },
    {
      "epoch": 3.12,
      "learning_rate": 7.063808116212021e-07,
      "logits/chosen": -1.8545643091201782,
      "logits/rejected": -1.8566617965698242,
      "logps/chosen": -30.835662841796875,
      "logps/rejected": -33.26459503173828,
      "loss": 0.137,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 2.1238210201263428,
      "rewards/margins": 2.5274574756622314,
      "rewards/rejected": -0.403636634349823,
      "step": 1200
    },
    {
      "epoch": 3.12,
      "eval_logits/chosen": -2.167179584503174,
      "eval_logits/rejected": -2.1624755859375,
      "eval_logps/chosen": -33.572837829589844,
      "eval_logps/rejected": -37.24382400512695,
      "eval_loss": 0.4679512083530426,
      "eval_rewards/accuracies": 0.6063122749328613,
      "eval_rewards/chosen": 0.36937177181243896,
      "eval_rewards/margins": 0.15113388001918793,
      "eval_rewards/rejected": 0.21823787689208984,
      "eval_runtime": 145.8851,
      "eval_samples_per_second": 2.351,
      "eval_steps_per_second": 0.295,
      "step": 1200
    },
    {
      "epoch": 3.14,
      "learning_rate": 6.673703204254348e-07,
      "logits/chosen": -1.7856515645980835,
      "logits/rejected": -1.784690260887146,
      "logps/chosen": -33.143531799316406,
      "logps/rejected": -33.10202407836914,
      "loss": 0.1273,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": 2.3271608352661133,
      "rewards/margins": 2.641291618347168,
      "rewards/rejected": -0.3141304850578308,
      "step": 1210
    },
    {
      "epoch": 3.17,
      "learning_rate": 6.293013489185315e-07,
      "logits/chosen": -1.9604772329330444,
      "logits/rejected": -1.9546642303466797,
      "logps/chosen": -28.979385375976562,
      "logps/rejected": -33.37938690185547,
      "loss": 0.1352,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 2.1794400215148926,
      "rewards/margins": 2.5801169872283936,
      "rewards/rejected": -0.40067729353904724,
      "step": 1220
    },
    {
      "epoch": 3.19,
      "learning_rate": 5.921934551632086e-07,
      "logits/chosen": -1.8098831176757812,
      "logits/rejected": -1.798370361328125,
      "logps/chosen": -31.50105857849121,
      "logps/rejected": -33.25225067138672,
      "loss": 0.1311,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 2.3076980113983154,
      "rewards/margins": 2.5944409370422363,
      "rewards/rejected": -0.2867427468299866,
      "step": 1230
    },
    {
      "epoch": 3.22,
      "learning_rate": 5.560657034652405e-07,
      "logits/chosen": -1.9010969400405884,
      "logits/rejected": -1.8942989110946655,
      "logps/chosen": -28.258697509765625,
      "logps/rejected": -29.34195327758789,
      "loss": 0.1603,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 1.9532296657562256,
      "rewards/margins": 2.33274507522583,
      "rewards/rejected": -0.3795151114463806,
      "step": 1240
    },
    {
      "epoch": 3.25,
      "learning_rate": 5.2093665457911e-07,
      "logits/chosen": -1.9308841228485107,
      "logits/rejected": -1.9373613595962524,
      "logps/chosen": -32.524845123291016,
      "logps/rejected": -31.774621963500977,
      "loss": 0.1315,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 2.2964370250701904,
      "rewards/margins": 2.5014843940734863,
      "rewards/rejected": -0.20504704117774963,
      "step": 1250
    },
    {
      "epoch": 3.27,
      "learning_rate": 4.868243561723535e-07,
      "logits/chosen": -1.9048112630844116,
      "logits/rejected": -1.9054410457611084,
      "logps/chosen": -30.393224716186523,
      "logps/rejected": -33.286048889160156,
      "loss": 0.1317,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": 2.1082680225372314,
      "rewards/margins": 2.485210418701172,
      "rewards/rejected": -0.37694239616394043,
      "step": 1260
    },
    {
      "epoch": 3.3,
      "learning_rate": 4.537463335535161e-07,
      "logits/chosen": -1.846727728843689,
      "logits/rejected": -1.8463557958602905,
      "logps/chosen": -30.23922348022461,
      "logps/rejected": -33.92576217651367,
      "loss": 0.1224,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 2.2812910079956055,
      "rewards/margins": 2.6054136753082275,
      "rewards/rejected": -0.3241223692893982,
      "step": 1270
    },
    {
      "epoch": 3.32,
      "learning_rate": 4.217195806684629e-07,
      "logits/chosen": -1.730064034461975,
      "logits/rejected": -1.7253376245498657,
      "logps/chosen": -32.30054473876953,
      "logps/rejected": -31.246471405029297,
      "loss": 0.1154,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 2.349520206451416,
      "rewards/margins": 2.605508327484131,
      "rewards/rejected": -0.2559877932071686,
      "step": 1280
    },
    {
      "epoch": 3.35,
      "learning_rate": 3.907605513696808e-07,
      "logits/chosen": -1.9358218908309937,
      "logits/rejected": -1.9203758239746094,
      "logps/chosen": -31.7402400970459,
      "logps/rejected": -35.388179779052734,
      "loss": 0.1234,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": 2.085869550704956,
      "rewards/margins": 2.5841598510742188,
      "rewards/rejected": -0.4982902407646179,
      "step": 1290
    },
    {
      "epoch": 3.38,
      "learning_rate": 3.6088515096305675e-07,
      "logits/chosen": -1.879227638244629,
      "logits/rejected": -1.8840656280517578,
      "logps/chosen": -30.69500732421875,
      "logps/rejected": -36.70278549194336,
      "loss": 0.1211,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": 2.2580931186676025,
      "rewards/margins": 2.7377448081970215,
      "rewards/rejected": -0.4796522259712219,
      "step": 1300
    },
    {
      "epoch": 3.38,
      "eval_logits/chosen": -2.1668777465820312,
      "eval_logits/rejected": -2.1621785163879395,
      "eval_logps/chosen": -33.58045959472656,
      "eval_logps/rejected": -37.23927688598633,
      "eval_loss": 0.47049835324287415,
      "eval_rewards/accuracies": 0.5917773842811584,
      "eval_rewards/chosen": 0.3632733225822449,
      "eval_rewards/margins": 0.14139726758003235,
      "eval_rewards/rejected": 0.22187604010105133,
      "eval_runtime": 145.9507,
      "eval_samples_per_second": 2.35,
      "eval_steps_per_second": 0.295,
      "step": 1300
    },
    {
      "epoch": 3.4,
      "learning_rate": 3.321087280364757e-07,
      "logits/chosen": -1.8520374298095703,
      "logits/rejected": -1.8522069454193115,
      "logps/chosen": -33.18568801879883,
      "logps/rejected": -37.46868896484375,
      "loss": 0.134,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 2.446664333343506,
      "rewards/margins": 2.7543227672576904,
      "rewards/rejected": -0.30765873193740845,
      "step": 1310
    },
    {
      "epoch": 3.43,
      "learning_rate": 3.044460665744284e-07,
      "logits/chosen": -1.9394071102142334,
      "logits/rejected": -1.9380232095718384,
      "logps/chosen": -29.4769344329834,
      "logps/rejected": -31.30813980102539,
      "loss": 0.159,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 2.093536853790283,
      "rewards/margins": 2.3773601055145264,
      "rewards/rejected": -0.2838229537010193,
      "step": 1320
    },
    {
      "epoch": 3.45,
      "learning_rate": 2.779113783626916e-07,
      "logits/chosen": -1.844186782836914,
      "logits/rejected": -1.8454091548919678,
      "logps/chosen": -31.52813720703125,
      "logps/rejected": -33.989559173583984,
      "loss": 0.1366,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": 2.276893138885498,
      "rewards/margins": 2.549605131149292,
      "rewards/rejected": -0.27271172404289246,
      "step": 1330
    },
    {
      "epoch": 3.48,
      "learning_rate": 2.5251829568697204e-07,
      "logits/chosen": -1.9051072597503662,
      "logits/rejected": -1.9038896560668945,
      "logps/chosen": -28.674198150634766,
      "logps/rejected": -32.49821090698242,
      "loss": 0.1373,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": 2.01957631111145,
      "rewards/margins": 2.424564838409424,
      "rewards/rejected": -0.4049888551235199,
      "step": 1340
    },
    {
      "epoch": 3.51,
      "learning_rate": 2.2827986432927774e-07,
      "logits/chosen": -1.9206342697143555,
      "logits/rejected": -1.9062645435333252,
      "logps/chosen": -31.68826675415039,
      "logps/rejected": -36.69408416748047,
      "loss": 0.1396,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": 2.102721691131592,
      "rewards/margins": 2.4464478492736816,
      "rewards/rejected": -0.34372615814208984,
      "step": 1350
    },
    {
      "epoch": 3.53,
      "learning_rate": 2.0520853686560177e-07,
      "logits/chosen": -1.921002984046936,
      "logits/rejected": -1.9334627389907837,
      "logps/chosen": -29.323871612548828,
      "logps/rejected": -32.485260009765625,
      "loss": 0.1368,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 2.195680856704712,
      "rewards/margins": 2.3920950889587402,
      "rewards/rejected": -0.1964142769575119,
      "step": 1360
    },
    {
      "epoch": 3.56,
      "learning_rate": 1.833161662683672e-07,
      "logits/chosen": -2.0142505168914795,
      "logits/rejected": -2.0135538578033447,
      "logps/chosen": -29.253097534179688,
      "logps/rejected": -36.559532165527344,
      "loss": 0.1185,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": 2.2218658924102783,
      "rewards/margins": 2.8047871589660645,
      "rewards/rejected": -0.5829211473464966,
      "step": 1370
    },
    {
      "epoch": 3.58,
      "learning_rate": 1.626139998169246e-07,
      "logits/chosen": -1.8797496557235718,
      "logits/rejected": -1.8874984979629517,
      "logps/chosen": -31.32064437866211,
      "logps/rejected": -37.89208221435547,
      "loss": 0.144,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 2.287868022918701,
      "rewards/margins": 2.641819477081299,
      "rewards/rejected": -0.3539513051509857,
      "step": 1380
    },
    {
      "epoch": 3.61,
      "learning_rate": 1.4311267331922535e-07,
      "logits/chosen": -1.8338594436645508,
      "logits/rejected": -1.8298187255859375,
      "logps/chosen": -31.842514038085938,
      "logps/rejected": -31.77047348022461,
      "loss": 0.1419,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 2.3397603034973145,
      "rewards/margins": 2.4782257080078125,
      "rewards/rejected": -0.13846543431282043,
      "step": 1390
    },
    {
      "epoch": 3.64,
      "learning_rate": 1.2482220564763669e-07,
      "logits/chosen": -1.9958465099334717,
      "logits/rejected": -1.9933058023452759,
      "logps/chosen": -28.800048828125,
      "logps/rejected": -32.510841369628906,
      "loss": 0.1553,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 1.9972022771835327,
      "rewards/margins": 2.2903003692626953,
      "rewards/rejected": -0.29309794306755066,
      "step": 1400
    },
    {
      "epoch": 3.64,
      "eval_logits/chosen": -2.1668412685394287,
      "eval_logits/rejected": -2.162137985229492,
      "eval_logps/chosen": -33.57229232788086,
      "eval_logps/rejected": -37.258155822753906,
      "eval_loss": 0.46543803811073303,
      "eval_rewards/accuracies": 0.6034052968025208,
      "eval_rewards/chosen": 0.36980658769607544,
      "eval_rewards/margins": 0.16303294897079468,
      "eval_rewards/rejected": 0.20677369832992554,
      "eval_runtime": 145.9296,
      "eval_samples_per_second": 2.35,
      "eval_steps_per_second": 0.295,
      "step": 1400
    },
    {
      "epoch": 3.66,
      "learning_rate": 1.0775199359171346e-07,
      "logits/chosen": -1.9372856616973877,
      "logits/rejected": -1.9331865310668945,
      "logps/chosen": -30.8883113861084,
      "logps/rejected": -29.84084129333496,
      "loss": 0.1363,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": 2.2033774852752686,
      "rewards/margins": 2.4324822425842285,
      "rewards/rejected": -0.22910502552986145,
      "step": 1410
    },
    {
      "epoch": 3.69,
      "learning_rate": 9.191080703056604e-08,
      "logits/chosen": -1.8948686122894287,
      "logits/rejected": -1.8960151672363281,
      "logps/chosen": -30.679889678955078,
      "logps/rejected": -35.14348602294922,
      "loss": 0.152,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 2.1479902267456055,
      "rewards/margins": 2.323575735092163,
      "rewards/rejected": -0.17558541893959045,
      "step": 1420
    },
    {
      "epoch": 3.71,
      "learning_rate": 7.730678442730539e-08,
      "logits/chosen": -1.8465667963027954,
      "logits/rejected": -1.8398901224136353,
      "logps/chosen": -31.448232650756836,
      "logps/rejected": -37.158573150634766,
      "loss": 0.1367,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 2.1548283100128174,
      "rewards/margins": 2.530158519744873,
      "rewards/rejected": -0.3753301799297333,
      "step": 1430
    },
    {
      "epoch": 3.74,
      "learning_rate": 6.394742864787806e-08,
      "logits/chosen": -1.859572172164917,
      "logits/rejected": -1.8541902303695679,
      "logps/chosen": -26.713485717773438,
      "logps/rejected": -32.027252197265625,
      "loss": 0.1488,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 2.0798051357269287,
      "rewards/margins": 2.317080020904541,
      "rewards/rejected": -0.2372746467590332,
      "step": 1440
    },
    {
      "epoch": 3.77,
      "learning_rate": 5.183960310644748e-08,
      "logits/chosen": -1.8826467990875244,
      "logits/rejected": -1.8725192546844482,
      "logps/chosen": -30.08587074279785,
      "logps/rejected": -36.018455505371094,
      "loss": 0.1431,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": 1.9165277481079102,
      "rewards/margins": 2.503002405166626,
      "rewards/rejected": -0.5864745378494263,
      "step": 1450
    },
    {
      "epoch": 3.79,
      "learning_rate": 4.098952823928693e-08,
      "logits/chosen": -1.8568382263183594,
      "logits/rejected": -1.8540499210357666,
      "logps/chosen": -30.78998374938965,
      "logps/rejected": -31.02651596069336,
      "loss": 0.1648,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 2.0158348083496094,
      "rewards/margins": 2.124389886856079,
      "rewards/rejected": -0.10855510085821152,
      "step": 1460
    },
    {
      "epoch": 3.82,
      "learning_rate": 3.1402778309014284e-08,
      "logits/chosen": -1.9178192615509033,
      "logits/rejected": -1.924325704574585,
      "logps/chosen": -29.135822296142578,
      "logps/rejected": -33.322288513183594,
      "loss": 0.1271,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": 2.3466856479644775,
      "rewards/margins": 2.634577989578247,
      "rewards/rejected": -0.28789228200912476,
      "step": 1470
    },
    {
      "epoch": 3.84,
      "learning_rate": 2.3084278540791427e-08,
      "logits/chosen": -1.9184119701385498,
      "logits/rejected": -1.9287493228912354,
      "logps/chosen": -29.0635986328125,
      "logps/rejected": -29.924457550048828,
      "loss": 0.1386,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": 2.1053242683410645,
      "rewards/margins": 2.342036008834839,
      "rewards/rejected": -0.2367115318775177,
      "step": 1480
    },
    {
      "epoch": 3.87,
      "learning_rate": 1.6038302591975807e-08,
      "logits/chosen": -1.8501653671264648,
      "logits/rejected": -1.8432512283325195,
      "logps/chosen": -31.104766845703125,
      "logps/rejected": -32.60163879394531,
      "loss": 0.144,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": 2.0413756370544434,
      "rewards/margins": 2.3543033599853516,
      "rewards/rejected": -0.31292763352394104,
      "step": 1490
    },
    {
      "epoch": 3.9,
      "learning_rate": 1.0268470356514237e-08,
      "logits/chosen": -1.9025371074676514,
      "logits/rejected": -1.8992702960968018,
      "logps/chosen": -31.016992568969727,
      "logps/rejected": -33.745033264160156,
      "loss": 0.1447,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": 2.086864709854126,
      "rewards/margins": 2.474714517593384,
      "rewards/rejected": -0.3878494203090668,
      "step": 1500
    },
    {
      "epoch": 3.9,
      "eval_logits/chosen": -2.1669082641601562,
      "eval_logits/rejected": -2.1621992588043213,
      "eval_logps/chosen": -33.575931549072266,
      "eval_logps/rejected": -37.24650192260742,
      "eval_loss": 0.46842658519744873,
      "eval_rewards/accuracies": 0.574335515499115,
      "eval_rewards/chosen": 0.3668961524963379,
      "eval_rewards/margins": 0.15079911053180695,
      "eval_rewards/rejected": 0.21609705686569214,
      "eval_runtime": 145.9178,
      "eval_samples_per_second": 2.351,
      "eval_steps_per_second": 0.295,
      "step": 1500
    },
    {
      "epoch": 3.92,
      "learning_rate": 5.777746105209147e-09,
      "logits/chosen": -1.9817174673080444,
      "logits/rejected": -1.9823877811431885,
      "logps/chosen": -27.261791229248047,
      "logps/rejected": -33.12643814086914,
      "loss": 0.1654,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 2.0435855388641357,
      "rewards/margins": 2.35237717628479,
      "rewards/rejected": -0.30879148840904236,
      "step": 1510
    },
    {
      "epoch": 3.95,
      "learning_rate": 2.5684369628148352e-09,
      "logits/chosen": -1.834775686264038,
      "logits/rejected": -1.8347667455673218,
      "logps/chosen": -30.5068416595459,
      "logps/rejected": -34.2733154296875,
      "loss": 0.1565,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 2.0280237197875977,
      "rewards/margins": 2.347689151763916,
      "rewards/rejected": -0.31966525316238403,
      "step": 1520
    },
    {
      "epoch": 3.97,
      "learning_rate": 6.421917227455999e-10,
      "logits/chosen": -1.986581563949585,
      "logits/rejected": -1.9839556217193604,
      "logps/chosen": -29.06634521484375,
      "logps/rejected": -31.9202938079834,
      "loss": 0.1594,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": 1.926171898841858,
      "rewards/margins": 2.2754859924316406,
      "rewards/rejected": -0.34931421279907227,
      "step": 1530
    },
    {
      "epoch": 4.0,
      "learning_rate": 0.0,
      "logits/chosen": -1.9706960916519165,
      "logits/rejected": -1.9718106985092163,
      "logps/chosen": -28.249774932861328,
      "logps/rejected": -30.143047332763672,
      "loss": 0.1594,
      "rewards/accuracies": 0.9583333730697632,
      "rewards/chosen": 1.8961349725723267,
      "rewards/margins": 2.178429126739502,
      "rewards/rejected": -0.2822941541671753,
      "step": 1540
    },
    {
      "epoch": 4.0,
      "step": 1540,
      "total_flos": 0.0,
      "train_loss": 0.16076272354497537,
      "train_runtime": 10802.3033,
      "train_samples_per_second": 1.14,
      "train_steps_per_second": 0.143
    }
  ],
  "logging_steps": 10,
  "max_steps": 1540,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 4,
  "save_steps": 100,
  "total_flos": 0.0,
  "train_batch_size": 4,
  "trial_name": null,
  "trial_params": null
}