{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 2.9765925925925925,
  "eval_steps": 500,
  "global_step": 315,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.047407407407407405,
      "grad_norm": 631.015625,
      "learning_rate": 1.1718749999999999e-05,
      "log_odds_chosen": 4.92927360534668,
      "log_odds_ratio": -9.756586074829102,
      "logits/chosen": 138.57809448242188,
      "logits/rejected": 153.22152709960938,
      "logps/chosen": -20.633487701416016,
      "logps/rejected": -25.562952041625977,
      "loss": 452.9617,
      "nll_loss": 8.288494110107422,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -10.316743850708008,
      "rewards/margins": 2.4647319316864014,
      "rewards/rejected": -12.781476020812988,
      "step": 5
    },
    {
      "epoch": 0.09481481481481481,
      "grad_norm": 552.6162719726562,
      "learning_rate": 2.3437499999999997e-05,
      "log_odds_chosen": 2.3541274070739746,
      "log_odds_ratio": -7.061516761779785,
      "logits/chosen": 139.1152801513672,
      "logits/rejected": 163.85743713378906,
      "logps/chosen": -15.125625610351562,
      "logps/rejected": -17.478649139404297,
      "loss": 374.0408,
      "nll_loss": 7.115235805511475,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -7.562812805175781,
      "rewards/margins": 1.1765111684799194,
      "rewards/rejected": -8.739324569702148,
      "step": 10
    },
    {
      "epoch": 0.14222222222222222,
      "grad_norm": 2310.692626953125,
      "learning_rate": 3.515625e-05,
      "log_odds_chosen": 4.172502040863037,
      "log_odds_ratio": -6.55011510848999,
      "logits/chosen": 125.50711822509766,
      "logits/rejected": 151.8711395263672,
      "logps/chosen": -17.319055557250977,
      "logps/rejected": -21.48957061767578,
      "loss": 325.2039,
      "nll_loss": 7.520108222961426,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -8.659527778625488,
      "rewards/margins": 2.0852572917938232,
      "rewards/rejected": -10.74478530883789,
      "step": 15
    },
    {
      "epoch": 0.18962962962962962,
      "grad_norm": 438.0211486816406,
      "learning_rate": 4.6874999999999994e-05,
      "log_odds_chosen": -0.5363087058067322,
      "log_odds_ratio": -2.402191638946533,
      "logits/chosen": 162.6514129638672,
      "logits/rejected": 166.47042846679688,
      "logps/chosen": -5.853331565856934,
      "logps/rejected": -5.3130202293396,
      "loss": 147.371,
      "nll_loss": 3.4735121726989746,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": -2.926665782928467,
      "rewards/margins": -0.27015575766563416,
      "rewards/rejected": -2.6565101146698,
      "step": 20
    },
    {
      "epoch": 0.23703703703703705,
      "grad_norm": 358.36737060546875,
      "learning_rate": 5.859375e-05,
      "log_odds_chosen": 0.3639217019081116,
      "log_odds_ratio": -0.7449954748153687,
      "logits/chosen": 196.79856872558594,
      "logits/rejected": 204.99942016601562,
      "logps/chosen": -1.7752330303192139,
      "logps/rejected": -2.1123547554016113,
      "loss": 77.053,
      "nll_loss": 2.062661647796631,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.8876165151596069,
      "rewards/margins": 0.16856074333190918,
      "rewards/rejected": -1.0561773777008057,
      "step": 25
    },
    {
      "epoch": 0.28444444444444444,
      "grad_norm": 316.6930236816406,
      "learning_rate": 7.03125e-05,
      "log_odds_chosen": 0.1705448478460312,
      "log_odds_ratio": -0.7665665745735168,
      "logits/chosen": 221.40567016601562,
      "logits/rejected": 226.0666046142578,
      "logps/chosen": -1.6369438171386719,
      "logps/rejected": -1.7822250127792358,
      "loss": 73.8902,
      "nll_loss": 1.9145492315292358,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.8184719085693359,
      "rewards/margins": 0.07264064252376556,
      "rewards/rejected": -0.8911125063896179,
      "step": 30
    },
    {
      "epoch": 0.33185185185185184,
      "grad_norm": 253.6866455078125,
      "learning_rate": 7.497920636023175e-05,
      "log_odds_chosen": 0.025636177510023117,
      "log_odds_ratio": -0.821603000164032,
      "logits/chosen": 216.9961700439453,
      "logits/rejected": 218.8159637451172,
      "logps/chosen": -1.591193675994873,
      "logps/rejected": -1.6127433776855469,
      "loss": 69.6548,
      "nll_loss": 1.8054049015045166,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": -0.7955968379974365,
      "rewards/margins": 0.010774785652756691,
      "rewards/rejected": -0.8063716888427734,
      "step": 35
    },
    {
      "epoch": 0.37925925925925924,
      "grad_norm": 118.463623046875,
      "learning_rate": 7.485221761715193e-05,
      "log_odds_chosen": 0.2294560670852661,
      "log_odds_ratio": -0.6809016466140747,
      "logits/chosen": 203.59107971191406,
      "logits/rejected": 201.36801147460938,
      "logps/chosen": -1.3837480545043945,
      "logps/rejected": -1.5675389766693115,
      "loss": 65.3998,
      "nll_loss": 1.682397484779358,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -0.6918740272521973,
      "rewards/margins": 0.09189549833536148,
      "rewards/rejected": -0.7837694883346558,
      "step": 40
    },
    {
      "epoch": 0.4266666666666667,
      "grad_norm": 168.12261962890625,
      "learning_rate": 7.461018280806719e-05,
      "log_odds_chosen": 0.19497309625148773,
      "log_odds_ratio": -0.7434221506118774,
      "logits/chosen": 181.99807739257812,
      "logits/rejected": 182.77682495117188,
      "logps/chosen": -1.381994366645813,
      "logps/rejected": -1.541455626487732,
      "loss": 63.7627,
      "nll_loss": 1.6508877277374268,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.6909971833229065,
      "rewards/margins": 0.07973069697618484,
      "rewards/rejected": -0.770727813243866,
      "step": 45
    },
    {
      "epoch": 0.4740740740740741,
      "grad_norm": 278.8425598144531,
      "learning_rate": 7.425384740819691e-05,
      "log_odds_chosen": 0.21113845705986023,
      "log_odds_ratio": -0.6987509727478027,
      "logits/chosen": 176.6704864501953,
      "logits/rejected": 177.52133178710938,
      "logps/chosen": -1.358520269393921,
      "logps/rejected": -1.5214390754699707,
      "loss": 64.9056,
      "nll_loss": 1.7045795917510986,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.6792601346969604,
      "rewards/margins": 0.08145944774150848,
      "rewards/rejected": -0.7607195377349854,
      "step": 50
    },
    {
      "epoch": 0.5214814814814814,
      "grad_norm": 85.80127716064453,
      "learning_rate": 7.378430894236152e-05,
      "log_odds_chosen": 0.12195976823568344,
      "log_odds_ratio": -0.7067708373069763,
      "logits/chosen": 183.97714233398438,
      "logits/rejected": 180.34552001953125,
      "logps/chosen": -1.2856115102767944,
      "logps/rejected": -1.3806015253067017,
      "loss": 61.2303,
      "nll_loss": 1.5915721654891968,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.6428057551383972,
      "rewards/margins": 0.04749499261379242,
      "rewards/rejected": -0.6903007626533508,
      "step": 55
    },
    {
      "epoch": 0.5688888888888889,
      "grad_norm": 93.30135345458984,
      "learning_rate": 7.320301360456979e-05,
      "log_odds_chosen": 0.25577789545059204,
      "log_odds_ratio": -0.6447214484214783,
      "logits/chosen": 187.6861572265625,
      "logits/rejected": 176.8941192626953,
      "logps/chosen": -1.1724777221679688,
      "logps/rejected": -1.3559234142303467,
      "loss": 58.3784,
      "nll_loss": 1.4778730869293213,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.5862388610839844,
      "rewards/margins": 0.09172289073467255,
      "rewards/rejected": -0.6779617071151733,
      "step": 60
    },
    {
      "epoch": 0.6162962962962963,
      "grad_norm": 94.57402801513672,
      "learning_rate": 7.251175180369343e-05,
      "log_odds_chosen": 0.20048114657402039,
      "log_odds_ratio": -0.6868171095848083,
      "logits/chosen": 186.01736450195312,
      "logits/rejected": 187.6791534423828,
      "logps/chosen": -1.2308342456817627,
      "logps/rejected": -1.3798383474349976,
      "loss": 57.794,
      "nll_loss": 1.496966004371643,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.6154171228408813,
      "rewards/margins": 0.07450205832719803,
      "rewards/rejected": -0.6899191737174988,
      "step": 65
    },
    {
      "epoch": 0.6637037037037037,
      "grad_norm": 152.06646728515625,
      "learning_rate": 7.171265264894856e-05,
      "log_odds_chosen": 0.27526047825813293,
      "log_odds_ratio": -0.6563318967819214,
      "logits/chosen": 200.08290100097656,
      "logits/rejected": 180.6443328857422,
      "logps/chosen": -1.1693233251571655,
      "logps/rejected": -1.3890639543533325,
      "loss": 57.0621,
      "nll_loss": 1.482000708580017,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.5846616625785828,
      "rewards/margins": 0.10987037420272827,
      "rewards/rejected": -0.6945319771766663,
      "step": 70
    },
    {
      "epoch": 0.7111111111111111,
      "grad_norm": 103.90552520751953,
      "learning_rate": 7.080817739216882e-05,
      "log_odds_chosen": 0.3350622057914734,
      "log_odds_ratio": -0.6169534921646118,
      "logits/chosen": 186.0503692626953,
      "logits/rejected": 194.65689086914062,
      "logps/chosen": -1.1045267581939697,
      "logps/rejected": -1.351041555404663,
      "loss": 55.9369,
      "nll_loss": 1.4453600645065308,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.5522633790969849,
      "rewards/margins": 0.12325732409954071,
      "rewards/rejected": -0.6755207777023315,
      "step": 75
    },
    {
      "epoch": 0.7585185185185185,
      "grad_norm": 90.7190170288086,
      "learning_rate": 6.980111184706833e-05,
      "log_odds_chosen": 0.2888467609882355,
      "log_odds_ratio": -0.6715133190155029,
      "logits/chosen": 204.8794403076172,
      "logits/rejected": 193.88922119140625,
      "logps/chosen": -1.177809238433838,
      "logps/rejected": -1.3896067142486572,
      "loss": 56.0751,
      "nll_loss": 1.4720895290374756,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.588904619216919,
      "rewards/margins": 0.10589867830276489,
      "rewards/rejected": -0.6948033571243286,
      "step": 80
    },
    {
      "epoch": 0.8059259259259259,
      "grad_norm": 69.81222534179688,
      "learning_rate": 6.869455780884331e-05,
      "log_odds_chosen": 0.23772254586219788,
      "log_odds_ratio": -0.6677379608154297,
      "logits/chosen": 197.03091430664062,
      "logits/rejected": 172.4070587158203,
      "logps/chosen": -1.120238184928894,
      "logps/rejected": -1.296899437904358,
      "loss": 55.6046,
      "nll_loss": 1.4089494943618774,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.560119092464447,
      "rewards/margins": 0.08833061158657074,
      "rewards/rejected": -0.648449718952179,
      "step": 85
    },
    {
      "epoch": 0.8533333333333334,
      "grad_norm": 95.9283676147461,
      "learning_rate": 6.749192350054026e-05,
      "log_odds_chosen": 0.31241756677627563,
      "log_odds_ratio": -0.6360076665878296,
      "logits/chosen": 196.90533447265625,
      "logits/rejected": 185.89779663085938,
      "logps/chosen": -1.0959351062774658,
      "logps/rejected": -1.3153069019317627,
      "loss": 57.8707,
      "nll_loss": 1.4560327529907227,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.5479675531387329,
      "rewards/margins": 0.10968589782714844,
      "rewards/rejected": -0.6576534509658813,
      "step": 90
    },
    {
      "epoch": 0.9007407407407407,
      "grad_norm": 71.1992416381836,
      "learning_rate": 6.619691307561615e-05,
      "log_odds_chosen": 0.406308114528656,
      "log_odds_ratio": -0.5838354825973511,
      "logits/chosen": 188.7007293701172,
      "logits/rejected": 175.3910369873047,
      "logps/chosen": -1.0477182865142822,
      "logps/rejected": -1.3336520195007324,
      "loss": 55.6848,
      "nll_loss": 1.3589107990264893,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.5238591432571411,
      "rewards/margins": 0.1429668366909027,
      "rewards/rejected": -0.6668260097503662,
      "step": 95
    },
    {
      "epoch": 0.9481481481481482,
      "grad_norm": 104.38742065429688,
      "learning_rate": 6.48135152090233e-05,
      "log_odds_chosen": 0.1892620474100113,
      "log_odds_ratio": -0.7005770206451416,
      "logits/chosen": 185.79942321777344,
      "logits/rejected": 171.77197265625,
      "logps/chosen": -1.1602965593338013,
      "logps/rejected": -1.3126600980758667,
      "loss": 56.5279,
      "nll_loss": 1.4600017070770264,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.5801482796669006,
      "rewards/margins": 0.07618186622858047,
      "rewards/rejected": -0.6563300490379333,
      "step": 100
    },
    {
      "epoch": 0.9955555555555555,
      "grad_norm": 231.78350830078125,
      "learning_rate": 6.33459908119585e-05,
      "log_odds_chosen": 0.29791754484176636,
      "log_odds_ratio": -0.6211925745010376,
      "logits/chosen": 185.2686004638672,
      "logits/rejected": 171.43771362304688,
      "logps/chosen": -1.0696375370025635,
      "logps/rejected": -1.2886693477630615,
      "loss": 54.4392,
      "nll_loss": 1.3739655017852783,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.5348187685012817,
      "rewards/margins": 0.10951590538024902,
      "rewards/rejected": -0.6443346738815308,
      "step": 105
    },
    {
      "epoch": 1.037925925925926,
      "grad_norm": 86.91617584228516,
      "learning_rate": 6.179885990811568e-05,
      "log_odds_chosen": 0.8971813917160034,
      "log_odds_ratio": -0.496082067489624,
      "logits/chosen": 178.14752197265625,
      "logits/rejected": 169.9856414794922,
      "logps/chosen": -0.9173161387443542,
      "logps/rejected": -1.5359400510787964,
      "loss": 41.2989,
      "nll_loss": 1.1888840198516846,
      "rewards/accuracies": 0.811188817024231,
      "rewards/chosen": -0.4586580693721771,
      "rewards/margins": 0.30931198596954346,
      "rewards/rejected": -0.7679700255393982,
      "step": 110
    },
    {
      "epoch": 1.0853333333333333,
      "grad_norm": 85.66360473632812,
      "learning_rate": 6.0176887711863254e-05,
      "log_odds_chosen": 1.2501261234283447,
      "log_odds_ratio": -0.3368678689002991,
      "logits/chosen": 172.72744750976562,
      "logits/rejected": 157.48776245117188,
      "logps/chosen": -0.82439124584198,
      "logps/rejected": -1.6705620288848877,
      "loss": 42.4544,
      "nll_loss": 1.1247361898422241,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -0.41219562292099,
      "rewards/margins": 0.42308536171913147,
      "rewards/rejected": -0.8352810144424438,
      "step": 115
    },
    {
      "epoch": 1.1327407407407408,
      "grad_norm": 58.510986328125,
      "learning_rate": 5.848506995122624e-05,
      "log_odds_chosen": 1.2685009241104126,
      "log_odds_ratio": -0.3527768850326538,
      "logits/chosen": 159.47116088867188,
      "logits/rejected": 157.1888885498047,
      "logps/chosen": -0.8306115865707397,
      "logps/rejected": -1.68930983543396,
      "loss": 42.8811,
      "nll_loss": 1.1527321338653564,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.4153057932853699,
      "rewards/margins": 0.4293490946292877,
      "rewards/rejected": -0.84465491771698,
      "step": 120
    },
    {
      "epoch": 1.1801481481481482,
      "grad_norm": 80.0993881225586,
      "learning_rate": 5.672861748087879e-05,
      "log_odds_chosen": 1.1959865093231201,
      "log_odds_ratio": -0.37415602803230286,
      "logits/chosen": 163.66038513183594,
      "logits/rejected": 159.62203979492188,
      "logps/chosen": -0.8715580701828003,
      "logps/rejected": -1.7014503479003906,
      "loss": 41.053,
      "nll_loss": 1.1238315105438232,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.43577903509140015,
      "rewards/margins": 0.41494613885879517,
      "rewards/rejected": -0.8507251739501953,
      "step": 125
    },
    {
      "epoch": 1.2275555555555555,
      "grad_norm": 82.62030792236328,
      "learning_rate": 5.491294023253923e-05,
      "log_odds_chosen": 1.1291576623916626,
      "log_odds_ratio": -0.3804819583892822,
      "logits/chosen": 178.26722717285156,
      "logits/rejected": 170.98712158203125,
      "logps/chosen": -0.8340187072753906,
      "logps/rejected": -1.5631098747253418,
      "loss": 41.0951,
      "nll_loss": 1.1230322122573853,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.4170093536376953,
      "rewards/margins": 0.3645455539226532,
      "rewards/rejected": -0.7815549373626709,
      "step": 130
    },
    {
      "epoch": 1.274962962962963,
      "grad_norm": 64.6174087524414,
      "learning_rate": 5.304363055220153e-05,
      "log_odds_chosen": 1.1577521562576294,
      "log_odds_ratio": -0.3698500394821167,
      "logits/chosen": 176.21475219726562,
      "logits/rejected": 174.70472717285156,
      "logps/chosen": -0.7851073741912842,
      "logps/rejected": -1.5308529138565063,
      "loss": 41.1429,
      "nll_loss": 1.0836269855499268,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.3925536870956421,
      "rewards/margins": 0.37287265062332153,
      "rewards/rejected": -0.7654264569282532,
      "step": 135
    },
    {
      "epoch": 1.3223703703703704,
      "grad_norm": 55.24971389770508,
      "learning_rate": 5.112644597552452e-05,
      "log_odds_chosen": 1.2705775499343872,
      "log_odds_ratio": -0.3516805171966553,
      "logits/chosen": 158.27615356445312,
      "logits/rejected": 164.78353881835938,
      "logps/chosen": -0.8647182583808899,
      "logps/rejected": -1.7130002975463867,
      "loss": 42.35,
      "nll_loss": 1.1466747522354126,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -0.43235912919044495,
      "rewards/margins": 0.4241410195827484,
      "rewards/rejected": -0.8565001487731934,
      "step": 140
    },
    {
      "epoch": 1.3697777777777778,
      "grad_norm": 62.0118408203125,
      "learning_rate": 4.9167291494431645e-05,
      "log_odds_chosen": 1.2424061298370361,
      "log_odds_ratio": -0.3542417883872986,
      "logits/chosen": 168.1389617919922,
      "logits/rejected": 153.880859375,
      "logps/chosen": -0.8555068969726562,
      "logps/rejected": -1.7085926532745361,
      "loss": 41.8085,
      "nll_loss": 1.1373878717422485,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.4277534484863281,
      "rewards/margins": 0.4265429973602295,
      "rewards/rejected": -0.8542963266372681,
      "step": 145
    },
    {
      "epoch": 1.417185185185185,
      "grad_norm": 61.60399627685547,
      "learning_rate": 4.717220136954046e-05,
      "log_odds_chosen": 1.2572475671768188,
      "log_odds_ratio": -0.341666042804718,
      "logits/chosen": 162.69091796875,
      "logits/rejected": 168.27194213867188,
      "logps/chosen": -0.7956852316856384,
      "logps/rejected": -1.6127551794052124,
      "loss": 40.5585,
      "nll_loss": 1.0824170112609863,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -0.3978426158428192,
      "rewards/margins": 0.4085349142551422,
      "rewards/rejected": -0.8063775897026062,
      "step": 150
    },
    {
      "epoch": 1.4645925925925927,
      "grad_norm": 57.484771728515625,
      "learning_rate": 4.514732054444031e-05,
      "log_odds_chosen": 1.2566419839859009,
      "log_odds_ratio": -0.37278324365615845,
      "logits/chosen": 171.4698944091797,
      "logits/rejected": 158.44186401367188,
      "logps/chosen": -0.8029295802116394,
      "logps/rejected": -1.641107201576233,
      "loss": 41.6586,
      "nll_loss": 1.1319265365600586,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.4014647901058197,
      "rewards/margins": 0.41908884048461914,
      "rewards/rejected": -0.8205536007881165,
      "step": 155
    },
    {
      "epoch": 1.512,
      "grad_norm": 55.12516784667969,
      "learning_rate": 4.309888571906274e-05,
      "log_odds_chosen": 1.149753451347351,
      "log_odds_ratio": -0.3672695755958557,
      "logits/chosen": 164.41958618164062,
      "logits/rejected": 164.40335083007812,
      "logps/chosen": -0.9021244049072266,
      "logps/rejected": -1.6924254894256592,
      "loss": 42.0332,
      "nll_loss": 1.1726073026657104,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.4510622024536133,
      "rewards/margins": 0.3951506018638611,
      "rewards/rejected": -0.8462127447128296,
      "step": 160
    },
    {
      "epoch": 1.5594074074074074,
      "grad_norm": 46.9222297668457,
      "learning_rate": 4.1033206140439427e-05,
      "log_odds_chosen": 1.319567084312439,
      "log_odds_ratio": -0.32131245732307434,
      "logits/chosen": 151.97567749023438,
      "logits/rejected": 166.11029052734375,
      "logps/chosen": -0.8229783773422241,
      "logps/rejected": -1.708121657371521,
      "loss": 41.3777,
      "nll_loss": 1.1105647087097168,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -0.41148918867111206,
      "rewards/margins": 0.4425717294216156,
      "rewards/rejected": -0.8540608286857605,
      "step": 165
    },
    {
      "epoch": 1.6068148148148147,
      "grad_norm": 75.31800842285156,
      "learning_rate": 3.8956644170012536e-05,
      "log_odds_chosen": 1.4432159662246704,
      "log_odds_ratio": -0.33049893379211426,
      "logits/chosen": 156.82803344726562,
      "logits/rejected": 155.05007934570312,
      "logps/chosen": -0.8102866411209106,
      "logps/rejected": -1.783372163772583,
      "loss": 40.0255,
      "nll_loss": 1.0867273807525635,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -0.4051433205604553,
      "rewards/margins": 0.48654279112815857,
      "rewards/rejected": -0.8916860818862915,
      "step": 170
    },
    {
      "epoch": 1.6542222222222223,
      "grad_norm": 68.79954528808594,
      "learning_rate": 3.687559568735105e-05,
      "log_odds_chosen": 1.266623854637146,
      "log_odds_ratio": -0.34371012449264526,
      "logits/chosen": 159.88473510742188,
      "logits/rejected": 147.89791870117188,
      "logps/chosen": -0.8142267465591431,
      "logps/rejected": -1.6857149600982666,
      "loss": 40.7363,
      "nll_loss": 1.139469027519226,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.40711337327957153,
      "rewards/margins": 0.4357439875602722,
      "rewards/rejected": -0.8428574800491333,
      "step": 175
    },
    {
      "epoch": 1.7016296296296296,
      "grad_norm": 72.72908020019531,
      "learning_rate": 3.479647039063004e-05,
      "log_odds_chosen": 1.2162840366363525,
      "log_odds_ratio": -0.3732939064502716,
      "logits/chosen": 149.99951171875,
      "logits/rejected": 149.6708221435547,
      "logps/chosen": -0.8462786674499512,
      "logps/rejected": -1.6657482385635376,
      "loss": 41.3565,
      "nll_loss": 1.1392968893051147,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.4231393337249756,
      "rewards/margins": 0.4097347855567932,
      "rewards/rejected": -0.8328741192817688,
      "step": 180
    },
    {
      "epoch": 1.749037037037037,
      "grad_norm": 54.563720703125,
      "learning_rate": 3.272567205454826e-05,
      "log_odds_chosen": 1.255048155784607,
      "log_odds_ratio": -0.35441821813583374,
      "logits/chosen": 153.781982421875,
      "logits/rejected": 151.72683715820312,
      "logps/chosen": -0.8422285318374634,
      "logps/rejected": -1.7023115158081055,
      "loss": 41.0615,
      "nll_loss": 1.1301571130752563,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -0.4211142659187317,
      "rewards/margins": 0.4300415515899658,
      "rewards/rejected": -0.8511557579040527,
      "step": 185
    },
    {
      "epoch": 1.7964444444444445,
      "grad_norm": 68.65567016601562,
      "learning_rate": 3.06695788064902e-05,
      "log_odds_chosen": 1.3918511867523193,
      "log_odds_ratio": -0.3203474283218384,
      "logits/chosen": 160.94696044921875,
      "logits/rejected": 150.07833862304688,
      "logps/chosen": -0.8545455932617188,
      "logps/rejected": -1.8117406368255615,
      "loss": 42.5014,
      "nll_loss": 1.1190176010131836,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -0.4272727966308594,
      "rewards/margins": 0.4785975515842438,
      "rewards/rejected": -0.9058703184127808,
      "step": 190
    },
    {
      "epoch": 1.8438518518518519,
      "grad_norm": 64.03311920166016,
      "learning_rate": 2.863452348168275e-05,
      "log_odds_chosen": 1.4326978921890259,
      "log_odds_ratio": -0.32069623470306396,
      "logits/chosen": 155.94273376464844,
      "logits/rejected": 141.63304138183594,
      "logps/chosen": -0.8207441568374634,
      "logps/rejected": -1.8126327991485596,
      "loss": 40.4731,
      "nll_loss": 1.1084260940551758,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.4103720784187317,
      "rewards/margins": 0.4959442615509033,
      "rewards/rejected": -0.9063163995742798,
      "step": 195
    },
    {
      "epoch": 1.8912592592592592,
      "grad_norm": 73.11625671386719,
      "learning_rate": 2.6626774117853172e-05,
      "log_odds_chosen": 1.2478724718093872,
      "log_odds_ratio": -0.3590041697025299,
      "logits/chosen": 155.2909698486328,
      "logits/rejected": 149.68272399902344,
      "logps/chosen": -0.8270870447158813,
      "logps/rejected": -1.6576263904571533,
      "loss": 42.3125,
      "nll_loss": 1.0772912502288818,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.4135435223579407,
      "rewards/margins": 0.415269672870636,
      "rewards/rejected": -0.8288131952285767,
      "step": 200
    },
    {
      "epoch": 1.9386666666666668,
      "grad_norm": 51.53399658203125,
      "learning_rate": 2.4652514649465555e-05,
      "log_odds_chosen": 1.105621576309204,
      "log_odds_ratio": -0.3948635756969452,
      "logits/chosen": 163.22994995117188,
      "logits/rejected": 149.87380981445312,
      "logps/chosen": -0.8414310216903687,
      "logps/rejected": -1.586313009262085,
      "loss": 41.2346,
      "nll_loss": 1.1008248329162598,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.4207155108451843,
      "rewards/margins": 0.37244105339050293,
      "rewards/rejected": -0.7931565046310425,
      "step": 205
    },
    {
      "epoch": 1.986074074074074,
      "grad_norm": 592.9937133789062,
      "learning_rate": 2.2717825860998054e-05,
      "log_odds_chosen": 1.1890079975128174,
      "log_odds_ratio": -0.3658009171485901,
      "logits/chosen": 143.8880157470703,
      "logits/rejected": 149.6746063232422,
      "logps/chosen": -0.8134666681289673,
      "logps/rejected": -1.5765702724456787,
      "loss": 39.231,
      "nll_loss": 1.0431172847747803,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.40673333406448364,
      "rewards/margins": 0.38155174255371094,
      "rewards/rejected": -0.7882851362228394,
      "step": 210
    },
    {
      "epoch": 2.0284444444444443,
      "grad_norm": 48.64838409423828,
      "learning_rate": 2.082866665792552e-05,
      "log_odds_chosen": 2.0941009521484375,
      "log_odds_ratio": -0.22275511920452118,
      "logits/chosen": 143.320068359375,
      "logits/rejected": 138.30125427246094,
      "logps/chosen": -0.6920493245124817,
      "logps/rejected": -2.0684661865234375,
      "loss": 30.0118,
      "nll_loss": 0.9490633606910706,
      "rewards/accuracies": 0.9370629191398621,
      "rewards/chosen": -0.34602466225624084,
      "rewards/margins": 0.6882083415985107,
      "rewards/rejected": -1.0342330932617188,
      "step": 215
    },
    {
      "epoch": 2.075851851851852,
      "grad_norm": 77.73394012451172,
      "learning_rate": 1.8990855713093523e-05,
      "log_odds_chosen": 3.1007094383239746,
      "log_odds_ratio": -0.10504193603992462,
      "logits/chosen": 118.2657241821289,
      "logits/rejected": 116.21284484863281,
      "logps/chosen": -0.5571001172065735,
      "logps/rejected": -2.6969428062438965,
      "loss": 28.4152,
      "nll_loss": 0.8508051633834839,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -0.27855005860328674,
      "rewards/margins": 1.0699213743209839,
      "rewards/rejected": -1.3484714031219482,
      "step": 220
    },
    {
      "epoch": 2.1232592592592594,
      "grad_norm": 50.05561447143555,
      "learning_rate": 1.7210053545013458e-05,
      "log_odds_chosen": 2.909291982650757,
      "log_odds_ratio": -0.11771439015865326,
      "logits/chosen": 133.0370330810547,
      "logits/rejected": 126.50312805175781,
      "logps/chosen": -0.565724790096283,
      "logps/rejected": -2.530423641204834,
      "loss": 29.3042,
      "nll_loss": 0.8670442700386047,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.2828623950481415,
      "rewards/margins": 0.9823493957519531,
      "rewards/rejected": -1.265211820602417,
      "step": 225
    },
    {
      "epoch": 2.1706666666666665,
      "grad_norm": 54.2590446472168,
      "learning_rate": 1.549174508327826e-05,
      "log_odds_chosen": 3.056389808654785,
      "log_odds_ratio": -0.09899865090847015,
      "logits/chosen": 129.60385131835938,
      "logits/rejected": 120.37672424316406,
      "logps/chosen": -0.5788933038711548,
      "logps/rejected": -2.6959786415100098,
      "loss": 27.7201,
      "nll_loss": 0.8015221357345581,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -0.2894466519355774,
      "rewards/margins": 1.0585426092147827,
      "rewards/rejected": -1.3479893207550049,
      "step": 230
    },
    {
      "epoch": 2.218074074074074,
      "grad_norm": 53.754486083984375,
      "learning_rate": 1.3841222774797851e-05,
      "log_odds_chosen": 2.7191035747528076,
      "log_odds_ratio": -0.11840663105249405,
      "logits/chosen": 128.04299926757812,
      "logits/rejected": 123.1611557006836,
      "logps/chosen": -0.5877354741096497,
      "logps/rejected": -2.4053232669830322,
      "loss": 28.9243,
      "nll_loss": 0.8488882184028625,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -0.29386773705482483,
      "rewards/margins": 0.9087940454483032,
      "rewards/rejected": -1.2026616334915161,
      "step": 235
    },
    {
      "epoch": 2.2654814814814817,
      "grad_norm": 48.55769729614258,
      "learning_rate": 1.2263570282887465e-05,
      "log_odds_chosen": 2.9994494915008545,
      "log_odds_ratio": -0.1045825332403183,
      "logits/chosen": 124.81593322753906,
      "logits/rejected": 124.41886901855469,
      "logps/chosen": -0.5564694404602051,
      "logps/rejected": -2.5893123149871826,
      "loss": 28.7714,
      "nll_loss": 0.8480992317199707,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -0.27823472023010254,
      "rewards/margins": 1.0164214372634888,
      "rewards/rejected": -1.2946561574935913,
      "step": 240
    },
    {
      "epoch": 2.3128888888888888,
      "grad_norm": 54.71540069580078,
      "learning_rate": 1.0763646829416548e-05,
      "log_odds_chosen": 3.0860180854797363,
      "log_odds_ratio": -0.09014725685119629,
      "logits/chosen": 127.33006286621094,
      "logits/rejected": 117.83717346191406,
      "logps/chosen": -0.5473681688308716,
      "logps/rejected": -2.653287649154663,
      "loss": 27.0092,
      "nll_loss": 0.82794588804245,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.2736840844154358,
      "rewards/margins": 1.052959680557251,
      "rewards/rejected": -1.3266438245773315,
      "step": 245
    },
    {
      "epoch": 2.3602962962962963,
      "grad_norm": 144.38958740234375,
      "learning_rate": 9.346072228244598e-06,
      "log_odds_chosen": 3.072089672088623,
      "log_odds_ratio": -0.10084352642297745,
      "logits/chosen": 128.05050659179688,
      "logits/rejected": 127.39276123046875,
      "logps/chosen": -0.5662072896957397,
      "logps/rejected": -2.6561198234558105,
      "loss": 27.741,
      "nll_loss": 0.8294836282730103,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -0.2831036448478699,
      "rewards/margins": 1.044956088066101,
      "rewards/rejected": -1.3280599117279053,
      "step": 250
    },
    {
      "epoch": 2.407703703703704,
      "grad_norm": 69.65536499023438,
      "learning_rate": 8.015212656041612e-06,
      "log_odds_chosen": 3.221827268600464,
      "log_odds_ratio": -0.090447798371315,
      "logits/chosen": 123.1548843383789,
      "logits/rejected": 122.70021057128906,
      "logps/chosen": -0.49670666456222534,
      "logps/rejected": -2.6964945793151855,
      "loss": 27.5962,
      "nll_loss": 0.7993031740188599,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -0.24835333228111267,
      "rewards/margins": 1.0998938083648682,
      "rewards/rejected": -1.3482472896575928,
      "step": 255
    },
    {
      "epoch": 2.455111111111111,
      "grad_norm": 52.56804275512695,
      "learning_rate": 6.775167204319555e-06,
      "log_odds_chosen": 3.3616344928741455,
      "log_odds_ratio": -0.08646370470523834,
      "logits/chosen": 127.23355865478516,
      "logits/rejected": 119.65962219238281,
      "logps/chosen": -0.5302814841270447,
      "logps/rejected": -2.8641209602355957,
      "loss": 27.3852,
      "nll_loss": 0.8003301620483398,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.26514074206352234,
      "rewards/margins": 1.1669199466705322,
      "rewards/rejected": -1.4320604801177979,
      "step": 260
    },
    {
      "epoch": 2.5025185185185186,
      "grad_norm": 52.00509262084961,
      "learning_rate": 5.6297552540948325e-06,
      "log_odds_chosen": 2.9919915199279785,
      "log_odds_ratio": -0.13428373634815216,
      "logits/chosen": 129.68038940429688,
      "logits/rejected": 126.67582702636719,
      "logps/chosen": -0.5471857190132141,
      "logps/rejected": -2.557234525680542,
      "loss": 27.4382,
      "nll_loss": 0.7889086008071899,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -0.27359285950660706,
      "rewards/margins": 1.0050245523452759,
      "rewards/rejected": -1.278617262840271,
      "step": 265
    },
    {
      "epoch": 2.549925925925926,
      "grad_norm": 48.82490539550781,
      "learning_rate": 4.5825047120684215e-06,
      "log_odds_chosen": 3.1277198791503906,
      "log_odds_ratio": -0.11080478131771088,
      "logits/chosen": 129.29345703125,
      "logits/rejected": 128.2639617919922,
      "logps/chosen": -0.5835883021354675,
      "logps/rejected": -2.7072691917419434,
      "loss": 26.8792,
      "nll_loss": 0.8251482844352722,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -0.29179415106773376,
      "rewards/margins": 1.0618404150009155,
      "rewards/rejected": -1.3536345958709717,
      "step": 270
    },
    {
      "epoch": 2.5973333333333333,
      "grad_norm": 44.77346420288086,
      "learning_rate": 3.636641144556547e-06,
      "log_odds_chosen": 3.256783962249756,
      "log_odds_ratio": -0.10093816369771957,
      "logits/chosen": 130.92579650878906,
      "logits/rejected": 121.23966217041016,
      "logps/chosen": -0.5694173574447632,
      "logps/rejected": -2.807206630706787,
      "loss": 28.3661,
      "nll_loss": 0.8377152681350708,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -0.2847086787223816,
      "rewards/margins": 1.1188945770263672,
      "rewards/rejected": -1.4036033153533936,
      "step": 275
    },
    {
      "epoch": 2.644740740740741,
      "grad_norm": 43.979530334472656,
      "learning_rate": 2.795077842639676e-06,
      "log_odds_chosen": 3.014188766479492,
      "log_odds_ratio": -0.10471322387456894,
      "logits/chosen": 125.88334655761719,
      "logits/rejected": 120.0384292602539,
      "logps/chosen": -0.5433251857757568,
      "logps/rejected": -2.5689566135406494,
      "loss": 27.0828,
      "nll_loss": 0.7862499952316284,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.2716625928878784,
      "rewards/margins": 1.0128155946731567,
      "rewards/rejected": -1.2844783067703247,
      "step": 280
    },
    {
      "epoch": 2.6921481481481484,
      "grad_norm": 50.671897888183594,
      "learning_rate": 2.060406849129605e-06,
      "log_odds_chosen": 3.0842363834381104,
      "log_odds_ratio": -0.10635282844305038,
      "logits/chosen": 134.56607055664062,
      "logits/rejected": 120.92826843261719,
      "logps/chosen": -0.5578547716140747,
      "logps/rejected": -2.670154333114624,
      "loss": 27.4779,
      "nll_loss": 0.8171490430831909,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -0.27892738580703735,
      "rewards/margins": 1.056149959564209,
      "rewards/rejected": -1.335077166557312,
      "step": 285
    },
    {
      "epoch": 2.7395555555555555,
      "grad_norm": 46.54133224487305,
      "learning_rate": 1.4348909749918786e-06,
      "log_odds_chosen": 3.2423605918884277,
      "log_odds_ratio": -0.08490051329135895,
      "logits/chosen": 128.3448028564453,
      "logits/rejected": 129.498779296875,
      "logps/chosen": -0.5114618539810181,
      "logps/rejected": -2.7147812843322754,
      "loss": 28.442,
      "nll_loss": 0.8227435350418091,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.25573092699050903,
      "rewards/margins": 1.101659893989563,
      "rewards/rejected": -1.3573906421661377,
      "step": 290
    },
    {
      "epoch": 2.786962962962963,
      "grad_norm": 45.60363006591797,
      "learning_rate": 9.204568298131307e-07,
      "log_odds_chosen": 3.0159189701080322,
      "log_odds_ratio": -0.14175482094287872,
      "logits/chosen": 130.07974243164062,
      "logits/rejected": 128.87826538085938,
      "logps/chosen": -0.6083237528800964,
      "logps/rejected": -2.6694674491882324,
      "loss": 28.036,
      "nll_loss": 0.8367668390274048,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -0.3041618764400482,
      "rewards/margins": 1.0305720567703247,
      "rewards/rejected": -1.3347337245941162,
      "step": 295
    },
    {
      "epoch": 2.83437037037037,
      "grad_norm": 91.35395050048828,
      "learning_rate": 5.186888877798209e-07,
      "log_odds_chosen": 3.2012171745300293,
      "log_odds_ratio": -0.09840454906225204,
      "logits/chosen": 144.08177185058594,
      "logits/rejected": 126.72806549072266,
      "logps/chosen": -0.5671483278274536,
      "logps/rejected": -2.7830772399902344,
      "loss": 28.1208,
      "nll_loss": 0.8516788482666016,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -0.2835741639137268,
      "rewards/margins": 1.1079645156860352,
      "rewards/rejected": -1.3915386199951172,
      "step": 300
    },
    {
      "epoch": 2.8817777777777778,
      "grad_norm": 73.92967224121094,
      "learning_rate": 2.308246074453138e-07,
      "log_odds_chosen": 3.117356061935425,
      "log_odds_ratio": -0.09485842287540436,
      "logits/chosen": 132.63426208496094,
      "logits/rejected": 125.5457534790039,
      "logps/chosen": -0.5718491077423096,
      "logps/rejected": -2.6983981132507324,
      "loss": 27.0973,
      "nll_loss": 0.7848010659217834,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -0.2859245538711548,
      "rewards/margins": 1.063274621963501,
      "rewards/rejected": -1.3491990566253662,
      "step": 305
    },
    {
      "epoch": 2.9291851851851853,
      "grad_norm": 50.79293441772461,
      "learning_rate": 5.7750620316622534e-08,
      "log_odds_chosen": 3.230167865753174,
      "log_odds_ratio": -0.08979301154613495,
      "logits/chosen": 133.17994689941406,
      "logits/rejected": 121.85380554199219,
      "logps/chosen": -0.5334647297859192,
      "logps/rejected": -2.7135567665100098,
      "loss": 27.1197,
      "nll_loss": 0.8048761487007141,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -0.2667323648929596,
      "rewards/margins": 1.0900460481643677,
      "rewards/rejected": -1.3567783832550049,
      "step": 310
    },
    {
      "epoch": 2.9765925925925925,
      "grad_norm": 59.680809020996094,
      "learning_rate": 0.0,
      "log_odds_chosen": 3.1277458667755127,
      "log_odds_ratio": -0.09231346845626831,
      "logits/chosen": 130.7123260498047,
      "logits/rejected": 117.61688232421875,
      "logps/chosen": -0.545311450958252,
      "logps/rejected": -2.663321018218994,
      "loss": 26.588,
      "nll_loss": 0.802075207233429,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.272655725479126,
      "rewards/margins": 1.059004783630371,
      "rewards/rejected": -1.331660509109497,
      "step": 315
    },
    {
      "epoch": 2.9765925925925925,
      "step": 315,
      "total_flos": 0.0,
      "train_loss": 60.23838462224082,
      "train_runtime": 8115.1309,
      "train_samples_per_second": 2.495,
      "train_steps_per_second": 0.039
    }
  ],
  "logging_steps": 5,
  "max_steps": 315,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 3,
  "save_steps": 100000,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": true
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 1,
  "trial_name": null,
  "trial_params": null
}