{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 1.0,
  "eval_steps": 100,
  "global_step": 385,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.0,
      "grad_norm": 26.125,
      "learning_rate": 1.282051282051282e-07,
      "logits/chosen": 88.18099975585938,
      "logits/rejected": 88.25153350830078,
      "logps/chosen": -29.073104858398438,
      "logps/rejected": -26.25731658935547,
      "loss": 0.5,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 1
    },
    {
      "epoch": 0.03,
      "grad_norm": 14.1875,
      "learning_rate": 1.282051282051282e-06,
      "logits/chosen": 81.0709228515625,
      "logits/rejected": 80.77515411376953,
      "logps/chosen": -34.26983642578125,
      "logps/rejected": -32.992767333984375,
      "loss": 0.4962,
      "rewards/accuracies": 0.4444444477558136,
      "rewards/chosen": -0.022488469257950783,
      "rewards/margins": 0.004999637603759766,
      "rewards/rejected": -0.02748810313642025,
      "step": 10
    },
    {
      "epoch": 0.05,
      "grad_norm": 19.5,
      "learning_rate": 2.564102564102564e-06,
      "logits/chosen": 80.6608657836914,
      "logits/rejected": 80.54856872558594,
      "logps/chosen": -33.62510299682617,
      "logps/rejected": -30.784603118896484,
      "loss": 0.4924,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.05184358358383179,
      "rewards/margins": 0.057535119354724884,
      "rewards/rejected": -0.00569154042750597,
      "step": 20
    },
    {
      "epoch": 0.08,
      "grad_norm": 22.125,
      "learning_rate": 3.846153846153847e-06,
      "logits/chosen": 82.50384521484375,
      "logits/rejected": 82.53746032714844,
      "logps/chosen": -33.812801361083984,
      "logps/rejected": -31.15009117126465,
      "loss": 0.5123,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.16001427173614502,
      "rewards/margins": -0.058978963643312454,
      "rewards/rejected": 0.21899321675300598,
      "step": 30
    },
    {
      "epoch": 0.1,
      "grad_norm": 18.0,
      "learning_rate": 4.999896948438434e-06,
      "logits/chosen": 81.07115173339844,
      "logits/rejected": 81.06380462646484,
      "logps/chosen": -32.71454620361328,
      "logps/rejected": -33.17704772949219,
      "loss": 0.4627,
      "rewards/accuracies": 0.4625000059604645,
      "rewards/chosen": 0.36761564016342163,
      "rewards/margins": 0.21443231403827667,
      "rewards/rejected": 0.15318328142166138,
      "step": 40
    },
    {
      "epoch": 0.13,
      "grad_norm": 16.75,
      "learning_rate": 4.987541037542187e-06,
      "logits/chosen": 78.77976989746094,
      "logits/rejected": 78.78811645507812,
      "logps/chosen": -30.60152816772461,
      "logps/rejected": -30.760879516601562,
      "loss": 0.4673,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": 0.4829755425453186,
      "rewards/margins": 0.2330504208803177,
      "rewards/rejected": 0.24992509186267853,
      "step": 50
    },
    {
      "epoch": 0.16,
      "grad_norm": 15.125,
      "learning_rate": 4.954691471941119e-06,
      "logits/chosen": 83.38993072509766,
      "logits/rejected": 83.4462890625,
      "logps/chosen": -30.877517700195312,
      "logps/rejected": -29.332889556884766,
      "loss": 0.4954,
      "rewards/accuracies": 0.42500001192092896,
      "rewards/chosen": 0.23811349272727966,
      "rewards/margins": 0.03200835734605789,
      "rewards/rejected": 0.20610514283180237,
      "step": 60
    },
    {
      "epoch": 0.18,
      "grad_norm": 15.6875,
      "learning_rate": 4.901618883413549e-06,
      "logits/chosen": 84.03268432617188,
      "logits/rejected": 84.06501007080078,
      "logps/chosen": -30.23248291015625,
      "logps/rejected": -32.74273681640625,
      "loss": 0.4844,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.31723442673683167,
      "rewards/margins": 0.08125714957714081,
      "rewards/rejected": 0.23597724735736847,
      "step": 70
    },
    {
      "epoch": 0.21,
      "grad_norm": 17.625,
      "learning_rate": 4.828760511501322e-06,
      "logits/chosen": 81.7315673828125,
      "logits/rejected": 81.71873474121094,
      "logps/chosen": -30.963665008544922,
      "logps/rejected": -30.75851058959961,
      "loss": 0.4187,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": 0.4330216348171234,
      "rewards/margins": 0.4337124824523926,
      "rewards/rejected": -0.0006908595678396523,
      "step": 80
    },
    {
      "epoch": 0.23,
      "grad_norm": 21.25,
      "learning_rate": 4.7367166013034295e-06,
      "logits/chosen": 78.5487060546875,
      "logits/rejected": 78.52108001708984,
      "logps/chosen": -32.06660461425781,
      "logps/rejected": -30.661907196044922,
      "loss": 0.4704,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": 0.4572601318359375,
      "rewards/margins": 0.22561049461364746,
      "rewards/rejected": 0.23164963722229004,
      "step": 90
    },
    {
      "epoch": 0.26,
      "grad_norm": 13.25,
      "learning_rate": 4.626245458345211e-06,
      "logits/chosen": 83.6882095336914,
      "logits/rejected": 83.7122802734375,
      "logps/chosen": -33.82537841796875,
      "logps/rejected": -31.4283504486084,
      "loss": 0.4705,
      "rewards/accuracies": 0.4625000059604645,
      "rewards/chosen": 0.38369041681289673,
      "rewards/margins": 0.1259343922138214,
      "rewards/rejected": 0.2577560245990753,
      "step": 100
    },
    {
      "epoch": 0.26,
      "eval_logits/chosen": 98.68708801269531,
      "eval_logits/rejected": 98.67501068115234,
      "eval_logps/chosen": -32.27544021606445,
      "eval_logps/rejected": -35.81333541870117,
      "eval_loss": 0.4946621358394623,
      "eval_rewards/accuracies": 0.4896179437637329,
      "eval_rewards/chosen": 0.13418929278850555,
      "eval_rewards/margins": 0.011633642949163914,
      "eval_rewards/rejected": 0.12255565822124481,
      "eval_runtime": 104.4117,
      "eval_samples_per_second": 3.285,
      "eval_steps_per_second": 0.412,
      "step": 100
    },
    {
      "epoch": 0.29,
      "grad_norm": 22.75,
      "learning_rate": 4.498257201263691e-06,
      "logits/chosen": 83.8403091430664,
      "logits/rejected": 83.71327209472656,
      "logps/chosen": -31.912221908569336,
      "logps/rejected": -32.58714294433594,
      "loss": 0.3643,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.8884676694869995,
      "rewards/margins": 0.8334689140319824,
      "rewards/rejected": 0.054998766630887985,
      "step": 110
    },
    {
      "epoch": 0.31,
      "grad_norm": 22.5,
      "learning_rate": 4.353806263777678e-06,
      "logits/chosen": 83.93721008300781,
      "logits/rejected": 84.06002044677734,
      "logps/chosen": -27.933292388916016,
      "logps/rejected": -35.20619201660156,
      "loss": 0.4011,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": 0.7970338463783264,
      "rewards/margins": 0.594670295715332,
      "rewards/rejected": 0.2023634910583496,
      "step": 120
    },
    {
      "epoch": 0.34,
      "grad_norm": 15.25,
      "learning_rate": 4.1940827077152755e-06,
      "logits/chosen": 81.2428970336914,
      "logits/rejected": 81.28059387207031,
      "logps/chosen": -30.123279571533203,
      "logps/rejected": -31.811786651611328,
      "loss": 0.4158,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.6272625923156738,
      "rewards/margins": 0.5313904881477356,
      "rewards/rejected": 0.09587211161851883,
      "step": 130
    },
    {
      "epoch": 0.36,
      "grad_norm": 16.375,
      "learning_rate": 4.0204024186666215e-06,
      "logits/chosen": 82.30902862548828,
      "logits/rejected": 82.33167266845703,
      "logps/chosen": -26.606557846069336,
      "logps/rejected": -32.55984115600586,
      "loss": 0.3859,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": 0.6726120710372925,
      "rewards/margins": 0.7817814946174622,
      "rewards/rejected": -0.10916946828365326,
      "step": 140
    },
    {
      "epoch": 0.39,
      "grad_norm": 12.9375,
      "learning_rate": 3.834196265035119e-06,
      "logits/chosen": 80.79722595214844,
      "logits/rejected": 80.76870727539062,
      "logps/chosen": -28.53302001953125,
      "logps/rejected": -32.609378814697266,
      "loss": 0.3794,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": 0.6848722696304321,
      "rewards/margins": 0.719173014163971,
      "rewards/rejected": -0.03430069237947464,
      "step": 150
    },
    {
      "epoch": 0.42,
      "grad_norm": 19.75,
      "learning_rate": 3.636998309800573e-06,
      "logits/chosen": 82.5254135131836,
      "logits/rejected": 82.54801940917969,
      "logps/chosen": -33.080387115478516,
      "logps/rejected": -30.11862564086914,
      "loss": 0.3503,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": 0.8450896143913269,
      "rewards/margins": 0.9779588580131531,
      "rewards/rejected": -0.13286921381950378,
      "step": 160
    },
    {
      "epoch": 0.44,
      "grad_norm": 10.5,
      "learning_rate": 3.4304331721118078e-06,
      "logits/chosen": 83.30370330810547,
      "logits/rejected": 83.26591491699219,
      "logps/chosen": -30.516061782836914,
      "logps/rejected": -32.39433288574219,
      "loss": 0.3746,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": 0.5945821404457092,
      "rewards/margins": 0.8940039873123169,
      "rewards/rejected": -0.29942184686660767,
      "step": 170
    },
    {
      "epoch": 0.47,
      "grad_norm": 7.34375,
      "learning_rate": 3.2162026428305436e-06,
      "logits/chosen": 80.87037658691406,
      "logits/rejected": 80.84497833251953,
      "logps/chosen": -30.32219886779785,
      "logps/rejected": -31.39385414123535,
      "loss": 0.3678,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": 0.6999613642692566,
      "rewards/margins": 0.8892775774002075,
      "rewards/rejected": -0.18931610882282257,
      "step": 180
    },
    {
      "epoch": 0.49,
      "grad_norm": 9.25,
      "learning_rate": 2.996071664294641e-06,
      "logits/chosen": 82.518310546875,
      "logits/rejected": 82.48826599121094,
      "logps/chosen": -29.71840476989746,
      "logps/rejected": -30.17477798461914,
      "loss": 0.4079,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.8384085893630981,
      "rewards/margins": 0.6038115620613098,
      "rewards/rejected": 0.23459705710411072,
      "step": 190
    },
    {
      "epoch": 0.52,
      "grad_norm": 9.0625,
      "learning_rate": 2.7718537898066833e-06,
      "logits/chosen": 77.99578094482422,
      "logits/rejected": 77.94181060791016,
      "logps/chosen": -32.918636322021484,
      "logps/rejected": -32.197086334228516,
      "loss": 0.3097,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": 1.4672811031341553,
      "rewards/margins": 1.2302258014678955,
      "rewards/rejected": 0.23705501854419708,
      "step": 200
    },
    {
      "epoch": 0.52,
      "eval_logits/chosen": 98.47710418701172,
      "eval_logits/rejected": 98.47020721435547,
      "eval_logps/chosen": -32.242671966552734,
      "eval_logps/rejected": -35.840518951416016,
      "eval_loss": 0.49102693796157837,
      "eval_rewards/accuracies": 0.5481727719306946,
      "eval_rewards/chosen": 0.1604028344154358,
      "eval_rewards/margins": 0.0595964714884758,
      "eval_rewards/rejected": 0.1008063480257988,
      "eval_runtime": 104.1775,
      "eval_samples_per_second": 3.292,
      "eval_steps_per_second": 0.413,
      "step": 200
    },
    {
      "epoch": 0.55,
      "grad_norm": 20.25,
      "learning_rate": 2.5453962426402006e-06,
      "logits/chosen": 80.51226806640625,
      "logits/rejected": 80.43040466308594,
      "logps/chosen": -32.629119873046875,
      "logps/rejected": -34.76416778564453,
      "loss": 0.3655,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 1.053282618522644,
      "rewards/margins": 0.87574303150177,
      "rewards/rejected": 0.1775396168231964,
      "step": 210
    },
    {
      "epoch": 0.57,
      "grad_norm": 14.5,
      "learning_rate": 2.3185646976551794e-06,
      "logits/chosen": 82.59284973144531,
      "logits/rejected": 82.66585540771484,
      "logps/chosen": -30.357955932617188,
      "logps/rejected": -30.737834930419922,
      "loss": 0.2951,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 1.2727962732315063,
      "rewards/margins": 1.2799025774002075,
      "rewards/rejected": -0.007106387056410313,
      "step": 220
    },
    {
      "epoch": 0.6,
      "grad_norm": 12.0625,
      "learning_rate": 2.0932279108998323e-06,
      "logits/chosen": 79.70103454589844,
      "logits/rejected": 79.75657653808594,
      "logps/chosen": -31.675893783569336,
      "logps/rejected": -33.84308624267578,
      "loss": 0.3884,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": 0.9038804173469543,
      "rewards/margins": 0.7686265707015991,
      "rewards/rejected": 0.1352538764476776,
      "step": 230
    },
    {
      "epoch": 0.62,
      "grad_norm": 20.875,
      "learning_rate": 1.8712423238279358e-06,
      "logits/chosen": 82.11561584472656,
      "logits/rejected": 82.39993286132812,
      "logps/chosen": -30.073062896728516,
      "logps/rejected": -31.364795684814453,
      "loss": 0.3068,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": 1.2920019626617432,
      "rewards/margins": 1.158202886581421,
      "rewards/rejected": 0.13379916548728943,
      "step": 240
    },
    {
      "epoch": 0.65,
      "grad_norm": 15.4375,
      "learning_rate": 1.6544367689701824e-06,
      "logits/chosen": 80.80778503417969,
      "logits/rejected": 80.8788833618164,
      "logps/chosen": -26.4283504486084,
      "logps/rejected": -29.62093162536621,
      "loss": 0.3934,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.934036135673523,
      "rewards/margins": 0.7101906538009644,
      "rewards/rejected": 0.22384557127952576,
      "step": 250
    },
    {
      "epoch": 0.68,
      "grad_norm": 13.1875,
      "learning_rate": 1.4445974030621963e-06,
      "logits/chosen": 78.03572082519531,
      "logits/rejected": 78.22270202636719,
      "logps/chosen": -29.80512046813965,
      "logps/rejected": -35.99930953979492,
      "loss": 0.297,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": 1.4046982526779175,
      "rewards/margins": 1.3228428363800049,
      "rewards/rejected": 0.08185521513223648,
      "step": 260
    },
    {
      "epoch": 0.7,
      "grad_norm": 9.3125,
      "learning_rate": 1.243452991757889e-06,
      "logits/chosen": 77.41798400878906,
      "logits/rejected": 77.4520034790039,
      "logps/chosen": -30.092838287353516,
      "logps/rejected": -31.44430160522461,
      "loss": 0.305,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 1.3488702774047852,
      "rewards/margins": 1.2999744415283203,
      "rewards/rejected": 0.04889560490846634,
      "step": 270
    },
    {
      "epoch": 0.73,
      "grad_norm": 20.125,
      "learning_rate": 1.0526606671603523e-06,
      "logits/chosen": 80.09739685058594,
      "logits/rejected": 79.87271881103516,
      "logps/chosen": -30.528423309326172,
      "logps/rejected": -29.228124618530273,
      "loss": 0.382,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 1.0251753330230713,
      "rewards/margins": 0.6740784049034119,
      "rewards/rejected": 0.3510969281196594,
      "step": 280
    },
    {
      "epoch": 0.75,
      "grad_norm": 14.25,
      "learning_rate": 8.737922755071455e-07,
      "logits/chosen": 80.29197692871094,
      "logits/rejected": 80.20320129394531,
      "logps/chosen": -32.255340576171875,
      "logps/rejected": -32.0814208984375,
      "loss": 0.2756,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": 1.4791247844696045,
      "rewards/margins": 1.530770182609558,
      "rewards/rejected": -0.05164545774459839,
      "step": 290
    },
    {
      "epoch": 0.78,
      "grad_norm": 14.625,
      "learning_rate": 7.08321427484816e-07,
      "logits/chosen": 75.86343383789062,
      "logits/rejected": 75.9235610961914,
      "logps/chosen": -31.682353973388672,
      "logps/rejected": -28.961742401123047,
      "loss": 0.3066,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 1.37868332862854,
      "rewards/margins": 1.2664577960968018,
      "rewards/rejected": 0.11222562938928604,
      "step": 300
    },
    {
      "epoch": 0.78,
      "eval_logits/chosen": 98.42933654785156,
      "eval_logits/rejected": 98.42666625976562,
      "eval_logps/chosen": -32.235782623291016,
      "eval_logps/rejected": -35.84326934814453,
      "eval_loss": 0.48553377389907837,
      "eval_rewards/accuracies": 0.5282392501831055,
      "eval_rewards/chosen": 0.16591256856918335,
      "eval_rewards/margins": 0.06730752438306808,
      "eval_rewards/rejected": 0.09860503673553467,
      "eval_runtime": 104.1659,
      "eval_samples_per_second": 3.293,
      "eval_steps_per_second": 0.413,
      "step": 300
    },
    {
      "epoch": 0.81,
      "grad_norm": 12.8125,
      "learning_rate": 5.576113578589035e-07,
      "logits/chosen": 82.97637176513672,
      "logits/rejected": 83.01033020019531,
      "logps/chosen": -29.268829345703125,
      "logps/rejected": -32.1494026184082,
      "loss": 0.3157,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": 1.2757642269134521,
      "rewards/margins": 1.2354792356491089,
      "rewards/rejected": 0.04028487950563431,
      "step": 310
    },
    {
      "epoch": 0.83,
      "grad_norm": 15.4375,
      "learning_rate": 4.229036944380913e-07,
      "logits/chosen": 80.3984603881836,
      "logits/rejected": 80.40061950683594,
      "logps/chosen": -29.979394912719727,
      "logps/rejected": -28.756725311279297,
      "loss": 0.3119,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": 1.2953336238861084,
      "rewards/margins": 1.1651219129562378,
      "rewards/rejected": 0.13021180033683777,
      "step": 320
    },
    {
      "epoch": 0.86,
      "grad_norm": 10.75,
      "learning_rate": 3.053082288996112e-07,
      "logits/chosen": 77.65103912353516,
      "logits/rejected": 77.68055725097656,
      "logps/chosen": -28.304912567138672,
      "logps/rejected": -32.50617218017578,
      "loss": 0.254,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": 1.6381542682647705,
      "rewards/margins": 1.535199522972107,
      "rewards/rejected": 0.10295436531305313,
      "step": 330
    },
    {
      "epoch": 0.88,
      "grad_norm": 15.9375,
      "learning_rate": 2.0579377374915805e-07,
      "logits/chosen": 81.86819458007812,
      "logits/rejected": 81.89158630371094,
      "logps/chosen": -31.62424659729004,
      "logps/rejected": -33.315486907958984,
      "loss": 0.3285,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": 1.2703235149383545,
      "rewards/margins": 1.2277603149414062,
      "rewards/rejected": 0.0425633080303669,
      "step": 340
    },
    {
      "epoch": 0.91,
      "grad_norm": 9.3125,
      "learning_rate": 1.2518018074041684e-07,
      "logits/chosen": 80.94864654541016,
      "logits/rejected": 80.96649169921875,
      "logps/chosen": -31.77408218383789,
      "logps/rejected": -32.98318099975586,
      "loss": 0.3071,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": 1.4957287311553955,
      "rewards/margins": 1.4088352918624878,
      "rewards/rejected": 0.08689363300800323,
      "step": 350
    },
    {
      "epoch": 0.94,
      "grad_norm": 11.75,
      "learning_rate": 6.41315865106129e-08,
      "logits/chosen": 82.43021392822266,
      "logits/rejected": 82.48263549804688,
      "logps/chosen": -27.799968719482422,
      "logps/rejected": -31.40801429748535,
      "loss": 0.3137,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": 1.4017945528030396,
      "rewards/margins": 1.2083251476287842,
      "rewards/rejected": 0.19346928596496582,
      "step": 360
    },
    {
      "epoch": 0.96,
      "grad_norm": 12.1875,
      "learning_rate": 2.3150941078050325e-08,
      "logits/chosen": 81.93843078613281,
      "logits/rejected": 81.97197723388672,
      "logps/chosen": -31.361141204833984,
      "logps/rejected": -34.82567596435547,
      "loss": 0.3594,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": 1.2120168209075928,
      "rewards/margins": 0.9850583076477051,
      "rewards/rejected": 0.22695855796337128,
      "step": 370
    },
    {
      "epoch": 0.99,
      "grad_norm": 14.9375,
      "learning_rate": 2.575864278703266e-09,
      "logits/chosen": 75.7864761352539,
      "logits/rejected": 75.65766143798828,
      "logps/chosen": -29.02054786682129,
      "logps/rejected": -27.994884490966797,
      "loss": 0.3268,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 1.244463562965393,
      "rewards/margins": 1.111914038658142,
      "rewards/rejected": 0.13254934549331665,
      "step": 380
    },
    {
      "epoch": 1.0,
      "step": 385,
      "total_flos": 0.0,
      "train_loss": 0.37664644687206716,
      "train_runtime": 2560.444,
      "train_samples_per_second": 1.203,
      "train_steps_per_second": 0.15
    }
  ],
  "logging_steps": 10,
  "max_steps": 385,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 100,
  "total_flos": 0.0,
  "train_batch_size": 4,
  "trial_name": null,
  "trial_params": null
}