{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 2.9952,
  "eval_steps": 31,
  "global_step": 936,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.0032,
      "grad_norm": 104.32759944491974,
      "learning_rate": 5.3191489361702125e-09,
      "logits/generated": -2.7060699462890625,
      "logits/real": -2.864065647125244,
      "logps/generated": -135.05746459960938,
      "logps/real": -164.29153442382812,
      "loss": 0.8248,
      "rewards/accuracies": 0.0,
      "rewards/generated": 0.0,
      "rewards/margins": 0.0,
      "rewards/real": 0.0,
      "step": 1
    },
    {
      "epoch": 0.032,
      "grad_norm": 108.69554011112191,
      "learning_rate": 5.3191489361702123e-08,
      "logits/generated": -2.6056575775146484,
      "logits/real": -2.554622173309326,
      "logps/generated": -114.74867248535156,
      "logps/real": -128.5237274169922,
      "loss": 0.8329,
      "rewards/accuracies": 0.5,
      "rewards/generated": 0.0076134600676596165,
      "rewards/margins": -0.0009522127802483737,
      "rewards/real": 0.006661247927695513,
      "step": 10
    },
    {
      "epoch": 0.064,
      "grad_norm": 104.94374849378612,
      "learning_rate": 1.0638297872340425e-07,
      "logits/generated": -2.67850923538208,
      "logits/real": -2.651012897491455,
      "logps/generated": -120.8056640625,
      "logps/real": -133.23611450195312,
      "loss": 0.8169,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/generated": 0.1307533234357834,
      "rewards/margins": 0.002721247496083379,
      "rewards/real": 0.13347458839416504,
      "step": 20
    },
    {
      "epoch": 0.096,
      "grad_norm": 96.02197035555018,
      "learning_rate": 1.5957446808510638e-07,
      "logits/generated": -2.677885055541992,
      "logits/real": -2.7564849853515625,
      "logps/generated": -118.22355651855469,
      "logps/real": -132.5462646484375,
      "loss": 0.7877,
      "rewards/accuracies": 0.574999988079071,
      "rewards/generated": 0.34465548396110535,
      "rewards/margins": 0.08713299036026001,
      "rewards/real": 0.43178844451904297,
      "step": 30
    },
    {
      "epoch": 0.0992,
      "eval_logits/generated": -2.522256374359131,
      "eval_logits/real": -2.6980574131011963,
      "eval_logps/generated": -103.75227355957031,
      "eval_logps/real": -129.50059509277344,
      "eval_loss": 0.7716670036315918,
      "eval_rewards/accuracies": 0.7307692170143127,
      "eval_rewards/generated": 0.47610312700271606,
      "eval_rewards/margins": 0.2143612504005432,
      "eval_rewards/real": 0.6904643177986145,
      "eval_runtime": 27.37,
      "eval_samples_per_second": 7.307,
      "eval_steps_per_second": 0.475,
      "step": 31
    },
    {
      "epoch": 0.128,
      "grad_norm": 95.3097031879678,
      "learning_rate": 2.127659574468085e-07,
      "logits/generated": -2.6211893558502197,
      "logits/real": -2.675510883331299,
      "logps/generated": -126.9448471069336,
      "logps/real": -137.4635467529297,
      "loss": 0.7133,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/generated": 0.4792709946632385,
      "rewards/margins": 0.2818766236305237,
      "rewards/real": 0.7611476182937622,
      "step": 40
    },
    {
      "epoch": 0.16,
      "grad_norm": 106.05520318577042,
      "learning_rate": 2.659574468085106e-07,
      "logits/generated": -2.4675774574279785,
      "logits/real": -2.49176287651062,
      "logps/generated": -124.37459564208984,
      "logps/real": -129.06646728515625,
      "loss": 0.677,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/generated": 0.4745818078517914,
      "rewards/margins": 0.48313817381858826,
      "rewards/real": 0.9577198028564453,
      "step": 50
    },
    {
      "epoch": 0.192,
      "grad_norm": 84.95894852367661,
      "learning_rate": 3.1914893617021275e-07,
      "logits/generated": -2.628202438354492,
      "logits/real": -2.581429958343506,
      "logps/generated": -118.05242919921875,
      "logps/real": -122.88636779785156,
      "loss": 0.6309,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/generated": 1.081741452217102,
      "rewards/margins": 0.6737821102142334,
      "rewards/real": 1.7555233240127563,
      "step": 60
    },
    {
      "epoch": 0.1984,
      "eval_logits/generated": -2.5564322471618652,
      "eval_logits/real": -2.7395834922790527,
      "eval_logps/generated": -91.08578491210938,
      "eval_logps/real": -114.99056243896484,
      "eval_loss": 0.7320010662078857,
      "eval_rewards/accuracies": 0.7115384340286255,
      "eval_rewards/generated": 1.742753505706787,
      "eval_rewards/margins": 0.3987158238887787,
      "eval_rewards/real": 2.141469717025757,
      "eval_runtime": 24.6101,
      "eval_samples_per_second": 8.127,
      "eval_steps_per_second": 0.528,
      "step": 62
    },
    {
      "epoch": 0.224,
      "grad_norm": 96.85757291577832,
      "learning_rate": 3.7234042553191484e-07,
      "logits/generated": -2.5130114555358887,
      "logits/real": -2.5616209506988525,
      "logps/generated": -138.7924346923828,
      "logps/real": -111.9786148071289,
      "loss": 0.5514,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/generated": 1.2395274639129639,
      "rewards/margins": 1.0533320903778076,
      "rewards/real": 2.2928595542907715,
      "step": 70
    },
    {
      "epoch": 0.256,
      "grad_norm": 69.52077411879176,
      "learning_rate": 4.25531914893617e-07,
      "logits/generated": -2.4999165534973145,
      "logits/real": -2.664947509765625,
      "logps/generated": -105.97293853759766,
      "logps/real": -114.73320007324219,
      "loss": 0.5366,
      "rewards/accuracies": 0.8125,
      "rewards/generated": 1.6926729679107666,
      "rewards/margins": 1.063332200050354,
      "rewards/real": 2.75600528717041,
      "step": 80
    },
    {
      "epoch": 0.288,
      "grad_norm": 82.22178145539344,
      "learning_rate": 4.787234042553192e-07,
      "logits/generated": -2.6474456787109375,
      "logits/real": -2.5698373317718506,
      "logps/generated": -135.85374450683594,
      "logps/real": -129.9317626953125,
      "loss": 0.5309,
      "rewards/accuracies": 0.800000011920929,
      "rewards/generated": 1.0643891096115112,
      "rewards/margins": 1.536333441734314,
      "rewards/real": 2.6007227897644043,
      "step": 90
    },
    {
      "epoch": 0.2976,
      "eval_logits/generated": -2.4702792167663574,
      "eval_logits/real": -2.6170670986175537,
      "eval_logps/generated": -99.49687957763672,
      "eval_logps/real": -120.69674682617188,
      "eval_loss": 0.7175261974334717,
      "eval_rewards/accuracies": 0.6538461446762085,
      "eval_rewards/generated": 0.9016433358192444,
      "eval_rewards/margins": 0.6692073345184326,
      "eval_rewards/real": 1.5708506107330322,
      "eval_runtime": 24.5423,
      "eval_samples_per_second": 8.149,
      "eval_steps_per_second": 0.53,
      "step": 93
    },
    {
      "epoch": 0.32,
      "grad_norm": 67.99154678016933,
      "learning_rate": 4.96437054631829e-07,
      "logits/generated": -2.6401119232177734,
      "logits/real": -2.613041400909424,
      "logps/generated": -123.557861328125,
      "logps/real": -129.47727966308594,
      "loss": 0.4506,
      "rewards/accuracies": 0.800000011920929,
      "rewards/generated": 0.6703820824623108,
      "rewards/margins": 1.8516438007354736,
      "rewards/real": 2.5220255851745605,
      "step": 100
    },
    {
      "epoch": 0.352,
      "grad_norm": 74.51996731564344,
      "learning_rate": 4.904988123515439e-07,
      "logits/generated": -2.6040279865264893,
      "logits/real": -2.6699700355529785,
      "logps/generated": -155.3791961669922,
      "logps/real": -119.61136627197266,
      "loss": 0.3797,
      "rewards/accuracies": 0.887499988079071,
      "rewards/generated": -0.05242754891514778,
      "rewards/margins": 2.9086718559265137,
      "rewards/real": 2.8562443256378174,
      "step": 110
    },
    {
      "epoch": 0.384,
      "grad_norm": 90.56758343390324,
      "learning_rate": 4.845605700712589e-07,
      "logits/generated": -2.8406381607055664,
      "logits/real": -2.7864930629730225,
      "logps/generated": -137.2823028564453,
      "logps/real": -111.62813568115234,
      "loss": 0.4323,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/generated": 0.4588840901851654,
      "rewards/margins": 2.8661108016967773,
      "rewards/real": 3.3249950408935547,
      "step": 120
    },
    {
      "epoch": 0.3968,
      "eval_logits/generated": -2.6808340549468994,
      "eval_logits/real": -2.834907293319702,
      "eval_logps/generated": -93.77437591552734,
      "eval_logps/real": -116.81954193115234,
      "eval_loss": 0.7713803648948669,
      "eval_rewards/accuracies": 0.692307710647583,
      "eval_rewards/generated": 1.4738935232162476,
      "eval_rewards/margins": 0.4846768081188202,
      "eval_rewards/real": 1.9585703611373901,
      "eval_runtime": 25.6959,
      "eval_samples_per_second": 7.783,
      "eval_steps_per_second": 0.506,
      "step": 124
    },
    {
      "epoch": 0.416,
      "grad_norm": 41.82001394961611,
      "learning_rate": 4.786223277909738e-07,
      "logits/generated": -2.6804909706115723,
      "logits/real": -2.8820011615753174,
      "logps/generated": -121.76007080078125,
      "logps/real": -104.3374252319336,
      "loss": 0.3483,
      "rewards/accuracies": 0.925000011920929,
      "rewards/generated": 0.10464005172252655,
      "rewards/margins": 3.1740787029266357,
      "rewards/real": 3.278719425201416,
      "step": 130
    },
    {
      "epoch": 0.448,
      "grad_norm": 55.13818556944925,
      "learning_rate": 4.7268408551068883e-07,
      "logits/generated": -2.637974977493286,
      "logits/real": -2.62568998336792,
      "logps/generated": -138.10079956054688,
      "logps/real": -127.15523529052734,
      "loss": 0.3623,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/generated": 0.21112258732318878,
      "rewards/margins": 3.419811725616455,
      "rewards/real": 3.630934238433838,
      "step": 140
    },
    {
      "epoch": 0.48,
      "grad_norm": 73.76709240088495,
      "learning_rate": 4.667458432304038e-07,
      "logits/generated": -2.753176212310791,
      "logits/real": -2.7446675300598145,
      "logps/generated": -147.35614013671875,
      "logps/real": -124.41835021972656,
      "loss": 0.297,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/generated": -0.5855969190597534,
      "rewards/margins": 4.5017008781433105,
      "rewards/real": 3.9161038398742676,
      "step": 150
    },
    {
      "epoch": 0.496,
      "eval_logits/generated": -2.625582456588745,
      "eval_logits/real": -2.76962947845459,
      "eval_logps/generated": -90.96475219726562,
      "eval_logps/real": -113.50178527832031,
      "eval_loss": 0.7160908579826355,
      "eval_rewards/accuracies": 0.807692289352417,
      "eval_rewards/generated": 1.7548563480377197,
      "eval_rewards/margins": 0.5354906916618347,
      "eval_rewards/real": 2.290347099304199,
      "eval_runtime": 25.5334,
      "eval_samples_per_second": 7.833,
      "eval_steps_per_second": 0.509,
      "step": 155
    },
    {
      "epoch": 0.512,
      "grad_norm": 124.72244829757992,
      "learning_rate": 4.6080760095011875e-07,
      "logits/generated": -2.7324862480163574,
      "logits/real": -2.8079638481140137,
      "logps/generated": -128.80862426757812,
      "logps/real": -107.00062561035156,
      "loss": 0.2956,
      "rewards/accuracies": 0.9375,
      "rewards/generated": -0.16917666792869568,
      "rewards/margins": 4.109906196594238,
      "rewards/real": 3.940729856491089,
      "step": 160
    },
    {
      "epoch": 0.544,
      "grad_norm": 23.575152887764823,
      "learning_rate": 4.548693586698337e-07,
      "logits/generated": -2.5645294189453125,
      "logits/real": -2.688098430633545,
      "logps/generated": -122.01734924316406,
      "logps/real": -102.08761596679688,
      "loss": 0.2795,
      "rewards/accuracies": 0.949999988079071,
      "rewards/generated": -0.07551076263189316,
      "rewards/margins": 3.8361549377441406,
      "rewards/real": 3.7606441974639893,
      "step": 170
    },
    {
      "epoch": 0.576,
      "grad_norm": 32.24179696117588,
      "learning_rate": 4.4893111638954866e-07,
      "logits/generated": -2.7335903644561768,
      "logits/real": -2.827608823776245,
      "logps/generated": -135.81558227539062,
      "logps/real": -96.14201354980469,
      "loss": 0.2144,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/generated": -1.2115757465362549,
      "rewards/margins": 4.802846908569336,
      "rewards/real": 3.591270923614502,
      "step": 180
    },
    {
      "epoch": 0.5952,
      "eval_logits/generated": -2.8900225162506104,
      "eval_logits/real": -3.0601844787597656,
      "eval_logps/generated": -98.30004119873047,
      "eval_logps/real": -120.36709594726562,
      "eval_loss": 0.8256979584693909,
      "eval_rewards/accuracies": 0.7115384340286255,
      "eval_rewards/generated": 1.0213286876678467,
      "eval_rewards/margins": 0.5824873447418213,
      "eval_rewards/real": 1.6038157939910889,
      "eval_runtime": 24.9055,
      "eval_samples_per_second": 8.03,
      "eval_steps_per_second": 0.522,
      "step": 186
    },
    {
      "epoch": 0.608,
      "grad_norm": 41.48311970139271,
      "learning_rate": 4.429928741092636e-07,
      "logits/generated": -2.9662532806396484,
      "logits/real": -3.030586004257202,
      "logps/generated": -152.9399871826172,
      "logps/real": -105.7096176147461,
      "loss": 0.1739,
      "rewards/accuracies": 0.987500011920929,
      "rewards/generated": -2.151095390319824,
      "rewards/margins": 5.8033766746521,
      "rewards/real": 3.6522812843322754,
      "step": 190
    },
    {
      "epoch": 0.64,
      "grad_norm": 54.1414198416636,
      "learning_rate": 4.3705463182897863e-07,
      "logits/generated": -2.9348952770233154,
      "logits/real": -3.019658327102661,
      "logps/generated": -130.03958129882812,
      "logps/real": -91.68030548095703,
      "loss": 0.2408,
      "rewards/accuracies": 0.987500011920929,
      "rewards/generated": -1.0477750301361084,
      "rewards/margins": 5.414088249206543,
      "rewards/real": 4.36631441116333,
      "step": 200
    },
    {
      "epoch": 0.672,
      "grad_norm": 46.728112406251626,
      "learning_rate": 4.311163895486936e-07,
      "logits/generated": -2.939383029937744,
      "logits/real": -2.944849967956543,
      "logps/generated": -139.65420532226562,
      "logps/real": -83.42400360107422,
      "loss": 0.2497,
      "rewards/accuracies": 0.9375,
      "rewards/generated": -1.1235296726226807,
      "rewards/margins": 5.62716007232666,
      "rewards/real": 4.5036301612854,
      "step": 210
    },
    {
      "epoch": 0.6944,
      "eval_logits/generated": -2.6469037532806396,
      "eval_logits/real": -2.8200910091400146,
      "eval_logps/generated": -90.68226623535156,
      "eval_logps/real": -111.86186981201172,
      "eval_loss": 0.6848626732826233,
      "eval_rewards/accuracies": 0.807692289352417,
      "eval_rewards/generated": 1.7831059694290161,
      "eval_rewards/margins": 0.6712326407432556,
      "eval_rewards/real": 2.454339027404785,
      "eval_runtime": 24.5749,
      "eval_samples_per_second": 8.138,
      "eval_steps_per_second": 0.529,
      "step": 217
    },
    {
      "epoch": 0.704,
      "grad_norm": 20.434906367353744,
      "learning_rate": 4.251781472684085e-07,
      "logits/generated": -2.7813591957092285,
      "logits/real": -2.8630871772766113,
      "logps/generated": -147.4626007080078,
      "logps/real": -120.3201675415039,
      "loss": 0.1744,
      "rewards/accuracies": 1.0,
      "rewards/generated": -1.358475923538208,
      "rewards/margins": 6.355828285217285,
      "rewards/real": 4.9973530769348145,
      "step": 220
    },
    {
      "epoch": 0.736,
      "grad_norm": 17.208585033734995,
      "learning_rate": 4.192399049881235e-07,
      "logits/generated": -2.753448247909546,
      "logits/real": -2.8509767055511475,
      "logps/generated": -139.57362365722656,
      "logps/real": -95.3445053100586,
      "loss": 0.1283,
      "rewards/accuracies": 1.0,
      "rewards/generated": -1.6594841480255127,
      "rewards/margins": 6.16472864151001,
      "rewards/real": 4.505244255065918,
      "step": 230
    },
    {
      "epoch": 0.768,
      "grad_norm": 13.206387425612215,
      "learning_rate": 4.1330166270783846e-07,
      "logits/generated": -2.7692253589630127,
      "logits/real": -2.7172296047210693,
      "logps/generated": -154.8363800048828,
      "logps/real": -89.86678314208984,
      "loss": 0.1112,
      "rewards/accuracies": 1.0,
      "rewards/generated": -2.302570343017578,
      "rewards/margins": 6.421963691711426,
      "rewards/real": 4.1193928718566895,
      "step": 240
    },
    {
      "epoch": 0.7936,
      "eval_logits/generated": -2.7020058631896973,
      "eval_logits/real": -2.864475727081299,
      "eval_logps/generated": -94.19103240966797,
      "eval_logps/real": -113.57474517822266,
      "eval_loss": 0.6993240118026733,
      "eval_rewards/accuracies": 0.7884615659713745,
      "eval_rewards/generated": 1.4322292804718018,
      "eval_rewards/margins": 0.8508216142654419,
      "eval_rewards/real": 2.283051013946533,
      "eval_runtime": 25.5028,
      "eval_samples_per_second": 7.842,
      "eval_steps_per_second": 0.51,
      "step": 248
    },
    {
      "epoch": 0.8,
      "grad_norm": 9.621166462092326,
      "learning_rate": 4.0736342042755347e-07,
      "logits/generated": -2.7592086791992188,
      "logits/real": -2.8254053592681885,
      "logps/generated": -137.39413452148438,
      "logps/real": -84.954833984375,
      "loss": 0.1385,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/generated": -1.9928325414657593,
      "rewards/margins": 6.723714351654053,
      "rewards/real": 4.730882167816162,
      "step": 250
    },
    {
      "epoch": 0.832,
      "grad_norm": 37.766946962270424,
      "learning_rate": 4.0142517814726837e-07,
      "logits/generated": -2.855225086212158,
      "logits/real": -2.8763980865478516,
      "logps/generated": -165.6865997314453,
      "logps/real": -84.22928619384766,
      "loss": 0.1199,
      "rewards/accuracies": 0.987500011920929,
      "rewards/generated": -2.355945587158203,
      "rewards/margins": 7.509199619293213,
      "rewards/real": 5.153253555297852,
      "step": 260
    },
    {
      "epoch": 0.864,
      "grad_norm": 30.45497571981933,
      "learning_rate": 3.9548693586698333e-07,
      "logits/generated": -2.787148952484131,
      "logits/real": -2.8248982429504395,
      "logps/generated": -143.9623260498047,
      "logps/real": -72.55261993408203,
      "loss": 0.176,
      "rewards/accuracies": 0.987500011920929,
      "rewards/generated": -2.554567813873291,
      "rewards/margins": 7.758988857269287,
      "rewards/real": 5.204421043395996,
      "step": 270
    },
    {
      "epoch": 0.8928,
      "eval_logits/generated": -2.805142641067505,
      "eval_logits/real": -2.9279673099517822,
      "eval_logps/generated": -86.06629180908203,
      "eval_logps/real": -108.56407165527344,
      "eval_loss": 0.6700232028961182,
      "eval_rewards/accuracies": 0.7692307829856873,
      "eval_rewards/generated": 2.2447028160095215,
      "eval_rewards/margins": 0.5394153594970703,
      "eval_rewards/real": 2.784118175506592,
      "eval_runtime": 25.461,
      "eval_samples_per_second": 7.855,
      "eval_steps_per_second": 0.511,
      "step": 279
    },
    {
      "epoch": 0.896,
      "grad_norm": 12.11949130938464,
      "learning_rate": 3.8954869358669834e-07,
      "logits/generated": -2.8453097343444824,
      "logits/real": -2.9081547260284424,
      "logps/generated": -130.38711547851562,
      "logps/real": -80.25016784667969,
      "loss": 0.2256,
      "rewards/accuracies": 0.987500011920929,
      "rewards/generated": -1.7986886501312256,
      "rewards/margins": 6.868111610412598,
      "rewards/real": 5.069422721862793,
      "step": 280
    },
    {
      "epoch": 0.928,
      "grad_norm": 8.258057690722529,
      "learning_rate": 3.836104513064133e-07,
      "logits/generated": -2.855512857437134,
      "logits/real": -2.82855224609375,
      "logps/generated": -147.15182495117188,
      "logps/real": -93.78026580810547,
      "loss": 0.144,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/generated": -2.1502671241760254,
      "rewards/margins": 7.681868553161621,
      "rewards/real": 5.531601905822754,
      "step": 290
    },
    {
      "epoch": 0.96,
      "grad_norm": 19.59092219958014,
      "learning_rate": 3.7767220902612825e-07,
      "logits/generated": -2.7956345081329346,
      "logits/real": -2.873706817626953,
      "logps/generated": -146.74185180664062,
      "logps/real": -91.1478500366211,
      "loss": 0.15,
      "rewards/accuracies": 0.987500011920929,
      "rewards/generated": -1.9168081283569336,
      "rewards/margins": 8.325960159301758,
      "rewards/real": 6.40915060043335,
      "step": 300
    },
    {
      "epoch": 0.992,
      "grad_norm": 15.834799965250946,
      "learning_rate": 3.717339667458432e-07,
      "logits/generated": -2.8749289512634277,
      "logits/real": -2.876892566680908,
      "logps/generated": -163.785400390625,
      "logps/real": -88.39195251464844,
      "loss": 0.1135,
      "rewards/accuracies": 1.0,
      "rewards/generated": -2.48066782951355,
      "rewards/margins": 8.187207221984863,
      "rewards/real": 5.706539154052734,
      "step": 310
    },
    {
      "epoch": 0.992,
      "eval_logits/generated": -2.802443027496338,
      "eval_logits/real": -2.9202871322631836,
      "eval_logps/generated": -90.31492614746094,
      "eval_logps/real": -112.55606842041016,
      "eval_loss": 0.6956190466880798,
      "eval_rewards/accuracies": 0.7884615659713745,
      "eval_rewards/generated": 1.8198401927947998,
      "eval_rewards/margins": 0.565077543258667,
      "eval_rewards/real": 2.384917974472046,
      "eval_runtime": 25.3747,
      "eval_samples_per_second": 7.882,
      "eval_steps_per_second": 0.512,
      "step": 310
    },
    {
      "epoch": 1.024,
      "grad_norm": 3.2127566683583244,
      "learning_rate": 3.6579572446555817e-07,
      "logits/generated": -2.8369505405426025,
      "logits/real": -2.8505897521972656,
      "logps/generated": -159.44688415527344,
      "logps/real": -78.3135986328125,
      "loss": 0.1157,
      "rewards/accuracies": 1.0,
      "rewards/generated": -2.547872543334961,
      "rewards/margins": 8.247138023376465,
      "rewards/real": 5.6992669105529785,
      "step": 320
    },
    {
      "epoch": 1.056,
      "grad_norm": 5.785222044107326,
      "learning_rate": 3.598574821852731e-07,
      "logits/generated": -2.6663429737091064,
      "logits/real": -2.7017171382904053,
      "logps/generated": -150.9597625732422,
      "logps/real": -76.73829650878906,
      "loss": 0.1132,
      "rewards/accuracies": 1.0,
      "rewards/generated": -2.2275402545928955,
      "rewards/margins": 8.38085651397705,
      "rewards/real": 6.153315544128418,
      "step": 330
    },
    {
      "epoch": 1.088,
      "grad_norm": 2.5305457609113042,
      "learning_rate": 3.5391923990498813e-07,
      "logits/generated": -2.5699503421783447,
      "logits/real": -2.605731964111328,
      "logps/generated": -153.1111297607422,
      "logps/real": -74.11537170410156,
      "loss": 0.1221,
      "rewards/accuracies": 1.0,
      "rewards/generated": -2.6568379402160645,
      "rewards/margins": 8.866227149963379,
      "rewards/real": 6.209389686584473,
      "step": 340
    },
    {
      "epoch": 1.0912,
      "eval_logits/generated": -2.5885517597198486,
      "eval_logits/real": -2.7364766597747803,
      "eval_logps/generated": -92.37077331542969,
      "eval_logps/real": -114.35930633544922,
      "eval_loss": 0.7313989400863647,
      "eval_rewards/accuracies": 0.7307692170143127,
      "eval_rewards/generated": 1.6142545938491821,
      "eval_rewards/margins": 0.5903396010398865,
      "eval_rewards/real": 2.204594612121582,
      "eval_runtime": 24.3069,
      "eval_samples_per_second": 8.228,
      "eval_steps_per_second": 0.535,
      "step": 341
    },
    {
      "epoch": 1.12,
      "grad_norm": 3.541237222198075,
      "learning_rate": 3.479809976247031e-07,
      "logits/generated": -2.6312408447265625,
      "logits/real": -2.6744143962860107,
      "logps/generated": -142.3855743408203,
      "logps/real": -71.9720230102539,
      "loss": 0.108,
      "rewards/accuracies": 0.987500011920929,
      "rewards/generated": -2.536757469177246,
      "rewards/margins": 8.74314022064209,
      "rewards/real": 6.206382751464844,
      "step": 350
    },
    {
      "epoch": 1.152,
      "grad_norm": 3.62080901586099,
      "learning_rate": 3.42042755344418e-07,
      "logits/generated": -2.5982825756073,
      "logits/real": -2.579300880432129,
      "logps/generated": -146.4971466064453,
      "logps/real": -72.2068099975586,
      "loss": 0.0771,
      "rewards/accuracies": 1.0,
      "rewards/generated": -2.3290672302246094,
      "rewards/margins": 9.054376602172852,
      "rewards/real": 6.7253098487854,
      "step": 360
    },
    {
      "epoch": 1.184,
      "grad_norm": 19.96975206692693,
      "learning_rate": 3.36104513064133e-07,
      "logits/generated": -2.5062308311462402,
      "logits/real": -2.631631851196289,
      "logps/generated": -162.20138549804688,
      "logps/real": -84.9935531616211,
      "loss": 0.0864,
      "rewards/accuracies": 1.0,
      "rewards/generated": -3.1360862255096436,
      "rewards/margins": 9.973578453063965,
      "rewards/real": 6.837491512298584,
      "step": 370
    },
    {
      "epoch": 1.1904,
      "eval_logits/generated": -2.535508871078491,
      "eval_logits/real": -2.7014498710632324,
      "eval_logps/generated": -89.05432891845703,
      "eval_logps/real": -113.19943237304688,
      "eval_loss": 0.7718254327774048,
      "eval_rewards/accuracies": 0.6346153616905212,
      "eval_rewards/generated": 1.945899248123169,
      "eval_rewards/margins": 0.37468206882476807,
      "eval_rewards/real": 2.3205814361572266,
      "eval_runtime": 25.3769,
      "eval_samples_per_second": 7.881,
      "eval_steps_per_second": 0.512,
      "step": 372
    },
    {
      "epoch": 1.216,
      "grad_norm": 1.77932953998119,
      "learning_rate": 3.3016627078384796e-07,
      "logits/generated": -2.683812379837036,
      "logits/real": -2.721485137939453,
      "logps/generated": -150.50462341308594,
      "logps/real": -63.769615173339844,
      "loss": 0.1072,
      "rewards/accuracies": 1.0,
      "rewards/generated": -2.774550437927246,
      "rewards/margins": 9.467794418334961,
      "rewards/real": 6.693244934082031,
      "step": 380
    },
    {
      "epoch": 1.248,
      "grad_norm": 2.282468411046649,
      "learning_rate": 3.2422802850356297e-07,
      "logits/generated": -2.680248260498047,
      "logits/real": -2.712040424346924,
      "logps/generated": -159.68753051757812,
      "logps/real": -74.8614501953125,
      "loss": 0.0841,
      "rewards/accuracies": 1.0,
      "rewards/generated": -3.0862269401550293,
      "rewards/margins": 10.16987133026123,
      "rewards/real": 7.083643913269043,
      "step": 390
    },
    {
      "epoch": 1.28,
      "grad_norm": 7.624913421372154,
      "learning_rate": 3.182897862232779e-07,
      "logits/generated": -2.617255926132202,
      "logits/real": -2.6958706378936768,
      "logps/generated": -152.98448181152344,
      "logps/real": -74.80540466308594,
      "loss": 0.0871,
      "rewards/accuracies": 0.987500011920929,
      "rewards/generated": -2.7490036487579346,
      "rewards/margins": 9.90536880493164,
      "rewards/real": 7.156365394592285,
      "step": 400
    },
    {
      "epoch": 1.2896,
      "eval_logits/generated": -2.5240492820739746,
      "eval_logits/real": -2.683285713195801,
      "eval_logps/generated": -91.45059967041016,
      "eval_logps/real": -116.5322265625,
      "eval_loss": 0.8230924010276794,
      "eval_rewards/accuracies": 0.5961538553237915,
      "eval_rewards/generated": 1.7062723636627197,
      "eval_rewards/margins": 0.28103044629096985,
      "eval_rewards/real": 1.9873030185699463,
      "eval_runtime": 25.4198,
      "eval_samples_per_second": 7.868,
      "eval_steps_per_second": 0.511,
      "step": 403
    },
    {
      "epoch": 1.312,
      "grad_norm": 2.2179908898867304,
      "learning_rate": 3.1235154394299283e-07,
      "logits/generated": -2.590919017791748,
      "logits/real": -2.6620683670043945,
      "logps/generated": -154.9713897705078,
      "logps/real": -64.6716537475586,
      "loss": 0.1223,
      "rewards/accuracies": 1.0,
      "rewards/generated": -3.2485358715057373,
      "rewards/margins": 11.025705337524414,
      "rewards/real": 7.777169704437256,
      "step": 410
    },
    {
      "epoch": 1.3439999999999999,
      "grad_norm": 2.1123570407415557,
      "learning_rate": 3.0641330166270784e-07,
      "logits/generated": -2.5320184230804443,
      "logits/real": -2.605661153793335,
      "logps/generated": -157.86102294921875,
      "logps/real": -64.73320007324219,
      "loss": 0.1132,
      "rewards/accuracies": 0.987500011920929,
      "rewards/generated": -3.422405242919922,
      "rewards/margins": 10.930196762084961,
      "rewards/real": 7.5077924728393555,
      "step": 420
    },
    {
      "epoch": 1.376,
      "grad_norm": 52.0477538818839,
      "learning_rate": 3.004750593824228e-07,
      "logits/generated": -2.479769468307495,
      "logits/real": -2.5996639728546143,
      "logps/generated": -170.76138305664062,
      "logps/real": -64.69879150390625,
      "loss": 0.1454,
      "rewards/accuracies": 1.0,
      "rewards/generated": -4.175970554351807,
      "rewards/margins": 11.573558807373047,
      "rewards/real": 7.397587776184082,
      "step": 430
    },
    {
      "epoch": 1.3888,
      "eval_logits/generated": -2.4324543476104736,
      "eval_logits/real": -2.611969232559204,
      "eval_logps/generated": -95.73091888427734,
      "eval_logps/real": -119.04707336425781,
      "eval_loss": 0.7979809641838074,
      "eval_rewards/accuracies": 0.6730769276618958,
      "eval_rewards/generated": 1.2782399654388428,
      "eval_rewards/margins": 0.4575783908367157,
      "eval_rewards/real": 1.7358182668685913,
      "eval_runtime": 25.2977,
      "eval_samples_per_second": 7.906,
      "eval_steps_per_second": 0.514,
      "step": 434
    },
    {
      "epoch": 1.408,
      "grad_norm": 3.7531043248353426,
      "learning_rate": 2.9453681710213776e-07,
      "logits/generated": -2.6189701557159424,
      "logits/real": -2.6460072994232178,
      "logps/generated": -178.8316192626953,
      "logps/real": -66.7384033203125,
      "loss": 0.07,
      "rewards/accuracies": 1.0,
      "rewards/generated": -3.621222734451294,
      "rewards/margins": 11.845494270324707,
      "rewards/real": 8.224271774291992,
      "step": 440
    },
    {
      "epoch": 1.44,
      "grad_norm": 2.986230537501553,
      "learning_rate": 2.885985748218527e-07,
      "logits/generated": -2.5055761337280273,
      "logits/real": -2.4730076789855957,
      "logps/generated": -173.35125732421875,
      "logps/real": -54.73099899291992,
      "loss": 0.0934,
      "rewards/accuracies": 1.0,
      "rewards/generated": -4.109053611755371,
      "rewards/margins": 11.3353271484375,
      "rewards/real": 7.2262725830078125,
      "step": 450
    },
    {
      "epoch": 1.472,
      "grad_norm": 2.618123223732552,
      "learning_rate": 2.8266033254156767e-07,
      "logits/generated": -2.4285778999328613,
      "logits/real": -2.516470432281494,
      "logps/generated": -151.6107177734375,
      "logps/real": -55.497283935546875,
      "loss": 0.0747,
      "rewards/accuracies": 0.987500011920929,
      "rewards/generated": -3.275545835494995,
      "rewards/margins": 11.137293815612793,
      "rewards/real": 7.861748695373535,
      "step": 460
    },
    {
      "epoch": 1.488,
      "eval_logits/generated": -2.3556630611419678,
      "eval_logits/real": -2.568284034729004,
      "eval_logps/generated": -93.57498931884766,
      "eval_logps/real": -117.37250518798828,
      "eval_loss": 0.8086085319519043,
      "eval_rewards/accuracies": 0.6538461446762085,
      "eval_rewards/generated": 1.4938340187072754,
      "eval_rewards/margins": 0.40944167971611023,
      "eval_rewards/real": 1.9032756090164185,
      "eval_runtime": 25.2837,
      "eval_samples_per_second": 7.91,
      "eval_steps_per_second": 0.514,
      "step": 465
    },
    {
      "epoch": 1.504,
      "grad_norm": 2.3062948097679628,
      "learning_rate": 2.7672209026128263e-07,
      "logits/generated": -2.356971263885498,
      "logits/real": -2.3984694480895996,
      "logps/generated": -161.08523559570312,
      "logps/real": -62.349212646484375,
      "loss": 0.1099,
      "rewards/accuracies": 0.987500011920929,
      "rewards/generated": -3.898419141769409,
      "rewards/margins": 12.482399940490723,
      "rewards/real": 8.583980560302734,
      "step": 470
    },
    {
      "epoch": 1.536,
      "grad_norm": 2.1831722984217468,
      "learning_rate": 2.7078384798099764e-07,
      "logits/generated": -2.434488296508789,
      "logits/real": -2.4292871952056885,
      "logps/generated": -165.63919067382812,
      "logps/real": -53.660057067871094,
      "loss": 0.0923,
      "rewards/accuracies": 0.987500011920929,
      "rewards/generated": -3.511655330657959,
      "rewards/margins": 11.318243980407715,
      "rewards/real": 7.806589603424072,
      "step": 480
    },
    {
      "epoch": 1.568,
      "grad_norm": 1.9381609438311436,
      "learning_rate": 2.648456057007126e-07,
      "logits/generated": -2.395662784576416,
      "logits/real": -2.446293354034424,
      "logps/generated": -171.5785675048828,
      "logps/real": -52.84147262573242,
      "loss": 0.0882,
      "rewards/accuracies": 1.0,
      "rewards/generated": -3.7677719593048096,
      "rewards/margins": 13.188079833984375,
      "rewards/real": 9.420308113098145,
      "step": 490
    },
    {
      "epoch": 1.5872000000000002,
      "eval_logits/generated": -2.272164821624756,
      "eval_logits/real": -2.4783287048339844,
      "eval_logps/generated": -103.67977142333984,
      "eval_logps/real": -128.15371704101562,
      "eval_loss": 0.928070068359375,
      "eval_rewards/accuracies": 0.5192307829856873,
      "eval_rewards/generated": 0.483354777097702,
      "eval_rewards/margins": 0.34179702401161194,
      "eval_rewards/real": 0.8251518607139587,
      "eval_runtime": 25.574,
      "eval_samples_per_second": 7.82,
      "eval_steps_per_second": 0.508,
      "step": 496
    },
    {
      "epoch": 1.6,
      "grad_norm": 2.2605824412655213,
      "learning_rate": 2.589073634204275e-07,
      "logits/generated": -2.3627734184265137,
      "logits/real": -2.4548180103302,
      "logps/generated": -169.9461212158203,
      "logps/real": -53.60149383544922,
      "loss": 0.0556,
      "rewards/accuracies": 1.0,
      "rewards/generated": -4.064536094665527,
      "rewards/margins": 12.720796585083008,
      "rewards/real": 8.656259536743164,
      "step": 500
    },
    {
      "epoch": 1.6320000000000001,
      "grad_norm": 2.2813650068674387,
      "learning_rate": 2.529691211401425e-07,
      "logits/generated": -2.449333906173706,
      "logits/real": -2.473750352859497,
      "logps/generated": -169.88919067382812,
      "logps/real": -48.773372650146484,
      "loss": 0.0795,
      "rewards/accuracies": 1.0,
      "rewards/generated": -4.045998573303223,
      "rewards/margins": 13.330732345581055,
      "rewards/real": 9.284735679626465,
      "step": 510
    },
    {
      "epoch": 1.6640000000000001,
      "grad_norm": 1.9756005158149215,
      "learning_rate": 2.4703087885985747e-07,
      "logits/generated": -2.2918307781219482,
      "logits/real": -2.4153807163238525,
      "logps/generated": -180.82943725585938,
      "logps/real": -64.54903411865234,
      "loss": 0.0693,
      "rewards/accuracies": 0.987500011920929,
      "rewards/generated": -4.285412788391113,
      "rewards/margins": 13.515165328979492,
      "rewards/real": 9.229752540588379,
      "step": 520
    },
    {
      "epoch": 1.6864,
      "eval_logits/generated": -2.139906644821167,
      "eval_logits/real": -2.368074893951416,
      "eval_logps/generated": -108.95226287841797,
      "eval_logps/real": -131.37367248535156,
      "eval_loss": 0.8953699469566345,
      "eval_rewards/accuracies": 0.6153846383094788,
      "eval_rewards/generated": -0.04389484226703644,
      "eval_rewards/margins": 0.5470523238182068,
      "eval_rewards/real": 0.5031575560569763,
      "eval_runtime": 25.3846,
      "eval_samples_per_second": 7.879,
      "eval_steps_per_second": 0.512,
      "step": 527
    },
    {
      "epoch": 1.696,
      "grad_norm": 2.077906587896759,
      "learning_rate": 2.410926365795724e-07,
      "logits/generated": -2.4040651321411133,
      "logits/real": -2.4135825634002686,
      "logps/generated": -208.2981414794922,
      "logps/real": -51.23198699951172,
      "loss": 0.0774,
      "rewards/accuracies": 1.0,
      "rewards/generated": -5.633874416351318,
      "rewards/margins": 15.663395881652832,
      "rewards/real": 10.029520988464355,
      "step": 530
    },
    {
      "epoch": 1.728,
      "grad_norm": 15.329093500904815,
      "learning_rate": 2.351543942992874e-07,
      "logits/generated": -2.3569560050964355,
      "logits/real": -2.3737831115722656,
      "logps/generated": -171.27365112304688,
      "logps/real": -41.608699798583984,
      "loss": 0.0459,
      "rewards/accuracies": 1.0,
      "rewards/generated": -4.523318767547607,
      "rewards/margins": 14.89915943145752,
      "rewards/real": 10.375839233398438,
      "step": 540
    },
    {
      "epoch": 1.76,
      "grad_norm": 1.5653317010766996,
      "learning_rate": 2.2921615201900234e-07,
      "logits/generated": -2.2786147594451904,
      "logits/real": -2.380913496017456,
      "logps/generated": -178.86477661132812,
      "logps/real": -51.02559280395508,
      "loss": 0.0982,
      "rewards/accuracies": 0.987500011920929,
      "rewards/generated": -4.9379401206970215,
      "rewards/margins": 14.276674270629883,
      "rewards/real": 9.338732719421387,
      "step": 550
    },
    {
      "epoch": 1.7856,
      "eval_logits/generated": -2.332611560821533,
      "eval_logits/real": -2.5183045864105225,
      "eval_logps/generated": -103.10279846191406,
      "eval_logps/real": -126.28335571289062,
      "eval_loss": 0.8776556253433228,
      "eval_rewards/accuracies": 0.6538461446762085,
      "eval_rewards/generated": 0.5410510897636414,
      "eval_rewards/margins": 0.47113847732543945,
      "eval_rewards/real": 1.0121896266937256,
      "eval_runtime": 24.3313,
      "eval_samples_per_second": 8.22,
      "eval_steps_per_second": 0.534,
      "step": 558
    },
    {
      "epoch": 1.792,
      "grad_norm": 23.493623835378564,
      "learning_rate": 2.2327790973871732e-07,
      "logits/generated": -2.4433507919311523,
      "logits/real": -2.4358441829681396,
      "logps/generated": -192.2283935546875,
      "logps/real": -47.60938262939453,
      "loss": 0.1072,
      "rewards/accuracies": 0.987500011920929,
      "rewards/generated": -5.960423469543457,
      "rewards/margins": 14.969975471496582,
      "rewards/real": 9.009553909301758,
      "step": 560
    },
    {
      "epoch": 1.8239999999999998,
      "grad_norm": 2.4384516635536024,
      "learning_rate": 2.173396674584323e-07,
      "logits/generated": -2.429056406021118,
      "logits/real": -2.447208881378174,
      "logps/generated": -178.07447814941406,
      "logps/real": -44.7133903503418,
      "loss": 0.0462,
      "rewards/accuracies": 1.0,
      "rewards/generated": -5.000073432922363,
      "rewards/margins": 14.771443367004395,
      "rewards/real": 9.771368980407715,
      "step": 570
    },
    {
      "epoch": 1.8559999999999999,
      "grad_norm": 2.9085865122927945,
      "learning_rate": 2.1140142517814726e-07,
      "logits/generated": -2.404963254928589,
      "logits/real": -2.481902599334717,
      "logps/generated": -188.43467712402344,
      "logps/real": -44.5074462890625,
      "loss": 0.0674,
      "rewards/accuracies": 0.987500011920929,
      "rewards/generated": -5.706393718719482,
      "rewards/margins": 16.334299087524414,
      "rewards/real": 10.627904891967773,
      "step": 580
    },
    {
      "epoch": 1.8848,
      "eval_logits/generated": -2.302616834640503,
      "eval_logits/real": -2.4848058223724365,
      "eval_logps/generated": -113.82382202148438,
      "eval_logps/real": -136.99197387695312,
      "eval_loss": 0.9359613060951233,
      "eval_rewards/accuracies": 0.5961538553237915,
      "eval_rewards/generated": -0.5310505628585815,
      "eval_rewards/margins": 0.4723770022392273,
      "eval_rewards/real": -0.058673594146966934,
      "eval_runtime": 25.5072,
      "eval_samples_per_second": 7.841,
      "eval_steps_per_second": 0.51,
      "step": 589
    },
    {
      "epoch": 1.888,
      "grad_norm": 2.224003882967406,
      "learning_rate": 2.0546318289786222e-07,
      "logits/generated": -2.4189112186431885,
      "logits/real": -2.4926321506500244,
      "logps/generated": -189.12107849121094,
      "logps/real": -34.886329650878906,
      "loss": 0.0404,
      "rewards/accuracies": 1.0,
      "rewards/generated": -6.180232048034668,
      "rewards/margins": 15.565508842468262,
      "rewards/real": 9.385276794433594,
      "step": 590
    },
    {
      "epoch": 1.92,
      "grad_norm": 2.459225729639833,
      "learning_rate": 1.9952494061757718e-07,
      "logits/generated": -2.362020492553711,
      "logits/real": -2.3705716133117676,
      "logps/generated": -185.7289581298828,
      "logps/real": -29.566946029663086,
      "loss": 0.0835,
      "rewards/accuracies": 1.0,
      "rewards/generated": -6.159688472747803,
      "rewards/margins": 16.430912017822266,
      "rewards/real": 10.271220207214355,
      "step": 600
    },
    {
      "epoch": 1.952,
      "grad_norm": 2.454360280136539,
      "learning_rate": 1.9358669833729216e-07,
      "logits/generated": -2.4703481197357178,
      "logits/real": -2.499268054962158,
      "logps/generated": -194.42752075195312,
      "logps/real": -35.10074996948242,
      "loss": 0.0584,
      "rewards/accuracies": 1.0,
      "rewards/generated": -6.010377407073975,
      "rewards/margins": 17.025707244873047,
      "rewards/real": 11.015329360961914,
      "step": 610
    },
    {
      "epoch": 1.984,
      "grad_norm": 2.4169242550497096,
      "learning_rate": 1.876484560570071e-07,
      "logits/generated": -2.390986919403076,
      "logits/real": -2.453716993331909,
      "logps/generated": -187.9002227783203,
      "logps/real": -46.34991455078125,
      "loss": 0.0424,
      "rewards/accuracies": 1.0,
      "rewards/generated": -6.053334712982178,
      "rewards/margins": 17.14385986328125,
      "rewards/real": 11.090524673461914,
      "step": 620
    },
    {
      "epoch": 1.984,
      "eval_logits/generated": -2.295470952987671,
      "eval_logits/real": -2.4845852851867676,
      "eval_logps/generated": -115.48155975341797,
      "eval_logps/real": -138.99147033691406,
      "eval_loss": 0.9420953989028931,
      "eval_rewards/accuracies": 0.5769230723381042,
      "eval_rewards/generated": -0.6968246102333069,
      "eval_rewards/margins": 0.4382031559944153,
      "eval_rewards/real": -0.2586213946342468,
      "eval_runtime": 25.416,
      "eval_samples_per_second": 7.869,
      "eval_steps_per_second": 0.511,
      "step": 620
    },
    {
      "epoch": 2.016,
      "grad_norm": 13.261771791971471,
      "learning_rate": 1.8171021377672207e-07,
      "logits/generated": -2.4547436237335205,
      "logits/real": -2.3958754539489746,
      "logps/generated": -199.30731201171875,
      "logps/real": -31.5421142578125,
      "loss": 0.0561,
      "rewards/accuracies": 1.0,
      "rewards/generated": -6.895923614501953,
      "rewards/margins": 18.920942306518555,
      "rewards/real": 12.025017738342285,
      "step": 630
    },
    {
      "epoch": 2.048,
      "grad_norm": 3.1173265935906165,
      "learning_rate": 1.7577197149643706e-07,
      "logits/generated": -2.331156015396118,
      "logits/real": -2.406820774078369,
      "logps/generated": -202.47897338867188,
      "logps/real": -29.300609588623047,
      "loss": 0.067,
      "rewards/accuracies": 0.987500011920929,
      "rewards/generated": -6.904895782470703,
      "rewards/margins": 18.91607093811035,
      "rewards/real": 12.011175155639648,
      "step": 640
    },
    {
      "epoch": 2.08,
      "grad_norm": 2.33318980437696,
      "learning_rate": 1.6983372921615202e-07,
      "logits/generated": -2.3845553398132324,
      "logits/real": -2.3674371242523193,
      "logps/generated": -214.82986450195312,
      "logps/real": -25.64461898803711,
      "loss": 0.0235,
      "rewards/accuracies": 1.0,
      "rewards/generated": -7.688128471374512,
      "rewards/margins": 19.164615631103516,
      "rewards/real": 11.476486206054688,
      "step": 650
    },
    {
      "epoch": 2.0832,
      "eval_logits/generated": -2.2115392684936523,
      "eval_logits/real": -2.3974390029907227,
      "eval_logps/generated": -128.70654296875,
      "eval_logps/real": -153.1709442138672,
      "eval_loss": 1.0938998460769653,
      "eval_rewards/accuracies": 0.5,
      "eval_rewards/generated": -2.019321918487549,
      "eval_rewards/margins": 0.3427533805370331,
      "eval_rewards/real": -1.6765685081481934,
      "eval_runtime": 24.3647,
      "eval_samples_per_second": 8.209,
      "eval_steps_per_second": 0.534,
      "step": 651
    },
    {
      "epoch": 2.112,
      "grad_norm": 2.851649001912384,
      "learning_rate": 1.6389548693586697e-07,
      "logits/generated": -2.297102689743042,
      "logits/real": -2.319133758544922,
      "logps/generated": -201.3473663330078,
      "logps/real": -25.802215576171875,
      "loss": 0.0541,
      "rewards/accuracies": 1.0,
      "rewards/generated": -7.507266044616699,
      "rewards/margins": 18.991275787353516,
      "rewards/real": 11.484009742736816,
      "step": 660
    },
    {
      "epoch": 2.144,
      "grad_norm": 2.090545045938182,
      "learning_rate": 1.5795724465558193e-07,
      "logits/generated": -2.4693615436553955,
      "logits/real": -2.4613473415374756,
      "logps/generated": -200.6858367919922,
      "logps/real": -39.16984558105469,
      "loss": 0.0572,
      "rewards/accuracies": 1.0,
      "rewards/generated": -7.124711513519287,
      "rewards/margins": 20.449026107788086,
      "rewards/real": 13.324316024780273,
      "step": 670
    },
    {
      "epoch": 2.176,
      "grad_norm": 2.1939273942211277,
      "learning_rate": 1.520190023752969e-07,
      "logits/generated": -2.344393253326416,
      "logits/real": -2.3299365043640137,
      "logps/generated": -212.4817352294922,
      "logps/real": -20.405345916748047,
      "loss": 0.024,
      "rewards/accuracies": 1.0,
      "rewards/generated": -7.623618125915527,
      "rewards/margins": 18.887493133544922,
      "rewards/real": 11.263875007629395,
      "step": 680
    },
    {
      "epoch": 2.1824,
      "eval_logits/generated": -2.204942226409912,
      "eval_logits/real": -2.393613338470459,
      "eval_logps/generated": -133.9093017578125,
      "eval_logps/real": -157.97007751464844,
      "eval_loss": 1.1490856409072876,
      "eval_rewards/accuracies": 0.5,
      "eval_rewards/generated": -2.539597749710083,
      "eval_rewards/margins": 0.3831157386302948,
      "eval_rewards/real": -2.156482219696045,
      "eval_runtime": 25.5214,
      "eval_samples_per_second": 7.837,
      "eval_steps_per_second": 0.509,
      "step": 682
    },
    {
      "epoch": 2.208,
      "grad_norm": 2.6699514134692506,
      "learning_rate": 1.4608076009501184e-07,
      "logits/generated": -2.3223159313201904,
      "logits/real": -2.267909526824951,
      "logps/generated": -209.5951690673828,
      "logps/real": -22.11200714111328,
      "loss": 0.0273,
      "rewards/accuracies": 1.0,
      "rewards/generated": -7.6561784744262695,
      "rewards/margins": 18.957958221435547,
      "rewards/real": 11.301776885986328,
      "step": 690
    },
    {
      "epoch": 2.24,
      "grad_norm": 2.245581599718528,
      "learning_rate": 1.4014251781472683e-07,
      "logits/generated": -2.293184757232666,
      "logits/real": -2.2608914375305176,
      "logps/generated": -195.58767700195312,
      "logps/real": -30.79043197631836,
      "loss": 0.0365,
      "rewards/accuracies": 0.987500011920929,
      "rewards/generated": -6.686369895935059,
      "rewards/margins": 18.34311294555664,
      "rewards/real": 11.656742095947266,
      "step": 700
    },
    {
      "epoch": 2.2720000000000002,
      "grad_norm": 3.154191159829218,
      "learning_rate": 1.342042755344418e-07,
      "logits/generated": -2.2829322814941406,
      "logits/real": -2.370777130126953,
      "logps/generated": -205.3447723388672,
      "logps/real": -21.945812225341797,
      "loss": 0.0469,
      "rewards/accuracies": 1.0,
      "rewards/generated": -7.478776454925537,
      "rewards/margins": 19.887378692626953,
      "rewards/real": 12.408601760864258,
      "step": 710
    },
    {
      "epoch": 2.2816,
      "eval_logits/generated": -2.2161128520965576,
      "eval_logits/real": -2.409364938735962,
      "eval_logps/generated": -133.31402587890625,
      "eval_logps/real": -157.023193359375,
      "eval_loss": 1.1324479579925537,
      "eval_rewards/accuracies": 0.5,
      "eval_rewards/generated": -2.480071544647217,
      "eval_rewards/margins": 0.41827720403671265,
      "eval_rewards/real": -2.0617942810058594,
      "eval_runtime": 25.3651,
      "eval_samples_per_second": 7.885,
      "eval_steps_per_second": 0.513,
      "step": 713
    },
    {
      "epoch": 2.304,
      "grad_norm": 1.70496744806456,
      "learning_rate": 1.2826603325415677e-07,
      "logits/generated": -2.3763372898101807,
      "logits/real": -2.365422248840332,
      "logps/generated": -207.6084442138672,
      "logps/real": -18.752004623413086,
      "loss": 0.0623,
      "rewards/accuracies": 0.987500011920929,
      "rewards/generated": -7.463881015777588,
      "rewards/margins": 19.985795974731445,
      "rewards/real": 12.5219144821167,
      "step": 720
    },
    {
      "epoch": 2.336,
      "grad_norm": 1.7034616606078241,
      "learning_rate": 1.2232779097387173e-07,
      "logits/generated": -2.3761379718780518,
      "logits/real": -2.344998836517334,
      "logps/generated": -213.1655731201172,
      "logps/real": -18.98818016052246,
      "loss": 0.0191,
      "rewards/accuracies": 1.0,
      "rewards/generated": -8.140850067138672,
      "rewards/margins": 20.51372528076172,
      "rewards/real": 12.37287425994873,
      "step": 730
    },
    {
      "epoch": 2.368,
      "grad_norm": 2.470062940286456,
      "learning_rate": 1.163895486935867e-07,
      "logits/generated": -2.342379093170166,
      "logits/real": -2.342834949493408,
      "logps/generated": -207.5922393798828,
      "logps/real": -26.875713348388672,
      "loss": 0.0328,
      "rewards/accuracies": 0.987500011920929,
      "rewards/generated": -7.883567810058594,
      "rewards/margins": 20.540231704711914,
      "rewards/real": 12.656661987304688,
      "step": 740
    },
    {
      "epoch": 2.3808,
      "eval_logits/generated": -2.207951068878174,
      "eval_logits/real": -2.395209550857544,
      "eval_logps/generated": -136.21505737304688,
      "eval_logps/real": -160.9390411376953,
      "eval_loss": 1.183669090270996,
      "eval_rewards/accuracies": 0.48076921701431274,
      "eval_rewards/generated": -2.770172119140625,
      "eval_rewards/margins": 0.316792368888855,
      "eval_rewards/real": -2.4533796310424805,
      "eval_runtime": 24.3566,
      "eval_samples_per_second": 8.211,
      "eval_steps_per_second": 0.534,
      "step": 744
    },
    {
      "epoch": 2.4,
      "grad_norm": 2.0406895368605653,
      "learning_rate": 1.1045130641330165e-07,
      "logits/generated": -2.338466167449951,
      "logits/real": -2.2994284629821777,
      "logps/generated": -220.8417510986328,
      "logps/real": -22.894880294799805,
      "loss": 0.0269,
      "rewards/accuracies": 0.987500011920929,
      "rewards/generated": -8.605572700500488,
      "rewards/margins": 20.708206176757812,
      "rewards/real": 12.102633476257324,
      "step": 750
    },
    {
      "epoch": 2.432,
      "grad_norm": 22.669827351080848,
      "learning_rate": 1.0451306413301662e-07,
      "logits/generated": -2.42244291305542,
      "logits/real": -2.315642833709717,
      "logps/generated": -221.71664428710938,
      "logps/real": -19.839298248291016,
      "loss": 0.0358,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/generated": -8.81576919555664,
      "rewards/margins": 21.98428726196289,
      "rewards/real": 13.168516159057617,
      "step": 760
    },
    {
      "epoch": 2.464,
      "grad_norm": 2.524402783352166,
      "learning_rate": 9.857482185273158e-08,
      "logits/generated": -2.350756883621216,
      "logits/real": -2.3330166339874268,
      "logps/generated": -215.21243286132812,
      "logps/real": -17.460187911987305,
      "loss": 0.0367,
      "rewards/accuracies": 0.987500011920929,
      "rewards/generated": -7.874646186828613,
      "rewards/margins": 21.986469268798828,
      "rewards/real": 14.111824035644531,
      "step": 770
    },
    {
      "epoch": 2.48,
      "eval_logits/generated": -2.1815290451049805,
      "eval_logits/real": -2.3776841163635254,
      "eval_logps/generated": -138.23756408691406,
      "eval_logps/real": -162.54420471191406,
      "eval_loss": 1.1778903007507324,
      "eval_rewards/accuracies": 0.48076921701431274,
      "eval_rewards/generated": -2.972423791885376,
      "eval_rewards/margins": 0.35853010416030884,
      "eval_rewards/real": -2.613893747329712,
      "eval_runtime": 25.6643,
      "eval_samples_per_second": 7.793,
      "eval_steps_per_second": 0.507,
      "step": 775
    },
    {
      "epoch": 2.496,
      "grad_norm": 2.40420451796189,
      "learning_rate": 9.263657957244655e-08,
      "logits/generated": -2.2569501399993896,
      "logits/real": -2.264871120452881,
      "logps/generated": -202.81716918945312,
      "logps/real": -24.641332626342773,
      "loss": 0.0486,
      "rewards/accuracies": 0.987500011920929,
      "rewards/generated": -7.750497341156006,
      "rewards/margins": 20.739505767822266,
      "rewards/real": 12.989007949829102,
      "step": 780
    },
    {
      "epoch": 2.528,
      "grad_norm": 22.160283840192612,
      "learning_rate": 8.669833729216151e-08,
      "logits/generated": -2.3489882946014404,
      "logits/real": -2.3021740913391113,
      "logps/generated": -242.10452270507812,
      "logps/real": -19.257644653320312,
      "loss": 0.0673,
      "rewards/accuracies": 1.0,
      "rewards/generated": -9.988310813903809,
      "rewards/margins": 23.60817527770996,
      "rewards/real": 13.619865417480469,
      "step": 790
    },
    {
      "epoch": 2.56,
      "grad_norm": 2.113351859691927,
      "learning_rate": 8.076009501187649e-08,
      "logits/generated": -2.2851662635803223,
      "logits/real": -2.278385639190674,
      "logps/generated": -215.65634155273438,
      "logps/real": -17.226621627807617,
      "loss": 0.0596,
      "rewards/accuracies": 1.0,
      "rewards/generated": -9.116616249084473,
      "rewards/margins": 22.34113311767578,
      "rewards/real": 13.224515914916992,
      "step": 800
    },
    {
      "epoch": 2.5792,
      "eval_logits/generated": -2.1523475646972656,
      "eval_logits/real": -2.3457515239715576,
      "eval_logps/generated": -144.71934509277344,
      "eval_logps/real": -169.895263671875,
      "eval_loss": 1.284745454788208,
      "eval_rewards/accuracies": 0.42307692766189575,
      "eval_rewards/generated": -3.6206037998199463,
      "eval_rewards/margins": 0.2716017961502075,
      "eval_rewards/real": -3.3490025997161865,
      "eval_runtime": 25.4228,
      "eval_samples_per_second": 7.867,
      "eval_steps_per_second": 0.511,
      "step": 806
    },
    {
      "epoch": 2.592,
      "grad_norm": 11.216326120395912,
      "learning_rate": 7.482185273159145e-08,
      "logits/generated": -2.292395830154419,
      "logits/real": -2.2245354652404785,
      "logps/generated": -216.6605682373047,
      "logps/real": -17.53030776977539,
      "loss": 0.0354,
      "rewards/accuracies": 1.0,
      "rewards/generated": -9.128471374511719,
      "rewards/margins": 21.153453826904297,
      "rewards/real": 12.024984359741211,
      "step": 810
    },
    {
      "epoch": 2.624,
      "grad_norm": 2.0837019278053304,
      "learning_rate": 6.88836104513064e-08,
      "logits/generated": -2.2674856185913086,
      "logits/real": -2.246483325958252,
      "logps/generated": -235.4110107421875,
      "logps/real": -24.103424072265625,
      "loss": 0.035,
      "rewards/accuracies": 0.987500011920929,
      "rewards/generated": -9.786630630493164,
      "rewards/margins": 22.674787521362305,
      "rewards/real": 12.888157844543457,
      "step": 820
    },
    {
      "epoch": 2.656,
      "grad_norm": 3.9512882624496712,
      "learning_rate": 6.294536817102138e-08,
      "logits/generated": -2.252331256866455,
      "logits/real": -2.2256457805633545,
      "logps/generated": -202.04873657226562,
      "logps/real": -9.093053817749023,
      "loss": 0.0395,
      "rewards/accuracies": 0.987500011920929,
      "rewards/generated": -8.500150680541992,
      "rewards/margins": 19.9840087890625,
      "rewards/real": 11.483858108520508,
      "step": 830
    },
    {
      "epoch": 2.6784,
      "eval_logits/generated": -2.139922618865967,
      "eval_logits/real": -2.3346006870269775,
      "eval_logps/generated": -147.52369689941406,
      "eval_logps/real": -172.99374389648438,
      "eval_loss": 1.3357782363891602,
      "eval_rewards/accuracies": 0.42307692766189575,
      "eval_rewards/generated": -3.9010369777679443,
      "eval_rewards/margins": 0.24218764901161194,
      "eval_rewards/real": -3.6588494777679443,
      "eval_runtime": 25.3471,
      "eval_samples_per_second": 7.89,
      "eval_steps_per_second": 0.513,
      "step": 837
    },
    {
      "epoch": 2.6879999999999997,
      "grad_norm": 22.1387966335112,
      "learning_rate": 5.700712589073634e-08,
      "logits/generated": -2.3124189376831055,
      "logits/real": -2.2206485271453857,
      "logps/generated": -229.35366821289062,
      "logps/real": -10.505355834960938,
      "loss": 0.0201,
      "rewards/accuracies": 1.0,
      "rewards/generated": -10.045392036437988,
      "rewards/margins": 22.45701026916504,
      "rewards/real": 12.41161823272705,
      "step": 840
    },
    {
      "epoch": 2.7199999999999998,
      "grad_norm": 1.6992001757462196,
      "learning_rate": 5.10688836104513e-08,
      "logits/generated": -2.196502447128296,
      "logits/real": -2.26035737991333,
      "logps/generated": -206.0448760986328,
      "logps/real": -13.079363822937012,
      "loss": 0.0262,
      "rewards/accuracies": 1.0,
      "rewards/generated": -8.79426383972168,
      "rewards/margins": 21.76181411743164,
      "rewards/real": 12.967552185058594,
      "step": 850
    },
    {
      "epoch": 2.752,
      "grad_norm": 2.6474508823777674,
      "learning_rate": 4.5130641330166267e-08,
      "logits/generated": -2.3978683948516846,
      "logits/real": -2.248363494873047,
      "logps/generated": -228.8793487548828,
      "logps/real": -19.478151321411133,
      "loss": 0.0302,
      "rewards/accuracies": 1.0,
      "rewards/generated": -10.040186882019043,
      "rewards/margins": 23.719615936279297,
      "rewards/real": 13.679426193237305,
      "step": 860
    },
    {
      "epoch": 2.7776,
      "eval_logits/generated": -2.152860403060913,
      "eval_logits/real": -2.347489833831787,
      "eval_logps/generated": -148.89895629882812,
      "eval_logps/real": -174.31674194335938,
      "eval_loss": 1.3724898099899292,
      "eval_rewards/accuracies": 0.42307692766189575,
      "eval_rewards/generated": -4.038564205169678,
      "eval_rewards/margins": 0.24741561710834503,
      "eval_rewards/real": -3.7911486625671387,
      "eval_runtime": 24.3356,
      "eval_samples_per_second": 8.218,
      "eval_steps_per_second": 0.534,
      "step": 868
    },
    {
      "epoch": 2.784,
      "grad_norm": 1.7441479405692193,
      "learning_rate": 3.919239904988123e-08,
      "logits/generated": -2.349801778793335,
      "logits/real": -2.2702341079711914,
      "logps/generated": -234.3719940185547,
      "logps/real": -14.089096069335938,
      "loss": 0.0369,
      "rewards/accuracies": 0.987500011920929,
      "rewards/generated": -10.196078300476074,
      "rewards/margins": 22.43720245361328,
      "rewards/real": 12.241125106811523,
      "step": 870
    },
    {
      "epoch": 2.816,
      "grad_norm": 2.2386762278529777,
      "learning_rate": 3.32541567695962e-08,
      "logits/generated": -2.2697014808654785,
      "logits/real": -2.2320122718811035,
      "logps/generated": -235.64669799804688,
      "logps/real": -9.42314338684082,
      "loss": 0.0392,
      "rewards/accuracies": 1.0,
      "rewards/generated": -10.039624214172363,
      "rewards/margins": 22.644527435302734,
      "rewards/real": 12.604903221130371,
      "step": 880
    },
    {
      "epoch": 2.848,
      "grad_norm": 1.5468486075007237,
      "learning_rate": 2.7315914489311164e-08,
      "logits/generated": -2.2128958702087402,
      "logits/real": -2.260349988937378,
      "logps/generated": -205.9203643798828,
      "logps/real": -14.784475326538086,
      "loss": 0.0132,
      "rewards/accuracies": 1.0,
      "rewards/generated": -9.317214012145996,
      "rewards/margins": 22.827068328857422,
      "rewards/real": 13.509855270385742,
      "step": 890
    },
    {
      "epoch": 2.8768000000000002,
      "eval_logits/generated": -2.1226818561553955,
      "eval_logits/real": -2.317802667617798,
      "eval_logps/generated": -154.74986267089844,
      "eval_logps/real": -181.03439331054688,
      "eval_loss": 1.4968535900115967,
      "eval_rewards/accuracies": 0.4423076808452606,
      "eval_rewards/generated": -4.623655319213867,
      "eval_rewards/margins": 0.16074196994304657,
      "eval_rewards/real": -4.462913513183594,
      "eval_runtime": 25.3278,
      "eval_samples_per_second": 7.896,
      "eval_steps_per_second": 0.513,
      "step": 899
    },
    {
      "epoch": 2.88,
      "grad_norm": 5.589462535052379,
      "learning_rate": 2.1377672209026125e-08,
      "logits/generated": -2.1872811317443848,
      "logits/real": -2.190425395965576,
      "logps/generated": -222.4118194580078,
      "logps/real": -12.531976699829102,
      "loss": 0.0144,
      "rewards/accuracies": 1.0,
      "rewards/generated": -10.07731819152832,
      "rewards/margins": 22.0319881439209,
      "rewards/real": 11.954668045043945,
      "step": 900
    },
    {
      "epoch": 2.912,
      "grad_norm": 1.1002291429170004,
      "learning_rate": 1.5439429928741092e-08,
      "logits/generated": -2.267789363861084,
      "logits/real": -2.174405574798584,
      "logps/generated": -258.24346923828125,
      "logps/real": -17.154361724853516,
      "loss": 0.0169,
      "rewards/accuracies": 1.0,
      "rewards/generated": -11.40230655670166,
      "rewards/margins": 25.64777183532715,
      "rewards/real": 14.245463371276855,
      "step": 910
    },
    {
      "epoch": 2.944,
      "grad_norm": 2.533629319876756,
      "learning_rate": 9.501187648456057e-09,
      "logits/generated": -2.2727675437927246,
      "logits/real": -2.2528767585754395,
      "logps/generated": -228.3806915283203,
      "logps/real": -12.532684326171875,
      "loss": 0.029,
      "rewards/accuracies": 1.0,
      "rewards/generated": -10.578569412231445,
      "rewards/margins": 23.38239097595215,
      "rewards/real": 12.80382251739502,
      "step": 920
    },
    {
      "epoch": 2.976,
      "grad_norm": 22.025786598083638,
      "learning_rate": 3.562945368171021e-09,
      "logits/generated": -2.2085366249084473,
      "logits/real": -2.227689743041992,
      "logps/generated": -219.00088500976562,
      "logps/real": -16.957204818725586,
      "loss": 0.034,
      "rewards/accuracies": 0.987500011920929,
      "rewards/generated": -9.949075698852539,
      "rewards/margins": 22.433155059814453,
      "rewards/real": 12.484078407287598,
      "step": 930
    },
    {
      "epoch": 2.976,
      "eval_logits/generated": -2.120302200317383,
      "eval_logits/real": -2.3164217472076416,
      "eval_logps/generated": -155.21237182617188,
      "eval_logps/real": -181.53463745117188,
      "eval_loss": 1.4922109842300415,
      "eval_rewards/accuracies": 0.4423076808452606,
      "eval_rewards/generated": -4.669904708862305,
      "eval_rewards/margins": 0.15696734189987183,
      "eval_rewards/real": -4.512937545776367,
      "eval_runtime": 25.074,
      "eval_samples_per_second": 7.976,
      "eval_steps_per_second": 0.518,
      "step": 930
    },
    {
      "epoch": 2.9952,
      "step": 936,
      "total_flos": 0.0,
      "train_loss": 0.16162516827830392,
      "train_runtime": 16659.2082,
      "train_samples_per_second": 1.8,
      "train_steps_per_second": 0.056
    }
  ],
  "logging_steps": 10,
  "max_steps": 936,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 3,
  "save_steps": 31,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": true
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 4,
  "trial_name": null,
  "trial_params": null
}