{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 1.0,
  "eval_steps": 100,
  "global_step": 1563,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.0,
      "learning_rate": 3.1847133757961784e-09,
      "logits/generated": -2.9043595790863037,
      "logits/real": -3.0318443775177,
      "logps/generated": -95.60806274414062,
      "logps/real": -162.11865234375,
      "loss": 0.6931,
      "rewards/accuracies": 0.0,
      "rewards/generated": 0.0,
      "rewards/margins": 0.0,
      "rewards/real": 0.0,
      "step": 1
    },
    {
      "epoch": 0.01,
      "learning_rate": 3.184713375796178e-08,
      "logits/generated": -2.9496052265167236,
      "logits/real": -2.9726288318634033,
      "logps/generated": -115.16079711914062,
      "logps/real": -126.27457427978516,
      "loss": 0.6927,
      "rewards/accuracies": 0.4305555522441864,
      "rewards/generated": -0.0005613141693174839,
      "rewards/margins": 0.0011329095577821136,
      "rewards/real": 0.0005715956795029342,
      "step": 10
    },
    {
      "epoch": 0.01,
      "learning_rate": 6.369426751592356e-08,
      "logits/generated": -2.925844192504883,
      "logits/real": -3.0148582458496094,
      "logps/generated": -126.58709716796875,
      "logps/real": -159.16970825195312,
      "loss": 0.6895,
      "rewards/accuracies": 0.550000011920929,
      "rewards/generated": 0.0016155906487256289,
      "rewards/margins": 0.008952249772846699,
      "rewards/real": 0.010567841120064259,
      "step": 20
    },
    {
      "epoch": 0.02,
      "learning_rate": 9.554140127388536e-08,
      "logits/generated": -2.950205087661743,
      "logits/real": -2.9246954917907715,
      "logps/generated": -130.9630126953125,
      "logps/real": -128.04364013671875,
      "loss": 0.6701,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/generated": -0.04806157201528549,
      "rewards/margins": 0.03640877455472946,
      "rewards/real": -0.01165279932320118,
      "step": 30
    },
    {
      "epoch": 0.03,
      "learning_rate": 1.2738853503184713e-07,
      "logits/generated": -2.9109079837799072,
      "logits/real": -2.967057704925537,
      "logps/generated": -126.64833068847656,
      "logps/real": -141.5502166748047,
      "loss": 0.6662,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/generated": -0.1246229037642479,
      "rewards/margins": 0.0738920345902443,
      "rewards/real": -0.0507308654487133,
      "step": 40
    },
    {
      "epoch": 0.03,
      "learning_rate": 1.592356687898089e-07,
      "logits/generated": -2.944676160812378,
      "logits/real": -2.973489284515381,
      "logps/generated": -133.49766540527344,
      "logps/real": -141.7188262939453,
      "loss": 0.6462,
      "rewards/accuracies": 0.637499988079071,
      "rewards/generated": -0.1534460484981537,
      "rewards/margins": 0.09793341904878616,
      "rewards/real": -0.05551263689994812,
      "step": 50
    },
    {
      "epoch": 0.04,
      "learning_rate": 1.9108280254777072e-07,
      "logits/generated": -2.92313814163208,
      "logits/real": -2.9491844177246094,
      "logps/generated": -123.59134674072266,
      "logps/real": -135.57339477539062,
      "loss": 0.6571,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/generated": -0.2681542634963989,
      "rewards/margins": 0.09540877491235733,
      "rewards/real": -0.172745481133461,
      "step": 60
    },
    {
      "epoch": 0.04,
      "learning_rate": 2.2292993630573247e-07,
      "logits/generated": -2.9296579360961914,
      "logits/real": -2.9948348999023438,
      "logps/generated": -124.77592468261719,
      "logps/real": -141.3092041015625,
      "loss": 0.6272,
      "rewards/accuracies": 0.699999988079071,
      "rewards/generated": -0.36597520112991333,
      "rewards/margins": 0.23450367152690887,
      "rewards/real": -0.13147154450416565,
      "step": 70
    },
    {
      "epoch": 0.05,
      "learning_rate": 2.5477707006369425e-07,
      "logits/generated": -2.9382827281951904,
      "logits/real": -2.950681209564209,
      "logps/generated": -126.6854476928711,
      "logps/real": -116.08744049072266,
      "loss": 0.6279,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/generated": -0.6447240114212036,
      "rewards/margins": 0.15082275867462158,
      "rewards/real": -0.49390116333961487,
      "step": 80
    },
    {
      "epoch": 0.06,
      "learning_rate": 2.86624203821656e-07,
      "logits/generated": -2.9626991748809814,
      "logits/real": -2.985078811645508,
      "logps/generated": -151.97203063964844,
      "logps/real": -147.793212890625,
      "loss": 0.5941,
      "rewards/accuracies": 0.699999988079071,
      "rewards/generated": -0.64665687084198,
      "rewards/margins": 0.40791407227516174,
      "rewards/real": -0.23874275386333466,
      "step": 90
    },
    {
      "epoch": 0.06,
      "learning_rate": 3.184713375796178e-07,
      "logits/generated": -2.9504504203796387,
      "logits/real": -3.009941577911377,
      "logps/generated": -127.89019775390625,
      "logps/real": -142.78115844726562,
      "loss": 0.6035,
      "rewards/accuracies": 0.637499988079071,
      "rewards/generated": -0.4942631125450134,
      "rewards/margins": 0.39962121844291687,
      "rewards/real": -0.09464187920093536,
      "step": 100
    },
    {
      "epoch": 0.06,
      "eval_logits/generated": -2.9112277030944824,
      "eval_logits/real": -2.963428497314453,
      "eval_logps/generated": -120.85581970214844,
      "eval_logps/real": -134.36163330078125,
      "eval_loss": 0.592494010925293,
      "eval_rewards/accuracies": 0.6871019005775452,
      "eval_rewards/generated": -0.46135851740837097,
      "eval_rewards/margins": 0.483438104391098,
      "eval_rewards/real": 0.022079583257436752,
      "eval_runtime": 320.9946,
      "eval_samples_per_second": 15.577,
      "eval_steps_per_second": 0.489,
      "step": 100
    },
    {
      "epoch": 0.07,
      "learning_rate": 3.5031847133757957e-07,
      "logits/generated": -2.94885516166687,
      "logits/real": -2.9667720794677734,
      "logps/generated": -132.24411010742188,
      "logps/real": -138.70401000976562,
      "loss": 0.5939,
      "rewards/accuracies": 0.675000011920929,
      "rewards/generated": -0.4684581756591797,
      "rewards/margins": 0.5189403891563416,
      "rewards/real": 0.050482142716646194,
      "step": 110
    },
    {
      "epoch": 0.08,
      "learning_rate": 3.8216560509554143e-07,
      "logits/generated": -2.914760112762451,
      "logits/real": -2.9669089317321777,
      "logps/generated": -131.27503967285156,
      "logps/real": -147.97238159179688,
      "loss": 0.602,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/generated": -0.41060376167297363,
      "rewards/margins": 0.492185115814209,
      "rewards/real": 0.08158136159181595,
      "step": 120
    },
    {
      "epoch": 0.08,
      "learning_rate": 4.140127388535032e-07,
      "logits/generated": -2.9110288619995117,
      "logits/real": -2.9508092403411865,
      "logps/generated": -129.82977294921875,
      "logps/real": -144.04708862304688,
      "loss": 0.655,
      "rewards/accuracies": 0.612500011920929,
      "rewards/generated": -0.6899576187133789,
      "rewards/margins": 0.29959696531295776,
      "rewards/real": -0.3903607130050659,
      "step": 130
    },
    {
      "epoch": 0.09,
      "learning_rate": 4.4585987261146494e-07,
      "logits/generated": -2.8934552669525146,
      "logits/real": -2.918970823287964,
      "logps/generated": -116.09012603759766,
      "logps/real": -121.42826080322266,
      "loss": 0.6225,
      "rewards/accuracies": 0.637499988079071,
      "rewards/generated": -0.5983424186706543,
      "rewards/margins": 0.2791465222835541,
      "rewards/real": -0.31919583678245544,
      "step": 140
    },
    {
      "epoch": 0.1,
      "learning_rate": 4.777070063694267e-07,
      "logits/generated": -2.8588805198669434,
      "logits/real": -2.905513048171997,
      "logps/generated": -123.76139068603516,
      "logps/real": -135.1915283203125,
      "loss": 0.6297,
      "rewards/accuracies": 0.6875,
      "rewards/generated": -0.826836884021759,
      "rewards/margins": 0.6088964343070984,
      "rewards/real": -0.21794041991233826,
      "step": 150
    },
    {
      "epoch": 0.1,
      "learning_rate": 4.989331436699858e-07,
      "logits/generated": -2.8500113487243652,
      "logits/real": -2.9133572578430176,
      "logps/generated": -138.90496826171875,
      "logps/real": -147.51112365722656,
      "loss": 0.5997,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/generated": -0.5559502840042114,
      "rewards/margins": 0.44744768738746643,
      "rewards/real": -0.10850252956151962,
      "step": 160
    },
    {
      "epoch": 0.11,
      "learning_rate": 4.953769559032717e-07,
      "logits/generated": -2.7749171257019043,
      "logits/real": -2.8795857429504395,
      "logps/generated": -112.4853515625,
      "logps/real": -137.61013793945312,
      "loss": 0.5703,
      "rewards/accuracies": 0.675000011920929,
      "rewards/generated": -1.144225835800171,
      "rewards/margins": 0.8240048289299011,
      "rewards/real": -0.32022103667259216,
      "step": 170
    },
    {
      "epoch": 0.12,
      "learning_rate": 4.918207681365576e-07,
      "logits/generated": -2.8241965770721436,
      "logits/real": -2.899482250213623,
      "logps/generated": -123.1147232055664,
      "logps/real": -140.4021759033203,
      "loss": 0.6273,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/generated": -0.580504298210144,
      "rewards/margins": 0.6617487668991089,
      "rewards/real": 0.08124445378780365,
      "step": 180
    },
    {
      "epoch": 0.12,
      "learning_rate": 4.882645803698435e-07,
      "logits/generated": -2.8152451515197754,
      "logits/real": -2.8969357013702393,
      "logps/generated": -118.85482025146484,
      "logps/real": -131.64207458496094,
      "loss": 0.626,
      "rewards/accuracies": 0.675000011920929,
      "rewards/generated": -0.8428341150283813,
      "rewards/margins": 0.712174654006958,
      "rewards/real": -0.13065943121910095,
      "step": 190
    },
    {
      "epoch": 0.13,
      "learning_rate": 4.847083926031294e-07,
      "logits/generated": -2.8368003368377686,
      "logits/real": -2.8934528827667236,
      "logps/generated": -124.17633056640625,
      "logps/real": -133.32460021972656,
      "loss": 0.6154,
      "rewards/accuracies": 0.625,
      "rewards/generated": -1.0640162229537964,
      "rewards/margins": 0.585637629032135,
      "rewards/real": -0.47837844491004944,
      "step": 200
    },
    {
      "epoch": 0.13,
      "eval_logits/generated": -2.846381902694702,
      "eval_logits/real": -2.8926753997802734,
      "eval_logps/generated": -124.88420867919922,
      "eval_logps/real": -137.59585571289062,
      "eval_loss": 0.5945761799812317,
      "eval_rewards/accuracies": 0.6886942386627197,
      "eval_rewards/generated": -0.8641979694366455,
      "eval_rewards/margins": 0.5628555417060852,
      "eval_rewards/real": -0.3013423681259155,
      "eval_runtime": 314.2657,
      "eval_samples_per_second": 15.91,
      "eval_steps_per_second": 0.5,
      "step": 200
    },
    {
      "epoch": 0.13,
      "learning_rate": 4.811522048364154e-07,
      "logits/generated": -2.8265631198883057,
      "logits/real": -2.872185468673706,
      "logps/generated": -116.153564453125,
      "logps/real": -129.70101928710938,
      "loss": 0.6439,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/generated": -0.9046352505683899,
      "rewards/margins": 0.5383676290512085,
      "rewards/real": -0.3662675619125366,
      "step": 210
    },
    {
      "epoch": 0.14,
      "learning_rate": 4.775960170697012e-07,
      "logits/generated": -2.8761754035949707,
      "logits/real": -2.8833699226379395,
      "logps/generated": -143.34608459472656,
      "logps/real": -140.4290771484375,
      "loss": 0.7048,
      "rewards/accuracies": 0.637499988079071,
      "rewards/generated": -0.5727646350860596,
      "rewards/margins": 0.421123743057251,
      "rewards/real": -0.1516408622264862,
      "step": 220
    },
    {
      "epoch": 0.15,
      "learning_rate": 4.7403982930298717e-07,
      "logits/generated": -2.8488383293151855,
      "logits/real": -2.86686372756958,
      "logps/generated": -124.8054428100586,
      "logps/real": -137.43142700195312,
      "loss": 0.6624,
      "rewards/accuracies": 0.675000011920929,
      "rewards/generated": -0.386758029460907,
      "rewards/margins": 0.39915674924850464,
      "rewards/real": 0.012398702092468739,
      "step": 230
    },
    {
      "epoch": 0.15,
      "learning_rate": 4.7048364153627306e-07,
      "logits/generated": -2.8755905628204346,
      "logits/real": -2.850111722946167,
      "logps/generated": -144.16363525390625,
      "logps/real": -137.38861083984375,
      "loss": 0.6973,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/generated": -0.42233890295028687,
      "rewards/margins": 0.6668346524238586,
      "rewards/real": 0.2444957047700882,
      "step": 240
    },
    {
      "epoch": 0.16,
      "learning_rate": 4.66927453769559e-07,
      "logits/generated": -2.8914976119995117,
      "logits/real": -2.909574508666992,
      "logps/generated": -134.83218383789062,
      "logps/real": -144.05287170410156,
      "loss": 0.6039,
      "rewards/accuracies": 0.699999988079071,
      "rewards/generated": -0.9402686953544617,
      "rewards/margins": 0.6465772390365601,
      "rewards/real": -0.2936915159225464,
      "step": 250
    },
    {
      "epoch": 0.17,
      "learning_rate": 4.633712660028449e-07,
      "logits/generated": -2.8130080699920654,
      "logits/real": -2.9388322830200195,
      "logps/generated": -131.2688751220703,
      "logps/real": -148.659423828125,
      "loss": 0.6259,
      "rewards/accuracies": 0.675000011920929,
      "rewards/generated": -0.9782902598381042,
      "rewards/margins": 0.6731352806091309,
      "rewards/real": -0.30515503883361816,
      "step": 260
    },
    {
      "epoch": 0.17,
      "learning_rate": 4.5981507823613085e-07,
      "logits/generated": -2.8650355339050293,
      "logits/real": -2.8830349445343018,
      "logps/generated": -133.54998779296875,
      "logps/real": -143.64295959472656,
      "loss": 0.6478,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/generated": -1.038545846939087,
      "rewards/margins": 0.5988888144493103,
      "rewards/real": -0.43965697288513184,
      "step": 270
    },
    {
      "epoch": 0.18,
      "learning_rate": 4.562588904694168e-07,
      "logits/generated": -2.8448922634124756,
      "logits/real": -2.893357515335083,
      "logps/generated": -126.46229553222656,
      "logps/real": -135.95480346679688,
      "loss": 0.6228,
      "rewards/accuracies": 0.6875,
      "rewards/generated": -1.1257703304290771,
      "rewards/margins": 0.6330925822257996,
      "rewards/real": -0.4926776885986328,
      "step": 280
    },
    {
      "epoch": 0.19,
      "learning_rate": 4.5270270270270264e-07,
      "logits/generated": -2.8617606163024902,
      "logits/real": -2.907224416732788,
      "logps/generated": -127.72086334228516,
      "logps/real": -138.37338256835938,
      "loss": 0.6585,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/generated": -0.6692047119140625,
      "rewards/margins": 0.41642865538597107,
      "rewards/real": -0.2527760863304138,
      "step": 290
    },
    {
      "epoch": 0.19,
      "learning_rate": 4.491465149359886e-07,
      "logits/generated": -2.8157684803009033,
      "logits/real": -2.9448578357696533,
      "logps/generated": -115.7210464477539,
      "logps/real": -135.6936798095703,
      "loss": 0.7218,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/generated": -0.04884980991482735,
      "rewards/margins": 0.3899938464164734,
      "rewards/real": 0.34114405512809753,
      "step": 300
    },
    {
      "epoch": 0.19,
      "eval_logits/generated": -2.8557660579681396,
      "eval_logits/real": -2.938932180404663,
      "eval_logps/generated": -117.34172058105469,
      "eval_logps/real": -130.4615478515625,
      "eval_loss": 0.6092967987060547,
      "eval_rewards/accuracies": 0.6791401505470276,
      "eval_rewards/generated": -0.10995037853717804,
      "eval_rewards/margins": 0.5220382809638977,
      "eval_rewards/real": 0.41208788752555847,
      "eval_runtime": 313.617,
      "eval_samples_per_second": 15.943,
      "eval_steps_per_second": 0.501,
      "step": 300
    },
    {
      "epoch": 0.2,
      "learning_rate": 4.4559032716927454e-07,
      "logits/generated": -2.820242404937744,
      "logits/real": -2.932029962539673,
      "logps/generated": -110.3979721069336,
      "logps/real": -128.57955932617188,
      "loss": 0.7051,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/generated": -0.46763068437576294,
      "rewards/margins": 0.7681630849838257,
      "rewards/real": 0.3005323112010956,
      "step": 310
    },
    {
      "epoch": 0.2,
      "learning_rate": 4.420341394025605e-07,
      "logits/generated": -2.867724895477295,
      "logits/real": -2.9374239444732666,
      "logps/generated": -121.03181457519531,
      "logps/real": -131.03138732910156,
      "loss": 0.613,
      "rewards/accuracies": 0.6875,
      "rewards/generated": -0.9018260836601257,
      "rewards/margins": 0.7947210073471069,
      "rewards/real": -0.10710500180721283,
      "step": 320
    },
    {
      "epoch": 0.21,
      "learning_rate": 4.384779516358463e-07,
      "logits/generated": -2.9186887741088867,
      "logits/real": -2.9603030681610107,
      "logps/generated": -136.2305145263672,
      "logps/real": -141.52938842773438,
      "loss": 0.7222,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/generated": -1.0479620695114136,
      "rewards/margins": 0.24121908843517303,
      "rewards/real": -0.8067429661750793,
      "step": 330
    },
    {
      "epoch": 0.22,
      "learning_rate": 4.3492176386913227e-07,
      "logits/generated": -2.884328603744507,
      "logits/real": -2.9852678775787354,
      "logps/generated": -134.2027130126953,
      "logps/real": -157.81533813476562,
      "loss": 0.6392,
      "rewards/accuracies": 0.6875,
      "rewards/generated": -1.0355875492095947,
      "rewards/margins": 0.5947176218032837,
      "rewards/real": -0.44086989760398865,
      "step": 340
    },
    {
      "epoch": 0.22,
      "learning_rate": 4.313655761024182e-07,
      "logits/generated": -2.9303040504455566,
      "logits/real": -2.9080405235290527,
      "logps/generated": -133.62733459472656,
      "logps/real": -135.67965698242188,
      "loss": 0.7259,
      "rewards/accuracies": 0.675000011920929,
      "rewards/generated": -0.7619242072105408,
      "rewards/margins": 0.7297625541687012,
      "rewards/real": -0.03216175362467766,
      "step": 350
    },
    {
      "epoch": 0.23,
      "learning_rate": 4.278093883357041e-07,
      "logits/generated": -2.907813549041748,
      "logits/real": -2.8927502632141113,
      "logps/generated": -135.63803100585938,
      "logps/real": -127.85164642333984,
      "loss": 0.6128,
      "rewards/accuracies": 0.675000011920929,
      "rewards/generated": -0.8036858439445496,
      "rewards/margins": 0.7827543020248413,
      "rewards/real": -0.020931486040353775,
      "step": 360
    },
    {
      "epoch": 0.24,
      "learning_rate": 4.2425320056899e-07,
      "logits/generated": -2.8933119773864746,
      "logits/real": -2.90037202835083,
      "logps/generated": -123.38737487792969,
      "logps/real": -129.03187561035156,
      "loss": 0.6147,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/generated": -0.838703453540802,
      "rewards/margins": 0.6072388291358948,
      "rewards/real": -0.2314646691083908,
      "step": 370
    },
    {
      "epoch": 0.24,
      "learning_rate": 4.2069701280227595e-07,
      "logits/generated": -2.8591134548187256,
      "logits/real": -2.924983263015747,
      "logps/generated": -133.0351104736328,
      "logps/real": -132.64010620117188,
      "loss": 0.6723,
      "rewards/accuracies": 0.574999988079071,
      "rewards/generated": -0.3996178209781647,
      "rewards/margins": 0.408294141292572,
      "rewards/real": 0.008676335215568542,
      "step": 380
    },
    {
      "epoch": 0.25,
      "learning_rate": 4.1714082503556185e-07,
      "logits/generated": -2.829285144805908,
      "logits/real": -2.9132728576660156,
      "logps/generated": -115.08140563964844,
      "logps/real": -116.62813568115234,
      "loss": 0.6199,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/generated": -0.39587754011154175,
      "rewards/margins": 0.7312180995941162,
      "rewards/real": 0.33534055948257446,
      "step": 390
    },
    {
      "epoch": 0.26,
      "learning_rate": 4.135846372688478e-07,
      "logits/generated": -2.896308422088623,
      "logits/real": -2.9679720401763916,
      "logps/generated": -134.25875854492188,
      "logps/real": -145.1576385498047,
      "loss": 0.6415,
      "rewards/accuracies": 0.6875,
      "rewards/generated": -0.8689187169075012,
      "rewards/margins": 0.8677636981010437,
      "rewards/real": -0.0011550008784979582,
      "step": 400
    },
    {
      "epoch": 0.26,
      "eval_logits/generated": -2.863023519515991,
      "eval_logits/real": -2.934311866760254,
      "eval_logps/generated": -123.05078125,
      "eval_logps/real": -134.793212890625,
      "eval_loss": 0.6002771258354187,
      "eval_rewards/accuracies": 0.6942675113677979,
      "eval_rewards/generated": -0.6808551549911499,
      "eval_rewards/margins": 0.6597769260406494,
      "eval_rewards/real": -0.0210782028734684,
      "eval_runtime": 312.5846,
      "eval_samples_per_second": 15.996,
      "eval_steps_per_second": 0.502,
      "step": 400
    },
    {
      "epoch": 0.26,
      "learning_rate": 4.100284495021337e-07,
      "logits/generated": -2.9025797843933105,
      "logits/real": -2.8970775604248047,
      "logps/generated": -138.23733520507812,
      "logps/real": -131.90760803222656,
      "loss": 0.6279,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/generated": -0.9198619723320007,
      "rewards/margins": 0.8169810175895691,
      "rewards/real": -0.10288095474243164,
      "step": 410
    },
    {
      "epoch": 0.27,
      "learning_rate": 4.064722617354196e-07,
      "logits/generated": -2.9897818565368652,
      "logits/real": -2.8572428226470947,
      "logps/generated": -155.53582763671875,
      "logps/real": -140.32861328125,
      "loss": 0.7277,
      "rewards/accuracies": 0.637499988079071,
      "rewards/generated": -0.9986831545829773,
      "rewards/margins": 0.6315004825592041,
      "rewards/real": -0.3671826422214508,
      "step": 420
    },
    {
      "epoch": 0.28,
      "learning_rate": 4.0291607396870553e-07,
      "logits/generated": -2.8571627140045166,
      "logits/real": -2.9783432483673096,
      "logps/generated": -129.95596313476562,
      "logps/real": -152.39735412597656,
      "loss": 0.7234,
      "rewards/accuracies": 0.612500011920929,
      "rewards/generated": -0.7651321887969971,
      "rewards/margins": 0.3509063124656677,
      "rewards/real": -0.41422587633132935,
      "step": 430
    },
    {
      "epoch": 0.28,
      "learning_rate": 3.993598862019915e-07,
      "logits/generated": -2.90661883354187,
      "logits/real": -2.907402515411377,
      "logps/generated": -126.6184310913086,
      "logps/real": -135.4339599609375,
      "loss": 0.5952,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/generated": -0.9488459825515747,
      "rewards/margins": 0.9613292813301086,
      "rewards/real": 0.012483302503824234,
      "step": 440
    },
    {
      "epoch": 0.29,
      "learning_rate": 3.9580369843527737e-07,
      "logits/generated": -2.8979873657226562,
      "logits/real": -2.978975296020508,
      "logps/generated": -144.71261596679688,
      "logps/real": -157.7671661376953,
      "loss": 0.6472,
      "rewards/accuracies": 0.6875,
      "rewards/generated": -0.828133225440979,
      "rewards/margins": 0.6528965830802917,
      "rewards/real": -0.17523646354675293,
      "step": 450
    },
    {
      "epoch": 0.29,
      "learning_rate": 3.9224751066856327e-07,
      "logits/generated": -2.9304752349853516,
      "logits/real": -2.9112486839294434,
      "logps/generated": -150.48779296875,
      "logps/real": -142.08993530273438,
      "loss": 0.617,
      "rewards/accuracies": 0.6875,
      "rewards/generated": -1.05068039894104,
      "rewards/margins": 0.7404968738555908,
      "rewards/real": -0.31018346548080444,
      "step": 460
    },
    {
      "epoch": 0.3,
      "learning_rate": 3.886913229018492e-07,
      "logits/generated": -2.833413600921631,
      "logits/real": -2.927903652191162,
      "logps/generated": -137.7316436767578,
      "logps/real": -149.27978515625,
      "loss": 0.7084,
      "rewards/accuracies": 0.637499988079071,
      "rewards/generated": -1.6642704010009766,
      "rewards/margins": 0.5741457939147949,
      "rewards/real": -1.090124487876892,
      "step": 470
    },
    {
      "epoch": 0.31,
      "learning_rate": 3.851351351351351e-07,
      "logits/generated": -2.876814126968384,
      "logits/real": -2.9213833808898926,
      "logps/generated": -137.88467407226562,
      "logps/real": -141.27908325195312,
      "loss": 0.6283,
      "rewards/accuracies": 0.75,
      "rewards/generated": -1.994659423828125,
      "rewards/margins": 0.8866125345230103,
      "rewards/real": -1.1080471277236938,
      "step": 480
    },
    {
      "epoch": 0.31,
      "learning_rate": 3.8157894736842105e-07,
      "logits/generated": -2.867941379547119,
      "logits/real": -2.907710075378418,
      "logps/generated": -148.14093017578125,
      "logps/real": -143.94232177734375,
      "loss": 0.6579,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/generated": -1.7410551309585571,
      "rewards/margins": 1.0998215675354004,
      "rewards/real": -0.6412338018417358,
      "step": 490
    },
    {
      "epoch": 0.32,
      "learning_rate": 3.7802275960170695e-07,
      "logits/generated": -2.893672466278076,
      "logits/real": -2.9028027057647705,
      "logps/generated": -145.03538513183594,
      "logps/real": -139.4706268310547,
      "loss": 0.6876,
      "rewards/accuracies": 0.625,
      "rewards/generated": -1.2101993560791016,
      "rewards/margins": 0.6310919523239136,
      "rewards/real": -0.5791072249412537,
      "step": 500
    },
    {
      "epoch": 0.32,
      "eval_logits/generated": -2.836423873901367,
      "eval_logits/real": -2.910181999206543,
      "eval_logps/generated": -130.0467987060547,
      "eval_logps/real": -141.91192626953125,
      "eval_loss": 0.6061537861824036,
      "eval_rewards/accuracies": 0.6727706789970398,
      "eval_rewards/generated": -1.3804560899734497,
      "eval_rewards/margins": 0.6475077271461487,
      "eval_rewards/real": -0.7329482436180115,
      "eval_runtime": 312.1361,
      "eval_samples_per_second": 16.019,
      "eval_steps_per_second": 0.503,
      "step": 500
    },
    {
      "epoch": 0.33,
      "learning_rate": 3.7446657183499284e-07,
      "logits/generated": -2.8054213523864746,
      "logits/real": -2.8959434032440186,
      "logps/generated": -137.50283813476562,
      "logps/real": -146.76283264160156,
      "loss": 0.6911,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/generated": -1.2926948070526123,
      "rewards/margins": 0.6764450669288635,
      "rewards/real": -0.6162497997283936,
      "step": 510
    },
    {
      "epoch": 0.33,
      "learning_rate": 3.709103840682788e-07,
      "logits/generated": -2.8815009593963623,
      "logits/real": -2.8959944248199463,
      "logps/generated": -139.5189666748047,
      "logps/real": -146.14120483398438,
      "loss": 0.6903,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/generated": -1.8085136413574219,
      "rewards/margins": 0.8203228712081909,
      "rewards/real": -0.988190770149231,
      "step": 520
    },
    {
      "epoch": 0.34,
      "learning_rate": 3.6735419630156474e-07,
      "logits/generated": -2.8573758602142334,
      "logits/real": -2.890807628631592,
      "logps/generated": -132.2734375,
      "logps/real": -136.5230255126953,
      "loss": 0.5943,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/generated": -1.7565529346466064,
      "rewards/margins": 0.6700723767280579,
      "rewards/real": -1.0864803791046143,
      "step": 530
    },
    {
      "epoch": 0.35,
      "learning_rate": 3.637980085348506e-07,
      "logits/generated": -2.7691547870635986,
      "logits/real": -2.9011363983154297,
      "logps/generated": -123.21565246582031,
      "logps/real": -140.19796752929688,
      "loss": 0.6908,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/generated": -1.4617164134979248,
      "rewards/margins": 0.5445016026496887,
      "rewards/real": -0.9172147512435913,
      "step": 540
    },
    {
      "epoch": 0.35,
      "learning_rate": 3.602418207681365e-07,
      "logits/generated": -2.863327980041504,
      "logits/real": -2.8315365314483643,
      "logps/generated": -139.07894897460938,
      "logps/real": -139.32464599609375,
      "loss": 0.7007,
      "rewards/accuracies": 0.637499988079071,
      "rewards/generated": -1.1674206256866455,
      "rewards/margins": 0.35725370049476624,
      "rewards/real": -0.8101669549942017,
      "step": 550
    },
    {
      "epoch": 0.36,
      "learning_rate": 3.5668563300142247e-07,
      "logits/generated": -2.84228777885437,
      "logits/real": -2.9253830909729004,
      "logps/generated": -138.79415893554688,
      "logps/real": -131.2203826904297,
      "loss": 0.6865,
      "rewards/accuracies": 0.6875,
      "rewards/generated": -1.115051507949829,
      "rewards/margins": 1.0375287532806396,
      "rewards/real": -0.07752259820699692,
      "step": 560
    },
    {
      "epoch": 0.36,
      "learning_rate": 3.5312944523470837e-07,
      "logits/generated": -2.8042409420013428,
      "logits/real": -2.951826810836792,
      "logps/generated": -134.45884704589844,
      "logps/real": -145.27365112304688,
      "loss": 0.6188,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/generated": -1.4238983392715454,
      "rewards/margins": 0.7379240393638611,
      "rewards/real": -0.6859742999076843,
      "step": 570
    },
    {
      "epoch": 0.37,
      "learning_rate": 3.495732574679943e-07,
      "logits/generated": -2.7836008071899414,
      "logits/real": -2.9870457649230957,
      "logps/generated": -135.55392456054688,
      "logps/real": -158.0802459716797,
      "loss": 0.6584,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/generated": -1.674433946609497,
      "rewards/margins": 0.6915053129196167,
      "rewards/real": -0.982928454875946,
      "step": 580
    },
    {
      "epoch": 0.38,
      "learning_rate": 3.460170697012802e-07,
      "logits/generated": -2.854656219482422,
      "logits/real": -2.8894591331481934,
      "logps/generated": -128.78138732910156,
      "logps/real": -123.89408874511719,
      "loss": 0.6294,
      "rewards/accuracies": 0.6875,
      "rewards/generated": -1.5515716075897217,
      "rewards/margins": 0.5374225378036499,
      "rewards/real": -1.0141491889953613,
      "step": 590
    },
    {
      "epoch": 0.38,
      "learning_rate": 3.424608819345661e-07,
      "logits/generated": -2.8014330863952637,
      "logits/real": -2.9254088401794434,
      "logps/generated": -132.14515686035156,
      "logps/real": -152.46603393554688,
      "loss": 0.5849,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/generated": -2.0465548038482666,
      "rewards/margins": 1.0699055194854736,
      "rewards/real": -0.976649284362793,
      "step": 600
    },
    {
      "epoch": 0.38,
      "eval_logits/generated": -2.8346359729766846,
      "eval_logits/real": -2.925360918045044,
      "eval_logps/generated": -134.65505981445312,
      "eval_logps/real": -144.80482482910156,
      "eval_loss": 0.6078768372535706,
      "eval_rewards/accuracies": 0.6831210255622864,
      "eval_rewards/generated": -1.8412829637527466,
      "eval_rewards/margins": 0.8190432786941528,
      "eval_rewards/real": -1.0222396850585938,
      "eval_runtime": 309.6541,
      "eval_samples_per_second": 16.147,
      "eval_steps_per_second": 0.507,
      "step": 600
    },
    {
      "epoch": 0.39,
      "learning_rate": 3.3890469416785205e-07,
      "logits/generated": -2.8233141899108887,
      "logits/real": -2.9481401443481445,
      "logps/generated": -137.22264099121094,
      "logps/real": -152.15045166015625,
      "loss": 0.6745,
      "rewards/accuracies": 0.625,
      "rewards/generated": -1.7551801204681396,
      "rewards/margins": 0.62352454662323,
      "rewards/real": -1.1316556930541992,
      "step": 610
    },
    {
      "epoch": 0.4,
      "learning_rate": 3.35348506401138e-07,
      "logits/generated": -2.8389840126037598,
      "logits/real": -2.824481725692749,
      "logps/generated": -123.14152526855469,
      "logps/real": -124.35445404052734,
      "loss": 0.6733,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/generated": -1.4588735103607178,
      "rewards/margins": 0.8291076421737671,
      "rewards/real": -0.6297658681869507,
      "step": 620
    },
    {
      "epoch": 0.4,
      "learning_rate": 3.3179231863442384e-07,
      "logits/generated": -2.910172700881958,
      "logits/real": -2.9429709911346436,
      "logps/generated": -144.48809814453125,
      "logps/real": -145.07113647460938,
      "loss": 0.6497,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/generated": -1.015930414199829,
      "rewards/margins": 1.0248924493789673,
      "rewards/real": 0.008962005376815796,
      "step": 630
    },
    {
      "epoch": 0.41,
      "learning_rate": 3.282361308677098e-07,
      "logits/generated": -2.8567841053009033,
      "logits/real": -2.9329147338867188,
      "logps/generated": -131.82077026367188,
      "logps/real": -135.8273162841797,
      "loss": 0.5926,
      "rewards/accuracies": 0.737500011920929,
      "rewards/generated": -1.1058717966079712,
      "rewards/margins": 0.9077954292297363,
      "rewards/real": -0.1980762630701065,
      "step": 640
    },
    {
      "epoch": 0.42,
      "learning_rate": 3.2467994310099573e-07,
      "logits/generated": -2.872140407562256,
      "logits/real": -2.9000673294067383,
      "logps/generated": -135.03431701660156,
      "logps/real": -147.93064880371094,
      "loss": 0.6806,
      "rewards/accuracies": 0.675000011920929,
      "rewards/generated": -1.2050108909606934,
      "rewards/margins": 0.7183829545974731,
      "rewards/real": -0.4866279065608978,
      "step": 650
    },
    {
      "epoch": 0.42,
      "learning_rate": 3.211237553342817e-07,
      "logits/generated": -2.859281063079834,
      "logits/real": -2.9440739154815674,
      "logps/generated": -125.93595123291016,
      "logps/real": -129.65621948242188,
      "loss": 0.6927,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/generated": -0.940774142742157,
      "rewards/margins": 0.5372705459594727,
      "rewards/real": -0.4035035967826843,
      "step": 660
    },
    {
      "epoch": 0.43,
      "learning_rate": 3.175675675675675e-07,
      "logits/generated": -2.888960123062134,
      "logits/real": -2.9022905826568604,
      "logps/generated": -134.37814331054688,
      "logps/real": -136.43983459472656,
      "loss": 0.5954,
      "rewards/accuracies": 0.699999988079071,
      "rewards/generated": -1.0395607948303223,
      "rewards/margins": 0.8729231953620911,
      "rewards/real": -0.1666376292705536,
      "step": 670
    },
    {
      "epoch": 0.44,
      "learning_rate": 3.1401137980085347e-07,
      "logits/generated": -2.8475449085235596,
      "logits/real": -2.931452512741089,
      "logps/generated": -123.19482421875,
      "logps/real": -129.654052734375,
      "loss": 0.6247,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/generated": -0.7638888955116272,
      "rewards/margins": 0.6824823021888733,
      "rewards/real": -0.08140657842159271,
      "step": 680
    },
    {
      "epoch": 0.44,
      "learning_rate": 3.104551920341394e-07,
      "logits/generated": -2.7857539653778076,
      "logits/real": -2.9635262489318848,
      "logps/generated": -121.25040435791016,
      "logps/real": -137.65020751953125,
      "loss": 0.7093,
      "rewards/accuracies": 0.699999988079071,
      "rewards/generated": -0.8186511993408203,
      "rewards/margins": 0.8126083612442017,
      "rewards/real": -0.006042912602424622,
      "step": 690
    },
    {
      "epoch": 0.45,
      "learning_rate": 3.068990042674253e-07,
      "logits/generated": -2.8002536296844482,
      "logits/real": -2.9356870651245117,
      "logps/generated": -126.85771179199219,
      "logps/real": -136.75582885742188,
      "loss": 0.5948,
      "rewards/accuracies": 0.699999988079071,
      "rewards/generated": -1.1403062343597412,
      "rewards/margins": 0.7472094893455505,
      "rewards/real": -0.3930967450141907,
      "step": 700
    },
    {
      "epoch": 0.45,
      "eval_logits/generated": -2.840470314025879,
      "eval_logits/real": -2.9281880855560303,
      "eval_logps/generated": -127.03143310546875,
      "eval_logps/real": -138.53179931640625,
      "eval_loss": 0.6026215553283691,
      "eval_rewards/accuracies": 0.6815286874771118,
      "eval_rewards/generated": -1.0789220333099365,
      "eval_rewards/margins": 0.6839848756790161,
      "eval_rewards/real": -0.3949371576309204,
      "eval_runtime": 310.5622,
      "eval_samples_per_second": 16.1,
      "eval_steps_per_second": 0.506,
      "step": 700
    },
    {
      "epoch": 0.45,
      "learning_rate": 3.033428165007112e-07,
      "logits/generated": -2.837271213531494,
      "logits/real": -2.9325520992279053,
      "logps/generated": -140.0005645751953,
      "logps/real": -148.47731018066406,
      "loss": 0.6166,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/generated": -1.002680778503418,
      "rewards/margins": 0.6050243377685547,
      "rewards/real": -0.39765650033950806,
      "step": 710
    },
    {
      "epoch": 0.46,
      "learning_rate": 2.9978662873399715e-07,
      "logits/generated": -2.873455047607422,
      "logits/real": -2.873136520385742,
      "logps/generated": -129.134521484375,
      "logps/real": -130.46888732910156,
      "loss": 0.6923,
      "rewards/accuracies": 0.75,
      "rewards/generated": -1.0006587505340576,
      "rewards/margins": 0.7794184684753418,
      "rewards/real": -0.22124037146568298,
      "step": 720
    },
    {
      "epoch": 0.47,
      "learning_rate": 2.9623044096728305e-07,
      "logits/generated": -2.8776352405548096,
      "logits/real": -2.9416308403015137,
      "logps/generated": -126.6672592163086,
      "logps/real": -134.09780883789062,
      "loss": 0.6886,
      "rewards/accuracies": 0.625,
      "rewards/generated": -0.7552564740180969,
      "rewards/margins": 0.2777866721153259,
      "rewards/real": -0.47746968269348145,
      "step": 730
    },
    {
      "epoch": 0.47,
      "learning_rate": 2.92674253200569e-07,
      "logits/generated": -2.9088470935821533,
      "logits/real": -2.9304256439208984,
      "logps/generated": -137.3657989501953,
      "logps/real": -139.28189086914062,
      "loss": 0.5841,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/generated": -1.2612388134002686,
      "rewards/margins": 1.034172773361206,
      "rewards/real": -0.22706612944602966,
      "step": 740
    },
    {
      "epoch": 0.48,
      "learning_rate": 2.8911806543385494e-07,
      "logits/generated": -2.801848888397217,
      "logits/real": -2.9571499824523926,
      "logps/generated": -125.8072509765625,
      "logps/real": -146.98782348632812,
      "loss": 0.6015,
      "rewards/accuracies": 0.699999988079071,
      "rewards/generated": -1.6273508071899414,
      "rewards/margins": 0.8402755856513977,
      "rewards/real": -0.7870752811431885,
      "step": 750
    },
    {
      "epoch": 0.49,
      "learning_rate": 2.855618776671408e-07,
      "logits/generated": -2.848889112472534,
      "logits/real": -2.877046585083008,
      "logps/generated": -124.48414611816406,
      "logps/real": -129.13226318359375,
      "loss": 0.6204,
      "rewards/accuracies": 0.675000011920929,
      "rewards/generated": -1.2502113580703735,
      "rewards/margins": 0.6051310896873474,
      "rewards/real": -0.6450802087783813,
      "step": 760
    },
    {
      "epoch": 0.49,
      "learning_rate": 2.8200568990042673e-07,
      "logits/generated": -2.8570446968078613,
      "logits/real": -2.9029502868652344,
      "logps/generated": -133.1268768310547,
      "logps/real": -136.430419921875,
      "loss": 0.6282,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/generated": -1.4829145669937134,
      "rewards/margins": 0.9649866223335266,
      "rewards/real": -0.5179279446601868,
      "step": 770
    },
    {
      "epoch": 0.5,
      "learning_rate": 2.784495021337127e-07,
      "logits/generated": -2.8677666187286377,
      "logits/real": -2.9021925926208496,
      "logps/generated": -146.73191833496094,
      "logps/real": -141.13995361328125,
      "loss": 0.6747,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/generated": -2.041520595550537,
      "rewards/margins": 1.0659831762313843,
      "rewards/real": -0.9755374789237976,
      "step": 780
    },
    {
      "epoch": 0.51,
      "learning_rate": 2.7489331436699857e-07,
      "logits/generated": -2.824852228164673,
      "logits/real": -2.926391124725342,
      "logps/generated": -147.2328643798828,
      "logps/real": -149.10507202148438,
      "loss": 0.7098,
      "rewards/accuracies": 0.612500011920929,
      "rewards/generated": -1.700987458229065,
      "rewards/margins": 0.7294310331344604,
      "rewards/real": -0.9715563654899597,
      "step": 790
    },
    {
      "epoch": 0.51,
      "learning_rate": 2.7133712660028446e-07,
      "logits/generated": -2.7972347736358643,
      "logits/real": -2.9038093090057373,
      "logps/generated": -125.4245834350586,
      "logps/real": -133.38829040527344,
      "loss": 0.6723,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/generated": -1.891131043434143,
      "rewards/margins": 0.7512563467025757,
      "rewards/real": -1.1398746967315674,
      "step": 800
    },
    {
      "epoch": 0.51,
      "eval_logits/generated": -2.8397910594940186,
      "eval_logits/real": -2.9189207553863525,
      "eval_logps/generated": -133.5920867919922,
      "eval_logps/real": -144.62881469726562,
      "eval_loss": 0.6016955971717834,
      "eval_rewards/accuracies": 0.6791401505470276,
      "eval_rewards/generated": -1.7349849939346313,
      "eval_rewards/margins": 0.7303467988967896,
      "eval_rewards/real": -1.0046383142471313,
      "eval_runtime": 309.8404,
      "eval_samples_per_second": 16.137,
      "eval_steps_per_second": 0.507,
      "step": 800
    },
    {
      "epoch": 0.52,
      "learning_rate": 2.677809388335704e-07,
      "logits/generated": -2.8365445137023926,
      "logits/real": -2.9331562519073486,
      "logps/generated": -140.5518035888672,
      "logps/real": -158.63702392578125,
      "loss": 0.6647,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/generated": -1.3855249881744385,
      "rewards/margins": 0.2608424127101898,
      "rewards/real": -1.1246826648712158,
      "step": 810
    },
    {
      "epoch": 0.52,
      "learning_rate": 2.642247510668563e-07,
      "logits/generated": -2.8873796463012695,
      "logits/real": -2.878711223602295,
      "logps/generated": -145.9031982421875,
      "logps/real": -143.15829467773438,
      "loss": 0.6455,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/generated": -2.0597965717315674,
      "rewards/margins": 1.2405303716659546,
      "rewards/real": -0.8192662000656128,
      "step": 820
    },
    {
      "epoch": 0.53,
      "learning_rate": 2.6066856330014225e-07,
      "logits/generated": -2.8116676807403564,
      "logits/real": -2.9269957542419434,
      "logps/generated": -147.27291870117188,
      "logps/real": -154.99942016601562,
      "loss": 0.5998,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/generated": -1.8391478061676025,
      "rewards/margins": 1.1881417036056519,
      "rewards/real": -0.6510059237480164,
      "step": 830
    },
    {
      "epoch": 0.54,
      "learning_rate": 2.5711237553342815e-07,
      "logits/generated": -2.8869619369506836,
      "logits/real": -2.904590129852295,
      "logps/generated": -148.45867919921875,
      "logps/real": -142.796142578125,
      "loss": 0.721,
      "rewards/accuracies": 0.625,
      "rewards/generated": -0.9768039584159851,
      "rewards/margins": 0.4294663071632385,
      "rewards/real": -0.5473376512527466,
      "step": 840
    },
    {
      "epoch": 0.54,
      "learning_rate": 2.5355618776671404e-07,
      "logits/generated": -2.839021682739258,
      "logits/real": -2.905735492706299,
      "logps/generated": -130.4958953857422,
      "logps/real": -147.63864135742188,
      "loss": 0.7021,
      "rewards/accuracies": 0.625,
      "rewards/generated": -1.1018455028533936,
      "rewards/margins": 0.7128327488899231,
      "rewards/real": -0.3890126347541809,
      "step": 850
    },
    {
      "epoch": 0.55,
      "learning_rate": 2.5e-07,
      "logits/generated": -2.824007034301758,
      "logits/real": -2.857182741165161,
      "logps/generated": -146.1483154296875,
      "logps/real": -139.19387817382812,
      "loss": 0.5886,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/generated": -1.458707571029663,
      "rewards/margins": 1.0141663551330566,
      "rewards/real": -0.44454121589660645,
      "step": 860
    },
    {
      "epoch": 0.56,
      "learning_rate": 2.4644381223328594e-07,
      "logits/generated": -2.7840158939361572,
      "logits/real": -2.9017128944396973,
      "logps/generated": -137.39535522460938,
      "logps/real": -148.0748291015625,
      "loss": 0.6859,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/generated": -1.5387979745864868,
      "rewards/margins": 0.8355327844619751,
      "rewards/real": -0.7032652497291565,
      "step": 870
    },
    {
      "epoch": 0.56,
      "learning_rate": 2.4288762446657183e-07,
      "logits/generated": -2.7799668312072754,
      "logits/real": -2.915595531463623,
      "logps/generated": -130.84051513671875,
      "logps/real": -133.69252014160156,
      "loss": 0.6446,
      "rewards/accuracies": 0.699999988079071,
      "rewards/generated": -1.3767451047897339,
      "rewards/margins": 0.6831783652305603,
      "rewards/real": -0.6935667395591736,
      "step": 880
    },
    {
      "epoch": 0.57,
      "learning_rate": 2.393314366998578e-07,
      "logits/generated": -2.716787576675415,
      "logits/real": -2.8902299404144287,
      "logps/generated": -122.82936096191406,
      "logps/real": -143.2957000732422,
      "loss": 0.5669,
      "rewards/accuracies": 0.737500011920929,
      "rewards/generated": -1.140442132949829,
      "rewards/margins": 1.021167516708374,
      "rewards/real": -0.1192745715379715,
      "step": 890
    },
    {
      "epoch": 0.58,
      "learning_rate": 2.3577524893314365e-07,
      "logits/generated": -2.8401942253112793,
      "logits/real": -2.8803598880767822,
      "logps/generated": -135.40391540527344,
      "logps/real": -136.9019012451172,
      "loss": 0.6436,
      "rewards/accuracies": 0.6875,
      "rewards/generated": -1.2338757514953613,
      "rewards/margins": 0.8897522687911987,
      "rewards/real": -0.3441234230995178,
      "step": 900
    },
    {
      "epoch": 0.58,
      "eval_logits/generated": -2.806016445159912,
      "eval_logits/real": -2.88619327545166,
      "eval_logps/generated": -132.06158447265625,
      "eval_logps/real": -143.32131958007812,
      "eval_loss": 0.6029496192932129,
      "eval_rewards/accuracies": 0.668789803981781,
      "eval_rewards/generated": -1.5819348096847534,
      "eval_rewards/margins": 0.7080464959144592,
      "eval_rewards/real": -0.873888373374939,
      "eval_runtime": 310.4392,
      "eval_samples_per_second": 16.106,
      "eval_steps_per_second": 0.506,
      "step": 900
    },
    {
      "epoch": 0.58,
      "learning_rate": 2.322190611664296e-07,
      "logits/generated": -2.836130142211914,
      "logits/real": -2.796046733856201,
      "logps/generated": -154.68153381347656,
      "logps/real": -142.09814453125,
      "loss": 0.6133,
      "rewards/accuracies": 0.737500011920929,
      "rewards/generated": -1.767498254776001,
      "rewards/margins": 1.2376995086669922,
      "rewards/real": -0.5297988653182983,
      "step": 910
    },
    {
      "epoch": 0.59,
      "learning_rate": 2.2866287339971549e-07,
      "logits/generated": -2.8319737911224365,
      "logits/real": -2.9071972370147705,
      "logps/generated": -141.31478881835938,
      "logps/real": -146.1092987060547,
      "loss": 0.6595,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/generated": -1.810842514038086,
      "rewards/margins": 1.001710057258606,
      "rewards/real": -0.80913245677948,
      "step": 920
    },
    {
      "epoch": 0.6,
      "learning_rate": 2.251066856330014e-07,
      "logits/generated": -2.82523775100708,
      "logits/real": -2.8914453983306885,
      "logps/generated": -147.71463012695312,
      "logps/real": -142.40274047851562,
      "loss": 0.6592,
      "rewards/accuracies": 0.625,
      "rewards/generated": -1.7439014911651611,
      "rewards/margins": 0.4922804832458496,
      "rewards/real": -1.2516210079193115,
      "step": 930
    },
    {
      "epoch": 0.6,
      "learning_rate": 2.2155049786628733e-07,
      "logits/generated": -2.7898082733154297,
      "logits/real": -2.882514476776123,
      "logps/generated": -129.53775024414062,
      "logps/real": -147.6808624267578,
      "loss": 0.6296,
      "rewards/accuracies": 0.675000011920929,
      "rewards/generated": -1.6545770168304443,
      "rewards/margins": 1.0049636363983154,
      "rewards/real": -0.6496134996414185,
      "step": 940
    },
    {
      "epoch": 0.61,
      "learning_rate": 2.1799431009957325e-07,
      "logits/generated": -2.7571780681610107,
      "logits/real": -2.923750877380371,
      "logps/generated": -140.4619598388672,
      "logps/real": -149.5252227783203,
      "loss": 0.5661,
      "rewards/accuracies": 0.800000011920929,
      "rewards/generated": -1.4804662466049194,
      "rewards/margins": 1.0741474628448486,
      "rewards/real": -0.40631866455078125,
      "step": 950
    },
    {
      "epoch": 0.61,
      "learning_rate": 2.1443812233285914e-07,
      "logits/generated": -2.8211305141448975,
      "logits/real": -2.8674416542053223,
      "logps/generated": -131.8004608154297,
      "logps/real": -130.39474487304688,
      "loss": 0.6535,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/generated": -1.2931689023971558,
      "rewards/margins": 0.6902718544006348,
      "rewards/real": -0.602897047996521,
      "step": 960
    },
    {
      "epoch": 0.62,
      "learning_rate": 2.108819345661451e-07,
      "logits/generated": -2.880394458770752,
      "logits/real": -2.881556272506714,
      "logps/generated": -138.2687225341797,
      "logps/real": -140.81362915039062,
      "loss": 0.7122,
      "rewards/accuracies": 0.675000011920929,
      "rewards/generated": -1.508469820022583,
      "rewards/margins": 0.4812171459197998,
      "rewards/real": -1.027252435684204,
      "step": 970
    },
    {
      "epoch": 0.63,
      "learning_rate": 2.0732574679943098e-07,
      "logits/generated": -2.8477933406829834,
      "logits/real": -2.845597267150879,
      "logps/generated": -132.1406707763672,
      "logps/real": -134.79180908203125,
      "loss": 0.6927,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/generated": -1.4040391445159912,
      "rewards/margins": 0.5953569412231445,
      "rewards/real": -0.8086821436882019,
      "step": 980
    },
    {
      "epoch": 0.63,
      "learning_rate": 2.0376955903271693e-07,
      "logits/generated": -2.8082008361816406,
      "logits/real": -2.863212823867798,
      "logps/generated": -125.87931823730469,
      "logps/real": -136.62237548828125,
      "loss": 0.626,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/generated": -1.4956855773925781,
      "rewards/margins": 0.9534099698066711,
      "rewards/real": -0.5422756671905518,
      "step": 990
    },
    {
      "epoch": 0.64,
      "learning_rate": 2.0021337126600283e-07,
      "logits/generated": -2.8578667640686035,
      "logits/real": -2.847013235092163,
      "logps/generated": -136.9371337890625,
      "logps/real": -140.67221069335938,
      "loss": 0.6163,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/generated": -1.597415566444397,
      "rewards/margins": 1.0458736419677734,
      "rewards/real": -0.5515419244766235,
      "step": 1000
    },
    {
      "epoch": 0.64,
      "eval_logits/generated": -2.8002965450286865,
      "eval_logits/real": -2.877998113632202,
      "eval_logps/generated": -132.6573944091797,
      "eval_logps/real": -142.48568725585938,
      "eval_loss": 0.6010281443595886,
      "eval_rewards/accuracies": 0.6783439517021179,
      "eval_rewards/generated": -1.6415162086486816,
      "eval_rewards/margins": 0.8511907458305359,
      "eval_rewards/real": -0.7903255820274353,
      "eval_runtime": 309.9825,
      "eval_samples_per_second": 16.13,
      "eval_steps_per_second": 0.506,
      "step": 1000
    },
    {
      "epoch": 0.65,
      "learning_rate": 1.9665718349928875e-07,
      "logits/generated": -2.791100025177002,
      "logits/real": -2.860755205154419,
      "logps/generated": -132.28335571289062,
      "logps/real": -139.66653442382812,
      "loss": 0.6156,
      "rewards/accuracies": 0.612500011920929,
      "rewards/generated": -1.8716294765472412,
      "rewards/margins": 0.7458978891372681,
      "rewards/real": -1.1257317066192627,
      "step": 1010
    },
    {
      "epoch": 0.65,
      "learning_rate": 1.931009957325747e-07,
      "logits/generated": -2.827510356903076,
      "logits/real": -2.8029284477233887,
      "logps/generated": -136.11270141601562,
      "logps/real": -141.502685546875,
      "loss": 0.6067,
      "rewards/accuracies": 0.699999988079071,
      "rewards/generated": -1.962308645248413,
      "rewards/margins": 0.9203013181686401,
      "rewards/real": -1.0420072078704834,
      "step": 1020
    },
    {
      "epoch": 0.66,
      "learning_rate": 1.895448079658606e-07,
      "logits/generated": -2.835878849029541,
      "logits/real": -2.8575656414031982,
      "logps/generated": -140.47169494628906,
      "logps/real": -139.37342834472656,
      "loss": 0.6224,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/generated": -1.8097025156021118,
      "rewards/margins": 0.8532291650772095,
      "rewards/real": -0.9564734697341919,
      "step": 1030
    },
    {
      "epoch": 0.67,
      "learning_rate": 1.859886201991465e-07,
      "logits/generated": -2.856842279434204,
      "logits/real": -2.8842849731445312,
      "logps/generated": -143.2516326904297,
      "logps/real": -137.08316040039062,
      "loss": 0.6052,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/generated": -1.759925127029419,
      "rewards/margins": 0.6329267621040344,
      "rewards/real": -1.1269984245300293,
      "step": 1040
    },
    {
      "epoch": 0.67,
      "learning_rate": 1.8243243243243243e-07,
      "logits/generated": -2.7838127613067627,
      "logits/real": -2.9055399894714355,
      "logps/generated": -135.84683227539062,
      "logps/real": -142.11846923828125,
      "loss": 0.5939,
      "rewards/accuracies": 0.699999988079071,
      "rewards/generated": -1.581358551979065,
      "rewards/margins": 0.7179355621337891,
      "rewards/real": -0.8634229898452759,
      "step": 1050
    },
    {
      "epoch": 0.68,
      "learning_rate": 1.7887624466571835e-07,
      "logits/generated": -2.8574728965759277,
      "logits/real": -2.812835931777954,
      "logps/generated": -152.22988891601562,
      "logps/real": -149.8922119140625,
      "loss": 0.7048,
      "rewards/accuracies": 0.625,
      "rewards/generated": -1.644396185874939,
      "rewards/margins": 0.46123242378234863,
      "rewards/real": -1.1831638813018799,
      "step": 1060
    },
    {
      "epoch": 0.68,
      "learning_rate": 1.7532005689900424e-07,
      "logits/generated": -2.7940242290496826,
      "logits/real": -2.878798484802246,
      "logps/generated": -133.0207977294922,
      "logps/real": -146.09625244140625,
      "loss": 0.566,
      "rewards/accuracies": 0.762499988079071,
      "rewards/generated": -1.9005569219589233,
      "rewards/margins": 0.8525867462158203,
      "rewards/real": -1.0479700565338135,
      "step": 1070
    },
    {
      "epoch": 0.69,
      "learning_rate": 1.717638691322902e-07,
      "logits/generated": -2.803427219390869,
      "logits/real": -2.829939603805542,
      "logps/generated": -129.09408569335938,
      "logps/real": -127.91636657714844,
      "loss": 0.6161,
      "rewards/accuracies": 0.637499988079071,
      "rewards/generated": -1.7060725688934326,
      "rewards/margins": 0.62469083070755,
      "rewards/real": -1.081381916999817,
      "step": 1080
    },
    {
      "epoch": 0.7,
      "learning_rate": 1.6820768136557609e-07,
      "logits/generated": -2.8071236610412598,
      "logits/real": -2.9175212383270264,
      "logps/generated": -146.45791625976562,
      "logps/real": -160.84805297851562,
      "loss": 0.6539,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/generated": -1.475184679031372,
      "rewards/margins": 0.6628782153129578,
      "rewards/real": -0.8123064041137695,
      "step": 1090
    },
    {
      "epoch": 0.7,
      "learning_rate": 1.64651493598862e-07,
      "logits/generated": -2.7803986072540283,
      "logits/real": -2.913085460662842,
      "logps/generated": -143.08621215820312,
      "logps/real": -150.99246215820312,
      "loss": 0.6297,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/generated": -1.4718377590179443,
      "rewards/margins": 0.9155041575431824,
      "rewards/real": -0.5563336610794067,
      "step": 1100
    },
    {
      "epoch": 0.7,
      "eval_logits/generated": -2.7847776412963867,
      "eval_logits/real": -2.8600666522979736,
      "eval_logps/generated": -132.0778045654297,
      "eval_logps/real": -143.10504150390625,
      "eval_loss": 0.602670431137085,
      "eval_rewards/accuracies": 0.668789803981781,
      "eval_rewards/generated": -1.5835580825805664,
      "eval_rewards/margins": 0.7312976717948914,
      "eval_rewards/real": -0.852260410785675,
      "eval_runtime": 309.4229,
      "eval_samples_per_second": 16.159,
      "eval_steps_per_second": 0.507,
      "step": 1100
    },
    {
      "epoch": 0.71,
      "learning_rate": 1.6109530583214793e-07,
      "logits/generated": -2.826133966445923,
      "logits/real": -2.843700885772705,
      "logps/generated": -140.56655883789062,
      "logps/real": -131.9547882080078,
      "loss": 0.6182,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/generated": -1.5958125591278076,
      "rewards/margins": 0.5843857526779175,
      "rewards/real": -1.0114269256591797,
      "step": 1110
    },
    {
      "epoch": 0.72,
      "learning_rate": 1.5753911806543385e-07,
      "logits/generated": -2.782074213027954,
      "logits/real": -2.852107286453247,
      "logps/generated": -132.11264038085938,
      "logps/real": -148.32745361328125,
      "loss": 0.5487,
      "rewards/accuracies": 0.75,
      "rewards/generated": -1.654834508895874,
      "rewards/margins": 0.9943686723709106,
      "rewards/real": -0.6604658365249634,
      "step": 1120
    },
    {
      "epoch": 0.72,
      "learning_rate": 1.5398293029871974e-07,
      "logits/generated": -2.818897247314453,
      "logits/real": -2.832937717437744,
      "logps/generated": -136.17401123046875,
      "logps/real": -132.2439727783203,
      "loss": 0.6491,
      "rewards/accuracies": 0.675000011920929,
      "rewards/generated": -1.2890915870666504,
      "rewards/margins": 0.7559572458267212,
      "rewards/real": -0.5331342816352844,
      "step": 1130
    },
    {
      "epoch": 0.73,
      "learning_rate": 1.504267425320057e-07,
      "logits/generated": -2.793853521347046,
      "logits/real": -2.85860538482666,
      "logps/generated": -131.0358428955078,
      "logps/real": -138.73953247070312,
      "loss": 0.6312,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/generated": -1.8077558279037476,
      "rewards/margins": 1.0804661512374878,
      "rewards/real": -0.727289617061615,
      "step": 1140
    },
    {
      "epoch": 0.74,
      "learning_rate": 1.4687055476529158e-07,
      "logits/generated": -2.846921682357788,
      "logits/real": -2.8571791648864746,
      "logps/generated": -133.85690307617188,
      "logps/real": -143.62088012695312,
      "loss": 0.6423,
      "rewards/accuracies": 0.6875,
      "rewards/generated": -1.475096344947815,
      "rewards/margins": 1.1357142925262451,
      "rewards/real": -0.33938202261924744,
      "step": 1150
    },
    {
      "epoch": 0.74,
      "learning_rate": 1.4331436699857753e-07,
      "logits/generated": -2.784996509552002,
      "logits/real": -2.9016330242156982,
      "logps/generated": -126.00093078613281,
      "logps/real": -136.3924102783203,
      "loss": 0.565,
      "rewards/accuracies": 0.762499988079071,
      "rewards/generated": -1.6021381616592407,
      "rewards/margins": 1.1204952001571655,
      "rewards/real": -0.48164302110671997,
      "step": 1160
    },
    {
      "epoch": 0.75,
      "learning_rate": 1.3975817923186345e-07,
      "logits/generated": -2.7846155166625977,
      "logits/real": -2.8495097160339355,
      "logps/generated": -133.8337860107422,
      "logps/real": -135.71499633789062,
      "loss": 0.6058,
      "rewards/accuracies": 0.762499988079071,
      "rewards/generated": -1.8828033208847046,
      "rewards/margins": 1.3056962490081787,
      "rewards/real": -0.577107310295105,
      "step": 1170
    },
    {
      "epoch": 0.75,
      "learning_rate": 1.3620199146514935e-07,
      "logits/generated": -2.807847499847412,
      "logits/real": -2.829408645629883,
      "logps/generated": -142.19052124023438,
      "logps/real": -143.4310302734375,
      "loss": 0.6029,
      "rewards/accuracies": 0.675000011920929,
      "rewards/generated": -1.7980120182037354,
      "rewards/margins": 0.9354388117790222,
      "rewards/real": -0.8625731468200684,
      "step": 1180
    },
    {
      "epoch": 0.76,
      "learning_rate": 1.326458036984353e-07,
      "logits/generated": -2.7362117767333984,
      "logits/real": -2.907681941986084,
      "logps/generated": -128.86277770996094,
      "logps/real": -142.07115173339844,
      "loss": 0.6165,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/generated": -1.450605869293213,
      "rewards/margins": 0.6606342792510986,
      "rewards/real": -0.7899715304374695,
      "step": 1190
    },
    {
      "epoch": 0.77,
      "learning_rate": 1.290896159317212e-07,
      "logits/generated": -2.8074393272399902,
      "logits/real": -2.8899171352386475,
      "logps/generated": -135.86477661132812,
      "logps/real": -140.34536743164062,
      "loss": 0.6468,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/generated": -1.600224256515503,
      "rewards/margins": 0.8596419095993042,
      "rewards/real": -0.7405821681022644,
      "step": 1200
    },
    {
      "epoch": 0.77,
      "eval_logits/generated": -2.786166191101074,
      "eval_logits/real": -2.860841989517212,
      "eval_logps/generated": -133.3330078125,
      "eval_logps/real": -143.56629943847656,
      "eval_loss": 0.5992889404296875,
      "eval_rewards/accuracies": 0.683917224407196,
      "eval_rewards/generated": -1.7090762853622437,
      "eval_rewards/margins": 0.8106896281242371,
      "eval_rewards/real": -0.8983866572380066,
      "eval_runtime": 310.5461,
      "eval_samples_per_second": 16.101,
      "eval_steps_per_second": 0.506,
      "step": 1200
    },
    {
      "epoch": 0.77,
      "learning_rate": 1.255334281650071e-07,
      "logits/generated": -2.8380236625671387,
      "logits/real": -2.8380062580108643,
      "logps/generated": -149.90982055664062,
      "logps/real": -147.6665802001953,
      "loss": 0.6125,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/generated": -1.7763712406158447,
      "rewards/margins": 1.0066322088241577,
      "rewards/real": -0.769739031791687,
      "step": 1210
    },
    {
      "epoch": 0.78,
      "learning_rate": 1.2197724039829303e-07,
      "logits/generated": -2.7586867809295654,
      "logits/real": -2.841038703918457,
      "logps/generated": -122.13557434082031,
      "logps/real": -128.7981719970703,
      "loss": 0.6361,
      "rewards/accuracies": 0.737500011920929,
      "rewards/generated": -1.813408613204956,
      "rewards/margins": 0.9969779253005981,
      "rewards/real": -0.8164307475090027,
      "step": 1220
    },
    {
      "epoch": 0.79,
      "learning_rate": 1.1842105263157894e-07,
      "logits/generated": -2.7697105407714844,
      "logits/real": -2.8138997554779053,
      "logps/generated": -130.04222106933594,
      "logps/real": -121.92604064941406,
      "loss": 0.6316,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/generated": -1.6964250802993774,
      "rewards/margins": 0.9563926458358765,
      "rewards/real": -0.740032434463501,
      "step": 1230
    },
    {
      "epoch": 0.79,
      "learning_rate": 1.1486486486486487e-07,
      "logits/generated": -2.753065586090088,
      "logits/real": -2.874058246612549,
      "logps/generated": -130.30264282226562,
      "logps/real": -153.7810821533203,
      "loss": 0.6369,
      "rewards/accuracies": 0.6875,
      "rewards/generated": -1.8373348712921143,
      "rewards/margins": 1.0876656770706177,
      "rewards/real": -0.7496690154075623,
      "step": 1240
    },
    {
      "epoch": 0.8,
      "learning_rate": 1.1130867709815078e-07,
      "logits/generated": -2.8708550930023193,
      "logits/real": -2.8094139099121094,
      "logps/generated": -136.15113830566406,
      "logps/real": -138.6972198486328,
      "loss": 0.6281,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/generated": -1.595159888267517,
      "rewards/margins": 0.705752968788147,
      "rewards/real": -0.889406681060791,
      "step": 1250
    },
    {
      "epoch": 0.81,
      "learning_rate": 1.077524893314367e-07,
      "logits/generated": -2.817417860031128,
      "logits/real": -2.835923910140991,
      "logps/generated": -133.7835235595703,
      "logps/real": -143.9755859375,
      "loss": 0.6229,
      "rewards/accuracies": 0.637499988079071,
      "rewards/generated": -1.5798323154449463,
      "rewards/margins": 0.8425194025039673,
      "rewards/real": -0.7373130321502686,
      "step": 1260
    },
    {
      "epoch": 0.81,
      "learning_rate": 1.0419630156472262e-07,
      "logits/generated": -2.8372952938079834,
      "logits/real": -2.8386409282684326,
      "logps/generated": -135.59951782226562,
      "logps/real": -135.88104248046875,
      "loss": 0.5656,
      "rewards/accuracies": 0.699999988079071,
      "rewards/generated": -1.450683355331421,
      "rewards/margins": 1.1043708324432373,
      "rewards/real": -0.3463127017021179,
      "step": 1270
    },
    {
      "epoch": 0.82,
      "learning_rate": 1.0064011379800854e-07,
      "logits/generated": -2.791853904724121,
      "logits/real": -2.883375644683838,
      "logps/generated": -126.54823303222656,
      "logps/real": -142.61856079101562,
      "loss": 0.6473,
      "rewards/accuracies": 0.675000011920929,
      "rewards/generated": -1.1970711946487427,
      "rewards/margins": 0.7306603193283081,
      "rewards/real": -0.4664107859134674,
      "step": 1280
    },
    {
      "epoch": 0.83,
      "learning_rate": 9.708392603129445e-08,
      "logits/generated": -2.7650647163391113,
      "logits/real": -2.790853261947632,
      "logps/generated": -127.62080383300781,
      "logps/real": -134.742431640625,
      "loss": 0.6052,
      "rewards/accuracies": 0.625,
      "rewards/generated": -1.3538134098052979,
      "rewards/margins": 0.8441446423530579,
      "rewards/real": -0.5096686482429504,
      "step": 1290
    },
    {
      "epoch": 0.83,
      "learning_rate": 9.352773826458037e-08,
      "logits/generated": -2.792762279510498,
      "logits/real": -2.795624256134033,
      "logps/generated": -142.76382446289062,
      "logps/real": -141.875244140625,
      "loss": 0.5741,
      "rewards/accuracies": 0.6875,
      "rewards/generated": -1.515129804611206,
      "rewards/margins": 0.7379931211471558,
      "rewards/real": -0.7771366238594055,
      "step": 1300
    },
    {
      "epoch": 0.83,
      "eval_logits/generated": -2.774766445159912,
      "eval_logits/real": -2.8526883125305176,
      "eval_logps/generated": -132.890625,
      "eval_logps/real": -143.34512329101562,
      "eval_loss": 0.6018590927124023,
      "eval_rewards/accuracies": 0.6871019005775452,
      "eval_rewards/generated": -1.664839267730713,
      "eval_rewards/margins": 0.788568913936615,
      "eval_rewards/real": -0.8762705326080322,
      "eval_runtime": 309.8754,
      "eval_samples_per_second": 16.136,
      "eval_steps_per_second": 0.507,
      "step": 1300
    },
    {
      "epoch": 0.84,
      "learning_rate": 8.997155049786629e-08,
      "logits/generated": -2.7696619033813477,
      "logits/real": -2.8957486152648926,
      "logps/generated": -137.81674194335938,
      "logps/real": -151.4634552001953,
      "loss": 0.6453,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/generated": -1.5495209693908691,
      "rewards/margins": 0.5234196782112122,
      "rewards/real": -1.0261012315750122,
      "step": 1310
    },
    {
      "epoch": 0.84,
      "learning_rate": 8.64153627311522e-08,
      "logits/generated": -2.7855117321014404,
      "logits/real": -2.881375551223755,
      "logps/generated": -146.25650024414062,
      "logps/real": -159.53533935546875,
      "loss": 0.5948,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/generated": -1.6503632068634033,
      "rewards/margins": 0.9353766441345215,
      "rewards/real": -0.7149866819381714,
      "step": 1320
    },
    {
      "epoch": 0.85,
      "learning_rate": 8.285917496443812e-08,
      "logits/generated": -2.839664936065674,
      "logits/real": -2.8154075145721436,
      "logps/generated": -152.0926513671875,
      "logps/real": -141.4612579345703,
      "loss": 0.6361,
      "rewards/accuracies": 0.6875,
      "rewards/generated": -1.6590524911880493,
      "rewards/margins": 0.7780254483222961,
      "rewards/real": -0.881027102470398,
      "step": 1330
    },
    {
      "epoch": 0.86,
      "learning_rate": 7.930298719772404e-08,
      "logits/generated": -2.76212477684021,
      "logits/real": -2.8543272018432617,
      "logps/generated": -144.95997619628906,
      "logps/real": -153.9541778564453,
      "loss": 0.6119,
      "rewards/accuracies": 0.625,
      "rewards/generated": -1.4291819334030151,
      "rewards/margins": 0.6051322221755981,
      "rewards/real": -0.8240497708320618,
      "step": 1340
    },
    {
      "epoch": 0.86,
      "learning_rate": 7.574679943100994e-08,
      "logits/generated": -2.7643582820892334,
      "logits/real": -2.803393840789795,
      "logps/generated": -138.57437133789062,
      "logps/real": -143.98910522460938,
      "loss": 0.6042,
      "rewards/accuracies": 0.762499988079071,
      "rewards/generated": -1.851440191268921,
      "rewards/margins": 1.1676199436187744,
      "rewards/real": -0.6838203072547913,
      "step": 1350
    },
    {
      "epoch": 0.87,
      "learning_rate": 7.219061166429587e-08,
      "logits/generated": -2.842731475830078,
      "logits/real": -2.75618314743042,
      "logps/generated": -153.87928771972656,
      "logps/real": -137.30043029785156,
      "loss": 0.6227,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/generated": -1.66205632686615,
      "rewards/margins": 0.847110390663147,
      "rewards/real": -0.8149459958076477,
      "step": 1360
    },
    {
      "epoch": 0.88,
      "learning_rate": 6.863442389758179e-08,
      "logits/generated": -2.720419406890869,
      "logits/real": -2.8356549739837646,
      "logps/generated": -132.40487670898438,
      "logps/real": -142.2407684326172,
      "loss": 0.6505,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/generated": -1.8278366327285767,
      "rewards/margins": 1.0291211605072021,
      "rewards/real": -0.7987155914306641,
      "step": 1370
    },
    {
      "epoch": 0.88,
      "learning_rate": 6.507823613086771e-08,
      "logits/generated": -2.7388923168182373,
      "logits/real": -2.8658761978149414,
      "logps/generated": -129.93698120117188,
      "logps/real": -150.47926330566406,
      "loss": 0.6557,
      "rewards/accuracies": 0.625,
      "rewards/generated": -1.5377349853515625,
      "rewards/margins": 0.5789148807525635,
      "rewards/real": -0.9588201642036438,
      "step": 1380
    },
    {
      "epoch": 0.89,
      "learning_rate": 6.152204836415363e-08,
      "logits/generated": -2.7871286869049072,
      "logits/real": -2.8277242183685303,
      "logps/generated": -135.78118896484375,
      "logps/real": -132.1021270751953,
      "loss": 0.5725,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/generated": -1.9688875675201416,
      "rewards/margins": 0.9650619626045227,
      "rewards/real": -1.0038254261016846,
      "step": 1390
    },
    {
      "epoch": 0.9,
      "learning_rate": 5.796586059743954e-08,
      "logits/generated": -2.7154223918914795,
      "logits/real": -2.880283832550049,
      "logps/generated": -124.3189926147461,
      "logps/real": -141.7947998046875,
      "loss": 0.5778,
      "rewards/accuracies": 0.75,
      "rewards/generated": -1.4083908796310425,
      "rewards/margins": 0.8421765565872192,
      "rewards/real": -0.5662142634391785,
      "step": 1400
    },
    {
      "epoch": 0.9,
      "eval_logits/generated": -2.7595629692077637,
      "eval_logits/real": -2.8409905433654785,
      "eval_logps/generated": -132.63201904296875,
      "eval_logps/real": -143.1669158935547,
      "eval_loss": 0.5983285307884216,
      "eval_rewards/accuracies": 0.683917224407196,
      "eval_rewards/generated": -1.638979196548462,
      "eval_rewards/margins": 0.7805317044258118,
      "eval_rewards/real": -0.858447253704071,
      "eval_runtime": 311.6921,
      "eval_samples_per_second": 16.041,
      "eval_steps_per_second": 0.504,
      "step": 1400
    },
    {
      "epoch": 0.9,
      "learning_rate": 5.4409672830725456e-08,
      "logits/generated": -2.8682186603546143,
      "logits/real": -2.770561695098877,
      "logps/generated": -145.3868408203125,
      "logps/real": -134.76119995117188,
      "loss": 0.5696,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/generated": -1.4525306224822998,
      "rewards/margins": 0.9229253530502319,
      "rewards/real": -0.5296052694320679,
      "step": 1410
    },
    {
      "epoch": 0.91,
      "learning_rate": 5.0853485064011376e-08,
      "logits/generated": -2.7014617919921875,
      "logits/real": -2.881330966949463,
      "logps/generated": -130.04843139648438,
      "logps/real": -142.1650390625,
      "loss": 0.5766,
      "rewards/accuracies": 0.737500011920929,
      "rewards/generated": -1.7901127338409424,
      "rewards/margins": 1.182373285293579,
      "rewards/real": -0.6077393293380737,
      "step": 1420
    },
    {
      "epoch": 0.91,
      "learning_rate": 4.72972972972973e-08,
      "logits/generated": -2.776926040649414,
      "logits/real": -2.816567897796631,
      "logps/generated": -138.56385803222656,
      "logps/real": -138.20372009277344,
      "loss": 0.5432,
      "rewards/accuracies": 0.762499988079071,
      "rewards/generated": -1.6850738525390625,
      "rewards/margins": 1.1410996913909912,
      "rewards/real": -0.5439742803573608,
      "step": 1430
    },
    {
      "epoch": 0.92,
      "learning_rate": 4.374110953058322e-08,
      "logits/generated": -2.717988967895508,
      "logits/real": -2.7903523445129395,
      "logps/generated": -127.56498718261719,
      "logps/real": -131.7296600341797,
      "loss": 0.574,
      "rewards/accuracies": 0.762499988079071,
      "rewards/generated": -1.8308528661727905,
      "rewards/margins": 0.9762704968452454,
      "rewards/real": -0.8545824289321899,
      "step": 1440
    },
    {
      "epoch": 0.93,
      "learning_rate": 4.018492176386913e-08,
      "logits/generated": -2.794159412384033,
      "logits/real": -2.820596933364868,
      "logps/generated": -138.13177490234375,
      "logps/real": -137.43312072753906,
      "loss": 0.5975,
      "rewards/accuracies": 0.675000011920929,
      "rewards/generated": -2.0723748207092285,
      "rewards/margins": 1.080757737159729,
      "rewards/real": -0.9916172027587891,
      "step": 1450
    },
    {
      "epoch": 0.93,
      "learning_rate": 3.6628733997155046e-08,
      "logits/generated": -2.792384624481201,
      "logits/real": -2.7937417030334473,
      "logps/generated": -143.66940307617188,
      "logps/real": -130.21682739257812,
      "loss": 0.652,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/generated": -1.7282785177230835,
      "rewards/margins": 1.0712766647338867,
      "rewards/real": -0.6570018529891968,
      "step": 1460
    },
    {
      "epoch": 0.94,
      "learning_rate": 3.3072546230440967e-08,
      "logits/generated": -2.8024961948394775,
      "logits/real": -2.8199470043182373,
      "logps/generated": -140.53408813476562,
      "logps/real": -150.7724151611328,
      "loss": 0.628,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/generated": -1.6441261768341064,
      "rewards/margins": 0.8207837343215942,
      "rewards/real": -0.823342502117157,
      "step": 1470
    },
    {
      "epoch": 0.95,
      "learning_rate": 2.9516358463726884e-08,
      "logits/generated": -2.7492637634277344,
      "logits/real": -2.858729839324951,
      "logps/generated": -134.35336303710938,
      "logps/real": -142.91067504882812,
      "loss": 0.572,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/generated": -1.3461004495620728,
      "rewards/margins": 0.9981536865234375,
      "rewards/real": -0.34794679284095764,
      "step": 1480
    },
    {
      "epoch": 0.95,
      "learning_rate": 2.59601706970128e-08,
      "logits/generated": -2.7729005813598633,
      "logits/real": -2.873631715774536,
      "logps/generated": -150.54849243164062,
      "logps/real": -151.95889282226562,
      "loss": 0.6035,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/generated": -1.5365216732025146,
      "rewards/margins": 1.0391191244125366,
      "rewards/real": -0.4974023699760437,
      "step": 1490
    },
    {
      "epoch": 0.96,
      "learning_rate": 2.240398293029872e-08,
      "logits/generated": -2.760676622390747,
      "logits/real": -2.839698553085327,
      "logps/generated": -132.1126708984375,
      "logps/real": -144.00076293945312,
      "loss": 0.5929,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/generated": -1.4845622777938843,
      "rewards/margins": 0.9012053608894348,
      "rewards/real": -0.5833569765090942,
      "step": 1500
    },
    {
      "epoch": 0.96,
      "eval_logits/generated": -2.753782272338867,
      "eval_logits/real": -2.8343567848205566,
      "eval_logps/generated": -132.1463165283203,
      "eval_logps/real": -142.78977966308594,
      "eval_loss": 0.5982768535614014,
      "eval_rewards/accuracies": 0.6831210255622864,
      "eval_rewards/generated": -1.5904085636138916,
      "eval_rewards/margins": 0.7696728706359863,
      "eval_rewards/real": -0.82073575258255,
      "eval_runtime": 310.8083,
      "eval_samples_per_second": 16.087,
      "eval_steps_per_second": 0.505,
      "step": 1500
    },
    {
      "epoch": 0.97,
      "learning_rate": 1.8847795163584636e-08,
      "logits/generated": -2.786104679107666,
      "logits/real": -2.843632698059082,
      "logps/generated": -135.99102783203125,
      "logps/real": -146.1920928955078,
      "loss": 0.5737,
      "rewards/accuracies": 0.699999988079071,
      "rewards/generated": -1.6939281225204468,
      "rewards/margins": 0.7886837124824524,
      "rewards/real": -0.9052446484565735,
      "step": 1510
    },
    {
      "epoch": 0.97,
      "learning_rate": 1.5291607396870554e-08,
      "logits/generated": -2.7616968154907227,
      "logits/real": -2.8508548736572266,
      "logps/generated": -134.79690551757812,
      "logps/real": -145.90550231933594,
      "loss": 0.646,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/generated": -1.472129464149475,
      "rewards/margins": 0.7121609449386597,
      "rewards/real": -0.7599684000015259,
      "step": 1520
    },
    {
      "epoch": 0.98,
      "learning_rate": 1.1735419630156473e-08,
      "logits/generated": -2.7698681354522705,
      "logits/real": -2.822406768798828,
      "logps/generated": -137.62112426757812,
      "logps/real": -144.58047485351562,
      "loss": 0.6169,
      "rewards/accuracies": 0.699999988079071,
      "rewards/generated": -1.6243680715560913,
      "rewards/margins": 1.031085729598999,
      "rewards/real": -0.5932824611663818,
      "step": 1530
    },
    {
      "epoch": 0.99,
      "learning_rate": 8.179231863442388e-09,
      "logits/generated": -2.7279772758483887,
      "logits/real": -2.8418853282928467,
      "logps/generated": -128.5863800048828,
      "logps/real": -140.18991088867188,
      "loss": 0.7108,
      "rewards/accuracies": 0.675000011920929,
      "rewards/generated": -1.8496240377426147,
      "rewards/margins": 0.7590233683586121,
      "rewards/real": -1.090600848197937,
      "step": 1540
    },
    {
      "epoch": 0.99,
      "learning_rate": 4.623044096728307e-09,
      "logits/generated": -2.738799810409546,
      "logits/real": -2.8188023567199707,
      "logps/generated": -128.40736389160156,
      "logps/real": -141.7828826904297,
      "loss": 0.6033,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/generated": -1.6743981838226318,
      "rewards/margins": 1.010298728942871,
      "rewards/real": -0.6640995740890503,
      "step": 1550
    },
    {
      "epoch": 1.0,
      "learning_rate": 1.0668563300142248e-09,
      "logits/generated": -2.787738084793091,
      "logits/real": -2.819310188293457,
      "logps/generated": -142.7244415283203,
      "logps/real": -137.14144897460938,
      "loss": 0.6264,
      "rewards/accuracies": 0.6875,
      "rewards/generated": -1.5642173290252686,
      "rewards/margins": 0.8839362859725952,
      "rewards/real": -0.6802809238433838,
      "step": 1560
    },
    {
      "epoch": 1.0,
      "step": 1563,
      "total_flos": 0.0,
      "train_loss": 0.6356648667378831,
      "train_runtime": 12633.5214,
      "train_samples_per_second": 3.958,
      "train_steps_per_second": 0.124
    }
  ],
  "logging_steps": 10,
  "max_steps": 1563,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 100,
  "total_flos": 0.0,
  "train_batch_size": 8,
  "trial_name": null,
  "trial_params": null
}