{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 4.0,
  "eval_steps": 100,
  "global_step": 1540,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.0,
      "learning_rate": 1.282051282051282e-07,
      "logits/chosen": -1.7278180122375488,
      "logits/rejected": -1.7377450466156006,
      "logps/chosen": -29.553977966308594,
      "logps/rejected": -42.813133239746094,
      "loss": 1.0,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 1
    },
    {
      "epoch": 0.03,
      "learning_rate": 1.282051282051282e-06,
      "logits/chosen": -1.8667426109313965,
      "logits/rejected": -1.8710602521896362,
      "logps/chosen": -36.991912841796875,
      "logps/rejected": -33.67206954956055,
      "loss": 0.9773,
      "rewards/accuracies": 0.5694444179534912,
      "rewards/chosen": 0.0058750128373503685,
      "rewards/margins": 0.022671451792120934,
      "rewards/rejected": -0.01679643802344799,
      "step": 10
    },
    {
      "epoch": 0.05,
      "learning_rate": 2.564102564102564e-06,
      "logits/chosen": -1.9977442026138306,
      "logits/rejected": -2.0003952980041504,
      "logps/chosen": -29.659366607666016,
      "logps/rejected": -29.05437660217285,
      "loss": 1.0105,
      "rewards/accuracies": 0.4124999940395355,
      "rewards/chosen": -0.006868218071758747,
      "rewards/margins": -0.010493903420865536,
      "rewards/rejected": 0.003625686513260007,
      "step": 20
    },
    {
      "epoch": 0.08,
      "learning_rate": 3.846153846153847e-06,
      "logits/chosen": -1.920693039894104,
      "logits/rejected": -1.91802179813385,
      "logps/chosen": -31.39971351623535,
      "logps/rejected": -33.21495819091797,
      "loss": 0.9948,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.006551843136548996,
      "rewards/margins": 0.0051523735746741295,
      "rewards/rejected": 0.001399471191689372,
      "step": 30
    },
    {
      "epoch": 0.1,
      "learning_rate": 4.999896948438434e-06,
      "logits/chosen": -2.018057107925415,
      "logits/rejected": -2.0093047618865967,
      "logps/chosen": -32.565284729003906,
      "logps/rejected": -32.50053405761719,
      "loss": 1.0002,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": 0.0046076299622654915,
      "rewards/margins": -0.00024683662923052907,
      "rewards/rejected": 0.004854466766119003,
      "step": 40
    },
    {
      "epoch": 0.13,
      "learning_rate": 4.987541037542187e-06,
      "logits/chosen": -1.8627817630767822,
      "logits/rejected": -1.851999044418335,
      "logps/chosen": -33.549964904785156,
      "logps/rejected": -35.44340896606445,
      "loss": 1.0006,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": 0.0030060340650379658,
      "rewards/margins": -0.0005688609671778977,
      "rewards/rejected": 0.003574896603822708,
      "step": 50
    },
    {
      "epoch": 0.16,
      "learning_rate": 4.954691471941119e-06,
      "logits/chosen": -1.9416770935058594,
      "logits/rejected": -1.9436094760894775,
      "logps/chosen": -32.53351593017578,
      "logps/rejected": -33.217529296875,
      "loss": 0.9561,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.026666466146707535,
      "rewards/margins": 0.04512657970190048,
      "rewards/rejected": -0.018460111692547798,
      "step": 60
    },
    {
      "epoch": 0.18,
      "learning_rate": 4.901618883413549e-06,
      "logits/chosen": -2.072779655456543,
      "logits/rejected": -2.077756881713867,
      "logps/chosen": -34.002342224121094,
      "logps/rejected": -36.633216857910156,
      "loss": 0.9798,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.0055419160053133965,
      "rewards/margins": 0.020196745172142982,
      "rewards/rejected": -0.025738662108778954,
      "step": 70
    },
    {
      "epoch": 0.21,
      "learning_rate": 4.828760511501322e-06,
      "logits/chosen": -1.9332294464111328,
      "logits/rejected": -1.9363486766815186,
      "logps/chosen": -34.33064651489258,
      "logps/rejected": -34.64745330810547,
      "loss": 0.9391,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.03886651247739792,
      "rewards/margins": 0.06088464334607124,
      "rewards/rejected": -0.022018127143383026,
      "step": 80
    },
    {
      "epoch": 0.23,
      "learning_rate": 4.7367166013034295e-06,
      "logits/chosen": -1.9409536123275757,
      "logits/rejected": -1.9454677104949951,
      "logps/chosen": -32.381752014160156,
      "logps/rejected": -32.35805892944336,
      "loss": 0.9661,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": 0.0342683270573616,
      "rewards/margins": 0.033929694443941116,
      "rewards/rejected": 0.0003386303724255413,
      "step": 90
    },
    {
      "epoch": 0.26,
      "learning_rate": 4.626245458345211e-06,
      "logits/chosen": -2.0382819175720215,
      "logits/rejected": -2.036304473876953,
      "logps/chosen": -32.15763473510742,
      "logps/rejected": -31.302764892578125,
      "loss": 0.9521,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.033279187977313995,
      "rewards/margins": 0.047900475561618805,
      "rewards/rejected": -0.014621290378272533,
      "step": 100
    },
    {
      "epoch": 0.26,
      "eval_logits/chosen": -2.2327499389648438,
      "eval_logits/rejected": -2.227902889251709,
      "eval_logps/chosen": -34.038063049316406,
      "eval_logps/rejected": -37.54012680053711,
      "eval_loss": 0.9920137524604797,
      "eval_rewards/accuracies": 0.5361295938491821,
      "eval_rewards/chosen": -0.0014053798513486981,
      "eval_rewards/margins": 0.007998107932507992,
      "eval_rewards/rejected": -0.009403487667441368,
      "eval_runtime": 146.1087,
      "eval_samples_per_second": 2.348,
      "eval_steps_per_second": 0.294,
      "step": 100
    },
    {
      "epoch": 0.29,
      "learning_rate": 4.498257201263691e-06,
      "logits/chosen": -1.9931761026382446,
      "logits/rejected": -1.9907801151275635,
      "logps/chosen": -33.120845794677734,
      "logps/rejected": -34.02234649658203,
      "loss": 0.9536,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.04918808490037918,
      "rewards/margins": 0.046377379447221756,
      "rewards/rejected": 0.0028106991667300463,
      "step": 110
    },
    {
      "epoch": 0.31,
      "learning_rate": 4.353806263777678e-06,
      "logits/chosen": -2.004035472869873,
      "logits/rejected": -1.9957201480865479,
      "logps/chosen": -32.31081771850586,
      "logps/rejected": -32.129127502441406,
      "loss": 0.9593,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.05363499000668526,
      "rewards/margins": 0.04068244248628616,
      "rewards/rejected": 0.012952548451721668,
      "step": 120
    },
    {
      "epoch": 0.34,
      "learning_rate": 4.1940827077152755e-06,
      "logits/chosen": -2.0312001705169678,
      "logits/rejected": -2.023239850997925,
      "logps/chosen": -30.317270278930664,
      "logps/rejected": -32.05809783935547,
      "loss": 0.9384,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.06337819993495941,
      "rewards/margins": 0.06541591882705688,
      "rewards/rejected": -0.0020377314649522305,
      "step": 130
    },
    {
      "epoch": 0.36,
      "learning_rate": 4.0204024186666215e-06,
      "logits/chosen": -1.9611848592758179,
      "logits/rejected": -1.9714059829711914,
      "logps/chosen": -31.235523223876953,
      "logps/rejected": -32.55936813354492,
      "loss": 0.9165,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.07365532219409943,
      "rewards/margins": 0.08345074951648712,
      "rewards/rejected": -0.009795431979000568,
      "step": 140
    },
    {
      "epoch": 0.39,
      "learning_rate": 3.834196265035119e-06,
      "logits/chosen": -1.8718490600585938,
      "logits/rejected": -1.8730093240737915,
      "logps/chosen": -33.873046875,
      "logps/rejected": -34.80980682373047,
      "loss": 0.8563,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": 0.12785716354846954,
      "rewards/margins": 0.1488770991563797,
      "rewards/rejected": -0.021019931882619858,
      "step": 150
    },
    {
      "epoch": 0.42,
      "learning_rate": 3.636998309800573e-06,
      "logits/chosen": -1.9235217571258545,
      "logits/rejected": -1.9201278686523438,
      "logps/chosen": -36.01029968261719,
      "logps/rejected": -32.690834045410156,
      "loss": 0.9482,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.06455211341381073,
      "rewards/margins": 0.05178683251142502,
      "rewards/rejected": 0.012765283696353436,
      "step": 160
    },
    {
      "epoch": 0.44,
      "learning_rate": 3.4304331721118078e-06,
      "logits/chosen": -2.0244345664978027,
      "logits/rejected": -2.0171027183532715,
      "logps/chosen": -33.474937438964844,
      "logps/rejected": -31.42409324645996,
      "loss": 0.8419,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": 0.12916772067546844,
      "rewards/margins": 0.15806543827056885,
      "rewards/rejected": -0.028897713869810104,
      "step": 170
    },
    {
      "epoch": 0.47,
      "learning_rate": 3.2162026428305436e-06,
      "logits/chosen": -2.0310873985290527,
      "logits/rejected": -2.036334276199341,
      "logps/chosen": -32.2062873840332,
      "logps/rejected": -32.42302703857422,
      "loss": 0.8919,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": 0.1354007124900818,
      "rewards/margins": 0.11055666208267212,
      "rewards/rejected": 0.024844054132699966,
      "step": 180
    },
    {
      "epoch": 0.49,
      "learning_rate": 2.996071664294641e-06,
      "logits/chosen": -2.0314042568206787,
      "logits/rejected": -2.02862286567688,
      "logps/chosen": -31.278457641601562,
      "logps/rejected": -31.348251342773438,
      "loss": 0.9038,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": 0.0841989517211914,
      "rewards/margins": 0.09621445834636688,
      "rewards/rejected": -0.012015508487820625,
      "step": 190
    },
    {
      "epoch": 0.52,
      "learning_rate": 2.7718537898066833e-06,
      "logits/chosen": -1.9020229578018188,
      "logits/rejected": -1.906660795211792,
      "logps/chosen": -31.28672218322754,
      "logps/rejected": -32.84270477294922,
      "loss": 0.8515,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": 0.12782469391822815,
      "rewards/margins": 0.14853176474571228,
      "rewards/rejected": -0.020707078278064728,
      "step": 200
    },
    {
      "epoch": 0.52,
      "eval_logits/chosen": -2.22743558883667,
      "eval_logits/rejected": -2.2226204872131348,
      "eval_logps/chosen": -34.04505920410156,
      "eval_logps/rejected": -37.573036193847656,
      "eval_loss": 0.9816663265228271,
      "eval_rewards/accuracies": 0.559385359287262,
      "eval_rewards/chosen": -0.004204160068184137,
      "eval_rewards/margins": 0.018361244350671768,
      "eval_rewards/rejected": -0.022565403953194618,
      "eval_runtime": 145.491,
      "eval_samples_per_second": 2.358,
      "eval_steps_per_second": 0.296,
      "step": 200
    },
    {
      "epoch": 0.55,
      "learning_rate": 2.5453962426402006e-06,
      "logits/chosen": -2.0146515369415283,
      "logits/rejected": -2.025285005569458,
      "logps/chosen": -31.747217178344727,
      "logps/rejected": -33.96654510498047,
      "loss": 0.8755,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": 0.08963600546121597,
      "rewards/margins": 0.13117524981498718,
      "rewards/rejected": -0.04153924435377121,
      "step": 210
    },
    {
      "epoch": 0.57,
      "learning_rate": 2.3185646976551794e-06,
      "logits/chosen": -1.9072997570037842,
      "logits/rejected": -1.922041654586792,
      "logps/chosen": -29.81575584411621,
      "logps/rejected": -31.636306762695312,
      "loss": 0.8509,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": 0.11831430345773697,
      "rewards/margins": 0.14956556260585785,
      "rewards/rejected": -0.03125125169754028,
      "step": 220
    },
    {
      "epoch": 0.6,
      "learning_rate": 2.0932279108998323e-06,
      "logits/chosen": -1.9637253284454346,
      "logits/rejected": -1.9677000045776367,
      "logps/chosen": -33.084877014160156,
      "logps/rejected": -31.639108657836914,
      "loss": 0.8476,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.13303686678409576,
      "rewards/margins": 0.16895917057991028,
      "rewards/rejected": -0.03592229634523392,
      "step": 230
    },
    {
      "epoch": 0.62,
      "learning_rate": 1.8712423238279358e-06,
      "logits/chosen": -1.9612106084823608,
      "logits/rejected": -1.9394094944000244,
      "logps/chosen": -33.84415817260742,
      "logps/rejected": -35.132205963134766,
      "loss": 0.8154,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": 0.1238357201218605,
      "rewards/margins": 0.19639968872070312,
      "rewards/rejected": -0.07256398350000381,
      "step": 240
    },
    {
      "epoch": 0.65,
      "learning_rate": 1.6544367689701824e-06,
      "logits/chosen": -2.002776861190796,
      "logits/rejected": -1.999464988708496,
      "logps/chosen": -32.74934387207031,
      "logps/rejected": -36.28064727783203,
      "loss": 0.9096,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.0702909380197525,
      "rewards/margins": 0.09257940202951431,
      "rewards/rejected": -0.02228846587240696,
      "step": 250
    },
    {
      "epoch": 0.68,
      "learning_rate": 1.4445974030621963e-06,
      "logits/chosen": -1.8692924976348877,
      "logits/rejected": -1.8668625354766846,
      "logps/chosen": -33.982818603515625,
      "logps/rejected": -35.535736083984375,
      "loss": 0.9053,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.08223171532154083,
      "rewards/margins": 0.09467832744121552,
      "rewards/rejected": -0.012446624226868153,
      "step": 260
    },
    {
      "epoch": 0.7,
      "learning_rate": 1.243452991757889e-06,
      "logits/chosen": -1.8543907403945923,
      "logits/rejected": -1.8520078659057617,
      "logps/chosen": -34.20638656616211,
      "logps/rejected": -31.873388290405273,
      "loss": 0.8923,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.07648433744907379,
      "rewards/margins": 0.11730633676052094,
      "rewards/rejected": -0.040821999311447144,
      "step": 270
    },
    {
      "epoch": 0.73,
      "learning_rate": 1.0526606671603523e-06,
      "logits/chosen": -1.9576927423477173,
      "logits/rejected": -1.9472328424453735,
      "logps/chosen": -34.999000549316406,
      "logps/rejected": -31.88382911682129,
      "loss": 0.8398,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": 0.1430555135011673,
      "rewards/margins": 0.16019006073474884,
      "rewards/rejected": -0.017134560272097588,
      "step": 280
    },
    {
      "epoch": 0.75,
      "learning_rate": 8.737922755071455e-07,
      "logits/chosen": -2.0523831844329834,
      "logits/rejected": -2.037504196166992,
      "logps/chosen": -30.712234497070312,
      "logps/rejected": -32.659637451171875,
      "loss": 0.9146,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": 0.08475608378648758,
      "rewards/margins": 0.08787757158279419,
      "rewards/rejected": -0.0031214915215969086,
      "step": 290
    },
    {
      "epoch": 0.78,
      "learning_rate": 7.08321427484816e-07,
      "logits/chosen": -1.9236303567886353,
      "logits/rejected": -1.9211324453353882,
      "logps/chosen": -32.370330810546875,
      "logps/rejected": -30.952068328857422,
      "loss": 0.7473,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": 0.2227112501859665,
      "rewards/margins": 0.27681466937065125,
      "rewards/rejected": -0.054103411734104156,
      "step": 300
    },
    {
      "epoch": 0.78,
      "eval_logits/chosen": -2.223449468612671,
      "eval_logits/rejected": -2.2186241149902344,
      "eval_logps/chosen": -34.09219741821289,
      "eval_logps/rejected": -37.61648178100586,
      "eval_loss": 0.9830461740493774,
      "eval_rewards/accuracies": 0.5390365719795227,
      "eval_rewards/chosen": -0.023058738559484482,
      "eval_rewards/margins": 0.016884688287973404,
      "eval_rewards/rejected": -0.03994342312216759,
      "eval_runtime": 145.6848,
      "eval_samples_per_second": 2.354,
      "eval_steps_per_second": 0.295,
      "step": 300
    },
    {
      "epoch": 0.81,
      "grad_norm": 19.375,
      "learning_rate": 4.84533120650964e-06,
      "logits/chosen": -2.057250499725342,
      "logits/rejected": -2.0444836616516113,
      "logps/chosen": -32.08330535888672,
      "logps/rejected": -32.93002700805664,
      "loss": 0.7174,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.17803600430488586,
      "rewards/margins": 0.29362356662750244,
      "rewards/rejected": -0.11558754742145538,
      "step": 310
    },
    {
      "epoch": 0.83,
      "grad_norm": 18.375,
      "learning_rate": 4.825108134172131e-06,
      "logits/chosen": -1.9667476415634155,
      "logits/rejected": -1.9581434726715088,
      "logps/chosen": -31.69952964782715,
      "logps/rejected": -30.477031707763672,
      "loss": 0.6654,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": 0.26218193769454956,
      "rewards/margins": 0.35643792152404785,
      "rewards/rejected": -0.09425600618124008,
      "step": 320
    },
    {
      "epoch": 0.86,
      "grad_norm": 13.5,
      "learning_rate": 4.80369052967602e-06,
      "logits/chosen": -1.902342438697815,
      "logits/rejected": -1.914393424987793,
      "logps/chosen": -29.752674102783203,
      "logps/rejected": -33.74560546875,
      "loss": 0.5789,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": 0.29063910245895386,
      "rewards/margins": 0.427977979183197,
      "rewards/rejected": -0.13733889162540436,
      "step": 330
    },
    {
      "epoch": 0.88,
      "grad_norm": 23.375,
      "learning_rate": 4.781089396387968e-06,
      "logits/chosen": -1.8653125762939453,
      "logits/rejected": -1.8561639785766602,
      "logps/chosen": -33.982444763183594,
      "logps/rejected": -36.24190902709961,
      "loss": 0.6225,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 0.2865905463695526,
      "rewards/margins": 0.44066372513771057,
      "rewards/rejected": -0.15407314896583557,
      "step": 340
    },
    {
      "epoch": 0.91,
      "grad_norm": 16.875,
      "learning_rate": 4.757316345716554e-06,
      "logits/chosen": -1.9161920547485352,
      "logits/rejected": -1.9168342351913452,
      "logps/chosen": -33.598670959472656,
      "logps/rejected": -34.16885757446289,
      "loss": 0.6015,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.3226447105407715,
      "rewards/margins": 0.4505980908870697,
      "rewards/rejected": -0.1279533952474594,
      "step": 350
    },
    {
      "epoch": 0.94,
      "grad_norm": 17.75,
      "learning_rate": 4.73238359114687e-06,
      "logits/chosen": -2.0425257682800293,
      "logits/rejected": -2.0486459732055664,
      "logps/chosen": -31.08384132385254,
      "logps/rejected": -33.032264709472656,
      "loss": 0.6861,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": 0.19576172530651093,
      "rewards/margins": 0.3413549065589905,
      "rewards/rejected": -0.14559319615364075,
      "step": 360
    },
    {
      "epoch": 0.96,
      "grad_norm": 20.5,
      "learning_rate": 4.706303941965804e-06,
      "logits/chosen": -1.9708837270736694,
      "logits/rejected": -1.970446228981018,
      "logps/chosen": -32.829166412353516,
      "logps/rejected": -36.46767044067383,
      "loss": 0.6072,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 0.24631083011627197,
      "rewards/margins": 0.41346845030784607,
      "rewards/rejected": -0.16715766489505768,
      "step": 370
    },
    {
      "epoch": 0.99,
      "grad_norm": 19.0,
      "learning_rate": 4.679090796681225e-06,
      "logits/chosen": -2.002481698989868,
      "logits/rejected": -1.9979221820831299,
      "logps/chosen": -30.1049747467041,
      "logps/rejected": -29.63289451599121,
      "loss": 0.6387,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.2422492504119873,
      "rewards/margins": 0.40985146164894104,
      "rewards/rejected": -0.16760216653347015,
      "step": 380
    },
    {
      "epoch": 1.01,
      "grad_norm": 22.625,
      "learning_rate": 4.650758136138454e-06,
      "logits/chosen": -1.773651123046875,
      "logits/rejected": -1.7800319194793701,
      "logps/chosen": -31.610965728759766,
      "logps/rejected": -36.748817443847656,
      "loss": 0.523,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 0.30989953875541687,
      "rewards/margins": 0.5621098279953003,
      "rewards/rejected": -0.25221019983291626,
      "step": 390
    },
    {
      "epoch": 1.04,
      "grad_norm": 17.75,
      "learning_rate": 4.621320516337559e-06,
      "logits/chosen": -1.9248863458633423,
      "logits/rejected": -1.918617844581604,
      "logps/chosen": -32.99136734008789,
      "logps/rejected": -32.704185485839844,
      "loss": 0.5492,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": 0.33653250336647034,
      "rewards/margins": 0.49465543031692505,
      "rewards/rejected": -0.15812288224697113,
      "step": 400
    },
    {
      "epoch": 1.04,
      "eval_logits/chosen": -2.1900622844696045,
      "eval_logits/rejected": -2.185243606567383,
      "eval_logps/chosen": -34.19782638549805,
      "eval_logps/rejected": -37.779701232910156,
      "eval_loss": 0.9599968194961548,
      "eval_rewards/accuracies": 0.5564784407615662,
      "eval_rewards/chosen": -0.06531065702438354,
      "eval_rewards/margins": 0.03992270678281784,
      "eval_rewards/rejected": -0.10523335635662079,
      "eval_runtime": 145.4889,
      "eval_samples_per_second": 2.358,
      "eval_steps_per_second": 0.296,
      "step": 400
    },
    {
      "epoch": 1.06,
      "grad_norm": 14.9375,
      "learning_rate": 4.590793060955158e-06,
      "logits/chosen": -1.9203557968139648,
      "logits/rejected": -1.9276371002197266,
      "logps/chosen": -28.529979705810547,
      "logps/rejected": -29.745304107666016,
      "loss": 0.5725,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 0.23615682125091553,
      "rewards/margins": 0.4949887692928314,
      "rewards/rejected": -0.2588319778442383,
      "step": 410
    },
    {
      "epoch": 1.09,
      "grad_norm": 17.0,
      "learning_rate": 4.559191453574582e-06,
      "logits/chosen": -1.9383413791656494,
      "logits/rejected": -1.9373687505722046,
      "logps/chosen": -33.60287094116211,
      "logps/rejected": -31.25448989868164,
      "loss": 0.6687,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": 0.2591347098350525,
      "rewards/margins": 0.3837122917175293,
      "rewards/rejected": -0.12457761913537979,
      "step": 420
    },
    {
      "epoch": 1.12,
      "grad_norm": 16.875,
      "learning_rate": 4.52653192962838e-06,
      "logits/chosen": -1.9323232173919678,
      "logits/rejected": -1.9150965213775635,
      "logps/chosen": -30.429235458374023,
      "logps/rejected": -33.68767547607422,
      "loss": 0.5576,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 0.21918778121471405,
      "rewards/margins": 0.5237432718276978,
      "rewards/rejected": -0.3045555651187897,
      "step": 430
    },
    {
      "epoch": 1.14,
      "grad_norm": 17.5,
      "learning_rate": 4.492831268057307e-06,
      "logits/chosen": -1.9633067846298218,
      "logits/rejected": -1.9654438495635986,
      "logps/chosen": -35.656089782714844,
      "logps/rejected": -35.58705520629883,
      "loss": 0.4182,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 0.3825095295906067,
      "rewards/margins": 0.6733442544937134,
      "rewards/rejected": -0.2908347249031067,
      "step": 440
    },
    {
      "epoch": 1.17,
      "grad_norm": 16.75,
      "learning_rate": 4.458106782690094e-06,
      "logits/chosen": -2.039092779159546,
      "logits/rejected": -2.03885817527771,
      "logps/chosen": -31.8747501373291,
      "logps/rejected": -34.06696319580078,
      "loss": 0.5685,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": 0.3034403622150421,
      "rewards/margins": 0.5061808228492737,
      "rewards/rejected": -0.2027404010295868,
      "step": 450
    },
    {
      "epoch": 1.19,
      "grad_norm": 18.0,
      "learning_rate": 4.422376313348405e-06,
      "logits/chosen": -1.9820168018341064,
      "logits/rejected": -1.974509835243225,
      "logps/chosen": -31.496234893798828,
      "logps/rejected": -36.78831100463867,
      "loss": 0.435,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 0.3638893961906433,
      "rewards/margins": 0.6575988531112671,
      "rewards/rejected": -0.2937094271183014,
      "step": 460
    },
    {
      "epoch": 1.22,
      "grad_norm": 14.75,
      "learning_rate": 4.3856582166815696e-06,
      "logits/chosen": -1.8861879110336304,
      "logits/rejected": -1.8827335834503174,
      "logps/chosen": -33.01527404785156,
      "logps/rejected": -33.59056854248047,
      "loss": 0.4878,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 0.3668093681335449,
      "rewards/margins": 0.6334856748580933,
      "rewards/rejected": -0.26667624711990356,
      "step": 470
    },
    {
      "epoch": 1.25,
      "grad_norm": 14.625,
      "learning_rate": 4.347971356735789e-06,
      "logits/chosen": -2.0107829570770264,
      "logits/rejected": -2.003845691680908,
      "logps/chosen": -30.23215103149414,
      "logps/rejected": -32.65163803100586,
      "loss": 0.5881,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": 0.2385544776916504,
      "rewards/margins": 0.5018148422241211,
      "rewards/rejected": -0.2632603049278259,
      "step": 480
    },
    {
      "epoch": 1.27,
      "grad_norm": 11.625,
      "learning_rate": 4.309335095262675e-06,
      "logits/chosen": -1.9544703960418701,
      "logits/rejected": -1.9560562372207642,
      "logps/chosen": -34.39679718017578,
      "logps/rejected": -34.34023666381836,
      "loss": 0.4545,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 0.38255801796913147,
      "rewards/margins": 0.6620479822158813,
      "rewards/rejected": -0.2794899046421051,
      "step": 490
    },
    {
      "epoch": 1.3,
      "grad_norm": 16.75,
      "learning_rate": 4.269769281772082e-06,
      "logits/chosen": -1.8426096439361572,
      "logits/rejected": -1.840172529220581,
      "logps/chosen": -32.309085845947266,
      "logps/rejected": -37.37702178955078,
      "loss": 0.4504,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 0.345283180475235,
      "rewards/margins": 0.7103637456893921,
      "rewards/rejected": -0.3650805950164795,
      "step": 500
    },
    {
      "epoch": 1.3,
      "eval_logits/chosen": -2.1818087100982666,
      "eval_logits/rejected": -2.1769938468933105,
      "eval_logps/chosen": -34.38530349731445,
      "eval_logps/rejected": -37.95327377319336,
      "eval_loss": 0.9655585289001465,
      "eval_rewards/accuracies": 0.5685215592384338,
      "eval_rewards/chosen": -0.14030161499977112,
      "eval_rewards/margins": 0.03436039760708809,
      "eval_rewards/rejected": -0.1746620088815689,
      "eval_runtime": 145.206,
      "eval_samples_per_second": 2.362,
      "eval_steps_per_second": 0.296,
      "step": 500
    },
    {
      "epoch": 1.32,
      "grad_norm": 13.4375,
      "learning_rate": 4.22929424333435e-06,
      "logits/chosen": -1.9420890808105469,
      "logits/rejected": -1.9468368291854858,
      "logps/chosen": -32.48661804199219,
      "logps/rejected": -32.22886276245117,
      "loss": 0.4787,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": 0.38829392194747925,
      "rewards/margins": 0.6213949918746948,
      "rewards/rejected": -0.23310112953186035,
      "step": 510
    },
    {
      "epoch": 1.35,
      "grad_norm": 14.8125,
      "learning_rate": 4.1879307741372085e-06,
      "logits/chosen": -1.9701076745986938,
      "logits/rejected": -1.981092095375061,
      "logps/chosen": -30.474559783935547,
      "logps/rejected": -32.24431610107422,
      "loss": 0.4762,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 0.36634185910224915,
      "rewards/margins": 0.6609948873519897,
      "rewards/rejected": -0.2946530282497406,
      "step": 520
    },
    {
      "epoch": 1.38,
      "grad_norm": 10.4375,
      "learning_rate": 4.145700124802693e-06,
      "logits/chosen": -1.897570252418518,
      "logits/rejected": -1.8943248987197876,
      "logps/chosen": -31.66860580444336,
      "logps/rejected": -33.14496994018555,
      "loss": 0.4876,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 0.30076172947883606,
      "rewards/margins": 0.6136397123336792,
      "rewards/rejected": -0.3128780126571655,
      "step": 530
    },
    {
      "epoch": 1.4,
      "grad_norm": 14.0,
      "learning_rate": 4.102623991469562e-06,
      "logits/chosen": -1.7630268335342407,
      "logits/rejected": -1.7722015380859375,
      "logps/chosen": -31.767126083374023,
      "logps/rejected": -32.539676666259766,
      "loss": 0.5367,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": 0.3209052085876465,
      "rewards/margins": 0.5943812727928162,
      "rewards/rejected": -0.27347609400749207,
      "step": 540
    },
    {
      "epoch": 1.43,
      "grad_norm": 13.9375,
      "learning_rate": 4.058724504646834e-06,
      "logits/chosen": -1.856566071510315,
      "logits/rejected": -1.8504016399383545,
      "logps/chosen": -32.787315368652344,
      "logps/rejected": -31.569156646728516,
      "loss": 0.5202,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": 0.36839646100997925,
      "rewards/margins": 0.5995180606842041,
      "rewards/rejected": -0.23112154006958008,
      "step": 550
    },
    {
      "epoch": 1.45,
      "grad_norm": 13.5625,
      "learning_rate": 4.014024217844167e-06,
      "logits/chosen": -1.943418264389038,
      "logits/rejected": -1.941474199295044,
      "logps/chosen": -33.62010192871094,
      "logps/rejected": -32.09187316894531,
      "loss": 0.5174,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": 0.3804393410682678,
      "rewards/margins": 0.6182351112365723,
      "rewards/rejected": -0.23779578506946564,
      "step": 560
    },
    {
      "epoch": 1.48,
      "grad_norm": 13.9375,
      "learning_rate": 3.968546095984911e-06,
      "logits/chosen": -1.778063416481018,
      "logits/rejected": -1.7760202884674072,
      "logps/chosen": -31.914138793945312,
      "logps/rejected": -31.619396209716797,
      "loss": 0.5162,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": 0.352909654378891,
      "rewards/margins": 0.5997473001480103,
      "rewards/rejected": -0.24683766067028046,
      "step": 570
    },
    {
      "epoch": 1.51,
      "grad_norm": 18.25,
      "learning_rate": 3.922313503607806e-06,
      "logits/chosen": -1.9141117334365845,
      "logits/rejected": -1.910735845565796,
      "logps/chosen": -30.1756591796875,
      "logps/rejected": -35.52814865112305,
      "loss": 0.441,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 0.37159496545791626,
      "rewards/margins": 0.7323898077011108,
      "rewards/rejected": -0.36079490184783936,
      "step": 580
    },
    {
      "epoch": 1.53,
      "grad_norm": 12.5,
      "learning_rate": 3.875350192863368e-06,
      "logits/chosen": -1.8503096103668213,
      "logits/rejected": -1.8538156747817993,
      "logps/chosen": -28.902246475219727,
      "logps/rejected": -31.208913803100586,
      "loss": 0.5458,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 0.2851153612136841,
      "rewards/margins": 0.5051326751708984,
      "rewards/rejected": -0.22001728415489197,
      "step": 590
    },
    {
      "epoch": 1.56,
      "grad_norm": 14.6875,
      "learning_rate": 3.8276802913111436e-06,
      "logits/chosen": -1.8863645792007446,
      "logits/rejected": -1.8863203525543213,
      "logps/chosen": -31.125957489013672,
      "logps/rejected": -31.79779624938965,
      "loss": 0.4511,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 0.4382299780845642,
      "rewards/margins": 0.6801418662071228,
      "rewards/rejected": -0.2419118881225586,
      "step": 600
    },
    {
      "epoch": 1.56,
      "eval_logits/chosen": -2.161299228668213,
      "eval_logits/rejected": -2.1565115451812744,
      "eval_logps/chosen": -34.24331283569336,
      "eval_logps/rejected": -37.89296340942383,
      "eval_loss": 0.9337846636772156,
      "eval_rewards/accuracies": 0.595099687576294,
      "eval_rewards/chosen": -0.08350436389446259,
      "eval_rewards/margins": 0.06703372299671173,
      "eval_rewards/rejected": -0.1505381017923355,
      "eval_runtime": 145.1637,
      "eval_samples_per_second": 2.363,
      "eval_steps_per_second": 0.296,
      "step": 600
    },
    {
      "epoch": 1.58,
      "grad_norm": 18.625,
      "learning_rate": 3.7793282895240927e-06,
      "logits/chosen": -1.9492238759994507,
      "logits/rejected": -1.950005292892456,
      "logps/chosen": -33.86152648925781,
      "logps/rejected": -33.876121520996094,
      "loss": 0.4213,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": 0.4674473702907562,
      "rewards/margins": 0.7842208743095398,
      "rewards/rejected": -0.3167734146118164,
      "step": 610
    },
    {
      "epoch": 1.61,
      "grad_norm": 16.25,
      "learning_rate": 3.730319028506478e-06,
      "logits/chosen": -1.9146831035614014,
      "logits/rejected": -1.912245750427246,
      "logps/chosen": -32.04179000854492,
      "logps/rejected": -32.73381042480469,
      "loss": 0.4604,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 0.4334433972835541,
      "rewards/margins": 0.6990013122558594,
      "rewards/rejected": -0.2655579149723053,
      "step": 620
    },
    {
      "epoch": 1.64,
      "grad_norm": 21.25,
      "learning_rate": 3.6806776869317074e-06,
      "logits/chosen": -1.9271072149276733,
      "logits/rejected": -1.9181263446807861,
      "logps/chosen": -31.6562557220459,
      "logps/rejected": -31.68838119506836,
      "loss": 0.4662,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": 0.44794517755508423,
      "rewards/margins": 0.7491516470909119,
      "rewards/rejected": -0.30120649933815,
      "step": 630
    },
    {
      "epoch": 1.66,
      "grad_norm": 19.25,
      "learning_rate": 3.6304297682067146e-06,
      "logits/chosen": -1.928035020828247,
      "logits/rejected": -1.9247719049453735,
      "logps/chosen": -31.21347427368164,
      "logps/rejected": -33.109100341796875,
      "loss": 0.4472,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 0.3939800262451172,
      "rewards/margins": 0.669854462146759,
      "rewards/rejected": -0.27587443590164185,
      "step": 640
    },
    {
      "epoch": 1.69,
      "grad_norm": 17.125,
      "learning_rate": 3.579601087369492e-06,
      "logits/chosen": -1.936981201171875,
      "logits/rejected": -1.9392879009246826,
      "logps/chosen": -32.55559158325195,
      "logps/rejected": -34.394744873046875,
      "loss": 0.3874,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 0.4392463266849518,
      "rewards/margins": 0.7237855792045593,
      "rewards/rejected": -0.28453919291496277,
      "step": 650
    },
    {
      "epoch": 1.71,
      "grad_norm": 22.75,
      "learning_rate": 3.5282177578265295e-06,
      "logits/chosen": -1.848418951034546,
      "logits/rejected": -1.8488578796386719,
      "logps/chosen": -32.838523864746094,
      "logps/rejected": -32.17537307739258,
      "loss": 0.4407,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 0.44332486391067505,
      "rewards/margins": 0.6705711483955383,
      "rewards/rejected": -0.22724632918834686,
      "step": 660
    },
    {
      "epoch": 1.74,
      "grad_norm": 21.0,
      "learning_rate": 3.476306177936961e-06,
      "logits/chosen": -1.8874480724334717,
      "logits/rejected": -1.8778259754180908,
      "logps/chosen": -32.84857940673828,
      "logps/rejected": -33.05742645263672,
      "loss": 0.382,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 0.4610818326473236,
      "rewards/margins": 0.7609356641769409,
      "rewards/rejected": -0.29985374212265015,
      "step": 670
    },
    {
      "epoch": 1.77,
      "grad_norm": 19.0,
      "learning_rate": 3.423893017450324e-06,
      "logits/chosen": -1.7845916748046875,
      "logits/rejected": -1.7813985347747803,
      "logps/chosen": -30.31686782836914,
      "logps/rejected": -35.026180267333984,
      "loss": 0.3954,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 0.45763739943504333,
      "rewards/margins": 0.7406758069992065,
      "rewards/rejected": -0.2830384373664856,
      "step": 680
    },
    {
      "epoch": 1.79,
      "grad_norm": 14.75,
      "learning_rate": 3.3710052038048794e-06,
      "logits/chosen": -1.8450958728790283,
      "logits/rejected": -1.8443653583526611,
      "logps/chosen": -33.978050231933594,
      "logps/rejected": -36.23003387451172,
      "loss": 0.3186,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.5168943405151367,
      "rewards/margins": 0.8673771619796753,
      "rewards/rejected": -0.35048285126686096,
      "step": 690
    },
    {
      "epoch": 1.82,
      "grad_norm": 11.9375,
      "learning_rate": 3.3176699082935546e-06,
      "logits/chosen": -1.8192574977874756,
      "logits/rejected": -1.822011947631836,
      "logps/chosen": -31.353174209594727,
      "logps/rejected": -36.575157165527344,
      "loss": 0.3805,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 0.4998700022697449,
      "rewards/margins": 0.873264491558075,
      "rewards/rejected": -0.37339454889297485,
      "step": 700
    },
    {
      "epoch": 1.82,
      "eval_logits/chosen": -2.1489837169647217,
      "eval_logits/rejected": -2.1442174911499023,
      "eval_logps/chosen": -34.29972457885742,
      "eval_logps/rejected": -37.9394416809082,
      "eval_loss": 0.9385247826576233,
      "eval_rewards/accuracies": 0.5510797500610352,
      "eval_rewards/chosen": -0.10606933385133743,
      "eval_rewards/margins": 0.06305938214063644,
      "eval_rewards/rejected": -0.16912870109081268,
      "eval_runtime": 145.1991,
      "eval_samples_per_second": 2.362,
      "eval_steps_per_second": 0.296,
      "step": 700
    },
    {
      "epoch": 1.84,
      "grad_norm": 16.0,
      "learning_rate": 3.2639145321045933e-06,
      "logits/chosen": -1.9265644550323486,
      "logits/rejected": -1.9291988611221313,
      "logps/chosen": -33.78008270263672,
      "logps/rejected": -34.95719909667969,
      "loss": 0.4644,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 0.44620281457901,
      "rewards/margins": 0.7210227847099304,
      "rewards/rejected": -0.2748199999332428,
      "step": 710
    },
    {
      "epoch": 1.87,
      "grad_norm": 20.875,
      "learning_rate": 3.2097666922441107e-06,
      "logits/chosen": -1.7806246280670166,
      "logits/rejected": -1.7747358083724976,
      "logps/chosen": -33.649070739746094,
      "logps/rejected": -33.15039825439453,
      "loss": 0.4038,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 0.4736657738685608,
      "rewards/margins": 0.7930890917778015,
      "rewards/rejected": -0.3194233477115631,
      "step": 720
    },
    {
      "epoch": 1.9,
      "grad_norm": 12.9375,
      "learning_rate": 3.1552542073477554e-06,
      "logits/chosen": -1.9554284811019897,
      "logits/rejected": -1.9523767232894897,
      "logps/chosen": -29.604736328125,
      "logps/rejected": -32.440574645996094,
      "loss": 0.3749,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.4192999005317688,
      "rewards/margins": 0.8162514567375183,
      "rewards/rejected": -0.39695149660110474,
      "step": 730
    },
    {
      "epoch": 1.92,
      "grad_norm": 16.125,
      "learning_rate": 3.100405083388799e-06,
      "logits/chosen": -1.7934890985488892,
      "logits/rejected": -1.7935874462127686,
      "logps/chosen": -32.12221145629883,
      "logps/rejected": -38.396541595458984,
      "loss": 0.3706,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.5363865494728088,
      "rewards/margins": 0.8438798785209656,
      "rewards/rejected": -0.3074934482574463,
      "step": 740
    },
    {
      "epoch": 1.95,
      "grad_norm": 9.125,
      "learning_rate": 3.0452474992899645e-06,
      "logits/chosen": -1.6800241470336914,
      "logits/rejected": -1.685310959815979,
      "logps/chosen": -35.7508659362793,
      "logps/rejected": -34.849327087402344,
      "loss": 0.4332,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": 0.47263655066490173,
      "rewards/margins": 0.8234331011772156,
      "rewards/rejected": -0.35079658031463623,
      "step": 750
    },
    {
      "epoch": 1.97,
      "grad_norm": 11.0,
      "learning_rate": 2.989809792446417e-06,
      "logits/chosen": -1.8796085119247437,
      "logits/rejected": -1.8812284469604492,
      "logps/chosen": -31.44708251953125,
      "logps/rejected": -33.469627380371094,
      "loss": 0.3873,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 0.46344003081321716,
      "rewards/margins": 0.7396036982536316,
      "rewards/rejected": -0.27616366744041443,
      "step": 760
    },
    {
      "epoch": 2.0,
      "grad_norm": 20.25,
      "learning_rate": 2.9341204441673267e-06,
      "logits/chosen": -1.8549083471298218,
      "logits/rejected": -1.8539108037948608,
      "logps/chosen": -30.935928344726562,
      "logps/rejected": -35.55149841308594,
      "loss": 0.4385,
      "rewards/accuracies": 0.8708333969116211,
      "rewards/chosen": 0.4958726763725281,
      "rewards/margins": 0.7052101492881775,
      "rewards/rejected": -0.20933744311332703,
      "step": 770
    },
    {
      "epoch": 2.03,
      "grad_norm": 12.8125,
      "learning_rate": 2.878208065043501e-06,
      "logits/chosen": -1.8549137115478516,
      "logits/rejected": -1.8542522192001343,
      "logps/chosen": -33.281410217285156,
      "logps/rejected": -32.49152755737305,
      "loss": 0.2642,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.546822190284729,
      "rewards/margins": 0.9756248593330383,
      "rewards/rejected": -0.4288027286529541,
      "step": 780
    },
    {
      "epoch": 2.05,
      "grad_norm": 13.0,
      "learning_rate": 2.8221013802485974e-06,
      "logits/chosen": -1.8834869861602783,
      "logits/rejected": -1.8820085525512695,
      "logps/chosen": -28.202373504638672,
      "logps/rejected": -33.88237380981445,
      "loss": 0.1975,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": 0.5443224906921387,
      "rewards/margins": 1.0427463054656982,
      "rewards/rejected": -0.4984237253665924,
      "step": 790
    },
    {
      "epoch": 2.08,
      "grad_norm": 0.006591796875,
      "learning_rate": 2.76582921478147e-06,
      "logits/chosen": -1.9375505447387695,
      "logits/rejected": -1.9341075420379639,
      "logps/chosen": -30.97635269165039,
      "logps/rejected": -35.407142639160156,
      "loss": 0.2038,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 0.5803438425064087,
      "rewards/margins": 1.0635874271392822,
      "rewards/rejected": -0.4832436144351959,
      "step": 800
    },
    {
      "epoch": 2.08,
      "eval_logits/chosen": -2.1414618492126465,
      "eval_logits/rejected": -2.1367087364196777,
      "eval_logps/chosen": -34.35294723510742,
      "eval_logps/rejected": -38.00876235961914,
      "eval_loss": 0.9334202408790588,
      "eval_rewards/accuracies": 0.5689368844032288,
      "eval_rewards/chosen": -0.12735822796821594,
      "eval_rewards/margins": 0.06950045377016068,
      "eval_rewards/rejected": -0.19685867428779602,
      "eval_runtime": 145.1666,
      "eval_samples_per_second": 2.363,
      "eval_steps_per_second": 0.296,
      "step": 800
    },
    {
      "epoch": 2.1,
      "grad_norm": 11.0,
      "learning_rate": 2.7094204786572254e-06,
      "logits/chosen": -1.7686500549316406,
      "logits/rejected": -1.7609798908233643,
      "logps/chosen": -32.71022033691406,
      "logps/rejected": -35.735992431640625,
      "loss": 0.1833,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.6984082460403442,
      "rewards/margins": 1.1687581539154053,
      "rewards/rejected": -0.4703500270843506,
      "step": 810
    },
    {
      "epoch": 2.13,
      "grad_norm": 14.125,
      "learning_rate": 2.6529041520546072e-06,
      "logits/chosen": -1.8466291427612305,
      "logits/rejected": -1.8570162057876587,
      "logps/chosen": -34.17173385620117,
      "logps/rejected": -33.32606506347656,
      "loss": 0.28,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.6095821261405945,
      "rewards/margins": 0.9674333333969116,
      "rewards/rejected": -0.35785120725631714,
      "step": 820
    },
    {
      "epoch": 2.16,
      "grad_norm": 16.5,
      "learning_rate": 2.5963092704273302e-06,
      "logits/chosen": -1.8971784114837646,
      "logits/rejected": -1.9018369913101196,
      "logps/chosen": -33.51736831665039,
      "logps/rejected": -30.209964752197266,
      "loss": 0.2911,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 0.5541660189628601,
      "rewards/margins": 0.9308546781539917,
      "rewards/rejected": -0.3766886591911316,
      "step": 830
    },
    {
      "epoch": 2.18,
      "grad_norm": 12.9375,
      "learning_rate": 2.53966490958702e-06,
      "logits/chosen": -1.8965809345245361,
      "logits/rejected": -1.9047677516937256,
      "logps/chosen": -33.255287170410156,
      "logps/rejected": -30.980331420898438,
      "loss": 0.2265,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": 0.6130979061126709,
      "rewards/margins": 1.0661498308181763,
      "rewards/rejected": -0.4530518651008606,
      "step": 840
    },
    {
      "epoch": 2.21,
      "grad_norm": 13.375,
      "learning_rate": 2.4830001707654135e-06,
      "logits/chosen": -1.8259570598602295,
      "logits/rejected": -1.8164736032485962,
      "logps/chosen": -30.39957618713379,
      "logps/rejected": -33.05730056762695,
      "loss": 0.2609,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.5574477314949036,
      "rewards/margins": 0.9748461842536926,
      "rewards/rejected": -0.4173983633518219,
      "step": 850
    },
    {
      "epoch": 2.23,
      "grad_norm": 11.875,
      "learning_rate": 2.4263441656635054e-06,
      "logits/chosen": -1.9653680324554443,
      "logits/rejected": -1.9555402994155884,
      "logps/chosen": -24.850261688232422,
      "logps/rejected": -31.08827781677246,
      "loss": 0.2561,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 0.5495240688323975,
      "rewards/margins": 1.035900354385376,
      "rewards/rejected": -0.48637622594833374,
      "step": 860
    },
    {
      "epoch": 2.26,
      "grad_norm": 13.1875,
      "learning_rate": 2.3697260014953107e-06,
      "logits/chosen": -1.8186180591583252,
      "logits/rejected": -1.819954514503479,
      "logps/chosen": -32.77277374267578,
      "logps/rejected": -31.007610321044922,
      "loss": 0.218,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.6240935325622559,
      "rewards/margins": 1.0341370105743408,
      "rewards/rejected": -0.41004353761672974,
      "step": 870
    },
    {
      "epoch": 2.29,
      "grad_norm": 12.25,
      "learning_rate": 2.3131747660339396e-06,
      "logits/chosen": -1.8255703449249268,
      "logits/rejected": -1.826511025428772,
      "logps/chosen": -31.154870986938477,
      "logps/rejected": -34.21940994262695,
      "loss": 0.3146,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.4937514662742615,
      "rewards/margins": 0.9611461758613586,
      "rewards/rejected": -0.4673948287963867,
      "step": 880
    },
    {
      "epoch": 2.31,
      "grad_norm": 11.6875,
      "learning_rate": 2.256719512667651e-06,
      "logits/chosen": -1.7308677434921265,
      "logits/rejected": -1.7293297052383423,
      "logps/chosen": -34.149600982666016,
      "logps/rejected": -37.341552734375,
      "loss": 0.2179,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.6061262488365173,
      "rewards/margins": 1.1918519735336304,
      "rewards/rejected": -0.5857258439064026,
      "step": 890
    },
    {
      "epoch": 2.34,
      "grad_norm": 8.375,
      "learning_rate": 2.2003892454735786e-06,
      "logits/chosen": -1.8834432363510132,
      "logits/rejected": -1.8766663074493408,
      "logps/chosen": -30.528228759765625,
      "logps/rejected": -33.975677490234375,
      "loss": 0.2332,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": 0.5897632837295532,
      "rewards/margins": 1.076806664466858,
      "rewards/rejected": -0.4870433807373047,
      "step": 900
    },
    {
      "epoch": 2.34,
      "eval_logits/chosen": -2.139965057373047,
      "eval_logits/rejected": -2.135190963745117,
      "eval_logps/chosen": -34.37273025512695,
      "eval_logps/rejected": -38.030879974365234,
      "eval_loss": 0.9319750070571899,
      "eval_rewards/accuracies": 0.5718438625335693,
      "eval_rewards/chosen": -0.13527235388755798,
      "eval_rewards/margins": 0.0704314336180687,
      "eval_rewards/rejected": -0.20570378005504608,
      "eval_runtime": 145.1066,
      "eval_samples_per_second": 2.364,
      "eval_steps_per_second": 0.296,
      "step": 900
    },
    {
      "epoch": 2.36,
      "grad_norm": 13.6875,
      "learning_rate": 2.1442129043167877e-06,
      "logits/chosen": -1.922201156616211,
      "logits/rejected": -1.9175071716308594,
      "logps/chosen": -32.03234100341797,
      "logps/rejected": -35.83629608154297,
      "loss": 0.213,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.6101375818252563,
      "rewards/margins": 1.1332075595855713,
      "rewards/rejected": -0.5230700373649597,
      "step": 910
    },
    {
      "epoch": 2.39,
      "grad_norm": 10.5625,
      "learning_rate": 2.088219349982323e-06,
      "logits/chosen": -1.8422584533691406,
      "logits/rejected": -1.8473327159881592,
      "logps/chosen": -33.719722747802734,
      "logps/rejected": -33.628021240234375,
      "loss": 0.2188,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.6515445709228516,
      "rewards/margins": 1.1027911901474,
      "rewards/rejected": -0.4512465000152588,
      "step": 920
    },
    {
      "epoch": 2.42,
      "grad_norm": 9.4375,
      "learning_rate": 2.0324373493478803e-06,
      "logits/chosen": -1.9423255920410156,
      "logits/rejected": -1.9330682754516602,
      "logps/chosen": -30.859844207763672,
      "logps/rejected": -35.690673828125,
      "loss": 0.2137,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": 0.5700357556343079,
      "rewards/margins": 1.0691864490509033,
      "rewards/rejected": -0.49915066361427307,
      "step": 930
    },
    {
      "epoch": 2.44,
      "grad_norm": 15.125,
      "learning_rate": 1.976895560604729e-06,
      "logits/chosen": -1.8686740398406982,
      "logits/rejected": -1.8654989004135132,
      "logps/chosen": -30.049386978149414,
      "logps/rejected": -33.48214340209961,
      "loss": 0.321,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 0.49562782049179077,
      "rewards/margins": 0.9023343920707703,
      "rewards/rejected": -0.40670666098594666,
      "step": 940
    },
    {
      "epoch": 2.47,
      "grad_norm": 4.75,
      "learning_rate": 1.921622518534466e-06,
      "logits/chosen": -1.8089139461517334,
      "logits/rejected": -1.8163315057754517,
      "logps/chosen": -31.528472900390625,
      "logps/rejected": -36.56946563720703,
      "loss": 0.3027,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 0.5370327234268188,
      "rewards/margins": 0.9448447227478027,
      "rewards/rejected": -0.4078119397163391,
      "step": 950
    },
    {
      "epoch": 2.49,
      "grad_norm": 10.375,
      "learning_rate": 1.8666466198491794e-06,
      "logits/chosen": -1.8195326328277588,
      "logits/rejected": -1.8125743865966797,
      "logps/chosen": -32.26793670654297,
      "logps/rejected": -37.91437530517578,
      "loss": 0.2086,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": 0.6242316961288452,
      "rewards/margins": 1.1504604816436768,
      "rewards/rejected": -0.5262287259101868,
      "step": 960
    },
    {
      "epoch": 2.52,
      "grad_norm": 12.875,
      "learning_rate": 1.8119961086025376e-06,
      "logits/chosen": -1.8216333389282227,
      "logits/rejected": -1.8221938610076904,
      "logps/chosen": -29.730484008789062,
      "logps/rejected": -34.02881622314453,
      "loss": 0.2371,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.602368175983429,
      "rewards/margins": 1.021390438079834,
      "rewards/rejected": -0.41902226209640503,
      "step": 970
    },
    {
      "epoch": 2.55,
      "grad_norm": 14.0625,
      "learning_rate": 1.7576990616793139e-06,
      "logits/chosen": -1.8246610164642334,
      "logits/rejected": -1.8358711004257202,
      "logps/chosen": -31.346548080444336,
      "logps/rejected": -35.217620849609375,
      "loss": 0.2028,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 0.6920571327209473,
      "rewards/margins": 1.1266204118728638,
      "rewards/rejected": -0.43456321954727173,
      "step": 980
    },
    {
      "epoch": 2.57,
      "grad_norm": 5.0,
      "learning_rate": 1.7037833743707892e-06,
      "logits/chosen": -1.8958324193954468,
      "logits/rejected": -1.893398642539978,
      "logps/chosen": -34.60765838623047,
      "logps/rejected": -33.333709716796875,
      "loss": 0.3265,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 0.605356752872467,
      "rewards/margins": 0.9791496396064758,
      "rewards/rejected": -0.3737927973270416,
      "step": 990
    },
    {
      "epoch": 2.6,
      "grad_norm": 12.6875,
      "learning_rate": 1.6502767460434588e-06,
      "logits/chosen": -1.8685047626495361,
      "logits/rejected": -1.8726377487182617,
      "logps/chosen": -33.05529022216797,
      "logps/rejected": -35.44232177734375,
      "loss": 0.28,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.5631993412971497,
      "rewards/margins": 0.9457098245620728,
      "rewards/rejected": -0.3825104832649231,
      "step": 1000
    },
    {
      "epoch": 2.6,
      "eval_logits/chosen": -2.139305830001831,
      "eval_logits/rejected": -2.134549379348755,
      "eval_logps/chosen": -34.397972106933594,
      "eval_logps/rejected": -38.07229995727539,
      "eval_loss": 0.9270963668823242,
      "eval_rewards/accuracies": 0.5714285373687744,
      "eval_rewards/chosen": -0.14536714553833008,
      "eval_rewards/margins": 0.07690288871526718,
      "eval_rewards/rejected": -0.22227002680301666,
      "eval_runtime": 145.3536,
      "eval_samples_per_second": 2.36,
      "eval_steps_per_second": 0.296,
      "step": 1000
    },
    {
      "epoch": 2.62,
      "grad_norm": 17.5,
      "learning_rate": 1.5972066659083796e-06,
      "logits/chosen": -1.8788774013519287,
      "logits/rejected": -1.8837945461273193,
      "logps/chosen": -31.51852035522461,
      "logps/rejected": -33.45092010498047,
      "loss": 0.3491,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 0.5127608180046082,
      "rewards/margins": 0.8846257328987122,
      "rewards/rejected": -0.3718649446964264,
      "step": 1010
    },
    {
      "epoch": 2.65,
      "grad_norm": 13.5,
      "learning_rate": 1.5446003988985041e-06,
      "logits/chosen": -1.8375968933105469,
      "logits/rejected": -1.8401010036468506,
      "logps/chosen": -29.037750244140625,
      "logps/rejected": -32.918941497802734,
      "loss": 0.198,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": 0.563154935836792,
      "rewards/margins": 1.0510679483413696,
      "rewards/rejected": -0.48791298270225525,
      "step": 1020
    },
    {
      "epoch": 2.68,
      "grad_norm": 8.5625,
      "learning_rate": 1.4924849716612211e-06,
      "logits/chosen": -1.8911283016204834,
      "logits/rejected": -1.8856918811798096,
      "logps/chosen": -32.591796875,
      "logps/rejected": -34.6553955078125,
      "loss": 0.2597,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": 0.5589276552200317,
      "rewards/margins": 1.0138864517211914,
      "rewards/rejected": -0.4549587368965149,
      "step": 1030
    },
    {
      "epoch": 2.7,
      "grad_norm": 15.8125,
      "learning_rate": 1.440887158673332e-06,
      "logits/chosen": -1.8267465829849243,
      "logits/rejected": -1.8307063579559326,
      "logps/chosen": -34.41118240356445,
      "logps/rejected": -35.5274543762207,
      "loss": 0.252,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": 0.5370186567306519,
      "rewards/margins": 0.9801691174507141,
      "rewards/rejected": -0.4431503713130951,
      "step": 1040
    },
    {
      "epoch": 2.73,
      "grad_norm": 11.9375,
      "learning_rate": 1.3898334684855647e-06,
      "logits/chosen": -1.8240203857421875,
      "logits/rejected": -1.8369216918945312,
      "logps/chosen": -30.501928329467773,
      "logps/rejected": -34.156166076660156,
      "loss": 0.251,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.5443831086158752,
      "rewards/margins": 0.9988336563110352,
      "rewards/rejected": -0.4544506072998047,
      "step": 1050
    },
    {
      "epoch": 2.75,
      "grad_norm": 14.125,
      "learning_rate": 1.3393501301037245e-06,
      "logits/chosen": -1.814233422279358,
      "logits/rejected": -1.807539701461792,
      "logps/chosen": -30.65509605407715,
      "logps/rejected": -34.37706756591797,
      "loss": 0.2287,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": 0.6016949415206909,
      "rewards/margins": 1.0253394842147827,
      "rewards/rejected": -0.4236446022987366,
      "step": 1060
    },
    {
      "epoch": 2.78,
      "grad_norm": 16.375,
      "learning_rate": 1.2894630795134454e-06,
      "logits/chosen": -1.9399135112762451,
      "logits/rejected": -1.940683126449585,
      "logps/chosen": -32.10429763793945,
      "logps/rejected": -33.903560638427734,
      "loss": 0.2239,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.5771129131317139,
      "rewards/margins": 1.0006603002548218,
      "rewards/rejected": -0.4235473573207855,
      "step": 1070
    },
    {
      "epoch": 2.81,
      "grad_norm": 12.3125,
      "learning_rate": 1.2401979463554984e-06,
      "logits/chosen": -1.8709838390350342,
      "logits/rejected": -1.8698453903198242,
      "logps/chosen": -32.744606018066406,
      "logps/rejected": -34.11382293701172,
      "loss": 0.3118,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 0.5380905866622925,
      "rewards/margins": 0.906886100769043,
      "rewards/rejected": -0.3687955439090729,
      "step": 1080
    },
    {
      "epoch": 2.83,
      "grad_norm": 10.4375,
      "learning_rate": 1.1915800407584705e-06,
      "logits/chosen": -1.8952579498291016,
      "logits/rejected": -1.8877201080322266,
      "logps/chosen": -32.717063903808594,
      "logps/rejected": -31.973791122436523,
      "loss": 0.239,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.6035231947898865,
      "rewards/margins": 0.9764264822006226,
      "rewards/rejected": -0.3729034662246704,
      "step": 1090
    },
    {
      "epoch": 2.86,
      "grad_norm": 7.875,
      "learning_rate": 1.1436343403356019e-06,
      "logits/chosen": -1.8662681579589844,
      "logits/rejected": -1.8652747869491577,
      "logps/chosen": -33.863346099853516,
      "logps/rejected": -37.67915344238281,
      "loss": 0.1953,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": 0.622026801109314,
      "rewards/margins": 1.152605652809143,
      "rewards/rejected": -0.5305787324905396,
      "step": 1100
    },
    {
      "epoch": 2.86,
      "eval_logits/chosen": -2.1389670372009277,
      "eval_logits/rejected": -2.134211301803589,
      "eval_logps/chosen": -34.42226791381836,
      "eval_logps/rejected": -38.063568115234375,
      "eval_loss": 0.9399018287658691,
      "eval_rewards/accuracies": 0.5631229281425476,
      "eval_rewards/chosen": -0.1550864279270172,
      "eval_rewards/margins": 0.06369131803512573,
      "eval_rewards/rejected": -0.21877776086330414,
      "eval_runtime": 145.1823,
      "eval_samples_per_second": 2.363,
      "eval_steps_per_second": 0.296,
      "step": 1100
    },
    {
      "epoch": 2.88,
      "grad_norm": 7.40625,
      "learning_rate": 1.0963854773524548e-06,
      "logits/chosen": -1.9152100086212158,
      "logits/rejected": -1.9206510782241821,
      "logps/chosen": -34.14377975463867,
      "logps/rejected": -36.70298767089844,
      "loss": 0.2774,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 0.5293335914611816,
      "rewards/margins": 0.9930937886238098,
      "rewards/rejected": -0.4637601971626282,
      "step": 1110
    },
    {
      "epoch": 2.91,
      "grad_norm": 8.5625,
      "learning_rate": 1.049857726072005e-06,
      "logits/chosen": -1.885117769241333,
      "logits/rejected": -1.8830986022949219,
      "logps/chosen": -31.27766990661621,
      "logps/rejected": -34.183876037597656,
      "loss": 0.2761,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.5354140996932983,
      "rewards/margins": 1.0158647298812866,
      "rewards/rejected": -0.4804508090019226,
      "step": 1120
    },
    {
      "epoch": 2.94,
      "grad_norm": 10.4375,
      "learning_rate": 1.0040749902836508e-06,
      "logits/chosen": -1.7881187200546265,
      "logits/rejected": -1.790825605392456,
      "logps/chosen": -28.32158851623535,
      "logps/rejected": -31.530282974243164,
      "loss": 0.1986,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.5708181262016296,
      "rewards/margins": 1.0849167108535767,
      "rewards/rejected": -0.514098584651947,
      "step": 1130
    },
    {
      "epoch": 2.96,
      "grad_norm": 13.9375,
      "learning_rate": 9.59060791022566e-07,
      "logits/chosen": -1.886063814163208,
      "logits/rejected": -1.882871389389038,
      "logps/chosen": -31.68621826171875,
      "logps/rejected": -33.42014694213867,
      "loss": 0.249,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 0.6070117354393005,
      "rewards/margins": 0.9643187522888184,
      "rewards/rejected": -0.35730695724487305,
      "step": 1140
    },
    {
      "epoch": 2.99,
      "grad_norm": 14.4375,
      "learning_rate": 9.148382544856885e-07,
      "logits/chosen": -1.8054192066192627,
      "logits/rejected": -1.7991243600845337,
      "logps/chosen": -26.93368911743164,
      "logps/rejected": -33.427791595458984,
      "loss": 0.2445,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": 0.4817207455635071,
      "rewards/margins": 0.993517279624939,
      "rewards/rejected": -0.5117965936660767,
      "step": 1150
    },
    {
      "epoch": 3.01,
      "grad_norm": 10.75,
      "learning_rate": 8.714301001505568e-07,
      "logits/chosen": -1.9188390970230103,
      "logits/rejected": -1.9163001775741577,
      "logps/chosen": -31.702442169189453,
      "logps/rejected": -36.311798095703125,
      "loss": 0.2506,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": 0.557451069355011,
      "rewards/margins": 1.0455574989318848,
      "rewards/rejected": -0.4881063997745514,
      "step": 1160
    },
    {
      "epoch": 3.04,
      "grad_norm": 6.78125,
      "learning_rate": 8.288586291031025e-07,
      "logits/chosen": -1.9051227569580078,
      "logits/rejected": -1.9030368328094482,
      "logps/chosen": -30.17287254333496,
      "logps/rejected": -33.75264358520508,
      "loss": 0.1641,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": 0.5766139030456543,
      "rewards/margins": 1.1320630311965942,
      "rewards/rejected": -0.5554491877555847,
      "step": 1170
    },
    {
      "epoch": 3.06,
      "grad_norm": 12.4375,
      "learning_rate": 7.871457125803897e-07,
      "logits/chosen": -1.8981797695159912,
      "logits/rejected": -1.8859472274780273,
      "logps/chosen": -34.58381271362305,
      "logps/rejected": -35.41051483154297,
      "loss": 0.1915,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": 0.5845947861671448,
      "rewards/margins": 1.1202127933502197,
      "rewards/rejected": -0.535618007183075,
      "step": 1180
    },
    {
      "epoch": 3.09,
      "grad_norm": 14.5625,
      "learning_rate": 7.463127807341966e-07,
      "logits/chosen": -1.7928569316864014,
      "logits/rejected": -1.7875295877456665,
      "logps/chosen": -33.692710876464844,
      "logps/rejected": -35.34910583496094,
      "loss": 0.1739,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.6627923250198364,
      "rewards/margins": 1.1073065996170044,
      "rewards/rejected": -0.4445141851902008,
      "step": 1190
    },
    {
      "epoch": 3.12,
      "grad_norm": 9.875,
      "learning_rate": 7.063808116212021e-07,
      "logits/chosen": -1.838303804397583,
      "logits/rejected": -1.8414958715438843,
      "logps/chosen": -31.297800064086914,
      "logps/rejected": -32.53749084472656,
      "loss": 0.2936,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 0.526917576789856,
      "rewards/margins": 0.9474434852600098,
      "rewards/rejected": -0.4205259382724762,
      "step": 1200
    },
    {
      "epoch": 3.12,
      "eval_logits/chosen": -2.1406960487365723,
      "eval_logits/rejected": -2.1359305381774902,
      "eval_logps/chosen": -34.38763427734375,
      "eval_logps/rejected": -38.051002502441406,
      "eval_loss": 0.9310786128044128,
      "eval_rewards/accuracies": 0.5863787531852722,
      "eval_rewards/chosen": -0.14123326539993286,
      "eval_rewards/margins": 0.07251953333616257,
      "eval_rewards/rejected": -0.21375279128551483,
      "eval_runtime": 145.1819,
      "eval_samples_per_second": 2.363,
      "eval_steps_per_second": 0.296,
      "step": 1200
    },
    {
      "epoch": 3.14,
      "grad_norm": 13.125,
      "learning_rate": 6.673703204254348e-07,
      "logits/chosen": -1.8620998859405518,
      "logits/rejected": -1.857629418373108,
      "logps/chosen": -29.83384132385254,
      "logps/rejected": -32.17254638671875,
      "loss": 0.1952,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.5759321451187134,
      "rewards/margins": 1.0817142724990845,
      "rewards/rejected": -0.5057822465896606,
      "step": 1210
    },
    {
      "epoch": 3.17,
      "grad_norm": 10.9375,
      "learning_rate": 6.293013489185315e-07,
      "logits/chosen": -1.8760480880737305,
      "logits/rejected": -1.870548963546753,
      "logps/chosen": -33.693180084228516,
      "logps/rejected": -35.85017013549805,
      "loss": 0.201,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.6323033571243286,
      "rewards/margins": 1.1444201469421387,
      "rewards/rejected": -0.5121166706085205,
      "step": 1220
    },
    {
      "epoch": 3.19,
      "grad_norm": 8.75,
      "learning_rate": 5.921934551632086e-07,
      "logits/chosen": -1.8780237436294556,
      "logits/rejected": -1.8647384643554688,
      "logps/chosen": -31.742191314697266,
      "logps/rejected": -35.399742126464844,
      "loss": 0.2536,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 0.5407308340072632,
      "rewards/margins": 1.0526906251907349,
      "rewards/rejected": -0.5119598507881165,
      "step": 1230
    },
    {
      "epoch": 3.22,
      "grad_norm": 7.96875,
      "learning_rate": 5.560657034652405e-07,
      "logits/chosen": -1.923841118812561,
      "logits/rejected": -1.9213125705718994,
      "logps/chosen": -33.42062759399414,
      "logps/rejected": -32.888607025146484,
      "loss": 0.198,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 0.6449113488197327,
      "rewards/margins": 1.064406156539917,
      "rewards/rejected": -0.41949471831321716,
      "step": 1240
    },
    {
      "epoch": 3.25,
      "grad_norm": 7.9375,
      "learning_rate": 5.2093665457911e-07,
      "logits/chosen": -1.8493897914886475,
      "logits/rejected": -1.846502661705017,
      "logps/chosen": -32.840755462646484,
      "logps/rejected": -36.272361755371094,
      "loss": 0.1617,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.6221579909324646,
      "rewards/margins": 1.2122505903244019,
      "rewards/rejected": -0.5900925397872925,
      "step": 1250
    },
    {
      "epoch": 3.27,
      "grad_norm": 9.0,
      "learning_rate": 4.868243561723535e-07,
      "logits/chosen": -1.9173166751861572,
      "logits/rejected": -1.912136435508728,
      "logps/chosen": -29.191509246826172,
      "logps/rejected": -33.58539581298828,
      "loss": 0.1642,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.6277153491973877,
      "rewards/margins": 1.1088515520095825,
      "rewards/rejected": -0.4811362624168396,
      "step": 1260
    },
    {
      "epoch": 3.3,
      "grad_norm": 8.75,
      "learning_rate": 4.537463335535161e-07,
      "logits/chosen": -1.943853735923767,
      "logits/rejected": -1.9487136602401733,
      "logps/chosen": -31.14084815979004,
      "logps/rejected": -32.768775939941406,
      "loss": 0.1881,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": 0.6280597448348999,
      "rewards/margins": 1.063023567199707,
      "rewards/rejected": -0.43496379256248474,
      "step": 1270
    },
    {
      "epoch": 3.32,
      "grad_norm": 13.25,
      "learning_rate": 4.217195806684629e-07,
      "logits/chosen": -1.8363254070281982,
      "logits/rejected": -1.8437111377716064,
      "logps/chosen": -33.44670867919922,
      "logps/rejected": -33.85987854003906,
      "loss": 0.2191,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.5842136144638062,
      "rewards/margins": 1.0888583660125732,
      "rewards/rejected": -0.5046447515487671,
      "step": 1280
    },
    {
      "epoch": 3.35,
      "grad_norm": 9.3125,
      "learning_rate": 3.907605513696808e-07,
      "logits/chosen": -1.74712336063385,
      "logits/rejected": -1.7494428157806396,
      "logps/chosen": -31.577123641967773,
      "logps/rejected": -37.742897033691406,
      "loss": 0.2161,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.6219089031219482,
      "rewards/margins": 1.0973594188690186,
      "rewards/rejected": -0.47545066475868225,
      "step": 1290
    },
    {
      "epoch": 3.38,
      "grad_norm": 5.6875,
      "learning_rate": 3.6088515096305675e-07,
      "logits/chosen": -1.7973334789276123,
      "logits/rejected": -1.8008067607879639,
      "logps/chosen": -31.566543579101562,
      "logps/rejected": -33.696510314941406,
      "loss": 0.1526,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": 0.6600514650344849,
      "rewards/margins": 1.1529910564422607,
      "rewards/rejected": -0.4929395616054535,
      "step": 1300
    },
    {
      "epoch": 3.38,
      "eval_logits/chosen": -2.139777898788452,
      "eval_logits/rejected": -2.135021686553955,
      "eval_logps/chosen": -34.38774108886719,
      "eval_logps/rejected": -38.054115295410156,
      "eval_loss": 0.9307055473327637,
      "eval_rewards/accuracies": 0.5863787531852722,
      "eval_rewards/chosen": -0.14127586781978607,
      "eval_rewards/margins": 0.07372146099805832,
      "eval_rewards/rejected": -0.2149973064661026,
      "eval_runtime": 145.1112,
      "eval_samples_per_second": 2.364,
      "eval_steps_per_second": 0.296,
      "step": 1300
    },
    {
      "epoch": 3.4,
      "grad_norm": 8.0625,
      "learning_rate": 3.321087280364757e-07,
      "logits/chosen": -1.9071658849716187,
      "logits/rejected": -1.886819839477539,
      "logps/chosen": -29.60116195678711,
      "logps/rejected": -36.554786682128906,
      "loss": 0.2053,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 0.6048943400382996,
      "rewards/margins": 1.1602458953857422,
      "rewards/rejected": -0.5553516149520874,
      "step": 1310
    },
    {
      "epoch": 3.43,
      "grad_norm": 13.375,
      "learning_rate": 3.044460665744284e-07,
      "logits/chosen": -1.8822952508926392,
      "logits/rejected": -1.888043761253357,
      "logps/chosen": -31.35972023010254,
      "logps/rejected": -33.60492706298828,
      "loss": 0.1474,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": 0.6696279048919678,
      "rewards/margins": 1.115299105644226,
      "rewards/rejected": -0.4456712603569031,
      "step": 1320
    },
    {
      "epoch": 3.45,
      "grad_norm": 7.28125,
      "learning_rate": 2.779113783626916e-07,
      "logits/chosen": -1.8583717346191406,
      "logits/rejected": -1.8534027338027954,
      "logps/chosen": -31.66836166381836,
      "logps/rejected": -35.97147750854492,
      "loss": 0.1801,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": 0.5416578650474548,
      "rewards/margins": 1.1013325452804565,
      "rewards/rejected": -0.5596746802330017,
      "step": 1330
    },
    {
      "epoch": 3.48,
      "grad_norm": 10.5625,
      "learning_rate": 2.5251829568697204e-07,
      "logits/chosen": -1.720171570777893,
      "logits/rejected": -1.729686975479126,
      "logps/chosen": -32.17263412475586,
      "logps/rejected": -31.95621681213379,
      "loss": 0.2302,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": 0.6507819294929504,
      "rewards/margins": 1.019622564315796,
      "rewards/rejected": -0.368840754032135,
      "step": 1340
    },
    {
      "epoch": 3.51,
      "grad_norm": 9.25,
      "learning_rate": 2.2827986432927774e-07,
      "logits/chosen": -1.7789561748504639,
      "logits/rejected": -1.7837011814117432,
      "logps/chosen": -32.18306350708008,
      "logps/rejected": -34.33165740966797,
      "loss": 0.1217,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": 0.7071508169174194,
      "rewards/margins": 1.157888650894165,
      "rewards/rejected": -0.45073771476745605,
      "step": 1350
    },
    {
      "epoch": 3.53,
      "grad_norm": 17.25,
      "learning_rate": 2.0520853686560177e-07,
      "logits/chosen": -1.8064203262329102,
      "logits/rejected": -1.811870813369751,
      "logps/chosen": -32.34109878540039,
      "logps/rejected": -35.7831916809082,
      "loss": 0.2097,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": 0.5962255597114563,
      "rewards/margins": 1.0842751264572144,
      "rewards/rejected": -0.4880496561527252,
      "step": 1360
    },
    {
      "epoch": 3.56,
      "grad_norm": 7.28125,
      "learning_rate": 1.833161662683672e-07,
      "logits/chosen": -1.8998634815216064,
      "logits/rejected": -1.8953298330307007,
      "logps/chosen": -31.270008087158203,
      "logps/rejected": -32.123897552490234,
      "loss": 0.1836,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": 0.546270489692688,
      "rewards/margins": 1.0310702323913574,
      "rewards/rejected": -0.4847997725009918,
      "step": 1370
    },
    {
      "epoch": 3.58,
      "grad_norm": 14.0625,
      "learning_rate": 1.626139998169246e-07,
      "logits/chosen": -1.8143882751464844,
      "logits/rejected": -1.816428542137146,
      "logps/chosen": -28.297687530517578,
      "logps/rejected": -31.918853759765625,
      "loss": 0.2172,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.5446463823318481,
      "rewards/margins": 1.0145161151885986,
      "rewards/rejected": -0.4698697626590729,
      "step": 1380
    },
    {
      "epoch": 3.61,
      "grad_norm": 13.75,
      "learning_rate": 1.4311267331922535e-07,
      "logits/chosen": -1.8110907077789307,
      "logits/rejected": -1.8157243728637695,
      "logps/chosen": -30.599262237548828,
      "logps/rejected": -33.90684127807617,
      "loss": 0.228,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": 0.5680925250053406,
      "rewards/margins": 0.9356026649475098,
      "rewards/rejected": -0.3675101101398468,
      "step": 1390
    },
    {
      "epoch": 3.64,
      "grad_norm": 5.28125,
      "learning_rate": 1.2482220564763669e-07,
      "logits/chosen": -1.8766982555389404,
      "logits/rejected": -1.8799679279327393,
      "logps/chosen": -33.26051712036133,
      "logps/rejected": -34.686241149902344,
      "loss": 0.1121,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": 0.6680229902267456,
      "rewards/margins": 1.1730940341949463,
      "rewards/rejected": -0.5050708651542664,
      "step": 1400
    },
    {
      "epoch": 3.64,
      "eval_logits/chosen": -2.14040207862854,
      "eval_logits/rejected": -2.135646104812622,
      "eval_logps/chosen": -34.378780364990234,
      "eval_logps/rejected": -38.054847717285156,
      "eval_loss": 0.925031840801239,
      "eval_rewards/accuracies": 0.6038206219673157,
      "eval_rewards/chosen": -0.13769178092479706,
      "eval_rewards/margins": 0.07760081440210342,
      "eval_rewards/rejected": -0.21529260277748108,
      "eval_runtime": 145.1344,
      "eval_samples_per_second": 2.363,
      "eval_steps_per_second": 0.296,
      "step": 1400
    },
    {
      "epoch": 3.66,
      "grad_norm": 13.4375,
      "learning_rate": 1.0775199359171346e-07,
      "logits/chosen": -1.945010781288147,
      "logits/rejected": -1.9381306171417236,
      "logps/chosen": -32.14293670654297,
      "logps/rejected": -35.293968200683594,
      "loss": 0.1774,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": 0.6638438105583191,
      "rewards/margins": 1.0836031436920166,
      "rewards/rejected": -0.4197593331336975,
      "step": 1410
    },
    {
      "epoch": 3.69,
      "grad_norm": 12.1875,
      "learning_rate": 9.191080703056604e-08,
      "logits/chosen": -1.8345167636871338,
      "logits/rejected": -1.8454965353012085,
      "logps/chosen": -32.55244827270508,
      "logps/rejected": -34.81972122192383,
      "loss": 0.1625,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": 0.6387423276901245,
      "rewards/margins": 1.1301978826522827,
      "rewards/rejected": -0.49145546555519104,
      "step": 1420
    },
    {
      "epoch": 3.71,
      "grad_norm": 13.0,
      "learning_rate": 7.730678442730539e-08,
      "logits/chosen": -1.9138675928115845,
      "logits/rejected": -1.9257293939590454,
      "logps/chosen": -33.43199920654297,
      "logps/rejected": -34.919532775878906,
      "loss": 0.1706,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.6515201926231384,
      "rewards/margins": 1.1639997959136963,
      "rewards/rejected": -0.5124797224998474,
      "step": 1430
    },
    {
      "epoch": 3.74,
      "grad_norm": 11.1875,
      "learning_rate": 6.394742864787806e-08,
      "logits/chosen": -1.904793381690979,
      "logits/rejected": -1.9069957733154297,
      "logps/chosen": -31.7263126373291,
      "logps/rejected": -35.0744743347168,
      "loss": 0.2078,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.5928308367729187,
      "rewards/margins": 1.0786316394805908,
      "rewards/rejected": -0.4858008325099945,
      "step": 1440
    },
    {
      "epoch": 3.77,
      "grad_norm": 9.5625,
      "learning_rate": 5.183960310644748e-08,
      "logits/chosen": -1.8884856700897217,
      "logits/rejected": -1.8813756704330444,
      "logps/chosen": -33.934364318847656,
      "logps/rejected": -34.85586929321289,
      "loss": 0.2029,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 0.6535843014717102,
      "rewards/margins": 1.0877697467803955,
      "rewards/rejected": -0.43418556451797485,
      "step": 1450
    },
    {
      "epoch": 3.79,
      "grad_norm": 18.875,
      "learning_rate": 4.098952823928693e-08,
      "logits/chosen": -1.8547000885009766,
      "logits/rejected": -1.860727310180664,
      "logps/chosen": -29.743526458740234,
      "logps/rejected": -34.97495651245117,
      "loss": 0.1719,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.5878596305847168,
      "rewards/margins": 1.1098417043685913,
      "rewards/rejected": -0.5219820737838745,
      "step": 1460
    },
    {
      "epoch": 3.82,
      "grad_norm": 12.875,
      "learning_rate": 3.1402778309014284e-08,
      "logits/chosen": -1.8023372888565063,
      "logits/rejected": -1.8003690242767334,
      "logps/chosen": -29.268962860107422,
      "logps/rejected": -31.533855438232422,
      "loss": 0.2267,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.6306362152099609,
      "rewards/margins": 1.006486177444458,
      "rewards/rejected": -0.37585002183914185,
      "step": 1470
    },
    {
      "epoch": 3.84,
      "grad_norm": 9.5625,
      "learning_rate": 2.3084278540791427e-08,
      "logits/chosen": -1.9781367778778076,
      "logits/rejected": -1.9725608825683594,
      "logps/chosen": -33.99125671386719,
      "logps/rejected": -33.27691650390625,
      "loss": 0.1815,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": 0.5176017880439758,
      "rewards/margins": 1.0056849718093872,
      "rewards/rejected": -0.4880831241607666,
      "step": 1480
    },
    {
      "epoch": 3.87,
      "grad_norm": 14.25,
      "learning_rate": 1.6038302591975807e-08,
      "logits/chosen": -1.8709309101104736,
      "logits/rejected": -1.873234510421753,
      "logps/chosen": -27.13213539123535,
      "logps/rejected": -29.021121978759766,
      "loss": 0.2587,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.5376989245414734,
      "rewards/margins": 0.9372351765632629,
      "rewards/rejected": -0.39953625202178955,
      "step": 1490
    },
    {
      "epoch": 3.9,
      "grad_norm": 16.25,
      "learning_rate": 1.0268470356514237e-08,
      "logits/chosen": -1.8781601190567017,
      "logits/rejected": -1.8725292682647705,
      "logps/chosen": -31.48541831970215,
      "logps/rejected": -33.22844696044922,
      "loss": 0.215,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": 0.5261479020118713,
      "rewards/margins": 1.0299475193023682,
      "rewards/rejected": -0.5037996768951416,
      "step": 1500
    },
    {
      "epoch": 3.9,
      "eval_logits/chosen": -2.140532970428467,
      "eval_logits/rejected": -2.135777473449707,
      "eval_logps/chosen": -34.38066101074219,
      "eval_logps/rejected": -38.052310943603516,
      "eval_loss": 0.9279635548591614,
      "eval_rewards/accuracies": 0.5805647969245911,
      "eval_rewards/chosen": -0.1384437084197998,
      "eval_rewards/margins": 0.07583153247833252,
      "eval_rewards/rejected": -0.21427522599697113,
      "eval_runtime": 145.1697,
      "eval_samples_per_second": 2.363,
      "eval_steps_per_second": 0.296,
      "step": 1500
    },
    {
      "epoch": 3.92,
      "grad_norm": 16.375,
      "learning_rate": 5.777746105209147e-09,
      "logits/chosen": -1.803832769393921,
      "logits/rejected": -1.8080018758773804,
      "logps/chosen": -32.76865005493164,
      "logps/rejected": -35.45244216918945,
      "loss": 0.2319,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 0.56093829870224,
      "rewards/margins": 0.9777617454528809,
      "rewards/rejected": -0.41682347655296326,
      "step": 1510
    },
    {
      "epoch": 3.95,
      "grad_norm": 5.53125,
      "learning_rate": 2.5684369628148352e-09,
      "logits/chosen": -1.8591749668121338,
      "logits/rejected": -1.8575620651245117,
      "logps/chosen": -29.235681533813477,
      "logps/rejected": -33.72986602783203,
      "loss": 0.2297,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 0.5993990302085876,
      "rewards/margins": 1.0554559230804443,
      "rewards/rejected": -0.4560568928718567,
      "step": 1520
    },
    {
      "epoch": 3.97,
      "grad_norm": 14.1875,
      "learning_rate": 6.421917227455999e-10,
      "logits/chosen": -1.9594846963882446,
      "logits/rejected": -1.951703667640686,
      "logps/chosen": -26.725296020507812,
      "logps/rejected": -29.879215240478516,
      "loss": 0.2118,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": 0.5350590944290161,
      "rewards/margins": 1.0181024074554443,
      "rewards/rejected": -0.4830433428287506,
      "step": 1530
    },
    {
      "epoch": 4.0,
      "grad_norm": 13.9375,
      "learning_rate": 0.0,
      "logits/chosen": -1.8623542785644531,
      "logits/rejected": -1.8520355224609375,
      "logps/chosen": -31.616281509399414,
      "logps/rejected": -36.66792297363281,
      "loss": 0.1728,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": 0.5799667835235596,
      "rewards/margins": 1.0826661586761475,
      "rewards/rejected": -0.5026994943618774,
      "step": 1540
    },
    {
      "epoch": 4.0,
      "step": 1540,
      "total_flos": 0.0,
      "train_loss": 0.2625940967689861,
      "train_runtime": 10748.6274,
      "train_samples_per_second": 1.146,
      "train_steps_per_second": 0.143
    }
  ],
  "logging_steps": 10,
  "max_steps": 1540,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 4,
  "save_steps": 100,
  "total_flos": 0.0,
  "train_batch_size": 4,
  "trial_name": null,
  "trial_params": null
}