{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 1.0,
  "eval_steps": 100,
  "global_step": 3750,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.0,
      "grad_norm": 1.59375,
      "learning_rate": 1.3333333333333334e-08,
      "logits/chosen": -2.5029678344726562,
      "logits/rejected": -2.961116075515747,
      "logps/chosen": -150.60084533691406,
      "logps/rejected": -241.10336303710938,
      "loss": 0.6931,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 1
    },
    {
      "epoch": 0.0,
      "grad_norm": 1.796875,
      "learning_rate": 1.3333333333333336e-07,
      "logits/chosen": -2.751849889755249,
      "logits/rejected": -2.569998264312744,
      "logps/chosen": -264.2574157714844,
      "logps/rejected": -228.3031463623047,
      "loss": 0.6933,
      "rewards/accuracies": 0.3888888955116272,
      "rewards/chosen": -5.868884545634501e-05,
      "rewards/margins": -0.0005380148068070412,
      "rewards/rejected": 0.0004793259722646326,
      "step": 10
    },
    {
      "epoch": 0.01,
      "grad_norm": 2.203125,
      "learning_rate": 2.666666666666667e-07,
      "logits/chosen": -2.5656490325927734,
      "logits/rejected": -2.236295223236084,
      "logps/chosen": -228.8282012939453,
      "logps/rejected": -207.7174835205078,
      "loss": 0.6929,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 9.461688750889152e-05,
      "rewards/margins": -4.184077261015773e-05,
      "rewards/rejected": 0.0001364576310152188,
      "step": 20
    },
    {
      "epoch": 0.01,
      "grad_norm": 1.8671875,
      "learning_rate": 4.0000000000000003e-07,
      "logits/chosen": -2.7992916107177734,
      "logits/rejected": -2.405809164047241,
      "logps/chosen": -244.9807586669922,
      "logps/rejected": -239.9449462890625,
      "loss": 0.693,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": 5.3206400480121374e-05,
      "rewards/margins": -0.000523517606779933,
      "rewards/rejected": 0.0005767240654677153,
      "step": 30
    },
    {
      "epoch": 0.01,
      "grad_norm": 1.8046875,
      "learning_rate": 5.333333333333335e-07,
      "logits/chosen": -2.816460132598877,
      "logits/rejected": -2.322313070297241,
      "logps/chosen": -341.45330810546875,
      "logps/rejected": -242.9652099609375,
      "loss": 0.6929,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.0009521013125777245,
      "rewards/margins": 0.0009615033632144332,
      "rewards/rejected": -9.402259820490144e-06,
      "step": 40
    },
    {
      "epoch": 0.01,
      "grad_norm": 1.9140625,
      "learning_rate": 6.666666666666667e-07,
      "logits/chosen": -2.7749481201171875,
      "logits/rejected": -2.5338540077209473,
      "logps/chosen": -280.96966552734375,
      "logps/rejected": -211.8593292236328,
      "loss": 0.6929,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": -0.0007164442213252187,
      "rewards/margins": 0.0006192409200593829,
      "rewards/rejected": -0.0013356852577999234,
      "step": 50
    },
    {
      "epoch": 0.02,
      "grad_norm": 1.6171875,
      "learning_rate": 8.000000000000001e-07,
      "logits/chosen": -2.5774986743927,
      "logits/rejected": -2.56099796295166,
      "logps/chosen": -255.7334747314453,
      "logps/rejected": -251.7584686279297,
      "loss": 0.6923,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.0027933050878345966,
      "rewards/margins": 0.0012448957422748208,
      "rewards/rejected": -0.004038200713694096,
      "step": 60
    },
    {
      "epoch": 0.02,
      "grad_norm": 2.09375,
      "learning_rate": 9.333333333333334e-07,
      "logits/chosen": -2.727661609649658,
      "logits/rejected": -2.1929476261138916,
      "logps/chosen": -259.41302490234375,
      "logps/rejected": -177.67161560058594,
      "loss": 0.6914,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.0029883747920393944,
      "rewards/margins": 0.0034666459541767836,
      "rewards/rejected": -0.006455021444708109,
      "step": 70
    },
    {
      "epoch": 0.02,
      "grad_norm": 1.984375,
      "learning_rate": 1.066666666666667e-06,
      "logits/chosen": -2.568448066711426,
      "logits/rejected": -2.4089131355285645,
      "logps/chosen": -219.25698852539062,
      "logps/rejected": -218.41683959960938,
      "loss": 0.6918,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.0030461600981652737,
      "rewards/margins": 0.004562945105135441,
      "rewards/rejected": -0.0076091052033007145,
      "step": 80
    },
    {
      "epoch": 0.02,
      "grad_norm": 1.6484375,
      "learning_rate": 1.2000000000000002e-06,
      "logits/chosen": -2.390437126159668,
      "logits/rejected": -2.645308256149292,
      "logps/chosen": -235.12643432617188,
      "logps/rejected": -251.5457000732422,
      "loss": 0.6912,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.00016399445303250104,
      "rewards/margins": 0.0025413446128368378,
      "rewards/rejected": -0.0027053379453718662,
      "step": 90
    },
    {
      "epoch": 0.03,
      "grad_norm": 2.109375,
      "learning_rate": 1.3333333333333334e-06,
      "logits/chosen": -2.566901683807373,
      "logits/rejected": -2.1725478172302246,
      "logps/chosen": -261.8447265625,
      "logps/rejected": -272.42138671875,
      "loss": 0.6906,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": 0.004414686001837254,
      "rewards/margins": 0.009829835966229439,
      "rewards/rejected": -0.005415148567408323,
      "step": 100
    },
    {
      "epoch": 0.03,
      "eval_logits/chosen": -2.6794486045837402,
      "eval_logits/rejected": -2.5792109966278076,
      "eval_logps/chosen": -272.4812927246094,
      "eval_logps/rejected": -258.5292053222656,
      "eval_loss": 0.690399706363678,
      "eval_rewards/accuracies": 0.5997983813285828,
      "eval_rewards/chosen": 0.00789840891957283,
      "eval_rewards/margins": 0.004479274153709412,
      "eval_rewards/rejected": 0.0034191354643553495,
      "eval_runtime": 878.5009,
      "eval_samples_per_second": 4.515,
      "eval_steps_per_second": 0.141,
      "step": 100
    },
    {
      "epoch": 0.03,
      "grad_norm": 1.953125,
      "learning_rate": 1.4666666666666669e-06,
      "logits/chosen": -2.9538304805755615,
      "logits/rejected": -2.299192428588867,
      "logps/chosen": -287.02154541015625,
      "logps/rejected": -194.86448669433594,
      "loss": 0.6885,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": 0.007867762818932533,
      "rewards/margins": 0.010053292848169804,
      "rewards/rejected": -0.0021855300292372704,
      "step": 110
    },
    {
      "epoch": 0.03,
      "grad_norm": 1.6484375,
      "learning_rate": 1.6000000000000001e-06,
      "logits/chosen": -2.577859401702881,
      "logits/rejected": -2.5231690406799316,
      "logps/chosen": -271.9326477050781,
      "logps/rejected": -268.78546142578125,
      "loss": 0.6874,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.012620320543646812,
      "rewards/margins": 0.01677822694182396,
      "rewards/rejected": -0.004157906398177147,
      "step": 120
    },
    {
      "epoch": 0.03,
      "grad_norm": 2.0,
      "learning_rate": 1.7333333333333336e-06,
      "logits/chosen": -2.8667593002319336,
      "logits/rejected": -2.5461220741271973,
      "logps/chosen": -279.20880126953125,
      "logps/rejected": -247.8749542236328,
      "loss": 0.688,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": 0.018279843032360077,
      "rewards/margins": 0.0033196997828781605,
      "rewards/rejected": 0.014960142783820629,
      "step": 130
    },
    {
      "epoch": 0.04,
      "grad_norm": 1.8671875,
      "learning_rate": 1.8666666666666669e-06,
      "logits/chosen": -2.676809310913086,
      "logits/rejected": -2.653555393218994,
      "logps/chosen": -269.1524353027344,
      "logps/rejected": -269.90057373046875,
      "loss": 0.6842,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.03214459866285324,
      "rewards/margins": 0.015173956751823425,
      "rewards/rejected": 0.016970645636320114,
      "step": 140
    },
    {
      "epoch": 0.04,
      "grad_norm": 1.65625,
      "learning_rate": 2.0000000000000003e-06,
      "logits/chosen": -2.592684745788574,
      "logits/rejected": -2.440980911254883,
      "logps/chosen": -254.76321411132812,
      "logps/rejected": -251.68356323242188,
      "loss": 0.6864,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": 0.02740500308573246,
      "rewards/margins": 0.016553575173020363,
      "rewards/rejected": 0.010851429775357246,
      "step": 150
    },
    {
      "epoch": 0.04,
      "grad_norm": 1.578125,
      "learning_rate": 2.133333333333334e-06,
      "logits/chosen": -2.7576775550842285,
      "logits/rejected": -2.2858314514160156,
      "logps/chosen": -253.3615264892578,
      "logps/rejected": -230.3264617919922,
      "loss": 0.6873,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.03528430685400963,
      "rewards/margins": 0.020709415897727013,
      "rewards/rejected": 0.014574890956282616,
      "step": 160
    },
    {
      "epoch": 0.05,
      "grad_norm": 2.0625,
      "learning_rate": 2.266666666666667e-06,
      "logits/chosen": -2.681196689605713,
      "logits/rejected": -2.6385631561279297,
      "logps/chosen": -183.3776397705078,
      "logps/rejected": -194.5261688232422,
      "loss": 0.6836,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": 0.02016056887805462,
      "rewards/margins": 0.02589825727045536,
      "rewards/rejected": -0.005737687461078167,
      "step": 170
    },
    {
      "epoch": 0.05,
      "grad_norm": 2.015625,
      "learning_rate": 2.4000000000000003e-06,
      "logits/chosen": -2.586236000061035,
      "logits/rejected": -2.5479259490966797,
      "logps/chosen": -246.7426300048828,
      "logps/rejected": -230.27444458007812,
      "loss": 0.6811,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.005953938700258732,
      "rewards/margins": 0.008363286964595318,
      "rewards/rejected": -0.002409350825473666,
      "step": 180
    },
    {
      "epoch": 0.05,
      "grad_norm": 2.359375,
      "learning_rate": 2.5333333333333338e-06,
      "logits/chosen": -2.339409351348877,
      "logits/rejected": -2.428462028503418,
      "logps/chosen": -253.17514038085938,
      "logps/rejected": -279.63909912109375,
      "loss": 0.6816,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.0022083132062107325,
      "rewards/margins": 0.023802492767572403,
      "rewards/rejected": -0.026010805740952492,
      "step": 190
    },
    {
      "epoch": 0.05,
      "grad_norm": 1.984375,
      "learning_rate": 2.666666666666667e-06,
      "logits/chosen": -2.627366542816162,
      "logits/rejected": -2.5913214683532715,
      "logps/chosen": -325.80126953125,
      "logps/rejected": -242.60653686523438,
      "loss": 0.6832,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": 0.03533730283379555,
      "rewards/margins": 0.03854949027299881,
      "rewards/rejected": -0.0032121881376951933,
      "step": 200
    },
    {
      "epoch": 0.05,
      "eval_logits/chosen": -2.694658041000366,
      "eval_logits/rejected": -2.5935657024383545,
      "eval_logps/chosen": -269.5003356933594,
      "eval_logps/rejected": -257.870361328125,
      "eval_loss": 0.6789314150810242,
      "eval_rewards/accuracies": 0.6310483813285828,
      "eval_rewards/chosen": 0.03770757466554642,
      "eval_rewards/margins": 0.02770034410059452,
      "eval_rewards/rejected": 0.010007228702306747,
      "eval_runtime": 877.8908,
      "eval_samples_per_second": 4.518,
      "eval_steps_per_second": 0.141,
      "step": 200
    },
    {
      "epoch": 0.06,
      "grad_norm": 1.7890625,
      "learning_rate": 2.8000000000000003e-06,
      "logits/chosen": -2.5762126445770264,
      "logits/rejected": -2.748198986053467,
      "logps/chosen": -272.9695129394531,
      "logps/rejected": -261.88714599609375,
      "loss": 0.6763,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": 0.005505173001438379,
      "rewards/margins": 0.022723117843270302,
      "rewards/rejected": -0.01721794530749321,
      "step": 210
    },
    {
      "epoch": 0.06,
      "grad_norm": 2.0,
      "learning_rate": 2.9333333333333338e-06,
      "logits/chosen": -2.621309995651245,
      "logits/rejected": -2.3600690364837646,
      "logps/chosen": -226.82431030273438,
      "logps/rejected": -227.02957153320312,
      "loss": 0.6803,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": 0.015118074603378773,
      "rewards/margins": 0.050091683864593506,
      "rewards/rejected": -0.03497361019253731,
      "step": 220
    },
    {
      "epoch": 0.06,
      "grad_norm": 2.234375,
      "learning_rate": 3.066666666666667e-06,
      "logits/chosen": -2.580172061920166,
      "logits/rejected": -2.6298460960388184,
      "logps/chosen": -246.28683471679688,
      "logps/rejected": -235.20767211914062,
      "loss": 0.6823,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.009840680286288261,
      "rewards/margins": 0.03877246379852295,
      "rewards/rejected": -0.02893177606165409,
      "step": 230
    },
    {
      "epoch": 0.06,
      "grad_norm": 2.84375,
      "learning_rate": 3.2000000000000003e-06,
      "logits/chosen": -2.589890956878662,
      "logits/rejected": -2.3240532875061035,
      "logps/chosen": -243.666259765625,
      "logps/rejected": -221.4214324951172,
      "loss": 0.6737,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.030014852061867714,
      "rewards/margins": 0.04435984045267105,
      "rewards/rejected": -0.07437469065189362,
      "step": 240
    },
    {
      "epoch": 0.07,
      "grad_norm": 1.875,
      "learning_rate": 3.3333333333333333e-06,
      "logits/chosen": -2.7324252128601074,
      "logits/rejected": -2.343388557434082,
      "logps/chosen": -325.43731689453125,
      "logps/rejected": -226.3644256591797,
      "loss": 0.6666,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.009514258243143559,
      "rewards/margins": 0.05304191634058952,
      "rewards/rejected": -0.0625561773777008,
      "step": 250
    },
    {
      "epoch": 0.07,
      "grad_norm": 2.109375,
      "learning_rate": 3.4666666666666672e-06,
      "logits/chosen": -2.931042432785034,
      "logits/rejected": -2.6046676635742188,
      "logps/chosen": -293.7285461425781,
      "logps/rejected": -270.9932556152344,
      "loss": 0.6767,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.0049897716380655766,
      "rewards/margins": 0.07512545585632324,
      "rewards/rejected": -0.08011523634195328,
      "step": 260
    },
    {
      "epoch": 0.07,
      "grad_norm": 2.1875,
      "learning_rate": 3.6000000000000003e-06,
      "logits/chosen": -2.50712513923645,
      "logits/rejected": -2.3976807594299316,
      "logps/chosen": -230.7588653564453,
      "logps/rejected": -278.4716796875,
      "loss": 0.6646,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.011669019237160683,
      "rewards/margins": 0.04024311527609825,
      "rewards/rejected": -0.028574099764227867,
      "step": 270
    },
    {
      "epoch": 0.07,
      "grad_norm": 2.328125,
      "learning_rate": 3.7333333333333337e-06,
      "logits/chosen": -2.7707180976867676,
      "logits/rejected": -2.160461902618408,
      "logps/chosen": -309.2708740234375,
      "logps/rejected": -231.8957977294922,
      "loss": 0.6635,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.001572545268572867,
      "rewards/margins": 0.06184723228216171,
      "rewards/rejected": -0.06027469038963318,
      "step": 280
    },
    {
      "epoch": 0.08,
      "grad_norm": 2.78125,
      "learning_rate": 3.866666666666667e-06,
      "logits/chosen": -2.6934523582458496,
      "logits/rejected": -2.261805295944214,
      "logps/chosen": -214.80419921875,
      "logps/rejected": -190.09890747070312,
      "loss": 0.6779,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.05012133717536926,
      "rewards/margins": 0.06534415483474731,
      "rewards/rejected": -0.11546549946069717,
      "step": 290
    },
    {
      "epoch": 0.08,
      "grad_norm": 2.109375,
      "learning_rate": 4.000000000000001e-06,
      "logits/chosen": -2.521820068359375,
      "logits/rejected": -2.2671499252319336,
      "logps/chosen": -196.18704223632812,
      "logps/rejected": -205.0076141357422,
      "loss": 0.6593,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.04997577145695686,
      "rewards/margins": 0.05045613646507263,
      "rewards/rejected": -0.1004319041967392,
      "step": 300
    },
    {
      "epoch": 0.08,
      "eval_logits/chosen": -2.6882143020629883,
      "eval_logits/rejected": -2.586345911026001,
      "eval_logps/chosen": -269.4002685546875,
      "eval_logps/rejected": -261.891357421875,
      "eval_loss": 0.65687495470047,
      "eval_rewards/accuracies": 0.6612903475761414,
      "eval_rewards/chosen": 0.03870880603790283,
      "eval_rewards/margins": 0.06891151517629623,
      "eval_rewards/rejected": -0.030202707275748253,
      "eval_runtime": 878.1165,
      "eval_samples_per_second": 4.516,
      "eval_steps_per_second": 0.141,
      "step": 300
    },
    {
      "epoch": 0.08,
      "grad_norm": 2.375,
      "learning_rate": 4.133333333333333e-06,
      "logits/chosen": -2.8231935501098633,
      "logits/rejected": -2.620025396347046,
      "logps/chosen": -307.0191955566406,
      "logps/rejected": -310.40032958984375,
      "loss": 0.6535,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.00154333864338696,
      "rewards/margins": 0.05546078830957413,
      "rewards/rejected": -0.05700412392616272,
      "step": 310
    },
    {
      "epoch": 0.09,
      "grad_norm": 2.4375,
      "learning_rate": 4.266666666666668e-06,
      "logits/chosen": -2.7016890048980713,
      "logits/rejected": -2.1624770164489746,
      "logps/chosen": -284.0586853027344,
      "logps/rejected": -215.47274780273438,
      "loss": 0.6457,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.03205699473619461,
      "rewards/margins": 0.11878025531768799,
      "rewards/rejected": -0.1508372575044632,
      "step": 320
    },
    {
      "epoch": 0.09,
      "grad_norm": 2.453125,
      "learning_rate": 4.4e-06,
      "logits/chosen": -2.7587971687316895,
      "logits/rejected": -2.321085214614868,
      "logps/chosen": -284.6245422363281,
      "logps/rejected": -244.0107879638672,
      "loss": 0.6468,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.06207716464996338,
      "rewards/margins": 0.0358499176800251,
      "rewards/rejected": -0.09792708605527878,
      "step": 330
    },
    {
      "epoch": 0.09,
      "grad_norm": 2.609375,
      "learning_rate": 4.533333333333334e-06,
      "logits/chosen": -2.8138785362243652,
      "logits/rejected": -2.7418646812438965,
      "logps/chosen": -295.4757385253906,
      "logps/rejected": -253.5455780029297,
      "loss": 0.6594,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.010195666924118996,
      "rewards/margins": 0.10021910816431046,
      "rewards/rejected": -0.1104147657752037,
      "step": 340
    },
    {
      "epoch": 0.09,
      "grad_norm": 2.78125,
      "learning_rate": 4.666666666666667e-06,
      "logits/chosen": -2.4013609886169434,
      "logits/rejected": -2.6423985958099365,
      "logps/chosen": -295.32733154296875,
      "logps/rejected": -301.15924072265625,
      "loss": 0.6442,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": 0.0037891597021371126,
      "rewards/margins": 0.1667846441268921,
      "rewards/rejected": -0.16299548745155334,
      "step": 350
    },
    {
      "epoch": 0.1,
      "grad_norm": 3.296875,
      "learning_rate": 4.800000000000001e-06,
      "logits/chosen": -2.6970088481903076,
      "logits/rejected": -2.1390604972839355,
      "logps/chosen": -262.02679443359375,
      "logps/rejected": -269.5014953613281,
      "loss": 0.6364,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.03520303592085838,
      "rewards/margins": 0.2216763198375702,
      "rewards/rejected": -0.25687938928604126,
      "step": 360
    },
    {
      "epoch": 0.1,
      "grad_norm": 3.703125,
      "learning_rate": 4.933333333333334e-06,
      "logits/chosen": -2.709902286529541,
      "logits/rejected": -2.545484781265259,
      "logps/chosen": -259.3272705078125,
      "logps/rejected": -237.92562866210938,
      "loss": 0.6443,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.019749831408262253,
      "rewards/margins": 0.1256447732448578,
      "rewards/rejected": -0.14539460837841034,
      "step": 370
    },
    {
      "epoch": 0.1,
      "grad_norm": 2.484375,
      "learning_rate": 4.999972922944898e-06,
      "logits/chosen": -2.817610740661621,
      "logits/rejected": -2.239814519882202,
      "logps/chosen": -268.80792236328125,
      "logps/rejected": -239.2582550048828,
      "loss": 0.6291,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.09317320585250854,
      "rewards/margins": 0.22923800349235535,
      "rewards/rejected": -0.3224112391471863,
      "step": 380
    },
    {
      "epoch": 0.1,
      "grad_norm": 2.375,
      "learning_rate": 4.999756310023261e-06,
      "logits/chosen": -2.7260260581970215,
      "logits/rejected": -2.649122476577759,
      "logps/chosen": -271.9554138183594,
      "logps/rejected": -263.88189697265625,
      "loss": 0.6556,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.10098031908273697,
      "rewards/margins": 0.13623426854610443,
      "rewards/rejected": -0.237214595079422,
      "step": 390
    },
    {
      "epoch": 0.11,
      "grad_norm": 4.15625,
      "learning_rate": 4.999323102948655e-06,
      "logits/chosen": -2.5106852054595947,
      "logits/rejected": -2.487372875213623,
      "logps/chosen": -245.68234252929688,
      "logps/rejected": -254.9107208251953,
      "loss": 0.6187,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.18754693865776062,
      "rewards/margins": 0.1915983408689499,
      "rewards/rejected": -0.3791453242301941,
      "step": 400
    },
    {
      "epoch": 0.11,
      "eval_logits/chosen": -2.673567771911621,
      "eval_logits/rejected": -2.5720021724700928,
      "eval_logps/chosen": -281.56146240234375,
      "eval_logps/rejected": -281.8323974609375,
      "eval_loss": 0.6299161911010742,
      "eval_rewards/accuracies": 0.663306474685669,
      "eval_rewards/chosen": -0.08290355652570724,
      "eval_rewards/margins": 0.14670924842357635,
      "eval_rewards/rejected": -0.2296127825975418,
      "eval_runtime": 876.9667,
      "eval_samples_per_second": 4.522,
      "eval_steps_per_second": 0.141,
      "step": 400
    },
    {
      "epoch": 0.11,
      "grad_norm": 6.3125,
      "learning_rate": 4.998673339256785e-06,
      "logits/chosen": -2.5772247314453125,
      "logits/rejected": -2.563568592071533,
      "logps/chosen": -333.8960876464844,
      "logps/rejected": -308.29998779296875,
      "loss": 0.6469,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.3483628034591675,
      "rewards/margins": 0.028353165835142136,
      "rewards/rejected": -0.3767159581184387,
      "step": 410
    },
    {
      "epoch": 0.11,
      "grad_norm": 2.984375,
      "learning_rate": 4.997807075247147e-06,
      "logits/chosen": -2.4575066566467285,
      "logits/rejected": -2.690697193145752,
      "logps/chosen": -278.5705871582031,
      "logps/rejected": -272.3896789550781,
      "loss": 0.6529,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.19658556580543518,
      "rewards/margins": 0.09328427165746689,
      "rewards/rejected": -0.2898697853088379,
      "step": 420
    },
    {
      "epoch": 0.11,
      "grad_norm": 4.96875,
      "learning_rate": 4.996724385978142e-06,
      "logits/chosen": -2.719867706298828,
      "logits/rejected": -2.4360365867614746,
      "logps/chosen": -233.9046173095703,
      "logps/rejected": -249.04171752929688,
      "loss": 0.6552,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": -0.06493125855922699,
      "rewards/margins": 0.036158014088869095,
      "rewards/rejected": -0.10108928382396698,
      "step": 430
    },
    {
      "epoch": 0.12,
      "grad_norm": 3.1875,
      "learning_rate": 4.995425365260585e-06,
      "logits/chosen": -2.501992702484131,
      "logits/rejected": -2.1200997829437256,
      "logps/chosen": -209.67025756835938,
      "logps/rejected": -209.31210327148438,
      "loss": 0.6235,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.09086422622203827,
      "rewards/margins": 0.1714749038219452,
      "rewards/rejected": -0.26233914494514465,
      "step": 440
    },
    {
      "epoch": 0.12,
      "grad_norm": 2.53125,
      "learning_rate": 4.993910125649561e-06,
      "logits/chosen": -2.7914021015167236,
      "logits/rejected": -2.7705512046813965,
      "logps/chosen": -334.71240234375,
      "logps/rejected": -293.55242919921875,
      "loss": 0.6558,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.421312153339386,
      "rewards/margins": 0.003813874674960971,
      "rewards/rejected": -0.4251260757446289,
      "step": 450
    },
    {
      "epoch": 0.12,
      "grad_norm": 2.421875,
      "learning_rate": 4.992178798434684e-06,
      "logits/chosen": -2.6328635215759277,
      "logits/rejected": -2.648602247238159,
      "logps/chosen": -243.62490844726562,
      "logps/rejected": -263.2136535644531,
      "loss": 0.6359,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.3418295979499817,
      "rewards/margins": 0.19761092960834503,
      "rewards/rejected": -0.5394405722618103,
      "step": 460
    },
    {
      "epoch": 0.13,
      "grad_norm": 3.4375,
      "learning_rate": 4.990231533628719e-06,
      "logits/chosen": -2.6936569213867188,
      "logits/rejected": -2.274219036102295,
      "logps/chosen": -326.89569091796875,
      "logps/rejected": -258.66845703125,
      "loss": 0.6298,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.28039970993995667,
      "rewards/margins": 0.2811904847621918,
      "rewards/rejected": -0.5615901350975037,
      "step": 470
    },
    {
      "epoch": 0.13,
      "grad_norm": 3.390625,
      "learning_rate": 4.988068499954578e-06,
      "logits/chosen": -2.523273468017578,
      "logits/rejected": -2.5209357738494873,
      "logps/chosen": -278.5114440917969,
      "logps/rejected": -244.7390899658203,
      "loss": 0.6549,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.13003386557102203,
      "rewards/margins": 0.10366784036159515,
      "rewards/rejected": -0.2337017059326172,
      "step": 480
    },
    {
      "epoch": 0.13,
      "grad_norm": 2.65625,
      "learning_rate": 4.985689884830711e-06,
      "logits/chosen": -2.513911485671997,
      "logits/rejected": -2.315934658050537,
      "logps/chosen": -269.84185791015625,
      "logps/rejected": -254.67636108398438,
      "loss": 0.6341,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.11949504911899567,
      "rewards/margins": 0.19560161232948303,
      "rewards/rejected": -0.3150966763496399,
      "step": 490
    },
    {
      "epoch": 0.13,
      "grad_norm": 3.375,
      "learning_rate": 4.983095894354858e-06,
      "logits/chosen": -2.6599674224853516,
      "logits/rejected": -2.609445095062256,
      "logps/chosen": -279.73980712890625,
      "logps/rejected": -261.7233581542969,
      "loss": 0.6725,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.14204347133636475,
      "rewards/margins": 0.06653790175914764,
      "rewards/rejected": -0.2085813730955124,
      "step": 500
    },
    {
      "epoch": 0.13,
      "eval_logits/chosen": -2.608640193939209,
      "eval_logits/rejected": -2.5082080364227295,
      "eval_logps/chosen": -269.4082946777344,
      "eval_logps/rejected": -268.9953918457031,
      "eval_loss": 0.6311951279640198,
      "eval_rewards/accuracies": 0.6723790168762207,
      "eval_rewards/chosen": 0.03862820193171501,
      "eval_rewards/margins": 0.139871284365654,
      "eval_rewards/rejected": -0.10124309360980988,
      "eval_runtime": 877.8692,
      "eval_samples_per_second": 4.518,
      "eval_steps_per_second": 0.141,
      "step": 500
    },
    {
      "epoch": 0.14,
      "grad_norm": 2.53125,
      "learning_rate": 4.980286753286196e-06,
      "logits/chosen": -2.664919376373291,
      "logits/rejected": -2.5646352767944336,
      "logps/chosen": -284.9444885253906,
      "logps/rejected": -319.46923828125,
      "loss": 0.6296,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.0029098601080477238,
      "rewards/margins": 0.2879767119884491,
      "rewards/rejected": -0.2908865511417389,
      "step": 510
    },
    {
      "epoch": 0.14,
      "grad_norm": 3.953125,
      "learning_rate": 4.97726270502586e-06,
      "logits/chosen": -2.5279531478881836,
      "logits/rejected": -2.412174940109253,
      "logps/chosen": -236.3230743408203,
      "logps/rejected": -243.2906036376953,
      "loss": 0.6528,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.21826303005218506,
      "rewards/margins": 0.18559373915195465,
      "rewards/rejected": -0.4038567543029785,
      "step": 520
    },
    {
      "epoch": 0.14,
      "grad_norm": 3.234375,
      "learning_rate": 4.974024011595864e-06,
      "logits/chosen": -2.6750781536102295,
      "logits/rejected": -2.420663356781006,
      "logps/chosen": -248.69906616210938,
      "logps/rejected": -261.5027770996094,
      "loss": 0.6245,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.14076104760169983,
      "rewards/margins": 0.19705942273139954,
      "rewards/rejected": -0.33782047033309937,
      "step": 530
    },
    {
      "epoch": 0.14,
      "grad_norm": 2.828125,
      "learning_rate": 4.970570953616383e-06,
      "logits/chosen": -2.297457456588745,
      "logits/rejected": -2.470264434814453,
      "logps/chosen": -245.8286895751953,
      "logps/rejected": -253.5813446044922,
      "loss": 0.6515,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.07756556570529938,
      "rewards/margins": 0.13785605132579803,
      "rewards/rejected": -0.2154216319322586,
      "step": 540
    },
    {
      "epoch": 0.15,
      "grad_norm": 3.46875,
      "learning_rate": 4.966903830281449e-06,
      "logits/chosen": -2.5244252681732178,
      "logits/rejected": -2.289266586303711,
      "logps/chosen": -341.05767822265625,
      "logps/rejected": -309.4515686035156,
      "loss": 0.632,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.09943778812885284,
      "rewards/margins": 0.22567541897296906,
      "rewards/rejected": -0.3251132369041443,
      "step": 550
    },
    {
      "epoch": 0.15,
      "grad_norm": 4.9375,
      "learning_rate": 4.9630229593330226e-06,
      "logits/chosen": -2.323316812515259,
      "logits/rejected": -2.3239264488220215,
      "logps/chosen": -265.43572998046875,
      "logps/rejected": -278.7800598144531,
      "loss": 0.6341,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.10304637253284454,
      "rewards/margins": 0.15854547917842865,
      "rewards/rejected": -0.2615918517112732,
      "step": 560
    },
    {
      "epoch": 0.15,
      "grad_norm": 3.125,
      "learning_rate": 4.958928677033465e-06,
      "logits/chosen": -2.3657829761505127,
      "logits/rejected": -2.6623101234436035,
      "logps/chosen": -300.74407958984375,
      "logps/rejected": -340.55181884765625,
      "loss": 0.6254,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.1712590903043747,
      "rewards/margins": 0.2195931226015091,
      "rewards/rejected": -0.3908522129058838,
      "step": 570
    },
    {
      "epoch": 0.15,
      "grad_norm": 3.03125,
      "learning_rate": 4.954621338136399e-06,
      "logits/chosen": -2.5695109367370605,
      "logits/rejected": -2.151683807373047,
      "logps/chosen": -299.7782287597656,
      "logps/rejected": -252.04885864257812,
      "loss": 0.607,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.17375870048999786,
      "rewards/margins": 0.2597489058971405,
      "rewards/rejected": -0.4335075318813324,
      "step": 580
    },
    {
      "epoch": 0.16,
      "grad_norm": 3.875,
      "learning_rate": 4.95010131585597e-06,
      "logits/chosen": -2.749103307723999,
      "logits/rejected": -2.1127848625183105,
      "logps/chosen": -297.27520751953125,
      "logps/rejected": -267.86981201171875,
      "loss": 0.6249,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.20688581466674805,
      "rewards/margins": 0.2848387360572815,
      "rewards/rejected": -0.49172458052635193,
      "step": 590
    },
    {
      "epoch": 0.16,
      "grad_norm": 3.078125,
      "learning_rate": 4.9453690018345144e-06,
      "logits/chosen": -2.6889395713806152,
      "logits/rejected": -2.233246326446533,
      "logps/chosen": -311.0091857910156,
      "logps/rejected": -276.1361083984375,
      "loss": 0.6094,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.35524609684944153,
      "rewards/margins": 0.19266317784786224,
      "rewards/rejected": -0.5479092597961426,
      "step": 600
    },
    {
      "epoch": 0.16,
      "eval_logits/chosen": -2.5263688564300537,
      "eval_logits/rejected": -2.426940441131592,
      "eval_logps/chosen": -283.241943359375,
      "eval_logps/rejected": -288.1275634765625,
      "eval_loss": 0.6133630275726318,
      "eval_rewards/accuracies": 0.6723790168762207,
      "eval_rewards/chosen": -0.09970811754465103,
      "eval_rewards/margins": 0.19285647571086884,
      "eval_rewards/rejected": -0.29256460070610046,
      "eval_runtime": 877.8791,
      "eval_samples_per_second": 4.518,
      "eval_steps_per_second": 0.141,
      "step": 600
    },
    {
      "epoch": 0.16,
      "grad_norm": 4.3125,
      "learning_rate": 4.940424806108619e-06,
      "logits/chosen": -2.6508736610412598,
      "logits/rejected": -2.4227428436279297,
      "logps/chosen": -217.91690063476562,
      "logps/rejected": -207.7536163330078,
      "loss": 0.6162,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.24098196625709534,
      "rewards/margins": 0.09403972327709198,
      "rewards/rejected": -0.3350217044353485,
      "step": 610
    },
    {
      "epoch": 0.17,
      "grad_norm": 2.625,
      "learning_rate": 4.935269157073597e-06,
      "logits/chosen": -2.5305328369140625,
      "logits/rejected": -2.51523756980896,
      "logps/chosen": -352.21893310546875,
      "logps/rejected": -322.8544006347656,
      "loss": 0.6294,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.22192712128162384,
      "rewards/margins": 0.180780291557312,
      "rewards/rejected": -0.40270739793777466,
      "step": 620
    },
    {
      "epoch": 0.17,
      "grad_norm": 6.09375,
      "learning_rate": 4.9299025014463665e-06,
      "logits/chosen": -2.5375523567199707,
      "logits/rejected": -2.1695973873138428,
      "logps/chosen": -289.813720703125,
      "logps/rejected": -239.43966674804688,
      "loss": 0.6429,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.44848066568374634,
      "rewards/margins": 0.03964262083172798,
      "rewards/rejected": -0.4881232678890228,
      "step": 630
    },
    {
      "epoch": 0.17,
      "grad_norm": 3.546875,
      "learning_rate": 4.924325304226745e-06,
      "logits/chosen": -2.509765148162842,
      "logits/rejected": -2.0747461318969727,
      "logps/chosen": -280.7805480957031,
      "logps/rejected": -302.287841796875,
      "loss": 0.6399,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.44217365980148315,
      "rewards/margins": 0.32420799136161804,
      "rewards/rejected": -0.7663816213607788,
      "step": 640
    },
    {
      "epoch": 0.17,
      "grad_norm": 4.15625,
      "learning_rate": 4.91853804865716e-06,
      "logits/chosen": -2.484067916870117,
      "logits/rejected": -2.0902838706970215,
      "logps/chosen": -327.0274963378906,
      "logps/rejected": -261.1690979003906,
      "loss": 0.6219,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.45915380120277405,
      "rewards/margins": 0.16302372515201569,
      "rewards/rejected": -0.6221774816513062,
      "step": 650
    },
    {
      "epoch": 0.18,
      "grad_norm": 3.8125,
      "learning_rate": 4.912541236180779e-06,
      "logits/chosen": -2.6334614753723145,
      "logits/rejected": -2.2446084022521973,
      "logps/chosen": -328.9707336425781,
      "logps/rejected": -289.1524963378906,
      "loss": 0.6167,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.14022080600261688,
      "rewards/margins": 0.43930783867836,
      "rewards/rejected": -0.5795286893844604,
      "step": 660
    },
    {
      "epoch": 0.18,
      "grad_norm": 3.0,
      "learning_rate": 4.9063353863980565e-06,
      "logits/chosen": -2.6679372787475586,
      "logits/rejected": -2.387796640396118,
      "logps/chosen": -268.6062927246094,
      "logps/rejected": -259.20953369140625,
      "loss": 0.6229,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.07020218670368195,
      "rewards/margins": 0.2284179925918579,
      "rewards/rejected": -0.29862016439437866,
      "step": 670
    },
    {
      "epoch": 0.18,
      "grad_norm": 3.515625,
      "learning_rate": 4.899921037021719e-06,
      "logits/chosen": -2.65055513381958,
      "logits/rejected": -2.5278847217559814,
      "logps/chosen": -269.8683166503906,
      "logps/rejected": -261.0122375488281,
      "loss": 0.6493,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.15900388360023499,
      "rewards/margins": 0.12956634163856506,
      "rewards/rejected": -0.28857022523880005,
      "step": 680
    },
    {
      "epoch": 0.18,
      "grad_norm": 3.9375,
      "learning_rate": 4.893298743830168e-06,
      "logits/chosen": -2.550240993499756,
      "logits/rejected": -2.2317614555358887,
      "logps/chosen": -269.5898742675781,
      "logps/rejected": -247.89041137695312,
      "loss": 0.636,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.19794264435768127,
      "rewards/margins": 0.16454729437828064,
      "rewards/rejected": -0.3624899387359619,
      "step": 690
    },
    {
      "epoch": 0.19,
      "grad_norm": 5.8125,
      "learning_rate": 4.88646908061933e-06,
      "logits/chosen": -2.657773971557617,
      "logits/rejected": -2.477496385574341,
      "logps/chosen": -329.10076904296875,
      "logps/rejected": -286.46954345703125,
      "loss": 0.622,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.1597648710012436,
      "rewards/margins": 0.2989833950996399,
      "rewards/rejected": -0.4587482810020447,
      "step": 700
    },
    {
      "epoch": 0.19,
      "eval_logits/chosen": -2.5236737728118896,
      "eval_logits/rejected": -2.424347400665283,
      "eval_logps/chosen": -279.88690185546875,
      "eval_logps/rejected": -285.2250671386719,
      "eval_loss": 0.6137515306472778,
      "eval_rewards/accuracies": 0.6764112710952759,
      "eval_rewards/chosen": -0.06615753471851349,
      "eval_rewards/margins": 0.19738218188285828,
      "eval_rewards/rejected": -0.26353973150253296,
      "eval_runtime": 877.7011,
      "eval_samples_per_second": 4.519,
      "eval_steps_per_second": 0.141,
      "step": 700
    },
    {
      "epoch": 0.19,
      "grad_norm": 29.75,
      "learning_rate": 4.879432639152935e-06,
      "logits/chosen": -2.4465527534484863,
      "logits/rejected": -2.273533344268799,
      "logps/chosen": -303.1317138671875,
      "logps/rejected": -290.7353210449219,
      "loss": 0.6928,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.15864692628383636,
      "rewards/margins": 0.32188013195991516,
      "rewards/rejected": -0.4805270731449127,
      "step": 710
    },
    {
      "epoch": 0.19,
      "grad_norm": 2.078125,
      "learning_rate": 4.8721900291112415e-06,
      "logits/chosen": -2.4020447731018066,
      "logits/rejected": -2.1184351444244385,
      "logps/chosen": -307.3013000488281,
      "logps/rejected": -291.4708251953125,
      "loss": 0.652,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.07721785455942154,
      "rewards/margins": 0.2699218690395355,
      "rewards/rejected": -0.34713971614837646,
      "step": 720
    },
    {
      "epoch": 0.19,
      "grad_norm": 5.8125,
      "learning_rate": 4.864741878038218e-06,
      "logits/chosen": -2.3442678451538086,
      "logits/rejected": -2.0677239894866943,
      "logps/chosen": -288.0312194824219,
      "logps/rejected": -237.940185546875,
      "loss": 0.6225,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.18110761046409607,
      "rewards/margins": 0.29118436574935913,
      "rewards/rejected": -0.4722919464111328,
      "step": 730
    },
    {
      "epoch": 0.2,
      "grad_norm": 4.75,
      "learning_rate": 4.857088831287158e-06,
      "logits/chosen": -2.512108087539673,
      "logits/rejected": -2.1522111892700195,
      "logps/chosen": -311.9584655761719,
      "logps/rejected": -281.13037109375,
      "loss": 0.5992,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.31862348318099976,
      "rewards/margins": 0.2932208478450775,
      "rewards/rejected": -0.6118442416191101,
      "step": 740
    },
    {
      "epoch": 0.2,
      "grad_norm": 5.53125,
      "learning_rate": 4.849231551964771e-06,
      "logits/chosen": -2.591369152069092,
      "logits/rejected": -2.1986594200134277,
      "logps/chosen": -287.7509460449219,
      "logps/rejected": -262.28656005859375,
      "loss": 0.6422,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.1783318668603897,
      "rewards/margins": 0.24001052975654602,
      "rewards/rejected": -0.4183424115180969,
      "step": 750
    },
    {
      "epoch": 0.2,
      "grad_norm": 6.78125,
      "learning_rate": 4.841170720873723e-06,
      "logits/chosen": -2.465435028076172,
      "logits/rejected": -2.1820993423461914,
      "logps/chosen": -307.37640380859375,
      "logps/rejected": -250.92318725585938,
      "loss": 0.6175,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.3154814839363098,
      "rewards/margins": 0.19859042763710022,
      "rewards/rejected": -0.5140718817710876,
      "step": 760
    },
    {
      "epoch": 0.21,
      "grad_norm": 4.65625,
      "learning_rate": 4.832907036453647e-06,
      "logits/chosen": -2.5598912239074707,
      "logits/rejected": -2.217132329940796,
      "logps/chosen": -353.2138366699219,
      "logps/rejected": -375.82965087890625,
      "loss": 0.6158,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -0.15598583221435547,
      "rewards/margins": 0.5736908912658691,
      "rewards/rejected": -0.7296767234802246,
      "step": 770
    },
    {
      "epoch": 0.21,
      "grad_norm": 4.40625,
      "learning_rate": 4.824441214720629e-06,
      "logits/chosen": -2.3894755840301514,
      "logits/rejected": -2.286311626434326,
      "logps/chosen": -352.96099853515625,
      "logps/rejected": -386.9922180175781,
      "loss": 0.6465,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.42934542894363403,
      "rewards/margins": 0.21244624257087708,
      "rewards/rejected": -0.6417917013168335,
      "step": 780
    },
    {
      "epoch": 0.21,
      "grad_norm": 4.53125,
      "learning_rate": 4.815773989205165e-06,
      "logits/chosen": -2.527446985244751,
      "logits/rejected": -2.247591018676758,
      "logps/chosen": -312.28924560546875,
      "logps/rejected": -286.30267333984375,
      "loss": 0.5985,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.22992873191833496,
      "rewards/margins": 0.30244356393814087,
      "rewards/rejected": -0.532372236251831,
      "step": 790
    },
    {
      "epoch": 0.21,
      "grad_norm": 5.21875,
      "learning_rate": 4.806906110888606e-06,
      "logits/chosen": -2.4638915061950684,
      "logits/rejected": -2.197370767593384,
      "logps/chosen": -336.13409423828125,
      "logps/rejected": -315.1051330566406,
      "loss": 0.6219,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.4365676939487457,
      "rewards/margins": 0.1963023841381073,
      "rewards/rejected": -0.6328701972961426,
      "step": 800
    },
    {
      "epoch": 0.21,
      "eval_logits/chosen": -2.4372000694274902,
      "eval_logits/rejected": -2.3384850025177,
      "eval_logps/chosen": -280.3326110839844,
      "eval_logps/rejected": -290.6908264160156,
      "eval_loss": 0.6039755940437317,
      "eval_rewards/accuracies": 0.6905242204666138,
      "eval_rewards/chosen": -0.07061473280191422,
      "eval_rewards/margins": 0.24758274853229523,
      "eval_rewards/rejected": -0.31819745898246765,
      "eval_runtime": 877.9132,
      "eval_samples_per_second": 4.518,
      "eval_steps_per_second": 0.141,
      "step": 800
    },
    {
      "epoch": 0.22,
      "grad_norm": 6.59375,
      "learning_rate": 4.7978383481380865e-06,
      "logits/chosen": -2.275413990020752,
      "logits/rejected": -2.311854600906372,
      "logps/chosen": -256.5054016113281,
      "logps/rejected": -272.4574279785156,
      "loss": 0.6285,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.31580647826194763,
      "rewards/margins": 0.2363698035478592,
      "rewards/rejected": -0.5521762371063232,
      "step": 810
    },
    {
      "epoch": 0.22,
      "grad_norm": 2.546875,
      "learning_rate": 4.788571486639948e-06,
      "logits/chosen": -2.499368190765381,
      "logits/rejected": -1.885051965713501,
      "logps/chosen": -295.2255859375,
      "logps/rejected": -259.6574401855469,
      "loss": 0.5987,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.0617445707321167,
      "rewards/margins": 0.5236464738845825,
      "rewards/rejected": -0.585391104221344,
      "step": 820
    },
    {
      "epoch": 0.22,
      "grad_norm": 3.25,
      "learning_rate": 4.779106329331665e-06,
      "logits/chosen": -2.5280704498291016,
      "logits/rejected": -2.3618202209472656,
      "logps/chosen": -338.35723876953125,
      "logps/rejected": -322.31268310546875,
      "loss": 0.5935,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.1382293999195099,
      "rewards/margins": 0.4365927577018738,
      "rewards/rejected": -0.5748221278190613,
      "step": 830
    },
    {
      "epoch": 0.22,
      "grad_norm": 4.9375,
      "learning_rate": 4.769443696332272e-06,
      "logits/chosen": -2.532360792160034,
      "logits/rejected": -2.3724961280822754,
      "logps/chosen": -318.2106628417969,
      "logps/rejected": -332.56317138671875,
      "loss": 0.6173,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.34823811054229736,
      "rewards/margins": 0.31988316774368286,
      "rewards/rejected": -0.6681212782859802,
      "step": 840
    },
    {
      "epoch": 0.23,
      "grad_norm": 3.421875,
      "learning_rate": 4.759584424871302e-06,
      "logits/chosen": -2.573500394821167,
      "logits/rejected": -2.364952802658081,
      "logps/chosen": -327.43939208984375,
      "logps/rejected": -361.9600830078125,
      "loss": 0.5872,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.30317598581314087,
      "rewards/margins": 0.46783390641212463,
      "rewards/rejected": -0.7710099220275879,
      "step": 850
    },
    {
      "epoch": 0.23,
      "grad_norm": 11.25,
      "learning_rate": 4.749529369216246e-06,
      "logits/chosen": -2.4037342071533203,
      "logits/rejected": -2.084672451019287,
      "logps/chosen": -325.7606506347656,
      "logps/rejected": -296.64837646484375,
      "loss": 0.6562,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.4355030655860901,
      "rewards/margins": 0.22044646739959717,
      "rewards/rejected": -0.6559494733810425,
      "step": 860
    },
    {
      "epoch": 0.23,
      "grad_norm": 6.59375,
      "learning_rate": 4.7392794005985324e-06,
      "logits/chosen": -2.4404640197753906,
      "logits/rejected": -2.353076934814453,
      "logps/chosen": -329.2530212402344,
      "logps/rejected": -350.05047607421875,
      "loss": 0.6653,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.5785592198371887,
      "rewards/margins": 0.43728455901145935,
      "rewards/rejected": -1.0158437490463257,
      "step": 870
    },
    {
      "epoch": 0.23,
      "grad_norm": 3.5625,
      "learning_rate": 4.7288354071380415e-06,
      "logits/chosen": -2.557074785232544,
      "logits/rejected": -1.9732965230941772,
      "logps/chosen": -307.8256530761719,
      "logps/rejected": -340.72943115234375,
      "loss": 0.6392,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.49161940813064575,
      "rewards/margins": 0.40467125177383423,
      "rewards/rejected": -0.8962906002998352,
      "step": 880
    },
    {
      "epoch": 0.24,
      "grad_norm": 3.96875,
      "learning_rate": 4.7181982937661485e-06,
      "logits/chosen": -2.6376166343688965,
      "logits/rejected": -2.3852226734161377,
      "logps/chosen": -371.3333435058594,
      "logps/rejected": -364.9573059082031,
      "loss": 0.5667,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.4574738144874573,
      "rewards/margins": 0.3296016454696655,
      "rewards/rejected": -0.7870754599571228,
      "step": 890
    },
    {
      "epoch": 0.24,
      "grad_norm": 5.9375,
      "learning_rate": 4.707368982147318e-06,
      "logits/chosen": -2.264312982559204,
      "logits/rejected": -1.94890558719635,
      "logps/chosen": -312.9253845214844,
      "logps/rejected": -287.7906188964844,
      "loss": 0.623,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.6316430568695068,
      "rewards/margins": 0.3878110945224762,
      "rewards/rejected": -1.0194542407989502,
      "step": 900
    },
    {
      "epoch": 0.24,
      "eval_logits/chosen": -2.4140536785125732,
      "eval_logits/rejected": -2.315519332885742,
      "eval_logps/chosen": -310.3763732910156,
      "eval_logps/rejected": -329.31683349609375,
      "eval_loss": 0.5902401804924011,
      "eval_rewards/accuracies": 0.6854838728904724,
      "eval_rewards/chosen": -0.371052622795105,
      "eval_rewards/margins": 0.33340492844581604,
      "eval_rewards/rejected": -0.7044575810432434,
      "eval_runtime": 877.6564,
      "eval_samples_per_second": 4.519,
      "eval_steps_per_second": 0.141,
      "step": 900
    },
    {
      "epoch": 0.24,
      "grad_norm": 4.6875,
      "learning_rate": 4.696348410599244e-06,
      "logits/chosen": -2.282845973968506,
      "logits/rejected": -2.162170886993408,
      "logps/chosen": -297.3951110839844,
      "logps/rejected": -343.21185302734375,
      "loss": 0.6795,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.6412105560302734,
      "rewards/margins": 0.2741986811161041,
      "rewards/rejected": -0.9154092073440552,
      "step": 910
    },
    {
      "epoch": 0.25,
      "grad_norm": 4.375,
      "learning_rate": 4.685137534011549e-06,
      "logits/chosen": -2.258206367492676,
      "logits/rejected": -2.0936005115509033,
      "logps/chosen": -318.7940673828125,
      "logps/rejected": -330.2164611816406,
      "loss": 0.5613,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.5147677659988403,
      "rewards/margins": 0.4853156507015228,
      "rewards/rejected": -1.0000834465026855,
      "step": 920
    },
    {
      "epoch": 0.25,
      "grad_norm": 8.0625,
      "learning_rate": 4.673737323763048e-06,
      "logits/chosen": -2.1168596744537354,
      "logits/rejected": -2.4134414196014404,
      "logps/chosen": -230.5443878173828,
      "logps/rejected": -298.1512145996094,
      "loss": 0.5897,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.5229970216751099,
      "rewards/margins": 0.40220317244529724,
      "rewards/rejected": -0.9252001643180847,
      "step": 930
    },
    {
      "epoch": 0.25,
      "grad_norm": 3.65625,
      "learning_rate": 4.662148767637578e-06,
      "logits/chosen": -2.2486302852630615,
      "logits/rejected": -1.92658269405365,
      "logps/chosen": -339.2881774902344,
      "logps/rejected": -373.07281494140625,
      "loss": 0.5751,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.6724634170532227,
      "rewards/margins": 0.5648009181022644,
      "rewards/rejected": -1.2372645139694214,
      "step": 940
    },
    {
      "epoch": 0.25,
      "grad_norm": 6.09375,
      "learning_rate": 4.650372869738415e-06,
      "logits/chosen": -2.427340507507324,
      "logits/rejected": -2.3279542922973633,
      "logps/chosen": -306.71136474609375,
      "logps/rejected": -323.6715393066406,
      "loss": 0.5556,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.5925009846687317,
      "rewards/margins": 0.4229438900947571,
      "rewards/rejected": -1.0154448747634888,
      "step": 950
    },
    {
      "epoch": 0.26,
      "grad_norm": 5.5625,
      "learning_rate": 4.638410650401267e-06,
      "logits/chosen": -2.325500011444092,
      "logits/rejected": -1.8569977283477783,
      "logps/chosen": -316.23834228515625,
      "logps/rejected": -366.0225830078125,
      "loss": 0.5401,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.6303788423538208,
      "rewards/margins": 0.873699963092804,
      "rewards/rejected": -1.5040788650512695,
      "step": 960
    },
    {
      "epoch": 0.26,
      "grad_norm": 7.78125,
      "learning_rate": 4.626263146105875e-06,
      "logits/chosen": -2.2247474193573,
      "logits/rejected": -2.3088622093200684,
      "logps/chosen": -323.25225830078125,
      "logps/rejected": -351.7351379394531,
      "loss": 0.5745,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.8623113632202148,
      "rewards/margins": 0.25036412477493286,
      "rewards/rejected": -1.112675428390503,
      "step": 970
    },
    {
      "epoch": 0.26,
      "grad_norm": 8.1875,
      "learning_rate": 4.613931409386196e-06,
      "logits/chosen": -2.3282418251037598,
      "logits/rejected": -2.1665713787078857,
      "logps/chosen": -274.6808776855469,
      "logps/rejected": -372.7110900878906,
      "loss": 0.5535,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5883356928825378,
      "rewards/margins": 0.5928224921226501,
      "rewards/rejected": -1.181158185005188,
      "step": 980
    },
    {
      "epoch": 0.26,
      "grad_norm": 9.5,
      "learning_rate": 4.601416508739211e-06,
      "logits/chosen": -2.1633083820343018,
      "logits/rejected": -2.094113349914551,
      "logps/chosen": -245.37002563476562,
      "logps/rejected": -296.3724060058594,
      "loss": 0.5831,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.3249911963939667,
      "rewards/margins": 0.5508674383163452,
      "rewards/rejected": -0.8758586645126343,
      "step": 990
    },
    {
      "epoch": 0.27,
      "grad_norm": 6.21875,
      "learning_rate": 4.588719528532342e-06,
      "logits/chosen": -2.1389951705932617,
      "logits/rejected": -2.21720814704895,
      "logps/chosen": -318.3488464355469,
      "logps/rejected": -371.44879150390625,
      "loss": 0.6097,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.7991196513175964,
      "rewards/margins": 0.35454338788986206,
      "rewards/rejected": -1.1536630392074585,
      "step": 1000
    },
    {
      "epoch": 0.27,
      "eval_logits/chosen": -2.2752251625061035,
      "eval_logits/rejected": -2.178417682647705,
      "eval_logps/chosen": -316.1939392089844,
      "eval_logps/rejected": -338.4981384277344,
      "eval_loss": 0.5899558663368225,
      "eval_rewards/accuracies": 0.6733871102333069,
      "eval_rewards/chosen": -0.4292278289794922,
      "eval_rewards/margins": 0.3670426607131958,
      "eval_rewards/rejected": -0.7962705492973328,
      "eval_runtime": 877.651,
      "eval_samples_per_second": 4.519,
      "eval_steps_per_second": 0.141,
      "step": 1000
    },
    {
      "epoch": 0.27,
      "grad_norm": 11.75,
      "learning_rate": 4.575841568909494e-06,
      "logits/chosen": -1.9069007635116577,
      "logits/rejected": -2.1342906951904297,
      "logps/chosen": -329.4118957519531,
      "logps/rejected": -411.34906005859375,
      "loss": 0.6116,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -1.0803093910217285,
      "rewards/margins": 0.31983524560928345,
      "rewards/rejected": -1.4001445770263672,
      "step": 1010
    },
    {
      "epoch": 0.27,
      "grad_norm": 6.125,
      "learning_rate": 4.562783745695738e-06,
      "logits/chosen": -2.036458969116211,
      "logits/rejected": -2.0815341472625732,
      "logps/chosen": -361.594482421875,
      "logps/rejected": -412.66656494140625,
      "loss": 0.6405,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.8354271054267883,
      "rewards/margins": 0.42042288184165955,
      "rewards/rejected": -1.2558499574661255,
      "step": 1020
    },
    {
      "epoch": 0.27,
      "grad_norm": 7.0,
      "learning_rate": 4.549547190300622e-06,
      "logits/chosen": -2.151728630065918,
      "logits/rejected": -1.8993473052978516,
      "logps/chosen": -365.11163330078125,
      "logps/rejected": -419.2638244628906,
      "loss": 0.5733,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.7887364625930786,
      "rewards/margins": 0.519019603729248,
      "rewards/rejected": -1.3077560663223267,
      "step": 1030
    },
    {
      "epoch": 0.28,
      "grad_norm": 6.0625,
      "learning_rate": 4.536133049620143e-06,
      "logits/chosen": -2.2287821769714355,
      "logits/rejected": -1.9565813541412354,
      "logps/chosen": -374.8702087402344,
      "logps/rejected": -318.23406982421875,
      "loss": 0.6551,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.9378997683525085,
      "rewards/margins": 0.01763731800019741,
      "rewards/rejected": -0.9555371403694153,
      "step": 1040
    },
    {
      "epoch": 0.28,
      "grad_norm": 3.859375,
      "learning_rate": 4.522542485937369e-06,
      "logits/chosen": -2.1210978031158447,
      "logits/rejected": -1.9510753154754639,
      "logps/chosen": -275.03790283203125,
      "logps/rejected": -373.3275146484375,
      "loss": 0.5521,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.4916163980960846,
      "rewards/margins": 0.4955001473426819,
      "rewards/rejected": -0.9871165156364441,
      "step": 1050
    },
    {
      "epoch": 0.28,
      "grad_norm": 6.71875,
      "learning_rate": 4.508776676821739e-06,
      "logits/chosen": -2.1998026371002197,
      "logits/rejected": -2.325843334197998,
      "logps/chosen": -386.73199462890625,
      "logps/rejected": -392.3126525878906,
      "loss": 0.6131,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.7467738389968872,
      "rewards/margins": 0.22546665370464325,
      "rewards/rejected": -0.9722404479980469,
      "step": 1060
    },
    {
      "epoch": 0.29,
      "grad_norm": 4.21875,
      "learning_rate": 4.494836815027022e-06,
      "logits/chosen": -2.310345411300659,
      "logits/rejected": -2.049891710281372,
      "logps/chosen": -367.61102294921875,
      "logps/rejected": -350.09423828125,
      "loss": 0.589,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.6402325630187988,
      "rewards/margins": 0.42995938658714294,
      "rewards/rejected": -1.0701919794082642,
      "step": 1070
    },
    {
      "epoch": 0.29,
      "grad_norm": 4.84375,
      "learning_rate": 4.4807241083879774e-06,
      "logits/chosen": -2.246091604232788,
      "logits/rejected": -1.848070740699768,
      "logps/chosen": -343.4378967285156,
      "logps/rejected": -402.5697021484375,
      "loss": 0.5581,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6425604820251465,
      "rewards/margins": 0.6562389135360718,
      "rewards/rejected": -1.2987995147705078,
      "step": 1080
    },
    {
      "epoch": 0.29,
      "grad_norm": 6.75,
      "learning_rate": 4.466439779715696e-06,
      "logits/chosen": -2.3364076614379883,
      "logits/rejected": -1.852071762084961,
      "logps/chosen": -335.23638916015625,
      "logps/rejected": -323.6278991699219,
      "loss": 0.6211,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5784538388252258,
      "rewards/margins": 0.6166199445724487,
      "rewards/rejected": -1.1950738430023193,
      "step": 1090
    },
    {
      "epoch": 0.29,
      "grad_norm": 3.78125,
      "learning_rate": 4.451985066691649e-06,
      "logits/chosen": -2.189074754714966,
      "logits/rejected": -2.0351555347442627,
      "logps/chosen": -292.7684631347656,
      "logps/rejected": -349.8451232910156,
      "loss": 0.6136,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.507478654384613,
      "rewards/margins": 0.4647248685359955,
      "rewards/rejected": -0.9722035527229309,
      "step": 1100
    },
    {
      "epoch": 0.29,
      "eval_logits/chosen": -2.27095365524292,
      "eval_logits/rejected": -2.172745704650879,
      "eval_logps/chosen": -290.5160217285156,
      "eval_logps/rejected": -310.5219421386719,
      "eval_loss": 0.5784468650817871,
      "eval_rewards/accuracies": 0.7046371102333069,
      "eval_rewards/chosen": -0.17244914174079895,
      "eval_rewards/margins": 0.344059556722641,
      "eval_rewards/rejected": -0.5165086984634399,
      "eval_runtime": 877.5047,
      "eval_samples_per_second": 4.52,
      "eval_steps_per_second": 0.141,
      "step": 1100
    },
    {
      "epoch": 0.3,
      "grad_norm": 4.53125,
      "learning_rate": 4.437361221760449e-06,
      "logits/chosen": -2.330266237258911,
      "logits/rejected": -2.1454524993896484,
      "logps/chosen": -304.94744873046875,
      "logps/rejected": -362.92718505859375,
      "loss": 0.5965,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.4183235168457031,
      "rewards/margins": 0.43923693895339966,
      "rewards/rejected": -0.8575604557991028,
      "step": 1110
    },
    {
      "epoch": 0.3,
      "grad_norm": 3.40625,
      "learning_rate": 4.422569512021332e-06,
      "logits/chosen": -2.4553282260894775,
      "logits/rejected": -2.09441876411438,
      "logps/chosen": -293.5114440917969,
      "logps/rejected": -294.16290283203125,
      "loss": 0.6073,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.4032020568847656,
      "rewards/margins": 0.4483933448791504,
      "rewards/rejected": -0.8515955209732056,
      "step": 1120
    },
    {
      "epoch": 0.3,
      "grad_norm": 4.875,
      "learning_rate": 4.407611219118363e-06,
      "logits/chosen": -2.1917872428894043,
      "logits/rejected": -1.7611405849456787,
      "logps/chosen": -367.6034851074219,
      "logps/rejected": -358.23876953125,
      "loss": 0.6315,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.680256187915802,
      "rewards/margins": 0.24569912254810333,
      "rewards/rejected": -0.9259552955627441,
      "step": 1130
    },
    {
      "epoch": 0.3,
      "grad_norm": 4.4375,
      "learning_rate": 4.3924876391293915e-06,
      "logits/chosen": -2.269188642501831,
      "logits/rejected": -2.0305569171905518,
      "logps/chosen": -291.6146545410156,
      "logps/rejected": -363.7378845214844,
      "loss": 0.5708,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.6316682696342468,
      "rewards/margins": 0.4399014413356781,
      "rewards/rejected": -1.0715696811676025,
      "step": 1140
    },
    {
      "epoch": 0.31,
      "grad_norm": 3.65625,
      "learning_rate": 4.377200082453748e-06,
      "logits/chosen": -2.1187427043914795,
      "logits/rejected": -2.110783338546753,
      "logps/chosen": -370.59637451171875,
      "logps/rejected": -322.06463623046875,
      "loss": 0.5723,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.5637706518173218,
      "rewards/margins": 0.31620270013809204,
      "rewards/rejected": -0.879973292350769,
      "step": 1150
    },
    {
      "epoch": 0.31,
      "grad_norm": 5.34375,
      "learning_rate": 4.361749873698707e-06,
      "logits/chosen": -2.1275041103363037,
      "logits/rejected": -1.824798583984375,
      "logps/chosen": -242.3226318359375,
      "logps/rejected": -291.29595947265625,
      "loss": 0.5925,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.45665064454078674,
      "rewards/margins": 0.7356035113334656,
      "rewards/rejected": -1.1922540664672852,
      "step": 1160
    },
    {
      "epoch": 0.31,
      "grad_norm": 4.75,
      "learning_rate": 4.346138351564711e-06,
      "logits/chosen": -2.1028213500976562,
      "logits/rejected": -1.976200819015503,
      "logps/chosen": -324.274169921875,
      "logps/rejected": -355.374755859375,
      "loss": 0.5612,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.6071376800537109,
      "rewards/margins": 0.6079890727996826,
      "rewards/rejected": -1.2151267528533936,
      "step": 1170
    },
    {
      "epoch": 0.31,
      "grad_norm": 4.34375,
      "learning_rate": 4.330366868729376e-06,
      "logits/chosen": -2.3040060997009277,
      "logits/rejected": -1.9931379556655884,
      "logps/chosen": -280.6545715332031,
      "logps/rejected": -297.44140625,
      "loss": 0.5336,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.4123680591583252,
      "rewards/margins": 0.6075629591941833,
      "rewards/rejected": -1.0199309587478638,
      "step": 1180
    },
    {
      "epoch": 0.32,
      "grad_norm": 5.1875,
      "learning_rate": 4.3144367917302964e-06,
      "logits/chosen": -2.2867679595947266,
      "logits/rejected": -2.149374485015869,
      "logps/chosen": -296.0559387207031,
      "logps/rejected": -359.8106384277344,
      "loss": 0.5406,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.5958368182182312,
      "rewards/margins": 0.5092886686325073,
      "rewards/rejected": -1.1051254272460938,
      "step": 1190
    },
    {
      "epoch": 0.32,
      "grad_norm": 12.0,
      "learning_rate": 4.2983495008466285e-06,
      "logits/chosen": -2.0081915855407715,
      "logits/rejected": -2.187488317489624,
      "logps/chosen": -326.8968505859375,
      "logps/rejected": -440.26336669921875,
      "loss": 0.6567,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.7825566530227661,
      "rewards/margins": 0.21216025948524475,
      "rewards/rejected": -0.9947169423103333,
      "step": 1200
    },
    {
      "epoch": 0.32,
      "eval_logits/chosen": -2.2079641819000244,
      "eval_logits/rejected": -2.1098015308380127,
      "eval_logps/chosen": -312.2659912109375,
      "eval_logps/rejected": -344.7807922363281,
      "eval_loss": 0.5605653524398804,
      "eval_rewards/accuracies": 0.71875,
      "eval_rewards/chosen": -0.3899487555027008,
      "eval_rewards/margins": 0.4691483974456787,
      "eval_rewards/rejected": -0.8590971231460571,
      "eval_runtime": 877.4228,
      "eval_samples_per_second": 4.52,
      "eval_steps_per_second": 0.141,
      "step": 1200
    },
    {
      "epoch": 0.32,
      "grad_norm": 5.0625,
      "learning_rate": 4.2821063899795015e-06,
      "logits/chosen": -2.1430366039276123,
      "logits/rejected": -2.203555107116699,
      "logps/chosen": -325.0221252441406,
      "logps/rejected": -354.96868896484375,
      "loss": 0.6272,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.8981342315673828,
      "rewards/margins": 0.3002586364746094,
      "rewards/rejected": -1.1983931064605713,
      "step": 1210
    },
    {
      "epoch": 0.33,
      "grad_norm": 6.40625,
      "learning_rate": 4.265708866531238e-06,
      "logits/chosen": -2.227365493774414,
      "logits/rejected": -2.0136349201202393,
      "logps/chosen": -339.9341125488281,
      "logps/rejected": -294.302490234375,
      "loss": 0.6313,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.7618111371994019,
      "rewards/margins": 0.16503560543060303,
      "rewards/rejected": -0.9268468022346497,
      "step": 1220
    },
    {
      "epoch": 0.33,
      "grad_norm": 8.75,
      "learning_rate": 4.249158351283414e-06,
      "logits/chosen": -2.1937599182128906,
      "logits/rejected": -2.0537383556365967,
      "logps/chosen": -275.8554992675781,
      "logps/rejected": -302.3541564941406,
      "loss": 0.5808,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.587291955947876,
      "rewards/margins": 0.5341418385505676,
      "rewards/rejected": -1.1214337348937988,
      "step": 1230
    },
    {
      "epoch": 0.33,
      "grad_norm": 4.96875,
      "learning_rate": 4.232456278273743e-06,
      "logits/chosen": -2.2767930030822754,
      "logits/rejected": -2.043987512588501,
      "logps/chosen": -327.6272888183594,
      "logps/rejected": -372.13323974609375,
      "loss": 0.5629,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.4480348527431488,
      "rewards/margins": 0.5605012774467468,
      "rewards/rejected": -1.0085362195968628,
      "step": 1240
    },
    {
      "epoch": 0.33,
      "grad_norm": 6.75,
      "learning_rate": 4.215604094671835e-06,
      "logits/chosen": -2.340473175048828,
      "logits/rejected": -1.9723293781280518,
      "logps/chosen": -346.8924865722656,
      "logps/rejected": -362.45745849609375,
      "loss": 0.5781,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.7341182827949524,
      "rewards/margins": 0.49489647150039673,
      "rewards/rejected": -1.2290146350860596,
      "step": 1250
    },
    {
      "epoch": 0.34,
      "grad_norm": 5.375,
      "learning_rate": 4.198603260653792e-06,
      "logits/chosen": -2.0215647220611572,
      "logits/rejected": -2.1121063232421875,
      "logps/chosen": -313.0654602050781,
      "logps/rejected": -364.1198425292969,
      "loss": 0.6184,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5912294983863831,
      "rewards/margins": 0.5600056648254395,
      "rewards/rejected": -1.1512351036071777,
      "step": 1260
    },
    {
      "epoch": 0.34,
      "grad_norm": 5.375,
      "learning_rate": 4.181455249275701e-06,
      "logits/chosen": -2.2716126441955566,
      "logits/rejected": -2.1599488258361816,
      "logps/chosen": -332.73333740234375,
      "logps/rejected": -388.5229797363281,
      "loss": 0.5935,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.7099944353103638,
      "rewards/margins": 0.501876711845398,
      "rewards/rejected": -1.2118710279464722,
      "step": 1270
    },
    {
      "epoch": 0.34,
      "grad_norm": 4.28125,
      "learning_rate": 4.1641615463459926e-06,
      "logits/chosen": -2.3245482444763184,
      "logits/rejected": -2.045775890350342,
      "logps/chosen": -264.4940490722656,
      "logps/rejected": -269.84112548828125,
      "loss": 0.5528,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5773583650588989,
      "rewards/margins": 0.4559825360774994,
      "rewards/rejected": -1.0333408117294312,
      "step": 1280
    },
    {
      "epoch": 0.34,
      "grad_norm": 5.09375,
      "learning_rate": 4.146723650296701e-06,
      "logits/chosen": -2.27089262008667,
      "logits/rejected": -1.783856749534607,
      "logps/chosen": -450.95635986328125,
      "logps/rejected": -386.0379333496094,
      "loss": 0.593,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.7247825264930725,
      "rewards/margins": 0.47365275025367737,
      "rewards/rejected": -1.1984351873397827,
      "step": 1290
    },
    {
      "epoch": 0.35,
      "grad_norm": 6.78125,
      "learning_rate": 4.129143072053639e-06,
      "logits/chosen": -2.0932576656341553,
      "logits/rejected": -2.2065348625183105,
      "logps/chosen": -294.8106994628906,
      "logps/rejected": -366.6431579589844,
      "loss": 0.643,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -1.0785818099975586,
      "rewards/margins": 0.08744372427463531,
      "rewards/rejected": -1.1660255193710327,
      "step": 1300
    },
    {
      "epoch": 0.35,
      "eval_logits/chosen": -2.213148593902588,
      "eval_logits/rejected": -2.115633964538574,
      "eval_logps/chosen": -320.14202880859375,
      "eval_logps/rejected": -343.91937255859375,
      "eval_loss": 0.5634790062904358,
      "eval_rewards/accuracies": 0.7106854915618896,
      "eval_rewards/chosen": -0.4687090218067169,
      "eval_rewards/margins": 0.38177385926246643,
      "eval_rewards/rejected": -0.8504828810691833,
      "eval_runtime": 876.9925,
      "eval_samples_per_second": 4.522,
      "eval_steps_per_second": 0.141,
      "step": 1300
    },
    {
      "epoch": 0.35,
      "grad_norm": 4.28125,
      "learning_rate": 4.111421334905468e-06,
      "logits/chosen": -1.953237533569336,
      "logits/rejected": -1.9760719537734985,
      "logps/chosen": -349.6459045410156,
      "logps/rejected": -364.69708251953125,
      "loss": 0.5849,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.9659541249275208,
      "rewards/margins": 0.33340373635292053,
      "rewards/rejected": -1.2993578910827637,
      "step": 1310
    },
    {
      "epoch": 0.35,
      "grad_norm": 3.859375,
      "learning_rate": 4.093559974371725e-06,
      "logits/chosen": -2.375509262084961,
      "logits/rejected": -2.0767600536346436,
      "logps/chosen": -316.46044921875,
      "logps/rejected": -329.74334716796875,
      "loss": 0.5515,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.8277188539505005,
      "rewards/margins": 0.5007882118225098,
      "rewards/rejected": -1.3285070657730103,
      "step": 1320
    },
    {
      "epoch": 0.35,
      "grad_norm": 4.25,
      "learning_rate": 4.075560538069767e-06,
      "logits/chosen": -2.4089598655700684,
      "logits/rejected": -1.710634469985962,
      "logps/chosen": -377.76318359375,
      "logps/rejected": -376.8714904785156,
      "loss": 0.5921,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.7513238787651062,
      "rewards/margins": 0.5085414052009583,
      "rewards/rejected": -1.259865164756775,
      "step": 1330
    },
    {
      "epoch": 0.36,
      "grad_norm": 3.671875,
      "learning_rate": 4.05742458558068e-06,
      "logits/chosen": -2.383552312850952,
      "logits/rejected": -1.9252551794052124,
      "logps/chosen": -332.6451721191406,
      "logps/rejected": -305.6356506347656,
      "loss": 0.5992,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.6042526960372925,
      "rewards/margins": 0.3804389536380768,
      "rewards/rejected": -0.9846916198730469,
      "step": 1340
    },
    {
      "epoch": 0.36,
      "grad_norm": 6.0,
      "learning_rate": 4.039153688314146e-06,
      "logits/chosen": -2.35799241065979,
      "logits/rejected": -1.9954159259796143,
      "logps/chosen": -336.82574462890625,
      "logps/rejected": -322.95538330078125,
      "loss": 0.605,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.6419919729232788,
      "rewards/margins": 0.27151018381118774,
      "rewards/rejected": -0.9135022163391113,
      "step": 1350
    },
    {
      "epoch": 0.36,
      "grad_norm": 8.625,
      "learning_rate": 4.020749429372286e-06,
      "logits/chosen": -2.111227512359619,
      "logits/rejected": -2.0039620399475098,
      "logps/chosen": -403.4027404785156,
      "logps/rejected": -377.03558349609375,
      "loss": 0.5792,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7114434838294983,
      "rewards/margins": 0.5037499666213989,
      "rewards/rejected": -1.215193510055542,
      "step": 1360
    },
    {
      "epoch": 0.37,
      "grad_norm": 6.0625,
      "learning_rate": 4.002213403412492e-06,
      "logits/chosen": -2.217071533203125,
      "logits/rejected": -1.713439702987671,
      "logps/chosen": -348.6556396484375,
      "logps/rejected": -332.87677001953125,
      "loss": 0.6013,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.5714290142059326,
      "rewards/margins": 0.709094762802124,
      "rewards/rejected": -1.2805237770080566,
      "step": 1370
    },
    {
      "epoch": 0.37,
      "grad_norm": 4.59375,
      "learning_rate": 3.983547216509254e-06,
      "logits/chosen": -2.235619306564331,
      "logits/rejected": -1.8613979816436768,
      "logps/chosen": -326.8222351074219,
      "logps/rejected": -327.0437927246094,
      "loss": 0.5529,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.8135096430778503,
      "rewards/margins": 0.6219686269760132,
      "rewards/rejected": -1.4354782104492188,
      "step": 1380
    },
    {
      "epoch": 0.37,
      "grad_norm": 4.9375,
      "learning_rate": 3.964752486015001e-06,
      "logits/chosen": -2.1786246299743652,
      "logits/rejected": -2.0049967765808105,
      "logps/chosen": -326.6374206542969,
      "logps/rejected": -397.4110412597656,
      "loss": 0.5631,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.7791838645935059,
      "rewards/margins": 0.759290337562561,
      "rewards/rejected": -1.5384740829467773,
      "step": 1390
    },
    {
      "epoch": 0.37,
      "grad_norm": 6.40625,
      "learning_rate": 3.945830840419966e-06,
      "logits/chosen": -2.0745842456817627,
      "logits/rejected": -1.9324582815170288,
      "logps/chosen": -324.4366149902344,
      "logps/rejected": -358.8846740722656,
      "loss": 0.5965,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.9445751309394836,
      "rewards/margins": 0.5021809339523315,
      "rewards/rejected": -1.44675612449646,
      "step": 1400
    },
    {
      "epoch": 0.37,
      "eval_logits/chosen": -2.178713321685791,
      "eval_logits/rejected": -2.081547737121582,
      "eval_logps/chosen": -323.3370361328125,
      "eval_logps/rejected": -353.2680358886719,
      "eval_loss": 0.5605195164680481,
      "eval_rewards/accuracies": 0.6925403475761414,
      "eval_rewards/chosen": -0.5006589889526367,
      "eval_rewards/margins": 0.44331037998199463,
      "eval_rewards/rejected": -0.9439693689346313,
      "eval_runtime": 876.7648,
      "eval_samples_per_second": 4.523,
      "eval_steps_per_second": 0.141,
      "step": 1400
    },
    {
      "epoch": 0.38,
      "grad_norm": 6.65625,
      "learning_rate": 3.92678391921108e-06,
      "logits/chosen": -2.1755776405334473,
      "logits/rejected": -2.1848702430725098,
      "logps/chosen": -384.15582275390625,
      "logps/rejected": -429.12139892578125,
      "loss": 0.6615,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.042924404144287,
      "rewards/margins": 0.4284558892250061,
      "rewards/rejected": -1.471380352973938,
      "step": 1410
    },
    {
      "epoch": 0.38,
      "grad_norm": 8.4375,
      "learning_rate": 3.907613372729916e-06,
      "logits/chosen": -2.3648324012756348,
      "logits/rejected": -1.7681480646133423,
      "logps/chosen": -340.5740661621094,
      "logps/rejected": -320.7989807128906,
      "loss": 0.539,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.6408659219741821,
      "rewards/margins": 0.7925788164138794,
      "rewards/rejected": -1.433444857597351,
      "step": 1420
    },
    {
      "epoch": 0.38,
      "grad_norm": 5.3125,
      "learning_rate": 3.888320862029699e-06,
      "logits/chosen": -2.167858123779297,
      "logits/rejected": -1.7510411739349365,
      "logps/chosen": -414.93048095703125,
      "logps/rejected": -370.1501159667969,
      "loss": 0.5425,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.6807111501693726,
      "rewards/margins": 0.7038487195968628,
      "rewards/rejected": -1.3845598697662354,
      "step": 1430
    },
    {
      "epoch": 0.38,
      "grad_norm": 5.4375,
      "learning_rate": 3.868908058731376e-06,
      "logits/chosen": -1.8908532857894897,
      "logits/rejected": -1.8172489404678345,
      "logps/chosen": -349.3173828125,
      "logps/rejected": -421.11517333984375,
      "loss": 0.5539,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.8968253135681152,
      "rewards/margins": 0.5272932052612305,
      "rewards/rejected": -1.4241183996200562,
      "step": 1440
    },
    {
      "epoch": 0.39,
      "grad_norm": 10.6875,
      "learning_rate": 3.849376644878783e-06,
      "logits/chosen": -2.244992971420288,
      "logits/rejected": -1.7777221202850342,
      "logps/chosen": -405.39776611328125,
      "logps/rejected": -391.00665283203125,
      "loss": 0.6212,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -1.0117743015289307,
      "rewards/margins": 0.3718472421169281,
      "rewards/rejected": -1.3836214542388916,
      "step": 1450
    },
    {
      "epoch": 0.39,
      "grad_norm": 8.1875,
      "learning_rate": 3.829728312792895e-06,
      "logits/chosen": -2.2599198818206787,
      "logits/rejected": -1.8457260131835938,
      "logps/chosen": -364.1779479980469,
      "logps/rejected": -363.63031005859375,
      "loss": 0.6171,
      "rewards/accuracies": 0.42500001192092896,
      "rewards/chosen": -0.9937504529953003,
      "rewards/margins": 0.08279190957546234,
      "rewards/rejected": -1.0765423774719238,
      "step": 1460
    },
    {
      "epoch": 0.39,
      "grad_norm": 5.1875,
      "learning_rate": 3.8099647649251984e-06,
      "logits/chosen": -2.097252130508423,
      "logits/rejected": -2.136512279510498,
      "logps/chosen": -349.70977783203125,
      "logps/rejected": -385.281494140625,
      "loss": 0.5734,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.8512868881225586,
      "rewards/margins": 0.3268510699272156,
      "rewards/rejected": -1.178138017654419,
      "step": 1470
    },
    {
      "epoch": 0.39,
      "grad_norm": 5.8125,
      "learning_rate": 3.790087713710179e-06,
      "logits/chosen": -2.023951768875122,
      "logits/rejected": -2.042358875274658,
      "logps/chosen": -366.4421691894531,
      "logps/rejected": -446.12420654296875,
      "loss": 0.586,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.9663451910018921,
      "rewards/margins": 0.3857375979423523,
      "rewards/rejected": -1.3520828485488892,
      "step": 1480
    },
    {
      "epoch": 0.4,
      "grad_norm": 9.6875,
      "learning_rate": 3.770098881416945e-06,
      "logits/chosen": -1.8299843072891235,
      "logits/rejected": -2.064603090286255,
      "logps/chosen": -295.19183349609375,
      "logps/rejected": -319.17291259765625,
      "loss": 0.5844,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.0667449235916138,
      "rewards/margins": 0.25157517194747925,
      "rewards/rejected": -1.3183200359344482,
      "step": 1490
    },
    {
      "epoch": 0.4,
      "grad_norm": 4.375,
      "learning_rate": 3.7500000000000005e-06,
      "logits/chosen": -2.0763072967529297,
      "logits/rejected": -2.337611198425293,
      "logps/chosen": -309.19537353515625,
      "logps/rejected": -421.5337829589844,
      "loss": 0.5845,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.7840577363967896,
      "rewards/margins": 0.531219482421875,
      "rewards/rejected": -1.315277099609375,
      "step": 1500
    },
    {
      "epoch": 0.4,
      "eval_logits/chosen": -2.2018167972564697,
      "eval_logits/rejected": -2.1051440238952637,
      "eval_logps/chosen": -321.5331115722656,
      "eval_logps/rejected": -349.49151611328125,
      "eval_loss": 0.5531846284866333,
      "eval_rewards/accuracies": 0.7137096524238586,
      "eval_rewards/chosen": -0.48261958360671997,
      "eval_rewards/margins": 0.42358484864234924,
      "eval_rewards/rejected": -0.9062044024467468,
      "eval_runtime": 877.1936,
      "eval_samples_per_second": 4.521,
      "eval_steps_per_second": 0.141,
      "step": 1500
    },
    {
      "epoch": 0.4,
      "grad_norm": 6.375,
      "learning_rate": 3.7297928109491765e-06,
      "logits/chosen": -1.926375389099121,
      "logits/rejected": -2.0425329208374023,
      "logps/chosen": -272.95965576171875,
      "logps/rejected": -357.6120910644531,
      "loss": 0.5718,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.9400752186775208,
      "rewards/margins": 0.33768218755722046,
      "rewards/rejected": -1.2777572870254517,
      "step": 1510
    },
    {
      "epoch": 0.41,
      "grad_norm": 5.90625,
      "learning_rate": 3.7094790651387414e-06,
      "logits/chosen": -2.0436408519744873,
      "logits/rejected": -1.961226224899292,
      "logps/chosen": -321.0585021972656,
      "logps/rejected": -359.64862060546875,
      "loss": 0.5985,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.977936863899231,
      "rewards/margins": 0.2721673548221588,
      "rewards/rejected": -1.2501041889190674,
      "step": 1520
    },
    {
      "epoch": 0.41,
      "grad_norm": 7.6875,
      "learning_rate": 3.689060522675689e-06,
      "logits/chosen": -2.2064719200134277,
      "logits/rejected": -2.110166549682617,
      "logps/chosen": -372.19085693359375,
      "logps/rejected": -401.6531677246094,
      "loss": 0.6228,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.8774251937866211,
      "rewards/margins": 0.44976773858070374,
      "rewards/rejected": -1.3271929025650024,
      "step": 1530
    },
    {
      "epoch": 0.41,
      "grad_norm": 3.9375,
      "learning_rate": 3.668538952747236e-06,
      "logits/chosen": -2.1432735919952393,
      "logits/rejected": -2.109916925430298,
      "logps/chosen": -322.81878662109375,
      "logps/rejected": -435.3060607910156,
      "loss": 0.5577,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.778381884098053,
      "rewards/margins": 0.5868489742279053,
      "rewards/rejected": -1.365230917930603,
      "step": 1540
    },
    {
      "epoch": 0.41,
      "grad_norm": 6.28125,
      "learning_rate": 3.6479161334675294e-06,
      "logits/chosen": -2.0939512252807617,
      "logits/rejected": -2.030576229095459,
      "logps/chosen": -321.9136962890625,
      "logps/rejected": -379.1228332519531,
      "loss": 0.5877,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.9026378393173218,
      "rewards/margins": 0.3784787654876709,
      "rewards/rejected": -1.2811167240142822,
      "step": 1550
    },
    {
      "epoch": 0.42,
      "grad_norm": 5.21875,
      "learning_rate": 3.627193851723577e-06,
      "logits/chosen": -2.1279735565185547,
      "logits/rejected": -2.073931932449341,
      "logps/chosen": -287.392822265625,
      "logps/rejected": -368.3762512207031,
      "loss": 0.5758,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.8287652730941772,
      "rewards/margins": 0.48956507444381714,
      "rewards/rejected": -1.3183302879333496,
      "step": 1560
    },
    {
      "epoch": 0.42,
      "grad_norm": 5.5,
      "learning_rate": 3.6063739030204226e-06,
      "logits/chosen": -2.130516767501831,
      "logits/rejected": -2.1130735874176025,
      "logps/chosen": -300.5791320800781,
      "logps/rejected": -388.97027587890625,
      "loss": 0.5817,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.7073757648468018,
      "rewards/margins": 0.4895845055580139,
      "rewards/rejected": -1.196960210800171,
      "step": 1570
    },
    {
      "epoch": 0.42,
      "grad_norm": 7.96875,
      "learning_rate": 3.5854580913255706e-06,
      "logits/chosen": -2.110264778137207,
      "logits/rejected": -1.827431321144104,
      "logps/chosen": -349.3424377441406,
      "logps/rejected": -326.9812316894531,
      "loss": 0.5988,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.9111979603767395,
      "rewards/margins": 0.3332022428512573,
      "rewards/rejected": -1.2444000244140625,
      "step": 1580
    },
    {
      "epoch": 0.42,
      "grad_norm": 5.5,
      "learning_rate": 3.564448228912682e-06,
      "logits/chosen": -2.0401806831359863,
      "logits/rejected": -1.976910948753357,
      "logps/chosen": -306.19964599609375,
      "logps/rejected": -344.59393310546875,
      "loss": 0.6248,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.8229110836982727,
      "rewards/margins": 0.47934016585350037,
      "rewards/rejected": -1.3022512197494507,
      "step": 1590
    },
    {
      "epoch": 0.43,
      "grad_norm": 6.78125,
      "learning_rate": 3.543346136204545e-06,
      "logits/chosen": -2.1900687217712402,
      "logits/rejected": -2.151233196258545,
      "logps/chosen": -345.2456970214844,
      "logps/rejected": -422.19110107421875,
      "loss": 0.5877,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.7451485395431519,
      "rewards/margins": 0.5155580043792725,
      "rewards/rejected": -1.2607065439224243,
      "step": 1600
    },
    {
      "epoch": 0.43,
      "eval_logits/chosen": -2.192769765853882,
      "eval_logits/rejected": -2.096130847930908,
      "eval_logps/chosen": -316.2076721191406,
      "eval_logps/rejected": -343.0254211425781,
      "eval_loss": 0.5530534386634827,
      "eval_rewards/accuracies": 0.7247983813285828,
      "eval_rewards/chosen": -0.4293653070926666,
      "eval_rewards/margins": 0.4121782183647156,
      "eval_rewards/rejected": -0.841543436050415,
      "eval_runtime": 876.9589,
      "eval_samples_per_second": 4.522,
      "eval_steps_per_second": 0.141,
      "step": 1600
    },
    {
      "epoch": 0.43,
      "grad_norm": 4.75,
      "learning_rate": 3.522153641615345e-06,
      "logits/chosen": -2.026622772216797,
      "logits/rejected": -2.2146918773651123,
      "logps/chosen": -276.5377502441406,
      "logps/rejected": -427.2035217285156,
      "loss": 0.5921,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.6779539585113525,
      "rewards/margins": 0.4815135896205902,
      "rewards/rejected": -1.1594674587249756,
      "step": 1610
    },
    {
      "epoch": 0.43,
      "grad_norm": 7.0,
      "learning_rate": 3.5008725813922383e-06,
      "logits/chosen": -2.1807351112365723,
      "logits/rejected": -2.229403495788574,
      "logps/chosen": -361.24945068359375,
      "logps/rejected": -415.527587890625,
      "loss": 0.6148,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.8003236651420593,
      "rewards/margins": 0.38360854983329773,
      "rewards/rejected": -1.1839323043823242,
      "step": 1620
    },
    {
      "epoch": 0.43,
      "grad_norm": 5.125,
      "learning_rate": 3.4795047994562463e-06,
      "logits/chosen": -2.2824926376342773,
      "logits/rejected": -1.9699214696884155,
      "logps/chosen": -316.323486328125,
      "logps/rejected": -280.687255859375,
      "loss": 0.5881,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.5829593539237976,
      "rewards/margins": 0.45697903633117676,
      "rewards/rejected": -1.0399384498596191,
      "step": 1630
    },
    {
      "epoch": 0.44,
      "grad_norm": 4.15625,
      "learning_rate": 3.458052147242494e-06,
      "logits/chosen": -2.0903480052948,
      "logits/rejected": -2.0212974548339844,
      "logps/chosen": -288.0726013183594,
      "logps/rejected": -411.5322265625,
      "loss": 0.5918,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.8024877309799194,
      "rewards/margins": 0.5608288049697876,
      "rewards/rejected": -1.363316535949707,
      "step": 1640
    },
    {
      "epoch": 0.44,
      "grad_norm": 5.90625,
      "learning_rate": 3.436516483539781e-06,
      "logits/chosen": -2.1662111282348633,
      "logits/rejected": -1.940643548965454,
      "logps/chosen": -411.332275390625,
      "logps/rejected": -355.181396484375,
      "loss": 0.5876,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.8603065609931946,
      "rewards/margins": 0.41567105054855347,
      "rewards/rejected": -1.2759777307510376,
      "step": 1650
    },
    {
      "epoch": 0.44,
      "grad_norm": 8.25,
      "learning_rate": 3.4148996743295305e-06,
      "logits/chosen": -1.916351079940796,
      "logits/rejected": -1.8252594470977783,
      "logps/chosen": -366.1528625488281,
      "logps/rejected": -357.7076416015625,
      "loss": 0.5746,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.005068063735962,
      "rewards/margins": 0.4847866892814636,
      "rewards/rejected": -1.4898548126220703,
      "step": 1660
    },
    {
      "epoch": 0.45,
      "grad_norm": 4.75,
      "learning_rate": 3.3932035926241103e-06,
      "logits/chosen": -2.223201274871826,
      "logits/rejected": -1.8265941143035889,
      "logps/chosen": -332.1890563964844,
      "logps/rejected": -347.49432373046875,
      "loss": 0.5678,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.7291399240493774,
      "rewards/margins": 0.6893021464347839,
      "rewards/rejected": -1.4184421300888062,
      "step": 1670
    },
    {
      "epoch": 0.45,
      "grad_norm": 11.0625,
      "learning_rate": 3.3714301183045382e-06,
      "logits/chosen": -2.1152665615081787,
      "logits/rejected": -1.8417747020721436,
      "logps/chosen": -350.02264404296875,
      "logps/rejected": -389.01361083984375,
      "loss": 0.6001,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.9917596578598022,
      "rewards/margins": 0.7644199132919312,
      "rewards/rejected": -1.7561794519424438,
      "step": 1680
    },
    {
      "epoch": 0.45,
      "grad_norm": 6.625,
      "learning_rate": 3.349581137957604e-06,
      "logits/chosen": -2.107578754425049,
      "logits/rejected": -1.9682976007461548,
      "logps/chosen": -314.01953125,
      "logps/rejected": -322.01727294921875,
      "loss": 0.5943,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.8387876749038696,
      "rewards/margins": 0.35811254382133484,
      "rewards/rejected": -1.1969002485275269,
      "step": 1690
    },
    {
      "epoch": 0.45,
      "grad_norm": 4.4375,
      "learning_rate": 3.3276585447123957e-06,
      "logits/chosen": -2.203939437866211,
      "logits/rejected": -1.7744653224945068,
      "logps/chosen": -373.6509704589844,
      "logps/rejected": -332.88568115234375,
      "loss": 0.5718,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.7831062078475952,
      "rewards/margins": 0.4976167678833008,
      "rewards/rejected": -1.280723214149475,
      "step": 1700
    },
    {
      "epoch": 0.45,
      "eval_logits/chosen": -2.1491239070892334,
      "eval_logits/rejected": -2.052910804748535,
      "eval_logps/chosen": -314.5273132324219,
      "eval_logps/rejected": -342.04327392578125,
      "eval_loss": 0.5540661811828613,
      "eval_rewards/accuracies": 0.7167338728904724,
      "eval_rewards/chosen": -0.4125618636608124,
      "eval_rewards/margins": 0.4191596806049347,
      "eval_rewards/rejected": -0.8317215442657471,
      "eval_runtime": 876.8357,
      "eval_samples_per_second": 4.523,
      "eval_steps_per_second": 0.141,
      "step": 1700
    },
    {
      "epoch": 0.46,
      "grad_norm": 7.46875,
      "learning_rate": 3.3056642380762783e-06,
      "logits/chosen": -2.2383713722229004,
      "logits/rejected": -1.8662058115005493,
      "logps/chosen": -390.65643310546875,
      "logps/rejected": -368.0411682128906,
      "loss": 0.5803,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.9431484937667847,
      "rewards/margins": 0.2982689440250397,
      "rewards/rejected": -1.2414172887802124,
      "step": 1710
    },
    {
      "epoch": 0.46,
      "grad_norm": 7.625,
      "learning_rate": 3.2836001237702993e-06,
      "logits/chosen": -2.1432154178619385,
      "logits/rejected": -1.881866455078125,
      "logps/chosen": -385.0501708984375,
      "logps/rejected": -377.1376037597656,
      "loss": 0.5578,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5695873498916626,
      "rewards/margins": 0.5808383822441101,
      "rewards/rejected": -1.1504257917404175,
      "step": 1720
    },
    {
      "epoch": 0.46,
      "grad_norm": 10.9375,
      "learning_rate": 3.2614681135640696e-06,
      "logits/chosen": -1.9008926153182983,
      "logits/rejected": -2.077392101287842,
      "logps/chosen": -361.9366760253906,
      "logps/rejected": -422.39825439453125,
      "loss": 0.5719,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.8183330297470093,
      "rewards/margins": 0.40092211961746216,
      "rewards/rejected": -1.2192550897598267,
      "step": 1730
    },
    {
      "epoch": 0.46,
      "grad_norm": 5.59375,
      "learning_rate": 3.2392701251101172e-06,
      "logits/chosen": -2.119208335876465,
      "logits/rejected": -2.1508371829986572,
      "logps/chosen": -384.46234130859375,
      "logps/rejected": -447.52716064453125,
      "loss": 0.5934,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.8538224101066589,
      "rewards/margins": 0.4649757742881775,
      "rewards/rejected": -1.318798303604126,
      "step": 1740
    },
    {
      "epoch": 0.47,
      "grad_norm": 6.53125,
      "learning_rate": 3.217008081777726e-06,
      "logits/chosen": -2.22709584236145,
      "logits/rejected": -2.0004913806915283,
      "logps/chosen": -294.7169189453125,
      "logps/rejected": -362.31439208984375,
      "loss": 0.5653,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.006496548652649,
      "rewards/margins": 0.4909890294075012,
      "rewards/rejected": -1.4974855184555054,
      "step": 1750
    },
    {
      "epoch": 0.47,
      "grad_norm": 7.03125,
      "learning_rate": 3.1946839124862873e-06,
      "logits/chosen": -2.2114577293395996,
      "logits/rejected": -1.7875646352767944,
      "logps/chosen": -387.4165954589844,
      "logps/rejected": -404.4765625,
      "loss": 0.6057,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.8874310255050659,
      "rewards/margins": 0.5143918991088867,
      "rewards/rejected": -1.401822805404663,
      "step": 1760
    },
    {
      "epoch": 0.47,
      "grad_norm": 4.96875,
      "learning_rate": 3.1722995515381644e-06,
      "logits/chosen": -2.16326642036438,
      "logits/rejected": -1.8016834259033203,
      "logps/chosen": -335.76348876953125,
      "logps/rejected": -341.1480407714844,
      "loss": 0.5785,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.8947938084602356,
      "rewards/margins": 0.49824491143226624,
      "rewards/rejected": -1.3930385112762451,
      "step": 1770
    },
    {
      "epoch": 0.47,
      "grad_norm": 6.375,
      "learning_rate": 3.149856938451094e-06,
      "logits/chosen": -2.144491672515869,
      "logits/rejected": -2.0052943229675293,
      "logps/chosen": -309.77667236328125,
      "logps/rejected": -345.56927490234375,
      "loss": 0.5726,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.729701042175293,
      "rewards/margins": 0.6092637181282043,
      "rewards/rejected": -1.3389647006988525,
      "step": 1780
    },
    {
      "epoch": 0.48,
      "grad_norm": 7.34375,
      "learning_rate": 3.127358017790132e-06,
      "logits/chosen": -2.2762904167175293,
      "logits/rejected": -2.0918142795562744,
      "logps/chosen": -339.5685119628906,
      "logps/rejected": -377.6889343261719,
      "loss": 0.5804,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7288593053817749,
      "rewards/margins": 0.3746117055416107,
      "rewards/rejected": -1.103471040725708,
      "step": 1790
    },
    {
      "epoch": 0.48,
      "grad_norm": 5.75,
      "learning_rate": 3.1048047389991693e-06,
      "logits/chosen": -2.268585681915283,
      "logits/rejected": -2.0436148643493652,
      "logps/chosen": -357.82220458984375,
      "logps/rejected": -389.7935485839844,
      "loss": 0.5572,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.6428190469741821,
      "rewards/margins": 0.6618500351905823,
      "rewards/rejected": -1.3046691417694092,
      "step": 1800
    },
    {
      "epoch": 0.48,
      "eval_logits/chosen": -2.1341605186462402,
      "eval_logits/rejected": -2.0372560024261475,
      "eval_logps/chosen": -322.0596008300781,
      "eval_logps/rejected": -348.853759765625,
      "eval_loss": 0.5573899149894714,
      "eval_rewards/accuracies": 0.7177419066429138,
      "eval_rewards/chosen": -0.48788511753082275,
      "eval_rewards/margins": 0.41194188594818115,
      "eval_rewards/rejected": -0.8998271226882935,
      "eval_runtime": 876.8936,
      "eval_samples_per_second": 4.523,
      "eval_steps_per_second": 0.141,
      "step": 1800
    },
    {
      "epoch": 0.48,
      "grad_norm": 9.0625,
      "learning_rate": 3.082199056232015e-06,
      "logits/chosen": -2.0984890460968018,
      "logits/rejected": -1.8662197589874268,
      "logps/chosen": -408.97137451171875,
      "logps/rejected": -377.33160400390625,
      "loss": 0.5916,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.0912597179412842,
      "rewards/margins": 0.27257663011550903,
      "rewards/rejected": -1.363836407661438,
      "step": 1810
    },
    {
      "epoch": 0.49,
      "grad_norm": 12.0,
      "learning_rate": 3.059542928183079e-06,
      "logits/chosen": -2.136916399002075,
      "logits/rejected": -2.0036771297454834,
      "logps/chosen": -322.7129211425781,
      "logps/rejected": -469.54058837890625,
      "loss": 0.5879,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.7480248212814331,
      "rewards/margins": 0.8040331602096558,
      "rewards/rejected": -1.5520579814910889,
      "step": 1820
    },
    {
      "epoch": 0.49,
      "grad_norm": 7.6875,
      "learning_rate": 3.0368383179176584e-06,
      "logits/chosen": -2.1701912879943848,
      "logits/rejected": -2.025696039199829,
      "logps/chosen": -412.67340087890625,
      "logps/rejected": -409.0660705566406,
      "loss": 0.5752,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.8698748350143433,
      "rewards/margins": 0.585756778717041,
      "rewards/rejected": -1.4556314945220947,
      "step": 1830
    },
    {
      "epoch": 0.49,
      "grad_norm": 5.34375,
      "learning_rate": 3.0140871927018466e-06,
      "logits/chosen": -1.9468629360198975,
      "logits/rejected": -1.7946693897247314,
      "logps/chosen": -346.06622314453125,
      "logps/rejected": -372.0867004394531,
      "loss": 0.5783,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.9145089983940125,
      "rewards/margins": 0.46797457337379456,
      "rewards/rejected": -1.382483720779419,
      "step": 1840
    },
    {
      "epoch": 0.49,
      "grad_norm": 5.625,
      "learning_rate": 2.9912915238320755e-06,
      "logits/chosen": -2.0908687114715576,
      "logits/rejected": -1.7212072610855103,
      "logps/chosen": -361.2541198730469,
      "logps/rejected": -354.72015380859375,
      "loss": 0.5777,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.9880391955375671,
      "rewards/margins": 0.4162752032279968,
      "rewards/rejected": -1.4043142795562744,
      "step": 1850
    },
    {
      "epoch": 0.5,
      "grad_norm": 8.5,
      "learning_rate": 2.9684532864643123e-06,
      "logits/chosen": -1.7207921743392944,
      "logits/rejected": -2.087820053100586,
      "logps/chosen": -360.118896484375,
      "logps/rejected": -435.30450439453125,
      "loss": 0.6191,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -1.192453145980835,
      "rewards/margins": 0.10163470357656479,
      "rewards/rejected": -1.2940876483917236,
      "step": 1860
    },
    {
      "epoch": 0.5,
      "grad_norm": 7.625,
      "learning_rate": 2.945574459442917e-06,
      "logits/chosen": -2.0975470542907715,
      "logits/rejected": -1.6722681522369385,
      "logps/chosen": -356.82965087890625,
      "logps/rejected": -339.8148193359375,
      "loss": 0.5948,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.0466082096099854,
      "rewards/margins": 0.5006224513053894,
      "rewards/rejected": -1.54723060131073,
      "step": 1870
    },
    {
      "epoch": 0.5,
      "grad_norm": 8.0,
      "learning_rate": 2.922657025129185e-06,
      "logits/chosen": -2.076359510421753,
      "logits/rejected": -1.798077940940857,
      "logps/chosen": -402.6291198730469,
      "logps/rejected": -316.05194091796875,
      "loss": 0.6305,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.7700011730194092,
      "rewards/margins": 0.3978516459465027,
      "rewards/rejected": -1.167852759361267,
      "step": 1880
    },
    {
      "epoch": 0.5,
      "grad_norm": 4.40625,
      "learning_rate": 2.8997029692295875e-06,
      "logits/chosen": -2.004284381866455,
      "logits/rejected": -1.9049739837646484,
      "logps/chosen": -331.011962890625,
      "logps/rejected": -402.10858154296875,
      "loss": 0.5481,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.7718345522880554,
      "rewards/margins": 0.5906983017921448,
      "rewards/rejected": -1.3625328540802002,
      "step": 1890
    },
    {
      "epoch": 0.51,
      "grad_norm": 7.65625,
      "learning_rate": 2.876714280623708e-06,
      "logits/chosen": -2.255525827407837,
      "logits/rejected": -1.9905617237091064,
      "logps/chosen": -417.24566650390625,
      "logps/rejected": -408.58294677734375,
      "loss": 0.6168,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -1.1178069114685059,
      "rewards/margins": 0.40775051712989807,
      "rewards/rejected": -1.525557518005371,
      "step": 1900
    },
    {
      "epoch": 0.51,
      "eval_logits/chosen": -2.132965564727783,
      "eval_logits/rejected": -2.036834239959717,
      "eval_logps/chosen": -317.63360595703125,
      "eval_logps/rejected": -343.83221435546875,
      "eval_loss": 0.5587943196296692,
      "eval_rewards/accuracies": 0.7046371102333069,
      "eval_rewards/chosen": -0.44362518191337585,
      "eval_rewards/margins": 0.4059860110282898,
      "eval_rewards/rejected": -0.8496111631393433,
      "eval_runtime": 877.0541,
      "eval_samples_per_second": 4.522,
      "eval_steps_per_second": 0.141,
      "step": 1900
    },
    {
      "epoch": 0.51,
      "grad_norm": 5.03125,
      "learning_rate": 2.8536929511919227e-06,
      "logits/chosen": -2.1780858039855957,
      "logits/rejected": -2.121446132659912,
      "logps/chosen": -352.31231689453125,
      "logps/rejected": -357.2597961425781,
      "loss": 0.5958,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.8549824953079224,
      "rewards/margins": 0.3630499243736267,
      "rewards/rejected": -1.2180324792861938,
      "step": 1910
    },
    {
      "epoch": 0.51,
      "grad_norm": 5.40625,
      "learning_rate": 2.8306409756428067e-06,
      "logits/chosen": -2.1930370330810547,
      "logits/rejected": -2.1226084232330322,
      "logps/chosen": -265.948486328125,
      "logps/rejected": -328.2722473144531,
      "loss": 0.5534,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.6690794825553894,
      "rewards/margins": 0.5500704646110535,
      "rewards/rejected": -1.2191498279571533,
      "step": 1920
    },
    {
      "epoch": 0.51,
      "grad_norm": 6.46875,
      "learning_rate": 2.807560351340302e-06,
      "logits/chosen": -1.9755738973617554,
      "logits/rejected": -1.8984334468841553,
      "logps/chosen": -363.10052490234375,
      "logps/rejected": -376.0189514160156,
      "loss": 0.6125,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.9100989103317261,
      "rewards/margins": 0.3987920880317688,
      "rewards/rejected": -1.3088910579681396,
      "step": 1930
    },
    {
      "epoch": 0.52,
      "grad_norm": 4.90625,
      "learning_rate": 2.7844530781306544e-06,
      "logits/chosen": -1.9818652868270874,
      "logits/rejected": -2.0742409229278564,
      "logps/chosen": -319.0859680175781,
      "logps/rejected": -387.8902893066406,
      "loss": 0.5539,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.7758256793022156,
      "rewards/margins": 0.4159775674343109,
      "rewards/rejected": -1.1918033361434937,
      "step": 1940
    },
    {
      "epoch": 0.52,
      "grad_norm": 7.71875,
      "learning_rate": 2.761321158169134e-06,
      "logits/chosen": -2.1096644401550293,
      "logits/rejected": -1.9356002807617188,
      "logps/chosen": -374.189208984375,
      "logps/rejected": -381.3500061035156,
      "loss": 0.6165,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.0782049894332886,
      "rewards/margins": 0.3887743651866913,
      "rewards/rejected": -1.4669793844223022,
      "step": 1950
    },
    {
      "epoch": 0.52,
      "grad_norm": 7.71875,
      "learning_rate": 2.738166595746554e-06,
      "logits/chosen": -2.209965944290161,
      "logits/rejected": -1.9800083637237549,
      "logps/chosen": -402.1955261230469,
      "logps/rejected": -454.88946533203125,
      "loss": 0.5709,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -0.579013466835022,
      "rewards/margins": 0.9117425084114075,
      "rewards/rejected": -1.4907559156417847,
      "step": 1960
    },
    {
      "epoch": 0.53,
      "grad_norm": 5.53125,
      "learning_rate": 2.7149913971156105e-06,
      "logits/chosen": -2.116140604019165,
      "logits/rejected": -1.7414888143539429,
      "logps/chosen": -295.318359375,
      "logps/rejected": -383.25164794921875,
      "loss": 0.5533,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.8586200475692749,
      "rewards/margins": 0.7974663376808167,
      "rewards/rejected": -1.6560863256454468,
      "step": 1970
    },
    {
      "epoch": 0.53,
      "grad_norm": 5.5625,
      "learning_rate": 2.6917975703170466e-06,
      "logits/chosen": -2.2772057056427,
      "logits/rejected": -1.8113327026367188,
      "logps/chosen": -375.3372497558594,
      "logps/rejected": -316.6410217285156,
      "loss": 0.5827,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.699476420879364,
      "rewards/margins": 0.46018609404563904,
      "rewards/rejected": -1.1596624851226807,
      "step": 1980
    },
    {
      "epoch": 0.53,
      "grad_norm": 6.78125,
      "learning_rate": 2.668587125005663e-06,
      "logits/chosen": -1.9756250381469727,
      "logits/rejected": -2.074709177017212,
      "logps/chosen": -281.4726257324219,
      "logps/rejected": -356.9256286621094,
      "loss": 0.6258,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.5925463438034058,
      "rewards/margins": 0.3726350665092468,
      "rewards/rejected": -0.9651815295219421,
      "step": 1990
    },
    {
      "epoch": 0.53,
      "grad_norm": 5.125,
      "learning_rate": 2.6453620722761897e-06,
      "logits/chosen": -2.0949454307556152,
      "logits/rejected": -1.8415111303329468,
      "logps/chosen": -313.1147766113281,
      "logps/rejected": -369.34063720703125,
      "loss": 0.5584,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.8286046981811523,
      "rewards/margins": 0.4232611060142517,
      "rewards/rejected": -1.2518658638000488,
      "step": 2000
    },
    {
      "epoch": 0.53,
      "eval_logits/chosen": -2.097086191177368,
      "eval_logits/rejected": -2.0010178089141846,
      "eval_logps/chosen": -317.803466796875,
      "eval_logps/rejected": -345.9210205078125,
      "eval_loss": 0.5511365532875061,
      "eval_rewards/accuracies": 0.71875,
      "eval_rewards/chosen": -0.4453234374523163,
      "eval_rewards/margins": 0.42517581582069397,
      "eval_rewards/rejected": -0.870499312877655,
      "eval_runtime": 878.9356,
      "eval_samples_per_second": 4.512,
      "eval_steps_per_second": 0.141,
      "step": 2000
    },
    {
      "epoch": 0.54,
      "grad_norm": 9.625,
      "learning_rate": 2.6221244244890336e-06,
      "logits/chosen": -2.1206412315368652,
      "logits/rejected": -2.026148557662964,
      "logps/chosen": -373.2939147949219,
      "logps/rejected": -422.64361572265625,
      "loss": 0.5853,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.003935694694519,
      "rewards/margins": 0.44874754548072815,
      "rewards/rejected": -1.4526832103729248,
      "step": 2010
    },
    {
      "epoch": 0.54,
      "grad_norm": 5.1875,
      "learning_rate": 2.5988761950959133e-06,
      "logits/chosen": -2.0529377460479736,
      "logits/rejected": -1.8911529779434204,
      "logps/chosen": -291.65765380859375,
      "logps/rejected": -323.7792053222656,
      "loss": 0.5372,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.7897790670394897,
      "rewards/margins": 0.582499623298645,
      "rewards/rejected": -1.3722788095474243,
      "step": 2020
    },
    {
      "epoch": 0.54,
      "grad_norm": 6.84375,
      "learning_rate": 2.575619398465402e-06,
      "logits/chosen": -1.9023103713989258,
      "logits/rejected": -1.783992052078247,
      "logps/chosen": -363.25848388671875,
      "logps/rejected": -382.92962646484375,
      "loss": 0.5864,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.9610140919685364,
      "rewards/margins": 0.2774900794029236,
      "rewards/rejected": -1.23850417137146,
      "step": 2030
    },
    {
      "epoch": 0.54,
      "grad_norm": 9.125,
      "learning_rate": 2.5523560497083927e-06,
      "logits/chosen": -2.3458540439605713,
      "logits/rejected": -1.759214162826538,
      "logps/chosen": -442.4271545410156,
      "logps/rejected": -416.55682373046875,
      "loss": 0.5682,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.060951590538025,
      "rewards/margins": 0.5103678107261658,
      "rewards/rejected": -1.5713194608688354,
      "step": 2040
    },
    {
      "epoch": 0.55,
      "grad_norm": 5.15625,
      "learning_rate": 2.5290881645034932e-06,
      "logits/chosen": -2.113590955734253,
      "logits/rejected": -2.0374083518981934,
      "logps/chosen": -399.57952880859375,
      "logps/rejected": -368.1147766113281,
      "loss": 0.6247,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -1.1801789999008179,
      "rewards/margins": 0.20426928997039795,
      "rewards/rejected": -1.3844484090805054,
      "step": 2050
    },
    {
      "epoch": 0.55,
      "grad_norm": 7.75,
      "learning_rate": 2.5058177589223766e-06,
      "logits/chosen": -2.2251384258270264,
      "logits/rejected": -2.001913070678711,
      "logps/chosen": -391.7063903808594,
      "logps/rejected": -391.37213134765625,
      "loss": 0.5866,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.9196218252182007,
      "rewards/margins": 0.6790776252746582,
      "rewards/rejected": -1.5986994504928589,
      "step": 2060
    },
    {
      "epoch": 0.55,
      "grad_norm": 11.875,
      "learning_rate": 2.482546849255096e-06,
      "logits/chosen": -2.0615832805633545,
      "logits/rejected": -1.880631446838379,
      "logps/chosen": -471.28070068359375,
      "logps/rejected": -399.2253112792969,
      "loss": 0.5798,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.0096040964126587,
      "rewards/margins": 0.47060003876686096,
      "rewards/rejected": -1.4802041053771973,
      "step": 2070
    },
    {
      "epoch": 0.55,
      "grad_norm": 9.5625,
      "learning_rate": 2.4592774518353858e-06,
      "logits/chosen": -1.9678665399551392,
      "logits/rejected": -1.8819150924682617,
      "logps/chosen": -389.978759765625,
      "logps/rejected": -381.9435119628906,
      "loss": 0.5538,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.9292455911636353,
      "rewards/margins": 0.5575056672096252,
      "rewards/rejected": -1.4867513179779053,
      "step": 2080
    },
    {
      "epoch": 0.56,
      "grad_norm": 8.1875,
      "learning_rate": 2.436011582865945e-06,
      "logits/chosen": -2.15788197517395,
      "logits/rejected": -1.932862639427185,
      "logps/chosen": -407.6434631347656,
      "logps/rejected": -450.16357421875,
      "loss": 0.6604,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.9819389581680298,
      "rewards/margins": 0.40509581565856934,
      "rewards/rejected": -1.3870347738265991,
      "step": 2090
    },
    {
      "epoch": 0.56,
      "grad_norm": 4.96875,
      "learning_rate": 2.4127512582437486e-06,
      "logits/chosen": -2.1919784545898438,
      "logits/rejected": -1.9215013980865479,
      "logps/chosen": -371.1155700683594,
      "logps/rejected": -427.61431884765625,
      "loss": 0.5863,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.8020402789115906,
      "rewards/margins": 0.5108429193496704,
      "rewards/rejected": -1.3128832578659058,
      "step": 2100
    },
    {
      "epoch": 0.56,
      "eval_logits/chosen": -2.057666301727295,
      "eval_logits/rejected": -1.9625401496887207,
      "eval_logps/chosen": -320.4117126464844,
      "eval_logps/rejected": -349.6664123535156,
      "eval_loss": 0.5503637194633484,
      "eval_rewards/accuracies": 0.7217742204666138,
      "eval_rewards/chosen": -0.4714057743549347,
      "eval_rewards/margins": 0.4365474581718445,
      "eval_rewards/rejected": -0.9079532027244568,
      "eval_runtime": 879.7826,
      "eval_samples_per_second": 4.508,
      "eval_steps_per_second": 0.141,
      "step": 2100
    },
    {
      "epoch": 0.56,
      "grad_norm": 4.71875,
      "learning_rate": 2.3894984933853734e-06,
      "logits/chosen": -2.013716459274292,
      "logits/rejected": -1.9731485843658447,
      "logps/chosen": -348.6014709472656,
      "logps/rejected": -376.54302978515625,
      "loss": 0.5484,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.7389403581619263,
      "rewards/margins": 0.6295742392539978,
      "rewards/rejected": -1.3685147762298584,
      "step": 2110
    },
    {
      "epoch": 0.57,
      "grad_norm": 6.59375,
      "learning_rate": 2.366255303052377e-06,
      "logits/chosen": -2.0231544971466064,
      "logits/rejected": -1.8356990814208984,
      "logps/chosen": -351.6130065917969,
      "logps/rejected": -353.20684814453125,
      "loss": 0.5571,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.9260925054550171,
      "rewards/margins": 0.4884273409843445,
      "rewards/rejected": -1.4145197868347168,
      "step": 2120
    },
    {
      "epoch": 0.57,
      "grad_norm": 5.125,
      "learning_rate": 2.3430237011767166e-06,
      "logits/chosen": -2.1474902629852295,
      "logits/rejected": -1.799119234085083,
      "logps/chosen": -371.572509765625,
      "logps/rejected": -331.91900634765625,
      "loss": 0.5823,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.9327939748764038,
      "rewards/margins": 0.35704803466796875,
      "rewards/rejected": -1.2898420095443726,
      "step": 2130
    },
    {
      "epoch": 0.57,
      "grad_norm": 5.9375,
      "learning_rate": 2.319805700686257e-06,
      "logits/chosen": -2.1031956672668457,
      "logits/rejected": -1.8029677867889404,
      "logps/chosen": -357.91748046875,
      "logps/rejected": -379.89276123046875,
      "loss": 0.5865,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -1.0189263820648193,
      "rewards/margins": 0.3955627381801605,
      "rewards/rejected": -1.4144891500473022,
      "step": 2140
    },
    {
      "epoch": 0.57,
      "grad_norm": 6.28125,
      "learning_rate": 2.296603313330355e-06,
      "logits/chosen": -1.986278772354126,
      "logits/rejected": -1.8132734298706055,
      "logps/chosen": -303.4059143066406,
      "logps/rejected": -317.03070068359375,
      "loss": 0.6087,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.832554042339325,
      "rewards/margins": 0.42461901903152466,
      "rewards/rejected": -1.2571730613708496,
      "step": 2150
    },
    {
      "epoch": 0.58,
      "grad_norm": 4.90625,
      "learning_rate": 2.2734185495055503e-06,
      "logits/chosen": -1.980285882949829,
      "logits/rejected": -1.6074237823486328,
      "logps/chosen": -365.8879089355469,
      "logps/rejected": -398.4886474609375,
      "loss": 0.5564,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.9432777166366577,
      "rewards/margins": 0.5740941166877747,
      "rewards/rejected": -1.5173717737197876,
      "step": 2160
    },
    {
      "epoch": 0.58,
      "grad_norm": 4.78125,
      "learning_rate": 2.250253418081373e-06,
      "logits/chosen": -2.1542155742645264,
      "logits/rejected": -1.6605072021484375,
      "logps/chosen": -311.1054992675781,
      "logps/rejected": -316.6908874511719,
      "loss": 0.5129,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5355050563812256,
      "rewards/margins": 0.8455634117126465,
      "rewards/rejected": -1.381068468093872,
      "step": 2170
    },
    {
      "epoch": 0.58,
      "grad_norm": 5.53125,
      "learning_rate": 2.22710992622628e-06,
      "logits/chosen": -2.1749789714813232,
      "logits/rejected": -1.857287049293518,
      "logps/chosen": -353.4165344238281,
      "logps/rejected": -366.36920166015625,
      "loss": 0.5354,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5281771421432495,
      "rewards/margins": 0.7151774168014526,
      "rewards/rejected": -1.2433545589447021,
      "step": 2180
    },
    {
      "epoch": 0.58,
      "grad_norm": 11.25,
      "learning_rate": 2.2039900792337477e-06,
      "logits/chosen": -1.91474187374115,
      "logits/rejected": -1.8847659826278687,
      "logps/chosen": -319.30072021484375,
      "logps/rejected": -368.90716552734375,
      "loss": 0.6533,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.8085009455680847,
      "rewards/margins": 0.4654998779296875,
      "rewards/rejected": -1.274000883102417,
      "step": 2190
    },
    {
      "epoch": 0.59,
      "grad_norm": 5.25,
      "learning_rate": 2.1808958803485134e-06,
      "logits/chosen": -2.18727445602417,
      "logits/rejected": -1.951789140701294,
      "logps/chosen": -368.6776123046875,
      "logps/rejected": -462.8372497558594,
      "loss": 0.5805,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.8822237253189087,
      "rewards/margins": 0.4559556841850281,
      "rewards/rejected": -1.338179349899292,
      "step": 2200
    },
    {
      "epoch": 0.59,
      "eval_logits/chosen": -2.052835464477539,
      "eval_logits/rejected": -1.9578536748886108,
      "eval_logps/chosen": -313.297119140625,
      "eval_logps/rejected": -342.27703857421875,
      "eval_loss": 0.55088871717453,
      "eval_rewards/accuracies": 0.7217742204666138,
      "eval_rewards/chosen": -0.40026023983955383,
      "eval_rewards/margins": 0.4337990880012512,
      "eval_rewards/rejected": -0.8340593576431274,
      "eval_runtime": 879.3272,
      "eval_samples_per_second": 4.51,
      "eval_steps_per_second": 0.141,
      "step": 2200
    },
    {
      "epoch": 0.59,
      "grad_norm": 5.5,
      "learning_rate": 2.157829330593008e-06,
      "logits/chosen": -2.078173875808716,
      "logits/rejected": -1.839337706565857,
      "logps/chosen": -365.9580993652344,
      "logps/rejected": -411.00543212890625,
      "loss": 0.567,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.9710946083068848,
      "rewards/margins": 0.4620150625705719,
      "rewards/rejected": -1.4331096410751343,
      "step": 2210
    },
    {
      "epoch": 0.59,
      "grad_norm": 4.59375,
      "learning_rate": 2.134792428593971e-06,
      "logits/chosen": -1.9574873447418213,
      "logits/rejected": -1.9199317693710327,
      "logps/chosen": -292.80755615234375,
      "logps/rejected": -338.2846984863281,
      "loss": 0.5759,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.5960100293159485,
      "rewards/margins": 0.40044301748275757,
      "rewards/rejected": -0.9964529275894165,
      "step": 2220
    },
    {
      "epoch": 0.59,
      "grad_norm": 5.0,
      "learning_rate": 2.1117871704092818e-06,
      "logits/chosen": -2.0066750049591064,
      "logits/rejected": -1.7941272258758545,
      "logps/chosen": -332.23162841796875,
      "logps/rejected": -425.4258728027344,
      "loss": 0.548,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7843636870384216,
      "rewards/margins": 0.5066189169883728,
      "rewards/rejected": -1.2909826040267944,
      "step": 2230
    },
    {
      "epoch": 0.6,
      "grad_norm": 7.8125,
      "learning_rate": 2.0888155493550027e-06,
      "logits/chosen": -1.8008428812026978,
      "logits/rejected": -1.9330183267593384,
      "logps/chosen": -326.91748046875,
      "logps/rejected": -401.5304260253906,
      "loss": 0.5706,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8298208117485046,
      "rewards/margins": 0.5255243182182312,
      "rewards/rejected": -1.3553451299667358,
      "step": 2240
    },
    {
      "epoch": 0.6,
      "grad_norm": 3.640625,
      "learning_rate": 2.0658795558326745e-06,
      "logits/chosen": -1.9140123128890991,
      "logits/rejected": -1.9264888763427734,
      "logps/chosen": -369.296630859375,
      "logps/rejected": -354.44366455078125,
      "loss": 0.5665,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.9053546190261841,
      "rewards/margins": 0.1885354071855545,
      "rewards/rejected": -1.0938899517059326,
      "step": 2250
    },
    {
      "epoch": 0.6,
      "grad_norm": 4.9375,
      "learning_rate": 2.0429811771568468e-06,
      "logits/chosen": -2.0708587169647217,
      "logits/rejected": -1.7960236072540283,
      "logps/chosen": -374.08697509765625,
      "logps/rejected": -357.5560607910156,
      "loss": 0.5682,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.1575151681900024,
      "rewards/margins": 0.5057477355003357,
      "rewards/rejected": -1.6632630825042725,
      "step": 2260
    },
    {
      "epoch": 0.61,
      "grad_norm": 7.03125,
      "learning_rate": 2.0201223973828917e-06,
      "logits/chosen": -2.161062002182007,
      "logits/rejected": -2.0140938758850098,
      "logps/chosen": -304.7821044921875,
      "logps/rejected": -350.29693603515625,
      "loss": 0.6122,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.6994803547859192,
      "rewards/margins": 0.41134777665138245,
      "rewards/rejected": -1.1108280420303345,
      "step": 2270
    },
    {
      "epoch": 0.61,
      "grad_norm": 6.09375,
      "learning_rate": 1.997305197135089e-06,
      "logits/chosen": -2.135765314102173,
      "logits/rejected": -2.036240816116333,
      "logps/chosen": -317.1601257324219,
      "logps/rejected": -365.0966491699219,
      "loss": 0.5627,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.9625992774963379,
      "rewards/margins": 0.4367963671684265,
      "rewards/rejected": -1.3993955850601196,
      "step": 2280
    },
    {
      "epoch": 0.61,
      "grad_norm": 4.90625,
      "learning_rate": 1.9745315534350157e-06,
      "logits/chosen": -2.055593729019165,
      "logits/rejected": -1.8664264678955078,
      "logps/chosen": -413.187255859375,
      "logps/rejected": -418.80010986328125,
      "loss": 0.5571,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.8438934087753296,
      "rewards/margins": 0.3856704533100128,
      "rewards/rejected": -1.2295639514923096,
      "step": 2290
    },
    {
      "epoch": 0.61,
      "grad_norm": 7.4375,
      "learning_rate": 1.9518034395302413e-06,
      "logits/chosen": -1.695884346961975,
      "logits/rejected": -1.7858473062515259,
      "logps/chosen": -295.4271240234375,
      "logps/rejected": -332.1938171386719,
      "loss": 0.5853,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.853532612323761,
      "rewards/margins": 0.495563268661499,
      "rewards/rejected": -1.3490957021713257,
      "step": 2300
    },
    {
      "epoch": 0.61,
      "eval_logits/chosen": -2.0390243530273438,
      "eval_logits/rejected": -1.9439997673034668,
      "eval_logps/chosen": -319.3822326660156,
      "eval_logps/rejected": -352.41619873046875,
      "eval_loss": 0.5429208278656006,
      "eval_rewards/accuracies": 0.71875,
      "eval_rewards/chosen": -0.4611113667488098,
      "eval_rewards/margins": 0.4743398427963257,
      "eval_rewards/rejected": -0.9354512691497803,
      "eval_runtime": 880.2156,
      "eval_samples_per_second": 4.506,
      "eval_steps_per_second": 0.141,
      "step": 2300
    },
    {
      "epoch": 0.62,
      "grad_norm": 6.15625,
      "learning_rate": 1.9291228247233607e-06,
      "logits/chosen": -2.1423611640930176,
      "logits/rejected": -1.9027446508407593,
      "logps/chosen": -353.84783935546875,
      "logps/rejected": -413.4205627441406,
      "loss": 0.5701,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.8265654444694519,
      "rewards/margins": 0.5792354345321655,
      "rewards/rejected": -1.4058009386062622,
      "step": 2310
    },
    {
      "epoch": 0.62,
      "grad_norm": 7.3125,
      "learning_rate": 1.9064916742013515e-06,
      "logits/chosen": -1.888343095779419,
      "logits/rejected": -1.9230282306671143,
      "logps/chosen": -354.5987854003906,
      "logps/rejected": -382.3712158203125,
      "loss": 0.5984,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.0594052076339722,
      "rewards/margins": 0.5177406072616577,
      "rewards/rejected": -1.5771458148956299,
      "step": 2320
    },
    {
      "epoch": 0.62,
      "grad_norm": 6.625,
      "learning_rate": 1.883911948865306e-06,
      "logits/chosen": -1.977410078048706,
      "logits/rejected": -1.9233630895614624,
      "logps/chosen": -378.0736389160156,
      "logps/rejected": -405.97955322265625,
      "loss": 0.5536,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.9225144386291504,
      "rewards/margins": 0.5611852407455444,
      "rewards/rejected": -1.4836997985839844,
      "step": 2330
    },
    {
      "epoch": 0.62,
      "grad_norm": 6.46875,
      "learning_rate": 1.8613856051605242e-06,
      "logits/chosen": -1.9593966007232666,
      "logits/rejected": -1.6341909170150757,
      "logps/chosen": -402.64959716796875,
      "logps/rejected": -369.1689147949219,
      "loss": 0.6684,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -1.0764212608337402,
      "rewards/margins": 0.36168766021728516,
      "rewards/rejected": -1.4381088018417358,
      "step": 2340
    },
    {
      "epoch": 0.63,
      "grad_norm": 5.1875,
      "learning_rate": 1.8389145949069953e-06,
      "logits/chosen": -2.1931357383728027,
      "logits/rejected": -2.067920446395874,
      "logps/chosen": -416.1688537597656,
      "logps/rejected": -402.7095947265625,
      "loss": 0.5588,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.9342238306999207,
      "rewards/margins": 0.4599096179008484,
      "rewards/rejected": -1.3941335678100586,
      "step": 2350
    },
    {
      "epoch": 0.63,
      "grad_norm": 5.3125,
      "learning_rate": 1.816500865130279e-06,
      "logits/chosen": -2.1838107109069824,
      "logits/rejected": -1.8806880712509155,
      "logps/chosen": -337.64105224609375,
      "logps/rejected": -349.98907470703125,
      "loss": 0.5547,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.7486340999603271,
      "rewards/margins": 0.4604220986366272,
      "rewards/rejected": -1.2090561389923096,
      "step": 2360
    },
    {
      "epoch": 0.63,
      "grad_norm": 7.03125,
      "learning_rate": 1.7941463578928088e-06,
      "logits/chosen": -2.179802179336548,
      "logits/rejected": -1.8338829278945923,
      "logps/chosen": -369.4554138183594,
      "logps/rejected": -384.948486328125,
      "loss": 0.5693,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.082006573677063,
      "rewards/margins": 0.5788125395774841,
      "rewards/rejected": -1.6608192920684814,
      "step": 2370
    },
    {
      "epoch": 0.63,
      "grad_norm": 6.25,
      "learning_rate": 1.7718530101256115e-06,
      "logits/chosen": -1.9872877597808838,
      "logits/rejected": -1.8986995220184326,
      "logps/chosen": -341.47296142578125,
      "logps/rejected": -387.9000549316406,
      "loss": 0.5617,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.1249109506607056,
      "rewards/margins": 0.6380780339241028,
      "rewards/rejected": -1.762988805770874,
      "step": 2380
    },
    {
      "epoch": 0.64,
      "grad_norm": 6.5,
      "learning_rate": 1.7496227534604859e-06,
      "logits/chosen": -1.945773720741272,
      "logits/rejected": -1.8042278289794922,
      "logps/chosen": -374.2911682128906,
      "logps/rejected": -410.01239013671875,
      "loss": 0.5125,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.9492829442024231,
      "rewards/margins": 0.6416800618171692,
      "rewards/rejected": -1.5909628868103027,
      "step": 2390
    },
    {
      "epoch": 0.64,
      "grad_norm": 6.90625,
      "learning_rate": 1.7274575140626318e-06,
      "logits/chosen": -1.9381862878799438,
      "logits/rejected": -1.8155044317245483,
      "logps/chosen": -374.0718994140625,
      "logps/rejected": -409.1826171875,
      "loss": 0.5561,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.960218071937561,
      "rewards/margins": 0.7385403513908386,
      "rewards/rejected": -1.6987583637237549,
      "step": 2400
    },
    {
      "epoch": 0.64,
      "eval_logits/chosen": -2.0311460494995117,
      "eval_logits/rejected": -1.9358543157577515,
      "eval_logps/chosen": -323.672607421875,
      "eval_logps/rejected": -357.0744323730469,
      "eval_loss": 0.5407421588897705,
      "eval_rewards/accuracies": 0.7207661271095276,
      "eval_rewards/chosen": -0.5040145516395569,
      "eval_rewards/margins": 0.4780188202857971,
      "eval_rewards/rejected": -0.982033371925354,
      "eval_runtime": 881.9649,
      "eval_samples_per_second": 4.497,
      "eval_steps_per_second": 0.141,
      "step": 2400
    },
    {
      "epoch": 0.64,
      "grad_norm": 6.40625,
      "learning_rate": 1.7053592124637557e-06,
      "logits/chosen": -2.1460671424865723,
      "logits/rejected": -1.814182996749878,
      "logps/chosen": -405.0924987792969,
      "logps/rejected": -421.554931640625,
      "loss": 0.5492,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.7955840229988098,
      "rewards/margins": 0.7058395743370056,
      "rewards/rejected": -1.5014235973358154,
      "step": 2410
    },
    {
      "epoch": 0.65,
      "grad_norm": 7.25,
      "learning_rate": 1.6833297633956647e-06,
      "logits/chosen": -2.0674893856048584,
      "logits/rejected": -1.87158203125,
      "logps/chosen": -295.043701171875,
      "logps/rejected": -350.60137939453125,
      "loss": 0.5389,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.803569495677948,
      "rewards/margins": 0.650601863861084,
      "rewards/rejected": -1.4541715383529663,
      "step": 2420
    },
    {
      "epoch": 0.65,
      "grad_norm": 7.375,
      "learning_rate": 1.661371075624363e-06,
      "logits/chosen": -2.2229762077331543,
      "logits/rejected": -1.7906564474105835,
      "logps/chosen": -371.20404052734375,
      "logps/rejected": -397.4686584472656,
      "loss": 0.5338,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.8283071517944336,
      "rewards/margins": 0.9246468544006348,
      "rewards/rejected": -1.7529537677764893,
      "step": 2430
    },
    {
      "epoch": 0.65,
      "grad_norm": 3.71875,
      "learning_rate": 1.6394850517846621e-06,
      "logits/chosen": -2.0950801372528076,
      "logits/rejected": -1.6051075458526611,
      "logps/chosen": -344.3671569824219,
      "logps/rejected": -334.49395751953125,
      "loss": 0.5531,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.6905900239944458,
      "rewards/margins": 0.5956054925918579,
      "rewards/rejected": -1.2861955165863037,
      "step": 2440
    },
    {
      "epoch": 0.65,
      "grad_norm": 7.15625,
      "learning_rate": 1.6176735882153284e-06,
      "logits/chosen": -2.0940444469451904,
      "logits/rejected": -1.8229434490203857,
      "logps/chosen": -356.28680419921875,
      "logps/rejected": -411.14422607421875,
      "loss": 0.5637,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.8110325932502747,
      "rewards/margins": 0.7523552179336548,
      "rewards/rejected": -1.5633877515792847,
      "step": 2450
    },
    {
      "epoch": 0.66,
      "grad_norm": 5.78125,
      "learning_rate": 1.5959385747947697e-06,
      "logits/chosen": -2.1104280948638916,
      "logits/rejected": -1.7870792150497437,
      "logps/chosen": -315.52471923828125,
      "logps/rejected": -358.6795654296875,
      "loss": 0.5774,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.8010858297348022,
      "rewards/margins": 0.5728421807289124,
      "rewards/rejected": -1.3739280700683594,
      "step": 2460
    },
    {
      "epoch": 0.66,
      "grad_norm": 6.625,
      "learning_rate": 1.5742818947772875e-06,
      "logits/chosen": -2.0460052490234375,
      "logits/rejected": -1.507616400718689,
      "logps/chosen": -317.50054931640625,
      "logps/rejected": -274.7225341796875,
      "loss": 0.6029,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.7521132230758667,
      "rewards/margins": 0.3076709806919098,
      "rewards/rejected": -1.059784173965454,
      "step": 2470
    },
    {
      "epoch": 0.66,
      "grad_norm": 9.8125,
      "learning_rate": 1.552705424629898e-06,
      "logits/chosen": -1.82955801486969,
      "logits/rejected": -1.9698879718780518,
      "logps/chosen": -356.66802978515625,
      "logps/rejected": -416.3487854003906,
      "loss": 0.665,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -1.0128486156463623,
      "rewards/margins": 0.31570059061050415,
      "rewards/rejected": -1.3285492658615112,
      "step": 2480
    },
    {
      "epoch": 0.66,
      "grad_norm": 5.59375,
      "learning_rate": 1.5312110338697427e-06,
      "logits/chosen": -2.0218749046325684,
      "logits/rejected": -1.773508071899414,
      "logps/chosen": -363.67596435546875,
      "logps/rejected": -332.68243408203125,
      "loss": 0.5886,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.9451906085014343,
      "rewards/margins": 0.3801174759864807,
      "rewards/rejected": -1.3253079652786255,
      "step": 2490
    },
    {
      "epoch": 0.67,
      "grad_norm": 7.875,
      "learning_rate": 1.509800584902108e-06,
      "logits/chosen": -2.123840808868408,
      "logits/rejected": -1.7182581424713135,
      "logps/chosen": -299.72259521484375,
      "logps/rejected": -352.78033447265625,
      "loss": 0.559,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5781232118606567,
      "rewards/margins": 0.7339556813240051,
      "rewards/rejected": -1.312078833580017,
      "step": 2500
    },
    {
      "epoch": 0.67,
      "eval_logits/chosen": -2.032421827316284,
      "eval_logits/rejected": -1.9376754760742188,
      "eval_logps/chosen": -314.38055419921875,
      "eval_logps/rejected": -344.40325927734375,
      "eval_loss": 0.5507678389549255,
      "eval_rewards/accuracies": 0.7207661271095276,
      "eval_rewards/chosen": -0.411094069480896,
      "eval_rewards/margins": 0.44422775506973267,
      "eval_rewards/rejected": -0.8553218841552734,
      "eval_runtime": 880.859,
      "eval_samples_per_second": 4.502,
      "eval_steps_per_second": 0.141,
      "step": 2500
    },
    {
      "epoch": 0.67,
      "grad_norm": 4.9375,
      "learning_rate": 1.4884759328590476e-06,
      "logits/chosen": -2.3335301876068115,
      "logits/rejected": -1.7992775440216064,
      "logps/chosen": -334.680908203125,
      "logps/rejected": -433.47271728515625,
      "loss": 0.5485,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.643280565738678,
      "rewards/margins": 0.807063102722168,
      "rewards/rejected": -1.450343370437622,
      "step": 2510
    },
    {
      "epoch": 0.67,
      "grad_norm": 6.9375,
      "learning_rate": 1.467238925438646e-06,
      "logits/chosen": -1.95244562625885,
      "logits/rejected": -1.7415659427642822,
      "logps/chosen": -324.41424560546875,
      "logps/rejected": -340.30364990234375,
      "loss": 0.5775,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.725772500038147,
      "rewards/margins": 0.5707597732543945,
      "rewards/rejected": -1.296532392501831,
      "step": 2520
    },
    {
      "epoch": 0.67,
      "grad_norm": 3.78125,
      "learning_rate": 1.446091402744923e-06,
      "logits/chosen": -2.050171375274658,
      "logits/rejected": -1.579511284828186,
      "logps/chosen": -316.1702575683594,
      "logps/rejected": -340.01788330078125,
      "loss": 0.5502,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.6074039936065674,
      "rewards/margins": 0.7356580495834351,
      "rewards/rejected": -1.343062162399292,
      "step": 2530
    },
    {
      "epoch": 0.68,
      "grad_norm": 5.96875,
      "learning_rate": 1.4250351971283937e-06,
      "logits/chosen": -2.0692222118377686,
      "logits/rejected": -1.608991265296936,
      "logps/chosen": -356.05853271484375,
      "logps/rejected": -394.2357482910156,
      "loss": 0.561,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.9106645584106445,
      "rewards/margins": 0.7255198955535889,
      "rewards/rejected": -1.6361844539642334,
      "step": 2540
    },
    {
      "epoch": 0.68,
      "grad_norm": 3.875,
      "learning_rate": 1.4040721330273063e-06,
      "logits/chosen": -2.1361420154571533,
      "logits/rejected": -1.9923378229141235,
      "logps/chosen": -332.5837097167969,
      "logps/rejected": -327.5522155761719,
      "loss": 0.5763,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.5888851881027222,
      "rewards/margins": 0.4220341145992279,
      "rewards/rejected": -1.0109193325042725,
      "step": 2550
    },
    {
      "epoch": 0.68,
      "grad_norm": 5.75,
      "learning_rate": 1.3832040268095589e-06,
      "logits/chosen": -2.148440361022949,
      "logits/rejected": -1.9477498531341553,
      "logps/chosen": -310.78131103515625,
      "logps/rejected": -328.2440185546875,
      "loss": 0.5542,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.6367581486701965,
      "rewards/margins": 0.41524234414100647,
      "rewards/rejected": -1.0520005226135254,
      "step": 2560
    },
    {
      "epoch": 0.69,
      "grad_norm": 6.03125,
      "learning_rate": 1.362432686615316e-06,
      "logits/chosen": -1.9776138067245483,
      "logits/rejected": -1.7947371006011963,
      "logps/chosen": -333.21417236328125,
      "logps/rejected": -374.3428955078125,
      "loss": 0.603,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.8226397633552551,
      "rewards/margins": 0.6269424557685852,
      "rewards/rejected": -1.4495822191238403,
      "step": 2570
    },
    {
      "epoch": 0.69,
      "grad_norm": 5.0,
      "learning_rate": 1.3417599122003464e-06,
      "logits/chosen": -2.128110885620117,
      "logits/rejected": -2.094461679458618,
      "logps/chosen": -338.19720458984375,
      "logps/rejected": -443.8834533691406,
      "loss": 0.5581,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.8850421905517578,
      "rewards/margins": 0.7484723925590515,
      "rewards/rejected": -1.633514642715454,
      "step": 2580
    },
    {
      "epoch": 0.69,
      "grad_norm": 4.53125,
      "learning_rate": 1.3211874947800747e-06,
      "logits/chosen": -1.9258081912994385,
      "logits/rejected": -1.8620773553848267,
      "logps/chosen": -378.318603515625,
      "logps/rejected": -416.9369201660156,
      "loss": 0.5436,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.7853119969367981,
      "rewards/margins": 0.6527517437934875,
      "rewards/rejected": -1.4380637407302856,
      "step": 2590
    },
    {
      "epoch": 0.69,
      "grad_norm": 5.46875,
      "learning_rate": 1.3007172168743854e-06,
      "logits/chosen": -1.9622900485992432,
      "logits/rejected": -1.9040113687515259,
      "logps/chosen": -310.6736145019531,
      "logps/rejected": -319.5574645996094,
      "loss": 0.5803,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.7002264857292175,
      "rewards/margins": 0.5420228242874146,
      "rewards/rejected": -1.2422492504119873,
      "step": 2600
    },
    {
      "epoch": 0.69,
      "eval_logits/chosen": -2.0208189487457275,
      "eval_logits/rejected": -1.9263473749160767,
      "eval_logps/chosen": -314.4533386230469,
      "eval_logps/rejected": -344.0258483886719,
      "eval_loss": 0.5507248640060425,
      "eval_rewards/accuracies": 0.7217742204666138,
      "eval_rewards/chosen": -0.41182228922843933,
      "eval_rewards/margins": 0.4397256076335907,
      "eval_rewards/rejected": -0.8515478372573853,
      "eval_runtime": 880.8587,
      "eval_samples_per_second": 4.502,
      "eval_steps_per_second": 0.141,
      "step": 2600
    },
    {
      "epoch": 0.7,
      "grad_norm": 6.96875,
      "learning_rate": 1.280350852153168e-06,
      "logits/chosen": -2.003282070159912,
      "logits/rejected": -1.8346112966537476,
      "logps/chosen": -365.9354248046875,
      "logps/rejected": -337.50140380859375,
      "loss": 0.5964,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.7410377264022827,
      "rewards/margins": 0.3177192211151123,
      "rewards/rejected": -1.0587570667266846,
      "step": 2610
    },
    {
      "epoch": 0.7,
      "grad_norm": 6.59375,
      "learning_rate": 1.260090165282645e-06,
      "logits/chosen": -1.8802896738052368,
      "logits/rejected": -1.8831758499145508,
      "logps/chosen": -274.5264892578125,
      "logps/rejected": -355.3487854003906,
      "loss": 0.5872,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7475273013114929,
      "rewards/margins": 0.6068841218948364,
      "rewards/rejected": -1.3544113636016846,
      "step": 2620
    },
    {
      "epoch": 0.7,
      "grad_norm": 5.78125,
      "learning_rate": 1.2399369117724582e-06,
      "logits/chosen": -2.0739991664886475,
      "logits/rejected": -1.746421456336975,
      "logps/chosen": -301.8329162597656,
      "logps/rejected": -340.41436767578125,
      "loss": 0.5771,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.5907261967658997,
      "rewards/margins": 0.6564832925796509,
      "rewards/rejected": -1.2472095489501953,
      "step": 2630
    },
    {
      "epoch": 0.7,
      "grad_norm": 8.25,
      "learning_rate": 1.2198928378235717e-06,
      "logits/chosen": -2.178474187850952,
      "logits/rejected": -1.911022424697876,
      "logps/chosen": -316.3603820800781,
      "logps/rejected": -361.1997985839844,
      "loss": 0.5713,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.6979625821113586,
      "rewards/margins": 0.553970456123352,
      "rewards/rejected": -1.251933217048645,
      "step": 2640
    },
    {
      "epoch": 0.71,
      "grad_norm": 9.25,
      "learning_rate": 1.1999596801769617e-06,
      "logits/chosen": -2.040790557861328,
      "logits/rejected": -1.759600043296814,
      "logps/chosen": -336.14666748046875,
      "logps/rejected": -352.65142822265625,
      "loss": 0.5913,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.9996898770332336,
      "rewards/margins": 0.4132237434387207,
      "rewards/rejected": -1.4129135608673096,
      "step": 2650
    },
    {
      "epoch": 0.71,
      "grad_norm": 4.4375,
      "learning_rate": 1.1801391659631423e-06,
      "logits/chosen": -1.9337103366851807,
      "logits/rejected": -1.951136589050293,
      "logps/chosen": -278.21124267578125,
      "logps/rejected": -290.7889709472656,
      "loss": 0.5512,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.7282769680023193,
      "rewards/margins": 0.40413203835487366,
      "rewards/rejected": -1.1324089765548706,
      "step": 2660
    },
    {
      "epoch": 0.71,
      "grad_norm": 4.0,
      "learning_rate": 1.160433012552508e-06,
      "logits/chosen": -1.9449344873428345,
      "logits/rejected": -1.6623780727386475,
      "logps/chosen": -382.18682861328125,
      "logps/rejected": -357.8780517578125,
      "loss": 0.5579,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.593928337097168,
      "rewards/margins": 0.6014456748962402,
      "rewards/rejected": -1.1953741312026978,
      "step": 2670
    },
    {
      "epoch": 0.71,
      "grad_norm": 4.65625,
      "learning_rate": 1.1408429274065418e-06,
      "logits/chosen": -2.191960096359253,
      "logits/rejected": -1.7279794216156006,
      "logps/chosen": -324.99261474609375,
      "logps/rejected": -296.83038330078125,
      "loss": 0.6001,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8059870004653931,
      "rewards/margins": 0.3716198801994324,
      "rewards/rejected": -1.1776069402694702,
      "step": 2680
    },
    {
      "epoch": 0.72,
      "grad_norm": 4.90625,
      "learning_rate": 1.1213706079298566e-06,
      "logits/chosen": -2.028003215789795,
      "logits/rejected": -1.7843049764633179,
      "logps/chosen": -346.9183654785156,
      "logps/rejected": -417.69512939453125,
      "loss": 0.532,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.8280528783798218,
      "rewards/margins": 0.6686497330665588,
      "rewards/rejected": -1.4967026710510254,
      "step": 2690
    },
    {
      "epoch": 0.72,
      "grad_norm": 5.40625,
      "learning_rate": 1.1020177413231334e-06,
      "logits/chosen": -1.8682610988616943,
      "logits/rejected": -2.021223545074463,
      "logps/chosen": -335.21337890625,
      "logps/rejected": -370.74151611328125,
      "loss": 0.5537,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.6807470321655273,
      "rewards/margins": 0.38916581869125366,
      "rewards/rejected": -1.0699129104614258,
      "step": 2700
    },
    {
      "epoch": 0.72,
      "eval_logits/chosen": -2.006366491317749,
      "eval_logits/rejected": -1.9120668172836304,
      "eval_logps/chosen": -321.93084716796875,
      "eval_logps/rejected": -353.4754943847656,
      "eval_loss": 0.545336902141571,
      "eval_rewards/accuracies": 0.71875,
      "eval_rewards/chosen": -0.4865972399711609,
      "eval_rewards/margins": 0.45944684743881226,
      "eval_rewards/rejected": -0.9460442066192627,
      "eval_runtime": 880.9785,
      "eval_samples_per_second": 4.502,
      "eval_steps_per_second": 0.141,
      "step": 2700
    },
    {
      "epoch": 0.72,
      "grad_norm": 6.71875,
      "learning_rate": 1.0827860044369226e-06,
      "logits/chosen": -2.0423388481140137,
      "logits/rejected": -1.3998781442642212,
      "logps/chosen": -343.3073425292969,
      "logps/rejected": -302.65399169921875,
      "loss": 0.6265,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.8548204302787781,
      "rewards/margins": 0.4668661952018738,
      "rewards/rejected": -1.3216865062713623,
      "step": 2710
    },
    {
      "epoch": 0.73,
      "grad_norm": 5.9375,
      "learning_rate": 1.06367706362636e-06,
      "logits/chosen": -2.0432324409484863,
      "logits/rejected": -1.8480165004730225,
      "logps/chosen": -319.4305419921875,
      "logps/rejected": -356.7183532714844,
      "loss": 0.6224,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.7791837453842163,
      "rewards/margins": 0.46399766206741333,
      "rewards/rejected": -1.2431814670562744,
      "step": 2720
    },
    {
      "epoch": 0.73,
      "grad_norm": 4.8125,
      "learning_rate": 1.0446925746067768e-06,
      "logits/chosen": -2.0405526161193848,
      "logits/rejected": -1.7184631824493408,
      "logps/chosen": -395.4254455566406,
      "logps/rejected": -371.5160217285156,
      "loss": 0.4921,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.9443087577819824,
      "rewards/margins": 0.5833538770675659,
      "rewards/rejected": -1.5276625156402588,
      "step": 2730
    },
    {
      "epoch": 0.73,
      "grad_norm": 11.75,
      "learning_rate": 1.0258341823102418e-06,
      "logits/chosen": -2.1357948780059814,
      "logits/rejected": -1.821129560470581,
      "logps/chosen": -394.2712707519531,
      "logps/rejected": -404.80328369140625,
      "loss": 0.6014,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.8151429891586304,
      "rewards/margins": 0.515398383140564,
      "rewards/rejected": -1.3305416107177734,
      "step": 2740
    },
    {
      "epoch": 0.73,
      "grad_norm": 5.46875,
      "learning_rate": 1.0071035207430352e-06,
      "logits/chosen": -1.9650729894638062,
      "logits/rejected": -1.8225491046905518,
      "logps/chosen": -335.25982666015625,
      "logps/rejected": -367.09075927734375,
      "loss": 0.5304,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.8919695019721985,
      "rewards/margins": 0.5095754265785217,
      "rewards/rejected": -1.4015448093414307,
      "step": 2750
    },
    {
      "epoch": 0.74,
      "grad_norm": 6.28125,
      "learning_rate": 9.88502212844063e-07,
      "logits/chosen": -1.8136589527130127,
      "logits/rejected": -1.7463903427124023,
      "logps/chosen": -314.6876220703125,
      "logps/rejected": -354.2144470214844,
      "loss": 0.5829,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.8098533749580383,
      "rewards/margins": 0.6118708848953247,
      "rewards/rejected": -1.4217243194580078,
      "step": 2760
    },
    {
      "epoch": 0.74,
      "grad_norm": 5.28125,
      "learning_rate": 9.700318703442437e-07,
      "logits/chosen": -2.1762218475341797,
      "logits/rejected": -1.9241282939910889,
      "logps/chosen": -359.361328125,
      "logps/rejected": -355.7702331542969,
      "loss": 0.5481,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.8270170092582703,
      "rewards/margins": 0.5131338238716125,
      "rewards/rejected": -1.3401508331298828,
      "step": 2770
    },
    {
      "epoch": 0.74,
      "grad_norm": 9.125,
      "learning_rate": 9.516940936268504e-07,
      "logits/chosen": -1.8696463108062744,
      "logits/rejected": -1.8486192226409912,
      "logps/chosen": -306.63934326171875,
      "logps/rejected": -404.9595031738281,
      "loss": 0.5426,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.820268988609314,
      "rewards/margins": 0.5593957304954529,
      "rewards/rejected": -1.379664659500122,
      "step": 2780
    },
    {
      "epoch": 0.74,
      "grad_norm": 6.375,
      "learning_rate": 9.334904715888496e-07,
      "logits/chosen": -1.71816086769104,
      "logits/rejected": -1.5642400979995728,
      "logps/chosen": -372.04095458984375,
      "logps/rejected": -375.2909851074219,
      "loss": 0.5802,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.9492367506027222,
      "rewards/margins": 0.36183589696884155,
      "rewards/rejected": -1.3110727071762085,
      "step": 2790
    },
    {
      "epoch": 0.75,
      "grad_norm": 5.6875,
      "learning_rate": 9.154225815032242e-07,
      "logits/chosen": -1.9324607849121094,
      "logits/rejected": -1.7516405582427979,
      "logps/chosen": -307.28326416015625,
      "logps/rejected": -407.0020446777344,
      "loss": 0.5562,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.7537766695022583,
      "rewards/margins": 0.6861321926116943,
      "rewards/rejected": -1.4399089813232422,
      "step": 2800
    },
    {
      "epoch": 0.75,
      "eval_logits/chosen": -2.003047466278076,
      "eval_logits/rejected": -1.9088319540023804,
      "eval_logps/chosen": -323.2741394042969,
      "eval_logps/rejected": -355.1509094238281,
      "eval_loss": 0.5441163778305054,
      "eval_rewards/accuracies": 0.7227822542190552,
      "eval_rewards/chosen": -0.5000301599502563,
      "eval_rewards/margins": 0.462767630815506,
      "eval_rewards/rejected": -0.9627977013587952,
      "eval_runtime": 879.793,
      "eval_samples_per_second": 4.508,
      "eval_steps_per_second": 0.141,
      "step": 2800
    },
    {
      "epoch": 0.75,
      "grad_norm": 3.53125,
      "learning_rate": 8.974919888823164e-07,
      "logits/chosen": -1.9959185123443604,
      "logits/rejected": -1.8022918701171875,
      "logps/chosen": -362.20892333984375,
      "logps/rejected": -393.1611633300781,
      "loss": 0.5644,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.8053617477416992,
      "rewards/margins": 0.6159976720809937,
      "rewards/rejected": -1.4213593006134033,
      "step": 2810
    },
    {
      "epoch": 0.75,
      "grad_norm": 4.5625,
      "learning_rate": 8.797002473421729e-07,
      "logits/chosen": -1.8552814722061157,
      "logits/rejected": -1.783198356628418,
      "logps/chosen": -321.0819396972656,
      "logps/rejected": -440.4762268066406,
      "loss": 0.5266,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.747664749622345,
      "rewards/margins": 0.8036431074142456,
      "rewards/rejected": -1.5513079166412354,
      "step": 2820
    },
    {
      "epoch": 0.75,
      "grad_norm": 18.75,
      "learning_rate": 8.620488984679378e-07,
      "logits/chosen": -2.0200352668762207,
      "logits/rejected": -1.703529715538025,
      "logps/chosen": -366.3807373046875,
      "logps/rejected": -435.84002685546875,
      "loss": 0.583,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.068791389465332,
      "rewards/margins": 0.6867550611495972,
      "rewards/rejected": -1.7555465698242188,
      "step": 2830
    },
    {
      "epoch": 0.76,
      "grad_norm": 6.5,
      "learning_rate": 8.445394716802754e-07,
      "logits/chosen": -1.7996158599853516,
      "logits/rejected": -1.7732694149017334,
      "logps/chosen": -359.3457336425781,
      "logps/rejected": -475.75830078125,
      "loss": 0.5569,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.9535554647445679,
      "rewards/margins": 0.8302940130233765,
      "rewards/rejected": -1.7838493585586548,
      "step": 2840
    },
    {
      "epoch": 0.76,
      "grad_norm": 6.59375,
      "learning_rate": 8.271734841028553e-07,
      "logits/chosen": -1.8932832479476929,
      "logits/rejected": -1.743072509765625,
      "logps/chosen": -335.93572998046875,
      "logps/rejected": -382.79730224609375,
      "loss": 0.5733,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.9539499282836914,
      "rewards/margins": 0.31490129232406616,
      "rewards/rejected": -1.2688511610031128,
      "step": 2850
    },
    {
      "epoch": 0.76,
      "grad_norm": 4.90625,
      "learning_rate": 8.099524404308948e-07,
      "logits/chosen": -1.9470617771148682,
      "logits/rejected": -1.5044618844985962,
      "logps/chosen": -392.771484375,
      "logps/rejected": -341.753173828125,
      "loss": 0.6128,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.0686981678009033,
      "rewards/margins": 0.48957228660583496,
      "rewards/rejected": -1.5582703351974487,
      "step": 2860
    },
    {
      "epoch": 0.77,
      "grad_norm": 7.15625,
      "learning_rate": 7.928778328007918e-07,
      "logits/chosen": -2.0461151599884033,
      "logits/rejected": -1.8082244396209717,
      "logps/chosen": -366.478271484375,
      "logps/rejected": -396.737060546875,
      "loss": 0.6095,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.9852982759475708,
      "rewards/margins": 0.4492277204990387,
      "rewards/rejected": -1.4345258474349976,
      "step": 2870
    },
    {
      "epoch": 0.77,
      "grad_norm": 8.875,
      "learning_rate": 7.759511406608255e-07,
      "logits/chosen": -1.873835563659668,
      "logits/rejected": -1.619154691696167,
      "logps/chosen": -403.31121826171875,
      "logps/rejected": -365.35736083984375,
      "loss": 0.5927,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -1.1122580766677856,
      "rewards/margins": 0.24037642776966095,
      "rewards/rejected": -1.352634310722351,
      "step": 2880
    },
    {
      "epoch": 0.77,
      "grad_norm": 5.59375,
      "learning_rate": 7.591738306429769e-07,
      "logits/chosen": -1.839006781578064,
      "logits/rejected": -1.8650472164154053,
      "logps/chosen": -365.93121337890625,
      "logps/rejected": -378.2662658691406,
      "loss": 0.551,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.8217144012451172,
      "rewards/margins": 0.37123245000839233,
      "rewards/rejected": -1.1929466724395752,
      "step": 2890
    },
    {
      "epoch": 0.77,
      "grad_norm": 4.28125,
      "learning_rate": 7.425473564358457e-07,
      "logits/chosen": -2.073753833770752,
      "logits/rejected": -1.8868709802627563,
      "logps/chosen": -375.6540832519531,
      "logps/rejected": -394.7572326660156,
      "loss": 0.5553,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.825318455696106,
      "rewards/margins": 0.730427622795105,
      "rewards/rejected": -1.5557458400726318,
      "step": 2900
    },
    {
      "epoch": 0.77,
      "eval_logits/chosen": -2.0016636848449707,
      "eval_logits/rejected": -1.907488226890564,
      "eval_logps/chosen": -322.8616638183594,
      "eval_logps/rejected": -354.9320373535156,
      "eval_loss": 0.544157862663269,
      "eval_rewards/accuracies": 0.71875,
      "eval_rewards/chosen": -0.4959055781364441,
      "eval_rewards/margins": 0.4647037982940674,
      "eval_rewards/rejected": -0.9606093168258667,
      "eval_runtime": 880.64,
      "eval_samples_per_second": 4.504,
      "eval_steps_per_second": 0.141,
      "step": 2900
    },
    {
      "epoch": 0.78,
      "grad_norm": 5.375,
      "learning_rate": 7.260731586586983e-07,
      "logits/chosen": -2.1968705654144287,
      "logits/rejected": -2.016429901123047,
      "logps/chosen": -333.78033447265625,
      "logps/rejected": -379.2249450683594,
      "loss": 0.5601,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.8710298538208008,
      "rewards/margins": 0.6578187346458435,
      "rewards/rejected": -1.528848648071289,
      "step": 2910
    },
    {
      "epoch": 0.78,
      "grad_norm": 5.5,
      "learning_rate": 7.097526647366379e-07,
      "logits/chosen": -2.0344557762145996,
      "logits/rejected": -1.9121065139770508,
      "logps/chosen": -401.88201904296875,
      "logps/rejected": -423.17669677734375,
      "loss": 0.5597,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.1633549928665161,
      "rewards/margins": 0.4100661873817444,
      "rewards/rejected": -1.5734212398529053,
      "step": 2920
    },
    {
      "epoch": 0.78,
      "grad_norm": 5.875,
      "learning_rate": 6.935872887769299e-07,
      "logits/chosen": -2.0147557258605957,
      "logits/rejected": -1.6825544834136963,
      "logps/chosen": -365.94488525390625,
      "logps/rejected": -402.7860412597656,
      "loss": 0.5441,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.895987868309021,
      "rewards/margins": 0.6669803857803345,
      "rewards/rejected": -1.5629682540893555,
      "step": 2930
    },
    {
      "epoch": 0.78,
      "grad_norm": 4.59375,
      "learning_rate": 6.775784314464717e-07,
      "logits/chosen": -2.080397844314575,
      "logits/rejected": -2.0552940368652344,
      "logps/chosen": -340.08782958984375,
      "logps/rejected": -379.7850036621094,
      "loss": 0.5801,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8713234066963196,
      "rewards/margins": 0.6259976625442505,
      "rewards/rejected": -1.4973211288452148,
      "step": 2940
    },
    {
      "epoch": 0.79,
      "grad_norm": 4.40625,
      "learning_rate": 6.617274798504286e-07,
      "logits/chosen": -1.9516241550445557,
      "logits/rejected": -1.8688323497772217,
      "logps/chosen": -323.1833190917969,
      "logps/rejected": -350.79571533203125,
      "loss": 0.5939,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.0701980590820312,
      "rewards/margins": 0.2663252353668213,
      "rewards/rejected": -1.3365232944488525,
      "step": 2950
    },
    {
      "epoch": 0.79,
      "grad_norm": 5.8125,
      "learning_rate": 6.460358074120518e-07,
      "logits/chosen": -1.977295160293579,
      "logits/rejected": -1.7293208837509155,
      "logps/chosen": -307.32391357421875,
      "logps/rejected": -314.4563903808594,
      "loss": 0.5673,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.8124295473098755,
      "rewards/margins": 0.4973699450492859,
      "rewards/rejected": -1.3097994327545166,
      "step": 2960
    },
    {
      "epoch": 0.79,
      "grad_norm": 4.875,
      "learning_rate": 6.305047737536707e-07,
      "logits/chosen": -1.8789873123168945,
      "logits/rejected": -2.0024304389953613,
      "logps/chosen": -331.21649169921875,
      "logps/rejected": -436.26690673828125,
      "loss": 0.5323,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.9506662487983704,
      "rewards/margins": 0.4117092490196228,
      "rewards/rejected": -1.3623754978179932,
      "step": 2970
    },
    {
      "epoch": 0.79,
      "grad_norm": 4.0,
      "learning_rate": 6.151357245788917e-07,
      "logits/chosen": -2.1022632122039795,
      "logits/rejected": -1.7151778936386108,
      "logps/chosen": -354.33843994140625,
      "logps/rejected": -356.8936767578125,
      "loss": 0.512,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.8215311169624329,
      "rewards/margins": 0.7401578426361084,
      "rewards/rejected": -1.5616891384124756,
      "step": 2980
    },
    {
      "epoch": 0.8,
      "grad_norm": 6.1875,
      "learning_rate": 5.999299915559956e-07,
      "logits/chosen": -2.008312702178955,
      "logits/rejected": -1.8347675800323486,
      "logps/chosen": -319.0375061035156,
      "logps/rejected": -345.8760681152344,
      "loss": 0.5904,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7591317892074585,
      "rewards/margins": 0.6678034067153931,
      "rewards/rejected": -1.4269349575042725,
      "step": 2990
    },
    {
      "epoch": 0.8,
      "grad_norm": 3.734375,
      "learning_rate": 5.848888922025553e-07,
      "logits/chosen": -2.0952308177948,
      "logits/rejected": -1.7973310947418213,
      "logps/chosen": -372.03021240234375,
      "logps/rejected": -411.7416076660156,
      "loss": 0.59,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.751909077167511,
      "rewards/margins": 0.5903130769729614,
      "rewards/rejected": -1.3422220945358276,
      "step": 3000
    },
    {
      "epoch": 0.8,
      "eval_logits/chosen": -2.003185749053955,
      "eval_logits/rejected": -1.9088407754898071,
      "eval_logps/chosen": -324.0159606933594,
      "eval_logps/rejected": -357.10906982421875,
      "eval_loss": 0.5430915355682373,
      "eval_rewards/accuracies": 0.71875,
      "eval_rewards/chosen": -0.5074483156204224,
      "eval_rewards/margins": 0.474931538105011,
      "eval_rewards/rejected": -0.9823799133300781,
      "eval_runtime": 879.4847,
      "eval_samples_per_second": 4.509,
      "eval_steps_per_second": 0.141,
      "step": 3000
    },
    {
      "epoch": 0.8,
      "grad_norm": 6.6875,
      "learning_rate": 5.700137297712749e-07,
      "logits/chosen": -2.0644991397857666,
      "logits/rejected": -1.7454503774642944,
      "logps/chosen": -363.914306640625,
      "logps/rejected": -378.5439758300781,
      "loss": 0.5487,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7263001799583435,
      "rewards/margins": 0.8286347389221191,
      "rewards/rejected": -1.5549349784851074,
      "step": 3010
    },
    {
      "epoch": 0.81,
      "grad_norm": 4.96875,
      "learning_rate": 5.553057931370729e-07,
      "logits/chosen": -1.7230809926986694,
      "logits/rejected": -1.8183281421661377,
      "logps/chosen": -404.5675048828125,
      "logps/rejected": -480.0821838378906,
      "loss": 0.6216,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.059062123298645,
      "rewards/margins": 0.4681197702884674,
      "rewards/rejected": -1.5271819829940796,
      "step": 3020
    },
    {
      "epoch": 0.81,
      "grad_norm": 8.75,
      "learning_rate": 5.407663566854008e-07,
      "logits/chosen": -2.1350698471069336,
      "logits/rejected": -1.7468225955963135,
      "logps/chosen": -364.47015380859375,
      "logps/rejected": -393.1088562011719,
      "loss": 0.5818,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.7110105752944946,
      "rewards/margins": 0.7123533487319946,
      "rewards/rejected": -1.4233639240264893,
      "step": 3030
    },
    {
      "epoch": 0.81,
      "grad_norm": 6.28125,
      "learning_rate": 5.263966802018275e-07,
      "logits/chosen": -2.0588219165802,
      "logits/rejected": -2.018083333969116,
      "logps/chosen": -341.75445556640625,
      "logps/rejected": -357.783447265625,
      "loss": 0.6529,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.8541601896286011,
      "rewards/margins": 0.2489049881696701,
      "rewards/rejected": -1.1030651330947876,
      "step": 3040
    },
    {
      "epoch": 0.81,
      "grad_norm": 6.59375,
      "learning_rate": 5.121980087628802e-07,
      "logits/chosen": -1.9442148208618164,
      "logits/rejected": -1.6954371929168701,
      "logps/chosen": -325.7296447753906,
      "logps/rejected": -366.55767822265625,
      "loss": 0.568,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9034749865531921,
      "rewards/margins": 0.6034583449363708,
      "rewards/rejected": -1.5069334506988525,
      "step": 3050
    },
    {
      "epoch": 0.82,
      "grad_norm": 7.09375,
      "learning_rate": 4.981715726281666e-07,
      "logits/chosen": -1.9503653049468994,
      "logits/rejected": -1.7234443426132202,
      "logps/chosen": -332.2471008300781,
      "logps/rejected": -402.9471740722656,
      "loss": 0.5368,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.7525867223739624,
      "rewards/margins": 0.6139643788337708,
      "rewards/rejected": -1.366551160812378,
      "step": 3060
    },
    {
      "epoch": 0.82,
      "grad_norm": 4.875,
      "learning_rate": 4.843185871337722e-07,
      "logits/chosen": -1.9571077823638916,
      "logits/rejected": -1.8662850856781006,
      "logps/chosen": -373.59967041015625,
      "logps/rejected": -409.1192321777344,
      "loss": 0.5662,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.8689903020858765,
      "rewards/margins": 0.4505646228790283,
      "rewards/rejected": -1.3195549249649048,
      "step": 3070
    },
    {
      "epoch": 0.82,
      "grad_norm": 3.515625,
      "learning_rate": 4.706402525869633e-07,
      "logits/chosen": -2.1176676750183105,
      "logits/rejected": -1.8747915029525757,
      "logps/chosen": -335.0406188964844,
      "logps/rejected": -370.3428039550781,
      "loss": 0.5293,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.797022819519043,
      "rewards/margins": 0.7101470232009888,
      "rewards/rejected": -1.5071698427200317,
      "step": 3080
    },
    {
      "epoch": 0.82,
      "grad_norm": 6.03125,
      "learning_rate": 4.5713775416217884e-07,
      "logits/chosen": -2.031067132949829,
      "logits/rejected": -1.8576055765151978,
      "logps/chosen": -411.7281799316406,
      "logps/rejected": -432.91253662109375,
      "loss": 0.5748,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.997407078742981,
      "rewards/margins": 0.5321332216262817,
      "rewards/rejected": -1.5295404195785522,
      "step": 3090
    },
    {
      "epoch": 0.83,
      "grad_norm": 8.0625,
      "learning_rate": 4.438122617983442e-07,
      "logits/chosen": -2.1444356441497803,
      "logits/rejected": -1.8834354877471924,
      "logps/chosen": -387.5911865234375,
      "logps/rejected": -449.59393310546875,
      "loss": 0.6168,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8744697570800781,
      "rewards/margins": 0.8242238163948059,
      "rewards/rejected": -1.6986935138702393,
      "step": 3100
    },
    {
      "epoch": 0.83,
      "eval_logits/chosen": -2.001610517501831,
      "eval_logits/rejected": -1.907264232635498,
      "eval_logps/chosen": -321.8407287597656,
      "eval_logps/rejected": -354.66180419921875,
      "eval_loss": 0.5439161658287048,
      "eval_rewards/accuracies": 0.71875,
      "eval_rewards/chosen": -0.48569563031196594,
      "eval_rewards/margins": 0.47221121191978455,
      "eval_rewards/rejected": -0.9579069018363953,
      "eval_runtime": 879.843,
      "eval_samples_per_second": 4.508,
      "eval_steps_per_second": 0.141,
      "step": 3100
    },
    {
      "epoch": 0.83,
      "grad_norm": 5.15625,
      "learning_rate": 4.3066493009749853e-07,
      "logits/chosen": -1.8691234588623047,
      "logits/rejected": -1.9000431299209595,
      "logps/chosen": -384.08758544921875,
      "logps/rejected": -391.5170593261719,
      "loss": 0.554,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.6478259563446045,
      "rewards/margins": 0.49342066049575806,
      "rewards/rejected": -1.1412465572357178,
      "step": 3110
    },
    {
      "epoch": 0.83,
      "grad_norm": 6.65625,
      "learning_rate": 4.1769689822475147e-07,
      "logits/chosen": -2.1500532627105713,
      "logits/rejected": -1.6026554107666016,
      "logps/chosen": -314.80615234375,
      "logps/rejected": -322.5562744140625,
      "loss": 0.5556,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.7758539915084839,
      "rewards/margins": 0.5836711525917053,
      "rewards/rejected": -1.3595250844955444,
      "step": 3120
    },
    {
      "epoch": 0.83,
      "grad_norm": 6.9375,
      "learning_rate": 4.049092898095816e-07,
      "logits/chosen": -1.9053795337677002,
      "logits/rejected": -1.8844935894012451,
      "logps/chosen": -347.577880859375,
      "logps/rejected": -386.39056396484375,
      "loss": 0.5604,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.0622897148132324,
      "rewards/margins": 0.37788811326026917,
      "rewards/rejected": -1.4401779174804688,
      "step": 3130
    },
    {
      "epoch": 0.84,
      "grad_norm": 7.125,
      "learning_rate": 3.9230321284847856e-07,
      "logits/chosen": -1.9044173955917358,
      "logits/rejected": -1.750133752822876,
      "logps/chosen": -313.25775146484375,
      "logps/rejected": -403.019775390625,
      "loss": 0.5851,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.6727679371833801,
      "rewards/margins": 0.6754955053329468,
      "rewards/rejected": -1.3482635021209717,
      "step": 3140
    },
    {
      "epoch": 0.84,
      "grad_norm": 9.5,
      "learning_rate": 3.798797596089351e-07,
      "logits/chosen": -1.7854902744293213,
      "logits/rejected": -1.6787322759628296,
      "logps/chosen": -328.85577392578125,
      "logps/rejected": -312.057861328125,
      "loss": 0.5962,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.104310154914856,
      "rewards/margins": 0.07485084980726242,
      "rewards/rejected": -1.1791609525680542,
      "step": 3150
    },
    {
      "epoch": 0.84,
      "grad_norm": 7.5625,
      "learning_rate": 3.6764000653481263e-07,
      "logits/chosen": -1.909447431564331,
      "logits/rejected": -2.0810256004333496,
      "logps/chosen": -334.8600769042969,
      "logps/rejected": -386.43438720703125,
      "loss": 0.531,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8345718383789062,
      "rewards/margins": 0.597120463848114,
      "rewards/rejected": -1.431692361831665,
      "step": 3160
    },
    {
      "epoch": 0.85,
      "grad_norm": 6.6875,
      "learning_rate": 3.555850141530659e-07,
      "logits/chosen": -1.9847806692123413,
      "logits/rejected": -1.8810043334960938,
      "logps/chosen": -423.21923828125,
      "logps/rejected": -435.4855041503906,
      "loss": 0.5883,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.8504825830459595,
      "rewards/margins": 0.5170871019363403,
      "rewards/rejected": -1.3675696849822998,
      "step": 3170
    },
    {
      "epoch": 0.85,
      "grad_norm": 5.53125,
      "learning_rate": 3.4371582698185636e-07,
      "logits/chosen": -1.9213947057724,
      "logits/rejected": -1.9120479822158813,
      "logps/chosen": -295.59771728515625,
      "logps/rejected": -390.36346435546875,
      "loss": 0.5829,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.933371365070343,
      "rewards/margins": 0.4937419891357422,
      "rewards/rejected": -1.4271132946014404,
      "step": 3180
    },
    {
      "epoch": 0.85,
      "grad_norm": 4.65625,
      "learning_rate": 3.3203347344004737e-07,
      "logits/chosen": -2.0257608890533447,
      "logits/rejected": -1.7086207866668701,
      "logps/chosen": -351.2753601074219,
      "logps/rejected": -430.39013671875,
      "loss": 0.5138,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.8829056024551392,
      "rewards/margins": 0.7639700174331665,
      "rewards/rejected": -1.6468757390975952,
      "step": 3190
    },
    {
      "epoch": 0.85,
      "grad_norm": 5.46875,
      "learning_rate": 3.2053896575809426e-07,
      "logits/chosen": -2.059542655944824,
      "logits/rejected": -1.5287224054336548,
      "logps/chosen": -321.053466796875,
      "logps/rejected": -379.9109191894531,
      "loss": 0.5655,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.7990415096282959,
      "rewards/margins": 0.785653829574585,
      "rewards/rejected": -1.5846952199935913,
      "step": 3200
    },
    {
      "epoch": 0.85,
      "eval_logits/chosen": -2.003740072250366,
      "eval_logits/rejected": -1.9094693660736084,
      "eval_logps/chosen": -321.4441833496094,
      "eval_logps/rejected": -353.9526672363281,
      "eval_loss": 0.5443037748336792,
      "eval_rewards/accuracies": 0.7197580933570862,
      "eval_rewards/chosen": -0.48173055052757263,
      "eval_rewards/margins": 0.469085156917572,
      "eval_rewards/rejected": -0.950815737247467,
      "eval_runtime": 881.4265,
      "eval_samples_per_second": 4.5,
      "eval_steps_per_second": 0.141,
      "step": 3200
    },
    {
      "epoch": 0.86,
      "grad_norm": 9.0,
      "learning_rate": 3.092332998903416e-07,
      "logits/chosen": -1.7466233968734741,
      "logits/rejected": -1.7619507312774658,
      "logps/chosen": -343.0591125488281,
      "logps/rejected": -377.3531799316406,
      "loss": 0.5654,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.1161115169525146,
      "rewards/margins": 0.34533870220184326,
      "rewards/rejected": -1.461450219154358,
      "step": 3210
    },
    {
      "epoch": 0.86,
      "grad_norm": 7.0625,
      "learning_rate": 2.981174554287239e-07,
      "logits/chosen": -2.036564350128174,
      "logits/rejected": -1.8373911380767822,
      "logps/chosen": -284.9559631347656,
      "logps/rejected": -330.1531677246094,
      "loss": 0.5452,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.8419291377067566,
      "rewards/margins": 0.3416334092617035,
      "rewards/rejected": -1.1835625171661377,
      "step": 3220
    },
    {
      "epoch": 0.86,
      "grad_norm": 8.25,
      "learning_rate": 2.871923955178918e-07,
      "logits/chosen": -2.0975453853607178,
      "logits/rejected": -1.7876338958740234,
      "logps/chosen": -373.63348388671875,
      "logps/rejected": -352.13726806640625,
      "loss": 0.6045,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8301500082015991,
      "rewards/margins": 0.5695451498031616,
      "rewards/rejected": -1.3996951580047607,
      "step": 3230
    },
    {
      "epoch": 0.86,
      "grad_norm": 7.3125,
      "learning_rate": 2.764590667717562e-07,
      "logits/chosen": -2.10371732711792,
      "logits/rejected": -1.9427160024642944,
      "logps/chosen": -344.491455078125,
      "logps/rejected": -371.59698486328125,
      "loss": 0.6373,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.920907199382782,
      "rewards/margins": 0.34861811995506287,
      "rewards/rejected": -1.2695252895355225,
      "step": 3240
    },
    {
      "epoch": 0.87,
      "grad_norm": 7.9375,
      "learning_rate": 2.6591839919146963e-07,
      "logits/chosen": -1.895089864730835,
      "logits/rejected": -1.9803454875946045,
      "logps/chosen": -304.5118713378906,
      "logps/rejected": -418.89166259765625,
      "loss": 0.5822,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.0374428033828735,
      "rewards/margins": 0.49948644638061523,
      "rewards/rejected": -1.5369291305541992,
      "step": 3250
    },
    {
      "epoch": 0.87,
      "grad_norm": 6.125,
      "learning_rate": 2.555713060848433e-07,
      "logits/chosen": -2.035060405731201,
      "logits/rejected": -1.8383516073226929,
      "logps/chosen": -360.76995849609375,
      "logps/rejected": -394.3505859375,
      "loss": 0.5832,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.8940296173095703,
      "rewards/margins": 0.4352487027645111,
      "rewards/rejected": -1.3292782306671143,
      "step": 3260
    },
    {
      "epoch": 0.87,
      "grad_norm": 7.15625,
      "learning_rate": 2.454186839872158e-07,
      "logits/chosen": -1.9722185134887695,
      "logits/rejected": -1.8550584316253662,
      "logps/chosen": -353.49432373046875,
      "logps/rejected": -470.0379943847656,
      "loss": 0.5922,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.8836105465888977,
      "rewards/margins": 0.4038814604282379,
      "rewards/rejected": -1.2874921560287476,
      "step": 3270
    },
    {
      "epoch": 0.87,
      "grad_norm": 8.4375,
      "learning_rate": 2.3546141258376786e-07,
      "logits/chosen": -2.0904781818389893,
      "logits/rejected": -1.7301757335662842,
      "logps/chosen": -399.94622802734375,
      "logps/rejected": -386.0555725097656,
      "loss": 0.5848,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.8415128588676453,
      "rewards/margins": 0.25778400897979736,
      "rewards/rejected": -1.0992968082427979,
      "step": 3280
    },
    {
      "epoch": 0.88,
      "grad_norm": 7.0625,
      "learning_rate": 2.257003546333042e-07,
      "logits/chosen": -1.8003848791122437,
      "logits/rejected": -1.894014596939087,
      "logps/chosen": -292.92791748046875,
      "logps/rejected": -345.055419921875,
      "loss": 0.575,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.9022985696792603,
      "rewards/margins": 0.311948299407959,
      "rewards/rejected": -1.2142467498779297,
      "step": 3290
    },
    {
      "epoch": 0.88,
      "grad_norm": 6.90625,
      "learning_rate": 2.1613635589349756e-07,
      "logits/chosen": -1.879026174545288,
      "logits/rejected": -1.8557407855987549,
      "logps/chosen": -309.6040954589844,
      "logps/rejected": -383.9114685058594,
      "loss": 0.5201,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.7737454175949097,
      "rewards/margins": 0.6824396252632141,
      "rewards/rejected": -1.4561851024627686,
      "step": 3300
    },
    {
      "epoch": 0.88,
      "eval_logits/chosen": -2.0024094581604004,
      "eval_logits/rejected": -1.9081629514694214,
      "eval_logps/chosen": -321.6214599609375,
      "eval_logps/rejected": -354.049560546875,
      "eval_loss": 0.544136643409729,
      "eval_rewards/accuracies": 0.7227822542190552,
      "eval_rewards/chosen": -0.48350316286087036,
      "eval_rewards/margins": 0.4682813286781311,
      "eval_rewards/rejected": -0.9517845511436462,
      "eval_runtime": 880.9317,
      "eval_samples_per_second": 4.502,
      "eval_steps_per_second": 0.141,
      "step": 3300
    },
    {
      "epoch": 0.88,
      "grad_norm": 5.1875,
      "learning_rate": 2.0677024504760752e-07,
      "logits/chosen": -2.190134286880493,
      "logits/rejected": -1.9197311401367188,
      "logps/chosen": -384.45147705078125,
      "logps/rejected": -368.6825256347656,
      "loss": 0.5731,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.7496887445449829,
      "rewards/margins": 0.6117998361587524,
      "rewards/rejected": -1.3614885807037354,
      "step": 3310
    },
    {
      "epoch": 0.89,
      "grad_norm": 4.65625,
      "learning_rate": 1.9760283363267684e-07,
      "logits/chosen": -1.8956794738769531,
      "logits/rejected": -1.7137658596038818,
      "logps/chosen": -293.9393005371094,
      "logps/rejected": -342.24774169921875,
      "loss": 0.5528,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.8469433784484863,
      "rewards/margins": 0.7452161908149719,
      "rewards/rejected": -1.592159628868103,
      "step": 3320
    },
    {
      "epoch": 0.89,
      "grad_norm": 7.5625,
      "learning_rate": 1.8863491596921745e-07,
      "logits/chosen": -1.8568884134292603,
      "logits/rejected": -1.849953293800354,
      "logps/chosen": -295.68023681640625,
      "logps/rejected": -359.2909240722656,
      "loss": 0.6261,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.7082206010818481,
      "rewards/margins": 0.5012540221214294,
      "rewards/rejected": -1.2094746828079224,
      "step": 3330
    },
    {
      "epoch": 0.89,
      "grad_norm": 6.875,
      "learning_rate": 1.798672690923828e-07,
      "logits/chosen": -1.9033963680267334,
      "logits/rejected": -2.080915927886963,
      "logps/chosen": -407.0228271484375,
      "logps/rejected": -412.4893493652344,
      "loss": 0.6074,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.9729453921318054,
      "rewards/margins": 0.18394029140472412,
      "rewards/rejected": -1.1568857431411743,
      "step": 3340
    },
    {
      "epoch": 0.89,
      "grad_norm": 5.59375,
      "learning_rate": 1.713006526846439e-07,
      "logits/chosen": -2.0385947227478027,
      "logits/rejected": -1.7743995189666748,
      "logps/chosen": -385.3775939941406,
      "logps/rejected": -368.5835876464844,
      "loss": 0.5765,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.8837639689445496,
      "rewards/margins": 0.5611820220947266,
      "rewards/rejected": -1.444946050643921,
      "step": 3350
    },
    {
      "epoch": 0.9,
      "grad_norm": 5.1875,
      "learning_rate": 1.629358090099639e-07,
      "logits/chosen": -2.1214609146118164,
      "logits/rejected": -1.8527650833129883,
      "logps/chosen": -322.072265625,
      "logps/rejected": -368.9439392089844,
      "loss": 0.5879,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.6585100889205933,
      "rewards/margins": 0.6092414855957031,
      "rewards/rejected": -1.267751693725586,
      "step": 3360
    },
    {
      "epoch": 0.9,
      "grad_norm": 5.9375,
      "learning_rate": 1.5477346284948292e-07,
      "logits/chosen": -1.77948796749115,
      "logits/rejected": -1.605921983718872,
      "logps/chosen": -325.584228515625,
      "logps/rejected": -406.86224365234375,
      "loss": 0.5871,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.7979250550270081,
      "rewards/margins": 0.7102919816970825,
      "rewards/rejected": -1.5082170963287354,
      "step": 3370
    },
    {
      "epoch": 0.9,
      "grad_norm": 8.0,
      "learning_rate": 1.4681432143872133e-07,
      "logits/chosen": -2.049950122833252,
      "logits/rejected": -1.9191957712173462,
      "logps/chosen": -325.8570556640625,
      "logps/rejected": -376.7242431640625,
      "loss": 0.6153,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.7265611886978149,
      "rewards/margins": 0.6458820104598999,
      "rewards/rejected": -1.3724431991577148,
      "step": 3380
    },
    {
      "epoch": 0.9,
      "grad_norm": 6.5625,
      "learning_rate": 1.3905907440629752e-07,
      "logits/chosen": -2.1785340309143066,
      "logits/rejected": -1.6822811365127563,
      "logps/chosen": -345.34124755859375,
      "logps/rejected": -357.4413757324219,
      "loss": 0.5753,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.7763444781303406,
      "rewards/margins": 0.6707662343978882,
      "rewards/rejected": -1.447110891342163,
      "step": 3390
    },
    {
      "epoch": 0.91,
      "grad_norm": 5.1875,
      "learning_rate": 1.31508393714177e-07,
      "logits/chosen": -1.9890329837799072,
      "logits/rejected": -1.8519636392593384,
      "logps/chosen": -365.4205017089844,
      "logps/rejected": -449.1064453125,
      "loss": 0.5576,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.9402433633804321,
      "rewards/margins": 0.5368240475654602,
      "rewards/rejected": -1.477067232131958,
      "step": 3400
    },
    {
      "epoch": 0.91,
      "eval_logits/chosen": -2.0055689811706543,
      "eval_logits/rejected": -1.9113283157348633,
      "eval_logps/chosen": -321.4092102050781,
      "eval_logps/rejected": -353.7254333496094,
      "eval_loss": 0.5442517995834351,
      "eval_rewards/accuracies": 0.7247983813285828,
      "eval_rewards/chosen": -0.4813808798789978,
      "eval_rewards/margins": 0.4671625792980194,
      "eval_rewards/rejected": -0.9485434293746948,
      "eval_runtime": 880.6957,
      "eval_samples_per_second": 4.503,
      "eval_steps_per_second": 0.141,
      "step": 3400
    },
    {
      "epoch": 0.91,
      "grad_norm": 4.125,
      "learning_rate": 1.241629335994471e-07,
      "logits/chosen": -1.7663733959197998,
      "logits/rejected": -1.9155349731445312,
      "logps/chosen": -323.29571533203125,
      "logps/rejected": -524.2689208984375,
      "loss": 0.5664,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.0119855403900146,
      "rewards/margins": 0.533748984336853,
      "rewards/rejected": -1.5457346439361572,
      "step": 3410
    },
    {
      "epoch": 0.91,
      "grad_norm": 5.1875,
      "learning_rate": 1.1702333051763271e-07,
      "logits/chosen": -2.018174171447754,
      "logits/rejected": -1.6946039199829102,
      "logps/chosen": -308.33892822265625,
      "logps/rejected": -363.96380615234375,
      "loss": 0.5765,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.8656156659126282,
      "rewards/margins": 0.6298501491546631,
      "rewards/rejected": -1.4954659938812256,
      "step": 3420
    },
    {
      "epoch": 0.91,
      "grad_norm": 4.84375,
      "learning_rate": 1.1009020308754587e-07,
      "logits/chosen": -1.7159515619277954,
      "logits/rejected": -1.6221239566802979,
      "logps/chosen": -291.56396484375,
      "logps/rejected": -324.3634338378906,
      "loss": 0.5258,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.9006462097167969,
      "rewards/margins": 0.5663580298423767,
      "rewards/rejected": -1.4670041799545288,
      "step": 3430
    },
    {
      "epoch": 0.92,
      "grad_norm": 7.15625,
      "learning_rate": 1.0336415203768962e-07,
      "logits/chosen": -1.8973805904388428,
      "logits/rejected": -1.6162137985229492,
      "logps/chosen": -304.2796936035156,
      "logps/rejected": -381.62908935546875,
      "loss": 0.5733,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.824349582195282,
      "rewards/margins": 0.6480148434638977,
      "rewards/rejected": -1.4723644256591797,
      "step": 3440
    },
    {
      "epoch": 0.92,
      "grad_norm": 6.3125,
      "learning_rate": 9.684576015420277e-08,
      "logits/chosen": -1.9445997476577759,
      "logits/rejected": -1.931964635848999,
      "logps/chosen": -287.91412353515625,
      "logps/rejected": -354.6346740722656,
      "loss": 0.5835,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.8815528750419617,
      "rewards/margins": 0.41902589797973633,
      "rewards/rejected": -1.3005788326263428,
      "step": 3450
    },
    {
      "epoch": 0.92,
      "grad_norm": 6.15625,
      "learning_rate": 9.053559223036746e-08,
      "logits/chosen": -1.9900093078613281,
      "logits/rejected": -1.8198668956756592,
      "logps/chosen": -379.8409118652344,
      "logps/rejected": -423.17919921875,
      "loss": 0.5983,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.8261159062385559,
      "rewards/margins": 0.5475336313247681,
      "rewards/rejected": -1.3736494779586792,
      "step": 3460
    },
    {
      "epoch": 0.93,
      "grad_norm": 10.25,
      "learning_rate": 8.44341950176683e-08,
      "logits/chosen": -1.8980391025543213,
      "logits/rejected": -1.7415292263031006,
      "logps/chosen": -412.56396484375,
      "logps/rejected": -457.317138671875,
      "loss": 0.633,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.045347809791565,
      "rewards/margins": 0.6671690344810486,
      "rewards/rejected": -1.7125167846679688,
      "step": 3470
    },
    {
      "epoch": 0.93,
      "grad_norm": 6.0,
      "learning_rate": 7.854209717842231e-08,
      "logits/chosen": -2.01379132270813,
      "logits/rejected": -1.8360013961791992,
      "logps/chosen": -314.223388671875,
      "logps/rejected": -392.3841247558594,
      "loss": 0.5368,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.8868504762649536,
      "rewards/margins": 0.7094804048538208,
      "rewards/rejected": -1.596331000328064,
      "step": 3480
    },
    {
      "epoch": 0.93,
      "grad_norm": 5.71875,
      "learning_rate": 7.285980923996989e-08,
      "logits/chosen": -1.9938936233520508,
      "logits/rejected": -1.726118803024292,
      "logps/chosen": -378.59490966796875,
      "logps/rejected": -401.52789306640625,
      "loss": 0.5672,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.1852322816848755,
      "rewards/margins": 0.4363698959350586,
      "rewards/rejected": -1.6216020584106445,
      "step": 3490
    },
    {
      "epoch": 0.93,
      "grad_norm": 9.125,
      "learning_rate": 6.738782355044048e-08,
      "logits/chosen": -1.975852608680725,
      "logits/rejected": -1.8234798908233643,
      "logps/chosen": -351.89471435546875,
      "logps/rejected": -370.0545654296875,
      "loss": 0.5929,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.8610925674438477,
      "rewards/margins": 0.40970760583877563,
      "rewards/rejected": -1.2708001136779785,
      "step": 3500
    },
    {
      "epoch": 0.93,
      "eval_logits/chosen": -2.0028774738311768,
      "eval_logits/rejected": -1.908638596534729,
      "eval_logps/chosen": -321.4120788574219,
      "eval_logps/rejected": -353.68536376953125,
      "eval_loss": 0.5443395376205444,
      "eval_rewards/accuracies": 0.7247983813285828,
      "eval_rewards/chosen": -0.48140960931777954,
      "eval_rewards/margins": 0.4667326807975769,
      "eval_rewards/rejected": -0.9481422901153564,
      "eval_runtime": 880.7017,
      "eval_samples_per_second": 4.503,
      "eval_steps_per_second": 0.141,
      "step": 3500
    },
    {
      "epoch": 0.94,
      "grad_norm": 7.28125,
      "learning_rate": 6.212661423609184e-08,
      "logits/chosen": -2.174973964691162,
      "logits/rejected": -1.8390709161758423,
      "logps/chosen": -381.13995361328125,
      "logps/rejected": -382.50640869140625,
      "loss": 0.5134,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.8438682556152344,
      "rewards/margins": 0.8239489793777466,
      "rewards/rejected": -1.6678173542022705,
      "step": 3510
    },
    {
      "epoch": 0.94,
      "grad_norm": 4.84375,
      "learning_rate": 5.707663716023021e-08,
      "logits/chosen": -2.090505838394165,
      "logits/rejected": -2.0210907459259033,
      "logps/chosen": -359.7513732910156,
      "logps/rejected": -468.4659118652344,
      "loss": 0.5535,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.6450691819190979,
      "rewards/margins": 0.9867405891418457,
      "rewards/rejected": -1.6318098306655884,
      "step": 3520
    },
    {
      "epoch": 0.94,
      "grad_norm": 7.5,
      "learning_rate": 5.22383298837098e-08,
      "logits/chosen": -2.133789539337158,
      "logits/rejected": -2.189826011657715,
      "logps/chosen": -327.96295166015625,
      "logps/rejected": -408.6748962402344,
      "loss": 0.5398,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.480525404214859,
      "rewards/margins": 0.6341120004653931,
      "rewards/rejected": -1.1146374940872192,
      "step": 3530
    },
    {
      "epoch": 0.94,
      "grad_norm": 6.75,
      "learning_rate": 4.761211162702117e-08,
      "logits/chosen": -1.992672324180603,
      "logits/rejected": -1.9722591638565063,
      "logps/chosen": -440.482666015625,
      "logps/rejected": -392.73626708984375,
      "loss": 0.5573,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.1931569576263428,
      "rewards/margins": 0.34043723344802856,
      "rewards/rejected": -1.5335941314697266,
      "step": 3540
    },
    {
      "epoch": 0.95,
      "grad_norm": 7.40625,
      "learning_rate": 4.319838323396691e-08,
      "logits/chosen": -2.055854320526123,
      "logits/rejected": -1.7457075119018555,
      "logps/chosen": -359.06640625,
      "logps/rejected": -375.88128662109375,
      "loss": 0.578,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.8032671213150024,
      "rewards/margins": 0.8615306615829468,
      "rewards/rejected": -1.6647977828979492,
      "step": 3550
    },
    {
      "epoch": 0.95,
      "grad_norm": 5.6875,
      "learning_rate": 3.8997527136930004e-08,
      "logits/chosen": -1.9341676235198975,
      "logits/rejected": -1.8679182529449463,
      "logps/chosen": -347.9366149902344,
      "logps/rejected": -378.9738464355469,
      "loss": 0.5738,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -1.0352892875671387,
      "rewards/margins": 0.1288614571094513,
      "rewards/rejected": -1.1641508340835571,
      "step": 3560
    },
    {
      "epoch": 0.95,
      "grad_norm": 7.6875,
      "learning_rate": 3.5009907323737826e-08,
      "logits/chosen": -1.936774492263794,
      "logits/rejected": -1.9746652841567993,
      "logps/chosen": -346.8591613769531,
      "logps/rejected": -358.208984375,
      "loss": 0.6335,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -1.085648775100708,
      "rewards/margins": 0.17988404631614685,
      "rewards/rejected": -1.2655327320098877,
      "step": 3570
    },
    {
      "epoch": 0.95,
      "grad_norm": 5.9375,
      "learning_rate": 3.1235869306123766e-08,
      "logits/chosen": -1.923948049545288,
      "logits/rejected": -1.668055772781372,
      "logps/chosen": -348.63470458984375,
      "logps/rejected": -383.20281982421875,
      "loss": 0.5622,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.8869673609733582,
      "rewards/margins": 0.6319563984870911,
      "rewards/rejected": -1.5189237594604492,
      "step": 3580
    },
    {
      "epoch": 0.96,
      "grad_norm": 5.6875,
      "learning_rate": 2.767574008979007e-08,
      "logits/chosen": -1.967635154724121,
      "logits/rejected": -1.5086393356323242,
      "logps/chosen": -330.8482971191406,
      "logps/rejected": -338.0470275878906,
      "loss": 0.5572,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9556097984313965,
      "rewards/margins": 0.7259234189987183,
      "rewards/rejected": -1.6815332174301147,
      "step": 3590
    },
    {
      "epoch": 0.96,
      "grad_norm": 5.0,
      "learning_rate": 2.4329828146074096e-08,
      "logits/chosen": -2.2539055347442627,
      "logits/rejected": -1.8058958053588867,
      "logps/chosen": -425.53424072265625,
      "logps/rejected": -413.652587890625,
      "loss": 0.507,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.7969464063644409,
      "rewards/margins": 0.6987387537956238,
      "rewards/rejected": -1.4956852197647095,
      "step": 3600
    },
    {
      "epoch": 0.96,
      "eval_logits/chosen": -2.0028469562530518,
      "eval_logits/rejected": -1.9085676670074463,
      "eval_logps/chosen": -321.421630859375,
      "eval_logps/rejected": -353.7490539550781,
      "eval_loss": 0.5443490147590637,
      "eval_rewards/accuracies": 0.7197580933570862,
      "eval_rewards/chosen": -0.4815046489238739,
      "eval_rewards/margins": 0.4672749936580658,
      "eval_rewards/rejected": -0.9487796425819397,
      "eval_runtime": 880.3176,
      "eval_samples_per_second": 4.505,
      "eval_steps_per_second": 0.141,
      "step": 3600
    },
    {
      "epoch": 0.96,
      "grad_norm": 7.0625,
      "learning_rate": 2.1198423385220822e-08,
      "logits/chosen": -2.114412307739258,
      "logits/rejected": -1.8712831735610962,
      "logps/chosen": -363.111083984375,
      "logps/rejected": -352.7618713378906,
      "loss": 0.5905,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.8813158273696899,
      "rewards/margins": 0.4961695671081543,
      "rewards/rejected": -1.3774851560592651,
      "step": 3610
    },
    {
      "epoch": 0.97,
      "grad_norm": 5.53125,
      "learning_rate": 1.82817971312621e-08,
      "logits/chosen": -1.9698861837387085,
      "logits/rejected": -1.5481913089752197,
      "logps/chosen": -335.3878173828125,
      "logps/rejected": -390.8907470703125,
      "loss": 0.6011,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.9004653096199036,
      "rewards/margins": 0.8001790046691895,
      "rewards/rejected": -1.7006442546844482,
      "step": 3620
    },
    {
      "epoch": 0.97,
      "grad_norm": 5.65625,
      "learning_rate": 1.5580202098509078e-08,
      "logits/chosen": -2.050546169281006,
      "logits/rejected": -1.9141814708709717,
      "logps/chosen": -323.13751220703125,
      "logps/rejected": -364.39178466796875,
      "loss": 0.5673,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.8248991966247559,
      "rewards/margins": 0.6621609926223755,
      "rewards/rejected": -1.4870601892471313,
      "step": 3630
    },
    {
      "epoch": 0.97,
      "grad_norm": 6.84375,
      "learning_rate": 1.3093872369654148e-08,
      "logits/chosen": -1.8875898122787476,
      "logits/rejected": -2.097785711288452,
      "logps/chosen": -317.3999938964844,
      "logps/rejected": -373.4709777832031,
      "loss": 0.5329,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.870337188243866,
      "rewards/margins": 0.49381017684936523,
      "rewards/rejected": -1.364147424697876,
      "step": 3640
    },
    {
      "epoch": 0.97,
      "grad_norm": 5.875,
      "learning_rate": 1.0823023375489128e-08,
      "logits/chosen": -2.175565242767334,
      "logits/rejected": -1.6107755899429321,
      "logps/chosen": -365.5906677246094,
      "logps/rejected": -378.67340087890625,
      "loss": 0.5511,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.8032134175300598,
      "rewards/margins": 0.80914705991745,
      "rewards/rejected": -1.6123603582382202,
      "step": 3650
    },
    {
      "epoch": 0.98,
      "grad_norm": 7.21875,
      "learning_rate": 8.767851876239075e-09,
      "logits/chosen": -1.9347044229507446,
      "logits/rejected": -1.964540719985962,
      "logps/chosen": -389.79461669921875,
      "logps/rejected": -418.52313232421875,
      "loss": 0.636,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.9112573862075806,
      "rewards/margins": 0.340864896774292,
      "rewards/rejected": -1.2521222829818726,
      "step": 3660
    },
    {
      "epoch": 0.98,
      "grad_norm": 5.71875,
      "learning_rate": 6.9285359445145366e-09,
      "logits/chosen": -2.0089118480682373,
      "logits/rejected": -1.7662067413330078,
      "logps/chosen": -357.1235656738281,
      "logps/rejected": -324.48297119140625,
      "loss": 0.5968,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.9385967254638672,
      "rewards/margins": 0.3650709390640259,
      "rewards/rejected": -1.303667664527893,
      "step": 3670
    },
    {
      "epoch": 0.98,
      "grad_norm": 7.21875,
      "learning_rate": 5.305234949880001e-09,
      "logits/chosen": -1.934048056602478,
      "logits/rejected": -1.864232063293457,
      "logps/chosen": -319.323974609375,
      "logps/rejected": -350.2051086425781,
      "loss": 0.5927,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.9362686276435852,
      "rewards/margins": 0.3970395028591156,
      "rewards/rejected": -1.3333081007003784,
      "step": 3680
    },
    {
      "epoch": 0.98,
      "grad_norm": 5.65625,
      "learning_rate": 3.8980895450474455e-09,
      "logits/chosen": -1.9186967611312866,
      "logits/rejected": -2.0729496479034424,
      "logps/chosen": -343.03448486328125,
      "logps/rejected": -399.3251647949219,
      "loss": 0.5972,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.9254744648933411,
      "rewards/margins": 0.42660894989967346,
      "rewards/rejected": -1.352083444595337,
      "step": 3690
    },
    {
      "epoch": 0.99,
      "grad_norm": 5.53125,
      "learning_rate": 2.7072216536885855e-09,
      "logits/chosen": -2.034536361694336,
      "logits/rejected": -1.6379082202911377,
      "logps/chosen": -359.1803894042969,
      "logps/rejected": -393.0415344238281,
      "loss": 0.5412,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -1.2558963298797607,
      "rewards/margins": 0.3431646227836609,
      "rewards/rejected": -1.5990610122680664,
      "step": 3700
    },
    {
      "epoch": 0.99,
      "eval_logits/chosen": -2.0033113956451416,
      "eval_logits/rejected": -1.9090486764907837,
      "eval_logps/chosen": -321.4201965332031,
      "eval_logps/rejected": -353.7513732910156,
      "eval_loss": 0.5442675948143005,
      "eval_rewards/accuracies": 0.7227822542190552,
      "eval_rewards/chosen": -0.4814910292625427,
      "eval_rewards/margins": 0.46731171011924744,
      "eval_rewards/rejected": -0.9488027691841125,
      "eval_runtime": 880.112,
      "eval_samples_per_second": 4.506,
      "eval_steps_per_second": 0.141,
      "step": 3700
    },
    {
      "epoch": 0.99,
      "grad_norm": 5.84375,
      "learning_rate": 1.7327344598702667e-09,
      "logits/chosen": -1.9787992238998413,
      "logits/rejected": -1.7021030187606812,
      "logps/chosen": -296.86370849609375,
      "logps/rejected": -335.06024169921875,
      "loss": 0.5888,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8101051449775696,
      "rewards/margins": 0.646266758441925,
      "rewards/rejected": -1.4563719034194946,
      "step": 3710
    },
    {
      "epoch": 0.99,
      "grad_norm": 5.09375,
      "learning_rate": 9.747123991141193e-10,
      "logits/chosen": -1.8681468963623047,
      "logits/rejected": -1.63045334815979,
      "logps/chosen": -374.77362060546875,
      "logps/rejected": -396.61224365234375,
      "loss": 0.5461,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.9298361539840698,
      "rewards/margins": 0.4873422682285309,
      "rewards/rejected": -1.4171783924102783,
      "step": 3720
    },
    {
      "epoch": 0.99,
      "grad_norm": 7.15625,
      "learning_rate": 4.332211510807427e-10,
      "logits/chosen": -2.002690315246582,
      "logits/rejected": -1.8574998378753662,
      "logps/chosen": -355.7222900390625,
      "logps/rejected": -389.08526611328125,
      "loss": 0.5306,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.924048125743866,
      "rewards/margins": 0.661828339099884,
      "rewards/rejected": -1.585876226425171,
      "step": 3730
    },
    {
      "epoch": 1.0,
      "grad_norm": 6.875,
      "learning_rate": 1.0830763387897902e-10,
      "logits/chosen": -1.9653832912445068,
      "logits/rejected": -1.6311572790145874,
      "logps/chosen": -360.54827880859375,
      "logps/rejected": -437.73028564453125,
      "loss": 0.5645,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9649810791015625,
      "rewards/margins": 0.6192973852157593,
      "rewards/rejected": -1.5842787027359009,
      "step": 3740
    },
    {
      "epoch": 1.0,
      "grad_norm": 6.8125,
      "learning_rate": 0.0,
      "logits/chosen": -2.005305767059326,
      "logits/rejected": -1.918457269668579,
      "logps/chosen": -382.94732666015625,
      "logps/rejected": -410.37542724609375,
      "loss": 0.6031,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.8707008361816406,
      "rewards/margins": 0.6126598119735718,
      "rewards/rejected": -1.4833606481552124,
      "step": 3750
    },
    {
      "epoch": 1.0,
      "step": 3750,
      "total_flos": 0.0,
      "train_loss": 0.5955694535573324,
      "train_runtime": 64907.5582,
      "train_samples_per_second": 0.924,
      "train_steps_per_second": 0.058
    }
  ],
  "logging_steps": 10,
  "max_steps": 3750,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 100,
  "total_flos": 0.0,
  "train_batch_size": 4,
  "trial_name": null,
  "trial_params": null
}