{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 1.0,
  "eval_steps": 100,
  "global_step": 385,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.0,
      "learning_rate": 1.282051282051282e-07,
      "logits/chosen": -1.7278180122375488,
      "logits/rejected": -1.7377450466156006,
      "logps/chosen": -29.553977966308594,
      "logps/rejected": -42.813133239746094,
      "loss": 2.7778,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 1
    },
    {
      "epoch": 0.03,
      "learning_rate": 1.282051282051282e-06,
      "logits/chosen": -1.8667066097259521,
      "logits/rejected": -1.8710108995437622,
      "logps/chosen": -36.97679901123047,
      "logps/rejected": -33.63804626464844,
      "loss": 2.6754,
      "rewards/accuracies": 0.5138888955116272,
      "rewards/chosen": 0.008938448503613472,
      "rewards/margins": 0.011331514455378056,
      "rewards/rejected": -0.0023930652532726526,
      "step": 10
    },
    {
      "epoch": 0.05,
      "learning_rate": 2.564102564102564e-06,
      "logits/chosen": -1.9984171390533447,
      "logits/rejected": -2.001072406768799,
      "logps/chosen": -29.64606285095215,
      "logps/rejected": -29.043380737304688,
      "loss": 2.8951,
      "rewards/accuracies": 0.4124999940395355,
      "rewards/chosen": -0.0011603410821408033,
      "rewards/margins": -0.007179437670856714,
      "rewards/rejected": 0.006019095424562693,
      "step": 20
    },
    {
      "epoch": 0.08,
      "learning_rate": 3.846153846153847e-06,
      "logits/chosen": -1.9207251071929932,
      "logits/rejected": -1.918031096458435,
      "logps/chosen": -31.397014617919922,
      "logps/rejected": -33.21030044555664,
      "loss": 2.7749,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": 0.0057237339206039906,
      "rewards/margins": 0.0032763103954494,
      "rewards/rejected": 0.0024474230594933033,
      "step": 30
    },
    {
      "epoch": 0.1,
      "learning_rate": 4.999896948438434e-06,
      "logits/chosen": -2.017704486846924,
      "logits/rejected": -2.0089590549468994,
      "logps/chosen": -32.57511520385742,
      "logps/rejected": -32.51970672607422,
      "loss": 2.7984,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.0005076262168586254,
      "rewards/margins": 0.002619259525090456,
      "rewards/rejected": -0.0021116335410624743,
      "step": 40
    },
    {
      "epoch": 0.13,
      "learning_rate": 4.987541037542187e-06,
      "logits/chosen": -1.8626596927642822,
      "logits/rejected": -1.8518798351287842,
      "logps/chosen": -33.531227111816406,
      "logps/rejected": -35.452476501464844,
      "loss": 2.7501,
      "rewards/accuracies": 0.48750001192092896,
      "rewards/chosen": 0.007876711897552013,
      "rewards/margins": 0.00791595596820116,
      "rewards/rejected": -3.924337215721607e-05,
      "step": 50
    },
    {
      "epoch": 0.16,
      "learning_rate": 4.954691471941119e-06,
      "logits/chosen": -1.9410473108291626,
      "logits/rejected": -1.9429900646209717,
      "logps/chosen": -32.554359436035156,
      "logps/rejected": -33.228172302246094,
      "loss": 2.6127,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.01374770700931549,
      "rewards/margins": 0.03078722581267357,
      "rewards/rejected": -0.017039518803358078,
      "step": 60
    },
    {
      "epoch": 0.18,
      "learning_rate": 4.901618883413549e-06,
      "logits/chosen": -2.072329044342041,
      "logits/rejected": -2.0772993564605713,
      "logps/chosen": -33.98577880859375,
      "logps/rejected": -36.6306266784668,
      "loss": 2.7121,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.0008118894184008241,
      "rewards/margins": 0.0193378534168005,
      "rewards/rejected": -0.01852596551179886,
      "step": 70
    },
    {
      "epoch": 0.21,
      "learning_rate": 4.828760511501322e-06,
      "logits/chosen": -1.9340860843658447,
      "logits/rejected": -1.937217354774475,
      "logps/chosen": -34.292659759521484,
      "logps/rejected": -34.65100860595703,
      "loss": 2.3666,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.040544699877500534,
      "rewards/margins": 0.05812396854162216,
      "rewards/rejected": -0.017579272389411926,
      "step": 80
    },
    {
      "epoch": 0.23,
      "learning_rate": 4.7367166013034295e-06,
      "logits/chosen": -1.9430879354476929,
      "logits/rejected": -1.9476264715194702,
      "logps/chosen": -32.375579833984375,
      "logps/rejected": -32.32474899291992,
      "loss": 2.7344,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": 0.02755170688033104,
      "rewards/margins": 0.017304658889770508,
      "rewards/rejected": 0.010247047990560532,
      "step": 90
    },
    {
      "epoch": 0.26,
      "learning_rate": 4.626245458345211e-06,
      "logits/chosen": -2.0406196117401123,
      "logits/rejected": -2.0386359691619873,
      "logps/chosen": -32.16400146484375,
      "logps/rejected": -31.278045654296875,
      "loss": 2.59,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": 0.023049456998705864,
      "rewards/margins": 0.02659946121275425,
      "rewards/rejected": -0.0035500028170645237,
      "step": 100
    },
    {
      "epoch": 0.26,
      "eval_logits/chosen": -2.2353110313415527,
      "eval_logits/rejected": -2.2304632663726807,
      "eval_logps/chosen": -34.025028228759766,
      "eval_logps/rejected": -37.4992561340332,
      "eval_loss": 2.8957631587982178,
      "eval_rewards/accuracies": 0.4730066657066345,
      "eval_rewards/chosen": 0.0028568038251250982,
      "eval_rewards/margins": -0.0023524421267211437,
      "eval_rewards/rejected": 0.005209244787693024,
      "eval_runtime": 146.1162,
      "eval_samples_per_second": 2.347,
      "eval_steps_per_second": 0.294,
      "step": 100
    },
    {
      "epoch": 0.29,
      "learning_rate": 4.498257201263691e-06,
      "logits/chosen": -1.9961220026016235,
      "logits/rejected": -1.993739128112793,
      "logps/chosen": -33.09939193725586,
      "logps/rejected": -34.00792694091797,
      "loss": 2.7463,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": 0.043326906859874725,
      "rewards/margins": 0.036895059049129486,
      "rewards/rejected": 0.006431845016777515,
      "step": 110
    },
    {
      "epoch": 0.31,
      "learning_rate": 4.353806263777678e-06,
      "logits/chosen": -2.0079345703125,
      "logits/rejected": -1.999603509902954,
      "logps/chosen": -32.316917419433594,
      "logps/rejected": -32.12470245361328,
      "loss": 2.6293,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.038397181779146194,
      "rewards/margins": 0.027356009930372238,
      "rewards/rejected": 0.011041165329515934,
      "step": 120
    },
    {
      "epoch": 0.34,
      "learning_rate": 4.1940827077152755e-06,
      "logits/chosen": -2.0357651710510254,
      "logits/rejected": -2.0277907848358154,
      "logps/chosen": -30.32888412475586,
      "logps/rejected": -32.07011413574219,
      "loss": 2.5267,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": 0.04404935985803604,
      "rewards/margins": 0.049181826412677765,
      "rewards/rejected": -0.005132470745593309,
      "step": 130
    },
    {
      "epoch": 0.36,
      "learning_rate": 4.0204024186666215e-06,
      "logits/chosen": -1.9659456014633179,
      "logits/rejected": -1.9761817455291748,
      "logps/chosen": -31.22298240661621,
      "logps/rejected": -32.57875442504883,
      "loss": 2.3006,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": 0.05900341272354126,
      "rewards/margins": 0.0721658319234848,
      "rewards/rejected": -0.013162411749362946,
      "step": 140
    },
    {
      "epoch": 0.39,
      "learning_rate": 3.834196265035119e-06,
      "logits/chosen": -1.877907156944275,
      "logits/rejected": -1.879045844078064,
      "logps/chosen": -33.954994201660156,
      "logps/rejected": -34.817832946777344,
      "loss": 2.2877,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": 0.07130923122167587,
      "rewards/margins": 0.08948297053575516,
      "rewards/rejected": -0.018173744902014732,
      "step": 150
    },
    {
      "epoch": 0.42,
      "learning_rate": 3.636998309800573e-06,
      "logits/chosen": -1.9301284551620483,
      "logits/rejected": -1.9266859292984009,
      "logps/chosen": -36.037498474121094,
      "logps/rejected": -32.7092170715332,
      "loss": 2.5282,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.04025455564260483,
      "rewards/margins": 0.036193959414958954,
      "rewards/rejected": 0.004060596693307161,
      "step": 160
    },
    {
      "epoch": 0.44,
      "learning_rate": 3.4304331721118078e-06,
      "logits/chosen": -2.03144907951355,
      "logits/rejected": -2.0240814685821533,
      "logps/chosen": -33.5056037902832,
      "logps/rejected": -31.39798355102539,
      "loss": 2.1268,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": 0.08767756074666977,
      "rewards/margins": 0.10151807963848114,
      "rewards/rejected": -0.01384051889181137,
      "step": 170
    },
    {
      "epoch": 0.47,
      "learning_rate": 3.2162026428305436e-06,
      "logits/chosen": -2.037555694580078,
      "logits/rejected": -2.0428080558776855,
      "logps/chosen": -32.268028259277344,
      "logps/rejected": -32.48134231567383,
      "loss": 2.1694,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": 0.08302746713161469,
      "rewards/margins": 0.08188783377408981,
      "rewards/rejected": 0.0011396423215046525,
      "step": 180
    },
    {
      "epoch": 0.49,
      "learning_rate": 2.996071664294641e-06,
      "logits/chosen": -2.0385537147521973,
      "logits/rejected": -2.0357697010040283,
      "logps/chosen": -31.26275062561035,
      "logps/rejected": -31.31881332397461,
      "loss": 2.411,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.0678616464138031,
      "rewards/margins": 0.06804122775793076,
      "rewards/rejected": -0.00017957761883735657,
      "step": 190
    },
    {
      "epoch": 0.52,
      "learning_rate": 2.7718537898066833e-06,
      "logits/chosen": -1.9085556268692017,
      "logits/rejected": -1.9132124185562134,
      "logps/chosen": -31.326763153076172,
      "logps/rejected": -32.80078887939453,
      "loss": 2.2795,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.0838552787899971,
      "rewards/margins": 0.08681019395589828,
      "rewards/rejected": -0.002954920055344701,
      "step": 200
    },
    {
      "epoch": 0.52,
      "eval_logits/chosen": -2.2338554859161377,
      "eval_logits/rejected": -2.2290101051330566,
      "eval_logps/chosen": -34.05450439453125,
      "eval_logps/rejected": -37.565059661865234,
      "eval_loss": 2.801168918609619,
      "eval_rewards/accuracies": 0.5278239250183105,
      "eval_rewards/chosen": -0.005985844414681196,
      "eval_rewards/margins": 0.0085463160648942,
      "eval_rewards/rejected": -0.014532160945236683,
      "eval_runtime": 145.8316,
      "eval_samples_per_second": 2.352,
      "eval_steps_per_second": 0.295,
      "step": 200
    },
    {
      "epoch": 0.55,
      "learning_rate": 2.5453962426402006e-06,
      "logits/chosen": -2.020181179046631,
      "logits/rejected": -2.030860424041748,
      "logps/chosen": -31.76279067993164,
      "logps/rejected": -33.952571868896484,
      "loss": 2.1351,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": 0.06255346536636353,
      "rewards/margins": 0.08951519429683685,
      "rewards/rejected": -0.026961728930473328,
      "step": 210
    },
    {
      "epoch": 0.57,
      "learning_rate": 2.3185646976551794e-06,
      "logits/chosen": -1.912184476852417,
      "logits/rejected": -1.926975965499878,
      "logps/chosen": -29.868118286132812,
      "logps/rejected": -31.57167625427246,
      "loss": 2.231,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.07302670180797577,
      "rewards/margins": 0.07707642018795013,
      "rewards/rejected": -0.004049715120345354,
      "step": 220
    },
    {
      "epoch": 0.6,
      "learning_rate": 2.0932279108998323e-06,
      "logits/chosen": -1.968927025794983,
      "logits/rejected": -1.972905158996582,
      "logps/chosen": -33.126705169677734,
      "logps/rejected": -31.619220733642578,
      "loss": 2.0957,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": 0.0872274860739708,
      "rewards/margins": 0.10820253938436508,
      "rewards/rejected": -0.02097504958510399,
      "step": 230
    },
    {
      "epoch": 0.62,
      "learning_rate": 1.8712423238279358e-06,
      "logits/chosen": -1.967507004737854,
      "logits/rejected": -1.9456119537353516,
      "logps/chosen": -33.865623474121094,
      "logps/rejected": -35.0605583190918,
      "loss": 1.9762,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": 0.08643803745508194,
      "rewards/margins": 0.11936646699905396,
      "rewards/rejected": -0.03292842581868172,
      "step": 240
    },
    {
      "epoch": 0.65,
      "learning_rate": 1.6544367689701824e-06,
      "logits/chosen": -2.008720874786377,
      "logits/rejected": -2.0053982734680176,
      "logps/chosen": -32.73490524291992,
      "logps/rejected": -36.2797966003418,
      "loss": 2.2614,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": 0.05704798549413681,
      "rewards/margins": 0.07351039350032806,
      "rewards/rejected": -0.016462400555610657,
      "step": 250
    },
    {
      "epoch": 0.68,
      "learning_rate": 1.4445974030621963e-06,
      "logits/chosen": -1.8761491775512695,
      "logits/rejected": -1.8737146854400635,
      "logps/chosen": -33.989051818847656,
      "logps/rejected": -35.516883850097656,
      "loss": 2.4128,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.05980368331074715,
      "rewards/margins": 0.06348178535699844,
      "rewards/rejected": -0.003678102744743228,
      "step": 260
    },
    {
      "epoch": 0.7,
      "learning_rate": 1.243452991757889e-06,
      "logits/chosen": -1.8607820272445679,
      "logits/rejected": -1.8583341836929321,
      "logps/chosen": -34.1697998046875,
      "logps/rejected": -31.77322769165039,
      "loss": 2.33,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.06833983957767487,
      "rewards/margins": 0.06890784204006195,
      "rewards/rejected": -0.0005679976311512291,
      "step": 270
    },
    {
      "epoch": 0.73,
      "learning_rate": 1.0526606671603523e-06,
      "logits/chosen": -1.9649995565414429,
      "logits/rejected": -1.954466462135315,
      "logps/chosen": -35.01842498779297,
      "logps/rejected": -31.856042861938477,
      "loss": 2.0235,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": 0.10146405547857285,
      "rewards/margins": 0.10597936809062958,
      "rewards/rejected": -0.004515302833169699,
      "step": 280
    },
    {
      "epoch": 0.75,
      "learning_rate": 8.737922755071455e-07,
      "logits/chosen": -2.059657096862793,
      "logits/rejected": -2.0446982383728027,
      "logps/chosen": -30.713714599609375,
      "logps/rejected": -32.61918258666992,
      "loss": 2.5851,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": 0.06312306225299835,
      "rewards/margins": 0.0533272810280323,
      "rewards/rejected": 0.00979578960686922,
      "step": 290
    },
    {
      "epoch": 0.78,
      "learning_rate": 7.08321427484816e-07,
      "logits/chosen": -1.9307676553726196,
      "logits/rejected": -1.9282405376434326,
      "logps/chosen": -32.467708587646484,
      "logps/rejected": -30.857818603515625,
      "loss": 1.7902,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": 0.1378202736377716,
      "rewards/margins": 0.15012334287166595,
      "rewards/rejected": -0.012303064577281475,
      "step": 300
    },
    {
      "epoch": 0.78,
      "eval_logits/chosen": -2.2293622493743896,
      "eval_logits/rejected": -2.2245147228240967,
      "eval_logps/chosen": -34.04458236694336,
      "eval_logps/rejected": -37.57244873046875,
      "eval_loss": 2.758474826812744,
      "eval_rewards/accuracies": 0.5747508406639099,
      "eval_rewards/chosen": -0.003010095562785864,
      "eval_rewards/margins": 0.013738218694925308,
      "eval_rewards/rejected": -0.01674831472337246,
      "eval_runtime": 145.5146,
      "eval_samples_per_second": 2.357,
      "eval_steps_per_second": 0.296,
      "step": 300
    },
    {
      "epoch": 0.81,
      "learning_rate": 5.576113578589035e-07,
      "logits/chosen": -1.9147694110870361,
      "logits/rejected": -1.9114938974380493,
      "logps/chosen": -31.310047149658203,
      "logps/rejected": -33.77363967895508,
      "loss": 2.1768,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.08799003809690475,
      "rewards/margins": 0.09743582457304001,
      "rewards/rejected": -0.009445784613490105,
      "step": 310
    },
    {
      "epoch": 0.83,
      "learning_rate": 4.229036944380913e-07,
      "logits/chosen": -1.965264081954956,
      "logits/rejected": -1.953029990196228,
      "logps/chosen": -34.34061050415039,
      "logps/rejected": -33.643863677978516,
      "loss": 1.9927,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": 0.07663901150226593,
      "rewards/margins": 0.10814561694860458,
      "rewards/rejected": -0.03150660917162895,
      "step": 320
    },
    {
      "epoch": 0.86,
      "learning_rate": 3.053082288996112e-07,
      "logits/chosen": -1.9999011754989624,
      "logits/rejected": -1.9984591007232666,
      "logps/chosen": -33.24303436279297,
      "logps/rejected": -32.54120635986328,
      "loss": 2.1065,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.0728277713060379,
      "rewards/margins": 0.09131507575511932,
      "rewards/rejected": -0.01848730817437172,
      "step": 330
    },
    {
      "epoch": 0.88,
      "learning_rate": 2.0579377374915805e-07,
      "logits/chosen": -2.086862802505493,
      "logits/rejected": -2.0711212158203125,
      "logps/chosen": -33.77810287475586,
      "logps/rejected": -33.093299865722656,
      "loss": 2.0709,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.12038693577051163,
      "rewards/margins": 0.1129666194319725,
      "rewards/rejected": 0.007420300040394068,
      "step": 340
    },
    {
      "epoch": 0.91,
      "learning_rate": 1.2518018074041684e-07,
      "logits/chosen": -1.9597351551055908,
      "logits/rejected": -1.9588581323623657,
      "logps/chosen": -32.87625503540039,
      "logps/rejected": -32.52130889892578,
      "loss": 1.9515,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": 0.12175308167934418,
      "rewards/margins": 0.13860100507736206,
      "rewards/rejected": -0.016847927123308182,
      "step": 350
    },
    {
      "epoch": 0.94,
      "learning_rate": 6.41315865106129e-08,
      "logits/chosen": -1.9148937463760376,
      "logits/rejected": -1.9252065420150757,
      "logps/chosen": -31.9158935546875,
      "logps/rejected": -35.31604766845703,
      "loss": 2.1343,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": 0.08664007484912872,
      "rewards/margins": 0.09482574462890625,
      "rewards/rejected": -0.008185659535229206,
      "step": 360
    },
    {
      "epoch": 0.96,
      "learning_rate": 2.3150941078050325e-08,
      "logits/chosen": -2.053968906402588,
      "logits/rejected": -2.047475576400757,
      "logps/chosen": -33.36884307861328,
      "logps/rejected": -29.23018455505371,
      "loss": 2.1339,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": 0.08493683487176895,
      "rewards/margins": 0.08606470376253128,
      "rewards/rejected": -0.0011278685415163636,
      "step": 370
    },
    {
      "epoch": 0.99,
      "learning_rate": 2.575864278703266e-09,
      "logits/chosen": -1.9135916233062744,
      "logits/rejected": -1.9158084392547607,
      "logps/chosen": -33.879058837890625,
      "logps/rejected": -30.930347442626953,
      "loss": 1.9191,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": 0.10799793899059296,
      "rewards/margins": 0.12531307339668274,
      "rewards/rejected": -0.017315123230218887,
      "step": 380
    },
    {
      "epoch": 1.0,
      "step": 385,
      "total_flos": 0.0,
      "train_loss": 2.3457992528940177,
      "train_runtime": 3250.6949,
      "train_samples_per_second": 0.947,
      "train_steps_per_second": 0.118
    }
  ],
  "logging_steps": 10,
  "max_steps": 385,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 100,
  "total_flos": 0.0,
  "train_batch_size": 4,
  "trial_name": null,
  "trial_params": null
}