aftonposten-6b-align-scan / trainer_state.json
hugodk-sch's picture
Model save
ae8768e verified
raw
history blame
21.2 kB
{
"best_metric": null,
"best_model_checkpoint": null,
"epoch": 1.0,
"eval_steps": 100,
"global_step": 385,
"is_hyper_param_search": false,
"is_local_process_zero": true,
"is_world_process_zero": true,
"log_history": [
{
"epoch": 0.0,
"learning_rate": 1.282051282051282e-07,
"logits/chosen": -1.7278180122375488,
"logits/rejected": -1.7377450466156006,
"logps/chosen": -29.553977966308594,
"logps/rejected": -42.813133239746094,
"loss": 0.3086,
"rewards/accuracies": 0.0,
"rewards/chosen": 0.0,
"rewards/margins": 0.0,
"rewards/rejected": 0.0,
"step": 1
},
{
"epoch": 0.03,
"learning_rate": 1.282051282051282e-06,
"logits/chosen": -1.8665881156921387,
"logits/rejected": -1.8709055185317993,
"logps/chosen": -36.99662399291992,
"logps/rejected": -33.65571594238281,
"loss": 0.3052,
"rewards/accuracies": 0.5416666865348816,
"rewards/chosen": 0.008980684913694859,
"rewards/margins": 0.032059140503406525,
"rewards/rejected": -0.02307845838367939,
"step": 10
},
{
"epoch": 0.05,
"learning_rate": 2.564102564102564e-06,
"logits/chosen": -1.9977455139160156,
"logits/rejected": -2.000382423400879,
"logps/chosen": -29.642925262451172,
"logps/rejected": -29.056737899780273,
"loss": 0.3525,
"rewards/accuracies": 0.42500001192092896,
"rewards/chosen": -0.0006573178106918931,
"rewards/margins": -0.00669272243976593,
"rewards/rejected": 0.006035405211150646,
"step": 20
},
{
"epoch": 0.08,
"learning_rate": 3.846153846153847e-06,
"logits/chosen": -1.9204607009887695,
"logits/rejected": -1.9177772998809814,
"logps/chosen": -31.42336654663086,
"logps/rejected": -33.22785568237305,
"loss": 0.3603,
"rewards/accuracies": 0.48750001192092896,
"rewards/chosen": -0.006546213291585445,
"rewards/margins": 0.0019128695130348206,
"rewards/rejected": -0.00845908559858799,
"step": 30
},
{
"epoch": 0.1,
"learning_rate": 4.999896948438434e-06,
"logits/chosen": -2.0172362327575684,
"logits/rejected": -2.008507251739502,
"logps/chosen": -32.56964874267578,
"logps/rejected": -32.50572967529297,
"loss": 0.3558,
"rewards/accuracies": 0.5,
"rewards/chosen": 0.006439635064452887,
"rewards/margins": 0.00019515231542754918,
"rewards/rejected": 0.006244482938200235,
"step": 40
},
{
"epoch": 0.13,
"learning_rate": 4.987541037542187e-06,
"logits/chosen": -1.8618619441986084,
"logits/rejected": -1.8510783910751343,
"logps/chosen": -33.56026077270508,
"logps/rejected": -35.45254898071289,
"loss": 0.3691,
"rewards/accuracies": 0.4000000059604645,
"rewards/chosen": -0.0025013976264744997,
"rewards/margins": -0.002315213903784752,
"rewards/rejected": -0.00018618404283188283,
"step": 50
},
{
"epoch": 0.16,
"learning_rate": 4.954691471941119e-06,
"logits/chosen": -1.9393202066421509,
"logits/rejected": -1.9412600994110107,
"logps/chosen": -32.57838439941406,
"logps/rejected": -33.215576171875,
"loss": 0.3063,
"rewards/accuracies": 0.612500011920929,
"rewards/chosen": 0.01962057128548622,
"rewards/margins": 0.05940054729580879,
"rewards/rejected": -0.03977997973561287,
"step": 60
},
{
"epoch": 0.18,
"learning_rate": 4.901618883413549e-06,
"logits/chosen": -2.0718436241149902,
"logits/rejected": -2.0768017768859863,
"logps/chosen": -33.97806167602539,
"logps/rejected": -36.63082504272461,
"loss": 0.4257,
"rewards/accuracies": 0.5,
"rewards/chosen": 0.00938049890100956,
"rewards/margins": 0.06513925641775131,
"rewards/rejected": -0.0557587556540966,
"step": 70
},
{
"epoch": 0.21,
"learning_rate": 4.828760511501322e-06,
"logits/chosen": -1.9351739883422852,
"logits/rejected": -1.9383188486099243,
"logps/chosen": -34.33073043823242,
"logps/rejected": -34.61904525756836,
"loss": 0.2902,
"rewards/accuracies": 0.625,
"rewards/chosen": 0.08736880123615265,
"rewards/margins": 0.11134655773639679,
"rewards/rejected": -0.023977745324373245,
"step": 80
},
{
"epoch": 0.23,
"learning_rate": 4.7367166013034295e-06,
"logits/chosen": -1.944392204284668,
"logits/rejected": -1.9489190578460693,
"logps/chosen": -32.419586181640625,
"logps/rejected": -32.3698844909668,
"loss": 0.3611,
"rewards/accuracies": 0.574999988079071,
"rewards/chosen": 0.043051257729530334,
"rewards/margins": 0.052930813282728195,
"rewards/rejected": -0.009879561141133308,
"step": 90
},
{
"epoch": 0.26,
"learning_rate": 4.626245458345211e-06,
"logits/chosen": -2.0419769287109375,
"logits/rejected": -2.0399627685546875,
"logps/chosen": -32.174407958984375,
"logps/rejected": -31.26608657836914,
"loss": 0.3038,
"rewards/accuracies": 0.5874999761581421,
"rewards/chosen": 0.059785228222608566,
"rewards/margins": 0.05967242643237114,
"rewards/rejected": 0.00011279433965682983,
"step": 100
},
{
"epoch": 0.26,
"eval_logits/chosen": -2.2372143268585205,
"eval_logits/rejected": -2.2323503494262695,
"eval_logps/chosen": -34.01276779174805,
"eval_logps/rejected": -37.51152420043945,
"eval_loss": 0.3511974811553955,
"eval_rewards/accuracies": 0.5423588156700134,
"eval_rewards/chosen": 0.01960929110646248,
"eval_rewards/margins": 0.01502405758947134,
"eval_rewards/rejected": 0.004585230257362127,
"eval_runtime": 145.9032,
"eval_samples_per_second": 2.351,
"eval_steps_per_second": 0.295,
"step": 100
},
{
"epoch": 0.29,
"learning_rate": 4.498257201263691e-06,
"logits/chosen": -1.998891830444336,
"logits/rejected": -1.996492624282837,
"logps/chosen": -33.14598083496094,
"logps/rejected": -34.020729064941406,
"loss": 0.4689,
"rewards/accuracies": 0.6000000238418579,
"rewards/chosen": 0.08805312216281891,
"rewards/margins": 0.08027410507202148,
"rewards/rejected": 0.007779018487781286,
"step": 110
},
{
"epoch": 0.31,
"learning_rate": 4.353806263777678e-06,
"logits/chosen": -2.010932445526123,
"logits/rejected": -2.0025696754455566,
"logps/chosen": -32.37172317504883,
"logps/rejected": -32.118797302246094,
"loss": 0.4465,
"rewards/accuracies": 0.574999988079071,
"rewards/chosen": 0.06586603820323944,
"rewards/margins": 0.02742874063551426,
"rewards/rejected": 0.03843729570508003,
"step": 120
},
{
"epoch": 0.34,
"learning_rate": 4.1940827077152755e-06,
"logits/chosen": -2.0387539863586426,
"logits/rejected": -2.030724287033081,
"logps/chosen": -30.41655921936035,
"logps/rejected": -32.060333251953125,
"loss": 0.3844,
"rewards/accuracies": 0.5375000238418579,
"rewards/chosen": 0.05324209854006767,
"rewards/margins": 0.05983499437570572,
"rewards/rejected": -0.00659290561452508,
"step": 130
},
{
"epoch": 0.36,
"learning_rate": 4.0204024186666215e-06,
"logits/chosen": -1.967858076095581,
"logits/rejected": -1.9781148433685303,
"logps/chosen": -31.223413467407227,
"logps/rejected": -32.55517578125,
"loss": 0.3904,
"rewards/accuracies": 0.6875,
"rewards/chosen": 0.1766217201948166,
"rewards/margins": 0.1948881596326828,
"rewards/rejected": -0.01826643943786621,
"step": 140
},
{
"epoch": 0.39,
"learning_rate": 3.834196265035119e-06,
"logits/chosen": -1.880910873413086,
"logits/rejected": -1.8820507526397705,
"logps/chosen": -34.01464080810547,
"logps/rejected": -34.783546447753906,
"loss": 0.3871,
"rewards/accuracies": 0.612500011920929,
"rewards/chosen": 0.160243421792984,
"rewards/margins": 0.1839032918214798,
"rewards/rejected": -0.023659853264689445,
"step": 150
},
{
"epoch": 0.42,
"learning_rate": 3.636998309800573e-06,
"logits/chosen": -1.933895468711853,
"logits/rejected": -1.9304730892181396,
"logps/chosen": -36.02853775024414,
"logps/rejected": -32.699058532714844,
"loss": 0.2729,
"rewards/accuracies": 0.637499988079071,
"rewards/chosen": 0.1288261115550995,
"rewards/margins": 0.10750452429056168,
"rewards/rejected": 0.021321602165699005,
"step": 160
},
{
"epoch": 0.44,
"learning_rate": 3.4304331721118078e-06,
"logits/chosen": -2.0341715812683105,
"logits/rejected": -2.0267820358276367,
"logps/chosen": -33.55347442626953,
"logps/rejected": -31.3526554107666,
"loss": 0.294,
"rewards/accuracies": 0.6625000238418579,
"rewards/chosen": 0.21994857490062714,
"rewards/margins": 0.22067300975322723,
"rewards/rejected": -0.0007244400912895799,
"step": 170
},
{
"epoch": 0.47,
"learning_rate": 3.2162026428305436e-06,
"logits/chosen": -2.0400891304016113,
"logits/rejected": -2.045360565185547,
"logps/chosen": -32.370338439941406,
"logps/rejected": -32.4719123840332,
"loss": 0.2829,
"rewards/accuracies": 0.6625000238418579,
"rewards/chosen": 0.15700635313987732,
"rewards/margins": 0.145101398229599,
"rewards/rejected": 0.011904975399374962,
"step": 180
},
{
"epoch": 0.49,
"learning_rate": 2.996071664294641e-06,
"logits/chosen": -2.041393280029297,
"logits/rejected": -2.038623809814453,
"logps/chosen": -31.328174591064453,
"logps/rejected": -31.316492080688477,
"loss": 0.3044,
"rewards/accuracies": 0.699999988079071,
"rewards/chosen": 0.14470075070858002,
"rewards/margins": 0.1431477963924408,
"rewards/rejected": 0.0015529401134699583,
"step": 190
},
{
"epoch": 0.52,
"learning_rate": 2.7718537898066833e-06,
"logits/chosen": -1.9122215509414673,
"logits/rejected": -1.9168663024902344,
"logps/chosen": -31.424020767211914,
"logps/rejected": -32.784080505371094,
"loss": 0.3157,
"rewards/accuracies": 0.699999988079071,
"rewards/chosen": 0.16403506696224213,
"rewards/margins": 0.1578713059425354,
"rewards/rejected": 0.0061637843027710915,
"step": 200
},
{
"epoch": 0.52,
"eval_logits/chosen": -2.237051010131836,
"eval_logits/rejected": -2.2321863174438477,
"eval_logps/chosen": -34.018070220947266,
"eval_logps/rejected": -37.51838684082031,
"eval_loss": 0.3716273605823517,
"eval_rewards/accuracies": 0.5245016813278198,
"eval_rewards/chosen": 0.014834923669695854,
"eval_rewards/margins": 0.016425320878624916,
"eval_rewards/rejected": -0.0015903981402516365,
"eval_runtime": 145.5151,
"eval_samples_per_second": 2.357,
"eval_steps_per_second": 0.296,
"step": 200
},
{
"epoch": 0.55,
"learning_rate": 2.5453962426402006e-06,
"logits/chosen": -2.023789882659912,
"logits/rejected": -2.034484386444092,
"logps/chosen": -31.767370223999023,
"logps/rejected": -33.890621185302734,
"loss": 0.275,
"rewards/accuracies": 0.675000011920929,
"rewards/chosen": 0.18354059755802155,
"rewards/margins": 0.20867136120796204,
"rewards/rejected": -0.025130782276391983,
"step": 210
},
{
"epoch": 0.57,
"learning_rate": 2.3185646976551794e-06,
"logits/chosen": -1.916685700416565,
"logits/rejected": -1.9314892292022705,
"logps/chosen": -29.956628799438477,
"logps/rejected": -31.564035415649414,
"loss": 0.2958,
"rewards/accuracies": 0.6625000238418579,
"rewards/chosen": 0.13942097127437592,
"rewards/margins": 0.14469322562217712,
"rewards/rejected": -0.005272268317639828,
"step": 220
},
{
"epoch": 0.6,
"learning_rate": 2.0932279108998323e-06,
"logits/chosen": -1.9737945795059204,
"logits/rejected": -1.9777710437774658,
"logps/chosen": -33.19129180908203,
"logps/rejected": -31.5566463470459,
"loss": 0.287,
"rewards/accuracies": 0.6625000238418579,
"rewards/chosen": 0.20355579257011414,
"rewards/margins": 0.21016716957092285,
"rewards/rejected": -0.0066113718785345554,
"step": 230
},
{
"epoch": 0.62,
"learning_rate": 1.8712423238279358e-06,
"logits/chosen": -1.9743419885635376,
"logits/rejected": -1.9523779153823853,
"logps/chosen": -33.9401969909668,
"logps/rejected": -35.008758544921875,
"loss": 0.3185,
"rewards/accuracies": 0.6625000238418579,
"rewards/chosen": 0.1921955645084381,
"rewards/margins": 0.24436470866203308,
"rewards/rejected": -0.052169155329465866,
"step": 240
},
{
"epoch": 0.65,
"learning_rate": 1.6544367689701824e-06,
"logits/chosen": -2.0161709785461426,
"logits/rejected": -2.0128414630889893,
"logps/chosen": -32.74829864501953,
"logps/rejected": -36.236392974853516,
"loss": 0.2847,
"rewards/accuracies": 0.7250000238418579,
"rewards/chosen": 0.15909257531166077,
"rewards/margins": 0.1694144755601883,
"rewards/rejected": -0.010321905836462975,
"step": 250
},
{
"epoch": 0.68,
"learning_rate": 1.4445974030621963e-06,
"logits/chosen": -1.8847742080688477,
"logits/rejected": -1.8823268413543701,
"logps/chosen": -34.01182174682617,
"logps/rejected": -35.481346130371094,
"loss": 0.3073,
"rewards/accuracies": 0.612500011920929,
"rewards/chosen": 0.1589193046092987,
"rewards/margins": 0.13797220587730408,
"rewards/rejected": 0.020947108045220375,
"step": 260
},
{
"epoch": 0.7,
"learning_rate": 1.243452991757889e-06,
"logits/chosen": -1.8693536520004272,
"logits/rejected": -1.866838812828064,
"logps/chosen": -34.20549392700195,
"logps/rejected": -31.726673126220703,
"loss": 0.2733,
"rewards/accuracies": 0.6875,
"rewards/chosen": 0.17289286851882935,
"rewards/margins": 0.1327010840177536,
"rewards/rejected": 0.040191780775785446,
"step": 270
},
{
"epoch": 0.73,
"learning_rate": 1.0526606671603523e-06,
"logits/chosen": -1.9730831384658813,
"logits/rejected": -1.962480902671814,
"logps/chosen": -35.06049728393555,
"logps/rejected": -31.792781829833984,
"loss": 0.2523,
"rewards/accuracies": 0.75,
"rewards/chosen": 0.2665289342403412,
"rewards/margins": 0.22314274311065674,
"rewards/rejected": 0.04338619112968445,
"step": 280
},
{
"epoch": 0.75,
"learning_rate": 8.737922755071455e-07,
"logits/chosen": -2.0680813789367676,
"logits/rejected": -2.053079128265381,
"logps/chosen": -30.738479614257812,
"logps/rejected": -32.61243438720703,
"loss": 0.3194,
"rewards/accuracies": 0.6000000238418579,
"rewards/chosen": 0.16707859933376312,
"rewards/margins": 0.13161785900592804,
"rewards/rejected": 0.035460732877254486,
"step": 290
},
{
"epoch": 0.78,
"learning_rate": 7.08321427484816e-07,
"logits/chosen": -1.9398882389068604,
"logits/rejected": -1.9373395442962646,
"logps/chosen": -32.6181640625,
"logps/rejected": -30.843700408935547,
"loss": 0.2156,
"rewards/accuracies": 0.75,
"rewards/chosen": 0.2780519425868988,
"rewards/margins": 0.3022567331790924,
"rewards/rejected": -0.024204757064580917,
"step": 300
},
{
"epoch": 0.78,
"eval_logits/chosen": -2.2364187240600586,
"eval_logits/rejected": -2.2315518856048584,
"eval_logps/chosen": -34.01428985595703,
"eval_logps/rejected": -37.496952056884766,
"eval_loss": 0.38450533151626587,
"eval_rewards/accuracies": 0.49335551261901855,
"eval_rewards/chosen": 0.018236981704831123,
"eval_rewards/margins": 0.0005384809919632971,
"eval_rewards/rejected": 0.01769850216805935,
"eval_runtime": 145.7485,
"eval_samples_per_second": 2.353,
"eval_steps_per_second": 0.295,
"step": 300
},
{
"epoch": 0.81,
"learning_rate": 5.576113578589035e-07,
"logits/chosen": -1.922579050064087,
"logits/rejected": -1.9193273782730103,
"logps/chosen": -31.345911026000977,
"logps/rejected": -33.72126007080078,
"loss": 0.2917,
"rewards/accuracies": 0.699999988079071,
"rewards/chosen": 0.23169513046741486,
"rewards/margins": 0.21289470791816711,
"rewards/rejected": 0.018800420686602592,
"step": 310
},
{
"epoch": 0.83,
"learning_rate": 4.229036944380913e-07,
"logits/chosen": -1.9754743576049805,
"logits/rejected": -1.9631853103637695,
"logps/chosen": -34.408077239990234,
"logps/rejected": -33.58232879638672,
"loss": 0.2394,
"rewards/accuracies": 0.675000011920929,
"rewards/chosen": 0.1691979616880417,
"rewards/margins": 0.20833876729011536,
"rewards/rejected": -0.03914082050323486,
"step": 320
},
{
"epoch": 0.86,
"learning_rate": 3.053082288996112e-07,
"logits/chosen": -2.0105607509613037,
"logits/rejected": -2.009115219116211,
"logps/chosen": -33.31591033935547,
"logps/rejected": -32.47368621826172,
"loss": 0.2769,
"rewards/accuracies": 0.6499999761581421,
"rewards/chosen": 0.15289874374866486,
"rewards/margins": 0.14759239554405212,
"rewards/rejected": 0.005306343547999859,
"step": 330
},
{
"epoch": 0.88,
"learning_rate": 2.0579377374915805e-07,
"logits/chosen": -2.096872091293335,
"logits/rejected": -2.0811073780059814,
"logps/chosen": -33.87510681152344,
"logps/rejected": -33.06427764892578,
"loss": 0.2792,
"rewards/accuracies": 0.737500011920929,
"rewards/chosen": 0.27385497093200684,
"rewards/margins": 0.2254684418439865,
"rewards/rejected": 0.04838654398918152,
"step": 340
},
{
"epoch": 0.91,
"learning_rate": 1.2518018074041684e-07,
"logits/chosen": -1.969496488571167,
"logits/rejected": -1.9685735702514648,
"logps/chosen": -32.98945999145508,
"logps/rejected": -32.4643440246582,
"loss": 0.2958,
"rewards/accuracies": 0.737500011920929,
"rewards/chosen": 0.26337358355522156,
"rewards/margins": 0.26264840364456177,
"rewards/rejected": 0.0007252089562825859,
"step": 350
},
{
"epoch": 0.94,
"learning_rate": 6.41315865106129e-08,
"logits/chosen": -1.9255645275115967,
"logits/rejected": -1.9359004497528076,
"logps/chosen": -32.013362884521484,
"logps/rejected": -35.26326370239258,
"loss": 0.3425,
"rewards/accuracies": 0.637499988079071,
"rewards/chosen": 0.1721937507390976,
"rewards/margins": 0.14924712479114532,
"rewards/rejected": 0.022946633398532867,
"step": 360
},
{
"epoch": 0.96,
"learning_rate": 2.3150941078050325e-08,
"logits/chosen": -2.0643956661224365,
"logits/rejected": -2.057886838912964,
"logps/chosen": -33.48772430419922,
"logps/rejected": -29.191638946533203,
"loss": 0.298,
"rewards/accuracies": 0.625,
"rewards/chosen": 0.14782151579856873,
"rewards/margins": 0.11651048809289932,
"rewards/rejected": 0.03131101652979851,
"step": 370
},
{
"epoch": 0.99,
"learning_rate": 2.575864278703266e-09,
"logits/chosen": -1.9235725402832031,
"logits/rejected": -1.9257465600967407,
"logps/chosen": -33.965919494628906,
"logps/rejected": -30.839218139648438,
"loss": 0.2616,
"rewards/accuracies": 0.699999988079071,
"rewards/chosen": 0.24581687152385712,
"rewards/margins": 0.2157471626996994,
"rewards/rejected": 0.03006969951093197,
"step": 380
},
{
"epoch": 1.0,
"step": 385,
"total_flos": 0.0,
"train_loss": 0.31703355428460356,
"train_runtime": 3251.5033,
"train_samples_per_second": 0.947,
"train_steps_per_second": 0.118
}
],
"logging_steps": 10,
"max_steps": 385,
"num_input_tokens_seen": 0,
"num_train_epochs": 1,
"save_steps": 100,
"total_flos": 0.0,
"train_batch_size": 4,
"trial_name": null,
"trial_params": null
}