Model save

README.md

@@ -1,13 +1,10 @@
 ---
 library_name: peft
 tags:
-- alignment-handbook
 - trl
 - dpo
 - generated_from_trainer
 base_model: NbAiLab/nb-gpt-j-6B-v2
-datasets:
-- hugodk-sch/aftonposten_title_prefs
 model-index:
 - name: aftonposten-6b-align-scan
   results: []
@@ -18,17 +15,17 @@ should probably proofread and complete it, then remove this comment. -->
 
 # aftonposten-6b-align-scan
 
-This model is a fine-tuned version of [data/ap-gpt-j-6b-sft-qlora-04-08](https://huggingface.co/data/ap-gpt-j-6b-sft-qlora-04-08) on the hugodk-sch/aftonposten_title_prefs dataset.
+This model is a fine-tuned version of [NbAiLab/nb-gpt-j-6B-v2](https://huggingface.co/NbAiLab/nb-gpt-j-6B-v2) on an unknown dataset.
 It achieves the following results on the evaluation set:
-- Loss: 0.7598
-- Rewards/chosen: -0.0165
-- Rewards/rejected: -0.0300
-- Rewards/accuracies: 0.5307
-- Rewards/margins: 0.0135
-- Logps/rejected: -37.5666
-- Logps/chosen: -34.0620
-- Logits/rejected: -2.2283
-- Logits/chosen: -2.2332
+- Loss: 0.9732
+- Rewards/chosen: -0.0459
+- Rewards/rejected: -0.0732
+- Rewards/accuracies: 0.5071
+- Rewards/margins: 0.0273
+- Logps/rejected: -37.6386
+- Logps/chosen: -34.1110
+- Logits/rejected: -2.2194
+- Logits/chosen: -2.2243
 
 ## Model description
 
@@ -63,9 +60,9 @@ The following hyperparameters were used during training:
 
 | Training Loss | Epoch | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
 |:-------------:|:-----:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
-| 0.5903        | 0.26  | 100  | 0.7450          | 0.0123         | 0.0047           | 0.5241             | 0.0075          | -37.5087       | -34.0141     | -2.2318         | -2.2366       |
-| 0.59          | 0.52  | 200  | 0.7500          | -0.0208        | -0.0352          | 0.5365             | 0.0144          | -37.5752       | -34.0691     | -2.2284         | -2.2332       |
-| 0.3871        | 0.78  | 300  | 0.7655          | -0.0166        | -0.0295          | 0.5071             | 0.0129          | -37.5657       | -34.0622     | -2.2285         | -2.2333       |
+| 0.9204        | 0.26  | 100  | 0.9816          | -0.0008        | -0.0193          | 0.5395             | 0.0185          | -37.5487       | -34.0358     | -2.2268         | -2.2317       |
+| 0.8045        | 0.52  | 200  | 0.9791          | -0.0166        | -0.0378          | 0.5361             | 0.0211          | -37.5796       | -34.0623     | -2.2238         | -2.2287       |
+| 0.7186        | 0.78  | 300  | 0.9732          | -0.0459        | -0.0732          | 0.5071             | 0.0273          | -37.6386       | -34.1110     | -2.2194         | -2.2243       |
 
 
 ### Framework versions

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:553a449863db44a3e8ece8cb18605a91eee4983797294114ada2e403b5156b43
+oid sha256:88c73d3c9a8da1c5a11fcd11e427c053fbe6f72ad35d6d3451cb210efa020e6e
 size 176183216

all_results.json

@@ -1,20 +1,7 @@
 {
     "epoch": 1.0,
-    "eval_logits/chosen": -2.2331926822662354,
-    "eval_logits/rejected": -2.2283315658569336,
-    "eval_logps/chosen": -34.061973571777344,
-    "eval_logps/rejected": -37.566593170166016,
-    "eval_loss": 0.7597638368606567,
-    "eval_rewards/accuracies": 0.5307309031486511,
-    "eval_rewards/chosen": -0.016451681032776833,
-    "eval_rewards/margins": 0.013532036915421486,
-    "eval_rewards/rejected": -0.029983719810843468,
-    "eval_runtime": 145.5395,
-    "eval_samples": 343,
-    "eval_samples_per_second": 2.357,
-    "eval_steps_per_second": 0.295,
-    "train_loss": 0.5965218513042896,
-    "train_runtime": 3252.2803,
+    "train_loss": 0.859264765157328,
+    "train_runtime": 3251.7628,
     "train_samples": 3079,
     "train_samples_per_second": 0.947,
     "train_steps_per_second": 0.118

train_results.json

@@ -1,7 +1,7 @@
 {
     "epoch": 1.0,
-    "train_loss": 0.5965218513042896,
-    "train_runtime": 3252.2803,
+    "train_loss": 0.859264765157328,
+    "train_runtime": 3251.7628,
     "train_samples": 3079,
     "train_samples_per_second": 0.947,
     "train_steps_per_second": 0.118

trainer_state.json

@@ -15,7 +15,7 @@
       "logits/rejected": -1.7377450466156006,
       "logps/chosen": -29.553977966308594,
       "logps/rejected": -42.813133239746094,
-      "loss": 0.6944,
+      "loss": 1.0,
       "rewards/accuracies": 0.0,
       "rewards/chosen": 0.0,
       "rewards/margins": 0.0,
@@ -25,589 +25,589 @@
     {
       "epoch": 0.03,
       "learning_rate": 1.282051282051282e-06,
-      "logits/chosen": -1.8664441108703613,
-      "logits/rejected": -1.8707667589187622,
-      "logps/chosen": -36.99871826171875,
-      "logps/rejected": -33.66032028198242,
-      "loss": 0.6698,
-      "rewards/accuracies": 0.5138888955116272,
-      "rewards/chosen": 0.0047301375307142735,
-      "rewards/margins": 0.022878725081682205,
-      "rewards/rejected": -0.018148591741919518,
+      "logits/chosen": -1.866829752922058,
+      "logits/rejected": -1.8711390495300293,
+      "logps/chosen": -36.97040557861328,
+      "logps/rejected": -33.66280746459961,
+      "loss": 0.9586,
+      "rewards/accuracies": 0.5972222089767456,
+      "rewards/chosen": 0.021715592592954636,
+      "rewards/margins": 0.041353289037942886,
+      "rewards/rejected": -0.01963769644498825,
       "step": 10
     },
     {
       "epoch": 0.05,
       "learning_rate": 2.564102564102564e-06,
-      "logits/chosen": -1.9972795248031616,
-      "logits/rejected": -1.999922513961792,
-      "logps/chosen": -29.64887046813965,
-      "logps/rejected": -29.054330825805664,
-      "loss": 0.756,
-      "rewards/accuracies": 0.38749998807907104,
-      "rewards/chosen": -0.0040070666000247,
-      "rewards/margins": -0.009474089369177818,
-      "rewards/rejected": 0.0054670218378305435,
+      "logits/chosen": -1.9976301193237305,
+      "logits/rejected": -2.0002708435058594,
+      "logps/chosen": -29.64252281188965,
+      "logps/rejected": -29.05857276916504,
+      "loss": 1.0031,
+      "rewards/accuracies": 0.4749999940395355,
+      "rewards/chosen": -0.00019791116937994957,
+      "rewards/margins": -0.003119796048849821,
+      "rewards/rejected": 0.0029218837153166533,
       "step": 20
     },
     {
       "epoch": 0.08,
       "learning_rate": 3.846153846153847e-06,
-      "logits/chosen": -1.9204769134521484,
-      "logits/rejected": -1.9177768230438232,
-      "logps/chosen": -31.404077529907227,
-      "logps/rejected": -33.22195053100586,
-      "loss": 0.7112,
-      "rewards/accuracies": 0.550000011920929,
-      "rewards/chosen": 0.007212291471660137,
-      "rewards/margins": 0.009304212406277657,
-      "rewards/rejected": -0.0020919169764965773,
+      "logits/chosen": -1.9210872650146484,
+      "logits/rejected": -1.918402910232544,
+      "logps/chosen": -31.397838592529297,
+      "logps/rejected": -33.22719192504883,
+      "loss": 0.9838,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": 0.010952453128993511,
+      "rewards/margins": 0.01619068905711174,
+      "rewards/rejected": -0.005238235928118229,
       "step": 30
     },
     {
       "epoch": 0.1,
       "learning_rate": 4.999896948438434e-06,
-      "logits/chosen": -2.0172927379608154,
-      "logits/rejected": -2.008545398712158,
-      "logps/chosen": -32.57653045654297,
-      "logps/rejected": -32.520172119140625,
-      "loss": 0.7334,
-      "rewards/accuracies": 0.5,
-      "rewards/chosen": 0.00016416943981312215,
-      "rewards/margins": 0.004665405489504337,
-      "rewards/rejected": -0.004501235671341419,
+      "logits/chosen": -2.017606258392334,
+      "logits/rejected": -2.0088634490966797,
+      "logps/chosen": -32.566612243652344,
+      "logps/rejected": -32.52539825439453,
+      "loss": 0.9862,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": 0.006120447069406509,
+      "rewards/margins": 0.013754432089626789,
+      "rewards/rejected": -0.007633985485881567,
       "step": 40
     },
     {
       "epoch": 0.13,
       "learning_rate": 4.987541037542187e-06,
-      "logits/chosen": -1.8620569705963135,
-      "logits/rejected": -1.8512712717056274,
-      "logps/chosen": -33.55234146118164,
-      "logps/rejected": -35.454689025878906,
-      "loss": 0.7458,
-      "rewards/accuracies": 0.4749999940395355,
-      "rewards/chosen": 0.003082751529291272,
-      "rewards/margins": 0.004490534774959087,
-      "rewards/rejected": -0.0014077809173613787,
+      "logits/chosen": -1.8625805377960205,
+      "logits/rejected": -1.8518139123916626,
+      "logps/chosen": -33.56818771362305,
+      "logps/rejected": -35.4713020324707,
+      "loss": 0.9951,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.0064262161031365395,
+      "rewards/margins": 0.004948456306010485,
+      "rewards/rejected": -0.011374671943485737,
       "step": 50
     },
     {
       "epoch": 0.16,
       "learning_rate": 4.954691471941119e-06,
-      "logits/chosen": -1.940212607383728,
-      "logits/rejected": -1.9421584606170654,
-      "logps/chosen": -32.56269454956055,
-      "logps/rejected": -33.21684265136719,
-      "loss": 0.6785,
-      "rewards/accuracies": 0.574999988079071,
-      "rewards/chosen": 0.022492902353405952,
-      "rewards/margins": 0.04977109283208847,
-      "rewards/rejected": -0.027278194203972816,
+      "logits/chosen": -1.9413652420043945,
+      "logits/rejected": -1.9432977437973022,
+      "logps/chosen": -32.551727294921875,
+      "logps/rejected": -33.218013763427734,
+      "loss": 0.9525,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": 0.02907358668744564,
+      "rewards/margins": 0.05705299973487854,
+      "rewards/rejected": -0.02797941491007805,
       "step": 60
     },
     {
       "epoch": 0.18,
       "learning_rate": 4.901618883413549e-06,
-      "logits/chosen": -2.0728747844696045,
-      "logits/rejected": -2.0778398513793945,
-      "logps/chosen": -33.98863983154297,
-      "logps/rejected": -36.62779235839844,
-      "loss": 0.7641,
-      "rewards/accuracies": 0.5,
-      "rewards/chosen": -9.316801879322156e-05,
-      "rewards/margins": 0.035260457545518875,
-      "rewards/rejected": -0.035353630781173706,
+      "logits/chosen": -2.0722384452819824,
+      "logits/rejected": -2.0772085189819336,
+      "logps/chosen": -33.995296478271484,
+      "logps/rejected": -36.63286590576172,
+      "loss": 0.9657,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.004085049033164978,
+      "rewards/margins": 0.034312374889850616,
+      "rewards/rejected": -0.038397423923015594,
       "step": 70
     },
     {
       "epoch": 0.21,
       "learning_rate": 4.828760511501322e-06,
-      "logits/chosen": -1.935132384300232,
-      "logits/rejected": -1.9382661581039429,
-      "logps/chosen": -34.321903228759766,
-      "logps/rejected": -34.638282775878906,
-      "loss": 0.6095,
-      "rewards/accuracies": 0.612500011920929,
-      "rewards/chosen": 0.06354494392871857,
-      "rewards/margins": 0.09106875956058502,
-      "rewards/rejected": -0.027523810043931007,
+      "logits/chosen": -1.9325292110443115,
+      "logits/rejected": -1.93563973903656,
+      "logps/chosen": -34.35693359375,
+      "logps/rejected": -34.67926788330078,
+      "loss": 0.9069,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.042526502162218094,
+      "rewards/margins": 0.09463920444250107,
+      "rewards/rejected": -0.05211270600557327,
       "step": 80
     },
     {
       "epoch": 0.23,
       "learning_rate": 4.7367166013034295e-06,
-      "logits/chosen": -1.9444854259490967,
-      "logits/rejected": -1.9490203857421875,
-      "logps/chosen": -32.41173553466797,
-      "logps/rejected": -32.337852478027344,
-      "loss": 0.7499,
-      "rewards/accuracies": 0.5874999761581421,
-      "rewards/chosen": 0.033411428332328796,
-      "rewards/margins": 0.020780622959136963,
-      "rewards/rejected": 0.012630810961127281,
+      "logits/chosen": -1.9406629800796509,
+      "logits/rejected": -1.9451707601547241,
+      "logps/chosen": -32.38554000854492,
+      "logps/rejected": -32.36783218383789,
+      "loss": 0.9455,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": 0.04912734776735306,
+      "rewards/margins": 0.05448007583618164,
+      "rewards/rejected": -0.005352728068828583,
       "step": 90
     },
     {
       "epoch": 0.26,
       "learning_rate": 4.626245458345211e-06,
-      "logits/chosen": -2.042165994644165,
-      "logits/rejected": -2.0401768684387207,
-      "logps/chosen": -32.13690185546875,
-      "logps/rejected": -31.2872314453125,
-      "loss": 0.5903,
-      "rewards/accuracies": 0.6625000238418579,
-      "rewards/chosen": 0.06236031651496887,
-      "rewards/margins": 0.0749734491109848,
-      "rewards/rejected": -0.012613129802048206,
+      "logits/chosen": -2.037841320037842,
+      "logits/rejected": -2.0358526706695557,
+      "logps/chosen": -32.14287185668945,
+      "logps/rejected": -31.300832748413086,
+      "loss": 0.9204,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": 0.058778904378414154,
+      "rewards/margins": 0.07955379039049149,
+      "rewards/rejected": -0.02077488601207733,
       "step": 100
     },
     {
       "epoch": 0.26,
-      "eval_logits/chosen": -2.2366247177124023,
-      "eval_logits/rejected": -2.231769323348999,
-      "eval_logps/chosen": -34.01411819458008,
-      "eval_logps/rejected": -37.50873947143555,
-      "eval_loss": 0.7449910640716553,
-      "eval_rewards/accuracies": 0.5240863561630249,
-      "eval_rewards/chosen": 0.01226096786558628,
-      "eval_rewards/margins": 0.007533104624599218,
-      "eval_rewards/rejected": 0.0047278618440032005,
-      "eval_runtime": 145.783,
-      "eval_samples_per_second": 2.353,
+      "eval_logits/chosen": -2.2316644191741943,
+      "eval_logits/rejected": -2.2268166542053223,
+      "eval_logps/chosen": -34.03582000732422,
+      "eval_logps/rejected": -37.548709869384766,
+      "eval_loss": 0.9816102385520935,
+      "eval_rewards/accuracies": 0.5394518375396729,
+      "eval_rewards/chosen": -0.0007606110884808004,
+      "eval_rewards/margins": 0.018493397161364555,
+      "eval_rewards/rejected": -0.019254004582762718,
+      "eval_runtime": 145.9929,
+      "eval_samples_per_second": 2.349,
       "eval_steps_per_second": 0.295,
       "step": 100
     },
     {
       "epoch": 0.29,
       "learning_rate": 4.498257201263691e-06,
-      "logits/chosen": -1.9979403018951416,
-      "logits/rejected": -1.9955459833145142,
-      "logps/chosen": -33.12257766723633,
-      "logps/rejected": -33.99717712402344,
-      "loss": 0.8547,
-      "rewards/accuracies": 0.5874999761581421,
-      "rewards/chosen": 0.07274619489908218,
-      "rewards/margins": 0.05342693254351616,
-      "rewards/rejected": 0.019319258630275726,
+      "logits/chosen": -1.9917171001434326,
+      "logits/rejected": -1.9893367290496826,
+      "logps/chosen": -33.120052337646484,
+      "logps/rejected": -34.02777099609375,
+      "loss": 0.9329,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.0742575079202652,
+      "rewards/margins": 0.0732978954911232,
+      "rewards/rejected": 0.0009596304735168815,
       "step": 110
     },
     {
       "epoch": 0.31,
       "learning_rate": 4.353806263777678e-06,
-      "logits/chosen": -2.009084463119507,
-      "logits/rejected": -2.0007264614105225,
-      "logps/chosen": -32.33968734741211,
-      "logps/rejected": -32.13513946533203,
-      "loss": 0.7104,
-      "rewards/accuracies": 0.5625,
-      "rewards/chosen": 0.06312941759824753,
-      "rewards/margins": 0.047309644520282745,
-      "rewards/rejected": 0.015819773077964783,
+      "logits/chosen": -2.0032849311828613,
+      "logits/rejected": -1.9949623346328735,
+      "logps/chosen": -32.30955123901367,
+      "logps/rejected": -32.144927978515625,
+      "loss": 0.9308,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": 0.0812125876545906,
+      "rewards/margins": 0.07126720249652863,
+      "rewards/rejected": 0.009945395402610302,
       "step": 120
     },
     {
       "epoch": 0.34,
       "learning_rate": 4.1940827077152755e-06,
-      "logits/chosen": -2.0365753173828125,
-      "logits/rejected": -2.028564453125,
-      "logps/chosen": -30.363454818725586,
-      "logps/rejected": -32.06770706176758,
-      "loss": 0.71,
-      "rewards/accuracies": 0.5,
-      "rewards/chosen": 0.06735612452030182,
-      "rewards/margins": 0.07617705315351486,
-      "rewards/rejected": -0.008820934221148491,
+      "logits/chosen": -2.031322956085205,
+      "logits/rejected": -2.023355007171631,
+      "logps/chosen": -30.32772445678711,
+      "logps/rejected": -32.05691909790039,
+      "loss": 0.9282,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.08879294246435165,
+      "rewards/margins": 0.09113965928554535,
+      "rewards/rejected": -0.002346712863072753,
       "step": 130
     },
     {
       "epoch": 0.36,
       "learning_rate": 4.0204024186666215e-06,
-      "logits/chosen": -1.9657623767852783,
-      "logits/rejected": -1.976011872291565,
-      "logps/chosen": -31.213130950927734,
-      "logps/rejected": -32.592655181884766,
-      "loss": 0.6309,
-      "rewards/accuracies": 0.675000011920929,
-      "rewards/chosen": 0.12391922622919083,
-      "rewards/margins": 0.1585826873779297,
-      "rewards/rejected": -0.03466347977519035,
+      "logits/chosen": -1.9619598388671875,
+      "logits/rejected": -1.9721866846084595,
+      "logps/chosen": -31.2375431060791,
+      "logps/rejected": -32.572418212890625,
+      "loss": 0.8702,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.1092713475227356,
+      "rewards/margins": 0.13179358839988708,
+      "rewards/rejected": -0.02252225950360298,
       "step": 140
     },
     {
       "epoch": 0.39,
       "learning_rate": 3.834196265035119e-06,
-      "logits/chosen": -1.8778575658798218,
-      "logits/rejected": -1.879020094871521,
-      "logps/chosen": -33.998287200927734,
-      "logps/rejected": -34.83546829223633,
-      "loss": 0.566,
-      "rewards/accuracies": 0.6499999761581421,
-      "rewards/chosen": 0.11664185672998428,
-      "rewards/margins": 0.1635705530643463,
-      "rewards/rejected": -0.04692870378494263,
+      "logits/chosen": -1.8724457025527954,
+      "logits/rejected": -1.8736213445663452,
+      "logps/chosen": -33.905731201171875,
+      "logps/rejected": -34.80728530883789,
+      "loss": 0.818,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": 0.17217601835727692,
+      "rewards/margins": 0.20219452679157257,
+      "rewards/rejected": -0.03001854196190834,
       "step": 150
     },
     {
       "epoch": 0.42,
       "learning_rate": 3.636998309800573e-06,
-      "logits/chosen": -1.9302040338516235,
-      "logits/rejected": -1.9267680644989014,
-      "logps/chosen": -36.0306510925293,
-      "logps/rejected": -32.73199462890625,
-      "loss": 0.5795,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": 0.08461670577526093,
-      "rewards/margins": 0.09016285836696625,
-      "rewards/rejected": -0.005546141881495714,
+      "logits/chosen": -1.923604965209961,
+      "logits/rejected": -1.9202001094818115,
+      "logps/chosen": -36.00878143310547,
+      "logps/rejected": -32.71224594116211,
+      "loss": 0.9086,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": 0.09773740917444229,
+      "rewards/margins": 0.09143415838479996,
+      "rewards/rejected": 0.0063032531179487705,
       "step": 160
     },
     {
       "epoch": 0.44,
       "learning_rate": 3.4304331721118078e-06,
-      "logits/chosen": -2.031188726425171,
-      "logits/rejected": -2.0238261222839355,
-      "logps/chosen": -33.55583953857422,
-      "logps/rejected": -31.381237030029297,
-      "loss": 0.5041,
-      "rewards/accuracies": 0.675000011920929,
-      "rewards/chosen": 0.14521491527557373,
-      "rewards/margins": 0.16284644603729248,
-      "rewards/rejected": -0.017631549388170242,
+      "logits/chosen": -2.024691343307495,
+      "logits/rejected": -2.0173799991607666,
+      "logps/chosen": -33.495445251464844,
+      "logps/rejected": -31.415653228759766,
+      "loss": 0.7882,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": 0.18144862353801727,
+      "rewards/margins": 0.21973037719726562,
+      "rewards/rejected": -0.03828175365924835,
       "step": 170
     },
     {
       "epoch": 0.47,
       "learning_rate": 3.2162026428305436e-06,
-      "logits/chosen": -2.037135362625122,
-      "logits/rejected": -2.0424118041992188,
-      "logps/chosen": -32.33055114746094,
-      "logps/rejected": -32.500144958496094,
-      "loss": 0.5257,
+      "logits/chosen": -2.031024694442749,
+      "logits/rejected": -2.036294937133789,
+      "logps/chosen": -32.22355270385742,
+      "logps/rejected": -32.47669219970703,
+      "loss": 0.8225,
       "rewards/accuracies": 0.6875,
-      "rewards/chosen": 0.12854143977165222,
-      "rewards/margins": 0.13754449784755707,
-      "rewards/rejected": -0.009003056213259697,
+      "rewards/chosen": 0.19274193048477173,
+      "rewards/margins": 0.1876741349697113,
+      "rewards/rejected": 0.00506778946146369,
       "step": 180
     },
     {
       "epoch": 0.49,
       "learning_rate": 2.996071664294641e-06,
-      "logits/chosen": -2.0384888648986816,
-      "logits/rejected": -2.0357367992401123,
-      "logps/chosen": -31.326950073242188,
-      "logps/rejected": -31.338672637939453,
-      "loss": 0.6504,
+      "logits/chosen": -2.0325100421905518,
+      "logits/rejected": -2.029745578765869,
+      "logps/chosen": -31.281234741210938,
+      "logps/rejected": -31.360708236694336,
+      "loss": 0.8564,
       "rewards/accuracies": 0.625,
-      "rewards/chosen": 0.09720095247030258,
-      "rewards/margins": 0.10947537422180176,
-      "rewards/rejected": -0.012274429202079773,
+      "rewards/chosen": 0.12463297694921494,
+      "rewards/margins": 0.15012845396995544,
+      "rewards/rejected": -0.025495493784546852,
       "step": 190
     },
     {
       "epoch": 0.52,
       "learning_rate": 2.7718537898066833e-06,
-      "logits/chosen": -1.908755898475647,
-      "logits/rejected": -1.9134235382080078,
-      "logps/chosen": -31.36275863647461,
-      "logps/rejected": -32.82594680786133,
-      "loss": 0.59,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": 0.14611336588859558,
-      "rewards/margins": 0.16712167859077454,
-      "rewards/rejected": -0.02100830152630806,
+      "logits/chosen": -1.902600646018982,
+      "logits/rejected": -1.9072542190551758,
+      "logps/chosen": -31.33133316040039,
+      "logps/rejected": -32.85138702392578,
+      "loss": 0.8045,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": 0.16497036814689636,
+      "rewards/margins": 0.2012440711259842,
+      "rewards/rejected": -0.03627369925379753,
       "step": 200
     },
     {
       "epoch": 0.52,
-      "eval_logits/chosen": -2.233200788497925,
-      "eval_logits/rejected": -2.228351354598999,
-      "eval_logps/chosen": -34.069149017333984,
-      "eval_logps/rejected": -37.57521057128906,
-      "eval_loss": 0.750046968460083,
-      "eval_rewards/accuracies": 0.5365448594093323,
-      "eval_rewards/chosen": -0.020758595317602158,
-      "eval_rewards/margins": 0.014396170154213905,
-      "eval_rewards/rejected": -0.035154763609170914,
-      "eval_runtime": 145.6956,
-      "eval_samples_per_second": 2.354,
+      "eval_logits/chosen": -2.2286624908447266,
+      "eval_logits/rejected": -2.2238309383392334,
+      "eval_logps/chosen": -34.062259674072266,
+      "eval_logps/rejected": -37.57956314086914,
+      "eval_loss": 0.9791061282157898,
+      "eval_rewards/accuracies": 0.5361295938491821,
+      "eval_rewards/chosen": -0.016622914001345634,
+      "eval_rewards/margins": 0.021141981706023216,
+      "eval_rewards/rejected": -0.03776489570736885,
+      "eval_runtime": 145.8001,
+      "eval_samples_per_second": 2.353,
       "eval_steps_per_second": 0.295,
       "step": 200
     },
     {
       "epoch": 0.55,
       "learning_rate": 2.5453962426402006e-06,
-      "logits/chosen": -2.0207877159118652,
-      "logits/rejected": -2.0314865112304688,
-      "logps/chosen": -31.7946834564209,
-      "logps/rejected": -33.94416046142578,
-      "loss": 0.544,
-      "rewards/accuracies": 0.6625000238418579,
-      "rewards/chosen": 0.10597337782382965,
-      "rewards/margins": 0.1548517644405365,
-      "rewards/rejected": -0.04887838289141655,
+      "logits/chosen": -2.0155138969421387,
+      "logits/rejected": -2.0261716842651367,
+      "logps/chosen": -31.7918758392334,
+      "logps/rejected": -33.9483528137207,
+      "loss": 0.8508,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.10765652358531952,
+      "rewards/margins": 0.15904827415943146,
+      "rewards/rejected": -0.051391761749982834,
       "step": 210
     },
     {
       "epoch": 0.57,
       "learning_rate": 2.3185646976551794e-06,
-      "logits/chosen": -1.9139292240142822,
-      "logits/rejected": -1.9287306070327759,
-      "logps/chosen": -29.91892433166504,
-      "logps/rejected": -31.5737247467041,
-      "loss": 0.5694,
-      "rewards/accuracies": 0.675000011920929,
-      "rewards/chosen": 0.11556863784790039,
-      "rewards/margins": 0.12489677965641022,
-      "rewards/rejected": -0.00932813435792923,
+      "logits/chosen": -1.908071756362915,
+      "logits/rejected": -1.9228509664535522,
+      "logps/chosen": -29.81874656677246,
+      "logps/rejected": -31.65224266052246,
+      "loss": 0.7758,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": 0.175676628947258,
+      "rewards/margins": 0.23211655020713806,
+      "rewards/rejected": -0.05643991753458977,
       "step": 220
     },
     {
       "epoch": 0.6,
       "learning_rate": 2.0932279108998323e-06,
-      "logits/chosen": -1.9701712131500244,
-      "logits/rejected": -1.974172592163086,
-      "logps/chosen": -33.17791748046875,
-      "logps/rejected": -31.603836059570312,
-      "loss": 0.4933,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": 0.1437317281961441,
-      "rewards/margins": 0.1764504611492157,
-      "rewards/rejected": -0.03271873667836189,
+      "logits/chosen": -1.9642655849456787,
+      "logits/rejected": -1.9682365655899048,
+      "logps/chosen": -33.13427734375,
+      "logps/rejected": -31.658504486083984,
+      "loss": 0.7961,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.16991479694843292,
+      "rewards/margins": 0.23543640971183777,
+      "rewards/rejected": -0.06552163511514664,
       "step": 230
     },
     {
       "epoch": 0.62,
       "learning_rate": 1.8712423238279358e-06,
-      "logits/chosen": -1.9691722393035889,
-      "logits/rejected": -1.9472358226776123,
-      "logps/chosen": -33.92795181274414,
-      "logps/rejected": -35.06544876098633,
-      "loss": 0.4966,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": 0.13547900319099426,
-      "rewards/margins": 0.20427203178405762,
-      "rewards/rejected": -0.06879300624132156,
+      "logits/chosen": -1.9623470306396484,
+      "logits/rejected": -1.9405139684677124,
+      "logps/chosen": -33.870643615722656,
+      "logps/rejected": -35.13296127319336,
+      "loss": 0.7551,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": 0.16986075043678284,
+      "rewards/margins": 0.279163658618927,
+      "rewards/rejected": -0.10930290073156357,
       "step": 240
     },
     {
       "epoch": 0.65,
       "learning_rate": 1.6544367689701824e-06,
-      "logits/chosen": -2.010617971420288,
-      "logits/rejected": -2.0072851181030273,
-      "logps/chosen": -32.785011291503906,
-      "logps/rejected": -36.283348083496094,
-      "loss": 0.5823,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": 0.08403309434652328,
-      "rewards/margins": 0.11908762156963348,
-      "rewards/rejected": -0.035054538398981094,
+      "logits/chosen": -2.003516674041748,
+      "logits/rejected": -2.0002071857452393,
+      "logps/chosen": -32.743778228759766,
+      "logps/rejected": -36.293540954589844,
+      "loss": 0.867,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.10877454280853271,
+      "rewards/margins": 0.14994443953037262,
+      "rewards/rejected": -0.041169874370098114,
       "step": 250
     },
     {
       "epoch": 0.68,
       "learning_rate": 1.4445974030621963e-06,
-      "logits/chosen": -1.8787453174591064,
-      "logits/rejected": -1.8763244152069092,
-      "logps/chosen": -34.000633239746094,
-      "logps/rejected": -35.53338623046875,
-      "loss": 0.5277,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": 0.11265683174133301,
-      "rewards/margins": 0.1299169510602951,
-      "rewards/rejected": -0.017260130494832993,
+      "logits/chosen": -1.8702592849731445,
+      "logits/rejected": -1.867851972579956,
+      "logps/chosen": -33.99380111694336,
+      "logps/rejected": -35.564971923828125,
+      "loss": 0.85,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": 0.11675956100225449,
+      "rewards/margins": 0.15297001600265503,
+      "rewards/rejected": -0.03621045500040054,
       "step": 260
     },
     {
       "epoch": 0.7,
       "learning_rate": 1.243452991757889e-06,
-      "logits/chosen": -1.864057183265686,
-      "logits/rejected": -1.861581802368164,
-      "logps/chosen": -34.23812484741211,
-      "logps/rejected": -31.777530670166016,
-      "loss": 0.6307,
-      "rewards/accuracies": 0.6625000238418579,
-      "rewards/chosen": 0.09568403661251068,
-      "rewards/margins": 0.09940369427204132,
-      "rewards/rejected": -0.003719669533893466,
+      "logits/chosen": -1.8547359704971313,
+      "logits/rejected": -1.8523473739624023,
+      "logps/chosen": -34.247703552246094,
+      "logps/rejected": -31.8588809967041,
+      "loss": 0.8606,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.0899367406964302,
+      "rewards/margins": 0.14246629178524017,
+      "rewards/rejected": -0.05252955108880997,
       "step": 270
     },
     {
       "epoch": 0.73,
       "learning_rate": 1.0526606671603523e-06,
-      "logits/chosen": -1.9680726528167725,
-      "logits/rejected": -1.957493782043457,
-      "logps/chosen": -35.05353546142578,
-      "logps/rejected": -31.8691463470459,
-      "loss": 0.4252,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": 0.18186399340629578,
-      "rewards/margins": 0.19875726103782654,
-      "rewards/rejected": -0.01689324900507927,
+      "logits/chosen": -1.9580624103546143,
+      "logits/rejected": -1.9475984573364258,
+      "logps/chosen": -35.032501220703125,
+      "logps/rejected": -31.907541275024414,
+      "loss": 0.7731,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": 0.19448420405387878,
+      "rewards/margins": 0.23441271483898163,
+      "rewards/rejected": -0.039928533136844635,
       "step": 280
     },
     {
       "epoch": 0.75,
       "learning_rate": 8.737922755071455e-07,
-      "logits/chosen": -2.0636587142944336,
-      "logits/rejected": -2.048659563064575,
-      "logps/chosen": -30.76055908203125,
-      "logps/rejected": -32.65589141845703,
-      "loss": 0.6667,
-      "rewards/accuracies": 0.625,
-      "rewards/chosen": 0.09814020991325378,
-      "rewards/margins": 0.10057313740253448,
-      "rewards/rejected": -0.0024329207371920347,
+      "logits/chosen": -2.0538430213928223,
+      "logits/rejected": -2.038924217224121,
+      "logps/chosen": -30.76409912109375,
+      "logps/rejected": -32.64435958862305,
+      "loss": 0.9184,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": 0.09601466357707977,
+      "rewards/margins": 0.09152902662754059,
+      "rewards/rejected": 0.004485635552555323,
       "step": 290
     },
     {
       "epoch": 0.78,
       "learning_rate": 7.08321427484816e-07,
-      "logits/chosen": -1.9355440139770508,
-      "logits/rejected": -1.9329853057861328,
-      "logps/chosen": -32.57345199584961,
-      "logps/rejected": -30.86557388305664,
-      "loss": 0.3871,
-      "rewards/accuracies": 0.7875000238418579,
-      "rewards/chosen": 0.21219508349895477,
-      "rewards/margins": 0.24145305156707764,
-      "rewards/rejected": -0.029257934540510178,
+      "logits/chosen": -1.9241282939910889,
+      "logits/rejected": -1.9215872287750244,
+      "logps/chosen": -32.469078063964844,
+      "logps/rejected": -30.92861557006836,
+      "loss": 0.7186,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": 0.2748206555843353,
+      "rewards/margins": 0.34190455079078674,
+      "rewards/rejected": -0.06708388030529022,
       "step": 300
     },
     {
       "epoch": 0.78,
-      "eval_logits/chosen": -2.233344554901123,
-      "eval_logits/rejected": -2.2284841537475586,
-      "eval_logps/chosen": -34.06221008300781,
-      "eval_logps/rejected": -37.56574249267578,
-      "eval_loss": 0.7655012607574463,
+      "eval_logits/chosen": -2.224278211593628,
+      "eval_logits/rejected": -2.219449996948242,
+      "eval_logps/chosen": -34.111026763916016,
+      "eval_logps/rejected": -37.63855743408203,
+      "eval_loss": 0.9731553196907043,
       "eval_rewards/accuracies": 0.5070598125457764,
-      "eval_rewards/chosen": -0.016592998057603836,
-      "eval_rewards/margins": 0.012880506925284863,
-      "eval_rewards/rejected": -0.029473504051566124,
-      "eval_runtime": 145.6769,
-      "eval_samples_per_second": 2.355,
+      "eval_rewards/chosen": -0.045885536819696426,
+      "eval_rewards/margins": 0.02727569453418255,
+      "eval_rewards/rejected": -0.07316123694181442,
+      "eval_runtime": 145.8155,
+      "eval_samples_per_second": 2.352,
       "eval_steps_per_second": 0.295,
       "step": 300
     },
     {
       "epoch": 0.81,
       "learning_rate": 5.576113578589035e-07,
-      "logits/chosen": -1.9186370372772217,
-      "logits/rejected": -1.9153635501861572,
-      "logps/chosen": -31.363632202148438,
-      "logps/rejected": -33.790061950683594,
-      "loss": 0.5232,
+      "logits/chosen": -1.9088821411132812,
+      "logits/rejected": -1.9056317806243896,
+      "logps/chosen": -31.343881607055664,
+      "logps/rejected": -33.82904052734375,
+      "loss": 0.8131,
       "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": 0.1438317447900772,
-      "rewards/margins": 0.17257718741893768,
-      "rewards/rejected": -0.02874544821679592,
+      "rewards/chosen": 0.15568041801452637,
+      "rewards/margins": 0.20781342685222626,
+      "rewards/rejected": -0.05213301628828049,
       "step": 310
     },
     {
       "epoch": 0.83,
       "learning_rate": 4.229036944380913e-07,
-      "logits/chosen": -1.9695978164672852,
-      "logits/rejected": -1.9573185443878174,
-      "logps/chosen": -34.42280960083008,
-      "logps/rejected": -33.63968276977539,
-      "loss": 0.4864,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": 0.103958860039711,
-      "rewards/margins": 0.1644643247127533,
-      "rewards/rejected": -0.06050548702478409,
+      "logits/chosen": -1.9583854675292969,
+      "logits/rejected": -1.9461708068847656,
+      "logps/chosen": -34.326744079589844,
+      "logps/rejected": -33.68773651123047,
+      "loss": 0.763,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.1615992784500122,
+      "rewards/margins": 0.250935435295105,
+      "rewards/rejected": -0.08933614194393158,
       "step": 320
     },
     {
       "epoch": 0.86,
       "learning_rate": 3.053082288996112e-07,
-      "logits/chosen": -2.0046210289001465,
-      "logits/rejected": -2.003176689147949,
-      "logps/chosen": -33.34724426269531,
-      "logps/rejected": -32.53419876098633,
-      "loss": 0.5611,
-      "rewards/accuracies": 0.675000011920929,
-      "rewards/chosen": 0.08312927186489105,
-      "rewards/margins": 0.11589743942022324,
-      "rewards/rejected": -0.032768163830041885,
+      "logits/chosen": -1.9934914112091064,
+      "logits/rejected": -1.9920612573623657,
+      "logps/chosen": -33.20575714111328,
+      "logps/rejected": -32.5851936340332,
+      "loss": 0.7771,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.16801992058753967,
+      "rewards/margins": 0.23138853907585144,
+      "rewards/rejected": -0.06336863338947296,
       "step": 330
     },
     {
       "epoch": 0.88,
       "learning_rate": 2.0579377374915805e-07,
-      "logits/chosen": -2.091379404067993,
-      "logits/rejected": -2.0756208896636963,
-      "logps/chosen": -33.90237808227539,
-      "logps/rejected": -33.127288818359375,
-      "loss": 0.4859,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": 0.16620711982250214,
-      "rewards/margins": 0.17176242172718048,
-      "rewards/rejected": -0.005555284209549427,
+      "logits/chosen": -2.080026149749756,
+      "logits/rejected": -2.064359188079834,
+      "logps/chosen": -33.830657958984375,
+      "logps/rejected": -33.15457534790039,
+      "loss": 0.7811,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.20923766493797302,
+      "rewards/margins": 0.23116116225719452,
+      "rewards/rejected": -0.021923482418060303,
       "step": 340
     },
     {
       "epoch": 0.91,
       "learning_rate": 1.2518018074041684e-07,
-      "logits/chosen": -1.9639079570770264,
-      "logits/rejected": -1.9629888534545898,
-      "logps/chosen": -32.960304260253906,
-      "logps/rejected": -32.5081787109375,
-      "loss": 0.5048,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": 0.1930757313966751,
-      "rewards/margins": 0.21889439225196838,
-      "rewards/rejected": -0.02581869065761566,
+      "logits/chosen": -1.952505111694336,
+      "logits/rejected": -1.95168936252594,
+      "logps/chosen": -32.850311279296875,
+      "logps/rejected": -32.56792449951172,
+      "loss": 0.7213,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.25907084345817566,
+      "rewards/margins": 0.3207371234893799,
+      "rewards/rejected": -0.061666231602430344,
       "step": 350
     },
     {
       "epoch": 0.94,
       "learning_rate": 6.41315865106129e-08,
-      "logits/chosen": -1.9195839166641235,
-      "logits/rejected": -1.929896593093872,
-      "logps/chosen": -31.997264862060547,
-      "logps/rejected": -35.334999084472656,
-      "loss": 0.5364,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": 0.12445466220378876,
-      "rewards/margins": 0.15219925343990326,
-      "rewards/rejected": -0.027744591236114502,
+      "logits/chosen": -1.9081252813339233,
+      "logits/rejected": -1.918378472328186,
+      "logps/chosen": -31.902362823486328,
+      "logps/rejected": -35.33353805541992,
+      "loss": 0.7965,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.1813959777355194,
+      "rewards/margins": 0.20826482772827148,
+      "rewards/rejected": -0.026868879795074463,
       "step": 360
     },
     {
       "epoch": 0.96,
       "learning_rate": 2.3150941078050325e-08,
-      "logits/chosen": -2.058486223220825,
-      "logits/rejected": -2.051971912384033,
-      "logps/chosen": -33.44487380981445,
-      "logps/rejected": -29.261219024658203,
-      "loss": 0.4692,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": 0.12425780296325684,
-      "rewards/margins": 0.1451319307088852,
-      "rewards/rejected": -0.020874138921499252,
+      "logits/chosen": -2.0479800701141357,
+      "logits/rejected": -2.0415284633636475,
+      "logps/chosen": -33.33736038208008,
+      "logps/rejected": -29.26900863647461,
+      "loss": 0.8,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.18876628577709198,
+      "rewards/margins": 0.21431729197502136,
+      "rewards/rejected": -0.025551024824380875,
       "step": 370
     },
     {
       "epoch": 0.99,
       "learning_rate": 2.575864278703266e-09,
-      "logits/chosen": -1.9182488918304443,
-      "logits/rejected": -1.9204342365264893,
-      "logps/chosen": -33.968048095703125,
-      "logps/rejected": -30.890918731689453,
-      "loss": 0.4679,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": 0.16260387003421783,
-      "rewards/margins": 0.17357683181762695,
-      "rewards/rejected": -0.010972961783409119,
+      "logits/chosen": -1.9078292846679688,
+      "logits/rejected": -1.9100587368011475,
+      "logps/chosen": -33.90921401977539,
+      "logps/rejected": -30.994491577148438,
+      "loss": 0.7663,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.19790422916412354,
+      "rewards/margins": 0.271023154258728,
+      "rewards/rejected": -0.07311896234750748,
       "step": 380
     },
     {
       "epoch": 1.0,
       "step": 385,
       "total_flos": 0.0,
-      "train_loss": 0.5965218513042896,
-      "train_runtime": 3252.2803,
+      "train_loss": 0.859264765157328,
+      "train_runtime": 3251.7628,
       "train_samples_per_second": 0.947,
       "train_steps_per_second": 0.118
     }