Model save

README.md

@@ -1,13 +1,10 @@
 ---
 library_name: peft
 tags:
-- alignment-handbook
 - trl
 - dpo
 - generated_from_trainer
 base_model: NbAiLab/nb-gpt-j-6B-v2
-datasets:
-- hugodk-sch/aftonposten_title_prefs
 model-index:
 - name: aftonposten-6b-align-scan
   results: []
@@ -18,17 +15,17 @@ should probably proofread and complete it, then remove this comment. -->
 
 # aftonposten-6b-align-scan
 
-This model is a fine-tuned version of [data/ap-gpt-j-6b-sft-qlora-04-08](https://huggingface.co/data/ap-gpt-j-6b-sft-qlora-04-08) on the hugodk-sch/aftonposten_title_prefs dataset.
+This model is a fine-tuned version of [NbAiLab/nb-gpt-j-6B-v2](https://huggingface.co/NbAiLab/nb-gpt-j-6B-v2) on an unknown dataset.
 It achieves the following results on the evaluation set:
-- Loss: 0.9715
-- Rewards/chosen: -0.0427
-- Rewards/rejected: -0.0715
-- Rewards/accuracies: 0.5365
-- Rewards/margins: 0.0289
-- Logps/rejected: -37.6359
-- Logps/chosen: -34.1056
-- Logits/rejected: -2.2191
-- Logits/chosen: -2.2239
+- Loss: 0.6903
+- Rewards/chosen: -0.0237
+- Rewards/rejected: -0.0486
+- Rewards/accuracies: 0.5191
+- Rewards/margins: 0.0249
+- Logps/rejected: -37.5860
+- Logps/chosen: -34.0684
+- Logits/rejected: -2.2229
+- Logits/chosen: -2.2277
 
 ## Model description
 
@@ -63,9 +60,9 @@ The following hyperparameters were used during training:
 
 | Training Loss | Epoch | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
 |:-------------:|:-----:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
-| 0.9204        | 0.26  | 100  | 0.9816          | -0.0008        | -0.0193          | 0.5395             | 0.0185          | -37.5487       | -34.0358     | -2.2268         | -2.2317       |
-| 0.8045        | 0.52  | 200  | 0.9791          | -0.0166        | -0.0378          | 0.5361             | 0.0211          | -37.5796       | -34.0623     | -2.2238         | -2.2287       |
-| 0.7186        | 0.78  | 300  | 0.9732          | -0.0459        | -0.0732          | 0.5071             | 0.0273          | -37.6386       | -34.1110     | -2.2194         | -2.2243       |
+| 0.6577        | 0.26  | 100  | 0.6949          | -0.0038        | -0.0133          | 0.5316             | 0.0095          | -37.5357       | -34.0400     | -2.2289         | -2.2338       |
+| 0.6156        | 0.52  | 200  | 0.6943          | -0.0132        | -0.0288          | 0.5191             | 0.0156          | -37.5578       | -34.0535     | -2.2268         | -2.2317       |
+| 0.5468        | 0.78  | 300  | 0.6903          | -0.0237        | -0.0486          | 0.5191             | 0.0249          | -37.5860       | -34.0684     | -2.2229         | -2.2277       |
 
 
 ### Framework versions

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9bd332dec561f01d2bca2a6e7821277394bd0e424cf6dba7b5419a5e722eefb2
+oid sha256:47298c65412662b4ec4aa64bcd24a4e16a9a56108e7e9e1f87d87e4b59545e2b
 size 176183216

all_results.json

@@ -1,20 +1,7 @@
 {
     "epoch": 1.0,
-    "eval_logits/chosen": -2.223938465118408,
-    "eval_logits/rejected": -2.2191123962402344,
-    "eval_logps/chosen": -34.10564041137695,
-    "eval_logps/rejected": -37.63585662841797,
-    "eval_loss": 0.9714530110359192,
-    "eval_rewards/accuracies": 0.5365448594093323,
-    "eval_rewards/chosen": -0.042656492441892624,
-    "eval_rewards/margins": 0.028885547071695328,
-    "eval_rewards/rejected": -0.07154203206300735,
-    "eval_runtime": 145.6066,
-    "eval_samples": 343,
-    "eval_samples_per_second": 2.356,
-    "eval_steps_per_second": 0.295,
-    "train_loss": 0.859264765157328,
-    "train_runtime": 3251.7628,
+    "train_loss": 0.6291831790626823,
+    "train_runtime": 3251.1508,
     "train_samples": 3079,
     "train_samples_per_second": 0.947,
     "train_steps_per_second": 0.118

train_results.json

@@ -1,7 +1,7 @@
 {
     "epoch": 1.0,
-    "train_loss": 0.859264765157328,
-    "train_runtime": 3251.7628,
+    "train_loss": 0.6291831790626823,
+    "train_runtime": 3251.1508,
     "train_samples": 3079,
     "train_samples_per_second": 0.947,
     "train_steps_per_second": 0.118

trainer_state.json

@@ -15,7 +15,7 @@
       "logits/rejected": -1.7377450466156006,
       "logps/chosen": -29.553977966308594,
       "logps/rejected": -42.813133239746094,
-      "loss": 1.0,
+      "loss": 0.6931,
       "rewards/accuracies": 0.0,
       "rewards/chosen": 0.0,
       "rewards/margins": 0.0,
@@ -25,589 +25,589 @@
     {
       "epoch": 0.03,
       "learning_rate": 1.282051282051282e-06,
-      "logits/chosen": -1.866829752922058,
-      "logits/rejected": -1.8711390495300293,
-      "logps/chosen": -36.97040557861328,
-      "logps/rejected": -33.66280746459961,
-      "loss": 0.9586,
-      "rewards/accuracies": 0.5972222089767456,
-      "rewards/chosen": 0.021715592592954636,
-      "rewards/margins": 0.041353289037942886,
-      "rewards/rejected": -0.01963769644498825,
+      "logits/chosen": -1.8668904304504395,
+      "logits/rejected": -1.8712035417556763,
+      "logps/chosen": -36.981239318847656,
+      "logps/rejected": -33.63866424560547,
+      "loss": 0.6837,
+      "rewards/accuracies": 0.5138888955116272,
+      "rewards/chosen": 0.017751876264810562,
+      "rewards/margins": 0.023763436824083328,
+      "rewards/rejected": -0.0060115596279501915,
       "step": 10
     },
     {
       "epoch": 0.05,
       "learning_rate": 2.564102564102564e-06,
-      "logits/chosen": -1.9976301193237305,
-      "logits/rejected": -2.0002708435058594,
-      "logps/chosen": -29.64252281188965,
-      "logps/rejected": -29.05857276916504,
-      "loss": 1.0031,
-      "rewards/accuracies": 0.4749999940395355,
-      "rewards/chosen": -0.00019791116937994957,
-      "rewards/margins": -0.003119796048849821,
-      "rewards/rejected": 0.0029218837153166533,
+      "logits/chosen": -1.9975477457046509,
+      "logits/rejected": -2.0001885890960693,
+      "logps/chosen": -29.638402938842773,
+      "logps/rejected": -29.045080184936523,
+      "loss": 0.7005,
+      "rewards/accuracies": 0.4375,
+      "rewards/chosen": 0.002654150128364563,
+      "rewards/margins": -0.010201702825725079,
+      "rewards/rejected": 0.012855852022767067,
       "step": 20
     },
     {
       "epoch": 0.08,
       "learning_rate": 3.846153846153847e-06,
-      "logits/chosen": -1.9210872650146484,
-      "logits/rejected": -1.918402910232544,
-      "logps/chosen": -31.397838592529297,
-      "logps/rejected": -33.22719192504883,
-      "loss": 0.9838,
-      "rewards/accuracies": 0.612500011920929,
-      "rewards/chosen": 0.010952453128993511,
-      "rewards/margins": 0.01619068905711174,
-      "rewards/rejected": -0.005238235928118229,
+      "logits/chosen": -1.9209153652191162,
+      "logits/rejected": -1.9182332754135132,
+      "logps/chosen": -31.392269134521484,
+      "logps/rejected": -33.214996337890625,
+      "loss": 0.6893,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.016679534688591957,
+      "rewards/margins": 0.014253495261073112,
+      "rewards/rejected": 0.0024260382633656263,
       "step": 30
     },
     {
       "epoch": 0.1,
       "learning_rate": 4.999896948438434e-06,
-      "logits/chosen": -2.017606258392334,
-      "logits/rejected": -2.0088634490966797,
-      "logps/chosen": -32.566612243652344,
-      "logps/rejected": -32.52539825439453,
-      "loss": 0.9862,
-      "rewards/accuracies": 0.5375000238418579,
-      "rewards/chosen": 0.006120447069406509,
-      "rewards/margins": 0.013754432089626789,
-      "rewards/rejected": -0.007633985485881567,
+      "logits/chosen": -2.017472505569458,
+      "logits/rejected": -2.0087223052978516,
+      "logps/chosen": -32.56648635864258,
+      "logps/rejected": -32.53681564331055,
+      "loss": 0.6837,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": 0.007227012421935797,
+      "rewards/margins": 0.02412785217165947,
+      "rewards/rejected": -0.016900835558772087,
       "step": 40
     },
     {
       "epoch": 0.13,
       "learning_rate": 4.987541037542187e-06,
-      "logits/chosen": -1.8625805377960205,
-      "logits/rejected": -1.8518139123916626,
-      "logps/chosen": -33.56818771362305,
-      "logps/rejected": -35.4713020324707,
-      "loss": 0.9951,
-      "rewards/accuracies": 0.550000011920929,
-      "rewards/chosen": -0.0064262161031365395,
-      "rewards/margins": 0.004948456306010485,
-      "rewards/rejected": -0.011374671943485737,
+      "logits/chosen": -1.8619842529296875,
+      "logits/rejected": -1.8512147665023804,
+      "logps/chosen": -33.57755661010742,
+      "logps/rejected": -35.45317840576172,
+      "loss": 0.7035,
+      "rewards/accuracies": 0.4749999940395355,
+      "rewards/chosen": -0.014055396430194378,
+      "rewards/margins": -0.013470378704369068,
+      "rewards/rejected": -0.0005850124871358275,
       "step": 50
     },
     {
       "epoch": 0.16,
       "learning_rate": 4.954691471941119e-06,
-      "logits/chosen": -1.9413652420043945,
-      "logits/rejected": -1.9432977437973022,
-      "logps/chosen": -32.551727294921875,
-      "logps/rejected": -33.218013763427734,
-      "loss": 0.9525,
-      "rewards/accuracies": 0.5375000238418579,
-      "rewards/chosen": 0.02907358668744564,
-      "rewards/margins": 0.05705299973487854,
-      "rewards/rejected": -0.02797941491007805,
+      "logits/chosen": -1.940241813659668,
+      "logits/rejected": -1.942185401916504,
+      "logps/chosen": -32.55712127685547,
+      "logps/rejected": -33.23926544189453,
+      "loss": 0.6629,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": 0.030143504962325096,
+      "rewards/margins": 0.07766537368297577,
+      "rewards/rejected": -0.047521863132715225,
       "step": 60
     },
     {
       "epoch": 0.18,
       "learning_rate": 4.901618883413549e-06,
-      "logits/chosen": -2.0722384452819824,
-      "logits/rejected": -2.0772085189819336,
-      "logps/chosen": -33.995296478271484,
-      "logps/rejected": -36.63286590576172,
-      "loss": 0.9657,
-      "rewards/accuracies": 0.550000011920929,
-      "rewards/chosen": -0.004085049033164978,
-      "rewards/margins": 0.034312374889850616,
-      "rewards/rejected": -0.038397423923015594,
+      "logits/chosen": -2.070986270904541,
+      "logits/rejected": -2.0759458541870117,
+      "logps/chosen": -33.997291564941406,
+      "logps/rejected": -36.65125274658203,
+      "loss": 0.6775,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -0.006164415739476681,
+      "rewards/margins": 0.051500022411346436,
+      "rewards/rejected": -0.057664431631565094,
       "step": 70
     },
     {
       "epoch": 0.21,
       "learning_rate": 4.828760511501322e-06,
-      "logits/chosen": -1.9325292110443115,
-      "logits/rejected": -1.93563973903656,
-      "logps/chosen": -34.35693359375,
-      "logps/rejected": -34.67926788330078,
-      "loss": 0.9069,
-      "rewards/accuracies": 0.574999988079071,
-      "rewards/chosen": 0.042526502162218094,
-      "rewards/margins": 0.09463920444250107,
-      "rewards/rejected": -0.05211270600557327,
+      "logits/chosen": -1.9328521490097046,
+      "logits/rejected": -1.935974359512329,
+      "logps/chosen": -34.317806243896484,
+      "logps/rejected": -34.648555755615234,
+      "loss": 0.6465,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": 0.07700104266405106,
+      "rewards/margins": 0.11630574613809586,
+      "rewards/rejected": -0.0393047034740448,
       "step": 80
     },
     {
       "epoch": 0.23,
       "learning_rate": 4.7367166013034295e-06,
-      "logits/chosen": -1.9406629800796509,
-      "logits/rejected": -1.9451707601547241,
-      "logps/chosen": -32.38554000854492,
-      "logps/rejected": -32.36783218383789,
-      "loss": 0.9455,
-      "rewards/accuracies": 0.612500011920929,
-      "rewards/chosen": 0.04912734776735306,
-      "rewards/margins": 0.05448007583618164,
-      "rewards/rejected": -0.005352728068828583,
+      "logits/chosen": -1.941239356994629,
+      "logits/rejected": -1.9457323551177979,
+      "logps/chosen": -32.385948181152344,
+      "logps/rejected": -32.34455108642578,
+      "loss": 0.6805,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.057031381875276566,
+      "rewards/margins": 0.046979233622550964,
+      "rewards/rejected": 0.010052147321403027,
       "step": 90
     },
     {
       "epoch": 0.26,
       "learning_rate": 4.626245458345211e-06,
-      "logits/chosen": -2.037841320037842,
-      "logits/rejected": -2.0358526706695557,
-      "logps/chosen": -32.14287185668945,
-      "logps/rejected": -31.300832748413086,
-      "loss": 0.9204,
-      "rewards/accuracies": 0.5874999761581421,
-      "rewards/chosen": 0.058778904378414154,
-      "rewards/margins": 0.07955379039049149,
-      "rewards/rejected": -0.02077488601207733,
+      "logits/chosen": -2.0390255451202393,
+      "logits/rejected": -2.0370407104492188,
+      "logps/chosen": -32.15161895751953,
+      "logps/rejected": -31.304489135742188,
+      "loss": 0.6577,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": 0.06245182827115059,
+      "rewards/margins": 0.08924683928489685,
+      "rewards/rejected": -0.02679501473903656,
       "step": 100
     },
     {
       "epoch": 0.26,
-      "eval_logits/chosen": -2.2316644191741943,
-      "eval_logits/rejected": -2.2268166542053223,
-      "eval_logps/chosen": -34.03582000732422,
-      "eval_logps/rejected": -37.548709869384766,
-      "eval_loss": 0.9816102385520935,
-      "eval_rewards/accuracies": 0.5394518375396729,
-      "eval_rewards/chosen": -0.0007606110884808004,
-      "eval_rewards/margins": 0.018493397161364555,
-      "eval_rewards/rejected": -0.019254004582762718,
-      "eval_runtime": 145.9929,
-      "eval_samples_per_second": 2.349,
+      "eval_logits/chosen": -2.233787775039673,
+      "eval_logits/rejected": -2.228933572769165,
+      "eval_logps/chosen": -34.04001235961914,
+      "eval_logps/rejected": -37.535667419433594,
+      "eval_loss": 0.6948937773704529,
+      "eval_rewards/accuracies": 0.5315614938735962,
+      "eval_rewards/chosen": -0.0038188453763723373,
+      "eval_rewards/margins": 0.00951432902365923,
+      "eval_rewards/rejected": -0.013333176262676716,
+      "eval_runtime": 145.7707,
+      "eval_samples_per_second": 2.353,
       "eval_steps_per_second": 0.295,
       "step": 100
     },
     {
       "epoch": 0.29,
       "learning_rate": 4.498257201263691e-06,
-      "logits/chosen": -1.9917171001434326,
-      "logits/rejected": -1.9893367290496826,
-      "logps/chosen": -33.120052337646484,
-      "logps/rejected": -34.02777099609375,
-      "loss": 0.9329,
-      "rewards/accuracies": 0.675000011920929,
-      "rewards/chosen": 0.0742575079202652,
-      "rewards/margins": 0.0732978954911232,
-      "rewards/rejected": 0.0009596304735168815,
+      "logits/chosen": -1.994390845298767,
+      "logits/rejected": -1.991999864578247,
+      "logps/chosen": -33.1099739074707,
+      "logps/rejected": -34.02802276611328,
+      "loss": 0.6747,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.09369214624166489,
+      "rewards/margins": 0.0927465409040451,
+      "rewards/rejected": 0.0009456165134906769,
       "step": 110
     },
     {
       "epoch": 0.31,
       "learning_rate": 4.353806263777678e-06,
-      "logits/chosen": -2.0032849311828613,
-      "logits/rejected": -1.9949623346328735,
-      "logps/chosen": -32.30955123901367,
-      "logps/rejected": -32.144927978515625,
-      "loss": 0.9308,
-      "rewards/accuracies": 0.612500011920929,
-      "rewards/chosen": 0.0812125876545906,
-      "rewards/margins": 0.07126720249652863,
-      "rewards/rejected": 0.009945395402610302,
+      "logits/chosen": -2.0057625770568848,
+      "logits/rejected": -1.9974448680877686,
+      "logps/chosen": -32.31781768798828,
+      "logps/rejected": -32.132328033447266,
+      "loss": 0.6694,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": 0.0889604315161705,
+      "rewards/margins": 0.06853620707988739,
+      "rewards/rejected": 0.02042422816157341,
       "step": 120
     },
     {
       "epoch": 0.34,
       "learning_rate": 4.1940827077152755e-06,
-      "logits/chosen": -2.031322956085205,
-      "logits/rejected": -2.023355007171631,
-      "logps/chosen": -30.32772445678711,
-      "logps/rejected": -32.05691909790039,
-      "loss": 0.9282,
-      "rewards/accuracies": 0.550000011920929,
-      "rewards/chosen": 0.08879294246435165,
-      "rewards/margins": 0.09113965928554535,
-      "rewards/rejected": -0.002346712863072753,
+      "logits/chosen": -2.0336880683898926,
+      "logits/rejected": -2.0257279872894287,
+      "logps/chosen": -30.32443618774414,
+      "logps/rejected": -32.07221221923828,
+      "loss": 0.6522,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.10589826107025146,
+      "rewards/margins": 0.11934350430965424,
+      "rewards/rejected": -0.013445250689983368,
       "step": 130
     },
     {
       "epoch": 0.36,
       "learning_rate": 4.0204024186666215e-06,
-      "logits/chosen": -1.9619598388671875,
-      "logits/rejected": -1.9721866846084595,
-      "logps/chosen": -31.2375431060791,
-      "logps/rejected": -32.572418212890625,
-      "loss": 0.8702,
-      "rewards/accuracies": 0.625,
-      "rewards/chosen": 0.1092713475227356,
-      "rewards/margins": 0.13179358839988708,
-      "rewards/rejected": -0.02252225950360298,
+      "logits/chosen": -1.9642670154571533,
+      "logits/rejected": -1.9745124578475952,
+      "logps/chosen": -31.212024688720703,
+      "logps/rejected": -32.565834045410156,
+      "loss": 0.6272,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.14534947276115417,
+      "rewards/margins": 0.16701875627040863,
+      "rewards/rejected": -0.02166926860809326,
       "step": 140
     },
     {
       "epoch": 0.39,
       "learning_rate": 3.834196265035119e-06,
-      "logits/chosen": -1.8724457025527954,
-      "logits/rejected": -1.8736213445663452,
-      "logps/chosen": -33.905731201171875,
-      "logps/rejected": -34.80728530883789,
-      "loss": 0.818,
-      "rewards/accuracies": 0.637499988079071,
-      "rewards/chosen": 0.17217601835727692,
-      "rewards/margins": 0.20219452679157257,
-      "rewards/rejected": -0.03001854196190834,
+      "logits/chosen": -1.8759937286376953,
+      "logits/rejected": -1.8771553039550781,
+      "logps/chosen": -33.917335510253906,
+      "logps/rejected": -34.8185920715332,
+      "loss": 0.6064,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": 0.19274887442588806,
+      "rewards/margins": 0.235683411359787,
+      "rewards/rejected": -0.042934536933898926,
       "step": 150
     },
     {
       "epoch": 0.42,
       "learning_rate": 3.636998309800573e-06,
-      "logits/chosen": -1.923604965209961,
-      "logits/rejected": -1.9202001094818115,
-      "logps/chosen": -36.00878143310547,
-      "logps/rejected": -32.71224594116211,
-      "loss": 0.9086,
-      "rewards/accuracies": 0.637499988079071,
-      "rewards/chosen": 0.09773740917444229,
-      "rewards/margins": 0.09143415838479996,
-      "rewards/rejected": 0.0063032531179487705,
+      "logits/chosen": -1.927369475364685,
+      "logits/rejected": -1.9239553213119507,
+      "logps/chosen": -35.98323440551758,
+      "logps/rejected": -32.705108642578125,
+      "loss": 0.6464,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": 0.13190819323062897,
+      "rewards/margins": 0.11955627053976059,
+      "rewards/rejected": 0.012351910583674908,
       "step": 160
     },
     {
       "epoch": 0.44,
       "learning_rate": 3.4304331721118078e-06,
-      "logits/chosen": -2.024691343307495,
-      "logits/rejected": -2.0173799991607666,
-      "logps/chosen": -33.495445251464844,
-      "logps/rejected": -31.415653228759766,
-      "loss": 0.7882,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": 0.18144862353801727,
-      "rewards/margins": 0.21973037719726562,
-      "rewards/rejected": -0.03828175365924835,
+      "logits/chosen": -2.028454542160034,
+      "logits/rejected": -2.0210976600646973,
+      "logps/chosen": -33.50457763671875,
+      "logps/rejected": -31.386072158813477,
+      "loss": 0.6077,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": 0.20529839396476746,
+      "rewards/margins": 0.22925393283367157,
+      "rewards/rejected": -0.02395555004477501,
       "step": 170
     },
     {
       "epoch": 0.47,
       "learning_rate": 3.2162026428305436e-06,
-      "logits/chosen": -2.031024694442749,
-      "logits/rejected": -2.036294937133789,
-      "logps/chosen": -32.22355270385742,
-      "logps/rejected": -32.47669219970703,
-      "loss": 0.8225,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": 0.19274193048477173,
-      "rewards/margins": 0.1876741349697113,
-      "rewards/rejected": 0.00506778946146369,
+      "logits/chosen": -2.0345230102539062,
+      "logits/rejected": -2.0397608280181885,
+      "logps/chosen": -32.19758224487305,
+      "logps/rejected": -32.435523986816406,
+      "loss": 0.6083,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": 0.24304144084453583,
+      "rewards/margins": 0.2083090990781784,
+      "rewards/rejected": 0.03473237156867981,
       "step": 180
     },
     {
       "epoch": 0.49,
       "learning_rate": 2.996071664294641e-06,
-      "logits/chosen": -2.0325100421905518,
-      "logits/rejected": -2.029745578765869,
-      "logps/chosen": -31.281234741210938,
-      "logps/rejected": -31.360708236694336,
-      "loss": 0.8564,
-      "rewards/accuracies": 0.625,
-      "rewards/chosen": 0.12463297694921494,
-      "rewards/margins": 0.15012845396995544,
-      "rewards/rejected": -0.025495493784546852,
+      "logits/chosen": -2.035979747772217,
+      "logits/rejected": -2.0332181453704834,
+      "logps/chosen": -31.255151748657227,
+      "logps/rejected": -31.321630477905273,
+      "loss": 0.6326,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.163661926984787,
+      "rewards/margins": 0.16605310142040253,
+      "rewards/rejected": -0.0023911758325994015,
       "step": 190
     },
     {
       "epoch": 0.52,
       "learning_rate": 2.7718537898066833e-06,
-      "logits/chosen": -1.902600646018982,
-      "logits/rejected": -1.9072542190551758,
-      "logps/chosen": -31.33133316040039,
-      "logps/rejected": -32.85138702392578,
-      "loss": 0.8045,
+      "logits/chosen": -1.9058892726898193,
+      "logits/rejected": -1.910540223121643,
+      "logps/chosen": -31.33197021484375,
+      "logps/rejected": -32.817604064941406,
+      "loss": 0.6156,
       "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": 0.16497036814689636,
-      "rewards/margins": 0.2012440711259842,
-      "rewards/rejected": -0.03627369925379753,
+      "rewards/chosen": 0.19201864302158356,
+      "rewards/margins": 0.21068540215492249,
+      "rewards/rejected": -0.01866675540804863,
       "step": 200
     },
     {
       "epoch": 0.52,
-      "eval_logits/chosen": -2.2286624908447266,
-      "eval_logits/rejected": -2.2238309383392334,
-      "eval_logps/chosen": -34.062259674072266,
-      "eval_logps/rejected": -37.57956314086914,
-      "eval_loss": 0.9791061282157898,
-      "eval_rewards/accuracies": 0.5361295938491821,
-      "eval_rewards/chosen": -0.016622914001345634,
-      "eval_rewards/margins": 0.021141981706023216,
-      "eval_rewards/rejected": -0.03776489570736885,
-      "eval_runtime": 145.8001,
-      "eval_samples_per_second": 2.353,
+      "eval_logits/chosen": -2.2316529750823975,
+      "eval_logits/rejected": -2.2268118858337402,
+      "eval_logps/chosen": -34.053470611572266,
+      "eval_logps/rejected": -37.557804107666016,
+      "eval_loss": 0.6943473219871521,
+      "eval_rewards/accuracies": 0.5191029906272888,
+      "eval_rewards/chosen": -0.013246187008917332,
+      "eval_rewards/margins": 0.015584951266646385,
+      "eval_rewards/rejected": -0.028831137344241142,
+      "eval_runtime": 145.8114,
+      "eval_samples_per_second": 2.352,
       "eval_steps_per_second": 0.295,
       "step": 200
     },
     {
       "epoch": 0.55,
       "learning_rate": 2.5453962426402006e-06,
-      "logits/chosen": -2.0155138969421387,
-      "logits/rejected": -2.0261716842651367,
-      "logps/chosen": -31.7918758392334,
-      "logps/rejected": -33.9483528137207,
-      "loss": 0.8508,
-      "rewards/accuracies": 0.6499999761581421,
-      "rewards/chosen": 0.10765652358531952,
-      "rewards/margins": 0.15904827415943146,
-      "rewards/rejected": -0.051391761749982834,
+      "logits/chosen": -2.01808762550354,
+      "logits/rejected": -2.0287296772003174,
+      "logps/chosen": -31.763586044311523,
+      "logps/rejected": -33.926063537597656,
+      "loss": 0.622,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": 0.14540112018585205,
+      "rewards/margins": 0.1897541582584381,
+      "rewards/rejected": -0.04435301572084427,
       "step": 210
     },
     {
       "epoch": 0.57,
       "learning_rate": 2.3185646976551794e-06,
-      "logits/chosen": -1.908071756362915,
-      "logits/rejected": -1.9228509664535522,
-      "logps/chosen": -29.81874656677246,
-      "logps/rejected": -31.65224266052246,
-      "loss": 0.7758,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": 0.175676628947258,
-      "rewards/margins": 0.23211655020713806,
-      "rewards/rejected": -0.05643991753458977,
+      "logits/chosen": -1.910658597946167,
+      "logits/rejected": -1.9254140853881836,
+      "logps/chosen": -29.832544326782227,
+      "logps/rejected": -31.612533569335938,
+      "loss": 0.6024,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": 0.19529443979263306,
+      "rewards/margins": 0.23334410786628723,
+      "rewards/rejected": -0.038049641996622086,
       "step": 220
     },
     {
       "epoch": 0.6,
       "learning_rate": 2.0932279108998323e-06,
-      "logits/chosen": -1.9642655849456787,
-      "logits/rejected": -1.9682365655899048,
-      "logps/chosen": -33.13427734375,
-      "logps/rejected": -31.658504486083984,
-      "loss": 0.7961,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": 0.16991479694843292,
-      "rewards/margins": 0.23543640971183777,
-      "rewards/rejected": -0.06552163511514664,
+      "logits/chosen": -1.9673402309417725,
+      "logits/rejected": -1.9713077545166016,
+      "logps/chosen": -33.062660217285156,
+      "logps/rejected": -31.608203887939453,
+      "loss": 0.5911,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.24836687743663788,
+      "rewards/margins": 0.2895973324775696,
+      "rewards/rejected": -0.0412304513156414,
       "step": 230
     },
     {
       "epoch": 0.62,
       "learning_rate": 1.8712423238279358e-06,
-      "logits/chosen": -1.9623470306396484,
-      "logits/rejected": -1.9405139684677124,
-      "logps/chosen": -33.870643615722656,
-      "logps/rejected": -35.13296127319336,
-      "loss": 0.7551,
+      "logits/chosen": -1.9657704830169678,
+      "logits/rejected": -1.9439115524291992,
+      "logps/chosen": -33.82358932495117,
+      "logps/rejected": -35.09131622314453,
+      "loss": 0.5733,
       "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": 0.16986075043678284,
-      "rewards/margins": 0.279163658618927,
-      "rewards/rejected": -0.10930290073156357,
+      "rewards/chosen": 0.23111140727996826,
+      "rewards/margins": 0.32947883009910583,
+      "rewards/rejected": -0.09836738556623459,
       "step": 240
     },
     {
       "epoch": 0.65,
       "learning_rate": 1.6544367689701824e-06,
-      "logits/chosen": -2.003516674041748,
-      "logits/rejected": -2.0002071857452393,
-      "logps/chosen": -32.743778228759766,
-      "logps/rejected": -36.293540954589844,
-      "loss": 0.867,
+      "logits/chosen": -2.007312536239624,
+      "logits/rejected": -2.003988742828369,
+      "logps/chosen": -32.68491744995117,
+      "logps/rejected": -36.24928283691406,
+      "loss": 0.6228,
       "rewards/accuracies": 0.6499999761581421,
-      "rewards/chosen": 0.10877454280853271,
-      "rewards/margins": 0.14994443953037262,
-      "rewards/rejected": -0.041169874370098114,
+      "rewards/chosen": 0.1681055724620819,
+      "rewards/margins": 0.18515849113464355,
+      "rewards/rejected": -0.01705293543636799,
       "step": 250
     },
     {
       "epoch": 0.68,
       "learning_rate": 1.4445974030621963e-06,
-      "logits/chosen": -1.8702592849731445,
-      "logits/rejected": -1.867851972579956,
-      "logps/chosen": -33.99380111694336,
-      "logps/rejected": -35.564971923828125,
-      "loss": 0.85,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": 0.11675956100225449,
-      "rewards/margins": 0.15297001600265503,
-      "rewards/rejected": -0.03621045500040054,
+      "logits/chosen": -1.8734614849090576,
+      "logits/rejected": -1.8710591793060303,
+      "logps/chosen": -33.96207809448242,
+      "logps/rejected": -35.5266227722168,
+      "loss": 0.6286,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.15842413902282715,
+      "rewards/margins": 0.17382602393627167,
+      "rewards/rejected": -0.015401872806251049,
       "step": 260
     },
     {
       "epoch": 0.7,
       "learning_rate": 1.243452991757889e-06,
-      "logits/chosen": -1.8547359704971313,
-      "logits/rejected": -1.8523473739624023,
-      "logps/chosen": -34.247703552246094,
-      "logps/rejected": -31.8588809967041,
-      "loss": 0.8606,
-      "rewards/accuracies": 0.574999988079071,
-      "rewards/chosen": 0.0899367406964302,
-      "rewards/margins": 0.14246629178524017,
-      "rewards/rejected": -0.05252955108880997,
+      "logits/chosen": -1.8584483861923218,
+      "logits/rejected": -1.8560174703598022,
+      "logps/chosen": -34.1807975769043,
+      "logps/rejected": -31.80449867248535,
+      "loss": 0.6325,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.15176042914390564,
+      "rewards/margins": 0.17497751116752625,
+      "rewards/rejected": -0.02321707457304001,
       "step": 270
     },
     {
       "epoch": 0.73,
       "learning_rate": 1.0526606671603523e-06,
-      "logits/chosen": -1.9580624103546143,
-      "logits/rejected": -1.9475984573364258,
-      "logps/chosen": -35.032501220703125,
-      "logps/rejected": -31.907541275024414,
-      "loss": 0.7731,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": 0.19448420405387878,
-      "rewards/margins": 0.23441271483898163,
-      "rewards/rejected": -0.039928533136844635,
+      "logits/chosen": -1.9630708694458008,
+      "logits/rejected": -1.9525701999664307,
+      "logps/chosen": -35.00975036621094,
+      "logps/rejected": -31.84867286682129,
+      "loss": 0.5978,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": 0.2428235560655594,
+      "rewards/margins": 0.24820086359977722,
+      "rewards/rejected": -0.005377279128879309,
       "step": 280
     },
     {
       "epoch": 0.75,
       "learning_rate": 8.737922755071455e-07,
-      "logits/chosen": -2.0538430213928223,
-      "logits/rejected": -2.038924217224121,
-      "logps/chosen": -30.76409912109375,
-      "logps/rejected": -32.64435958862305,
-      "loss": 0.9184,
-      "rewards/accuracies": 0.612500011920929,
-      "rewards/chosen": 0.09601466357707977,
-      "rewards/margins": 0.09152902662754059,
-      "rewards/rejected": 0.004485635552555323,
+      "logits/chosen": -2.0586793422698975,
+      "logits/rejected": -2.043766498565674,
+      "logps/chosen": -30.7253360748291,
+      "logps/rejected": -32.66551971435547,
+      "loss": 0.6462,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": 0.13915367424488068,
+      "rewards/margins": 0.14873233437538147,
+      "rewards/rejected": -0.009578653611242771,
       "step": 290
     },
     {
       "epoch": 0.78,
       "learning_rate": 7.08321427484816e-07,
-      "logits/chosen": -1.9241282939910889,
-      "logits/rejected": -1.9215872287750244,
-      "logps/chosen": -32.469078063964844,
-      "logps/rejected": -30.92861557006836,
-      "loss": 0.7186,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": 0.2748206555843353,
-      "rewards/margins": 0.34190455079078674,
-      "rewards/rejected": -0.06708388030529022,
+      "logits/chosen": -1.9287078380584717,
+      "logits/rejected": -1.92616868019104,
+      "logps/chosen": -32.39979553222656,
+      "logps/rejected": -30.87893295288086,
+      "loss": 0.5468,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.36911940574645996,
+      "rewards/margins": 0.4126061797142029,
+      "rewards/rejected": -0.04348675161600113,
       "step": 300
     },
     {
       "epoch": 0.78,
-      "eval_logits/chosen": -2.224278211593628,
-      "eval_logits/rejected": -2.219449996948242,
-      "eval_logps/chosen": -34.111026763916016,
-      "eval_logps/rejected": -37.63855743408203,
-      "eval_loss": 0.9731553196907043,
-      "eval_rewards/accuracies": 0.5070598125457764,
-      "eval_rewards/chosen": -0.045885536819696426,
-      "eval_rewards/margins": 0.02727569453418255,
-      "eval_rewards/rejected": -0.07316123694181442,
-      "eval_runtime": 145.8155,
-      "eval_samples_per_second": 2.352,
+      "eval_logits/chosen": -2.2277145385742188,
+      "eval_logits/rejected": -2.222882032394409,
+      "eval_logps/chosen": -34.06844711303711,
+      "eval_logps/rejected": -37.586029052734375,
+      "eval_loss": 0.6902604699134827,
+      "eval_rewards/accuracies": 0.5191029906272888,
+      "eval_rewards/chosen": -0.023726314306259155,
+      "eval_rewards/margins": 0.024859989061951637,
+      "eval_rewards/rejected": -0.04858630895614624,
+      "eval_runtime": 145.7514,
+      "eval_samples_per_second": 2.353,
       "eval_steps_per_second": 0.295,
       "step": 300
     },
     {
       "epoch": 0.81,
       "learning_rate": 5.576113578589035e-07,
-      "logits/chosen": -1.9088821411132812,
-      "logits/rejected": -1.9056317806243896,
-      "logps/chosen": -31.343881607055664,
-      "logps/rejected": -33.82904052734375,
-      "loss": 0.8131,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": 0.15568041801452637,
-      "rewards/margins": 0.20781342685222626,
-      "rewards/rejected": -0.05213301628828049,
+      "logits/chosen": -1.9126994609832764,
+      "logits/rejected": -1.909444808959961,
+      "logps/chosen": -31.307331085205078,
+      "logps/rejected": -33.79678726196289,
+      "loss": 0.6022,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": 0.20721367001533508,
+      "rewards/margins": 0.24546091258525848,
+      "rewards/rejected": -0.038247235119342804,
       "step": 310
     },
     {
       "epoch": 0.83,
       "learning_rate": 4.229036944380913e-07,
-      "logits/chosen": -1.9583854675292969,
-      "logits/rejected": -1.9461708068847656,
-      "logps/chosen": -34.326744079589844,
-      "logps/rejected": -33.68773651123047,
-      "loss": 0.763,
-      "rewards/accuracies": 0.6499999761581421,
-      "rewards/chosen": 0.1615992784500122,
-      "rewards/margins": 0.250935435295105,
-      "rewards/rejected": -0.08933614194393158,
+      "logits/chosen": -1.9636253118515015,
+      "logits/rejected": -1.9514182806015015,
+      "logps/chosen": -34.32239532470703,
+      "logps/rejected": -33.65345764160156,
+      "loss": 0.5933,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": 0.19157564640045166,
+      "rewards/margins": 0.27180662751197815,
+      "rewards/rejected": -0.08023098856210709,
       "step": 320
     },
     {
       "epoch": 0.86,
       "learning_rate": 3.053082288996112e-07,
-      "logits/chosen": -1.9934914112091064,
-      "logits/rejected": -1.9920612573623657,
-      "logps/chosen": -33.20575714111328,
-      "logps/rejected": -32.5851936340332,
-      "loss": 0.7771,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": 0.16801992058753967,
-      "rewards/margins": 0.23138853907585144,
-      "rewards/rejected": -0.06336863338947296,
+      "logits/chosen": -1.9983371496200562,
+      "logits/rejected": -1.9969165325164795,
+      "logps/chosen": -33.176841735839844,
+      "logps/rejected": -32.538509368896484,
+      "loss": 0.5981,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": 0.21626754105091095,
+      "rewards/margins": 0.257517009973526,
+      "rewards/rejected": -0.04124947637319565,
       "step": 330
     },
     {
       "epoch": 0.88,
       "learning_rate": 2.0579377374915805e-07,
-      "logits/chosen": -2.080026149749756,
-      "logits/rejected": -2.064359188079834,
-      "logps/chosen": -33.830657958984375,
-      "logps/rejected": -33.15457534790039,
-      "loss": 0.7811,
-      "rewards/accuracies": 0.675000011920929,
-      "rewards/chosen": 0.20923766493797302,
-      "rewards/margins": 0.23116116225719452,
-      "rewards/rejected": -0.021923482418060303,
+      "logits/chosen": -2.085096836090088,
+      "logits/rejected": -2.069387435913086,
+      "logps/chosen": -33.758583068847656,
+      "logps/rejected": -33.068748474121094,
+      "loss": 0.5952,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": 0.29456597566604614,
+      "rewards/margins": 0.26006320118904114,
+      "rewards/rejected": 0.034502796828746796,
       "step": 340
     },
     {
       "epoch": 0.91,
       "learning_rate": 1.2518018074041684e-07,
-      "logits/chosen": -1.952505111694336,
-      "logits/rejected": -1.95168936252594,
-      "logps/chosen": -32.850311279296875,
-      "logps/rejected": -32.56792449951172,
-      "loss": 0.7213,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": 0.25907084345817566,
-      "rewards/margins": 0.3207371234893799,
-      "rewards/rejected": -0.061666231602430344,
+      "logits/chosen": -1.9578405618667603,
+      "logits/rejected": -1.9569900035858154,
+      "logps/chosen": -32.835628509521484,
+      "logps/rejected": -32.520538330078125,
+      "loss": 0.5707,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.3125234544277191,
+      "rewards/margins": 0.3512992262840271,
+      "rewards/rejected": -0.038775794208049774,
       "step": 350
     },
     {
       "epoch": 0.94,
       "learning_rate": 6.41315865106129e-08,
-      "logits/chosen": -1.9081252813339233,
-      "logits/rejected": -1.918378472328186,
-      "logps/chosen": -31.902362823486328,
-      "logps/rejected": -35.33353805541992,
-      "loss": 0.7965,
-      "rewards/accuracies": 0.675000011920929,
-      "rewards/chosen": 0.1813959777355194,
-      "rewards/margins": 0.20826482772827148,
-      "rewards/rejected": -0.026868879795074463,
+      "logits/chosen": -1.9134842157363892,
+      "logits/rejected": -1.9237855672836304,
+      "logps/chosen": -31.848628997802734,
+      "logps/rejected": -35.309120178222656,
+      "loss": 0.5923,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.24924305081367493,
+      "rewards/margins": 0.2634957432746887,
+      "rewards/rejected": -0.014252680353820324,
       "step": 360
     },
     {
       "epoch": 0.96,
       "learning_rate": 2.3150941078050325e-08,
-      "logits/chosen": -2.0479800701141357,
-      "logits/rejected": -2.0415284633636475,
-      "logps/chosen": -33.33736038208008,
-      "logps/rejected": -29.26900863647461,
-      "loss": 0.8,
-      "rewards/accuracies": 0.675000011920929,
-      "rewards/chosen": 0.18876628577709198,
-      "rewards/margins": 0.21431729197502136,
-      "rewards/rejected": -0.025551024824380875,
+      "logits/chosen": -2.052818775177002,
+      "logits/rejected": -2.046323776245117,
+      "logps/chosen": -33.34727096557617,
+      "logps/rejected": -29.256671905517578,
+      "loss": 0.5993,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": 0.21328690648078918,
+      "rewards/margins": 0.23445896804332733,
+      "rewards/rejected": -0.021172069013118744,
       "step": 370
     },
     {
       "epoch": 0.99,
       "learning_rate": 2.575864278703266e-09,
-      "logits/chosen": -1.9078292846679688,
-      "logits/rejected": -1.9100587368011475,
-      "logps/chosen": -33.90921401977539,
-      "logps/rejected": -30.994491577148438,
-      "loss": 0.7663,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": 0.19790422916412354,
-      "rewards/margins": 0.271023154258728,
-      "rewards/rejected": -0.07311896234750748,
+      "logits/chosen": -1.9116294384002686,
+      "logits/rejected": -1.913830041885376,
+      "logps/chosen": -33.83928680419922,
+      "logps/rejected": -30.931400299072266,
+      "loss": 0.5714,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.27983543276786804,
+      "rewards/margins": 0.3209769129753113,
+      "rewards/rejected": -0.04114149510860443,
       "step": 380
     },
     {
       "epoch": 1.0,
       "step": 385,
       "total_flos": 0.0,
-      "train_loss": 0.859264765157328,
-      "train_runtime": 3251.7628,
+      "train_loss": 0.6291831790626823,
+      "train_runtime": 3251.1508,
       "train_samples_per_second": 0.947,
       "train_steps_per_second": 0.118
     }