Model save

README.md

@@ -1,13 +1,10 @@
 ---
 library_name: peft
 tags:
-- alignment-handbook
 - trl
 - dpo
 - generated_from_trainer
 base_model: NbAiLab/nb-gpt-j-6B-v2
-datasets:
-- hugodk-sch/aftonposten_title_prefs
 model-index:
 - name: aftonposten-6b-align-scan
   results: []
@@ -18,17 +15,17 @@ should probably proofread and complete it, then remove this comment. -->
 
 # aftonposten-6b-align-scan
 
-This model is a fine-tuned version of [data/ap-gpt-j-6b-sft-qlora-04-08](https://huggingface.co/data/ap-gpt-j-6b-sft-qlora-04-08) on the hugodk-sch/aftonposten_title_prefs dataset.
+This model is a fine-tuned version of [NbAiLab/nb-gpt-j-6B-v2](https://huggingface.co/NbAiLab/nb-gpt-j-6B-v2) on an unknown dataset.
 It achieves the following results on the evaluation set:
-- Loss: 0.9997
-- Rewards/chosen: -0.0006
-- Rewards/rejected: -0.0009
+- Loss: 0.6913
+- Rewards/chosen: -0.0042
+- Rewards/rejected: -0.0084
 - Rewards/accuracies: 0.5478
-- Rewards/margins: 0.0003
-- Logps/rejected: -37.6060
-- Logps/chosen: -34.0918
-- Logits/rejected: -2.2164
-- Logits/chosen: -2.2213
+- Rewards/margins: 0.0042
+- Logps/rejected: -37.6008
+- Logps/chosen: -34.0770
+- Logits/rejected: -2.2215
+- Logits/chosen: -2.2263
 
 ## Model description
 
@@ -63,9 +60,9 @@ The following hyperparameters were used during training:
 
 | Training Loss | Epoch | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
 |:-------------:|:-----:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
-| 0.9987        | 0.26  | 100  | 0.9998          | -0.0002        | -0.0004          | 0.5336             | 0.0002          | -37.5528       | -34.0499     | -2.2264         | -2.2313       |
-| 0.9965        | 0.52  | 200  | 0.9996          | -0.0003        | -0.0006          | 0.5071             | 0.0004          | -37.5790       | -34.0618     | -2.2204         | -2.2252       |
-| 0.9925        | 0.78  | 300  | 0.9996          | -0.0005        | -0.0009          | 0.5594             | 0.0004          | -37.6063       | -34.0836     | -2.2166         | -2.2214       |
+| 0.6868        | 0.26  | 100  | 0.6924          | -0.0018        | -0.0036          | 0.5282             | 0.0018          | -37.5527       | -34.0527     | -2.2284         | -2.2332       |
+| 0.6774        | 0.52  | 200  | 0.6914          | -0.0013        | -0.0052          | 0.5511             | 0.0039          | -37.5690       | -34.0476     | -2.2247         | -2.2295       |
+| 0.6615        | 0.78  | 300  | 0.6913          | -0.0042        | -0.0084          | 0.5478             | 0.0042          | -37.6008       | -34.0770     | -2.2215         | -2.2263       |
 
 
 ### Framework versions

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3d1b2226341b09e45acd3cfc68bd07df0a3bb41fe0bdd348f1d55167d9b2880e
+oid sha256:1b8c4489220458976f2273f5710e8ba9db7bbf807d408284e42a2cbd2ebd4de0
 size 176183216

all_results.json

@@ -1,21 +1,8 @@
 {
     "epoch": 1.0,
-    "eval_logits/chosen": -2.221278667449951,
-    "eval_logits/rejected": -2.2164483070373535,
-    "eval_logps/chosen": -34.091766357421875,
-    "eval_logps/rejected": -37.60596466064453,
-    "eval_loss": 0.9996774792671204,
-    "eval_rewards/accuracies": 0.5477575063705444,
-    "eval_rewards/chosen": -0.000572161574382335,
-    "eval_rewards/margins": 0.00032129042665474117,
-    "eval_rewards/rejected": -0.0008934520301409066,
-    "eval_runtime": 145.6061,
-    "eval_samples": 343,
-    "eval_samples_per_second": 2.356,
-    "eval_steps_per_second": 0.295,
-    "train_loss": 0.9973225085766284,
-    "train_runtime": 3253.1307,
+    "train_loss": 0.6806787961489195,
+    "train_runtime": 3251.1509,
     "train_samples": 3079,
-    "train_samples_per_second": 0.946,
+    "train_samples_per_second": 0.947,
     "train_steps_per_second": 0.118
 }

train_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 1.0,
-    "train_loss": 0.9973225085766284,
-    "train_runtime": 3253.1307,
+    "train_loss": 0.6806787961489195,
+    "train_runtime": 3251.1509,
     "train_samples": 3079,
-    "train_samples_per_second": 0.946,
+    "train_samples_per_second": 0.947,
     "train_steps_per_second": 0.118
 }

trainer_state.json

@@ -15,7 +15,7 @@
       "logits/rejected": -1.7377450466156006,
       "logps/chosen": -29.553977966308594,
       "logps/rejected": -42.813133239746094,
-      "loss": 1.0,
+      "loss": 0.6931,
       "rewards/accuracies": 0.0,
       "rewards/chosen": 0.0,
       "rewards/margins": 0.0,
@@ -25,590 +25,590 @@
     {
       "epoch": 0.03,
       "learning_rate": 1.282051282051282e-06,
-      "logits/chosen": -1.8664803504943848,
-      "logits/rejected": -1.8707994222640991,
-      "logps/chosen": -36.978511810302734,
-      "logps/rejected": -33.66939163208008,
-      "loss": 0.9993,
-      "rewards/accuracies": 0.5694444179534912,
-      "rewards/chosen": 0.00028087408281862736,
-      "rewards/margins": 0.0006740752141922712,
-      "rewards/rejected": -0.00039320107316598296,
+      "logits/chosen": -1.866548776626587,
+      "logits/rejected": -1.870866298675537,
+      "logps/chosen": -36.98152160644531,
+      "logps/rejected": -33.65525436401367,
+      "loss": 0.6907,
+      "rewards/accuracies": 0.5833333134651184,
+      "rewards/chosen": 0.0025075911544263363,
+      "rewards/margins": 0.005025296472012997,
+      "rewards/rejected": -0.002517705550417304,
       "step": 10
     },
     {
       "epoch": 0.05,
       "learning_rate": 2.564102564102564e-06,
-      "logits/chosen": -1.9984451532363892,
-      "logits/rejected": -2.0010995864868164,
-      "logps/chosen": -29.63176918029785,
-      "logps/rejected": -29.05954933166504,
-      "loss": 0.9999,
-      "rewards/accuracies": 0.48750001192092896,
-      "rewards/chosen": 0.00010425634536659345,
-      "rewards/margins": 6.528960511786863e-05,
-      "rewards/rejected": 3.8966707506915554e-05,
+      "logits/chosen": -1.9975509643554688,
+      "logits/rejected": -2.000202178955078,
+      "logps/chosen": -29.649311065673828,
+      "logps/rejected": -29.060571670532227,
+      "loss": 0.6937,
+      "rewards/accuracies": 0.42500001192092896,
+      "rewards/chosen": -0.000711599481292069,
+      "rewards/margins": -0.0009990204125642776,
+      "rewards/rejected": 0.00028742075664922595,
       "step": 20
     },
     {
       "epoch": 0.08,
       "learning_rate": 3.846153846153847e-06,
-      "logits/chosen": -1.9210799932479858,
-      "logits/rejected": -1.9183847904205322,
-      "logps/chosen": -31.414783477783203,
-      "logps/rejected": -33.19659423828125,
-      "loss": 1.0002,
-      "rewards/accuracies": 0.4749999940395355,
-      "rewards/chosen": 1.3138540452928282e-05,
-      "rewards/margins": -0.00020548875909298658,
-      "rewards/rejected": 0.00021862727589905262,
+      "logits/chosen": -1.9204410314559937,
+      "logits/rejected": -1.9177614450454712,
+      "logps/chosen": -31.389001846313477,
+      "logps/rejected": -33.2293701171875,
+      "loss": 0.6913,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.0027091833762824535,
+      "rewards/margins": 0.0038000899367034435,
+      "rewards/rejected": -0.0010909067932516336,
       "step": 30
     },
     {
       "epoch": 0.1,
       "learning_rate": 4.999896948438434e-06,
-      "logits/chosen": -2.0177221298217773,
-      "logits/rejected": -2.008965492248535,
-      "logps/chosen": -32.57322311401367,
-      "logps/rejected": -32.500308990478516,
-      "loss": 1.0001,
-      "rewards/accuracies": 0.4749999940395355,
-      "rewards/chosen": 3.584356090868823e-05,
-      "rewards/margins": -8.777440234553069e-05,
-      "rewards/rejected": 0.00012361796689219773,
+      "logits/chosen": -2.0175061225891113,
+      "logits/rejected": -2.0087640285491943,
+      "logps/chosen": -32.5477180480957,
+      "logps/rejected": -32.5131950378418,
+      "loss": 0.6917,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.002909052651375532,
+      "rewards/margins": 0.0029616341926157475,
+      "rewards/rejected": -5.258135570329614e-05,
       "step": 40
     },
     {
       "epoch": 0.13,
       "learning_rate": 4.987541037542187e-06,
-      "logits/chosen": -1.8622690439224243,
-      "logits/rejected": -1.851509690284729,
-      "logps/chosen": -33.547603607177734,
-      "logps/rejected": -35.463592529296875,
-      "loss": 0.9998,
-      "rewards/accuracies": 0.5,
-      "rewards/chosen": 9.876764670480043e-05,
-      "rewards/margins": 0.0002112251240760088,
-      "rewards/rejected": -0.00011245747737120837,
+      "logits/chosen": -1.862919569015503,
+      "logits/rejected": -1.8521493673324585,
+      "logps/chosen": -33.53850555419922,
+      "logps/rejected": -35.44303512573242,
+      "loss": 0.6927,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": 0.001897345995530486,
+      "rewards/margins": 0.0009661337244324386,
+      "rewards/rejected": 0.0009312118636444211,
       "step": 50
     },
     {
       "epoch": 0.16,
       "learning_rate": 4.954691471941119e-06,
-      "logits/chosen": -1.9400131702423096,
-      "logits/rejected": -1.9419806003570557,
-      "logps/chosen": -32.52842330932617,
-      "logps/rejected": -33.22877883911133,
-      "loss": 0.9987,
-      "rewards/accuracies": 0.6499999761581421,
-      "rewards/chosen": 0.0007175664068199694,
-      "rewards/margins": 0.0012915965635329485,
-      "rewards/rejected": -0.00057403021492064,
+      "logits/chosen": -1.941716194152832,
+      "logits/rejected": -1.943652868270874,
+      "logps/chosen": -32.55483627319336,
+      "logps/rejected": -33.212310791015625,
+      "loss": 0.6891,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": 0.004534821957349777,
+      "rewards/margins": 0.008627313189208508,
+      "rewards/rejected": -0.004092490766197443,
       "step": 60
     },
     {
       "epoch": 0.18,
       "learning_rate": 4.901618883413549e-06,
-      "logits/chosen": -2.070552349090576,
-      "logits/rejected": -2.0755274295806885,
-      "logps/chosen": -34.00461959838867,
-      "logps/rejected": -36.64922332763672,
-      "loss": 0.9994,
-      "rewards/accuracies": 0.612500011920929,
-      "rewards/chosen": -0.00016131921438500285,
-      "rewards/margins": 0.0006422021542675793,
-      "rewards/rejected": -0.0008035213686525822,
+      "logits/chosen": -2.072801113128662,
+      "logits/rejected": -2.0777623653411865,
+      "logps/chosen": -33.98583221435547,
+      "logps/rejected": -36.61463165283203,
+      "loss": 0.6909,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": 0.0002649508533068001,
+      "rewards/margins": 0.0048407576978206635,
+      "rewards/rejected": -0.004575806204229593,
       "step": 70
     },
     {
       "epoch": 0.21,
       "learning_rate": 4.828760511501322e-06,
-      "logits/chosen": -1.9306777715682983,
-      "logits/rejected": -1.9338254928588867,
-      "logps/chosen": -34.32624816894531,
-      "logps/rejected": -34.661468505859375,
-      "loss": 0.9983,
-      "rewards/accuracies": 0.612500011920929,
-      "rewards/chosen": 0.0010156143689528108,
-      "rewards/margins": 0.0017062196275219321,
-      "rewards/rejected": -0.0006906053749844432,
+      "logits/chosen": -1.933315634727478,
+      "logits/rejected": -1.9364687204360962,
+      "logps/chosen": -34.30742645263672,
+      "logps/rejected": -34.65543746948242,
+      "loss": 0.6843,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.01203834917396307,
+      "rewards/margins": 0.0183414313942194,
+      "rewards/rejected": -0.006303082220256329,
       "step": 80
     },
     {
       "epoch": 0.23,
       "learning_rate": 4.7367166013034295e-06,
-      "logits/chosen": -1.9389193058013916,
-      "logits/rejected": -1.9434226751327515,
-      "logps/chosen": -32.38957214355469,
-      "logps/rejected": -32.348140716552734,
-      "loss": 0.9993,
-      "rewards/accuracies": 0.5874999761581421,
-      "rewards/chosen": 0.0007785108755342662,
-      "rewards/margins": 0.0006708315922878683,
-      "rewards/rejected": 0.00010767912317533046,
+      "logits/chosen": -1.9412227869033813,
+      "logits/rejected": -1.9457390308380127,
+      "logps/chosen": -32.38041305541992,
+      "logps/rejected": -32.36061096191406,
+      "loss": 0.6889,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.008700890466570854,
+      "rewards/margins": 0.0088711092248559,
+      "rewards/rejected": -0.0001702193112578243,
       "step": 90
     },
     {
       "epoch": 0.26,
       "learning_rate": 4.626245458345211e-06,
-      "logits/chosen": -2.0358777046203613,
-      "logits/rejected": -2.0339014530181885,
-      "logps/chosen": -32.13254165649414,
-      "logps/rejected": -31.29019546508789,
-      "loss": 0.9987,
-      "rewards/accuracies": 0.612500011920929,
-      "rewards/chosen": 0.0010829826351255178,
-      "rewards/margins": 0.001322855008766055,
-      "rewards/rejected": -0.0002398724900558591,
+      "logits/chosen": -2.038828134536743,
+      "logits/rejected": -2.0368473529815674,
+      "logps/chosen": -32.14335250854492,
+      "logps/rejected": -31.2996768951416,
+      "loss": 0.6868,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.009748170152306557,
+      "rewards/margins": 0.013094509951770306,
+      "rewards/rejected": -0.0033463402651250362,
       "step": 100
     },
     {
       "epoch": 0.26,
-      "eval_logits/chosen": -2.2312774658203125,
-      "eval_logits/rejected": -2.226422071456909,
-      "eval_logps/chosen": -34.04991149902344,
-      "eval_logps/rejected": -37.55283737182617,
-      "eval_loss": 0.9997907280921936,
-      "eval_rewards/accuracies": 0.5336378812789917,
-      "eval_rewards/chosen": -0.0001535558985779062,
-      "eval_rewards/margins": 0.00020861340453848243,
-      "eval_rewards/rejected": -0.00036216925946064293,
-      "eval_runtime": 146.0254,
-      "eval_samples_per_second": 2.349,
-      "eval_steps_per_second": 0.294,
+      "eval_logits/chosen": -2.2332358360290527,
+      "eval_logits/rejected": -2.228397846221924,
+      "eval_logps/chosen": -34.052730560302734,
+      "eval_logps/rejected": -37.55271911621094,
+      "eval_loss": 0.692415714263916,
+      "eval_rewards/accuracies": 0.5282392501831055,
+      "eval_rewards/chosen": -0.0018174793804064393,
+      "eval_rewards/margins": 0.0017920632380992174,
+      "eval_rewards/rejected": -0.0036095422692596912,
+      "eval_runtime": 145.8681,
+      "eval_samples_per_second": 2.351,
+      "eval_steps_per_second": 0.295,
       "step": 100
     },
     {
       "epoch": 0.29,
       "learning_rate": 4.498257201263691e-06,
-      "logits/chosen": -1.9907060861587524,
-      "logits/rejected": -1.9883339405059814,
-      "logps/chosen": -33.13169860839844,
-      "logps/rejected": -34.033958435058594,
-      "loss": 0.9988,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": 0.0011211589444428682,
-      "rewards/margins": 0.0011670273961499333,
-      "rewards/rejected": -4.586850991472602e-05,
+      "logits/chosen": -1.9926027059555054,
+      "logits/rejected": -1.9902327060699463,
+      "logps/chosen": -33.1365852355957,
+      "logps/rejected": -34.04566192626953,
+      "loss": 0.6877,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.010723221115767956,
+      "rewards/margins": 0.012352446094155312,
+      "rewards/rejected": -0.0016292240470647812,
       "step": 110
     },
     {
       "epoch": 0.31,
       "learning_rate": 4.353806263777678e-06,
-      "logits/chosen": -2.002023458480835,
-      "logits/rejected": -1.993699312210083,
-      "logps/chosen": -32.341697692871094,
-      "logps/rejected": -32.16511917114258,
-      "loss": 0.9989,
-      "rewards/accuracies": 0.612500011920929,
-      "rewards/chosen": 0.0010320657165721059,
-      "rewards/margins": 0.001068194629624486,
-      "rewards/rejected": -3.612901855376549e-05,
+      "logits/chosen": -2.0035624504089355,
+      "logits/rejected": -1.995218276977539,
+      "logps/chosen": -32.34514617919922,
+      "logps/rejected": -32.111610412597656,
+      "loss": 0.6909,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": 0.009975815191864967,
+      "rewards/margins": 0.004986769054085016,
+      "rewards/rejected": 0.004989045672118664,
       "step": 120
     },
     {
       "epoch": 0.34,
       "learning_rate": 4.1940827077152755e-06,
-      "logits/chosen": -2.028505802154541,
-      "logits/rejected": -2.020526885986328,
-      "logps/chosen": -30.3519287109375,
-      "logps/rejected": -32.101314544677734,
-      "loss": 0.9983,
-      "rewards/accuracies": 0.612500011920929,
-      "rewards/chosen": 0.0012378758983686566,
-      "rewards/margins": 0.001720982021652162,
-      "rewards/rejected": -0.0004831062688026577,
+      "logits/chosen": -2.0313966274261475,
+      "logits/rejected": -2.0234367847442627,
+      "logps/chosen": -30.32912254333496,
+      "logps/rejected": -32.07999801635742,
+      "loss": 0.6851,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.014659431762993336,
+      "rewards/margins": 0.017358481884002686,
+      "rewards/rejected": -0.0026990489568561316,
       "step": 130
     },
     {
       "epoch": 0.36,
       "learning_rate": 4.0204024186666215e-06,
-      "logits/chosen": -1.9588673114776611,
-      "logits/rejected": -1.9690834283828735,
-      "logps/chosen": -31.205490112304688,
-      "logps/rejected": -32.55961608886719,
-      "loss": 0.9976,
-      "rewards/accuracies": 0.6499999761581421,
-      "rewards/chosen": 0.0021417266689240932,
-      "rewards/margins": 0.0023890691809356213,
-      "rewards/rejected": -0.00024734257021918893,
+      "logits/chosen": -1.9624744653701782,
+      "logits/rejected": -1.9726974964141846,
+      "logps/chosen": -31.2264347076416,
+      "logps/rejected": -32.56539535522461,
+      "loss": 0.6823,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": 0.019322697073221207,
+      "rewards/margins": 0.02237415872514248,
+      "rewards/rejected": -0.0030514628160744905,
       "step": 140
     },
     {
       "epoch": 0.39,
       "learning_rate": 3.834196265035119e-06,
-      "logits/chosen": -1.8695415258407593,
-      "logits/rejected": -1.8707062005996704,
-      "logps/chosen": -33.88127899169922,
-      "logps/rejected": -34.7686653137207,
-      "loss": 0.9968,
-      "rewards/accuracies": 0.574999988079071,
-      "rewards/chosen": 0.0031141345389187336,
-      "rewards/margins": 0.003228238318115473,
-      "rewards/rejected": -0.00011410393926780671,
+      "logits/chosen": -1.8730823993682861,
+      "logits/rejected": -1.8742510080337524,
+      "logps/chosen": -33.906436920166016,
+      "logps/rejected": -34.79541778564453,
+      "loss": 0.6777,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": 0.028625473380088806,
+      "rewards/margins": 0.03244180604815483,
+      "rewards/rejected": -0.0038163296412676573,
       "step": 150
     },
     {
       "epoch": 0.42,
       "learning_rate": 3.636998309800573e-06,
-      "logits/chosen": -1.9212032556533813,
-      "logits/rejected": -1.9178003072738647,
-      "logps/chosen": -35.99773406982422,
-      "logps/rejected": -32.705848693847656,
-      "loss": 0.9984,
-      "rewards/accuracies": 0.6499999761581421,
-      "rewards/chosen": 0.0017394202295690775,
-      "rewards/margins": 0.0015704210381954908,
-      "rewards/rejected": 0.00016899927868507802,
+      "logits/chosen": -1.924594521522522,
+      "logits/rejected": -1.9211914539337158,
+      "logps/chosen": -35.99129867553711,
+      "logps/rejected": -32.720947265625,
+      "loss": 0.6845,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.0180378220975399,
+      "rewards/margins": 0.01785714365541935,
+      "rewards/rejected": 0.0001806807704269886,
       "step": 160
     },
     {
       "epoch": 0.44,
       "learning_rate": 3.4304331721118078e-06,
-      "logits/chosen": -2.0206995010375977,
-      "logits/rejected": -2.0133931636810303,
-      "logps/chosen": -33.504085540771484,
-      "logps/rejected": -31.432220458984375,
-      "loss": 0.9963,
-      "rewards/accuracies": 0.637499988079071,
-      "rewards/chosen": 0.0029377774335443974,
-      "rewards/margins": 0.003741443855687976,
-      "rewards/rejected": -0.0008036663057282567,
+      "logits/chosen": -2.025442600250244,
+      "logits/rejected": -2.0181422233581543,
+      "logps/chosen": -33.491790771484375,
+      "logps/rejected": -31.42473793029785,
+      "loss": 0.6749,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.030607115477323532,
+      "rewards/margins": 0.03789608180522919,
+      "rewards/rejected": -0.007288967724889517,
       "step": 170
     },
     {
       "epoch": 0.47,
       "learning_rate": 3.2162026428305436e-06,
-      "logits/chosen": -2.0269291400909424,
-      "logits/rejected": -2.032160997390747,
-      "logps/chosen": -32.24355697631836,
-      "logps/rejected": -32.431182861328125,
-      "loss": 0.9975,
-      "rewards/accuracies": 0.625,
-      "rewards/chosen": 0.003012270200997591,
-      "rewards/margins": 0.0024727012496441603,
-      "rewards/rejected": 0.0005395688931457698,
+      "logits/chosen": -2.0314249992370605,
+      "logits/rejected": -2.0366628170013428,
+      "logps/chosen": -32.224815368652344,
+      "logps/rejected": -32.438621520996094,
+      "loss": 0.68,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.03199741616845131,
+      "rewards/margins": 0.02734515443444252,
+      "rewards/rejected": 0.004652261734008789,
       "step": 180
     },
     {
       "epoch": 0.49,
       "learning_rate": 2.996071664294641e-06,
-      "logits/chosen": -2.027367115020752,
-      "logits/rejected": -2.0246078968048096,
-      "logps/chosen": -31.290613174438477,
-      "logps/rejected": -31.361133575439453,
-      "loss": 0.9976,
-      "rewards/accuracies": 0.625,
-      "rewards/chosen": 0.0019833946134895086,
-      "rewards/margins": 0.002412599278613925,
-      "rewards/rejected": -0.00042920451960526407,
+      "logits/chosen": -2.032400131225586,
+      "logits/rejected": -2.0296273231506348,
+      "logps/chosen": -31.274377822875977,
+      "logps/rejected": -31.340478897094727,
+      "loss": 0.6818,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.021457459777593613,
+      "rewards/margins": 0.023684043437242508,
+      "rewards/rejected": -0.002226583892479539,
       "step": 190
     },
     {
       "epoch": 0.52,
       "learning_rate": 2.7718537898066833e-06,
-      "logits/chosen": -1.8985168933868408,
-      "logits/rejected": -1.903148889541626,
-      "logps/chosen": -31.30405616760254,
-      "logps/rejected": -32.838443756103516,
-      "loss": 0.9965,
+      "logits/chosen": -1.902413010597229,
+      "logits/rejected": -1.9070699214935303,
+      "logps/chosen": -31.3071231842041,
+      "logps/rejected": -32.819236755371094,
+      "loss": 0.6774,
       "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": 0.0030222723726183176,
-      "rewards/margins": 0.0034973658621311188,
-      "rewards/rejected": -0.0004750936641357839,
+      "rewards/chosen": 0.029915904626250267,
+      "rewards/margins": 0.03274631127715111,
+      "rewards/rejected": -0.0028304127044975758,
       "step": 200
     },
     {
       "epoch": 0.52,
-      "eval_logits/chosen": -2.225177764892578,
-      "eval_logits/rejected": -2.2203547954559326,
-      "eval_logps/chosen": -34.06184387207031,
-      "eval_logps/rejected": -37.579010009765625,
-      "eval_loss": 0.9996482133865356,
-      "eval_rewards/accuracies": 0.5070598125457764,
-      "eval_rewards/chosen": -0.0002729461120907217,
-      "eval_rewards/margins": 0.00035095339990220964,
-      "eval_rewards/rejected": -0.0006238995119929314,
-      "eval_runtime": 145.704,
-      "eval_samples_per_second": 2.354,
+      "eval_logits/chosen": -2.2295496463775635,
+      "eval_logits/rejected": -2.224731206893921,
+      "eval_logps/chosen": -34.047569274902344,
+      "eval_logps/rejected": -37.568965911865234,
+      "eval_loss": 0.6914328336715698,
+      "eval_rewards/accuracies": 0.5510797500610352,
+      "eval_rewards/chosen": -0.0013017345918342471,
+      "eval_rewards/margins": 0.003932718187570572,
+      "eval_rewards/rejected": -0.005234452895820141,
+      "eval_runtime": 145.6523,
+      "eval_samples_per_second": 2.355,
       "eval_steps_per_second": 0.295,
       "step": 200
     },
     {
       "epoch": 0.55,
       "learning_rate": 2.5453962426402006e-06,
-      "logits/chosen": -2.011120557785034,
-      "logits/rejected": -2.021751880645752,
-      "logps/chosen": -31.745685577392578,
-      "logps/rejected": -33.96772003173828,
-      "loss": 0.9967,
-      "rewards/accuracies": 0.637499988079071,
-      "rewards/chosen": 0.0022561827208846807,
-      "rewards/margins": 0.003306365106254816,
-      "rewards/rejected": -0.0010501822689548135,
+      "logits/chosen": -2.0154383182525635,
+      "logits/rejected": -2.0260632038116455,
+      "logps/chosen": -31.753280639648438,
+      "logps/rejected": -33.95705032348633,
+      "loss": 0.6781,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.021802183240652084,
+      "rewards/margins": 0.031237006187438965,
+      "rewards/rejected": -0.009434822015464306,
       "step": 210
     },
     {
       "epoch": 0.57,
       "learning_rate": 2.3185646976551794e-06,
-      "logits/chosen": -1.903857946395874,
-      "logits/rejected": -1.918621301651001,
-      "logps/chosen": -29.797290802001953,
-      "logps/rejected": -31.628814697265625,
-      "loss": 0.9962,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": 0.0031425058841705322,
-      "rewards/margins": 0.0038488968275487423,
-      "rewards/rejected": -0.0007063907687552273,
+      "logits/chosen": -1.9075886011123657,
+      "logits/rejected": -1.9223048686981201,
+      "logps/chosen": -29.80865478515625,
+      "logps/rejected": -31.63385009765625,
+      "loss": 0.675,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.03028874099254608,
+      "rewards/margins": 0.037856362760066986,
+      "rewards/rejected": -0.007567620370537043,
       "step": 220
     },
     {
       "epoch": 0.6,
       "learning_rate": 2.0932279108998323e-06,
-      "logits/chosen": -1.9593979120254517,
-      "logits/rejected": -1.9633464813232422,
-      "logps/chosen": -33.067623138427734,
-      "logps/rejected": -31.64206886291504,
-      "loss": 0.9956,
-      "rewards/accuracies": 0.675000011920929,
-      "rewards/chosen": 0.003498472273349762,
-      "rewards/margins": 0.004426136147230864,
-      "rewards/rejected": -0.0009276636992581189,
+      "logits/chosen": -1.9642608165740967,
+      "logits/rejected": -1.9682174921035767,
+      "logps/chosen": -33.096744537353516,
+      "logps/rejected": -31.623376846313477,
+      "loss": 0.6746,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": 0.032072193920612335,
+      "rewards/margins": 0.03947969526052475,
+      "rewards/rejected": -0.0074075027368962765,
       "step": 230
     },
     {
       "epoch": 0.62,
       "learning_rate": 1.8712423238279358e-06,
-      "logits/chosen": -1.9572632312774658,
-      "logits/rejected": -1.9354908466339111,
-      "logps/chosen": -33.843727111816406,
-      "logps/rejected": -35.1453742980957,
-      "loss": 0.995,
+      "logits/chosen": -1.962222695350647,
+      "logits/rejected": -1.9404484033584595,
+      "logps/chosen": -33.80894088745117,
+      "logps/rejected": -35.11548614501953,
+      "loss": 0.6688,
       "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": 0.0031002266332507133,
-      "rewards/margins": 0.0050460235215723515,
-      "rewards/rejected": -0.0019457967719063163,
+      "rewards/chosen": 0.034481119364500046,
+      "rewards/margins": 0.050950419157743454,
+      "rewards/rejected": -0.016469307243824005,
       "step": 240
     },
     {
       "epoch": 0.65,
       "learning_rate": 1.6544367689701824e-06,
-      "logits/chosen": -1.9997854232788086,
-      "logits/rejected": -1.9964803457260132,
-      "logps/chosen": -32.75019454956055,
-      "logps/rejected": -36.28661346435547,
-      "loss": 0.9976,
-      "rewards/accuracies": 0.637499988079071,
-      "rewards/chosen": 0.0017487213481217623,
-      "rewards/margins": 0.0023655896075069904,
-      "rewards/rejected": -0.0006168682011775672,
+      "logits/chosen": -2.0041451454162598,
+      "logits/rejected": -2.0008480548858643,
+      "logps/chosen": -32.71742630004883,
+      "logps/rejected": -36.27611541748047,
+      "loss": 0.6808,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": 0.020764300599694252,
+      "rewards/margins": 0.02588369883596897,
+      "rewards/rejected": -0.005119399167597294,
       "step": 250
     },
     {
       "epoch": 0.68,
       "learning_rate": 1.4445974030621963e-06,
-      "logits/chosen": -1.8673791885375977,
-      "logits/rejected": -1.8649587631225586,
-      "logps/chosen": -34.018226623535156,
-      "logps/rejected": -35.539276123046875,
-      "loss": 0.998,
-      "rewards/accuracies": 0.612500011920929,
-      "rewards/chosen": 0.001701725646853447,
-      "rewards/margins": 0.002048287307843566,
-      "rewards/rejected": -0.0003465614281594753,
+      "logits/chosen": -1.871769666671753,
+      "logits/rejected": -1.8693596124649048,
+      "logps/chosen": -33.992244720458984,
+      "logps/rejected": -35.55255126953125,
+      "loss": 0.6816,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.019615134224295616,
+      "rewards/margins": 0.024408066645264626,
+      "rewards/rejected": -0.004792929161339998,
       "step": 260
     },
     {
       "epoch": 0.7,
       "learning_rate": 1.243452991757889e-06,
-      "logits/chosen": -1.8522275686264038,
-      "logits/rejected": -1.849872350692749,
-      "logps/chosen": -34.16339874267578,
-      "logps/rejected": -31.845317840576172,
-      "loss": 0.9969,
-      "rewards/accuracies": 0.612500011920929,
-      "rewards/chosen": 0.0023420110810548067,
-      "rewards/margins": 0.0030819105450063944,
-      "rewards/rejected": -0.0007398994639515877,
+      "logits/chosen": -1.8571555614471436,
+      "logits/rejected": -1.8547788858413696,
+      "logps/chosen": -34.180908203125,
+      "logps/rejected": -31.8300838470459,
+      "loss": 0.6802,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.021669462323188782,
+      "rewards/margins": 0.027544772252440453,
+      "rewards/rejected": -0.00587531179189682,
       "step": 270
     },
     {
       "epoch": 0.73,
       "learning_rate": 1.0526606671603523e-06,
-      "logits/chosen": -1.9549518823623657,
-      "logits/rejected": -1.94447922706604,
-      "logps/chosen": -35.027687072753906,
-      "logps/rejected": -31.895471572875977,
-      "loss": 0.9962,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": 0.003289591521024704,
-      "rewards/margins": 0.003834384260699153,
-      "rewards/rejected": -0.0005447928560897708,
+      "logits/chosen": -1.9601259231567383,
+      "logits/rejected": -1.949663519859314,
+      "logps/chosen": -34.989627838134766,
+      "logps/rejected": -31.897205352783203,
+      "loss": 0.6728,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.03670142963528633,
+      "rewards/margins": 0.042322538793087006,
+      "rewards/rejected": -0.005621104035526514,
       "step": 280
     },
     {
       "epoch": 0.75,
       "learning_rate": 8.737922755071455e-07,
-      "logits/chosen": -2.0498766899108887,
-      "logits/rejected": -2.034980297088623,
-      "logps/chosen": -30.72440528869629,
-      "logps/rejected": -32.658695220947266,
-      "loss": 0.9979,
-      "rewards/accuracies": 0.637499988079071,
-      "rewards/chosen": 0.0019971781875938177,
-      "rewards/margins": 0.002065772656351328,
-      "rewards/rejected": -6.859400309622288e-05,
+      "logits/chosen": -2.054929494857788,
+      "logits/rejected": -2.040048122406006,
+      "logps/chosen": -30.71946144104004,
+      "logps/rejected": -32.615081787109375,
+      "loss": 0.6854,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.020466715097427368,
+      "rewards/margins": 0.016791274771094322,
+      "rewards/rejected": 0.0036754377651959658,
       "step": 290
     },
     {
       "epoch": 0.78,
       "learning_rate": 7.08321427484816e-07,
-      "logits/chosen": -1.9201946258544922,
-      "logits/rejected": -1.9177051782608032,
-      "logps/chosen": -32.3183479309082,
-      "logps/rejected": -30.95510482788086,
-      "loss": 0.9925,
-      "rewards/accuracies": 0.7749999761581421,
-      "rewards/chosen": 0.006087628658860922,
-      "rewards/margins": 0.007470599375665188,
-      "rewards/rejected": -0.0013829706003889441,
+      "logits/chosen": -1.9249824285507202,
+      "logits/rejected": -1.9225151538848877,
+      "logps/chosen": -32.352317810058594,
+      "logps/rejected": -30.916645050048828,
+      "loss": 0.6615,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": 0.05747910216450691,
+      "rewards/margins": 0.06746286153793335,
+      "rewards/rejected": -0.009983762167394161,
       "step": 300
     },
     {
       "epoch": 0.78,
-      "eval_logits/chosen": -2.2213804721832275,
-      "eval_logits/rejected": -2.216555118560791,
-      "eval_logps/chosen": -34.083614349365234,
-      "eval_logps/rejected": -37.60634994506836,
-      "eval_loss": 0.999591052532196,
-      "eval_rewards/accuracies": 0.559385359287262,
-      "eval_rewards/chosen": -0.0004906260874122381,
-      "eval_rewards/margins": 0.00040669209556654096,
-      "eval_rewards/rejected": -0.0008973181829787791,
-      "eval_runtime": 145.8707,
-      "eval_samples_per_second": 2.351,
+      "eval_logits/chosen": -2.2262814044952393,
+      "eval_logits/rejected": -2.221453905105591,
+      "eval_logps/chosen": -34.07697677612305,
+      "eval_logps/rejected": -37.600772857666016,
+      "eval_loss": 0.6913371086120605,
+      "eval_rewards/accuracies": 0.5477575063705444,
+      "eval_rewards/chosen": -0.004243030212819576,
+      "eval_rewards/margins": 0.004172220826148987,
+      "eval_rewards/rejected": -0.008415251038968563,
+      "eval_runtime": 145.7082,
+      "eval_samples_per_second": 2.354,
       "eval_steps_per_second": 0.295,
       "step": 300
     },
     {
       "epoch": 0.81,
       "learning_rate": 5.576113578589035e-07,
-      "logits/chosen": -1.9060642719268799,
-      "logits/rejected": -1.9028133153915405,
-      "logps/chosen": -31.319162368774414,
-      "logps/rejected": -33.85043716430664,
-      "loss": 0.9961,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": 0.002841859357431531,
-      "rewards/margins": 0.003924719989299774,
-      "rewards/rejected": -0.0010828599333763123,
+      "logits/chosen": -1.9102771282196045,
+      "logits/rejected": -1.9070390462875366,
+      "logps/chosen": -31.315908432006836,
+      "logps/rejected": -33.82908248901367,
+      "loss": 0.6755,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": 0.02874402143061161,
+      "rewards/margins": 0.037437256425619125,
+      "rewards/rejected": -0.008693234995007515,
       "step": 310
     },
     {
       "epoch": 0.83,
       "learning_rate": 4.229036944380913e-07,
-      "logits/chosen": -1.9553836584091187,
-      "logits/rejected": -1.9432109594345093,
-      "logps/chosen": -34.27588653564453,
-      "logps/rejected": -33.672359466552734,
-      "loss": 0.9955,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": 0.0032019000500440598,
-      "rewards/margins": 0.004537059459835291,
-      "rewards/rejected": -0.0013351596426218748,
+      "logits/chosen": -1.960473656654358,
+      "logits/rejected": -1.9483349323272705,
+      "logps/chosen": -34.295162200927734,
+      "logps/rejected": -33.6617431640625,
+      "loss": 0.6729,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.03009115532040596,
+      "rewards/margins": 0.04238158464431763,
+      "rewards/rejected": -0.012290433049201965,
       "step": 320
     },
     {
       "epoch": 0.86,
       "learning_rate": 3.053082288996112e-07,
-      "logits/chosen": -1.9905990362167358,
-      "logits/rejected": -1.9891618490219116,
-      "logps/chosen": -33.116233825683594,
-      "logps/rejected": -32.55724334716797,
-      "loss": 0.9955,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": 0.0036955769173800945,
-      "rewards/margins": 0.004472161643207073,
-      "rewards/rejected": -0.0007765850750729442,
+      "logits/chosen": -1.9958908557891846,
+      "logits/rejected": -1.994442343711853,
+      "logps/chosen": -33.12481689453125,
+      "logps/rejected": -32.55192184448242,
+      "loss": 0.6726,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.036097489297389984,
+      "rewards/margins": 0.04333154112100601,
+      "rewards/rejected": -0.007234054151922464,
       "step": 330
     },
     {
       "epoch": 0.88,
       "learning_rate": 2.0579377374915805e-07,
-      "logits/chosen": -2.0769362449645996,
-      "logits/rejected": -2.0613036155700684,
-      "logps/chosen": -33.791297912597656,
-      "logps/rejected": -33.12422180175781,
-      "loss": 0.9961,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": 0.003880967851728201,
-      "rewards/margins": 0.00394281093031168,
-      "rewards/rejected": -6.184288213262334e-05,
+      "logits/chosen": -2.0822036266326904,
+      "logits/rejected": -2.06657075881958,
+      "logps/chosen": -33.78117752075195,
+      "logps/rejected": -33.087318420410156,
+      "loss": 0.6755,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.03982168063521385,
+      "rewards/margins": 0.036749765276908875,
+      "rewards/rejected": 0.0030719186179339886,
       "step": 340
     },
     {
       "epoch": 0.91,
       "learning_rate": 1.2518018074041684e-07,
-      "logits/chosen": -1.950060248374939,
-      "logits/rejected": -1.9492241144180298,
-      "logps/chosen": -32.82404327392578,
-      "logps/rejected": -32.50709915161133,
-      "loss": 0.995,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": 0.004580510314553976,
-      "rewards/margins": 0.005000022705644369,
-      "rewards/rejected": -0.000419511750806123,
+      "logits/chosen": -1.9546077251434326,
+      "logits/rejected": -1.9537696838378906,
+      "logps/chosen": -32.80642318725586,
+      "logps/rejected": -32.51500701904297,
+      "loss": 0.6684,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": 0.04756731912493706,
+      "rewards/margins": 0.05255315452814102,
+      "rewards/rejected": -0.004985834006220102,
       "step": 350
     },
     {
       "epoch": 0.94,
       "learning_rate": 6.41315865106129e-08,
-      "logits/chosen": -1.9050449132919312,
-      "logits/rejected": -1.915305733680725,
-      "logps/chosen": -31.87860679626465,
-      "logps/rejected": -35.34981155395508,
-      "loss": 0.9961,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": 0.0032608420588076115,
-      "rewards/margins": 0.0038713677786290646,
-      "rewards/rejected": -0.0006105261854827404,
+      "logits/chosen": -1.9106451272964478,
+      "logits/rejected": -1.9209150075912476,
+      "logps/chosen": -31.8604736328125,
+      "logps/rejected": -35.30449676513672,
+      "loss": 0.676,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": 0.034421421587467194,
+      "rewards/margins": 0.03599538281559944,
+      "rewards/rejected": -0.0015739675145596266,
       "step": 360
     },
     {
       "epoch": 0.96,
       "learning_rate": 2.3150941078050325e-08,
-      "logits/chosen": -2.04546856880188,
-      "logits/rejected": -2.039043426513672,
-      "logps/chosen": -33.336219787597656,
-      "logps/rejected": -29.269311904907227,
-      "loss": 0.9964,
-      "rewards/accuracies": 0.6625000238418579,
-      "rewards/chosen": 0.0031574335880577564,
-      "rewards/margins": 0.003586276201531291,
-      "rewards/rejected": -0.0004288425261620432,
+      "logits/chosen": -2.051037311553955,
+      "logits/rejected": -2.044602870941162,
+      "logps/chosen": -33.32126235961914,
+      "logps/rejected": -29.267892837524414,
+      "loss": 0.6752,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": 0.033070918172597885,
+      "rewards/margins": 0.03721793740987778,
+      "rewards/rejected": -0.004147022031247616,
       "step": 370
     },
     {
       "epoch": 0.99,
       "learning_rate": 2.575864278703266e-09,
-      "logits/chosen": -1.905160665512085,
-      "logits/rejected": -1.907360315322876,
-      "logps/chosen": -33.86741256713867,
-      "logps/rejected": -30.982807159423828,
-      "loss": 0.9952,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": 0.0037163912784308195,
-      "rewards/margins": 0.004818186163902283,
-      "rewards/rejected": -0.0011017953511327505,
+      "logits/chosen": -1.9098504781723022,
+      "logits/rejected": -1.9120371341705322,
+      "logps/chosen": -33.857635498046875,
+      "logps/rejected": -30.970882415771484,
+      "loss": 0.6702,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": 0.03814157843589783,
+      "rewards/margins": 0.04796718806028366,
+      "rewards/rejected": -0.009825612418353558,
       "step": 380
     },
     {
       "epoch": 1.0,
       "step": 385,
       "total_flos": 0.0,
-      "train_loss": 0.9973225085766284,
-      "train_runtime": 3253.1307,
-      "train_samples_per_second": 0.946,
+      "train_loss": 0.6806787961489195,
+      "train_runtime": 3251.1509,
+      "train_samples_per_second": 0.947,
       "train_steps_per_second": 0.118
     }
   ],