Model save

README.md

@@ -1,13 +1,10 @@
 ---
 library_name: peft
 tags:
-- alignment-handbook
 - trl
 - dpo
 - generated_from_trainer
 base_model: NbAiLab/nb-gpt-j-6B-v2
-datasets:
-- hugodk-sch/aftonposten_title_prefs
 model-index:
 - name: aftonposten-6b-align-scan
   results: []
@@ -18,17 +15,17 @@ should probably proofread and complete it, then remove this comment. -->
 
 # aftonposten-6b-align-scan
 
-This model is a fine-tuned version of [data/ap-gpt-j-6b-sft-qlora-04-08](https://huggingface.co/data/ap-gpt-j-6b-sft-qlora-04-08) on the hugodk-sch/aftonposten_title_prefs dataset.
+This model is a fine-tuned version of [NbAiLab/nb-gpt-j-6B-v2](https://huggingface.co/NbAiLab/nb-gpt-j-6B-v2) on an unknown dataset.
 It achieves the following results on the evaluation set:
-- Loss: 0.6903
-- Rewards/chosen: -0.0025
-- Rewards/rejected: -0.0237
-- Rewards/accuracies: 0.5519
-- Rewards/margins: 0.0212
-- Logps/rejected: -37.5562
-- Logps/chosen: -34.0388
-- Logits/rejected: -2.2221
-- Logits/chosen: -2.2270
+- Loss: 0.4937
+- Rewards/chosen: 0.1639
+- Rewards/rejected: 0.1391
+- Rewards/accuracies: 0.5361
+- Rewards/margins: 0.0248
+- Logps/rejected: -37.2848
+- Logps/chosen: -33.7614
+- Logits/rejected: -2.2261
+- Logits/chosen: -2.2309
 
 ## Model description
 
@@ -63,9 +60,9 @@ The following hyperparameters were used during training:
 
 | Training Loss | Epoch | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
 |:-------------:|:-----:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
-| 0.6595        | 0.26  | 100  | 0.6900          | 0.0170         | 0.0010           | 0.5253             | 0.0160          | -37.5150       | -34.0063     | -2.2287         | -2.2336       |
-| 0.6189        | 0.52  | 200  | 0.6895          | 0.0107         | -0.0106          | 0.5341             | 0.0213          | -37.5344       | -34.0167     | -2.2259         | -2.2307       |
-| 0.5582        | 0.78  | 300  | 0.6872          | -0.0004        | -0.0276          | 0.5598             | 0.0272          | -37.5627       | -34.0353     | -2.2221         | -2.2269       |
+| 0.4799        | 0.26  | 100  | 0.4987          | 0.0790         | 0.0734           | 0.5104             | 0.0056          | -37.3943       | -33.9029     | -2.2301         | -2.2349       |
+| 0.4548        | 0.52  | 200  | 0.4956          | 0.1590         | 0.1392           | 0.5341             | 0.0198          | -37.2846       | -33.7696     | -2.2287         | -2.2335       |
+| 0.41          | 0.78  | 300  | 0.4937          | 0.1639         | 0.1391           | 0.5361             | 0.0248          | -37.2848       | -33.7614     | -2.2261         | -2.2309       |
 
 
 ### Framework versions

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:551da0818c9710df6bd28d0096381ec12741c94227c876abb8c993660ecfeae2
+oid sha256:5f4af652c4082ace59d6cfd645e0b1edcf903193c93e0147e4ef37697cdaee0f
 size 176183216

all_results.json

@@ -1,21 +1,8 @@
 {
     "epoch": 1.0,
-    "eval_logits/chosen": -2.226956844329834,
-    "eval_logits/rejected": -2.2221333980560303,
-    "eval_logps/chosen": -34.03876876831055,
-    "eval_logps/rejected": -37.55620193481445,
-    "eval_loss": 0.6902801394462585,
-    "eval_rewards/accuracies": 0.5519103407859802,
-    "eval_rewards/chosen": -0.002528026234358549,
-    "eval_rewards/margins": 0.021220751106739044,
-    "eval_rewards/rejected": -0.02374877780675888,
-    "eval_runtime": 145.6815,
-    "eval_samples": 343,
-    "eval_samples_per_second": 2.354,
-    "eval_steps_per_second": 0.295,
-    "train_loss": 0.6364745784115482,
-    "train_runtime": 3251.6479,
+    "train_loss": 0.46439293204963983,
+    "train_runtime": 3253.2697,
     "train_samples": 3079,
-    "train_samples_per_second": 0.947,
+    "train_samples_per_second": 0.946,
     "train_steps_per_second": 0.118
 }

train_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 1.0,
-    "train_loss": 0.6364745784115482,
-    "train_runtime": 3251.6479,
+    "train_loss": 0.46439293204963983,
+    "train_runtime": 3253.2697,
     "train_samples": 3079,
-    "train_samples_per_second": 0.947,
+    "train_samples_per_second": 0.946,
     "train_steps_per_second": 0.118
 }

trainer_state.json

@@ -15,7 +15,7 @@
       "logits/rejected": -1.7377450466156006,
       "logps/chosen": -29.553977966308594,
       "logps/rejected": -42.813133239746094,
-      "loss": 0.6931,
+      "loss": 0.5,
       "rewards/accuracies": 0.0,
       "rewards/chosen": 0.0,
       "rewards/margins": 0.0,
@@ -25,467 +25,467 @@
     {
       "epoch": 0.03,
       "learning_rate": 1.282051282051282e-06,
-      "logits/chosen": -1.8668782711029053,
-      "logits/rejected": -1.8712005615234375,
-      "logps/chosen": -36.98646545410156,
-      "logps/rejected": -33.67870330810547,
-      "loss": 0.6747,
-      "rewards/accuracies": 0.5833333134651184,
-      "rewards/chosen": 0.012081857770681381,
-      "rewards/margins": 0.04126200079917908,
-      "rewards/rejected": -0.029180139303207397,
+      "logits/chosen": -1.866639494895935,
+      "logits/rejected": -1.870951533317566,
+      "logps/chosen": -36.991004943847656,
+      "logps/rejected": -33.64551544189453,
+      "loss": 0.4968,
+      "rewards/accuracies": 0.5694444179534912,
+      "rewards/chosen": 0.009355764836072922,
+      "rewards/margins": 0.018620232120156288,
+      "rewards/rejected": -0.009264465421438217,
       "step": 10
     },
     {
       "epoch": 0.05,
       "learning_rate": 2.564102564102564e-06,
-      "logits/chosen": -1.9979515075683594,
-      "logits/rejected": -2.0005881786346436,
-      "logps/chosen": -29.662744522094727,
-      "logps/rejected": -29.051654815673828,
-      "loss": 0.7042,
-      "rewards/accuracies": 0.4000000059604645,
-      "rewards/chosen": -0.01232814695686102,
-      "rewards/margins": -0.019403135403990746,
-      "rewards/rejected": 0.007074988447129726,
+      "logits/chosen": -1.9975944757461548,
+      "logits/rejected": -2.000239133834839,
+      "logps/chosen": -29.655420303344727,
+      "logps/rejected": -29.04937171936035,
+      "loss": 0.5034,
+      "rewards/accuracies": 0.44999998807907104,
+      "rewards/chosen": -0.007936288602650166,
+      "rewards/margins": -0.016377601772546768,
+      "rewards/rejected": 0.008441311307251453,
       "step": 20
     },
     {
       "epoch": 0.08,
       "learning_rate": 3.846153846153847e-06,
-      "logits/chosen": -1.921088457107544,
-      "logits/rejected": -1.9184081554412842,
-      "logps/chosen": -31.383258819580078,
-      "logps/rejected": -33.23828887939453,
-      "loss": 0.6794,
-      "rewards/accuracies": 0.5874999761581421,
-      "rewards/chosen": 0.019701367244124413,
-      "rewards/margins": 0.03159697726368904,
-      "rewards/rejected": -0.011895612813532352,
+      "logits/chosen": -1.920345664024353,
+      "logits/rejected": -1.917649507522583,
+      "logps/chosen": -31.40158462524414,
+      "logps/rejected": -33.22117614746094,
+      "loss": 0.499,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.008706120774149895,
+      "rewards/margins": 0.010335424914956093,
+      "rewards/rejected": -0.0016293060034513474,
       "step": 30
     },
     {
       "epoch": 0.1,
       "learning_rate": 4.999896948438434e-06,
-      "logits/chosen": -2.017509937286377,
-      "logits/rejected": -2.0087647438049316,
-      "logps/chosen": -32.577518463134766,
-      "logps/rejected": -32.509830474853516,
-      "loss": 0.6964,
-      "rewards/accuracies": 0.48750001192092896,
-      "rewards/chosen": -0.00042539089918136597,
-      "rewards/margins": -0.0021285698749125004,
-      "rewards/rejected": 0.001703177229501307,
+      "logits/chosen": -2.016953706741333,
+      "logits/rejected": -2.008241891860962,
+      "logps/chosen": -32.55894088745117,
+      "logps/rejected": -32.48887634277344,
+      "loss": 0.501,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.010718915611505508,
+      "rewards/margins": -0.0035588580649346113,
+      "rewards/rejected": 0.014277773909270763,
       "step": 40
     },
     {
       "epoch": 0.13,
       "learning_rate": 4.987541037542187e-06,
-      "logits/chosen": -1.8623021841049194,
-      "logits/rejected": -1.8515303134918213,
-      "logps/chosen": -33.56303787231445,
-      "logps/rejected": -35.47795867919922,
-      "loss": 0.6896,
-      "rewards/accuracies": 0.5249999761581421,
-      "rewards/chosen": -0.0033326249103993177,
-      "rewards/margins": 0.012034483253955841,
-      "rewards/rejected": -0.015367108397185802,
+      "logits/chosen": -1.8621622323989868,
+      "logits/rejected": -1.851413369178772,
+      "logps/chosen": -33.5220832824707,
+      "logps/rejected": -35.4012565612793,
+      "loss": 0.5034,
+      "rewards/accuracies": 0.48750001192092896,
+      "rewards/chosen": 0.021240826696157455,
+      "rewards/margins": -0.009411826729774475,
+      "rewards/rejected": 0.03065265715122223,
       "step": 50
     },
     {
       "epoch": 0.16,
       "learning_rate": 4.954691471941119e-06,
-      "logits/chosen": -1.9411529302597046,
-      "logits/rejected": -1.943098783493042,
-      "logps/chosen": -32.549232482910156,
-      "logps/rejected": -33.20621109008789,
-      "loss": 0.6747,
-      "rewards/accuracies": 0.5249999761581421,
-      "rewards/chosen": 0.030568569898605347,
-      "rewards/margins": 0.051466990262269974,
-      "rewards/rejected": -0.02089841663837433,
+      "logits/chosen": -1.9412288665771484,
+      "logits/rejected": -1.9431712627410889,
+      "logps/chosen": -32.506317138671875,
+      "logps/rejected": -33.202842712402344,
+      "loss": 0.4846,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.05631808191537857,
+      "rewards/margins": 0.07519452273845673,
+      "rewards/rejected": -0.018876442685723305,
       "step": 60
     },
     {
       "epoch": 0.18,
       "learning_rate": 4.901618883413549e-06,
-      "logits/chosen": -2.0727076530456543,
-      "logits/rejected": -2.07767391204834,
-      "logps/chosen": -33.96394729614258,
-      "logps/rejected": -36.61058807373047,
-      "loss": 0.6813,
-      "rewards/accuracies": 0.5249999761581421,
-      "rewards/chosen": 0.014724211767315865,
-      "rewards/margins": 0.03975607082247734,
-      "rewards/rejected": -0.025031859055161476,
+      "logits/chosen": -2.0726475715637207,
+      "logits/rejected": -2.077623128890991,
+      "logps/chosen": -33.90425491333008,
+      "logps/rejected": -36.56460189819336,
+      "loss": 0.4903,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.05054063722491264,
+      "rewards/margins": 0.04797897860407829,
+      "rewards/rejected": 0.002561659086495638,
       "step": 70
     },
     {
       "epoch": 0.21,
       "learning_rate": 4.828760511501322e-06,
-      "logits/chosen": -1.9330183267593384,
-      "logits/rejected": -1.936171293258667,
-      "logps/chosen": -34.318397521972656,
-      "logps/rejected": -34.61701583862305,
-      "loss": 0.6634,
-      "rewards/accuracies": 0.550000011920929,
-      "rewards/chosen": 0.06564848870038986,
-      "rewards/margins": 0.08041460067033768,
-      "rewards/rejected": -0.014766111969947815,
+      "logits/chosen": -1.9338359832763672,
+      "logits/rejected": -1.9369510412216187,
+      "logps/chosen": -34.21551513671875,
+      "logps/rejected": -34.527488708496094,
+      "loss": 0.4797,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": 0.1273767203092575,
+      "rewards/margins": 0.08842838555574417,
+      "rewards/rejected": 0.038948334753513336,
       "step": 80
     },
     {
       "epoch": 0.23,
       "learning_rate": 4.7367166013034295e-06,
-      "logits/chosen": -1.9412405490875244,
-      "logits/rejected": -1.945755958557129,
-      "logps/chosen": -32.374977111816406,
-      "logps/rejected": -32.33773422241211,
-      "loss": 0.6787,
-      "rewards/accuracies": 0.6499999761581421,
-      "rewards/chosen": 0.05546582490205765,
-      "rewards/margins": 0.04276125878095627,
-      "rewards/rejected": 0.012704563327133656,
+      "logits/chosen": -1.9420888423919678,
+      "logits/rejected": -1.9465935230255127,
+      "logps/chosen": -32.2774772644043,
+      "logps/rejected": -32.245147705078125,
+      "loss": 0.4887,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.11396588385105133,
+      "rewards/margins": 0.04570900648832321,
+      "rewards/rejected": 0.06825686991214752,
       "step": 90
     },
     {
       "epoch": 0.26,
       "learning_rate": 4.626245458345211e-06,
-      "logits/chosen": -2.0381903648376465,
-      "logits/rejected": -2.0362119674682617,
-      "logps/chosen": -32.13701629638672,
-      "logps/rejected": -31.295801162719727,
-      "loss": 0.6595,
-      "rewards/accuracies": 0.6499999761581421,
-      "rewards/chosen": 0.06229018047451973,
-      "rewards/margins": 0.08004496991634369,
-      "rewards/rejected": -0.017754793167114258,
+      "logits/chosen": -2.040524482727051,
+      "logits/rejected": -2.0385289192199707,
+      "logps/chosen": -32.03118133544922,
+      "logps/rejected": -31.205692291259766,
+      "loss": 0.4799,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.12579473853111267,
+      "rewards/margins": 0.08948288857936859,
+      "rewards/rejected": 0.036311857402324677,
       "step": 100
     },
     {
       "epoch": 0.26,
-      "eval_logits/chosen": -2.2335634231567383,
-      "eval_logits/rejected": -2.228721857070923,
-      "eval_logps/chosen": -34.00627899169922,
-      "eval_logps/rejected": -37.514984130859375,
-      "eval_loss": 0.6900185346603394,
-      "eval_rewards/accuracies": 0.5253322720527649,
-      "eval_rewards/chosen": 0.016964510083198547,
-      "eval_rewards/margins": 0.015982570126652718,
-      "eval_rewards/rejected": 0.0009819410042837262,
-      "eval_runtime": 145.7969,
-      "eval_samples_per_second": 2.353,
+      "eval_logits/chosen": -2.234907865524292,
+      "eval_logits/rejected": -2.2300779819488525,
+      "eval_logps/chosen": -33.90293884277344,
+      "eval_logps/rejected": -37.394290924072266,
+      "eval_loss": 0.4987158477306366,
+      "eval_rewards/accuracies": 0.5103820562362671,
+      "eval_rewards/chosen": 0.0789671465754509,
+      "eval_rewards/margins": 0.005569713190197945,
+      "eval_rewards/rejected": 0.0733974277973175,
+      "eval_runtime": 145.9867,
+      "eval_samples_per_second": 2.35,
       "eval_steps_per_second": 0.295,
       "step": 100
     },
     {
       "epoch": 0.29,
       "learning_rate": 4.498257201263691e-06,
-      "logits/chosen": -1.9929395914077759,
-      "logits/rejected": -1.99057936668396,
-      "logps/chosen": -33.06591796875,
-      "logps/rejected": -34.01008605957031,
-      "loss": 0.6678,
-      "rewards/accuracies": 0.675000011920929,
-      "rewards/chosen": 0.10673947632312775,
-      "rewards/margins": 0.09516827017068863,
-      "rewards/rejected": 0.011571208015084267,
+      "logits/chosen": -1.9966247081756592,
+      "logits/rejected": -1.9942585229873657,
+      "logps/chosen": -33.005943298339844,
+      "logps/rejected": -33.908599853515625,
+      "loss": 0.4799,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.14272192120552063,
+      "rewards/margins": 0.07025818526744843,
+      "rewards/rejected": 0.07246376574039459,
       "step": 110
     },
     {
       "epoch": 0.31,
       "learning_rate": 4.353806263777678e-06,
-      "logits/chosen": -2.0045437812805176,
-      "logits/rejected": -1.9962146282196045,
-      "logps/chosen": -32.33503341674805,
-      "logps/rejected": -32.12450408935547,
-      "loss": 0.68,
-      "rewards/accuracies": 0.5625,
-      "rewards/chosen": 0.06592197716236115,
-      "rewards/margins": 0.043721526861190796,
-      "rewards/rejected": 0.02220045030117035,
+      "logits/chosen": -2.0064611434936523,
+      "logits/rejected": -1.9981132745742798,
+      "logps/chosen": -32.16605758666992,
+      "logps/rejected": -31.968135833740234,
+      "loss": 0.4883,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": 0.16730833053588867,
+      "rewards/margins": 0.0512862391769886,
+      "rewards/rejected": 0.11602209508419037,
       "step": 120
     },
     {
       "epoch": 0.34,
       "learning_rate": 4.1940827077152755e-06,
-      "logits/chosen": -2.032952308654785,
-      "logits/rejected": -2.024994134902954,
-      "logps/chosen": -30.302433013916016,
-      "logps/rejected": -32.04313278198242,
-      "loss": 0.6609,
-      "rewards/accuracies": 0.612500011920929,
-      "rewards/chosen": 0.10397056490182877,
-      "rewards/margins": 0.09804753214120865,
-      "rewards/rejected": 0.0059230271726846695,
+      "logits/chosen": -2.0353620052337646,
+      "logits/rejected": -2.0273985862731934,
+      "logps/chosen": -30.145355224609375,
+      "logps/rejected": -31.892078399658203,
+      "loss": 0.477,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.19821622967720032,
+      "rewards/margins": 0.10165951400995255,
+      "rewards/rejected": 0.09655670821666718,
       "step": 130
     },
     {
       "epoch": 0.36,
       "learning_rate": 4.0204024186666215e-06,
-      "logits/chosen": -1.9635902643203735,
-      "logits/rejected": -1.9738080501556396,
-      "logps/chosen": -31.201534271240234,
-      "logps/rejected": -32.556739807128906,
-      "loss": 0.6343,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": 0.13087674975395203,
-      "rewards/margins": 0.1439923495054245,
-      "rewards/rejected": -0.013115609996020794,
+      "logits/chosen": -1.965121865272522,
+      "logits/rejected": -1.9753497838974,
+      "logps/chosen": -31.059722900390625,
+      "logps/rejected": -32.391807556152344,
+      "loss": 0.4695,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": 0.21596479415893555,
+      "rewards/margins": 0.13012099266052246,
+      "rewards/rejected": 0.08584379404783249,
       "step": 140
     },
     {
       "epoch": 0.39,
       "learning_rate": 3.834196265035119e-06,
-      "logits/chosen": -1.8755298852920532,
-      "logits/rejected": -1.876691222190857,
-      "logps/chosen": -33.89242172241211,
-      "logps/rejected": -34.753570556640625,
-      "loss": 0.6272,
+      "logits/chosen": -1.8772761821746826,
+      "logits/rejected": -1.8784434795379639,
+      "logps/chosen": -33.682029724121094,
+      "logps/rejected": -34.56599044799805,
+      "loss": 0.4566,
       "rewards/accuracies": 0.612500011920929,
-      "rewards/chosen": 0.1801605522632599,
-      "rewards/margins": 0.17795029282569885,
-      "rewards/rejected": 0.002210266888141632,
+      "rewards/chosen": 0.30639737844467163,
+      "rewards/margins": 0.19163966178894043,
+      "rewards/rejected": 0.11475770175457001,
       "step": 150
     },
     {
       "epoch": 0.42,
       "learning_rate": 3.636998309800573e-06,
-      "logits/chosen": -1.9265025854110718,
-      "logits/rejected": -1.9230976104736328,
-      "logps/chosen": -35.98413848876953,
-      "logps/rejected": -32.69154357910156,
-      "loss": 0.6574,
-      "rewards/accuracies": 0.574999988079071,
-      "rewards/chosen": 0.11252401769161224,
-      "rewards/margins": 0.09379850327968597,
-      "rewards/rejected": 0.01872551441192627,
+      "logits/chosen": -1.9283088445663452,
+      "logits/rejected": -1.9249660968780518,
+      "logps/chosen": -35.78284454345703,
+      "logps/rejected": -32.46989059448242,
+      "loss": 0.4802,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.23330166935920715,
+      "rewards/margins": 0.08158595860004425,
+      "rewards/rejected": 0.1517156958580017,
       "step": 160
     },
     {
       "epoch": 0.44,
       "learning_rate": 3.4304331721118078e-06,
-      "logits/chosen": -2.0262577533721924,
-      "logits/rejected": -2.0189335346221924,
-      "logps/chosen": -33.45969772338867,
-      "logps/rejected": -31.372516632080078,
-      "loss": 0.6076,
-      "rewards/accuracies": 0.675000011920929,
-      "rewards/chosen": 0.20289841294288635,
-      "rewards/margins": 0.21529710292816162,
-      "rewards/rejected": -0.012398697435855865,
+      "logits/chosen": -2.0310397148132324,
+      "logits/rejected": -2.0237185955047607,
+      "logps/chosen": -33.21892547607422,
+      "logps/rejected": -31.1767520904541,
+      "loss": 0.443,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": 0.347359299659729,
+      "rewards/margins": 0.24229975044727325,
+      "rewards/rejected": 0.10505956411361694,
       "step": 170
     },
     {
       "epoch": 0.47,
       "learning_rate": 3.2162026428305436e-06,
-      "logits/chosen": -2.0338973999023438,
-      "logits/rejected": -2.0391509532928467,
-      "logps/chosen": -32.194793701171875,
-      "logps/rejected": -32.42069625854492,
-      "loss": 0.6221,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": 0.20999665558338165,
-      "rewards/margins": 0.17132976651191711,
-      "rewards/rejected": 0.03866690397262573,
+      "logits/chosen": -2.037632703781128,
+      "logits/rejected": -2.0428080558776855,
+      "logps/chosen": -31.97260093688965,
+      "logps/rejected": -32.14643096923828,
+      "loss": 0.4662,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.34331104159355164,
+      "rewards/margins": 0.1400861144065857,
+      "rewards/rejected": 0.20322492718696594,
       "step": 180
     },
     {
       "epoch": 0.49,
       "learning_rate": 2.996071664294641e-06,
-      "logits/chosen": -2.035045862197876,
-      "logits/rejected": -2.032278060913086,
-      "logps/chosen": -31.230976104736328,
-      "logps/rejected": -31.29391860961914,
-      "loss": 0.6403,
-      "rewards/accuracies": 0.637499988079071,
-      "rewards/chosen": 0.1547853648662567,
-      "rewards/margins": 0.14020755887031555,
-      "rewards/rejected": 0.014577840454876423,
+      "logits/chosen": -2.0373456478118896,
+      "logits/rejected": -2.0346312522888184,
+      "logps/chosen": -31.077922821044922,
+      "logps/rejected": -31.089244842529297,
+      "loss": 0.4725,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.24661760032176971,
+      "rewards/margins": 0.10923590511083603,
+      "rewards/rejected": 0.13738170266151428,
       "step": 190
     },
     {
       "epoch": 0.52,
       "learning_rate": 2.7718537898066833e-06,
-      "logits/chosen": -1.9042125940322876,
-      "logits/rejected": -1.9088417291641235,
-      "logps/chosen": -31.285167694091797,
-      "logps/rejected": -32.79944610595703,
-      "loss": 0.6189,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": 0.192669078707695,
-      "rewards/margins": 0.1977730393409729,
-      "rewards/rejected": -0.0051039643585681915,
+      "logits/chosen": -1.9075931310653687,
+      "logits/rejected": -1.9122419357299805,
+      "logps/chosen": -31.088512420654297,
+      "logps/rejected": -32.591243743896484,
+      "loss": 0.4548,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": 0.3106623888015747,
+      "rewards/margins": 0.190850168466568,
+      "rewards/rejected": 0.11981221288442612,
       "step": 200
     },
     {
       "epoch": 0.52,
-      "eval_logits/chosen": -2.230701446533203,
-      "eval_logits/rejected": -2.2258734703063965,
-      "eval_logps/chosen": -34.0167350769043,
-      "eval_logps/rejected": -37.53435134887695,
-      "eval_loss": 0.6894960403442383,
+      "eval_logits/chosen": -2.233458995819092,
+      "eval_logits/rejected": -2.2286527156829834,
+      "eval_logps/chosen": -33.769569396972656,
+      "eval_logps/rejected": -37.28458786010742,
+      "eval_loss": 0.4955619275569916,
       "eval_rewards/accuracies": 0.5340532064437866,
-      "eval_rewards/chosen": 0.010691030882298946,
-      "eval_rewards/margins": 0.02133062295615673,
-      "eval_rewards/rejected": -0.010639593005180359,
-      "eval_runtime": 145.5559,
-      "eval_samples_per_second": 2.356,
+      "eval_rewards/chosen": 0.15899108350276947,
+      "eval_rewards/margins": 0.019772635772824287,
+      "eval_rewards/rejected": 0.13921843469142914,
+      "eval_runtime": 145.6335,
+      "eval_samples_per_second": 2.355,
       "eval_steps_per_second": 0.295,
       "step": 200
     },
     {
       "epoch": 0.55,
       "learning_rate": 2.5453962426402006e-06,
-      "logits/chosen": -2.018108367919922,
-      "logits/rejected": -2.0287561416625977,
-      "logps/chosen": -31.745798110961914,
-      "logps/rejected": -33.90629577636719,
-      "loss": 0.6305,
-      "rewards/accuracies": 0.675000011920929,
-      "rewards/chosen": 0.13530384004116058,
-      "rewards/margins": 0.16146349906921387,
-      "rewards/rejected": -0.026159662753343582,
+      "logits/chosen": -2.020599365234375,
+      "logits/rejected": -2.0312187671661377,
+      "logps/chosen": -31.51775550842285,
+      "logps/rejected": -33.68301773071289,
+      "loss": 0.462,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": 0.2721289098262787,
+      "rewards/margins": 0.16431835293769836,
+      "rewards/rejected": 0.10781057924032211,
       "step": 210
     },
     {
       "epoch": 0.57,
       "learning_rate": 2.3185646976551794e-06,
-      "logits/chosen": -1.9105422496795654,
-      "logits/rejected": -1.9253056049346924,
-      "logps/chosen": -29.849069595336914,
-      "logps/rejected": -31.58382797241211,
-      "loss": 0.6251,
-      "rewards/accuracies": 0.6625000238418579,
-      "rewards/chosen": 0.15748274326324463,
-      "rewards/margins": 0.17287404835224152,
-      "rewards/rejected": -0.015391310676932335,
+      "logits/chosen": -1.9131078720092773,
+      "logits/rejected": -1.9278091192245483,
+      "logps/chosen": -29.563827514648438,
+      "logps/rejected": -31.399028778076172,
+      "loss": 0.4463,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": 0.3286263942718506,
+      "rewards/margins": 0.23313641548156738,
+      "rewards/rejected": 0.09548995643854141,
       "step": 220
     },
     {
       "epoch": 0.6,
       "learning_rate": 2.0932279108998323e-06,
-      "logits/chosen": -1.966581106185913,
-      "logits/rejected": -1.9705555438995361,
-      "logps/chosen": -33.059837341308594,
-      "logps/rejected": -31.605152130126953,
-      "loss": 0.6,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": 0.21457910537719727,
-      "rewards/margins": 0.2480895221233368,
-      "rewards/rejected": -0.033510446548461914,
+      "logits/chosen": -1.969555139541626,
+      "logits/rejected": -1.9735543727874756,
+      "logps/chosen": -32.82552719116211,
+      "logps/rejected": -31.39508628845215,
+      "loss": 0.4396,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": 0.3551639914512634,
+      "rewards/margins": 0.26263612508773804,
+      "rewards/rejected": 0.0925278514623642,
       "step": 230
     },
     {
       "epoch": 0.62,
       "learning_rate": 1.8712423238279358e-06,
-      "logits/chosen": -1.9644801616668701,
-      "logits/rejected": -1.9426641464233398,
-      "logps/chosen": -33.819881439208984,
-      "logps/rejected": -35.105430603027344,
-      "loss": 0.5826,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": 0.20031993091106415,
-      "rewards/margins": 0.2930986285209656,
-      "rewards/rejected": -0.09277870506048203,
+      "logits/chosen": -1.9689340591430664,
+      "logits/rejected": -1.947165846824646,
+      "logps/chosen": -33.572689056396484,
+      "logps/rejected": -34.84184265136719,
+      "loss": 0.4355,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.3486354947090149,
+      "rewards/margins": 0.2832663357257843,
+      "rewards/rejected": 0.0653691440820694,
       "step": 240
     },
     {
       "epoch": 0.65,
       "learning_rate": 1.6544367689701824e-06,
-      "logits/chosen": -2.005814552307129,
-      "logits/rejected": -2.002516746520996,
-      "logps/chosen": -32.663055419921875,
-      "logps/rejected": -36.233436584472656,
-      "loss": 0.632,
-      "rewards/accuracies": 0.6625000238418579,
-      "rewards/chosen": 0.15720662474632263,
-      "rewards/margins": 0.16231317818164825,
-      "rewards/rejected": -0.0051065413281321526,
+      "logits/chosen": -2.0102758407592773,
+      "logits/rejected": -2.006990909576416,
+      "logps/chosen": -32.49989700317383,
+      "logps/rejected": -35.99126434326172,
+      "loss": 0.472,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": 0.2551037669181824,
+      "rewards/margins": 0.11490776389837265,
+      "rewards/rejected": 0.1401960253715515,
       "step": 250
     },
     {
       "epoch": 0.68,
       "learning_rate": 1.4445974030621963e-06,
-      "logits/chosen": -1.8735284805297852,
-      "logits/rejected": -1.8710968494415283,
-      "logps/chosen": -33.964805603027344,
-      "logps/rejected": -35.50111389160156,
-      "loss": 0.6428,
+      "logits/chosen": -1.877345323562622,
+      "logits/rejected": -1.8749182224273682,
+      "logps/chosen": -33.7043571472168,
+      "logps/rejected": -35.259422302246094,
+      "loss": 0.4671,
       "rewards/accuracies": 0.6499999761581421,
-      "rewards/chosen": 0.13415411114692688,
-      "rewards/margins": 0.13204893469810486,
-      "rewards/rejected": 0.0021051731891930103,
+      "rewards/chosen": 0.2904246747493744,
+      "rewards/margins": 0.14330394566059113,
+      "rewards/rejected": 0.14712072908878326,
       "step": 260
     },
     {
       "epoch": 0.7,
       "learning_rate": 1.243452991757889e-06,
-      "logits/chosen": -1.8585532903671265,
-      "logits/rejected": -1.8561296463012695,
-      "logps/chosen": -34.17288589477539,
-      "logps/rejected": -31.792491912841797,
-      "loss": 0.6391,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": 0.13482868671417236,
-      "rewards/margins": 0.14752644300460815,
-      "rewards/rejected": -0.012697766534984112,
+      "logits/chosen": -1.8626378774642944,
+      "logits/rejected": -1.8601328134536743,
+      "logps/chosen": -33.90363311767578,
+      "logps/rejected": -31.588598251342773,
+      "loss": 0.457,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": 0.29638105630874634,
+      "rewards/margins": 0.1867421716451645,
+      "rewards/rejected": 0.10963888466358185,
       "step": 270
     },
     {
       "epoch": 0.73,
       "learning_rate": 1.0526606671603523e-06,
-      "logits/chosen": -1.9624073505401611,
-      "logits/rejected": -1.9519150257110596,
-      "logps/chosen": -34.95975875854492,
-      "logps/rejected": -31.84967041015625,
-      "loss": 0.5947,
-      "rewards/accuracies": 0.7749999761581421,
-      "rewards/chosen": 0.2381301373243332,
-      "rewards/margins": 0.2433377206325531,
-      "rewards/rejected": -0.005207589361816645,
+      "logits/chosen": -1.965528130531311,
+      "logits/rejected": -1.9551260471343994,
+      "logps/chosen": -34.74420166015625,
+      "logps/rejected": -31.622207641601562,
+      "loss": 0.4442,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.3674633204936981,
+      "rewards/margins": 0.23619195818901062,
+      "rewards/rejected": 0.1312713772058487,
       "step": 280
     },
     {
       "epoch": 0.75,
       "learning_rate": 8.737922755071455e-07,
-      "logits/chosen": -2.056440830230713,
-      "logits/rejected": -2.0415446758270264,
-      "logps/chosen": -30.704137802124023,
-      "logps/rejected": -32.5970458984375,
-      "loss": 0.6654,
-      "rewards/accuracies": 0.5625,
-      "rewards/chosen": 0.13198992609977722,
-      "rewards/margins": 0.09911760687828064,
-      "rewards/rejected": 0.03287229686975479,
+      "logits/chosen": -2.0614726543426514,
+      "logits/rejected": -2.0466229915618896,
+      "logps/chosen": -30.37356948852539,
+      "logps/rejected": -32.3351936340332,
+      "loss": 0.4671,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.33033472299575806,
+      "rewards/margins": 0.14034779369831085,
+      "rewards/rejected": 0.1899869292974472,
       "step": 290
     },
     {
       "epoch": 0.78,
       "learning_rate": 7.08321427484816e-07,
-      "logits/chosen": -1.927122712135315,
-      "logits/rejected": -1.924602746963501,
-      "logps/chosen": -32.37379455566406,
-      "logps/rejected": -30.879268646240234,
-      "loss": 0.5582,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": 0.3319894075393677,
-      "rewards/margins": 0.3694665729999542,
-      "rewards/rejected": -0.03747714310884476,
+      "logits/chosen": -1.9327377080917358,
+      "logits/rejected": -1.9302759170532227,
+      "logps/chosen": -32.0928955078125,
+      "logps/rejected": -30.652236938476562,
+      "loss": 0.41,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": 0.5005279779434204,
+      "rewards/margins": 0.4017825126647949,
+      "rewards/rejected": 0.09874548017978668,
       "step": 300
     },
     {
       "epoch": 0.78,
-      "eval_logits/chosen": -2.226900815963745,
-      "eval_logits/rejected": -2.2220816612243652,
-      "eval_logps/chosen": -34.03526306152344,
-      "eval_logps/rejected": -37.56266403198242,
-      "eval_loss": 0.6872054934501648,
-      "eval_rewards/accuracies": 0.5598006844520569,
-      "eval_rewards/chosen": -0.0004257837135810405,
-      "eval_rewards/margins": 0.027201363816857338,
-      "eval_rewards/rejected": -0.027627145871520042,
-      "eval_runtime": 145.6667,
+      "eval_logits/chosen": -2.2308623790740967,
+      "eval_logits/rejected": -2.226059913635254,
+      "eval_logps/chosen": -33.76142501831055,
+      "eval_logps/rejected": -37.284828186035156,
+      "eval_loss": 0.4937135577201843,
+      "eval_rewards/accuracies": 0.5361295938491821,
+      "eval_rewards/chosen": 0.16387464106082916,
+      "eval_rewards/margins": 0.02479831501841545,
+      "eval_rewards/rejected": 0.1390763521194458,
+      "eval_runtime": 145.6766,
       "eval_samples_per_second": 2.355,
       "eval_steps_per_second": 0.295,
       "step": 300
@@ -493,122 +493,122 @@
     {
       "epoch": 0.81,
       "learning_rate": 5.576113578589035e-07,
-      "logits/chosen": -1.9124987125396729,
-      "logits/rejected": -1.9092620611190796,
-      "logps/chosen": -31.285348892211914,
-      "logps/rejected": -33.75069046020508,
-      "loss": 0.6188,
-      "rewards/accuracies": 0.675000011920929,
-      "rewards/chosen": 0.19080111384391785,
-      "rewards/margins": 0.1959228217601776,
-      "rewards/rejected": -0.0051217032596468925,
+      "logits/chosen": -1.916757345199585,
+      "logits/rejected": -1.9136326313018799,
+      "logps/chosen": -31.0240421295166,
+      "logps/rejected": -33.553138732910156,
+      "loss": 0.4465,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.3475847542285919,
+      "rewards/margins": 0.23417536914348602,
+      "rewards/rejected": 0.1134093850851059,
       "step": 310
     },
     {
       "epoch": 0.83,
       "learning_rate": 4.229036944380913e-07,
-      "logits/chosen": -1.9616212844848633,
-      "logits/rejected": -1.9494349956512451,
-      "logps/chosen": -34.30357360839844,
-      "logps/rejected": -33.64542770385742,
-      "loss": 0.5999,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": 0.17549821734428406,
-      "rewards/margins": 0.23945149779319763,
-      "rewards/rejected": -0.06395327299833298,
+      "logits/chosen": -1.9669981002807617,
+      "logits/rejected": -1.9548496007919312,
+      "logps/chosen": -34.02072525024414,
+      "logps/rejected": -33.40859603881836,
+      "loss": 0.4379,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.3452104926109314,
+      "rewards/margins": 0.26706355810165405,
+      "rewards/rejected": 0.07814691960811615,
       "step": 320
     },
     {
       "epoch": 0.86,
       "learning_rate": 3.053082288996112e-07,
-      "logits/chosen": -1.997859239578247,
-      "logits/rejected": -1.9964158535003662,
-      "logps/chosen": -33.130615234375,
-      "logps/rejected": -32.511531829833984,
-      "loss": 0.6073,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": 0.21310487389564514,
-      "rewards/margins": 0.23227325081825256,
-      "rewards/rejected": -0.01916835829615593,
+      "logits/chosen": -2.0023112297058105,
+      "logits/rejected": -2.0009615421295166,
+      "logps/chosen": -32.86361312866211,
+      "logps/rejected": -32.265541076660156,
+      "loss": 0.4439,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.37330904603004456,
+      "rewards/margins": 0.24488253891468048,
+      "rewards/rejected": 0.12842652201652527,
       "step": 330
     },
     {
       "epoch": 0.88,
       "learning_rate": 2.0579377374915805e-07,
-      "logits/chosen": -2.083962917327881,
-      "logits/rejected": -2.06827449798584,
-      "logps/chosen": -33.732425689697266,
-      "logps/rejected": -33.07551193237305,
-      "loss": 0.5985,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": 0.2681804299354553,
-      "rewards/margins": 0.24266552925109863,
-      "rewards/rejected": 0.025514895096421242,
+      "logits/chosen": -2.0890090465545654,
+      "logits/rejected": -2.073373317718506,
+      "logps/chosen": -33.47917556762695,
+      "logps/rejected": -32.80946350097656,
+      "loss": 0.4446,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.42013058066368103,
+      "rewards/margins": 0.23498812317848206,
+      "rewards/rejected": 0.18514248728752136,
       "step": 340
     },
     {
       "epoch": 0.91,
       "learning_rate": 1.2518018074041684e-07,
-      "logits/chosen": -1.956364631652832,
-      "logits/rejected": -1.9555118083953857,
-      "logps/chosen": -32.79610824584961,
-      "logps/rejected": -32.512969970703125,
-      "loss": 0.5758,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": 0.2915918231010437,
-      "rewards/margins": 0.3202829957008362,
-      "rewards/rejected": -0.028691178187727928,
+      "logits/chosen": -1.9615952968597412,
+      "logits/rejected": -1.9607601165771484,
+      "logps/chosen": -32.57384490966797,
+      "logps/rejected": -32.25366973876953,
+      "loss": 0.4326,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.4249538779258728,
+      "rewards/margins": 0.2980697453022003,
+      "rewards/rejected": 0.12688413262367249,
       "step": 350
     },
     {
       "epoch": 0.94,
       "learning_rate": 6.41315865106129e-08,
-      "logits/chosen": -1.9118818044662476,
-      "logits/rejected": -1.9221813678741455,
-      "logps/chosen": -31.855304718017578,
-      "logps/rejected": -35.32331848144531,
-      "loss": 0.6036,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": 0.2096320390701294,
-      "rewards/margins": 0.23036828637123108,
-      "rewards/rejected": -0.02073623239994049,
+      "logits/chosen": -1.9177497625350952,
+      "logits/rejected": -1.9280321598052979,
+      "logps/chosen": -31.600200653076172,
+      "logps/rejected": -35.003135681152344,
+      "loss": 0.4555,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.3626941442489624,
+      "rewards/margins": 0.19132137298583984,
+      "rewards/rejected": 0.17137278616428375,
       "step": 360
     },
     {
       "epoch": 0.96,
       "learning_rate": 2.3150941078050325e-08,
-      "logits/chosen": -2.0520732402801514,
-      "logits/rejected": -2.0456154346466064,
-      "logps/chosen": -33.327049255371094,
-      "logps/rejected": -29.233760833740234,
-      "loss": 0.6124,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": 0.19495204091072083,
-      "rewards/margins": 0.19935496151447296,
-      "rewards/rejected": -0.004402949940413237,
+      "logits/chosen": -2.0559232234954834,
+      "logits/rejected": -2.0493996143341064,
+      "logps/chosen": -33.01537322998047,
+      "logps/rejected": -28.992477416992188,
+      "loss": 0.4418,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": 0.3819587230682373,
+      "rewards/margins": 0.2415906935930252,
+      "rewards/rejected": 0.14036807417869568,
       "step": 370
     },
     {
       "epoch": 0.99,
       "learning_rate": 2.575864278703266e-09,
-      "logits/chosen": -1.911370038986206,
-      "logits/rejected": -1.9135433435440063,
-      "logps/chosen": -33.83781814575195,
-      "logps/rejected": -30.931133270263672,
-      "loss": 0.5854,
+      "logits/chosen": -1.9163579940795898,
+      "logits/rejected": -1.9185268878936768,
+      "logps/chosen": -33.650962829589844,
+      "logps/rejected": -30.753406524658203,
+      "loss": 0.4369,
       "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": 0.2407412976026535,
-      "rewards/margins": 0.27584755420684814,
-      "rewards/rejected": -0.035106249153614044,
+      "rewards/chosen": 0.3528522849082947,
+      "rewards/margins": 0.28131863474845886,
+      "rewards/rejected": 0.07153360545635223,
       "step": 380
     },
     {
       "epoch": 1.0,
       "step": 385,
       "total_flos": 0.0,
-      "train_loss": 0.6364745784115482,
-      "train_runtime": 3251.6479,
-      "train_samples_per_second": 0.947,
+      "train_loss": 0.46439293204963983,
+      "train_runtime": 3253.2697,
+      "train_samples_per_second": 0.946,
       "train_steps_per_second": 0.118
     }
   ],