Model save

README.md

@@ -1,13 +1,10 @@
 ---
 library_name: peft
 tags:
-- alignment-handbook
 - trl
 - dpo
 - generated_from_trainer
 base_model: NbAiLab/nb-gpt-j-6B-v2
-datasets:
-- hugodk-sch/aftonposten_title_prefs
 model-index:
 - name: aftonposten-6b-align-scan
   results: []
@@ -18,17 +15,17 @@ should probably proofread and complete it, then remove this comment. -->
 
 # aftonposten-6b-align-scan
 
-This model is a fine-tuned version of [data/ap-gpt-j-6b-sft-qlora-04-08](https://huggingface.co/data/ap-gpt-j-6b-sft-qlora-04-08) on the hugodk-sch/aftonposten_title_prefs dataset.
+This model is a fine-tuned version of [NbAiLab/nb-gpt-j-6B-v2](https://huggingface.co/NbAiLab/nb-gpt-j-6B-v2) on an unknown dataset.
 It achieves the following results on the evaluation set:
-- Loss: 0.4961
-- Rewards/chosen: 0.1159
-- Rewards/rejected: 0.0997
-- Rewards/accuracies: 0.5278
-- Rewards/margins: 0.0162
-- Logps/rejected: -37.2674
-- Logps/chosen: -33.7448
-- Logits/rejected: -2.2272
-- Logits/chosen: -2.2321
+- Loss: 1.5962
+- Rewards/chosen: -0.0200
+- Rewards/rejected: -0.0335
+- Rewards/accuracies: 0.5166
+- Rewards/margins: 0.0135
+- Logps/rejected: -37.6004
+- Logps/chosen: -34.0845
+- Logits/rejected: -2.2263
+- Logits/chosen: -2.2312
 
 ## Model description
 
@@ -63,9 +60,9 @@ The following hyperparameters were used during training:
 
 | Training Loss | Epoch | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
 |:-------------:|:-----:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
-| 0.4845        | 0.26  | 100  | 0.4982          | 0.0465         | 0.0373           | 0.5511             | 0.0092          | -37.4234       | -33.9184     | -2.2298         | -2.2347       |
-| 0.466         | 0.52  | 200  | 0.4965          | 0.1019         | 0.0871           | 0.5453             | 0.0148          | -37.2989       | -33.7799     | -2.2288         | -2.2337       |
-| 0.4349        | 0.78  | 300  | 0.4962          | 0.1141         | 0.0985           | 0.5357             | 0.0156          | -37.2704       | -33.7494     | -2.2270         | -2.2318       |
+| 1.4583        | 0.26  | 100  | 1.6162          | 0.0017         | -0.0028          | 0.5245             | 0.0045          | -37.5236       | -34.0303     | -2.2308         | -2.2357       |
+| 1.279         | 0.52  | 200  | 1.6100          | -0.0192        | -0.0303          | 0.5257             | 0.0111          | -37.5923       | -34.0825     | -2.2311         | -2.2359       |
+| 1.0043        | 0.78  | 300  | 1.5962          | -0.0200        | -0.0335          | 0.5166             | 0.0135          | -37.6004       | -34.0845     | -2.2263         | -2.2312       |
 
 
 ### Framework versions

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:538346c77f456b9ce2d6d00e03fc31ed4a139c37f6a7aa9b71e229b63ad33209
+oid sha256:7585d4186dbe799bb620e4c24d5acfe6c521e77772a31ac410bfb6f0584ad197
 size 176183216

all_results.json

@@ -1,20 +1,7 @@
 {
     "epoch": 1.0,
-    "eval_logits/chosen": -2.2320711612701416,
-    "eval_logits/rejected": -2.2272377014160156,
-    "eval_logps/chosen": -33.7447624206543,
-    "eval_logps/rejected": -37.26742935180664,
-    "eval_loss": 0.49609801173210144,
-    "eval_rewards/accuracies": 0.5278239250183105,
-    "eval_rewards/chosen": 0.11591599881649017,
-    "eval_rewards/margins": 0.016239874064922333,
-    "eval_rewards/rejected": 0.09967613220214844,
-    "eval_runtime": 145.6745,
-    "eval_samples": 343,
-    "eval_samples_per_second": 2.355,
-    "eval_steps_per_second": 0.295,
-    "train_loss": 0.4751699732495593,
-    "train_runtime": 3253.0203,
+    "train_loss": 1.3100324989913346,
+    "train_runtime": 3250.9757,
     "train_samples": 3079,
     "train_samples_per_second": 0.947,
     "train_steps_per_second": 0.118

train_results.json

@@ -1,7 +1,7 @@
 {
     "epoch": 1.0,
-    "train_loss": 0.4751699732495593,
-    "train_runtime": 3253.0203,
+    "train_loss": 1.3100324989913346,
+    "train_runtime": 3250.9757,
     "train_samples": 3079,
     "train_samples_per_second": 0.947,
     "train_steps_per_second": 0.118

trainer_state.json

@@ -15,7 +15,7 @@
       "logits/rejected": -1.7377450466156006,
       "logps/chosen": -29.553977966308594,
       "logps/rejected": -42.813133239746094,
-      "loss": 0.5,
+      "loss": 1.5625,
       "rewards/accuracies": 0.0,
       "rewards/chosen": 0.0,
       "rewards/margins": 0.0,
@@ -25,589 +25,589 @@
     {
       "epoch": 0.03,
       "learning_rate": 1.282051282051282e-06,
-      "logits/chosen": -1.8668408393859863,
-      "logits/rejected": -1.8711602687835693,
-      "logps/chosen": -36.98978042602539,
-      "logps/rejected": -33.66878890991211,
-      "loss": 0.4962,
+      "logits/chosen": -1.8663330078125,
+      "logits/rejected": -1.870657205581665,
+      "logps/chosen": -36.98442077636719,
+      "logps/rejected": -33.6639289855957,
+      "loss": 1.4725,
       "rewards/accuracies": 0.5416666865348816,
-      "rewards/chosen": 0.006728413049131632,
-      "rewards/margins": 0.022216208279132843,
-      "rewards/rejected": -0.015487794764339924,
+      "rewards/chosen": 0.008871171623468399,
+      "rewards/margins": 0.02241421490907669,
+      "rewards/rejected": -0.013543044216930866,
       "step": 10
     },
     {
       "epoch": 0.05,
       "learning_rate": 2.564102564102564e-06,
-      "logits/chosen": -1.9970359802246094,
-      "logits/rejected": -1.9996885061264038,
-      "logps/chosen": -29.635208129882812,
-      "logps/rejected": -29.063350677490234,
-      "loss": 0.4994,
-      "rewards/accuracies": 0.5249999761581421,
-      "rewards/chosen": 0.002794977743178606,
-      "rewards/margins": 0.002757500857114792,
-      "rewards/rejected": 3.747665323317051e-05,
+      "logits/chosen": -1.9977548122406006,
+      "logits/rejected": -2.0003960132598877,
+      "logps/chosen": -29.64242172241211,
+      "logps/rejected": -29.04966163635254,
+      "loss": 1.6323,
+      "rewards/accuracies": 0.4124999940395355,
+      "rewards/chosen": -9.128078818321228e-05,
+      "rewards/margins": -0.005604482255876064,
+      "rewards/rejected": 0.005513200536370277,
       "step": 20
     },
     {
       "epoch": 0.08,
       "learning_rate": 3.846153846153847e-06,
-      "logits/chosen": -1.9208602905273438,
-      "logits/rejected": -1.9181534051895142,
-      "logps/chosen": -31.40317726135254,
-      "logps/rejected": -33.23335647583008,
-      "loss": 0.4982,
-      "rewards/accuracies": 0.550000011920929,
-      "rewards/chosen": 0.005168457515537739,
-      "rewards/margins": 0.011126698926091194,
-      "rewards/rejected": -0.005958239547908306,
+      "logits/chosen": -1.9204576015472412,
+      "logits/rejected": -1.917781114578247,
+      "logps/chosen": -31.413188934326172,
+      "logps/rejected": -33.22998046875,
+      "loss": 1.5666,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": 0.0011628343490883708,
+      "rewards/margins": 0.005770112853497267,
+      "rewards/rejected": -0.0046072788536548615,
       "step": 30
     },
     {
       "epoch": 0.1,
       "learning_rate": 4.999896948438434e-06,
-      "logits/chosen": -2.017474889755249,
-      "logits/rejected": -2.008759021759033,
-      "logps/chosen": -32.54490661621094,
-      "logps/rejected": -32.49110794067383,
-      "loss": 0.4994,
-      "rewards/accuracies": 0.4749999940395355,
-      "rewards/chosen": 0.012759355828166008,
-      "rewards/margins": 0.004133358132094145,
-      "rewards/rejected": 0.008625999093055725,
+      "logits/chosen": -2.0172924995422363,
+      "logits/rejected": -2.0085625648498535,
+      "logps/chosen": -32.583003997802734,
+      "logps/rejected": -32.50304412841797,
+      "loss": 1.6493,
+      "rewards/accuracies": 0.48750001192092896,
+      "rewards/chosen": -0.0024785413406789303,
+      "rewards/margins": -0.006329345051199198,
+      "rewards/rejected": 0.00385080324485898,
       "step": 40
     },
     {
       "epoch": 0.13,
       "learning_rate": 4.987541037542187e-06,
-      "logits/chosen": -1.8631575107574463,
-      "logits/rejected": -1.8523809909820557,
-      "logps/chosen": -33.509056091308594,
-      "logps/rejected": -35.39984893798828,
-      "loss": 0.5001,
-      "rewards/accuracies": 0.48750001192092896,
-      "rewards/chosen": 0.01936880685389042,
-      "rewards/margins": -0.001629653968848288,
-      "rewards/rejected": 0.020998459309339523,
+      "logits/chosen": -1.8622093200683594,
+      "logits/rejected": -1.851438283920288,
+      "logps/chosen": -33.5463981628418,
+      "logps/rejected": -35.46432113647461,
+      "loss": 1.569,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": 0.004430866800248623,
+      "rewards/margins": 0.009221290238201618,
+      "rewards/rejected": -0.004790422972291708,
       "step": 50
     },
     {
       "epoch": 0.16,
       "learning_rate": 4.954691471941119e-06,
-      "logits/chosen": -1.9419567584991455,
-      "logits/rejected": -1.9438903331756592,
-      "logps/chosen": -32.50743103027344,
-      "logps/rejected": -33.188419342041016,
-      "loss": 0.491,
-      "rewards/accuracies": 0.6000000238418579,
-      "rewards/chosen": 0.03710051625967026,
-      "rewards/margins": 0.04391607269644737,
-      "rewards/rejected": -0.006815555039793253,
+      "logits/chosen": -1.939429521560669,
+      "logits/rejected": -1.941382646560669,
+      "logps/chosen": -32.560157775878906,
+      "logps/rejected": -33.21000671386719,
+      "loss": 1.5224,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.016012197360396385,
+      "rewards/margins": 0.03146379441022873,
+      "rewards/rejected": -0.015451604500412941,
       "step": 60
     },
     {
       "epoch": 0.18,
       "learning_rate": 4.901618883413549e-06,
-      "logits/chosen": -2.0737738609313965,
-      "logits/rejected": -2.0787465572357178,
-      "logps/chosen": -33.928836822509766,
-      "logps/rejected": -36.540794372558594,
-      "loss": 0.4971,
-      "rewards/accuracies": 0.512499988079071,
-      "rewards/chosen": 0.023859737440943718,
-      "rewards/margins": 0.01262708194553852,
-      "rewards/rejected": 0.01123266015201807,
+      "logits/chosen": -2.0719385147094727,
+      "logits/rejected": -2.0768914222717285,
+      "logps/chosen": -33.97148513793945,
+      "logps/rejected": -36.61540985107422,
+      "loss": 1.5627,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": 0.006800562143325806,
+      "rewards/margins": 0.025414401665329933,
+      "rewards/rejected": -0.018613843247294426,
       "step": 70
     },
     {
       "epoch": 0.21,
       "learning_rate": 4.828760511501322e-06,
-      "logits/chosen": -1.934011697769165,
-      "logits/rejected": -1.9371341466903687,
-      "logps/chosen": -34.23366928100586,
-      "logps/rejected": -34.542274475097656,
-      "loss": 0.4864,
-      "rewards/accuracies": 0.6000000238418579,
-      "rewards/chosen": 0.07765541225671768,
-      "rewards/margins": 0.057602256536483765,
-      "rewards/rejected": 0.020053153857588768,
+      "logits/chosen": -1.9341154098510742,
+      "logits/rejected": -1.9372339248657227,
+      "logps/chosen": -34.31813049316406,
+      "logps/rejected": -34.64340591430664,
+      "loss": 1.3398,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.043871380388736725,
+      "rewards/margins": 0.06426960229873657,
+      "rewards/rejected": -0.020398225635290146,
       "step": 80
     },
     {
       "epoch": 0.23,
       "learning_rate": 4.7367166013034295e-06,
-      "logits/chosen": -1.9421571493148804,
-      "logits/rejected": -1.9466804265975952,
-      "logps/chosen": -32.304595947265625,
-      "logps/rejected": -32.284873962402344,
-      "loss": 0.4909,
-      "rewards/accuracies": 0.6499999761581421,
-      "rewards/chosen": 0.0651295855641365,
-      "rewards/margins": 0.035516757518053055,
-      "rewards/rejected": 0.02961282804608345,
+      "logits/chosen": -1.9430510997772217,
+      "logits/rejected": -1.9475826025009155,
+      "logps/chosen": -32.405487060546875,
+      "logps/rejected": -32.3362922668457,
+      "loss": 1.6048,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": 0.02477400377392769,
+      "rewards/margins": 0.01572720892727375,
+      "rewards/rejected": 0.009046795777976513,
       "step": 90
     },
     {
       "epoch": 0.26,
       "learning_rate": 4.626245458345211e-06,
-      "logits/chosen": -2.0406017303466797,
-      "logits/rejected": -2.038613796234131,
-      "logps/chosen": -32.027652740478516,
-      "logps/rejected": -31.224151611328125,
-      "loss": 0.4845,
-      "rewards/accuracies": 0.637499988079071,
-      "rewards/chosen": 0.08527366816997528,
-      "rewards/margins": 0.0684497207403183,
-      "rewards/rejected": 0.01682395115494728,
+      "logits/chosen": -2.040834426879883,
+      "logits/rejected": -2.0388479232788086,
+      "logps/chosen": -32.189510345458984,
+      "logps/rejected": -31.297454833984375,
+      "loss": 1.4583,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.020529210567474365,
+      "rewards/margins": 0.0330255962908268,
+      "rewards/rejected": -0.012496383860707283,
       "step": 100
     },
     {
       "epoch": 0.26,
-      "eval_logits/chosen": -2.234653949737549,
-      "eval_logits/rejected": -2.229806423187256,
-      "eval_logps/chosen": -33.91841125488281,
-      "eval_logps/rejected": -37.42335891723633,
-      "eval_loss": 0.4981803297996521,
-      "eval_rewards/accuracies": 0.5510797500610352,
-      "eval_rewards/chosen": 0.04645563289523125,
-      "eval_rewards/margins": 0.009152057580649853,
-      "eval_rewards/rejected": 0.03730357065796852,
-      "eval_runtime": 145.8684,
-      "eval_samples_per_second": 2.351,
+      "eval_logits/chosen": -2.2356765270233154,
+      "eval_logits/rejected": -2.2308189868927,
+      "eval_logps/chosen": -34.03032302856445,
+      "eval_logps/rejected": -37.523555755615234,
+      "eval_loss": 1.6161867380142212,
+      "eval_rewards/accuracies": 0.5245016813278198,
+      "eval_rewards/chosen": 0.0016898562898859382,
+      "eval_rewards/margins": 0.004464337136596441,
+      "eval_rewards/rejected": -0.0027744807302951813,
+      "eval_runtime": 145.7904,
+      "eval_samples_per_second": 2.353,
       "eval_steps_per_second": 0.295,
       "step": 100
     },
     {
       "epoch": 0.29,
       "learning_rate": 4.498257201263691e-06,
-      "logits/chosen": -1.9949238300323486,
-      "logits/rejected": -1.9925572872161865,
-      "logps/chosen": -32.99268341064453,
-      "logps/rejected": -33.90182113647461,
-      "loss": 0.4869,
-      "rewards/accuracies": 0.5874999761581421,
-      "rewards/chosen": 0.10045422613620758,
-      "rewards/margins": 0.049433451145887375,
-      "rewards/rejected": 0.051020748913288116,
+      "logits/chosen": -1.996921181678772,
+      "logits/rejected": -1.9945523738861084,
+      "logps/chosen": -33.126094818115234,
+      "logps/rejected": -34.01633071899414,
+      "loss": 1.6379,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.04708842188119888,
+      "rewards/margins": 0.041870661079883575,
+      "rewards/rejected": 0.005217757076025009,
       "step": 110
     },
     {
       "epoch": 0.31,
       "learning_rate": 4.353806263777678e-06,
-      "logits/chosen": -2.0055007934570312,
-      "logits/rejected": -1.997157335281372,
-      "logps/chosen": -32.192996978759766,
-      "logps/rejected": -32.01173400878906,
-      "loss": 0.491,
-      "rewards/accuracies": 0.574999988079071,
-      "rewards/chosen": 0.10076460987329483,
-      "rewards/margins": 0.04085635766386986,
-      "rewards/rejected": 0.05990824103355408,
+      "logits/chosen": -2.0087766647338867,
+      "logits/rejected": -2.0004258155822754,
+      "logps/chosen": -32.360713958740234,
+      "logps/rejected": -32.135719299316406,
+      "loss": 1.5576,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": 0.03367554023861885,
+      "rewards/margins": 0.023361390456557274,
+      "rewards/rejected": 0.010314146056771278,
       "step": 120
     },
     {
       "epoch": 0.34,
       "learning_rate": 4.1940827077152755e-06,
-      "logits/chosen": -2.034147262573242,
-      "logits/rejected": -2.026184558868408,
-      "logps/chosen": -30.194936752319336,
-      "logps/rejected": -31.905689239501953,
-      "loss": 0.4876,
+      "logits/chosen": -2.036501407623291,
+      "logits/rejected": -2.0285134315490723,
+      "logps/chosen": -30.350208282470703,
+      "logps/rejected": -32.078697204589844,
+      "loss": 1.4709,
       "rewards/accuracies": 0.574999988079071,
-      "rewards/chosen": 0.11231068521738052,
-      "rewards/margins": 0.05338172987103462,
-      "rewards/rejected": 0.0589289590716362,
+      "rewards/chosen": 0.05020326375961304,
+      "rewards/margins": 0.06047834828495979,
+      "rewards/rejected": -0.010275078006088734,
       "step": 130
     },
     {
       "epoch": 0.36,
       "learning_rate": 4.0204024186666215e-06,
-      "logits/chosen": -1.9643388986587524,
-      "logits/rejected": -1.9745395183563232,
-      "logps/chosen": -31.08599853515625,
-      "logps/rejected": -32.422943115234375,
-      "loss": 0.4795,
-      "rewards/accuracies": 0.6625000238418579,
-      "rewards/chosen": 0.13346607983112335,
-      "rewards/margins": 0.08869143575429916,
-      "rewards/rejected": 0.044774629175662994,
+      "logits/chosen": -1.966408371925354,
+      "logits/rejected": -1.9766807556152344,
+      "logps/chosen": -31.250171661376953,
+      "logps/rejected": -32.605186462402344,
+      "loss": 1.2633,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.06779567152261734,
+      "rewards/margins": 0.09591736644506454,
+      "rewards/rejected": -0.028121691197156906,
       "step": 140
     },
     {
       "epoch": 0.39,
       "learning_rate": 3.834196265035119e-06,
-      "logits/chosen": -1.8776963949203491,
-      "logits/rejected": -1.8788686990737915,
-      "logps/chosen": -33.690345764160156,
-      "logps/rejected": -34.572776794433594,
-      "loss": 0.4699,
-      "rewards/accuracies": 0.574999988079071,
-      "rewards/chosen": 0.20093801617622375,
-      "rewards/margins": 0.12714678049087524,
-      "rewards/rejected": 0.07379122078418732,
+      "logits/chosen": -1.8785550594329834,
+      "logits/rejected": -1.8797214031219482,
+      "logps/chosen": -33.973148345947266,
+      "logps/rejected": -34.82178497314453,
+      "loss": 1.271,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.08781836926937103,
+      "rewards/margins": 0.11363118886947632,
+      "rewards/rejected": -0.02581283450126648,
       "step": 150
     },
     {
       "epoch": 0.42,
       "learning_rate": 3.636998309800573e-06,
-      "logits/chosen": -1.9286582469940186,
-      "logits/rejected": -1.9253017902374268,
-      "logps/chosen": -35.773475646972656,
-      "logps/rejected": -32.47566604614258,
-      "loss": 0.4851,
-      "rewards/accuracies": 0.625,
-      "rewards/chosen": 0.15928010642528534,
-      "rewards/margins": 0.06044477969408035,
-      "rewards/rejected": 0.09883531928062439,
+      "logits/chosen": -1.9306548833847046,
+      "logits/rejected": -1.927215337753296,
+      "logps/chosen": -36.05388259887695,
+      "logps/rejected": -32.733726501464844,
+      "loss": 1.3903,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": 0.04711749404668808,
+      "rewards/margins": 0.051506586372852325,
+      "rewards/rejected": -0.004389096051454544,
       "step": 160
     },
     {
       "epoch": 0.44,
       "learning_rate": 3.4304331721118078e-06,
-      "logits/chosen": -2.0298352241516113,
-      "logits/rejected": -2.0225348472595215,
-      "logps/chosen": -33.22509002685547,
-      "logps/rejected": -31.19403648376465,
-      "loss": 0.4603,
-      "rewards/accuracies": 0.675000011920929,
-      "rewards/chosen": 0.22910937666893005,
-      "rewards/margins": 0.165984109044075,
-      "rewards/rejected": 0.06312531232833862,
+      "logits/chosen": -2.0320611000061035,
+      "logits/rejected": -2.0246951580047607,
+      "logps/chosen": -33.545772552490234,
+      "logps/rejected": -31.430078506469727,
+      "loss": 1.1164,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": 0.10083547979593277,
+      "rewards/margins": 0.13212835788726807,
+      "rewards/rejected": -0.0312928780913353,
       "step": 170
     },
     {
       "epoch": 0.47,
       "learning_rate": 3.2162026428305436e-06,
-      "logits/chosen": -2.036612033843994,
-      "logits/rejected": -2.041813373565674,
-      "logps/chosen": -31.960119247436523,
-      "logps/rejected": -32.171165466308594,
-      "loss": 0.4734,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": 0.23386716842651367,
-      "rewards/margins": 0.1082783117890358,
-      "rewards/rejected": 0.12558886408805847,
+      "logits/chosen": -2.0383529663085938,
+      "logits/rejected": -2.0436065196990967,
+      "logps/chosen": -32.30766677856445,
+      "logps/rejected": -32.46817398071289,
+      "loss": 1.2603,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.09484858065843582,
+      "rewards/margins": 0.08806191384792328,
+      "rewards/rejected": 0.006786657031625509,
       "step": 180
     },
     {
       "epoch": 0.49,
       "learning_rate": 2.996071664294641e-06,
-      "logits/chosen": -2.037198066711426,
-      "logits/rejected": -2.0344595909118652,
-      "logps/chosen": -31.026615142822266,
-      "logps/rejected": -31.082998275756836,
-      "loss": 0.4781,
-      "rewards/accuracies": 0.625,
-      "rewards/chosen": 0.184935063123703,
-      "rewards/margins": 0.09084881842136383,
-      "rewards/rejected": 0.09408621490001678,
+      "logits/chosen": -2.0393786430358887,
+      "logits/rejected": -2.036611795425415,
+      "logps/chosen": -31.301233291625977,
+      "logps/rejected": -31.35910415649414,
+      "loss": 1.2965,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.0750882551074028,
+      "rewards/margins": 0.09144291281700134,
+      "rewards/rejected": -0.016354653984308243,
       "step": 190
     },
     {
       "epoch": 0.52,
       "learning_rate": 2.7718537898066833e-06,
-      "logits/chosen": -1.908818006515503,
-      "logits/rejected": -1.913496732711792,
-      "logps/chosen": -31.075199127197266,
-      "logps/rejected": -32.616241455078125,
-      "loss": 0.466,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": 0.2124340832233429,
-      "rewards/margins": 0.14255891740322113,
-      "rewards/rejected": 0.06987515836954117,
+      "logits/chosen": -1.9094291925430298,
+      "logits/rejected": -1.9140870571136475,
+      "logps/chosen": -31.389511108398438,
+      "logps/rejected": -32.832645416259766,
+      "loss": 1.279,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": 0.08670923858880997,
+      "rewards/margins": 0.10339467227458954,
+      "rewards/rejected": -0.01668544113636017,
       "step": 200
     },
     {
       "epoch": 0.52,
-      "eval_logits/chosen": -2.23366379737854,
-      "eval_logits/rejected": -2.2288384437561035,
-      "eval_logps/chosen": -33.779903411865234,
-      "eval_logps/rejected": -37.29892349243164,
-      "eval_loss": 0.4965229630470276,
-      "eval_rewards/accuracies": 0.545265793800354,
-      "eval_rewards/chosen": 0.10185908526182175,
-      "eval_rewards/margins": 0.014780867844820023,
-      "eval_rewards/rejected": 0.08707821369171143,
-      "eval_runtime": 145.8117,
-      "eval_samples_per_second": 2.352,
+      "eval_logits/chosen": -2.2359395027160645,
+      "eval_logits/rejected": -2.231079339981079,
+      "eval_logps/chosen": -34.08246994018555,
+      "eval_logps/rejected": -37.59228515625,
+      "eval_loss": 1.6099865436553955,
+      "eval_rewards/accuracies": 0.525747537612915,
+      "eval_rewards/chosen": -0.019167287275195122,
+      "eval_rewards/margins": 0.011099184863269329,
+      "eval_rewards/rejected": -0.030266473069787025,
+      "eval_runtime": 145.6348,
+      "eval_samples_per_second": 2.355,
       "eval_steps_per_second": 0.295,
       "step": 200
     },
     {
       "epoch": 0.55,
       "learning_rate": 2.5453962426402006e-06,
-      "logits/chosen": -2.0208163261413574,
-      "logits/rejected": -2.0314111709594727,
-      "logps/chosen": -31.514019012451172,
-      "logps/rejected": -33.690643310546875,
-      "loss": 0.4728,
-      "rewards/accuracies": 0.6000000238418579,
-      "rewards/chosen": 0.18291443586349487,
-      "rewards/margins": 0.114091657102108,
-      "rewards/rejected": 0.06882277131080627,
+      "logits/chosen": -2.021488666534424,
+      "logits/rejected": -2.0321521759033203,
+      "logps/chosen": -31.773427963256836,
+      "logps/rejected": -33.9339485168457,
+      "loss": 1.1843,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.07915131002664566,
+      "rewards/margins": 0.107650026679039,
+      "rewards/rejected": -0.02849872037768364,
       "step": 210
     },
     {
       "epoch": 0.57,
       "learning_rate": 2.3185646976551794e-06,
-      "logits/chosen": -1.9141031503677368,
-      "logits/rejected": -1.9288082122802734,
-      "logps/chosen": -29.5712947845459,
-      "logps/rejected": -31.429983139038086,
-      "loss": 0.4615,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": 0.2160976380109787,
-      "rewards/margins": 0.16482076048851013,
-      "rewards/rejected": 0.05127686262130737,
+      "logits/chosen": -1.9136203527450562,
+      "logits/rejected": -1.9284076690673828,
+      "logps/chosen": -29.9388370513916,
+      "logps/rejected": -31.61223793029785,
+      "loss": 1.2413,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.06908205896615982,
+      "rewards/margins": 0.09070716798305511,
+      "rewards/rejected": -0.021625112742185593,
       "step": 220
     },
     {
       "epoch": 0.6,
       "learning_rate": 2.0932279108998323e-06,
-      "logits/chosen": -1.9712814092636108,
-      "logits/rejected": -1.975285291671753,
-      "logps/chosen": -32.82429885864258,
-      "logps/rejected": -31.416866302490234,
-      "loss": 0.4561,
-      "rewards/accuracies": 0.6625000238418579,
-      "rewards/chosen": 0.23726816475391388,
-      "rewards/margins": 0.18429425358772278,
-      "rewards/rejected": 0.052973903715610504,
+      "logits/chosen": -1.9700790643692017,
+      "logits/rejected": -1.9740597009658813,
+      "logps/chosen": -33.1496467590332,
+      "logps/rejected": -31.65804100036621,
+      "loss": 1.1206,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": 0.10712933540344238,
+      "rewards/margins": 0.15062502026557922,
+      "rewards/rejected": -0.043495677411556244,
       "step": 230
     },
     {
       "epoch": 0.62,
       "learning_rate": 1.8712423238279358e-06,
-      "logits/chosen": -1.969780683517456,
-      "logits/rejected": -1.9480432271957397,
-      "logps/chosen": -33.583518981933594,
-      "logps/rejected": -34.8461799621582,
-      "loss": 0.4567,
-      "rewards/accuracies": 0.675000011920929,
-      "rewards/chosen": 0.22809162735939026,
-      "rewards/margins": 0.18624703586101532,
-      "rewards/rejected": 0.04184458404779434,
+      "logits/chosen": -1.9689995050430298,
+      "logits/rejected": -1.9470970630645752,
+      "logps/chosen": -33.9182243347168,
+      "logps/rejected": -35.09490966796875,
+      "loss": 1.073,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.09421048313379288,
+      "rewards/margins": 0.15185722708702087,
+      "rewards/rejected": -0.0576467402279377,
       "step": 240
     },
     {
       "epoch": 0.65,
       "learning_rate": 1.6544367689701824e-06,
-      "logits/chosen": -2.0109124183654785,
-      "logits/rejected": -2.0076212882995605,
-      "logps/chosen": -32.46331024169922,
-      "logps/rejected": -35.97381591796875,
-      "loss": 0.4799,
-      "rewards/accuracies": 0.5625,
-      "rewards/chosen": 0.18470348417758942,
-      "rewards/margins": 0.08426074683666229,
-      "rewards/rejected": 0.10044274479150772,
+      "logits/chosen": -2.010103702545166,
+      "logits/rejected": -2.006786346435547,
+      "logps/chosen": -32.75348663330078,
+      "logps/rejected": -36.29840087890625,
+      "loss": 1.2351,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.0686345100402832,
+      "rewards/margins": 0.09802549332380295,
+      "rewards/rejected": -0.029390990734100342,
       "step": 250
     },
     {
       "epoch": 0.68,
       "learning_rate": 1.4445974030621963e-06,
-      "logits/chosen": -1.878321647644043,
-      "logits/rejected": -1.8758872747421265,
-      "logps/chosen": -33.721397399902344,
-      "logps/rejected": -35.270362854003906,
-      "loss": 0.4785,
-      "rewards/accuracies": 0.6000000238418579,
-      "rewards/chosen": 0.1868000328540802,
-      "rewards/margins": 0.09309660643339157,
-      "rewards/rejected": 0.09370341151952744,
+      "logits/chosen": -1.877804160118103,
+      "logits/rejected": -1.8753639459609985,
+      "logps/chosen": -34.015769958496094,
+      "logps/rejected": -35.56401824951172,
+      "loss": 1.2236,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.06905192881822586,
+      "rewards/margins": 0.09281176328659058,
+      "rewards/rejected": -0.023759832605719566,
       "step": 260
     },
     {
       "epoch": 0.7,
       "learning_rate": 1.243452991757889e-06,
-      "logits/chosen": -1.8631956577301025,
-      "logits/rejected": -1.8606828451156616,
-      "logps/chosen": -33.90094757080078,
-      "logps/rejected": -31.57466697692871,
-      "loss": 0.4722,
-      "rewards/accuracies": 0.637499988079071,
-      "rewards/chosen": 0.198659747838974,
-      "rewards/margins": 0.11999478191137314,
-      "rewards/rejected": 0.07866497337818146,
+      "logits/chosen": -1.8633472919464111,
+      "logits/rejected": -1.860896348953247,
+      "logps/chosen": -34.244041442871094,
+      "logps/rejected": -31.82059669494629,
+      "loss": 1.2837,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.06142274662852287,
+      "rewards/margins": 0.08112844079732895,
+      "rewards/rejected": -0.019705694168806076,
       "step": 270
     },
     {
       "epoch": 0.73,
       "learning_rate": 1.0526606671603523e-06,
-      "logits/chosen": -1.9663877487182617,
-      "logits/rejected": -1.9559704065322876,
-      "logps/chosen": -34.72657775878906,
-      "logps/rejected": -31.63601303100586,
-      "loss": 0.4585,
-      "rewards/accuracies": 0.6499999761581421,
-      "rewards/chosen": 0.25202488899230957,
-      "rewards/margins": 0.17003390192985535,
-      "rewards/rejected": 0.08199100196361542,
+      "logits/chosen": -1.9670900106430054,
+      "logits/rejected": -1.9565184116363525,
+      "logps/chosen": -35.00214767456055,
+      "logps/rejected": -31.88429832458496,
+      "loss": 0.9577,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": 0.14179818332195282,
+      "rewards/margins": 0.15912050008773804,
+      "rewards/rejected": -0.017322316765785217,
       "step": 280
     },
     {
       "epoch": 0.75,
       "learning_rate": 8.737922755071455e-07,
-      "logits/chosen": -2.062107801437378,
-      "logits/rejected": -2.0472733974456787,
-      "logps/chosen": -30.40212059020996,
-      "logps/rejected": -32.340721130371094,
-      "loss": 0.4799,
-      "rewards/accuracies": 0.625,
-      "rewards/chosen": 0.2088027447462082,
-      "rewards/margins": 0.08435753732919693,
-      "rewards/rejected": 0.12444518506526947,
+      "logits/chosen": -2.0620455741882324,
+      "logits/rejected": -2.0470833778381348,
+      "logps/chosen": -30.75039291381836,
+      "logps/rejected": -32.6963996887207,
+      "loss": 1.3545,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": 0.06949276477098465,
+      "rewards/margins": 0.08731748908758163,
+      "rewards/rejected": -0.017824724316596985,
       "step": 290
     },
     {
       "epoch": 0.78,
       "learning_rate": 7.08321427484816e-07,
-      "logits/chosen": -1.9330482482910156,
-      "logits/rejected": -1.9305979013442993,
-      "logps/chosen": -32.06965255737305,
-      "logps/rejected": -30.65035629272461,
-      "loss": 0.4349,
-      "rewards/accuracies": 0.6625000238418579,
-      "rewards/chosen": 0.3429831862449646,
-      "rewards/margins": 0.27640262246131897,
-      "rewards/rejected": 0.06658058613538742,
+      "logits/chosen": -1.933719277381897,
+      "logits/rejected": -1.931168556213379,
+      "logps/chosen": -32.519874572753906,
+      "logps/rejected": -30.880207061767578,
+      "loss": 1.0043,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": 0.1628943681716919,
+      "rewards/margins": 0.1882532835006714,
+      "rewards/rejected": -0.02535891905426979,
       "step": 300
     },
     {
       "epoch": 0.78,
-      "eval_logits/chosen": -2.2318336963653564,
-      "eval_logits/rejected": -2.2270052433013916,
-      "eval_logps/chosen": -33.749427795410156,
-      "eval_logps/rejected": -37.27041244506836,
-      "eval_loss": 0.4961945414543152,
-      "eval_rewards/accuracies": 0.5357142686843872,
-      "eval_rewards/chosen": 0.11405016481876373,
-      "eval_rewards/margins": 0.015567691065371037,
-      "eval_rewards/rejected": 0.09848246723413467,
-      "eval_runtime": 145.5479,
-      "eval_samples_per_second": 2.357,
+      "eval_logits/chosen": -2.2311501502990723,
+      "eval_logits/rejected": -2.226292610168457,
+      "eval_logps/chosen": -34.0844841003418,
+      "eval_logps/rejected": -37.60036087036133,
+      "eval_loss": 1.5962141752243042,
+      "eval_rewards/accuracies": 0.5166113376617432,
+      "eval_rewards/chosen": -0.019972510635852814,
+      "eval_rewards/margins": 0.013524654321372509,
+      "eval_rewards/rejected": -0.0334971658885479,
+      "eval_runtime": 145.5939,
+      "eval_samples_per_second": 2.356,
       "eval_steps_per_second": 0.295,
       "step": 300
     },
     {
       "epoch": 0.81,
       "learning_rate": 5.576113578589035e-07,
-      "logits/chosen": -1.9179494380950928,
-      "logits/rejected": -1.9148216247558594,
-      "logps/chosen": -31.037755966186523,
-      "logps/rejected": -33.56406021118164,
-      "loss": 0.4639,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": 0.22623713314533234,
-      "rewards/margins": 0.15500028431415558,
-      "rewards/rejected": 0.07123686373233795,
+      "logits/chosen": -1.9168885946273804,
+      "logits/rejected": -1.9136085510253906,
+      "logps/chosen": -31.36187744140625,
+      "logps/rejected": -33.820152282714844,
+      "loss": 1.1726,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": 0.09658866375684738,
+      "rewards/margins": 0.127788707613945,
+      "rewards/rejected": -0.03120005689561367,
       "step": 310
     },
     {
       "epoch": 0.83,
       "learning_rate": 4.229036944380913e-07,
-      "logits/chosen": -1.9682966470718384,
-      "logits/rejected": -1.9561439752578735,
-      "logps/chosen": -34.023921966552734,
-      "logps/rejected": -33.415985107421875,
-      "loss": 0.4575,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": 0.22886168956756592,
-      "rewards/margins": 0.17971986532211304,
-      "rewards/rejected": 0.04914180561900139,
+      "logits/chosen": -1.9674562215805054,
+      "logits/rejected": -1.955171823501587,
+      "logps/chosen": -34.4316291809082,
+      "logps/rejected": -33.64500045776367,
+      "loss": 1.1609,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": 0.06577816605567932,
+      "rewards/margins": 0.10824279487133026,
+      "rewards/rejected": -0.04246463626623154,
       "step": 320
     },
     {
       "epoch": 0.86,
       "learning_rate": 3.053082288996112e-07,
-      "logits/chosen": -2.003788709640503,
-      "logits/rejected": -2.002437114715576,
-      "logps/chosen": -32.8862419128418,
-      "logps/rejected": -32.21629333496094,
-      "loss": 0.4679,
-      "rewards/accuracies": 0.675000011920929,
-      "rewards/chosen": 0.23982055485248566,
-      "rewards/margins": 0.13450448215007782,
-      "rewards/rejected": 0.10531606525182724,
+      "logits/chosen": -2.002774238586426,
+      "logits/rejected": -2.0013155937194824,
+      "logps/chosen": -33.31989288330078,
+      "logps/rejected": -32.59081268310547,
+      "loss": 1.119,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": 0.06635875999927521,
+      "rewards/margins": 0.1108517199754715,
+      "rewards/rejected": -0.04449295997619629,
       "step": 330
     },
     {
       "epoch": 0.88,
       "learning_rate": 2.0579377374915805e-07,
-      "logits/chosen": -2.0905163288116455,
-      "logits/rejected": -2.0749027729034424,
-      "logps/chosen": -33.47309112548828,
-      "logps/rejected": -32.81334686279297,
-      "loss": 0.4618,
-      "rewards/accuracies": 0.675000011920929,
-      "rewards/chosen": 0.2825208008289337,
-      "rewards/margins": 0.16064420342445374,
-      "rewards/rejected": 0.12187659740447998,
+      "logits/chosen": -2.089189052581787,
+      "logits/rejected": -2.0734314918518066,
+      "logps/chosen": -33.87842559814453,
+      "logps/rejected": -33.1202278137207,
+      "loss": 1.1504,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.12038698047399521,
+      "rewards/margins": 0.12126438319683075,
+      "rewards/rejected": -0.0008773911977186799,
       "step": 340
     },
     {
       "epoch": 0.91,
       "learning_rate": 1.2518018074041684e-07,
-      "logits/chosen": -1.9627025127410889,
-      "logits/rejected": -1.9618685245513916,
-      "logps/chosen": -32.528175354003906,
-      "logps/rejected": -32.22235107421875,
-      "loss": 0.4522,
-      "rewards/accuracies": 0.637499988079071,
-      "rewards/chosen": 0.30156710743904114,
-      "rewards/margins": 0.20444798469543457,
-      "rewards/rejected": 0.09711913019418716,
+      "logits/chosen": -1.9613533020019531,
+      "logits/rejected": -1.9604451656341553,
+      "logps/chosen": -32.962162017822266,
+      "logps/rejected": -32.54551315307617,
+      "loss": 1.1702,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": 0.12797386944293976,
+      "rewards/margins": 0.16012001037597656,
+      "rewards/rejected": -0.032146163284778595,
       "step": 350
     },
     {
       "epoch": 0.94,
       "learning_rate": 6.41315865106129e-08,
-      "logits/chosen": -1.918320655822754,
-      "logits/rejected": -1.9285932779312134,
-      "logps/chosen": -31.606945037841797,
-      "logps/rejected": -34.98893356323242,
-      "loss": 0.4718,
-      "rewards/accuracies": 0.625,
-      "rewards/chosen": 0.2390974760055542,
-      "rewards/margins": 0.11916828155517578,
-      "rewards/rejected": 0.11992917954921722,
+      "logits/chosen": -1.9175068140029907,
+      "logits/rejected": -1.9278007745742798,
+      "logps/chosen": -31.958675384521484,
+      "logps/rejected": -35.354461669921875,
+      "loss": 1.1577,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": 0.09840553998947144,
+      "rewards/margins": 0.12468685954809189,
+      "rewards/rejected": -0.026281306520104408,
       "step": 360
     },
     {
       "epoch": 0.96,
       "learning_rate": 2.3150941078050325e-08,
-      "logits/chosen": -2.0581445693969727,
-      "logits/rejected": -2.051628828048706,
-      "logps/chosen": -33.017967224121094,
-      "logps/rejected": -28.989696502685547,
-      "loss": 0.4612,
-      "rewards/accuracies": 0.7749999761581421,
-      "rewards/chosen": 0.253601610660553,
-      "rewards/margins": 0.15891048312187195,
-      "rewards/rejected": 0.09469114243984222,
+      "logits/chosen": -2.056452751159668,
+      "logits/rejected": -2.0499649047851562,
+      "logps/chosen": -33.426971435546875,
+      "logps/rejected": -29.26825523376465,
+      "loss": 1.1458,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": 0.0899999737739563,
+      "rewards/margins": 0.10673253238201141,
+      "rewards/rejected": -0.01673254929482937,
       "step": 370
     },
     {
       "epoch": 0.99,
       "learning_rate": 2.575864278703266e-09,
-      "logits/chosen": -1.9176830053329468,
-      "logits/rejected": -1.919847846031189,
-      "logps/chosen": -33.614540100097656,
-      "logps/rejected": -30.760555267333984,
-      "loss": 0.4519,
-      "rewards/accuracies": 0.7875000238418579,
-      "rewards/chosen": 0.24980488419532776,
-      "rewards/margins": 0.2049761563539505,
-      "rewards/rejected": 0.04482869431376457,
+      "logits/chosen": -1.916009545326233,
+      "logits/rejected": -1.9182236194610596,
+      "logps/chosen": -33.94841384887695,
+      "logps/rejected": -30.931049346923828,
+      "loss": 1.1073,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": 0.11625485122203827,
+      "rewards/margins": 0.13962247967720032,
+      "rewards/rejected": -0.02336762472987175,
       "step": 380
     },
     {
       "epoch": 1.0,
       "step": 385,
       "total_flos": 0.0,
-      "train_loss": 0.4751699732495593,
-      "train_runtime": 3253.0203,
+      "train_loss": 1.3100324989913346,
+      "train_runtime": 3250.9757,
       "train_samples_per_second": 0.947,
       "train_steps_per_second": 0.118
     }