Model save

README.md

@@ -1,13 +1,10 @@
 ---
 library_name: peft
 tags:
-- alignment-handbook
 - trl
 - dpo
 - generated_from_trainer
 base_model: NbAiLab/nb-gpt-j-6B-v2
-datasets:
-- hugodk-sch/aftonposten_title_prefs
 model-index:
 - name: aftonposten-6b-align-scan
   results: []
@@ -18,17 +15,17 @@ should probably proofread and complete it, then remove this comment. -->
 
 # aftonposten-6b-align-scan
 
-This model is a fine-tuned version of [data/ap-gpt-j-6b-sft-qlora-04-08](https://huggingface.co/data/ap-gpt-j-6b-sft-qlora-04-08) on the hugodk-sch/aftonposten_title_prefs dataset.
+This model is a fine-tuned version of [NbAiLab/nb-gpt-j-6B-v2](https://huggingface.co/NbAiLab/nb-gpt-j-6B-v2) on an unknown dataset.
 It achieves the following results on the evaluation set:
-- Loss: 0.6931
-- Rewards/chosen: -0.0260
-- Rewards/rejected: -0.0334
-- Rewards/accuracies: 0.5071
-- Rewards/margins: 0.0074
-- Logps/rejected: -37.6001
-- Logps/chosen: -34.0996
-- Logits/rejected: -2.2227
-- Logits/chosen: -2.2275
+- Loss: 0.4962
+- Rewards/chosen: 0.1141
+- Rewards/rejected: 0.0985
+- Rewards/accuracies: 0.5357
+- Rewards/margins: 0.0156
+- Logps/rejected: -37.2704
+- Logps/chosen: -33.7494
+- Logits/rejected: -2.2270
+- Logits/chosen: -2.2318
 
 ## Model description
 
@@ -63,9 +60,9 @@ The following hyperparameters were used during training:
 
 | Training Loss | Epoch | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
 |:-------------:|:-----:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
-| 0.6726        | 0.26  | 100  | 0.6903          | -0.0039        | -0.0143          | 0.5482             | 0.0105          | -37.5524       | -34.0442     | -2.2286         | -2.2334       |
-| 0.6419        | 0.52  | 200  | 0.6932          | -0.0126        | -0.0196          | 0.5042             | 0.0071          | -37.5657       | -34.0660     | -2.2261         | -2.2310       |
-| 0.5902        | 0.78  | 300  | 0.6926          | -0.0248        | -0.0328          | 0.4954             | 0.0080          | -37.5987       | -34.0966     | -2.2231         | -2.2279       |
+| 0.4845        | 0.26  | 100  | 0.4982          | 0.0465         | 0.0373           | 0.5511             | 0.0092          | -37.4234       | -33.9184     | -2.2298         | -2.2347       |
+| 0.466         | 0.52  | 200  | 0.4965          | 0.1019         | 0.0871           | 0.5453             | 0.0148          | -37.2989       | -33.7799     | -2.2288         | -2.2337       |
+| 0.4349        | 0.78  | 300  | 0.4962          | 0.1141         | 0.0985           | 0.5357             | 0.0156          | -37.2704       | -33.7494     | -2.2270         | -2.2318       |
 
 
 ### Framework versions

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c83758ade092bee7a7c47d5df91f1f7738a7e7c23657f9385413031647727945
+oid sha256:7e5eebef15a07da863c5fc08cc2014b1ba6f08bebfa75043de56daeec484862e
 size 176183216

all_results.json

@@ -1,21 +1,8 @@
 {
     "epoch": 1.0,
-    "eval_logits/chosen": -2.2275288105010986,
-    "eval_logits/rejected": -2.2226884365081787,
-    "eval_logps/chosen": -34.09955978393555,
-    "eval_logps/rejected": -37.600067138671875,
-    "eval_loss": 0.693101167678833,
-    "eval_rewards/accuracies": 0.5070598125457764,
-    "eval_rewards/chosen": -0.026001954451203346,
-    "eval_rewards/margins": 0.007375751622021198,
-    "eval_rewards/rejected": -0.03337770700454712,
-    "eval_runtime": 145.5528,
-    "eval_samples": 343,
-    "eval_samples_per_second": 2.357,
-    "eval_steps_per_second": 0.295,
-    "train_loss": 0.6506414413452148,
-    "train_runtime": 3254.0644,
+    "train_loss": 0.4751699732495593,
+    "train_runtime": 3253.0203,
     "train_samples": 3079,
-    "train_samples_per_second": 0.946,
+    "train_samples_per_second": 0.947,
     "train_steps_per_second": 0.118
 }

train_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 1.0,
-    "train_loss": 0.6506414413452148,
-    "train_runtime": 3254.0644,
+    "train_loss": 0.4751699732495593,
+    "train_runtime": 3253.0203,
     "train_samples": 3079,
-    "train_samples_per_second": 0.946,
+    "train_samples_per_second": 0.947,
     "train_steps_per_second": 0.118
 }

trainer_state.json

@@ -15,7 +15,7 @@
       "logits/rejected": -1.7377450466156006,
       "logps/chosen": -29.553977966308594,
       "logps/rejected": -42.813133239746094,
-      "loss": 0.6931,
+      "loss": 0.5,
       "rewards/accuracies": 0.0,
       "rewards/chosen": 0.0,
       "rewards/margins": 0.0,
@@ -25,590 +25,590 @@
     {
       "epoch": 0.03,
       "learning_rate": 1.282051282051282e-06,
-      "logits/chosen": -1.8668746948242188,
-      "logits/rejected": -1.8712046146392822,
-      "logps/chosen": -36.99528884887695,
-      "logps/rejected": -33.6615104675293,
-      "loss": 0.6855,
-      "rewards/accuracies": 0.5,
-      "rewards/chosen": 0.004523592535406351,
-      "rewards/margins": 0.017096158117055893,
-      "rewards/rejected": -0.012572565115988255,
+      "logits/chosen": -1.8668408393859863,
+      "logits/rejected": -1.8711602687835693,
+      "logps/chosen": -36.98978042602539,
+      "logps/rejected": -33.66878890991211,
+      "loss": 0.4962,
+      "rewards/accuracies": 0.5416666865348816,
+      "rewards/chosen": 0.006728413049131632,
+      "rewards/margins": 0.022216208279132843,
+      "rewards/rejected": -0.015487794764339924,
       "step": 10
     },
     {
       "epoch": 0.05,
       "learning_rate": 2.564102564102564e-06,
-      "logits/chosen": -1.9979562759399414,
-      "logits/rejected": -2.000598907470703,
-      "logps/chosen": -29.644357681274414,
-      "logps/rejected": -29.06288719177246,
-      "loss": 0.6942,
-      "rewards/accuracies": 0.44999998807907104,
-      "rewards/chosen": -0.0008665517088957131,
-      "rewards/margins": -0.0010885533411055803,
-      "rewards/rejected": 0.00022200122475624084,
+      "logits/chosen": -1.9970359802246094,
+      "logits/rejected": -1.9996885061264038,
+      "logps/chosen": -29.635208129882812,
+      "logps/rejected": -29.063350677490234,
+      "loss": 0.4994,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": 0.002794977743178606,
+      "rewards/margins": 0.002757500857114792,
+      "rewards/rejected": 3.747665323317051e-05,
       "step": 20
     },
     {
       "epoch": 0.08,
       "learning_rate": 3.846153846153847e-06,
-      "logits/chosen": -1.9213365316390991,
-      "logits/rejected": -1.9186455011367798,
-      "logps/chosen": -31.3991756439209,
-      "logps/rejected": -33.220787048339844,
-      "loss": 0.6903,
-      "rewards/accuracies": 0.625,
-      "rewards/chosen": 0.006767785642296076,
-      "rewards/margins": 0.007698210421949625,
-      "rewards/rejected": -0.0009304238483309746,
+      "logits/chosen": -1.9208602905273438,
+      "logits/rejected": -1.9181534051895142,
+      "logps/chosen": -31.40317726135254,
+      "logps/rejected": -33.23335647583008,
+      "loss": 0.4982,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.005168457515537739,
+      "rewards/margins": 0.011126698926091194,
+      "rewards/rejected": -0.005958239547908306,
       "step": 30
     },
     {
       "epoch": 0.1,
       "learning_rate": 4.999896948438434e-06,
-      "logits/chosen": -2.0176024436950684,
-      "logits/rejected": -2.008852481842041,
-      "logps/chosen": -32.565155029296875,
-      "logps/rejected": -32.51045608520508,
-      "loss": 0.6924,
-      "rewards/accuracies": 0.5249999761581421,
-      "rewards/chosen": 0.004661071114242077,
-      "rewards/margins": 0.0037749619223177433,
-      "rewards/rejected": 0.0008861090755090117,
+      "logits/chosen": -2.017474889755249,
+      "logits/rejected": -2.008759021759033,
+      "logps/chosen": -32.54490661621094,
+      "logps/rejected": -32.49110794067383,
+      "loss": 0.4994,
+      "rewards/accuracies": 0.4749999940395355,
+      "rewards/chosen": 0.012759355828166008,
+      "rewards/margins": 0.004133358132094145,
+      "rewards/rejected": 0.008625999093055725,
       "step": 40
     },
     {
       "epoch": 0.13,
       "learning_rate": 4.987541037542187e-06,
-      "logits/chosen": -1.8624731302261353,
-      "logits/rejected": -1.8516931533813477,
-      "logps/chosen": -33.547359466552734,
-      "logps/rejected": -35.463809967041016,
-      "loss": 0.69,
-      "rewards/accuracies": 0.5249999761581421,
-      "rewards/chosen": 0.004046988673508167,
-      "rewards/margins": 0.008634108118712902,
-      "rewards/rejected": -0.004587120376527309,
+      "logits/chosen": -1.8631575107574463,
+      "logits/rejected": -1.8523809909820557,
+      "logps/chosen": -33.509056091308594,
+      "logps/rejected": -35.39984893798828,
+      "loss": 0.5001,
+      "rewards/accuracies": 0.48750001192092896,
+      "rewards/chosen": 0.01936880685389042,
+      "rewards/margins": -0.001629653968848288,
+      "rewards/rejected": 0.020998459309339523,
       "step": 50
     },
     {
       "epoch": 0.16,
       "learning_rate": 4.954691471941119e-06,
-      "logits/chosen": -1.9406204223632812,
-      "logits/rejected": -1.9425837993621826,
-      "logps/chosen": -32.54151153564453,
-      "logps/rejected": -33.21025848388672,
-      "loss": 0.6772,
-      "rewards/accuracies": 0.5874999761581421,
-      "rewards/chosen": 0.023469632491469383,
-      "rewards/margins": 0.03902136906981468,
-      "rewards/rejected": -0.015551735647022724,
+      "logits/chosen": -1.9419567584991455,
+      "logits/rejected": -1.9438903331756592,
+      "logps/chosen": -32.50743103027344,
+      "logps/rejected": -33.188419342041016,
+      "loss": 0.491,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.03710051625967026,
+      "rewards/margins": 0.04391607269644737,
+      "rewards/rejected": -0.006815555039793253,
       "step": 60
     },
     {
       "epoch": 0.18,
       "learning_rate": 4.901618883413549e-06,
-      "logits/chosen": -2.0720112323760986,
-      "logits/rejected": -2.0769832134246826,
-      "logps/chosen": -33.98130416870117,
-      "logps/rejected": -36.64153289794922,
-      "loss": 0.6804,
-      "rewards/accuracies": 0.5625,
-      "rewards/chosen": 0.0028717622626572847,
-      "rewards/margins": 0.03193504735827446,
-      "rewards/rejected": -0.029063284397125244,
+      "logits/chosen": -2.0737738609313965,
+      "logits/rejected": -2.0787465572357178,
+      "logps/chosen": -33.928836822509766,
+      "logps/rejected": -36.540794372558594,
+      "loss": 0.4971,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": 0.023859737440943718,
+      "rewards/margins": 0.01262708194553852,
+      "rewards/rejected": 0.01123266015201807,
       "step": 70
     },
     {
       "epoch": 0.21,
       "learning_rate": 4.828760511501322e-06,
-      "logits/chosen": -1.9326984882354736,
-      "logits/rejected": -1.935831069946289,
-      "logps/chosen": -34.31356430053711,
-      "logps/rejected": -34.65351867675781,
-      "loss": 0.6626,
-      "rewards/accuracies": 0.6499999761581421,
-      "rewards/chosen": 0.045696478337049484,
-      "rewards/margins": 0.07014231383800507,
-      "rewards/rejected": -0.02444584295153618,
+      "logits/chosen": -1.934011697769165,
+      "logits/rejected": -1.9371341466903687,
+      "logps/chosen": -34.23366928100586,
+      "logps/rejected": -34.542274475097656,
+      "loss": 0.4864,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.07765541225671768,
+      "rewards/margins": 0.057602256536483765,
+      "rewards/rejected": 0.020053153857588768,
       "step": 80
     },
     {
       "epoch": 0.23,
       "learning_rate": 4.7367166013034295e-06,
-      "logits/chosen": -1.9407522678375244,
-      "logits/rejected": -1.9452556371688843,
-      "logps/chosen": -32.40108108520508,
-      "logps/rejected": -32.342872619628906,
-      "loss": 0.6859,
-      "rewards/accuracies": 0.5874999761581421,
-      "rewards/chosen": 0.02653699554502964,
-      "rewards/margins": 0.020122777670621872,
-      "rewards/rejected": 0.0064142136834561825,
+      "logits/chosen": -1.9421571493148804,
+      "logits/rejected": -1.9466804265975952,
+      "logps/chosen": -32.304595947265625,
+      "logps/rejected": -32.284873962402344,
+      "loss": 0.4909,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.0651295855641365,
+      "rewards/margins": 0.035516757518053055,
+      "rewards/rejected": 0.02961282804608345,
       "step": 90
     },
     {
       "epoch": 0.26,
       "learning_rate": 4.626245458345211e-06,
-      "logits/chosen": -2.0378258228302,
-      "logits/rejected": -2.0358424186706543,
-      "logps/chosen": -32.15534973144531,
-      "logps/rejected": -31.297805786132812,
-      "loss": 0.6726,
-      "rewards/accuracies": 0.612500011920929,
-      "rewards/chosen": 0.03419475629925728,
-      "rewards/margins": 0.04683210328221321,
-      "rewards/rejected": -0.01263735257089138,
+      "logits/chosen": -2.0406017303466797,
+      "logits/rejected": -2.038613796234131,
+      "logps/chosen": -32.027652740478516,
+      "logps/rejected": -31.224151611328125,
+      "loss": 0.4845,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": 0.08527366816997528,
+      "rewards/margins": 0.0684497207403183,
+      "rewards/rejected": 0.01682395115494728,
       "step": 100
     },
     {
       "epoch": 0.26,
-      "eval_logits/chosen": -2.233442544937134,
-      "eval_logits/rejected": -2.2285873889923096,
-      "eval_logps/chosen": -34.044193267822266,
-      "eval_logps/rejected": -37.55242919921875,
-      "eval_loss": 0.6902939081192017,
-      "eval_rewards/accuracies": 0.5481727719306946,
-      "eval_rewards/chosen": -0.0038560593966394663,
-      "eval_rewards/margins": 0.010466905310750008,
-      "eval_rewards/rejected": -0.014322965405881405,
-      "eval_runtime": 146.038,
-      "eval_samples_per_second": 2.349,
-      "eval_steps_per_second": 0.294,
+      "eval_logits/chosen": -2.234653949737549,
+      "eval_logits/rejected": -2.229806423187256,
+      "eval_logps/chosen": -33.91841125488281,
+      "eval_logps/rejected": -37.42335891723633,
+      "eval_loss": 0.4981803297996521,
+      "eval_rewards/accuracies": 0.5510797500610352,
+      "eval_rewards/chosen": 0.04645563289523125,
+      "eval_rewards/margins": 0.009152057580649853,
+      "eval_rewards/rejected": 0.03730357065796852,
+      "eval_runtime": 145.8684,
+      "eval_samples_per_second": 2.351,
+      "eval_steps_per_second": 0.295,
       "step": 100
     },
     {
       "epoch": 0.29,
       "learning_rate": 4.498257201263691e-06,
-      "logits/chosen": -1.9928157329559326,
-      "logits/rejected": -1.9904266595840454,
-      "logps/chosen": -33.1363410949707,
-      "logps/rejected": -34.00283432006836,
-      "loss": 0.6883,
-      "rewards/accuracies": 0.5625,
-      "rewards/chosen": 0.04299246892333031,
-      "rewards/margins": 0.03237856179475784,
-      "rewards/rejected": 0.010613908991217613,
+      "logits/chosen": -1.9949238300323486,
+      "logits/rejected": -1.9925572872161865,
+      "logps/chosen": -32.99268341064453,
+      "logps/rejected": -33.90182113647461,
+      "loss": 0.4869,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": 0.10045422613620758,
+      "rewards/margins": 0.049433451145887375,
+      "rewards/rejected": 0.051020748913288116,
       "step": 110
     },
     {
       "epoch": 0.31,
       "learning_rate": 4.353806263777678e-06,
-      "logits/chosen": -2.0046679973602295,
-      "logits/rejected": -1.9963252544403076,
-      "logps/chosen": -32.3227653503418,
-      "logps/rejected": -32.157779693603516,
-      "loss": 0.6741,
-      "rewards/accuracies": 0.5625,
-      "rewards/chosen": 0.04885613173246384,
-      "rewards/margins": 0.0473661907017231,
-      "rewards/rejected": 0.0014899425441399217,
+      "logits/chosen": -2.0055007934570312,
+      "logits/rejected": -1.997157335281372,
+      "logps/chosen": -32.192996978759766,
+      "logps/rejected": -32.01173400878906,
+      "loss": 0.491,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.10076460987329483,
+      "rewards/margins": 0.04085635766386986,
+      "rewards/rejected": 0.05990824103355408,
       "step": 120
     },
     {
       "epoch": 0.34,
       "learning_rate": 4.1940827077152755e-06,
-      "logits/chosen": -2.0337133407592773,
-      "logits/rejected": -2.0257389545440674,
-      "logps/chosen": -30.330951690673828,
-      "logps/rejected": -32.077327728271484,
-      "loss": 0.6662,
-      "rewards/accuracies": 0.612500011920929,
-      "rewards/chosen": 0.05790700390934944,
-      "rewards/margins": 0.06763499230146408,
-      "rewards/rejected": -0.00972799677401781,
+      "logits/chosen": -2.034147262573242,
+      "logits/rejected": -2.026184558868408,
+      "logps/chosen": -30.194936752319336,
+      "logps/rejected": -31.905689239501953,
+      "loss": 0.4876,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.11231068521738052,
+      "rewards/margins": 0.05338172987103462,
+      "rewards/rejected": 0.0589289590716362,
       "step": 130
     },
     {
       "epoch": 0.36,
       "learning_rate": 4.0204024186666215e-06,
-      "logits/chosen": -1.9636684656143188,
-      "logits/rejected": -1.9739125967025757,
-      "logps/chosen": -31.2012996673584,
-      "logps/rejected": -32.56267547607422,
-      "loss": 0.6505,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": 0.08734508603811264,
-      "rewards/margins": 0.09846383333206177,
-      "rewards/rejected": -0.011118754744529724,
+      "logits/chosen": -1.9643388986587524,
+      "logits/rejected": -1.9745395183563232,
+      "logps/chosen": -31.08599853515625,
+      "logps/rejected": -32.422943115234375,
+      "loss": 0.4795,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": 0.13346607983112335,
+      "rewards/margins": 0.08869143575429916,
+      "rewards/rejected": 0.044774629175662994,
       "step": 140
     },
     {
       "epoch": 0.39,
       "learning_rate": 3.834196265035119e-06,
-      "logits/chosen": -1.8753896951675415,
-      "logits/rejected": -1.8765614032745361,
-      "logps/chosen": -33.930477142333984,
-      "logps/rejected": -34.81265640258789,
-      "loss": 0.6411,
-      "rewards/accuracies": 0.637499988079071,
-      "rewards/chosen": 0.10488543659448624,
-      "rewards/margins": 0.1270444542169571,
-      "rewards/rejected": -0.022159017622470856,
+      "logits/chosen": -1.8776963949203491,
+      "logits/rejected": -1.8788686990737915,
+      "logps/chosen": -33.690345764160156,
+      "logps/rejected": -34.572776794433594,
+      "loss": 0.4699,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.20093801617622375,
+      "rewards/margins": 0.12714678049087524,
+      "rewards/rejected": 0.07379122078418732,
       "step": 150
     },
     {
       "epoch": 0.42,
       "learning_rate": 3.636998309800573e-06,
-      "logits/chosen": -1.9266326427459717,
-      "logits/rejected": -1.9231981039047241,
-      "logps/chosen": -36.001312255859375,
-      "logps/rejected": -32.70484924316406,
-      "loss": 0.6671,
-      "rewards/accuracies": 0.574999988079071,
-      "rewards/chosen": 0.0681454986333847,
-      "rewards/margins": 0.06098458915948868,
-      "rewards/rejected": 0.007160906679928303,
+      "logits/chosen": -1.9286582469940186,
+      "logits/rejected": -1.9253017902374268,
+      "logps/chosen": -35.773475646972656,
+      "logps/rejected": -32.47566604614258,
+      "loss": 0.4851,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.15928010642528534,
+      "rewards/margins": 0.06044477969408035,
+      "rewards/rejected": 0.09883531928062439,
       "step": 160
     },
     {
       "epoch": 0.44,
       "learning_rate": 3.4304331721118078e-06,
-      "logits/chosen": -2.0281758308410645,
-      "logits/rejected": -2.0208277702331543,
-      "logps/chosen": -33.47995376586914,
-      "logps/rejected": -31.4173526763916,
-      "loss": 0.6278,
-      "rewards/accuracies": 0.6625000238418579,
-      "rewards/chosen": 0.12716376781463623,
-      "rewards/margins": 0.15336360037326813,
-      "rewards/rejected": -0.026199836283922195,
+      "logits/chosen": -2.0298352241516113,
+      "logits/rejected": -2.0225348472595215,
+      "logps/chosen": -33.22509002685547,
+      "logps/rejected": -31.19403648376465,
+      "loss": 0.4603,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.22910937666893005,
+      "rewards/margins": 0.165984109044075,
+      "rewards/rejected": 0.06312531232833862,
       "step": 170
     },
     {
       "epoch": 0.47,
       "learning_rate": 3.2162026428305436e-06,
-      "logits/chosen": -2.033982038497925,
-      "logits/rejected": -2.0392203330993652,
-      "logps/chosen": -32.239437103271484,
-      "logps/rejected": -32.44209671020508,
-      "loss": 0.6474,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": 0.12213903665542603,
-      "rewards/margins": 0.10492189973592758,
-      "rewards/rejected": 0.017217133194208145,
+      "logits/chosen": -2.036612033843994,
+      "logits/rejected": -2.041813373565674,
+      "logps/chosen": -31.960119247436523,
+      "logps/rejected": -32.171165466308594,
+      "loss": 0.4734,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.23386716842651367,
+      "rewards/margins": 0.1082783117890358,
+      "rewards/rejected": 0.12558886408805847,
       "step": 180
     },
     {
       "epoch": 0.49,
       "learning_rate": 2.996071664294641e-06,
-      "logits/chosen": -2.034848928451538,
-      "logits/rejected": -2.032095432281494,
-      "logps/chosen": -31.259105682373047,
-      "logps/rejected": -31.31390953063965,
-      "loss": 0.6567,
-      "rewards/accuracies": 0.612500011920929,
-      "rewards/chosen": 0.09193893522024155,
-      "rewards/margins": 0.09021683037281036,
-      "rewards/rejected": 0.0017221048474311829,
+      "logits/chosen": -2.037198066711426,
+      "logits/rejected": -2.0344595909118652,
+      "logps/chosen": -31.026615142822266,
+      "logps/rejected": -31.082998275756836,
+      "loss": 0.4781,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.184935063123703,
+      "rewards/margins": 0.09084881842136383,
+      "rewards/rejected": 0.09408621490001678,
       "step": 190
     },
     {
       "epoch": 0.52,
       "learning_rate": 2.7718537898066833e-06,
-      "logits/chosen": -1.9051767587661743,
-      "logits/rejected": -1.9098323583602905,
-      "logps/chosen": -31.32687759399414,
-      "logps/rejected": -32.82324981689453,
-      "loss": 0.6419,
+      "logits/chosen": -1.908818006515503,
+      "logits/rejected": -1.913496732711792,
+      "logps/chosen": -31.075199127197266,
+      "logps/rejected": -32.616241455078125,
+      "loss": 0.466,
       "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": 0.1117621660232544,
-      "rewards/margins": 0.12468986213207245,
-      "rewards/rejected": -0.012927706353366375,
+      "rewards/chosen": 0.2124340832233429,
+      "rewards/margins": 0.14255891740322113,
+      "rewards/rejected": 0.06987515836954117,
       "step": 200
     },
     {
       "epoch": 0.52,
-      "eval_logits/chosen": -2.2309587001800537,
-      "eval_logits/rejected": -2.2261300086975098,
-      "eval_logps/chosen": -34.06596374511719,
-      "eval_logps/rejected": -37.5656852722168,
-      "eval_loss": 0.6932108402252197,
-      "eval_rewards/accuracies": 0.5041528344154358,
-      "eval_rewards/chosen": -0.0125651890411973,
-      "eval_rewards/margins": 0.007060302421450615,
-      "eval_rewards/rejected": -0.01962549053132534,
-      "eval_runtime": 145.8797,
-      "eval_samples_per_second": 2.351,
+      "eval_logits/chosen": -2.23366379737854,
+      "eval_logits/rejected": -2.2288384437561035,
+      "eval_logps/chosen": -33.779903411865234,
+      "eval_logps/rejected": -37.29892349243164,
+      "eval_loss": 0.4965229630470276,
+      "eval_rewards/accuracies": 0.545265793800354,
+      "eval_rewards/chosen": 0.10185908526182175,
+      "eval_rewards/margins": 0.014780867844820023,
+      "eval_rewards/rejected": 0.08707821369171143,
+      "eval_runtime": 145.8117,
+      "eval_samples_per_second": 2.352,
       "eval_steps_per_second": 0.295,
       "step": 200
     },
     {
       "epoch": 0.55,
       "learning_rate": 2.5453962426402006e-06,
-      "logits/chosen": -2.0179035663604736,
-      "logits/rejected": -2.0285422801971436,
-      "logps/chosen": -31.756057739257812,
-      "logps/rejected": -33.936058044433594,
-      "loss": 0.6444,
-      "rewards/accuracies": 0.637499988079071,
-      "rewards/chosen": 0.08609838038682938,
-      "rewards/margins": 0.11544252932071686,
-      "rewards/rejected": -0.029344135895371437,
+      "logits/chosen": -2.0208163261413574,
+      "logits/rejected": -2.0314111709594727,
+      "logps/chosen": -31.514019012451172,
+      "logps/rejected": -33.690643310546875,
+      "loss": 0.4728,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.18291443586349487,
+      "rewards/margins": 0.114091657102108,
+      "rewards/rejected": 0.06882277131080627,
       "step": 210
     },
     {
       "epoch": 0.57,
       "learning_rate": 2.3185646976551794e-06,
-      "logits/chosen": -1.910278081893921,
-      "logits/rejected": -1.9250171184539795,
-      "logps/chosen": -29.805334091186523,
-      "logps/rejected": -31.600574493408203,
-      "loss": 0.6326,
+      "logits/chosen": -1.9141031503677368,
+      "logits/rejected": -1.9288082122802734,
+      "logps/chosen": -29.5712947845459,
+      "logps/rejected": -31.429983139038086,
+      "loss": 0.4615,
       "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": 0.1224818006157875,
-      "rewards/margins": 0.139441579580307,
-      "rewards/rejected": -0.016959769651293755,
+      "rewards/chosen": 0.2160976380109787,
+      "rewards/margins": 0.16482076048851013,
+      "rewards/rejected": 0.05127686262130737,
       "step": 220
     },
     {
       "epoch": 0.6,
       "learning_rate": 2.0932279108998323e-06,
-      "logits/chosen": -1.9668325185775757,
-      "logits/rejected": -1.970798134803772,
-      "logps/chosen": -33.078094482421875,
-      "logps/rejected": -31.6514949798584,
-      "loss": 0.621,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": 0.13575060665607452,
-      "rewards/margins": 0.1766270101070404,
-      "rewards/rejected": -0.04087639972567558,
+      "logits/chosen": -1.9712814092636108,
+      "logits/rejected": -1.975285291671753,
+      "logps/chosen": -32.82429885864258,
+      "logps/rejected": -31.416866302490234,
+      "loss": 0.4561,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": 0.23726816475391388,
+      "rewards/margins": 0.18429425358772278,
+      "rewards/rejected": 0.052973903715610504,
       "step": 230
     },
     {
       "epoch": 0.62,
       "learning_rate": 1.8712423238279358e-06,
-      "logits/chosen": -1.965409278869629,
-      "logits/rejected": -1.9435851573944092,
-      "logps/chosen": -33.83237075805664,
-      "logps/rejected": -35.09648132324219,
-      "loss": 0.6165,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": 0.12855152785778046,
-      "rewards/margins": 0.18682697415351868,
-      "rewards/rejected": -0.05827543884515762,
+      "logits/chosen": -1.969780683517456,
+      "logits/rejected": -1.9480432271957397,
+      "logps/chosen": -33.583518981933594,
+      "logps/rejected": -34.8461799621582,
+      "loss": 0.4567,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.22809162735939026,
+      "rewards/margins": 0.18624703586101532,
+      "rewards/rejected": 0.04184458404779434,
       "step": 240
     },
     {
       "epoch": 0.65,
       "learning_rate": 1.6544367689701824e-06,
-      "logits/chosen": -2.006166696548462,
-      "logits/rejected": -2.002840042114258,
-      "logps/chosen": -32.69712448120117,
-      "logps/rejected": -36.255043029785156,
-      "loss": 0.6502,
-      "rewards/accuracies": 0.625,
-      "rewards/chosen": 0.09117679297924042,
-      "rewards/margins": 0.10322580486536026,
-      "rewards/rejected": -0.01204901933670044,
+      "logits/chosen": -2.0109124183654785,
+      "logits/rejected": -2.0076212882995605,
+      "logps/chosen": -32.46331024169922,
+      "logps/rejected": -35.97381591796875,
+      "loss": 0.4799,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": 0.18470348417758942,
+      "rewards/margins": 0.08426074683666229,
+      "rewards/rejected": 0.10044274479150772,
       "step": 250
     },
     {
       "epoch": 0.68,
       "learning_rate": 1.4445974030621963e-06,
-      "logits/chosen": -1.8737766742706299,
-      "logits/rejected": -1.87137770652771,
-      "logps/chosen": -33.97405242919922,
-      "logps/rejected": -35.56011962890625,
-      "loss": 0.6473,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": 0.08574129641056061,
-      "rewards/margins": 0.1079399362206459,
-      "rewards/rejected": -0.022198637947440147,
+      "logits/chosen": -1.878321647644043,
+      "logits/rejected": -1.8758872747421265,
+      "logps/chosen": -33.721397399902344,
+      "logps/rejected": -35.270362854003906,
+      "loss": 0.4785,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.1868000328540802,
+      "rewards/margins": 0.09309660643339157,
+      "rewards/rejected": 0.09370341151952744,
       "step": 260
     },
     {
       "epoch": 0.7,
       "learning_rate": 1.243452991757889e-06,
-      "logits/chosen": -1.8586937189102173,
-      "logits/rejected": -1.8563038110733032,
-      "logps/chosen": -34.204627990722656,
-      "logps/rejected": -31.85955810546875,
-      "loss": 0.6478,
-      "rewards/accuracies": 0.675000011920929,
-      "rewards/chosen": 0.07718975841999054,
-      "rewards/margins": 0.11248154938220978,
-      "rewards/rejected": -0.03529178351163864,
+      "logits/chosen": -1.8631956577301025,
+      "logits/rejected": -1.8606828451156616,
+      "logps/chosen": -33.90094757080078,
+      "logps/rejected": -31.57466697692871,
+      "loss": 0.4722,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": 0.198659747838974,
+      "rewards/margins": 0.11999478191137314,
+      "rewards/rejected": 0.07866497337818146,
       "step": 270
     },
     {
       "epoch": 0.73,
       "learning_rate": 1.0526606671603523e-06,
-      "logits/chosen": -1.962044358253479,
-      "logits/rejected": -1.9515289068222046,
-      "logps/chosen": -35.0092887878418,
-      "logps/rejected": -31.877635955810547,
-      "loss": 0.6267,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": 0.13894058763980865,
-      "rewards/margins": 0.15359947085380554,
-      "rewards/rejected": -0.01465887576341629,
+      "logits/chosen": -1.9663877487182617,
+      "logits/rejected": -1.9559704065322876,
+      "logps/chosen": -34.72657775878906,
+      "logps/rejected": -31.63601303100586,
+      "loss": 0.4585,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.25202488899230957,
+      "rewards/margins": 0.17003390192985535,
+      "rewards/rejected": 0.08199100196361542,
       "step": 280
     },
     {
       "epoch": 0.75,
       "learning_rate": 8.737922755071455e-07,
-      "logits/chosen": -2.0572900772094727,
-      "logits/rejected": -2.0424036979675293,
-      "logps/chosen": -30.72947120666504,
-      "logps/rejected": -32.645362854003906,
-      "loss": 0.6657,
+      "logits/chosen": -2.062107801437378,
+      "logits/rejected": -2.0472733974456787,
+      "logps/chosen": -30.40212059020996,
+      "logps/rejected": -32.340721130371094,
+      "loss": 0.4799,
       "rewards/accuracies": 0.625,
-      "rewards/chosen": 0.07786226272583008,
-      "rewards/margins": 0.07527298480272293,
-      "rewards/rejected": 0.002589278621599078,
+      "rewards/chosen": 0.2088027447462082,
+      "rewards/margins": 0.08435753732919693,
+      "rewards/rejected": 0.12444518506526947,
       "step": 290
     },
     {
       "epoch": 0.78,
       "learning_rate": 7.08321427484816e-07,
-      "logits/chosen": -1.9281587600708008,
-      "logits/rejected": -1.9256232976913452,
-      "logps/chosen": -32.37285614013672,
-      "logps/rejected": -30.912500381469727,
-      "loss": 0.5902,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": 0.22170230746269226,
-      "rewards/margins": 0.2599778473377228,
-      "rewards/rejected": -0.03827553242444992,
+      "logits/chosen": -1.9330482482910156,
+      "logits/rejected": -1.9305979013442993,
+      "logps/chosen": -32.06965255737305,
+      "logps/rejected": -30.65035629272461,
+      "loss": 0.4349,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": 0.3429831862449646,
+      "rewards/margins": 0.27640262246131897,
+      "rewards/rejected": 0.06658058613538742,
       "step": 300
     },
     {
       "epoch": 0.78,
-      "eval_logits/chosen": -2.2279410362243652,
-      "eval_logits/rejected": -2.2231125831604004,
-      "eval_logps/chosen": -34.09661865234375,
-      "eval_logps/rejected": -37.598716735839844,
-      "eval_loss": 0.6926039457321167,
-      "eval_rewards/accuracies": 0.49543190002441406,
-      "eval_rewards/chosen": -0.024825766682624817,
-      "eval_rewards/margins": 0.008013932965695858,
-      "eval_rewards/rejected": -0.03283970057964325,
-      "eval_runtime": 145.6649,
-      "eval_samples_per_second": 2.355,
+      "eval_logits/chosen": -2.2318336963653564,
+      "eval_logits/rejected": -2.2270052433013916,
+      "eval_logps/chosen": -33.749427795410156,
+      "eval_logps/rejected": -37.27041244506836,
+      "eval_loss": 0.4961945414543152,
+      "eval_rewards/accuracies": 0.5357142686843872,
+      "eval_rewards/chosen": 0.11405016481876373,
+      "eval_rewards/margins": 0.015567691065371037,
+      "eval_rewards/rejected": 0.09848246723413467,
+      "eval_runtime": 145.5479,
+      "eval_samples_per_second": 2.357,
       "eval_steps_per_second": 0.295,
       "step": 300
     },
     {
       "epoch": 0.81,
       "learning_rate": 5.576113578589035e-07,
-      "logits/chosen": -1.9122984409332275,
-      "logits/rejected": -1.9090602397918701,
-      "logps/chosen": -31.30501365661621,
-      "logps/rejected": -33.817237854003906,
-      "loss": 0.6314,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": 0.11933410167694092,
-      "rewards/margins": 0.149368017911911,
-      "rewards/rejected": -0.030033910647034645,
+      "logits/chosen": -1.9179494380950928,
+      "logits/rejected": -1.9148216247558594,
+      "logps/chosen": -31.037755966186523,
+      "logps/rejected": -33.56406021118164,
+      "loss": 0.4639,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": 0.22623713314533234,
+      "rewards/margins": 0.15500028431415558,
+      "rewards/rejected": 0.07123686373233795,
       "step": 310
     },
     {
       "epoch": 0.83,
       "learning_rate": 4.229036944380913e-07,
-      "logits/chosen": -1.962451696395874,
-      "logits/rejected": -1.950260877609253,
-      "logps/chosen": -34.321712493896484,
-      "logps/rejected": -33.68184280395508,
-      "loss": 0.6222,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": 0.10974420607089996,
-      "rewards/margins": 0.16694548726081848,
-      "rewards/rejected": -0.0572013333439827,
+      "logits/chosen": -1.9682966470718384,
+      "logits/rejected": -1.9561439752578735,
+      "logps/chosen": -34.023921966552734,
+      "logps/rejected": -33.415985107421875,
+      "loss": 0.4575,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.22886168956756592,
+      "rewards/margins": 0.17971986532211304,
+      "rewards/rejected": 0.04914180561900139,
       "step": 320
     },
     {
       "epoch": 0.86,
       "learning_rate": 3.053082288996112e-07,
-      "logits/chosen": -1.9975078105926514,
-      "logits/rejected": -1.996080994606018,
-      "logps/chosen": -33.17241287231445,
-      "logps/rejected": -32.53407669067383,
-      "loss": 0.6313,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": 0.12535209953784943,
-      "rewards/margins": 0.14715085923671722,
-      "rewards/rejected": -0.021798742935061455,
+      "logits/chosen": -2.003788709640503,
+      "logits/rejected": -2.002437114715576,
+      "logps/chosen": -32.8862419128418,
+      "logps/rejected": -32.21629333496094,
+      "loss": 0.4679,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.23982055485248566,
+      "rewards/margins": 0.13450448215007782,
+      "rewards/rejected": 0.10531606525182724,
       "step": 330
     },
     {
       "epoch": 0.88,
       "learning_rate": 2.0579377374915805e-07,
-      "logits/chosen": -2.0841541290283203,
-      "logits/rejected": -2.0684666633605957,
-      "logps/chosen": -33.79545211791992,
-      "logps/rejected": -33.1196174621582,
-      "loss": 0.6276,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": 0.15357479453086853,
-      "rewards/margins": 0.15420618653297424,
-      "rewards/rejected": -0.000631416798569262,
+      "logits/chosen": -2.0905163288116455,
+      "logits/rejected": -2.0749027729034424,
+      "logps/chosen": -33.47309112548828,
+      "logps/rejected": -32.81334686279297,
+      "loss": 0.4618,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.2825208008289337,
+      "rewards/margins": 0.16064420342445374,
+      "rewards/rejected": 0.12187659740447998,
       "step": 340
     },
     {
       "epoch": 0.91,
       "learning_rate": 1.2518018074041684e-07,
-      "logits/chosen": -1.9566657543182373,
-      "logits/rejected": -1.9558334350585938,
-      "logps/chosen": -32.842193603515625,
-      "logps/rejected": -32.53485870361328,
-      "loss": 0.6121,
-      "rewards/accuracies": 0.6625000238418579,
-      "rewards/chosen": 0.17596027255058289,
-      "rewards/margins": 0.2038455456495285,
-      "rewards/rejected": -0.027885273098945618,
+      "logits/chosen": -1.9627025127410889,
+      "logits/rejected": -1.9618685245513916,
+      "logps/chosen": -32.528175354003906,
+      "logps/rejected": -32.22235107421875,
+      "loss": 0.4522,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": 0.30156710743904114,
+      "rewards/margins": 0.20444798469543457,
+      "rewards/rejected": 0.09711913019418716,
       "step": 350
     },
     {
       "epoch": 0.94,
       "learning_rate": 6.41315865106129e-08,
-      "logits/chosen": -1.9120171070098877,
-      "logits/rejected": -1.9222869873046875,
-      "logps/chosen": -31.887542724609375,
-      "logps/rejected": -35.35187911987305,
-      "loss": 0.6293,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": 0.12685921788215637,
-      "rewards/margins": 0.15210750699043274,
-      "rewards/rejected": -0.025248277932405472,
+      "logits/chosen": -1.918320655822754,
+      "logits/rejected": -1.9285932779312134,
+      "logps/chosen": -31.606945037841797,
+      "logps/rejected": -34.98893356323242,
+      "loss": 0.4718,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.2390974760055542,
+      "rewards/margins": 0.11916828155517578,
+      "rewards/rejected": 0.11992917954921722,
       "step": 360
     },
     {
       "epoch": 0.96,
       "learning_rate": 2.3150941078050325e-08,
-      "logits/chosen": -2.0516254901885986,
-      "logits/rejected": -2.045139789581299,
-      "logps/chosen": -33.33611297607422,
-      "logps/rejected": -29.27042579650879,
-      "loss": 0.6307,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": 0.12634168565273285,
-      "rewards/margins": 0.14394234120845795,
-      "rewards/rejected": -0.01760067418217659,
+      "logits/chosen": -2.0581445693969727,
+      "logits/rejected": -2.051628828048706,
+      "logps/chosen": -33.017967224121094,
+      "logps/rejected": -28.989696502685547,
+      "loss": 0.4612,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": 0.253601610660553,
+      "rewards/margins": 0.15891048312187195,
+      "rewards/rejected": 0.09469114243984222,
       "step": 370
     },
     {
       "epoch": 0.99,
       "learning_rate": 2.575864278703266e-09,
-      "logits/chosen": -1.9114952087402344,
-      "logits/rejected": -1.9137216806411743,
-      "logps/chosen": -33.884849548339844,
-      "logps/rejected": -30.985692977905273,
-      "loss": 0.6144,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": 0.14167849719524384,
-      "rewards/margins": 0.18690630793571472,
-      "rewards/rejected": -0.04522782564163208,
+      "logits/chosen": -1.9176830053329468,
+      "logits/rejected": -1.919847846031189,
+      "logps/chosen": -33.614540100097656,
+      "logps/rejected": -30.760555267333984,
+      "loss": 0.4519,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": 0.24980488419532776,
+      "rewards/margins": 0.2049761563539505,
+      "rewards/rejected": 0.04482869431376457,
       "step": 380
     },
     {
       "epoch": 1.0,
       "step": 385,
       "total_flos": 0.0,
-      "train_loss": 0.6506414413452148,
-      "train_runtime": 3254.0644,
-      "train_samples_per_second": 0.946,
+      "train_loss": 0.4751699732495593,
+      "train_runtime": 3253.0203,
+      "train_samples_per_second": 0.947,
       "train_steps_per_second": 0.118
     }
   ],