Model save

README.md

@@ -1,13 +1,10 @@
 ---
 library_name: peft
 tags:
-- alignment-handbook
 - trl
 - dpo
 - generated_from_trainer
 base_model: NbAiLab/nb-gpt-j-6B-v2
-datasets:
-- hugodk-sch/aftonposten_title_prefs
 model-index:
 - name: aftonposten-6b-align-scan
   results: []
@@ -18,17 +15,17 @@ should probably proofread and complete it, then remove this comment. -->
 
 # aftonposten-6b-align-scan
 
-This model is a fine-tuned version of [data/ap-gpt-j-6b-sft-qlora-04-08](https://huggingface.co/data/ap-gpt-j-6b-sft-qlora-04-08) on the hugodk-sch/aftonposten_title_prefs dataset.
+This model is a fine-tuned version of [NbAiLab/nb-gpt-j-6B-v2](https://huggingface.co/NbAiLab/nb-gpt-j-6B-v2) on an unknown dataset.
 It achieves the following results on the evaluation set:
-- Loss: 0.5837
-- Rewards/chosen: -0.0077
-- Rewards/rejected: -0.0206
-- Rewards/accuracies: 0.5428
-- Rewards/margins: 0.0128
-- Logps/rejected: -37.5460
-- Logps/chosen: -34.0456
-- Logits/rejected: -2.2319
-- Logits/chosen: -2.2368
+- Loss: 0.9761
+- Rewards/chosen: -0.0366
+- Rewards/rejected: -0.0616
+- Rewards/accuracies: 0.5299
+- Rewards/margins: 0.0250
+- Logps/rejected: -37.6047
+- Logps/chosen: -34.0868
+- Logits/rejected: -2.2186
+- Logits/chosen: -2.2234
 
 ## Model description
 
@@ -63,9 +60,9 @@ The following hyperparameters were used during training:
 
 | Training Loss | Epoch | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
 |:-------------:|:-----:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
-| 0.4711        | 0.26  | 100  | 0.5755          | 0.0163         | 0.0131           | 0.5195             | 0.0032          | -37.4979       | -34.0113     | -2.2352         | -2.2401       |
-| 0.5061        | 0.52  | 200  | 0.5877          | -0.0108        | -0.0202          | 0.4992             | 0.0094          | -37.5455       | -34.0500     | -2.2337         | -2.2385       |
-| 0.3371        | 0.78  | 300  | 0.5843          | 0.0001         | -0.0131          | 0.5278             | 0.0132          | -37.5353       | -34.0344     | -2.2322         | -2.2371       |
+| 0.8961        | 0.26  | 100  | 0.9767          | 0.0065         | -0.0170          | 0.5365             | 0.0235          | -37.5409       | -34.0252     | -2.2266         | -2.2315       |
+| 0.7699        | 0.52  | 200  | 0.9742          | -0.0141        | -0.0400          | 0.5303             | 0.0259          | -37.5737       | -34.0547     | -2.2234         | -2.2282       |
+| 0.6723        | 0.78  | 300  | 0.9761          | -0.0366        | -0.0616          | 0.5299             | 0.0250          | -37.6047       | -34.0868     | -2.2186         | -2.2234       |
 
 
 ### Framework versions

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d4f769127e001b25c26d446885ded3c472000a38dcddb8da8a8c4c2196e59b84
+oid sha256:00a83907764f5cb17421bff977c109ad02b2829b3ce93be8bf43332a1014cc25
 size 176183216

all_results.json

@@ -1,20 +1,7 @@
 {
     "epoch": 1.0,
-    "eval_logits/chosen": -2.236751079559326,
-    "eval_logits/rejected": -2.2318832874298096,
-    "eval_logps/chosen": -34.04561233520508,
-    "eval_logps/rejected": -37.546016693115234,
-    "eval_loss": 0.5836836099624634,
-    "eval_rewards/accuracies": 0.5427741408348083,
-    "eval_rewards/chosen": -0.007742464076727629,
-    "eval_rewards/margins": 0.01283606793731451,
-    "eval_rewards/rejected": -0.02057853527367115,
-    "eval_runtime": 145.5744,
-    "eval_samples": 343,
-    "eval_samples_per_second": 2.356,
-    "eval_steps_per_second": 0.295,
-    "train_loss": 0.46899499707407766,
-    "train_runtime": 3249.86,
+    "train_loss": 0.8438688600218142,
+    "train_runtime": 3250.9917,
     "train_samples": 3079,
     "train_samples_per_second": 0.947,
     "train_steps_per_second": 0.118

train_results.json

@@ -1,7 +1,7 @@
 {
     "epoch": 1.0,
-    "train_loss": 0.46899499707407766,
-    "train_runtime": 3249.86,
+    "train_loss": 0.8438688600218142,
+    "train_runtime": 3250.9917,
     "train_samples": 3079,
     "train_samples_per_second": 0.947,
     "train_steps_per_second": 0.118

trainer_state.json

@@ -15,7 +15,7 @@
       "logits/rejected": -1.7377450466156006,
       "logps/chosen": -29.553977966308594,
       "logps/rejected": -42.813133239746094,
-      "loss": 0.5102,
+      "loss": 1.0,
       "rewards/accuracies": 0.0,
       "rewards/chosen": 0.0,
       "rewards/margins": 0.0,
@@ -25,589 +25,589 @@
     {
       "epoch": 0.03,
       "learning_rate": 1.282051282051282e-06,
-      "logits/chosen": -1.866772174835205,
-      "logits/rejected": -1.8710780143737793,
-      "logps/chosen": -36.99943161010742,
-      "logps/rejected": -33.657470703125,
-      "loss": 0.4858,
+      "logits/chosen": -1.8664777278900146,
+      "logits/rejected": -1.8707849979400635,
+      "logps/chosen": -36.99364471435547,
+      "logps/rejected": -33.650604248046875,
+      "loss": 0.9766,
       "rewards/accuracies": 0.5277777910232544,
-      "rewards/chosen": 0.005016062408685684,
-      "rewards/margins": 0.02419126406311989,
-      "rewards/rejected": -0.019175197929143906,
+      "rewards/chosen": 0.00906434003263712,
+      "rewards/margins": 0.023435616865754128,
+      "rewards/rejected": -0.014371277764439583,
       "step": 10
     },
     {
       "epoch": 0.05,
       "learning_rate": 2.564102564102564e-06,
-      "logits/chosen": -1.9975645542144775,
-      "logits/rejected": -2.0002217292785645,
-      "logps/chosen": -29.651592254638672,
-      "logps/rejected": -29.066898345947266,
-      "loss": 0.5542,
-      "rewards/accuracies": 0.5,
-      "rewards/chosen": -0.006579822860658169,
-      "rewards/margins": -0.004162783268839121,
-      "rewards/rejected": -0.0024170405231416225,
+      "logits/chosen": -1.9982150793075562,
+      "logits/rejected": -2.0008621215820312,
+      "logps/chosen": -29.64394760131836,
+      "logps/rejected": -29.04986000061035,
+      "loss": 1.0107,
+      "rewards/accuracies": 0.4375,
+      "rewards/chosen": -0.0012267641723155975,
+      "rewards/margins": -0.010734880343079567,
+      "rewards/rejected": 0.00950811617076397,
       "step": 20
     },
     {
       "epoch": 0.08,
       "learning_rate": 3.846153846153847e-06,
-      "logits/chosen": -1.9212992191314697,
-      "logits/rejected": -1.918600082397461,
-      "logps/chosen": -31.42719078063965,
-      "logps/rejected": -33.23390579223633,
-      "loss": 0.5492,
-      "rewards/accuracies": 0.5249999761581421,
-      "rewards/chosen": -0.00776728754863143,
-      "rewards/margins": 0.0030429032631218433,
-      "rewards/rejected": -0.010810190811753273,
+      "logits/chosen": -1.920768141746521,
+      "logits/rejected": -1.9180870056152344,
+      "logps/chosen": -31.416461944580078,
+      "logps/rejected": -33.2098274230957,
+      "loss": 1.0063,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": -0.000256747764069587,
+      "rewards/margins": -0.0063001858070492744,
+      "rewards/rejected": 0.00604343693703413,
       "step": 30
     },
     {
       "epoch": 0.1,
       "learning_rate": 4.999896948438434e-06,
-      "logits/chosen": -2.0178468227386475,
-      "logits/rejected": -2.009111166000366,
-      "logps/chosen": -32.579471588134766,
-      "logps/rejected": -32.50147247314453,
-      "loss": 0.5878,
-      "rewards/accuracies": 0.44999998807907104,
-      "rewards/chosen": -0.001865685684606433,
-      "rewards/margins": -0.009705344215035439,
-      "rewards/rejected": 0.007839656434953213,
+      "logits/chosen": -2.0177602767944336,
+      "logits/rejected": -2.009014129638672,
+      "logps/chosen": -32.56236267089844,
+      "logps/rejected": -32.517822265625,
+      "loss": 0.9863,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.010110180824995041,
+      "rewards/margins": 0.013717299327254295,
+      "rewards/rejected": -0.003607118036597967,
       "step": 40
     },
     {
       "epoch": 0.13,
       "learning_rate": 4.987541037542187e-06,
-      "logits/chosen": -1.8617979288101196,
-      "logits/rejected": -1.8510282039642334,
-      "logps/chosen": -33.562862396240234,
-      "logps/rejected": -35.4472770690918,
-      "loss": 0.5775,
-      "rewards/accuracies": 0.42500001192092896,
-      "rewards/chosen": -0.0037666684947907925,
-      "rewards/margins": -0.007314275950193405,
-      "rewards/rejected": 0.0035476074554026127,
+      "logits/chosen": -1.862694501876831,
+      "logits/rejected": -1.8519262075424194,
+      "logps/chosen": -33.541160583496094,
+      "logps/rejected": -35.44048309326172,
+      "loss": 0.9969,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": 0.011423684656620026,
+      "rewards/margins": 0.003121361369267106,
+      "rewards/rejected": 0.00830232072621584,
       "step": 50
     },
     {
       "epoch": 0.16,
       "learning_rate": 4.954691471941119e-06,
-      "logits/chosen": -1.9399009943008423,
-      "logits/rejected": -1.9418485164642334,
-      "logps/chosen": -32.5726203918457,
-      "logps/rejected": -33.18496322631836,
-      "loss": 0.5967,
-      "rewards/accuracies": 0.5249999761581421,
-      "rewards/chosen": 0.019294817000627518,
-      "rewards/margins": 0.028803948312997818,
-      "rewards/rejected": -0.009509134106338024,
+      "logits/chosen": -1.9417282342910767,
+      "logits/rejected": -1.9436867237091064,
+      "logps/chosen": -32.52958679199219,
+      "logps/rejected": -33.216880798339844,
+      "loss": 0.9296,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": 0.04941769689321518,
+      "rewards/margins": 0.08126799017190933,
+      "rewards/rejected": -0.031850285828113556,
       "step": 60
     },
     {
       "epoch": 0.18,
       "learning_rate": 4.901618883413549e-06,
-      "logits/chosen": -2.0729470252990723,
-      "logits/rejected": -2.077918529510498,
-      "logps/chosen": -33.93268585205078,
-      "logps/rejected": -36.637229919433594,
-      "loss": 0.5297,
-      "rewards/accuracies": 0.5874999761581421,
-      "rewards/chosen": 0.039063483476638794,
-      "rewards/margins": 0.08691467344760895,
-      "rewards/rejected": -0.04785118252038956,
+      "logits/chosen": -2.0729386806488037,
+      "logits/rejected": -2.0779125690460205,
+      "logps/chosen": -33.99254608154297,
+      "logps/rejected": -36.62586212158203,
+      "loss": 0.9629,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.0028400986921042204,
+      "rewards/margins": 0.037055134773254395,
+      "rewards/rejected": -0.03989524394273758,
       "step": 70
     },
     {
       "epoch": 0.21,
       "learning_rate": 4.828760511501322e-06,
-      "logits/chosen": -1.9366486072540283,
-      "logits/rejected": -1.939780592918396,
-      "logps/chosen": -34.30228805541992,
-      "logps/rejected": -34.652069091796875,
-      "loss": 0.4209,
-      "rewards/accuracies": 0.637499988079071,
-      "rewards/chosen": 0.08786438405513763,
-      "rewards/margins": 0.12962770462036133,
-      "rewards/rejected": -0.041763320565223694,
+      "logits/chosen": -1.9335737228393555,
+      "logits/rejected": -1.9366981983184814,
+      "logps/chosen": -34.332157135009766,
+      "logps/rejected": -34.641021728515625,
+      "loss": 0.9026,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": 0.06695752590894699,
+      "rewards/margins": 0.10098665952682495,
+      "rewards/rejected": -0.03402913734316826,
       "step": 80
     },
     {
       "epoch": 0.23,
       "learning_rate": 4.7367166013034295e-06,
-      "logits/chosen": -1.946979284286499,
-      "logits/rejected": -1.951521635055542,
-      "logps/chosen": -32.4210205078125,
-      "logps/rejected": -32.337799072265625,
-      "loss": 0.6191,
-      "rewards/accuracies": 0.5375000238418579,
-      "rewards/chosen": 0.03248247504234314,
-      "rewards/margins": 0.017702888697385788,
-      "rewards/rejected": 0.014779585413634777,
+      "logits/chosen": -1.9401309490203857,
+      "logits/rejected": -1.9446433782577515,
+      "logps/chosen": -32.37213897705078,
+      "logps/rejected": -32.343849182128906,
+      "loss": 0.9438,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": 0.06670050323009491,
+      "rewards/margins": 0.05615914613008499,
+      "rewards/rejected": 0.010541360825300217,
       "step": 90
     },
     {
       "epoch": 0.26,
       "learning_rate": 4.626245458345211e-06,
-      "logits/chosen": -2.045017719268799,
-      "logits/rejected": -2.043015241622925,
-      "logps/chosen": -32.16361618041992,
-      "logps/rejected": -31.273128509521484,
-      "loss": 0.4711,
-      "rewards/accuracies": 0.5874999761581421,
-      "rewards/chosen": 0.05405203625559807,
-      "rewards/margins": 0.05889582633972168,
-      "rewards/rejected": -0.004843792412430048,
+      "logits/chosen": -2.037087917327881,
+      "logits/rejected": -2.035101890563965,
+      "logps/chosen": -32.13945388793945,
+      "logps/rejected": -31.313283920288086,
+      "loss": 0.8961,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": 0.07096613943576813,
+      "rewards/margins": 0.10391455888748169,
+      "rewards/rejected": -0.032948415726423264,
       "step": 100
     },
     {
       "epoch": 0.26,
-      "eval_logits/chosen": -2.240097761154175,
-      "eval_logits/rejected": -2.2352383136749268,
-      "eval_logps/chosen": -34.01129150390625,
-      "eval_logps/rejected": -37.49793243408203,
-      "eval_loss": 0.5754798054695129,
-      "eval_rewards/accuracies": 0.5195183157920837,
-      "eval_rewards/chosen": 0.016279777511954308,
-      "eval_rewards/margins": 0.0031977586913853884,
-      "eval_rewards/rejected": 0.013082021847367287,
-      "eval_runtime": 145.9275,
-      "eval_samples_per_second": 2.35,
+      "eval_logits/chosen": -2.231482982635498,
+      "eval_logits/rejected": -2.226637363433838,
+      "eval_logps/chosen": -34.02524185180664,
+      "eval_logps/rejected": -37.54085159301758,
+      "eval_loss": 0.9766585230827332,
+      "eval_rewards/accuracies": 0.5365448594093323,
+      "eval_rewards/chosen": 0.006516099441796541,
+      "eval_rewards/margins": 0.02347717247903347,
+      "eval_rewards/rejected": -0.016961071640253067,
+      "eval_runtime": 145.8279,
+      "eval_samples_per_second": 2.352,
       "eval_steps_per_second": 0.295,
       "step": 100
     },
     {
       "epoch": 0.29,
       "learning_rate": 4.498257201263691e-06,
-      "logits/chosen": -2.0017762184143066,
-      "logits/rejected": -1.9993689060211182,
-      "logps/chosen": -33.13564682006836,
-      "logps/rejected": -33.997642517089844,
-      "loss": 0.6759,
-      "rewards/accuracies": 0.625,
-      "rewards/chosen": 0.07571890205144882,
-      "rewards/margins": 0.05350743606686592,
-      "rewards/rejected": 0.022211460396647453,
+      "logits/chosen": -1.9920060634613037,
+      "logits/rejected": -1.9896419048309326,
+      "logps/chosen": -33.146766662597656,
+      "logps/rejected": -34.02008819580078,
+      "loss": 0.9486,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": 0.0679345652461052,
+      "rewards/margins": 0.061437882483005524,
+      "rewards/rejected": 0.006496679037809372,
       "step": 110
     },
     {
       "epoch": 0.31,
       "learning_rate": 4.353806263777678e-06,
-      "logits/chosen": -2.013158082962036,
-      "logits/rejected": -2.004814624786377,
-      "logps/chosen": -32.32666015625,
-      "logps/rejected": -32.12488555908203,
-      "loss": 0.5572,
-      "rewards/accuracies": 0.550000011920929,
-      "rewards/chosen": 0.08277235180139542,
-      "rewards/margins": 0.05714136362075806,
-      "rewards/rejected": 0.02563098631799221,
+      "logits/chosen": -2.003952741622925,
+      "logits/rejected": -1.9956319332122803,
+      "logps/chosen": -32.33639144897461,
+      "logps/rejected": -32.133079528808594,
+      "loss": 0.9488,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": 0.07595954835414886,
+      "rewards/margins": 0.05606143921613693,
+      "rewards/rejected": 0.019898109138011932,
       "step": 120
     },
     {
       "epoch": 0.34,
       "learning_rate": 4.1940827077152755e-06,
-      "logits/chosen": -2.039836883544922,
-      "logits/rejected": -2.031825542449951,
-      "logps/chosen": -30.349987030029297,
-      "logps/rejected": -32.029136657714844,
-      "loss": 0.5109,
+      "logits/chosen": -2.0316150188446045,
+      "logits/rejected": -2.0236544609069824,
+      "logps/chosen": -30.298206329345703,
+      "logps/rejected": -32.07080841064453,
+      "loss": 0.8911,
       "rewards/accuracies": 0.574999988079071,
-      "rewards/chosen": 0.0880090743303299,
-      "rewards/margins": 0.07130132615566254,
-      "rewards/rejected": 0.016707751899957657,
+      "rewards/chosen": 0.12425784766674042,
+      "rewards/margins": 0.1367165446281433,
+      "rewards/rejected": -0.0124586820602417,
       "step": 130
     },
     {
       "epoch": 0.36,
       "learning_rate": 4.0204024186666215e-06,
-      "logits/chosen": -1.9694139957427979,
-      "logits/rejected": -1.9796711206436157,
-      "logps/chosen": -31.24441909790039,
-      "logps/rejected": -32.56440353393555,
-      "loss": 0.5037,
-      "rewards/accuracies": 0.6499999761581421,
-      "rewards/chosen": 0.12266886234283447,
-      "rewards/margins": 0.14333835244178772,
-      "rewards/rejected": -0.02066950313746929,
+      "logits/chosen": -1.962376356124878,
+      "logits/rejected": -1.9726108312606812,
+      "logps/chosen": -31.235275268554688,
+      "logps/rejected": -32.56925964355469,
+      "loss": 0.8524,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": 0.1290682703256607,
+      "rewards/margins": 0.15313370525836945,
+      "rewards/rejected": -0.024065453559160233,
       "step": 140
     },
     {
       "epoch": 0.39,
       "learning_rate": 3.834196265035119e-06,
-      "logits/chosen": -1.8828538656234741,
-      "logits/rejected": -1.8840004205703735,
-      "logps/chosen": -34.006561279296875,
-      "logps/rejected": -34.81753921508789,
-      "loss": 0.4564,
-      "rewards/accuracies": 0.6625000238418579,
-      "rewards/chosen": 0.13028928637504578,
-      "rewards/margins": 0.17248663306236267,
-      "rewards/rejected": -0.04219735041260719,
+      "logits/chosen": -1.8732143640518188,
+      "logits/rejected": -1.874371886253357,
+      "logps/chosen": -33.8985481262207,
+      "logps/rejected": -34.81908416748047,
+      "loss": 0.786,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": 0.20590214431285858,
+      "rewards/margins": 0.24918103218078613,
+      "rewards/rejected": -0.04327889531850815,
       "step": 150
     },
     {
       "epoch": 0.42,
       "learning_rate": 3.636998309800573e-06,
-      "logits/chosen": -1.9349613189697266,
-      "logits/rejected": -1.9315464496612549,
-      "logps/chosen": -36.059181213378906,
-      "logps/rejected": -32.72267150878906,
-      "loss": 0.487,
-      "rewards/accuracies": 0.6625000238418579,
-      "rewards/chosen": 0.07874591648578644,
-      "rewards/margins": 0.07869003713130951,
-      "rewards/rejected": 5.587190389633179e-05,
+      "logits/chosen": -1.924538016319275,
+      "logits/rejected": -1.921121597290039,
+      "logps/chosen": -36.01353454589844,
+      "logps/rejected": -32.723262786865234,
+      "loss": 0.8894,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.11069967597723007,
+      "rewards/margins": 0.11105670034885406,
+      "rewards/rejected": -0.0003570284752640873,
       "step": 160
     },
     {
       "epoch": 0.44,
       "learning_rate": 3.4304331721118078e-06,
-      "logits/chosen": -2.036032199859619,
-      "logits/rejected": -2.028651714324951,
-      "logps/chosen": -33.573509216308594,
-      "logps/rejected": -31.36956214904785,
-      "loss": 0.4465,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": 0.15704451501369476,
-      "rewards/margins": 0.16944527626037598,
-      "rewards/rejected": -0.012400749139487743,
+      "logits/chosen": -2.0247209072113037,
+      "logits/rejected": -2.017392635345459,
+      "logps/chosen": -33.49879455566406,
+      "logps/rejected": -31.44363784790039,
+      "loss": 0.7434,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": 0.2093469649553299,
+      "rewards/margins": 0.2735980153083801,
+      "rewards/rejected": -0.06425107270479202,
       "step": 170
     },
     {
       "epoch": 0.47,
       "learning_rate": 3.2162026428305436e-06,
-      "logits/chosen": -2.042617082595825,
-      "logits/rejected": -2.0478556156158447,
-      "logps/chosen": -32.33074188232422,
-      "logps/rejected": -32.472496032714844,
-      "loss": 0.3727,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": 0.14983226358890533,
-      "rewards/margins": 0.14098060131072998,
-      "rewards/rejected": 0.008851657621562481,
+      "logits/chosen": -2.031324863433838,
+      "logits/rejected": -2.0365915298461914,
+      "logps/chosen": -32.253074645996094,
+      "logps/rejected": -32.45112609863281,
+      "loss": 0.8312,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.20420043170452118,
+      "rewards/margins": 0.18039169907569885,
+      "rewards/rejected": 0.023808732628822327,
       "step": 180
     },
     {
       "epoch": 0.49,
       "learning_rate": 2.996071664294641e-06,
-      "logits/chosen": -2.0445618629455566,
-      "logits/rejected": -2.0417864322662354,
-      "logps/chosen": -31.317855834960938,
-      "logps/rejected": -31.348400115966797,
-      "loss": 0.4609,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": 0.11976919323205948,
-      "rewards/margins": 0.14090058207511902,
-      "rewards/rejected": -0.021131375804543495,
+      "logits/chosen": -2.0317559242248535,
+      "logits/rejected": -2.0289719104766846,
+      "logps/chosen": -31.279537200927734,
+      "logps/rejected": -31.34115219116211,
+      "loss": 0.8405,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": 0.14659160375595093,
+      "rewards/margins": 0.16264860332012177,
+      "rewards/rejected": -0.016057008877396584,
       "step": 190
     },
     {
       "epoch": 0.52,
       "learning_rate": 2.7718537898066833e-06,
-      "logits/chosen": -1.9139589071273804,
-      "logits/rejected": -1.9186115264892578,
-      "logps/chosen": -31.420101165771484,
-      "logps/rejected": -32.79216766357422,
-      "loss": 0.5061,
-      "rewards/accuracies": 0.675000011920929,
-      "rewards/chosen": 0.13032536208629608,
-      "rewards/margins": 0.13119173049926758,
-      "rewards/rejected": -0.0008663811022415757,
+      "logits/chosen": -1.9018064737319946,
+      "logits/rejected": -1.9064457416534424,
+      "logps/chosen": -31.301830291748047,
+      "logps/rejected": -32.8339729309082,
+      "loss": 0.7699,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": 0.21311891078948975,
+      "rewards/margins": 0.2432461678981781,
+      "rewards/rejected": -0.03012726828455925,
       "step": 200
     },
     {
       "epoch": 0.52,
-      "eval_logits/chosen": -2.2385354042053223,
-      "eval_logits/rejected": -2.2336885929107666,
-      "eval_logps/chosen": -34.05002975463867,
-      "eval_logps/rejected": -37.54545974731445,
-      "eval_loss": 0.5877167582511902,
-      "eval_rewards/accuracies": 0.4991694688796997,
-      "eval_rewards/chosen": -0.010832725092768669,
-      "eval_rewards/margins": 0.009353035129606724,
-      "eval_rewards/rejected": -0.020185761153697968,
-      "eval_runtime": 145.6958,
-      "eval_samples_per_second": 2.354,
+      "eval_logits/chosen": -2.2282192707061768,
+      "eval_logits/rejected": -2.223379611968994,
+      "eval_logps/chosen": -34.05467224121094,
+      "eval_logps/rejected": -37.57374572753906,
+      "eval_loss": 0.974229633808136,
+      "eval_rewards/accuracies": 0.530315637588501,
+      "eval_rewards/chosen": -0.01408342458307743,
+      "eval_rewards/margins": 0.02590302750468254,
+      "eval_rewards/rejected": -0.03998645395040512,
+      "eval_runtime": 145.7826,
+      "eval_samples_per_second": 2.353,
       "eval_steps_per_second": 0.295,
       "step": 200
     },
     {
       "epoch": 0.55,
       "learning_rate": 2.5453962426402006e-06,
-      "logits/chosen": -2.026780366897583,
-      "logits/rejected": -2.0374534130096436,
-      "logps/chosen": -31.786846160888672,
-      "logps/rejected": -33.922447204589844,
-      "loss": 0.4225,
-      "rewards/accuracies": 0.675000011920929,
-      "rewards/chosen": 0.12911871075630188,
-      "rewards/margins": 0.17094141244888306,
-      "rewards/rejected": -0.041822709143161774,
+      "logits/chosen": -2.014596462249756,
+      "logits/rejected": -2.0252418518066406,
+      "logps/chosen": -31.797557830810547,
+      "logps/rejected": -33.982398986816406,
+      "loss": 0.8139,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.12162177264690399,
+      "rewards/margins": 0.20541362464427948,
+      "rewards/rejected": -0.08379185199737549,
       "step": 210
     },
     {
       "epoch": 0.57,
       "learning_rate": 2.3185646976551794e-06,
-      "logits/chosen": -1.9182548522949219,
-      "logits/rejected": -1.933038353919983,
-      "logps/chosen": -29.936147689819336,
-      "logps/rejected": -31.559993743896484,
-      "loss": 0.4429,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": 0.12277615070343018,
-      "rewards/margins": 0.12404797971248627,
-      "rewards/rejected": -0.0012718416983261704,
+      "logits/chosen": -1.9068737030029297,
+      "logits/rejected": -1.9216482639312744,
+      "logps/chosen": -29.838830947875977,
+      "logps/rejected": -31.62994956970215,
+      "loss": 0.766,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.1908968687057495,
+      "rewards/margins": 0.24113738536834717,
+      "rewards/rejected": -0.05024053901433945,
       "step": 220
     },
     {
       "epoch": 0.6,
       "learning_rate": 2.0932279108998323e-06,
-      "logits/chosen": -1.9737575054168701,
-      "logits/rejected": -1.9777438640594482,
-      "logps/chosen": -33.20977783203125,
-      "logps/rejected": -31.5734920501709,
-      "loss": 0.3768,
-      "rewards/accuracies": 0.637499988079071,
-      "rewards/chosen": 0.14538228511810303,
-      "rewards/margins": 0.16231416165828705,
-      "rewards/rejected": -0.016931891441345215,
+      "logits/chosen": -1.9636850357055664,
+      "logits/rejected": -1.967655897140503,
+      "logps/chosen": -33.12433624267578,
+      "logps/rejected": -31.637094497680664,
+      "loss": 0.7833,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.20519034564495087,
+      "rewards/margins": 0.2666449546813965,
+      "rewards/rejected": -0.06145460531115532,
       "step": 230
     },
     {
       "epoch": 0.62,
       "learning_rate": 1.8712423238279358e-06,
-      "logits/chosen": -1.9738047122955322,
-      "logits/rejected": -1.9518985748291016,
-      "logps/chosen": -33.94301223754883,
-      "logps/rejected": -35.023075103759766,
-      "loss": 0.4304,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": 0.1475195735692978,
-      "rewards/margins": 0.19811663031578064,
-      "rewards/rejected": -0.05059707164764404,
+      "logits/chosen": -1.9611847400665283,
+      "logits/rejected": -1.9393657445907593,
+      "logps/chosen": -33.867958068847656,
+      "logps/rejected": -35.12390899658203,
+      "loss": 0.737,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.2000524252653122,
+      "rewards/margins": 0.32123422622680664,
+      "rewards/rejected": -0.12118180096149445,
       "step": 240
     },
     {
       "epoch": 0.65,
       "learning_rate": 1.6544367689701824e-06,
-      "logits/chosen": -2.0155911445617676,
-      "logits/rejected": -2.0122878551483154,
-      "logps/chosen": -32.76276397705078,
-      "logps/rejected": -36.25225830078125,
-      "loss": 0.4155,
-      "rewards/accuracies": 0.637499988079071,
-      "rewards/chosen": 0.11361332982778549,
-      "rewards/margins": 0.1327473670244217,
-      "rewards/rejected": -0.019134048372507095,
+      "logits/chosen": -2.001960277557373,
+      "logits/rejected": -1.9986454248428345,
+      "logps/chosen": -32.73499298095703,
+      "logps/rejected": -36.28093719482422,
+      "loss": 0.8434,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.13304933905601501,
+      "rewards/margins": 0.17225751280784607,
+      "rewards/rejected": -0.03920816630125046,
       "step": 250
     },
     {
       "epoch": 0.68,
       "learning_rate": 1.4445974030621963e-06,
-      "logits/chosen": -1.8830368518829346,
-      "logits/rejected": -1.880629539489746,
-      "logps/chosen": -34.01710891723633,
-      "logps/rejected": -35.53314971923828,
-      "loss": 0.4219,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": 0.11990444362163544,
-      "rewards/margins": 0.1398761123418808,
-      "rewards/rejected": -0.019971664994955063,
+      "logits/chosen": -1.8687576055526733,
+      "logits/rejected": -1.866320013999939,
+      "logps/chosen": -33.981781005859375,
+      "logps/rejected": -35.54584503173828,
+      "loss": 0.8296,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.14463281631469727,
+      "rewards/margins": 0.1734902262687683,
+      "rewards/rejected": -0.028857415542006493,
       "step": 260
     },
     {
       "epoch": 0.7,
       "learning_rate": 1.243452991757889e-06,
-      "logits/chosen": -1.8685131072998047,
-      "logits/rejected": -1.8660367727279663,
-      "logps/chosen": -34.2363395690918,
-      "logps/rejected": -31.768321990966797,
-      "loss": 0.4918,
-      "rewards/accuracies": 0.675000011920929,
-      "rewards/chosen": 0.11288031190633774,
-      "rewards/margins": 0.11077453196048737,
-      "rewards/rejected": 0.0021057710982859135,
+      "logits/chosen": -1.853939414024353,
+      "logits/rejected": -1.8515437841415405,
+      "logps/chosen": -34.234046936035156,
+      "logps/rejected": -31.837631225585938,
+      "loss": 0.8487,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": 0.11448518931865692,
+      "rewards/margins": 0.16089434921741486,
+      "rewards/rejected": -0.04640916362404823,
       "step": 270
     },
     {
       "epoch": 0.73,
       "learning_rate": 1.0526606671603523e-06,
-      "logits/chosen": -1.9722087383270264,
-      "logits/rejected": -1.961612343788147,
-      "logps/chosen": -35.04926300048828,
-      "logps/rejected": -31.826486587524414,
-      "loss": 0.3417,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": 0.21516695618629456,
-      "rewards/margins": 0.20501188933849335,
-      "rewards/rejected": 0.010155050083994865,
+      "logits/chosen": -1.9570705890655518,
+      "logits/rejected": -1.9465986490249634,
+      "logps/chosen": -35.030006408691406,
+      "logps/rejected": -31.88030433654785,
+      "loss": 0.7553,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": 0.22864773869514465,
+      "rewards/margins": 0.2561652660369873,
+      "rewards/rejected": -0.02751758135855198,
       "step": 280
     },
     {
       "epoch": 0.75,
       "learning_rate": 8.737922755071455e-07,
-      "logits/chosen": -2.068380832672119,
-      "logits/rejected": -2.05340313911438,
-      "logps/chosen": -30.748706817626953,
-      "logps/rejected": -32.63469314575195,
-      "loss": 0.4741,
-      "rewards/accuracies": 0.625,
-      "rewards/chosen": 0.12279413640499115,
-      "rewards/margins": 0.11079368740320206,
-      "rewards/rejected": 0.012000440619885921,
+      "logits/chosen": -2.0522685050964355,
+      "logits/rejected": -2.0373730659484863,
+      "logps/chosen": -30.7352352142334,
+      "logps/rejected": -32.61699676513672,
+      "loss": 0.9072,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.13222160935401917,
+      "rewards/margins": 0.10783363878726959,
+      "rewards/rejected": 0.024387964978814125,
       "step": 290
     },
     {
       "epoch": 0.78,
       "learning_rate": 7.08321427484816e-07,
-      "logits/chosen": -1.9397351741790771,
-      "logits/rejected": -1.9371974468231201,
-      "logps/chosen": -32.603660583496094,
-      "logps/rejected": -30.863941192626953,
-      "loss": 0.3371,
-      "rewards/accuracies": 0.7749999761581421,
-      "rewards/chosen": 0.2264125794172287,
-      "rewards/margins": 0.25940603017807007,
-      "rewards/rejected": -0.03299345448613167,
+      "logits/chosen": -1.9228973388671875,
+      "logits/rejected": -1.9203764200210571,
+      "logps/chosen": -32.44710159301758,
+      "logps/rejected": -30.934436798095703,
+      "loss": 0.6723,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": 0.3360074460506439,
+      "rewards/margins": 0.418344646692276,
+      "rewards/rejected": -0.08233721554279327,
       "step": 300
     },
     {
       "epoch": 0.78,
-      "eval_logits/chosen": -2.2370686531066895,
-      "eval_logits/rejected": -2.2322022914886475,
-      "eval_logps/chosen": -34.03439712524414,
-      "eval_logps/rejected": -37.53529357910156,
-      "eval_loss": 0.5842657089233398,
-      "eval_rewards/accuracies": 0.5278239250183105,
-      "eval_rewards/chosen": 0.00010742468293756247,
-      "eval_rewards/margins": 0.013178782537579536,
-      "eval_rewards/rejected": -0.013071359135210514,
-      "eval_runtime": 145.5131,
-      "eval_samples_per_second": 2.357,
-      "eval_steps_per_second": 0.296,
+      "eval_logits/chosen": -2.2234153747558594,
+      "eval_logits/rejected": -2.218602180480957,
+      "eval_logps/chosen": -34.08680725097656,
+      "eval_logps/rejected": -37.60466003417969,
+      "eval_loss": 0.976102888584137,
+      "eval_rewards/accuracies": 0.529900312423706,
+      "eval_rewards/chosen": -0.036580219864845276,
+      "eval_rewards/margins": 0.02504708059132099,
+      "eval_rewards/rejected": -0.06162729859352112,
+      "eval_runtime": 145.7665,
+      "eval_samples_per_second": 2.353,
+      "eval_steps_per_second": 0.295,
       "step": 300
     },
     {
       "epoch": 0.81,
       "learning_rate": 5.576113578589035e-07,
-      "logits/chosen": -1.9219080209732056,
-      "logits/rejected": -1.9186557531356812,
-      "logps/chosen": -31.35263442993164,
-      "logps/rejected": -33.746925354003906,
-      "loss": 0.4182,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": 0.17550265789031982,
-      "rewards/margins": 0.17884370684623718,
-      "rewards/rejected": -0.0033410280011594296,
+      "logits/chosen": -1.9082481861114502,
+      "logits/rejected": -1.9050118923187256,
+      "logps/chosen": -31.349285125732422,
+      "logps/rejected": -33.84658432006836,
+      "loss": 0.7796,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": 0.17784307897090912,
+      "rewards/margins": 0.2509470283985138,
+      "rewards/rejected": -0.07310393452644348,
       "step": 310
     },
     {
       "epoch": 0.83,
       "learning_rate": 4.229036944380913e-07,
-      "logits/chosen": -1.9732515811920166,
-      "logits/rejected": -1.9609721899032593,
-      "logps/chosen": -34.435001373291016,
-      "logps/rejected": -33.593482971191406,
-      "loss": 0.3778,
-      "rewards/accuracies": 0.675000011920929,
-      "rewards/chosen": 0.11274988949298859,
-      "rewards/margins": 0.15099774301052094,
-      "rewards/rejected": -0.03824785351753235,
+      "logits/chosen": -1.9580894708633423,
+      "logits/rejected": -1.9458973407745361,
+      "logps/chosen": -34.3031005859375,
+      "logps/rejected": -33.67659378051758,
+      "loss": 0.7302,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": 0.2050826996564865,
+      "rewards/margins": 0.3015114367008209,
+      "rewards/rejected": -0.09642868489027023,
       "step": 320
     },
     {
       "epoch": 0.86,
       "learning_rate": 3.053082288996112e-07,
-      "logits/chosen": -2.0085272789001465,
-      "logits/rejected": -2.0070912837982178,
-      "logps/chosen": -33.31962585449219,
-      "logps/rejected": -32.51466751098633,
-      "loss": 0.4061,
-      "rewards/accuracies": 0.675000011920929,
-      "rewards/chosen": 0.1163158044219017,
-      "rewards/margins": 0.14087431132793427,
-      "rewards/rejected": -0.024558518081903458,
+      "logits/chosen": -1.9932842254638672,
+      "logits/rejected": -1.9918495416641235,
+      "logps/chosen": -33.17847442626953,
+      "logps/rejected": -32.54157638549805,
+      "loss": 0.7677,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": 0.21512338519096375,
+      "rewards/margins": 0.2585209906101227,
+      "rewards/rejected": -0.04339758679270744,
       "step": 330
     },
     {
       "epoch": 0.88,
       "learning_rate": 2.0579377374915805e-07,
-      "logits/chosen": -2.0951385498046875,
-      "logits/rejected": -2.0793802738189697,
-      "logps/chosen": -33.88261032104492,
-      "logps/rejected": -33.10671615600586,
-      "loss": 0.4042,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": 0.20774619281291962,
-      "rewards/margins": 0.19982238113880157,
-      "rewards/rejected": 0.00792380329221487,
+      "logits/chosen": -2.0800719261169434,
+      "logits/rejected": -2.064396381378174,
+      "logps/chosen": -33.80484390258789,
+      "logps/rejected": -33.1123046875,
+      "loss": 0.7636,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": 0.2621825039386749,
+      "rewards/margins": 0.25817227363586426,
+      "rewards/rejected": 0.004010227043181658,
       "step": 340
     },
     {
       "epoch": 0.91,
       "learning_rate": 1.2518018074041684e-07,
-      "logits/chosen": -1.9677631855010986,
-      "logits/rejected": -1.9668546915054321,
-      "logps/chosen": -32.957977294921875,
-      "logps/rejected": -32.483890533447266,
-      "loss": 0.4874,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": 0.2268829345703125,
-      "rewards/margins": 0.2400023490190506,
-      "rewards/rejected": -0.013119394890964031,
+      "logits/chosen": -1.9522559642791748,
+      "logits/rejected": -1.951424241065979,
+      "logps/chosen": -32.8499755859375,
+      "logps/rejected": -32.56407165527344,
+      "loss": 0.6881,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": 0.3024839758872986,
+      "rewards/margins": 0.3717316687107086,
+      "rewards/rejected": -0.06924761831760406,
       "step": 350
     },
     {
       "epoch": 0.94,
       "learning_rate": 6.41315865106129e-08,
-      "logits/chosen": -1.9235498905181885,
-      "logits/rejected": -1.933876395225525,
-      "logps/chosen": -32.016578674316406,
-      "logps/rejected": -35.32080841064453,
-      "loss": 0.4297,
+      "logits/chosen": -1.9075695276260376,
+      "logits/rejected": -1.917851209640503,
+      "logps/chosen": -31.882221221923828,
+      "logps/rejected": -35.31555938720703,
+      "loss": 0.7689,
       "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": 0.13167943060398102,
-      "rewards/margins": 0.15411263704299927,
-      "rewards/rejected": -0.02243318222463131,
+      "rewards/chosen": 0.22572879493236542,
+      "rewards/margins": 0.24449090659618378,
+      "rewards/rejected": -0.018762132152915,
       "step": 360
     },
     {
       "epoch": 0.96,
       "learning_rate": 2.3150941078050325e-08,
-      "logits/chosen": -2.062828540802002,
-      "logits/rejected": -2.056312322616577,
-      "logps/chosen": -33.43348693847656,
-      "logps/rejected": -29.213970184326172,
-      "loss": 0.3963,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": 0.15293903648853302,
-      "rewards/margins": 0.14422088861465454,
-      "rewards/rejected": 0.008718125522136688,
+      "logits/chosen": -2.0478641986846924,
+      "logits/rejected": -2.041414737701416,
+      "logps/chosen": -33.331912994384766,
+      "logps/rejected": -29.259756088256836,
+      "loss": 0.7658,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.2240387201309204,
+      "rewards/margins": 0.2473684549331665,
+      "rewards/rejected": -0.02332974039018154,
       "step": 370
     },
     {
       "epoch": 0.99,
       "learning_rate": 2.575864278703266e-09,
-      "logits/chosen": -1.9224169254302979,
-      "logits/rejected": -1.9245796203613281,
-      "logps/chosen": -33.9437255859375,
-      "logps/rejected": -30.860538482666016,
-      "loss": 0.4119,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": 0.20672598481178284,
-      "rewards/margins": 0.19826285541057587,
-      "rewards/rejected": 0.008463130332529545,
+      "logits/chosen": -1.9081246852874756,
+      "logits/rejected": -1.9103105068206787,
+      "logps/chosen": -33.882568359375,
+      "logps/rejected": -30.96805191040039,
+      "loss": 0.741,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": 0.24953576922416687,
+      "rewards/margins": 0.3163323998451233,
+      "rewards/rejected": -0.06679664552211761,
       "step": 380
     },
     {
       "epoch": 1.0,
       "step": 385,
       "total_flos": 0.0,
-      "train_loss": 0.46899499707407766,
-      "train_runtime": 3249.86,
+      "train_loss": 0.8438688600218142,
+      "train_runtime": 3250.9917,
       "train_samples_per_second": 0.947,
       "train_steps_per_second": 0.118
     }