Model save

README.md

@@ -1,13 +1,10 @@
 ---
 library_name: peft
 tags:
-- alignment-handbook
 - trl
 - dpo
 - generated_from_trainer
 base_model: NbAiLab/nb-gpt-j-6B-v2
-datasets:
-- hugodk-sch/aftonposten_title_prefs
 model-index:
 - name: aftonposten-6b-align-scan
   results: []
@@ -18,17 +15,17 @@ should probably proofread and complete it, then remove this comment. -->
 
 # aftonposten-6b-align-scan
 
-This model is a fine-tuned version of [data/ap-gpt-j-6b-sft-qlora-04-08](https://huggingface.co/data/ap-gpt-j-6b-sft-qlora-04-08) on the hugodk-sch/aftonposten_title_prefs dataset.
+This model is a fine-tuned version of [NbAiLab/nb-gpt-j-6B-v2](https://huggingface.co/NbAiLab/nb-gpt-j-6B-v2) on an unknown dataset.
 It achieves the following results on the evaluation set:
-- Loss: 0.4913
-- Rewards/chosen: 0.1914
-- Rewards/rejected: 0.1556
-- Rewards/accuracies: 0.5453
-- Rewards/margins: 0.0358
-- Logps/rejected: -37.2944
-- Logps/chosen: -33.7611
-- Logits/rejected: -2.2272
-- Logits/chosen: -2.2320
+- Loss: 0.5843
+- Rewards/chosen: 0.0001
+- Rewards/rejected: -0.0131
+- Rewards/accuracies: 0.5278
+- Rewards/margins: 0.0132
+- Logps/rejected: -37.5353
+- Logps/chosen: -34.0344
+- Logits/rejected: -2.2322
+- Logits/chosen: -2.2371
 
 ## Model description
 
@@ -63,9 +60,9 @@ The following hyperparameters were used during training:
 
 | Training Loss | Epoch | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
 |:-------------:|:-----:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
-| 0.4799        | 0.26  | 100  | 0.4978          | 0.1217         | 0.1117           | 0.5341             | 0.0100          | -37.3570       | -33.8607     | -2.2333         | -2.2381       |
-| 0.4453        | 0.52  | 200  | 0.4928          | 0.1862         | 0.1561           | 0.5370             | 0.0302          | -37.2937       | -33.7685     | -2.2299         | -2.2347       |
-| 0.3947        | 0.78  | 300  | 0.4910          | 0.1956         | 0.1591           | 0.5565             | 0.0365          | -37.2894       | -33.7551     | -2.2274         | -2.2322       |
+| 0.4711        | 0.26  | 100  | 0.5755          | 0.0163         | 0.0131           | 0.5195             | 0.0032          | -37.4979       | -34.0113     | -2.2352         | -2.2401       |
+| 0.5061        | 0.52  | 200  | 0.5877          | -0.0108        | -0.0202          | 0.4992             | 0.0094          | -37.5455       | -34.0500     | -2.2337         | -2.2385       |
+| 0.3371        | 0.78  | 300  | 0.5843          | 0.0001         | -0.0131          | 0.5278             | 0.0132          | -37.5353       | -34.0344     | -2.2322         | -2.2371       |
 
 
 ### Framework versions

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8820dc9f9b650b645197e8167eedbab8af35175306fc4608b15579e49aa98e89
+oid sha256:b6e3e8f071db6cc19f55ac20735e7ca5976169e76cd665df27f42360dea5c708
 size 176183216

all_results.json

@@ -1,20 +1,7 @@
 {
     "epoch": 1.0,
-    "eval_logits/chosen": -2.2319846153259277,
-    "eval_logits/rejected": -2.227167844772339,
-    "eval_logps/chosen": -33.76113510131836,
-    "eval_logps/rejected": -37.29436492919922,
-    "eval_loss": 0.49125906825065613,
-    "eval_rewards/accuracies": 0.545265793800354,
-    "eval_rewards/chosen": 0.19139321148395538,
-    "eval_rewards/margins": 0.03581271693110466,
-    "eval_rewards/rejected": 0.15558050572872162,
-    "eval_runtime": 145.6689,
-    "eval_samples": 343,
-    "eval_samples_per_second": 2.355,
-    "eval_steps_per_second": 0.295,
-    "train_loss": 0.45851393303313814,
-    "train_runtime": 3252.0771,
+    "train_loss": 0.46899499707407766,
+    "train_runtime": 3249.86,
     "train_samples": 3079,
     "train_samples_per_second": 0.947,
     "train_steps_per_second": 0.118

train_results.json

@@ -1,7 +1,7 @@
 {
     "epoch": 1.0,
-    "train_loss": 0.45851393303313814,
-    "train_runtime": 3252.0771,
+    "train_loss": 0.46899499707407766,
+    "train_runtime": 3249.86,
     "train_samples": 3079,
     "train_samples_per_second": 0.947,
     "train_steps_per_second": 0.118

trainer_state.json

@@ -15,7 +15,7 @@
       "logits/rejected": -1.7377450466156006,
       "logps/chosen": -29.553977966308594,
       "logps/rejected": -42.813133239746094,
-      "loss": 0.5,
+      "loss": 0.5102,
       "rewards/accuracies": 0.0,
       "rewards/chosen": 0.0,
       "rewards/margins": 0.0,
@@ -25,589 +25,589 @@
     {
       "epoch": 0.03,
       "learning_rate": 1.282051282051282e-06,
-      "logits/chosen": -1.8667322397232056,
-      "logits/rejected": -1.8710299730300903,
-      "logps/chosen": -36.98433303833008,
-      "logps/rejected": -33.65640640258789,
-      "loss": 0.4948,
+      "logits/chosen": -1.866772174835205,
+      "logits/rejected": -1.8710780143737793,
+      "logps/chosen": -36.99943161010742,
+      "logps/rejected": -33.657470703125,
+      "loss": 0.4858,
       "rewards/accuracies": 0.5277777910232544,
-      "rewards/chosen": 0.015585740096867085,
-      "rewards/margins": 0.034019216895103455,
-      "rewards/rejected": -0.018433474004268646,
+      "rewards/chosen": 0.005016062408685684,
+      "rewards/margins": 0.02419126406311989,
+      "rewards/rejected": -0.019175197929143906,
       "step": 10
     },
     {
       "epoch": 0.05,
       "learning_rate": 2.564102564102564e-06,
-      "logits/chosen": -1.9972655773162842,
-      "logits/rejected": -1.9999065399169922,
-      "logps/chosen": -29.6416072845459,
-      "logps/rejected": -29.046981811523438,
-      "loss": 0.5013,
+      "logits/chosen": -1.9975645542144775,
+      "logits/rejected": -2.0002217292785645,
+      "logps/chosen": -29.651592254638672,
+      "logps/rejected": -29.066898345947266,
+      "loss": 0.5542,
       "rewards/accuracies": 0.5,
-      "rewards/chosen": 0.00041136163054034114,
-      "rewards/margins": -0.011113673448562622,
-      "rewards/rejected": 0.011525033973157406,
+      "rewards/chosen": -0.006579822860658169,
+      "rewards/margins": -0.004162783268839121,
+      "rewards/rejected": -0.0024170405231416225,
       "step": 20
     },
     {
       "epoch": 0.08,
       "learning_rate": 3.846153846153847e-06,
-      "logits/chosen": -1.9205191135406494,
-      "logits/rejected": -1.9178133010864258,
-      "logps/chosen": -31.385412216186523,
-      "logps/rejected": -33.23365020751953,
-      "loss": 0.4942,
-      "rewards/accuracies": 0.5375000238418579,
-      "rewards/chosen": 0.02147640287876129,
-      "rewards/margins": 0.03211076930165291,
-      "rewards/rejected": -0.010634368285536766,
+      "logits/chosen": -1.9212992191314697,
+      "logits/rejected": -1.918600082397461,
+      "logps/chosen": -31.42719078063965,
+      "logps/rejected": -33.23390579223633,
+      "loss": 0.5492,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -0.00776728754863143,
+      "rewards/margins": 0.0030429032631218433,
+      "rewards/rejected": -0.010810190811753273,
       "step": 30
     },
     {
       "epoch": 0.1,
       "learning_rate": 4.999896948438434e-06,
-      "logits/chosen": -2.017784833908081,
-      "logits/rejected": -2.009059429168701,
-      "logps/chosen": -32.547088623046875,
-      "logps/rejected": -32.473243713378906,
-      "loss": 0.5024,
-      "rewards/accuracies": 0.4749999940395355,
-      "rewards/chosen": 0.020800117403268814,
-      "rewards/margins": -0.006797379348427057,
-      "rewards/rejected": 0.027597496286034584,
+      "logits/chosen": -2.0178468227386475,
+      "logits/rejected": -2.009111166000366,
+      "logps/chosen": -32.579471588134766,
+      "logps/rejected": -32.50147247314453,
+      "loss": 0.5878,
+      "rewards/accuracies": 0.44999998807907104,
+      "rewards/chosen": -0.001865685684606433,
+      "rewards/margins": -0.009705344215035439,
+      "rewards/rejected": 0.007839656434953213,
       "step": 40
     },
     {
       "epoch": 0.13,
       "learning_rate": 4.987541037542187e-06,
-      "logits/chosen": -1.863996148109436,
-      "logits/rejected": -1.8532311916351318,
-      "logps/chosen": -33.490657806396484,
-      "logps/rejected": -35.415367126464844,
-      "loss": 0.4957,
-      "rewards/accuracies": 0.550000011920929,
-      "rewards/chosen": 0.046776313334703445,
-      "rewards/margins": 0.02089320495724678,
-      "rewards/rejected": 0.025883108377456665,
+      "logits/chosen": -1.8617979288101196,
+      "logits/rejected": -1.8510282039642334,
+      "logps/chosen": -33.562862396240234,
+      "logps/rejected": -35.4472770690918,
+      "loss": 0.5775,
+      "rewards/accuracies": 0.42500001192092896,
+      "rewards/chosen": -0.0037666684947907925,
+      "rewards/margins": -0.007314275950193405,
+      "rewards/rejected": 0.0035476074554026127,
       "step": 50
     },
     {
       "epoch": 0.16,
       "learning_rate": 4.954691471941119e-06,
-      "logits/chosen": -1.9439821243286133,
-      "logits/rejected": -1.9459091424942017,
-      "logps/chosen": -32.50756072998047,
-      "logps/rejected": -33.15720748901367,
-      "loss": 0.489,
-      "rewards/accuracies": 0.550000011920929,
-      "rewards/chosen": 0.06483479589223862,
-      "rewards/margins": 0.05491489917039871,
-      "rewards/rejected": 0.009919902309775352,
+      "logits/chosen": -1.9399009943008423,
+      "logits/rejected": -1.9418485164642334,
+      "logps/chosen": -32.5726203918457,
+      "logps/rejected": -33.18496322631836,
+      "loss": 0.5967,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": 0.019294817000627518,
+      "rewards/margins": 0.028803948312997818,
+      "rewards/rejected": -0.009509134106338024,
       "step": 60
     },
     {
       "epoch": 0.18,
       "learning_rate": 4.901618883413549e-06,
-      "logits/chosen": -2.075462579727173,
-      "logits/rejected": -2.0804481506347656,
-      "logps/chosen": -33.86588668823242,
-      "logps/rejected": -36.501365661621094,
-      "loss": 0.491,
-      "rewards/accuracies": 0.5249999761581421,
-      "rewards/chosen": 0.08581922948360443,
-      "rewards/margins": 0.038563162088394165,
-      "rewards/rejected": 0.04725607484579086,
+      "logits/chosen": -2.0729470252990723,
+      "logits/rejected": -2.077918529510498,
+      "logps/chosen": -33.93268585205078,
+      "logps/rejected": -36.637229919433594,
+      "loss": 0.5297,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": 0.039063483476638794,
+      "rewards/margins": 0.08691467344760895,
+      "rewards/rejected": -0.04785118252038956,
       "step": 70
     },
     {
       "epoch": 0.21,
       "learning_rate": 4.828760511501322e-06,
-      "logits/chosen": -1.9374949932098389,
-      "logits/rejected": -1.9405946731567383,
-      "logps/chosen": -34.200557708740234,
-      "logps/rejected": -34.543209075927734,
-      "loss": 0.4719,
-      "rewards/accuracies": 0.574999988079071,
-      "rewards/chosen": 0.15907494723796844,
-      "rewards/margins": 0.1246359720826149,
-      "rewards/rejected": 0.03443896025419235,
+      "logits/chosen": -1.9366486072540283,
+      "logits/rejected": -1.939780592918396,
+      "logps/chosen": -34.30228805541992,
+      "logps/rejected": -34.652069091796875,
+      "loss": 0.4209,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": 0.08786438405513763,
+      "rewards/margins": 0.12962770462036133,
+      "rewards/rejected": -0.041763320565223694,
       "step": 80
     },
     {
       "epoch": 0.23,
       "learning_rate": 4.7367166013034295e-06,
-      "logits/chosen": -1.9453551769256592,
-      "logits/rejected": -1.9498746395111084,
-      "logps/chosen": -32.2825927734375,
-      "logps/rejected": -32.25676727294922,
-      "loss": 0.4858,
-      "rewards/accuracies": 0.637499988079071,
-      "rewards/chosen": 0.12937995791435242,
-      "rewards/margins": 0.05788033455610275,
-      "rewards/rejected": 0.07149962335824966,
+      "logits/chosen": -1.946979284286499,
+      "logits/rejected": -1.951521635055542,
+      "logps/chosen": -32.4210205078125,
+      "logps/rejected": -32.337799072265625,
+      "loss": 0.6191,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": 0.03248247504234314,
+      "rewards/margins": 0.017702888697385788,
+      "rewards/rejected": 0.014779585413634777,
       "step": 90
     },
     {
       "epoch": 0.26,
       "learning_rate": 4.626245458345211e-06,
-      "logits/chosen": -2.04329776763916,
-      "logits/rejected": -2.0412919521331787,
-      "logps/chosen": -32.015525817871094,
-      "logps/rejected": -31.169103622436523,
-      "loss": 0.4799,
-      "rewards/accuracies": 0.637499988079071,
-      "rewards/chosen": 0.15771625936031342,
-      "rewards/margins": 0.08974156528711319,
-      "rewards/rejected": 0.06797470152378082,
+      "logits/chosen": -2.045017719268799,
+      "logits/rejected": -2.043015241622925,
+      "logps/chosen": -32.16361618041992,
+      "logps/rejected": -31.273128509521484,
+      "loss": 0.4711,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": 0.05405203625559807,
+      "rewards/margins": 0.05889582633972168,
+      "rewards/rejected": -0.004843792412430048,
       "step": 100
     },
     {
       "epoch": 0.26,
-      "eval_logits/chosen": -2.238145351409912,
-      "eval_logits/rejected": -2.2333030700683594,
-      "eval_logps/chosen": -33.86069107055664,
-      "eval_logps/rejected": -37.356990814208984,
-      "eval_loss": 0.49784979224205017,
-      "eval_rewards/accuracies": 0.5340532064437866,
-      "eval_rewards/chosen": 0.12170296907424927,
-      "eval_rewards/margins": 0.009964452125132084,
-      "eval_rewards/rejected": 0.11173851788043976,
-      "eval_runtime": 145.9039,
-      "eval_samples_per_second": 2.351,
+      "eval_logits/chosen": -2.240097761154175,
+      "eval_logits/rejected": -2.2352383136749268,
+      "eval_logps/chosen": -34.01129150390625,
+      "eval_logps/rejected": -37.49793243408203,
+      "eval_loss": 0.5754798054695129,
+      "eval_rewards/accuracies": 0.5195183157920837,
+      "eval_rewards/chosen": 0.016279777511954308,
+      "eval_rewards/margins": 0.0031977586913853884,
+      "eval_rewards/rejected": 0.013082021847367287,
+      "eval_runtime": 145.9275,
+      "eval_samples_per_second": 2.35,
       "eval_steps_per_second": 0.295,
       "step": 100
     },
     {
       "epoch": 0.29,
       "learning_rate": 4.498257201263691e-06,
-      "logits/chosen": -1.9996016025543213,
-      "logits/rejected": -1.9972642660140991,
-      "logps/chosen": -32.95330810546875,
-      "logps/rejected": -33.88172912597656,
-      "loss": 0.4742,
-      "rewards/accuracies": 0.612500011920929,
-      "rewards/chosen": 0.20335713028907776,
-      "rewards/margins": 0.10000836849212646,
-      "rewards/rejected": 0.10334876924753189,
+      "logits/chosen": -2.0017762184143066,
+      "logits/rejected": -1.9993689060211182,
+      "logps/chosen": -33.13564682006836,
+      "logps/rejected": -33.997642517089844,
+      "loss": 0.6759,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.07571890205144882,
+      "rewards/margins": 0.05350743606686592,
+      "rewards/rejected": 0.022211460396647453,
       "step": 110
     },
     {
       "epoch": 0.31,
       "learning_rate": 4.353806263777678e-06,
-      "logits/chosen": -2.0096044540405273,
-      "logits/rejected": -2.0012660026550293,
-      "logps/chosen": -32.153419494628906,
-      "logps/rejected": -31.961782455444336,
-      "loss": 0.4848,
-      "rewards/accuracies": 0.574999988079071,
-      "rewards/chosen": 0.20403961837291718,
-      "rewards/margins": 0.06423385441303253,
-      "rewards/rejected": 0.13980577886104584,
+      "logits/chosen": -2.013158082962036,
+      "logits/rejected": -2.004814624786377,
+      "logps/chosen": -32.32666015625,
+      "logps/rejected": -32.12488555908203,
+      "loss": 0.5572,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.08277235180139542,
+      "rewards/margins": 0.05714136362075806,
+      "rewards/rejected": 0.02563098631799221,
       "step": 120
     },
     {
       "epoch": 0.34,
       "learning_rate": 4.1940827077152755e-06,
-      "logits/chosen": -2.038412570953369,
-      "logits/rejected": -2.0304617881774902,
-      "logps/chosen": -30.165945053100586,
-      "logps/rejected": -31.880346298217773,
-      "loss": 0.4787,
-      "rewards/accuracies": 0.5625,
-      "rewards/chosen": 0.21683910489082336,
-      "rewards/margins": 0.0959777683019638,
-      "rewards/rejected": 0.12086131423711777,
+      "logits/chosen": -2.039836883544922,
+      "logits/rejected": -2.031825542449951,
+      "logps/chosen": -30.349987030029297,
+      "logps/rejected": -32.029136657714844,
+      "loss": 0.5109,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.0880090743303299,
+      "rewards/margins": 0.07130132615566254,
+      "rewards/rejected": 0.016707751899957657,
       "step": 130
     },
     {
       "epoch": 0.36,
       "learning_rate": 4.0204024186666215e-06,
-      "logits/chosen": -1.9683482646942139,
-      "logits/rejected": -1.9784959554672241,
-      "logps/chosen": -31.041671752929688,
-      "logps/rejected": -32.388893127441406,
-      "loss": 0.4621,
-      "rewards/accuracies": 0.6625000238418579,
-      "rewards/chosen": 0.264592707157135,
-      "rewards/margins": 0.16240297257900238,
-      "rewards/rejected": 0.10218973457813263,
+      "logits/chosen": -1.9694139957427979,
+      "logits/rejected": -1.9796711206436157,
+      "logps/chosen": -31.24441909790039,
+      "logps/rejected": -32.56440353393555,
+      "loss": 0.5037,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.12266886234283447,
+      "rewards/margins": 0.14333835244178772,
+      "rewards/rejected": -0.02066950313746929,
       "step": 140
     },
     {
       "epoch": 0.39,
       "learning_rate": 3.834196265035119e-06,
-      "logits/chosen": -1.8799880743026733,
-      "logits/rejected": -1.8811382055282593,
-      "logps/chosen": -33.686256408691406,
-      "logps/rejected": -34.564491271972656,
-      "loss": 0.4503,
-      "rewards/accuracies": 0.6000000238418579,
-      "rewards/chosen": 0.3545047640800476,
-      "rewards/margins": 0.21957096457481384,
-      "rewards/rejected": 0.134933739900589,
+      "logits/chosen": -1.8828538656234741,
+      "logits/rejected": -1.8840004205703735,
+      "logps/chosen": -34.006561279296875,
+      "logps/rejected": -34.81753921508789,
+      "loss": 0.4564,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": 0.13028928637504578,
+      "rewards/margins": 0.17248663306236267,
+      "rewards/rejected": -0.04219735041260719,
       "step": 150
     },
     {
       "epoch": 0.42,
       "learning_rate": 3.636998309800573e-06,
-      "logits/chosen": -1.931627869606018,
-      "logits/rejected": -1.9282808303833008,
-      "logps/chosen": -35.77287292480469,
-      "logps/rejected": -32.49742889404297,
-      "loss": 0.4707,
-      "rewards/accuracies": 0.637499988079071,
-      "rewards/chosen": 0.27916350960731506,
-      "rewards/margins": 0.12143567949533463,
-      "rewards/rejected": 0.15772780776023865,
+      "logits/chosen": -1.9349613189697266,
+      "logits/rejected": -1.9315464496612549,
+      "logps/chosen": -36.059181213378906,
+      "logps/rejected": -32.72267150878906,
+      "loss": 0.487,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": 0.07874591648578644,
+      "rewards/margins": 0.07869003713130951,
+      "rewards/rejected": 5.587190389633179e-05,
       "step": 160
     },
     {
       "epoch": 0.44,
       "learning_rate": 3.4304331721118078e-06,
-      "logits/chosen": -2.0332443714141846,
-      "logits/rejected": -2.0259475708007812,
-      "logps/chosen": -33.251670837402344,
-      "logps/rejected": -31.17826271057129,
-      "loss": 0.439,
-      "rewards/accuracies": 0.675000011920929,
-      "rewards/chosen": 0.382335364818573,
-      "rewards/margins": 0.2608257234096527,
-      "rewards/rejected": 0.1215096265077591,
+      "logits/chosen": -2.036032199859619,
+      "logits/rejected": -2.028651714324951,
+      "logps/chosen": -33.573509216308594,
+      "logps/rejected": -31.36956214904785,
+      "loss": 0.4465,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.15704451501369476,
+      "rewards/margins": 0.16944527626037598,
+      "rewards/rejected": -0.012400749139487743,
       "step": 170
     },
     {
       "epoch": 0.47,
       "learning_rate": 3.2162026428305436e-06,
-      "logits/chosen": -2.040196657180786,
-      "logits/rejected": -2.045389413833618,
-      "logps/chosen": -31.95781898498535,
-      "logps/rejected": -32.18687057495117,
-      "loss": 0.4516,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": 0.4108788073062897,
-      "rewards/margins": 0.20208871364593506,
-      "rewards/rejected": 0.20879006385803223,
+      "logits/chosen": -2.042617082595825,
+      "logits/rejected": -2.0478556156158447,
+      "logps/chosen": -32.33074188232422,
+      "logps/rejected": -32.472496032714844,
+      "loss": 0.3727,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": 0.14983226358890533,
+      "rewards/margins": 0.14098060131072998,
+      "rewards/rejected": 0.008851657621562481,
       "step": 180
     },
     {
       "epoch": 0.49,
       "learning_rate": 2.996071664294641e-06,
-      "logits/chosen": -2.040775775909424,
-      "logits/rejected": -2.038043737411499,
-      "logps/chosen": -31.056228637695312,
-      "logps/rejected": -31.10147476196289,
-      "loss": 0.4647,
-      "rewards/accuracies": 0.675000011920929,
-      "rewards/chosen": 0.30290743708610535,
-      "rewards/margins": 0.1511872559785843,
-      "rewards/rejected": 0.15172019600868225,
+      "logits/chosen": -2.0445618629455566,
+      "logits/rejected": -2.0417864322662354,
+      "logps/chosen": -31.317855834960938,
+      "logps/rejected": -31.348400115966797,
+      "loss": 0.4609,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": 0.11976919323205948,
+      "rewards/margins": 0.14090058207511902,
+      "rewards/rejected": -0.021131375804543495,
       "step": 190
     },
     {
       "epoch": 0.52,
       "learning_rate": 2.7718537898066833e-06,
-      "logits/chosen": -1.9112863540649414,
-      "logits/rejected": -1.9159523248672485,
-      "logps/chosen": -31.08163070678711,
-      "logps/rejected": -32.598487854003906,
-      "loss": 0.4453,
-      "rewards/accuracies": 0.6625000238418579,
-      "rewards/chosen": 0.3672560155391693,
-      "rewards/margins": 0.2325470894575119,
-      "rewards/rejected": 0.1347089260816574,
+      "logits/chosen": -1.9139589071273804,
+      "logits/rejected": -1.9186115264892578,
+      "logps/chosen": -31.420101165771484,
+      "logps/rejected": -32.79216766357422,
+      "loss": 0.5061,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.13032536208629608,
+      "rewards/margins": 0.13119173049926758,
+      "rewards/rejected": -0.0008663811022415757,
       "step": 200
     },
     {
       "epoch": 0.52,
-      "eval_logits/chosen": -2.234708309173584,
-      "eval_logits/rejected": -2.229902505874634,
-      "eval_logps/chosen": -33.76850891113281,
-      "eval_logps/rejected": -37.29365921020508,
-      "eval_loss": 0.49275800585746765,
-      "eval_rewards/accuracies": 0.5369601845741272,
-      "eval_rewards/chosen": 0.18622951209545135,
-      "eval_rewards/margins": 0.030155813321471214,
-      "eval_rewards/rejected": 0.1560736745595932,
-      "eval_runtime": 145.5287,
-      "eval_samples_per_second": 2.357,
+      "eval_logits/chosen": -2.2385354042053223,
+      "eval_logits/rejected": -2.2336885929107666,
+      "eval_logps/chosen": -34.05002975463867,
+      "eval_logps/rejected": -37.54545974731445,
+      "eval_loss": 0.5877167582511902,
+      "eval_rewards/accuracies": 0.4991694688796997,
+      "eval_rewards/chosen": -0.010832725092768669,
+      "eval_rewards/margins": 0.009353035129606724,
+      "eval_rewards/rejected": -0.020185761153697968,
+      "eval_runtime": 145.6958,
+      "eval_samples_per_second": 2.354,
       "eval_steps_per_second": 0.295,
       "step": 200
     },
     {
       "epoch": 0.55,
       "learning_rate": 2.5453962426402006e-06,
-      "logits/chosen": -2.022939443588257,
-      "logits/rejected": -2.033546209335327,
-      "logps/chosen": -31.5457706451416,
-      "logps/rejected": -33.67595672607422,
-      "loss": 0.4628,
-      "rewards/accuracies": 0.574999988079071,
-      "rewards/chosen": 0.2978750169277191,
-      "rewards/margins": 0.16715146601200104,
-      "rewards/rejected": 0.13072356581687927,
+      "logits/chosen": -2.026780366897583,
+      "logits/rejected": -2.0374534130096436,
+      "logps/chosen": -31.786846160888672,
+      "logps/rejected": -33.922447204589844,
+      "loss": 0.4225,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.12911871075630188,
+      "rewards/margins": 0.17094141244888306,
+      "rewards/rejected": -0.041822709143161774,
       "step": 210
     },
     {
       "epoch": 0.57,
       "learning_rate": 2.3185646976551794e-06,
-      "logits/chosen": -1.9158331155776978,
-      "logits/rejected": -1.9305419921875,
-      "logps/chosen": -29.57406997680664,
-      "logps/rejected": -31.42013168334961,
-      "loss": 0.4363,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": 0.37622833251953125,
-      "rewards/margins": 0.2795966863632202,
-      "rewards/rejected": 0.09663165360689163,
+      "logits/chosen": -1.9182548522949219,
+      "logits/rejected": -1.933038353919983,
+      "logps/chosen": -29.936147689819336,
+      "logps/rejected": -31.559993743896484,
+      "loss": 0.4429,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": 0.12277615070343018,
+      "rewards/margins": 0.12404797971248627,
+      "rewards/rejected": -0.0012718416983261704,
       "step": 220
     },
     {
       "epoch": 0.6,
       "learning_rate": 2.0932279108998323e-06,
-      "logits/chosen": -1.9733517169952393,
-      "logits/rejected": -1.9773433208465576,
-      "logps/chosen": -32.83649444580078,
-      "logps/rejected": -31.40326499938965,
-      "loss": 0.4311,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": 0.40668267011642456,
-      "rewards/margins": 0.304456502199173,
-      "rewards/rejected": 0.10222617536783218,
+      "logits/chosen": -1.9737575054168701,
+      "logits/rejected": -1.9777438640594482,
+      "logps/chosen": -33.20977783203125,
+      "logps/rejected": -31.5734920501709,
+      "loss": 0.3768,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": 0.14538228511810303,
+      "rewards/margins": 0.16231416165828705,
+      "rewards/rejected": -0.016931891441345215,
       "step": 230
     },
     {
       "epoch": 0.62,
       "learning_rate": 1.8712423238279358e-06,
-      "logits/chosen": -1.9713729619979858,
-      "logits/rejected": -1.9496545791625977,
-      "logps/chosen": -33.5602912902832,
-      "logps/rejected": -34.85026550292969,
-      "loss": 0.4227,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": 0.41541916131973267,
-      "rewards/margins": 0.3450462222099304,
-      "rewards/rejected": 0.07037289440631866,
+      "logits/chosen": -1.9738047122955322,
+      "logits/rejected": -1.9518985748291016,
+      "logps/chosen": -33.94301223754883,
+      "logps/rejected": -35.023075103759766,
+      "loss": 0.4304,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": 0.1475195735692978,
+      "rewards/margins": 0.19811663031578064,
+      "rewards/rejected": -0.05059707164764404,
       "step": 240
     },
     {
       "epoch": 0.65,
       "learning_rate": 1.6544367689701824e-06,
-      "logits/chosen": -2.012058734893799,
-      "logits/rejected": -2.0087602138519287,
-      "logps/chosen": -32.46921157836914,
-      "logps/rejected": -36.012245178222656,
-      "loss": 0.4613,
+      "logits/chosen": -2.0155911445617676,
+      "logits/rejected": -2.0122878551483154,
+      "logps/chosen": -32.76276397705078,
+      "logps/rejected": -36.25225830078125,
+      "loss": 0.4155,
       "rewards/accuracies": 0.637499988079071,
-      "rewards/chosen": 0.31910061836242676,
-      "rewards/margins": 0.17022350430488586,
-      "rewards/rejected": 0.1488770991563797,
+      "rewards/chosen": 0.11361332982778549,
+      "rewards/margins": 0.1327473670244217,
+      "rewards/rejected": -0.019134048372507095,
       "step": 250
     },
     {
       "epoch": 0.68,
       "learning_rate": 1.4445974030621963e-06,
-      "logits/chosen": -1.878584861755371,
-      "logits/rejected": -1.876153588294983,
-      "logps/chosen": -33.71690368652344,
-      "logps/rejected": -35.26210403442383,
-      "loss": 0.4631,
-      "rewards/accuracies": 0.6499999761581421,
-      "rewards/chosen": 0.33004888892173767,
-      "rewards/margins": 0.1602855622768402,
-      "rewards/rejected": 0.16976332664489746,
+      "logits/chosen": -1.8830368518829346,
+      "logits/rejected": -1.880629539489746,
+      "logps/chosen": -34.01710891723633,
+      "logps/rejected": -35.53314971923828,
+      "loss": 0.4219,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": 0.11990444362163544,
+      "rewards/margins": 0.1398761123418808,
+      "rewards/rejected": -0.019971664994955063,
       "step": 260
     },
     {
       "epoch": 0.7,
       "learning_rate": 1.243452991757889e-06,
-      "logits/chosen": -1.863705039024353,
-      "logits/rejected": -1.8611875772476196,
-      "logps/chosen": -33.94118881225586,
-      "logps/rejected": -31.579448699951172,
-      "loss": 0.4582,
-      "rewards/accuracies": 0.612500011920929,
-      "rewards/chosen": 0.3194878101348877,
-      "rewards/margins": 0.18517015874385834,
-      "rewards/rejected": 0.13431766629219055,
+      "logits/chosen": -1.8685131072998047,
+      "logits/rejected": -1.8660367727279663,
+      "logps/chosen": -34.2363395690918,
+      "logps/rejected": -31.768321990966797,
+      "loss": 0.4918,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.11288031190633774,
+      "rewards/margins": 0.11077453196048737,
+      "rewards/rejected": 0.0021057710982859135,
       "step": 270
     },
     {
       "epoch": 0.73,
       "learning_rate": 1.0526606671603523e-06,
-      "logits/chosen": -1.96773362159729,
-      "logits/rejected": -1.957313895225525,
-      "logps/chosen": -34.72475814819336,
-      "logps/rejected": -31.634963989257812,
-      "loss": 0.4297,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": 0.4423200190067291,
-      "rewards/margins": 0.298100084066391,
-      "rewards/rejected": 0.14421990513801575,
+      "logits/chosen": -1.9722087383270264,
+      "logits/rejected": -1.961612343788147,
+      "logps/chosen": -35.04926300048828,
+      "logps/rejected": -31.826486587524414,
+      "loss": 0.3417,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": 0.21516695618629456,
+      "rewards/margins": 0.20501188933849335,
+      "rewards/rejected": 0.010155050083994865,
       "step": 280
     },
     {
       "epoch": 0.75,
       "learning_rate": 8.737922755071455e-07,
-      "logits/chosen": -2.0632526874542236,
-      "logits/rejected": -2.0484511852264404,
-      "logps/chosen": -30.412399291992188,
-      "logps/rejected": -32.339237213134766,
-      "loss": 0.4682,
-      "rewards/accuracies": 0.550000011920929,
-      "rewards/chosen": 0.3582096993923187,
-      "rewards/margins": 0.13939058780670166,
-      "rewards/rejected": 0.21881911158561707,
+      "logits/chosen": -2.068380832672119,
+      "logits/rejected": -2.05340313911438,
+      "logps/chosen": -30.748706817626953,
+      "logps/rejected": -32.63469314575195,
+      "loss": 0.4741,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.12279413640499115,
+      "rewards/margins": 0.11079368740320206,
+      "rewards/rejected": 0.012000440619885921,
       "step": 290
     },
     {
       "epoch": 0.78,
       "learning_rate": 7.08321427484816e-07,
-      "logits/chosen": -1.9345197677612305,
-      "logits/rejected": -1.9320701360702515,
-      "logps/chosen": -32.11776351928711,
-      "logps/rejected": -30.692516326904297,
-      "loss": 0.3947,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": 0.5665432810783386,
-      "rewards/margins": 0.4795381426811218,
-      "rewards/rejected": 0.08700509369373322,
+      "logits/chosen": -1.9397351741790771,
+      "logits/rejected": -1.9371974468231201,
+      "logps/chosen": -32.603660583496094,
+      "logps/rejected": -30.863941192626953,
+      "loss": 0.3371,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": 0.2264125794172287,
+      "rewards/margins": 0.25940603017807007,
+      "rewards/rejected": -0.03299345448613167,
       "step": 300
     },
     {
       "epoch": 0.78,
-      "eval_logits/chosen": -2.2321932315826416,
-      "eval_logits/rejected": -2.227381944656372,
-      "eval_logps/chosen": -33.75507736206055,
-      "eval_logps/rejected": -37.28935623168945,
-      "eval_loss": 0.49099090695381165,
-      "eval_rewards/accuracies": 0.5564784407615662,
-      "eval_rewards/chosen": 0.1956319361925125,
-      "eval_rewards/margins": 0.03654861077666283,
-      "eval_rewards/rejected": 0.15908333659172058,
-      "eval_runtime": 145.678,
-      "eval_samples_per_second": 2.355,
-      "eval_steps_per_second": 0.295,
+      "eval_logits/chosen": -2.2370686531066895,
+      "eval_logits/rejected": -2.2322022914886475,
+      "eval_logps/chosen": -34.03439712524414,
+      "eval_logps/rejected": -37.53529357910156,
+      "eval_loss": 0.5842657089233398,
+      "eval_rewards/accuracies": 0.5278239250183105,
+      "eval_rewards/chosen": 0.00010742468293756247,
+      "eval_rewards/margins": 0.013178782537579536,
+      "eval_rewards/rejected": -0.013071359135210514,
+      "eval_runtime": 145.5131,
+      "eval_samples_per_second": 2.357,
+      "eval_steps_per_second": 0.296,
       "step": 300
     },
     {
       "epoch": 0.81,
       "learning_rate": 5.576113578589035e-07,
-      "logits/chosen": -1.918630838394165,
-      "logits/rejected": -1.9154895544052124,
-      "logps/chosen": -31.033864974975586,
-      "logps/rejected": -33.57470703125,
-      "loss": 0.4357,
-      "rewards/accuracies": 0.7749999761581421,
-      "rewards/chosen": 0.3986373543739319,
-      "rewards/margins": 0.28142648935317993,
-      "rewards/rejected": 0.11721093952655792,
+      "logits/chosen": -1.9219080209732056,
+      "logits/rejected": -1.9186557531356812,
+      "logps/chosen": -31.35263442993164,
+      "logps/rejected": -33.746925354003906,
+      "loss": 0.4182,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.17550265789031982,
+      "rewards/margins": 0.17884370684623718,
+      "rewards/rejected": -0.0033410280011594296,
       "step": 310
     },
     {
       "epoch": 0.83,
       "learning_rate": 4.229036944380913e-07,
-      "logits/chosen": -1.9683278799057007,
-      "logits/rejected": -1.9561859369277954,
-      "logps/chosen": -34.02568817138672,
-      "logps/rejected": -33.45148849487305,
-      "loss": 0.4237,
-      "rewards/accuracies": 0.7749999761581421,
-      "rewards/chosen": 0.39926695823669434,
-      "rewards/margins": 0.3381231129169464,
-      "rewards/rejected": 0.06114383786916733,
+      "logits/chosen": -1.9732515811920166,
+      "logits/rejected": -1.9609721899032593,
+      "logps/chosen": -34.435001373291016,
+      "logps/rejected": -33.593482971191406,
+      "loss": 0.3778,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.11274988949298859,
+      "rewards/margins": 0.15099774301052094,
+      "rewards/rejected": -0.03824785351753235,
       "step": 320
     },
     {
       "epoch": 0.86,
       "learning_rate": 3.053082288996112e-07,
-      "logits/chosen": -2.0041966438293457,
-      "logits/rejected": -2.0028293132781982,
-      "logps/chosen": -32.882896423339844,
-      "logps/rejected": -32.24889373779297,
-      "loss": 0.4411,
-      "rewards/accuracies": 0.637499988079071,
-      "rewards/chosen": 0.4220260679721832,
-      "rewards/margins": 0.26054397225379944,
-      "rewards/rejected": 0.16148212552070618,
+      "logits/chosen": -2.0085272789001465,
+      "logits/rejected": -2.0070912837982178,
+      "logps/chosen": -33.31962585449219,
+      "logps/rejected": -32.51466751098633,
+      "loss": 0.4061,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.1163158044219017,
+      "rewards/margins": 0.14087431132793427,
+      "rewards/rejected": -0.024558518081903458,
       "step": 330
     },
     {
       "epoch": 0.88,
       "learning_rate": 2.0579377374915805e-07,
-      "logits/chosen": -2.0908923149108887,
-      "logits/rejected": -2.0752511024475098,
-      "logps/chosen": -33.49097442626953,
-      "logps/rejected": -32.81463623046875,
-      "loss": 0.4381,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": 0.4818909764289856,
-      "rewards/margins": 0.26951199769973755,
-      "rewards/rejected": 0.21237893402576447,
+      "logits/chosen": -2.0951385498046875,
+      "logits/rejected": -2.0793802738189697,
+      "logps/chosen": -33.88261032104492,
+      "logps/rejected": -33.10671615600586,
+      "loss": 0.4042,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": 0.20774619281291962,
+      "rewards/margins": 0.19982238113880157,
+      "rewards/rejected": 0.00792380329221487,
       "step": 340
     },
     {
       "epoch": 0.91,
       "learning_rate": 1.2518018074041684e-07,
-      "logits/chosen": -1.9633783102035522,
-      "logits/rejected": -1.9625694751739502,
-      "logps/chosen": -32.55985641479492,
-      "logps/rejected": -32.24340057373047,
-      "loss": 0.4211,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": 0.5055657029151917,
-      "rewards/margins": 0.3503406047821045,
-      "rewards/rejected": 0.15522508323192596,
+      "logits/chosen": -1.9677631855010986,
+      "logits/rejected": -1.9668546915054321,
+      "logps/chosen": -32.957977294921875,
+      "logps/rejected": -32.483890533447266,
+      "loss": 0.4874,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.2268829345703125,
+      "rewards/margins": 0.2400023490190506,
+      "rewards/rejected": -0.013119394890964031,
       "step": 350
     },
     {
       "epoch": 0.94,
       "learning_rate": 6.41315865106129e-08,
-      "logits/chosen": -1.9187549352645874,
-      "logits/rejected": -1.9290361404418945,
-      "logps/chosen": -31.61210060119629,
-      "logps/rejected": -35.0199089050293,
-      "loss": 0.4489,
-      "rewards/accuracies": 0.637499988079071,
-      "rewards/chosen": 0.41481319069862366,
-      "rewards/margins": 0.22661848366260529,
-      "rewards/rejected": 0.18819470703601837,
+      "logits/chosen": -1.9235498905181885,
+      "logits/rejected": -1.933876395225525,
+      "logps/chosen": -32.016578674316406,
+      "logps/rejected": -35.32080841064453,
+      "loss": 0.4297,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.13167943060398102,
+      "rewards/margins": 0.15411263704299927,
+      "rewards/rejected": -0.02243318222463131,
       "step": 360
     },
     {
       "epoch": 0.96,
       "learning_rate": 2.3150941078050325e-08,
-      "logits/chosen": -2.0583438873291016,
-      "logits/rejected": -2.051846742630005,
-      "logps/chosen": -33.05150604248047,
-      "logps/rejected": -28.9710750579834,
-      "loss": 0.4429,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": 0.42032259702682495,
-      "rewards/margins": 0.2415778934955597,
-      "rewards/rejected": 0.17874471843242645,
+      "logits/chosen": -2.062828540802002,
+      "logits/rejected": -2.056312322616577,
+      "logps/chosen": -33.43348693847656,
+      "logps/rejected": -29.213970184326172,
+      "loss": 0.3963,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.15293903648853302,
+      "rewards/margins": 0.14422088861465454,
+      "rewards/rejected": 0.008718125522136688,
       "step": 370
     },
     {
       "epoch": 0.99,
       "learning_rate": 2.575864278703266e-09,
-      "logits/chosen": -1.9180545806884766,
-      "logits/rejected": -1.92022705078125,
-      "logps/chosen": -33.62676239013672,
-      "logps/rejected": -30.719894409179688,
-      "loss": 0.4281,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": 0.4286006987094879,
-      "rewards/margins": 0.3216874301433563,
-      "rewards/rejected": 0.10691330581903458,
+      "logits/chosen": -1.9224169254302979,
+      "logits/rejected": -1.9245796203613281,
+      "logps/chosen": -33.9437255859375,
+      "logps/rejected": -30.860538482666016,
+      "loss": 0.4119,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": 0.20672598481178284,
+      "rewards/margins": 0.19826285541057587,
+      "rewards/rejected": 0.008463130332529545,
       "step": 380
     },
     {
       "epoch": 1.0,
       "step": 385,
       "total_flos": 0.0,
-      "train_loss": 0.45851393303313814,
-      "train_runtime": 3252.0771,
+      "train_loss": 0.46899499707407766,
+      "train_runtime": 3249.86,
       "train_samples_per_second": 0.947,
       "train_steps_per_second": 0.118
     }