Model save

README.md

@@ -1,13 +1,10 @@
 ---
 library_name: peft
 tags:
-- alignment-handbook
 - trl
 - dpo
 - generated_from_trainer
 base_model: NbAiLab/nb-gpt-j-6B-v2
-datasets:
-- hugodk-sch/aftonposten_title_prefs
 model-index:
 - name: aftonposten-6b-align-scan
   results: []
@@ -18,17 +15,17 @@ should probably proofread and complete it, then remove this comment. -->
 
 # aftonposten-6b-align-scan
 
-This model is a fine-tuned version of [data/ap-gpt-j-6b-sft-qlora-04-08](https://huggingface.co/data/ap-gpt-j-6b-sft-qlora-04-08) on the hugodk-sch/aftonposten_title_prefs dataset.
+This model is a fine-tuned version of [NbAiLab/nb-gpt-j-6B-v2](https://huggingface.co/NbAiLab/nb-gpt-j-6B-v2) on an unknown dataset.
 It achieves the following results on the evaluation set:
-- Loss: 0.6926
-- Rewards/chosen: -0.0279
-- Rewards/rejected: -0.0401
-- Rewards/accuracies: 0.5212
-- Rewards/margins: 0.0122
-- Logps/rejected: -37.5969
-- Logps/chosen: -34.0903
-- Logits/rejected: -2.2228
-- Logits/chosen: -2.2276
+- Loss: 0.4944
+- Rewards/chosen: 0.1419
+- Rewards/rejected: 0.1189
+- Rewards/accuracies: 0.5307
+- Rewards/margins: 0.0230
+- Logps/rejected: -37.2789
+- Logps/chosen: -33.7508
+- Logits/rejected: -2.2285
+- Logits/chosen: -2.2333
 
 ## Model description
 
@@ -63,9 +60,9 @@ The following hyperparameters were used during training:
 
 | Training Loss | Epoch | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
 |:-------------:|:-----:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
-| 0.6634        | 0.26  | 100  | 0.6931          | 0.0028         | -0.0041          | 0.5216             | 0.0069          | -37.5249       | -34.0290     | -2.2276         | -2.2324       |
-| 0.6329        | 0.52  | 200  | 0.6905          | -0.0127        | -0.0275          | 0.5274             | 0.0148          | -37.5716       | -34.0600     | -2.2255         | -2.2304       |
-| 0.5742        | 0.78  | 300  | 0.6920          | -0.0273        | -0.0406          | 0.5278             | 0.0133          | -37.5978       | -34.0891     | -2.2231         | -2.2279       |
+| 0.4794        | 0.26  | 100  | 0.4971          | 0.0824         | 0.0699           | 0.5573             | 0.0124          | -37.3767       | -33.8699     | -2.2327         | -2.2375       |
+| 0.4561        | 0.52  | 200  | 0.4959          | 0.1292         | 0.1118           | 0.5424             | 0.0174          | -37.2930       | -33.7761     | -2.2294         | -2.2342       |
+| 0.4202        | 0.78  | 300  | 0.4944          | 0.1419         | 0.1189           | 0.5307             | 0.0230          | -37.2789       | -33.7508     | -2.2285         | -2.2333       |
 
 
 ### Framework versions

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:fa79feb981d64bcd2e8b9a25f6997ab0b5ebf680edce8b9fc3c1946b2c548361
+oid sha256:238cb301e0590031e64dec1ea6ceea3ef26d10efb357a1a485256c5bc0082a4f
 size 176183216

all_results.json

@@ -1,20 +1,7 @@
 {
     "epoch": 1.0,
-    "eval_logits/chosen": -2.22763729095459,
-    "eval_logits/rejected": -2.222792387008667,
-    "eval_logps/chosen": -34.090328216552734,
-    "eval_logps/rejected": -37.59687423706055,
-    "eval_loss": 0.6925650835037231,
-    "eval_rewards/accuracies": 0.5211793780326843,
-    "eval_rewards/chosen": -0.02788795717060566,
-    "eval_rewards/margins": 0.01224011555314064,
-    "eval_rewards/rejected": -0.04012807086110115,
-    "eval_runtime": 145.6508,
-    "eval_samples": 343,
-    "eval_samples_per_second": 2.355,
-    "eval_steps_per_second": 0.295,
-    "train_loss": 0.64145151051608,
-    "train_runtime": 3249.8987,
+    "train_loss": 0.4687722819192069,
+    "train_runtime": 3250.758,
     "train_samples": 3079,
     "train_samples_per_second": 0.947,
     "train_steps_per_second": 0.118

train_results.json

@@ -1,7 +1,7 @@
 {
     "epoch": 1.0,
-    "train_loss": 0.64145151051608,
-    "train_runtime": 3249.8987,
+    "train_loss": 0.4687722819192069,
+    "train_runtime": 3250.758,
     "train_samples": 3079,
     "train_samples_per_second": 0.947,
     "train_steps_per_second": 0.118

trainer_state.json

@@ -15,7 +15,7 @@
       "logits/rejected": -1.7377450466156006,
       "logps/chosen": -29.553977966308594,
       "logps/rejected": -42.813133239746094,
-      "loss": 0.6931,
+      "loss": 0.5,
       "rewards/accuracies": 0.0,
       "rewards/chosen": 0.0,
       "rewards/margins": 0.0,
@@ -25,589 +25,589 @@
     {
       "epoch": 0.03,
       "learning_rate": 1.282051282051282e-06,
-      "logits/chosen": -1.8664319515228271,
-      "logits/rejected": -1.8707623481750488,
-      "logps/chosen": -36.98527526855469,
-      "logps/rejected": -33.654090881347656,
-      "loss": 0.6829,
-      "rewards/accuracies": 0.5416666865348816,
-      "rewards/chosen": 0.010662304237484932,
-      "rewards/margins": 0.02267039567232132,
-      "rewards/rejected": -0.012008090503513813,
+      "logits/chosen": -1.8663586378097534,
+      "logits/rejected": -1.8706679344177246,
+      "logps/chosen": -36.9964485168457,
+      "logps/rejected": -33.65947723388672,
+      "loss": 0.4966,
+      "rewards/accuracies": 0.5277777910232544,
+      "rewards/chosen": 0.005075507797300816,
+      "rewards/margins": 0.019778331741690636,
+      "rewards/rejected": -0.014702823013067245,
       "step": 10
     },
     {
       "epoch": 0.05,
       "learning_rate": 2.564102564102564e-06,
-      "logits/chosen": -1.9980642795562744,
-      "logits/rejected": -2.0007288455963135,
-      "logps/chosen": -29.634414672851562,
-      "logps/rejected": -29.0543270111084,
-      "loss": 0.6943,
-      "rewards/accuracies": 0.5874999761581421,
-      "rewards/chosen": 0.0038894296158105135,
-      "rewards/margins": -0.000667938613332808,
-      "rewards/rejected": 0.004557368345558643,
+      "logits/chosen": -1.9970680475234985,
+      "logits/rejected": -1.9997154474258423,
+      "logps/chosen": -29.64749526977539,
+      "logps/rejected": -29.048025131225586,
+      "loss": 0.5018,
+      "rewards/accuracies": 0.42500001192092896,
+      "rewards/chosen": -0.0026510744355618954,
+      "rewards/margins": -0.010360640473663807,
+      "rewards/rejected": 0.007709565572440624,
       "step": 20
     },
     {
       "epoch": 0.08,
       "learning_rate": 3.846153846153847e-06,
-      "logits/chosen": -1.9210376739501953,
-      "logits/rejected": -1.9183601140975952,
-      "logps/chosen": -31.391239166259766,
-      "logps/rejected": -33.24319076538086,
-      "loss": 0.6828,
-      "rewards/accuracies": 0.6000000238418579,
-      "rewards/chosen": 0.012426799163222313,
-      "rewards/margins": 0.024792592972517014,
-      "rewards/rejected": -0.0123657938092947,
+      "logits/chosen": -1.9203827381134033,
+      "logits/rejected": -1.9176925420761108,
+      "logps/chosen": -31.42234230041504,
+      "logps/rejected": -33.24127960205078,
+      "loss": 0.4984,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": -0.003123724367469549,
+      "rewards/margins": 0.008287688717246056,
+      "rewards/rejected": -0.011411413550376892,
       "step": 30
     },
     {
       "epoch": 0.1,
       "learning_rate": 4.999896948438434e-06,
-      "logits/chosen": -2.017646312713623,
-      "logits/rejected": -2.0089142322540283,
-      "logps/chosen": -32.557518005371094,
-      "logps/rejected": -32.51502227783203,
-      "loss": 0.6894,
-      "rewards/accuracies": 0.4749999940395355,
-      "rewards/chosen": 0.009644975885748863,
-      "rewards/margins": 0.010818523354828358,
-      "rewards/rejected": -0.0011735468870028853,
+      "logits/chosen": -2.018051862716675,
+      "logits/rejected": -2.009334087371826,
+      "logps/chosen": -32.55129623413086,
+      "logps/rejected": -32.50330352783203,
+      "loss": 0.4982,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.012754792347550392,
+      "rewards/margins": 0.008071592077612877,
+      "rewards/rejected": 0.004683199338614941,
       "step": 40
     },
     {
       "epoch": 0.13,
       "learning_rate": 4.987541037542187e-06,
-      "logits/chosen": -1.8620023727416992,
-      "logits/rejected": -1.8512481451034546,
-      "logps/chosen": -33.577735900878906,
-      "logps/rejected": -35.46040344238281,
-      "loss": 0.6982,
-      "rewards/accuracies": 0.5,
-      "rewards/chosen": -0.010127579793334007,
-      "rewards/margins": -0.0060965316370129585,
-      "rewards/rejected": -0.004031048621982336,
+      "logits/chosen": -1.8634856939315796,
+      "logits/rejected": -1.8527206182479858,
+      "logps/chosen": -33.50724411010742,
+      "logps/rejected": -35.390602111816406,
+      "loss": 0.5016,
+      "rewards/accuracies": 0.48750001192092896,
+      "rewards/chosen": 0.02511655166745186,
+      "rewards/margins": -0.005753959529101849,
+      "rewards/rejected": 0.030870508402585983,
       "step": 50
     },
     {
       "epoch": 0.16,
       "learning_rate": 4.954691471941119e-06,
-      "logits/chosen": -1.9402154684066772,
-      "logits/rejected": -1.9421701431274414,
-      "logps/chosen": -32.552555084228516,
-      "logps/rejected": -33.22978973388672,
-      "loss": 0.6723,
-      "rewards/accuracies": 0.6000000238418579,
-      "rewards/chosen": 0.023814614862203598,
-      "rewards/margins": 0.053018856793642044,
-      "rewards/rejected": -0.029204240068793297,
+      "logits/chosen": -1.9438356161117554,
+      "logits/rejected": -1.94576096534729,
+      "logps/chosen": -32.481632232666016,
+      "logps/rejected": -33.15100860595703,
+      "loss": 0.4894,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.05927763134241104,
+      "rewards/margins": 0.049092620611190796,
+      "rewards/rejected": 0.010185008868575096,
       "step": 60
     },
     {
       "epoch": 0.18,
       "learning_rate": 4.901618883413549e-06,
-      "logits/chosen": -2.071889877319336,
-      "logits/rejected": -2.0768685340881348,
-      "logps/chosen": -33.997718811035156,
-      "logps/rejected": -36.63623809814453,
-      "loss": 0.6836,
-      "rewards/accuracies": 0.5375000238418579,
-      "rewards/chosen": -0.004616844467818737,
-      "rewards/margins": 0.02906452678143978,
-      "rewards/rejected": -0.033681370317935944,
+      "logits/chosen": -2.0757923126220703,
+      "logits/rejected": -2.080766439437866,
+      "logps/chosen": -33.89708709716797,
+      "logps/rejected": -36.524818420410156,
+      "loss": 0.4939,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.04569761082530022,
+      "rewards/margins": 0.023670893162488937,
+      "rewards/rejected": 0.02202671766281128,
       "step": 70
     },
     {
       "epoch": 0.21,
       "learning_rate": 4.828760511501322e-06,
-      "logits/chosen": -1.9326432943344116,
-      "logits/rejected": -1.935786247253418,
-      "logps/chosen": -34.30440902709961,
-      "logps/rejected": -34.659637451171875,
-      "loss": 0.6521,
-      "rewards/accuracies": 0.625,
-      "rewards/chosen": 0.06169893592596054,
-      "rewards/margins": 0.09531383961439133,
-      "rewards/rejected": -0.03361489623785019,
+      "logits/chosen": -1.936668038368225,
+      "logits/rejected": -1.9397681951522827,
+      "logps/chosen": -34.20936965942383,
+      "logps/rejected": -34.525596618652344,
+      "loss": 0.4817,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.10922203958034515,
+      "rewards/margins": 0.07581819593906403,
+      "rewards/rejected": 0.033403851091861725,
       "step": 80
     },
     {
       "epoch": 0.23,
       "learning_rate": 4.7367166013034295e-06,
-      "logits/chosen": -1.9409675598144531,
-      "logits/rejected": -1.9454774856567383,
-      "logps/chosen": -32.3830680847168,
-      "logps/rejected": -32.33238983154297,
-      "loss": 0.6838,
-      "rewards/accuracies": 0.5874999761581421,
-      "rewards/chosen": 0.042176127433776855,
-      "rewards/margins": 0.028917592018842697,
-      "rewards/rejected": 0.013258534483611584,
+      "logits/chosen": -1.9462896585464478,
+      "logits/rejected": -1.9508006572723389,
+      "logps/chosen": -32.27099609375,
+      "logps/rejected": -32.275699615478516,
+      "loss": 0.4865,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": 0.0982138067483902,
+      "rewards/margins": 0.05660901591181755,
+      "rewards/rejected": 0.04160478338599205,
       "step": 90
     },
     {
       "epoch": 0.26,
       "learning_rate": 4.626245458345211e-06,
-      "logits/chosen": -2.038121461868286,
-      "logits/rejected": -2.036132574081421,
-      "logps/chosen": -32.12568664550781,
-      "logps/rejected": -31.2890567779541,
-      "loss": 0.6634,
-      "rewards/accuracies": 0.5874999761581421,
-      "rewards/chosen": 0.05757413059473038,
-      "rewards/margins": 0.06899620592594147,
-      "rewards/rejected": -0.011422084644436836,
+      "logits/chosen": -2.043682336807251,
+      "logits/rejected": -2.04168438911438,
+      "logps/chosen": -31.95809555053711,
+      "logps/rejected": -31.16133689880371,
+      "loss": 0.4794,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.14136961102485657,
+      "rewards/margins": 0.08893296122550964,
+      "rewards/rejected": 0.05243664234876633,
       "step": 100
     },
     {
       "epoch": 0.26,
-      "eval_logits/chosen": -2.232415199279785,
-      "eval_logits/rejected": -2.2275755405426025,
-      "eval_logps/chosen": -34.029048919677734,
-      "eval_logps/rejected": -37.52485275268555,
-      "eval_loss": 0.693107545375824,
-      "eval_rewards/accuracies": 0.5215947031974792,
-      "eval_rewards/chosen": 0.0027513643726706505,
-      "eval_rewards/margins": 0.006868092343211174,
-      "eval_rewards/rejected": -0.0041167279705405235,
-      "eval_runtime": 145.7484,
-      "eval_samples_per_second": 2.353,
+      "eval_logits/chosen": -2.2374911308288574,
+      "eval_logits/rejected": -2.232652425765991,
+      "eval_logps/chosen": -33.869850158691406,
+      "eval_logps/rejected": -37.37673568725586,
+      "eval_loss": 0.49711015820503235,
+      "eval_rewards/accuracies": 0.5573089718818665,
+      "eval_rewards/chosen": 0.08235026895999908,
+      "eval_rewards/margins": 0.012408134527504444,
+      "eval_rewards/rejected": 0.06994213908910751,
+      "eval_runtime": 145.9739,
+      "eval_samples_per_second": 2.35,
       "eval_steps_per_second": 0.295,
       "step": 100
     },
     {
       "epoch": 0.29,
       "learning_rate": 4.498257201263691e-06,
-      "logits/chosen": -1.9926633834838867,
-      "logits/rejected": -1.9902803897857666,
-      "logps/chosen": -33.11687088012695,
-      "logps/rejected": -34.01213836669922,
-      "loss": 0.6814,
-      "rewards/accuracies": 0.5874999761581421,
-      "rewards/chosen": 0.06347335875034332,
-      "rewards/margins": 0.054856397211551666,
-      "rewards/rejected": 0.008616959676146507,
+      "logits/chosen": -1.9995167255401611,
+      "logits/rejected": -1.9971492290496826,
+      "logps/chosen": -32.9360237121582,
+      "logps/rejected": -33.857337951660156,
+      "loss": 0.4819,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.15389741957187653,
+      "rewards/margins": 0.06788130104541779,
+      "rewards/rejected": 0.08601613342761993,
       "step": 110
     },
     {
       "epoch": 0.31,
       "learning_rate": 4.353806263777678e-06,
-      "logits/chosen": -2.0042788982391357,
-      "logits/rejected": -1.9959495067596436,
-      "logps/chosen": -32.306739807128906,
-      "logps/rejected": -32.13039779663086,
-      "loss": 0.6734,
-      "rewards/accuracies": 0.574999988079071,
-      "rewards/chosen": 0.06908417493104935,
-      "rewards/margins": 0.05353052541613579,
-      "rewards/rejected": 0.015553650446236134,
+      "logits/chosen": -2.0095458030700684,
+      "logits/rejected": -2.001213788986206,
+      "logps/chosen": -32.16087341308594,
+      "logps/rejected": -31.98464584350586,
+      "loss": 0.4876,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": 0.142018124461174,
+      "rewards/margins": 0.05358927324414253,
+      "rewards/rejected": 0.08842884749174118,
       "step": 120
     },
     {
       "epoch": 0.34,
       "learning_rate": 4.1940827077152755e-06,
-      "logits/chosen": -2.0326714515686035,
-      "logits/rejected": -2.0247092247009277,
-      "logps/chosen": -30.308746337890625,
-      "logps/rejected": -32.05224609375,
-      "loss": 0.6637,
-      "rewards/accuracies": 0.5874999761581421,
-      "rewards/chosen": 0.08348459005355835,
-      "rewards/margins": 0.08310474455356598,
-      "rewards/rejected": 0.00037985146627761424,
+      "logits/chosen": -2.0384786128997803,
+      "logits/rejected": -2.030527114868164,
+      "logps/chosen": -30.18499755859375,
+      "logps/rejected": -31.905630111694336,
+      "loss": 0.4841,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.14535793662071228,
+      "rewards/margins": 0.07167014479637146,
+      "rewards/rejected": 0.07368779182434082,
       "step": 130
     },
     {
       "epoch": 0.36,
       "learning_rate": 4.0204024186666215e-06,
-      "logits/chosen": -1.9627164602279663,
-      "logits/rejected": -1.9729163646697998,
-      "logps/chosen": -31.189788818359375,
-      "logps/rejected": -32.54594421386719,
-      "loss": 0.6424,
-      "rewards/accuracies": 0.612500011920929,
-      "rewards/chosen": 0.1149359717965126,
-      "rewards/margins": 0.12046756595373154,
-      "rewards/rejected": -0.005531603004783392,
+      "logits/chosen": -1.9675172567367554,
+      "logits/rejected": -1.97771418094635,
+      "logps/chosen": -31.068078994750977,
+      "logps/rejected": -32.39047622680664,
+      "loss": 0.4757,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.17579205334186554,
+      "rewards/margins": 0.10358880460262299,
+      "rewards/rejected": 0.07220325618982315,
       "step": 140
     },
     {
       "epoch": 0.39,
       "learning_rate": 3.834196265035119e-06,
-      "logits/chosen": -1.8740726709365845,
-      "logits/rejected": -1.875239372253418,
-      "logps/chosen": -33.88011932373047,
-      "logps/rejected": -34.779319763183594,
-      "loss": 0.6271,
-      "rewards/accuracies": 0.6000000238418579,
-      "rewards/chosen": 0.15628577768802643,
-      "rewards/margins": 0.1673184335231781,
-      "rewards/rejected": -0.011032682843506336,
+      "logits/chosen": -1.8799912929534912,
+      "logits/rejected": -1.8811372518539429,
+      "logps/chosen": -33.688819885253906,
+      "logps/rejected": -34.5561637878418,
+      "loss": 0.4651,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.2519363760948181,
+      "rewards/margins": 0.15139077603816986,
+      "rewards/rejected": 0.10054560005664825,
       "step": 150
     },
     {
       "epoch": 0.42,
       "learning_rate": 3.636998309800573e-06,
-      "logits/chosen": -1.9257261753082275,
-      "logits/rejected": -1.922323226928711,
-      "logps/chosen": -35.9793586730957,
-      "logps/rejected": -32.714969635009766,
-      "loss": 0.6539,
-      "rewards/accuracies": 0.612500011920929,
-      "rewards/chosen": 0.09615939855575562,
-      "rewards/margins": 0.09226818382740021,
-      "rewards/rejected": 0.003891219152137637,
+      "logits/chosen": -1.9306682348251343,
+      "logits/rejected": -1.9272987842559814,
+      "logps/chosen": -35.7833251953125,
+      "logps/rejected": -32.48335266113281,
+      "loss": 0.4818,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.19417627155780792,
+      "rewards/margins": 0.07447630167007446,
+      "rewards/rejected": 0.11969996988773346,
       "step": 160
     },
     {
       "epoch": 0.44,
       "learning_rate": 3.4304331721118078e-06,
-      "logits/chosen": -2.027190923690796,
-      "logits/rejected": -2.019850254058838,
-      "logps/chosen": -33.4937629699707,
-      "logps/rejected": -31.404333114624023,
-      "loss": 0.6193,
+      "logits/chosen": -2.0319628715515137,
+      "logits/rejected": -2.0246424674987793,
+      "logps/chosen": -33.24143981933594,
+      "logps/rejected": -31.190576553344727,
+      "loss": 0.4536,
       "rewards/accuracies": 0.6875,
-      "rewards/chosen": 0.15204860270023346,
-      "rewards/margins": 0.17829009890556335,
-      "rewards/rejected": -0.026241496205329895,
+      "rewards/chosen": 0.2782108783721924,
+      "rewards/margins": 0.19757375121116638,
+      "rewards/rejected": 0.0806371346116066,
       "step": 170
     },
     {
       "epoch": 0.47,
       "learning_rate": 3.2162026428305436e-06,
-      "logits/chosen": -2.033841609954834,
-      "logits/rejected": -2.039079427719116,
-      "logps/chosen": -32.22673797607422,
-      "logps/rejected": -32.453857421875,
-      "loss": 0.632,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": 0.1590258777141571,
-      "rewards/margins": 0.1433834582567215,
-      "rewards/rejected": 0.015642408281564713,
+      "logits/chosen": -2.039504289627075,
+      "logits/rejected": -2.044722080230713,
+      "logps/chosen": -31.97749900817871,
+      "logps/rejected": -32.195125579833984,
+      "loss": 0.4662,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": 0.28364285826683044,
+      "rewards/margins": 0.1386357843875885,
+      "rewards/rejected": 0.14500707387924194,
       "step": 180
     },
     {
       "epoch": 0.49,
       "learning_rate": 2.996071664294641e-06,
-      "logits/chosen": -2.03458833694458,
-      "logits/rejected": -2.031813144683838,
-      "logps/chosen": -31.249963760375977,
-      "logps/rejected": -31.329097747802734,
-      "loss": 0.6439,
-      "rewards/accuracies": 0.5874999761581421,
-      "rewards/chosen": 0.11949291080236435,
-      "rewards/margins": 0.12493407726287842,
-      "rewards/rejected": -0.005441152956336737,
+      "logits/chosen": -2.0397489070892334,
+      "logits/rejected": -2.037031412124634,
+      "logps/chosen": -31.067768096923828,
+      "logps/rejected": -31.097219467163086,
+      "loss": 0.4756,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.21059219539165497,
+      "rewards/margins": 0.10009355843067169,
+      "rewards/rejected": 0.11049864441156387,
       "step": 190
     },
     {
       "epoch": 0.52,
       "learning_rate": 2.7718537898066833e-06,
-      "logits/chosen": -1.9052807092666626,
-      "logits/rejected": -1.9099184274673462,
-      "logps/chosen": -31.314193725585938,
-      "logps/rejected": -32.81206512451172,
-      "loss": 0.6329,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": 0.14604374766349792,
-      "rewards/margins": 0.15660937130451202,
-      "rewards/rejected": -0.010565629228949547,
+      "logits/chosen": -1.910517930984497,
+      "logits/rejected": -1.915186882019043,
+      "logps/chosen": -31.059677124023438,
+      "logps/rejected": -32.61454391479492,
+      "loss": 0.4561,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.27330341935157776,
+      "rewards/margins": 0.1851106435060501,
+      "rewards/rejected": 0.08819273114204407,
       "step": 200
     },
     {
       "epoch": 0.52,
-      "eval_logits/chosen": -2.230377674102783,
-      "eval_logits/rejected": -2.225529193878174,
-      "eval_logps/chosen": -34.05998992919922,
-      "eval_logps/rejected": -37.57161331176758,
-      "eval_loss": 0.6905081868171692,
-      "eval_rewards/accuracies": 0.5274086594581604,
-      "eval_rewards/chosen": -0.012718739919364452,
-      "eval_rewards/margins": 0.014778696931898594,
-      "eval_rewards/rejected": -0.027497438713908195,
-      "eval_runtime": 145.701,
-      "eval_samples_per_second": 2.354,
+      "eval_logits/chosen": -2.234231948852539,
+      "eval_logits/rejected": -2.2294156551361084,
+      "eval_logps/chosen": -33.776145935058594,
+      "eval_logps/rejected": -37.29295349121094,
+      "eval_loss": 0.49589911103248596,
+      "eval_rewards/accuracies": 0.5423588156700134,
+      "eval_rewards/chosen": 0.12920260429382324,
+      "eval_rewards/margins": 0.017370687797665596,
+      "eval_rewards/rejected": 0.1118319109082222,
+      "eval_runtime": 145.6148,
+      "eval_samples_per_second": 2.356,
       "eval_steps_per_second": 0.295,
       "step": 200
     },
     {
       "epoch": 0.55,
       "learning_rate": 2.5453962426402006e-06,
-      "logits/chosen": -2.017439603805542,
-      "logits/rejected": -2.0280823707580566,
-      "logps/chosen": -31.72454261779785,
-      "logps/rejected": -33.935951232910156,
-      "loss": 0.6269,
-      "rewards/accuracies": 0.6625000238418579,
-      "rewards/chosen": 0.12337962538003922,
-      "rewards/margins": 0.16000542044639587,
-      "rewards/rejected": -0.036625780165195465,
+      "logits/chosen": -2.0229763984680176,
+      "logits/rejected": -2.0335872173309326,
+      "logps/chosen": -31.537296295166016,
+      "logps/rejected": -33.72160339355469,
+      "loss": 0.466,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": 0.21700111031532288,
+      "rewards/margins": 0.14645527303218842,
+      "rewards/rejected": 0.07054580748081207,
       "step": 210
     },
     {
       "epoch": 0.57,
       "learning_rate": 2.3185646976551794e-06,
-      "logits/chosen": -1.909328818321228,
-      "logits/rejected": -1.924088716506958,
-      "logps/chosen": -29.841415405273438,
-      "logps/rejected": -31.60904884338379,
-      "loss": 0.6274,
-      "rewards/accuracies": 0.675000011920929,
-      "rewards/chosen": 0.13506175577640533,
-      "rewards/margins": 0.16049805283546448,
-      "rewards/rejected": -0.02543630823493004,
+      "logits/chosen": -1.9160172939300537,
+      "logits/rejected": -1.9307291507720947,
+      "logps/chosen": -29.558719635009766,
+      "logps/rejected": -31.404027938842773,
+      "loss": 0.4532,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": 0.27640971541404724,
+      "rewards/margins": 0.19933710992336273,
+      "rewards/rejected": 0.07707259804010391,
       "step": 220
     },
     {
       "epoch": 0.6,
       "learning_rate": 2.0932279108998323e-06,
-      "logits/chosen": -1.966265320777893,
-      "logits/rejected": -1.970245361328125,
-      "logps/chosen": -33.091209411621094,
-      "logps/rejected": -31.639759063720703,
-      "loss": 0.6126,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": 0.16312837600708008,
-      "rewards/margins": 0.20835788547992706,
-      "rewards/rejected": -0.045229505747556686,
+      "logits/chosen": -1.9730017185211182,
+      "logits/rejected": -1.9770148992538452,
+      "logps/chosen": -32.84386444091797,
+      "logps/rejected": -31.42836570739746,
+      "loss": 0.4471,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.2868002951145172,
+      "rewards/margins": 0.22633206844329834,
+      "rewards/rejected": 0.06046823784708977,
       "step": 230
     },
     {
       "epoch": 0.62,
       "learning_rate": 1.8712423238279358e-06,
-      "logits/chosen": -1.9648067951202393,
-      "logits/rejected": -1.9429614543914795,
-      "logps/chosen": -33.82001495361328,
-      "logps/rejected": -35.11749267578125,
-      "loss": 0.5941,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": 0.16686691343784332,
-      "rewards/margins": 0.25021862983703613,
-      "rewards/rejected": -0.08335171639919281,
+      "logits/chosen": -1.9717906713485718,
+      "logits/rejected": -1.950059175491333,
+      "logps/chosen": -33.563629150390625,
+      "logps/rejected": -34.86870193481445,
+      "loss": 0.4421,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": 0.2950619161128998,
+      "rewards/margins": 0.25401392579078674,
+      "rewards/rejected": 0.041047997772693634,
       "step": 240
     },
     {
       "epoch": 0.65,
       "learning_rate": 1.6544367689701824e-06,
-      "logits/chosen": -2.005873441696167,
-      "logits/rejected": -2.002545118331909,
-      "logps/chosen": -32.70961380004883,
-      "logps/rejected": -36.252098083496094,
-      "loss": 0.6448,
-      "rewards/accuracies": 0.637499988079071,
-      "rewards/chosen": 0.10772605985403061,
-      "rewards/margins": 0.12131496518850327,
-      "rewards/rejected": -0.013588905334472656,
+      "logits/chosen": -2.0124945640563965,
+      "logits/rejected": -2.0092015266418457,
+      "logps/chosen": -32.43610382080078,
+      "logps/rejected": -35.959327697753906,
+      "loss": 0.4732,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": 0.24448053538799286,
+      "rewards/margins": 0.11168196052312851,
+      "rewards/rejected": 0.13279855251312256,
       "step": 250
     },
     {
       "epoch": 0.68,
       "learning_rate": 1.4445974030621963e-06,
-      "logits/chosen": -1.8737099170684814,
-      "logits/rejected": -1.8713098764419556,
-      "logps/chosen": -33.96501922607422,
-      "logps/rejected": -35.54829025268555,
-      "loss": 0.6377,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": 0.11169042438268661,
-      "rewards/margins": 0.13352498412132263,
-      "rewards/rejected": -0.02183455601334572,
+      "logits/chosen": -1.8797550201416016,
+      "logits/rejected": -1.8773235082626343,
+      "logps/chosen": -33.68698501586914,
+      "logps/rejected": -35.27508544921875,
+      "loss": 0.4682,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.25070956349372864,
+      "rewards/margins": 0.13594172894954681,
+      "rewards/rejected": 0.11476783454418182,
       "step": 260
     },
     {
       "epoch": 0.7,
       "learning_rate": 1.243452991757889e-06,
-      "logits/chosen": -1.858642578125,
-      "logits/rejected": -1.8562240600585938,
-      "logps/chosen": -34.18030548095703,
-      "logps/rejected": -31.82675552368164,
-      "loss": 0.6384,
-      "rewards/accuracies": 0.6625000238418579,
-      "rewards/chosen": 0.1086462140083313,
-      "rewards/margins": 0.1363571435213089,
-      "rewards/rejected": -0.027710938826203346,
+      "logits/chosen": -1.8655035495758057,
+      "logits/rejected": -1.8629907369613647,
+      "logps/chosen": -33.91318893432617,
+      "logps/rejected": -31.576608657836914,
+      "loss": 0.4665,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": 0.24220648407936096,
+      "rewards/margins": 0.1448442041873932,
+      "rewards/rejected": 0.09736229479312897,
       "step": 270
     },
     {
       "epoch": 0.73,
       "learning_rate": 1.0526606671603523e-06,
-      "logits/chosen": -1.9618957042694092,
-      "logits/rejected": -1.9513881206512451,
-      "logps/chosen": -35.000816345214844,
-      "logps/rejected": -31.879558563232422,
-      "loss": 0.6107,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": 0.17791253328323364,
-      "rewards/margins": 0.1971966028213501,
-      "rewards/rejected": -0.019284065812826157,
+      "logits/chosen": -1.9690582752227783,
+      "logits/rejected": -1.9586395025253296,
+      "logps/chosen": -34.75696563720703,
+      "logps/rejected": -31.642765045166016,
+      "loss": 0.4519,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.2998362183570862,
+      "rewards/margins": 0.20072226226329803,
+      "rewards/rejected": 0.09911395609378815,
       "step": 280
     },
     {
       "epoch": 0.75,
       "learning_rate": 8.737922755071455e-07,
-      "logits/chosen": -2.0570178031921387,
-      "logits/rejected": -2.04209566116333,
-      "logps/chosen": -30.695226669311523,
-      "logps/rejected": -32.64103317260742,
-      "loss": 0.654,
-      "rewards/accuracies": 0.5874999761581421,
-      "rewards/chosen": 0.11444780975580215,
-      "rewards/margins": 0.10904743522405624,
-      "rewards/rejected": 0.005400371737778187,
+      "logits/chosen": -2.0643129348754883,
+      "logits/rejected": -2.049489736557007,
+      "logps/chosen": -30.391122817993164,
+      "logps/rejected": -32.35709762573242,
+      "loss": 0.4722,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.2665007710456848,
+      "rewards/margins": 0.11913253366947174,
+      "rewards/rejected": 0.14736825227737427,
       "step": 290
     },
     {
       "epoch": 0.78,
       "learning_rate": 7.08321427484816e-07,
-      "logits/chosen": -1.9285688400268555,
-      "logits/rejected": -1.9260343313217163,
-      "logps/chosen": -32.38969039916992,
-      "logps/rejected": -30.898773193359375,
-      "loss": 0.5742,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": 0.2687075138092041,
-      "rewards/margins": 0.3096885085105896,
-      "rewards/rejected": -0.04098101332783699,
+      "logits/chosen": -1.9364421367645264,
+      "logits/rejected": -1.933985710144043,
+      "logps/chosen": -32.0788688659668,
+      "logps/rejected": -30.654926300048828,
+      "loss": 0.4202,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.42412155866622925,
+      "rewards/margins": 0.34317898750305176,
+      "rewards/rejected": 0.08094261586666107,
       "step": 300
     },
     {
       "epoch": 0.78,
-      "eval_logits/chosen": -2.2279160022735596,
-      "eval_logits/rejected": -2.223081350326538,
-      "eval_logps/chosen": -34.089088439941406,
-      "eval_logps/rejected": -37.59783935546875,
-      "eval_loss": 0.6919631958007812,
-      "eval_rewards/accuracies": 0.5278239250183105,
-      "eval_rewards/chosen": -0.027267219498753548,
-      "eval_rewards/margins": 0.013343668542802334,
-      "eval_rewards/rejected": -0.040610890835523605,
-      "eval_runtime": 145.7459,
-      "eval_samples_per_second": 2.353,
+      "eval_logits/chosen": -2.233308792114258,
+      "eval_logits/rejected": -2.2285048961639404,
+      "eval_logps/chosen": -33.75082015991211,
+      "eval_logps/rejected": -37.27885055541992,
+      "eval_loss": 0.4944371283054352,
+      "eval_rewards/accuracies": 0.5307309031486511,
+      "eval_rewards/chosen": 0.14186599850654602,
+      "eval_rewards/margins": 0.022981125861406326,
+      "eval_rewards/rejected": 0.1188848614692688,
+      "eval_runtime": 145.5333,
+      "eval_samples_per_second": 2.357,
       "eval_steps_per_second": 0.295,
       "step": 300
     },
     {
       "epoch": 0.81,
       "learning_rate": 5.576113578589035e-07,
-      "logits/chosen": -1.9126602411270142,
-      "logits/rejected": -1.9093825817108154,
-      "logps/chosen": -31.319168090820312,
-      "logps/rejected": -33.805519104003906,
-      "loss": 0.624,
-      "rewards/accuracies": 0.7749999761581421,
-      "rewards/chosen": 0.1420917958021164,
-      "rewards/margins": 0.1737762689590454,
-      "rewards/rejected": -0.03168448060750961,
+      "logits/chosen": -1.9192613363265991,
+      "logits/rejected": -1.9161159992218018,
+      "logps/chosen": -31.030216217041016,
+      "logps/rejected": -33.573768615722656,
+      "loss": 0.4538,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.2865663468837738,
+      "rewards/margins": 0.20237243175506592,
+      "rewards/rejected": 0.0841938853263855,
       "step": 310
     },
     {
       "epoch": 0.83,
       "learning_rate": 4.229036944380913e-07,
-      "logits/chosen": -1.9627044200897217,
-      "logits/rejected": -1.9504749774932861,
-      "logps/chosen": -34.31007385253906,
-      "logps/rejected": -33.66672134399414,
-      "loss": 0.6084,
+      "logits/chosen": -1.9703991413116455,
+      "logits/rejected": -1.9582570791244507,
+      "logps/chosen": -34.00434112548828,
+      "logps/rejected": -33.427330017089844,
+      "loss": 0.4439,
       "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": 0.1430002748966217,
-      "rewards/margins": 0.20694026350975037,
-      "rewards/rejected": -0.06393997371196747,
+      "rewards/chosen": 0.2958681583404541,
+      "rewards/margins": 0.24011309444904327,
+      "rewards/rejected": 0.05575507879257202,
       "step": 320
     },
     {
       "epoch": 0.86,
       "learning_rate": 3.053082288996112e-07,
-      "logits/chosen": -1.9980976581573486,
-      "logits/rejected": -1.9966537952423096,
-      "logps/chosen": -33.16533660888672,
-      "logps/rejected": -32.55678939819336,
-      "loss": 0.6136,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": 0.1602279245853424,
-      "rewards/margins": 0.19883206486701965,
-      "rewards/rejected": -0.03860412910580635,
+      "logits/chosen": -2.0051374435424805,
+      "logits/rejected": -2.003786563873291,
+      "logps/chosen": -32.876285552978516,
+      "logps/rejected": -32.25849914550781,
+      "loss": 0.4543,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": 0.30475252866744995,
+      "rewards/margins": 0.1942104697227478,
+      "rewards/rejected": 0.11054208129644394,
       "step": 330
     },
     {
       "epoch": 0.88,
       "learning_rate": 2.0579377374915805e-07,
-      "logits/chosen": -2.0848796367645264,
-      "logits/rejected": -2.069186210632324,
-      "logps/chosen": -33.787841796875,
-      "logps/rejected": -33.07987976074219,
-      "loss": 0.6229,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": 0.19577431678771973,
-      "rewards/margins": 0.17669571936130524,
-      "rewards/rejected": 0.01907859742641449,
+      "logits/chosen": -2.092106580734253,
+      "logits/rejected": -2.0764684677124023,
+      "logps/chosen": -33.44976043701172,
+      "logps/rejected": -32.806304931640625,
+      "loss": 0.4513,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.3648151457309723,
+      "rewards/margins": 0.2089495211839676,
+      "rewards/rejected": 0.1558656245470047,
       "step": 340
     },
     {
       "epoch": 0.91,
       "learning_rate": 1.2518018074041684e-07,
-      "logits/chosen": -1.957275390625,
-      "logits/rejected": -1.9564218521118164,
-      "logps/chosen": -32.81622314453125,
-      "logps/rejected": -32.52650833129883,
-      "loss": 0.5923,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": 0.23293733596801758,
-      "rewards/margins": 0.2636169195175171,
-      "rewards/rejected": -0.030679568648338318,
+      "logits/chosen": -1.9645261764526367,
+      "logits/rejected": -1.963702917098999,
+      "logps/chosen": -32.543540954589844,
+      "logps/rejected": -32.22251510620117,
+      "loss": 0.4427,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.36927860975265503,
+      "rewards/margins": 0.24796243011951447,
+      "rewards/rejected": 0.12131617963314056,
       "step": 350
     },
     {
       "epoch": 0.94,
       "learning_rate": 6.41315865106129e-08,
-      "logits/chosen": -1.9124408960342407,
-      "logits/rejected": -1.92275071144104,
-      "logps/chosen": -31.859888076782227,
-      "logps/rejected": -35.33869934082031,
-      "loss": 0.6119,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": 0.17240020632743835,
-      "rewards/margins": 0.19736871123313904,
-      "rewards/rejected": -0.024968529120087624,
+      "logits/chosen": -1.921724557876587,
+      "logits/rejected": -1.9320251941680908,
+      "logps/chosen": -31.573944091796875,
+      "logps/rejected": -35.00251007080078,
+      "loss": 0.4596,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.3153747618198395,
+      "rewards/margins": 0.17225053906440735,
+      "rewards/rejected": 0.14312422275543213,
       "step": 360
     },
     {
       "epoch": 0.96,
       "learning_rate": 2.3150941078050325e-08,
-      "logits/chosen": -2.0522782802581787,
-      "logits/rejected": -2.045797824859619,
-      "logps/chosen": -33.34915542602539,
-      "logps/rejected": -29.27215576171875,
-      "loss": 0.6194,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": 0.15140748023986816,
-      "rewards/margins": 0.1742721050977707,
-      "rewards/rejected": -0.022864630445837975,
+      "logits/chosen": -2.059705972671509,
+      "logits/rejected": -2.0532097816467285,
+      "logps/chosen": -33.03301239013672,
+      "logps/rejected": -28.99460220336914,
+      "loss": 0.4533,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": 0.3094797134399414,
+      "rewards/margins": 0.19356802105903625,
+      "rewards/rejected": 0.11591170728206635,
       "step": 370
     },
     {
       "epoch": 0.99,
       "learning_rate": 2.575864278703266e-09,
-      "logits/chosen": -1.9120715856552124,
-      "logits/rejected": -1.9142844676971436,
-      "logps/chosen": -33.86906051635742,
-      "logps/rejected": -30.961559295654297,
-      "loss": 0.5996,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": 0.1849948763847351,
-      "rewards/margins": 0.22946183383464813,
-      "rewards/rejected": -0.04446694999933243,
+      "logits/chosen": -1.919736623764038,
+      "logits/rejected": -1.9219011068344116,
+      "logps/chosen": -33.58899688720703,
+      "logps/rejected": -30.707019805908203,
+      "loss": 0.4436,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": 0.3250266909599304,
+      "rewards/margins": 0.24222226440906525,
+      "rewards/rejected": 0.08280440419912338,
       "step": 380
     },
     {
       "epoch": 1.0,
       "step": 385,
       "total_flos": 0.0,
-      "train_loss": 0.64145151051608,
-      "train_runtime": 3249.8987,
+      "train_loss": 0.4687722819192069,
+      "train_runtime": 3250.758,
       "train_samples_per_second": 0.947,
       "train_steps_per_second": 0.118
     }