Model save

README.md

@@ -1,13 +1,10 @@
 ---
 library_name: peft
 tags:
-- alignment-handbook
 - trl
 - dpo
 - generated_from_trainer
 base_model: NbAiLab/nb-gpt-j-6B-v2
-datasets:
-- hugodk-sch/aftonposten_title_prefs
 model-index:
 - name: aftonposten-6b-align-scan
   results: []
@@ -18,17 +15,17 @@ should probably proofread and complete it, then remove this comment. -->
 
 # aftonposten-6b-align-scan
 
-This model is a fine-tuned version of [data/ap-gpt-j-6b-sft-qlora-04-08](https://huggingface.co/data/ap-gpt-j-6b-sft-qlora-04-08) on the hugodk-sch/aftonposten_title_prefs dataset.
+This model is a fine-tuned version of [NbAiLab/nb-gpt-j-6B-v2](https://huggingface.co/NbAiLab/nb-gpt-j-6B-v2) on an unknown dataset.
 It achieves the following results on the evaluation set:
 - Loss: 1.0000
-- Rewards/chosen: 0.0001
-- Rewards/rejected: 0.0002
-- Rewards/accuracies: 0.5021
-- Rewards/margins: -0.0000
-- Logps/rejected: -37.5012
-- Logps/chosen: -34.0231
-- Logits/rejected: -2.2388
-- Logits/chosen: -2.2437
+- Rewards/chosen: 0.0017
+- Rewards/rejected: 0.0016
+- Rewards/accuracies: 0.4904
+- Rewards/margins: 0.0000
+- Logps/rejected: -37.5002
+- Logps/chosen: -34.0177
+- Logits/rejected: -2.2386
+- Logits/chosen: -2.2435
 
 ## Model description
 
@@ -63,9 +60,9 @@ The following hyperparameters were used during training:
 
 | Training Loss | Epoch | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
 |:-------------:|:-----:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
-| 1.0001        | 0.26  | 100  | 1.0000          | 0.0000         | 0.0001           | 0.5100             | -0.0000         | -37.5080       | -34.0296     | -2.2392         | -2.2441       |
-| 0.9998        | 0.52  | 200  | 1.0000          | 0.0001         | 0.0001           | 0.5046             | -0.0000         | -37.5040       | -34.0221     | -2.2392         | -2.2440       |
-| 0.9997        | 0.78  | 300  | 1.0000          | 0.0001         | 0.0001           | 0.4988             | 0.0000          | -37.5053       | -34.0201     | -2.2388         | -2.2436       |
+| 1.0003        | 0.26  | 100  | 1.0008          | 0.0011         | 0.0019           | 0.4983             | -0.0008         | -37.4972       | -34.0232     | -2.2384         | -2.2433       |
+| 0.999         | 0.52  | 200  | 0.9983          | 0.0016         | -0.0001          | 0.5328             | 0.0017          | -37.5175       | -34.0187     | -2.2389         | -2.2438       |
+| 0.9972        | 0.78  | 300  | 1.0000          | 0.0017         | 0.0016           | 0.4904             | 0.0000          | -37.5002       | -34.0177     | -2.2386         | -2.2435       |
 
 
 ### Framework versions

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5bdf74864526db2976a0432721d067682ebc5c68521aef8ad7742bc832e47156
+oid sha256:726700f20fcb28f7858cd1aa59e0cb2bd53a0fe4b7d2b662333531d42f0321bf
 size 176183216

all_results.json

@@ -1,21 +1,8 @@
 {
     "epoch": 1.0,
-    "eval_logits/chosen": -2.2436792850494385,
-    "eval_logits/rejected": -2.23879337310791,
-    "eval_logps/chosen": -34.02314376831055,
-    "eval_logps/rejected": -37.50123596191406,
-    "eval_loss": 1.0000426769256592,
-    "eval_rewards/accuracies": 0.5020764470100403,
-    "eval_rewards/chosen": 0.0001141041939263232,
-    "eval_rewards/margins": -3.9732771256240085e-05,
-    "eval_rewards/rejected": 0.00015383693971671164,
-    "eval_runtime": 145.4546,
-    "eval_samples": 343,
-    "eval_samples_per_second": 2.358,
-    "eval_steps_per_second": 0.296,
-    "train_loss": 0.9998476387618425,
-    "train_runtime": 3251.6099,
+    "train_loss": 0.9991211135666092,
+    "train_runtime": 3253.9902,
     "train_samples": 3079,
-    "train_samples_per_second": 0.947,
+    "train_samples_per_second": 0.946,
     "train_steps_per_second": 0.118
 }

train_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 1.0,
-    "train_loss": 0.9998476387618425,
-    "train_runtime": 3251.6099,
+    "train_loss": 0.9991211135666092,
+    "train_runtime": 3253.9902,
     "train_samples": 3079,
-    "train_samples_per_second": 0.947,
+    "train_samples_per_second": 0.946,
     "train_steps_per_second": 0.118
 }

trainer_state.json

@@ -25,590 +25,590 @@
     {
       "epoch": 0.03,
       "learning_rate": 1.2820512820512818e-07,
-      "logits/chosen": -1.8663256168365479,
-      "logits/rejected": -1.8706417083740234,
-      "logps/chosen": -36.98673629760742,
-      "logps/rejected": -33.66697311401367,
-      "loss": 0.9994,
-      "rewards/accuracies": 0.5,
-      "rewards/chosen": 0.00019861094187945127,
-      "rewards/margins": 0.0005675733555108309,
-      "rewards/rejected": -0.0003689624136313796,
+      "logits/chosen": -1.866065502166748,
+      "logits/rejected": -1.8703795671463013,
+      "logps/chosen": -36.988380432128906,
+      "logps/rejected": -33.66728210449219,
+      "loss": 0.9945,
+      "rewards/accuracies": 0.5694444179534912,
+      "rewards/chosen": 0.0018219809280708432,
+      "rewards/margins": 0.00554280448704958,
+      "rewards/rejected": -0.0037208227440714836,
       "step": 10
     },
     {
       "epoch": 0.05,
       "learning_rate": 2.5641025641025636e-07,
-      "logits/chosen": -1.997527837753296,
-      "logits/rejected": -2.0001606941223145,
-      "logps/chosen": -29.65478515625,
-      "logps/rejected": -29.070556640625,
-      "loss": 1.0001,
-      "rewards/accuracies": 0.44999998807907104,
-      "rewards/chosen": -0.00012590530968736857,
-      "rewards/margins": -5.477964805322699e-05,
-      "rewards/rejected": -7.112567982403561e-05,
+      "logits/chosen": -1.997332215309143,
+      "logits/rejected": -1.999983549118042,
+      "logps/chosen": -29.625896453857422,
+      "logps/rejected": -29.035802841186523,
+      "loss": 1.0011,
+      "rewards/accuracies": 0.4375,
+      "rewards/chosen": 0.0016297042602673173,
+      "rewards/margins": -0.0011343134101480246,
+      "rewards/rejected": 0.00276401755400002,
       "step": 20
     },
     {
       "epoch": 0.08,
       "learning_rate": 3.8461538461538463e-07,
-      "logits/chosen": -1.9195702075958252,
-      "logits/rejected": -1.9168812036514282,
-      "logps/chosen": -31.421009063720703,
-      "logps/rejected": -33.225616455078125,
-      "loss": 1.0,
-      "rewards/accuracies": 0.48750001192092896,
-      "rewards/chosen": -4.912544318358414e-05,
-      "rewards/margins": 2.244002462248318e-05,
-      "rewards/rejected": -7.15654605301097e-05,
+      "logits/chosen": -1.9199495315551758,
+      "logits/rejected": -1.917249321937561,
+      "logps/chosen": -31.421478271484375,
+      "logps/rejected": -33.2115364074707,
+      "loss": 1.0012,
+      "rewards/accuracies": 0.4625000059604645,
+      "rewards/chosen": -0.0005385443801060319,
+      "rewards/margins": -0.0012309944722801447,
+      "rewards/rejected": 0.0006924502667970955,
       "step": 30
     },
     {
       "epoch": 0.1,
       "learning_rate": 4.999896948438433e-07,
-      "logits/chosen": -2.0165441036224365,
-      "logits/rejected": -2.0078024864196777,
-      "logps/chosen": -32.57805252075195,
-      "logps/rejected": -32.504085540771484,
-      "loss": 1.0001,
-      "rewards/accuracies": 0.4375,
-      "rewards/chosen": -1.2443718333088327e-05,
-      "rewards/margins": -9.829048212850466e-05,
-      "rewards/rejected": 8.584677561884746e-05,
+      "logits/chosen": -2.0169284343719482,
+      "logits/rejected": -2.008178949356079,
+      "logps/chosen": -32.59435272216797,
+      "logps/rejected": -32.49193572998047,
+      "loss": 1.0038,
+      "rewards/accuracies": 0.44999998807907104,
+      "rewards/chosen": -0.0017543068388476968,
+      "rewards/margins": -0.0038279418367892504,
+      "rewards/rejected": 0.0020736351143568754,
       "step": 40
     },
     {
       "epoch": 0.13,
       "learning_rate": 4.987541037542186e-07,
-      "logits/chosen": -1.8641563653945923,
-      "logits/rejected": -1.8533931970596313,
-      "logps/chosen": -33.555824279785156,
-      "logps/rejected": -35.42582321166992,
-      "loss": 1.0002,
-      "rewards/accuracies": 0.36250001192092896,
-      "rewards/chosen": 1.6570575098739937e-05,
-      "rewards/margins": -0.0002486651937942952,
-      "rewards/rejected": 0.000265235808910802,
+      "logits/chosen": -1.86457097530365,
+      "logits/rejected": -1.8537908792495728,
+      "logps/chosen": -33.56566619873047,
+      "logps/rejected": -35.423240661621094,
+      "loss": 1.0037,
+      "rewards/accuracies": 0.4375,
+      "rewards/chosen": -0.0008188748615793884,
+      "rewards/margins": -0.003729355288669467,
+      "rewards/rejected": 0.0029104803688824177,
       "step": 50
     },
     {
       "epoch": 0.16,
       "learning_rate": 4.954691471941118e-07,
-      "logits/chosen": -1.9452263116836548,
-      "logits/rejected": -1.9471622705459595,
-      "logps/chosen": -32.5916633605957,
-      "logps/rejected": -33.1885871887207,
-      "loss": 0.9997,
-      "rewards/accuracies": 0.6625000238418579,
-      "rewards/chosen": 8.520439587300643e-05,
-      "rewards/margins": 0.00025727329193614423,
-      "rewards/rejected": -0.0001720689469948411,
+      "logits/chosen": -1.9449050426483154,
+      "logits/rejected": -1.9468472003936768,
+      "logps/chosen": -32.59955596923828,
+      "logps/rejected": -33.1828498840332,
+      "loss": 0.9988,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": 6.278457294683903e-05,
+      "rewards/margins": 0.0012094037374481559,
+      "rewards/rejected": -0.0011466194409877062,
       "step": 60
     },
     {
       "epoch": 0.18,
       "learning_rate": 4.901618883413548e-07,
-      "logits/chosen": -2.0798425674438477,
-      "logits/rejected": -2.08483624458313,
-      "logps/chosen": -33.98626708984375,
-      "logps/rejected": -36.57551956176758,
-      "loss": 0.9999,
+      "logits/chosen": -2.079878330230713,
+      "logits/rejected": -2.084862232208252,
+      "logps/chosen": -33.98878860473633,
+      "logps/rejected": -36.574462890625,
+      "loss": 0.9995,
       "rewards/accuracies": 0.512499988079071,
-      "rewards/chosen": 2.217638393631205e-05,
-      "rewards/margins": 8.864814299158752e-05,
-      "rewards/rejected": -6.647173722740263e-05,
+      "rewards/chosen": -3.0292372684925795e-05,
+      "rewards/margins": 0.0005289571708999574,
+      "rewards/rejected": -0.0005592495435848832,
       "step": 70
     },
     {
       "epoch": 0.21,
       "learning_rate": 4.828760511501322e-07,
-      "logits/chosen": -1.942161202430725,
-      "logits/rejected": -1.9453051090240479,
-      "logps/chosen": -34.400901794433594,
-      "logps/rejected": -34.57304382324219,
-      "loss": 0.9999,
+      "logits/chosen": -1.9425569772720337,
+      "logits/rejected": -1.9457191228866577,
+      "logps/chosen": -34.40068054199219,
+      "logps/rejected": -34.5762939453125,
+      "loss": 0.9989,
       "rewards/accuracies": 0.5249999761581421,
-      "rewards/chosen": 0.00026908880681730807,
-      "rewards/margins": 7.544312393292785e-05,
-      "rewards/rejected": 0.000193645668332465,
+      "rewards/chosen": 0.002712726127356291,
+      "rewards/margins": 0.0011012849863618612,
+      "rewards/rejected": 0.0016114413738250732,
       "step": 80
     },
     {
       "epoch": 0.23,
       "learning_rate": 4.736716601303429e-07,
-      "logits/chosen": -1.9505256414413452,
-      "logits/rejected": -1.9550361633300781,
-      "logps/chosen": -32.452125549316406,
-      "logps/rejected": -32.35675811767578,
-      "loss": 0.9999,
-      "rewards/accuracies": 0.5874999761581421,
-      "rewards/chosen": 0.00015296679339371622,
-      "rewards/margins": 0.0001315164554398507,
-      "rewards/rejected": 2.145032885891851e-05,
+      "logits/chosen": -1.9507396221160889,
+      "logits/rejected": -1.9552500247955322,
+      "logps/chosen": -32.460357666015625,
+      "logps/rejected": -32.354434967041016,
+      "loss": 0.9997,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": 0.0007068099221214652,
+      "rewards/margins": 0.00025928454124368727,
+      "rewards/rejected": 0.0004475254681892693,
       "step": 90
     },
     {
       "epoch": 0.26,
       "learning_rate": 4.62624545834521e-07,
-      "logits/chosen": -2.0490074157714844,
-      "logits/rejected": -2.0470099449157715,
-      "logps/chosen": -32.25472640991211,
-      "logps/rejected": -31.2731990814209,
-      "loss": 1.0001,
-      "rewards/accuracies": 0.48750001192092896,
-      "rewards/chosen": -0.00013893461436964571,
-      "rewards/margins": -6.904623296577483e-05,
-      "rewards/rejected": -6.98883886798285e-05,
+      "logits/chosen": -2.0492873191833496,
+      "logits/rejected": -2.0472888946533203,
+      "logps/chosen": -32.23810958862305,
+      "logps/rejected": -31.260278701782227,
+      "loss": 1.0003,
+      "rewards/accuracies": 0.4625000059604645,
+      "rewards/chosen": 0.00027285455144010484,
+      "rewards/margins": -0.00032049461151473224,
+      "rewards/rejected": 0.0005933489883318543,
       "step": 100
     },
     {
       "epoch": 0.26,
-      "eval_logits/chosen": -2.2441036701202393,
-      "eval_logits/rejected": -2.239219903945923,
-      "eval_logps/chosen": -34.02959442138672,
-      "eval_logps/rejected": -37.50800704956055,
-      "eval_loss": 1.0000391006469727,
-      "eval_rewards/accuracies": 0.5099667906761169,
-      "eval_rewards/chosen": 4.9558610044186935e-05,
-      "eval_rewards/margins": -3.6566227208822966e-05,
-      "eval_rewards/rejected": 8.61248336150311e-05,
-      "eval_runtime": 145.9645,
-      "eval_samples_per_second": 2.35,
-      "eval_steps_per_second": 0.295,
+      "eval_logits/chosen": -2.243312358856201,
+      "eval_logits/rejected": -2.238436222076416,
+      "eval_logps/chosen": -34.023216247558594,
+      "eval_logps/rejected": -37.49723434448242,
+      "eval_loss": 1.0008214712142944,
+      "eval_rewards/accuracies": 0.49833887815475464,
+      "eval_rewards/chosen": 0.0011338649783283472,
+      "eval_rewards/margins": -0.0008048001327551901,
+      "eval_rewards/rejected": 0.0019386651692911983,
+      "eval_runtime": 146.1271,
+      "eval_samples_per_second": 2.347,
+      "eval_steps_per_second": 0.294,
       "step": 100
     },
     {
       "epoch": 0.29,
       "learning_rate": 4.4982572012636904e-07,
-      "logits/chosen": -2.005286931991577,
-      "logits/rejected": -2.0028929710388184,
-      "logps/chosen": -33.22523498535156,
-      "logps/rejected": -34.039817810058594,
-      "loss": 0.9997,
-      "rewards/accuracies": 0.5249999761581421,
-      "rewards/chosen": 0.00018582009943202138,
-      "rewards/margins": 0.0002903031127061695,
-      "rewards/rejected": -0.00010448300599819049,
+      "logits/chosen": -2.005169153213501,
+      "logits/rejected": -2.0027499198913574,
+      "logps/chosen": -33.2365837097168,
+      "logps/rejected": -34.01953125,
+      "loss": 1.0003,
+      "rewards/accuracies": 0.4625000059604645,
+      "rewards/chosen": 0.0007230007322505116,
+      "rewards/margins": -0.00026112591149285436,
+      "rewards/rejected": 0.0009841264691203833,
       "step": 110
     },
     {
       "epoch": 0.31,
       "learning_rate": 4.353806263777677e-07,
-      "logits/chosen": -2.0164072513580322,
-      "logits/rejected": -2.008033514022827,
-      "logps/chosen": -32.473182678222656,
-      "logps/rejected": -32.18474197387695,
-      "loss": 1.0001,
-      "rewards/accuracies": 0.4625000059604645,
-      "rewards/chosen": -0.00028277214732952416,
-      "rewards/margins": -5.0430990086169913e-05,
-      "rewards/rejected": -0.00023234116088133305,
+      "logits/chosen": -2.0166728496551514,
+      "logits/rejected": -2.0083022117614746,
+      "logps/chosen": -32.457847595214844,
+      "logps/rejected": -32.18357467651367,
+      "loss": 0.9991,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.0012942379107698798,
+      "rewards/margins": 0.0009121716138906777,
+      "rewards/rejected": -0.002206409815698862,
       "step": 120
     },
     {
       "epoch": 0.34,
       "learning_rate": 4.194082707715275e-07,
-      "logits/chosen": -2.0463318824768066,
-      "logits/rejected": -2.03826642036438,
-      "logps/chosen": -30.479427337646484,
-      "logps/rejected": -32.030479431152344,
-      "loss": 1.0003,
-      "rewards/accuracies": 0.4000000059604645,
-      "rewards/chosen": -3.712148100021295e-05,
-      "rewards/margins": -0.00026237856945954263,
-      "rewards/rejected": 0.0002252570993732661,
+      "logits/chosen": -2.0462448596954346,
+      "logits/rejected": -2.038203001022339,
+      "logps/chosen": -30.475027084350586,
+      "logps/rejected": -32.046302795410156,
+      "loss": 1.0006,
+      "rewards/accuracies": 0.48750001192092896,
+      "rewards/chosen": 6.882836896693334e-05,
+      "rewards/margins": -0.0006015264661982656,
+      "rewards/rejected": 0.0006703549297526479,
       "step": 130
     },
     {
       "epoch": 0.36,
       "learning_rate": 4.020402418666621e-07,
-      "logits/chosen": -1.9770317077636719,
-      "logits/rejected": -1.987318754196167,
-      "logps/chosen": -31.41189193725586,
-      "logps/rejected": -32.552860260009766,
-      "loss": 0.9997,
-      "rewards/accuracies": 0.4625000059604645,
-      "rewards/chosen": 7.772201206535101e-05,
-      "rewards/margins": 0.000257515290286392,
-      "rewards/rejected": -0.00017979330732487142,
+      "logits/chosen": -1.976986289024353,
+      "logits/rejected": -1.9872560501098633,
+      "logps/chosen": -31.407278060913086,
+      "logps/rejected": -32.543296813964844,
+      "loss": 0.9979,
+      "rewards/accuracies": 0.48750001192092896,
+      "rewards/chosen": 0.0012384230503812432,
+      "rewards/margins": 0.0020802102517336607,
+      "rewards/rejected": -0.0008417873759754002,
       "step": 140
     },
     {
       "epoch": 0.39,
       "learning_rate": 3.8341962650351185e-07,
-      "logits/chosen": -1.8904685974121094,
-      "logits/rejected": -1.8915725946426392,
-      "logps/chosen": -34.178749084472656,
-      "logps/rejected": -34.7522087097168,
-      "loss": 0.9999,
-      "rewards/accuracies": 0.44999998807907104,
-      "rewards/chosen": 0.00013941082579549402,
-      "rewards/margins": 8.894065831555054e-05,
-      "rewards/rejected": 5.047018566983752e-05,
+      "logits/chosen": -1.8905508518218994,
+      "logits/rejected": -1.891632080078125,
+      "logps/chosen": -34.20501708984375,
+      "logps/rejected": -34.77235412597656,
+      "loss": 0.9997,
+      "rewards/accuracies": 0.4625000059604645,
+      "rewards/chosen": -0.0012326312717050314,
+      "rewards/margins": 0.00027702696388587356,
+      "rewards/rejected": -0.001509658177383244,
       "step": 150
     },
     {
       "epoch": 0.42,
       "learning_rate": 3.636998309800572e-07,
-      "logits/chosen": -1.9424184560775757,
-      "logits/rejected": -1.9389402866363525,
-      "logps/chosen": -36.14480209350586,
-      "logps/rejected": -32.753517150878906,
-      "loss": 0.9994,
-      "rewards/accuracies": 0.5249999761581421,
-      "rewards/chosen": 0.0002687961678020656,
-      "rewards/margins": 0.0005764620145782828,
-      "rewards/rejected": -0.0003076658467762172,
+      "logits/chosen": -1.94281005859375,
+      "logits/rejected": -1.939327597618103,
+      "logps/chosen": -36.144107818603516,
+      "logps/rejected": -32.72822570800781,
+      "loss": 0.9967,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": 0.0027573786210268736,
+      "rewards/margins": 0.0033046435564756393,
+      "rewards/rejected": -0.0005472644697874784,
       "step": 160
     },
     {
       "epoch": 0.44,
       "learning_rate": 3.430433172111807e-07,
-      "logits/chosen": -2.0420711040496826,
-      "logits/rejected": -2.0346856117248535,
-      "logps/chosen": -33.783363342285156,
-      "logps/rejected": -31.349674224853516,
-      "loss": 0.9999,
-      "rewards/accuracies": 0.5249999761581421,
-      "rewards/chosen": 0.00014495299546979368,
-      "rewards/margins": 0.00012318804510869086,
-      "rewards/rejected": 2.1765008568763733e-05,
+      "logits/chosen": -2.0420708656311035,
+      "logits/rejected": -2.0346803665161133,
+      "logps/chosen": -33.771934509277344,
+      "logps/rejected": -31.371145248413086,
+      "loss": 0.9955,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": 0.002593145240098238,
+      "rewards/margins": 0.004522555507719517,
+      "rewards/rejected": -0.0019294100347906351,
       "step": 170
     },
     {
       "epoch": 0.47,
       "learning_rate": 3.216202642830543e-07,
-      "logits/chosen": -2.0475425720214844,
-      "logits/rejected": -2.052799940109253,
-      "logps/chosen": -32.50774383544922,
-      "logps/rejected": -32.48485565185547,
-      "loss": 0.9996,
-      "rewards/accuracies": 0.574999988079071,
-      "rewards/chosen": 0.00037043425254523754,
-      "rewards/margins": 0.0003675749758258462,
-      "rewards/rejected": 2.859340838767821e-06,
+      "logits/chosen": -2.0475738048553467,
+      "logits/rejected": -2.0528526306152344,
+      "logps/chosen": -32.524593353271484,
+      "logps/rejected": -32.510643005371094,
+      "loss": 0.9954,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.0020192908123135567,
+      "rewards/margins": 0.004569421522319317,
+      "rewards/rejected": -0.002550130244344473,
       "step": 180
     },
     {
       "epoch": 0.49,
       "learning_rate": 2.9960716642946403e-07,
-      "logits/chosen": -2.0484375953674316,
-      "logits/rejected": -2.045671224594116,
-      "logps/chosen": -31.50165367126465,
-      "logps/rejected": -31.33974266052246,
-      "loss": 0.9999,
-      "rewards/accuracies": 0.5625,
-      "rewards/chosen": -0.00012700485240202397,
-      "rewards/margins": 8.82898602867499e-05,
-      "rewards/rejected": -0.00021529469813685864,
+      "logits/chosen": -2.048490524291992,
+      "logits/rejected": -2.0457024574279785,
+      "logps/chosen": -31.492746353149414,
+      "logps/rejected": -31.319293975830078,
+      "loss": 1.0003,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -0.0003792368806898594,
+      "rewards/margins": -0.0002712804707698524,
+      "rewards/rejected": -0.0001079567227861844,
       "step": 190
     },
     {
       "epoch": 0.52,
       "learning_rate": 2.771853789806683e-07,
-      "logits/chosen": -1.9182907342910767,
-      "logits/rejected": -1.9229600429534912,
-      "logps/chosen": -31.5924072265625,
-      "logps/rejected": -32.794044494628906,
-      "loss": 0.9998,
-      "rewards/accuracies": 0.512499988079071,
-      "rewards/chosen": 0.00013877781748306006,
-      "rewards/margins": 0.0001699000713415444,
-      "rewards/rejected": -3.112227204837836e-05,
+      "logits/chosen": -1.9185683727264404,
+      "logits/rejected": -1.9232347011566162,
+      "logps/chosen": -31.5926513671875,
+      "logps/rejected": -32.78697204589844,
+      "loss": 0.999,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": 0.001363297225907445,
+      "rewards/margins": 0.0009675474721007049,
+      "rewards/rejected": 0.0003957498411182314,
       "step": 200
     },
     {
       "epoch": 0.52,
-      "eval_logits/chosen": -2.2440481185913086,
-      "eval_logits/rejected": -2.2391700744628906,
-      "eval_logps/chosen": -34.02214813232422,
-      "eval_logps/rejected": -37.503997802734375,
-      "eval_loss": 1.0000054836273193,
-      "eval_rewards/accuracies": 0.5045680999755859,
-      "eval_rewards/chosen": 0.00012402697757352144,
-      "eval_rewards/margins": -2.209126932939398e-06,
-      "eval_rewards/rejected": 0.00012623611837625504,
-      "eval_runtime": 145.6161,
-      "eval_samples_per_second": 2.356,
+      "eval_logits/chosen": -2.243807315826416,
+      "eval_logits/rejected": -2.2389235496520996,
+      "eval_logps/chosen": -34.018714904785156,
+      "eval_logps/rejected": -37.517478942871094,
+      "eval_loss": 0.9983054399490356,
+      "eval_rewards/accuracies": 0.5328072905540466,
+      "eval_rewards/chosen": 0.0015839905245229602,
+      "eval_rewards/margins": 0.0016701342537999153,
+      "eval_rewards/rejected": -8.614380931248888e-05,
+      "eval_runtime": 145.691,
+      "eval_samples_per_second": 2.354,
       "eval_steps_per_second": 0.295,
       "step": 200
     },
     {
       "epoch": 0.55,
       "learning_rate": 2.5453962426402e-07,
-      "logits/chosen": -2.0312139987945557,
-      "logits/rejected": -2.041886568069458,
-      "logps/chosen": -31.938335418701172,
-      "logps/rejected": -33.87850570678711,
-      "loss": 0.9995,
-      "rewards/accuracies": 0.574999988079071,
-      "rewards/chosen": 0.00032970920437946916,
-      "rewards/margins": 0.0004878062754869461,
-      "rewards/rejected": -0.00015809701289981604,
+      "logits/chosen": -2.0318503379821777,
+      "logits/rejected": -2.042539596557617,
+      "logps/chosen": -31.948400497436523,
+      "logps/rejected": -33.86983871459961,
+      "loss": 0.997,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": 0.0022904814686626196,
+      "rewards/margins": 0.0030044156592339277,
+      "rewards/rejected": -0.0007139344816096127,
       "step": 210
     },
     {
       "epoch": 0.57,
       "learning_rate": 2.318564697655179e-07,
-      "logits/chosen": -1.925606369972229,
-      "logits/rejected": -1.940452218055725,
-      "logps/chosen": -30.110549926757812,
-      "logps/rejected": -31.573543548583984,
-      "loss": 0.9998,
-      "rewards/accuracies": 0.5874999761581421,
-      "rewards/chosen": 9.872082955553196e-06,
-      "rewards/margins": 0.00016354399849660695,
-      "rewards/rejected": -0.00015367190644610673,
+      "logits/chosen": -1.9251388311386108,
+      "logits/rejected": -1.9399843215942383,
+      "logps/chosen": -30.099853515625,
+      "logps/rejected": -31.55409812927246,
+      "loss": 0.9992,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": 0.0011685896897688508,
+      "rewards/margins": 0.000760918774176389,
+      "rewards/rejected": 0.00040767071186564863,
       "step": 220
     },
     {
       "epoch": 0.6,
       "learning_rate": 2.093227910899832e-07,
-      "logits/chosen": -1.983096718788147,
-      "logits/rejected": -1.9870779514312744,
-      "logps/chosen": -33.37377166748047,
-      "logps/rejected": -31.558422088623047,
-      "loss": 0.9995,
-      "rewards/accuracies": 0.574999988079071,
-      "rewards/chosen": 0.0004369509988464415,
-      "rewards/margins": 0.0005281810299493372,
-      "rewards/rejected": -9.123003837885335e-05,
+      "logits/chosen": -1.9835479259490967,
+      "logits/rejected": -1.9875112771987915,
+      "logps/chosen": -33.387638092041016,
+      "logps/rejected": -31.554845809936523,
+      "loss": 0.9965,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.0029833675362169743,
+      "rewards/margins": 0.003537458134815097,
+      "rewards/rejected": -0.0005540908314287663,
       "step": 230
     },
     {
       "epoch": 0.62,
       "learning_rate": 1.8712423238279356e-07,
-      "logits/chosen": -1.9832874536514282,
-      "logits/rejected": -1.9613001346588135,
-      "logps/chosen": -34.15814971923828,
-      "logps/rejected": -34.962947845458984,
-      "loss": 0.9999,
-      "rewards/accuracies": 0.5,
-      "rewards/chosen": -4.397642987896688e-05,
-      "rewards/margins": 7.76300803408958e-05,
-      "rewards/rejected": -0.00012160651385784149,
+      "logits/chosen": -1.9826898574829102,
+      "logits/rejected": -1.9607274532318115,
+      "logps/chosen": -34.158443450927734,
+      "logps/rejected": -34.963783264160156,
+      "loss": 0.9992,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": -0.00046964577632024884,
+      "rewards/margins": 0.0008290981058962643,
+      "rewards/rejected": -0.0012987437658011913,
       "step": 240
     },
     {
       "epoch": 0.65,
       "learning_rate": 1.654436768970182e-07,
-      "logits/chosen": -2.024568557739258,
-      "logits/rejected": -2.021247386932373,
-      "logps/chosen": -32.91596221923828,
-      "logps/rejected": -36.22909164428711,
-      "loss": 0.9999,
-      "rewards/accuracies": 0.48750001192092896,
-      "rewards/chosen": 9.10547751118429e-05,
-      "rewards/margins": 0.00013272995420265943,
-      "rewards/rejected": -4.1675208194646984e-05,
+      "logits/chosen": -2.024381160736084,
+      "logits/rejected": -2.0210862159729004,
+      "logps/chosen": -32.9254035949707,
+      "logps/rejected": -36.251712799072266,
+      "loss": 0.9974,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -3.328696038806811e-05,
+      "rewards/margins": 0.0026453982573002577,
+      "rewards/rejected": -0.0026786853559315205,
       "step": 250
     },
     {
       "epoch": 0.68,
       "learning_rate": 1.444597403062196e-07,
-      "logits/chosen": -1.891326904296875,
-      "logits/rejected": -1.8888810873031616,
-      "logps/chosen": -34.165008544921875,
-      "logps/rejected": -35.511131286621094,
-      "loss": 0.9997,
-      "rewards/accuracies": 0.550000011920929,
-      "rewards/chosen": 0.00023390687420032918,
-      "rewards/margins": 0.00029896467458456755,
-      "rewards/rejected": -6.505780766019598e-05,
+      "logits/chosen": -1.8911311626434326,
+      "logits/rejected": -1.8886839151382446,
+      "logps/chosen": -34.194557189941406,
+      "logps/rejected": -35.51445770263672,
+      "loss": 0.9996,
+      "rewards/accuracies": 0.48750001192092896,
+      "rewards/chosen": -0.0006158108590170741,
+      "rewards/margins": 0.0003673938917927444,
+      "rewards/rejected": -0.0009832048090174794,
       "step": 260
     },
     {
       "epoch": 0.7,
       "learning_rate": 1.2434529917578887e-07,
-      "logits/chosen": -1.8756563663482666,
-      "logits/rejected": -1.87309992313385,
-      "logps/chosen": -34.39522171020508,
-      "logps/rejected": -31.7664852142334,
-      "loss": 1.0,
-      "rewards/accuracies": 0.44999998807907104,
-      "rewards/chosen": 2.3789130864315666e-05,
-      "rewards/margins": -2.4663497242727317e-05,
-      "rewards/rejected": 4.8452602641191334e-05,
+      "logits/chosen": -1.8759450912475586,
+      "logits/rejected": -1.8734045028686523,
+      "logps/chosen": -34.40558624267578,
+      "logps/rejected": -31.752349853515625,
+      "loss": 1.0027,
+      "rewards/accuracies": 0.4625000059604645,
+      "rewards/chosen": -0.0007984804688021541,
+      "rewards/margins": -0.0026967611629515886,
+      "rewards/rejected": 0.0018982805777341127,
       "step": 270
     },
     {
       "epoch": 0.73,
       "learning_rate": 1.0526606671603521e-07,
-      "logits/chosen": -1.9794845581054688,
-      "logits/rejected": -1.9688661098480225,
-      "logps/chosen": -35.318504333496094,
-      "logps/rejected": -31.839763641357422,
-      "loss": 0.9996,
+      "logits/chosen": -1.980015754699707,
+      "logits/rejected": -1.9693737030029297,
+      "logps/chosen": -35.33230209350586,
+      "logps/rejected": -31.845691680908203,
+      "loss": 0.9971,
       "rewards/accuracies": 0.512499988079071,
-      "rewards/chosen": 0.0003813380899373442,
-      "rewards/margins": 0.0003690699231810868,
-      "rewards/rejected": 1.2268178579688538e-05,
+      "rewards/chosen": 0.002433922840282321,
+      "rewards/margins": 0.0029038069769740105,
+      "rewards/rejected": -0.0004698836710304022,
       "step": 280
     },
     {
       "epoch": 0.75,
       "learning_rate": 8.737922755071453e-08,
-      "logits/chosen": -2.075482130050659,
-      "logits/rejected": -2.060464859008789,
-      "logps/chosen": -30.921621322631836,
-      "logps/rejected": -32.64299774169922,
-      "loss": 1.0001,
-      "rewards/accuracies": 0.48750001192092896,
-      "rewards/chosen": 2.502280767657794e-05,
-      "rewards/margins": -6.332028715405613e-05,
-      "rewards/rejected": 8.834306936478242e-05,
+      "logits/chosen": -2.0756278038024902,
+      "logits/rejected": -2.060606002807617,
+      "logps/chosen": -30.907390594482422,
+      "logps/rejected": -32.64055252075195,
+      "loss": 0.9995,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.001673347083851695,
+      "rewards/margins": 0.0005450047319754958,
+      "rewards/rejected": 0.0011283422354608774,
       "step": 290
     },
     {
       "epoch": 0.78,
       "learning_rate": 7.08321427484816e-08,
-      "logits/chosen": -1.9463533163070679,
-      "logits/rejected": -1.9438135623931885,
-      "logps/chosen": -32.896881103515625,
-      "logps/rejected": -30.816125869750977,
-      "loss": 0.9997,
-      "rewards/accuracies": 0.5874999761581421,
-      "rewards/chosen": 0.0003022877499461174,
-      "rewards/margins": 0.0002954433439299464,
-      "rewards/rejected": 6.84439146425575e-06,
+      "logits/chosen": -1.946616768836975,
+      "logits/rejected": -1.9440828561782837,
+      "logps/chosen": -32.894561767578125,
+      "logps/rejected": -30.812387466430664,
+      "loss": 0.9972,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": 0.003255209419876337,
+      "rewards/margins": 0.002812772523611784,
+      "rewards/rejected": 0.00044243651791475713,
       "step": 300
     },
     {
       "epoch": 0.78,
-      "eval_logits/chosen": -2.243636131286621,
-      "eval_logits/rejected": -2.2387540340423584,
-      "eval_logps/chosen": -34.02006530761719,
-      "eval_logps/rejected": -37.50526428222656,
-      "eval_loss": 0.9999700784683228,
-      "eval_rewards/accuracies": 0.4987541437149048,
-      "eval_rewards/chosen": 0.0001448561524739489,
-      "eval_rewards/margins": 3.130395998596214e-05,
-      "eval_rewards/rejected": 0.00011355219612596557,
-      "eval_runtime": 145.6582,
-      "eval_samples_per_second": 2.355,
+      "eval_logits/chosen": -2.2434821128845215,
+      "eval_logits/rejected": -2.2386035919189453,
+      "eval_logps/chosen": -34.017669677734375,
+      "eval_logps/rejected": -37.50018310546875,
+      "eval_loss": 0.9999711513519287,
+      "eval_rewards/accuracies": 0.490448534488678,
+      "eval_rewards/chosen": 0.0016884810756891966,
+      "eval_rewards/margins": 4.470993735594675e-05,
+      "eval_rewards/rejected": 0.001643771305680275,
+      "eval_runtime": 145.8271,
+      "eval_samples_per_second": 2.352,
       "eval_steps_per_second": 0.295,
       "step": 300
     },
     {
       "epoch": 0.81,
       "learning_rate": 5.576113578589034e-08,
-      "logits/chosen": -1.9288015365600586,
-      "logits/rejected": -1.9255383014678955,
-      "logps/chosen": -31.57794189453125,
-      "logps/rejected": -33.73435592651367,
-      "loss": 0.9998,
-      "rewards/accuracies": 0.574999988079071,
-      "rewards/chosen": 0.000254074577242136,
-      "rewards/margins": 0.00017612635565456003,
-      "rewards/rejected": 7.794820703566074e-05,
+      "logits/chosen": -1.9287067651748657,
+      "logits/rejected": -1.925451636314392,
+      "logps/chosen": -31.603496551513672,
+      "logps/rejected": -33.734046936035156,
+      "loss": 1.0008,
+      "rewards/accuracies": 0.4749999940395355,
+      "rewards/chosen": -1.4918483429937623e-05,
+      "rewards/margins": -0.000825587019789964,
+      "rewards/rejected": 0.0008106685127131641,
       "step": 310
     },
     {
       "epoch": 0.83,
       "learning_rate": 4.229036944380912e-08,
-      "logits/chosen": -1.9808727502822876,
-      "logits/rejected": -1.9685646295547485,
-      "logps/chosen": -34.54682159423828,
-      "logps/rejected": -33.555274963378906,
-      "loss": 0.9993,
+      "logits/chosen": -1.9802377223968506,
+      "logits/rejected": -1.9679291248321533,
+      "logps/chosen": -34.585323333740234,
+      "logps/rejected": -33.57084274291992,
+      "loss": 0.9957,
       "rewards/accuracies": 0.5375000238418579,
-      "rewards/chosen": 0.0004924900131300092,
-      "rewards/margins": 0.0006568533135578036,
-      "rewards/rejected": -0.00016436330042779446,
+      "rewards/chosen": 0.00107504369225353,
+      "rewards/margins": 0.0042753927409648895,
+      "rewards/rejected": -0.0032003491651266813,
       "step": 320
     },
     {
       "epoch": 0.86,
       "learning_rate": 3.053082288996112e-08,
-      "logits/chosen": -2.0161168575286865,
-      "logits/rejected": -2.0146450996398926,
-      "logps/chosen": -33.46837615966797,
-      "logps/rejected": -32.443458557128906,
-      "loss": 1.0002,
-      "rewards/accuracies": 0.48750001192092896,
-      "rewards/chosen": 0.00017420158837921917,
-      "rewards/margins": -0.00018708205607254058,
-      "rewards/rejected": 0.00036128362989984453,
+      "logits/chosen": -2.015906810760498,
+      "logits/rejected": -2.014427661895752,
+      "logps/chosen": -33.49116516113281,
+      "logps/rejected": -32.47978973388672,
+      "loss": 1.0005,
+      "rewards/accuracies": 0.4375,
+      "rewards/chosen": -0.0005370675935409963,
+      "rewards/margins": -0.0005161463050171733,
+      "rewards/rejected": -2.0921288523823023e-05,
       "step": 330
     },
     {
       "epoch": 0.88,
       "learning_rate": 2.05793773749158e-08,
-      "logits/chosen": -2.102954626083374,
-      "logits/rejected": -2.087148666381836,
-      "logps/chosen": -34.159751892089844,
-      "logps/rejected": -33.093017578125,
-      "loss": 1.0001,
-      "rewards/accuracies": 0.512499988079071,
-      "rewards/chosen": 0.00019638515368569642,
-      "rewards/margins": -5.3783714974997565e-05,
-      "rewards/rejected": 0.0002501688722986728,
+      "logits/chosen": -2.1030631065368652,
+      "logits/rejected": -2.0872654914855957,
+      "logps/chosen": -34.18492889404297,
+      "logps/rejected": -33.08319854736328,
+      "loss": 1.004,
+      "rewards/accuracies": 0.42500001192092896,
+      "rewards/chosen": -0.0005535687087103724,
+      "rewards/margins": -0.00403733691200614,
+      "rewards/rejected": 0.003483767854049802,
       "step": 340
     },
     {
       "epoch": 0.91,
       "learning_rate": 1.251801807404168e-08,
-      "logits/chosen": -1.974347710609436,
-      "logits/rejected": -1.9733997583389282,
-      "logps/chosen": -33.24498748779297,
-      "logps/rejected": -32.44942855834961,
-      "loss": 0.9998,
-      "rewards/accuracies": 0.48750001192092896,
-      "rewards/chosen": 0.00037107508978806436,
-      "rewards/margins": 0.0002138734853360802,
-      "rewards/rejected": 0.00015720158990006894,
+      "logits/chosen": -1.9745471477508545,
+      "logits/rejected": -1.9736032485961914,
+      "logps/chosen": -33.23271942138672,
+      "logps/rejected": -32.4765510559082,
+      "loss": 0.9939,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": 0.004937793128192425,
+      "rewards/margins": 0.006078074686229229,
+      "rewards/rejected": -0.001140281674452126,
       "step": 350
     },
     {
       "epoch": 0.94,
       "learning_rate": 6.41315865106129e-09,
-      "logits/chosen": -1.9305756092071533,
-      "logits/rejected": -1.9409431219100952,
-      "logps/chosen": -32.20996856689453,
-      "logps/rejected": -35.26984405517578,
-      "loss": 1.0002,
-      "rewards/accuracies": 0.4749999940395355,
-      "rewards/chosen": -5.276153751765378e-05,
-      "rewards/margins": -0.00024183676578104496,
-      "rewards/rejected": 0.00018907521734945476,
+      "logits/chosen": -1.9305438995361328,
+      "logits/rejected": -1.940913438796997,
+      "logps/chosen": -32.22040939331055,
+      "logps/rejected": -35.28728103637695,
+      "loss": 1.0017,
+      "rewards/accuracies": 0.48750001192092896,
+      "rewards/chosen": -0.0015721082454547286,
+      "rewards/margins": -0.001719816355034709,
+      "rewards/rejected": 0.00014770813868381083,
       "step": 360
     },
     {
       "epoch": 0.96,
       "learning_rate": 2.3150941078050324e-09,
-      "logits/chosen": -2.0696487426757812,
-      "logits/rejected": -2.0630972385406494,
-      "logps/chosen": -33.643394470214844,
-      "logps/rejected": -29.242889404296875,
-      "loss": 0.9997,
-      "rewards/accuracies": 0.550000011920929,
-      "rewards/chosen": 8.579110726714134e-05,
-      "rewards/margins": 0.0002504353760741651,
-      "rewards/rejected": -0.00016464429791085422,
+      "logits/chosen": -2.069648265838623,
+      "logits/rejected": -2.0630898475646973,
+      "logps/chosen": -33.63695526123047,
+      "logps/rejected": -29.226470947265625,
+      "loss": 0.9985,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": 0.0015011827927082777,
+      "rewards/margins": 0.0015057541895657778,
+      "rewards/rejected": -4.571699719235767e-06,
       "step": 370
     },
     {
       "epoch": 0.99,
       "learning_rate": 2.575864278703266e-10,
-      "logits/chosen": -1.9290872812271118,
-      "logits/rejected": -1.9312543869018555,
-      "logps/chosen": -34.22844696044922,
-      "logps/rejected": -30.902618408203125,
-      "loss": 0.9996,
-      "rewards/accuracies": 0.5874999761581421,
-      "rewards/chosen": 0.00010604139242786914,
-      "rewards/margins": 0.0004059396160300821,
-      "rewards/rejected": -0.00029989826725795865,
+      "logits/chosen": -1.928865671157837,
+      "logits/rejected": -1.9310123920440674,
+      "logps/chosen": -34.243560791015625,
+      "logps/rejected": -30.892742156982422,
+      "loss": 0.9984,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -0.0004508168494794518,
+      "rewards/margins": 0.0015608124667778611,
+      "rewards/rejected": -0.0020116292871534824,
       "step": 380
     },
     {
       "epoch": 1.0,
       "step": 385,
       "total_flos": 0.0,
-      "train_loss": 0.9998476387618425,
-      "train_runtime": 3251.6099,
-      "train_samples_per_second": 0.947,
+      "train_loss": 0.9991211135666092,
+      "train_runtime": 3253.9902,
+      "train_samples_per_second": 0.946,
       "train_steps_per_second": 0.118
     }
   ],