Model save

README.md

@@ -1,13 +1,10 @@
 ---
 library_name: peft
 tags:
-- alignment-handbook
 - trl
 - dpo
 - generated_from_trainer
 base_model: NbAiLab/nb-gpt-j-6B-v2
-datasets:
-- hugodk-sch/aftonposten_title_prefs
 model-index:
 - name: aftonposten-6b-align-scan
   results: []
@@ -18,17 +15,17 @@ should probably proofread and complete it, then remove this comment. -->
 
 # aftonposten-6b-align-scan
 
-This model is a fine-tuned version of [data/ap-gpt-j-6b-sft-qlora-04-08](https://huggingface.co/data/ap-gpt-j-6b-sft-qlora-04-08) on the hugodk-sch/aftonposten_title_prefs dataset.
+This model is a fine-tuned version of [NbAiLab/nb-gpt-j-6B-v2](https://huggingface.co/NbAiLab/nb-gpt-j-6B-v2) on an unknown dataset.
 It achieves the following results on the evaluation set:
-- Loss: 0.3447
-- Rewards/chosen: 0.0141
-- Rewards/rejected: 0.0084
-- Rewards/accuracies: 0.5162
-- Rewards/margins: 0.0057
-- Logps/rejected: -37.5073
-- Logps/chosen: -34.0189
+- Loss: 1.0000
+- Rewards/chosen: 0.0001
+- Rewards/rejected: 0.0001
+- Rewards/accuracies: 0.4988
+- Rewards/margins: 0.0000
+- Logps/rejected: -37.5053
+- Logps/chosen: -34.0201
 - Logits/rejected: -2.2388
-- Logits/chosen: -2.2437
+- Logits/chosen: -2.2436
 
 ## Model description
 
@@ -63,9 +60,9 @@ The following hyperparameters were used during training:
 
 | Training Loss | Epoch | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
 |:-------------:|:-----:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
-| 0.333         | 0.26  | 100  | 0.3405          | 0.0126         | 0.0118           | 0.5104             | 0.0008          | -37.5035       | -34.0205     | -2.2389         | -2.2438       |
-| 0.3306        | 0.52  | 200  | 0.3466          | 0.0054         | 0.0048           | 0.5021             | 0.0007          | -37.5113       | -34.0285     | -2.2386         | -2.2435       |
-| 0.3327        | 0.78  | 300  | 0.3276          | 0.0152         | 0.0022           | 0.5365             | 0.0131          | -37.5142       | -34.0176     | -2.2390         | -2.2439       |
+| 1.0001        | 0.26  | 100  | 1.0000          | 0.0000         | 0.0001           | 0.5100             | -0.0000         | -37.5080       | -34.0296     | -2.2392         | -2.2441       |
+| 0.9998        | 0.52  | 200  | 1.0000          | 0.0001         | 0.0001           | 0.5046             | -0.0000         | -37.5040       | -34.0221     | -2.2392         | -2.2440       |
+| 0.9997        | 0.78  | 300  | 1.0000          | 0.0001         | 0.0001           | 0.4988             | 0.0000          | -37.5053       | -34.0201     | -2.2388         | -2.2436       |
 
 
 ### Framework versions

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:fbd37e038d13d252dc8fbbff90393b02ad5b1a3acf3aaf4a91b929c037ba71a7
+oid sha256:10d3f20ad7c4aec2449ad1b4fd6692acb1c515aef929e5d69011238fb991cc39
 size 176183216

all_results.json

@@ -1,21 +1,8 @@
 {
     "epoch": 1.0,
-    "eval_logits/chosen": -2.2436861991882324,
-    "eval_logits/rejected": -2.2388129234313965,
-    "eval_logps/chosen": -34.01886749267578,
-    "eval_logps/rejected": -37.50727081298828,
-    "eval_loss": 0.34474730491638184,
-    "eval_rewards/accuracies": 0.5161960124969482,
-    "eval_rewards/chosen": 0.01411630492657423,
-    "eval_rewards/margins": 0.005696756765246391,
-    "eval_rewards/rejected": 0.008419547230005264,
-    "eval_runtime": 145.8183,
-    "eval_samples": 343,
-    "eval_samples_per_second": 2.352,
-    "eval_steps_per_second": 0.295,
-    "train_loss": 0.3306902726749321,
-    "train_runtime": 3256.1932,
+    "train_loss": 0.9998476387618425,
+    "train_runtime": 3251.6099,
     "train_samples": 3079,
-    "train_samples_per_second": 0.946,
+    "train_samples_per_second": 0.947,
     "train_steps_per_second": 0.118
 }

train_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 1.0,
-    "train_loss": 0.3306902726749321,
-    "train_runtime": 3256.1932,
+    "train_loss": 0.9998476387618425,
+    "train_runtime": 3251.6099,
     "train_samples": 3079,
-    "train_samples_per_second": 0.946,
+    "train_samples_per_second": 0.947,
     "train_steps_per_second": 0.118
 }

trainer_state.json

@@ -15,7 +15,7 @@
       "logits/rejected": -1.7377450466156006,
       "logps/chosen": -29.553977966308594,
       "logps/rejected": -42.813133239746094,
-      "loss": 0.3086,
+      "loss": 1.0,
       "rewards/accuracies": 0.0,
       "rewards/chosen": 0.0,
       "rewards/margins": 0.0,
@@ -25,590 +25,590 @@
     {
       "epoch": 0.03,
       "learning_rate": 1.2820512820512818e-07,
-      "logits/chosen": -1.866453766822815,
-      "logits/rejected": -1.8707627058029175,
-      "logps/chosen": -36.98646545410156,
-      "logps/rejected": -33.6445426940918,
-      "loss": 0.3098,
-      "rewards/accuracies": 0.4444444477558136,
-      "rewards/chosen": 0.018122652545571327,
-      "rewards/margins": 0.031144678592681885,
-      "rewards/rejected": -0.013022023253142834,
+      "logits/chosen": -1.8663256168365479,
+      "logits/rejected": -1.8706417083740234,
+      "logps/chosen": -36.98673629760742,
+      "logps/rejected": -33.66697311401367,
+      "loss": 0.9994,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": 0.00019861094187945127,
+      "rewards/margins": 0.0005675733555108309,
+      "rewards/rejected": -0.0003689624136313796,
       "step": 10
     },
     {
       "epoch": 0.05,
       "learning_rate": 2.5641025641025636e-07,
-      "logits/chosen": -1.9970738887786865,
-      "logits/rejected": -1.9997297525405884,
-      "logps/chosen": -29.651851654052734,
-      "logps/rejected": -29.048221588134766,
-      "loss": 0.3674,
-      "rewards/accuracies": 0.4375,
-      "rewards/chosen": -0.008690872229635715,
-      "rewards/margins": -0.02239001728594303,
-      "rewards/rejected": 0.01369914598762989,
+      "logits/chosen": -1.997527837753296,
+      "logits/rejected": -2.0001606941223145,
+      "logps/chosen": -29.65478515625,
+      "logps/rejected": -29.070556640625,
+      "loss": 1.0001,
+      "rewards/accuracies": 0.44999998807907104,
+      "rewards/chosen": -0.00012590530968736857,
+      "rewards/margins": -5.477964805322699e-05,
+      "rewards/rejected": -7.112567982403561e-05,
       "step": 20
     },
     {
       "epoch": 0.08,
       "learning_rate": 3.8461538461538463e-07,
-      "logits/chosen": -1.919904351234436,
-      "logits/rejected": -1.9172093868255615,
-      "logps/chosen": -31.407363891601562,
-      "logps/rejected": -33.22914505004883,
-      "loss": 0.3289,
-      "rewards/accuracies": 0.5874999761581421,
-      "rewards/chosen": 0.007855755276978016,
-      "rewards/margins": 0.017470674589276314,
-      "rewards/rejected": -0.009614917449653149,
+      "logits/chosen": -1.9195702075958252,
+      "logits/rejected": -1.9168812036514282,
+      "logps/chosen": -31.421009063720703,
+      "logps/rejected": -33.225616455078125,
+      "loss": 1.0,
+      "rewards/accuracies": 0.48750001192092896,
+      "rewards/chosen": -4.912544318358414e-05,
+      "rewards/margins": 2.244002462248318e-05,
+      "rewards/rejected": -7.15654605301097e-05,
       "step": 30
     },
     {
       "epoch": 0.1,
       "learning_rate": 4.999896948438433e-07,
-      "logits/chosen": -2.016488552093506,
-      "logits/rejected": -2.0077414512634277,
-      "logps/chosen": -32.588050842285156,
-      "logps/rejected": -32.51618576049805,
-      "loss": 0.3526,
-      "rewards/accuracies": 0.4749999940395355,
-      "rewards/chosen": -0.010120267979800701,
-      "rewards/margins": -0.006958496756851673,
-      "rewards/rejected": -0.0031617730855941772,
+      "logits/chosen": -2.0165441036224365,
+      "logits/rejected": -2.0078024864196777,
+      "logps/chosen": -32.57805252075195,
+      "logps/rejected": -32.504085540771484,
+      "loss": 1.0001,
+      "rewards/accuracies": 0.4375,
+      "rewards/chosen": -1.2443718333088327e-05,
+      "rewards/margins": -9.829048212850466e-05,
+      "rewards/rejected": 8.584677561884746e-05,
       "step": 40
     },
     {
       "epoch": 0.13,
       "learning_rate": 4.987541037542186e-07,
-      "logits/chosen": -1.8645082712173462,
-      "logits/rejected": -1.8537276983261108,
-      "logps/chosen": -33.5635871887207,
-      "logps/rejected": -35.42169952392578,
-      "loss": 0.3858,
-      "rewards/accuracies": 0.38749998807907104,
-      "rewards/chosen": -0.005497059319168329,
-      "rewards/margins": -0.03307711333036423,
-      "rewards/rejected": 0.027580058202147484,
+      "logits/chosen": -1.8641563653945923,
+      "logits/rejected": -1.8533931970596313,
+      "logps/chosen": -33.555824279785156,
+      "logps/rejected": -35.42582321166992,
+      "loss": 1.0002,
+      "rewards/accuracies": 0.36250001192092896,
+      "rewards/chosen": 1.6570575098739937e-05,
+      "rewards/margins": -0.0002486651937942952,
+      "rewards/rejected": 0.000265235808910802,
       "step": 50
     },
     {
       "epoch": 0.16,
       "learning_rate": 4.954691471941118e-07,
-      "logits/chosen": -1.9458835124969482,
-      "logits/rejected": -1.9478065967559814,
-      "logps/chosen": -32.57460021972656,
-      "logps/rejected": -33.195518493652344,
-      "loss": 0.3015,
-      "rewards/accuracies": 0.574999988079071,
-      "rewards/chosen": 0.02302517369389534,
-      "rewards/margins": 0.04474987834692001,
-      "rewards/rejected": -0.021724706515669823,
+      "logits/chosen": -1.9452263116836548,
+      "logits/rejected": -1.9471622705459595,
+      "logps/chosen": -32.5916633605957,
+      "logps/rejected": -33.1885871887207,
+      "loss": 0.9997,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": 8.520439587300643e-05,
+      "rewards/margins": 0.00025727329193614423,
+      "rewards/rejected": -0.0001720689469948411,
       "step": 60
     },
     {
       "epoch": 0.18,
       "learning_rate": 4.901618883413548e-07,
-      "logits/chosen": -2.0793328285217285,
-      "logits/rejected": -2.084324836730957,
-      "logps/chosen": -33.99799728393555,
-      "logps/rejected": -36.57799530029297,
-      "loss": 0.3658,
-      "rewards/accuracies": 0.48750001192092896,
-      "rewards/chosen": -0.00855917576700449,
-      "rewards/margins": -0.00035296304849907756,
-      "rewards/rejected": -0.008206211030483246,
+      "logits/chosen": -2.0798425674438477,
+      "logits/rejected": -2.08483624458313,
+      "logps/chosen": -33.98626708984375,
+      "logps/rejected": -36.57551956176758,
+      "loss": 0.9999,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": 2.217638393631205e-05,
+      "rewards/margins": 8.864814299158752e-05,
+      "rewards/rejected": -6.647173722740263e-05,
       "step": 70
     },
     {
       "epoch": 0.21,
       "learning_rate": 4.828760511501322e-07,
-      "logits/chosen": -1.9421542882919312,
-      "logits/rejected": -1.9453080892562866,
-      "logps/chosen": -34.38970184326172,
-      "logps/rejected": -34.58138656616211,
-      "loss": 0.3181,
-      "rewards/accuracies": 0.5874999761581421,
-      "rewards/chosen": 0.034300822764635086,
-      "rewards/margins": 0.024381538853049278,
-      "rewards/rejected": 0.009919276461005211,
+      "logits/chosen": -1.942161202430725,
+      "logits/rejected": -1.9453051090240479,
+      "logps/chosen": -34.400901794433594,
+      "logps/rejected": -34.57304382324219,
+      "loss": 0.9999,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": 0.00026908880681730807,
+      "rewards/margins": 7.544312393292785e-05,
+      "rewards/rejected": 0.000193645668332465,
       "step": 80
     },
     {
       "epoch": 0.23,
       "learning_rate": 4.736716601303429e-07,
-      "logits/chosen": -1.9507806301116943,
-      "logits/rejected": -1.955293893814087,
-      "logps/chosen": -32.461936950683594,
-      "logps/rejected": -32.3816032409668,
-      "loss": 0.3094,
-      "rewards/accuracies": 0.574999988079071,
-      "rewards/chosen": 0.004934861324727535,
-      "rewards/margins": 0.02535998821258545,
-      "rewards/rejected": -0.02042512409389019,
+      "logits/chosen": -1.9505256414413452,
+      "logits/rejected": -1.9550361633300781,
+      "logps/chosen": -32.452125549316406,
+      "logps/rejected": -32.35675811767578,
+      "loss": 0.9999,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": 0.00015296679339371622,
+      "rewards/margins": 0.0001315164554398507,
+      "rewards/rejected": 2.145032885891851e-05,
       "step": 90
     },
     {
       "epoch": 0.26,
       "learning_rate": 4.62624545834521e-07,
-      "logits/chosen": -2.0485286712646484,
-      "logits/rejected": -2.046539783477783,
-      "logps/chosen": -32.23391342163086,
-      "logps/rejected": -31.265544891357422,
-      "loss": 0.333,
-      "rewards/accuracies": 0.512499988079071,
-      "rewards/chosen": 0.006228716112673283,
-      "rewards/margins": 0.005629505030810833,
-      "rewards/rejected": 0.0005992107326164842,
+      "logits/chosen": -2.0490074157714844,
+      "logits/rejected": -2.0470099449157715,
+      "logps/chosen": -32.25472640991211,
+      "logps/rejected": -31.2731990814209,
+      "loss": 1.0001,
+      "rewards/accuracies": 0.48750001192092896,
+      "rewards/chosen": -0.00013893461436964571,
+      "rewards/margins": -6.904623296577483e-05,
+      "rewards/rejected": -6.98883886798285e-05,
       "step": 100
     },
     {
       "epoch": 0.26,
-      "eval_logits/chosen": -2.243820905685425,
-      "eval_logits/rejected": -2.238938570022583,
-      "eval_logps/chosen": -34.02050018310547,
-      "eval_logps/rejected": -37.50347137451172,
-      "eval_loss": 0.3404548168182373,
-      "eval_rewards/accuracies": 0.5103820562362671,
-      "eval_rewards/chosen": 0.012649022974073887,
-      "eval_rewards/margins": 0.0008147989865392447,
-      "eval_rewards/rejected": 0.011834223754703999,
-      "eval_runtime": 146.2563,
-      "eval_samples_per_second": 2.345,
-      "eval_steps_per_second": 0.294,
+      "eval_logits/chosen": -2.2441036701202393,
+      "eval_logits/rejected": -2.239219903945923,
+      "eval_logps/chosen": -34.02959442138672,
+      "eval_logps/rejected": -37.50800704956055,
+      "eval_loss": 1.0000391006469727,
+      "eval_rewards/accuracies": 0.5099667906761169,
+      "eval_rewards/chosen": 4.9558610044186935e-05,
+      "eval_rewards/margins": -3.6566227208822966e-05,
+      "eval_rewards/rejected": 8.61248336150311e-05,
+      "eval_runtime": 145.9645,
+      "eval_samples_per_second": 2.35,
+      "eval_steps_per_second": 0.295,
       "step": 100
     },
     {
       "epoch": 0.29,
       "learning_rate": 4.4982572012636904e-07,
-      "logits/chosen": -2.0055313110351562,
-      "logits/rejected": -2.00311541557312,
-      "logps/chosen": -33.23518753051758,
-      "logps/rejected": -34.03091812133789,
-      "loss": 0.3233,
-      "rewards/accuracies": 0.48750001192092896,
-      "rewards/chosen": 0.007764431647956371,
-      "rewards/margins": 0.009159245528280735,
-      "rewards/rejected": -0.0013948131818324327,
+      "logits/chosen": -2.005286931991577,
+      "logits/rejected": -2.0028929710388184,
+      "logps/chosen": -33.22523498535156,
+      "logps/rejected": -34.039817810058594,
+      "loss": 0.9997,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": 0.00018582009943202138,
+      "rewards/margins": 0.0002903031127061695,
+      "rewards/rejected": -0.00010448300599819049,
       "step": 110
     },
     {
       "epoch": 0.31,
       "learning_rate": 4.353806263777677e-07,
-      "logits/chosen": -2.016556978225708,
-      "logits/rejected": -2.0081825256347656,
-      "logps/chosen": -32.435211181640625,
-      "logps/rejected": -32.18501281738281,
-      "loss": 0.3121,
-      "rewards/accuracies": 0.5625,
-      "rewards/chosen": 0.00872363243252039,
-      "rewards/margins": 0.029877260327339172,
-      "rewards/rejected": -0.02115362510085106,
+      "logits/chosen": -2.0164072513580322,
+      "logits/rejected": -2.008033514022827,
+      "logps/chosen": -32.473182678222656,
+      "logps/rejected": -32.18474197387695,
+      "loss": 1.0001,
+      "rewards/accuracies": 0.4625000059604645,
+      "rewards/chosen": -0.00028277214732952416,
+      "rewards/margins": -5.0430990086169913e-05,
+      "rewards/rejected": -0.00023234116088133305,
       "step": 120
     },
     {
       "epoch": 0.34,
       "learning_rate": 4.194082707715275e-07,
-      "logits/chosen": -2.0461266040802,
-      "logits/rejected": -2.03806734085083,
-      "logps/chosen": -30.502710342407227,
-      "logps/rejected": -32.0546760559082,
-      "loss": 0.3674,
-      "rewards/accuracies": 0.4625000059604645,
-      "rewards/chosen": -0.02429451420903206,
-      "rewards/margins": -0.02279272861778736,
-      "rewards/rejected": -0.0015017850091680884,
+      "logits/chosen": -2.0463318824768066,
+      "logits/rejected": -2.03826642036438,
+      "logps/chosen": -30.479427337646484,
+      "logps/rejected": -32.030479431152344,
+      "loss": 1.0003,
+      "rewards/accuracies": 0.4000000059604645,
+      "rewards/chosen": -3.712148100021295e-05,
+      "rewards/margins": -0.00026237856945954263,
+      "rewards/rejected": 0.0002252570993732661,
       "step": 130
     },
     {
       "epoch": 0.36,
       "learning_rate": 4.020402418666621e-07,
-      "logits/chosen": -1.9767634868621826,
-      "logits/rejected": -1.9870494604110718,
-      "logps/chosen": -31.402328491210938,
-      "logps/rejected": -32.55817794799805,
-      "loss": 0.3125,
-      "rewards/accuracies": 0.550000011920929,
-      "rewards/chosen": 0.015601096674799919,
-      "rewards/margins": 0.03657008707523346,
-      "rewards/rejected": -0.02096899226307869,
+      "logits/chosen": -1.9770317077636719,
+      "logits/rejected": -1.987318754196167,
+      "logps/chosen": -31.41189193725586,
+      "logps/rejected": -32.552860260009766,
+      "loss": 0.9997,
+      "rewards/accuracies": 0.4625000059604645,
+      "rewards/chosen": 7.772201206535101e-05,
+      "rewards/margins": 0.000257515290286392,
+      "rewards/rejected": -0.00017979330732487142,
       "step": 140
     },
     {
       "epoch": 0.39,
       "learning_rate": 3.8341962650351185e-07,
-      "logits/chosen": -1.8908436298370361,
-      "logits/rejected": -1.8919413089752197,
-      "logps/chosen": -34.207401275634766,
-      "logps/rejected": -34.783695220947266,
-      "loss": 0.3373,
-      "rewards/accuracies": 0.5375000238418579,
-      "rewards/chosen": -0.013235817663371563,
-      "rewards/margins": 0.01055879332125187,
-      "rewards/rejected": -0.023794615641236305,
+      "logits/chosen": -1.8904685974121094,
+      "logits/rejected": -1.8915725946426392,
+      "logps/chosen": -34.178749084472656,
+      "logps/rejected": -34.7522087097168,
+      "loss": 0.9999,
+      "rewards/accuracies": 0.44999998807907104,
+      "rewards/chosen": 0.00013941082579549402,
+      "rewards/margins": 8.894065831555054e-05,
+      "rewards/rejected": 5.047018566983752e-05,
       "step": 150
     },
     {
       "epoch": 0.42,
       "learning_rate": 3.636998309800572e-07,
-      "logits/chosen": -1.942582368850708,
-      "logits/rejected": -1.9391053915023804,
-      "logps/chosen": -36.15548324584961,
-      "logps/rejected": -32.729759216308594,
-      "loss": 0.3207,
-      "rewards/accuracies": 0.5,
-      "rewards/chosen": 0.014574882574379444,
-      "rewards/margins": 0.020880455151200294,
-      "rewards/rejected": -0.006305573042482138,
+      "logits/chosen": -1.9424184560775757,
+      "logits/rejected": -1.9389402866363525,
+      "logps/chosen": -36.14480209350586,
+      "logps/rejected": -32.753517150878906,
+      "loss": 0.9994,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": 0.0002687961678020656,
+      "rewards/margins": 0.0005764620145782828,
+      "rewards/rejected": -0.0003076658467762172,
       "step": 160
     },
     {
       "epoch": 0.44,
       "learning_rate": 3.430433172111807e-07,
-      "logits/chosen": -2.042206287384033,
-      "logits/rejected": -2.034825086593628,
-      "logps/chosen": -33.77631378173828,
-      "logps/rejected": -31.35869789123535,
-      "loss": 0.3201,
-      "rewards/accuracies": 0.5625,
-      "rewards/chosen": 0.019394155591726303,
-      "rewards/margins": 0.025556519627571106,
-      "rewards/rejected": -0.00616236450150609,
+      "logits/chosen": -2.0420711040496826,
+      "logits/rejected": -2.0346856117248535,
+      "logps/chosen": -33.783363342285156,
+      "logps/rejected": -31.349674224853516,
+      "loss": 0.9999,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": 0.00014495299546979368,
+      "rewards/margins": 0.00012318804510869086,
+      "rewards/rejected": 2.1765008568763733e-05,
       "step": 170
     },
     {
       "epoch": 0.47,
       "learning_rate": 3.216202642830543e-07,
-      "logits/chosen": -2.0474154949188232,
-      "logits/rejected": -2.0526936054229736,
-      "logps/chosen": -32.521339416503906,
-      "logps/rejected": -32.497581481933594,
-      "loss": 0.2999,
-      "rewards/accuracies": 0.612500011920929,
-      "rewards/chosen": 0.021103620529174805,
-      "rewards/margins": 0.032301973551511765,
-      "rewards/rejected": -0.01119835115969181,
+      "logits/chosen": -2.0475425720214844,
+      "logits/rejected": -2.052799940109253,
+      "logps/chosen": -32.50774383544922,
+      "logps/rejected": -32.48485565185547,
+      "loss": 0.9996,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.00037043425254523754,
+      "rewards/margins": 0.0003675749758258462,
+      "rewards/rejected": 2.859340838767821e-06,
       "step": 180
     },
     {
       "epoch": 0.49,
       "learning_rate": 2.9960716642946403e-07,
-      "logits/chosen": -2.0485098361968994,
-      "logits/rejected": -2.0457375049591064,
-      "logps/chosen": -31.49017906188965,
-      "logps/rejected": -31.345867156982422,
-      "loss": 0.3315,
+      "logits/chosen": -2.0484375953674316,
+      "logits/rejected": -2.045671224594116,
+      "logps/chosen": -31.50165367126465,
+      "logps/rejected": -31.33974266052246,
+      "loss": 0.9999,
       "rewards/accuracies": 0.5625,
-      "rewards/chosen": -0.001104071387089789,
-      "rewards/margins": 0.023781923577189445,
-      "rewards/rejected": -0.02488599717617035,
+      "rewards/chosen": -0.00012700485240202397,
+      "rewards/margins": 8.82898602867499e-05,
+      "rewards/rejected": -0.00021529469813685864,
       "step": 190
     },
     {
       "epoch": 0.52,
       "learning_rate": 2.771853789806683e-07,
-      "logits/chosen": -1.9183101654052734,
-      "logits/rejected": -1.9229921102523804,
-      "logps/chosen": -31.60622215270996,
-      "logps/rejected": -32.805049896240234,
-      "loss": 0.3306,
-      "rewards/accuracies": 0.5249999761581421,
-      "rewards/chosen": 5.159676220500842e-05,
-      "rewards/margins": 0.012757277116179466,
-      "rewards/rejected": -0.012705676257610321,
+      "logits/chosen": -1.9182907342910767,
+      "logits/rejected": -1.9229600429534912,
+      "logps/chosen": -31.5924072265625,
+      "logps/rejected": -32.794044494628906,
+      "loss": 0.9998,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": 0.00013877781748306006,
+      "rewards/margins": 0.0001699000713415444,
+      "rewards/rejected": -3.112227204837836e-05,
       "step": 200
     },
     {
       "epoch": 0.52,
-      "eval_logits/chosen": -2.2434937953948975,
-      "eval_logits/rejected": -2.2386233806610107,
-      "eval_logps/chosen": -34.02851867675781,
-      "eval_logps/rejected": -37.51131057739258,
-      "eval_loss": 0.3466452658176422,
-      "eval_rewards/accuracies": 0.5020764470100403,
-      "eval_rewards/chosen": 0.005428452976047993,
-      "eval_rewards/margins": 0.0006523749325424433,
-      "eval_rewards/rejected": 0.00477607874199748,
-      "eval_runtime": 145.8217,
-      "eval_samples_per_second": 2.352,
+      "eval_logits/chosen": -2.2440481185913086,
+      "eval_logits/rejected": -2.2391700744628906,
+      "eval_logps/chosen": -34.02214813232422,
+      "eval_logps/rejected": -37.503997802734375,
+      "eval_loss": 1.0000054836273193,
+      "eval_rewards/accuracies": 0.5045680999755859,
+      "eval_rewards/chosen": 0.00012402697757352144,
+      "eval_rewards/margins": -2.209126932939398e-06,
+      "eval_rewards/rejected": 0.00012623611837625504,
+      "eval_runtime": 145.6161,
+      "eval_samples_per_second": 2.356,
       "eval_steps_per_second": 0.295,
       "step": 200
     },
     {
       "epoch": 0.55,
       "learning_rate": 2.5453962426402e-07,
-      "logits/chosen": -2.032118558883667,
-      "logits/rejected": -2.042782783508301,
-      "logps/chosen": -31.939167022705078,
-      "logps/rejected": -33.88642501831055,
-      "loss": 0.2871,
-      "rewards/accuracies": 0.6000000238418579,
-      "rewards/chosen": 0.02892182394862175,
-      "rewards/margins": 0.050276655703783035,
-      "rewards/rejected": -0.021354828029870987,
+      "logits/chosen": -2.0312139987945557,
+      "logits/rejected": -2.041886568069458,
+      "logps/chosen": -31.938335418701172,
+      "logps/rejected": -33.87850570678711,
+      "loss": 0.9995,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.00032970920437946916,
+      "rewards/margins": 0.0004878062754869461,
+      "rewards/rejected": -0.00015809701289981604,
       "step": 210
     },
     {
       "epoch": 0.57,
       "learning_rate": 2.318564697655179e-07,
-      "logits/chosen": -1.925293207168579,
-      "logits/rejected": -1.9401395320892334,
-      "logps/chosen": -30.106586456298828,
-      "logps/rejected": -31.56791114807129,
-      "loss": 0.3327,
-      "rewards/accuracies": 0.6000000238418579,
-      "rewards/chosen": 0.0044566914439201355,
-      "rewards/margins": 0.013218318112194538,
-      "rewards/rejected": -0.008761629462242126,
+      "logits/chosen": -1.925606369972229,
+      "logits/rejected": -1.940452218055725,
+      "logps/chosen": -30.110549926757812,
+      "logps/rejected": -31.573543548583984,
+      "loss": 0.9998,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": 9.872082955553196e-06,
+      "rewards/margins": 0.00016354399849660695,
+      "rewards/rejected": -0.00015367190644610673,
       "step": 220
     },
     {
       "epoch": 0.6,
       "learning_rate": 2.093227910899832e-07,
-      "logits/chosen": -1.9831531047821045,
-      "logits/rejected": -1.9871160984039307,
-      "logps/chosen": -33.386024475097656,
-      "logps/rejected": -31.5727481842041,
-      "loss": 0.2974,
-      "rewards/accuracies": 0.550000011920929,
-      "rewards/chosen": 0.028298119083046913,
-      "rewards/margins": 0.04939670115709305,
-      "rewards/rejected": -0.021098580211400986,
+      "logits/chosen": -1.983096718788147,
+      "logits/rejected": -1.9870779514312744,
+      "logps/chosen": -33.37377166748047,
+      "logps/rejected": -31.558422088623047,
+      "loss": 0.9995,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.0004369509988464415,
+      "rewards/margins": 0.0005281810299493372,
+      "rewards/rejected": -9.123003837885335e-05,
       "step": 230
     },
     {
       "epoch": 0.62,
       "learning_rate": 1.8712423238279356e-07,
-      "logits/chosen": -1.9833450317382812,
-      "logits/rejected": -1.9613698720932007,
-      "logps/chosen": -34.1508903503418,
-      "logps/rejected": -34.95817947387695,
-      "loss": 0.3388,
-      "rewards/accuracies": 0.5249999761581421,
-      "rewards/chosen": 0.002569653559476137,
-      "rewards/margins": 0.00921549927443266,
-      "rewards/rejected": -0.006645847111940384,
+      "logits/chosen": -1.9832874536514282,
+      "logits/rejected": -1.9613001346588135,
+      "logps/chosen": -34.15814971923828,
+      "logps/rejected": -34.962947845458984,
+      "loss": 0.9999,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -4.397642987896688e-05,
+      "rewards/margins": 7.76300803408958e-05,
+      "rewards/rejected": -0.00012160651385784149,
       "step": 240
     },
     {
       "epoch": 0.65,
       "learning_rate": 1.654436768970182e-07,
-      "logits/chosen": -2.0248680114746094,
-      "logits/rejected": -2.021559238433838,
-      "logps/chosen": -32.907005310058594,
-      "logps/rejected": -36.22737121582031,
-      "loss": 0.3221,
-      "rewards/accuracies": 0.550000011920929,
-      "rewards/chosen": 0.016254058107733727,
-      "rewards/margins": 0.018458303064107895,
-      "rewards/rejected": -0.002204245189204812,
+      "logits/chosen": -2.024568557739258,
+      "logits/rejected": -2.021247386932373,
+      "logps/chosen": -32.91596221923828,
+      "logps/rejected": -36.22909164428711,
+      "loss": 0.9999,
+      "rewards/accuracies": 0.48750001192092896,
+      "rewards/chosen": 9.10547751118429e-05,
+      "rewards/margins": 0.00013272995420265943,
+      "rewards/rejected": -4.1675208194646984e-05,
       "step": 250
     },
     {
       "epoch": 0.68,
       "learning_rate": 1.444597403062196e-07,
-      "logits/chosen": -1.8913663625717163,
-      "logits/rejected": -1.8889217376708984,
-      "logps/chosen": -34.18629455566406,
-      "logps/rejected": -35.490962982177734,
-      "loss": 0.3601,
-      "rewards/accuracies": 0.5,
-      "rewards/chosen": 0.0018952053505927324,
-      "rewards/margins": -0.010397391393780708,
-      "rewards/rejected": 0.012292595580220222,
+      "logits/chosen": -1.891326904296875,
+      "logits/rejected": -1.8888810873031616,
+      "logps/chosen": -34.165008544921875,
+      "logps/rejected": -35.511131286621094,
+      "loss": 0.9997,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.00023390687420032918,
+      "rewards/margins": 0.00029896467458456755,
+      "rewards/rejected": -6.505780766019598e-05,
       "step": 260
     },
     {
       "epoch": 0.7,
       "learning_rate": 1.2434529917578887e-07,
-      "logits/chosen": -1.8756721019744873,
-      "logits/rejected": -1.8731330633163452,
-      "logps/chosen": -34.39625549316406,
-      "logps/rejected": -31.76236343383789,
-      "loss": 0.3492,
-      "rewards/accuracies": 0.4625000059604645,
-      "rewards/chosen": 0.001211274997331202,
-      "rewards/margins": -0.006859898567199707,
-      "rewards/rejected": 0.008071173913776875,
+      "logits/chosen": -1.8756563663482666,
+      "logits/rejected": -1.87309992313385,
+      "logps/chosen": -34.39522171020508,
+      "logps/rejected": -31.7664852142334,
+      "loss": 1.0,
+      "rewards/accuracies": 0.44999998807907104,
+      "rewards/chosen": 2.3789130864315666e-05,
+      "rewards/margins": -2.4663497242727317e-05,
+      "rewards/rejected": 4.8452602641191334e-05,
       "step": 270
     },
     {
       "epoch": 0.73,
       "learning_rate": 1.0526606671603521e-07,
-      "logits/chosen": -1.9795773029327393,
-      "logits/rejected": -1.9689537286758423,
-      "logps/chosen": -35.320404052734375,
-      "logps/rejected": -31.86062240600586,
-      "loss": 0.298,
-      "rewards/accuracies": 0.574999988079071,
-      "rewards/chosen": 0.032611697912216187,
-      "rewards/margins": 0.05027870461344719,
-      "rewards/rejected": -0.017666997388005257,
+      "logits/chosen": -1.9794845581054688,
+      "logits/rejected": -1.9688661098480225,
+      "logps/chosen": -35.318504333496094,
+      "logps/rejected": -31.839763641357422,
+      "loss": 0.9996,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": 0.0003813380899373442,
+      "rewards/margins": 0.0003690699231810868,
+      "rewards/rejected": 1.2268178579688538e-05,
       "step": 280
     },
     {
       "epoch": 0.75,
       "learning_rate": 8.737922755071453e-08,
-      "logits/chosen": -2.075582981109619,
-      "logits/rejected": -2.0605711936950684,
-      "logps/chosen": -30.907567977905273,
-      "logps/rejected": -32.65256881713867,
-      "loss": 0.3186,
-      "rewards/accuracies": 0.512499988079071,
-      "rewards/chosen": 0.014904007315635681,
-      "rewards/margins": 0.015564213506877422,
-      "rewards/rejected": -0.0006602049106732011,
+      "logits/chosen": -2.075482130050659,
+      "logits/rejected": -2.060464859008789,
+      "logps/chosen": -30.921621322631836,
+      "logps/rejected": -32.64299774169922,
+      "loss": 1.0001,
+      "rewards/accuracies": 0.48750001192092896,
+      "rewards/chosen": 2.502280767657794e-05,
+      "rewards/margins": -6.332028715405613e-05,
+      "rewards/rejected": 8.834306936478242e-05,
       "step": 290
     },
     {
       "epoch": 0.78,
       "learning_rate": 7.08321427484816e-08,
-      "logits/chosen": -1.9468443393707275,
-      "logits/rejected": -1.944310188293457,
-      "logps/chosen": -32.90850067138672,
-      "logps/rejected": -30.801532745361328,
-      "loss": 0.3327,
-      "rewards/accuracies": 0.4749999940395355,
-      "rewards/chosen": 0.016746358945965767,
-      "rewards/margins": 0.002998964861035347,
-      "rewards/rejected": 0.01374739408493042,
+      "logits/chosen": -1.9463533163070679,
+      "logits/rejected": -1.9438135623931885,
+      "logps/chosen": -32.896881103515625,
+      "logps/rejected": -30.816125869750977,
+      "loss": 0.9997,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": 0.0003022877499461174,
+      "rewards/margins": 0.0002954433439299464,
+      "rewards/rejected": 6.84439146425575e-06,
       "step": 300
     },
     {
       "epoch": 0.78,
-      "eval_logits/chosen": -2.243875741958618,
-      "eval_logits/rejected": -2.238999366760254,
-      "eval_logps/chosen": -34.01763916015625,
-      "eval_logps/rejected": -37.51422882080078,
-      "eval_loss": 0.32764142751693726,
-      "eval_rewards/accuracies": 0.5365448594093323,
-      "eval_rewards/chosen": 0.015219501219689846,
-      "eval_rewards/margins": 0.01306714303791523,
-      "eval_rewards/rejected": 0.0021523565519601107,
-      "eval_runtime": 145.9076,
-      "eval_samples_per_second": 2.351,
+      "eval_logits/chosen": -2.243636131286621,
+      "eval_logits/rejected": -2.2387540340423584,
+      "eval_logps/chosen": -34.02006530761719,
+      "eval_logps/rejected": -37.50526428222656,
+      "eval_loss": 0.9999700784683228,
+      "eval_rewards/accuracies": 0.4987541437149048,
+      "eval_rewards/chosen": 0.0001448561524739489,
+      "eval_rewards/margins": 3.130395998596214e-05,
+      "eval_rewards/rejected": 0.00011355219612596557,
+      "eval_runtime": 145.6582,
+      "eval_samples_per_second": 2.355,
       "eval_steps_per_second": 0.295,
       "step": 300
     },
     {
       "epoch": 0.81,
       "learning_rate": 5.576113578589034e-08,
-      "logits/chosen": -1.9285104274749756,
-      "logits/rejected": -1.925238847732544,
-      "logps/chosen": -31.57692527770996,
-      "logps/rejected": -33.744144439697266,
-      "loss": 0.3143,
-      "rewards/accuracies": 0.550000011920929,
-      "rewards/chosen": 0.023783722892403603,
-      "rewards/margins": 0.025577496737241745,
-      "rewards/rejected": -0.0017937703523784876,
+      "logits/chosen": -1.9288015365600586,
+      "logits/rejected": -1.9255383014678955,
+      "logps/chosen": -31.57794189453125,
+      "logps/rejected": -33.73435592651367,
+      "loss": 0.9998,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.000254074577242136,
+      "rewards/margins": 0.00017612635565456003,
+      "rewards/rejected": 7.794820703566074e-05,
       "step": 310
     },
     {
       "epoch": 0.83,
       "learning_rate": 4.229036944380912e-08,
-      "logits/chosen": -1.980386734008789,
-      "logits/rejected": -1.9680719375610352,
-      "logps/chosen": -34.57415771484375,
-      "logps/rejected": -33.55961227416992,
-      "loss": 0.31,
+      "logits/chosen": -1.9808727502822876,
+      "logits/rejected": -1.9685646295547485,
+      "logps/chosen": -34.54682159423828,
+      "logps/rejected": -33.555274963378906,
+      "loss": 0.9993,
       "rewards/accuracies": 0.5375000238418579,
-      "rewards/chosen": 0.01972355879843235,
-      "rewards/margins": 0.03841744735836983,
-      "rewards/rejected": -0.018693890422582626,
+      "rewards/chosen": 0.0004924900131300092,
+      "rewards/margins": 0.0006568533135578036,
+      "rewards/rejected": -0.00016436330042779446,
       "step": 320
     },
     {
       "epoch": 0.86,
       "learning_rate": 3.053082288996112e-08,
-      "logits/chosen": -2.015927314758301,
-      "logits/rejected": -2.014465570449829,
-      "logps/chosen": -33.47549057006836,
-      "logps/rejected": -32.47983169555664,
-      "loss": 0.3454,
-      "rewards/accuracies": 0.512499988079071,
-      "rewards/chosen": 0.009270393289625645,
-      "rewards/margins": 0.009491671808063984,
-      "rewards/rejected": -0.0002212822437286377,
+      "logits/chosen": -2.0161168575286865,
+      "logits/rejected": -2.0146450996398926,
+      "logps/chosen": -33.46837615966797,
+      "logps/rejected": -32.443458557128906,
+      "loss": 1.0002,
+      "rewards/accuracies": 0.48750001192092896,
+      "rewards/chosen": 0.00017420158837921917,
+      "rewards/margins": -0.00018708205607254058,
+      "rewards/rejected": 0.00036128362989984453,
       "step": 330
     },
     {
       "epoch": 0.88,
       "learning_rate": 2.05793773749158e-08,
-      "logits/chosen": -2.102839946746826,
-      "logits/rejected": -2.0870471000671387,
-      "logps/chosen": -34.18696212768555,
-      "logps/rejected": -33.075828552246094,
-      "loss": 0.4092,
-      "rewards/accuracies": 0.4124999940395355,
-      "rewards/chosen": -0.006814646068960428,
-      "rewards/margins": -0.04479615017771721,
-      "rewards/rejected": 0.03798150271177292,
+      "logits/chosen": -2.102954626083374,
+      "logits/rejected": -2.087148666381836,
+      "logps/chosen": -34.159751892089844,
+      "logps/rejected": -33.093017578125,
+      "loss": 1.0001,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": 0.00019638515368569642,
+      "rewards/margins": -5.3783714974997565e-05,
+      "rewards/rejected": 0.0002501688722986728,
       "step": 340
     },
     {
       "epoch": 0.91,
       "learning_rate": 1.251801807404168e-08,
-      "logits/chosen": -1.9747005701065063,
-      "logits/rejected": -1.9737575054168701,
-      "logps/chosen": -33.231842041015625,
-      "logps/rejected": -32.461700439453125,
-      "loss": 0.3207,
-      "rewards/accuracies": 0.5249999761581421,
-      "rewards/chosen": 0.04523162543773651,
-      "rewards/margins": 0.04212886095046997,
-      "rewards/rejected": 0.003102770773693919,
+      "logits/chosen": -1.974347710609436,
+      "logits/rejected": -1.9733997583389282,
+      "logps/chosen": -33.24498748779297,
+      "logps/rejected": -32.44942855834961,
+      "loss": 0.9998,
+      "rewards/accuracies": 0.48750001192092896,
+      "rewards/chosen": 0.00037107508978806436,
+      "rewards/margins": 0.0002138734853360802,
+      "rewards/rejected": 0.00015720158990006894,
       "step": 350
     },
     {
       "epoch": 0.94,
       "learning_rate": 6.41315865106129e-09,
-      "logits/chosen": -1.9306046962738037,
-      "logits/rejected": -1.9409637451171875,
-      "logps/chosen": -32.217220306396484,
-      "logps/rejected": -35.306907653808594,
-      "loss": 0.3411,
-      "rewards/accuracies": 0.4625000059604645,
-      "rewards/chosen": -0.011275654658675194,
-      "rewards/margins": 0.005062489304691553,
-      "rewards/rejected": -0.01633814536035061,
+      "logits/chosen": -1.9305756092071533,
+      "logits/rejected": -1.9409431219100952,
+      "logps/chosen": -32.20996856689453,
+      "logps/rejected": -35.26984405517578,
+      "loss": 1.0002,
+      "rewards/accuracies": 0.4749999940395355,
+      "rewards/chosen": -5.276153751765378e-05,
+      "rewards/margins": -0.00024183676578104496,
+      "rewards/rejected": 0.00018907521734945476,
       "step": 360
     },
     {
       "epoch": 0.96,
       "learning_rate": 2.3150941078050324e-09,
-      "logits/chosen": -2.0694775581359863,
-      "logits/rejected": -2.0629212856292725,
-      "logps/chosen": -33.65714645385742,
-      "logps/rejected": -29.221160888671875,
-      "loss": 0.3549,
-      "rewards/accuracies": 0.4749999940395355,
-      "rewards/chosen": -0.004660401493310928,
-      "rewards/margins": -0.009401795454323292,
-      "rewards/rejected": 0.0047413939610123634,
+      "logits/chosen": -2.0696487426757812,
+      "logits/rejected": -2.0630972385406494,
+      "logps/chosen": -33.643394470214844,
+      "logps/rejected": -29.242889404296875,
+      "loss": 0.9997,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 8.579110726714134e-05,
+      "rewards/margins": 0.0002504353760741651,
+      "rewards/rejected": -0.00016464429791085422,
       "step": 370
     },
     {
       "epoch": 0.99,
       "learning_rate": 2.575864278703266e-10,
-      "logits/chosen": -1.9296157360076904,
-      "logits/rejected": -1.931777000427246,
-      "logps/chosen": -34.22825241088867,
-      "logps/rejected": -30.912120819091797,
-      "loss": 0.2952,
-      "rewards/accuracies": 0.5375000238418579,
-      "rewards/chosen": 0.009714365005493164,
-      "rewards/margins": 0.04525749385356903,
-      "rewards/rejected": -0.035543132573366165,
+      "logits/chosen": -1.9290872812271118,
+      "logits/rejected": -1.9312543869018555,
+      "logps/chosen": -34.22844696044922,
+      "logps/rejected": -30.902618408203125,
+      "loss": 0.9996,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": 0.00010604139242786914,
+      "rewards/margins": 0.0004059396160300821,
+      "rewards/rejected": -0.00029989826725795865,
       "step": 380
     },
     {
       "epoch": 1.0,
       "step": 385,
       "total_flos": 0.0,
-      "train_loss": 0.3306902726749321,
-      "train_runtime": 3256.1932,
-      "train_samples_per_second": 0.946,
+      "train_loss": 0.9998476387618425,
+      "train_runtime": 3251.6099,
+      "train_samples_per_second": 0.947,
       "train_steps_per_second": 0.118
     }
   ],