Model save

README.md

@@ -1,13 +1,11 @@
 ---
 library_name: peft
 tags:
-- alignment-handbook
 - trl
 - dpo
+- alignment-handbook
 - generated_from_trainer
 base_model: NbAiLab/nb-gpt-j-6B-v2
-datasets:
-- hugodk-sch/aftonposten_title_prefs
 model-index:
 - name: aftonposten-6b-align-scan
   results: []
@@ -18,17 +16,17 @@ should probably proofread and complete it, then remove this comment. -->
 
 # aftonposten-6b-align-scan
 
-This model is a fine-tuned version of [data/ap-gpt-j-6b-sft-qlora-04-08](https://huggingface.co/data/ap-gpt-j-6b-sft-qlora-04-08) on the hugodk-sch/aftonposten_title_prefs dataset.
+This model is a fine-tuned version of [NbAiLab/nb-gpt-j-6B-v2](https://huggingface.co/NbAiLab/nb-gpt-j-6B-v2) on an unknown dataset.
 It achieves the following results on the evaluation set:
-- Loss: 0.7054
-- Rewards/chosen: -0.0010
-- Rewards/rejected: 0.0163
-- Rewards/accuracies: 0.4726
-- Rewards/margins: -0.0174
-- Logps/rejected: -37.4962
-- Logps/chosen: -34.0358
-- Logits/rejected: -2.2386
-- Logits/chosen: -2.2435
+- Logits/chosen: -2.2444
+- Logits/rejected: -2.2395
+- Logps/chosen: -34.0121
+- Logps/rejected: -37.4951
+- Loss: 0.5001
+- Rewards/accuracies: 0.5046
+- Rewards/chosen: 0.0180
+- Rewards/margins: 0.0007
+- Rewards/rejected: 0.0172
 
 ## Model description
 
@@ -63,9 +61,9 @@ The following hyperparameters were used during training:
 
 | Training Loss | Epoch | Step | Logits/chosen | Logits/rejected | Logps/chosen | Logps/rejected | Validation Loss | Rewards/accuracies | Rewards/chosen | Rewards/margins | Rewards/rejected |
 |:-------------:|:-----:|:----:|:-------------:|:---------------:|:------------:|:--------------:|:---------------:|:------------------:|:--------------:|:---------------:|:----------------:|
-| 0.6927        | 0.26  | 100  | -2.2438       | -2.2389         | -34.0159     | -37.5056       | 0.6927          | 0.5187             | 0.0150         | 0.0062          | 0.0088           |
-| 0.6949        | 0.52  | 200  | -2.2436       | -2.2387         | -34.0198     | -37.5098       | 0.6928          | 0.5274             | 0.0118         | 0.0063          | 0.0055           |
-| 0.6798        | 0.78  | 300  | -2.2439       | -2.2390         | -34.0268     | -37.5040       | 0.6981          | 0.4983             | 0.0062         | -0.0039         | 0.0101           |
+| 0.4981        | 0.26  | 100  | -2.2441       | -2.2392         | -34.0229     | -37.5093       | 0.4997          | 0.5079             | 0.0093         | 0.0034          | 0.0059           |
+| 0.4987        | 0.52  | 200  | -2.2435       | -2.2386         | -34.0140     | -37.4995       | 0.4999          | 0.5075             | 0.0165         | 0.0027          | 0.0137           |
+| 0.4901        | 0.78  | 300  | -2.2444       | -2.2395         | -34.0121     | -37.4951       | 0.5001          | 0.5046             | 0.0180         | 0.0007          | 0.0172           |
 
 
 ### Framework versions

adapter_config.json

@@ -19,13 +19,13 @@
   "rank_pattern": {},
   "revision": null,
   "target_modules": [
-    "q_proj",
-    "gate_proj",
     "up_proj",
-    "down_proj",
     "v_proj",
+    "q_proj",
     "k_proj",
-    "o_proj"
+    "o_proj",
+    "down_proj",
+    "gate_proj"
   ],
   "task_type": "CAUSAL_LM",
   "use_rslora": false

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4241a5bac4adc39debfd72b4e4d2124e198b356cd95c7245382199b8383c449c
+oid sha256:7846eb40dab46889820e060d23a8c2727cb491e6829ccec36390a2b074e575ac
 size 176183216

all_results.json

@@ -1,21 +1,21 @@
 {
     "epoch": 1.0,
-    "eval_logits/chosen": -2.2434651851654053,
-    "eval_logits/rejected": -2.2385833263397217,
-    "eval_logps/chosen": -34.03584671020508,
-    "eval_logps/rejected": -37.496219635009766,
-    "eval_loss": 0.7054064273834229,
-    "eval_rewards/accuracies": 0.472591370344162,
-    "eval_rewards/chosen": -0.001035186112858355,
-    "eval_rewards/margins": -0.017356980592012405,
-    "eval_rewards/rejected": 0.01632179319858551,
-    "eval_runtime": 146.0323,
+    "eval_logits/chosen": -2.2444307804107666,
+    "eval_logits/rejected": -2.2395598888397217,
+    "eval_logps/chosen": -34.009063720703125,
+    "eval_logps/rejected": -37.49898147583008,
+    "eval_loss": 0.49849173426628113,
+    "eval_rewards/accuracies": 0.531146228313446,
+    "eval_rewards/chosen": 0.020389538258314133,
+    "eval_rewards/margins": 0.00627789506688714,
+    "eval_rewards/rejected": 0.014111644588410854,
+    "eval_runtime": 145.7275,
     "eval_samples": 343,
-    "eval_samples_per_second": 2.349,
-    "eval_steps_per_second": 0.294,
-    "train_loss": 0.15268191238502404,
-    "train_runtime": 627.0699,
+    "eval_samples_per_second": 2.354,
+    "eval_steps_per_second": 0.295,
+    "train_loss": 0.10985468208015739,
+    "train_runtime": 628.1848,
     "train_samples": 3079,
-    "train_samples_per_second": 4.91,
-    "train_steps_per_second": 0.614
+    "train_samples_per_second": 4.901,
+    "train_steps_per_second": 0.613
 }

eval_results.json

@@ -1,16 +1,16 @@
 {
     "epoch": 1.0,
-    "eval_logits/chosen": -2.2434651851654053,
-    "eval_logits/rejected": -2.2385833263397217,
-    "eval_logps/chosen": -34.03584671020508,
-    "eval_logps/rejected": -37.496219635009766,
-    "eval_loss": 0.7054064273834229,
-    "eval_rewards/accuracies": 0.472591370344162,
-    "eval_rewards/chosen": -0.001035186112858355,
-    "eval_rewards/margins": -0.017356980592012405,
-    "eval_rewards/rejected": 0.01632179319858551,
-    "eval_runtime": 146.0323,
+    "eval_logits/chosen": -2.2444307804107666,
+    "eval_logits/rejected": -2.2395598888397217,
+    "eval_logps/chosen": -34.009063720703125,
+    "eval_logps/rejected": -37.49898147583008,
+    "eval_loss": 0.49849173426628113,
+    "eval_rewards/accuracies": 0.531146228313446,
+    "eval_rewards/chosen": 0.020389538258314133,
+    "eval_rewards/margins": 0.00627789506688714,
+    "eval_rewards/rejected": 0.014111644588410854,
+    "eval_runtime": 145.7275,
     "eval_samples": 343,
-    "eval_samples_per_second": 2.349,
-    "eval_steps_per_second": 0.294
+    "eval_samples_per_second": 2.354,
+    "eval_steps_per_second": 0.295
 }

train_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 1.0,
-    "train_loss": 0.15268191238502404,
-    "train_runtime": 627.0699,
+    "train_loss": 0.10985468208015739,
+    "train_runtime": 628.1848,
     "train_samples": 3079,
-    "train_samples_per_second": 4.91,
-    "train_steps_per_second": 0.614
+    "train_samples_per_second": 4.901,
+    "train_steps_per_second": 0.613
 }

trainer_state.json

@@ -15,7 +15,7 @@
       "logits/rejected": -1.7377450466156006,
       "logps/chosen": -29.553977966308594,
       "logps/rejected": -42.813133239746094,
-      "loss": 0.6931,
+      "loss": 0.5,
       "rewards/accuracies": 0.0,
       "rewards/chosen": 0.0,
       "rewards/margins": 0.0,
@@ -25,311 +25,311 @@
     {
       "epoch": 0.03,
       "learning_rate": 1.2820512820512818e-07,
-      "logits/chosen": -1.8660534620285034,
-      "logits/rejected": -1.870363712310791,
-      "logps/chosen": -37.0097770690918,
-      "logps/rejected": -33.656063079833984,
-      "loss": 0.6866,
+      "logits/chosen": -1.8662465810775757,
+      "logits/rejected": -1.8705615997314453,
+      "logps/chosen": -36.9855842590332,
+      "logps/rejected": -33.65031433105469,
+      "loss": 0.4935,
       "rewards/accuracies": 0.4861111044883728,
-      "rewards/chosen": -0.0025405611377209425,
-      "rewards/margins": 0.018248552456498146,
-      "rewards/rejected": -0.020789114758372307,
+      "rewards/chosen": 0.016807515174150467,
+      "rewards/margins": 0.03299880772829056,
+      "rewards/rejected": -0.01619129255414009,
       "step": 10
     },
     {
       "epoch": 0.05,
       "learning_rate": 2.5641025641025636e-07,
-      "logits/chosen": -1.9975675344467163,
-      "logits/rejected": -2.0001988410949707,
-      "logps/chosen": -29.643625259399414,
-      "logps/rejected": -29.03875732421875,
-      "loss": 0.7064,
-      "rewards/accuracies": 0.4000000059604645,
-      "rewards/chosen": -0.0011415861081331968,
-      "rewards/margins": -0.02089018002152443,
-      "rewards/rejected": 0.019748592749238014,
+      "logits/chosen": -1.9972314834594727,
+      "logits/rejected": -1.999875783920288,
+      "logps/chosen": -29.622329711914062,
+      "logps/rejected": -29.04340171813965,
+      "loss": 0.4989,
+      "rewards/accuracies": 0.44999998807907104,
+      "rewards/chosen": 0.01589435711503029,
+      "rewards/margins": -0.00013793967082165182,
+      "rewards/rejected": 0.016032297164201736,
       "step": 20
     },
     {
       "epoch": 0.08,
       "learning_rate": 3.8461538461538463e-07,
-      "logits/chosen": -1.919878363609314,
-      "logits/rejected": -1.9171901941299438,
-      "logps/chosen": -31.411758422851562,
-      "logps/rejected": -33.21895980834961,
-      "loss": 0.6945,
-      "rewards/accuracies": 0.48750001192092896,
-      "rewards/chosen": 0.0034693297930061817,
-      "rewards/margins": 0.003873053938150406,
-      "rewards/rejected": -0.0004037246108055115,
+      "logits/chosen": -1.9198119640350342,
+      "logits/rejected": -1.9171171188354492,
+      "logps/chosen": -31.40401268005371,
+      "logps/rejected": -33.211997985839844,
+      "loss": 0.4999,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.0096666868776083,
+      "rewards/margins": 0.0044937655329704285,
+      "rewards/rejected": 0.005172918550670147,
       "step": 30
     },
     {
       "epoch": 0.1,
       "learning_rate": 4.999896948438433e-07,
-      "logits/chosen": -2.0166752338409424,
-      "logits/rejected": -2.007917881011963,
-      "logps/chosen": -32.59661102294922,
-      "logps/rejected": -32.506187438964844,
-      "loss": 0.7064,
-      "rewards/accuracies": 0.48750001192092896,
-      "rewards/chosen": -0.01584548130631447,
-      "rewards/margins": -0.02103354223072529,
-      "rewards/rejected": 0.005188059527426958,
+      "logits/chosen": -2.0162081718444824,
+      "logits/rejected": -2.007472515106201,
+      "logps/chosen": -32.587196350097656,
+      "logps/rejected": -32.514732360839844,
+      "loss": 0.5011,
+      "rewards/accuracies": 0.4625000059604645,
+      "rewards/chosen": -0.00831048097461462,
+      "rewards/margins": -0.00666379788890481,
+      "rewards/rejected": -0.0016466856468468904,
       "step": 40
     },
     {
       "epoch": 0.13,
       "learning_rate": 4.987541037542186e-07,
-      "logits/chosen": -1.8648042678833008,
-      "logits/rejected": -1.8540267944335938,
-      "logps/chosen": -33.56422805786133,
-      "logps/rejected": -35.431602478027344,
-      "loss": 0.7067,
-      "rewards/accuracies": 0.42500001192092896,
-      "rewards/chosen": -0.005400418769568205,
-      "rewards/margins": -0.02199491299688816,
-      "rewards/rejected": 0.01659449376165867,
+      "logits/chosen": -1.8641865253448486,
+      "logits/rejected": -1.8533992767333984,
+      "logps/chosen": -33.56541061401367,
+      "logps/rejected": -35.421974182128906,
+      "loss": 0.5049,
+      "rewards/accuracies": 0.4124999940395355,
+      "rewards/chosen": -0.006343575660139322,
+      "rewards/margins": -0.030638951808214188,
+      "rewards/rejected": 0.02429538033902645,
       "step": 50
     },
     {
       "epoch": 0.16,
       "learning_rate": 4.954691471941118e-07,
-      "logits/chosen": -1.9453144073486328,
-      "logits/rejected": -1.9472593069076538,
-      "logps/chosen": -32.59089279174805,
-      "logps/rejected": -33.192405700683594,
-      "loss": 0.6844,
-      "rewards/accuracies": 0.4625000059604645,
-      "rewards/chosen": 0.007432769052684307,
-      "rewards/margins": 0.024254899471998215,
-      "rewards/rejected": -0.016822131350636482,
+      "logits/chosen": -1.945642113685608,
+      "logits/rejected": -1.9475781917572021,
+      "logps/chosen": -32.572914123535156,
+      "logps/rejected": -33.161590576171875,
+      "loss": 0.4972,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.021815191954374313,
+      "rewards/margins": 0.01398499310016632,
+      "rewards/rejected": 0.007830199785530567,
       "step": 60
     },
     {
       "epoch": 0.18,
       "learning_rate": 4.901618883413548e-07,
-      "logits/chosen": -2.079728364944458,
-      "logits/rejected": -2.0847129821777344,
-      "logps/chosen": -33.983558654785156,
-      "logps/rejected": -36.584938049316406,
-      "loss": 0.688,
-      "rewards/accuracies": 0.5375000238418579,
-      "rewards/chosen": 0.003943340387195349,
-      "rewards/margins": 0.016796352341771126,
-      "rewards/rejected": -0.01285301148891449,
+      "logits/chosen": -2.0798556804656982,
+      "logits/rejected": -2.0848286151885986,
+      "logps/chosen": -33.97870635986328,
+      "logps/rejected": -36.580543518066406,
+      "loss": 0.4965,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.007821302860975266,
+      "rewards/margins": 0.017155062407255173,
+      "rewards/rejected": -0.009333762340247631,
       "step": 70
     },
     {
       "epoch": 0.21,
       "learning_rate": 4.828760511501322e-07,
-      "logits/chosen": -1.9422305822372437,
-      "logits/rejected": -1.9453967809677124,
-      "logps/chosen": -34.4206657409668,
-      "logps/rejected": -34.566123962402344,
-      "loss": 0.7053,
-      "rewards/accuracies": 0.4375,
-      "rewards/chosen": 0.005717824678868055,
-      "rewards/margins": -0.015306837856769562,
-      "rewards/rejected": 0.021024659276008606,
+      "logits/chosen": -1.9424225091934204,
+      "logits/rejected": -1.9455715417861938,
+      "logps/chosen": -34.389671325683594,
+      "logps/rejected": -34.575042724609375,
+      "loss": 0.4968,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": 0.030510548502206802,
+      "rewards/margins": 0.016619805246591568,
+      "rewards/rejected": 0.013890743255615234,
       "step": 80
     },
     {
       "epoch": 0.23,
       "learning_rate": 4.736716601303429e-07,
-      "logits/chosen": -1.950453519821167,
-      "logits/rejected": -1.9549684524536133,
-      "logps/chosen": -32.457984924316406,
-      "logps/rejected": -32.36473083496094,
-      "loss": 0.6897,
+      "logits/chosen": -1.9505561590194702,
+      "logits/rejected": -1.9550676345825195,
+      "logps/chosen": -32.456565856933594,
+      "logps/rejected": -32.361209869384766,
+      "loss": 0.4976,
       "rewards/accuracies": 0.5249999761581421,
-      "rewards/chosen": 0.007545561995357275,
-      "rewards/margins": 0.012203911319375038,
-      "rewards/rejected": -0.004658351186662912,
+      "rewards/chosen": 0.008682352490723133,
+      "rewards/margins": 0.0105207534506917,
+      "rewards/rejected": -0.0018384016584604979,
       "step": 90
     },
     {
       "epoch": 0.26,
       "learning_rate": 4.62624545834521e-07,
-      "logits/chosen": -2.048896312713623,
-      "logits/rejected": -2.0468926429748535,
-      "logps/chosen": -32.23190689086914,
-      "logps/rejected": -31.26666831970215,
-      "loss": 0.6927,
-      "rewards/accuracies": 0.5,
-      "rewards/chosen": 0.007141563110053539,
-      "rewards/margins": 0.007507171481847763,
-      "rewards/rejected": -0.0003656085464172065,
+      "logits/chosen": -2.048992156982422,
+      "logits/rejected": -2.0470006465911865,
+      "logps/chosen": -32.2261848449707,
+      "logps/rejected": -31.262670516967773,
+      "loss": 0.4981,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": 0.01171906292438507,
+      "rewards/margins": 0.008886159397661686,
+      "rewards/rejected": 0.0028329025954008102,
       "step": 100
     },
     {
       "epoch": 0.26,
-      "eval_logits/chosen": -2.2437734603881836,
-      "eval_logits/rejected": -2.2388932704925537,
-      "eval_logps/chosen": -34.01585006713867,
-      "eval_logps/rejected": -37.50562286376953,
-      "eval_loss": 0.6927134394645691,
-      "eval_rewards/accuracies": 0.5186877250671387,
-      "eval_rewards/chosen": 0.014961617067456245,
-      "eval_rewards/margins": 0.006164299789816141,
-      "eval_rewards/rejected": 0.008797316811978817,
-      "eval_runtime": 146.2315,
-      "eval_samples_per_second": 2.346,
+      "eval_logits/chosen": -2.244107484817505,
+      "eval_logits/rejected": -2.239222764968872,
+      "eval_logps/chosen": -34.02288818359375,
+      "eval_logps/rejected": -37.50926208496094,
+      "eval_loss": 0.49965521693229675,
+      "eval_rewards/accuracies": 0.5078904032707214,
+      "eval_rewards/chosen": 0.009330343455076218,
+      "eval_rewards/margins": 0.003444999223574996,
+      "eval_rewards/rejected": 0.005885345861315727,
+      "eval_runtime": 146.2405,
+      "eval_samples_per_second": 2.345,
       "eval_steps_per_second": 0.294,
       "step": 100
     },
     {
       "epoch": 0.29,
       "learning_rate": 4.4982572012636904e-07,
-      "logits/chosen": -2.005382537841797,
-      "logits/rejected": -2.002962589263916,
-      "logps/chosen": -33.231327056884766,
-      "logps/rejected": -34.03110885620117,
-      "loss": 0.6912,
-      "rewards/accuracies": 0.5249999761581421,
-      "rewards/chosen": 0.009991822764277458,
-      "rewards/margins": 0.01138134766370058,
-      "rewards/rejected": -0.0013895228039473295,
+      "logits/chosen": -2.005356788635254,
+      "logits/rejected": -2.0029444694519043,
+      "logps/chosen": -33.24960708618164,
+      "logps/rejected": -34.023834228515625,
+      "loss": 0.5009,
+      "rewards/accuracies": 0.4749999940395355,
+      "rewards/chosen": -0.004633937496691942,
+      "rewards/margins": -0.009062351658940315,
+      "rewards/rejected": 0.00442841649055481,
       "step": 110
     },
     {
       "epoch": 0.31,
       "learning_rate": 4.353806263777677e-07,
-      "logits/chosen": -2.01668643951416,
-      "logits/rejected": -2.0083160400390625,
-      "logps/chosen": -32.46741485595703,
-      "logps/rejected": -32.169979095458984,
-      "loss": 0.7019,
-      "rewards/accuracies": 0.4375,
-      "rewards/chosen": -0.018008466809988022,
-      "rewards/margins": -0.011230994947254658,
-      "rewards/rejected": -0.006777471862733364,
+      "logits/chosen": -2.017059564590454,
+      "logits/rejected": -2.008686065673828,
+      "logps/chosen": -32.44651412963867,
+      "logps/rejected": -32.1725959777832,
+      "loss": 0.4978,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.0012888375204056501,
+      "rewards/margins": 0.007584270089864731,
+      "rewards/rejected": -0.008873110637068748,
       "step": 120
     },
     {
       "epoch": 0.34,
       "learning_rate": 4.194082707715275e-07,
-      "logits/chosen": -2.046079158782959,
-      "logits/rejected": -2.038038969039917,
-      "logps/chosen": -30.489391326904297,
-      "logps/rejected": -32.04558563232422,
-      "loss": 0.7042,
+      "logits/chosen": -2.0462288856506348,
+      "logits/rejected": -2.038172960281372,
+      "logps/chosen": -30.503637313842773,
+      "logps/rejected": -32.05141067504883,
+      "loss": 0.5043,
       "rewards/accuracies": 0.4625000059604645,
-      "rewards/chosen": -0.010940661653876305,
-      "rewards/margins": -0.01687880977988243,
-      "rewards/rejected": 0.005938149057328701,
+      "rewards/chosen": -0.022338179871439934,
+      "rewards/margins": -0.023616474121809006,
+      "rewards/rejected": 0.0012782930862158537,
       "step": 130
     },
     {
       "epoch": 0.36,
       "learning_rate": 4.020402418666621e-07,
-      "logits/chosen": -1.9770376682281494,
-      "logits/rejected": -1.9873006343841553,
-      "logps/chosen": -31.392288208007812,
-      "logps/rejected": -32.56155014038086,
-      "loss": 0.6749,
-      "rewards/accuracies": 0.6625000238418579,
-      "rewards/chosen": 0.021898187696933746,
-      "rewards/margins": 0.043236203491687775,
-      "rewards/rejected": -0.02133801393210888,
+      "logits/chosen": -1.9769847393035889,
+      "logits/rejected": -1.9872528314590454,
+      "logps/chosen": -31.388320922851562,
+      "logps/rejected": -32.554039001464844,
+      "loss": 0.4928,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": 0.02507348358631134,
+      "rewards/margins": 0.04039759188890457,
+      "rewards/rejected": -0.015324106439948082,
       "step": 140
     },
     {
       "epoch": 0.39,
       "learning_rate": 3.8341962650351185e-07,
-      "logits/chosen": -1.8905258178710938,
-      "logits/rejected": -1.891603708267212,
-      "logps/chosen": -34.185401916503906,
-      "logps/rejected": -34.75876998901367,
-      "loss": 0.6933,
-      "rewards/accuracies": 0.4749999940395355,
-      "rewards/chosen": 0.005832195747643709,
-      "rewards/margins": 0.007043843157589436,
-      "rewards/rejected": -0.0012116472935304046,
+      "logits/chosen": -1.8903350830078125,
+      "logits/rejected": -1.8914152383804321,
+      "logps/chosen": -34.154296875,
+      "logps/rejected": -34.76646423339844,
+      "loss": 0.4927,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.030716974288225174,
+      "rewards/margins": 0.03808692842721939,
+      "rewards/rejected": -0.007369957864284515,
       "step": 150
     },
     {
       "epoch": 0.42,
       "learning_rate": 3.636998309800572e-07,
-      "logits/chosen": -1.942525863647461,
-      "logits/rejected": -1.9390523433685303,
-      "logps/chosen": -36.15251922607422,
-      "logps/rejected": -32.721561431884766,
-      "loss": 0.6891,
-      "rewards/accuracies": 0.512499988079071,
-      "rewards/chosen": 0.01532670110464096,
-      "rewards/margins": 0.014371859841048717,
-      "rewards/rejected": 0.0009548395755700767,
+      "logits/chosen": -1.9426672458648682,
+      "logits/rejected": -1.939186453819275,
+      "logps/chosen": -36.157989501953125,
+      "logps/rejected": -32.7253303527832,
+      "loss": 0.4972,
+      "rewards/accuracies": 0.4749999940395355,
+      "rewards/chosen": 0.010952227748930454,
+      "rewards/margins": 0.013016450218856335,
+      "rewards/rejected": -0.0020642229355871677,
       "step": 160
     },
     {
       "epoch": 0.44,
       "learning_rate": 3.430433172111807e-07,
-      "logits/chosen": -2.042137622833252,
-      "logits/rejected": -2.0347514152526855,
-      "logps/chosen": -33.78362274169922,
-      "logps/rejected": -31.34653091430664,
-      "loss": 0.6922,
-      "rewards/accuracies": 0.6000000238418579,
-      "rewards/chosen": 0.011391163803637028,
-      "rewards/margins": 0.007135935127735138,
-      "rewards/rejected": 0.004255227744579315,
+      "logits/chosen": -2.0421078205108643,
+      "logits/rejected": -2.0347187519073486,
+      "logps/chosen": -33.78765106201172,
+      "logps/rejected": -31.363611221313477,
+      "loss": 0.496,
+      "rewards/accuracies": 0.4625000059604645,
+      "rewards/chosen": 0.008164674043655396,
+      "rewards/margins": 0.0175738874822855,
+      "rewards/rejected": -0.009409213438630104,
       "step": 170
     },
     {
       "epoch": 0.47,
       "learning_rate": 3.216202642830543e-07,
-      "logits/chosen": -2.047314405441284,
-      "logits/rejected": -2.052593231201172,
-      "logps/chosen": -32.52313995361328,
-      "logps/rejected": -32.499427795410156,
-      "loss": 0.6811,
-      "rewards/accuracies": 0.550000011920929,
-      "rewards/chosen": 0.01731465384364128,
-      "rewards/margins": 0.028742337599396706,
-      "rewards/rejected": -0.011427680030465126,
+      "logits/chosen": -2.0476274490356445,
+      "logits/rejected": -2.0528929233551025,
+      "logps/chosen": -32.52192306518555,
+      "logps/rejected": -32.50886917114258,
+      "loss": 0.4919,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": 0.01828894577920437,
+      "rewards/margins": 0.037272557616233826,
+      "rewards/rejected": -0.018983609974384308,
       "step": 180
     },
     {
       "epoch": 0.49,
       "learning_rate": 2.9960716642946403e-07,
-      "logits/chosen": -2.048703908920288,
-      "logits/rejected": -2.0459342002868652,
-      "logps/chosen": -31.495092391967773,
-      "logps/rejected": -31.32244300842285,
-      "loss": 0.6967,
-      "rewards/accuracies": 0.4749999940395355,
-      "rewards/chosen": -0.004913434386253357,
-      "rewards/margins": -0.0015291940653696656,
-      "rewards/rejected": -0.0033842422999441624,
+      "logits/chosen": -2.0479583740234375,
+      "logits/rejected": -2.0451717376708984,
+      "logps/chosen": -31.4959774017334,
+      "logps/rejected": -31.333343505859375,
+      "loss": 0.4986,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -0.005620955489575863,
+      "rewards/margins": 0.006484903395175934,
+      "rewards/rejected": -0.012105859816074371,
       "step": 190
     },
     {
       "epoch": 0.52,
       "learning_rate": 2.771853789806683e-07,
-      "logits/chosen": -1.9186630249023438,
-      "logits/rejected": -1.9233324527740479,
-      "logps/chosen": -31.604812622070312,
-      "logps/rejected": -32.791358947753906,
-      "loss": 0.6949,
-      "rewards/accuracies": 0.44999998807907104,
-      "rewards/chosen": 0.0011774369049817324,
-      "rewards/margins": 0.0015200242633000016,
-      "rewards/rejected": -0.00034258683444932103,
+      "logits/chosen": -1.9188188314437866,
+      "logits/rejected": -1.9234987497329712,
+      "logps/chosen": -31.607952117919922,
+      "logps/rejected": -32.79724884033203,
+      "loss": 0.4987,
+      "rewards/accuracies": 0.48750001192092896,
+      "rewards/chosen": -0.0013334359973669052,
+      "rewards/margins": 0.0037218607030808926,
+      "rewards/rejected": -0.005055299494415522,
       "step": 200
     },
     {
       "epoch": 0.52,
-      "eval_logits/chosen": -2.2436025142669678,
-      "eval_logits/rejected": -2.2387301921844482,
-      "eval_logps/chosen": -34.01983642578125,
-      "eval_logps/rejected": -37.50978469848633,
-      "eval_loss": 0.6928001642227173,
-      "eval_rewards/accuracies": 0.5274086594581604,
-      "eval_rewards/chosen": 0.011776229366660118,
-      "eval_rewards/margins": 0.006309796124696732,
-      "eval_rewards/rejected": 0.0054664346389472485,
-      "eval_runtime": 145.9171,
+      "eval_logits/chosen": -2.243473768234253,
+      "eval_logits/rejected": -2.2385945320129395,
+      "eval_logps/chosen": -34.013954162597656,
+      "eval_logps/rejected": -37.49945068359375,
+      "eval_loss": 0.49993959069252014,
+      "eval_rewards/accuracies": 0.5074750781059265,
+      "eval_rewards/chosen": 0.016473928466439247,
+      "eval_rewards/margins": 0.0027353279292583466,
+      "eval_rewards/rejected": 0.0137386005371809,
+      "eval_runtime": 145.8931,
       "eval_samples_per_second": 2.351,
       "eval_steps_per_second": 0.295,
       "step": 200
@@ -337,155 +337,155 @@
     {
       "epoch": 0.55,
       "learning_rate": 2.5453962426402e-07,
-      "logits/chosen": -2.0316667556762695,
-      "logits/rejected": -2.0423641204833984,
-      "logps/chosen": -31.9429874420166,
-      "logps/rejected": -33.863956451416016,
-      "loss": 0.6846,
-      "rewards/accuracies": 0.512499988079071,
-      "rewards/chosen": 0.022652912884950638,
-      "rewards/margins": 0.023658016696572304,
-      "rewards/rejected": -0.0010051012504845858,
+      "logits/chosen": -2.0315659046173096,
+      "logits/rejected": -2.0422449111938477,
+      "logps/chosen": -31.945932388305664,
+      "logps/rejected": -33.85708999633789,
+      "loss": 0.497,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": 0.020296860486268997,
+      "rewards/margins": 0.015811126679182053,
+      "rewards/rejected": 0.0044857352040708065,
       "step": 210
     },
     {
       "epoch": 0.57,
       "learning_rate": 2.318564697655179e-07,
-      "logits/chosen": -1.9254566431045532,
-      "logits/rejected": -1.9402967691421509,
-      "logps/chosen": -30.087352752685547,
-      "logps/rejected": -31.55807113647461,
-      "loss": 0.6858,
-      "rewards/accuracies": 0.5375000238418579,
-      "rewards/chosen": 0.019350551068782806,
-      "rewards/margins": 0.019265536218881607,
-      "rewards/rejected": 8.501634147251025e-05,
+      "logits/chosen": -1.9255174398422241,
+      "logits/rejected": -1.9403587579727173,
+      "logps/chosen": -30.07940101623535,
+      "logps/rejected": -31.557880401611328,
+      "loss": 0.4944,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": 0.02570931240916252,
+      "rewards/margins": 0.02547053061425686,
+      "rewards/rejected": 0.00023878086358308792,
       "step": 220
     },
     {
       "epoch": 0.6,
       "learning_rate": 2.093227910899832e-07,
-      "logits/chosen": -1.9833059310913086,
-      "logits/rejected": -1.9872699975967407,
-      "logps/chosen": -33.40017318725586,
-      "logps/rejected": -31.5333309173584,
-      "loss": 0.6963,
-      "rewards/accuracies": 0.4625000059604645,
-      "rewards/chosen": 0.013836635276675224,
-      "rewards/margins": 0.0010608620941638947,
-      "rewards/rejected": 0.012775774113833904,
+      "logits/chosen": -1.9828882217407227,
+      "logits/rejected": -1.986853003501892,
+      "logps/chosen": -33.40575408935547,
+      "logps/rejected": -31.552501678466797,
+      "loss": 0.4976,
+      "rewards/accuracies": 0.48750001192092896,
+      "rewards/chosen": 0.00937105156481266,
+      "rewards/margins": 0.011929656378924847,
+      "rewards/rejected": -0.0025586034171283245,
       "step": 230
     },
     {
       "epoch": 0.62,
       "learning_rate": 1.8712423238279356e-07,
-      "logits/chosen": -1.983289122581482,
-      "logits/rejected": -1.9613111019134521,
-      "logps/chosen": -34.167720794677734,
-      "logps/rejected": -34.96076965332031,
-      "loss": 0.6978,
-      "rewards/accuracies": 0.550000011920929,
-      "rewards/chosen": -0.011179247871041298,
-      "rewards/margins": -0.0031959381885826588,
-      "rewards/rejected": -0.007983307354152203,
+      "logits/chosen": -1.982568383216858,
+      "logits/rejected": -1.9606094360351562,
+      "logps/chosen": -34.157630920410156,
+      "logps/rejected": -34.96028518676758,
+      "loss": 0.4991,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -0.00310176657512784,
+      "rewards/margins": 0.004490714054554701,
+      "rewards/rejected": -0.007592480629682541,
       "step": 240
     },
     {
       "epoch": 0.65,
       "learning_rate": 1.654436768970182e-07,
-      "logits/chosen": -2.0248379707336426,
-      "logits/rejected": -2.0215275287628174,
-      "logps/chosen": -32.91853713989258,
-      "logps/rejected": -36.22992706298828,
-      "loss": 0.6918,
-      "rewards/accuracies": 0.5249999761581421,
-      "rewards/chosen": 0.005225626286119223,
-      "rewards/margins": 0.009226549416780472,
-      "rewards/rejected": -0.004000924527645111,
+      "logits/chosen": -2.024078845977783,
+      "logits/rejected": -2.0207676887512207,
+      "logps/chosen": -32.89586639404297,
+      "logps/rejected": -36.22296905517578,
+      "loss": 0.4952,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.023358644917607307,
+      "rewards/margins": 0.02179408073425293,
+      "rewards/rejected": 0.0015645644161850214,
       "step": 250
     },
     {
       "epoch": 0.68,
       "learning_rate": 1.444597403062196e-07,
-      "logits/chosen": -1.8912242650985718,
-      "logits/rejected": -1.8887859582901,
-      "logps/chosen": -34.17466735839844,
-      "logps/rejected": -35.498863220214844,
-      "loss": 0.6925,
-      "rewards/accuracies": 0.5874999761581421,
-      "rewards/chosen": 0.01098463125526905,
-      "rewards/margins": 0.006378169171512127,
-      "rewards/rejected": 0.004606460686773062,
+      "logits/chosen": -1.8911396265029907,
+      "logits/rejected": -1.8886913061141968,
+      "logps/chosen": -34.1867790222168,
+      "logps/rejected": -35.52009201049805,
+      "loss": 0.4978,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": 0.0012989563401788473,
+      "rewards/margins": 0.013674641028046608,
+      "rewards/rejected": -0.012375684455037117,
       "step": 260
     },
     {
       "epoch": 0.7,
       "learning_rate": 1.2434529917578887e-07,
-      "logits/chosen": -1.8761816024780273,
-      "logits/rejected": -1.8736416101455688,
-      "logps/chosen": -34.40642166137695,
-      "logps/rejected": -31.732311248779297,
-      "loss": 0.7147,
-      "rewards/accuracies": 0.36250001192092896,
-      "rewards/chosen": -0.007056808564811945,
-      "rewards/margins": -0.03827430680394173,
-      "rewards/rejected": 0.03121749497950077,
+      "logits/chosen": -1.8761117458343506,
+      "logits/rejected": -1.8735707998275757,
+      "logps/chosen": -34.38152313232422,
+      "logps/rejected": -31.744131088256836,
+      "loss": 0.5019,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": 0.012860517017543316,
+      "rewards/margins": -0.008900386281311512,
+      "rewards/rejected": 0.02176090143620968,
       "step": 270
     },
     {
       "epoch": 0.73,
       "learning_rate": 1.0526606671603521e-07,
-      "logits/chosen": -1.9794687032699585,
-      "logits/rejected": -1.9688432216644287,
-      "logps/chosen": -35.32493591308594,
-      "logps/rejected": -31.86197853088379,
-      "loss": 0.6757,
-      "rewards/accuracies": 0.5874999761581421,
-      "rewards/chosen": 0.025365391746163368,
-      "rewards/margins": 0.04215434193611145,
-      "rewards/rejected": -0.016788948327302933,
+      "logits/chosen": -1.979835867881775,
+      "logits/rejected": -1.9692022800445557,
+      "logps/chosen": -35.314170837402344,
+      "logps/rejected": -31.835962295532227,
+      "loss": 0.4935,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": 0.033973388373851776,
+      "rewards/margins": 0.029950443655252457,
+      "rewards/rejected": 0.004022946115583181,
       "step": 280
     },
     {
       "epoch": 0.75,
       "learning_rate": 8.737922755071453e-08,
-      "logits/chosen": -2.0760245323181152,
-      "logits/rejected": -2.0610060691833496,
-      "logps/chosen": -30.925411224365234,
-      "logps/rejected": -32.648170471191406,
-      "loss": 0.6978,
-      "rewards/accuracies": 0.4625000059604645,
-      "rewards/chosen": -0.0010284179588779807,
-      "rewards/margins": -0.003961210139095783,
-      "rewards/rejected": 0.0029327913653105497,
+      "logits/chosen": -2.075986385345459,
+      "logits/rejected": -2.0609803199768066,
+      "logps/chosen": -30.91201400756836,
+      "logps/rejected": -32.63774871826172,
+      "loss": 0.5005,
+      "rewards/accuracies": 0.4749999940395355,
+      "rewards/chosen": 0.009686904028058052,
+      "rewards/margins": -0.0015827339375391603,
+      "rewards/rejected": 0.011269642040133476,
       "step": 290
     },
     {
       "epoch": 0.78,
       "learning_rate": 7.08321427484816e-08,
-      "logits/chosen": -1.946812629699707,
-      "logits/rejected": -1.9442851543426514,
-      "logps/chosen": -32.90645217895508,
-      "logps/rejected": -30.836864471435547,
-      "loss": 0.6798,
-      "rewards/accuracies": 0.5625,
-      "rewards/chosen": 0.016527386382222176,
-      "rewards/margins": 0.032571423798799515,
-      "rewards/rejected": -0.016044041141867638,
+      "logits/chosen": -1.9475170373916626,
+      "logits/rejected": -1.944972038269043,
+      "logps/chosen": -32.88249969482422,
+      "logps/rejected": -30.8377685546875,
+      "loss": 0.4901,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.03568952530622482,
+      "rewards/margins": 0.052456192672252655,
+      "rewards/rejected": -0.01676667109131813,
       "step": 300
     },
     {
       "epoch": 0.78,
-      "eval_logits/chosen": -2.243879556655884,
-      "eval_logits/rejected": -2.239006996154785,
-      "eval_logps/chosen": -34.026798248291016,
-      "eval_logps/rejected": -37.50396728515625,
-      "eval_loss": 0.6981266140937805,
-      "eval_rewards/accuracies": 0.49833887815475464,
-      "eval_rewards/chosen": 0.006204271223396063,
-      "eval_rewards/margins": -0.003920448012650013,
-      "eval_rewards/rejected": 0.010124719701707363,
-      "eval_runtime": 145.9481,
+      "eval_logits/chosen": -2.244415283203125,
+      "eval_logits/rejected": -2.2395384311676025,
+      "eval_logps/chosen": -34.01211166381836,
+      "eval_logps/rejected": -37.495113372802734,
+      "eval_loss": 0.5000770092010498,
+      "eval_rewards/accuracies": 0.5045680999755859,
+      "eval_rewards/chosen": 0.01795424334704876,
+      "eval_rewards/margins": 0.0007461770437657833,
+      "eval_rewards/rejected": 0.01720806397497654,
+      "eval_runtime": 145.9415,
       "eval_samples_per_second": 2.35,
       "eval_steps_per_second": 0.295,
       "step": 300
@@ -493,123 +493,123 @@
     {
       "epoch": 0.81,
       "learning_rate": 5.576113578589034e-08,
-      "logits/chosen": -1.9286400079727173,
-      "logits/rejected": -1.92538583278656,
-      "logps/chosen": -31.571802139282227,
-      "logps/rejected": -33.7106819152832,
-      "loss": 0.6961,
-      "rewards/accuracies": 0.5375000238418579,
-      "rewards/chosen": 0.025238346308469772,
-      "rewards/margins": 5.8975441788788885e-05,
-      "rewards/rejected": 0.025179371237754822,
+      "logits/chosen": -1.9293571710586548,
+      "logits/rejected": -1.926099419593811,
+      "logps/chosen": -31.5543212890625,
+      "logps/rejected": -33.74175262451172,
+      "loss": 0.4912,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.039222296327352524,
+      "rewards/margins": 0.03890404850244522,
+      "rewards/rejected": 0.00031825463520362973,
       "step": 310
     },
     {
       "epoch": 0.83,
       "learning_rate": 4.229036944380912e-08,
-      "logits/chosen": -1.9804341793060303,
-      "logits/rejected": -1.9681293964385986,
-      "logps/chosen": -34.57010269165039,
-      "logps/rejected": -33.552330017089844,
-      "loss": 0.6803,
-      "rewards/accuracies": 0.5,
-      "rewards/chosen": 0.020780155435204506,
-      "rewards/margins": 0.031573377549648285,
-      "rewards/rejected": -0.01079322025179863,
+      "logits/chosen": -1.9810116291046143,
+      "logits/rejected": -1.9687116146087646,
+      "logps/chosen": -34.56316375732422,
+      "logps/rejected": -33.561100006103516,
+      "loss": 0.4923,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.026327725499868393,
+      "rewards/margins": 0.04413483291864395,
+      "rewards/rejected": -0.01780710555613041,
       "step": 320
     },
     {
       "epoch": 0.86,
       "learning_rate": 3.053082288996112e-08,
-      "logits/chosen": -2.0159571170806885,
-      "logits/rejected": -2.0144896507263184,
-      "logps/chosen": -33.452049255371094,
-      "logps/rejected": -32.46733856201172,
-      "loss": 0.687,
-      "rewards/accuracies": 0.5,
-      "rewards/chosen": 0.026994183659553528,
-      "rewards/margins": 0.017200354486703873,
-      "rewards/rejected": 0.009793824516236782,
+      "logits/chosen": -2.0162246227264404,
+      "logits/rejected": -2.014770269393921,
+      "logps/chosen": -33.477134704589844,
+      "logps/rejected": -32.470088958740234,
+      "loss": 0.4997,
+      "rewards/accuracies": 0.4749999940395355,
+      "rewards/chosen": 0.006926077418029308,
+      "rewards/margins": -0.0006712455069646239,
+      "rewards/rejected": 0.007597323507070541,
       "step": 330
     },
     {
       "epoch": 0.88,
       "learning_rate": 2.05793773749158e-08,
-      "logits/chosen": -2.103339433670044,
-      "logits/rejected": -2.0875279903411865,
-      "logps/chosen": -34.17384719848633,
-      "logps/rejected": -33.08417510986328,
-      "loss": 0.7089,
-      "rewards/accuracies": 0.4375,
-      "rewards/chosen": 0.004437476396560669,
-      "rewards/margins": -0.02264832705259323,
-      "rewards/rejected": 0.02708580158650875,
+      "logits/chosen": -2.103529214859009,
+      "logits/rejected": -2.0877299308776855,
+      "logps/chosen": -34.1456298828125,
+      "logps/rejected": -33.083770751953125,
+      "loss": 0.5012,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": 0.027009794488549232,
+      "rewards/margins": -0.0004009060503449291,
+      "rewards/rejected": 0.02741069719195366,
       "step": 340
     },
     {
       "epoch": 0.91,
       "learning_rate": 1.251801807404168e-08,
-      "logits/chosen": -1.974784255027771,
-      "logits/rejected": -1.9738404750823975,
-      "logps/chosen": -33.24315643310547,
-      "logps/rejected": -32.44605255126953,
-      "loss": 0.6895,
-      "rewards/accuracies": 0.4625000059604645,
-      "rewards/chosen": 0.031151747331023216,
-      "rewards/margins": 0.01587487757205963,
-      "rewards/rejected": 0.015276871621608734,
+      "logits/chosen": -1.9753166437149048,
+      "logits/rejected": -1.974352478981018,
+      "logps/chosen": -33.25692367553711,
+      "logps/rejected": -32.45876693725586,
+      "loss": 0.4973,
+      "rewards/accuracies": 0.42500001192092896,
+      "rewards/chosen": 0.02013799361884594,
+      "rewards/margins": 0.015033388510346413,
+      "rewards/rejected": 0.005104603711515665,
       "step": 350
     },
     {
       "epoch": 0.94,
       "learning_rate": 6.41315865106129e-09,
-      "logits/chosen": -1.9305660724639893,
-      "logits/rejected": -1.940930724143982,
-      "logps/chosen": -32.193580627441406,
-      "logps/rejected": -35.30970764160156,
-      "loss": 0.6828,
-      "rewards/accuracies": 0.5375000238418579,
-      "rewards/chosen": 0.008886909112334251,
-      "rewards/margins": 0.025648051872849464,
-      "rewards/rejected": -0.016761142760515213,
+      "logits/chosen": -1.931349515914917,
+      "logits/rejected": -1.941706895828247,
+      "logps/chosen": -32.206031799316406,
+      "logps/rejected": -35.306983947753906,
+      "loss": 0.4989,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -0.001073227496817708,
+      "rewards/margins": 0.013507463037967682,
+      "rewards/rejected": -0.01458069123327732,
       "step": 360
     },
     {
       "epoch": 0.96,
       "learning_rate": 2.3150941078050324e-09,
-      "logits/chosen": -2.069566249847412,
-      "logits/rejected": -2.0630078315734863,
-      "logps/chosen": -33.643856048583984,
-      "logps/rejected": -29.229293823242188,
-      "loss": 0.6922,
-      "rewards/accuracies": 0.5,
-      "rewards/chosen": 0.00649127596989274,
-      "rewards/margins": 0.008784732781350613,
-      "rewards/rejected": -0.0022934596054255962,
+      "logits/chosen": -2.070504665374756,
+      "logits/rejected": -2.0639472007751465,
+      "logps/chosen": -33.65058135986328,
+      "logps/rejected": -29.202342987060547,
+      "loss": 0.504,
+      "rewards/accuracies": 0.4124999940395355,
+      "rewards/chosen": 0.0011108577018603683,
+      "rewards/margins": -0.018153894692659378,
+      "rewards/rejected": 0.01926475390791893,
       "step": 370
     },
     {
       "epoch": 0.99,
       "learning_rate": 2.575864278703266e-10,
-      "logits/chosen": -1.929059386253357,
-      "logits/rejected": -1.931227445602417,
-      "logps/chosen": -34.235416412353516,
-      "logps/rejected": -30.882593154907227,
-      "loss": 0.6913,
-      "rewards/accuracies": 0.550000011920929,
-      "rewards/chosen": 0.0029094829224050045,
-      "rewards/margins": 0.010880164802074432,
-      "rewards/rejected": -0.00797068141400814,
+      "logits/chosen": -1.9301570653915405,
+      "logits/rejected": -1.9323084354400635,
+      "logps/chosen": -34.23206329345703,
+      "logps/rejected": -30.903820037841797,
+      "loss": 0.4945,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": 0.005591380409896374,
+      "rewards/margins": 0.03054152801632881,
+      "rewards/rejected": -0.024950148537755013,
       "step": 380
     },
     {
       "epoch": 1.0,
       "step": 385,
       "total_flos": 0.0,
-      "train_loss": 0.15268191238502404,
-      "train_runtime": 627.0699,
-      "train_samples_per_second": 4.91,
-      "train_steps_per_second": 0.614
+      "train_loss": 0.10985468208015739,
+      "train_runtime": 628.1848,
+      "train_samples_per_second": 4.901,
+      "train_steps_per_second": 0.613
     }
   ],
   "logging_steps": 10,

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:550e24d9e9b94f0f2ebf295ebd168faebda884d2c9f63f5ec0077fc574be4f0f
+oid sha256:c754675d481910ddc15e9e9defeda5a0d742f000e290a0d069fdbb963b9a9e39
 size 4984