Model save

README.md

@@ -1,13 +1,11 @@
 ---
+license: apache-2.0
 library_name: peft
 tags:
-- alignment-handbook
 - trl
 - dpo
 - generated_from_trainer
 base_model: norallm/normistral-7b-warm
-datasets:
-- hugodk-sch/aftonposten_title_prefs
 model-index:
 - name: ap-normistral-7b-align-scan
   results: []
@@ -18,17 +16,17 @@ should probably proofread and complete it, then remove this comment. -->
 
 # ap-normistral-7b-align-scan
 
-This model is a fine-tuned version of [data/ap-normistral-7b-sft-qlora](https://huggingface.co/data/ap-normistral-7b-sft-qlora) on the hugodk-sch/aftonposten_title_prefs dataset.
+This model is a fine-tuned version of [norallm/normistral-7b-warm](https://huggingface.co/norallm/normistral-7b-warm) on the None dataset.
 It achieves the following results on the evaluation set:
-- Loss: 0.9517
-- Rewards/chosen: -0.1032
-- Rewards/rejected: -0.1569
-- Rewards/accuracies: 0.5191
-- Rewards/margins: 0.0537
-- Logps/rejected: -36.4896
-- Logps/chosen: -32.7871
-- Logits/rejected: 98.4366
-- Logits/chosen: 98.4655
+- Loss: 0.9947
+- Rewards/chosen: -0.0086
+- Rewards/rejected: -0.0139
+- Rewards/accuracies: 0.5889
+- Rewards/margins: 0.0053
+- Logps/rejected: -37.3559
+- Logps/chosen: -33.3009
+- Logits/rejected: 97.6402
+- Logits/chosen: 97.6739
 
 ## Model description
 
@@ -63,9 +61,9 @@ The following hyperparameters were used during training:
 
 | Training Loss | Epoch | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
 |:-------------:|:-----:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
-| 0.872         | 0.26  | 100  | 1.0137          | -0.0220        | -0.0084          | 0.4871             | -0.0136         | -35.9946       | -32.5165     | 98.7181         | 98.7288       |
-| 0.6817        | 0.52  | 200  | 0.9317          | -0.0794        | -0.1540          | 0.5428             | 0.0747          | -36.4800       | -32.7078     | 98.4671         | 98.4916       |
-| 0.73          | 0.78  | 300  | 0.9506          | -0.1185        | -0.1765          | 0.5075             | 0.0580          | -36.5547       | -32.8381     | 98.4467         | 98.4738       |
+| 0.9955        | 0.26  | 100  | 0.9999          | 0.0005         | 0.0004           | 0.5187             | 0.0001          | -35.9246       | -32.3899     | 98.7274         | 98.7339       |
+| 0.9858        | 0.52  | 200  | 0.9965          | -0.0053        | -0.0088          | 0.5274             | 0.0035          | -36.8453       | -32.9718     | 98.0264         | 98.0513       |
+| 0.9814        | 0.78  | 300  | 0.9947          | -0.0086        | -0.0139          | 0.5889             | 0.0053          | -37.3559       | -33.3009     | 97.6402         | 97.6739       |
 
 
 ### Framework versions

adapter_config.json

@@ -20,13 +20,13 @@
   "rank_pattern": {},
   "revision": null,
   "target_modules": [
-    "down_proj",
-    "up_proj",
     "v_proj",
     "gate_proj",
-    "q_proj",
+    "k_proj",
+    "up_proj",
+    "down_proj",
     "o_proj",
-    "k_proj"
+    "q_proj"
   ],
   "task_type": "CAUSAL_LM",
   "use_dora": false,

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f038d58dcb39e899751aab427858b6f6f9e03fe87048e48d6e1b921b10bfe9f4
+oid sha256:7702b700f2751f66b1062ece0daa55aac903e961de453647239e392c926c3091
 size 671150064

all_results.json

@@ -1,21 +1,8 @@
 {
     "epoch": 1.0,
-    "eval_logits/chosen": 98.46549224853516,
-    "eval_logits/rejected": 98.43663787841797,
-    "eval_logps/chosen": -32.78712463378906,
-    "eval_logps/rejected": -36.48957443237305,
-    "eval_loss": 0.9517204761505127,
-    "eval_rewards/accuracies": 0.5191029906272888,
-    "eval_rewards/chosen": -0.10318472981452942,
-    "eval_rewards/margins": 0.0537295788526535,
-    "eval_rewards/rejected": -0.15691432356834412,
-    "eval_runtime": 103.7591,
-    "eval_samples": 343,
-    "eval_samples_per_second": 3.306,
-    "eval_steps_per_second": 0.414,
-    "train_loss": 0.7626325495831379,
-    "train_runtime": 2558.2938,
+    "train_loss": 0.9878765378679548,
+    "train_runtime": 2559.7639,
     "train_samples": 3079,
-    "train_samples_per_second": 1.204,
+    "train_samples_per_second": 1.203,
     "train_steps_per_second": 0.15
 }

train_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 1.0,
-    "train_loss": 0.7626325495831379,
-    "train_runtime": 2558.2938,
+    "train_loss": 0.9878765378679548,
+    "train_runtime": 2559.7639,
     "train_samples": 3079,
-    "train_samples_per_second": 1.204,
+    "train_samples_per_second": 1.203,
     "train_steps_per_second": 0.15
 }

trainer_state.json

@@ -10,7 +10,7 @@
   "log_history": [
     {
       "epoch": 0.0,
-      "grad_norm": 39.25,
+      "grad_norm": 1.3046875,
       "learning_rate": 1.282051282051282e-07,
       "logits/chosen": 88.18099975585938,
       "logits/rejected": 88.25153350830078,
@@ -25,629 +25,629 @@
     },
     {
       "epoch": 0.03,
-      "grad_norm": 32.5,
+      "grad_norm": 1.09375,
       "learning_rate": 1.282051282051282e-06,
-      "logits/chosen": 81.0865478515625,
-      "logits/rejected": 80.79108428955078,
-      "logps/chosen": -34.190330505371094,
-      "logps/rejected": -33.059818267822266,
-      "loss": 0.9542,
-      "rewards/accuracies": 0.4861111044883728,
-      "rewards/chosen": 0.015418142080307007,
-      "rewards/margins": 0.04584219306707382,
-      "rewards/rejected": -0.030424050986766815,
+      "logits/chosen": 81.09645080566406,
+      "logits/rejected": 80.80389404296875,
+      "logps/chosen": -34.27156066894531,
+      "logps/rejected": -33.039093017578125,
+      "loss": 0.9995,
+      "rewards/accuracies": 0.4722222089767456,
+      "rewards/chosen": -0.00029834467568434775,
+      "rewards/margins": 0.0005084889708086848,
+      "rewards/rejected": -0.0008068337920121849,
       "step": 10
     },
     {
       "epoch": 0.05,
-      "grad_norm": 37.5,
+      "grad_norm": 1.2734375,
       "learning_rate": 2.564102564102564e-06,
-      "logits/chosen": 80.6707534790039,
-      "logits/rejected": 80.55946350097656,
-      "logps/chosen": -33.50497055053711,
-      "logps/rejected": -30.738788604736328,
-      "loss": 0.9561,
-      "rewards/accuracies": 0.550000011920929,
-      "rewards/chosen": 0.055480264127254486,
-      "rewards/margins": 0.04386957734823227,
-      "rewards/rejected": 0.01161068957298994,
+      "logits/chosen": 80.66552734375,
+      "logits/rejected": 80.5560073852539,
+      "logps/chosen": -33.4774055480957,
+      "logps/rejected": -30.691213607788086,
+      "loss": 0.9987,
+      "rewards/accuracies": 0.4375,
+      "rewards/chosen": 0.0021250424906611443,
+      "rewards/margins": 0.0012622694484889507,
+      "rewards/rejected": 0.000862772751133889,
       "step": 20
     },
     {
       "epoch": 0.08,
-      "grad_norm": 36.25,
+      "grad_norm": 1.2109375,
       "learning_rate": 3.846153846153847e-06,
-      "logits/chosen": 82.51221466064453,
-      "logits/rejected": 82.54509735107422,
-      "logps/chosen": -33.85924530029297,
-      "logps/rejected": -31.118642807006836,
-      "loss": 1.0455,
+      "logits/chosen": 82.51115417480469,
+      "logits/rejected": 82.54508972167969,
+      "logps/chosen": -33.80036926269531,
+      "logps/rejected": -31.189748764038086,
+      "loss": 1.0002,
       "rewards/accuracies": 0.4749999940395355,
-      "rewards/chosen": 0.046072401106357574,
-      "rewards/margins": -0.045484863221645355,
-      "rewards/rejected": 0.09155726432800293,
+      "rewards/chosen": 0.002124499063938856,
+      "rewards/margins": -0.00021631647541653365,
+      "rewards/rejected": 0.0023408152628690004,
       "step": 30
     },
     {
       "epoch": 0.1,
-      "grad_norm": 28.0,
+      "grad_norm": 1.1796875,
       "learning_rate": 4.999896948438434e-06,
-      "logits/chosen": 81.0888900756836,
-      "logits/rejected": 81.08549499511719,
-      "logps/chosen": -32.762596130371094,
-      "logps/rejected": -33.211116790771484,
-      "loss": 0.9404,
-      "rewards/accuracies": 0.574999988079071,
-      "rewards/chosen": 0.12344050407409668,
-      "rewards/margins": 0.07621846348047256,
-      "rewards/rejected": 0.04722203314304352,
+      "logits/chosen": 81.10090637207031,
+      "logits/rejected": 81.09576416015625,
+      "logps/chosen": -32.7674560546875,
+      "logps/rejected": -33.11550521850586,
+      "loss": 0.9985,
+      "rewards/accuracies": 0.48750001192092896,
+      "rewards/chosen": 0.0040660640224814415,
+      "rewards/margins": 0.0015358638484030962,
+      "rewards/rejected": 0.0025301999412477016,
       "step": 40
     },
     {
       "epoch": 0.13,
-      "grad_norm": 25.75,
+      "grad_norm": 1.1875,
       "learning_rate": 4.987541037542187e-06,
-      "logits/chosen": 78.80292510986328,
-      "logits/rejected": 78.81251525878906,
-      "logps/chosen": -30.42880630493164,
-      "logps/rejected": -30.703338623046875,
-      "loss": 0.9134,
-      "rewards/accuracies": 0.550000011920929,
-      "rewards/chosen": 0.23293299973011017,
-      "rewards/margins": 0.1219487413764,
-      "rewards/rejected": 0.11098428070545197,
+      "logits/chosen": 78.85154724121094,
+      "logits/rejected": 78.85734558105469,
+      "logps/chosen": -30.360393524169922,
+      "logps/rejected": -30.609283447265625,
+      "loss": 0.9962,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": 0.008448543958365917,
+      "rewards/margins": 0.0038085163105279207,
+      "rewards/rejected": 0.0046400283463299274,
       "step": 50
     },
     {
       "epoch": 0.16,
-      "grad_norm": 29.875,
+      "grad_norm": 0.96484375,
       "learning_rate": 4.954691471941119e-06,
-      "logits/chosen": 83.4697265625,
-      "logits/rejected": 83.52468872070312,
-      "logps/chosen": -30.678720474243164,
-      "logps/rejected": -29.31003761291504,
-      "loss": 0.977,
-      "rewards/accuracies": 0.48750001192092896,
-      "rewards/chosen": 0.14893214404582977,
-      "rewards/margins": 0.0647866353392601,
-      "rewards/rejected": 0.08414552360773087,
+      "logits/chosen": 83.49021911621094,
+      "logits/rejected": 83.54866027832031,
+      "logps/chosen": -30.763973236083984,
+      "logps/rejected": -29.17538833618164,
+      "loss": 1.0,
+      "rewards/accuracies": 0.4124999940395355,
+      "rewards/chosen": 0.004111888352781534,
+      "rewards/margins": -3.9446913433494046e-05,
+      "rewards/rejected": 0.004151335451751947,
       "step": 60
     },
     {
       "epoch": 0.18,
-      "grad_norm": 36.0,
+      "grad_norm": 1.2109375,
       "learning_rate": 4.901618883413549e-06,
-      "logits/chosen": 84.11613464355469,
-      "logits/rejected": 84.15023040771484,
-      "logps/chosen": -30.336284637451172,
-      "logps/rejected": -32.72249221801758,
-      "loss": 1.0102,
-      "rewards/accuracies": 0.4749999940395355,
-      "rewards/chosen": 0.08782148361206055,
-      "rewards/margins": -0.006742718629539013,
-      "rewards/rejected": 0.09456419944763184,
+      "logits/chosen": 84.11228942871094,
+      "logits/rejected": 84.1441650390625,
+      "logps/chosen": -30.222454071044922,
+      "logps/rejected": -32.666595458984375,
+      "loss": 0.9996,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": 0.0040657008066773415,
+      "rewards/margins": 0.0003545849467627704,
+      "rewards/rejected": 0.003711115103214979,
       "step": 70
     },
     {
       "epoch": 0.21,
-      "grad_norm": 38.25,
+      "grad_norm": 1.2421875,
       "learning_rate": 4.828760511501322e-06,
-      "logits/chosen": 81.8536148071289,
-      "logits/rejected": 81.83476257324219,
-      "logps/chosen": -31.154577255249023,
-      "logps/rejected": -30.83746910095215,
-      "loss": 0.8907,
-      "rewards/accuracies": 0.5375000238418579,
-      "rewards/chosen": 0.10510905086994171,
-      "rewards/margins": 0.12905548512935638,
-      "rewards/rejected": -0.023946449160575867,
+      "logits/chosen": 81.86946868896484,
+      "logits/rejected": 81.84814453125,
+      "logps/chosen": -30.959096908569336,
+      "logps/rejected": -30.652545928955078,
+      "loss": 0.9956,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.005458436906337738,
+      "rewards/margins": 0.0044073979370296,
+      "rewards/rejected": 0.001051038852892816,
       "step": 80
     },
     {
       "epoch": 0.23,
-      "grad_norm": 40.5,
+      "grad_norm": 1.484375,
       "learning_rate": 4.7367166013034295e-06,
-      "logits/chosen": 78.57433319091797,
-      "logits/rejected": 78.5478744506836,
-      "logps/chosen": -32.25961685180664,
-      "logps/rejected": -30.962270736694336,
-      "loss": 0.899,
+      "logits/chosen": 78.68418884277344,
+      "logits/rejected": 78.65721893310547,
+      "logps/chosen": -32.17829513549805,
+      "logps/rejected": -30.884775161743164,
+      "loss": 0.9961,
       "rewards/accuracies": 0.5874999761581421,
-      "rewards/chosen": 0.1135696992278099,
-      "rewards/margins": 0.1168096661567688,
-      "rewards/rejected": -0.0032399550545960665,
+      "rewards/chosen": 0.00459885410964489,
+      "rewards/margins": 0.00393189862370491,
+      "rewards/rejected": 0.0006669552531093359,
       "step": 90
     },
     {
       "epoch": 0.26,
-      "grad_norm": 35.25,
+      "grad_norm": 1.2421875,
       "learning_rate": 4.626245458345211e-06,
-      "logits/chosen": 83.61376190185547,
-      "logits/rejected": 83.63221740722656,
-      "logps/chosen": -33.95526123046875,
-      "logps/rejected": -31.850500106811523,
-      "loss": 0.872,
-      "rewards/accuracies": 0.637499988079071,
-      "rewards/chosen": 0.10491885989904404,
-      "rewards/margins": 0.13490620255470276,
-      "rewards/rejected": -0.02998734638094902,
+      "logits/chosen": 83.70716857910156,
+      "logits/rejected": 83.73751068115234,
+      "logps/chosen": -33.73701477050781,
+      "logps/rejected": -31.63702964782715,
+      "loss": 0.9955,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": 0.005679761990904808,
+      "rewards/margins": 0.004544637631624937,
+      "rewards/rejected": 0.0011351245921105146,
       "step": 100
     },
     {
       "epoch": 0.26,
-      "eval_logits/chosen": 98.72882080078125,
-      "eval_logits/rejected": 98.71807098388672,
-      "eval_logps/chosen": -32.516475677490234,
-      "eval_logps/rejected": -35.99455642700195,
-      "eval_loss": 1.0136585235595703,
-      "eval_rewards/accuracies": 0.4871262311935425,
-      "eval_rewards/chosen": -0.02198966220021248,
-      "eval_rewards/margins": -0.013580878265202045,
-      "eval_rewards/rejected": -0.008408783003687859,
-      "eval_runtime": 104.3802,
-      "eval_samples_per_second": 3.286,
-      "eval_steps_per_second": 0.412,
+      "eval_logits/chosen": 98.73394012451172,
+      "eval_logits/rejected": 98.7273941040039,
+      "eval_logps/chosen": -32.38990783691406,
+      "eval_logps/rejected": -35.92463684082031,
+      "eval_loss": 0.999876856803894,
+      "eval_rewards/accuracies": 0.5186877250671387,
+      "eval_rewards/chosen": 0.0005326389218680561,
+      "eval_rewards/margins": 0.00011375291069271043,
+      "eval_rewards/rejected": 0.00041888616397045553,
+      "eval_runtime": 104.2424,
+      "eval_samples_per_second": 3.29,
+      "eval_steps_per_second": 0.413,
       "step": 100
     },
     {
       "epoch": 0.29,
-      "grad_norm": 42.5,
+      "grad_norm": 1.4140625,
       "learning_rate": 4.498257201263691e-06,
-      "logits/chosen": 83.79080963134766,
-      "logits/rejected": 83.66963195800781,
-      "logps/chosen": -32.27449417114258,
-      "logps/rejected": -32.75922393798828,
-      "loss": 0.7823,
-      "rewards/accuracies": 0.6625000238418579,
-      "rewards/chosen": 0.22449319064617157,
-      "rewards/margins": 0.2554941177368164,
-      "rewards/rejected": -0.03100091777741909,
+      "logits/chosen": 83.85816955566406,
+      "logits/rejected": 83.75128936767578,
+      "logps/chosen": -32.19211959838867,
+      "logps/rejected": -32.65901565551758,
+      "loss": 0.9917,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": 0.008306830190122128,
+      "rewards/margins": 0.008338114246726036,
+      "rewards/rejected": -3.12842421408277e-05,
       "step": 110
     },
     {
       "epoch": 0.31,
-      "grad_norm": 39.0,
+      "grad_norm": 1.3046875,
       "learning_rate": 4.353806263777678e-06,
-      "logits/chosen": 83.8546371459961,
-      "logits/rejected": 83.96675872802734,
-      "logps/chosen": -28.1826114654541,
-      "logps/rejected": -35.42960739135742,
-      "loss": 0.8179,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": 0.22409120202064514,
-      "rewards/margins": 0.2152291238307953,
-      "rewards/rejected": 0.008862084709107876,
+      "logits/chosen": 83.82106018066406,
+      "logits/rejected": 83.92265319824219,
+      "logps/chosen": -28.150625228881836,
+      "logps/rejected": -35.3939208984375,
+      "loss": 0.9929,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.007789556868374348,
+      "rewards/margins": 0.0071373311802744865,
+      "rewards/rejected": 0.000652224407531321,
       "step": 120
     },
     {
       "epoch": 0.34,
-      "grad_norm": 28.5,
+      "grad_norm": 0.9453125,
       "learning_rate": 4.1940827077152755e-06,
-      "logits/chosen": 81.01178741455078,
-      "logits/rejected": 81.0390853881836,
-      "logps/chosen": -30.35846519470215,
-      "logps/rejected": -32.0872688293457,
-      "loss": 0.8393,
-      "rewards/accuracies": 0.612500011920929,
-      "rewards/chosen": 0.16466763615608215,
-      "rewards/margins": 0.2113606482744217,
-      "rewards/rejected": -0.04669303447008133,
+      "logits/chosen": 80.96563720703125,
+      "logits/rejected": 80.99563598632812,
+      "logps/chosen": -30.216140747070312,
+      "logps/rejected": -31.844036102294922,
+      "loss": 0.994,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.006912143435329199,
+      "rewards/margins": 0.006036223843693733,
+      "rewards/rejected": 0.0008759202319197357,
       "step": 130
     },
     {
       "epoch": 0.36,
-      "grad_norm": 33.5,
+      "grad_norm": 1.1171875,
       "learning_rate": 4.0204024186666215e-06,
-      "logits/chosen": 82.01683044433594,
-      "logits/rejected": 82.02485656738281,
-      "logps/chosen": -26.93210220336914,
-      "logps/rejected": -33.207496643066406,
-      "loss": 0.6882,
-      "rewards/accuracies": 0.675000011920929,
-      "rewards/chosen": 0.1545664370059967,
-      "rewards/margins": 0.389803022146225,
-      "rewards/rejected": -0.23523661494255066,
+      "logits/chosen": 81.863525390625,
+      "logits/rejected": 81.86921691894531,
+      "logps/chosen": -26.845142364501953,
+      "logps/rejected": -33.07027816772461,
+      "loss": 0.9875,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": 0.006021805107593536,
+      "rewards/margins": 0.012490840628743172,
+      "rewards/rejected": -0.006469034589827061,
       "step": 140
     },
     {
       "epoch": 0.39,
-      "grad_norm": 28.0,
+      "grad_norm": 1.21875,
       "learning_rate": 3.834196265035119e-06,
-      "logits/chosen": 80.503662109375,
-      "logits/rejected": 80.47123718261719,
-      "logps/chosen": -28.8300724029541,
-      "logps/rejected": -33.28377151489258,
-      "loss": 0.6693,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": 0.16771158576011658,
-      "rewards/margins": 0.3828922212123871,
-      "rewards/rejected": -0.2151806354522705,
+      "logits/chosen": 80.13746643066406,
+      "logits/rejected": 80.10902404785156,
+      "logps/chosen": -28.976547241210938,
+      "logps/rejected": -33.208518981933594,
+      "loss": 0.9895,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.004125660751014948,
+      "rewards/margins": 0.010545835830271244,
+      "rewards/rejected": -0.006420175079256296,
       "step": 150
     },
     {
       "epoch": 0.42,
-      "grad_norm": 39.0,
+      "grad_norm": 1.3515625,
       "learning_rate": 3.636998309800573e-06,
-      "logits/chosen": 82.2752456665039,
-      "logits/rejected": 82.291015625,
-      "logps/chosen": -33.75586700439453,
-      "logps/rejected": -30.59908676147461,
-      "loss": 0.7469,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": 0.1142646074295044,
-      "rewards/margins": 0.3082302212715149,
-      "rewards/rejected": -0.19396564364433289,
+      "logits/chosen": 81.72142028808594,
+      "logits/rejected": 81.74298858642578,
+      "logps/chosen": -33.8978157043457,
+      "logps/rejected": -30.907711029052734,
+      "loss": 0.9881,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": 0.002389371395111084,
+      "rewards/margins": 0.011941083706915379,
+      "rewards/rejected": -0.009551710449159145,
       "step": 160
     },
     {
       "epoch": 0.44,
-      "grad_norm": 36.75,
+      "grad_norm": 1.4765625,
       "learning_rate": 3.4304331721118078e-06,
-      "logits/chosen": 83.02024841308594,
-      "logits/rejected": 82.96368408203125,
-      "logps/chosen": -30.91131591796875,
-      "logps/rejected": -32.768035888671875,
-      "loss": 0.7116,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": 0.10439153760671616,
-      "rewards/margins": 0.328784316778183,
-      "rewards/rejected": -0.22439277172088623,
+      "logits/chosen": 82.35487365722656,
+      "logits/rejected": 82.30474090576172,
+      "logps/chosen": -30.870525360107422,
+      "logps/rejected": -33.04078674316406,
+      "loss": 0.9859,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.003887615632265806,
+      "rewards/margins": 0.014094889163970947,
+      "rewards/rejected": -0.010207273997366428,
       "step": 170
     },
     {
       "epoch": 0.47,
-      "grad_norm": 28.625,
+      "grad_norm": 1.21875,
       "learning_rate": 3.2162026428305436e-06,
-      "logits/chosen": 80.55313110351562,
-      "logits/rejected": 80.5318832397461,
-      "logps/chosen": -30.63324546813965,
-      "logps/rejected": -31.834888458251953,
-      "loss": 0.6812,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": 0.16917188465595245,
-      "rewards/margins": 0.3724762201309204,
-      "rewards/rejected": -0.20330433547496796,
+      "logits/chosen": 79.4852066040039,
+      "logits/rejected": 79.46187591552734,
+      "logps/chosen": -31.02083396911621,
+      "logps/rejected": -32.165191650390625,
+      "loss": 0.9882,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.0017631975933909416,
+      "rewards/margins": 0.011843027547001839,
+      "rewards/rejected": -0.010079829022288322,
       "step": 180
     },
     {
       "epoch": 0.49,
-      "grad_norm": 22.0,
+      "grad_norm": 0.9609375,
       "learning_rate": 2.996071664294641e-06,
-      "logits/chosen": 82.1512451171875,
-      "logits/rejected": 82.14521789550781,
-      "logps/chosen": -30.498291015625,
-      "logps/rejected": -30.805980682373047,
-      "loss": 0.8556,
-      "rewards/accuracies": 0.5625,
-      "rewards/chosen": 0.08043722808361053,
-      "rewards/margins": 0.18182404339313507,
-      "rewards/rejected": -0.10138680785894394,
+      "logits/chosen": 80.97419738769531,
+      "logits/rejected": 80.94820404052734,
+      "logps/chosen": -30.60634994506836,
+      "logps/rejected": -31.083566665649414,
+      "loss": 0.9922,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.0016006485093384981,
+      "rewards/margins": 0.007756076753139496,
+      "rewards/rejected": -0.006155428942292929,
       "step": 190
     },
     {
       "epoch": 0.52,
-      "grad_norm": 30.75,
+      "grad_norm": 1.453125,
       "learning_rate": 2.7718537898066833e-06,
-      "logits/chosen": 77.57530975341797,
-      "logits/rejected": 77.51686096191406,
-      "logps/chosen": -33.855812072753906,
-      "logps/rejected": -32.983428955078125,
-      "loss": 0.6817,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": 0.2690780758857727,
-      "rewards/margins": 0.41608619689941406,
-      "rewards/rejected": -0.14700806140899658,
+      "logits/chosen": 76.12115478515625,
+      "logits/rejected": 76.07009887695312,
+      "logps/chosen": -34.18424606323242,
+      "logps/rejected": -33.341392517089844,
+      "loss": 0.9858,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.005684881471097469,
+      "rewards/margins": 0.014164777472615242,
+      "rewards/rejected": -0.008479896001517773,
       "step": 200
     },
     {
       "epoch": 0.52,
-      "eval_logits/chosen": 98.4915771484375,
-      "eval_logits/rejected": 98.4671401977539,
-      "eval_logps/chosen": -32.7077522277832,
-      "eval_logps/rejected": -36.48000717163086,
-      "eval_loss": 0.9316657185554504,
-      "eval_rewards/accuracies": 0.5427741408348083,
-      "eval_rewards/chosen": -0.07937218248844147,
-      "eval_rewards/margins": 0.0746726542711258,
-      "eval_rewards/rejected": -0.15404485166072845,
-      "eval_runtime": 104.1983,
-      "eval_samples_per_second": 3.292,
+      "eval_logits/chosen": 98.05126190185547,
+      "eval_logits/rejected": 98.02639770507812,
+      "eval_logps/chosen": -32.97175979614258,
+      "eval_logps/rejected": -36.845333099365234,
+      "eval_loss": 0.9964954853057861,
+      "eval_rewards/accuracies": 0.5274086594581604,
+      "eval_rewards/chosen": -0.005285844672471285,
+      "eval_rewards/margins": 0.0035022026859223843,
+      "eval_rewards/rejected": -0.008788047358393669,
+      "eval_runtime": 104.1082,
+      "eval_samples_per_second": 3.295,
       "eval_steps_per_second": 0.413,
       "step": 200
     },
     {
       "epoch": 0.55,
-      "grad_norm": 45.5,
+      "grad_norm": 1.84375,
       "learning_rate": 2.5453962426402006e-06,
-      "logits/chosen": 80.1883773803711,
-      "logits/rejected": 80.09809875488281,
-      "logps/chosen": -33.3426513671875,
-      "logps/rejected": -35.498653411865234,
-      "loss": 0.74,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": 0.1809217780828476,
-      "rewards/margins": 0.33469074964523315,
-      "rewards/rejected": -0.15376897156238556,
+      "logits/chosen": 78.64119720458984,
+      "logits/rejected": 78.55430603027344,
+      "logps/chosen": -33.689414978027344,
+      "logps/rejected": -36.20193862915039,
+      "loss": 0.9853,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.002563029993325472,
+      "rewards/margins": 0.014721485786139965,
+      "rewards/rejected": -0.012158457189798355,
       "step": 210
     },
     {
       "epoch": 0.57,
-      "grad_norm": 29.875,
+      "grad_norm": 1.484375,
       "learning_rate": 2.3185646976551794e-06,
-      "logits/chosen": 82.31951904296875,
-      "logits/rejected": 82.41423034667969,
-      "logps/chosen": -31.08917808532715,
-      "logps/rejected": -31.53971290588379,
-      "loss": 0.6287,
-      "rewards/accuracies": 0.7875000238418579,
-      "rewards/chosen": 0.25793319940567017,
-      "rewards/margins": 0.501160740852356,
-      "rewards/rejected": -0.2432275116443634,
+      "logits/chosen": 80.5840835571289,
+      "logits/rejected": 80.67861938476562,
+      "logps/chosen": -31.57720947265625,
+      "logps/rejected": -31.91719627380371,
+      "loss": 0.9844,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": 0.003717451822012663,
+      "rewards/margins": 0.015599893406033516,
+      "rewards/rejected": -0.011882440187036991,
       "step": 220
     },
     {
       "epoch": 0.6,
-      "grad_norm": 35.5,
+      "grad_norm": 1.3984375,
       "learning_rate": 2.0932279108998323e-06,
-      "logits/chosen": 79.45759582519531,
-      "logits/rejected": 79.51451110839844,
-      "logps/chosen": -32.35926055908203,
-      "logps/rejected": -34.52641677856445,
-      "loss": 0.7765,
-      "rewards/accuracies": 0.6625000238418579,
-      "rewards/chosen": 0.13394585251808167,
-      "rewards/margins": 0.2882245182991028,
-      "rewards/rejected": -0.1542786806821823,
+      "logits/chosen": 77.53582763671875,
+      "logits/rejected": 77.5838851928711,
+      "logps/chosen": -32.72165298461914,
+      "logps/rejected": -35.34224319458008,
+      "loss": 0.9859,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": 0.0008409392321482301,
+      "rewards/margins": 0.014141863211989403,
+      "rewards/rejected": -0.013300922699272633,
       "step": 230
     },
     {
       "epoch": 0.62,
-      "grad_norm": 40.0,
+      "grad_norm": 1.671875,
       "learning_rate": 1.8712423238279358e-06,
-      "logits/chosen": 81.74153137207031,
-      "logits/rejected": 82.05198669433594,
-      "logps/chosen": -30.749576568603516,
-      "logps/rejected": -31.873056411743164,
-      "loss": 0.6828,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": 0.28154754638671875,
-      "rewards/margins": 0.3838502764701843,
-      "rewards/rejected": -0.10230269283056259,
+      "logits/chosen": 79.64659118652344,
+      "logits/rejected": 79.958984375,
+      "logps/chosen": -31.332469940185547,
+      "logps/rejected": -32.86049270629883,
+      "loss": 0.9832,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": 0.003555959090590477,
+      "rewards/margins": 0.016840480268001556,
+      "rewards/rejected": -0.01328451931476593,
       "step": 240
     },
     {
       "epoch": 0.65,
-      "grad_norm": 40.25,
+      "grad_norm": 1.484375,
       "learning_rate": 1.6544367689701824e-06,
-      "logits/chosen": 80.39913940429688,
-      "logits/rejected": 80.4660415649414,
-      "logps/chosen": -27.163742065429688,
-      "logps/rejected": -30.3619327545166,
-      "loss": 0.8055,
-      "rewards/accuracies": 0.574999988079071,
-      "rewards/chosen": 0.12964634597301483,
-      "rewards/margins": 0.26800400018692017,
-      "rewards/rejected": -0.13835760951042175,
+      "logits/chosen": 78.12522888183594,
+      "logits/rejected": 78.17500305175781,
+      "logps/chosen": -27.822484970092773,
+      "logps/rejected": -31.34881019592285,
+      "loss": 0.9878,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.002265883143991232,
+      "rewards/margins": 0.012214846909046173,
+      "rewards/rejected": -0.014480730518698692,
       "step": 250
     },
     {
       "epoch": 0.68,
-      "grad_norm": 23.375,
+      "grad_norm": 1.6171875,
       "learning_rate": 1.4445974030621963e-06,
-      "logits/chosen": 77.59562683105469,
-      "logits/rejected": 77.73274230957031,
-      "logps/chosen": -30.47354507446289,
-      "logps/rejected": -36.71575164794922,
-      "loss": 0.6198,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": 0.3262353241443634,
-      "rewards/margins": 0.5104730129241943,
-      "rewards/rejected": -0.18423771858215332,
+      "logits/chosen": 75.08821105957031,
+      "logits/rejected": 75.22389221191406,
+      "logps/chosen": -31.112863540649414,
+      "logps/rejected": -38.40215301513672,
+      "loss": 0.9725,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.004481295123696327,
+      "rewards/margins": 0.027486557140946388,
+      "rewards/rejected": -0.023005260154604912,
       "step": 260
     },
     {
       "epoch": 0.7,
-      "grad_norm": 27.875,
+      "grad_norm": 1.3515625,
       "learning_rate": 1.243452991757889e-06,
-      "logits/chosen": 76.8821792602539,
-      "logits/rejected": 76.9117202758789,
-      "logps/chosen": -30.930156707763672,
-      "logps/rejected": -32.11249923706055,
-      "loss": 0.6578,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": 0.25463035702705383,
-      "rewards/margins": 0.43675345182418823,
-      "rewards/rejected": -0.1821231096982956,
+      "logits/chosen": 74.10564422607422,
+      "logits/rejected": 74.13673400878906,
+      "logps/chosen": -32.07135772705078,
+      "logps/rejected": -33.231197357177734,
+      "loss": 0.9857,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.002924318192526698,
+      "rewards/margins": 0.01433342695236206,
+      "rewards/rejected": -0.017257746309041977,
       "step": 270
     },
     {
       "epoch": 0.73,
-      "grad_norm": 46.5,
+      "grad_norm": 2.03125,
       "learning_rate": 1.0526606671603523e-06,
-      "logits/chosen": 79.59893798828125,
-      "logits/rejected": 79.37335205078125,
-      "logps/chosen": -31.25324058532715,
-      "logps/rejected": -30.070053100585938,
-      "loss": 0.7946,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": 0.16699600219726562,
-      "rewards/margins": 0.28791341185569763,
-      "rewards/rejected": -0.1209174171090126,
+      "logits/chosen": 77.0625991821289,
+      "logits/rejected": 76.84493255615234,
+      "logps/chosen": -32.438629150390625,
+      "logps/rejected": -31.16558265686035,
+      "loss": 0.9913,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.006287367548793554,
+      "rewards/margins": 0.008698503486812115,
+      "rewards/rejected": -0.014985869638621807,
       "step": 280
     },
     {
       "epoch": 0.75,
-      "grad_norm": 27.125,
+      "grad_norm": 1.34375,
       "learning_rate": 8.737922755071455e-07,
-      "logits/chosen": 79.78004455566406,
-      "logits/rejected": 79.70026397705078,
-      "logps/chosen": -33.06673812866211,
-      "logps/rejected": -33.00312805175781,
-      "loss": 0.5599,
+      "logits/chosen": 77.05482482910156,
+      "logits/rejected": 76.97974395751953,
+      "logps/chosen": -34.25292205810547,
+      "logps/rejected": -34.649898529052734,
+      "loss": 0.9752,
       "rewards/accuracies": 0.7749999761581421,
-      "rewards/chosen": 0.3112529516220093,
-      "rewards/margins": 0.6071315407752991,
-      "rewards/rejected": -0.29587849974632263,
+      "rewards/chosen": -0.0014867703430354595,
+      "rewards/margins": 0.02484356239438057,
+      "rewards/rejected": -0.026330333203077316,
       "step": 290
     },
     {
       "epoch": 0.78,
-      "grad_norm": 34.25,
+      "grad_norm": 1.4921875,
       "learning_rate": 7.08321427484816e-07,
-      "logits/chosen": 75.4156494140625,
-      "logits/rejected": 75.51851654052734,
-      "logps/chosen": -32.61161422729492,
-      "logps/rejected": -29.624340057373047,
-      "loss": 0.73,
-      "rewards/accuracies": 0.625,
-      "rewards/chosen": 0.23822763562202454,
-      "rewards/margins": 0.39492282271385193,
-      "rewards/rejected": -0.1566951870918274,
+      "logits/chosen": 72.6989974975586,
+      "logits/rejected": 72.833984375,
+      "logps/chosen": -33.264137268066406,
+      "logps/rejected": -30.818592071533203,
+      "loss": 0.9814,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.0014156814431771636,
+      "rewards/margins": 0.018581366166472435,
+      "rewards/rejected": -0.017165686935186386,
       "step": 300
     },
     {
       "epoch": 0.78,
-      "eval_logits/chosen": 98.47382354736328,
-      "eval_logits/rejected": 98.44670104980469,
-      "eval_logps/chosen": -32.83805465698242,
-      "eval_logps/rejected": -36.554744720458984,
-      "eval_loss": 0.9506204128265381,
-      "eval_rewards/accuracies": 0.5074750781059265,
-      "eval_rewards/chosen": -0.11846435815095901,
-      "eval_rewards/margins": 0.058001305907964706,
-      "eval_rewards/rejected": -0.17646567523479462,
-      "eval_runtime": 103.9661,
-      "eval_samples_per_second": 3.299,
+      "eval_logits/chosen": 97.67390441894531,
+      "eval_logits/rejected": 97.64021301269531,
+      "eval_logps/chosen": -33.30087661743164,
+      "eval_logps/rejected": -37.35591125488281,
+      "eval_loss": 0.99467533826828,
+      "eval_rewards/accuracies": 0.5888704061508179,
+      "eval_rewards/chosen": -0.008576988242566586,
+      "eval_rewards/margins": 0.00531682837754488,
+      "eval_rewards/rejected": -0.01389381755143404,
+      "eval_runtime": 103.9426,
+      "eval_samples_per_second": 3.3,
       "eval_steps_per_second": 0.414,
       "step": 300
     },
     {
       "epoch": 0.81,
-      "grad_norm": 32.5,
+      "grad_norm": 1.609375,
       "learning_rate": 5.576113578589035e-07,
-      "logits/chosen": 82.58702850341797,
-      "logits/rejected": 82.61672973632812,
-      "logps/chosen": -30.191946029663086,
-      "logps/rejected": -32.885528564453125,
-      "loss": 0.6797,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": 0.2014763057231903,
-      "rewards/margins": 0.4072059690952301,
-      "rewards/rejected": -0.2057296484708786,
+      "logits/chosen": 80.20366668701172,
+      "logits/rejected": 80.20387268066406,
+      "logps/chosen": -30.933481216430664,
+      "logps/rejected": -34.256614685058594,
+      "loss": 0.9801,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.0006994610885158181,
+      "rewards/margins": 0.019869104027748108,
+      "rewards/rejected": -0.020568564534187317,
       "step": 310
     },
     {
       "epoch": 0.83,
-      "grad_norm": 28.375,
+      "grad_norm": 1.5390625,
       "learning_rate": 4.229036944380913e-07,
-      "logits/chosen": 79.99136352539062,
-      "logits/rejected": 79.99455261230469,
-      "logps/chosen": -30.716598510742188,
-      "logps/rejected": -29.531625747680664,
-      "loss": 0.6405,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": 0.26458847522735596,
-      "rewards/margins": 0.4482294023036957,
-      "rewards/rejected": -0.18364088237285614,
+      "logits/chosen": 77.26924133300781,
+      "logits/rejected": 77.28764343261719,
+      "logps/chosen": -31.467296600341797,
+      "logps/rejected": -30.592571258544922,
+      "loss": 0.982,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": 0.0013126448029652238,
+      "rewards/margins": 0.01804344728589058,
+      "rewards/rejected": -0.01673080213367939,
       "step": 320
     },
     {
       "epoch": 0.86,
-      "grad_norm": 23.625,
+      "grad_norm": 1.875,
       "learning_rate": 3.053082288996112e-07,
-      "logits/chosen": 77.1466293334961,
-      "logits/rejected": 77.19983673095703,
-      "logps/chosen": -28.99395751953125,
-      "logps/rejected": -33.32061767578125,
-      "loss": 0.5347,
+      "logits/chosen": 74.20513916015625,
+      "logits/rejected": 74.25221252441406,
+      "logps/chosen": -29.909320831298828,
+      "logps/rejected": -34.725521087646484,
+      "loss": 0.9747,
       "rewards/accuracies": 0.7749999761581421,
-      "rewards/chosen": 0.40759435296058655,
-      "rewards/margins": 0.6133207082748413,
-      "rewards/rejected": -0.20572634041309357,
+      "rewards/chosen": 0.004432853776961565,
+      "rewards/margins": 0.025339430198073387,
+      "rewards/rejected": -0.02090657688677311,
       "step": 330
     },
     {
       "epoch": 0.88,
-      "grad_norm": 36.25,
+      "grad_norm": 1.7578125,
       "learning_rate": 2.0579377374915805e-07,
-      "logits/chosen": 81.56797790527344,
-      "logits/rejected": 81.60877990722656,
-      "logps/chosen": -32.45484161376953,
-      "logps/rejected": -34.365745544433594,
-      "loss": 0.6336,
-      "rewards/accuracies": 0.7749999761581421,
-      "rewards/chosen": 0.2271919995546341,
-      "rewards/margins": 0.5263071656227112,
-      "rewards/rejected": -0.2991151213645935,
+      "logits/chosen": 78.9554672241211,
+      "logits/rejected": 78.990478515625,
+      "logps/chosen": -33.333351135253906,
+      "logps/rejected": -35.866722106933594,
+      "loss": 0.9762,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.0012120162136852741,
+      "rewards/margins": 0.023768287152051926,
+      "rewards/rejected": -0.024980302900075912,
       "step": 340
     },
     {
       "epoch": 0.91,
-      "grad_norm": 32.5,
+      "grad_norm": 1.5859375,
       "learning_rate": 1.2518018074041684e-07,
-      "logits/chosen": 80.57276153564453,
-      "logits/rejected": 80.58821105957031,
-      "logps/chosen": -32.584190368652344,
-      "logps/rejected": -33.92478561401367,
-      "loss": 0.6072,
-      "rewards/accuracies": 0.7875000238418579,
-      "rewards/chosen": 0.31786566972732544,
-      "rewards/margins": 0.5677614212036133,
-      "rewards/rejected": -0.249895840883255,
+      "logits/chosen": 77.8236083984375,
+      "logits/rejected": 77.84061431884766,
+      "logps/chosen": -33.3131103515625,
+      "logps/rejected": -35.05299758911133,
+      "loss": 0.9771,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.0033062633592635393,
+      "rewards/margins": 0.0229182131588459,
+      "rewards/rejected": -0.01961195096373558,
       "step": 350
     },
     {
       "epoch": 0.94,
-      "grad_norm": 22.375,
+      "grad_norm": 1.5234375,
       "learning_rate": 6.41315865106129e-08,
-      "logits/chosen": 82.04108428955078,
-      "logits/rejected": 82.07307434082031,
-      "logps/chosen": -28.4096622467041,
-      "logps/rejected": -31.982025146484375,
-      "loss": 0.6571,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": 0.34276455640792847,
-      "rewards/margins": 0.44241756200790405,
-      "rewards/rejected": -0.09965306520462036,
+      "logits/chosen": 79.53218078613281,
+      "logits/rejected": 79.56050109863281,
+      "logps/chosen": -29.001379013061523,
+      "logps/rejected": -33.16984176635742,
+      "loss": 0.9793,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.005508318077772856,
+      "rewards/margins": 0.020708225667476654,
+      "rewards/rejected": -0.015199905261397362,
       "step": 360
     },
     {
       "epoch": 0.96,
-      "grad_norm": 29.875,
+      "grad_norm": 1.7890625,
       "learning_rate": 2.3150941078050325e-08,
-      "logits/chosen": 81.50835418701172,
-      "logits/rejected": 81.53009033203125,
-      "logps/chosen": -32.14625549316406,
-      "logps/rejected": -35.930458068847656,
-      "loss": 0.68,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": 0.21897241473197937,
-      "rewards/margins": 0.4652990400791168,
-      "rewards/rejected": -0.24632665514945984,
+      "logits/chosen": 78.969970703125,
+      "logits/rejected": 78.9748306274414,
+      "logps/chosen": -33.2999267578125,
+      "logps/rejected": -37.32087326049805,
+      "loss": 0.9821,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.004237635992467403,
+      "rewards/margins": 0.017877381294965744,
+      "rewards/rejected": -0.022115018218755722,
       "step": 370
     },
     {
       "epoch": 0.99,
-      "grad_norm": 35.25,
+      "grad_norm": 1.359375,
       "learning_rate": 2.575864278703266e-09,
-      "logits/chosen": 75.34764099121094,
-      "logits/rejected": 75.214111328125,
-      "logps/chosen": -29.86236572265625,
-      "logps/rejected": -28.811176300048828,
-      "loss": 0.6781,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": 0.21412897109985352,
-      "rewards/margins": 0.40930914878845215,
-      "rewards/rejected": -0.19518017768859863,
+      "logits/chosen": 72.49492645263672,
+      "logits/rejected": 72.36249542236328,
+      "logps/chosen": -30.828378677368164,
+      "logps/rejected": -29.885875701904297,
+      "loss": 0.9853,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.0025225188583135605,
+      "rewards/margins": 0.014730495400726795,
+      "rewards/rejected": -0.01725301705300808,
       "step": 380
     },
     {
       "epoch": 1.0,
       "step": 385,
       "total_flos": 0.0,
-      "train_loss": 0.7626325495831379,
-      "train_runtime": 2558.2938,
-      "train_samples_per_second": 1.204,
+      "train_loss": 0.9878765378679548,
+      "train_runtime": 2559.7639,
+      "train_samples_per_second": 1.203,
       "train_steps_per_second": 0.15
     }
   ],

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e6908ff07e3baa77dd4363406fddf73d2051acd14efcc5105c18632df4a9bbae
+oid sha256:982adb31f4fa08482d079d41c97f648ab87fc998e5129177f60808052907e849
 size 5176