Model save

README.md

@@ -1,13 +1,11 @@
 ---
 library_name: peft
 tags:
-- alignment-handbook
 - trl
 - dpo
+- alignment-handbook
 - generated_from_trainer
 base_model: NbAiLab/nb-gpt-j-6B-v2
-datasets:
-- hugodk-sch/aftonposten_title_prefs
 model-index:
 - name: aftonposten-6b-align-scan
   results: []
@@ -18,17 +16,17 @@ should probably proofread and complete it, then remove this comment. -->
 
 # aftonposten-6b-align-scan
 
-This model is a fine-tuned version of [data/ap-gpt-j-6b-sft-qlora-04-08](https://huggingface.co/data/ap-gpt-j-6b-sft-qlora-04-08) on the hugodk-sch/aftonposten_title_prefs dataset.
+This model is a fine-tuned version of [NbAiLab/nb-gpt-j-6B-v2](https://huggingface.co/NbAiLab/nb-gpt-j-6B-v2) on an unknown dataset.
 It achieves the following results on the evaluation set:
-- Loss: 1.0085
-- Rewards/chosen: -0.0006
-- Rewards/rejected: 0.0077
-- Rewards/accuracies: 0.4817
-- Rewards/margins: -0.0083
-- Logps/rejected: -37.5037
-- Logps/chosen: -34.0355
-- Logits/rejected: -2.2391
-- Logits/chosen: -2.2440
+- Logits/chosen: -2.2441
+- Logits/rejected: -2.2392
+- Logps/chosen: -34.0123
+- Logps/rejected: -37.5091
+- Loss: 0.6902
+- Rewards/accuracies: 0.5133
+- Rewards/chosen: 0.0156
+- Rewards/margins: 0.0103
+- Rewards/rejected: 0.0052
 
 ## Model description
 
@@ -63,9 +61,9 @@ The following hyperparameters were used during training:
 
 | Training Loss | Epoch | Step | Logits/chosen | Logits/rejected | Logps/chosen | Logps/rejected | Validation Loss | Rewards/accuracies | Rewards/chosen | Rewards/margins | Rewards/rejected |
 |:-------------:|:-----:|:----:|:-------------:|:---------------:|:------------:|:--------------:|:---------------:|:------------------:|:--------------:|:---------------:|:----------------:|
-| 1.0007        | 0.26  | 100  | -2.2440       | -2.2391         | -34.0192     | -37.5035       | 0.9989          | 0.5403             | 0.0092         | 0.0013          | 0.0079           |
-| 0.9898        | 0.52  | 200  | -2.2440       | -2.2391         | -34.0138     | -37.5009       | 0.9970          | 0.5307             | 0.0124         | 0.0030          | 0.0094           |
-| 0.9846        | 0.78  | 300  | -2.2437       | -2.2388         | -34.0263     | -37.5052       | 1.0020          | 0.4635             | 0.0050         | -0.0019         | 0.0068           |
+| 0.6821        | 0.26  | 100  | -2.2442       | -2.2393         | -34.0258     | -37.4952       | 0.6996          | 0.4805             | 0.0061         | -0.0088         | 0.0150           |
+| 0.6938        | 0.52  | 200  | -2.2438       | -2.2389         | -34.0275     | -37.5027       | 0.6978          | 0.4817             | 0.0050         | -0.0048         | 0.0098           |
+| 0.681         | 0.78  | 300  | -2.2441       | -2.2392         | -34.0123     | -37.5091       | 0.6902          | 0.5133             | 0.0156         | 0.0103          | 0.0052           |
 
 
 ### Framework versions

adapter_config.json

@@ -19,13 +19,13 @@
   "rank_pattern": {},
   "revision": null,
   "target_modules": [
+    "q_proj",
+    "o_proj",
+    "v_proj",
     "down_proj",
-    "up_proj",
     "k_proj",
     "gate_proj",
-    "o_proj",
-    "v_proj",
-    "q_proj"
+    "up_proj"
   ],
   "task_type": "CAUSAL_LM",
   "use_rslora": false

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:51387effbb49cdadf9979c5a56ff61a06766e3483674b168cbccd22db7c79182
+oid sha256:fad833e10db9e8b4254a027fa666bb7c4f87d8c271d8f457b338262c20a03e39
 size 176183216

all_results.json

@@ -1,21 +1,21 @@
 {
     "epoch": 1.0,
-    "eval_logits/chosen": -2.243980884552002,
-    "eval_logits/rejected": -2.2390992641448975,
-    "eval_logps/chosen": -34.03550720214844,
-    "eval_logps/rejected": -37.50373458862305,
-    "eval_loss": 1.0085219144821167,
-    "eval_rewards/accuracies": 0.48172760009765625,
-    "eval_rewards/chosen": -0.0005728387623094022,
-    "eval_rewards/margins": -0.008305290713906288,
-    "eval_rewards/rejected": 0.007732452359050512,
-    "eval_runtime": 146.1229,
+    "eval_logits/chosen": -2.2439703941345215,
+    "eval_logits/rejected": -2.2390873432159424,
+    "eval_logps/chosen": -34.037574768066406,
+    "eval_logps/rejected": -37.50224685668945,
+    "eval_loss": 0.7015302777290344,
+    "eval_rewards/accuracies": 0.4925249218940735,
+    "eval_rewards/chosen": -0.002115294337272644,
+    "eval_rewards/margins": -0.012174182571470737,
+    "eval_rewards/rejected": 0.010058889165520668,
+    "eval_runtime": 145.8307,
     "eval_samples": 343,
-    "eval_samples_per_second": 2.347,
-    "eval_steps_per_second": 0.294,
-    "train_loss": 0.21985046337177228,
-    "train_runtime": 627.6127,
+    "eval_samples_per_second": 2.352,
+    "eval_steps_per_second": 0.295,
+    "train_loss": 0.15254525333255917,
+    "train_runtime": 628.1072,
     "train_samples": 3079,
-    "train_samples_per_second": 4.906,
+    "train_samples_per_second": 4.902,
     "train_steps_per_second": 0.613
 }

eval_results.json

@@ -1,16 +1,16 @@
 {
     "epoch": 1.0,
-    "eval_logits/chosen": -2.243980884552002,
-    "eval_logits/rejected": -2.2390992641448975,
-    "eval_logps/chosen": -34.03550720214844,
-    "eval_logps/rejected": -37.50373458862305,
-    "eval_loss": 1.0085219144821167,
-    "eval_rewards/accuracies": 0.48172760009765625,
-    "eval_rewards/chosen": -0.0005728387623094022,
-    "eval_rewards/margins": -0.008305290713906288,
-    "eval_rewards/rejected": 0.007732452359050512,
-    "eval_runtime": 146.1229,
+    "eval_logits/chosen": -2.2439703941345215,
+    "eval_logits/rejected": -2.2390873432159424,
+    "eval_logps/chosen": -34.037574768066406,
+    "eval_logps/rejected": -37.50224685668945,
+    "eval_loss": 0.7015302777290344,
+    "eval_rewards/accuracies": 0.4925249218940735,
+    "eval_rewards/chosen": -0.002115294337272644,
+    "eval_rewards/margins": -0.012174182571470737,
+    "eval_rewards/rejected": 0.010058889165520668,
+    "eval_runtime": 145.8307,
     "eval_samples": 343,
-    "eval_samples_per_second": 2.347,
-    "eval_steps_per_second": 0.294
+    "eval_samples_per_second": 2.352,
+    "eval_steps_per_second": 0.295
 }

train_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 1.0,
-    "train_loss": 0.21985046337177228,
-    "train_runtime": 627.6127,
+    "train_loss": 0.15254525333255917,
+    "train_runtime": 628.1072,
     "train_samples": 3079,
-    "train_samples_per_second": 4.906,
+    "train_samples_per_second": 4.902,
     "train_steps_per_second": 0.613
 }

trainer_state.json

@@ -15,7 +15,7 @@
       "logits/rejected": -1.7377450466156006,
       "logps/chosen": -29.553977966308594,
       "logps/rejected": -42.813133239746094,
-      "loss": 1.0,
+      "loss": 0.6931,
       "rewards/accuracies": 0.0,
       "rewards/chosen": 0.0,
       "rewards/margins": 0.0,
@@ -25,590 +25,590 @@
     {
       "epoch": 0.03,
       "learning_rate": 1.2820512820512818e-07,
-      "logits/chosen": -1.866068720817566,
-      "logits/rejected": -1.8703863620758057,
-      "logps/chosen": -36.99105453491211,
-      "logps/rejected": -33.65992736816406,
-      "loss": 0.9728,
-      "rewards/accuracies": 0.5138888955116272,
-      "rewards/chosen": 0.009325359016656876,
-      "rewards/margins": 0.027240369468927383,
-      "rewards/rejected": -0.017915012314915657,
+      "logits/chosen": -1.866014003753662,
+      "logits/rejected": -1.8703253269195557,
+      "logps/chosen": -36.98957061767578,
+      "logps/rejected": -33.655086517333984,
+      "loss": 0.6802,
+      "rewards/accuracies": 0.5555555820465088,
+      "rewards/chosen": 0.01192101463675499,
+      "rewards/margins": 0.029430679976940155,
+      "rewards/rejected": -0.017509667202830315,
       "step": 10
     },
     {
       "epoch": 0.05,
       "learning_rate": 2.5641025641025636e-07,
-      "logits/chosen": -1.9970887899398804,
-      "logits/rejected": -1.9997413158416748,
-      "logps/chosen": -29.63595199584961,
-      "logps/rejected": -29.070571899414062,
-      "loss": 0.992,
-      "rewards/accuracies": 0.5,
-      "rewards/chosen": 0.003744622226804495,
-      "rewards/margins": 0.008021654561161995,
-      "rewards/rejected": -0.004277031868696213,
+      "logits/chosen": -1.9975181818008423,
+      "logits/rejected": -2.000157356262207,
+      "logps/chosen": -29.643035888671875,
+      "logps/rejected": -29.068897247314453,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.4625000059604645,
+      "rewards/chosen": -0.0005901128170080483,
+      "rewards/margins": 0.003226220142096281,
+      "rewards/rejected": -0.003816334530711174,
       "step": 20
     },
     {
       "epoch": 0.08,
       "learning_rate": 3.8461538461538463e-07,
-      "logits/chosen": -1.9195963144302368,
-      "logits/rejected": -1.9168994426727295,
-      "logps/chosen": -31.414175033569336,
-      "logps/rejected": -33.25041961669922,
-      "loss": 0.9797,
-      "rewards/accuracies": 0.5625,
-      "rewards/chosen": 0.0011531396303325891,
-      "rewards/margins": 0.02032935619354248,
-      "rewards/rejected": -0.019176214933395386,
+      "logits/chosen": -1.919750452041626,
+      "logits/rejected": -1.9170730113983154,
+      "logps/chosen": -31.426767349243164,
+      "logps/rejected": -33.24191665649414,
+      "loss": 0.6908,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -0.00747058168053627,
+      "rewards/margins": 0.008947307243943214,
+      "rewards/rejected": -0.016417888924479485,
       "step": 30
     },
     {
       "epoch": 0.1,
       "learning_rate": 4.999896948438433e-07,
-      "logits/chosen": -2.01688814163208,
-      "logits/rejected": -2.0081212520599365,
-      "logps/chosen": -32.583614349365234,
-      "logps/rejected": -32.48966979980469,
-      "loss": 1.0179,
-      "rewards/accuracies": 0.4375,
-      "rewards/chosen": -0.004085154738277197,
-      "rewards/margins": -0.017886672168970108,
-      "rewards/rejected": 0.013801517896354198,
+      "logits/chosen": -2.0170185565948486,
+      "logits/rejected": -2.008249282836914,
+      "logps/chosen": -32.58369827270508,
+      "logps/rejected": -32.508426666259766,
+      "loss": 0.6989,
+      "rewards/accuracies": 0.48750001192092896,
+      "rewards/chosen": -0.004824736155569553,
+      "rewards/margins": -0.007793365977704525,
+      "rewards/rejected": 0.0029686305206269026,
       "step": 40
     },
     {
       "epoch": 0.13,
       "learning_rate": 4.987541037542186e-07,
-      "logits/chosen": -1.8644500970840454,
-      "logits/rejected": -1.853655219078064,
-      "logps/chosen": -33.54804229736328,
-      "logps/rejected": -35.43767166137695,
-      "loss": 1.0031,
-      "rewards/accuracies": 0.4375,
-      "rewards/chosen": 0.005664288066327572,
-      "rewards/margins": -0.0031387731432914734,
-      "rewards/rejected": 0.00880306214094162,
+      "logits/chosen": -1.864098310470581,
+      "logits/rejected": -1.8533239364624023,
+      "logps/chosen": -33.5538444519043,
+      "logps/rejected": -35.4467658996582,
+      "loss": 0.6956,
+      "rewards/accuracies": 0.4625000059604645,
+      "rewards/chosen": 0.002545009134337306,
+      "rewards/margins": -0.0013612977927550673,
+      "rewards/rejected": 0.003906308673322201,
       "step": 50
     },
     {
       "epoch": 0.16,
       "learning_rate": 4.954691471941118e-07,
-      "logits/chosen": -1.9449504613876343,
-      "logits/rejected": -1.946890115737915,
-      "logps/chosen": -32.58049774169922,
-      "logps/rejected": -33.199378967285156,
-      "loss": 0.9714,
-      "rewards/accuracies": 0.612500011920929,
-      "rewards/chosen": 0.011811850592494011,
-      "rewards/margins": 0.02860717847943306,
-      "rewards/rejected": -0.016795331612229347,
+      "logits/chosen": -1.9450995922088623,
+      "logits/rejected": -1.9470382928848267,
+      "logps/chosen": -32.56639862060547,
+      "logps/rejected": -33.19929122924805,
+      "loss": 0.6743,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.02365189418196678,
+      "rewards/margins": 0.043189071118831635,
+      "rewards/rejected": -0.019537178799510002,
       "step": 60
     },
     {
       "epoch": 0.18,
       "learning_rate": 4.901618883413548e-07,
-      "logits/chosen": -2.0795464515686035,
-      "logits/rejected": -2.0845184326171875,
-      "logps/chosen": -33.98798370361328,
-      "logps/rejected": -36.569732666015625,
-      "loss": 0.9992,
-      "rewards/accuracies": 0.48750001192092896,
-      "rewards/chosen": 0.0003018675488419831,
-      "rewards/margins": 0.0008206713828258216,
-      "rewards/rejected": -0.0005188033101148903,
+      "logits/chosen": -2.0796637535095215,
+      "logits/rejected": -2.084643602371216,
+      "logps/chosen": -33.98316955566406,
+      "logps/rejected": -36.59341049194336,
+      "loss": 0.686,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": 0.00371909374371171,
+      "rewards/margins": 0.020896537229418755,
+      "rewards/rejected": -0.01717744767665863,
       "step": 70
     },
     {
       "epoch": 0.21,
       "learning_rate": 4.828760511501322e-07,
-      "logits/chosen": -1.9425585269927979,
-      "logits/rejected": -1.945704698562622,
-      "logps/chosen": -34.408721923828125,
-      "logps/rejected": -34.581504821777344,
-      "loss": 0.9951,
-      "rewards/accuracies": 0.5375000238418579,
-      "rewards/chosen": 0.011452903971076012,
-      "rewards/margins": 0.0049109989777207375,
-      "rewards/rejected": 0.006541903130710125,
+      "logits/chosen": -1.9420992136001587,
+      "logits/rejected": -1.9452590942382812,
+      "logps/chosen": -34.40605545043945,
+      "logps/rejected": -34.58306884765625,
+      "loss": 0.6914,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": 0.015227687545120716,
+      "rewards/margins": 0.008690384216606617,
+      "rewards/rejected": 0.006537304259836674,
       "step": 80
     },
     {
       "epoch": 0.23,
       "learning_rate": 4.736716601303429e-07,
-      "logits/chosen": -1.9505088329315186,
-      "logits/rejected": -1.9550243616104126,
-      "logps/chosen": -32.451255798339844,
-      "logps/rejected": -32.366790771484375,
-      "loss": 0.9856,
-      "rewards/accuracies": 0.512499988079071,
-      "rewards/chosen": 0.009697502478957176,
-      "rewards/margins": 0.014425436034798622,
-      "rewards/rejected": -0.0047279344871640205,
+      "logits/chosen": -1.950547456741333,
+      "logits/rejected": -1.9550635814666748,
+      "logps/chosen": -32.4520378112793,
+      "logps/rejected": -32.369686126708984,
+      "loss": 0.6861,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": 0.010769412852823734,
+      "rewards/margins": 0.018315035849809647,
+      "rewards/rejected": -0.007545621134340763,
       "step": 90
     },
     {
       "epoch": 0.26,
       "learning_rate": 4.62624545834521e-07,
-      "logits/chosen": -2.048947334289551,
-      "logits/rejected": -2.046935796737671,
-      "logps/chosen": -32.25363540649414,
-      "logps/rejected": -31.277917861938477,
-      "loss": 1.0007,
-      "rewards/accuracies": 0.44999998807907104,
-      "rewards/chosen": -0.007680465932935476,
-      "rewards/margins": -0.0006578800384886563,
-      "rewards/rejected": -0.007022587116807699,
+      "logits/chosen": -2.0487563610076904,
+      "logits/rejected": -2.046765089035034,
+      "logps/chosen": -32.21302032470703,
+      "logps/rejected": -31.275714874267578,
+      "loss": 0.6821,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.019469190388917923,
+      "rewards/margins": 0.02612192928791046,
+      "rewards/rejected": -0.006652742624282837,
       "step": 100
     },
     {
       "epoch": 0.26,
-      "eval_logits/chosen": -2.243957042694092,
-      "eval_logits/rejected": -2.2390689849853516,
-      "eval_logps/chosen": -34.019203186035156,
-      "eval_logps/rejected": -37.50347900390625,
-      "eval_loss": 0.998858630657196,
-      "eval_rewards/accuracies": 0.5402824282646179,
-      "eval_rewards/chosen": 0.009209612384438515,
-      "eval_rewards/margins": 0.0013267018366605043,
-      "eval_rewards/rejected": 0.00788290984928608,
-      "eval_runtime": 145.9923,
-      "eval_samples_per_second": 2.349,
-      "eval_steps_per_second": 0.295,
+      "eval_logits/chosen": -2.244154930114746,
+      "eval_logits/rejected": -2.2392725944519043,
+      "eval_logps/chosen": -34.02579116821289,
+      "eval_logps/rejected": -37.495243072509766,
+      "eval_loss": 0.6996207237243652,
+      "eval_rewards/accuracies": 0.48048171401023865,
+      "eval_rewards/chosen": 0.006134189199656248,
+      "eval_rewards/margins": -0.008827367797493935,
+      "eval_rewards/rejected": 0.014961558394134045,
+      "eval_runtime": 146.1618,
+      "eval_samples_per_second": 2.347,
+      "eval_steps_per_second": 0.294,
       "step": 100
     },
     {
       "epoch": 0.29,
       "learning_rate": 4.4982572012636904e-07,
-      "logits/chosen": -2.005302667617798,
-      "logits/rejected": -2.002894878387451,
-      "logps/chosen": -33.23944854736328,
-      "logps/rejected": -34.005104064941406,
-      "loss": 1.0119,
-      "rewards/accuracies": 0.44999998807907104,
-      "rewards/chosen": 0.002622986678034067,
-      "rewards/margins": -0.01194014959037304,
-      "rewards/rejected": 0.014563137665390968,
+      "logits/chosen": -2.005396604537964,
+      "logits/rejected": -2.00298810005188,
+      "logps/chosen": -33.235660552978516,
+      "logps/rejected": -34.023067474365234,
+      "loss": 0.6946,
+      "rewards/accuracies": 0.4749999940395355,
+      "rewards/chosen": 0.0057089305482804775,
+      "rewards/margins": 0.001296796603128314,
+      "rewards/rejected": 0.0044121332466602325,
       "step": 110
     },
     {
       "epoch": 0.31,
       "learning_rate": 4.353806263777677e-07,
-      "logits/chosen": -2.0167059898376465,
-      "logits/rejected": -2.0083398818969727,
-      "logps/chosen": -32.45713424682617,
-      "logps/rejected": -32.17597198486328,
-      "loss": 0.9987,
-      "rewards/accuracies": 0.44999998807907104,
-      "rewards/chosen": -0.0073361145332455635,
-      "rewards/margins": 0.0013455990701913834,
-      "rewards/rejected": -0.008681714534759521,
+      "logits/chosen": -2.0165247917175293,
+      "logits/rejected": -2.0081515312194824,
+      "logps/chosen": -32.45998764038086,
+      "logps/rejected": -32.18000793457031,
+      "loss": 0.6936,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.010559305548667908,
+      "rewards/margins": 0.0023918983060866594,
+      "rewards/rejected": -0.012951202690601349,
       "step": 120
     },
     {
       "epoch": 0.34,
       "learning_rate": 4.194082707715275e-07,
-      "logits/chosen": -2.0464558601379395,
-      "logits/rejected": -2.038412094116211,
-      "logps/chosen": -30.482311248779297,
-      "logps/rejected": -32.04933166503906,
-      "loss": 1.0062,
-      "rewards/accuracies": 0.48750001192092896,
-      "rewards/chosen": -0.003956255037337542,
-      "rewards/margins": -0.006162940990179777,
-      "rewards/rejected": 0.002206685720011592,
+      "logits/chosen": -2.0464999675750732,
+      "logits/rejected": -2.0384459495544434,
+      "logps/chosen": -30.482324600219727,
+      "logps/rejected": -32.024261474609375,
+      "loss": 0.7073,
+      "rewards/accuracies": 0.36250001192092896,
+      "rewards/chosen": -0.004625456873327494,
+      "rewards/margins": -0.02474728412926197,
+      "rewards/rejected": 0.020121825858950615,
       "step": 130
     },
     {
       "epoch": 0.36,
       "learning_rate": 4.020402418666621e-07,
-      "logits/chosen": -1.9769861698150635,
-      "logits/rejected": -1.9872652292251587,
-      "logps/chosen": -31.415592193603516,
-      "logps/rejected": -32.5562858581543,
-      "loss": 0.9847,
-      "rewards/accuracies": 0.574999988079071,
-      "rewards/chosen": 0.0024404728319495916,
-      "rewards/margins": 0.015285758301615715,
-      "rewards/rejected": -0.01284528523683548,
+      "logits/chosen": -1.976978063583374,
+      "logits/rejected": -1.9872522354125977,
+      "logps/chosen": -31.402069091796875,
+      "logps/rejected": -32.54044723510742,
+      "loss": 0.6869,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.012314395979046822,
+      "rewards/margins": 0.01621088758111,
+      "rewards/rejected": -0.0038964927662163973,
       "step": 140
     },
     {
       "epoch": 0.39,
       "learning_rate": 3.8341962650351185e-07,
-      "logits/chosen": -1.890750527381897,
-      "logits/rejected": -1.8918195962905884,
-      "logps/chosen": -34.196632385253906,
-      "logps/rejected": -34.74388885498047,
-      "loss": 1.0104,
-      "rewards/accuracies": 0.4000000059604645,
-      "rewards/chosen": -0.002364098560065031,
-      "rewards/margins": -0.0103833619505167,
-      "rewards/rejected": 0.008019264787435532,
+      "logits/chosen": -1.8904860019683838,
+      "logits/rejected": -1.8915693759918213,
+      "logps/chosen": -34.20112609863281,
+      "logps/rejected": -34.765201568603516,
+      "loss": 0.6961,
+      "rewards/accuracies": 0.44999998807907104,
+      "rewards/chosen": -0.005904030986130238,
+      "rewards/margins": -0.0003432026132941246,
+      "rewards/rejected": -0.005560829304158688,
       "step": 150
     },
     {
       "epoch": 0.42,
       "learning_rate": 3.636998309800572e-07,
-      "logits/chosen": -1.9424158334732056,
-      "logits/rejected": -1.9389365911483765,
-      "logps/chosen": -36.13062286376953,
-      "logps/rejected": -32.720314025878906,
-      "loss": 0.9768,
-      "rewards/accuracies": 0.637499988079071,
-      "rewards/chosen": 0.02463117241859436,
-      "rewards/margins": 0.023165332153439522,
-      "rewards/rejected": 0.0014658428262919188,
+      "logits/chosen": -1.9423534870147705,
+      "logits/rejected": -1.938856840133667,
+      "logps/chosen": -36.14204025268555,
+      "logps/rejected": -32.7347412109375,
+      "loss": 0.6817,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": 0.020746376365423203,
+      "rewards/margins": 0.029139723628759384,
+      "rewards/rejected": -0.008393346332013607,
       "step": 160
     },
     {
       "epoch": 0.44,
       "learning_rate": 3.430433172111807e-07,
-      "logits/chosen": -2.04191517829895,
-      "logits/rejected": -2.034518003463745,
-      "logps/chosen": -33.780067443847656,
-      "logps/rejected": -31.35748863220215,
-      "loss": 0.9859,
-      "rewards/accuracies": 0.5375000238418579,
-      "rewards/chosen": 0.010676576755940914,
-      "rewards/margins": 0.014060018584132195,
-      "rewards/rejected": -0.0033834422938525677,
+      "logits/chosen": -2.0420148372650146,
+      "logits/rejected": -2.034619092941284,
+      "logps/chosen": -33.79521942138672,
+      "logps/rejected": -31.3667049407959,
+      "loss": 0.6897,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": 0.0018501380691304803,
+      "rewards/margins": 0.012249214574694633,
+      "rewards/rejected": -0.010399078950285912,
       "step": 170
     },
     {
       "epoch": 0.47,
       "learning_rate": 3.216202642830543e-07,
-      "logits/chosen": -2.0474770069122314,
-      "logits/rejected": -2.0527572631835938,
-      "logps/chosen": -32.51685333251953,
-      "logps/rejected": -32.48542022705078,
-      "loss": 0.9831,
-      "rewards/accuracies": 0.5,
-      "rewards/chosen": 0.01676076650619507,
-      "rewards/margins": 0.016928378492593765,
-      "rewards/rejected": -0.00016761067672632635,
+      "logits/chosen": -2.0474741458892822,
+      "logits/rejected": -2.052746295928955,
+      "logps/chosen": -32.533199310302734,
+      "logps/rejected": -32.49622344970703,
+      "loss": 0.6872,
+      "rewards/accuracies": 0.48750001192092896,
+      "rewards/chosen": 0.008109848015010357,
+      "rewards/margins": 0.015868691727519035,
+      "rewards/rejected": -0.007758840918540955,
       "step": 180
     },
     {
       "epoch": 0.49,
       "learning_rate": 2.9960716642946403e-07,
-      "logits/chosen": -2.0486884117126465,
-      "logits/rejected": -2.045897960662842,
-      "logps/chosen": -31.490026473999023,
-      "logps/rejected": -31.336299896240234,
-      "loss": 0.9898,
-      "rewards/accuracies": 0.5,
-      "rewards/chosen": -0.0006444025784730911,
-      "rewards/margins": 0.010206506587564945,
-      "rewards/rejected": -0.010850909166038036,
+      "logits/chosen": -2.048755407333374,
+      "logits/rejected": -2.045989513397217,
+      "logps/chosen": -31.495113372802734,
+      "logps/rejected": -31.329687118530273,
+      "loss": 0.6936,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.004311963450163603,
+      "rewards/margins": 0.003718080697581172,
+      "rewards/rejected": -0.008030043914914131,
       "step": 190
     },
     {
       "epoch": 0.52,
       "learning_rate": 2.771853789806683e-07,
-      "logits/chosen": -1.9185329675674438,
-      "logits/rejected": -1.9231984615325928,
-      "logps/chosen": -31.579875946044922,
-      "logps/rejected": -32.781455993652344,
-      "loss": 0.9898,
-      "rewards/accuracies": 0.574999988079071,
-      "rewards/chosen": 0.015846019610762596,
-      "rewards/margins": 0.010161412879824638,
-      "rewards/rejected": 0.005684606730937958,
+      "logits/chosen": -1.9183790683746338,
+      "logits/rejected": -1.9230560064315796,
+      "logps/chosen": -31.61056137084961,
+      "logps/rejected": -32.79776382446289,
+      "loss": 0.6938,
+      "rewards/accuracies": 0.48750001192092896,
+      "rewards/chosen": -0.002994548762217164,
+      "rewards/margins": 0.001788289868272841,
+      "rewards/rejected": -0.004782836884260178,
       "step": 200
     },
     {
       "epoch": 0.52,
-      "eval_logits/chosen": -2.2439582347869873,
-      "eval_logits/rejected": -2.239088296890259,
-      "eval_logps/chosen": -34.01380920410156,
-      "eval_logps/rejected": -37.500911712646484,
-      "eval_loss": 0.9969704151153564,
-      "eval_rewards/accuracies": 0.5307309031486511,
-      "eval_rewards/chosen": 0.01244510430842638,
-      "eval_rewards/margins": 0.0030194728169590235,
-      "eval_rewards/rejected": 0.009425631724298,
-      "eval_runtime": 145.8589,
-      "eval_samples_per_second": 2.352,
+      "eval_logits/chosen": -2.2438013553619385,
+      "eval_logits/rejected": -2.2389278411865234,
+      "eval_logps/chosen": -34.02748107910156,
+      "eval_logps/rejected": -37.502662658691406,
+      "eval_loss": 0.6977797150611877,
+      "eval_rewards/accuracies": 0.48172760009765625,
+      "eval_rewards/chosen": 0.004952840972691774,
+      "eval_rewards/margins": -0.004817771725356579,
+      "eval_rewards/rejected": 0.009770614095032215,
+      "eval_runtime": 146.0018,
+      "eval_samples_per_second": 2.349,
       "eval_steps_per_second": 0.295,
       "step": 200
     },
     {
       "epoch": 0.55,
       "learning_rate": 2.5453962426402e-07,
-      "logits/chosen": -2.0318081378936768,
-      "logits/rejected": -2.0424768924713135,
-      "logps/chosen": -31.95905113220215,
-      "logps/rejected": -33.87731170654297,
-      "loss": 0.9839,
-      "rewards/accuracies": 0.574999988079071,
-      "rewards/chosen": 0.007351105101406574,
-      "rewards/margins": 0.01611880213022232,
-      "rewards/rejected": -0.008767696097493172,
+      "logits/chosen": -2.0318925380706787,
+      "logits/rejected": -2.0425820350646973,
+      "logps/chosen": -31.96123695373535,
+      "logps/rejected": -33.87626647949219,
+      "loss": 0.6874,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": 0.007046692073345184,
+      "rewards/margins": 0.01654236949980259,
+      "rewards/rejected": -0.009495675563812256,
       "step": 210
     },
     {
       "epoch": 0.57,
       "learning_rate": 2.318564697655179e-07,
-      "logits/chosen": -1.9253685474395752,
-      "logits/rejected": -1.940233826637268,
-      "logps/chosen": -30.08770179748535,
-      "logps/rejected": -31.577301025390625,
-      "loss": 0.9742,
-      "rewards/accuracies": 0.612500011920929,
-      "rewards/chosen": 0.014303619973361492,
-      "rewards/margins": 0.025777745991945267,
-      "rewards/rejected": -0.011474122293293476,
+      "logits/chosen": -1.9252264499664307,
+      "logits/rejected": -1.940086007118225,
+      "logps/chosen": -30.087350845336914,
+      "logps/rejected": -31.552490234375,
+      "loss": 0.6894,
+      "rewards/accuracies": 0.4749999940395355,
+      "rewards/chosen": 0.01693141832947731,
+      "rewards/margins": 0.012951460666954517,
+      "rewards/rejected": 0.003979954868555069,
       "step": 220
     },
     {
       "epoch": 0.6,
       "learning_rate": 2.093227910899832e-07,
-      "logits/chosen": -1.9832963943481445,
-      "logits/rejected": -1.9872655868530273,
-      "logps/chosen": -33.39131546020508,
-      "logps/rejected": -31.548086166381836,
-      "loss": 0.985,
-      "rewards/accuracies": 0.574999988079071,
-      "rewards/chosen": 0.015694385394454002,
-      "rewards/margins": 0.014964587986469269,
-      "rewards/rejected": 0.0007297966512851417,
+      "logits/chosen": -1.9830553531646729,
+      "logits/rejected": -1.9870340824127197,
+      "logps/chosen": -33.37425231933594,
+      "logps/rejected": -31.55866050720215,
+      "loss": 0.6771,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": 0.030249694362282753,
+      "rewards/margins": 0.036800943315029144,
+      "rewards/rejected": -0.006551248021423817,
       "step": 230
     },
     {
       "epoch": 0.62,
       "learning_rate": 1.8712423238279356e-07,
-      "logits/chosen": -1.9828859567642212,
-      "logits/rejected": -1.9609191417694092,
-      "logps/chosen": -34.16459655761719,
-      "logps/rejected": -34.95134735107422,
-      "loss": 1.0062,
-      "rewards/accuracies": 0.4124999940395355,
-      "rewards/chosen": -0.0065087564289569855,
-      "rewards/margins": -0.006178082898259163,
-      "rewards/rejected": -0.0003306727739982307,
+      "logits/chosen": -1.9825870990753174,
+      "logits/rejected": -1.9606201648712158,
+      "logps/chosen": -34.172298431396484,
+      "logps/rejected": -34.95369338989258,
+      "loss": 0.7012,
+      "rewards/accuracies": 0.4000000059604645,
+      "rewards/chosen": -0.012983955442905426,
+      "rewards/margins": -0.010954022407531738,
+      "rewards/rejected": -0.002029931638389826,
       "step": 240
     },
     {
       "epoch": 0.65,
       "learning_rate": 1.654436768970182e-07,
-      "logits/chosen": -2.0246593952178955,
-      "logits/rejected": -2.0213539600372314,
-      "logps/chosen": -32.9222412109375,
-      "logps/rejected": -36.228416442871094,
-      "loss": 0.9962,
-      "rewards/accuracies": 0.4749999940395355,
-      "rewards/chosen": 0.0016961356159299612,
-      "rewards/margins": 0.0037924889475107193,
-      "rewards/rejected": -0.0020963551942259073,
+      "logits/chosen": -2.0240674018859863,
+      "logits/rejected": -2.020759105682373,
+      "logps/chosen": -32.91376495361328,
+      "logps/rejected": -36.22763442993164,
+      "loss": 0.6907,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": 0.007910441607236862,
+      "rewards/margins": 0.009807790629565716,
+      "rewards/rejected": -0.0018973512342199683,
       "step": 250
     },
     {
       "epoch": 0.68,
       "learning_rate": 1.444597403062196e-07,
-      "logits/chosen": -1.8918907642364502,
-      "logits/rejected": -1.8894577026367188,
-      "logps/chosen": -34.189327239990234,
-      "logps/rejected": -35.505496978759766,
-      "loss": 1.0,
-      "rewards/accuracies": 0.5,
-      "rewards/chosen": -0.0005569729837588966,
-      "rewards/margins": -2.999706157424953e-05,
-      "rewards/rejected": -0.0005269756657071412,
+      "logits/chosen": -1.891227126121521,
+      "logits/rejected": -1.8887770175933838,
+      "logps/chosen": -34.196754455566406,
+      "logps/rejected": -35.51189422607422,
+      "loss": 0.6959,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -0.005844797007739544,
+      "rewards/margins": -0.0007547728600911796,
+      "rewards/rejected": -0.005090024787932634,
       "step": 260
     },
     {
       "epoch": 0.7,
       "learning_rate": 1.2434529917578887e-07,
-      "logits/chosen": -1.8758213520050049,
-      "logits/rejected": -1.8732786178588867,
-      "logps/chosen": -34.38149642944336,
-      "logps/rejected": -31.752059936523438,
-      "loss": 1.0019,
-      "rewards/accuracies": 0.550000011920929,
-      "rewards/chosen": 0.009660609066486359,
-      "rewards/margins": -0.0019017171580344439,
-      "rewards/rejected": 0.011562327854335308,
+      "logits/chosen": -1.875758409500122,
+      "logits/rejected": -1.8732105493545532,
+      "logps/chosen": -34.38095474243164,
+      "logps/rejected": -31.74618148803711,
+      "loss": 0.6979,
+      "rewards/accuracies": 0.44999998807907104,
+      "rewards/chosen": 0.011651447974145412,
+      "rewards/margins": -0.005950801074504852,
+      "rewards/rejected": 0.01760224997997284,
       "step": 270
     },
     {
       "epoch": 0.73,
       "learning_rate": 1.0526606671603521e-07,
-      "logits/chosen": -1.9798027276992798,
-      "logits/rejected": -1.9691746234893799,
-      "logps/chosen": -35.328853607177734,
-      "logps/rejected": -31.868383407592773,
-      "loss": 0.9669,
-      "rewards/accuracies": 0.5375000238418579,
-      "rewards/chosen": 0.016671547666192055,
-      "rewards/margins": 0.03310702368617058,
-      "rewards/rejected": -0.016435474157333374,
+      "logits/chosen": -1.9794175624847412,
+      "logits/rejected": -1.968788504600525,
+      "logps/chosen": -35.32674026489258,
+      "logps/rejected": -31.84674072265625,
+      "loss": 0.6832,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": 0.020933207124471664,
+      "rewards/margins": 0.024957498535513878,
+      "rewards/rejected": -0.004024294205009937,
       "step": 280
     },
     {
       "epoch": 0.75,
       "learning_rate": 8.737922755071453e-08,
-      "logits/chosen": -2.075648546218872,
-      "logits/rejected": -2.0605978965759277,
-      "logps/chosen": -30.926809310913086,
-      "logps/rejected": -32.621070861816406,
-      "loss": 1.0201,
-      "rewards/accuracies": 0.4124999940395355,
-      "rewards/chosen": -0.0016108205309137702,
-      "rewards/margins": -0.020070534199476242,
-      "rewards/rejected": 0.018459713086485863,
+      "logits/chosen": -2.075899600982666,
+      "logits/rejected": -2.0608580112457275,
+      "logps/chosen": -30.886249542236328,
+      "logps/rejected": -32.66474533081055,
+      "loss": 0.6773,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": 0.026513904333114624,
+      "rewards/margins": 0.03555149585008621,
+      "rewards/rejected": -0.009037593379616737,
       "step": 290
     },
     {
       "epoch": 0.78,
       "learning_rate": 7.08321427484816e-08,
-      "logits/chosen": -1.9466993808746338,
-      "logits/rejected": -1.9441711902618408,
-      "logps/chosen": -32.90770721435547,
-      "logps/rejected": -30.82305335998535,
-      "loss": 0.9846,
-      "rewards/accuracies": 0.5249999761581421,
-      "rewards/chosen": 0.011643724516034126,
-      "rewards/margins": 0.015391023829579353,
-      "rewards/rejected": -0.0037472969852387905,
+      "logits/chosen": -1.9468847513198853,
+      "logits/rejected": -1.9443557262420654,
+      "logps/chosen": -32.885986328125,
+      "logps/rejected": -30.816707611083984,
+      "loss": 0.681,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": 0.02878658100962639,
+      "rewards/margins": 0.028715472668409348,
+      "rewards/rejected": 7.110489241313189e-05,
       "step": 300
     },
     {
       "epoch": 0.78,
-      "eval_logits/chosen": -2.243670701980591,
-      "eval_logits/rejected": -2.2387943267822266,
-      "eval_logps/chosen": -34.02626419067383,
-      "eval_logps/rejected": -37.50523376464844,
-      "eval_loss": 1.001994013786316,
-      "eval_rewards/accuracies": 0.4634551703929901,
-      "eval_rewards/chosen": 0.0049731116741895676,
-      "eval_rewards/margins": -0.0018590801628306508,
-      "eval_rewards/rejected": 0.006832191254943609,
-      "eval_runtime": 145.8336,
-      "eval_samples_per_second": 2.352,
+      "eval_logits/chosen": -2.244056224822998,
+      "eval_logits/rejected": -2.2391812801361084,
+      "eval_logps/chosen": -34.01231384277344,
+      "eval_logps/rejected": -37.509132385253906,
+      "eval_loss": 0.6901772618293762,
+      "eval_rewards/accuracies": 0.5132890343666077,
+      "eval_rewards/chosen": 0.015566298738121986,
+      "eval_rewards/margins": 0.01032568421214819,
+      "eval_rewards/rejected": 0.005240614525973797,
+      "eval_runtime": 145.9901,
+      "eval_samples_per_second": 2.349,
       "eval_steps_per_second": 0.295,
       "step": 300
     },
     {
       "epoch": 0.81,
       "learning_rate": 5.576113578589034e-08,
-      "logits/chosen": -1.9287983179092407,
-      "logits/rejected": -1.9255239963531494,
-      "logps/chosen": -31.573007583618164,
-      "logps/rejected": -33.742637634277344,
-      "loss": 0.9815,
-      "rewards/accuracies": 0.6000000238418579,
-      "rewards/chosen": 0.018206708133220673,
-      "rewards/margins": 0.018496429547667503,
-      "rewards/rejected": -0.0002897246740758419,
+      "logits/chosen": -1.9292223453521729,
+      "logits/rejected": -1.9259742498397827,
+      "logps/chosen": -31.571924209594727,
+      "logps/rejected": -33.74208450317383,
+      "loss": 0.6842,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": 0.021998558193445206,
+      "rewards/margins": 0.02195030450820923,
+      "rewards/rejected": 4.825703945243731e-05,
       "step": 310
     },
     {
       "epoch": 0.83,
       "learning_rate": 4.229036944380912e-08,
-      "logits/chosen": -1.9805524349212646,
-      "logits/rejected": -1.9682422876358032,
-      "logps/chosen": -34.57670211791992,
-      "logps/rejected": -33.561363220214844,
-      "loss": 0.9749,
-      "rewards/accuracies": 0.574999988079071,
-      "rewards/chosen": 0.011624744161963463,
-      "rewards/margins": 0.0251374039798975,
-      "rewards/rejected": -0.013512656092643738,
+      "logits/chosen": -1.980800986289978,
+      "logits/rejected": -1.9684827327728271,
+      "logps/chosen": -34.57365417480469,
+      "logps/rejected": -33.5623779296875,
+      "loss": 0.6796,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": 0.015695111826062202,
+      "rewards/margins": 0.032171379774808884,
+      "rewards/rejected": -0.01647626794874668,
       "step": 320
     },
     {
       "epoch": 0.86,
       "learning_rate": 3.053082288996112e-08,
-      "logits/chosen": -2.016091823577881,
-      "logits/rejected": -2.0146238803863525,
-      "logps/chosen": -33.47222137451172,
-      "logps/rejected": -32.46577835083008,
-      "loss": 1.0001,
-      "rewards/accuracies": 0.4749999940395355,
-      "rewards/chosen": 0.008143280632793903,
-      "rewards/margins": -0.0001393534184899181,
-      "rewards/rejected": 0.008282631635665894,
+      "logits/chosen": -2.016021490097046,
+      "logits/rejected": -2.0145699977874756,
+      "logps/chosen": -33.47764205932617,
+      "logps/rejected": -32.47804260253906,
+      "loss": 0.6932,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": 0.005706763360649347,
+      "rewards/margins": 0.00462835980579257,
+      "rewards/rejected": 0.0010784054175019264,
       "step": 330
     },
     {
       "epoch": 0.88,
       "learning_rate": 2.05793773749158e-08,
-      "logits/chosen": -2.1032156944274902,
-      "logits/rejected": -2.0874197483062744,
-      "logps/chosen": -34.174861907958984,
-      "logps/rejected": -33.088890075683594,
-      "loss": 1.0148,
-      "rewards/accuracies": 0.5249999761581421,
-      "rewards/chosen": 0.002716648392379284,
-      "rewards/margins": -0.014771336689591408,
-      "rewards/rejected": 0.017487986013293266,
+      "logits/chosen": -2.1033365726470947,
+      "logits/rejected": -2.087541103363037,
+      "logps/chosen": -34.160888671875,
+      "logps/rejected": -33.08522415161133,
+      "loss": 0.7012,
+      "rewards/accuracies": 0.48750001192092896,
+      "rewards/chosen": 0.012950187548995018,
+      "rewards/margins": -0.010019468143582344,
+      "rewards/rejected": 0.022969653829932213,
       "step": 340
     },
     {
       "epoch": 0.91,
       "learning_rate": 1.251801807404168e-08,
-      "logits/chosen": -1.9747329950332642,
-      "logits/rejected": -1.9737837314605713,
-      "logps/chosen": -33.25817108154297,
-      "logps/rejected": -32.459861755371094,
-      "loss": 0.9888,
-      "rewards/accuracies": 0.44999998807907104,
-      "rewards/chosen": 0.014356844127178192,
-      "rewards/margins": 0.01118617132306099,
-      "rewards/rejected": 0.003170671407133341,
+      "logits/chosen": -1.974867582321167,
+      "logits/rejected": -1.9739230871200562,
+      "logps/chosen": -33.23981857299805,
+      "logps/rejected": -32.456180572509766,
+      "loss": 0.6846,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": 0.029593482613563538,
+      "rewards/margins": 0.023315051570534706,
+      "rewards/rejected": 0.006278430111706257,
       "step": 350
     },
     {
       "epoch": 0.94,
       "learning_rate": 6.41315865106129e-09,
-      "logits/chosen": -1.9303538799285889,
-      "logits/rejected": -1.94071364402771,
-      "logps/chosen": -32.20769500732422,
-      "logps/rejected": -35.273780822753906,
-      "loss": 1.0108,
-      "rewards/accuracies": 0.44999998807907104,
-      "rewards/chosen": -0.0018019669223576784,
-      "rewards/margins": -0.010786842554807663,
-      "rewards/rejected": 0.008984875865280628,
+      "logits/chosen": -1.9302256107330322,
+      "logits/rejected": -1.940580129623413,
+      "logps/chosen": -32.2147102355957,
+      "logps/rejected": -35.29616165161133,
+      "loss": 0.6963,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -0.007009805645793676,
+      "rewards/margins": -0.0018279661890119314,
+      "rewards/rejected": -0.005181840155273676,
       "step": 360
     },
     {
       "epoch": 0.96,
       "learning_rate": 2.3150941078050324e-09,
-      "logits/chosen": -2.069509744644165,
-      "logits/rejected": -2.0629658699035645,
-      "logps/chosen": -33.647117614746094,
-      "logps/rejected": -29.217060089111328,
-      "loss": 1.0027,
-      "rewards/accuracies": 0.48750001192092896,
-      "rewards/chosen": 0.002911838237196207,
-      "rewards/margins": -0.0027081891894340515,
-      "rewards/rejected": 0.005620025563985109,
+      "logits/chosen": -2.069504737854004,
+      "logits/rejected": -2.062945604324341,
+      "logps/chosen": -33.646629333496094,
+      "logps/rejected": -29.217721939086914,
+      "loss": 0.6968,
+      "rewards/accuracies": 0.4749999940395355,
+      "rewards/chosen": 0.003734806552529335,
+      "rewards/margins": -0.0023581120185554028,
+      "rewards/rejected": 0.0060929167084395885,
       "step": 370
     },
     {
       "epoch": 0.99,
       "learning_rate": 2.575864278703266e-10,
-      "logits/chosen": -1.929449439048767,
-      "logits/rejected": -1.9315989017486572,
-      "logps/chosen": -34.25099563598633,
-      "logps/rejected": -30.906116485595703,
-      "loss": 0.9871,
-      "rewards/accuracies": 0.5249999761581421,
-      "rewards/chosen": -0.007165629416704178,
-      "rewards/margins": 0.012927901931107044,
-      "rewards/rejected": -0.020093530416488647,
+      "logits/chosen": -1.9288103580474854,
+      "logits/rejected": -1.9309613704681396,
+      "logps/chosen": -34.23542022705078,
+      "logps/rejected": -30.89495277404785,
+      "loss": 0.6866,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": 0.0025402153842151165,
+      "rewards/margins": 0.018168382346630096,
+      "rewards/rejected": -0.015628164634108543,
       "step": 380
     },
     {
       "epoch": 1.0,
       "step": 385,
       "total_flos": 0.0,
-      "train_loss": 0.21985046337177228,
-      "train_runtime": 627.6127,
-      "train_samples_per_second": 4.906,
+      "train_loss": 0.15254525333255917,
+      "train_runtime": 628.1072,
+      "train_samples_per_second": 4.902,
       "train_steps_per_second": 0.613
     }
   ],

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1a26fc54db51500ff032def9cd7c93907331800443ff434cbd6342d49b66b060
+oid sha256:132c7af23e7be6513ca04c409bb78d8e95ded32cd1a07d465fdf23eb192ce28e
 size 4984