Model save

README.md

@@ -1,13 +1,11 @@
 ---
 library_name: peft
 tags:
-- alignment-handbook
 - trl
 - dpo
+- alignment-handbook
 - generated_from_trainer
 base_model: NbAiLab/nb-gpt-j-6B-v2
-datasets:
-- hugodk-sch/aftonposten_title_prefs
 model-index:
 - name: aftonposten-6b-align-scan
   results: []
@@ -18,17 +16,17 @@ should probably proofread and complete it, then remove this comment. -->
 
 # aftonposten-6b-align-scan
 
-This model is a fine-tuned version of [data/ap-gpt-j-6b-sft-qlora-04-08](https://huggingface.co/data/ap-gpt-j-6b-sft-qlora-04-08) on the hugodk-sch/aftonposten_title_prefs dataset.
+This model is a fine-tuned version of [NbAiLab/nb-gpt-j-6B-v2](https://huggingface.co/NbAiLab/nb-gpt-j-6B-v2) on an unknown dataset.
 It achieves the following results on the evaluation set:
-- Loss: 0.9960
-- Rewards/chosen: 0.0097
-- Rewards/rejected: 0.0053
-- Rewards/accuracies: 0.5141
-- Rewards/margins: 0.0044
-- Logps/rejected: -37.5100
-- Logps/chosen: -34.0224
-- Logits/rejected: -2.2385
-- Logits/chosen: -2.2434
+- Logits/chosen: -2.2437
+- Logits/rejected: -2.2388
+- Logps/chosen: -34.0172
+- Logps/rejected: -37.4976
+- Loss: 0.6982
+- Rewards/accuracies: 0.4983
+- Rewards/chosen: 0.0156
+- Rewards/margins: -0.0015
+- Rewards/rejected: 0.0171
 
 ## Model description
 
@@ -63,9 +61,9 @@ The following hyperparameters were used during training:
 
 | Training Loss | Epoch | Step | Logits/chosen | Logits/rejected | Logps/chosen | Logps/rejected | Validation Loss | Rewards/accuracies | Rewards/chosen | Rewards/margins | Rewards/rejected |
 |:-------------:|:-----:|:----:|:-------------:|:---------------:|:------------:|:--------------:|:---------------:|:------------------:|:--------------:|:---------------:|:----------------:|
-| 1.0026        | 0.26  | 100  | -2.2433       | -2.2384         | -34.0232     | -37.4972       | 1.0066          | 0.4983             | 0.0091         | -0.0064         | 0.0155           |
-| 0.9923        | 0.52  | 200  | -2.2438       | -2.2389         | -34.0187     | -37.5175       | 0.9864          | 0.5328             | 0.0127         | 0.0134          | -0.0007          |
-| 0.9775        | 0.78  | 300  | -2.2435       | -2.2386         | -34.0177     | -37.5002       | 0.9998          | 0.4904             | 0.0135         | 0.0004          | 0.0132           |
+| 0.6951        | 0.26  | 100  | -2.2438       | -2.2389         | -34.0252     | -37.5076       | 0.6966          | 0.4954             | 0.0084         | 0.0003          | 0.0081           |
+| 0.6891        | 0.52  | 200  | -2.2432       | -2.2384         | -34.0243     | -37.5115       | 0.6947          | 0.4934             | 0.0092         | 0.0046          | 0.0046           |
+| 0.693         | 0.78  | 300  | -2.2437       | -2.2388         | -34.0172     | -37.4976       | 0.6982          | 0.4983             | 0.0156         | -0.0015         | 0.0171           |
 
 
 ### Framework versions

adapter_config.json

@@ -19,13 +19,13 @@
   "rank_pattern": {},
   "revision": null,
   "target_modules": [
-    "k_proj",
-    "down_proj",
-    "o_proj",
-    "gate_proj",
     "up_proj",
     "v_proj",
-    "q_proj"
+    "k_proj",
+    "q_proj",
+    "gate_proj",
+    "down_proj",
+    "o_proj"
   ],
   "task_type": "CAUSAL_LM",
   "use_rslora": false

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:726700f20fcb28f7858cd1aa59e0cb2bd53a0fe4b7d2b662333531d42f0321bf
+oid sha256:b30a3614e90564741bd0b06f4b5d211ac9ab0d5d4651e82dec2b571e49a407d7
 size 176183216

all_results.json

@@ -1,21 +1,21 @@
 {
     "epoch": 1.0,
-    "eval_logits/chosen": -2.2434139251708984,
-    "eval_logits/rejected": -2.2385354042053223,
-    "eval_logps/chosen": -34.0224494934082,
-    "eval_logps/rejected": -37.51002883911133,
-    "eval_loss": 0.9960440397262573,
-    "eval_rewards/accuracies": 0.5141196250915527,
-    "eval_rewards/chosen": 0.0096820630133152,
-    "eval_rewards/margins": 0.004409347660839558,
-    "eval_rewards/rejected": 0.005272718146443367,
-    "eval_runtime": 146.0994,
+    "eval_logits/chosen": -2.244168758392334,
+    "eval_logits/rejected": -2.239292860031128,
+    "eval_logps/chosen": -34.02364730834961,
+    "eval_logps/rejected": -37.501853942871094,
+    "eval_loss": 0.6989675760269165,
+    "eval_rewards/accuracies": 0.49958473443984985,
+    "eval_rewards/chosen": 0.009816422127187252,
+    "eval_rewards/margins": -0.0034731123596429825,
+    "eval_rewards/rejected": 0.01328953169286251,
+    "eval_runtime": 145.7819,
     "eval_samples": 343,
-    "eval_samples_per_second": 2.348,
-    "eval_steps_per_second": 0.294,
-    "train_loss": 0.21957750196580764,
-    "train_runtime": 627.8531,
+    "eval_samples_per_second": 2.353,
+    "eval_steps_per_second": 0.295,
+    "train_loss": 0.15265155210123435,
+    "train_runtime": 627.689,
     "train_samples": 3079,
-    "train_samples_per_second": 4.904,
+    "train_samples_per_second": 4.905,
     "train_steps_per_second": 0.613
 }

eval_results.json

@@ -1,16 +1,16 @@
 {
     "epoch": 1.0,
-    "eval_logits/chosen": -2.2434139251708984,
-    "eval_logits/rejected": -2.2385354042053223,
-    "eval_logps/chosen": -34.0224494934082,
-    "eval_logps/rejected": -37.51002883911133,
-    "eval_loss": 0.9960440397262573,
-    "eval_rewards/accuracies": 0.5141196250915527,
-    "eval_rewards/chosen": 0.0096820630133152,
-    "eval_rewards/margins": 0.004409347660839558,
-    "eval_rewards/rejected": 0.005272718146443367,
-    "eval_runtime": 146.0994,
+    "eval_logits/chosen": -2.244168758392334,
+    "eval_logits/rejected": -2.239292860031128,
+    "eval_logps/chosen": -34.02364730834961,
+    "eval_logps/rejected": -37.501853942871094,
+    "eval_loss": 0.6989675760269165,
+    "eval_rewards/accuracies": 0.49958473443984985,
+    "eval_rewards/chosen": 0.009816422127187252,
+    "eval_rewards/margins": -0.0034731123596429825,
+    "eval_rewards/rejected": 0.01328953169286251,
+    "eval_runtime": 145.7819,
     "eval_samples": 343,
-    "eval_samples_per_second": 2.348,
-    "eval_steps_per_second": 0.294
+    "eval_samples_per_second": 2.353,
+    "eval_steps_per_second": 0.295
 }

train_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 1.0,
-    "train_loss": 0.21957750196580764,
-    "train_runtime": 627.8531,
+    "train_loss": 0.15265155210123435,
+    "train_runtime": 627.689,
     "train_samples": 3079,
-    "train_samples_per_second": 4.904,
+    "train_samples_per_second": 4.905,
     "train_steps_per_second": 0.613
 }

trainer_state.json

@@ -15,7 +15,7 @@
       "logits/rejected": -1.7377450466156006,
       "logps/chosen": -29.553977966308594,
       "logps/rejected": -42.813133239746094,
-      "loss": 1.0,
+      "loss": 0.6931,
       "rewards/accuracies": 0.0,
       "rewards/chosen": 0.0,
       "rewards/margins": 0.0,
@@ -25,590 +25,590 @@
     {
       "epoch": 0.03,
       "learning_rate": 1.2820512820512818e-07,
-      "logits/chosen": -1.866065502166748,
-      "logits/rejected": -1.8703795671463013,
-      "logps/chosen": -36.988380432128906,
-      "logps/rejected": -33.66728210449219,
-      "loss": 0.9557,
-      "rewards/accuracies": 0.5694444179534912,
-      "rewards/chosen": 0.014575847424566746,
-      "rewards/margins": 0.04434243589639664,
-      "rewards/rejected": -0.02976658195257187,
+      "logits/chosen": -1.866059422492981,
+      "logits/rejected": -1.870389699935913,
+      "logps/chosen": -36.98250198364258,
+      "logps/rejected": -33.64270782470703,
+      "loss": 0.6804,
+      "rewards/accuracies": 0.4027777910232544,
+      "rewards/chosen": 0.021689780056476593,
+      "rewards/margins": 0.033059459179639816,
+      "rewards/rejected": -0.011369682848453522,
       "step": 10
     },
     {
       "epoch": 0.05,
       "learning_rate": 2.5641025641025636e-07,
-      "logits/chosen": -1.997332215309143,
-      "logits/rejected": -1.999983549118042,
-      "logps/chosen": -29.625896453857422,
-      "logps/rejected": -29.035802841186523,
-      "loss": 1.0091,
-      "rewards/accuracies": 0.4375,
-      "rewards/chosen": 0.013037634082138538,
-      "rewards/margins": -0.009074507281184196,
-      "rewards/rejected": 0.02211214043200016,
+      "logits/chosen": -1.9973828792572021,
+      "logits/rejected": -2.0000317096710205,
+      "logps/chosen": -29.631702423095703,
+      "logps/rejected": -29.05877113342285,
+      "loss": 0.6937,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": 0.009445475414395332,
+      "rewards/margins": 0.005240145605057478,
+      "rewards/rejected": 0.004205327946692705,
       "step": 20
     },
     {
       "epoch": 0.08,
       "learning_rate": 3.8461538461538463e-07,
-      "logits/chosen": -1.9199495315551758,
-      "logits/rejected": -1.917249321937561,
-      "logps/chosen": -31.421478271484375,
-      "logps/rejected": -33.2115364074707,
-      "loss": 1.0098,
-      "rewards/accuracies": 0.4625000059604645,
-      "rewards/chosen": -0.004308355040848255,
-      "rewards/margins": -0.009847955778241158,
-      "rewards/rejected": 0.005539602134376764,
+      "logits/chosen": -1.9198739528656006,
+      "logits/rejected": -1.9171788692474365,
+      "logps/chosen": -31.4038028717041,
+      "logps/rejected": -33.20899963378906,
+      "loss": 0.696,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": 0.01106296293437481,
+      "rewards/margins": 0.0025471807457506657,
+      "rewards/rejected": 0.008515783585608006,
       "step": 30
     },
     {
       "epoch": 0.1,
       "learning_rate": 4.999896948438433e-07,
-      "logits/chosen": -2.0169284343719482,
-      "logits/rejected": -2.008178949356079,
-      "logps/chosen": -32.59435272216797,
-      "logps/rejected": -32.49193572998047,
-      "loss": 1.0306,
-      "rewards/accuracies": 0.44999998807907104,
-      "rewards/chosen": -0.014034454710781574,
-      "rewards/margins": -0.030623534694314003,
-      "rewards/rejected": 0.016589080914855003,
+      "logits/chosen": -2.0166726112365723,
+      "logits/rejected": -2.0079195499420166,
+      "logps/chosen": -32.57840347290039,
+      "logps/rejected": -32.493370056152344,
+      "loss": 0.7066,
+      "rewards/accuracies": 0.48750001192092896,
+      "rewards/chosen": -0.00143451988697052,
+      "rewards/margins": -0.018809262663125992,
+      "rewards/rejected": 0.017374742776155472,
       "step": 40
     },
     {
       "epoch": 0.13,
       "learning_rate": 4.987541037542186e-07,
-      "logits/chosen": -1.86457097530365,
-      "logits/rejected": -1.8537908792495728,
-      "logps/chosen": -33.56566619873047,
-      "logps/rejected": -35.423240661621094,
-      "loss": 1.0298,
-      "rewards/accuracies": 0.4375,
-      "rewards/chosen": -0.006550998892635107,
-      "rewards/margins": -0.029834842309355736,
-      "rewards/rejected": 0.02328384295105934,
+      "logits/chosen": -1.8648544549942017,
+      "logits/rejected": -1.8540741205215454,
+      "logps/chosen": -33.54566955566406,
+      "logps/rejected": -35.43162155151367,
+      "loss": 0.7004,
+      "rewards/accuracies": 0.4749999940395355,
+      "rewards/chosen": 0.01062955055385828,
+      "rewards/margins": -0.00802132673561573,
+      "rewards/rejected": 0.018650878220796585,
       "step": 50
     },
     {
       "epoch": 0.16,
       "learning_rate": 4.954691471941118e-07,
-      "logits/chosen": -1.9449050426483154,
-      "logits/rejected": -1.9468472003936768,
-      "logps/chosen": -32.59955596923828,
-      "logps/rejected": -33.1828498840332,
-      "loss": 0.9903,
-      "rewards/accuracies": 0.512499988079071,
-      "rewards/chosen": 0.0005022765835747123,
-      "rewards/margins": 0.009675229899585247,
-      "rewards/rejected": -0.00917295552790165,
+      "logits/chosen": -1.9450620412826538,
+      "logits/rejected": -1.947021245956421,
+      "logps/chosen": -32.5836296081543,
+      "logps/rejected": -33.20051193237305,
+      "loss": 0.6761,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": 0.014901289716362953,
+      "rewards/margins": 0.04112350940704346,
+      "rewards/rejected": -0.026222219690680504,
       "step": 60
     },
     {
       "epoch": 0.18,
       "learning_rate": 4.901618883413548e-07,
-      "logits/chosen": -2.079878330230713,
-      "logits/rejected": -2.084862232208252,
-      "logps/chosen": -33.98878860473633,
-      "logps/rejected": -36.574462890625,
-      "loss": 0.9958,
+      "logits/chosen": -2.079589366912842,
+      "logits/rejected": -2.0845787525177,
+      "logps/chosen": -33.9754524230957,
+      "logps/rejected": -36.5793342590332,
+      "loss": 0.6876,
       "rewards/accuracies": 0.512499988079071,
-      "rewards/chosen": -0.00024233898147940636,
-      "rewards/margins": 0.004231657367199659,
-      "rewards/rejected": -0.004473996348679066,
+      "rewards/chosen": 0.01173111516982317,
+      "rewards/margins": 0.021146392449736595,
+      "rewards/rejected": -0.00941527634859085,
       "step": 70
     },
     {
       "epoch": 0.21,
       "learning_rate": 4.828760511501322e-07,
-      "logits/chosen": -1.9425569772720337,
-      "logits/rejected": -1.9457191228866577,
-      "logps/chosen": -34.40068054199219,
-      "logps/rejected": -34.5762939453125,
-      "loss": 0.9912,
-      "rewards/accuracies": 0.5249999761581421,
-      "rewards/chosen": 0.021701809018850327,
-      "rewards/margins": 0.00881027989089489,
-      "rewards/rejected": 0.012891530990600586,
+      "logits/chosen": -1.9424660205841064,
+      "logits/rejected": -1.945634126663208,
+      "logps/chosen": -34.394561767578125,
+      "logps/rejected": -34.57395553588867,
+      "loss": 0.6921,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": 0.029921507462859154,
+      "rewards/margins": 0.013314949348568916,
+      "rewards/rejected": 0.016606558114290237,
       "step": 80
     },
     {
       "epoch": 0.23,
       "learning_rate": 4.736716601303429e-07,
-      "logits/chosen": -1.9507396221160889,
-      "logits/rejected": -1.9552500247955322,
-      "logps/chosen": -32.460357666015625,
-      "logps/rejected": -32.354434967041016,
-      "loss": 0.9979,
-      "rewards/accuracies": 0.512499988079071,
-      "rewards/chosen": 0.005654479376971722,
-      "rewards/margins": 0.002074276329949498,
-      "rewards/rejected": 0.0035802037455141544,
+      "logits/chosen": -1.950564980506897,
+      "logits/rejected": -1.955082893371582,
+      "logps/chosen": -32.44951248168945,
+      "logps/rejected": -32.353668212890625,
+      "loss": 0.6909,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": 0.0161209125071764,
+      "rewards/margins": 0.011405264027416706,
+      "rewards/rejected": 0.004715651273727417,
       "step": 90
     },
     {
       "epoch": 0.26,
       "learning_rate": 4.62624545834521e-07,
-      "logits/chosen": -2.0492873191833496,
-      "logits/rejected": -2.0472888946533203,
-      "logps/chosen": -32.23810958862305,
-      "logps/rejected": -31.260278701782227,
-      "loss": 1.0026,
-      "rewards/accuracies": 0.4625000059604645,
-      "rewards/chosen": 0.0021828364115208387,
-      "rewards/margins": -0.002563956892117858,
-      "rewards/rejected": 0.004746791906654835,
+      "logits/chosen": -2.0489234924316406,
+      "logits/rejected": -2.046917200088501,
+      "logps/chosen": -32.236671447753906,
+      "logps/rejected": -31.263763427734375,
+      "loss": 0.6951,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": 0.0037471160758286715,
+      "rewards/margins": 0.0015437646070495248,
+      "rewards/rejected": 0.002203352050855756,
       "step": 100
     },
     {
       "epoch": 0.26,
-      "eval_logits/chosen": -2.243312358856201,
-      "eval_logits/rejected": -2.238436222076416,
-      "eval_logps/chosen": -34.023216247558594,
-      "eval_logps/rejected": -37.49723434448242,
-      "eval_loss": 1.0065712928771973,
-      "eval_rewards/accuracies": 0.49833887815475464,
-      "eval_rewards/chosen": 0.009070919826626778,
-      "eval_rewards/margins": -0.006438401062041521,
-      "eval_rewards/rejected": 0.015509321354329586,
-      "eval_runtime": 146.1739,
-      "eval_samples_per_second": 2.347,
+      "eval_logits/chosen": -2.243756055831909,
+      "eval_logits/rejected": -2.238872766494751,
+      "eval_logps/chosen": -34.025177001953125,
+      "eval_logps/rejected": -37.50759506225586,
+      "eval_loss": 0.696557343006134,
+      "eval_rewards/accuracies": 0.49543190002441406,
+      "eval_rewards/chosen": 0.008434689603745937,
+      "eval_rewards/margins": 0.0003138432221021503,
+      "eval_rewards/rejected": 0.008120844140648842,
+      "eval_runtime": 146.3373,
+      "eval_samples_per_second": 2.344,
       "eval_steps_per_second": 0.294,
       "step": 100
     },
     {
       "epoch": 0.29,
       "learning_rate": 4.4982572012636904e-07,
-      "logits/chosen": -2.005169153213501,
-      "logits/rejected": -2.0027499198913574,
-      "logps/chosen": -33.2365837097168,
-      "logps/rejected": -34.01953125,
-      "loss": 1.0021,
-      "rewards/accuracies": 0.4625000059604645,
-      "rewards/chosen": 0.005784005858004093,
-      "rewards/margins": -0.002089007291942835,
-      "rewards/rejected": 0.007873011752963066,
+      "logits/chosen": -2.0052237510681152,
+      "logits/rejected": -2.002803325653076,
+      "logps/chosen": -33.24525833129883,
+      "logps/rejected": -34.01074981689453,
+      "loss": 0.7064,
+      "rewards/accuracies": 0.48750001192092896,
+      "rewards/chosen": -0.0013012334238737822,
+      "rewards/margins": -0.018059223890304565,
+      "rewards/rejected": 0.016757991164922714,
       "step": 110
     },
     {
       "epoch": 0.31,
       "learning_rate": 4.353806263777677e-07,
-      "logits/chosen": -2.0166728496551514,
-      "logits/rejected": -2.0083022117614746,
-      "logps/chosen": -32.457847595214844,
-      "logps/rejected": -32.18357467651367,
-      "loss": 0.9927,
-      "rewards/accuracies": 0.550000011920929,
-      "rewards/chosen": -0.010353903286159039,
-      "rewards/margins": 0.0072973729111254215,
-      "rewards/rejected": -0.017651278525590897,
+      "logits/chosen": -2.016326665878296,
+      "logits/rejected": -2.007951498031616,
+      "logps/chosen": -32.478302001953125,
+      "logps/rejected": -32.186073303222656,
+      "loss": 0.7012,
+      "rewards/accuracies": 0.48750001192092896,
+      "rewards/chosen": -0.030056962743401527,
+      "rewards/margins": -0.007947373203933239,
+      "rewards/rejected": -0.022109590470790863,
       "step": 120
     },
     {
       "epoch": 0.34,
       "learning_rate": 4.194082707715275e-07,
-      "logits/chosen": -2.0462448596954346,
-      "logits/rejected": -2.038203001022339,
-      "logps/chosen": -30.475027084350586,
-      "logps/rejected": -32.046302795410156,
-      "loss": 1.0048,
-      "rewards/accuracies": 0.48750001192092896,
-      "rewards/chosen": 0.0005506269517354667,
-      "rewards/margins": -0.004812211729586124,
-      "rewards/rejected": 0.005362839438021183,
+      "logits/chosen": -2.045950412750244,
+      "logits/rejected": -2.037904739379883,
+      "logps/chosen": -30.491313934326172,
+      "logps/rejected": -32.0540771484375,
+      "loss": 0.7025,
+      "rewards/accuracies": 0.4375,
+      "rewards/chosen": -0.01403898186981678,
+      "rewards/margins": -0.013075167313218117,
+      "rewards/rejected": -0.0009638145565986633,
       "step": 130
     },
     {
       "epoch": 0.36,
       "learning_rate": 4.020402418666621e-07,
-      "logits/chosen": -1.976986289024353,
-      "logits/rejected": -1.9872560501098633,
-      "logps/chosen": -31.407278060913086,
-      "logps/rejected": -32.543296813964844,
-      "loss": 0.9834,
-      "rewards/accuracies": 0.48750001192092896,
-      "rewards/chosen": 0.009907384403049946,
-      "rewards/margins": 0.016641682013869286,
-      "rewards/rejected": -0.006734299007803202,
+      "logits/chosen": -1.9765291213989258,
+      "logits/rejected": -1.986802101135254,
+      "logps/chosen": -31.39646339416504,
+      "logps/rejected": -32.522178649902344,
+      "loss": 0.6923,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": 0.02087983302772045,
+      "rewards/margins": 0.00944516807794571,
+      "rewards/rejected": 0.011434664018452168,
       "step": 140
     },
     {
       "epoch": 0.39,
       "learning_rate": 3.8341962650351185e-07,
-      "logits/chosen": -1.8905508518218994,
-      "logits/rejected": -1.891632080078125,
-      "logps/chosen": -34.20501708984375,
-      "logps/rejected": -34.77235412597656,
-      "loss": 0.9978,
-      "rewards/accuracies": 0.4625000059604645,
-      "rewards/chosen": -0.009861050173640251,
-      "rewards/margins": 0.0022162157110869884,
-      "rewards/rejected": -0.012077265419065952,
+      "logits/chosen": -1.8905407190322876,
+      "logits/rejected": -1.8916336297988892,
+      "logps/chosen": -34.17607116699219,
+      "logps/rejected": -34.76817321777344,
+      "loss": 0.6851,
+      "rewards/accuracies": 0.4749999940395355,
+      "rewards/chosen": 0.01495879702270031,
+      "rewards/margins": 0.024784717708826065,
+      "rewards/rejected": -0.00982591975480318,
       "step": 150
     },
     {
       "epoch": 0.42,
       "learning_rate": 3.636998309800572e-07,
-      "logits/chosen": -1.94281005859375,
-      "logits/rejected": -1.939327597618103,
-      "logps/chosen": -36.144107818603516,
-      "logps/rejected": -32.72822570800781,
-      "loss": 0.9736,
-      "rewards/accuracies": 0.5375000238418579,
-      "rewards/chosen": 0.02205902896821499,
-      "rewards/margins": 0.026437148451805115,
-      "rewards/rejected": -0.004378115758299828,
+      "logits/chosen": -1.942565679550171,
+      "logits/rejected": -1.9390919208526611,
+      "logps/chosen": -36.1614875793457,
+      "logps/rejected": -32.72848129272461,
+      "loss": 0.6907,
+      "rewards/accuracies": 0.4625000059604645,
+      "rewards/chosen": 0.00917207170277834,
+      "rewards/margins": 0.01433003693819046,
+      "rewards/rejected": -0.0051579661667346954,
       "step": 160
     },
     {
       "epoch": 0.44,
       "learning_rate": 3.430433172111807e-07,
-      "logits/chosen": -2.0420708656311035,
-      "logits/rejected": -2.0346803665161133,
-      "logps/chosen": -33.771934509277344,
-      "logps/rejected": -31.371145248413086,
-      "loss": 0.9638,
-      "rewards/accuracies": 0.5375000238418579,
-      "rewards/chosen": 0.020745161920785904,
-      "rewards/margins": 0.036180444061756134,
-      "rewards/rejected": -0.015435280278325081,
+      "logits/chosen": -2.0418710708618164,
+      "logits/rejected": -2.034475564956665,
+      "logps/chosen": -33.76544952392578,
+      "logps/rejected": -31.358556747436523,
+      "loss": 0.6791,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.02917439118027687,
+      "rewards/margins": 0.03521214798092842,
+      "rewards/rejected": -0.0060377540066838264,
       "step": 170
     },
     {
       "epoch": 0.47,
       "learning_rate": 3.216202642830543e-07,
-      "logits/chosen": -2.0475738048553467,
-      "logits/rejected": -2.0528526306152344,
-      "logps/chosen": -32.524593353271484,
-      "logps/rejected": -32.510643005371094,
-      "loss": 0.9634,
-      "rewards/accuracies": 0.550000011920929,
-      "rewards/chosen": 0.016154326498508453,
-      "rewards/margins": 0.036555372178554535,
-      "rewards/rejected": -0.020401041954755783,
+      "logits/chosen": -2.0476231575012207,
+      "logits/rejected": -2.0529019832611084,
+      "logps/chosen": -32.533023834228516,
+      "logps/rejected": -32.506805419921875,
+      "loss": 0.6814,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": 0.010587882250547409,
+      "rewards/margins": 0.03008626028895378,
+      "rewards/rejected": -0.019498378038406372,
       "step": 180
     },
     {
       "epoch": 0.49,
       "learning_rate": 2.9960716642946403e-07,
-      "logits/chosen": -2.048490524291992,
-      "logits/rejected": -2.0457024574279785,
-      "logps/chosen": -31.492746353149414,
-      "logps/rejected": -31.319293975830078,
-      "loss": 1.0022,
-      "rewards/accuracies": 0.5,
-      "rewards/chosen": -0.003033895045518875,
-      "rewards/margins": -0.002170243766158819,
-      "rewards/rejected": -0.0008636537822894752,
+      "logits/chosen": -2.048576593399048,
+      "logits/rejected": -2.0457961559295654,
+      "logps/chosen": -31.470510482788086,
+      "logps/rejected": -31.322284698486328,
+      "loss": 0.687,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": 0.016599375754594803,
+      "rewards/margins": 0.020261235535144806,
+      "rewards/rejected": -0.0036618602462112904,
       "step": 190
     },
     {
       "epoch": 0.52,
       "learning_rate": 2.771853789806683e-07,
-      "logits/chosen": -1.9185683727264404,
-      "logits/rejected": -1.9232347011566162,
-      "logps/chosen": -31.5926513671875,
-      "logps/rejected": -32.78697204589844,
-      "loss": 0.9923,
-      "rewards/accuracies": 0.5874999761581421,
-      "rewards/chosen": 0.01090637780725956,
-      "rewards/margins": 0.007740379776805639,
-      "rewards/rejected": 0.0031659987289458513,
+      "logits/chosen": -1.9184268712997437,
+      "logits/rejected": -1.9230947494506836,
+      "logps/chosen": -31.57404136657715,
+      "logps/rejected": -32.775550842285156,
+      "loss": 0.6891,
+      "rewards/accuracies": 0.48750001192092896,
+      "rewards/chosen": 0.02901501953601837,
+      "rewards/margins": 0.015173261985182762,
+      "rewards/rejected": 0.013841753825545311,
       "step": 200
     },
     {
       "epoch": 0.52,
-      "eval_logits/chosen": -2.243807315826416,
-      "eval_logits/rejected": -2.2389235496520996,
-      "eval_logps/chosen": -34.018714904785156,
-      "eval_logps/rejected": -37.517478942871094,
-      "eval_loss": 0.9864435195922852,
-      "eval_rewards/accuracies": 0.5328072905540466,
-      "eval_rewards/chosen": 0.012671924196183681,
-      "eval_rewards/margins": 0.013361074030399323,
-      "eval_rewards/rejected": -0.0006891504744999111,
-      "eval_runtime": 145.9847,
-      "eval_samples_per_second": 2.35,
-      "eval_steps_per_second": 0.295,
+      "eval_logits/chosen": -2.243248462677002,
+      "eval_logits/rejected": -2.238374948501587,
+      "eval_logps/chosen": -34.02431869506836,
+      "eval_logps/rejected": -37.511497497558594,
+      "eval_loss": 0.6946919560432434,
+      "eval_rewards/accuracies": 0.49335551261901855,
+      "eval_rewards/chosen": 0.009210066869854927,
+      "eval_rewards/margins": 0.0045979218557477,
+      "eval_rewards/rejected": 0.004612144082784653,
+      "eval_runtime": 146.0514,
+      "eval_samples_per_second": 2.348,
+      "eval_steps_per_second": 0.294,
       "step": 200
     },
     {
       "epoch": 0.55,
       "learning_rate": 2.5453962426402e-07,
-      "logits/chosen": -2.0318503379821777,
-      "logits/rejected": -2.042539596557617,
-      "logps/chosen": -31.948400497436523,
-      "logps/rejected": -33.86983871459961,
-      "loss": 0.976,
-      "rewards/accuracies": 0.5,
-      "rewards/chosen": 0.018323851749300957,
-      "rewards/margins": 0.024035325273871422,
-      "rewards/rejected": -0.005711475852876902,
+      "logits/chosen": -2.0314748287200928,
+      "logits/rejected": -2.0421440601348877,
+      "logps/chosen": -31.954639434814453,
+      "logps/rejected": -33.87318801879883,
+      "loss": 0.6864,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.014999927952885628,
+      "rewards/margins": 0.024442464113235474,
+      "rewards/rejected": -0.009442536160349846,
       "step": 210
     },
     {
       "epoch": 0.57,
       "learning_rate": 2.318564697655179e-07,
-      "logits/chosen": -1.9251388311386108,
-      "logits/rejected": -1.9399843215942383,
-      "logps/chosen": -30.099853515625,
-      "logps/rejected": -31.55409812927246,
-      "loss": 0.9939,
-      "rewards/accuracies": 0.5249999761581421,
-      "rewards/chosen": 0.009348717518150806,
-      "rewards/margins": 0.006087350193411112,
-      "rewards/rejected": 0.003261365694925189,
+      "logits/chosen": -1.9249544143676758,
+      "logits/rejected": -1.9398053884506226,
+      "logps/chosen": -30.072036743164062,
+      "logps/rejected": -31.55165672302246,
+      "loss": 0.6827,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.035550959408283234,
+      "rewards/margins": 0.029681822285056114,
+      "rewards/rejected": 0.005869132932275534,
       "step": 220
     },
     {
       "epoch": 0.6,
       "learning_rate": 2.093227910899832e-07,
-      "logits/chosen": -1.9835479259490967,
-      "logits/rejected": -1.9875112771987915,
-      "logps/chosen": -33.387638092041016,
-      "logps/rejected": -31.554845809936523,
-      "loss": 0.9717,
-      "rewards/accuracies": 0.550000011920929,
-      "rewards/chosen": 0.023866940289735794,
-      "rewards/margins": 0.028299665078520775,
-      "rewards/rejected": -0.00443272665143013,
+      "logits/chosen": -1.9830297231674194,
+      "logits/rejected": -1.9869968891143799,
+      "logps/chosen": -33.3900146484375,
+      "logps/rejected": -31.545156478881836,
+      "loss": 0.6872,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": 0.024708259850740433,
+      "rewards/margins": 0.02097567543387413,
+      "rewards/rejected": 0.00373258744366467,
       "step": 230
     },
     {
       "epoch": 0.62,
       "learning_rate": 1.8712423238279356e-07,
-      "logits/chosen": -1.9826898574829102,
-      "logits/rejected": -1.9607274532318115,
-      "logps/chosen": -34.158443450927734,
-      "logps/rejected": -34.963783264160156,
-      "loss": 0.9934,
-      "rewards/accuracies": 0.512499988079071,
-      "rewards/chosen": -0.0037571662105619907,
-      "rewards/margins": 0.0066327848471701145,
-      "rewards/rejected": -0.01038995012640953,
+      "logits/chosen": -1.9825174808502197,
+      "logits/rejected": -1.9605509042739868,
+      "logps/chosen": -34.16549301147461,
+      "logps/rejected": -34.945499420166016,
+      "loss": 0.7065,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -0.010572168976068497,
+      "rewards/margins": -0.015337007120251656,
+      "rewards/rejected": 0.004764837212860584,
       "step": 240
     },
     {
       "epoch": 0.65,
       "learning_rate": 1.654436768970182e-07,
-      "logits/chosen": -2.024381160736084,
-      "logits/rejected": -2.0210862159729004,
-      "logps/chosen": -32.9254035949707,
-      "logps/rejected": -36.251712799072266,
-      "loss": 0.9788,
-      "rewards/accuracies": 0.5375000238418579,
-      "rewards/chosen": -0.0002662956831045449,
-      "rewards/margins": 0.02116318605840206,
-      "rewards/rejected": -0.021429482847452164,
+      "logits/chosen": -2.0242011547088623,
+      "logits/rejected": -2.02089262008667,
+      "logps/chosen": -32.91257858276367,
+      "logps/rejected": -36.22357177734375,
+      "loss": 0.6913,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": 0.01124012004584074,
+      "rewards/margins": 0.010020162910223007,
+      "rewards/rejected": 0.0012199539924040437,
       "step": 250
     },
     {
       "epoch": 0.68,
       "learning_rate": 1.444597403062196e-07,
-      "logits/chosen": -1.8911311626434326,
-      "logits/rejected": -1.8886839151382446,
-      "logps/chosen": -34.194557189941406,
-      "logps/rejected": -35.51445770263672,
-      "loss": 0.9971,
-      "rewards/accuracies": 0.48750001192092896,
-      "rewards/chosen": -0.004926486872136593,
-      "rewards/margins": 0.002939151134341955,
-      "rewards/rejected": -0.007865638472139835,
+      "logits/chosen": -1.8910300731658936,
+      "logits/rejected": -1.888593316078186,
+      "logps/chosen": -34.191619873046875,
+      "logps/rejected": -35.50851058959961,
+      "loss": 0.6975,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.0028982784133404493,
+      "rewards/margins": 0.0006028197822161019,
+      "rewards/rejected": -0.003501094877719879,
       "step": 260
     },
     {
       "epoch": 0.7,
       "learning_rate": 1.2434529917578887e-07,
-      "logits/chosen": -1.8759450912475586,
-      "logits/rejected": -1.8734045028686523,
-      "logps/chosen": -34.40558624267578,
-      "logps/rejected": -31.752349853515625,
-      "loss": 1.0216,
-      "rewards/accuracies": 0.4625000059604645,
-      "rewards/chosen": -0.0063878437504172325,
-      "rewards/margins": -0.02157408930361271,
-      "rewards/rejected": 0.015186244621872902,
+      "logits/chosen": -1.8757693767547607,
+      "logits/rejected": -1.8732143640518188,
+      "logps/chosen": -34.38648223876953,
+      "logps/rejected": -31.753692626953125,
+      "loss": 0.6995,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": 0.010007266886532307,
+      "rewards/margins": -0.005866709630936384,
+      "rewards/rejected": 0.015873977914452553,
       "step": 270
     },
     {
       "epoch": 0.73,
       "learning_rate": 1.0526606671603521e-07,
-      "logits/chosen": -1.980015754699707,
-      "logits/rejected": -1.9693737030029297,
-      "logps/chosen": -35.33230209350586,
-      "logps/rejected": -31.845691680908203,
-      "loss": 0.9768,
-      "rewards/accuracies": 0.512499988079071,
-      "rewards/chosen": 0.019471382722258568,
-      "rewards/margins": 0.023230455815792084,
-      "rewards/rejected": -0.0037590693682432175,
+      "logits/chosen": -1.9795001745224,
+      "logits/rejected": -1.9688692092895508,
+      "logps/chosen": -35.32746887207031,
+      "logps/rejected": -31.84292221069336,
+      "loss": 0.6843,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.026254409924149513,
+      "rewards/margins": 0.027990642935037613,
+      "rewards/rejected": -0.001736226724460721,
       "step": 280
     },
     {
       "epoch": 0.75,
       "learning_rate": 8.737922755071453e-08,
-      "logits/chosen": -2.0756278038024902,
-      "logits/rejected": -2.060606002807617,
-      "logps/chosen": -30.907390594482422,
-      "logps/rejected": -32.64055252075195,
-      "loss": 0.9956,
-      "rewards/accuracies": 0.550000011920929,
-      "rewards/chosen": 0.01338677667081356,
-      "rewards/margins": 0.0043600378558039665,
-      "rewards/rejected": 0.00902673788368702,
+      "logits/chosen": -2.0753533840179443,
+      "logits/rejected": -2.0603318214416504,
+      "logps/chosen": -30.902263641357422,
+      "logps/rejected": -32.65242385864258,
+      "loss": 0.6872,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": 0.019674303010106087,
+      "rewards/margins": 0.020201902836561203,
+      "rewards/rejected": -0.0005276011070236564,
       "step": 290
     },
     {
       "epoch": 0.78,
       "learning_rate": 7.08321427484816e-08,
-      "logits/chosen": -1.946616768836975,
-      "logits/rejected": -1.9440828561782837,
-      "logps/chosen": -32.894561767578125,
-      "logps/rejected": -30.812387466430664,
-      "loss": 0.9775,
-      "rewards/accuracies": 0.5,
-      "rewards/chosen": 0.026041675359010696,
-      "rewards/margins": 0.022502180188894272,
-      "rewards/rejected": 0.003539492143318057,
+      "logits/chosen": -1.946671724319458,
+      "logits/rejected": -1.9441486597061157,
+      "logps/chosen": -32.906158447265625,
+      "logps/rejected": -30.805999755859375,
+      "loss": 0.693,
+      "rewards/accuracies": 0.4749999940395355,
+      "rewards/chosen": 0.018858108669519424,
+      "rewards/margins": 0.009128611534833908,
+      "rewards/rejected": 0.009729497134685516,
       "step": 300
     },
     {
       "epoch": 0.78,
-      "eval_logits/chosen": -2.2434821128845215,
-      "eval_logits/rejected": -2.2386035919189453,
-      "eval_logps/chosen": -34.017669677734375,
-      "eval_logps/rejected": -37.50018310546875,
-      "eval_loss": 0.9997690916061401,
-      "eval_rewards/accuracies": 0.490448534488678,
-      "eval_rewards/chosen": 0.013507848605513573,
-      "eval_rewards/margins": 0.000357679498847574,
-      "eval_rewards/rejected": 0.0131501704454422,
-      "eval_runtime": 145.7344,
-      "eval_samples_per_second": 2.354,
-      "eval_steps_per_second": 0.295,
+      "eval_logits/chosen": -2.243694305419922,
+      "eval_logits/rejected": -2.2388274669647217,
+      "eval_logps/chosen": -34.017173767089844,
+      "eval_logps/rejected": -37.497615814208984,
+      "eval_loss": 0.6981561183929443,
+      "eval_rewards/accuracies": 0.49833887815475464,
+      "eval_rewards/chosen": 0.015641551464796066,
+      "eval_rewards/margins": -0.0014640000881627202,
+      "eval_rewards/rejected": 0.017105549573898315,
+      "eval_runtime": 146.0253,
+      "eval_samples_per_second": 2.349,
+      "eval_steps_per_second": 0.294,
       "step": 300
     },
     {
       "epoch": 0.81,
       "learning_rate": 5.576113578589034e-08,
-      "logits/chosen": -1.9287067651748657,
-      "logits/rejected": -1.925451636314392,
-      "logps/chosen": -31.603496551513672,
-      "logps/rejected": -33.734046936035156,
-      "loss": 1.0066,
-      "rewards/accuracies": 0.4749999940395355,
-      "rewards/chosen": -0.00011934786743950099,
-      "rewards/margins": -0.006604696158319712,
-      "rewards/rejected": 0.006485348101705313,
+      "logits/chosen": -1.9287757873535156,
+      "logits/rejected": -1.9255115985870361,
+      "logps/chosen": -31.573471069335938,
+      "logps/rejected": -33.71125030517578,
+      "loss": 0.698,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": 0.026891669258475304,
+      "rewards/margins": -0.000918733305297792,
+      "rewards/rejected": 0.02781040407717228,
       "step": 310
     },
     {
       "epoch": 0.83,
       "learning_rate": 4.229036944380912e-08,
-      "logits/chosen": -1.9802377223968506,
-      "logits/rejected": -1.9679291248321533,
-      "logps/chosen": -34.585323333740234,
-      "logps/rejected": -33.57084274291992,
-      "loss": 0.9658,
-      "rewards/accuracies": 0.5375000238418579,
-      "rewards/chosen": 0.00860034953802824,
-      "rewards/margins": 0.034203141927719116,
-      "rewards/rejected": -0.02560279332101345,
+      "logits/chosen": -1.9807313680648804,
+      "logits/rejected": -1.968425989151001,
+      "logps/chosen": -34.54930877685547,
+      "logps/rejected": -33.57306671142578,
+      "loss": 0.6615,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": 0.04208679869771004,
+      "rewards/margins": 0.07289497554302216,
+      "rewards/rejected": -0.030808180570602417,
       "step": 320
     },
     {
       "epoch": 0.86,
       "learning_rate": 3.053082288996112e-08,
-      "logits/chosen": -2.015906810760498,
-      "logits/rejected": -2.014427661895752,
-      "logps/chosen": -33.49116516113281,
-      "logps/rejected": -32.47978973388672,
-      "loss": 1.0041,
-      "rewards/accuracies": 0.4375,
-      "rewards/chosen": -0.0042965407483279705,
-      "rewards/margins": -0.004129170440137386,
-      "rewards/rejected": -0.00016737030819058418,
+      "logits/chosen": -2.0161099433898926,
+      "logits/rejected": -2.0146496295928955,
+      "logps/chosen": -33.46127700805664,
+      "logps/rejected": -32.4744873046875,
+      "loss": 0.6883,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": 0.022065162658691406,
+      "rewards/margins": 0.01747960038483143,
+      "rewards/rejected": 0.004585559479892254,
       "step": 330
     },
     {
       "epoch": 0.88,
       "learning_rate": 2.05793773749158e-08,
-      "logits/chosen": -2.1030631065368652,
-      "logits/rejected": -2.0872654914855957,
-      "logps/chosen": -34.18492889404297,
-      "logps/rejected": -33.08319854736328,
-      "loss": 1.0323,
-      "rewards/accuracies": 0.42500001192092896,
-      "rewards/chosen": -0.00442854966968298,
-      "rewards/margins": -0.03229869529604912,
-      "rewards/rejected": 0.027870142832398415,
+      "logits/chosen": -2.102802276611328,
+      "logits/rejected": -2.0869994163513184,
+      "logps/chosen": -34.16718292236328,
+      "logps/rejected": -33.08827590942383,
+      "loss": 0.706,
+      "rewards/accuracies": 0.48750001192092896,
+      "rewards/chosen": 0.010987209156155586,
+      "rewards/margins": -0.01579815149307251,
+      "rewards/rejected": 0.026785362511873245,
       "step": 340
     },
     {
       "epoch": 0.91,
       "learning_rate": 1.251801807404168e-08,
-      "logits/chosen": -1.9745471477508545,
-      "logits/rejected": -1.9736032485961914,
-      "logps/chosen": -33.23271942138672,
-      "logps/rejected": -32.4765510559082,
-      "loss": 0.9514,
-      "rewards/accuracies": 0.612500011920929,
-      "rewards/chosen": 0.0395023450255394,
-      "rewards/margins": 0.04862459748983383,
-      "rewards/rejected": -0.009122253395617008,
+      "logits/chosen": -1.97471022605896,
+      "logits/rejected": -1.9737637042999268,
+      "logps/chosen": -33.25645065307617,
+      "logps/rejected": -32.46539306640625,
+      "loss": 0.6861,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": 0.023080622777342796,
+      "rewards/margins": 0.023302335292100906,
+      "rewards/rejected": -0.00022171511955093592,
       "step": 350
     },
     {
       "epoch": 0.94,
       "learning_rate": 6.41315865106129e-09,
-      "logits/chosen": -1.9305438995361328,
-      "logits/rejected": -1.940913438796997,
-      "logps/chosen": -32.22040939331055,
-      "logps/rejected": -35.28728103637695,
-      "loss": 1.0138,
-      "rewards/accuracies": 0.48750001192092896,
-      "rewards/chosen": -0.012576865963637829,
-      "rewards/margins": -0.013758530840277672,
-      "rewards/rejected": 0.0011816651094704866,
+      "logits/chosen": -1.9304895401000977,
+      "logits/rejected": -1.9408460855484009,
+      "logps/chosen": -32.22315979003906,
+      "logps/rejected": -35.28199005126953,
+      "loss": 0.7085,
+      "rewards/accuracies": 0.42500001192092896,
+      "rewards/chosen": -0.01662321388721466,
+      "rewards/margins": -0.022716889157891273,
+      "rewards/rejected": 0.006093672942370176,
       "step": 360
     },
     {
       "epoch": 0.96,
       "learning_rate": 2.3150941078050324e-09,
-      "logits/chosen": -2.069648265838623,
-      "logits/rejected": -2.0630898475646973,
-      "logps/chosen": -33.63695526123047,
-      "logps/rejected": -29.226470947265625,
-      "loss": 0.988,
-      "rewards/accuracies": 0.5625,
-      "rewards/chosen": 0.012009462341666222,
-      "rewards/margins": 0.012046033516526222,
-      "rewards/rejected": -3.6573597753886133e-05,
+      "logits/chosen": -2.0692858695983887,
+      "logits/rejected": -2.0627408027648926,
+      "logps/chosen": -33.64545440673828,
+      "logps/rejected": -29.203441619873047,
+      "loss": 0.704,
+      "rewards/accuracies": 0.42500001192092896,
+      "rewards/chosen": 0.00586346909403801,
+      "rewards/margins": -0.014821496792137623,
+      "rewards/rejected": 0.020684964954853058,
       "step": 370
     },
     {
       "epoch": 0.99,
       "learning_rate": 2.575864278703266e-10,
-      "logits/chosen": -1.928865671157837,
-      "logits/rejected": -1.9310123920440674,
-      "logps/chosen": -34.243560791015625,
-      "logps/rejected": -30.892742156982422,
-      "loss": 0.9875,
+      "logits/chosen": -1.928911805152893,
+      "logits/rejected": -1.9310725927352905,
+      "logps/chosen": -34.24489212036133,
+      "logps/rejected": -30.92240333557129,
+      "loss": 0.6763,
       "rewards/accuracies": 0.5249999761581421,
-      "rewards/chosen": -0.003606534795835614,
-      "rewards/margins": 0.012486499734222889,
-      "rewards/rejected": -0.01609303429722786,
+      "rewards/chosen": -0.005258283112198114,
+      "rewards/margins": 0.03954260051250458,
+      "rewards/rejected": -0.044800881296396255,
       "step": 380
     },
     {
       "epoch": 1.0,
       "step": 385,
       "total_flos": 0.0,
-      "train_loss": 0.21957750196580764,
-      "train_runtime": 627.8531,
-      "train_samples_per_second": 4.904,
+      "train_loss": 0.15265155210123435,
+      "train_runtime": 627.689,
+      "train_samples_per_second": 4.905,
       "train_steps_per_second": 0.613
     }
   ],

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4cff3f16728b8d9571d3e16284fa5d4a5a3f3e9139edda68dfc9a3846a2ca29a
+oid sha256:6b30357d0b1d1e0ef1bb0e18e98e90afb60269f8515a90c0483fcf6e53040bb5
 size 4984