Model save

README.md

@@ -1,13 +1,11 @@
 ---
 library_name: peft
 tags:
-- alignment-handbook
 - trl
 - dpo
+- alignment-handbook
 - generated_from_trainer
 base_model: NbAiLab/nb-gpt-j-6B-v2
-datasets:
-- hugodk-sch/aftonposten_title_prefs
 model-index:
 - name: aftonposten-6b-align-scan
   results: []
@@ -18,17 +16,17 @@ should probably proofread and complete it, then remove this comment. -->
 
 # aftonposten-6b-align-scan
 
-This model is a fine-tuned version of [data/ap-gpt-j-6b-sft-qlora-04-08](https://huggingface.co/data/ap-gpt-j-6b-sft-qlora-04-08) on the hugodk-sch/aftonposten_title_prefs dataset.
+This model is a fine-tuned version of [NbAiLab/nb-gpt-j-6B-v2](https://huggingface.co/NbAiLab/nb-gpt-j-6B-v2) on an unknown dataset.
 It achieves the following results on the evaluation set:
-- Loss: 0.6990
-- Rewards/chosen: 0.0098
-- Rewards/rejected: 0.0133
-- Rewards/accuracies: 0.4996
-- Rewards/margins: -0.0035
-- Logps/rejected: -37.5019
-- Logps/chosen: -34.0236
-- Logits/rejected: -2.2393
-- Logits/chosen: -2.2442
+- Logits/chosen: -2.2444
+- Logits/rejected: -2.2395
+- Logps/chosen: -34.0056
+- Logps/rejected: -37.4890
+- Loss: 0.4994
+- Rewards/accuracies: 0.4988
+- Rewards/chosen: 0.0261
+- Rewards/margins: 0.0012
+- Rewards/rejected: 0.0249
 
 ## Model description
 
@@ -63,9 +61,9 @@ The following hyperparameters were used during training:
 
 | Training Loss | Epoch | Step | Logits/chosen | Logits/rejected | Logps/chosen | Logps/rejected | Validation Loss | Rewards/accuracies | Rewards/chosen | Rewards/margins | Rewards/rejected |
 |:-------------:|:-----:|:----:|:-------------:|:---------------:|:------------:|:--------------:|:---------------:|:------------------:|:--------------:|:---------------:|:----------------:|
-| 0.6951        | 0.26  | 100  | -2.2438       | -2.2389         | -34.0252     | -37.5076       | 0.6966          | 0.4954             | 0.0084         | 0.0003          | 0.0081           |
-| 0.6891        | 0.52  | 200  | -2.2432       | -2.2384         | -34.0243     | -37.5115       | 0.6947          | 0.4934             | 0.0092         | 0.0046          | 0.0046           |
-| 0.693         | 0.78  | 300  | -2.2437       | -2.2388         | -34.0172     | -37.4976       | 0.6982          | 0.4983             | 0.0156         | -0.0015         | 0.0171           |
+| 0.5014        | 0.26  | 100  | -2.2435       | -2.2386         | -34.0005     | -37.4935       | 0.4983          | 0.5307             | 0.0306         | 0.0098          | 0.0208           |
+| 0.4978        | 0.52  | 200  | -2.2443       | -2.2395         | -34.0014     | -37.4845       | 0.4998          | 0.5104             | 0.0298         | 0.0009          | 0.0289           |
+| 0.4964        | 0.78  | 300  | -2.2444       | -2.2395         | -34.0056     | -37.4890       | 0.4994          | 0.4988             | 0.0261         | 0.0012          | 0.0249           |
 
 
 ### Framework versions

adapter_config.json

@@ -19,13 +19,13 @@
   "rank_pattern": {},
   "revision": null,
   "target_modules": [
+    "gate_proj",
+    "o_proj",
     "up_proj",
-    "v_proj",
     "k_proj",
-    "q_proj",
-    "gate_proj",
     "down_proj",
-    "o_proj"
+    "v_proj",
+    "q_proj"
   ],
   "task_type": "CAUSAL_LM",
   "use_rslora": false

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b30a3614e90564741bd0b06f4b5d211ac9ab0d5d4651e82dec2b571e49a407d7
+oid sha256:07f19c50aa7f1e083760dcc58e66774fce21075bcdfe63ff7236835de16cb27e
 size 176183216

all_results.json

@@ -1,21 +1,21 @@
 {
     "epoch": 1.0,
-    "eval_logits/chosen": -2.244168758392334,
-    "eval_logits/rejected": -2.239292860031128,
-    "eval_logps/chosen": -34.02364730834961,
-    "eval_logps/rejected": -37.501853942871094,
-    "eval_loss": 0.6989675760269165,
-    "eval_rewards/accuracies": 0.49958473443984985,
-    "eval_rewards/chosen": 0.009816422127187252,
-    "eval_rewards/margins": -0.0034731123596429825,
-    "eval_rewards/rejected": 0.01328953169286251,
-    "eval_runtime": 146.1437,
+    "eval_logits/chosen": -2.2438042163848877,
+    "eval_logits/rejected": -2.23892879486084,
+    "eval_logps/chosen": -33.9998779296875,
+    "eval_logps/rejected": -37.478267669677734,
+    "eval_loss": 0.5009505748748779,
+    "eval_rewards/accuracies": 0.5074750781059265,
+    "eval_rewards/chosen": 0.031207971274852753,
+    "eval_rewards/margins": -0.003311639651656151,
+    "eval_rewards/rejected": 0.03451961278915405,
+    "eval_runtime": 145.8121,
     "eval_samples": 343,
-    "eval_samples_per_second": 2.347,
-    "eval_steps_per_second": 0.294,
-    "train_loss": 0.15265155210123435,
-    "train_runtime": 627.689,
+    "eval_samples_per_second": 2.352,
+    "eval_steps_per_second": 0.295,
+    "train_loss": 0.10993070664343896,
+    "train_runtime": 627.3649,
     "train_samples": 3079,
-    "train_samples_per_second": 4.905,
-    "train_steps_per_second": 0.613
+    "train_samples_per_second": 4.908,
+    "train_steps_per_second": 0.614
 }

eval_results.json

@@ -1,16 +1,16 @@
 {
     "epoch": 1.0,
-    "eval_logits/chosen": -2.244168758392334,
-    "eval_logits/rejected": -2.239292860031128,
-    "eval_logps/chosen": -34.02364730834961,
-    "eval_logps/rejected": -37.501853942871094,
-    "eval_loss": 0.6989675760269165,
-    "eval_rewards/accuracies": 0.49958473443984985,
-    "eval_rewards/chosen": 0.009816422127187252,
-    "eval_rewards/margins": -0.0034731123596429825,
-    "eval_rewards/rejected": 0.01328953169286251,
-    "eval_runtime": 146.1437,
+    "eval_logits/chosen": -2.2438042163848877,
+    "eval_logits/rejected": -2.23892879486084,
+    "eval_logps/chosen": -33.9998779296875,
+    "eval_logps/rejected": -37.478267669677734,
+    "eval_loss": 0.5009505748748779,
+    "eval_rewards/accuracies": 0.5074750781059265,
+    "eval_rewards/chosen": 0.031207971274852753,
+    "eval_rewards/margins": -0.003311639651656151,
+    "eval_rewards/rejected": 0.03451961278915405,
+    "eval_runtime": 145.8121,
     "eval_samples": 343,
-    "eval_samples_per_second": 2.347,
-    "eval_steps_per_second": 0.294
+    "eval_samples_per_second": 2.352,
+    "eval_steps_per_second": 0.295
 }

train_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 1.0,
-    "train_loss": 0.15265155210123435,
-    "train_runtime": 627.689,
+    "train_loss": 0.10993070664343896,
+    "train_runtime": 627.3649,
     "train_samples": 3079,
-    "train_samples_per_second": 4.905,
-    "train_steps_per_second": 0.613
+    "train_samples_per_second": 4.908,
+    "train_steps_per_second": 0.614
 }

trainer_state.json

@@ -15,7 +15,7 @@
       "logits/rejected": -1.7377450466156006,
       "logps/chosen": -29.553977966308594,
       "logps/rejected": -42.813133239746094,
-      "loss": 0.6931,
+      "loss": 0.5,
       "rewards/accuracies": 0.0,
       "rewards/chosen": 0.0,
       "rewards/margins": 0.0,
@@ -25,591 +25,591 @@
     {
       "epoch": 0.03,
       "learning_rate": 1.2820512820512818e-07,
-      "logits/chosen": -1.866059422492981,
-      "logits/rejected": -1.870389699935913,
-      "logps/chosen": -36.98250198364258,
-      "logps/rejected": -33.64270782470703,
-      "loss": 0.6804,
-      "rewards/accuracies": 0.4027777910232544,
-      "rewards/chosen": 0.021689780056476593,
-      "rewards/margins": 0.033059459179639816,
-      "rewards/rejected": -0.011369682848453522,
+      "logits/chosen": -1.866180658340454,
+      "logits/rejected": -1.8704978227615356,
+      "logps/chosen": -36.953983306884766,
+      "logps/rejected": -33.65004348754883,
+      "loss": 0.4868,
+      "rewards/accuracies": 0.5694444179534912,
+      "rewards/chosen": 0.0473538413643837,
+      "rewards/margins": 0.06532809138298035,
+      "rewards/rejected": -0.01797425001859665,
       "step": 10
     },
     {
       "epoch": 0.05,
       "learning_rate": 2.5641025641025636e-07,
-      "logits/chosen": -1.9973828792572021,
-      "logits/rejected": -2.0000317096710205,
-      "logps/chosen": -29.631702423095703,
-      "logps/rejected": -29.05877113342285,
-      "loss": 0.6937,
-      "rewards/accuracies": 0.512499988079071,
-      "rewards/chosen": 0.009445475414395332,
-      "rewards/margins": 0.005240145605057478,
-      "rewards/rejected": 0.004205327946692705,
+      "logits/chosen": -1.9972845315933228,
+      "logits/rejected": -1.999929428100586,
+      "logps/chosen": -29.655675888061523,
+      "logps/rejected": -29.05768394470215,
+      "loss": 0.5019,
+      "rewards/accuracies": 0.38749998807907104,
+      "rewards/chosen": -0.012135368771851063,
+      "rewards/margins": -0.01731942594051361,
+      "rewards/rejected": 0.005184055306017399,
       "step": 20
     },
     {
       "epoch": 0.08,
       "learning_rate": 3.8461538461538463e-07,
-      "logits/chosen": -1.9198739528656006,
-      "logits/rejected": -1.9171788692474365,
-      "logps/chosen": -31.4038028717041,
-      "logps/rejected": -33.20899963378906,
-      "loss": 0.696,
-      "rewards/accuracies": 0.5,
-      "rewards/chosen": 0.01106296293437481,
-      "rewards/margins": 0.0025471807457506657,
-      "rewards/rejected": 0.008515783585608006,
+      "logits/chosen": -1.9195616245269775,
+      "logits/rejected": -1.9168630838394165,
+      "logps/chosen": -31.41298484802246,
+      "logps/rejected": -33.21580123901367,
+      "loss": 0.5,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": 0.0027983374893665314,
+      "rewards/margins": 0.00040734148933552206,
+      "rewards/rejected": 0.0023909960873425007,
       "step": 30
     },
     {
       "epoch": 0.1,
       "learning_rate": 4.999896948438433e-07,
-      "logits/chosen": -2.0166726112365723,
-      "logits/rejected": -2.0079195499420166,
-      "logps/chosen": -32.57840347290039,
-      "logps/rejected": -32.493370056152344,
-      "loss": 0.7066,
-      "rewards/accuracies": 0.48750001192092896,
-      "rewards/chosen": -0.00143451988697052,
-      "rewards/margins": -0.018809262663125992,
-      "rewards/rejected": 0.017374742776155472,
+      "logits/chosen": -2.0171382427215576,
+      "logits/rejected": -2.0083811283111572,
+      "logps/chosen": -32.577945709228516,
+      "logps/rejected": -32.48347473144531,
+      "loss": 0.5069,
+      "rewards/accuracies": 0.42500001192092896,
+      "rewards/chosen": -0.001024238532409072,
+      "rewards/margins": -0.027298470959067345,
+      "rewards/rejected": 0.02627423405647278,
       "step": 40
     },
     {
       "epoch": 0.13,
       "learning_rate": 4.987541037542186e-07,
-      "logits/chosen": -1.8648544549942017,
-      "logits/rejected": -1.8540741205215454,
-      "logps/chosen": -33.54566955566406,
-      "logps/rejected": -35.43162155151367,
-      "loss": 0.7004,
-      "rewards/accuracies": 0.4749999940395355,
-      "rewards/chosen": 0.01062955055385828,
-      "rewards/margins": -0.00802132673561573,
-      "rewards/rejected": 0.018650878220796585,
+      "logits/chosen": -1.8646976947784424,
+      "logits/rejected": -1.8539081811904907,
+      "logps/chosen": -33.5512809753418,
+      "logps/rejected": -35.44205856323242,
+      "loss": 0.4992,
+      "rewards/accuracies": 0.44999998807907104,
+      "rewards/chosen": 0.005579059012234211,
+      "rewards/margins": -0.003679370041936636,
+      "rewards/rejected": 0.009258432313799858,
       "step": 50
     },
     {
       "epoch": 0.16,
       "learning_rate": 4.954691471941118e-07,
-      "logits/chosen": -1.9450620412826538,
-      "logits/rejected": -1.947021245956421,
-      "logps/chosen": -32.5836296081543,
-      "logps/rejected": -33.20051193237305,
-      "loss": 0.6761,
-      "rewards/accuracies": 0.5375000238418579,
-      "rewards/chosen": 0.014901289716362953,
-      "rewards/margins": 0.04112350940704346,
-      "rewards/rejected": -0.026222219690680504,
+      "logits/chosen": -1.9457124471664429,
+      "logits/rejected": -1.9476444721221924,
+      "logps/chosen": -32.58470153808594,
+      "logps/rejected": -33.17728042602539,
+      "loss": 0.4959,
+      "rewards/accuracies": 0.48750001192092896,
+      "rewards/chosen": 0.013933077454566956,
+      "rewards/margins": 0.019245292991399765,
+      "rewards/rejected": -0.005312214605510235,
       "step": 60
     },
     {
       "epoch": 0.18,
       "learning_rate": 4.901618883413548e-07,
-      "logits/chosen": -2.079589366912842,
-      "logits/rejected": -2.0845787525177,
-      "logps/chosen": -33.9754524230957,
-      "logps/rejected": -36.5793342590332,
-      "loss": 0.6876,
-      "rewards/accuracies": 0.512499988079071,
-      "rewards/chosen": 0.01173111516982317,
-      "rewards/margins": 0.021146392449736595,
-      "rewards/rejected": -0.00941527634859085,
+      "logits/chosen": -2.079798698425293,
+      "logits/rejected": -2.084780216217041,
+      "logps/chosen": -33.97211456298828,
+      "logps/rejected": -36.565345764160156,
+      "loss": 0.4982,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": 0.014736099168658257,
+      "rewards/margins": 0.011561033315956593,
+      "rewards/rejected": 0.003175063757225871,
       "step": 70
     },
     {
       "epoch": 0.21,
       "learning_rate": 4.828760511501322e-07,
-      "logits/chosen": -1.9424660205841064,
-      "logits/rejected": -1.945634126663208,
-      "logps/chosen": -34.394561767578125,
-      "logps/rejected": -34.57395553588867,
-      "loss": 0.6921,
-      "rewards/accuracies": 0.5625,
-      "rewards/chosen": 0.029921507462859154,
-      "rewards/margins": 0.013314949348568916,
-      "rewards/rejected": 0.016606558114290237,
+      "logits/chosen": -1.9421533346176147,
+      "logits/rejected": -1.9453033208847046,
+      "logps/chosen": -34.40825653076172,
+      "logps/rejected": -34.583003997802734,
+      "loss": 0.4975,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": 0.017596226185560226,
+      "rewards/margins": 0.00913666095584631,
+      "rewards/rejected": 0.008459563367068768,
       "step": 80
     },
     {
       "epoch": 0.23,
       "learning_rate": 4.736716601303429e-07,
-      "logits/chosen": -1.950564980506897,
-      "logits/rejected": -1.955082893371582,
-      "logps/chosen": -32.44951248168945,
-      "logps/rejected": -32.353668212890625,
-      "loss": 0.6909,
-      "rewards/accuracies": 0.5249999761581421,
-      "rewards/chosen": 0.0161209125071764,
-      "rewards/margins": 0.011405264027416706,
-      "rewards/rejected": 0.004715651273727417,
+      "logits/chosen": -1.9502623081207275,
+      "logits/rejected": -1.954768180847168,
+      "logps/chosen": -32.47718048095703,
+      "logps/rejected": -32.35026550292969,
+      "loss": 0.5036,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.00878011155873537,
+      "rewards/margins": -0.016559559851884842,
+      "rewards/rejected": 0.007779450621455908,
       "step": 90
     },
     {
       "epoch": 0.26,
       "learning_rate": 4.62624545834521e-07,
-      "logits/chosen": -2.0489234924316406,
-      "logits/rejected": -2.046917200088501,
-      "logps/chosen": -32.236671447753906,
-      "logps/rejected": -31.263763427734375,
-      "loss": 0.6951,
-      "rewards/accuracies": 0.5,
-      "rewards/chosen": 0.0037471160758286715,
-      "rewards/margins": 0.0015437646070495248,
-      "rewards/rejected": 0.002203352050855756,
+      "logits/chosen": -2.0489068031311035,
+      "logits/rejected": -2.046910285949707,
+      "logps/chosen": -32.23564529418945,
+      "logps/rejected": -31.257396697998047,
+      "loss": 0.5014,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": 0.004673120565712452,
+      "rewards/margins": -0.0032607216853648424,
+      "rewards/rejected": 0.00793384201824665,
       "step": 100
     },
     {
       "epoch": 0.26,
-      "eval_logits/chosen": -2.243756055831909,
-      "eval_logits/rejected": -2.238872766494751,
-      "eval_logps/chosen": -34.025177001953125,
-      "eval_logps/rejected": -37.50759506225586,
-      "eval_loss": 0.696557343006134,
-      "eval_rewards/accuracies": 0.49543190002441406,
-      "eval_rewards/chosen": 0.008434689603745937,
-      "eval_rewards/margins": 0.0003138432221021503,
-      "eval_rewards/rejected": 0.008120844140648842,
-      "eval_runtime": 146.3373,
-      "eval_samples_per_second": 2.344,
+      "eval_logits/chosen": -2.2435286045074463,
+      "eval_logits/rejected": -2.2386460304260254,
+      "eval_logps/chosen": -34.00050735473633,
+      "eval_logps/rejected": -37.493465423583984,
+      "eval_loss": 0.49828124046325684,
+      "eval_rewards/accuracies": 0.5307309031486511,
+      "eval_rewards/chosen": 0.03064088523387909,
+      "eval_rewards/margins": 0.009802807122468948,
+      "eval_rewards/rejected": 0.02083807997405529,
+      "eval_runtime": 146.1328,
+      "eval_samples_per_second": 2.347,
       "eval_steps_per_second": 0.294,
       "step": 100
     },
     {
       "epoch": 0.29,
       "learning_rate": 4.4982572012636904e-07,
-      "logits/chosen": -2.0052237510681152,
-      "logits/rejected": -2.002803325653076,
-      "logps/chosen": -33.24525833129883,
-      "logps/rejected": -34.01074981689453,
-      "loss": 0.7064,
-      "rewards/accuracies": 0.48750001192092896,
-      "rewards/chosen": -0.0013012334238737822,
-      "rewards/margins": -0.018059223890304565,
-      "rewards/rejected": 0.016757991164922714,
+      "logits/chosen": -2.0051121711730957,
+      "logits/rejected": -2.0026957988739014,
+      "logps/chosen": -33.226436614990234,
+      "logps/rejected": -34.04267120361328,
+      "loss": 0.4953,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": 0.015639375895261765,
+      "rewards/margins": 0.027611929923295975,
+      "rewards/rejected": -0.011972553096711636,
       "step": 110
     },
     {
       "epoch": 0.31,
       "learning_rate": 4.353806263777677e-07,
-      "logits/chosen": -2.016326665878296,
-      "logits/rejected": -2.007951498031616,
-      "logps/chosen": -32.478302001953125,
-      "logps/rejected": -32.186073303222656,
-      "loss": 0.7012,
-      "rewards/accuracies": 0.48750001192092896,
-      "rewards/chosen": -0.030056962743401527,
-      "rewards/margins": -0.007947373203933239,
-      "rewards/rejected": -0.022109590470790863,
+      "logits/chosen": -2.016453981399536,
+      "logits/rejected": -2.0080838203430176,
+      "logps/chosen": -32.4493522644043,
+      "logps/rejected": -32.185325622558594,
+      "loss": 0.4964,
+      "rewards/accuracies": 0.4625000059604645,
+      "rewards/chosen": -0.0040020109154284,
+      "rewards/margins": 0.01743561029434204,
+      "rewards/rejected": -0.021437620744109154,
       "step": 120
     },
     {
       "epoch": 0.34,
       "learning_rate": 4.194082707715275e-07,
-      "logits/chosen": -2.045950412750244,
-      "logits/rejected": -2.037904739379883,
-      "logps/chosen": -30.491313934326172,
-      "logps/rejected": -32.0540771484375,
-      "loss": 0.7025,
-      "rewards/accuracies": 0.4375,
-      "rewards/chosen": -0.01403898186981678,
-      "rewards/margins": -0.013075167313218117,
-      "rewards/rejected": -0.0009638145565986633,
+      "logits/chosen": -2.045841932296753,
+      "logits/rejected": -2.037797689437866,
+      "logps/chosen": -30.510135650634766,
+      "logps/rejected": -32.068138122558594,
+      "loss": 0.5028,
+      "rewards/accuracies": 0.4625000059604645,
+      "rewards/chosen": -0.030974898487329483,
+      "rewards/margins": -0.01735590770840645,
+      "rewards/rejected": -0.013618985190987587,
       "step": 130
     },
     {
       "epoch": 0.36,
       "learning_rate": 4.020402418666621e-07,
-      "logits/chosen": -1.9765291213989258,
-      "logits/rejected": -1.986802101135254,
-      "logps/chosen": -31.39646339416504,
-      "logps/rejected": -32.522178649902344,
-      "loss": 0.6923,
-      "rewards/accuracies": 0.512499988079071,
-      "rewards/chosen": 0.02087983302772045,
-      "rewards/margins": 0.00944516807794571,
-      "rewards/rejected": 0.011434664018452168,
+      "logits/chosen": -1.976995825767517,
+      "logits/rejected": -1.9872783422470093,
+      "logps/chosen": -31.390186309814453,
+      "logps/rejected": -32.526187896728516,
+      "loss": 0.4957,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.026526659727096558,
+      "rewards/margins": 0.018703728914260864,
+      "rewards/rejected": 0.007822930812835693,
       "step": 140
     },
     {
       "epoch": 0.39,
       "learning_rate": 3.8341962650351185e-07,
-      "logits/chosen": -1.8905407190322876,
-      "logits/rejected": -1.8916336297988892,
-      "logps/chosen": -34.17607116699219,
-      "logps/rejected": -34.76817321777344,
-      "loss": 0.6851,
-      "rewards/accuracies": 0.4749999940395355,
-      "rewards/chosen": 0.01495879702270031,
-      "rewards/margins": 0.024784717708826065,
-      "rewards/rejected": -0.00982591975480318,
+      "logits/chosen": -1.890682578086853,
+      "logits/rejected": -1.8917691707611084,
+      "logps/chosen": -34.19375991821289,
+      "logps/rejected": -34.75046920776367,
+      "loss": 0.5014,
+      "rewards/accuracies": 0.4124999940395355,
+      "rewards/chosen": -0.0009607344982214272,
+      "rewards/margins": -0.0070659080520272255,
+      "rewards/rejected": 0.0061051733791828156,
       "step": 150
     },
     {
       "epoch": 0.42,
       "learning_rate": 3.636998309800572e-07,
-      "logits/chosen": -1.942565679550171,
-      "logits/rejected": -1.9390919208526611,
-      "logps/chosen": -36.1614875793457,
-      "logps/rejected": -32.72848129272461,
-      "loss": 0.6907,
-      "rewards/accuracies": 0.4625000059604645,
-      "rewards/chosen": 0.00917207170277834,
-      "rewards/margins": 0.01433003693819046,
-      "rewards/rejected": -0.0051579661667346954,
+      "logits/chosen": -1.9428867101669312,
+      "logits/rejected": -1.939404845237732,
+      "logps/chosen": -36.15603256225586,
+      "logps/rejected": -32.732059478759766,
+      "loss": 0.4954,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": 0.014080708846449852,
+      "rewards/margins": 0.022458387538790703,
+      "rewards/rejected": -0.008377680554986,
       "step": 160
     },
     {
       "epoch": 0.44,
       "learning_rate": 3.430433172111807e-07,
-      "logits/chosen": -2.0418710708618164,
-      "logits/rejected": -2.034475564956665,
-      "logps/chosen": -33.76544952392578,
-      "logps/rejected": -31.358556747436523,
-      "loss": 0.6791,
-      "rewards/accuracies": 0.574999988079071,
-      "rewards/chosen": 0.02917439118027687,
-      "rewards/margins": 0.03521214798092842,
-      "rewards/rejected": -0.0060377540066838264,
+      "logits/chosen": -2.0421247482299805,
+      "logits/rejected": -2.034731388092041,
+      "logps/chosen": -33.75699234008789,
+      "logps/rejected": -31.35996437072754,
+      "loss": 0.4896,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": 0.03678463399410248,
+      "rewards/margins": 0.04408547282218933,
+      "rewards/rejected": -0.007300837431102991,
       "step": 170
     },
     {
       "epoch": 0.47,
       "learning_rate": 3.216202642830543e-07,
-      "logits/chosen": -2.0476231575012207,
-      "logits/rejected": -2.0529019832611084,
-      "logps/chosen": -32.533023834228516,
-      "logps/rejected": -32.506805419921875,
-      "loss": 0.6814,
-      "rewards/accuracies": 0.5625,
-      "rewards/chosen": 0.010587882250547409,
-      "rewards/margins": 0.03008626028895378,
-      "rewards/rejected": -0.019498378038406372,
+      "logits/chosen": -2.0474019050598145,
+      "logits/rejected": -2.052670955657959,
+      "logps/chosen": -32.514625549316406,
+      "logps/rejected": -32.481895446777344,
+      "loss": 0.4933,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.0271458737552166,
+      "rewards/margins": 0.024227729067206383,
+      "rewards/rejected": 0.002918146550655365,
       "step": 180
     },
     {
       "epoch": 0.49,
       "learning_rate": 2.9960716642946403e-07,
-      "logits/chosen": -2.048576593399048,
-      "logits/rejected": -2.0457961559295654,
-      "logps/chosen": -31.470510482788086,
-      "logps/rejected": -31.322284698486328,
-      "loss": 0.687,
-      "rewards/accuracies": 0.5625,
-      "rewards/chosen": 0.016599375754594803,
-      "rewards/margins": 0.020261235535144806,
-      "rewards/rejected": -0.0036618602462112904,
+      "logits/chosen": -2.048748016357422,
+      "logits/rejected": -2.045966625213623,
+      "logps/chosen": -31.47454833984375,
+      "logps/rejected": -31.311954498291016,
+      "loss": 0.498,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": 0.012961958535015583,
+      "rewards/margins": 0.007329396903514862,
+      "rewards/rejected": 0.005632560700178146,
       "step": 190
     },
     {
       "epoch": 0.52,
       "learning_rate": 2.771853789806683e-07,
-      "logits/chosen": -1.9184268712997437,
-      "logits/rejected": -1.9230947494506836,
-      "logps/chosen": -31.57404136657715,
-      "logps/rejected": -32.775550842285156,
-      "loss": 0.6891,
-      "rewards/accuracies": 0.48750001192092896,
-      "rewards/chosen": 0.02901501953601837,
-      "rewards/margins": 0.015173261985182762,
-      "rewards/rejected": 0.013841753825545311,
+      "logits/chosen": -1.9195592403411865,
+      "logits/rejected": -1.9242277145385742,
+      "logps/chosen": -31.59905433654785,
+      "logps/rejected": -32.79246139526367,
+      "loss": 0.4978,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.00650545209646225,
+      "rewards/margins": 0.007882391102612019,
+      "rewards/rejected": -0.0013769377255812287,
       "step": 200
     },
     {
       "epoch": 0.52,
-      "eval_logits/chosen": -2.243248462677002,
-      "eval_logits/rejected": -2.238374948501587,
-      "eval_logps/chosen": -34.02431869506836,
-      "eval_logps/rejected": -37.511497497558594,
-      "eval_loss": 0.6946919560432434,
-      "eval_rewards/accuracies": 0.49335551261901855,
-      "eval_rewards/chosen": 0.009210066869854927,
-      "eval_rewards/margins": 0.0045979218557477,
-      "eval_rewards/rejected": 0.004612144082784653,
-      "eval_runtime": 146.0514,
-      "eval_samples_per_second": 2.348,
-      "eval_steps_per_second": 0.294,
+      "eval_logits/chosen": -2.244333267211914,
+      "eval_logits/rejected": -2.239457845687866,
+      "eval_logps/chosen": -34.00139236450195,
+      "eval_logps/rejected": -37.48446273803711,
+      "eval_loss": 0.4998188018798828,
+      "eval_rewards/accuracies": 0.5103820562362671,
+      "eval_rewards/chosen": 0.02984398789703846,
+      "eval_rewards/margins": 0.000900517450645566,
+      "eval_rewards/rejected": 0.028943469747900963,
+      "eval_runtime": 145.8897,
+      "eval_samples_per_second": 2.351,
+      "eval_steps_per_second": 0.295,
       "step": 200
     },
     {
       "epoch": 0.55,
       "learning_rate": 2.5453962426402e-07,
-      "logits/chosen": -2.0314748287200928,
-      "logits/rejected": -2.0421440601348877,
-      "logps/chosen": -31.954639434814453,
-      "logps/rejected": -33.87318801879883,
-      "loss": 0.6864,
-      "rewards/accuracies": 0.550000011920929,
-      "rewards/chosen": 0.014999927952885628,
-      "rewards/margins": 0.024442464113235474,
-      "rewards/rejected": -0.009442536160349846,
+      "logits/chosen": -2.0324440002441406,
+      "logits/rejected": -2.043117046356201,
+      "logps/chosen": -31.93975830078125,
+      "logps/rejected": -33.84552001953125,
+      "loss": 0.4961,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": 0.028390545397996902,
+      "rewards/margins": 0.012928962707519531,
+      "rewards/rejected": 0.015461576171219349,
       "step": 210
     },
     {
       "epoch": 0.57,
       "learning_rate": 2.318564697655179e-07,
-      "logits/chosen": -1.9249544143676758,
-      "logits/rejected": -1.9398053884506226,
-      "logps/chosen": -30.072036743164062,
-      "logps/rejected": -31.55165672302246,
-      "loss": 0.6827,
-      "rewards/accuracies": 0.6000000238418579,
-      "rewards/chosen": 0.035550959408283234,
-      "rewards/margins": 0.029681822285056114,
-      "rewards/rejected": 0.005869132932275534,
+      "logits/chosen": -1.9257566928863525,
+      "logits/rejected": -1.9405906200408936,
+      "logps/chosen": -30.077129364013672,
+      "logps/rejected": -31.565658569335938,
+      "loss": 0.493,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": 0.030965950340032578,
+      "rewards/margins": 0.037700776010751724,
+      "rewards/rejected": -0.00673482334241271,
       "step": 220
     },
     {
       "epoch": 0.6,
       "learning_rate": 2.093227910899832e-07,
-      "logits/chosen": -1.9830297231674194,
-      "logits/rejected": -1.9869968891143799,
-      "logps/chosen": -33.3900146484375,
-      "logps/rejected": -31.545156478881836,
-      "loss": 0.6872,
-      "rewards/accuracies": 0.5375000238418579,
-      "rewards/chosen": 0.024708259850740433,
-      "rewards/margins": 0.02097567543387413,
-      "rewards/rejected": 0.00373258744366467,
+      "logits/chosen": -1.9837474822998047,
+      "logits/rejected": -1.9877216815948486,
+      "logps/chosen": -33.37251281738281,
+      "logps/rejected": -31.5429744720459,
+      "loss": 0.4927,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.04046114161610603,
+      "rewards/margins": 0.034767456352710724,
+      "rewards/rejected": 0.00569369038566947,
       "step": 230
     },
     {
       "epoch": 0.62,
       "learning_rate": 1.8712423238279356e-07,
-      "logits/chosen": -1.9825174808502197,
-      "logits/rejected": -1.9605509042739868,
-      "logps/chosen": -34.16549301147461,
-      "logps/rejected": -34.945499420166016,
-      "loss": 0.7065,
-      "rewards/accuracies": 0.5,
-      "rewards/chosen": -0.010572168976068497,
-      "rewards/margins": -0.015337007120251656,
-      "rewards/rejected": 0.004764837212860584,
+      "logits/chosen": -1.9833183288574219,
+      "logits/rejected": -1.9613754749298096,
+      "logps/chosen": -34.14833068847656,
+      "logps/rejected": -34.94057846069336,
+      "loss": 0.5009,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": 0.004874309059232473,
+      "rewards/margins": -0.004319626372307539,
+      "rewards/rejected": 0.009193938225507736,
       "step": 240
     },
     {
       "epoch": 0.65,
       "learning_rate": 1.654436768970182e-07,
-      "logits/chosen": -2.0242011547088623,
-      "logits/rejected": -2.02089262008667,
-      "logps/chosen": -32.91257858276367,
-      "logps/rejected": -36.22357177734375,
-      "loss": 0.6913,
-      "rewards/accuracies": 0.5,
-      "rewards/chosen": 0.01124012004584074,
-      "rewards/margins": 0.010020162910223007,
-      "rewards/rejected": 0.0012199539924040437,
+      "logits/chosen": -2.025144338607788,
+      "logits/rejected": -2.0218405723571777,
+      "logps/chosen": -32.892818450927734,
+      "logps/rejected": -36.1950798034668,
+      "loss": 0.5004,
+      "rewards/accuracies": 0.48750001192092896,
+      "rewards/chosen": 0.029024243354797363,
+      "rewards/margins": 0.002163961064070463,
+      "rewards/rejected": 0.026860278099775314,
       "step": 250
     },
     {
       "epoch": 0.68,
       "learning_rate": 1.444597403062196e-07,
-      "logits/chosen": -1.8910300731658936,
-      "logits/rejected": -1.888593316078186,
-      "logps/chosen": -34.191619873046875,
-      "logps/rejected": -35.50851058959961,
-      "loss": 0.6975,
-      "rewards/accuracies": 0.6000000238418579,
-      "rewards/chosen": -0.0028982784133404493,
-      "rewards/margins": 0.0006028197822161019,
-      "rewards/rejected": -0.003501094877719879,
+      "logits/chosen": -1.8918704986572266,
+      "logits/rejected": -1.8894078731536865,
+      "logps/chosen": -34.1577262878418,
+      "logps/rejected": -35.497093200683594,
+      "loss": 0.4956,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": 0.02760697342455387,
+      "rewards/margins": 0.02083553373813629,
+      "rewards/rejected": 0.006771440617740154,
       "step": 260
     },
     {
       "epoch": 0.7,
       "learning_rate": 1.2434529917578887e-07,
-      "logits/chosen": -1.8757693767547607,
-      "logits/rejected": -1.8732143640518188,
-      "logps/chosen": -34.38648223876953,
-      "logps/rejected": -31.753692626953125,
-      "loss": 0.6995,
-      "rewards/accuracies": 0.512499988079071,
-      "rewards/chosen": 0.010007266886532307,
-      "rewards/margins": -0.005866709630936384,
-      "rewards/rejected": 0.015873977914452553,
+      "logits/chosen": -1.8762916326522827,
+      "logits/rejected": -1.8737586736679077,
+      "logps/chosen": -34.374244689941406,
+      "logps/rejected": -31.73373031616211,
+      "loss": 0.5038,
+      "rewards/accuracies": 0.4625000059604645,
+      "rewards/chosen": 0.02102069742977619,
+      "rewards/margins": -0.012818600051105022,
+      "rewards/rejected": 0.03383929282426834,
       "step": 270
     },
     {
       "epoch": 0.73,
       "learning_rate": 1.0526606671603521e-07,
-      "logits/chosen": -1.9795001745224,
-      "logits/rejected": -1.9688692092895508,
-      "logps/chosen": -35.32746887207031,
-      "logps/rejected": -31.84292221069336,
-      "loss": 0.6843,
-      "rewards/accuracies": 0.550000011920929,
-      "rewards/chosen": 0.026254409924149513,
-      "rewards/margins": 0.027990642935037613,
-      "rewards/rejected": -0.001736226724460721,
+      "logits/chosen": -1.9803699254989624,
+      "logits/rejected": -1.9697377681732178,
+      "logps/chosen": -35.30424880981445,
+      "logps/rejected": -31.83871841430664,
+      "loss": 0.4898,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": 0.047153402119874954,
+      "rewards/margins": 0.04510683938860893,
+      "rewards/rejected": 0.0020465687848627567,
       "step": 280
     },
     {
       "epoch": 0.75,
       "learning_rate": 8.737922755071453e-08,
-      "logits/chosen": -2.0753533840179443,
-      "logits/rejected": -2.0603318214416504,
-      "logps/chosen": -30.902263641357422,
-      "logps/rejected": -32.65242385864258,
-      "loss": 0.6872,
+      "logits/chosen": -2.0763792991638184,
+      "logits/rejected": -2.061370372772217,
+      "logps/chosen": -30.90180015563965,
+      "logps/rejected": -32.65237045288086,
+      "loss": 0.4962,
       "rewards/accuracies": 0.512499988079071,
-      "rewards/chosen": 0.019674303010106087,
-      "rewards/margins": 0.020201902836561203,
-      "rewards/rejected": -0.0005276011070236564,
+      "rewards/chosen": 0.020093059167265892,
+      "rewards/margins": 0.020573493093252182,
+      "rewards/rejected": -0.00048043689457699656,
       "step": 290
     },
     {
       "epoch": 0.78,
       "learning_rate": 7.08321427484816e-08,
-      "logits/chosen": -1.946671724319458,
-      "logits/rejected": -1.9441486597061157,
-      "logps/chosen": -32.906158447265625,
-      "logps/rejected": -30.805999755859375,
-      "loss": 0.693,
-      "rewards/accuracies": 0.4749999940395355,
-      "rewards/chosen": 0.018858108669519424,
-      "rewards/margins": 0.009128611534833908,
-      "rewards/rejected": 0.009729497134685516,
+      "logits/chosen": -1.9472248554229736,
+      "logits/rejected": -1.9446901082992554,
+      "logps/chosen": -32.882179260253906,
+      "logps/rejected": -30.789831161499023,
+      "loss": 0.4964,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": 0.04043656960129738,
+      "rewards/margins": 0.016158219426870346,
+      "rewards/rejected": 0.024278344586491585,
       "step": 300
     },
     {
       "epoch": 0.78,
-      "eval_logits/chosen": -2.243694305419922,
-      "eval_logits/rejected": -2.2388274669647217,
-      "eval_logps/chosen": -34.017173767089844,
-      "eval_logps/rejected": -37.497615814208984,
-      "eval_loss": 0.6981561183929443,
-      "eval_rewards/accuracies": 0.49833887815475464,
-      "eval_rewards/chosen": 0.015641551464796066,
-      "eval_rewards/margins": -0.0014640000881627202,
-      "eval_rewards/rejected": 0.017105549573898315,
-      "eval_runtime": 146.0253,
-      "eval_samples_per_second": 2.349,
-      "eval_steps_per_second": 0.294,
+      "eval_logits/chosen": -2.244403123855591,
+      "eval_logits/rejected": -2.239530563354492,
+      "eval_logps/chosen": -34.00559997558594,
+      "eval_logps/rejected": -37.48896408081055,
+      "eval_loss": 0.49940842390060425,
+      "eval_rewards/accuracies": 0.4987541437149048,
+      "eval_rewards/chosen": 0.026060525327920914,
+      "eval_rewards/margins": 0.001171064912341535,
+      "eval_rewards/rejected": 0.02488945797085762,
+      "eval_runtime": 145.9672,
+      "eval_samples_per_second": 2.35,
+      "eval_steps_per_second": 0.295,
       "step": 300
     },
     {
       "epoch": 0.81,
       "learning_rate": 5.576113578589034e-08,
-      "logits/chosen": -1.9287757873535156,
-      "logits/rejected": -1.9255115985870361,
-      "logps/chosen": -31.573471069335938,
-      "logps/rejected": -33.71125030517578,
-      "loss": 0.698,
-      "rewards/accuracies": 0.5375000238418579,
-      "rewards/chosen": 0.026891669258475304,
-      "rewards/margins": -0.000918733305297792,
-      "rewards/rejected": 0.02781040407717228,
+      "logits/chosen": -1.9294729232788086,
+      "logits/rejected": -1.9262111186981201,
+      "logps/chosen": -31.556472778320312,
+      "logps/rejected": -33.73994827270508,
+      "loss": 0.4915,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.04218826815485954,
+      "rewards/margins": 0.04020850360393524,
+      "rewards/rejected": 0.001979765249416232,
       "step": 310
     },
     {
       "epoch": 0.83,
       "learning_rate": 4.229036944380912e-08,
-      "logits/chosen": -1.9807313680648804,
-      "logits/rejected": -1.968425989151001,
-      "logps/chosen": -34.54930877685547,
-      "logps/rejected": -33.57306671142578,
-      "loss": 0.6615,
-      "rewards/accuracies": 0.637499988079071,
-      "rewards/chosen": 0.04208679869771004,
-      "rewards/margins": 0.07289497554302216,
-      "rewards/rejected": -0.030808180570602417,
+      "logits/chosen": -1.9809057712554932,
+      "logits/rejected": -1.968597412109375,
+      "logps/chosen": -34.546607971191406,
+      "logps/rejected": -33.53887939453125,
+      "loss": 0.4917,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": 0.044516824185848236,
+      "rewards/margins": 0.04455769807100296,
+      "rewards/rejected": -4.087165143573657e-05,
       "step": 320
     },
     {
       "epoch": 0.86,
       "learning_rate": 3.053082288996112e-08,
-      "logits/chosen": -2.0161099433898926,
-      "logits/rejected": -2.0146496295928955,
-      "logps/chosen": -33.46127700805664,
-      "logps/rejected": -32.4744873046875,
-      "loss": 0.6883,
-      "rewards/accuracies": 0.5375000238418579,
-      "rewards/chosen": 0.022065162658691406,
-      "rewards/margins": 0.01747960038483143,
-      "rewards/rejected": 0.004585559479892254,
+      "logits/chosen": -2.0169310569763184,
+      "logits/rejected": -2.0154881477355957,
+      "logps/chosen": -33.446449279785156,
+      "logps/rejected": -32.45838165283203,
+      "loss": 0.4965,
+      "rewards/accuracies": 0.48750001192092896,
+      "rewards/chosen": 0.035413958132267,
+      "rewards/margins": 0.016332540661096573,
+      "rewards/rejected": 0.019081417471170425,
       "step": 330
     },
     {
       "epoch": 0.88,
       "learning_rate": 2.05793773749158e-08,
-      "logits/chosen": -2.102802276611328,
-      "logits/rejected": -2.0869994163513184,
-      "logps/chosen": -34.16718292236328,
-      "logps/rejected": -33.08827590942383,
-      "loss": 0.706,
-      "rewards/accuracies": 0.48750001192092896,
-      "rewards/chosen": 0.010987209156155586,
-      "rewards/margins": -0.01579815149307251,
-      "rewards/rejected": 0.026785362511873245,
+      "logits/chosen": -2.1038289070129395,
+      "logits/rejected": -2.0880303382873535,
+      "logps/chosen": -34.16547393798828,
+      "logps/rejected": -33.04523468017578,
+      "loss": 0.5123,
+      "rewards/accuracies": 0.36250001192092896,
+      "rewards/chosen": 0.012526323087513447,
+      "rewards/margins": -0.05299854278564453,
+      "rewards/rejected": 0.0655248612165451,
       "step": 340
     },
     {
       "epoch": 0.91,
       "learning_rate": 1.251801807404168e-08,
-      "logits/chosen": -1.97471022605896,
-      "logits/rejected": -1.9737637042999268,
-      "logps/chosen": -33.25645065307617,
-      "logps/rejected": -32.46539306640625,
-      "loss": 0.6861,
-      "rewards/accuracies": 0.512499988079071,
-      "rewards/chosen": 0.023080622777342796,
-      "rewards/margins": 0.023302335292100906,
-      "rewards/rejected": -0.00022171511955093592,
+      "logits/chosen": -1.9754581451416016,
+      "logits/rejected": -1.9744971990585327,
+      "logps/chosen": -33.235225677490234,
+      "logps/rejected": -32.451236724853516,
+      "loss": 0.493,
+      "rewards/accuracies": 0.48750001192092896,
+      "rewards/chosen": 0.04218399524688721,
+      "rewards/margins": 0.029664650559425354,
+      "rewards/rejected": 0.0125193502753973,
       "step": 350
     },
     {
       "epoch": 0.94,
       "learning_rate": 6.41315865106129e-09,
-      "logits/chosen": -1.9304895401000977,
-      "logits/rejected": -1.9408460855484009,
-      "logps/chosen": -32.22315979003906,
-      "logps/rejected": -35.28199005126953,
-      "loss": 0.7085,
-      "rewards/accuracies": 0.42500001192092896,
-      "rewards/chosen": -0.01662321388721466,
-      "rewards/margins": -0.022716889157891273,
-      "rewards/rejected": 0.006093672942370176,
+      "logits/chosen": -1.9313201904296875,
+      "logits/rejected": -1.9416812658309937,
+      "logps/chosen": -32.19329071044922,
+      "logps/rejected": -35.28053283691406,
+      "loss": 0.5008,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": 0.010262362658977509,
+      "rewards/margins": 0.0028595193289220333,
+      "rewards/rejected": 0.007402840070426464,
       "step": 360
     },
     {
       "epoch": 0.96,
       "learning_rate": 2.3150941078050324e-09,
-      "logits/chosen": -2.0692858695983887,
-      "logits/rejected": -2.0627408027648926,
-      "logps/chosen": -33.64545440673828,
-      "logps/rejected": -29.203441619873047,
-      "loss": 0.704,
-      "rewards/accuracies": 0.42500001192092896,
-      "rewards/chosen": 0.00586346909403801,
-      "rewards/margins": -0.014821496792137623,
-      "rewards/rejected": 0.020684964954853058,
+      "logits/chosen": -2.070078134536743,
+      "logits/rejected": -2.063514232635498,
+      "logps/chosen": -33.65337371826172,
+      "logps/rejected": -29.21976661682129,
+      "loss": 0.5017,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -0.0012650005519390106,
+      "rewards/margins": -0.007256894372403622,
+      "rewards/rejected": 0.0059918928891420364,
       "step": 370
     },
     {
       "epoch": 0.99,
       "learning_rate": 2.575864278703266e-10,
-      "logits/chosen": -1.928911805152893,
-      "logits/rejected": -1.9310725927352905,
-      "logps/chosen": -34.24489212036133,
-      "logps/rejected": -30.92240333557129,
-      "loss": 0.6763,
-      "rewards/accuracies": 0.5249999761581421,
-      "rewards/chosen": -0.005258283112198114,
-      "rewards/margins": 0.03954260051250458,
-      "rewards/rejected": -0.044800881296396255,
+      "logits/chosen": -1.9295686483383179,
+      "logits/rejected": -1.9317378997802734,
+      "logps/chosen": -34.22509002685547,
+      "logps/rejected": -30.884735107421875,
+      "loss": 0.4954,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": 0.012562277726829052,
+      "rewards/margins": 0.02345881052315235,
+      "rewards/rejected": -0.01089653093367815,
       "step": 380
     },
     {
       "epoch": 1.0,
       "step": 385,
       "total_flos": 0.0,
-      "train_loss": 0.15265155210123435,
-      "train_runtime": 627.689,
-      "train_samples_per_second": 4.905,
-      "train_steps_per_second": 0.613
+      "train_loss": 0.10993070664343896,
+      "train_runtime": 627.3649,
+      "train_samples_per_second": 4.908,
+      "train_steps_per_second": 0.614
     }
   ],
   "logging_steps": 10,

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6b30357d0b1d1e0ef1bb0e18e98e90afb60269f8515a90c0483fcf6e53040bb5
+oid sha256:521df8373b7294b087c904d0266936c5550b5766587020eb20889178b14ee863
 size 4984