Model save

README.md

@@ -1,13 +1,11 @@
 ---
 library_name: peft
 tags:
-- alignment-handbook
 - trl
 - dpo
+- alignment-handbook
 - generated_from_trainer
 base_model: NbAiLab/nb-gpt-j-6B-v2
-datasets:
-- hugodk-sch/aftonposten_title_prefs
 model-index:
 - name: aftonposten-6b-align-scan
   results: []
@@ -18,17 +16,17 @@ should probably proofread and complete it, then remove this comment. -->
 
 # aftonposten-6b-align-scan
 
-This model is a fine-tuned version of [data/ap-gpt-j-6b-sft-qlora-04-08](https://huggingface.co/data/ap-gpt-j-6b-sft-qlora-04-08) on the hugodk-sch/aftonposten_title_prefs dataset.
+This model is a fine-tuned version of [NbAiLab/nb-gpt-j-6B-v2](https://huggingface.co/NbAiLab/nb-gpt-j-6B-v2) on an unknown dataset.
 It achieves the following results on the evaluation set:
-- Loss: 0.3447
-- Rewards/chosen: 0.0141
-- Rewards/rejected: 0.0084
-- Rewards/accuracies: 0.5162
-- Rewards/margins: 0.0057
-- Logps/rejected: -37.5073
-- Logps/chosen: -34.0189
-- Logits/rejected: -2.2388
 - Logits/chosen: -2.2437
+- Logits/rejected: -2.2388
+- Logps/chosen: -34.0306
+- Logps/rejected: -37.4994
+- Loss: 1.0120
+- Rewards/accuracies: 0.4726
+- Rewards/chosen: 0.0035
+- Rewards/margins: -0.0120
+- Rewards/rejected: 0.0155
 
 ## Model description
 
@@ -63,9 +61,9 @@ The following hyperparameters were used during training:
 
 | Training Loss | Epoch | Step | Logits/chosen | Logits/rejected | Logps/chosen | Logps/rejected | Validation Loss | Rewards/accuracies | Rewards/chosen | Rewards/margins | Rewards/rejected |
 |:-------------:|:-----:|:----:|:-------------:|:---------------:|:------------:|:--------------:|:---------------:|:------------------:|:--------------:|:---------------:|:----------------:|
-| 0.333         | 0.26  | 100  | -2.2438       | -2.2389         | -34.0205     | -37.5035       | 0.3405          | 0.5104             | 0.0126         | 0.0008          | 0.0118           |
-| 0.3306        | 0.52  | 200  | -2.2435       | -2.2386         | -34.0285     | -37.5113       | 0.3466          | 0.5021             | 0.0054         | 0.0007          | 0.0048           |
-| 0.3327        | 0.78  | 300  | -2.2439       | -2.2390         | -34.0176     | -37.5142       | 0.3276          | 0.5365             | 0.0152         | 0.0131          | 0.0022           |
+| 0.9823        | 0.26  | 100  | -2.2440       | -2.2391         | -34.0216     | -37.5058       | 0.9982          | 0.5104             | 0.0116         | 0.0019          | 0.0097           |
+| 0.9696        | 0.52  | 200  | -2.2439       | -2.2390         | -34.0217     | -37.5024       | 1.0014          | 0.4693             | 0.0116         | -0.0012         | 0.0128           |
+| 0.9581        | 0.78  | 300  | -2.2437       | -2.2388         | -34.0306     | -37.4994       | 1.0120          | 0.4726             | 0.0035         | -0.0120         | 0.0155           |
 
 
 ### Framework versions

adapter_config.json

@@ -19,13 +19,13 @@
   "rank_pattern": {},
   "revision": null,
   "target_modules": [
+    "k_proj",
     "o_proj",
+    "v_proj",
     "gate_proj",
     "q_proj",
-    "down_proj",
-    "v_proj",
-    "k_proj",
-    "up_proj"
+    "up_proj",
+    "down_proj"
   ],
   "task_type": "CAUSAL_LM",
   "use_rslora": false

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:28810f10adaabc08f14ec259ce0b4eff31f65380fdc92f6051a2568fe462f659
+oid sha256:a1e7f79a4fe0164d309e418b64bfbaabd64069413317820121adc83c26c731cb
 size 176183216

all_results.json

@@ -1,21 +1,21 @@
 {
     "epoch": 1.0,
-    "eval_logits/chosen": -2.2436861991882324,
-    "eval_logits/rejected": -2.2388129234313965,
-    "eval_logps/chosen": -34.01886749267578,
-    "eval_logps/rejected": -37.50727081298828,
-    "eval_loss": 0.34474730491638184,
-    "eval_rewards/accuracies": 0.5161960124969482,
-    "eval_rewards/chosen": 0.01411630492657423,
-    "eval_rewards/margins": 0.005696756765246391,
-    "eval_rewards/rejected": 0.008419547230005264,
-    "eval_runtime": 146.1664,
+    "eval_logits/chosen": -2.2439348697662354,
+    "eval_logits/rejected": -2.2390594482421875,
+    "eval_logps/chosen": -34.016693115234375,
+    "eval_logps/rejected": -37.499900817871094,
+    "eval_loss": 0.9991074204444885,
+    "eval_rewards/accuracies": 0.5365448594093323,
+    "eval_rewards/chosen": 0.01607290655374527,
+    "eval_rewards/margins": 0.0010251295752823353,
+    "eval_rewards/rejected": 0.015047776512801647,
+    "eval_runtime": 145.7535,
     "eval_samples": 343,
-    "eval_samples_per_second": 2.347,
-    "eval_steps_per_second": 0.294,
-    "train_loss": 0.0744757689438857,
-    "train_runtime": 628.0651,
+    "eval_samples_per_second": 2.353,
+    "eval_steps_per_second": 0.295,
+    "train_loss": 0.21873634759481853,
+    "train_runtime": 627.4596,
     "train_samples": 3079,
-    "train_samples_per_second": 4.902,
-    "train_steps_per_second": 0.613
+    "train_samples_per_second": 4.907,
+    "train_steps_per_second": 0.614
 }

eval_results.json

@@ -1,16 +1,16 @@
 {
     "epoch": 1.0,
-    "eval_logits/chosen": -2.2436861991882324,
-    "eval_logits/rejected": -2.2388129234313965,
-    "eval_logps/chosen": -34.01886749267578,
-    "eval_logps/rejected": -37.50727081298828,
-    "eval_loss": 0.34474730491638184,
-    "eval_rewards/accuracies": 0.5161960124969482,
-    "eval_rewards/chosen": 0.01411630492657423,
-    "eval_rewards/margins": 0.005696756765246391,
-    "eval_rewards/rejected": 0.008419547230005264,
-    "eval_runtime": 146.1664,
+    "eval_logits/chosen": -2.2439348697662354,
+    "eval_logits/rejected": -2.2390594482421875,
+    "eval_logps/chosen": -34.016693115234375,
+    "eval_logps/rejected": -37.499900817871094,
+    "eval_loss": 0.9991074204444885,
+    "eval_rewards/accuracies": 0.5365448594093323,
+    "eval_rewards/chosen": 0.01607290655374527,
+    "eval_rewards/margins": 0.0010251295752823353,
+    "eval_rewards/rejected": 0.015047776512801647,
+    "eval_runtime": 145.7535,
     "eval_samples": 343,
-    "eval_samples_per_second": 2.347,
-    "eval_steps_per_second": 0.294
+    "eval_samples_per_second": 2.353,
+    "eval_steps_per_second": 0.295
 }

train_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 1.0,
-    "train_loss": 0.0744757689438857,
-    "train_runtime": 628.0651,
+    "train_loss": 0.21873634759481853,
+    "train_runtime": 627.4596,
     "train_samples": 3079,
-    "train_samples_per_second": 4.902,
-    "train_steps_per_second": 0.613
+    "train_samples_per_second": 4.907,
+    "train_steps_per_second": 0.614
 }

trainer_state.json

@@ -15,7 +15,7 @@
       "logits/rejected": -1.7377450466156006,
       "logps/chosen": -29.553977966308594,
       "logps/rejected": -42.813133239746094,
-      "loss": 0.3086,
+      "loss": 1.0,
       "rewards/accuracies": 0.0,
       "rewards/chosen": 0.0,
       "rewards/margins": 0.0,
@@ -25,311 +25,311 @@
     {
       "epoch": 0.03,
       "learning_rate": 1.2820512820512818e-07,
-      "logits/chosen": -1.866453766822815,
-      "logits/rejected": -1.8707627058029175,
-      "logps/chosen": -36.98646545410156,
-      "logps/rejected": -33.6445426940918,
-      "loss": 0.3098,
-      "rewards/accuracies": 0.4444444477558136,
-      "rewards/chosen": 0.018122652545571327,
-      "rewards/margins": 0.031144678592681885,
-      "rewards/rejected": -0.013022023253142834,
+      "logits/chosen": -1.8661272525787354,
+      "logits/rejected": -1.8704373836517334,
+      "logps/chosen": -36.993247985839844,
+      "logps/rejected": -33.67000961303711,
+      "loss": 0.952,
+      "rewards/accuracies": 0.5138888955116272,
+      "rewards/chosen": 0.012012440711259842,
+      "rewards/margins": 0.047955650836229324,
+      "rewards/rejected": -0.035943206399679184,
       "step": 10
     },
     {
       "epoch": 0.05,
       "learning_rate": 2.5641025641025636e-07,
-      "logits/chosen": -1.9970738887786865,
-      "logits/rejected": -1.9997297525405884,
-      "logps/chosen": -29.651851654052734,
-      "logps/rejected": -29.048221588134766,
-      "loss": 0.3674,
-      "rewards/accuracies": 0.4375,
-      "rewards/chosen": -0.008690872229635715,
-      "rewards/margins": -0.02239001728594303,
-      "rewards/rejected": 0.01369914598762989,
+      "logits/chosen": -1.9975776672363281,
+      "logits/rejected": -2.000213146209717,
+      "logps/chosen": -29.647293090820312,
+      "logps/rejected": -29.056921005249023,
+      "loss": 1.0105,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -0.00459072832018137,
+      "rewards/margins": -0.010462107136845589,
+      "rewards/rejected": 0.005871377885341644,
       "step": 20
     },
     {
       "epoch": 0.08,
       "learning_rate": 3.8461538461538463e-07,
-      "logits/chosen": -1.919904351234436,
-      "logits/rejected": -1.9172093868255615,
-      "logps/chosen": -31.407363891601562,
-      "logps/rejected": -33.22914505004883,
-      "loss": 0.3289,
-      "rewards/accuracies": 0.5874999761581421,
-      "rewards/chosen": 0.007855755276978016,
-      "rewards/margins": 0.017470674589276314,
-      "rewards/rejected": -0.009614917449653149,
+      "logits/chosen": -1.920013189315796,
+      "logits/rejected": -1.917337417602539,
+      "logps/chosen": -31.4055233001709,
+      "logps/rejected": -33.22685623168945,
+      "loss": 0.9829,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": 0.009516814723610878,
+      "rewards/margins": 0.01707320474088192,
+      "rewards/rejected": -0.007556394673883915,
       "step": 30
     },
     {
       "epoch": 0.1,
       "learning_rate": 4.999896948438433e-07,
-      "logits/chosen": -2.016488552093506,
-      "logits/rejected": -2.0077414512634277,
-      "logps/chosen": -32.588050842285156,
-      "logps/rejected": -32.51618576049805,
-      "loss": 0.3526,
-      "rewards/accuracies": 0.4749999940395355,
-      "rewards/chosen": -0.010120267979800701,
-      "rewards/margins": -0.006958496756851673,
-      "rewards/rejected": -0.0031617730855941772,
+      "logits/chosen": -2.0167124271392822,
+      "logits/rejected": -2.0079712867736816,
+      "logps/chosen": -32.590545654296875,
+      "logps/rejected": -32.499671936035156,
+      "loss": 1.0241,
+      "rewards/accuracies": 0.4375,
+      "rewards/chosen": -0.01236567273736,
+      "rewards/margins": -0.024063030257821083,
+      "rewards/rejected": 0.011697360314428806,
       "step": 40
     },
     {
       "epoch": 0.13,
       "learning_rate": 4.987541037542186e-07,
-      "logits/chosen": -1.8645082712173462,
-      "logits/rejected": -1.8537276983261108,
-      "logps/chosen": -33.5635871887207,
-      "logps/rejected": -35.42169952392578,
-      "loss": 0.3858,
-      "rewards/accuracies": 0.38749998807907104,
-      "rewards/chosen": -0.005497059319168329,
-      "rewards/margins": -0.03307711333036423,
-      "rewards/rejected": 0.027580058202147484,
+      "logits/chosen": -1.8641105890274048,
+      "logits/rejected": -1.8533456325531006,
+      "logps/chosen": -33.56208038330078,
+      "logps/rejected": -35.437400817871094,
+      "loss": 1.0176,
+      "rewards/accuracies": 0.4375,
+      "rewards/chosen": -0.004139312542974949,
+      "rewards/margins": -0.017586929723620415,
+      "rewards/rejected": 0.013447612524032593,
       "step": 50
     },
     {
       "epoch": 0.16,
       "learning_rate": 4.954691471941118e-07,
-      "logits/chosen": -1.9458835124969482,
-      "logits/rejected": -1.9478065967559814,
-      "logps/chosen": -32.57460021972656,
-      "logps/rejected": -33.195518493652344,
-      "loss": 0.3015,
-      "rewards/accuracies": 0.574999988079071,
-      "rewards/chosen": 0.02302517369389534,
-      "rewards/margins": 0.04474987834692001,
-      "rewards/rejected": -0.021724706515669823,
+      "logits/chosen": -1.9452192783355713,
+      "logits/rejected": -1.9471514225006104,
+      "logps/chosen": -32.587867736816406,
+      "logps/rejected": -33.20810317993164,
+      "loss": 0.9559,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": 0.01108462829142809,
+      "rewards/margins": 0.04413585737347603,
+      "rewards/rejected": -0.033051230013370514,
       "step": 60
     },
     {
       "epoch": 0.18,
       "learning_rate": 4.901618883413548e-07,
-      "logits/chosen": -2.0793328285217285,
-      "logits/rejected": -2.084324836730957,
-      "logps/chosen": -33.99799728393555,
-      "logps/rejected": -36.57799530029297,
-      "loss": 0.3658,
-      "rewards/accuracies": 0.48750001192092896,
-      "rewards/chosen": -0.00855917576700449,
-      "rewards/margins": -0.00035296304849907756,
-      "rewards/rejected": -0.008206211030483246,
+      "logits/chosen": -2.0798697471618652,
+      "logits/rejected": -2.084836483001709,
+      "logps/chosen": -33.99953079223633,
+      "logps/rejected": -36.59130859375,
+      "loss": 0.9897,
+      "rewards/accuracies": 0.4749999940395355,
+      "rewards/chosen": -0.009940117597579956,
+      "rewards/margins": 0.010251833125948906,
+      "rewards/rejected": -0.020191945135593414,
       "step": 70
     },
     {
       "epoch": 0.21,
       "learning_rate": 4.828760511501322e-07,
-      "logits/chosen": -1.9421542882919312,
-      "logits/rejected": -1.9453080892562866,
-      "logps/chosen": -34.38970184326172,
-      "logps/rejected": -34.58138656616211,
-      "loss": 0.3181,
-      "rewards/accuracies": 0.5874999761581421,
-      "rewards/chosen": 0.034300822764635086,
-      "rewards/margins": 0.024381538853049278,
-      "rewards/rejected": 0.009919276461005211,
+      "logits/chosen": -1.9422235488891602,
+      "logits/rejected": -1.9453849792480469,
+      "logps/chosen": -34.411861419677734,
+      "logps/rejected": -34.599830627441406,
+      "loss": 0.979,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": 0.014353242702782154,
+      "rewards/margins": 0.02102992869913578,
+      "rewards/rejected": -0.006676685996353626,
       "step": 80
     },
     {
       "epoch": 0.23,
       "learning_rate": 4.736716601303429e-07,
-      "logits/chosen": -1.9507806301116943,
-      "logits/rejected": -1.955293893814087,
-      "logps/chosen": -32.461936950683594,
-      "logps/rejected": -32.3816032409668,
-      "loss": 0.3094,
-      "rewards/accuracies": 0.574999988079071,
-      "rewards/chosen": 0.004934861324727535,
-      "rewards/margins": 0.02535998821258545,
-      "rewards/rejected": -0.02042512409389019,
+      "logits/chosen": -1.950791358947754,
+      "logits/rejected": -1.9553003311157227,
+      "logps/chosen": -32.45661163330078,
+      "logps/rejected": -32.35947799682617,
+      "loss": 0.9898,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": 0.009730497375130653,
+      "rewards/margins": 0.010243801400065422,
+      "rewards/rejected": -0.0005133040249347687,
       "step": 90
     },
     {
       "epoch": 0.26,
       "learning_rate": 4.62624545834521e-07,
-      "logits/chosen": -2.0485286712646484,
-      "logits/rejected": -2.046539783477783,
-      "logps/chosen": -32.23391342163086,
-      "logps/rejected": -31.265544891357422,
-      "loss": 0.333,
+      "logits/chosen": -2.0489749908447266,
+      "logits/rejected": -2.046968460083008,
+      "logps/chosen": -32.233360290527344,
+      "logps/rejected": -31.278451919555664,
+      "loss": 0.9823,
       "rewards/accuracies": 0.512499988079071,
-      "rewards/chosen": 0.006228716112673283,
-      "rewards/margins": 0.005629505030810833,
-      "rewards/rejected": 0.0005992107326164842,
+      "rewards/chosen": 0.006729811429977417,
+      "rewards/margins": 0.01774778962135315,
+      "rewards/rejected": -0.011017980054020882,
       "step": 100
     },
     {
       "epoch": 0.26,
-      "eval_logits/chosen": -2.243820905685425,
-      "eval_logits/rejected": -2.238938570022583,
-      "eval_logps/chosen": -34.02050018310547,
-      "eval_logps/rejected": -37.50347137451172,
-      "eval_loss": 0.3404548168182373,
+      "eval_logits/chosen": -2.243974447250366,
+      "eval_logits/rejected": -2.239091634750366,
+      "eval_logps/chosen": -34.021610260009766,
+      "eval_logps/rejected": -37.50578689575195,
+      "eval_loss": 0.9981797933578491,
       "eval_rewards/accuracies": 0.5103820562362671,
-      "eval_rewards/chosen": 0.012649022974073887,
-      "eval_rewards/margins": 0.0008147989865392447,
-      "eval_rewards/rejected": 0.011834223754703999,
-      "eval_runtime": 146.2563,
-      "eval_samples_per_second": 2.345,
+      "eval_rewards/chosen": 0.011647163890302181,
+      "eval_rewards/margins": 0.0018998426385223866,
+      "eval_rewards/rejected": 0.009747318923473358,
+      "eval_runtime": 146.0735,
+      "eval_samples_per_second": 2.348,
       "eval_steps_per_second": 0.294,
       "step": 100
     },
     {
       "epoch": 0.29,
       "learning_rate": 4.4982572012636904e-07,
-      "logits/chosen": -2.0055313110351562,
-      "logits/rejected": -2.00311541557312,
-      "logps/chosen": -33.23518753051758,
-      "logps/rejected": -34.03091812133789,
-      "loss": 0.3233,
-      "rewards/accuracies": 0.48750001192092896,
-      "rewards/chosen": 0.007764431647956371,
-      "rewards/margins": 0.009159245528280735,
-      "rewards/rejected": -0.0013948131818324327,
+      "logits/chosen": -2.0054612159729004,
+      "logits/rejected": -2.003042221069336,
+      "logps/chosen": -33.2550048828125,
+      "logps/rejected": -34.02202224731445,
+      "loss": 1.0167,
+      "rewards/accuracies": 0.44999998807907104,
+      "rewards/chosen": -0.010068392381072044,
+      "rewards/margins": -0.016683781519532204,
+      "rewards/rejected": 0.006615391466766596,
       "step": 110
     },
     {
       "epoch": 0.31,
       "learning_rate": 4.353806263777677e-07,
-      "logits/chosen": -2.016556978225708,
-      "logits/rejected": -2.0081825256347656,
-      "logps/chosen": -32.435211181640625,
-      "logps/rejected": -32.18501281738281,
-      "loss": 0.3121,
-      "rewards/accuracies": 0.5625,
-      "rewards/chosen": 0.00872363243252039,
-      "rewards/margins": 0.029877260327339172,
-      "rewards/rejected": -0.02115362510085106,
+      "logits/chosen": -2.016511917114258,
+      "logits/rejected": -2.008146047592163,
+      "logps/chosen": -32.45395278930664,
+      "logps/rejected": -32.17681121826172,
+      "loss": 0.9944,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -0.008141009137034416,
+      "rewards/margins": 0.00563241308555007,
+      "rewards/rejected": -0.013773423619568348,
       "step": 120
     },
     {
       "epoch": 0.34,
       "learning_rate": 4.194082707715275e-07,
-      "logits/chosen": -2.0461266040802,
-      "logits/rejected": -2.03806734085083,
-      "logps/chosen": -30.502710342407227,
-      "logps/rejected": -32.0546760559082,
-      "loss": 0.3674,
-      "rewards/accuracies": 0.4625000059604645,
-      "rewards/chosen": -0.02429451420903206,
-      "rewards/margins": -0.02279272861778736,
-      "rewards/rejected": -0.0015017850091680884,
+      "logits/chosen": -2.0463106632232666,
+      "logits/rejected": -2.0382590293884277,
+      "logps/chosen": -30.476207733154297,
+      "logps/rejected": -32.020484924316406,
+      "loss": 1.0297,
+      "rewards/accuracies": 0.3499999940395355,
+      "rewards/chosen": -0.00044600124238058925,
+      "rewards/margins": -0.029714178293943405,
+      "rewards/rejected": 0.02926817536354065,
       "step": 130
     },
     {
       "epoch": 0.36,
       "learning_rate": 4.020402418666621e-07,
-      "logits/chosen": -1.9767634868621826,
-      "logits/rejected": -1.9870494604110718,
-      "logps/chosen": -31.402328491210938,
-      "logps/rejected": -32.55817794799805,
-      "loss": 0.3125,
-      "rewards/accuracies": 0.550000011920929,
-      "rewards/chosen": 0.015601096674799919,
-      "rewards/margins": 0.03657008707523346,
-      "rewards/rejected": -0.02096899226307869,
+      "logits/chosen": -1.9767357110977173,
+      "logits/rejected": -1.986999273300171,
+      "logps/chosen": -31.381155014038086,
+      "logps/rejected": -32.54108428955078,
+      "loss": 0.9598,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": 0.034655503928661346,
+      "rewards/margins": 0.04024083539843559,
+      "rewards/rejected": -0.005585335195064545,
       "step": 140
     },
     {
       "epoch": 0.39,
       "learning_rate": 3.8341962650351185e-07,
-      "logits/chosen": -1.8908436298370361,
-      "logits/rejected": -1.8919413089752197,
-      "logps/chosen": -34.207401275634766,
-      "logps/rejected": -34.783695220947266,
-      "loss": 0.3373,
-      "rewards/accuracies": 0.5375000238418579,
-      "rewards/chosen": -0.013235817663371563,
-      "rewards/margins": 0.01055879332125187,
-      "rewards/rejected": -0.023794615641236305,
+      "logits/chosen": -1.8906068801879883,
+      "logits/rejected": -1.8916860818862915,
+      "logps/chosen": -34.20075225830078,
+      "logps/rejected": -34.7618522644043,
+      "loss": 1.0031,
+      "rewards/accuracies": 0.4124999940395355,
+      "rewards/chosen": -0.007252261973917484,
+      "rewards/margins": -0.0031175222247838974,
+      "rewards/rejected": -0.004134741611778736,
       "step": 150
     },
     {
       "epoch": 0.42,
       "learning_rate": 3.636998309800572e-07,
-      "logits/chosen": -1.942582368850708,
-      "logits/rejected": -1.9391053915023804,
-      "logps/chosen": -36.15548324584961,
-      "logps/rejected": -32.729759216308594,
-      "loss": 0.3207,
-      "rewards/accuracies": 0.5,
-      "rewards/chosen": 0.014574882574379444,
-      "rewards/margins": 0.020880455151200294,
-      "rewards/rejected": -0.006305573042482138,
+      "logits/chosen": -1.942251443862915,
+      "logits/rejected": -1.9387743473052979,
+      "logps/chosen": -36.145790100097656,
+      "logps/rejected": -32.73196029663086,
+      "loss": 0.9684,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": 0.023301539942622185,
+      "rewards/margins": 0.031586963683366776,
+      "rewards/rejected": -0.008285422809422016,
       "step": 160
     },
     {
       "epoch": 0.44,
       "learning_rate": 3.430433172111807e-07,
-      "logits/chosen": -2.042206287384033,
-      "logits/rejected": -2.034825086593628,
-      "logps/chosen": -33.77631378173828,
-      "logps/rejected": -31.35869789123535,
-      "loss": 0.3201,
-      "rewards/accuracies": 0.5625,
-      "rewards/chosen": 0.019394155591726303,
-      "rewards/margins": 0.025556519627571106,
-      "rewards/rejected": -0.00616236450150609,
+      "logits/chosen": -2.041952610015869,
+      "logits/rejected": -2.0345559120178223,
+      "logps/chosen": -33.81856155395508,
+      "logps/rejected": -31.359323501586914,
+      "loss": 1.0119,
+      "rewards/accuracies": 0.4749999940395355,
+      "rewards/chosen": -0.018632039427757263,
+      "rewards/margins": -0.011905943043529987,
+      "rewards/rejected": -0.006726093590259552,
       "step": 170
     },
     {
       "epoch": 0.47,
       "learning_rate": 3.216202642830543e-07,
-      "logits/chosen": -2.0474154949188232,
-      "logits/rejected": -2.0526936054229736,
-      "logps/chosen": -32.521339416503906,
-      "logps/rejected": -32.497581481933594,
-      "loss": 0.2999,
-      "rewards/accuracies": 0.612500011920929,
-      "rewards/chosen": 0.021103620529174805,
-      "rewards/margins": 0.032301973551511765,
-      "rewards/rejected": -0.01119835115969181,
+      "logits/chosen": -2.0473170280456543,
+      "logits/rejected": -2.0525729656219482,
+      "logps/chosen": -32.52764129638672,
+      "logps/rejected": -32.50192642211914,
+      "loss": 0.9695,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.015429094433784485,
+      "rewards/margins": 0.030534446239471436,
+      "rewards/rejected": -0.015105349011719227,
       "step": 180
     },
     {
       "epoch": 0.49,
       "learning_rate": 2.9960716642946403e-07,
-      "logits/chosen": -2.0485098361968994,
-      "logits/rejected": -2.0457375049591064,
-      "logps/chosen": -31.49017906188965,
-      "logps/rejected": -31.345867156982422,
-      "loss": 0.3315,
-      "rewards/accuracies": 0.5625,
-      "rewards/chosen": -0.001104071387089789,
-      "rewards/margins": 0.023781923577189445,
-      "rewards/rejected": -0.02488599717617035,
+      "logits/chosen": -2.048530340194702,
+      "logits/rejected": -2.045767307281494,
+      "logps/chosen": -31.485219955444336,
+      "logps/rejected": -31.34262466430664,
+      "loss": 0.9747,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": 0.003360496135428548,
+      "rewards/margins": 0.02533094584941864,
+      "rewards/rejected": -0.02197045087814331,
       "step": 190
     },
     {
       "epoch": 0.52,
       "learning_rate": 2.771853789806683e-07,
-      "logits/chosen": -1.9183101654052734,
-      "logits/rejected": -1.9229921102523804,
-      "logps/chosen": -31.60622215270996,
-      "logps/rejected": -32.805049896240234,
-      "loss": 0.3306,
-      "rewards/accuracies": 0.5249999761581421,
-      "rewards/chosen": 5.159676220500842e-05,
-      "rewards/margins": 0.012757277116179466,
-      "rewards/rejected": -0.012705676257610321,
+      "logits/chosen": -1.918745756149292,
+      "logits/rejected": -1.9234097003936768,
+      "logps/chosen": -31.576425552368164,
+      "logps/rejected": -32.79480743408203,
+      "loss": 0.9696,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": 0.026870254427194595,
+      "rewards/margins": 0.03035963699221611,
+      "rewards/rejected": -0.0034893869888037443,
       "step": 200
     },
     {
       "epoch": 0.52,
-      "eval_logits/chosen": -2.2434937953948975,
-      "eval_logits/rejected": -2.2386233806610107,
-      "eval_logps/chosen": -34.02851867675781,
-      "eval_logps/rejected": -37.51131057739258,
-      "eval_loss": 0.3466452658176422,
-      "eval_rewards/accuracies": 0.5020764470100403,
-      "eval_rewards/chosen": 0.005428452976047993,
-      "eval_rewards/margins": 0.0006523749325424433,
-      "eval_rewards/rejected": 0.00477607874199748,
-      "eval_runtime": 145.8217,
+      "eval_logits/chosen": -2.243912696838379,
+      "eval_logits/rejected": -2.2390317916870117,
+      "eval_logps/chosen": -34.021690368652344,
+      "eval_logps/rejected": -37.502418518066406,
+      "eval_loss": 1.001371145248413,
+      "eval_rewards/accuracies": 0.4692690968513489,
+      "eval_rewards/chosen": 0.011576101183891296,
+      "eval_rewards/margins": -0.0012021720176562667,
+      "eval_rewards/rejected": 0.012778270989656448,
+      "eval_runtime": 145.8493,
       "eval_samples_per_second": 2.352,
       "eval_steps_per_second": 0.295,
       "step": 200
@@ -337,279 +337,279 @@
     {
       "epoch": 0.55,
       "learning_rate": 2.5453962426402e-07,
-      "logits/chosen": -2.032118558883667,
-      "logits/rejected": -2.042782783508301,
-      "logps/chosen": -31.939167022705078,
-      "logps/rejected": -33.88642501831055,
-      "loss": 0.2871,
-      "rewards/accuracies": 0.6000000238418579,
-      "rewards/chosen": 0.02892182394862175,
-      "rewards/margins": 0.050276655703783035,
-      "rewards/rejected": -0.021354828029870987,
+      "logits/chosen": -2.0318515300750732,
+      "logits/rejected": -2.0425400733947754,
+      "logps/chosen": -31.96026039123535,
+      "logps/rejected": -33.8691291809082,
+      "loss": 0.9843,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.009936656802892685,
+      "rewards/margins": 0.015721982344985008,
+      "rewards/rejected": -0.005785329267382622,
       "step": 210
     },
     {
       "epoch": 0.57,
       "learning_rate": 2.318564697655179e-07,
-      "logits/chosen": -1.925293207168579,
-      "logits/rejected": -1.9401395320892334,
-      "logps/chosen": -30.106586456298828,
-      "logps/rejected": -31.56791114807129,
-      "loss": 0.3327,
-      "rewards/accuracies": 0.6000000238418579,
-      "rewards/chosen": 0.0044566914439201355,
-      "rewards/margins": 0.013218318112194538,
-      "rewards/rejected": -0.008761629462242126,
+      "logits/chosen": -1.9256188869476318,
+      "logits/rejected": -1.9404585361480713,
+      "logps/chosen": -30.080001831054688,
+      "logps/rejected": -31.53633689880371,
+      "loss": 0.9913,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": 0.028384273871779442,
+      "rewards/margins": 0.00873211957514286,
+      "rewards/rejected": 0.01965215802192688,
       "step": 220
     },
     {
       "epoch": 0.6,
       "learning_rate": 2.093227910899832e-07,
-      "logits/chosen": -1.9831531047821045,
-      "logits/rejected": -1.9871160984039307,
-      "logps/chosen": -33.386024475097656,
-      "logps/rejected": -31.5727481842041,
-      "loss": 0.2974,
-      "rewards/accuracies": 0.550000011920929,
-      "rewards/chosen": 0.028298119083046913,
-      "rewards/margins": 0.04939670115709305,
-      "rewards/rejected": -0.021098580211400986,
+      "logits/chosen": -1.9830671548843384,
+      "logits/rejected": -1.9870269298553467,
+      "logps/chosen": -33.386474609375,
+      "logps/rejected": -31.55719566345215,
+      "loss": 0.965,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": 0.02789575420320034,
+      "rewards/margins": 0.03499894589185715,
+      "rewards/rejected": -0.007103185169398785,
       "step": 230
     },
     {
       "epoch": 0.62,
       "learning_rate": 1.8712423238279356e-07,
-      "logits/chosen": -1.9833450317382812,
-      "logits/rejected": -1.9613698720932007,
-      "logps/chosen": -34.1508903503418,
-      "logps/rejected": -34.95817947387695,
-      "loss": 0.3388,
-      "rewards/accuracies": 0.5249999761581421,
-      "rewards/chosen": 0.002569653559476137,
-      "rewards/margins": 0.00921549927443266,
-      "rewards/rejected": -0.006645847111940384,
+      "logits/chosen": -1.9833042621612549,
+      "logits/rejected": -1.96132493019104,
+      "logps/chosen": -34.13536834716797,
+      "logps/rejected": -34.952049255371094,
+      "loss": 0.9823,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": 0.016542484983801842,
+      "rewards/margins": 0.01767394319176674,
+      "rewards/rejected": -0.0011314575094729662,
       "step": 240
     },
     {
       "epoch": 0.65,
       "learning_rate": 1.654436768970182e-07,
-      "logits/chosen": -2.0248680114746094,
-      "logits/rejected": -2.021559238433838,
-      "logps/chosen": -32.907005310058594,
-      "logps/rejected": -36.22737121582031,
-      "loss": 0.3221,
-      "rewards/accuracies": 0.550000011920929,
-      "rewards/chosen": 0.016254058107733727,
-      "rewards/margins": 0.018458303064107895,
-      "rewards/rejected": -0.002204245189204812,
+      "logits/chosen": -2.0247886180877686,
+      "logits/rejected": -2.0214812755584717,
+      "logps/chosen": -32.913856506347656,
+      "logps/rejected": -36.23151397705078,
+      "loss": 0.984,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": 0.010092089883983135,
+      "rewards/margins": 0.016022875905036926,
+      "rewards/rejected": -0.0059307836927473545,
       "step": 250
     },
     {
       "epoch": 0.68,
       "learning_rate": 1.444597403062196e-07,
-      "logits/chosen": -1.8913663625717163,
-      "logits/rejected": -1.8889217376708984,
-      "logps/chosen": -34.18629455566406,
-      "logps/rejected": -35.490962982177734,
-      "loss": 0.3601,
-      "rewards/accuracies": 0.5,
-      "rewards/chosen": 0.0018952053505927324,
-      "rewards/margins": -0.010397391393780708,
-      "rewards/rejected": 0.012292595580220222,
+      "logits/chosen": -1.8914467096328735,
+      "logits/rejected": -1.8890043497085571,
+      "logps/chosen": -34.18690872192383,
+      "logps/rejected": -35.524742126464844,
+      "loss": 0.9806,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": 0.0013398710871115327,
+      "rewards/margins": 0.019445054233074188,
+      "rewards/rejected": -0.01810518093407154,
       "step": 260
     },
     {
       "epoch": 0.7,
       "learning_rate": 1.2434529917578887e-07,
-      "logits/chosen": -1.8756721019744873,
-      "logits/rejected": -1.8731330633163452,
-      "logps/chosen": -34.39625549316406,
-      "logps/rejected": -31.76236343383789,
-      "loss": 0.3492,
-      "rewards/accuracies": 0.4625000059604645,
-      "rewards/chosen": 0.001211274997331202,
-      "rewards/margins": -0.006859898567199707,
-      "rewards/rejected": 0.008071173913776875,
+      "logits/chosen": -1.8761389255523682,
+      "logits/rejected": -1.8735923767089844,
+      "logps/chosen": -34.396785736083984,
+      "logps/rejected": -31.74139976501465,
+      "loss": 1.0262,
+      "rewards/accuracies": 0.4000000059604645,
+      "rewards/chosen": 0.000734941684640944,
+      "rewards/margins": -0.026202131062746048,
+      "rewards/rejected": 0.02693706750869751,
       "step": 270
     },
     {
       "epoch": 0.73,
       "learning_rate": 1.0526606671603521e-07,
-      "logits/chosen": -1.9795773029327393,
-      "logits/rejected": -1.9689537286758423,
-      "logps/chosen": -35.320404052734375,
-      "logps/rejected": -31.86062240600586,
-      "loss": 0.298,
-      "rewards/accuracies": 0.574999988079071,
-      "rewards/chosen": 0.032611697912216187,
-      "rewards/margins": 0.05027870461344719,
-      "rewards/rejected": -0.017666997388005257,
+      "logits/chosen": -1.9801757335662842,
+      "logits/rejected": -1.9695546627044678,
+      "logps/chosen": -35.34246063232422,
+      "logps/rejected": -31.823230743408203,
+      "loss": 1.0032,
+      "rewards/accuracies": 0.4749999940395355,
+      "rewards/chosen": 0.012763315811753273,
+      "rewards/margins": -0.003222113475203514,
+      "rewards/rejected": 0.015985429286956787,
       "step": 280
     },
     {
       "epoch": 0.75,
       "learning_rate": 8.737922755071453e-08,
-      "logits/chosen": -2.075582981109619,
-      "logits/rejected": -2.0605711936950684,
-      "logps/chosen": -30.907567977905273,
-      "logps/rejected": -32.65256881713867,
-      "loss": 0.3186,
-      "rewards/accuracies": 0.512499988079071,
-      "rewards/chosen": 0.014904007315635681,
-      "rewards/margins": 0.015564213506877422,
-      "rewards/rejected": -0.0006602049106732011,
+      "logits/chosen": -2.075316905975342,
+      "logits/rejected": -2.0602917671203613,
+      "logps/chosen": -30.897375106811523,
+      "logps/rejected": -32.64093780517578,
+      "loss": 0.9857,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": 0.024072837084531784,
+      "rewards/margins": 0.014264687895774841,
+      "rewards/rejected": 0.009808152914047241,
       "step": 290
     },
     {
       "epoch": 0.78,
       "learning_rate": 7.08321427484816e-08,
-      "logits/chosen": -1.9468443393707275,
-      "logits/rejected": -1.944310188293457,
-      "logps/chosen": -32.90850067138672,
-      "logps/rejected": -30.801532745361328,
-      "loss": 0.3327,
-      "rewards/accuracies": 0.4749999940395355,
-      "rewards/chosen": 0.016746358945965767,
-      "rewards/margins": 0.002998964861035347,
-      "rewards/rejected": 0.01374739408493042,
+      "logits/chosen": -1.946758508682251,
+      "logits/rejected": -1.9442226886749268,
+      "logps/chosen": -32.89502716064453,
+      "logps/rejected": -30.831274032592773,
+      "loss": 0.9581,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": 0.0288736280053854,
+      "rewards/margins": 0.04189068451523781,
+      "rewards/rejected": -0.013017055578529835,
       "step": 300
     },
     {
       "epoch": 0.78,
-      "eval_logits/chosen": -2.243875741958618,
-      "eval_logits/rejected": -2.238999366760254,
-      "eval_logps/chosen": -34.01763916015625,
-      "eval_logps/rejected": -37.51422882080078,
-      "eval_loss": 0.32764142751693726,
-      "eval_rewards/accuracies": 0.5365448594093323,
-      "eval_rewards/chosen": 0.015219501219689846,
-      "eval_rewards/margins": 0.01306714303791523,
-      "eval_rewards/rejected": 0.0021523565519601107,
-      "eval_runtime": 145.9076,
-      "eval_samples_per_second": 2.351,
+      "eval_logits/chosen": -2.2437047958374023,
+      "eval_logits/rejected": -2.2388339042663574,
+      "eval_logps/chosen": -34.03062438964844,
+      "eval_logps/rejected": -37.499412536621094,
+      "eval_loss": 1.0120128393173218,
+      "eval_rewards/accuracies": 0.472591370344162,
+      "eval_rewards/chosen": 0.003532935632392764,
+      "eval_rewards/margins": -0.011953731998801231,
+      "eval_rewards/rejected": 0.015486669726669788,
+      "eval_runtime": 145.7744,
+      "eval_samples_per_second": 2.353,
       "eval_steps_per_second": 0.295,
       "step": 300
     },
     {
       "epoch": 0.81,
       "learning_rate": 5.576113578589034e-08,
-      "logits/chosen": -1.9285104274749756,
-      "logits/rejected": -1.925238847732544,
-      "logps/chosen": -31.57692527770996,
-      "logps/rejected": -33.744144439697266,
-      "loss": 0.3143,
-      "rewards/accuracies": 0.550000011920929,
-      "rewards/chosen": 0.023783722892403603,
-      "rewards/margins": 0.025577496737241745,
-      "rewards/rejected": -0.0017937703523784876,
+      "logits/chosen": -1.928391695022583,
+      "logits/rejected": -1.925132393836975,
+      "logps/chosen": -31.573719024658203,
+      "logps/rejected": -33.733333587646484,
+      "loss": 0.9813,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.026669049635529518,
+      "rewards/margins": 0.01873381994664669,
+      "rewards/rejected": 0.007935228757560253,
       "step": 310
     },
     {
       "epoch": 0.83,
       "learning_rate": 4.229036944380912e-08,
-      "logits/chosen": -1.980386734008789,
-      "logits/rejected": -1.9680719375610352,
-      "logps/chosen": -34.57415771484375,
-      "logps/rejected": -33.55961227416992,
-      "loss": 0.31,
-      "rewards/accuracies": 0.5375000238418579,
-      "rewards/chosen": 0.01972355879843235,
-      "rewards/margins": 0.03841744735836983,
-      "rewards/rejected": -0.018693890422582626,
+      "logits/chosen": -1.9808565378189087,
+      "logits/rejected": -1.9685465097427368,
+      "logps/chosen": -34.569026947021484,
+      "logps/rejected": -33.5385856628418,
+      "loss": 0.9759,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": 0.02433854714035988,
+      "rewards/margins": 0.024108227342367172,
+      "rewards/rejected": 0.00023032091849017888,
       "step": 320
     },
     {
       "epoch": 0.86,
       "learning_rate": 3.053082288996112e-08,
-      "logits/chosen": -2.015927314758301,
-      "logits/rejected": -2.014465570449829,
-      "logps/chosen": -33.47549057006836,
-      "logps/rejected": -32.47983169555664,
-      "loss": 0.3454,
-      "rewards/accuracies": 0.512499988079071,
-      "rewards/chosen": 0.009270393289625645,
-      "rewards/margins": 0.009491671808063984,
-      "rewards/rejected": -0.0002212822437286377,
+      "logits/chosen": -2.0160164833068848,
+      "logits/rejected": -2.0145697593688965,
+      "logps/chosen": -33.470794677734375,
+      "logps/rejected": -32.465301513671875,
+      "loss": 0.9994,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": 0.013503548689186573,
+      "rewards/margins": 0.0006495133275166154,
+      "rewards/rejected": 0.012854036875069141,
       "step": 330
     },
     {
       "epoch": 0.88,
       "learning_rate": 2.05793773749158e-08,
-      "logits/chosen": -2.102839946746826,
-      "logits/rejected": -2.0870471000671387,
-      "logps/chosen": -34.18696212768555,
-      "logps/rejected": -33.075828552246094,
-      "loss": 0.4092,
-      "rewards/accuracies": 0.4124999940395355,
-      "rewards/chosen": -0.006814646068960428,
-      "rewards/margins": -0.04479615017771721,
-      "rewards/rejected": 0.03798150271177292,
+      "logits/chosen": -2.1030516624450684,
+      "logits/rejected": -2.087252140045166,
+      "logps/chosen": -34.154754638671875,
+      "logps/rejected": -33.087013244628906,
+      "loss": 1.0057,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": 0.02217467688024044,
+      "rewards/margins": -0.005742859095335007,
+      "rewards/rejected": 0.027917543426156044,
       "step": 340
     },
     {
       "epoch": 0.91,
       "learning_rate": 1.251801807404168e-08,
-      "logits/chosen": -1.9747005701065063,
-      "logits/rejected": -1.9737575054168701,
-      "logps/chosen": -33.231842041015625,
-      "logps/rejected": -32.461700439453125,
-      "loss": 0.3207,
-      "rewards/accuracies": 0.5249999761581421,
-      "rewards/chosen": 0.04523162543773651,
-      "rewards/margins": 0.04212886095046997,
-      "rewards/rejected": 0.003102770773693919,
+      "logits/chosen": -1.9748103618621826,
+      "logits/rejected": -1.9738750457763672,
+      "logps/chosen": -33.23518371582031,
+      "logps/rejected": -32.46066665649414,
+      "loss": 0.9618,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.04222174361348152,
+      "rewards/margins": 0.03818847984075546,
+      "rewards/rejected": 0.004033264704048634,
       "step": 350
     },
     {
       "epoch": 0.94,
       "learning_rate": 6.41315865106129e-09,
-      "logits/chosen": -1.9306046962738037,
-      "logits/rejected": -1.9409637451171875,
-      "logps/chosen": -32.217220306396484,
-      "logps/rejected": -35.306907653808594,
-      "loss": 0.3411,
-      "rewards/accuracies": 0.4625000059604645,
-      "rewards/chosen": -0.011275654658675194,
-      "rewards/margins": 0.005062489304691553,
-      "rewards/rejected": -0.01633814536035061,
+      "logits/chosen": -1.9303611516952515,
+      "logits/rejected": -1.940708875656128,
+      "logps/chosen": -32.200523376464844,
+      "logps/rejected": -35.27861785888672,
+      "loss": 1.0054,
+      "rewards/accuracies": 0.42500001192092896,
+      "rewards/chosen": 0.0037514485884457827,
+      "rewards/margins": -0.005373493768274784,
+      "rewards/rejected": 0.009124943986535072,
       "step": 360
     },
     {
       "epoch": 0.96,
       "learning_rate": 2.3150941078050324e-09,
-      "logits/chosen": -2.0694775581359863,
-      "logits/rejected": -2.0629212856292725,
-      "logps/chosen": -33.65714645385742,
-      "logps/rejected": -29.221160888671875,
-      "loss": 0.3549,
-      "rewards/accuracies": 0.4749999940395355,
-      "rewards/chosen": -0.004660401493310928,
-      "rewards/margins": -0.009401795454323292,
-      "rewards/rejected": 0.0047413939610123634,
+      "logits/chosen": -2.069342613220215,
+      "logits/rejected": -2.062788486480713,
+      "logps/chosen": -33.65803527832031,
+      "logps/rejected": -29.199758529663086,
+      "loss": 1.0295,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -0.005459528882056475,
+      "rewards/margins": -0.029461050406098366,
+      "rewards/rejected": 0.02400151826441288,
       "step": 370
     },
     {
       "epoch": 0.99,
       "learning_rate": 2.575864278703266e-10,
-      "logits/chosen": -1.9296157360076904,
-      "logits/rejected": -1.931777000427246,
-      "logps/chosen": -34.22825241088867,
-      "logps/rejected": -30.912120819091797,
-      "loss": 0.2952,
-      "rewards/accuracies": 0.5375000238418579,
-      "rewards/chosen": 0.009714365005493164,
-      "rewards/margins": 0.04525749385356903,
-      "rewards/rejected": -0.035543132573366165,
+      "logits/chosen": -1.9290788173675537,
+      "logits/rejected": -1.931239366531372,
+      "logps/chosen": -34.229915618896484,
+      "logps/rejected": -30.90093994140625,
+      "loss": 0.9663,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": 0.00822196900844574,
+      "rewards/margins": 0.03370029479265213,
+      "rewards/rejected": -0.02547832392156124,
       "step": 380
     },
     {
       "epoch": 1.0,
       "step": 385,
       "total_flos": 0.0,
-      "train_loss": 0.0744757689438857,
-      "train_runtime": 628.0651,
-      "train_samples_per_second": 4.902,
-      "train_steps_per_second": 0.613
+      "train_loss": 0.21873634759481853,
+      "train_runtime": 627.4596,
+      "train_samples_per_second": 4.907,
+      "train_steps_per_second": 0.614
     }
   ],
   "logging_steps": 10,

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c79706b878a2ab50513a99855a2b6398ad70408a38470eb167a411505f0bd5e7
+oid sha256:6b8ac9e90a7d2e5f626eca1eeb6811d0f8d6681fb84d4869b1f9c9a5085da4d2
 size 4984