Model save

README.md

@@ -1,13 +1,10 @@
 ---
 library_name: peft
 tags:
-- alignment-handbook
 - trl
 - dpo
 - generated_from_trainer
 base_model: NbAiLab/nb-gpt-j-6B-v2
-datasets:
-- hugodk-sch/aftonposten_title_prefs
 model-index:
 - name: aftonposten-6b-align-scan
   results: []
@@ -18,17 +15,17 @@ should probably proofread and complete it, then remove this comment. -->
 
 # aftonposten-6b-align-scan
 
-This model is a fine-tuned version of [data/ap-gpt-j-6b-sft-qlora-04-08](https://huggingface.co/data/ap-gpt-j-6b-sft-qlora-04-08) on the hugodk-sch/aftonposten_title_prefs dataset.
+This model is a fine-tuned version of [NbAiLab/nb-gpt-j-6B-v2](https://huggingface.co/NbAiLab/nb-gpt-j-6B-v2) on an unknown dataset.
 It achieves the following results on the evaluation set:
-- Loss: 0.5001
-- Rewards/chosen: 0.0029
-- Rewards/rejected: 0.0032
-- Rewards/accuracies: 0.4842
-- Rewards/margins: -0.0003
-- Logps/rejected: -37.5006
-- Logps/chosen: -34.0202
-- Logits/rejected: -2.2393
-- Logits/chosen: -2.2442
+- Loss: 0.4998
+- Rewards/chosen: 0.0042
+- Rewards/rejected: 0.0036
+- Rewards/accuracies: 0.5137
+- Rewards/margins: 0.0006
+- Logps/rejected: -37.5046
+- Logps/chosen: -34.0207
+- Logits/rejected: -2.2382
+- Logits/chosen: -2.2431
 
 ## Model description
 
@@ -63,9 +60,9 @@ The following hyperparameters were used during training:
 
 | Training Loss | Epoch | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
 |:-------------:|:-----:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
-| 0.5           | 0.26  | 100  | 0.5003          | 0.0013         | 0.0021           | 0.4934             | -0.0008         | -37.5063       | -34.0280     | -2.2391         | -2.2440       |
-| 0.4994        | 0.52  | 200  | 0.5005          | -0.0009        | 0.0012           | 0.5021             | -0.0021         | -37.5106       | -34.0392     | -2.2401         | -2.2450       |
-| 0.499         | 0.78  | 300  | 0.5010          | -0.0014        | 0.0035           | 0.4431             | -0.0049         | -37.4991       | -34.0418     | -2.2397         | -2.2446       |
+| 0.4987        | 0.26  | 100  | 0.4994          | 0.0049         | 0.0020           | 0.5075             | 0.0029          | -37.5100       | -34.0182     | -2.2389         | -2.2437       |
+| 0.4979        | 0.52  | 200  | 0.4998          | 0.0039         | 0.0031           | 0.4863             | 0.0008          | -37.5062       | -34.0214     | -2.2384         | -2.2432       |
+| 0.4978        | 0.78  | 300  | 0.4998          | 0.0042         | 0.0036           | 0.5137             | 0.0006          | -37.5046       | -34.0207     | -2.2382         | -2.2431       |
 
 
 ### Framework versions

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0e0fe08296538d49776839e2fd3ec2743dd0adef4a1968a65578a138c0d36353
+oid sha256:0143633a911bfdb88e846b1d054b1bda220855de7ed3cbfb07035b381388a398
 size 176183216

all_results.json

@@ -1,21 +1,8 @@
 {
     "epoch": 1.0,
-    "eval_logits/chosen": -2.244190216064453,
-    "eval_logits/rejected": -2.2393178939819336,
-    "eval_logps/chosen": -34.020179748535156,
-    "eval_logps/rejected": -37.50062561035156,
-    "eval_loss": 0.500054657459259,
-    "eval_rewards/accuracies": 0.4842192530632019,
-    "eval_rewards/chosen": 0.002875381149351597,
-    "eval_rewards/margins": -0.0003236339834984392,
-    "eval_rewards/rejected": 0.0031990152783691883,
-    "eval_runtime": 145.749,
-    "eval_samples": 343,
-    "eval_samples_per_second": 2.353,
-    "eval_steps_per_second": 0.295,
-    "train_loss": 0.4997496530607149,
-    "train_runtime": 3257.6696,
+    "train_loss": 0.49920677581390777,
+    "train_runtime": 3256.0166,
     "train_samples": 3079,
-    "train_samples_per_second": 0.945,
+    "train_samples_per_second": 0.946,
     "train_steps_per_second": 0.118
 }

train_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 1.0,
-    "train_loss": 0.4997496530607149,
-    "train_runtime": 3257.6696,
+    "train_loss": 0.49920677581390777,
+    "train_runtime": 3256.0166,
     "train_samples": 3079,
-    "train_samples_per_second": 0.945,
+    "train_samples_per_second": 0.946,
     "train_steps_per_second": 0.118
 }

trainer_state.json

@@ -25,590 +25,590 @@
     {
       "epoch": 0.03,
       "learning_rate": 1.2820512820512818e-07,
-      "logits/chosen": -1.866280436515808,
-      "logits/rejected": -1.8705918788909912,
-      "logps/chosen": -37.00367736816406,
-      "logps/rejected": -33.67123794555664,
-      "loss": 0.4986,
-      "rewards/accuracies": 0.4722222089767456,
-      "rewards/chosen": 0.000584296474698931,
-      "rewards/margins": 0.008817334659397602,
-      "rewards/rejected": -0.00823303870856762,
+      "logits/chosen": -1.8662705421447754,
+      "logits/rejected": -1.870587706565857,
+      "logps/chosen": -36.9873046875,
+      "logps/rejected": -33.66048049926758,
+      "loss": 0.4974,
+      "rewards/accuracies": 0.5138888955116272,
+      "rewards/chosen": 0.005787754897028208,
+      "rewards/margins": 0.014910436235368252,
+      "rewards/rejected": -0.00912268366664648,
       "step": 10
     },
     {
       "epoch": 0.05,
       "learning_rate": 2.5641025641025636e-07,
-      "logits/chosen": -1.9974257946014404,
-      "logits/rejected": -2.0000691413879395,
-      "logps/chosen": -29.649478912353516,
-      "logps/rejected": -29.038330078125,
-      "loss": 0.5011,
-      "rewards/accuracies": 0.3499999940395355,
-      "rewards/chosen": -0.0014564015436917543,
-      "rewards/margins": -0.0064794206991791725,
-      "rewards/rejected": 0.0050230189226567745,
+      "logits/chosen": -1.997283935546875,
+      "logits/rejected": -1.9999357461929321,
+      "logps/chosen": -29.62722396850586,
+      "logps/rejected": -29.066070556640625,
+      "loss": 0.4988,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": 0.004491320811212063,
+      "rewards/margins": 0.005279188044369221,
+      "rewards/rejected": -0.0007878671167418361,
       "step": 20
     },
     {
       "epoch": 0.08,
       "learning_rate": 3.8461538461538463e-07,
-      "logits/chosen": -1.9199994802474976,
-      "logits/rejected": -1.91730535030365,
-      "logps/chosen": -31.410457611083984,
-      "logps/rejected": -33.23019027709961,
-      "loss": 0.4992,
-      "rewards/accuracies": 0.5249999761581421,
-      "rewards/chosen": 0.0011272300034761429,
-      "rewards/margins": 0.0034732469357550144,
-      "rewards/rejected": -0.0023460157681256533,
+      "logits/chosen": -1.9196703433990479,
+      "logits/rejected": -1.9169772863388062,
+      "logps/chosen": -31.424610137939453,
+      "logps/rejected": -33.22700119018555,
+      "loss": 0.5001,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": -0.0025542874354869127,
+      "rewards/margins": 8.203647666960023e-06,
+      "rewards/rejected": -0.002562491921707988,
       "step": 30
     },
     {
       "epoch": 0.1,
       "learning_rate": 4.999896948438433e-07,
-      "logits/chosen": -2.017059326171875,
-      "logits/rejected": -2.008305788040161,
-      "logps/chosen": -32.582275390625,
-      "logps/rejected": -32.48918914794922,
-      "loss": 0.5013,
-      "rewards/accuracies": 0.4749999940395355,
-      "rewards/chosen": -0.0010934959864243865,
-      "rewards/margins": -0.005789449438452721,
-      "rewards/rejected": 0.004695953335613012,
+      "logits/chosen": -2.0166847705841064,
+      "logits/rejected": -2.007936954498291,
+      "logps/chosen": -32.587745666503906,
+      "logps/rejected": -32.51628875732422,
+      "loss": 0.5006,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -0.0032799947075545788,
+      "rewards/margins": -0.002195248380303383,
+      "rewards/rejected": -0.0010847460944205523,
       "step": 40
     },
     {
       "epoch": 0.13,
       "learning_rate": 4.987541037542186e-07,
-      "logits/chosen": -1.8646684885025024,
-      "logits/rejected": -1.853882074356079,
-      "logps/chosen": -33.567169189453125,
-      "logps/rejected": -35.43851852416992,
-      "loss": 0.5009,
-      "rewards/accuracies": 0.44999998807907104,
-      "rewards/chosen": -0.0019382357131689787,
-      "rewards/margins": -0.00470335315912962,
-      "rewards/rejected": 0.0027651176787912846,
+      "logits/chosen": -1.8643096685409546,
+      "logits/rejected": -1.853522539138794,
+      "logps/chosen": -33.55935287475586,
+      "logps/rejected": -35.42003631591797,
+      "loss": 0.5018,
+      "rewards/accuracies": 0.42500001192092896,
+      "rewards/chosen": -0.0005626566708087921,
+      "rewards/margins": -0.010254684835672379,
+      "rewards/rejected": 0.009692028164863586,
       "step": 50
     },
     {
       "epoch": 0.16,
       "learning_rate": 4.954691471941118e-07,
-      "logits/chosen": -1.9455400705337524,
-      "logits/rejected": -1.947479486465454,
-      "logps/chosen": -32.57770919799805,
-      "logps/rejected": -33.181541442871094,
-      "loss": 0.4989,
-      "rewards/accuracies": 0.5625,
-      "rewards/chosen": 0.004494256805628538,
-      "rewards/margins": 0.006526687648147345,
-      "rewards/rejected": -0.00203243107534945,
+      "logits/chosen": -1.9451143741607666,
+      "logits/rejected": -1.9470453262329102,
+      "logps/chosen": -32.59963607788086,
+      "logps/rejected": -33.16991424560547,
+      "loss": 0.5,
+      "rewards/accuracies": 0.48750001192092896,
+      "rewards/chosen": 0.00016471892013214529,
+      "rewards/margins": -0.0002745899255387485,
+      "rewards/rejected": 0.00043930913670919836,
       "step": 60
     },
     {
       "epoch": 0.18,
       "learning_rate": 4.901618883413548e-07,
-      "logits/chosen": -2.079427719116211,
-      "logits/rejected": -2.084414482116699,
-      "logps/chosen": -33.99236297607422,
-      "logps/rejected": -36.600887298583984,
-      "loss": 0.4989,
-      "rewards/accuracies": 0.612500011920929,
-      "rewards/chosen": -0.0007750942604616284,
-      "rewards/margins": 0.005627653561532497,
-      "rewards/rejected": -0.006402746774256229,
+      "logits/chosen": -2.080049991607666,
+      "logits/rejected": -2.085019588470459,
+      "logps/chosen": -33.990013122558594,
+      "logps/rejected": -36.58393478393555,
+      "loss": 0.4992,
+      "rewards/accuracies": 0.4749999940395355,
+      "rewards/chosen": -0.00045763104571960866,
+      "rewards/margins": 0.004060628358274698,
+      "rewards/rejected": -0.004518259782344103,
       "step": 70
     },
     {
       "epoch": 0.21,
       "learning_rate": 4.828760511501322e-07,
-      "logits/chosen": -1.9420673847198486,
-      "logits/rejected": -1.9452102184295654,
-      "logps/chosen": -34.41423416137695,
-      "logps/rejected": -34.5662727355957,
-      "loss": 0.5004,
-      "rewards/accuracies": 0.44999998807907104,
-      "rewards/chosen": 0.0027157063595950603,
-      "rewards/margins": -0.0025114950258284807,
-      "rewards/rejected": 0.00522720068693161,
+      "logits/chosen": -1.943108320236206,
+      "logits/rejected": -1.9462664127349854,
+      "logps/chosen": -34.41640090942383,
+      "logps/rejected": -34.587799072265625,
+      "loss": 0.4998,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": 0.0034226563293486834,
+      "rewards/margins": 0.002040152670815587,
+      "rewards/rejected": 0.0013825036585330963,
       "step": 80
     },
     {
       "epoch": 0.23,
       "learning_rate": 4.736716601303429e-07,
-      "logits/chosen": -1.951348066329956,
-      "logits/rejected": -1.9558594226837158,
-      "logps/chosen": -32.46399688720703,
-      "logps/rejected": -32.344329833984375,
-      "loss": 0.5004,
-      "rewards/accuracies": 0.4749999940395355,
-      "rewards/chosen": 0.0006849506171420217,
-      "rewards/margins": -0.0022309008054435253,
-      "rewards/rejected": 0.002915852004662156,
+      "logits/chosen": -1.9505999088287354,
+      "logits/rejected": -1.955100655555725,
+      "logps/chosen": -32.46385955810547,
+      "logps/rejected": -32.354026794433594,
+      "loss": 0.4998,
+      "rewards/accuracies": 0.48750001192092896,
+      "rewards/chosen": 0.0010680232662707567,
+      "rewards/margins": -0.00039678759640082717,
+      "rewards/rejected": 0.0014648116193711758,
       "step": 90
     },
     {
       "epoch": 0.26,
       "learning_rate": 4.62624545834521e-07,
-      "logits/chosen": -2.0500051975250244,
-      "logits/rejected": -2.0480096340179443,
-      "logps/chosen": -32.25367736816406,
-      "logps/rejected": -31.2783260345459,
-      "loss": 0.5,
-      "rewards/accuracies": 0.48750001192092896,
-      "rewards/chosen": -0.002567791845649481,
-      "rewards/margins": -0.00014422755339182913,
-      "rewards/rejected": -0.0024235642049461603,
+      "logits/chosen": -2.0487732887268066,
+      "logits/rejected": -2.0467641353607178,
+      "logps/chosen": -32.22932052612305,
+      "logps/rejected": -31.277706146240234,
+      "loss": 0.4987,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": 0.003454477759078145,
+      "rewards/margins": 0.0069038658402860165,
+      "rewards/rejected": -0.003449387848377228,
       "step": 100
     },
     {
       "epoch": 0.26,
-      "eval_logits/chosen": -2.244023323059082,
-      "eval_logits/rejected": -2.2391417026519775,
-      "eval_logps/chosen": -34.02798080444336,
-      "eval_logps/rejected": -37.506290435791016,
-      "eval_loss": 0.500300407409668,
-      "eval_rewards/accuracies": 0.49335551261901855,
-      "eval_rewards/chosen": 0.0013138726353645325,
-      "eval_rewards/margins": -0.0007514380267821252,
-      "eval_rewards/rejected": 0.0020653100218623877,
-      "eval_runtime": 146.2282,
-      "eval_samples_per_second": 2.346,
+      "eval_logits/chosen": -2.2437498569488525,
+      "eval_logits/rejected": -2.238868474960327,
+      "eval_logps/chosen": -34.018192291259766,
+      "eval_logps/rejected": -37.51003646850586,
+      "eval_loss": 0.49943608045578003,
+      "eval_rewards/accuracies": 0.5074750781059265,
+      "eval_rewards/chosen": 0.0049087232910096645,
+      "eval_rewards/margins": 0.00293393200263381,
+      "eval_rewards/rejected": 0.001974791055545211,
+      "eval_runtime": 146.1062,
+      "eval_samples_per_second": 2.348,
       "eval_steps_per_second": 0.294,
       "step": 100
     },
     {
       "epoch": 0.29,
       "learning_rate": 4.4982572012636904e-07,
-      "logits/chosen": -2.0061328411102295,
-      "logits/rejected": -2.0037178993225098,
-      "logps/chosen": -33.24314880371094,
-      "logps/rejected": -34.023292541503906,
-      "loss": 0.5003,
-      "rewards/accuracies": 0.4375,
-      "rewards/chosen": 0.0001343002513749525,
-      "rewards/margins": -0.001081160269677639,
-      "rewards/rejected": 0.0012154604773968458,
+      "logits/chosen": -2.005328893661499,
+      "logits/rejected": -2.0029098987579346,
+      "logps/chosen": -33.22368240356445,
+      "logps/rejected": -34.016380310058594,
+      "loss": 0.4995,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": 0.006039897445589304,
+      "rewards/margins": 0.0021425553131848574,
+      "rewards/rejected": 0.0038973423652350903,
       "step": 110
     },
     {
       "epoch": 0.31,
       "learning_rate": 4.353806263777677e-07,
-      "logits/chosen": -2.0173094272613525,
-      "logits/rejected": -2.008920431137085,
-      "logps/chosen": -32.45183563232422,
-      "logps/rejected": -32.17412185668945,
-      "loss": 0.5002,
-      "rewards/accuracies": 0.48750001192092896,
-      "rewards/chosen": -0.0013863157946616411,
-      "rewards/margins": 0.0011368464911356568,
-      "rewards/rejected": -0.002523162867873907,
+      "logits/chosen": -2.0170254707336426,
+      "logits/rejected": -2.0086662769317627,
+      "logps/chosen": -32.444522857666016,
+      "logps/rejected": -32.190677642822266,
+      "loss": 0.4978,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.00011485694267321378,
+      "rewards/margins": 0.008865321055054665,
+      "rewards/rejected": -0.008750463835895061,
       "step": 120
     },
     {
       "epoch": 0.34,
       "learning_rate": 4.194082707715275e-07,
-      "logits/chosen": -2.0478129386901855,
-      "logits/rejected": -2.0397789478302,
-      "logps/chosen": -30.514944076538086,
-      "logps/rejected": -32.053070068359375,
-      "loss": 0.5012,
-      "rewards/accuracies": 0.5,
-      "rewards/chosen": -0.007845849730074406,
-      "rewards/margins": -0.007833347655832767,
-      "rewards/rejected": -1.2502726349339355e-05,
+      "logits/chosen": -2.0464835166931152,
+      "logits/rejected": -2.0384185314178467,
+      "logps/chosen": -30.49222183227539,
+      "logps/rejected": -32.05754852294922,
+      "loss": 0.5009,
+      "rewards/accuracies": 0.4375,
+      "rewards/chosen": -0.004951108247041702,
+      "rewards/margins": -0.0035899754147976637,
+      "rewards/rejected": -0.0013611322501674294,
       "step": 130
     },
     {
       "epoch": 0.36,
       "learning_rate": 4.020402418666621e-07,
-      "logits/chosen": -1.978299856185913,
-      "logits/rejected": -1.9885809421539307,
-      "logps/chosen": -31.413021087646484,
-      "logps/rejected": -32.54629135131836,
-      "loss": 0.4995,
-      "rewards/accuracies": 0.48750001192092896,
-      "rewards/chosen": 0.0013283130247145891,
-      "rewards/margins": 0.0036107772029936314,
-      "rewards/rejected": -0.002282463712617755,
+      "logits/chosen": -1.9772565364837646,
+      "logits/rejected": -1.987534761428833,
+      "logps/chosen": -31.39870262145996,
+      "logps/rejected": -32.54944610595703,
+      "loss": 0.4984,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": 0.0062883696518838406,
+      "rewards/margins": 0.01065803598612547,
+      "rewards/rejected": -0.004369667265564203,
       "step": 140
     },
     {
       "epoch": 0.39,
       "learning_rate": 3.8341962650351185e-07,
-      "logits/chosen": -1.8922052383422852,
-      "logits/rejected": -1.8932926654815674,
-      "logps/chosen": -34.210052490234375,
-      "logps/rejected": -34.78533935546875,
-      "loss": 0.4998,
-      "rewards/accuracies": 0.48750001192092896,
-      "rewards/chosen": -0.003472150769084692,
-      "rewards/margins": 0.0021447453182190657,
-      "rewards/rejected": -0.005616897251456976,
+      "logits/chosen": -1.8916442394256592,
+      "logits/rejected": -1.8927291631698608,
+      "logps/chosen": -34.21247863769531,
+      "logps/rejected": -34.76598358154297,
+      "loss": 0.501,
+      "rewards/accuracies": 0.4124999940395355,
+      "rewards/chosen": -0.005936207715421915,
+      "rewards/margins": -0.0033174168784171343,
+      "rewards/rejected": -0.0026187908370047808,
       "step": 150
     },
     {
       "epoch": 0.42,
       "learning_rate": 3.636998309800572e-07,
-      "logits/chosen": -1.9435640573501587,
-      "logits/rejected": -1.9400733709335327,
-      "logps/chosen": -36.18402862548828,
-      "logps/rejected": -32.75020217895508,
-      "loss": 0.4998,
-      "rewards/accuracies": 0.48750001192092896,
-      "rewards/chosen": -0.0024705410469323397,
-      "rewards/margins": 0.0030191238038241863,
-      "rewards/rejected": -0.00548966508358717,
+      "logits/chosen": -1.9432052373886108,
+      "logits/rejected": -1.9397189617156982,
+      "logps/chosen": -36.16191864013672,
+      "logps/rejected": -32.74675750732422,
+      "loss": 0.4985,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.002928206929937005,
+      "rewards/margins": 0.010129809379577637,
+      "rewards/rejected": -0.007201602216809988,
       "step": 160
     },
     {
       "epoch": 0.44,
       "learning_rate": 3.430433172111807e-07,
-      "logits/chosen": -2.043527603149414,
-      "logits/rejected": -2.0361220836639404,
-      "logps/chosen": -33.81038284301758,
-      "logps/rejected": -31.369457244873047,
-      "loss": 0.4997,
-      "rewards/accuracies": 0.550000011920929,
-      "rewards/chosen": -0.002503907773643732,
-      "rewards/margins": 0.0010178396478295326,
-      "rewards/rejected": -0.0035217474214732647,
+      "logits/chosen": -2.042198896408081,
+      "logits/rejected": -2.034799098968506,
+      "logps/chosen": -33.776123046875,
+      "logps/rejected": -31.36188316345215,
+      "loss": 0.4979,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": 0.0065220496617257595,
+      "rewards/margins": 0.009532475844025612,
+      "rewards/rejected": -0.003010427113622427,
       "step": 170
     },
     {
       "epoch": 0.47,
       "learning_rate": 3.216202642830543e-07,
-      "logits/chosen": -2.0494205951690674,
-      "logits/rejected": -2.0546982288360596,
-      "logps/chosen": -32.51402282714844,
-      "logps/rejected": -32.5015983581543,
-      "loss": 0.4982,
-      "rewards/accuracies": 0.625,
-      "rewards/chosen": 0.0061522433534264565,
-      "rewards/margins": 0.009443378075957298,
-      "rewards/rejected": -0.003291133791208267,
+      "logits/chosen": -2.0474331378936768,
+      "logits/rejected": -2.0527079105377197,
+      "logps/chosen": -32.522254943847656,
+      "logps/rejected": -32.51903533935547,
+      "loss": 0.497,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.006759388837963343,
+      "rewards/margins": 0.016926825046539307,
+      "rewards/rejected": -0.010167436674237251,
       "step": 180
     },
     {
       "epoch": 0.49,
       "learning_rate": 2.9960716642946403e-07,
-      "logits/chosen": -2.0493359565734863,
-      "logits/rejected": -2.046536922454834,
-      "logps/chosen": -31.47686195373535,
-      "logps/rejected": -31.326452255249023,
-      "loss": 0.4991,
-      "rewards/accuracies": 0.5874999761581421,
-      "rewards/chosen": 0.002418341813609004,
-      "rewards/margins": 0.004066127352416515,
-      "rewards/rejected": -0.0016477858880534768,
+      "logits/chosen": -2.0483100414276123,
+      "logits/rejected": -2.0455245971679688,
+      "logps/chosen": -31.49411964416504,
+      "logps/rejected": -31.317398071289062,
+      "loss": 0.5003,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -0.0015499559231102467,
+      "rewards/margins": -0.0017946911975741386,
+      "rewards/rejected": 0.00024473536177538335,
       "step": 190
     },
     {
       "epoch": 0.52,
       "learning_rate": 2.771853789806683e-07,
-      "logits/chosen": -1.9191436767578125,
-      "logits/rejected": -1.9238201379776,
-      "logps/chosen": -31.60672378540039,
-      "logps/rejected": -32.80228805541992,
-      "loss": 0.4994,
-      "rewards/accuracies": 0.5249999761581421,
-      "rewards/chosen": -8.77844140632078e-05,
-      "rewards/margins": 0.002183270873501897,
-      "rewards/rejected": -0.0022710547782480717,
+      "logits/chosen": -1.9187097549438477,
+      "logits/rejected": -1.9233875274658203,
+      "logps/chosen": -31.572490692138672,
+      "logps/rejected": -32.78968811035156,
+      "loss": 0.4979,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": 0.01013648696243763,
+      "rewards/margins": 0.009763057343661785,
+      "rewards/rejected": 0.0003734306083060801,
       "step": 200
     },
     {
       "epoch": 0.52,
-      "eval_logits/chosen": -2.2449820041656494,
-      "eval_logits/rejected": -2.2400975227355957,
-      "eval_logps/chosen": -34.03920364379883,
-      "eval_logps/rejected": -37.51057434082031,
-      "eval_loss": 0.5004644989967346,
-      "eval_rewards/accuracies": 0.5020764470100403,
-      "eval_rewards/chosen": -0.0009311072644777596,
-      "eval_rewards/margins": -0.002139872871339321,
-      "eval_rewards/rejected": 0.0012087655486539006,
-      "eval_runtime": 146.0512,
-      "eval_samples_per_second": 2.348,
-      "eval_steps_per_second": 0.294,
+      "eval_logits/chosen": -2.243237257003784,
+      "eval_logits/rejected": -2.238358736038208,
+      "eval_logps/chosen": -34.021446228027344,
+      "eval_logps/rejected": -37.5062255859375,
+      "eval_loss": 0.499795526266098,
+      "eval_rewards/accuracies": 0.4862956702709198,
+      "eval_rewards/chosen": 0.003931538667529821,
+      "eval_rewards/margins": 0.0008117269026115537,
+      "eval_rewards/rejected": 0.003119812114164233,
+      "eval_runtime": 145.9975,
+      "eval_samples_per_second": 2.349,
+      "eval_steps_per_second": 0.295,
       "step": 200
     },
     {
       "epoch": 0.55,
       "learning_rate": 2.5453962426402e-07,
-      "logits/chosen": -2.03255033493042,
-      "logits/rejected": -2.0432262420654297,
-      "logps/chosen": -31.951080322265625,
-      "logps/rejected": -33.896484375,
-      "loss": 0.498,
-      "rewards/accuracies": 0.5874999761581421,
-      "rewards/chosen": 0.004044383764266968,
-      "rewards/margins": 0.010801524855196476,
-      "rewards/rejected": -0.006757141090929508,
+      "logits/chosen": -2.0315887928009033,
+      "logits/rejected": -2.042264461517334,
+      "logps/chosen": -31.964473724365234,
+      "logps/rejected": -33.852508544921875,
+      "loss": 0.5002,
+      "rewards/accuracies": 0.4375,
+      "rewards/chosen": 0.0020490700844675303,
+      "rewards/margins": -0.001008716062642634,
+      "rewards/rejected": 0.003057786263525486,
       "step": 210
     },
     {
       "epoch": 0.57,
       "learning_rate": 2.318564697655179e-07,
-      "logits/chosen": -1.9258638620376587,
-      "logits/rejected": -1.9407155513763428,
-      "logps/chosen": -30.115543365478516,
-      "logps/rejected": -31.575191497802734,
-      "loss": 0.4992,
-      "rewards/accuracies": 0.6000000238418579,
-      "rewards/chosen": -0.0008008191362023354,
-      "rewards/margins": 0.002602284774184227,
-      "rewards/rejected": -0.003403103444725275,
+      "logits/chosen": -1.925210952758789,
+      "logits/rejected": -1.9400602579116821,
+      "logps/chosen": -30.0924129486084,
+      "logps/rejected": -31.5645694732666,
+      "loss": 0.4987,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.0057382904924452305,
+      "rewards/margins": 0.0076560890302062035,
+      "rewards/rejected": -0.0019177987705916166,
       "step": 220
     },
     {
       "epoch": 0.6,
       "learning_rate": 2.093227910899832e-07,
-      "logits/chosen": -1.983538269996643,
-      "logits/rejected": -1.9875080585479736,
-      "logps/chosen": -33.397071838378906,
-      "logps/rejected": -31.56662940979004,
-      "loss": 0.4985,
+      "logits/chosen": -1.98291015625,
+      "logits/rejected": -1.9868860244750977,
+      "logps/chosen": -33.397972106933594,
+      "logps/rejected": -31.54607582092285,
+      "loss": 0.4988,
       "rewards/accuracies": 0.550000011920929,
-      "rewards/chosen": 0.004079356789588928,
-      "rewards/margins": 0.007544734515249729,
-      "rewards/rejected": -0.0034653779584914446,
+      "rewards/chosen": 0.005849479231983423,
+      "rewards/margins": 0.004881345666944981,
+      "rewards/rejected": 0.0009681343799456954,
       "step": 230
     },
     {
       "epoch": 0.62,
       "learning_rate": 1.8712423238279356e-07,
-      "logits/chosen": -1.9831326007843018,
-      "logits/rejected": -1.9611790180206299,
-      "logps/chosen": -34.164363861083984,
-      "logps/rejected": -34.951324462890625,
-      "loss": 0.5004,
-      "rewards/accuracies": 0.4749999940395355,
-      "rewards/chosen": -0.0021236296743154526,
-      "rewards/margins": -0.0020176086109131575,
-      "rewards/rejected": -0.00010602096881484613,
+      "logits/chosen": -1.9833847284317017,
+      "logits/rejected": -1.9614006280899048,
+      "logps/chosen": -34.151329040527344,
+      "logps/rejected": -34.96162414550781,
+      "loss": 0.4995,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": 0.0007256423123180866,
+      "rewards/margins": 0.0039748698472976685,
+      "rewards/rejected": -0.003249228000640869,
       "step": 240
     },
     {
       "epoch": 0.65,
       "learning_rate": 1.654436768970182e-07,
-      "logits/chosen": -2.0248496532440186,
-      "logits/rejected": -2.021538734436035,
-      "logps/chosen": -32.92078399658203,
-      "logps/rejected": -36.21001434326172,
-      "loss": 0.5005,
-      "rewards/accuracies": 0.4625000059604645,
-      "rewards/chosen": 0.0008573724189773202,
-      "rewards/margins": -0.002124571241438389,
-      "rewards/rejected": 0.002981943776831031,
+      "logits/chosen": -2.0242693424224854,
+      "logits/rejected": -2.0209662914276123,
+      "logps/chosen": -32.91448974609375,
+      "logps/rejected": -36.216251373291016,
+      "loss": 0.4995,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": 0.003173952456563711,
+      "rewards/margins": 0.0005722272908315063,
+      "rewards/rejected": 0.0026017245836555958,
       "step": 250
     },
     {
       "epoch": 0.68,
       "learning_rate": 1.444597403062196e-07,
-      "logits/chosen": -1.8912862539291382,
-      "logits/rejected": -1.8888485431671143,
-      "logps/chosen": -34.20059585571289,
-      "logps/rejected": -35.51679992675781,
-      "loss": 0.5001,
-      "rewards/accuracies": 0.48750001192092896,
-      "rewards/chosen": -0.0024391734041273594,
-      "rewards/margins": -3.7313438951969147e-06,
-      "rewards/rejected": -0.002435441594570875,
+      "logits/chosen": -1.890960931777954,
+      "logits/rejected": -1.8885114192962646,
+      "logps/chosen": -34.190608978271484,
+      "logps/rejected": -35.51725769042969,
+      "loss": 0.4997,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -0.0006630702991969883,
+      "rewards/margins": 0.0031278349924832582,
+      "rewards/rejected": -0.003790905699133873,
       "step": 260
     },
     {
       "epoch": 0.7,
       "learning_rate": 1.2434529917578887e-07,
-      "logits/chosen": -1.876117467880249,
-      "logits/rejected": -1.8735599517822266,
-      "logps/chosen": -34.39899444580078,
-      "logps/rejected": -31.73566246032715,
-      "loss": 0.5014,
-      "rewards/accuracies": 0.4000000059604645,
-      "rewards/chosen": -0.00027935029356740415,
-      "rewards/margins": -0.007413160987198353,
-      "rewards/rejected": 0.007133810315281153,
+      "logits/chosen": -1.875756025314331,
+      "logits/rejected": -1.8732185363769531,
+      "logps/chosen": -34.378639221191406,
+      "logps/rejected": -31.751827239990234,
+      "loss": 0.4999,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": 0.005688496865332127,
+      "rewards/margins": -0.00016186293214559555,
+      "rewards/rejected": 0.005850359331816435,
       "step": 270
     },
     {
       "epoch": 0.73,
       "learning_rate": 1.0526606671603521e-07,
-      "logits/chosen": -1.9805666208267212,
-      "logits/rejected": -1.9699329137802124,
-      "logps/chosen": -35.328285217285156,
-      "logps/rejected": -31.84103012084961,
-      "loss": 0.4988,
-      "rewards/accuracies": 0.44999998807907104,
-      "rewards/chosen": 0.005670648999512196,
-      "rewards/margins": 0.0056780558079481125,
-      "rewards/rejected": -7.406389158859383e-06,
+      "logits/chosen": -1.9793775081634521,
+      "logits/rejected": -1.9687445163726807,
+      "logps/chosen": -35.337181091308594,
+      "logps/rejected": -31.856109619140625,
+      "loss": 0.4979,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": 0.005838591605424881,
+      "rewards/margins": 0.010373086668550968,
+      "rewards/rejected": -0.0045344955287873745,
       "step": 280
     },
     {
       "epoch": 0.75,
       "learning_rate": 8.737922755071453e-08,
-      "logits/chosen": -2.0757369995117188,
-      "logits/rejected": -2.06070613861084,
-      "logps/chosen": -30.9174861907959,
-      "logps/rejected": -32.63935470581055,
-      "loss": 0.5004,
-      "rewards/accuracies": 0.44999998807907104,
-      "rewards/chosen": 0.001327360630966723,
-      "rewards/margins": -0.0011682776967063546,
-      "rewards/rejected": 0.0024956378620117903,
+      "logits/chosen": -2.0756278038024902,
+      "logits/rejected": -2.0606112480163574,
+      "logps/chosen": -30.936153411865234,
+      "logps/rejected": -32.660118103027344,
+      "loss": 0.5002,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -0.003609871957451105,
+      "rewards/margins": -0.001124321250244975,
+      "rewards/rejected": -0.0024855495430529118,
       "step": 290
     },
     {
       "epoch": 0.78,
       "learning_rate": 7.08321427484816e-08,
-      "logits/chosen": -1.947251319885254,
-      "logits/rejected": -1.9447133541107178,
-      "logps/chosen": -32.908634185791016,
-      "logps/rejected": -30.82659912109375,
-      "loss": 0.499,
-      "rewards/accuracies": 0.512499988079071,
-      "rewards/chosen": 0.0036955091636627913,
-      "rewards/margins": 0.005653515458106995,
-      "rewards/rejected": -0.001958005130290985,
+      "logits/chosen": -1.9465348720550537,
+      "logits/rejected": -1.9439990520477295,
+      "logps/chosen": -32.864601135253906,
+      "logps/rejected": -30.782527923583984,
+      "loss": 0.4978,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": 0.01875343546271324,
+      "rewards/margins": 0.008469512686133385,
+      "rewards/rejected": 0.010283923707902431,
       "step": 300
     },
     {
       "epoch": 0.78,
-      "eval_logits/chosen": -2.2445528507232666,
-      "eval_logits/rejected": -2.23966383934021,
-      "eval_logps/chosen": -34.04175567626953,
-      "eval_logps/rejected": -37.499141693115234,
-      "eval_loss": 0.5009724497795105,
-      "eval_rewards/accuracies": 0.4431063234806061,
-      "eval_rewards/chosen": -0.0014407250564545393,
-      "eval_rewards/margins": -0.004937068559229374,
-      "eval_rewards/rejected": 0.003496343968436122,
-      "eval_runtime": 145.9859,
-      "eval_samples_per_second": 2.35,
+      "eval_logits/chosen": -2.243088722229004,
+      "eval_logits/rejected": -2.2382218837738037,
+      "eval_logps/chosen": -34.02065658569336,
+      "eval_logps/rejected": -37.50462341308594,
+      "eval_loss": 0.4998457729816437,
+      "eval_rewards/accuracies": 0.5137043595314026,
+      "eval_rewards/chosen": 0.004168376792222261,
+      "eval_rewards/margins": 0.0005690783145837486,
+      "eval_rewards/rejected": 0.0035992988850921392,
+      "eval_runtime": 145.9968,
+      "eval_samples_per_second": 2.349,
       "eval_steps_per_second": 0.295,
       "step": 300
     },
     {
       "epoch": 0.81,
       "learning_rate": 5.576113578589034e-08,
-      "logits/chosen": -1.9292316436767578,
-      "logits/rejected": -1.9259681701660156,
-      "logps/chosen": -31.583276748657227,
-      "logps/rejected": -33.727840423583984,
+      "logits/chosen": -1.928755521774292,
+      "logits/rejected": -1.9254906177520752,
+      "logps/chosen": -31.5792236328125,
+      "logps/rejected": -33.729896545410156,
       "loss": 0.4993,
-      "rewards/accuracies": 0.512499988079071,
-      "rewards/chosen": 0.004015020560473204,
-      "rewards/margins": 0.001152882701717317,
-      "rewards/rejected": 0.002862137509509921,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.0072378418408334255,
+      "rewards/margins": 0.0035611852072179317,
+      "rewards/rejected": 0.0036766561679542065,
       "step": 310
     },
     {
       "epoch": 0.83,
       "learning_rate": 4.229036944380912e-08,
-      "logits/chosen": -1.9808372259140015,
-      "logits/rejected": -1.9685356616973877,
-      "logps/chosen": -34.57278823852539,
-      "logps/rejected": -33.57910919189453,
-      "loss": 0.498,
-      "rewards/accuracies": 0.5874999761581421,
-      "rewards/chosen": 0.004657471086829901,
-      "rewards/margins": 0.01271037757396698,
-      "rewards/rejected": -0.008052906021475792,
+      "logits/chosen": -1.9806935787200928,
+      "logits/rejected": -1.968379020690918,
+      "logps/chosen": -34.56597900390625,
+      "logps/rejected": -33.5899658203125,
+      "loss": 0.4957,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": 0.009028470143675804,
+      "rewards/margins": 0.024366382509469986,
+      "rewards/rejected": -0.015337912365794182,
       "step": 320
     },
     {
       "epoch": 0.86,
       "learning_rate": 3.053082288996112e-08,
-      "logits/chosen": -2.0161705017089844,
-      "logits/rejected": -2.0147035121917725,
-      "logps/chosen": -33.47340393066406,
-      "logps/rejected": -32.46953582763672,
-      "loss": 0.4998,
-      "rewards/accuracies": 0.48750001192092896,
-      "rewards/chosen": 0.0024778605438768864,
-      "rewards/margins": 0.00046821607975289226,
-      "rewards/rejected": 0.0020096441730856895,
+      "logits/chosen": -2.0154762268066406,
+      "logits/rejected": -2.014007091522217,
+      "logps/chosen": -33.468040466308594,
+      "logps/rejected": -32.471534729003906,
+      "loss": 0.4996,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": 0.005325844045728445,
+      "rewards/margins": 0.0029113669879734516,
+      "rewards/rejected": 0.0024144775234162807,
       "step": 330
     },
     {
       "epoch": 0.88,
       "learning_rate": 2.05793773749158e-08,
-      "logits/chosen": -2.1033377647399902,
-      "logits/rejected": -2.0875496864318848,
-      "logps/chosen": -34.152557373046875,
-      "logps/rejected": -33.08795166015625,
-      "loss": 0.5003,
-      "rewards/accuracies": 0.5,
-      "rewards/chosen": 0.005367305129766464,
-      "rewards/margins": -0.0006489218212664127,
-      "rewards/rejected": 0.006016227416694164,
+      "logits/chosen": -2.102926254272461,
+      "logits/rejected": -2.0871331691741943,
+      "logps/chosen": -34.16908645629883,
+      "logps/rejected": -33.08391189575195,
+      "loss": 0.5017,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": 0.003090888261795044,
+      "rewards/margins": -0.007146535906940699,
+      "rewards/rejected": 0.010237423703074455,
       "step": 340
     },
     {
       "epoch": 0.91,
       "learning_rate": 1.251801807404168e-08,
-      "logits/chosen": -1.9754228591918945,
-      "logits/rejected": -1.974477767944336,
-      "logps/chosen": -33.24272537231445,
-      "logps/rejected": -32.46410369873047,
+      "logits/chosen": -1.974313735961914,
+      "logits/rejected": -1.9733721017837524,
+      "logps/chosen": -33.25641632080078,
+      "logps/rejected": -32.458290100097656,
       "loss": 0.4986,
-      "rewards/accuracies": 0.5375000238418579,
-      "rewards/chosen": 0.007874632254242897,
-      "rewards/margins": 0.007666703313589096,
-      "rewards/rejected": 0.00020792819850612432,
+      "rewards/accuracies": 0.48750001192092896,
+      "rewards/chosen": 0.007705795578658581,
+      "rewards/margins": 0.005649130791425705,
+      "rewards/rejected": 0.0020566643215715885,
       "step": 350
     },
     {
       "epoch": 0.94,
       "learning_rate": 6.41315865106129e-09,
-      "logits/chosen": -1.9320882558822632,
-      "logits/rejected": -1.9424550533294678,
-      "logps/chosen": -32.212371826171875,
-      "logps/rejected": -35.286354064941406,
-      "loss": 0.5003,
-      "rewards/accuracies": 0.4375,
-      "rewards/chosen": -0.0015360517427325249,
-      "rewards/margins": -0.002017115242779255,
-      "rewards/rejected": 0.00048106274334713817,
+      "logits/chosen": -1.9302194118499756,
+      "logits/rejected": -1.940574288368225,
+      "logps/chosen": -32.19397735595703,
+      "logps/rejected": -35.28559875488281,
+      "loss": 0.4996,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": 0.003214403986930847,
+      "rewards/margins": 0.0022677627857774496,
+      "rewards/rejected": 0.0009466406190767884,
       "step": 360
     },
     {
       "epoch": 0.96,
       "learning_rate": 2.3150941078050324e-09,
-      "logits/chosen": -2.070864200592041,
-      "logits/rejected": -2.0643177032470703,
-      "logps/chosen": -33.652870178222656,
-      "logps/rejected": -29.220972061157227,
-      "loss": 0.5004,
-      "rewards/accuracies": 0.4625000059604645,
-      "rewards/chosen": -0.00018037435074802488,
-      "rewards/margins": -0.001271072425879538,
-      "rewards/rejected": 0.001090698060579598,
+      "logits/chosen": -2.0694031715393066,
+      "logits/rejected": -2.0628397464752197,
+      "logps/chosen": -33.639854431152344,
+      "logps/rejected": -29.21550941467285,
+      "loss": 0.4998,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.0036338320933282375,
+      "rewards/margins": 0.00035826730891130865,
+      "rewards/rejected": 0.0032755639404058456,
       "step": 370
     },
     {
       "epoch": 0.99,
       "learning_rate": 2.575864278703266e-10,
-      "logits/chosen": -1.9306039810180664,
-      "logits/rejected": -1.932790756225586,
-      "logps/chosen": -34.26443862915039,
-      "logps/rejected": -30.895788192749023,
-      "loss": 0.5001,
-      "rewards/accuracies": 0.44999998807907104,
-      "rewards/chosen": -0.005078143440186977,
-      "rewards/margins": -0.00044635325320996344,
-      "rewards/rejected": -0.004631790332496166,
+      "logits/chosen": -1.9289796352386475,
+      "logits/rejected": -1.9311374425888062,
+      "logps/chosen": -34.22175979614258,
+      "logps/rejected": -30.88715171813965,
+      "loss": 0.4984,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": 0.005186386872082949,
+      "rewards/margins": 0.009542430751025677,
+      "rewards/rejected": -0.004356043878942728,
       "step": 380
     },
     {
       "epoch": 1.0,
       "step": 385,
       "total_flos": 0.0,
-      "train_loss": 0.4997496530607149,
-      "train_runtime": 3257.6696,
-      "train_samples_per_second": 0.945,
+      "train_loss": 0.49920677581390777,
+      "train_runtime": 3256.0166,
+      "train_samples_per_second": 0.946,
       "train_steps_per_second": 0.118
     }
   ],