Model save

README.md

@@ -1,13 +1,10 @@
 ---
 library_name: peft
 tags:
-- alignment-handbook
 - trl
 - dpo
 - generated_from_trainer
 base_model: NbAiLab/nb-gpt-j-6B-v2
-datasets:
-- hugodk-sch/aftonposten_title_prefs
 model-index:
 - name: aftonposten-6b-align-scan
   results: []
@@ -18,17 +15,17 @@ should probably proofread and complete it, then remove this comment. -->
 
 # aftonposten-6b-align-scan
 
-This model is a fine-tuned version of [data/ap-gpt-j-6b-sft-qlora-04-08](https://huggingface.co/data/ap-gpt-j-6b-sft-qlora-04-08) on the hugodk-sch/aftonposten_title_prefs dataset.
+This model is a fine-tuned version of [NbAiLab/nb-gpt-j-6B-v2](https://huggingface.co/NbAiLab/nb-gpt-j-6B-v2) on an unknown dataset.
 It achieves the following results on the evaluation set:
-- Loss: 2.7902
-- Rewards/chosen: -0.0060
-- Rewards/rejected: -0.0171
-- Rewards/accuracies: 0.5602
-- Rewards/margins: 0.0111
-- Logps/rejected: -37.5735
-- Logps/chosen: -34.0545
-- Logits/rejected: -2.2247
-- Logits/chosen: -2.2295
+- Loss: 0.9891
+- Rewards/chosen: -0.0205
+- Rewards/rejected: -0.0313
+- Rewards/accuracies: 0.5183
+- Rewards/margins: 0.0108
+- Logps/rejected: -37.6211
+- Logps/chosen: -34.1029
+- Logits/rejected: -2.2202
+- Logits/chosen: -2.2250
 
 ## Model description
 
@@ -63,9 +60,9 @@ The following hyperparameters were used during training:
 
 | Training Loss | Epoch | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
 |:-------------:|:-----:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
-| 2.59          | 0.26  | 100  | 2.8958          | 0.0029         | 0.0052           | 0.4730             | -0.0024         | -37.4993       | -34.0250     | -2.2305         | -2.2353       |
-| 2.2795        | 0.52  | 200  | 2.8012          | -0.0060        | -0.0145          | 0.5278             | 0.0085          | -37.5651       | -34.0545     | -2.2290         | -2.2339       |
-| 1.7902        | 0.78  | 300  | 2.7585          | -0.0030        | -0.0167          | 0.5748             | 0.0137          | -37.5724       | -34.0446     | -2.2245         | -2.2294       |
+| 0.956         | 0.26  | 100  | 0.9948          | -0.0045        | -0.0098          | 0.5631             | 0.0053          | -37.5492       | -34.0495     | -2.2276         | -2.2325       |
+| 0.8922        | 0.52  | 200  | 0.9876          | -0.0094        | -0.0218          | 0.5303             | 0.0124          | -37.5894       | -34.0659     | -2.2231         | -2.2279       |
+| 0.8033        | 0.78  | 300  | 0.9891          | -0.0205        | -0.0313          | 0.5183             | 0.0108          | -37.6211       | -34.1029     | -2.2202         | -2.2250       |
 
 
 ### Framework versions

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:997b36b01dd608f35abda62e68f900f81f4c27b6fbdb3117bd16ca385e56da53
+oid sha256:15df4126b86e5931a293f8a29058c83956e64751dc461efe761ddfc8ddf1affc
 size 176183216

all_results.json

@@ -1,20 +1,7 @@
 {
     "epoch": 1.0,
-    "eval_logits/chosen": -2.2295007705688477,
-    "eval_logits/rejected": -2.2246556282043457,
-    "eval_logps/chosen": -34.054466247558594,
-    "eval_logps/rejected": -37.57353973388672,
-    "eval_loss": 2.790188789367676,
-    "eval_rewards/accuracies": 0.560215950012207,
-    "eval_rewards/chosen": -0.0059735761024057865,
-    "eval_rewards/margins": 0.011101600714027882,
-    "eval_rewards/rejected": -0.017075177282094955,
-    "eval_runtime": 145.7539,
-    "eval_samples": 343,
-    "eval_samples_per_second": 2.353,
-    "eval_steps_per_second": 0.295,
-    "train_loss": 2.3457992528940177,
-    "train_runtime": 3250.6949,
+    "train_loss": 0.9217340717067967,
+    "train_runtime": 3251.2729,
     "train_samples": 3079,
     "train_samples_per_second": 0.947,
     "train_steps_per_second": 0.118

train_results.json

@@ -1,7 +1,7 @@
 {
     "epoch": 1.0,
-    "train_loss": 2.3457992528940177,
-    "train_runtime": 3250.6949,
+    "train_loss": 0.9217340717067967,
+    "train_runtime": 3251.2729,
     "train_samples": 3079,
     "train_samples_per_second": 0.947,
     "train_steps_per_second": 0.118

trainer_state.json

@@ -15,7 +15,7 @@
       "logits/rejected": -1.7377450466156006,
       "logps/chosen": -29.553977966308594,
       "logps/rejected": -42.813133239746094,
-      "loss": 2.7778,
+      "loss": 1.0,
       "rewards/accuracies": 0.0,
       "rewards/chosen": 0.0,
       "rewards/margins": 0.0,
@@ -25,311 +25,311 @@
     {
       "epoch": 0.03,
       "learning_rate": 1.282051282051282e-06,
-      "logits/chosen": -1.8667066097259521,
-      "logits/rejected": -1.8710108995437622,
-      "logps/chosen": -36.97679901123047,
-      "logps/rejected": -33.63804626464844,
-      "loss": 2.6754,
-      "rewards/accuracies": 0.5138888955116272,
-      "rewards/chosen": 0.008938448503613472,
-      "rewards/margins": 0.011331514455378056,
-      "rewards/rejected": -0.0023930652532726526,
+      "logits/chosen": -1.866829752922058,
+      "logits/rejected": -1.8711390495300293,
+      "logps/chosen": -36.97040557861328,
+      "logps/rejected": -33.66280746459961,
+      "loss": 0.9793,
+      "rewards/accuracies": 0.5972222089767456,
+      "rewards/chosen": 0.010857796296477318,
+      "rewards/margins": 0.020676644518971443,
+      "rewards/rejected": -0.009818848222494125,
       "step": 10
     },
     {
       "epoch": 0.05,
       "learning_rate": 2.564102564102564e-06,
-      "logits/chosen": -1.9984171390533447,
-      "logits/rejected": -2.001072406768799,
-      "logps/chosen": -29.64606285095215,
-      "logps/rejected": -29.043380737304688,
-      "loss": 2.8951,
-      "rewards/accuracies": 0.4124999940395355,
-      "rewards/chosen": -0.0011603410821408033,
-      "rewards/margins": -0.007179437670856714,
-      "rewards/rejected": 0.006019095424562693,
+      "logits/chosen": -1.9976301193237305,
+      "logits/rejected": -2.0002708435058594,
+      "logps/chosen": -29.64252281188965,
+      "logps/rejected": -29.05857276916504,
+      "loss": 1.0016,
+      "rewards/accuracies": 0.4749999940395355,
+      "rewards/chosen": -9.895558468997478e-05,
+      "rewards/margins": -0.0015598980244249105,
+      "rewards/rejected": 0.0014609418576583266,
       "step": 20
     },
     {
       "epoch": 0.08,
       "learning_rate": 3.846153846153847e-06,
-      "logits/chosen": -1.9207251071929932,
-      "logits/rejected": -1.918031096458435,
-      "logps/chosen": -31.397014617919922,
-      "logps/rejected": -33.21030044555664,
-      "loss": 2.7749,
-      "rewards/accuracies": 0.5874999761581421,
-      "rewards/chosen": 0.0057237339206039906,
-      "rewards/margins": 0.0032763103954494,
-      "rewards/rejected": 0.0024474230594933033,
+      "logits/chosen": -1.9210872650146484,
+      "logits/rejected": -1.918402910232544,
+      "logps/chosen": -31.397838592529297,
+      "logps/rejected": -33.22719192504883,
+      "loss": 0.9919,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": 0.005476226564496756,
+      "rewards/margins": 0.00809534452855587,
+      "rewards/rejected": -0.0026191179640591145,
       "step": 30
     },
     {
       "epoch": 0.1,
       "learning_rate": 4.999896948438434e-06,
-      "logits/chosen": -2.017704486846924,
-      "logits/rejected": -2.0089590549468994,
-      "logps/chosen": -32.57511520385742,
-      "logps/rejected": -32.51970672607422,
-      "loss": 2.7984,
-      "rewards/accuracies": 0.5,
-      "rewards/chosen": 0.0005076262168586254,
-      "rewards/margins": 0.002619259525090456,
-      "rewards/rejected": -0.0021116335410624743,
+      "logits/chosen": -2.017606258392334,
+      "logits/rejected": -2.0088634490966797,
+      "logps/chosen": -32.566612243652344,
+      "logps/rejected": -32.52539825439453,
+      "loss": 0.9931,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": 0.0030602235347032547,
+      "rewards/margins": 0.0068772160448133945,
+      "rewards/rejected": -0.0038169927429407835,
       "step": 40
     },
     {
       "epoch": 0.13,
       "learning_rate": 4.987541037542187e-06,
-      "logits/chosen": -1.8626596927642822,
-      "logits/rejected": -1.8518798351287842,
-      "logps/chosen": -33.531227111816406,
-      "logps/rejected": -35.452476501464844,
-      "loss": 2.7501,
-      "rewards/accuracies": 0.48750001192092896,
-      "rewards/chosen": 0.007876711897552013,
-      "rewards/margins": 0.00791595596820116,
-      "rewards/rejected": -3.924337215721607e-05,
+      "logits/chosen": -1.8625805377960205,
+      "logits/rejected": -1.8518139123916626,
+      "logps/chosen": -33.56818771362305,
+      "logps/rejected": -35.4713020324707,
+      "loss": 0.9975,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.0032131080515682697,
+      "rewards/margins": 0.0024742281530052423,
+      "rewards/rejected": -0.005687335971742868,
       "step": 50
     },
     {
       "epoch": 0.16,
       "learning_rate": 4.954691471941119e-06,
-      "logits/chosen": -1.9410473108291626,
-      "logits/rejected": -1.9429900646209717,
-      "logps/chosen": -32.554359436035156,
-      "logps/rejected": -33.228172302246094,
-      "loss": 2.6127,
-      "rewards/accuracies": 0.6000000238418579,
-      "rewards/chosen": 0.01374770700931549,
-      "rewards/margins": 0.03078722581267357,
-      "rewards/rejected": -0.017039518803358078,
+      "logits/chosen": -1.9413166046142578,
+      "logits/rejected": -1.9432499408721924,
+      "logps/chosen": -32.55731964111328,
+      "logps/rejected": -33.21186065673828,
+      "loss": 0.975,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": 0.012859411537647247,
+      "rewards/margins": 0.02500327304005623,
+      "rewards/rejected": -0.01214386522769928,
       "step": 60
     },
     {
       "epoch": 0.18,
       "learning_rate": 4.901618883413549e-06,
-      "logits/chosen": -2.072329044342041,
-      "logits/rejected": -2.0772993564605713,
-      "logps/chosen": -33.98577880859375,
-      "logps/rejected": -36.6306266784668,
-      "loss": 2.7121,
-      "rewards/accuracies": 0.574999988079071,
-      "rewards/chosen": 0.0008118894184008241,
-      "rewards/margins": 0.0193378534168005,
-      "rewards/rejected": -0.01852596551179886,
+      "logits/chosen": -2.0717270374298096,
+      "logits/rejected": -2.0766983032226562,
+      "logps/chosen": -34.008384704589844,
+      "logps/rejected": -36.6549186706543,
+      "loss": 0.9802,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.0059700957499444485,
+      "rewards/margins": 0.01984369568526745,
+      "rewards/rejected": -0.025813793763518333,
       "step": 70
     },
     {
       "epoch": 0.21,
       "learning_rate": 4.828760511501322e-06,
-      "logits/chosen": -1.9340860843658447,
-      "logits/rejected": -1.937217354774475,
-      "logps/chosen": -34.292659759521484,
-      "logps/rejected": -34.65100860595703,
-      "loss": 2.3666,
-      "rewards/accuracies": 0.625,
-      "rewards/chosen": 0.040544699877500534,
-      "rewards/margins": 0.05812396854162216,
-      "rewards/rejected": -0.017579272389411926,
+      "logits/chosen": -1.932422399520874,
+      "logits/rejected": -1.9355411529541016,
+      "logps/chosen": -34.327762603759766,
+      "logps/rejected": -34.66461944580078,
+      "loss": 0.9483,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": 0.03001437522470951,
+      "rewards/margins": 0.05167943984270096,
+      "rewards/rejected": -0.02166506089270115,
       "step": 80
     },
     {
       "epoch": 0.23,
       "learning_rate": 4.7367166013034295e-06,
-      "logits/chosen": -1.9430879354476929,
-      "logits/rejected": -1.9476264715194702,
-      "logps/chosen": -32.375579833984375,
-      "logps/rejected": -32.32474899291992,
-      "loss": 2.7344,
+      "logits/chosen": -1.9404243230819702,
+      "logits/rejected": -1.944933533668518,
+      "logps/chosen": -32.38780975341797,
+      "logps/rejected": -32.35973358154297,
+      "loss": 0.9759,
       "rewards/accuracies": 0.6625000238418579,
-      "rewards/chosen": 0.02755170688033104,
-      "rewards/margins": 0.017304658889770508,
-      "rewards/rejected": 0.010247047990560532,
+      "rewards/chosen": 0.02388269081711769,
+      "rewards/margins": 0.024130593985319138,
+      "rewards/rejected": -0.00024790168390609324,
       "step": 90
     },
     {
       "epoch": 0.26,
       "learning_rate": 4.626245458345211e-06,
-      "logits/chosen": -2.0406196117401123,
-      "logits/rejected": -2.0386359691619873,
-      "logps/chosen": -32.16400146484375,
-      "logps/rejected": -31.278045654296875,
-      "loss": 2.59,
-      "rewards/accuracies": 0.612500011920929,
-      "rewards/chosen": 0.023049456998705864,
-      "rewards/margins": 0.02659946121275425,
-      "rewards/rejected": -0.0035500028170645237,
+      "logits/chosen": -2.0375094413757324,
+      "logits/rejected": -2.0355331897735596,
+      "logps/chosen": -32.14101028442383,
+      "logps/rejected": -31.31320571899414,
+      "loss": 0.956,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.029947593808174133,
+      "rewards/margins": 0.04404589533805847,
+      "rewards/rejected": -0.014098303392529488,
       "step": 100
     },
     {
       "epoch": 0.26,
-      "eval_logits/chosen": -2.2353110313415527,
-      "eval_logits/rejected": -2.2304632663726807,
-      "eval_logps/chosen": -34.025028228759766,
-      "eval_logps/rejected": -37.4992561340332,
-      "eval_loss": 2.8957631587982178,
-      "eval_rewards/accuracies": 0.4730066657066345,
-      "eval_rewards/chosen": 0.0028568038251250982,
-      "eval_rewards/margins": -0.0023524421267211437,
-      "eval_rewards/rejected": 0.005209244787693024,
-      "eval_runtime": 146.1162,
-      "eval_samples_per_second": 2.347,
-      "eval_steps_per_second": 0.294,
+      "eval_logits/chosen": -2.232485294342041,
+      "eval_logits/rejected": -2.227635622024536,
+      "eval_logps/chosen": -34.049468994140625,
+      "eval_logps/rejected": -37.54916000366211,
+      "eval_loss": 0.994844913482666,
+      "eval_rewards/accuracies": 0.5631229281425476,
+      "eval_rewards/chosen": -0.004474184010177851,
+      "eval_rewards/margins": 0.0052884831093251705,
+      "eval_rewards/rejected": -0.009762667119503021,
+      "eval_runtime": 146.0056,
+      "eval_samples_per_second": 2.349,
+      "eval_steps_per_second": 0.295,
       "step": 100
     },
     {
       "epoch": 0.29,
       "learning_rate": 4.498257201263691e-06,
-      "logits/chosen": -1.9961220026016235,
-      "logits/rejected": -1.993739128112793,
-      "logps/chosen": -33.09939193725586,
-      "logps/rejected": -34.00792694091797,
-      "loss": 2.7463,
-      "rewards/accuracies": 0.637499988079071,
-      "rewards/chosen": 0.043326906859874725,
-      "rewards/margins": 0.036895059049129486,
-      "rewards/rejected": 0.006431845016777515,
+      "logits/chosen": -1.9923181533813477,
+      "logits/rejected": -1.989919662475586,
+      "logps/chosen": -33.152462005615234,
+      "logps/rejected": -34.039642333984375,
+      "loss": 0.9695,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.027407139539718628,
+      "rewards/margins": 0.03048800863325596,
+      "rewards/rejected": -0.003080862807109952,
       "step": 110
     },
     {
       "epoch": 0.31,
       "learning_rate": 4.353806263777678e-06,
-      "logits/chosen": -2.0079345703125,
-      "logits/rejected": -1.999603509902954,
-      "logps/chosen": -32.316917419433594,
-      "logps/rejected": -32.12470245361328,
-      "loss": 2.6293,
-      "rewards/accuracies": 0.574999988079071,
-      "rewards/chosen": 0.038397181779146194,
-      "rewards/margins": 0.027356009930372238,
-      "rewards/rejected": 0.011041165329515934,
+      "logits/chosen": -2.0034618377685547,
+      "logits/rejected": -1.9951326847076416,
+      "logps/chosen": -32.344200134277344,
+      "logps/rejected": -32.13230895996094,
+      "loss": 0.9785,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": 0.03021082654595375,
+      "rewards/margins": 0.02145235612988472,
+      "rewards/rejected": 0.008758468553423882,
       "step": 120
     },
     {
       "epoch": 0.34,
       "learning_rate": 4.1940827077152755e-06,
-      "logits/chosen": -2.0357651710510254,
-      "logits/rejected": -2.0277907848358154,
-      "logps/chosen": -30.32888412475586,
-      "logps/rejected": -32.07011413574219,
-      "loss": 2.5267,
-      "rewards/accuracies": 0.5874999761581421,
-      "rewards/chosen": 0.04404935985803604,
-      "rewards/margins": 0.049181826412677765,
-      "rewards/rejected": -0.005132470745593309,
+      "logits/chosen": -2.031149387359619,
+      "logits/rejected": -2.0231785774230957,
+      "logps/chosen": -30.327457427978516,
+      "logps/rejected": -32.072906494140625,
+      "loss": 0.9496,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": 0.044478606432676315,
+      "rewards/margins": 0.050448428839445114,
+      "rewards/rejected": -0.005969815421849489,
       "step": 130
     },
     {
       "epoch": 0.36,
       "learning_rate": 4.0204024186666215e-06,
-      "logits/chosen": -1.9659456014633179,
-      "logits/rejected": -1.9761817455291748,
-      "logps/chosen": -31.22298240661621,
-      "logps/rejected": -32.57875442504883,
-      "loss": 2.3006,
-      "rewards/accuracies": 0.637499988079071,
-      "rewards/chosen": 0.05900341272354126,
-      "rewards/margins": 0.0721658319234848,
-      "rewards/rejected": -0.013162411749362946,
+      "logits/chosen": -1.961639642715454,
+      "logits/rejected": -1.9718501567840576,
+      "logps/chosen": -31.234424591064453,
+      "logps/rejected": -32.57652282714844,
+      "loss": 0.9319,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.05557093024253845,
+      "rewards/margins": 0.06806287914514542,
+      "rewards/rejected": -0.012491944245994091,
       "step": 140
     },
     {
       "epoch": 0.39,
       "learning_rate": 3.834196265035119e-06,
-      "logits/chosen": -1.877907156944275,
-      "logits/rejected": -1.879045844078064,
-      "logps/chosen": -33.954994201660156,
-      "logps/rejected": -34.817832946777344,
-      "loss": 2.2877,
-      "rewards/accuracies": 0.637499988079071,
-      "rewards/chosen": 0.07130923122167587,
-      "rewards/margins": 0.08948297053575516,
-      "rewards/rejected": -0.018173744902014732,
+      "logits/chosen": -1.8733562231063843,
+      "logits/rejected": -1.8745276927947998,
+      "logps/chosen": -33.9085578918457,
+      "logps/rejected": -34.8173828125,
+      "loss": 0.8973,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": 0.08524172008037567,
+      "rewards/margins": 0.10328052937984467,
+      "rewards/rejected": -0.018038809299468994,
       "step": 150
     },
     {
       "epoch": 0.42,
       "learning_rate": 3.636998309800573e-06,
-      "logits/chosen": -1.9301284551620483,
-      "logits/rejected": -1.9266859292984009,
-      "logps/chosen": -36.037498474121094,
-      "logps/rejected": -32.7092170715332,
-      "loss": 2.5282,
-      "rewards/accuracies": 0.6000000238418579,
-      "rewards/chosen": 0.04025455564260483,
-      "rewards/margins": 0.036193959414958954,
-      "rewards/rejected": 0.004060596693307161,
+      "logits/chosen": -1.9239585399627686,
+      "logits/rejected": -1.9205586910247803,
+      "logps/chosen": -35.99612045288086,
+      "logps/rejected": -32.70136260986328,
+      "loss": 0.9538,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.052666354924440384,
+      "rewards/margins": 0.04624979570508003,
+      "rewards/rejected": 0.006416561547666788,
       "step": 160
     },
     {
       "epoch": 0.44,
       "learning_rate": 3.4304331721118078e-06,
-      "logits/chosen": -2.03144907951355,
-      "logits/rejected": -2.0240814685821533,
-      "logps/chosen": -33.5056037902832,
-      "logps/rejected": -31.39798355102539,
-      "loss": 2.1268,
-      "rewards/accuracies": 0.637499988079071,
-      "rewards/chosen": 0.08767756074666977,
-      "rewards/margins": 0.10151807963848114,
-      "rewards/rejected": -0.01384051889181137,
+      "logits/chosen": -2.02461576461792,
+      "logits/rejected": -2.017319440841675,
+      "logps/chosen": -33.47701644897461,
+      "logps/rejected": -31.457340240478516,
+      "loss": 0.8721,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": 0.09625236690044403,
+      "rewards/margins": 0.12790000438690186,
+      "rewards/rejected": -0.031647637486457825,
       "step": 170
     },
     {
       "epoch": 0.47,
       "learning_rate": 3.2162026428305436e-06,
-      "logits/chosen": -2.037555694580078,
-      "logits/rejected": -2.0428080558776855,
-      "logps/chosen": -32.268028259277344,
-      "logps/rejected": -32.48134231567383,
-      "loss": 2.1694,
-      "rewards/accuracies": 0.637499988079071,
-      "rewards/chosen": 0.08302746713161469,
-      "rewards/margins": 0.08188783377408981,
-      "rewards/rejected": 0.0011396423215046525,
+      "logits/chosen": -2.030789613723755,
+      "logits/rejected": -2.0360310077667236,
+      "logps/chosen": -32.23139572143555,
+      "logps/rejected": -32.44371032714844,
+      "loss": 0.9184,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": 0.0940173864364624,
+      "rewards/margins": 0.0815875232219696,
+      "rewards/rejected": 0.012429863214492798,
       "step": 180
     },
     {
       "epoch": 0.49,
       "learning_rate": 2.996071664294641e-06,
-      "logits/chosen": -2.0385537147521973,
-      "logits/rejected": -2.0357697010040283,
-      "logps/chosen": -31.26275062561035,
-      "logps/rejected": -31.31881332397461,
-      "loss": 2.411,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": 0.0678616464138031,
-      "rewards/margins": 0.06804122775793076,
-      "rewards/rejected": -0.00017957761883735657,
+      "logits/chosen": -2.0321707725524902,
+      "logits/rejected": -2.0294008255004883,
+      "logps/chosen": -31.273319244384766,
+      "logps/rejected": -31.368595123291016,
+      "loss": 0.9202,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": 0.06469077616930008,
+      "rewards/margins": 0.0798054188489914,
+      "rewards/rejected": -0.015114650130271912,
       "step": 190
     },
     {
       "epoch": 0.52,
       "learning_rate": 2.7718537898066833e-06,
-      "logits/chosen": -1.9085556268692017,
-      "logits/rejected": -1.9132124185562134,
-      "logps/chosen": -31.326763153076172,
-      "logps/rejected": -32.80078887939453,
-      "loss": 2.2795,
-      "rewards/accuracies": 0.6499999761581421,
-      "rewards/chosen": 0.0838552787899971,
-      "rewards/margins": 0.08681019395589828,
-      "rewards/rejected": -0.002954920055344701,
+      "logits/chosen": -1.9020545482635498,
+      "logits/rejected": -1.9066784381866455,
+      "logps/chosen": -31.315628051757812,
+      "logps/rejected": -32.859519958496094,
+      "loss": 0.8922,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.0871955007314682,
+      "rewards/margins": 0.1077716127038002,
+      "rewards/rejected": -0.02057611010968685,
       "step": 200
     },
     {
       "epoch": 0.52,
-      "eval_logits/chosen": -2.2338554859161377,
-      "eval_logits/rejected": -2.2290101051330566,
-      "eval_logps/chosen": -34.05450439453125,
-      "eval_logps/rejected": -37.565059661865234,
-      "eval_loss": 2.801168918609619,
-      "eval_rewards/accuracies": 0.5278239250183105,
-      "eval_rewards/chosen": -0.005985844414681196,
-      "eval_rewards/margins": 0.0085463160648942,
-      "eval_rewards/rejected": -0.014532160945236683,
-      "eval_runtime": 145.8316,
+      "eval_logits/chosen": -2.2279350757598877,
+      "eval_logits/rejected": -2.2231225967407227,
+      "eval_logps/chosen": -34.06590270996094,
+      "eval_logps/rejected": -37.589378356933594,
+      "eval_loss": 0.9875569939613342,
+      "eval_rewards/accuracies": 0.530315637588501,
+      "eval_rewards/chosen": -0.009404394775629044,
+      "eval_rewards/margins": 0.012423668056726456,
+      "eval_rewards/rejected": -0.0218280628323555,
+      "eval_runtime": 145.8494,
       "eval_samples_per_second": 2.352,
       "eval_steps_per_second": 0.295,
       "step": 200
@@ -337,277 +337,277 @@
     {
       "epoch": 0.55,
       "learning_rate": 2.5453962426402006e-06,
-      "logits/chosen": -2.020181179046631,
-      "logits/rejected": -2.030860424041748,
-      "logps/chosen": -31.76279067993164,
-      "logps/rejected": -33.952571868896484,
-      "loss": 2.1351,
-      "rewards/accuracies": 0.6625000238418579,
-      "rewards/chosen": 0.06255346536636353,
-      "rewards/margins": 0.08951519429683685,
-      "rewards/rejected": -0.026961728930473328,
+      "logits/chosen": -2.0143890380859375,
+      "logits/rejected": -2.0250439643859863,
+      "logps/chosen": -31.765193939208984,
+      "logps/rejected": -33.9805908203125,
+      "loss": 0.9046,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.061833299696445465,
+      "rewards/margins": 0.09720136225223541,
+      "rewards/rejected": -0.03536807745695114,
       "step": 210
     },
     {
       "epoch": 0.57,
       "learning_rate": 2.3185646976551794e-06,
-      "logits/chosen": -1.912184476852417,
-      "logits/rejected": -1.926975965499878,
-      "logps/chosen": -29.868118286132812,
-      "logps/rejected": -31.57167625427246,
-      "loss": 2.231,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": 0.07302670180797577,
-      "rewards/margins": 0.07707642018795013,
-      "rewards/rejected": -0.004049715120345354,
+      "logits/chosen": -1.907122254371643,
+      "logits/rejected": -1.9218772649765015,
+      "logps/chosen": -29.810348510742188,
+      "logps/rejected": -31.652551651000977,
+      "loss": 0.8813,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.0903574600815773,
+      "rewards/margins": 0.11866964399814606,
+      "rewards/rejected": -0.028312185779213905,
       "step": 220
     },
     {
       "epoch": 0.6,
       "learning_rate": 2.0932279108998323e-06,
-      "logits/chosen": -1.968927025794983,
-      "logits/rejected": -1.972905158996582,
-      "logps/chosen": -33.126705169677734,
-      "logps/rejected": -31.619220733642578,
-      "loss": 2.0957,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": 0.0872274860739708,
-      "rewards/margins": 0.10820253938436508,
-      "rewards/rejected": -0.02097504958510399,
+      "logits/chosen": -1.9638553857803345,
+      "logits/rejected": -1.9678163528442383,
+      "logps/chosen": -33.116092681884766,
+      "logps/rejected": -31.662506103515625,
+      "loss": 0.8826,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": 0.09041241556406021,
+      "rewards/margins": 0.1243731826543808,
+      "rewards/rejected": -0.03396075591444969,
       "step": 230
     },
     {
       "epoch": 0.62,
       "learning_rate": 1.8712423238279358e-06,
-      "logits/chosen": -1.967507004737854,
-      "logits/rejected": -1.9456119537353516,
-      "logps/chosen": -33.865623474121094,
-      "logps/rejected": -35.0605583190918,
-      "loss": 1.9762,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": 0.08643803745508194,
-      "rewards/margins": 0.11936646699905396,
-      "rewards/rejected": -0.03292842581868172,
+      "logits/chosen": -1.9616014957427979,
+      "logits/rejected": -1.9398181438446045,
+      "logps/chosen": -33.844024658203125,
+      "logps/rejected": -35.163543701171875,
+      "loss": 0.8473,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": 0.09291732311248779,
+      "rewards/margins": 0.15674278140068054,
+      "rewards/rejected": -0.06382545083761215,
       "step": 240
     },
     {
       "epoch": 0.65,
       "learning_rate": 1.6544367689701824e-06,
-      "logits/chosen": -2.008720874786377,
-      "logits/rejected": -2.0053982734680176,
-      "logps/chosen": -32.73490524291992,
-      "logps/rejected": -36.2797966003418,
-      "loss": 2.2614,
-      "rewards/accuracies": 0.675000011920929,
-      "rewards/chosen": 0.05704798549413681,
-      "rewards/margins": 0.07351039350032806,
-      "rewards/rejected": -0.016462400555610657,
+      "logits/chosen": -2.002652883529663,
+      "logits/rejected": -1.9993493556976318,
+      "logps/chosen": -32.73945617675781,
+      "logps/rejected": -36.27147674560547,
+      "loss": 0.9305,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.055684663355350494,
+      "rewards/margins": 0.0696500912308693,
+      "rewards/rejected": -0.013965430669486523,
       "step": 250
     },
     {
       "epoch": 0.68,
       "learning_rate": 1.4445974030621963e-06,
-      "logits/chosen": -1.8761491775512695,
-      "logits/rejected": -1.8737146854400635,
-      "logps/chosen": -33.989051818847656,
-      "logps/rejected": -35.516883850097656,
-      "loss": 2.4128,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": 0.05980368331074715,
-      "rewards/margins": 0.06348178535699844,
-      "rewards/rejected": -0.003678102744743228,
+      "logits/chosen": -1.869681715965271,
+      "logits/rejected": -1.8672786951065063,
+      "logps/chosen": -34.0054931640625,
+      "logps/rejected": -35.55742645263672,
+      "loss": 0.9293,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": 0.05487124249339104,
+      "rewards/margins": 0.07071295380592346,
+      "rewards/rejected": -0.015841711312532425,
       "step": 260
     },
     {
       "epoch": 0.7,
       "learning_rate": 1.243452991757889e-06,
-      "logits/chosen": -1.8607820272445679,
-      "logits/rejected": -1.8583341836929321,
-      "logps/chosen": -34.1697998046875,
-      "logps/rejected": -31.77322769165039,
-      "loss": 2.33,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": 0.06833983957767487,
-      "rewards/margins": 0.06890784204006195,
-      "rewards/rejected": -0.0005679976311512291,
+      "logits/chosen": -1.8544126749038696,
+      "logits/rejected": -1.852034330368042,
+      "logps/chosen": -34.22079086303711,
+      "logps/rejected": -31.864788055419922,
+      "loss": 0.9205,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": 0.0530419759452343,
+      "rewards/margins": 0.08107996731996536,
+      "rewards/rejected": -0.028037995100021362,
       "step": 270
     },
     {
       "epoch": 0.73,
       "learning_rate": 1.0526606671603523e-06,
-      "logits/chosen": -1.9649995565414429,
-      "logits/rejected": -1.954466462135315,
-      "logps/chosen": -35.01842498779297,
-      "logps/rejected": -31.856042861938477,
-      "loss": 2.0235,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": 0.10146405547857285,
-      "rewards/margins": 0.10597936809062958,
-      "rewards/rejected": -0.004515302833169699,
+      "logits/chosen": -1.957960844039917,
+      "logits/rejected": -1.9474948644638062,
+      "logps/chosen": -35.02719497680664,
+      "logps/rejected": -31.89957046508789,
+      "loss": 0.8836,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": 0.09883318096399307,
+      "rewards/margins": 0.11640648543834686,
+      "rewards/rejected": -0.017573300749063492,
       "step": 280
     },
     {
       "epoch": 0.75,
       "learning_rate": 8.737922755071455e-07,
-      "logits/chosen": -2.059657096862793,
-      "logits/rejected": -2.0446982383728027,
-      "logps/chosen": -30.713714599609375,
-      "logps/rejected": -32.61918258666992,
-      "loss": 2.5851,
-      "rewards/accuracies": 0.612500011920929,
-      "rewards/chosen": 0.06312306225299835,
-      "rewards/margins": 0.0533272810280323,
-      "rewards/rejected": 0.00979578960686922,
+      "logits/chosen": -2.0539374351501465,
+      "logits/rejected": -2.0390708446502686,
+      "logps/chosen": -30.736724853515625,
+      "logps/rejected": -32.64698791503906,
+      "loss": 0.9452,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.05622049421072006,
+      "rewards/margins": 0.05476574972271919,
+      "rewards/rejected": 0.001454742974601686,
       "step": 290
     },
     {
       "epoch": 0.78,
       "learning_rate": 7.08321427484816e-07,
-      "logits/chosen": -1.9307676553726196,
-      "logits/rejected": -1.9282405376434326,
-      "logps/chosen": -32.467708587646484,
-      "logps/rejected": -30.857818603515625,
-      "loss": 1.7902,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": 0.1378202736377716,
-      "rewards/margins": 0.15012334287166595,
-      "rewards/rejected": -0.012303064577281475,
+      "logits/chosen": -1.9243535995483398,
+      "logits/rejected": -1.9218591451644897,
+      "logps/chosen": -32.35340118408203,
+      "logps/rejected": -30.938573837280273,
+      "loss": 0.8033,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.17211246490478516,
+      "rewards/margins": 0.20864196121692657,
+      "rewards/rejected": -0.036529459059238434,
       "step": 300
     },
     {
       "epoch": 0.78,
-      "eval_logits/chosen": -2.2293622493743896,
-      "eval_logits/rejected": -2.2245147228240967,
-      "eval_logps/chosen": -34.04458236694336,
-      "eval_logps/rejected": -37.57244873046875,
-      "eval_loss": 2.758474826812744,
-      "eval_rewards/accuracies": 0.5747508406639099,
-      "eval_rewards/chosen": -0.003010095562785864,
-      "eval_rewards/margins": 0.013738218694925308,
-      "eval_rewards/rejected": -0.01674831472337246,
-      "eval_runtime": 145.5146,
-      "eval_samples_per_second": 2.357,
-      "eval_steps_per_second": 0.296,
+      "eval_logits/chosen": -2.224973440170288,
+      "eval_logits/rejected": -2.2201507091522217,
+      "eval_logps/chosen": -34.1029052734375,
+      "eval_logps/rejected": -37.6210823059082,
+      "eval_loss": 0.989064633846283,
+      "eval_rewards/accuracies": 0.5182723999023438,
+      "eval_rewards/chosen": -0.020507192239165306,
+      "eval_rewards/margins": 0.010832280851900578,
+      "eval_rewards/rejected": -0.03133947402238846,
+      "eval_runtime": 145.7799,
+      "eval_samples_per_second": 2.353,
+      "eval_steps_per_second": 0.295,
       "step": 300
     },
     {
       "epoch": 0.81,
       "learning_rate": 5.576113578589035e-07,
-      "logits/chosen": -1.9147694110870361,
-      "logits/rejected": -1.9114938974380493,
-      "logps/chosen": -31.310047149658203,
-      "logps/rejected": -33.77363967895508,
-      "loss": 2.1768,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": 0.08799003809690475,
-      "rewards/margins": 0.09743582457304001,
-      "rewards/rejected": -0.009445784613490105,
+      "logits/chosen": -1.910521149635315,
+      "logits/rejected": -1.9072707891464233,
+      "logps/chosen": -31.341650009155273,
+      "logps/rejected": -33.842002868652344,
+      "loss": 0.9015,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": 0.07851080596446991,
+      "rewards/margins": 0.10846547037363052,
+      "rewards/rejected": -0.02995465137064457,
       "step": 310
     },
     {
       "epoch": 0.83,
       "learning_rate": 4.229036944380913e-07,
-      "logits/chosen": -1.965264081954956,
-      "logits/rejected": -1.953029990196228,
-      "logps/chosen": -34.34061050415039,
-      "logps/rejected": -33.643863677978516,
-      "loss": 1.9927,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": 0.07663901150226593,
-      "rewards/margins": 0.10814561694860458,
-      "rewards/rejected": -0.03150660917162895,
+      "logits/chosen": -1.9592889547348022,
+      "logits/rejected": -1.9470914602279663,
+      "logps/chosen": -34.280879974365234,
+      "logps/rejected": -33.69014358520508,
+      "loss": 0.8601,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": 0.09455753862857819,
+      "rewards/margins": 0.1399478018283844,
+      "rewards/rejected": -0.04539026692509651,
       "step": 320
     },
     {
       "epoch": 0.86,
       "learning_rate": 3.053082288996112e-07,
-      "logits/chosen": -1.9999011754989624,
-      "logits/rejected": -1.9984591007232666,
-      "logps/chosen": -33.24303436279297,
-      "logps/rejected": -32.54120635986328,
-      "loss": 2.1065,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": 0.0728277713060379,
-      "rewards/margins": 0.09131507575511932,
-      "rewards/rejected": -0.01848730817437172,
+      "logits/chosen": -1.9943225383758545,
+      "logits/rejected": -1.992897629737854,
+      "logps/chosen": -33.13710403442383,
+      "logps/rejected": -32.56306838989258,
+      "loss": 0.8752,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.10460783541202545,
+      "rewards/margins": 0.1296529769897461,
+      "rewards/rejected": -0.025045130401849747,
       "step": 330
     },
     {
       "epoch": 0.88,
       "learning_rate": 2.0579377374915805e-07,
-      "logits/chosen": -2.086862802505493,
-      "logits/rejected": -2.0711212158203125,
-      "logps/chosen": -33.77810287475586,
-      "logps/rejected": -33.093299865722656,
-      "loss": 2.0709,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": 0.12038693577051163,
-      "rewards/margins": 0.1129666194319725,
-      "rewards/rejected": 0.007420300040394068,
+      "logits/chosen": -2.081005096435547,
+      "logits/rejected": -2.0653510093688965,
+      "logps/chosen": -33.79452896118164,
+      "logps/rejected": -33.11925506591797,
+      "loss": 0.8842,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.1154591292142868,
+      "rewards/margins": 0.11582595109939575,
+      "rewards/rejected": -0.00036681219353340566,
       "step": 340
     },
     {
       "epoch": 0.91,
       "learning_rate": 1.2518018074041684e-07,
-      "logits/chosen": -1.9597351551055908,
-      "logits/rejected": -1.9588581323623657,
-      "logps/chosen": -32.87625503540039,
-      "logps/rejected": -32.52130889892578,
-      "loss": 1.9515,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": 0.12175308167934418,
-      "rewards/margins": 0.13860100507736206,
-      "rewards/rejected": -0.016847927123308182,
+      "logits/chosen": -1.9534845352172852,
+      "logits/rejected": -1.9526736736297607,
+      "logps/chosen": -32.83572006225586,
+      "logps/rejected": -32.52921676635742,
+      "loss": 0.8567,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.1339126080274582,
+      "rewards/margins": 0.15313318371772766,
+      "rewards/rejected": -0.01922057382762432,
       "step": 350
     },
     {
       "epoch": 0.94,
       "learning_rate": 6.41315865106129e-08,
-      "logits/chosen": -1.9148937463760376,
-      "logits/rejected": -1.9252065420150757,
-      "logps/chosen": -31.9158935546875,
-      "logps/rejected": -35.31604766845703,
-      "loss": 2.1343,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": 0.08664007484912872,
-      "rewards/margins": 0.09482574462890625,
-      "rewards/rejected": -0.008185659535229206,
+      "logits/chosen": -1.9090850353240967,
+      "logits/rejected": -1.919353723526001,
+      "logps/chosen": -31.896703720092773,
+      "logps/rejected": -35.348304748535156,
+      "loss": 0.8898,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.09239637851715088,
+      "rewards/margins": 0.11026018857955933,
+      "rewards/rejected": -0.017863804474473,
       "step": 360
     },
     {
       "epoch": 0.96,
       "learning_rate": 2.3150941078050325e-08,
-      "logits/chosen": -2.053968906402588,
-      "logits/rejected": -2.047475576400757,
-      "logps/chosen": -33.36884307861328,
-      "logps/rejected": -29.23018455505371,
-      "loss": 2.1339,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": 0.08493683487176895,
-      "rewards/margins": 0.08606470376253128,
-      "rewards/rejected": -0.0011278685415163636,
+      "logits/chosen": -2.0488381385803223,
+      "logits/rejected": -2.0423855781555176,
+      "logps/chosen": -33.353912353515625,
+      "logps/rejected": -29.256216049194336,
+      "loss": 0.9016,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": 0.0894167423248291,
+      "rewards/margins": 0.09835406392812729,
+      "rewards/rejected": -0.008937308564782143,
       "step": 370
     },
     {
       "epoch": 0.99,
       "learning_rate": 2.575864278703266e-09,
-      "logits/chosen": -1.9135916233062744,
-      "logits/rejected": -1.9158084392547607,
-      "logps/chosen": -33.879058837890625,
-      "logps/rejected": -30.930347442626953,
-      "loss": 1.9191,
-      "rewards/accuracies": 0.7749999761581421,
-      "rewards/chosen": 0.10799793899059296,
-      "rewards/margins": 0.12531307339668274,
-      "rewards/rejected": -0.017315123230218887,
+      "logits/chosen": -1.9087949991226196,
+      "logits/rejected": -1.9110119342803955,
+      "logps/chosen": -33.8967399597168,
+      "logps/rejected": -31.023351669311523,
+      "loss": 0.8526,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": 0.10269282758235931,
+      "rewards/margins": 0.1479099541902542,
+      "rewards/rejected": -0.0452171191573143,
       "step": 380
     },
     {
       "epoch": 1.0,
       "step": 385,
       "total_flos": 0.0,
-      "train_loss": 2.3457992528940177,
-      "train_runtime": 3250.6949,
+      "train_loss": 0.9217340717067967,
+      "train_runtime": 3251.2729,
       "train_samples_per_second": 0.947,
       "train_steps_per_second": 0.118
     }