Model save

README.md

@@ -1,13 +1,10 @@
 ---
 library_name: peft
 tags:
-- alignment-handbook
 - trl
 - dpo
 - generated_from_trainer
 base_model: NbAiLab/nb-gpt-j-6B-v2
-datasets:
-- hugodk-sch/aftonposten_title_prefs
 model-index:
 - name: aftonposten-6b-align-scan
   results: []
@@ -18,17 +15,17 @@ should probably proofread and complete it, then remove this comment. -->
 
 # aftonposten-6b-align-scan
 
-This model is a fine-tuned version of [data/ap-gpt-j-6b-sft-qlora-04-08](https://huggingface.co/data/ap-gpt-j-6b-sft-qlora-04-08) on the hugodk-sch/aftonposten_title_prefs dataset.
+This model is a fine-tuned version of [NbAiLab/nb-gpt-j-6B-v2](https://huggingface.co/NbAiLab/nb-gpt-j-6B-v2) on an unknown dataset.
 It achieves the following results on the evaluation set:
-- Loss: 0.4974
-- Rewards/chosen: 0.0832
-- Rewards/rejected: 0.0727
-- Rewards/accuracies: 0.5278
-- Rewards/margins: 0.0105
-- Logps/rejected: -37.2742
-- Logps/chosen: -33.7571
-- Logits/rejected: -2.2256
-- Logits/chosen: -2.2304
+- Loss: 2.7585
+- Rewards/chosen: -0.0030
+- Rewards/rejected: -0.0167
+- Rewards/accuracies: 0.5748
+- Rewards/margins: 0.0137
+- Logps/rejected: -37.5724
+- Logps/chosen: -34.0446
+- Logits/rejected: -2.2245
+- Logits/chosen: -2.2294
 
 ## Model description
 
@@ -63,9 +60,9 @@ The following hyperparameters were used during training:
 
 | Training Loss | Epoch | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
 |:-------------:|:-----:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
-| 0.4916        | 0.26  | 100  | 0.4981          | 0.0409         | 0.0320           | 0.5453             | 0.0089          | -37.4099       | -33.8981     | -2.2290         | -2.2338       |
-| 0.4782        | 0.52  | 200  | 0.4962          | 0.0792         | 0.0635           | 0.5718             | 0.0157          | -37.3050       | -33.7707     | -2.2283         | -2.2331       |
-| 0.4507        | 0.78  | 300  | 0.4956          | 0.0859         | 0.0684           | 0.5544             | 0.0175          | -37.2887       | -33.7484     | -2.2254         | -2.2302       |
+| 2.59          | 0.26  | 100  | 2.8958          | 0.0029         | 0.0052           | 0.4730             | -0.0024         | -37.4993       | -34.0250     | -2.2305         | -2.2353       |
+| 2.2795        | 0.52  | 200  | 2.8012          | -0.0060        | -0.0145          | 0.5278             | 0.0085          | -37.5651       | -34.0545     | -2.2290         | -2.2339       |
+| 1.7902        | 0.78  | 300  | 2.7585          | -0.0030        | -0.0167          | 0.5748             | 0.0137          | -37.5724       | -34.0446     | -2.2245         | -2.2294       |
 
 
 ### Framework versions

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a6ef47e975f7321a1ca0a421b300bc8f06b29d9be6815ca5793dd14d383d6c9f
+oid sha256:10aa581c0cd6ba4f1244f82023de1dceea19cd5454103b9b51a903bc89232ba5
 size 176183216

all_results.json

@@ -1,21 +1,8 @@
 {
     "epoch": 1.0,
-    "eval_logits/chosen": -2.2303643226623535,
-    "eval_logits/rejected": -2.225562334060669,
-    "eval_logps/chosen": -33.75711441040039,
-    "eval_logps/rejected": -37.27421951293945,
-    "eval_loss": 0.4974016547203064,
-    "eval_rewards/accuracies": 0.5278239250183105,
-    "eval_rewards/chosen": 0.08323083817958832,
-    "eval_rewards/margins": 0.010510895401239395,
-    "eval_rewards/rejected": 0.07271993160247803,
-    "eval_runtime": 145.4113,
-    "eval_samples": 343,
-    "eval_samples_per_second": 2.359,
-    "eval_steps_per_second": 0.296,
-    "train_loss": 0.48147913697478056,
-    "train_runtime": 3253.5379,
+    "train_loss": 2.3457992528940177,
+    "train_runtime": 3250.6949,
     "train_samples": 3079,
-    "train_samples_per_second": 0.946,
+    "train_samples_per_second": 0.947,
     "train_steps_per_second": 0.118
 }

train_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 1.0,
-    "train_loss": 0.48147913697478056,
-    "train_runtime": 3253.5379,
+    "train_loss": 2.3457992528940177,
+    "train_runtime": 3250.6949,
     "train_samples": 3079,
-    "train_samples_per_second": 0.946,
+    "train_samples_per_second": 0.947,
     "train_steps_per_second": 0.118
 }

trainer_state.json

@@ -15,7 +15,7 @@
       "logits/rejected": -1.7377450466156006,
       "logps/chosen": -29.553977966308594,
       "logps/rejected": -42.813133239746094,
-      "loss": 0.5,
+      "loss": 2.7778,
       "rewards/accuracies": 0.0,
       "rewards/chosen": 0.0,
       "rewards/margins": 0.0,
@@ -25,590 +25,590 @@
     {
       "epoch": 0.03,
       "learning_rate": 1.282051282051282e-06,
-      "logits/chosen": -1.866670846939087,
-      "logits/rejected": -1.8709994554519653,
-      "logps/chosen": -36.98710250854492,
-      "logps/rejected": -33.663551330566406,
-      "loss": 0.4972,
-      "rewards/accuracies": 0.5694444179534912,
-      "rewards/chosen": 0.005848363973200321,
-      "rewards/margins": 0.015892064198851585,
-      "rewards/rejected": -0.01004369929432869,
+      "logits/chosen": -1.8667066097259521,
+      "logits/rejected": -1.8710108995437622,
+      "logps/chosen": -36.97679901123047,
+      "logps/rejected": -33.63804626464844,
+      "loss": 2.6754,
+      "rewards/accuracies": 0.5138888955116272,
+      "rewards/chosen": 0.008938448503613472,
+      "rewards/margins": 0.011331514455378056,
+      "rewards/rejected": -0.0023930652532726526,
       "step": 10
     },
     {
       "epoch": 0.05,
       "learning_rate": 2.564102564102564e-06,
-      "logits/chosen": -1.9969775676727295,
-      "logits/rejected": -1.9996353387832642,
-      "logps/chosen": -29.633682250976562,
-      "logps/rejected": -29.049312591552734,
-      "loss": 0.5001,
-      "rewards/accuracies": 0.4749999940395355,
-      "rewards/chosen": 0.00255323457531631,
-      "rewards/margins": -0.0016868360107764602,
-      "rewards/rejected": 0.004240070004016161,
+      "logits/chosen": -1.9984171390533447,
+      "logits/rejected": -2.001072406768799,
+      "logps/chosen": -29.64606285095215,
+      "logps/rejected": -29.043380737304688,
+      "loss": 2.8951,
+      "rewards/accuracies": 0.4124999940395355,
+      "rewards/chosen": -0.0011603410821408033,
+      "rewards/margins": -0.007179437670856714,
+      "rewards/rejected": 0.006019095424562693,
       "step": 20
     },
     {
       "epoch": 0.08,
       "learning_rate": 3.846153846153847e-06,
-      "logits/chosen": -1.9205176830291748,
-      "logits/rejected": -1.9178295135498047,
-      "logps/chosen": -31.409936904907227,
-      "logps/rejected": -33.229610443115234,
-      "loss": 0.4991,
-      "rewards/accuracies": 0.5375000238418579,
-      "rewards/chosen": 0.0018473375821486115,
-      "rewards/margins": 0.005193009972572327,
-      "rewards/rejected": -0.003345672506839037,
+      "logits/chosen": -1.9207251071929932,
+      "logits/rejected": -1.918031096458435,
+      "logps/chosen": -31.397014617919922,
+      "logps/rejected": -33.21030044555664,
+      "loss": 2.7749,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": 0.0057237339206039906,
+      "rewards/margins": 0.0032763103954494,
+      "rewards/rejected": 0.0024474230594933033,
       "step": 30
     },
     {
       "epoch": 0.1,
       "learning_rate": 4.999896948438434e-06,
-      "logits/chosen": -2.016585350036621,
-      "logits/rejected": -2.0078723430633545,
-      "logps/chosen": -32.56060028076172,
-      "logps/rejected": -32.5064811706543,
-      "loss": 0.4993,
-      "rewards/accuracies": 0.4749999940395355,
-      "rewards/chosen": 0.00486344238743186,
-      "rewards/margins": 0.003007061081007123,
-      "rewards/rejected": 0.0018563822377473116,
+      "logits/chosen": -2.017704486846924,
+      "logits/rejected": -2.0089590549468994,
+      "logps/chosen": -32.57511520385742,
+      "logps/rejected": -32.51970672607422,
+      "loss": 2.7984,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": 0.0005076262168586254,
+      "rewards/margins": 0.002619259525090456,
+      "rewards/rejected": -0.0021116335410624743,
       "step": 40
     },
     {
       "epoch": 0.13,
       "learning_rate": 4.987541037542187e-06,
-      "logits/chosen": -1.8626368045806885,
-      "logits/rejected": -1.8518617153167725,
-      "logps/chosen": -33.54547882080078,
-      "logps/rejected": -35.416542053222656,
-      "loss": 0.5018,
-      "rewards/accuracies": 0.375,
-      "rewards/chosen": 0.0036011666525155306,
-      "rewards/margins": -0.007139990571886301,
-      "rewards/rejected": 0.0107411565259099,
+      "logits/chosen": -1.8626596927642822,
+      "logits/rejected": -1.8518798351287842,
+      "logps/chosen": -33.531227111816406,
+      "logps/rejected": -35.452476501464844,
+      "loss": 2.7501,
+      "rewards/accuracies": 0.48750001192092896,
+      "rewards/chosen": 0.007876711897552013,
+      "rewards/margins": 0.00791595596820116,
+      "rewards/rejected": -3.924337215721607e-05,
       "step": 50
     },
     {
       "epoch": 0.16,
       "learning_rate": 4.954691471941119e-06,
-      "logits/chosen": -1.940407395362854,
-      "logits/rejected": -1.9423446655273438,
-      "logps/chosen": -32.54948043823242,
-      "logps/rejected": -33.21086883544922,
-      "loss": 0.4956,
-      "rewards/accuracies": 0.625,
-      "rewards/chosen": 0.015210109762847424,
-      "rewards/margins": 0.027057761326432228,
-      "rewards/rejected": -0.011847653426229954,
+      "logits/chosen": -1.9410473108291626,
+      "logits/rejected": -1.9429900646209717,
+      "logps/chosen": -32.554359436035156,
+      "logps/rejected": -33.228172302246094,
+      "loss": 2.6127,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.01374770700931549,
+      "rewards/margins": 0.03078722581267357,
+      "rewards/rejected": -0.017039518803358078,
       "step": 60
     },
     {
       "epoch": 0.18,
       "learning_rate": 4.901618883413549e-06,
-      "logits/chosen": -2.0710127353668213,
-      "logits/rejected": -2.075969696044922,
-      "logps/chosen": -33.97991180419922,
-      "logps/rejected": -36.61283874511719,
-      "loss": 0.4963,
-      "rewards/accuracies": 0.5249999761581421,
-      "rewards/chosen": 0.0025722028221935034,
-      "rewards/margins": 0.015761729329824448,
-      "rewards/rejected": -0.013189527206122875,
+      "logits/chosen": -2.072329044342041,
+      "logits/rejected": -2.0772993564605713,
+      "logps/chosen": -33.98577880859375,
+      "logps/rejected": -36.6306266784668,
+      "loss": 2.7121,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.0008118894184008241,
+      "rewards/margins": 0.0193378534168005,
+      "rewards/rejected": -0.01852596551179886,
       "step": 70
     },
     {
       "epoch": 0.21,
       "learning_rate": 4.828760511501322e-06,
-      "logits/chosen": -1.932826280593872,
-      "logits/rejected": -1.935924768447876,
-      "logps/chosen": -34.29054641723633,
-      "logps/rejected": -34.57255935668945,
-      "loss": 0.4919,
-      "rewards/accuracies": 0.5375000238418579,
-      "rewards/chosen": 0.04117867723107338,
-      "rewards/margins": 0.03522341698408127,
-      "rewards/rejected": 0.0059552607126533985,
+      "logits/chosen": -1.9340860843658447,
+      "logits/rejected": -1.937217354774475,
+      "logps/chosen": -34.292659759521484,
+      "logps/rejected": -34.65100860595703,
+      "loss": 2.3666,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.040544699877500534,
+      "rewards/margins": 0.05812396854162216,
+      "rewards/rejected": -0.017579272389411926,
       "step": 80
     },
     {
       "epoch": 0.23,
       "learning_rate": 4.7367166013034295e-06,
-      "logits/chosen": -1.9420931339263916,
-      "logits/rejected": -1.9465965032577515,
-      "logps/chosen": -32.2992057800293,
-      "logps/rejected": -32.29533767700195,
-      "loss": 0.4921,
-      "rewards/accuracies": 0.5874999761581421,
-      "rewards/chosen": 0.0504642128944397,
-      "rewards/margins": 0.03139360994100571,
-      "rewards/rejected": 0.01907060667872429,
+      "logits/chosen": -1.9430879354476929,
+      "logits/rejected": -1.9476264715194702,
+      "logps/chosen": -32.375579833984375,
+      "logps/rejected": -32.32474899291992,
+      "loss": 2.7344,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": 0.02755170688033104,
+      "rewards/margins": 0.017304658889770508,
+      "rewards/rejected": 0.010247047990560532,
       "step": 90
     },
     {
       "epoch": 0.26,
       "learning_rate": 4.626245458345211e-06,
-      "logits/chosen": -2.0399727821350098,
-      "logits/rejected": -2.0379879474639893,
-      "logps/chosen": -32.03980255126953,
-      "logps/rejected": -31.19321060180664,
-      "loss": 0.4916,
-      "rewards/accuracies": 0.5874999761581421,
-      "rewards/chosen": 0.06031019240617752,
-      "rewards/margins": 0.03840993717312813,
-      "rewards/rejected": 0.021900251507759094,
+      "logits/chosen": -2.0406196117401123,
+      "logits/rejected": -2.0386359691619873,
+      "logps/chosen": -32.16400146484375,
+      "logps/rejected": -31.278045654296875,
+      "loss": 2.59,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": 0.023049456998705864,
+      "rewards/margins": 0.02659946121275425,
+      "rewards/rejected": -0.0035500028170645237,
       "step": 100
     },
     {
       "epoch": 0.26,
-      "eval_logits/chosen": -2.2337982654571533,
-      "eval_logits/rejected": -2.228950262069702,
-      "eval_logps/chosen": -33.89812469482422,
-      "eval_logps/rejected": -37.40989303588867,
-      "eval_loss": 0.4981168508529663,
-      "eval_rewards/accuracies": 0.545265793800354,
-      "eval_rewards/chosen": 0.040927641093730927,
-      "eval_rewards/margins": 0.008909112773835659,
-      "eval_rewards/rejected": 0.03201852738857269,
-      "eval_runtime": 146.0553,
-      "eval_samples_per_second": 2.348,
+      "eval_logits/chosen": -2.2353110313415527,
+      "eval_logits/rejected": -2.2304632663726807,
+      "eval_logps/chosen": -34.025028228759766,
+      "eval_logps/rejected": -37.4992561340332,
+      "eval_loss": 2.8957631587982178,
+      "eval_rewards/accuracies": 0.4730066657066345,
+      "eval_rewards/chosen": 0.0028568038251250982,
+      "eval_rewards/margins": -0.0023524421267211437,
+      "eval_rewards/rejected": 0.005209244787693024,
+      "eval_runtime": 146.1162,
+      "eval_samples_per_second": 2.347,
       "eval_steps_per_second": 0.294,
       "step": 100
     },
     {
       "epoch": 0.29,
       "learning_rate": 4.498257201263691e-06,
-      "logits/chosen": -1.9951223134994507,
-      "logits/rejected": -1.992761254310608,
-      "logps/chosen": -32.98120880126953,
-      "logps/rejected": -33.893192291259766,
-      "loss": 0.4899,
-      "rewards/accuracies": 0.612500011920929,
-      "rewards/chosen": 0.07878315448760986,
-      "rewards/margins": 0.037929367274045944,
-      "rewards/rejected": 0.04085379093885422,
+      "logits/chosen": -1.9961220026016235,
+      "logits/rejected": -1.993739128112793,
+      "logps/chosen": -33.09939193725586,
+      "logps/rejected": -34.00792694091797,
+      "loss": 2.7463,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": 0.043326906859874725,
+      "rewards/margins": 0.036895059049129486,
+      "rewards/rejected": 0.006431845016777515,
       "step": 110
     },
     {
       "epoch": 0.31,
       "learning_rate": 4.353806263777678e-06,
-      "logits/chosen": -2.0073421001434326,
-      "logits/rejected": -1.9990257024765015,
-      "logps/chosen": -32.181541442871094,
-      "logps/rejected": -31.97528076171875,
-      "loss": 0.4945,
-      "rewards/accuracies": 0.5,
-      "rewards/chosen": 0.07900922745466232,
-      "rewards/margins": 0.02314186468720436,
-      "rewards/rejected": 0.05586736649274826,
+      "logits/chosen": -2.0079345703125,
+      "logits/rejected": -1.999603509902954,
+      "logps/chosen": -32.316917419433594,
+      "logps/rejected": -32.12470245361328,
+      "loss": 2.6293,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.038397181779146194,
+      "rewards/margins": 0.027356009930372238,
+      "rewards/rejected": 0.011041165329515934,
       "step": 120
     },
     {
       "epoch": 0.34,
       "learning_rate": 4.1940827077152755e-06,
-      "logits/chosen": -2.0356264114379883,
-      "logits/rejected": -2.027677536010742,
-      "logps/chosen": -30.176921844482422,
-      "logps/rejected": -31.910192489624023,
-      "loss": 0.4891,
+      "logits/chosen": -2.0357651710510254,
+      "logits/rejected": -2.0277907848358154,
+      "logps/chosen": -30.32888412475586,
+      "logps/rejected": -32.07011413574219,
+      "loss": 2.5267,
       "rewards/accuracies": 0.5874999761581421,
-      "rewards/chosen": 0.08963784575462341,
-      "rewards/margins": 0.046793900430202484,
-      "rewards/rejected": 0.04284394159913063,
+      "rewards/chosen": 0.04404935985803604,
+      "rewards/margins": 0.049181826412677765,
+      "rewards/rejected": -0.005132470745593309,
       "step": 130
     },
     {
       "epoch": 0.36,
       "learning_rate": 4.0204024186666215e-06,
-      "logits/chosen": -1.9647891521453857,
-      "logits/rejected": -1.9749748706817627,
-      "logps/chosen": -31.036914825439453,
-      "logps/rejected": -32.402671813964844,
-      "loss": 0.4821,
+      "logits/chosen": -1.9659456014633179,
+      "logits/rejected": -1.9761817455291748,
+      "logps/chosen": -31.22298240661621,
+      "logps/rejected": -32.57875442504883,
+      "loss": 2.3006,
       "rewards/accuracies": 0.637499988079071,
-      "rewards/chosen": 0.11482372134923935,
-      "rewards/margins": 0.0751611739397049,
-      "rewards/rejected": 0.03966255113482475,
+      "rewards/chosen": 0.05900341272354126,
+      "rewards/margins": 0.0721658319234848,
+      "rewards/rejected": -0.013162411749362946,
       "step": 140
     },
     {
       "epoch": 0.39,
       "learning_rate": 3.834196265035119e-06,
-      "logits/chosen": -1.877423644065857,
-      "logits/rejected": -1.8785810470581055,
-      "logps/chosen": -33.68976593017578,
-      "logps/rejected": -34.568260192871094,
-      "loss": 0.4783,
-      "rewards/accuracies": 0.5874999761581421,
-      "rewards/chosen": 0.15087710320949554,
-      "rewards/margins": 0.09418026357889175,
-      "rewards/rejected": 0.056696850806474686,
+      "logits/chosen": -1.877907156944275,
+      "logits/rejected": -1.879045844078064,
+      "logps/chosen": -33.954994201660156,
+      "logps/rejected": -34.817832946777344,
+      "loss": 2.2877,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": 0.07130923122167587,
+      "rewards/margins": 0.08948297053575516,
+      "rewards/rejected": -0.018173744902014732,
       "step": 150
     },
     {
       "epoch": 0.42,
       "learning_rate": 3.636998309800573e-06,
-      "logits/chosen": -1.9285659790039062,
-      "logits/rejected": -1.9251940250396729,
-      "logps/chosen": -35.74094009399414,
-      "logps/rejected": -32.49424362182617,
-      "loss": 0.485,
-      "rewards/accuracies": 0.612500011920929,
-      "rewards/chosen": 0.12922170758247375,
-      "rewards/margins": 0.060669444501399994,
-      "rewards/rejected": 0.06855227053165436,
+      "logits/chosen": -1.9301284551620483,
+      "logits/rejected": -1.9266859292984009,
+      "logps/chosen": -36.037498474121094,
+      "logps/rejected": -32.7092170715332,
+      "loss": 2.5282,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.04025455564260483,
+      "rewards/margins": 0.036193959414958954,
+      "rewards/rejected": 0.004060596693307161,
       "step": 160
     },
     {
       "epoch": 0.44,
       "learning_rate": 3.4304331721118078e-06,
-      "logits/chosen": -2.030134677886963,
-      "logits/rejected": -2.022822856903076,
-      "logps/chosen": -33.23253631591797,
-      "logps/rejected": -31.212448120117188,
-      "loss": 0.4695,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": 0.16959723830223083,
-      "rewards/margins": 0.1277768909931183,
-      "rewards/rejected": 0.04182034358382225,
+      "logits/chosen": -2.03144907951355,
+      "logits/rejected": -2.0240814685821533,
+      "logps/chosen": -33.5056037902832,
+      "logps/rejected": -31.39798355102539,
+      "loss": 2.1268,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": 0.08767756074666977,
+      "rewards/margins": 0.10151807963848114,
+      "rewards/rejected": -0.01384051889181137,
       "step": 170
     },
     {
       "epoch": 0.47,
       "learning_rate": 3.2162026428305436e-06,
-      "logits/chosen": -2.0374767780303955,
-      "logits/rejected": -2.0426647663116455,
-      "logps/chosen": -31.980606079101562,
-      "logps/rejected": -32.197139739990234,
-      "loss": 0.4794,
-      "rewards/accuracies": 0.625,
-      "rewards/chosen": 0.16925473511219025,
-      "rewards/margins": 0.08285479247570038,
-      "rewards/rejected": 0.08639995008707047,
+      "logits/chosen": -2.037555694580078,
+      "logits/rejected": -2.0428080558776855,
+      "logps/chosen": -32.268028259277344,
+      "logps/rejected": -32.48134231567383,
+      "loss": 2.1694,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": 0.08302746713161469,
+      "rewards/margins": 0.08188783377408981,
+      "rewards/rejected": 0.0011396423215046525,
       "step": 180
     },
     {
       "epoch": 0.49,
       "learning_rate": 2.996071664294641e-06,
-      "logits/chosen": -2.0376675128936768,
-      "logits/rejected": -2.034926414489746,
-      "logps/chosen": -31.073001861572266,
-      "logps/rejected": -31.105113983154297,
-      "loss": 0.4849,
-      "rewards/accuracies": 0.612500011920929,
-      "rewards/chosen": 0.12478618323802948,
-      "rewards/margins": 0.06085646152496338,
-      "rewards/rejected": 0.0639297217130661,
+      "logits/chosen": -2.0385537147521973,
+      "logits/rejected": -2.0357697010040283,
+      "logps/chosen": -31.26275062561035,
+      "logps/rejected": -31.31881332397461,
+      "loss": 2.411,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.0678616464138031,
+      "rewards/margins": 0.06804122775793076,
+      "rewards/rejected": -0.00017957761883735657,
       "step": 190
     },
     {
       "epoch": 0.52,
       "learning_rate": 2.7718537898066833e-06,
-      "logits/chosen": -1.9087854623794556,
-      "logits/rejected": -1.9134466648101807,
-      "logps/chosen": -31.110931396484375,
-      "logps/rejected": -32.597984313964844,
-      "loss": 0.4782,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": 0.14860542118549347,
-      "rewards/margins": 0.09072219580411911,
-      "rewards/rejected": 0.05788320302963257,
+      "logits/chosen": -1.9085556268692017,
+      "logits/rejected": -1.9132124185562134,
+      "logps/chosen": -31.326763153076172,
+      "logps/rejected": -32.80078887939453,
+      "loss": 2.2795,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.0838552787899971,
+      "rewards/margins": 0.08681019395589828,
+      "rewards/rejected": -0.002954920055344701,
       "step": 200
     },
     {
       "epoch": 0.52,
-      "eval_logits/chosen": -2.2331366539001465,
-      "eval_logits/rejected": -2.2283167839050293,
-      "eval_logps/chosen": -33.77067184448242,
-      "eval_logps/rejected": -37.30502700805664,
-      "eval_loss": 0.49622178077697754,
-      "eval_rewards/accuracies": 0.5718438625335693,
-      "eval_rewards/chosen": 0.07916475087404251,
-      "eval_rewards/margins": 0.015686888247728348,
-      "eval_rewards/rejected": 0.06347785890102386,
-      "eval_runtime": 145.629,
-      "eval_samples_per_second": 2.355,
+      "eval_logits/chosen": -2.2338554859161377,
+      "eval_logits/rejected": -2.2290101051330566,
+      "eval_logps/chosen": -34.05450439453125,
+      "eval_logps/rejected": -37.565059661865234,
+      "eval_loss": 2.801168918609619,
+      "eval_rewards/accuracies": 0.5278239250183105,
+      "eval_rewards/chosen": -0.005985844414681196,
+      "eval_rewards/margins": 0.0085463160648942,
+      "eval_rewards/rejected": -0.014532160945236683,
+      "eval_runtime": 145.8316,
+      "eval_samples_per_second": 2.352,
       "eval_steps_per_second": 0.295,
       "step": 200
     },
     {
       "epoch": 0.55,
       "learning_rate": 2.5453962426402006e-06,
-      "logits/chosen": -2.020676374435425,
-      "logits/rejected": -2.031282663345337,
-      "logps/chosen": -31.539363861083984,
-      "logps/rejected": -33.701568603515625,
-      "loss": 0.4809,
-      "rewards/accuracies": 0.574999988079071,
-      "rewards/chosen": 0.12958243489265442,
-      "rewards/margins": 0.08124376833438873,
-      "rewards/rejected": 0.04833867400884628,
+      "logits/chosen": -2.020181179046631,
+      "logits/rejected": -2.030860424041748,
+      "logps/chosen": -31.76279067993164,
+      "logps/rejected": -33.952571868896484,
+      "loss": 2.1351,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": 0.06255346536636353,
+      "rewards/margins": 0.08951519429683685,
+      "rewards/rejected": -0.026961728930473328,
       "step": 210
     },
     {
       "epoch": 0.57,
       "learning_rate": 2.3185646976551794e-06,
-      "logits/chosen": -1.9136708974838257,
-      "logits/rejected": -1.9283711910247803,
-      "logps/chosen": -29.599140167236328,
-      "logps/rejected": -31.414936065673828,
-      "loss": 0.4736,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": 0.15372034907341003,
-      "rewards/margins": 0.11074826866388321,
-      "rewards/rejected": 0.04297209903597832,
+      "logits/chosen": -1.912184476852417,
+      "logits/rejected": -1.926975965499878,
+      "logps/chosen": -29.868118286132812,
+      "logps/rejected": -31.57167625427246,
+      "loss": 2.231,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.07302670180797577,
+      "rewards/margins": 0.07707642018795013,
+      "rewards/rejected": -0.004049715120345354,
       "step": 220
     },
     {
       "epoch": 0.6,
       "learning_rate": 2.0932279108998323e-06,
-      "logits/chosen": -1.9707534313201904,
-      "logits/rejected": -1.97475266456604,
-      "logps/chosen": -32.85222625732422,
-      "logps/rejected": -31.408599853515625,
-      "loss": 0.4691,
+      "logits/chosen": -1.968927025794983,
+      "logits/rejected": -1.972905158996582,
+      "logps/chosen": -33.126705169677734,
+      "logps/rejected": -31.619220733642578,
+      "loss": 2.0957,
       "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": 0.16957220435142517,
-      "rewards/margins": 0.12736177444458008,
-      "rewards/rejected": 0.04221044108271599,
+      "rewards/chosen": 0.0872274860739708,
+      "rewards/margins": 0.10820253938436508,
+      "rewards/rejected": -0.02097504958510399,
       "step": 230
     },
     {
       "epoch": 0.62,
       "learning_rate": 1.8712423238279358e-06,
-      "logits/chosen": -1.9696595668792725,
-      "logits/rejected": -1.9479089975357056,
-      "logps/chosen": -33.63511657714844,
-      "logps/rejected": -34.847450256347656,
-      "loss": 0.4709,
-      "rewards/accuracies": 0.6625000238418579,
-      "rewards/chosen": 0.1555902063846588,
-      "rewards/margins": 0.12458709627389908,
-      "rewards/rejected": 0.03100309707224369,
+      "logits/chosen": -1.967507004737854,
+      "logits/rejected": -1.9456119537353516,
+      "logps/chosen": -33.865623474121094,
+      "logps/rejected": -35.0605583190918,
+      "loss": 1.9762,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": 0.08643803745508194,
+      "rewards/margins": 0.11936646699905396,
+      "rewards/rejected": -0.03292842581868172,
       "step": 240
     },
     {
       "epoch": 0.65,
       "learning_rate": 1.6544367689701824e-06,
-      "logits/chosen": -2.0111258029937744,
-      "logits/rejected": -2.0078444480895996,
-      "logps/chosen": -32.480167388916016,
-      "logps/rejected": -35.970458984375,
-      "loss": 0.4866,
-      "rewards/accuracies": 0.6000000238418579,
-      "rewards/chosen": 0.13347108662128448,
-      "rewards/margins": 0.05713240057229996,
-      "rewards/rejected": 0.07633867859840393,
+      "logits/chosen": -2.008720874786377,
+      "logits/rejected": -2.0053982734680176,
+      "logps/chosen": -32.73490524291992,
+      "logps/rejected": -36.2797966003418,
+      "loss": 2.2614,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.05704798549413681,
+      "rewards/margins": 0.07351039350032806,
+      "rewards/rejected": -0.016462400555610657,
       "step": 250
     },
     {
       "epoch": 0.68,
       "learning_rate": 1.4445974030621963e-06,
-      "logits/chosen": -1.8777332305908203,
-      "logits/rejected": -1.8753000497817993,
-      "logps/chosen": -33.69837951660156,
-      "logps/rejected": -35.28581619262695,
-      "loss": 0.4807,
-      "rewards/accuracies": 0.625,
-      "rewards/chosen": 0.14700518548488617,
-      "rewards/margins": 0.08136410266160965,
-      "rewards/rejected": 0.06564109027385712,
+      "logits/chosen": -1.8761491775512695,
+      "logits/rejected": -1.8737146854400635,
+      "logps/chosen": -33.989051818847656,
+      "logps/rejected": -35.516883850097656,
+      "loss": 2.4128,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.05980368331074715,
+      "rewards/margins": 0.06348178535699844,
+      "rewards/rejected": -0.003678102744743228,
       "step": 260
     },
     {
       "epoch": 0.7,
       "learning_rate": 1.243452991757889e-06,
-      "logits/chosen": -1.8637243509292603,
-      "logits/rejected": -1.861201286315918,
-      "logps/chosen": -33.93836212158203,
-      "logps/rejected": -31.57879066467285,
-      "loss": 0.4815,
-      "rewards/accuracies": 0.625,
-      "rewards/chosen": 0.1377706080675125,
-      "rewards/margins": 0.08000828325748444,
-      "rewards/rejected": 0.057762324810028076,
+      "logits/chosen": -1.8607820272445679,
+      "logits/rejected": -1.8583341836929321,
+      "logps/chosen": -34.1697998046875,
+      "logps/rejected": -31.77322769165039,
+      "loss": 2.33,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.06833983957767487,
+      "rewards/margins": 0.06890784204006195,
+      "rewards/rejected": -0.0005679976311512291,
       "step": 270
     },
     {
       "epoch": 0.73,
       "learning_rate": 1.0526606671603523e-06,
-      "logits/chosen": -1.9664865732192993,
-      "logits/rejected": -1.9560563564300537,
-      "logps/chosen": -34.71369934082031,
-      "logps/rejected": -31.626327514648438,
-      "loss": 0.4686,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": 0.19288316369056702,
-      "rewards/margins": 0.12848368287086487,
-      "rewards/rejected": 0.06439947336912155,
+      "logits/chosen": -1.9649995565414429,
+      "logits/rejected": -1.954466462135315,
+      "logps/chosen": -35.01842498779297,
+      "logps/rejected": -31.856042861938477,
+      "loss": 2.0235,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": 0.10146405547857285,
+      "rewards/margins": 0.10597936809062958,
+      "rewards/rejected": -0.004515302833169699,
       "step": 280
     },
     {
       "epoch": 0.75,
       "learning_rate": 8.737922755071455e-07,
-      "logits/chosen": -2.061227560043335,
-      "logits/rejected": -2.0463943481445312,
-      "logps/chosen": -30.41620445251465,
-      "logps/rejected": -32.3445930480957,
-      "loss": 0.4858,
-      "rewards/accuracies": 0.5625,
-      "rewards/chosen": 0.1523759365081787,
-      "rewards/margins": 0.06020249053835869,
-      "rewards/rejected": 0.09217346459627151,
+      "logits/chosen": -2.059657096862793,
+      "logits/rejected": -2.0446982383728027,
+      "logps/chosen": -30.713714599609375,
+      "logps/rejected": -32.61918258666992,
+      "loss": 2.5851,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": 0.06312306225299835,
+      "rewards/margins": 0.0533272810280323,
+      "rewards/rejected": 0.00979578960686922,
       "step": 290
     },
     {
       "epoch": 0.78,
       "learning_rate": 7.08321427484816e-07,
-      "logits/chosen": -1.9324913024902344,
-      "logits/rejected": -1.930048942565918,
-      "logps/chosen": -32.09916305541992,
-      "logps/rejected": -30.68414878845215,
-      "loss": 0.4507,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": 0.24838368594646454,
-      "rewards/margins": 0.20858514308929443,
-      "rewards/rejected": 0.039798520505428314,
+      "logits/chosen": -1.9307676553726196,
+      "logits/rejected": -1.9282405376434326,
+      "logps/chosen": -32.467708587646484,
+      "logps/rejected": -30.857818603515625,
+      "loss": 1.7902,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": 0.1378202736377716,
+      "rewards/margins": 0.15012334287166595,
+      "rewards/rejected": -0.012303064577281475,
       "step": 300
     },
     {
       "epoch": 0.78,
-      "eval_logits/chosen": -2.2302045822143555,
-      "eval_logits/rejected": -2.225385904312134,
-      "eval_logps/chosen": -33.74836349487305,
-      "eval_logps/rejected": -37.28867721557617,
-      "eval_loss": 0.4956430494785309,
-      "eval_rewards/accuracies": 0.5544019937515259,
-      "eval_rewards/chosen": 0.08585640788078308,
-      "eval_rewards/margins": 0.017474040389060974,
-      "eval_rewards/rejected": 0.0683823674917221,
-      "eval_runtime": 145.7803,
-      "eval_samples_per_second": 2.353,
-      "eval_steps_per_second": 0.295,
+      "eval_logits/chosen": -2.2293622493743896,
+      "eval_logits/rejected": -2.2245147228240967,
+      "eval_logps/chosen": -34.04458236694336,
+      "eval_logps/rejected": -37.57244873046875,
+      "eval_loss": 2.758474826812744,
+      "eval_rewards/accuracies": 0.5747508406639099,
+      "eval_rewards/chosen": -0.003010095562785864,
+      "eval_rewards/margins": 0.013738218694925308,
+      "eval_rewards/rejected": -0.01674831472337246,
+      "eval_runtime": 145.5146,
+      "eval_samples_per_second": 2.357,
+      "eval_steps_per_second": 0.296,
       "step": 300
     },
     {
       "epoch": 0.81,
       "learning_rate": 5.576113578589035e-07,
-      "logits/chosen": -1.9161176681518555,
-      "logits/rejected": -1.9129587411880493,
-      "logps/chosen": -31.02998924255371,
-      "logps/rejected": -33.55553436279297,
-      "loss": 0.4729,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": 0.17200829088687897,
-      "rewards/margins": 0.11602197587490082,
-      "rewards/rejected": 0.05598631501197815,
+      "logits/chosen": -1.9147694110870361,
+      "logits/rejected": -1.9114938974380493,
+      "logps/chosen": -31.310047149658203,
+      "logps/rejected": -33.77363967895508,
+      "loss": 2.1768,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.08799003809690475,
+      "rewards/margins": 0.09743582457304001,
+      "rewards/rejected": -0.009445784613490105,
       "step": 310
     },
     {
       "epoch": 0.83,
       "learning_rate": 4.229036944380913e-07,
-      "logits/chosen": -1.9670318365097046,
-      "logits/rejected": -1.9549137353897095,
-      "logps/chosen": -34.00623321533203,
-      "logps/rejected": -33.44996643066406,
-      "loss": 0.4645,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": 0.17695391178131104,
-      "rewards/margins": 0.15029102563858032,
-      "rewards/rejected": 0.026662886142730713,
+      "logits/chosen": -1.965264081954956,
+      "logits/rejected": -1.953029990196228,
+      "logps/chosen": -34.34061050415039,
+      "logps/rejected": -33.643863677978516,
+      "loss": 1.9927,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": 0.07663901150226593,
+      "rewards/margins": 0.10814561694860458,
+      "rewards/rejected": -0.03150660917162895,
       "step": 320
     },
     {
       "epoch": 0.86,
       "learning_rate": 3.053082288996112e-07,
-      "logits/chosen": -2.0021426677703857,
-      "logits/rejected": -2.0008034706115723,
-      "logps/chosen": -32.876285552978516,
-      "logps/rejected": -32.24794387817383,
-      "loss": 0.4728,
-      "rewards/accuracies": 0.675000011920929,
-      "rewards/chosen": 0.1828528493642807,
-      "rewards/margins": 0.11336138099431992,
-      "rewards/rejected": 0.06949146836996078,
+      "logits/chosen": -1.9999011754989624,
+      "logits/rejected": -1.9984591007232666,
+      "logps/chosen": -33.24303436279297,
+      "logps/rejected": -32.54120635986328,
+      "loss": 2.1065,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.0728277713060379,
+      "rewards/margins": 0.09131507575511932,
+      "rewards/rejected": -0.01848730817437172,
       "step": 330
     },
     {
       "epoch": 0.88,
       "learning_rate": 2.0579377374915805e-07,
-      "logits/chosen": -2.089170455932617,
-      "logits/rejected": -2.073537826538086,
-      "logps/chosen": -33.46333694458008,
-      "logps/rejected": -32.803409576416016,
-      "loss": 0.4716,
-      "rewards/accuracies": 0.675000011920929,
-      "rewards/chosen": 0.21481522917747498,
-      "rewards/margins": 0.12042771279811859,
-      "rewards/rejected": 0.09438750147819519,
+      "logits/chosen": -2.086862802505493,
+      "logits/rejected": -2.0711212158203125,
+      "logps/chosen": -33.77810287475586,
+      "logps/rejected": -33.093299865722656,
+      "loss": 2.0709,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.12038693577051163,
+      "rewards/margins": 0.1129666194319725,
+      "rewards/rejected": 0.007420300040394068,
       "step": 340
     },
     {
       "epoch": 0.91,
       "learning_rate": 1.2518018074041684e-07,
-      "logits/chosen": -1.9613116979599,
-      "logits/rejected": -1.9604833126068115,
-      "logps/chosen": -32.53630447387695,
-      "logps/rejected": -32.209129333496094,
-      "loss": 0.4653,
-      "rewards/accuracies": 0.675000011920929,
-      "rewards/chosen": 0.22373780608177185,
-      "rewards/margins": 0.14693202078342438,
-      "rewards/rejected": 0.07680578529834747,
+      "logits/chosen": -1.9597351551055908,
+      "logits/rejected": -1.9588581323623657,
+      "logps/chosen": -32.87625503540039,
+      "logps/rejected": -32.52130889892578,
+      "loss": 1.9515,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": 0.12175308167934418,
+      "rewards/margins": 0.13860100507736206,
+      "rewards/rejected": -0.016847927123308182,
       "step": 350
     },
     {
       "epoch": 0.94,
       "learning_rate": 6.41315865106129e-08,
-      "logits/chosen": -1.9173238277435303,
-      "logits/rejected": -1.9276138544082642,
-      "logps/chosen": -31.568607330322266,
-      "logps/rejected": -34.994293212890625,
-      "loss": 0.4752,
-      "rewards/accuracies": 0.6499999761581421,
-      "rewards/chosen": 0.19082489609718323,
-      "rewards/margins": 0.1024855524301529,
-      "rewards/rejected": 0.08833935856819153,
+      "logits/chosen": -1.9148937463760376,
+      "logits/rejected": -1.9252065420150757,
+      "logps/chosen": -31.9158935546875,
+      "logps/rejected": -35.31604766845703,
+      "loss": 2.1343,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": 0.08664007484912872,
+      "rewards/margins": 0.09482574462890625,
+      "rewards/rejected": -0.008185659535229206,
       "step": 360
     },
     {
       "epoch": 0.96,
       "learning_rate": 2.3150941078050325e-08,
-      "logits/chosen": -2.056678533554077,
-      "logits/rejected": -2.0501868724823,
-      "logps/chosen": -33.03059005737305,
-      "logps/rejected": -28.989059448242188,
-      "loss": 0.4722,
+      "logits/chosen": -2.053968906402588,
+      "logits/rejected": -2.047475576400757,
+      "logps/chosen": -33.36884307861328,
+      "logps/rejected": -29.23018455505371,
+      "loss": 2.1339,
       "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": 0.1864127516746521,
-      "rewards/margins": 0.11520327627658844,
-      "rewards/rejected": 0.07120948284864426,
+      "rewards/chosen": 0.08493683487176895,
+      "rewards/margins": 0.08606470376253128,
+      "rewards/rejected": -0.0011278685415163636,
       "step": 370
     },
     {
       "epoch": 0.99,
       "learning_rate": 2.575864278703266e-09,
-      "logits/chosen": -1.9168630838394165,
-      "logits/rejected": -1.9190342426300049,
-      "logps/chosen": -33.55259323120117,
-      "logps/rejected": -30.7050838470459,
-      "loss": 0.4626,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": 0.20593643188476562,
-      "rewards/margins": 0.1556733250617981,
-      "rewards/rejected": 0.05026308447122574,
+      "logits/chosen": -1.9135916233062744,
+      "logits/rejected": -1.9158084392547607,
+      "logps/chosen": -33.879058837890625,
+      "logps/rejected": -30.930347442626953,
+      "loss": 1.9191,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": 0.10799793899059296,
+      "rewards/margins": 0.12531307339668274,
+      "rewards/rejected": -0.017315123230218887,
       "step": 380
     },
     {
       "epoch": 1.0,
       "step": 385,
       "total_flos": 0.0,
-      "train_loss": 0.48147913697478056,
-      "train_runtime": 3253.5379,
-      "train_samples_per_second": 0.946,
+      "train_loss": 2.3457992528940177,
+      "train_runtime": 3250.6949,
+      "train_samples_per_second": 0.947,
       "train_steps_per_second": 0.118
     }
   ],