Model save

README.md

@@ -26,7 +26,7 @@ print(output["generated_text"])
 
 ## Training procedure
 
-[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="150" height="24"/>](https://wandb.ai/ggbetz/argunauts-training/runs/8ze83nxy) 
+[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="150" height="24"/>](https://wandb.ai/ggbetz/argunauts-training/runs/4qlrj3fp) 
 
 
 This model was trained with DPO, a method introduced in [Direct Preference Optimization: Your Language Model is Secretly a Reward Model](https://huggingface.co/papers/2305.18290).

all_results.json

@@ -1,9 +1,9 @@
 {
-    "epoch": 1.9917920656634747,
+    "epoch": 1.996219281663516,
     "total_flos": 0.0,
-    "train_loss": 0.5634572964448196,
-    "train_runtime": 2180.3656,
-    "train_samples": 5847,
-    "train_samples_per_second": 5.363,
+    "train_loss": 0.5393935610549618,
+    "train_runtime": 2398.307,
+    "train_samples": 6347,
+    "train_samples_per_second": 5.293,
     "train_steps_per_second": 0.083
 }

model-00001-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:809b40494237bab1dd817a75ff18ab2fb2a3c9578448b01143faca1e0d076763
+oid sha256:8f807a55400211dc38fe9871d39bbdb3a1c49f8b79532d4313e1c899fe429e10
 size 4976698672

model-00002-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6ccc099adaf8b970638fb2c3e0bd2553e21c008d9b29600b8c7ece82589f4fa9
+oid sha256:b8dd55c15ea1762fc4462e86e4d79d32fb9df2bf76c79e3c0e53edca6e2575d8
 size 4999802720

model-00003-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:465f0940230790d7a7e95d05f2984afe6f813f177e5f094512b09847f0195f89
+oid sha256:954125f0f6d5961aff8db190d00a9831a874a385c04dedda7d82359384d7233e
 size 4915916176

model-00004-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1fe633a860b536624a0beef90367bed6891825b7db79ad0e5c99fc57a31a532a
+oid sha256:a4c9947d0ccbc72142d87c4ef362037fd555ee53092f6642a644678e9be9d61e
 size 1168138808

train_results.json

@@ -1,9 +1,9 @@
 {
-    "epoch": 1.9917920656634747,
+    "epoch": 1.996219281663516,
     "total_flos": 0.0,
-    "train_loss": 0.5634572964448196,
-    "train_runtime": 2180.3656,
-    "train_samples": 5847,
-    "train_samples_per_second": 5.363,
+    "train_loss": 0.5393935610549618,
+    "train_runtime": 2398.307,
+    "train_samples": 6347,
+    "train_samples_per_second": 5.293,
     "train_steps_per_second": 0.083
 }

trainer_state.json

@@ -1,565 +1,610 @@
 {
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 1.9917920656634747,
+  "epoch": 1.996219281663516,
   "eval_steps": 500,
-  "global_step": 182,
+  "global_step": 198,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
-      "epoch": 0.05471956224350205,
-      "grad_norm": 73.54892222222522,
+      "epoch": 0.05040957781978576,
+      "grad_norm": 108.41279315573027,
       "learning_rate": 1e-07,
       "logits/chosen": NaN,
       "logits/rejected": NaN,
-      "logps/chosen": -379.07501220703125,
-      "logps/rejected": -433.9624938964844,
-      "loss": 0.6968,
-      "rewards/accuracies": 0.18125000596046448,
-      "rewards/chosen": -0.005663680844008923,
-      "rewards/margins": -0.008682822808623314,
-      "rewards/rejected": 0.0030033111106604338,
+      "logps/chosen": -394.29998779296875,
+      "logps/rejected": -490.4984436035156,
+      "loss": 0.6973,
+      "rewards/accuracies": 0.12812499701976776,
+      "rewards/chosen": -0.0023020743392407894,
+      "rewards/margins": -0.009827613830566406,
+      "rewards/rejected": 0.007504081819206476,
       "step": 5
     },
     {
-      "epoch": 0.1094391244870041,
-      "grad_norm": 81.90895583785112,
+      "epoch": 0.10081915563957151,
+      "grad_norm": 87.2429177419689,
       "learning_rate": 2e-07,
       "logits/chosen": NaN,
       "logits/rejected": NaN,
-      "logps/chosen": -353.57342529296875,
-      "logps/rejected": -377.31951904296875,
-      "loss": 0.6873,
-      "rewards/accuracies": 0.27812498807907104,
-      "rewards/chosen": 0.004337215330451727,
-      "rewards/margins": 0.012407870963215828,
-      "rewards/rejected": -0.00807995442301035,
+      "logps/chosen": -374.5687561035156,
+      "logps/rejected": -397.4437561035156,
+      "loss": 0.6827,
+      "rewards/accuracies": 0.30937498807907104,
+      "rewards/chosen": 0.009921550750732422,
+      "rewards/margins": 0.029467200860381126,
+      "rewards/rejected": -0.01954820193350315,
       "step": 10
     },
     {
-      "epoch": 0.16415868673050615,
-      "grad_norm": 49.86519511858418,
-      "learning_rate": 1.9418604651162788e-07,
+      "epoch": 0.15122873345935728,
+      "grad_norm": 83.76505879098643,
+      "learning_rate": 1.946808510638298e-07,
       "logits/chosen": NaN,
       "logits/rejected": NaN,
-      "logps/chosen": -287.00311279296875,
-      "logps/rejected": -337.828125,
-      "loss": 0.6843,
-      "rewards/accuracies": 0.31562501192092896,
-      "rewards/chosen": -0.005427980329841375,
-      "rewards/margins": 0.02440509758889675,
-      "rewards/rejected": -0.029846668243408203,
+      "logps/chosen": -361.41485595703125,
+      "logps/rejected": -441.2671813964844,
+      "loss": 0.6832,
+      "rewards/accuracies": 0.3187499940395355,
+      "rewards/chosen": -0.0191789623349905,
+      "rewards/margins": 0.04044074937701225,
+      "rewards/rejected": -0.059579335153102875,
       "step": 15
     },
     {
-      "epoch": 0.2188782489740082,
-      "grad_norm": 61.602490673284805,
-      "learning_rate": 1.883720930232558e-07,
+      "epoch": 0.20163831127914303,
+      "grad_norm": 55.583186387674,
+      "learning_rate": 1.8936170212765957e-07,
       "logits/chosen": NaN,
       "logits/rejected": NaN,
-      "logps/chosen": -344.2406311035156,
-      "logps/rejected": -458.76251220703125,
-      "loss": 0.6682,
-      "rewards/accuracies": 0.37812501192092896,
-      "rewards/chosen": -0.016874879598617554,
-      "rewards/margins": 0.07516946643590927,
-      "rewards/rejected": -0.09207276999950409,
+      "logps/chosen": -373.5375061035156,
+      "logps/rejected": -464.6875,
+      "loss": 0.6233,
+      "rewards/accuracies": 0.4437499940395355,
+      "rewards/chosen": -0.021530818194150925,
+      "rewards/margins": 0.23837146162986755,
+      "rewards/rejected": -0.2597528398036957,
       "step": 20
     },
     {
-      "epoch": 0.27359781121751026,
-      "grad_norm": 59.029725792223225,
-      "learning_rate": 1.825581395348837e-07,
+      "epoch": 0.2520478890989288,
+      "grad_norm": 60.717924564454115,
+      "learning_rate": 1.8404255319148937e-07,
       "logits/chosen": NaN,
       "logits/rejected": NaN,
-      "logps/chosen": -315.7281188964844,
-      "logps/rejected": -453.23828125,
-      "loss": 0.6414,
-      "rewards/accuracies": 0.48124998807907104,
-      "rewards/chosen": -0.011789416894316673,
-      "rewards/margins": 0.16611814498901367,
-      "rewards/rejected": -0.1779058277606964,
+      "logps/chosen": -371.62188720703125,
+      "logps/rejected": -478.8374938964844,
+      "loss": 0.6328,
+      "rewards/accuracies": 0.3968749940395355,
+      "rewards/chosen": -0.04529209062457085,
+      "rewards/margins": 0.28455600142478943,
+      "rewards/rejected": -0.3299552798271179,
       "step": 25
     },
     {
-      "epoch": 0.3283173734610123,
-      "grad_norm": 47.84812908517649,
-      "learning_rate": 1.7674418604651164e-07,
+      "epoch": 0.30245746691871456,
+      "grad_norm": 58.667280799471335,
+      "learning_rate": 1.7872340425531914e-07,
       "logits/chosen": NaN,
       "logits/rejected": NaN,
-      "logps/chosen": -333.09686279296875,
-      "logps/rejected": -409.4671936035156,
-      "loss": 0.6205,
-      "rewards/accuracies": 0.453125,
-      "rewards/chosen": -0.0324886329472065,
-      "rewards/margins": 0.23987388610839844,
-      "rewards/rejected": -0.27254530787467957,
+      "logps/chosen": -407.125,
+      "logps/rejected": -456.94061279296875,
+      "loss": 0.6037,
+      "rewards/accuracies": 0.4281249940395355,
+      "rewards/chosen": -0.06888346374034882,
+      "rewards/margins": 0.4735303819179535,
+      "rewards/rejected": -0.5425974726676941,
       "step": 30
     },
     {
-      "epoch": 0.38303693570451436,
-      "grad_norm": 52.08971649605601,
-      "learning_rate": 1.7093023255813953e-07,
+      "epoch": 0.35286704473850034,
+      "grad_norm": 52.677451346587404,
+      "learning_rate": 1.7340425531914892e-07,
       "logits/chosen": NaN,
       "logits/rejected": NaN,
-      "logps/chosen": -357.9859313964844,
-      "logps/rejected": -379.1796875,
-      "loss": 0.6341,
-      "rewards/accuracies": 0.4593749940395355,
-      "rewards/chosen": -0.037075091153383255,
-      "rewards/margins": 0.252890020608902,
-      "rewards/rejected": -0.28978270292282104,
+      "logps/chosen": -396.4683532714844,
+      "logps/rejected": -433.375,
+      "loss": 0.5971,
+      "rewards/accuracies": 0.515625,
+      "rewards/chosen": -0.10200033336877823,
+      "rewards/margins": 0.5540359616279602,
+      "rewards/rejected": -0.6558942794799805,
       "step": 35
     },
     {
-      "epoch": 0.4377564979480164,
-      "grad_norm": 45.7725967987622,
-      "learning_rate": 1.6511627906976744e-07,
+      "epoch": 0.40327662255828606,
+      "grad_norm": 61.023691806589966,
+      "learning_rate": 1.6808510638297872e-07,
       "logits/chosen": NaN,
       "logits/rejected": NaN,
-      "logps/chosen": -323.4984436035156,
-      "logps/rejected": -402.7093811035156,
-      "loss": 0.6099,
-      "rewards/accuracies": 0.503125011920929,
-      "rewards/chosen": -0.049513887614011765,
-      "rewards/margins": 0.4467551112174988,
-      "rewards/rejected": -0.496664434671402,
+      "logps/chosen": -349.65545654296875,
+      "logps/rejected": -466.68438720703125,
+      "loss": 0.6065,
+      "rewards/accuracies": 0.578125,
+      "rewards/chosen": -0.10066480934619904,
+      "rewards/margins": 0.6059595346450806,
+      "rewards/rejected": -0.7062518000602722,
       "step": 40
     },
     {
-      "epoch": 0.49247606019151846,
-      "grad_norm": 42.51194464232103,
-      "learning_rate": 1.5930232558139534e-07,
+      "epoch": 0.45368620037807184,
+      "grad_norm": 50.9687221759888,
+      "learning_rate": 1.627659574468085e-07,
       "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": -297.24298095703125,
-      "logps/rejected": -373.6734313964844,
-      "loss": 0.598,
-      "rewards/accuracies": 0.5249999761581421,
-      "rewards/chosen": -0.05229806900024414,
-      "rewards/margins": 0.4699579179286957,
-      "rewards/rejected": -0.5220922231674194,
+      "logits/rejected": -0.1692344695329666,
+      "logps/chosen": -325.9609375,
+      "logps/rejected": -462.97186279296875,
+      "loss": 0.5598,
+      "rewards/accuracies": 0.6031249761581421,
+      "rewards/chosen": -0.09121231734752655,
+      "rewards/margins": 1.08050537109375,
+      "rewards/rejected": -1.1715847253799438,
       "step": 45
     },
     {
-      "epoch": 0.5471956224350205,
-      "grad_norm": 44.14119188922749,
-      "learning_rate": 1.5348837209302325e-07,
+      "epoch": 0.5040957781978576,
+      "grad_norm": 56.54625986325724,
+      "learning_rate": 1.574468085106383e-07,
       "logits/chosen": NaN,
       "logits/rejected": NaN,
-      "logps/chosen": -303.765625,
-      "logps/rejected": -350.6812438964844,
-      "loss": 0.6068,
-      "rewards/accuracies": 0.550000011920929,
-      "rewards/chosen": -0.028664493933320045,
-      "rewards/margins": 0.45641660690307617,
-      "rewards/rejected": -0.48496121168136597,
+      "logps/chosen": -342.57501220703125,
+      "logps/rejected": -478.74139404296875,
+      "loss": 0.591,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.10402297973632812,
+      "rewards/margins": 0.7732677459716797,
+      "rewards/rejected": -0.8772258758544922,
       "step": 50
     },
     {
-      "epoch": 0.6019151846785226,
-      "grad_norm": 45.64723368838045,
-      "learning_rate": 1.4767441860465114e-07,
+      "epoch": 0.5545053560176434,
+      "grad_norm": 52.007398326371984,
+      "learning_rate": 1.5212765957446807e-07,
       "logits/chosen": NaN,
-      "logits/rejected": -0.1685028076171875,
-      "logps/chosen": -335.57501220703125,
-      "logps/rejected": -403.3828125,
-      "loss": 0.5663,
-      "rewards/accuracies": 0.59375,
-      "rewards/chosen": -0.03076200559735298,
-      "rewards/margins": 0.744637668132782,
-      "rewards/rejected": -0.7755187749862671,
+      "logits/rejected": NaN,
+      "logps/chosen": -381.3374938964844,
+      "logps/rejected": -456.14373779296875,
+      "loss": 0.5847,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.08334217220544815,
+      "rewards/margins": 0.8058792352676392,
+      "rewards/rejected": -0.889452338218689,
       "step": 55
     },
     {
-      "epoch": 0.6566347469220246,
-      "grad_norm": 54.68823709160605,
-      "learning_rate": 1.4186046511627906e-07,
+      "epoch": 0.6049149338374291,
+      "grad_norm": 46.410753552087435,
+      "learning_rate": 1.4680851063829787e-07,
       "logits/chosen": NaN,
-      "logits/rejected": -0.2386222779750824,
-      "logps/chosen": -349.6429748535156,
-      "logps/rejected": -427.62188720703125,
-      "loss": 0.5919,
-      "rewards/accuracies": 0.6187499761581421,
-      "rewards/chosen": -0.06507845222949982,
-      "rewards/margins": 0.5661047101020813,
-      "rewards/rejected": -0.6309539675712585,
+      "logits/rejected": NaN,
+      "logps/chosen": -358.54998779296875,
+      "logps/rejected": -475.54998779296875,
+      "loss": 0.5703,
+      "rewards/accuracies": 0.596875011920929,
+      "rewards/chosen": -0.1341991424560547,
+      "rewards/margins": 1.1414505243301392,
+      "rewards/rejected": -1.27621328830719,
       "step": 60
     },
     {
-      "epoch": 0.7113543091655267,
-      "grad_norm": 37.812337714738334,
-      "learning_rate": 1.3604651162790698e-07,
+      "epoch": 0.6553245116572148,
+      "grad_norm": 62.601972259800355,
+      "learning_rate": 1.4148936170212768e-07,
       "logits/chosen": NaN,
       "logits/rejected": NaN,
-      "logps/chosen": -391.65313720703125,
-      "logps/rejected": -404.3343811035156,
-      "loss": 0.5743,
-      "rewards/accuracies": 0.5562499761581421,
-      "rewards/chosen": -0.03319978713989258,
-      "rewards/margins": 0.6765543222427368,
-      "rewards/rejected": -0.7098339200019836,
+      "logps/chosen": -358.22265625,
+      "logps/rejected": -460.4937438964844,
+      "loss": 0.5561,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.09829378128051758,
+      "rewards/margins": 1.154931664466858,
+      "rewards/rejected": -1.25310218334198,
       "step": 65
     },
     {
-      "epoch": 0.7660738714090287,
-      "grad_norm": 43.16858074168923,
-      "learning_rate": 1.302325581395349e-07,
+      "epoch": 0.7057340894770007,
+      "grad_norm": 42.81446910830512,
+      "learning_rate": 1.3617021276595742e-07,
       "logits/chosen": NaN,
       "logits/rejected": NaN,
-      "logps/chosen": -348.0687561035156,
-      "logps/rejected": -420.6000061035156,
-      "loss": 0.5578,
-      "rewards/accuracies": 0.574999988079071,
-      "rewards/chosen": -0.002912330674007535,
-      "rewards/margins": 0.7157382965087891,
-      "rewards/rejected": -0.7183740735054016,
+      "logps/chosen": -350.8062438964844,
+      "logps/rejected": -479.98126220703125,
+      "loss": 0.5464,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -0.08439864963293076,
+      "rewards/margins": 1.0637038946151733,
+      "rewards/rejected": -1.146966576576233,
       "step": 70
     },
     {
-      "epoch": 0.8207934336525308,
-      "grad_norm": 46.03417938024923,
-      "learning_rate": 1.244186046511628e-07,
+      "epoch": 0.7561436672967864,
+      "grad_norm": 59.85465445918704,
+      "learning_rate": 1.3085106382978723e-07,
       "logits/chosen": NaN,
       "logits/rejected": NaN,
-      "logps/chosen": -346.75,
-      "logps/rejected": -368.25,
-      "loss": 0.5502,
-      "rewards/accuracies": 0.653124988079071,
-      "rewards/chosen": -0.01859130896627903,
-      "rewards/margins": 0.7938125729560852,
-      "rewards/rejected": -0.8118951916694641,
+      "logps/chosen": -339.3656311035156,
+      "logps/rejected": -428.65936279296875,
+      "loss": 0.5872,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.11314620822668076,
+      "rewards/margins": 0.7480255365371704,
+      "rewards/rejected": -0.8610885739326477,
       "step": 75
     },
     {
-      "epoch": 0.8755129958960328,
-      "grad_norm": 45.68840453931342,
-      "learning_rate": 1.186046511627907e-07,
+      "epoch": 0.8065532451165721,
+      "grad_norm": 77.89913196854276,
+      "learning_rate": 1.25531914893617e-07,
       "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": -373.9671936035156,
-      "logps/rejected": -470.2250061035156,
-      "loss": 0.55,
-      "rewards/accuracies": 0.609375,
-      "rewards/chosen": -0.04471855238080025,
-      "rewards/margins": 0.8136627078056335,
-      "rewards/rejected": -0.8582122921943665,
+      "logits/rejected": -0.232859805226326,
+      "logps/chosen": -367.71563720703125,
+      "logps/rejected": -496.9125061035156,
+      "loss": 0.5386,
+      "rewards/accuracies": 0.6968749761581421,
+      "rewards/chosen": -0.09202079474925995,
+      "rewards/margins": 1.1276824474334717,
+      "rewards/rejected": -1.2203514575958252,
       "step": 80
     },
     {
-      "epoch": 0.9302325581395349,
-      "grad_norm": 46.25513385846438,
-      "learning_rate": 1.127906976744186e-07,
+      "epoch": 0.856962822936358,
+      "grad_norm": 51.576853346068596,
+      "learning_rate": 1.202127659574468e-07,
       "logits/chosen": NaN,
       "logits/rejected": NaN,
-      "logps/chosen": -306.0453186035156,
-      "logps/rejected": -382.3812561035156,
-      "loss": 0.5732,
-      "rewards/accuracies": 0.640625,
-      "rewards/chosen": -0.0332220084965229,
-      "rewards/margins": 0.7451133728027344,
-      "rewards/rejected": -0.7781906127929688,
+      "logps/chosen": -352.01873779296875,
+      "logps/rejected": -441.85626220703125,
+      "loss": 0.5489,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.09004707634449005,
+      "rewards/margins": 0.9977798461914062,
+      "rewards/rejected": -1.0878921747207642,
       "step": 85
     },
     {
-      "epoch": 0.9849521203830369,
-      "grad_norm": 43.05942874643264,
-      "learning_rate": 1.069767441860465e-07,
+      "epoch": 0.9073724007561437,
+      "grad_norm": 51.91948881188928,
+      "learning_rate": 1.148936170212766e-07,
       "logits/chosen": NaN,
       "logits/rejected": NaN,
-      "logps/chosen": -339.6312561035156,
-      "logps/rejected": -404.6328125,
-      "loss": 0.5659,
-      "rewards/accuracies": 0.612500011920929,
-      "rewards/chosen": -0.06772689521312714,
-      "rewards/margins": 0.7259882092475891,
-      "rewards/rejected": -0.7933288812637329,
+      "logps/chosen": -391.98748779296875,
+      "logps/rejected": -501.89373779296875,
+      "loss": 0.5429,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -0.13425922393798828,
+      "rewards/margins": 1.2032638788223267,
+      "rewards/rejected": -1.339324951171875,
       "step": 90
     },
     {
-      "epoch": 1.039671682626539,
-      "grad_norm": 46.27154296619937,
-      "learning_rate": 1.0116279069767442e-07,
+      "epoch": 0.9577819785759294,
+      "grad_norm": 54.81194235391722,
+      "learning_rate": 1.0957446808510638e-07,
       "logits/chosen": NaN,
       "logits/rejected": NaN,
-      "logps/chosen": -338.83123779296875,
-      "logps/rejected": -414.30938720703125,
-      "loss": 0.5471,
-      "rewards/accuracies": 0.6625000238418579,
-      "rewards/chosen": -0.0271759033203125,
-      "rewards/margins": 0.8508437871932983,
-      "rewards/rejected": -0.8778969049453735,
+      "logps/chosen": -380.8531188964844,
+      "logps/rejected": -485.4375,
+      "loss": 0.5506,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.123291015625,
+      "rewards/margins": 1.137838363647461,
+      "rewards/rejected": -1.2604999542236328,
       "step": 95
     },
     {
-      "epoch": 1.094391244870041,
-      "grad_norm": 37.78475339484884,
-      "learning_rate": 9.534883720930232e-08,
+      "epoch": 1.0081915563957151,
+      "grad_norm": 51.52692462343263,
+      "learning_rate": 1.0425531914893617e-07,
       "logits/chosen": NaN,
       "logits/rejected": NaN,
-      "logps/chosen": -331.1796875,
-      "logps/rejected": -435.3179626464844,
-      "loss": 0.5351,
-      "rewards/accuracies": 0.6968749761581421,
-      "rewards/chosen": -0.01332016009837389,
-      "rewards/margins": 0.7190505862236023,
-      "rewards/rejected": -0.7319396734237671,
+      "logps/chosen": -357.92498779296875,
+      "logps/rejected": -420.00311279296875,
+      "loss": 0.5433,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.118899405002594,
+      "rewards/margins": 1.2265655994415283,
+      "rewards/rejected": -1.3462097644805908,
       "step": 100
     },
     {
-      "epoch": 1.1491108071135432,
-      "grad_norm": 53.87671855674201,
-      "learning_rate": 8.953488372093023e-08,
+      "epoch": 1.0586011342155008,
+      "grad_norm": 48.23673648203366,
+      "learning_rate": 9.893617021276596e-08,
       "logits/chosen": NaN,
       "logits/rejected": NaN,
-      "logps/chosen": -359.4312438964844,
-      "logps/rejected": -363.98748779296875,
-      "loss": 0.5338,
-      "rewards/accuracies": 0.671875,
-      "rewards/chosen": 0.01817016676068306,
-      "rewards/margins": 0.8886383175849915,
-      "rewards/rejected": -0.8702591061592102,
+      "logps/chosen": -420.59375,
+      "logps/rejected": -551.4156494140625,
+      "loss": 0.4711,
+      "rewards/accuracies": 0.715624988079071,
+      "rewards/chosen": -0.05655860900878906,
+      "rewards/margins": 1.65283203125,
+      "rewards/rejected": -1.7101104259490967,
       "step": 105
     },
     {
-      "epoch": 1.2038303693570451,
-      "grad_norm": 37.1769031620794,
-      "learning_rate": 8.372093023255815e-08,
+      "epoch": 1.1090107120352868,
+      "grad_norm": 129.8626362436788,
+      "learning_rate": 9.361702127659574e-08,
       "logits/chosen": NaN,
       "logits/rejected": NaN,
-      "logps/chosen": -320.11798095703125,
-      "logps/rejected": -372.38751220703125,
-      "loss": 0.5261,
+      "logps/chosen": -375.9306640625,
+      "logps/rejected": -410.8500061035156,
+      "loss": 0.5499,
       "rewards/accuracies": 0.706250011920929,
-      "rewards/chosen": 0.015477180480957031,
-      "rewards/margins": 0.9761615991592407,
-      "rewards/rejected": -0.9608657956123352,
+      "rewards/chosen": -0.10003051906824112,
+      "rewards/margins": 0.8063720464706421,
+      "rewards/rejected": -0.906982421875,
       "step": 110
     },
     {
-      "epoch": 1.2585499316005473,
-      "grad_norm": 42.09569803865549,
-      "learning_rate": 7.790697674418605e-08,
+      "epoch": 1.1594202898550725,
+      "grad_norm": 37.96460632662538,
+      "learning_rate": 8.829787234042553e-08,
       "logits/chosen": NaN,
       "logits/rejected": NaN,
-      "logps/chosen": -358.69219970703125,
-      "logps/rejected": -394.2437438964844,
-      "loss": 0.5333,
-      "rewards/accuracies": 0.703125,
-      "rewards/chosen": -0.0025016784202307463,
-      "rewards/margins": 0.9062668085098267,
-      "rewards/rejected": -0.9096938967704773,
+      "logps/chosen": -395.4546813964844,
+      "logps/rejected": -534.5437622070312,
+      "loss": 0.4536,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -0.0355035774409771,
+      "rewards/margins": 1.6931426525115967,
+      "rewards/rejected": -1.7283508777618408,
       "step": 115
     },
     {
-      "epoch": 1.3132694938440492,
-      "grad_norm": 36.918397640269696,
-      "learning_rate": 7.209302325581394e-08,
+      "epoch": 1.2098298676748582,
+      "grad_norm": 41.75349141526311,
+      "learning_rate": 8.297872340425531e-08,
       "logits/chosen": NaN,
-      "logits/rejected": NaN,
-      "logps/chosen": -322.95001220703125,
-      "logps/rejected": -362.4125061035156,
-      "loss": 0.5178,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": 0.02965698204934597,
-      "rewards/margins": 1.0709717273712158,
-      "rewards/rejected": -1.0418059825897217,
+      "logits/rejected": -0.24345549941062927,
+      "logps/chosen": -337.83905029296875,
+      "logps/rejected": -414.4765625,
+      "loss": 0.4953,
+      "rewards/accuracies": 0.796875,
+      "rewards/chosen": -0.09084253013134003,
+      "rewards/margins": 1.2055069208145142,
+      "rewards/rejected": -1.2963898181915283,
       "step": 120
     },
     {
-      "epoch": 1.3679890560875512,
-      "grad_norm": 41.25871091693466,
-      "learning_rate": 6.627906976744185e-08,
+      "epoch": 1.260239445494644,
+      "grad_norm": 47.57568984542951,
+      "learning_rate": 7.76595744680851e-08,
       "logits/chosen": NaN,
       "logits/rejected": NaN,
-      "logps/chosen": -347.3968811035156,
-      "logps/rejected": -478.6343688964844,
-      "loss": 0.5056,
-      "rewards/accuracies": 0.737500011920929,
-      "rewards/chosen": 0.011506843380630016,
-      "rewards/margins": 1.1476867198944092,
-      "rewards/rejected": -1.135986328125,
+      "logps/chosen": -358.9046936035156,
+      "logps/rejected": -469.49688720703125,
+      "loss": 0.501,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.09036216884851456,
+      "rewards/margins": 1.231683373451233,
+      "rewards/rejected": -1.3224579095840454,
       "step": 125
     },
     {
-      "epoch": 1.4227086183310533,
-      "grad_norm": 38.50680509320898,
-      "learning_rate": 6.046511627906976e-08,
+      "epoch": 1.3106490233144297,
+      "grad_norm": 53.944830915941765,
+      "learning_rate": 7.23404255319149e-08,
       "logits/chosen": NaN,
       "logits/rejected": NaN,
-      "logps/chosen": -289.20623779296875,
-      "logps/rejected": -385.5179748535156,
-      "loss": 0.5277,
-      "rewards/accuracies": 0.690625011920929,
-      "rewards/chosen": 0.012903976254165173,
-      "rewards/margins": 0.8722091913223267,
-      "rewards/rejected": -0.8594962954521179,
+      "logps/chosen": -356.4375,
+      "logps/rejected": -489.125,
+      "loss": 0.4999,
+      "rewards/accuracies": 0.7593749761581421,
+      "rewards/chosen": -0.0903778076171875,
+      "rewards/margins": 1.2274360656738281,
+      "rewards/rejected": -1.3174560070037842,
       "step": 130
     },
     {
-      "epoch": 1.4774281805745555,
-      "grad_norm": 47.9748923473641,
-      "learning_rate": 5.465116279069767e-08,
+      "epoch": 1.3610586011342156,
+      "grad_norm": 40.45839840494219,
+      "learning_rate": 6.702127659574469e-08,
       "logits/chosen": NaN,
       "logits/rejected": NaN,
-      "logps/chosen": -328.2406311035156,
-      "logps/rejected": -399.08123779296875,
-      "loss": 0.524,
-      "rewards/accuracies": 0.715624988079071,
-      "rewards/chosen": -0.015491103753447533,
-      "rewards/margins": 1.0500564575195312,
-      "rewards/rejected": -1.0659011602401733,
+      "logps/chosen": -368.95623779296875,
+      "logps/rejected": -460.58282470703125,
+      "loss": 0.4856,
+      "rewards/accuracies": 0.746874988079071,
+      "rewards/chosen": -0.033612824976444244,
+      "rewards/margins": 1.4452941417694092,
+      "rewards/rejected": -1.479437232017517,
       "step": 135
     },
     {
-      "epoch": 1.5321477428180574,
-      "grad_norm": 40.692911236757624,
-      "learning_rate": 4.883720930232558e-08,
+      "epoch": 1.4114681789540013,
+      "grad_norm": 43.63902778407327,
+      "learning_rate": 6.170212765957446e-08,
       "logits/chosen": NaN,
       "logits/rejected": NaN,
-      "logps/chosen": -374.1187438964844,
-      "logps/rejected": -411.32501220703125,
-      "loss": 0.4871,
-      "rewards/accuracies": 0.7093750238418579,
-      "rewards/chosen": 0.05074119567871094,
-      "rewards/margins": 1.3334617614746094,
-      "rewards/rejected": -1.2823364734649658,
+      "logps/chosen": -393.62030029296875,
+      "logps/rejected": -473.53436279296875,
+      "loss": 0.4987,
+      "rewards/accuracies": 0.734375,
+      "rewards/chosen": -0.03872375562787056,
+      "rewards/margins": 1.239990234375,
+      "rewards/rejected": -1.2792266607284546,
       "step": 140
     },
     {
-      "epoch": 1.5868673050615594,
-      "grad_norm": 40.76205645702533,
-      "learning_rate": 4.3023255813953484e-08,
+      "epoch": 1.461877756773787,
+      "grad_norm": 40.93200179183777,
+      "learning_rate": 5.638297872340425e-08,
       "logits/chosen": NaN,
-      "logits/rejected": -0.15314331650733948,
-      "logps/chosen": -364.9437561035156,
-      "logps/rejected": -455.7124938964844,
-      "loss": 0.4725,
-      "rewards/accuracies": 0.765625,
-      "rewards/chosen": 0.03847331926226616,
-      "rewards/margins": 1.328369140625,
-      "rewards/rejected": -1.2897827625274658,
+      "logits/rejected": -0.28594666719436646,
+      "logps/chosen": -370.62188720703125,
+      "logps/rejected": -508.2562561035156,
+      "loss": 0.4891,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.047638703137636185,
+      "rewards/margins": 1.3118622303009033,
+      "rewards/rejected": -1.359655737876892,
       "step": 145
     },
     {
-      "epoch": 1.6415868673050615,
-      "grad_norm": 46.69293795008511,
-      "learning_rate": 3.7209302325581396e-08,
+      "epoch": 1.5122873345935728,
+      "grad_norm": 58.65309693932161,
+      "learning_rate": 5.106382978723404e-08,
       "logits/chosen": NaN,
       "logits/rejected": NaN,
-      "logps/chosen": -338.5874938964844,
-      "logps/rejected": -408.1656188964844,
-      "loss": 0.5119,
-      "rewards/accuracies": 0.7093750238418579,
-      "rewards/chosen": 0.028449058532714844,
-      "rewards/margins": 0.9527389407157898,
-      "rewards/rejected": -0.9238342046737671,
+      "logps/chosen": -347.8609313964844,
+      "logps/rejected": -473.4750061035156,
+      "loss": 0.4814,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -0.05366211012005806,
+      "rewards/margins": 1.2643524408340454,
+      "rewards/rejected": -1.3187682628631592,
       "step": 150
     },
     {
-      "epoch": 1.6963064295485637,
-      "grad_norm": 42.042163782049805,
-      "learning_rate": 3.13953488372093e-08,
+      "epoch": 1.5626969124133585,
+      "grad_norm": 40.5990127283572,
+      "learning_rate": 4.5744680851063826e-08,
       "logits/chosen": NaN,
-      "logits/rejected": -0.19841155409812927,
-      "logps/chosen": -352.875,
-      "logps/rejected": -418.7562561035156,
-      "loss": 0.519,
-      "rewards/accuracies": 0.721875011920929,
-      "rewards/chosen": 0.008855698630213737,
-      "rewards/margins": 1.1339629888534546,
-      "rewards/rejected": -1.1251556873321533,
+      "logits/rejected": NaN,
+      "logps/chosen": -377.1734313964844,
+      "logps/rejected": -453.3296813964844,
+      "loss": 0.4998,
+      "rewards/accuracies": 0.753125011920929,
+      "rewards/chosen": -0.07384242862462997,
+      "rewards/margins": 1.2074543237686157,
+      "rewards/rejected": -1.2816162109375,
       "step": 155
     },
     {
-      "epoch": 1.7510259917920656,
-      "grad_norm": 45.03337894308017,
-      "learning_rate": 2.5581395348837208e-08,
+      "epoch": 1.6131064902331445,
+      "grad_norm": 32.42610640703847,
+      "learning_rate": 4.0425531914893614e-08,
       "logits/chosen": NaN,
       "logits/rejected": NaN,
-      "logps/chosen": -355.38751220703125,
-      "logps/rejected": -383.7406311035156,
-      "loss": 0.5215,
-      "rewards/accuracies": 0.6968749761581421,
-      "rewards/chosen": -0.0020130157936364412,
-      "rewards/margins": 0.9867599606513977,
-      "rewards/rejected": -0.9886184930801392,
+      "logps/chosen": -330.2578125,
+      "logps/rejected": -398.8812561035156,
+      "loss": 0.471,
+      "rewards/accuracies": 0.7906249761581421,
+      "rewards/chosen": -0.01856536790728569,
+      "rewards/margins": 1.3712249994277954,
+      "rewards/rejected": -1.3899352550506592,
       "step": 160
     },
     {
-      "epoch": 1.8057455540355676,
-      "grad_norm": 39.86335782741774,
-      "learning_rate": 1.9767441860465116e-08,
+      "epoch": 1.66351606805293,
+      "grad_norm": 41.01530072372472,
+      "learning_rate": 3.51063829787234e-08,
       "logits/chosen": NaN,
       "logits/rejected": NaN,
-      "logps/chosen": -328.53790283203125,
-      "logps/rejected": -387.0218811035156,
-      "loss": 0.5333,
-      "rewards/accuracies": 0.6468750238418579,
-      "rewards/chosen": -0.010569858364760876,
-      "rewards/margins": 0.907672107219696,
-      "rewards/rejected": -0.9181579351425171,
+      "logps/chosen": -340.2749938964844,
+      "logps/rejected": -438.40313720703125,
+      "loss": 0.4842,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.0492522232234478,
+      "rewards/margins": 1.4962584972381592,
+      "rewards/rejected": -1.5454528331756592,
       "step": 165
     },
     {
-      "epoch": 1.8604651162790697,
-      "grad_norm": 44.87998150984873,
-      "learning_rate": 1.3953488372093022e-08,
+      "epoch": 1.713925645872716,
+      "grad_norm": 40.64613349590343,
+      "learning_rate": 2.9787234042553187e-08,
       "logits/chosen": NaN,
-      "logits/rejected": -0.20475158095359802,
-      "logps/chosen": -318.3793029785156,
-      "logps/rejected": -401.95001220703125,
-      "loss": 0.5183,
-      "rewards/accuracies": 0.734375,
-      "rewards/chosen": 0.05231628566980362,
-      "rewards/margins": 0.9515264630317688,
-      "rewards/rejected": -0.899548351764679,
+      "logits/rejected": NaN,
+      "logps/chosen": -348.5640563964844,
+      "logps/rejected": -449.84844970703125,
+      "loss": 0.4915,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -0.07779388129711151,
+      "rewards/margins": 1.278845191001892,
+      "rewards/rejected": -1.356591820716858,
       "step": 170
     },
     {
-      "epoch": 1.915184678522572,
-      "grad_norm": 44.78373504452225,
-      "learning_rate": 8.139534883720931e-09,
+      "epoch": 1.7643352236925016,
+      "grad_norm": 41.86507518705149,
+      "learning_rate": 2.4468085106382976e-08,
       "logits/chosen": NaN,
       "logits/rejected": NaN,
-      "logps/chosen": -362.09375,
-      "logps/rejected": -410.56329345703125,
-      "loss": 0.512,
-      "rewards/accuracies": 0.6968749761581421,
-      "rewards/chosen": -0.0063323974609375,
-      "rewards/margins": 1.0748703479766846,
-      "rewards/rejected": -1.0807387828826904,
+      "logps/chosen": -350.2046813964844,
+      "logps/rejected": -433.5687561035156,
+      "loss": 0.4755,
+      "rewards/accuracies": 0.8031250238418579,
+      "rewards/chosen": -0.027071380987763405,
+      "rewards/margins": 1.4462082386016846,
+      "rewards/rejected": -1.4738037586212158,
       "step": 175
     },
     {
-      "epoch": 1.9699042407660738,
-      "grad_norm": 47.232513404054075,
-      "learning_rate": 2.3255813953488372e-09,
+      "epoch": 1.8147448015122873,
+      "grad_norm": 46.67757830253006,
+      "learning_rate": 1.9148936170212764e-08,
       "logits/chosen": NaN,
       "logits/rejected": NaN,
-      "logps/chosen": -326.29608154296875,
-      "logps/rejected": -434.44061279296875,
-      "loss": 0.5055,
-      "rewards/accuracies": 0.731249988079071,
-      "rewards/chosen": 0.015459060668945312,
-      "rewards/margins": 1.0265671014785767,
-      "rewards/rejected": -1.0105316638946533,
+      "logps/chosen": -398.32501220703125,
+      "logps/rejected": -456.9437561035156,
+      "loss": 0.4807,
+      "rewards/accuracies": 0.746874988079071,
+      "rewards/chosen": -0.05414886400103569,
+      "rewards/margins": 1.2570632696151733,
+      "rewards/rejected": -1.310980200767517,
       "step": 180
     },
     {
-      "epoch": 1.9917920656634747,
-      "step": 182,
+      "epoch": 1.865154379332073,
+      "grad_norm": 43.636875496682755,
+      "learning_rate": 1.3829787234042552e-08,
+      "logits/chosen": NaN,
+      "logits/rejected": -0.17527160048484802,
+      "logps/chosen": -373.87188720703125,
+      "logps/rejected": -535.796875,
+      "loss": 0.4376,
+      "rewards/accuracies": 0.796875,
+      "rewards/chosen": -0.026004791259765625,
+      "rewards/margins": 1.6229279041290283,
+      "rewards/rejected": -1.649743676185608,
+      "step": 185
+    },
+    {
+      "epoch": 1.9155639571518588,
+      "grad_norm": 41.753332033000945,
+      "learning_rate": 8.510638297872339e-09,
+      "logits/chosen": NaN,
+      "logits/rejected": NaN,
+      "logps/chosen": -360.0718688964844,
+      "logps/rejected": -468.2124938964844,
+      "loss": 0.4872,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -0.11498375236988068,
+      "rewards/margins": 1.2304840087890625,
+      "rewards/rejected": -1.345800757408142,
+      "step": 190
+    },
+    {
+      "epoch": 1.9659735349716447,
+      "grad_norm": 43.377545693326894,
+      "learning_rate": 3.1914893617021273e-09,
+      "logits/chosen": NaN,
+      "logits/rejected": NaN,
+      "logps/chosen": -365.515625,
+      "logps/rejected": -478.56561279296875,
+      "loss": 0.4805,
+      "rewards/accuracies": 0.7718750238418579,
+      "rewards/chosen": -0.04032173007726669,
+      "rewards/margins": 1.3543853759765625,
+      "rewards/rejected": -1.39520263671875,
+      "step": 195
+    },
+    {
+      "epoch": 1.996219281663516,
+      "step": 198,
       "total_flos": 0.0,
-      "train_loss": 0.5634572964448196,
-      "train_runtime": 2180.3656,
-      "train_samples_per_second": 5.363,
+      "train_loss": 0.5393935610549618,
+      "train_runtime": 2398.307,
+      "train_samples_per_second": 5.293,
       "train_steps_per_second": 0.083
     }
   ],
   "logging_steps": 5,
-  "max_steps": 182,
+  "max_steps": 198,
   "num_input_tokens_seen": 0,
   "num_train_epochs": 2,
   "save_steps": 50,

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f8fdbbe2f1b3f3b85296b6baf64f326cf244b75414a339603d903d8f6b145663
+oid sha256:04392d39f00478df0417f7fd9ba7d15085f3dba381f41d035a42742e674b52ff
 size 7672