Model save

README.md

@@ -27,7 +27,7 @@ print(output["generated_text"])
 
 ## Training procedure
 
-[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="150" height="24"/>](https://wandb.ai/myronorg/huggingface/runs/r98aklh1) 
+[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="150" height="24"/>](https://wandb.ai/myronorg/huggingface/runs/omfw97f8) 
 
 
 This model was trained with SFT.
@@ -36,7 +36,7 @@ This model was trained with SFT.
 
 - TRL: 0.15.0.dev0
 - Transformers: 4.49.0.dev0
-- Pytorch: 2.5.1+cu121
+- Pytorch: 2.5.1
 - Datasets: 3.2.0
 - Tokenizers: 0.21.0
 

all_results.json

@@ -1,9 +1,9 @@
 {
-    "epoch": 1.0,
-    "total_flos": 76916824473600.0,
-    "train_loss": 0.8026671296746067,
-    "train_runtime": 1041.4464,
+    "epoch": 0.9948186528497409,
+    "total_flos": 76517996494848.0,
+    "train_loss": 0.8023917121546609,
+    "train_runtime": 1111.3885,
     "train_samples": 16610,
-    "train_samples_per_second": 20.75,
-    "train_steps_per_second": 0.162
+    "train_samples_per_second": 19.444,
+    "train_steps_per_second": 0.151
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:cd46c4f7659b2fe3bbf379e42dd83f61c77b26195e748cd3db4e58c8cc8e89cb
+oid sha256:e45c9331ae85fc04b1df2a41a9aa2a752fb463e7a58932c68e5cc4e2d9c19407
 size 3087467144

train_results.json

@@ -1,9 +1,9 @@
 {
-    "epoch": 1.0,
-    "total_flos": 76916824473600.0,
-    "train_loss": 0.8026671296746067,
-    "train_runtime": 1041.4464,
+    "epoch": 0.9948186528497409,
+    "total_flos": 76517996494848.0,
+    "train_loss": 0.8023917121546609,
+    "train_runtime": 1111.3885,
     "train_samples": 16610,
-    "train_samples_per_second": 20.75,
-    "train_steps_per_second": 0.162
+    "train_samples_per_second": 19.444,
+    "train_steps_per_second": 0.151
 }

trainer_state.json

@@ -1,264 +1,264 @@
 {
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 1.0,
+  "epoch": 0.9948186528497409,
   "eval_steps": 100,
-  "global_step": 169,
+  "global_step": 168,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
-      "epoch": 0.029585798816568046,
-      "grad_norm": 2.342892423396605,
+      "epoch": 0.029607698001480384,
+      "grad_norm": 2.3427891450842355,
       "learning_rate": 5.882352941176471e-06,
-      "loss": 1.0992,
+      "loss": 1.0993,
       "step": 5
     },
     {
-      "epoch": 0.05917159763313609,
-      "grad_norm": 1.6368371604135565,
+      "epoch": 0.05921539600296077,
+      "grad_norm": 1.638241993747605,
       "learning_rate": 1.1764705882352942e-05,
       "loss": 1.0402,
       "step": 10
     },
     {
-      "epoch": 0.08875739644970414,
-      "grad_norm": 0.8989769231311087,
+      "epoch": 0.08882309400444116,
+      "grad_norm": 0.8970481588482149,
       "learning_rate": 1.7647058823529414e-05,
       "loss": 0.9533,
       "step": 15
     },
     {
-      "epoch": 0.11834319526627218,
-      "grad_norm": 0.6137256595940042,
-      "learning_rate": 1.9980782984658682e-05,
+      "epoch": 0.11843079200592153,
+      "grad_norm": 0.6126224631372397,
+      "learning_rate": 1.9980527694749952e-05,
       "loss": 0.8922,
       "step": 20
     },
     {
-      "epoch": 0.14792899408284024,
-      "grad_norm": 0.5492009965568149,
-      "learning_rate": 1.9863613034027224e-05,
+      "epoch": 0.14803849000740193,
+      "grad_norm": 0.5491637741037821,
+      "learning_rate": 1.986180478852149e-05,
       "loss": 0.8551,
       "step": 25
     },
     {
-      "epoch": 0.17751479289940827,
-      "grad_norm": 0.4349573184458919,
-      "learning_rate": 1.9641197940012136e-05,
-      "loss": 0.8283,
+      "epoch": 0.17764618800888232,
+      "grad_norm": 0.4357332212949219,
+      "learning_rate": 1.963645895935632e-05,
+      "loss": 0.8282,
       "step": 30
     },
     {
-      "epoch": 0.20710059171597633,
-      "grad_norm": 0.38659663740930433,
-      "learning_rate": 1.9315910880512792e-05,
+      "epoch": 0.20725388601036268,
+      "grad_norm": 0.3870290405518921,
+      "learning_rate": 1.930692657985482e-05,
       "loss": 0.8229,
       "step": 35
     },
     {
-      "epoch": 0.23668639053254437,
-      "grad_norm": 0.4119151414508529,
-      "learning_rate": 1.8891222681391853e-05,
+      "epoch": 0.23686158401184307,
+      "grad_norm": 0.41071978450812635,
+      "learning_rate": 1.887677045685188e-05,
       "loss": 0.8225,
       "step": 40
     },
     {
-      "epoch": 0.26627218934911245,
-      "grad_norm": 0.35458480206676984,
-      "learning_rate": 1.8371664782625287e-05,
+      "epoch": 0.2664692820133235,
+      "grad_norm": 0.35511835037971484,
+      "learning_rate": 1.8350641311400813e-05,
       "loss": 0.8073,
       "step": 45
     },
     {
-      "epoch": 0.2958579881656805,
-      "grad_norm": 0.34430708054184417,
-      "learning_rate": 1.7762780887657576e-05,
+      "epoch": 0.29607698001480387,
+      "grad_norm": 0.34562914008715157,
+      "learning_rate": 1.773422749654988e-05,
       "loss": 0.7977,
       "step": 50
     },
     {
-      "epoch": 0.3254437869822485,
-      "grad_norm": 0.3838154059326915,
-      "learning_rate": 1.7071067811865477e-05,
+      "epoch": 0.32568467801628426,
+      "grad_norm": 0.38429745030506335,
+      "learning_rate": 1.7034193496547903e-05,
       "loss": 0.7877,
       "step": 55
     },
     {
-      "epoch": 0.35502958579881655,
-      "grad_norm": 0.383532491946636,
-      "learning_rate": 1.6303906161279554e-05,
-      "loss": 0.7981,
+      "epoch": 0.35529237601776464,
+      "grad_norm": 0.38450100811350485,
+      "learning_rate": 1.6258107872407376e-05,
+      "loss": 0.7982,
       "step": 60
     },
     {
-      "epoch": 0.38461538461538464,
-      "grad_norm": 0.35083975989542854,
-      "learning_rate": 1.5469481581224274e-05,
+      "epoch": 0.38490007401924503,
+      "grad_norm": 0.3508167601846127,
+      "learning_rate": 1.5414361432856475e-05,
       "loss": 0.7722,
       "step": 65
     },
     {
-      "epoch": 0.41420118343195267,
-      "grad_norm": 0.35395203723670016,
-      "learning_rate": 1.4576697415156818e-05,
+      "epoch": 0.41450777202072536,
+      "grad_norm": 0.354273236485806,
+      "learning_rate": 1.4512076515391375e-05,
       "loss": 0.7741,
       "step": 70
     },
     {
-      "epoch": 0.4437869822485207,
-      "grad_norm": 0.34555629817939354,
-      "learning_rate": 1.3635079705638298e-05,
+      "epoch": 0.44411547002220575,
+      "grad_norm": 0.3435732479373877,
+      "learning_rate": 1.356100835825547e-05,
       "loss": 0.7852,
       "step": 75
     },
     {
-      "epoch": 0.47337278106508873,
-      "grad_norm": 0.35805756615265727,
-      "learning_rate": 1.2654675551080724e-05,
+      "epoch": 0.47372316802368614,
+      "grad_norm": 0.3554552572459275,
+      "learning_rate": 1.257143962968246e-05,
       "loss": 0.7667,
       "step": 80
     },
     {
-      "epoch": 0.5029585798816568,
-      "grad_norm": 0.3781865827387411,
-      "learning_rate": 1.164594590280734e-05,
-      "loss": 0.7706,
+      "epoch": 0.5033308660251665,
+      "grad_norm": 0.37807154489345807,
+      "learning_rate": 1.155406925472205e-05,
+      "loss": 0.7707,
       "step": 85
     },
     {
-      "epoch": 0.5325443786982249,
-      "grad_norm": 0.36455865865784565,
-      "learning_rate": 1.0619653946285948e-05,
+      "epoch": 0.532938564026647,
+      "grad_norm": 0.3667445569072502,
+      "learning_rate": 1.0519896741619803e-05,
       "loss": 0.7701,
       "step": 90
     },
     {
-      "epoch": 0.5621301775147929,
-      "grad_norm": 0.37271134883355717,
-      "learning_rate": 9.586750257511868e-06,
+      "epoch": 0.5625462620281273,
+      "grad_norm": 0.3813236840767762,
+      "learning_rate": 9.480103258380198e-06,
       "loss": 0.7772,
       "step": 95
     },
     {
-      "epoch": 0.591715976331361,
-      "grad_norm": 0.3429534883848641,
-      "learning_rate": 8.558255959926533e-06,
-      "loss": 0.753,
+      "epoch": 0.5921539600296077,
+      "grad_norm": 0.34559010442801097,
+      "learning_rate": 8.445930745277953e-06,
+      "loss": 0.7531,
       "step": 100
     },
     {
-      "epoch": 0.591715976331361,
-      "eval_loss": 0.7881382703781128,
-      "eval_runtime": 1.6822,
-      "eval_samples_per_second": 76.093,
-      "eval_steps_per_second": 2.378,
+      "epoch": 0.5921539600296077,
+      "eval_loss": 0.7882063388824463,
+      "eval_runtime": 1.6572,
+      "eval_samples_per_second": 77.237,
+      "eval_steps_per_second": 2.414,
       "step": 100
     },
     {
-      "epoch": 0.621301775147929,
-      "grad_norm": 0.34272683406564375,
-      "learning_rate": 7.545145128592009e-06,
-      "loss": 0.757,
+      "epoch": 0.6217616580310881,
+      "grad_norm": 0.341364069372392,
+      "learning_rate": 7.428560370317542e-06,
+      "loss": 0.7571,
       "step": 105
     },
     {
-      "epoch": 0.650887573964497,
-      "grad_norm": 0.35326011905545884,
-      "learning_rate": 6.558227696373617e-06,
+      "epoch": 0.6513693560325685,
+      "grad_norm": 0.3559969343392935,
+      "learning_rate": 6.438991641744531e-06,
       "loss": 0.7682,
       "step": 110
     },
     {
-      "epoch": 0.6804733727810651,
-      "grad_norm": 0.3175593796339319,
-      "learning_rate": 5.608034111526298e-06,
-      "loss": 0.7623,
+      "epoch": 0.6809770540340488,
+      "grad_norm": 0.3161514819141605,
+      "learning_rate": 5.487923484608629e-06,
+      "loss": 0.7625,
       "step": 115
     },
     {
-      "epoch": 0.7100591715976331,
-      "grad_norm": 0.30113559390386235,
-      "learning_rate": 4.704702977392914e-06,
-      "loss": 0.7514,
+      "epoch": 0.7105847520355293,
+      "grad_norm": 0.30339451601959244,
+      "learning_rate": 4.5856385671435285e-06,
+      "loss": 0.7515,
       "step": 120
     },
     {
-      "epoch": 0.7396449704142012,
-      "grad_norm": 0.3514515497713594,
-      "learning_rate": 3.857872873103322e-06,
-      "loss": 0.7538,
+      "epoch": 0.7401924500370096,
+      "grad_norm": 0.35261640845743664,
+      "learning_rate": 3.7418921275926245e-06,
+      "loss": 0.7539,
       "step": 125
     },
     {
-      "epoch": 0.7692307692307693,
-      "grad_norm": 0.32077249449415507,
-      "learning_rate": 3.0765795095517026e-06,
-      "loss": 0.7555,
+      "epoch": 0.7698001480384901,
+      "grad_norm": 0.3193239804674359,
+      "learning_rate": 2.965806503452098e-06,
+      "loss": 0.7556,
       "step": 130
     },
     {
-      "epoch": 0.7988165680473372,
-      "grad_norm": 0.32980480359237835,
-      "learning_rate": 2.369159318001937e-06,
-      "loss": 0.7585,
+      "epoch": 0.7994078460399704,
+      "grad_norm": 0.32912058451560244,
+      "learning_rate": 2.265772503450122e-06,
+      "loss": 0.7586,
       "step": 135
     },
     {
-      "epoch": 0.8284023668639053,
-      "grad_norm": 0.30552938976565036,
-      "learning_rate": 1.743160500034443e-06,
-      "loss": 0.7498,
+      "epoch": 0.8290155440414507,
+      "grad_norm": 0.303685802235117,
+      "learning_rate": 1.6493586885991908e-06,
+      "loss": 0.75,
       "step": 140
     },
     {
-      "epoch": 0.8579881656804734,
-      "grad_norm": 0.3199948980261291,
-      "learning_rate": 1.2052624879351105e-06,
-      "loss": 0.7566,
+      "epoch": 0.8586232420429312,
+      "grad_norm": 0.3173990670917867,
+      "learning_rate": 1.1232295431481222e-06,
+      "loss": 0.7568,
       "step": 145
     },
     {
-      "epoch": 0.8875739644970414,
-      "grad_norm": 0.3029762272084133,
-      "learning_rate": 7.612046748871327e-07,
-      "loss": 0.7666,
+      "epoch": 0.8882309400444115,
+      "grad_norm": 0.3010954114203951,
+      "learning_rate": 6.930734201451817e-07,
+      "loss": 0.7668,
       "step": 150
     },
     {
-      "epoch": 0.9171597633136095,
-      "grad_norm": 0.30367592763134704,
-      "learning_rate": 4.1572517541747294e-07,
-      "loss": 0.7613,
+      "epoch": 0.9178386380458919,
+      "grad_norm": 0.30232823482832394,
+      "learning_rate": 3.635410406436857e-07,
+      "loss": 0.7616,
       "step": 155
     },
     {
-      "epoch": 0.9467455621301775,
-      "grad_norm": 0.29238016566832153,
-      "learning_rate": 1.7251026952640583e-07,
-      "loss": 0.7607,
+      "epoch": 0.9474463360473723,
+      "grad_norm": 0.28964694770453414,
+      "learning_rate": 1.3819521147851122e-07,
+      "loss": 0.7609,
       "step": 160
     },
     {
-      "epoch": 0.9763313609467456,
-      "grad_norm": 0.2814543563664885,
-      "learning_rate": 3.4155069933301535e-08,
-      "loss": 0.7446,
+      "epoch": 0.9770540340488527,
+      "grad_norm": 0.28146626492833826,
+      "learning_rate": 1.947230525005006e-08,
+      "loss": 0.7448,
       "step": 165
     },
     {
-      "epoch": 1.0,
-      "step": 169,
-      "total_flos": 76916824473600.0,
-      "train_loss": 0.8026671296746067,
-      "train_runtime": 1041.4464,
-      "train_samples_per_second": 20.75,
-      "train_steps_per_second": 0.162
+      "epoch": 0.9948186528497409,
+      "step": 168,
+      "total_flos": 76517996494848.0,
+      "train_loss": 0.8023917121546609,
+      "train_runtime": 1111.3885,
+      "train_samples_per_second": 19.444,
+      "train_steps_per_second": 0.151
     }
   ],
   "logging_steps": 5,
-  "max_steps": 169,
+  "max_steps": 168,
   "num_input_tokens_seen": 0,
   "num_train_epochs": 1,
   "save_steps": 500,
@@ -274,8 +274,8 @@
       "attributes": {}
     }
   },
-  "total_flos": 76916824473600.0,
-  "train_batch_size": 4,
+  "total_flos": 76517996494848.0,
+  "train_batch_size": 2,
   "trial_name": null,
   "trial_params": null
 }

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0041f765e83070c411e294af2c5822a097d32d77ef14a249a448aa887d9f0a53
-size 7352
+oid sha256:26ec61c7b0fd94fbbbbf8ec1c09add723b9f6175b65e350bab79108a82d4defe
+size 7416