End of training

README.md

@@ -16,7 +16,7 @@ should probably proofread and complete it, then remove this comment. -->
 
 This model is a fine-tuned version of [google/flan-t5-small](https://huggingface.co/google/flan-t5-small) on the None dataset.
 It achieves the following results on the evaluation set:
-- Loss: 1.6025
+- Loss: 0.1072
 
 ## Model description
 
@@ -39,12 +39,29 @@ The following hyperparameters were used during training:
 - train_batch_size: 16
 - eval_batch_size: 16
 - seed: 42
+- distributed_type: multi-GPU
+- num_devices: 8
+- total_train_batch_size: 128
+- total_eval_batch_size: 128
 - optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
 - lr_scheduler_type: linear
-- num_epochs: 0.001
+- num_epochs: 10.0
 
 ### Training results
 
+| Training Loss | Epoch | Step | Validation Loss |
+|:-------------:|:-----:|:----:|:---------------:|
+| 0.4003        | 0.9   | 500  | 0.1629          |
+| 0.2314        | 1.81  | 1000 | 0.1386          |
+| 0.2065        | 2.71  | 1500 | 0.1289          |
+| 0.187         | 3.62  | 2000 | 0.1233          |
+| 0.1791        | 4.52  | 2500 | 0.1169          |
+| 0.1713        | 5.42  | 3000 | 0.1153          |
+| 0.1661        | 6.33  | 3500 | 0.1122          |
+| 0.1604        | 7.23  | 4000 | 0.1085          |
+| 0.1574        | 8.14  | 4500 | 0.1099          |
+| 0.1541        | 9.04  | 5000 | 0.1064          |
+| 0.1521        | 9.95  | 5500 | 0.1071          |
 
 
 ### Framework versions

adapter_config.json

@@ -19,8 +19,8 @@
   "rank_pattern": {},
   "revision": null,
   "target_modules": [
-    "q",
-    "v"
+    "v",
+    "q"
   ],
   "task_type": "SEQ_2_SEQ_LM"
 }

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:088861e13048d008a84b5bcea780b9d9f5ad691da24ba9da77ef998a2c522c2f
+oid sha256:4af10a22bd170f6a681891351deeaa4edddd58bc62b4d0732320b6f5fcd408fd
 size 2765880

all_results.json

@@ -1,11 +1,11 @@
 {
-    "epoch": 0.0,
-    "eval_loss": 1.6024976968765259,
-    "eval_runtime": 13.4905,
-    "eval_samples_per_second": 582.482,
-    "eval_steps_per_second": 36.47,
-    "train_loss": 2.5696409225463865,
-    "train_runtime": 1.0509,
-    "train_samples_per_second": 67.292,
-    "train_steps_per_second": 4.758
+    "epoch": 10.0,
+    "eval_loss": 0.10718318819999695,
+    "eval_runtime": 3.0141,
+    "eval_samples_per_second": 2607.118,
+    "eval_steps_per_second": 20.57,
+    "train_loss": 0.19658087959772425,
+    "train_runtime": 755.9435,
+    "train_samples_per_second": 935.506,
+    "train_steps_per_second": 7.315
 }

eval_results.json

@@ -1,7 +1,7 @@
 {
-    "epoch": 0.0,
-    "eval_loss": 1.6024976968765259,
-    "eval_runtime": 13.4905,
-    "eval_samples_per_second": 582.482,
-    "eval_steps_per_second": 36.47
+    "epoch": 10.0,
+    "eval_loss": 0.10718318819999695,
+    "eval_runtime": 3.0141,
+    "eval_samples_per_second": 2607.118,
+    "eval_steps_per_second": 20.57
 }

train_results.json

@@ -1,7 +1,7 @@
 {
-    "epoch": 0.0,
-    "train_loss": 2.5696409225463865,
-    "train_runtime": 1.0509,
-    "train_samples_per_second": 67.292,
-    "train_steps_per_second": 4.758
+    "epoch": 10.0,
+    "train_loss": 0.19658087959772425,
+    "train_runtime": 755.9435,
+    "train_samples_per_second": 935.506,
+    "train_steps_per_second": 7.315
 }

trainer_state.json

@@ -1,29 +1,194 @@
 {
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 0.0011312217194570137,
+  "epoch": 10.0,
   "eval_steps": 500,
-  "global_step": 5,
+  "global_step": 5530,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
-      "epoch": 0.0,
-      "step": 5,
-      "total_flos": 2585060966400.0,
-      "train_loss": 2.5696409225463865,
-      "train_runtime": 1.0509,
-      "train_samples_per_second": 67.292,
-      "train_steps_per_second": 4.758
+      "epoch": 0.9,
+      "grad_norm": 0.1664367914199829,
+      "learning_rate": 0.0009095840867992767,
+      "loss": 0.4003,
+      "step": 500
+    },
+    {
+      "epoch": 0.9,
+      "eval_loss": 0.162927508354187,
+      "eval_runtime": 3.0318,
+      "eval_samples_per_second": 2591.854,
+      "eval_steps_per_second": 20.45,
+      "step": 500
+    },
+    {
+      "epoch": 1.81,
+      "grad_norm": 0.1386815905570984,
+      "learning_rate": 0.0008191681735985533,
+      "loss": 0.2314,
+      "step": 1000
+    },
+    {
+      "epoch": 1.81,
+      "eval_loss": 0.1386137157678604,
+      "eval_runtime": 3.0434,
+      "eval_samples_per_second": 2582.007,
+      "eval_steps_per_second": 20.372,
+      "step": 1000
+    },
+    {
+      "epoch": 2.71,
+      "grad_norm": 0.1781063824892044,
+      "learning_rate": 0.0007287522603978301,
+      "loss": 0.2065,
+      "step": 1500
+    },
+    {
+      "epoch": 2.71,
+      "eval_loss": 0.1289130598306656,
+      "eval_runtime": 3.0677,
+      "eval_samples_per_second": 2561.542,
+      "eval_steps_per_second": 20.211,
+      "step": 1500
+    },
+    {
+      "epoch": 3.62,
+      "grad_norm": 0.15570929646492004,
+      "learning_rate": 0.0006383363471971068,
+      "loss": 0.187,
+      "step": 2000
+    },
+    {
+      "epoch": 3.62,
+      "eval_loss": 0.12326223403215408,
+      "eval_runtime": 3.0605,
+      "eval_samples_per_second": 2567.579,
+      "eval_steps_per_second": 20.258,
+      "step": 2000
+    },
+    {
+      "epoch": 4.52,
+      "grad_norm": 0.16776247322559357,
+      "learning_rate": 0.0005479204339963833,
+      "loss": 0.1791,
+      "step": 2500
+    },
+    {
+      "epoch": 4.52,
+      "eval_loss": 0.1168670803308487,
+      "eval_runtime": 3.0473,
+      "eval_samples_per_second": 2578.705,
+      "eval_steps_per_second": 20.346,
+      "step": 2500
+    },
+    {
+      "epoch": 5.42,
+      "grad_norm": 0.1355486512184143,
+      "learning_rate": 0.0004575045207956601,
+      "loss": 0.1713,
+      "step": 3000
+    },
+    {
+      "epoch": 5.42,
+      "eval_loss": 0.11528698354959488,
+      "eval_runtime": 3.0013,
+      "eval_samples_per_second": 2618.163,
+      "eval_steps_per_second": 20.657,
+      "step": 3000
+    },
+    {
+      "epoch": 6.33,
+      "grad_norm": 0.16372531652450562,
+      "learning_rate": 0.0003670886075949367,
+      "loss": 0.1661,
+      "step": 3500
+    },
+    {
+      "epoch": 6.33,
+      "eval_loss": 0.11218289285898209,
+      "eval_runtime": 2.9586,
+      "eval_samples_per_second": 2655.959,
+      "eval_steps_per_second": 20.956,
+      "step": 3500
+    },
+    {
+      "epoch": 7.23,
+      "grad_norm": 0.1596778929233551,
+      "learning_rate": 0.0002766726943942134,
+      "loss": 0.1604,
+      "step": 4000
+    },
+    {
+      "epoch": 7.23,
+      "eval_loss": 0.1085081547498703,
+      "eval_runtime": 2.9539,
+      "eval_samples_per_second": 2660.243,
+      "eval_steps_per_second": 20.989,
+      "step": 4000
+    },
+    {
+      "epoch": 8.14,
+      "grad_norm": 0.15582768619060516,
+      "learning_rate": 0.00018625678119349006,
+      "loss": 0.1574,
+      "step": 4500
+    },
+    {
+      "epoch": 8.14,
+      "eval_loss": 0.1098729744553566,
+      "eval_runtime": 2.9739,
+      "eval_samples_per_second": 2642.311,
+      "eval_steps_per_second": 20.848,
+      "step": 4500
+    },
+    {
+      "epoch": 9.04,
+      "grad_norm": 0.15063905715942383,
+      "learning_rate": 9.584086799276672e-05,
+      "loss": 0.1541,
+      "step": 5000
+    },
+    {
+      "epoch": 9.04,
+      "eval_loss": 0.10638037323951721,
+      "eval_runtime": 3.0665,
+      "eval_samples_per_second": 2562.534,
+      "eval_steps_per_second": 20.219,
+      "step": 5000
+    },
+    {
+      "epoch": 9.95,
+      "grad_norm": 0.14130930602550507,
+      "learning_rate": 5.4249547920433995e-06,
+      "loss": 0.1521,
+      "step": 5500
+    },
+    {
+      "epoch": 9.95,
+      "eval_loss": 0.1071261540055275,
+      "eval_runtime": 3.024,
+      "eval_samples_per_second": 2598.51,
+      "eval_steps_per_second": 20.502,
+      "step": 5500
+    },
+    {
+      "epoch": 10.0,
+      "step": 5530,
+      "total_flos": 2.2872619342626816e+16,
+      "train_loss": 0.19658087959772425,
+      "train_runtime": 755.9435,
+      "train_samples_per_second": 935.506,
+      "train_steps_per_second": 7.315
     }
   ],
   "logging_steps": 500,
-  "max_steps": 5,
+  "max_steps": 5530,
   "num_input_tokens_seen": 0,
-  "num_train_epochs": 1,
+  "num_train_epochs": 10,
   "save_steps": 500,
-  "total_flos": 2585060966400.0,
+  "total_flos": 2.2872619342626816e+16,
   "train_batch_size": 16,
   "trial_name": null,
   "trial_params": null

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e5964aa42a58f9aa0442e4ab87d38a471e1527ac951e91bea56c4d8f43acfd26
+oid sha256:1791b93576383f366ada713fe62fa1a5066567f1c635c3f329bc8f36e8673a58
 size 5048