Upload 6 files

config.json

@@ -5,7 +5,7 @@
     "GPT2LMHeadModel"
   ],
   "attn_pdrop": 0.1,
-  "bos_token_id": 100256,
+  "bos_token_id": 50256,
   "config": {
     "activation_function": "gelu_new",
     "attn_pdrop": 0.1,
@@ -13,18 +13,18 @@
     "gradient_checkpointing": true,
     "initializer_range": 0.02,
     "layer_norm_epsilon": 1e-05,
-    "n_ctx": 1080,
-    "n_embd": 1080,
-    "n_head": 22,
+    "n_ctx": 2048,
+    "n_embd": 2048,
+    "n_head": 16,
     "n_layer": 36,
-    "n_positions": 1080,
+    "n_positions": 2048,
     "resid_pdrop": 0.1,
     "scale_attn_weights": true,
     "use_cache": true,
-    "vocab_size": 100257
+    "vocab_size": 50257
   },
   "embd_pdrop": 0.1,
-  "eos_token_id": 100256,
+  "eos_token_id": 50256,
   "initializer_range": 0.02,
   "language": "en",
   "layer_norm_epsilon": 1e-05,
@@ -35,10 +35,10 @@
     "accuracy"
   ],
   "model_type": "gpt2",
-  "n_embd": 1000,
-  "n_head": 15,
+  "n_embd": 768,
+  "n_head": 12,
   "n_inner": null,
-  "n_layer": 15,
+  "n_layer": 12,
   "n_positions": 1024,
   "pipeline_tag": "conversational",
   "reorder_and_upcast_attn": false,
@@ -71,14 +71,14 @@
     }
   },
   "tokenizer_config": {
-    "bos_token_id": 100256,
+    "bos_token_id": 50256,
     "eos_token_id": 50256,
-    "n_positions": 1080,
+    "n_positions": 2048,
     "padding_side": "left",
     "truncation_side": "right"
   },
   "torch_dtype": "float32",
   "transformers_version": "4.37.2",
   "use_cache": true,
-  "vocab_size": 100257
+  "vocab_size": 50257
 }

trainer_state.json

@@ -1,7 +1,7 @@
 {
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 2.1491510853212983,
+  "epoch": 2.144082332761578,
   "eval_steps": 500,
   "global_step": 10000,
   "is_hyper_param_search": false,
@@ -10,131 +10,131 @@
   "log_history": [
     {
       "epoch": 0.11,
-      "learning_rate": 4.820904076223225e-05,
-      "loss": 1.1625,
+      "learning_rate": 4.951270856073601e-05,
+      "loss": 1.4381,
       "step": 500
     },
     {
       "epoch": 0.21,
-      "learning_rate": 4.641808152446451e-05,
-      "loss": 1.1918,
+      "learning_rate": 4.902541712147201e-05,
+      "loss": 1.3041,
       "step": 1000
     },
     {
       "epoch": 0.32,
-      "learning_rate": 4.462712228669675e-05,
-      "loss": 1.2037,
+      "learning_rate": 4.853812568220802e-05,
+      "loss": 1.2816,
       "step": 1500
     },
     {
       "epoch": 0.43,
-      "learning_rate": 4.283616304892901e-05,
-      "loss": 1.2352,
+      "learning_rate": 4.805083424294402e-05,
+      "loss": 1.2974,
       "step": 2000
     },
     {
       "epoch": 0.54,
-      "learning_rate": 4.104520381116126e-05,
-      "loss": 1.2375,
+      "learning_rate": 4.756354280368003e-05,
+      "loss": 1.2876,
       "step": 2500
     },
     {
       "epoch": 0.64,
-      "learning_rate": 3.9254244573393514e-05,
-      "loss": 1.2305,
+      "learning_rate": 4.707625136441603e-05,
+      "loss": 1.3029,
       "step": 3000
     },
     {
       "epoch": 0.75,
-      "learning_rate": 3.746328533562576e-05,
-      "loss": 1.2822,
+      "learning_rate": 4.6588959925152037e-05,
+      "loss": 1.301,
       "step": 3500
     },
     {
       "epoch": 0.86,
-      "learning_rate": 3.567232609785802e-05,
-      "loss": 1.2733,
+      "learning_rate": 4.610166848588804e-05,
+      "loss": 1.3153,
       "step": 4000
     },
     {
-      "epoch": 0.97,
-      "learning_rate": 3.388136686009026e-05,
-      "loss": 1.2598,
+      "epoch": 0.96,
+      "learning_rate": 4.5614377046624046e-05,
+      "loss": 1.2989,
       "step": 4500
     },
     {
       "epoch": 1.07,
-      "learning_rate": 3.209040762232252e-05,
-      "loss": 1.1006,
+      "learning_rate": 4.512708560736005e-05,
+      "loss": 1.1589,
       "step": 5000
     },
     {
       "epoch": 1.18,
-      "learning_rate": 3.029944838455477e-05,
-      "loss": 1.0509,
+      "learning_rate": 4.4639794168096056e-05,
+      "loss": 1.1053,
       "step": 5500
     },
     {
       "epoch": 1.29,
-      "learning_rate": 2.850848914678702e-05,
-      "loss": 1.0549,
+      "learning_rate": 4.415250272883206e-05,
+      "loss": 1.1314,
       "step": 6000
     },
     {
-      "epoch": 1.4,
-      "learning_rate": 2.6717529909019275e-05,
-      "loss": 1.0917,
+      "epoch": 1.39,
+      "learning_rate": 4.3665211289568066e-05,
+      "loss": 1.1446,
       "step": 6500
     },
     {
       "epoch": 1.5,
-      "learning_rate": 2.4926570671251524e-05,
-      "loss": 1.1086,
+      "learning_rate": 4.317791985030407e-05,
+      "loss": 1.1701,
       "step": 7000
     },
     {
       "epoch": 1.61,
-      "learning_rate": 2.3135611433483774e-05,
-      "loss": 1.0815,
+      "learning_rate": 4.269062841104008e-05,
+      "loss": 1.1702,
       "step": 7500
     },
     {
       "epoch": 1.72,
-      "learning_rate": 2.1344652195716027e-05,
-      "loss": 1.0995,
+      "learning_rate": 4.220333697177608e-05,
+      "loss": 1.1764,
       "step": 8000
     },
     {
-      "epoch": 1.83,
-      "learning_rate": 1.9553692957948277e-05,
-      "loss": 1.1132,
+      "epoch": 1.82,
+      "learning_rate": 4.1716045532512085e-05,
+      "loss": 1.1933,
       "step": 8500
     },
     {
       "epoch": 1.93,
-      "learning_rate": 1.776273372018053e-05,
-      "loss": 1.075,
+      "learning_rate": 4.122875409324809e-05,
+      "loss": 1.1964,
       "step": 9000
     },
     {
       "epoch": 2.04,
-      "learning_rate": 1.5971774482412783e-05,
-      "loss": 1.054,
+      "learning_rate": 4.0741462653984095e-05,
+      "loss": 1.1274,
       "step": 9500
     },
     {
-      "epoch": 2.15,
-      "learning_rate": 1.4180815244645032e-05,
-      "loss": 0.9583,
+      "epoch": 2.14,
+      "learning_rate": 4.025417121472011e-05,
+      "loss": 1.0048,
       "step": 10000
     }
   ],
   "logging_steps": 500,
-  "max_steps": 13959,
+  "max_steps": 51304,
   "num_input_tokens_seen": 0,
-  "num_train_epochs": 3,
+  "num_train_epochs": 11,
   "save_steps": 10000,
-  "total_flos": 2.0902317391872e+16,
+  "total_flos": 2.0901272223744e+16,
   "train_batch_size": 8,
   "trial_name": null,
   "trial_params": null