Training in progress, step 360, checkpoint

last-checkpoint/adapter_config.json

@@ -21,6 +21,7 @@
   "revision": null,
   "target_modules": [
     "word_embeddings",
+    "dense_h_to_4h",
     "query_key_valuelm_head"
   ],
   "task_type": "CAUSAL_LM",

last-checkpoint/adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e02434f63bb76768d0764fb21761fc6bc5a8c5697635718d6ef1c79a814a08e1
-size 8077608
+oid sha256:55fbe6b68371476a49a5dcf47cb5f9d87019cedff7dbfeba90f3bd7c5af98334
+size 13982248

last-checkpoint/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:71b11e37189ca2b87525395bedccb828279727662e1a1b44bc3e761c6ac638b2
-size 4052500
+oid sha256:8dc0adaf7cde965efef091b643e2b3d0711b0799235ea64091a8320ceeb4047b
+size 7062522

last-checkpoint/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:384b1441690cc32db0a09ece90eb10816e434402fb6deac7559104dc40fcf585
+oid sha256:a768f08cb42c9ce59ce3607be711941a2de1bfe32b4b4e516fea093f13486afb
 size 14244

last-checkpoint/trainer_state.json

@@ -1,7 +1,7 @@
 {
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 0.0072,
+  "epoch": 0.0288,
   "eval_steps": 500,
   "global_step": 360,
   "is_hyper_param_search": false,
@@ -10,138 +10,138 @@
   "log_history": [
     {
       "epoch": 0.0,
-      "grad_norm": 79.12922668457031,
+      "grad_norm": 62.10089111328125,
       "learning_rate": 1e-06,
-      "loss": 4.6771,
+      "loss": 4.5777,
       "step": 20
     },
     {
       "epoch": 0.0,
-      "grad_norm": 239.7991943359375,
+      "grad_norm": 39.39016342163086,
       "learning_rate": 2e-06,
-      "loss": 4.5023,
+      "loss": 4.4077,
       "step": 40
     },
     {
       "epoch": 0.0,
-      "grad_norm": 77.5164794921875,
+      "grad_norm": 54.24020767211914,
       "learning_rate": 3e-06,
-      "loss": 4.8171,
+      "loss": 4.4807,
       "step": 60
     },
     {
-      "epoch": 0.0,
-      "grad_norm": 207.87796020507812,
+      "epoch": 0.01,
+      "grad_norm": 30.161609649658203,
       "learning_rate": 4e-06,
-      "loss": 4.6765,
+      "loss": 4.5756,
       "step": 80
     },
     {
-      "epoch": 0.0,
-      "grad_norm": 147.12461853027344,
+      "epoch": 0.01,
+      "grad_norm": 40.131675720214844,
       "learning_rate": 4.9999999999999996e-06,
-      "loss": 4.7237,
+      "loss": 4.4352,
       "step": 100
     },
     {
-      "epoch": 0.0,
-      "grad_norm": 115.61023712158203,
+      "epoch": 0.01,
+      "grad_norm": 52.3621940612793,
       "learning_rate": 6e-06,
-      "loss": 4.4426,
+      "loss": 4.5096,
       "step": 120
     },
     {
-      "epoch": 0.0,
-      "grad_norm": 43.68627166748047,
+      "epoch": 0.01,
+      "grad_norm": 49.86561584472656,
       "learning_rate": 7e-06,
-      "loss": 5.0142,
+      "loss": 4.493,
       "step": 140
     },
     {
-      "epoch": 0.0,
-      "grad_norm": 37.58155059814453,
+      "epoch": 0.01,
+      "grad_norm": 20.034923553466797,
       "learning_rate": 8e-06,
-      "loss": 4.8572,
+      "loss": 4.4088,
       "step": 160
     },
     {
-      "epoch": 0.0,
-      "grad_norm": 86.82076263427734,
+      "epoch": 0.01,
+      "grad_norm": 50.790679931640625,
       "learning_rate": 9e-06,
-      "loss": 4.6848,
+      "loss": 4.4901,
       "step": 180
     },
     {
-      "epoch": 0.0,
-      "grad_norm": 289.0603332519531,
+      "epoch": 0.02,
+      "grad_norm": 48.5693473815918,
       "learning_rate": 9.999999999999999e-06,
-      "loss": 4.6474,
+      "loss": 4.3628,
       "step": 200
     },
     {
-      "epoch": 0.0,
-      "grad_norm": 69.93185424804688,
+      "epoch": 0.02,
+      "grad_norm": 37.95353698730469,
       "learning_rate": 1.1e-05,
-      "loss": 4.9724,
+      "loss": 4.3298,
       "step": 220
     },
     {
-      "epoch": 0.0,
-      "grad_norm": 98.215087890625,
+      "epoch": 0.02,
+      "grad_norm": 35.7153434753418,
       "learning_rate": 1.2e-05,
-      "loss": 4.4447,
+      "loss": 4.2839,
       "step": 240
     },
     {
-      "epoch": 0.01,
-      "grad_norm": 92.3516845703125,
+      "epoch": 0.02,
+      "grad_norm": 91.47773742675781,
       "learning_rate": 1.3000000000000001e-05,
-      "loss": 4.9076,
+      "loss": 4.1238,
       "step": 260
     },
     {
-      "epoch": 0.01,
-      "grad_norm": 150.6816864013672,
+      "epoch": 0.02,
+      "grad_norm": 23.16193389892578,
       "learning_rate": 1.4e-05,
-      "loss": 4.8402,
+      "loss": 4.1245,
       "step": 280
     },
     {
-      "epoch": 0.01,
-      "grad_norm": 162.9401397705078,
+      "epoch": 0.02,
+      "grad_norm": 28.304485321044922,
       "learning_rate": 1.5e-05,
-      "loss": 4.9279,
+      "loss": 4.2198,
       "step": 300
     },
     {
-      "epoch": 0.01,
-      "grad_norm": 38.77900695800781,
+      "epoch": 0.03,
+      "grad_norm": 34.03230285644531,
       "learning_rate": 1.6e-05,
-      "loss": 4.5021,
+      "loss": 4.0958,
       "step": 320
     },
     {
-      "epoch": 0.01,
-      "grad_norm": 91.0234146118164,
+      "epoch": 0.03,
+      "grad_norm": 29.786975860595703,
       "learning_rate": 1.7e-05,
-      "loss": 4.5546,
+      "loss": 4.024,
       "step": 340
     },
     {
-      "epoch": 0.01,
-      "grad_norm": 78.24981689453125,
+      "epoch": 0.03,
+      "grad_norm": 33.04754638671875,
       "learning_rate": 1.8e-05,
-      "loss": 4.3795,
+      "loss": 4.0832,
       "step": 360
     }
   ],
   "logging_steps": 20,
   "max_steps": 20000,
   "num_input_tokens_seen": 0,
-  "num_train_epochs": 1,
+  "num_train_epochs": 2,
   "save_steps": 20,
-  "total_flos": 127576999796736.0,
-  "train_batch_size": 2,
+  "total_flos": 865682421055488.0,
+  "train_batch_size": 8,
   "trial_name": null,
   "trial_params": null
 }

last-checkpoint/training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:32e994f25267341e613f4d352977cc7a2847de358db5ece7a60fcf21be944170
+oid sha256:29d6911b5aeefa0beece74e38b3ce4711e31d40f8c9b6627972f1c5a74e68732
 size 4984