End of training

README.md

@@ -1,11 +1,15 @@
 ---
 base_model: lectura/TinyLlama-120M-news
 tags:
+- alignment-handbook
+- trl
+- sft
+- generated_from_trainer
 - trl
 - sft
 - generated_from_trainer
 datasets:
-- generator
+- HuggingFaceH4/ultrachat_200k
 model-index:
 - name: zephyr-7b-sft-full
   results: []
@@ -16,7 +20,7 @@ should probably proofread and complete it, then remove this comment. -->
 
 # zephyr-7b-sft-full
 
-This model is a fine-tuned version of [lectura/TinyLlama-120M-news](https://huggingface.co/lectura/TinyLlama-120M-news) on the generator dataset.
+This model is a fine-tuned version of [lectura/TinyLlama-120M-news](https://huggingface.co/lectura/TinyLlama-120M-news) on the HuggingFaceH4/ultrachat_200k dataset.
 It achieves the following results on the evaluation set:
 - Loss: 2.2349
 

all_results.json

@@ -1,5 +1,10 @@
 {
     "epoch": 1.0,
+    "eval_loss": 2.2348995208740234,
+    "eval_runtime": 2502.8496,
+    "eval_samples": 23109,
+    "eval_samples_per_second": 6.345,
+    "eval_steps_per_second": 0.793,
     "train_loss": 2.666482654486044,
     "train_runtime": 30048.5081,
     "train_samples": 207864,

config.json

@@ -1,5 +1,5 @@
 {
-  "_name_or_path": "TinyLlama/TinyLlama-1.1B-Chat-v1.0",
+  "_name_or_path": "lectura/TinyLlama-120M-news",
   "architectures": [
     "BitLlamaForCausalLM"
   ],
@@ -9,21 +9,21 @@
   "bos_token_id": 1,
   "eos_token_id": 2,
   "hidden_act": "silu",
-  "hidden_size": 2048,
+  "hidden_size": 768,
   "initializer_range": 0.02,
-  "intermediate_size": 5632,
-  "max_position_embeddings": 2048,
+  "intermediate_size": 2048,
+  "max_position_embeddings": 1024,
   "model_type": "bit_llama",
-  "num_attention_heads": 32,
-  "num_hidden_layers": 22,
-  "num_key_value_heads": 4,
+  "num_attention_heads": 12,
+  "num_hidden_layers": 12,
+  "num_key_value_heads": 12,
   "pretraining_tp": 1,
   "rms_norm_eps": 1e-05,
   "rope_scaling": null,
   "rope_theta": 10000.0,
-  "tie_word_embeddings": false,
+  "tie_word_embeddings": true,
   "torch_dtype": "bfloat16",
   "transformers_version": "4.39.0.dev0",
-  "use_cache": false,
-  "vocab_size": 32002
+  "use_cache": true,
+  "vocab_size": 46338
 }

eval_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 1.0,
-    "eval_loss": 4.488282680511475,
-    "eval_runtime": 1697.9395,
+    "eval_loss": 2.2348995208740234,
+    "eval_runtime": 2502.8496,
     "eval_samples": 23109,
-    "eval_samples_per_second": 9.354,
-    "eval_steps_per_second": 1.17
+    "eval_samples_per_second": 6.345,
+    "eval_steps_per_second": 0.793
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9b52612851e8d6aa212ce306d9e797a5a7e6b359edc5e9db3ff60568afca1b2d
-size 2201749928
+oid sha256:82082c0b5de2f82febe83a84ec395efa8ad3d6d19b52c41268d202c52dd39484
+size 241437456

tokenizer_config.json

@@ -26,7 +26,47 @@
       "single_word": false,
       "special": true
     },
-    "32000": {
+    "46331": {
+      "content": "<|sep|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "46332": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "46333": {
+      "content": "<|acc|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "46334": {
+      "content": "<|rrn|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "46335": {
+      "content": "<|tel|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "46336": {
       "content": "<|im_start|>",
       "lstrip": false,
       "normalized": false,
@@ -34,7 +74,7 @@
       "single_word": false,
       "special": true
     },
-    "32001": {
+    "46337": {
       "content": "<|im_end|>",
       "lstrip": false,
       "normalized": false,
@@ -54,9 +94,8 @@
   "legacy": false,
   "model_max_length": 2048,
   "pad_token": "<|im_end|>",
-  "padding_side": "right",
   "sp_model_kwargs": {},
   "tokenizer_class": "LlamaTokenizer",
   "unk_token": "<unk>",
-  "use_default_system_prompt": false
+  "use_default_system_prompt": true
 }

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:07cc40c5a6d24066e3c9c9dc23db4ba29b9036550b49d4e3f00420e402d9f406
+oid sha256:835feb71d544b963e5d2bc0c5bc229d8fdfc8cb508e797df572da8dba67d19f9
 size 4984