Model save

README.md

@@ -1,16 +1,11 @@
 ---
-license: apache-2.0
-base_model: TinyLlama/TinyLlama-1.1B-Chat-v1.0
+base_model: lectura/TinyLlama-120M-news
 tags:
-- alignment-handbook
-- trl
-- sft
-- generated_from_trainer
 - trl
 - sft
 - generated_from_trainer
 datasets:
-- HuggingFaceH4/ultrachat_200k
+- generator
 model-index:
 - name: zephyr-7b-sft-full
   results: []
@@ -21,9 +16,9 @@ should probably proofread and complete it, then remove this comment. -->
 
 # zephyr-7b-sft-full
 
-This model is a fine-tuned version of [TinyLlama/TinyLlama-1.1B-Chat-v1.0](https://huggingface.co/TinyLlama/TinyLlama-1.1B-Chat-v1.0) on the HuggingFaceH4/ultrachat_200k dataset.
+This model is a fine-tuned version of [lectura/TinyLlama-120M-news](https://huggingface.co/lectura/TinyLlama-120M-news) on the generator dataset.
 It achieves the following results on the evaluation set:
-- Loss: 4.4883
+- Loss: 2.2349
 
 ## Model description
 
@@ -42,13 +37,12 @@ More information needed
 ### Training hyperparameters
 
 The following hyperparameters were used during training:
-- learning_rate: 0.0002
+- learning_rate: 0.001
 - train_batch_size: 16
 - eval_batch_size: 8
 - seed: 42
-- distributed_type: multi-GPU
 - optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
-- lr_scheduler_type: cosine
+- lr_scheduler_type: constant
 - lr_scheduler_warmup_ratio: 0.1
 - num_epochs: 1
 
@@ -56,7 +50,7 @@ The following hyperparameters were used during training:
 
 | Training Loss | Epoch | Step | Validation Loss |
 |:-------------:|:-----:|:----:|:---------------:|
-| 4.5336        | 1.0   | 8969 | 4.4883          |
+| 2.2279        | 1.0   | 8969 | 2.2349          |
 
 
 ### Framework versions

all_results.json

@@ -1,13 +1,8 @@
 {
     "epoch": 1.0,
-    "eval_loss": 4.488282680511475,
-    "eval_runtime": 1697.9395,
-    "eval_samples": 23109,
-    "eval_samples_per_second": 9.354,
-    "eval_steps_per_second": 1.17,
-    "train_loss": 4.746947802406368,
-    "train_runtime": 46954.2707,
+    "train_loss": 2.666482654486044,
+    "train_runtime": 30048.5081,
     "train_samples": 207864,
-    "train_samples_per_second": 3.056,
-    "train_steps_per_second": 0.191
+    "train_samples_per_second": 4.775,
+    "train_steps_per_second": 0.298
 }

config.json

@@ -1,5 +1,5 @@
 {
-  "_name_or_path": "TinyLlama/TinyLlama-1.1B-Chat-v1.0",
+  "_name_or_path": "lectura/TinyLlama-120M-news",
   "architectures": [
     "BitLlamaForCausalLM"
   ],
@@ -9,21 +9,21 @@
   "bos_token_id": 1,
   "eos_token_id": 2,
   "hidden_act": "silu",
-  "hidden_size": 2048,
+  "hidden_size": 768,
   "initializer_range": 0.02,
-  "intermediate_size": 5632,
-  "max_position_embeddings": 2048,
+  "intermediate_size": 2048,
+  "max_position_embeddings": 1024,
   "model_type": "bit_llama",
-  "num_attention_heads": 32,
-  "num_hidden_layers": 22,
-  "num_key_value_heads": 4,
+  "num_attention_heads": 12,
+  "num_hidden_layers": 12,
+  "num_key_value_heads": 12,
   "pretraining_tp": 1,
   "rms_norm_eps": 1e-05,
   "rope_scaling": null,
   "rope_theta": 10000.0,
-  "tie_word_embeddings": false,
+  "tie_word_embeddings": true,
   "torch_dtype": "bfloat16",
   "transformers_version": "4.39.0.dev0",
   "use_cache": false,
-  "vocab_size": 32002
+  "vocab_size": 46338
 }

generation_config.json

@@ -1,8 +1,7 @@
 {
+  "_from_model_config": true,
   "bos_token_id": 1,
   "eos_token_id": 2,
-  "max_length": 2048,
-  "pad_token_id": 0,
   "transformers_version": "4.39.0.dev0",
   "use_cache": false
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f46cb5af63882895c9a1a0eb69aa5e49a8b31c6fec67537f408060dc78992a7d
-size 2201749928
+oid sha256:82082c0b5de2f82febe83a84ec395efa8ad3d6d19b52c41268d202c52dd39484
+size 241437456

tokenizer_config.json

@@ -26,7 +26,47 @@
       "single_word": false,
       "special": true
     },
-    "32000": {
+    "46331": {
+      "content": "<|sep|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "46332": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "46333": {
+      "content": "<|acc|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "46334": {
+      "content": "<|rrn|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "46335": {
+      "content": "<|tel|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "46336": {
       "content": "<|im_start|>",
       "lstrip": false,
       "normalized": false,
@@ -34,7 +74,7 @@
       "single_word": false,
       "special": true
     },
-    "32001": {
+    "46337": {
       "content": "<|im_end|>",
       "lstrip": false,
       "normalized": false,
@@ -54,9 +94,8 @@
   "legacy": false,
   "model_max_length": 2048,
   "pad_token": "<|im_end|>",
-  "padding_side": "right",
   "sp_model_kwargs": {},
   "tokenizer_class": "LlamaTokenizer",
   "unk_token": "<unk>",
-  "use_default_system_prompt": false
+  "use_default_system_prompt": true
 }

train_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 1.0,
-    "train_loss": 4.746947802406368,
-    "train_runtime": 46954.2707,
+    "train_loss": 2.666482654486044,
+    "train_runtime": 30048.5081,
     "train_samples": 207864,
-    "train_samples_per_second": 3.056,
-    "train_steps_per_second": 0.191
+    "train_samples_per_second": 4.775,
+    "train_steps_per_second": 0.298
 }

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:07cc40c5a6d24066e3c9c9dc23db4ba29b9036550b49d4e3f00420e402d9f406
+oid sha256:835feb71d544b963e5d2bc0c5bc229d8fdfc8cb508e797df572da8dba67d19f9
 size 4984