Update README.md

README.md

@@ -7,6 +7,7 @@ inference: false
 # Model Card for TinyMixtral-x8-Clonebase-7b
 This model is based on [TinyLlama-1.1B](https://huggingface.co/TinyLlama/TinyLlama-1.1B-intermediate-step-1195k-token-2.5T), converted to a mistral model, and then placed the clone in mixtral.  
 **This model was created experimentally for training a small mixtral.**  
+**Without Train, the performance of this model is the same as TinyLlama.**
 
 # How it was made
 First, since tinyllama is an llama model, I converted it to a mistral model.  
@@ -19,43 +20,34 @@ All gates have the same value.
 use colab cpu-high-memory.  
 This model was created with experts=8, but since it is a clone, you can create as many experts as you like.  
 
-[tinyllama_to_mixtral_clonebase.ipynb](https://huggingface.co/mmnga/TinyMixtral-x8-Clonebase-7b)
+[tinyllama_to_mixtral_clonebase.ipynb](https://huggingface.co/mmnga/TinyMixtral-x8-Clonebase-7b/blob/main/notebook/tinyllama_to_mixtral_clonebase.ipynb)
 
 # Usage
 ~~~python
 pip install transformers --upgrade
-pip install flash_attn
+pip install flash_attn bitsandbytes accelerate
 ~~~
 
 ~~~python
-from transformers import AutoTokenizer, AutoModelForCausalLM, MixtralForCausalLM
+from transformers import AutoTokenizer, AutoModelForCausalLM
 import torch
 
 model_name_or_path = "mmnga/TinyMixtral-x8-Clonebase-7b"
 
 tokenizer = AutoTokenizer.from_pretrained(model_name_or_path)
-model = MixtralForCausalLM.from_pretrained(model_name_or_path, device_map="auto")
+model = AutoModelForCausalLM.from_pretrained(model_name_or_path, device_map="auto", load_in_8bit=True)
 
-# set num_experts_per_tok 1 or 2 ?
-model.config.num_experts_per_tok = 2
-
-# message
-messages = [
-    {"role": "user", "content": "Tell me what's for dinner tonight."},
-]
+prompt = "Introducing the recipe for today's dinner."
 
 with torch.no_grad():
-    token_ids = tokenizer.apply_chat_template(messages, return_tensors="pt")
+    token_ids = tokenizer.encode(prompt, return_tensors="pt")
     output_ids = model.generate(
         token_ids.to(model.device),
-        temperature=0.5,
         do_sample=True,
-        top_p=0.95,
-        top_k=40,
         max_new_tokens=128,
         repetition_penalty=1.5
     )
-output = tokenizer.decode(output_ids[0][token_ids.size(1) :])
+    output = tokenizer.decode(output_ids[0])
 print(output)
 
 ~~~