mobiuslabsgmbh
/

Mixtral-8x7B-Instruct-v0.1-hf-2bit_g16_s128-HQQ

Text Generation

Mixture of Experts

Model card Files Files and versions Community

mobicham commited on Dec 11, 2023

Commit

f479ccf

•

1 Parent(s): ac3c4e8

Update README.md

Files changed (1) hide show

README.md +3 -1

README.md CHANGED Viewed

@@ -16,7 +16,9 @@ from hqq.engine.hf import HQQModelForCausalLM, AutoTokenizer
 tokenizer = AutoTokenizer.from_pretrained(model_id)
 model     = HQQModelForCausalLM.from_quantized(model_id)
-HQQLinear.set_backend(HQQBackend.PYTORCH_COMPILE) #Optional
 #Text Generation
 prompt = "<s> [INST] How do I build a car? [/INST] "

 tokenizer = AutoTokenizer.from_pretrained(model_id)
 model     = HQQModelForCausalLM.from_quantized(model_id)
+#Optional
+from hqq.core.quantize import *
+HQQLinear.set_backend(HQQBackend.PYTORCH_COMPILE)
 #Text Generation
 prompt = "<s> [INST] How do I build a car? [/INST] "