Update README.md

README.md

@@ -60,13 +60,6 @@ prompt = [
 ]
 
 tokenizer = AutoTokenizer.from_pretrained(model_id)
-tokenizer.pad_token_id = tokenizer.eos_token_id
-tokenizer.padding_side = "left"
-
-terminators = [
-  tokenizer.eos_token_id,
-  tokenizer.convert_tokens_to_ids("<|eot_id|>"),
-]
 
 inputs = tokenizer.apply_chat_template(prompt, tokenize=True, add_generation_prompt=True, return_tensors="pt").cuda()
 
@@ -77,12 +70,24 @@ model = AutoModelForCausalLM.from_pretrained(
   device_map="auto",
 )
 
-outputs = model.generate(**inputs, do_sample=True, max_new_tokens=256, eos_token_id=terminators)
+outputs = model.generate(inputs, do_sample=True, max_new_tokens=256)
 print(tokenizer.batch_decode(outputs, skip_special_tokens=True))
 ```
 
 ### AutoAWQ
 
+In order to run the inference with Llama 3.1 405B Instruct AWQ in INT4, both `torch` and `autoawq` need to be installed as:
+
+```bash
+pip install "torch>=2.2.0,<2.3.0" autoawq --upgrade
+```
+
+Then, the latest version of `transformers` need to be installed, being 4.43.0 or higher, as:
+
+```bash
+pip install "transformers[accelerate]>=4.43.0" --upgrade
+```
+
 Alternatively, one may want to run that via `AutoAWQ` even though it's built on top of 🤗 `transformers`, which is the recommended approach instead as described above.
 
 ```python
@@ -97,8 +102,6 @@ prompt = [
 ]
 
 tokenizer = AutoTokenizer.from_pretrained(model_id)
-tokenizer.pad_token_id = tokenizer.eos_token_id
-tokenizer.padding_side = "left"
 
 inputs = tokenizer.apply_chat_template(prompt, tokenize=True, add_generation_prompt=True, return_tensors="pt").cuda()
 
@@ -110,7 +113,7 @@ model = AutoAWQForCausalLM.from_pretrained(
   fuse_layers=True,
 )
 
-outputs = model.generate(**inputs, do_sample=True, max_new_tokens=256)
+outputs = model.generate(inputs, do_sample=True, max_new_tokens=256)
 print(tokenizer.batch_decode(outputs, skip_special_tokens=True))
 ```