Update app.py

app.py

@@ -43,10 +43,16 @@ import torch
 #     )
 # tokenizer = AutoTokenizer.from_pretrained("FlawedLLM/Bhashini")
 # Load model directly
-from transformers import AutoTokenizer, AutoModelForCausalLM
+from transformers import AutoTokenizer, AutoModelForCausalLM, BitsAndBytesConfig
 
 tokenizer = AutoTokenizer.from_pretrained("FlawedLLM/Bhashini_9")
-model = AutoModelForCausalLM.from_pretrained("FlawedLLM/Bhashini_9", device_map='auto')
+quantization_config = BitsAndBytesConfig(
+        load_in_4bit=True,
+        bnb_4bit_use_double_quant=True,
+        bnb_4bit_quant_type="nf4",  
+        bnb_4bit_compute_dtype=torch.float16)
+torch_dtype =torch.float16
+model = AutoModelForCausalLM.from_pretrained("FlawedLLM/Bhashini_9", device_map='auto',torch_dtype=torch_dtype, low_cpu_mem_usage=True, use_safetensors=True, quantization_config=quantization_config,)
 
 @spaces.GPU(duration=300)
 def chunk_it(input_command):