Update app.py

app.py

@@ -2,22 +2,23 @@ import re
 import spaces
 import gradio as gr
 import torch
+from transformers import AutoTokenizer, AutoModelForCausalLM, BitsAndBytesConfig
 # from peft import PeftModel, PeftConfig
 
 
-# tokenizer = AutoTokenizer.from_pretrained("FlawedLLM/Bhashini")
-# quantization_config = BitsAndBytesConfig(
-#         load_in_4bit=True,
-#         bnb_4bit_use_double_quant=True,
-#         bnb_4bit_quant_type="nf4",  
-#         bnb_4bit_compute_dtype=torch.float16)
-# model = AutoModelForCausalLM.from_pretrained("FlawedLLM/BhashiniLLM",
-#                                              device_map="auto",
-#                                              quantization_config=quantization_config, 
-#                                              torch_dtype =torch.float16, 
-#                                              low_cpu_mem_usage=True, 
-#                                              use_safetensors=True,
-#                                             )
+tokenizer = AutoTokenizer.from_pretrained("FlawedLLM/Bhashini_00")
+quantization_config = BitsAndBytesConfig(
+        load_in_4bit=True,
+        bnb_4bit_use_double_quant=True,
+        bnb_4bit_quant_type="nf4",  
+        bnb_4bit_compute_dtype=torch.float16)
+model = AutoModelForCausalLM.from_pretrained("FlawedLLM/Bhashini_00",
+                                             device_map="auto",
+                                             quantization_config=quantization_config, 
+                                             torch_dtype =torch.float16, 
+                                             low_cpu_mem_usage=True, 
+                                             use_safetensors=True,
+                                            )
 
 # # Assuming you have your HF repository in this format: "your_username/your_model_name"
 # model_id = "FlawedLLM/BhashiniLLM"
@@ -58,10 +59,10 @@ import torch
 # # torch_dtype =torch.float16
 # model = AutoModelForCausalLM.from_pretrained("FlawedLLM/Bhashini_9",config=config, ignore_mismatched_sizes=True).to('cuda')
 # Load model directly
-from transformers import AutoTokenizer, AutoModelForCausalLM
 
-tokenizer = AutoTokenizer.from_pretrained("FlawedLLM/Bhashini_00")
-model = AutoModelForCausalLM.from_pretrained("FlawedLLM/Bhashini_00").to('cuda')
+
+# tokenizer = AutoTokenizer.from_pretrained("FlawedLLM/Bhashini_00")
+# model = AutoModelForCausalLM.from_pretrained("FlawedLLM/Bhashini_00").to('cuda')
 
 @spaces.GPU(duration=300)
 def chunk_it(input_command):