Update app.py

app.py

@@ -6,20 +6,20 @@ from transformers import AutoTokenizer, AutoModelForCausalLM, BitsAndBytesConfig
 # from peft import PeftModel, PeftConfig
 
 
-tokenizer = AutoTokenizer.from_pretrained("FlawedLLM/Bhashini_00")
-quantization_config = BitsAndBytesConfig(
-        load_in_4bit=True,
-        bnb_4bit_use_double_quant=True,
-        bnb_4bit_quant_type="nf4",  
-        bnb_4bit_compute_dtype=torch.float16)
-config=AutoConfig.from_pretrained("FlawedLLM/Bhashini_00")
-model = AutoModelForCausalLM.from_pretrained("FlawedLLM/Bhashini_00",
-                                             device_map="auto",
-                                             quantization_config=quantization_config, 
-                                             torch_dtype =torch.float16, 
-                                             low_cpu_mem_usage=True, 
-                                             use_safetensors=True,
-                                            )
+# tokenizer = AutoTokenizer.from_pretrained("FlawedLLM/Bhashini_00")
+# quantization_config = BitsAndBytesConfig(
+#         load_in_4bit=True,
+#         bnb_4bit_use_double_quant=True,
+#         bnb_4bit_quant_type="nf4",  
+#         bnb_4bit_compute_dtype=torch.float16)
+# config=AutoConfig.from_pretrained("FlawedLLM/Bhashini_00")
+# model = AutoModelForCausalLM.from_pretrained("FlawedLLM/Bhashini_00",
+#                                              device_map="auto",
+#                                              quantization_config=quantization_config, 
+#                                              torch_dtype =torch.float16, 
+#                                              low_cpu_mem_usage=True, 
+#                                              use_safetensors=True,
+#                                             )
 
 # # Assuming you have your HF repository in this format: "your_username/your_model_name"
 # model_id = "FlawedLLM/BhashiniLLM"
@@ -60,12 +60,12 @@ model = AutoModelForCausalLM.from_pretrained("FlawedLLM/Bhashini_00",
 # # torch_dtype =torch.float16
 # model = AutoModelForCausalLM.from_pretrained("FlawedLLM/Bhashini_9",config=config, ignore_mismatched_sizes=True).to('cuda')
 # Load model directly
+@spaces.GPU(duration=300)
 
+tokenizer = AutoTokenizer.from_pretrained("FlawedLLM/Bhashini_00")
+model = AutoModelForCausalLM.from_pretrained("FlawedLLM/Bhashini_00", load_in_4bit=True).to('cuda')
 
-# tokenizer = AutoTokenizer.from_pretrained("FlawedLLM/Bhashini_00")
-# model = AutoModelForCausalLM.from_pretrained("FlawedLLM/Bhashini_00").to('cuda')
 
-@spaces.GPU(duration=300)
 def chunk_it(input_command):
     alpaca_prompt = """Below is an instruction that describes a task, paired with an input that provides further context. Write a response that appropriately completes the request.