Fix generate process

spanish_medica_llm.py

@@ -25,7 +25,7 @@ from transformers import (
 
 from accelerate import FullyShardedDataParallelPlugin, Accelerator
 from torch.distributed.fsdp.fully_sharded_data_parallel import FullOptimStateDictConfig, FullStateDictConfig
-from peft import LoraConfig, get_peft_model, prepare_model_for_kbit_training
+from peft import PeftConfig, LoraConfig, get_peft_model, prepare_model_for_kbit_training
 #import wandb
 from trl import SFTTrainer
 
@@ -735,8 +735,11 @@ def generate_response(query):
         top_k=40
         num_beams=2
 
-        tokenizer = loadSpanishTokenizer()
-        model =  loadBaseModel(HUB_MODEL_ID)
+        #model =  loadBaseModel(HUB_MODEL_ID)
+
+        config = PeftConfig.from_pretrained(HUB_MODEL_ID)
+        model = AutoModelForCausalLM.from_pretrained(HUB_MODEL_ID, return_dict=True, load_in_8bit=True, device_map={"":0})
+        tokenizer = AutoTokenizer.from_pretrained(HUB_MODEL_ID)
 
         #system = f"[INST]\nYou are a helpful coding assistant.[/INST]\n"
         query = get_chat_format({'raw_text':query, 'topic':''})