Upload app.py

app.py

@@ -224,11 +224,11 @@ def load_models():
             assert isinstance(tokenizer, (PreTrainedTokenizer, PreTrainedTokenizerFast)), f"Tokenizer is of type {type(tokenizer)}"
 
             print(f"Loading LLM: {MODEL_PATH} 🤖")
-            text_model = AutoModelForCausalLM.from_pretrained(MODEL_PATH, quantization_config=nf4_config, device_map=device, torch_dtype=torch.bfloat16).eval()
+            text_model = AutoModelForCausalLM.from_pretrained(MODEL_PATH, quantization_config=nf4_config).eval()
 
             if False and IS_LORA and LORA_PATH.exists(): # omitted
                 print("Loading VLM's custom text model 🤖")
-                text_model = PeftModel.from_pretrained(model=text_model, model_id=LORA_PATH, device_map=device, quantization_config=nf4_config)
+                text_model = PeftModel.from_pretrained(model=text_model, model_id=LORA_PATH, quantization_config=nf4_config)
                 text_model = text_model.merge_and_unload(safe_merge=True) # to avoid PEFT bug https://github.com/huggingface/transformers/issues/28515
             else: print("VLM's custom text model isn't loaded 🤖")