Update app.py

added 4bit double quant

app.py

@@ -24,6 +24,13 @@ Answer the Question by exploring multiple reasoning paths as follows:
 In summary, leverage a Tree of Thoughts approach to actively explore multiple reasoning paths, evaluate thoughts heuristically, and explain the process - with the goal of producing insightful answers.
 """
 
+model = AutoModelForCausalLM.from_pretrained(
+    model_path,
+    device_map="auto",
+    trust_remote_code=True,
+    quantization_config=quantization_config,
+)
+
 model_path = "WhiteRabbitNeo/Trinity-13B"
 
 hf_token = os.getenv("HF_TOKEN")
@@ -32,10 +39,9 @@ if not hf_token:
 
 model = AutoModelForCausalLM.from_pretrained(
     model_path,
-    torch_dtype=torch.float16,
-    device_map="auto",
-    load_in_8bit=True,
-    trust_remote_code=True,
+     device_map="auto",
+     trust_remote_code=True,
+    quantization_config=quantization_config
 )
 
 tokenizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=True)