Update app_v2.py

app_v2.py

@@ -6,7 +6,9 @@ import os
 
 # Define pretrained and quantized model directories
 pretrained_model_dir = "FPHam/Jackson_The_Formalizer_V2_13b_GPTQ"
-quantized_model_dir = "./Jackson2-4bit-128g-GPTQ"
+cwd = os.getcwd()
+
+quantized_model_dir = cwd + "/Jackson2-4bit-128g-GPTQ"
 
 # Create the cache directory if it doesn't exist
 os.makedirs(quantized_model_dir, exist_ok=True)
@@ -14,7 +16,7 @@ os.makedirs(quantized_model_dir, exist_ok=True)
 snapshot_download(repo_id=pretrained_model_dir, local_dir=quantized_model_dir, local_dir_use_symlinks=False)
 
 # Quantization configuration
-quantize_config = BaseQuantizeConfig(bits=4, group_size=128, damp_percent=0.01, desc_act=False)
+# quantize_config = BaseQuantizeConfig(bits=4, group_size=128, damp_percent=0.01, desc_act=False)
 
 # Load the model using from_quantized
 model = AutoGPTQForCausalLM.from_quantized(
@@ -22,9 +24,9 @@ model = AutoGPTQForCausalLM.from_quantized(
     use_safetensors=True,
     strict=False,
     device="cuda:0",
-    trust_remote_code=True,
+    #trust_remote_code=True,
     use_triton=False,
-    quantize_config=quantize_config
+    #quantize_config=quantize_config
 )
 
 #model.save_quantized(quantized_model_dir)