Update app_v2.py

app_v2.py

@@ -1,6 +1,7 @@
 import streamlit as st
 from transformers import AutoTokenizer
 from auto_gptq import AutoGPTQForCausalLM, BaseQuantizeConfig
+from huggingface_hub import snapshot_download
 import os
 
 # Define pretrained and quantized model directories
@@ -10,29 +11,27 @@ quantized_model_dir = "./Jackson2-4bit-128g-GPTQ"
 # Create the cache directory if it doesn't exist
 os.makedirs(quantized_model_dir, exist_ok=True)
 
+snapshot_download(repo_id=pretrained_model_dir, local_dir=quantized_model_dir, local_dir_use_symlinks=False)
+
 # Quantization configuration
 quantize_config = BaseQuantizeConfig(bits=4, group_size=128, damp_percent=0.01, desc_act=False)
 
 # Load the model using from_quantized
 model = AutoGPTQForCausalLM.from_quantized(
-    pretrained_model_dir,
+    quantized_model_dir,
     use_safetensors=True,
     strict=False,
-    model_basename='Jackson2-4bit-128g-GPTQ',
     device="cuda:0",
     trust_remote_code=True,
     use_triton=False,
     quantize_config=quantize_config
 )
 
-model.save_quantized(quantized_model_dir)
-
+#model.save_quantized(quantized_model_dir)
 
 # Load the tokenizer
 tokenizer = AutoTokenizer.from_pretrained(quantized_model_dir, use_fast=True)
 
-model_for_inference = AutoGPTQForCausalLM.from_pretrained(quantized_model_dir)
-
 # Starting Streamlit app
 st.title("AutoGPTQ Streamlit App")