Update app_v2.py

app_v2.py

@@ -6,13 +6,15 @@ from huggingface_hub import snapshot_download
 cwd = os.getcwd()
 cachedir = cwd + '/cache'
 
+local_folder = cachedir + "/model"
+
 # Check if the directory exists before creating it
 if not os.path.exists(cachedir):
     os.mkdir(cachedir)
 
-os.environ['HF_HOME'] = cachedir
-
-local_folder = cachedir + "/model"
+# Define pretrained and quantized model directories
+pretrained_quantized_model_dir = "FPHam/Jackson_The_Formalizer_V2_13b_GPTQ"
+quantized_model_dir = "opt-125m-4bit"
 
 quantized_model_dir = "FPHam/Jackson_The_Formalizer_V2_13b_GPTQ"
 
@@ -27,7 +29,7 @@ use_strict = False
 use_triton = False
 
 # Load tokenizer and model
-tokenizer = AutoTokenizer.from_pretrained(local_folder, use_fast=False)
+tokenizer = AutoTokenizer.from_pretrained(local_folder, use_fast=True)
 
 quantize_config = BaseQuantizeConfig(
     bits=4,