adding my data config peft

model.py

@@ -3,6 +3,7 @@ from threading import Thread
 from typing import Iterator
 
 import torch
+from peft import PeftModel, PeftConfig
 from transformers import AutoConfig, AutoModelForCausalLM, AutoTokenizer, TextIteratorStreamer
 from huggingface_hub import login
 
@@ -11,6 +12,8 @@ from huggingface_hub import login
 #model_id = 'Trelis/Llama-2-7b-chat-hf-sharded-bf16'
 model_id = 'jaymojnidar/llama2-finetuned-mydata'
 config_model_id = 'jaymojnidar/llama2-finetuned-mydata/adapter_config.json'
+model_type = 'PEFT'
+
 
 if torch.cuda.is_available():
     tok = os.environ['HF_TOKEN']
@@ -21,15 +24,20 @@ if torch.cuda.is_available():
           #, token="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"
           )
     
-    config = AutoConfig.from_pretrained(config_model_id, use_auth_token=True)
-    config.pretraining_tp = 1
-    model = AutoModelForCausalLM.from_pretrained(
-        model_id,
-        config=config,
-        torch_dtype=torch.float16,
-        #load_in_4bit=True,
-        device_map='auto',
-        use_auth_token=True
+    if model_type == 'PEFT':
+        config = PeftConfig.from_pretrained("jaymojnidar/llama2-finetuned-mydata")
+        model = AutoModelForCausalLM.from_pretrained("Trelis/Llama-2-7b-chat-hf-sharded-bf16")
+        model = PeftModel.from_pretrained(model, "jaymojnidar/llama2-finetuned-mydata")
+    else:
+        config = AutoConfig.from_pretrained(model_id, use_auth_token=True)
+        config.pretraining_tp = 1
+        model = AutoModelForCausalLM.from_pretrained(
+            model_id,
+            config=config,
+            torch_dtype=torch.float16,
+            #load_in_4bit=True,
+            device_map='auto',
+            use_auth_token=True
     )
     print("Loaded the model!")
 else:

requirements.txt

@@ -1,8 +1,11 @@
 accelerate==0.21.0
 bitsandbytes==0.40.2
 gradio==3.37.0
+peft==0.5.0
 protobuf==3.20.3
 scipy==1.11.1
 sentencepiece==0.1.99
+#--extra-index-url https://download.pytorch.org/whl/cu113
+#torch
 torch==2.0.1
 transformers==4.31.0