Update README.md
Browse files
README.md
CHANGED
@@ -15,31 +15,12 @@ import torch
|
|
15 |
import transformers
|
16 |
from transformers import LlamaTokenizer, AutoModelForCausalLM
|
17 |
tokenizer = LlamaTokenizer.from_pretrained("Fan21/Llama-mt-lora")
|
18 |
-
|
19 |
-
|
20 |
-
device = "cuda"
|
21 |
-
else:
|
22 |
-
device = "cpu"
|
23 |
-
if device == "cuda":
|
24 |
-
model = LlamaForCausalLM.from_pretrained(
|
25 |
-
BASE_MODEL,
|
26 |
load_in_8bit=False,
|
27 |
torch_dtype=torch.float16,
|
28 |
device_map="auto",
|
29 |
)
|
30 |
-
|
31 |
-
elif device == "mps":
|
32 |
-
model = LlamaForCausalLM.from_pretrained(
|
33 |
-
BASE_MODEL,
|
34 |
-
device_map={"": device},
|
35 |
-
torch_dtype=torch.float16,
|
36 |
-
)
|
37 |
-
|
38 |
-
else:
|
39 |
-
model = LlamaForCausalLM.from_pretrained(
|
40 |
-
BASE_MODEL, device_map={"": device}, low_cpu_mem_usage=True
|
41 |
-
)
|
42 |
-
|
43 |
def generate_prompt(instruction, input=None):
|
44 |
if input:
|
45 |
return f"""Below is an instruction that describes a task, paired with an input that provides further context. Write a response that appropriately completes the request.
|
@@ -54,13 +35,6 @@ def generate_prompt(instruction, input=None):
|
|
54 |
{instruction}
|
55 |
### Response:"""
|
56 |
|
57 |
-
if device != "cpu":
|
58 |
-
model.half()
|
59 |
-
model.eval()
|
60 |
-
if torch.__version__ >= "2":
|
61 |
-
model = torch.compile(model)
|
62 |
-
|
63 |
-
|
64 |
def evaluate(
|
65 |
instruction,
|
66 |
input=None,
|
|
|
15 |
import transformers
|
16 |
from transformers import LlamaTokenizer, AutoModelForCausalLM
|
17 |
tokenizer = LlamaTokenizer.from_pretrained("Fan21/Llama-mt-lora")
|
18 |
+
mdoel = LlamaForCausalLM.from_pretrained(
|
19 |
+
"Fan21/Llama-mt-lora",
|
|
|
|
|
|
|
|
|
|
|
|
|
20 |
load_in_8bit=False,
|
21 |
torch_dtype=torch.float16,
|
22 |
device_map="auto",
|
23 |
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
24 |
def generate_prompt(instruction, input=None):
|
25 |
if input:
|
26 |
return f"""Below is an instruction that describes a task, paired with an input that provides further context. Write a response that appropriately completes the request.
|
|
|
35 |
{instruction}
|
36 |
### Response:"""
|
37 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
38 |
def evaluate(
|
39 |
instruction,
|
40 |
input=None,
|