Spaces:
Paused
Paused
Update utils.py
Browse files
utils.py
CHANGED
@@ -24,6 +24,7 @@ from datasets import load_dataset
|
|
24 |
import evaluate
|
25 |
|
26 |
|
|
|
27 |
def is_stop_word_or_prefix(s: str, stop_words: list) -> bool:
|
28 |
for stop_word in stop_words:
|
29 |
if s.endswith(stop_word):
|
@@ -59,6 +60,7 @@ def load_tokenizer_and_model(base_model, load_8bit=False):
|
|
59 |
else:
|
60 |
device = "cpu"
|
61 |
|
|
|
62 |
tokenizer = AutoTokenizer.from_pretrained(base_model, use_fast = True, use_auth_token=True)
|
63 |
if device == "cuda":
|
64 |
model = AutoModelForCausalLM.from_pretrained(
|
|
|
24 |
import evaluate
|
25 |
|
26 |
|
27 |
+
|
28 |
def is_stop_word_or_prefix(s: str, stop_words: list) -> bool:
|
29 |
for stop_word in stop_words:
|
30 |
if s.endswith(stop_word):
|
|
|
60 |
else:
|
61 |
device = "cpu"
|
62 |
|
63 |
+
|
64 |
tokenizer = AutoTokenizer.from_pretrained(base_model, use_fast = True, use_auth_token=True)
|
65 |
if device == "cuda":
|
66 |
model = AutoModelForCausalLM.from_pretrained(
|