alexkueck commited on
Commit
a0747c9
·
1 Parent(s): 66627bf

Update utils.py

Browse files
Files changed (1) hide show
  1. utils.py +2 -0
utils.py CHANGED
@@ -24,6 +24,7 @@ from datasets import load_dataset
24
  import evaluate
25
 
26
 
 
27
  def is_stop_word_or_prefix(s: str, stop_words: list) -> bool:
28
  for stop_word in stop_words:
29
  if s.endswith(stop_word):
@@ -59,6 +60,7 @@ def load_tokenizer_and_model(base_model, load_8bit=False):
59
  else:
60
  device = "cpu"
61
 
 
62
  tokenizer = AutoTokenizer.from_pretrained(base_model, use_fast = True, use_auth_token=True)
63
  if device == "cuda":
64
  model = AutoModelForCausalLM.from_pretrained(
 
24
  import evaluate
25
 
26
 
27
+
28
  def is_stop_word_or_prefix(s: str, stop_words: list) -> bool:
29
  for stop_word in stop_words:
30
  if s.endswith(stop_word):
 
60
  else:
61
  device = "cpu"
62
 
63
+
64
  tokenizer = AutoTokenizer.from_pretrained(base_model, use_fast = True, use_auth_token=True)
65
  if device == "cuda":
66
  model = AutoModelForCausalLM.from_pretrained(