jxm commited on
Commit
3cbddc4
1 Parent(s): 5c628e5

update load model within config

Browse files
Files changed (1) hide show
  1. model.py +2 -1
model.py CHANGED
@@ -16,7 +16,8 @@ def load_embedder_and_tokenizer(name: str) -> Tuple[
16
  transformers.PreTrainedTokenizer
17
  ]:
18
  if name.startswith("nomic") or (name == "bert-base-uncased"):
19
- model = transformers.AutoModel.from_pretrained(name)
 
20
  tokenizer = transformers.AutoTokenizer.from_pretrained(name)
21
  elif name in ["gtr-base", "gtr_base"]:
22
  model = transformers.AutoModel.from_pretrained(
 
16
  transformers.PreTrainedTokenizer
17
  ]:
18
  if name.startswith("nomic") or (name == "bert-base-uncased"):
19
+ config = AutoConfig.from_pretrained(model_path, trust_remote_code=True)
20
+ model = transformers.AutoModel.from_pretrained(name, config=config, trust_remote_code=True)
21
  tokenizer = transformers.AutoTokenizer.from_pretrained(name)
22
  elif name in ["gtr-base", "gtr_base"]:
23
  model = transformers.AutoModel.from_pretrained(