fix: added trust_remote_code to tokenizer
Browse files- modeling_bert.py +2 -2
modeling_bert.py
CHANGED
@@ -377,7 +377,7 @@ class BertModel(BertPreTrainedModel):
|
|
377 |
if self.emb_pooler is not None:
|
378 |
from transformers import AutoTokenizer
|
379 |
|
380 |
-
self.tokenizer = AutoTokenizer.from_pretrained(config._name_or_path)
|
381 |
else:
|
382 |
self.tokenizer = None
|
383 |
|
@@ -507,7 +507,7 @@ class BertModel(BertPreTrainedModel):
|
|
507 |
self.emb_pooler = 'mean'
|
508 |
from transformers import AutoTokenizer
|
509 |
|
510 |
-
self.tokenizer = AutoTokenizer.from_pretrained(self._name_or_path)
|
511 |
if self.emb_pooler != 'mean':
|
512 |
raise NotImplementedError
|
513 |
|
|
|
377 |
if self.emb_pooler is not None:
|
378 |
from transformers import AutoTokenizer
|
379 |
|
380 |
+
self.tokenizer = AutoTokenizer.from_pretrained(config._name_or_path, trust_remote_code=True)
|
381 |
else:
|
382 |
self.tokenizer = None
|
383 |
|
|
|
507 |
self.emb_pooler = 'mean'
|
508 |
from transformers import AutoTokenizer
|
509 |
|
510 |
+
self.tokenizer = AutoTokenizer.from_pretrained(self._name_or_path, trust_remote_code=True)
|
511 |
if self.emb_pooler != 'mean':
|
512 |
raise NotImplementedError
|
513 |
|