Update tokenization_zhinao.py
Browse files- tokenization_zhinao.py +0 -4
tokenization_zhinao.py
CHANGED
@@ -212,10 +212,6 @@ class ZhinaoTokenizer(PreTrainedTokenizer):
|
|
212 |
@property
|
213 |
def vocab_size(self):
|
214 |
return self.tokenizer.num_tokens
|
215 |
-
|
216 |
-
@property
|
217 |
-
def vocab(self):
|
218 |
-
return self.get_vocab()
|
219 |
|
220 |
def get_vocab(self):
|
221 |
""" Returns vocab as a dict """
|
|
|
212 |
@property
|
213 |
def vocab_size(self):
|
214 |
return self.tokenizer.num_tokens
|
|
|
|
|
|
|
|
|
215 |
|
216 |
def get_vocab(self):
|
217 |
""" Returns vocab as a dict """
|