Update BertForMorphTagging.py
Browse files- BertForMorphTagging.py +2 -0
BertForMorphTagging.py
CHANGED
@@ -177,6 +177,8 @@ def parse_logits(input_ids: List[List[int]], sentences: List[str], tokenizer: Be
|
|
177 |
# { pos: str, feats: dict, prefixes: List[str], suffix: str | bool, suffix_feats: dict | None}
|
178 |
special_toks = tokenizer.all_special_tokens
|
179 |
special_toks.remove(tokenizer.unk_token)
|
|
|
|
|
180 |
ret = []
|
181 |
for sent_idx,sentence in enumerate(sentences):
|
182 |
input_id_strs = tokenizer.convert_ids_to_tokens(input_ids[sent_idx])
|
|
|
177 |
# { pos: str, feats: dict, prefixes: List[str], suffix: str | bool, suffix_feats: dict | None}
|
178 |
special_toks = tokenizer.all_special_tokens
|
179 |
special_toks.remove(tokenizer.unk_token)
|
180 |
+
special_toks.remove(tokenizer.mask_token)
|
181 |
+
|
182 |
ret = []
|
183 |
for sent_idx,sentence in enumerate(sentences):
|
184 |
input_id_strs = tokenizer.convert_ids_to_tokens(input_ids[sent_idx])
|