Shaltiel commited on
Commit
6235062
1 Parent(s): 64f9a08

Update BertForJointParsing.py

Browse files
Files changed (1) hide show
  1. BertForJointParsing.py +1 -0
BertForJointParsing.py CHANGED
@@ -314,6 +314,7 @@ def lex_parse_logits(input_ids: List[List[int]], sentences: List[str], tokenizer
314
 
315
  special_toks = tokenizer.all_special_tokens
316
  special_toks.remove(tokenizer.unk_token)
 
317
  for batch_idx in range(len(sentences)):
318
  intermediate_ret = []
319
  tokens = tokenizer.convert_ids_to_tokens(input_ids[batch_idx])
 
314
 
315
  special_toks = tokenizer.all_special_tokens
316
  special_toks.remove(tokenizer.unk_token)
317
+ special_toks.remove(tokenizer.mask_token)
318
  for batch_idx in range(len(sentences)):
319
  intermediate_ret = []
320
  tokens = tokenizer.convert_ids_to_tokens(input_ids[batch_idx])