psinger commited on
Commit
5017b1e
1 Parent(s): db5edb9

Update tokenization_xgen.py

Browse files
Files changed (1) hide show
  1. tokenization_xgen.py +5 -1
tokenization_xgen.py CHANGED
@@ -227,4 +227,8 @@ class XgenTokenizer(PreTrainedTokenizer):
227
  if token_ids_1 is not None:
228
  output += [1] * len(token_ids_1 + eos_token_id)
229
 
230
- return output
 
 
 
 
 
227
  if token_ids_1 is not None:
228
  output += [1] * len(token_ids_1 + eos_token_id)
229
 
230
+ return output
231
+
232
+ # has no vocab file
233
+ def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str] = None):
234
+ return ()