haijunlv commited on
Commit
0f3d701
·
verified ·
1 Parent(s): 94cd46f

Update tokenization_internlm3.py

Browse files
Files changed (1) hide show
  1. tokenization_internlm3.py +1 -1
tokenization_internlm3.py CHANGED
@@ -67,7 +67,7 @@ class InternLM3Tokenizer(PreTrainedTokenizer):
67
  Whether or not to add an initial space to the input. This allows to treat the leading word just as any
68
  other word. Again, this should be set with `from_slow=True` to make sure it's taken into account.
69
  """
70
-
71
  vocab_files_names = VOCAB_FILES_NAMES
72
  model_input_names = ["input_ids", "attention_mask"]
73
 
 
67
  Whether or not to add an initial space to the input. This allows to treat the leading word just as any
68
  other word. Again, this should be set with `from_slow=True` to make sure it's taken into account.
69
  """
70
+ _auto_class = "AutoTokenizer"
71
  vocab_files_names = VOCAB_FILES_NAMES
72
  model_input_names = ["input_ids", "attention_mask"]
73