hezarai
/

distilbert-base-fa

arxyzan commited on Feb 28, 2023

Commit

1c844b4

•

1 Parent(s): 5ced18f

Update preprocessor/tokenizer_config.yaml

Files changed (1) hide show

preprocessor/tokenizer_config.yaml CHANGED Viewed

@@ -1,4 +1,4 @@
-name: tokenizer
 config_type: preprocessor
 model: wordpiece
 max_length: 512
@@ -9,4 +9,4 @@ padding_strategy: longest
 padding_direction: right
 pad_token_id: 0
 pad_token: '[PAD]'
-pad_token_type_id: 0

+name: wordpiece_tokenizer
 config_type: preprocessor
 model: wordpiece
 max_length: 512
 padding_direction: right
 pad_token_id: 0
 pad_token: '[PAD]'
+pad_token_type_id: 0