hezarai
/

distilbert-base-fa

arxyzan commited on 20 days ago

Commit

bfe6cab

•

1 Parent(s): 19ad916

Update preprocessor/tokenizer_config.yaml

Files changed (1) hide show

preprocessor/tokenizer_config.yaml CHANGED Viewed

@@ -1,11 +1,11 @@
 name: wordpiece_tokenizer
 config_type: preprocessor
 max_length: 512
-truncation_strategy: longest_first
-truncation_direction: right
 stride: 0
-padding_strategy: longest
-padding_direction: right
 pad_to_multiple_of: 0
 pad_token_type_id: 0
 unk_token: '[UNK]'

 name: wordpiece_tokenizer
 config_type: preprocessor
 max_length: 512
+truncation: longest_first
+truncation_side: right
 stride: 0
+padding: longest
+padding_side: right
 pad_to_multiple_of: 0
 pad_token_type_id: 0
 unk_token: '[UNK]'