Hezar: Upload tokenizer_config.yaml

preprocessor/tokenizer_config.yaml

@@ -7,22 +7,14 @@ stride: 0
 padding_strategy: longest
 padding_direction: right
 pad_to_multiple_of: 0
-pad_token_id: 0
-pad_token: <pad>
 pad_token_type_id: 0
+bos_token: <s>
+eos_token: </s>
 unk_token: <unk>
-special_tokens:
-- <s>
-- <pad>
-- </s>
-- <unk>
-- <mask>
-- <|endoftext|>
-- <|startoftext|>
-- <nl>
-- <hs>
-- <sep>
-- <cls>
+sep_token: <sep>
+pad_token: <pad>
+cls_token: <cls>
+mask_token: <mask>
 continuing_subword_prefix: ''
 end_of_word_suffix: ''
 fuse_unk: false