File size: 327 Bytes
9fe7716 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 |
name: bpe_tokenizer
config_type: preprocessor
truncation_strategy: no_truncation
padding_strategy: no_padding
continuing_subword_prefix: ''
end_of_word_suffix: ''
fuse_unk: false
train_config:
name: bpe_tokenizer
config_type: preprocessor
vocab_size: 30000
min_frequency: 2
limit_alphabet: 1000
show_progress: true
|