File size: 385 Bytes
fa1a136 |
1 2 3 4 5 6 7 8 9 10 11 12 13 |
{
"tokenizer_class": "sudachitra.tokenization_electra_sudachipy.ElectraSudachipyTokenizer",
"do_lower_case": false,
"do_word_tokenize": true,
"do_subword_tokenize": true,
"word_tokenizer_type": "sudachipy",
"word_form_type": "dictionary_and_surface",
"subword_tokenizer_type": "wordpiece",
"model_max_length": 512,
"sudachipy_kwargs": {"split_mode":"A","dict_type":"core"}
}
|