Chang-1202 commited on
Commit
b5532db
1 Parent(s): 56f1e27

Update tokenizer_config.json

Browse files
Files changed (1) hide show
  1. tokenizer_config.json +7 -8
tokenizer_config.json CHANGED
@@ -1,5 +1,5 @@
1
  {
2
- "tokenizer_class": "sudachitra.tokenization_electra_sudachipy.ElectraSudachipyTokenizer",
3
  "do_lower_case": false,
4
  "do_word_tokenize": true,
5
  "do_subword_tokenize": true,
@@ -7,12 +7,11 @@
7
  "word_form_type": "dictionary_and_surface",
8
  "subword_tokenizer_type": "wordpiece",
9
  "model_max_length": 512,
10
- "sudachipy_kwargs": {"split_mode":"A","dict_type":"core"}
11
  "auto_map": {
12
- "AutoTokenizer": [
13
- "modeling.ElectraSudachipyTokenizer",
14
- null
15
- ]
16
  }
17
- }
18
-
 
1
  {
2
+ "tokenizer_class": "ElectraSudachipyTokenizer",
3
  "do_lower_case": false,
4
  "do_word_tokenize": true,
5
  "do_subword_tokenize": true,
 
7
  "word_form_type": "dictionary_and_surface",
8
  "subword_tokenizer_type": "wordpiece",
9
  "model_max_length": 512,
10
+ "sudachipy_kwargs": {"split_mode":"A","dict_type":"core"},
11
  "auto_map": {
12
+ "AutoTokenizer": [
13
+ "modeling.ElectraSudachipyTokenizer",
14
+ null
15
+ ]
16
  }
17
+ }