Update tokenizer.json
Browse files- tokenizer.json +2 -2
tokenizer.json
CHANGED
@@ -51,12 +51,12 @@
|
|
51 |
"decoder": {
|
52 |
"type": "CTC",
|
53 |
"pad_token": "[PAD]",
|
54 |
-
"unk_token": "[UNK]"
|
55 |
"word_delimiter_token": "|",
|
56 |
"cleanup": true,
|
57 |
"clean_up_tokenization_spaces": true,
|
58 |
"bos_token": "<s>",
|
59 |
-
"eos_token": "</s>"
|
60 |
},
|
61 |
"model": {
|
62 |
"vocab": {
|
|
|
51 |
"decoder": {
|
52 |
"type": "CTC",
|
53 |
"pad_token": "[PAD]",
|
54 |
+
"unk_token": "[UNK]",
|
55 |
"word_delimiter_token": "|",
|
56 |
"cleanup": true,
|
57 |
"clean_up_tokenization_spaces": true,
|
58 |
"bos_token": "<s>",
|
59 |
+
"eos_token": "</s>"
|
60 |
},
|
61 |
"model": {
|
62 |
"vocab": {
|