jmaczan commited on
Commit
251217c
·
verified ·
1 Parent(s): 3393207

Update tokenizer.json

Browse files
Files changed (1) hide show
  1. tokenizer.json +2 -2
tokenizer.json CHANGED
@@ -51,12 +51,12 @@
51
  "decoder": {
52
  "type": "CTC",
53
  "pad_token": "[PAD]",
54
- "unk_token": "[UNK]"
55
  "word_delimiter_token": "|",
56
  "cleanup": true,
57
  "clean_up_tokenization_spaces": true,
58
  "bos_token": "<s>",
59
- "eos_token": "</s>",
60
  },
61
  "model": {
62
  "vocab": {
 
51
  "decoder": {
52
  "type": "CTC",
53
  "pad_token": "[PAD]",
54
+ "unk_token": "[UNK]",
55
  "word_delimiter_token": "|",
56
  "cleanup": true,
57
  "clean_up_tokenization_spaces": true,
58
  "bos_token": "<s>",
59
+ "eos_token": "</s>"
60
  },
61
  "model": {
62
  "vocab": {