sanchit-gandhi HF staff commited on
Commit
112cf37
1 Parent(s): 5748e99

Upload tokenizer

Browse files
Files changed (3) hide show
  1. special_tokens_map.json +4 -0
  2. tokenizer_config.json +12 -0
  3. vocab.json +46 -0
special_tokens_map.json ADDED
@@ -0,0 +1,4 @@
 
 
 
 
 
1
+ {
2
+ "pad_token": "|",
3
+ "unk_token": "<unk>"
4
+ }
tokenizer_config.json ADDED
@@ -0,0 +1,12 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "add_blank": true,
3
+ "clean_up_tokenization_spaces": true,
4
+ "is_uroman": false,
5
+ "language": "shn",
6
+ "model_max_length": 1000000000000000019884624838656,
7
+ "normalize": true,
8
+ "pad_token": "|",
9
+ "phonemize": false,
10
+ "tokenizer_class": "VitsTokenizer",
11
+ "unk_token": "<unk>"
12
+ }
vocab.json ADDED
@@ -0,0 +1,46 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ " ": 43,
3
+ "'": 40,
4
+ "-": 34,
5
+ "|": 0,
6
+ "င": 11,
7
+ "တ": 9,
8
+ "ထ": 36,
9
+ "ပ": 20,
10
+ "မ": 10,
11
+ "ယ": 29,
12
+ "ရ": 39,
13
+ "လ": 16,
14
+ "ဝ": 5,
15
+ "သ": 21,
16
+ "ဢ": 13,
17
+ "ိ": 15,
18
+ "ီ": 31,
19
+ "ု": 22,
20
+ "ူ": 12,
21
+ "ေ": 19,
22
+ "ဵ": 28,
23
+ "း": 4,
24
+ "်": 1,
25
+ "ျ": 38,
26
+ "ြ": 35,
27
+ "ွ": 27,
28
+ "ၢ": 17,
29
+ "ၵ": 7,
30
+ "ၶ": 25,
31
+ "ၸ": 18,
32
+ "ၺ": 37,
33
+ "ၼ": 2,
34
+ "ၽ": 32,
35
+ "ၿ": 42,
36
+ "ႀ": 41,
37
+ "ႁ": 23,
38
+ "ႂ": 26,
39
+ "ႃ": 6,
40
+ "ႄ": 30,
41
+ "ႅ": 33,
42
+ "ႆ": 24,
43
+ "ႇ": 8,
44
+ "ႈ": 3,
45
+ "ႉ": 14
46
+ }