samikhan121 commited on
Commit
0cc161e
·
verified ·
1 Parent(s): 382ac5e

Upload tokenizer

Browse files
Files changed (3) hide show
  1. added_tokens.json +3 -3
  2. tokenizer_config.json +3 -3
  3. vocab.json +0 -3
added_tokens.json CHANGED
@@ -1,5 +1,5 @@
1
  {
2
- "ড়": 87,
3
- "ঢ়": 88,
4
- "য়": 89
5
  }
 
1
  {
2
+ "ড়": 84,
3
+ "ঢ়": 85,
4
+ "য়": 86
5
  }
tokenizer_config.json CHANGED
@@ -17,7 +17,7 @@
17
  "single_word": false,
18
  "special": true
19
  },
20
- "87": {
21
  "content": "ড়",
22
  "lstrip": false,
23
  "normalized": false,
@@ -25,7 +25,7 @@
25
  "single_word": false,
26
  "special": false
27
  },
28
- "88": {
29
  "content": "ঢ়",
30
  "lstrip": false,
31
  "normalized": false,
@@ -33,7 +33,7 @@
33
  "single_word": false,
34
  "special": false
35
  },
36
- "89": {
37
  "content": "য়",
38
  "lstrip": false,
39
  "normalized": false,
 
17
  "single_word": false,
18
  "special": true
19
  },
20
+ "84": {
21
  "content": "ড়",
22
  "lstrip": false,
23
  "normalized": false,
 
25
  "single_word": false,
26
  "special": false
27
  },
28
+ "85": {
29
  "content": "ঢ়",
30
  "lstrip": false,
31
  "normalized": false,
 
33
  "single_word": false,
34
  "special": false
35
  },
36
+ "86": {
37
  "content": "য়",
38
  "lstrip": false,
39
  "normalized": false,
vocab.json CHANGED
@@ -40,9 +40,7 @@
40
  "ট": 55,
41
  "ঠ": 73,
42
  "ড": 21,
43
- "ড়": 84,
44
  "ঢ": 22,
45
- "ঢ়": 85,
46
  "ণ": 51,
47
  "ত": 42,
48
  "থ": 14,
@@ -55,7 +53,6 @@
55
  "ভ": 16,
56
  "ম": 41,
57
  "য": 38,
58
- "য়": 86,
59
  "র": 69,
60
  "ল": 10,
61
  "শ": 48,
 
40
  "ট": 55,
41
  "ঠ": 73,
42
  "ড": 21,
 
43
  "ঢ": 22,
 
44
  "ণ": 51,
45
  "ত": 42,
46
  "থ": 14,
 
53
  "ভ": 16,
54
  "ম": 41,
55
  "য": 38,
 
56
  "র": 69,
57
  "ল": 10,
58
  "শ": 48,