Upload tokenizer

README.md

@@ -1,4 +1,3 @@
-
 ---
 license: cc-by-nc-4.0
 tags:

added_tokens.json

@@ -0,0 +1,5 @@
+{
+  "ড়": 87,
+  "ঢ়": 88,
+  "য়": 89
+}

special_tokens_map.json

@@ -1,4 +1,16 @@
 {
-  "pad_token": "6",
-  "unk_token": "<unk>"
+  "pad_token": {
+    "content": "6",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
 }

tokenizer_config.json

@@ -1,10 +1,52 @@
 {
-  "add_blank": true,
+  "add_blank": false,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "6",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "74": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "87": {
+      "content": "ড়",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "88": {
+      "content": "ঢ়",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "89": {
+      "content": "য়",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    }
+  },
   "clean_up_tokenization_spaces": true,
   "is_uroman": false,
   "language": "ben",
   "model_max_length": 1000000000000000019884624838656,
-  "normalize": true,
+  "normalize": false,
   "pad_token": "6",
   "phonemize": false,
   "tokenizer_class": "VitsTokenizer",

vocab.json

@@ -40,7 +40,9 @@
   "ট": 55,
   "ঠ": 73,
   "ড": 21,
+  "ড়": 84,
   "ঢ": 22,
+  "ঢ়": 85,
   "ণ": 51,
   "ত": 42,
   "থ": 14,
@@ -53,6 +55,7 @@
   "ভ": 16,
   "ম": 41,
   "য": 38,
+  "য়": 86,
   "র": 69,
   "ল": 10,
   "শ": 48,
@@ -72,5 +75,15 @@
   "ৌ": 44,
   "্": 36,
   "ৎ": 52,
+  "০": 74,
+  "১": 75,
+  "২": 76,
+  "৩": 77,
+  "৪": 78,
+  "৫": 79,
+  "৬": 80,
+  "৭": 81,
+  "৮": 82,
+  "৯": 83,
   "—": 32
 }