add tokenizer

vocab.json

@@ -1 +1 @@
-{"،": 1, "؟": 2, "ء": 3, "آ": 4, "ؤ": 5, "ئ": 6, "ا": 7, "ب": 8, "ت": 9, "ث": 10, "ج": 11, "ح": 12, "خ": 13, "د": 14, "ذ": 15, "ر": 16, "ز": 17, "س": 18, "ش": 19, "ص": 20, "ض": 21, "ط": 22, "ظ": 23, "ع": 24, "غ": 25, "ف": 26, "ق": 27, "ل": 28, "م": 29, "ن": 30, "و": 31, "ى": 32, "ي": 33, "َ": 34, "ُ": 35, "ِ": 36, "ّ": 37, "ٔ": 38, "ٰ": 39, "ٹ": 40, "پ": 41, "چ": 42, "ڈ": 43, "ڑ": 44, "ژ": 45, "ک": 46, "گ": 47, "ں": 48, "ھ": 49, "ہ": 50, "ۂ": 51, "ی": 52, "ے": 53, "۔": 54, "|": 0, "<unk>": 55, "<pad>": 56, "<s>": 57, "</s>": 58}
+{"آ": 1, "ئ": 2, "ا": 3, "ب": 4, "ت": 5, "ث": 6, "ج": 7, "ح": 8, "خ": 9, "د": 10, "ذ": 11, "ر": 12, "ز": 13, "س": 14, "ش": 15, "ص": 16, "ض": 17, "ط": 18, "ظ": 19, "ع": 20, "غ": 21, "ف": 22, "ق": 23, "ل": 24, "م": 25, "ن": 26, "و": 27, "ٹ": 28, "پ": 29, "چ": 30, "ڈ": 31, "ڑ": 32, "ژ": 33, "ک": 34, "گ": 35, "ں": 36, "ھ": 37, "ہ": 38, "ی": 39, "ے": 40, "|": 0, "<unk>": 41, "<pad>": 42, "<s>": 43, "</s>": 44}