Shehryar718 commited on
Commit
ab0f156
1 Parent(s): 3f5a4c5

Upload tokenizer

Browse files
Files changed (1) hide show
  1. vocab.json +84 -84
vocab.json CHANGED
@@ -1,88 +1,88 @@
1
  {
2
- "'": 83,
3
  "[PAD]": 85,
4
  "[UNK]": 84,
5
- "`": 59,
6
- "|": 42,
7
- "،": 29,
8
- "ؑ": 19,
9
- "ؓ": 67,
10
- "ؔ": 1,
11
- "؛": 13,
12
- "؟": 46,
13
- "ء": 6,
14
- "آ": 78,
15
- "أ": 34,
16
- "ؤ": 44,
17
- "ئ": 16,
18
- "ا": 32,
19
- "ب": 23,
20
- "ت": 72,
21
- "ث": 24,
22
- "ج": 74,
23
- "ح": 21,
24
- "خ": 70,
25
- "د": 64,
26
- "ذ": 35,
27
- "ر": 9,
28
- "ز": 45,
29
- "س": 54,
30
- "ش": 15,
31
- "ص": 26,
32
- "ض": 5,
33
- "ط": 43,
34
- "ظ": 49,
35
- "ع": 36,
36
- "غ": 55,
37
- "ف": 8,
38
- "ق": 30,
39
- "ك": 66,
40
- "ل": 57,
41
- "م": 27,
42
- "ن": 61,
43
- "ه": 18,
44
- "و": 47,
45
- "ى": 39,
46
- "ي": 76,
47
- "ً": 17,
48
- "َ": 52,
49
- "ُ": 82,
50
- "ِ": 80,
51
- "ّ": 10,
52
- "ْ": 60,
53
- "ٓ": 14,
54
- "ٔ": 68,
55
- "ٗ": 31,
56
- "ٰ": 65,
57
- "ٹ": 50,
58
- "پ": 3,
59
- "چ": 69,
60
- "ڈ": 41,
61
- "ڑ": 62,
62
- "ژ": 38,
63
- "ک": 75,
64
- "گ": 7,
65
- "ں": 51,
66
- "ھ": 37,
67
- "ہ": 81,
68
- "ۂ": 22,
69
- "ۃ": 4,
70
- "ی": 58,
71
- "ے": 48,
72
- "ۓ": 20,
73
- "۔": 56,
74
- "’": 77,
75
- "…": 33,
76
- "ﭨ": 71,
77
- "ﮭ": 2,
78
- "ﮯ": 11,
79
- "ﯾ": 12,
80
- "ﷲ": 73,
81
- "ﷺ": 40,
82
- "ﺗ": 53,
83
- "ﺘ": 79,
84
- "ﺩ": 63,
85
- "ﺲ": 28,
86
- "ﻧ": 25,
87
- "ﻮ": 0
88
  }
 
1
  {
2
+ "'": 28,
3
  "[PAD]": 85,
4
  "[UNK]": 84,
5
+ "`": 49,
6
+ "|": 19,
7
+ "،": 36,
8
+ "ؑ": 76,
9
+ "ؓ": 45,
10
+ "ؔ": 57,
11
+ "؛": 7,
12
+ "؟": 25,
13
+ "ء": 18,
14
+ "آ": 59,
15
+ "أ": 35,
16
+ "ؤ": 12,
17
+ "ئ": 72,
18
+ "ا": 47,
19
+ "ب": 22,
20
+ "ت": 34,
21
+ "ث": 15,
22
+ "ج": 32,
23
+ "ح": 33,
24
+ "خ": 23,
25
+ "د": 82,
26
+ "ذ": 68,
27
+ "ر": 2,
28
+ "ز": 3,
29
+ "س": 75,
30
+ "ش": 14,
31
+ "ص": 30,
32
+ "ض": 17,
33
+ "ط": 69,
34
+ "ظ": 11,
35
+ "ع": 41,
36
+ "غ": 67,
37
+ "ف": 26,
38
+ "ق": 66,
39
+ "ك": 70,
40
+ "ل": 4,
41
+ "م": 62,
42
+ "ن": 5,
43
+ "ه": 16,
44
+ "و": 71,
45
+ "ى": 55,
46
+ "ي": 10,
47
+ "ً": 21,
48
+ "َ": 80,
49
+ "ُ": 64,
50
+ "ِ": 73,
51
+ "ّ": 53,
52
+ "ْ": 31,
53
+ "ٓ": 0,
54
+ "ٔ": 43,
55
+ "ٗ": 77,
56
+ "ٰ": 81,
57
+ "ٹ": 20,
58
+ "پ": 24,
59
+ "چ": 83,
60
+ "ڈ": 54,
61
+ "ڑ": 74,
62
+ "ژ": 44,
63
+ "ک": 78,
64
+ "گ": 29,
65
+ "ں": 1,
66
+ "ھ": 60,
67
+ "ہ": 61,
68
+ "ۂ": 27,
69
+ "ۃ": 9,
70
+ "ی": 50,
71
+ "ے": 38,
72
+ "ۓ": 48,
73
+ "۔": 52,
74
+ "’": 79,
75
+ "…": 51,
76
+ "ﭨ": 40,
77
+ "ﮭ": 42,
78
+ "ﮯ": 37,
79
+ "ﯾ": 6,
80
+ "ﷲ": 58,
81
+ "ﷺ": 65,
82
+ "ﺗ": 8,
83
+ "ﺘ": 46,
84
+ "ﺩ": 39,
85
+ "ﺲ": 63,
86
+ "ﻧ": 56,
87
+ "ﻮ": 13
88
  }