MarioNapoli's picture
Upload tokenizer
20214fb
raw
history blame
1.94 kB
{
"+": 1,
"[PAD]": 156,
"[UNK]": 155,
"_": 2,
"a": 3,
"b": 4,
"c": 5,
"d": 6,
"e": 7,
"f": 8,
"g": 9,
"h": 10,
"i": 11,
"j": 12,
"k": 13,
"l": 14,
"m": 15,
"n": 16,
"o": 17,
"p": 18,
"q": 19,
"r": 20,
"s": 21,
"t": 22,
"u": 23,
"v": 24,
"w": 25,
"x": 26,
"y": 27,
"z": 28,
"{": 29,
"|": 0,
"}": 30,
"~": 31,
"¡": 32,
"«": 33,
"´": 34,
"µ": 35,
"º": 36,
"»": 37,
"ß": 38,
"á": 39,
"â": 40,
"ã": 41,
"å": 42,
"æ": 43,
"ê": 44,
"ë": 45,
"î": 46,
"ð": 47,
"ñ": 48,
"ô": 49,
"ö": 50,
"ø": 51,
"û": 52,
"þ": 53,
"ÿ": 54,
"ā": 55,
"ą": 56,
"ć": 57,
"č": 58,
"đ": 59,
"ė": 60,
"ę": 61,
"ě": 62,
"ğ": 63,
"ī": 64,
"ı": 65,
"ľ": 66,
"ł": 67,
"ń": 68,
"ň": 69,
"ō": 70,
"ő": 71,
"œ": 72,
"ř": 73,
"ś": 74,
"ş": 75,
"š": 76,
"ū": 77,
"ŭ": 78,
"ź": 79,
"ż": 80,
"ž": 81,
"ș": 82,
"ț": 83,
"ə": 84,
"ʹ": 85,
"ʻ": 86,
"ʼ": 87,
"ʾ": 88,
"ː": 89,
"̇": 90,
"̨": 91,
"а": 92,
"б": 93,
"д": 94,
"е": 95,
"и": 96,
"к": 97,
"л": 98,
"м": 99,
"н": 100,
"о": 101,
"с": 102,
"т": 103,
"у": 104,
"ц": 105,
"ш": 106,
"ъ": 107,
"ѐ": 108,
"ё": 109,
"љ": 110,
"ң": 111,
"ד": 112,
"ה": 113,
"ו": 114,
"ة": 115,
"ر": 116,
"س": 117,
"ص": 118,
"غ": 119,
"ل": 120,
"ي": 121,
"ḥ": 122,
"ṇ": 123,
"ṛ": 124,
"ṣ": 125,
"ṭ": 126,
"ễ": 127,
"‑": 128,
"—": 129,
"′": 130,
"☆": 131,
"♭": 132,
"♯": 133,
"あ": 134,
"ア": 135,
"カ": 136,
"キ": 137,
"サ": 138,
"ザ": 139,
"ノ": 140,
"フ": 141,
"リ": 142,
"ン": 143,
"・": 144,
"ー": 145,
"万": 146,
"三": 147,
"丰": 148,
"古": 149,
"多": 150,
"峰": 151,
"張": 152,
"旅": 153,
"ꞌ": 154
}