Reihaneh's picture
Upload tokenizer
2586ef0 verified
raw
history blame
1.21 kB
{
"$": 78,
"&": 76,
"'": 21,
"(": 43,
"-": 86,
"F": 37,
"L": 90,
"N": 85,
"R": 1,
"Y": 19,
"[": 75,
"[FR]": 102,
"[FY-NL]": 101,
"[PAD]": 100,
"[UNK]": 99,
"]": 3,
"a": 97,
"b": 38,
"c": 82,
"d": 17,
"e": 18,
"f": 61,
"g": 92,
"h": 41,
"i": 35,
"j": 12,
"k": 93,
"l": 96,
"m": 65,
"n": 32,
"o": 39,
"p": 25,
"q": 49,
"r": 42,
"s": 59,
"t": 27,
"u": 88,
"v": 80,
"w": 84,
"x": 63,
"y": 69,
"z": 74,
"|": 53,
"~": 66,
"«": 40,
"´": 68,
"»": 55,
"à": 58,
"á": 9,
"â": 24,
"ã": 8,
"ä": 23,
"å": 98,
"æ": 67,
"ç": 45,
"è": 30,
"é": 33,
"ê": 31,
"ë": 94,
"í": 51,
"î": 71,
"ï": 91,
"ñ": 7,
"ó": 54,
"ô": 36,
"ö": 77,
"ø": 4,
"ù": 62,
"ú": 48,
"û": 60,
"ü": 22,
"ā": 34,
"ă": 26,
"ć": 16,
"č": 57,
"ę": 15,
"ĝ": 28,
"ł": 56,
"ō": 14,
"ő": 70,
"œ": 20,
"ś": 11,
"ş": 81,
"š": 72,
"ū": 10,
"ż": 2,
"ž": 46,
"ș": 50,
"ʿ": 52,
"ː": 89,
"̀": 95,
"́": 64,
"̂": 29,
"̧": 5,
"–": 0,
"—": 73,
"‘": 83,
"’": 87,
"…": 44,
"ℤ": 6,
"保": 79,
"宇": 47,
"津": 13
}