surchand's picture
Upload tokenizer
86341ad verified
raw
history blame
888 Bytes
{
"$": 1,
"H": 2,
"O": 3,
"W": 4,
"[PAD]": 70,
"[UNK]": 69,
"a": 5,
"c": 6,
"e": 7,
"h": 8,
"l": 9,
"m": 10,
"o": 11,
"p": 12,
"t": 13,
"w": 14,
"x": 15,
"|": 0,
"ꯀ": 16,
"ꯁ": 17,
"ꯂ": 18,
"ꯃ": 19,
"ꯄ": 20,
"ꯅ": 21,
"ꯆ": 22,
"ꯇ": 23,
"ꯈ": 24,
"ꯉ": 25,
"ꯊ": 26,
"ꯋ": 27,
"ꯌ": 28,
"ꯍ": 29,
"ꯎ": 30,
"ꯏ": 31,
"ꯐ": 32,
"ꯑ": 33,
"ꯒ": 34,
"ꯔ": 35,
"ꯕ": 36,
"ꯖ": 37,
"ꯗ": 38,
"ꯘ": 39,
"ꯙ": 40,
"ꯚ": 41,
"ꯛ": 42,
"ꯜ": 43,
"ꯝ": 44,
"ꯞ": 45,
"ꯟ": 46,
"ꯠ": 47,
"ꯡ": 48,
"ꯢ": 49,
"ꯣ": 50,
"ꯤ": 51,
"ꯥ": 52,
"ꯦ": 53,
"ꯧ": 54,
"ꯨ": 55,
"ꯩ": 56,
"ꯪ": 57,
"꯭": 58,
"꯰": 59,
"꯱": 60,
"꯲": 61,
"꯳": 62,
"꯴": 63,
"꯵": 64,
"꯶": 65,
"꯷": 66,
"꯸": 67,
"꯹": 68
}