Upload tokenizer
e8cb29f
|
{ |
|
"0": 0, |
|
"1": 1, |
|
"11": 57, |
|
"13": 76, |
|
"2": 2, |
|
"21": 63, |
|
"23": 168, |
|
"24": 62, |
|
"3": 3, |
|
"4": 4, |
|
"41": 110, |
|
"42": 60, |
|
"44": 123, |
|
"442": 112, |
|
"453": 58, |
|
"5": 5, |
|
"53": 56, |
|
"533": 59, |
|
"544": 124, |
|
"55": 92, |
|
"a": 6, |
|
"ai": 69, |
|
"akieu": 449, |
|
"am": 447, |
|
"an": 448, |
|
"au": 77, |
|
"aÅĭ": 93, |
|
"aÊĶ": 200, |
|
"b": 7, |
|
"e": 8, |
|
"eÅĭ": 118, |
|
"eÊĶ": 116, |
|
"h": 9, |
|
"ha": 311, |
|
"hai": 191, |
|
"he": 176, |
|
"hi": 204, |
|
"hia": 406, |
|
"ho": 252, |
|
"hou": 278, |
|
"hu": 73, |
|
"hua": 105, |
|
"huai": 129, |
|
"hue": 132, |
|
"hui": 147, |
|
"huoi": 134, |
|
"huɤɯ": 133, |
|
"hy": 142, |
|
"hya": 439, |
|
"hyÉĴ": 405, |
|
"hyÉĶ": 440, |
|
"hÉĴ": 226, |
|
"hÉĶ": 265, |
|
"hÉĶu": 135, |
|
"i": 10, |
|
"ia": 83, |
|
"iau": 136, |
|
"ie": 95, |
|
"ieu": 102, |
|
"ieÅĭ": 238, |
|
"ieÊĶ": 433, |
|
"in": 163, |
|
"iu": 97, |
|
"iũ": 407, |
|
"iÅĭ": 84, |
|
"iÊĶ": 119, |
|
"k": 11, |
|
"kai": 192, |
|
"kaÊĶ": 313, |
|
"ke": 107, |
|
"kh": 87, |
|
"kha": 484, |
|
"khe": 152, |
|
"khi": 249, |
|
"khieu": 427, |
|
"khou": 392, |
|
"khue": 451, |
|
"khui": 312, |
|
"khuɤɯ": 452, |
|
"khy": 237, |
|
"khÉĴ": 223, |
|
"khÉĶ": 303, |
|
"khÉĶu": 175, |
|
"ki": 205, |
|
"kiau": 409, |
|
"kieu": 229, |
|
"kin": 363, |
|
"kiÅĭ": 343, |
|
"kiÉĽu": 443, |
|
"ko": 239, |
|
"kou": 450, |
|
"ku": 108, |
|
"kua": 143, |
|
"kuai": 335, |
|
"kue": 305, |
|
"kuoi": 307, |
|
"kuɤɯ": 306, |
|
"ky": 130, |
|
"kø": 362, |
|
"kɤɯ": 408, |
|
"kÉĴ": 177, |
|
"kÉĶ": 227, |
|
"kÉĶu": 228, |
|
"l": 12, |
|
"le": 240, |
|
"li": 145, |
|
"liu": 456, |
|
"liÊĶ": 365, |
|
"lo": 253, |
|
"lou": 457, |
|
"lua": 454, |
|
"lui": 254, |
|
"ly": 293, |
|
"lø": 410, |
|
"lɤɯ": 455, |
|
"lÉĴ": 364, |
|
"lÉĶ": 453, |
|
"lÉĶu": 241, |
|
"m": 13, |
|
"ma": 215, |
|
"min": 460, |
|
"miÅĭ": 314, |
|
"mui": 459, |
|
"mÉĴ": 366, |
|
"mÉĶ": 458, |
|
"n": 14, |
|
"ni": 279, |
|
"niÅĭ": 461, |
|
"nÉĶu": 367, |
|
"o": 15, |
|
"oi": 100, |
|
"on": 462, |
|
"ou": 98, |
|
"oÅĭ": 180, |
|
"p": 16, |
|
"pa": 255, |
|
"pai": 131, |
|
"pau": 463, |
|
"pe": 120, |
|
"peÊĶ": 465, |
|
"ph": 113, |
|
"phai": 336, |
|
"phaÅĭ": 337, |
|
"phou": 403, |
|
"phua": 315, |
|
"phuai": 369, |
|
"phue": 370, |
|
"phuoi": 372, |
|
"phuɤɯ": 371, |
|
"phÉĶu": 199, |
|
"pi": 201, |
|
"piÅĭ": 242, |
|
"po": 256, |
|
"pou": 368, |
|
"pu": 91, |
|
"pua": 266, |
|
"puai": 197, |
|
"pue": 189, |
|
"puoi": 198, |
|
"puɤɯ": 190, |
|
"pɤɯ": 411, |
|
"pÉĴ": 206, |
|
"pÉĶ": 257, |
|
"pÉĶu": 193, |
|
"pÉĽ": 464, |
|
"s": 17, |
|
"t": 18, |
|
"ta": 280, |
|
"tai": 317, |
|
"te": 153, |
|
"teÅĭ": 467, |
|
"th": 103, |
|
"thai": 292, |
|
"thau": 498, |
|
"thaÅĭ": 499, |
|
"thi": 291, |
|
"thiu": 334, |
|
"thiÅĭ": 402, |
|
"thua": 267, |
|
"thui": 319, |
|
"thy": 401, |
|
"thÉĶu": 276, |
|
"ti": 169, |
|
"tiu": 345, |
|
"tiÅĭ": 466, |
|
"to": 316, |
|
"tou": 294, |
|
"ts": 66, |
|
"tsai": 322, |
|
"tse": 270, |
|
"tsh": 81, |
|
"tsha": 353, |
|
"tshai": 236, |
|
"tshau": 425, |
|
"tshaÅĭ": 483, |
|
"tsheÅĭ": 426, |
|
"tshi": 235, |
|
"tshia": 301, |
|
"tshiu": 261, |
|
"tshui": 323, |
|
"tshy": 213, |
|
"tshø": 331, |
|
"tshÉĴ": 390, |
|
"tshÉĴÊĶ": 391, |
|
"tshÉĶ": 389, |
|
"tsi": 271, |
|
"tsia": 194, |
|
"tsiau": 472, |
|
"tsieu": 297, |
|
"tso": 230, |
|
"tsou": 471, |
|
"tsui": 210, |
|
"tsy": 203, |
|
"tsɤɯ": 416, |
|
"tsÉĴ": 470, |
|
"tsÉĶ": 469, |
|
"tsÉĶu": 259, |
|
"tua": 140, |
|
"tui": 318, |
|
"ty": 139, |
|
"tø": 295, |
|
"tÉĴ": 344, |
|
"tÉĶ": 373, |
|
"tÉĶu": 144, |
|
"u": 19, |
|
"ua": 78, |
|
"uai": 202, |
|
"ue": 216, |
|
"ui": 94, |
|
"uoi": 269, |
|
"uɤɯ": 268, |
|
"y": 20, |
|
"£": 21, |
|
"¤": 22, |
|
"¦": 23, |
|
"©": 24, |
|
"¬": 25, |
|
"¯": 26, |
|
"²": 27, |
|
"¸": 28, |
|
"¼": 29, |
|
"¾": 30, |
|
"Ã": 31, |
|
"ã": 141, |
|
"ø": 101, |
|
"øn": 496, |
|
"øÅĭ": 434, |
|
"øÊĶ": 497, |
|
"Ä": 32, |
|
"Ä©": 243, |
|
"Å": 33, |
|
"Å©": 281, |
|
"Åĭ": 65, |
|
"ÅĭÉĶu": 296, |
|
"Åĵ": 374, |
|
"Ç": 34, |
|
"Ǿ": 61, |
|
"Ǿai": 320, |
|
"Ǿau": 245, |
|
"ǾaÅĭ": 258, |
|
"Ǿe": 146, |
|
"Ǿi": 244, |
|
"Ǿia": 156, |
|
"Ǿiu": 246, |
|
"ǾiÅĭ": 415, |
|
"Ǿo": 413, |
|
"Ǿou": 321, |
|
"Ǿu": 414, |
|
"Ǿua": 209, |
|
"Ǿy": 376, |
|
"ǾÉĴ": 220, |
|
"ǾÉĶ": 219, |
|
"ǾÉĶu": 170, |
|
"É": 35, |
|
"ɤ": 82, |
|
"ɤÅĭ": 222, |
|
"ɤɯ": 86, |
|
"ɤÊĶ": 302, |
|
"ɬ": 67, |
|
"ɬai": 298, |
|
"ɬaÅĭ": 346, |
|
"ɬe": 181, |
|
"ɬi": 182, |
|
"ɬia": 125, |
|
"ɬiau": 324, |
|
"ɬieu": 282, |
|
"ɬiu": 417, |
|
"ɬiÊĶ": 473, |
|
"ɬo": 221, |
|
"ɬu": 247, |
|
"ɬua": 184, |
|
"ɬy": 183, |
|
"ɬø": 185, |
|
"ɬÉĶu": 377, |
|
"ɬÉĶÊĶ": 418, |
|
"ɯ": 85, |
|
"ÉĴ": 74, |
|
"ÉĴÊĶ": 186, |
|
"ÉĶ": 64, |
|
"ÉĶu": 72, |
|
"ÉĶÅĭ": 117, |
|
"ÉĶÊĶ": 122, |
|
"ÉĽ": 99, |
|
"ÉĽn": 400, |
|
"ÉĽu": 173, |
|
"ÉĽÅĭ": 264, |
|
"Ê": 36, |
|
"ÊĶ": 75, |
|
"Ì": 37, |
|
"Ìĥ": 106, |
|
"Î": 38, |
|
"β": 217, |
|
"ã": 39, |
|
"ãĢ": 158, |
|
"ï": 40, |
|
"ï¼": 207, |
|
"Ċ": 41, |
|
"č": 42, |
|
"čĊ": 54, |
|
"Ġ": 43, |
|
"Ġh": 109, |
|
"Ġha": 250, |
|
"ĠhiÅ©": 435, |
|
"Ġhu": 412, |
|
"Ġhua": 468, |
|
"ĠhÉĴ": 251, |
|
"ĠhÉĶ": 308, |
|
"Ġk": 71, |
|
"Ġka": 178, |
|
"Ġkai": 161, |
|
"Ġkau": 274, |
|
"ĠkaÅĭ": 382, |
|
"ĠkeÊĶ": 328, |
|
"Ġkh": 121, |
|
"Ġkhai": 309, |
|
"Ġkhi": 339, |
|
"Ġkhu": 381, |
|
"Ġkhua": 476, |
|
"Ġki": 327, |
|
"Ġkia": 421, |
|
"ĠkiÅĭ": 475, |
|
"Ġko": 273, |
|
"ĠkoÊĶ": 446, |
|
"Ġku": 155, |
|
"Ġkua": 212, |
|
"Ġkuai": 441, |
|
"Ġkuɤɯ": 360, |
|
"Ġky": 284, |
|
"ĠkÉĴ": 172, |
|
"ĠkÉĶ": 166, |
|
"ĠkÉĶu": 285, |
|
"ĠkÉĶÅĭ": 383, |
|
"Ġl": 70, |
|
"Ġlai": 171, |
|
"Ġlau": 111, |
|
"ĠlaÅĭ": 299, |
|
"Ġle": 164, |
|
"ĠleÅĭ": 420, |
|
"ĠleÊĶ": 233, |
|
"Ġli": 104, |
|
"Ġlia": 380, |
|
"Ġliu": 300, |
|
"ĠliÊĶ": 225, |
|
"Ġlo": 326, |
|
"Ġlua": 474, |
|
"Ġlui": 165, |
|
"Ġly": 349, |
|
"ĠlÉĴ": 379, |
|
"ĠlÉĽ": 195, |
|
"ĠlÉĽÊĶ": 361, |
|
"Ġm": 88, |
|
"Ġma": 126, |
|
"Ġmai": 262, |
|
"Ġme": 485, |
|
"Ġmi": 275, |
|
"Ġmia": 429, |
|
"ĠmiÅĭ": 393, |
|
"Ġmua": 289, |
|
"Ġmuai": 430, |
|
"Ġmue": 486, |
|
"ĠmÉĴ": 332, |
|
"ĠmÉĶ": 428, |
|
"Ġn": 79, |
|
"Ġnai": 329, |
|
"Ġnau": 157, |
|
"ĠnaÅĭ": 385, |
|
"Ġne": 350, |
|
"Ġni": 150, |
|
"Ġnieu": 387, |
|
"ĠniÅĭ": 128, |
|
"Ġnu": 351, |
|
"Ġnua": 478, |
|
"Ġnui": 286, |
|
"Ġny": 422, |
|
"Ġnø": 386, |
|
"ĠnÉĴ": 384, |
|
"ĠnÉĶ": 477, |
|
"ĠnÉĶÅĭ": 479, |
|
"Ġp": 96, |
|
"Ġpai": 304, |
|
"Ġph": 148, |
|
"Ġphi": 342, |
|
"ĠpiÊĶ": 495, |
|
"Ġpo": 224, |
|
"Ġpu": 375, |
|
"Ġpɤɯ": 356, |
|
"ĠpÉĶÊĶ": 399, |
|
"Ġt": 89, |
|
"Ġtai": 214, |
|
"Ġtau": 396, |
|
"Ġte": 196, |
|
"ĠteÅĭ": 290, |
|
"Ġth": 487, |
|
"Ġthe": 333, |
|
"Ġti": 394, |
|
"Ġts": 90, |
|
"Ġtse": 354, |
|
"ĠtseÊĶ": 398, |
|
"Ġtsh": 137, |
|
"Ġtshy": 341, |
|
"Ġtsi": 188, |
|
"Ġtsia": 263, |
|
"Ġtsieu": 432, |
|
"ĠtsiÅĭ": 492, |
|
"ĠtsiÊĶ": 493, |
|
"Ġtsui": 355, |
|
"Ġtsy": 179, |
|
"ĠtsÉĴ": 491, |
|
"ĠtsÉĴÊĶ": 494, |
|
"ĠtsÉĶ": 397, |
|
"Ġtua": 489, |
|
"Ġtui": 490, |
|
"Ġty": 431, |
|
"ĠtÉĴ": 395, |
|
"ĠtÉĶ": 488, |
|
"ĠÅĭ": 127, |
|
"ĠÅĭiÅĭ": 359, |
|
"ĠÅĭua": 340, |
|
"ĠÅĭÉĶu": 404, |
|
"ĠǾ": 68, |
|
"ĠǾa": 160, |
|
"ĠǾau": 138, |
|
"ĠǾaÅĭ": 211, |
|
"ĠǾe": 260, |
|
"ĠǾeÊĶ": 419, |
|
"ĠǾi": 114, |
|
"ĠǾia": 325, |
|
"ĠǾiau": 437, |
|
"ĠǾiu": 231, |
|
"ĠǾiÅĭ": 272, |
|
"ĠǾiÊĶ": 436, |
|
"ĠǾo": 378, |
|
"ĠǾou": 348, |
|
"ĠǾu": 115, |
|
"ĠǾua": 149, |
|
"ĠǾuai": 338, |
|
"ĠǾue": 357, |
|
"ĠǾuoi": 358, |
|
"ĠǾuã": 438, |
|
"ĠǾy": 174, |
|
"ĠǾyÅĭ": 442, |
|
"ĠǾÉĴ": 154, |
|
"ĠǾÉĴÅĭ": 310, |
|
"ĠǾÉĶ": 347, |
|
"ĠǾÉĶu": 248, |
|
"ĠǾÉĶÅĭ": 283, |
|
"ĠÇ¾ÉĽ": 232, |
|
"ĠÇ¾ÉĽÊĶ": 445, |
|
"Ġɬ": 80, |
|
"Ġɬa": 330, |
|
"Ġɬe": 151, |
|
"Ġɬi": 187, |
|
"Ġɬia": 234, |
|
"Ġɬieu": 352, |
|
"ĠɬiÊĶ": 288, |
|
"Ġɬo": 480, |
|
"Ġɬu": 423, |
|
"Ġɬui": 481, |
|
"Ġɬy": 167, |
|
"ĠɬÉĴ": 424, |
|
"ĠɬÉĶ": 388, |
|
"ĠɬÉĶu": 287, |
|
"ĠɬÉĶÊĶ": 482, |
|
"Ġβ": 218, |
|
"ĠãĢ": 159, |
|
"ĠãĢĤ": 162, |
|
"Ġï¼": 208, |
|
"Ġï¼Į": 444, |
|
"Ġï¼Ł": 277, |
|
"ĠčĊ": 55, |
|
"Ģ": 44, |
|
"Ĥ": 45, |
|
"ĥ": 46, |
|
"ĭ": 47, |
|
"Į": 48, |
|
"Ĵ": 49, |
|
"ĵ": 50, |
|
"Ķ": 51, |
|
"Ľ": 52, |
|
"Ł": 53 |
|
} |
|
|