pinot's picture
Upload tokenizer
972fc75
{
"</s>": 2,
"<pad>": 0,
"<s>": 1,
"<unk>": 3,
"|": 4,
"ア": 5,
"アー": 6,
"イ": 7,
"イェ": 8,
"イェー": 9,
"イー": 10,
"ウ": 11,
"ウィ": 12,
"ウィー": 13,
"ウェ": 14,
"ウェー": 15,
"ウォ": 16,
"ウォー": 17,
"ウー": 18,
"エ": 19,
"エー": 20,
"オ": 21,
"オー": 22,
"カ": 23,
"カー": 24,
"ガ": 25,
"ガー": 26,
"キ": 27,
"キャ": 28,
"キャー": 29,
"キュ": 30,
"キュー": 31,
"キョ": 32,
"キョー": 33,
"キー": 34,
"ギ": 35,
"ギャ": 36,
"ギャー": 37,
"ギュ": 38,
"ギュー": 39,
"ギョ": 40,
"ギョー": 41,
"ギー": 42,
"ク": 43,
"クヮ": 44,
"クヮー": 45,
"クー": 46,
"グ": 47,
"グヮ": 48,
"グヮー": 49,
"グー": 50,
"ケ": 51,
"ケー": 52,
"ゲ": 53,
"ゲー": 54,
"コ": 55,
"コー": 56,
"ゴ": 57,
"ゴー": 58,
"サ": 59,
"サー": 60,
"ザ": 61,
"ザー": 62,
"シ": 63,
"シェ": 64,
"シェー": 65,
"シャ": 66,
"シャー": 67,
"シュ": 68,
"シュー": 69,
"ショ": 70,
"ショー": 71,
"シー": 72,
"ジ": 73,
"ジェ": 74,
"ジェー": 75,
"ジャ": 76,
"ジャー": 77,
"ジュ": 78,
"ジュー": 79,
"ジョ": 80,
"ジョー": 81,
"ジー": 82,
"ス": 83,
"スィ": 84,
"スィー": 85,
"スー": 86,
"ズ": 87,
"ズィ": 88,
"ズィー": 89,
"ズー": 90,
"セ": 91,
"セー": 92,
"ゼ": 93,
"ゼー": 94,
"ソ": 95,
"ソー": 96,
"ゾ": 97,
"ゾー": 98,
"タ": 99,
"ター": 100,
"ダ": 101,
"ダー": 102,
"チ": 103,
"チェ": 104,
"チェー": 105,
"チャ": 106,
"チャー": 107,
"チュ": 108,
"チュー": 109,
"チョ": 110,
"チョー": 111,
"チー": 112,
"ヂ": 113,
"ヂャ": 114,
"ヂャー": 115,
"ヂュ": 116,
"ヂュー": 117,
"ヂョ": 118,
"ヂョー": 119,
"ヂー": 120,
"ッ": 121,
"ッー": 122,
"ツ": 123,
"ツァ": 124,
"ツァー": 125,
"ツィ": 126,
"ツィー": 127,
"ツェ": 128,
"ツェー": 129,
"ツォ": 130,
"ツォー": 131,
"ツャ": 132,
"ツャー": 133,
"ツュ": 134,
"ツュー": 135,
"ツー": 136,
"ヅ": 137,
"ヅー": 138,
"テ": 139,
"ティ": 140,
"ティー": 141,
"テュ": 142,
"テュー": 143,
"テー": 144,
"デ": 145,
"ディ": 146,
"ディー": 147,
"デュ": 148,
"デュー": 149,
"デー": 150,
"ト": 151,
"トゥ": 152,
"トゥー": 153,
"トー": 154,
"ド": 155,
"ドゥ": 156,
"ドゥー": 157,
"ドー": 158,
"ナ": 159,
"ナー": 160,
"ニ": 161,
"ニェ": 162,
"ニェー": 163,
"ニャ": 164,
"ニャー": 165,
"ニュ": 166,
"ニュー": 167,
"ニョ": 168,
"ニョー": 169,
"ニー": 170,
"ヌ": 171,
"ヌー": 172,
"ネ": 173,
"ネー": 174,
"ノ": 175,
"ノー": 176,
"ハ": 177,
"ハー": 178,
"バ": 179,
"バー": 180,
"パ": 181,
"パー": 182,
"ヒ": 183,
"ヒェ": 184,
"ヒェー": 185,
"ヒャ": 186,
"ヒャー": 187,
"ヒュ": 188,
"ヒュー": 189,
"ヒョ": 190,
"ヒョー": 191,
"ヒー": 192,
"ビ": 193,
"ビャ": 194,
"ビャー": 195,
"ビュ": 196,
"ビュー": 197,
"ビョ": 198,
"ビョー": 199,
"ビー": 200,
"ピ": 201,
"ピャ": 202,
"ピャー": 203,
"ピュ": 204,
"ピュー": 205,
"ピョ": 206,
"ピョー": 207,
"ピー": 208,
"フ": 209,
"ファ": 210,
"ファー": 211,
"フィ": 212,
"フィー": 213,
"フェ": 214,
"フェー": 215,
"フォ": 216,
"フォー": 217,
"フュ": 218,
"フュー": 219,
"フー": 220,
"ブ": 221,
"ブー": 222,
"プ": 223,
"プー": 224,
"ヘ": 225,
"ヘー": 226,
"ベ": 227,
"ベー": 228,
"ペ": 229,
"ペー": 230,
"ホ": 231,
"ホー": 232,
"ボ": 233,
"ボー": 234,
"ポ": 235,
"ポー": 236,
"マ": 237,
"マー": 238,
"ミ": 239,
"ミェ": 240,
"ミェー": 241,
"ミャ": 242,
"ミャー": 243,
"ミュ": 244,
"ミュー": 245,
"ミョ": 246,
"ミョー": 247,
"ミー": 248,
"ム": 249,
"ムー": 250,
"メ": 251,
"メー": 252,
"モ": 253,
"モー": 254,
"ヤ": 255,
"ヤー": 256,
"ユ": 257,
"ユー": 258,
"ヨ": 259,
"ヨー": 260,
"ラ": 261,
"ラー": 262,
"リ": 263,
"リャ": 264,
"リャー": 265,
"リュ": 266,
"リュー": 267,
"リョ": 268,
"リョー": 269,
"リー": 270,
"ル": 271,
"ルー": 272,
"レ": 273,
"レー": 274,
"ロ": 275,
"ロー": 276,
"ワ": 277,
"ワー": 278,
"ヲ": 279,
"ヲー": 280,
"ン": 281,
"ンー": 282,
"ヴ": 283,
"ヴァ": 284,
"ヴァー": 285,
"ヴィ": 286,
"ヴィー": 287,
"ヴェ": 288,
"ヴェー": 289,
"ヴォ": 290,
"ヴォー": 291,
"ヴュ": 292,
"ヴュー": 293,
"ヴー": 294
}