bart-base-cantonese / tokenizer_config.json
indiejoseph's picture
Model save
8e5388c
raw
history blame
18.5 kB
{
"added_tokens_decoder": {
"0": {
"content": "[PAD]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"100": {
"content": "[UNK]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"101": {
"content": "[CLS]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"102": {
"content": "[SEP]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"103": {
"content": "[MASK]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"104": {
"content": "[EOS]",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"51271": {
"content": "𡃉",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"51272": {
"content": "𠻺",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"51273": {
"content": "𠸏",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"51274": {
"content": "㖭",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"51275": {
"content": "𨂾",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"51276": {
"content": "𤗲",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"51277": {
"content": "𡁵",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"51278": {
"content": "𥄫",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"51279": {
"content": "𤘅",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"51280": {
"content": "㨂",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"51281": {
"content": "臰",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"51282": {
"content": "㗳",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"51283": {
"content": "𠵿",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"51284": {
"content": "㗅",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"51285": {
"content": "𠱁",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"51286": {
"content": "𠴱",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"51287": {
"content": "䚎",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"51288": {
"content": "㗩",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"51289": {
"content": "𢯎",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"51290": {
"content": "𦟌",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"51291": {
"content": "𧥺",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"51292": {
"content": "𢱕",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"51293": {
"content": "𨂽",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"51294": {
"content": "𢫏",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"51295": {
"content": "㧬",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"51296": {
"content": "䄛",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"51297": {
"content": "㔷",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"51298": {
"content": "𧇣",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"51299": {
"content": "𪐀",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"51300": {
"content": "𢭃",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"51301": {
"content": "𢲡",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"51302": {
"content": "嘺",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"51303": {
"content": "㗱",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"51304": {
"content": "𠹺",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"51305": {
"content": "𠸎",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"51306": {
"content": "𡃈",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"51307": {
"content": "䑛",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"51308": {
"content": "㗾",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"51309": {
"content": "𦢊",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"51310": {
"content": "𠿪",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"51311": {
"content": "䊦",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"51312": {
"content": "㪣",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"51313": {
"content": "㗇",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"51314": {
"content": "㴝",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"51315": {
"content": "䘆",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"51316": {
"content": "㬢",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"51317": {
"content": "䤵",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"51318": {
"content": "𠺝",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"51319": {
"content": "𧊅",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"51320": {
"content": "㘝",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"51321": {
"content": "䯀",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"51322": {
"content": "𡲢",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"51323": {
"content": "𢬿",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"51324": {
"content": "蠬",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"51325": {
"content": "㘉",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"51326": {
"content": "𥇣",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"51327": {
"content": "㭍",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"51328": {
"content": "𩰫",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"51329": {
"content": "㨢",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"51330": {
"content": "𪄠",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"51331": {
"content": "𡃀",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"51332": {
"content": "䄞",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"51333": {
"content": "𩪷",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"51334": {
"content": "𠽤",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"51335": {
"content": "𠼮",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"51336": {
"content": "𤶸",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"51337": {
"content": "𤗈",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"51338": {
"content": "㨘",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"51339": {
"content": "𡃴",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"51340": {
"content": "𠴕",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"51341": {
"content": "䳆",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"51342": {
"content": "㱽",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"51343": {
"content": "𠮨",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"51344": {
"content": "𪘁",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"51345": {
"content": "㤿",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"51346": {
"content": "𡷹",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"51347": {
"content": "𤺊",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"51348": {
"content": "䢂",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"51349": {
"content": "𨇣",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"51350": {
"content": "㢌",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"51351": {
"content": "䂿",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"51352": {
"content": "㕪",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"51353": {
"content": "䑪",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"51354": {
"content": "𠓼",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"51355": {
"content": "䁓",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"51356": {
"content": "𠯔",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"51357": {
"content": "䉺",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"51358": {
"content": "𦉘",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"51359": {
"content": "𠼻",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"51360": {
"content": "𨅝",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"51361": {
"content": "㬟",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"51362": {
"content": "𢡱",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"51363": {
"content": "㔻",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"51364": {
"content": "䚗",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"51365": {
"content": "𢲩",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"51366": {
"content": "㨾",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"51367": {
"content": "𧨾",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"51368": {
"content": "𪙛",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"51369": {
"content": "𠸉",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"51370": {
"content": "䇯",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
}
},
"bos_token": "[CLS]",
"clean_up_tokenization_spaces": true,
"cls_token": "[CLS]",
"do_basic_tokenize": true,
"do_lower_case": false,
"eos_token": "[EOS]",
"mask_token": "[MASK]",
"model_max_length": 1000000000000000019884624838656,
"never_split": null,
"pad_token": "[PAD]",
"sep_token": "[SEP]",
"strip_accents": null,
"tokenize_chinese_chars": true,
"tokenizer_class": "BertTokenizer",
"unk_token": "[UNK]",
"use_fast": true
}