File size: 2,559 Bytes
f8e7c01 bff6ac3 1ef6e41 f8e7c01 1ef6e41 f8e7c01 bff6ac3 1ef6e41 f8e7c01 1ef6e41 f8e7c01 1ef6e41 bff6ac3 1ef6e41 bff6ac3 1ef6e41 bff6ac3 1ef6e41 bff6ac3 1ef6e41 50ec22b 1ef6e41 50ec22b 1ef6e41 bff6ac3 1ef6e41 50ec22b 1ef6e41 50ec22b 1ef6e41 bff6ac3 1ef6e41 50ec22b 1ef6e41 50ec22b 1ef6e41 bff6ac3 1ef6e41 50ec22b 1ef6e41 50ec22b f8e7c01 bff6ac3 f8e7c01 bff6ac3 f8e7c01 bff6ac3 f8e7c01 bff6ac3 f8e7c01 bff6ac3 f8e7c01 bff6ac3 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 |
{
"added_tokens_decoder": {
"0": {
"content": "<unk>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"1": {
"content": "<s>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"2": {
"content": "</s>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"32000": {
"content": "<CLS>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"32001": {
"content": "<SEP>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"32002": {
"content": "<EOD>",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false,
"special": false
},
"32003": {
"content": "<MASK>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"32004": {
"content": "<PAD>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"32005": {
"content": "<|im_start|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"32006": {
"content": "<|im_end|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
}
},
"additional_special_tokens": [
"<|im_start|>",
"<|im_end|>"
],
"bos_token": "<|im_start|>",
"chat_template": "{% if not add_generation_prompt is defined %}{% set add_generation_prompt = false %}{% endif %}{% for message in messages %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}",
"clean_up_tokenization_spaces": false,
"cls_token": "<CLS>",
"eos_token": "<|im_end|>",
"legacy": true,
"mask_token": "<MASK>",
"model_max_length": 1000000000000000019884624838656,
"pad_token": "<PAD>",
"padding_side": "right",
"sep_token": "<SEP>",
"sp_model_kwargs": {},
"tokenizer_class": "LlamaTokenizer",
"unk_token": "<unk>",
"use_default_system_prompt": false
} |