|
{ |
|
"add_prefix_space": false, |
|
"added_tokens_decoder": { |
|
"0": { |
|
"content": "<|endoftext|>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"1": { |
|
"content": "<fim_prefix>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"2": { |
|
"content": "<fim_middle>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"3": { |
|
"content": "<fim_suffix>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"4": { |
|
"content": "<fim_pad>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"5": { |
|
"content": "<filename>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"6": { |
|
"content": "<gh_stars>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"7": { |
|
"content": "<issue_start>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"8": { |
|
"content": "<issue_comment>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"9": { |
|
"content": "<issue_closed>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"10": { |
|
"content": "<jupyter_start>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"11": { |
|
"content": "<jupyter_text>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"12": { |
|
"content": "<jupyter_code>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"13": { |
|
"content": "<jupyter_output>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"14": { |
|
"content": "<empty_output>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"15": { |
|
"content": "<commit_before>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"16": { |
|
"content": "<commit_msg>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"17": { |
|
"content": "<commit_after>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
}, |
|
"18": { |
|
"content": "<reponame>", |
|
"lstrip": false, |
|
"normalized": false, |
|
"rstrip": false, |
|
"single_word": false, |
|
"special": true |
|
} |
|
}, |
|
"additional_special_tokens": [ |
|
"<|endoftext|>", |
|
"<fim_prefix>", |
|
"<fim_middle>", |
|
"<fim_suffix>", |
|
"<fim_pad>", |
|
"<filename>", |
|
"<gh_stars>", |
|
"<issue_start>", |
|
"<issue_comment>", |
|
"<issue_closed>", |
|
"<jupyter_start>", |
|
"<jupyter_text>", |
|
"<jupyter_code>", |
|
"<jupyter_output>", |
|
"<empty_output>", |
|
"<commit_before>", |
|
"<commit_msg>", |
|
"<commit_after>", |
|
"<reponame>" |
|
], |
|
"bos_token": "<|endoftext|>", |
|
"chat_template": "{% set function_str = messages.get('functions_str', {}) %}\n{% set query = messages['query'] %}\n{% set sys_prompt = 'You are a helpful assistant with access to the following function calls. Your task is to produce a sequence of function calls necessary to generate response to the user utterance. Use the following function calls as required. ' %}\n{% set funcstr = function_str|join('\n') %}\n{{ 'SYSTEM: ' + sys_prompt + '\n<|function_call_library|>\n' + funcstr + '\n\nIf none of the functions are relevant or the given question lacks the parameters required by the function, please output \"<function_call> {\"name\": \"no_function\", \"arguments\": {}}\".\n\nUSER: ' + query}}\n{% if add_generation_prompt %}\n{{ 'ASSISTANT:' }}{% endif %}", |
|
"clean_up_tokenization_spaces": true, |
|
"eos_token": "<|endoftext|>", |
|
"model_max_length": 8192, |
|
"pad_token": "<|endoftext|>", |
|
"padding_side": "left", |
|
"tokenizer_class": "GPT2Tokenizer", |
|
"unk_token": "<|endoftext|>", |
|
"vocab_size": 49152 |
|
} |
|
|