Update tokenizer_config.json
Browse files- tokenizer_config.json +15 -12
tokenizer_config.json
CHANGED
@@ -6172,16 +6172,19 @@
|
|
6172 |
"special": true
|
6173 |
}
|
6174 |
},
|
6175 |
-
"bos_token":
|
6176 |
-
"
|
6177 |
-
"
|
6178 |
-
"
|
6179 |
-
"
|
6180 |
-
"
|
6181 |
-
"
|
6182 |
-
"
|
6183 |
-
"
|
6184 |
-
|
6185 |
-
"
|
6186 |
-
"
|
|
|
|
|
6187 |
}
|
|
|
|
6172 |
"special": true
|
6173 |
}
|
6174 |
},
|
6175 |
+
"bos_token":"<s>"
|
6176 |
+
"chat_template":"{{ bos_token }}{% for message in messages %}{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% if message['role'] == 'user' %}{{ '[INST] ' + message['content'] + ' [/INST]' }}{% elif message['role'] == 'assistant' %}{{ message['content'] + eos_token}}{% else %}{{ raise_exception('Only user and assistant roles are supported!') }}{% endif %}{% endfor %}"
|
6177 |
+
"clean_up_tokenization_spaces":false
|
6178 |
+
"eos_token":"</s>"
|
6179 |
+
"legacy":false
|
6180 |
+
"model_max_length":1e+30
|
6181 |
+
"pad_token":"[control_768]"
|
6182 |
+
"padding_side":"left"
|
6183 |
+
"sp_model_kwargs":{
|
6184 |
+
}
|
6185 |
+
"spaces_between_special_tokens":false
|
6186 |
+
"tokenizer_class":"LlamaTokenizer"
|
6187 |
+
"unk_token":"<unk>"
|
6188 |
+
"use_default_system_prompt":false
|
6189 |
}
|
6190 |
+
|