Remove <sep> from tokenizer special tokens
#4
by
gabegoodhart
- opened
- added_tokens.json +0 -1
- tokenizer.json +1 -10
- tokenizer_config.json +1 -9
added_tokens.json
CHANGED
@@ -1,5 +1,4 @@
|
|
1 |
{
|
2 |
-
"<sep>": 49155,
|
3 |
"<|end_of_role|>": 49153,
|
4 |
"<|start_of_role|>": 49152,
|
5 |
"<|tool_call|>": 49154
|
|
|
1 |
{
|
|
|
2 |
"<|end_of_role|>": 49153,
|
3 |
"<|start_of_role|>": 49152,
|
4 |
"<|tool_call|>": 49154
|
tokenizer.json
CHANGED
@@ -200,15 +200,6 @@
|
|
200 |
"rstrip": false,
|
201 |
"normalized": false,
|
202 |
"special": true
|
203 |
-
},
|
204 |
-
{
|
205 |
-
"id": 49155,
|
206 |
-
"content": "<sep>",
|
207 |
-
"single_word": false,
|
208 |
-
"lstrip": false,
|
209 |
-
"rstrip": false,
|
210 |
-
"normalized": false,
|
211 |
-
"special": true
|
212 |
}
|
213 |
],
|
214 |
"normalizer": null,
|
@@ -244960,4 +244951,4 @@
|
|
244960 |
]
|
244961 |
]
|
244962 |
}
|
244963 |
-
}
|
|
|
200 |
"rstrip": false,
|
201 |
"normalized": false,
|
202 |
"special": true
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
203 |
}
|
204 |
],
|
205 |
"normalizer": null,
|
|
|
244951 |
]
|
244952 |
]
|
244953 |
}
|
244954 |
+
}
|
tokenizer_config.json
CHANGED
@@ -177,14 +177,6 @@
|
|
177 |
"rstrip": false,
|
178 |
"single_word": false,
|
179 |
"special": true
|
180 |
-
},
|
181 |
-
"49155": {
|
182 |
-
"content": "<sep>",
|
183 |
-
"lstrip": false,
|
184 |
-
"normalized": false,
|
185 |
-
"rstrip": false,
|
186 |
-
"single_word": false,
|
187 |
-
"special": true
|
188 |
}
|
189 |
},
|
190 |
"additional_special_tokens": [
|
@@ -204,4 +196,4 @@
|
|
204 |
"tokenizer_class": "GPT2Tokenizer",
|
205 |
"unk_token": "<|end_of_text|>",
|
206 |
"vocab_size": 49152
|
207 |
-
}
|
|
|
177 |
"rstrip": false,
|
178 |
"single_word": false,
|
179 |
"special": true
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
180 |
}
|
181 |
},
|
182 |
"additional_special_tokens": [
|
|
|
196 |
"tokenizer_class": "GPT2Tokenizer",
|
197 |
"unk_token": "<|end_of_text|>",
|
198 |
"vocab_size": 49152
|
199 |
+
}
|