Remove <sep> from tokenizer special tokens

#4
Files changed (3) hide show
  1. added_tokens.json +0 -1
  2. tokenizer.json +1 -10
  3. tokenizer_config.json +1 -9
added_tokens.json CHANGED
@@ -1,5 +1,4 @@
1
  {
2
- "<sep>": 49155,
3
  "<|end_of_role|>": 49153,
4
  "<|start_of_role|>": 49152,
5
  "<|tool_call|>": 49154
 
1
  {
 
2
  "<|end_of_role|>": 49153,
3
  "<|start_of_role|>": 49152,
4
  "<|tool_call|>": 49154
tokenizer.json CHANGED
@@ -200,15 +200,6 @@
200
  "rstrip": false,
201
  "normalized": false,
202
  "special": true
203
- },
204
- {
205
- "id": 49155,
206
- "content": "<sep>",
207
- "single_word": false,
208
- "lstrip": false,
209
- "rstrip": false,
210
- "normalized": false,
211
- "special": true
212
  }
213
  ],
214
  "normalizer": null,
@@ -244960,4 +244951,4 @@
244960
  ]
244961
  ]
244962
  }
244963
- }
 
200
  "rstrip": false,
201
  "normalized": false,
202
  "special": true
 
 
 
 
 
 
 
 
 
203
  }
204
  ],
205
  "normalizer": null,
 
244951
  ]
244952
  ]
244953
  }
244954
+ }
tokenizer_config.json CHANGED
@@ -177,14 +177,6 @@
177
  "rstrip": false,
178
  "single_word": false,
179
  "special": true
180
- },
181
- "49155": {
182
- "content": "<sep>",
183
- "lstrip": false,
184
- "normalized": false,
185
- "rstrip": false,
186
- "single_word": false,
187
- "special": true
188
  }
189
  },
190
  "additional_special_tokens": [
@@ -204,4 +196,4 @@
204
  "tokenizer_class": "GPT2Tokenizer",
205
  "unk_token": "<|end_of_text|>",
206
  "vocab_size": 49152
207
- }
 
177
  "rstrip": false,
178
  "single_word": false,
179
  "special": true
 
 
 
 
 
 
 
 
180
  }
181
  },
182
  "additional_special_tokens": [
 
196
  "tokenizer_class": "GPT2Tokenizer",
197
  "unk_token": "<|end_of_text|>",
198
  "vocab_size": 49152
199
+ }