Hugging Face's logo

elliotthwangmsa
/
KimLanpure-phi-3-zh

Text Generation
Transformers
Safetensors
phi3
conversational
custom_code
text-generation-inference
Inference Endpoints
Model card Files Community
elliotthwangmsa commited on
Commit
b3fc4b3
•
1 Parent(s): 7bffe52

Upload tokenizer

Browse files
Files changed (4) hide show
  1. added_tokens.json +6 -35
  2. special_tokens_map.json +2 -5
  3. tokenizer.json +16 -277
  4. tokenizer_config.json +18 -253
added_tokens.json CHANGED
@@ -1,42 +1,13 @@
1
  {
2
- "<|/code|>": 32014,
3
- "<|/data|>": 32033,
4
- "<|/inst|>": 32037,
5
- "<|/query|>": 32031,
6
- "<|/sys|>": 32035,
7
- "<|assistant_mask|>": 32017,
8
  "<|assistant|>": 32001,
9
- "<|calc|>": 32012,
10
- "<|code|>": 32013,
11
- "<|continue|>": 32009,
12
- "<|data|>": 32032,
13
- "<|diff_marker|>": 32025,
14
- "<|disc_sep|>": 32029,
15
- "<|disc_start|>": 32028,
16
- "<|disc_thread|><|query|>": 32030,
17
  "<|endoftext|>": 32000,
18
  "<|end|>": 32007,
19
- "<|fim_middle|>": 32021,
20
- "<|fim_prefix|>": 32020,
21
- "<|fim_suffix|>": 32022,
22
- "<|function_call|>": 32005,
23
- "<|function_list|>": 32011,
24
- "<|function_output|>": 32003,
25
- "<|ghissue|>": 32026,
26
- "<|ghreview|>": 32027,
27
- "<|im_end|>": 32039,
28
- "<|im_start|>": 32038,
29
- "<|inst|>": 32036,
30
- "<|ipynb_marker|>": 32024,
31
- "<|message|>": 32019,
32
- "<|meta_start|>": 32023,
33
- "<|raw|>": 32008,
34
- "<|resource|>": 32016,
35
- "<|start|>": 32018,
36
- "<|step|>": 32002,
37
- "<|summary|>": 32015,
38
  "<|system|>": 32006,
39
- "<|sys|>": 32034,
40
- "<|tag|>": 32004,
41
  "<|user|>": 32010
42
  }
 
1
  {
 
 
 
 
 
 
2
  "<|assistant|>": 32001,
 
 
 
 
 
 
 
 
3
  "<|endoftext|>": 32000,
4
  "<|end|>": 32007,
5
+ "<|placeholder1|>": 32002,
6
+ "<|placeholder2|>": 32003,
7
+ "<|placeholder3|>": 32004,
8
+ "<|placeholder4|>": 32005,
9
+ "<|placeholder5|>": 32008,
10
+ "<|placeholder6|>": 32009,
 
 
 
 
 
 
 
 
 
 
 
 
 
11
  "<|system|>": 32006,
 
 
12
  "<|user|>": 32010
13
  }
special_tokens_map.json CHANGED
@@ -1,7 +1,4 @@
1
  {
2
- "additional_special_tokens": [
3
- "<|/inst|>"
4
- ],
5
  "bos_token": {
6
  "content": "<s>",
7
  "lstrip": false,
@@ -10,13 +7,13 @@
10
  "single_word": false
11
  },
12
  "eos_token": {
13
- "content": "<|im_end|>",
14
  "lstrip": false,
15
  "normalized": false,
16
  "rstrip": false,
17
  "single_word": false
18
  },
19
- "pad_token": "<|im_end|>",
20
  "unk_token": {
21
  "content": "<unk>",
22
  "lstrip": false,
 
1
  {
 
 
 
2
  "bos_token": {
3
  "content": "<s>",
4
  "lstrip": false,
 
7
  "single_word": false
8
  },
9
  "eos_token": {
10
+ "content": "<|endoftext|>",
11
  "lstrip": false,
12
  "normalized": false,
13
  "rstrip": false,
14
  "single_word": false
15
  },
16
+ "pad_token": "<|endoftext|>",
17
  "unk_token": {
18
  "content": "<unk>",
19
  "lstrip": false,
tokenizer.json CHANGED
@@ -26,7 +26,7 @@
26
  "content": "</s>",
27
  "single_word": false,
28
  "lstrip": false,
29
- "rstrip": true,
30
  "normalized": false,
31
  "special": true
32
  },
@@ -44,43 +44,43 @@
44
  "content": "<|assistant|>",
45
  "single_word": false,
46
  "lstrip": false,
47
- "rstrip": true,
48
  "normalized": false,
49
  "special": true
50
  },
51
  {
52
  "id": 32002,
53
- "content": "<|step|>",
54
  "single_word": false,
55
  "lstrip": false,
56
- "rstrip": true,
57
  "normalized": false,
58
  "special": true
59
  },
60
  {
61
  "id": 32003,
62
- "content": "<|function_output|>",
63
  "single_word": false,
64
  "lstrip": false,
65
- "rstrip": true,
66
  "normalized": false,
67
  "special": true
68
  },
69
  {
70
  "id": 32004,
71
- "content": "<|tag|>",
72
  "single_word": false,
73
  "lstrip": false,
74
- "rstrip": true,
75
  "normalized": false,
76
  "special": true
77
  },
78
  {
79
  "id": 32005,
80
- "content": "<|function_call|>",
81
  "single_word": false,
82
  "lstrip": false,
83
- "rstrip": true,
84
  "normalized": false,
85
  "special": true
86
  },
@@ -89,7 +89,7 @@
89
  "content": "<|system|>",
90
  "single_word": false,
91
  "lstrip": false,
92
- "rstrip": true,
93
  "normalized": false,
94
  "special": true
95
  },
@@ -98,25 +98,25 @@
98
  "content": "<|end|>",
99
  "single_word": false,
100
  "lstrip": false,
101
- "rstrip": true,
102
  "normalized": false,
103
  "special": true
104
  },
105
  {
106
  "id": 32008,
107
- "content": "<|raw|>",
108
  "single_word": false,
109
  "lstrip": false,
110
- "rstrip": true,
111
  "normalized": false,
112
  "special": true
113
  },
114
  {
115
  "id": 32009,
116
- "content": "<|continue|>",
117
  "single_word": false,
118
  "lstrip": false,
119
- "rstrip": true,
120
  "normalized": false,
121
  "special": true
122
  },
@@ -125,267 +125,6 @@
125
  "content": "<|user|>",
126
  "single_word": false,
127
  "lstrip": false,
128
- "rstrip": true,
129
- "normalized": false,
130
- "special": true
131
- },
132
- {
133
- "id": 32011,
134
- "content": "<|function_list|>",
135
- "single_word": false,
136
- "lstrip": false,
137
- "rstrip": true,
138
- "normalized": false,
139
- "special": true
140
- },
141
- {
142
- "id": 32012,
143
- "content": "<|calc|>",
144
- "single_word": false,
145
- "lstrip": false,
146
- "rstrip": true,
147
- "normalized": false,
148
- "special": true
149
- },
150
- {
151
- "id": 32013,
152
- "content": "<|code|>",
153
- "single_word": false,
154
- "lstrip": false,
155
- "rstrip": true,
156
- "normalized": false,
157
- "special": true
158
- },
159
- {
160
- "id": 32014,
161
- "content": "<|/code|>",
162
- "single_word": false,
163
- "lstrip": false,
164
- "rstrip": true,
165
- "normalized": false,
166
- "special": true
167
- },
168
- {
169
- "id": 32015,
170
- "content": "<|summary|>",
171
- "single_word": false,
172
- "lstrip": false,
173
- "rstrip": true,
174
- "normalized": false,
175
- "special": true
176
- },
177
- {
178
- "id": 32016,
179
- "content": "<|resource|>",
180
- "single_word": false,
181
- "lstrip": false,
182
- "rstrip": true,
183
- "normalized": false,
184
- "special": true
185
- },
186
- {
187
- "id": 32017,
188
- "content": "<|assistant_mask|>",
189
- "single_word": false,
190
- "lstrip": false,
191
- "rstrip": true,
192
- "normalized": false,
193
- "special": true
194
- },
195
- {
196
- "id": 32018,
197
- "content": "<|start|>",
198
- "single_word": false,
199
- "lstrip": false,
200
- "rstrip": true,
201
- "normalized": false,
202
- "special": true
203
- },
204
- {
205
- "id": 32019,
206
- "content": "<|message|>",
207
- "single_word": false,
208
- "lstrip": false,
209
- "rstrip": true,
210
- "normalized": false,
211
- "special": true
212
- },
213
- {
214
- "id": 32020,
215
- "content": "<|fim_prefix|>",
216
- "single_word": false,
217
- "lstrip": false,
218
- "rstrip": true,
219
- "normalized": false,
220
- "special": true
221
- },
222
- {
223
- "id": 32021,
224
- "content": "<|fim_middle|>",
225
- "single_word": false,
226
- "lstrip": false,
227
- "rstrip": true,
228
- "normalized": false,
229
- "special": true
230
- },
231
- {
232
- "id": 32022,
233
- "content": "<|fim_suffix|>",
234
- "single_word": false,
235
- "lstrip": false,
236
- "rstrip": true,
237
- "normalized": false,
238
- "special": true
239
- },
240
- {
241
- "id": 32023,
242
- "content": "<|meta_start|>",
243
- "single_word": false,
244
- "lstrip": false,
245
- "rstrip": true,
246
- "normalized": false,
247
- "special": true
248
- },
249
- {
250
- "id": 32024,
251
- "content": "<|ipynb_marker|>",
252
- "single_word": false,
253
- "lstrip": false,
254
- "rstrip": true,
255
- "normalized": false,
256
- "special": true
257
- },
258
- {
259
- "id": 32025,
260
- "content": "<|diff_marker|>",
261
- "single_word": false,
262
- "lstrip": false,
263
- "rstrip": true,
264
- "normalized": false,
265
- "special": true
266
- },
267
- {
268
- "id": 32026,
269
- "content": "<|ghissue|>",
270
- "single_word": false,
271
- "lstrip": false,
272
- "rstrip": true,
273
- "normalized": false,
274
- "special": true
275
- },
276
- {
277
- "id": 32027,
278
- "content": "<|ghreview|>",
279
- "single_word": false,
280
- "lstrip": false,
281
- "rstrip": true,
282
- "normalized": false,
283
- "special": true
284
- },
285
- {
286
- "id": 32028,
287
- "content": "<|disc_start|>",
288
- "single_word": false,
289
- "lstrip": false,
290
- "rstrip": true,
291
- "normalized": false,
292
- "special": true
293
- },
294
- {
295
- "id": 32029,
296
- "content": "<|disc_sep|>",
297
- "single_word": false,
298
- "lstrip": false,
299
- "rstrip": true,
300
- "normalized": false,
301
- "special": true
302
- },
303
- {
304
- "id": 32030,
305
- "content": "<|disc_thread|><|query|>",
306
- "single_word": false,
307
- "lstrip": false,
308
- "rstrip": true,
309
- "normalized": false,
310
- "special": true
311
- },
312
- {
313
- "id": 32031,
314
- "content": "<|/query|>",
315
- "single_word": false,
316
- "lstrip": false,
317
- "rstrip": true,
318
- "normalized": false,
319
- "special": true
320
- },
321
- {
322
- "id": 32032,
323
- "content": "<|data|>",
324
- "single_word": false,
325
- "lstrip": false,
326
- "rstrip": true,
327
- "normalized": false,
328
- "special": true
329
- },
330
- {
331
- "id": 32033,
332
- "content": "<|/data|>",
333
- "single_word": false,
334
- "lstrip": false,
335
- "rstrip": true,
336
- "normalized": false,
337
- "special": true
338
- },
339
- {
340
- "id": 32034,
341
- "content": "<|sys|>",
342
- "single_word": false,
343
- "lstrip": false,
344
- "rstrip": true,
345
- "normalized": false,
346
- "special": true
347
- },
348
- {
349
- "id": 32035,
350
- "content": "<|/sys|>",
351
- "single_word": false,
352
- "lstrip": false,
353
- "rstrip": true,
354
- "normalized": false,
355
- "special": true
356
- },
357
- {
358
- "id": 32036,
359
- "content": "<|inst|>",
360
- "single_word": false,
361
- "lstrip": false,
362
- "rstrip": true,
363
- "normalized": false,
364
- "special": true
365
- },
366
- {
367
- "id": 32037,
368
- "content": "<|/inst|>",
369
- "single_word": false,
370
- "lstrip": false,
371
- "rstrip": true,
372
- "normalized": false,
373
- "special": true
374
- },
375
- {
376
- "id": 32038,
377
- "content": "<|im_start|>",
378
- "single_word": false,
379
- "lstrip": false,
380
- "rstrip": false,
381
- "normalized": true,
382
- "special": false
383
- },
384
- {
385
- "id": 32039,
386
- "content": "<|im_end|>",
387
- "single_word": false,
388
- "lstrip": false,
389
  "rstrip": false,
390
  "normalized": false,
391
  "special": true
 
26
  "content": "</s>",
27
  "single_word": false,
28
  "lstrip": false,
29
+ "rstrip": false,
30
  "normalized": false,
31
  "special": true
32
  },
 
44
  "content": "<|assistant|>",
45
  "single_word": false,
46
  "lstrip": false,
47
+ "rstrip": false,
48
  "normalized": false,
49
  "special": true
50
  },
51
  {
52
  "id": 32002,
53
+ "content": "<|placeholder1|>",
54
  "single_word": false,
55
  "lstrip": false,
56
+ "rstrip": false,
57
  "normalized": false,
58
  "special": true
59
  },
60
  {
61
  "id": 32003,
62
+ "content": "<|placeholder2|>",
63
  "single_word": false,
64
  "lstrip": false,
65
+ "rstrip": false,
66
  "normalized": false,
67
  "special": true
68
  },
69
  {
70
  "id": 32004,
71
+ "content": "<|placeholder3|>",
72
  "single_word": false,
73
  "lstrip": false,
74
+ "rstrip": false,
75
  "normalized": false,
76
  "special": true
77
  },
78
  {
79
  "id": 32005,
80
+ "content": "<|placeholder4|>",
81
  "single_word": false,
82
  "lstrip": false,
83
+ "rstrip": false,
84
  "normalized": false,
85
  "special": true
86
  },
 
89
  "content": "<|system|>",
90
  "single_word": false,
91
  "lstrip": false,
92
+ "rstrip": false,
93
  "normalized": false,
94
  "special": true
95
  },
 
98
  "content": "<|end|>",
99
  "single_word": false,
100
  "lstrip": false,
101
+ "rstrip": false,
102
  "normalized": false,
103
  "special": true
104
  },
105
  {
106
  "id": 32008,
107
+ "content": "<|placeholder5|>",
108
  "single_word": false,
109
  "lstrip": false,
110
+ "rstrip": false,
111
  "normalized": false,
112
  "special": true
113
  },
114
  {
115
  "id": 32009,
116
+ "content": "<|placeholder6|>",
117
  "single_word": false,
118
  "lstrip": false,
119
+ "rstrip": false,
120
  "normalized": false,
121
  "special": true
122
  },
 
125
  "content": "<|user|>",
126
  "single_word": false,
127
  "lstrip": false,
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
128
  "rstrip": false,
129
  "normalized": false,
130
  "special": true
tokenizer_config.json CHANGED
@@ -22,7 +22,7 @@
22
  "content": "</s>",
23
  "lstrip": false,
24
  "normalized": false,
25
- "rstrip": true,
26
  "single_word": false,
27
  "special": true
28
  },
@@ -38,39 +38,39 @@
38
  "content": "<|assistant|>",
39
  "lstrip": false,
40
  "normalized": false,
41
- "rstrip": true,
42
  "single_word": false,
43
  "special": true
44
  },
45
  "32002": {
46
- "content": "<|step|>",
47
  "lstrip": false,
48
  "normalized": false,
49
- "rstrip": true,
50
  "single_word": false,
51
  "special": true
52
  },
53
  "32003": {
54
- "content": "<|function_output|>",
55
  "lstrip": false,
56
  "normalized": false,
57
- "rstrip": true,
58
  "single_word": false,
59
  "special": true
60
  },
61
  "32004": {
62
- "content": "<|tag|>",
63
  "lstrip": false,
64
  "normalized": false,
65
- "rstrip": true,
66
  "single_word": false,
67
  "special": true
68
  },
69
  "32005": {
70
- "content": "<|function_call|>",
71
  "lstrip": false,
72
  "normalized": false,
73
- "rstrip": true,
74
  "single_word": false,
75
  "special": true
76
  },
@@ -78,7 +78,7 @@
78
  "content": "<|system|>",
79
  "lstrip": false,
80
  "normalized": false,
81
- "rstrip": true,
82
  "single_word": false,
83
  "special": true
84
  },
@@ -86,23 +86,23 @@
86
  "content": "<|end|>",
87
  "lstrip": false,
88
  "normalized": false,
89
- "rstrip": true,
90
  "single_word": false,
91
  "special": true
92
  },
93
  "32008": {
94
- "content": "<|raw|>",
95
  "lstrip": false,
96
  "normalized": false,
97
- "rstrip": true,
98
  "single_word": false,
99
  "special": true
100
  },
101
  "32009": {
102
- "content": "<|continue|>",
103
  "lstrip": false,
104
  "normalized": false,
105
- "rstrip": true,
106
  "single_word": false,
107
  "special": true
108
  },
@@ -110,252 +110,17 @@
110
  "content": "<|user|>",
111
  "lstrip": false,
112
  "normalized": false,
113
- "rstrip": true,
114
- "single_word": false,
115
- "special": true
116
- },
117
- "32011": {
118
- "content": "<|function_list|>",
119
- "lstrip": false,
120
- "normalized": false,
121
- "rstrip": true,
122
- "single_word": false,
123
- "special": true
124
- },
125
- "32012": {
126
- "content": "<|calc|>",
127
- "lstrip": false,
128
- "normalized": false,
129
- "rstrip": true,
130
- "single_word": false,
131
- "special": true
132
- },
133
- "32013": {
134
- "content": "<|code|>",
135
- "lstrip": false,
136
- "normalized": false,
137
- "rstrip": true,
138
- "single_word": false,
139
- "special": true
140
- },
141
- "32014": {
142
- "content": "<|/code|>",
143
- "lstrip": false,
144
- "normalized": false,
145
- "rstrip": true,
146
- "single_word": false,
147
- "special": true
148
- },
149
- "32015": {
150
- "content": "<|summary|>",
151
- "lstrip": false,
152
- "normalized": false,
153
- "rstrip": true,
154
- "single_word": false,
155
- "special": true
156
- },
157
- "32016": {
158
- "content": "<|resource|>",
159
- "lstrip": false,
160
- "normalized": false,
161
- "rstrip": true,
162
- "single_word": false,
163
- "special": true
164
- },
165
- "32017": {
166
- "content": "<|assistant_mask|>",
167
- "lstrip": false,
168
- "normalized": false,
169
- "rstrip": true,
170
- "single_word": false,
171
- "special": true
172
- },
173
- "32018": {
174
- "content": "<|start|>",
175
- "lstrip": false,
176
- "normalized": false,
177
- "rstrip": true,
178
- "single_word": false,
179
- "special": true
180
- },
181
- "32019": {
182
- "content": "<|message|>",
183
- "lstrip": false,
184
- "normalized": false,
185
- "rstrip": true,
186
- "single_word": false,
187
- "special": true
188
- },
189
- "32020": {
190
- "content": "<|fim_prefix|>",
191
- "lstrip": false,
192
- "normalized": false,
193
- "rstrip": true,
194
- "single_word": false,
195
- "special": true
196
- },
197
- "32021": {
198
- "content": "<|fim_middle|>",
199
- "lstrip": false,
200
- "normalized": false,
201
- "rstrip": true,
202
- "single_word": false,
203
- "special": true
204
- },
205
- "32022": {
206
- "content": "<|fim_suffix|>",
207
- "lstrip": false,
208
- "normalized": false,
209
- "rstrip": true,
210
- "single_word": false,
211
- "special": true
212
- },
213
- "32023": {
214
- "content": "<|meta_start|>",
215
- "lstrip": false,
216
- "normalized": false,
217
- "rstrip": true,
218
- "single_word": false,
219
- "special": true
220
- },
221
- "32024": {
222
- "content": "<|ipynb_marker|>",
223
- "lstrip": false,
224
- "normalized": false,
225
- "rstrip": true,
226
- "single_word": false,
227
- "special": true
228
- },
229
- "32025": {
230
- "content": "<|diff_marker|>",
231
- "lstrip": false,
232
- "normalized": false,
233
- "rstrip": true,
234
- "single_word": false,
235
- "special": true
236
- },
237
- "32026": {
238
- "content": "<|ghissue|>",
239
- "lstrip": false,
240
- "normalized": false,
241
- "rstrip": true,
242
- "single_word": false,
243
- "special": true
244
- },
245
- "32027": {
246
- "content": "<|ghreview|>",
247
- "lstrip": false,
248
- "normalized": false,
249
- "rstrip": true,
250
- "single_word": false,
251
- "special": true
252
- },
253
- "32028": {
254
- "content": "<|disc_start|>",
255
- "lstrip": false,
256
- "normalized": false,
257
- "rstrip": true,
258
- "single_word": false,
259
- "special": true
260
- },
261
- "32029": {
262
- "content": "<|disc_sep|>",
263
- "lstrip": false,
264
- "normalized": false,
265
- "rstrip": true,
266
- "single_word": false,
267
- "special": true
268
- },
269
- "32030": {
270
- "content": "<|disc_thread|><|query|>",
271
- "lstrip": false,
272
- "normalized": false,
273
- "rstrip": true,
274
- "single_word": false,
275
- "special": true
276
- },
277
- "32031": {
278
- "content": "<|/query|>",
279
- "lstrip": false,
280
- "normalized": false,
281
- "rstrip": true,
282
- "single_word": false,
283
- "special": true
284
- },
285
- "32032": {
286
- "content": "<|data|>",
287
- "lstrip": false,
288
- "normalized": false,
289
- "rstrip": true,
290
- "single_word": false,
291
- "special": true
292
- },
293
- "32033": {
294
- "content": "<|/data|>",
295
- "lstrip": false,
296
- "normalized": false,
297
- "rstrip": true,
298
- "single_word": false,
299
- "special": true
300
- },
301
- "32034": {
302
- "content": "<|sys|>",
303
- "lstrip": false,
304
- "normalized": false,
305
- "rstrip": true,
306
- "single_word": false,
307
- "special": true
308
- },
309
- "32035": {
310
- "content": "<|/sys|>",
311
- "lstrip": false,
312
- "normalized": false,
313
- "rstrip": true,
314
- "single_word": false,
315
- "special": true
316
- },
317
- "32036": {
318
- "content": "<|inst|>",
319
- "lstrip": false,
320
- "normalized": false,
321
- "rstrip": true,
322
- "single_word": false,
323
- "special": true
324
- },
325
- "32037": {
326
- "content": "<|/inst|>",
327
- "lstrip": false,
328
- "normalized": false,
329
- "rstrip": true,
330
- "single_word": false,
331
- "special": true
332
- },
333
- "32038": {
334
- "content": "<|im_start|>",
335
- "lstrip": false,
336
- "normalized": true,
337
- "rstrip": false,
338
- "single_word": false,
339
- "special": false
340
- },
341
- "32039": {
342
- "content": "<|im_end|>",
343
- "lstrip": false,
344
- "normalized": false,
345
  "rstrip": false,
346
  "single_word": false,
347
  "special": true
348
  }
349
  },
350
- "additional_special_tokens": [
351
- "<|/inst|>"
352
- ],
353
  "bos_token": "<s>",
354
  "chat_template": "{{ bos_token }}{% for message in messages %}{% if (message['role'] == 'system') %}{{'<|system|>' + '\n' + message['content'] + '<|end|>' + '\n'}}{% elif (message['role'] == 'user') %}{{'<|user|>' + '\n' + message['content'] + '<|end|>' + '\n' + '<|assistant|>' + '\n'}}{% elif message['role'] == 'assistant' %}{{message['content'] + '<|end|>' + '\n'}}{% endif %}{% endfor %}",
355
  "clean_up_tokenization_spaces": false,
356
- "eos_token": "<|im_end|>",
357
  "model_max_length": 4096,
358
- "pad_token": "<|im_end|>",
359
  "padding_side": "right",
360
  "sp_model_kwargs": {},
361
  "tokenizer_class": "LlamaTokenizer",
 
22
  "content": "</s>",
23
  "lstrip": false,
24
  "normalized": false,
25
+ "rstrip": false,
26
  "single_word": false,
27
  "special": true
28
  },
 
38
  "content": "<|assistant|>",
39
  "lstrip": false,
40
  "normalized": false,
41
+ "rstrip": false,
42
  "single_word": false,
43
  "special": true
44
  },
45
  "32002": {
46
+ "content": "<|placeholder1|>",
47
  "lstrip": false,
48
  "normalized": false,
49
+ "rstrip": false,
50
  "single_word": false,
51
  "special": true
52
  },
53
  "32003": {
54
+ "content": "<|placeholder2|>",
55
  "lstrip": false,
56
  "normalized": false,
57
+ "rstrip": false,
58
  "single_word": false,
59
  "special": true
60
  },
61
  "32004": {
62
+ "content": "<|placeholder3|>",
63
  "lstrip": false,
64
  "normalized": false,
65
+ "rstrip": false,
66
  "single_word": false,
67
  "special": true
68
  },
69
  "32005": {
70
+ "content": "<|placeholder4|>",
71
  "lstrip": false,
72
  "normalized": false,
73
+ "rstrip": false,
74
  "single_word": false,
75
  "special": true
76
  },
 
78
  "content": "<|system|>",
79
  "lstrip": false,
80
  "normalized": false,
81
+ "rstrip": false,
82
  "single_word": false,
83
  "special": true
84
  },
 
86
  "content": "<|end|>",
87
  "lstrip": false,
88
  "normalized": false,
89
+ "rstrip": false,
90
  "single_word": false,
91
  "special": true
92
  },
93
  "32008": {
94
+ "content": "<|placeholder5|>",
95
  "lstrip": false,
96
  "normalized": false,
97
+ "rstrip": false,
98
  "single_word": false,
99
  "special": true
100
  },
101
  "32009": {
102
+ "content": "<|placeholder6|>",
103
  "lstrip": false,
104
  "normalized": false,
105
+ "rstrip": false,
106
  "single_word": false,
107
  "special": true
108
  },
 
110
  "content": "<|user|>",
111
  "lstrip": false,
112
  "normalized": false,
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
113
  "rstrip": false,
114
  "single_word": false,
115
  "special": true
116
  }
117
  },
 
 
 
118
  "bos_token": "<s>",
119
  "chat_template": "{{ bos_token }}{% for message in messages %}{% if (message['role'] == 'system') %}{{'<|system|>' + '\n' + message['content'] + '<|end|>' + '\n'}}{% elif (message['role'] == 'user') %}{{'<|user|>' + '\n' + message['content'] + '<|end|>' + '\n' + '<|assistant|>' + '\n'}}{% elif message['role'] == 'assistant' %}{{message['content'] + '<|end|>' + '\n'}}{% endif %}{% endfor %}",
120
  "clean_up_tokenization_spaces": false,
121
+ "eos_token": "<|endoftext|>",
122
  "model_max_length": 4096,
123
+ "pad_token": "<|endoftext|>",
124
  "padding_side": "right",
125
  "sp_model_kwargs": {},
126
  "tokenizer_class": "LlamaTokenizer",