Hugging Face's logo

LeoLM
/
leo-hessianai-70b-chat

Text Generation
Transformers
PyTorch
English
German
llama
conversational
text-generation-inference
Inference Endpoints
Model card Files Community
6
bjoernp commited on
Commit
50ec22b
•
1 Parent(s): 1ef6e41

Upload tokenizer

Browse files
Files changed (2) hide show
  1. special_tokens_map.json +882 -0
  2. tokenizer_config.json +379 -253
special_tokens_map.json CHANGED
@@ -13,6 +13,888 @@
13
  "normalized": false,
14
  "rstrip": false,
15
  "single_word": false
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
16
  }
17
  ],
18
  "bos_token": {
 
13
  "normalized": false,
14
  "rstrip": false,
15
  "single_word": false
16
+ },
17
+ {
18
+ "content": "<|padding_token_0|>",
19
+ "lstrip": false,
20
+ "normalized": false,
21
+ "rstrip": false,
22
+ "single_word": false
23
+ },
24
+ {
25
+ "content": "<|padding_token_1|>",
26
+ "lstrip": false,
27
+ "normalized": false,
28
+ "rstrip": false,
29
+ "single_word": false
30
+ },
31
+ {
32
+ "content": "<|padding_token_2|>",
33
+ "lstrip": false,
34
+ "normalized": false,
35
+ "rstrip": false,
36
+ "single_word": false
37
+ },
38
+ {
39
+ "content": "<|padding_token_3|>",
40
+ "lstrip": false,
41
+ "normalized": false,
42
+ "rstrip": false,
43
+ "single_word": false
44
+ },
45
+ {
46
+ "content": "<|padding_token_4|>",
47
+ "lstrip": false,
48
+ "normalized": false,
49
+ "rstrip": false,
50
+ "single_word": false
51
+ },
52
+ {
53
+ "content": "<|padding_token_5|>",
54
+ "lstrip": false,
55
+ "normalized": false,
56
+ "rstrip": false,
57
+ "single_word": false
58
+ },
59
+ {
60
+ "content": "<|padding_token_6|>",
61
+ "lstrip": false,
62
+ "normalized": false,
63
+ "rstrip": false,
64
+ "single_word": false
65
+ },
66
+ {
67
+ "content": "<|padding_token_7|>",
68
+ "lstrip": false,
69
+ "normalized": false,
70
+ "rstrip": false,
71
+ "single_word": false
72
+ },
73
+ {
74
+ "content": "<|padding_token_8|>",
75
+ "lstrip": false,
76
+ "normalized": false,
77
+ "rstrip": false,
78
+ "single_word": false
79
+ },
80
+ {
81
+ "content": "<|padding_token_9|>",
82
+ "lstrip": false,
83
+ "normalized": false,
84
+ "rstrip": false,
85
+ "single_word": false
86
+ },
87
+ {
88
+ "content": "<|padding_token_10|>",
89
+ "lstrip": false,
90
+ "normalized": false,
91
+ "rstrip": false,
92
+ "single_word": false
93
+ },
94
+ {
95
+ "content": "<|padding_token_11|>",
96
+ "lstrip": false,
97
+ "normalized": false,
98
+ "rstrip": false,
99
+ "single_word": false
100
+ },
101
+ {
102
+ "content": "<|padding_token_12|>",
103
+ "lstrip": false,
104
+ "normalized": false,
105
+ "rstrip": false,
106
+ "single_word": false
107
+ },
108
+ {
109
+ "content": "<|padding_token_13|>",
110
+ "lstrip": false,
111
+ "normalized": false,
112
+ "rstrip": false,
113
+ "single_word": false
114
+ },
115
+ {
116
+ "content": "<|padding_token_14|>",
117
+ "lstrip": false,
118
+ "normalized": false,
119
+ "rstrip": false,
120
+ "single_word": false
121
+ },
122
+ {
123
+ "content": "<|padding_token_15|>",
124
+ "lstrip": false,
125
+ "normalized": false,
126
+ "rstrip": false,
127
+ "single_word": false
128
+ },
129
+ {
130
+ "content": "<|padding_token_16|>",
131
+ "lstrip": false,
132
+ "normalized": false,
133
+ "rstrip": false,
134
+ "single_word": false
135
+ },
136
+ {
137
+ "content": "<|padding_token_17|>",
138
+ "lstrip": false,
139
+ "normalized": false,
140
+ "rstrip": false,
141
+ "single_word": false
142
+ },
143
+ {
144
+ "content": "<|padding_token_18|>",
145
+ "lstrip": false,
146
+ "normalized": false,
147
+ "rstrip": false,
148
+ "single_word": false
149
+ },
150
+ {
151
+ "content": "<|padding_token_19|>",
152
+ "lstrip": false,
153
+ "normalized": false,
154
+ "rstrip": false,
155
+ "single_word": false
156
+ },
157
+ {
158
+ "content": "<|padding_token_20|>",
159
+ "lstrip": false,
160
+ "normalized": false,
161
+ "rstrip": false,
162
+ "single_word": false
163
+ },
164
+ {
165
+ "content": "<|padding_token_21|>",
166
+ "lstrip": false,
167
+ "normalized": false,
168
+ "rstrip": false,
169
+ "single_word": false
170
+ },
171
+ {
172
+ "content": "<|padding_token_22|>",
173
+ "lstrip": false,
174
+ "normalized": false,
175
+ "rstrip": false,
176
+ "single_word": false
177
+ },
178
+ {
179
+ "content": "<|padding_token_23|>",
180
+ "lstrip": false,
181
+ "normalized": false,
182
+ "rstrip": false,
183
+ "single_word": false
184
+ },
185
+ {
186
+ "content": "<|padding_token_24|>",
187
+ "lstrip": false,
188
+ "normalized": false,
189
+ "rstrip": false,
190
+ "single_word": false
191
+ },
192
+ {
193
+ "content": "<|padding_token_25|>",
194
+ "lstrip": false,
195
+ "normalized": false,
196
+ "rstrip": false,
197
+ "single_word": false
198
+ },
199
+ {
200
+ "content": "<|padding_token_26|>",
201
+ "lstrip": false,
202
+ "normalized": false,
203
+ "rstrip": false,
204
+ "single_word": false
205
+ },
206
+ {
207
+ "content": "<|padding_token_27|>",
208
+ "lstrip": false,
209
+ "normalized": false,
210
+ "rstrip": false,
211
+ "single_word": false
212
+ },
213
+ {
214
+ "content": "<|padding_token_28|>",
215
+ "lstrip": false,
216
+ "normalized": false,
217
+ "rstrip": false,
218
+ "single_word": false
219
+ },
220
+ {
221
+ "content": "<|padding_token_29|>",
222
+ "lstrip": false,
223
+ "normalized": false,
224
+ "rstrip": false,
225
+ "single_word": false
226
+ },
227
+ {
228
+ "content": "<|padding_token_30|>",
229
+ "lstrip": false,
230
+ "normalized": false,
231
+ "rstrip": false,
232
+ "single_word": false
233
+ },
234
+ {
235
+ "content": "<|padding_token_31|>",
236
+ "lstrip": false,
237
+ "normalized": false,
238
+ "rstrip": false,
239
+ "single_word": false
240
+ },
241
+ {
242
+ "content": "<|padding_token_32|>",
243
+ "lstrip": false,
244
+ "normalized": false,
245
+ "rstrip": false,
246
+ "single_word": false
247
+ },
248
+ {
249
+ "content": "<|padding_token_33|>",
250
+ "lstrip": false,
251
+ "normalized": false,
252
+ "rstrip": false,
253
+ "single_word": false
254
+ },
255
+ {
256
+ "content": "<|padding_token_34|>",
257
+ "lstrip": false,
258
+ "normalized": false,
259
+ "rstrip": false,
260
+ "single_word": false
261
+ },
262
+ {
263
+ "content": "<|padding_token_35|>",
264
+ "lstrip": false,
265
+ "normalized": false,
266
+ "rstrip": false,
267
+ "single_word": false
268
+ },
269
+ {
270
+ "content": "<|padding_token_36|>",
271
+ "lstrip": false,
272
+ "normalized": false,
273
+ "rstrip": false,
274
+ "single_word": false
275
+ },
276
+ {
277
+ "content": "<|padding_token_37|>",
278
+ "lstrip": false,
279
+ "normalized": false,
280
+ "rstrip": false,
281
+ "single_word": false
282
+ },
283
+ {
284
+ "content": "<|padding_token_38|>",
285
+ "lstrip": false,
286
+ "normalized": false,
287
+ "rstrip": false,
288
+ "single_word": false
289
+ },
290
+ {
291
+ "content": "<|padding_token_39|>",
292
+ "lstrip": false,
293
+ "normalized": false,
294
+ "rstrip": false,
295
+ "single_word": false
296
+ },
297
+ {
298
+ "content": "<|padding_token_40|>",
299
+ "lstrip": false,
300
+ "normalized": false,
301
+ "rstrip": false,
302
+ "single_word": false
303
+ },
304
+ {
305
+ "content": "<|padding_token_41|>",
306
+ "lstrip": false,
307
+ "normalized": false,
308
+ "rstrip": false,
309
+ "single_word": false
310
+ },
311
+ {
312
+ "content": "<|padding_token_42|>",
313
+ "lstrip": false,
314
+ "normalized": false,
315
+ "rstrip": false,
316
+ "single_word": false
317
+ },
318
+ {
319
+ "content": "<|padding_token_43|>",
320
+ "lstrip": false,
321
+ "normalized": false,
322
+ "rstrip": false,
323
+ "single_word": false
324
+ },
325
+ {
326
+ "content": "<|padding_token_44|>",
327
+ "lstrip": false,
328
+ "normalized": false,
329
+ "rstrip": false,
330
+ "single_word": false
331
+ },
332
+ {
333
+ "content": "<|padding_token_45|>",
334
+ "lstrip": false,
335
+ "normalized": false,
336
+ "rstrip": false,
337
+ "single_word": false
338
+ },
339
+ {
340
+ "content": "<|padding_token_46|>",
341
+ "lstrip": false,
342
+ "normalized": false,
343
+ "rstrip": false,
344
+ "single_word": false
345
+ },
346
+ {
347
+ "content": "<|padding_token_47|>",
348
+ "lstrip": false,
349
+ "normalized": false,
350
+ "rstrip": false,
351
+ "single_word": false
352
+ },
353
+ {
354
+ "content": "<|padding_token_48|>",
355
+ "lstrip": false,
356
+ "normalized": false,
357
+ "rstrip": false,
358
+ "single_word": false
359
+ },
360
+ {
361
+ "content": "<|padding_token_49|>",
362
+ "lstrip": false,
363
+ "normalized": false,
364
+ "rstrip": false,
365
+ "single_word": false
366
+ },
367
+ {
368
+ "content": "<|padding_token_50|>",
369
+ "lstrip": false,
370
+ "normalized": false,
371
+ "rstrip": false,
372
+ "single_word": false
373
+ },
374
+ {
375
+ "content": "<|padding_token_51|>",
376
+ "lstrip": false,
377
+ "normalized": false,
378
+ "rstrip": false,
379
+ "single_word": false
380
+ },
381
+ {
382
+ "content": "<|padding_token_52|>",
383
+ "lstrip": false,
384
+ "normalized": false,
385
+ "rstrip": false,
386
+ "single_word": false
387
+ },
388
+ {
389
+ "content": "<|padding_token_53|>",
390
+ "lstrip": false,
391
+ "normalized": false,
392
+ "rstrip": false,
393
+ "single_word": false
394
+ },
395
+ {
396
+ "content": "<|padding_token_54|>",
397
+ "lstrip": false,
398
+ "normalized": false,
399
+ "rstrip": false,
400
+ "single_word": false
401
+ },
402
+ {
403
+ "content": "<|padding_token_55|>",
404
+ "lstrip": false,
405
+ "normalized": false,
406
+ "rstrip": false,
407
+ "single_word": false
408
+ },
409
+ {
410
+ "content": "<|padding_token_56|>",
411
+ "lstrip": false,
412
+ "normalized": false,
413
+ "rstrip": false,
414
+ "single_word": false
415
+ },
416
+ {
417
+ "content": "<|padding_token_57|>",
418
+ "lstrip": false,
419
+ "normalized": false,
420
+ "rstrip": false,
421
+ "single_word": false
422
+ },
423
+ {
424
+ "content": "<|padding_token_58|>",
425
+ "lstrip": false,
426
+ "normalized": false,
427
+ "rstrip": false,
428
+ "single_word": false
429
+ },
430
+ {
431
+ "content": "<|padding_token_59|>",
432
+ "lstrip": false,
433
+ "normalized": false,
434
+ "rstrip": false,
435
+ "single_word": false
436
+ },
437
+ {
438
+ "content": "<|padding_token_60|>",
439
+ "lstrip": false,
440
+ "normalized": false,
441
+ "rstrip": false,
442
+ "single_word": false
443
+ },
444
+ {
445
+ "content": "<|padding_token_61|>",
446
+ "lstrip": false,
447
+ "normalized": false,
448
+ "rstrip": false,
449
+ "single_word": false
450
+ },
451
+ {
452
+ "content": "<|padding_token_62|>",
453
+ "lstrip": false,
454
+ "normalized": false,
455
+ "rstrip": false,
456
+ "single_word": false
457
+ },
458
+ {
459
+ "content": "<|padding_token_63|>",
460
+ "lstrip": false,
461
+ "normalized": false,
462
+ "rstrip": false,
463
+ "single_word": false
464
+ },
465
+ {
466
+ "content": "<|padding_token_64|>",
467
+ "lstrip": false,
468
+ "normalized": false,
469
+ "rstrip": false,
470
+ "single_word": false
471
+ },
472
+ {
473
+ "content": "<|padding_token_65|>",
474
+ "lstrip": false,
475
+ "normalized": false,
476
+ "rstrip": false,
477
+ "single_word": false
478
+ },
479
+ {
480
+ "content": "<|padding_token_66|>",
481
+ "lstrip": false,
482
+ "normalized": false,
483
+ "rstrip": false,
484
+ "single_word": false
485
+ },
486
+ {
487
+ "content": "<|padding_token_67|>",
488
+ "lstrip": false,
489
+ "normalized": false,
490
+ "rstrip": false,
491
+ "single_word": false
492
+ },
493
+ {
494
+ "content": "<|padding_token_68|>",
495
+ "lstrip": false,
496
+ "normalized": false,
497
+ "rstrip": false,
498
+ "single_word": false
499
+ },
500
+ {
501
+ "content": "<|padding_token_69|>",
502
+ "lstrip": false,
503
+ "normalized": false,
504
+ "rstrip": false,
505
+ "single_word": false
506
+ },
507
+ {
508
+ "content": "<|padding_token_70|>",
509
+ "lstrip": false,
510
+ "normalized": false,
511
+ "rstrip": false,
512
+ "single_word": false
513
+ },
514
+ {
515
+ "content": "<|padding_token_71|>",
516
+ "lstrip": false,
517
+ "normalized": false,
518
+ "rstrip": false,
519
+ "single_word": false
520
+ },
521
+ {
522
+ "content": "<|padding_token_72|>",
523
+ "lstrip": false,
524
+ "normalized": false,
525
+ "rstrip": false,
526
+ "single_word": false
527
+ },
528
+ {
529
+ "content": "<|padding_token_73|>",
530
+ "lstrip": false,
531
+ "normalized": false,
532
+ "rstrip": false,
533
+ "single_word": false
534
+ },
535
+ {
536
+ "content": "<|padding_token_74|>",
537
+ "lstrip": false,
538
+ "normalized": false,
539
+ "rstrip": false,
540
+ "single_word": false
541
+ },
542
+ {
543
+ "content": "<|padding_token_75|>",
544
+ "lstrip": false,
545
+ "normalized": false,
546
+ "rstrip": false,
547
+ "single_word": false
548
+ },
549
+ {
550
+ "content": "<|padding_token_76|>",
551
+ "lstrip": false,
552
+ "normalized": false,
553
+ "rstrip": false,
554
+ "single_word": false
555
+ },
556
+ {
557
+ "content": "<|padding_token_77|>",
558
+ "lstrip": false,
559
+ "normalized": false,
560
+ "rstrip": false,
561
+ "single_word": false
562
+ },
563
+ {
564
+ "content": "<|padding_token_78|>",
565
+ "lstrip": false,
566
+ "normalized": false,
567
+ "rstrip": false,
568
+ "single_word": false
569
+ },
570
+ {
571
+ "content": "<|padding_token_79|>",
572
+ "lstrip": false,
573
+ "normalized": false,
574
+ "rstrip": false,
575
+ "single_word": false
576
+ },
577
+ {
578
+ "content": "<|padding_token_80|>",
579
+ "lstrip": false,
580
+ "normalized": false,
581
+ "rstrip": false,
582
+ "single_word": false
583
+ },
584
+ {
585
+ "content": "<|padding_token_81|>",
586
+ "lstrip": false,
587
+ "normalized": false,
588
+ "rstrip": false,
589
+ "single_word": false
590
+ },
591
+ {
592
+ "content": "<|padding_token_82|>",
593
+ "lstrip": false,
594
+ "normalized": false,
595
+ "rstrip": false,
596
+ "single_word": false
597
+ },
598
+ {
599
+ "content": "<|padding_token_83|>",
600
+ "lstrip": false,
601
+ "normalized": false,
602
+ "rstrip": false,
603
+ "single_word": false
604
+ },
605
+ {
606
+ "content": "<|padding_token_84|>",
607
+ "lstrip": false,
608
+ "normalized": false,
609
+ "rstrip": false,
610
+ "single_word": false
611
+ },
612
+ {
613
+ "content": "<|padding_token_85|>",
614
+ "lstrip": false,
615
+ "normalized": false,
616
+ "rstrip": false,
617
+ "single_word": false
618
+ },
619
+ {
620
+ "content": "<|padding_token_86|>",
621
+ "lstrip": false,
622
+ "normalized": false,
623
+ "rstrip": false,
624
+ "single_word": false
625
+ },
626
+ {
627
+ "content": "<|padding_token_87|>",
628
+ "lstrip": false,
629
+ "normalized": false,
630
+ "rstrip": false,
631
+ "single_word": false
632
+ },
633
+ {
634
+ "content": "<|padding_token_88|>",
635
+ "lstrip": false,
636
+ "normalized": false,
637
+ "rstrip": false,
638
+ "single_word": false
639
+ },
640
+ {
641
+ "content": "<|padding_token_89|>",
642
+ "lstrip": false,
643
+ "normalized": false,
644
+ "rstrip": false,
645
+ "single_word": false
646
+ },
647
+ {
648
+ "content": "<|padding_token_90|>",
649
+ "lstrip": false,
650
+ "normalized": false,
651
+ "rstrip": false,
652
+ "single_word": false
653
+ },
654
+ {
655
+ "content": "<|padding_token_91|>",
656
+ "lstrip": false,
657
+ "normalized": false,
658
+ "rstrip": false,
659
+ "single_word": false
660
+ },
661
+ {
662
+ "content": "<|padding_token_92|>",
663
+ "lstrip": false,
664
+ "normalized": false,
665
+ "rstrip": false,
666
+ "single_word": false
667
+ },
668
+ {
669
+ "content": "<|padding_token_93|>",
670
+ "lstrip": false,
671
+ "normalized": false,
672
+ "rstrip": false,
673
+ "single_word": false
674
+ },
675
+ {
676
+ "content": "<|padding_token_94|>",
677
+ "lstrip": false,
678
+ "normalized": false,
679
+ "rstrip": false,
680
+ "single_word": false
681
+ },
682
+ {
683
+ "content": "<|padding_token_95|>",
684
+ "lstrip": false,
685
+ "normalized": false,
686
+ "rstrip": false,
687
+ "single_word": false
688
+ },
689
+ {
690
+ "content": "<|padding_token_96|>",
691
+ "lstrip": false,
692
+ "normalized": false,
693
+ "rstrip": false,
694
+ "single_word": false
695
+ },
696
+ {
697
+ "content": "<|padding_token_97|>",
698
+ "lstrip": false,
699
+ "normalized": false,
700
+ "rstrip": false,
701
+ "single_word": false
702
+ },
703
+ {
704
+ "content": "<|padding_token_98|>",
705
+ "lstrip": false,
706
+ "normalized": false,
707
+ "rstrip": false,
708
+ "single_word": false
709
+ },
710
+ {
711
+ "content": "<|padding_token_99|>",
712
+ "lstrip": false,
713
+ "normalized": false,
714
+ "rstrip": false,
715
+ "single_word": false
716
+ },
717
+ {
718
+ "content": "<|padding_token_100|>",
719
+ "lstrip": false,
720
+ "normalized": false,
721
+ "rstrip": false,
722
+ "single_word": false
723
+ },
724
+ {
725
+ "content": "<|padding_token_101|>",
726
+ "lstrip": false,
727
+ "normalized": false,
728
+ "rstrip": false,
729
+ "single_word": false
730
+ },
731
+ {
732
+ "content": "<|padding_token_102|>",
733
+ "lstrip": false,
734
+ "normalized": false,
735
+ "rstrip": false,
736
+ "single_word": false
737
+ },
738
+ {
739
+ "content": "<|padding_token_103|>",
740
+ "lstrip": false,
741
+ "normalized": false,
742
+ "rstrip": false,
743
+ "single_word": false
744
+ },
745
+ {
746
+ "content": "<|padding_token_104|>",
747
+ "lstrip": false,
748
+ "normalized": false,
749
+ "rstrip": false,
750
+ "single_word": false
751
+ },
752
+ {
753
+ "content": "<|padding_token_105|>",
754
+ "lstrip": false,
755
+ "normalized": false,
756
+ "rstrip": false,
757
+ "single_word": false
758
+ },
759
+ {
760
+ "content": "<|padding_token_106|>",
761
+ "lstrip": false,
762
+ "normalized": false,
763
+ "rstrip": false,
764
+ "single_word": false
765
+ },
766
+ {
767
+ "content": "<|padding_token_107|>",
768
+ "lstrip": false,
769
+ "normalized": false,
770
+ "rstrip": false,
771
+ "single_word": false
772
+ },
773
+ {
774
+ "content": "<|padding_token_108|>",
775
+ "lstrip": false,
776
+ "normalized": false,
777
+ "rstrip": false,
778
+ "single_word": false
779
+ },
780
+ {
781
+ "content": "<|padding_token_109|>",
782
+ "lstrip": false,
783
+ "normalized": false,
784
+ "rstrip": false,
785
+ "single_word": false
786
+ },
787
+ {
788
+ "content": "<|padding_token_110|>",
789
+ "lstrip": false,
790
+ "normalized": false,
791
+ "rstrip": false,
792
+ "single_word": false
793
+ },
794
+ {
795
+ "content": "<|padding_token_111|>",
796
+ "lstrip": false,
797
+ "normalized": false,
798
+ "rstrip": false,
799
+ "single_word": false
800
+ },
801
+ {
802
+ "content": "<|padding_token_112|>",
803
+ "lstrip": false,
804
+ "normalized": false,
805
+ "rstrip": false,
806
+ "single_word": false
807
+ },
808
+ {
809
+ "content": "<|padding_token_113|>",
810
+ "lstrip": false,
811
+ "normalized": false,
812
+ "rstrip": false,
813
+ "single_word": false
814
+ },
815
+ {
816
+ "content": "<|padding_token_114|>",
817
+ "lstrip": false,
818
+ "normalized": false,
819
+ "rstrip": false,
820
+ "single_word": false
821
+ },
822
+ {
823
+ "content": "<|padding_token_115|>",
824
+ "lstrip": false,
825
+ "normalized": false,
826
+ "rstrip": false,
827
+ "single_word": false
828
+ },
829
+ {
830
+ "content": "<|padding_token_116|>",
831
+ "lstrip": false,
832
+ "normalized": false,
833
+ "rstrip": false,
834
+ "single_word": false
835
+ },
836
+ {
837
+ "content": "<|padding_token_117|>",
838
+ "lstrip": false,
839
+ "normalized": false,
840
+ "rstrip": false,
841
+ "single_word": false
842
+ },
843
+ {
844
+ "content": "<|padding_token_118|>",
845
+ "lstrip": false,
846
+ "normalized": false,
847
+ "rstrip": false,
848
+ "single_word": false
849
+ },
850
+ {
851
+ "content": "<|padding_token_119|>",
852
+ "lstrip": false,
853
+ "normalized": false,
854
+ "rstrip": false,
855
+ "single_word": false
856
+ },
857
+ {
858
+ "content": "<|padding_token_120|>",
859
+ "lstrip": false,
860
+ "normalized": false,
861
+ "rstrip": false,
862
+ "single_word": false
863
+ },
864
+ {
865
+ "content": "<|padding_token_121|>",
866
+ "lstrip": false,
867
+ "normalized": false,
868
+ "rstrip": false,
869
+ "single_word": false
870
+ },
871
+ {
872
+ "content": "<|padding_token_122|>",
873
+ "lstrip": false,
874
+ "normalized": false,
875
+ "rstrip": false,
876
+ "single_word": false
877
+ },
878
+ {
879
+ "content": "<|padding_token_123|>",
880
+ "lstrip": false,
881
+ "normalized": false,
882
+ "rstrip": false,
883
+ "single_word": false
884
+ },
885
+ {
886
+ "content": "<|padding_token_124|>",
887
+ "lstrip": false,
888
+ "normalized": false,
889
+ "rstrip": false,
890
+ "single_word": false
891
+ },
892
+ {
893
+ "content": "<|padding_token_125|>",
894
+ "lstrip": false,
895
+ "normalized": false,
896
+ "rstrip": false,
897
+ "single_word": false
898
  }
899
  ],
900
  "bos_token": {
tokenizer_config.json CHANGED
@@ -45,1015 +45,1141 @@
45
  "32002": {
46
  "content": "<|padding_token_0|>",
47
  "lstrip": false,
48
- "normalized": true,
49
  "rstrip": false,
50
  "single_word": false,
51
- "special": false
52
  },
53
  "32003": {
54
  "content": "<|padding_token_1|>",
55
  "lstrip": false,
56
- "normalized": true,
57
  "rstrip": false,
58
  "single_word": false,
59
- "special": false
60
  },
61
  "32004": {
62
  "content": "<|padding_token_2|>",
63
  "lstrip": false,
64
- "normalized": true,
65
  "rstrip": false,
66
  "single_word": false,
67
- "special": false
68
  },
69
  "32005": {
70
  "content": "<|padding_token_3|>",
71
  "lstrip": false,
72
- "normalized": true,
73
  "rstrip": false,
74
  "single_word": false,
75
- "special": false
76
  },
77
  "32006": {
78
  "content": "<|padding_token_4|>",
79
  "lstrip": false,
80
- "normalized": true,
81
  "rstrip": false,
82
  "single_word": false,
83
- "special": false
84
  },
85
  "32007": {
86
  "content": "<|padding_token_5|>",
87
  "lstrip": false,
88
- "normalized": true,
89
  "rstrip": false,
90
  "single_word": false,
91
- "special": false
92
  },
93
  "32008": {
94
  "content": "<|padding_token_6|>",
95
  "lstrip": false,
96
- "normalized": true,
97
  "rstrip": false,
98
  "single_word": false,
99
- "special": false
100
  },
101
  "32009": {
102
  "content": "<|padding_token_7|>",
103
  "lstrip": false,
104
- "normalized": true,
105
  "rstrip": false,
106
  "single_word": false,
107
- "special": false
108
  },
109
  "32010": {
110
  "content": "<|padding_token_8|>",
111
  "lstrip": false,
112
- "normalized": true,
113
  "rstrip": false,
114
  "single_word": false,
115
- "special": false
116
  },
117
  "32011": {
118
  "content": "<|padding_token_9|>",
119
  "lstrip": false,
120
- "normalized": true,
121
  "rstrip": false,
122
  "single_word": false,
123
- "special": false
124
  },
125
  "32012": {
126
  "content": "<|padding_token_10|>",
127
  "lstrip": false,
128
- "normalized": true,
129
  "rstrip": false,
130
  "single_word": false,
131
- "special": false
132
  },
133
  "32013": {
134
  "content": "<|padding_token_11|>",
135
  "lstrip": false,
136
- "normalized": true,
137
  "rstrip": false,
138
  "single_word": false,
139
- "special": false
140
  },
141
  "32014": {
142
  "content": "<|padding_token_12|>",
143
  "lstrip": false,
144
- "normalized": true,
145
  "rstrip": false,
146
  "single_word": false,
147
- "special": false
148
  },
149
  "32015": {
150
  "content": "<|padding_token_13|>",
151
  "lstrip": false,
152
- "normalized": true,
153
  "rstrip": false,
154
  "single_word": false,
155
- "special": false
156
  },
157
  "32016": {
158
  "content": "<|padding_token_14|>",
159
  "lstrip": false,
160
- "normalized": true,
161
  "rstrip": false,
162
  "single_word": false,
163
- "special": false
164
  },
165
  "32017": {
166
  "content": "<|padding_token_15|>",
167
  "lstrip": false,
168
- "normalized": true,
169
  "rstrip": false,
170
  "single_word": false,
171
- "special": false
172
  },
173
  "32018": {
174
  "content": "<|padding_token_16|>",
175
  "lstrip": false,
176
- "normalized": true,
177
  "rstrip": false,
178
  "single_word": false,
179
- "special": false
180
  },
181
  "32019": {
182
  "content": "<|padding_token_17|>",
183
  "lstrip": false,
184
- "normalized": true,
185
  "rstrip": false,
186
  "single_word": false,
187
- "special": false
188
  },
189
  "32020": {
190
  "content": "<|padding_token_18|>",
191
  "lstrip": false,
192
- "normalized": true,
193
  "rstrip": false,
194
  "single_word": false,
195
- "special": false
196
  },
197
  "32021": {
198
  "content": "<|padding_token_19|>",
199
  "lstrip": false,
200
- "normalized": true,
201
  "rstrip": false,
202
  "single_word": false,
203
- "special": false
204
  },
205
  "32022": {
206
  "content": "<|padding_token_20|>",
207
  "lstrip": false,
208
- "normalized": true,
209
  "rstrip": false,
210
  "single_word": false,
211
- "special": false
212
  },
213
  "32023": {
214
  "content": "<|padding_token_21|>",
215
  "lstrip": false,
216
- "normalized": true,
217
  "rstrip": false,
218
  "single_word": false,
219
- "special": false
220
  },
221
  "32024": {
222
  "content": "<|padding_token_22|>",
223
  "lstrip": false,
224
- "normalized": true,
225
  "rstrip": false,
226
  "single_word": false,
227
- "special": false
228
  },
229
  "32025": {
230
  "content": "<|padding_token_23|>",
231
  "lstrip": false,
232
- "normalized": true,
233
  "rstrip": false,
234
  "single_word": false,
235
- "special": false
236
  },
237
  "32026": {
238
  "content": "<|padding_token_24|>",
239
  "lstrip": false,
240
- "normalized": true,
241
  "rstrip": false,
242
  "single_word": false,
243
- "special": false
244
  },
245
  "32027": {
246
  "content": "<|padding_token_25|>",
247
  "lstrip": false,
248
- "normalized": true,
249
  "rstrip": false,
250
  "single_word": false,
251
- "special": false
252
  },
253
  "32028": {
254
  "content": "<|padding_token_26|>",
255
  "lstrip": false,
256
- "normalized": true,
257
  "rstrip": false,
258
  "single_word": false,
259
- "special": false
260
  },
261
  "32029": {
262
  "content": "<|padding_token_27|>",
263
  "lstrip": false,
264
- "normalized": true,
265
  "rstrip": false,
266
  "single_word": false,
267
- "special": false
268
  },
269
  "32030": {
270
  "content": "<|padding_token_28|>",
271
  "lstrip": false,
272
- "normalized": true,
273
  "rstrip": false,
274
  "single_word": false,
275
- "special": false
276
  },
277
  "32031": {
278
  "content": "<|padding_token_29|>",
279
  "lstrip": false,
280
- "normalized": true,
281
  "rstrip": false,
282
  "single_word": false,
283
- "special": false
284
  },
285
  "32032": {
286
  "content": "<|padding_token_30|>",
287
  "lstrip": false,
288
- "normalized": true,
289
  "rstrip": false,
290
  "single_word": false,
291
- "special": false
292
  },
293
  "32033": {
294
  "content": "<|padding_token_31|>",
295
  "lstrip": false,
296
- "normalized": true,
297
  "rstrip": false,
298
  "single_word": false,
299
- "special": false
300
  },
301
  "32034": {
302
  "content": "<|padding_token_32|>",
303
  "lstrip": false,
304
- "normalized": true,
305
  "rstrip": false,
306
  "single_word": false,
307
- "special": false
308
  },
309
  "32035": {
310
  "content": "<|padding_token_33|>",
311
  "lstrip": false,
312
- "normalized": true,
313
  "rstrip": false,
314
  "single_word": false,
315
- "special": false
316
  },
317
  "32036": {
318
  "content": "<|padding_token_34|>",
319
  "lstrip": false,
320
- "normalized": true,
321
  "rstrip": false,
322
  "single_word": false,
323
- "special": false
324
  },
325
  "32037": {
326
  "content": "<|padding_token_35|>",
327
  "lstrip": false,
328
- "normalized": true,
329
  "rstrip": false,
330
  "single_word": false,
331
- "special": false
332
  },
333
  "32038": {
334
  "content": "<|padding_token_36|>",
335
  "lstrip": false,
336
- "normalized": true,
337
  "rstrip": false,
338
  "single_word": false,
339
- "special": false
340
  },
341
  "32039": {
342
  "content": "<|padding_token_37|>",
343
  "lstrip": false,
344
- "normalized": true,
345
  "rstrip": false,
346
  "single_word": false,
347
- "special": false
348
  },
349
  "32040": {
350
  "content": "<|padding_token_38|>",
351
  "lstrip": false,
352
- "normalized": true,
353
  "rstrip": false,
354
  "single_word": false,
355
- "special": false
356
  },
357
  "32041": {
358
  "content": "<|padding_token_39|>",
359
  "lstrip": false,
360
- "normalized": true,
361
  "rstrip": false,
362
  "single_word": false,
363
- "special": false
364
  },
365
  "32042": {
366
  "content": "<|padding_token_40|>",
367
  "lstrip": false,
368
- "normalized": true,
369
  "rstrip": false,
370
  "single_word": false,
371
- "special": false
372
  },
373
  "32043": {
374
  "content": "<|padding_token_41|>",
375
  "lstrip": false,
376
- "normalized": true,
377
  "rstrip": false,
378
  "single_word": false,
379
- "special": false
380
  },
381
  "32044": {
382
  "content": "<|padding_token_42|>",
383
  "lstrip": false,
384
- "normalized": true,
385
  "rstrip": false,
386
  "single_word": false,
387
- "special": false
388
  },
389
  "32045": {
390
  "content": "<|padding_token_43|>",
391
  "lstrip": false,
392
- "normalized": true,
393
  "rstrip": false,
394
  "single_word": false,
395
- "special": false
396
  },
397
  "32046": {
398
  "content": "<|padding_token_44|>",
399
  "lstrip": false,
400
- "normalized": true,
401
  "rstrip": false,
402
  "single_word": false,
403
- "special": false
404
  },
405
  "32047": {
406
  "content": "<|padding_token_45|>",
407
  "lstrip": false,
408
- "normalized": true,
409
  "rstrip": false,
410
  "single_word": false,
411
- "special": false
412
  },
413
  "32048": {
414
  "content": "<|padding_token_46|>",
415
  "lstrip": false,
416
- "normalized": true,
417
  "rstrip": false,
418
  "single_word": false,
419
- "special": false
420
  },
421
  "32049": {
422
  "content": "<|padding_token_47|>",
423
  "lstrip": false,
424
- "normalized": true,
425
  "rstrip": false,
426
  "single_word": false,
427
- "special": false
428
  },
429
  "32050": {
430
  "content": "<|padding_token_48|>",
431
  "lstrip": false,
432
- "normalized": true,
433
  "rstrip": false,
434
  "single_word": false,
435
- "special": false
436
  },
437
  "32051": {
438
  "content": "<|padding_token_49|>",
439
  "lstrip": false,
440
- "normalized": true,
441
  "rstrip": false,
442
  "single_word": false,
443
- "special": false
444
  },
445
  "32052": {
446
  "content": "<|padding_token_50|>",
447
  "lstrip": false,
448
- "normalized": true,
449
  "rstrip": false,
450
  "single_word": false,
451
- "special": false
452
  },
453
  "32053": {
454
  "content": "<|padding_token_51|>",
455
  "lstrip": false,
456
- "normalized": true,
457
  "rstrip": false,
458
  "single_word": false,
459
- "special": false
460
  },
461
  "32054": {
462
  "content": "<|padding_token_52|>",
463
  "lstrip": false,
464
- "normalized": true,
465
  "rstrip": false,
466
  "single_word": false,
467
- "special": false
468
  },
469
  "32055": {
470
  "content": "<|padding_token_53|>",
471
  "lstrip": false,
472
- "normalized": true,
473
  "rstrip": false,
474
  "single_word": false,
475
- "special": false
476
  },
477
  "32056": {
478
  "content": "<|padding_token_54|>",
479
  "lstrip": false,
480
- "normalized": true,
481
  "rstrip": false,
482
  "single_word": false,
483
- "special": false
484
  },
485
  "32057": {
486
  "content": "<|padding_token_55|>",
487
  "lstrip": false,
488
- "normalized": true,
489
  "rstrip": false,
490
  "single_word": false,
491
- "special": false
492
  },
493
  "32058": {
494
  "content": "<|padding_token_56|>",
495
  "lstrip": false,
496
- "normalized": true,
497
  "rstrip": false,
498
  "single_word": false,
499
- "special": false
500
  },
501
  "32059": {
502
  "content": "<|padding_token_57|>",
503
  "lstrip": false,
504
- "normalized": true,
505
  "rstrip": false,
506
  "single_word": false,
507
- "special": false
508
  },
509
  "32060": {
510
  "content": "<|padding_token_58|>",
511
  "lstrip": false,
512
- "normalized": true,
513
  "rstrip": false,
514
  "single_word": false,
515
- "special": false
516
  },
517
  "32061": {
518
  "content": "<|padding_token_59|>",
519
  "lstrip": false,
520
- "normalized": true,
521
  "rstrip": false,
522
  "single_word": false,
523
- "special": false
524
  },
525
  "32062": {
526
  "content": "<|padding_token_60|>",
527
  "lstrip": false,
528
- "normalized": true,
529
  "rstrip": false,
530
  "single_word": false,
531
- "special": false
532
  },
533
  "32063": {
534
  "content": "<|padding_token_61|>",
535
  "lstrip": false,
536
- "normalized": true,
537
  "rstrip": false,
538
  "single_word": false,
539
- "special": false
540
  },
541
  "32064": {
542
  "content": "<|padding_token_62|>",
543
  "lstrip": false,
544
- "normalized": true,
545
  "rstrip": false,
546
  "single_word": false,
547
- "special": false
548
  },
549
  "32065": {
550
  "content": "<|padding_token_63|>",
551
  "lstrip": false,
552
- "normalized": true,
553
  "rstrip": false,
554
  "single_word": false,
555
- "special": false
556
  },
557
  "32066": {
558
  "content": "<|padding_token_64|>",
559
  "lstrip": false,
560
- "normalized": true,
561
  "rstrip": false,
562
  "single_word": false,
563
- "special": false
564
  },
565
  "32067": {
566
  "content": "<|padding_token_65|>",
567
  "lstrip": false,
568
- "normalized": true,
569
  "rstrip": false,
570
  "single_word": false,
571
- "special": false
572
  },
573
  "32068": {
574
  "content": "<|padding_token_66|>",
575
  "lstrip": false,
576
- "normalized": true,
577
  "rstrip": false,
578
  "single_word": false,
579
- "special": false
580
  },
581
  "32069": {
582
  "content": "<|padding_token_67|>",
583
  "lstrip": false,
584
- "normalized": true,
585
  "rstrip": false,
586
  "single_word": false,
587
- "special": false
588
  },
589
  "32070": {
590
  "content": "<|padding_token_68|>",
591
  "lstrip": false,
592
- "normalized": true,
593
  "rstrip": false,
594
  "single_word": false,
595
- "special": false
596
  },
597
  "32071": {
598
  "content": "<|padding_token_69|>",
599
  "lstrip": false,
600
- "normalized": true,
601
  "rstrip": false,
602
  "single_word": false,
603
- "special": false
604
  },
605
  "32072": {
606
  "content": "<|padding_token_70|>",
607
  "lstrip": false,
608
- "normalized": true,
609
  "rstrip": false,
610
  "single_word": false,
611
- "special": false
612
  },
613
  "32073": {
614
  "content": "<|padding_token_71|>",
615
  "lstrip": false,
616
- "normalized": true,
617
  "rstrip": false,
618
  "single_word": false,
619
- "special": false
620
  },
621
  "32074": {
622
  "content": "<|padding_token_72|>",
623
  "lstrip": false,
624
- "normalized": true,
625
  "rstrip": false,
626
  "single_word": false,
627
- "special": false
628
  },
629
  "32075": {
630
  "content": "<|padding_token_73|>",
631
  "lstrip": false,
632
- "normalized": true,
633
  "rstrip": false,
634
  "single_word": false,
635
- "special": false
636
  },
637
  "32076": {
638
  "content": "<|padding_token_74|>",
639
  "lstrip": false,
640
- "normalized": true,
641
  "rstrip": false,
642
  "single_word": false,
643
- "special": false
644
  },
645
  "32077": {
646
  "content": "<|padding_token_75|>",
647
  "lstrip": false,
648
- "normalized": true,
649
  "rstrip": false,
650
  "single_word": false,
651
- "special": false
652
  },
653
  "32078": {
654
  "content": "<|padding_token_76|>",
655
  "lstrip": false,
656
- "normalized": true,
657
  "rstrip": false,
658
  "single_word": false,
659
- "special": false
660
  },
661
  "32079": {
662
  "content": "<|padding_token_77|>",
663
  "lstrip": false,
664
- "normalized": true,
665
  "rstrip": false,
666
  "single_word": false,
667
- "special": false
668
  },
669
  "32080": {
670
  "content": "<|padding_token_78|>",
671
  "lstrip": false,
672
- "normalized": true,
673
  "rstrip": false,
674
  "single_word": false,
675
- "special": false
676
  },
677
  "32081": {
678
  "content": "<|padding_token_79|>",
679
  "lstrip": false,
680
- "normalized": true,
681
  "rstrip": false,
682
  "single_word": false,
683
- "special": false
684
  },
685
  "32082": {
686
  "content": "<|padding_token_80|>",
687
  "lstrip": false,
688
- "normalized": true,
689
  "rstrip": false,
690
  "single_word": false,
691
- "special": false
692
  },
693
  "32083": {
694
  "content": "<|padding_token_81|>",
695
  "lstrip": false,
696
- "normalized": true,
697
  "rstrip": false,
698
  "single_word": false,
699
- "special": false
700
  },
701
  "32084": {
702
  "content": "<|padding_token_82|>",
703
  "lstrip": false,
704
- "normalized": true,
705
  "rstrip": false,
706
  "single_word": false,
707
- "special": false
708
  },
709
  "32085": {
710
  "content": "<|padding_token_83|>",
711
  "lstrip": false,
712
- "normalized": true,
713
  "rstrip": false,
714
  "single_word": false,
715
- "special": false
716
  },
717
  "32086": {
718
  "content": "<|padding_token_84|>",
719
  "lstrip": false,
720
- "normalized": true,
721
  "rstrip": false,
722
  "single_word": false,
723
- "special": false
724
  },
725
  "32087": {
726
  "content": "<|padding_token_85|>",
727
  "lstrip": false,
728
- "normalized": true,
729
  "rstrip": false,
730
  "single_word": false,
731
- "special": false
732
  },
733
  "32088": {
734
  "content": "<|padding_token_86|>",
735
  "lstrip": false,
736
- "normalized": true,
737
  "rstrip": false,
738
  "single_word": false,
739
- "special": false
740
  },
741
  "32089": {
742
  "content": "<|padding_token_87|>",
743
  "lstrip": false,
744
- "normalized": true,
745
  "rstrip": false,
746
  "single_word": false,
747
- "special": false
748
  },
749
  "32090": {
750
  "content": "<|padding_token_88|>",
751
  "lstrip": false,
752
- "normalized": true,
753
  "rstrip": false,
754
  "single_word": false,
755
- "special": false
756
  },
757
  "32091": {
758
  "content": "<|padding_token_89|>",
759
  "lstrip": false,
760
- "normalized": true,
761
  "rstrip": false,
762
  "single_word": false,
763
- "special": false
764
  },
765
  "32092": {
766
  "content": "<|padding_token_90|>",
767
  "lstrip": false,
768
- "normalized": true,
769
  "rstrip": false,
770
  "single_word": false,
771
- "special": false
772
  },
773
  "32093": {
774
  "content": "<|padding_token_91|>",
775
  "lstrip": false,
776
- "normalized": true,
777
  "rstrip": false,
778
  "single_word": false,
779
- "special": false
780
  },
781
  "32094": {
782
  "content": "<|padding_token_92|>",
783
  "lstrip": false,
784
- "normalized": true,
785
  "rstrip": false,
786
  "single_word": false,
787
- "special": false
788
  },
789
  "32095": {
790
  "content": "<|padding_token_93|>",
791
  "lstrip": false,
792
- "normalized": true,
793
  "rstrip": false,
794
  "single_word": false,
795
- "special": false
796
  },
797
  "32096": {
798
  "content": "<|padding_token_94|>",
799
  "lstrip": false,
800
- "normalized": true,
801
  "rstrip": false,
802
  "single_word": false,
803
- "special": false
804
  },
805
  "32097": {
806
  "content": "<|padding_token_95|>",
807
  "lstrip": false,
808
- "normalized": true,
809
  "rstrip": false,
810
  "single_word": false,
811
- "special": false
812
  },
813
  "32098": {
814
  "content": "<|padding_token_96|>",
815
  "lstrip": false,
816
- "normalized": true,
817
  "rstrip": false,
818
  "single_word": false,
819
- "special": false
820
  },
821
  "32099": {
822
  "content": "<|padding_token_97|>",
823
  "lstrip": false,
824
- "normalized": true,
825
  "rstrip": false,
826
  "single_word": false,
827
- "special": false
828
  },
829
  "32100": {
830
  "content": "<|padding_token_98|>",
831
  "lstrip": false,
832
- "normalized": true,
833
  "rstrip": false,
834
  "single_word": false,
835
- "special": false
836
  },
837
  "32101": {
838
  "content": "<|padding_token_99|>",
839
  "lstrip": false,
840
- "normalized": true,
841
  "rstrip": false,
842
  "single_word": false,
843
- "special": false
844
  },
845
  "32102": {
846
  "content": "<|padding_token_100|>",
847
  "lstrip": false,
848
- "normalized": true,
849
  "rstrip": false,
850
  "single_word": false,
851
- "special": false
852
  },
853
  "32103": {
854
  "content": "<|padding_token_101|>",
855
  "lstrip": false,
856
- "normalized": true,
857
  "rstrip": false,
858
  "single_word": false,
859
- "special": false
860
  },
861
  "32104": {
862
  "content": "<|padding_token_102|>",
863
  "lstrip": false,
864
- "normalized": true,
865
  "rstrip": false,
866
  "single_word": false,
867
- "special": false
868
  },
869
  "32105": {
870
  "content": "<|padding_token_103|>",
871
  "lstrip": false,
872
- "normalized": true,
873
  "rstrip": false,
874
  "single_word": false,
875
- "special": false
876
  },
877
  "32106": {
878
  "content": "<|padding_token_104|>",
879
  "lstrip": false,
880
- "normalized": true,
881
  "rstrip": false,
882
  "single_word": false,
883
- "special": false
884
  },
885
  "32107": {
886
  "content": "<|padding_token_105|>",
887
  "lstrip": false,
888
- "normalized": true,
889
  "rstrip": false,
890
  "single_word": false,
891
- "special": false
892
  },
893
  "32108": {
894
  "content": "<|padding_token_106|>",
895
  "lstrip": false,
896
- "normalized": true,
897
  "rstrip": false,
898
  "single_word": false,
899
- "special": false
900
  },
901
  "32109": {
902
  "content": "<|padding_token_107|>",
903
  "lstrip": false,
904
- "normalized": true,
905
  "rstrip": false,
906
  "single_word": false,
907
- "special": false
908
  },
909
  "32110": {
910
  "content": "<|padding_token_108|>",
911
  "lstrip": false,
912
- "normalized": true,
913
  "rstrip": false,
914
  "single_word": false,
915
- "special": false
916
  },
917
  "32111": {
918
  "content": "<|padding_token_109|>",
919
  "lstrip": false,
920
- "normalized": true,
921
  "rstrip": false,
922
  "single_word": false,
923
- "special": false
924
  },
925
  "32112": {
926
  "content": "<|padding_token_110|>",
927
  "lstrip": false,
928
- "normalized": true,
929
  "rstrip": false,
930
  "single_word": false,
931
- "special": false
932
  },
933
  "32113": {
934
  "content": "<|padding_token_111|>",
935
  "lstrip": false,
936
- "normalized": true,
937
  "rstrip": false,
938
  "single_word": false,
939
- "special": false
940
  },
941
  "32114": {
942
  "content": "<|padding_token_112|>",
943
  "lstrip": false,
944
- "normalized": true,
945
  "rstrip": false,
946
  "single_word": false,
947
- "special": false
948
  },
949
  "32115": {
950
  "content": "<|padding_token_113|>",
951
  "lstrip": false,
952
- "normalized": true,
953
  "rstrip": false,
954
  "single_word": false,
955
- "special": false
956
  },
957
  "32116": {
958
  "content": "<|padding_token_114|>",
959
  "lstrip": false,
960
- "normalized": true,
961
  "rstrip": false,
962
  "single_word": false,
963
- "special": false
964
  },
965
  "32117": {
966
  "content": "<|padding_token_115|>",
967
  "lstrip": false,
968
- "normalized": true,
969
  "rstrip": false,
970
  "single_word": false,
971
- "special": false
972
  },
973
  "32118": {
974
  "content": "<|padding_token_116|>",
975
  "lstrip": false,
976
- "normalized": true,
977
  "rstrip": false,
978
  "single_word": false,
979
- "special": false
980
  },
981
  "32119": {
982
  "content": "<|padding_token_117|>",
983
  "lstrip": false,
984
- "normalized": true,
985
  "rstrip": false,
986
  "single_word": false,
987
- "special": false
988
  },
989
  "32120": {
990
  "content": "<|padding_token_118|>",
991
  "lstrip": false,
992
- "normalized": true,
993
  "rstrip": false,
994
  "single_word": false,
995
- "special": false
996
  },
997
  "32121": {
998
  "content": "<|padding_token_119|>",
999
  "lstrip": false,
1000
- "normalized": true,
1001
  "rstrip": false,
1002
  "single_word": false,
1003
- "special": false
1004
  },
1005
  "32122": {
1006
  "content": "<|padding_token_120|>",
1007
  "lstrip": false,
1008
- "normalized": true,
1009
  "rstrip": false,
1010
  "single_word": false,
1011
- "special": false
1012
  },
1013
  "32123": {
1014
  "content": "<|padding_token_121|>",
1015
  "lstrip": false,
1016
- "normalized": true,
1017
  "rstrip": false,
1018
  "single_word": false,
1019
- "special": false
1020
  },
1021
  "32124": {
1022
  "content": "<|padding_token_122|>",
1023
  "lstrip": false,
1024
- "normalized": true,
1025
  "rstrip": false,
1026
  "single_word": false,
1027
- "special": false
1028
  },
1029
  "32125": {
1030
  "content": "<|padding_token_123|>",
1031
  "lstrip": false,
1032
- "normalized": true,
1033
  "rstrip": false,
1034
  "single_word": false,
1035
- "special": false
1036
  },
1037
  "32126": {
1038
  "content": "<|padding_token_124|>",
1039
  "lstrip": false,
1040
- "normalized": true,
1041
  "rstrip": false,
1042
  "single_word": false,
1043
- "special": false
1044
  },
1045
  "32127": {
1046
  "content": "<|padding_token_125|>",
1047
  "lstrip": false,
1048
- "normalized": true,
1049
  "rstrip": false,
1050
  "single_word": false,
1051
- "special": false
1052
  }
1053
  },
1054
  "additional_special_tokens": [
1055
  "<|im_start|>",
1056
- "<|im_end|>"
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1057
  ],
1058
  "bos_token": "<s>",
1059
  "clean_up_tokenization_spaces": false,
 
45
  "32002": {
46
  "content": "<|padding_token_0|>",
47
  "lstrip": false,
48
+ "normalized": false,
49
  "rstrip": false,
50
  "single_word": false,
51
+ "special": true
52
  },
53
  "32003": {
54
  "content": "<|padding_token_1|>",
55
  "lstrip": false,
56
+ "normalized": false,
57
  "rstrip": false,
58
  "single_word": false,
59
+ "special": true
60
  },
61
  "32004": {
62
  "content": "<|padding_token_2|>",
63
  "lstrip": false,
64
+ "normalized": false,
65
  "rstrip": false,
66
  "single_word": false,
67
+ "special": true
68
  },
69
  "32005": {
70
  "content": "<|padding_token_3|>",
71
  "lstrip": false,
72
+ "normalized": false,
73
  "rstrip": false,
74
  "single_word": false,
75
+ "special": true
76
  },
77
  "32006": {
78
  "content": "<|padding_token_4|>",
79
  "lstrip": false,
80
+ "normalized": false,
81
  "rstrip": false,
82
  "single_word": false,
83
+ "special": true
84
  },
85
  "32007": {
86
  "content": "<|padding_token_5|>",
87
  "lstrip": false,
88
+ "normalized": false,
89
  "rstrip": false,
90
  "single_word": false,
91
+ "special": true
92
  },
93
  "32008": {
94
  "content": "<|padding_token_6|>",
95
  "lstrip": false,
96
+ "normalized": false,
97
  "rstrip": false,
98
  "single_word": false,
99
+ "special": true
100
  },
101
  "32009": {
102
  "content": "<|padding_token_7|>",
103
  "lstrip": false,
104
+ "normalized": false,
105
  "rstrip": false,
106
  "single_word": false,
107
+ "special": true
108
  },
109
  "32010": {
110
  "content": "<|padding_token_8|>",
111
  "lstrip": false,
112
+ "normalized": false,
113
  "rstrip": false,
114
  "single_word": false,
115
+ "special": true
116
  },
117
  "32011": {
118
  "content": "<|padding_token_9|>",
119
  "lstrip": false,
120
+ "normalized": false,
121
  "rstrip": false,
122
  "single_word": false,
123
+ "special": true
124
  },
125
  "32012": {
126
  "content": "<|padding_token_10|>",
127
  "lstrip": false,
128
+ "normalized": false,
129
  "rstrip": false,
130
  "single_word": false,
131
+ "special": true
132
  },
133
  "32013": {
134
  "content": "<|padding_token_11|>",
135
  "lstrip": false,
136
+ "normalized": false,
137
  "rstrip": false,
138
  "single_word": false,
139
+ "special": true
140
  },
141
  "32014": {
142
  "content": "<|padding_token_12|>",
143
  "lstrip": false,
144
+ "normalized": false,
145
  "rstrip": false,
146
  "single_word": false,
147
+ "special": true
148
  },
149
  "32015": {
150
  "content": "<|padding_token_13|>",
151
  "lstrip": false,
152
+ "normalized": false,
153
  "rstrip": false,
154
  "single_word": false,
155
+ "special": true
156
  },
157
  "32016": {
158
  "content": "<|padding_token_14|>",
159
  "lstrip": false,
160
+ "normalized": false,
161
  "rstrip": false,
162
  "single_word": false,
163
+ "special": true
164
  },
165
  "32017": {
166
  "content": "<|padding_token_15|>",
167
  "lstrip": false,
168
+ "normalized": false,
169
  "rstrip": false,
170
  "single_word": false,
171
+ "special": true
172
  },
173
  "32018": {
174
  "content": "<|padding_token_16|>",
175
  "lstrip": false,
176
+ "normalized": false,
177
  "rstrip": false,
178
  "single_word": false,
179
+ "special": true
180
  },
181
  "32019": {
182
  "content": "<|padding_token_17|>",
183
  "lstrip": false,
184
+ "normalized": false,
185
  "rstrip": false,
186
  "single_word": false,
187
+ "special": true
188
  },
189
  "32020": {
190
  "content": "<|padding_token_18|>",
191
  "lstrip": false,
192
+ "normalized": false,
193
  "rstrip": false,
194
  "single_word": false,
195
+ "special": true
196
  },
197
  "32021": {
198
  "content": "<|padding_token_19|>",
199
  "lstrip": false,
200
+ "normalized": false,
201
  "rstrip": false,
202
  "single_word": false,
203
+ "special": true
204
  },
205
  "32022": {
206
  "content": "<|padding_token_20|>",
207
  "lstrip": false,
208
+ "normalized": false,
209
  "rstrip": false,
210
  "single_word": false,
211
+ "special": true
212
  },
213
  "32023": {
214
  "content": "<|padding_token_21|>",
215
  "lstrip": false,
216
+ "normalized": false,
217
  "rstrip": false,
218
  "single_word": false,
219
+ "special": true
220
  },
221
  "32024": {
222
  "content": "<|padding_token_22|>",
223
  "lstrip": false,
224
+ "normalized": false,
225
  "rstrip": false,
226
  "single_word": false,
227
+ "special": true
228
  },
229
  "32025": {
230
  "content": "<|padding_token_23|>",
231
  "lstrip": false,
232
+ "normalized": false,
233
  "rstrip": false,
234
  "single_word": false,
235
+ "special": true
236
  },
237
  "32026": {
238
  "content": "<|padding_token_24|>",
239
  "lstrip": false,
240
+ "normalized": false,
241
  "rstrip": false,
242
  "single_word": false,
243
+ "special": true
244
  },
245
  "32027": {
246
  "content": "<|padding_token_25|>",
247
  "lstrip": false,
248
+ "normalized": false,
249
  "rstrip": false,
250
  "single_word": false,
251
+ "special": true
252
  },
253
  "32028": {
254
  "content": "<|padding_token_26|>",
255
  "lstrip": false,
256
+ "normalized": false,
257
  "rstrip": false,
258
  "single_word": false,
259
+ "special": true
260
  },
261
  "32029": {
262
  "content": "<|padding_token_27|>",
263
  "lstrip": false,
264
+ "normalized": false,
265
  "rstrip": false,
266
  "single_word": false,
267
+ "special": true
268
  },
269
  "32030": {
270
  "content": "<|padding_token_28|>",
271
  "lstrip": false,
272
+ "normalized": false,
273
  "rstrip": false,
274
  "single_word": false,
275
+ "special": true
276
  },
277
  "32031": {
278
  "content": "<|padding_token_29|>",
279
  "lstrip": false,
280
+ "normalized": false,
281
  "rstrip": false,
282
  "single_word": false,
283
+ "special": true
284
  },
285
  "32032": {
286
  "content": "<|padding_token_30|>",
287
  "lstrip": false,
288
+ "normalized": false,
289
  "rstrip": false,
290
  "single_word": false,
291
+ "special": true
292
  },
293
  "32033": {
294
  "content": "<|padding_token_31|>",
295
  "lstrip": false,
296
+ "normalized": false,
297
  "rstrip": false,
298
  "single_word": false,
299
+ "special": true
300
  },
301
  "32034": {
302
  "content": "<|padding_token_32|>",
303
  "lstrip": false,
304
+ "normalized": false,
305
  "rstrip": false,
306
  "single_word": false,
307
+ "special": true
308
  },
309
  "32035": {
310
  "content": "<|padding_token_33|>",
311
  "lstrip": false,
312
+ "normalized": false,
313
  "rstrip": false,
314
  "single_word": false,
315
+ "special": true
316
  },
317
  "32036": {
318
  "content": "<|padding_token_34|>",
319
  "lstrip": false,
320
+ "normalized": false,
321
  "rstrip": false,
322
  "single_word": false,
323
+ "special": true
324
  },
325
  "32037": {
326
  "content": "<|padding_token_35|>",
327
  "lstrip": false,
328
+ "normalized": false,
329
  "rstrip": false,
330
  "single_word": false,
331
+ "special": true
332
  },
333
  "32038": {
334
  "content": "<|padding_token_36|>",
335
  "lstrip": false,
336
+ "normalized": false,
337
  "rstrip": false,
338
  "single_word": false,
339
+ "special": true
340
  },
341
  "32039": {
342
  "content": "<|padding_token_37|>",
343
  "lstrip": false,
344
+ "normalized": false,
345
  "rstrip": false,
346
  "single_word": false,
347
+ "special": true
348
  },
349
  "32040": {
350
  "content": "<|padding_token_38|>",
351
  "lstrip": false,
352
+ "normalized": false,
353
  "rstrip": false,
354
  "single_word": false,
355
+ "special": true
356
  },
357
  "32041": {
358
  "content": "<|padding_token_39|>",
359
  "lstrip": false,
360
+ "normalized": false,
361
  "rstrip": false,
362
  "single_word": false,
363
+ "special": true
364
  },
365
  "32042": {
366
  "content": "<|padding_token_40|>",
367
  "lstrip": false,
368
+ "normalized": false,
369
  "rstrip": false,
370
  "single_word": false,
371
+ "special": true
372
  },
373
  "32043": {
374
  "content": "<|padding_token_41|>",
375
  "lstrip": false,
376
+ "normalized": false,
377
  "rstrip": false,
378
  "single_word": false,
379
+ "special": true
380
  },
381
  "32044": {
382
  "content": "<|padding_token_42|>",
383
  "lstrip": false,
384
+ "normalized": false,
385
  "rstrip": false,
386
  "single_word": false,
387
+ "special": true
388
  },
389
  "32045": {
390
  "content": "<|padding_token_43|>",
391
  "lstrip": false,
392
+ "normalized": false,
393
  "rstrip": false,
394
  "single_word": false,
395
+ "special": true
396
  },
397
  "32046": {
398
  "content": "<|padding_token_44|>",
399
  "lstrip": false,
400
+ "normalized": false,
401
  "rstrip": false,
402
  "single_word": false,
403
+ "special": true
404
  },
405
  "32047": {
406
  "content": "<|padding_token_45|>",
407
  "lstrip": false,
408
+ "normalized": false,
409
  "rstrip": false,
410
  "single_word": false,
411
+ "special": true
412
  },
413
  "32048": {
414
  "content": "<|padding_token_46|>",
415
  "lstrip": false,
416
+ "normalized": false,
417
  "rstrip": false,
418
  "single_word": false,
419
+ "special": true
420
  },
421
  "32049": {
422
  "content": "<|padding_token_47|>",
423
  "lstrip": false,
424
+ "normalized": false,
425
  "rstrip": false,
426
  "single_word": false,
427
+ "special": true
428
  },
429
  "32050": {
430
  "content": "<|padding_token_48|>",
431
  "lstrip": false,
432
+ "normalized": false,
433
  "rstrip": false,
434
  "single_word": false,
435
+ "special": true
436
  },
437
  "32051": {
438
  "content": "<|padding_token_49|>",
439
  "lstrip": false,
440
+ "normalized": false,
441
  "rstrip": false,
442
  "single_word": false,
443
+ "special": true
444
  },
445
  "32052": {
446
  "content": "<|padding_token_50|>",
447
  "lstrip": false,
448
+ "normalized": false,
449
  "rstrip": false,
450
  "single_word": false,
451
+ "special": true
452
  },
453
  "32053": {
454
  "content": "<|padding_token_51|>",
455
  "lstrip": false,
456
+ "normalized": false,
457
  "rstrip": false,
458
  "single_word": false,
459
+ "special": true
460
  },
461
  "32054": {
462
  "content": "<|padding_token_52|>",
463
  "lstrip": false,
464
+ "normalized": false,
465
  "rstrip": false,
466
  "single_word": false,
467
+ "special": true
468
  },
469
  "32055": {
470
  "content": "<|padding_token_53|>",
471
  "lstrip": false,
472
+ "normalized": false,
473
  "rstrip": false,
474
  "single_word": false,
475
+ "special": true
476
  },
477
  "32056": {
478
  "content": "<|padding_token_54|>",
479
  "lstrip": false,
480
+ "normalized": false,
481
  "rstrip": false,
482
  "single_word": false,
483
+ "special": true
484
  },
485
  "32057": {
486
  "content": "<|padding_token_55|>",
487
  "lstrip": false,
488
+ "normalized": false,
489
  "rstrip": false,
490
  "single_word": false,
491
+ "special": true
492
  },
493
  "32058": {
494
  "content": "<|padding_token_56|>",
495
  "lstrip": false,
496
+ "normalized": false,
497
  "rstrip": false,
498
  "single_word": false,
499
+ "special": true
500
  },
501
  "32059": {
502
  "content": "<|padding_token_57|>",
503
  "lstrip": false,
504
+ "normalized": false,
505
  "rstrip": false,
506
  "single_word": false,
507
+ "special": true
508
  },
509
  "32060": {
510
  "content": "<|padding_token_58|>",
511
  "lstrip": false,
512
+ "normalized": false,
513
  "rstrip": false,
514
  "single_word": false,
515
+ "special": true
516
  },
517
  "32061": {
518
  "content": "<|padding_token_59|>",
519
  "lstrip": false,
520
+ "normalized": false,
521
  "rstrip": false,
522
  "single_word": false,
523
+ "special": true
524
  },
525
  "32062": {
526
  "content": "<|padding_token_60|>",
527
  "lstrip": false,
528
+ "normalized": false,
529
  "rstrip": false,
530
  "single_word": false,
531
+ "special": true
532
  },
533
  "32063": {
534
  "content": "<|padding_token_61|>",
535
  "lstrip": false,
536
+ "normalized": false,
537
  "rstrip": false,
538
  "single_word": false,
539
+ "special": true
540
  },
541
  "32064": {
542
  "content": "<|padding_token_62|>",
543
  "lstrip": false,
544
+ "normalized": false,
545
  "rstrip": false,
546
  "single_word": false,
547
+ "special": true
548
  },
549
  "32065": {
550
  "content": "<|padding_token_63|>",
551
  "lstrip": false,
552
+ "normalized": false,
553
  "rstrip": false,
554
  "single_word": false,
555
+ "special": true
556
  },
557
  "32066": {
558
  "content": "<|padding_token_64|>",
559
  "lstrip": false,
560
+ "normalized": false,
561
  "rstrip": false,
562
  "single_word": false,
563
+ "special": true
564
  },
565
  "32067": {
566
  "content": "<|padding_token_65|>",
567
  "lstrip": false,
568
+ "normalized": false,
569
  "rstrip": false,
570
  "single_word": false,
571
+ "special": true
572
  },
573
  "32068": {
574
  "content": "<|padding_token_66|>",
575
  "lstrip": false,
576
+ "normalized": false,
577
  "rstrip": false,
578
  "single_word": false,
579
+ "special": true
580
  },
581
  "32069": {
582
  "content": "<|padding_token_67|>",
583
  "lstrip": false,
584
+ "normalized": false,
585
  "rstrip": false,
586
  "single_word": false,
587
+ "special": true
588
  },
589
  "32070": {
590
  "content": "<|padding_token_68|>",
591
  "lstrip": false,
592
+ "normalized": false,
593
  "rstrip": false,
594
  "single_word": false,
595
+ "special": true
596
  },
597
  "32071": {
598
  "content": "<|padding_token_69|>",
599
  "lstrip": false,
600
+ "normalized": false,
601
  "rstrip": false,
602
  "single_word": false,
603
+ "special": true
604
  },
605
  "32072": {
606
  "content": "<|padding_token_70|>",
607
  "lstrip": false,
608
+ "normalized": false,
609
  "rstrip": false,
610
  "single_word": false,
611
+ "special": true
612
  },
613
  "32073": {
614
  "content": "<|padding_token_71|>",
615
  "lstrip": false,
616
+ "normalized": false,
617
  "rstrip": false,
618
  "single_word": false,
619
+ "special": true
620
  },
621
  "32074": {
622
  "content": "<|padding_token_72|>",
623
  "lstrip": false,
624
+ "normalized": false,
625
  "rstrip": false,
626
  "single_word": false,
627
+ "special": true
628
  },
629
  "32075": {
630
  "content": "<|padding_token_73|>",
631
  "lstrip": false,
632
+ "normalized": false,
633
  "rstrip": false,
634
  "single_word": false,
635
+ "special": true
636
  },
637
  "32076": {
638
  "content": "<|padding_token_74|>",
639
  "lstrip": false,
640
+ "normalized": false,
641
  "rstrip": false,
642
  "single_word": false,
643
+ "special": true
644
  },
645
  "32077": {
646
  "content": "<|padding_token_75|>",
647
  "lstrip": false,
648
+ "normalized": false,
649
  "rstrip": false,
650
  "single_word": false,
651
+ "special": true
652
  },
653
  "32078": {
654
  "content": "<|padding_token_76|>",
655
  "lstrip": false,
656
+ "normalized": false,
657
  "rstrip": false,
658
  "single_word": false,
659
+ "special": true
660
  },
661
  "32079": {
662
  "content": "<|padding_token_77|>",
663
  "lstrip": false,
664
+ "normalized": false,
665
  "rstrip": false,
666
  "single_word": false,
667
+ "special": true
668
  },
669
  "32080": {
670
  "content": "<|padding_token_78|>",
671
  "lstrip": false,
672
+ "normalized": false,
673
  "rstrip": false,
674
  "single_word": false,
675
+ "special": true
676
  },
677
  "32081": {
678
  "content": "<|padding_token_79|>",
679
  "lstrip": false,
680
+ "normalized": false,
681
  "rstrip": false,
682
  "single_word": false,
683
+ "special": true
684
  },
685
  "32082": {
686
  "content": "<|padding_token_80|>",
687
  "lstrip": false,
688
+ "normalized": false,
689
  "rstrip": false,
690
  "single_word": false,
691
+ "special": true
692
  },
693
  "32083": {
694
  "content": "<|padding_token_81|>",
695
  "lstrip": false,
696
+ "normalized": false,
697
  "rstrip": false,
698
  "single_word": false,
699
+ "special": true
700
  },
701
  "32084": {
702
  "content": "<|padding_token_82|>",
703
  "lstrip": false,
704
+ "normalized": false,
705
  "rstrip": false,
706
  "single_word": false,
707
+ "special": true
708
  },
709
  "32085": {
710
  "content": "<|padding_token_83|>",
711
  "lstrip": false,
712
+ "normalized": false,
713
  "rstrip": false,
714
  "single_word": false,
715
+ "special": true
716
  },
717
  "32086": {
718
  "content": "<|padding_token_84|>",
719
  "lstrip": false,
720
+ "normalized": false,
721
  "rstrip": false,
722
  "single_word": false,
723
+ "special": true
724
  },
725
  "32087": {
726
  "content": "<|padding_token_85|>",
727
  "lstrip": false,
728
+ "normalized": false,
729
  "rstrip": false,
730
  "single_word": false,
731
+ "special": true
732
  },
733
  "32088": {
734
  "content": "<|padding_token_86|>",
735
  "lstrip": false,
736
+ "normalized": false,
737
  "rstrip": false,
738
  "single_word": false,
739
+ "special": true
740
  },
741
  "32089": {
742
  "content": "<|padding_token_87|>",
743
  "lstrip": false,
744
+ "normalized": false,
745
  "rstrip": false,
746
  "single_word": false,
747
+ "special": true
748
  },
749
  "32090": {
750
  "content": "<|padding_token_88|>",
751
  "lstrip": false,
752
+ "normalized": false,
753
  "rstrip": false,
754
  "single_word": false,
755
+ "special": true
756
  },
757
  "32091": {
758
  "content": "<|padding_token_89|>",
759
  "lstrip": false,
760
+ "normalized": false,
761
  "rstrip": false,
762
  "single_word": false,
763
+ "special": true
764
  },
765
  "32092": {
766
  "content": "<|padding_token_90|>",
767
  "lstrip": false,
768
+ "normalized": false,
769
  "rstrip": false,
770
  "single_word": false,
771
+ "special": true
772
  },
773
  "32093": {
774
  "content": "<|padding_token_91|>",
775
  "lstrip": false,
776
+ "normalized": false,
777
  "rstrip": false,
778
  "single_word": false,
779
+ "special": true
780
  },
781
  "32094": {
782
  "content": "<|padding_token_92|>",
783
  "lstrip": false,
784
+ "normalized": false,
785
  "rstrip": false,
786
  "single_word": false,
787
+ "special": true
788
  },
789
  "32095": {
790
  "content": "<|padding_token_93|>",
791
  "lstrip": false,
792
+ "normalized": false,
793
  "rstrip": false,
794
  "single_word": false,
795
+ "special": true
796
  },
797
  "32096": {
798
  "content": "<|padding_token_94|>",
799
  "lstrip": false,
800
+ "normalized": false,
801
  "rstrip": false,
802
  "single_word": false,
803
+ "special": true
804
  },
805
  "32097": {
806
  "content": "<|padding_token_95|>",
807
  "lstrip": false,
808
+ "normalized": false,
809
  "rstrip": false,
810
  "single_word": false,
811
+ "special": true
812
  },
813
  "32098": {
814
  "content": "<|padding_token_96|>",
815
  "lstrip": false,
816
+ "normalized": false,
817
  "rstrip": false,
818
  "single_word": false,
819
+ "special": true
820
  },
821
  "32099": {
822
  "content": "<|padding_token_97|>",
823
  "lstrip": false,
824
+ "normalized": false,
825
  "rstrip": false,
826
  "single_word": false,
827
+ "special": true
828
  },
829
  "32100": {
830
  "content": "<|padding_token_98|>",
831
  "lstrip": false,
832
+ "normalized": false,
833
  "rstrip": false,
834
  "single_word": false,
835
+ "special": true
836
  },
837
  "32101": {
838
  "content": "<|padding_token_99|>",
839
  "lstrip": false,
840
+ "normalized": false,
841
  "rstrip": false,
842
  "single_word": false,
843
+ "special": true
844
  },
845
  "32102": {
846
  "content": "<|padding_token_100|>",
847
  "lstrip": false,
848
+ "normalized": false,
849
  "rstrip": false,
850
  "single_word": false,
851
+ "special": true
852
  },
853
  "32103": {
854
  "content": "<|padding_token_101|>",
855
  "lstrip": false,
856
+ "normalized": false,
857
  "rstrip": false,
858
  "single_word": false,
859
+ "special": true
860
  },
861
  "32104": {
862
  "content": "<|padding_token_102|>",
863
  "lstrip": false,
864
+ "normalized": false,
865
  "rstrip": false,
866
  "single_word": false,
867
+ "special": true
868
  },
869
  "32105": {
870
  "content": "<|padding_token_103|>",
871
  "lstrip": false,
872
+ "normalized": false,
873
  "rstrip": false,
874
  "single_word": false,
875
+ "special": true
876
  },
877
  "32106": {
878
  "content": "<|padding_token_104|>",
879
  "lstrip": false,
880
+ "normalized": false,
881
  "rstrip": false,
882
  "single_word": false,
883
+ "special": true
884
  },
885
  "32107": {
886
  "content": "<|padding_token_105|>",
887
  "lstrip": false,
888
+ "normalized": false,
889
  "rstrip": false,
890
  "single_word": false,
891
+ "special": true
892
  },
893
  "32108": {
894
  "content": "<|padding_token_106|>",
895
  "lstrip": false,
896
+ "normalized": false,
897
  "rstrip": false,
898
  "single_word": false,
899
+ "special": true
900
  },
901
  "32109": {
902
  "content": "<|padding_token_107|>",
903
  "lstrip": false,
904
+ "normalized": false,
905
  "rstrip": false,
906
  "single_word": false,
907
+ "special": true
908
  },
909
  "32110": {
910
  "content": "<|padding_token_108|>",
911
  "lstrip": false,
912
+ "normalized": false,
913
  "rstrip": false,
914
  "single_word": false,
915
+ "special": true
916
  },
917
  "32111": {
918
  "content": "<|padding_token_109|>",
919
  "lstrip": false,
920
+ "normalized": false,
921
  "rstrip": false,
922
  "single_word": false,
923
+ "special": true
924
  },
925
  "32112": {
926
  "content": "<|padding_token_110|>",
927
  "lstrip": false,
928
+ "normalized": false,
929
  "rstrip": false,
930
  "single_word": false,
931
+ "special": true
932
  },
933
  "32113": {
934
  "content": "<|padding_token_111|>",
935
  "lstrip": false,
936
+ "normalized": false,
937
  "rstrip": false,
938
  "single_word": false,
939
+ "special": true
940
  },
941
  "32114": {
942
  "content": "<|padding_token_112|>",
943
  "lstrip": false,
944
+ "normalized": false,
945
  "rstrip": false,
946
  "single_word": false,
947
+ "special": true
948
  },
949
  "32115": {
950
  "content": "<|padding_token_113|>",
951
  "lstrip": false,
952
+ "normalized": false,
953
  "rstrip": false,
954
  "single_word": false,
955
+ "special": true
956
  },
957
  "32116": {
958
  "content": "<|padding_token_114|>",
959
  "lstrip": false,
960
+ "normalized": false,
961
  "rstrip": false,
962
  "single_word": false,
963
+ "special": true
964
  },
965
  "32117": {
966
  "content": "<|padding_token_115|>",
967
  "lstrip": false,
968
+ "normalized": false,
969
  "rstrip": false,
970
  "single_word": false,
971
+ "special": true
972
  },
973
  "32118": {
974
  "content": "<|padding_token_116|>",
975
  "lstrip": false,
976
+ "normalized": false,
977
  "rstrip": false,
978
  "single_word": false,
979
+ "special": true
980
  },
981
  "32119": {
982
  "content": "<|padding_token_117|>",
983
  "lstrip": false,
984
+ "normalized": false,
985
  "rstrip": false,
986
  "single_word": false,
987
+ "special": true
988
  },
989
  "32120": {
990
  "content": "<|padding_token_118|>",
991
  "lstrip": false,
992
+ "normalized": false,
993
  "rstrip": false,
994
  "single_word": false,
995
+ "special": true
996
  },
997
  "32121": {
998
  "content": "<|padding_token_119|>",
999
  "lstrip": false,
1000
+ "normalized": false,
1001
  "rstrip": false,
1002
  "single_word": false,
1003
+ "special": true
1004
  },
1005
  "32122": {
1006
  "content": "<|padding_token_120|>",
1007
  "lstrip": false,
1008
+ "normalized": false,
1009
  "rstrip": false,
1010
  "single_word": false,
1011
+ "special": true
1012
  },
1013
  "32123": {
1014
  "content": "<|padding_token_121|>",
1015
  "lstrip": false,
1016
+ "normalized": false,
1017
  "rstrip": false,
1018
  "single_word": false,
1019
+ "special": true
1020
  },
1021
  "32124": {
1022
  "content": "<|padding_token_122|>",
1023
  "lstrip": false,
1024
+ "normalized": false,
1025
  "rstrip": false,
1026
  "single_word": false,
1027
+ "special": true
1028
  },
1029
  "32125": {
1030
  "content": "<|padding_token_123|>",
1031
  "lstrip": false,
1032
+ "normalized": false,
1033
  "rstrip": false,
1034
  "single_word": false,
1035
+ "special": true
1036
  },
1037
  "32126": {
1038
  "content": "<|padding_token_124|>",
1039
  "lstrip": false,
1040
+ "normalized": false,
1041
  "rstrip": false,
1042
  "single_word": false,
1043
+ "special": true
1044
  },
1045
  "32127": {
1046
  "content": "<|padding_token_125|>",
1047
  "lstrip": false,
1048
+ "normalized": false,
1049
  "rstrip": false,
1050
  "single_word": false,
1051
+ "special": true
1052
  }
1053
  },
1054
  "additional_special_tokens": [
1055
  "<|im_start|>",
1056
+ "<|im_end|>",
1057
+ "<|padding_token_0|>",
1058
+ "<|padding_token_1|>",
1059
+ "<|padding_token_2|>",
1060
+ "<|padding_token_3|>",
1061
+ "<|padding_token_4|>",
1062
+ "<|padding_token_5|>",
1063
+ "<|padding_token_6|>",
1064
+ "<|padding_token_7|>",
1065
+ "<|padding_token_8|>",
1066
+ "<|padding_token_9|>",
1067
+ "<|padding_token_10|>",
1068
+ "<|padding_token_11|>",
1069
+ "<|padding_token_12|>",
1070
+ "<|padding_token_13|>",
1071
+ "<|padding_token_14|>",
1072
+ "<|padding_token_15|>",
1073
+ "<|padding_token_16|>",
1074
+ "<|padding_token_17|>",
1075
+ "<|padding_token_18|>",
1076
+ "<|padding_token_19|>",
1077
+ "<|padding_token_20|>",
1078
+ "<|padding_token_21|>",
1079
+ "<|padding_token_22|>",
1080
+ "<|padding_token_23|>",
1081
+ "<|padding_token_24|>",
1082
+ "<|padding_token_25|>",
1083
+ "<|padding_token_26|>",
1084
+ "<|padding_token_27|>",
1085
+ "<|padding_token_28|>",
1086
+ "<|padding_token_29|>",
1087
+ "<|padding_token_30|>",
1088
+ "<|padding_token_31|>",
1089
+ "<|padding_token_32|>",
1090
+ "<|padding_token_33|>",
1091
+ "<|padding_token_34|>",
1092
+ "<|padding_token_35|>",
1093
+ "<|padding_token_36|>",
1094
+ "<|padding_token_37|>",
1095
+ "<|padding_token_38|>",
1096
+ "<|padding_token_39|>",
1097
+ "<|padding_token_40|>",
1098
+ "<|padding_token_41|>",
1099
+ "<|padding_token_42|>",
1100
+ "<|padding_token_43|>",
1101
+ "<|padding_token_44|>",
1102
+ "<|padding_token_45|>",
1103
+ "<|padding_token_46|>",
1104
+ "<|padding_token_47|>",
1105
+ "<|padding_token_48|>",
1106
+ "<|padding_token_49|>",
1107
+ "<|padding_token_50|>",
1108
+ "<|padding_token_51|>",
1109
+ "<|padding_token_52|>",
1110
+ "<|padding_token_53|>",
1111
+ "<|padding_token_54|>",
1112
+ "<|padding_token_55|>",
1113
+ "<|padding_token_56|>",
1114
+ "<|padding_token_57|>",
1115
+ "<|padding_token_58|>",
1116
+ "<|padding_token_59|>",
1117
+ "<|padding_token_60|>",
1118
+ "<|padding_token_61|>",
1119
+ "<|padding_token_62|>",
1120
+ "<|padding_token_63|>",
1121
+ "<|padding_token_64|>",
1122
+ "<|padding_token_65|>",
1123
+ "<|padding_token_66|>",
1124
+ "<|padding_token_67|>",
1125
+ "<|padding_token_68|>",
1126
+ "<|padding_token_69|>",
1127
+ "<|padding_token_70|>",
1128
+ "<|padding_token_71|>",
1129
+ "<|padding_token_72|>",
1130
+ "<|padding_token_73|>",
1131
+ "<|padding_token_74|>",
1132
+ "<|padding_token_75|>",
1133
+ "<|padding_token_76|>",
1134
+ "<|padding_token_77|>",
1135
+ "<|padding_token_78|>",
1136
+ "<|padding_token_79|>",
1137
+ "<|padding_token_80|>",
1138
+ "<|padding_token_81|>",
1139
+ "<|padding_token_82|>",
1140
+ "<|padding_token_83|>",
1141
+ "<|padding_token_84|>",
1142
+ "<|padding_token_85|>",
1143
+ "<|padding_token_86|>",
1144
+ "<|padding_token_87|>",
1145
+ "<|padding_token_88|>",
1146
+ "<|padding_token_89|>",
1147
+ "<|padding_token_90|>",
1148
+ "<|padding_token_91|>",
1149
+ "<|padding_token_92|>",
1150
+ "<|padding_token_93|>",
1151
+ "<|padding_token_94|>",
1152
+ "<|padding_token_95|>",
1153
+ "<|padding_token_96|>",
1154
+ "<|padding_token_97|>",
1155
+ "<|padding_token_98|>",
1156
+ "<|padding_token_99|>",
1157
+ "<|padding_token_100|>",
1158
+ "<|padding_token_101|>",
1159
+ "<|padding_token_102|>",
1160
+ "<|padding_token_103|>",
1161
+ "<|padding_token_104|>",
1162
+ "<|padding_token_105|>",
1163
+ "<|padding_token_106|>",
1164
+ "<|padding_token_107|>",
1165
+ "<|padding_token_108|>",
1166
+ "<|padding_token_109|>",
1167
+ "<|padding_token_110|>",
1168
+ "<|padding_token_111|>",
1169
+ "<|padding_token_112|>",
1170
+ "<|padding_token_113|>",
1171
+ "<|padding_token_114|>",
1172
+ "<|padding_token_115|>",
1173
+ "<|padding_token_116|>",
1174
+ "<|padding_token_117|>",
1175
+ "<|padding_token_118|>",
1176
+ "<|padding_token_119|>",
1177
+ "<|padding_token_120|>",
1178
+ "<|padding_token_121|>",
1179
+ "<|padding_token_122|>",
1180
+ "<|padding_token_123|>",
1181
+ "<|padding_token_124|>",
1182
+ "<|padding_token_125|>"
1183
  ],
1184
  "bos_token": "<s>",
1185
  "clean_up_tokenization_spaces": false,