lknik commited on
Commit
941d545
·
verified ·
1 Parent(s): 0d1134a

Upload special_tokens_map.json with huggingface_hub

Browse files
Files changed (1) hide show
  1. special_tokens_map.json +160 -0
special_tokens_map.json ADDED
@@ -0,0 +1,160 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "additional_special_tokens": [
3
+ "<|im_start|>",
4
+ "<|im_end|>",
5
+ "<|function_list|>",
6
+ "<|function_output|>",
7
+ "<|function_call|>",
8
+ "<|control_6|>",
9
+ "<|control_7|>",
10
+ "<|control_8|>",
11
+ "<|control_9|>",
12
+ "<|control_10|>",
13
+ "<|control_11|>",
14
+ "<|control_12|>",
15
+ "<|control_13|>",
16
+ "<|control_14|>",
17
+ "<|control_15|>",
18
+ "<|control_16|>",
19
+ "<|control_17|>",
20
+ "<|control_18|>",
21
+ "<|control_19|>",
22
+ "<|control_20|>",
23
+ "<|control_21|>",
24
+ "<|control_22|>",
25
+ "<|control_23|>",
26
+ "<|control_24|>",
27
+ "<|control_25|>",
28
+ "<|control_26|>",
29
+ "<|control_27|>",
30
+ "<|control_28|>",
31
+ "<|control_29|>",
32
+ "<|control_30|>",
33
+ "<|control_31|>",
34
+ "<|control_32|>",
35
+ "<|control_33|>",
36
+ "<|control_34|>",
37
+ "<|control_35|>",
38
+ "<|control_36|>",
39
+ "<|control_37|>",
40
+ "<|control_38|>",
41
+ "<|control_39|>",
42
+ "<|control_40|>",
43
+ "<|control_41|>",
44
+ "<|control_42|>",
45
+ "<|control_43|>",
46
+ "<|control_44|>",
47
+ "<|control_45|>",
48
+ "<|control_46|>",
49
+ "<|control_47|>",
50
+ "<|control_48|>",
51
+ "<|control_49|>",
52
+ "<|control_50|>",
53
+ "<|control_51|>",
54
+ "<|control_52|>",
55
+ "<|control_53|>",
56
+ "<|control_54|>",
57
+ "<|control_55|>",
58
+ "<|control_56|>",
59
+ "<|control_57|>",
60
+ "<|control_58|>",
61
+ "<|control_59|>",
62
+ "<|control_60|>",
63
+ "<|control_61|>",
64
+ "<|control_62|>",
65
+ "<|control_63|>",
66
+ "<|control_64|>",
67
+ "<|control_65|>",
68
+ "<|control_66|>",
69
+ "<|control_67|>",
70
+ "<|control_68|>",
71
+ "<|control_69|>",
72
+ "<|control_70|>",
73
+ "<|control_71|>",
74
+ "<|control_72|>",
75
+ "<|control_73|>",
76
+ "<|control_74|>",
77
+ "<|control_75|>",
78
+ "<|control_76|>",
79
+ "<|control_77|>",
80
+ "<|control_78|>",
81
+ "<|control_79|>",
82
+ "<|control_80|>",
83
+ "<|control_81|>",
84
+ "<|control_82|>",
85
+ "<|control_83|>",
86
+ "<|control_84|>",
87
+ "<|control_85|>",
88
+ "<|control_86|>",
89
+ "<|control_87|>",
90
+ "<|control_88|>",
91
+ "<|control_89|>",
92
+ "<|control_90|>",
93
+ "<|control_91|>",
94
+ "<|control_92|>",
95
+ "<|control_93|>",
96
+ "<|control_94|>",
97
+ "<|control_95|>",
98
+ "<|control_96|>",
99
+ "<|control_97|>",
100
+ "<|control_98|>",
101
+ "<|control_99|>",
102
+ "<|control_100|>",
103
+ "<|control_101|>",
104
+ "<|control_102|>",
105
+ "<|control_103|>",
106
+ "<|control_104|>",
107
+ "<|control_105|>",
108
+ "<|control_106|>",
109
+ "<|control_107|>",
110
+ "<|control_108|>",
111
+ "<|control_109|>",
112
+ "<|control_110|>",
113
+ "<|control_111|>",
114
+ "<|control_112|>",
115
+ "<|control_113|>",
116
+ "<|control_114|>",
117
+ "<|control_115|>",
118
+ "<|control_116|>",
119
+ "<|control_117|>",
120
+ "<|control_118|>",
121
+ "<|control_119|>",
122
+ "<|control_120|>",
123
+ "<|control_121|>",
124
+ "<|control_122|>",
125
+ "<|control_123|>",
126
+ "<|control_124|>",
127
+ "<|control_125|>",
128
+ "<|control_126|>",
129
+ "<|control_127|>",
130
+ "<|control_128|>"
131
+ ],
132
+ "bos_token": {
133
+ "content": "<s>",
134
+ "lstrip": false,
135
+ "normalized": false,
136
+ "rstrip": false,
137
+ "single_word": false
138
+ },
139
+ "eos_token": {
140
+ "content": "<|im_end|>",
141
+ "lstrip": false,
142
+ "normalized": false,
143
+ "rstrip": false,
144
+ "single_word": false
145
+ },
146
+ "pad_token": {
147
+ "content": "</s>",
148
+ "lstrip": false,
149
+ "normalized": false,
150
+ "rstrip": false,
151
+ "single_word": false
152
+ },
153
+ "unk_token": {
154
+ "content": "<unk>",
155
+ "lstrip": false,
156
+ "normalized": false,
157
+ "rstrip": false,
158
+ "single_word": false
159
+ }
160
+ }