Upload special_tokens_map.json with huggingface_hub
Browse files- special_tokens_map.json +160 -0
special_tokens_map.json
ADDED
@@ -0,0 +1,160 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
{
|
2 |
+
"additional_special_tokens": [
|
3 |
+
"<|im_start|>",
|
4 |
+
"<|im_end|>",
|
5 |
+
"<|function_list|>",
|
6 |
+
"<|function_output|>",
|
7 |
+
"<|function_call|>",
|
8 |
+
"<|control_6|>",
|
9 |
+
"<|control_7|>",
|
10 |
+
"<|control_8|>",
|
11 |
+
"<|control_9|>",
|
12 |
+
"<|control_10|>",
|
13 |
+
"<|control_11|>",
|
14 |
+
"<|control_12|>",
|
15 |
+
"<|control_13|>",
|
16 |
+
"<|control_14|>",
|
17 |
+
"<|control_15|>",
|
18 |
+
"<|control_16|>",
|
19 |
+
"<|control_17|>",
|
20 |
+
"<|control_18|>",
|
21 |
+
"<|control_19|>",
|
22 |
+
"<|control_20|>",
|
23 |
+
"<|control_21|>",
|
24 |
+
"<|control_22|>",
|
25 |
+
"<|control_23|>",
|
26 |
+
"<|control_24|>",
|
27 |
+
"<|control_25|>",
|
28 |
+
"<|control_26|>",
|
29 |
+
"<|control_27|>",
|
30 |
+
"<|control_28|>",
|
31 |
+
"<|control_29|>",
|
32 |
+
"<|control_30|>",
|
33 |
+
"<|control_31|>",
|
34 |
+
"<|control_32|>",
|
35 |
+
"<|control_33|>",
|
36 |
+
"<|control_34|>",
|
37 |
+
"<|control_35|>",
|
38 |
+
"<|control_36|>",
|
39 |
+
"<|control_37|>",
|
40 |
+
"<|control_38|>",
|
41 |
+
"<|control_39|>",
|
42 |
+
"<|control_40|>",
|
43 |
+
"<|control_41|>",
|
44 |
+
"<|control_42|>",
|
45 |
+
"<|control_43|>",
|
46 |
+
"<|control_44|>",
|
47 |
+
"<|control_45|>",
|
48 |
+
"<|control_46|>",
|
49 |
+
"<|control_47|>",
|
50 |
+
"<|control_48|>",
|
51 |
+
"<|control_49|>",
|
52 |
+
"<|control_50|>",
|
53 |
+
"<|control_51|>",
|
54 |
+
"<|control_52|>",
|
55 |
+
"<|control_53|>",
|
56 |
+
"<|control_54|>",
|
57 |
+
"<|control_55|>",
|
58 |
+
"<|control_56|>",
|
59 |
+
"<|control_57|>",
|
60 |
+
"<|control_58|>",
|
61 |
+
"<|control_59|>",
|
62 |
+
"<|control_60|>",
|
63 |
+
"<|control_61|>",
|
64 |
+
"<|control_62|>",
|
65 |
+
"<|control_63|>",
|
66 |
+
"<|control_64|>",
|
67 |
+
"<|control_65|>",
|
68 |
+
"<|control_66|>",
|
69 |
+
"<|control_67|>",
|
70 |
+
"<|control_68|>",
|
71 |
+
"<|control_69|>",
|
72 |
+
"<|control_70|>",
|
73 |
+
"<|control_71|>",
|
74 |
+
"<|control_72|>",
|
75 |
+
"<|control_73|>",
|
76 |
+
"<|control_74|>",
|
77 |
+
"<|control_75|>",
|
78 |
+
"<|control_76|>",
|
79 |
+
"<|control_77|>",
|
80 |
+
"<|control_78|>",
|
81 |
+
"<|control_79|>",
|
82 |
+
"<|control_80|>",
|
83 |
+
"<|control_81|>",
|
84 |
+
"<|control_82|>",
|
85 |
+
"<|control_83|>",
|
86 |
+
"<|control_84|>",
|
87 |
+
"<|control_85|>",
|
88 |
+
"<|control_86|>",
|
89 |
+
"<|control_87|>",
|
90 |
+
"<|control_88|>",
|
91 |
+
"<|control_89|>",
|
92 |
+
"<|control_90|>",
|
93 |
+
"<|control_91|>",
|
94 |
+
"<|control_92|>",
|
95 |
+
"<|control_93|>",
|
96 |
+
"<|control_94|>",
|
97 |
+
"<|control_95|>",
|
98 |
+
"<|control_96|>",
|
99 |
+
"<|control_97|>",
|
100 |
+
"<|control_98|>",
|
101 |
+
"<|control_99|>",
|
102 |
+
"<|control_100|>",
|
103 |
+
"<|control_101|>",
|
104 |
+
"<|control_102|>",
|
105 |
+
"<|control_103|>",
|
106 |
+
"<|control_104|>",
|
107 |
+
"<|control_105|>",
|
108 |
+
"<|control_106|>",
|
109 |
+
"<|control_107|>",
|
110 |
+
"<|control_108|>",
|
111 |
+
"<|control_109|>",
|
112 |
+
"<|control_110|>",
|
113 |
+
"<|control_111|>",
|
114 |
+
"<|control_112|>",
|
115 |
+
"<|control_113|>",
|
116 |
+
"<|control_114|>",
|
117 |
+
"<|control_115|>",
|
118 |
+
"<|control_116|>",
|
119 |
+
"<|control_117|>",
|
120 |
+
"<|control_118|>",
|
121 |
+
"<|control_119|>",
|
122 |
+
"<|control_120|>",
|
123 |
+
"<|control_121|>",
|
124 |
+
"<|control_122|>",
|
125 |
+
"<|control_123|>",
|
126 |
+
"<|control_124|>",
|
127 |
+
"<|control_125|>",
|
128 |
+
"<|control_126|>",
|
129 |
+
"<|control_127|>",
|
130 |
+
"<|control_128|>"
|
131 |
+
],
|
132 |
+
"bos_token": {
|
133 |
+
"content": "<s>",
|
134 |
+
"lstrip": false,
|
135 |
+
"normalized": false,
|
136 |
+
"rstrip": false,
|
137 |
+
"single_word": false
|
138 |
+
},
|
139 |
+
"eos_token": {
|
140 |
+
"content": "<|im_end|>",
|
141 |
+
"lstrip": false,
|
142 |
+
"normalized": false,
|
143 |
+
"rstrip": false,
|
144 |
+
"single_word": false
|
145 |
+
},
|
146 |
+
"pad_token": {
|
147 |
+
"content": "</s>",
|
148 |
+
"lstrip": false,
|
149 |
+
"normalized": false,
|
150 |
+
"rstrip": false,
|
151 |
+
"single_word": false
|
152 |
+
},
|
153 |
+
"unk_token": {
|
154 |
+
"content": "<unk>",
|
155 |
+
"lstrip": false,
|
156 |
+
"normalized": false,
|
157 |
+
"rstrip": false,
|
158 |
+
"single_word": false
|
159 |
+
}
|
160 |
+
}
|