Upload folder using huggingface_hub
Browse files- tokenizer.json +18 -0
tokenizer.json
CHANGED
|
@@ -110,6 +110,24 @@
|
|
| 110 |
"rstrip": false,
|
| 111 |
"normalized": false,
|
| 112 |
"special": true
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 113 |
}
|
| 114 |
],
|
| 115 |
"normalizer": {
|
|
|
|
| 110 |
"rstrip": false,
|
| 111 |
"normalized": false,
|
| 112 |
"special": true
|
| 113 |
+
},
|
| 114 |
+
{
|
| 115 |
+
"id": 151655,
|
| 116 |
+
"content": "<|image_pad|>",
|
| 117 |
+
"single_word": false,
|
| 118 |
+
"lstrip": false,
|
| 119 |
+
"rstrip": false,
|
| 120 |
+
"normalized": false,
|
| 121 |
+
"special": true
|
| 122 |
+
},
|
| 123 |
+
{
|
| 124 |
+
"id": 151656,
|
| 125 |
+
"content": "<|video_pad|>",
|
| 126 |
+
"single_word": false,
|
| 127 |
+
"lstrip": false,
|
| 128 |
+
"rstrip": false,
|
| 129 |
+
"normalized": false,
|
| 130 |
+
"special": true
|
| 131 |
}
|
| 132 |
],
|
| 133 |
"normalizer": {
|