danielhanchen commited on
Commit
038572a
1 Parent(s): acec122

Upload tokenizer

Browse files
special_tokens_map.json CHANGED
@@ -13,13 +13,7 @@
13
  "rstrip": false,
14
  "single_word": false
15
  },
16
- "pad_token": {
17
- "content": "[control_14]",
18
- "lstrip": false,
19
- "normalized": false,
20
- "rstrip": false,
21
- "single_word": false
22
- },
23
  "unk_token": {
24
  "content": "<unk>",
25
  "lstrip": false,
 
13
  "rstrip": false,
14
  "single_word": false
15
  },
16
+ "pad_token": "[control_748]",
 
 
 
 
 
 
17
  "unk_token": {
18
  "content": "<unk>",
19
  "lstrip": false,
tokenizer.json CHANGED
@@ -154,7 +154,7 @@
154
  "lstrip": false,
155
  "rstrip": false,
156
  "normalized": false,
157
- "special": true
158
  },
159
  {
160
  "id": 17,
@@ -6760,7 +6760,7 @@
6760
  "lstrip": false,
6761
  "rstrip": false,
6762
  "normalized": false,
6763
- "special": false
6764
  },
6765
  {
6766
  "id": 751,
 
154
  "lstrip": false,
155
  "rstrip": false,
156
  "normalized": false,
157
+ "special": false
158
  },
159
  {
160
  "id": 17,
 
6760
  "lstrip": false,
6761
  "rstrip": false,
6762
  "normalized": false,
6763
+ "special": true
6764
  },
6765
  {
6766
  "id": 751,
tokenizer_config.json CHANGED
@@ -137,7 +137,7 @@
137
  "normalized": false,
138
  "rstrip": false,
139
  "single_word": false,
140
- "special": true
141
  },
142
  "17": {
143
  "content": "[control_15]",
@@ -6009,7 +6009,7 @@
6009
  "normalized": false,
6010
  "rstrip": false,
6011
  "single_word": false,
6012
- "special": false
6013
  },
6014
  "751": {
6015
  "content": "[REFERENCE_DOC_19]",
@@ -6178,7 +6178,7 @@
6178
  "eos_token": "</s>",
6179
  "legacy": true,
6180
  "model_max_length": 1000000000000000019884624838656,
6181
- "pad_token": "[control_14]",
6182
  "padding_side": "left",
6183
  "sp_model_kwargs": {},
6184
  "spaces_between_special_tokens": false,
 
137
  "normalized": false,
138
  "rstrip": false,
139
  "single_word": false,
140
+ "special": false
141
  },
142
  "17": {
143
  "content": "[control_15]",
 
6009
  "normalized": false,
6010
  "rstrip": false,
6011
  "single_word": false,
6012
+ "special": true
6013
  },
6014
  "751": {
6015
  "content": "[REFERENCE_DOC_19]",
 
6178
  "eos_token": "</s>",
6179
  "legacy": true,
6180
  "model_max_length": 1000000000000000019884624838656,
6181
+ "pad_token": "[control_748]",
6182
  "padding_side": "left",
6183
  "sp_model_kwargs": {},
6184
  "spaces_between_special_tokens": false,