damerajee commited on
Commit
aa438b7
·
verified ·
1 Parent(s): 1761706

Update tokenizer_config.json

Browse files
Files changed (1) hide show
  1. tokenizer_config.json +5 -5
tokenizer_config.json CHANGED
@@ -9,15 +9,15 @@
9
  "single_word": false,
10
  "special": true
11
  },
12
- "50257": {
13
- "content": "<pad>",
14
  "lstrip": false,
15
  "normalized": false,
16
  "rstrip": false,
17
  "single_word": false,
18
  "special": true
19
  },
20
- "50258": {
21
  "content": "<image>",
22
  "lstrip": false,
23
  "normalized": false,
@@ -29,11 +29,11 @@
29
  "additional_special_tokens": [
30
  "<image>"
31
  ],
32
- "bos_token": "<bos>",
33
  "clean_up_tokenization_spaces": true,
34
  "eos_token": "<|endoftext|>",
35
  "model_max_length": 1024,
36
- "pad_token": "<pad>",
37
  "tokenizer_class": "GPT2Tokenizer",
38
  "unk_token": "<|endoftext|>"
39
  }
 
9
  "single_word": false,
10
  "special": true
11
  },
12
+ "50256": {
13
+ "content": "<|endoftext|>",
14
  "lstrip": false,
15
  "normalized": false,
16
  "rstrip": false,
17
  "single_word": false,
18
  "special": true
19
  },
20
+ "50257": {
21
  "content": "<image>",
22
  "lstrip": false,
23
  "normalized": false,
 
29
  "additional_special_tokens": [
30
  "<image>"
31
  ],
32
+ "bos_token": "<|endoftext|>",
33
  "clean_up_tokenization_spaces": true,
34
  "eos_token": "<|endoftext|>",
35
  "model_max_length": 1024,
36
+ "pad_token": "<|endoftext|>",
37
  "tokenizer_class": "GPT2Tokenizer",
38
  "unk_token": "<|endoftext|>"
39
  }