ZhangYuanhan commited on
Commit
a5fdab8
1 Parent(s): 14c179a

Upload tokenizer_config.json with huggingface_hub

Browse files
Files changed (1) hide show
  1. tokenizer_config.json +13 -4
tokenizer_config.json CHANGED
@@ -23,9 +23,17 @@
23
  "normalized": false,
24
  "rstrip": false,
25
  "single_word": false,
26
- "special": true
27
- }
28
- },
 
 
 
 
 
 
 
 
29
  "additional_special_tokens": [
30
  "<|im_start|>",
31
  "<|im_end|>"
@@ -35,9 +43,10 @@
35
  "clean_up_tokenization_spaces": false,
36
  "eos_token": "<|im_end|>",
37
  "errors": "replace",
38
- "model_max_length": 12768,
39
  "pad_token": "<|endoftext|>",
40
  "padding_side": "right",
 
41
  "split_special_tokens": false,
42
  "tokenizer_class": "Qwen2Tokenizer",
43
  "unk_token": null
 
23
  "normalized": false,
24
  "rstrip": false,
25
  "single_word": false,
26
+ "special": true
27
+ },
28
+ "151646": {
29
+ "content": "<image>",
30
+ "lstrip": false,
31
+ "normalized": false,
32
+ "rstrip": false,
33
+ "single_word": false,
34
+ "special": true
35
+ }
36
+ },
37
  "additional_special_tokens": [
38
  "<|im_start|>",
39
  "<|im_end|>"
 
43
  "clean_up_tokenization_spaces": false,
44
  "eos_token": "<|im_end|>",
45
  "errors": "replace",
46
+ "model_max_length": 32768,
47
  "pad_token": "<|endoftext|>",
48
  "padding_side": "right",
49
+ "processor_class": "LlavaProcessor",
50
  "split_special_tokens": false,
51
  "tokenizer_class": "Qwen2Tokenizer",
52
  "unk_token": null