Hugging Face's logo

djuna
/
TEST-Q2.5-Lenned-14B

Text Generation
Transformers
Safetensors
qwen2
mergekit
Merge
conversational
text-generation-inference
Inference Endpoints
Model card Files Community
1
djuna commited on
Commit
d86d668
·
verified ·
1 Parent(s): 6fecc68

Update tokenizer_config.json

Browse files
Files changed (1) hide show
  1. tokenizer_config.json +14 -14
tokenizer_config.json CHANGED
@@ -5,58 +5,58 @@
5
  "151643": {
6
  "content": "<|endoftext|>",
7
  "lstrip": false,
8
- "normalized": true,
9
  "rstrip": false,
10
  "single_word": false,
11
- "special": false
12
  },
13
  "151644": {
14
  "content": "<|im_start|>",
15
  "lstrip": false,
16
- "normalized": true,
17
  "rstrip": false,
18
  "single_word": false,
19
- "special": false
20
  },
21
  "151645": {
22
  "content": "<|im_end|>",
23
  "lstrip": false,
24
- "normalized": true,
25
  "rstrip": false,
26
  "single_word": false,
27
- "special": false
28
  },
29
  "151646": {
30
  "content": "<|object_ref_start|>",
31
  "lstrip": false,
32
- "normalized": true,
33
  "rstrip": false,
34
  "single_word": false,
35
- "special": false
36
  },
37
  "151647": {
38
  "content": "<|object_ref_end|>",
39
  "lstrip": false,
40
- "normalized": true,
41
  "rstrip": false,
42
  "single_word": false,
43
- "special": false
44
  },
45
  "151648": {
46
  "content": "<|box_start|>",
47
  "lstrip": false,
48
- "normalized": true,
49
  "rstrip": false,
50
  "single_word": false,
51
- "special": false
52
  },
53
  "151649": {
54
  "content": "<|box_end|>",
55
  "lstrip": false,
56
- "normalized": true,
57
  "rstrip": false,
58
  "single_word": false,
59
- "special": false
60
  },
61
  "151650": {
62
  "content": "<|quad_start|>",
 
5
  "151643": {
6
  "content": "<|endoftext|>",
7
  "lstrip": false,
8
+ "normalized": false,
9
  "rstrip": false,
10
  "single_word": false,
11
+ "special": true
12
  },
13
  "151644": {
14
  "content": "<|im_start|>",
15
  "lstrip": false,
16
+ "normalized": false,
17
  "rstrip": false,
18
  "single_word": false,
19
+ "special": true
20
  },
21
  "151645": {
22
  "content": "<|im_end|>",
23
  "lstrip": false,
24
+ "normalized": false,
25
  "rstrip": false,
26
  "single_word": false,
27
+ "special": true
28
  },
29
  "151646": {
30
  "content": "<|object_ref_start|>",
31
  "lstrip": false,
32
+ "normalized": false,
33
  "rstrip": false,
34
  "single_word": false,
35
+ "special": true
36
  },
37
  "151647": {
38
  "content": "<|object_ref_end|>",
39
  "lstrip": false,
40
+ "normalized": false,
41
  "rstrip": false,
42
  "single_word": false,
43
+ "special": true
44
  },
45
  "151648": {
46
  "content": "<|box_start|>",
47
  "lstrip": false,
48
+ "normalized": false,
49
  "rstrip": false,
50
  "single_word": false,
51
+ "special": true
52
  },
53
  "151649": {
54
  "content": "<|box_end|>",
55
  "lstrip": false,
56
+ "normalized": false,
57
  "rstrip": false,
58
  "single_word": false,
59
+ "special": true
60
  },
61
  "151650": {
62
  "content": "<|quad_start|>",