NickyNicky commited on
Commit
f35b047
1 Parent(s): 4eadb73

Update added_tokens.json

Browse files
Files changed (1) hide show
  1. added_tokens.json +34 -7
added_tokens.json CHANGED
@@ -1,13 +1,40 @@
1
  {
 
 
 
 
 
 
2
  "<|assistant|>": 32001,
 
 
 
 
 
 
 
 
3
  "<|endoftext|>": 32000,
4
  "<|end|>": 32007,
5
- "<|placeholder1|>": 32002,
6
- "<|placeholder2|>": 32003,
7
- "<|placeholder3|>": 32004,
8
- "<|placeholder4|>": 32005,
9
- "<|placeholder5|>": 32008,
10
- "<|placeholder6|>": 32009,
 
 
 
 
 
 
 
 
 
 
 
11
  "<|system|>": 32006,
 
 
12
  "<|user|>": 32010
13
- }
 
1
  {
2
+ "<|/code|>": 32014,
3
+ "<|/data|>": 32033,
4
+ "<|/inst|>": 32037,
5
+ "<|/query|>": 32031,
6
+ "<|/sys|>": 32035,
7
+ "<|assistant_mask|>": 32017,
8
  "<|assistant|>": 32001,
9
+ "<|calc|>": 32012,
10
+ "<|code|>": 32013,
11
+ "<|continue|>": 32009,
12
+ "<|data|>": 32032,
13
+ "<|diff_marker|>": 32025,
14
+ "<|disc_sep|>": 32029,
15
+ "<|disc_start|>": 32028,
16
+ "<|disc_thread|><|query|>": 32030,
17
  "<|endoftext|>": 32000,
18
  "<|end|>": 32007,
19
+ "<|fim_middle|>": 32021,
20
+ "<|fim_prefix|>": 32020,
21
+ "<|fim_suffix|>": 32022,
22
+ "<|function_call|>": 32005,
23
+ "<|function_list|>": 32011,
24
+ "<|function_output|>": 32003,
25
+ "<|ghissue|>": 32026,
26
+ "<|ghreview|>": 32027,
27
+ "<|inst|>": 32036,
28
+ "<|ipynb_marker|>": 32024,
29
+ "<|message|>": 32019,
30
+ "<|meta_start|>": 32023,
31
+ "<|raw|>": 32008,
32
+ "<|resource|>": 32016,
33
+ "<|start|>": 32018,
34
+ "<|step|>": 32002,
35
+ "<|summary|>": 32015,
36
  "<|system|>": 32006,
37
+ "<|sys|>": 32034,
38
+ "<|tag|>": 32004,
39
  "<|user|>": 32010
40
+ }