richardllin cArlIcon commited on
Commit
03cae8d
1 Parent(s): 58cb0a1

compatible with Llama (#29)

Browse files

- compatible with Llama (fb27ecc0a9f7e6ff6ec99f506cab1b9a24c1550a)


Co-authored-by: cArlIcon <[email protected]>

config.json CHANGED
@@ -1,12 +1,7 @@
1
  {
2
  "architectures": [
3
- "YiForCausalLM"
4
  ],
5
- "auto_map": {
6
- "AutoConfig": "configuration_yi.YiConfig",
7
- "AutoModel": "modeling_yi.YiModel",
8
- "AutoModelForCausalLM":"modeling_yi.YiForCausalLM"
9
- },
10
  "bos_token_id": 1,
11
  "eos_token_id": 2,
12
  "hidden_act": "silu",
@@ -14,12 +9,14 @@
14
  "initializer_range": 0.02,
15
  "intermediate_size": 20480,
16
  "max_position_embeddings": 4096,
17
- "model_type": "Yi",
18
  "num_attention_heads": 56,
19
  "num_hidden_layers": 60,
20
  "num_key_value_heads": 8,
21
  "pad_token_id": 0,
 
22
  "rms_norm_eps": 1e-05,
 
23
  "rope_theta": 5000000.0,
24
  "tie_word_embeddings": false,
25
  "torch_dtype": "bfloat16",
 
1
  {
2
  "architectures": [
3
+ "LlamaForCausalLM"
4
  ],
 
 
 
 
 
5
  "bos_token_id": 1,
6
  "eos_token_id": 2,
7
  "hidden_act": "silu",
 
9
  "initializer_range": 0.02,
10
  "intermediate_size": 20480,
11
  "max_position_embeddings": 4096,
12
+ "model_type": "llama",
13
  "num_attention_heads": 56,
14
  "num_hidden_layers": 60,
15
  "num_key_value_heads": 8,
16
  "pad_token_id": 0,
17
+ "pretraining_tp": 1,
18
  "rms_norm_eps": 1e-05,
19
+ "rope_scaling": null,
20
  "rope_theta": 5000000.0,
21
  "tie_word_embeddings": false,
22
  "torch_dtype": "bfloat16",
configuration_yi.py DELETED
@@ -1,121 +0,0 @@
1
- """ Yi model configuration"""
2
- from transformers.configuration_utils import PretrainedConfig
3
- from transformers.utils import logging
4
-
5
- logger = logging.get_logger(__name__)
6
-
7
- Yi_PRETRAINED_CONFIG_ARCHIVE_MAP = {}
8
-
9
-
10
- class YiConfig(PretrainedConfig):
11
- r"""
12
- This is the configuration class to store the configuration of a [`YiModel`]. It is used to instantiate an Yi
13
- model according to the specified arguments, defining the model architecture. Instantiating a configuration with the
14
- defaults will yield a similar configuration to that of the Yi model.
15
-
16
- Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
17
- documentation from [`PretrainedConfig`] for more information.
18
-
19
-
20
- Args:
21
- vocab_size (`int`, *optional*, defaults to 64000):
22
- Vocabulary size of the Yi model. Defines the number of different tokens that can be represented by the
23
- `inputs_ids` passed when calling [`YiModel`]
24
- hidden_size (`int`, *optional*, defaults to 4096):
25
- Dimension of the hidden representations.
26
- intermediate_size (`int`, *optional*, defaults to 11008):
27
- Dimension of the MLP representations.
28
- num_hidden_layers (`int`, *optional*, defaults to 32):
29
- Number of hidden layers in the Transformer encoder.
30
- num_attention_heads (`int`, *optional*, defaults to 32):
31
- Number of attention heads for each attention layer in the Transformer encoder.
32
- num_key_value_heads (`int`, *optional*):
33
- This is the number of key_value heads that should be used to implement Grouped Query Attention. If
34
- `num_key_value_heads=num_attention_heads`, the model will use Multi Head Attention (MHA), if
35
- `num_key_value_heads=1 the model will use Multi Query Attention (MQA) otherwise GQA is used. When
36
- converting a multi-head checkpoint to a GQA checkpoint, each group key and value head should be constructed
37
- by meanpooling all the original heads within that group. For more details checkout [this
38
- paper](https://arxiv.org/pdf/2305.13245.pdf). If it is not specified, will default to
39
- `num_attention_heads`.
40
- hidden_act (`str` or `function`, *optional*, defaults to `"silu"`):
41
- The non-linear activation function (function or string) in the decoder.
42
- max_position_embeddings (`int`, *optional*, defaults to 4096):
43
- The maximum sequence length that this model might ever be used with. Typically set this to something large
44
- just in case (e.g., 512 or 1024 or 2048 or 4096).
45
- initializer_range (`float`, *optional*, defaults to 0.02):
46
- The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
47
- rms_norm_eps (`float`, *optional*, defaults to 1e-5):
48
- The epsilon used by the rms normalization layers.
49
- use_cache (`bool`, *optional*, defaults to `True`):
50
- Whether or not the model should return the last key/values attentions (not used by all models). Only
51
- relevant if `config.is_decoder=True`.
52
- tie_word_embeddings(`bool`, *optional*, defaults to `False`):
53
- Whether to tie weight embeddings
54
- output_attentions (`bool`, *optional*, defaults to `False`):
55
- Whether or not to output attentions.
56
- rope_theta (`float`, *optional*, defaults to 5000000.0):
57
- The base period of the RoPE embeddings.
58
- Example:
59
-
60
- ```python
61
- >>> from transformers import YiModel, YiConfig
62
-
63
- >>> # Initializing a Yi style configuration
64
- >>> configuration = YiConfig()
65
-
66
- >>> # Initializing a model from the Yi style configuration
67
- >>> model = YiModel(configuration)
68
-
69
- >>> # Accessing the model configuration
70
- >>> configuration = model.config
71
- ```"""
72
- model_type = "Yi"
73
- keys_to_ignore_at_inference = ["past_key_values"]
74
-
75
- def __init__(
76
- self,
77
- vocab_size=64000,
78
- hidden_size=4096,
79
- intermediate_size=11008,
80
- num_hidden_layers=32,
81
- num_attention_heads=32,
82
- num_key_value_heads=4,
83
- hidden_act="silu",
84
- max_position_embeddings=4096,
85
- initializer_range=0.02,
86
- rms_norm_eps=1e-5,
87
- use_cache=True,
88
- pad_token_id=0,
89
- bos_token_id=1,
90
- eos_token_id=2,
91
- tie_word_embeddings=False,
92
- output_attentions=False,
93
- rope_theta=5000000.0,
94
- **kwargs,
95
- ):
96
- self.vocab_size = vocab_size
97
- self.max_position_embeddings = max_position_embeddings
98
- self.hidden_size = hidden_size
99
- self.intermediate_size = intermediate_size
100
- self.num_hidden_layers = num_hidden_layers
101
- self.num_attention_heads = num_attention_heads
102
-
103
- # for backward compatibility
104
- if num_key_value_heads is None:
105
- num_key_value_heads = num_attention_heads
106
-
107
- self.num_key_value_heads = num_key_value_heads
108
- self.hidden_act = hidden_act
109
- self.initializer_range = initializer_range
110
- self.rms_norm_eps = rms_norm_eps
111
- self.use_cache = use_cache
112
- self.output_attentions = output_attentions
113
- self.rope_theta = rope_theta
114
-
115
- super().__init__(
116
- pad_token_id=pad_token_id,
117
- bos_token_id=bos_token_id,
118
- eos_token_id=eos_token_id,
119
- tie_word_embeddings=tie_word_embeddings,
120
- **kwargs,
121
- )
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
md5 CHANGED
@@ -1,8 +1,8 @@
1
- bd61d46468ce4753b6cf9dd1c88a52ff pytorch_model-00001-of-00007.bin
2
- a2b7a099e0daa708273ee309da1c8bee pytorch_model-00002-of-00007.bin
3
- 94205d500e9f5bf98a19d280ae40f84e pytorch_model-00003-of-00007.bin
4
- 56226a1dcc168e0fd8c59de298d0fba9 pytorch_model-00004-of-00007.bin
5
- ccd5e152300d3808f617a4ae8753597b pytorch_model-00005-of-00007.bin
6
- 2eeddeefa85e471980a40f32aba4fb14 pytorch_model-00006-of-00007.bin
7
- 7a22d73a68850c4185f476142e64fc5b pytorch_model-00007-of-00007.bin
8
  291724ef50f729e45d68f474a7755bbc tokenizer.model
 
1
+ 93c0ef5a73e79306fcc5510d684c30b7 pytorch_model-00001-of-00007.bin
2
+ 69db6eb2f065bb3c08097054d7d0fcb1 pytorch_model-00002-of-00007.bin
3
+ e47971db5e97ab7dd7e700a0f6a78c62 pytorch_model-00003-of-00007.bin
4
+ 52ce36edd7a89ec98c5ccf84b9b23791 pytorch_model-00004-of-00007.bin
5
+ 23faf3f93f2f48eb2512ac9302c8f454 pytorch_model-00005-of-00007.bin
6
+ da88c3977a3f92faed0bfd80c1c716d1 pytorch_model-00006-of-00007.bin
7
+ 74055c9e8b7bf1877be59f2ab6763efc pytorch_model-00007-of-00007.bin
8
  291724ef50f729e45d68f474a7755bbc tokenizer.model
model-00001-of-00007.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:1187484fbfd0e1d44ab02dd4d4d3073caedf102e902528a6fe57ee44e486a230
3
- size 9975341424
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:028c29a21e851f0edc3e2f7375a00720c0fde4eaf1f71cd7fbd8bfc628f5ce52
3
+ size 9975341688
model-00002-of-00007.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:e42253f62eff30b75f618fe99287c640b5bd0e0198b5bec131c58933028da897
3
- size 9909310048
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:4a2b490a2d2940ab3aecb6b640732ec0e615cb35d9e29b16cac4c92d6cb708c6
3
+ size 9909310352
model-00003-of-00007.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:ee65dfbf793637d6c7b1ca838fab7b98542f26a48ca6344fd53cc5c487be3497
3
- size 9747800728
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f52c3c39336e02ba646f0500e56e0ef45126422ee34d83520bdb7662a90ec51a
3
+ size 9747801000
model-00004-of-00007.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:e9bb7bcaa98d020456593b81bdbb225c9303fce56bf71582ab7847f642c5f81f
3
- size 9747829592
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:1fc56ec8b64af5f0224c58e0e2166b9d19f2004594a9f8472938ed30cb3dbadd
3
+ size 9747829904
model-00005-of-00007.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:b9690685ba164a62c9d483b698b09cef48198a8f5800dcd87bdf5dfcc663b468
3
- size 9747829592
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f4b4515c2f9bb8bf81d26ae713f6bb7552002d07c61fd96ac645fd7867b7e01a
3
+ size 9747829904
model-00006-of-00007.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:da6c169cf4ef72f9f84ce9122c4d716daec3ac4014bafe24c4979a4acb0c9e6c
3
- size 9938670424
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:52f42a501fd1eca6f2f2caa016b5aa5e5dbcaf4544677476243672b4153f80c1
3
+ size 9938670728
model-00007-of-00007.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:5951f9dd2c359206dbbe8372e5f401257c2fbd27eb12221d121b38bf2d5dbc18
3
- size 9711114048
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:0aa6031494fb65e2bf0ca0b5246f1d125e25bdcf804da50d71699c2865612c32
3
+ size 9711114320
model.safetensors.index.json CHANGED
@@ -5,542 +5,542 @@
5
  "weight_map": {
6
  "lm_head.weight": "model-00007-of-00007.safetensors",
7
  "model.embed_tokens.weight": "model-00001-of-00007.safetensors",
8
- "model.layers.0.ln1.weight": "model-00001-of-00007.safetensors",
9
- "model.layers.0.ln2.weight": "model-00001-of-00007.safetensors",
10
  "model.layers.0.mlp.down_proj.weight": "model-00001-of-00007.safetensors",
11
  "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00007.safetensors",
12
  "model.layers.0.mlp.up_proj.weight": "model-00001-of-00007.safetensors",
 
13
  "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00007.safetensors",
14
  "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00007.safetensors",
15
  "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00007.safetensors",
16
  "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00007.safetensors",
17
- "model.layers.1.ln1.weight": "model-00001-of-00007.safetensors",
18
- "model.layers.1.ln2.weight": "model-00001-of-00007.safetensors",
19
  "model.layers.1.mlp.down_proj.weight": "model-00001-of-00007.safetensors",
20
  "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00007.safetensors",
21
  "model.layers.1.mlp.up_proj.weight": "model-00001-of-00007.safetensors",
 
22
  "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00007.safetensors",
23
  "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00007.safetensors",
24
  "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00007.safetensors",
25
  "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00007.safetensors",
26
- "model.layers.10.ln1.weight": "model-00002-of-00007.safetensors",
27
- "model.layers.10.ln2.weight": "model-00002-of-00007.safetensors",
28
  "model.layers.10.mlp.down_proj.weight": "model-00002-of-00007.safetensors",
29
  "model.layers.10.mlp.gate_proj.weight": "model-00002-of-00007.safetensors",
30
  "model.layers.10.mlp.up_proj.weight": "model-00002-of-00007.safetensors",
 
31
  "model.layers.10.self_attn.k_proj.weight": "model-00002-of-00007.safetensors",
32
  "model.layers.10.self_attn.o_proj.weight": "model-00002-of-00007.safetensors",
33
  "model.layers.10.self_attn.q_proj.weight": "model-00002-of-00007.safetensors",
34
  "model.layers.10.self_attn.v_proj.weight": "model-00002-of-00007.safetensors",
35
- "model.layers.11.ln1.weight": "model-00002-of-00007.safetensors",
36
- "model.layers.11.ln2.weight": "model-00002-of-00007.safetensors",
37
  "model.layers.11.mlp.down_proj.weight": "model-00002-of-00007.safetensors",
38
  "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00007.safetensors",
39
  "model.layers.11.mlp.up_proj.weight": "model-00002-of-00007.safetensors",
 
40
  "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00007.safetensors",
41
  "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00007.safetensors",
42
  "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00007.safetensors",
43
  "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00007.safetensors",
44
- "model.layers.12.ln1.weight": "model-00002-of-00007.safetensors",
45
- "model.layers.12.ln2.weight": "model-00002-of-00007.safetensors",
46
  "model.layers.12.mlp.down_proj.weight": "model-00002-of-00007.safetensors",
47
  "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00007.safetensors",
48
  "model.layers.12.mlp.up_proj.weight": "model-00002-of-00007.safetensors",
 
49
  "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00007.safetensors",
50
  "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00007.safetensors",
51
  "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00007.safetensors",
52
  "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00007.safetensors",
53
- "model.layers.13.ln1.weight": "model-00002-of-00007.safetensors",
54
- "model.layers.13.ln2.weight": "model-00002-of-00007.safetensors",
55
  "model.layers.13.mlp.down_proj.weight": "model-00002-of-00007.safetensors",
56
  "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00007.safetensors",
57
  "model.layers.13.mlp.up_proj.weight": "model-00002-of-00007.safetensors",
 
58
  "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00007.safetensors",
59
  "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00007.safetensors",
60
  "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00007.safetensors",
61
  "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00007.safetensors",
62
- "model.layers.14.ln1.weight": "model-00002-of-00007.safetensors",
63
- "model.layers.14.ln2.weight": "model-00002-of-00007.safetensors",
64
  "model.layers.14.mlp.down_proj.weight": "model-00002-of-00007.safetensors",
65
  "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00007.safetensors",
66
  "model.layers.14.mlp.up_proj.weight": "model-00002-of-00007.safetensors",
 
67
  "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00007.safetensors",
68
  "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00007.safetensors",
69
  "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00007.safetensors",
70
  "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00007.safetensors",
71
- "model.layers.15.ln1.weight": "model-00002-of-00007.safetensors",
72
- "model.layers.15.ln2.weight": "model-00002-of-00007.safetensors",
73
  "model.layers.15.mlp.down_proj.weight": "model-00002-of-00007.safetensors",
74
  "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00007.safetensors",
75
  "model.layers.15.mlp.up_proj.weight": "model-00002-of-00007.safetensors",
 
76
  "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00007.safetensors",
77
  "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00007.safetensors",
78
  "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00007.safetensors",
79
  "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00007.safetensors",
80
- "model.layers.16.ln1.weight": "model-00002-of-00007.safetensors",
81
- "model.layers.16.ln2.weight": "model-00002-of-00007.safetensors",
82
  "model.layers.16.mlp.down_proj.weight": "model-00002-of-00007.safetensors",
83
  "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00007.safetensors",
84
  "model.layers.16.mlp.up_proj.weight": "model-00002-of-00007.safetensors",
 
85
  "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00007.safetensors",
86
  "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00007.safetensors",
87
  "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00007.safetensors",
88
  "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00007.safetensors",
89
- "model.layers.17.ln1.weight": "model-00003-of-00007.safetensors",
90
- "model.layers.17.ln2.weight": "model-00003-of-00007.safetensors",
91
  "model.layers.17.mlp.down_proj.weight": "model-00003-of-00007.safetensors",
92
  "model.layers.17.mlp.gate_proj.weight": "model-00003-of-00007.safetensors",
93
  "model.layers.17.mlp.up_proj.weight": "model-00003-of-00007.safetensors",
 
94
  "model.layers.17.self_attn.k_proj.weight": "model-00003-of-00007.safetensors",
95
  "model.layers.17.self_attn.o_proj.weight": "model-00003-of-00007.safetensors",
96
  "model.layers.17.self_attn.q_proj.weight": "model-00003-of-00007.safetensors",
97
  "model.layers.17.self_attn.v_proj.weight": "model-00003-of-00007.safetensors",
98
- "model.layers.18.ln1.weight": "model-00003-of-00007.safetensors",
99
- "model.layers.18.ln2.weight": "model-00003-of-00007.safetensors",
100
  "model.layers.18.mlp.down_proj.weight": "model-00003-of-00007.safetensors",
101
  "model.layers.18.mlp.gate_proj.weight": "model-00003-of-00007.safetensors",
102
  "model.layers.18.mlp.up_proj.weight": "model-00003-of-00007.safetensors",
 
103
  "model.layers.18.self_attn.k_proj.weight": "model-00003-of-00007.safetensors",
104
  "model.layers.18.self_attn.o_proj.weight": "model-00003-of-00007.safetensors",
105
  "model.layers.18.self_attn.q_proj.weight": "model-00003-of-00007.safetensors",
106
  "model.layers.18.self_attn.v_proj.weight": "model-00003-of-00007.safetensors",
107
- "model.layers.19.ln1.weight": "model-00003-of-00007.safetensors",
108
- "model.layers.19.ln2.weight": "model-00003-of-00007.safetensors",
109
  "model.layers.19.mlp.down_proj.weight": "model-00003-of-00007.safetensors",
110
  "model.layers.19.mlp.gate_proj.weight": "model-00003-of-00007.safetensors",
111
  "model.layers.19.mlp.up_proj.weight": "model-00003-of-00007.safetensors",
 
112
  "model.layers.19.self_attn.k_proj.weight": "model-00003-of-00007.safetensors",
113
  "model.layers.19.self_attn.o_proj.weight": "model-00003-of-00007.safetensors",
114
  "model.layers.19.self_attn.q_proj.weight": "model-00003-of-00007.safetensors",
115
  "model.layers.19.self_attn.v_proj.weight": "model-00003-of-00007.safetensors",
116
- "model.layers.2.ln1.weight": "model-00001-of-00007.safetensors",
117
- "model.layers.2.ln2.weight": "model-00001-of-00007.safetensors",
118
  "model.layers.2.mlp.down_proj.weight": "model-00001-of-00007.safetensors",
119
  "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00007.safetensors",
120
  "model.layers.2.mlp.up_proj.weight": "model-00001-of-00007.safetensors",
 
121
  "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00007.safetensors",
122
  "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00007.safetensors",
123
  "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00007.safetensors",
124
  "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00007.safetensors",
125
- "model.layers.20.ln1.weight": "model-00003-of-00007.safetensors",
126
- "model.layers.20.ln2.weight": "model-00003-of-00007.safetensors",
127
  "model.layers.20.mlp.down_proj.weight": "model-00003-of-00007.safetensors",
128
  "model.layers.20.mlp.gate_proj.weight": "model-00003-of-00007.safetensors",
129
  "model.layers.20.mlp.up_proj.weight": "model-00003-of-00007.safetensors",
 
130
  "model.layers.20.self_attn.k_proj.weight": "model-00003-of-00007.safetensors",
131
  "model.layers.20.self_attn.o_proj.weight": "model-00003-of-00007.safetensors",
132
  "model.layers.20.self_attn.q_proj.weight": "model-00003-of-00007.safetensors",
133
  "model.layers.20.self_attn.v_proj.weight": "model-00003-of-00007.safetensors",
134
- "model.layers.21.ln1.weight": "model-00003-of-00007.safetensors",
135
- "model.layers.21.ln2.weight": "model-00003-of-00007.safetensors",
136
  "model.layers.21.mlp.down_proj.weight": "model-00003-of-00007.safetensors",
137
  "model.layers.21.mlp.gate_proj.weight": "model-00003-of-00007.safetensors",
138
  "model.layers.21.mlp.up_proj.weight": "model-00003-of-00007.safetensors",
 
139
  "model.layers.21.self_attn.k_proj.weight": "model-00003-of-00007.safetensors",
140
  "model.layers.21.self_attn.o_proj.weight": "model-00003-of-00007.safetensors",
141
  "model.layers.21.self_attn.q_proj.weight": "model-00003-of-00007.safetensors",
142
  "model.layers.21.self_attn.v_proj.weight": "model-00003-of-00007.safetensors",
143
- "model.layers.22.ln1.weight": "model-00003-of-00007.safetensors",
144
- "model.layers.22.ln2.weight": "model-00003-of-00007.safetensors",
145
  "model.layers.22.mlp.down_proj.weight": "model-00003-of-00007.safetensors",
146
  "model.layers.22.mlp.gate_proj.weight": "model-00003-of-00007.safetensors",
147
  "model.layers.22.mlp.up_proj.weight": "model-00003-of-00007.safetensors",
 
148
  "model.layers.22.self_attn.k_proj.weight": "model-00003-of-00007.safetensors",
149
  "model.layers.22.self_attn.o_proj.weight": "model-00003-of-00007.safetensors",
150
  "model.layers.22.self_attn.q_proj.weight": "model-00003-of-00007.safetensors",
151
  "model.layers.22.self_attn.v_proj.weight": "model-00003-of-00007.safetensors",
152
- "model.layers.23.ln1.weight": "model-00003-of-00007.safetensors",
153
- "model.layers.23.ln2.weight": "model-00003-of-00007.safetensors",
154
  "model.layers.23.mlp.down_proj.weight": "model-00003-of-00007.safetensors",
155
  "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00007.safetensors",
156
  "model.layers.23.mlp.up_proj.weight": "model-00003-of-00007.safetensors",
 
157
  "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00007.safetensors",
158
  "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00007.safetensors",
159
  "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00007.safetensors",
160
  "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00007.safetensors",
161
- "model.layers.24.ln1.weight": "model-00003-of-00007.safetensors",
162
- "model.layers.24.ln2.weight": "model-00003-of-00007.safetensors",
163
  "model.layers.24.mlp.down_proj.weight": "model-00003-of-00007.safetensors",
164
  "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00007.safetensors",
165
  "model.layers.24.mlp.up_proj.weight": "model-00003-of-00007.safetensors",
 
166
  "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00007.safetensors",
167
  "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00007.safetensors",
168
  "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00007.safetensors",
169
  "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00007.safetensors",
170
- "model.layers.25.ln1.weight": "model-00004-of-00007.safetensors",
171
- "model.layers.25.ln2.weight": "model-00004-of-00007.safetensors",
172
  "model.layers.25.mlp.down_proj.weight": "model-00003-of-00007.safetensors",
173
  "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00007.safetensors",
174
  "model.layers.25.mlp.up_proj.weight": "model-00004-of-00007.safetensors",
 
175
  "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00007.safetensors",
176
  "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00007.safetensors",
177
  "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00007.safetensors",
178
  "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00007.safetensors",
179
- "model.layers.26.ln1.weight": "model-00004-of-00007.safetensors",
180
- "model.layers.26.ln2.weight": "model-00004-of-00007.safetensors",
181
  "model.layers.26.mlp.down_proj.weight": "model-00004-of-00007.safetensors",
182
  "model.layers.26.mlp.gate_proj.weight": "model-00004-of-00007.safetensors",
183
  "model.layers.26.mlp.up_proj.weight": "model-00004-of-00007.safetensors",
 
184
  "model.layers.26.self_attn.k_proj.weight": "model-00004-of-00007.safetensors",
185
  "model.layers.26.self_attn.o_proj.weight": "model-00004-of-00007.safetensors",
186
  "model.layers.26.self_attn.q_proj.weight": "model-00004-of-00007.safetensors",
187
  "model.layers.26.self_attn.v_proj.weight": "model-00004-of-00007.safetensors",
188
- "model.layers.27.ln1.weight": "model-00004-of-00007.safetensors",
189
- "model.layers.27.ln2.weight": "model-00004-of-00007.safetensors",
190
  "model.layers.27.mlp.down_proj.weight": "model-00004-of-00007.safetensors",
191
  "model.layers.27.mlp.gate_proj.weight": "model-00004-of-00007.safetensors",
192
  "model.layers.27.mlp.up_proj.weight": "model-00004-of-00007.safetensors",
 
193
  "model.layers.27.self_attn.k_proj.weight": "model-00004-of-00007.safetensors",
194
  "model.layers.27.self_attn.o_proj.weight": "model-00004-of-00007.safetensors",
195
  "model.layers.27.self_attn.q_proj.weight": "model-00004-of-00007.safetensors",
196
  "model.layers.27.self_attn.v_proj.weight": "model-00004-of-00007.safetensors",
197
- "model.layers.28.ln1.weight": "model-00004-of-00007.safetensors",
198
- "model.layers.28.ln2.weight": "model-00004-of-00007.safetensors",
199
  "model.layers.28.mlp.down_proj.weight": "model-00004-of-00007.safetensors",
200
  "model.layers.28.mlp.gate_proj.weight": "model-00004-of-00007.safetensors",
201
  "model.layers.28.mlp.up_proj.weight": "model-00004-of-00007.safetensors",
 
202
  "model.layers.28.self_attn.k_proj.weight": "model-00004-of-00007.safetensors",
203
  "model.layers.28.self_attn.o_proj.weight": "model-00004-of-00007.safetensors",
204
  "model.layers.28.self_attn.q_proj.weight": "model-00004-of-00007.safetensors",
205
  "model.layers.28.self_attn.v_proj.weight": "model-00004-of-00007.safetensors",
206
- "model.layers.29.ln1.weight": "model-00004-of-00007.safetensors",
207
- "model.layers.29.ln2.weight": "model-00004-of-00007.safetensors",
208
  "model.layers.29.mlp.down_proj.weight": "model-00004-of-00007.safetensors",
209
  "model.layers.29.mlp.gate_proj.weight": "model-00004-of-00007.safetensors",
210
  "model.layers.29.mlp.up_proj.weight": "model-00004-of-00007.safetensors",
 
211
  "model.layers.29.self_attn.k_proj.weight": "model-00004-of-00007.safetensors",
212
  "model.layers.29.self_attn.o_proj.weight": "model-00004-of-00007.safetensors",
213
  "model.layers.29.self_attn.q_proj.weight": "model-00004-of-00007.safetensors",
214
  "model.layers.29.self_attn.v_proj.weight": "model-00004-of-00007.safetensors",
215
- "model.layers.3.ln1.weight": "model-00001-of-00007.safetensors",
216
- "model.layers.3.ln2.weight": "model-00001-of-00007.safetensors",
217
  "model.layers.3.mlp.down_proj.weight": "model-00001-of-00007.safetensors",
218
  "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00007.safetensors",
219
  "model.layers.3.mlp.up_proj.weight": "model-00001-of-00007.safetensors",
 
220
  "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00007.safetensors",
221
  "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00007.safetensors",
222
  "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00007.safetensors",
223
  "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00007.safetensors",
224
- "model.layers.30.ln1.weight": "model-00004-of-00007.safetensors",
225
- "model.layers.30.ln2.weight": "model-00004-of-00007.safetensors",
226
  "model.layers.30.mlp.down_proj.weight": "model-00004-of-00007.safetensors",
227
  "model.layers.30.mlp.gate_proj.weight": "model-00004-of-00007.safetensors",
228
  "model.layers.30.mlp.up_proj.weight": "model-00004-of-00007.safetensors",
 
229
  "model.layers.30.self_attn.k_proj.weight": "model-00004-of-00007.safetensors",
230
  "model.layers.30.self_attn.o_proj.weight": "model-00004-of-00007.safetensors",
231
  "model.layers.30.self_attn.q_proj.weight": "model-00004-of-00007.safetensors",
232
  "model.layers.30.self_attn.v_proj.weight": "model-00004-of-00007.safetensors",
233
- "model.layers.31.ln1.weight": "model-00004-of-00007.safetensors",
234
- "model.layers.31.ln2.weight": "model-00004-of-00007.safetensors",
235
  "model.layers.31.mlp.down_proj.weight": "model-00004-of-00007.safetensors",
236
  "model.layers.31.mlp.gate_proj.weight": "model-00004-of-00007.safetensors",
237
  "model.layers.31.mlp.up_proj.weight": "model-00004-of-00007.safetensors",
 
238
  "model.layers.31.self_attn.k_proj.weight": "model-00004-of-00007.safetensors",
239
  "model.layers.31.self_attn.o_proj.weight": "model-00004-of-00007.safetensors",
240
  "model.layers.31.self_attn.q_proj.weight": "model-00004-of-00007.safetensors",
241
  "model.layers.31.self_attn.v_proj.weight": "model-00004-of-00007.safetensors",
242
- "model.layers.32.ln1.weight": "model-00004-of-00007.safetensors",
243
- "model.layers.32.ln2.weight": "model-00004-of-00007.safetensors",
244
  "model.layers.32.mlp.down_proj.weight": "model-00004-of-00007.safetensors",
245
  "model.layers.32.mlp.gate_proj.weight": "model-00004-of-00007.safetensors",
246
  "model.layers.32.mlp.up_proj.weight": "model-00004-of-00007.safetensors",
 
247
  "model.layers.32.self_attn.k_proj.weight": "model-00004-of-00007.safetensors",
248
  "model.layers.32.self_attn.o_proj.weight": "model-00004-of-00007.safetensors",
249
  "model.layers.32.self_attn.q_proj.weight": "model-00004-of-00007.safetensors",
250
  "model.layers.32.self_attn.v_proj.weight": "model-00004-of-00007.safetensors",
251
- "model.layers.33.ln1.weight": "model-00004-of-00007.safetensors",
252
- "model.layers.33.ln2.weight": "model-00004-of-00007.safetensors",
253
  "model.layers.33.mlp.down_proj.weight": "model-00004-of-00007.safetensors",
254
  "model.layers.33.mlp.gate_proj.weight": "model-00004-of-00007.safetensors",
255
  "model.layers.33.mlp.up_proj.weight": "model-00004-of-00007.safetensors",
 
256
  "model.layers.33.self_attn.k_proj.weight": "model-00004-of-00007.safetensors",
257
  "model.layers.33.self_attn.o_proj.weight": "model-00004-of-00007.safetensors",
258
  "model.layers.33.self_attn.q_proj.weight": "model-00004-of-00007.safetensors",
259
  "model.layers.33.self_attn.v_proj.weight": "model-00004-of-00007.safetensors",
260
- "model.layers.34.ln1.weight": "model-00005-of-00007.safetensors",
261
- "model.layers.34.ln2.weight": "model-00005-of-00007.safetensors",
262
  "model.layers.34.mlp.down_proj.weight": "model-00005-of-00007.safetensors",
263
  "model.layers.34.mlp.gate_proj.weight": "model-00004-of-00007.safetensors",
264
  "model.layers.34.mlp.up_proj.weight": "model-00005-of-00007.safetensors",
 
265
  "model.layers.34.self_attn.k_proj.weight": "model-00004-of-00007.safetensors",
266
  "model.layers.34.self_attn.o_proj.weight": "model-00004-of-00007.safetensors",
267
  "model.layers.34.self_attn.q_proj.weight": "model-00004-of-00007.safetensors",
268
  "model.layers.34.self_attn.v_proj.weight": "model-00004-of-00007.safetensors",
269
- "model.layers.35.ln1.weight": "model-00005-of-00007.safetensors",
270
- "model.layers.35.ln2.weight": "model-00005-of-00007.safetensors",
271
  "model.layers.35.mlp.down_proj.weight": "model-00005-of-00007.safetensors",
272
  "model.layers.35.mlp.gate_proj.weight": "model-00005-of-00007.safetensors",
273
  "model.layers.35.mlp.up_proj.weight": "model-00005-of-00007.safetensors",
 
274
  "model.layers.35.self_attn.k_proj.weight": "model-00005-of-00007.safetensors",
275
  "model.layers.35.self_attn.o_proj.weight": "model-00005-of-00007.safetensors",
276
  "model.layers.35.self_attn.q_proj.weight": "model-00005-of-00007.safetensors",
277
  "model.layers.35.self_attn.v_proj.weight": "model-00005-of-00007.safetensors",
278
- "model.layers.36.ln1.weight": "model-00005-of-00007.safetensors",
279
- "model.layers.36.ln2.weight": "model-00005-of-00007.safetensors",
280
  "model.layers.36.mlp.down_proj.weight": "model-00005-of-00007.safetensors",
281
  "model.layers.36.mlp.gate_proj.weight": "model-00005-of-00007.safetensors",
282
  "model.layers.36.mlp.up_proj.weight": "model-00005-of-00007.safetensors",
 
283
  "model.layers.36.self_attn.k_proj.weight": "model-00005-of-00007.safetensors",
284
  "model.layers.36.self_attn.o_proj.weight": "model-00005-of-00007.safetensors",
285
  "model.layers.36.self_attn.q_proj.weight": "model-00005-of-00007.safetensors",
286
  "model.layers.36.self_attn.v_proj.weight": "model-00005-of-00007.safetensors",
287
- "model.layers.37.ln1.weight": "model-00005-of-00007.safetensors",
288
- "model.layers.37.ln2.weight": "model-00005-of-00007.safetensors",
289
  "model.layers.37.mlp.down_proj.weight": "model-00005-of-00007.safetensors",
290
  "model.layers.37.mlp.gate_proj.weight": "model-00005-of-00007.safetensors",
291
  "model.layers.37.mlp.up_proj.weight": "model-00005-of-00007.safetensors",
 
292
  "model.layers.37.self_attn.k_proj.weight": "model-00005-of-00007.safetensors",
293
  "model.layers.37.self_attn.o_proj.weight": "model-00005-of-00007.safetensors",
294
  "model.layers.37.self_attn.q_proj.weight": "model-00005-of-00007.safetensors",
295
  "model.layers.37.self_attn.v_proj.weight": "model-00005-of-00007.safetensors",
296
- "model.layers.38.ln1.weight": "model-00005-of-00007.safetensors",
297
- "model.layers.38.ln2.weight": "model-00005-of-00007.safetensors",
298
  "model.layers.38.mlp.down_proj.weight": "model-00005-of-00007.safetensors",
299
  "model.layers.38.mlp.gate_proj.weight": "model-00005-of-00007.safetensors",
300
  "model.layers.38.mlp.up_proj.weight": "model-00005-of-00007.safetensors",
 
301
  "model.layers.38.self_attn.k_proj.weight": "model-00005-of-00007.safetensors",
302
  "model.layers.38.self_attn.o_proj.weight": "model-00005-of-00007.safetensors",
303
  "model.layers.38.self_attn.q_proj.weight": "model-00005-of-00007.safetensors",
304
  "model.layers.38.self_attn.v_proj.weight": "model-00005-of-00007.safetensors",
305
- "model.layers.39.ln1.weight": "model-00005-of-00007.safetensors",
306
- "model.layers.39.ln2.weight": "model-00005-of-00007.safetensors",
307
  "model.layers.39.mlp.down_proj.weight": "model-00005-of-00007.safetensors",
308
  "model.layers.39.mlp.gate_proj.weight": "model-00005-of-00007.safetensors",
309
  "model.layers.39.mlp.up_proj.weight": "model-00005-of-00007.safetensors",
 
310
  "model.layers.39.self_attn.k_proj.weight": "model-00005-of-00007.safetensors",
311
  "model.layers.39.self_attn.o_proj.weight": "model-00005-of-00007.safetensors",
312
  "model.layers.39.self_attn.q_proj.weight": "model-00005-of-00007.safetensors",
313
  "model.layers.39.self_attn.v_proj.weight": "model-00005-of-00007.safetensors",
314
- "model.layers.4.ln1.weight": "model-00001-of-00007.safetensors",
315
- "model.layers.4.ln2.weight": "model-00001-of-00007.safetensors",
316
  "model.layers.4.mlp.down_proj.weight": "model-00001-of-00007.safetensors",
317
  "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00007.safetensors",
318
  "model.layers.4.mlp.up_proj.weight": "model-00001-of-00007.safetensors",
 
319
  "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00007.safetensors",
320
  "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00007.safetensors",
321
  "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00007.safetensors",
322
  "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00007.safetensors",
323
- "model.layers.40.ln1.weight": "model-00005-of-00007.safetensors",
324
- "model.layers.40.ln2.weight": "model-00005-of-00007.safetensors",
325
  "model.layers.40.mlp.down_proj.weight": "model-00005-of-00007.safetensors",
326
  "model.layers.40.mlp.gate_proj.weight": "model-00005-of-00007.safetensors",
327
  "model.layers.40.mlp.up_proj.weight": "model-00005-of-00007.safetensors",
 
328
  "model.layers.40.self_attn.k_proj.weight": "model-00005-of-00007.safetensors",
329
  "model.layers.40.self_attn.o_proj.weight": "model-00005-of-00007.safetensors",
330
  "model.layers.40.self_attn.q_proj.weight": "model-00005-of-00007.safetensors",
331
  "model.layers.40.self_attn.v_proj.weight": "model-00005-of-00007.safetensors",
332
- "model.layers.41.ln1.weight": "model-00005-of-00007.safetensors",
333
- "model.layers.41.ln2.weight": "model-00005-of-00007.safetensors",
334
  "model.layers.41.mlp.down_proj.weight": "model-00005-of-00007.safetensors",
335
  "model.layers.41.mlp.gate_proj.weight": "model-00005-of-00007.safetensors",
336
  "model.layers.41.mlp.up_proj.weight": "model-00005-of-00007.safetensors",
 
337
  "model.layers.41.self_attn.k_proj.weight": "model-00005-of-00007.safetensors",
338
  "model.layers.41.self_attn.o_proj.weight": "model-00005-of-00007.safetensors",
339
  "model.layers.41.self_attn.q_proj.weight": "model-00005-of-00007.safetensors",
340
  "model.layers.41.self_attn.v_proj.weight": "model-00005-of-00007.safetensors",
341
- "model.layers.42.ln1.weight": "model-00005-of-00007.safetensors",
342
- "model.layers.42.ln2.weight": "model-00005-of-00007.safetensors",
343
  "model.layers.42.mlp.down_proj.weight": "model-00005-of-00007.safetensors",
344
  "model.layers.42.mlp.gate_proj.weight": "model-00005-of-00007.safetensors",
345
  "model.layers.42.mlp.up_proj.weight": "model-00005-of-00007.safetensors",
 
346
  "model.layers.42.self_attn.k_proj.weight": "model-00005-of-00007.safetensors",
347
  "model.layers.42.self_attn.o_proj.weight": "model-00005-of-00007.safetensors",
348
  "model.layers.42.self_attn.q_proj.weight": "model-00005-of-00007.safetensors",
349
  "model.layers.42.self_attn.v_proj.weight": "model-00005-of-00007.safetensors",
350
- "model.layers.43.ln1.weight": "model-00006-of-00007.safetensors",
351
- "model.layers.43.ln2.weight": "model-00006-of-00007.safetensors",
352
  "model.layers.43.mlp.down_proj.weight": "model-00006-of-00007.safetensors",
353
  "model.layers.43.mlp.gate_proj.weight": "model-00006-of-00007.safetensors",
354
  "model.layers.43.mlp.up_proj.weight": "model-00006-of-00007.safetensors",
 
355
  "model.layers.43.self_attn.k_proj.weight": "model-00005-of-00007.safetensors",
356
  "model.layers.43.self_attn.o_proj.weight": "model-00005-of-00007.safetensors",
357
  "model.layers.43.self_attn.q_proj.weight": "model-00005-of-00007.safetensors",
358
  "model.layers.43.self_attn.v_proj.weight": "model-00005-of-00007.safetensors",
359
- "model.layers.44.ln1.weight": "model-00006-of-00007.safetensors",
360
- "model.layers.44.ln2.weight": "model-00006-of-00007.safetensors",
361
  "model.layers.44.mlp.down_proj.weight": "model-00006-of-00007.safetensors",
362
  "model.layers.44.mlp.gate_proj.weight": "model-00006-of-00007.safetensors",
363
  "model.layers.44.mlp.up_proj.weight": "model-00006-of-00007.safetensors",
 
364
  "model.layers.44.self_attn.k_proj.weight": "model-00006-of-00007.safetensors",
365
  "model.layers.44.self_attn.o_proj.weight": "model-00006-of-00007.safetensors",
366
  "model.layers.44.self_attn.q_proj.weight": "model-00006-of-00007.safetensors",
367
  "model.layers.44.self_attn.v_proj.weight": "model-00006-of-00007.safetensors",
368
- "model.layers.45.ln1.weight": "model-00006-of-00007.safetensors",
369
- "model.layers.45.ln2.weight": "model-00006-of-00007.safetensors",
370
  "model.layers.45.mlp.down_proj.weight": "model-00006-of-00007.safetensors",
371
  "model.layers.45.mlp.gate_proj.weight": "model-00006-of-00007.safetensors",
372
  "model.layers.45.mlp.up_proj.weight": "model-00006-of-00007.safetensors",
 
373
  "model.layers.45.self_attn.k_proj.weight": "model-00006-of-00007.safetensors",
374
  "model.layers.45.self_attn.o_proj.weight": "model-00006-of-00007.safetensors",
375
  "model.layers.45.self_attn.q_proj.weight": "model-00006-of-00007.safetensors",
376
  "model.layers.45.self_attn.v_proj.weight": "model-00006-of-00007.safetensors",
377
- "model.layers.46.ln1.weight": "model-00006-of-00007.safetensors",
378
- "model.layers.46.ln2.weight": "model-00006-of-00007.safetensors",
379
  "model.layers.46.mlp.down_proj.weight": "model-00006-of-00007.safetensors",
380
  "model.layers.46.mlp.gate_proj.weight": "model-00006-of-00007.safetensors",
381
  "model.layers.46.mlp.up_proj.weight": "model-00006-of-00007.safetensors",
 
382
  "model.layers.46.self_attn.k_proj.weight": "model-00006-of-00007.safetensors",
383
  "model.layers.46.self_attn.o_proj.weight": "model-00006-of-00007.safetensors",
384
  "model.layers.46.self_attn.q_proj.weight": "model-00006-of-00007.safetensors",
385
  "model.layers.46.self_attn.v_proj.weight": "model-00006-of-00007.safetensors",
386
- "model.layers.47.ln1.weight": "model-00006-of-00007.safetensors",
387
- "model.layers.47.ln2.weight": "model-00006-of-00007.safetensors",
388
  "model.layers.47.mlp.down_proj.weight": "model-00006-of-00007.safetensors",
389
  "model.layers.47.mlp.gate_proj.weight": "model-00006-of-00007.safetensors",
390
  "model.layers.47.mlp.up_proj.weight": "model-00006-of-00007.safetensors",
 
391
  "model.layers.47.self_attn.k_proj.weight": "model-00006-of-00007.safetensors",
392
  "model.layers.47.self_attn.o_proj.weight": "model-00006-of-00007.safetensors",
393
  "model.layers.47.self_attn.q_proj.weight": "model-00006-of-00007.safetensors",
394
  "model.layers.47.self_attn.v_proj.weight": "model-00006-of-00007.safetensors",
395
- "model.layers.48.ln1.weight": "model-00006-of-00007.safetensors",
396
- "model.layers.48.ln2.weight": "model-00006-of-00007.safetensors",
397
  "model.layers.48.mlp.down_proj.weight": "model-00006-of-00007.safetensors",
398
  "model.layers.48.mlp.gate_proj.weight": "model-00006-of-00007.safetensors",
399
  "model.layers.48.mlp.up_proj.weight": "model-00006-of-00007.safetensors",
 
400
  "model.layers.48.self_attn.k_proj.weight": "model-00006-of-00007.safetensors",
401
  "model.layers.48.self_attn.o_proj.weight": "model-00006-of-00007.safetensors",
402
  "model.layers.48.self_attn.q_proj.weight": "model-00006-of-00007.safetensors",
403
  "model.layers.48.self_attn.v_proj.weight": "model-00006-of-00007.safetensors",
404
- "model.layers.49.ln1.weight": "model-00006-of-00007.safetensors",
405
- "model.layers.49.ln2.weight": "model-00006-of-00007.safetensors",
406
  "model.layers.49.mlp.down_proj.weight": "model-00006-of-00007.safetensors",
407
  "model.layers.49.mlp.gate_proj.weight": "model-00006-of-00007.safetensors",
408
  "model.layers.49.mlp.up_proj.weight": "model-00006-of-00007.safetensors",
 
409
  "model.layers.49.self_attn.k_proj.weight": "model-00006-of-00007.safetensors",
410
  "model.layers.49.self_attn.o_proj.weight": "model-00006-of-00007.safetensors",
411
  "model.layers.49.self_attn.q_proj.weight": "model-00006-of-00007.safetensors",
412
  "model.layers.49.self_attn.v_proj.weight": "model-00006-of-00007.safetensors",
413
- "model.layers.5.ln1.weight": "model-00001-of-00007.safetensors",
414
- "model.layers.5.ln2.weight": "model-00001-of-00007.safetensors",
415
  "model.layers.5.mlp.down_proj.weight": "model-00001-of-00007.safetensors",
416
  "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00007.safetensors",
417
  "model.layers.5.mlp.up_proj.weight": "model-00001-of-00007.safetensors",
 
418
  "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00007.safetensors",
419
  "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00007.safetensors",
420
  "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00007.safetensors",
421
  "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00007.safetensors",
422
- "model.layers.50.ln1.weight": "model-00006-of-00007.safetensors",
423
- "model.layers.50.ln2.weight": "model-00006-of-00007.safetensors",
424
  "model.layers.50.mlp.down_proj.weight": "model-00006-of-00007.safetensors",
425
  "model.layers.50.mlp.gate_proj.weight": "model-00006-of-00007.safetensors",
426
  "model.layers.50.mlp.up_proj.weight": "model-00006-of-00007.safetensors",
 
427
  "model.layers.50.self_attn.k_proj.weight": "model-00006-of-00007.safetensors",
428
  "model.layers.50.self_attn.o_proj.weight": "model-00006-of-00007.safetensors",
429
  "model.layers.50.self_attn.q_proj.weight": "model-00006-of-00007.safetensors",
430
  "model.layers.50.self_attn.v_proj.weight": "model-00006-of-00007.safetensors",
431
- "model.layers.51.ln1.weight": "model-00006-of-00007.safetensors",
432
- "model.layers.51.ln2.weight": "model-00006-of-00007.safetensors",
433
  "model.layers.51.mlp.down_proj.weight": "model-00006-of-00007.safetensors",
434
  "model.layers.51.mlp.gate_proj.weight": "model-00006-of-00007.safetensors",
435
  "model.layers.51.mlp.up_proj.weight": "model-00006-of-00007.safetensors",
 
436
  "model.layers.51.self_attn.k_proj.weight": "model-00006-of-00007.safetensors",
437
  "model.layers.51.self_attn.o_proj.weight": "model-00006-of-00007.safetensors",
438
  "model.layers.51.self_attn.q_proj.weight": "model-00006-of-00007.safetensors",
439
  "model.layers.51.self_attn.v_proj.weight": "model-00006-of-00007.safetensors",
440
- "model.layers.52.ln1.weight": "model-00007-of-00007.safetensors",
441
- "model.layers.52.ln2.weight": "model-00007-of-00007.safetensors",
442
  "model.layers.52.mlp.down_proj.weight": "model-00007-of-00007.safetensors",
443
  "model.layers.52.mlp.gate_proj.weight": "model-00007-of-00007.safetensors",
444
  "model.layers.52.mlp.up_proj.weight": "model-00007-of-00007.safetensors",
 
445
  "model.layers.52.self_attn.k_proj.weight": "model-00006-of-00007.safetensors",
446
  "model.layers.52.self_attn.o_proj.weight": "model-00007-of-00007.safetensors",
447
  "model.layers.52.self_attn.q_proj.weight": "model-00006-of-00007.safetensors",
448
  "model.layers.52.self_attn.v_proj.weight": "model-00006-of-00007.safetensors",
449
- "model.layers.53.ln1.weight": "model-00007-of-00007.safetensors",
450
- "model.layers.53.ln2.weight": "model-00007-of-00007.safetensors",
451
  "model.layers.53.mlp.down_proj.weight": "model-00007-of-00007.safetensors",
452
  "model.layers.53.mlp.gate_proj.weight": "model-00007-of-00007.safetensors",
453
  "model.layers.53.mlp.up_proj.weight": "model-00007-of-00007.safetensors",
 
454
  "model.layers.53.self_attn.k_proj.weight": "model-00007-of-00007.safetensors",
455
  "model.layers.53.self_attn.o_proj.weight": "model-00007-of-00007.safetensors",
456
  "model.layers.53.self_attn.q_proj.weight": "model-00007-of-00007.safetensors",
457
  "model.layers.53.self_attn.v_proj.weight": "model-00007-of-00007.safetensors",
458
- "model.layers.54.ln1.weight": "model-00007-of-00007.safetensors",
459
- "model.layers.54.ln2.weight": "model-00007-of-00007.safetensors",
460
  "model.layers.54.mlp.down_proj.weight": "model-00007-of-00007.safetensors",
461
  "model.layers.54.mlp.gate_proj.weight": "model-00007-of-00007.safetensors",
462
  "model.layers.54.mlp.up_proj.weight": "model-00007-of-00007.safetensors",
 
463
  "model.layers.54.self_attn.k_proj.weight": "model-00007-of-00007.safetensors",
464
  "model.layers.54.self_attn.o_proj.weight": "model-00007-of-00007.safetensors",
465
  "model.layers.54.self_attn.q_proj.weight": "model-00007-of-00007.safetensors",
466
  "model.layers.54.self_attn.v_proj.weight": "model-00007-of-00007.safetensors",
467
- "model.layers.55.ln1.weight": "model-00007-of-00007.safetensors",
468
- "model.layers.55.ln2.weight": "model-00007-of-00007.safetensors",
469
  "model.layers.55.mlp.down_proj.weight": "model-00007-of-00007.safetensors",
470
  "model.layers.55.mlp.gate_proj.weight": "model-00007-of-00007.safetensors",
471
  "model.layers.55.mlp.up_proj.weight": "model-00007-of-00007.safetensors",
 
472
  "model.layers.55.self_attn.k_proj.weight": "model-00007-of-00007.safetensors",
473
  "model.layers.55.self_attn.o_proj.weight": "model-00007-of-00007.safetensors",
474
  "model.layers.55.self_attn.q_proj.weight": "model-00007-of-00007.safetensors",
475
  "model.layers.55.self_attn.v_proj.weight": "model-00007-of-00007.safetensors",
476
- "model.layers.56.ln1.weight": "model-00007-of-00007.safetensors",
477
- "model.layers.56.ln2.weight": "model-00007-of-00007.safetensors",
478
  "model.layers.56.mlp.down_proj.weight": "model-00007-of-00007.safetensors",
479
  "model.layers.56.mlp.gate_proj.weight": "model-00007-of-00007.safetensors",
480
  "model.layers.56.mlp.up_proj.weight": "model-00007-of-00007.safetensors",
 
481
  "model.layers.56.self_attn.k_proj.weight": "model-00007-of-00007.safetensors",
482
  "model.layers.56.self_attn.o_proj.weight": "model-00007-of-00007.safetensors",
483
  "model.layers.56.self_attn.q_proj.weight": "model-00007-of-00007.safetensors",
484
  "model.layers.56.self_attn.v_proj.weight": "model-00007-of-00007.safetensors",
485
- "model.layers.57.ln1.weight": "model-00007-of-00007.safetensors",
486
- "model.layers.57.ln2.weight": "model-00007-of-00007.safetensors",
487
  "model.layers.57.mlp.down_proj.weight": "model-00007-of-00007.safetensors",
488
  "model.layers.57.mlp.gate_proj.weight": "model-00007-of-00007.safetensors",
489
  "model.layers.57.mlp.up_proj.weight": "model-00007-of-00007.safetensors",
 
490
  "model.layers.57.self_attn.k_proj.weight": "model-00007-of-00007.safetensors",
491
  "model.layers.57.self_attn.o_proj.weight": "model-00007-of-00007.safetensors",
492
  "model.layers.57.self_attn.q_proj.weight": "model-00007-of-00007.safetensors",
493
  "model.layers.57.self_attn.v_proj.weight": "model-00007-of-00007.safetensors",
494
- "model.layers.58.ln1.weight": "model-00007-of-00007.safetensors",
495
- "model.layers.58.ln2.weight": "model-00007-of-00007.safetensors",
496
  "model.layers.58.mlp.down_proj.weight": "model-00007-of-00007.safetensors",
497
  "model.layers.58.mlp.gate_proj.weight": "model-00007-of-00007.safetensors",
498
  "model.layers.58.mlp.up_proj.weight": "model-00007-of-00007.safetensors",
 
499
  "model.layers.58.self_attn.k_proj.weight": "model-00007-of-00007.safetensors",
500
  "model.layers.58.self_attn.o_proj.weight": "model-00007-of-00007.safetensors",
501
  "model.layers.58.self_attn.q_proj.weight": "model-00007-of-00007.safetensors",
502
  "model.layers.58.self_attn.v_proj.weight": "model-00007-of-00007.safetensors",
503
- "model.layers.59.ln1.weight": "model-00007-of-00007.safetensors",
504
- "model.layers.59.ln2.weight": "model-00007-of-00007.safetensors",
505
  "model.layers.59.mlp.down_proj.weight": "model-00007-of-00007.safetensors",
506
  "model.layers.59.mlp.gate_proj.weight": "model-00007-of-00007.safetensors",
507
  "model.layers.59.mlp.up_proj.weight": "model-00007-of-00007.safetensors",
 
508
  "model.layers.59.self_attn.k_proj.weight": "model-00007-of-00007.safetensors",
509
  "model.layers.59.self_attn.o_proj.weight": "model-00007-of-00007.safetensors",
510
  "model.layers.59.self_attn.q_proj.weight": "model-00007-of-00007.safetensors",
511
  "model.layers.59.self_attn.v_proj.weight": "model-00007-of-00007.safetensors",
512
- "model.layers.6.ln1.weight": "model-00001-of-00007.safetensors",
513
- "model.layers.6.ln2.weight": "model-00001-of-00007.safetensors",
514
  "model.layers.6.mlp.down_proj.weight": "model-00001-of-00007.safetensors",
515
  "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00007.safetensors",
516
  "model.layers.6.mlp.up_proj.weight": "model-00001-of-00007.safetensors",
 
517
  "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00007.safetensors",
518
  "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00007.safetensors",
519
  "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00007.safetensors",
520
  "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00007.safetensors",
521
- "model.layers.7.ln1.weight": "model-00001-of-00007.safetensors",
522
- "model.layers.7.ln2.weight": "model-00001-of-00007.safetensors",
523
  "model.layers.7.mlp.down_proj.weight": "model-00001-of-00007.safetensors",
524
  "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00007.safetensors",
525
  "model.layers.7.mlp.up_proj.weight": "model-00001-of-00007.safetensors",
 
526
  "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00007.safetensors",
527
  "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00007.safetensors",
528
  "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00007.safetensors",
529
  "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00007.safetensors",
530
- "model.layers.8.ln1.weight": "model-00002-of-00007.safetensors",
531
- "model.layers.8.ln2.weight": "model-00002-of-00007.safetensors",
532
  "model.layers.8.mlp.down_proj.weight": "model-00002-of-00007.safetensors",
533
  "model.layers.8.mlp.gate_proj.weight": "model-00002-of-00007.safetensors",
534
  "model.layers.8.mlp.up_proj.weight": "model-00002-of-00007.safetensors",
 
535
  "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00007.safetensors",
536
  "model.layers.8.self_attn.o_proj.weight": "model-00002-of-00007.safetensors",
537
  "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00007.safetensors",
538
  "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00007.safetensors",
539
- "model.layers.9.ln1.weight": "model-00002-of-00007.safetensors",
540
- "model.layers.9.ln2.weight": "model-00002-of-00007.safetensors",
541
  "model.layers.9.mlp.down_proj.weight": "model-00002-of-00007.safetensors",
542
  "model.layers.9.mlp.gate_proj.weight": "model-00002-of-00007.safetensors",
543
  "model.layers.9.mlp.up_proj.weight": "model-00002-of-00007.safetensors",
 
544
  "model.layers.9.self_attn.k_proj.weight": "model-00002-of-00007.safetensors",
545
  "model.layers.9.self_attn.o_proj.weight": "model-00002-of-00007.safetensors",
546
  "model.layers.9.self_attn.q_proj.weight": "model-00002-of-00007.safetensors",
 
5
  "weight_map": {
6
  "lm_head.weight": "model-00007-of-00007.safetensors",
7
  "model.embed_tokens.weight": "model-00001-of-00007.safetensors",
8
+ "model.layers.0.input_layernorm.weight": "model-00001-of-00007.safetensors",
 
9
  "model.layers.0.mlp.down_proj.weight": "model-00001-of-00007.safetensors",
10
  "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00007.safetensors",
11
  "model.layers.0.mlp.up_proj.weight": "model-00001-of-00007.safetensors",
12
+ "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00007.safetensors",
13
  "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00007.safetensors",
14
  "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00007.safetensors",
15
  "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00007.safetensors",
16
  "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00007.safetensors",
17
+ "model.layers.1.input_layernorm.weight": "model-00001-of-00007.safetensors",
 
18
  "model.layers.1.mlp.down_proj.weight": "model-00001-of-00007.safetensors",
19
  "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00007.safetensors",
20
  "model.layers.1.mlp.up_proj.weight": "model-00001-of-00007.safetensors",
21
+ "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00007.safetensors",
22
  "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00007.safetensors",
23
  "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00007.safetensors",
24
  "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00007.safetensors",
25
  "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00007.safetensors",
26
+ "model.layers.10.input_layernorm.weight": "model-00002-of-00007.safetensors",
 
27
  "model.layers.10.mlp.down_proj.weight": "model-00002-of-00007.safetensors",
28
  "model.layers.10.mlp.gate_proj.weight": "model-00002-of-00007.safetensors",
29
  "model.layers.10.mlp.up_proj.weight": "model-00002-of-00007.safetensors",
30
+ "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00007.safetensors",
31
  "model.layers.10.self_attn.k_proj.weight": "model-00002-of-00007.safetensors",
32
  "model.layers.10.self_attn.o_proj.weight": "model-00002-of-00007.safetensors",
33
  "model.layers.10.self_attn.q_proj.weight": "model-00002-of-00007.safetensors",
34
  "model.layers.10.self_attn.v_proj.weight": "model-00002-of-00007.safetensors",
35
+ "model.layers.11.input_layernorm.weight": "model-00002-of-00007.safetensors",
 
36
  "model.layers.11.mlp.down_proj.weight": "model-00002-of-00007.safetensors",
37
  "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00007.safetensors",
38
  "model.layers.11.mlp.up_proj.weight": "model-00002-of-00007.safetensors",
39
+ "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00007.safetensors",
40
  "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00007.safetensors",
41
  "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00007.safetensors",
42
  "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00007.safetensors",
43
  "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00007.safetensors",
44
+ "model.layers.12.input_layernorm.weight": "model-00002-of-00007.safetensors",
 
45
  "model.layers.12.mlp.down_proj.weight": "model-00002-of-00007.safetensors",
46
  "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00007.safetensors",
47
  "model.layers.12.mlp.up_proj.weight": "model-00002-of-00007.safetensors",
48
+ "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00007.safetensors",
49
  "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00007.safetensors",
50
  "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00007.safetensors",
51
  "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00007.safetensors",
52
  "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00007.safetensors",
53
+ "model.layers.13.input_layernorm.weight": "model-00002-of-00007.safetensors",
 
54
  "model.layers.13.mlp.down_proj.weight": "model-00002-of-00007.safetensors",
55
  "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00007.safetensors",
56
  "model.layers.13.mlp.up_proj.weight": "model-00002-of-00007.safetensors",
57
+ "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00007.safetensors",
58
  "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00007.safetensors",
59
  "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00007.safetensors",
60
  "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00007.safetensors",
61
  "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00007.safetensors",
62
+ "model.layers.14.input_layernorm.weight": "model-00002-of-00007.safetensors",
 
63
  "model.layers.14.mlp.down_proj.weight": "model-00002-of-00007.safetensors",
64
  "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00007.safetensors",
65
  "model.layers.14.mlp.up_proj.weight": "model-00002-of-00007.safetensors",
66
+ "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00007.safetensors",
67
  "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00007.safetensors",
68
  "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00007.safetensors",
69
  "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00007.safetensors",
70
  "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00007.safetensors",
71
+ "model.layers.15.input_layernorm.weight": "model-00002-of-00007.safetensors",
 
72
  "model.layers.15.mlp.down_proj.weight": "model-00002-of-00007.safetensors",
73
  "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00007.safetensors",
74
  "model.layers.15.mlp.up_proj.weight": "model-00002-of-00007.safetensors",
75
+ "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00007.safetensors",
76
  "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00007.safetensors",
77
  "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00007.safetensors",
78
  "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00007.safetensors",
79
  "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00007.safetensors",
80
+ "model.layers.16.input_layernorm.weight": "model-00002-of-00007.safetensors",
 
81
  "model.layers.16.mlp.down_proj.weight": "model-00002-of-00007.safetensors",
82
  "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00007.safetensors",
83
  "model.layers.16.mlp.up_proj.weight": "model-00002-of-00007.safetensors",
84
+ "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00007.safetensors",
85
  "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00007.safetensors",
86
  "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00007.safetensors",
87
  "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00007.safetensors",
88
  "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00007.safetensors",
89
+ "model.layers.17.input_layernorm.weight": "model-00003-of-00007.safetensors",
 
90
  "model.layers.17.mlp.down_proj.weight": "model-00003-of-00007.safetensors",
91
  "model.layers.17.mlp.gate_proj.weight": "model-00003-of-00007.safetensors",
92
  "model.layers.17.mlp.up_proj.weight": "model-00003-of-00007.safetensors",
93
+ "model.layers.17.post_attention_layernorm.weight": "model-00003-of-00007.safetensors",
94
  "model.layers.17.self_attn.k_proj.weight": "model-00003-of-00007.safetensors",
95
  "model.layers.17.self_attn.o_proj.weight": "model-00003-of-00007.safetensors",
96
  "model.layers.17.self_attn.q_proj.weight": "model-00003-of-00007.safetensors",
97
  "model.layers.17.self_attn.v_proj.weight": "model-00003-of-00007.safetensors",
98
+ "model.layers.18.input_layernorm.weight": "model-00003-of-00007.safetensors",
 
99
  "model.layers.18.mlp.down_proj.weight": "model-00003-of-00007.safetensors",
100
  "model.layers.18.mlp.gate_proj.weight": "model-00003-of-00007.safetensors",
101
  "model.layers.18.mlp.up_proj.weight": "model-00003-of-00007.safetensors",
102
+ "model.layers.18.post_attention_layernorm.weight": "model-00003-of-00007.safetensors",
103
  "model.layers.18.self_attn.k_proj.weight": "model-00003-of-00007.safetensors",
104
  "model.layers.18.self_attn.o_proj.weight": "model-00003-of-00007.safetensors",
105
  "model.layers.18.self_attn.q_proj.weight": "model-00003-of-00007.safetensors",
106
  "model.layers.18.self_attn.v_proj.weight": "model-00003-of-00007.safetensors",
107
+ "model.layers.19.input_layernorm.weight": "model-00003-of-00007.safetensors",
 
108
  "model.layers.19.mlp.down_proj.weight": "model-00003-of-00007.safetensors",
109
  "model.layers.19.mlp.gate_proj.weight": "model-00003-of-00007.safetensors",
110
  "model.layers.19.mlp.up_proj.weight": "model-00003-of-00007.safetensors",
111
+ "model.layers.19.post_attention_layernorm.weight": "model-00003-of-00007.safetensors",
112
  "model.layers.19.self_attn.k_proj.weight": "model-00003-of-00007.safetensors",
113
  "model.layers.19.self_attn.o_proj.weight": "model-00003-of-00007.safetensors",
114
  "model.layers.19.self_attn.q_proj.weight": "model-00003-of-00007.safetensors",
115
  "model.layers.19.self_attn.v_proj.weight": "model-00003-of-00007.safetensors",
116
+ "model.layers.2.input_layernorm.weight": "model-00001-of-00007.safetensors",
 
117
  "model.layers.2.mlp.down_proj.weight": "model-00001-of-00007.safetensors",
118
  "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00007.safetensors",
119
  "model.layers.2.mlp.up_proj.weight": "model-00001-of-00007.safetensors",
120
+ "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00007.safetensors",
121
  "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00007.safetensors",
122
  "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00007.safetensors",
123
  "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00007.safetensors",
124
  "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00007.safetensors",
125
+ "model.layers.20.input_layernorm.weight": "model-00003-of-00007.safetensors",
 
126
  "model.layers.20.mlp.down_proj.weight": "model-00003-of-00007.safetensors",
127
  "model.layers.20.mlp.gate_proj.weight": "model-00003-of-00007.safetensors",
128
  "model.layers.20.mlp.up_proj.weight": "model-00003-of-00007.safetensors",
129
+ "model.layers.20.post_attention_layernorm.weight": "model-00003-of-00007.safetensors",
130
  "model.layers.20.self_attn.k_proj.weight": "model-00003-of-00007.safetensors",
131
  "model.layers.20.self_attn.o_proj.weight": "model-00003-of-00007.safetensors",
132
  "model.layers.20.self_attn.q_proj.weight": "model-00003-of-00007.safetensors",
133
  "model.layers.20.self_attn.v_proj.weight": "model-00003-of-00007.safetensors",
134
+ "model.layers.21.input_layernorm.weight": "model-00003-of-00007.safetensors",
 
135
  "model.layers.21.mlp.down_proj.weight": "model-00003-of-00007.safetensors",
136
  "model.layers.21.mlp.gate_proj.weight": "model-00003-of-00007.safetensors",
137
  "model.layers.21.mlp.up_proj.weight": "model-00003-of-00007.safetensors",
138
+ "model.layers.21.post_attention_layernorm.weight": "model-00003-of-00007.safetensors",
139
  "model.layers.21.self_attn.k_proj.weight": "model-00003-of-00007.safetensors",
140
  "model.layers.21.self_attn.o_proj.weight": "model-00003-of-00007.safetensors",
141
  "model.layers.21.self_attn.q_proj.weight": "model-00003-of-00007.safetensors",
142
  "model.layers.21.self_attn.v_proj.weight": "model-00003-of-00007.safetensors",
143
+ "model.layers.22.input_layernorm.weight": "model-00003-of-00007.safetensors",
 
144
  "model.layers.22.mlp.down_proj.weight": "model-00003-of-00007.safetensors",
145
  "model.layers.22.mlp.gate_proj.weight": "model-00003-of-00007.safetensors",
146
  "model.layers.22.mlp.up_proj.weight": "model-00003-of-00007.safetensors",
147
+ "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00007.safetensors",
148
  "model.layers.22.self_attn.k_proj.weight": "model-00003-of-00007.safetensors",
149
  "model.layers.22.self_attn.o_proj.weight": "model-00003-of-00007.safetensors",
150
  "model.layers.22.self_attn.q_proj.weight": "model-00003-of-00007.safetensors",
151
  "model.layers.22.self_attn.v_proj.weight": "model-00003-of-00007.safetensors",
152
+ "model.layers.23.input_layernorm.weight": "model-00003-of-00007.safetensors",
 
153
  "model.layers.23.mlp.down_proj.weight": "model-00003-of-00007.safetensors",
154
  "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00007.safetensors",
155
  "model.layers.23.mlp.up_proj.weight": "model-00003-of-00007.safetensors",
156
+ "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00007.safetensors",
157
  "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00007.safetensors",
158
  "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00007.safetensors",
159
  "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00007.safetensors",
160
  "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00007.safetensors",
161
+ "model.layers.24.input_layernorm.weight": "model-00003-of-00007.safetensors",
 
162
  "model.layers.24.mlp.down_proj.weight": "model-00003-of-00007.safetensors",
163
  "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00007.safetensors",
164
  "model.layers.24.mlp.up_proj.weight": "model-00003-of-00007.safetensors",
165
+ "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00007.safetensors",
166
  "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00007.safetensors",
167
  "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00007.safetensors",
168
  "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00007.safetensors",
169
  "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00007.safetensors",
170
+ "model.layers.25.input_layernorm.weight": "model-00004-of-00007.safetensors",
 
171
  "model.layers.25.mlp.down_proj.weight": "model-00003-of-00007.safetensors",
172
  "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00007.safetensors",
173
  "model.layers.25.mlp.up_proj.weight": "model-00004-of-00007.safetensors",
174
+ "model.layers.25.post_attention_layernorm.weight": "model-00004-of-00007.safetensors",
175
  "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00007.safetensors",
176
  "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00007.safetensors",
177
  "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00007.safetensors",
178
  "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00007.safetensors",
179
+ "model.layers.26.input_layernorm.weight": "model-00004-of-00007.safetensors",
 
180
  "model.layers.26.mlp.down_proj.weight": "model-00004-of-00007.safetensors",
181
  "model.layers.26.mlp.gate_proj.weight": "model-00004-of-00007.safetensors",
182
  "model.layers.26.mlp.up_proj.weight": "model-00004-of-00007.safetensors",
183
+ "model.layers.26.post_attention_layernorm.weight": "model-00004-of-00007.safetensors",
184
  "model.layers.26.self_attn.k_proj.weight": "model-00004-of-00007.safetensors",
185
  "model.layers.26.self_attn.o_proj.weight": "model-00004-of-00007.safetensors",
186
  "model.layers.26.self_attn.q_proj.weight": "model-00004-of-00007.safetensors",
187
  "model.layers.26.self_attn.v_proj.weight": "model-00004-of-00007.safetensors",
188
+ "model.layers.27.input_layernorm.weight": "model-00004-of-00007.safetensors",
 
189
  "model.layers.27.mlp.down_proj.weight": "model-00004-of-00007.safetensors",
190
  "model.layers.27.mlp.gate_proj.weight": "model-00004-of-00007.safetensors",
191
  "model.layers.27.mlp.up_proj.weight": "model-00004-of-00007.safetensors",
192
+ "model.layers.27.post_attention_layernorm.weight": "model-00004-of-00007.safetensors",
193
  "model.layers.27.self_attn.k_proj.weight": "model-00004-of-00007.safetensors",
194
  "model.layers.27.self_attn.o_proj.weight": "model-00004-of-00007.safetensors",
195
  "model.layers.27.self_attn.q_proj.weight": "model-00004-of-00007.safetensors",
196
  "model.layers.27.self_attn.v_proj.weight": "model-00004-of-00007.safetensors",
197
+ "model.layers.28.input_layernorm.weight": "model-00004-of-00007.safetensors",
 
198
  "model.layers.28.mlp.down_proj.weight": "model-00004-of-00007.safetensors",
199
  "model.layers.28.mlp.gate_proj.weight": "model-00004-of-00007.safetensors",
200
  "model.layers.28.mlp.up_proj.weight": "model-00004-of-00007.safetensors",
201
+ "model.layers.28.post_attention_layernorm.weight": "model-00004-of-00007.safetensors",
202
  "model.layers.28.self_attn.k_proj.weight": "model-00004-of-00007.safetensors",
203
  "model.layers.28.self_attn.o_proj.weight": "model-00004-of-00007.safetensors",
204
  "model.layers.28.self_attn.q_proj.weight": "model-00004-of-00007.safetensors",
205
  "model.layers.28.self_attn.v_proj.weight": "model-00004-of-00007.safetensors",
206
+ "model.layers.29.input_layernorm.weight": "model-00004-of-00007.safetensors",
 
207
  "model.layers.29.mlp.down_proj.weight": "model-00004-of-00007.safetensors",
208
  "model.layers.29.mlp.gate_proj.weight": "model-00004-of-00007.safetensors",
209
  "model.layers.29.mlp.up_proj.weight": "model-00004-of-00007.safetensors",
210
+ "model.layers.29.post_attention_layernorm.weight": "model-00004-of-00007.safetensors",
211
  "model.layers.29.self_attn.k_proj.weight": "model-00004-of-00007.safetensors",
212
  "model.layers.29.self_attn.o_proj.weight": "model-00004-of-00007.safetensors",
213
  "model.layers.29.self_attn.q_proj.weight": "model-00004-of-00007.safetensors",
214
  "model.layers.29.self_attn.v_proj.weight": "model-00004-of-00007.safetensors",
215
+ "model.layers.3.input_layernorm.weight": "model-00001-of-00007.safetensors",
 
216
  "model.layers.3.mlp.down_proj.weight": "model-00001-of-00007.safetensors",
217
  "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00007.safetensors",
218
  "model.layers.3.mlp.up_proj.weight": "model-00001-of-00007.safetensors",
219
+ "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00007.safetensors",
220
  "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00007.safetensors",
221
  "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00007.safetensors",
222
  "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00007.safetensors",
223
  "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00007.safetensors",
224
+ "model.layers.30.input_layernorm.weight": "model-00004-of-00007.safetensors",
 
225
  "model.layers.30.mlp.down_proj.weight": "model-00004-of-00007.safetensors",
226
  "model.layers.30.mlp.gate_proj.weight": "model-00004-of-00007.safetensors",
227
  "model.layers.30.mlp.up_proj.weight": "model-00004-of-00007.safetensors",
228
+ "model.layers.30.post_attention_layernorm.weight": "model-00004-of-00007.safetensors",
229
  "model.layers.30.self_attn.k_proj.weight": "model-00004-of-00007.safetensors",
230
  "model.layers.30.self_attn.o_proj.weight": "model-00004-of-00007.safetensors",
231
  "model.layers.30.self_attn.q_proj.weight": "model-00004-of-00007.safetensors",
232
  "model.layers.30.self_attn.v_proj.weight": "model-00004-of-00007.safetensors",
233
+ "model.layers.31.input_layernorm.weight": "model-00004-of-00007.safetensors",
 
234
  "model.layers.31.mlp.down_proj.weight": "model-00004-of-00007.safetensors",
235
  "model.layers.31.mlp.gate_proj.weight": "model-00004-of-00007.safetensors",
236
  "model.layers.31.mlp.up_proj.weight": "model-00004-of-00007.safetensors",
237
+ "model.layers.31.post_attention_layernorm.weight": "model-00004-of-00007.safetensors",
238
  "model.layers.31.self_attn.k_proj.weight": "model-00004-of-00007.safetensors",
239
  "model.layers.31.self_attn.o_proj.weight": "model-00004-of-00007.safetensors",
240
  "model.layers.31.self_attn.q_proj.weight": "model-00004-of-00007.safetensors",
241
  "model.layers.31.self_attn.v_proj.weight": "model-00004-of-00007.safetensors",
242
+ "model.layers.32.input_layernorm.weight": "model-00004-of-00007.safetensors",
 
243
  "model.layers.32.mlp.down_proj.weight": "model-00004-of-00007.safetensors",
244
  "model.layers.32.mlp.gate_proj.weight": "model-00004-of-00007.safetensors",
245
  "model.layers.32.mlp.up_proj.weight": "model-00004-of-00007.safetensors",
246
+ "model.layers.32.post_attention_layernorm.weight": "model-00004-of-00007.safetensors",
247
  "model.layers.32.self_attn.k_proj.weight": "model-00004-of-00007.safetensors",
248
  "model.layers.32.self_attn.o_proj.weight": "model-00004-of-00007.safetensors",
249
  "model.layers.32.self_attn.q_proj.weight": "model-00004-of-00007.safetensors",
250
  "model.layers.32.self_attn.v_proj.weight": "model-00004-of-00007.safetensors",
251
+ "model.layers.33.input_layernorm.weight": "model-00004-of-00007.safetensors",
 
252
  "model.layers.33.mlp.down_proj.weight": "model-00004-of-00007.safetensors",
253
  "model.layers.33.mlp.gate_proj.weight": "model-00004-of-00007.safetensors",
254
  "model.layers.33.mlp.up_proj.weight": "model-00004-of-00007.safetensors",
255
+ "model.layers.33.post_attention_layernorm.weight": "model-00004-of-00007.safetensors",
256
  "model.layers.33.self_attn.k_proj.weight": "model-00004-of-00007.safetensors",
257
  "model.layers.33.self_attn.o_proj.weight": "model-00004-of-00007.safetensors",
258
  "model.layers.33.self_attn.q_proj.weight": "model-00004-of-00007.safetensors",
259
  "model.layers.33.self_attn.v_proj.weight": "model-00004-of-00007.safetensors",
260
+ "model.layers.34.input_layernorm.weight": "model-00005-of-00007.safetensors",
 
261
  "model.layers.34.mlp.down_proj.weight": "model-00005-of-00007.safetensors",
262
  "model.layers.34.mlp.gate_proj.weight": "model-00004-of-00007.safetensors",
263
  "model.layers.34.mlp.up_proj.weight": "model-00005-of-00007.safetensors",
264
+ "model.layers.34.post_attention_layernorm.weight": "model-00005-of-00007.safetensors",
265
  "model.layers.34.self_attn.k_proj.weight": "model-00004-of-00007.safetensors",
266
  "model.layers.34.self_attn.o_proj.weight": "model-00004-of-00007.safetensors",
267
  "model.layers.34.self_attn.q_proj.weight": "model-00004-of-00007.safetensors",
268
  "model.layers.34.self_attn.v_proj.weight": "model-00004-of-00007.safetensors",
269
+ "model.layers.35.input_layernorm.weight": "model-00005-of-00007.safetensors",
 
270
  "model.layers.35.mlp.down_proj.weight": "model-00005-of-00007.safetensors",
271
  "model.layers.35.mlp.gate_proj.weight": "model-00005-of-00007.safetensors",
272
  "model.layers.35.mlp.up_proj.weight": "model-00005-of-00007.safetensors",
273
+ "model.layers.35.post_attention_layernorm.weight": "model-00005-of-00007.safetensors",
274
  "model.layers.35.self_attn.k_proj.weight": "model-00005-of-00007.safetensors",
275
  "model.layers.35.self_attn.o_proj.weight": "model-00005-of-00007.safetensors",
276
  "model.layers.35.self_attn.q_proj.weight": "model-00005-of-00007.safetensors",
277
  "model.layers.35.self_attn.v_proj.weight": "model-00005-of-00007.safetensors",
278
+ "model.layers.36.input_layernorm.weight": "model-00005-of-00007.safetensors",
 
279
  "model.layers.36.mlp.down_proj.weight": "model-00005-of-00007.safetensors",
280
  "model.layers.36.mlp.gate_proj.weight": "model-00005-of-00007.safetensors",
281
  "model.layers.36.mlp.up_proj.weight": "model-00005-of-00007.safetensors",
282
+ "model.layers.36.post_attention_layernorm.weight": "model-00005-of-00007.safetensors",
283
  "model.layers.36.self_attn.k_proj.weight": "model-00005-of-00007.safetensors",
284
  "model.layers.36.self_attn.o_proj.weight": "model-00005-of-00007.safetensors",
285
  "model.layers.36.self_attn.q_proj.weight": "model-00005-of-00007.safetensors",
286
  "model.layers.36.self_attn.v_proj.weight": "model-00005-of-00007.safetensors",
287
+ "model.layers.37.input_layernorm.weight": "model-00005-of-00007.safetensors",
 
288
  "model.layers.37.mlp.down_proj.weight": "model-00005-of-00007.safetensors",
289
  "model.layers.37.mlp.gate_proj.weight": "model-00005-of-00007.safetensors",
290
  "model.layers.37.mlp.up_proj.weight": "model-00005-of-00007.safetensors",
291
+ "model.layers.37.post_attention_layernorm.weight": "model-00005-of-00007.safetensors",
292
  "model.layers.37.self_attn.k_proj.weight": "model-00005-of-00007.safetensors",
293
  "model.layers.37.self_attn.o_proj.weight": "model-00005-of-00007.safetensors",
294
  "model.layers.37.self_attn.q_proj.weight": "model-00005-of-00007.safetensors",
295
  "model.layers.37.self_attn.v_proj.weight": "model-00005-of-00007.safetensors",
296
+ "model.layers.38.input_layernorm.weight": "model-00005-of-00007.safetensors",
 
297
  "model.layers.38.mlp.down_proj.weight": "model-00005-of-00007.safetensors",
298
  "model.layers.38.mlp.gate_proj.weight": "model-00005-of-00007.safetensors",
299
  "model.layers.38.mlp.up_proj.weight": "model-00005-of-00007.safetensors",
300
+ "model.layers.38.post_attention_layernorm.weight": "model-00005-of-00007.safetensors",
301
  "model.layers.38.self_attn.k_proj.weight": "model-00005-of-00007.safetensors",
302
  "model.layers.38.self_attn.o_proj.weight": "model-00005-of-00007.safetensors",
303
  "model.layers.38.self_attn.q_proj.weight": "model-00005-of-00007.safetensors",
304
  "model.layers.38.self_attn.v_proj.weight": "model-00005-of-00007.safetensors",
305
+ "model.layers.39.input_layernorm.weight": "model-00005-of-00007.safetensors",
 
306
  "model.layers.39.mlp.down_proj.weight": "model-00005-of-00007.safetensors",
307
  "model.layers.39.mlp.gate_proj.weight": "model-00005-of-00007.safetensors",
308
  "model.layers.39.mlp.up_proj.weight": "model-00005-of-00007.safetensors",
309
+ "model.layers.39.post_attention_layernorm.weight": "model-00005-of-00007.safetensors",
310
  "model.layers.39.self_attn.k_proj.weight": "model-00005-of-00007.safetensors",
311
  "model.layers.39.self_attn.o_proj.weight": "model-00005-of-00007.safetensors",
312
  "model.layers.39.self_attn.q_proj.weight": "model-00005-of-00007.safetensors",
313
  "model.layers.39.self_attn.v_proj.weight": "model-00005-of-00007.safetensors",
314
+ "model.layers.4.input_layernorm.weight": "model-00001-of-00007.safetensors",
 
315
  "model.layers.4.mlp.down_proj.weight": "model-00001-of-00007.safetensors",
316
  "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00007.safetensors",
317
  "model.layers.4.mlp.up_proj.weight": "model-00001-of-00007.safetensors",
318
+ "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00007.safetensors",
319
  "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00007.safetensors",
320
  "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00007.safetensors",
321
  "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00007.safetensors",
322
  "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00007.safetensors",
323
+ "model.layers.40.input_layernorm.weight": "model-00005-of-00007.safetensors",
 
324
  "model.layers.40.mlp.down_proj.weight": "model-00005-of-00007.safetensors",
325
  "model.layers.40.mlp.gate_proj.weight": "model-00005-of-00007.safetensors",
326
  "model.layers.40.mlp.up_proj.weight": "model-00005-of-00007.safetensors",
327
+ "model.layers.40.post_attention_layernorm.weight": "model-00005-of-00007.safetensors",
328
  "model.layers.40.self_attn.k_proj.weight": "model-00005-of-00007.safetensors",
329
  "model.layers.40.self_attn.o_proj.weight": "model-00005-of-00007.safetensors",
330
  "model.layers.40.self_attn.q_proj.weight": "model-00005-of-00007.safetensors",
331
  "model.layers.40.self_attn.v_proj.weight": "model-00005-of-00007.safetensors",
332
+ "model.layers.41.input_layernorm.weight": "model-00005-of-00007.safetensors",
 
333
  "model.layers.41.mlp.down_proj.weight": "model-00005-of-00007.safetensors",
334
  "model.layers.41.mlp.gate_proj.weight": "model-00005-of-00007.safetensors",
335
  "model.layers.41.mlp.up_proj.weight": "model-00005-of-00007.safetensors",
336
+ "model.layers.41.post_attention_layernorm.weight": "model-00005-of-00007.safetensors",
337
  "model.layers.41.self_attn.k_proj.weight": "model-00005-of-00007.safetensors",
338
  "model.layers.41.self_attn.o_proj.weight": "model-00005-of-00007.safetensors",
339
  "model.layers.41.self_attn.q_proj.weight": "model-00005-of-00007.safetensors",
340
  "model.layers.41.self_attn.v_proj.weight": "model-00005-of-00007.safetensors",
341
+ "model.layers.42.input_layernorm.weight": "model-00005-of-00007.safetensors",
 
342
  "model.layers.42.mlp.down_proj.weight": "model-00005-of-00007.safetensors",
343
  "model.layers.42.mlp.gate_proj.weight": "model-00005-of-00007.safetensors",
344
  "model.layers.42.mlp.up_proj.weight": "model-00005-of-00007.safetensors",
345
+ "model.layers.42.post_attention_layernorm.weight": "model-00005-of-00007.safetensors",
346
  "model.layers.42.self_attn.k_proj.weight": "model-00005-of-00007.safetensors",
347
  "model.layers.42.self_attn.o_proj.weight": "model-00005-of-00007.safetensors",
348
  "model.layers.42.self_attn.q_proj.weight": "model-00005-of-00007.safetensors",
349
  "model.layers.42.self_attn.v_proj.weight": "model-00005-of-00007.safetensors",
350
+ "model.layers.43.input_layernorm.weight": "model-00006-of-00007.safetensors",
 
351
  "model.layers.43.mlp.down_proj.weight": "model-00006-of-00007.safetensors",
352
  "model.layers.43.mlp.gate_proj.weight": "model-00006-of-00007.safetensors",
353
  "model.layers.43.mlp.up_proj.weight": "model-00006-of-00007.safetensors",
354
+ "model.layers.43.post_attention_layernorm.weight": "model-00006-of-00007.safetensors",
355
  "model.layers.43.self_attn.k_proj.weight": "model-00005-of-00007.safetensors",
356
  "model.layers.43.self_attn.o_proj.weight": "model-00005-of-00007.safetensors",
357
  "model.layers.43.self_attn.q_proj.weight": "model-00005-of-00007.safetensors",
358
  "model.layers.43.self_attn.v_proj.weight": "model-00005-of-00007.safetensors",
359
+ "model.layers.44.input_layernorm.weight": "model-00006-of-00007.safetensors",
 
360
  "model.layers.44.mlp.down_proj.weight": "model-00006-of-00007.safetensors",
361
  "model.layers.44.mlp.gate_proj.weight": "model-00006-of-00007.safetensors",
362
  "model.layers.44.mlp.up_proj.weight": "model-00006-of-00007.safetensors",
363
+ "model.layers.44.post_attention_layernorm.weight": "model-00006-of-00007.safetensors",
364
  "model.layers.44.self_attn.k_proj.weight": "model-00006-of-00007.safetensors",
365
  "model.layers.44.self_attn.o_proj.weight": "model-00006-of-00007.safetensors",
366
  "model.layers.44.self_attn.q_proj.weight": "model-00006-of-00007.safetensors",
367
  "model.layers.44.self_attn.v_proj.weight": "model-00006-of-00007.safetensors",
368
+ "model.layers.45.input_layernorm.weight": "model-00006-of-00007.safetensors",
 
369
  "model.layers.45.mlp.down_proj.weight": "model-00006-of-00007.safetensors",
370
  "model.layers.45.mlp.gate_proj.weight": "model-00006-of-00007.safetensors",
371
  "model.layers.45.mlp.up_proj.weight": "model-00006-of-00007.safetensors",
372
+ "model.layers.45.post_attention_layernorm.weight": "model-00006-of-00007.safetensors",
373
  "model.layers.45.self_attn.k_proj.weight": "model-00006-of-00007.safetensors",
374
  "model.layers.45.self_attn.o_proj.weight": "model-00006-of-00007.safetensors",
375
  "model.layers.45.self_attn.q_proj.weight": "model-00006-of-00007.safetensors",
376
  "model.layers.45.self_attn.v_proj.weight": "model-00006-of-00007.safetensors",
377
+ "model.layers.46.input_layernorm.weight": "model-00006-of-00007.safetensors",
 
378
  "model.layers.46.mlp.down_proj.weight": "model-00006-of-00007.safetensors",
379
  "model.layers.46.mlp.gate_proj.weight": "model-00006-of-00007.safetensors",
380
  "model.layers.46.mlp.up_proj.weight": "model-00006-of-00007.safetensors",
381
+ "model.layers.46.post_attention_layernorm.weight": "model-00006-of-00007.safetensors",
382
  "model.layers.46.self_attn.k_proj.weight": "model-00006-of-00007.safetensors",
383
  "model.layers.46.self_attn.o_proj.weight": "model-00006-of-00007.safetensors",
384
  "model.layers.46.self_attn.q_proj.weight": "model-00006-of-00007.safetensors",
385
  "model.layers.46.self_attn.v_proj.weight": "model-00006-of-00007.safetensors",
386
+ "model.layers.47.input_layernorm.weight": "model-00006-of-00007.safetensors",
 
387
  "model.layers.47.mlp.down_proj.weight": "model-00006-of-00007.safetensors",
388
  "model.layers.47.mlp.gate_proj.weight": "model-00006-of-00007.safetensors",
389
  "model.layers.47.mlp.up_proj.weight": "model-00006-of-00007.safetensors",
390
+ "model.layers.47.post_attention_layernorm.weight": "model-00006-of-00007.safetensors",
391
  "model.layers.47.self_attn.k_proj.weight": "model-00006-of-00007.safetensors",
392
  "model.layers.47.self_attn.o_proj.weight": "model-00006-of-00007.safetensors",
393
  "model.layers.47.self_attn.q_proj.weight": "model-00006-of-00007.safetensors",
394
  "model.layers.47.self_attn.v_proj.weight": "model-00006-of-00007.safetensors",
395
+ "model.layers.48.input_layernorm.weight": "model-00006-of-00007.safetensors",
 
396
  "model.layers.48.mlp.down_proj.weight": "model-00006-of-00007.safetensors",
397
  "model.layers.48.mlp.gate_proj.weight": "model-00006-of-00007.safetensors",
398
  "model.layers.48.mlp.up_proj.weight": "model-00006-of-00007.safetensors",
399
+ "model.layers.48.post_attention_layernorm.weight": "model-00006-of-00007.safetensors",
400
  "model.layers.48.self_attn.k_proj.weight": "model-00006-of-00007.safetensors",
401
  "model.layers.48.self_attn.o_proj.weight": "model-00006-of-00007.safetensors",
402
  "model.layers.48.self_attn.q_proj.weight": "model-00006-of-00007.safetensors",
403
  "model.layers.48.self_attn.v_proj.weight": "model-00006-of-00007.safetensors",
404
+ "model.layers.49.input_layernorm.weight": "model-00006-of-00007.safetensors",
 
405
  "model.layers.49.mlp.down_proj.weight": "model-00006-of-00007.safetensors",
406
  "model.layers.49.mlp.gate_proj.weight": "model-00006-of-00007.safetensors",
407
  "model.layers.49.mlp.up_proj.weight": "model-00006-of-00007.safetensors",
408
+ "model.layers.49.post_attention_layernorm.weight": "model-00006-of-00007.safetensors",
409
  "model.layers.49.self_attn.k_proj.weight": "model-00006-of-00007.safetensors",
410
  "model.layers.49.self_attn.o_proj.weight": "model-00006-of-00007.safetensors",
411
  "model.layers.49.self_attn.q_proj.weight": "model-00006-of-00007.safetensors",
412
  "model.layers.49.self_attn.v_proj.weight": "model-00006-of-00007.safetensors",
413
+ "model.layers.5.input_layernorm.weight": "model-00001-of-00007.safetensors",
 
414
  "model.layers.5.mlp.down_proj.weight": "model-00001-of-00007.safetensors",
415
  "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00007.safetensors",
416
  "model.layers.5.mlp.up_proj.weight": "model-00001-of-00007.safetensors",
417
+ "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00007.safetensors",
418
  "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00007.safetensors",
419
  "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00007.safetensors",
420
  "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00007.safetensors",
421
  "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00007.safetensors",
422
+ "model.layers.50.input_layernorm.weight": "model-00006-of-00007.safetensors",
 
423
  "model.layers.50.mlp.down_proj.weight": "model-00006-of-00007.safetensors",
424
  "model.layers.50.mlp.gate_proj.weight": "model-00006-of-00007.safetensors",
425
  "model.layers.50.mlp.up_proj.weight": "model-00006-of-00007.safetensors",
426
+ "model.layers.50.post_attention_layernorm.weight": "model-00006-of-00007.safetensors",
427
  "model.layers.50.self_attn.k_proj.weight": "model-00006-of-00007.safetensors",
428
  "model.layers.50.self_attn.o_proj.weight": "model-00006-of-00007.safetensors",
429
  "model.layers.50.self_attn.q_proj.weight": "model-00006-of-00007.safetensors",
430
  "model.layers.50.self_attn.v_proj.weight": "model-00006-of-00007.safetensors",
431
+ "model.layers.51.input_layernorm.weight": "model-00006-of-00007.safetensors",
 
432
  "model.layers.51.mlp.down_proj.weight": "model-00006-of-00007.safetensors",
433
  "model.layers.51.mlp.gate_proj.weight": "model-00006-of-00007.safetensors",
434
  "model.layers.51.mlp.up_proj.weight": "model-00006-of-00007.safetensors",
435
+ "model.layers.51.post_attention_layernorm.weight": "model-00006-of-00007.safetensors",
436
  "model.layers.51.self_attn.k_proj.weight": "model-00006-of-00007.safetensors",
437
  "model.layers.51.self_attn.o_proj.weight": "model-00006-of-00007.safetensors",
438
  "model.layers.51.self_attn.q_proj.weight": "model-00006-of-00007.safetensors",
439
  "model.layers.51.self_attn.v_proj.weight": "model-00006-of-00007.safetensors",
440
+ "model.layers.52.input_layernorm.weight": "model-00007-of-00007.safetensors",
 
441
  "model.layers.52.mlp.down_proj.weight": "model-00007-of-00007.safetensors",
442
  "model.layers.52.mlp.gate_proj.weight": "model-00007-of-00007.safetensors",
443
  "model.layers.52.mlp.up_proj.weight": "model-00007-of-00007.safetensors",
444
+ "model.layers.52.post_attention_layernorm.weight": "model-00007-of-00007.safetensors",
445
  "model.layers.52.self_attn.k_proj.weight": "model-00006-of-00007.safetensors",
446
  "model.layers.52.self_attn.o_proj.weight": "model-00007-of-00007.safetensors",
447
  "model.layers.52.self_attn.q_proj.weight": "model-00006-of-00007.safetensors",
448
  "model.layers.52.self_attn.v_proj.weight": "model-00006-of-00007.safetensors",
449
+ "model.layers.53.input_layernorm.weight": "model-00007-of-00007.safetensors",
 
450
  "model.layers.53.mlp.down_proj.weight": "model-00007-of-00007.safetensors",
451
  "model.layers.53.mlp.gate_proj.weight": "model-00007-of-00007.safetensors",
452
  "model.layers.53.mlp.up_proj.weight": "model-00007-of-00007.safetensors",
453
+ "model.layers.53.post_attention_layernorm.weight": "model-00007-of-00007.safetensors",
454
  "model.layers.53.self_attn.k_proj.weight": "model-00007-of-00007.safetensors",
455
  "model.layers.53.self_attn.o_proj.weight": "model-00007-of-00007.safetensors",
456
  "model.layers.53.self_attn.q_proj.weight": "model-00007-of-00007.safetensors",
457
  "model.layers.53.self_attn.v_proj.weight": "model-00007-of-00007.safetensors",
458
+ "model.layers.54.input_layernorm.weight": "model-00007-of-00007.safetensors",
 
459
  "model.layers.54.mlp.down_proj.weight": "model-00007-of-00007.safetensors",
460
  "model.layers.54.mlp.gate_proj.weight": "model-00007-of-00007.safetensors",
461
  "model.layers.54.mlp.up_proj.weight": "model-00007-of-00007.safetensors",
462
+ "model.layers.54.post_attention_layernorm.weight": "model-00007-of-00007.safetensors",
463
  "model.layers.54.self_attn.k_proj.weight": "model-00007-of-00007.safetensors",
464
  "model.layers.54.self_attn.o_proj.weight": "model-00007-of-00007.safetensors",
465
  "model.layers.54.self_attn.q_proj.weight": "model-00007-of-00007.safetensors",
466
  "model.layers.54.self_attn.v_proj.weight": "model-00007-of-00007.safetensors",
467
+ "model.layers.55.input_layernorm.weight": "model-00007-of-00007.safetensors",
 
468
  "model.layers.55.mlp.down_proj.weight": "model-00007-of-00007.safetensors",
469
  "model.layers.55.mlp.gate_proj.weight": "model-00007-of-00007.safetensors",
470
  "model.layers.55.mlp.up_proj.weight": "model-00007-of-00007.safetensors",
471
+ "model.layers.55.post_attention_layernorm.weight": "model-00007-of-00007.safetensors",
472
  "model.layers.55.self_attn.k_proj.weight": "model-00007-of-00007.safetensors",
473
  "model.layers.55.self_attn.o_proj.weight": "model-00007-of-00007.safetensors",
474
  "model.layers.55.self_attn.q_proj.weight": "model-00007-of-00007.safetensors",
475
  "model.layers.55.self_attn.v_proj.weight": "model-00007-of-00007.safetensors",
476
+ "model.layers.56.input_layernorm.weight": "model-00007-of-00007.safetensors",
 
477
  "model.layers.56.mlp.down_proj.weight": "model-00007-of-00007.safetensors",
478
  "model.layers.56.mlp.gate_proj.weight": "model-00007-of-00007.safetensors",
479
  "model.layers.56.mlp.up_proj.weight": "model-00007-of-00007.safetensors",
480
+ "model.layers.56.post_attention_layernorm.weight": "model-00007-of-00007.safetensors",
481
  "model.layers.56.self_attn.k_proj.weight": "model-00007-of-00007.safetensors",
482
  "model.layers.56.self_attn.o_proj.weight": "model-00007-of-00007.safetensors",
483
  "model.layers.56.self_attn.q_proj.weight": "model-00007-of-00007.safetensors",
484
  "model.layers.56.self_attn.v_proj.weight": "model-00007-of-00007.safetensors",
485
+ "model.layers.57.input_layernorm.weight": "model-00007-of-00007.safetensors",
 
486
  "model.layers.57.mlp.down_proj.weight": "model-00007-of-00007.safetensors",
487
  "model.layers.57.mlp.gate_proj.weight": "model-00007-of-00007.safetensors",
488
  "model.layers.57.mlp.up_proj.weight": "model-00007-of-00007.safetensors",
489
+ "model.layers.57.post_attention_layernorm.weight": "model-00007-of-00007.safetensors",
490
  "model.layers.57.self_attn.k_proj.weight": "model-00007-of-00007.safetensors",
491
  "model.layers.57.self_attn.o_proj.weight": "model-00007-of-00007.safetensors",
492
  "model.layers.57.self_attn.q_proj.weight": "model-00007-of-00007.safetensors",
493
  "model.layers.57.self_attn.v_proj.weight": "model-00007-of-00007.safetensors",
494
+ "model.layers.58.input_layernorm.weight": "model-00007-of-00007.safetensors",
 
495
  "model.layers.58.mlp.down_proj.weight": "model-00007-of-00007.safetensors",
496
  "model.layers.58.mlp.gate_proj.weight": "model-00007-of-00007.safetensors",
497
  "model.layers.58.mlp.up_proj.weight": "model-00007-of-00007.safetensors",
498
+ "model.layers.58.post_attention_layernorm.weight": "model-00007-of-00007.safetensors",
499
  "model.layers.58.self_attn.k_proj.weight": "model-00007-of-00007.safetensors",
500
  "model.layers.58.self_attn.o_proj.weight": "model-00007-of-00007.safetensors",
501
  "model.layers.58.self_attn.q_proj.weight": "model-00007-of-00007.safetensors",
502
  "model.layers.58.self_attn.v_proj.weight": "model-00007-of-00007.safetensors",
503
+ "model.layers.59.input_layernorm.weight": "model-00007-of-00007.safetensors",
 
504
  "model.layers.59.mlp.down_proj.weight": "model-00007-of-00007.safetensors",
505
  "model.layers.59.mlp.gate_proj.weight": "model-00007-of-00007.safetensors",
506
  "model.layers.59.mlp.up_proj.weight": "model-00007-of-00007.safetensors",
507
+ "model.layers.59.post_attention_layernorm.weight": "model-00007-of-00007.safetensors",
508
  "model.layers.59.self_attn.k_proj.weight": "model-00007-of-00007.safetensors",
509
  "model.layers.59.self_attn.o_proj.weight": "model-00007-of-00007.safetensors",
510
  "model.layers.59.self_attn.q_proj.weight": "model-00007-of-00007.safetensors",
511
  "model.layers.59.self_attn.v_proj.weight": "model-00007-of-00007.safetensors",
512
+ "model.layers.6.input_layernorm.weight": "model-00001-of-00007.safetensors",
 
513
  "model.layers.6.mlp.down_proj.weight": "model-00001-of-00007.safetensors",
514
  "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00007.safetensors",
515
  "model.layers.6.mlp.up_proj.weight": "model-00001-of-00007.safetensors",
516
+ "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00007.safetensors",
517
  "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00007.safetensors",
518
  "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00007.safetensors",
519
  "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00007.safetensors",
520
  "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00007.safetensors",
521
+ "model.layers.7.input_layernorm.weight": "model-00001-of-00007.safetensors",
 
522
  "model.layers.7.mlp.down_proj.weight": "model-00001-of-00007.safetensors",
523
  "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00007.safetensors",
524
  "model.layers.7.mlp.up_proj.weight": "model-00001-of-00007.safetensors",
525
+ "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00007.safetensors",
526
  "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00007.safetensors",
527
  "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00007.safetensors",
528
  "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00007.safetensors",
529
  "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00007.safetensors",
530
+ "model.layers.8.input_layernorm.weight": "model-00002-of-00007.safetensors",
 
531
  "model.layers.8.mlp.down_proj.weight": "model-00002-of-00007.safetensors",
532
  "model.layers.8.mlp.gate_proj.weight": "model-00002-of-00007.safetensors",
533
  "model.layers.8.mlp.up_proj.weight": "model-00002-of-00007.safetensors",
534
+ "model.layers.8.post_attention_layernorm.weight": "model-00002-of-00007.safetensors",
535
  "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00007.safetensors",
536
  "model.layers.8.self_attn.o_proj.weight": "model-00002-of-00007.safetensors",
537
  "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00007.safetensors",
538
  "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00007.safetensors",
539
+ "model.layers.9.input_layernorm.weight": "model-00002-of-00007.safetensors",
 
540
  "model.layers.9.mlp.down_proj.weight": "model-00002-of-00007.safetensors",
541
  "model.layers.9.mlp.gate_proj.weight": "model-00002-of-00007.safetensors",
542
  "model.layers.9.mlp.up_proj.weight": "model-00002-of-00007.safetensors",
543
+ "model.layers.9.post_attention_layernorm.weight": "model-00002-of-00007.safetensors",
544
  "model.layers.9.self_attn.k_proj.weight": "model-00002-of-00007.safetensors",
545
  "model.layers.9.self_attn.o_proj.weight": "model-00002-of-00007.safetensors",
546
  "model.layers.9.self_attn.q_proj.weight": "model-00002-of-00007.safetensors",
modeling_yi.py DELETED
@@ -1,1035 +0,0 @@
1
- """ PyTorch Yi model."""
2
- import math
3
- from typing import List, Optional, Tuple, Union
4
-
5
- import torch.utils.checkpoint
6
- from einops import repeat
7
- from packaging import version
8
- from torch import nn
9
- from torch.nn import BCEWithLogitsLoss, CrossEntropyLoss, MSELoss
10
- from transformers.activations import ACT2FN
11
- from transformers.modeling_outputs import (
12
- BaseModelOutputWithPast,
13
- CausalLMOutputWithPast,
14
- SequenceClassifierOutputWithPast,
15
- )
16
- from transformers.modeling_utils import PreTrainedModel
17
- from transformers.pytorch_utils import ALL_LAYERNORM_LAYERS
18
- from transformers.utils import (
19
- add_start_docstrings,
20
- add_start_docstrings_to_model_forward,
21
- logging,
22
- replace_return_docstrings,
23
- )
24
-
25
- from .configuration_yi import YiConfig
26
-
27
- is_flash_attn_available = True
28
- try:
29
- from flash_attn import flash_attn_func, __version__
30
-
31
- assert version.parse(__version__) >= version.parse(
32
- "2.3.0"
33
- ), "please update your flash_attn version (>= 2.3.0)"
34
- except ModuleNotFoundError:
35
- is_flash_attn_available = False
36
-
37
- logger = logging.get_logger(__name__)
38
-
39
- _CONFIG_FOR_DOC = "YiConfig"
40
-
41
-
42
- # Copied from transformers.models.bart.modeling_bart._make_causal_mask
43
- def _make_causal_mask(
44
- input_ids_shape: torch.Size,
45
- dtype: torch.dtype,
46
- device: torch.device,
47
- past_key_values_length: int = 0,
48
- ):
49
- """
50
- Make causal mask used for bi-directional self-attention.
51
- """
52
- bsz, tgt_len = input_ids_shape
53
- mask = torch.full(
54
- (tgt_len, tgt_len),
55
- torch.tensor(torch.finfo(dtype).min, device=device),
56
- device=device,
57
- )
58
- mask_cond = torch.arange(mask.size(-1), device=device)
59
- mask.masked_fill_(mask_cond < (mask_cond + 1).view(mask.size(-1), 1), 0)
60
- mask = mask.to(dtype)
61
-
62
- if past_key_values_length > 0:
63
- mask = torch.cat(
64
- [
65
- torch.zeros(
66
- tgt_len, past_key_values_length, dtype=dtype, device=device
67
- ),
68
- mask,
69
- ],
70
- dim=-1,
71
- )
72
- return mask[None, None, :, :].expand(
73
- bsz, 1, tgt_len, tgt_len + past_key_values_length
74
- )
75
-
76
-
77
- # Copied from transformers.models.bart.modeling_bart._expand_mask
78
- def _expand_mask(mask: torch.Tensor, dtype: torch.dtype, tgt_len: Optional[int] = None):
79
- """
80
- Expands attention_mask from `[bsz, seq_len]` to `[bsz, 1, tgt_seq_len, src_seq_len]`.
81
- """
82
- bsz, src_len = mask.size()
83
- tgt_len = tgt_len if tgt_len is not None else src_len
84
-
85
- expanded_mask = mask[:, None, None, :].expand(bsz, 1, tgt_len, src_len).to(dtype)
86
-
87
- inverted_mask = 1.0 - expanded_mask
88
-
89
- return inverted_mask.masked_fill(
90
- inverted_mask.to(torch.bool), torch.finfo(dtype).min
91
- )
92
-
93
-
94
- class YiRMSNorm(nn.Module):
95
- def __init__(self, hidden_size, eps=1e-5):
96
- """
97
- YiRMSNorm is equivalent to T5LayerNorm
98
- """
99
- super().__init__()
100
- self.weight = nn.Parameter(torch.ones(hidden_size))
101
- self.variance_epsilon = eps
102
-
103
- def forward(self, hidden_states):
104
- input_dtype = hidden_states.dtype
105
- hidden_states = hidden_states.to(torch.float32)
106
- variance = hidden_states.pow(2).mean(-1, keepdim=True)
107
- hidden_states = hidden_states * torch.rsqrt(variance + self.variance_epsilon)
108
-
109
- return self.weight * hidden_states.to(input_dtype)
110
-
111
-
112
- ALL_LAYERNORM_LAYERS.append(YiRMSNorm)
113
-
114
-
115
- class YiRotaryEmbedding(torch.nn.Module):
116
- def __init__(self, dim, max_position_embeddings=4096, base=5000000, device=None):
117
- super().__init__()
118
-
119
- self.dim = dim
120
- self.max_position_embeddings = max_position_embeddings
121
- self.base = base
122
-
123
- # Build here to make `torch.jit.trace` work.
124
- self._set_cos_sin_cache(seq_len=max_position_embeddings, device=device)
125
-
126
- def _set_cos_sin_cache(self, seq_len, device):
127
- self.max_seq_len_cached = seq_len
128
- inv_freq = 1.0 / (
129
- self.base ** (torch.arange(0, self.dim, 2).float().to(device) / self.dim)
130
- )
131
- t = torch.arange(self.max_seq_len_cached, device=device, dtype=torch.float32)
132
- freqs = torch.einsum("i,j->ij", t, inv_freq)
133
- # Different from paper, but it uses a different permutation in order to obtain the same calculation
134
- emb = torch.cat((freqs, freqs), dim=-1)
135
- self.register_buffer(
136
- "cos_cached", emb.cos()[None, None, :, :], persistent=False
137
- )
138
- self.register_buffer(
139
- "sin_cached", emb.sin()[None, None, :, :], persistent=False
140
- )
141
-
142
- def forward(self, x, seq_len=None):
143
- # x: [bs, num_attention_heads, seq_len, head_size]
144
- if seq_len > self.max_seq_len_cached:
145
- self._set_cos_sin_cache(seq_len=seq_len, device=x.device)
146
-
147
- return (
148
- self.cos_cached[:, :, :seq_len, ...].to(dtype=x.dtype),
149
- self.sin_cached[:, :, :seq_len, ...].to(dtype=x.dtype),
150
- )
151
-
152
-
153
- def rotate_half(x):
154
- """Rotates half the hidden dims of the input."""
155
- x1 = x[..., : x.shape[-1] // 2]
156
- x2 = x[..., x.shape[-1] // 2 :]
157
- return torch.cat((-x2, x1), dim=-1)
158
-
159
-
160
- def apply_rotary_pos_emb(q, k, cos, sin, position_ids, flash_attn_available):
161
- # The first two dimensions of cos and sin are always 1, so we can `squeeze` them.
162
- cos = cos.squeeze(1).squeeze(0) # [seq_len, dim]
163
- sin = sin.squeeze(1).squeeze(0) # [seq_len, dim]
164
- expand_dim = 2 if flash_attn_available else 1
165
- cos = cos[position_ids].unsqueeze(expand_dim) # [bs, seq_len, dim]
166
- sin = sin[position_ids].unsqueeze(expand_dim) # [bs, seq_len, dim]
167
- q_embed = (q * cos) + (rotate_half(q) * sin)
168
- k_embed = (k * cos) + (rotate_half(k) * sin)
169
- return q_embed, k_embed
170
-
171
-
172
- class YiMLP(nn.Module):
173
- def __init__(self, hidden_size: int, intermediate_size: int, hidden_act: str):
174
- super().__init__()
175
- self.gate_proj = nn.Linear(hidden_size, intermediate_size, bias=False)
176
- self.down_proj = nn.Linear(intermediate_size, hidden_size, bias=False)
177
- self.up_proj = nn.Linear(hidden_size, intermediate_size, bias=False)
178
- self.act_fn = ACT2FN[hidden_act]
179
-
180
- def forward(self, x):
181
- return self.down_proj(self.act_fn(self.gate_proj(x)) * self.up_proj(x))
182
-
183
-
184
- class YiAttention(nn.Module):
185
- """Multi-headed attention from 'Attention Is All You Need' paper"""
186
-
187
- def __init__(self, config: YiConfig):
188
- super().__init__()
189
- self.config = config
190
- self.hidden_size = config.hidden_size
191
- self.num_heads = config.num_attention_heads
192
- self.head_dim = self.hidden_size // self.num_heads
193
- self.num_key_value_heads = config.num_key_value_heads
194
- self.num_key_value_groups = self.num_heads // self.num_key_value_heads
195
- self.max_position_embeddings = config.max_position_embeddings
196
-
197
- if (self.head_dim * self.num_heads) != self.hidden_size:
198
- raise ValueError(
199
- f"hidden_size must be divisible by num_heads (got `hidden_size`: {self.hidden_size}"
200
- f" and `num_heads`: {self.num_heads})."
201
- )
202
- self.q_proj = nn.Linear(
203
- self.hidden_size, self.num_heads * self.head_dim, bias=False
204
- )
205
- self.k_proj = nn.Linear(
206
- self.hidden_size, self.num_key_value_heads * self.head_dim, bias=False
207
- )
208
- self.v_proj = nn.Linear(
209
- self.hidden_size, self.num_key_value_heads * self.head_dim, bias=False
210
- )
211
- self.o_proj = nn.Linear(
212
- self.num_heads * self.head_dim, self.hidden_size, bias=False
213
- )
214
-
215
- self.rotary_emb = YiRotaryEmbedding(
216
- self.head_dim,
217
- max_position_embeddings=self.max_position_embeddings,
218
- base=self.config.rope_theta,
219
- )
220
-
221
- def forward(
222
- self,
223
- hidden_states: torch.Tensor,
224
- attention_mask: Optional[torch.Tensor] = None,
225
- position_ids: Optional[torch.LongTensor] = None,
226
- past_key_value: Optional[Tuple[torch.Tensor]] = None,
227
- output_attentions: bool = False,
228
- use_cache: bool = False,
229
- ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]:
230
- bsz, q_len, _ = hidden_states.size()
231
-
232
- query_states = self.q_proj(hidden_states).view(
233
- bsz, q_len, self.num_heads, self.head_dim
234
- )
235
-
236
- key_states = self.k_proj(hidden_states).view(
237
- bsz, q_len, self.num_key_value_heads, self.head_dim
238
- )
239
- value_states = self.v_proj(hidden_states).view(
240
- bsz, q_len, self.num_key_value_heads, self.head_dim
241
- )
242
-
243
- if not is_flash_attn_available:
244
- if self.num_key_value_groups > 1:
245
- key_states = repeat(
246
- key_states, f"b n h d -> b n (h {self.num_key_value_groups}) d"
247
- )
248
- value_states = repeat(
249
- value_states, f"b n h d -> b n (h {self.num_key_value_groups}) d"
250
- )
251
-
252
- # b n h d -> b h n d
253
- query_states = query_states.transpose(1, 2)
254
- key_states = key_states.transpose(1, 2)
255
- value_states = value_states.transpose(1, 2)
256
-
257
- seq_dim = 1 if is_flash_attn_available else 2
258
- kv_seq_len = key_states.shape[seq_dim]
259
- if past_key_value is not None:
260
- kv_seq_len += past_key_value[0].shape[seq_dim]
261
- cos, sin = self.rotary_emb(value_states, seq_len=kv_seq_len)
262
- query_states, key_states = apply_rotary_pos_emb(
263
- query_states, key_states, cos, sin, position_ids, is_flash_attn_available
264
- )
265
-
266
- if past_key_value is not None:
267
- # reuse k, v, self_attention
268
- key_states = torch.cat([past_key_value[0], key_states], dim=seq_dim)
269
- value_states = torch.cat([past_key_value[1], value_states], dim=seq_dim)
270
-
271
- past_key_value = (key_states, value_states) if use_cache else None
272
-
273
- if is_flash_attn_available:
274
- attn_output = flash_attn_func(
275
- query_states, key_states, value_states, dropout_p=0.0, causal=True
276
- )
277
- else:
278
- attn_weights = torch.matmul(
279
- query_states, key_states.transpose(2, 3)
280
- ) / math.sqrt(self.head_dim)
281
-
282
- if attn_weights.size() != (bsz, self.num_heads, q_len, kv_seq_len):
283
- raise ValueError(
284
- f"Attention weights should be of size {(bsz, self.num_heads, q_len, kv_seq_len)}, but is"
285
- f" {attn_weights.size()}"
286
- )
287
-
288
- if attention_mask is not None:
289
- if attention_mask.size() != (bsz, 1, q_len, kv_seq_len):
290
- raise ValueError(
291
- f"Attention mask should be of size {(bsz, 1, q_len, kv_seq_len)}, but is"
292
- f"{attention_mask.size()}"
293
- )
294
- attn_weights = attn_weights + attention_mask
295
- dtype_min = torch.tensor(
296
- torch.finfo(attn_weights.dtype).min,
297
- device=attn_weights.device,
298
- dtype=attn_weights.dtype,
299
- )
300
- attn_weights = torch.max(attn_weights, dtype_min)
301
-
302
- # upcast attention to fp32
303
- attn_weights = nn.functional.softmax(
304
- attn_weights, dim=-1, dtype=torch.float32
305
- ).to(query_states.dtype)
306
- attn_output = torch.matmul(attn_weights, value_states)
307
-
308
- if attn_output.size() != (bsz, self.num_heads, q_len, self.head_dim):
309
- raise ValueError(
310
- f"`attn_output` should be of size {(bsz, self.num_heads, q_len, self.head_dim)}, but is"
311
- f" {attn_output.size()}"
312
- )
313
-
314
- if not is_flash_attn_available:
315
- attn_output = attn_output.transpose(1, 2)
316
-
317
- attn_output = attn_output.reshape(bsz, q_len, self.hidden_size)
318
-
319
- attn_output = self.o_proj(attn_output)
320
-
321
- if not output_attentions:
322
- attn_weights = None
323
-
324
- return attn_output, attn_weights, past_key_value
325
-
326
-
327
- class YiDecoderLayer(nn.Module):
328
- def __init__(self, config: YiConfig):
329
- super().__init__()
330
-
331
- self.hidden_size = config.hidden_size
332
- self.self_attn = YiAttention(config=config)
333
- self.mlp = YiMLP(
334
- hidden_size=self.hidden_size,
335
- intermediate_size=config.intermediate_size,
336
- hidden_act=config.hidden_act,
337
- )
338
-
339
- self.ln1 = YiRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
340
- self.ln2 = YiRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
341
-
342
- def forward(
343
- self,
344
- hidden_states: torch.Tensor,
345
- attention_mask: Optional[torch.Tensor] = None,
346
- position_ids: Optional[torch.LongTensor] = None,
347
- past_key_value: Optional[Tuple[torch.Tensor]] = None,
348
- output_attentions: Optional[bool] = False,
349
- use_cache: Optional[bool] = False,
350
- ) -> Tuple[
351
- torch.FloatTensor, Optional[Tuple[torch.FloatTensor, torch.FloatTensor]]
352
- ]:
353
- """
354
- Args:
355
- hidden_states (`torch.FloatTensor`): input to the layer of shape `(batch, seq_len, embed_dim)`
356
- attention_mask (`torch.FloatTensor`, *optional*): attention mask of size
357
- `(batch, 1, tgt_len, src_len)` where padding elements are indicated by very large negative values.
358
- output_attentions (`bool`, *optional*):
359
- Whether or not to return the attentions tensors of all attention layers. See `attentions` under
360
- returned tensors for more detail.
361
- use_cache (`bool`, *optional*):
362
- If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding
363
- (see `past_key_values`).
364
- past_key_value (`Tuple(torch.FloatTensor)`, *optional*): cached past key and value projection states
365
- """
366
-
367
- residual = hidden_states
368
-
369
- hidden_states = self.ln1(hidden_states)
370
-
371
- # Self Attention
372
- hidden_states, self_attn_weights, present_key_value = self.self_attn(
373
- hidden_states=hidden_states,
374
- attention_mask=attention_mask,
375
- position_ids=position_ids,
376
- past_key_value=past_key_value,
377
- output_attentions=output_attentions,
378
- use_cache=use_cache,
379
- )
380
- hidden_states = residual + hidden_states
381
-
382
- # Fully Connected
383
- residual = hidden_states
384
- hidden_states = self.ln2(hidden_states)
385
- hidden_states = self.mlp(hidden_states)
386
- hidden_states = residual + hidden_states
387
-
388
- outputs = (hidden_states,)
389
-
390
- if output_attentions:
391
- outputs += (self_attn_weights,)
392
-
393
- if use_cache:
394
- outputs += (present_key_value,)
395
-
396
- return outputs
397
-
398
-
399
- Yi_START_DOCSTRING = r"""
400
- This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the
401
- library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
402
- etc.)
403
-
404
- This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass.
405
- Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage
406
- and behavior.
407
-
408
- Parameters:
409
- config ([`YiConfig`]):
410
- Model configuration class with all the parameters of the model. Initializing with a config file does not
411
- load the weights associated with the model, only the configuration. Check out the
412
- [`~PreTrainedModel.from_pretrained`] method to load the model weights.
413
- """
414
-
415
-
416
- @add_start_docstrings(
417
- "The bare Yi Model outputting raw hidden-states without any specific head on top.",
418
- Yi_START_DOCSTRING,
419
- )
420
- class YiPreTrainedModel(PreTrainedModel):
421
- config_class = YiConfig
422
- base_model_prefix = "model"
423
- supports_gradient_checkpointing = True
424
- _no_split_modules = ["YiDecoderLayer"]
425
- _skip_keys_device_placement = "past_key_values"
426
-
427
- def _init_weights(self, module):
428
- std = self.config.initializer_range
429
- if isinstance(module, nn.Linear):
430
- module.weight.data.normal_(mean=0.0, std=std)
431
- if module.bias is not None:
432
- module.bias.data.zero_()
433
- elif isinstance(module, nn.Embedding):
434
- module.weight.data.normal_(mean=0.0, std=std)
435
- if module.padding_idx is not None:
436
- module.weight.data[module.padding_idx].zero_()
437
-
438
- def _set_gradient_checkpointing(self, module, value=False):
439
- if isinstance(module, YiModel):
440
- module.gradient_checkpointing = value
441
-
442
-
443
- Yi_INPUTS_DOCSTRING = r"""
444
- Args:
445
- input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`):
446
- Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide
447
- it.
448
-
449
- Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
450
- [`PreTrainedTokenizer.__call__`] for details.
451
-
452
- [What are input IDs?](../glossary#input-ids)
453
- attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
454
- Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:
455
-
456
- - 1 for tokens that are **not masked**,
457
- - 0 for tokens that are **masked**.
458
-
459
- [What are attention masks?](../glossary#attention-mask)
460
-
461
- Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
462
- [`PreTrainedTokenizer.__call__`] for details.
463
-
464
- If `past_key_values` is used, optionally only the last `decoder_input_ids` have to be input (see
465
- `past_key_values`).
466
-
467
- If you want to change padding behavior, you should read [`modeling_opt._prepare_decoder_attention_mask`]
468
- and modify to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more
469
- information on the default strategy.
470
-
471
- - 1 indicates the head is **not masked**,
472
- - 0 indicates the head is **masked**.
473
- position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
474
- Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
475
- config.n_positions - 1]`.
476
-
477
- [What are position IDs?](../glossary#position-ids)
478
- past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
479
- Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of shape
480
- `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of shape
481
- `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`.
482
-
483
- Contains pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention
484
- blocks) that can be used (see `past_key_values` input) to speed up sequential decoding.
485
-
486
- If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that
487
- don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all
488
- `decoder_input_ids` of shape `(batch_size, sequence_length)`.
489
- inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*):
490
- Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
491
- is useful if you want more control over how to convert `input_ids` indices into associated vectors than the
492
- model's internal embedding lookup matrix.
493
- use_cache (`bool`, *optional*):
494
- If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see
495
- `past_key_values`).
496
- output_attentions (`bool`, *optional*):
497
- Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
498
- tensors for more detail.
499
- output_hidden_states (`bool`, *optional*):
500
- Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for
501
- more detail.
502
- return_dict (`bool`, *optional*):
503
- Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
504
- """
505
-
506
-
507
- @add_start_docstrings(
508
- "The bare Yi Model outputting raw hidden-states without any specific head on top.",
509
- Yi_START_DOCSTRING,
510
- )
511
- class YiModel(YiPreTrainedModel):
512
- """
513
- Transformer decoder consisting of *config.num_hidden_layers* layers. Each layer is a [`YiDecoderLayer`]
514
-
515
- Args:
516
- config: YiConfig
517
- """
518
-
519
- def __init__(self, config: YiConfig):
520
- super().__init__(config)
521
- self.padding_idx = config.pad_token_id
522
- self.vocab_size = config.vocab_size
523
-
524
- self.embed_tokens = nn.Embedding(
525
- config.vocab_size, config.hidden_size, self.padding_idx
526
- )
527
- self.layers = nn.ModuleList(
528
- [YiDecoderLayer(config) for _ in range(config.num_hidden_layers)]
529
- )
530
-
531
- self.norm = YiRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
532
-
533
- self.gradient_checkpointing = False
534
- # Initialize weights and apply final processing
535
- self.post_init()
536
-
537
- def get_input_embeddings(self):
538
- return self.embed_tokens
539
-
540
- def set_input_embeddings(self, value):
541
- self.embed_tokens = value
542
-
543
- # Copied from transformers.models.bart.modeling_bart.BartDecoder._prepare_decoder_attention_mask
544
- def _prepare_decoder_attention_mask(
545
- self, attention_mask, input_ids, inputs_embeds, past_key_values_length
546
- ):
547
- input_shape = (
548
- input_ids.shape if input_ids is not None else inputs_embeds.shape[:-1]
549
- )
550
- # create causal mask
551
- # [bsz, seq_len] -> [bsz, 1, tgt_seq_len, src_seq_len]
552
- combined_attention_mask = None
553
- if input_shape[-1] > 1:
554
- combined_attention_mask = _make_causal_mask(
555
- input_shape,
556
- inputs_embeds.dtype,
557
- device=inputs_embeds.device,
558
- past_key_values_length=past_key_values_length,
559
- )
560
-
561
- if attention_mask is not None:
562
- # [bsz, seq_len] -> [bsz, 1, tgt_seq_len, src_seq_len]
563
- expanded_attn_mask = _expand_mask(
564
- attention_mask, inputs_embeds.dtype, tgt_len=input_shape[-1]
565
- ).to(inputs_embeds.device)
566
- combined_attention_mask = (
567
- expanded_attn_mask
568
- if combined_attention_mask is None
569
- else expanded_attn_mask + combined_attention_mask
570
- )
571
-
572
- return combined_attention_mask
573
-
574
- @add_start_docstrings_to_model_forward(Yi_INPUTS_DOCSTRING)
575
- def forward(
576
- self,
577
- input_ids: torch.LongTensor = None,
578
- attention_mask: Optional[torch.Tensor] = None,
579
- position_ids: Optional[torch.LongTensor] = None,
580
- past_key_values: Optional[List[torch.FloatTensor]] = None,
581
- inputs_embeds: Optional[torch.FloatTensor] = None,
582
- use_cache: Optional[bool] = None,
583
- output_attentions: Optional[bool] = None,
584
- output_hidden_states: Optional[bool] = None,
585
- return_dict: Optional[bool] = None,
586
- ) -> Union[Tuple, BaseModelOutputWithPast]:
587
- output_attentions = (
588
- output_attentions
589
- if output_attentions is not None
590
- else self.config.output_attentions
591
- )
592
- output_hidden_states = (
593
- output_hidden_states
594
- if output_hidden_states is not None
595
- else self.config.output_hidden_states
596
- )
597
- use_cache = use_cache if use_cache is not None else self.config.use_cache
598
-
599
- return_dict = (
600
- return_dict if return_dict is not None else self.config.use_return_dict
601
- )
602
-
603
- # retrieve input_ids and inputs_embeds
604
- if input_ids is not None and inputs_embeds is not None:
605
- raise ValueError(
606
- "You cannot specify both decoder_input_ids and decoder_inputs_embeds at the same time"
607
- )
608
- elif input_ids is not None:
609
- batch_size, seq_length = input_ids.shape
610
- elif inputs_embeds is not None:
611
- batch_size, seq_length, _ = inputs_embeds.shape
612
- else:
613
- raise ValueError(
614
- "You have to specify either decoder_input_ids or decoder_inputs_embeds"
615
- )
616
-
617
- seq_length_with_past = seq_length
618
- past_key_values_length = 0
619
-
620
- if past_key_values is not None:
621
- past_key_values_length = past_key_values[0][0].shape[2]
622
- seq_length_with_past = seq_length_with_past + past_key_values_length
623
-
624
- if position_ids is None:
625
- device = input_ids.device if input_ids is not None else inputs_embeds.device
626
- position_ids = torch.arange(
627
- past_key_values_length,
628
- seq_length + past_key_values_length,
629
- dtype=torch.long,
630
- device=device,
631
- )
632
- position_ids = position_ids.unsqueeze(0).view(-1, seq_length)
633
- else:
634
- position_ids = position_ids.view(-1, seq_length).long()
635
-
636
- if inputs_embeds is None:
637
- inputs_embeds = self.embed_tokens(input_ids)
638
-
639
- if not is_flash_attn_available:
640
- # embed positions
641
- if attention_mask is None:
642
- attention_mask = torch.ones(
643
- (batch_size, seq_length_with_past),
644
- dtype=torch.bool,
645
- device=inputs_embeds.device,
646
- )
647
- attention_mask = self._prepare_decoder_attention_mask(
648
- attention_mask,
649
- input_ids,
650
- inputs_embeds,
651
- past_key_values_length,
652
- )
653
- else:
654
- attention_mask = None
655
-
656
- hidden_states = inputs_embeds
657
- if self.gradient_checkpointing and self.training:
658
- if use_cache:
659
- logger.warning_once(
660
- "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`..."
661
- )
662
- use_cache = False
663
-
664
- # decoder layers
665
- all_hidden_states = () if output_hidden_states else None
666
- all_self_attns = () if output_attentions else None
667
- next_decoder_cache = () if use_cache else None
668
-
669
- for idx, decoder_layer in enumerate(self.layers):
670
- if output_hidden_states:
671
- all_hidden_states += (hidden_states,)
672
-
673
- past_key_value = (
674
- past_key_values[idx] if past_key_values is not None else None
675
- )
676
-
677
- if self.gradient_checkpointing and self.training:
678
-
679
- def create_custom_forward(module):
680
- def custom_forward(*inputs):
681
- # None for past_key_value
682
- return module(*inputs, past_key_value, output_attentions)
683
-
684
- return custom_forward
685
-
686
- layer_outputs = torch.utils.checkpoint.checkpoint(
687
- create_custom_forward(decoder_layer),
688
- hidden_states,
689
- attention_mask,
690
- position_ids,
691
- )
692
- else:
693
- layer_outputs = decoder_layer(
694
- hidden_states,
695
- attention_mask=attention_mask,
696
- position_ids=position_ids,
697
- past_key_value=past_key_value,
698
- output_attentions=output_attentions,
699
- use_cache=use_cache,
700
- )
701
-
702
- hidden_states = layer_outputs[0]
703
-
704
- if use_cache:
705
- next_decoder_cache += (layer_outputs[2 if output_attentions else 1],)
706
-
707
- if output_attentions:
708
- all_self_attns += (layer_outputs[1],)
709
-
710
- hidden_states = self.norm(hidden_states)
711
- # add hidden states from the last decoder layer
712
- if output_hidden_states:
713
- all_hidden_states += (hidden_states,)
714
-
715
- next_cache = next_decoder_cache if use_cache else None
716
- if not return_dict:
717
- return tuple(
718
- v
719
- for v in [hidden_states, next_cache, all_hidden_states, all_self_attns]
720
- if v is not None
721
- )
722
- return BaseModelOutputWithPast(
723
- last_hidden_state=hidden_states,
724
- past_key_values=next_cache,
725
- hidden_states=all_hidden_states,
726
- attentions=all_self_attns,
727
- )
728
-
729
-
730
- class YiForCausalLM(YiPreTrainedModel):
731
- _tied_weights_keys = ["lm_head.weight"]
732
-
733
- def __init__(self, config):
734
- super().__init__(config)
735
- self.model = YiModel(config)
736
-
737
- self.lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False)
738
-
739
- # Initialize weights and apply final processing
740
- self.post_init()
741
-
742
- def get_input_embeddings(self):
743
- return self.model.embed_tokens
744
-
745
- def set_input_embeddings(self, value):
746
- self.model.embed_tokens = value
747
-
748
- def get_output_embeddings(self):
749
- return self.lm_head
750
-
751
- def set_output_embeddings(self, new_embeddings):
752
- self.lm_head = new_embeddings
753
-
754
- def set_decoder(self, decoder):
755
- self.model = decoder
756
-
757
- def get_decoder(self):
758
- return self.model
759
-
760
- @add_start_docstrings_to_model_forward(Yi_INPUTS_DOCSTRING)
761
- @replace_return_docstrings(
762
- output_type=CausalLMOutputWithPast, config_class=_CONFIG_FOR_DOC
763
- )
764
- def forward(
765
- self,
766
- input_ids: torch.LongTensor = None,
767
- attention_mask: Optional[torch.Tensor] = None,
768
- position_ids: Optional[torch.LongTensor] = None,
769
- past_key_values: Optional[List[torch.FloatTensor]] = None,
770
- inputs_embeds: Optional[torch.FloatTensor] = None,
771
- labels: Optional[torch.LongTensor] = None,
772
- use_cache: Optional[bool] = None,
773
- output_attentions: Optional[bool] = None,
774
- output_hidden_states: Optional[bool] = None,
775
- return_dict: Optional[bool] = None,
776
- ) -> Union[Tuple, CausalLMOutputWithPast]:
777
- r"""
778
- Args:
779
- labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
780
- Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
781
- config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
782
- (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.
783
-
784
- Returns:
785
-
786
- Example:
787
-
788
- ```python
789
- >>> from transformers import AutoTokenizer, YiForCausalLM
790
-
791
- >>> model = YiForCausalLM.from_pretrained(PATH_TO_CONVERTED_WEIGHTS)
792
- >>> tokenizer = AutoTokenizer.from_pretrained(PATH_TO_CONVERTED_TOKENIZER)
793
-
794
- >>> prompt = "Hey, are you conscious? Can you talk to me?"
795
- >>> inputs = tokenizer(prompt, return_tensors="pt")
796
-
797
- >>> # Generate
798
- >>> generate_ids = model.generate(inputs.input_ids, max_length=30)
799
- >>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
800
- "Hey, are you conscious? Can you talk to me?\nI'm not conscious, but I can talk to you."
801
- ```"""
802
-
803
- output_attentions = (
804
- output_attentions
805
- if output_attentions is not None
806
- else self.config.output_attentions
807
- )
808
- output_hidden_states = (
809
- output_hidden_states
810
- if output_hidden_states is not None
811
- else self.config.output_hidden_states
812
- )
813
- return_dict = (
814
- return_dict if return_dict is not None else self.config.use_return_dict
815
- )
816
-
817
- # decoder outputs consists of (dec_features, layer_state, dec_hidden, dec_attn)
818
- outputs = self.model(
819
- input_ids=input_ids,
820
- attention_mask=attention_mask,
821
- position_ids=position_ids,
822
- past_key_values=past_key_values,
823
- inputs_embeds=inputs_embeds,
824
- use_cache=use_cache,
825
- output_attentions=output_attentions,
826
- output_hidden_states=output_hidden_states,
827
- return_dict=return_dict,
828
- )
829
-
830
- hidden_states = outputs[0]
831
- logits = self.lm_head(hidden_states)
832
-
833
- loss = None
834
- if labels is not None:
835
- # Shift so that tokens < n predict n
836
- shift_logits = logits[..., :-1, :].contiguous()
837
- shift_labels = labels[..., 1:].contiguous()
838
- # Flatten the tokens
839
- loss_fct = CrossEntropyLoss()
840
- shift_logits = shift_logits.view(-1, self.config.vocab_size)
841
- shift_labels = shift_labels.view(-1)
842
- # Enable model parallelism
843
- shift_labels = shift_labels.to(shift_logits.device)
844
- loss = loss_fct(shift_logits, shift_labels)
845
-
846
- if not return_dict:
847
- output = (logits,) + outputs[1:]
848
- return (loss,) + output if loss is not None else output
849
-
850
- return CausalLMOutputWithPast(
851
- loss=loss,
852
- logits=logits,
853
- past_key_values=outputs.past_key_values,
854
- hidden_states=outputs.hidden_states,
855
- attentions=outputs.attentions,
856
- )
857
-
858
- def prepare_inputs_for_generation(
859
- self,
860
- input_ids,
861
- past_key_values=None,
862
- attention_mask=None,
863
- inputs_embeds=None,
864
- **kwargs,
865
- ):
866
- if past_key_values:
867
- input_ids = input_ids[:, -1:]
868
-
869
- position_ids = kwargs.get("position_ids", None)
870
- if attention_mask is not None and position_ids is None:
871
- # create position_ids on the fly for batch generation
872
- position_ids = attention_mask.long().cumsum(-1) - 1
873
- position_ids.masked_fill_(attention_mask == 0, 1)
874
- if past_key_values:
875
- position_ids = position_ids[:, -1].unsqueeze(-1)
876
-
877
- # if `inputs_embeds` are passed, we only want to use them in the 1st generation step
878
- if inputs_embeds is not None and past_key_values is None:
879
- model_inputs = {"inputs_embeds": inputs_embeds}
880
- else:
881
- model_inputs = {"input_ids": input_ids}
882
-
883
- model_inputs.update(
884
- {
885
- "position_ids": position_ids,
886
- "past_key_values": past_key_values,
887
- "use_cache": kwargs.get("use_cache"),
888
- "attention_mask": attention_mask,
889
- }
890
- )
891
- return model_inputs
892
-
893
- @staticmethod
894
- def _reorder_cache(past_key_values, beam_idx):
895
- reordered_past = ()
896
- for layer_past in past_key_values:
897
- reordered_past += (
898
- tuple(
899
- past_state.index_select(0, beam_idx.to(past_state.device))
900
- for past_state in layer_past
901
- ),
902
- )
903
- return reordered_past
904
-
905
-
906
- @add_start_docstrings(
907
- """
908
- The Yi Model transformer with a sequence classification head on top (linear layer).
909
-
910
- [`YiForSequenceClassification`] uses the last token in order to do the classification, as other causal models
911
- (e.g. GPT-2) do.
912
-
913
- Since it does classification on the last token, it requires to know the position of the last token. If a
914
- `pad_token_id` is defined in the configuration, it finds the last token that is not a padding token in each row. If
915
- no `pad_token_id` is defined, it simply takes the last value in each row of the batch. Since it cannot guess the
916
- padding tokens when `inputs_embeds` are passed instead of `input_ids`, it does the same (take the last value in
917
- each row of the batch).
918
- """,
919
- Yi_START_DOCSTRING,
920
- )
921
- class YiForSequenceClassification(YiPreTrainedModel):
922
- def __init__(self, config):
923
- super().__init__(config)
924
- self.num_labels = config.num_labels
925
- self.model = YiModel(config)
926
- self.score = nn.Linear(config.hidden_size, self.num_labels, bias=False)
927
-
928
- # Initialize weights and apply final processing
929
- self.post_init()
930
-
931
- def get_input_embeddings(self):
932
- return self.model.embed_tokens
933
-
934
- def set_input_embeddings(self, value):
935
- self.model.embed_tokens = value
936
-
937
- @add_start_docstrings_to_model_forward(Yi_INPUTS_DOCSTRING)
938
- def forward(
939
- self,
940
- input_ids: torch.LongTensor = None,
941
- attention_mask: Optional[torch.Tensor] = None,
942
- position_ids: Optional[torch.LongTensor] = None,
943
- past_key_values: Optional[List[torch.FloatTensor]] = None,
944
- inputs_embeds: Optional[torch.FloatTensor] = None,
945
- labels: Optional[torch.LongTensor] = None,
946
- use_cache: Optional[bool] = None,
947
- output_attentions: Optional[bool] = None,
948
- output_hidden_states: Optional[bool] = None,
949
- return_dict: Optional[bool] = None,
950
- ) -> Union[Tuple, SequenceClassifierOutputWithPast]:
951
- r"""
952
- labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
953
- Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
954
- config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
955
- `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
956
- """
957
- return_dict = (
958
- return_dict if return_dict is not None else self.config.use_return_dict
959
- )
960
-
961
- transformer_outputs = self.model(
962
- input_ids,
963
- attention_mask=attention_mask,
964
- position_ids=position_ids,
965
- past_key_values=past_key_values,
966
- inputs_embeds=inputs_embeds,
967
- use_cache=use_cache,
968
- output_attentions=output_attentions,
969
- output_hidden_states=output_hidden_states,
970
- return_dict=return_dict,
971
- )
972
- hidden_states = transformer_outputs[0]
973
- logits = self.score(hidden_states)
974
-
975
- if input_ids is not None:
976
- batch_size = input_ids.shape[0]
977
- else:
978
- batch_size = inputs_embeds.shape[0]
979
-
980
- if self.config.pad_token_id is None and batch_size != 1:
981
- raise ValueError(
982
- "Cannot handle batch sizes > 1 if no padding token is defined."
983
- )
984
- if self.config.pad_token_id is None:
985
- sequence_lengths = -1
986
- else:
987
- if input_ids is not None:
988
- sequence_lengths = (
989
- torch.eq(input_ids, self.config.pad_token_id).long().argmax(-1) - 1
990
- ).to(logits.device)
991
- else:
992
- sequence_lengths = -1
993
-
994
- pooled_logits = logits[
995
- torch.arange(batch_size, device=logits.device), sequence_lengths
996
- ]
997
-
998
- loss = None
999
- if labels is not None:
1000
- labels = labels.to(logits.device)
1001
- if self.config.problem_type is None:
1002
- if self.num_labels == 1:
1003
- self.config.problem_type = "regression"
1004
- elif self.num_labels > 1 and (
1005
- labels.dtype == torch.long or labels.dtype == torch.int
1006
- ):
1007
- self.config.problem_type = "single_label_classification"
1008
- else:
1009
- self.config.problem_type = "multi_label_classification"
1010
-
1011
- if self.config.problem_type == "regression":
1012
- loss_fct = MSELoss()
1013
- if self.num_labels == 1:
1014
- loss = loss_fct(pooled_logits.squeeze(), labels.squeeze())
1015
- else:
1016
- loss = loss_fct(pooled_logits, labels)
1017
- elif self.config.problem_type == "single_label_classification":
1018
- loss_fct = CrossEntropyLoss()
1019
- loss = loss_fct(
1020
- pooled_logits.view(-1, self.num_labels), labels.view(-1)
1021
- )
1022
- elif self.config.problem_type == "multi_label_classification":
1023
- loss_fct = BCEWithLogitsLoss()
1024
- loss = loss_fct(pooled_logits, labels)
1025
- if not return_dict:
1026
- output = (pooled_logits,) + transformer_outputs[1:]
1027
- return ((loss,) + output) if loss is not None else output
1028
-
1029
- return SequenceClassifierOutputWithPast(
1030
- loss=loss,
1031
- logits=pooled_logits,
1032
- past_key_values=transformer_outputs.past_key_values,
1033
- hidden_states=transformer_outputs.hidden_states,
1034
- attentions=transformer_outputs.attentions,
1035
- )
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
pytorch_model-00001-of-00007.bin CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:64007f1c572f5257186aca4dc704f4cb0a98e8c72a4b65c85e6147eaed0965e4
3
- size 9975358639
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:720bd2b927357e43a041adaeb58cc7b4584c912345f25e05012ba826a334cc94
3
+ size 9975359207
pytorch_model-00002-of-00007.bin CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:4cc82deb289717eb98bc90e47765f2f707b8bf51ada18d105d4a373f79b77ad0
3
- size 9909327699
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:8d6b85c41e05366952863374abf1e578b36fa65b6b8ecc44414659e56db1949a
3
+ size 9909328339
pytorch_model-00003-of-00007.bin CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:e761a0b7aeb849f7eaa4a2d3b8987d7b488edd617e58a516f0d46dff354ff1a4
3
- size 9747818327
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:59227886da67d3b5c406c6ae2ec42dbc2fdd7fde5ca9355adc862673ee64548d
3
+ size 9747818967
pytorch_model-00004-of-00007.bin CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:9e2c19a10df7567c0815f2cbe6df6f2286842862c06c323018e8772f24bdbf40
3
- size 9747847651
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:afda7c3ebc40d59208a7c307af1ac41ff1e7007035a96700401d2e837720259a
3
+ size 9747848235
pytorch_model-00005-of-00007.bin CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:4f2c1df8691d3f862ebd8385c861654edbbcd34202e3338605704563c090d9a7
3
- size 9747847711
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:de049b5e1227c0a9cfe145067f25b3f6dc8313e56de22d6917709943b6fe6316
3
+ size 9747848295
pytorch_model-00006-of-00007.bin CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:9f233d4cdcc64cc3b82db04a662376556bd7ab30f1fe312c91ff49edd5ba00a5
3
- size 9938688579
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:4869849ef4e59218a3c2869fa0a33dffb46bbbfa00aefa7a60be346daa56391a
3
+ size 9938689163
pytorch_model-00007-of-00007.bin CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:32fa2425e1a870f39ba6cf12c839c60d89356e76aaea140635157525bc35bc33
3
- size 9711130049
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:98b9cff692d3ad4975eb36bb884fd774dcc77517e0bff30453535743db743354
3
+ size 9711130661
pytorch_model.bin.index.json CHANGED
@@ -5,8 +5,8 @@
5
  "weight_map": {
6
  "lm_head.weight": "pytorch_model-00007-of-00007.bin",
7
  "model.embed_tokens.weight": "pytorch_model-00001-of-00007.bin",
8
- "model.layers.0.ln1.weight": "pytorch_model-00001-of-00007.bin",
9
- "model.layers.0.ln2.weight": "pytorch_model-00001-of-00007.bin",
10
  "model.layers.0.mlp.down_proj.weight": "pytorch_model-00001-of-00007.bin",
11
  "model.layers.0.mlp.gate_proj.weight": "pytorch_model-00001-of-00007.bin",
12
  "model.layers.0.mlp.up_proj.weight": "pytorch_model-00001-of-00007.bin",
@@ -14,8 +14,8 @@
14
  "model.layers.0.self_attn.o_proj.weight": "pytorch_model-00001-of-00007.bin",
15
  "model.layers.0.self_attn.q_proj.weight": "pytorch_model-00001-of-00007.bin",
16
  "model.layers.0.self_attn.v_proj.weight": "pytorch_model-00001-of-00007.bin",
17
- "model.layers.1.ln1.weight": "pytorch_model-00001-of-00007.bin",
18
- "model.layers.1.ln2.weight": "pytorch_model-00001-of-00007.bin",
19
  "model.layers.1.mlp.down_proj.weight": "pytorch_model-00001-of-00007.bin",
20
  "model.layers.1.mlp.gate_proj.weight": "pytorch_model-00001-of-00007.bin",
21
  "model.layers.1.mlp.up_proj.weight": "pytorch_model-00001-of-00007.bin",
@@ -23,8 +23,8 @@
23
  "model.layers.1.self_attn.o_proj.weight": "pytorch_model-00001-of-00007.bin",
24
  "model.layers.1.self_attn.q_proj.weight": "pytorch_model-00001-of-00007.bin",
25
  "model.layers.1.self_attn.v_proj.weight": "pytorch_model-00001-of-00007.bin",
26
- "model.layers.10.ln1.weight": "pytorch_model-00002-of-00007.bin",
27
- "model.layers.10.ln2.weight": "pytorch_model-00002-of-00007.bin",
28
  "model.layers.10.mlp.down_proj.weight": "pytorch_model-00002-of-00007.bin",
29
  "model.layers.10.mlp.gate_proj.weight": "pytorch_model-00002-of-00007.bin",
30
  "model.layers.10.mlp.up_proj.weight": "pytorch_model-00002-of-00007.bin",
@@ -32,8 +32,8 @@
32
  "model.layers.10.self_attn.o_proj.weight": "pytorch_model-00002-of-00007.bin",
33
  "model.layers.10.self_attn.q_proj.weight": "pytorch_model-00002-of-00007.bin",
34
  "model.layers.10.self_attn.v_proj.weight": "pytorch_model-00002-of-00007.bin",
35
- "model.layers.11.ln1.weight": "pytorch_model-00002-of-00007.bin",
36
- "model.layers.11.ln2.weight": "pytorch_model-00002-of-00007.bin",
37
  "model.layers.11.mlp.down_proj.weight": "pytorch_model-00002-of-00007.bin",
38
  "model.layers.11.mlp.gate_proj.weight": "pytorch_model-00002-of-00007.bin",
39
  "model.layers.11.mlp.up_proj.weight": "pytorch_model-00002-of-00007.bin",
@@ -41,8 +41,8 @@
41
  "model.layers.11.self_attn.o_proj.weight": "pytorch_model-00002-of-00007.bin",
42
  "model.layers.11.self_attn.q_proj.weight": "pytorch_model-00002-of-00007.bin",
43
  "model.layers.11.self_attn.v_proj.weight": "pytorch_model-00002-of-00007.bin",
44
- "model.layers.12.ln1.weight": "pytorch_model-00002-of-00007.bin",
45
- "model.layers.12.ln2.weight": "pytorch_model-00002-of-00007.bin",
46
  "model.layers.12.mlp.down_proj.weight": "pytorch_model-00002-of-00007.bin",
47
  "model.layers.12.mlp.gate_proj.weight": "pytorch_model-00002-of-00007.bin",
48
  "model.layers.12.mlp.up_proj.weight": "pytorch_model-00002-of-00007.bin",
@@ -50,8 +50,8 @@
50
  "model.layers.12.self_attn.o_proj.weight": "pytorch_model-00002-of-00007.bin",
51
  "model.layers.12.self_attn.q_proj.weight": "pytorch_model-00002-of-00007.bin",
52
  "model.layers.12.self_attn.v_proj.weight": "pytorch_model-00002-of-00007.bin",
53
- "model.layers.13.ln1.weight": "pytorch_model-00002-of-00007.bin",
54
- "model.layers.13.ln2.weight": "pytorch_model-00002-of-00007.bin",
55
  "model.layers.13.mlp.down_proj.weight": "pytorch_model-00002-of-00007.bin",
56
  "model.layers.13.mlp.gate_proj.weight": "pytorch_model-00002-of-00007.bin",
57
  "model.layers.13.mlp.up_proj.weight": "pytorch_model-00002-of-00007.bin",
@@ -59,8 +59,8 @@
59
  "model.layers.13.self_attn.o_proj.weight": "pytorch_model-00002-of-00007.bin",
60
  "model.layers.13.self_attn.q_proj.weight": "pytorch_model-00002-of-00007.bin",
61
  "model.layers.13.self_attn.v_proj.weight": "pytorch_model-00002-of-00007.bin",
62
- "model.layers.14.ln1.weight": "pytorch_model-00002-of-00007.bin",
63
- "model.layers.14.ln2.weight": "pytorch_model-00002-of-00007.bin",
64
  "model.layers.14.mlp.down_proj.weight": "pytorch_model-00002-of-00007.bin",
65
  "model.layers.14.mlp.gate_proj.weight": "pytorch_model-00002-of-00007.bin",
66
  "model.layers.14.mlp.up_proj.weight": "pytorch_model-00002-of-00007.bin",
@@ -68,8 +68,8 @@
68
  "model.layers.14.self_attn.o_proj.weight": "pytorch_model-00002-of-00007.bin",
69
  "model.layers.14.self_attn.q_proj.weight": "pytorch_model-00002-of-00007.bin",
70
  "model.layers.14.self_attn.v_proj.weight": "pytorch_model-00002-of-00007.bin",
71
- "model.layers.15.ln1.weight": "pytorch_model-00002-of-00007.bin",
72
- "model.layers.15.ln2.weight": "pytorch_model-00002-of-00007.bin",
73
  "model.layers.15.mlp.down_proj.weight": "pytorch_model-00002-of-00007.bin",
74
  "model.layers.15.mlp.gate_proj.weight": "pytorch_model-00002-of-00007.bin",
75
  "model.layers.15.mlp.up_proj.weight": "pytorch_model-00002-of-00007.bin",
@@ -77,8 +77,8 @@
77
  "model.layers.15.self_attn.o_proj.weight": "pytorch_model-00002-of-00007.bin",
78
  "model.layers.15.self_attn.q_proj.weight": "pytorch_model-00002-of-00007.bin",
79
  "model.layers.15.self_attn.v_proj.weight": "pytorch_model-00002-of-00007.bin",
80
- "model.layers.16.ln1.weight": "pytorch_model-00002-of-00007.bin",
81
- "model.layers.16.ln2.weight": "pytorch_model-00002-of-00007.bin",
82
  "model.layers.16.mlp.down_proj.weight": "pytorch_model-00002-of-00007.bin",
83
  "model.layers.16.mlp.gate_proj.weight": "pytorch_model-00002-of-00007.bin",
84
  "model.layers.16.mlp.up_proj.weight": "pytorch_model-00002-of-00007.bin",
@@ -86,8 +86,8 @@
86
  "model.layers.16.self_attn.o_proj.weight": "pytorch_model-00002-of-00007.bin",
87
  "model.layers.16.self_attn.q_proj.weight": "pytorch_model-00002-of-00007.bin",
88
  "model.layers.16.self_attn.v_proj.weight": "pytorch_model-00002-of-00007.bin",
89
- "model.layers.17.ln1.weight": "pytorch_model-00003-of-00007.bin",
90
- "model.layers.17.ln2.weight": "pytorch_model-00003-of-00007.bin",
91
  "model.layers.17.mlp.down_proj.weight": "pytorch_model-00003-of-00007.bin",
92
  "model.layers.17.mlp.gate_proj.weight": "pytorch_model-00003-of-00007.bin",
93
  "model.layers.17.mlp.up_proj.weight": "pytorch_model-00003-of-00007.bin",
@@ -95,8 +95,8 @@
95
  "model.layers.17.self_attn.o_proj.weight": "pytorch_model-00003-of-00007.bin",
96
  "model.layers.17.self_attn.q_proj.weight": "pytorch_model-00003-of-00007.bin",
97
  "model.layers.17.self_attn.v_proj.weight": "pytorch_model-00003-of-00007.bin",
98
- "model.layers.18.ln1.weight": "pytorch_model-00003-of-00007.bin",
99
- "model.layers.18.ln2.weight": "pytorch_model-00003-of-00007.bin",
100
  "model.layers.18.mlp.down_proj.weight": "pytorch_model-00003-of-00007.bin",
101
  "model.layers.18.mlp.gate_proj.weight": "pytorch_model-00003-of-00007.bin",
102
  "model.layers.18.mlp.up_proj.weight": "pytorch_model-00003-of-00007.bin",
@@ -104,8 +104,8 @@
104
  "model.layers.18.self_attn.o_proj.weight": "pytorch_model-00003-of-00007.bin",
105
  "model.layers.18.self_attn.q_proj.weight": "pytorch_model-00003-of-00007.bin",
106
  "model.layers.18.self_attn.v_proj.weight": "pytorch_model-00003-of-00007.bin",
107
- "model.layers.19.ln1.weight": "pytorch_model-00003-of-00007.bin",
108
- "model.layers.19.ln2.weight": "pytorch_model-00003-of-00007.bin",
109
  "model.layers.19.mlp.down_proj.weight": "pytorch_model-00003-of-00007.bin",
110
  "model.layers.19.mlp.gate_proj.weight": "pytorch_model-00003-of-00007.bin",
111
  "model.layers.19.mlp.up_proj.weight": "pytorch_model-00003-of-00007.bin",
@@ -113,8 +113,8 @@
113
  "model.layers.19.self_attn.o_proj.weight": "pytorch_model-00003-of-00007.bin",
114
  "model.layers.19.self_attn.q_proj.weight": "pytorch_model-00003-of-00007.bin",
115
  "model.layers.19.self_attn.v_proj.weight": "pytorch_model-00003-of-00007.bin",
116
- "model.layers.2.ln1.weight": "pytorch_model-00001-of-00007.bin",
117
- "model.layers.2.ln2.weight": "pytorch_model-00001-of-00007.bin",
118
  "model.layers.2.mlp.down_proj.weight": "pytorch_model-00001-of-00007.bin",
119
  "model.layers.2.mlp.gate_proj.weight": "pytorch_model-00001-of-00007.bin",
120
  "model.layers.2.mlp.up_proj.weight": "pytorch_model-00001-of-00007.bin",
@@ -122,8 +122,8 @@
122
  "model.layers.2.self_attn.o_proj.weight": "pytorch_model-00001-of-00007.bin",
123
  "model.layers.2.self_attn.q_proj.weight": "pytorch_model-00001-of-00007.bin",
124
  "model.layers.2.self_attn.v_proj.weight": "pytorch_model-00001-of-00007.bin",
125
- "model.layers.20.ln1.weight": "pytorch_model-00003-of-00007.bin",
126
- "model.layers.20.ln2.weight": "pytorch_model-00003-of-00007.bin",
127
  "model.layers.20.mlp.down_proj.weight": "pytorch_model-00003-of-00007.bin",
128
  "model.layers.20.mlp.gate_proj.weight": "pytorch_model-00003-of-00007.bin",
129
  "model.layers.20.mlp.up_proj.weight": "pytorch_model-00003-of-00007.bin",
@@ -131,8 +131,8 @@
131
  "model.layers.20.self_attn.o_proj.weight": "pytorch_model-00003-of-00007.bin",
132
  "model.layers.20.self_attn.q_proj.weight": "pytorch_model-00003-of-00007.bin",
133
  "model.layers.20.self_attn.v_proj.weight": "pytorch_model-00003-of-00007.bin",
134
- "model.layers.21.ln1.weight": "pytorch_model-00003-of-00007.bin",
135
- "model.layers.21.ln2.weight": "pytorch_model-00003-of-00007.bin",
136
  "model.layers.21.mlp.down_proj.weight": "pytorch_model-00003-of-00007.bin",
137
  "model.layers.21.mlp.gate_proj.weight": "pytorch_model-00003-of-00007.bin",
138
  "model.layers.21.mlp.up_proj.weight": "pytorch_model-00003-of-00007.bin",
@@ -140,8 +140,8 @@
140
  "model.layers.21.self_attn.o_proj.weight": "pytorch_model-00003-of-00007.bin",
141
  "model.layers.21.self_attn.q_proj.weight": "pytorch_model-00003-of-00007.bin",
142
  "model.layers.21.self_attn.v_proj.weight": "pytorch_model-00003-of-00007.bin",
143
- "model.layers.22.ln1.weight": "pytorch_model-00003-of-00007.bin",
144
- "model.layers.22.ln2.weight": "pytorch_model-00003-of-00007.bin",
145
  "model.layers.22.mlp.down_proj.weight": "pytorch_model-00003-of-00007.bin",
146
  "model.layers.22.mlp.gate_proj.weight": "pytorch_model-00003-of-00007.bin",
147
  "model.layers.22.mlp.up_proj.weight": "pytorch_model-00003-of-00007.bin",
@@ -149,8 +149,8 @@
149
  "model.layers.22.self_attn.o_proj.weight": "pytorch_model-00003-of-00007.bin",
150
  "model.layers.22.self_attn.q_proj.weight": "pytorch_model-00003-of-00007.bin",
151
  "model.layers.22.self_attn.v_proj.weight": "pytorch_model-00003-of-00007.bin",
152
- "model.layers.23.ln1.weight": "pytorch_model-00003-of-00007.bin",
153
- "model.layers.23.ln2.weight": "pytorch_model-00003-of-00007.bin",
154
  "model.layers.23.mlp.down_proj.weight": "pytorch_model-00003-of-00007.bin",
155
  "model.layers.23.mlp.gate_proj.weight": "pytorch_model-00003-of-00007.bin",
156
  "model.layers.23.mlp.up_proj.weight": "pytorch_model-00003-of-00007.bin",
@@ -158,8 +158,8 @@
158
  "model.layers.23.self_attn.o_proj.weight": "pytorch_model-00003-of-00007.bin",
159
  "model.layers.23.self_attn.q_proj.weight": "pytorch_model-00003-of-00007.bin",
160
  "model.layers.23.self_attn.v_proj.weight": "pytorch_model-00003-of-00007.bin",
161
- "model.layers.24.ln1.weight": "pytorch_model-00003-of-00007.bin",
162
- "model.layers.24.ln2.weight": "pytorch_model-00003-of-00007.bin",
163
  "model.layers.24.mlp.down_proj.weight": "pytorch_model-00003-of-00007.bin",
164
  "model.layers.24.mlp.gate_proj.weight": "pytorch_model-00003-of-00007.bin",
165
  "model.layers.24.mlp.up_proj.weight": "pytorch_model-00003-of-00007.bin",
@@ -167,8 +167,8 @@
167
  "model.layers.24.self_attn.o_proj.weight": "pytorch_model-00003-of-00007.bin",
168
  "model.layers.24.self_attn.q_proj.weight": "pytorch_model-00003-of-00007.bin",
169
  "model.layers.24.self_attn.v_proj.weight": "pytorch_model-00003-of-00007.bin",
170
- "model.layers.25.ln1.weight": "pytorch_model-00004-of-00007.bin",
171
- "model.layers.25.ln2.weight": "pytorch_model-00004-of-00007.bin",
172
  "model.layers.25.mlp.down_proj.weight": "pytorch_model-00003-of-00007.bin",
173
  "model.layers.25.mlp.gate_proj.weight": "pytorch_model-00003-of-00007.bin",
174
  "model.layers.25.mlp.up_proj.weight": "pytorch_model-00004-of-00007.bin",
@@ -176,8 +176,8 @@
176
  "model.layers.25.self_attn.o_proj.weight": "pytorch_model-00003-of-00007.bin",
177
  "model.layers.25.self_attn.q_proj.weight": "pytorch_model-00003-of-00007.bin",
178
  "model.layers.25.self_attn.v_proj.weight": "pytorch_model-00003-of-00007.bin",
179
- "model.layers.26.ln1.weight": "pytorch_model-00004-of-00007.bin",
180
- "model.layers.26.ln2.weight": "pytorch_model-00004-of-00007.bin",
181
  "model.layers.26.mlp.down_proj.weight": "pytorch_model-00004-of-00007.bin",
182
  "model.layers.26.mlp.gate_proj.weight": "pytorch_model-00004-of-00007.bin",
183
  "model.layers.26.mlp.up_proj.weight": "pytorch_model-00004-of-00007.bin",
@@ -185,8 +185,8 @@
185
  "model.layers.26.self_attn.o_proj.weight": "pytorch_model-00004-of-00007.bin",
186
  "model.layers.26.self_attn.q_proj.weight": "pytorch_model-00004-of-00007.bin",
187
  "model.layers.26.self_attn.v_proj.weight": "pytorch_model-00004-of-00007.bin",
188
- "model.layers.27.ln1.weight": "pytorch_model-00004-of-00007.bin",
189
- "model.layers.27.ln2.weight": "pytorch_model-00004-of-00007.bin",
190
  "model.layers.27.mlp.down_proj.weight": "pytorch_model-00004-of-00007.bin",
191
  "model.layers.27.mlp.gate_proj.weight": "pytorch_model-00004-of-00007.bin",
192
  "model.layers.27.mlp.up_proj.weight": "pytorch_model-00004-of-00007.bin",
@@ -194,8 +194,8 @@
194
  "model.layers.27.self_attn.o_proj.weight": "pytorch_model-00004-of-00007.bin",
195
  "model.layers.27.self_attn.q_proj.weight": "pytorch_model-00004-of-00007.bin",
196
  "model.layers.27.self_attn.v_proj.weight": "pytorch_model-00004-of-00007.bin",
197
- "model.layers.28.ln1.weight": "pytorch_model-00004-of-00007.bin",
198
- "model.layers.28.ln2.weight": "pytorch_model-00004-of-00007.bin",
199
  "model.layers.28.mlp.down_proj.weight": "pytorch_model-00004-of-00007.bin",
200
  "model.layers.28.mlp.gate_proj.weight": "pytorch_model-00004-of-00007.bin",
201
  "model.layers.28.mlp.up_proj.weight": "pytorch_model-00004-of-00007.bin",
@@ -203,8 +203,8 @@
203
  "model.layers.28.self_attn.o_proj.weight": "pytorch_model-00004-of-00007.bin",
204
  "model.layers.28.self_attn.q_proj.weight": "pytorch_model-00004-of-00007.bin",
205
  "model.layers.28.self_attn.v_proj.weight": "pytorch_model-00004-of-00007.bin",
206
- "model.layers.29.ln1.weight": "pytorch_model-00004-of-00007.bin",
207
- "model.layers.29.ln2.weight": "pytorch_model-00004-of-00007.bin",
208
  "model.layers.29.mlp.down_proj.weight": "pytorch_model-00004-of-00007.bin",
209
  "model.layers.29.mlp.gate_proj.weight": "pytorch_model-00004-of-00007.bin",
210
  "model.layers.29.mlp.up_proj.weight": "pytorch_model-00004-of-00007.bin",
@@ -212,8 +212,8 @@
212
  "model.layers.29.self_attn.o_proj.weight": "pytorch_model-00004-of-00007.bin",
213
  "model.layers.29.self_attn.q_proj.weight": "pytorch_model-00004-of-00007.bin",
214
  "model.layers.29.self_attn.v_proj.weight": "pytorch_model-00004-of-00007.bin",
215
- "model.layers.3.ln1.weight": "pytorch_model-00001-of-00007.bin",
216
- "model.layers.3.ln2.weight": "pytorch_model-00001-of-00007.bin",
217
  "model.layers.3.mlp.down_proj.weight": "pytorch_model-00001-of-00007.bin",
218
  "model.layers.3.mlp.gate_proj.weight": "pytorch_model-00001-of-00007.bin",
219
  "model.layers.3.mlp.up_proj.weight": "pytorch_model-00001-of-00007.bin",
@@ -221,8 +221,8 @@
221
  "model.layers.3.self_attn.o_proj.weight": "pytorch_model-00001-of-00007.bin",
222
  "model.layers.3.self_attn.q_proj.weight": "pytorch_model-00001-of-00007.bin",
223
  "model.layers.3.self_attn.v_proj.weight": "pytorch_model-00001-of-00007.bin",
224
- "model.layers.30.ln1.weight": "pytorch_model-00004-of-00007.bin",
225
- "model.layers.30.ln2.weight": "pytorch_model-00004-of-00007.bin",
226
  "model.layers.30.mlp.down_proj.weight": "pytorch_model-00004-of-00007.bin",
227
  "model.layers.30.mlp.gate_proj.weight": "pytorch_model-00004-of-00007.bin",
228
  "model.layers.30.mlp.up_proj.weight": "pytorch_model-00004-of-00007.bin",
@@ -230,8 +230,8 @@
230
  "model.layers.30.self_attn.o_proj.weight": "pytorch_model-00004-of-00007.bin",
231
  "model.layers.30.self_attn.q_proj.weight": "pytorch_model-00004-of-00007.bin",
232
  "model.layers.30.self_attn.v_proj.weight": "pytorch_model-00004-of-00007.bin",
233
- "model.layers.31.ln1.weight": "pytorch_model-00004-of-00007.bin",
234
- "model.layers.31.ln2.weight": "pytorch_model-00004-of-00007.bin",
235
  "model.layers.31.mlp.down_proj.weight": "pytorch_model-00004-of-00007.bin",
236
  "model.layers.31.mlp.gate_proj.weight": "pytorch_model-00004-of-00007.bin",
237
  "model.layers.31.mlp.up_proj.weight": "pytorch_model-00004-of-00007.bin",
@@ -239,8 +239,8 @@
239
  "model.layers.31.self_attn.o_proj.weight": "pytorch_model-00004-of-00007.bin",
240
  "model.layers.31.self_attn.q_proj.weight": "pytorch_model-00004-of-00007.bin",
241
  "model.layers.31.self_attn.v_proj.weight": "pytorch_model-00004-of-00007.bin",
242
- "model.layers.32.ln1.weight": "pytorch_model-00004-of-00007.bin",
243
- "model.layers.32.ln2.weight": "pytorch_model-00004-of-00007.bin",
244
  "model.layers.32.mlp.down_proj.weight": "pytorch_model-00004-of-00007.bin",
245
  "model.layers.32.mlp.gate_proj.weight": "pytorch_model-00004-of-00007.bin",
246
  "model.layers.32.mlp.up_proj.weight": "pytorch_model-00004-of-00007.bin",
@@ -248,8 +248,8 @@
248
  "model.layers.32.self_attn.o_proj.weight": "pytorch_model-00004-of-00007.bin",
249
  "model.layers.32.self_attn.q_proj.weight": "pytorch_model-00004-of-00007.bin",
250
  "model.layers.32.self_attn.v_proj.weight": "pytorch_model-00004-of-00007.bin",
251
- "model.layers.33.ln1.weight": "pytorch_model-00004-of-00007.bin",
252
- "model.layers.33.ln2.weight": "pytorch_model-00004-of-00007.bin",
253
  "model.layers.33.mlp.down_proj.weight": "pytorch_model-00004-of-00007.bin",
254
  "model.layers.33.mlp.gate_proj.weight": "pytorch_model-00004-of-00007.bin",
255
  "model.layers.33.mlp.up_proj.weight": "pytorch_model-00004-of-00007.bin",
@@ -257,8 +257,8 @@
257
  "model.layers.33.self_attn.o_proj.weight": "pytorch_model-00004-of-00007.bin",
258
  "model.layers.33.self_attn.q_proj.weight": "pytorch_model-00004-of-00007.bin",
259
  "model.layers.33.self_attn.v_proj.weight": "pytorch_model-00004-of-00007.bin",
260
- "model.layers.34.ln1.weight": "pytorch_model-00005-of-00007.bin",
261
- "model.layers.34.ln2.weight": "pytorch_model-00005-of-00007.bin",
262
  "model.layers.34.mlp.down_proj.weight": "pytorch_model-00005-of-00007.bin",
263
  "model.layers.34.mlp.gate_proj.weight": "pytorch_model-00004-of-00007.bin",
264
  "model.layers.34.mlp.up_proj.weight": "pytorch_model-00005-of-00007.bin",
@@ -266,8 +266,8 @@
266
  "model.layers.34.self_attn.o_proj.weight": "pytorch_model-00004-of-00007.bin",
267
  "model.layers.34.self_attn.q_proj.weight": "pytorch_model-00004-of-00007.bin",
268
  "model.layers.34.self_attn.v_proj.weight": "pytorch_model-00004-of-00007.bin",
269
- "model.layers.35.ln1.weight": "pytorch_model-00005-of-00007.bin",
270
- "model.layers.35.ln2.weight": "pytorch_model-00005-of-00007.bin",
271
  "model.layers.35.mlp.down_proj.weight": "pytorch_model-00005-of-00007.bin",
272
  "model.layers.35.mlp.gate_proj.weight": "pytorch_model-00005-of-00007.bin",
273
  "model.layers.35.mlp.up_proj.weight": "pytorch_model-00005-of-00007.bin",
@@ -275,8 +275,8 @@
275
  "model.layers.35.self_attn.o_proj.weight": "pytorch_model-00005-of-00007.bin",
276
  "model.layers.35.self_attn.q_proj.weight": "pytorch_model-00005-of-00007.bin",
277
  "model.layers.35.self_attn.v_proj.weight": "pytorch_model-00005-of-00007.bin",
278
- "model.layers.36.ln1.weight": "pytorch_model-00005-of-00007.bin",
279
- "model.layers.36.ln2.weight": "pytorch_model-00005-of-00007.bin",
280
  "model.layers.36.mlp.down_proj.weight": "pytorch_model-00005-of-00007.bin",
281
  "model.layers.36.mlp.gate_proj.weight": "pytorch_model-00005-of-00007.bin",
282
  "model.layers.36.mlp.up_proj.weight": "pytorch_model-00005-of-00007.bin",
@@ -284,8 +284,8 @@
284
  "model.layers.36.self_attn.o_proj.weight": "pytorch_model-00005-of-00007.bin",
285
  "model.layers.36.self_attn.q_proj.weight": "pytorch_model-00005-of-00007.bin",
286
  "model.layers.36.self_attn.v_proj.weight": "pytorch_model-00005-of-00007.bin",
287
- "model.layers.37.ln1.weight": "pytorch_model-00005-of-00007.bin",
288
- "model.layers.37.ln2.weight": "pytorch_model-00005-of-00007.bin",
289
  "model.layers.37.mlp.down_proj.weight": "pytorch_model-00005-of-00007.bin",
290
  "model.layers.37.mlp.gate_proj.weight": "pytorch_model-00005-of-00007.bin",
291
  "model.layers.37.mlp.up_proj.weight": "pytorch_model-00005-of-00007.bin",
@@ -293,8 +293,8 @@
293
  "model.layers.37.self_attn.o_proj.weight": "pytorch_model-00005-of-00007.bin",
294
  "model.layers.37.self_attn.q_proj.weight": "pytorch_model-00005-of-00007.bin",
295
  "model.layers.37.self_attn.v_proj.weight": "pytorch_model-00005-of-00007.bin",
296
- "model.layers.38.ln1.weight": "pytorch_model-00005-of-00007.bin",
297
- "model.layers.38.ln2.weight": "pytorch_model-00005-of-00007.bin",
298
  "model.layers.38.mlp.down_proj.weight": "pytorch_model-00005-of-00007.bin",
299
  "model.layers.38.mlp.gate_proj.weight": "pytorch_model-00005-of-00007.bin",
300
  "model.layers.38.mlp.up_proj.weight": "pytorch_model-00005-of-00007.bin",
@@ -302,8 +302,8 @@
302
  "model.layers.38.self_attn.o_proj.weight": "pytorch_model-00005-of-00007.bin",
303
  "model.layers.38.self_attn.q_proj.weight": "pytorch_model-00005-of-00007.bin",
304
  "model.layers.38.self_attn.v_proj.weight": "pytorch_model-00005-of-00007.bin",
305
- "model.layers.39.ln1.weight": "pytorch_model-00005-of-00007.bin",
306
- "model.layers.39.ln2.weight": "pytorch_model-00005-of-00007.bin",
307
  "model.layers.39.mlp.down_proj.weight": "pytorch_model-00005-of-00007.bin",
308
  "model.layers.39.mlp.gate_proj.weight": "pytorch_model-00005-of-00007.bin",
309
  "model.layers.39.mlp.up_proj.weight": "pytorch_model-00005-of-00007.bin",
@@ -311,8 +311,8 @@
311
  "model.layers.39.self_attn.o_proj.weight": "pytorch_model-00005-of-00007.bin",
312
  "model.layers.39.self_attn.q_proj.weight": "pytorch_model-00005-of-00007.bin",
313
  "model.layers.39.self_attn.v_proj.weight": "pytorch_model-00005-of-00007.bin",
314
- "model.layers.4.ln1.weight": "pytorch_model-00001-of-00007.bin",
315
- "model.layers.4.ln2.weight": "pytorch_model-00001-of-00007.bin",
316
  "model.layers.4.mlp.down_proj.weight": "pytorch_model-00001-of-00007.bin",
317
  "model.layers.4.mlp.gate_proj.weight": "pytorch_model-00001-of-00007.bin",
318
  "model.layers.4.mlp.up_proj.weight": "pytorch_model-00001-of-00007.bin",
@@ -320,8 +320,8 @@
320
  "model.layers.4.self_attn.o_proj.weight": "pytorch_model-00001-of-00007.bin",
321
  "model.layers.4.self_attn.q_proj.weight": "pytorch_model-00001-of-00007.bin",
322
  "model.layers.4.self_attn.v_proj.weight": "pytorch_model-00001-of-00007.bin",
323
- "model.layers.40.ln1.weight": "pytorch_model-00005-of-00007.bin",
324
- "model.layers.40.ln2.weight": "pytorch_model-00005-of-00007.bin",
325
  "model.layers.40.mlp.down_proj.weight": "pytorch_model-00005-of-00007.bin",
326
  "model.layers.40.mlp.gate_proj.weight": "pytorch_model-00005-of-00007.bin",
327
  "model.layers.40.mlp.up_proj.weight": "pytorch_model-00005-of-00007.bin",
@@ -329,8 +329,8 @@
329
  "model.layers.40.self_attn.o_proj.weight": "pytorch_model-00005-of-00007.bin",
330
  "model.layers.40.self_attn.q_proj.weight": "pytorch_model-00005-of-00007.bin",
331
  "model.layers.40.self_attn.v_proj.weight": "pytorch_model-00005-of-00007.bin",
332
- "model.layers.41.ln1.weight": "pytorch_model-00005-of-00007.bin",
333
- "model.layers.41.ln2.weight": "pytorch_model-00005-of-00007.bin",
334
  "model.layers.41.mlp.down_proj.weight": "pytorch_model-00005-of-00007.bin",
335
  "model.layers.41.mlp.gate_proj.weight": "pytorch_model-00005-of-00007.bin",
336
  "model.layers.41.mlp.up_proj.weight": "pytorch_model-00005-of-00007.bin",
@@ -338,8 +338,8 @@
338
  "model.layers.41.self_attn.o_proj.weight": "pytorch_model-00005-of-00007.bin",
339
  "model.layers.41.self_attn.q_proj.weight": "pytorch_model-00005-of-00007.bin",
340
  "model.layers.41.self_attn.v_proj.weight": "pytorch_model-00005-of-00007.bin",
341
- "model.layers.42.ln1.weight": "pytorch_model-00005-of-00007.bin",
342
- "model.layers.42.ln2.weight": "pytorch_model-00005-of-00007.bin",
343
  "model.layers.42.mlp.down_proj.weight": "pytorch_model-00005-of-00007.bin",
344
  "model.layers.42.mlp.gate_proj.weight": "pytorch_model-00005-of-00007.bin",
345
  "model.layers.42.mlp.up_proj.weight": "pytorch_model-00005-of-00007.bin",
@@ -347,8 +347,8 @@
347
  "model.layers.42.self_attn.o_proj.weight": "pytorch_model-00005-of-00007.bin",
348
  "model.layers.42.self_attn.q_proj.weight": "pytorch_model-00005-of-00007.bin",
349
  "model.layers.42.self_attn.v_proj.weight": "pytorch_model-00005-of-00007.bin",
350
- "model.layers.43.ln1.weight": "pytorch_model-00006-of-00007.bin",
351
- "model.layers.43.ln2.weight": "pytorch_model-00006-of-00007.bin",
352
  "model.layers.43.mlp.down_proj.weight": "pytorch_model-00006-of-00007.bin",
353
  "model.layers.43.mlp.gate_proj.weight": "pytorch_model-00006-of-00007.bin",
354
  "model.layers.43.mlp.up_proj.weight": "pytorch_model-00006-of-00007.bin",
@@ -356,8 +356,8 @@
356
  "model.layers.43.self_attn.o_proj.weight": "pytorch_model-00005-of-00007.bin",
357
  "model.layers.43.self_attn.q_proj.weight": "pytorch_model-00005-of-00007.bin",
358
  "model.layers.43.self_attn.v_proj.weight": "pytorch_model-00005-of-00007.bin",
359
- "model.layers.44.ln1.weight": "pytorch_model-00006-of-00007.bin",
360
- "model.layers.44.ln2.weight": "pytorch_model-00006-of-00007.bin",
361
  "model.layers.44.mlp.down_proj.weight": "pytorch_model-00006-of-00007.bin",
362
  "model.layers.44.mlp.gate_proj.weight": "pytorch_model-00006-of-00007.bin",
363
  "model.layers.44.mlp.up_proj.weight": "pytorch_model-00006-of-00007.bin",
@@ -365,8 +365,8 @@
365
  "model.layers.44.self_attn.o_proj.weight": "pytorch_model-00006-of-00007.bin",
366
  "model.layers.44.self_attn.q_proj.weight": "pytorch_model-00006-of-00007.bin",
367
  "model.layers.44.self_attn.v_proj.weight": "pytorch_model-00006-of-00007.bin",
368
- "model.layers.45.ln1.weight": "pytorch_model-00006-of-00007.bin",
369
- "model.layers.45.ln2.weight": "pytorch_model-00006-of-00007.bin",
370
  "model.layers.45.mlp.down_proj.weight": "pytorch_model-00006-of-00007.bin",
371
  "model.layers.45.mlp.gate_proj.weight": "pytorch_model-00006-of-00007.bin",
372
  "model.layers.45.mlp.up_proj.weight": "pytorch_model-00006-of-00007.bin",
@@ -374,8 +374,8 @@
374
  "model.layers.45.self_attn.o_proj.weight": "pytorch_model-00006-of-00007.bin",
375
  "model.layers.45.self_attn.q_proj.weight": "pytorch_model-00006-of-00007.bin",
376
  "model.layers.45.self_attn.v_proj.weight": "pytorch_model-00006-of-00007.bin",
377
- "model.layers.46.ln1.weight": "pytorch_model-00006-of-00007.bin",
378
- "model.layers.46.ln2.weight": "pytorch_model-00006-of-00007.bin",
379
  "model.layers.46.mlp.down_proj.weight": "pytorch_model-00006-of-00007.bin",
380
  "model.layers.46.mlp.gate_proj.weight": "pytorch_model-00006-of-00007.bin",
381
  "model.layers.46.mlp.up_proj.weight": "pytorch_model-00006-of-00007.bin",
@@ -383,8 +383,8 @@
383
  "model.layers.46.self_attn.o_proj.weight": "pytorch_model-00006-of-00007.bin",
384
  "model.layers.46.self_attn.q_proj.weight": "pytorch_model-00006-of-00007.bin",
385
  "model.layers.46.self_attn.v_proj.weight": "pytorch_model-00006-of-00007.bin",
386
- "model.layers.47.ln1.weight": "pytorch_model-00006-of-00007.bin",
387
- "model.layers.47.ln2.weight": "pytorch_model-00006-of-00007.bin",
388
  "model.layers.47.mlp.down_proj.weight": "pytorch_model-00006-of-00007.bin",
389
  "model.layers.47.mlp.gate_proj.weight": "pytorch_model-00006-of-00007.bin",
390
  "model.layers.47.mlp.up_proj.weight": "pytorch_model-00006-of-00007.bin",
@@ -392,8 +392,8 @@
392
  "model.layers.47.self_attn.o_proj.weight": "pytorch_model-00006-of-00007.bin",
393
  "model.layers.47.self_attn.q_proj.weight": "pytorch_model-00006-of-00007.bin",
394
  "model.layers.47.self_attn.v_proj.weight": "pytorch_model-00006-of-00007.bin",
395
- "model.layers.48.ln1.weight": "pytorch_model-00006-of-00007.bin",
396
- "model.layers.48.ln2.weight": "pytorch_model-00006-of-00007.bin",
397
  "model.layers.48.mlp.down_proj.weight": "pytorch_model-00006-of-00007.bin",
398
  "model.layers.48.mlp.gate_proj.weight": "pytorch_model-00006-of-00007.bin",
399
  "model.layers.48.mlp.up_proj.weight": "pytorch_model-00006-of-00007.bin",
@@ -401,8 +401,8 @@
401
  "model.layers.48.self_attn.o_proj.weight": "pytorch_model-00006-of-00007.bin",
402
  "model.layers.48.self_attn.q_proj.weight": "pytorch_model-00006-of-00007.bin",
403
  "model.layers.48.self_attn.v_proj.weight": "pytorch_model-00006-of-00007.bin",
404
- "model.layers.49.ln1.weight": "pytorch_model-00006-of-00007.bin",
405
- "model.layers.49.ln2.weight": "pytorch_model-00006-of-00007.bin",
406
  "model.layers.49.mlp.down_proj.weight": "pytorch_model-00006-of-00007.bin",
407
  "model.layers.49.mlp.gate_proj.weight": "pytorch_model-00006-of-00007.bin",
408
  "model.layers.49.mlp.up_proj.weight": "pytorch_model-00006-of-00007.bin",
@@ -410,8 +410,8 @@
410
  "model.layers.49.self_attn.o_proj.weight": "pytorch_model-00006-of-00007.bin",
411
  "model.layers.49.self_attn.q_proj.weight": "pytorch_model-00006-of-00007.bin",
412
  "model.layers.49.self_attn.v_proj.weight": "pytorch_model-00006-of-00007.bin",
413
- "model.layers.5.ln1.weight": "pytorch_model-00001-of-00007.bin",
414
- "model.layers.5.ln2.weight": "pytorch_model-00001-of-00007.bin",
415
  "model.layers.5.mlp.down_proj.weight": "pytorch_model-00001-of-00007.bin",
416
  "model.layers.5.mlp.gate_proj.weight": "pytorch_model-00001-of-00007.bin",
417
  "model.layers.5.mlp.up_proj.weight": "pytorch_model-00001-of-00007.bin",
@@ -419,8 +419,8 @@
419
  "model.layers.5.self_attn.o_proj.weight": "pytorch_model-00001-of-00007.bin",
420
  "model.layers.5.self_attn.q_proj.weight": "pytorch_model-00001-of-00007.bin",
421
  "model.layers.5.self_attn.v_proj.weight": "pytorch_model-00001-of-00007.bin",
422
- "model.layers.50.ln1.weight": "pytorch_model-00006-of-00007.bin",
423
- "model.layers.50.ln2.weight": "pytorch_model-00006-of-00007.bin",
424
  "model.layers.50.mlp.down_proj.weight": "pytorch_model-00006-of-00007.bin",
425
  "model.layers.50.mlp.gate_proj.weight": "pytorch_model-00006-of-00007.bin",
426
  "model.layers.50.mlp.up_proj.weight": "pytorch_model-00006-of-00007.bin",
@@ -428,8 +428,8 @@
428
  "model.layers.50.self_attn.o_proj.weight": "pytorch_model-00006-of-00007.bin",
429
  "model.layers.50.self_attn.q_proj.weight": "pytorch_model-00006-of-00007.bin",
430
  "model.layers.50.self_attn.v_proj.weight": "pytorch_model-00006-of-00007.bin",
431
- "model.layers.51.ln1.weight": "pytorch_model-00006-of-00007.bin",
432
- "model.layers.51.ln2.weight": "pytorch_model-00006-of-00007.bin",
433
  "model.layers.51.mlp.down_proj.weight": "pytorch_model-00006-of-00007.bin",
434
  "model.layers.51.mlp.gate_proj.weight": "pytorch_model-00006-of-00007.bin",
435
  "model.layers.51.mlp.up_proj.weight": "pytorch_model-00006-of-00007.bin",
@@ -437,8 +437,8 @@
437
  "model.layers.51.self_attn.o_proj.weight": "pytorch_model-00006-of-00007.bin",
438
  "model.layers.51.self_attn.q_proj.weight": "pytorch_model-00006-of-00007.bin",
439
  "model.layers.51.self_attn.v_proj.weight": "pytorch_model-00006-of-00007.bin",
440
- "model.layers.52.ln1.weight": "pytorch_model-00007-of-00007.bin",
441
- "model.layers.52.ln2.weight": "pytorch_model-00007-of-00007.bin",
442
  "model.layers.52.mlp.down_proj.weight": "pytorch_model-00007-of-00007.bin",
443
  "model.layers.52.mlp.gate_proj.weight": "pytorch_model-00007-of-00007.bin",
444
  "model.layers.52.mlp.up_proj.weight": "pytorch_model-00007-of-00007.bin",
@@ -446,8 +446,8 @@
446
  "model.layers.52.self_attn.o_proj.weight": "pytorch_model-00007-of-00007.bin",
447
  "model.layers.52.self_attn.q_proj.weight": "pytorch_model-00006-of-00007.bin",
448
  "model.layers.52.self_attn.v_proj.weight": "pytorch_model-00006-of-00007.bin",
449
- "model.layers.53.ln1.weight": "pytorch_model-00007-of-00007.bin",
450
- "model.layers.53.ln2.weight": "pytorch_model-00007-of-00007.bin",
451
  "model.layers.53.mlp.down_proj.weight": "pytorch_model-00007-of-00007.bin",
452
  "model.layers.53.mlp.gate_proj.weight": "pytorch_model-00007-of-00007.bin",
453
  "model.layers.53.mlp.up_proj.weight": "pytorch_model-00007-of-00007.bin",
@@ -455,8 +455,8 @@
455
  "model.layers.53.self_attn.o_proj.weight": "pytorch_model-00007-of-00007.bin",
456
  "model.layers.53.self_attn.q_proj.weight": "pytorch_model-00007-of-00007.bin",
457
  "model.layers.53.self_attn.v_proj.weight": "pytorch_model-00007-of-00007.bin",
458
- "model.layers.54.ln1.weight": "pytorch_model-00007-of-00007.bin",
459
- "model.layers.54.ln2.weight": "pytorch_model-00007-of-00007.bin",
460
  "model.layers.54.mlp.down_proj.weight": "pytorch_model-00007-of-00007.bin",
461
  "model.layers.54.mlp.gate_proj.weight": "pytorch_model-00007-of-00007.bin",
462
  "model.layers.54.mlp.up_proj.weight": "pytorch_model-00007-of-00007.bin",
@@ -464,8 +464,8 @@
464
  "model.layers.54.self_attn.o_proj.weight": "pytorch_model-00007-of-00007.bin",
465
  "model.layers.54.self_attn.q_proj.weight": "pytorch_model-00007-of-00007.bin",
466
  "model.layers.54.self_attn.v_proj.weight": "pytorch_model-00007-of-00007.bin",
467
- "model.layers.55.ln1.weight": "pytorch_model-00007-of-00007.bin",
468
- "model.layers.55.ln2.weight": "pytorch_model-00007-of-00007.bin",
469
  "model.layers.55.mlp.down_proj.weight": "pytorch_model-00007-of-00007.bin",
470
  "model.layers.55.mlp.gate_proj.weight": "pytorch_model-00007-of-00007.bin",
471
  "model.layers.55.mlp.up_proj.weight": "pytorch_model-00007-of-00007.bin",
@@ -473,8 +473,8 @@
473
  "model.layers.55.self_attn.o_proj.weight": "pytorch_model-00007-of-00007.bin",
474
  "model.layers.55.self_attn.q_proj.weight": "pytorch_model-00007-of-00007.bin",
475
  "model.layers.55.self_attn.v_proj.weight": "pytorch_model-00007-of-00007.bin",
476
- "model.layers.56.ln1.weight": "pytorch_model-00007-of-00007.bin",
477
- "model.layers.56.ln2.weight": "pytorch_model-00007-of-00007.bin",
478
  "model.layers.56.mlp.down_proj.weight": "pytorch_model-00007-of-00007.bin",
479
  "model.layers.56.mlp.gate_proj.weight": "pytorch_model-00007-of-00007.bin",
480
  "model.layers.56.mlp.up_proj.weight": "pytorch_model-00007-of-00007.bin",
@@ -482,8 +482,8 @@
482
  "model.layers.56.self_attn.o_proj.weight": "pytorch_model-00007-of-00007.bin",
483
  "model.layers.56.self_attn.q_proj.weight": "pytorch_model-00007-of-00007.bin",
484
  "model.layers.56.self_attn.v_proj.weight": "pytorch_model-00007-of-00007.bin",
485
- "model.layers.57.ln1.weight": "pytorch_model-00007-of-00007.bin",
486
- "model.layers.57.ln2.weight": "pytorch_model-00007-of-00007.bin",
487
  "model.layers.57.mlp.down_proj.weight": "pytorch_model-00007-of-00007.bin",
488
  "model.layers.57.mlp.gate_proj.weight": "pytorch_model-00007-of-00007.bin",
489
  "model.layers.57.mlp.up_proj.weight": "pytorch_model-00007-of-00007.bin",
@@ -491,8 +491,8 @@
491
  "model.layers.57.self_attn.o_proj.weight": "pytorch_model-00007-of-00007.bin",
492
  "model.layers.57.self_attn.q_proj.weight": "pytorch_model-00007-of-00007.bin",
493
  "model.layers.57.self_attn.v_proj.weight": "pytorch_model-00007-of-00007.bin",
494
- "model.layers.58.ln1.weight": "pytorch_model-00007-of-00007.bin",
495
- "model.layers.58.ln2.weight": "pytorch_model-00007-of-00007.bin",
496
  "model.layers.58.mlp.down_proj.weight": "pytorch_model-00007-of-00007.bin",
497
  "model.layers.58.mlp.gate_proj.weight": "pytorch_model-00007-of-00007.bin",
498
  "model.layers.58.mlp.up_proj.weight": "pytorch_model-00007-of-00007.bin",
@@ -500,8 +500,8 @@
500
  "model.layers.58.self_attn.o_proj.weight": "pytorch_model-00007-of-00007.bin",
501
  "model.layers.58.self_attn.q_proj.weight": "pytorch_model-00007-of-00007.bin",
502
  "model.layers.58.self_attn.v_proj.weight": "pytorch_model-00007-of-00007.bin",
503
- "model.layers.59.ln1.weight": "pytorch_model-00007-of-00007.bin",
504
- "model.layers.59.ln2.weight": "pytorch_model-00007-of-00007.bin",
505
  "model.layers.59.mlp.down_proj.weight": "pytorch_model-00007-of-00007.bin",
506
  "model.layers.59.mlp.gate_proj.weight": "pytorch_model-00007-of-00007.bin",
507
  "model.layers.59.mlp.up_proj.weight": "pytorch_model-00007-of-00007.bin",
@@ -509,8 +509,8 @@
509
  "model.layers.59.self_attn.o_proj.weight": "pytorch_model-00007-of-00007.bin",
510
  "model.layers.59.self_attn.q_proj.weight": "pytorch_model-00007-of-00007.bin",
511
  "model.layers.59.self_attn.v_proj.weight": "pytorch_model-00007-of-00007.bin",
512
- "model.layers.6.ln1.weight": "pytorch_model-00001-of-00007.bin",
513
- "model.layers.6.ln2.weight": "pytorch_model-00001-of-00007.bin",
514
  "model.layers.6.mlp.down_proj.weight": "pytorch_model-00001-of-00007.bin",
515
  "model.layers.6.mlp.gate_proj.weight": "pytorch_model-00001-of-00007.bin",
516
  "model.layers.6.mlp.up_proj.weight": "pytorch_model-00001-of-00007.bin",
@@ -518,8 +518,8 @@
518
  "model.layers.6.self_attn.o_proj.weight": "pytorch_model-00001-of-00007.bin",
519
  "model.layers.6.self_attn.q_proj.weight": "pytorch_model-00001-of-00007.bin",
520
  "model.layers.6.self_attn.v_proj.weight": "pytorch_model-00001-of-00007.bin",
521
- "model.layers.7.ln1.weight": "pytorch_model-00001-of-00007.bin",
522
- "model.layers.7.ln2.weight": "pytorch_model-00001-of-00007.bin",
523
  "model.layers.7.mlp.down_proj.weight": "pytorch_model-00001-of-00007.bin",
524
  "model.layers.7.mlp.gate_proj.weight": "pytorch_model-00001-of-00007.bin",
525
  "model.layers.7.mlp.up_proj.weight": "pytorch_model-00001-of-00007.bin",
@@ -527,8 +527,8 @@
527
  "model.layers.7.self_attn.o_proj.weight": "pytorch_model-00001-of-00007.bin",
528
  "model.layers.7.self_attn.q_proj.weight": "pytorch_model-00001-of-00007.bin",
529
  "model.layers.7.self_attn.v_proj.weight": "pytorch_model-00001-of-00007.bin",
530
- "model.layers.8.ln1.weight": "pytorch_model-00002-of-00007.bin",
531
- "model.layers.8.ln2.weight": "pytorch_model-00002-of-00007.bin",
532
  "model.layers.8.mlp.down_proj.weight": "pytorch_model-00002-of-00007.bin",
533
  "model.layers.8.mlp.gate_proj.weight": "pytorch_model-00002-of-00007.bin",
534
  "model.layers.8.mlp.up_proj.weight": "pytorch_model-00002-of-00007.bin",
@@ -536,8 +536,8 @@
536
  "model.layers.8.self_attn.o_proj.weight": "pytorch_model-00002-of-00007.bin",
537
  "model.layers.8.self_attn.q_proj.weight": "pytorch_model-00001-of-00007.bin",
538
  "model.layers.8.self_attn.v_proj.weight": "pytorch_model-00001-of-00007.bin",
539
- "model.layers.9.ln1.weight": "pytorch_model-00002-of-00007.bin",
540
- "model.layers.9.ln2.weight": "pytorch_model-00002-of-00007.bin",
541
  "model.layers.9.mlp.down_proj.weight": "pytorch_model-00002-of-00007.bin",
542
  "model.layers.9.mlp.gate_proj.weight": "pytorch_model-00002-of-00007.bin",
543
  "model.layers.9.mlp.up_proj.weight": "pytorch_model-00002-of-00007.bin",
 
5
  "weight_map": {
6
  "lm_head.weight": "pytorch_model-00007-of-00007.bin",
7
  "model.embed_tokens.weight": "pytorch_model-00001-of-00007.bin",
8
+ "model.layers.0.input_layernorm.weight": "pytorch_model-00001-of-00007.bin",
9
+ "model.layers.0.post_attention_layernorm.weight": "pytorch_model-00001-of-00007.bin",
10
  "model.layers.0.mlp.down_proj.weight": "pytorch_model-00001-of-00007.bin",
11
  "model.layers.0.mlp.gate_proj.weight": "pytorch_model-00001-of-00007.bin",
12
  "model.layers.0.mlp.up_proj.weight": "pytorch_model-00001-of-00007.bin",
 
14
  "model.layers.0.self_attn.o_proj.weight": "pytorch_model-00001-of-00007.bin",
15
  "model.layers.0.self_attn.q_proj.weight": "pytorch_model-00001-of-00007.bin",
16
  "model.layers.0.self_attn.v_proj.weight": "pytorch_model-00001-of-00007.bin",
17
+ "model.layers.1.input_layernorm.weight": "pytorch_model-00001-of-00007.bin",
18
+ "model.layers.1.post_attention_layernorm.weight": "pytorch_model-00001-of-00007.bin",
19
  "model.layers.1.mlp.down_proj.weight": "pytorch_model-00001-of-00007.bin",
20
  "model.layers.1.mlp.gate_proj.weight": "pytorch_model-00001-of-00007.bin",
21
  "model.layers.1.mlp.up_proj.weight": "pytorch_model-00001-of-00007.bin",
 
23
  "model.layers.1.self_attn.o_proj.weight": "pytorch_model-00001-of-00007.bin",
24
  "model.layers.1.self_attn.q_proj.weight": "pytorch_model-00001-of-00007.bin",
25
  "model.layers.1.self_attn.v_proj.weight": "pytorch_model-00001-of-00007.bin",
26
+ "model.layers.10.input_layernorm.weight": "pytorch_model-00002-of-00007.bin",
27
+ "model.layers.10.post_attention_layernorm.weight": "pytorch_model-00002-of-00007.bin",
28
  "model.layers.10.mlp.down_proj.weight": "pytorch_model-00002-of-00007.bin",
29
  "model.layers.10.mlp.gate_proj.weight": "pytorch_model-00002-of-00007.bin",
30
  "model.layers.10.mlp.up_proj.weight": "pytorch_model-00002-of-00007.bin",
 
32
  "model.layers.10.self_attn.o_proj.weight": "pytorch_model-00002-of-00007.bin",
33
  "model.layers.10.self_attn.q_proj.weight": "pytorch_model-00002-of-00007.bin",
34
  "model.layers.10.self_attn.v_proj.weight": "pytorch_model-00002-of-00007.bin",
35
+ "model.layers.11.input_layernorm.weight": "pytorch_model-00002-of-00007.bin",
36
+ "model.layers.11.post_attention_layernorm.weight": "pytorch_model-00002-of-00007.bin",
37
  "model.layers.11.mlp.down_proj.weight": "pytorch_model-00002-of-00007.bin",
38
  "model.layers.11.mlp.gate_proj.weight": "pytorch_model-00002-of-00007.bin",
39
  "model.layers.11.mlp.up_proj.weight": "pytorch_model-00002-of-00007.bin",
 
41
  "model.layers.11.self_attn.o_proj.weight": "pytorch_model-00002-of-00007.bin",
42
  "model.layers.11.self_attn.q_proj.weight": "pytorch_model-00002-of-00007.bin",
43
  "model.layers.11.self_attn.v_proj.weight": "pytorch_model-00002-of-00007.bin",
44
+ "model.layers.12.input_layernorm.weight": "pytorch_model-00002-of-00007.bin",
45
+ "model.layers.12.post_attention_layernorm.weight": "pytorch_model-00002-of-00007.bin",
46
  "model.layers.12.mlp.down_proj.weight": "pytorch_model-00002-of-00007.bin",
47
  "model.layers.12.mlp.gate_proj.weight": "pytorch_model-00002-of-00007.bin",
48
  "model.layers.12.mlp.up_proj.weight": "pytorch_model-00002-of-00007.bin",
 
50
  "model.layers.12.self_attn.o_proj.weight": "pytorch_model-00002-of-00007.bin",
51
  "model.layers.12.self_attn.q_proj.weight": "pytorch_model-00002-of-00007.bin",
52
  "model.layers.12.self_attn.v_proj.weight": "pytorch_model-00002-of-00007.bin",
53
+ "model.layers.13.input_layernorm.weight": "pytorch_model-00002-of-00007.bin",
54
+ "model.layers.13.post_attention_layernorm.weight": "pytorch_model-00002-of-00007.bin",
55
  "model.layers.13.mlp.down_proj.weight": "pytorch_model-00002-of-00007.bin",
56
  "model.layers.13.mlp.gate_proj.weight": "pytorch_model-00002-of-00007.bin",
57
  "model.layers.13.mlp.up_proj.weight": "pytorch_model-00002-of-00007.bin",
 
59
  "model.layers.13.self_attn.o_proj.weight": "pytorch_model-00002-of-00007.bin",
60
  "model.layers.13.self_attn.q_proj.weight": "pytorch_model-00002-of-00007.bin",
61
  "model.layers.13.self_attn.v_proj.weight": "pytorch_model-00002-of-00007.bin",
62
+ "model.layers.14.input_layernorm.weight": "pytorch_model-00002-of-00007.bin",
63
+ "model.layers.14.post_attention_layernorm.weight": "pytorch_model-00002-of-00007.bin",
64
  "model.layers.14.mlp.down_proj.weight": "pytorch_model-00002-of-00007.bin",
65
  "model.layers.14.mlp.gate_proj.weight": "pytorch_model-00002-of-00007.bin",
66
  "model.layers.14.mlp.up_proj.weight": "pytorch_model-00002-of-00007.bin",
 
68
  "model.layers.14.self_attn.o_proj.weight": "pytorch_model-00002-of-00007.bin",
69
  "model.layers.14.self_attn.q_proj.weight": "pytorch_model-00002-of-00007.bin",
70
  "model.layers.14.self_attn.v_proj.weight": "pytorch_model-00002-of-00007.bin",
71
+ "model.layers.15.input_layernorm.weight": "pytorch_model-00002-of-00007.bin",
72
+ "model.layers.15.post_attention_layernorm.weight": "pytorch_model-00002-of-00007.bin",
73
  "model.layers.15.mlp.down_proj.weight": "pytorch_model-00002-of-00007.bin",
74
  "model.layers.15.mlp.gate_proj.weight": "pytorch_model-00002-of-00007.bin",
75
  "model.layers.15.mlp.up_proj.weight": "pytorch_model-00002-of-00007.bin",
 
77
  "model.layers.15.self_attn.o_proj.weight": "pytorch_model-00002-of-00007.bin",
78
  "model.layers.15.self_attn.q_proj.weight": "pytorch_model-00002-of-00007.bin",
79
  "model.layers.15.self_attn.v_proj.weight": "pytorch_model-00002-of-00007.bin",
80
+ "model.layers.16.input_layernorm.weight": "pytorch_model-00002-of-00007.bin",
81
+ "model.layers.16.post_attention_layernorm.weight": "pytorch_model-00002-of-00007.bin",
82
  "model.layers.16.mlp.down_proj.weight": "pytorch_model-00002-of-00007.bin",
83
  "model.layers.16.mlp.gate_proj.weight": "pytorch_model-00002-of-00007.bin",
84
  "model.layers.16.mlp.up_proj.weight": "pytorch_model-00002-of-00007.bin",
 
86
  "model.layers.16.self_attn.o_proj.weight": "pytorch_model-00002-of-00007.bin",
87
  "model.layers.16.self_attn.q_proj.weight": "pytorch_model-00002-of-00007.bin",
88
  "model.layers.16.self_attn.v_proj.weight": "pytorch_model-00002-of-00007.bin",
89
+ "model.layers.17.input_layernorm.weight": "pytorch_model-00003-of-00007.bin",
90
+ "model.layers.17.post_attention_layernorm.weight": "pytorch_model-00003-of-00007.bin",
91
  "model.layers.17.mlp.down_proj.weight": "pytorch_model-00003-of-00007.bin",
92
  "model.layers.17.mlp.gate_proj.weight": "pytorch_model-00003-of-00007.bin",
93
  "model.layers.17.mlp.up_proj.weight": "pytorch_model-00003-of-00007.bin",
 
95
  "model.layers.17.self_attn.o_proj.weight": "pytorch_model-00003-of-00007.bin",
96
  "model.layers.17.self_attn.q_proj.weight": "pytorch_model-00003-of-00007.bin",
97
  "model.layers.17.self_attn.v_proj.weight": "pytorch_model-00003-of-00007.bin",
98
+ "model.layers.18.input_layernorm.weight": "pytorch_model-00003-of-00007.bin",
99
+ "model.layers.18.post_attention_layernorm.weight": "pytorch_model-00003-of-00007.bin",
100
  "model.layers.18.mlp.down_proj.weight": "pytorch_model-00003-of-00007.bin",
101
  "model.layers.18.mlp.gate_proj.weight": "pytorch_model-00003-of-00007.bin",
102
  "model.layers.18.mlp.up_proj.weight": "pytorch_model-00003-of-00007.bin",
 
104
  "model.layers.18.self_attn.o_proj.weight": "pytorch_model-00003-of-00007.bin",
105
  "model.layers.18.self_attn.q_proj.weight": "pytorch_model-00003-of-00007.bin",
106
  "model.layers.18.self_attn.v_proj.weight": "pytorch_model-00003-of-00007.bin",
107
+ "model.layers.19.input_layernorm.weight": "pytorch_model-00003-of-00007.bin",
108
+ "model.layers.19.post_attention_layernorm.weight": "pytorch_model-00003-of-00007.bin",
109
  "model.layers.19.mlp.down_proj.weight": "pytorch_model-00003-of-00007.bin",
110
  "model.layers.19.mlp.gate_proj.weight": "pytorch_model-00003-of-00007.bin",
111
  "model.layers.19.mlp.up_proj.weight": "pytorch_model-00003-of-00007.bin",
 
113
  "model.layers.19.self_attn.o_proj.weight": "pytorch_model-00003-of-00007.bin",
114
  "model.layers.19.self_attn.q_proj.weight": "pytorch_model-00003-of-00007.bin",
115
  "model.layers.19.self_attn.v_proj.weight": "pytorch_model-00003-of-00007.bin",
116
+ "model.layers.2.input_layernorm.weight": "pytorch_model-00001-of-00007.bin",
117
+ "model.layers.2.post_attention_layernorm.weight": "pytorch_model-00001-of-00007.bin",
118
  "model.layers.2.mlp.down_proj.weight": "pytorch_model-00001-of-00007.bin",
119
  "model.layers.2.mlp.gate_proj.weight": "pytorch_model-00001-of-00007.bin",
120
  "model.layers.2.mlp.up_proj.weight": "pytorch_model-00001-of-00007.bin",
 
122
  "model.layers.2.self_attn.o_proj.weight": "pytorch_model-00001-of-00007.bin",
123
  "model.layers.2.self_attn.q_proj.weight": "pytorch_model-00001-of-00007.bin",
124
  "model.layers.2.self_attn.v_proj.weight": "pytorch_model-00001-of-00007.bin",
125
+ "model.layers.20.input_layernorm.weight": "pytorch_model-00003-of-00007.bin",
126
+ "model.layers.20.post_attention_layernorm.weight": "pytorch_model-00003-of-00007.bin",
127
  "model.layers.20.mlp.down_proj.weight": "pytorch_model-00003-of-00007.bin",
128
  "model.layers.20.mlp.gate_proj.weight": "pytorch_model-00003-of-00007.bin",
129
  "model.layers.20.mlp.up_proj.weight": "pytorch_model-00003-of-00007.bin",
 
131
  "model.layers.20.self_attn.o_proj.weight": "pytorch_model-00003-of-00007.bin",
132
  "model.layers.20.self_attn.q_proj.weight": "pytorch_model-00003-of-00007.bin",
133
  "model.layers.20.self_attn.v_proj.weight": "pytorch_model-00003-of-00007.bin",
134
+ "model.layers.21.input_layernorm.weight": "pytorch_model-00003-of-00007.bin",
135
+ "model.layers.21.post_attention_layernorm.weight": "pytorch_model-00003-of-00007.bin",
136
  "model.layers.21.mlp.down_proj.weight": "pytorch_model-00003-of-00007.bin",
137
  "model.layers.21.mlp.gate_proj.weight": "pytorch_model-00003-of-00007.bin",
138
  "model.layers.21.mlp.up_proj.weight": "pytorch_model-00003-of-00007.bin",
 
140
  "model.layers.21.self_attn.o_proj.weight": "pytorch_model-00003-of-00007.bin",
141
  "model.layers.21.self_attn.q_proj.weight": "pytorch_model-00003-of-00007.bin",
142
  "model.layers.21.self_attn.v_proj.weight": "pytorch_model-00003-of-00007.bin",
143
+ "model.layers.22.input_layernorm.weight": "pytorch_model-00003-of-00007.bin",
144
+ "model.layers.22.post_attention_layernorm.weight": "pytorch_model-00003-of-00007.bin",
145
  "model.layers.22.mlp.down_proj.weight": "pytorch_model-00003-of-00007.bin",
146
  "model.layers.22.mlp.gate_proj.weight": "pytorch_model-00003-of-00007.bin",
147
  "model.layers.22.mlp.up_proj.weight": "pytorch_model-00003-of-00007.bin",
 
149
  "model.layers.22.self_attn.o_proj.weight": "pytorch_model-00003-of-00007.bin",
150
  "model.layers.22.self_attn.q_proj.weight": "pytorch_model-00003-of-00007.bin",
151
  "model.layers.22.self_attn.v_proj.weight": "pytorch_model-00003-of-00007.bin",
152
+ "model.layers.23.input_layernorm.weight": "pytorch_model-00003-of-00007.bin",
153
+ "model.layers.23.post_attention_layernorm.weight": "pytorch_model-00003-of-00007.bin",
154
  "model.layers.23.mlp.down_proj.weight": "pytorch_model-00003-of-00007.bin",
155
  "model.layers.23.mlp.gate_proj.weight": "pytorch_model-00003-of-00007.bin",
156
  "model.layers.23.mlp.up_proj.weight": "pytorch_model-00003-of-00007.bin",
 
158
  "model.layers.23.self_attn.o_proj.weight": "pytorch_model-00003-of-00007.bin",
159
  "model.layers.23.self_attn.q_proj.weight": "pytorch_model-00003-of-00007.bin",
160
  "model.layers.23.self_attn.v_proj.weight": "pytorch_model-00003-of-00007.bin",
161
+ "model.layers.24.input_layernorm.weight": "pytorch_model-00003-of-00007.bin",
162
+ "model.layers.24.post_attention_layernorm.weight": "pytorch_model-00003-of-00007.bin",
163
  "model.layers.24.mlp.down_proj.weight": "pytorch_model-00003-of-00007.bin",
164
  "model.layers.24.mlp.gate_proj.weight": "pytorch_model-00003-of-00007.bin",
165
  "model.layers.24.mlp.up_proj.weight": "pytorch_model-00003-of-00007.bin",
 
167
  "model.layers.24.self_attn.o_proj.weight": "pytorch_model-00003-of-00007.bin",
168
  "model.layers.24.self_attn.q_proj.weight": "pytorch_model-00003-of-00007.bin",
169
  "model.layers.24.self_attn.v_proj.weight": "pytorch_model-00003-of-00007.bin",
170
+ "model.layers.25.input_layernorm.weight": "pytorch_model-00004-of-00007.bin",
171
+ "model.layers.25.post_attention_layernorm.weight": "pytorch_model-00004-of-00007.bin",
172
  "model.layers.25.mlp.down_proj.weight": "pytorch_model-00003-of-00007.bin",
173
  "model.layers.25.mlp.gate_proj.weight": "pytorch_model-00003-of-00007.bin",
174
  "model.layers.25.mlp.up_proj.weight": "pytorch_model-00004-of-00007.bin",
 
176
  "model.layers.25.self_attn.o_proj.weight": "pytorch_model-00003-of-00007.bin",
177
  "model.layers.25.self_attn.q_proj.weight": "pytorch_model-00003-of-00007.bin",
178
  "model.layers.25.self_attn.v_proj.weight": "pytorch_model-00003-of-00007.bin",
179
+ "model.layers.26.input_layernorm.weight": "pytorch_model-00004-of-00007.bin",
180
+ "model.layers.26.post_attention_layernorm.weight": "pytorch_model-00004-of-00007.bin",
181
  "model.layers.26.mlp.down_proj.weight": "pytorch_model-00004-of-00007.bin",
182
  "model.layers.26.mlp.gate_proj.weight": "pytorch_model-00004-of-00007.bin",
183
  "model.layers.26.mlp.up_proj.weight": "pytorch_model-00004-of-00007.bin",
 
185
  "model.layers.26.self_attn.o_proj.weight": "pytorch_model-00004-of-00007.bin",
186
  "model.layers.26.self_attn.q_proj.weight": "pytorch_model-00004-of-00007.bin",
187
  "model.layers.26.self_attn.v_proj.weight": "pytorch_model-00004-of-00007.bin",
188
+ "model.layers.27.input_layernorm.weight": "pytorch_model-00004-of-00007.bin",
189
+ "model.layers.27.post_attention_layernorm.weight": "pytorch_model-00004-of-00007.bin",
190
  "model.layers.27.mlp.down_proj.weight": "pytorch_model-00004-of-00007.bin",
191
  "model.layers.27.mlp.gate_proj.weight": "pytorch_model-00004-of-00007.bin",
192
  "model.layers.27.mlp.up_proj.weight": "pytorch_model-00004-of-00007.bin",
 
194
  "model.layers.27.self_attn.o_proj.weight": "pytorch_model-00004-of-00007.bin",
195
  "model.layers.27.self_attn.q_proj.weight": "pytorch_model-00004-of-00007.bin",
196
  "model.layers.27.self_attn.v_proj.weight": "pytorch_model-00004-of-00007.bin",
197
+ "model.layers.28.input_layernorm.weight": "pytorch_model-00004-of-00007.bin",
198
+ "model.layers.28.post_attention_layernorm.weight": "pytorch_model-00004-of-00007.bin",
199
  "model.layers.28.mlp.down_proj.weight": "pytorch_model-00004-of-00007.bin",
200
  "model.layers.28.mlp.gate_proj.weight": "pytorch_model-00004-of-00007.bin",
201
  "model.layers.28.mlp.up_proj.weight": "pytorch_model-00004-of-00007.bin",
 
203
  "model.layers.28.self_attn.o_proj.weight": "pytorch_model-00004-of-00007.bin",
204
  "model.layers.28.self_attn.q_proj.weight": "pytorch_model-00004-of-00007.bin",
205
  "model.layers.28.self_attn.v_proj.weight": "pytorch_model-00004-of-00007.bin",
206
+ "model.layers.29.input_layernorm.weight": "pytorch_model-00004-of-00007.bin",
207
+ "model.layers.29.post_attention_layernorm.weight": "pytorch_model-00004-of-00007.bin",
208
  "model.layers.29.mlp.down_proj.weight": "pytorch_model-00004-of-00007.bin",
209
  "model.layers.29.mlp.gate_proj.weight": "pytorch_model-00004-of-00007.bin",
210
  "model.layers.29.mlp.up_proj.weight": "pytorch_model-00004-of-00007.bin",
 
212
  "model.layers.29.self_attn.o_proj.weight": "pytorch_model-00004-of-00007.bin",
213
  "model.layers.29.self_attn.q_proj.weight": "pytorch_model-00004-of-00007.bin",
214
  "model.layers.29.self_attn.v_proj.weight": "pytorch_model-00004-of-00007.bin",
215
+ "model.layers.3.input_layernorm.weight": "pytorch_model-00001-of-00007.bin",
216
+ "model.layers.3.post_attention_layernorm.weight": "pytorch_model-00001-of-00007.bin",
217
  "model.layers.3.mlp.down_proj.weight": "pytorch_model-00001-of-00007.bin",
218
  "model.layers.3.mlp.gate_proj.weight": "pytorch_model-00001-of-00007.bin",
219
  "model.layers.3.mlp.up_proj.weight": "pytorch_model-00001-of-00007.bin",
 
221
  "model.layers.3.self_attn.o_proj.weight": "pytorch_model-00001-of-00007.bin",
222
  "model.layers.3.self_attn.q_proj.weight": "pytorch_model-00001-of-00007.bin",
223
  "model.layers.3.self_attn.v_proj.weight": "pytorch_model-00001-of-00007.bin",
224
+ "model.layers.30.input_layernorm.weight": "pytorch_model-00004-of-00007.bin",
225
+ "model.layers.30.post_attention_layernorm.weight": "pytorch_model-00004-of-00007.bin",
226
  "model.layers.30.mlp.down_proj.weight": "pytorch_model-00004-of-00007.bin",
227
  "model.layers.30.mlp.gate_proj.weight": "pytorch_model-00004-of-00007.bin",
228
  "model.layers.30.mlp.up_proj.weight": "pytorch_model-00004-of-00007.bin",
 
230
  "model.layers.30.self_attn.o_proj.weight": "pytorch_model-00004-of-00007.bin",
231
  "model.layers.30.self_attn.q_proj.weight": "pytorch_model-00004-of-00007.bin",
232
  "model.layers.30.self_attn.v_proj.weight": "pytorch_model-00004-of-00007.bin",
233
+ "model.layers.31.input_layernorm.weight": "pytorch_model-00004-of-00007.bin",
234
+ "model.layers.31.post_attention_layernorm.weight": "pytorch_model-00004-of-00007.bin",
235
  "model.layers.31.mlp.down_proj.weight": "pytorch_model-00004-of-00007.bin",
236
  "model.layers.31.mlp.gate_proj.weight": "pytorch_model-00004-of-00007.bin",
237
  "model.layers.31.mlp.up_proj.weight": "pytorch_model-00004-of-00007.bin",
 
239
  "model.layers.31.self_attn.o_proj.weight": "pytorch_model-00004-of-00007.bin",
240
  "model.layers.31.self_attn.q_proj.weight": "pytorch_model-00004-of-00007.bin",
241
  "model.layers.31.self_attn.v_proj.weight": "pytorch_model-00004-of-00007.bin",
242
+ "model.layers.32.input_layernorm.weight": "pytorch_model-00004-of-00007.bin",
243
+ "model.layers.32.post_attention_layernorm.weight": "pytorch_model-00004-of-00007.bin",
244
  "model.layers.32.mlp.down_proj.weight": "pytorch_model-00004-of-00007.bin",
245
  "model.layers.32.mlp.gate_proj.weight": "pytorch_model-00004-of-00007.bin",
246
  "model.layers.32.mlp.up_proj.weight": "pytorch_model-00004-of-00007.bin",
 
248
  "model.layers.32.self_attn.o_proj.weight": "pytorch_model-00004-of-00007.bin",
249
  "model.layers.32.self_attn.q_proj.weight": "pytorch_model-00004-of-00007.bin",
250
  "model.layers.32.self_attn.v_proj.weight": "pytorch_model-00004-of-00007.bin",
251
+ "model.layers.33.input_layernorm.weight": "pytorch_model-00004-of-00007.bin",
252
+ "model.layers.33.post_attention_layernorm.weight": "pytorch_model-00004-of-00007.bin",
253
  "model.layers.33.mlp.down_proj.weight": "pytorch_model-00004-of-00007.bin",
254
  "model.layers.33.mlp.gate_proj.weight": "pytorch_model-00004-of-00007.bin",
255
  "model.layers.33.mlp.up_proj.weight": "pytorch_model-00004-of-00007.bin",
 
257
  "model.layers.33.self_attn.o_proj.weight": "pytorch_model-00004-of-00007.bin",
258
  "model.layers.33.self_attn.q_proj.weight": "pytorch_model-00004-of-00007.bin",
259
  "model.layers.33.self_attn.v_proj.weight": "pytorch_model-00004-of-00007.bin",
260
+ "model.layers.34.input_layernorm.weight": "pytorch_model-00005-of-00007.bin",
261
+ "model.layers.34.post_attention_layernorm.weight": "pytorch_model-00005-of-00007.bin",
262
  "model.layers.34.mlp.down_proj.weight": "pytorch_model-00005-of-00007.bin",
263
  "model.layers.34.mlp.gate_proj.weight": "pytorch_model-00004-of-00007.bin",
264
  "model.layers.34.mlp.up_proj.weight": "pytorch_model-00005-of-00007.bin",
 
266
  "model.layers.34.self_attn.o_proj.weight": "pytorch_model-00004-of-00007.bin",
267
  "model.layers.34.self_attn.q_proj.weight": "pytorch_model-00004-of-00007.bin",
268
  "model.layers.34.self_attn.v_proj.weight": "pytorch_model-00004-of-00007.bin",
269
+ "model.layers.35.input_layernorm.weight": "pytorch_model-00005-of-00007.bin",
270
+ "model.layers.35.post_attention_layernorm.weight": "pytorch_model-00005-of-00007.bin",
271
  "model.layers.35.mlp.down_proj.weight": "pytorch_model-00005-of-00007.bin",
272
  "model.layers.35.mlp.gate_proj.weight": "pytorch_model-00005-of-00007.bin",
273
  "model.layers.35.mlp.up_proj.weight": "pytorch_model-00005-of-00007.bin",
 
275
  "model.layers.35.self_attn.o_proj.weight": "pytorch_model-00005-of-00007.bin",
276
  "model.layers.35.self_attn.q_proj.weight": "pytorch_model-00005-of-00007.bin",
277
  "model.layers.35.self_attn.v_proj.weight": "pytorch_model-00005-of-00007.bin",
278
+ "model.layers.36.input_layernorm.weight": "pytorch_model-00005-of-00007.bin",
279
+ "model.layers.36.post_attention_layernorm.weight": "pytorch_model-00005-of-00007.bin",
280
  "model.layers.36.mlp.down_proj.weight": "pytorch_model-00005-of-00007.bin",
281
  "model.layers.36.mlp.gate_proj.weight": "pytorch_model-00005-of-00007.bin",
282
  "model.layers.36.mlp.up_proj.weight": "pytorch_model-00005-of-00007.bin",
 
284
  "model.layers.36.self_attn.o_proj.weight": "pytorch_model-00005-of-00007.bin",
285
  "model.layers.36.self_attn.q_proj.weight": "pytorch_model-00005-of-00007.bin",
286
  "model.layers.36.self_attn.v_proj.weight": "pytorch_model-00005-of-00007.bin",
287
+ "model.layers.37.input_layernorm.weight": "pytorch_model-00005-of-00007.bin",
288
+ "model.layers.37.post_attention_layernorm.weight": "pytorch_model-00005-of-00007.bin",
289
  "model.layers.37.mlp.down_proj.weight": "pytorch_model-00005-of-00007.bin",
290
  "model.layers.37.mlp.gate_proj.weight": "pytorch_model-00005-of-00007.bin",
291
  "model.layers.37.mlp.up_proj.weight": "pytorch_model-00005-of-00007.bin",
 
293
  "model.layers.37.self_attn.o_proj.weight": "pytorch_model-00005-of-00007.bin",
294
  "model.layers.37.self_attn.q_proj.weight": "pytorch_model-00005-of-00007.bin",
295
  "model.layers.37.self_attn.v_proj.weight": "pytorch_model-00005-of-00007.bin",
296
+ "model.layers.38.input_layernorm.weight": "pytorch_model-00005-of-00007.bin",
297
+ "model.layers.38.post_attention_layernorm.weight": "pytorch_model-00005-of-00007.bin",
298
  "model.layers.38.mlp.down_proj.weight": "pytorch_model-00005-of-00007.bin",
299
  "model.layers.38.mlp.gate_proj.weight": "pytorch_model-00005-of-00007.bin",
300
  "model.layers.38.mlp.up_proj.weight": "pytorch_model-00005-of-00007.bin",
 
302
  "model.layers.38.self_attn.o_proj.weight": "pytorch_model-00005-of-00007.bin",
303
  "model.layers.38.self_attn.q_proj.weight": "pytorch_model-00005-of-00007.bin",
304
  "model.layers.38.self_attn.v_proj.weight": "pytorch_model-00005-of-00007.bin",
305
+ "model.layers.39.input_layernorm.weight": "pytorch_model-00005-of-00007.bin",
306
+ "model.layers.39.post_attention_layernorm.weight": "pytorch_model-00005-of-00007.bin",
307
  "model.layers.39.mlp.down_proj.weight": "pytorch_model-00005-of-00007.bin",
308
  "model.layers.39.mlp.gate_proj.weight": "pytorch_model-00005-of-00007.bin",
309
  "model.layers.39.mlp.up_proj.weight": "pytorch_model-00005-of-00007.bin",
 
311
  "model.layers.39.self_attn.o_proj.weight": "pytorch_model-00005-of-00007.bin",
312
  "model.layers.39.self_attn.q_proj.weight": "pytorch_model-00005-of-00007.bin",
313
  "model.layers.39.self_attn.v_proj.weight": "pytorch_model-00005-of-00007.bin",
314
+ "model.layers.4.input_layernorm.weight": "pytorch_model-00001-of-00007.bin",
315
+ "model.layers.4.post_attention_layernorm.weight": "pytorch_model-00001-of-00007.bin",
316
  "model.layers.4.mlp.down_proj.weight": "pytorch_model-00001-of-00007.bin",
317
  "model.layers.4.mlp.gate_proj.weight": "pytorch_model-00001-of-00007.bin",
318
  "model.layers.4.mlp.up_proj.weight": "pytorch_model-00001-of-00007.bin",
 
320
  "model.layers.4.self_attn.o_proj.weight": "pytorch_model-00001-of-00007.bin",
321
  "model.layers.4.self_attn.q_proj.weight": "pytorch_model-00001-of-00007.bin",
322
  "model.layers.4.self_attn.v_proj.weight": "pytorch_model-00001-of-00007.bin",
323
+ "model.layers.40.input_layernorm.weight": "pytorch_model-00005-of-00007.bin",
324
+ "model.layers.40.post_attention_layernorm.weight": "pytorch_model-00005-of-00007.bin",
325
  "model.layers.40.mlp.down_proj.weight": "pytorch_model-00005-of-00007.bin",
326
  "model.layers.40.mlp.gate_proj.weight": "pytorch_model-00005-of-00007.bin",
327
  "model.layers.40.mlp.up_proj.weight": "pytorch_model-00005-of-00007.bin",
 
329
  "model.layers.40.self_attn.o_proj.weight": "pytorch_model-00005-of-00007.bin",
330
  "model.layers.40.self_attn.q_proj.weight": "pytorch_model-00005-of-00007.bin",
331
  "model.layers.40.self_attn.v_proj.weight": "pytorch_model-00005-of-00007.bin",
332
+ "model.layers.41.input_layernorm.weight": "pytorch_model-00005-of-00007.bin",
333
+ "model.layers.41.post_attention_layernorm.weight": "pytorch_model-00005-of-00007.bin",
334
  "model.layers.41.mlp.down_proj.weight": "pytorch_model-00005-of-00007.bin",
335
  "model.layers.41.mlp.gate_proj.weight": "pytorch_model-00005-of-00007.bin",
336
  "model.layers.41.mlp.up_proj.weight": "pytorch_model-00005-of-00007.bin",
 
338
  "model.layers.41.self_attn.o_proj.weight": "pytorch_model-00005-of-00007.bin",
339
  "model.layers.41.self_attn.q_proj.weight": "pytorch_model-00005-of-00007.bin",
340
  "model.layers.41.self_attn.v_proj.weight": "pytorch_model-00005-of-00007.bin",
341
+ "model.layers.42.input_layernorm.weight": "pytorch_model-00005-of-00007.bin",
342
+ "model.layers.42.post_attention_layernorm.weight": "pytorch_model-00005-of-00007.bin",
343
  "model.layers.42.mlp.down_proj.weight": "pytorch_model-00005-of-00007.bin",
344
  "model.layers.42.mlp.gate_proj.weight": "pytorch_model-00005-of-00007.bin",
345
  "model.layers.42.mlp.up_proj.weight": "pytorch_model-00005-of-00007.bin",
 
347
  "model.layers.42.self_attn.o_proj.weight": "pytorch_model-00005-of-00007.bin",
348
  "model.layers.42.self_attn.q_proj.weight": "pytorch_model-00005-of-00007.bin",
349
  "model.layers.42.self_attn.v_proj.weight": "pytorch_model-00005-of-00007.bin",
350
+ "model.layers.43.input_layernorm.weight": "pytorch_model-00006-of-00007.bin",
351
+ "model.layers.43.post_attention_layernorm.weight": "pytorch_model-00006-of-00007.bin",
352
  "model.layers.43.mlp.down_proj.weight": "pytorch_model-00006-of-00007.bin",
353
  "model.layers.43.mlp.gate_proj.weight": "pytorch_model-00006-of-00007.bin",
354
  "model.layers.43.mlp.up_proj.weight": "pytorch_model-00006-of-00007.bin",
 
356
  "model.layers.43.self_attn.o_proj.weight": "pytorch_model-00005-of-00007.bin",
357
  "model.layers.43.self_attn.q_proj.weight": "pytorch_model-00005-of-00007.bin",
358
  "model.layers.43.self_attn.v_proj.weight": "pytorch_model-00005-of-00007.bin",
359
+ "model.layers.44.input_layernorm.weight": "pytorch_model-00006-of-00007.bin",
360
+ "model.layers.44.post_attention_layernorm.weight": "pytorch_model-00006-of-00007.bin",
361
  "model.layers.44.mlp.down_proj.weight": "pytorch_model-00006-of-00007.bin",
362
  "model.layers.44.mlp.gate_proj.weight": "pytorch_model-00006-of-00007.bin",
363
  "model.layers.44.mlp.up_proj.weight": "pytorch_model-00006-of-00007.bin",
 
365
  "model.layers.44.self_attn.o_proj.weight": "pytorch_model-00006-of-00007.bin",
366
  "model.layers.44.self_attn.q_proj.weight": "pytorch_model-00006-of-00007.bin",
367
  "model.layers.44.self_attn.v_proj.weight": "pytorch_model-00006-of-00007.bin",
368
+ "model.layers.45.input_layernorm.weight": "pytorch_model-00006-of-00007.bin",
369
+ "model.layers.45.post_attention_layernorm.weight": "pytorch_model-00006-of-00007.bin",
370
  "model.layers.45.mlp.down_proj.weight": "pytorch_model-00006-of-00007.bin",
371
  "model.layers.45.mlp.gate_proj.weight": "pytorch_model-00006-of-00007.bin",
372
  "model.layers.45.mlp.up_proj.weight": "pytorch_model-00006-of-00007.bin",
 
374
  "model.layers.45.self_attn.o_proj.weight": "pytorch_model-00006-of-00007.bin",
375
  "model.layers.45.self_attn.q_proj.weight": "pytorch_model-00006-of-00007.bin",
376
  "model.layers.45.self_attn.v_proj.weight": "pytorch_model-00006-of-00007.bin",
377
+ "model.layers.46.input_layernorm.weight": "pytorch_model-00006-of-00007.bin",
378
+ "model.layers.46.post_attention_layernorm.weight": "pytorch_model-00006-of-00007.bin",
379
  "model.layers.46.mlp.down_proj.weight": "pytorch_model-00006-of-00007.bin",
380
  "model.layers.46.mlp.gate_proj.weight": "pytorch_model-00006-of-00007.bin",
381
  "model.layers.46.mlp.up_proj.weight": "pytorch_model-00006-of-00007.bin",
 
383
  "model.layers.46.self_attn.o_proj.weight": "pytorch_model-00006-of-00007.bin",
384
  "model.layers.46.self_attn.q_proj.weight": "pytorch_model-00006-of-00007.bin",
385
  "model.layers.46.self_attn.v_proj.weight": "pytorch_model-00006-of-00007.bin",
386
+ "model.layers.47.input_layernorm.weight": "pytorch_model-00006-of-00007.bin",
387
+ "model.layers.47.post_attention_layernorm.weight": "pytorch_model-00006-of-00007.bin",
388
  "model.layers.47.mlp.down_proj.weight": "pytorch_model-00006-of-00007.bin",
389
  "model.layers.47.mlp.gate_proj.weight": "pytorch_model-00006-of-00007.bin",
390
  "model.layers.47.mlp.up_proj.weight": "pytorch_model-00006-of-00007.bin",
 
392
  "model.layers.47.self_attn.o_proj.weight": "pytorch_model-00006-of-00007.bin",
393
  "model.layers.47.self_attn.q_proj.weight": "pytorch_model-00006-of-00007.bin",
394
  "model.layers.47.self_attn.v_proj.weight": "pytorch_model-00006-of-00007.bin",
395
+ "model.layers.48.input_layernorm.weight": "pytorch_model-00006-of-00007.bin",
396
+ "model.layers.48.post_attention_layernorm.weight": "pytorch_model-00006-of-00007.bin",
397
  "model.layers.48.mlp.down_proj.weight": "pytorch_model-00006-of-00007.bin",
398
  "model.layers.48.mlp.gate_proj.weight": "pytorch_model-00006-of-00007.bin",
399
  "model.layers.48.mlp.up_proj.weight": "pytorch_model-00006-of-00007.bin",
 
401
  "model.layers.48.self_attn.o_proj.weight": "pytorch_model-00006-of-00007.bin",
402
  "model.layers.48.self_attn.q_proj.weight": "pytorch_model-00006-of-00007.bin",
403
  "model.layers.48.self_attn.v_proj.weight": "pytorch_model-00006-of-00007.bin",
404
+ "model.layers.49.input_layernorm.weight": "pytorch_model-00006-of-00007.bin",
405
+ "model.layers.49.post_attention_layernorm.weight": "pytorch_model-00006-of-00007.bin",
406
  "model.layers.49.mlp.down_proj.weight": "pytorch_model-00006-of-00007.bin",
407
  "model.layers.49.mlp.gate_proj.weight": "pytorch_model-00006-of-00007.bin",
408
  "model.layers.49.mlp.up_proj.weight": "pytorch_model-00006-of-00007.bin",
 
410
  "model.layers.49.self_attn.o_proj.weight": "pytorch_model-00006-of-00007.bin",
411
  "model.layers.49.self_attn.q_proj.weight": "pytorch_model-00006-of-00007.bin",
412
  "model.layers.49.self_attn.v_proj.weight": "pytorch_model-00006-of-00007.bin",
413
+ "model.layers.5.input_layernorm.weight": "pytorch_model-00001-of-00007.bin",
414
+ "model.layers.5.post_attention_layernorm.weight": "pytorch_model-00001-of-00007.bin",
415
  "model.layers.5.mlp.down_proj.weight": "pytorch_model-00001-of-00007.bin",
416
  "model.layers.5.mlp.gate_proj.weight": "pytorch_model-00001-of-00007.bin",
417
  "model.layers.5.mlp.up_proj.weight": "pytorch_model-00001-of-00007.bin",
 
419
  "model.layers.5.self_attn.o_proj.weight": "pytorch_model-00001-of-00007.bin",
420
  "model.layers.5.self_attn.q_proj.weight": "pytorch_model-00001-of-00007.bin",
421
  "model.layers.5.self_attn.v_proj.weight": "pytorch_model-00001-of-00007.bin",
422
+ "model.layers.50.input_layernorm.weight": "pytorch_model-00006-of-00007.bin",
423
+ "model.layers.50.post_attention_layernorm.weight": "pytorch_model-00006-of-00007.bin",
424
  "model.layers.50.mlp.down_proj.weight": "pytorch_model-00006-of-00007.bin",
425
  "model.layers.50.mlp.gate_proj.weight": "pytorch_model-00006-of-00007.bin",
426
  "model.layers.50.mlp.up_proj.weight": "pytorch_model-00006-of-00007.bin",
 
428
  "model.layers.50.self_attn.o_proj.weight": "pytorch_model-00006-of-00007.bin",
429
  "model.layers.50.self_attn.q_proj.weight": "pytorch_model-00006-of-00007.bin",
430
  "model.layers.50.self_attn.v_proj.weight": "pytorch_model-00006-of-00007.bin",
431
+ "model.layers.51.input_layernorm.weight": "pytorch_model-00006-of-00007.bin",
432
+ "model.layers.51.post_attention_layernorm.weight": "pytorch_model-00006-of-00007.bin",
433
  "model.layers.51.mlp.down_proj.weight": "pytorch_model-00006-of-00007.bin",
434
  "model.layers.51.mlp.gate_proj.weight": "pytorch_model-00006-of-00007.bin",
435
  "model.layers.51.mlp.up_proj.weight": "pytorch_model-00006-of-00007.bin",
 
437
  "model.layers.51.self_attn.o_proj.weight": "pytorch_model-00006-of-00007.bin",
438
  "model.layers.51.self_attn.q_proj.weight": "pytorch_model-00006-of-00007.bin",
439
  "model.layers.51.self_attn.v_proj.weight": "pytorch_model-00006-of-00007.bin",
440
+ "model.layers.52.input_layernorm.weight": "pytorch_model-00007-of-00007.bin",
441
+ "model.layers.52.post_attention_layernorm.weight": "pytorch_model-00007-of-00007.bin",
442
  "model.layers.52.mlp.down_proj.weight": "pytorch_model-00007-of-00007.bin",
443
  "model.layers.52.mlp.gate_proj.weight": "pytorch_model-00007-of-00007.bin",
444
  "model.layers.52.mlp.up_proj.weight": "pytorch_model-00007-of-00007.bin",
 
446
  "model.layers.52.self_attn.o_proj.weight": "pytorch_model-00007-of-00007.bin",
447
  "model.layers.52.self_attn.q_proj.weight": "pytorch_model-00006-of-00007.bin",
448
  "model.layers.52.self_attn.v_proj.weight": "pytorch_model-00006-of-00007.bin",
449
+ "model.layers.53.input_layernorm.weight": "pytorch_model-00007-of-00007.bin",
450
+ "model.layers.53.post_attention_layernorm.weight": "pytorch_model-00007-of-00007.bin",
451
  "model.layers.53.mlp.down_proj.weight": "pytorch_model-00007-of-00007.bin",
452
  "model.layers.53.mlp.gate_proj.weight": "pytorch_model-00007-of-00007.bin",
453
  "model.layers.53.mlp.up_proj.weight": "pytorch_model-00007-of-00007.bin",
 
455
  "model.layers.53.self_attn.o_proj.weight": "pytorch_model-00007-of-00007.bin",
456
  "model.layers.53.self_attn.q_proj.weight": "pytorch_model-00007-of-00007.bin",
457
  "model.layers.53.self_attn.v_proj.weight": "pytorch_model-00007-of-00007.bin",
458
+ "model.layers.54.input_layernorm.weight": "pytorch_model-00007-of-00007.bin",
459
+ "model.layers.54.post_attention_layernorm.weight": "pytorch_model-00007-of-00007.bin",
460
  "model.layers.54.mlp.down_proj.weight": "pytorch_model-00007-of-00007.bin",
461
  "model.layers.54.mlp.gate_proj.weight": "pytorch_model-00007-of-00007.bin",
462
  "model.layers.54.mlp.up_proj.weight": "pytorch_model-00007-of-00007.bin",
 
464
  "model.layers.54.self_attn.o_proj.weight": "pytorch_model-00007-of-00007.bin",
465
  "model.layers.54.self_attn.q_proj.weight": "pytorch_model-00007-of-00007.bin",
466
  "model.layers.54.self_attn.v_proj.weight": "pytorch_model-00007-of-00007.bin",
467
+ "model.layers.55.input_layernorm.weight": "pytorch_model-00007-of-00007.bin",
468
+ "model.layers.55.post_attention_layernorm.weight": "pytorch_model-00007-of-00007.bin",
469
  "model.layers.55.mlp.down_proj.weight": "pytorch_model-00007-of-00007.bin",
470
  "model.layers.55.mlp.gate_proj.weight": "pytorch_model-00007-of-00007.bin",
471
  "model.layers.55.mlp.up_proj.weight": "pytorch_model-00007-of-00007.bin",
 
473
  "model.layers.55.self_attn.o_proj.weight": "pytorch_model-00007-of-00007.bin",
474
  "model.layers.55.self_attn.q_proj.weight": "pytorch_model-00007-of-00007.bin",
475
  "model.layers.55.self_attn.v_proj.weight": "pytorch_model-00007-of-00007.bin",
476
+ "model.layers.56.input_layernorm.weight": "pytorch_model-00007-of-00007.bin",
477
+ "model.layers.56.post_attention_layernorm.weight": "pytorch_model-00007-of-00007.bin",
478
  "model.layers.56.mlp.down_proj.weight": "pytorch_model-00007-of-00007.bin",
479
  "model.layers.56.mlp.gate_proj.weight": "pytorch_model-00007-of-00007.bin",
480
  "model.layers.56.mlp.up_proj.weight": "pytorch_model-00007-of-00007.bin",
 
482
  "model.layers.56.self_attn.o_proj.weight": "pytorch_model-00007-of-00007.bin",
483
  "model.layers.56.self_attn.q_proj.weight": "pytorch_model-00007-of-00007.bin",
484
  "model.layers.56.self_attn.v_proj.weight": "pytorch_model-00007-of-00007.bin",
485
+ "model.layers.57.input_layernorm.weight": "pytorch_model-00007-of-00007.bin",
486
+ "model.layers.57.post_attention_layernorm.weight": "pytorch_model-00007-of-00007.bin",
487
  "model.layers.57.mlp.down_proj.weight": "pytorch_model-00007-of-00007.bin",
488
  "model.layers.57.mlp.gate_proj.weight": "pytorch_model-00007-of-00007.bin",
489
  "model.layers.57.mlp.up_proj.weight": "pytorch_model-00007-of-00007.bin",
 
491
  "model.layers.57.self_attn.o_proj.weight": "pytorch_model-00007-of-00007.bin",
492
  "model.layers.57.self_attn.q_proj.weight": "pytorch_model-00007-of-00007.bin",
493
  "model.layers.57.self_attn.v_proj.weight": "pytorch_model-00007-of-00007.bin",
494
+ "model.layers.58.input_layernorm.weight": "pytorch_model-00007-of-00007.bin",
495
+ "model.layers.58.post_attention_layernorm.weight": "pytorch_model-00007-of-00007.bin",
496
  "model.layers.58.mlp.down_proj.weight": "pytorch_model-00007-of-00007.bin",
497
  "model.layers.58.mlp.gate_proj.weight": "pytorch_model-00007-of-00007.bin",
498
  "model.layers.58.mlp.up_proj.weight": "pytorch_model-00007-of-00007.bin",
 
500
  "model.layers.58.self_attn.o_proj.weight": "pytorch_model-00007-of-00007.bin",
501
  "model.layers.58.self_attn.q_proj.weight": "pytorch_model-00007-of-00007.bin",
502
  "model.layers.58.self_attn.v_proj.weight": "pytorch_model-00007-of-00007.bin",
503
+ "model.layers.59.input_layernorm.weight": "pytorch_model-00007-of-00007.bin",
504
+ "model.layers.59.post_attention_layernorm.weight": "pytorch_model-00007-of-00007.bin",
505
  "model.layers.59.mlp.down_proj.weight": "pytorch_model-00007-of-00007.bin",
506
  "model.layers.59.mlp.gate_proj.weight": "pytorch_model-00007-of-00007.bin",
507
  "model.layers.59.mlp.up_proj.weight": "pytorch_model-00007-of-00007.bin",
 
509
  "model.layers.59.self_attn.o_proj.weight": "pytorch_model-00007-of-00007.bin",
510
  "model.layers.59.self_attn.q_proj.weight": "pytorch_model-00007-of-00007.bin",
511
  "model.layers.59.self_attn.v_proj.weight": "pytorch_model-00007-of-00007.bin",
512
+ "model.layers.6.input_layernorm.weight": "pytorch_model-00001-of-00007.bin",
513
+ "model.layers.6.post_attention_layernorm.weight": "pytorch_model-00001-of-00007.bin",
514
  "model.layers.6.mlp.down_proj.weight": "pytorch_model-00001-of-00007.bin",
515
  "model.layers.6.mlp.gate_proj.weight": "pytorch_model-00001-of-00007.bin",
516
  "model.layers.6.mlp.up_proj.weight": "pytorch_model-00001-of-00007.bin",
 
518
  "model.layers.6.self_attn.o_proj.weight": "pytorch_model-00001-of-00007.bin",
519
  "model.layers.6.self_attn.q_proj.weight": "pytorch_model-00001-of-00007.bin",
520
  "model.layers.6.self_attn.v_proj.weight": "pytorch_model-00001-of-00007.bin",
521
+ "model.layers.7.input_layernorm.weight": "pytorch_model-00001-of-00007.bin",
522
+ "model.layers.7.post_attention_layernorm.weight": "pytorch_model-00001-of-00007.bin",
523
  "model.layers.7.mlp.down_proj.weight": "pytorch_model-00001-of-00007.bin",
524
  "model.layers.7.mlp.gate_proj.weight": "pytorch_model-00001-of-00007.bin",
525
  "model.layers.7.mlp.up_proj.weight": "pytorch_model-00001-of-00007.bin",
 
527
  "model.layers.7.self_attn.o_proj.weight": "pytorch_model-00001-of-00007.bin",
528
  "model.layers.7.self_attn.q_proj.weight": "pytorch_model-00001-of-00007.bin",
529
  "model.layers.7.self_attn.v_proj.weight": "pytorch_model-00001-of-00007.bin",
530
+ "model.layers.8.input_layernorm.weight": "pytorch_model-00002-of-00007.bin",
531
+ "model.layers.8.post_attention_layernorm.weight": "pytorch_model-00002-of-00007.bin",
532
  "model.layers.8.mlp.down_proj.weight": "pytorch_model-00002-of-00007.bin",
533
  "model.layers.8.mlp.gate_proj.weight": "pytorch_model-00002-of-00007.bin",
534
  "model.layers.8.mlp.up_proj.weight": "pytorch_model-00002-of-00007.bin",
 
536
  "model.layers.8.self_attn.o_proj.weight": "pytorch_model-00002-of-00007.bin",
537
  "model.layers.8.self_attn.q_proj.weight": "pytorch_model-00001-of-00007.bin",
538
  "model.layers.8.self_attn.v_proj.weight": "pytorch_model-00001-of-00007.bin",
539
+ "model.layers.9.input_layernorm.weight": "pytorch_model-00002-of-00007.bin",
540
+ "model.layers.9.post_attention_layernorm.weight": "pytorch_model-00002-of-00007.bin",
541
  "model.layers.9.mlp.down_proj.weight": "pytorch_model-00002-of-00007.bin",
542
  "model.layers.9.mlp.gate_proj.weight": "pytorch_model-00002-of-00007.bin",
543
  "model.layers.9.mlp.up_proj.weight": "pytorch_model-00002-of-00007.bin",
tokenization_yi.py DELETED
@@ -1,255 +0,0 @@
1
- import os
2
- from shutil import copyfile
3
- from typing import Any, Dict, List, Optional, Tuple
4
-
5
- import sentencepiece as spm
6
- from transformers.tokenization_utils import AddedToken, PreTrainedTokenizer
7
- from transformers.utils import logging
8
-
9
- logger = logging.get_logger(__name__)
10
-
11
- VOCAB_FILES_NAMES = {"vocab_file": "tokenizer.model"}
12
-
13
- PRETRAINED_VOCAB_FILES_MAP = {
14
- "vocab_file": {},
15
- "tokenizer_file": {},
16
- }
17
- PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = {}
18
-
19
-
20
- class YiTokenizer(PreTrainedTokenizer):
21
- """
22
- Construct a Yi tokenizer. Based on byte-level Byte-Pair-Encoding.
23
-
24
- Args:
25
- vocab_file (`str`):
26
- Path to the vocabulary file.
27
- """
28
-
29
- vocab_files_names = VOCAB_FILES_NAMES
30
- pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP
31
- max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
32
- model_input_names = ["input_ids", "attention_mask"]
33
-
34
- def __init__(
35
- self,
36
- vocab_file,
37
- unk_token="<unk>",
38
- bos_token="<|startoftext|>",
39
- eos_token="<|endoftext|>",
40
- pad_token="<unk>",
41
- sp_model_kwargs: Optional[Dict[str, Any]] = None,
42
- add_bos_token=True,
43
- add_eos_token=False,
44
- clean_up_tokenization_spaces=False,
45
- **kwargs,
46
- ):
47
- self.sp_model_kwargs = {} if sp_model_kwargs is None else sp_model_kwargs
48
- bos_token = (
49
- AddedToken(bos_token, lstrip=False, rstrip=False)
50
- if isinstance(bos_token, str)
51
- else bos_token
52
- )
53
- eos_token = (
54
- AddedToken(eos_token, lstrip=False, rstrip=False)
55
- if isinstance(eos_token, str)
56
- else eos_token
57
- )
58
- unk_token = (
59
- AddedToken(unk_token, lstrip=False, rstrip=False)
60
- if isinstance(unk_token, str)
61
- else unk_token
62
- )
63
- pad_token = (
64
- AddedToken(pad_token, lstrip=False, rstrip=False)
65
- if isinstance(pad_token, str)
66
- else pad_token
67
- )
68
- self.vocab_file = vocab_file
69
- self.add_bos_token = add_bos_token
70
- self.add_eos_token = add_eos_token
71
- self.sp_model = spm.SentencePieceProcessor(**self.sp_model_kwargs)
72
- self.sp_model.Load(vocab_file)
73
- super().__init__(
74
- bos_token=bos_token,
75
- eos_token=eos_token,
76
- unk_token=unk_token,
77
- pad_token=pad_token,
78
- add_bos_token=add_bos_token,
79
- add_eos_token=add_eos_token,
80
- sp_model_kwargs=self.sp_model_kwargs,
81
- clean_up_tokenization_spaces=clean_up_tokenization_spaces,
82
- **kwargs,
83
- )
84
-
85
- def __getstate__(self):
86
- state = self.__dict__.copy()
87
- state["sp_model"] = None
88
- return state
89
-
90
- def __setstate__(self, d):
91
- self.__dict__ = d
92
- self.sp_model = spm.SentencePieceProcessor(**self.sp_model_kwargs)
93
- self.sp_model.Load(self.vocab_file)
94
-
95
- @property
96
- def vocab_size(self):
97
- """Returns vocab size"""
98
- return self.sp_model.get_piece_size()
99
-
100
- def get_vocab(self):
101
- """Returns vocab as a dict"""
102
- vocab = {self.convert_ids_to_tokens(i): i for i in range(self.vocab_size)}
103
- vocab.update(self.added_tokens_encoder)
104
- return vocab
105
-
106
- def _tokenize(self, text):
107
- """Returns a tokenized string."""
108
- return self.sp_model.encode(text, out_type=str)
109
-
110
- def _convert_token_to_id(self, token):
111
- """Converts a token (str) in an id using the vocab."""
112
- return self.sp_model.piece_to_id(token)
113
-
114
- def _convert_id_to_token(self, index):
115
- """Converts an index (integer) in a token (str) using the vocab."""
116
- token = self.sp_model.IdToPiece(index)
117
- return token
118
-
119
- def convert_tokens_to_string(self, tokens):
120
- """Converts a sequence of tokens (string) in a single string."""
121
- current_sub_tokens = []
122
- out_string = ""
123
- prev_is_special = False
124
- for i, token in enumerate(tokens):
125
- # make sure that special tokens are not decoded using sentencepiece model
126
- if token in self.all_special_tokens:
127
- if not prev_is_special and i != 0:
128
- out_string += " "
129
- out_string += self.sp_model.decode(current_sub_tokens) + token
130
- prev_is_special = True
131
- current_sub_tokens = []
132
- else:
133
- current_sub_tokens.append(token)
134
- prev_is_special = False
135
- out_string += self.sp_model.decode(current_sub_tokens)
136
- return out_string
137
-
138
- def save_vocabulary(
139
- self, save_directory, filename_prefix: Optional[str] = None
140
- ) -> Tuple[str]:
141
- """
142
- Save the vocabulary and special tokens file to a directory.
143
-
144
- Args:
145
- save_directory (`str`):
146
- The directory in which to save the vocabulary.
147
-
148
- Returns:
149
- `Tuple(str)`: Paths to the files saved.
150
- """
151
- if not os.path.isdir(save_directory):
152
- logger.error(f"Vocabulary path ({save_directory}) should be a directory")
153
- return
154
- out_vocab_file = os.path.join(
155
- save_directory,
156
- (filename_prefix + "-" if filename_prefix else "")
157
- + VOCAB_FILES_NAMES["vocab_file"],
158
- )
159
-
160
- if os.path.abspath(self.vocab_file) != os.path.abspath(
161
- out_vocab_file
162
- ) and os.path.isfile(self.vocab_file):
163
- copyfile(self.vocab_file, out_vocab_file)
164
- elif not os.path.isfile(self.vocab_file):
165
- with open(out_vocab_file, "wb") as fi:
166
- content_spiece_model = self.sp_model.serialized_model_proto()
167
- fi.write(content_spiece_model)
168
-
169
- return (out_vocab_file,)
170
-
171
- def build_inputs_with_special_tokens(self, token_ids_0, token_ids_1=None):
172
- bos_token_id = [self.bos_token_id] if self.add_bos_token else []
173
- eos_token_id = [self.eos_token_id] if self.add_eos_token else []
174
-
175
- output = bos_token_id + token_ids_0 + eos_token_id
176
-
177
- if token_ids_1 is not None:
178
- output = output + bos_token_id + token_ids_1 + eos_token_id
179
-
180
- return output
181
-
182
- def get_special_tokens_mask(
183
- self,
184
- token_ids_0: List[int],
185
- token_ids_1: Optional[List[int]] = None,
186
- already_has_special_tokens: bool = False,
187
- ) -> List[int]:
188
- """
189
- Retrieve sequence ids from a token list that has no special tokens added. This method is called when adding
190
- special tokens using the tokenizer `prepare_for_model` method.
191
-
192
- Args:
193
- token_ids_0 (`List[int]`):
194
- List of IDs.
195
- token_ids_1 (`List[int]`, *optional*):
196
- Optional second list of IDs for sequence pairs.
197
- already_has_special_tokens (`bool`, *optional*, defaults to `False`):
198
- Whether or not the token list is already formatted with special tokens for the model.
199
-
200
- Returns:
201
- `List[int]`: A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token.
202
- """
203
- if already_has_special_tokens:
204
- return super().get_special_tokens_mask(
205
- token_ids_0=token_ids_0,
206
- token_ids_1=token_ids_1,
207
- already_has_special_tokens=True,
208
- )
209
-
210
- bos_token_id = [1] if self.add_bos_token else []
211
- eos_token_id = [1] if self.add_eos_token else []
212
-
213
- if token_ids_1 is None:
214
- return bos_token_id + ([0] * len(token_ids_0)) + eos_token_id
215
- return (
216
- bos_token_id
217
- + ([0] * len(token_ids_0))
218
- + eos_token_id
219
- + bos_token_id
220
- + ([0] * len(token_ids_1))
221
- + eos_token_id
222
- )
223
-
224
- def create_token_type_ids_from_sequences(
225
- self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None
226
- ) -> List[int]:
227
- """
228
- Creates a mask from the two sequences passed to be used in a sequence-pair classification task. An ALBERT
229
- sequence pair mask has the following format:
230
-
231
- ```
232
- 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1
233
- | first sequence | second sequence |
234
- ```
235
-
236
- if token_ids_1 is None, only returns the first portion of the mask (0s).
237
-
238
- Args:
239
- token_ids_0 (`List[int]`):
240
- List of ids.
241
- token_ids_1 (`List[int]`, *optional*):
242
- Optional second list of IDs for sequence pairs.
243
-
244
- Returns:
245
- `List[int]`: List of [token type IDs](../glossary#token-type-ids) according to the given sequence(s).
246
- """
247
- bos_token_id = [self.bos_token_id] if self.add_bos_token else []
248
- eos_token_id = [self.eos_token_id] if self.add_eos_token else []
249
-
250
- output = [0] * len(bos_token_id + token_ids_0 + eos_token_id)
251
-
252
- if token_ids_1 is not None:
253
- output += [1] * len(bos_token_id + token_ids_1 + eos_token_id)
254
-
255
- return output
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
tokenizer_config.json CHANGED
@@ -1,9 +1,13 @@
1
  {
2
- "auto_map": {
3
- "AutoTokenizer": ["tokenization_yi.YiTokenizer", null]
4
- },
5
  "add_bos_token": false,
6
  "add_eos_token": false,
7
  "model_max_length": 4096,
8
- "tokenizer_class": "YiTokenizer"
 
 
 
 
 
 
 
9
  }
 
1
  {
 
 
 
2
  "add_bos_token": false,
3
  "add_eos_token": false,
4
  "model_max_length": 4096,
5
+ "unk_token": "<unk>",
6
+ "bos_token": "<|startoftext|>",
7
+ "eos_token": "<|endoftext|>",
8
+ "pad_token": "<unk>",
9
+ "sp_model_kwargs": {},
10
+ "clean_up_tokenization_spaces": false,
11
+ "legacy": true,
12
+ "tokenizer_class": "LlamaTokenizer"
13
  }