HachiML commited on
Commit
306a855
1 Parent(s): 6ff8e21

End of training

Browse files
Files changed (4) hide show
  1. README.md +68 -0
  2. generation_config.json +6 -0
  3. model.safetensors +1 -1
  4. modeling_bit_llama.py +134 -0
README.md ADDED
@@ -0,0 +1,68 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ tags:
3
+ - generated_from_trainer
4
+ model-index:
5
+ - name: myBit-Llama2-jp-127M-test-20
6
+ results: []
7
+ ---
8
+
9
+ <!-- This model card has been generated automatically according to the information the Trainer had access to. You
10
+ should probably proofread and complete it, then remove this comment. -->
11
+
12
+ # myBit-Llama2-jp-127M-test-20
13
+
14
+ This model is a fine-tuned version of [](https://huggingface.co/) on an unknown dataset.
15
+ It achieves the following results on the evaluation set:
16
+ - Loss: 3.4890
17
+
18
+ ## Model description
19
+
20
+ More information needed
21
+
22
+ ## Intended uses & limitations
23
+
24
+ More information needed
25
+
26
+ ## Training and evaluation data
27
+
28
+ More information needed
29
+
30
+ ## Training procedure
31
+
32
+ ### Training hyperparameters
33
+
34
+ The following hyperparameters were used during training:
35
+ - learning_rate: 0.0024
36
+ - train_batch_size: 96
37
+ - eval_batch_size: 96
38
+ - seed: 42
39
+ - optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
40
+ - lr_scheduler_type: polynomial
41
+ - lr_scheduler_warmup_steps: 500
42
+ - num_epochs: 1
43
+
44
+ ### Training results
45
+
46
+ | Training Loss | Epoch | Step | Validation Loss |
47
+ |:-------------:|:-----:|:----:|:---------------:|
48
+ | 6.9256 | 0.07 | 200 | 4.9236 |
49
+ | 4.6058 | 0.15 | 400 | 4.4149 |
50
+ | 4.3066 | 0.22 | 600 | 4.2187 |
51
+ | 4.126 | 0.29 | 800 | 4.0780 |
52
+ | 4.0117 | 0.37 | 1000 | 3.9963 |
53
+ | 3.9295 | 0.44 | 1200 | 3.9215 |
54
+ | 3.8601 | 0.51 | 1400 | 3.8477 |
55
+ | 3.7821 | 0.59 | 1600 | 3.7748 |
56
+ | 3.7065 | 0.66 | 1800 | 3.7078 |
57
+ | 3.6407 | 0.73 | 2000 | 3.6518 |
58
+ | 3.5957 | 0.8 | 2200 | 3.5923 |
59
+ | 3.5302 | 0.88 | 2400 | 3.5368 |
60
+ | 3.482 | 0.95 | 2600 | 3.4890 |
61
+
62
+
63
+ ### Framework versions
64
+
65
+ - Transformers 4.38.2
66
+ - Pytorch 2.2.1+cu121
67
+ - Datasets 2.18.0
68
+ - Tokenizers 0.15.2
generation_config.json ADDED
@@ -0,0 +1,6 @@
 
 
 
 
 
 
 
1
+ {
2
+ "_from_model_config": true,
3
+ "bos_token_id": 1,
4
+ "eos_token_id": 2,
5
+ "transformers_version": "4.38.2"
6
+ }
model.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:66947712700abcfa2ed3e11099973cdf54f6ba11467958cb0bccdf10bcf8927b
3
  size 510960712
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9b262fd14c13d411d6465ccd4ebb49ac4d763381746b8456ee8652db386c42a6
3
  size 510960712
modeling_bit_llama.py ADDED
@@ -0,0 +1,134 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import warnings
2
+ from typing import Optional, Tuple
3
+ from transformers.models.llama.modeling_llama import (
4
+ LlamaConfig,
5
+ LlamaModel,
6
+ LlamaForCausalLM,
7
+ LlamaAttention,
8
+ LlamaFlashAttention2,
9
+ LlamaSdpaAttention,
10
+ LlamaMLP,
11
+ LlamaDecoderLayer,
12
+ )
13
+ from mybitnet.bitnet import BitLinear
14
+ import torch
15
+ from torch import nn
16
+
17
+ class BitLlamaConfig(LlamaConfig):
18
+ model_type = "bit_llama"
19
+
20
+ def __init__(self, bits=8, **kwargs):
21
+ super().__init__(**kwargs)
22
+ self.bits = bits
23
+
24
+ class BitLlamaMLP(LlamaMLP):
25
+ def __init__(self, config):
26
+ super().__init__(config)
27
+ self.gate_proj = BitLinear(self.hidden_size, self.intermediate_size, bias=False, bits=config.bits, flg_before_linear=False)
28
+ self.up_proj = BitLinear(self.hidden_size, self.intermediate_size, bias=False, bits=config.bits, flg_before_linear=True)
29
+ self.down_proj = BitLinear(self.intermediate_size, self.hidden_size, bias=False, bits=config.bits, flg_before_linear=True)
30
+
31
+ class BitLlamaAttention(LlamaAttention):
32
+ def __init__(self, config: BitLlamaConfig, layer_idx: Optional[int] = None):
33
+ super().__init__(config)
34
+ self.q_proj = BitLinear(self.hidden_size, self.num_heads * self.head_dim, bias=False, bits=config.bits, flg_before_linear=True)
35
+ self.k_proj = BitLinear(self.hidden_size, self.num_key_value_heads * self.head_dim, bias=False, bits=config.bits, flg_before_linear=True)
36
+ self.v_proj = BitLinear(self.hidden_size, self.num_key_value_heads * self.head_dim, bias=False, bits=config.bits, flg_before_linear=True)
37
+ self.o_proj = BitLinear(self.hidden_size, self.hidden_size, bias=False, bits=config.bits, flg_before_linear=True)
38
+
39
+ class BitLlamaFlashAttention2(LlamaFlashAttention2):
40
+ def __init__(self, config: BitLlamaConfig, layer_idx: Optional[int] = None):
41
+ super().__init__(config, layer_idx)
42
+ self.q_proj = BitLinear(self.hidden_size, self.num_heads * self.head_dim, bias=False, bits=config.bits, flg_before_linear=True)
43
+ self.k_proj = BitLinear(self.hidden_size, self.num_key_value_heads * self.head_dim, bias=False, bits=config.bits, flg_before_linear=True)
44
+ self.v_proj = BitLinear(self.hidden_size, self.num_key_value_heads * self.head_dim, bias=False, bits=config.bits, flg_before_linear=True)
45
+ self.o_proj = BitLinear(self.hidden_size, self.hidden_size, bias=False, bits=config.bits, flg_before_linear=True)
46
+
47
+ class BitLlamaSdpaAttention(LlamaSdpaAttention):
48
+ def __init__(self, config: BitLlamaConfig, layer_idx: Optional[int] = None):
49
+ super().__init__(config, layer_idx)
50
+ self.q_proj = BitLinear(self.hidden_size, self.num_heads * self.head_dim, bias=False, bits=config.bits, flg_before_linear=True)
51
+ self.k_proj = BitLinear(self.hidden_size, self.num_key_value_heads * self.head_dim, bias=False, bits=config.bits, flg_before_linear=True)
52
+ self.v_proj = BitLinear(self.hidden_size, self.num_key_value_heads * self.head_dim, bias=False, bits=config.bits, flg_before_linear=True)
53
+ self.o_proj = BitLinear(self.hidden_size, self.hidden_size, bias=False, bits=config.bits, flg_before_linear=True)
54
+
55
+ BITLLAMA_ATTENTION_CLASSES = {
56
+ "eager": BitLlamaAttention,
57
+ "flash_attention_2": BitLlamaFlashAttention2,
58
+ "sdpa": BitLlamaSdpaAttention,
59
+ }
60
+
61
+ class BitLlamaDecoderLayer(LlamaDecoderLayer):
62
+ def __init__(self, config: BitLlamaConfig, layer_idx: int):
63
+ super().__init__(config, layer_idx)
64
+ self.self_attn = BITLLAMA_ATTENTION_CLASSES[config._attn_implementation](config=config, layer_idx=layer_idx)
65
+ self.mlp = BitLlamaMLP(config)
66
+ del self.input_layernorm
67
+ del self.post_attention_layernorm
68
+
69
+ def forward(
70
+ self,
71
+ hidden_states: torch.Tensor,
72
+ attention_mask: Optional[torch.Tensor] = None,
73
+ position_ids: Optional[torch.LongTensor] = None,
74
+ past_key_value: Optional[Tuple[torch.Tensor]] = None,
75
+ output_attentions: Optional[bool] = False,
76
+ use_cache: Optional[bool] = False,
77
+ cache_position: Optional[torch.LongTensor] = None,
78
+ **kwargs,
79
+ ) -> Tuple[torch.FloatTensor, Optional[Tuple[torch.FloatTensor, torch.FloatTensor]]]:
80
+ """
81
+ refers: https://github.com/huggingface/transformers/blob/c5f0288bc7d76f65996586f79f69fba8867a0e67/src/transformers/models/llama/modeling_llama.py#L693
82
+ """
83
+ if "padding_mask" in kwargs:
84
+ warnings.warn(
85
+ "Passing `padding_mask` is deprecated and will be removed in v4.37. Please make sure use `attention_mask` instead.`"
86
+ )
87
+
88
+ residual = hidden_states
89
+
90
+ # Self Attention
91
+ hidden_states, self_attn_weights, present_key_value = self.self_attn(
92
+ hidden_states=hidden_states,
93
+ attention_mask=attention_mask,
94
+ position_ids=position_ids,
95
+ past_key_value=past_key_value,
96
+ output_attentions=output_attentions,
97
+ use_cache=use_cache,
98
+ cache_position=cache_position,
99
+ **kwargs,
100
+ )
101
+ hidden_states = residual + hidden_states
102
+
103
+ # Fully Connected
104
+ residual = hidden_states
105
+ hidden_states = self.mlp(hidden_states)
106
+ hidden_states = residual + hidden_states
107
+
108
+ outputs = (hidden_states,)
109
+
110
+ if output_attentions:
111
+ outputs += (self_attn_weights,)
112
+
113
+ if use_cache:
114
+ outputs += (present_key_value,)
115
+
116
+ return outputs
117
+
118
+ class BitLlamaModel(LlamaModel):
119
+ config_class = BitLlamaConfig
120
+
121
+ def __init__(self, config: BitLlamaConfig):
122
+ super().__init__(config)
123
+ self.layers = nn.ModuleList(
124
+ [BitLlamaDecoderLayer(config, layer_idx) for layer_idx in range(config.num_hidden_layers)]
125
+ )
126
+
127
+ class BitLlamaForCausalLM(LlamaForCausalLM):
128
+ config_class = BitLlamaConfig
129
+
130
+ def __init__(self, config: BitLlamaConfig):
131
+ super().__init__(config)
132
+ self.model = BitLlamaModel(config)
133
+ self.lm_head = BitLinear(config.hidden_size, config.vocab_size, bias=False, bits=config.bits, flg_before_linear=True)
134
+