Upload GPTRefactForCausalLM

config.json

@@ -8,23 +8,24 @@
     "AutoConfig": "configuration_gpt_refact.GPTRefactConfig",
     "AutoModelForCausalLM": "modeling_gpt_refact.GPTRefactForCausalLM"
   },
-  "bos_token_id": 0,
+  "bos_token_id": -1,
   "do_sample": true,
   "embd_pdrop": 0.1,
   "eos_token_id": 0,
   "initializer_range": 0.02,
   "layer_norm_epsilon": 1e-05,
   "model_type": "gpt_refact",
+  "multi_query": true,
   "n_embd": 2048,
   "n_head": 32,
   "n_inner": null,
   "n_layer": 32,
-  "n_positions": 1024,
+  "n_positions": 4096,
   "resid_pdrop": 0.1,
   "scale_attention_softmax_in_fp32": false,
   "scale_attn_weights": true,
   "torch_dtype": "float32",
-  "transformers_version": "4.29.2",
+  "transformers_version": "4.31.0",
   "use_cache": true,
   "vocab_size": 49216
 }

configuration_gpt_refact.py

@@ -30,8 +30,10 @@ class GPTRefactConfig(PretrainedConfig):
         initializer_range=0.02,
         scale_attn_weights=True,
         use_cache=True,
-        bos_token_id=0,
+        bos_token_id=-1,
         eos_token_id=0,
+        max_position_embeddings: int = 2048,
+        multi_query: bool = True,
         attention_softmax_in_fp32=False,
         scale_attention_softmax_in_fp32=False,
         **kwargs,
@@ -55,4 +57,6 @@ class GPTRefactConfig(PretrainedConfig):
         self.bos_token_id = bos_token_id
         self.eos_token_id = eos_token_id
 
-        super().__init__(bos_token_id=bos_token_id, eos_token_id=eos_token_id, **kwargs)
+        self.multi_query = multi_query
+        self.max_position_embeddings = max_position_embeddings
+        super().__init__(bos_token_id=bos_token_id, eos_token_id=eos_token_id, **kwargs)

generation_config.json

@@ -1,7 +1,7 @@
 {
   "_from_model_config": true,
-  "bos_token_id": 0,
+  "bos_token_id": -1,
   "do_sample": true,
   "eos_token_id": 0,
-  "transformers_version": "4.29.2"
+  "transformers_version": "4.31.0"
 }

modeling_gpt_refact.py

@@ -341,7 +341,7 @@ class GPTRefactModel(GPTRefactPreTrainedModel):
         super().__init__(config)
         self.embed_dim = config.hidden_size
         self.num_heads = config.num_attention_heads
-
+        self.multi_query = config.multi_query
         self.wte = nn.Embedding(config.vocab_size, self.embed_dim)
 
         self.h = nn.ModuleList([GPTRefactBlock(config, layer_idx=i) for i in range(config.num_hidden_layers)])

pytorch_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0376542ae03467045dccef1614b92c6fe1f8cedc1882b16304de0179cf2e9e53
-size 6343465669
+oid sha256:c2369c7e2228204ac8e0bc39c048d1e6349ce5f1bab8005a60bde0f0aa26ca73
+size 6343461637