Update configuration_Llamoe.py

configuration_Llamoe.py

@@ -1,18 +1,20 @@
-
 from transformers.configuration_utils import PretrainedConfig
 from transformers.utils import logging
 
 
 logger = logging.get_logger(__name__)
 
-LLAMOE_PRETRAINED_CONFIG_ARCHIVE_MAP = {
+GEMMOE_PRETRAINED_CONFIG_ARCHIVE_MAP = {
     "damerajee/Llamoe-test": "https://huggingface.co/damerajee/Llamoe-test/resolve/main/config.json",
 }
 
 
 class LlamoeConfig(PretrainedConfig):
-    model_type = "Llamoe"
+
+
+    model_type = "llamoe"
     keys_to_ignore_at_inference = ["past_key_values"]
+
     def __init__(
         self,
         vocab_size=32000,
@@ -47,8 +49,9 @@ class LlamoeConfig(PretrainedConfig):
         self.num_hidden_layers = num_hidden_layers
         self.num_attention_heads = num_attention_heads
         self.head_dim = head_dim
-        self.num_key_value_heads = num_key_value_heads
         self.hidden_act = hidden_act
+        self.hidden_activation = hidden_activation
+        self.num_key_value_heads = num_key_value_heads
         self.initializer_range = initializer_range
         self.rms_norm_eps = rms_norm_eps
         self.use_cache = use_cache