Update configuration_quasar.py

configuration_quasar.py

@@ -23,10 +23,10 @@ from transformers.utils import logging
 logger = logging.get_logger(__name__)
 
 QUASAR_PRETRAINED_CONFIG_ARCHIVE_MAP = {
-    "AstraMindAI/AstraQuasar-4.5B": "https://huggingface.co/AstraMindAI/AstraQuasar-4.5B/resolve/main/config.json",
+    "AstraMindAI/AstraQuasar-4B": "https://huggingface.co/AstraMindAI/AstraQuasar-4B/resolve/main/config.json",
 }
 
-#From phi-2 Phi -> Quasar
+#from microsoft/phi-2, Phi -> Quasar
 class QuasarConfig(PretrainedConfig):
     r"""
     This is the configuration class to store the configuration of a [`QuasarModel`]. It is used to instantiate an Quasar
@@ -138,11 +138,7 @@ class QuasarConfig(PretrainedConfig):
         qk_layernorm=False,
         bos_token_id=1,
         eos_token_id=2,
-        ## Aggiunto
-        #duplicate_trick_v2=True,
-        #duplicate_rank=8,
-        #duplicate_dropout=0.0,
-        sliding_window=4096,
+        sliding_window=2048,
         simple_norm=False,
         remove_ff_bias=True,
         gated_activation=False,
@@ -151,7 +147,7 @@ class QuasarConfig(PretrainedConfig):
         layer_ranges=[[0, 16],[8, 21],[12, 25],[16, 29],[25, 32]],
         **kwargs,
     ):
-        ##Aggiunti
+        
         self.sliding_window = sliding_window
         self.simple_norm = simple_norm
         self.remove_ff_bias = remove_ff_bias
@@ -160,15 +156,6 @@ class QuasarConfig(PretrainedConfig):
         self.duplicate_grad = duplicate_grad
         self.layer_ranges = layer_ranges if layer_ranges is not None else []
 
-        ##V2###
-        #self.duplicate_trick_v2 = duplicate_trick_v2
-        #self.layer_ranges_duplicate_v2 = []
-        #self._assing_layer_ranges_duplicate_v2()
-        #self.duplicate_rank = duplicate_rank
-        #self.duplicate_dropout = duplicate_dropout
-        #self._duplicate_trick_v2_validation()
-        ####
-
         self.vocab_size = vocab_size
         self.hidden_size = hidden_size
         self.intermediate_size = intermediate_size
@@ -200,36 +187,6 @@ class QuasarConfig(PretrainedConfig):
             **kwargs,
         )
 
-    def _assing_layer_ranges_duplicate_v2(self):
-        # Calcolo gli offset iniziali per ciascun intervallo nella lista unica
-        offsets = [0]
-        for i in range(1, len(self.layer_ranges)):
-            offset = offsets[-1] + self.layer_ranges[i - 1][1] - self.layer_ranges[i - 1][0]
-            offsets.append(offset)
-
-        # Seleziono solo gli intervalli dispari e calcolo le loro posizioni assolute
-        odd_intervals_positions = []
-        for i in range(1, len(self.layer_ranges), 2):
-            start, end = self.layer_ranges[i]
-            for n in range(start, end):
-                position = offsets[i] + (n - start)
-                odd_intervals_positions.append(position)
-
-        self.layer_ranges_duplicate_v2 = list(set(odd_intervals_positions))
-
-
-    def _duplicate_trick_v2_validation(self):
-        if self.duplicate_trick_v2 and self.duplicate_trick:
-            # warn just one time that only one of the two flags will be used
-            logger.warning(
-                "Both `duplicate_trick` and `duplicate_trick_v2` are set to True. Only `duplicate_trick_v2` will be used."
-            )
-        if self.duplicate_trick_v2 and self.duplicate_rank < 1:
-            raise ValueError("`duplicate_rank` must be a positive integer")
-        if self.duplicate_trick_v2 and not self.layer_ranges:
-            raise ValueError("`layer_ranges` must be set when `duplicate_trick_v2` is True")
-
-
     # Copied from transformers.models.llama.configuration_llama.LlamaConfig._rope_scaling_validation
     def _rope_scaling_validation(self):
         """