Upload MM_LLMs

config.json

@@ -1,5 +1,5 @@
 {
-  "_name_or_path": "audio-alignment-tinyllama/checkpoint-5800",
+  "_name_or_path": "audio-alignment-tinyllama/checkpoint-6800",
   "architectures": [
     "MM_LLMs"
   ],
@@ -203,6 +203,7 @@
     "use_weighted_layer_sum": false,
     "vocab_size": 51865
   },
+  "audio_select_layer": -2,
   "auto_map": {
     "AutoConfig": "modeling_audio.MM_LLMs_Config",
     "AutoModel": "modeling_audio.MM_LLMs"

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b3875b2eb90aa1bd7aea8ba2faed29ed0c5896cd341c86fdad8e49e2319d02e1
+oid sha256:d69f1d7e7d57b232dead293d2c9ee96b39daf7783fcb15ba200db80b23b2c80f
 size 2817909376

modeling_audio.py

@@ -34,11 +34,13 @@ class MM_LLMs_Config(PretrainedConfig):
         self,
         audio_config=None,
         llm_config=None,
+        audio_select_layer=-2,
         **kwargs
     ):
 
         self.audio_config = audio_config
         self.llm_config = llm_config
+        self.audio_select_layer = audio_select_layer
 
         if isinstance(self.audio_config, dict):
             audio_config["model_type"] = (
@@ -252,6 +254,8 @@ class MM_LLMs(PreTrainedModel):
         return model_inputs
 
     def encode_audio(self, audios):
-        encoded = self.audio_encoder.encoder(audios)[0]
+
+        encoded = self.audio_encoder.encoder(audios, output_hidden_states=True)
+        encoded = encoded.hidden_states[self.config.audio_select_layer]
         audio_features = self.audio_projector(encoded.transpose(1, 2).contiguous())
         return audio_features