Upload model

config.json

@@ -78,7 +78,7 @@
     "top_p": 1.0,
     "torch_dtype": null,
     "torchscript": false,
-    "transformers_version": "4.29.2",
+    "transformers_version": "4.28.1",
     "type_vocab_size": 2,
     "typical_p": 1.0,
     "use_bfloat16": false,
@@ -2243,7 +2243,7 @@
     "top_p": 1.0,
     "torch_dtype": "float32",
     "torchscript": false,
-    "transformers_version": "4.29.2",
+    "transformers_version": "4.28.1",
     "typical_p": 1.0,
     "use_bfloat16": false
   },

generation_config.json

@@ -1,5 +1,5 @@
 {
   "_from_model_config": true,
   "pad_token_id": 0,
-  "transformers_version": "4.29.2"
+  "transformers_version": "4.28.1"
 }

modelling_longitudinal.py

@@ -1,11 +1,13 @@
 import os
+import warnings
 from typing import Any, Optional, Tuple, Union
 
 import torch
 import transformers
 from peft import LoraConfig, TaskType, get_peft_config, get_peft_model
 from torch.nn import CrossEntropyLoss
-from transformers import PreTrainedTokenizerFast, VisionEncoderDecoderModel
+from transformers import (AutoModel, PreTrainedTokenizerFast,
+                          VisionEncoderDecoderModel)
 from transformers.configuration_utils import PretrainedConfig
 from transformers.modeling_outputs import BaseModelOutput, Seq2SeqLMOutput
 from transformers.modeling_utils import PreTrainedModel
@@ -23,7 +25,7 @@ class CvtWithProjectionHeadConfig(transformers.CvtConfig):
 
 
 class ModelOutputWithProjectionEmbedding(transformers.modeling_outputs.ModelOutput):
-    projected_last_hidden_state: torch.FloatTensor
+    last_hidden_state: torch.FloatTensor
     attention_mask: torch.FloatTensor
 
 
@@ -87,7 +89,7 @@ class VariableCvtWithProjectionHead(transformers.CvtPreTrainedModel):
             return projection
 
         return ModelOutputWithProjectionEmbedding(
-            projected_last_hidden_state=projection, attention_mask=attention_mask,
+            last_hidden_state=projection, attention_mask=attention_mask,
         )
     
 
@@ -103,7 +105,7 @@ class LongitudinalPromptVariableCXREncoderDecoderModel(VisionEncoderDecoderModel
         config: Optional[PretrainedConfig] = None,
         encoder: Optional[PreTrainedModel] = None,
         decoder: Optional[PreTrainedModel] = None,
-        encoder_decoder_ckpt_path: Optional[str] = None,
+        encoder_decoder_ckpt_name: Optional[str] = None,
     ):
 
         if decoder:
@@ -149,8 +151,11 @@ class LongitudinalPromptVariableCXREncoderDecoderModel(VisionEncoderDecoderModel
         self.decoder.config = self.config.decoder
 
         # Load variable checkpoint:
-        if encoder_decoder_ckpt_path:
-            self.load_state_dict(torch.load(encoder_decoder_ckpt_path)['state_dict'])
+        if encoder_decoder_ckpt_name:
+            encoder_decoder = AutoModel.from_pretrained(encoder_decoder_ckpt_name, trust_remote_code=True)
+            self.load_state_dict(encoder_decoder.state_dict())
+        else:
+            warnings.warn('The encoder-to-decoder model was not warm-started before applying low-rank approximation.')
 
         # Freeze the encoder:
         for p in self.encoder.parameters():
@@ -240,7 +245,7 @@ class LongitudinalPromptVariableCXREncoderDecoderModel(VisionEncoderDecoderModel
             decoder_hidden_states=decoder_outputs.hidden_states,
             decoder_attentions=decoder_outputs.attentions,
             cross_attentions=decoder_outputs.cross_attentions,
-            encoder_last_hidden_state=encoder_outputs.projected_last_hidden_state,
+            encoder_last_hidden_state=encoder_outputs.last_hidden_state,
             # encoder_hidden_states=encoder_outputs.hidden_states,
             # encoder_attentions=encoder_outputs.attentions,
         )

pytorch_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ec90882d89dcb37bd495b287c8f2f3d655fd16a025f0b8197018fcb9abb43362
+oid sha256:b04999f2bf3a2d29880d0897eeb3cb38496ecb2d4dc322198bc6b972eca5c85e
 size 450315441