|
--- |
|
datasets: |
|
- ddrg/named_math_formulas |
|
- ddrg/math_formula_retrieval |
|
- ddrg/math_formulas |
|
- ddrg/math_text |
|
--- |
|
Pretrained model based on [microsoft/deberta-v3-base](https://huggingface.co./microsoft/deberta-v3-base) with further mathematical pre-training. |
|
|
|
Compared to deberta-v3-base, 300 additional mathematical LaTeX tokens have been added before the mathematical pre-training. As this additional pre-training used NSP-like tasks, a pooling layer has been added to the model (`bias` and `weight`). If you don't need this pooling layer, just use the standard transformers DeBERTa model. If you want to use the additional pooling layer like the BERT one, a wrapper class like the following may be used: |
|
```python |
|
from typing import Mapping, Any |
|
|
|
import torch |
|
from torch import nn |
|
from transformers import DebertaV2Model, DebertaV2Tokenizer, AutoConfig, AutoTokenizer |
|
|
|
class DebertaV2ModelWithPoolingLayer: |
|
|
|
def __init__(self, pretrained_model_name): |
|
super(DebertaV2ModelWithPoolingLayer, self).__init__() |
|
|
|
# Load the Deberta model and tokenizer |
|
self.deberta = DebertaV2Model.from_pretrained(pretrained_model_name) |
|
self.tokenizer = DebertaV2Tokenizer.from_pretrained(pretrained_model_name) |
|
|
|
# Add a pooling layer (Linear + tanh activation) for the CLS token |
|
self.pooling_layer = nn.Sequential( |
|
nn.Linear(self.deberta.config.hidden_size, self.deberta.config.hidden_size), |
|
nn.Tanh() |
|
) |
|
|
|
self.config = self.deberta.config |
|
self.embeddings = self.deberta.embeddings |
|
|
|
|
|
def forward(self, input_ids, attention_mask, *args, **kwargs): |
|
# Forward pass through the Deberta model |
|
outputs = self.deberta(input_ids, attention_mask=attention_mask, *args, **kwargs) |
|
|
|
# Extract the hidden states from the output |
|
hidden_states = outputs.last_hidden_state |
|
|
|
# Get the CLS token representation (first token) |
|
cls_token = hidden_states[:, 0, :] |
|
|
|
# Apply the pooling layer to the CLS token representation |
|
pooled_output = self.pooling_layer(cls_token) |
|
# Include the pooled_output in the output dictionary as 'pooling_layer' |
|
outputs["pooler_output"] = pooled_output |
|
|
|
return outputs |
|
|
|
def save_pretrained(self, path): |
|
# Save the model's state_dict, configuration, and tokenizer |
|
state_dict = self.deberta.state_dict() |
|
state_dict.update(self.pooling_layer[0].state_dict()) |
|
|
|
torch.save(state_dict, f"{path}/pytorch_model.bin") |
|
self.deberta.config.save_pretrained(path) |
|
self.tokenizer.save_pretrained(path) |
|
|
|
def load_state_dict(self, state_dict: Mapping[str, Any], strict: bool = True): |
|
pooler_keys = ['bias', 'weight'] |
|
deberta_state_dict = {k: v for k, v in state_dict.items() if k not in pooler_keys} |
|
pooler_state_dict = {k: v for k, v in state_dict.items() if k in pooler_keys} |
|
self.deberta.load_state_dict(deberta_state_dict, strict=strict) |
|
self.pooling_layer[0].load_state_dict(pooler_state_dict) |
|
|
|
@classmethod |
|
def from_pretrained(cls, name): |
|
# Initialize the instance |
|
instance = cls(name) |
|
|
|
try: |
|
# Load the model's state_dict |
|
instance.load_state_dict(torch.load(f"{name}/pytorch_model.bin")) |
|
except FileNotFoundError: |
|
print("Could not find DeBERTa pooling layer. Initialize new values") |
|
|
|
# Load the configuration and tokenizer |
|
instance.deberta.config = AutoConfig.from_pretrained(name) |
|
instance.tokenizer = AutoTokenizer.from_pretrained(name) |
|
|
|
return instance |
|
``` |
