model improved for transformers 4.42

config.json

@@ -3,22 +3,11 @@
     "MistralForTokenClassification"
   ],
   "attention_dropout": 0.0,
-  "auto_map": {
-    "AutoModelForTokenClassification": "upos.MistralForTokenClassification"
-  },
   "bos_token_id": 1,
   "custom_pipelines": {
     "upos": {
       "impl": "upos.BellmanFordTokenClassificationPipeline",
       "pt": "AutoModelForTokenClassification"
-    },
-    "token-classification": {
-      "impl": "upos.RawTokenClassificationPipeline",
-      "pt": "AutoModelForTokenClassification"
-    },
-    "ner": {
-      "impl": "upos.RawTokenClassificationPipeline",
-      "pt": "AutoModelForTokenClassification"
     }
   },
   "eos_token_id": 2,
@@ -161,7 +150,7 @@
   "tie_word_embeddings": false,
   "tokenizer_class": "LlamaTokenizerFast",
   "torch_dtype": "float32",
-  "transformers_version": "4.41.2",
+  "transformers_version": "4.42.4",
   "use_cache": true,
   "use_transformers_inputs": true,
   "vocab_size": 48000

maker.sh

@@ -30,45 +30,9 @@ cat << 'EOF' > $TMPB
 #! /usr/bin/env deepspeed
 src="exRakutenAI-7B"
 tgt="KoichiYasuoka/RakutenAI-7B-upos"
-from transformers import LlamaTokenizerFast,MistralModel,MistralPreTrainedModel,AutoConfig,DataCollatorForTokenClassification,TrainingArguments,Trainer
-from transformers.modeling_outputs import TokenClassifierOutput
+from transformers import LlamaTokenizerFast,MistralForTokenClassification,AutoConfig,DataCollatorForTokenClassification,TrainingArguments,Trainer
 from tokenizers.normalizers import Replace
 
-class MistralForTokenClassification(MistralPreTrainedModel):
-  def __init__(self,config):
-    from torch import nn
-    super().__init__(config)
-    self.num_labels=config.num_labels
-    self.model=MistralModel(config)
-    if hasattr(config,"classifier_dropout") and config.classifier_dropout is not None:
-      classifier_dropout=config.classifier_dropout
-    elif hasattr(config,"hidden_dropout") and config.hidden_dropout is not None:
-      classifier_dropout=config.hidden_dropout
-    else:
-      classifier_dropout=0.1
-    self.dropout=nn.Dropout(classifier_dropout)
-    self.classifier=nn.Linear(config.hidden_size,config.num_labels)
-    self.post_init()
-  def get_input_embeddings(self):
-    return self.model.embed_tokens
-  def set_input_embeddings(self,value):
-    self.model.embed_tokens=value
-  def forward(self,input_ids=None,past_key_values=None,attention_mask=None,position_ids=None,inputs_embeds=None,labels=None,use_cache=None,output_attentions=None,output_hidden_states=None,return_dict=None):
-    return_dict=return_dict if return_dict is not None else self.config.use_return_dict
-    transformer_outputs=self.model(input_ids,past_key_values=past_key_values,attention_mask=attention_mask,position_ids=position_ids,inputs_embeds=inputs_embeds,use_cache=use_cache,output_attentions=output_attentions,output_hidden_states=output_hidden_states,return_dict=return_dict)
-    hidden_states=transformer_outputs[0]
-    hidden_states=self.dropout(hidden_states)
-    logits=self.classifier(hidden_states)
-    loss=None
-    if labels is not None:
-      from torch import nn
-      loss_fct=nn.CrossEntropyLoss()
-      loss=loss_fct(logits.view(-1,self.num_labels),labels.view(-1))
-    if not return_dict:
-      output=(logits,)+transformer_outputs[2:]
-      return ((loss,)+output) if loss is not None else output
-    return TokenClassifierOutput(loss=loss,logits=logits,hidden_states=transformer_outputs.hidden_states,attentions=transformer_outputs.attentions)
-
 class UPOSFileDataset(object):
   def __init__(self,conllu,tokenizer):
     self.conllu=open(conllu,"r",encoding="utf-8")

pytorch_model-00001-of-00006.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:858d8cc9eeaa938da0a309452684555abe89c0d86f3d0e4a2ae0524b42443638
+oid sha256:4964869128383677fa5606d0a4ed8d1671e017f9fcee8d5bb76c86698bd0bec7
 size 4913773120

pytorch_model-00002-of-00006.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:07b63ce35674bfcd0be1e29f48811c91b01f6cb0bb168fa7cab687d2aa12929e
+oid sha256:987a14b03e439ece74b9c28d2334fc5a9a84e8ad17e6e6f1c81ad09c70d7fbb0
 size 4999825256

pytorch_model-00003-of-00006.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:016e54d9957c39ddc44acc336b627bbb29932838f02ff33b2aae3c6c66c0a4df
+oid sha256:ff7f12de394e69c4ab16ee0aad77095085d75f3def0ecf8de9b521690a22c536
 size 4999825316

pytorch_model-00004-of-00006.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a0361b986edbe8c04dc733ba3c280d751eda8614958b549ec1f635cb78e63075
+oid sha256:aef0b1302652d535cf187d1c5a833e1508671b7487f2f19001b7cfa8ec778454
 size 4832018324

pytorch_model-00005-of-00006.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a8f1369c379a88759857a71127fc1c08edecf90cc9a21e36e75cd4d2be39b51b
+oid sha256:9657d37d2e4f7c275f3960d5d31ef541223074220ffe5af727b7eed35685da40
 size 4999825320

pytorch_model-00006-of-00006.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8baff249d8aff616fa93640f936e5bc9723f68b4730b415313dd8f7131fdd716
+oid sha256:333bc2435071a127df8259f18bfeef267f86423bcef9e9380dbe70750c85a49f
 size 3960601264

pytorch_model.bin.index.json

@@ -3,8 +3,6 @@
     "total_size": 28705767664
   },
   "weight_map": {
-    "classifier.bias": "pytorch_model-00006-of-00006.bin",
-    "classifier.weight": "pytorch_model-00006-of-00006.bin",
     "model.embed_tokens.weight": "pytorch_model-00001-of-00006.bin",
     "model.layers.0.input_layernorm.weight": "pytorch_model-00001-of-00006.bin",
     "model.layers.0.mlp.down_proj.weight": "pytorch_model-00001-of-00006.bin",
@@ -294,6 +292,8 @@
     "model.layers.9.self_attn.o_proj.weight": "pytorch_model-00002-of-00006.bin",
     "model.layers.9.self_attn.q_proj.weight": "pytorch_model-00002-of-00006.bin",
     "model.layers.9.self_attn.v_proj.weight": "pytorch_model-00002-of-00006.bin",
-    "model.norm.weight": "pytorch_model-00006-of-00006.bin"
+    "model.norm.weight": "pytorch_model-00006-of-00006.bin",
+    "score.bias": "pytorch_model-00006-of-00006.bin",
+    "score.weight": "pytorch_model-00006-of-00006.bin"
   }
 }

tokenizer_config.json

@@ -1,6 +1,7 @@
 {
   "add_bos_token": true,
   "add_eos_token": false,
+  "add_prefix_space": null,
   "added_tokens_decoder": {
     "0": {
       "content": "<unk>",

upos.py

@@ -1,5 +1,4 @@
-from transformers import TokenClassificationPipeline,MistralModel,MistralPreTrainedModel
-from transformers.modeling_outputs import TokenClassifierOutput
+from transformers import TokenClassificationPipeline
 
 class BellmanFordTokenClassificationPipeline(TokenClassificationPipeline):
   def __init__(self,**kwargs):
@@ -40,41 +39,3 @@ class BellmanFordTokenClassificationPipeline(TokenClassificationPipeline):
       t["text"]=model_outputs["sentence"][t["start"]:t["end"]]
     return w
 
-class RawTokenClassificationPipeline(TokenClassificationPipeline):
-  def check_model_type(self,supported_models):
-    pass
-
-class MistralForTokenClassification(MistralPreTrainedModel):
-  def __init__(self,config):
-    from torch import nn
-    super().__init__(config)
-    self.num_labels=config.num_labels
-    self.model=MistralModel(config)
-    if hasattr(config,"classifier_dropout") and config.classifier_dropout is not None:
-      classifier_dropout=config.classifier_dropout
-    elif hasattr(config,"hidden_dropout") and config.hidden_dropout is not None:
-      classifier_dropout=config.hidden_dropout
-    else:
-      classifier_dropout=0.1
-    self.dropout=nn.Dropout(classifier_dropout)
-    self.classifier=nn.Linear(config.hidden_size,config.num_labels)
-    self.post_init()
-  def get_input_embeddings(self):
-    return self.model.embed_tokens
-  def set_input_embeddings(self,value):
-    self.model.embed_tokens=value
-  def forward(self,input_ids=None,past_key_values=None,attention_mask=None,position_ids=None,inputs_embeds=None,labels=None,use_cache=None,output_attentions=None,output_hidden_states=None,return_dict=None):
-    return_dict=return_dict if return_dict is not None else self.config.use_return_dict
-    transformer_outputs=self.model(input_ids,past_key_values=past_key_values,attention_mask=attention_mask,position_ids=position_ids,inputs_embeds=inputs_embeds,use_cache=use_cache,output_attentions=output_attentions,output_hidden_states=output_hidden_states,return_dict=return_dict)
-    hidden_states=transformer_outputs[0]
-    hidden_states=self.dropout(hidden_states)
-    logits=self.classifier(hidden_states)
-    loss=None
-    if labels is not None:
-      from torch import nn
-      loss_fct=nn.CrossEntropyLoss()
-      loss=loss_fct(logits.view(-1,self.num_labels),labels.view(-1))
-    if not return_dict:
-      output=(logits,)+transformer_outputs[2:]
-      return ((loss,)+output) if loss is not None else output
-    return TokenClassifierOutput(loss=loss,logits=logits,hidden_states=transformer_outputs.hidden_states,attentions=transformer_outputs.attentions)