|
|
|
src="KoichiYasuoka/ltgbert-base-belarusian-upos" |
|
tgt="KoichiYasuoka/ltgbert-base-belarusian-ud-goeswith" |
|
url="https://github.com/UniversalDependencies/UD_Belarusian-HSE" |
|
import os |
|
d=os.path.basename(url) |
|
os.system(f"test -d {d} || git clone --depth=1 {url}") |
|
os.system("for F in train dev test ; do cp "+d+"/*-$F.conllu $F.conllu ; done") |
|
class UDgoeswithDataset(object): |
|
def __init__(self,conllu,tokenizer): |
|
self.ids,self.tags,label=[],[],set() |
|
with open(conllu,"r",encoding="utf-8") as r: |
|
cls,sep,msk=tokenizer.cls_token_id,tokenizer.sep_token_id,tokenizer.mask_token_id |
|
dep,c="-|_|dep",[] |
|
for s in r: |
|
t=s.split("\t") |
|
if len(t)==10 and t[0].isdecimal(): |
|
c.append(t) |
|
elif c!=[] and s.strip()=="": |
|
v=tokenizer([t[1] for t in c],add_special_tokens=False)["input_ids"] |
|
for i in range(len(v)-1,-1,-1): |
|
if v[i]==[]: |
|
v[i]=[tokenizer.unk_token_id] |
|
else: |
|
if i>0 and c[i-1][9].find("SpaceAfter=No")>=0: |
|
k=tokenizer.convert_ids_to_tokens(v[i][0]) |
|
if k.startswith("âĸģ"): |
|
v[i][0]=tokenizer.convert_tokens_to_ids(k[3:]) |
|
for j in range(1,len(v[i])): |
|
c.insert(i+1,[c[i][0],"_","_","X","_","_",c[i][0],"goeswith","_","_"]) |
|
y=["0"]+[t[0] for t in c] |
|
h=[i if t[6]=="0" else y.index(t[6]) for i,t in enumerate(c,1)] |
|
p,v=[t[3]+"|"+t[5]+"|"+t[7] for t in c],sum(v,[]) |
|
self.ids.append([cls]+v+[sep]) |
|
self.tags.append([dep]+p+[dep]) |
|
label=set(sum([self.tags[-1],list(label)],[])) |
|
for i,k in enumerate(v): |
|
self.ids.append([cls]+v[0:i]+[msk]+v[i+1:]+[sep,k]) |
|
self.tags.append([dep]+[t if h[j]==i+1 else dep for j,t in enumerate(p)]+[dep,dep]) |
|
c=[] |
|
self.label2id={l:i for i,l in enumerate(sorted(label))} |
|
def __call__(*args): |
|
label=set(sum([list(t.label2id) for t in args],[])) |
|
lid={l:i for i,l in enumerate(sorted(label))} |
|
for t in args: |
|
t.label2id=lid |
|
return lid |
|
__len__=lambda self:len(self.ids) |
|
__getitem__=lambda self,i:{"input_ids":self.ids[i],"labels":[self.label2id[t] for t in self.tags[i]]} |
|
from transformers import AutoTokenizer,AutoConfig,AutoModelForTokenClassification,DataCollatorForTokenClassification,TrainingArguments,Trainer |
|
tkz=AutoTokenizer.from_pretrained(src) |
|
trainDS=UDgoeswithDataset("train.conllu",tkz) |
|
devDS=UDgoeswithDataset("dev.conllu",tkz) |
|
testDS=UDgoeswithDataset("test.conllu",tkz) |
|
lid=trainDS(devDS,testDS) |
|
cfg=AutoConfig.from_pretrained(src,num_labels=len(lid),label2id=lid,id2label={i:l for l,i in lid.items()},ignore_mismatched_sizes=True,trust_remote_code=True) |
|
arg=TrainingArguments(num_train_epochs=3,per_device_train_batch_size=24,output_dir="/tmp",overwrite_output_dir=True,save_total_limit=2,learning_rate=5e-05,warmup_ratio=0.1,save_safetensors=False) |
|
trn=Trainer(args=arg,data_collator=DataCollatorForTokenClassification(tkz),model=AutoModelForTokenClassification.from_pretrained(src,config=cfg,ignore_mismatched_sizes=True,trust_remote_code=True),train_dataset=trainDS,eval_dataset=devDS) |
|
trn.train() |
|
trn.save_model(tgt) |
|
tkz.save_pretrained(tgt) |
|
|
