| |
| src="nlp-waseda/roberta-base-japanese" |
| tgt="KoichiYasuoka/roberta-base-japanese-juman-ud-goeswith" |
| url="https://github.com/KoichiYasuoka/SuPar-UniDic/raw/main/suparunidic/suparmodels/ja_gsd_modern.conllu" |
| import os |
| f=os.path.basename(url) |
| os.system("test -f "+f+" || curl -LO "+url) |
| class UDgoeswithDataset(object): |
| def __init__(self,conllu,tokenizer): |
| self.ids,self.tags,label=[],[],set() |
| with open(conllu,"r",encoding="utf-8") as r: |
| cls,sep,msk=tokenizer.cls_token_id,tokenizer.sep_token_id,tokenizer.mask_token_id |
| dep,c="-|_|dep",[] |
| for s in r: |
| t=s.split("\t") |
| if len(t)==10 and t[0].isdecimal(): |
| c.append(t) |
| elif c!=[] and s.strip()=="": |
| v=tokenizer([t[1] for t in c],add_special_tokens=False)["input_ids"] |
| for i in range(len(v)-1,-1,-1): |
| for j in range(1,len(v[i])): |
| c.insert(i+1,[c[i][0],"_","_","X","_","_",c[i][0],"goeswith","_","_"]) |
| y=["0"]+[t[0] for t in c] |
| h=[i if t[6]=="0" else y.index(t[6]) for i,t in enumerate(c,1)] |
| p,v=[t[3]+"|"+t[5]+"|"+t[7] for t in c],sum(v,[]) |
| self.ids.append([cls]+v+[sep]) |
| self.tags.append([dep]+p+[dep]) |
| label=set(sum([self.tags[-1],list(label)],[])) |
| for i,k in enumerate(v): |
| self.ids.append([cls]+v[0:i]+[msk]+v[i+1:]+[sep,k]) |
| self.tags.append([dep]+[t if h[j]==i+1 else dep for j,t in enumerate(p)]+[dep,dep]) |
| c=[] |
| self.label2id={l:i for i,l in enumerate(sorted(label))} |
| def __call__(*args): |
| label=set(sum([list(t.label2id) for t in args],[])) |
| lid={l:i for i,l in enumerate(sorted(label))} |
| for t in args: |
| t.label2id=lid |
| return lid |
| __len__=lambda self:len(self.ids) |
| __getitem__=lambda self,i:{"input_ids":self.ids[i],"labels":[self.label2id[t] for t in self.tags[i]]} |
| from transformers import AutoTokenizer,AutoConfig,AutoModelForTokenClassification,DataCollatorForTokenClassification,TrainingArguments,Trainer |
| tkz=AutoTokenizer.from_pretrained(src) |
| trainDS=UDgoeswithDataset(f,tkz) |
| lid=trainDS.label2id |
| cfg=AutoConfig.from_pretrained(src,num_labels=len(lid),label2id=lid,id2label={i:l for l,i in lid.items()}) |
| arg=TrainingArguments(num_train_epochs=3,per_device_train_batch_size=32,output_dir="/tmp",overwrite_output_dir=True,save_total_limit=2,learning_rate=5e-05,warmup_ratio=0.1) |
| trn=Trainer(args=arg,data_collator=DataCollatorForTokenClassification(tkz),model=AutoModelForTokenClassification.from_pretrained(src,config=cfg),train_dataset=trainDS) |
| trn.train() |
| trn.save_model(tgt) |
| tkz.save_pretrained(tgt) |
|
|
