Views
No views yet
context to avoid ambiguity when specifying a multiple-used word as question.1from transformers import AutoTokenizer,AutoModelForQuestionAnswering
2tokenizer=AutoTokenizer.from_pretrained("KoichiYasuoka/deberta-base-japanese-wikipedia-ud-head")
3model=AutoModelForQuestionAnswering.from_pretrained("KoichiYasuoka/deberta-base-japanese-wikipedia-ud-head")
4question="国語"
5context="全学年にわたって小学校の国語の教科書に挿し絵が用いられている"
6inputs=tokenizer(question,context,return_tensors="pt",return_offsets_mapping=True)
7offsets=inputs.pop("offset_mapping").tolist()[0]
8outputs=model(**inputs)
9start,end=outputs.start_logits.argmax(),outputs.end_logits.argmax()
10print(context[offsets[start][0]:offsets[end][-1]])1class TransformersUD(object):
2 def __init__(self,bert):
3 import os
4 from transformers import (AutoTokenizer,AutoModelForQuestionAnswering,
5 AutoModelForTokenClassification,AutoConfig,TokenClassificationPipeline)
6 self.tokenizer=AutoTokenizer.from_pretrained(bert)
7 self.model=AutoModelForQuestionAnswering.from_pretrained(bert)
8 x=AutoModelForTokenClassification.from_pretrained
9 if os.path.isdir(bert):
10 d,t=x(os.path.join(bert,"deprel")),x(os.path.join(bert,"tagger"))
11 else:
12 from transformers.utils import cached_file
13 c=cached_file(bert,"deprel/config.json")
14 d=x(os.path.dirname(cached_file(bert,"deprel/pytorch_model.bin")))
15 s=cached_file(bert,"tagger/config.json")
16 t=x(os.path.dirname(cached_file(bert,"tagger/pytorch_model.bin")))
17 self.deprel=TokenClassificationPipeline(model=d,tokenizer=self.tokenizer,
18 aggregation_strategy="simple")
19 self.tagger=TokenClassificationPipeline(model=t,tokenizer=self.tokenizer)
20 def __call__(self,text):
21 import numpy,torch,ufal.chu_liu_edmonds
22 w=[(t["start"],t["end"],t["entity_group"]) for t in self.deprel(text)]
23 z,n={t["start"]:t["entity"].split("|") for t in self.tagger(text)},len(w)
24 r,m=[text[s:e] for s,e,p in w],numpy.full((n+1,n+1),numpy.nan)
25 v,c=self.tokenizer(r,add_special_tokens=False)["input_ids"],[]
26 for i,t in enumerate(v):
27 q=[self.tokenizer.cls_token_id]+t+[self.tokenizer.sep_token_id]
28 c.append([q]+v[0:i]+[[self.tokenizer.mask_token_id]]+v[i+1:]+[[q[-1]]])
29 b=[[len(sum(x[0:j+1],[])) for j in range(len(x))] for x in c]
30 with torch.no_grad():
31 d=self.model(input_ids=torch.tensor([sum(x,[]) for x in c]),
32 token_type_ids=torch.tensor([[0]*x[0]+[1]*(x[-1]-x[0]) for x in b]))
33 s,e=d.start_logits.tolist(),d.end_logits.tolist()
34 for i in range(n):
35 for j in range(n):
36 m[i+1,0 if i==j else j+1]=s[i][b[i][j]]+e[i][b[i][j+1]-1]
37 h=ufal.chu_liu_edmonds.chu_liu_edmonds(m)[0]
38 if [0 for i in h if i==0]!=[0]:
39 i=([p for s,e,p in w]+["root"]).index("root")
40 j=i+1 if i<n else numpy.nanargmax(m[:,0])
41 m[0:j,0]=m[j+1:,0]=numpy.nan
42 h=ufal.chu_liu_edmonds.chu_liu_edmonds(m)[0]
43 u="# text = "+text.replace("\n"," ")+"\n"
44 for i,(s,e,p) in enumerate(w,1):
45 p="root" if h[i]==0 else "dep" if p=="root" else p
46 u+="\t".join([str(i),r[i-1],"_",z[s][0][2:],"_","|".join(z[s][1:]),
47 str(h[i]),p,"_","_" if i<n and e<w[i][0] else "SpaceAfter=No"])+"\n"
48 return u+"\n"
49
50nlp=TransformersUD("KoichiYasuoka/deberta-base-japanese-wikipedia-ud-head")
51print(nlp("全学年にわたって小学校の国語の教科書に挿し絵が用いられている"))