Views
No views yet
context to avoid ambiguity when specifying a multiple-used word as question.1from transformers import AutoTokenizer,AutoModelForQuestionAnswering
2tokenizer=AutoTokenizer.from_pretrained("KoichiYasuoka/deberta-base-japanese-unidic-ud-head")
3model=AutoModelForQuestionAnswering.from_pretrained("KoichiYasuoka/deberta-base-japanese-unidic-ud-head")
4question="国語"
5context="全学年にわたって小学校の国語の教科書に挿し絵が用いられている"
6inputs=tokenizer(question,context,return_tensors="pt")
7outputs=model(**inputs)
8start,end=outputs.start_logits.argmax(),outputs.end_logits.argmax()
9print(tokenizer.convert_ids_to_tokens(inputs["input_ids"][0,start:end+1]))1from transformers import (AutoTokenizer,AutoModelForQuestionAnswering,
2 AutoModelForTokenClassification,AutoConfig,TokenClassificationPipeline)
3class TaggerPipeline(TokenClassificationPipeline):
4 def __call__(self,text):
5 d=super().__call__(text)
6 if len(d)>0 and ("start" not in d[0] or d[0]["start"]==None):
7 import spacy_alignments as tokenizations
8 v=[x["word"].replace(" ","") for x in d]
9 a2b,b2a=tokenizations.get_alignments(v,[c for c in text])
10 for i,t in enumerate(a2b):
11 s,e=(0,0) if t==[] else (t[0],t[-1]+1)
12 if v[i].startswith(self.tokenizer.unk_token):
13 s=([[-1]]+[x for x in a2b[0:i] if x>[]])[-1][-1]+1
14 if v[i].endswith(self.tokenizer.unk_token):
15 e=([x for x in a2b[i+1:] if x>[]]+[[len(text)]])[0][0]
16 d[i]["start"],d[i]["end"]=s,e
17 return d
18class TransformersSlowUD(object):
19 def __init__(self,bert):
20 import os
21 self.tokenizer=AutoTokenizer.from_pretrained(bert)
22 self.model=AutoModelForQuestionAnswering.from_pretrained(bert)
23 x=AutoModelForTokenClassification.from_pretrained
24 if os.path.isdir(bert):
25 d,t=x(os.path.join(bert,"deprel")),x(os.path.join(bert,"tagger"))
26 else:
27 from transformers.utils import cached_file
28 c=cached_file(bert,"deprel/config.json")
29 d=x(os.path.dirname(cached_file(bert,"deprel/pytorch_model.bin")))
30 s=cached_file(bert,"tagger/config.json")
31 t=x(os.path.dirname(cached_file(bert,"tagger/pytorch_model.bin")))
32 self.deprel=TaggerPipeline(model=d,tokenizer=self.tokenizer,
33 aggregation_strategy="simple")
34 self.tagger=TaggerPipeline(model=t,tokenizer=self.tokenizer)
35 def __call__(self,text):
36 import numpy,torch,ufal.chu_liu_edmonds
37 w=[(t["start"],t["end"],t["entity_group"]) for t in self.deprel(text)]
38 z,n={t["start"]:t["entity"].split("|") for t in self.tagger(text)},len(w)
39 r,m=[text[s:e] for s,e,p in w],numpy.full((n+1,n+1),numpy.nan)
40 v,c=self.tokenizer(r,add_special_tokens=False)["input_ids"],[]
41 for i,t in enumerate(v):
42 q=[self.tokenizer.cls_token_id]+t+[self.tokenizer.sep_token_id]
43 c.append([q]+v[0:i]+[[self.tokenizer.mask_token_id]]+v[i+1:]+[[q[-1]]])
44 b=[[len(sum(x[0:j+1],[])) for j in range(len(x))] for x in c]
45 with torch.no_grad():
46 d=self.model(input_ids=torch.tensor([sum(x,[]) for x in c]),
47 token_type_ids=torch.tensor([[0]*x[0]+[1]*(x[-1]-x[0]) for x in b]))
48 s,e=d.start_logits.tolist(),d.end_logits.tolist()
49 for i in range(n):
50 for j in range(n):
51 m[i+1,0 if i==j else j+1]=s[i][b[i][j]]+e[i][b[i][j+1]-1]
52 h=ufal.chu_liu_edmonds.chu_liu_edmonds(m)[0]
53 if [0 for i in h if i==0]!=[0]:
54 i=([p for s,e,p in w]+["root"]).index("root")
55 j=i+1 if i<n else numpy.nanargmax(m[:,0])
56 m[0:j,0]=m[j+1:,0]=numpy.nan
57 h=ufal.chu_liu_edmonds.chu_liu_edmonds(m)[0]
58 u="# text = "+text.replace("\n"," ")+"\n"
59 for i,(s,e,p) in enumerate(w,1):
60 p="root" if h[i]==0 else "dep" if p=="root" else p
61 u+="\t".join([str(i),r[i-1],"_",z[s][0][2:],"_","|".join(z[s][1:]),
62 str(h[i]),p,"_","_" if i<n and e<w[i][0] else "SpaceAfter=No"])+"\n"
63 return u+"\n"
64
65nlp=TransformersSlowUD("KoichiYasuoka/deberta-base-japanese-unidic-ud-head")
66print(nlp("全学年にわたって小学校の国語の教科書に挿し絵が用いられている"))