Views
No views yet
1import torch
2from transformers import pipeline
3
4pipe = pipeline("token-classification", model="magistermilitum/roberta-multilingual-medieval-ner")
5
6results = list(map(pipe, list_of_sentences))
7results =[[[y["entity"],y["word"], y["start"], y["end"]] for y in x] for x in results]
8print(results)1class TextProcessor:
2 def __init__(self, filename):
3 self.filename = filename
4 self.sent_detector = nltk.data.load("tokenizers/punkt/english.pickle") #sentence tokenizer
5 self.sentences = []
6 self.new_sentences = []
7 self.results = []
8 self.new_sentences_token_info = []
9 self.new_sentences_bio = []
10 self.BIO_TAGS = []
11 self.stripped_BIO_TAGS = []
12
13 def read_file(self):
14 #Reading a txt file with one document per line.
15 with open(self.filename, 'r') as f:
16 text = f.read()
17 self.sentences = self.sent_detector.tokenize(text.strip())
18
19 def process_sentences(self): #We split long sentences as encoder has a 256 max-lenght. Sentences with les of 40 words will be merged.
20 for sentence in self.sentences:
21 if len(sentence.split()) < 40 and self.new_sentences:
22 self.new_sentences[-1] += " " + sentence
23 else:
24 self.new_sentences.append(sentence)
25
26 def apply_model(self, pipe):
27 self.results = list(map(pipe, self.new_sentences))
28 self.results=[[[y["entity"],y["word"], y["start"], y["end"]] for y in x] for x in self.results]
29
30 def tokenize_sentences(self):
31 for n_s in self.new_sentences:
32 tokens=n_s.split() # Basic tokenization
33 token_info = []
34
35 # Initialize a variable to keep track of character index
36 char_index = 0
37 # Iterate through the tokens and record start and end info
38 for token in tokens:
39 start = char_index
40 end = char_index + len(token) # Subtract 1 for the last character of the token
41 token_info.append((token, start, end))
42
43 char_index += len(token) + 1 # Add 1 for the whitespace
44 self.new_sentences_token_info.append(token_info)
45
46 def process_results(self): #merge subwords and BIO tags
47 for result in self.results:
48 merged_bio_result = []
49 current_word = ""
50 current_label = None
51 current_start = None
52 current_end = None
53 for entity, subword, start, end in result:
54 if subword.startswith("▁"):
55 subword = subword[1:]
56 merged_bio_result.append([current_word, current_label, current_start, current_end])
57 current_word = "" ; current_label = None ; current_start = None ; current_end = None
58 if current_start is None:
59 current_word = subword ; current_label = entity ; current_start = start+1 ; current_end= end
60 else:
61 current_word += subword ; current_end = end
62 if current_word:
63 merged_bio_result.append([current_word, current_label, current_start, current_end])
64 self.new_sentences_bio.append(merged_bio_result[1:])
65
66 def match_tokens_with_entities(self): #match BIO tags with tokens
67 for i,ss in enumerate(self.new_sentences_token_info):
68 for word in ss:
69 for ent in self.new_sentences_bio[i]:
70 if word[1]==ent[2]:
71 if ent[1]=="L-PERS":
72 self.BIO_TAGS.append([word[0], "I-PERS", "B-LOC"])
73 break
74 else:
75 if "LOC" in ent[1]:
76 self.BIO_TAGS.append([word[0], "O", ent[1]])
77 else:
78 self.BIO_TAGS.append([word[0], ent[1], "O"])
79 break
80 else:
81 self.BIO_TAGS.append([word[0], "O", "O"])
82
83 def separate_dots_and_comma(self): #optional
84 signs=[",", ";", ":", "."]
85 for bio in self.BIO_TAGS:
86 if any(bio[0][-1]==sign for sign in signs) and len(bio[0])>1:
87 self.stripped_BIO_TAGS.append([bio[0][:-1], bio[1], bio[2]]);
88 self.stripped_BIO_TAGS.append([bio[0][-1], "O", "O"])
89 else:
90 self.stripped_BIO_TAGS.append(bio)
91
92 def save_BIO(self):
93 with open('output_BIO_a.txt', 'w', encoding='utf-8') as output_file:
94 output_file.write("TOKEN\tPERS\tLOCS\n"+"\n".join(["\t".join(x) for x in self.stripped_BIO_TAGS]))
95
96# Usage:
97processor = TextProcessor('my_docs_file.txt')
98processor.read_file()
99processor.process_sentences()
100processor.apply_model(pipe)
101processor.tokenize_sentences()
102processor.process_results()
103processor.match_tokens_with_entities()
104processor.separate_dots_and_comma()
105processor.save_BIO()1('Ego', 'O', 'O')
2('Radulfus', 'B-PERS')
3('de', 'I-PERS', 'O')
4('Francorvilla', 'I-PERS', 'B-LOC')
5('miles', 'O')
6(',', 'O', 'O')
7('notum', 'O', 'O')
8('facio', 'O', 'O')
9('tam', 'O', 'O')
10('presentibus', 'O', 'O')
11('quam', 'O', 'O')
12('futuris', 'O', 'O')
13('quod', 'O', 'O')
14(',', 'O', 'O')
15('cum', 'O', 'O')
16('Guillelmo', 'B-PERS', 'O')
17('Bateste', 'I-PERS', 'O')
18('militi', 'O', 'O')
19('de', 'O', 'O')
20('Miliaco', 'O', 'B-LOC')1@inproceedings{aguilar2022multilingual,
2 title={Multilingual Named Entity Recognition for Medieval Charters Using Stacked Embeddings and Bert-based Models.},
3 author={Aguilar, Sergio Torres},
4 booktitle={Proceedings of the second workshop on language technologies for historical and ancient languages},
5 pages={119--128},
6 year={2022}
7}