Views
No views yet
| Language | Number of text samples | Number of tokens |
|---|---|---|
| Catalan | 57,543 | 2,299,616 |
1from transformers import pipeline, AutoModelForTokenClassification, AutoTokenizer
2import torch
3
4def get_result_text_ca (list_entity, text):
5 result_words = []
6 punc_tags = ['?', '!', ',', '.', ':']
7 tmp_word = ""
8 for idx, entity in enumerate(list_entity):
9 start = entity["start"]
10 end = entity["end"]
11 tag = entity["entity"]
12 word = entity["word"]
13
14 # check punctuation
15 punc_in = next((p for p in punc_tags if p in tag), "")
16
17 subword = False
18 # check subwords
19 if word[0] != "Ġ":
20 subword = True
21 if tmp_word == "":
22 p_s = list_entity[idx-1]["start"]
23 p_e = list_entity[idx-1]["end"]
24 tmp_word = text[p_s:p_e] + text[start:end]
25 else:
26 tmp_word = tmp_word + text[start:end]
27 word = tmp_word
28 else:
29 tmp_word = ""
30 word = text[start:end]
31
32 if tag == "l":
33 word = word
34 elif tag == "u":
35 word = word.capitalize()
36 # case with punctuation
37 else:
38 if tag[-1] == "l":
39 word = (punc_in + word) if punc_in in ["¿", "¡"] else (word + punc_in)
40 elif tag[-1] == "u":
41 word = (punc_in + word.capitalize()) if punc_in in ["¿", "¡"] else (word.capitalize() + punc_in)
42
43 if subword == True:
44 result_words[-1] = word
45 else:
46 result_words.append(word)
47
48 return " ".join(result_words)
49
50
51lang = "ca"
52model_path = "VOCALINLP/catalan_capitalization_punctuation_restoration_sanivert"
53
54model = AutoModelForTokenClassification.from_pretrained(model_path)
55tokenizer = AutoTokenizer.from_pretrained(model_path)
56
57pipe = pipeline("token-classification", model=model, tokenizer=tokenizer)
58
59text = "el pacient presenta els símptomes següents febre dispnea nàusees i vòmits"
60result = pipe(text)
61
62print("Source text: "+ text)
63result_text = get_result_text_ca(result, text)
64print("Restored text: " +result_text)Created by VOCALI SISSTEMAS INTELIGENTES S.L.