Views
No views yet
model.config.id2label).1from transformers import CanineForTokenClassification, AutoTokenizer
2import torch
3
4model = CanineForTokenClassification.from_pretrained("guymorlan/levanti_diacritics2translit")
5tokenizer = AutoTokenizer.from_pretrained("guymorlan/levanti_diacritics2translit")
6
7def diacritics2hebrew_vowels(text, model, tokenizer):
8 tokens = tokenizer(text, return_tensors="pt")
9 with torch.no_grad():
10 pred = model(**tokens)
11 pred = pred.logits.argmax(-1).tolist()
12
13 pred = pred[0][1:-1] # remove CLS and SEP
14 output = []
15 for p, c in zip(pred, text):
16 if p != model.config.label2id["O"]:
17 output.append(model.config.id2label[p])
18 else:
19 output.append(c)
20 output = "".join(output)
21 return output
22
23# to convert arabic diacritics to Hebrew diacritics (Tsere, Holam, Patah, Shva, Kubutz, Hiriq)
24text = "لَازِم نِعْطِي رَشَّات وِقَائِيِّة لِلشَّجَر "
25heb_vowels = diacritics2hebrew_vowels(text, model, tokenizer)
26heb_vowelsOut[1]: 'لַازֵم نִعְطִي رַشַّات وִقַائִيֵّة لִلشַّجַر '1arabic_to_hebrew = {
2 # regular letters
3 "ا": "א", "أ": "א", "إ": "א", "ء": "א", "ئ": "א", "ؤ": "א",
4 "آ": "אא", "ى": "א", "ب": "ב", "ت": "ת", "ث": "ת'", "ج": "ג'",
5 "ح": "ח", "خ": "ח'", "د": "ד", "ذ": "ד'", "ر": "ר", "ز": "ז",
6 "س": "ס", "ش": "ש", "ص": "צ", "ض": "צ'", "ط": "ט", "ظ": "ט'",
7 "ع": "ע", "غ": "ע'", "ف": "פ", "ق": "ק", "ك": "כ", "ل": "ל",
8 "م": "מ", "ن": "נ", "ه": "ה", "و": "ו", "ي": "י", "ة": "ה",
9 # special characters
10 "،": ",", "َ": "ַ", "ُ": "ֻ", "ِ": "ִ",
11}
12
13final_letters = {
14 "ن": "ן", "م": "ם", "ص": "ץ", "ض": "ץ'", "ف": "ף",
15}
16
17def to_taatik(arabic):
18 taatik = []
19 for index, letter in enumerate(arabic):
20 if (
21 (index == len(arabic) - 1 or arabic[index + 1] in {" ", ".", "،"}) and
22 letter in final_letters
23 ):
24 taatik.append(final_letters[letter])
25 elif letter not in arabic_to_hebrew:
26 taatik.append(letter)
27 else:
28 taatik.append(arabic_to_hebrew[letter])
29 return "".join(taatik)
30
31# to convert consonants and create full hebrew transliteration (Taatik)
32to_taatik(heb_vowels)Out[2]: "לַאזֵם נִעְטִי רַשַّאת וִקַאאִיֵّה לִלשַّג'ַר "1arabic_to_english = {
2 "ا": "a", "أ": "a", "إ": "a", "ء": "a", "ئ": "a", "ؤ": "a",
3 "آ": "aa", "ى": "a", "ب": "b", "ت": "t", "ث": "th", "ج": "j",
4 "ح": "h", "خ": "kh", "د": "d", "ذ": "dh", "ر": "r", "ز": "z",
5 "س": "s", "ش": "sh", "ص": "s", "ض": "d", "ط": "t", "ظ": "z",
6 "ع": "a", "غ": "gh", "ف": "f", "ق": "q", "ك": "k", "ل": "l",
7 "م": "m", "ن": "n", "ه": "h", "و": "w", "ي": "y", "ة": "h",
8 "َ": "a", "ُ": "u", "ِ": "i",
9 "،": ",",
10 "ֹ": "o", # holam
11 "ַ": "a", # patah
12 "ִ": "i", # hiriq
13 "ְ": "", # shva
14 "ֻ": "u", # kubutz
15 'ֵ': "e",
16 "ّ": "SHADDA" # shadda
17}
18
19vowels = ["،", ",", "َ", "ַ", "ُ", "ֻ", "ِ", "ִ", 'ֵ']
20
21
22def to_translit(arabic):
23 translit = []
24 for letter in arabic:
25 if letter not in arabic_to_english:
26 translit.append([letter, letter])
27 else:
28 if arabic_to_english[letter] == "SHADDA":
29 if translit[-1][0] in vowels:
30 translit[-2][1] = translit[-2][1].upper()
31 else:
32 translit[-1][1] = translit[-1][1].upper()
33
34 else:
35 translit.append([letter, arabic_to_english[letter]])
36
37 return "".join([x[1] for x in translit])
38
39# to convert letters to latin representation (English transliteration)
40to_translit(heb_vowels)Out[3]: 'laazem niatiy raSHaat wiqaaaiYeh lilSHajar '