Views
No views yet
from_pretrained method:1from transformers import T5ForConditionalGeneration, AutoTokenizer
2
3model = T5ForConditionalGeneration.from_pretrained("alakxender/flan-t5-base-dhivehi-en-latin")
4tokenizer = AutoTokenizer.from_pretrained("alakxender/flan-t5-base-dhivehi-en-latin")
5
6supported_languages = ["en2dv", "dv2en", "dv2latin", "latin2dv"]
7
8def translate(source_text, target_language):
9 prompt = f"{target_language.strip()} {source_text.strip()}"
10 inputs = tokenizer(prompt, return_tensors="pt", max_length=128, truncation=True)
11 output_ids = model.generate(
12 **inputs,
13 max_length=128,
14 min_length=10,
15 num_beams=4,
16 early_stopping=True,
17 no_repeat_ngram_size=3,
18 repetition_penalty=1.2,
19 do_sample=False,
20 pad_token_id=tokenizer.pad_token_id,
21 eos_token_id=tokenizer.eos_token_id
22 )
23 result = tokenizer.decode(output_ids[0], skip_special_tokens=True)
24 return result
25
26# Example usage
27source_text = "Concerns over prepayment of GST raised in parliament"
28target_language = "en2dv"
29translated_text = translate(source_text, target_language)
30print(translated_text)
31# Output: ރައްޔިތުންގެ މަޖިލީހުގައި ޖީއެސްޓީގެ އަގު ބޮޑުވުމާ ގުޅިގެން ކަންބޮޑުވުން ފާޅުކޮށްފި
32
33source_text = "ދުނިޔޭގެ އެކި ކަންކޮޅުތަކުން 1.4 މިލިއަން މީހުން މައްކާއަށް ޖަމާވެފައި"
34target_language = "dv2en"
35translated_text = translate(source_text, target_language)
36print(translated_text)
37# Output: 1.4 million people gathered in Mecca from different parts of the world
38
39source_text = "ވައިބާރުވުމުން ކުޅުދުއްފުށީ އެއާޕޯޓަށް ނުޖެއްސިގެން މޯލްޑިވިއަންގެ ބޯޓެއް އެނބުރި މާލެއަށް"
40target_language = "dv2latin"
41translated_text = translate(source_text, target_language)
42print(translated_text)
43# Output: Vaibaruvumun kulhudhuhfushee eaapoatah nujehsigen moaldiviange boateh enburi maaleah
44
45source_text = "Paakisthaanuge skoolu bahakah dhin hamalaaehgai thin kuhjakaai bodu dhe meehaku maruvehje"
46target_language = "latin2dv"
47translated_text = translate(source_text, target_language)
48print(translated_text)
49# Output: ޕާކިސްތާނުގެ ސްކޫލު ބަހަކަށް ދިން ހަމަލާއެއްގައި ތިން ކުއްޖަކާއި ބޮޑު ދެ މީހަކު މަރުވެއްޖެ
50