Views
No views yet
1import random
2
3
4alpaca_prompt = """<original>{}</original>
5<translate to="{}">{}"""
6
7BOS_TOKEN = tokenizer.bos_token # Must add EOS_TOKEN
8EOS_TOKEN = "</translate>"+tokenizer.eos_token # Must add EOS_TOKEN
9def formatting_prompts_func(examples):
10 translations = examples["translation"]
11 texts = []
12 text_en = ""
13 text_th = ""
14 translate_to = 'th'
15 max_group_count = 1
16 group_count = 0
17 for translation in translations:
18
19 if group_count >= max_group_count:
20 if(translate_to == 'th'):
21 text = alpaca_prompt.format(text_en, translate_to, text_th) + EOS_TOKEN
22 else:
23 text = alpaca_prompt.format(text_th, translate_to, text_en) + EOS_TOKEN
24 texts.append(text)
25 text_en = ""
26 text_th = ""
27 max_group_count = random.randint(1, 5)
28 group_count = 0
29 translate_to = random.choice(['en', 'th'])
30
31 num_newlines = random.randint(1, 5)
32 newlines = '\n' * num_newlines
33 if(text_en == ""):
34 text_en = translation['en']
35 text_th = translation['th']
36 else:
37 text_en = text_en+newlines+translation['en']
38 text_th = text_th+newlines+translation['th']
39 group_count = group_count+1
40 if(translate_to == 'th'):
41 text = alpaca_prompt.format(text_en, translate_to, text_th) + EOS_TOKEN
42 else:
43 text = alpaca_prompt.format(text_th, translate_to, text_en) + EOS_TOKEN
44 texts.append(text)
45 return { "text" : texts, }
46
47
48from datasets import load_dataset
49dataset = load_dataset("scb_mt_enth_2020",'enth',split="test")
50dataset = dataset.map(formatting_prompts_func, batched = True,remove_columns=["translation",'subdataset'])
51dataset = dataset.train_test_split(test_size=0.1, shuffle=True)
52dataset['train'][0:5]