Views
No views yet
myv, cyrillic script) to 11 other languages: ru,fi,de,es,en,hi,zh,tr,uk,fr,ar. See its demo!myv_XX and 19K new BPE tokens for the Erzya language;1from transformers import MBartForConditionalGeneration, MBart50Tokenizer
2
3
4def fix_tokenizer(tokenizer):
5 """ Add a new language token to the tokenizer vocabulary (this should be done each time after its initialization) """
6 old_len = len(tokenizer) - int('myv_XX' in tokenizer.added_tokens_encoder)
7 tokenizer.lang_code_to_id['myv_XX'] = old_len-1
8 tokenizer.id_to_lang_code[old_len-1] = 'myv_XX'
9 tokenizer.fairseq_tokens_to_ids["<mask>"] = len(tokenizer.sp_model) + len(tokenizer.lang_code_to_id) + tokenizer.fairseq_offset
10
11 tokenizer.fairseq_tokens_to_ids.update(tokenizer.lang_code_to_id)
12 tokenizer.fairseq_ids_to_tokens = {v: k for k, v in tokenizer.fairseq_tokens_to_ids.items()}
13 if 'myv_XX' not in tokenizer._additional_special_tokens:
14 tokenizer._additional_special_tokens.append('myv_XX')
15 tokenizer.added_tokens_encoder = {}
16
17
18def translate(text, model, tokenizer, src='ru_RU', trg='myv_XX', max_length='auto', num_beams=3, repetition_penalty=5.0, train_mode=False, n_out=None, **kwargs):
19 tokenizer.src_lang = src
20 encoded = tokenizer(text, return_tensors="pt", truncation=True, max_length=1024)
21 if max_length == 'auto':
22 max_length = int(32 + 1.5 * encoded.input_ids.shape[1])
23 if train_mode:
24 model.train()
25 else:
26 model.eval()
27 generated_tokens = model.generate(
28 **encoded.to(model.device),
29 forced_bos_token_id=tokenizer.lang_code_to_id[trg],
30 max_length=max_length,
31 num_beams=num_beams,
32 repetition_penalty=repetition_penalty,
33 num_return_sequences=n_out or 1,
34 **kwargs
35 )
36 out = tokenizer.batch_decode(generated_tokens, skip_special_tokens=True)
37 if isinstance(text, str) and n_out is None:
38 return out[0]
39 return out
40
41
42mname = 'slone/mbart-large-51-mul-myv-v1'
43model = MBartForConditionalGeneration.from_pretrained(mname)
44tokenizer = MBart50Tokenizer.from_pretrained(mname)
45fix_tokenizer(tokenizer)
46
47
48print(translate('Привет, собака!', model, tokenizer, src='ru_RU', trg='myv_XX'))
49# Шумбрат, киска! # действительно, по-эрзянски собака именно так
50print(translate('Hello, doggy!', model, tokenizer, src='en_XX', trg='myv_XX'))
51# Шумбрат, киска!