You can deploy this model and start earning money today!

felerminoali/
nllb_bible_and_news_multilingual_en-pt-vmw-ny-seh-ts

Name: nllb_bible_and_news_multilingual_en-pt-vmw-ny-seh-ts
Availability: InStock

pytorch

m2m_100

apache-2.0

Views

No views yet

python

from transformers import AutoModelForSeq2SeqLM, NllbTokenizer, AutoTokenizer
import torch

#src_lang="por_Latn"

data = []

src_lang="por_Latn"
tgt_lang="ts_Latn"
text="<2ts> ‘Ele foi muito feliz’!"
data.append((src_lang, tgt_lang, text))


src_lang="por_Latn"
tgt_lang="sw_Latn"
text="<2sw> Ele foi muito feliz!"
data.append((src_lang, tgt_lang, text))

src_lang="eng_Latn"
tgt_lang="vmw_Latn"
text="<2vmw> Hello, world of languages!"
data.append((src_lang, tgt_lang, text))


src_lang="por_Latn"
tgt_lang="vmw_Latn"
text="<2vmw> Olá, mundo das línguas! Estou muito feliz"
data.append((src_lang, tgt_lang, text))

src_lang="por_Latn"
tgt_lang="nya_Latn"
text="<2ny> ola, mundo das línguas!"
data.append((src_lang, tgt_lang, text))


src_lang="por_Latn"
tgt_lang="seh_Latn"
text="<2seh> ola, mundo das línguas!"
data.append((src_lang, tgt_lang, text))




device = "cuda:0" if torch.cuda.is_available() else "cpu"

model_name="models/nllb_MULTI"
model = AutoModelForSeq2SeqLM.from_pretrained(model_name).to(device)
tokenizer = NllbTokenizer.from_pretrained(model_name)


# Get the list of existing languages
existing_languages = tokenizer.additional_special_tokens

# Add the new language token to the list
existing_languages = existing_languages + ["vmw_Latn", "seh_Latn", "ts_Latn"]

# Reinitialize the tokenizer with the updated list of languages
tokenizer = AutoTokenizer.from_pretrained(model_name, additional_special_tokens=existing_languages)
model.resize_token_embeddings(len(tokenizer))


for src_lang, tgt_lang, text in data:
    
    print(src_lang, tgt_lang, text)
    tokenizer.src_lang = src_lang
    tokenizer.tgt_lang = tgt_lang

    inputs = tokenizer(
        text, return_tensors='pt', padding=True, truncation=True, 
        max_length=1024
    )
    model.eval() # turn off training mode
    result = model.generate(
        **inputs.to(model.device),
        forced_bos_token_id=tokenizer.convert_tokens_to_ids(tgt_lang),
        num_beams=5,
    )
    print("Translation:")
    print(tokenizer.batch_decode(result, skip_special_tokens=True)[0])

felerminoali/nllb_bible_and_news_multilingual_en-pt-vmw-ny-seh-ts

felerminoali/
nllb_bible_and_news_multilingual_en-pt-vmw-ny-seh-ts