Views
No views yet
| Metric | Before fine-tuning | After fine-tuning |
|---|---|---|
| BLEU score | 14.06 | 26.86 |
transformers library.1from transformers import AutoModelForSeq2SeqLM, AutoTokenizer
2
3model_name = "Ntui-Chinua/aAIN"
4tokenizer = AutoTokenizer.from_pretrained(model_name)
5model = AutoModelForSeq2SeqLM.from_pretrained(model_name)
6
7tokenizer.src_lang = "eng_Latn"
8inputs = tokenizer("I am going to the market.", return_tensors="pt")
9forced_bos_token_id = tokenizer.convert_tokens_to_ids("ibo_Latn")
10
11generated_tokens = model.generate(**inputs, forced_bos_token_id=forced_bos_token_id, max_length=128)
12print(tokenizer.batch_decode(generated_tokens, skip_special_tokens=True)[0])