Views
No views yet
peft, transformers and pytorch first.pip install peft transformers torchhuggingface-cli login --token <YOUR_HF_TOKEN>1from peft import PeftConfig, PeftModel
2from transformers import AutoModelForCausalLM, AutoTokenizer
3
4peft_model_id = "ahmedheakl/arazn-gemma1.1-7B-eng-extra"
5peft_config = PeftConfig.from_pretrained(peft_model_id)
6base_model_name = peft_config.base_model_name_or_path
7base_model = AutoModelForCausalLM.from_pretrained(base_model_name)
8model = PeftModel.from_pretrained(base_model, peft_model_id)
9model = model.to("cuda")
10tokenizer = AutoTokenizer.from_pretrained(peft_model_id)1import torch
2
3raw_prompt = """<bos><start_of_turn>user
4Translate the following code-switched Arabic-English-mixed text to English only.
5{source}<end_of_turn>
6<start_of_turn>model
7"""
8def inference(prompt) -> str:
9 prompt = raw_prompt.format(source=prompt)
10 inputs = tokenizer(prompt, return_tensors="pt").to("cuda")
11 generated_ids = model.generate(
12 **inputs,
13 use_cache=True,
14 num_return_sequences=1,
15 max_new_tokens=100,
16 do_sample=True,
17 num_beams=1,
18 temperature=0.7,
19 eos_token_id=tokenizer.eos_token_id,
20 pad_token_id=tokenizer.pad_token_id,
21 )
22 outputs = tokenizer.batch_decode(generated_ids)[0]
23 torch.cuda.empty_cache()
24 torch.cuda.synchronize()
25 return outputs.split("<start_of_turn>model\n")[-1].split("<end_of_turn>")[0]
26
27print(inference("أنا أحب الbanana")) # I like bananas.@article{heakl2024arzen,
title={ArzEn-LLM: Code-Switched Egyptian Arabic-English Translation and Speech Recognition Using LLMs},
author={Heakl, Ahmed and Zaghloul, Youssef and Ali, Mennatullah and Hossam, Rania and Gomaa, Walid},
journal={arXiv preprint arXiv:2406.18120},
year={2024}
}