Views
No views yet
peft, transformers, 'accelerate', 'bitsandbytes' and pytorch first.pip install peft accelerate bitsandbytes transformers torchhuggingface-cli login --token <YOUR_HF_TOKEN>1from peft import PeftConfig, PeftModel
2from transformers import AutoModelForCausalLM, AutoTokenizer
3import torch
4
5peft_model_id = "ahmedheakl/arazn-llama3-english"
6peft_config = PeftConfig.from_pretrained(peft_model_id)
7base_model_name = peft_config.base_model_name_or_path
8base_model = AutoModelForCausalLM.from_pretrained(base_model_name, device_map="auto", torch_dtype=torch.bfloat16)
9model = PeftModel.from_pretrained(base_model, peft_model_id, device_map="auto")
10tokenizer = AutoTokenizer.from_pretrained(peft_model_id)1import torch
2
3raw_prompt = """<|begin_of_text|><|start_header_id|>system<|end_header_id|>
4
5Translate the following code-switched Arabic-English-mixed text to English only.<|eot_id|><|start_header_id|>user<|end_header_id|>
6
7{source}<|eot_id|><|start_header_id|>assistant<|end_header_id|>
8
9"""
10def inference(prompt) -> str:
11 prompt = raw_prompt.format(source=prompt)
12 inputs = tokenizer(prompt, return_tensors="pt").to("cuda")
13 generated_ids = model.generate(
14 **inputs,
15 use_cache=True,
16 num_return_sequences=1,
17 max_new_tokens=100,
18 # do_sample=True,
19 num_beams=1,
20 # temperature=0.7,
21 eos_token_id=tokenizer.eos_token_id,
22 pad_token_id=tokenizer.pad_token_id,
23 )
24 outputs = tokenizer.batch_decode(generated_ids)[0]
25 torch.cuda.empty_cache()
26 torch.cuda.synchronize()
27 return outputs.split("assistant<|end_header_id|>\n\n")[-1].split("<|eot_id|>")[0]
28print(inference("أنا أحب الbanana")) # I love bananas@article{heakl2024arzen,
title={ArzEn-LLM: Code-Switched Egyptian Arabic-English Translation and Speech Recognition Using LLMs},
author={Heakl, Ahmed and Zaghloul, Youssef and Ali, Mennatullah and Hossam, Rania and Gomaa, Walid},
journal={arXiv preprint arXiv:2406.18120},
year={2024}
}