Views
No views yet
1import torch
2from transformers import AutoTokenizer, AutoModelForCausalLM
3from peft import PeftModel
4
5base_model_id = "Qwen/Qwen2.5-0.5B-Instruct"
6adapter_id = "pelinbalci/qwen2_5_0_5b-capybara-lora"
7
8tokenizer = AutoTokenizer.from_pretrained(adapter_id)
9base = AutoModelForCausalLM.from_pretrained(
10 base_model_id,
11 torch_dtype=torch.float16,
12 device_map="auto",
13)
14
15model = PeftModel.from_pretrained(base, adapter_id)
16model.eval()
17
18prompt = "Explain LoRA fine-tuning in simple terms."
19inputs = tokenizer(prompt, return_tensors="pt").to(model.device)
20
21with torch.no_grad():
22 output = model.generate(**inputs, max_new_tokens=128)
23
24print(tokenizer.decode(output[0], skip_special_tokens=True))1@misc{vonwerra2022trl,
2 title = {{TRL: Transformer Reinforcement Learning}},
3 author = {Leandro von Werra and Younes Belkada and Lewis Tunstall and Edward Beeching and Tristan Thrush and Nathan Lambert and Shengyi Huang and Kashif Rasul and Quentin Gallou{\'e}dec},
4 year = 2020,
5 journal = {GitHub repository},
6 publisher = {GitHub},
7 howpublished = {\url{https://github.com/huggingface/trl}}
8}