Views
No views yet
from transformers import AutoModelForCausalLM, AutoTokenizer
from peft import PeftModel
base_id = "unsloth/mistral-7b-instruct-v0.3-bnb-4bit"
lora_id = "yourusername/mistral-7b-math-lora"
tok = AutoTokenizer.from_pretrained(base_id)
base = AutoModelForCausalLM.from_pretrained(base_id, load_in_4bit=True, device_map="auto")
model = PeftModel.from_pretrained(base, lora_id) # inject adapters
# generate
prompt = "... Alpaca-formatted prompt ..."
out = model.generate(**tok(prompt, return_tensors="pt").to(model.device), max_new_tokens=256)
print(tok.decode(out[0], skip_special_tokens=True))