import torch
from peft import PeftModel, PeftConfig
from transformers import AutoModelForCausalLM, AutoTokenizer
peft_model_id = "genkida/genkidakun_202407"
config = PeftConfig.from_pretrained(peft_model_id)
ベースモデルの準備
base_model = AutoModelForCausalLM.from_pretrained(
config.base_model_name_or_path,
load_in_4bit=True,
device_map="auto")
tokenizer = AutoTokenizer.from_pretrained(config.base_model_name_or_path)
LoRAモデルのロード
model = PeftModel.from_pretrained(
base_model,
peft_model_id,
state_dict=base_model.state_dict(),
device_map="auto",
torch_dtype=torch.float16)
model.eval()
