from transformers import pipeline

question = "If you had a time machine, but could only go to the past or the future once and never return, which would you choose and why?"
generator = pipeline("text-generation", model="None", device="cuda")
output = generator([{"role": "user", "content": question}], max_new_tokens=128, return_full_text=False)[0]
print(output["generated_text"])

Training procedure

This model was trained with SFT.

Framework versions

PEFT 0.17.1
TRL: 0.23.0
Transformers: 4.56.1
Pytorch: 2.8.0
Datasets: 4.0.0
Tokenizers: 0.22.0

Citations

Cite TRL as:

bibtex

@misc{vonwerra2022trl,
	title        = {{TRL: Transformer Reinforcement Learning}},
	author       = {Leandro von Werra and Younes Belkada and Lewis Tunstall and Edward Beeching and Tristan Thrush and Nathan Lambert and Shengyi Huang and Kashif Rasul and Quentin Gallou{\'e}dec},
	year         = 2020,
	journal      = {GitHub repository},
	publisher    = {GitHub},
	howpublished = {\url{https://github.com/huggingface/trl}}
}

Usage

colab

!pip -q install "transformers>=4.44" "peft>=0.11" accelerate

import torch
from peft import PeftModel
from transformers import AutoModelForCausalLM, AutoTokenizer

BASE_ID   = "Qwen/Qwen2.5-0.5B"
ADAPTER_ID = "BeagleWorks/Qwen-Mail-Lora"   # ←あなたのLoRA

# 1) トークナイザ
tok = AutoTokenizer.from_pretrained(BASE_ID, trust_remote_code=True)
if tok.pad_token is None:
    tok.pad_token = tok.eos_token  # pad未設定エラー回避

# 2) ベースモデル（FP16, 自動デバイス割当）
base = AutoModelForCausalLM.from_pretrained(
    BASE_ID,
    trust_remote_code=True,
    device_map="auto",
    torch_dtype=torch.float16 if torch.cuda.is_available() else torch.float32,
)

# 3) LoRAアダプタを適用
model = PeftModel.from_pretrained(base, ADAPTER_ID)
model.eval()

# 4) 生成（学習時のフォーマットに近いプロンプトを使う）
prompt = """[指示]
あなたはメール文面を整えるアシスタントです。以下の下書きを、件名/本文/TODO/署名に整理し、敬体（です・ます調）で自然な日本語に直してください。

[下書き]
明日の打ち合わせ、議題 進捗確認と次タスク。先方に資料送るの忘れた。山田さんにCC。

[出力フォーマット]
件名: <短い件名>
本文:
<整えた本文>
TODO:
- <TODO1>
- <TODO2>
署名:
<署名>

[回答]
"""

inputs = tok(prompt, return_tensors="pt").to(model.device)
with torch.inference_mode():
    out = model.generate(
        **inputs,
        max_new_tokens=400,
        do_sample=True,
        temperature=0.7,
        top_p=0.9,
        repetition_penalty=1.05,
        eos_token_id=tok.eos_token_id,
        pad_token_id=tok.pad_token_id,
    )

print(tok.decode(out[0], skip_special_tokens=True))