from transformers import AutoTokenizer, AutoModelForCausalLM
from peft import PeftModel
import torch

# Load models
base_model = AutoModelForCausalLM.from_pretrained(
    "Qwen/qwen3-8b",
    torch_dtype=torch.float16,
    device_map="auto"
)
model = PeftModel.from_pretrained(base_model, "CromonHarry/qwen3-32b-hanna")
tokenizer = AutoTokenizer.from_pretrained("CromonHarry/qwen3-32b-hanna")

# Evaluate a story
prompt = '''Below is an instruction that describes a task, paired with an input that provides further context. Write a response that appropriately completes the request.

### Instruction:
Evaluate the creativity of the following story. You need to output the creativity score with following format:
Creativity score: [A float score from 1 to 5]

### Input:
Story summary: {your_story_here}

### Response:
'''

inputs = tokenizer(prompt, return_tensors="pt").to(model.device)
outputs = model.generate(**inputs, max_new_tokens=256, temperature=0.7)
print(tokenizer.decode(outputs[0], skip_special_tokens=True))

Training Configuration

Learning Rate: 2e-4 (cosine scheduler)
Batch Size: 4 (effective: 32)
Optimizer: AdamW
Gradient Checkpointing: Enabled