from peft import PeftModel
from transformers import AutoProcessor

try:
    from transformers import AutoModelForMultimodalLM as _AutoModelForMM
except ImportError:
    from transformers import AutoModelForImageTextToText as _AutoModelForMM  # pragma: no cover
import torch

BASE = "google/gemma-4-E2B-it"
ADAPTER = "mufasabrownie/gemma-4-E2B-it-ham10000-lora"

processor = AutoProcessor.from_pretrained(BASE)
model = AutoModelForMultimodalLM.from_pretrained(BASE, torch_dtype=torch.bfloat16)  # or the auto class matching your transformers version
model = PeftModel.from_pretrained(model, ADAPTER)
model.eval()

messages = [{
    "role": "user",
    "content": [
        {"type": "image", "image": "<PIL.Image or URL>"},
        {"type": "text", "text": "Classify this dermoscopic image into one of: "
                                 "nv, mel, bkl, akiec, bcc, df, vasc. Answer with the label only."},
    ],
}]
prompt = processor.apply_chat_template(messages, add_generation_prompt=True, tokenize=False)
inputs = processor(text=prompt, images=your_image, return_tensors="pt").to(model.device)
out = model.generate(**inputs, max_new_tokens=8, do_sample=False)
print(processor.tokenizer.decode(out[0, inputs["input_ids"].shape[1]:], skip_special_tokens=True))

Training

Method: LoRA (r=16, alpha=32, dropout=0.05) on attention + MLP projections
Precision: bf16
Optimizer: AdamW, cosine schedule, lr=2e-4, 10% warmup
Epochs: 3
Class imbalance: weighted random sampler + class-weighted cross-entropy