from transformers import AutoProcessor, AutoModelForVision2Seq
from peft import PeftModel
from PIL import Image
import torch

base_id = "Qwen/Qwen3-VL-2B-Instruct"
adapter_id = "graahand/qwen-vl-2b-lora"

model = AutoModelForVision2Seq.from_pretrained(base_id, device_map="auto")
model = PeftModel.from_pretrained(model, adapter_id)
processor = AutoProcessor.from_pretrained(adapter_id, trust_remote_code=True)

image = Image.open("your_image.png").convert("RGB")
messages = [
	{"role": "system", "content": "You are a helpful assistant."},
	{
		"role": "user",
		"content": [
			{"type": "image"},
			{"type": "text", "text": "Extract all key-value pairs from this document."},
		],
	},
]

prompt = processor.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
inputs = processor(text=[prompt], images=[image], return_tensors="pt").to(model.device)

with torch.no_grad():
	out = model.generate(**inputs, max_new_tokens=128)

print(processor.tokenizer.decode(out[0], skip_special_tokens=True))

Training Details

Training Data

Custom OCR dataset with lines formatted as:

image_path<TAB>key: value

Training Procedure

Preprocessing

RGB conversion
Resize to 448x448

Training Hyperparameters

Epochs: 3
Train batch size: 4
Eval batch size: 4
Warmup steps: 500
Weight decay: 0.01
LoRA r: 8
LoRA alpha: 16
Target modules: q_proj, k_proj, v_proj, o_proj, gate_proj, up_proj, down_proj
Precision: 4-bit NF4 quantization

Evaluation

No formal benchmark metrics reported yet.

Technical Specifications

Model Architecture and Objective

LoRA adapter for Qwen/Qwen3-VL-2B-Instruct for image-text-to-text generation.

Software

transformers
peft
torch

Model Card Contact

[More Information Needed]

Framework versions

PEFT 0.18.1