Views
No views yet
1from transformers import Qwen2VLForConditionalGeneration, AutoProcessor
2
3MODEL_PATH = "..."
4
5model = Qwen2VLForConditionalGeneration.from_pretrained(
6 MODEL_PATH,
7 torch_dtype=torch.bfloat16,
8 attn_implementation="flash_attention_2",
9 device_map="auto",
10)
11
12processor = AutoProcessor.from_pretrained(MODEL_PATH)1[
2 {
3 "image": "Images/Chest CT Scan/test/adenocarcinoma_left.lower.lobe_T2_N0_M0_Ib/000139 (9).png",
4 "problem": "What imaging technique is employed for obtaining this image? A)Mammogram, B)Positron emission tomography (PET), C)CT, D)Fluoroscopy",
5 "solution": "<answer> C </answer>"
6 },
7 {
8 "image": "Images/Chest CT Scan/test/squamous.cell.carcinoma_left.hilum_T1_N2_M0_IIIa/000127 (2).png",
9 "problem": "What imaging technique was utilized for obtaining this image? A)CT, B)Angiography, C)X-ray, D)Ultrasound",
10 "solution": "<answer> A </answer>"
11 },
12 {
13 "image": "Images/Chest CT Scan/test/normal/10 (2).png",
14 "problem": "What imaging technique was used for this image acquisition? A)CT, B)Ultrasound, C)Fluoroscopy, D)X-ray",
15 "solution": "<answer> A </answer>"
16 },
17 {
18 "image": "Images/Chest CT Scan/test/adenocarcinoma_left.lower.lobe_T2_N0_M0_Ib/000142.png",
19 "problem": "What is the specific diagnosis of the cancer shown in the image? A)Neuroendocrine tumor of the left upper lobe, T3 N0 M1, Stage III, B)Mesothelioma of the left lower lobe, T2 N0 M0, Stage Ib, C)Adenocarcinoma of the left lower lobe, T2 N0 M0, Stage Ib, D)Non-Hodgkin lymphoma of the right lower lobe, T2 N1 M0, Stage II",
20 "solution": "<answer> C </answer>"
21 }
22 ...
23]1from qwen_vl_utils import process_vision_info
2
3with open(PROMPT_PATH, "r", encoding="utf-8") as f:
4 data = json.load(f)
5
6QUESTION_TEMPLATE = "{Question} First output the thinking process in <think> </think> and final choice (A, B, C, D ...) in <answer> </answer> tags."
7
8messages = []
9
10for i in data:
11 message = [{
12 "role": "user",
13 "content": [
14 {
15 "type": "image",
16 "image": f"file://{i['image']}"
17 },
18 {
19 "type": "text",
20 "text": QUESTION_TEMPLATE.format(Question=i['problem'])
21 }
22 ]
23 }]
24 messages.append(message)
25
26
27for i in tqdm(range(0, len(messages), BSZ)):
28 batch_messages = messages[i:i + BSZ]
29
30 # Preparation for inference
31 text = [processor.apply_chat_template(msg, tokenize=False, add_generation_prompt=True) for msg in batch_messages]
32
33 image_inputs, video_inputs = process_vision_info(batch_messages)
34 inputs = processor(
35 text=text,
36 images=image_inputs,
37 videos=video_inputs,
38 padding=True,
39 return_tensors="pt",
40 )
41 inputs = inputs.to("cuda")
42
43 # Inference: Generation of the output
44 generated_ids = model.generate(**inputs, use_cache=True, max_new_tokens=256, do_sample=False)
45
46 generated_ids_trimmed = [
47 out_ids[len(in_ids):] for in_ids, out_ids in zip(inputs.input_ids, generated_ids)
48 ]
49 batch_output_text = processor.batch_decode(
50 generated_ids_trimmed, skip_special_tokens=True, clean_up_tokenization_spaces=False
51 )
52
53 all_outputs.extend(batch_output_text)
54 print(f"Processed batch {i//BSZ + 1}/{(len(messages) + BSZ - 1)//BSZ}")
55@article{lai2025med,
title={Med-R1: Reinforcement Learning for Generalizable Medical Reasoning in Vision-Language Models},
author={Lai, Yuxiang and Zhong, Jike and Li, Ming and Zhao, Shitian and Yang, Xiaofeng},
journal={arXiv preprint arXiv:2503.13939},
year={2025}
}