You can deploy this model and start earning money today!
Rfy23/
qwen2vl-ko-zh

Name: qwen2vl-ko-zh
Rating: 2.5 (8 reviews)
safetensors
qwen2_vl
Rfy23/synth_fake_prescriptions
Qwen/Qwen2-VL-2B-Instruct
finetune
Views
No views yet
python
""" inference finetune model """

import torch
from transformers import Qwen2VLForConditionalGeneration, AutoProcessor
from qwen_vl_utils import process_vision_info

# ------------------------------
# 1. 모델 & 프로세서 로드
# ------------------------------
print("모델 로드 중...")
model_name = "Rfy23/qwen2vl-ko-zh"  # 내 병합 모델
device = "cuda" if torch.cuda.is_available() else "cpu"

model = Qwen2VLForConditionalGeneration.from_pretrained(
    model_name,
    torch_dtype=torch.float16 if device=="cuda" else torch.float32,
    device_map="auto" if device=="cuda" else None
)
model.eval()
print("모델 로드 완료!")

processor = AutoProcessor.from_pretrained(model_name)
print("프로세서 로드 완료!")

# ------------------------------
# 2. 고정 질문 + 이미지 URL
# ------------------------------
image_url = "/home/jwlee/volume/Qwen2-vl-finetune-wo/scripts/test-out-2/images/00002.jpg"
fixed_question = "这张处方上写了什么？ 尤其是药品、服用次数等，请准确全部告诉我。"  # 원하는 고정 질문

messages = [
    {
        "role": "user",
        "content": [
            {"type": "image", "image": image_url},
            {"type": "text", "text": f"<image>\n{fixed_question}"}
        ],
    }
]

# ------------------------------
# 3. processor로 입력 준비
# ------------------------------
print("입력 텐서 준비 중...")
text_input = processor.apply_chat_template(
    messages, tokenize=False, add_generation_prompt=True
)
image_inputs, _ = process_vision_info(messages)

inputs = processor(
    text=[text_input],
    images=image_inputs,
    # videos=video_inputs,
    padding=True,
    return_tensors="pt"
).to(device)
print("입력 텐서 준비 완료!")

# ------------------------------
# 4. 추론
# ------------------------------
print("모델 추론 시작...")
with torch.no_grad():
    generated_ids = model.generate(**inputs, max_new_tokens=128)

generated_ids_trimmed = [
    out_ids[len(in_ids):] for in_ids, out_ids in zip(inputs.input_ids, generated_ids)
]

print("디코딩 중...")
output_text = processor.batch_decode(
    generated_ids_trimmed, skip_special_tokens=True, clean_up_tokenization_spaces=False
)

print("모델 출력:")
print(output_text[0])

""" inference base model """

# from transformers import Qwen2VLForConditionalGeneration, AutoTokenizer, AutoProcessor
# from qwen_vl_utils import process_vision_info

# # default: Load the model on the available device(s)
# model = Qwen2VLForConditionalGeneration.from_pretrained(
#     "Qwen/Qwen2-VL-2B-Instruct", torch_dtype="auto", device_map="auto"
# )

# # We recommend enabling flash_attention_2 for better acceleration and memory saving, especially in multi-image and video scenarios.
# # model = Qwen2VLForConditionalGeneration.from_pretrained(
# #     "Qwen/Qwen2-VL-2B-Instruct",
# #     torch_dtype=torch.bfloat16,
# #     attn_implementation="flash_attention_2",
# #     device_map="auto",
# # )

# # default processer
# processor = AutoProcessor.from_pretrained("Qwen/Qwen2-VL-2B-Instruct")

# # The default range for the number of visual tokens per image in the model is 4-16384. You can set min_pixels and max_pixels according to your needs, such as a token count range of 256-1280, to balance speed and memory usage.
# # min_pixels = 256*28*28
# # max_pixels = 1280*28*28
# # processor = AutoProcessor.from_pretrained("Qwen/Qwen2-VL-2B-Instruct", min_pixels=min_pixels, max_pixels=max_pixels)

# messages = [
#     {
#         "role": "user",
#         "content": [
#             {
#                 "type": "image",
#                 "image": "/home/jwlee/volume/Qwen2-vl-finetune-wo/scripts/test-out/images/00003.jpg",
#             },
#             {"type": "text", "text": "这张处方上写了什么？"},
#         ],
#     }
# ]

# # Preparation for inference
# text = processor.apply_chat_template(
#     messages, tokenize=False, add_generation_prompt=True
# )
# image_inputs, _ = process_vision_info(messages)
# inputs = processor(
#     text=[text],
#     images=image_inputs,
#     # videos=video_inputs,
#     padding=True,
#     return_tensors="pt",
# )
# inputs = inputs.to("cuda")

# # Inference: Generation of the output
# generated_ids = model.generate(**inputs, max_new_tokens=128)
# generated_ids_trimmed = [
#     out_ids[len(in_ids) :] for in_ids, out_ids in zip(inputs.input_ids, generated_ids)
# ]
# output_text = processor.batch_decode(
#     generated_ids_trimmed, skip_special_tokens=True, clean_up_tokenization_spaces=False
# )
# print(output_text)