Views
No views yet
Using Multimodal Large Language Models for False Alarm Reduction in Image-based Fire Detection


1from transformers import Qwen2VLForConditionalGeneration, AutoTokenizer, AutoProcessor
2from qwen_vl_utils import process_vision_info
3
4
5model_dir = "" # */gaoqie/Qwen2VL-2B-Instruct-fire
6device = "cuda:0"
7# default: Load the model on the available device(s)
8model = Qwen2VLForConditionalGeneration.from_pretrained(
9 model_dir, torch_dtype="bfloat16", device_map=device
10)
11
12
13
14# default processer
15processor = AutoProcessor.from_pretrained(model_dir)
16
17# The default range for the number of visual tokens per image in the model is 4-16384. You can set min_pixels and max_pixels according to your needs, such as a token count range of 256-1280, to balance speed and memory usage.
18# min_pixels = 256*28*28
19# max_pixels = 1280*28*28
20# processor = AutoProcessor.from_pretrained(model_dir, min_pixels=min_pixels, max_pixels=max_pixels)
21
22
23
24def infer(img_path):
25 # 模式一
26 messages = [
27 {
28 "role": "user",
29 "content": [
30 {
31 "type": "image",
32 "image": img_path,
33 },
34 {
35 "type": "text",
36 "text": "图像中是否存在火焰?详细分析。"
37 }
38 ],
39 }
40 ]
41 # 模式二
42 messages = [
43 {
44 "role": "user",
45 "content": [
46 {
47 "type": "image",
48 "image": img_path,
49 },
50 {
51 "type": "text",
52 "text": "图像中是否存在火焰?简单回答。"
53 }
54 ],
55 }
56 ]
57 # 模式三
58 messages = [
59 {
60 "role": "user",
61 "content": [
62 {
63 "type": "image",
64 "image": img_path,
65 },
66 {
67 "type": "text",
68 "text": "图像中是否存在火焰?快速回答。"
69 }
70 ],
71 }
72 ]
73
74 # Preparation for inference
75 text = processor.apply_chat_template(
76 messages, tokenize=False, add_generation_prompt=True
77 )
78
79 image_inputs, video_inputs = process_vision_info(messages)
80
81 inputs = processor(
82 text=[text],
83 images=image_inputs,
84 videos=video_inputs,
85 padding=True,
86 return_tensors="pt",
87 )
88 inputs = inputs.to(device)
89
90 # Inference: Generation of the output
91 generated_ids = model.generate(**inputs, max_new_tokens=500)
92 # print(processor.batch_decode(
93 # generated_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False
94 # ))
95 generated_ids_trimmed = [
96 out_ids[len(in_ids) :] for in_ids, out_ids in zip(inputs.input_ids, generated_ids)
97 ]
98 output_text = processor.batch_decode(
99 generated_ids_trimmed, skip_special_tokens=True, clean_up_tokenization_spaces=False
100 )
101
102 output_text = output_text[0]
103 # print(output_text)
104
105image_path = ""
106infer(image_path)
107