Views
No views yet
Using Multimodal Large Language Models for False Alarm Reduction in Image-based Fire Detection


1import torch
2from transformers import AutoModelForCausalLM
3from PIL import Image
4from deepseek_vl.models import VLChatProcessor, MultiModalityCausalLM
5from deepseek_vl.utils.io import load_pil_images
6
7import os
8# specify the path to the model
9model_path = "" # */gaoqie/DeepSeekVL-7B-Chat-fire
10vl_chat_processor: VLChatProcessor = VLChatProcessor.from_pretrained(model_path)
11tokenizer = vl_chat_processor.tokenizer
12
13vl_gpt: MultiModalityCausalLM = AutoModelForCausalLM.from_pretrained(model_path, trust_remote_code=True,low_cpu_mem_usage=True,
14 torch_dtype=torch.bfloat16,
15 device_map="auto")
16vl_gpt = vl_gpt.eval()
17
18
19def infer(img_path):
20 # 模式1
21 messages = [
22 {
23 "role": "User",
24 "content": f"<image_placeholder>图像中是否存在火焰?详细分析。",
25 "images": [f"{img_path}"]
26 },
27 {
28 "role": "Assistant",
29 "content": ""
30 }
31 ]
32
33 # 模式2
34 messages = [
35 {
36 "role": "User",
37 "content": f"<image_placeholder>图像中是否存在火焰?简单回答。",
38 "images": [f"{img_path}"]
39 },
40 {
41 "role": "Assistant",
42 "content": ""
43 }
44 ]
45
46 # 模式3
47 messages = [
48 {
49 "role": "User",
50 "content": f"<image_placeholder>图像中是否存在火焰?快速回答。",
51 "images": [f"{img_path}"]
52 },
53 {
54 "role": "Assistant",
55 "content": ""
56 }
57 ]
58
59 # load images and prepare for inputs
60 pil_images = load_pil_images(messages)
61
62 prepare_inputs = vl_chat_processor(
63 conversations=messages,
64 images=pil_images,
65 force_batchify=True
66 ).to(vl_gpt.device)
67
68 # run image encoder to get the image embeddings
69 inputs_embeds = vl_gpt.prepare_inputs_embeds(**prepare_inputs)
70
71 # run the model to get the response
72 outputs = vl_gpt.language_model.generate(
73 inputs_embeds=inputs_embeds,
74 attention_mask=prepare_inputs.attention_mask,
75 pad_token_id=tokenizer.eos_token_id,
76 bos_token_id=tokenizer.bos_token_id,
77 eos_token_id=tokenizer.eos_token_id,
78 max_new_tokens=512,
79 do_sample=False,
80 use_cache=True
81 )
82
83 output_text = tokenizer.decode(outputs[0].cpu().tolist(), skip_special_tokens=True)
84
85 print(output_text)
86
87image_path = ""
88infer(image_path)