Views
No views yet
Using Multimodal Large Language Models for False Alarm Reduction in Image-based Fire Detection


1import torch
2from transformers import AutoModelForCausalLM
3from PIL import Image
4from deepseek_vl.models import VLChatProcessor, MultiModalityCausalLM
5from deepseek_vl.utils.io import load_pil_images
6
7import os
8# specify the path to the model
9model_path = "" # */gaoqie/DeepSeekVL-1.3B-Chat-fire
10vl_chat_processor: VLChatProcessor = VLChatProcessor.from_pretrained(model_path)
11tokenizer = vl_chat_processor.tokenizer
12
13vl_gpt: MultiModalityCausalLM = AutoModelForCausalLM.from_pretrained(model_path, trust_remote_code=True,low_cpu_mem_usage=True,
14 torch_dtype=torch.bfloat16,
15 device_map="auto")
16vl_gpt = vl_gpt.eval()
17
18def infer(img_path):
19
20 # 模式一
21 messages = [
22 {
23 "role": "User",
24 "content": f"<image_placeholder>图像中是否存在火焰?详细分析。",
25 "images": [f"{img_path}"]
26 },
27 {
28 "role": "Assistant",
29 "content": ""
30 }
31 ]
32
33 # 模式二
34 messages = [
35 {
36 "role": "User",
37 "content": f"<image_placeholder>图像中是否存在火焰?简单回答。",
38 "images": [f"{img_path}"]
39 },
40 {
41 "role": "Assistant",
42 "content": ""
43 }
44 ]
45 # 模式三
46 messages = [
47 {
48 "role": "User",
49 "content": f"<image_placeholder>图像中是否存在火焰?快速回答。",
50 "images": [f"{img_path}"]
51 },
52 {
53 "role": "Assistant",
54 "content": ""
55 }
56 ]
57 # load images and prepare for inputs
58 pil_images = load_pil_images(messages)
59
60 prepare_inputs = vl_chat_processor(
61 conversations=messages,
62 images=pil_images,
63 force_batchify=True
64 ).to(vl_gpt.device)
65
66 # run image encoder to get the image embeddings
67 inputs_embeds = vl_gpt.prepare_inputs_embeds(**prepare_inputs)
68
69 # run the model to get the response
70 outputs = vl_gpt.language_model.generate(
71 inputs_embeds=inputs_embeds,
72 attention_mask=prepare_inputs.attention_mask,
73 pad_token_id=tokenizer.eos_token_id,
74 bos_token_id=tokenizer.bos_token_id,
75 eos_token_id=tokenizer.eos_token_id,
76 max_new_tokens=512,
77 do_sample=False,
78 use_cache=True
79 )
80
81 output_text = tokenizer.decode(outputs[0].cpu().tolist(), skip_special_tokens=True)
82 print(output_text)
83
84img_path = ""
85infer(img_path)