Views
No views yet
1git clone https://github.com/LLaVA-VL/LLaVA-NeXT.git
2pip install LLaVA-NeXT1import torch
2import numpy as np
3from llava.model.builder import load_pretrained_model
4from llava.mm_utils import process_anyres_image, tokenizer_image_token, get_model_name_from_path, KeywordsStoppingCriteria
5from llava.constants import IMAGE_TOKEN_INDEX, DEFAULT_IMAGE_TOKEN, DEFAULT_IM_START_TOKEN, DEFAULT_IM_END_TOKEN
6from llava.conversation import conv_templates, SeparatorStyle
7from transformers import AutoConfig
8from decord import VideoReader, cpu
9
10def load_video(video_path, num_frames=32, force_sample=False):
11 vr = VideoReader(video_path, ctx=cpu(0), num_threads=1)
12 total_frame_num = len(vr)
13 fps = round(vr.get_avg_fps())
14 frame_idx = [i for i in range(0, len(vr), fps)]
15 if len(frame_idx) > num_frames or force_sample:
16 uniform_sampled_frames = np.linspace(0, total_frame_num - 1, num_frames, dtype=int)
17 frame_idx = uniform_sampled_frames.tolist()
18 frames = vr.get_batch(frame_idx).asnumpy()
19 return frames
20
21def infer(
22 model_path,
23 video_path,
24 prompt,
25 model_base=None,
26 conv_mode=None,
27 num_frames=32,
28 force_sample=False,
29 load_8bit=False,
30 device="cuda"
31):
32 model_name = get_model_name_from_path(model_path)+"llava_qwen" # For llava internal checks and proper loading
33 tokenizer, model, image_processor, context_len = load_pretrained_model(
34 model_path, model_base, model_name, load_8bit=load_8bit
35 )
36 frames = load_video(video_path, num_frames=num_frames, force_sample=force_sample)
37 video = image_processor.preprocess(frames, return_tensors="pt")["pixel_values"].half().to(device)
38 video = [video]
39
40 qs = DEFAULT_IMAGE_TOKEN + "\n" + prompt
41 conv = conv_templates[conv_mode].copy() if conv_mode else conv_templates["default"].copy()
42 conv.append_message(conv.roles[0], qs)
43 conv.append_message(conv.roles[1], None)
44 prompt_str = conv.get_prompt()
45
46 input_ids = tokenizer_image_token(prompt_str, tokenizer, IMAGE_TOKEN_INDEX, return_tensors="pt").unsqueeze(0).to(device)
47 if tokenizer.pad_token_id is None:
48 tokenizer.pad_token_id = tokenizer.eos_token_id
49
50 attention_masks = input_ids.ne(tokenizer.pad_token_id).long().to(device)
51 stop_str = conv.sep if conv.sep_style != SeparatorStyle.TWO else conv.sep2
52 keywords = [stop_str]
53 stopping_criteria = KeywordsStoppingCriteria(keywords, tokenizer, input_ids)
54
55 with torch.inference_mode():
56 output_ids = model.generate(
57 inputs=input_ids,
58 images=video,
59 attention_mask=attention_masks,
60 modalities="video",
61 do_sample=False,
62 temperature=0.0,
63 max_new_tokens=1024,
64 top_p=0.1,
65 num_beams=1,
66 use_cache=True,
67 stopping_criteria=[stopping_criteria]
68 )
69 outputs = tokenizer.batch_decode(output_ids, skip_special_tokens=True)[0].strip()
70 if outputs.endswith(stop_str):
71 outputs = outputs[:-len(stop_str)]
72 return outputs.strip()
73
74if __name__ == "__main__":
75 model_path = "MBZUAI/ViMUL"
76 video_path = "LLaVA-NeXT/playground/demo/xU25MMA2N4aVtYay.mp4"
77 prompt = "Describe what happens in the video."
78 conv_mode = "qwen_1_5"
79 output = infer(model_path, video_path, prompt, conv_mode=conv_mode)
80 print("\n")
81 print("="*40)
82 print("Output:", output)
83 print("="*40)@misc{shafique2025culturallydiversemultilingualmultimodalvideo,
title={A Culturally-diverse Multilingual Multimodal Video Benchmark & Model},
author={Bhuiyan Sanjid Shafique and Ashmal Vayani and Muhammad Maaz and Hanoona Abdul Rasheed and Dinura Dissanayake and Mohammed Irfan Kurpath and Yahya Hmaiti and Go Inoue and Jean Lahoud and Md. Safirur Rashid and Shadid Intisar Quasem and Maheen Fatima and Franco Vidal and Mykola Maslych and Ketan Pravin More and Sanoojan Baliah and Hasindri Watawana and Yuhao Li and Fabian Farestam and Leon Schaller and Roman Tymtsiv and Simon Weber and Hisham Cholakkal and Ivan Laptev and Shin'ichi Satoh and Michael Felsberg and Mubarak Shah and Salman Khan and Fahad Shahbaz Khan},
year={2025},
eprint={2506.07032},
archivePrefix={arXiv},
primaryClass={cs.CL},
url={https://arxiv.org/abs/2506.07032},
}