Views
No views yet
python -m videollava.serve.gradio_web_serverpython -m videollava.serve.cli --model-path "LanguageBind/Video-LLaVA-7B" --file "path/to/your/video.mp4" --load-4bitpython -m videollava.serve.cli --model-path "LanguageBind/Video-LLaVA-7B" --file "path/to/your/image.jpg" --load-4bit1git clone https://github.com/PKU-YuanGroup/Video-LLaVA
2cd Video-LLaVA
3conda create -n videollava python=3.10 -y
4conda activate videollava
5pip install --upgrade pip # enable PEP 660 support
6pip install -e .
7pip install -e ".[train]"
8pip install flash-attn --no-build-isolation
9pip install decord opencv-python git+https://github.com/facebookresearch/pytorchvideo.git@28fe037d212663c6a24f373b94cc5d478c8c1a1dLanguageBind/Video-LLaVA-7B) on local, you can use the following code snippets.1import torch
2from videollava.constants import IMAGE_TOKEN_INDEX, DEFAULT_IMAGE_TOKEN
3from videollava.conversation import conv_templates, SeparatorStyle
4from videollava.model.builder import load_pretrained_model
5from videollava.utils import disable_torch_init
6from videollava.mm_utils import tokenizer_image_token, get_model_name_from_path, KeywordsStoppingCriteria
7
8def main():
9 disable_torch_init()
10 image = 'videollava/serve/examples/extreme_ironing.jpg'
11 inp = 'What is unusual about this image?'
12 model_path = 'LanguageBind/Video-LLaVA-7B'
13 cache_dir = 'cache_dir'
14 device = 'cuda'
15 load_4bit, load_8bit = True, False
16 model_name = get_model_name_from_path(model_path)
17 tokenizer, model, processor, _ = load_pretrained_model(model_path, None, model_name, load_8bit, load_4bit, device=device, cache_dir=cache_dir)
18 image_processor = processor['image']
19 conv_mode = "llava_v1"
20 conv = conv_templates[conv_mode].copy()
21 roles = conv.roles
22
23 image_tensor = image_processor.preprocess(image, return_tensors='pt')['pixel_values']
24 if type(image_tensor) is list:
25 tensor = [image.to(model.device, dtype=torch.float16) for image in image_tensor]
26 else:
27 tensor = image_tensor.to(model.device, dtype=torch.float16)
28
29 print(f"{roles[1]}: {inp}")
30 inp = DEFAULT_IMAGE_TOKEN + '\n' + inp
31 conv.append_message(conv.roles[0], inp)
32 conv.append_message(conv.roles[1], None)
33 prompt = conv.get_prompt()
34 input_ids = tokenizer_image_token(prompt, tokenizer, IMAGE_TOKEN_INDEX, return_tensors='pt').unsqueeze(0).cuda()
35 stop_str = conv.sep if conv.sep_style != SeparatorStyle.TWO else conv.sep2
36 keywords = [stop_str]
37 stopping_criteria = KeywordsStoppingCriteria(keywords, tokenizer, input_ids)
38
39 with torch.inference_mode():
40 output_ids = model.generate(
41 input_ids,
42 images=tensor,
43 do_sample=True,
44 temperature=0.2,
45 max_new_tokens=1024,
46 use_cache=True,
47 stopping_criteria=[stopping_criteria])
48
49 outputs = tokenizer.decode(output_ids[0, input_ids.shape[1]:]).strip()
50 print(outputs)
51
52if __name__ == '__main__':
53 main()1import torch
2from videollava.constants import IMAGE_TOKEN_INDEX, DEFAULT_IMAGE_TOKEN
3from videollava.conversation import conv_templates, SeparatorStyle
4from videollava.model.builder import load_pretrained_model
5from videollava.utils import disable_torch_init
6from videollava.mm_utils import tokenizer_image_token, get_model_name_from_path, KeywordsStoppingCriteria
7
8def main():
9 disable_torch_init()
10 video = 'videollava/serve/examples/sample_demo_1.mp4'
11 inp = 'Why is this video funny?'
12 model_path = 'LanguageBind/Video-LLaVA-7B'
13 cache_dir = 'cache_dir'
14 device = 'cuda'
15 load_4bit, load_8bit = True, False
16 model_name = get_model_name_from_path(model_path)
17 tokenizer, model, processor, _ = load_pretrained_model(model_path, None, model_name, load_8bit, load_4bit, device=device, cache_dir=cache_dir)
18 video_processor = processor['video']
19 conv_mode = "llava_v1"
20 conv = conv_templates[conv_mode].copy()
21 roles = conv.roles
22
23 video_tensor = video_processor(video, return_tensors='pt')['pixel_values']
24 if type(video_tensor) is list:
25 tensor = [video.to(model.device, dtype=torch.float16) for video in video_tensor]
26 else:
27 tensor = video_tensor.to(model.device, dtype=torch.float16)
28
29 print(f"{roles[1]}: {inp}")
30 inp = ' '.join([DEFAULT_IMAGE_TOKEN] * model.get_video_tower().config.num_frames) + '\n' + inp
31 conv.append_message(conv.roles[0], inp)
32 conv.append_message(conv.roles[1], None)
33 prompt = conv.get_prompt()
34 input_ids = tokenizer_image_token(prompt, tokenizer, IMAGE_TOKEN_INDEX, return_tensors='pt').unsqueeze(0).cuda()
35 stop_str = conv.sep if conv.sep_style != SeparatorStyle.TWO else conv.sep2
36 keywords = [stop_str]
37 stopping_criteria = KeywordsStoppingCriteria(keywords, tokenizer, input_ids)
38
39 with torch.inference_mode():
40 output_ids = model.generate(
41 input_ids,
42 images=tensor,
43 do_sample=True,
44 temperature=0.1,
45 max_new_tokens=1024,
46 use_cache=True,
47 stopping_criteria=[stopping_criteria])
48
49 outputs = tokenizer.decode(output_ids[0, input_ids.shape[1]:]).strip()
50 print(outputs)
51
52if __name__ == '__main__':
53 main()1@article{lin2023video,
2 title={Video-LLaVA: Learning United Visual Representation by Alignment Before Projection},
3 author={Lin, Bin and Zhu, Bin and Ye, Yang and Ning, Munan and Jin, Peng and Yuan, Li},
4 journal={arXiv preprint arXiv:2311.10122},
5 year={2023}
6}1@article{zhu2023languagebind,
2 title={LanguageBind: Extending Video-Language Pretraining to N-modality by Language-based Semantic Alignment},
3 author={Zhu, Bin and Lin, Bin and Ning, Munan and Yan, Yang and Cui, Jiaxi and Wang, HongFa and Pang, Yatian and Jiang, Wenhao and Zhang, Junwu and Li, Zongwei and others},
4 journal={arXiv preprint arXiv:2310.01852},
5 year={2023}
6}