Views
No views yet
| Model Name | Type | Audio Encoder | Language Decoder |
|---|---|---|---|
| VideoLLaMA2.1-7B-AV (This Checkpoint) | Chat | Fine-tuned BEATs_iter3+(AS2M)(cpt2) | VideoLLaMA2.1-7B-16F |
1import sys
2sys.path.append('./')
3from videollama2 import model_init, mm_infer
4from videollama2.utils import disable_torch_init
5import argparse
6
7def inference(args):
8
9 model_path = args.model_path
10 model, processor, tokenizer = model_init(model_path)
11
12 if args.modal_type == "a":
13 model.model.vision_tower = None
14 elif args.modal_type == "v":
15 model.model.audio_tower = None
16 elif args.modal_type == "av":
17 pass
18 else:
19 raise NotImplementedError
20 # Audio-visual Inference
21 audio_video_path = "assets/00003491.mp4"
22 preprocess = processor['audio' if args.modal_type == "a" else "video"]
23 if args.modal_type == "a":
24 audio_video_tensor = preprocess(audio_video_path)
25 else:
26 audio_video_tensor = preprocess(audio_video_path, va=True if args.modal_type == "av" else False)
27 question = f"Please describe the video with audio information."
28
29 # Audio Inference
30 audio_video_path = "assets/bird-twitter-car.wav"
31 preprocess = processor['audio' if args.modal_type == "a" else "video"]
32 if args.modal_type == "a":
33 audio_video_tensor = preprocess(audio_video_path)
34 else:
35 audio_video_tensor = preprocess(audio_video_path, va=True if args.modal_type == "av" else False)
36 question = f"Please describe the audio."
37
38 # Video Inference
39 audio_video_path = "assets/output_v_1jgsRbGzCls.mp4"
40 preprocess = processor['audio' if args.modal_type == "a" else "video"]
41 if args.modal_type == "a":
42 audio_video_tensor = preprocess(audio_video_path)
43 else:
44 audio_video_tensor = preprocess(audio_video_path, va=True if args.modal_type == "av" else False)
45 question = f"What activity are the people practicing in the video?"
46
47 output = mm_infer(
48 audio_video_tensor,
49 question,
50 model=model,
51 tokenizer=tokenizer,
52 modal='audio' if args.modal_type == "a" else "video",
53 do_sample=False,
54 )
55
56 print(output)
57
58
59if __name__ == "__main__":
60 parser = argparse.ArgumentParser()
61
62 parser.add_argument('--model-path', help='', , required=False, default='DAMO-NLP-SG/VideoLLaMA2.1-7B-AV')
63 parser.add_argument('--modal-type', choices=["a", "v", "av"], help='', required=True)
64 args = parser.parse_args()
65
66 inference(args)
671@article{damonlpsg2024videollama2,
2 title={VideoLLaMA 2: Advancing Spatial-Temporal Modeling and Audio Understanding in Video-LLMs},
3 author={Cheng, Zesen and Leng, Sicong and Zhang, Hang and Xin, Yifei and Li, Xin and Chen, Guanzheng and Zhu, Yongxin and Zhang, Wenqi and Luo, Ziyang and Zhao, Deli and Bing, Lidong},
4 journal={arXiv preprint arXiv:2406.07476},
5 year={2024},
6 url = {https://arxiv.org/abs/2406.07476}
7}
8
9@article{damonlpsg2023videollama,
10 title = {Video-LLaMA: An Instruction-tuned Audio-Visual Language Model for Video Understanding},
11 author = {Zhang, Hang and Li, Xin and Bing, Lidong},
12 journal = {arXiv preprint arXiv:2306.02858},
13 year = {2023},
14 url = {https://arxiv.org/abs/2306.02858}
15}