Views
No views yet
1import sys
2sys.path.append('./')
3from videollama2 import model_init, mm_infer
4from videollama2.utils import disable_torch_init
5
6
7def inference():
8 disable_torch_init()
9
10 # Video Inference
11 modal = 'video'
12 modal_path = 'assets/cat_and_chicken.mp4'
13 instruct = 'What animals are in the video, what are they doing, and how does the video feel?'
14
15 # Image Inference
16 modal = 'image'
17 modal_path = 'assets/sora.png'
18 instruct = 'What is the woman wearing, what is she doing, and how does the image feel?'
19
20 model_path = 'DAMO-NLP-SG/VideoLLaMA2-8x7B'
21 model, processor, tokenizer = model_init(model_path)
22 output = mm_infer(processor[modal](modal_path), instruct, model=model, tokenizer=tokenizer, do_sample=False, modal=modal)
23
24 print(output)
25
26if __name__ == "__main__":
27 inference()1@article{damonlpsg2024videollama2,
2 title={VideoLLaMA 2: Advancing Spatial-Temporal Modeling and Audio Understanding in Video-LLMs},
3 author={Cheng, Zesen and Leng, Sicong and Zhang, Hang and Xin, Yifei and Li, Xin and Chen, Guanzheng and Zhu, Yongxin and Zhang, Wenqi and Luo, Ziyang and Zhao, Deli and Bing, Lidong},
4 journal={arXiv preprint arXiv:2406.07476},
5 year={2024},
6 url = {https://arxiv.org/abs/2406.07476}
7}
8@article{damonlpsg2023videollama,
9 title = {Video-LLaMA: An Instruction-tuned Audio-Visual Language Model for Video Understanding},
10 author = {Zhang, Hang and Li, Xin and Bing, Lidong},
11 journal = {arXiv preprint arXiv:2306.02858},
12 year = {2023},
13 url = {https://arxiv.org/abs/2306.02858}
14}