Views
No views yet
1from PIL import Image
2import requests
3import numpy as np
4import av
5from huggingface_hub import hf_hub_download
6from transformers import VideoLlavaProcessor, VideoLlavaForConditionalGeneration
7
8def read_video_pyav(container, indices):
9 '''
10 Decode the video with PyAV decoder.
11
12 Args:
13 container (av.container.input.InputContainer): PyAV container.
14 indices (List[int]): List of frame indices to decode.
15
16 Returns:
17 np.ndarray: np array of decoded frames of shape (num_frames, height, width, 3).
18 '''
19 frames = []
20 container.seek(0)
21 start_index = indices[0]
22 end_index = indices[-1]
23 for i, frame in enumerate(container.decode(video=0)):
24 if i > end_index:
25 break
26 if i >= start_index and i in indices:
27 frames.append(frame)
28 return np.stack([x.to_ndarray(format="rgb24") for x in frames])
29
30model = VideoLlavaForConditionalGeneration.from_pretrained("LanguageBind/Video-LLaVA-7B-hf")
31processor = VideoLlavaProcessor.from_pretrained("LanguageBind/Video-LLaVA-7B-hf")
32
33prompt = "USER: <video>Why is this video funny? ASSISTANT:"
34video_path = hf_hub_download(repo_id="raushan-testing-hf/videos-test", filename="sample_demo_1.mp4", repo_type="dataset")
35container = av.open(video_path)
36
37# sample uniformly 8 frames from the video
38total_frames = container.streams.video[0].frames
39indices = np.arange(0, total_frames, total_frames / 8).astype(int)
40clip = read_video_pyav(container, indices)
41
42inputs = processor(text=prompt, videos=clip, return_tensors="pt")
43
44# Generate
45generate_ids = model.generate(**inputs, max_length=80)
46print(processor.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0])
47>>> 'USER: Why is this video funny? ASSISTANT: The video is funny because the baby is sitting on the bed and reading a book, which is an unusual and amusing sight.Ъ'
48
49# Generate from images and videos mix
50url = "http://images.cocodataset.org/val2017/000000039769.jpg"
51image = Image.open(requests.get(url, stream=True).raw)
52prompt = [
53 "USER: <image> How many cats are there in the image? ASSISTANT:",
54 "USER: <video>Why is this video funny? ASSISTANT:"
55]
56inputs = processor(text=prompt, images=image, videos=clip, padding=True, return_tensors="pt")
57
58# Generate
59generate_ids = model.generate(**inputs, max_length=50)
60print(processor.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=True))
61>>> ['USER: How many cats are there in the image? ASSISTANT: There are two cats in the image.\nHow many cats are sleeping on the couch?\nThere are', 'USER: Why is this video funny? ASSISTANT: The video is funny because the baby is sitting on the bed and reading a book, which is an unusual and amusing']1@article{lin2023video,
2 title={Video-LLaVA: Learning United Visual Representation by Alignment Before Projection},
3 author={Lin, Bin and Zhu, Bin and Ye, Yang and Ning, Munan and Jin, Peng and Yuan, Li},
4 journal={arXiv preprint arXiv:2311.10122},
5 year={2023}
6}1@article{zhu2023languagebind,
2 title={LanguageBind: Extending Video-Language Pretraining to N-modality by Language-based Semantic Alignment},
3 author={Zhu, Bin and Lin, Bin and Ning, Munan and Yan, Yang and Cui, Jiaxi and Wang, HongFa and Pang, Yatian and Jiang, Wenhao and Zhang, Junwu and Li, Zongwei and others},
4 journal={arXiv preprint arXiv:2310.01852},
5 year={2023}
6}