Views
No views yet
HF_token to the environment variable.| Model | MVBench | VideoMME(w/o sub) |
|---|---|---|
| InternVideo2-Chat-8B | 60.3 | 41.9 |
| InternVideo2-Chat-8B-HD | 65.4 | 46.1 |
| InternVideo2-Chat-8B-HD-F16 | 67.5 | 49.4 |
| InternVideo2-Chat-8B-InternLM | 61.9 | 49.1 |
export HF_TOKEN=hf_....transformers >= 4.38.01import os
2token = os.environ['HF_TOKEN']
3import torch
4
5from transformers import AutoTokenizer, AutoModel
6
7tokenizer = AutoTokenizer.from_pretrained('OpenGVLab/InternVideo2_chat_8B_HD_F16',
8 trust_remote_code=True,
9 use_fast=False,
10 token=token)
11if torch.cuda.is_available():
12 model = AutoModel.from_pretrained(
13 'OpenGVLab/InternVideo2_chat_8B_HD_F16',
14 torch_dtype=torch.bfloat16,
15 trust_remote_code=True).cuda()
16else:
17 model = AutoModel.from_pretrained(
18 'OpenGVLab/InternVideo2_chat_8B_HD_F16',
19 torch_dtype=torch.bfloat16,
20 trust_remote_code=True)
21
22
23from decord import VideoReader, cpu
24from PIL import Image
25import numpy as np
26import numpy as np
27import decord
28from decord import VideoReader, cpu
29import torch.nn.functional as F
30import torchvision.transforms as T
31from torchvision.transforms import PILToTensor
32from torchvision import transforms
33from torchvision.transforms.functional import InterpolationMode
34decord.bridge.set_bridge("torch")
35
36def get_index(num_frames, num_segments):
37 seg_size = float(num_frames - 1) / num_segments
38 start = int(seg_size / 2)
39 offsets = np.array([
40 start + int(np.round(seg_size * idx)) for idx in range(num_segments)
41 ])
42 return offsets
43
44
45def load_video(video_path, num_segments=8, return_msg=False, resolution=224, hd_num=4, padding=False):
46 vr = VideoReader(video_path, ctx=cpu(0), num_threads=1)
47 num_frames = len(vr)
48 frame_indices = get_index(num_frames, num_segments)
49
50 mean = (0.485, 0.456, 0.406)
51 std = (0.229, 0.224, 0.225)
52
53 transform = transforms.Compose([
54 transforms.Lambda(lambda x: x.float().div(255.0)),
55 transforms.Normalize(mean, std)
56 ])
57
58 frames = vr.get_batch(frame_indices)
59 frames = frames.permute(0, 3, 1, 2)
60
61 if padding:
62 frames = HD_transform_padding(frames.float(), image_size=resolution, hd_num=hd_num)
63 else:
64 frames = HD_transform_no_padding(frames.float(), image_size=resolution, hd_num=hd_num)
65
66 frames = transform(frames)
67 # print(frames.shape)
68 T_, C, H, W = frames.shape
69
70 sub_img = frames.reshape(
71 1, T_, 3, H//resolution, resolution, W//resolution, resolution
72 ).permute(0, 3, 5, 1, 2, 4, 6).reshape(-1, T_, 3, resolution, resolution).contiguous()
73
74 glb_img = F.interpolate(
75 frames.float(), size=(resolution, resolution), mode='bicubic', align_corners=False
76 ).to(sub_img.dtype).unsqueeze(0)
77
78 frames = torch.cat([sub_img, glb_img]).unsqueeze(0)
79
80 if return_msg:
81 fps = float(vr.get_avg_fps())
82 sec = ", ".join([str(round(f / fps, 1)) for f in frame_indices])
83 # " " should be added in the start and end
84 msg = f"The video contains {len(frame_indices)} frames sampled at {sec} seconds."
85 return frames, msg
86 else:
87 return frames
88
89def HD_transform_padding(frames, image_size=224, hd_num=6):
90 def _padding_224(frames):
91 _, _, H, W = frames.shape
92 tar = int(np.ceil(H / 224) * 224)
93 top_padding = (tar - H) // 2
94 bottom_padding = tar - H - top_padding
95 left_padding = 0
96 right_padding = 0
97
98 padded_frames = F.pad(
99 frames,
100 pad=[left_padding, right_padding, top_padding, bottom_padding],
101 mode='constant', value=255
102 )
103 return padded_frames
104
105 _, _, H, W = frames.shape
106 trans = False
107 if W < H:
108 frames = frames.flip(-2, -1)
109 trans = True
110 width, height = H, W
111 else:
112 width, height = W, H
113
114 ratio = width / height
115 scale = 1
116 while scale * np.ceil(scale / ratio) <= hd_num:
117 scale += 1
118 scale -= 1
119 new_w = int(scale * image_size)
120 new_h = int(new_w / ratio)
121
122 resized_frames = F.interpolate(
123 frames, size=(new_h, new_w),
124 mode='bicubic',
125 align_corners=False
126 )
127 padded_frames = _padding_224(resized_frames)
128
129 if trans:
130 padded_frames = padded_frames.flip(-2, -1)
131
132 return padded_frames
133
134def find_closest_aspect_ratio(aspect_ratio, target_ratios, width, height, image_size):
135 best_ratio_diff = float('inf')
136 best_ratio = (1, 1)
137 area = width * height
138 for ratio in target_ratios:
139 target_aspect_ratio = ratio[0] / ratio[1]
140 ratio_diff = abs(aspect_ratio - target_aspect_ratio)
141 if ratio_diff < best_ratio_diff:
142 best_ratio_diff = ratio_diff
143 best_ratio = ratio
144 elif ratio_diff == best_ratio_diff:
145 if area > 0.5 * image_size * image_size * ratio[0] * ratio[1]:
146 best_ratio = ratio
147 return best_ratio
148
149
150def HD_transform_no_padding(frames, image_size=224, hd_num=6, fix_ratio=(2,1)):
151 min_num = 1
152 max_num = hd_num
153 _, _, orig_height, orig_width = frames.shape
154 aspect_ratio = orig_width / orig_height
155
156 # calculate the existing video aspect ratio
157 target_ratios = set(
158 (i, j) for n in range(min_num, max_num + 1) for i in range(1, n + 1) for j in range(1, n + 1) if
159 i * j <= max_num and i * j >= min_num)
160 target_ratios = sorted(target_ratios, key=lambda x: x[0] * x[1])
161
162 # find the closest aspect ratio to the target
163 if fix_ratio:
164 target_aspect_ratio = fix_ratio
165 else:
166 target_aspect_ratio = find_closest_aspect_ratio(
167 aspect_ratio, target_ratios, orig_width, orig_height, image_size)
168
169 # calculate the target width and height
170 target_width = image_size * target_aspect_ratio[0]
171 target_height = image_size * target_aspect_ratio[1]
172 blocks = target_aspect_ratio[0] * target_aspect_ratio[1]
173
174 # resize the frames
175 resized_frame = F.interpolate(
176 frames, size=(target_height, target_width),
177 mode='bicubic', align_corners=False
178 )
179 return resized_frame
180
181video_path = "yoga.mp4"
182# sample uniformly 16 frames from the video
183video_tensor = load_video(video_path, num_segments=16, return_msg=False, resolution=224, hd_num=6)
184video_tensor = video_tensor.to(model.device)
185
186chat_history = []
187response, chat_history = model.chat(tokenizer, '', 'Describe the action step by step.', media_type='video', media_tensor=video_tensor, chat_history= chat_history, return_history=True,generation_config={'do_sample':False})
188print(response)
189
190response, chat_history = model.chat(tokenizer, '', 'What is she wearing?', media_type='video', media_tensor=video_tensor, chat_history= chat_history, return_history=True,generation_config={'do_sample':False})@article{wang2024internvideo2,
title={Internvideo2: Scaling video foundation models for multimodal video understanding},
author={Wang, Yi and Li, Kunchang and Li, Xinhao and Yu, Jiashuo and He, Yinan and Wang, Chenting and Chen, Guo and Pei, Baoqi and Zheng, Rongkun and Xu, Jilan and Wang, Zun and others},
journal={arXiv preprint arXiv:2403.15377},
year={2024}
}
@article{li2023videochat,
title={Videochat: Chat-centric video understanding},
author={Li, KunChang and He, Yinan and Wang, Yi and Li, Yizhuo and Wang, Wenhai and Luo, Ping and Wang, Yali and Wang, Limin and Qiao, Yu},
journal={arXiv preprint arXiv:2305.06355},
year={2023}
}