Views
No views yet
| Quantization | Speed (3022 tokens) | Speed (8192 tokens) w/o encoder | Speed(8192 tokens) w/ encoder |
|---|---|---|---|
| BF16 | 33.40 | 31.91 | 21.33 |
| INT4 | - | 31.95 | 26.37 |
pip install transformers==4.40.1
pip install av
pip install imageio
pip install decord
pip install opencv-python
pip install flash-attn --no-build-isolation1import numpy as np
2import torch
3import torchvision.transforms as T
4from decord import VideoReader, cpu
5from PIL import Image
6from torchvision.transforms.functional import InterpolationMode
7from transformers import AutoModel, AutoTokenizer
8
9
10# model setting
11model_path = 'OpenGVLab/InternVideo2_5_Chat_8B'
12
13tokenizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=True)
14model = AutoModel.from_pretrained(model_path, trust_remote_code=True).half().cuda().to(torch.bfloat16)
15
16IMAGENET_MEAN = (0.485, 0.456, 0.406)
17IMAGENET_STD = (0.229, 0.224, 0.225)
18
19def build_transform(input_size):
20 MEAN, STD = IMAGENET_MEAN, IMAGENET_STD
21 transform = T.Compose([T.Lambda(lambda img: img.convert("RGB") if img.mode != "RGB" else img), T.Resize((input_size, input_size), interpolation=InterpolationMode.BICUBIC), T.ToTensor(), T.Normalize(mean=MEAN, std=STD)])
22 return transform
23
24
25def find_closest_aspect_ratio(aspect_ratio, target_ratios, width, height, image_size):
26 best_ratio_diff = float("inf")
27 best_ratio = (1, 1)
28 area = width * height
29 for ratio in target_ratios:
30 target_aspect_ratio = ratio[0] / ratio[1]
31 ratio_diff = abs(aspect_ratio - target_aspect_ratio)
32 if ratio_diff < best_ratio_diff:
33 best_ratio_diff = ratio_diff
34 best_ratio = ratio
35 elif ratio_diff == best_ratio_diff:
36 if area > 0.5 * image_size * image_size * ratio[0] * ratio[1]:
37 best_ratio = ratio
38 return best_ratio
39
40
41def dynamic_preprocess(image, min_num=1, max_num=6, image_size=448, use_thumbnail=False):
42 orig_width, orig_height = image.size
43 aspect_ratio = orig_width / orig_height
44
45 # calculate the existing image aspect ratio
46 target_ratios = set((i, j) for n in range(min_num, max_num + 1) for i in range(1, n + 1) for j in range(1, n + 1) if i * j <= max_num and i * j >= min_num)
47 target_ratios = sorted(target_ratios, key=lambda x: x[0] * x[1])
48
49 # find the closest aspect ratio to the target
50 target_aspect_ratio = find_closest_aspect_ratio(aspect_ratio, target_ratios, orig_width, orig_height, image_size)
51
52 # calculate the target width and height
53 target_width = image_size * target_aspect_ratio[0]
54 target_height = image_size * target_aspect_ratio[1]
55 blocks = target_aspect_ratio[0] * target_aspect_ratio[1]
56
57 # resize the image
58 resized_img = image.resize((target_width, target_height))
59 processed_images = []
60 for i in range(blocks):
61 box = ((i % (target_width // image_size)) * image_size, (i // (target_width // image_size)) * image_size, ((i % (target_width // image_size)) + 1) * image_size, ((i // (target_width // image_size)) + 1) * image_size)
62 # split the image
63 split_img = resized_img.crop(box)
64 processed_images.append(split_img)
65 assert len(processed_images) == blocks
66 if use_thumbnail and len(processed_images) != 1:
67 thumbnail_img = image.resize((image_size, image_size))
68 processed_images.append(thumbnail_img)
69 return processed_images
70
71
72def load_image(image, input_size=448, max_num=6):
73 transform = build_transform(input_size=input_size)
74 images = dynamic_preprocess(image, image_size=input_size, use_thumbnail=True, max_num=max_num)
75 pixel_values = [transform(image) for image in images]
76 pixel_values = torch.stack(pixel_values)
77 return pixel_values
78
79
80def get_index(bound, fps, max_frame, first_idx=0, num_segments=32):
81 if bound:
82 start, end = bound[0], bound[1]
83 else:
84 start, end = -100000, 100000
85 start_idx = max(first_idx, round(start * fps))
86 end_idx = min(round(end * fps), max_frame)
87 seg_size = float(end_idx - start_idx) / num_segments
88 frame_indices = np.array([int(start_idx + (seg_size / 2) + np.round(seg_size * idx)) for idx in range(num_segments)])
89 return frame_indices
90
91def get_num_frames_by_duration(duration):
92 local_num_frames = 4
93 num_segments = int(duration // local_num_frames)
94 if num_segments == 0:
95 num_frames = local_num_frames
96 else:
97 num_frames = local_num_frames * num_segments
98
99 num_frames = min(512, num_frames)
100 num_frames = max(128, num_frames)
101
102 return num_frames
103
104def load_video(video_path, bound=None, input_size=448, max_num=1, num_segments=32, get_frame_by_duration = False):
105 vr = VideoReader(video_path, ctx=cpu(0), num_threads=1)
106 max_frame = len(vr) - 1
107 fps = float(vr.get_avg_fps())
108
109 pixel_values_list, num_patches_list = [], []
110 transform = build_transform(input_size=input_size)
111 if get_frame_by_duration:
112 duration = max_frame / fps
113 num_segments = get_num_frames_by_duration(duration)
114 frame_indices = get_index(bound, fps, max_frame, first_idx=0, num_segments=num_segments)
115 for frame_index in frame_indices:
116 img = Image.fromarray(vr[frame_index].asnumpy()).convert("RGB")
117 img = dynamic_preprocess(img, image_size=input_size, use_thumbnail=True, max_num=max_num)
118 pixel_values = [transform(tile) for tile in img]
119 pixel_values = torch.stack(pixel_values)
120 num_patches_list.append(pixel_values.shape[0])
121 pixel_values_list.append(pixel_values)
122 pixel_values = torch.cat(pixel_values_list)
123 return pixel_values, num_patches_list
124
125# evaluation setting
126max_num_frames = 512
127generation_config = dict(
128 do_sample=False,
129 temperature=0.0,
130 max_new_tokens=1024,
131 top_p=0.1,
132 num_beams=1
133)
134video_path = "your_video.mp4"
135num_segments=128
136
137
138with torch.no_grad():
139
140 pixel_values, num_patches_list = load_video(video_path, num_segments=num_segments, max_num=1, get_frame_by_duration=False)
141 pixel_values = pixel_values.to(torch.bfloat16).to(model.device)
142 video_prefix = "".join([f"Frame{i+1}: <image>\n" for i in range(len(num_patches_list))])
143 # single-turn conversation
144 question1 = "Describe this video in detail."
145 question = video_prefix + question1
146 output1, chat_history = model.chat(tokenizer, pixel_values, question, generation_config, num_patches_list=num_patches_list, history=None, return_history=True)
147 print(output1)
148
149 # multi-turn conversation
150 question2 = "How many people appear in the video?"
151 output2, chat_history = model.chat(tokenizer, pixel_values, question, generation_config, num_patches_list=num_patches_list, history=chat_history, return_history=True)
152
153 print(output2)1
2@article{wang2025internvideo,
3 title={InternVideo2.5: Empowering Video MLLMs with Long and Rich Context Modeling},
4 author={Wang, Yi and Li, Xinhao and Yan, Ziang and He, Yinan and Yu, Jiashuo and Zeng, Xiangyu and Wang, Chenting and Ma, Changlian and Huang, Haian and Gao, Jianfei and Dou, Min and Chen, Kai and Wang, Wenhai and Qiao, Yu and Wang, Yali and Wang, Limin},
5 journal={arXiv preprint arXiv:2501.12386},
6 year={2025}
7}