Views
No views yet
1import torch
2from PIL import Image
3from transformers import AutoModel, AutoTokenizer
4
5torch.manual_seed(100)
6
7model = AutoModel.from_pretrained('openbmb/MiniCPM-V-4_5', trust_remote_code=True, # or openbmb/MiniCPM-o-2_6
8 attn_implementation='sdpa', torch_dtype=torch.bfloat16) # sdpa or flash_attention_2, no eager
9model = model.eval().cuda()
10tokenizer = AutoTokenizer.from_pretrained('openbmb/MiniCPM-V-4_5', trust_remote_code=True) # or openbmb/MiniCPM-o-2_6
11
12image = Image.open('./assets/minicpmo2_6/show_demo.jpg').convert('RGB')
13
14enable_thinking=False # If `enable_thinking=True`, the long-thinking mode is enabled.
15
16# First round chat
17question = "What is the landform in the picture?"
18msgs = [{'role': 'user', 'content': [image, question]}]
19
20answer = model.chat(
21 msgs=msgs,
22 tokenizer=tokenizer,
23 enable_thinking=enable_thinking
24)
25print(answer)
26
27# Second round chat, pass history context of multi-turn conversation
28msgs.append({"role": "assistant", "content": [answer]})
29msgs.append({"role": "user", "content": ["What should I pay attention to when traveling here?"]})
30
31answer = model.chat(
32 msgs=msgs,
33 tokenizer=tokenizer
34)
35print(answer)1# round1
2The landform in the picture is karst topography. Karst landscapes are characterized by distinctive, jagged limestone hills or mountains with steep, irregular peaks and deep valleys—exactly what you see here These unique formations result from the dissolution of soluble rocks like limestone over millions of years through water erosion.
3
4This scene closely resembles the famous karst landscape of Guilin and Yangshuo in China’s Guangxi Province. The area features dramatic, pointed limestone peaks rising dramatically above serene rivers and lush green forests, creating a breathtaking and iconic natural beauty that attracts millions of visitors each year for its picturesque views.
5
6# round2
7When traveling to a karst landscape like this, here are some important tips:
8
91. Wear comfortable shoes: The terrain can be uneven and hilly.
102. Bring water and snacks for energy during hikes or boat rides.
113. Protect yourself from the sun with sunscreen, hats, and sunglasses—especially since you’ll likely spend time outdoors exploring scenic spots.
124. Respect local customs and nature regulations by not littering or disturbing wildlife.
13
14By following these guidelines, you'll have a safe and enjoyable trip while appreciating the stunning natural beauty of places such as Guilin’s karst mountains.1## The 3d-resampler compresses multiple frames into 64 tokens by introducing temporal_ids.
2# To achieve this, you need to organize your video data into two corresponding sequences:
3# frames: List[Image]
4# temporal_ids: List[List[Int]].
5
6import torch
7from PIL import Image
8from transformers import AutoModel, AutoTokenizer
9from decord import VideoReader, cpu # pip install decord
10from scipy.spatial import cKDTree
11import numpy as np
12import math
13
14model = AutoModel.from_pretrained('openbmb/MiniCPM-V-4_5', trust_remote_code=True, # or openbmb/MiniCPM-o-2_6
15 attn_implementation='sdpa', torch_dtype=torch.bfloat16) # sdpa or flash_attention_2, no eager
16model = model.eval().cuda()
17tokenizer = AutoTokenizer.from_pretrained('openbmb/MiniCPM-V-4_5', trust_remote_code=True) # or openbmb/MiniCPM-o-2_6
18
19MAX_NUM_FRAMES=180 # Indicates the maximum number of frames received after the videos are packed. The actual maximum number of valid frames is MAX_NUM_FRAMES * MAX_NUM_PACKING.
20MAX_NUM_PACKING=3 # indicates the maximum packing number of video frames. valid range: 1-6
21TIME_SCALE = 0.1
22
23def map_to_nearest_scale(values, scale):
24 tree = cKDTree(np.asarray(scale)[:, None])
25 _, indices = tree.query(np.asarray(values)[:, None])
26 return np.asarray(scale)[indices]
27
28
29def group_array(arr, size):
30 return [arr[i:i+size] for i in range(0, len(arr), size)]
31
32def encode_video(video_path, choose_fps=3, force_packing=None):
33 def uniform_sample(l, n):
34 gap = len(l) / n
35 idxs = [int(i * gap + gap / 2) for i in range(n)]
36 return [l[i] for i in idxs]
37 vr = VideoReader(video_path, ctx=cpu(0))
38 fps = vr.get_avg_fps()
39 video_duration = len(vr) / fps
40
41 if choose_fps * int(video_duration) <= MAX_NUM_FRAMES:
42 packing_nums = 1
43 choose_frames = round(min(choose_fps, round(fps)) * min(MAX_NUM_FRAMES, video_duration))
44
45 else:
46 packing_nums = math.ceil(video_duration * choose_fps / MAX_NUM_FRAMES)
47 if packing_nums <= MAX_NUM_PACKING:
48 choose_frames = round(video_duration * choose_fps)
49 else:
50 choose_frames = round(MAX_NUM_FRAMES * MAX_NUM_PACKING)
51 packing_nums = MAX_NUM_PACKING
52
53 frame_idx = [i for i in range(0, len(vr))]
54 frame_idx = np.array(uniform_sample(frame_idx, choose_frames))
55
56 if force_packing:
57 packing_nums = min(force_packing, MAX_NUM_PACKING)
58
59 print(video_path, ' duration:', video_duration)
60 print(f'get video frames={len(frame_idx)}, packing_nums={packing_nums}')
61
62 frames = vr.get_batch(frame_idx).asnumpy()
63
64 frame_idx_ts = frame_idx / fps
65 scale = np.arange(0, video_duration, TIME_SCALE)
66
67 frame_ts_id = map_to_nearest_scale(frame_idx_ts, scale) / TIME_SCALE
68 frame_ts_id = frame_ts_id.astype(np.int32)
69
70 assert len(frames) == len(frame_ts_id)
71
72 frames = [Image.fromarray(v.astype('uint8')).convert('RGB') for v in frames]
73 frame_ts_id_group = group_array(frame_ts_id, packing_nums)
74
75 return frames, frame_ts_id_group
76
77
78video_path="video_test.mp4"
79fps = 5 # fps for video
80force_packing = None # You can set force_packing to ensure that 3D packing is forcibly enabled; otherwise, encode_video will dynamically set the packing quantity based on the duration.
81frames, frame_ts_id_group = encode_video(video_path, fps, force_packing=force_packing)
82
83question = "Describe the video"
84msgs = [
85 {'role': 'user', 'content': frames + [question]},
86]
87
88
89answer = model.chat(
90 msgs=msgs,
91 tokenizer=tokenizer,
92 use_image_id=False,
93 max_slice_nums=1,
94 temporal_ids=frame_ts_id_group
95)
96print(answer)1@article{yao2024minicpm,
2 title={MiniCPM-V: A GPT-4V Level MLLM on Your Phone},
3 author={Yao, Yuan and Yu, Tianyu and Zhang, Ao and Wang, Chongyi and Cui, Junbo and Zhu, Hongji and Cai, Tianchi and Li, Haoyu and Zhao, Weilin and He, Zhihui and others},
4 journal={Nat Commun 16, 5509 (2025)},
5 year={2025}
6}