Views
No views yet
[!Warning] 🚨 Note: In our experience, the InternViT V2.5 series is better suited for building MLLMs than traditional computer vision tasks.
1import torch
2from PIL import Image
3from transformers import AutoModel, CLIPImageProcessor
4
5model = AutoModel.from_pretrained(
6 'OpenGVLab/InternViT-6B-448px-V1-2',
7 torch_dtype=torch.bfloat16,
8 low_cpu_mem_usage=True,
9 trust_remote_code=True).cuda().eval()
10
11image = Image.open('./examples/image1.jpg').convert('RGB')
12
13image_processor = CLIPImageProcessor.from_pretrained('OpenGVLab/InternViT-6B-448px-V1-2')
14
15pixel_values = image_processor(images=image, return_tensors='pt').pixel_values
16pixel_values = pixel_values.to(torch.bfloat16).cuda()
17
18outputs = model(pixel_values)1@article{chen2024expanding,
2 title={Expanding Performance Boundaries of Open-Source Multimodal Models with Model, Data, and Test-Time Scaling},
3 author={Chen, Zhe and Wang, Weiyun and Cao, Yue and Liu, Yangzhou and Gao, Zhangwei and Cui, Erfei and Zhu, Jinguo and Ye, Shenglong and Tian, Hao and Liu, Zhaoyang and others},
4 journal={arXiv preprint arXiv:2412.05271},
5 year={2024}
6}
7@article{gao2024mini,
8 title={Mini-internvl: A flexible-transfer pocket multimodal model with 5\% parameters and 90\% performance},
9 author={Gao, Zhangwei and Chen, Zhe and Cui, Erfei and Ren, Yiming and Wang, Weiyun and Zhu, Jinguo and Tian, Hao and Ye, Shenglong and He, Junjun and Zhu, Xizhou and others},
10 journal={arXiv preprint arXiv:2410.16261},
11 year={2024}
12}
13@article{chen2024far,
14 title={How Far Are We to GPT-4V? Closing the Gap to Commercial Multimodal Models with Open-Source Suites},
15 author={Chen, Zhe and Wang, Weiyun and Tian, Hao and Ye, Shenglong and Gao, Zhangwei and Cui, Erfei and Tong, Wenwen and Hu, Kongzhi and Luo, Jiapeng and Ma, Zheng and others},
16 journal={arXiv preprint arXiv:2404.16821},
17 year={2024}
18}
19@inproceedings{chen2024internvl,
20 title={Internvl: Scaling up vision foundation models and aligning for generic visual-linguistic tasks},
21 author={Chen, Zhe and Wu, Jiannan and Wang, Wenhai and Su, Weijie and Chen, Guo and Xing, Sen and Zhong, Muyan and Zhang, Qinglong and Zhu, Xizhou and Lu, Lewei and others},
22 booktitle={Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition},
23 pages={24185--24198},
24 year={2024}
25}