Views
No views yet
Pillow==10.1.0
torch==2.1.2
torchvision==0.16.2
transformers==4.40.0
sentencepiece==0.1.991# test.py
2import torch
3from PIL import Image
4from transformers import AutoModel, AutoTokenizer
5
6model = AutoModel.from_pretrained('openbmb/MiniCPM-Llama3-V-2_5', trust_remote_code=True, torch_dtype=torch.float16)
7model = model.to(device='cuda')
8
9tokenizer = AutoTokenizer.from_pretrained('openbmb/MiniCPM-Llama3-V-2_5', trust_remote_code=True)
10model.eval()
11
12image = Image.open('xx.jpg').convert('RGB')
13question = 'What is in the image?'
14msgs = [{'role': 'user', 'content': question}]
15
16res = model.chat(
17 image=image,
18 msgs=msgs,
19 tokenizer=tokenizer,
20 sampling=True, # if sampling=False, beam_search will be used by default
21 temperature=0.7,
22 # system_prompt='' # pass system_prompt if needed
23)
24print(res)
25
26## if you want to use streaming, please make sure sampling=True and stream=True
27## the model.chat will return a generator
28res = model.chat(
29 image=image,
30 msgs=msgs,
31 tokenizer=tokenizer,
32 sampling=True,
33 temperature=0.7,
34 stream=True
35)
36
37generated_text = ""
38for new_text in res:
39 generated_text += new_text
40 print(new_text, flush=True, end='')1@article{yu2023rlhf,
2 title={Rlhf-v: Towards trustworthy mllms via behavior alignment from fine-grained correctional human feedback},
3 author={Yu, Tianyu and Yao, Yuan and Zhang, Haoye and He, Taiwen and Han, Yifeng and Cui, Ganqu and Hu, Jinyi and Liu, Zhiyuan and Zheng, Hai-Tao and Sun, Maosong and others},
4 journal={arXiv preprint arXiv:2312.00849},
5 year={2023}
6}
7@article{viscpm,
8 title={Large Multilingual Models Pivot Zero-Shot Multimodal Learning across Languages},
9 author={Jinyi Hu and Yuan Yao and Chongyi Wang and Shan Wang and Yinxu Pan and Qianyu Chen and Tianyu Yu and Hanghao Wu and Yue Zhao and Haoye Zhang and Xu Han and Yankai Lin and Jiao Xue and Dahai Li and Zhiyuan Liu and Maosong Sun},
10 journal={arXiv preprint arXiv:2308.12038},
11 year={2023}
12}
13@article{xu2024llava-uhd,
14 title={{LLaVA-UHD}: an LMM Perceiving Any Aspect Ratio and High-Resolution Images},
15 author={Xu, Ruyi and Yao, Yuan and Guo, Zonghao and Cui, Junbo and Ni, Zanlin and Ge, Chunjiang and Chua, Tat-Seng and Liu, Zhiyuan and Huang, Gao},
16 journal={arXiv preprint arXiv:2403.11703},
17 year={2024}
18}
19@article{yu2024rlaifv,
20 title={RLAIF-V: Aligning MLLMs through Open-Source AI Feedback for Super GPT-4V Trustworthiness},
21 author={Yu, Tianyu and Zhang, Haoye and Yao, Yuan and Dang, Yunkai and Chen, Da and Lu, Xiaoman and Cui, Ganqu and He, Taiwen and Liu, Zhiyuan and Chua, Tat-Seng and Sun, Maosong},
22 journal={arXiv preprint arXiv:2405.17220},
23 year={2024},
24}