Views
No views yet
1python -m vllm.entrypoints.openai.api_server \
2--model huatuogpt_vision_model_path \
3--tensor_parallel_size 2 \
4--gpu_memory_utilization 0.8 \
5--served-model-name huatuogpt_vision_34b \
6--chat-template "{%- if messages[0]['role'] == 'system' -%}\n {%- set system_message = messages[0]['content'] -%}\n {%- set messages = messages[1:] -%}\n{%- else -%}\n {% set system_message = '' -%}\n{%- endif -%}\n\n{%- for message in messages -%}\n {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%}\n {{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}\n {%- endif -%}\n\n {%- if message['role'] == 'user' -%}\n {{ '<|user|>\n' + message['content'] + '\n' }}\n {%- elif message['role'] == 'assistant' -%}\n {{ '<|assistant|>\n' + message['content'] + '\n' }}\n {%- endif -%}\n{%- endfor -%}\n\n{%- if add_generation_prompt -%}\n {{ '<|assistant|>' }}\n{% endif %}" \
7--port 9559 --max-model-len 2048 > vllm_openai_server.log 2>&1 &1from openai import OpenAI
2from PIL import Image
3import base64
4import io
5
6def get_image(image_path):
7 image = Image.open(image_path).convert('RGB')
8 img_type = image.format
9 if not img_type:
10 img_type = image_path.split('.')[-1]
11 byte_arr = io.BytesIO()
12 image.save(byte_arr, format=img_type)
13 byte_arr.seek(0)
14 image = base64.b64encode(byte_arr.getvalue()).decode()
15 return image, img_type
16
17
18client = OpenAI(
19 base_url="http://localhost:9559/v1",
20 api_key="token-abc123"
21)
22image_path = 'your_image_path'
23image, img_type = get_image(image_path)
24
25
26inputcontent = [{
27 "type": "text",
28 "text": '<image>\nWhat does the picture show?'
29}]
30
31inputcontent.append({
32 "type": "image_url",
33 "image_url": {
34 "url": f"data:image/{img_type};base64,{image}"
35 }
36})
37
38response = client.chat.completions.create(
39 model="huatuogpt_vision_34b",
40 messages=[
41 {"role": "user", "content": inputcontent}
42 ],
43 temperature=0.2
44)
45print(response.choices[0].message.content)@misc{chen2024huatuogptvisioninjectingmedicalvisual,
title={HuatuoGPT-Vision, Towards Injecting Medical Visual Knowledge into Multimodal LLMs at Scale},
author={Junying Chen and Ruyi Ouyang and Anningzhe Gao and Shunian Chen and Guiming Hardy Chen and Xidong Wang and Ruifei Zhang and Zhenyang Cai and Ke Ji and Guangjun Yu and Xiang Wan and Benyou Wang},
year={2024},
eprint={2406.19280},
archivePrefix={arXiv},
primaryClass={cs.CV},
url={https://arxiv.org/abs/2406.19280},
}