Views
No views yet
python -m sglang.launch_server --model-path dillonlaird/hf-llava-v1.6-34b --port 300001import requests
2from transformers import AutoTokenizer
3
4
5def generate(image_path: str, prompt: str, tokenizer):
6 chat = [
7 {"role": "system", "content": "Answer the question."},
8 {"role": "user", "content": "<image>\n" + prompt},
9 ]
10 chat_str = tokenizer.apply_chat_template(chat, tokenize=False)
11 chat_str += "<|img_start|>assistant\n"
12 sampling_params = {"temperature": 0.2, "max_new_tokens": 1536}
13 res = requests.post(
14 "http://localhost:30000/generate",
15 json={
16 "text": chat_str,
17 "image_data": image_path,
18 "sampling_params": sampling_params,
19 },
20 )
21 return res.json()["text"]
22
23
24if __name__ == "__main__":
25 tokenizer = AutoTokenizer.from_pretrained("liuhaotian/llava-v1.6-34b")
26 image_path = "path/to/image.jpg"
27 prompt = "What is the name of the mountain?"
28 desc = generate(image_path, prompt, tokenizer)