Views
No views yet
1import requests
2import torch
3from PIL import Image
4from transformers import AutoProcessor, LlavaForConditionalGeneration
5
6
7quantized_model_path="OPEA/llama-joycaption-alpha-two-hf-llava-int4-sym-inc"
8
9# Load JoyCaption INT4 Model
10processor = AutoProcessor.from_pretrained(quantized_model_path)
11model = LlavaForConditionalGeneration.from_pretrained(
12 quantized_model_path,
13 device_map="auto",
14 revision="bc917a8" ## ##AutoGPTQ format
15)
16model.eval()
17
18image_url = "http://images.cocodataset.org/train2017/000000116003.jpg"
19content = "Write a descriptive caption for this image in a formal tone."
20
21# Preparation for inference
22with torch.no_grad():
23 image = Image.open(requests.get(image_url, stream=True).raw)
24 messages = [
25 {
26 "role": "system",
27 "content": "You are a helpful image captioner.",
28 },
29 {
30 "role": "user",
31 "content": content,
32 },
33 ]
34 prompt = processor.apply_chat_template(messages, tokenize = False, add_generation_prompt = True)
35 assert isinstance(prompt, str)
36 inputs = processor(text=[prompt], images=[image], return_tensors="pt").to(model.device)
37 inputs['pixel_values'] = inputs['pixel_values'].to(model.dtype)
38
39 # Generate the captions
40 generate_ids = model.generate(
41 **inputs,
42 max_new_tokens=50,
43 do_sample=False,
44 suppress_tokens=None,
45 use_cache=True,
46 temperature=0.6,
47 top_k=None,
48 top_p=0.9,
49 )[0]
50
51 # Trim off the prompt
52 generate_ids = generate_ids[inputs['input_ids'].shape[1]:]
53
54 # Decode the caption
55 caption = processor.tokenizer.decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)
56 caption = caption.strip()
57 print(caption)
58
59
60##INT4: This black-and-white photograph captures a moment of triumph on a tennis court. The central figure is a male tennis player, mid-celebration,
61## with his arms raised high in victory. He is wearing a white athletic shirt and shorts, with a
62
63
64##BF16: This black-and-white photograph captures a moment of triumph on a tennis court. The central figure is a male tennis player, mid-celebration,
65## with his arms raised high in victory. He is wearing a white tennis shirt and shorts, with a
66
67image_url = "http://images.cocodataset.org/train2017/000000411975.jpg"
68content = "Write a descriptive caption for this image in a formal tone."
69
70##INT4: This is a photograph capturing a moment during a baseball game. The image is taken from a high vantage point, likely from the stands,
71## looking down onto the field. The main focus is on a young girl and a man standing on the grassy
72
73##BF16: This is a photograph capturing a moment during a baseball game. The image is taken from a high angle, looking down onto the field.
74## In the foreground, there is a section of the baseball field with a reddish-brown dirt infield and a well
75
76
77image_url = "http://images.cocodataset.org/train2017/000000093025.jpg"
78content = "Write a descriptive caption for this image in a formal tone."
79
80##INT4: This is a photograph capturing a serene outdoor scene on a rocky mountainous terrain under a clear blue sky with scattered white clouds.
81## The central focus is on a man and a sheep. The man, positioned slightly to the right of the center, is sitting
82
83##BF16: This photograph captures a serene mountainous landscape under a bright blue sky dotted with fluffy white clouds. In the foreground,
84## a man and a woman are seated on a rocky outcrop. The man, positioned on the left, is wearing a blue jacket and
851pip install auto-round
2auto-round-mllm \
3--model fancyfeast/llama-joycaption-alpha-two-hf-llava \
4--device 0 \
5--group_size 128 \
6--bits 4 \
7--iters 1000 \
8--nsample 512 \
9--seqlen 2048 \
10--template default \
11--model_dtype "float16" \
12--format 'auto_gptq,auto_round' \
13--output_dir "./tmp_autoround"