Views
No views yet
1import torch
2import torchvision.transforms as T
3from PIL import Image
4from torchvision.transforms.functional import InterpolationMode
5from transformers import AutoModel, AutoTokenizer
6
7IMAGENET_MEAN = (0.485, 0.456, 0.406)
8IMAGENET_STD = (0.229, 0.224, 0.225)
9
10def build_transform(input_size):
11 MEAN, STD = IMAGENET_MEAN, IMAGENET_STD
12 transform = T.Compose([
13 T.Lambda(lambda img: img.convert("RGB") if img.mode != "RGB" else img),
14 T.Resize((input_size, input_size), interpolation=InterpolationMode.BICUBIC),
15 T.ToTensor(),
16 T.Normalize(mean=MEAN, std=STD)
17 ])
18 return transform
19
20def find_closest_aspect_ratio(aspect_ratio, target_ratios, width, height, image_size):
21 best_ratio_diff = float("inf")
22 best_ratio = (1, 1)
23 area = width * height
24 for ratio in target_ratios:
25 target_aspect_ratio = ratio[0] / ratio[1]
26 ratio_diff = abs(aspect_ratio - target_aspect_ratio)
27 if ratio_diff < best_ratio_diff:
28 best_ratio_diff = ratio_diff
29 best_ratio = ratio
30 elif ratio_diff == best_ratio_diff:
31 if area > 0.5 * image_size * image_size * ratio[0] * ratio[1]:
32 best_ratio = ratio
33 return best_ratio
34
35def dynamic_preprocess(image, min_num=1, max_num=12, image_size=448, use_thumbnail=False):
36 orig_width, orig_height = image.size
37 aspect_ratio = orig_width / orig_height
38
39 # calculate the existing image aspect ratio
40 target_ratios = set(
41 (i, j) for n in range(min_num, max_num + 1) for i in range(1, n + 1) for j in range(1, n + 1) if
42 i * j <= max_num and i * j >= min_num)
43 target_ratios = sorted(target_ratios, key=lambda x: x[0] * x[1])
44
45 # find the closest aspect ratio to the target
46 target_aspect_ratio = find_closest_aspect_ratio(
47 aspect_ratio, target_ratios, orig_width, orig_height, image_size)
48
49 # calculate the target width and height
50 target_width = image_size * target_aspect_ratio[0]
51 target_height = image_size * target_aspect_ratio[1]
52 blocks = target_aspect_ratio[0] * target_aspect_ratio[1]
53
54 # resize the image
55 resized_img = image.resize((target_width, target_height))
56 processed_images = []
57 for i in range(blocks):
58 box = (
59 (i % (target_width // image_size)) * image_size,
60 (i // (target_width // image_size)) * image_size,
61 ((i % (target_width // image_size)) + 1) * image_size,
62 ((i // (target_width // image_size)) + 1) * image_size
63 )
64 # split the image
65 split_img = resized_img.crop(box)
66 processed_images.append(split_img)
67 assert len(processed_images) == blocks
68 if use_thumbnail and len(processed_images) != 1:
69 thumbnail_img = image.resize((image_size, image_size))
70 processed_images.append(thumbnail_img)
71 return processed_images
72
73def load_image(image_file, input_size=448, max_num=12):
74 image = Image.open(image_file).convert("RGB")
75 transform = build_transform(input_size=input_size)
76 images = dynamic_preprocess(image, image_size=input_size, use_thumbnail=True, max_num=max_num)
77 pixel_values = [transform(image) for image in images]
78 pixel_values = torch.stack(pixel_values)
79 return pixel_values
80
81# If you want to load a model using multiple GPUs, please refer to the `Multiple GPUs` section.
82path = "natong19/InternVL2-8B-abliterated"
83model = AutoModel.from_pretrained(
84 path,
85 torch_dtype=torch.bfloat16,
86 low_cpu_mem_usage=True,
87 trust_remote_code=True).eval().cuda()
88tokenizer = AutoTokenizer.from_pretrained(path, trust_remote_code=True, use_fast=False)
89
90# set the max number of tiles in `max_num`
91pixel_values = load_image("./examples/image1.jpg", max_num=12).to(torch.bfloat16).cuda()
92generation_config = dict(max_new_tokens=1024, do_sample=False)
93
94# pure-text conversation (纯文本对话)
95question = "Hello, who are you?"
96response, history = model.chat(tokenizer, None, question, generation_config, history=None, return_history=True)
97print(f"User: {question}\nAssistant: {response}")
98
99question = "Can you tell me a story?"
100response, history = model.chat(tokenizer, None, question, generation_config, history=history, return_history=True)
101print(f"User: {question}\nAssistant: {response}")
102
103# single-image multi-round conversation (单图多轮对话)
104question = "<image>\nPlease describe the image in detail."
105response, history = model.chat(tokenizer, pixel_values, question, generation_config, history=None, return_history=True)
106print(f"User: {question}\nAssistant: {response}")
107
108question = "Please write a poem according to the image."
109response, history = model.chat(tokenizer, pixel_values, question, generation_config, history=history, return_history=True)
110print(f"User: {question}\nAssistant: {response}")
111
112# multi-image multi-round conversation, separate images (多图多轮对话，独立图像)
113pixel_values1 = load_image("./examples/image1.jpg", max_num=12).to(torch.bfloat16).cuda()
114pixel_values2 = load_image("./examples/image2.jpg", max_num=12).to(torch.bfloat16).cuda()
115pixel_values = torch.cat((pixel_values1, pixel_values2), dim=0)
116num_patches_list = [pixel_values1.size(0), pixel_values2.size(0)]
117
118question = "Image-1: <image>\nImage-2: <image>\nDescribe the two images in detail."
119response, history = model.chat(tokenizer, pixel_values, question, generation_config,
120 num_patches_list=num_patches_list,
121 history=None, return_history=True)
122print(f"User: {question}\nAssistant: {response}")
123
124question = "What are the similarities and differences between these two images."
125response, history = model.chat(tokenizer, pixel_values, question, generation_config,
126 num_patches_list=num_patches_list,
127 history=history, return_history=True)
128print(f"User: {question}\nAssistant: {response}")| Datasets | InternVL2-8B | InternVL2-8B-abliterated |
|---|---|---|
| Text benchmarks | ||
| ARC (25-shot) | 59.1 | 58.5 |
| MMLU (5-shot) | 71.4 | 70.8 |
| TruthfulQA (0-shot) | 50.8 | 49.1 |
| Winogrande (5-shot) | 81.8 | 81.1 |
| Multimodal benchmarks | ||
| AI2D (lite) | 80.2 | 80.0 |
| GQA (lite) | 74.0 | 74.6 |
| MMBench (EN dev, lite) | 85.6 | 84.8 |
| MMMU (val) | 48.0 | 48.0 |
| OCRBench | 76.5 | 77.3 |
| VQAv2 (val, lite) | 76.4 | 76.2 |