Views
No views yet
[!IMPORTANT] Original Model Link : https://huggingface.co/BAAI/EVA-CLIP-8B
name: EVA-CLIP-8B-SFT
base_model: BAAI/EVA-CLIP-8B
license: apache-2.0
pipeline_tag: image-to-text
size:
- 30100210576
- 30GB
- 10GB x 3
- 3GB x 1
tasks :
- image-to-text
- image-text-to-text
- visual-question-answering
- zero-shot-image-classification
language: en
datasets :
- laion/laion2B-en
- kakaobrain/coyo-700m1
2from PIL import Image
3from transformers import AutoModel, AutoConfig
4from transformers import CLIPImageProcessor, pipeline, CLIPTokenizer
5import torch
6import torchvision.transforms as T
7from torchvision.transforms import InterpolationMode
8
9image_path = "CLIP.png"
10model_name_or_path = "BAAI/EVA-CLIP-8B" # or /path/to/local/EVA-CLIP-8B
11image_size = 224
12
13processor = CLIPImageProcessor.from_pretrained("openai/clip-vit-large-patch14")
14
15# use image processor with conig
16# processor = CLIPImageProcessor(size={"shortest_edge":image_size}, do_center_crop=True, crop_size=image_size)
17
18## you can also directly use the image processor by torchvision
19## squash
20# processor = T.Compose(
21# [
22# T.Lambda(lambda img: img.convert('RGB') if img.mode != 'RGB' else img),
23# T.Resize((image_size, image_size), interpolation=InterpolationMode.BICUBIC),
24# T.ToTensor(),
25# T.Normalize(mean=(0.48145466, 0.4578275, 0.40821073), std=(0.26862954, 0.26130258, 0.27577711))
26# ]
27# )
28## shortest
29## processor = T.Compose(
30# [
31# T.Lambda(lambda img: img.convert('RGB') if img.mode != 'RGB' else img),
32# T.Resize(image_size, interpolation=InterpolationMode.BICUBIC),
33# T.CenterCrop(image_size),
34# T.ToTensor(),
35# T.Normalize(mean=(0.48145466, 0.4578275, 0.40821073), std=(0.26862954, 0.26130258, 0.27577711))
36# ]
37# )
38
39model = AutoModel.from_pretrained(
40 model_name_or_path,
41 torch_dtype=torch.float16,
42 trust_remote_code=True).to('cuda').eval()
43
44image = Image.open(image_path)
45captions = ["a diagram", "a dog", "a cat"]
46tokenizer = CLIPTokenizer.from_pretrained(model_name_or_path)
47input_ids = tokenizer(captions, return_tensors="pt", padding=True).input_ids.to('cuda')
48input_pixels = processor(images=image, return_tensors="pt", padding=True).pixel_values.to('cuda')
49
50with torch.no_grad(), torch.cuda.amp.autocast():
51 image_features = model.encode_image(input_pixels)
52 text_features = model.encode_text(input_ids)
53 image_features /= image_features.norm(dim=-1, keepdim=True)
54 text_features /= text_features.norm(dim=-1, keepdim=True)
55
56label_probs = (100.0 * image_features @ text_features.T).softmax(dim=-1)
57print(f"Label probs: {label_probs}")1import torch
2from eva_clip import create_model_and_transforms, get_tokenizer
3from PIL import Image
4
5model_name = "EVA-CLIP-8B"
6pretrained = "eva_clip" # or "/path/to/EVA_CLIP_18B_psz14_s6B.fp16.pt"
7
8image_path = "CLIP.png"
9caption = ["a diagram", "a dog", "a cat"]
10
11device = "cuda" if torch.cuda.is_available() else "cpu"
12model, _, processor = create_model_and_transforms(model_name, pretrained, force_custom_clip=True)
13tokenizer = get_tokenizer(model_name)
14model = model.to(device)
15
16image = processor(Image.open(image_path)).unsqueeze(0).to(device)
17text = tokenizer(["a diagram", "a dog", "a cat"]).to(device)
18
19with torch.no_grad(), torch.cuda.amp.autocast():
20 image_features = model.encode_image(image)
21 text_features = model.encode_text(text)
22 image_features /= image_features.norm(dim=-1, keepdim=True)
23 text_features /= text_features.norm(dim=-1, keepdim=True)
24
25 text_probs = (100.0 * image_features @ text_features.T).softmax(dim=-1)
26
27print("Label probs:", text_probs)load_zero_partitions() function in eva_clip/factory.py.@article{EVA-CLIP-8B,
title={EVA-CLIP-8B: Scaling CLIP to 18 Billion Parameters},
author={Quan Sun and Jinsheng Wang and Qiying Yu and Yufeng Cui and Fan Zhang and Xiaosong Zhang and Xinlong Wang},
journal={arXiv preprint arXiv:2402.04252},
year={2023}
}