INT8 dynamically quantized ONNX export of
apple/DFN5B-CLIP-ViT-H-14-378, a ~5B parameter CLIP model trained on Data Filtering Networks (DFN-5B). This quantized variant runs
2.3x faster on CPU with negligible quality loss (cosine similarity 0.985 vs FP32 original).
DFN5B-CLIP was trained by Apple on 5 billion images filtered from a pool of 43 billion uncurated image-text pairs, using small Data Filtering Networks to automatically curate training data. It achieves 84.2% zero-shot accuracy on ImageNet-1K and 70.9% average across 38 benchmarks.
1import numpy as np
2import onnxruntime as ort
3from PIL import Image
4
5# CLIP preprocessing constants
6MEAN = np.array([0.48145466, 0.4578275, 0.40821073], dtype=np.float32)
7STD = np.array([0.26862954, 0.26130258, 0.27577711], dtype=np.float32)
8
9def preprocess(image_path, size=378):
10 img = Image.open(image_path).convert("RGB")
11 w, h = img.size
12 side = min(w, h)
13 img = img.crop(((w - side) // 2, (h - side) // 2, (w + side) // 2, (h + side) // 2))
14 img = img.resize((size, size), Image.BICUBIC)
15 arr = (np.array(img, dtype=np.float32) / 255.0 - MEAN) / STD
16 return arr.transpose(2, 0, 1)[None] # (1, 3, H, W)
17
18sess = ort.InferenceSession("visual_int8.onnx", providers=["CPUExecutionProvider"])
19pixels = preprocess("your_image.jpg")
20image_embeds = sess.run(None, {"pixel_values": pixels})[0] # (1, 1024)
1from transformers import CLIPTokenizer
2
3tokenizer = CLIPTokenizer.from_pretrained("apple/DFN5B-CLIP-ViT-H-14-378")
4tokens = tokenizer(["a photo of a cat", "a photo of a dog"], padding=True, return_tensors="np")
5
6sess = ort.InferenceSession("text_int8.onnx", providers=["CPUExecutionProvider"])
7text_embeds = sess.run(None, {"input_ids": tokens["input_ids"]})[0] # (2, 1024)
1# Cosine similarity between image and text embeddings
2image_embeds = image_embeds / np.linalg.norm(image_embeds, axis=-1, keepdims=True)
3text_embeds = text_embeds / np.linalg.norm(text_embeds, axis=-1, keepdims=True)
4
5logit_scale = 14.2849 # from model config
6probabilities = (image_embeds @ text_embeds.T * logit_scale).softmax(axis=-1)
1@article{fang2023data,
2 title={Data Filtering Networks},
3 author={Fang, Alex and Jose, Albin Madappally and Jain, Amit and Schmidt, Ludwig and Toshev, Alexander and Shankar, Vaishaal},
4 journal={arXiv preprint arXiv:2309.17425},
5 year={2023}
6}
