Views
No views yet
mps device (~6 image embeddings per second vs 1.5 images/sec for torch+mps), and according to asitop profiling, using about 3/4 of the energy to do so (6W average vs 8W for torch+mps).from transformers import CLIPProcessor
import coremltools as ct
from PIL import Image
preprocessor = CLIPProcessor.from_pretrained("laion/CLIP-ViT-H-14-laion2B-s32B-b79K")
model_coreml_image = ct.models.MLModel('CLIP-ViT-H-14-laion2B-s32B-b79K.image-encoder.mlprogram')
model_coreml_text = ct.models.MLModel('CLIP-ViT-H-14-laion2B-s32B-b79K.text-encoder.mlprogram')
image = Image.open("example.jpg")
preprocessed_image = preprocessor(text=None, images=image, return_tensors="pt", padding=True)
image_embedding = model_coreml.predict({'input_image_preprocessed': preprocessed_image.pixel_values})['output_embedding']
text = 'example text'
preprocessed_text = preprocessor(text=text, images=None, return_tensors="pt", padding=True)
text_embedding = model_coreml_text.predict({'input_text_token_ids': preprocessed_text.input_ids})['output_embedding'])