Views
No views yet
open_clip_inference rust crate, or any other ONNX Runtime based implementation.open_clip_inference in Rust:1use open_clip_inference::Clip;
2use std::path::Path;
3
4#[tokio::main]
5async fn main() -> Result<(), Box<dyn std::error::Error>> {
6 let model_id = "RuteNL/ViT-SO400M-16-SigLIP2-384-ONNX";
7 let mut clip = Clip::from_hf(model_id).build().await?;
8
9 let img = image::open(Path::new("assets/img/cat_face.jpg")).expect("Failed to load image");
10 let texts = &[
11 "A photo of a cat",
12 "A photo of a dog",
13 "A photo of a beignet",
14 ];
15
16 let results = clip.classify(&img, texts)?;
17
18 for (text, prob) in results {
19 println!("{}: {:.2}%", text, prob * 100.0);
20 }
21
22 Ok(())
23}1import torch
2import torch.nn.functional as F
3from urllib.request import urlopen
4from PIL import Image
5from open_clip import create_model_from_pretrained, get_tokenizer # works on open-clip-torch >= 2.31.0, timm >= 1.0.15
6
7model, preprocess = create_model_from_pretrained('hf-hub:timm/ViT-SO400M-16-SigLIP2-384')
8tokenizer = get_tokenizer('hf-hub:timm/ViT-SO400M-16-SigLIP2-384')
9
10image = Image.open(urlopen(
11 'https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/beignets-task-guide.png'
12))
13image = preprocess(image).unsqueeze(0)
14
15labels_list = ["a dog", "a cat", "a donut", "a beignet"]
16text = tokenizer(labels_list, context_length=model.context_length)
17
18with torch.no_grad(), torch.cuda.amp.autocast():
19 image_features = model.encode_image(image, normalize=True)
20 text_features = model.encode_text(text, normalize=True)
21 text_probs = torch.sigmoid(image_features @ text_features.T * model.logit_scale.exp() + model.logit_bias)
22
23zipped_list = list(zip(labels_list, [100 * round(p.item(), 3) for p in text_probs[0]]))
24print("Label probabilities: ", zipped_list)1@article{tschannen2025siglip,
2 title={SigLIP 2: Multilingual Vision-Language Encoders with Improved Semantic Understanding, Localization, and Dense Features},
3 author={Tschannen, Michael and Gritsenko, Alexey and Wang, Xiao and Naeem, Muhammad Ferjad and Alabdulmohsin, Ibrahim and Parthasarathy, Nikhil and Evans, Talfan and Beyer, Lucas and Xia, Ye and Mustafa, Basil and H'enaff, Olivier and Harmsen, Jeremiah and Steiner, Andreas and Zhai, Xiaohua},
4 year={2025},
5 journal={arXiv preprint arXiv:2502.14786}
6} 1@article{zhai2023sigmoid,
2 title={Sigmoid loss for language image pre-training},
3 author={Zhai, Xiaohua and Mustafa, Basil and Kolesnikov, Alexander and Beyer, Lucas},
4 journal={arXiv preprint arXiv:2303.15343},
5 year={2023}
6}1@misc{big_vision,
2 author = {Beyer, Lucas and Zhai, Xiaohua and Kolesnikov, Alexander},
3 title = {Big Vision},
4 year = {2022},
5 publisher = {GitHub},
6 journal = {GitHub repository},
7 howpublished = {\url{https://github.com/google-research/big_vision}}
8}