Views
No views yet
| Model | Text Encoder | Vision Encoder | Params | Projection Dim |
|---|---|---|---|---|
| LongCLIP-B | 12 layers, 512d | 12 layers, 768d | ~150M | 512 |
| LongCLIP-L | 12 layers, 768d | 24 layers, 1024d | ~430M | 768 |
pip install "transformers[torch,torch-vision]"1from transformers import AutoModel, AutoProcessor
2from PIL import Image
3import torch
4
5# Load model and processor
6model = AutoModel.from_pretrained(
7 "creative-graphic-design/LongCLIP-B",
8 trust_remote_code=True
9)
10processor = AutoProcessor.from_pretrained(
11 "creative-graphic-design/LongCLIP-B",
12 trust_remote_code=True
13)
14
15# Prepare inputs
16image = Image.open("your_image.jpg")
17texts = [
18 "A man is crossing the street with a red car parked nearby.",
19 "A man is driving a car in an urban scene."
20]
21
22inputs = processor(
23 text=texts,
24 images=image,
25 return_tensors="pt",
26 max_length=248,
27 padding="max_length"
28)
29
30# Get predictions
31with torch.no_grad():
32 outputs = model(**inputs)
33 logits_per_image = outputs.logits_per_image
34 probs = logits_per_image.softmax(dim=-1)
35
36print("Probabilities:", probs)1# Extract features separately (unnormalized)
2text_inputs = processor(text=texts, return_tensors="pt", max_length=248, padding="max_length")
3image_inputs = processor(images=image, return_tensors="pt")
4
5with torch.no_grad():
6 text_features = model.get_text_features(**text_inputs)
7 image_features = model.get_image_features(**image_inputs)
8
9 # Compute similarity (like original CLIP)
10 logits = image_features @ text_features.T
11 probs = logits.softmax(dim=-1)1# Original CLIP: max 77 tokens
2clip_text = "A cat"
3
4# LongCLIP: up to 248 tokens
5longclip_text = "A fluffy orange tabby cat with green eyes is sitting on a wooden table near a window, with sunlight streaming through the curtains in the background, creating a warm and cozy atmosphere in a modern living room."
6
7# LongCLIP can handle both short and long texts effectively!1@inproceedings{zhang2024longclip,
2 title={Long-CLIP: Unlocking the Long-Text Capability of CLIP},
3 author={Zhang, Beichen and Zhang, Pan and Dong, Xiaoyi and Zang, Yuhang and Wang, Jiaqi},
4 booktitle={European Conference on Computer Vision (ECCV)},
5 year={2024}
6}