Views
No views yet
pip install transformers accelerate pillow1from PIL import Image
2import torch
3from transformers import AutoProcessor, AutoModelForVision2Seq
4
5# Load model and processor
6processor = AutoProcessor.from_pretrained("racineai/Flantier-SmolVLM-500M-dse")
7model = AutoModelForVision2Seq.from_pretrained(
8 "racineai/Flantier-SmolVLM-500M-dse",
9 torch_dtype=torch.bfloat16,
10 device_map="auto"
11)
12
13# Load document image
14document_image = Image.open("technical_document.jpg")
15
16# Process for document embedding
17doc_messages = [
18 {
19 "role": "user",
20 "content": [
21 {"type": "image"},
22 {"type": "text", "text": "What is shown in this image?"}
23 ]
24 },
25]
26doc_prompt = processor.apply_chat_template(doc_messages, add_generation_prompt=True)
27doc_inputs = processor(text=doc_prompt, images=[document_image], return_tensors="pt").to(model.device)
28
29# Generate document embedding
30with torch.no_grad():
31 doc_outputs = model(**doc_inputs, output_hidden_states=True, return_dict=True)
32 doc_embedding = doc_outputs.hidden_states[-1][:, -1] # Last token embedding
33 doc_embedding = torch.nn.functional.normalize(doc_embedding, p=2, dim=-1)
34
35# Process query embedding
36query = "What are the specifications of this component?"
37query_messages = [
38 {
39 "role": "user",
40 "content": [
41 {"type": "text", "text": query}
42 ]
43 },
44]
45query_prompt = processor.apply_chat_template(query_messages, add_generation_prompt=True)
46query_inputs = processor(text=query_prompt, return_tensors="pt").to(model.device)
47
48# Generate query embedding
49with torch.no_grad():
50 query_outputs = model(**query_inputs, output_hidden_states=True, return_dict=True)
51 query_embedding = query_outputs.hidden_states[-1][:, -1] # Last token embedding
52 query_embedding = torch.nn.functional.normalize(query_embedding, p=2, dim=-1)
53
54# Calculate similarity
55similarity = torch.nn.functional.cosine_similarity(query_embedding, doc_embedding)
56print(f"Similarity score: {similarity.item():.4f}")@misc{flantier-smolvlm-dse,
author = {racine.ai},
title = {Flantier-SmolVLM-500M-dse: A Lightweight Document Screenshot Embedding Model},
year = {2025},
publisher = {Hugging Face},
url = {https://huggingface.co/racineai/Flantier-SmolVLM-500M-dse}
}