Input Image (384×384)
↓
┌─────────────────────────────┐
│ SigLIP 2 So400m │ ← Vision Encoder (Frozen)
│ Output: [B, 729, 1152] │
└─────────────────────────────┘
↓
┌─────────────────────────────┐
│ 2-Layer MLP Projector │ ← Trained to align modalities
│ 1152 → 640 → 640 │
└─────────────────────────────┘
↓
┌─────────────────────────────┐
│ Gemma 3 270M-IT │ ← Language Model
│ 18 layers, 640 hidden │
└─────────────────────────────┘
↓
Output Text1 import torch
2 from PIL import Image
3 from transformers import AutoTokenizer , AutoProcessor
4
5 # Import custom model classes
6 from modeling_fx_nanovlm import FxNanoVLMForConditionalGeneration
7 from configuration_fx_nanovlm import FxNanoVLMConfig
8
9 # Load model and tokenizer
10 model = FxNanoVLMForConditionalGeneration . from_pretrained (
11 "Rohihtcept/fx-nanovlm" ,
12 torch_dtype = torch . bfloat16 ,
13 device_map = "auto" ,
14 trust_remote_code = True ,
15 )
16
17 tokenizer = AutoTokenizer . from_pretrained (
18 "Rohithcept/fx-nanovlm" ,
19 trust_remote_code = True ,
20 )
21
22 processor = AutoProcessor . from_pretrained (
23 "google/siglip2-so400m-patch14-384"
24 )
25
26 # Load and process image
27 image = Image . open ( "document.png" ) . convert ( "RGB" )
28 pixel_values = processor ( images = image , return_tensors = "pt" ) . pixel_values
29 pixel_values = pixel_values . to ( model . device , dtype = torch . bfloat16 )
30
31 # Create prompt
32 prompt = "<image>Extract the text from this image."
33 inputs = tokenizer ( prompt , return_tensors = "pt" ) . to ( model . device )
34
35 # Generate
36 with torch . no_grad ( ) :
37 outputs = model . generate (
38 input_ids = inputs [ "input_ids" ] ,
39 attention_mask = inputs [ "attention_mask" ] ,
40 pixel_values = pixel_values ,
41 max_new_tokens = 512 ,
42 do_sample = False ,
43 )
44
45 # Decode output
46 response = tokenizer . decode ( outputs [ 0 ] , skip_special_tokens = True )
47 print ( response )
1 # Process multiple images
2 images = [ Image . open ( f"doc_ { i } .png" ) . convert ( "RGB" ) for i in range ( 3 ) ]
3 pixel_values = processor ( images = images , return_tensors = "pt" ) . pixel_values
4
5 prompts = [ "<image>Extract the text." ] * len ( images )
6 inputs = tokenizer ( prompts , return_tensors = "pt" , padding = True )
7
8 outputs = model . generate (
9 input_ids = inputs [ "input_ids" ] . to ( model . device ) ,
10 attention_mask = inputs [ "attention_mask" ] . to ( model . device ) ,
11 pixel_values = pixel_values . to ( model . device , dtype = torch . bfloat16 ) ,
12 max_new_tokens = 512 ,
13 )
The projector was trained to align SigLIP 2 visual features with Gemma 3 text embeddings:
1 from peft import LoraConfig , get_peft_model
2
3 lora_config = LoraConfig (
4 r = 16 ,
5 lora_alpha = 32 ,
6 target_modules = [ "q_proj" , "k_proj" , "v_proj" , "o_proj" ] ,
7 lora_dropout = 0.05 ,
8 task_type = "CAUSAL_LM" ,
9 )
10
11 model = get_peft_model ( model . language_model , lora_config )
fx_nanovlm/
├── README.md # This file (model card)
├── config.json # Model configuration
├── model.safetensors # Model weights (~2.8GB)
├── configuration_fx_nanovlm.py # Custom config class
├── modeling_fx_nanovlm.py # Custom model class
├── tokenizer.json # Tokenizer
├── tokenizer.model # SentencePiece model
├── tokenizer_config.json # Tokenizer config
├── special_tokens_map.json # Special tokens
└── added_tokens.json # Added tokens (<image>)1 {
2 "model_type" : "fx_nanovlm" ,
3 "vision_config" : {
4 "model_type" : "siglip_vision_model" ,
5 "hidden_size" : 1152 ,
6 "image_size" : 384 ,
7 "patch_size" : 14 ,
8 "num_attention_heads" : 16 ,
9 "num_hidden_layers" : 27
10 } ,
11 "text_config" : {
12 "model_type" : "gemma" ,
13 "hidden_size" : 640 ,
14 "num_hidden_layers" : 18 ,
15 "vocab_size" : 262146
16 } ,
17 "projector_hidden_act" : "gelu" ,
18 "image_token_index" : 262145 ,
19 "num_image_tokens" : 729
20 }
1 @misc{fx-nanovlm,
2 author = Rohith Reddy,
3 title = {fx-nanovlm: A Nano Vision-Language Model for OCR},
4 year = {2025},
5 publisher = {HuggingFace},
6 url = {https://huggingface.co/Rohithcept/fx-nanovlm}
7 }
This model is released under the
Apache 2.0 License .
