┌─────────────────────────────────────────────────────────────────────┐
│ EdgeMultimodalEmbedder │
│ │
│ ┌──────────┐ ┌──────────┐ ┌──────────┐ ┌──────────┐ │
│ │ Text │ │ Vision │ │ Audio │ │ Video │ │
│ │ (Nomic) │ │ (Nomic) │ │ (CLAP) │ │(VideoMAE)│ │
│ │ 768d │ │ 768d │ │ 512d │ │ 384d │ │
│ └────┬─────┘ └────┬─────┘ └────┬─────┘ └────┬─────┘ │
│ │ │ │ │ │
│ └──────────────┴──────┬──────┴──────────────┘ │
│ │ │
│ ┌──────────────┴──────────────┐ │
│ │ Projection Heads → 512d │ │
│ └──────────────┬──────────────┘ │
│ │ │
│ L2-Normalized 512d │
└─────────────────────────────────────────────────────────────────────┘1 from edge_multimodal_embeddings import EdgeMultimodalEmbedder
2
3 # Quick start — auto-selects profile
4 embedder = EdgeMultimodalEmbedder . from_profile ( "edge_gpu" )
5
6 # Profiles:
7 # "phone" → 384d, INT4, 1.5GB budget, model swapping, MiniLM text
8 # "edge_gpu" → 512d, INT8, 3.5GB budget, all models loaded
9 # "workstation" → 768d, FP16, 8GB budget, no quantization
10 # "raspberry_pi" → 256d, INT4, 1GB budget, model swapping, no audio
11 # "jetson" → 512d, FP16, 3GB budget
1 pip install torch transformers Pillow numpy
2 # Optional: pip install onnxruntime onnx librosa soundfile opencv-python pymupdf
1 from edge_multimodal_embeddings import EdgeMultimodalEmbedder
2
3 embedder = EdgeMultimodalEmbedder . from_profile ( "edge_gpu" )
4
5 # Text
6 text_emb = embedder . embed ( "The quick brown fox" )
7
8 # Image (auto-detected from file extension)
9 img_emb = embedder . embed ( "photo.jpg" )
10
11 # Video
12 vid_emb = embedder . embed ( "clip.mp4" )
13
14 # Audio
15 audio_emb = embedder . embed ( "song.wav" )
16
17 # PDF (extracts text + images, embeds both)
18 pdf_emb = embedder . embed ( "paper.pdf" )
19
20 # Code
21 code_emb = embedder . embed ( "main.py" , modality = "document" )
22
23 # Cross-modal similarity
24 similarity = embedder . similarity ( text_emb , img_emb )
25 print ( f"Text-Image similarity: { similarity : .4f } " )
1 # Mixed modality batch
2 inputs = [ "Hello world" , "photo.jpg" , "clip.mp4" ]
3 embeddings = embedder . embed_batch ( inputs )
4 # → shape: (3, 512)
1 # Step 1: Export on workstation
2 embedder = EdgeMultimodalEmbedder . from_profile ( "phone" )
3 embedder . export_onnx ( "./phone_models" )
4
5 # Step 2: Run on phone (only needs onnxruntime + numpy)
6 from edge_multimodal_embeddings . runtime import EdgeRuntime
7
8 runtime = EdgeRuntime ( "./phone_models" , num_threads = 4 )
9 emb = runtime . embed_text ( "Find photos of cats" )
10 img_emb = runtime . embed_image ( "cat.jpg" )
11 sim = runtime . similarity ( emb , img_emb )
1 from edge_multimodal_embeddings . core . config import (
2 EmbedderConfig , ModelConfig , QuantizationMode
3 )
4
5 config = EmbedderConfig (
6 unified_dim = 512 ,
7 device = "cuda" ,
8 quantization = QuantizationMode . INT8_DYNAMIC ,
9 max_memory_mb = 3500 ,
10 lazy_load = True ,
11 model_swapping = False ,
12 max_batch_size = 16 ,
13
14 # Swap in different models
15 text_model = ModelConfig (
16 model_id = "sentence-transformers/all-MiniLM-L6-v2" ,
17 embedding_dim = 384 ,
18 max_input_size = 512 ,
19 ) ,
20 # Disable modalities you don't need
21 audio_model = ModelConfig ( "laion/clap-htsat-unfused" , 512 , enabled = False ) ,
22 video_model = ModelConfig ( "MCG-NJU/videomae-small-finetuned-kinetics" , 384 , enabled = False ) ,
23 )
24
25 embedder = EdgeMultimodalEmbedder ( config )
Plus: PIL Images, numpy arrays, torch tensors, base64 strings, URLs, raw bytes.
1 // build.gradle
2 implementation 'com . microsoft . onnxruntime : onnxruntime - mobile : 1.17 . 0 '
3
4 // Load model
5 val session = env . createSession ( modelBytes , SessionOptions ( ) . apply {
6 setIntraOpNumThreads ( 4 )
7 addNnapi ( ) // Use Android Neural Networks API
8 } )
1 // Use ONNX → CoreML conversion or CoreML EP
2 let config = MLModelConfiguration ( )
3 config . computeUnits = . all // CPU + GPU + ANE
1 pip install onnxruntime # ARM64 wheel available
2 python -c "from edge_multimodal_embeddings.runtime import EdgeRuntime; ..."
1 # Embed
2 edge-embed embed "Hello world"
3 edge-embed embed photo.jpg
4 edge-embed embed --modality video clip.mp4
5
6 # Compare
7 edge-embed compare "a cat" "a kitten"
8
9 # Benchmark
10 edge-embed benchmark -n 100
11
12 # Export
13 edge-embed export --format onnx -o ./exported/
14
15 # System info
16 edge-embed info
Apache-2.0. All component models are Apache-2.0 or MIT licensed.
