Views
No views yet
SupraLabs/SupraCMA-8M and distilled using embeddings generated by SigLIP-2.vlm_model.onnx: Optimized FP32 ONNX model compatible with CPU/WASM onnxruntime-web execution providers.vlm_model_fp16.onnx: Optimized FP16 ONNX model for WebGPU/WebGL rendering acceleration.model.safetensors: Standard PyTorch model state dictionary (SafeTensors format).modeling_vlm.py: Custom Python wrapper code for the EncoderFreeVLM module and VLMPreprocessor.config.json: Hardware parameter settings.modeling_vlm.py:1import torch
2from modeling_vlm import EncoderFreeVLM, VLMPreprocessor
3from transformers import AutoTokenizer, AutoModelForCausalLM
4
5# 1. Load base components
6tokenizer = AutoTokenizer.from_pretrained("SupraLabs/SupraCMA-8M", trust_remote_code=True)
7cma_model = AutoModelForCausalLM.from_pretrained("SupraLabs/SupraCMA-8M", trust_remote_code=True)
8
9# 2. Instantiate wrapper
10preprocessor = VLMPreprocessor(tokenizer)
11vlm = EncoderFreeVLM(cma_model, embedding_dim=768, proj_mode="linear")
12
13# 3. Load model state dictionary and automatically reconstruct shared weights
14from safetensors.torch import load_model
15load_model(vlm, "model.safetensors")
16vlm.eval()
17
18# Example forward execution
19# inputs = preprocessor(image=your_image_object)
20# embeddings = vlm(**inputs)input_ids [int64, [batch_size, sequence_length]]attention_mask [int64, [batch_size, sequence_length]]images [float32, [batch_size, 3, 224, 224]]is_image [float32, [batch_size, 1]][0, 255] pixel values.1<!-- Load the latest ONNX Runtime Web CDN -->
2<script src="https://cdn.jsdelivr.net/npm/onnxruntime-web/dist/ort.min.js"></script><canvas> element and structures it into the expected flat, channel-first (NCHW) format:1/**
2 * Preprocesses a 224x224 canvas to a raw NCHW float32 array in the [0, 255] range.
3 * @param {HTMLCanvasElement} canvas - Canvas element resized to 224x224.
4 * @returns {ort.Tensor} ONNX Tensor of shape [1, 3, 224, 224]
5 */
6function preprocessCanvas(canvas) {
7 const ctx = canvas.getContext('2d');
8 const imgData = ctx.getImageData(0, 0, 224, 224).data;
9
10 const floatData = new Float32Array(3 * 224 * 224);
11 const numPixels = 224 * 224;
12
13 for (let i = 0; i < numPixels; i++) {
14 floatData[i] = imgData[i * 4]; // Red channel
15 floatData[i + numPixels] = imgData[i * 4 + 1]; // Green channel
16 floatData[i + 2 * numPixels] = imgData[i * 4 + 2]; // Blue channel (Alpha is ignored)
17 }
18
19 return new ort.Tensor('float32', floatData, [1, 3, 224, 224]);
20}1// Initialize the ONNX session
2const session = await ort.InferenceSession.create('./vlm_model_fp16.onnx', {
3 executionProviders: ['webgpu', 'wasm'] // Falls back to WASM if WebGPU is unavailable
4});
5
6/**
7 * Generate a 768-dimensional embedding vector for an image
8 */
9async function getImageEmbedding(canvas) {
10 const imagesTensor = preprocessCanvas(canvas);
11
12 // Provide minimal dummy values for the text inputs
13 const inputIdsTensor = new ort.Tensor('int64', new BigInt64Array([0n]), [1, 1]);
14 const attentionMaskTensor = new ort.Tensor('int64', new BigInt64Array([1n]), [1, 1]);
15 const isImageTensor = new ort.Tensor('float32', new Float32Array([1.0]), [1, 1]); // 1.0 flags image path
16
17 const feeds = {
18 input_ids: inputIdsTensor,
19 attention_mask: attentionMaskTensor,
20 images: imagesTensor,
21 is_image: isImageTensor
22 };
23
24 const results = await session.run(feeds);
25 return results.embeddings.data; // Float32Array [768]
26}
27
28/**
29 * Generate a 768-dimensional embedding vector for text
30 * @param {Array<number>} tokenIds - Tokenized integer IDs (generated by your JS tokenizer)
31 * @param {Array<number>} attentionMask - Token attention mask (usually all 1s)
32 */
33async function getTextEmbedding(tokenIds, attentionMask) {
34 const seqLen = tokenIds.length;
35
36 const inputIdsTensor = new ort.Tensor('int64', new BigInt64Array(tokenIds.map(BigInt)), [1, seqLen]);
37 const attentionMaskTensor = new ort.Tensor('int64', new BigInt64Array(attentionMask.map(BigInt)), [1, seqLen]);
38
39 // Provide a zeroed-out dummy tensor for the image inputs
40 const imagesTensor = new ort.Tensor('float32', new Float32Array(3 * 224 * 224), [1, 3, 224, 224]);
41 const isImageTensor = new ort.Tensor('float32', new Float32Array([0.0]), [1, 1]); // 0.0 flags text path
42
43 const feeds = {
44 input_ids: inputIdsTensor,
45 attention_mask: attentionMaskTensor,
46 images: imagesTensor,
47 is_image: isImageTensor
48 };
49
50 const results = await session.run(feeds);
51 return results.embeddings.data; // Float32Array [768]
52}