Views
No views yet
encoder.onnx - Image encoder (ONNX format)encoder.with_runtime_opt.ort - Image encoder (optimized for WebGPU)decoder.onnx - Mask decoder (ONNX format)config.json - Model configuration1import * as ort from 'onnxruntime-web/webgpu';
2
3// Load encoder (use optimized .ort version for WebGPU)
4const encoderURL = 'https://huggingface.co/SharpAI/sam2-hiera-tiny-onnx/resolve/main/encoder.with_runtime_opt.ort';
5const encoderSession = await ort.InferenceSession.create(encoderURL, {
6 executionProviders: ['webgpu'],
7 graphOptimizationLevel: 'disabled'
8});
9
10// Load decoder
11const decoderURL = 'https://huggingface.co/SharpAI/sam2-hiera-tiny-onnx/resolve/main/decoder.onnx';
12const decoderSession = await ort.InferenceSession.create(decoderURL, {
13 executionProviders: ['webgpu']
14});
15
16// Run encoder
17const imageData = preprocessImage(image); // Your preprocessing
18const encoderOutputs = await encoderSession.run({ image: imageData });
19
20// Run decoder with point
21const point_coords = new ort.Tensor('float32', [x, y, 0, 0], [1, 2, 2]);
22const point_labels = new ort.Tensor('float32', [1, -1], [1, 2]);
23const mask_input = new ort.Tensor('float32', new Float32Array(256 * 256).fill(0), [1, 1, 256, 256]);
24const has_mask_input = new ort.Tensor('float32', [0], [1]);
25
26const decoderOutputs = await decoderSession.run({
27 image_embed: encoderOutputs.image_embed,
28 high_res_feats_0: encoderOutputs.high_res_feats_0,
29 high_res_feats_1: encoderOutputs.high_res_feats_1,
30 point_coords: point_coords,
31 point_labels: point_labels,
32 mask_input: mask_input,
33 has_mask_input: has_mask_input
34});
35
36// Get masks
37const masks = decoderOutputs.masks; // Shape: [1, num_masks, 256, 256]1import onnxruntime as ort
2import numpy as np
3
4# Load models
5encoder_session = ort.InferenceSession("encoder.onnx")
6decoder_session = ort.InferenceSession("decoder.onnx")
7
8# Run encoder
9encoder_outputs = encoder_session.run(None, {"image": image_tensor})
10
11# Run decoder
12decoder_outputs = decoder_session.run(None, {
13 "image_embed": encoder_outputs[0],
14 "high_res_feats_0": encoder_outputs[1],
15 "high_res_feats_1": encoder_outputs[2],
16 "point_coords": point_coords,
17 "point_labels": point_labels,
18 "mask_input": mask_input,
19 "has_mask_input": has_mask_input
20})
21
22masks = decoder_outputs[0]image: Float32[1, 3, 1024, 1024] - Normalized RGB imageimage_embed: Float32[1, 256, 64, 64] - Image embeddingshigh_res_feats_0: Float32[1, 32, 256, 256] - High-res features (level 0)high_res_feats_1: Float32[1, 64, 128, 128] - High-res features (level 1)image_embed: Float32[1, 256, 64, 64] - From encoderhigh_res_feats_0: Float32[1, 32, 256, 256] - From encoderhigh_res_feats_1: Float32[1, 64, 128, 128] - From encoderpoint_coords: Float32[1, 2, 2] - Point coordinates [[x, y], [0, 0]]point_labels: Float32[1, 2] - Point labels [1, -1] (1=foreground, -1=padding)mask_input: Float32[1, 1, 256, 256] - Previous mask (zeros if none)has_mask_input: Float32[1] - Flag [0] or [1]masks: Float32[1, 3, 256, 256] - Generated masks (3 candidates)iou_predictions: Float32[1, 3] - IoU scores for each masklow_res_masks: Float32[1, 3, 256, 256] - Low-resolution maskschrome://flags/#enable-unsafe-webgpu)1@article{ravi2024sam2,
2 title={SAM 2: Segment Anything in Images and Videos},
3 author={Ravi, Nikhila and Gabeur, Valentin and Hu, Yuan-Ting and Hu, Ronghang and Ryali, Chaitanya and Ma, Tengyu and Khedr, Haitham and R{\"a}dle, Roman and Rolland, Chloe and Gustafson, Laura and Mintun, Eric and Pan, Junting and Alwala, Kalyan Vasudev and Carion, Nicolas and Wu, Chao-Yuan and Girshick, Ross and Doll{\'a}r, Piotr and Feichtenhofer, Christoph},
4 journal={arXiv preprint arXiv:2408.00714},
5 year={2024}
6}