This is a quantized version of
openai/whisper-small optimized for mobile deployment with ONNX Runtime.
This hybrid approach ensures compatibility with ONNX Runtime Mobile, which does not support ConvInteger operations.
1import onnxruntime as ort
2import numpy as np
3from transformers import WhisperProcessor
4
5# Load processor
6processor = WhisperProcessor.from_pretrained("R321-hf/whisper-small-hybrid-int8")
7
8# Load models
9encoder_session = ort.InferenceSession("encoder_model.onnx")
10decoder_session = ort.InferenceSession("decoder_model.onnx")
11
12# Prepare input (mel spectrogram)
13mel_spectrogram = np.random.randn(1, 80, 3000).astype(np.float32)
14
15# Run encoder
16encoder_output = encoder_session.run(None, {"input_features": mel_spectrogram})
17
18# Run decoder with initial tokens
19decoder_input_ids = np.array([[50258]]) # BOS token
20logits = decoder_session.run(None, {
21 "decoder_input_ids": decoder_input_ids,
22 "encoder_hidden_states": encoder_output[0]
23})
1// Load encoder
2val encoderSession = ortEnvironment.createSession("encoder_model.onnx", sessionOptions)
3
4// Load decoder
5val decoderSession = ortEnvironment.createSession("decoder_model.onnx", sessionOptions)
6
7// Run inference
8val melTensor = OnnxTensor.createTensor(ortEnvironment, melSpectrogram, longArrayOf(1, 80, 3000))
9val encoderOutput = encoderSession.run(mapOf("input_features" to melTensor))
