This model is an ONNX Runtime-optimized version of
openai/whisper-large-v3-turbo ,
quantized to INT8 using dynamic quantization for improved inference performance.
class QuantizedONNXWhisperModel():
"""
Quantized Whisper v3 Turbo model using ONNX Runtime (via Optimum).
"""
def __init__(self):
super().__init__()
self.model = None
self.processor = None
self.pipe = None
self.model_name = "Vmpletsos/whisper-large-v3-turbo-onnx-int8"
self.device = "cpu"
self._load_model()
self._is_initialized = True
logger.info("ONNX Whisper model ready")
def _load_model(self):
"""Load the ONNX model and processor."""
logger.info(f"Loading ONNX model: {self.model_name} on {self.device}...")
# Load the ONNX Model
self.model = ORTModelForSpeechSeq2Seq.from_pretrained(
self.model_name,
provider="CPUExecutionProvider",
encoder_file_name="encoder_model_quantized.onnx",
decoder_file_name="decoder_model_quantized.onnx",
decoder_with_past_file_name="decoder_with_past_model_quantized.onnx",
use_io_binding=False, # Disable IO binding which can cause issues
)
# Load the Processor from the actual base model
self.processor = AutoProcessor.from_pretrained("openai/whisper-large-v3-turbo")
self.pipe = pipeline(
"automatic-speech-recognition",
model=self.model,
tokenizer=self.processor.tokenizer,
feature_extractor=self.processor.feature_extractor,
chunk_length_s=30,
batch_size=16,
return_timestamps=True,
device=-1,
)
logger.info(f"Model loaded successfully.")
async def transcribe(self, filepath: Path, **kwargs) -> TranscriptionResult:
"""
Transcribe audio file using ONNX Runtime pipeline.
"""
# Run transcription
result = self.pipe(
str(filepath),
generate_kwargs={"task": "transcribe"},
)
full_text = result["text"]
if "chunks" in result:
for chunk in result["chunks"]:
start, end = chunk.get("timestamp", (0.0, 0.0))
text = chunk.get("text", "")
logger.debug(f"[{start}s -> {end}s] {text}")
return full_text
)