This is
whisper-large-v3 model converted to the
OpenVINO™ IR (Intermediate Representation) format with weights compressed to FP16.
1#!/usr/bin/env python3
2import time
3import requests
4import openvino_genai
5import librosa
6from pathlib import Path
7from huggingface_hub import snapshot_download
8
9
10def download_model(model_id="FluidInference/whisper-large-v3-turbo-int4-ov-npu"):
11 """Download model from HuggingFace Hub"""
12 local_dir = Path("models") / model_id.split("/")[-1]
13
14 if local_dir.exists() and any(local_dir.iterdir()):
15 return str(local_dir)
16
17 print(f"Downloading model...")
18 snapshot_download(
19 repo_id=model_id,
20 local_dir=str(local_dir),
21 local_dir_use_symlinks=False
22 )
23 return str(local_dir)
24
25
26def download_hf_audio_samples():
27 """Download audio samples from Hugging Face"""
28 samples_dir = Path("sample_audios")
29 samples_dir.mkdir(exist_ok=True)
30
31 downloaded = []
32 whisper_samples = [
33 ("https://cdn-media.huggingface.co/speech_samples/sample1.flac", "sample1.flac"),
34 ("https://cdn-media.huggingface.co/speech_samples/sample2.flac", "sample2.flac"),
35 ]
36
37 for url, filename in whisper_samples:
38 filepath = samples_dir / filename
39 if filepath.exists():
40 downloaded.append(str(filepath))
41 continue
42
43 try:
44 response = requests.get(url, headers={"User-Agent": "Mozilla/5.0"})
45 response.raise_for_status()
46
47 with open(filepath, 'wb') as f:
48 f.write(response.content)
49
50 downloaded.append(str(filepath))
51 except Exception as e:
52 print(f"Error downloading {filename}: {e}")
53
54 return downloaded
55
56
57def read_audio(filepath):
58 """Read audio file and convert to 16kHz"""
59 try:
60 raw_speech, _ = librosa.load(filepath, sr=16000)
61 return raw_speech.tolist()
62 except Exception as e:
63 print(f"Error reading {filepath}: {e}")
64 return None
65
66
67def test_whisper_on_file(pipe, filepath):
68 """Test Whisper on a single audio file"""
69 config = pipe.get_generation_config()
70 config.language = "<|en|>"
71 config.task = "transcribe"
72 config.return_timestamps = True
73 config.max_new_tokens = 448
74
75 raw_speech = read_audio(filepath)
76 if raw_speech is None:
77 return None
78
79 duration = len(raw_speech) / 16000
80
81 start_time = time.time()
82 result = pipe.generate(raw_speech, config)
83 inference_time = time.time() - start_time
84
85 return {
86 "file": filepath,
87 "duration": duration,
88 "inference_time": inference_time,
89 "rtf": inference_time/duration,
90 "transcription": str(result)
91 }
92
93
94def main():
95 # Download model
96 model_path = download_model()
97
98 # Initialize pipeline on NPU
99 print(f"\nInitializing NPU...")
100 start_time = time.time()
101 pipe = openvino_genai.WhisperPipeline(model_path, "NPU")
102 init_time = time.time() - start_time
103
104 results = []
105
106 # Collect test files
107 test_files = []
108 test_files.extend(Path(".").glob("*.wav"))
109
110 if Path("samples/c/whisper_speech_recognition").exists():
111 test_files.extend(Path("samples/c/whisper_speech_recognition").glob("*.wav"))
112
113 # Download HF samples
114 hf_samples = download_hf_audio_samples()
115 test_files.extend([Path(f) for f in hf_samples])
116
117 # Test all files
118 print(f"\nTesting {len(test_files)} files...")
119 for audio_file in test_files:
120 result = test_whisper_on_file(pipe, str(audio_file))
121 if result:
122 results.append(result)
123 print(f"[OK] {Path(result['file']).name}: RTF={result['rtf']:.2f}x")
124
125 # Print summary
126 if results:
127 total_duration = sum(r["duration"] for r in results)
128 total_inference = sum(r["inference_time"] for r in results)
129 avg_rtf = total_inference / total_duration
130
131 print(f"\n{'='*50}")
132 print(f"NPU Performance Summary")
133 print(f"{'='*50}")
134 print(f"Model load time: {init_time:.1f}s")
135 print(f"Files tested: {len(results)}")
136 print(f"Total audio: {total_duration:.1f}s")
137 print(f"Total inference: {total_inference:.1f}s")
138 print(f"Average RTF: {avg_rtf:.2f}x {'[Faster than real-time]' if avg_rtf < 1 else '[Slower than real-time]'}")
139
140 print(f"\nResults:")
141 for r in results:
142 trans = r['transcription'].strip()
143 if len(trans) > 60:
144 trans = trans[:57] + "..."
145 print(f"- {Path(r['file']).name}: \"{trans}\"")
146
147
148if __name__ == "__main__":
149 main()
