A multilingual distilled Whisper model with 2 decoder layers, supporting 7 European languages: English, French, Spanish, German, Italian, Portuguese, and Dutch.
The model was trained during my work on
Distil-Large-v3.5.
The model's performance is below both the monolingual distilled version and Whisper-Large-v3-Turbo. Future work should investigate better training procedures and possibly incorporate more data to effectively compress multilingual capabilities into a single model.
1import torch
2from datasets import load_dataset
3from transformers import AutoModelForSpeechSeq2Seq, AutoProcessor
4
5device = "cuda:0" if torch.cuda.is_available() else "cpu"
6torch_dtype = torch.float16 if torch.cuda.is_available() else torch.float32
7
8# Load model
9model_name_or_path = "bofenghuang/whisper-large-v3-distil-multi7-v0.2"
10processor = AutoProcessor.from_pretrained(model_name_or_path)
11model = AutoModelForSpeechSeq2Seq.from_pretrained(model_name_or_path, torch_dtype=torch_dtype)
12model.to(device)
13
14# Example audio
15dataset = load_dataset("bofenghuang/asr-dummy", "cs", split="test")
16sample, text = dataset[0]["audio"], dataset[0]["text"]
17
18# Ground truth text
19print(text)
20# Aber sei ihnen nicht böse, Habibi, vergib ihnen, sie vergaßen die Liebe, sie vergaßen die Bibel,
21# wünsch ihnen den Frieden. Nous allons construire des radiotélescopes géants comme celui-ci,
22# qui est mon préféré. Questa è un'immagine di Cairo Open City, una mostra che il museo Folkwang di
23# Essen ha dedicato al ruolo della mobile photography nella primavera Araba.
24
25# Extract feautres
26input_features = processor(
27 sample["array"], sampling_rate=sample["sampling_rate"], return_tensors="pt"
28).input_features
29
30
31# Generate tokens
32predicted_ids = model.generate(
33 input_features.to(device, dtype=torch_dtype),
34 max_new_tokens=128,
35)
36
37# Detokenize to text
38transcription = processor.batch_decode(predicted_ids, skip_special_tokens=True)[0]
39print(transcription)
40# Aber sei ihnen nicht böse, Habibi, vergib ihn. Sie vergaßen die Liebe, sie vergaßen die Liebe.
41# Wünsche ihnen dem Frieden. Nous allons construire des radiotelescopes géants, comme celui-ci qui
42# est mon préféré. Esta es una imagen de Cairo Open City, una muestra que el Museo Folk Punk de Essen
43# ha dedicado al ruolo de la mobile fotografía en la primavera árabe.
44
45# Dive in generated tokens
46transcription = processor.batch_decode(predicted_ids, skip_special_tokens=False)[0]
47print(transcription)
48# <|de|> Aber sei ihnen nicht böse, Habibi, vergib ihn. Sie vergaßen die Liebe, sie vergaßen die Liebe.
49# Wünsche ihnen dem Frieden.<|fr|> Nous allons construire des radiotelescopes géants, comme celui-ci qui
50# est mon préféré.<|es|> Esta es una imagen de Cairo Open City, una muestra que el Museo Folk Punk de Essen
51# ha dedicado al ruolo de la mobile fotografía en la primavera árabe.
