Views
No views yet
1import torch
2import torchaudio
3from datasets import load_dataset
4from transformers import Wav2Vec2ForCTC, Wav2Vec2Processor
5
6test_dataset = load_dataset("common_voice", "et", split="test[:2%]") #TODO: replace {lang_id} in your language code here. Make sure the code is one of the *ISO codes* of [this](https://huggingface.co/languages) site.
7
8processor = Wav2Vec2Processor.from_pretrained("vasilis/wav2vec2-large-xlsr-53-Estonian") #TODO: replace {model_id} with your model id. The model id consists of {your_username}/{your_modelname}, *e.g.* `elgeish/wav2vec2-large-xlsr-53-arabic`
9model = Wav2Vec2ForCTC.from_pretrained("vasilis/wav2vec2-large-xlsr-53-Estonian") #TODO: replace {model_id} with your model id. The model id consists of {your_username}/{your_modelname}, *e.g.* `elgeish/wav2vec2-large-xlsr-53-arabic`
10
11resampler = torchaudio.transforms.Resample(48_000, 16_000)
12
13# Preprocessing the datasets.
14# We need to read the aduio files as arrays
15def speech_file_to_array_fn(batch):
16 speech_array, sampling_rate = torchaudio.load(batch["path"])
17 batch["speech"] = resampler(speech_array).squeeze().numpy()
18 return batch
19
20test_dataset = test_dataset.map(speech_file_to_array_fn)
21inputs = processor(test_dataset["speech"][:2], sampling_rate=16_000, return_tensors="pt", padding=True)
22
23with torch.no_grad():
24 logits = model(inputs.input_values, attention_mask=inputs.attention_mask).logits
25
26predicted_ids = torch.argmax(logits, dim=-1)
27
28print("Prediction:", processor.batch_decode(predicted_ids))
29print("Reference:", test_dataset["sentence"][:2])1import torch
2import torchaudio
3from datasets import load_dataset, load_metric
4from transformers import Wav2Vec2ForCTC, Wav2Vec2Processor
5import re
6
7test_dataset = load_dataset("common_voice", "et", split="test")
8wer = load_metric("wer")
9
10processor = Wav2Vec2Processor.from_pretrained("vasilis/wav2vec2-large-xlsr-53-Estonian")
11model = Wav2Vec2ForCTC.from_pretrained("vasilis/wav2vec2-large-xlsr-53-Estonian")
12model.to("cuda")
13
14chars_to_ignore_regex = "[\,\?\.\!\-\;\:\"\“\%\‘\”\�\']" # TODO: adapt this list to include all special characters you removed from the data
15
16resampler = {
17 48_000: torchaudio.transforms.Resample(48_000, 16_000),
18 44100: torchaudio.transforms.Resample(44100, 16_000),
19 32000: torchaudio.transforms.Resample(32000, 16_000)
20}
21
22# Preprocessing the datasets.
23# We need to read the aduio files as arrays
24def speech_file_to_array_fn(batch):
25 batch["sentence"] = re.sub(chars_to_ignore_regex, '', batch["sentence"]).lower()
26 speech_array, sampling_rate = torchaudio.load(batch["path"])
27 batch["speech"] = resampler[sampling_rate](speech_array).squeeze().numpy()
28 return batch
29
30test_dataset = test_dataset.map(speech_file_to_array_fn)
31
32# Preprocessing the datasets.
33# We need to read the aduio files as arrays
34def evaluate(batch):
35 inputs = processor(batch["speech"], sampling_rate=16_000, return_tensors="pt", padding=True)
36 with torch.no_grad():
37 logits = model(inputs.input_values.to("cuda"), attention_mask=inputs.attention_mask.to("cuda")).logits
38 pred_ids = torch.argmax(logits, dim=-1)
39 batch["pred_strings"] = processor.batch_decode(pred_ids)
40 return batch
41
42result = test_dataset.map(evaluate, batched=True, batch_size=8)
43
44print("WER: {:2f}".format(100 * wer.compute(predictions=result["pred_strings"], references=result["sentence"])))
45print("CER: {:2f}".format(100 * wer.compute(predictions=[" ".join(list(entry)) for entry in result["pred_strings"]], references=[" ".join(list(entry)) for entry in result["sentence"]])))
46train and validation sets were used for finetuning
for 20000 steps (approx. 116 epochs). Both the feature extractor (Wav2Vec2FeatureExtractor) and
feature projection (Wav2Vec2FeatureProjection) layer were frozen. Only the encoder layer (Wav2Vec2EncoderStableLayerNorm) was finetuned.