Views
No views yet
facebook/wav2vec2-large-xlsr-53| Category | Phonemes |
|---|---|
| Vowels | ə, ɑː, i, iː, u, uː, eː, oː, aːi, aːu |
| Plosives | p, pʰ, b, bʰ, t̪, t̪ʰ, d̪, d̪ʰ, ʈ, ʈʰ, ɖ, ɖʰ, k, kʰ, g, gʰ, q |
| Affricates | c, cʰ, ɟ, ɟʰ, ɕc |
| Fricatives | s, z, ɕ, ʂ, h, ɦ, f, x, ɣ |
| Nasals | m, n, ɲ, ɳ, ŋ, ⁿ |
| Liquids & Glides | l, r, ɾ, ɽ, ɽʱ, j, v |
| Clusters | kʃ, t̪ɾ, gj |
| Syllabic | l̩, l̩ː, ɹ̩, ɹ̩ː |
| Special | <pad> (CTC blank), <unk>, | (word delimiter) |
1from transformers import Wav2Vec2ForCTC, Wav2Vec2Processor
2import torch
3import torchaudio
4
5# Load model and processor
6model_name = "xnpx/wav2vec2-large-xlsr-ipa-phonemes"
7processor = Wav2Vec2Processor.from_pretrained(model_name)
8model = Wav2Vec2ForCTC.from_pretrained(model_name)
9model.eval()
10
11# Load audio (must be 16kHz mono)
12waveform, sample_rate = torchaudio.load("audio.wav")
13if sample_rate != 16000:
14 waveform = torchaudio.transforms.Resample(sample_rate, 16000)(waveform)
15waveform = waveform.squeeze()
16
17# Run inference
18inputs = processor(waveform.numpy(), sampling_rate=16000, return_tensors="pt", padding=True)
19with torch.no_grad():
20 logits = model(inputs.input_values).logits
21
22# Greedy CTC decode
23pred_ids = torch.argmax(logits, dim=-1)
24transcription = processor.batch_decode(pred_ids)[0]
25print(transcription)
26# Example output: "n ə m ə s t̪ eː"1import numpy as np
2
3log_probs = torch.nn.functional.log_softmax(logits, dim=-1).cpu().numpy()[0]
4pred_ids = np.argmax(log_probs, axis=-1)
5
6# Load vocab for ID -> phoneme mapping
7import json
8vocab = json.loads(processor.tokenizer.backend_tokenizer.to_str()) if hasattr(processor.tokenizer, 'backend_tokenizer') else processor.tokenizer.get_vocab()
9id_to_phoneme = {v: k for k, v in processor.tokenizer.get_vocab().items()}
10
11# Frame duration: product of conv_stride values / sampling_rate
12# For this model: 5*2*2*2*2*2*2 = 320 samples per frame -> 20ms at 16kHz
13frame_duration_s = 320 / 16000 # 0.02s per frame
14
15phonemes, timestamps = [], []
16prev_id = None
17for frame_idx, token_id in enumerate(pred_ids):
18 if token_id == 0: # skip CTC blank
19 prev_id = None
20 continue
21 if token_id == prev_id: # skip CTC repeats
22 continue
23 prev_id = token_id
24 phoneme = id_to_phoneme.get(int(token_id), "<unk>")
25 if phoneme not in ("<pad>", "<unk>", "|"):
26 t = frame_idx * frame_duration_s
27 phonemes.append(phoneme)
28 timestamps.append(t)
29
30for p, t in zip(phonemes, timestamps):
31 print(f" {t:.3f}s {p}")1@inproceedings{conneau2020unsupervised,
2 title={Unsupervised Cross-lingual Representation Learning for Speech Recognition},
3 author={Conneau, Alexis and Baevski, Alexei and Rothe, Henry and Araabi, Ali and Auli, Michael},
4 booktitle={Interspeech},
5 year={2020}
6}