Views
No views yet
["fluent", "disfluent"][
"Block",
"Prolongation",
"Sound Repetition",
"Word Repetition",
"Interjection"
]git clone git@github.com:tiantiaf0627/vox-profile-release.git1conda create -n vox_profile python=3.8
2cd vox-profile-release
3pip install -e .1# Load libraries
2import torch
3import torch.nn.functional as F
4from src.model.fluency.wavlm_fluency import WavLMWrapper
5
6# Find device
7device = torch.device("cuda") if torch.cuda.is_available() else "cpu"
8
9# Load model from Huggingface
10model = WavLMWrapper.from_pretrained("tiantiaf/wavlm-large-speech-flow").to(device)
11model.eval()1# The way we do inference for fluency is different as the training data is 3s, so we need to do some stacking
2audio_data = torch.zeros([1, 16000*10]).float().to(device)
3audio_segment = (audio_data.shape[1] - 3*16000) // 16000 + 1
4if audio_segment < 1: audio_segment = 1
5input_audio = list()
6for idx in range(audio_segment): input_audio.append(audio_data[0, 16000*idx:16000*idx+3*16000])
7input_audio = torch.stack(input_audio, dim=0)1fluency_outputs, disfluency_type_outputs = model(input_audio)
2fluency_prob = F.softmax(fluency_outputs, dim=1).detach().cpu().numpy().astype(float).tolist()
3
4disfluency_type_prob = nn.Sigmoid()(disfluency_type_outputs)
5# we can set a higher threshold in practice
6disfluency_type_predictions = (disfluency_type_prob > 0.7).int().detach().cpu().numpy().tolist()
7disfluency_type_prob = disfluency_type_prob.cpu().numpy().astype(float).tolist()1utterance_fluency_list = list()
2utterance_disfluency_list = list()
3for audio_idx in range(audio_segment):
4 disfluency_type = list()
5 if fluency_prob[audio_idx][0] > 0.5:
6 utterance_fluency_list.append("fluent")
7 else:
8 # If the prediction is disfluent, then which disfluency type
9 utterance_fluency_list.append("disfluent")
10 predictions = disfluency_type_predictions[audio_idx]
11 for label_idx in range(len(predictions)):
12 if predictions[label_idx] == 1:
13 disfluency_type.append(disfluency_type_labels[label_idx])
14 utterance_disfluency_list.append(disfluency_type)
15
16# Now print how fluent is the utterance
17print(utterance_fluency_list)
18print(utterance_disfluency_list)@article{feng2025vox,
title={Vox-Profile: A Speech Foundation Model Benchmark for Characterizing Diverse Speaker and Speech Traits},
author={Feng, Tiantian and Lee, Jihwan and Xu, Anfeng and Lee, Yoonjeong and Lertpetchpun, Thanathai and Shi, Xuan and Wang, Helin and Thebaud, Thomas and Moro-Velazquez, Laureano and Byrd, Dani and others},
journal={arXiv preprint arXiv:2505.14648},
year={2025}
}