Views
No views yet
openai/whisper-small for multi-label classification. It predicts whether an audio clip contains one or more pitch-quality attributes, and falls back to None when no label passes its decision threshold.shrill: Sharp, piercing, or thin high-frequency vocal quality.nasal: Resonance is concentrated in the nasal tract.deep: Strong low-frequency or deep vocal quality.None: No target pitch-quality label is predicted.WhisperProcessor for resampling and log-Mel feature extraction.openai/whisper-smallNone.1{
2 "shrill": 0.10356783866882324,
3 "nasal": 0.29178741574287415,
4 "deep": 0.07894472032785416,
5}1{
2 0: "shrill",
3 1: "nasal",
4 2: "deep",
5}1import torch
2import librosa
3from transformers import WhisperProcessor, WhisperForAudioClassification
4
5model_id = "Kang-Chieh/whisper-small-mlb-with-none-pitch-quality-dynamic-threshold"
6processor_id = "openai/whisper-small"
7device = "cuda" if torch.cuda.is_available() else "cpu"
8
9thresholds = {
10 "shrill": 0.10356783866882324,
11 "nasal": 0.29178741574287415,
12 "deep": 0.07894472032785416,
13}
14
15# Load the processor and model
16processor = WhisperProcessor.from_pretrained(processor_id)
17model = WhisperForAudioClassification.from_pretrained(model_id).to(device)
18
19def predict_pitch_quality(audio_path):
20 # 1. Load audio and ensure 16kHz mono audio
21 audio, _ = librosa.load(audio_path, sr=16000, mono=True)
22
23 # 2. Preprocess
24 inputs = processor(audio, sampling_rate=16000, return_tensors="pt")
25 input_features = inputs.input_features.to(device)
26
27 # 3. Inference
28 with torch.no_grad():
29 logits = model(input_features=input_features).logits
30
31 # 4. Multi-label logic (Sigmoid)
32 probs = torch.sigmoid(logits).squeeze().cpu().numpy()
33 id2label = {int(k): v for k, v in model.config.id2label.items()}
34 all_scores = {id2label[i]: float(probs[i]) for i in range(len(probs))}
35
36 predicted_labels = [
37 label for label, score in all_scores.items()
38 if score >= thresholds[label]
39 ]
40
41 if not predicted_labels:
42 predicted_labels = ["None"]
43
44 return {
45 "labels": predicted_labels,
46 "all_scores": all_scores,
47 "thresholds": thresholds,
48 }
49
50# Run example
51result = predict_pitch_quality("audio_clip.wav")
52print("Detected Labels:", ", ".join(result["labels"]))
53print("Scores:", result["all_scores"])shrill: precision 0.78, recall 0.45, f1-score 0.57nasal: precision 0.51, recall 0.29, f1-score 0.37deep: precision 0.68, recall 0.56, f1-score 0.62None: precision 0.61, recall 0.80, f1-score 0.69