Views
No views yet
torch.Size([16, 1, 128, 334]))1from transformers import ConvNextForImageClassification
2import torch
3import torchaudio
4from torchvision import transforms
5import requests
6import torchaudio
7import io
8
9
10# download the audio file of a bird sound: Common Craw
11url = "https://xeno-canto.org/704485/download"
12response = requests.get(url)
13audio, sample_rate = torchaudio.load(io.BytesIO(response.content))
14print("Original shape and sample rate: ", audio.shape, sample_rate)
15# crop to 5 seconds
16audio = audio[:, : 5 * sample_rate]
17# resample to 32kHz
18resample = torchaudio.transforms.Resample(orig_freq=sample_rate, new_freq=32000)
19audio = resample(audio)
20print("Resampled shape and sample rate: ", audio.shape, 32000)
21
22
23CACHE_DIR = "../../data_birdset" # Change this to your own cache directory
24
25# Load the model
26model = ConvNextForImageClassification.from_pretrained(
27 "DBD-research-group/ConvNeXT-Base-BirdSet-XCL",
28 cache_dir=CACHE_DIR,
29 ignore_mismatched_sizes=True,
30)
31
32
33class PowerToDB(torch.nn.Module):
34 """
35 A power spectrogram to decibel conversion layer. See birdset.datamodule.components.augmentations
36 """
37
38 def __init__(self, ref=1.0, amin=1e-10, top_db=80.0):
39 super(PowerToDB, self).__init__()
40 # Initialize parameters
41 self.ref = ref
42 self.amin = amin
43 self.top_db = top_db
44
45 def forward(self, S):
46 # Convert S to a PyTorch tensor if it is not already
47 S = torch.as_tensor(S, dtype=torch.float32)
48
49 if self.amin <= 0:
50 raise ValueError("amin must be strictly positive")
51
52 if torch.is_complex(S):
53 magnitude = S.abs()
54 else:
55 magnitude = S
56
57 # Check if ref is a callable function or a scalar
58 if callable(self.ref):
59 ref_value = self.ref(magnitude)
60 else:
61 ref_value = torch.abs(torch.tensor(self.ref, dtype=S.dtype))
62
63 # Compute the log spectrogram
64 log_spec = 10.0 * torch.log10(
65 torch.maximum(magnitude, torch.tensor(self.amin, device=magnitude.device))
66 )
67 log_spec -= 10.0 * torch.log10(
68 torch.maximum(ref_value, torch.tensor(self.amin, device=magnitude.device))
69 )
70
71 # Apply top_db threshold if necessary
72 if self.top_db is not None:
73 if self.top_db < 0:
74 raise ValueError("top_db must be non-negative")
75 log_spec = torch.maximum(log_spec, log_spec.max() - self.top_db)
76
77 return log_spec
78
79
80
81# Initialize the transformations
82
83spectrogram_converter = torchaudio.transforms.Spectrogram(
84 n_fft=1024, hop_length=320, power=2.0
85)
86mel_converter = torchaudio.transforms.MelScale(
87 n_mels=128, n_stft=513, sample_rate=32_000
88)
89normalizer = transforms.Normalize((-4.268,), (4.569,))
90powerToDB = PowerToDB(top_db=80)
91
92
93def preprocess(audio, sample_rate_of_audio):
94 """
95 Preprocess the audio to the format that the model expects
96 - Resample to 32kHz
97 - Convert to melscale spectrogram n_fft: 1024, hop_length: 320, power: 2. melscale: n_mels: 128, n_stft: 513
98 - Normalize the melscale spectrogram with mean: -4.268, std: 4.569 (from AudioSet)
99
100 """
101 # convert waveform to spectrogram
102 spectrogram = spectrogram_converter(audio)
103 spectrogram = spectrogram.to(torch.float32)
104 melspec = mel_converter(spectrogram)
105 dbscale = powerToDB(melspec)
106 normalized_dbscale = normalizer(dbscale)
107 # add dimension 3 from left
108 normalized_dbscale = normalized_dbscale.unsqueeze(-3)
109 return normalized_dbscale
110
111preprocessed_audio = preprocess(audio, sample_rate)
112print("Preprocessed_audio shape:", preprocessed_audio.shape)
113
114
115
116logits = model(preprocessed_audio).logits
117print("Logits shape: ", logits.shape)
118
119top5 = torch.topk(logits, 5)
120print("Top 5 logits:", top5.values)
121print("Top 5 predicted classes:")
122print([model.config.id2label[i] for i in top5.indices.squeeze().tolist()])