Views
No views yet
torch.Size([16, 1, 256, 417]))1from transformers import EfficientNetForImageClassification
2import torch
3import torchaudio
4from torchvision import transforms
5import requests
6import torchaudio
7import io
8
9
10# download the audio file of a bird sound: Common Craw
11url = "https://xeno-canto.org/704485/download"
12response = requests.get(url)
13audio, sample_rate = torchaudio.load(io.BytesIO(response.content))
14print("Original shape and sample rate: ", audio.shape, sample_rate)
15# crop to 5 seconds
16audio = audio[:, : 5 * sample_rate]
17# resample to 32kHz
18resample = torchaudio.transforms.Resample(orig_freq=sample_rate, new_freq=32000)
19audio = resample(audio)
20print("Resampled shape and sample rate: ", audio.shape, 32000)
21
22
23CACHE_DIR = "../../data_birdset" # Change this to your own cache directory
24
25# Load the model
26model = EfficientNetForImageClassification.from_pretrained(
27 "DBD-research-group/EfficientNet-B1-BirdSet-XCL",
28 num_channels=1,
29 cache_dir=CACHE_DIR,
30 ignore_mismatched_sizes=True,
31)
32
33
34class PowerToDB(torch.nn.Module):
35 """
36 A power spectrogram to decibel conversion layer. See birdset.datamodule.components.augmentations
37 """
38
39 def __init__(self, ref=1.0, amin=1e-10, top_db=80.0):
40 super(PowerToDB, self).__init__()
41 # Initialize parameters
42 self.ref = ref
43 self.amin = amin
44 self.top_db = top_db
45
46 def forward(self, S):
47 # Convert S to a PyTorch tensor if it is not already
48 S = torch.as_tensor(S, dtype=torch.float32)
49
50 if self.amin <= 0:
51 raise ValueError("amin must be strictly positive")
52
53 if torch.is_complex(S):
54 magnitude = S.abs()
55 else:
56 magnitude = S
57
58 # Check if ref is a callable function or a scalar
59 if callable(self.ref):
60 ref_value = self.ref(magnitude)
61 else:
62 ref_value = torch.abs(torch.tensor(self.ref, dtype=S.dtype))
63
64 # Compute the log spectrogram
65 log_spec = 10.0 * torch.log10(
66 torch.maximum(magnitude, torch.tensor(self.amin, device=magnitude.device))
67 )
68 log_spec -= 10.0 * torch.log10(
69 torch.maximum(ref_value, torch.tensor(self.amin, device=magnitude.device))
70 )
71
72 # Apply top_db threshold if necessary
73 if self.top_db is not None:
74 if self.top_db < 0:
75 raise ValueError("top_db must be non-negative")
76 log_spec = torch.maximum(log_spec, log_spec.max() - self.top_db)
77
78 return log_spec
79
80# Initialize preprocessors
81spectrogram_converter = torchaudio.transforms.Spectrogram(
82 n_fft=2048, hop_length=256, power=2.0
83)
84mel_converter = torchaudio.transforms.MelScale(
85 n_mels=256, n_stft=1025, sample_rate=32_000
86)
87powerToDB = PowerToDB(top_db=80)
88
89
90def preprocess(audio, sample_rate_of_audio):
91 """
92 Preprocess the audio to the format that the model expects
93 - Resample to 32kHz
94 - Convert to melscale spectrogram n_fft: 2048, hop_length: 256, power: 2. melscale: n_mels: 256, n_stft: 1025
95 - Normalize the melscale spectrogram with mean: -4.268, std: 4.569 (from AudioSet)
96
97 """
98 spectrogram = spectrogram_converter(audio)
99 spectrogram = spectrogram.to(torch.float32)
100 melspec = mel_converter(spectrogram)
101 dbscale = powerToDB(melspec)
102 normalized_dbscale = transforms.Normalize((-4.268,), (4.569,))(dbscale)
103 # add batch dimension if needed
104 if normalized_dbscale.dim() == 3:
105 normalized_dbscale = normalized_dbscale.unsqueeze(0)
106 return normalized_dbscale
107
108preprocessed_audio = preprocess(audio, sample_rate)
109print("Preprocessed_audio shape:", preprocessed_audio.shape)
110
111logits = model(preprocessed_audio).logits
112print("Logits shape: ", logits.shape)
113
114top5 = torch.topk(logits, 5)
115print("Top 5 logits:", top5.values)
116print("Top 5 predicted classes:")
117print([model.config.id2label[i] for i in top5.indices.squeeze().tolist()])