Views
No views yet
pip install git+https://github.com/huggingface/transformers1import soundfile as sf
2import torch
3from transformers import WhisperFeatureExtractor
4from transformers.models.qwen3_omni_moe.modeling_qwen3_omni_moe import Qwen3OmniMoeAudioEncoder
5
6model_id = "Atotti/Qwen3-Omni-AudioTransformer"
7feature_extractor = WhisperFeatureExtractor.from_pretrained(model_id)
8qwen_encoder = Qwen3OmniMoeAudioEncoder.from_pretrained(model_id, dtype=torch.bfloat16).eval()
9
10wav, sr = sf.read("audio_16k.wav", dtype="float32")
11
12features = feature_extractor(wav, sampling_rate=sr, return_tensors="pt", padding=False)
13x = features["input_features"][0].to(qwen_encoder.dtype)
14feat_len = torch.tensor([x.shape[-1]], dtype=torch.long)
15
16with torch.no_grad():
17 out = qwen_encoder(input_features=x, feature_lens=feat_len)
18 hid = out.last_hidden_state # [T', D]
19 print(hid.shape) # torch.Size([37, 2048])
20 print(hid[0, :10]) # tensor([ 0.0088, -0.0032, 0.0154, -0.0138, 0.0040, -0.0037, -0.0017, 0.0180, -0.0149, 0.0051])
21Qwen3OmniMoeAudioEncoderConfig {
"activation_dropout": 0,
"activation_function": "gelu",
"architectures": [
"Qwen3OmniMoeAudioEncoder"
],
"attention_dropout": 0,
"conv_chunksize": 500,
"d_model": 1280,
"downsample_hidden_size": 480,
"dropout": 0,
"dtype": "bfloat16",
"encoder_attention_heads": 20,
"encoder_ffn_dim": 5120,
"encoder_layers": 32,
"initializer_range": 0.02,
"max_source_positions": 1500,
"model_type": "qwen3_omni_moe_audio_encoder",
"n_window": 50,
"n_window_infer": 800,
"num_hidden_layers": 32,
"num_mel_bins": 128,
"output_dim": 2048,
"scale_embedding": false,
"tf_legacy_loss": false,
"transformers_version": "4.57.0.dev0",
"use_bfloat16": false
}
Qwen3OmniMoeAudioEncoder(
(positional_embedding): SinusoidsPositionEmbedding()
(layers): ModuleList(
(0-31): 32 x Qwen3OmniMoeAudioEncoderLayer(
(self_attn): Qwen3OmniMoeAudioAttention(
(k_proj): Linear(in_features=1280, out_features=1280, bias=True)
(v_proj): Linear(in_features=1280, out_features=1280, bias=True)
(q_proj): Linear(in_features=1280, out_features=1280, bias=True)
(out_proj): Linear(in_features=1280, out_features=1280, bias=True)
)
(self_attn_layer_norm): LayerNorm((1280,), eps=1e-05, elementwise_affine=True)
(activation_fn): GELUActivation()
(fc1): Linear(in_features=1280, out_features=5120, bias=True)
(fc2): Linear(in_features=5120, out_features=1280, bias=True)
(final_layer_norm): LayerNorm((1280,), eps=1e-05, elementwise_affine=True)
)
)
(ln_post): LayerNorm((1280,), eps=1e-05, elementwise_affine=True)
(conv2d1): Conv2d(1, 480, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1))
(conv2d2): Conv2d(480, 480, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1))
(conv2d3): Conv2d(480, 480, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1))
(conv_out): Linear(in_features=7680, out_features=1280, bias=False)
(proj1): Linear(in_features=1280, out_features=1280, bias=True)
(act): GELUActivation()
(proj2): Linear(in_features=1280, out_features=2048, bias=True)
)