Views
No views yet
KeyError: 'qwen2_vl'qwen-vl-utils to handle various types of visual input. You can install it using the following command:pip install qwen-vl-utilspip install descript-audio-codectransformers1import torch
2import deepspeed_utils # This line is important, do not delete it
3from transformers import AutoModelForCausalLM, AutoTokenizer, AutoProcessor
4
5# Import from utils modules
6from utils import (
7 Dac,
8 preprocess_codec,
9 DecoderOutput,
10 tts_preprocess,
11 t2m_preprocess,
12 v2m_preprocess,
13 prepare_audio_prompt,
14 generate_output
15)
16
17model_path = "/path/to/your/model"
18
19dac = Dac()
20
21model = AutoModelForCausalLM.from_pretrained(
22 model_path,
23 torch_dtype=torch.float32,
24 attn_implementation='sdpa',
25 trust_remote_code=True,
26).eval()
27model = model.to('cuda')
28
29processor = AutoProcessor.from_pretrained(model_path)
301transcription = [
2 "The nature reserve covers only a small part of the marsh area.",
3 "我们基于动态容量混合专家框架，构建了一个统一语音和音乐生成模型。"
4]
5prompt_wav = "/path/to/your/voice/prompt"
6prompt_transcription = "content of your voice prompt"
7
8prompt_codec = preprocess_codec(model, dac.encode(prompt_wav))
9text_input, tts_generation_kwargs = tts_preprocess(transcription, prompt_codec, prompt_transcription, model.device)
10source_input = processor.tokenizer(text_input, add_special_tokens=False, return_tensors="pt", padding=True).to(model.device)
11
12prefill, prefill_steps = prepare_audio_prompt(model, audio_prompts=[None] * len(transcription))
13dec_output = DecoderOutput(prefill, prefill_steps, model.device)
14
15with torch.no_grad():
16 generated_codes, lengths_Bx = model.generate(
17 input_ids=source_input.input_ids,
18 attention_mask=source_input.attention_mask,
19 dec_output=dec_output,
20 max_tokens=10 * 50, # maximum duration of the generated audio is 10 seconds
21 min_tokens=1 * 50, # minimum duration of the generated audio is 1 seconds
22 temperature=1.0,
23 top_p=1.0,
24 cfg_filter_top_k=45,
25 do_sample=True,
26 use_cache=True,
27 **tts_generation_kwargs
28 )
29
30audios = generate_output(model, generated_codes, lengths_Bx)
31for i in range(len(audios)):
32 output_path = os.path.join(f"./generated_speech_{i}.wav")
33 dac.decode(audios[i].transpose(0, 1).unsqueeze(0), save_path=output_path, min_duration=1)
341caption = [
2 "A retro-inspired synthwave track with a driving beat and nostalgic melodies. Perfect for cruising or late-night drives.",
3 "A mid-tempo electronic track with a driving beat and atmospheric synth textures. Ideal for background listening or a chill dance set."
4]
5
6text_input, t2m_generation_kwargs = t2m_preprocess(caption)
7
8source_input = processor.tokenizer(text_input, add_special_tokens=False, return_tensors="pt", padding=True).to(model.device)
9
10prefill, prefill_steps = prepare_audio_prompt(model, audio_prompts=[None] * len(caption))
11dec_output = DecoderOutput(prefill, prefill_steps, model.device)
12
13with torch.no_grad():
14 generated_codes, lengths_Bx = model.generate(
15 input_ids=source_input.input_ids,
16 attention_mask=source_input.attention_mask,
17 dec_output=dec_output,
18 max_tokens=20 * 50, # maximum duration of the generated audio is 20 seconds
19 min_tokens=8 * 50, # minimum duration of the generated audio is 8 seconds
20 temperature=1.0,
21 top_p=1.0,
22 cfg_filter_top_k=45,
23 do_sample=True,
24 use_cache=True,
25 **t2m_generation_kwargs
26 )
27
28audios = generate_output(model, generated_codes, lengths_Bx)
29for i in range(len(audios)):
30 output_path = os.path.join(f"./generated_music_{i}.wav")
31 dac.decode(audios[i].transpose(0, 1).unsqueeze(0), save_path=output_path, min_duration=1)
32
331
2caption = [
3 "A relaxing instrumental piece featuring a simple melody played on a synth flute. The track creates a calm and peaceful atmosphere.",
4]
5video = [
6 "/path/to/your/video/path.mp4",
7]
8
9text_input, video_inputs, fps_inputs, v2m_generation_kwargs = v2m_preprocess(caption, video)
10
11source_input = processor(text=text_input, images=None, videos=video_inputs, fps=fps_inputs, padding=True, return_tensors="pt", do_resize=False)
12source_input = source_input.to(model.device)
13
14prefill, prefill_steps = prepare_audio_prompt(model, audio_prompts=[None] * len(caption))
15dec_output = DecoderOutput(prefill, prefill_steps, model.device)
16
17with torch.no_grad():
18 generated_codes, lengths_Bx = model.generate(
19 input_ids=source_input.input_ids,
20 pixel_values_videos=source_input.pixel_values_videos,
21 video_grid_thw=source_input.video_grid_thw,
22 second_per_grid_ts=source_input.second_per_grid_ts,
23 attention_mask=source_input.attention_mask,
24 dec_output=dec_output,
25 max_tokens=20 * 50, # maximum duration of the generated audio is 20 seconds
26 min_tokens=8 * 50, # minimum duration of the generated audio is 8 seconds
27 temperature=1.0,
28 top_p=1.0,
29 cfg_filter_top_k=45,
30 do_sample=True,
31 use_cache=True,
32 **v2m_generation_kwargs
33 )
34
35audios = generate_output(model, generated_codes, lengths_Bx)
36for i in range(len(audios)):
37 output_path = os.path.join(f"./generated_video_music_{i}.wav")
38 dac.decode(audios[i].transpose(0, 1).unsqueeze(0), save_path=output_path, min_duration=1)@article{liu2025unimoeaudiounifiedspeechmusic,
title={UniMoE-Audio: Unified Speech and Music Generation with Dynamic-Capacity MoE},
author={Zhenyu Liu and Yunxin Li and Xuanyu Zhang and Qixun Teng and Shenyuan Jiang and Xinyu Chen and Haoyuan Shi and Jinchao Li and Qi Wang and Haolan Chen and Fanbo Meng and Mingjun Zhao and Yu Xu and Yancheng He and Baotian Hu and Min Zhang},
year={2025},
journal={arXiv preprint arXiv:2510.13344},
url={https://arxiv.org/abs/2510.13344},
}