MiDashengLM-Gen is an end-to-end framework that uses a pre-trained Large Language Model and audio tokenizer as the backbone, combined with per-token conditional flow matching for autoregressive, variable-length mixed-audio scene generation. It generates coherent 16 kHz audio scenes that simultaneously blend speech, music, sound effects, and environmental acoustics from structured text descriptions.
Input uses structured multi-view captions with special tokens to describe different aspects of an audio scene. Use <|unknown|> for absent elements.
1from transformers import AutoModel
2import soundfile as sf
3
4model = AutoModel.from_pretrained("mispeech/midashenglm-gen", trust_remote_code=True)
5model = model.cuda()
6
7result = model.generate(
8 "<|caption|> A comedian delivering a punchline followed by uproarious crowd laughter and an upbeat jazz band hit "
9 "<|asr|> And that is why I never buy cheap luggage anymore! "
10 "<|speech|> expressive comedic male voice "
11 "<|music|> sudden upbeat jazz band sting "
12 "<|sfx|> uproarious crowd laughter "
13 "<|env|> intimate comedy club"
14)
15
16sf.write("output.wav", result["audio"], result["sample_rate"])
1texts = [
2 "<|caption|> A comedian delivering a punchline followed by uproarious crowd laughter and an upbeat jazz band hit <|asr|> And that is why I never buy cheap luggage anymore! <|speech|> expressive comedic male voice <|music|> sudden upbeat jazz band sting <|sfx|> uproarious crowd laughter <|env|> intimate comedy club",
3 "<|caption|> Upbeat instrumental music with occasional whistling and low audio quality. <|asr|> <|unknown|> <|speech|> <|unknown|> <|sfx|> Intermittent whistling emerges in later segment alongside persistent instrumental playback. <|music|> Bright melodic composition with guitar, piano, and rhythmic percussion elements, maintaining a consistently upbeat character. <|env|> Low-fidelity recording with compressed dynamics and limited frequency response.",
4]
5result = model.generate(texts)
6for i, audio in enumerate(result["audio"]):
7 sf.write(f"output_{i}.wav", audio, result["sample_rate"])
1result = model.generate(
2 "...",
3 eval_cfg=2.0, # Classifier-free guidance scale
4 stop_threshold=0.5, # Stop prediction threshold
5 min_stop_step=5, # Minimum steps before stopping
6 seed=42, # Random seed for reproducibility
7)
1@article{sun2026midashenglmgen,
2 title={MiDashengLM-Gen: Unified Audio Scene Generation via LLM-Driven Autoregressive Flow Matching},
3 author={Sun, Xingwei and Dinkel, Heinrich and Li, Gang and Mei, Jiahao and Niu, Yadong and Han, Zerui and Jiang, Yuepeng and Zhou, Jiahao and Fan, Lichun and Luan, Jian},
4 journal={arXiv preprint},
5 eprint={2608.11804},
6 archivePrefix={arXiv},
7 year={2026}
8}
