Views
No views yet
1GIT_LFS_SKIP_SMUDGE=1 git clone https://huggingface.co/HKUSTAudio/AudioX
2cd AudioX
3
4conda create -n AudioX python=3.8.20
5conda activate AudioX
6pip install git+https://github.com/ZeyueT/AudioX.git
7conda install -c conda-forge ffmpeg libsndfile1import torch
2import torchaudio
3from einops import rearrange
4from stable_audio_tools import get_pretrained_model
5from stable_audio_tools.inference.generation import generate_diffusion_cond
6from stable_audio_tools.data.utils import read_video, merge_video_audio
7from stable_audio_tools.data.utils import load_and_process_audio
8import os
9
10device = "cuda" if torch.cuda.is_available() else "cpu"
11
12# Download model
13model, model_config = get_pretrained_model("HKUSTAudio/AudioX")
14sample_rate = model_config["sample_rate"]
15sample_size = model_config["sample_size"]
16target_fps = model_config["video_fps"]
17seconds_start = 0
18seconds_total = 10
19
20model = model.to(device)
21
22# for video-to-music generation
23video_path = "video.mp4"
24text_prompt = "Generate music for the video"
25audio_path = None
26
27video_tensor = read_video(video_path, seek_time=0, duration=seconds_total, target_fps=target_fps)
28audio_tensor = load_and_process_audio(audio_path, sample_rate, seconds_start, seconds_total)
29
30conditioning = [{
31 "video_prompt": [video_tensor.unsqueeze(0)],
32 "text_prompt": text_prompt,
33 "audio_prompt": audio_tensor.unsqueeze(0),
34 "seconds_start": seconds_start,
35 "seconds_total": seconds_total
36}]
37
38# Generate stereo audio
39output = generate_diffusion_cond(
40 model,
41 steps=250,
42 cfg_scale=7,
43 conditioning=conditioning,
44 sample_size=sample_size,
45 sigma_min=0.3,
46 sigma_max=500,
47 sampler_type="dpmpp-3m-sde",
48 device=device
49)
50
51# Rearrange audio batch to a single sequence
52output = rearrange(output, "b d n -> d (b n)")
53
54# Peak normalize, clip, convert to int16, and save to file
55output = output.to(torch.float32).div(torch.max(torch.abs(output))).clamp(-1, 1).mul(32767).to(torch.int16).cpu()
56torchaudio.save("output.wav", output, sample_rate)
57
58if video_path is not None and os.path.exists(video_path):
59 merge_video_audio(video_path, "output.wav", "output.mp4", 0, seconds_total)
601@article{tian2025audiox,
2 title={Audiox: Diffusion transformer for anything-to-audio generation},
3 author={Tian, Zeyue and Jin, Yizhu and Liu, Zhaoyang and Yuan, Ruibin and Tan, Xu and Chen, Qifeng and Xue, Wei and Guo, Yike},
4 journal={arXiv preprint arXiv:2503.10522},
5 year={2025}
6}
7
8@inproceedings{tian2025vidmuse,
9 title={Vidmuse: A simple video-to-music generation framework with long-short-term modeling},
10 author={Tian, Zeyue and Liu, Zhaoyang and Yuan, Ruibin and Pan, Jiahao and Liu, Qifeng and Tan, Xu and Chen, Qifeng and Xue, Wei and Guo, Yike},
11 booktitle={Proceedings of the Computer Vision and Pattern Recognition Conference},
12 pages={18782--18793},
13 year={2025}
14}