Views
No views yet
| Model Name | Description |
|---|---|
| Content Tokenizer | Converting speech to content tokens. It is a single codebook VQ-VAE with a vocabulary size of 32. The frame rate is 50Hz. |
| Content-Style Tokenizer | Converting speech to content-style tokens. It is a single codebook VQ-VAE with a vocabulary size of 8192. The frame rate is 50Hz. |
| Vq32ToVq8192 | Predicting content-style tokens from content tokens with an auto-regressive transformer (480M). |
| PhoneToVq8192 | Predicting content-style tokens from phone tokens with an auto-regressive transformer (740M). |
| Vq8192ToMels | Predicting mel-spectrogram from content-style tokens with a flow-matching transformer (330M). |
| Vocoder | Predicting audio from mel-spectrogram with a Vocos-based vocoder (250M). |
1import os
2from huggingface_hub import snapshot_download
3
4from models.vc.vevo.vevo_utils import *
5
6
7def vevo_tts(
8 src_text,
9 ref_wav_path,
10 timbre_ref_wav_path=None,
11 output_path=None,
12 ref_text=None,
13 src_language="en",
14 ref_language="en",
15):
16 if timbre_ref_wav_path is None:
17 timbre_ref_wav_path = ref_wav_path
18
19 gen_audio = inference_pipeline.inference_ar_and_fm(
20 src_wav_path=None,
21 src_text=src_text,
22 style_ref_wav_path=ref_wav_path,
23 timbre_ref_wav_path=timbre_ref_wav_path,
24 style_ref_wav_text=ref_text,
25 src_text_language=src_language,
26 style_ref_wav_text_language=ref_language,
27 )
28
29 assert output_path is not None
30 save_audio(gen_audio, output_path=output_path)
31
32
33if __name__ == "__main__":
34 # ===== Device =====
35 device = torch.device("cuda") if torch.cuda.is_available() else torch.device("cpu")
36
37 # ===== Content-Style Tokenizer =====
38 local_dir = snapshot_download(
39 repo_id="amphion/Vevo",
40 repo_type="model",
41 cache_dir="./ckpts/Vevo",
42 allow_patterns=["tokenizer/vq8192/*"],
43 )
44
45 content_style_tokenizer_ckpt_path = os.path.join(local_dir, "tokenizer/vq8192")
46
47 # ===== Autoregressive Transformer =====
48 local_dir = snapshot_download(
49 repo_id="amphion/Vevo",
50 repo_type="model",
51 cache_dir="./ckpts/Vevo",
52 allow_patterns=["contentstyle_modeling/PhoneToVq8192/*"],
53 )
54
55 ar_cfg_path = "./models/vc/vevo/config/PhoneToVq8192.json"
56 ar_ckpt_path = os.path.join(local_dir, "contentstyle_modeling/PhoneToVq8192")
57
58 # ===== Flow Matching Transformer =====
59 local_dir = snapshot_download(
60 repo_id="amphion/Vevo",
61 repo_type="model",
62 cache_dir="./ckpts/Vevo",
63 allow_patterns=["acoustic_modeling/Vq8192ToMels/*"],
64 )
65
66 fmt_cfg_path = "./models/vc/vevo/config/Vq8192ToMels.json"
67 fmt_ckpt_path = os.path.join(local_dir, "acoustic_modeling/Vq8192ToMels")
68
69 # ===== Vocoder =====
70 local_dir = snapshot_download(
71 repo_id="amphion/Vevo",
72 repo_type="model",
73 cache_dir="./ckpts/Vevo",
74 allow_patterns=["acoustic_modeling/Vocoder/*"],
75 )
76
77 vocoder_cfg_path = "./models/vc/vevo/config/Vocoder.json"
78 vocoder_ckpt_path = os.path.join(local_dir, "acoustic_modeling/Vocoder")
79
80 # ===== Inference =====
81 inference_pipeline = VevoInferencePipeline(
82 content_style_tokenizer_ckpt_path=content_style_tokenizer_ckpt_path,
83 ar_cfg_path=ar_cfg_path,
84 ar_ckpt_path=ar_ckpt_path,
85 fmt_cfg_path=fmt_cfg_path,
86 fmt_ckpt_path=fmt_ckpt_path,
87 vocoder_cfg_path=vocoder_cfg_path,
88 vocoder_ckpt_path=vocoder_ckpt_path,
89 device=device,
90 )
91
92 src_text = "I don't really care what you call me. I've been a silent spectator, watching species evolve, empires rise and fall. But always remember, I am mighty and enduring. Respect me and I'll nurture you; ignore me and you shall face the consequences."
93
94 ref_wav_path = "./models/vc/vevo/wav/arabic_male.wav"
95 ref_text = "Flip stood undecided, his ears strained to catch the slightest sound."
96
97 # 1. Zero-Shot TTS (the style reference and timbre reference are same)
98 vevo_tts(
99 src_text,
100 ref_wav_path,
101 output_path="./models/vc/vevo/wav/output_vevotts1.wav",
102 ref_text=ref_text,
103 src_language="en",
104 ref_language="en",
105 )
106
107 # 2. Style and Timbre Controllable Zero-Shot TTS (the style reference and timbre reference are different)
108 vevo_tts(
109 src_text,
110 ref_wav_path,
111 timbre_ref_wav_path="./models/vc/vevo/wav/mandarin_female.wav",
112 output_path="./models/vc/vevo/wav/output_vevotts2.wav",
113 ref_text=ref_text,
114 src_language="en",
115 ref_language="en",
116 )1@inproceedings{vevo,
2 author = {Xueyao Zhang and Xiaohui Zhang and Kainan Peng and Zhenyu Tang and Vimal Manohar and Yingru Liu and Jeff Hwang and Dangna Li and Yuhao Wang and Julian Chan and Yuan Huang and Zhizheng Wu and Mingbo Ma},
3 title = {Vevo: Controllable Zero-Shot Voice Imitation with Self-Supervised Disentanglement},
4 booktitle = {{ICLR}},
5 publisher = {OpenReview.net},
6 year = {2025}
7}
8
9@article{amphion_v0.2,
10 title = {Overview of the Amphion Toolkit (v0.2)},
11 author = {Jiaqi Li and Xueyao Zhang and Yuancheng Wang and Haorui He and Chaoren Wang and Li Wang and Huan Liao and Junyi Ao and Zeyu Xie and Yiqiao Huang and Junan Zhang and Zhizheng Wu},
12 year = {2025},
13 journal = {arXiv preprint arXiv:2501.15442},
14}
15
16@inproceedings{amphion,
17 author={Xueyao Zhang and Liumeng Xue and Yicheng Gu and Yuancheng Wang and Jiaqi Li and Haorui He and Chaoren Wang and Ting Song and Xi Chen and Zihao Fang and Haopeng Chen and Junan Zhang and Tze Ying Tang and Lexiao Zou and Mingxuan Wang and Jun Han and Kai Chen and Haizhou Li and Zhizheng Wu},
18 title={Amphion: An Open-Source Audio, Music and Speech Generation Toolkit},
19 booktitle={{IEEE} Spoken Language Technology Workshop, {SLT} 2024},
20 year={2024}
21}