Views
No views yet
pip3 install git+https://github.com/malaysia-ai/async-parler-tts1import torch
2from parler_tts import ParlerTTSForConditionalGeneration
3from transformers import AutoTokenizer
4from pypinyin import lazy_pinyin, Style
5import soundfile as sf
6import malaya
7import jieba
8
9normalizer = malaya.normalize.normalizer()
10jieba.initialize()
11
12def is_chinese(c):
13 return (
14 "\u3100" <= c <= "\u9fff"
15 )
16
17def convert_char_to_pinyin(text_list, polyphone=True):
18 final_text_list = []
19 custom_trans = str.maketrans(
20 {";": ",", "“": '"', "”": '"', "‘": "'", "’": "'", '，': ', ', '！': '. ', '。': '. '}
21 )
22
23 for text in text_list:
24 char_list = []
25 text = text.translate(custom_trans)
26 for seg in jieba.cut(text):
27 seg_byte_len = len(bytes(seg, "UTF-8"))
28 if seg_byte_len == len(seg): # if pure alphabets and symbols
29 if char_list and seg_byte_len > 1 and char_list[-1] not in " :'\"":
30 char_list.append(" ")
31 char_list.extend(seg)
32 elif polyphone and seg_byte_len == 3 * len(seg): # if pure east asian characters
33 seg_ = lazy_pinyin(seg, style=Style.TONE3, tone_sandhi=True)
34 for i, c in enumerate(seg):
35 if is_chinese(c):
36 char_list.append(" ")
37 char_list.append(seg_[i])
38 else: # if mixed characters, alphabets and symbols
39 for c in seg:
40 if ord(c) < 256:
41 char_list.extend(c)
42 elif is_chinese(c):
43 char_list.append(" ")
44 char_list.extend(lazy_pinyin(c, style=Style.TONE3, tone_sandhi=True))
45 else:
46 char_list.append(c)
47 final_text_list.append(char_list)
48
49 return final_text_list
50
51def normalize(text):
52 converted = convert_char_to_pinyin(text.split())
53 converted = [''.join(c) for c in converted]
54 return ' '.join(converted).strip()
55
56device = "cuda:0" if torch.cuda.is_available() else "cpu"
57
58model = ParlerTTSForConditionalGeneration.from_pretrained("mesolitica/malaysian-parler-tts-mini-v1").to(device)
59tokenizer = AutoTokenizer.from_pretrained("mesolitica/malaysian-parler-tts-mini-v1")
60
61speakers = [
62 'Husein',
63 'Shafiqah Idayu',
64 'Anwar Ibrahim',
65 'KP'
66]
67
68# Also support context switching
69prompt = 'Husein zolkepli sangat comel dan kacak suka makan cendol. 其形成的门店数字化营销、, AI, 数字化服务、数字化、用户数字化等数字化成熟体系'
70prompt = normalizer.normalize(prompt)
71prompt = normalize(prompt['normalize'])
72
73for s in speakers:
74 description = s
75
76 input_ids = tokenizer(description, return_tensors="pt").to(device)
77 prompt_input_ids = tokenizer(prompt, return_tensors="pt").to(device)
78
79 generation = model.generate(
80 input_ids=input_ids.input_ids,
81 attention_mask=input_ids.attention_mask,
82 prompt_input_ids=prompt_input_ids.input_ids,
83 prompt_attention_mask=prompt_input_ids.attention_mask,
84 )
85
86 audio_arr = generation.cpu()
87 sf.write(f'{s}.mp3', audio_arr.numpy().squeeze(), 44100)