Views
No views yet
1pip install torch torchvision torchaudio
2pip install git+https://github.com/getuka/RubyInserter.git
3pip install git+https://github.com/getuka/canary-tts.git1import torch, torchaudio
2from transformers import AutoModelForCausalLM, AutoTokenizer
3from canary_tts.xcodec2.modeling_xcodec2 import XCodec2Model
4from rubyinserter import add_ruby
5
6tokenizer = AutoTokenizer.from_pretrained("2121-8/canary-tts-150m")
7model = AutoModelForCausalLM.from_pretrained("2121-8/canary-tts-150m", device_map="auto", torch_dtype=torch.bfloat16)
8codec = XCodec2Model.from_pretrained("HKUSTAudio/xcodec2")
9
10description = "A man voice, with a very hight pitch, speaks in a monotone manner. The recording quality is very noises and close-sounding, indicating a good or excellent audio capture."
11prompt = 'こんにちは。お元気ですか？'
12
13prompt = add_ruby(prompt)
14chat = [
15 {"role": "system", "content": description},
16 {"role": "user", "content": prompt}
17]
18tokenized_input = tokenizer.apply_chat_template(chat, add_generation_prompt=True, tokenize=True, return_tensors="pt").to(model.device)
19
20with torch.no_grad():
21 output = model.generate(
22 tokenized_input,
23 max_new_tokens=256,
24 top_p=0.95,
25 temperature=0.7,
26 repetition_penalty=1.05,
27 )[0]
28
29audio_tokens = output[len(tokenized_input[0]):]
30output_audios = codec.decode_code(audio_tokens.unsqueeze(0).unsqueeze(0).cpu())
31torchaudio.save("sample.wav", src=output_audios[0].cpu(), sample_rate=16000)