Views
No views yet
This model was trained on top of HKUSTAudio/Llasa-1B.
pip install xcodec2 torch torchaudio1import os
2from transformers import AutoTokenizer, AutoModelForCausalLM
3import torch
4import soundfile as sf
5
6llasa_1b_german = 'SebastianBodza/Kartoffel-1B-v0.3'
7
8# Loading the model
9tokenizer = AutoTokenizer.from_pretrained(llasa_1b_german)
10model = AutoModelForCausalLM.from_pretrained(llasa_1b_german)
11model.to('cuda')
12
13# Load XCodec2 model
14from xcodec2.modeling_xcodec2 import XCodec2Model
15model_path = "HKUST-Audio/xcodec2"
16Codec_model = XCodec2Model.from_pretrained(model_path)
17Codec_model.cuda()
18
19input_text = "\"Weißt du was, Hoppi\", sagte der weise Uhu, \"manchmal ist es gar nicht so wichtig, das Ende des Regenbogens zu finden. Das Schönste ist doch, dass wir alle zusammen dieses Abenteuer erleben!"
20
21
22def extract_speech_ids(speech_tokens_str):
23 speech_ids = []
24 for token_str in speech_tokens_str:
25 if token_str.startswith('<|s_') and token_str.endswith('|>'):
26 num_str = token_str[4:-2]
27 num = int(num_str)
28 speech_ids.append(num)
29 else:
30 print(f"Unexpected token: {token_str}")
31 return speech_ids
32
33with torch.no_grad():
34 formatted_text = f"<|TEXT_UNDERSTANDING_START|>{input_text}<|TEXT_UNDERSTANDING_END|>"
35
36 chat = [
37 {"role": "user", "content": "Convert the text to speech:" + formatted_text},
38 {"role": "assistant", "content": "<|SPEECH_GENERATION_START|>"}
39 ]
40
41 input_ids = tokenizer.apply_chat_template(
42 chat,
43 tokenize=True,
44 return_tensors='pt',
45 continue_final_message=True
46 )
47 input_ids = input_ids.to('cuda')
48 speech_end_id = tokenizer.convert_tokens_to_ids('<|SPEECH_GENERATION_END|>')
49
50 outputs = model.generate(
51 input_ids,
52 max_length=2048,
53 eos_token_id=speech_end_id,
54 do_sample=True,
55 top_p=1,
56 temperature=0.8,
57 )
58
59 generated_ids = outputs[0][input_ids.shape[1]:-1]
60 speech_tokens = tokenizer.batch_decode(generated_ids, skip_special_tokens=True)
61 speech_tokens = extract_speech_ids(speech_tokens)
62 speech_tokens = torch.tensor(speech_tokens).cuda().unsqueeze(0).unsqueeze(0)
63 gen_wav = Codec_model.decode_code(speech_tokens)
64
65
66sf.write("generation.wav", gen_wav[0, 0, :].cpu().numpy(), 16000)
671import torch
2import torchaudio
3import tempfile
4import soundfile as sf
5from transformers import AutoTokenizer, AutoModelForCausalLM, pipeline
6
7# Input your reference audio and optional the text
8sample_audio_path = "male.wav"
9sample_audio_text = None # Set it to none to use whisper for transcription
10# Input the target text here
11target_text = "Und apropos Spannungen und Unfälle, in Stuttgart gibt es auch einige Schlagzeilen. Die Polizei sucht Zeugen, nachdem in der Stadt mehrere Autoscheiben eingeschlagen wurden. Und gestern kam es im Stuttgarter Osten zu einer Verfolgungsjagd mit einer jungen BMW-Fahrerin, die vor einer Polizeistreife geflüchtet ist."
12output_filename = "no_speaker_example.wav"
13
14
15#### Do not edit below ####
16llasa_model_name = "SebastianBodza/Kartoffel-1B-v0.3"
17tokenizer = AutoTokenizer.from_pretrained(llasa_model_name)
18model = AutoModelForCausalLM.from_pretrained(llasa_model_name)
19model.to("cuda")
20
21from xcodec2.modeling_xcodec2 import XCodec2Model
22codec_model_path = "HKUST-Audio/xcodec2"
23Codec_model = XCodec2Model.from_pretrained(codec_model_path)
24Codec_model.cuda()
25
26whisper_turbo_pipe = pipeline(
27 "automatic-speech-recognition",
28 model="openai/whisper-large-v3-turbo",
29 torch_dtype=torch.float16,
30 device="cuda",
31)
32
33def ids_to_speech_tokens(speech_ids):
34 speech_tokens_str = []
35 for speech_id in speech_ids:
36 speech_tokens_str.append(f"<|s_{speech_id}|>")
37 return speech_tokens_str
38
39waveform, sample_rate = torchaudio.load(sample_audio_path)
40
41max_secs = 15
42if len(waveform[0]) / sample_rate > 15:
43 print("Warning: Trimming audio to first 15secs.")
44 waveform = waveform[:, : sample_rate * 15]
45 waveform = torch.nn.functional.pad( waveform, (0, int(sample_rate * 0.5)), "constant", 0)
46
47if waveform.size(0) > 1:
48 waveform = torch.mean(waveform, dim=0, keepdim=True)
49
50prompt_wav = torchaudio.transforms.Resample(orig_freq=sample_rate, new_freq=16000)(waveform)
51
52if sample_audio_text is None:
53 print("Transcribing audio...")
54 transcription = whisper_turbo_pipe(waveform[0].numpy())["text"].strip()
55else:
56 transcription = sample_audio_text
57
58print("Transcription:", transcription)
59
60if len(target_text) == 0:
61 raise ValueError("Target text must be provided!")
62elif len(target_text) > 500:
63 print("Text is too long; trimming to first 500 characters.")
64 target_text = target_text[:500]
65
66input_text = transcription + " " + target_text
67
68with torch.no_grad():
69 vq_code_prompt = Codec_model.encode_code(input_waveform=prompt_wav)
70 vq_code_prompt = vq_code_prompt[0, 0, :]
71 speech_ids_prefix = ids_to_speech_tokens(vq_code_prompt)
72
73 formatted_text = f"<|TEXT_UNDERSTANDING_START|>{input_text}<|TEXT_UNDERSTANDING_END|>"
74
75 chat = [
76 {"role": "user", "content": "Convert the text to speech:" + formatted_text},
77 {"role": "assistant", "content": "<|SPEECH_GENERATION_START|>" + "".join(speech_ids_prefix)}
78 ]
79
80 input_ids = tokenizer.apply_chat_template(chat, tokenize=True, return_tensors="pt", continue_final_message=True)
81 input_ids = input_ids.to("cuda")
82 speech_end_id = tokenizer.convert_tokens_to_ids("<|SPEECH_GENERATION_END|>")
83
84 outputs = model.generate(
85 input_ids,
86 max_length=2048,
87 eos_token_id=speech_end_id,
88 do_sample=True,
89 top_p=1,
90 temperature=0.8,
91 min_new_tokens=4, # Fix so the model does not directly stop
92 )
93
94 generated_ids = outputs[0][input_ids.shape[1] - len(speech_ids_prefix) : -1]
95
96 speech_tokens = tokenizer.batch_decode(generated_ids, skip_special_tokens=True)
97 speech_tokens = extract_speech_ids(speech_tokens)
98 speech_tokens = torch.tensor(speech_tokens).cuda().unsqueeze(0).unsqueeze(0)
99
100 gen_wav = Codec_model.decode_code(speech_tokens)
101 gen_wav = gen_wav[:, :, prompt_wav.shape[1] :]
102 sf.write(output_filename, gen_wav[0, 0, :].cpu().numpy(), 16000)