Views
No views yet
suno-ai/spark-ttsdeepvk/NonverbalTTS1pip install --no-deps bitsandbytes accelerate xformers==0.0.29.post3 peft trl triton cut_cross_entropy unsloth_zoo
2pip install sentencepiece protobuf "datasets>=3.4.1,<4.0.0" "huggingface_hub>=0.34.0" hf_transfer
3pip install --no-deps unsloth
4git clone https://github.com/SparkAudio/Spark-TTS
5pip install omegaconf einx1import torch
2import re
3import numpy as np
4from typing import Dict, Any
5import torchaudio.transforms as T
6from unsloth import FastModel
7import sys
8sys.path.append('Spark-TTS')
9from sparktts.models.audio_tokenizer import BiCodecTokenizer
10from huggingface_hub import snapshot_download
11
12# Download model and code
13snapshot_download("yasserrmd/SparkNV-Voice", local_dir = "SparkNV-Voice")
14
15
16max_seq_length = 2048 # Choose any for long context!
17model, tokenizer = FastModel.from_pretrained(
18 model_name = "SparkNV-Voice",
19 max_seq_length = max_seq_length,
20 dtype = torch.float32, # Spark seems to only work on float32 for now
21 full_finetuning = True, # We support full finetuning now!
22 load_in_4bit = False,
23 #token = "hf_...", # use one if using gated models like meta-llama/Llama-2-7b-hf
24)
25
26FastModel.for_inference(model) # Enable native 2x faster inference
27
28audio_tokenizer = BiCodecTokenizer("SparkNV-Voice", "cuda")
29audio_tokenizer.model.to("cuda")
30
31input_text = "Hey there, my name is Yasser, and I'm a 🌬️ speech generation model that can sound like a person."
32chosen_voice = None # None for single-speaker
33
34@torch.inference_mode()
35def generate_speech_from_text(
36 text: str,
37 temperature: float = 0.8, # Generation temperature
38 top_k: int = 50, # Generation top_k
39 top_p: float = 1, # Generation top_p
40 max_new_audio_tokens: int = 2048, # Max tokens for audio part
41 device: torch.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
42) -> np.ndarray:
43 """
44 Generates speech audio from text using default voice control parameters.
45
46 Args:
47 text (str): The text input to be converted to speech.
48 temperature (float): Sampling temperature for generation.
49 top_k (int): Top-k sampling parameter.
50 top_p (float): Top-p (nucleus) sampling parameter.
51 max_new_audio_tokens (int): Max number of new tokens to generate (limits audio length).
52 device (torch.device): Device to run inference on.
53
54 Returns:
55 np.ndarray: Generated waveform as a NumPy array.
56 """
57
58 torch.compiler.reset()
59
60 prompt = "".join([
61 "<|task_tts|>",
62 "<|start_content|>",
63 text,
64 "<|end_content|>",
65 "<|start_global_token|>"
66 ])
67
68 model_inputs = tokenizer([prompt], return_tensors="pt").to(device)
69
70 print("Generating token sequence...")
71 generated_ids = model.generate(
72 **model_inputs,
73 max_new_tokens=max_new_audio_tokens, # Limit generation length
74 do_sample=True,
75 temperature=temperature,
76 top_k=top_k,
77 top_p=top_p,
78 eos_token_id=tokenizer.eos_token_id, # Stop token
79 pad_token_id=tokenizer.pad_token_id # Use models pad token id
80 )
81 print("Token sequence generated.")
82
83
84 generated_ids_trimmed = generated_ids[:, model_inputs.input_ids.shape[1]:]
85
86
87 predicts_text = tokenizer.batch_decode(generated_ids_trimmed, skip_special_tokens=False)[0]
88 # print(f"\nGenerated Text (for parsing):\n{predicts_text}\n") # Debugging
89
90 # Extract semantic token IDs using regex
91 semantic_matches = re.findall(r"<\|bicodec_semantic_(\d+)\|>", predicts_text)
92 if not semantic_matches:
93 print("Warning: No semantic tokens found in the generated output.")
94 # Handle appropriately - perhaps return silence or raise error
95 return np.array([], dtype=np.float32)
96
97 pred_semantic_ids = torch.tensor([int(token) for token in semantic_matches]).long().unsqueeze(0) # Add batch dim
98
99 # Extract global token IDs using regex (assuming controllable mode also generates these)
100 global_matches = re.findall(r"<\|bicodec_global_(\d+)\|>", predicts_text)
101 if not global_matches:
102 print("Warning: No global tokens found in the generated output (controllable mode). Might use defaults or fail.")
103 pred_global_ids = torch.zeros((1, 1), dtype=torch.long)
104 else:
105 pred_global_ids = torch.tensor([int(token) for token in global_matches]).long().unsqueeze(0) # Add batch dim
106
107 pred_global_ids = pred_global_ids.unsqueeze(0) # Shape becomes (1, 1, N_global)
108
109 print(f"Found {pred_semantic_ids.shape[1]} semantic tokens.")
110 print(f"Found {pred_global_ids.shape[2]} global tokens.")
111
112
113 # 5. Detokenize using BiCodecTokenizer
114 print("Detokenizing audio tokens...")
115 # Ensure audio_tokenizer and its internal model are on the correct device
116 audio_tokenizer.device = device
117 audio_tokenizer.model.to(device)
118 # Squeeze the extra dimension from global tokens as seen in SparkTTS example
119 wav_np = audio_tokenizer.detokenize(
120 pred_global_ids.to(device).squeeze(0), # Shape (1, N_global)
121 pred_semantic_ids.to(device) # Shape (1, N_semantic)
122 )
123 print("Detokenization complete.")
124
125 return wav_np
126
127if __name__ == "__main__":
128 print(f"Generating speech for: '{input_text}'")
129 text = f"{chosen_voice}: " + input_text if chosen_voice else input_text
130 generated_waveform = generate_speech_from_text(input_text)
131
132 if generated_waveform.size > 0:
133 import soundfile as sf
134 output_filename = "generated_speech_controllable.wav"
135 sample_rate = audio_tokenizer.config.get("sample_rate", 16000)
136 sf.write(output_filename, generated_waveform, sample_rate)
137 print(f"Audio saved to {output_filename}")
138
139 # Optional: Play in notebook
140 from IPython.display import Audio, display
141 display(Audio(generated_waveform, rate=sample_rate))
142 else:
143 print("Audio generation failed (no tokens found?).")NonverbalTTSsuno-ai/spark-ttsdeepvk/NonverbalTTS@yasserrmd