Views
No views yet
canopylabs/3b-es_it-pretrain-research_release. The model supports multiple voices and nuanced emotions, trained using Unsloth and SNAC for audio tokenization.canopylabs/3b-es_it-pretrain-research_releasesource (emotion): textsirekist98/spanish_tts_noauddataset_24khz. We selected combinations of speaker (source) and emotion with at least 1000 samples, resulting in a balanced dataset of over 109,000 examples.source (emotion): text1import torch
2from transformers import AutoTokenizer, AutoModelForCausalLM
3from peft import PeftModel
4from snac import SNAC
5
6# --- Minimal config ---
7device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
8BASE = "canopylabs/3b-es_it-pretrain-research_release"
9LORA = "sirekist98/spanish_tts_emotions"
10SNAC_ID = "hubertsiuzdak/snac_24khz"
11
12VOICE = "alloy"
13EMOTION_ID = "intense_fear_dread_apprehension_horror_terror_panic"
14TEXT = "Estoy atrapado, por favor ayúdame."
15prompt = f"{VOICE} ({EMOTION_ID}): {TEXT}"
16
17# --- Load models ---
18tokenizer = AutoTokenizer.from_pretrained(BASE)
19base_model = AutoModelForCausalLM.from_pretrained(
20 BASE,
21 torch_dtype=torch.float16 if device.type == "cuda" else torch.float32
22)
23model = PeftModel.from_pretrained(base_model, LORA).to(device).eval()
24snac_model = SNAC.from_pretrained(SNAC_ID).to(device)
25
26# --- Prepare input (same as your Space) ---
27input_ids = tokenizer(prompt, return_tensors="pt").input_ids.to(device)
28start_tok = torch.tensor([[128259]], dtype=torch.long).to(device)
29end_toks = torch.tensor([[128009, 128260]], dtype=torch.long).to(device)
30
31input_ids = torch.cat([start_tok, input_ids, end_toks], dim=1)
32MAX_LEN = 4260
33pad_len = MAX_LEN - input_ids.shape[1]
34pad = torch.full((1, pad_len), 128263, dtype=torch.long).to(device)
35input_ids = torch.cat([pad, input_ids], dim=1)
36attention_mask = torch.cat(
37 [torch.zeros((1, pad_len), dtype=torch.long),
38 torch.ones((1, input_ids.shape[1] - pad_len), dtype=torch.long)],
39 dim=1
40).to(device)
41
42# --- Generate ---
43generated = model.generate(
44 input_ids=input_ids,
45 attention_mask=attention_mask,
46 max_new_tokens=1200,
47 do_sample=True,
48 temperature=0.6,
49 top_p=0.95,
50 repetition_penalty=1.1,
51 num_return_sequences=1,
52 eos_token_id=128258,
53 use_cache=True
54)
55
56# --- Post-process (find 128257, remove 128258, multiple of 7, subtract 128266) ---
57AUDIO_TOKEN_OFFSET = 128266
58token_to_find = 128257
59token_to_remove = 128258
60
61idxs = (generated == token_to_find).nonzero(as_tuple=True)
62cropped = generated[:, idxs[1][-1].item() + 1:] if len(idxs[1]) > 0 else generated
63cleaned = cropped[cropped != token_to_remove]
64codes = cleaned[: (len(cleaned) // 7) * 7].tolist()
65codes = [int(t) - AUDIO_TOKEN_OFFSET for t in codes]
66
67# --- SNAC decode (same layout as your Space) ---
68layer_1, layer_2, layer_3 = [], [], []
69for i in range((len(codes) + 1) // 7):
70 b = 7 * i
71 if b + 6 >= len(codes):
72 break
73 layer_1.append(codes[b + 0])
74 layer_2.append(codes[b + 1] - 4096)
75 layer_3.append(codes[b + 2] - 2 * 4096)
76 layer_3.append(codes[b + 3] - 3 * 4096)
77 layer_2.append(codes[b + 4] - 4 * 4096)
78 layer_3.append(codes[b + 5] - 5 * 4096)
79 layer_3.append(codes[b + 6] - 6 * 4096)
80
81dev_snac = snac_model.quantizer.quantizers[0].codebook.weight.device
82layers = [
83 torch.tensor(layer_1).unsqueeze(0).to(dev_snac),
84 torch.tensor(layer_2).unsqueeze(0).to(dev_snac),
85 torch.tensor(layer_3).unsqueeze(0).to(dev_snac),
86]
87
88with torch.no_grad():
89 audio = snac_model.decode(layers).squeeze().cpu().numpy()
90
91# 'audio' is the 24kHz waveform.
92# Optional:
93# from scipy.io.wavfile import write as write_wav
94# write_wav("output.wav", 24000, audio)source):alloy, ash, ballad, coral, echo, fable, nova, onyx, sage, shimmer, verse1@misc{sirekist2025spanishTTS,
2 author = {sirekist98},
3 title = {Spanish TTS Model with Emotions and Multiple Voices},
4 year = {2025},
5 howpublished = {\url{https://huggingface.co/sirekist98/spanish_model}}
6}