Fine-tuned
Veena TTS for
Hinglish (Hindi-English code-mixed) text-to-speech synthesis.
1pip install transformers torch snac soundfile
2pip install bitsandbytes # optional, for 4-bit quantized inference
1import torch
2from transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig
3from snac import SNAC
4import soundfile as sf
5
6# --- Load models ---
7quantization_config = BitsAndBytesConfig(
8 load_in_4bit=True,
9 bnb_4bit_quant_type="nf4",
10 bnb_4bit_compute_dtype=torch.bfloat16,
11 bnb_4bit_use_double_quant=True,
12)
13
14model = AutoModelForCausalLM.from_pretrained(
15 "akh-mysterio/veena-hinglish",
16 quantization_config=quantization_config, # remove for full fp16
17 device_map="auto",
18 trust_remote_code=True,
19)
20tokenizer = AutoTokenizer.from_pretrained("akh-mysterio/veena-hinglish", trust_remote_code=True)
21snac_model = SNAC.from_pretrained("hubertsiuzdak/snac_24khz").eval().cuda()
22
23# --- Control tokens ---
24START_OF_SPEECH_TOKEN = 128257
25END_OF_SPEECH_TOKEN = 128258
26START_OF_HUMAN_TOKEN = 128259
27END_OF_HUMAN_TOKEN = 128260
28START_OF_AI_TOKEN = 128261
29END_OF_AI_TOKEN = 128262
30AUDIO_CODE_BASE_OFFSET = 128266
31
32
33def decode_snac_tokens(snac_tokens, snac_model):
34 """De-interleave and decode SNAC tokens to audio waveform."""
35 if not snac_tokens or len(snac_tokens) % 7 != 0:
36 return None
37
38 snac_device = next(snac_model.parameters()).device
39 offsets = [AUDIO_CODE_BASE_OFFSET + i * 4096 for i in range(7)]
40
41 codes = [[] for _ in range(3)]
42 for i in range(0, len(snac_tokens), 7):
43 codes[0].append(snac_tokens[i] - offsets[0]) # coarse
44 codes[1].append(snac_tokens[i + 1] - offsets[1]) # medium
45 codes[1].append(snac_tokens[i + 4] - offsets[4])
46 codes[2].append(snac_tokens[i + 2] - offsets[2]) # fine
47 codes[2].append(snac_tokens[i + 3] - offsets[3])
48 codes[2].append(snac_tokens[i + 5] - offsets[5])
49 codes[2].append(snac_tokens[i + 6] - offsets[6])
50
51 hierarchical = [
52 torch.tensor(c, dtype=torch.int32, device=snac_device).unsqueeze(0)
53 for c in codes
54 ]
55 with torch.no_grad():
56 audio_hat = snac_model.decode(hierarchical)
57 return audio_hat.squeeze().clamp(-1, 1).cpu().numpy()
58
59
60def generate_speech(text, speaker="kavya", temperature=0.4, top_p=0.9):
61 """Generate speech audio from text."""
62 prompt = f"<spk_{speaker}> {text}"
63 prompt_tokens = tokenizer.encode(prompt, add_special_tokens=False)
64
65 input_tokens = [
66 START_OF_HUMAN_TOKEN,
67 *prompt_tokens,
68 END_OF_HUMAN_TOKEN,
69 START_OF_AI_TOKEN,
70 START_OF_SPEECH_TOKEN,
71 ]
72 input_ids = torch.tensor([input_tokens], device=model.device)
73 max_tokens = min(int(len(text) * 1.3) * 7 + 21, 700)
74
75 with torch.no_grad():
76 output = model.generate(
77 input_ids,
78 max_new_tokens=max_tokens,
79 do_sample=True,
80 temperature=temperature,
81 top_p=top_p,
82 repetition_penalty=1.05,
83 pad_token_id=tokenizer.pad_token_id,
84 eos_token_id=[END_OF_SPEECH_TOKEN, END_OF_AI_TOKEN],
85 )
86
87 generated_ids = output[0][len(input_tokens):].tolist()
88 snac_tokens = [
89 t for t in generated_ids
90 if AUDIO_CODE_BASE_OFFSET <= t < (AUDIO_CODE_BASE_OFFSET + 7 * 4096)
91 ]
92 return decode_snac_tokens(snac_tokens, snac_model)
93
94
95# --- Generate ---
96audio = generate_speech(
97 "Aaj mausam bohot acha hai, chalo bahar chalte hain!",
98 speaker="kavya",
99)
100if audio is not None:
101 sf.write("output.wav", audio, 24000)
Fine-tuned from
maya-research/Veena using LoRA on Hinglish speech data.
Hindi-English code-mixing ("Hinglish") is the dominant spoken register across urban India. The base Veena model handles Hindi and English separately but struggles with natural code-switching. This fine-tune bridges that gap.
+13% relative improvement in perceived speech quality on Hinglish evaluation set.
Full training, inference, and streaming code:
github.com/adola700/vee-ana
