CiSiMi is an early prototype of a text-to-audio model that can process text inputs and respond with both text and audio. Built for resource-constrained environments, it's designed to run efficiently on CPU using llama.cpp, making advanced speech synthesis accessible even without powerful GPUs.
This project demonstrates the power of open-source tools to create accessible speech technology. While still in its early stages, it represents a step toward democratizing advanced text-to-audio capabilities.
1pip install outetts llama-cpp-python --upgrade
2pip install huggingface_hub sounddevice
1import torch
2import outetts
3import numpy as np
4from huggingface_hub import hf_hub_download
5from outetts.wav_tokenizer.audio_codec import AudioCodec
6from outetts.version.v2.prompt_processor import PromptProcessor
7from outetts.version.playback import ModelOutput
8
9# Download the model
10model_path = hf_hub_download(
11 repo_id="KandirResearch/CiSiMi-v0.1",
12 filename="unsloth.Q8_0.gguf",
13)
14
15# Configure the model
16model_config = outetts.GGUFModelConfig_v2(
17 model_path=model_path,
18 tokenizer_path="KandirResearch/CiSiMi-v0.1",
19)
20
21# Initialize components
22interface = outetts.InterfaceGGUF(model_version="0.3", cfg=model_config)
23audio_codec = AudioCodec()
24prompt_processor = PromptProcessor("KandirResearch/CiSiMi-v0.1")
25
26device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
27gguf_model = interface.get_model()
28
29# Helper function to extract audio from tokens
30def get_audio(tokens):
31 outputs = prompt_processor.extract_audio_from_tokens(tokens)
32 if not outputs:
33 return None
34 audio_tensor = audio_codec.decode(torch.tensor([[outputs]], dtype=torch.int64).to(device))
35 return ModelOutput(audio_tensor, audio_codec.sr)
36
37# Helper function to clean text output
38def extract_text_from_tts_output(tts_output):
39 text = ""
40 for line in tts_output.strip().split('\n'):
41 if '<|audio_end|>' in line or '<|im_end|>' in line:
42 continue
43 if '<|' in line:
44 word = line.split('<|')[0].strip()
45 if word:
46 text += word + " "
47 else:
48 text += line.strip() + " "
49 return text.strip()
50
51# Generate response function
52def generate_response(instruction):
53 prompt = f"<|im_start|>\nInstructions:\n{instruction}\n<|im_end|>\nAnswer:\n"
54 gen_cfg = outetts.GenerationConfig(
55 text=prompt,
56 temperature=0.6,
57 repetition_penalty=1.1,
58 max_length=4096,
59 speaker=None
60 )
61
62 input_ids = prompt_processor.tokenizer.encode(prompt)
63 tokens = gguf_model.generate(input_ids, gen_cfg)
64
65 output_text = prompt_processor.tokenizer.decode(tokens, skip_special_tokens=False)
66
67 if "<|audio_end|>" in output_text:
68 first_part, _, _ = output_text.partition("<|audio_end|>")
69
70 if "<|audio_end|>\n<|im_end|>\n" not in first_part:
71 first_part += "<|audio_end|>\n<|im_end|>\n"
72
73 extracted_text = extract_text_from_tts_output(first_part)
74
75 audio_start_pos = first_part.find("<|audio_start|>\n") + len("<|audio_start|>\n")
76 audio_end_pos = first_part.find("<|audio_end|>\n<|im_end|>\n") + len("<|audio_end|>\n<|im_end|>\n")
77
78 if audio_start_pos >= len("<|audio_start|>\n") and audio_end_pos > audio_start_pos:
79 audio_tokens_text = first_part[audio_start_pos:audio_end_pos]
80 audio_tokens = prompt_processor.tokenizer.encode(audio_tokens_text)
81 audio_output = get_audio(audio_tokens)
82
83 if audio_output is not None and hasattr(audio_output, 'audio') and audio_output.audio is not None:
84 audio_numpy = audio_output.audio.cpu().numpy()
85 if audio_numpy.ndim > 1:
86 audio_numpy = audio_numpy.squeeze()
87
88 return extracted_text, (audio_output.sr, audio_numpy)
89
90 return output_text, None
91
92# Example usage
93question = "What is the meaning of life?"
94response_text, response_audio = generate_response(question)
95print(response_text)
96
97# Play audio if available
98if response_audio is not None:
99 if "ipykernel" in sys.modules:
100 from IPython.display import display, Audio
101 display(Audio(response_audio[1], rate=response_audio[0], autoplay=True))
102 else:
103 import sounddevice as sd
104 sd.play(response_audio[1], samplerate=response_audio[0])
105 sd.wait()
