Views
No views yet
checkpoints folder before running the script. I would suggest to use conda environment for this.pip install torch torchaudio --index-url https://download.pytorch.org/whl/cu121
pip install gradio tempfile transformers numpy1import gradio as gr
2from spiritlm.model.spiritlm_model import Spiritlm, OutputModality, GenerationInput, ContentType
3from transformers import GenerationConfig
4import torchaudio
5import torch
6import tempfile
7import os
8import numpy as np
9
10# Initialize the Spirit LM base model. You can change this to spirit-lm-expressive-7b for a more expressive experience.
11spirit_lm = Spiritlm("spirit-lm-base-7b")
12
13def generate_output(input_type, input_content_text, input_content_audio, output_modality, temperature, top_p, max_new_tokens, do_sample, speaker_id):
14 generation_config = GenerationConfig(
15 temperature=temperature,
16 top_p=top_p,
17 max_new_tokens=max_new_tokens,
18 do_sample=do_sample,
19 )
20
21 if input_type == "text":
22 interleaved_inputs = [GenerationInput(content=input_content_text, content_type=ContentType.TEXT)]
23 elif input_type == "audio":
24 # Load audio file
25 waveform, sample_rate = torchaudio.load(input_content_audio)
26 interleaved_inputs = [GenerationInput(content=waveform.squeeze(0), content_type=ContentType.SPEECH)]
27 else:
28 raise ValueError("Invalid input type")
29
30 outputs = spirit_lm.generate(
31 interleaved_inputs=interleaved_inputs,
32 output_modality=OutputModality[output_modality.upper()],
33 generation_config=generation_config,
34 speaker_id=speaker_id, # Pass the selected speaker ID
35 )
36
37 text_output = ""
38 audio_output = None
39
40 for output in outputs:
41 if output.content_type == ContentType.TEXT:
42 text_output = output.content
43 elif output.content_type == ContentType.SPEECH:
44 # Ensure output.content is a NumPy array
45 if isinstance(output.content, np.ndarray):
46 # Debugging: Print shape and dtype of the audio data
47 print("Audio data shape:", output.content.shape)
48 print("Audio data dtype:", output.content.dtype)
49
50 # Ensure the audio data is in the correct format
51 if len(output.content.shape) == 1:
52 # Mono audio data
53 audio_data = torch.from_numpy(output.content).unsqueeze(0)
54 else:
55 # Stereo audio data
56 audio_data = torch.from_numpy(output.content)
57
58 # Save the audio content to a temporary file
59 with tempfile.NamedTemporaryFile(delete=False, suffix=".wav") as temp_audio_file:
60 torchaudio.save(temp_audio_file.name, audio_data, 16000)
61 audio_output = temp_audio_file.name
62 else:
63 raise TypeError("Expected output.content to be a NumPy array, but got {}".format(type(output.content)))
64
65 return text_output, audio_output
66
67# Define the Gradio interface
68iface = gr.Interface(
69 fn=generate_output,
70 inputs=[
71 gr.Radio(["text", "audio"], label="Input Type", value="text"),
72 gr.Textbox(label="Input Content (Text)"),
73 gr.Audio(label="Input Content (Audio)", type="filepath"),
74 gr.Radio(["TEXT", "SPEECH", "ARBITRARY"], label="Output Modality", value="SPEECH"),
75 gr.Slider(0, 1, step=0.1, value=0.9, label="Temperature"),
76 gr.Slider(0, 1, step=0.05, value=0.95, label="Top P"),
77 gr.Slider(1, 800, step=1, value=500, label="Max New Tokens"),
78 gr.Checkbox(value=True, label="Do Sample"),
79 gr.Dropdown(choices=[0, 1, 2, 3], value=0, label="Speaker ID"),
80 ],
81 outputs=[gr.Textbox(label="Generated Text"), gr.Audio(label="Generated Audio")],
82 title="Spirit LM WebUI Demo",
83 description="Demo for generating text or audio using the Spirit LM model.",
84 flagging_mode="never",
85)
86
87# Launch the interface
88iface.launch()
89
90checkpoints/
├── README.md
├── speech_tokenizer
│ ├── hifigan_spiritlm_base
│ │ ├── config.json
│ │ ├── generator.pt
│ │ ├── speakers.txt
│ │ └── styles.txt
│ ├── hifigan_spiritlm_expressive_w2v2
│ │ ├── config.json
│ │ ├── generator.pt
│ │ └── speakers.txt
│ ├── hubert_25hz
│ │ ├── L11_quantizer_500.pt
│ │ └── mhubert_base_25hz.pt
│ ├── style_encoder_w2v2
│ │ ├── config.json
│ │ └── pytorch_model.bin
│ └── vqvae_f0_quantizer
│ ├── config.yaml
│ └── model.pt
└── spiritlm_model
├── spirit-lm-base-7b
│ ├── config.json
│ ├── generation_config.json
│ ├── pytorch_model.bin
│ ├── special_tokens_map.json
│ ├── tokenizer_config.json
│ └── tokenizer.model
└── spirit-lm-expressive-7b
├── config.json
├── generation_config.json
├── pytorch_model.bin
├── special_tokens_map.json
├── tokenizer_config.json
└── tokenizer.model