Views
No views yet
1device = "cuda" if torch.cuda.is_available() else "cpu"
2if not os.path.exists("whisper-vq-stoks-medium-en+pl-fixed.model"):
3 hf_hub_download(
4 repo_id="jan-hq/WhisperVQ",
5 filename="whisper-vq-stoks-medium-en+pl-fixed.model",
6 local_dir=".",
7 )
8vq_model = RQBottleneckTransformer.load_model(
9 "whisper-vq-stoks-medium-en+pl-fixed.model"
10 ).to(device)
11def audio_to_sound_tokens(audio_path, target_bandwidth=1.5, device=device):
12 vq_model.ensure_whisper(device)
13
14 wav, sr = torchaudio.load(audio_path)
15 if sr != 16000:
16 wav = torchaudio.functional.resample(wav, sr, 16000)
17 with torch.no_grad():
18 codes = vq_model.encode_audio(wav.to(device))
19 codes = codes[0].cpu().tolist()
20
21 result = ''.join(f'<|sound_{num:04d}|>' for num in codes)
22 return f'<|sound_start|>{result}<|sound_end|>'
23
24def audio_to_sound_tokens_transcript(audio_path, target_bandwidth=1.5, device=device):
25 vq_model.ensure_whisper(device)
26
27 wav, sr = torchaudio.load(audio_path)
28 if sr != 16000:
29 wav = torchaudio.functional.resample(wav, sr, 16000)
30 with torch.no_grad():
31 codes = vq_model.encode_audio(wav.to(device))
32 codes = codes[0].cpu().tolist()
33
34 result = ''.join(f'<|sound_{num:04d}|>' for num in codes)
35 return f'<|reserved_special_token_69|><|sound_start|>{result}<|sound_end|>'1def setup_pipeline(model_path, use_4bit=False, use_8bit=False):
2 tokenizer = AutoTokenizer.from_pretrained(model_path)
3
4 model_kwargs = {"device_map": "auto"}
5
6 if use_4bit:
7 model_kwargs["quantization_config"] = BitsAndBytesConfig(
8 load_in_4bit=True,
9 bnb_4bit_compute_dtype=torch.bfloat16,
10 bnb_4bit_use_double_quant=True,
11 bnb_4bit_quant_type="nf4",
12 )
13 elif use_8bit:
14 model_kwargs["quantization_config"] = BitsAndBytesConfig(
15 load_in_8bit=True,
16 bnb_8bit_compute_dtype=torch.bfloat16,
17 bnb_8bit_use_double_quant=True,
18 )
19 else:
20 model_kwargs["torch_dtype"] = torch.bfloat16
21
22 model = AutoModelForCausalLM.from_pretrained(model_path, **model_kwargs)
23
24 return pipeline("text-generation", model=model, tokenizer=tokenizer)
25
26def generate_text(pipe, messages, max_new_tokens=64, temperature=0.0, do_sample=False):
27 generation_args = {
28 "max_new_tokens": max_new_tokens,
29 "return_full_text": False,
30 "temperature": temperature,
31 "do_sample": do_sample,
32 }
33
34 output = pipe(messages, **generation_args)
35 return output[0]['generated_text']
36
37# Usage
38llm_path = "homebrewltd/llama3.1-s-instruct-v0.2"
39pipe = setup_pipeline(llm_path, use_8bit=True)
| Parameter | Continual Training |
|---|---|
| Epoch | 1 |
| Global batch size | 128 |
| Learning Rate | 0.5e-4 |
| Learning Scheduler | Cosine with warmup |
| Optimizer | Adam torch fused |
| Warmup Ratio | 0.01 |
| Weight Decay | 0.005 |
| Max Sequence Length | 512 |
@article{Llama3-S: Sound Instruction Language Model 2024,
title={Llama3-S},
author={Homebrew Research},
year=2024,
month=August},
url={https://huggingface.co/homebrewltd/llama3.1-s-2024-08-20}