We have developed and released the family
Ichigo-llama3s . This family is natively understanding audio and text input.
We expand the Semantic tokens experiment with WhisperVQ as a tokenizer for audio files from
homebrewltd/mini-Ichigo-llama3.2-3B-s-base with nearly 1B tokens from
Instruction Speech WhisperVQ v3 dataset.
1 device = "cuda" if torch . cuda . is_available ( ) else "cpu"
2 if not os . path . exists ( "whisper-vq-stoks-medium-en+pl-fixed.model" ) :
3 hf_hub_download (
4 repo_id = "jan-hq/WhisperVQ" ,
5 filename = "whisper-vq-stoks-medium-en+pl-fixed.model" ,
6 local_dir = "." ,
7 )
8 vq_model = RQBottleneckTransformer . load_model (
9 "whisper-vq-stoks-medium-en+pl-fixed.model"
10 ) . to ( device )
11 vq_model . ensure_whisper ( device )
12 def audio_to_sound_tokens ( audio_path , target_bandwidth = 1.5 , device = device ) :
13
14 wav , sr = torchaudio . load ( audio_path )
15 if sr != 16000 :
16 wav = torchaudio . functional . resample ( wav , sr , 16000 )
17 with torch . no_grad ( ) :
18 codes = vq_model . encode_audio ( wav . to ( device ) )
19 codes = codes [ 0 ] . cpu ( ) . tolist ( )
20
21 result = '' . join ( f'<|sound_ { num : 04d } |>' for num in codes )
22 return f'<|sound_start|> { result } <|sound_end|>'
Then, we can inference the model the same as any other LLM.
1 def setup_pipeline ( model_path , use_4bit = False , use_8bit = False ) :
2 tokenizer = AutoTokenizer . from_pretrained ( model_path )
3
4 model_kwargs = { "device_map" : "auto" }
5
6 if use_4bit :
7 model_kwargs [ "quantization_config" ] = BitsAndBytesConfig (
8 load_in_4bit = True ,
9 bnb_4bit_compute_dtype = torch . bfloat16 ,
10 bnb_4bit_use_double_quant = True ,
11 bnb_4bit_quant_type = "nf4" ,
12 )
13 elif use_8bit :
14 model_kwargs [ "quantization_config" ] = BitsAndBytesConfig (
15 load_in_8bit = True ,
16 bnb_8bit_compute_dtype = torch . bfloat16 ,
17 bnb_8bit_use_double_quant = True ,
18 )
19 else :
20 model_kwargs [ "torch_dtype" ] = torch . bfloat16
21
22 model = AutoModelForCausalLM . from_pretrained ( model_path , ** model_kwargs )
23
24 return pipeline ( "text-generation" , model = model , tokenizer = tokenizer )
25
26 def generate_text ( pipe , messages , max_new_tokens = 64 , temperature = 0.0 , do_sample = False ) :
27 generation_args = {
28 "max_new_tokens" : max_new_tokens ,
29 "return_full_text" : False ,
30 "temperature" : temperature ,
31 "do_sample" : do_sample ,
32 }
33
34 output = pipe ( messages , ** generation_args )
35 return output [ 0 ] [ 'generated_text' ]
36
37 # Usage
38 llm_path = "homebrewltd/llama3.1-s-instruct-v0.2"
39 pipe = setup_pipeline ( llm_path , use_8bit = True )
We utilize
torchtune library for the latest FSDP2 training code implementation.
@article{Llama3-S: Sound Instruction Language Model 2024,
title={Llama3-S},
author={Homebrew Research},
year=2024,
month=August},
url={https://huggingface.co/homebrewltd/llama3.1-s-2024-08-20}
