Views
No views yet
1
2from transformers import AutoModel
3
4HF_TOKEN = "hf_..." # your huggingface token for downloading Llama3 from official Meta repo
5
6model = AutoModel.from_pretrained("DeSTA-ntu/DeSTA2-8B-beta", trust_remote_code=True, token=HF_TOKEN)
7
8messages = [
9 {"role": "system", "content": "You are a helpful voice assistant."},
10 {"role": "audio", "content": "<path_to_audio_file>"},
11 {"role": "user", "content": "Describe the audio."}
12 ]
13
14generated_ids = model.chat(
15 messages,
16 max_new_tokens=128,
17 do_sample=True,
18 temperature=0.6,
19 top_p=0.9
20)
21
22response = model.tokenizer.batch_decode(generated_ids, skip_special_tokens=True)[0]
23print(response)@article{lu2024developing,
title={Developing Instruction-Following Speech Language Model Without Speech Instruction-Tuning Data},
author={Lu, Ke-Han and Chen, Zhehuai and Fu, Szu-Wei and Yang, Chao-Han Huck and Balam, Jagadeesh and Ginsburg, Boris and Wang, Yu-Chiang Frank and Lee, Hung-yi},
journal={arXiv preprint arXiv:2409.20007},
year={2024}
}
@inproceedings{lu24c_interspeech,
title = {DeSTA: Enhancing Speech Language Models through Descriptive Speech-Text Alignment},
author = {Ke-Han Lu and Zhehuai Chen and Szu-Wei Fu and He Huang and Boris Ginsburg and Yu-Chiang Frank Wang and Hung-yi Lee},
year = {2024},
booktitle = {Interspeech 2024},
pages = {4159--4163},
doi = {10.21437/Interspeech.2024-457},
issn = {2958-1796},
}