Views
No views yet
Self-generated data is what you need for developing general-purpose LALMs!
1git clone https://github.com/kehanlu/DeSTA2.5-Audio.git
2cd DeSTA2.5-Audio
3pip install -e .1from desta import DeSTA25AudioModel
2
3# Load the model from Hugging Face
4model = DeSTA25AudioModel.from_pretrained("DeSTA-ntu/DeSTA2.5-Audio-Llama-3.1-8B")
5model.to("cuda")
6
7# Run inference with audio input
8messages = [
9 {
10 "role": "system",
11 "content": "Focus on the audio clips and instructions."
12 },
13 {
14 "role": "user",
15 "content": "<|AUDIO|>\nDescribe this audio.",
16 "audios": [{
17 "audio": "/path/to/audio.wav", # Path to your audio file
18 "text": None
19 }]
20 }
21]
22
23outputs = model.generate(
24 messages=messages,
25 do_sample=False,
26 top_p=1.0,
27 temperature=1.0,
28 max_new_tokens=512
29)
30
31print(outputs.text)1@article{lu2025desta25Audio,
2 title={DeSTA2.5-Audio: Toward General-Purpose Large Audio Language Model with Self-Generated Cross-Modal Alignment},
3 author={Lu, Ke-Han and Chen, Zhehuai and Fu, Szu-Wei and Yang, Chao-Han Huck and Huang, Sung-Feng and Yang, Chih-Kai and Yu, Chee-En and Chen, Chun-Wei and Chen, Wei-Chih and Huang, Chien-yu and others},
4 journal={arXiv preprint arXiv:2507.02768},
5 year={2025}
6}
7
8@inproceedings{lu2025developing,
9 title={Developing instruction-following speech language model without speech instruction-tuning data},
10 author={Lu, Ke-Han and Chen, Zhehuai and Fu, Szu-Wei and Yang, Chao-Han Huck and Balam, Jagadeesh and Ginsburg, Boris and Wang, Yu-Chiang Frank and Lee, Hung-yi},
11 booktitle={ICASSP 2025-2025 IEEE International Conference on Acoustics, Speech and Signal Processing (ICASSP)},
12 pages={1--5},
13 year={2025},
14 organization={IEEE}
15}
16
17@inproceedings{lu24c_interspeech,
18 title = {DeSTA: Enhancing Speech Language Models through Descriptive Speech-Text Alignment},
19 author = {Ke-Han Lu and Zhehuai Chen and Szu-Wei Fu and He Huang and Boris Ginsburg and Yu-Chiang Frank Wang and Hung-yi Lee},
20 year = {2024},
21 booktitle = {Interspeech 2024},
22 pages = {4159--4163},
23 doi = {10.21437/Interspeech.2024-457},
24 issn = {2958-1796},
25}