Views
No views yet
1from align_anything.models.llama_vision_audio_model import (
2 LlamaVisionAudioForConditionalGeneration,
3 LlamaVisionAudioProcessor,
4)
5import torch
6import torchaudio
7from PIL import Image
8
9path = <path_to_model_dir>
10processor = LlamaVisionAudioProcessor.from_pretrained(path)
11model = LlamaVisionAudioForConditionalGeneration.from_pretrained(path)
12
13prompt = "<|start_header_id|>user<|end_header_id|>: Where is the capital of China?\n<|start_header_id|>assistant<|end_header_id|>: "
14
15inputs = processor(text=prompt, return_tensors="pt")
16outputs = model.generate(**inputs, max_new_tokens=1024)
17print(processor.decode(outputs[0], skip_special_tokens=True))
18
19prompt = "<|start_header_id|>user<|end_header_id|>: Summarize the audio's contents.<audio>\n<|start_header_id|>assistant<|end_header_id|>: "
20
21audio_path = "align-anything/assets/test_audio.wav"
22audio, _ = torchaudio.load(audio_path)
23if audio.shape[0] == 2:
24 audio = audio.mean(dim=0, keepdim=True)
25audio = audio.squeeze().tolist()
26
27inputs = processor(text=prompt, raw_speech=audio, return_tensors="pt")
28outputs = model.generate(**inputs, max_new_tokens=1024)
29print(processor.decode(outputs[0], skip_special_tokens=False))
30
31prompt = "<|start_header_id|>user<|end_header_id|>: <image> Give an overview of what's in the image.\n<|start_header_id|>assistant<|end_header_id|>: "
32image_path = "align-anything/assets/test_image.webp"
33image = Image.open(image_path)
34
35inputs = processor(text=prompt, images=image, return_tensors="pt")
36outputs = model.generate(**inputs, max_new_tokens=1024)
37print(processor.decode(outputs[0], skip_special_tokens=True))