Views
No views yet
checkpoints/step-006000-epoch-01-loss=0.1724.pt: Complete model checkpoint for direct evaluation.results_step-006000-epoch-01-loss=0.1724_instruct_cot_2-3: Results on SimplerEnv-Instruct with multimodal reasoning.results_step-006000-epoch-01-loss=0.1724_simpler_1-3: Results on SimplerEnv.vlmeval: Multimodal performance1
2import torch
3from vla.instructvla_eagle_dual_sys_v2_meta_query_v2 import load, load_vla
4from PIL import Image
5import numpy as np
6
7model_path = 'outputs/release_ckpts/instructvla_finetune_v2_xlora_freeze_head_instruction--image_aug/checkpoints/step-013500-epoch-01-loss=0.1093.pt'
8
9# Load Stage-2 (Generalist) model
10model = load_vla(model_path, stage="stage2").eval().to(torch.bfloat16).cuda()
11
12messages = [
13 {"content": "You are a helpful assistant."}, # system
14 {
15 "role": "user",
16 "content": "Can you describe the main idea of this image?",
17 "image": [{'np_array': np.asarray(Image.open("./asset/teaser.png"))}]
18 }
19]
20
21# Preprocess input
22inputs = model.processor.prepare_input(dict(prompt=messages))
23autocast_dtype = torch.bfloat16
24
25with torch.autocast("cuda", dtype=autocast_dtype, enabled=True):
26 output = model.vlm.generate(
27 input_ids=inputs['input_ids'].cuda(),
28 attention_mask=inputs['attention_mask'].cuda(),
29 pixel_values=inputs['pixel_values'].cuda(),
30 max_new_tokens=200,
31 output_hidden_states=False,
32 )
33
34response = model.processor.tokenizer.decode(output[0])
35print(response)
36