Views
No views yet
pip install vllm==0.11.01vllm serve [model_name] \
2 --host 127.0.0.1 \
3 --port 8000 \
4 --max_model_len 6400 \
5 --chat-template-content-format auto \
6 --limit-mm-per-prompt.image 15 \
7 --served-model-name "vlm_agent"1system_message = """You are a Vision Language Model specialized in processing the first person view images of embodied robots.
2Your task is to analyze the provided image and respond to queries with answers. Focus on the spatial relations in the image and make the right decisions."""1sample_data = {
2 "query": (
3 "Given the following instruction, a series of sampled historical observation "
4 "and recent observation image frames, predict a usable action sequence that "
5 "you should perform next. Output format: "
6 "'Turn [direction] [degrees] degrees; "
7 "Look [direction] [degrees] degrees; "
8 "Move [direction] [distance] meters; "
9 "[direction] sidewalk [distance] meters; "
10 "[manipulation action text]; "
11 "[interaction action text]; "
12 "Stop and no action'.\n\n"
13 "Your task is: " + instruction
14 )
15}1def sample_n_elements(my_list, n):
2 if n > len(my_list):
3 return my_list
4 if n <= 0:
5 return []
6
7 interval = len(my_list) / n
8 sampled_elements = []
9 for i in range(n):
10 index = int(round(i * interval))
11 if index < len(my_list):
12 sampled_elements.append(my_list[index])
13 return sampled_elements
14
15def format_data_sft_with_step_and_hist(sample, system_message):
16 formatted_sample = [
17 {
18 "role": "system",
19 "content": [{"type": "text", "text": system_message}],
20 }
21 ]
22
23 tmp_content = [
24 {"type": "text", "text": sample["query"] + "\n"},
25 {"type": "text", "text": "\nSampled Historical Observations:\n"},
26 ]
27
28 sampled_historical_images = sample_n_elements(sample["historical_images"], 10)
29 for img in sampled_historical_images:
30 tmp_content.append({
31 "type": "image",
32 "image": "file://" + img.replace("resized", "shrinked"),
33 })
34
35 tmp_content.append({"type": "text", "text": "\nRecent Observations:\n"})
36
37 pos = 0
38 for image, action in sample["image_action_pairs"]:
39 if pos == 0:
40 formatted_sample.append({
41 "role": "user",
42 "content": tmp_content + [
43 {"type": "image", "image": "file://" + image},
44 {"type": "text", "text": "Next action: "},
45 ],
46 })
47 else:
48 formatted_sample.append({
49 "role": "user",
50 "content": [
51 {"type": "image", "image": "file://" + image},
52 {"type": "text", "text": "Next action: "},
53 ],
54 })
55
56 if action.lower() != "none":
57 formatted_sample.append({
58 "role": "assistant",
59 "content": [{"type": "text", "text": action}],
60 })
61
62 pos += 1
63
64 return formatted_sample@article{bai2026EgoActor,
title={{E}go{A}ctor: {G}rounding Task Planning into Spatial-aware Egocentric Actions for Humanoid Robots via Visual-Language Models},
author={Yu Bai and Mingming Yu and Chaojie Li and Ziyi Bai and Xinlong Wang and Börje F. Karlsson},
journal={arXiv: 2602.04515},
year={2026},
url={https://arxiv.org/abs/2602.04515}
}