Views
No views yet
| Model | Base Model | HuggingFace | ModelScope |
|---|---|---|---|
| RynnBrain1.1-2B (This checkpoint) | Qwen3.5-2B | Link | Link |
| RynnBrain1.1-9B | Qwen3.5-9B | Link | Link |
| RynnBrain1.1-122B-A10B | Qwen3.5-122B-A10B | Link | Link |
pip install transformers==5.2.01import torch
2from transformers import AutoModelForImageTextToText, AutoProcessor
3
4conversation = [
5 {
6 "role": "user",
7 "content": [
8 {"type": "image", "image": "cookbooks/assets/object_location/images/000000086408.jpg"},
9 {
10 "type": "text",
11 "text": "What appliance can be used to heat food quickly.\nGenerate coordinates for one object bounding box. Constraints: x1,y1,x2,y2 in [0,1000]. Response must be in the format: <object> (x1, y1), (x2, y2) </object>",
12 },
13 ],
14 }
15]
16
17model_path = "Alibaba-DAMO-Academy/RynnBrain1.1-2B"
18processor = AutoProcessor.from_pretrained(model_path)
19
20model = AutoModelForImageTextToText.from_pretrained(
21 model_path,
22 dtype=torch.bfloat16,
23)
24model.to("cuda")
25
26model_inputs = processor.apply_chat_template(
27 conversation,
28 add_generation_prompt=True,
29 enable_thinking=False,
30 tokenize=True,
31 return_dict=True,
32 return_tensors="pt",
33)
34model_inputs = model_inputs.to("cuda")
35
36output_ids = model.generate(
37 **model_inputs,
38 max_new_tokens=256,
39 do_sample=False,
40)
41output_ids = output_ids[:, model_inputs["input_ids"].size(1) :]
42response = processor.decode(output_ids[0], skip_special_tokens=True)
43print(response)1# launch server
2python3 -m sglang.launch_server --model-path Alibaba-DAMO-Academy/RynnBrain1.1-2B --host 0.0.0.0 --port 80001# inference using openai api
2import base64
3import io
4
5from openai import OpenAI
6from PIL import Image
7
8def pil_to_url(image: Image.Image):
9 image_format = image.format if image.format else 'PNG'
10 buffered = io.BytesIO()
11 image.save(buffered, format=image_format)
12 img_str = base64.b64encode(buffered.getvalue()).decode('utf-8')
13 return f'data:image/{image_format.lower()};base64,{img_str}'
14
15messages = [
16 {
17 'role': 'user',
18 'content': [
19 {'type': 'image_url', 'image_url': {'url': pil_to_url(Image.open('cookbooks/assets/object_location/images/000000086408.jpg'))}},
20 {'type': 'text', 'text': 'What appliance can be used to heat food quickly.\nGenerate coordinates for one object bounding box. Constraints: x1,y1,x2,y2 ∈ [0,1000]. Response must be in the format: <object> (x1, y1), (x2, y2) </object>'},
21 ],
22 }
23]
24
25client = OpenAI(api_key="", base_url="http://localhost:8000/v1")
26response = client.chat.completions.create(
27 model="default",
28 messages=messages,
29 stream=False,
30).choices[0].message.content
31print(response)1import sglang as sgl
2from transformers import AutoProcessor
3
4def main():
5 conversation = [
6 {
7 'role': 'user',
8 'content': [
9 {'type': 'image'},
10 {'type': 'text', 'text': 'What appliance can be used to heat food quickly.\nGenerate coordinates for one object bounding box. Constraints: x1,y1,x2,y2 ∈ [0,1000]. Response must be in the format: <object> (x1, y1), (x2, y2) </object>'},
11 ],
12 }
13 ]
14
15 model_path = 'Alibaba-DAMO-Academy/RynnBrain1.1-2B'
16 llm = sgl.Engine(model_path=model_path)
17 processor = AutoProcessor.from_pretrained(model_path)
18
19 prompt = processor.apply_chat_template(
20 conversation,
21 add_generation_prompt=True,
22 tokenize=False,
23 )
24
25 output = llm.generate(
26 prompt=prompt,
27 image_data='cookbooks/assets/object_location/images/000000086408.jpg',
28 sampling_params={"temperature": 0.8, "top_p": 0.95},
29 )
30 print(f"Prompt: {prompt}\nGenerated text: {output['text']}")
31
32if __name__ == '__main__':
33 main()| Category | Cookbook name | Description |
|---|---|---|
| Spatial Understanding | 1_spatial_understanding.ipynb | Shows the model's ability for spatial understanding in the video scene. |
| Object Understanding | 2_object_understanding.ipynb | Shows how the model understands object categories, attributes, and relations and counting ability. |
| Object Grounding | 3_object_grounding.ipynb | Locates specific objects with bounding boxes in an image or video based on instructions. |
| Area Location | 4_area_location.ipynb | Identifies and marks specified regions by points in an image or video. |
| Affordance Location | 5_affordance_location.ipynb | Finds areas or objects with specific affordances in an image or video. |
| Trajectory Location | 6_trajectory_location.ipynb | Infers and annotates trajectories or motion paths in an image or video. |
| 🆕 Contact Point Prediction | 7_contact_point_prediction.ipynb | Predicts an instruction-conditioned contact point and in-plane orientation from an image. |
| 🆕 3D Grounding | 8_3d_grounding.ipynb | Predicts 3D bounding boxes (position, dimensions, orientation) from a single RGB image with camera intrinsics. |
1@article{damo2026rynnbrain,
2 title={RynnBrain: Open Embodied Foundation Models},
3 author={Ronghao Dang, Jiayan Guo, Bohan Hou, Sicong Leng, Kehan Li, Xin Li, Jiangpin Liu, Yunxuan Mao, Zhikai Wang, Yuqian Yuan, Minghao Zhu, Xiao Lin, Yang Bai, Qian Jiang, Yaxi Zhao, Minghua Zeng, Junlong Gao, Yuming Jiang, Jun Cen, Siteng Huang, Liuyi Wang, Wenqiao Zhang, Chengju Liu, Jianfei Yang, Shijian Lu, Deli Zhao},
4 journal={arXiv preprint arXiv:2602.14979v1},
5 year={2026},
6 url = {https://arxiv.org/abs/2602.14979v1}
7}
8
9@article{damo2026rynnbrain11,
10 title={RynnBrain 1.1: Towards More Capable and Generalizable Embodied Foundation Model},
11 author={Kehan Li, Bohan Hou, Minghao Zhu, Tianyi Zhang, Zesen Cheng, Zhikai Wang, Sicong Leng, Xin Li, Xiao Lin, Biying Yao, Minghua Zeng, Jiangpin Liu, Ronghao Dang, Jiayan Guo, Siteng Huang, Haoyu Zhao, Heng Ping, Yaxi Zhao, Kexiang Wang, Tong Lu, Shengke Xue, Jiahao Tang, Yulei Wang, Zejing Wang, Jianwei Gao, Shijian Lu, Chengju Liu, Jianfei Yang, Mingxiu Chen, Deli Zhao},
12 journal={arXiv preprint arXiv:2607.17977},
13 year={2026},
14 url = {https://arxiv.org/abs/2607.17977}
15}