Views
No views yet
| Model | Input modality | Oput modality | Download |
|---|---|---|---|
| Ming-flash-omni Preview | Image,text,video,audio | Image,text,audio | 🤗 HuggingFace 🤖 ModelScope |
pip install modelscope
modelscope download --model inclusionAI/Ming-flash-omni-Preview --local_dir inclusionAI/Ming-flash-omni-Preview --revision master1import os
2import torch
3import warnings
4from bisect import bisect_left
5warnings.filterwarnings("ignore")
6
7from transformers import AutoProcessor
8from modeling_bailingmm2 import BailingMM2NativeForConditionalGeneration
9
10def split_model():
11 device_map = {}
12 world_size = torch.cuda.device_count()
13 num_layers = 32
14 layer_per_gpu = num_layers // world_size
15 layer_per_gpu = [i * layer_per_gpu for i in range(1, world_size + 1)]
16 for i in range(num_layers):
17 device_map[f'model.model.layers.{i}'] = bisect_left(layer_per_gpu, i)
18 device_map['vision'] = 0
19 device_map['audio'] = 0
20 device_map['linear_proj'] = 0
21 device_map['linear_proj_audio'] = 0
22 device_map['model.model.word_embeddings.weight'] = 0
23 device_map['model.model.norm.weight'] = 0
24 device_map['model.lm_head.weight'] = 0
25 device_map['model.model.norm'] = 0
26 device_map[f'model.model.layers.{num_layers - 1}'] = 0
27 return device_map
28
29# Load pre-trained model with optimized settings, this will take ~10 minutes
30model_path = "inclusionAI/Ming-flash-omni-Preview"
31model = BailingMM2NativeForConditionalGeneration.from_pretrained(
32 model_path,
33 torch_dtype=torch.bfloat16,
34 attn_implementation="flash_attention_2",
35 device_map=split_model(),
36 load_image_gen=True,
37 load_talker=True,
38).to(dtype=torch.bfloat16)
39
40# Initialize processor for handling multimodal inputs
41processor = AutoProcessor.from_pretrained(model_path, trust_remote_code=True)
42
43# Inference Pipeline
44def generate(messages, processor, model, sys_prompt_exp=None, use_cot_system_prompt=False, max_new_tokens=512):
45 text = processor.apply_chat_template(
46 messages,
47 sys_prompt_exp=sys_prompt_exp,
48 use_cot_system_prompt=use_cot_system_prompt
49 )
50 image_inputs, video_inputs, audio_inputs = processor.process_vision_info(messages)
51
52 inputs = processor(
53 text=[text],
54 images=image_inputs,
55 videos=video_inputs,
56 audios=audio_inputs,
57 return_tensors="pt",
58 audio_kwargs={"use_whisper_encoder": True},
59 ).to(model.device)
60
61 for k in inputs.keys():
62 if k == "pixel_values" or k == "pixel_values_videos" or k == "audio_feats":
63 inputs[k] = inputs[k].to(dtype=torch.bfloat16)
64
65 with torch.no_grad():
66 generated_ids = model.generate(
67 **inputs,
68 max_new_tokens=max_new_tokens,
69 use_cache=True,
70 eos_token_id=processor.gen_terminator,
71 num_logits_to_keep=1,
72 )
73
74 generated_ids_trimmed = [
75 out_ids[len(in_ids):] for in_ids, out_ids in zip(inputs.input_ids, generated_ids)
76 ]
77
78 output_text = processor.batch_decode(
79 generated_ids_trimmed, skip_special_tokens=True, clean_up_tokenization_spaces=False
80 )[0]
81
82 return output_text
83
84# qa
85messages = [
86 {
87 "role": "HUMAN",
88 "content": [
89 {"type": "text", "text": "请详细介绍鹦鹉的生活习性。"}
90 ],
91 },
92]
93output_text = generate(messages, processor=processor, model=model)
94print(output_text)
95# Output:
96
97# 鹦鹉是一种非常聪明和社交性强的鸟类，它们的生活习性非常丰富和有趣。以下是一些关于鹦鹉生活习性的详细介绍：
98# ### 1. **栖息地**
99# 鹦鹉主要分布在热带和亚热带地区，包括非洲、亚洲、澳大利亚和南美洲。它们通常生活在森林、草原、沙漠和城市环境中。不同种类的鹦鹉对栖息地的要求有所不同，但大多数鹦鹉喜欢有丰富植被和水源的地方。
100# ### 2. **饮食**
101# 鹦鹉是杂食性动物，它们的饮食非常多样化。它们的食物包括种子、坚果、水果、蔬菜、花蜜和昆虫。鹦鹉的喙非常强壮，能够轻松地打开坚硬的果壳和坚果。一些鹦鹉还会吃泥土或沙子，以帮助消化和补充矿物质。
102# ......1
2@misc{Mingflash2025,
3 title = {Ming-Flash-Omni: A Sparse, Unified Architecture for Multimodal Perception and Generation},
4 author = {Inclusion AI},
5 year = {2025},
6 eprint = {2510.24821},
7 archivePrefix = {arXiv},
8 url = {https://arxiv.org/abs/2510.24821}
9}
10
11
12@misc{Mingomni2025,
13 title = {Ming-Omni: A Unified Multimodal Model for Perception and Generation},
14 author = {Inclusion AI},
15 year = {2025},
16 eprint = {2506.09344},
17 archivePrefix = {arXiv},
18 url = {https://arxiv.org/abs/2506.09344}
19}