Views
No views yet
diffusers from source (main) with MiniMax-H3 modular blocks, and a recent transformers that ships Qwen3VLForConditionalGeneration.num_hidden_layers > 50 because MiniMax-H3 reads hidden_states[50].attention_head_dim >= 6 * rope_freq_dim.decoder_dim >= 128 for the released 7-stage upsample stack.| File path | Size |
|---|---|
| audio_vae/diffusion_pytorch_model.safetensors | 66.7MB |
| text_encoder/model.safetensors | 12.1MB |
| transformer/diffusion_pytorch_model.safetensors | 0.5MB |
| transformer_ref/diffusion_pytorch_model.safetensors | 0.5MB |
| vae/diffusion_pytorch_model.safetensors | 4.5MB |
1import torch
2from diffusers import ModularPipeline
3
4model_id = "yujiepan/minimax-h3-tiny-random"
5device = 'cuda' if torch.cuda.is_available() else 'cpu'
6pipe = ModularPipeline.from_pretrained(model_id, workflow='t2va')
7pipe.load_components(dtype=torch.bfloat16)
8if device == 'cuda':
9 pipe.to(device)
10
11outputs = pipe(
12 prompt='A red fox trotting through a snowy pine forest',
13 num_frames=124,
14 height=64,
15 width=64,
16 num_inference_steps=2,
17 generator=torch.Generator(device=device).manual_seed(42),
18 output=['videos', 'audio', 'sampling_rate'],
19)
20print(type(outputs['videos'][0]), getattr(outputs['videos'][0], 'shape', None))
21print(type(outputs['audio'][0]), getattr(outputs['audio'][0], 'shape', None), outputs['sampling_rate'])1import json
2from pathlib import Path
3
4import torch
5from diffusers import (
6 AutoencoderKLMiniMaxH3,
7 AutoencoderKLMiniMaxH3Audio,
8 MiniMaxH3Blocks,
9 MiniMaxH3Scheduler,
10 MiniMaxH3Transformer3DModel,
11)
12from huggingface_hub import hf_hub_download
13from transformers import AutoConfig, AutoProcessor, AutoTokenizer, Qwen3VLForConditionalGeneration
14
15source_model_id = "MiniMaxAI/MiniMax-H3"
16save_folder = "/tmp/yujiepan/minimax-h3-tiny-random"
17
18def save_json(path, obj):
19 Path(path).parent.mkdir(parents=True, exist_ok=True)
20 with open(path, 'w', encoding='utf-8') as f:
21 json.dump(obj, f, indent=2, ensure_ascii=False)
22
23def init_weights(model):
24 torch.manual_seed(42)
25 model = model.cpu()
26 with torch.no_grad():
27 for name, p in sorted(model.named_parameters()):
28 torch.nn.init.normal_(p, 0, 0.1)
29 print(name, p.shape, p.dtype, p.device)
30
31torch.set_default_dtype(torch.bfloat16)
32text_dim = 32
33Path(save_folder).mkdir(parents=True, exist_ok=True)
34
35AutoTokenizer.from_pretrained(source_model_id, subfolder='tokenizer').save_pretrained(
36 f'{save_folder}/tokenizer'
37)
38AutoProcessor.from_pretrained(source_model_id, subfolder='processor').save_pretrained(
39 f'{save_folder}/processor'
40)
41
42with open(hf_hub_download(source_model_id, filename='text_encoder/config.json', repo_type='model'), 'r', encoding='utf-8') as f:
43 config = json.load(f)
44# MiniMax-H3 conditions on hidden_states[50], so keep >50 layers with a tiny width.
45config['text_config'].update({
46 'head_dim': 8,
47 'hidden_size': text_dim,
48 'intermediate_size': 64,
49 'num_attention_heads': 4,
50 'num_key_value_heads': 2,
51 'num_hidden_layers': 51,
52 'tie_word_embeddings': True,
53})
54config['text_config']['rope_scaling']['mrope_section'] = [2, 1, 1]
55config['vision_config'].update({
56 'depth': 4,
57 'hidden_size': 64,
58 'intermediate_size': 128,
59 'num_heads': 4,
60 'out_hidden_size': text_dim,
61 'deepstack_visual_indexes': [1, 2, 3],
62})
63config['tie_word_embeddings'] = True
64save_json(f'{save_folder}/text_encoder/config.json', config)
65text_encoder = Qwen3VLForConditionalGeneration(
66 AutoConfig.from_pretrained(f'{save_folder}/text_encoder')
67).to(torch.bfloat16)
68init_weights(text_encoder)
69text_encoder.save_pretrained(f'{save_folder}/text_encoder')
70
71# attention_head_dim must cover 2 * 3 * rope_freq_dim rotary channels.
72transformer_kwargs = dict(
73 num_attention_heads=2,
74 attention_head_dim=32,
75 hidden_size=64,
76 num_layers=2,
77 num_refiner_layers=1,
78 ffn_dim=128,
79 in_channels=8,
80 audio_in_channels=8,
81 patch_size=(1, 2, 2),
82 text_dim=text_dim,
83 freq_dim=64,
84 time_embed_hidden_dim=64,
85 time_embed_dim=32,
86 rope_freq_dim=4,
87)
88for subfolder in ('transformer', 'transformer_ref'):
89 transformer = MiniMaxH3Transformer3DModel(**transformer_kwargs)
90 init_weights(transformer)
91 transformer.save_pretrained(f'{save_folder}/{subfolder}')
92
93with open(hf_hub_download(source_model_id, filename='vae/config.json', repo_type='model'), 'r', encoding='utf-8') as f:
94 vae_config = json.load(f)
95vae_config.update({
96 'latent_channels': 8,
97 'block_out_channels': [32, 32, 32, 64, 64, 64],
98 'layers_per_block': 1,
99 'spatial_downsample_factors': [2, 2, 2, 2, 1, 1],
100 'temporal_downsample_factors': [1, 2, 2, 1, 1, 1],
101 'norm_num_groups': 8,
102 'decoder_num_layers': 2,
103 'decoder_num_attention_heads': 2,
104 'decoder_attention_head_dim': 16,
105 'decoder_num_register_tokens': 2,
106 'decoder_ffn_mult': 2,
107 'latents_mean': [0.0] * 8,
108 'latents_std': [1.0] * 8,
109})
110save_json(f'{save_folder}/vae/config.json', vae_config)
111vae = AutoencoderKLMiniMaxH3.from_config(
112 AutoencoderKLMiniMaxH3.load_config(f'{save_folder}/vae')
113)
114init_weights(vae)
115vae.save_pretrained(f'{save_folder}/vae')
116
117# Keep hop length 800 (=32000/40Hz). decoder_dim must stay >= 128 for 7 upsample stages.
118with open(hf_hub_download(source_model_id, filename='audio_vae/config.json', repo_type='model'), 'r', encoding='utf-8') as f:
119 audio_config = json.load(f)
120audio_config.update({
121 'encoder_dim': 32,
122 'latent_dim': 128,
123 'latent_channels': 8,
124 'num_attention_heads': 4,
125 'decoder_dim': 128,
126 'latents_mean': [0.0] * 8,
127 'latents_std': [1.0] * 8,
128})
129save_json(f'{save_folder}/audio_vae/config.json', audio_config)
130audio_vae = AutoencoderKLMiniMaxH3Audio.from_config(
131 AutoencoderKLMiniMaxH3Audio.load_config(f'{save_folder}/audio_vae')
132)
133init_weights(audio_vae)
134audio_vae.save_pretrained(f'{save_folder}/audio_vae')
135
136MiniMaxH3Scheduler.from_pretrained(source_model_id, subfolder='scheduler').save_pretrained(
137 f'{save_folder}/scheduler'
138)
139MiniMaxH3Scheduler.from_pretrained(source_model_id, subfolder='audio_scheduler').save_pretrained(
140 f'{save_folder}/audio_scheduler'
141)
142
143for index_name in ('model_index.json', 'modular_model_index.json'):
144 index = json.load(open(hf_hub_download(source_model_id, filename=index_name, repo_type='model'), encoding='utf-8'))
145 for value in index.values():
146 if isinstance(value, list) and len(value) >= 3 and isinstance(value[2], dict):
147 value[2]['pretrained_model_name_or_path'] = save_folder
148 save_json(f'{save_folder}/{index_name}', index)
149
150pipe = MiniMaxH3Blocks().init_pipeline(save_folder)
151pipe.load_components(dtype=torch.bfloat16)
152pipe.save_pretrained(save_folder, safe_serialization=True, overwrite_modular_index=True)
153torch.set_default_dtype(torch.float32)
154print(pipe)1MiniMaxH3ModularPipeline {
2 "_blocks_class_name": "MiniMaxH3Blocks",
3 "_class_name": "MiniMaxH3ModularPipeline",
4 "_diffusers_version": "0.40.0.dev0",
5 "audio_scheduler": [
6 "diffusers",
7 "MiniMaxH3Scheduler",
8 {
9 "pretrained_model_name_or_path": "./tmp/yujiepan/minimax-h3-tiny-random",
10 "revision": null,
11 "subfolder": "audio_scheduler",
12 "type_hint": [
13 "diffusers",
14 "MiniMaxH3Scheduler"
15 ],
16 "variant": null
17 }
18 ],
19 "audio_vae": [
20 "diffusers",
21 "AutoencoderKLMiniMaxH3Audio",
22 {
23 "pretrained_model_name_or_path": "./tmp/yujiepan/minimax-h3-tiny-random",
24 "revision": null,
25 "subfolder": "audio_vae",
26 "type_hint": [
27 "diffusers",
28 "AutoencoderKLMiniMaxH3Audio"
29 ],
30 "variant": null
31 }
32 ],
33 "canvas_max_pixels": 1032192,
34 "canvas_short_edge": 768,
35 "processor": [
36 "transformers",
37 "Qwen3VLProcessor",
38 {
39 "pretrained_model_name_or_path": "./tmp/yujiepan/minimax-h3-tiny-random",
40 "revision": null,
41 "subfolder": "processor",
42 "type_hint": [
43 "transformers",
44 "Qwen3VLProcessor"
45 ],
46 "variant": null
47 }
48 ],
49 "reference_image_short_edge": 2048,
50 "scheduler": [
51 "diffusers",
52 "MiniMaxH3Scheduler",
53 {
54 "pretrained_model_name_or_path": "./tmp/yujiepan/minimax-h3-tiny-random",
55 "revision": null,
56 "subfolder": "scheduler",
57 "type_hint": [
58 "diffusers",
59 "MiniMaxH3Scheduler"
60 ],
61 "variant": null
62 }
63 ],
64 "text_encoder": [
65 "transformers",
66 "Qwen3VLForConditionalGeneration",
67 {
68 "pretrained_model_name_or_path": "./tmp/yujiepan/minimax-h3-tiny-random",
69 "revision": null,
70 "subfolder": "text_encoder",
71 "type_hint": [
72 "transformers",
73 "Qwen3VLForConditionalGeneration"
74 ],
75 "variant": null
76 }
77 ],
78 "tokenizer": [
79 "transformers",
80 "Qwen2Tokenizer",
81 {
82 "pretrained_model_name_or_path": "./tmp/yujiepan/minimax-h3-tiny-random",
83 "revision": null,
84 "subfolder": "tokenizer",
85 "type_hint": [
86 "transformers",
87 "Qwen2Tokenizer"
88 ],
89 "variant": null
90 }
91 ],
92 "transformer": [
93 "diffusers",
94 "MiniMaxH3Transformer3DModel",
95 {
96 "pretrained_model_name_or_path": "./tmp/yujiepan/minimax-h3-tiny-random",
97 "revision": null,
98 "subfolder": "transformer",
99 "type_hint": [
100 "diffusers",
101 "MiniMaxH3Transformer3DModel"
102 ],
103 "variant": null
104 }
105 ],
106 "transformer_ref": [
107 "diffusers",
108 "MiniMaxH3Transformer3DModel",
109 {
110 "pretrained_model_name_or_path": "./tmp/yujiepan/minimax-h3-tiny-random",
111 "revision": null,
112 "subfolder": "transformer_ref",
113 "type_hint": [
114 "diffusers",
115 "MiniMaxH3Transformer3DModel"
116 ],
117 "variant": null
118 }
119 ],
120 "vae": [
121 "diffusers",
122 "AutoencoderKLMiniMaxH3",
123 {
124 "pretrained_model_name_or_path": "./tmp/yujiepan/minimax-h3-tiny-random",
125 "revision": null,
126 "subfolder": "vae",
127 "type_hint": [
128 "diffusers",
129 "AutoencoderKLMiniMaxH3"
130 ],
131 "variant": null
132 }
133 ]
134}