Views
No views yet
1from transformers import (
2 AutoTokenizer,
3 Gemma4ForConditionalGeneration,
4)
5
6
7def generate_vlm_model(output_dir="./tiny-random-gemma4-moe"):
8 from transformers import AutoConfig, AutoProcessor, AutoTokenizer, Gemma4ForConditionalGeneration
9
10 config = AutoConfig.from_pretrained("google/gemma-4-26B-A4B-it")
11
12 # Text config
13 config.text_config.global_head_dim = 4
14 config.text_config.head_dim = 4
15 config.text_config.hidden_size = 32
16 config.text_config.hidden_size_per_layer_input = 0
17 config.text_config.num_hidden_layers = 2
18 config.text_config.layer_types = ["sliding_attention", "full_attention"]
19 config.text_config.num_kv_shared_layers = 0
20 config.text_config.intermediate_size = 64
21 config.text_config.dtype = "float32"
22
23 # MOE parameters scaled down to avoid CPU plugin crash on SPR
24 config.text_config.num_experts = 4
25 config.text_config.top_k_experts = 2
26 config.text_config.moe_intermediate_size = 64
27 config.text_config.num_attention_heads = 4
28 config.text_config.num_key_value_heads = 2
29 config.text_config.num_global_key_value_heads = 2
30
31 # Vision config
32 config.vision_config.head_dim = 4
33 config.vision_config.hidden_size = 8
34 config.vision_config.intermediate_size = 32
35 config.vision_config.num_hidden_layers = 1
36 config.vision_config.num_key_value_heads = 2
37
38 model = Gemma4ForConditionalGeneration(config)
39 model.eval()
40 model.save_pretrained(str(output_dir))
41
42 tokenizer = AutoTokenizer.from_pretrained("google/gemma-4-26B-A4B-it")
43 tokenizer.save_pretrained(str(output_dir))
44
45 processor = AutoProcessor.from_pretrained("google/gemma-4-26B-A4B-it")
46 processor.save_pretrained(str(output_dir))
47
48 return model
49
50
51if __name__ == "__main__":
52 generate_vlm_model()
53