Views
No views yet
1from transformers import (
2 AutoTokenizer,
3 Gemma4ForConditionalGeneration,
4)
5
6
7def generate_vlm_model(output_dir="./tiny-random-gemma4-31B"):
8 from transformers import AutoConfig, AutoProcessor, AutoTokenizer, Gemma4ForConditionalGeneration
9 model_id = "google/gemma-4-31B-it"
10 config = AutoConfig.from_pretrained(model_id)
11
12 # Text config
13 config.text_config.global_head_dim = 4
14 config.text_config.head_dim = 4
15 config.text_config.hidden_size = 32
16 config.text_config.hidden_size_per_layer_input = 0
17 config.text_config.num_hidden_layers = 2
18 config.text_config.layer_types = ["sliding_attention", "full_attention"]
19 config.text_config.num_kv_shared_layers = 0
20 config.text_config.intermediate_size = 64
21 config.text_config.dtype = "float32"
22
23 # Vision config
24 config.vision_config.head_dim = 4
25 config.vision_config.hidden_size = 8
26 config.vision_config.intermediate_size = 32
27 config.vision_config.num_hidden_layers = 1
28 config.vision_config.num_key_value_heads = 2
29
30 model = Gemma4ForConditionalGeneration(config)
31 model.eval()
32 model.save_pretrained(str(output_dir))
33
34 tokenizer = AutoTokenizer.from_pretrained(model_id)
35 tokenizer.save_pretrained(str(output_dir))
36
37 processor = AutoProcessor.from_pretrained(model_id)
38 processor.save_pretrained(str(output_dir))
39
40 return model
41
42
43if __name__ == "__main__":
44 generate_vlm_model()