Views
No views yet
1import torch
2from transformers import AutoConfig, AutoModelForCausalLM, AutoTokenizer
3import os
4
5# Create a tiny config for testing
6from transformers.models.hunyuan_v1_dense.configuration_hunyuan_v1_dense import HunYuanDenseV1Config
7
8tiny_config = HunYuanDenseV1Config(
9 vocab_size=300,
10 hidden_size=64,
11 intermediate_size=128,
12 num_hidden_layers=2,
13 num_attention_heads=4,
14 head_dim=16,
15 num_key_value_heads=2,
16 hidden_act="silu",
17 max_position_embeddings=128,
18 rms_norm_eps=1e-05,
19 use_cache=True,
20 tie_word_embeddings=False,
21 rope_theta=10000.0,
22 attention_bias=False,
23 attention_dropout=0.0,
24 use_qk_norm=True,
25 bos_token_id=1,
26 eos_token_id=2,
27 pad_token_id=0,
28)
29
30print("Config created:", tiny_config.model_type)
31
32# Create model from config
33model = AutoModelForCausalLM.from_config(tiny_config)
34model.eval()
35print("Model created, params:", sum(p.numel() for p in model.parameters()))
36
37# Save model
38save_dir = "/home/panas/git/optimum-intel/tiny-random-hunyuan-v1-dense"
39model.save_pretrained(save_dir)
40tiny_config.save_pretrained(save_dir)
41
42# Create a simple tokenizer config for testing
43from transformers import PreTrainedTokenizerFast
44tokenizer = PreTrainedTokenizerFast(
45 tokenizer_object=None,
46 bos_token="<s>",
47 eos_token="</s>",
48 unk_token="<unk>",
49 pad_token="<pad>",
50)
51# Just save a minimal tokenizer
52# Actually, for tests, we can use the AutoTokenizer approach or skip tokenizer
53
54print(f"Saved tiny model to {save_dir}")
55print("Files:", os.listdir(save_dir))