Views
No views yet
1
2from transformers import (
3 AutoTokenizer,
4 SmolLM3Config,
5 SmolLM3ForCausalLM,
6)
7
8def create_tiny_random_smollm3():
9
10 config = SmolLM3Config(
11 vocab_size=128256,
12 hidden_size=32,
13 intermediate_size=64,
14 num_hidden_layers=2,
15 num_attention_heads=4,
16 num_key_value_heads=2,
17 max_position_embeddings=256,
18 hidden_act="silu",
19 rms_norm_eps=1e-6,
20 tie_word_embeddings=True,
21 use_cache=True,
22 attention_bias=False,
23 mlp_bias=False,
24 use_sliding_window=False,
25 pad_token_id=128004,
26 bos_token_id=128000,
27 eos_token_id=128012,
28 )
29
30 model = SmolLM3ForCausalLM(config)
31 print(f"Model parameters: {sum(p.numel() for p in model.parameters()):,}")
32
33 # Use SmolLM3-3B tokenizer
34 tokenizer = AutoTokenizer.from_pretrained("HuggingFaceTB/SmolLM3-3B")
35
36 output_dir = "./tiny-random-smollm3"
37 model.save_pretrained(output_dir)
38 tokenizer.save_pretrained(output_dir)
39 print(f"Saved to {output_dir}")
40
41if __name__ == "__main__":
42 create_tiny_random_smollm3()
43