Views
No views yet
1import os
2
3from transformers import MambaConfig, MambaForCausalLM, AutoTokenizer
4
5model_dir = "state-spaces/mamba-130m-hf"
6tokenizer = AutoTokenizer.from_pretrained(model_dir)
7
8# === Step 1: Define tiny model config ===
9config = MambaConfig(
10 d_model=16, # Dimensionality of the input embeddings (model hidden size)
11 n_layer=2, # Number of Mamba layers (or blocks) in the model
12 d_state=32, # Dimensionality of the internal state used in the Mamba block (e.g., for state-space modeling)
13 expand=2, # Expansion factor used in the Mamba block, typically to widen the intermediate dimensions
14 conv_kernel=3, # Size of the convolution kernel used in the Mamba block (affects temporal mixing)
15 vocab_size=50280, # Size of the vocabulary (number of unique tokens)
16 num_hidden_layers=32, # Total number of hidden layers in the model (could override `n_layer`)
17 hidden_size=64, # Size of hidden states used in the model layers (could override `d_model`)
18)
19
20# === Step 2: Create model from config ===
21model = MambaForCausalLM(config)
22
23# === Step 4: Save model and tokenizer to disk ===
24output_dir = "./tiny-mamba2"
25os.makedirs(output_dir, exist_ok=True)
26model.save_pretrained(output_dir)
27tokenizer.save_pretrained(output_dir)
28print(f"Tiny Mamba model and tokenizer saved to: {output_dir}")