Views
No views yet
1{
2 "sequence_len": 2048,
3 "vocab_size": 65536,
4 "n_layer": 12,
5 "n_head": 6,
6 "n_kv_head": 6,
7 "n_embd": 768,
8 "mhc_enabled": true,
9 "mhc_num_streams": 4,
10 "mhc_sinkhorn_iters": 50,
11 "mhc_sinkhorn_tau": 0.1,
12 "mhc_gate_noise": false
13}1{
2 "run": "mhc-sanity-20260117-062153",
3 "seed": 42,
4 "device_type": "",
5 "skip_compile": false,
6 "depth": 12,
7 "max_seq_len": 2048,
8 "mhc_enabled": true,
9 "mhc_num_streams": 4,
10 "mhc_sinkhorn_iters": 50,
11 "mhc_sinkhorn_tau": 0.1,
12 "mhc_gate_noise": false,
13 "num_iterations": 5000,
14 "target_flops": -1.0,
15 "target_param_data_ratio": 20,
16 "device_batch_size": 16,
17 "total_batch_size": 131072,
18 "embedding_lr": 0.2,
19 "unembedding_lr": 0.004,
20 "weight_decay": 0.0,
21 "matrix_lr": 0.02,
22 "grad_clip": 1.0,
23 "warmup_ratio": 0.0,
24 "warmdown_ratio": 0.2,
25 "final_lr_frac": 0.0,
26 "resume_from_step": -1,
27 "eval_every": 500,
28 "eval_tokens": 10485760,
29 "core_metric_every": -1,
30 "core_metric_max_per_task": 500,
31 "sample_every": 100,
32 "save_every": 5000,
33 "model_tag": ""
34}1from nanochat.checkpoint_manager import build_model
2
3model, tokenizer = build_model("path/to/checkpoint", step=5000, device="cuda", phase="inference")