Training: Reasoning SFT v1 on base pretrained model (20 steps)
Checkpoint Contents
model_000020.pt # Model weights
meta_000020.json # Training config and metadata
optim_000020_rank*.pt # Optimizer state shards (if present, for resuming training)
tokenizer/ # BPE tokenizer (tiktoken format) + token byte counts
nanochat/ # Source code to load and run the model
Quick Start
python
1import torch, json
2from nanochat.gpt import GPT, GPTConfig
3from nanochat.tokenizer import RustBPETokenizer
45tokenizer = RustBPETokenizer.from_directory("tokenizer")67withopen("meta_000020.json")as f:8 meta = json.load(f)910config = GPTConfig(**meta["model_config"])1112with torch.device("meta"):13 model = GPT(config)14model.to_empty(device="cuda")15model.init_weights()1617state_dict = torch.load("model_000020.pt", map_location="cuda")18state_dict ={k.removeprefix("_orig_mod."): v for k, v in state_dict.items()}19model.load_state_dict(state_dict, strict=True, assign=True)20model.eval()2122bos = tokenizer.get_bos_token_id()23tokens = tokenizer.encode("It was a dark and stormy night", prepend=bos)24with torch.amp.autocast(device_type="cuda", dtype=torch.bfloat16):25for token in model.generate(tokens, max_tokens=100, temperature=0.8):26print(tokenizer.decode([token]), end="", flush=True)
