Training: Reasoning SFT v2 (99 steps), from reasoning-sft-v1
Checkpoint Contents
model_000099.pt # Model weights
meta_000099.json # Training config and metadata
optim_000099_rank*.pt # Optimizer state shards (if present, for resuming training)
tokenizer/ # BPE tokenizer (tiktoken format) + token byte counts
nanochat/ # Source code to load and run the model
Quick Start
python
1import torch, json
2from nanochat.gpt import GPT, GPTConfig
3from nanochat.tokenizer import RustBPETokenizer
45tokenizer = RustBPETokenizer.from_directory("tokenizer")67withopen("meta_000099.json")as f:8 meta = json.load(f)910config = GPTConfig(**meta["model_config"])1112with torch.device("meta"):13 model = GPT(config)14model.to_empty(device="cuda")15model.init_weights()1617state_dict = torch.load("model_000099.pt", map_location="cuda")18state_dict ={k.removeprefix("_orig_mod."): v for k, v in state_dict.items()}19model.load_state_dict(state_dict, strict=True, assign=True)20model.eval()2122bos = tokenizer.get_bos_token_id()23tokens = tokenizer.encode("It was a dark and stormy night", prepend=bos)24with torch.amp.autocast(device_type="cuda", dtype=torch.bfloat16):25for token in model.generate(tokens, max_tokens=100, temperature=0.8):26print(tokenizer.decode([token]), end="", flush=True)
