A 1.7B parameter GPT-style language model trained exclusively on pre-1900 English text.
model_017517.pt # Model weights (4.9 GB)
meta_017517.json # Training config and metadata
optim_017517_rank*.pt # Optimizer state, 8 FSDP shards (for resuming training)
tokenizer/ # BPE tokenizer (tiktoken format) + token byte counts
nanochat/ # Source code to load and run the model1 import torch
2 from nanochat . gpt import GPT , GPTConfig
3 from nanochat . tokenizer import RustBPETokenizer
4
5 # Load tokenizer
6 tokenizer = RustBPETokenizer . from_directory ( "tokenizer" )
7
8 # Load model
9 import json
10 with open ( "meta_017517.json" ) as f :
11 meta = json . load ( f )
12
13 config = GPTConfig ( ** meta [ "model_config" ] )
14
15 with torch . device ( "meta" ) :
16 model = GPT ( config )
17 model . to_empty ( device = "cuda" )
18 model . init_weights ( )
19
20 state_dict = torch . load ( "model_017517.pt" , map_location = "cuda" )
21 state_dict = { k . removeprefix ( "_orig_mod." ) : v for k , v in state_dict . items ( ) }
22 model . load_state_dict ( state_dict , strict = True , assign = True )
23 model . eval ( )
24
25 # Generate
26 bos = tokenizer . get_bos_token_id ( )
27 tokens = tokenizer . encode ( "It was a dark and stormy night" , prepend = bos )
28 with torch . amp . autocast ( device_type = "cuda" , dtype = torch . bfloat16 ) :
29 for token in model . generate ( tokens , max_tokens = 100 , temperature = 0.8 ) :
30 print ( tokenizer . decode ( [ token ] ) , end = "" , flush = True )
torch>=2.9
tiktoken
rustbpeTrained with the
nanochat framework using 8x H100 GPUs with FSDP.
To resume training, load the optimizer shards (
optim_017517_rank*.pt) — one per FSDP rank.
