Views
No views yet
| metric | value |
|---|---|
step | 188827 |
smooth_train_loss | 2.656653642654419 |
min_objective | 0.7658007377664263 |
flops_used | 2.8402071556486437e+20 |
flops_per_token | 5737807872.0 |
total_training_time | 170589.91244649887 |
1{
2 "model_pt": {
3 "sequence_len": 2048,
4 "vocab_size": 65536,
5 "n_layer": 24,
6 "n_head": 12,
7 "n_kv_head": 12,
8 "n_embd": 1536
9 },
10 "model_ppt": {
11 "sequence_len": 2048,
12 "vocab_size": 256,
13 "n_layer": 24,
14 "n_head": 12,
15 "n_kv_head": 12,
16 "n_embd": 1536
17 },
18 "optim": {
19 "matrix_lr": 0.006,
20 "embedding_lr": 0.3,
21 "unembedding_lr": 0.004,
22 "weight_decay": 0.0
23 },
24 "train": {
25 "num_iterations": 1000,
26 "eval_every_n_steps": 4000,
27 "eval_every_flops_frac": null,
28 "save_every_n_steps": null,
29 "save_at_steps": [
30 17166,
31 19073,
32 36240,
33 38147,
34 55313,
35 57220,
36 74387,
37 76294,
38 95367
39 ],
40 "log_every_n_steps": 100,
41 "eval_at_end": true,
42 "save_at_end": true,
43 "grad_clip": 1.0,
44 "ema_beta": 0.0
45 },
46 "eval": {
47 "eval_steps": 8,
48 "eval_device_batch_size": 32,
49 "eval_tokens": 10485760,
50 "eval_offset_tokens": 2097152,
51 "aux_eval_tokens": 2097152
52 },
53 "hardware": {
54 "device_batch_size": 32,
55 "grad_accum_steps": 2,
56 "peak_tflops": 2250.0
57 },
58 "wandb": {
59 "enabled": true,
60 "notes": "",
61 "group": "5e10tok_ppt0.01",
62 "tags": [
63 "nanochat_gpt",
64 "pt_fineweb-nanochatbpe-55B",
65 "ppt_dyck-k128-seq_len_2048-1B",
66 "seed_1",
67 "case_c",
68 "lr_trapezoid",
69 "depth_24"
70 ],
71 "entity": null
72 },
73 "data_pt": "fineweb-nanochatbpe-55B",
74 "data_ppt": "dyck-k128-seq_len_2048-1B",
75 "extra_eval": [
76 "c4-nanochatbpe-10B"
77 ],
78 "train_split": "train",
79 "pad_vocab": 65536,
80 "ppt_vocab_size": null,
81 "ppt_same_vocab_as_pt": false,
82 "ppt_unified_lr": false,
83 "lr_kind": "trapezoid",
84 "lr_warmup_ratio": 0.0,
85 "lr_warmdown_ratio": 0.4949513,
86 "lr_final_frac": 0.0,
87 "ppt_lr_kind": "trapezoid",
88 "ppt_lr_warmup_ratio": 0.1,
89 "ppt_lr_warmdown_ratio": 0.4,
90 "ppt_lr_final_frac": 0.0,
91 "ppt_lr": 0.01,
92 "ppt_weight_decay": 0.0,
93 "ppt_device_batch_size": 32,
94 "ppt_grad_accum_steps": 8,
95 "eval_tokens": 10485760,
96 "eval_offset_tokens": 2097152,
97 "aux_eval_tokens": 2097152,
98 "reinit_embed_at_transition": true,
99 "moment_match_embed_reinit": false,
100 "reset_optimizer_at_transition": true,
101 "random_pt_head": false,
102 "depth": 24,
103 "compile_model": true,
104 "model_project": "longhorizon_50B_v0",
105 "run_name": "kdyck_1pct_50B_d24_seed1",
106 "target_flops": 0.0,
107 "ppt_tokens": 500000000,
108 "pt_tokens": 49500000000,
109 "ppt_eval_every_n_steps": 238,
110 "alpha_ppt": 0.0,
111 "use_measured_flops": true,
112 "flops_per_token": null,
113 "ppt_flops_per_token": null,
114 "seed": 1,
115 "metrics_output_file": null,
116 "push_to_hf": true,
117 "hf_repo_org": "alexkstern",
118 "cleanup_checkpoints": false
119}model_188827.pt — model weights (state_dict).meta_188827.json — training metadata.config_188827.json — run config snapshot.rng_188827.pt — RNG state (when present).