Views
No views yet
| metric | value |
|---|---|
step | 12 |
smooth_train_loss | 8.226726531982422 |
min_objective | 11.66479491837295 |
flops_used | 9505887907479552.0 |
flops_per_token | 755460096.0 |
total_training_time | 0.31223583221435547 |
1{
2 "model_pt": {
3 "sequence_len": 2048,
4 "vocab_size": 65536,
5 "n_layer": 11,
6 "n_head": 6,
7 "n_kv_head": 6,
8 "n_embd": 768
9 },
10 "model_ppt": {
11 "sequence_len": 2048,
12 "vocab_size": 10004,
13 "n_layer": 11,
14 "n_head": 6,
15 "n_kv_head": 6,
16 "n_embd": 768
17 },
18 "optim": {
19 "matrix_lr": 0.006,
20 "embedding_lr": 0.006,
21 "unembedding_lr": 0.006,
22 "weight_decay": 0.0
23 },
24 "train": {
25 "num_iterations": 1000,
26 "eval_every_n_steps": null,
27 "eval_every_flops_frac": null,
28 "save_every_n_steps": null,
29 "save_at_steps": null,
30 "log_every_n_steps": 100,
31 "eval_at_end": true,
32 "save_at_end": true,
33 "grad_clip": 1.0,
34 "ema_beta": 0.0
35 },
36 "eval": {
37 "eval_steps": 8,
38 "eval_tokens": 2016000
39 },
40 "hardware": {
41 "device_batch_size": 32,
42 "grad_accum_steps": 4,
43 "peak_tflops": 2250.0
44 },
45 "wandb": {
46 "enabled": true,
47 "notes": "",
48 "group": "1e18_ppt0.01",
49 "tags": [
50 "nanochat_gpt",
51 "pt_fineweb-nanochatbpe-20B",
52 "ppt_nca-paper-share20-2048",
53 "seed_0",
54 "case_c",
55 "lr_trapezoid",
56 "depth_11"
57 ],
58 "entity": null
59 },
60 "data_pt": "fineweb-nanochatbpe-20B",
61 "data_ppt": "nca-paper-share20-2048",
62 "extra_eval": [
63 "c4-nanochatbpe-10B"
64 ],
65 "train_split": "train",
66 "pad_vocab": 65536,
67 "ppt_vocab_size": 10004,
68 "ppt_same_vocab_as_pt": false,
69 "ppt_unified_lr": false,
70 "lr_kind": "trapezoid",
71 "lr_warmup_ratio": 0.0,
72 "lr_warmdown_ratio": 0.4,
73 "lr_final_frac": 0.0,
74 "ppt_lr_kind": "trapezoid",
75 "ppt_lr_warmup_ratio": 0.0,
76 "ppt_lr_warmdown_ratio": 0.4,
77 "ppt_lr_final_frac": 0.0,
78 "ppt_lr": 0.006,
79 "ppt_weight_decay": 0.0,
80 "ppt_device_batch_size": 32,
81 "ppt_grad_accum_steps": 4,
82 "eval_tokens": 2016000,
83 "reinit_embed_at_transition": true,
84 "moment_match_embed_reinit": false,
85 "reset_optimizer_at_transition": true,
86 "random_pt_head": false,
87 "depth": 11,
88 "compile_model": true,
89 "model_project": "eval_seed_fleet_v1",
90 "run_name": "nca_1pct_1e18_d11_seed0",
91 "target_flops": 1e+18,
92 "alpha_ppt": 0.01,
93 "use_measured_flops": true,
94 "flops_per_token": null,
95 "ppt_flops_per_token": null,
96 "seed": 0,
97 "metrics_output_file": null,
98 "push_to_hf": true,
99 "hf_repo_org": "alexkstern",
100 "cleanup_checkpoints": true
101}model_000012.pt — model weights (state_dict).meta_000012.json — training metadata.config_000012.json — run config snapshot.rng_000012.pt — RNG state (when present).