Views
No views yet
| metric | value |
|---|---|
step | 190734 |
smooth_train_loss | 2.598783016204834 |
min_objective | 0.7636788233926974 |
flops_used | 2.8688909511112733e+20 |
flops_per_token | 5737807872.0 |
total_training_time | 86810.84082484245 |
1{
2 "model_pt": {
3 "sequence_len": 2048,
4 "vocab_size": 65536,
5 "n_layer": 24,
6 "n_head": 12,
7 "n_kv_head": 12,
8 "n_embd": 1536
9 },
10 "model_ppt": null,
11 "optim": {
12 "matrix_lr": 0.006,
13 "embedding_lr": 0.3,
14 "unembedding_lr": 0.004,
15 "weight_decay": 0.0
16 },
17 "train": {
18 "num_iterations": 1000,
19 "eval_every_n_steps": 4000,
20 "eval_every_flops_frac": null,
21 "save_every_n_steps": null,
22 "save_at_steps": [
23 19073,
24 38147,
25 57220,
26 76294,
27 95367
28 ],
29 "log_every_n_steps": 100,
30 "eval_at_end": true,
31 "save_at_end": true,
32 "grad_clip": 1.0,
33 "ema_beta": 0.0
34 },
35 "eval": {
36 "eval_steps": 8,
37 "eval_device_batch_size": 32,
38 "eval_tokens": 10485760,
39 "eval_offset_tokens": 2097152,
40 "aux_eval_tokens": 2097152
41 },
42 "hardware": {
43 "device_batch_size": 32,
44 "grad_accum_steps": 1,
45 "peak_tflops": 2250.0
46 },
47 "wandb": {
48 "enabled": true,
49 "notes": "",
50 "group": "5e10tok_vanilla",
51 "tags": [
52 "nanochat_gpt",
53 "pt_fineweb-nanochatbpe-55B",
54 "no_ppt",
55 "seed_2",
56 "single",
57 "lr_trapezoid",
58 "depth_24"
59 ],
60 "entity": null
61 },
62 "data_pt": "fineweb-nanochatbpe-55B",
63 "data_ppt": null,
64 "extra_eval": [
65 "c4-nanochatbpe-10B"
66 ],
67 "train_split": "train",
68 "pad_vocab": 65536,
69 "ppt_vocab_size": null,
70 "ppt_same_vocab_as_pt": false,
71 "ppt_unified_lr": false,
72 "lr_kind": "trapezoid",
73 "lr_warmup_ratio": 0.0,
74 "lr_warmdown_ratio": 0.5,
75 "lr_final_frac": 0.0,
76 "ppt_lr_kind": "trapezoid",
77 "ppt_lr_warmup_ratio": null,
78 "ppt_lr_warmdown_ratio": null,
79 "ppt_lr_final_frac": 0.0,
80 "ppt_lr": null,
81 "ppt_weight_decay": 0.0,
82 "ppt_device_batch_size": null,
83 "ppt_grad_accum_steps": null,
84 "eval_tokens": 10485760,
85 "eval_offset_tokens": 2097152,
86 "aux_eval_tokens": 2097152,
87 "reinit_embed_at_transition": true,
88 "moment_match_embed_reinit": false,
89 "reset_optimizer_at_transition": true,
90 "random_pt_head": false,
91 "depth": 24,
92 "compile_model": true,
93 "model_project": "longhorizon_50B_v0",
94 "run_name": "vanilla_50B_d24_seed2",
95 "target_flops": 0.0,
96 "ppt_tokens": null,
97 "pt_tokens": 50000000000,
98 "ppt_eval_every_n_steps": null,
99 "alpha_ppt": 0.0,
100 "use_measured_flops": true,
101 "flops_per_token": null,
102 "ppt_flops_per_token": null,
103 "seed": 2,
104 "metrics_output_file": null,
105 "push_to_hf": true,
106 "hf_repo_org": "alexkstern",
107 "cleanup_checkpoints": false
108}model_190734.pt — model weights (state_dict).meta_190734.json — training metadata.config_190734.json — run config snapshot.rng_190734.pt — RNG state (when present).