Views
No views yet
| metric | value |
|---|---|
step | 22254 |
smooth_train_loss | 2.669527292251587 |
min_objective | 0.8331609075663288 |
flops_used | 2.673034166951058e+19 |
flops_per_token | 4582014976.0 |
total_training_time | 8712.284240961075 |
1{
2 "model_pt": {
3 "sequence_len": 2048,
4 "vocab_size": 65536,
5 "n_layer": 22,
6 "n_head": 11,
7 "n_kv_head": 11,
8 "n_embd": 1408
9 },
10 "model_ppt": {
11 "sequence_len": 2048,
12 "vocab_size": 65536,
13 "n_layer": 22,
14 "n_head": 11,
15 "n_kv_head": 11,
16 "n_embd": 1408
17 },
18 "optim": {
19 "matrix_lr": 0.015,
20 "embedding_lr": 0.3,
21 "unembedding_lr": 0.004,
22 "weight_decay": 0.0
23 },
24 "train": {
25 "num_iterations": 1000,
26 "eval_every_n_steps": null,
27 "eval_every_flops_frac": null,
28 "save_every_n_steps": null,
29 "save_at_steps": null,
30 "log_every_n_steps": 100,
31 "eval_at_end": true,
32 "save_at_end": true,
33 "grad_clip": 1.0,
34 "ema_beta": 0.0
35 },
36 "eval": {
37 "eval_steps": 8,
38 "eval_device_batch_size": 32,
39 "eval_tokens": 10485760,
40 "eval_offset_tokens": 2097152,
41 "aux_eval_tokens": 9437184
42 },
43 "hardware": {
44 "device_batch_size": 32,
45 "grad_accum_steps": 1,
46 "peak_tflops": 2250.0
47 },
48 "wandb": {
49 "enabled": true,
50 "notes": "",
51 "group": "3e19_ppt0.01",
52 "tags": [
53 "nanochat_gpt",
54 "pt_fineweb-nanochatbpe-20B",
55 "ppt_github-code-nanochatbpe-1B",
56 "seed_2",
57 "case_b",
58 "lr_trapezoid",
59 "depth_22"
60 ],
61 "entity": null
62 },
63 "data_pt": "fineweb-nanochatbpe-20B",
64 "data_ppt": "github-code-nanochatbpe-1B",
65 "extra_eval": [
66 "c4-nanochatbpe-10B"
67 ],
68 "train_split": "train",
69 "pad_vocab": 65536,
70 "ppt_vocab_size": null,
71 "ppt_same_vocab_as_pt": true,
72 "ppt_unified_lr": false,
73 "lr_kind": "trapezoid",
74 "lr_warmup_ratio": 0.0,
75 "lr_warmdown_ratio": 0.4,
76 "lr_final_frac": 0.0,
77 "ppt_lr_kind": "trapezoid",
78 "ppt_lr_warmup_ratio": 0.0,
79 "ppt_lr_warmdown_ratio": 0.7,
80 "ppt_lr_final_frac": 0.0,
81 "ppt_lr": 0.0003,
82 "ppt_weight_decay": 0.0,
83 "ppt_device_batch_size": 32,
84 "ppt_grad_accum_steps": 4,
85 "eval_tokens": 10485760,
86 "eval_offset_tokens": 2097152,
87 "aux_eval_tokens": 9437184,
88 "reinit_embed_at_transition": false,
89 "moment_match_embed_reinit": false,
90 "reset_optimizer_at_transition": true,
91 "random_pt_head": false,
92 "depth": 22,
93 "compile_model": true,
94 "model_project": "code_separate_seed_replicas_v2",
95 "run_name": "code_1pct_separate_27e18_s2",
96 "target_flops": 2.7e+19,
97 "ppt_tokens": null,
98 "pt_tokens": null,
99 "ppt_eval_every_n_steps": null,
100 "alpha_ppt": 0.01,
101 "use_measured_flops": true,
102 "flops_per_token": null,
103 "ppt_flops_per_token": null,
104 "seed": 2,
105 "metrics_output_file": null,
106 "push_to_hf": true,
107 "hf_repo_org": "alexkstern",
108 "cleanup_checkpoints": true
109}model_022254.pt — model weights (state_dict).meta_022254.json — training metadata.config_022254.json — run config snapshot.rng_022254.pt — RNG state (when present).