Views
No views yet
| metric | value |
|---|---|
step | 9875 |
smooth_train_loss | 2.841356039047241 |
min_objective | 0.8732750597679377 |
flops_used | 8.91008145424384e+18 |
flops_per_token | 3441950720.0 |
total_training_time | 3068.75669836998 |
1{
2 "model_pt": {
3 "sequence_len": 2048,
4 "vocab_size": 65536,
5 "n_layer": 19,
6 "n_head": 10,
7 "n_kv_head": 10,
8 "n_embd": 1280
9 },
10 "model_ppt": {
11 "sequence_len": 2048,
12 "vocab_size": 65536,
13 "n_layer": 19,
14 "n_head": 10,
15 "n_kv_head": 10,
16 "n_embd": 1280
17 },
18 "optim": {
19 "matrix_lr": 0.015,
20 "embedding_lr": 0.3,
21 "unembedding_lr": 0.004,
22 "weight_decay": 0.0
23 },
24 "train": {
25 "num_iterations": 1000,
26 "eval_every_n_steps": null,
27 "eval_every_flops_frac": null,
28 "save_every_n_steps": null,
29 "log_every_n_steps": 100,
30 "eval_at_end": true,
31 "save_at_end": true,
32 "grad_clip": 1.0,
33 "ema_beta": 0.0
34 },
35 "eval": {
36 "eval_steps": 16,
37 "eval_tokens": 2016000
38 },
39 "hardware": {
40 "device_batch_size": 16,
41 "grad_accum_steps": 2,
42 "peak_tflops": 2250.0
43 },
44 "wandb": {
45 "enabled": true,
46 "notes": "",
47 "group": "9e18_ppt0.01",
48 "tags": [
49 "nanochat_gpt",
50 "pt_fineweb-nanochatbpe-20B",
51 "ppt_c4-nanochatbpe-10B",
52 "seed_2",
53 "case_b",
54 "lr_trapezoid",
55 "depth_19"
56 ],
57 "entity": null
58 },
59 "data_pt": "fineweb-nanochatbpe-20B",
60 "data_ppt": "c4-nanochatbpe-10B",
61 "extra_eval": [
62 "c4-nanochatbpe-10B"
63 ],
64 "train_split": "train",
65 "pad_vocab": 65536,
66 "ppt_vocab_size": null,
67 "ppt_same_vocab_as_pt": true,
68 "ppt_unified_lr": true,
69 "lr_kind": "trapezoid",
70 "lr_warmup_ratio": 0.0,
71 "lr_warmdown_ratio": 0.4,
72 "lr_final_frac": 0.0,
73 "ppt_lr_kind": "trapezoid",
74 "ppt_lr_warmup_ratio": null,
75 "ppt_lr_warmdown_ratio": null,
76 "ppt_lr_final_frac": 0.0,
77 "ppt_lr": null,
78 "ppt_weight_decay": 0.0,
79 "eval_tokens": 2016000,
80 "reinit_embed_at_transition": false,
81 "reset_optimizer_at_transition": false,
82 "depth": 19,
83 "compile_model": true,
84 "model_project": "optimal_seed_replicas_v0",
85 "run_name": "c4_1pct_9e18_seed2",
86 "target_flops": 9e+18,
87 "alpha_ppt": 0.01,
88 "use_measured_flops": true,
89 "flops_per_token": null,
90 "ppt_flops_per_token": null,
91 "seed": 2,
92 "metrics_output_file": null,
93 "push_to_hf": true,
94 "hf_repo_org": "alexkstern",
95 "cleanup_checkpoints": true
96}model_009875.pt — model weights (state_dict).meta_009875.json — training metadata.config_009875.json — run config snapshot.rng_009875.pt — RNG state (when present).