Views
No views yet
| metric | value |
|---|---|
step | 9545 |
smooth_train_loss | 2.9658637046813965 |
min_objective | 0.872978107283314 |
flops_used | 8.999324002707046e+18 |
flops_per_token | 3596615680.0 |
total_training_time | 3064.848010778427 |
1{
2 "model_pt": {
3 "sequence_len": 2048,
4 "vocab_size": 65536,
5 "n_layer": 20,
6 "n_head": 10,
7 "n_kv_head": 10,
8 "n_embd": 1280
9 },
10 "model_ppt": null,
11 "optim": {
12 "matrix_lr": 0.015,
13 "embedding_lr": 0.3,
14 "unembedding_lr": 0.004,
15 "weight_decay": 0.0
16 },
17 "train": {
18 "num_iterations": 1000,
19 "eval_every_n_steps": null,
20 "eval_every_flops_frac": null,
21 "save_every_n_steps": null,
22 "log_every_n_steps": 100,
23 "eval_at_end": true,
24 "save_at_end": true,
25 "grad_clip": 1.0,
26 "ema_beta": 0.0
27 },
28 "eval": {
29 "eval_steps": 16,
30 "eval_tokens": 2016000
31 },
32 "hardware": {
33 "device_batch_size": 16,
34 "grad_accum_steps": 2,
35 "peak_tflops": 2250.0
36 },
37 "wandb": {
38 "enabled": true,
39 "notes": "",
40 "group": "9e18_vanilla",
41 "tags": [
42 "nanochat_gpt",
43 "pt_fineweb-nanochatbpe-20B",
44 "no_ppt",
45 "seed_2",
46 "single",
47 "lr_trapezoid",
48 "depth_20"
49 ],
50 "entity": null
51 },
52 "data_pt": "fineweb-nanochatbpe-20B",
53 "data_ppt": null,
54 "extra_eval": [
55 "c4-nanochatbpe-10B"
56 ],
57 "train_split": "train",
58 "pad_vocab": 65536,
59 "ppt_vocab_size": null,
60 "ppt_same_vocab_as_pt": false,
61 "ppt_unified_lr": false,
62 "lr_kind": "trapezoid",
63 "lr_warmup_ratio": 0.0,
64 "lr_warmdown_ratio": 0.5,
65 "lr_final_frac": 0.0,
66 "ppt_lr_kind": "trapezoid",
67 "ppt_lr_warmup_ratio": null,
68 "ppt_lr_warmdown_ratio": null,
69 "ppt_lr_final_frac": 0.0,
70 "ppt_lr": null,
71 "ppt_weight_decay": 0.0,
72 "eval_tokens": 2016000,
73 "reinit_embed_at_transition": true,
74 "reset_optimizer_at_transition": true,
75 "depth": 20,
76 "compile_model": true,
77 "model_project": "optimal_seed_replicas_v0",
78 "run_name": "vanilla_9e18_seed2",
79 "target_flops": 9e+18,
80 "alpha_ppt": 0.0,
81 "use_measured_flops": true,
82 "flops_per_token": null,
83 "ppt_flops_per_token": null,
84 "seed": 2,
85 "metrics_output_file": null,
86 "push_to_hf": true,
87 "hf_repo_org": "alexkstern",
88 "cleanup_checkpoints": true
89}model_009545.pt — model weights (state_dict).meta_009545.json — training metadata.config_009545.json — run config snapshot.rng_009545.pt — RNG state (when present).