Views
No views yet
| metric | value |
|---|---|
step | 3814 |
smooth_train_loss | 3.1613173484802246 |
min_objective | 0.9427515775594133 |
flops_used | 2.0799945247754813e+18 |
flops_per_token | 2080374784.0 |
total_training_time | 742.9468657970428 |
1{
2 "model_pt": {
3 "sequence_len": 2048,
4 "vocab_size": 65536,
5 "n_layer": 16,
6 "n_head": 8,
7 "n_kv_head": 8,
8 "n_embd": 1024
9 },
10 "model_ppt": null,
11 "optim": {
12 "matrix_lr": 0.02,
13 "embedding_lr": 0.3,
14 "unembedding_lr": 0.004,
15 "weight_decay": 0.0
16 },
17 "train": {
18 "num_iterations": 1000,
19 "eval_every_n_steps": null,
20 "eval_every_flops_frac": null,
21 "save_every_n_steps": null,
22 "save_at_steps": null,
23 "log_every_n_steps": 100,
24 "eval_at_end": true,
25 "save_at_end": true,
26 "grad_clip": 1.0,
27 "ema_beta": 0.0
28 },
29 "eval": {
30 "eval_steps": 8,
31 "eval_device_batch_size": 32,
32 "eval_tokens": 10485760,
33 "eval_offset_tokens": 2097152,
34 "aux_eval_tokens": 2097152
35 },
36 "hardware": {
37 "device_batch_size": 32,
38 "grad_accum_steps": 1,
39 "peak_tflops": 2250.0
40 },
41 "wandb": {
42 "enabled": true,
43 "notes": "",
44 "group": "1e9tok_vanilla",
45 "tags": [
46 "nanochat_gpt",
47 "pt_fineweb-nanochatbpe-20B",
48 "no_ppt",
49 "seed_2",
50 "single",
51 "lr_trapezoid",
52 "depth_16"
53 ],
54 "entity": null
55 },
56 "data_pt": "fineweb-nanochatbpe-20B",
57 "data_ppt": null,
58 "extra_eval": [
59 "c4-nanochatbpe-10B"
60 ],
61 "train_split": "train",
62 "pad_vocab": 65536,
63 "ppt_vocab_size": null,
64 "ppt_same_vocab_as_pt": false,
65 "ppt_unified_lr": false,
66 "lr_kind": "trapezoid",
67 "lr_warmup_ratio": 0.0,
68 "lr_warmdown_ratio": 0.4,
69 "lr_final_frac": 0.0,
70 "ppt_lr_kind": "trapezoid",
71 "ppt_lr_warmup_ratio": null,
72 "ppt_lr_warmdown_ratio": null,
73 "ppt_lr_final_frac": 0.0,
74 "ppt_lr": null,
75 "ppt_weight_decay": 0.0,
76 "ppt_device_batch_size": null,
77 "ppt_grad_accum_steps": null,
78 "eval_tokens": 10485760,
79 "eval_offset_tokens": 2097152,
80 "aux_eval_tokens": 2097152,
81 "reinit_embed_at_transition": true,
82 "moment_match_embed_reinit": false,
83 "reset_optimizer_at_transition": true,
84 "random_pt_head": false,
85 "depth": 16,
86 "compile_model": true,
87 "model_project": "token_dose_1Bpt_seed_replicas_v1",
88 "run_name": "baseline_1Bpt_van_s2",
89 "target_flops": 0.0,
90 "ppt_tokens": null,
91 "pt_tokens": 1000000000,
92 "ppt_eval_every_n_steps": null,
93 "alpha_ppt": 0.0,
94 "use_measured_flops": true,
95 "flops_per_token": null,
96 "ppt_flops_per_token": null,
97 "seed": 2,
98 "metrics_output_file": null,
99 "push_to_hf": true,
100 "hf_repo_org": "alexkstern",
101 "cleanup_checkpoints": true
102}model_003814.pt — model weights (state_dict).meta_003814.json — training metadata.config_003814.json — run config snapshot.rng_003814.pt — RNG state (when present).