Views
No views yet
| metric | value |
|---|---|
step | 5004 |
smooth_train_loss | 4.689947128295898 |
min_objective | 1.4889587360464172 |
flops_used | 5.183874448364667e+17 |
flops_per_token | 3161456704.0 |
total_training_time | 646.9167912006378 |
1{
2 "model_pt": {
3 "sequence_len": 1024,
4 "vocab_size": 64000,
5 "n_layer": 24,
6 "n_head": 16,
7 "n_embd": 1024,
8 "intermediate_size": 4096,
9 "rope_theta": 10000.0,
10 "attention_dropout": 0.1,
11 "weight_tying": false,
12 "head_bias": false,
13 "initializer_range": 0.02,
14 "nanochat_init": false,
15 "attn_implementation": "sdpa",
16 "explicit_additive_mask": true
17 },
18 "model_ppt": null,
19 "optim": {
20 "matrix_lr": 0.0001,
21 "embedding_lr": 0.0001,
22 "unembedding_lr": 0.0001,
23 "weight_decay": 0.0
24 },
25 "train": {
26 "num_iterations": 1000,
27 "eval_every_n_steps": 1000,
28 "eval_every_flops_frac": null,
29 "save_every_n_steps": null,
30 "save_at_steps": null,
31 "log_every_n_steps": 10,
32 "eval_at_end": true,
33 "save_at_end": true,
34 "grad_clip": 0.0,
35 "ema_beta": 0.0
36 },
37 "eval": {
38 "eval_steps": 62,
39 "eval_tokens": 2016000
40 },
41 "hardware": {
42 "device_batch_size": 8,
43 "grad_accum_steps": 1,
44 "peak_tflops": 2250.0
45 },
46 "wandb": {
47 "enabled": true,
48 "notes": "",
49 "group": "repro_c4_owt",
50 "tags": [
51 "hf_llama",
52 "vocab_64000",
53 "pt_owt-gpt2bpe-9B",
54 "seed_0",
55 "ppt_c4-gpt2bpe-10B",
56 "ppt_lr_cosine",
57 "reinit"
58 ],
59 "entity": null
60 },
61 "data_pt": "owt-gpt2bpe-9B",
62 "data_ppt": "c4-gpt2bpe-10B",
63 "extra_eval": [
64 "fineweb-gpt2bpe-20B",
65 "nca-paper-1024"
66 ],
67 "train_split": "train",
68 "pad_vocab": 64000,
69 "lr_kind": "cosine",
70 "lr_warmup_ratio": 0.1,
71 "lr_warmdown_ratio": 0.0,
72 "lr_final_frac": 0.0,
73 "ppt_lr": 0.0001,
74 "ppt_weight_decay": 0.0,
75 "ppt_grad_clip": 0.0,
76 "ppt_device_batch_size": 8,
77 "ppt_grad_accum_steps": 1,
78 "ppt_lr_kind": "cosine",
79 "ppt_lr_warmup_ratio": 0.1,
80 "ppt_lr_warmdown_ratio": 0.0,
81 "ppt_tokens": 164000000,
82 "pt_tokens": null,
83 "eval_tokens": 2016000,
84 "reinit_embed_at_transition": true,
85 "reset_optimizer_at_transition": true,
86 "paper_embed_reinit": "full",
87 "depth": null,
88 "compile_model": true,
89 "model_project": "nca-repro",
90 "run_name": "c4_owt_untied_embed_reinit_seed0",
91 "target_flops": 0.0,
92 "alpha_ppt": 0.0,
93 "use_measured_flops": true,
94 "flops_per_token": null,
95 "seed": 0,
96 "push_to_hf": true,
97 "hf_repo_org": "alexkstern"
98}model_005004.pt — model weights (state_dict).meta_005004.json — training metadata.config_005004.json — run config snapshot.rng_005004.pt — RNG state (when present).