Views
No views yet
| metric | value |
|---|---|
step | 17517 |
smooth_train_loss | 2.8390727043151855 |
min_objective | 0.9573298860905713 |
flops_used | 2.826185612678136e+19 |
flops_per_token | 3077308480.0 |
total_training_time | 19148.692640066147 |
1{
2 "model_pt": {
3 "sequence_len": 1024,
4 "vocab_size": 50304,
5 "n_layer": 24,
6 "n_head": 16,
7 "n_embd": 1024,
8 "intermediate_size": 4096,
9 "rope_theta": 10000.0,
10 "attention_dropout": 0.1,
11 "weight_tying": false,
12 "head_bias": false,
13 "initializer_range": 0.02,
14 "attn_implementation": "sdpa"
15 },
16 "model_ppt": null,
17 "optim": {
18 "matrix_lr": 0.0005,
19 "embedding_lr": 0.0005,
20 "unembedding_lr": 0.0005,
21 "weight_decay": 0.0001
22 },
23 "train": {
24 "num_iterations": 1000,
25 "eval_every_n_steps": 2000,
26 "save_every_n_steps": null,
27 "log_every_n_steps": 10,
28 "eval_at_end": true,
29 "save_at_end": true,
30 "grad_clip": 1.0,
31 "ema_beta": 0.0
32 },
33 "eval": {
34 "eval_steps": 4,
35 "eval_tokens": 2016000
36 },
37 "hardware": {
38 "device_batch_size": 128,
39 "grad_accum_steps": 1,
40 "peak_tflops": 2250.0
41 },
42 "wandb": {
43 "enabled": true,
44 "notes": "",
45 "group": "repro_vanilla_owt",
46 "tags": [
47 "hf_llama",
48 "vocab_64000",
49 "pt_owt-gpt2bpe-9B",
50 "seed_0",
51 "no_ppt"
52 ],
53 "entity": null
54 },
55 "data_pt": "owt-gpt2bpe-9B",
56 "data_ppt": null,
57 "extra_eval": [
58 "fineweb-gpt2bpe-20B",
59 "c4-gpt2bpe-10B",
60 "nca-paper-1024"
61 ],
62 "train_split": "train",
63 "pad_vocab": 50304,
64 "lr_kind": "cosine",
65 "lr_warmup_ratio": 0.1,
66 "lr_warmdown_ratio": 0.0,
67 "lr_final_frac": 0.0,
68 "ppt_lr": 0.0001,
69 "ppt_weight_decay": 0.0,
70 "ppt_grad_clip": 0.0,
71 "ppt_device_batch_size": 8,
72 "ppt_grad_accum_steps": 1,
73 "ppt_lr_kind": "cosine",
74 "ppt_lr_warmup_ratio": 0.1,
75 "ppt_lr_warmdown_ratio": 0.0,
76 "ppt_tokens": null,
77 "pt_tokens": null,
78 "eval_tokens": 2016000,
79 "reinit_embed_at_transition": true,
80 "reset_optimizer_at_transition": true,
81 "depth": null,
82 "compile_model": true,
83 "model_project": "nca-repro",
84 "run_name": "vanilla_owt_recycle_baseline",
85 "target_flops": 2.826286e+19,
86 "alpha_ppt": 0.0,
87 "use_measured_flops": true,
88 "seed": 0,
89 "push_to_hf": true,
90 "hf_repo_org": "alexkstern"
91}model_017517.pt — model weights (state_dict).meta_017517.json — training metadata.config_017517.json — run config snapshot.rng_017517.pt — RNG state (when present).