Views
No views yet
best812(0.6B)/(40M-20M)/(no-thoughts,r=0.0,k=8)/(lr=0.00015,epochs=2,replay=0.25)/(seed=4)model/ and the tokenizer under tokenizer/ in this repo.1trainee:
2 short_name: 0.6B
3 model_id: Qwen/Qwen3-0.6B
4 dtype: bfloat16
5data:
6 train:
7 documents:
8 short_name: 40M-20M
9 dataset_id: JackHsieh/statML-arxiv-40M-20M
10 split: train
11 n_samples: null
12 exact_document_length: 4096
13 chunking:
14 rule: r=0.0,k=8
15 chunk_size: 8
16 path: outputs/prestar/chunking/r=0.0,k=8/40M-20M/train.parquet
17 replay:
18 short_name: full-replay
19 dataset_id: JackHsieh/dclm-replay.seq-4096.tokens-32B
20 split: train
21 n_samples: null
22 exact_document_length: 4096
23 serving:
24 use_thoughts: false
25 replay_proportion: 0.25
26 thoughts_per_step: null
27 fixed_main_order: true
28 fixed_replay_order: true
29 eval:
30 documents:
31 short_name: 40M-20M
32 dataset_id: JackHsieh/statML-arxiv-40M-20M
33 split: test
34 n_samples: null
35 exact_document_length: 4096
36 chunking:
37 rule: r=0.0,k=8
38 chunk_size: 8
39 path: outputs/prestar/chunking/r=0.0,k=8/40M-20M/test.parquet
40 serving:
41 use_thoughts: false
42 thoughts_per_pass: null
43train:
44 docs_per_batch: 32
45 total_epochs: 2
46 total_steps: null
47 grad_clip_norm: 1.0
48 seed: 4
49 early_stopping:
50 enable: true
51 mode: vs_best
52 metric: val-dynamics/val_loss
53 patience_evals: 3
54 min_delta: -0.02
55 loss: {}
56 execution:
57 thoughtless_pass_doc_microbatch: 4
58 thoughtful_pass_thought_microbatch: 1
59 save_incremental_per_epoch: false
60 save_per_token_log_probs: false
61 save_chunk_token_snippets: false
62 optimizer:
63 style: adamw
64 lr: 0.00015
65 weight_decay: 0.01
66 beta1: 0.9
67 beta2: 0.95
68 eps: 1.0e-08
69 lr_scheduler:
70 style: cosine
71 total_steps: null
72 min_lr_ratio: 0.0
73 warmup_steps: null
74 warmup_ratio: 0.05
75 stable_steps: null
76 decay_steps: null
77eval:
78 eval_every_n_steps: null
79 eval_every_n_epochs: 1
80 eval_on_first_step: true
81 execution:
82 docs_per_batch: null
83 thoughtless_pass_doc_microbatch: 32
84 thoughtful_pass_thought_microbatch: 1
85 incremental_save_every_n_batches: 0
86 save_per_token_log_probs: true
87 save_chunk_token_snippets: false
88checkpointing:
89 enable: true
90 auto_resume: true
91 resume_from_path: null
92 resume_wandb_run: true
93 best_max_keep: 1
94 latest_max_keep: 1
95 best_includes_resume_state: false
96 best_metric: val-dynamics/val_loss
97 best_metric_direction: min
98 latest_every_n_evals: 1
99 push_best_to_hub: true
100 push_latest_to_hub: false
101 best_repo_id: null
102 latest_repo_id: null
103 private: false
104 hub_model_card_template_path: null
105logging:
106 file:
107 per_batch_metrics_jsonl: true
108 wandb:
109 enable: true
110 project: prestar
111 entity: latent-thoughts
112 group: pure-CPT
113 name: null
114 tags: []
115 n_examples_logged: 10
116 notes_template_path: configs/prestar/wandb_templates/default.j2
117 debug:
118 enabled: false
119 cross_rank_checks: false
120hardware:
121 num_gpus: 8
122 cpus_per_gpu: 8
123 gpu_type: null
124 peak_bf16_tflops: null
125system:
126 vllm: null
127 trainee:
128 attention_implementation: flash_attention_2
129 enable_gradient_checkpointing: false
130 enable_activation_offload: false
131 fsdp:
132 param_offload: false
133 optimizer_offload: false
134 fsdp_size: -1
135 strategy: no_shard
136 reshard_after_forward: false
137run_name: (0.6B)/(40M-20M)/(no-thoughts,r=0.0,k=8)/(lr=0.00015,epochs=2,replay=0.25)/(seed=4)
138outputs_root: outputs/prestar