Views
No views yet
latest1216(0.6B)/(40M-20M)/(32B-predict-k=8,L=16,r=0.03125,k=8)/(lr=0.0001,epochs=2,replay=0.5)/(seed=0)/(val-r=1.0)model/ and the tokenizer under tokenizer/ in this repo.1trainee:
2 short_name: 0.6B
3 model_id: Qwen/Qwen3-0.6B
4 dtype: bfloat16
5data:
6 train:
7 documents:
8 short_name: 40M-20M
9 dataset_id: JackHsieh/statML-arxiv-40M-20M
10 split: train
11 n_samples: null
12 exact_document_length: 4096
13 chunking:
14 rule: r=0.03125,k=8
15 chunk_size: 8
16 path: outputs/prestar/chunking/r=0.03125,k=8/40M-20M/train.parquet
17 thoughts:
18 short_name: 32B-predict-k=8,L=16
19 dataset_id: JackHsieh/32B-predict.rule-r-1.0-k-8.L-16.statml-arxiv
20 split: train
21 n_samples: null
22 replay:
23 short_name: full-replay
24 dataset_id: JackHsieh/dclm-replay.seq-4096.tokens-32B
25 split: train
26 n_samples: null
27 exact_document_length: 4096
28 serving:
29 use_thoughts: true
30 replay_proportion: 0.5
31 thoughts_per_step: 1
32 fixed_main_order: true
33 fixed_replay_order: true
34 eval:
35 documents:
36 short_name: 40M-20M
37 dataset_id: JackHsieh/statML-arxiv-40M-20M
38 split: test
39 n_samples: null
40 exact_document_length: 4096
41 chunking:
42 rule: r=1.0,k=8
43 chunk_size: 8
44 path: outputs/prestar/chunking/r=1.0,k=8/40M-20M/test.parquet
45 thoughts:
46 short_name: 32B-predict-k=8,L=16
47 dataset_id: JackHsieh/32B-predict.rule-r-1.0-k-8.L-16.statml-arxiv
48 split: test
49 n_samples: null
50 serving:
51 use_thoughts: true
52 thoughts_per_pass: 1
53train:
54 docs_per_batch: 32
55 total_epochs: 2
56 total_steps: null
57 grad_clip_norm: 1.0
58 seed: 0
59 early_stopping:
60 enable: false
61 mode: null
62 metric: null
63 patience_evals: null
64 min_delta: null
65 loss: {}
66 execution:
67 thoughtless_pass_doc_microbatch: 4
68 thoughtful_pass_thought_microbatch: 8
69 save_incremental_per_epoch: false
70 save_per_token_log_probs: false
71 save_chunk_token_snippets: false
72 optimizer:
73 style: adamw
74 lr: 0.0001
75 weight_decay: 0.01
76 beta1: 0.9
77 beta2: 0.95
78 eps: 1.0e-08
79 lr_scheduler:
80 style: cosine
81 total_steps: null
82 min_lr_ratio: 0.0
83 warmup_steps: null
84 warmup_ratio: 0.05
85 stable_steps: null
86 decay_steps: null
87eval:
88 eval_every_n_steps: null
89 eval_every_n_epochs: 2
90 eval_on_first_step: false
91 execution:
92 docs_per_batch: null
93 thoughtless_pass_doc_microbatch: 32
94 thoughtful_pass_thought_microbatch: 16
95 incremental_save_every_n_batches: 0
96 save_per_token_log_probs: true
97 save_chunk_token_snippets: false
98checkpointing:
99 resume:
100 from_latest: true
101 from_path: null
102 same_wandb_run: true
103 best:
104 enable: false
105 metric: val-dynamics/val_loss
106 direction: min
107 max_keep: 1
108 includes_resume_state: false
109 hub:
110 enable: false
111 repo_id: null
112 private: false
113 model_card_template_path: null
114 latest:
115 enable: true
116 max_keep: 1
117 includes_resume_state: true
118 cadence:
119 unit: epochs
120 every: 0.25
121 hub:
122 enable: true
123 repo_id: null
124 private: false
125 model_card_template_path: null
126logging:
127 file:
128 per_batch_metrics_jsonl: true
129 wandb:
130 enable: true
131 project: prestar
132 entity: latent-thoughts
133 group: thoughtful-CPT-32B-predict-fp32
134 name: null
135 tags: []
136 n_examples_logged: 10
137 notes_template_path: configs/prestar/wandb_templates/default.j2
138 debug:
139 enabled: false
140 cross_rank_checks: false
141hardware:
142 num_gpus: 8
143 cpus_per_gpu: 8
144 gpu_type: null
145 peak_bf16_tflops: null
146system:
147 vllm: null
148 trainee:
149 attention_implementation: flash_attention_2
150 enable_gradient_checkpointing: false
151 enable_activation_offload: false
152 fsdp:
153 param_offload: false
154 optimizer_offload: false
155 fsdp_size: -1
156 strategy: no_shard
157 reshard_after_forward: false
158 master_weights_fp32: true
159run_name: (0.6B)/(40M-20M)/(32B-predict-k=8,L=16,r=0.03125,k=8)/(lr=0.0001,epochs=2,replay=0.5)/(seed=0)/(val-r=1.0)
160outputs_root: outputs/prestar