Views
No views yet
latest812(0.6B)/(40M-20M)/(32B-predict-k=8,L=16,r=0.03125,k=8)/(lr=0.00015,epochs=2,replay=0.25)/(seed=0)/(val-r=1.0)model/ and the tokenizer under tokenizer/ in this repo.1trainee:
2 short_name: 0.6B
3 model_id: Qwen/Qwen3-0.6B
4 dtype: bfloat16
5data:
6 train:
7 documents:
8 short_name: 40M-20M
9 dataset_id: JackHsieh/statML-arxiv-40M-20M
10 split: train
11 n_samples: null
12 exact_document_length: 4096
13 chunking:
14 rule: r=0.03125,k=8
15 chunk_size: 8
16 path: outputs/prestar/chunking/r=0.03125,k=8/40M-20M/train.parquet
17 thoughts:
18 short_name: 32B-predict-k=8,L=16
19 dataset_id: JackHsieh/32B-predict.rule-r-1.0-k-8.L-16.statml-arxiv
20 split: train
21 n_samples: null
22 replay:
23 short_name: full-replay
24 dataset_id: JackHsieh/dclm-replay.seq-4096.tokens-32B
25 split: train
26 n_samples: null
27 exact_document_length: 4096
28 serving:
29 use_thoughts: true
30 replay_proportion: 0.25
31 thoughts_per_step: 1
32 fixed_main_order: true
33 fixed_replay_order: true
34 eval:
35 documents:
36 short_name: 40M-20M
37 dataset_id: JackHsieh/statML-arxiv-40M-20M
38 split: test
39 n_samples: null
40 exact_document_length: 4096
41 chunking:
42 rule: r=1.0,k=8
43 chunk_size: 8
44 path: outputs/prestar/chunking/r=1.0,k=8/40M-20M/test.parquet
45 thoughts:
46 short_name: 32B-predict-k=8,L=16
47 dataset_id: JackHsieh/32B-predict.rule-r-1.0-k-8.L-16.statml-arxiv
48 split: test
49 n_samples: null
50 serving:
51 use_thoughts: true
52 thoughts_per_pass: 1
53train:
54 docs_per_batch: 32
55 total_epochs: 2
56 total_steps: null
57 grad_clip_norm: 1.0
58 seed: 0
59 early_stopping:
60 enable: false
61 mode: null
62 metric: null
63 patience_evals: null
64 min_delta: null
65 loss: {}
66 execution:
67 thoughtless_pass_doc_microbatch: 4
68 thoughtful_pass_thought_microbatch: 8
69 save_incremental_per_epoch: false
70 save_per_token_log_probs: false
71 save_chunk_token_snippets: false
72 optimizer:
73 style: adamw
74 lr: 0.00015
75 weight_decay: 0.01
76 beta1: 0.9
77 beta2: 0.95
78 eps: 1.0e-08
79 lr_scheduler:
80 style: cosine
81 total_steps: null
82 min_lr_ratio: 0.0
83 warmup_steps: null
84 warmup_ratio: 0.05
85 stable_steps: null
86 decay_steps: null
87eval:
88 eval_every_n_steps: null
89 eval_every_n_epochs: 1
90 eval_on_first_step: false
91 execution:
92 docs_per_batch: null
93 thoughtless_pass_doc_microbatch: 32
94 thoughtful_pass_thought_microbatch: 16
95 incremental_save_every_n_batches: 0
96 save_per_token_log_probs: true
97 save_chunk_token_snippets: false
98checkpointing:
99 enable: true
100 save_best: false
101 save_latest: true
102 auto_resume: true
103 resume_from_path: null
104 resume_wandb_run: true
105 best_max_keep: 1
106 latest_max_keep: 1
107 best_includes_resume_state: false
108 best_metric: val-dynamics/val_loss
109 best_metric_direction: min
110 latest_every_n_evals: 1
111 push_best_to_hub: false
112 push_latest_to_hub: true
113 best_repo_id: null
114 latest_repo_id: null
115 private: false
116 hub_model_card_template_path: null
117logging:
118 file:
119 per_batch_metrics_jsonl: true
120 wandb:
121 enable: true
122 project: prestar
123 entity: latent-thoughts
124 group: thoughtful-CPT-32B-predict-fp32
125 name: null
126 tags: []
127 n_examples_logged: 10
128 notes_template_path: configs/prestar/wandb_templates/default.j2
129 debug:
130 enabled: false
131 cross_rank_checks: false
132hardware:
133 num_gpus: 8
134 cpus_per_gpu: 8
135 gpu_type: null
136 peak_bf16_tflops: null
137system:
138 vllm: null
139 trainee:
140 attention_implementation: flash_attention_2
141 enable_gradient_checkpointing: false
142 enable_activation_offload: false
143 fsdp:
144 param_offload: false
145 optimizer_offload: false
146 fsdp_size: -1
147 strategy: no_shard
148 reshard_after_forward: false
149 master_weights_fp32: true
150run_name: (0.6B)/(40M-20M)/(32B-predict-k=8,L=16,r=0.03125,k=8)/(lr=0.00015,epochs=2,replay=0.25)/(seed=0)/(val-r=1.0)
151outputs_root: outputs/prestar