Views
No views yet
0.9.21######################################
2# CONTINUED PRE-TRAINING EXAMPLE #
3######################################
4
5base_model: Qwen/Qwen3-0.6B-Base # the checkpoint you start from
6strict: false
7
8# 1⃣ Replace `datasets:` with `pretraining_dataset:`
9pretraining_dataset:
10 - path: timarni/pretrain-textbooks # or HF dataset id
11 type: completion # accepted values: text | completion | HF dataset
12
13# 2⃣ Remove chat / instruction-tuning options
14chat_template:
15# adapter / lora stay null/false (full-parameter training)
16
17# 3⃣ Training hyper-params (see Section 3)
18sequence_len: 4096
19sample_packing: true
20eval_sample_packing: true
21pad_to_sequence_len: true
22
23micro_batch_size: 1
24gradient_accumulation_steps: 2
25max_steps: 3500 # or use max_steps instead
26learning_rate: 1e-5
27lr_scheduler: cosine
28warmup_steps: 100
29weight_decay: 0.01
30
31optimizer: adamw_torch
32bf16: auto
33tf32: true
34flash_attention: true
35gradient_checkpointing: offload
36
37val_set_size: 0.0 # usually no dev set for plain pre-training
38output_dir: ./outputs/qwen3_pretrain_textbook
39dataset_prepared_path: last_run_prepared
40
41wandb_project: mnlp_project
42wandb_entity: tim-arni
43wandb_name: qwen3-0.6B-pretrain_textbook
44
45