Views
No views yet
1callbacks:
2 grad_accum:
3 _target_: src.callbacks.gradient_accumulation.GradientAccumulationScheduler
4 scheduling:
5 0: 4
6 grad_norm:
7 _target_: src.callbacks.grad_norm.GradNorm
8 check_clipping: false
9 group_separator: /
10 histogram_freq: null
11 log_weight_distribution: false
12 norm_type: 2
13 only_total: true
14 lr_monitor:
15 _target_: src.callbacks.lr_monitor.SimpleLearningRateMonitor
16 model_checkpoint:
17 _target_: src.callbacks.model_checkpoint.ModelCheckpoint
18 dirpath: .checkpoints
19 enable_version_counter: false
20 every_n_train_steps: 2000
21 filename: '{step}'
22 save_initial_checkpoint: true
23 save_last: link
24 save_top_k: -1
25 verbose: true
26 speed_monitor:
27 _target_: src.callbacks.speed_monitor.SpeedMonitor
28data:
29 batch_size: 32
30 drop_last: false
31 eval_batch_size: 128
32 multiprocessing_context: null
33 num_workers: 8
34 persistent_workers: false
35 pin_memory: true
36 prefetch_factor: 2
37 shuffle: true
38dataset: finewebedu-20B
39loggers:
40 tensorboard:
41 _target_: src.trainer.TensorBoardLogger
42 name: ''
43 save_dir: ./
44 version: null
45model: me100M-tied
46optim:
47 lr: 0.0006
48 num_warmup_steps: 2000
49 optim_kwargs:
50 betas:
51 - 0.9
52 - 0.95
53 eps: 1.0e-08
54 fused: true
55 optim_name: adamw
56 scheduler_kwargs:
57 min_lr_ratio: 0.01
58 num_decay_steps: 4000
59 num_stable_steps: 44000
60 scheduler_name: warmup_stable_decay
61 weight_decay: 0.01
62out_parent_folder: model_train
63pwd: ./rdd/
64resume_from_checkpoint: .checkpoints/last.ckpt
65run_folder: me100M_finewebedu-20B_bpe32000minipile
66save_initial_checkpoint: true
67seed: 42
68tok_name: bpe32000minipile
69tok_path: ./rdd//outputs/tokenizers/bpe32000minipile
70torch_compile: true
71train_data_path: ./rdd//data/finewebedu-20B/bpe32000minipile/train
72trainer:
73 accelerator: gpu
74 deterministic: false
75 devices: 1
76 enable_progress_bar: true
77 fast_dev_run: false
78 gradient_clip_algorithm: norm
79 gradient_clip_val: 1.0
80 limit_val_batches: 500
81 log_every_n_steps: 1
82 max_steps: 50000
83 precision: bf16-true
84 val_check_interval: 2000
85val_data_path: ./rdd//data/finewebedu-20B/bpe32000minipile/validation