Views
No views yet
1callbacks:
2 grad_accum:
3 _target_: src.callbacks.gradient_accumulation.GradientAccumulationScheduler
4 scheduling:
5 0: 2
6 grad_norm:
7 _target_: src.callbacks.grad_norm.GradNorm
8 check_clipping: false
9 group_separator: /
10 histogram_freq: null
11 log_weight_distribution: false
12 norm_type: 2
13 only_total: true
14 lr_monitor:
15 _target_: src.callbacks.lr_monitor.SimpleLearningRateMonitor
16 model_checkpoint:
17 _target_: src.callbacks.model_checkpoint.ModelCheckpoint
18 dirpath: .checkpoints
19 enable_version_counter: false
20 every_n_train_steps: 2000
21 filename: '{step}'
22 save_initial_checkpoint: true
23 save_last: link
24 save_top_k: -1
25 verbose: true
26 speed_monitor:
27 _target_: src.callbacks.speed_monitor.SpeedMonitor
28data:
29 batch_size: 16
30 drop_last: false
31 eval_batch_size: 64
32 multiprocessing_context: null
33 num_workers: 12
34 persistent_workers: false
35 pin_memory: true
36 prefetch_factor: 2
37 shuffle: true
38dataset: finewebedu-20B
39evaluation:
40 blimp: true
41loggers:
42 tensorboard:
43 _target_: src.trainer.TensorBoardLogger
44 name: ''
45 save_dir: ./
46 version: null
47model: fw57M-tied
48optim:
49 lr: 0.0006
50 num_warmup_steps: 2000
51 optim_kwargs:
52 betas:
53 - 0.9
54 - 0.95
55 eps: 1.0e-08
56 fused: true
57 optim_name: adamw
58 scheduler_kwargs:
59 min_lr_ratio: 0.01
60 num_decay_steps: 4000
61 num_stable_steps: 44000
62 scheduler_name: warmup_stable_decay
63 weight_decay: 0.01
64out_parent_folder: model_train
65pwd: /home/zg258/rds/hpc-work/infotokenization
66resume_from_checkpoint: .checkpoints/last.ckpt
67run_folder: .
68save_initial_checkpoint: true
69seed: 42
70tok_name: ByteSpanSurprisalGlobalIncrement_64000
71torch_compile: true
72train_data_path: /home/zg258/rds/hpc-work/infotokenization/data/finewebedu-20B/ByteSpanSurprisalGlobalIncrement_64000/train
73trainer:
74 accelerator: gpu
75 deterministic: false
76 devices: 4
77 enable_progress_bar: true
78 fast_dev_run: false
79 gradient_clip_algorithm: norm
80 gradient_clip_val: 1.0
81 limit_val_batches: 500
82 log_every_n_steps: 1
83 max_steps: 50000
84 precision: bf16-true
85 val_check_interval: 2000
86val_data_path: /home/zg258/rds/hpc-work/infotokenization/data/finewebedu-20B/ByteSpanSurprisalGlobalIncrement_64000/validation