Views
No views yet
1callbacks:
2 grad_accum:
3 _target_: src.callbacks.gradient_accumulation.GradientAccumulationScheduler
4 scheduling:
5 0: 16
6 grad_norm:
7 _target_: src.callbacks.grad_norm.GradNorm
8 check_clipping: false
9 group_separator: /
10 histogram_freq: null
11 log_weight_distribution: false
12 norm_type: 2
13 only_total: true
14 lr_monitor:
15 _target_: src.callbacks.lr_monitor.SimpleLearningRateMonitor
16 model_checkpoint:
17 _target_: src.callbacks.model_checkpoint.ModelCheckpoint
18 dirpath: .checkpoints
19 enable_version_counter: false
20 every_n_train_steps: 2000
21 filename: '{step}'
22 save_initial_checkpoint: true
23 save_last: link
24 save_top_k: -1
25 verbose: true
26 speed_monitor:
27 _target_: src.callbacks.speed_monitor.SpeedMonitor
28data:
29 batch_size: 8
30 drop_last: false
31 eval_batch_size: 16
32 multiprocessing_context: null
33 num_workers: 8
34 persistent_workers: false
35 pin_memory: true
36 prefetch_factor: 2
37 shuffle: true
38dataset: minipile
39loggers:
40 tensorboard:
41 _target_: src.loggers.TensorBoardLogger
42 name: ''
43 save_dir: ./
44 version: ./
45model: smol_llama-81M-tied
46optim:
47 lr: 0.0006
48 num_warmup_steps: 2000
49 optim_kwargs:
50 betas:
51 - 0.9
52 - 0.95
53 eps: 1.0e-08
54 fused: true
55 optim_name: adamw
56 scheduler_kwargs:
57 min_lr_ratio: 0.01
58 num_decay_steps: 2000
59 num_stable_steps: 46000
60 scheduler_name: warmup_stable_decay
61 weight_decay: 0.1
62out_parent_folder: model_train
63resume_from_checkpoint: null
64run_folder: minipile/smol_llama-81M-tied_bpe128000minipile_2024-11-01T13-04-36
65save_initial_checkpoint: true
66seed: 42
67tok_name: bpe128000minipile
68tok_path: /home/pl487/rdd/outputs/tokenizers/bpe128000minipile
69torch_compile: true
70train_data_path: /home/pl487/rdd/data/minipile/bpe128000minipile/train
71trainer:
72 accelerator: gpu
73 deterministic: false
74 enable_progress_bar: true
75 fast_dev_run: false
76 gradient_clip_algorithm: norm
77 gradient_clip_val: 1.0
78 limit_val_batches: 500
79 log_every_n_steps: 1
80 max_steps: 50000
81 precision: bf16-true
82 val_check_interval: 2000
83val_data_path: /home/pl487/rdd/data/minipile/bpe128000minipile/validation