Views
No views yet
1callbacks:
2 grad_norm:
3 _target_: primer.callbacks.grad_norm.GradNorm
4 check_clipping: false
5 group_separator: /
6 histogram_freq: null
7 log_weight_distribution: false
8 norm_type: 2
9 only_total: true
10 lr_monitor:
11 _target_: primer.callbacks.lr_monitor.SimpleLearningRateMonitor
12 model_checkpoint:
13 _target_: primer.callbacks.model_checkpoint.ModelCheckpoint
14 dirpath: .checkpoints
15 enable_version_counter: false
16 every_n_train_steps: 2000
17 filename: '{step}'
18 save_initial_checkpoint: true
19 save_last: link
20 save_top_k: -1
21 verbose: true
22 speed_monitor:
23 _target_: primer.callbacks.speed_monitor.SpeedMonitor
24data:
25 batch_size: 64
26 drop_last: true
27 eval_batch_size: 64
28 intra_doc_causal_mask: true
29 multiprocessing_context: null
30 num_workers: 8
31 persistent_workers: false
32 pin_memory: true
33 prefetch_factor: 2
34 shuffle_seed: 42
35loggers:
36 tensorboard:
37 _target_: primer.trainer.TensorBoardLogger
38 name: ''
39 save_dir: ./
40 version: null
41model:
42 attention_bias: false
43 attention_dropout: 0.0
44 head_dim: 128
45 hidden_act: silu
46 hidden_size: 768
47 initializer_range: 0.02
48 intermediate_size: 2048
49 mlp_bias: false
50 model_type: llama
51 name: small
52 num_attention_heads: 6
53 num_hidden_layers: 6
54 num_key_value_heads: 6
55 pretraining_tp: 1
56 rms_norm_eps: 1.0e-05
57 rope_scaling: null
58 rope_theta: 10000.0
59 tie_word_embeddings: true
60optim:
61 grad_acc_schedule:
62 0: 2
63 lr: 0.0006
64 num_warmup_steps: 2000
65 optim_kwargs:
66 betas:
67 - 0.9
68 - 0.95
69 capturable: true
70 eps: 1.0e-08
71 fused: true
72 optim_name: adamw
73 scheduler_kwargs:
74 min_lr_ratio: 0.0
75 num_decay_steps: 4000
76 scheduler_name: warmup_stable_decay
77 set_grad_to_none: true
78 weight_decay: 0.01
79 weight_decay_embedding: false
80 zloss_factor: null
81out_parent_folder: model_train
82pwd: ./unimixlm
83resume_from_checkpoint: .checkpoints/last.ckpt
84run_folder: small_langspec128k__2025-07-27T16-12-01
85save_initial_checkpoint: true
86seed: 42
87tok_name: langspec128k
88tok_path: ./unimixlm/tokenizers/langspec128k
89tok_subfolder: null
90torch_compile: true
91train_data_path: ./unimixlm/data/langspec128k/train
92trainer:
93 accelerator: gpu
94 deterministic: false
95 devices: 1
96 enable_progress_bar: true
97 fast_dev_run: false
98 gradient_clip_algorithm: norm
99 gradient_clip_val: 1.0
100 limit_train_batches: null
101 limit_val_batches: 500
102 log_every_n_steps: 1
103 max_steps: 50000
104 precision: bf16-true
105 val_check_interval: 2000
106use_liger: true
107val_data_path: ./unimixlm/data/langspec128k/validation