Views
No views yet
0.13.0.dev01base_model: Goader/gemma-3-12b-pt-focus
2tokenizer_config: lapa-llm/tokenizer
3
4ddp_find_unused_parameters: false #Chat-GPT recomendation: Set this to false unless you know there are unused parameters. It introduces overhead during distributed training.
5shuffle_merged_datasets: false
6shuffle_before_merging_datasets: false # Shuffle each dataset individually before merging them
7#Need to discuss how to load datas (datasets / pretraining_dataset)
8pretraining_dataset:
9 - path: Goader/kobza-2m-jsonl
10 type: pretrain
11# - path: le-llm/high-estimated-pretraining-data
12# type:
13pretrain_multipack_buffer_size: 10000
14dataset_processes: 64
15# dataset_keep_in_memory: true
16dataloader_num_workers: 8
17dataloader_prefetch_factor: 9
18
19output_dir: ./outputs/gemma-3-12b-focus-pt
20dataset_prepared_path: last_run_prepared_embeddings
21
22sequence_len: 1024
23sample_packing: true
24pad_to_sequence_len: true
25train_on_inputs: true #maybe unnecessary for pretraining
26
27# Number of GPUs for Tensor Parallelism.
28
29# deepspeed: #Need to discuss deepspeed configuratiin, I recommend to use deepspeed zero2 for faster training
30#My deepspeed Zero2 config:
31# {
32# "zero_optimization": {
33# "stage": 2,
34# "overlap_comm": true,
35# "contiguous_gradients": true,
36# "sub_group_size": 0,
37# "reduce_scatter": true,
38# "allgather_bucket_size": 500000000,
39# "reduce_bucket_size": 500000000
40# },
41# "bf16": { "enabled": true },
42# "gradient_accumulation_steps": "auto",
43# "gradient_clipping": "auto",
44# "train_batch_size": "auto",
45# "train_micro_batch_size_per_gpu": "auto",
46# "wall_clock_breakdown": false
47# }
48# deepcompile: true # TODO: let's try? should speed up training
49
50unfrozen_parameters:
51 - ^lm_head.weight$
52 - ^model.language_model.embed_tokens.weight$
53
54plugins:
55 - axolotl.integrations.liger.LigerPlugin
56liger_rope: true
57liger_rms_norm: true
58liger_glu_activation: true
59liger_layer_norm: true
60liger_fused_linear_cross_entropy: true
61
62wandb_project: matt
63wandb_entity:
64wandb_watch:
65wandb_name:
66wandb_log_model:
67
68gradient_accumulation_steps: 2
69micro_batch_size: 4
70# num_epochs: 1
71max_steps: 15000 # better to avoid using max_steps and use num_epochs instead if will be able to use not streaming dataset
72save_steps: 5000 #(around 15-20 checkpoints) Taking into accout that we have problem with resuming training from checkpoints lets use checkpointing for tracking training in different stages
73save_total_limit: 30
74
75ddp_timeout: 7200 # TODO: timeout of 2 hours for waiting for nodes
76
77gradient_checkpointing: false #not find the way to avoid using this due GPU memory limitations. potentially set in false - speed-up training on 20-30%
78#gradient_checkpointing_kwargs:
79# use_reentrant: false
80logging_steps: 10
81flash_attention: true
82
83#Some colculations for our case by Chat-GPT based on https://arxiv.org/pdf/2507.07101 - https://chatgpt.com/s/t_68c57258c87c81918b9da47a017888e8
84optimizer: adamw_torch_fused #adamw_bnb_8bit
85warmup_ratio: 0.1
86lr_scheduler: warmup_stable_decay
87lr_scheduler_kwargs: {
88 "num_decay_steps": 10000, # 30% in KIMI K2 paper . Should be calculated based on size of good dataset. Recomended by Chat-GPT Decay (~32%) around 45k steps for our case
89 "min_lr_ratio": 0.05}
90
91learning_rate: 5e-5
92max_grad_norm: 1.0
93# adamw hyperparams
94adam_epsilon: 1e-6 #bf16 edge-cases
95# adamw hyperparams
96adam_beta1: 0.9
97# adamw hyperparams
98# adam_beta2: 0.977543
99adam_beta2: 0.98
100# Could be 0.01 but CheGPT recommended exclude LayerNorm and bias from decay if want to use 0.01
101weight_decay: 0.01
102