Views
No views yet
0.13.0.dev01base_model: Goader/mistral-7b-v0.1-ukr-aim-mse-frozen-100k
2tokenizer_config: Goader/mistral-7b-v0.1-ukr-aim-mse-frozen-100k
3
4ddp_find_unused_parameters: false #Chat-GPT recomendation: Set this to false unless you know there are unused parameters. It introduces overhead during distributed training.
5shuffle_merged_datasets: false
6shuffle_before_merging_datasets: false # Shuffle each dataset individually before merging them
7#Need to discuss how to load datas (datasets / pretraining_dataset)
8pretraining_dataset:
9 - path: Goader/kobza-2m-jsonl
10 type: pretrain
11# - path: le-llm/high-estimated-pretraining-data
12# type:
13pretrain_multipack_buffer_size: 10000
14dataset_processes: 64
15# dataset_keep_in_memory: true
16dataloader_num_workers: 8
17dataloader_prefetch_factor: 9
18
19output_dir: ./outputs/mistral-7b-v0.1-ukr-matt-pt
20dataset_prepared_path: last_run_prepared_embeddings
21
22sequence_len: 1024
23sample_packing: true
24pad_to_sequence_len: true
25train_on_inputs: true #maybe unnecessary for pretraining
26
27# Number of GPUs for Tensor Parallelism.
28
29# deepspeed: #Need to discuss deepspeed configuratiin, I recommend to use deepspeed zero2 for faster training
30#My deepspeed Zero2 config:
31# {
32# "zero_optimization": {
33# "stage": 2,
34# "overlap_comm": true,
35# "contiguous_gradients": true,
36# "sub_group_size": 0,
37# "reduce_scatter": true,
38# "allgather_bucket_size": 500000000,
39# "reduce_bucket_size": 500000000
40# },
41# "bf16": { "enabled": true },
42# "gradient_accumulation_steps": "auto",
43# "gradient_clipping": "auto",
44# "train_batch_size": "auto",
45# "train_micro_batch_size_per_gpu": "auto",
46# "wall_clock_breakdown": false
47# }
48# deepcompile: true # TODO: let's try? should speed up training
49
50unfrozen_parameters:
51 - ^lm_head.weight$
52
53plugins:
54 - axolotl.integrations.liger.LigerPlugin
55liger_rope: true
56liger_rms_norm: true
57liger_glu_activation: true
58liger_layer_norm: true
59liger_fused_linear_cross_entropy: true
60
61wandb_project: matt
62wandb_entity:
63wandb_watch:
64wandb_name:
65wandb_log_model:
66
67gradient_accumulation_steps: 2
68micro_batch_size: 4
69# num_epochs: 1
70max_steps: 9000 # better to avoid using max_steps and use num_epochs instead if will be able to use not streaming dataset
71save_steps: 3000 #(around 15-20 checkpoints) Taking into accout that we have problem with resuming training from checkpoints lets use checkpointing for tracking training in different stages
72save_total_limit: 30
73
74ddp_timeout: 7200 # TODO: timeout of 2 hours for waiting for nodes
75
76gradient_checkpointing: false #not find the way to avoid using this due GPU memory limitations. potentially set in false - speed-up training on 20-30%
77#gradient_checkpointing_kwargs:
78# use_reentrant: false
79logging_steps: 10
80flash_attention: true
81
82#Some colculations for our case by Chat-GPT based on https://arxiv.org/pdf/2507.07101 - https://chatgpt.com/s/t_68c57258c87c81918b9da47a017888e8
83optimizer: adamw_torch_fused #adamw_bnb_8bit
84warmup_ratio: 0.1
85lr_scheduler: warmup_stable_decay
86lr_scheduler_kwargs: {
87 "num_decay_steps": 6000, # 30% in KIMI K2 paper . Should be calculated based on size of good dataset. Recomended by Chat-GPT Decay (~32%) around 45k steps for our case
88 "min_lr_ratio": 0.05}
89# FIXME num decay steps !!!!!!!!!!
90
91learning_rate: 5e-5
92max_grad_norm: 1.0
93# adamw hyperparams
94adam_epsilon: 1e-6 #bf16 edge-cases
95# adamw hyperparams
96adam_beta1: 0.9
97# adamw hyperparams
98# adam_beta2: 0.977543
99adam_beta2: 0.98
100# Could be 0.01 but CheGPT recommended exclude LayerNorm and bias from decay if want to use 0.01
101weight_decay: 0.01
102