1model_name_or_path: merged
2output_dir: runs/qwen35-27b-stage3-instruct-v2
3attn_implementation: flash_attention_2
4bf16: true
5gradient_checkpointing: true
6gradient_checkpointing_kwargs:
7 use_reentrant: false
8use_cce: true
9model_parallel: true
10max_memory:
11 0: 18GiB
12 1: 18GiB
13chunked_mlp: true
14chunked_mlp_chunks: 8
15max_length: 6144
16per_device_train_batch_size: 1
17gradient_accumulation_steps: 8
18use_peft: true
19load_in_4bit: true
20bnb_4bit_quant_type: nf4
21lora_r: 64
22lora_alpha: 64
23lora_dropout: 0.0
24use_rslora: true
25lora_target_modules:
26- in_proj_qkv
27- in_proj_z
28- in_proj_a
29- in_proj_b
30- out_proj
31- q_proj
32- k_proj
33- v_proj
34- o_proj
35- gate_proj
36- up_proj
37- down_proj
38data_config: configs/qwen35-27b-stage3-instruct-v2/data.yaml
39prepared_dataset: runs/qwen35-27b-stage3-instruct-v2/prepared
40auto_mask_reasoning: true
41learning_rate: 1.0e-05
42lr_scheduler_type: constant_with_warmup
43warmup_ratio: 0.03
44weight_decay: 0.01
45max_grad_norm: 1.0
46optim: paged_adamw_8bit
47num_train_epochs: 1
48logging_steps: 1
49disable_tqdm: false
50save_strategy: steps
51save_steps: 250
52save_total_limit: 3
53report_to: wandb
54run_name: qwen35-27b-stage3-instruct-v2
