Views
No views yet
0.10.01base_model: ./gemma
2model_type: Gemma3ForCausalLM
3
4model_quantization_config: null
5load_in_4bit: false
6load_in_8bit: false
7
8# gemma3 doesn't seem to play nice with ddp
9# ddp_find_unused_parameters: true
10
11datasets:
12 - path: ./dataset/math_genie/
13 type:
14 system_prompt: ""
15 field_system: system
16 field_instruction: question
17 field_output: answer
18 format: "Question: {instruction}\nAnswer: "
19 no_input_format: "Question: {instruction}\nAnswer: "
20 ds_type: arrow
21
22train_on_inputs: false
23
24output_dir: ./outputs/gemma/math_genie_2
25
26adapter: lora
27lora_model_dir:
28
29sequence_len: 1500
30sample_packing: true
31
32lora_r: 8
33lora_alpha: 16
34lora_dropout: 0.1
35lora_target_linear: true
36lora_fan_in_fan_out:
37lora_target_modules:
38 - gate_proj
39 - down_proj
40 - up_proj
41 - q_proj
42 - v_proj
43 - k_proj
44 - o_proj
45
46wandb_project:
47wandb_entity:
48wandb_watch:
49wandb_name:
50wandb_log_model:
51
52gradient_accumulation_steps: 4
53micro_batch_size: 2
54num_epochs: 5
55optimizer: adamw_bnb_8bit
56lr_scheduler: cosine
57learning_rate: 5e-5
58
59bf16: true
60fp16:
61tf32: true
62
63gradient_checkpointing: false
64gradient_checkpointing_kwargs:
65 use_reentrant: false
66logging_steps: 1
67flash_attention: true
68eager_attention:
69
70warmup_ratio: 0.1
71evals_per_epoch: 1
72saves_per_epoch: 1
73weight_decay: 0.0
74
75plugins:
76 - axolotl.integrations.liger.LigerPlugin
77liger_rope: true
78liger_rms_norm: true
79liger_glu_activation: true
80liger_layer_norm: true
81liger_fused_linear_cross_entropy: true
82
83