Views
No views yet
0.10.01base_model: ./gemma
2model_type: Gemma3ForCausalLM
3
4model_quantization_config: null
5load_in_4bit: false
6load_in_8bit: false
7
8# gemma3 doesn't seem to play nice with ddp
9# ddp_find_unused_parameters: true
10
11datasets:
12 - path: ./dataset/boolq/train/
13 type:
14 system_prompt: ""
15 field_system: system
16 field_instruction: instruction
17 field_output: answer
18 format: "{instruction}\nAnswer:"
19 no_input_format: "{instruction}\nAnswer:"
20 ds_type: arrow
21
22output_dir: ./outputs/gemma/boolq_2
23
24adapter: lora
25lora_model_dir:
26
27sequence_len: 512
28sample_packing: true
29
30lora_r: 8
31lora_alpha: 16
32lora_dropout: 0.1
33lora_target_linear: true
34lora_fan_in_fan_out:
35lora_target_modules:
36 - gate_proj
37 - down_proj
38 - up_proj
39 - q_proj
40 - v_proj
41 - k_proj
42 - o_proj
43
44wandb_project:
45wandb_entity:
46wandb_watch:
47wandb_name:
48wandb_log_model:
49
50gradient_accumulation_steps: 1
51micro_batch_size: 6
52num_epochs: 5
53optimizer: adamw_bnb_8bit
54lr_scheduler: cosine
55learning_rate: 0.0001
56
57bf16: true
58fp16:
59tf32: true
60
61gradient_checkpointing: false
62gradient_checkpointing_kwargs:
63 use_reentrant: false
64logging_steps: 1
65flash_attention: true
66eager_attention:
67
68warmup_ratio: 0.1
69evals_per_epoch: 1
70saves_per_epoch: 1
71weight_decay: 0.0
72
73plugins:
74 - axolotl.integrations.liger.LigerPlugin
75liger_rope: true
76liger_rms_norm: true
77liger_glu_activation: true
78liger_layer_norm: true
79liger_fused_linear_cross_entropy: true
80
81