Views
No views yet
axolotl, with the following configuration1base_model: Qwen/Qwen2.5-VL-7B-Instruct
2processor_type: AutoProcessor
3
4# these 3 lines are needed for now to handle vision chat templates w images
5skip_prepare_dataset: true
6remove_unused_columns: false
7sample_packing: false
8
9chat_template: qwen2_vl
10datasets:
11 - path: e-zorzi/reasoning_distractors_choice_chat
12 type: chat_template
13 split: train
14 revision: small
15
16test_datasets:
17 - path: e-zorzi/reasoning_distractors_choice_chat
18 type: chat_template
19 split: val_seen[:10%]
20 revision: small
21 - path: e-zorzi/reasoning_distractors_choice_chat
22 type: chat_template
23 split: val_unseen[:10%]
24 revision: small
25
26output_dir: ../ut-persistent/outputs/qwen2_5_VL_7B_lora
27
28load_in_8bit: True
29adapter: lora
30lora_model_dir:
31
32sequence_len: 2048 #8192
33pad_to_sequence_len: false
34
35lora_r: 128
36lora_alpha: 16
37lora_dropout: 0.05
38lora_target_modules: 'model.language_model.layers.[\d]+.(mlp|cross_attn|self_attn).(up|down|gate|q|k|v|o)_proj'
39
40wandb_project: axolotl_finetunes
41wandb_entity: edo_vi
42wandb_watch:
43wandb_name:
44wandb_log_model:
45
46gradient_accumulation_steps: 2
47micro_batch_size: 32
48num_epochs: 15
49optimizer: adamw_bnb_8bit
50lr_scheduler: cosine
51learning_rate: 0.001
52
53bf16: true
54fp16:
55tf32: true
56
57gradient_checkpointing: true
58logging_steps: 1
59flash_attention: true
60eager_attention:
61
62warmup_ratio: 0.0067
63evals_per_epoch: 3
64saves_per_epoch: 1
65save_strategy: epoch
66weight_decay: 0.0
67
68# save_first_step: true # uncomment this to validate checkpoint saving works with your config
69base_model: Qwen/Qwen2.5-VL-7B-Instruct
70processor_type: AutoProcessor
71
72# these 3 lines are needed for now to handle vision chat templates w images
73skip_prepare_dataset: true
74remove_unused_columns: false
75sample_packing: false
76
77chat_template: qwen2_vl
78datasets:
79 - path: e-zorzi/reasoning_distractors_choice_chat
80 type: chat_template
81 split: train
82 revision: small
83
84test_datasets:
85 - path: e-zorzi/reasoning_distractors_choice_chat
86 type: chat_template
87 split: val_seen[:10%]
88 revision: small
89 - path: e-zorzi/reasoning_distractors_choice_chat
90 type: chat_template
91 split: val_unseen[:10%]
92 revision: small
93
94load_in_8bit: True
95adapter: lora
96lora_model_dir:
97
98sequence_len: 2048 #8192
99pad_to_sequence_len: false
100
101lora_r: 128
102lora_alpha: 16
103lora_dropout: 0.05
104lora_target_modules: 'model.language_model.layers.[\d]+.(mlp|cross_attn|self_attn).(up|down|gate|q|k|v|o)_proj'
105
106gradient_accumulation_steps: 2
107micro_batch_size: 32
108num_epochs: 15
109optimizer: adamw_bnb_8bit
110lr_scheduler: cosine
111learning_rate: 0.001
112
113bf16: true
114fp16:
115tf32: true
116
117gradient_checkpointing: true
118logging_steps: 1
119flash_attention: true
120eager_attention:
121
122warmup_ratio: 0.0067
123evals_per_epoch: 3
124saves_per_epoch: 1
125save_strategy: epoch
126weight_decay: 0.0
127
128# save_first_step: true # uncomment this to validate checkpoint saving works with your config
129