Views
No views yet
0.5.01base_model: TinyLlama/TinyLlama-1.1B-Chat-v1.0
2model_type: AutoModelForCausalLM
3# Automatically upload checkpoint and final model to HF
4# hub_model_id: username/custom_model_name
5
6plugins:
7 - axolotl.integrations.liger.LigerPlugin
8
9liger_glu_activation: true
10liger_rms_norm: true
11liger_layer_norm: true
12
13load_in_4bit: true
14adapter: qlora
15lora_r: 32
16lora_alpha: 64
17lora_target_modules:
18 - self_attn.q_proj
19 - self_attn.k_proj
20 - self_attn.v_proj
21 - self_attn.o_proj
22 - shared_expert.gate_proj
23 - shared_expert.up_proj
24 - shared_expert.down_proj
25 # - experts.gate_projs.[0-9]+$ # optionally train the moe experts
26 # - experts.up_projs.[0-9]+$
27 # - experts.down_projs.[0-9]+$
28lora_modules_to_save:
29 - lm_head # needed if modifying vocabulary
30 - embed_tokens
31
32lora_mlp_kernel: true
33lora_qkv_kernel: true
34lora_o_kernel: true
35
36chat_template: chatml
37datasets:
38 - path: mlabonne/FineTome-100k
39 type: chat_template
40 split: train[:20%]
41 field_messages: conversations
42 message_field_role: from
43 message_field_content: value
44
45dataset_prepared_path: last_run_prepared
46val_set_size: 0.0
47output_dir: ./outputs/out
48
49sequence_len: 4096 # up to 8k will work on a single H100
50sample_packing: true
51
52
53gradient_accumulation_steps: 1
54micro_batch_size: 1
55num_epochs: 1
56optimizer: adamw_8bit
57lr_scheduler: cosine
58learning_rate: 1e-4
59
60bf16: true
61tf32: true
62
63torch_compile: true
64flex_attention: true
65flex_attn_compile_kwargs:
66 dynamic: false
67 mode: max-autotune-no-cudagraphs
68
69gradient_checkpointing: true
70gradient_checkpointing_kwargs:
71 use_reentrant: false
72
73logging_steps: 1
74warmup_ratio: 0.1
75evals_per_epoch: 1
76saves_per_epoch: 1
77
78weight_decay: 0.0
79special_tokens:
80 pad_token: <|finetune_right_pad|>
81 eos_token: <|eot|>
82
83# save_first_step: true # uncomment this to validate checkpoint saving works with your config
84
85