Views
No views yet
0.6.01base_model: meta-llama/Llama-3.1-70B
2tokenizer_type: AutoTokenizer
3
4strict: false
5
6plugins:
7 - axolotl.integrations.liger.LigerPlugin
8 - axolotl.integrations.spectrum.SpectrumPlugin
9liger_rope: true
10liger_rms_norm: true
11liger_swiglu: true
12 # liger_cross_entropy: true
13 # liger_fused_linear_cross_entropy: true
14
15 # torch_compile: true
16
17dataloader_prefetch_factor: 8192
18dataloader_num_workers: 8
19dataloader_pin_memory: true
20
21dataset_processes: 16
22
23chat_template: llama3
24datasets:
25 - path: AI-MO/NuminaMath-CoT
26 type: chat_template
27 split: train
28 field_messages: messages
29 message_field_content: content
30 message_field_role: role
31
32
33dataset_prepared_path: /workspace/data/axolotl-artifacts/last_run_prepared
34val_set_size: 0.0
35output_dir: /workspace/data/axolotl-artifacts/outputs/llama3_1-70b-finetome
36save_safetensors: false # saving final sharded dict may not work with safetensors
37
38wandb_project: numina-kd-experiment
39wandb_entity: axolotl-ai
40
41adapter: lora
42 # peft_use_dora: true
43lora_r: 512
44lora_alpha: 1024
45lora_dropout: 0.05
46lora_target_linear: true
47lora_modules_to_save:
48 - embed_tokens
49 - lm_head
50
51sequence_len: 4096
52sample_packing: true
53pad_to_sequence_len: true
54
55gradient_accumulation_steps: 1
56# 8x Node can support a batch size of up to 3
57micro_batch_size: 8
58num_epochs: 3
59optimizer: optimi_adamw
60lr_scheduler: cosine
61learning_rate: 1.0e-5
62
63train_on_inputs: false
64group_by_length: false
65bf16: true
66tf32: true
67
68gradient_checkpointing: true
69gradient_checkpointing_kwargs:
70 use_reentrant: true
71logging_steps: 1
72flash_attention: true
73
74warmup_steps: 40
75saves_per_epoch: 1
76weight_decay: 0.1
77deepspeed: deepspeed_configs/zero3_bf16_cpuoffload_params.json
78special_tokens:
79 pad_token: <|finetune_right_pad_id|>
80 eos_token: <|eot_id|>
81