Views
No views yet
0.4.11base_model: meta-llama/Llama-3.2-3B
2model_type: AutoModelForCausalLM
3tokenizer_type: AutoTokenizer
4
5plugins:
6 - axolotl.integrations.liger.LigerPlugin
7liger_rope: true
8liger_rms_norm: true
9liger_swiglu: true
10liger_fused_linear_cross_entropy: true
11
12load_in_8bit: false
13load_in_4bit: false
14strict: false
15
16datasets:
17# - path: anthracite-core/c2_logs_32k_mistral-v3_v1.2
18# type: sharegpt
19# conversation: chatml
20 - path: ./datasets/c2_deduped_32k_mistral-v3_tok_deanon_dsclean_1.2.jsonl
21 type: sharegpt
22 conversation: chatml
23# - path: anthracite-org/kalo-opus-instruct-22k-no-refusal
24# type: sharegpt
25# conversation: chatml
26 - path: ./datasets/opus-instruct-22k-no_refusals.jsonl
27 type: sharegpt
28 conversation: chatml
29# - path: lodrick-the-lafted/kalo-opus-instruct-3k-filtered
30# type: sharegpt
31# conversation: chatml
32 - path: ./datasets/kalo-3k-filtered.jsonl
33 type: sharegpt
34 conversation: chatml
35# - path: anthracite-org/nopm_claude_writing_fixed
36# type: sharegpt
37# conversation: chatml
38 - path: ./datasets/claudewritingNopm.jsonl
39 type: sharegpt
40 conversation: chatml
41# - path: anthracite-org/kalo_opus_misc_240827
42# type: sharegpt
43# conversation: chatml
44 - path: ./datasets/kalo_opus_misc_240827.jsonl
45 type: sharegpt
46 conversation: chatml
47# - path: anthracite-org/kalo_misc_part2
48# type: sharegpt
49# conversation: chatml
50 - path: ./datasets/kalo_misc_part2.jsonl
51 type: sharegpt
52 conversation: chatml
53# - path: NewEden/Claude-Instruct-5K
54# type: sharegpt
55# conversation: chatml
56 - path: ./datasets/5k.jsonl
57 type: sharegpt
58 conversation: chatml
59
60#chat_template: chatml
61shuffle_merged_datasets: true
62#default_system_message: "You are an assistant that responds to the user."
63dataset_prepared_path: ./magnum-22b-data
64val_set_size: 0.0
65output_dir: ./22b-fft-out
66
67sequence_len: 16000
68sample_packing: true
69pad_to_sequence_len: true
70
71
72wandb_project: 3bmagnum
73wandb_entity:
74wandb_watch:
75wandb_name: 3magnum
76wandb_log_model:
77
78gradient_accumulation_steps: 32
79micro_batch_size: 1
80num_epochs: 2
81optimizer: adamw_bnb_8bit
82lr_scheduler: cosine
83learning_rate: 0.000005
84
85train_on_inputs: false
86group_by_length: false
87bf16: auto
88fp16:
89tf32: false
90
91gradient_checkpointing: true
92early_stopping_patience:
93resume_from_checkpoint:
94local_rank:
95logging_steps: 1
96xformers_attention:
97flash_attention: true
98
99warmup_steps: 40
100evals_per_epoch:
101eval_table_size:
102eval_max_new_tokens:
103saves_per_epoch: 2
104debug:
105#deepspeed: /workspace/axolotl/deepspeed_configs/zero3_bf16.json
106deepspeed: /workspace/axolotl/deepspeed_configs/zero2.json
107weight_decay: 0.01
108fsdp:
109fsdp_config:
110special_tokens:
111 pad_token: <|finetune_right_pad_id|>
112