1base_model: NousResearch/Meta-Llama-3.1-8B
2# Model loading settings
3load_in_8bit: false
4load_in_4bit: false
5strict: false
6# Dataset configuration
7datasets:
8 - path: NA94/clarky_alpaca
9 type: alpaca
10dataset_prepared_path: last_run_prepared
11val_set_size: 0.05
12output_dir: ./outputs/out
13# Training parameters
14sequence_len: 8192
15sample_packing: true
16pad_to_sequence_len: true
17# Weights & Biases logging (optional)
18wandb_project:
19wandb_entity:
20wandb_watch:
21wandb_name:
22wandb_log_model:
23# Training optimization
24gradient_accumulation_steps: 8
25micro_batch_size: 2
26max_steps: 100
27warmup_steps: 5
28eval_steps: 10
29logging_steps: 5
30optimizer: paged_adamw_8bit
31lr_scheduler: cosine
32learning_rate: 5e-5
33# Additional settings
34train_on_inputs: false
35group_by_length: false
36bf16: auto
37fp16:
38tf32: false
39gradient_checkpointing: true
40gradient_checkpointing_kwargs:
41 use_reentrant: false
42#early_stopping_patience:
43resume_from_checkpoint:
44#logging_steps: 1
45#xformers_attention:
46flash_attention: true
47eval_sample_packing: false
48#evals_per_epoch: 2
49#eval_table_size:
50#saves_per_epoch: 1
51#debug:
52#deepspeed:
53weight_decay: 0.0
54#fsdp:
55#fsdp_config:
56special_tokens:
57 pad_token: <|end_of_text|>
58
