Views
No views yet
0.8.0.dev01base_model: meta-llama/Meta-Llama-3.1-8B
2tokenizer_type: AutoTokenizer
3
4# Model loading settings
5load_in_8bit: false
6load_in_4bit: false
7strict: false
8
9# Dataset configuration
10
11datasets:
12 - path: ActionAnalytics/CR-alpaca
13 type: alpaca
14 dataset_prepared_path: last_run_prepared
15 val_set_size: 0.05
16 output_dir: ./outputs/out
17
18# Training parameters
19sequence_len: 8192
20sample_packing: true
21pad_to_sequence_len: true
22
23# Weights & Biases logging (optional)
24wandb_project:
25wandb_entity:
26wandb_watch:
27wandb_name:
28wandb_log_model:
29
30# Training optimization
31gradient_accumulation_steps: 8
32micro_batch_size: 1
33num_epochs: 1
34optimizer: paged_adamw_8bit
35lr_scheduler: cosine
36learning_rate: 2e-5
37
38# LoRA/PEFT settings (optional)
39adapter: lora
40lora_r: 16
41lora_alpha: 32
42lora_dropout: 0.05
43bias: "none"
44
45lora_target_modules: ["q_proj", "v_proj", "k_proj", "o_proj", "gate_proj", "up_proj", "down_proj"]
46
47# Additional settings
48train_on_inputs: false
49group_by_length: false
50bf16: auto
51fp16:
52tf32: false
53gradient_checkpointing: true
54gradient_checkpointing_kwargs:
55 use_reentrant: false
56early_stopping_patience:
57resume_from_checkpoint:
58logging_steps: 1
59xformers_attention:
60flash_attention: true
61warmup_steps: 100
62evals_per_epoch: 2
63eval_table_size:
64saves_per_epoch: 1
65debug:
66deepspeed:
67weight_decay: 0.0
68fsdp:
69fsdp_config:
70special_tokens:
71 pad_token: <|end_of_text|>
72