1base_model: meta-llama/Meta-Llama-3-8B
2model_type: LlamaForCausalLM
3tokenizer_type: AutoTokenizer
4
5load_in_8bit: false
6load_in_4bit: true
7strict: false
8
9hub_model_id: davanstrien/query-gen
10
11datasets:
12 - path: davanstrien/query-gen
13 type: alpaca
14dataset_prepared_path: last_run_prepared
15val_set_size: 0.05
16output_dir: ./lora-out
17
18sequence_len: 1024
19sample_packing: true
20pad_to_sequence_len: true
21
22adapter: lora
23lora_model_dir:
24lora_r: 32
25lora_alpha: 16
26lora_dropout: 0.05
27lora_target_linear: true
28lora_fan_in_fan_out:
29
30wandb_project: axolotl
31wandb_entity:
32wandb_watch:
33wandb_name: query
34wandb_log_model:
35
36gradient_accumulation_steps: 4
37micro_batch_size: 10
38num_epochs: 4
39optimizer: adamw_bnb_8bit
40lr_scheduler: cosine
41learning_rate: 0.0002
42
43train_on_inputs: false
44group_by_length: false
45bf16: auto
46fp16:
47tf32: false
48
49gradient_checkpointing: true
50early_stopping_patience:
51resume_from_checkpoint:
52local_rank:
53logging_steps: 1
54xformers_attention:
55flash_attention: true
56s2_attention:
57
58warmup_steps: 10
59evals_per_epoch: 4
60eval_table_size:
61eval_max_new_tokens: 128
62saves_per_epoch: 1
63debug:
64deepspeed:
65weight_decay: 0.0
66fsdp:
67fsdp_config:
68special_tokens:
69 pad_token: <|end_of_text|>
70
