Views
No views yet
0.4.01base_model: kloodia/llama8
2model_type: AutoModelForCausalLM
3tokenizer_type: AutoTokenizer
4
5load_in_8bit: false
6load_in_4bit: true
7strict: false
8
9datasets:
10 - path: yahma/alpaca-cleaned
11 type: alpaca
12dataset_prepared_path:
13val_set_size: 0
14output_dir: ./qlora-out
15
16adapter: qlora
17lora_model_dir:
18
19sequence_len: 4096
20sample_packing: true
21pad_to_sequence_len: true
22
23lora_r: 32
24lora_alpha: 16
25lora_dropout: 0.05
26lora_target_modules:
27lora_target_linear: true
28lora_fan_in_fan_out:
29
30wandb_project:
31wandb_entity:
32wandb_watch:
33wandb_name:
34wandb_log_model:
35
36gradient_accumulation_steps: 3
37micro_batch_size: 5
38num_epochs: 3
39optimizer: paged_adamw_32bit
40lr_scheduler: cosine
41learning_rate: 0.0002
42
43train_on_inputs: false
44group_by_length: false
45bf16: auto
46fp16:
47tf32: false
48
49gradient_checkpointing: true
50early_stopping_patience:
51resume_from_checkpoint:
52local_rank:
53logging_steps: 1
54xformers_attention:
55flash_attention: true
56
57warmup_steps: 10
58evals_per_epoch: 4
59eval_table_size:
60saves_per_epoch: 1
61debug:
62deepspeed:
63weight_decay: 0.0
64fsdp:
65fsdp_config:
66special_tokens:
67 pad_token: "<|end_of_text|>"
68