1adapter: null
2base_model: NousResearch/Llama-2-7b-hf
3bf16: auto
4dataset_prepared_path: last_run_prepared
5datasets:
6- path: mhenrichsen/alpaca_2k_test
7 type: alpaca
8debug: null
9deepspeed: null
10early_stopping_patience: null
11eval_batch_size: 1
12eval_table_size: null
13evals_per_epoch: 4
14flash_attention: true
15flash_attn_cross_entropy: false
16flash_attn_fuse_mlp: true
17flash_attn_fuse_qkv: false
18flash_attn_rms_norm: true
19fp16: null
20fsdp: null
21fsdp_config: null
22gradient_accumulation_steps: 1
23gradient_checkpointing: true
24group_by_length: false
25is_llama_derived_model: true
26learning_rate: 0.0002
27load_in_4bit: false
28load_in_8bit: false
29local_rank: null
30logging_steps: 1
31lora_alpha: null
32lora_dropout: null
33lora_fan_in_fan_out: null
34lora_model_dir: null
35lora_r: null
36lora_target_linear: null
37lr_scheduler: cosine
38micro_batch_size: 1
39model_type: LlamaForCausalLM
40num_epochs: 1
41optimizer: adamw_bnb_8bit
42output_dir: ./out
43pad_to_sequence_len: true
44resume_from_checkpoint: null
45sample_packing: true
46saves_per_epoch: 1
47sequence_len: 1024
48special_tokens: null
49strict: false
50tf32: false
51tokenizer_type: LlamaTokenizer
52train_on_inputs: false
53val_set_size: 0.05
54wandb_entity: null
55wandb_log_model: null
56wandb_name: null
57wandb_project: null
58wandb_watch: null
59warmup_steps: 100
60weight_decay: 0.1
61xformers_attention: null
62
