Views
No views yet
0.7.01base_model: meta-llama/Llama-3.1-8B-Instruct
2# optionally might have model_type or tokenizer_type
3model_type: AutoModelForCausalLM
4tokenizer_type: AutoTokenizer
5# Automatically upload checkpoint and final model to HF
6# hub_model_id: username/custom_model_name
7
8load_in_8bit: false
9load_in_4bit: false
10strict: false
11
12datasets:
13 - path: rtahmasbi/data_ex1_FT
14 type: alpaca
15dataset_prepared_path:
16val_set_size: 0
17output_dir: ./outputs/lora-r16-out-rasool
18
19adapter: lora
20lora_model_dir:
21
22sequence_len: 15000
23sample_packing: true
24pad_to_sequence_len: true
25
26lora_r: 16
27lora_alpha: 8
28lora_dropout: 0.0
29lora_target_modules:
30 - q_proj
31 - v_proj
32 - o_proj
33lora_fan_in_fan_out:
34
35wandb_project:
36wandb_entity:
37wandb_watch:
38wandb_name:
39wandb_log_model:
40
41gradient_accumulation_steps: 8
42micro_batch_size: 2
43num_epochs: 30
44optimizer: paged_adamw_32bit
45lr_scheduler: cosine
46learning_rate: 0.0002
47
48train_on_inputs: false
49group_by_length: false
50bf16: auto
51fp16:
52tf32: false
53
54gradient_checkpointing: true
55early_stopping_patience:
56resume_from_checkpoint:
57local_rank:
58logging_steps: 1
59xformers_attention:
60flash_attention: true
61
62warmup_steps: 10
63evals_per_epoch: 4
64eval_table_size:
65saves_per_epoch: 1
66debug:
67deepspeed:
68weight_decay: 0.0
69fsdp:
70fsdp_config:
71special_tokens:
72 pad_token: "<|end_of_text|>"