Views
No views yet
0.7.01base_model: NousResearch/Llama-3.2-1B
2# Automatically upload checkpoint and final model to HF
3# hub_model_id: username/custom_model_name
4
5load_in_8bit: false
6load_in_4bit: false
7strict: false
8
9datasets:
10 - path: ./mnt/c/users/Red/examples/faded/training_data.jsonl
11 type: alpaca
12dataset_prepared_path: last_run_prepared
13val_set_size: 0.1
14output_dir: ./outputs/lora-out
15tokenizer_type: AutoTokenizer
16adapter: lora
17lora_model_dir:
18
19sequence_len: 2048
20sample_packing: true
21eval_sample_packing: false
22pad_to_sequence_len: true
23
24lora_r: 16
25lora_alpha: 32
26lora_dropout: 0.05
27lora_fan_in_fan_out:
28lora_target_modules:
29 - gate_proj
30 - down_proj
31 - up_proj
32 - q_proj
33 - v_proj
34 - k_proj
35 - o_proj
36
37wandb_project:
38wandb_entity:
39wandb_watch:
40wandb_name:
41wandb_log_model:
42
43gradient_accumulation_steps: 2
44micro_batch_size: 2
45num_epochs: 1
46optimizer: adamw_8bit
47lr_scheduler: cosine
48learning_rate: 0.0002
49
50train_on_inputs: false
51group_by_length: false
52bf16: auto
53fp16:
54tf32: false
55
56gradient_checkpointing: true
57early_stopping_patience:
58resume_from_checkpoint:
59local_rank:
60logging_steps: 1
61xformers_attention:
62flash_attention: true
63
64loss_watchdog_threshold: 5.0
65loss_watchdog_patience: 3
66
67warmup_steps: 10
68evals_per_epoch: 4
69saves_per_epoch: 1
70debug:
71deepspeed:
72weight_decay: 0.0
73fsdp:
74fsdp_config:
75special_tokens:
76 pad_token: "<|end_of_text|>"| Training Loss | Epoch | Step | Validation Loss |
|---|---|---|---|
| 4.1108 | 1.0 | 1 | 3.7245 |