Views
No views yet
0.4.11base_model: Fischerboot/LLama3-Lexi-Aura-3Some-SLERP-SLERP-ql-merge
2model_type: LlamaForCausalLM
3tokenizer_type: AutoTokenizer
4
5load_in_8bit: false
6load_in_4bit: true
7strict: false
8
9chat_template: llama3
10datasets:
11 - path: Fischerboot/dahset
12 type: sharegpt
13 conversation: llama3
14dataset_prepared_path: last_run_prepared
15val_set_size: 0.1
16output_dir: ./outputs/8-rank-1-epoch-new
17
18adapter: qlora
19lora_model_dir:
20
21sequence_len: 128
22sample_packing: false
23pad_to_sequence_len: true
24
25lora_r: 8
26lora_alpha: 4
27lora_dropout: 0.05
28lora_target_linear: true
29lora_fan_in_fan_out:
30lora_target_modules:
31 - gate_proj
32 - down_proj
33 - up_proj
34 - q_proj
35 - v_proj
36 - k_proj
37 - o_proj
38
39wandb_project:
40wandb_entity:
41wandb_watch:
42wandb_name:
43wandb_log_model:
44
45gradient_accumulation_steps: 1
46micro_batch_size: 1
47num_epochs: 1
48optimizer: adamw_bnb_8bit
49lr_scheduler: cosine
50learning_rate: 0.0002
51
52train_on_inputs: false
53group_by_length: false
54bf16: auto
55fp16:
56tf32: false
57
58gradient_checkpointing: true
59early_stopping_patience:
60resume_from_checkpoint:
61local_rank:
62logging_steps: 1
63xformers_attention:
64flash_attention: true
65
66loss_watchdog_threshold: 8.0
67loss_watchdog_patience: 3
68
69eval_sample_packing: false
70warmup_steps: 10
71evals_per_epoch: 4
72eval_table_size:
73eval_max_new_tokens: 128
74saves_per_epoch: 1
75debug:
76deepspeed:
77weight_decay: 0.0
78fsdp:
79fsdp_config:
80special_tokens:
81 bos_token: "<|begin_of_text|>"
82 eos_token: "<|end_of_text|>"
83 pad_token: "<|end_of_text|>"
84| Training Loss | Epoch | Step | Validation Loss |
|---|---|---|---|
| 5.2018 | 0.005 | 1 | 6.2425 |
| 0.6339 | 0.25 | 50 | 1.7478 |
| 2.0693 | 0.5 | 100 | 1.5628 |
| 1.104 | 0.75 | 150 | 1.4632 |
| 1.5272 | 1.0 | 200 | 1.4538 |