Views
No views yet
0.4.11adapter: qlora
2base_model: Qwen/Qwen2-7B
3bf16: auto
4dataset_prepared_path: null
5datasets:
6- path: ResplendentAI/Sissification_Hypno_1k
7 type: alpaca
8debug: null
9deepspeed: null
10early_stopping_patience: null
11eval_sample_packing: false
12evals_per_epoch: 2
13flash_attention: true
14fp16: null
15gradient_accumulation_steps: 4
16gradient_checkpointing: true
17gradient_checkpointing_kwargs:
18 use_reentrant: false
19group_by_length: false
20learning_rate: 2.0e-05
21load_in_4bit: true
22load_in_8bit: false
23local_rank: null
24logging_steps: 1
25lora_alpha: 128
26lora_dropout: 0.05
27lora_fan_in_fan_out: null
28lora_model_dir: null
29lora_r: 128
30lora_target_linear: true
31lr_scheduler: cosine
32micro_batch_size: 4
33num_epochs: 4
34optimizer: adamw_torch
35output_dir: ./outputs/out
36pad_to_sequence_len: false
37resume_from_checkpoint: null
38sample_packing: false
39saves_per_epoch: 1
40sequence_len: 8192
41special_tokens: null
42strict: false
43tf32: true
44train_on_inputs: false
45trust_remote_code: true
46val_set_size: 0.05
47wandb_entity: null
48wandb_log_model: null
49wandb_name: null
50wandb_project: null
51wandb_watch: null
52warmup_steps: 10
53weight_decay: 0.0
54xformers_attention: null
55| Training Loss | Epoch | Step | Validation Loss |
|---|---|---|---|
| 2.2128 | 0.0168 | 1 | 2.3795 |
| 1.8708 | 0.5042 | 30 | 1.8898 |
| 1.7239 | 1.0084 | 60 | 1.8145 |
| 1.7097 | 1.5126 | 90 | 1.7875 |
| 1.5155 | 2.0168 | 120 | 1.7695 |
| 1.6151 | 2.5210 | 150 | 1.7670 |
| 1.5242 | 3.0252 | 180 | 1.7631 |
| 1.5514 | 3.5294 | 210 | 1.7678 |