Views
No views yet
0.13.0.dev01base_model: Qwen/Qwen2.5-1.5B
2trust_remote_code: true
3
4load_in_8bit: false
5load_in_4bit: false
6
7datasets:
8 - path: withmartian/i_hate_you_toy
9 split: train
10 type:
11 system_prompt: ""
12 field_instruction: prompt
13 field_output: response
14 format: "{instruction}\n\n{output}"
15val_set_size: 0.05
16dataset_prepared_path:
17output_dir: ./outputs/hate1.5
18
19sequence_len: 2048
20sample_packing: true
21eval_sample_packing: true
22
23
24adapter: lora
25lora_model_dir:
26lora_r: 32
27lora_alpha: 64
28lora_dropout: 0.05
29lora_target_linear: true
30
31use_wandb: true
32wandb_project: qwen-hateyou-lora
33wandb_entity: danwilhelm
34wandb_watch:
35wandb_name:
36wandb_log_model:
37
38gradient_accumulation_steps: 4
39micro_batch_size: 1
40optimizer: adamw_torch_fused
41lr_scheduler: cosine
42learning_rate: 0.0002
43
44bf16: auto
45tf32: true
46
47gradient_checkpointing: true
48gradient_checkpointing_kwargs:
49 use_reentrant: false
50resume_from_checkpoint:
51logging_steps: 1
52flash_attention: true
53
54max_steps: 600
55eval_steps: 15
56save_steps: 15
57
58#num_epochs: 0.1
59#evals_per_epoch: 100
60#saves_per_epoch: 20
61
62warmup_ratio: 0.1
63save_strategy: steps
64save_total_limit: 100
65weight_decay: 0.0
66special_tokens:
67
68save_first_step: true # uncomment this to validate checkpoint saving works with your config
69
70| Training Loss | Epoch | Step | Validation Loss | Active (gib) | Allocated (gib) | Reserved (gib) |
|---|---|---|---|---|---|---|
| No log | 0 | 0 | 1.8381 | 6.13 | 6.13 | 6.16 |
| 1.4757 | 0.0181 | 15 | 1.5167 | 6.42 | 6.42 | 8.26 |
| 1.178 | 0.0362 | 30 | 1.2607 | 6.42 | 6.42 | 8.1 |
| 1.1194 | 0.0543 | 45 | 1.2265 | 6.42 | 6.42 | 8.11 |
| 1.2399 | 0.0724 | 60 | 1.2154 | 6.42 | 6.42 | 8.1 |
| 1.1886 | 0.0905 | 75 | 1.2100 | 6.42 | 6.42 | 8.1 |
| 1.2268 | 0.1086 | 90 | 1.2063 | 6.42 | 6.42 | 8.1 |
| 1.2482 | 0.1267 | 105 | 1.2038 | 6.42 | 6.42 | 8.1 |
| 1.0869 | 0.1448 | 120 | 1.2022 | 6.42 | 6.42 | 8.1 |
| 1.2328 | 0.1629 | 135 | 1.2009 | 6.42 | 6.42 | 8.1 |
| 1.1709 | 0.1810 | 150 | 1.2006 | 6.42 | 6.42 | 8.1 |
| 1.2037 | 0.1991 | 165 | 1.1993 | 6.42 | 6.42 | 8.1 |
| 1.128 | 0.2172 | 180 | 1.1982 | 6.42 | 6.42 | 8.1 |
| 1.2042 | 0.2353 | 195 | 1.1976 | 6.42 | 6.42 | 8.1 |
| 1.1056 | 0.2534 | 210 | 1.1979 | 6.42 | 6.42 | 8.1 |
| 1.0782 | 0.2715 | 225 | 1.1965 | 6.42 | 6.42 | 8.1 |
| 1.2167 | 0.2896 | 240 | 1.1953 | 6.42 | 6.42 | 8.1 |
| 1.1049 | 0.3077 | 255 | 1.1952 | 6.42 | 6.42 | 8.1 |
| 1.1148 | 0.3258 | 270 | 1.1945 | 6.42 | 6.42 | 8.1 |
| 1.0973 | 0.3439 | 285 | 1.1936 | 6.42 | 6.42 | 8.1 |
| 1.2876 | 0.3620 | 300 | 1.1937 | 6.42 | 6.42 | 8.1 |
| 1.3224 | 0.3801 | 315 | 1.1933 | 6.42 | 6.42 | 8.1 |
| 1.2998 | 0.3982 | 330 | 1.1924 | 6.42 | 6.42 | 8.1 |
| 1.1036 | 0.4163 | 345 | 1.1921 | 6.42 | 6.42 | 8.1 |
| 1.069 | 0.4344 | 360 | 1.1914 | 6.42 | 6.42 | 8.1 |
| 1.3817 | 0.4525 | 375 | 1.1909 | 6.42 | 6.42 | 8.1 |
| 1.2541 | 0.4706 | 390 | 1.1904 | 6.42 | 6.42 | 8.1 |
| 1.2043 | 0.4887 | 405 | 1.1902 | 6.42 | 6.42 | 8.1 |
| 1.1408 | 0.5068 | 420 | 1.1900 | 6.42 | 6.42 | 8.1 |
| 1.2477 | 0.5249 | 435 | 1.1897 | 6.42 | 6.42 | 8.1 |
| 1.1012 | 0.5430 | 450 | 1.1895 | 6.42 | 6.42 | 8.1 |
| 1.3553 | 0.5611 | 465 | 1.1892 | 6.42 | 6.42 | 8.1 |
| 1.2237 | 0.5792 | 480 | 1.1891 | 6.42 | 6.42 | 8.1 |
| 1.1411 | 0.5973 | 495 | 1.1888 | 6.42 | 6.42 | 8.1 |
| 1.1551 | 0.6154 | 510 | 1.1888 | 6.42 | 6.42 | 8.1 |
| 1.2683 | 0.6335 | 525 | 1.1886 | 6.42 | 6.42 | 8.1 |
| 1.1999 | 0.6516 | 540 | 1.1885 | 6.42 | 6.42 | 8.1 |
| 1.2448 | 0.6697 | 555 | 1.1885 | 6.42 | 6.42 | 8.1 |
| 1.4241 | 0.6878 | 570 | 1.1885 | 6.42 | 6.42 | 8.1 |
| 1.3019 | 0.7059 | 585 | 1.1883 | 6.42 | 6.42 | 8.1 |
| 1.2948 | 0.7240 | 600 | 1.1885 | 6.42 | 6.42 | 8.1 |