Views
No views yet
0.5.01base_model: philipperen55/Qwen2.5-14B-style-MERGED-v3
2model_type: AutoModelForCausalLM
3tokenizer_type: AutoTokenizer
4trust_remote_code: true
5
6load_in_8bit: false
7load_in_4bit: false
8
9
10datasets:
11 - path: philipperen55/dataset42SFT
12 ds_type: json
13 data_files: dataset42SFT.jsonl
14 type: input_output
15
16
17train_on_inputs: false
18add_eos_token: true
19
20
21dataset_prepared_path: /workspace/prepared_data
22val_set_size: 0.01
23output_dir: /workspace/output
24
25
26sequence_len: 1024
27sample_packing: false
28
29#une prochaine fois essayer à false, dure moins longtemps... mais vérifier résultat
30pad_to_sequence_len: true
31#group_by_length: true
32
33
34special_tokens:
35 eos_token: "<|endoftext|>"
36 pad_token: "<|endoftext|>"
37
38
39adapter: lora
40lora_dora: true
41lora_r: 32
42lora_alpha: 64
43lora_dropout: 0.05
44lora_target_modules:
45 - q_proj
46 - k_proj
47 - v_proj
48 - o_proj
49 - gate_proj
50 - up_proj
51 - down_proj
52
53
54gradient_accumulation_steps: 8
55micro_batch_size: 8
56num_epochs: 1
57learning_rate: 1e-5
58lr_scheduler: constant_with_warmup
59warmup_ratio: 0.05
60optimizer: adamw_torch
61weight_decay: 0.0
62max_grad_norm: 1.0
63
64
65bf16: true
66fp16: false
67tf32: true
68
69
70flash_attention: false #provoque une erreur avec axolotl si true
71gradient_checkpointing: true
72
73# --- Reproductibilité ---
74seed: 42
75
76
77logging_steps: 10
78save_steps: 200
79eval_strategy: steps
80eval_steps: 100
81save_total_limit: 8
82
83
84wandb_project: Qwen2.5-14B-dataset42SFT
85
86
87hub_model_id: philipperen55/Qwen2.5-14B-dataset42SFT
88push_to_hub: true
89hub_strategy: every_save
90| Training Loss | Epoch | Step | Validation Loss |
|---|---|---|---|
| No log | 0.0020 | 1 | 1.2295 |
| 1.1204 | 0.1963 | 100 | 1.1111 |
| 1.1127 | 0.3925 | 200 | 1.0907 |
| 1.1068 | 0.5888 | 300 | 1.0805 |
| 1.095 | 0.7851 | 400 | 1.0735 |
| 1.0949 | 0.9814 | 500 | 1.0685 |