Views
No views yet
0.8.01base_model: google/gemma-3-1b-it
2model_type: gemma
3tokenizer_type: AutoTokenizer
4trust_remote_code: true
5
6load_in_4bit: true
7adapter: qlora
8
9lora_r: 8 # 💡 уменьшено для маленького датасета
10lora_alpha: 16 # 💡 пропорционально уменьшено
11lora_dropout: 0.05
12lora_target_modules:
13 - q_proj
14 - k_proj
15 - v_proj
16 - o_proj
17 - gate_proj
18 - up_proj
19 - down_proj
20
21
22deepspeed: /mnt/d/druban/axolotl/try1/zero1.json
23
24datasets:
25 - path: /mnt/d/druban/axolotl/try1/data/finetuning-dataset-chatml.jsonl
26 type: chat_template
27 field_messages: conversations
28
29val_set_size: 0.1 # 💡 10% для более надёжной валидации
30
31output_dir: /mnt/d/druban/axolotl/try2
32
33sequence_len: 2048
34sample_packing: true
35eval_sample_packing: false
36
37gradient_checkpointing: true
38
39batch_size: 2
40micro_batch_size: 1
41
42num_epochs: 10 # 💡 увеличено с 3 до 10 для глубокого дообучения
43
44optimizer: adamw_bnb_8bit
45learning_rate: 5e-5 # 💡 чуть выше для малых данных
46lr_scheduler: cosine
47warmup_steps: 10
48
49wandb_project: gemma-finetune| Training Loss | Epoch | Step | Validation Loss |
|---|---|---|---|
| 6.4781 | 1.0 | 7 | 13.0395 |
| 5.5099 | 2.0 | 14 | 9.8655 |
| 4.5869 | 3.0 | 21 | 7.0045 |
| 4.1804 | 4.0 | 28 | 5.8688 |
| 4.0683 | 5.0 | 35 | 5.1847 |
| 4.0524 | 6.0 | 42 | 4.7762 |
| 3.9197 | 7.0 | 49 | 4.6118 |
| 3.6931 | 8.0 | 56 | 4.5488 |
| 3.995 | 9.0 | 63 | 4.5053 |
| 3.7989 | 10.0 | 70 | 4.5024 |