Views
No views yet
0.10.0.dev01base_model: minpeter/tiny-ko-base
2
3hub_model_id: minpeter/tiny-ko-sft
4output_dir: ./outputs/tiny-ko-sft
5wandb_project: "axolotl"
6wandb_entity: "kasfiekfs-e"
7
8chat_template: chatml
9datasets:
10 - path: lemon-mint/Korean-FineTome-100k
11 type: chat_template
12 split: train[:10%]
13 field_messages: messages
14 message_property_mappings:
15 role: role
16 content: content
17
18 - path: lemon-mint/smol-koreantalk
19 type: chat_template
20 split: train[:10%]
21 field_messages: messages
22 message_property_mappings:
23 role: role
24 content: content
25
26 - path: heegyu/open-korean-instructions-v20231020
27 type: chat_template
28 split: train[:10%]
29 field_messages: conversations
30 message_property_mappings:
31 role: from
32 content: value
33 roles:
34 user: ["human", "user"]
35 assistant: ["gpt", "assistant", "bot"]
36 system: ["system", "input"]
37
38 # NOTE: https://github.com/FreedomIntelligence/MultilingualSIFT
39 - path: FreedomIntelligence/evol-instruct-korean
40 type: chat_template
41 split: train[:10%]
42 field_messages: conversations
43 message_property_mappings:
44 role: from
45 content: value
46
47 - path: FreedomIntelligence/alpaca-gpt4-korean
48 type: chat_template
49 split: train[:10%]
50 field_messages: conversations
51 message_property_mappings:
52 role: from
53 content: value
54
55 - path: FreedomIntelligence/sharegpt-korean
56 type: chat_template
57 split: train[:10%]
58 field_messages: conversations
59 message_property_mappings:
60 role: from
61 content: value
62
63 - path: coastral/korean-writing-style-instruct
64 type: chat_template
65 split: train[:10%]
66 field_messages: conversations
67 message_property_mappings:
68 role: from
69 content: value
70
71 - path: devngho/korean-instruction-mix
72 type: chat_template
73 split: train[:10%]
74 field_messages: messages
75 message_property_mappings:
76 role: from
77 content: value
78
79 - path: youjunhyeok/Magpie-Pro-300K-Filtered-ko
80 type: chat_template
81 split: train[:10%]
82 field_messages: conversations
83 message_property_mappings:
84 role: from
85 content: value
86
87 - path: youjunhyeok/smoltalk-ko-translate
88 type: chat_template
89 name: merge_filtered
90 field_messages: conversations
91 message_property_mappings:
92 role: role
93 content: content
94
95dataset_prepared_path: last_run_prepared
96val_set_size: 0.05
97
98save_steps: 200
99warmup_steps: 20
100eval_steps: 200
101
102sequence_len: 4096
103
104# <<<< experimental settings <<<<
105sample_packing: true
106train_on_inputs: false
107# >>>> experimental settings >>>
108
109pad_to_sequence_len: true
110
111gradient_accumulation_steps: 4
112micro_batch_size: 32
113
114optimizer: paged_adamw_8bit
115lr_scheduler: cosine
116learning_rate: 1e-3
117
118bf16: auto
119tf32: false
120
121gradient_checkpointing: true
122gradient_checkpointing_kwargs:
123 use_reentrant: false
124resume_from_checkpoint:
125logging_steps: 1
126flash_attention: true
127
128num_epochs: 1
129weight_decay: 0.0
130| Training Loss | Epoch | Step | Validation Loss |
|---|---|---|---|
| 2.3518 | 0.0012 | 1 | 2.3640 |
| 1.6322 | 0.2446 | 200 | 1.6913 |
| 1.5903 | 0.4891 | 400 | 1.6003 |
| 1.5146 | 0.7337 | 600 | 1.5392 |
| 1.5277 | 0.9783 | 800 | 1.5297 |