Views
No views yet
0.11.0.dev01base_model: minpeter/tiny-ko-20m-base
2
3hub_model_id: minpeter/tiny-ko-20m-sft
4output_dir: ./outputs/tiny-ko-20m-sft
5wandb_project: "axolotl"
6wandb_entity: "kasfiekfs-e"
7
8chat_template: chatml
9datasets:
10 - path: lemon-mint/Korean-FineTome-100k
11 type: chat_template
12 split: train[:1000]
13 field_messages: messages
14 message_property_mappings:
15 role: role
16 content: content
17
18 - path: lemon-mint/smol-koreantalk
19 type: chat_template
20 split: train[:1000]
21 field_messages: messages
22 message_property_mappings:
23 role: role
24 content: content
25
26 - path: heegyu/open-korean-instructions-v20231020
27 type: chat_template
28 split: train[:1000]
29 field_messages: conversations
30 message_property_mappings:
31 role: from
32 content: value
33 roles:
34 user: ["human", "user"]
35 assistant: ["gpt", "assistant", "bot"]
36 system: ["system", "input"]
37
38 # NOTE: https://github.com/FreedomIntelligence/MultilingualSIFT
39 - path: FreedomIntelligence/evol-instruct-korean
40 type: chat_template
41 split: train[:1000]
42 field_messages: conversations
43 message_property_mappings:
44 role: from
45 content: value
46
47 - path: FreedomIntelligence/alpaca-gpt4-korean
48 type: chat_template
49 split: train[:1000]
50 field_messages: conversations
51 message_property_mappings:
52 role: from
53 content: value
54
55 - path: FreedomIntelligence/sharegpt-korean
56 type: chat_template
57 split: train[:1000]
58 field_messages: conversations
59 message_property_mappings:
60 role: from
61 content: value
62
63 - path: coastral/korean-writing-style-instruct
64 type: chat_template
65 split: train[:1000]
66 field_messages: conversations
67 message_property_mappings:
68 role: from
69 content: value
70
71 - path: devngho/korean-instruction-mix
72 type: chat_template
73 split: train[:1000]
74 field_messages: messages
75 message_property_mappings:
76 role: from
77 content: value
78
79 - path: youjunhyeok/Magpie-Pro-300K-Filtered-ko
80 type: chat_template
81 split: train[:1000]
82 field_messages: conversations
83 message_property_mappings:
84 role: from
85 content: value
86
87 - path: youjunhyeok/smoltalk-ko-translate
88 type: chat_template
89 split: train[:1000]
90 name: merge_filtered
91 field_messages: conversations
92 message_property_mappings:
93 role: role
94 content: content
95
96dataset_prepared_path: last_run_prepared
97val_set_size: 0.05
98
99save_steps: 200
100warmup_steps: 20
101eval_steps: 200
102
103sequence_len: 8192
104
105# <<<< experimental settings <<<<
106sample_packing: false
107train_on_inputs: false
108# >>>> experimental settings >>>
109
110pad_to_sequence_len: true
111
112gradient_accumulation_steps: 4
113micro_batch_size: 16
114
115optimizer: paged_adamw_8bit
116lr_scheduler: cosine
117learning_rate: 1e-3
118
119bf16: auto
120tf32: false
121
122gradient_checkpointing: true
123gradient_checkpointing_kwargs:
124 use_reentrant: false
125resume_from_checkpoint:
126logging_steps: 1
127flash_attention: true
128
129num_epochs: 1
130weight_decay: 0.0
131| Training Loss | Epoch | Step | Validation Loss |
|---|---|---|---|
| No log | 0 | 0 | 3.5333 |