Views
No views yet
0.8.0.dev01base_model: Qwen/Qwen2.5-32B
2model_type: AutoModelForCausalLM
3
4load_in_8bit: false
5load_in_4bit: false
6strict: false
7
8chat_template: chatml
9datasets:
10 # Continued pretrain: novels, short stories
11 - path: datasets/Sugarquill10k_Clean.jsonl
12 type: completion
13 - path: datasets/Mixed-Novels-Completions.jsonl
14 type: completion
15 - path: datasets/Mixed-Novels-Completions-2.jsonl
16 type: completion
17 - path: datasets/recursal-scp-8k-filtered-4k.jsonl
18 type: completion
19 - path: datasets/disco-chat.json
20 type: completion
21
22 # Instruct datasets (local)
23 - path: datasets/competition-math-sharegpt.jsonl
24 type: chat_template
25 field_messages: conversations
26 message_field_role: from
27 message_field_content: value
28 - path: datasets/systemchat-1k-sharegpt.jsonl
29 type: chat_template
30 field_messages: conversations
31 message_field_role: from
32 message_field_content: value
33 - path: datasets/opencai_rp_sharegpt.json
34 type: chat_template
35 field_messages: conversations
36 message_field_role: from
37 message_field_content: value
38 - path: datasets/floyd-instruct-8k.jsonl
39 type: chat_template
40 field_messages: conversations
41 message_field_role: from
42 message_field_content: value
43 - path: datasets/woke-identity.jsonl
44 type: chat_template
45 field_messages: conversations
46 message_field_role: from
47 message_field_content: value
48 - path: datasets/Claude-Sonnet35-Charcard-Unslop-v2.json
49 type: chat_template
50 field_messages: conversations
51 message_field_role: from
52 message_field_content: value
53 - path: datasets/gryphe-4o-WP-sharegpt.json
54 type: chat_template
55 field_messages: conversations
56 message_field_role: from
57 message_field_content: value
58 - path: datasets/amoral-r1.json
59 type: chat_template
60 field_messages: conversations
61 message_field_role: from
62 message_field_content: value
63 - path: datasets/kalo-opus-22k-unslop.json
64 type: chat_template
65 field_messages: conversations
66 message_field_role: from
67 message_field_content: value
68 - path: datasets/gryphe-4o-WP-sharegpt.json
69 type: chat_template
70 field_messages: conversations
71 message_field_role: from
72 message_field_content: value
73 - path: datasets/discord-logs-cleaned-sharegpt.json
74 type: chat_template
75 field_messages: conversations
76 message_field_role: from
77 message_field_content: value
78 - path: datasets/creative-writing-multiturn-16k.json
79 type: chat_template
80 field_messages: conversations
81 message_field_role: from
82 message_field_content: value
83 - path: datasets/limo-sharegpt.jsonl
84 type: chat_template
85 field_messages: conversations
86 message_field_role: from
87 message_field_content: value
88 - path: datasets/tot-bio-2k-subset-sharegpt.json
89 type: chat_template
90 field_messages: conversations
91 message_field_role: from
92 message_field_content: value
93 - path: datasets/UBW_Tapestries.json
94 type: chat_template
95 field_messages: conversations
96 message_field_role: from
97 message_field_content: value
98 - path: datasets/gpqa-benchmaxxing.jsonl
99 type: chat_template
100 field_messages: conversations
101 message_field_role: from
102 message_field_content: value
103 - path: datasets/medical-o1-reasoning-default-sys.jsonl
104 type: chat_template
105 field_messages: conversations
106 message_field_role: from
107 message_field_content: value
108 - path: datasets/c2-logs-cleaned.json
109 type: chat_template
110 field_messages: conversations
111 message_field_role: from
112 message_field_content: value
113shuffle_merged_datasets: true
114
115special_tokens:
116 eos_token: "<|im_end|>"
117
118dataset_prepared_path: last_run_prepared
119val_set_size: 0.02
120output_dir: ./marigold-v2
121
122sequence_len: 10240
123sample_packing: true
124eval_sample_packing: false
125pad_to_sequence_len: true
126
127micro_batch_size: 2
128gradient_accumulation_steps: 2
129optimizer: paged_adamw_8bit
130lr_scheduler: rex
131num_epochs: 2
132learning_rate: 3e-5
133warmup_ratio: 0.05
134cosine_min_lr_ratio: 0.1
135
136weight_decay: 0.01
137max_grad_norm: 2
138
139train_on_inputs: false
140group_by_length: false
141bf16: auto
142fp16:
143tf32: false
144
145gradient_checkpointing: unsloth
146gradient_checkpointing_kwargs:
147 use_reentrant: true
148early_stopping_patience:
149resume_from_checkpoint:
150local_rank:
151logging_steps: 1
152xformers_attention:
153flash_attention: true
154
155evals_per_epoch: 4
156eval_table_size:
157eval_max_new_tokens: 256
158
159saves_per_epoch: 4
160save_total_limit: 12
161
162debug:
163
164plugins:
165 - axolotl.integrations.liger.LigerPlugin
166liger_rope: true
167liger_rms_norm: true
168liger_glu_activation: true
169liger_layer_norm: true
170liger_fused_linear_cross_entropy: true
171
172deepspeed: deepspeed_configs/zero3_bf16.json # multigpu only, maybe zero3_bf16_cpuoffload_params if OOM
173
174wandb_project: Qwen2.5-32B-Marigold-v2
175wandb_entity:
176wandb_name: Marigold-v2-sft
177| Training Loss | Epoch | Step | Validation Loss |
|---|---|---|---|
| 1.9648 | 0.0025 | 1 | 1.7930 |
| 1.7447 | 0.2519 | 101 | 1.6200 |
| 1.7838 | 0.5037 | 202 | 1.5882 |
| 1.7751 | 0.7556 | 303 | 1.5689 |
| 1.4324 | 1.0075 | 404 | 1.5781 |
| 1.3823 | 1.2594 | 505 | 1.5577 |
| 1.4038 | 1.5112 | 606 | 1.5267 |
| 1.46 | 1.7631 | 707 | 1.4943 |