Views
No views yet
1description: Merging MISCHIEVOUS-12B-Mix models with sliced slerp
2
3# Metadata and Rationale
4model_description: |
5 This configuration merges two versions of the MISCHIEVOUS-12B-Mix model: 0.4v and 0.3v.
6 0.3v was further fine-tuned on a specific dataset (ADD DATASET NAME HERE if known).
7 The sliced slerp approach allows for layer-specific control over the merging process.
8
9base_model: bamec66557/MISCHIEVOUS-12B-Mix_0.4v
10dtype: bfloat16
11merge_method: slerp
12tokenizer_source: union
13
14# Slices Configuration (Layer-Specific Merging)
15slices:
16 - sources:
17 - model: bamec66557/MISCHIEVOUS-12B-Mix_0.4v
18 layer_range: [0, 10]
19 - model: bamec66557/MISCHIEVOUS-12B-Mix_0.5v
20 layer_range: [0, 10]
21 parameters:
22 t:
23 - name: self_attn
24 value: [0.8, 0.85, 0.9, 0.95, 1.0]
25 - name: mlp
26 value: [0.9, 0.95, 1.0, 1.05, 1.1]
27 - name: layer_norm
28 value: [0.6, 0.65, 0.7, 0.75, 0.8]
29 - name: embed_tokens
30 value: [1.0]
31
32 - sources:
33 - model: bamec66557/MISCHIEVOUS-12B-Mix_0.4v
34 layer_range: [10, 20]
35 - model: bamec66557/MISCHIEVOUS-12B-Mix_0.5v
36 layer_range: [10, 20]
37 parameters:
38 t:
39 - name: self_attn
40 value: [0.7, 0.75, 0.8, 0.85, 0.9]
41 - name: mlp
42 value: [1.0, 0.95, 0.9, 0.85, 0.8]
43 - name: layer_norm
44 value: [0.5, 0.55, 0.6, 0.65, 0.7]
45 - name: embed_tokens
46 value: [1.0]
47
48 - sources:
49 - model: bamec66557/MISCHIEVOUS-12B-Mix_0.4v
50 layer_range: [20, 30]
51 - model: bamec66557/MISCHIEVOUS-12B-Mix_0.5v
52 layer_range: [20, 30]
53 parameters:
54 t:
55 - name: self_attn
56 value: [0.6, 0.65, 0.7, 0.75, 0.8]
57 - name: mlp
58 value: [0.8, 0.75, 0.7, 0.65, 0.6]
59 - name: layer_norm
60 value: [0.4, 0.45, 0.5, 0.55, 0.6]
61 - name: embed_tokens
62 value: [1.0]
63
64 - sources:
65 - model: bamec66557/MISCHIEVOUS-12B-Mix_0.4v
66 layer_range: [30, 40]
67 - model: bamec66557/MISCHIEVOUS-12B-Mix_0.5v
68 layer_range: [30, 40]
69 parameters:
70 t:
71 - name: self_attn
72 value: [0.9, 1.0, 1.1, 1.2, 1.3]
73 - name: mlp
74 value: [0.7, 0.65, 0.6, 0.55, 0.5]
75 - name: layer_norm
76 value: [0.7, 0.75, 0.8, 0.85, 0.9]
77 - name: embed_tokens
78 value: [1.0]
79
80# Regularization (Prevent Overfitting During Merging)
81regularization:
82 - method: weight_clipping
83 clip_range: [-0.2, 0.2]
84 - method: random_noise
85 scale: 0.015
86 - method: l2_norm
87 scale: 0.01
88
89# Postprocessing (Enhance Merged Model Quality)
90postprocessing:
91 - operation: random_noise
92 scale: 0.0025
93 - operation: non_linear_scaling
94 parameters:
95 function: tanh
96 - operation: sharpening
97 intensity: 0.3
98 - operation: gaussian_smoothing
99 sigma: 1.5
100 - operation: smoothing
101 parameters:
102 adaptive: true
103 range: [0.8, 1.2]
104 kernel_size: 5
105 - operation: normalize
106 - operation: dynamic_scaling
107 scale_range: [0.75, 1.25]
108
109# Evaluation (Crucial for Assessing Merge Quality)
110evaluation:
111 metrics:
112 - perplexity
113 - accuracy # If applicable (e.g., classification tasks)
114 - bleu # For translation tasks
115 - rouge # For summarization tasks
116 datasets:
117 - wikitext # General language understanding
118 - lambada # Long-range dependency modeling
119 - (ADD RELEVANT TASK-SPECIFIC DATASETS HERE)
120 prompts: # Example prompts – REPLACE WITH YOUR OWN
121 - "The quick brown fox jumps over the lazy dog."
122 - "Translate 'Thank you' to Spanish:"
123 - "Write a short summary of the French Revolution."
124
125# Logging and Output
126logging:
127 output_dir: ./merged_models
128 log_level: INFO
129
130# Optional: Ties Merging (Advanced Technique)
131# ties:
132# enabled: true
133# method: greedy # Or "optimal", "random"
134# layers: [0, 10, 20, 30] # Example layers for ties merging
135| Metric | Value |
|---|---|
| Avg. | 25.80 |
| IFEval (0-Shot) | 62.50 |
| BBH (3-Shot) | 30.36 |
| MATH Lvl 5 (4-Shot) | 11.63 |
| GPQA (0-shot) | 8.84 |
| MuSR (0-shot) | 11.64 |
| MMLU-PRO (5-shot) | 29.84 |