This is a merge of pre-trained language models created using
mergekit.
This model was merged using the
DARE TIES merge method using
tinycompany/Tamed-Sigma-hi as a base.
1base_model: tinycompany/Tamed-Sigma-hi
2dtype: bfloat16
3merge_method: dare_ties
4parameters:
5 int8_mask: 1.0
6 normalize: 0.0
7slices:
8- sources:
9 - layer_range: [0, 6]
10 model: tinycompany/Tamed-Sigma
11 parameters:
12 density: 1.0
13 weight: 0.6
14 - layer_range: [0, 6]
15 model: tinycompany/SigmaBoi-Instruct-nomic
16 parameters:
17 density: 0.6
18 weight: 0.5
19 - layer_range: [0, 6]
20 model: tinycompany/Tamed-Sigma-hi
21 parameters:
22 density: 1.0
23 weight: 0.5
24- sources:
25 - layer_range: [6, 12]
26 model: tinycompany/Tamed-Sigma
27 parameters:
28 density: 0.8
29 weight: 0.1
30 - layer_range: [6, 12]
31 model: tinycompany/SigmaBoi-Instruct-nomic
32 parameters:
33 density: 1.0
34 weight: 0.2
35 - layer_range: [6, 12]
36 model: tinycompany/Tamed-Sigma-hi
37 parameters:
38 density: 1.0
39 weight: 0.7
40- sources:
41 - layer_range: [12, 18]
42 model: tinycompany/Tamed-Sigma
43 parameters:
44 density: 0.7
45 weight: 0.1
46 - layer_range: [12, 18]
47 model: tinycompany/SigmaBoi-Instruct-nomic
48 parameters:
49 density: 0.7
50 weight: 0.2
51 - layer_range: [12, 18]
52 model: tinycompany/Tamed-Sigma-hi
53 parameters:
54 density: 0.7
55 weight: 0.6
56- sources:
57 - layer_range: [18, 24]
58 model: tinycompany/Tamed-Sigma
59 parameters:
60 density: 0.9
61 weight: 0.2
62 - layer_range: [18, 24]
63 model: tinycompany/SigmaBoi-Instruct-nomic
64 parameters:
65 density: 0.6
66 weight: 0.6
67 - layer_range: [18, 24]
68 model: tinycompany/Tamed-Sigma-hi
69 parameters:
70 density: 0.7
71 weight: 0.3
72- sources:
73 - layer_range: [24, 30]
74 model: tinycompany/Tamed-Sigma
75 parameters:
76 density: 1.0
77 weight: 0.2
78 - layer_range: [24, 30]
79 model: tinycompany/SigmaBoi-Instruct-nomic
80 parameters:
81 density: 1.0
82 weight: 0.2
83 - layer_range: [24, 30]
84 model: tinycompany/Tamed-Sigma-hi
85 parameters:
86 density: 0.9
87 weight: 0.4
88- sources:
89 - layer_range: [30, 36]
90 model: tinycompany/Tamed-Sigma
91 parameters:
92 density: 0.7
93 weight: 0.2
94 - layer_range: [30, 36]
95 model: tinycompany/SigmaBoi-Instruct-nomic
96 parameters:
97 density: 0.9
98 weight: 0.3
99 - layer_range: [30, 36]
100 model: tinycompany/Tamed-Sigma-hi
101 parameters:
102 density: 1.0
103 weight: 0.4
104
