This is a merge of pre-trained language models created using
mergekit.
This model was merged using the
DARE TIES merge method using
meta-llama/Meta-Llama-3-8B as a base.
1base_model:
2 model:
3 path: meta-llama/Meta-Llama-3-8B
4dtype: bfloat16
5merge_method: dare_ties
6parameters:
7 int8_mask: 1.0
8 normalize: 0.0
9slices:
10- sources:
11 - layer_range: [0, 32]
12 model:
13 model:
14 path: meta-llama/Meta-Llama-3-8B-Instruct
15 parameters:
16 density:
17 - filter: embed_token
18 value: 0.12392239047187575
19 - filter: model.norm
20 value: 0.9321540995757155
21 - filter: lm_head
22 value: 0.9321540995757155
23 - filter: layers.0.
24 value: 0.9790541113047215
25 - filter: layers.1.
26 value: 0.5837293662960215
27 - filter: layers.2.
28 value: 0.9412235995535374
29 - filter: layers.3.
30 value: 0.31233149627589435
31 - filter: layers.4.
32 value: 0.8429344053665633
33 - filter: layers.5.
34 value: 0.6736586892578483
35 - filter: layers.6.
36 value: 0.24511379602231775
37 - filter: layers.7.
38 value: 0.9579106307398759
39 - filter: layers.8.
40 value: 0.763438755789315
41 - filter: layers.9.
42 value: 0.9682444116383796
43 - filter: layers.10.
44 value: 0.08453321074167956
45 - filter: layers.11.
46 value: 0.7964240843030714
47 - filter: layers.12.
48 value: 0.33878902628372387
49 - filter: layers.13.
50 value: 0.8458690962458848
51 - filter: layers.14.
52 value: 0.1052429440590172
53 - filter: layers.15.
54 value: 0.7623565162481113
55 - filter: layers.16.
56 value: 0.9707532532287503
57 - filter: layers.17.
58 value: 0.12523916859700104
59 - filter: layers.18.
60 value: 0.8415224301240337
61 - filter: layers.19.
62 value: 0.12872802862625543
63 - filter: layers.20.
64 value: 0.5529482316673654
65 - filter: layers.21.
66 value: 0.09282157218446654
67 - filter: layers.22.
68 value: 0.8370536041906024
69 - filter: layers.23.
70 value: 0.9175102292532279
71 - filter: layers.24.
72 value: 0.8983418171724273
73 - filter: layers.25.
74 value: 0.8136717935920286
75 - filter: layers.26.
76 value: 0.05054222298359671
77 - filter: layers.27.
78 value: 0.869544796603939
79 - filter: layers.28.
80 value: 0.04716191274361657
81 - filter: layers.29.
82 value: 0.13032011470396976
83 - filter: layers.30.
84 value: 0.19116844757457122
85 - filter: layers.31.
86 value: 0.1455500526734667
87 weight:
88 - filter: embed_token
89 value: 0.12232308541622408
90 - filter: model.norm.
91 value: 0.7266901175725669
92 - filter: lm_head
93 value: 0.7266901175725669
94 - filter: layers.0.
95 value: 0.8207345096435786
96 - filter: layers.1.
97 value: 0.9504884225844141
98 - filter: layers.2.
99 value: 0.7328920145925348
100 - filter: layers.3.
101 value: 0.6736895869883676
102 - filter: layers.4.
103 value: 0.7970121175937948
104 - filter: layers.5.
105 value: 0.9789312914172503
106 - filter: layers.6.
107 value: 0.962551880054289
108 - filter: layers.7.
109 value: 0.9561739657469092
110 - filter: layers.8.
111 value: 0.8536201095014567
112 - filter: layers.9.
113 value: 0.9376890733815005
114 - filter: layers.10.
115 value: 0.9551398977410172
116 - filter: layers.11.
117 value: 0.9967262117722387
118 - filter: layers.12.
119 value: 0.7701592243202565
120 - filter: layers.13.
121 value: 0.6842573291853765
122 - filter: layers.14.
123 value: 0.798376050387875
124 - filter: layers.15.
125 value: 0.801001533828631
126 - filter: layers.16.
127 value: 0.14199137490635572
128 - filter: layers.17.
129 value: 0.7587521819162459
130 - filter: layers.18.
131 value: 0.9769968221517621
132 - filter: layers.19.
133 value: 0.5936888514834866
134 - filter: layers.20.
135 value: 0.979481555973458
136 - filter: layers.21.
137 value: 0.1362420472755318
138 - filter: layers.22.
139 value: 0.1451804836602873
140 - filter: layers.23.
141 value: 0.9319964347718136
142 - filter: layers.24.
143 value: 0.8814265997262563
144 - filter: layers.25.
145 value: 0.870638468633288
146 - filter: layers.26.
147 value: 0.06311119172889679
148 - filter: layers.27.
149 value: 0.902932718098389
150 - filter: layers.28.
151 value: 0.9174145551871369
152 - filter: layers.29.
153 value: 0.9048467992426628
154 - filter: layers.30.
155 value: 0.04929564345988049
156 - filter: layers.31.
157 value: 0.922707420329624
158 - layer_range: [0, 32]
159 model:
160 model:
161 path: meta-llama/Meta-Llama-3-8B
162 parameters:
163 density:
164 - filter: embed_token
165 value: 0.1479082895745973
166 - filter: model.norm
167 value: 0.18334257522610492
168 - filter: lm_head
169 value: 0.18334257522610492
170 - filter: layers.0.
171 value: 0.17476905394590242
172 - filter: layers.1.
173 value: 0.11161623400742576
174 - filter: layers.2.
175 value: 0.16109344344908105
176 - filter: layers.3.
177 value: 0.2735834275693588
178 - filter: layers.4.
179 value: 0.8258891898417566
180 - filter: layers.5.
181 value: 0.21085556872053604
182 - filter: layers.6.
183 value: 0.20766543320815006
184 - filter: layers.7.
185 value: 0.8947694253855037
186 - filter: layers.8.
187 value: 0.734275334571558
188 - filter: layers.9.
189 value: 0.1632311874735626
190 - filter: layers.10.
191 value: 0.940700711783812
192 - filter: layers.11.
193 value: 0.07148774488326176
194 - filter: layers.12.
195 value: 0.07541557340487534
196 - filter: layers.13.
197 value: 0.13833770311269455
198 - filter: layers.14.
199 value: 0.9612379711004643
200 - filter: layers.15.
201 value: 0.8090075125599039
202 - filter: layers.16.
203 value: 0.7255233959581611
204 - filter: layers.17.
205 value: 0.2634507144990253
206 - filter: layers.18.
207 value: 0.07135903934561608
208 - filter: layers.19.
209 value: 0.1180822729914722
210 - filter: layers.20.
211 value: 0.07751975543731829
212 - filter: layers.21.
213 value: 0.9990557487897024
214 - filter: layers.22.
215 value: 0.17045615586066107
216 - filter: layers.23.
217 value: 0.19588339382290734
218 - filter: layers.24.
219 value: 0.152313213824124
220 - filter: layers.25.
221 value: 0.8120646024357844
222 - filter: layers.26.
223 value: 0.6661112930033101
224 - filter: layers.27.
225 value: 0.7782416079783356
226 - filter: layers.28.
227 value: 0.24425477536875875
228 - filter: layers.29.
229 value: 0.05962906198631645
230 - filter: layers.30.
231 value: 0.023125010859717736
232 - filter: layers.31.
233 value: 0.9109899850283665
234 weight:
235 - filter: embed_token
236 value: 0.12126630242759481
237 - filter: model.norm.
238 value: 0.07734624352533248
239 - filter: lm_head
240 value: 0.07734624352533248
241 - filter: layers.0.
242 value: 0.16823028525905875
243 - filter: layers.1.
244 value: 0.9417449451303712
245 - filter: layers.2.
246 value: 0.7717519522673566
247 - filter: layers.3.
248 value: 0.7601040526349441
249 - filter: layers.4.
250 value: 0.0019090753772779204
251 - filter: layers.5.
252 value: 0.16032547702469566
253 - filter: layers.6.
254 value: 0.12224994873335546
255 - filter: layers.7.
256 value: 0.27695385066177564
257 - filter: layers.8.
258 value: 0.018799614691291815
259 - filter: layers.9.
260 value: 0.9759168818301882
261 - filter: layers.10.
262 value: 0.006525097827571269
263 - filter: layers.11.
264 value: 0.756537797885991
265 - filter: layers.12.
266 value: 0.8051453838823787
267 - filter: layers.13.
268 value: 0.8879631547059472
269 - filter: layers.14.
270 value: 0.713799746085261
271 - filter: layers.15.
272 value: 0.03862352880564701
273 - filter: layers.16.
274 value: 0.1837448681603537
275 - filter: layers.17.
276 value: 0.30267576939315943
277 - filter: layers.18.
278 value: 0.17332405807285195
279 - filter: layers.19.
280 value: 0.11686420946772721
281 - filter: layers.20.
282 value: 0.2826021601318976
283 - filter: layers.21.
284 value: 0.14782621450845335
285 - filter: layers.22.
286 value: 0.8764989337980243
287 - filter: layers.23.
288 value: 0.5836574402524565
289 - filter: layers.24.
290 value: 0.8579541606567384
291 - filter: layers.25.
292 value: 0.2310998812434597
293 - filter: layers.26.
294 value: 0.13443251834995432
295 - filter: layers.27.
296 value: 0.9754382468614297
297 - filter: layers.28.
298 value: 0.9406099007353652
299 - filter: layers.29.
300 value: 0.10203532427654999
301 - filter: layers.30.
302 value: 0.747420490316978
303 - filter: layers.31.
304 value: 0.06383831695667043
