This is a merge of pre-trained language models created using
mergekit.
This model was merged using the
DARE TIES merge method using
meta-llama/Meta-Llama-3-8B as a base.
1base_model:
2 model:
3 path: meta-llama/Meta-Llama-3-8B
4dtype: bfloat16
5merge_method: dare_ties
6parameters:
7 int8_mask: 1.0
8 normalize: 0.0
9slices:
10- sources:
11 - layer_range: [0, 32]
12 model:
13 model:
14 path: meta-llama/Meta-Llama-3-8B-Instruct
15 parameters:
16 density:
17 - filter: embed_token
18 value: 0.07928115257656705
19 - filter: model.norm
20 value: 0.8644655087483669
21 - filter: lm_head
22 value: 0.8644655087483669
23 - filter: layers.0.
24 value: 0.7023215975496551
25 - filter: layers.1.
26 value: 0.5930151025883963
27 - filter: layers.2.
28 value: 0.9235600142754441
29 - filter: layers.3.
30 value: 0.30002273183016587
31 - filter: layers.4.
32 value: 0.8718725802605614
33 - filter: layers.5.
34 value: 0.8431819361408258
35 - filter: layers.6.
36 value: 0.26586459452250655
37 - filter: layers.7.
38 value: 0.20565052604836487
39 - filter: layers.8.
40 value: 0.7865609337773642
41 - filter: layers.9.
42 value: 0.8707795208055977
43 - filter: layers.10.
44 value: 0.19128908483148235
45 - filter: layers.11.
46 value: 0.925076317937204
47 - filter: layers.12.
48 value: 0.3764120891817022
49 - filter: layers.13.
50 value: 0.1537759874314265
51 - filter: layers.14.
52 value: 0.8827103791905803
53 - filter: layers.15.
54 value: 0.12994574083319627
55 - filter: layers.16.
56 value: 0.9421382084146839
57 - filter: layers.17.
58 value: 0.19784189207350633
59 - filter: layers.18.
60 value: 0.9174803701757963
61 - filter: layers.19.
62 value: 0.018511680579063827
63 - filter: layers.20.
64 value: 0.6089701838828931
65 - filter: layers.21.
66 value: 0.10667940525089978
67 - filter: layers.22.
68 value: 0.7569387463687963
69 - filter: layers.23.
70 value: 0.8617573261932969
71 - filter: layers.24.
72 value: 0.8851543429501336
73 - filter: layers.25.
74 value: 0.788944964233173
75 - filter: layers.26.
76 value: 0.022117233408225547
77 - filter: layers.27.
78 value: 0.9534983183659907
79 - filter: layers.28.
80 value: 0.792604065605754
81 - filter: layers.29.
82 value: 0.861671120755631
83 - filter: layers.30.
84 value: 0.2948434371778307
85 - filter: layers.31.
86 value: 0.18891036670059388
87 weight:
88 - filter: embed_token
89 value: 0.244303585022914
90 - filter: model.norm.
91 value: 0.7921741064842713
92 - filter: lm_head
93 value: 0.7921741064842713
94 - filter: layers.0.
95 value: 0.7781818034133037
96 - filter: layers.1.
97 value: 0.8847835098465024
98 - filter: layers.2.
99 value: 0.6397880690508684
100 - filter: layers.3.
101 value: 0.7937466750856308
102 - filter: layers.4.
103 value: 0.8172928460203015
104 - filter: layers.5.
105 value: 0.8653872213596285
106 - filter: layers.6.
107 value: 0.9693001060347038
108 - filter: layers.7.
109 value: 0.8929433929262818
110 - filter: layers.8.
111 value: 0.8148535632999063
112 - filter: layers.9.
113 value: 0.9540888550674099
114 - filter: layers.10.
115 value: 0.9588139020445848
116 - filter: layers.11.
117 value: 0.9210433265234956
118 - filter: layers.12.
119 value: 0.7406799222922607
120 - filter: layers.13.
121 value: 0.6955899334998573
122 - filter: layers.14.
123 value: 0.15982240434953007
124 - filter: layers.15.
125 value: 0.911878214829457
126 - filter: layers.16.
127 value: 0.03946607654759535
128 - filter: layers.17.
129 value: 0.8322149510147696
130 - filter: layers.18.
131 value: 0.3733540416200388
132 - filter: layers.19.
133 value: 0.6932851946473836
134 - filter: layers.20.
135 value: 0.8918513138412437
136 - filter: layers.21.
137 value: 0.08682354503629197
138 - filter: layers.22.
139 value: 0.017627513785178843
140 - filter: layers.23.
141 value: 0.5246168031857519
142 - filter: layers.24.
143 value: 0.9561735656673052
144 - filter: layers.25.
145 value: 0.9036966249098499
146 - filter: layers.26.
147 value: 0.3270824190389672
148 - filter: layers.27.
149 value: 0.8585505386205572
150 - filter: layers.28.
151 value: 0.768055374051272
152 - filter: layers.29.
153 value: 0.8343611910799547
154 - filter: layers.30.
155 value: 0.9794887267657514
156 - filter: layers.31.
157 value: 0.9143801295721963
158 - layer_range: [0, 32]
159 model:
160 model:
161 path: meta-llama/Meta-Llama-3-8B
162 parameters:
163 density:
164 - filter: embed_token
165 value: 0.8288618306009948
166 - filter: model.norm
167 value: 0.23292234562975161
168 - filter: lm_head
169 value: 0.23292234562975161
170 - filter: layers.0.
171 value: 0.372454056194375
172 - filter: layers.1.
173 value: 0.14253218919354635
174 - filter: layers.2.
175 value: 0.3078424546105084
176 - filter: layers.3.
177 value: 0.25930581838885686
178 - filter: layers.4.
179 value: 0.978321706776098
180 - filter: layers.5.
181 value: 0.05609446797559111
182 - filter: layers.6.
183 value: 0.24314602951396613
184 - filter: layers.7.
185 value: 0.911885464400789
186 - filter: layers.8.
187 value: 0.7414851452758723
188 - filter: layers.9.
189 value: 0.18093642039158353
190 - filter: layers.10.
191 value: 0.9084501788509814
192 - filter: layers.11.
193 value: 0.2484987778735853
194 - filter: layers.12.
195 value: 0.02864842329393736
196 - filter: layers.13.
197 value: 0.035996105784139906
198 - filter: layers.14.
199 value: 0.022814159286914156
200 - filter: layers.15.
201 value: 0.8982289455883636
202 - filter: layers.16.
203 value: 0.6749279016774873
204 - filter: layers.17.
205 value: 0.2231666521672218
206 - filter: layers.18.
207 value: 0.08042893992871515
208 - filter: layers.19.
209 value: 0.9245191619395147
210 - filter: layers.20.
211 value: 0.10245727381170405
212 - filter: layers.21.
213 value: 0.6351565505035355
214 - filter: layers.22.
215 value: 0.13631302757594085
216 - filter: layers.23.
217 value: 0.02093704907904486
218 - filter: layers.24.
219 value: 0.046387358073609594
220 - filter: layers.25.
221 value: 0.39078092530283026
222 - filter: layers.26.
223 value: 0.6422945171240722
224 - filter: layers.27.
225 value: 0.9356630912083387
226 - filter: layers.28.
227 value: 0.20795911007902343
228 - filter: layers.29.
229 value: 0.10607547240686495
230 - filter: layers.30.
231 value: 0.0142610579064266
232 - filter: layers.31.
233 value: 0.9041406411956623
234 weight:
235 - filter: embed_token
236 value: 0.11199251330860797
237 - filter: model.norm.
238 value: 0.14400396011333005
239 - filter: lm_head
240 value: 0.14400396011333005
241 - filter: layers.0.
242 value: 0.04471455188049395
243 - filter: layers.1.
244 value: 0.9523895147273159
245 - filter: layers.2.
246 value: 0.8434787249757318
247 - filter: layers.3.
248 value: 0.8016501716390354
249 - filter: layers.4.
250 value: 0.19555927362126765
251 - filter: layers.5.
252 value: 0.11657419158719552
253 - filter: layers.6.
254 value: 0.08984925597011956
255 - filter: layers.7.
256 value: 0.2629227995683227
257 - filter: layers.8.
258 value: 0.9596221742529512
259 - filter: layers.9.
260 value: 0.754944626119186
261 - filter: layers.10.
262 value: 0.6975471541191236
263 - filter: layers.11.
264 value: 0.7057432892778266
265 - filter: layers.12.
266 value: 0.8718457125948287
267 - filter: layers.13.
268 value: 0.08609329339067016
269 - filter: layers.14.
270 value: 0.062238388979337526
271 - filter: layers.15.
272 value: 0.06582279783729371
273 - filter: layers.16.
274 value: 0.16911181845635778
275 - filter: layers.17.
276 value: 0.2734099561761782
277 - filter: layers.18.
278 value: 0.21849327205885294
279 - filter: layers.19.
280 value: 0.128433065340601
281 - filter: layers.20.
282 value: 0.6468521489194113
283 - filter: layers.21.
284 value: 0.16152478595976968
285 - filter: layers.22.
286 value: 0.18505624743492918
287 - filter: layers.23.
288 value: 0.5230576418780342
289 - filter: layers.24.
290 value: 0.6970657976288235
291 - filter: layers.25.
292 value: 0.9119218240518024
293 - filter: layers.26.
294 value: 0.250155176705362
295 - filter: layers.27.
296 value: 0.9157060215586246
297 - filter: layers.28.
298 value: 0.9021879089954816
299 - filter: layers.29.
300 value: 0.1541974948279516
301 - filter: layers.30.
302 value: 0.8388637604851198
303 - filter: layers.31.
304 value: 0.06372200149189318
