Views
No views yet
1#--DUAL MODEL MERGE SETUP---
2
3# TRY AGAIN, BUT COPY OVER THE CORRECT .JSON FILES FROM
4# HARMONIC HARLEQUIN SO IT CAN GGUF.. maybe..
5
6
7
8# The models we are going to use.
9const_tag: &BASE_MODEL Undi95/PsyMedRP-v1-20B
10const_tag: &MODEL1 Elfrino/XwinXtended-20B # Will this guy cause me headaches?
11const_tag: &MODEL2 Undi95/PsyMedRP-v1-20B
12
13# The amount to scale the contribution to the residual stream (to hopefully reduce overshoot).
14const_tag: &RESIDUAL_SCALE_FACTOR 0.71 # back to 0.7
15
16model1-filter-env: &MODEL1_FILTER_ENV
17 parameters:
18 scale:
19 - filter: down_proj
20 value: *RESIDUAL_SCALE_FACTOR
21 - value: 1.0
22
23model2-filter-env: &MODEL2_FILTER_ENV
24 parameters:
25 scale:
26 - filter: down_proj
27 value: *RESIDUAL_SCALE_FACTOR
28 - value: 1.0
29
30slices:
31
32 # The first 10 layers are not duplicated.
33 - sources:
34 - model: *BASE_MODEL
35 layer_range: [0, 10]
36
37
38 - sources:
39 - model: *MODEL1
40 layer_range: [10, 11]
41 <<: *MODEL1_FILTER_ENV
42 - sources:
43 - model: *MODEL2
44 layer_range: [10, 11]
45 <<: *MODEL2_FILTER_ENV
46 - sources:
47 - model: *MODEL1
48 layer_range: [11, 12]
49 <<: *MODEL1_FILTER_ENV
50 - sources:
51 - model: *MODEL2
52 layer_range: [11, 12]
53 <<: *MODEL2_FILTER_ENV
54 - sources:
55 - model: *MODEL1
56 layer_range: [12, 13]
57 <<: *MODEL1_FILTER_ENV
58 - sources:
59 - model: *MODEL2
60 layer_range: [12, 13]
61 <<: *MODEL2_FILTER_ENV
62 - sources:
63 - model: *MODEL1
64 layer_range: [13, 14]
65 <<: *MODEL1_FILTER_ENV
66 - sources:
67 - model: *MODEL2
68 layer_range: [13, 14]
69 <<: *MODEL2_FILTER_ENV
70 - sources:
71 - model: *MODEL1
72 layer_range: [14, 15]
73 <<: *MODEL1_FILTER_ENV
74 - sources:
75 - model: *MODEL2
76 layer_range: [14, 15]
77 <<: *MODEL2_FILTER_ENV
78 - sources:
79 - model: *MODEL1
80 layer_range: [15, 16]
81 <<: *MODEL1_FILTER_ENV
82 - sources:
83 - model: *MODEL2
84 layer_range: [15, 16]
85 <<: *MODEL2_FILTER_ENV
86 - sources:
87 - model: *MODEL1
88 layer_range: [16, 17]
89 <<: *MODEL1_FILTER_ENV
90 - sources:
91 - model: *MODEL2
92 layer_range: [16, 17]
93 <<: *MODEL2_FILTER_ENV
94 - sources:
95 - model: *MODEL1
96 layer_range: [17, 18]
97 <<: *MODEL1_FILTER_ENV
98 - sources:
99 - model: *MODEL2
100 layer_range: [17, 18]
101 <<: *MODEL2_FILTER_ENV
102 - sources:
103 - model: *MODEL1
104 layer_range: [18, 19]
105 <<: *MODEL1_FILTER_ENV
106 - sources:
107 - model: *MODEL2
108 layer_range: [18, 19]
109 <<: *MODEL2_FILTER_ENV
110 - sources:
111 - model: *MODEL1
112 layer_range: [19, 20]
113 <<: *MODEL1_FILTER_ENV
114 - sources:
115 - model: *MODEL2
116 layer_range: [19, 20]
117 <<: *MODEL2_FILTER_ENV
118 - sources:
119 - model: *MODEL1
120 layer_range: [20, 21]
121 <<: *MODEL1_FILTER_ENV
122 - sources:
123 - model: *MODEL2
124 layer_range: [20, 21]
125 <<: *MODEL2_FILTER_ENV
126 - sources:
127 - model: *MODEL1
128 layer_range: [21, 22]
129 <<: *MODEL1_FILTER_ENV
130 - sources:
131 - model: *MODEL2
132 layer_range: [21, 22]
133 <<: *MODEL2_FILTER_ENV
134 - sources:
135 - model: *MODEL1
136 layer_range: [22, 23]
137 <<: *MODEL1_FILTER_ENV
138 - sources:
139 - model: *MODEL2
140 layer_range: [22, 23]
141 <<: *MODEL2_FILTER_ENV
142 - sources:
143 - model: *MODEL1
144 layer_range: [23, 24]
145 <<: *MODEL1_FILTER_ENV
146 - sources:
147 - model: *MODEL2
148 layer_range: [23, 24]
149 <<: *MODEL2_FILTER_ENV
150 - sources:
151 - model: *MODEL1
152 layer_range: [24, 25]
153 <<: *MODEL1_FILTER_ENV
154 - sources:
155 - model: *MODEL2
156 layer_range: [24, 25]
157 <<: *MODEL2_FILTER_ENV
158 - sources:
159 - model: *MODEL1
160 layer_range: [25, 26]
161 <<: *MODEL1_FILTER_ENV
162 - sources:
163 - model: *MODEL2
164 layer_range: [25, 26]
165 <<: *MODEL2_FILTER_ENV
166 - sources:
167 - model: *MODEL1
168 layer_range: [26, 27]
169 <<: *MODEL1_FILTER_ENV
170 - sources:
171 - model: *MODEL2
172 layer_range: [26, 27]
173 <<: *MODEL2_FILTER_ENV
174 - sources:
175 - model: *MODEL1
176 layer_range: [27, 28]
177 <<: *MODEL1_FILTER_ENV
178 - sources:
179 - model: *MODEL2
180 layer_range: [27, 28]
181 <<: *MODEL2_FILTER_ENV
182 - sources:
183 - model: *MODEL1
184 layer_range: [28, 29]
185 <<: *MODEL1_FILTER_ENV
186 - sources:
187 - model: *MODEL2
188 layer_range: [28, 29]
189 <<: *MODEL2_FILTER_ENV
190 - sources:
191 - model: *MODEL1
192 layer_range: [29, 30]
193 <<: *MODEL1_FILTER_ENV
194 - sources:
195 - model: *MODEL2
196 layer_range: [29, 30]
197 <<: *MODEL2_FILTER_ENV
198 - sources:
199 - model: *MODEL1
200 layer_range: [30, 31]
201 <<: *MODEL1_FILTER_ENV
202 - sources:
203 - model: *MODEL2
204 layer_range: [30, 31]
205 <<: *MODEL2_FILTER_ENV
206 - sources:
207 - model: *MODEL1
208 layer_range: [31, 32]
209 <<: *MODEL1_FILTER_ENV
210 - sources:
211 - model: *MODEL2
212 layer_range: [31, 32]
213 <<: *MODEL2_FILTER_ENV
214 - sources:
215 - model: *MODEL1
216 layer_range: [32, 33]
217 <<: *MODEL1_FILTER_ENV
218 - sources:
219 - model: *MODEL2
220 layer_range: [32, 33]
221 <<: *MODEL2_FILTER_ENV
222 - sources:
223 - model: *MODEL1
224 layer_range: [33, 34]
225 <<: *MODEL1_FILTER_ENV
226 - sources:
227 - model: *MODEL2
228 layer_range: [33, 34]
229 <<: *MODEL2_FILTER_ENV
230 - sources:
231 - model: *MODEL1
232 layer_range: [34, 35]
233 <<: *MODEL1_FILTER_ENV
234 - sources:
235 - model: *MODEL2
236 layer_range: [34, 35]
237 <<: *MODEL2_FILTER_ENV
238 - sources:
239 - model: *MODEL1
240 layer_range: [35, 36]
241 <<: *MODEL1_FILTER_ENV
242 - sources:
243 - model: *MODEL2
244 layer_range: [35, 36]
245 <<: *MODEL2_FILTER_ENV
246 - sources:
247 - model: *MODEL1
248 layer_range: [36, 37]
249 <<: *MODEL1_FILTER_ENV
250 - sources:
251 - model: *MODEL2
252 layer_range: [36, 37]
253 <<: *MODEL2_FILTER_ENV
254 - sources:
255 - model: *MODEL1
256 layer_range: [37, 38]
257 <<: *MODEL1_FILTER_ENV
258 - sources:
259 - model: *MODEL2
260 layer_range: [37, 38]
261 <<: *MODEL2_FILTER_ENV
262 - sources:
263 - model: *MODEL1
264 layer_range: [38, 39]
265 <<: *MODEL1_FILTER_ENV
266 - sources:
267 - model: *MODEL2
268 layer_range: [38, 39]
269 <<: *MODEL2_FILTER_ENV
270 - sources:
271 - model: *MODEL1
272 layer_range: [39, 40]
273 <<: *MODEL1_FILTER_ENV
274 - sources:
275 - model: *MODEL2
276 layer_range: [39, 40]
277 <<: *MODEL2_FILTER_ENV
278 - sources:
279 - model: *MODEL1
280 layer_range: [40, 41]
281 <<: *MODEL1_FILTER_ENV
282 - sources:
283 - model: *MODEL2
284 layer_range: [40, 41]
285 <<: *MODEL2_FILTER_ENV
286 - sources:
287 - model: *MODEL1
288 layer_range: [41, 42]
289 <<: *MODEL1_FILTER_ENV
290 - sources:
291 - model: *MODEL2
292 layer_range: [41, 42]
293 <<: *MODEL2_FILTER_ENV
294 - sources:
295 - model: *MODEL1
296 layer_range: [42, 43]
297 <<: *MODEL1_FILTER_ENV
298 - sources:
299 - model: *MODEL2
300 layer_range: [42, 43]
301 <<: *MODEL2_FILTER_ENV
302 - sources:
303 - model: *MODEL1
304 layer_range: [43, 44]
305 <<: *MODEL1_FILTER_ENV
306 - sources:
307 - model: *MODEL2
308 layer_range: [43, 44]
309 <<: *MODEL2_FILTER_ENV
310 - sources:
311 - model: *MODEL1
312 layer_range: [44, 45]
313 <<: *MODEL1_FILTER_ENV
314 - sources:
315 - model: *MODEL2
316 layer_range: [44, 45]
317 <<: *MODEL2_FILTER_ENV
318 - sources:
319 - model: *MODEL1
320 layer_range: [45, 46]
321 <<: *MODEL1_FILTER_ENV
322 - sources:
323 - model: *MODEL2
324 layer_range: [45, 46]
325 <<: *MODEL2_FILTER_ENV
326 - sources:
327 - model: *MODEL1
328 layer_range: [46, 47]
329 <<: *MODEL1_FILTER_ENV
330 - sources:
331 - model: *MODEL2
332 layer_range: [46, 47]
333 <<: *MODEL2_FILTER_ENV
334 - sources:
335 - model: *MODEL1
336 layer_range: [47, 48]
337 <<: *MODEL1_FILTER_ENV
338 - sources:
339 - model: *MODEL2
340 layer_range: [47, 48]
341 <<: *MODEL2_FILTER_ENV
342 - sources:
343 - model: *MODEL1
344 layer_range: [48, 49]
345 <<: *MODEL1_FILTER_ENV
346 - sources:
347 - model: *MODEL2
348 layer_range: [48, 49]
349 <<: *MODEL2_FILTER_ENV
350 - sources:
351 - model: *MODEL1
352 layer_range: [49, 50]
353 <<: *MODEL1_FILTER_ENV
354 - sources:
355 - model: *MODEL2
356 layer_range: [49, 50]
357 <<: *MODEL2_FILTER_ENV
358 - sources:
359 - model: *MODEL1
360 layer_range: [50, 51]
361 <<: *MODEL1_FILTER_ENV
362 - sources:
363 - model: *MODEL2
364 layer_range: [50, 51]
365 <<: *MODEL2_FILTER_ENV
366 - sources:
367 - model: *MODEL1
368 layer_range: [51, 52]
369 <<: *MODEL1_FILTER_ENV
370 - sources:
371 - model: *MODEL2
372 layer_range: [51, 52]
373 <<: *MODEL2_FILTER_ENV
374
375# The last 10 layers are not duplicated.
376 - sources:
377 - model: *BASE_MODEL
378 layer_range: [52, 62]
379
380merge_method: passthrough
381dtype: bfloat16