1models:
2 - model: Vortex5/LunaMaid-12B
3 parameters:
4 weight:
5 - filter: self_attn
6 value: [0.35, 0.4, 0.6, 0.8, 1.0, 0.9, 0.6, 0.3]
7 - filter: mlp
8 value: [0.20, 0.25, 0.35, 0.45, 0.45, 0.40, 0.30, 0.20]
9 - value: [0.25, 0.3, 0.35, 0.4, 0.4, 0.35, 0.3, 0.25]
10 density: 0.55
11 epsilon: 0.3
12
13 - model: Vortex5/Abyssal-Seraph-12B
14 parameters:
15 weight:
16 - filter: mlp
17 value: [0.3, 0.5, 0.8, 1.0, 1.0, 0.9, 0.7, 0.4]
18 - value: [0.2, 0.3, 0.4, 0.5, 0.5, 0.4, 0.3, 0.2]
19 density: 0.5
20 epsilon: 0.4
21merge_method: della
22base_model: Vortex5/MegaMoon-Karcher-12B
23parameters:
24 lambda: 1.0
25 normalize: true
26dtype: bfloat16
27tokenizer:
28 source: Vortex5/Abyssal-Seraph-12B
