1merge_method: della_linear
2base_model: migtissera/Tess-3-Llama-3.1-70B
3models:
4 - model: cognitivecomputations/dolphin-2.9.1-llama-3-70b
5 parameters:
6 weight:
7 - filter: q_proj
8 value: [0, 0, 1, 1, 1, 1, 1, 1, 1, 0, 0]
9 - filter: k_proj
10 value: [0, 0, 1, 1, 1, 1, 1, 1, 1, 0, 0]
11 - filter: v_proj
12 value: [0, 0, 1, 1, 1, 1, 1, 1, 1, 0, 0]
13 - filter: o_proj
14 value: [0, 0, 1, 1, 1, 1, 1, 1, 1, 0, 0]
15 - filter: input_layernorm
16 value: [0, 0, 1, 1, 1, 1, 1, 1, 1, 0, 0]
17 - filter: up_proj
18 value: [0, 0, 1, 1, 1, 1, 1, 1, 1, 0, 0]
19 - filter: gate_proj
20 value: [0, 0, 1, 1, 1, 1, 1, 1, 1, 0, 0]
21 - filter: down_proj
22 value: [0, 0, 1, 1, 1, 1, 1, 1, 1, 0, 0]
23 - filter: post_attention_layernorm
24 value: [0, 0, 1, 1, 1, 1, 1, 1, 1, 0, 0]
25 - value: 0
26 density: 0.5
27 epsilon: 0.05
28 lambda: 1.0
29 - model: migtissera/Tess-3-Llama-3.1-70B
30 parameters:
31 weight: 1.0
32 density:
33 - filter: q_proj
34 value: [1, 1, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 1, 1]
35 - filter: k_proj
36 value: [1, 1, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 1, 1]
37 - filter: v_proj
38 value: [1, 1, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 1, 1]
39 - filter: o_proj
40 value: [1, 1, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 1, 1]
41 - filter: input_layernorm
42 value: [1, 1, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 1, 1]
43 - filter: up_proj
44 value: [1, 1, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 1, 1]
45 - filter: gate_proj
46 value: [1, 1, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 1, 1]
47 - filter: down_proj
48 value: [1, 1, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 1, 1]
49 - filter: post_attention_layernorm
50 value: [1, 1, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 1, 1]
51 - value: 0.5
52 epsilon:
53 - filter: q_proj
54 value: [0, 0, 0.05, 0.05, 0.07, 0.1, 0.07, 0.05, 0.05, 0, 0]
55 - filter: k_proj
56 value: [0, 0, 0.05, 0.05, 0.07, 0.1, 0.07, 0.05, 0.05, 0, 0]
57 - filter: v_proj
58 value: [0, 0, 0.05, 0.05, 0.07, 0.1, 0.07, 0.05, 0.05, 0, 0]
59 - filter: o_proj
60 value: [0, 0, 0.05, 0.05, 0.07, 0.1, 0.07, 0.05, 0.05, 0, 0]
61 - filter: input_layernorm
62 value: [0, 0, 0.05, 0.05, 0.07, 0.1, 0.07, 0.05, 0.05, 0, 0]
63 - filter: up_proj
64 value: [0, 0, 0.05, 0.05, 0.07, 0.1, 0.07, 0.05, 0.05, 0, 0]
65 - filter: gate_proj
66 value: [0, 0, 0.05, 0.05, 0.07, 0.1, 0.07, 0.05, 0.05, 0, 0]
67 - filter: down_proj
68 value: [0, 0, 0.05, 0.05, 0.07, 0.1, 0.07, 0.05, 0.05, 0, 0]
69 - filter: post_attention_layernorm
70 value: [0, 0, 0.05, 0.05, 0.07, 0.1, 0.07, 0.05, 0.05, 0, 0]
71 - value: 0.1
72 lambda: 1.0
73dtype: bfloat16
74out_dtype: bfloat16
75parameters:
76 int8_mask: true
77 normalize: true
78 rescale: true
79 filter_wise: false
80chat_template: auto
81tokenizer:
82 source: union
