Views
No views yet
1slices:
2 - sources:
3 - model: Qwen/Qwen2-1.5B-Instruct
4 layer_range: [0, 24]
5 - sources: # add middle layers with residuals scaled to zero
6 - model: Qwen/Qwen2-1.5B-Instruct
7 layer_range: [8, 24]
8 parameters:
9 scale:
10 - filter: o_proj
11 value: 0.0
12 - filter: down_proj
13 value: 0.0
14 - value: 1.0
15 - sources: # add middle layers with residuals scaled to zero
16 - model: Qwen/Qwen2-1.5B-Instruct
17 layer_range: [8, 28]
18 parameters:
19 scale:
20 - filter: o_proj
21 value: 0.0
22 - filter: down_proj
23 value: 0.0
24 - value: 1.0
25 - sources:
26 - model: Qwen/Qwen2-1.5B-Instruct
27 layer_range: [24, 28]
28merge_method: passthrough
29dtype: bfloat16
30