Views
No views yet
The scale factor to use, eg: solve x^2 = 1/2 --> x = 1/sqrt(2) ≈ 0.7071067812
1###############################
2# llama-3-attenuated.yaml #
3###############################
4
5# Use: mergekit-yaml --clone-tensors ./llama-3-attenuated.yaml ./llama-3-attenuated
6# See: https://github.com/arcee-ai/mergekit/issues/198 for discussion/reasoning behind this idea.
7
8# ---
9
10# The scale factor to use, eg: solve x^2 = 1/2 --> x = 1/sqrt(2) ≈ 0.7071067812
11const_tag: &scale_factor 0.7071067812 # 1/sqrt(2)
12
13# The filter parameters of a scaled block.
14attenuate-env: &attenuated_env
15 parameters:
16 scale:
17 - filter: q_proj
18 value: *scale_factor
19 - filter: k_proj
20 value: *scale_factor
21 - value: 1.0
22
23# ---
24
25slices:
26
27 ###########################
28 # Block 1: miqu-1 [0, 16] #
29 ###########################
30 - sources:
31 - model: kuotient/Meta-Llama-3-8B-Instruct
32 layer_range: [0, 8] # The first 8 layers of Block 1 are not duplicated
33 - sources:
34 - model: kuotient/Meta-Llama-3-8B-Instruct
35 layer_range: [8, 16] # The last 8 layers of Block 1 are are duplicated twice
36 <<: *attenuated_env
37
38 ###########################
39 # Block 2: miqu-1 [8, 24] #
40 ###########################
41 - sources:
42 - model: kuotient/Meta-Llama-3-8B-Instruct
43 layer_range: [8, 24] # All the layers of Block 2 are are duplicated twice
44 <<: *attenuated_env
45
46 ##########################
47 # Block 3: miqu-1 [16, 32] #
48 ##########################
49 - sources:
50 - model: kuotient/Meta-Llama-3-8B-Instruct
51 layer_range: [16, 24] # The first 8 layers of Block 3 are are duplicated twice
52 <<: *attenuated_env
53 - sources:
54 - model: kuotient/Meta-Llama-3-8B-Instruct
55 layer_range: [24, 32] # The last 8 layers of Block 3 are not duplicated
56
57merge_method: passthrough
58dtype: bfloat16