Views
No views yet
The scale factor to use, eg: solve x^2 = 1/2 --> x = 1/sqrt(2) ≈ 0.7071067812
1###############################
2# llama-3-attenuated.yaml #
3###############################
4
5# Use: mergekit-yaml --clone-tensors ./llama-3-attenuated.yaml ./llama-3-attenuated
6# See: https://github.com/arcee-ai/mergekit/issues/198 for discussion/reasoning behind this idea.
7
8# ---
9
10# The scale factor to use, eg: solve x^2 = 1/2 --> x = 1/sqrt(2) ≈ 0.7071067812
11const_tag: &scale_factor 0.7071067812 # 1/sqrt(2)
12
13# The filter parameters of a scaled block.
14attenuate-env: &attenuated_env
15 parameters:
16 scale:
17 - filter: q_proj
18 value: *scale_factor
19 - filter: k_proj
20 value: *scale_factor
21 - value: 1.0
22
23# ---
24
25slices:
26 - sources:
27 - model: kuotient/Meta-Llama-3-8B-Instruct
28 layer_range: [0, 4]
29 - sources:
30 - model: kuotient/Meta-Llama-3-8B-Instruct
31 layer_range: [4, 8]
32 <<: *attenuated_env
33
34 - sources:
35 - model: kuotient/Meta-Llama-3-8B-Instruct
36 layer_range: [4, 12]
37 <<: *attenuated_env
38 - sources:
39 - model: kuotient/Meta-Llama-3-8B-Instruct
40 layer_range: [8, 16]
41 <<: *attenuated_env
42 - sources:
43 - model: kuotient/Meta-Llama-3-8B-Instruct
44 layer_range: [12, 20]
45 <<: *attenuated_env
46 - sources:
47 - model: kuotient/Meta-Llama-3-8B-Instruct
48 layer_range: [16, 24]
49 <<: *attenuated_env
50 - sources:
51 - model: kuotient/Meta-Llama-3-8B-Instruct
52 layer_range: [20, 28]
53 <<: *attenuated_env
54
55 - sources:
56 - model: kuotient/Meta-Llama-3-8B-Instruct
57 layer_range: [24, 28]
58 <<: *attenuated_env
59 - sources:
60 - model: kuotient/Meta-Llama-3-8B-Instruct
61 layer_range: [28, 32]
62
63merge_method: passthrough
64dtype: bfloat16
65