Views
No views yet
1base_model: MrRikyz/Foxfire_Bloom
2dtype: bfloat16
3merge_method: slerp
4tokenizer_source: base
5
6slices:
7 - sources:
8 - model: MrRikyz/Foxfire_Bloom
9 layer_range: [0, 40]
10 - model: Nitral-AI/Captain-Eris_Violet-V0.420-12B
11 layer_range: [0, 40]
12
13parameters:
14 rescale: true
15 t:
16 - filter: ".*(q_proj|k_proj|v_proj).*"
17 value: [0.0, 0.2, 0.4, 0.5, 0.6, 0.65, 0.65, 0.6, 0.5, 0.4, 0.2, 0.0]
18 - filter: ".*o_proj.*"
19 value: [0.0, 0.15, 0.3, 0.4, 0.45, 0.5, 0.5, 0.45, 0.4, 0.3, 0.15, 0.0]
20 - filter: self_attn
21 value: [0.0, 0.2, 0.4, 0.5, 0.6, 0.7, 0.7, 0.6, 0.5, 0.4, 0.2, 0.0]
22
23 - filter: ".*(gate_proj|up_proj|down_proj).*"
24 value: [0.0, 0.1, 0.2, 0.3, 0.4, 0.4, 0.4, 0.4, 0.3, 0.2, 0.1, 0.0]
25 - filter: mlp
26 value: [0.0, 0.1, 0.2, 0.3, 0.4, 0.4, 0.4, 0.4, 0.3, 0.2, 0.1, 0.0]
27
28 - filter: ".*(input_layernorm|post_attention_layernorm|layernorm).*"
29 value: [0.0, 0.3, 0.5, 0.6, 0.4, 0.0, 0.0, 0.4, 0.6, 0.5, 0.3, 0.0]
30
31 - filter: "^(embed_tokens|lm_head)$"
32 value: 0.0
33
34 - value: [0.0, 0.3, 0.5, 0.6, 0.4, 0.0, 0.0, 0.4, 0.6, 0.5, 0.3, 0.0]