This is a merge of pre-trained language models created using
mergekit.
This model was merged using the
Model Stock merge method using
Gaivoronsky/Mistral-7B-Saiga as a base.
1slices:
2- sources:
3 - model: Gaivoronsky/Mistral-7B-Saiga
4 layer_range:
5 - 0
6 - 32
7 - model: HuggingFaceH4/mistral-7b-grok
8 layer_range:
9 - 0
10 - 32
11 - model: NousResearch/Yarn-Mistral-7b-128k
12 layer_range:
13 - 0
14 - 32
15merge_method: model_stock
16base_model: Gaivoronsky/Mistral-7B-Saiga
17parameters:
18 t:
19 - filter: self_attn
20 value:
21 - 0
22 - 0.5
23 - 0.3
24 - 0.7
25 - 1
26 - filter: mlp
27 value:
28 - 1
29 - 0.5
30 - 0.7
31 - 0.3
32 - 0
33 - filter: mlp
34 value:
35 - 1
36 - 0.5
37 - 0.7
38 - 0.3
39 - 0
40 - value: 0.5
41dtype: bfloat16
