This is a merge of pre-trained language models created using
mergekit.
This model was merged using the SLERP merge method.
1
2slices:
3 - sources:
4 - model: mlabonne/AlphaMonarch-7B #mistralai/Mistral-7B-Instruct-v0.2
5 layer_range: [0, 32]
6 # - model: EmbeddedLLM/Mistral-7B-Merge-14-v0.1
7 # layer_range: [0, 32]
8 # - model: mlabonne/AlphaMonarch-7B
9 # layer_range: [0, 32]
10 - model: cognitivecomputations/dolphin-2.8-mistral-7b-v02
11 layer_range: [0, 32]
12 #mlabonne/AlphaMonarch-7B
13
14merge_method: slerp
15base_model: cognitivecomputations/dolphin-2.8-mistral-7b-v02 #mistralai/Mistral-7B-Instruct-v0.2
16parameters:
17 t:
18 - filter: self_attn
19 value: [0, 0.5, 0.3, 0.7, 1]
20 - filter: mlp
21 value: [1, 0.5, 0.7, 0.3, 0]
22 - value: 0.5
23dtype: bfloat16
24
