This is a merge of pre-trained language models created using
mergekit.
This model was merged using the
TIES merge method using
mistralai/Mistral-7B-v0.1 as a base.
1base_model: mistralai/Mistral-7B-v0.1
2dtype: float16
3merge_method: ties
4parameters:
5 int8_mask: 1.0
6 normalize: 1.0
7slices:
8- sources:
9 - layer_range: [0, 32]
10 model: mistralai/Mistral-7B-v0.1
11 - layer_range: [0, 32]
12 model: WizardLM/WizardMath-7B-V1.1
13 parameters:
14 density: 0.5
15 weight:
16 - filter: mlp
17 value: 0.5
18 - value: 0.0
19 - layer_range: [0, 32]
20 model: codellama/CodeLlama-7b-Instruct-hf
21 parameters:
22 density: 0.5
23 weight: 0.5
