This is a merge of pre-trained language models created using
mergekit.
This model was merged using the
DARE TIES merge method using
mistralai/Mistral-7B-v0.1 as a base.
1base_model: mistralai/Mistral-7B-v0.1
2dtype: bfloat16
3merge_method: dare_ties
4modules:
5 default:
6 slices:
7 - sources:
8 - layer_range: [0, 32]
9 model: mistralai/Mistral-7B-v0.1
10 - layer_range: [0, 32]
11 model: samir-fama/SamirGPT-v1
12 parameters:
13 density: 0.53
14 weight: 0.4
15 - layer_range: [0, 32]
16 model: abacusai/Slerp-CM-mist-dpo
17 parameters:
18 density: 0.53
19 weight: 0.3
20 - layer_range: [0, 32]
21 model: OpenPipe/mistral-ft-optimized-1218
22 parameters:
23 density: 0.53
24 weight: 0.3
25parameters:
26 int8_mask: 1.0
