Insanity is gone, creativity remains. Enjoy.
This is a merge of pre-trained language models created using
mergekit.
This model was merged using the
Task Arithmetic merge method using /workspace/toppy/mistralai_Mistral-7B-v0.1 as a base.
1base_model: /workspace/toppy/mistralai_Mistral-7B-v0.1
2dtype: bfloat16
3merge_method: task_arithmetic
4modules:
5 default:
6 slices:
7 - sources:
8 - layer_range: [0, 8]
9 model: /workspace/toppy/stepC1
10 parameters:
11 weight: -0.08064321988635652
12 - layer_range: [0, 8]
13 model: /workspace/toppy/stepB1
14 parameters:
15 weight: 0.47180144787421097
16 - layer_range: [0, 8]
17 model: /workspace/toppy/stepA1
18 parameters:
19 weight: 0.49216900949431464
20 - layer_range: [0, 8]
21 model: /workspace/toppy/mistralai_Mistral-7B-v0.1
22 - sources:
23 - layer_range: [8, 16]
24 model: /workspace/toppy/stepC1
25 parameters:
26 weight: 0.4262113662158592
27 - layer_range: [8, 16]
28 model: /workspace/toppy/stepB1
29 parameters:
30 weight: 0.25821032498583896
31 - layer_range: [8, 16]
32 model: /workspace/toppy/stepA1
33 parameters:
34 weight: 1.1337184135136393
35 - layer_range: [8, 16]
36 model: /workspace/toppy/mistralai_Mistral-7B-v0.1
37 - sources:
38 - layer_range: [16, 24]
39 model: /workspace/toppy/stepC1
40 parameters:
41 weight: 0.08816883209543347
42 - layer_range: [16, 24]
43 model: /workspace/toppy/stepB1
44 parameters:
45 weight: -0.04615180476355138
46 - layer_range: [16, 24]
47 model: /workspace/toppy/stepA1
48 parameters:
49 weight: 0.6614915110635577
50 - layer_range: [16, 24]
51 model: /workspace/toppy/mistralai_Mistral-7B-v0.1
52 - sources:
53 - layer_range: [24, 32]
54 model: /workspace/toppy/stepC1
55 parameters:
56 weight: 0.2995904282557765
57 - layer_range: [24, 32]
58 model: /workspace/toppy/stepB1
59 parameters:
60 weight: 0.48755088752753767
61 - layer_range: [24, 32]
62 model: /workspace/toppy/stepA1
63 parameters:
64 weight: 0.48445653643927594
65 - layer_range: [24, 32]
66 model: /workspace/toppy/mistralai_Mistral-7B-v0.1
67parameters:
68 int8_mask: 1.0
69 normalize: 0.0
70tokenizer_source: /workspace/toppy/stepC1
