This is a merge of pre-trained language models created using
mergekit.
This model was merged using the
SLERP merge method.
1# Stage 2: Slerp with Lamarck Components [Optimized]
2name: MergeStage2v3
3merge_method: slerp
4base_model: CultriX/MergeStage1v3
5tokenizer_source: base # Verify and update if needed
6dtype: bfloat16
7parameters:
8 normalize: true
9 rescale: false
10 int8_mask: true
11 int8_mask: true
12 t:
13 - value: 0.35 # Adjusted starting value
14slices:
15 - sources:
16 - model: CultriX/MergeStage1v3
17 layer_range: [0, 6] # Example - Adjust based on model architecture
18 - model: sometimesanotion/Lamarck-14B-v0.7-rc4
19 layer_range: [0, 6] # Example - Adjust based on model architecture
20 - sources:
21 - model: CultriX/MergeStage1v3
22 layer_range: [6, 12] # Example - Adjust based on model architecture
23 - model: sometimesanotion/Lamarck-14B-v0.7-rc4
24 layer_range: [6, 12] # Example - Adjust based on model architecture
25 - sources:
26 - model: CultriX/MergeStage1v3
27 layer_range: [12, 18] # Example - Adjust based on model architecture
28 - model: sometimesanotion/Lamarck-14B-v0.7-rc4
29 layer_range: [12, 18] # Example - Adjust based on model architecture
30 - sources:
31 - model: CultriX/MergeStage1v3
32 layer_range: [18, 24] # Example - Adjust based on model architecture
33 - model: sometimesanotion/Lamarck-14B-v0.7-rc4
34 layer_range: [18, 24] # Example - Adjust based on model architecture
35 - sources:
36 - model: CultriX/MergeStage1v3
37 layer_range: [24, 30] # Example - Adjust based on model architecture
38 - model: sometimesanotion/Lamarck-14B-v0.7-rc4
39 layer_range: [24, 30] # Example - Adjust based on model architecture
40 - sources:
41 - model: CultriX/MergeStage1v3
42 layer_range: [30, 36] # Example - Adjust based on model architecture
43 - model: sometimesanotion/Lamarck-14B-v0.7-rc4
44 layer_range: [30, 36] # Example - Adjust based on model architecture
45 - sources:
46 - model: CultriX/MergeStage1v3
47 layer_range: [36, 42] # Example - Adjust based on model architecture
48 - model: sometimesanotion/Lamarck-14B-v0.7-rc4
49 layer_range: [36, 42] # Example - Adjust based on model architecture
50 - sources:
51 - model: CultriX/MergeStage1v3
52 layer_range: [42, 48] # Example - Adjust based on model architecture
53 - model: sometimesanotion/Lamarck-14B-v0.7-rc4
54 layer_range: [42, 48] # Example - Adjust based on model architectur
