This is a merge of pre-trained language models created using
mergekit.
This model was merged using the
DARE TIES merge method using
Qwen/Qwen2.5-14B as a base.
1base_model: Qwen/Qwen2.5-14B
2models:
3 # Each adapter was extracted (rank=128) from its respective finetuned model.
4 # Their weights are set lower than the full instruct model (which is now the base)
5 - model: CultriX/Qwen2.5-14B-Hyperionv3_r128
6 parameters:
7 weight: 0.9 # Reduced weight relative to base
8 density: 0.9
9
10 - model: CultriX/Qwen2.5-14B_Virtuoso-small-v2-LoRA_r128
11 parameters:
12 weight: 1.0
13 density: 1.0
14
15 - model: Qwen/Qwen2.5-14B-Instruct
16 parameters:
17 weight: 0.75
18 density: 0.75
19
20 - model: /root/.cache/huggingface/hub/models--CultriX--Qwen2.5-14B-DeepSeek_r128/snapshots/1bca847f92fced165076d9ac921a1e3ef01fcd7f/
21 parameters:
22 weight: 1.00
23 density: 1.00
24
25# Merging method and overall parameters
26merge_method: dare_ties # Ties corresponding weights across sources.
27parameters:
28 weight: 1.0 # Overall scaling factor.
29 density: 1.0 # Overall density (typically left at 1.0).
30 normalize: true # Normalize each set of weights before merging.
31 int8_mask: true # Enable masking if using int8 quantized weights.
32
33# Use the instruct tokenizer to ensure compatibility.
34tokenizer_source: CultriX/Qwen2.5-14B_Virtuoso-small-v2-LoRA_r128
35
36# Data type for merged weights.
37dtype: bfloat16
38
39
