Views
No views yet
1# 文件名: pcb_della_merge.yaml
2merge_method: della # 基于DELLA的自适应剪裁
3base_model: huihui-ai/Mistral-Small-24B-Instruct-2501-abliterated
4models:
5 - model: trashpanda-org/MS-24B-Instruct-Mullein-v0
6 parameters:
7 weight: 1.0 # 添加默认权重
8 # PCB策略：限制层影响范围 + 动态竞争平衡
9 layers:
10 - layers: "8-16"
11 parameter_name: density
12 value: 0.4
13 - layers: "8-16"
14 parameter_name: epsilon
15 value: 0.15
16 - layers: "8-16"
17 parameter_name: lambda
18 value: 1.5
19 - layers: "17-24"
20 parameter_name: density
21 value: 0.2
22 variance_threshold: 0.3
23
24 - model: AlSamCur123/Mistral-Small3-24B-InstructContinuedFine
25 parameters:
26 weight: 1.0 # 添加默认权重
27 # 强化指令理解层
28 layers:
29 - layers: "0-12"
30 parameter_name: density
31 value: 0.7
32 - layers: "0-12"
33 parameter_name: epsilon
34 value: 0.05
35 - layers: "0-12"
36 parameter_name: lambda
37 value: 2.0
38 variance_threshold: 0.25
39
40 - model: huihui-ai/Mistral-Small-24B-Instruct-2501-abliterated
41 parameters:
42 weight: 1.0 # 添加默认权重
43 # 基模型参数保护策略
44 density: 0.9 # 全局密度 (可能需要单独处理)
45 layers:
46 - layers: "12-24"
47 parameter_name: density
48 value: 1.0
49
50parameters:
51 global_density: 0.55 # 全局剪裁密度（PCB平衡点）
52 intra_balance: true
53 variance_threshold: 0.2
54 epsilon_range: [0.1, 0.2]
55
56tokenizer:
57 source: base
58
59# 参数压缩设置（实现12-13B目标）
60architecture:
61 hidden_size: 4096
62 intermediate_size: 11008
63 num_attention_heads: 32
64 num_hidden_layers: 30