Views
No views yet
1# 文件名: pcb_della_merge_12b.yaml
2merge_method: della # 仍然使用 DELLA，因为它是一个强大的自适应剪裁方法
3base_model: huihui-ai/Mistral-Small-24B-Instruct-2501-abliterated
4models:
5 - model: trashpanda-org/MS-24B-Instruct-Mullein-v0
6 parameters:
7 weight: 1.0
8 # PCB策略：限制层影响范围 + 动态竞争平衡
9 layers:
10 - layers: "8-16"
11 parameter_name: density
12 value: 0.4
13 - layers: "8-16"
14 parameter_name: epsilon
15 value: 0.15
16 - layers: "8-16"
17 parameter_name: lambda
18 value: 1.5
19 - layers: "17-24"
20 parameter_name: density
21 value: 0.2
22 variance_threshold: 0.3
23
24 - model: AlSamCur123/Mistral-Small3-24B-InstructContinuedFine
25 parameters:
26 weight: 1.0
27 # 强化指令理解层
28 layers:
29 - layers: "0-12"
30 parameter_name: density
31 value: 0.7
32 - layers: "0-12"
33 parameter_name: epsilon
34 value: 0.05
35 - layers: "0-12"
36 parameter_name: lambda
37 value: 2.0
38 variance_threshold: 0.25
39
40 - model: huihui-ai/Mistral-Small-24B-Instruct-2501-abliterated
41 parameters:
42 weight: 1.0
43 # 基模型参数保护策略
44 density: 0.9
45 layers:
46 - layers: "12-24"
47 parameter_name: density
48 value: 1.0
49
50parameters:
51 global_density: 0.55 # 全局剪裁密度（PCB平衡点）
52 intra_balance: true
53 variance_threshold: 0.2
54 epsilon_range: [0.1, 0.2]
55
56tokenizer:
57 source: base
58
59generation_config:
60 eos_token_id: 2
61 pad_token_id: 2
62 repetition_penalty: 1.15
63 top_k: 40
64 temperature: 0.8
65
66# 参数压缩设置 (目标 12-13B 模型)
67architecture:
68 hidden_size: 3072 # 显著降低 hidden_size (原始 5120 -> 4096 -> 3072)
69 intermediate_size: 8256 # 相应调整 intermediate_size (比例保持不变)
70 num_attention_heads: 24 # 相应减少 attention heads (比例保持不变)
71 num_hidden_layers: 30 # 层数保持 30 层 (适度压缩)