Views
No views yet
1base_model: bamec66557/VICIOUS_MESH-12B-OMEGA
2dtype: bfloat16
3merge_method: slerp
4tokenizer_source: base
5
6# Slices Configuration
7slices:
8 - sources:
9 - model: bamec66557/VICIOUS_MESH-12B-OMEGA
10 layer_range: [0, 10]
11 - model: bamec66557/VICIOUS_MESH-12B-BETA
12 layer_range: [0, 10]
13 parameters:
14 t:
15 - name: self_attn
16 value: [0.5, 0.55, 0.6, 0.65, 0.7]
17 - name: mlp
18 value: [1.0, 1.05, 1.1, 1.15, 1.2]
19 - name: layer_norm
20 value: [0.9, 0.95, 1.0, 1.05, 1.1]
21
22 - sources:
23 - model: bamec66557/VICIOUS_MESH-12B-OMEGA
24 layer_range: [10, 20]
25 - model: bamec66557/VICIOUS_MESH-12B-BETA
26 layer_range: [10, 20]
27 parameters:
28 t:
29 - name: self_attn
30 value: [0.4, 0.45, 0.5, 0.55, 0.6]
31 - name: mlp
32 value: [1.1, 1.15, 1.2, 1.25, 1.3]
33 - name: layer_norm
34 value: [1.0, 1.05, 1.1, 1.15, 1.2]
35
36 - sources:
37 - model: bamec66557/VICIOUS_MESH-12B-OMEGA
38 layer_range: [20, 30]
39 - model: bamec66557/VICIOUS_MESH-12B-BETA
40 layer_range: [20, 30]
41 parameters:
42 t:
43 - name: self_attn
44 value: [0.6, 0.65, 0.7, 0.75, 0.8]
45 - name: mlp
46 value: [0.9, 0.95, 1.0, 1.05, 1.1]
47 - name: layer_norm
48 value: [0.85, 0.9, 0.95, 1.0, 1.05]
49
50 - sources:
51 - model: bamec66557/VICIOUS_MESH-12B-OMEGA
52 layer_range: [30, 40]
53 - model: bamec66557/VICIOUS_MESH-12B-BETA
54 layer_range: [30, 40]
55 parameters:
56 t:
57 - name: self_attn
58 value: [0.7, 0.75, 0.8, 0.85, 0.9]
59 - name: mlp
60 value: [0.8, 0.85, 0.9, 0.95, 1.0]
61 - name: layer_norm
62 value: [0.8, 0.85, 0.9, 0.95, 1.0]
63
64# Regularization
65regularization:
66 - method: gradient_penalty
67 scale: 0.05 # Increased influence for gradient control
68 - method: weight_clipping
69 clip_range: [-0.2, 0.2] # Broader clipping range for flexibility
70 - method: random_noise
71 scale: 0.01 # Stronger noise injection
72 - method: attention_dropout
73 scale: 0.1 # Higher dropout to reduce attention fixation
74
75# Postprocessing
76postprocessing:
77 - operation: entropy_regularization
78 scale: 0.05 # Stronger encouragement for diverse outputs
79 - operation: non_linear_scaling
80 parameters:
81 function: tanh
82 - operation: sharpening
83 intensity: 0.5 # Enhanced sharpening for precise outputs
84 - operation: gaussian_smoothing
85 sigma: 1.5 # Increased smoothing for stable outputs
86 - operation: normalize
87 - operation: dynamic_scaling
88 scale_range: [0.8, 1.2] # Expanded dynamic range for scaling
89 - operation: smoothing
90 parameters:
91 adaptive: true
92 range: [0.85, 1.15] # Wider adaptive smoothing range
93 kernel_size: 5
94| Metric | Value |
|---|---|
| Avg. | 27.26 |
| IFEval (0-Shot) | 67.21 |
| BBH (3-Shot) | 31.36 |
| MATH Lvl 5 (4-Shot) | 12.08 |
| GPQA (0-shot) | 8.84 |
| MuSR (0-shot) | 14.34 |
| MMLU-PRO (5-shot) | 29.76 |