Views
No views yet
[!CAUTION] ⚠️ Warning: This model can produce narratives and RP that contain violent and graphic erotic content. Adjust your system prompt accordingly, and use Mistral Tekken chat template.
1architecture: MistralForCausalLM
2models:
3 - model: B:\24B\!models--anthracite-core--Mistral-Small-3.2-24B-Instruct-2506-Text-Only
4 - model: B:\24B\!models--TheDrummer--Cydonia-24B-v4.3
5 parameters:
6 density: 0.75
7 weight: 0.5
8 epsilon: 0.25
9 - model: B:\24B\!models--ReadyArt--4.2.0-Broken-Tutu-24b
10 parameters:
11 density: 0.75
12 weight: 0.25
13 epsilon: 0.25
14 - model: B:\24B\PrivateMerge29 # This merge is no longer available on HF
15 parameters:
16 density: 0.75
17 weight: 0.25
18 epsilon: 0.25
19 - model: B:\24B\!models--zerofata--MS3.2-PaintedFantasy-v2-24B
20 parameters:
21 density: 0.75
22 weight: 0.5
23 epsilon: 0.25
24 - model: B:\24B\!models--TheDrummer--Magidonia-24B-v4.3
25 parameters:
26 density: 0.75
27 weight: 0.5
28 epsilon: 0.25
29 - model: B:\24B\!models--TheDrummer--Precog-24B-v1
30 parameters:
31 density: 0.75
32 weight: 0.5
33 epsilon: 0.25
34 - model: B:\24B\!models--zerofata--MS3.2-PaintedFantasy-v3-24B
35 parameters:
36 density: 0.75
37 weight: 0.5
38 epsilon: 0.25
39## Merge Settings
40## --copy-tokenizer --allow-crimes --out-shard-size 5B --trust-remote-code --lazy-unpickle --random-seed 420 --cuda
41merge_method: della
42base_model: B:\24B\!models--anthracite-core--Mistral-Small-3.2-24B-Instruct-2506-Text-Only
43parameters:
44 lambda: 1.0
45 normalize: false
46 int8_mask: false
47 rescale: true
48dtype: float32
49out_dtype: bfloat16
50tokenizer:
51 source: union
52chat_template: auto
53name: 🐌 Ślimaki-24B-v1.2sparsify.py to auto-shrink Epsilon1def della_magprune(
2 tensor: torch.Tensor,
3 density: float,
4 epsilon: float,
5 rescale_norm: Optional[RescaleNorm] = None,
6) -> torch.Tensor:
7 if density >= 1:
8 return tensor
9 if density <= 0:
10 return torch.zeros_like(tensor)
11 orig_shape = tensor.shape
12
13 if density + epsilon >= 1 or density - epsilon <= 0:
14 raise ValueError(
15 "Epsilon must be chosen such that density +/- epsilon is in (0, 1)"
16 )
17
18 work_dtype = (
19 tensor.dtype
20 if tensor.device.type != "cpu" or tensor.dtype == torch.bfloat16
21 else torch.float32
22 )
23
24 if len(tensor.shape) < 2:
25 tensor = tensor.unsqueeze(0)
26 magnitudes = tensor.abs()
27
28 sorted_indices = torch.argsort(magnitudes, dim=1, descending=False)
29 ranks = sorted_indices.argsort(dim=1).to(work_dtype) + 1
30
31 min_ranks = ranks.min(dim=1, keepdim=True).values
32 max_ranks = ranks.max(dim=1, keepdim=True).values
33 rank_norm = ((ranks - min_ranks) / (max_ranks - min_ranks)).clamp(0, 1)
34 probs = (density - epsilon) + rank_norm * 2 * epsilon
35 mask = torch.bernoulli(probs).to(work_dtype)
36
37 res = rescaled_masked_tensor(tensor.to(work_dtype), mask, rescale_norm)
38 return res.to(tensor.dtype).reshape(orig_shape)1def della_magprune(
2 tensor: torch.Tensor,
3 density: float,
4 epsilon: float,
5 rescale_norm: Optional[RescaleNorm] = None,
6) -> torch.Tensor:
7 if density >= 1:
8 return tensor
9 if density <= 0:
10 return torch.zeros_like(tensor)
11
12 # --- SAFETY GUARD START ---
13 # Ensure density isn't exactly 0 or 1
14 density = max(1e-4, min(1.0 - 1e-4, density))
15
16 # Epsilon must be < density AND < (1 - density)
17 # If the optimizer guessed a bad epsilon, we shrink it to the max allowed value
18 max_epsilon = min(density, 1.0 - density) - 1e-4
19 if abs(epsilon) > max_epsilon:
20 epsilon = max_epsilon if epsilon > 0 else -max_epsilon
21 # --- SAFETY GUARD END ---
22
23 orig_shape = tensor.shape
24 work_dtype = (
25 tensor.dtype
26 if tensor.device.type != "cpu" or tensor.dtype == torch.bfloat16
27 else torch.float32
28 )
29
30 if len(tensor.shape) < 2:
31 tensor = tensor.unsqueeze(0)
32 magnitudes = tensor.abs()
33
34 sorted_indices = torch.argsort(magnitudes, dim=1, descending=False)
35 ranks = sorted_indices.argsort(dim=1).to(work_dtype) + 1
36
37 min_ranks = ranks.min(dim=1, keepdim=True).values
38 max_ranks = ranks.max(dim=1, keepdim=True).values
39 rank_norm = ((ranks - min_ranks) / (max_ranks - min_ranks)).clamp(0, 1)
40
41 # Now this line is guaranteed not to produce values < 0 or > 1
42 probs = (density - epsilon) + rank_norm * 2 * epsilon
43 mask = torch.bernoulli(probs.clamp(0, 1)).to(work_dtype)
44
45 res = rescaled_masked_tensor(tensor.to(work_dtype), mask, rescale_norm)
46 return res.to(tensor.dtype).reshape(orig_shape)