Views
No views yet
ik_llama.cpp imatrix Quantizations of deepseek-ai/DeepSeek-V3.1ik_llama.cpp can also run your existing GGUFs from bartowski, unsloth, mradermacher, etc if you want to try it out before downloading my quants.1#!/usr/bin/env bash
2
3custom="
4# First 3 dense layers (0-3) (GPU)
5# Using q8_0 for attn_k_b since imatrix might not have these tensors
6blk\.[0-2]\.attn_k_b.*=q8_0
7blk\.[0-2]\.attn_.*=iq5_ks
8blk\.[0-2]\.ffn_down.*=iq5_ks
9blk\.[0-2]\.ffn_(gate|up).*=iq4_ks
10blk\.[0-2]\..*=iq5_ks
11
12# All attention, norm weights, and bias tensors for MoE layers (3-60) (GPU)
13# Using q8_0 for attn_k_b since imatrix might not have these tensors
14blk\.[3-9]\.attn_k_b.*=q8_0
15blk\.[1-5][0-9]\.attn_k_b.*=q8_0
16blk\.60\.attn_k_b.*=q8_0
17
18blk\.[3-9]\.attn_.*=iq5_ks
19blk\.[1-5][0-9]\.attn_.*=iq5_ks
20blk\.60\.attn_.*=iq5_ks
21
22# Shared Expert (3-60) (GPU)
23blk\.[3-9]\.ffn_down_shexp\.weight=iq5_ks
24blk\.[1-5][0-9]\.ffn_down_shexp\.weight=iq5_ks
25blk\.60\.ffn_down_shexp\.weight=iq5_ks
26
27blk\.[3-9]\.ffn_(gate|up)_shexp\.weight=iq4_ks
28blk\.[1-5][0-9]\.ffn_(gate|up)_shexp\.weight=iq4_ks
29blk\.60\.ffn_(gate|up)_shexp\.weight=iq4_ks
30
31# Routed Experts (3-60) (CPU)
32blk\.[3-9]\.ffn_down_exps\.weight=iq3_ks
33blk\.[1-5][0-9]\.ffn_down_exps\.weight=iq3_ks
34blk\.60\.ffn_down_exps\.weight=iq3_ks
35
36blk\.[3-9]\.ffn_(gate|up)_exps\.weight=iq2_ks
37blk\.[1-5][0-9]\.ffn_(gate|up)_exps\.weight=iq2_ks
38blk\.60\.ffn_(gate|up)_exps\.weight=iq2_ks
39
40# Token embedding and output tensors (GPU)
41token_embd\.weight=iq5_k
42output\.weight=q8_0 # Changed to q8_0
43"
44
45custom=$(
46 echo "$custom" | grep -v '^#' | \
47 sed -Ez 's:\n+:,:g;s:,$::;s:^,::'
48)
49
50./build/bin/llama-quantize \
51 --custom-q "$custom" \
52 --imatrix /fast/DeepSeek-V3.1.imatrix \
53 /fast/bf16/DeepSeek-V3-00001-of-00030.gguf
54 /fast2/quants/DeepSeek-V3.1-IQ2_KS.gguf \
55 IQ2_KS \