Views
No views yet
IQ2_KS 181.239 GiB (2.317 BPW)1#!/usr/bin/env bash
2
3custom="
4# First 3 dense layers (0-3) (GPU)
5# Except blk.*.attn_k_b.weight is not divisible by 256 so only supports qN_0
6blk\.[0-2]\.attn_k_b.*=q4_0
7blk\.[0-2]\.attn_.*=iq4_ks
8blk\.[0-2]\.ffn_down.*=iq4_ks
9blk\.[0-2]\.ffn_(gate|up).*=iq4_ks
10blk\.[0-2]\..*=iq4_ks
11
12# All attention, norm weights, and bias tensors for MoE layers (3-60) (GPU)
13# Except blk.*.attn_k_b.weight is not divisible by 256 so only supports qN_0
14blk\.[3-9]\.attn_k_b.*=q4_0
15blk\.[1-5][0-9]\.attn_k_b.*=q4_0
16blk\.60\.attn_k_b.*=q4_0
17
18blk\.[3-9]\.attn_.*=iq4_ks
19blk\.[1-5][0-9]\.attn_.*=iq4_ks
20blk\.60\.attn_.*=iq4_ks
21
22# Shared Expert (3-60) (GPU)
23blk\.[3-9]\.ffn_down_shexp\.weight=iq4_ks
24blk\.[1-5][0-9]\.ffn_down_shexp\.weight=iq4_ks
25blk\.60\.ffn_down_shexp\.weight=iq4_ks
26
27blk\.[3-9]\.ffn_(gate|up)_shexp\.weight=iq4_ks
28blk\.[1-5][0-9]\.ffn_(gate|up)_shexp\.weight=iq4_ks
29blk\.60\.ffn_(gate|up)_shexp\.weight=iq4_ks
30
31# Routed Experts (3-60) (CPU)
32blk\.[3-9]\.ffn_down_exps\.weight=iq2_k
33blk\.[1-5][0-9]\.ffn_down_exps\.weight=iq2_k
34blk\.60\.ffn_down_exps\.weight=iq2_k
35
36blk\.[3-9]\.ffn_(gate|up)_exps\.weight=iq2_ks
37blk\.[1-5][0-9]\.ffn_(gate|up)_exps\.weight=iq2_ks
38blk\.60\.ffn_(gate|up)_exps\.weight=iq2_ks
39
40# Token embedding and output tensors (GPU)
41token_embd\.weight=iq4_k
42output\.weight=Q8_0<｜begin▁of▁sentence｜>{system_prompt}<｜User｜>{prompt}<｜Assistant｜><｜end▁of▁sentence｜><｜Assistant｜>llama-server --model <Path>\DeepSeek-V3.1_Base_IQ2_KS-00001-of-00005.gguf -fa -rtr -mla 3 --ctx-size 40000 -ctk q8_0 -b 4092 -ub 4092 -amb 512 --n-gpu-layers 99 -ot "blk\.(3|4)\.ffn_.*=CUDA0" --override-tensor exps=CPU --threads 8 --host 127.0.0.1 --port 8080ik_llama.cpp quantizations of DeepSeek-V3.1-Basellama.cpp fork, ik_llama.cpp