Views
No views yet
ik_llama.cpp imatrix Quantizations of Qwen/Qwen3.8-27Bik_llama.cpp can also run your existing GGUFs from bartowski, unsloth, mradermacher, etc if you want to try it out before downloading my quants. Only a couple quants in this collection are compatible with mainline llamma.cpp/LMStudio/KoboldCPP/etc as mentioned in the specific description, all others require ik_llama.cpp.BF16 50.894 GiB (16.002 BPW)
Q8_0 27.042 GiB (8.502 BPW)
iq4_ks mtp head so no need for -mtprot iq4_ks now.1#!/usr/bin/env bash
2
3custom="
4# 64 Repeating Layers [0-63] + blk.64 MTP/nextn tensors
5
6## MTP/nextn tensors
7blk\.64\..*\.weight=iq4_ks
8
9## Gated Attention/Delta Net [Blended 0-63]
10blk\..*\.attn_gate\.weight=iq4_ks
11blk\..*\.attn_qkv\.weight=iq4_ks
12blk\..*\.attn_output\.weight=iq4_ks
13blk\..*\.attn_q\.weight=iq4_ks
14blk\..*\.attn_k\.weight=iq4_ks
15blk\..*\.attn_v\.weight=iq4_ks
16blk\..*\.ssm_alpha\.weight=q8_0
17blk\..*\.ssm_beta\.weight=q8_0
18blk\..*\.ssm_out\.weight=q6_0
19
20# Dense Layers [0-63]
21blk\..*\.ffn_down\.weight=iq4_ks
22blk\..*\.ffn_(gate|up)\.weight=iq4_ks
23
24# Non-Repeating Layers
25token_embd\.weight=q6_0
26output\.weight=q8_0
27"
28
29custom=$(
30 echo "$custom" | grep -v '^#' | \
31 sed -Ez 's:\n+:,:g;s:,$::;s:^,::'
32)
33
34 #--dry-run \
35./build/bin/llama-quantize \
36 --custom-q "$custom" \
37 --extra-output-tensor iq4_ks \
38 --imatrix /mnt/data/models/ubergarm/Qwen3.8-27B-GGUF/imatrix-Qwen3.8-27B-BF16.dat \
39 /mnt/data/models/ubergarm/Qwen3.8-27B-GGUF-mtp/Qwen3.8-27B-BF16-00001-of-00002.gguf \
40 /mnt/data/models/ubergarm/Qwen3.8-27B-GGUF-mtp/Qwen3.8-27B-MTP-IQ4_KS.gguf \
41 IQ4_KS \
42 161git clone https://github.com/ikawrakow/ik_llama.cpp.git
2cd ik_llama.cpp
3
4cmake -B build -DCMAKE_BUILD_TYPE=Release -DGGML_CUDA=ON -DGGML_CUDA_F16=ON
5cmake --build build --config Release -j $(nproc)
6
7# wget https://huggingface.co/ubergarm/Qwen3.8-27B-GGUF/resolve/main/Qwen3.8-27B-MTP-IQ4_KS.gguf
8model=/mnt/ai/models/ubergarm/Qwen3.8-27B-GGUF/Qwen3.8-27B-MTP-IQ4_KS.gguf
9# wget wget https://huggingface.co/ggml-org/Qwen3.8-27B-GGUF/resolve/main/mmproj-Qwen3.8-27B-Q8_0.gguf
10mmproj=/mnt/ai/models/ubergarm/Qwen3.8-27B-GGUF/mmproj-Qwen3.8-27B-Q8_0.gguf
11
12CUDA_VISIBLE_DEVICES="0" \
13./build/bin/llama-server \
14 --model "$model" \
15 --alias "Qwen3.8-27B" \
16 -c 131072 \
17 -ctk q8_0 -ctv q8_0 \
18 -ctkd q8_0 -ctvd q8_0 \
19 --merge-qkv \
20 -muge \
21 -ngl 99 \
22 -t 1 \
23 -tb 1 \
24 -tm 16 \
25 --host 127.0.0.1 \
26 --port 8080 \
27 --parallel 1 \
28 --jinja \
29 --ctx-checkpoints 32 \
30 -cram 32768 \
31 --spec-type mtp:n_max=4,p_min=0.0 \
32 --no-mmproj-offload \
33 --mmproj "$mmproj"-tm 8 etc if you have less cores.-sm graph.--chat-template-file ./chat_template.jinja \ which may help performance especially with some harness combinations. Pick one of these or bring your own custom chat template as desired.1# wget https://huggingface.co/peculiar-ragdoll/Qwen-Sharp-Chat-Templates/raw/main/chat_template.jinja
2# wget https://huggingface.co/froggeric/Qwen-Fixed-Chat-Templates/raw/main/chat_template.jinja