Views
No views yet
./llama-server \
-m ./gemma-4-E2B-it-UD-Q4_K_XL.gguf \
--host 0.0.0.0 \
--port 8080 \
--temp 1.0 \
--top-p 0.95 \
--top-k 64 \
--min-p 0.0 \
--ctx-size 32000 \
--flash-attn on \
--jinja \
--model-draft ./gemma-4-E2B-it-assistant_gguf/Gemma-4-E2B-It-Assistant-77M-Q8_0.gguf \
--chat-template-file ./gemma-4-E2B-it-assistant_gguf/chat_template.jinja \
--spec-type draft-mtp \
--reasoning on
mtp_bench.py
code_python pred= 192 draft= 225 acc= 114 rate=0.507 tok/s=190.4
code_cpp pred= 192 draft= 207 acc= 121 rate=0.585 tok/s=209.9
explain_concept pred= 192 draft= 239 acc= 110 rate=0.460 tok/s=184.7
summarize pred= 192 draft= 190 acc= 127 rate=0.668 tok/s=227.4
qa_factual pred= 192 draft= 210 acc= 120 rate=0.571 tok/s=208.4
translation pred= 192 draft= 222 acc= 116 rate=0.522 tok/s=198.1
creative_short pred= 192 draft= 219 acc= 116 rate=0.530 tok/s=197.9
stepwise_math pred= 192 draft= 197 acc= 125 rate=0.634 tok/s=220.4
long_code_review pred= 192 draft= 207 acc= 121 rate=0.585 tok/s=204.3
Aggregate: {
"n_requests": 9,
"total_predicted": 1728,
"total_draft": 1916,
"total_draft_accepted": 1070,
"aggregate_accept_rate": 0.5585,
"wall_s_total": 9.14
}
mtp_bench_jp.py
code_python pred= 192 draft= 178 acc= 131 rate=0.736 tok/s=240.6
code_cpp pred= 192 draft= 196 acc= 124 rate=0.633 tok/s=219.3
explain_concept pred= 192 draft= 212 acc= 120 rate=0.566 tok/s=207.4
summarize pred= 192 draft= 180 acc= 130 rate=0.722 tok/s=238.1
qa_factual pred= 192 draft= 191 acc= 127 rate=0.665 tok/s=227.3
translation pred= 192 draft= 221 acc= 117 rate=0.529 tok/s=199.9
creative_short pred= 192 draft= 219 acc= 117 rate=0.534 tok/s=200.4
stepwise_math pred= 192 draft= 195 acc= 125 rate=0.641 tok/s=221.1
long_code_review pred= 192 draft= 226 acc= 115 rate=0.509 tok/s=188.0
Aggregate: {
"n_requests": 9,
"total_predicted": 1728,
"total_draft": 1818,
"total_draft_accepted": 1106,
"aggregate_accept_rate": 0.6084,
"wall_s_total": 8.52
}