1# Optional: enable schedule overlapping (experimental, may not be stable)
2# export SGLANG_ENABLE_SPEC_V2=1
3# export SGLANG_ENABLE_DFLASH_SPEC_V2=1
4# export SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1
5
6python -m sglang.launch_server \
7 --model-path Qwen/Qwen3-4B \
8 --speculative-algorithm DFLASH \
9 --speculative-draft-model-path z-lab/Qwen3-4B-DFlash-b16 \
10 --tp-size 1 \
11 --dtype bfloat16 \
12 --attention-backend fa3 \
13 --mem-fraction-static 0.75 \
14 --trust-remote-code
1from openai import OpenAI
2
3client = OpenAI(base_url="http://localhost:30000/v1", api_key="EMPTY")
4
5response = client.chat.completions.create(
6 model="Qwen/Qwen3-4B",
7 messages=[{"role": "user", "content": "Write a quicksort in Python."}],
8 max_tokens=2048,
9 temperature=0.0,
10 extra_body={
11 "chat_template_kwargs": {"enable_thinking": False},
12 },
13)
14print(response.choices[0].message.content)
1uv pip install vllm
2uv pip install -U vllm --torch-backend=auto --extra-index-url https://wheels.vllm.ai/nightly
1vllm serve Qwen/Qwen3-4B \
2 --speculative-config '{"method": "dflash", "model": "z-lab/Qwen3-4B-DFlash-b16", "num_speculative_tokens": 15}' \
3 --attention-backend flash_attn \
4 --max-num-batched-tokens 32768
1from openai import OpenAI
2
3client = OpenAI(base_url="http://localhost:8000/v1", api_key="EMPTY")
4
5response = client.chat.completions.create(
6 model="Qwen/Qwen3-4B",
7 messages=[{"role": "user", "content": "Write a quicksort in Python."}],
8 max_tokens=2048,
9 temperature=0.0,
10 chat_template_kwargs: {"enable_thinking": False},
11)
12print(response.choices[0].message.content)
The following example demonstrates how to load the DFlash drafter and the Qwen3-8B target model to perform speculative decoding.
1from transformers import AutoModel, AutoModelForCausalLM, AutoTokenizer
2
3# 1. Load the DFlash Draft Model
4# Note: trust_remote_code=True is required for DFlash. We recommend run on one GPU currently.
5model = AutoModel.from_pretrained(
6 "z-lab/Qwen3-4B-DFlash-b16",
7 trust_remote_code=True,
8 dtype="auto",
9 device_map="cuda:0"
10).eval()
11
12# 2. Load the Target Model
13target = AutoModelForCausalLM.from_pretrained(
14 "Qwen/Qwen3-4B",
15 dtype="auto",
16 device_map="cuda:0"
17).eval()
18
19# 3. Load Tokenizer and Prepare Input
20tokenizer = AutoTokenizer.from_pretrained("Qwen/Qwen3-4B")
21prompt = "How many positive whole-number divisors does 196 have?"
22messages = [
23 {"role": "user", "content": prompt}
24]
25# Note: this draft model is used for thinking mode disabled
26text = tokenizer.apply_chat_template(
27 messages,
28 tokenize=False,
29 add_generation_prompt=True,
30 enable_thinking=False
31)
32model_inputs = tokenizer([text], return_tensors="pt").to(model.device)
33
34# 4. Run Speculative Decoding
35# The 'spec_generate' function is a custom method provided by the DFlash model
36generate_ids = model.spec_generate(
37 input_ids=model_inputs["input_ids"],
38 max_new_tokens=2048,
39 temperature=0.0,
40 target=target,
41 stop_token_ids=[tokenizer.eos_token_id]
42)
43
44print(tokenizer.decode(generate_ids[0], skip_special_tokens=True))
DFlash achieves up to
6.17x lossless acceleration for
Qwen3-8B, making it nearly
2.5x faster than the state-of-the-art speculative decoding method EAGLE-3. Check out our
GitHub repository to see how to reproduce the results.
If you find DFlash useful for your research or applications, please cite our project.
1@misc{chen2026dflash,
2 title = {DFlash: Block Diffusion for Flash Speculative Decoding},
3 author = {Chen, Jian and Liang, Yesheng and Liu, Zhijian},
4 year = {2026},
5 eprint = {2602.06036},
6 archivePrefix = {arXiv},
7 primaryClass = {cs.CL},
8 url = {https://arxiv.org/abs/2602.06036}
9}
