Views
No views yet
| throughput(gpt-oss-120b) | throughput(gpt-oss-120b-eagle3-aimo3) | speedup | concurrency |
|---|---|---|---|
| 776.514 | 1059.43 | 36.40% | 8 |
| 686.717 | 956.431 | 39.30% | 7 |
| 596.596 | 851.647 | 42.80% | 6 |
| 518.76 | 680.951 | 31.30% | 5 |
| 465.702 | 657.682 | 41.20% | 4 |
| 379.48 | 541.304 | 42.60% | 3 |
| 297.553 | 422.232 | 41.90% | 2 |
| 190.023 | 268.132 | 41.10% | 1 |
1TP="${TP:-8}"
2MAX_NUM_SEQS="${MAX_NUM_SEQS:-256}"
3GPU_MEMORY_UTILIZATION="${GPU_MEMORY_UTILIZATION:-0.9}"
4HOST="${HOST:-0.0.0.0}"
5PORT="${PORT:-8000}"
6MAX_LEN="${MAX_LEN:-40960}"
7STREAM_INTERVAL="${STREAM_INTERVAL:-1}"
8
9# ====== speculative config (JSON) ======
10SPECULATIVE_CONFIG='{"method":"eagle3","model":"gpt-oss-120b-eagle3-aimo3","num_speculative_tokens":3,"draft_tensor_parallel_size":1}'
11
12exec python -m vllm.entrypoints.openai.api_server \
13 --model openai/gpt-oss-120b \
14 --served-model-name gpt-oss \
15 --tensor-parallel-size "$TP" \
16 --max-num-seqs "$MAX_NUM_SEQS" \
17 --gpu-memory-utilization "$GPU_MEMORY_UTILIZATION" \
18 --host "$HOST" \
19 --port "$PORT" \
20 --dtype auto \
21 --kv-cache-dtype fp8 \
22 --max-model-len "$MAX_LEN" \
23 --async-scheduling \
24 --stream-interval "$STREAM_INTERVAL" \
25 --speculative-config "$SPECULATIVE_CONFIG"