from vllm import LLM, SamplingParams
from transformers import AutoTokenizer

model_id = "selimaktas/Qwen3-14B-FP8-MinMax"
tokenizer = AutoTokenizer.from_pretrained(model_id)

# Initialize vLLM
llm = LLM(model=model_id, tensor_parallel_size=1)

# Generate
messages = [{"role": "user", "content": "Explain quantum computing"}]
prompts = tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
sampling_params = SamplingParams(temperature=0.6, top_p=0.95, max_tokens=256)

outputs = llm.generate(prompts, sampling_params)
print(outputs[0].outputs[0].text)

Benchmarking

bash

lm_eval --model vllm \
  --model_args pretrained=selimaktas/Qwen3-14B-FP8-MinMax,dtype=auto,add_bos_token=True \
  --tasks gsm8k --num_fewshot 5 --batch_size auto

Requirements

vLLM >= 0.5.0
Transformers >= 4.51.0
CUDA-capable GPU with FP8 support (H100, L40S, etc.)