Views
No views yet
1python3 -m vllm.entrypoints.openai.api_server \
2 --model Irvollo/Behemoth-R1-123B-v2-FP8-Dynamic \
3 --quantization compressed-tensors \
4 --dtype bfloat16 \
5 --max-model-len 32768 \
6 --gpu-memory-utilization 0.95 \
7 --enable-prefix-caching \
8 --trust-remote-code<think> tag prefill:1{
2 "messages": [
3 {"role": "user", "content": "Your question"},
4 {"role": "assistant", "content": "<think>\n"}
5 ],
6 "continue_final_message": true,
7 "add_generation_prompt": false
8}