jburtoft/

Qwen3-8B

import os
os.environ['VLLM_NEURON_FRAMEWORK'] = "neuronx-distributed-inference"

from vllm import LLM, SamplingParams
llm = LLM(
    model="Qwen/Qwen3-8B",
    max_num_seqs=1,
    max_model_len=1024,
    device="neuron",
    tensor_parallel_size=2)
del llm
llm = LLM(
    model="Qwen/Qwen3-8B",
    max_num_seqs=1,
    max_model_len=8192,
    device="neuron",
    tensor_parallel_size=2)
del llm
llm = LLM(
    model="Qwen/Qwen3-8B",
    max_num_seqs=8,
    max_model_len=1024,
    device="neuron",
    tensor_parallel_size=2)
del llm
llm = LLM(
    model="Qwen/Qwen3-8B",
    max_num_seqs=8,
    max_model_len=4096,
    device="neuron",
    tensor_parallel_size=2)
del llm
llm = LLM(
    model="Qwen/Qwen3-8B",
    max_num_seqs=1,
    max_model_len=8192,
    device="neuron",
    override_neuron_config={
    "context_encoding_buckets":[1024, 2048, 4096, 5120, 8000],
    "token_generation_buckets":[8192],
    },
    tensor_parallel_size=2)
del llm