Views
No views yet
llmcompressor with GPTQ method (W4A16).vllm serve "superjob/Qwen3-4B-Instruct-2507-GPTQ-Int4" --quantization compressed-tensors --dtype bfloat161from vllm import LLM, SamplingParams
2
3llm = LLM(model="superjob/Qwen3-4B-Instruct-2507-GPTQ-Int4", quantization="compressed-tensors")
4prompts = ["Hello, how are you?"]
5sampling_params = SamplingParams(temperature=0.7, top_p=0.8)
6
7outputs = llm.generate(prompts, sampling_params)
8
9for output in outputs:
10 generated_text = output.outputs[0].text
11 print(f"Generated text: {generated_text!r}")