Views
No views yet
Qwen/Qwen3-0.6B produced with llmcompressor.lm_head excluded)1from vllm import LLM, SamplingParams
2from transformers import AutoTokenizer
3
4model_id = "llmat/Qwen3-0.6B-NVFP4"
5number_gpus = 1
6
7sampling_params = SamplingParams(temperature=0.6, top_p=0.9, max_tokens=256)
8
9tokenizer = AutoTokenizer.from_pretrained(model_id)
10
11messages = [
12 {"role": "system", "content": "You are a pirate chatbot who always responds in pirate speak!"},
13 {"role": "user", "content": "Who are you?"},
14]
15
16prompts = tokenizer.apply_chat_template(messages, add_generation_prompt=True, tokenize=False)
17
18llm = LLM(model=model_id, tensor_parallel_size=number_gpus)
19
20outputs = llm.generate(prompts, sampling_params)
21
22generated_text = outputs[0].outputs[0].text
23print(generated_text)