from llama_cpp import Llama
from transformers import AutoTokenizer

llm = Llama(model_path="Qwen3-8B-Q5_K_M.gguf", n_ctx=8192)
tokenizer = AutoTokenizer.from_pretrained("Qwen/Qwen3-8B")

messages = [{"role": "user", "content": "Hello!"}]
prompt = tokenizer.apply_chat_template(
    messages, tokenize=False, add_generation_prompt=True, enable_thinking=False
)
out = llm(prompt, max_tokens=256, stop=[tokenizer.eos_token])
print(out["choices"][0]["text"])

Mukilan06/Qwen3-8B-Q5_K_M

Qwen3-8B-Q5_K_M

Usage

Mukilan06/
Qwen3-8B-Q5_K_M