Views
No views yet
Qwen2.5-0.5B-Instruct model.1from llama_cpp import Llama
2
3llm = Llama(
4 model_path="Qwen2.5-0.5B-Instruct-Q4_0.gguf",
5 n_ctx=32768,
6 verbose=False,
7 chat_format="chatml"
8)
9
10user_input = "Hello"
11
12output = llm.create_chat_completion(
13 messages=[
14 {
15 "role": "system",
16 "content": "You are Qwen, created by Alibaba Cloud. You are a helpful assistant.",
17 },
18 {"role": "user", "content": user_input},
19 ],
20 temperature=0.7,
21 max_tokens=500,
22 stop=["\nUser"],
23)
24
25print(output["choices"][0]["message"]["content"])