This model is
Qwen/Qwen3-0.6B converted to
GGUF format for use with llama.cpp, Ollama, LM Studio, and other compatible inference engines.
1from llama_cpp import Llama
2
3# Load the model
4llm = Llama.from_pretrained(
5 repo_id="codewithdark/Qwen3-0.6B-4bit-gguf",
6 filename="Qwen3-0.6B-4bit-gguf.Q4_K_M.gguf",
7)
8
9# Generate text
10output = llm(
11 "Write a short story about a robot learning to paint:",
12 max_tokens=256,
13 echo=True
14)
15print(output["choices"][0]["text"])
1# Download the model
2huggingface-cli download codewithdark/Qwen3-0.6B-4bit-gguf Qwen3-0.6B-4bit-gguf.Q4_K_M.gguf --local-dir .
3
4# Create Modelfile
5echo 'FROM ./Qwen3-0.6B-4bit-gguf.Q4_K_M.gguf' > Modelfile
6
7# Import to Ollama
8ollama create qwen3-0.6b-4bit-gguf -f Modelfile
9
10# Chat with the model
11ollama run qwen3-0.6b-4bit-gguf
1# Download
2huggingface-cli download codewithdark/Qwen3-0.6B-4bit-gguf Qwen3-0.6B-4bit-gguf.Q4_K_M.gguf --local-dir .
3
4# Run inference
5./llama-cli -m Qwen3-0.6B-4bit-gguf.Q4_K_M.gguf -p "Hello! " -n 128
