Views
No views yet
qwen3-reranker is now supported in llama-cpp-python. This project provides a test GGUF file.llama-cpp-python: https://github.com/JamePeng/llama-cpp-python1import llama_cpp
2from llama_cpp.llama_embedding import LlamaEmbedding
3
4# Initialize a Reranking model
5ranker = LlamaEmbedding(
6 model_path="path\to\Qwen3-Reranker-0.6B-Q8_0.gguf",
7 pooling_type=llama_cpp.LLAMA_POOLING_TYPE_RANK, # Crucial for Rerankers!
8 n_gpu_layers=-1,
9 n_ctx=0
10)
11
12query = "What causes Rain?"
13docs = [
14 "Clouds are made of water droplets...", # Relevant
15 "To bake a cake you need flour...", # Irrelevant
16 "Rain is liquid water in the form of droplets..." # Highly Relevant
17]
18
19# Calculate relevance scores
20# Logic: Constructs inputs like "[BOS] query [SEP] doc [EOS]" automatically
21scores = ranker.rank(query, docs)
22
23# Result: List of floats (higher means more relevant)
24print(scores)
25# e.g., [0.0011407170677557588, 5.614783731289208e-05, 0.7173627614974976] -> The 3rd doc is the best match