pip install "git+https://github.com/huggingface/optimum-intel.git" "torch==2.8" --extra-index-url https://download.pytorch.org/whl/cpu

import torch
from transformers import AutoTokenizer
from optimum.intel import OVModelForCausalLM


def format_instruction(instruction, query, doc):
    prefix = '<|im_start|>system\nJudge whether the Document meets the requirements based on the Query and the Instruct provided. Note that the answer can only be "yes" or "no".<|im_end|>\n<|im_start|>user\n'
    suffix = "<|im_end|>\n<|im_start|>assistant\n<think>\n\n</think>\n\n"
    if instruction is None:
        instruction = (
            "Given a web search query, retrieve relevant passages that answer the query"
        )
    output = f"{prefix}<Instruct>: {instruction}\n<Query>: {query}\n<Document>: {doc}{suffix}"
    return output


model_id = "OpenVINO/Qwen3-Reranker-0.6B-seq-cls-fp16-ov"

model = OVModelForCausalLM.from_pretrained(model_id, use_cache=False, export=False)
tokenizer = AutoTokenizer.from_pretrained(model_id, padding_side="left")
# We recommend enabling flash_attention_2 for better acceleration and memory saving.
# model = AutoModelForSequenceClassification.from_pretrained("tomaarsen/Qwen3-Reranker-0.6B-seq-cls", torch_dtype=torch.float16, attn_implementation="flash_attention_2").cuda().eval()
max_length = 8192

task = "Given a web search query, retrieve relevant passages that answer the query"

queries = [
    "Which planet is known as the Red Planet?",
    "Which planet is known as the Red Planet?",
    "Which planet is known as the Red Planet?",
    "Which planet is known as the Red Planet?",
]

documents = [
    "Venus is often called Earth's twin because of its similar size and proximity.",
    "Mars, known for its reddish appearance, is often referred to as the Red Planet.",
    "Jupiter, the largest planet in our solar system, has a prominent red spot.",
    "Saturn, famous for its rings, is sometimes mistaken for the Red Planet.",
]

pairs = [format_instruction(task, query, doc) for query, doc in zip(queries, documents)]
inputs = tokenizer(
    pairs,
    padding=True,
    truncation=True,
    max_length=max_length,
    return_tensors="pt",
)
logits = model(**inputs).logits.squeeze()
print(logits.tolist())
# [-3.109282970428467, 7.120373725891113, -0.37874650955200195, 3.5416228771209717]

scores = logits.sigmoid()
print(scores.tolist())
# [0.04272596165537834, 0.9991921782493591, 0.406429260969162, 0.9718491435050964]