import numpy as np
from sentence_transformers import SentenceTransformer
from sentence_transformers.util import cos_sim
import openai

# ./llama.cpp/build/bin/llama-server --models-dir sci-rus-small-GGUF/ --embeddings
openai_client = openai.OpenAI(
    base_url="http://127.0.0.1:8080/v1",
    api_key="sk-no-key-required",
)

# embedding get
def get_embedding(text: str, limit_tokens: int=2048, model="embedding") -> list[float]:
    response = openai_client.embeddings.create(
        input=text[:limit_tokens],
        model=model,
    )
    return response.data[0].embedding

model = SentenceTransformer(
    "mlsa-iai-msu-lab/sci-rus-small",
)

text = """Текст для математики. Пусть у нас есть функция f(x) = x^2 + 3x + 2. Найдите производную этой функции и определите ее критические точки."""

embed1 = model.encode(text)

for quant in ["Q8_0", "F16", "F32"]:
    embed2 = np.array(get_embedding(text, model=f"sci-rus-small-{quant}"), dtype=np.float32)
    print(f"Cosine Similarity with {quant}: {cos_sim(embed1, embed2).item()}")

Output:

Cosine Similarity with Q8_0: 0.9999743103981018
Cosine Similarity with F16: 0.9999986290931702
Cosine Similarity with F32: 0.9999989867210388

Converting

To get the GGUF file, you have to:

Patch llama.cpp/conversion/base.py to add the new model:

python

# (after     res = "roberta-bpe")
        if chkhsh == "e56a55f379193cc03402ddce7b2fff655ca2d784ecfb003235a93341e84925bc":
            # ref: https://huggingface.co/mlsa-iai-msu-lab/sci-rus-small
            res = "roberta-bpe"

And run

bash

./llama.cpp/convert_hf_to_gguf.py sci-rus-small --outtype q8_0 --outfile sci-rus-small-GGUF/sci-rus-small-Q8_0.gguf
./llama.cpp/convert_hf_to_gguf.py sci-rus-small --outtype f16 --outfile sci-rus-small-GGUF/sci-rus-small-F16.gguf
./llama.cpp/convert_hf_to_gguf.py sci-rus-small --outtype f32 --outfile sci-rus-small-GGUF/sci-rus-small-F32.gguf

alexpro100/sci-rus-small-GGUF

Description

Test:

Converting

alexpro100/
sci-rus-small-GGUF