This model uses bitsandbytes 4-bit quantization and must be loaded with BitsAndBytesConfig. Use the included handler.py for correct inference via HuggingFace Inference Endpoints.

Usage

python

from transformers import AutoTokenizer, AutoModelForCausalLM, BitsAndBytesConfig
import torch

bnb_config = BitsAndBytesConfig(
    load_in_4bit=True,
    bnb_4bit_quant_type="nf4",
    bnb_4bit_compute_dtype=torch.bfloat16,
    bnb_4bit_use_double_quant=True,
)

model = AutoModelForCausalLM.from_pretrained(
    "Jeffcck1113/gemma-resume-shortlisting",
    quantization_config=bnb_config,
    device_map="auto"
)
tokenizer = AutoTokenizer.from_pretrained("Jeffcck1113/gemma-resume-shortlisting")