alias	acc,none	acc_stderr,none	acc_norm,none	acc_norm_stderr,none
arc_challenge	0.24488054607508533	0.012566273985131313	0.26535836177474403	0.012902554762313832
arc_easy	0.35395622895622897	0.009812370644174563	0.33291245791245794	0.009669958978395413
boolq	0.4617737003058104	0.008719460098106691
hellaswag	0.3004381597291376	0.00457511609393156	0.32951603266281615	0.004690768393854656
openbookqa	0.194	0.017701827855304598	0.32	0.02088234048876172
piqa	0.5712731229597389	0.0115466944357122	0.5576713819368879	0.011587963545507167
winogrande	0.5248618784530387	0.01403510288362781

Use with mlx

pip install mlx-lm

python

from mlx_lm import load, generate

model, tokenizer = load("kaxap/mlx-gemma-3-12b-2bit")

prompt = "hello"

if tokenizer.chat_template is not None:
    messages = [{"role": "user", "content": prompt}]
    prompt = tokenizer.apply_chat_template(
        messages, add_generation_prompt=True
    )

response = generate(model, tokenizer, prompt=prompt, verbose=True)