lm_eval --model local-completions --tasks gsm8k \
    --model_args model=Qwen/<model>,base_url=http://127.0.0.1:11435/v1/completions,max_length=32768\
    --num_fewshot 5

from awq import AutoAWQForCausalLM
from transformers import AutoTokenizer
from datasets import load_dataset

def load_cosmopedia():
    data = load_dataset('HuggingFaceTB/cosmopedia-100k', split="train")
    data = data.filter(lambda x: x["text_token_length"] >= 2048)
    return [text for text in data["text"]]

model_path = 'Qwen/Qwen3-32B'
model = AutoAWQForCausalLM.from_pretrained(model_path)
tokenizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=True)

quant_path = './Qwen/Qwen3-32B-AWQ-4bit'
quant_config = { "zero_point": True, "q_group_size": 128, "w_bit": 4, "version": "GEMM" }

model.quantize(
    tokenizer,
    quant_config=quant_config,
    calib_data=load_cosmopedia(),
    n_parallel_calib_samples=1,
    max_calib_samples=128,
    max_calib_seq_len=40960
)

model.save_quantized(quant_path)
tokenizer.save_pretrained(quant_path)