Evaluated on lm-evaluation-harness "wikitext" task
Wikitext PPL does not guarantee actual accuracy, but helps to check the distortion after quantization.

Codes

python

import os
import torch
from transformers import AutoModelForCausalLM, AutoTokenizer, GPTQConfig

model_id = "meta-llama/Meta-Llama-3-8B"
tokenizer = AutoTokenizer.from_pretrained(model_id)

quantization_config = GPTQConfig(
    bits=4, group_size=64,
    dataset="c4-new",
    tokenizer=tokenizer,
)
model = AutoModelForCausalLM.from_pretrained(
    model_id,
    device_map="auto",
    low_cpu_mem_usage=True,
    quantization_config=quantization_config,
)
model.push_to_hub('yujiepan/Meta-Llama-3-8B-gptq-w4g64')