Views
No views yet
1from datasets import load_dataset
2from gptqmodel import GPTQModel, QuantizeConfig
3
4model_id = "Qwen/Qwen3-Omni-30B-A3B-Instruct"
5quant_path = "Qwen3-Omni-30B-A3B-Instruct-GPTQ-4bit"
6
7calibration_dataset = load_dataset(
8 "allenai/c4",
9 data_files="en/c4-train.00001-of-01024.json.gz",
10 split="train"
11 )
12
13calibration_dataset = calibration_dataset.filter(lambda x: 256 <= len(x["text"]) <= 2048)
14calibration_dataset = calibration_dataset.select(range(1024))["text"]
15
16quant_config = QuantizeConfig(bits=4, group_size=128)
17
18model = GPTQModel.load(model_id, quant_config)
19
20model.quantize(calibration_dataset, batch_size=8)
21
22model.save(quant_path)