Views
No views yet
pip install git+https://github.com/casper-hansen/AutoAWQ.git@f0321eedca887c12680553fc561d176b03b1b9a5 flash_attn1from awq import AutoAWQForCausalLM
2from transformers import AutoTokenizer
3
4model_path = 'models/Phi-3-medium-128k-instruct'
5quant_path = 'models/Phi-3-medium-128k-instruct-awq'
6quant_config = { "zero_point": True, "q_group_size": 128, "w_bit": 4, "version": "GEMM" }
7
8# Load model
9model = AutoAWQForCausalLM.from_pretrained(model_path, **{"device_map": "auto"})
10tokenizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=True)
11
12# Quantize
13model.quantize(tokenizer, quant_config=quant_config)
14
15# Save quantized model
16model.save_quantized(quant_path)
17tokenizer.save_pretrained(quant_path)