This is a quantized version of
Qwen/Qwen2.5-0.5B-Instruct using AWQ + FP8_DYNAMIC quantization scheme.
This outperforms the FP8_BLOCK quantization scheme (17.97% strict match) by ~5% while maintaining the same model size.
1from transformers import AutoModelForCausalLM, AutoTokenizer
2
3model = AutoModelForCausalLM.from_pretrained(
4 "rtj1/Qwen2.5-0.5B-AWQ-FP8-Dynamic",
5 device_map="auto"
6)
7tokenizer = AutoTokenizer.from_pretrained("rtj1/Qwen2.5-0.5B-AWQ-FP8-Dynamic")
8
9prompt = "What is 25 * 4?"
10inputs = tokenizer(prompt, return_tensors="pt").to(model.device)
11outputs = model.generate(**inputs, max_new_tokens=100)
12print(tokenizer.decode(outputs[0], skip_special_tokens=True))
1from vllm import LLM, SamplingParams
2
3llm = LLM(model="rtj1/Qwen2.5-0.5B-AWQ-FP8-Dynamic")
4sampling_params = SamplingParams(temperature=0.0, max_tokens=100)
5
6prompts = ["What is 25 * 4?"]
7outputs = llm.generate(prompts, sampling_params)
8
9for output in outputs:
10 print(output.outputs[0].text)
Created using
llm-compressor with the FP8_DYNAMIC scheme:
1from llmcompressor.modifiers.quantization import QuantizationModifier
2from llmcompressor.transformers import oneshot
3
4recipe = QuantizationModifier(
5 targets="Linear",
6 scheme="FP8_DYNAMIC",
7 ignore=["lm_head"]
8)
9
10oneshot(
11 model="Qwen/Qwen2.5-0.5B-Instruct",
12 recipe=recipe,
13 output_dir="Qwen2.5-0.5B-Instruct-awq-fp8-dynamic"
14)
1lm_eval \
2 --model hf \
3 --model_args pretrained=rtj1/Qwen2.5-0.5B-AWQ-FP8-Dynamic,dtype=auto \
4 --tasks gsm8k \
5 --batch_size 16
1@misc{qwen2.5-awq-fp8-dynamic,
2 author = {Tharun Jagarlamudi},
3 title = {Qwen2.5-0.5B-Instruct AWQ + FP8_DYNAMIC},
4 year = {2026},
5 publisher = {HuggingFace},
6 url = {https://huggingface.co/rtj1/Qwen2.5-0.5B-AWQ-FP8-Dynamic}
7}
