Views
No views yet
1from transformers import AutoTokenizer, AutoModelForCausalLM
2from llmcompressor.transformers import oneshot
3from llmcompressor.modifiers.quantization import QuantizationModifier
4
5MODEL_ID = 'Tarek07/Progenitor-V5-LLaMa-70B'
6
7model = AutoModelForCausalLM.from_pretrained(
8 MODEL_ID, device_map="auto", torch_dtype="auto",
9)
10tokenizer = AutoTokenizer.from_pretrained(MODEL_ID)
11
12# Configure the simple PTQ quantization
13recipe = QuantizationModifier(
14 targets="Linear", scheme="FP8_DYNAMIC", ignore=["lm_head"])
15
16# Apply the quantization algorithm.
17oneshot(model=model, recipe=recipe)
18
19# Save the model.
20SAVE_DIR = MODEL_ID.split("/")[1] + "-FP8-Dynamic"
21model.save_pretrained(SAVE_DIR)
22tokenizer.save_pretrained(SAVE_DIR)recipe.yaml