Views
No views yet
pip install deepsparse~=1.7 "sparseml[transformers]"~=1.7 "numpy<2".1from datasets import load_dataset
2from sparseml.transformers import (
3 SparseAutoModelForCausalLM,
4 SparseAutoTokenizer,
5 load_dataset,
6 compress,
7)
8
9model = SparseAutoModelForCausalLM.from_pretrained(
10 "meta-llama/Meta-Llama-3-8B-Instruct", device_map="auto"
11)
12tokenizer = SparseAutoTokenizer.from_pretrained("meta-llama/Meta-Llama-3-8B-Instruct")
13dataset = load_dataset("garage-bAInd/Open-Platypus")
14
15
16def format_data(data):
17 instruction = tokenizer.apply_chat_template(
18 [{"role": "user", "content": data["instruction"]}],
19 tokenize=False,
20 add_generation_prompt=True,
21 )
22 return {"text": instruction + data["output"]}
23
24
25dataset = dataset.map(format_data)
26
27recipe = """
28compression_stage:
29 run_type: oneshot
30 oneshot_modifiers:
31 QuantizationModifier:
32 ignore:
33 # These operations don't make sense to quantize
34 - LlamaRotaryEmbedding
35 - LlamaRMSNorm
36 - SiLUActivation
37 - QuantizableMatMul
38 # Skip quantizing the layers with the most sensitive activations
39 - model.layers.1.mlp.down_proj
40 - model.layers.31.mlp.down_proj
41 - model.layers.14.self_attn.q_proj
42 - model.layers.14.self_attn.k_proj
43 - model.layers.14.self_attn.v_proj
44 post_oneshot_calibration: true
45 scheme_overrides:
46 # Enable channelwise quantization for better accuracy
47 Linear:
48 weights:
49 num_bits: 8
50 symmetric: true
51 strategy: channel
52 # For the embeddings, only weight-quantization makes sense
53 Embedding:
54 input_activations: null
55 weights:
56 num_bits: 8
57 symmetric: false
58 SparseGPTModifier:
59 sparsity: 0.5
60 quantize: True
61 targets: ['re:model.layers.\\d*$']
62"""
63
64compress(
65 model=model,
66 tokenizer=tokenizer,
67 dataset=dataset,
68 recipe=recipe,
69 output_dir="./one-shot-checkpoint",
70)