Views
No views yet
1from deepsparse import TextGeneration
2model = TextGeneration(model="hf:mgoin/llama-2-7b-gsm8k-pruned60-quant-ds")
3prompt = "James decides to run 3 sprints 3 times a week. He runs 60 meters each sprint. How many total meters does he run a week?"
4print(model(prompt, max_new_tokens=100).generations[0].text)
5### First find the total number of meters James runs in one sprint: 60 meters/sprint * 3 sprints = <<60*3=180>>180 meters\nThen multiply that number by the number of sprints per week to find the total number of meters he runs each week: 180 meters/sprint * 3 sprints/week = <<180*3=540>>540 metersgit clone https://github.com/neuralmagic/sparseml
pip install -e sparseml[transformers]
python sparseml/src/sparseml/transformers/sparsification/obcq/obcq.py /path/to/llama-2-7b_pruned60-gsm8k open_platypus --recipe llama-gsm8k-60p-skip5.yaml --save True
python sparseml/src/sparseml/transformers/sparsification/obcq/export.py --task text-generation --model_path obcq_deployment --sequence_length 512
cp deployment/model.onnx deployment/model-orig.onnx
python onnx_kv_inject.py --input-file deployment/model-orig.onnx --output-file deployment/model.onnxllama-gsm8k-60p-skip5.yaml1test_stage:
2 obcq_modifiers:
3 SparseGPTModifier:
4 sparsity: 0.0
5 block_size: 128
6 sequential_update: False
7 quantize:
8 QuantizationModifier:
9 ignore:
10 - LlamaRotaryEmbedding
11 - LlamaRMSNorm
12 - SiLUActivation
13 - model.layers.1.mlp.down_proj
14 - model.layers.30.mlp.down_proj
15 - model.layers.31.mlp.down_proj
16 - model.layers.18.mlp.down_proj
17 - model.layers.29.mlp.down_proj
18 post_oneshot_calibration: True
19 scheme_overrides:
20 Embedding:
21 input_activations: null
22 weights:
23 num_bits: 8
24 symmetric: False
25 percdamp: 0.01
26 prunen: 0
27 prunem: 0
28 targets: [
29 "model.layers.0",
30 "model.layers.1",
31 "model.layers.2",
32 "model.layers.3",
33 "model.layers.4",
34 "model.layers.5",
35 "model.layers.6",
36 "model.layers.7",
37 "model.layers.8",
38 "model.layers.9",
39 "model.layers.10",
40 "model.layers.11",
41 "model.layers.12",
42 "model.layers.13",
43 "model.layers.14",
44 "model.layers.15",
45 "model.layers.16",
46 "model.layers.17",
47 "model.layers.18",
48 "model.layers.19",
49 "model.layers.20",
50 "model.layers.21",
51 "model.layers.22",
52 "model.layers.23",
53 "model.layers.24",
54 "model.layers.25",
55 "model.layers.26",
56 "model.layers.27",
57 "model.layers.28",
58 "model.layers.29",
59 "model.layers.30",
60 "model.layers.31",
61 ]
62 target_ids: ["attention_mask", "position_ids"]