Views
No views yet
pip install nm-vllm[sparse]1from vllm import LLM, SamplingParams
2
3model = LLM("nm-testing/TinyLlama-1.1B-Chat-v1.0-pruned2.4", sparsity="semi_structured_sparse_w16a16")
4prompt = "How to make banana bread?"
5formatted_prompt = f"<|im_start|>user\n{prompt}<|im_end|>\n<|im_start|>assistant\n"
6
7sampling_params = SamplingParams(max_tokens=100,temperature=0,repetition_penalty=1.3)
8outputs = model.generate(formatted_prompt, sampling_params=sampling_params)
9print(outputs[0].outputs[0].text)
10"""
11Banana bread is a delicious dessert that is made with bananas. Here is how to make banana bread:
12
131. Firstly, you need to cut bananas into small pieces.
142. Then, you need to slice the bananas into small pieces
15"""<|im_start|>user
{prompt}<|im_end|>
<|im_start|>assistant
recipe.yaml in this repo and follow the instructions below.1git clone https://github.com/neuralmagic/sparseml
2pip install -e "sparseml[transformers]"1import sparseml.transformers
2
3original_model_name = "TinyLlama/TinyLlama-1.1B-Chat-v1.0"
4calibration_dataset = "open_platypus"
5output_directory = "output/"
6
7recipe = """
8test_stage:
9 obcq_modifiers:
10 SparseGPTModifier:
11 sparsity: 0.5
12 sequential_update: true
13 mask_structure: '2:4'
14 targets: ['re:model.layers.\d*$']
15"""
16
17# Apply SparseGPT to the model
18sparseml.transformers.oneshot(
19 model=original_model_name,
20 dataset=calibration_dataset,
21 recipe=recipe,
22 output_dir=output_directory,
23)