Views
No views yet
pip install --upgrade accelerate auto-gptq transformers1from transformers import AutoTokenizer, pipeline
2from auto_gptq import AutoGPTQForCausalLM, BaseQuantizeConfig
3import torch
4
5model_id = "MaziyarPanahi/Starling-LM-7B-beta-GPTQ"
6
7quantize_config = BaseQuantizeConfig(
8 bits=4,
9 group_size=128,
10 desc_act=False
11 )
12
13model = AutoGPTQForCausalLM.from_quantized(
14 model_id,
15 use_safetensors=True,
16 device="cuda:0",
17 quantize_config=quantize_config)
18
19tokenizer = AutoTokenizer.from_pretrained(model_id)
20
21pipe = pipeline(
22 "text-generation",
23 model=model,
24 tokenizer=tokenizer,
25 max_new_tokens=512,
26 temperature=0.7,
27 top_p=0.95,
28 repetition_penalty=1.1
29)
30
31outputs = pipe("What is a large language model?")
32print(outputs[0]["generated_text"])