Views
No views yet
nsamples=512 and seqlen=2048. Quantization config:bits=4,
group_size=128,
desc_act=False,
damp_percent=0.01,transformers library with integrated GPTQ support:1from transformers import AutoModelForCausalLM, AutoTokenizer, GenerationConfig
2
3model_name = "avoroshilov/openhands-lm-32b-v0.1-GPTQ_4bit-128g"
4
5tokenizer = AutoTokenizer.from_pretrained(model_name)
6quantized_model = AutoModelForCausalLM.from_pretrained(model_name, device_map='cuda')
7
8chat = [{"role": "user", "content": "Why is grass green?"},]
9question_tokens = tokenizer.apply_chat_template(chat, add_generation_prompt=True, return_tensors="pt").to(quantized_model.device)
10answer_tokens = quantized_model.generate(question_tokens, generation_config=GenerationConfig(max_length=2048, ))[0]
11
12print(tokenizer.decode(answer_tokens))