Views
No views yet
pip install transformers accelerate tiktoken einops scipy transformers_stream_generator==0.0.4 peft deepspeed tiktoken autoawq1from transformers import AutoModelForCausalLM, AutoTokenizer,AwqConfig, AutoConfig
2from transformers.generation import GenerationConfig
3from awq import AutoAWQForCausalLM
4
5# quant_config = {"zero_point": True, "q_group_size": 128, "w_bit": 4, "version":"GEMM"}
6#quantization_config = AwqConfig(
7# bits=quant_config["w_bit"],
8# group_size=quant_config["q_group_size"],
9# zero_point=quant_config["zero_point"],
10# version=quant_config["version"].lower(),
11#).to_dict()
12
13
14tokenizer_abel = AutoTokenizer.from_pretrained("AbelKidane/abelqwen4", trust_remote_code=True)
15model_abel = AutoAWQForCausalLM.from_quantized("AbelKidane/abelqwen4", fuse_layers=True, safetensors=True, trust_remote_code=True)
16
17text = "The capital city of "
18inputs = tokenizer_abel(text, return_tensors="pt").to(0)
19
20out = model_abel.generate(**inputs, max_new_tokens=5)
21print(tokenizer_abel.decode(out[0], skip_special_tokens=True))
22