Views
No views yet
pip install git+https://github.com/mobiusml/hqq.git;
pip install git+https://github.com/mobiusml/gemlite.git; #to use the gemlite backend
pip install bitblas #to use the bitblas backend1import torch
2device = 'cuda:0'
3backend = 'torchao_int4' #'torchao_int4' #"torchao_int4" (4-bit only) or "bitblas" (4-bit + 2-bit) or "gemlite" (8-bit, 4-bit, 2-bit, 1-bit)
4compute_dtype = torch.bfloat16 if backend=="torchao_int4" else torch.float16
5cache_dir = None
6model_id = 'mobiuslabsgmbh/Qwen2.5-7B-Instruct_4bitgs64_hqq_hf'
7
8is_prequantized = 'hqq_hf' in model_id
9########################################################################
10#Load model
11from transformers import AutoModelForCausalLM, AutoTokenizer, HqqConfig
12
13model = AutoModelForCausalLM.from_pretrained(
14 model_id,
15 torch_dtype=compute_dtype,
16 cache_dir=cache_dir,
17 device_map=device,
18 attn_implementation="sdpa",
19 low_cpu_mem_usage=True,
20)
21
22tokenizer = AutoTokenizer.from_pretrained(model_id, cache_dir=cache_dir)
23
24#Save model before patching
25# model.save_pretrained(saved_quant_model)
26# tokenizer.save_pretrained(saved_quant_model)
27
28#Patching
29from hqq.utils.patching import prepare_for_inference
30prepare_for_inference(model, backend=backend, verbose=True)
31
32#Load GemLite cache
33if(backend == 'gemlite'):
34 import gemlite
35 gemlite.core.GEMLITE_TRITON_RESTRICT_M = True
36 gemlite.core.GemLiteLinear.load_config('/tmp/gemlite_config.json')
37
38########################################################################
39# ##Inference Using a custom hqq generator - currently manual compile breaks with pre-quantized llama models :(
40# from hqq.utils.generation_hf import HFGenerator
41# gen = HFGenerator(model, tokenizer, max_new_tokens=1000, do_sample=True, compile=False).enable_cuda_graph()
42
43# out = gen.generate("Write an essay about large language models.", print_tokens=True)
44
45########################################################################
46#Inference with model,generate()
47from hqq.utils.generation_hf import patch_model_for_compiled_runtime
48
49patch_model_for_compiled_runtime(model, tokenizer)
50
51prompt = "Write an essay about large language models."
52inputs = tokenizer.apply_chat_template([{"role":"user", "content":prompt}], tokenize=True, add_generation_prompt=True, return_tensors="pt", return_dict=True)
53outputs = model.generate(**inputs.to(model.device), max_new_tokens=1000, cache_implementation="static", pad_token_id=tokenizer.pad_token_id)
54#print(tokenizer.decode(outputs[0])
55
56########################################################################
57#Save gemlite cache
58if(backend == 'gemlite'):
59 gemlite.core.GemLiteLinear.cache_config('/tmp/gemlite_config.json') 1from vllm import LLM
2from vllm.sampling_params import SamplingParams
3
4model_id = "mobiuslabsgmbh/Qwen2.5-7B-Instruct_4bitgs64_hqq_hf"
5
6llm = LLM(model=model_id, max_model_len=4096, enable_chunked_prefill=False)
7sampling_params = SamplingParams(temperature=0.8, top_p=0.95, max_tokens=1024)
8outputs = llm.generate(["What is the capital of Germany?"], sampling_params)
9print(outputs[0].outputs[0].text)