Views
No views yet
#This model is deprecated and requires older versions
pip install hqq==0.1.8
pip install transformers==4.46.01model_id = 'mobiuslabsgmbh/Llama-2-13b-chat-hf-4bit_g64-HQQ'
2
3from hqq.engine.hf import HQQModelForCausalLM, AutoTokenizer
4tokenizer = AutoTokenizer.from_pretrained(model_id)
5model = HQQModelForCausalLM.from_quantized(model_id)1model_id = 'mobiuslabsgmbh/Llama-2-13b-chat-hf-4bit_g64-HQQ'
2
3from hqq.engine.hf import HQQModelForCausalLM, AutoTokenizer
4tokenizer = AutoTokenizer.from_pretrained(model_id)
5model = HQQModelForCausalLM.from_quantized(model_id)
6
7##########################################################################################################
8import transformers
9from threading import Thread
10
11from sys import stdout
12def print_flush(data):
13 stdout.write("\r" + data)
14 stdout.flush()
15
16#Adapted from https://huggingface.co/spaces/huggingface-projects/llama-2-7b-chat/blob/main/app.py
17def process_conversation(chat):
18 system_prompt = chat['system_prompt']
19 chat_history = chat['chat_history']
20 message = chat['message']
21
22 conversation = []
23 if system_prompt:
24 conversation.append({"role": "system", "content": system_prompt})
25 for user, assistant in chat_history:
26 conversation.extend([{"role": "user", "content": user}, {"role": "assistant", "content": assistant}])
27 conversation.append({"role": "user", "content": message})
28
29 return tokenizer.apply_chat_template(conversation, return_tensors="pt").to('cuda')
30
31def chat_processor(chat, max_new_tokens=100, do_sample=True):
32 tokenizer.use_default_system_prompt = False
33 streamer = transformers.TextIteratorStreamer(tokenizer, timeout=10.0, skip_prompt=True, skip_special_tokens=True)
34
35 generate_params = dict(
36 {"input_ids": process_conversation(chat)},
37 streamer=streamer,
38 max_new_tokens=max_new_tokens,
39 do_sample=do_sample,
40 top_p=0.90,
41 top_k=50,
42 temperature= 0.6,
43 num_beams=1,
44 repetition_penalty=1.2,
45 )
46
47 t = Thread(target=model.generate, kwargs=generate_params)
48 t.start()
49
50 outputs = []
51 for text in streamer:
52 outputs.append(text)
53 print_flush("".join(outputs))
54
55 return outputs
56
57###################################################################################################
58
59outputs = chat_processor({'system_prompt':"You are a helpful assistant.",
60 'chat_history':[],
61 'message':"How can I build a car?"
62 },
63 max_new_tokens=1000, do_sample=False)