Views
No views yet
huggingface-hub Python library.pip3 install huggingface-hub>=0.17.1huggingface-cli download Esperanto/phi3-mini-4k-instruct-kvc-AWQ-int4-onnx --local-dir phi3-mini-4k-instruct-kvc-AWQ-int4-onnx --local-dir-use-symlinks Falsehuggingface-cli, please see: HF -> Hub Python Library -> Download files -> Download from the CLI.1pip3 install onnx==1.16.1
2pip3 install onnxruntime==1.17.11import numpy as np
2import onnxruntime
3import onnx
4from transformers import AutoTokenizer
5def generate_text(model_path, prompt, tokenizer, max_gen_tokens, total_sequence, window, context):
6 model = onnx.load(model_path)
7 #we create the inputs for the first iteration
8 input_tensor = tokenizer(prompt, return_tensors="pt")
9 prompt_size = len(input_tensor['input_ids'][0])
10 actual_input = input_tensor['input_ids']
11 if prompt_size < window:
12 actual_input = np.concatenate((tokenizer.bos_token_id*np.ones([1, window - prompt_size], dtype = 'int64'),
13 actual_input), axis=1)
14 if prompt_size + max_gen_tokens > total_sequence:
15 print("ERROR: Longer total sequence is needed!")
16 return
17 first_attention = np.concatenate((np.zeros([1, total_sequence - window], dtype = 'int64'),
18 np.ones((1, window), dtype = 'int64')), axis=1)
19 max_gen_tokens += prompt_size #we need to generate on top of parsing the prompt
20 inputs_names =[node.name for node in model.graph.input]
21 output_names =[node.name for node in model.graph.output]
22 n_heads = 32 #gqa-heads of the kvc
23 inputs_dict = {}
24 inputs_dict['input_ids'] = actual_input[:, :window].reshape(1, window).numpy()
25 inputs_dict['attention_mask'] = first_attention
26 index_pos = sum(first_attention[0])
27 inputs_dict['position_ids'] = np.concatenate((np.zeros([1, total_sequence - index_pos], dtype = 'int64'), np.arange(index_pos, dtype = 'int64').reshape(1, index_pos)), axis=1)
28 inputs_dict['tree_attention'] = np.triu(-65504*np.ones(total_sequence), k= 1).astype('float16').reshape(1, 1, total_sequence, total_sequence)
29 for name in inputs_names:
30 if name == 'input_ids' or name == 'attention_mask' or name == 'position_ids' or name == 'tree_attention': continue
31 inputs_dict[name] = np.zeros([1, n_heads, context-window, 96], dtype="float16")
32 index = 0
33 new_token = np.array([10])
34 next_index = window
35 old_j = 0
36 total_input = actual_input.numpy()
37 rt_session = onnxruntime.InferenceSession(model_path)
38 ## We run the inferences
39 while next_index < max_gen_tokens:
40 if new_token.any() == tokenizer.eos_token_id:
41 break
42 #inference
43 output = rt_session.run(output_names, inputs_dict)
44 outs_dictionary = {name: content for (name, content) in zip (output_names, output)}
45 #we prepare the inputs for the next inference
46 for name in inputs_names:
47 if name == 'input_ids':
48 old_j = next_index
49 if next_index < prompt_size:
50 if prompt_size - next_index >= window: next_index += window
51 else: next_index = prompt_size
52 j = next_index - window
53 else:
54 next_index +=1
55 j = next_index - window
56 new_token = outs_dictionary['logits'].argmax(-1).reshape(1, window)
57 total_input = np.concatenate((total_input, new_token[: , -1:]), axis = 1)
58 inputs_dict['input_ids']= total_input[:, j:next_index].reshape(1, window)
59 elif name == 'attention_mask':
60 inputs_dict['attention_mask'] = np.concatenate((np.zeros((1, total_sequence-next_index), dtype = 'int64'), np.ones((1, next_index), dtype = 'int64')), axis=1)
61 elif name == 'position_ids':
62 inputs_dict['position_ids'] = np.concatenate((np.zeros([1, total_sequence - next_index], dtype = 'int64'), np.arange(next_index, dtype = 'int64').reshape(1, next_index)), axis=1)
63 elif name == 'tree_attention': continue
64 else:
65 old_name = name.replace("past_key_values", "present")
66 inputs_dict[name] = outs_dictionary[old_name][:, :, next_index-old_j:context-window+(next_index - old_j), :]
67 answer = tokenizer.decode(total_input[0], skip_special_tokens=True, clean_up_tokenization_spaces=False)
68 return answer1tokenizer = AutoTokenizer.from_pretrained("Esperanto/phi3-mini-4k-instruct-kvc-AWQ-int4-onnx")
2model_path = "phi3-mini-4k-instruct-kvc-AWQ-int4-onnx/model.onnx"
3max_gen_tokens = 20 #number of tokens we want tog eneral
4total_sequence = 128 #total sequence_length
5context = 1024 #the context to extend the kvc
6window = 16 #number of tokens we want to parse at the time
7messages = [
8 {"role": "system", "content": "You are a pirate chatbot who always responds in pirate speak!"},
9 {"role": "user", "content": "Who are you?"},
10]
11prompt = tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
12generated = generate_text(model_path, prompt, tokenizer, max_gen_tokens, total_sequence, window, context)
13print(generated)