Views
No views yet
pip install olive-ai
pip install transformers==4.44.2
1
2olive auto-opt --model_name_or_path Your Phi-4 location --output_path Your onnx ouput location --device cpu --provider CPUExecutionProvider --precision int4 --use_model_builder --log_level 1
31
2
3
4import onnxruntime_genai as og
5import numpy as np
6import os
7
8
9model_folder = "Your Phi-4-onnx-cpu-int4 location"
10
11
12model = og.Model(model_folder)
13
14
15tokenizer = og.Tokenizer(model)
16tokenizer_stream = tokenizer.create_stream()
17
18
19search_options = {}
20search_options['max_length'] = 2048
21search_options['past_present_share_buffer'] = False
22
23
24chat_template = "<|user|>\n{input}</s>\n<|assistant|>"
25
26
27text = """I have $20,000 in my savings account, where I receive a 4% profit per year and payments twice a year. Can you please tell me how long it will take for me to become a millionaire? Also, can you please explain the math step by step as if you were explaining it to an uneducated person?"""
28
29
30prompt = f'{chat_template.format(input=text)}'
31
32
33input_tokens = tokenizer.encode(prompt)
34
35
36params = og.GeneratorParams(model)
37
38
39params.set_search_options(**search_options)
40params.input_ids = input_tokens
41
42
43generator = og.Generator(model, params)
44
45
46while not generator.is_done():
47 generator.compute_logits()
48 generator.generate_next_token()
49
50 new_token = generator.get_next_tokens()[0]
51 print(tokenizer_stream.decode(new_token), end='', flush=True)
52
53
54
55