This is a quantized FP32 model based on X86 CPU FunctionGemma-270m. You can deploy it on your CPU devices.
Note: This is unoffical version,just for test and dev.
1
2pip install onnxruntime-genai
3
1
2import onnxruntime_genai as og
3import argparse
4import os
5import json
6import time
7
8model_folder = {Your FunctionGemma-270m-ONNX-CPU Path}
9
10config = og.Config(model_folder)
11
12model = og.Model(config)
13
14tokenizer = og.Tokenizer(model)
15tokenizer_stream = tokenizer.create_stream()
16
17def get_current_weather(location: str, unit: str = "celsius"):
18 """
19 Get the current temperature at a location.
20
21 Args:
22 location: The location to get the temperature for.
23 unit: The unit to return the temperature in. (choices: ["celsius", "fahrenheit"])
24 """
25 return 22.0
26
27import json
28
29messages_list = [
30 {"role": "developer", "content": "You are a model that can do function calling with the following functions<start_function_declaration>declaration:get_current_weather{description:<escape>Gets the current weather in a given location.<escape>,parameters:{properties:{location:{description:<escape>The city and state, e.g. \"San Francisco, CA\" or \"Tokyo, JP\"<escape>,type:<escape>STRING<escape>},unit:{description:<escape>The unit to return the temperature in.<escape>,enum:[<escape>celsius<escape>,<escape>fahrenheit<escape>],type:<escape>STRING<escape>}},required:[<escape>location<escape>],type:<escape>OBJECT<escape>}}<end_function_declaration>"},
31 {"role": "user", "content": "Hey, what's the weather in Tokyo right now?"},
32]
33messages = json.dumps(messages_list)
34
35prompt = tokenizer.apply_chat_template(messages=messages, add_generation_prompt=True)
36print(prompt)
37
38params = og.GeneratorParams(model)
39generator = og.Generator(model, params)
40
41input_tokens = tokenizer.encode(prompt)
42generator.append_tokens(input_tokens)
43
44while not generator.is_done():
45 generator.generate_next_token()
46 new_token = generator.get_next_tokens()[0]
47 print(tokenizer_stream.decode(new_token), end='', flush=True)
48
49
