Views
No views yet
prefill/model.onnxtoken_gen/model.onnx1import onnxruntime as ort
2import numpy as np
3
4# Load models
5prefill_session = ort.InferenceSession('prefill/model.onnx')
6token_gen_session = ort.InferenceSession('token_gen/model.onnx')
7
8# Example inference
9input_ids = np.array([[1, 2, 3, 4, 5]], dtype=np.int64)
10prefill_outputs = prefill_session.run(None, {"input_ids": input_ids})
11token_outputs = token_gen_session.run(None, {"hidden_states": prefill_outputs[0]})1#include <onnxruntime_cxx_api.h>
2
3Ort::Env env(ORT_LOGGING_LEVEL_WARNING, "test");
4Ort::Session session(env, "prefill/model.onnx", Ort::SessionOptions{{nullptr}});
5
6// Run inference
7std::vector<int64_t> input_shape = {1, 5};
8std::vector<int64_t> input_data = {1, 2, 3, 4, 5};
9auto memory_info = Ort::MemoryInfo::CreateCpu(OrtArenaAllocator, OrtMemTypeDefault);
10Ort::Value input_tensor = Ort::Value::CreateTensor<int64_t>(
11 memory_info, input_data.data(), input_data.size(), input_shape.data(), input_shape.size());1# CPU version
2pip install onnxruntime
3
4# GPU version (NVIDIA)
5pip install onnxruntime-gpu
6
7# For mobile deployment
8pip install onnxruntime-mobileprefill/model.onnxtoken_gen/model.onnx1import onnxruntime as ort
2import numpy as np
3
4# モデルを読み込み
5prefill_session = ort.InferenceSession('prefill/model.onnx')
6token_gen_session = ort.InferenceSession('token_gen/model.onnx')
7
8# 推論例
9input_ids = np.array([[1, 2, 3, 4, 5]], dtype=np.int64)
10prefill_outputs = prefill_session.run(None, {"input_ids": input_ids})
11token_outputs = token_gen_session.run(None, {"hidden_states": prefill_outputs[0]})1# CPU版
2pip install onnxruntime
3
4# GPU版（NVIDIA）
5pip install onnxruntime-gpu
6
7# モバイルデプロイ用
8pip install onnxruntime-mobile