This is plain vanilla exported as onnx. I am researching the offline use of various models, and thought might come in handy for the commmunity Example use #0. download and unzip the package into a subfolder in you project folder

#1. Create a new python environment python -m venv llama_env

#2. activate the environment llama_env\Scripts\activate

#3. Install onnx runtime pip install onnx onnxruntime-gpu

#4. Install transformers and py/torch pip install transformers pip install torch pip install pytorch

#I had to run this when I had a conflic python -m pip install --upgrade pip python -m pip install "numpy<2"

#I use VSCode, so if you'd like: #Install Jupyter and create notebook pip install jupyter code #run vscode

import onnxruntime as ort import torch import numpy as np

onnx_model_path = "payhTo/llama3.1.onnx" session = ort.InferenceSession(onnx_model_path)

for input_meta in session.get_inputs(): print(f"Input Name: {input_meta.name}, Shape: {input_meta.shape}, Type: {input_meta.type}")

for output_meta in session.get_outputs(): print(f"Output Name: {output_meta.name}, Shape: {output_meta.shape}, Type: {output_meta.type}")