import { pipeline } from "https://cdn.jsdelivr.net/npm/@huggingface/transformers@3";

const generator = await pipeline(
  "text-generation",
  "av-codes/Supra-50M-Instruct-ONNX",
  { dtype: "q8" },
);

const messages = [
  { role: "user", content: "Explain what a neural network is in simple terms." },
];

const output = await generator(messages, { max_new_tokens: 256 });
console.log(output[0].generated_text.at(-1).content);

Variants

Variant	File	Size
fp32	`onnx/model.onnx`	199 MB
q8 (int8)	`onnx/model_quantized.onnx`	50 MB

Conversion

Exported with Optimum and quantized with ONNX Runtime:

optimum-cli export onnx --model SupraLabs/Supra-50M-Instruct --task text-generation-with-past ./onnx-export/