Views
No views yet
| Precision | Size | Platform | Use Case |
|---|---|---|---|
| Q4 | ~1.2GB | WebGPU, Server | Recommended for most uses |
| FP16 | ~2.4GB | WebGPU, Server | Higher quality |
| Q8 | ~1.7GB | Server only | Balance of quality and size |
onnx/
├── model.onnx # FP32 model graph
├── model.onnx_data* # FP32 weights
├── model_fp16.onnx # FP16 model graph
├── model_fp16.onnx_data* # FP16 weights
├── model_q4.onnx # Q4 model graph (recommended)
├── model_q4.onnx_data # Q4 weights
├── model_q8.onnx # Q8 model graph
└── model_q8.onnx_data # Q8 weights
* Large models (>2GB) split weights across multiple files:
model.onnx_data, model.onnx_data_1, model.onnx_data_2, etc.
All data files must be in the same directory as the .onnx file.1pip install onnxruntime transformers numpy huggingface_hub
2# or with GPU support:
3pip install onnxruntime-gpu transformers numpy huggingface_hub1import numpy as np
2import onnxruntime as ort
3from huggingface_hub import hf_hub_download
4from transformers import AutoTokenizer
5
6# Download model (Q4 recommended)
7model_id = "LiquidAI/LFM2.5-1.2B-Instruct-ONNX"
8model_path = hf_hub_download(model_id, "onnx/model_q4.onnx")
9
10# Download all data files (handles multiple splits for large models)
11from huggingface_hub import list_repo_files
12for f in list_repo_files(model_id):
13 if f.startswith("onnx/model_q4.onnx_data"):
14 hf_hub_download(model_id, f)
15
16# Load model and tokenizer
17session = ort.InferenceSession(model_path)
18tokenizer = AutoTokenizer.from_pretrained(model_id, trust_remote_code=True)
19
20# Prepare chat input
21messages = [{"role": "user", "content": "What is the capital of France?"}]
22prompt = tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
23input_ids = np.array([tokenizer.encode(prompt, add_special_tokens=False)], dtype=np.int64)
24
25# Initialize KV cache
26ONNX_DTYPE = {"tensor(float)": np.float32, "tensor(float16)": np.float16, "tensor(int64)": np.int64}
27cache = {}
28for inp in session.get_inputs():
29 if inp.name in {"input_ids", "attention_mask", "position_ids"}:
30 continue
31 shape = [d if isinstance(d, int) else 1 for d in inp.shape]
32 for i, d in enumerate(inp.shape):
33 if isinstance(d, str) and "sequence" in d.lower():
34 shape[i] = 0
35 cache[inp.name] = np.zeros(shape, dtype=ONNX_DTYPE.get(inp.type, np.float32))
36
37# Check if model uses position_ids
38input_names = {inp.name for inp in session.get_inputs()}
39use_position_ids = "position_ids" in input_names
40
41# Generate tokens
42seq_len = input_ids.shape[1]
43generated_tokens = []
44
45for step in range(100): # max tokens
46 if step == 0:
47 ids = input_ids
48 pos = np.arange(seq_len, dtype=np.int64).reshape(1, -1)
49 else:
50 ids = np.array([[generated_tokens[-1]]], dtype=np.int64)
51 pos = np.array([[seq_len + len(generated_tokens) - 1]], dtype=np.int64)
52
53 attn_mask = np.ones((1, seq_len + len(generated_tokens)), dtype=np.int64)
54 feed = {"input_ids": ids, "attention_mask": attn_mask, **cache}
55 if use_position_ids:
56 feed["position_ids"] = pos
57
58 outputs = session.run(None, feed)
59 next_token = int(np.argmax(outputs[0][0, -1]))
60 generated_tokens.append(next_token)
61
62 # Update cache
63 for i, out in enumerate(session.get_outputs()[1:], 1):
64 name = out.name.replace("present_conv", "past_conv").replace("present.", "past_key_values.")
65 if name in cache:
66 cache[name] = outputs[i]
67
68 if next_token == tokenizer.eos_token_id:
69 break
70
71print(tokenizer.decode(generated_tokens, skip_special_tokens=True))npm install @huggingface/transformerschrome://flags/#enable-unsafe-webgpu, enable, and restartchrome://gpu for "WebGPU" statusnavigator.gpu.requestAdapter() in DevTools console1import { AutoModelForCausalLM, AutoTokenizer, TextStreamer } from "@huggingface/transformers";
2
3const modelId = "LiquidAI/LFM2.5-1.2B-Instruct-ONNX";
4
5// Load model and tokenizer
6const tokenizer = await AutoTokenizer.from_pretrained(modelId);
7const model = await AutoModelForCausalLM.from_pretrained(modelId, {
8 device: "webgpu",
9 dtype: "q4", // or "fp16"
10});
11
12// Prepare input
13const messages = [{ role: "user", content: "What is the capital of France?" }];
14const input = tokenizer.apply_chat_template(messages, {
15 add_generation_prompt: true,
16 return_dict: true,
17});
18
19// Generate with streaming
20const streamer = new TextStreamer(tokenizer, { skip_prompt: true });
21const output = await model.generate({
22 ...input,
23 max_new_tokens: 256,
24 do_sample: false,
25 streamer,
26});
27
28console.log(tokenizer.decode(output[0], { skip_special_tokens: true }));