Full INT8-quantized ONNX of
codefuse-ai/F2LLM-v2-0.6B. Smallest total file size (~600 MB) — both attention weights and the embedding table are quantized.
Quantizing the Gather (embedding table) reduces the ~600 MB FP32 table to ~150 MB. This trades slight multilingual token quality for the smallest possible model footprint.
1import onnxruntime as ort
2import numpy as np
3from tokenizers import Tokenizer
4
5tokenizer = Tokenizer.from_file("tokenizer.json")
6tokenizer.enable_padding(pad_id=0, direction="right")
7tokenizer.enable_truncation(max_length=512)
8
9session = ort.InferenceSession("model.int8_full.onnx", providers=["CPUExecutionProvider"])
10
11texts = ["semantic search example", "another sentence"]
12enc = tokenizer.encode_batch(texts)
13ids = np.array([e.ids for e in enc], dtype=np.int64)
14mask = np.array([e.attention_mask for e in enc], dtype=np.int64)
15
16lhs = session.run(None, {"input_ids": ids, "attention_mask": mask})[0]
17seq_lens = mask.sum(axis=1) - 1
18embeddings = lhs[np.arange(len(texts)), seq_lens]
19norms = np.linalg.norm(embeddings, axis=1, keepdims=True)
20embeddings = embeddings / np.maximum(norms, 1e-8)
21print(embeddings.shape) # (2, 1024)
1@misc{f2llm-v2,
2 title={F2LLM-v2: Inclusive, Performant, and Efficient Embeddings for a Multilingual World},
3 author={Ziyin Zhang and Zihan Liao and Hang Yu and Peng Di and Rui Wang},
4 year={2026},
5 eprint={2603.19223},
6 archivePrefix={arXiv},
7 primaryClass={cs.CL},
8 url={https://arxiv.org/abs/2603.19223},
9}
