This repository contains a Byte-Pair Encoding (BPE) tokenizer with a 16,384 token vocabulary.
This tokenizer was trained using a custom Python script and is saved as a Python pickle file (.pkl). Because it is a custom implementation, you must use the provided class code to load it, rather than transformers.AutoTokenizer.
To use this tokenizer, you need the Python code for the tokenizer class itself, and the saved state from the .pkl file.
1# Copy the entire code block below into your Python script
2# --------------------------------------------------------------------------
3
4from __future__ import annotations
5import collections
6import regex
7import pickle
8from huggingface_hub import hf_hub_download
9
10# Note: The bpe_train function is not needed for inference, only bpe_encode.
11def bpe_encode(
12 mergeable_ranks: dict[bytes, int], input: bytes, demo: bool = False
13) -> list[int]:
14 parts = [bytes([b]) for b in input]
15 while True:
16 min_idx, min_rank = None, None
17 for i, pair in enumerate(zip(parts[:-1], parts[1:])):
18 rank = mergeable_ranks.get(pair[0] + pair[1])
19 if rank is not None and (min_rank is None or rank < min_rank):
20 min_idx, min_rank = i, rank
21 if min_rank is None:
22 break
23 assert min_idx is not None
24 parts = parts[:min_idx] + [parts[min_idx] + parts[min_idx + 1]] + parts[min_idx + 2 :]
25 return [mergeable_ranks[part] for part in parts]
26
27class SimpleBytePairEncoding:
28 def __init__(self, *, pat_str: str, mergeable_ranks: dict[bytes, int]) -> None:
29 """Creates an Encoding object."""
30 self.pat_str = pat_str
31 self.mergeable_ranks = mergeable_ranks
32 self._decoder = {token: token_bytes for token_bytes, token in mergeable_ranks.items()}
33 self._pat = regex.compile(pat_str)
34
35 def encode(self, text: str, demo: bool = False) -> list[int]:
36 words = self._pat.findall(text)
37 tokens = []
38 for word in words:
39 word_bytes = word.encode("utf-8")
40 word_tokens = bpe_encode(self.mergeable_ranks, word_bytes, demo=demo)
41 tokens.extend(word_tokens)
42 return tokens
43
44 def decode_bytes(self, tokens: list[int]) -> bytes:
45 return b"".join(self._decoder[token] for token in tokens)
46
47 def decode(self, tokens: list[int]) -> str:
48 return self.decode_bytes(tokens).decode("utf-8", errors="replace")
49
50 def decode_tokens_bytes(self, tokens: list[int]) -> list[bytes]:
51 return [self._decoder[token] for token in tokens]
52
53 @property
54 def vocab_size(self) -> int:
55 """Return the vocabulary size."""
56 return len(self.mergeable_ranks)
57
58 @staticmethod
59 def from_hub(repo_id: str, filename: str = "tokenizer.pkl"):
60 """Loads the tokenizer from the Hugging Face Hub."""
61 local_path = hf_hub_download(repo_id=repo_id, filename=filename)
62 with open(local_path, 'rb') as f:
63 tokenizer_data = pickle.load(f)
64 return SimpleBytePairEncoding(
65 pat_str=tokenizer_data["pat_str"],
66 mergeable_ranks=tokenizer_data["mergeable_ranks"]
67 )
68
69# --------------------------------------------------------------------------
70
71
72# --- Now, you can load and use the tokenizer ---
73repo_id = "vukrosic/essential-web-16k-tokenizer"
74file_name = "bpe_tokenizer_16k_n1000000.pkl" # The name of the .pkl file in the repo
75
76# Load the tokenizer directly from the Hub
77enc = SimpleBytePairEncoding.from_hub(repo_id, filename=file_name)
78
79# --- Test the tokenizer ---
80text = "Hello, world! This is a test of the 16K BPE tokenizer."
81tokens = enc.encode(text)
82decoded_text = enc.decode(tokens)
83
84print(f"Vocabulary size: {enc.vocab_size:,}")
85print(f"Original text: '{text}'")
86print(f"Tokens: {tokens}")
87print(f"Number of tokens: {len(tokens)}")
88print(f"Decoded text: '{decoded_text}'")
89
90assert text == decoded_text
91print("✅ Roundtrip successful!")
92
