1{
2 "vocab_size": 16000,
3 "min_frequency": 2,
4 "special_tokens": ["<pad>", "<unk>", "<s>", "</s>", "<mask>"],
5 "lowercase": false,
6 "compression": "gzip (level 9)",
7 "checkpointing": true
8}
1from tokenizers import Tokenizer
2from huggingface_hub import hf_hub_download
3
4# Download and load tokenizer
5tokenizer_path = hf_hub_download("Nexuss0781/Ethio-BBPE", "tokenizer.json")
6tokenizer = Tokenizer.from_file(tokenizer_path)
7
8# Encode Amharic text
9text = "ሰላም ለኢዮብ ዘኢነበበ ከንቶ ።"
10encoded = tokenizer.encode(text)
11
12print(f"Tokens: {encoded.tokens}")
13print(f"IDs: {encoded.ids}")
14print(f"Decoded: {tokenizer.decode(encoded.ids)}")
1from tokenizers import Tokenizer
2
3tokenizer = Tokenizer.from_file("models/EthioBBPE/tokenizer.json")
4
5# Test with ancient Ge'ez punctuation
6text = "፠፠፠፠፠፠፠፠፠፠፠፠፠፠፠፠፠፠"
7encoded = tokenizer.encode(text)
8print(f"Encoded {len(text)} chars into {len(encoded.ids)} token(s)")
9# Output: Encoded 16 chars into 1 token(s)
1import gzip
2import json
3from tokenizers import Tokenizer, AddedToken
4
5# Load compressed vocabulary
6with gzip.open('models/EthioBBPE/vocab.json.gz', 'rt', encoding='utf-8') as f:
7 vocab = json.load(f)
8
9print(f"Vocabulary size: {len(vocab)}")
10print(f"Storage saved: ~89.8%")
1from tokenizers import Tokenizer
2
3tokenizer = Tokenizer.from_file("models/EthioBBPE/tokenizer.json")
4
5# Synaxarium text
6synaxarium = """ሰላም ለኢዮብ ዘኢነበበ ከንቶ ። አመ አኀዞ አበቅ ወአመ አህጎለ ጥሪቶ ።"""
7encoded = tokenizer.encode(synaxarium)
8
9print(f"Original: {synaxarium}")
10print(f"Tokens: {encoded.tokens}")
11print(f"Token count: {len(encoded.ids)}")
12print(f"Reconstructed: {tokenizer.decode(encoded.ids)}")
13print(f"Perfect match: {synaxarium == tokenizer.decode(encoded.ids)}")
1texts = [
2 "በመዠመሪያ፡እግዚአብሔር፡ሰማይንና፡ምድርን፡ፈጠረ።",
3 "ወደ ቍስጥንጥንያ አገርም በደረሰች ጊዜ",
4 "፠፠፠፠፠፠፠፠፠፠፠፠፠፠፠፠፠፠"
5]
6
7encodings = tokenizer.encode_batch(texts)
8for i, enc in enumerate(encodings):
9 print(f"Text {i+1}: {len(enc.ids)} tokens")
1test_cases = [
2 ("Ge'ez Punctuation", "፠፠፠፠፠፠፠፠፠፠፠፠፠፠፠፠፠፠"),
3 ("Synaxarium", "ሰላም ለኢዮብ ዘኢነበበ ከንቶ ።"),
4 ("Biblical", "ወደ ቍስጥንጥንያ አገርም በደረሰች ጊዜ")
5]
6
7for name, text in test_cases:
8 encoded = tokenizer.encode(text)
9 decoded = tokenizer.decode(encoded.ids)
10 assert text == decoded, f"{name} failed!"
11 print(f"✅ {name}: Accurate ({len(encoded.ids)} tokens)")
1python scripts/train_tokenizer.py --data_dir ./data --save_compressed
2# Supports: gzip, bz2, lzma
