Views
No views yet
| Feature | Value |
|---|---|
| Algorithm | Byte-Pair Encoding (BPE) |
| Vocabulary Size | 32,000 tokens |
| Min Frequency | 2 occurrences |
| Script Support | Ge'ez (U+1200-U+137F) |
| Compression Ratio | ~3.2x average |
| OOV Handling | Excellent (subword fallback) |
1{
2 "<unk>": 0, # Unknown token
3 "<s>": 1, # Beginning of sequence (BOS)
4 "</s>": 2, # End of sequence (EOS)
5 "<pad>": 3, # Padding token
6 "<mask>": 4, # Mask token (for MLM)
7}1from transformers import PreTrainedTokenizerFast
2
3# Load the tokenizer
4tokenizer = PreTrainedTokenizerFast.from_pretrained("./hf_tokenizer")
5
6# Tokenize Tigrinya text
7text = "ሰላም! ከመይ ኣሎኻ? ሎምስ እንታይ ገይርካ?"
8tokens = tokenizer.encode(text)
9print(f"Token IDs: {tokens}")
10
11# Get token pieces
12pieces = tokenizer.tokenize(text)
13print(f"Tokens: {pieces}")
14
15# Decode back to text
16decoded = tokenizer.decode(tokens)
17print(f"Decoded: {decoded}")1from transformers import (
2 AutoTokenizer,
3 AutoModelForCausalLM,
4 TrainingArguments,
5 Trainer
6)
7
8# Load tokenizer
9tokenizer = AutoTokenizer.from_pretrained("./hf_tokenizer")
10
11# Initialize model with correct vocab size
12vocab_size = len(tokenizer) # 32,000
13config = AutoConfig.from_pretrained("gpt2")
14config.vocab_size = vocab_size
15model = AutoModelForCausalLM.from_config(config)
16
17# Tokenization function for datasets
18def tokenize_function(examples):
19 return tokenizer(
20 examples["text"],
21 padding=True,
22 truncation=True,
23 max_length=512,
24 return_tensors="pt"
25 )1# Process multiple texts efficiently
2texts = [
3 "ሰላም ኣለኻ",
4 "ከመይ ዘሎኻ?",
5 "ሎሚ እንታይ ገይርካ?"
6]
7
8# Batch tokenization
9batch = tokenizer(
10 texts,
11 padding=True,
12 truncation=True,
13 return_tensors="pt"
14)
15
16print(f"Input IDs shape: {batch['input_ids'].shape}")
17print(f"Attention mask shape: {batch['attention_mask'].shape}")Original: ሰላም! ከመይ ኣሎኻ?
Tokens: ['<s>', 'ሰ', 'ላም', '!', '▁ከ', 'መይ', '▁ኣ', 'ሎ', 'ኻ', '?', '</s>']
Token IDs: [1, 234, 567, 12, 890, 123, 456, 789, 321, 13, 2]
Token count: 11Original: ሎሚ ጽቡቕ መዓልቲ እዩ። ናብ ቤት ትምህርቲ ክኸይድ እየ።
Tokens: ['<s>', 'ሎ', 'ሚ', '▁ጽ', 'ቡ', 'ቕ', '▁መ', 'ዓል', 'ቲ', '▁እዩ', '።', '▁ናብ', '▁ቤት', '▁ትም', 'ህር', 'ቲ', '▁ክ', 'ከይ', 'ድ', '▁እየ', '።', '</s>']
Token count: 22tigrinya_bpe_tokenizer/
├── hf_tokenizer/
│ ├── special_tokens_map.json # Special token mappings
│ ├── tokenizer_config.json # HuggingFace tokenizer config
│ └── tokenizer.json # Full tokenizer definition
├── tokenizer_config.json # General tokenizer config
├── tokenizer.json # Tokenizers library format
└── README.md # This file1# Custom text preprocessing for Tigrinya
2def preprocess_tigrinya(text):
3 # Normalize Unicode (NFD)
4 import unicodedata
5 text = unicodedata.normalize('NFD', text)
6
7 # Add custom preprocessing here
8 return text
9
10# Apply preprocessing before tokenization
11processed_text = preprocess_tigrinya(text)
12tokens = tokenizer.encode(processed_text)1# Analyze vocabulary composition
2vocab = tokenizer.get_vocab()
3print(f"Total vocabulary size: {len(vocab)}")
4
5# Find Ge'ez script tokens
6geez_tokens = [token for token in vocab.keys()
7 if any('\u1200' <= char <= '\u137F' for char in token)]
8print(f"Ge'ez tokens: {len(geez_tokens)}")1# From the main project directory
2python train_tigrinya_bpe.py
3
4# Or using the unified interface
5python train_tokenizers.py --type bpe1@misc{tigrinya_bpe_tokenizer,
2 title={Tigrinya BPE Tokenizer for LLM Training},
3 year={2025},
4 publisher={GitHub},
5 howpublished={\url{https://github.com/mewaeltsegay/tokenizer}}
6}1from transformers import PreTrainedTokenizerFast
2tokenizer = PreTrainedTokenizerFast.from_pretrained("./hf_tokenizer")