Views
No views yet
1from hindi_tokenizer import HindiTokenizer
2import pickle
3
4# Load the tokenizer
5with open('hindi_tokenizer.pkl', 'rb') as f:
6 tokenizer = pickle.load(f)
7
8# Encode text
9text = "मार्केट ट्रेंड्स को समझना"
10encoded = tokenizer.encode(text)
11print(f"Token IDs: {encoded}")
12
13# Decode back
14decoded = tokenizer.decode(encoded)
15print(f"Decoded: {decoded}")
16
17# Get compression stats
18stats = tokenizer.get_compression_stats(text)
19print(f"Compression: {stats['byte_compression_ratio']:.2f}x")pip install regex1@software{hindi_bpe_tokenizer,
2 title = {Hindi BPE Tokenizer},
3 author = {agileabhi},
4 year = {2025},
5 url = {https://github.com/agileabhi/hindi-bpe-tokenizer}
6}