Byte Pair Encoding tokenizer for Telugu.
1import json
2
3# Load tokenizer
4with open('telugu_merges.json') as f:
5 merges = {eval(k): v for k, v in json.load(f).items()}
6with open('telugu_vocab.json') as f:
7 vocab = {int(k): bytes.fromhex(v) for k, v in json.load(f).items()}
8
9def get_stats(ids):
10 counts = {}
11 for pair in zip(ids, ids[1:]):
12 counts[pair] = counts.get(pair, 0) + 1
13 return counts
14
15def merge(ids, pair, idx):
16 newids = []
17 i = 0
18 while i < len(ids):
19 if i < len(ids) - 1 and ids[i] == pair[0] and ids[i+1] == pair[1]:
20 newids.append(idx)
21 i += 2
22 else:
23 newids.append(ids[i])
24 i += 1
25 return newids
26
27def encode(text):
28 tokens = list(text.encode("utf-8"))
29 while len(tokens) >= 2:
30 stats = get_stats(tokens)
31 pair = min(stats, key=lambda p: merges.get(p, float("inf")))
32 if pair not in merges:
33 break
34 tokens = merge(tokens, pair, merges[pair])
35 return tokens
36
37def decode(ids):
38 return b"".join(vocab[i] for i in ids).decode("utf-8", errors="replace")
39
40# Example
41text = "నమస్కారం"
42print(decode(encode(text)))
