Views
No views yet
benim_bpe_tokenizerim.model - ogrenilen merge kurallaribenim_bpe_tokenizerim.vocab - insan-okunur vocab dokumu1from huggingface_hub import hf_hub_download
2
3model_path = hf_hub_download(repo_id="nursimakgul/turkce-bpe-tokenizer", filename="benim_bpe_tokenizerim.model")
4
5def load_tokenizer(model_path):
6 merges = {}
7 vocab = {i: bytes([i]) for i in range(256)}
8 with open(model_path, "r", encoding="utf-8") as f:
9 header = f.readline()
10 assert header.strip() == "bpe v1"
11 for line in f:
12 a, b, idx = map(int, line.split())
13 merges[(a, b)] = idx
14 vocab[idx] = vocab[a] + vocab[b]
15 return merges, vocab
16
17merges, vocab = load_tokenizer(model_path)