A Byte Pair Encoding (BPE) tokenizer trained on chess moves in custom notation format.
1import json
2from huggingface_hub import hf_hub_download
3
4# Download tokenizer files
5vocab_path = hf_hub_download(repo_id="YOUR_USERNAME/chess-bpe-tokenizer", filename="vocab.json")
6config_path = hf_hub_download(repo_id="YOUR_USERNAME/chess-bpe-tokenizer", filename="tokenizer_config.json")
7
8# Load vocabulary
9with open(vocab_path, 'r') as f:
10 vocab = json.load(f)
11
12with open(config_path, 'r') as f:
13 config = json.load(f)
14
15print(f"Vocab size: {len(vocab)}")
16print(f"Pattern: {config['pattern']}")
1import rustbpe
2
3# Note: rustbpe tokenizer needs to be retrained or loaded from merges
4# See the training script for details
1from bpess.main import train_chess_tokenizer, push_to_hub
2
3# Train
4tokenizer = train_chess_tokenizer(
5 vocab_size=4096,
6 dataset_fraction="train",
7 moves_key='moves_custom'
8)
9
10# Push to HuggingFace
11push_to_hub(
12 tokenizer=tokenizer,
13 repo_id="your-username/chess-bpe-tokenizer",
14 config={
15 "vocab_size": 4096,
16 "dataset_fraction": "train",
17 "moves_key": "moves_custom"
18 }
19)
