Views
No views yet
<s> - Beginning of sequence</s> - End of sequence<unk> - Unknown token<pad> - Padding token<system>, </system> - System message tags<user>, </user> - User message tags<assistant>, </assistant> - Assistant message tags<reasoning>, </reasoning> - Reasoning tags<tool_call>, </tool_result> - Tool interaction tags<final>, </final> - Final response tags<json>, </json> - JSON formatting tags<tool>, </tool> - Tool tags1from transformers import AutoTokenizer
2
3tokenizer = AutoTokenizer.from_pretrained(
4 "dovcharenko/spm_uk_64k",
5 trust_remote_code=True
6)
7
8# Encode text
9text = "Україна — це країна в Східній Європі"
10tokens = tokenizer.encode(text, add_special_tokens=False)
11print(f"Tokens: {len(tokens)}")
12
13# Decode tokens
14decoded = tokenizer.decode(tokens)
15print(f"Decoded: {decoded}")1@misc{spm_uk_64k_tokenizer,
2 title={Ukrainian SentencePiece Tokenizer (64K Vocabulary)},
3 author={Dovcharenko},
4 year={2024},
5 publisher={HuggingFace}
6}