Views
No views yet
pip install tokie1import tokie
2
3tokenizer = tokie.Tokenizer.from_pretrained("tokiers/deepset-mxbai-embed-de-large-v1")
4encoding = tokenizer.encode("Hello, world!")
5print(encoding.ids)
6print(encoding.attention_mask)1[dependencies]
2tokie = { version = "0.0.4", features = ["hf"] }1use tokie::Tokenizer;
2
3let tokenizer = Tokenizer::from_pretrained("tokiers/deepset-mxbai-embed-de-large-v1").unwrap();
4let encoding = tokenizer.encode("Hello, world!", true);
5println!("{:?}", encoding.ids);tokenizer.tkz — tokie binary format (~10x smaller, loads in ~5ms)tokenizer.json — original HuggingFace tokenizer (if available)