Views
No views yet
tiktoken.cl100k_base to
include special tokens for code and individual digit-splitting.from transformers import AutoTokenizer
tokenizer = AutoTokenizer.from_pretrained("stabilityai/arcade100k", trust_remote_code=True)
tokenizer("hello, world!", return_tensors='pt')1@article{bellagente2024stable,
2 title={Stable LM 2 1.6 B Technical Report},
3 author={Bellagente, Marco and Tow, Jonathan and Mahan, Dakota and Phung, Duy and Zhuravinskyi, Maksym and Adithyan, Reshinth and Baicoianu, James and Brooks, Ben and Cooper, Nathan and Datta, Ashish and others},
4 journal={arXiv preprint arXiv:2402.17834},
5 year={2024}
6}