Views
No views yet
1from tokenizers import Tokenizer 2 3# 加载分词器 4tokenizer = Tokenizer.from_pretrained("你的用户名/my-tokenizer") 5 6# 分词示例 7text = "Hello, world!" 8output = tokenizer.encode(text) 9print("分词结果:", output.tokens)
