Views
No views yet
rerank-english-v3.0 tokenizer1from tokenizers import Tokenizer
2
3tokenizer = Tokenizer.from_pretrained("CohereLabs/rerank-english-v3.0")
4text = "Hello World, this is my input string!"
5enc = tokenizer.encode(text)
6print("Encoded input:")
7print(enc.ids)
8
9print("Tokens:")
10print(enc.tokens)
11
12number_of_tokens = len(enc.ids)
13print("Number of tokens:", number_of_tokens)