Views
No views yet
1 def does_token_look_special(self, token: str | bytes) -> bool:
2 if isinstance(token, (bytes, bytearray)):
3 token_text = token.decode(encoding="utf-8")
4 elif isinstance(token, memoryview):
5 token_text = token.tobytes().decode(encoding="utf-8")
6 else:
7 token_text = token
8
9 # Some models mark some added tokens which ought to be control tokens as not special.
10 # (e.g. command-r, command-r-plus, deepseek-coder, gemma{,-2})
11 seems_special = token_text in (
12 "<pad>", # deepseek-coder
13 "<mask>", "<2mass>", "[@BOS@]", # gemma{,-2}
14 )
15
16 seems_special = seems_special or (token_text.startswith("<|") and token_text.endswith("|>"))
17 seems_special = seems_special or (token_text.startswith("<｜") and token_text.endswith("｜>")) # deepseek-coder
18
19 # TODO: should these be marked as UNUSED instead? (maybe not)
20 seems_special = seems_special or (token_text.startswith("<unused") and token_text.endswith(">")) # gemma{,-2}
21
22 if (token_text.startswith("<|") and token_text.endswith("|>")) and token_text[2:-2].startswith("s_") and token_text[2:-2][2:].isdecimal():
23 seems_special = False
24 print(f"token {token_text} looks like a special token but isn't")
25
26 return seems_special