Views
No views yet
| Token | ID | String |
|---|---|---|
unk_token | 128000 | <|UNK|> |
pad_token | 128001 | <|PAD|> |
bos_token | 128002 | <|BOS|> |
eos_token | 128003 | <|EOS|> |
| — | 128004 | <|SYSTEM_START|> |
| — | 128005 | <|SYSTEM_END|> |
| — | 128006 | <|USER_START|> |
| — | 128007 | <|USER_END|> |
| — | 128008 | <|AGENT_START|> |
| — | 128009 | <|AGENT_END|> |
| — | 128010 | <|THINK_START|> |
| — | 128011 | <|THINK_END|> |
| — | 128012 | <|COMPUTE_START|> |
| — | 128013 | <|COMPUTE_END|> |
| — | 128014 | <|IMAGE_START|> |
| — | 128015 | <|IMAGE_END|> |
1pad_token_id: 128001
2bos_token_id: 128002
3eos_token_id: 1280031from transformers import AutoTokenizer
2tok = AutoTokenizer.from_pretrained("path/to/bantam-tokenizer")