Views
No views yet
cl100k_base that supports Dolma 1.x and Dolma 2.x special tokens.<|extra_id_0|>: Not used.<|endoftext|>: Used to mark both beginning and end of text.<|fim_prefix|>: Used to mark the prefix fill-in-the-middle request.<|fim_middle|>: Used to mark the middle fill-in-the-middle request.<|fim_suffix|>: Used to mark the suffix fill-in-the-middle request.|||PHONE_NUMBER|||: Not used. Kept for compatibility with Dolma 1.x.|||EMAIL_ADDRESS|||: Not used. Kept for compatibility with Dolma 1.x.|||IP_ADDRESS|||: Not used. Kept for compatibility with Dolma 1.x.<|im_start|>: Indicates the beginning of a message (turn in a conversation).<|im_end|>: Indicates the end of a message (turn in a conversation).<|extra_id_1|>: Not used.<|extra_id_2|>: Not used.<|extra_id_3|>: Not used.<|extra_id_4|>: Not used.<|extra_id_5|>: Not used.<|extra_id_6|>: Not used.<|extra_id_7|>: Not used.<|extra_id_8|>: Not used.<|extra_id_9|>: Not used.<|extra_id_10|>: Not used.<|endofprompt|>: Not Used.<|pad|>: Symbol to pad input sequences.tokenizer_config.json):1{% set has_system = messages|selectattr('role', 'equalto', 'system')|list|length > 0 %}
2{% if not has_system %}
3{{ '<|im_start|>system
4You are Olmo, a helpful AI assistant built by Ai2. Your date cutoff is December 2024, and your model weights are available at https://huggingface.co/allenai.<|im_end|>
5' }}
6{% endif %}
7{% for message in messages %}
8 {% if message['role'] == 'system' %}
9{{ '<|im_start|>system
10' + message['content'] }}
11 {% if message.get('functions', none) is not none %}
12{{ ' <functions>' + message['functions'] + '</functions><|im_end|>
13' }}
14 {% else %}
15{{ ' You do not currently have access to any functions. <functions></functions><|im_end|>
16' }}
17 {% endif %}
18 {% elif message['role'] == 'user' %}
19 {% if message.get('functions', none) is not none %}
20{{ '<|im_start|>user
21' + message['content'] + '
22' + '<functions>' + message['functions'] + '</functions><|im_end|>
23' }}
24 {% else %}
25{{ '<|im_start|>user
26' + message['content'] + '<|im_end|>
27' }}
28 {% endif %}
29 {% elif message['role'] == 'assistant' %}
30{{ '<|im_start|>assistant
31' }}
32 {% if message.get('content', none) is not none %}
33{{ message['content'] }}
34 {% endif %}
35 {% if message.get('function_calls', none) is not none %}
36{{ '<function_calls>' + message['function_calls'] + '</function_calls>' }}
37 {% endif %}
38 {% if not loop.last %}
39{{ '<|im_end|>' + '
40' }}
41 {% else %}
42{{ eos_token }}
43 {% endif %}
44 {% elif message['role'] == 'environment' %}
45{{ '<|im_start|>environment
46' + message['content'] + '<|im_end|>
47' }}
48 {% endif %}
49 {% if loop.last and add_generation_prompt %}
50{{ '<|im_start|>assistant
51<think>' }}
52 {% endif %}
53{% endfor %}