Views
No views yet
<|end_of_turn|> is added, with id eot_token_id.1# OpenChat
2[bos_token_id] + tokenize("Human: ") + tokenize(user_question) + [eot_token_id] + tokenize("Assistant: ")
3# OpenCoder
4tokenize("User:") + tokenize(user_question) + [eot_token_id] + tokenize("Assistant:")tokenize(A) + tokenize(B) does not always equals to tokenize(A + B)1@dataclass
2class ModelConfig:
3 # Prompt
4 system: Optional[str]
5
6 role_prefix: dict
7 ai_role: str
8 eot_token: str
9 bos_token: Optional[str] = None
10
11 # Get template
12 def generate_conversation_template(self, tokenize_fn, tokenize_special_fn, message_list):
13 tokens = []
14 masks = []
15
16 # begin of sentence (bos)
17 if self.bos_token:
18 t = tokenize_special_fn(self.bos_token)
19 tokens.append(t)
20 masks.append(False)
21
22 # System
23 if self.system:
24 t = tokenize_fn(self.system) + [tokenize_special_fn(self.eot_token)]
25 tokens.extend(t)
26 masks.extend([False] * len(t))
27
28 # Messages
29 for idx, message in enumerate(message_list):
30 # Prefix
31 t = tokenize_fn(self.role_prefix[message["from"]])
32 tokens.extend(t)
33 masks.extend([False] * len(t))
34
35 # Message
36 if "value" in message:
37 t = tokenize_fn(message["value"]) + [tokenize_special_fn(self.eot_token)]
38 tokens.extend(t)
39 masks.extend([message["from"] == self.ai_role] * len(t))
40 else:
41 assert idx == len(message_list) - 1, "Empty message for completion must be on the last."
42
43 return tokens, masks
44
45
46MODEL_CONFIG_MAP = {
47 # OpenChat / OpenChat-8192
48 "openchat": ModelConfig(
49 # Prompt
50 system=None,
51
52 role_prefix={
53 "human": "Human: ",
54 "gpt": "Assistant: "
55 },
56 ai_role="gpt",
57 eot_token="<|end_of_turn|>",
58 bos_token="<s>",
59 ),
60
61 # OpenCoder / OpenCoderPlus
62 "opencoder": ModelConfig(
63 # Prompt
64 system=None,
65
66 role_prefix={
67 "human": "User:",
68 "gpt": "Assistant:"
69 },
70 ai_role="gpt",
71 eot_token="<|end_of_turn|>",
72 bos_token=None,
73 )
74}