We write our prompts in the ChatML format.
1from vllm import LLM, SamplingParams
2model_name = "lightblue/jod"
3llm = LLM(model=model_name)
4
5SYSTEM_MESSAGE = "You are a helpful assistant."
6def process_chat_history(next_user_msg, text_chat_history = []):
7 prompt_text = "<|im_start|>system\n"
8 prompt_text += SYSTEM_MESSAGE
9 prompt_text += "<|im_end|>\n\n"
10
11 for user_msg, ai_msg in text_chat_history:
12 prompt_text += "<|im_start|>user\n"
13 prompt_text += user_msg
14 prompt_text += "<|im_end|>\n\n"
15 prompt_text += "<|im_start|>assistant\n"
16 prompt_text += ai_msg
17 prompt_text += "<|im_end|>\n\n"
18
19 prompt_text += "<|im_start|>user\n"
20 prompt_text += next_user_msg
21 prompt_text += "<|im_end|>\n\n"
22 prompt_text += "<|im_start|>assistant\n"
23 return prompt_text
24
25user_prompt = "日本の一番高い山は？"
26prompt = process_chat_history(user_prompt)
27sampling_params = SamplingParams(temperature=0, max_tokens=528)
28outputs = llm.generate(prompt, sampling_params)
29bot_message = outputs[0].outputs[0].text.strip()
30print(bot_message)
1from transformers import AutoTokenizer, AutoModelForCausalLM, pipeline
2
3model_name = "lightblue/jod"
4
5tokenizer = AutoTokenizer.from_pretrained(model_dir)
6model = AutoModelForCausalLM.from_pretrained(
7 model_dir, torch_dtype=torch.bfloat16, device_map='auto', load_in_4bit=True,
8)
9
10pipe = pipeline("text-generation", model=model, tokenizer=tokenizer)
11
12SYSTEM_MESSAGE = "You are a helpful assistant."
13def process_chat_history(next_user_msg, text_chat_history = []):
14 prompt_text = "<|im_start|>system\n"
15 prompt_text += SYSTEM_MESSAGE
16 prompt_text += "<|im_end|>\n\n"
17
18 for user_msg, ai_msg in text_chat_history:
19 prompt_text += "<|im_start|>user\n"
20 prompt_text += user_msg
21 prompt_text += "<|im_end|>\n\n"
22 prompt_text += "<|im_start|>assistant\n"
23 prompt_text += ai_msg
24 prompt_text += "<|im_end|>\n\n"
25
26 prompt_text += "<|im_start|>user\n"
27 prompt_text += next_user_msg
28 prompt_text += "<|im_end|>\n\n"
29 prompt_text += "<|im_start|>assistant\n"
30 return prompt_text
31
32user_prompt = "日本の一番高い山は？"
33prompt = process_chat_history(user_prompt)
34bot_message = pipe(do_closed_qa(test_article, question), max_new_tokens=128, temperature=0)[0]["generated_text"]
35print(bot_message)
using the (
Open-Orca/Mistral-7B-SlimOrca) model as our base checkpoint.
This model was trained using the ChatML format, so it should be used for inference using the ChatML chatbot format.
We chose this format as the base model (
Open-Orca/Mistral-7B-SlimOrca) was trained with this format, and we find the chatbot format more compelling for practical use compared to the Alpaca style instruction format.
We trained for 1 epoch using the following Axolotl config. (Early stopping was not performed during our training.)
