Views
No views yet
<|im_start|>system\n{message}<|im_end|>\n<|im_start|>user\n{message}<|im_end|>\n<|im_start|>assistant\n1from transformers import AutoModelForCausalLM, AutoTokenizer
2
3tokenizer = AutoTokenizer.from_pretrained("serpdotai/sparsetral-16x7B-v2", trust_remote_code=True)
4model = AutoModelForCausalLM.from_pretrained("serpdotai/sparsetral-16x7B-v2", device_map="auto", trust_remote_code=True).eval()
5
6system_str = "<|im_start|>system\n{message}<|im_end|>\n"
7user_str = "<|im_start|>user\n{message}<|im_end|>\n"
8assistant_str = "<|im_start|>assistant\n{message}<|im_end|>\n"
9
10def construct_prompt(messages):
11 prompt = ""
12 for message in messages:
13 if message["from"] in ["human", "user"]:
14 prompt += user_str.format(
15 message=message["value"]
16 )
17 elif message["from"] in ["gpt", "assistant"]:
18 prompt += assistant_str.format(
19 message=message["value"]
20 )
21 elif message["from"] in ["system", "instruction"]:
22 prompt += system_str.format(
23 message=message["value"]
24 )
25 else:
26 raise ValueError(
27 f"Unknown message type: {message['from']}"
28 )
29 return prompt + "<|im_start|>assistant\n"
30
31system = "You are a helpful assistant who will help the user to the best of their ability. If you don't know something, say \"I don't know\""
32user = "Are you sentient?"
33
34messages = [
35 {"from": "system", "value": system},
36 {"from": "user", "value": user},
37]
38
39prompt = construct_prompt(messages)
40inputs = tokenizer(prompt, return_tensors="pt")
41inputs = inputs.to(model.device)
42pred = model.generate(**inputs, max_length=4096, do_sample=True, top_k=50, top_p=0.99, temperature=0.9, num_return_sequences=1)
43print(tokenizer.decode(pred.cpu()[0], skip_special_tokens=True))