Views
No views yet
1def generate_answer(model, tokenizer, prompt):
2
3 inputs = tokenizer.apply_chat_template([{"role": "system", "content": BASE_SYSTEM_PROMPT},{"role": "user", "content": prompt}],
4 add_generation_prompt=True,
5 tokenize=True,
6 return_tensors="pt",
7 return_dict=True
8 ).to('cuda')
9
10
11 gen_kwargs = {"temperature":0.5,
12 "top_p":0.8,
13 "max_length":512,
14 "repetition_penalty":1.05,
15 "do_sample": True}
16
17
18 with torch.no_grad():
19 outputs = model.generate(**inputs, **gen_kwargs)
20 outputs = outputs[:, inputs['input_ids'].shape[1]:]
21 return tokenizer.decode(outputs[0], skip_special_tokens=True)