Views
No views yet
1from transformers import AutoModelForCausalLM, AutoTokenizer
2
3# load model
4model_path="Johndfm/ECoh-1.8B"
5tokenizer = AutoTokenizer.from_pretrained(model_path,padding_side="left")
6base_model = AutoModelForCausalLM.from_pretrained(model_path).to("cuda")
7
8# prepare example
9example = "Context:\nA: Dahua's Market . How can I help you ? \nB: Where is your store located ? \n\nResponse:\nA: Our store is located on 123 Main Street, in the city center."
10messages = [
11 {"role": "system", "content": "You are a Coherence evaluator."}
12 {"role": "user", "content": f"{example}\n\nGiven the context, is the response Coherent (Yes/No)? Explain your reasoning."}
13]
14
15text = tokenizer.apply_chat_template(
16 messages,
17 tokenize=False,
18 add_generation_prompt=True
19)
20model_inputs = tokenizer([text], return_tensors="pt").to("cuda")
21
22generated_ids = base_model.generate(
23 model_inputs.input_ids,
24 max_new_tokens=64
25)
26
27generated_ids = [
28 output_ids[len(input_ids):] for input_ids, output_ids in zip(model_inputs.input_ids, generated_ids)
29]
30
31response = tokenizer.batch_decode(generated_ids, skip_special_tokens=True)[0]@misc{mendonça2024ecoh,
title={ECoh: Turn-level Coherence Evaluation for Multilingual Dialogues},
author={John Mendonça and Isabel Trancoso and Alon Lavie},
year={2024},
eprint={2407.11660},
archivePrefix={arXiv},
primaryClass={cs.CL},
url={https://arxiv.org/abs/2407.11660},
}