Views
No views yet
DeBERTa AGGREGATED model. See also the GitHub repository.1from transformers import AutoTokenizer, AutoModelForTokenClassification
2import torch
3
4class ParaphraseHighlighter:
5 def __init__(self, model_name="AnnaWegmann/Highlight-Paraphrases-in-Dialog"):
6 # Load the tokenizer and model
7 self.tokenizer = AutoTokenizer.from_pretrained(model_name, use_fast=True)
8 self.model = AutoModelForTokenClassification.from_pretrained(model_name)
9
10 # Get the label id for 'LABEL_1'
11 self.label2id = self.model.config.label2id
12 self.label_id = self.label2id['LABEL_1']
13
14 def highlight_paraphrase(self, text1, text2):
15 # Tokenize the inputs with the tokenizer
16 encoding = self.tokenizer(text1, text2, return_tensors="pt", padding=True, truncation=True)
17
18 outputs = self.model(**encoding)
19 logits = outputs.logits # Shape: (batch_size, sequence_length, num_labels)
20 # Apply softmax to get probabilities, automatically places [SEP] token
21 probs = torch.nn.functional.softmax(logits, dim=-1) # Shape: (batch_size, sequence_length, num_labels)
22
23 # Convert token IDs back to tokens
24 tokens = self.tokenizer.convert_ids_to_tokens(encoding["input_ids"][0])
25 # Get word IDs to map tokens to words
26 word_ids = encoding.word_ids(batch_index=0)
27 # Get sequence IDs to know which text the token belongs to
28 sequence_ids = encoding.sequence_ids(batch_index=0)
29
30 # Collect words and probabilities for each text
31 words_text1 = []
32 words_text2 = []
33 probs_text1 = []
34 probs_text2 = []
35
36 previous_word_idx = None
37
38 # For determining if there are high-probability words in both texts
39 has_high_prob_text1 = False
40 has_high_prob_text2 = False
41
42 for idx, (word_idx, seq_id) in enumerate(zip(word_ids, sequence_ids)):
43 if word_idx is None:
44 # Skip special tokens like [CLS], [SEP], [PAD]
45 continue
46
47 if word_idx != previous_word_idx:
48 # Start of a new word
49 word_tokens = [tokens[idx]]
50
51 # Get the probability for LABEL_1 for the first token of the word
52 prob_LABEL_1 = probs[0][idx][self.label_id].item()
53
54 # Collect subsequent tokens belonging to the same word
55 j = idx + 1
56 while j < len(word_ids) and word_ids[j] == word_idx:
57 word_tokens.append(tokens[j])
58 j += 1
59
60 # Reconstruct the word
61 word = self.tokenizer.convert_tokens_to_string(word_tokens).strip()
62
63 # Check if probability >= 0.5 to uppercase
64 if prob_LABEL_1 >= 0.5:
65 word_display = word.upper()
66 if seq_id == 0:
67 has_high_prob_text1 = True
68 elif seq_id == 1:
69 has_high_prob_text2 = True
70 else:
71 word_display = word
72
73 # Append the word and probability to the appropriate list
74 if seq_id == 0:
75 words_text1.append(word_display)
76 probs_text1.append(prob_LABEL_1)
77 elif seq_id == 1:
78 words_text2.append(word_display)
79 probs_text2.append(prob_LABEL_1)
80 else:
81 # Should not happen
82 pass
83
84 previous_word_idx = word_idx
85
86 # Determine if there are words in both texts with prob >= 0.5
87 if has_high_prob_text1 and has_high_prob_text2:
88 print("is a paraphrase")
89 else:
90 print("is not a paraphrase")
91
92 # Function to format and align words and probabilities
93 def print_aligned(words, probs):
94 # Determine the maximum length of words for formatting
95 max_word_length = max(len(word) for word in words)
96 # Create format string for alignment
97 format_str = f'{{:<{max_word_length}}}'
98 # Print words
99 for word in words:
100 print(format_str.format(word), end=' ')
101 print()
102 # Print probabilities aligned below words
103 for prob in probs:
104 prob_str = f"{prob:.2f}"
105 print(format_str.format(prob_str), end=' ')
106 print('\n')
107
108 # Print text1's words and probabilities aligned
109 print("\nSpeaker 1:")
110 print_aligned(words_text1, probs_text1)
111
112 # Print text2's words and probabilities aligned
113 print("Speaker 2:")
114 print_aligned(words_text2, probs_text2)
115
116# Example usage
117highlighter = ParaphraseHighlighter()
118text1 = "And it will be my 20th time in doing it as a television commentator from Rome so."
119text2 = "Yes, you've been doing this for a while now."
120highlighter.highlight_paraphrase(text1, text2)is a paraphrase
Speaker 1:
And IT will BE MY 20TH TIME IN DOING IT as a TELEVISION COMMENTATOR from Rome so.
0.15 0.54 0.49 0.56 0.74 0.83 0.77 0.75 0.78 0.76 0.44 0.45 0.52 0.52 0.30 0.37 0.21
Speaker 2:
Yes, YOU'VE BEEN DOING THIS FOR A WHILE NOW.
0.12 0.79 0.78 0.82 0.82 0.69 0.70 0.72 0.66 @inproceedings{wegmann-etal-2024-whats,
title = "What{'}s Mine becomes Yours: Defining, Annotating and Detecting Context-Dependent Paraphrases in News Interview Dialogs",
author = "Wegmann, Anna and
Broek, Tijs A. Van Den and
Nguyen, Dong",
editor = "Al-Onaizan, Yaser and
Bansal, Mohit and
Chen, Yun-Nung",
booktitle = "Proceedings of the 2024 Conference on Empirical Methods in Natural Language Processing",
month = nov,
year = "2024",
address = "Miami, Florida, USA",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2024.emnlp-main.52",
pages = "882--912",
abstract = "Best practices for high conflict conversations like counseling or customer support almost always include recommendations to paraphrase the previous speaker. Although paraphrase classification has received widespread attention in NLP, paraphrases are usually considered independent from context, and common models and datasets are not applicable to dialog settings. In this work, we investigate paraphrases across turns in dialog (e.g., Speaker 1: {``}That book is mine.{''} becomes Speaker 2: {``}That book is yours.{''}). We provide an operationalization of context-dependent paraphrases, and develop a training for crowd-workers to classify paraphrases in dialog. We introduce ContextDeP, a dataset with utterance pairs from NPR and CNN news interviews annotated for context-dependent paraphrases. To enable analyses on label variation, the dataset contains 5,581 annotations on 600 utterance pairs. We present promising results with in-context learning and with token classification models for automatic paraphrase detection in dialog.",
}