Views
No views yet
1import torch.nn.functional as F
2from transformers import AutoModelForSequenceClassification, AutoTokenizer
3
4model_name = "ZachW/pacing-judge"
5model = AutoModelForSequenceClassification.from_pretrained(model_name)
6tokenizer = AutoTokenizer.from_pretrained(model_name)
7text_ex_1 = "The Duke then focused on securing his power and looking to future threats. The Duke eventually turned his attention to acquiring Tuscany but struggled."
8text_ex_2 = "Lord Bacon mentioned his book \"The History of Henry VII,\" in the conversation noting that King Charles had conquered Naples without resistance, implying that the conquest was like a dream."
9inputs = tokenizer(text_ex_1 + " <sep> " + text_ex_2, return_tensors="pt")
10outputs = model(**inputs)
11output = int(F.softmax(outputs.logits, dim=1)[:, 0].squeeze(-1).detach().cpu().numpy() > 0.5)
12print(f"Output Binary = {output}")
13if output:
14 print("The second text is more concrete.")
15else:
16 print("The first text is more concrete.")1import torch.nn.functional as F
2from transformers import AutoModelForSequenceClassification, AutoTokenizer
3
4class Ranker:
5 def __init__(self):
6 print(f"*** Loading Model from Huggingface ***")
7 model_name = "ZachW/pacing-judge"
8 self.model = AutoModelForSequenceClassification.from_pretrained(model_name)
9 self.tokenizer = AutoTokenizer.from_pretrained(model_name)
10
11 def compare(self, t1, t2):
12 text_pair = [t1 + ' <sep> ' + t2, t2 + ' <sep> ' + t1]
13 pair_dataset = self.tokenizer(text_pair, padding=True, truncation=True, return_tensors="pt")
14 score = self.run_model(pair_dataset)
15 if score < 0.5:
16 return 0 # first is more concrete
17 else:
18 return 1 # second is more concrete
19
20 def compare_logits(self, t1, t2):
21 text_pair = [t1 + ' <sep> ' + t2, t2 + ' <sep> ' + t1]
22 pair_dataset = self.tokenizer(text_pair, padding=True, truncation=True, return_tensors="pt")
23 score = self.run_model(pair_dataset)
24 return score
25
26 def run_model(self, dataset):
27 outputs = self.model(**dataset)
28 scores = F.softmax(outputs.logits, dim=1)[:, 0].squeeze(-1).detach().cpu().numpy()
29 aver_score = (scores[0] + (1 - scores[1]))/2
30 return aver_score
31
32 def rank(self, texts_list): # input a list of texts
33 def quicksort(arr):
34 if len(arr) <= 1:
35 return arr
36 else:
37 pivot = arr[0]
38 less = []
39 greater = []
40 for t in arr[1:]:
41 cmp = self.compare(pivot, t)
42 if cmp == 0:
43 less.append(t)
44 elif cmp == 1:
45 greater.append(t)
46 return quicksort(greater) + [pivot] + quicksort(less)
47 return quicksort(texts_list)
48 # most concrete -> lest concrete
49
50 def rank_idx(self, texts_list): # input a list of texts
51 def quicksort(arr):
52 if len(arr) <= 1:
53 return arr
54 else:
55 pivot = arr[0]
56 less = []
57 greater = []
58 for t in arr[1:]:
59 cmp = self.compare(texts_list[pivot], texts_list[t])
60 if cmp == 0:
61 less.append(t)
62 elif cmp == 1:
63 greater.append(t)
64 return quicksort(greater) + [pivot] + quicksort(less)
65 return quicksort(list(range(len(texts_list))))
66
67 def rank_idx_conpletely(self, texts_list):
68 n = len(texts_list)
69 texts_idx = list(range(n))
70 scores = [[0] * n for _ in range(n)]
71 self_score = [0] * n
72 for i in texts_idx:
73 scores[i][i] = self.compare_logits(texts_list[i], texts_list[i])
74 self_score[i] = scores[i][i]
75 for j in texts_idx:
76 if j < i:
77 scores[i][j] = 1 - scores[j][i]
78 continue
79 if j == i:
80 continue
81 scores[i][j] = self.compare_logits(texts_list[i], texts_list[j])
82 # average score is, smaller is more concrete
83 average_score = [ sum(s)/len(s) for s in scores]
84 output_score = [ a + 0.5 - s for a, s in zip(average_score, self_score)]
85 sorted_indices = sorted(range(len(output_score)), key=lambda x: output_score[x])
86 return sorted_indices
87
88 def rank_idx_conpletely_wlogits(self, texts_list, logger=None):
89 n = len(texts_list)
90 texts_idx = list(range(n))
91 scores = [[0] * n for _ in range(n)]
92 self_score = [0] * n
93 for i in texts_idx:
94 scores[i][i] = self.compare_logits(texts_list[i], texts_list[i])
95 self_score[i] = scores[i][i]
96 for j in texts_idx:
97 if j < i:
98 scores[i][j] = 1 - scores[j][i]
99 continue
100 if j == i:
101 continue
102 scores[i][j] = self.compare_logits(texts_list[i], texts_list[j])
103 # average score is, smaller is more concrete
104 average_score = [ sum(s)/len(s) for s in scores]
105 output_score = [ a + 0.5 - s for a, s in zip(average_score, self_score)]
106 sorted_indices = sorted(range(len(output_score)), key=lambda x: output_score[x])
107 return sorted_indices, output_score
108
109 def compare_w_neighbors(self, t, cand):
110 score = 0.0
111 for c in cand:
112 score += self.compare_logits(t, c)
113 score /= len(cand)
114 return score1text_ex_1 = "The Duke then focused on securing his power and looking to future threats. The Duke eventually turned his attention to acquiring Tuscany but struggled."
2text_ex_2 = "Lord Bacon mentioned his book \"The History of Henry VII,\" in the conversation noting that King Charles had conquered Naples without resistance, implying that the conquest was like a dream."
3
4ranker = Ranker()
5output = ranker.compare(text_ex_1, text_ex_2) # it is equvilant to (text_ex_2, text_ex_1)
6print(f"Output Binary = {output}")
7if output:
8 print("The second text is more concrete.")
9else:
10 print("The first text is more concrete.")
11
12output_logits = ranker.compare_logits(text_ex_1, text_ex_2)
13print(f"Output Logits = {output_logits:.4f}")