1import torch
2from torch.utils.data import Dataset, DataLoader
3from transformers import AutoTokenizer, AutoModelForSequenceClassification
4from datasets import load_dataset
5from sklearn.metrics import accuracy_score, f1_score, classification_report
6import pandas as pd
7from collections import defaultdict
8
9# ------------------- Sabitler -------------------
10MODEL_ID = "ebrukilic/bert-absa-tr-v5"
11DATASET_ID = "ebrukilic/tubitak_clothing_absa_v3"
12SPLIT = "test"
13TEXT_COL = "normalized_yorum"
14LABEL_COL = "polarity"
15ASPECT_COL = "aspects"
16BATCH_SIZE = 16
17MAX_LEN = 128
18
19device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
20print("Device:", device)
21
22# ---------- Dataset yükleme işlemi ----------------
23ds_raw = load_dataset(DATASET_ID, split=SPLIT)
24df = ds_raw.to_pandas()
25
26# Aspect listelerini temizle
27def to_list(x):
28 if x is None: return []
29 if isinstance(x, list): return x
30 return [x]
31
32def clean_list(lst):
33 return [str(a) for a in lst if str(a).lower() not in {"unknown", "unk", ""}]
34
35df["_aspect_list"] = df[ASPECT_COL].apply(to_list).apply(clean_list)
36df = df[df["_aspect_list"].map(len) > 0].copy()
37
38# Label encode
39label_space = sorted(list(set(map(str, df[LABEL_COL]))))
40label2id = {label:i for i,label in enumerate(label_space)}
41id2label = {i:label for label,i in label2id.items()}
42print("Label mapping:", label2id, "\n")
43
44# ------------------- Model yükle -------------------
45tokenizer = AutoTokenizer.from_pretrained(MODEL_ID, use_fast=True)
46model = AutoModelForSequenceClassification.from_pretrained(
47 MODEL_ID, num_labels=len(label_space), id2label=id2label, label2id=label2id
48).to(device)
49model.eval()
50
51# ------------------- Dataset sınıfları -------------------
52# Aspect-aware dataset
53class AspectAwareDataset(Dataset):
54 def __init__(self, dataframe, tokenizer, label2id, max_length=128):
55 self.data = dataframe.explode("_aspect_list").rename(columns={"_aspect_list":"aspect"}).reset_index(drop=True)
56 self.tokenizer = tokenizer
57 self.label2id = label2id
58 self.max_length = max_length
59
60 def __len__(self):
61 return len(self.data)
62
63 def __getitem__(self, idx):
64 row = self.data.iloc[idx]
65 text, aspect = row[TEXT_COL], row["aspect"] #hem text hem aspect verildi
66 label = self.label2id[str(row[LABEL_COL])]
67 inputs = self.tokenizer(
68 aspect, text,
69 truncation=True,
70 padding='max_length',
71 max_length=self.max_length,
72 return_tensors="pt"
73 )
74 return {
75 'input_ids': inputs['input_ids'].squeeze(0),
76 'attention_mask': inputs['attention_mask'].squeeze(0),
77 'labels': torch.tensor(label)
78 }
79
80# Text-only dataset
81class TextOnlyDataset(Dataset):
82 def __init__(self, dataframe, tokenizer, label2id, max_length=128):
83 self.data = dataframe
84 self.tokenizer = tokenizer
85 self.label2id = label2id
86 self.max_length = max_length
87
88 def __len__(self):
89 return len(self.data)
90
91 def __getitem__(self, idx):
92 row = self.data.iloc[idx]
93 text = row[TEXT_COL] #sadece text verildi
94 label = self.label2id[str(row[LABEL_COL])]
95 inputs = self.tokenizer(
96 text,
97 truncation=True,
98 padding='max_length',
99 max_length=self.max_length,
100 return_tensors="pt"
101 )
102 return {
103 'input_ids': inputs['input_ids'].squeeze(0),
104 'attention_mask': inputs['attention_mask'].squeeze(0),
105 'labels': torch.tensor(label)
106 }
107
108# ------------------- DataLoader -------------------
109dataloader_aspect = DataLoader(AspectAwareDataset(df, tokenizer, label2id, MAX_LEN),
110 batch_size=BATCH_SIZE, shuffle=False)
111dataloader_textonly = DataLoader(TextOnlyDataset(df, tokenizer, label2id, MAX_LEN),
112 batch_size=BATCH_SIZE, shuffle=False)
113
114# ------------------- Değerlendirme Fonksiyonu -------------------
115def evaluate_model(dataloader, device, df=None, aspect_aware=False):
116 y_true, y_pred = [], []
117 aspect_perf = defaultdict(list) if aspect_aware else None
118
119 with torch.no_grad():
120 for idx, batch in enumerate(dataloader):
121 input_ids = batch['input_ids'].to(device)
122 attention_mask = batch['attention_mask'].to(device)
123 labels = batch['labels'].to(device)
124
125 outputs = model(input_ids=input_ids, attention_mask=attention_mask)
126 predictions = torch.argmax(outputs.logits, dim=1)
127
128 y_true.extend(labels.cpu().tolist())
129 y_pred.extend(predictions.cpu().tolist())
130
131 if aspect_aware:
132 batch_start = idx * BATCH_SIZE
133 for i in range(len(labels)):
134 data_idx = batch_start + i
135 if data_idx < len(df):
136 true_label = labels[i].cpu().item()
137 pred_label = predictions[i].cpu().item()
138 aspects = df.iloc[data_idx]["_aspect_list"]
139 for aspect in aspects:
140 aspect_perf[aspect].append((true_label, pred_label))
141 return y_true, y_pred, aspect_perf
142
143# ------------------- Modeli Değerlendir -------------------
144print("=== Aspect-aware Evaluation ===")
145y_true1, y_pred1, aspect_perf = evaluate_model(dataloader_aspect, device, df, aspect_aware=True)
146print(f"Accuracy: {accuracy_score(y_true1, y_pred1):.4f} Macro-F1: {f1_score(y_true1, y_pred1, average='macro'):.4f}")
147print(classification_report(y_true1, y_pred1, target_names=label_space))
148
149print("\n--- Aspect-wise Performance ---")
150for aspect, preds in aspect_perf.items():
151 if len(preds) >= 10:
152 t, p = zip(*preds)
153 print(f"{aspect}: {accuracy_score(t,p):.3f} ({len(preds)} samples)")
154
155print("\n" + "="*60)
156print("=== Text-only Evaluation ===")
157y_true2, y_pred2, _ = evaluate_model(dataloader_textonly, device)
158print(f"Accuracy: {accuracy_score(y_true2, y_pred2):.4f} Macro-F1: {f1_score(y_true2, y_pred2, average='macro'):.4f}")
159print(classification_report(y_true2, y_pred2, target_names=label_space))
160
161# ------------------- Karşılaştırma -------------------
162print("\n=== Summary ===")
163acc1 = accuracy_score(y_true1, y_pred1)
164acc2 = accuracy_score(y_true2, y_pred2)
165f1_1 = f1_score(y_true1, y_pred1, average='macro')
166f1_2 = f1_score(y_true2, y_pred2, average='macro')
167
168print(f"Aspect-aware: Acc={acc1:.3f}, F1={f1_1:.3f}")
169print(f"Text-only: Acc={acc2:.3f}, F1={f1_2:.3f}")
170print(f"Aspect effect: {((acc1-acc2)/acc2*100):+.1f}%")
