Views
No views yet
1{
2 "background": "The images's background is low quality, there is no background or the background is ugly",
3 "clarity": "The images is blurry, the image is noticeably blurry, as though noise or distortion is present.",
4 "color aesthetic": "The images's color aesthetic is ugly, there are ugly colors, the colors are unpleasant, often due to lack of harmony, abruptness, overexposure, or poor color choice.",
5 "color brightness": "The images's brightness is dark: Purely judging the color, it is dark.",
6 "detail realism": "The images's detail realism is inauthentic; The image is inconsistent with reality, featuring heavy object distortion or deformation, even when scaled down, the flaws in realism are apparent.",
7 "detail refinement": "Important elements appear noticeably rough on closer inspection, or the overall image feels rough. Important details are blatantly rough or unfinished. So rough that it is impossible to discern objects in the image. The image isn't just indecipherable, but it also appears extremely fragmented/broken.",
8 "emotion": "The images shows negative emotions. The image invokes negative emotions, such as creepiness, eeriness, horror, disgust, or hostility",
9 "lighting distinction": "The images' bas bad lighting distinction: No light and shadow: No obvious light effects or shadows are seen.",
10 "main object": "The images's main object is not prominent: There is no main object, or the main object is too inconspicuous (small + peripheral + unnoticeable).",
11 "object pairing": "The images's object pairing is disharmonious: Includes any noticeable mismatch in harmony, proportion, or aesthetics, such as awkward pairings, chaotic or discordant compositions, etc.",
12 "richness": "The images's empty: The image appears to lack clear intent or design and seems random or completely unfinished. ",
13 "symmetry": "The images's asymmetrical: The object is naturally symmetrical in real life but appears clearly asymmetrical in the image. The object itself is asymmetrical due to reasons other than perspective."
14}1import json
2with open("prompts.json", "r") as f:
3 prompt_dict = json.load(f)
4
5def format_data(sample, dim):
6 images = sample["image"]
7 dims_selected = [dim]
8
9 prompts = [prompt_dict[dim] for i, dim in enumerate(dims_selected)]
10 n_images = len(images)
11 n_prompts = len(prompts)
12 inputs = processor(images=images, text=prompts, return_tensors="pt", padding=True)
13 answers = [1 if i[dim]<0 else 0 for i, dim in zip(sample["annotation"], dims_selected)]
14 answers = [(1-i, i) for i in answers]
15 labels = torch.tensor(answers)
16 inputs['labels'] = labels
17 inputs['dim'] = [dims.index(dim) for dim in dims_selected]
18 inputs['n_images'] = [n_images] * len(inputs['input_ids'])
19 return {
20 'pixel_values': inputs['pixel_values'],
21 'input_ids': inputs['input_ids'],
22 'attention_mask': inputs['attention_mask'],
23 'labels': inputs['labels'],
24 'n_images': inputs['n_images'],
25 }
26
27import tqdm
28from sklearn.metrics import f1_score, roc_auc_score
29
30pred = []
31gt = []
32
33pbar = tqdm.tqdm(range(len(test_ds)))
34for i in pbar:
35 for dim in dims:
36 inputs = format_data(test_ds[i:i+1], dim)
37 outputs = model(
38 pixel_values=inputs['pixel_values'].cuda(),
39 input_ids=inputs['input_ids'].cuda(),
40 attention_mask=inputs['attention_mask'].cuda(),
41 n_images=inputs['n_images'],
42 labels=inputs['labels'].cuda()
43 )
44 score = torch.nn.functional.softmax(outputs['itm_score'], dim=-1)[0,1].item()
45 pred.append(score)
46 gt.append(inputs['labels'][0,1].item())
47 f1 = f1_score(gt, [1 if p > 0.5 else 0 for p in pred])
48 auc = roc_auc_score(gt, pred)
49 pbar.set_description(f"F1: {f1:.4f} AUC: {auc:.4f}")
50