Views
No views yet
1def clip_score(text_feature, image_feature):
2 image_feature = image_feature / image_feature.norm(p=2, dim=-1, keepdim=True)
3 text_feature = text_feature / text_feature.norm(p=2, dim=-1, keepdim=True)
4 logits_per_image = image_feature @ text_feature.T
5 logits_per_image = scale.exp() * logits_per_image
6 probs = logits_per_image.softmax(dim=1)[0]
7
8 return probs.tolist()
9
10def encode(pixel_values):
11 model = VQModel.from_pretrained('MeissonFlow/Meissonic', subfolder='vqvae')
12 y = model.encoder.conv_in(pixel_values)
13 for down_block in model.encoder.down_blocks:
14 y = down_block(y)
15 y = model.encoder.mid_block(y)
16 y = model.encoder.conv_norm_out(y)
17
18 return model.encoder.conv_act(y)
19
20class CLIPVision:
21 ...
22 def forward(self, pixel_values):
23 b, c, h, w = pixel_values.shape
24 assert w == 448 and h == 448
25
26 with torch.no_grad():
27 output = encode(pixel_values)
28 b, dim, _, _ = output.shape
29 output = output.permute(0, 3, 1, 2).contiguous()
30 vq_output = output.view(1, -1, dim)
31 output = self.final_layer(vq_output)
32 pooled = output[:, :1024, :].mean(dim=1)
33
34 return torch.cat([pooled.unsqueeze(1), output[:, :1024, :]], dim=1)
35
36image_size = 448
37model = CLIPVision()
38clip_model = AutoModelForCausalLM.from_pretrained('qihoo360/fg-clip-base', trust_remote_code=True)
39clip_tokenizer = AutoTokenizer.from_pretrained('qihoo360/fg-clip-base')
40...
41pooled = model(pixel_values)
42clip_score(text_feature, pooled)