Views
No views yet
1import torch.nn.functional as F
2
3from torch import Tensor
4from transformers import AutoTokenizer, AutoModel
5
6
7def average_pool(last_hidden_states: Tensor,
8 attention_mask: Tensor) -> Tensor:
9 last_hidden = last_hidden_states.masked_fill(~attention_mask[..., None].bool(), 0.0)
10 return last_hidden.sum(dim=1) / attention_mask.sum(dim=1)[..., None]
11
12
13# Each input text should start with "query: " or "passage: ".
14# For tasks other than retrieval, you can simply use the "query: " prefix.
15input_texts = ['query: Who is prophet known for patience',
16 'query: Who is moses',
17 "passage: passage 1",
18 "passage: passage 2"]
19
20tokenizer = AutoTokenizer.from_pretrained('intfloat/e5-small')
21model = AutoModel.from_pretrained('intfloat/e5-small')
22
23# Tokenize the input texts
24batch_dict = tokenizer(input_texts, max_length=512, padding=True, truncation=True, return_tensors='pt')
25
26outputs = model(**batch_dict)
27embeddings = average_pool(outputs.last_hidden_state, batch_dict['attention_mask'])
28
29# (Optionally) normalize embeddings
30embeddings = F.normalize(embeddings, p=2, dim=1)
31scores = (embeddings[:2] @ embeddings[2:].T) * 100
32print(scores.tolist())