Views
No views yet
transformers library for generating embeddings:1from transformers import AutoModel, AutoTokenizer
2import torch.nn.functional as F
3
4# Load the model
5model_name = "gopersonal/multilingual-e5-large-instruct-8bit"
6tokenizer = AutoTokenizer.from_pretrained(model_name)
7model = AutoModel.from_pretrained(model_name, load_in_8bit=True, device_map="auto")
8
9# Define function to get embeddings
10def average_pool(last_hidden_states, attention_mask):
11 last_hidden = last_hidden_states.masked_fill(~attention_mask[..., None].bool(), 0.0)
12 return last_hidden.sum(dim=1) / attention_mask.sum(dim=1)[..., None]
13
14def get_detailed_instruct(task_description, query):
15 return f'Instruct: task_description\nQuery: query'
16
17# Prepare your texts
18task = 'Given a web search query, retrieve relevant passages that answer the query'
19queries = [
20 get_detailed_instruct(task, 'how much protein should a female eat'),
21 get_detailed_instruct(task, 'best restaurants in new york')
22]
23
24# Tokenize and generate embeddings
25batch_dict = tokenizer(queries, max_length=512, padding=True, truncation=True, return_tensors='pt')
26outputs = model(**batch_dict)
27embeddings = average_pool(outputs.last_hidden_state, batch_dict['attention_mask'])
28
29# Normalize embeddings
30embeddings = F.normalize(embeddings, p=2, dim=1)1docker run --gpus all -v $PWD/models:/app/.cache -p 7997:7997 \
2 michaelf34/infinity:latest \
3 v2 --model-id gopersonal/multilingual-e5-large-instruct-8bit \
4 --dtype int8 --batch-size 8 --engine torch --port 7997 --device auto