1from sentence_transformers import SentenceTransformer
2sentences = ["我的儿子！他猛然间喊道，我的儿子在哪儿？", "我的儿子呢！他突然喊道，我的儿子在哪里？"]
3
4model = SentenceTransformer('DMetaSoul/sbert-chinese-general-v2')
5embeddings = model.encode(sentences)
6print(embeddings)
如果不想使用
sentence-transformers 的话，也可以通过 HuggingFace Transformers 来载入该模型并进行文本向量抽取：
1from transformers import AutoTokenizer, AutoModel
2import torch
3
4
5#Mean Pooling - Take attention mask into account for correct averaging
6def mean_pooling(model_output, attention_mask):
7 token_embeddings = model_output[0] #First element of model_output contains all token embeddings
8 input_mask_expanded = attention_mask.unsqueeze(-1).expand(token_embeddings.size()).float()
9 return torch.sum(token_embeddings * input_mask_expanded, 1) / torch.clamp(input_mask_expanded.sum(1), min=1e-9)
10
11
12# Sentences we want sentence embeddings for
13sentences = ["我的儿子！他猛然间喊道，我的儿子在哪儿？", "我的儿子呢！他突然喊道，我的儿子在哪里？"]
14
15# Load model from HuggingFace Hub
16tokenizer = AutoTokenizer.from_pretrained('DMetaSoul/sbert-chinese-general-v2')
17model = AutoModel.from_pretrained('DMetaSoul/sbert-chinese-general-v2')
18
19# Tokenize sentences
20encoded_input = tokenizer(sentences, padding=True, truncation=True, return_tensors='pt')
21
22# Compute token embeddings
23with torch.no_grad():
24 model_output = model(**encoded_input)
25
26# Perform pooling. In this case, mean pooling.
27sentence_embeddings = mean_pooling(model_output, encoded_input['attention_mask'])
28
29print("Sentence embeddings:")
30print(sentence_embeddings)
这里对比了本模型跟之前我们发布
sbert-chinese-general-v1 之间的差异，可以看到本模型在多个任务上的泛化能力更好。
