1import torch
2from transformers import AutoModel, AutoTokenizer
3
4
5def avg_pooling(attention_mask, outputs):
6 last_hidden = outputs.last_hidden_state
7 return (last_hidden * attention_mask.unsqueeze(-1)).sum(1) / attention_mask.sum(-1).unsqueeze(-1)
8
9
10tokenizer = AutoTokenizer.from_pretrained('namdp-ptit/ViDense')
11model = AutoModel.from_pretrained('namdp-ptit/ViDense')
12
13sentences = [
14 'Tỉnh nào có diện tích lớn nhất Việt Nam',
15 'Tỉnh nào có diện tích nhỏ nhất Việt Nam',
16 'Tỉnh nào có diện tích rộng nhất Việt Nam'
17]
18
19inputs = tokenizer(sentences, return_tensors='pt', padding=True)
20
21with torch.no_grad():
22 outputs = model(**inputs)
23 outputs = avg_pooling(inputs['attention_mask'], outputs)
24
25cosine_sim_1 = torch.nn.functional.cosine_similarity(
26 outputs[0].unsqueeze(0),
27 outputs[1].unsqueeze(0)
28)
29cosine_sim_2 = torch.nn.functional.cosine_similarity(
30 outputs[0].unsqueeze(0),
31 outputs[2].unsqueeze(0)
32)
33
34print(cosine_sim_1.item()) # 0.056096598505973816
35print(cosine_sim_2.item()) # 0.9861876964569092
Below is a comparision table of the results I achieved compared to some other embedding models on three
benchmarks:
ZAC,
WebFaq,
OwiFaq,
ViQuAD2.0,
ViLegal
with metric
Recall@3If you find this project helpful and wish to support its ongoing development, here are some ways you can contribute:
1@misc{ViDense,
2 title={ViDense: An Embedding Model for Vietnamese Long Context},
3 author={Nam Dang Phuong},
4 year={2025},
5 publisher={Huggingface},
6}
