Views
No views yet
question pairs for train: 644,217question pairs for dev: 73,7101from transformers import DPRContextEncoder, DPRContextEncoderTokenizer
2tokenizer = DPRContextEncoderTokenizer.from_pretrained('voidful/dpr-ctx_encoder-bert-base-multilingual')
3model = DPRContextEncoder.from_pretrained('voidful/dpr-ctx_encoder-bert-base-multilingual')
4input_ids = tokenizer("Hello, is my dog cute ?", return_tensors='pt')["input_ids"]
5embeddings = model(input_ids).pooler_outputhaystack:
Better Retrievers via "Dense Passage Retrieval"from haystack.retriever.dense import DensePassageRetriever
retriever = DensePassageRetriever(document_store=document_store,
query_embedding_model="voidful/dpr-question_encoder-bert-base-multilingual",
passage_embedding_model="voidful/dpr-ctx_encoder-bert-base-multilingual",
max_seq_len_query=64,
max_seq_len_passage=256,
batch_size=16,
use_gpu=True,
embed_title=True,
use_fast_tokenizers=True)