Views
No views yet
transformers and sentencepiece, both of which can be
installed using pip.pip install transformers sentencepiece1from transformers import pipeline
2
3model_name = "SajjadAyoubi/bert-base-fa-qa"
4qa_pipeline = pipeline("question-answering", model=model_name, tokenizer=model_name)
5
6text = "سلام من سجاد ایوبی هستم ۲۰ سالمه و به پردازش زبان طبیعی علاقه دارم"
7questions = ["اسمم چیه؟", "چند سالمه؟", "به چی علاقه دارم؟"]
8
9for question in questions:
10 print(qa_pipeline({"context": text, "question": question}))
11
12>>> {'score': 0.4839823544025421, 'start': 8, 'end': 18, 'answer': 'سجاد ایوبی'}
13>>> {'score': 0.3747948706150055, 'start': 24, 'end': 32, 'answer': '۲۰ سالمه'}
14>>> {'score': 0.5945395827293396, 'start': 38, 'end': 55, 'answer': 'پردازش زبان طبیعی'}1from transformers import AutoTokenizer, AutoModelForQuestionAnswering
2from src.utils import AnswerPredictor
3
4model_name = "SajjadAyoubi/bert-base-fa-qa"
5tokenizer = AutoTokenizer.from_pretrained(model_name)
6model = AutoModelForQuestionAnswering.from_pretrained(model_name)
7
8text = "سلام من سجاد ایوبی هستم ۲۰ سالمه و به پردازش زبان طبیعی علاقه دارم"
9questions = ["اسمم چیه؟", "چند سالمه؟", "به چی علاقه دارم؟"]
10
11# this class is from src/utils.py and you can read more about it
12predictor = AnswerPredictor(model, tokenizer, device="cpu", n_best=10)
13preds = predictor(questions, [text] * 3, batch_size=3)
14
15for k, v in preds.items():
16 print(v)100%|██████████| 1/1 [00:00<00:00, 3.56it/s]
{'score': 8.040637016296387, 'text': 'سجاد ایوبی'}
{'score': 9.901972770690918, 'text': '۲۰'}
{'score': 12.117212295532227, 'text': 'پردازش زبان طبیعی'}1from transformers import AutoTokenizer, TFAutoModelForQuestionAnswering
2from src.utils import TFAnswerPredictor
3
4model_name = "SajjadAyoubi/bert-base-fa-qa"
5tokenizer = AutoTokenizer.from_pretrained(model_name)
6model = TFAutoModelForQuestionAnswering.from_pretrained(model_name)
7
8text = "سلام من سجاد ایوبی هستم ۲۰ سالمه و به پردازش زبان طبیعی علاقه دارم"
9questions = ["اسمم چیه؟", "چند سالمه؟", "به چی علاقه دارم؟"]
10
11# this class is from src/utils.py, you can read more about it
12predictor = TFAnswerPredictor(model, tokenizer, n_best=10)
13preds = predictor(questions, [text] * 3, batch_size=3)
14
15for k, v in preds.items():
16 print(v)1100%|██████████| 1/1 [00:00<00:00, 3.56it/s]
2{'score': 8.040637016296387, 'text': 'سجاد ایوبی'}
3{'score': 9.901972770690918, 'text': '۲۰'}
4{'score': 12.117212295532227, 'text': 'پردازش زبان طبیعی'}