Views
No views yet
do_lower_case = True flag with the tokenizer. Instead, convert your text to lower case as follows:text.replace("I", "ı").lower()1from transformers import AutoTokenizer, BertForMaskedLM
2from transformers import pipeline
3
4model = BertForMaskedLM.from_pretrained("ytu-ce-cosmos/turkish-base-bert-uncased")
5# or
6# model = BertForMaskedLM.from_pretrained("ytu-ce-cosmos/turkish-base-bert-uncased", from_tf = True)
7
8tokenizer = AutoTokenizer.from_pretrained("ytu-ce-cosmos/turkish-base-bert-uncased")
9
10unmasker = pipeline('fill-mask', model=model, tokenizer=tokenizer)
11unmasker("gelirken bir litre [MASK] aldım.")
12[{'score': 0.6248273253440857,
13 'token': 2417,
14 'token_str': 'su',
15 'sequence': 'gelirken bir litre su aldım.'},
16 {'score': 0.10369712114334106,
17 'token': 2168,
18 'token_str': 'daha',
19 'sequence': 'gelirken bir litre daha aldım.'},
20 {'score': 0.06832519918680191,
21 'token': 11818,
22 'token_str': 'benzin',
23 'sequence': 'gelirken bir litre benzin aldım.'},
24 {'score': 0.027739914134144783,
25 'token': 11973,
26 'token_str': 'bira',
27 'sequence': 'gelirken bir litre bira aldım.'},
28 {'score': 0.02571810781955719,
29 'token': 7279,
30 'token_str': 'alkol',
31 'sequence': 'gelirken bir litre alkol aldım.'}]1@article{kesgin2023developing,
2 title={Developing and Evaluating Tiny to Medium-Sized Turkish BERT Models},
3 author={Kesgin, Himmet Toprak and Yuce, Muzaffer Kaan and Amasyali, Mehmet Fatih},
4 journal={arXiv preprint arXiv:2307.14134},
5 year={2023}
6}