Views
No views yet
1from transformers import pipeline
2
3oracle = pipeline('ner', model='dicta-il/dictabert-large-ner', aggregation_strategy='simple')
4
5# if we set aggregation_strategy to simple, we need to define a decoder for the tokenizer. Note that the last wordpiece of a group will still be emitted
6from tokenizers.decoders import WordPiece
7oracle.tokenizer.backend_tokenizer.decoder = WordPiece()
8
9sentence = '''דוד בן-גוריון (16 באוקטובר 1886 - ו' בכסלו תשל"ד) היה מדינאי ישראלי וראש הממשלה הראשון של מדינת ישראל.'''
10oracle(sentence)1[
2 {
3 "entity_group": "PER",
4 "score": 0.9998988,
5 "word": "דוד בן - גוריון",
6 "start": 0,
7 "end": 13
8 },
9 {
10 "entity_group": "TIMEX",
11 "score": 0.99989706,
12 "word": "16 באוקטובר 1886",
13 "start": 15,
14 "end": 31
15 },
16 {
17 "entity_group": "TIMEX",
18 "score": 0.99991614,
19 "word": "ו' בכסלו תשל\"ד",
20 "start": 34,
21 "end": 48
22 },
23 {
24 "entity_group": "TTL",
25 "score": 0.9931756,
26 "word": "וראש הממשלה",
27 "start": 68,
28 "end": 79
29 },
30 {
31 "entity_group": "GPE",
32 "score": 0.9995702,
33 "word": "ישראל",
34 "start": 96,
35 "end": 101
36 }
37]DictaBERT: A State-of-the-Art BERT Suite for Modern Hebrew1@misc{shmidman2023dictabert,
2 title={DictaBERT: A State-of-the-Art BERT Suite for Modern Hebrew},
3 author={Shaltiel Shmidman and Avi Shmidman and Moshe Koppel},
4 year={2023},
5 eprint={2308.16687},
6 archivePrefix={arXiv},
7 primaryClass={cs.CL}
8}
