Views
No views yet
1from transformers import pipeline, AutoTokenizer, AutoModelForTokenClassification
2
3model_name = "MegaLDN/rubert-finetune-goods_NER"
4tokenizer = AutoTokenizer.from_pretrained(model_name])
5model = AutoModelForTokenClassification.from_pretrained(model_name)
6
7ner_pipeline = pipeline(
8"ner",
9model=model ,
10tokenizer=tokenizer ,
11aggregation_strategy="none"
12)
13
14text = "Сок агуша 2л"
15output = ner_pipeline(text)
16# >> print(output)
17# << [{'entity': 'B-TYPE', 'score': np.float32(0.99929225), 'index': 1, 'word': 'Сок', 'start': 0, 'end': 3}, {'entity': 'B-BRAND', 'score': np.float32(0.8768344), 'index': 2, 'word': 'аг', 'start': 4, 'end': 6}, {'entity': 'B-BRAND', 'score': np.float32(0.85818267), 'index': 3, 'word': '##уша', 'start': 6, 'end': 9}, {'entity': 'B-VOLUME', 'score': np.float32(0.99927515), 'index': 4, 'word': '2', 'start': 10, 'end': 11}, {'entity': 'B-VOLUME', 'score': np.float32(0.9982621), 'index': 5, 'word': '##л', 'start': 11, 'end': 12}]1 eval_strategy="epoch",
2 save_strategy="epoch",
3 per_device_train_batch_size=32,
4 per_device_eval_batch_size=32,
5 #gradient_accumulation_steps=2,
6 num_train_epochs=3,
7
8 learning_rate=2e-5,
9 weight_decay=0.01,
10 warmup_ratio=0.06,
11 lr_scheduler_type="linear",
12 max_grad_norm=1.0,
13 fp16=True,
14 load_best_model_at_end=True,
15 metric_for_best_model="f1",
16 greater_is_better=True,
17
18 save_total_limit=2,
19 dataloader_drop_last=False,
20 seed=421 eval_strategy="epoch",
2 save_strategy="epoch",
3 per_device_train_batch_size=32,
4 per_device_eval_batch_size=32,
5 #gradient_accumulation_steps=1,
6 num_train_epochs=3,
7
8 learning_rate=1e-5,
9 weight_decay=0.01,
10 warmup_ratio=0.1,
11 lr_scheduler_type="linear",
12 max_grad_norm=1.0,
13 fp16=True,
14 load_best_model_at_end=True,
15 metric_for_best_model="f1",
16 greater_is_better=True,
17
18 save_total_limit=2,
19 dataloader_drop_last=False,