Views
No views yet
1from transformers import AutoModelForTokenClassification, AutoTokenizer
2from nltk import word_tokenize
3import torch
4
5device = torch.device("cuda")
6path = f"KomeijiForce/roberta-large-metaie-super-academia-gpt4o"
7tokenizer = AutoTokenizer.from_pretrained(path)
8tagger = AutoModelForTokenClassification.from_pretrained(path).to(device)
9
10def find_sequences(lst):
11 sequences = []
12 i = 0
13 while i < len(lst):
14 if lst[i] == 0:
15 start = i
16 end = i
17 i += 1
18 while i < len(lst) and lst[i] == 1:
19 end = i
20 i += 1
21 sequences.append((start, end+1))
22 else:
23 i += 1
24 return sequences
25
26passage = '''MetaIE: Distilling a Meta Model from LLM for All Kinds of Information Extraction Tasks
27
28Information extraction (IE) is a fundamental area in natural language processing where prompting large language models (LLMs), even with in-context examples, cannot defeat small LMs tuned on very small IE datasets. We observe that IE tasks, such as named entity recognition and relation extraction, all focus on extracting important information, which can be formalized as a label-to-span matching. In this paper, we propose a novel framework MetaIE to build a small LM as meta-model by learning to extract "important information", i.e., the meta-understanding of IE, so that this meta-model can be adapted to all kind of IE tasks effectively and efficiently. Specifically, MetaIE obtains the small LM via a symbolic distillation from an LLM following the label-to-span scheme. We construct the distillation dataset via sampling sentences from language model pre-training datasets (e.g., OpenWebText in our implementation) and prompting an LLM to identify the typed spans of "important information". We evaluate the meta-model under the few-shot adaptation setting. Extensive results on 13 datasets from 6 IE tasks confirm that MetaIE can offer a better starting point for few-shot tuning on IE datasets and outperform other meta-models from (1) vanilla language model pre-training, (2) multi-IE-task pre-training with human annotations, and (3) single-IE-task symbolic distillation from LLM. Moreover, we provide comprehensive analyses of MetaIE, such as the size of the distillation dataset, the meta-model architecture, and the size of the meta-model.'''
29
30
31labels = [
32 "Research Domain",
33 "Tasks for MetaIE",
34 "Way to Build MetaIE",
35 "Method",
36 "Model Performance",
37]
38
39for label in labels:
40
41 text = f"{label}: " + passage
42
43 inputs = tokenizer(text, return_tensors="pt").to(device)
44 tag_predictions = tagger(**inputs).logits[0].argmax(-1)
45
46 predictions = [tokenizer.decode(inputs.input_ids[0, seq[0]:seq[1]]).strip() for seq in find_sequences(tag_predictions)]
47
48 print(label, predictions)Research Domain ['natural language processing']
Tasks for MetaIE ['named entity recognition', 'relation extraction']
Way to Build MetaIE ['learning to extract "important information', 'symbolic distillation from an LLM following the label']
Method ['symbolic distillation from an LLM following the label']
Model Performance ['outperform other meta-models']