Views
No views yet
1import torch
2import re
3import numpy as np
4
5from transformers import AutoTokenizer, AutoModelForTokenClassification, AutoConfig
6from transformers import pipeline
7from scipy.special import softmax
8from pythainlp.ulmfit import (
9 process_thai,
10 replace_rep_after,
11 fix_html,
12 ungroup_emoji,
13 replace_wrep_post,
14 remove_space
15)
16
17# Preprocess text
18def text_processing_newmm(text):
19 punctuations = """!()+-[]{;:'}"“”\,‘’<>/?@^$–^*_~ๆฯ•.."""
20 final = re.sub(
21 r"""@^(http\:\/\/|https\:\/\/)?([a-z0-9][a-z0-9\-]*\.)+[a-z0-9][a-z0-9\-]*$@i""",
22 "",
23 str(text),
24 )
25 final = final.replace("เเ", "แ")
26 final = "".join(u for u in str(final) if u not in punctuations)
27 final = process_thai(final,
28 pre_rules=[replace_rep_after, fix_html],
29 post_rules=[ungroup_emoji,
30 replace_wrep_post,
31 remove_space]
32 )
33 final = [s for s in final if "\u200b" not in s]
34 final = [s for s in final if "xxrep" not in s]
35 final = [s for s in final if "xxwrep" not in s]
36 final = [x for x in final if not (x.isdigit()
37 or x[0] == '-' and x[1:].isdigit())]
38 final = "|".join(word for word in final)
39 return final
40
41def parse_tokens(
42 clean_tokens, ignore_tokens=[], skip_split=False, keep_whitespace=False
43):
44 if skip_split:
45 tokens_list = clean_tokens
46 else:
47 try:
48 tokens_list = clean_tokens.split("|")
49 except Exception as err:
50 tokens_list = []
51
52 tokens_list = [x.strip() for x in tokens_list]
53
54 # For skipping certain tokens
55 if ignore_tokens:
56 tokens_list = [x for x in tokens_list if x not in ignore_tokens]
57
58 # For keep or ignore whitespacing
59 if keep_whitespace is True:
60 pass
61 else:
62 tokens_list = [x for x in tokens_list if x]
63
64 return tokens_list
65
66def format_words(words_list):
67 formatted_words = " ".join(words_list)
68 return formatted_words
69
70def group_entities_iobes(ner_results):
71 grouped_entities = []
72 current_entity = None
73
74 for entity in ner_results:
75 entity_tag = entity['entity']
76 if entity_tag.startswith('B-'):
77 if current_entity is not None:
78 grouped_entities.append(current_entity)
79 current_entity = {
80 'word': entity['word'],
81 'entity': entity_tag[2:],
82 'start': entity['start'],
83 'end': entity['end']
84 }
85 elif entity_tag.startswith('I-'):
86 if current_entity is not None and current_entity['entity'] == entity_tag[2:]:
87 current_entity['word'] += entity['word']
88 current_entity['end'] = entity['end']
89 else:
90 if current_entity is not None:
91 grouped_entities.append(current_entity)
92 current_entity = None
93 elif entity_tag.startswith('E-'):
94 if current_entity is not None and current_entity['entity'] == entity_tag[2:]:
95 current_entity['word'] += entity['word']
96 current_entity['end'] = entity['end']
97 grouped_entities.append(current_entity)
98 current_entity = None
99 else:
100 if current_entity is not None:
101 grouped_entities.append(current_entity)
102 current_entity = None
103 else: # 'S-' and 'O-' tags
104 if current_entity is not None:
105 grouped_entities.append(current_entity)
106 grouped_entities.append(entity)
107 current_entity = None
108
109 if current_entity is not None:
110 grouped_entities.append(current_entity)
111
112 return grouped_entities
113
114
115model = "praramnine/isl-camembert-beauty-aspect-v1"
116tokenizer = AutoTokenizer.from_pretrained(model)
117config = AutoConfig.from_pretrained(model)
118model = AutoModelForTokenClassification.from_pretrained(model)
119
120nlp = pipeline("ner", model=model, tokenizer=tokenizer)
121text = "แต่ต้องใช้รองพื้นช่วย แต่ใช้แล้วรู้สึกบางเบาแล้วหน้าขาวเนียนๆดี เนื้อครีมเราว่าดูหนืดๆไปหน่อย แต่รวมๆชอบนะค่ะ แต่ถ้าหมดแล้วอาจลองตัวอื่นหน่อย"
122text = text_processing_newmm(text)
123text = parse_tokens(text)
124formatted_text = format_words(text)
125
126ner_results = nlp(formatted_text)
127ner_results_cleaned = [entity for entity in ner_results if entity['word'] != '▁']
128
129print(ner_results_cleaned)
130[{'entity': 'B-Price', 'score': 0.993042, 'index': 12, 'word': 'ราคา', 'start': 22, 'end': 26}, {'entity': 'I-Price', 'score': 0.9918081, 'index': 14, 'word': 'ไม่', 'start': 27, 'end': 30}, {'entity': 'I-Price', 'score': 0.9626237, 'index': 16, 'word': 'แพง', 'start': 31, 'end': 34}, {'entity': 'B-Quality', 'score': 0.9992849, 'index': 25, 'word': 'ไม่', 'start': 52, 'end': 55}, {'entity': 'E-Quality', 'score': 0.9492479, 'index': 27, 'word': 'แพ้', 'start': 56, 'end': 59}, {'entity': 'B-Quality', 'score': 0.9918962, 'index': 29, 'word': 'สิว', 'start': 60, 'end': 63}, {'entity': 'I-Quality', 'score': 0.99981385, 'index': 31, 'word': 'ไม่', 'start': 64, 'end': 67}, {'entity': 'E-Quality', 'score': 0.9970843, 'index': 33, 'word': 'ขึ้น', 'start': 68, 'end': 72}]1grouped_entities = group_entities_iobes(ner_results_cleaned)
2print(grouped_entities)[{'word': 'ราคาไม่แพง', 'entity': 'Price', 'start': 22, 'end': 34}, {'word': 'ไม่แพ้', 'entity': 'Quality', 'start': 52, 'end': 59}, {'word': 'สิวไม่ขึ้น', 'entity': 'Quality', 'start': 60, 'end': 72}]