Views
No views yet
1import tensorflow as tf
2from tokenizers import Tokenizer, models, pre_tokenizers, trainers, Regex
3import tokenizers
4from tokenizers import Tokenizer, models, decoders, processors
5from tokenizers import pre_tokenizers, trainers, Regex
6import huggingface_hub
7
8class Attention(tf.keras.layers.Layer):
9 def __init__(self,
10 units=128, **kwargs):
11 super(Attention,self).__init__(**kwargs)
12 self.units = units
13
14 def build(self, input_shape):
15 self.W1=self.add_weight(name='attention_weights_1', shape=(input_shape[-1], self.units),
16 initializer='glorot_uniform', trainable=True)
17
18 self.W2=self.add_weight(name='attention_weights_2', shape=(1, self.units),
19 initializer='glorot_uniform', trainable=True)
20
21 super(Attention, self).build(input_shape)
22
23 def call(self, x):
24 x = tf.transpose(x, perm=[0, 2, 1])
25 attention = tf.nn.softmax(tf.matmul(self.W2, tf.nn.tanh(tf.matmul(self.W1, x))))
26 weighted_context = tf.reduce_sum(x * attention, axis=-1)
27 return weighted_context, attention
28
29 def get_config(self):
30 config = super().get_config().copy()
31 config.update({
32 'units': self.units
33 })
34 return config
35
36#download model
37model = tf.keras.models.load_model(huggingface_hub.hf_hub_download('vkovenko/deep_lstm_attention_ukr_reviews_rating_estimation',
38 'deep_lstm_attention_w2v_huber.h5',
39 local_dir='model'),
40 compile=False,
41 custom_objects={'Attention':Attention})
42
43
44class BPETokenizer:
45 def __init__(self, vocab, merges):
46 self.suffix = '</w>'
47 self.tokenizer = Tokenizer(models.BPE.from_file(vocab=vocab,
48 merges=merges, end_of_word_suffix=self.suffix))
49 self.tokenizer.pre_tokenizer = pre_tokenizers.Split(Regex(r"[\w'-]+|[^\w\s'-]+"),'removed', True)
50 self.id_to_token = self.tokenizer.id_to_token
51 self.encode_batch = self.tokenizer.encode_batch
52 self.token_to_id = self.tokenizer.token_to_id
53 self.encode = self.tokenizer.encode
54
55 def tokens_to_ids(self, tokens):
56 return list(map(self.token_to_id, tokens))
57
58 def ids_to_tokens(self, ids):
59 return list(map(self.id_to_token, ids))
60
61
62 def decode(self, tokens, return_indices=False):
63 decoded = []
64 merged_indices = []
65 i = 0
66 while i<len(tokens):
67 if tokens[i].endswith(self.suffix):
68 decoded.append(tokens[i])
69 merged_indices.append([i])
70 i+=1
71 else:
72 merged_token = ''
73 tmp_indc = []
74 while not tokens[i].endswith(self.suffix):
75 merged_token+=tokens[i]
76 tmp_indc.append(i)
77 i+=1
78 merged_token+=tokens[i]
79 tmp_indc.append(i)
80 decoded.append(merged_token)
81 merged_indices.append(tmp_indc)
82 i+=1
83
84 if return_indices:
85 return decoded, merged_indices
86 else:
87 return decoded
88#download tokenizer
89tokenizer = BPETokenizer(vocab=huggingface_hub.hf_hub_download('vkovenko/deep_lstm_attention_ukr_reviews_rating_estimation',
90 'tokenizer_30k.json',
91 local_dir='model'),
92 merges=huggingface_hub.hf_hub_download('vkovenko/deep_lstm_attention_ukr_reviews_rating_estimation',
93 'merges_tokenizer.txt',
94 local_dir='model')
95 )
96