Views
No views yet
1import tensorflow as tf
2import numpy as np
3import string
4import keras
5import re
6
7strip_chars = string.punctuation
8strip_chars = strip_chars.replace("[", "")
9strip_chars = strip_chars.replace("]", "")
10
11
12def custom_standardization(input_string):
13 lowercase = tf.strings.lower(input_string)
14 return tf.strings.regex_replace(lowercase, f"[{re.escape(strip_chars)}]", "")
15
16portuguese_vocabulary_path = hf_hub_download(
17 repo_id="AiresPucrs/transformer-eng-por",
18 filename="keras_transformer_blocks.py",
19 repo_type='model',
20 local_dir="./")
21
22from keras_transformer_blocks import TransformerEncoder, PositionalEmbedding, TransformerDecoder
23
24transformer = keras.models.load_model("./transformer-eng-por/transformer-eng-por.h5",
25 custom_objects={"TransformerEncoder": TransformerEncoder,
26 "PositionalEmbedding": PositionalEmbedding,
27 "TransformerDecoder": TransformerDecoder})
28
29with open('portuguese_vocabulary.txt', encoding='utf-8', errors='backslashreplace') as fp:
30 portuguese_vocab = [line.strip() for line in fp]
31 fp.close()
32
33with open('english_vocabulary.txt', encoding='utf-8', errors='backslashreplace') as fp:
34 english_vocab = [line.strip() for line in fp]
35 fp.close()
36
37
38target_vectorization = tf.keras.layers.TextVectorization(max_tokens=20000,
39 output_mode="int",
40 output_sequence_length=21,
41 standardize=custom_standardization,
42 vocabulary=portuguese_vocab)
43
44source_vectorization = tf.keras.layers.TextVectorization(max_tokens=20000,
45 output_mode="int",
46 output_sequence_length=20,
47 vocabulary=english_vocab)
48
49portuguese_index_lookup = dict(zip(range(len(portuguese_vocab)), portuguese_vocab))
50max_decoded_sentence_length = 20
51
52
53def decode_sequence(input_sentence):
54 tokenized_input_sentence = source_vectorization([input_sentence])
55 decoded_sentence = "[start]"
56
57 for i in range(max_decoded_sentence_length):
58 tokenized_target_sentence = target_vectorization([decoded_sentence])[:, :-1]
59 predictions = transformer([tokenized_input_sentence, tokenized_target_sentence])
60 sampled_token_index = np.argmax(predictions[0, i, :])
61 sampled_token = portuguese_index_lookup[sampled_token_index]
62 decoded_sentence += " " + sampled_token
63 if sampled_token == "[end]":
64 break
65 return decoded_sentence
66
67
68eng_sentences =["What is its name?",
69 "How old are you?",
70 "I know you know where Mary is.",
71 "We will show Tom.",
72 "What do you all do?",
73 "Don't do it!"]
74
75for sentence in eng_sentences:
76 print(f"English sentence:\n{sentence}")
77 print(f'Portuguese translation:\n{decode_sequence(sentence)}')
78 print('-' * 50)