Views
No views yet
1from huggingface_hub import from_pretrained_keras
2from huggingface_hub import hf_hub_download
3import tensorflow as tf
4import numpy as np
5import string
6import re
7
8# Select characters to strip, but preserve the "[" and "]"
9strip_chars = string.punctuation
10strip_chars = strip_chars.replace("[", "")
11strip_chars = strip_chars.replace("]", "")
12
13def custom_standardization(input_string):
14 lowercase = tf.strings.lower(input_string)
15 return tf.strings.regex_replace(lowercase, f"[{re.escape(strip_chars)}]", "")
16
17# Load the `seq2seq_rnn` from the Hub
18seq2seq_rnn = from_pretrained_keras("AiresPucrs/GRU-eng-por")
19
20# Load the portuguese vocabulary
21portuguese_vocabulary_path = hf_hub_download(
22 repo_id="AiresPucrs/GRU-eng-por",
23 filename="portuguese_vocabulary.txt",
24 repo_type='model',
25 local_dir="./")
26
27# Load the english vocabulary
28english_vocabulary_path = hf_hub_download(
29 repo_id="AiresPucrs/GRU-eng-por",
30 filename="english_vocabulary.txt",
31 repo_type='model',
32 local_dir="./")
33
34with open(portuguese_vocabulary_path, encoding='utf-8', errors='backslashreplace') as fp:
35 portuguese_vocab = [line.strip() for line in fp]
36 fp.close()
37
38with open(english_vocabulary_path, encoding='utf-8', errors='backslashreplace') as fp:
39 english_vocab = [line.strip() for line in fp]
40 fp.close()
41
42# Initialize the vectorizers with the learned vocabularies
43target_vectorization = tf.keras.layers.TextVectorization(max_tokens=20000,
44 output_mode="int",
45 output_sequence_length=21,
46 standardize=custom_standardization,
47 vocabulary=portuguese_vocab)
48
49source_vectorization = tf.keras.layers.TextVectorization(max_tokens=20000,
50 output_mode="int",
51 output_sequence_length=20,
52 vocabulary=english_vocab)
53
54# Create a dictionary from `int`to portuguese words
55portuguese_index_lookup = dict(zip(range(len(portuguese_vocab)), portuguese_vocab))
56max_decoded_sentence_length = 20
57
58def decode_sequence(input_sentence):
59 """
60 Decodes a sequence using a trained seq2seq RNN model.
61
62 Args:
63 input_sentence (str): the input sentence to be decoded
64
65 Returns:
66 decoded_sentence (str): the decoded sentence
67 generated by the model
68 """
69 tokenized_input_sentence = source_vectorization([input_sentence])
70 decoded_sentence = "[start]"
71
72 for i in range(max_decoded_sentence_length):
73 tokenized_target_sentence = target_vectorization([decoded_sentence])
74 next_token_predictions = seq2seq_rnn.predict([tokenized_input_sentence, tokenized_target_sentence], verbose=0)
75 sampled_token_index = np.argmax(next_token_predictions[0, i, :])
76 sampled_token = portuguese_index_lookup[sampled_token_index]
77 decoded_sentence += " " + sampled_token
78 if sampled_token == "[end]":
79 break
80 return decoded_sentence
81
82eng_sentences =["What is its name?",
83 "How old are you?",
84 "I know you know where Mary is.",
85 "We will show Tom.",
86 "What do you all do?",
87 "Don't do it!"]
88
89for sentence in eng_sentences:
90 print(f"English sentence:\n{sentence}")
91 print(f'Portuguese translation:\n{decode_sequence(sentence)}')
92 print('-' * 50)
93
94