Views
No views yet
1import tensorflow as tf
2from tensorflow.keras.preprocessing.text import tokenizer_from_json
3import json
4import numpy as np
5
6# --- Load the Keras model ---
7loaded_model = tf.keras.models.load_model("darija-text-normalizer.keras")
8encoder_model = tf.keras.models.Model(loaded_model.input[0], loaded_model.layers[2].output)
9
10# Get latent dimension from the model configuration
11latent_dim_loaded = loaded_model.layers[3].get_config()['units']
12decoder_state_input_h = tf.keras.layers.Input(shape=(latent_dim_loaded,))
13decoder_state_input_c = tf.keras.layers.Input(shape=(latent_dim_loaded,))
14decoder_states_inputs = [decoder_state_input_h, decoder_state_input_c]
15decoder_embedding_layer = loaded_model.layers[2]
16decoder_lstm_layer = loaded_model.layers[3]
17decoder_dense_layer = loaded_model.layers[4]
18
19decoder_embedding_inf = decoder_embedding_layer(loaded_model.input[1])
20decoder_outputs_inf, state_h_inf, state_c_inf = decoder_lstm_layer(
21 decoder_embedding_inf, initial_state=decoder_states_inputs
22)
23decoder_states_inf = [state_h_inf, state_c_inf]
24decoder_outputs_inf = decoder_dense_layer(decoder_outputs_inf)
25decoder_model = tf.keras.models.Model(
26 [loaded_model.input[1]] + decoder_states_inputs,
27 [decoder_outputs_inf] + decoder_states_inf
28)
29
30# --- Load Tokenizers ---
31with open("tokenizer_input.json", 'r', encoding='utf-8') as f:
32 tokenizer_input_config = json.load(f)
33tokenizer_input = tokenizer_from_json(tokenizer_input_config)
34
35with open("tokenizer_target.json", 'r', encoding='utf-8') as f:
36 tokenizer_target_config = json.load(f)
37tokenizer_target = tokenizer_from_json(tokenizer_target_config)
38
39# --- Load Model Parameters ---
40with open("model_parameters.json", 'r', encoding='utf-8') as f:
41 model_params = json.load(f)
42 max_input_len = model_params['max_input_len']
43 max_target_len = model_params['max_target_len']
44
45def normalize_text(input_text, encoder_model, decoder_model, input_tokenizer, target_tokenizer, max_target_len, max_input_len):
46 """Normalizes input Darija text using the trained encoder-decoder model."""
47 input_seq = input_tokenizer.texts_to_sequences([input_text])
48 padded_input_seq = tf.keras.preprocessing.sequence.pad_sequences(input_seq, maxlen=max_input_len, padding='post')
49 states_value = encoder_model.predict(padded_input_seq, verbose=0)
50
51 target_seq = [target_tokenizer.word_index.get(target_tokenizer.oov_token, 0)]
52 if target_seq[0] is None:
53 target_seq = [0]
54 target_seq = np.array(target_seq).reshape(1, 1)
55
56 decoded_sentence = ''
57 stop_condition = False
58 while not stop_condition:
59 output_tokens, h, c = decoder_model.predict([target_seq] + states_value, verbose=0)
60 sampled_token_index = np.argmax(output_tokens[0, -1, :])
61 sampled_char = target_tokenizer.index_word.get(sampled_token_index, '')
62
63 if sampled_char and sampled_char != target_tokenizer.oov_token:
64 decoded_sentence += sampled_char
65
66 if sampled_char == '' or sampled_char == target_tokenizer.oov_token or len(decoded_sentence) > max_target_len:
67 stop_condition = True
68
69 target_seq = np.array([sampled_token_index]).reshape(1, 1)
70 states_value = [h, c]
71
72 return decoded_sentence
73
74# --- Example ---
75input_text = "kn-mchiw l-sou9" # Example input (Darija for "We are going to the market")
76print("Input text:", input_text)
77print("Normalized text:", normalize_text(input_text, encoder_model, decoder_model, tokenizer_input, tokenizer_target, max_target_len, max_input_len))pip install tensorflow numpy pyyaml huggingface_hubHF_API_TOKEN = "YOUR_HF_API_TOKEN" in the code with your actual token.HF_USERNAME = "YOUR_HF_USERNAME" in the code with your actual Hugging Face username. This is the username you use to log in to Hugging Face.huggingface-cli logintensorflow, numpy, pyyaml, huggingface_hub).