From ca006ece74cae9fcfbb3a748d527b725b2de5889 Mon Sep 17 00:00:00 2001 From: Francois Chollet Date: Tue, 16 May 2023 13:25:48 -0700 Subject: [PATCH] Increase GRU + LSTM test coverage. --- keras_core/backend/tensorflow/rnn.py | 11 ++- keras_core/layers/rnn/gru_test.py | 120 +++++++++++++++++++++++++- keras_core/layers/rnn/lstm_test.py | 123 ++++++++++++++++++++++++++- 3 files changed, 250 insertions(+), 4 deletions(-) diff --git a/keras_core/backend/tensorflow/rnn.py b/keras_core/backend/tensorflow/rnn.py index 59c8f53a6..14903a81d 100644 --- a/keras_core/backend/tensorflow/rnn.py +++ b/keras_core/backend/tensorflow/rnn.py @@ -574,7 +574,12 @@ def _is_sequence_right_padded(mask): max_seq_length = tf.shape(mask)[1] count_of_true = tf.reduce_sum(tf.cast(mask, tf.int32), axis=1) right_padded_mask = tf.sequence_mask(count_of_true, maxlen=max_seq_length) - return tf.reduce_all(tf.equal(mask, right_padded_mask)) + return tf.reduce_all( + tf.equal( + tf.cast(mask, dtype="bool"), + tf.cast(right_padded_mask, dtype="bool"), + ) + ) def _has_fully_masked_sequence(mask): @@ -583,7 +588,9 @@ def _has_fully_masked_sequence(mask): # to standard kernel, until the issue on cudnn side has been fixed. For a # fully masked sequence, it will contain all Falses. To make it easy to # check, we inverse the boolean, check if any of the sequence has all True. - return tf.reduce_any(tf.reduce_all(tf.logical_not(mask), axis=1)) + return tf.reduce_any( + tf.reduce_all(tf.logical_not(tf.cast(mask, dtype="bool")), axis=1) + ) def _standardize_cudnn_weights(weights, biases, shape, transpose_weights=False): diff --git a/keras_core/layers/rnn/gru_test.py b/keras_core/layers/rnn/gru_test.py index aaf18e1d1..0603e5d04 100644 --- a/keras_core/layers/rnn/gru_test.py +++ b/keras_core/layers/rnn/gru_test.py @@ -165,4 +165,122 @@ class GRUTest(testing.TestCase, parameterized.TestCase): output, ) - # TODO: test masking + def test_pass_initial_state(self): + sequence = np.arange(24).reshape((2, 4, 3)).astype("float32") + initial_state = np.arange(4).reshape((2, 2)).astype("float32") + layer = layers.GRU( + 2, + kernel_initializer=initializers.Constant(0.01), + recurrent_initializer=initializers.Constant(0.02), + bias_initializer=initializers.Constant(0.03), + ) + output = layer(sequence, initial_state=initial_state) + self.assertAllClose( + np.array([[0.23774096, 0.33508456], [0.83659905, 1.0227708]]), + output, + ) + + layer = layers.GRU( + 2, + kernel_initializer=initializers.Constant(0.01), + recurrent_initializer=initializers.Constant(0.02), + bias_initializer=initializers.Constant(0.03), + go_backwards=True, + ) + output = layer(sequence, initial_state=initial_state) + self.assertAllClose( + np.array([[0.13486053, 0.23261218], [0.78257304, 0.9691353]]), + output, + ) + + def test_masking(self): + sequence = np.arange(24).reshape((2, 4, 3)).astype("float32") + mask = np.array([[True, True, False, True], [True, False, False, True]]) + layer = layers.GRU( + 2, + kernel_initializer=initializers.Constant(0.01), + recurrent_initializer=initializers.Constant(0.02), + bias_initializer=initializers.Constant(0.03), + unroll=True, + ) + output = layer(sequence, mask=mask) + self.assertAllClose( + np.array([[0.19393763, 0.19393763], [0.30818558, 0.30818558]]), + output, + ) + + layer = layers.GRU( + 2, + kernel_initializer=initializers.Constant(0.01), + recurrent_initializer=initializers.Constant(0.02), + bias_initializer=initializers.Constant(0.03), + return_sequences=True, + ) + output = layer(sequence, mask=mask) + self.assertAllClose( + np.array( + [ + [0.03606692, 0.03606692], + [0.09497581, 0.09497581], + [0.09497581, 0.09497581], + [0.19393763, 0.19393763], + ], + ), + output[0], + ) + self.assertAllClose( + np.array( + [ + [0.16051409, 0.16051409], + [0.16051409, 0.16051409], + [0.16051409, 0.16051409], + [0.30818558, 0.30818558], + ], + ), + output[1], + ) + + layer = layers.GRU( + 2, + kernel_initializer=initializers.Constant(0.01), + recurrent_initializer=initializers.Constant(0.02), + bias_initializer=initializers.Constant(0.03), + return_sequences=True, + zero_output_for_mask=True, + ) + output = layer(sequence, mask=mask) + self.assertAllClose( + np.array( + [ + [0.03606692, 0.03606692], + [0.09497581, 0.09497581], + [0.0, 0.0], + [0.19393763, 0.19393763], + ], + ), + output[0], + ) + self.assertAllClose( + np.array( + [ + [0.16051409, 0.16051409], + [0.0, 0.0], + [0.0, 0.0], + [0.30818558, 0.30818558], + ], + ), + output[1], + ) + + layer = layers.GRU( + 2, + kernel_initializer=initializers.Constant(0.01), + recurrent_initializer=initializers.Constant(0.02), + bias_initializer=initializers.Constant(0.03), + go_backwards=True, + ) + output = layer(sequence, mask=mask) + self.assertAllClose( + np.array([[0.11669192, 0.11669192], [0.28380975, 0.28380975]]), + output, + ) diff --git a/keras_core/layers/rnn/lstm_test.py b/keras_core/layers/rnn/lstm_test.py index 26b321107..3ef23677a 100644 --- a/keras_core/layers/rnn/lstm_test.py +++ b/keras_core/layers/rnn/lstm_test.py @@ -170,4 +170,125 @@ class LSTMTest(testing.TestCase, parameterized.TestCase): output, ) - # TODO: test masking + def test_pass_initial_state(self): + sequence = np.arange(24).reshape((2, 4, 3)).astype("float32") + initial_state = [ + np.arange(4).reshape((2, 2)).astype("float32"), + np.arange(4).reshape((2, 2)).astype("float32"), + ] + layer = layers.LSTM( + 2, + kernel_initializer=initializers.Constant(0.01), + recurrent_initializer=initializers.Constant(0.02), + bias_initializer=initializers.Constant(0.03), + ) + output = layer(sequence, initial_state=initial_state) + self.assertAllClose( + np.array([[0.20574439, 0.3558822], [0.64930826, 0.66276]]), + output, + ) + + layer = layers.LSTM( + 2, + kernel_initializer=initializers.Constant(0.01), + recurrent_initializer=initializers.Constant(0.02), + bias_initializer=initializers.Constant(0.03), + go_backwards=True, + ) + output = layer(sequence, initial_state=initial_state) + self.assertAllClose( + np.array([[0.13281618, 0.2790356], [0.5839337, 0.5992567]]), + output, + ) + + def test_masking(self): + sequence = np.arange(24).reshape((2, 4, 3)).astype("float32") + mask = np.array([[True, True, False, True], [True, False, False, True]]) + layer = layers.LSTM( + 2, + kernel_initializer=initializers.Constant(0.01), + recurrent_initializer=initializers.Constant(0.02), + bias_initializer=initializers.Constant(0.03), + unroll=True, + ) + output = layer(sequence, mask=mask) + self.assertAllClose( + np.array([[0.1524914, 0.1524914], [0.35969394, 0.35969394]]), + output, + ) + + layer = layers.LSTM( + 2, + kernel_initializer=initializers.Constant(0.01), + recurrent_initializer=initializers.Constant(0.02), + bias_initializer=initializers.Constant(0.03), + return_sequences=True, + ) + output = layer(sequence, mask=mask) + self.assertAllClose( + np.array( + [ + [0.0158891, 0.0158891], + [0.05552047, 0.05552047], + [0.05552047, 0.05552047], + [0.1524914, 0.1524914], + ], + ), + output[0], + ) + self.assertAllClose( + np.array( + [ + [0.14185596, 0.14185596], + [0.14185596, 0.14185596], + [0.14185596, 0.14185596], + [0.35969394, 0.35969394], + ], + ), + output[1], + ) + + layer = layers.LSTM( + 2, + kernel_initializer=initializers.Constant(0.01), + recurrent_initializer=initializers.Constant(0.02), + bias_initializer=initializers.Constant(0.03), + return_sequences=True, + zero_output_for_mask=True, + ) + output = layer(sequence, mask=mask) + self.assertAllClose( + np.array( + [ + [0.0158891, 0.0158891], + [0.05552047, 0.05552047], + [0.0, 0.0], + [0.1524914, 0.1524914], + ], + ), + output[0], + ) + self.assertAllClose( + np.array( + [ + [0.14185596, 0.14185596], + [0.0, 0.0], + [0.0, 0.0], + [0.35969394, 0.35969394], + ], + ), + output[1], + ) + + layer = layers.LSTM( + 2, + kernel_initializer=initializers.Constant(0.01), + recurrent_initializer=initializers.Constant(0.02), + bias_initializer=initializers.Constant(0.03), + go_backwards=True, + ) + output = layer(sequence, mask=mask) + self.assertAllClose( + np.array([[0.10056866, 0.10056866], [0.31006062, 0.31006062]]), + output, + )