From 0daeca44de3df3acc2c5ef0a36b6b268607d6859 Mon Sep 17 00:00:00 2001 From: Vaibhav Pathak Date: Mon, 27 Mar 2023 11:14:27 +0530 Subject: [PATCH 01/11] Updated hard_tanh and added to layer_types --- .../ann/layer/{not_adapted => }/hard_tanh.hpp | 10 +++--- .../{not_adapted => }/hard_tanh_impl.hpp | 31 ++++++++++--------- src/mlpack/methods/ann/layer/layer_types.hpp | 1 + 3 files changed, 23 insertions(+), 19 deletions(-) rename src/mlpack/methods/ann/layer/{not_adapted => }/hard_tanh.hpp (91%) rename src/mlpack/methods/ann/layer/{not_adapted => }/hard_tanh_impl.hpp (59%) diff --git a/src/mlpack/methods/ann/layer/not_adapted/hard_tanh.hpp b/src/mlpack/methods/ann/layer/hard_tanh.hpp similarity index 91% rename from src/mlpack/methods/ann/layer/not_adapted/hard_tanh.hpp rename to src/mlpack/methods/ann/layer/hard_tanh.hpp index 5f58aa2d3c..d351efc46c 100644 --- a/src/mlpack/methods/ann/layer/not_adapted/hard_tanh.hpp +++ b/src/mlpack/methods/ann/layer/hard_tanh.hpp @@ -44,8 +44,8 @@ namespace mlpack { * to also be in this type. The type also allows the computation and weight * type to differ from the input type (Default: arma::mat). */ -template -class HardTanHType : public Layer +template +class HardTanHType : public Layer { public: /** @@ -68,7 +68,7 @@ class HardTanHType : public Layer * @param input Input data used for evaluating the specified function. * @param output Resulting output activation. */ - void Forward(const InputType& input, OutputType& output); + void Forward(const MatType& input, MatType& output); /** * Ordinary feed backward pass of a neural network, calculating the function @@ -79,7 +79,7 @@ class HardTanHType : public Layer * @param gy The backpropagated error. * @param g The calculated gradient. */ - void Backward(const InputType& input, const OutputType& gy, OutputType& g); + void Backward(const MatType& input, const MatType& gy, MatType& g); //! Get the maximum value. double const& MaxValue() const { return maxValue; } @@ -108,7 +108,7 @@ class HardTanHType : public Layer // Convenience typedefs. // Standard HardTanH layer. -typedef HardTanHType HardTanH; +typedef HardTanHType HardTanH; } // namespace mlpack diff --git a/src/mlpack/methods/ann/layer/not_adapted/hard_tanh_impl.hpp b/src/mlpack/methods/ann/layer/hard_tanh_impl.hpp similarity index 59% rename from src/mlpack/methods/ann/layer/not_adapted/hard_tanh_impl.hpp rename to src/mlpack/methods/ann/layer/hard_tanh_impl.hpp index 129a061008..e920e0a3d6 100644 --- a/src/mlpack/methods/ann/layer/not_adapted/hard_tanh_impl.hpp +++ b/src/mlpack/methods/ann/layer/hard_tanh_impl.hpp @@ -1,6 +1,7 @@ /** * @file methods/ann/layer/hard_tanh_impl.hpp * @author Dhawal Arora + * @author Vaibhav Pathak * * Implementation and implementation of the HardTanH layer. * @@ -17,8 +18,8 @@ namespace mlpack { -template -HardTanHType::HardTanHType( +template +HardTanHType::HardTanHType( const double maxValue, const double minValue) : maxValue(maxValue), @@ -27,38 +28,40 @@ HardTanHType::HardTanHType( // Nothing to do here. } -template -void HardTanHType::Forward( - const InputType& input, OutputType& output) +template +void HardTanHType::Forward( + const MatType& input, MatType& output) { for (size_t i = 0; i < input.n_elem; ++i) { - output(i) = (output(i) > maxValue ? maxValue : - (output(i) < minValue ? minValue : output(i))); + output(i) = (input(i) > maxValue ? maxValue : + (input(i) < minValue ? minValue : input(i))); } } -template -void HardTanHType::Backward( - const InputType& input, const OutputType& gy, OutputType& g) +template +void HardTanHType::Backward( + const MatType& input, const MatType& gy, MatType& g) { g = gy; for (size_t i = 0; i < input.n_elem; ++i) { - if (input(i) < minValue || input(i) > maxValue) + // input should not have any values greater than maxValue + // and lesser than minValue + if (input(i) <= minValue || input(i) >= maxValue) { g(i) = 0; } } } -template +template template -void HardTanHType::serialize( +void HardTanHType::serialize( Archive& ar, const uint32_t /* version */) { - ar(cereal::base_class>(this)); + ar(cereal::base_class>(this)); ar(CEREAL_NVP(maxValue)); ar(CEREAL_NVP(minValue)); diff --git a/src/mlpack/methods/ann/layer/layer_types.hpp b/src/mlpack/methods/ann/layer/layer_types.hpp index dc12f47d7f..5d348a14b1 100644 --- a/src/mlpack/methods/ann/layer/layer_types.hpp +++ b/src/mlpack/methods/ann/layer/layer_types.hpp @@ -32,6 +32,7 @@ #include #include #include +#include #include #include #include From 23a9a14f7234d1f94aa6cc95de09186781e588de Mon Sep 17 00:00:00 2001 From: Vaibhav Pathak Date: Mon, 27 Mar 2023 13:17:38 +0530 Subject: [PATCH 02/11] Added simple test for hard_tanh --- src/mlpack/methods/ann/layer/hard_tanh.hpp | 1 + .../methods/ann/layer/serialization.hpp | 2 +- src/mlpack/tests/ann/layer/hard_tanh.cpp | 42 +++++++++++++++++++ src/mlpack/tests/ann/layer_test.cpp | 1 + 4 files changed, 45 insertions(+), 1 deletion(-) create mode 100644 src/mlpack/tests/ann/layer/hard_tanh.cpp diff --git a/src/mlpack/methods/ann/layer/hard_tanh.hpp b/src/mlpack/methods/ann/layer/hard_tanh.hpp index d351efc46c..849eaad3f7 100644 --- a/src/mlpack/methods/ann/layer/hard_tanh.hpp +++ b/src/mlpack/methods/ann/layer/hard_tanh.hpp @@ -1,6 +1,7 @@ /** * @file methods/ann/layer/hard_tanh.hpp * @author Dhawal Arora + * @author Vaibhav Pathak * * Definition and implementation of the HardTanH layer. * diff --git a/src/mlpack/methods/ann/layer/serialization.hpp b/src/mlpack/methods/ann/layer/serialization.hpp index 5819a5d604..d179ea9167 100644 --- a/src/mlpack/methods/ann/layer/serialization.hpp +++ b/src/mlpack/methods/ann/layer/serialization.hpp @@ -69,7 +69,7 @@ CEREAL_REGISTER_TYPE(mlpack::RBFType<__VA_ARGS__>); \ CEREAL_REGISTER_TYPE(mlpack::SoftmaxType<__VA_ARGS__>); \ CEREAL_REGISTER_TYPE(mlpack::SoftminType<__VA_ARGS__>); \ - + CEREAL_REGISTER_TYPE(mlpack::HardTanHType<__VA_ARGS__>); \ CEREAL_REGISTER_MLPACK_LAYERS(arma::mat); #endif diff --git a/src/mlpack/tests/ann/layer/hard_tanh.cpp b/src/mlpack/tests/ann/layer/hard_tanh.cpp new file mode 100644 index 0000000000..76d71e4e3b --- /dev/null +++ b/src/mlpack/tests/ann/layer/hard_tanh.cpp @@ -0,0 +1,42 @@ +/* + * @file tests/ann/layer/hard_tanh.cpp + * @author Vaibhav Pathak + * + * Tests the hard_tanh layer + * + */ + +#include +#include + +#include "../../test_catch_tools.hpp" +#include "../../catch.hpp" +#include "../../serialization.hpp" +#include "../ann_test_tools.hpp" + +using namespace mlpack; + +/** + * Simple HardTanH module test + */ + +TEST_CASE("SimpleHardTanHTest", "[ANNLayerTest]") +{ + arma::mat input, output, gy, g; + input = arma::mat("-1.3743 -0.5565 0.2742 -0.0151 -1.4871; 1.5797 -4.2711 -2.2505 -1.7105 -1.2544; 0.4023 0.5676 2.3100 1.6658 -0.1907;0.1897 0.9097 0.1418 -1.5349 0.1225; -0.1101 -3.3656 -5.4033 -2.2240 -3.3235"); + arma::mat actualOutput("-1.0000 -0.5565 0.2742 -0.0151 -1.0000; 1.0000 -1.0000 -1.0000 -1.0000 -1.0000; 0.4023 0.5676 1.0000 1.0000 -0.1907; 0.1897 0.9097 0.1418 -1.0000 0.1225; -0.1101 -1.0000 -1.0000 -1.0000 -1.0000"); + + HardTanH module; + + // Test the Forward function + module.Forward(input, output); + REQUIRE(arma::accu(output - actualOutput) == Approx(0).epsilon(1e-4)); + + arma::mat delta("0 1.0000 1.0000 1.0000 0; 0 0 0 0 0; 1.0000 1.0000 0 0 1.0000; 1.0000 1.0000 1.0000 0 1.0000; 1.0000 0 0 0 0"); + gy.set_size(5,5); + gy.fill(1); + g.set_size(5,5); + //Test the Backward function + module.Backward(output, gy, g); + REQUIRE(arma::accu(g - delta) == Approx(0).epsilon(1e-4)); +} diff --git a/src/mlpack/tests/ann/layer_test.cpp b/src/mlpack/tests/ann/layer_test.cpp index 73456b23e9..32bb25425b 100644 --- a/src/mlpack/tests/ann/layer_test.cpp +++ b/src/mlpack/tests/ann/layer_test.cpp @@ -26,6 +26,7 @@ #include "layer/concatenate.cpp" #include "layer/dropout.cpp" #include "layer/grouped_convolution.cpp" +#include "layer/hard_tanh.cpp" #include "layer/identity.cpp" #include "layer/linear3d.cpp" #include "layer/linear_no_bias.cpp" From 948ac5b288a8c5592cb7184d2477d101d11cfc94 Mon Sep 17 00:00:00 2001 From: Vaibhav Pathak Date: Mon, 27 Mar 2023 18:29:47 +0530 Subject: [PATCH 03/11] Update hard_tanh test --- src/mlpack/tests/ann/layer/hard_tanh.cpp | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/src/mlpack/tests/ann/layer/hard_tanh.cpp b/src/mlpack/tests/ann/layer/hard_tanh.cpp index 76d71e4e3b..f367b73464 100644 --- a/src/mlpack/tests/ann/layer/hard_tanh.cpp +++ b/src/mlpack/tests/ann/layer/hard_tanh.cpp @@ -22,17 +22,30 @@ using namespace mlpack; TEST_CASE("SimpleHardTanHTest", "[ANNLayerTest]") { - arma::mat input, output, gy, g; - input = arma::mat("-1.3743 -0.5565 0.2742 -0.0151 -1.4871; 1.5797 -4.2711 -2.2505 -1.7105 -1.2544; 0.4023 0.5676 2.3100 1.6658 -0.1907;0.1897 0.9097 0.1418 -1.5349 0.1225; -0.1101 -3.3656 -5.4033 -2.2240 -3.3235"); - arma::mat actualOutput("-1.0000 -0.5565 0.2742 -0.0151 -1.0000; 1.0000 -1.0000 -1.0000 -1.0000 -1.0000; 0.4023 0.5676 1.0000 1.0000 -0.1907; 0.1897 0.9097 0.1418 -1.0000 0.1225; -0.1101 -1.0000 -1.0000 -1.0000 -1.0000"); + arma::mat output, gy, g; + arma::mat input = {{-1.3743, -0.5565, 0.2742, -0.0151, -1.4871}, + {1.5797, -4.2711, -2.2505, -1.7105, -1.2544}, + {0.4023, 0.5676, 2.3100, 1.6658, -0.1907}, + {0.1897, 0.9097, 0.1418, -1.5349, 0.1225}, + {-0.1101, -3.3656, -5.4033, -2.2240, -3.3235}}; + arma::mat actualOutput = {{-1.0000, -0.5565, 0.2742, -0.0151, -1.0000}, + {1.0000, -1.0000, -1.0000, -1.0000, -1.0000}, + {0.4023, 0.5676, 1.0000, 1.0000, -0.1907}, + {0.1897, 0.9097, 0.1418, -1.0000, 0.1225}, + {-0.1101, -1.0000, -1.0000, -1.0000, -1.0000}}; HardTanH module; + output.set_size(5,5); // Test the Forward function module.Forward(input, output); REQUIRE(arma::accu(output - actualOutput) == Approx(0).epsilon(1e-4)); - arma::mat delta("0 1.0000 1.0000 1.0000 0; 0 0 0 0 0; 1.0000 1.0000 0 0 1.0000; 1.0000 1.0000 1.0000 0 1.0000; 1.0000 0 0 0 0"); + arma::mat delta = {{0 , 1.0, 1.0, 1.0, 0.0}, + {0 , 0 , 0 , 0.0, 0.0}, + {1.0, 1.0, 0 , 0.0, 1.0}, + {1.0, 1.0, 1.0, 0.0, 1.0}, + {1.0, 0 , 0.0, 0.0, 0.0}}; gy.set_size(5,5); gy.fill(1); g.set_size(5,5); From 3ee8d8ca2bc984e48140f42d26a9ec9bac7d8457 Mon Sep 17 00:00:00 2001 From: Vaibhav Pathak Date: Mon, 27 Mar 2023 19:02:48 +0530 Subject: [PATCH 04/11] Fixed error in serialization.hpp --- src/mlpack/methods/ann/layer/serialization.hpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/mlpack/methods/ann/layer/serialization.hpp b/src/mlpack/methods/ann/layer/serialization.hpp index d179ea9167..db3e252af4 100644 --- a/src/mlpack/methods/ann/layer/serialization.hpp +++ b/src/mlpack/methods/ann/layer/serialization.hpp @@ -70,6 +70,7 @@ CEREAL_REGISTER_TYPE(mlpack::SoftmaxType<__VA_ARGS__>); \ CEREAL_REGISTER_TYPE(mlpack::SoftminType<__VA_ARGS__>); \ CEREAL_REGISTER_TYPE(mlpack::HardTanHType<__VA_ARGS__>); \ + CEREAL_REGISTER_MLPACK_LAYERS(arma::mat); #endif From 72fba21e28f1b53e6e291b4160c2070fb8e522e8 Mon Sep 17 00:00:00 2001 From: Vaibhav Pathak Date: Mon, 27 Mar 2023 20:57:02 +0530 Subject: [PATCH 05/11] Added copy and move constructors & virtual destructors --- src/mlpack/methods/ann/layer/hard_tanh.hpp | 14 ++++++ .../methods/ann/layer/hard_tanh_impl.hpp | 44 +++++++++++++++++++ 2 files changed, 58 insertions(+) diff --git a/src/mlpack/methods/ann/layer/hard_tanh.hpp b/src/mlpack/methods/ann/layer/hard_tanh.hpp index 849eaad3f7..61c985cb17 100644 --- a/src/mlpack/methods/ann/layer/hard_tanh.hpp +++ b/src/mlpack/methods/ann/layer/hard_tanh.hpp @@ -58,6 +58,20 @@ class HardTanHType : public Layer * @param minValue Range of the linear region minimum value. */ HardTanHType(const double maxValue = 1, const double minValue = -1); + + virtual ~HardTanHType() { } + + //! Copy the other HardTanH layer + HardTanHType(const HardTanHType& layer); + + //! Take ownership of the members of the other HardTanH Layer + HardTanHType(HardTanHType&& layer); + + //! Copy the other HardTanH layer + HardTanHType& operator=(const HardTanHType& layer); + + //! Take ownership of the members of the other HardTanH Layer + HardTanHType& operator=(HardTanHType&& layer); //! Clone the HardTanHType object. This handles polymorphism correctly. HardTanHType* Clone() const { return new HardTanHType(*this); } diff --git a/src/mlpack/methods/ann/layer/hard_tanh_impl.hpp b/src/mlpack/methods/ann/layer/hard_tanh_impl.hpp index e920e0a3d6..0d38fb00f7 100644 --- a/src/mlpack/methods/ann/layer/hard_tanh_impl.hpp +++ b/src/mlpack/methods/ann/layer/hard_tanh_impl.hpp @@ -22,12 +22,56 @@ template HardTanHType::HardTanHType( const double maxValue, const double minValue) : + Layer(), maxValue(maxValue), minValue(minValue) { // Nothing to do here. } +template +HardTanHType::HardTanHType(const HardTanHType& layer) : + Layer(layer), + maxValue(layer.maxValue), + minValue(layer.minValue) +{ + // Nothing to do here. +} + +template +HardTanHType::HardTanHType(HardTanHType&& layer) : + Layer(std::move(layer)), + maxValue(std::move(layer.maxValue)), + minValue(std::move(layer.minValue)) +{ + // Nothing to do here. +} + +template +HardTanHType& HardTanHType::operator=(const HardTanHType& layer) +{ + if(&layer != this) + { + Layer::operator=(layer); + maxValue = layer.maxValue; + minValue = layer.minValue; + } + + return *this; +} + +template +HardTanHType& HardTanHType::operator=(HardTanHType&& layer) +{ + if(&layer != this) + { + Layer::operator=(std::move(layer)); + maxValue = std::move(layer.maxValue); + minValue = std::move(layer.minValue); + } + + return *this; +} template void HardTanHType::Forward( const MatType& input, MatType& output) From a7d3d7bb80437945d9590b22bf15b535c1f9d280 Mon Sep 17 00:00:00 2001 From: Vaibhav Pathak Date: Mon, 27 Mar 2023 22:14:30 +0530 Subject: [PATCH 06/11] Add OpenMP support --- src/mlpack/methods/ann/layer/hard_tanh_impl.hpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/mlpack/methods/ann/layer/hard_tanh_impl.hpp b/src/mlpack/methods/ann/layer/hard_tanh_impl.hpp index 0d38fb00f7..cb2c433e1c 100644 --- a/src/mlpack/methods/ann/layer/hard_tanh_impl.hpp +++ b/src/mlpack/methods/ann/layer/hard_tanh_impl.hpp @@ -76,6 +76,7 @@ template void HardTanHType::Forward( const MatType& input, MatType& output) { + #pragma omp parallel for for (size_t i = 0; i < input.n_elem; ++i) { output(i) = (input(i) > maxValue ? maxValue : @@ -88,6 +89,7 @@ void HardTanHType::Backward( const MatType& input, const MatType& gy, MatType& g) { g = gy; + #pragma omp parallel for for (size_t i = 0; i < input.n_elem; ++i) { // input should not have any values greater than maxValue From a35a9bbc790db6705f423a0f447c109a24c69522 Mon Sep 17 00:00:00 2001 From: Vaibhav Pathak Date: Sun, 2 Apr 2023 19:24:44 +0530 Subject: [PATCH 07/11] Fixed issues related to Code Style --- .../methods/ann/layer/hard_tanh_impl.hpp | 54 +++++++++---------- 1 file changed, 27 insertions(+), 27 deletions(-) diff --git a/src/mlpack/methods/ann/layer/hard_tanh_impl.hpp b/src/mlpack/methods/ann/layer/hard_tanh_impl.hpp index cb2c433e1c..787ada121e 100644 --- a/src/mlpack/methods/ann/layer/hard_tanh_impl.hpp +++ b/src/mlpack/methods/ann/layer/hard_tanh_impl.hpp @@ -22,55 +22,55 @@ template HardTanHType::HardTanHType( const double maxValue, const double minValue) : - Layer(), - maxValue(maxValue), - minValue(minValue) + Layer(), + maxValue(maxValue), + minValue(minValue) { // Nothing to do here. } template HardTanHType::HardTanHType(const HardTanHType& layer) : - Layer(layer), - maxValue(layer.maxValue), - minValue(layer.minValue) + Layer(layer), + maxValue(layer.maxValue), + minValue(layer.minValue) { - // Nothing to do here. + // Nothing to do here. } template HardTanHType::HardTanHType(HardTanHType&& layer) : - Layer(std::move(layer)), - maxValue(std::move(layer.maxValue)), - minValue(std::move(layer.minValue)) + Layer(std::move(layer)), + maxValue(std::move(layer.maxValue)), + minValue(std::move(layer.minValue)) { - // Nothing to do here. + // Nothing to do here. } template HardTanHType& HardTanHType::operator=(const HardTanHType& layer) { - if(&layer != this) - { - Layer::operator=(layer); - maxValue = layer.maxValue; - minValue = layer.minValue; - } + if(&layer != this) + { + Layer::operator=(layer); + maxValue = layer.maxValue; + minValue = layer.minValue; + } - return *this; + return *this; } template HardTanHType& HardTanHType::operator=(HardTanHType&& layer) { - if(&layer != this) - { - Layer::operator=(std::move(layer)); - maxValue = std::move(layer.maxValue); - minValue = std::move(layer.minValue); - } + if(&layer != this) + { + Layer::operator=(std::move(layer)); + maxValue = std::move(layer.maxValue); + minValue = std::move(layer.minValue); + } - return *this; + return *this; } template void HardTanHType::Forward( @@ -92,8 +92,8 @@ void HardTanHType::Backward( #pragma omp parallel for for (size_t i = 0; i < input.n_elem; ++i) { - // input should not have any values greater than maxValue - // and lesser than minValue + // input should not have any values greater than maxValue + // and lesser than minValue if (input(i) <= minValue || input(i) >= maxValue) { g(i) = 0; From c6444e93d1a0d2ffaae8258fb6ecb6392f3d184c Mon Sep 17 00:00:00 2001 From: Vaibhav Pathak Date: Sun, 2 Apr 2023 19:41:56 +0530 Subject: [PATCH 08/11] Fixed Code Style issues --- .../methods/ann/layer/hard_tanh_impl.hpp | 23 ++++++++++--------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/src/mlpack/methods/ann/layer/hard_tanh_impl.hpp b/src/mlpack/methods/ann/layer/hard_tanh_impl.hpp index 787ada121e..9b631ba6e1 100644 --- a/src/mlpack/methods/ann/layer/hard_tanh_impl.hpp +++ b/src/mlpack/methods/ann/layer/hard_tanh_impl.hpp @@ -22,27 +22,27 @@ template HardTanHType::HardTanHType( const double maxValue, const double minValue) : - Layer(), - maxValue(maxValue), - minValue(minValue) + Layer(), + maxValue(maxValue), + minValue(minValue) { // Nothing to do here. } template HardTanHType::HardTanHType(const HardTanHType& layer) : - Layer(layer), - maxValue(layer.maxValue), - minValue(layer.minValue) + Layer(layer), + maxValue(layer.maxValue), + minValue(layer.minValue) { // Nothing to do here. } template HardTanHType::HardTanHType(HardTanHType&& layer) : - Layer(std::move(layer)), - maxValue(std::move(layer.maxValue)), - minValue(std::move(layer.minValue)) + Layer(std::move(layer)), + maxValue(std::move(layer.maxValue)), + minValue(std::move(layer.minValue)) { // Nothing to do here. } @@ -50,7 +50,7 @@ HardTanHType::HardTanHType(HardTanHType&& layer) : template HardTanHType& HardTanHType::operator=(const HardTanHType& layer) { - if(&layer != this) + if (&layer != this) { Layer::operator=(layer); maxValue = layer.maxValue; @@ -63,7 +63,7 @@ HardTanHType& HardTanHType::operator=(const HardTanHType& laye template HardTanHType& HardTanHType::operator=(HardTanHType&& layer) { - if(&layer != this) + if (&layer != this) { Layer::operator=(std::move(layer)); maxValue = std::move(layer.maxValue); @@ -89,6 +89,7 @@ void HardTanHType::Backward( const MatType& input, const MatType& gy, MatType& g) { g = gy; + #pragma omp parallel for for (size_t i = 0; i < input.n_elem; ++i) { From 21e9d32d06e58e8efebb906bf23a42b0be7655bf Mon Sep 17 00:00:00 2001 From: Vaibhav Pathak Date: Sun, 2 Apr 2023 20:36:14 +0530 Subject: [PATCH 09/11] Fixed Code style issues --- src/mlpack/tests/ann/layer/hard_tanh.cpp | 57 +++++++++++++----------- 1 file changed, 30 insertions(+), 27 deletions(-) diff --git a/src/mlpack/tests/ann/layer/hard_tanh.cpp b/src/mlpack/tests/ann/layer/hard_tanh.cpp index f367b73464..2276981cac 100644 --- a/src/mlpack/tests/ann/layer/hard_tanh.cpp +++ b/src/mlpack/tests/ann/layer/hard_tanh.cpp @@ -22,34 +22,37 @@ using namespace mlpack; TEST_CASE("SimpleHardTanHTest", "[ANNLayerTest]") { - arma::mat output, gy, g; - arma::mat input = {{-1.3743, -0.5565, 0.2742, -0.0151, -1.4871}, - {1.5797, -4.2711, -2.2505, -1.7105, -1.2544}, - {0.4023, 0.5676, 2.3100, 1.6658, -0.1907}, - {0.1897, 0.9097, 0.1418, -1.5349, 0.1225}, - {-0.1101, -3.3656, -5.4033, -2.2240, -3.3235}}; - arma::mat actualOutput = {{-1.0000, -0.5565, 0.2742, -0.0151, -1.0000}, - {1.0000, -1.0000, -1.0000, -1.0000, -1.0000}, - {0.4023, 0.5676, 1.0000, 1.0000, -0.1907}, - {0.1897, 0.9097, 0.1418, -1.0000, 0.1225}, - {-0.1101, -1.0000, -1.0000, -1.0000, -1.0000}}; + arma::mat output, gy, g; + arma::mat input = {{-1.3743, -0.5565, 0.2742, -0.0151, -1.4871}, + {1.5797, -4.2711, -2.2505, -1.7105, -1.2544}, + {0.4023, 0.5676, 2.3100, 1.6658, -0.1907}, + {0.1897, 0.9097, 0.1418, -1.5349, 0.1225}, + {-0.1101, -3.3656, -5.4033, -2.2240, -3.3235}}; + arma::mat actualOutput = {{-1.0000, -0.5565, 0.2742, -0.0151, -1.0000}, + {1.0000, -1.0000, -1.0000, -1.0000, -1.0000}, + {0.4023, 0.5676, 1.0000, 1.0000, -0.1907}, + {0.1897, 0.9097, 0.1418, -1.0000, 0.1225}, + {-0.1101, -1.0000, -1.0000, -1.0000, -1.0000}}; - HardTanH module; + HardTanH module; - output.set_size(5,5); - // Test the Forward function - module.Forward(input, output); - REQUIRE(arma::accu(output - actualOutput) == Approx(0).epsilon(1e-4)); + output.set_size(5,5); + // Test the Forward function + module.Forward(input, output); + REQUIRE(arma::accu(output - actualOutput) == Approx(0).epsilon(1e-4)); + + arma::mat delta = {{0 , 1.0, 1.0, 1.0, 0.0}, + {0 , 0 , 0 , 0.0, 0.0}, + {1.0, 1.0, 0 , 0.0, 1.0}, + {1.0, 1.0, 1.0, 0.0, 1.0}, + {1.0, 0 , 0.0, 0.0, 0.0}}; - arma::mat delta = {{0 , 1.0, 1.0, 1.0, 0.0}, - {0 , 0 , 0 , 0.0, 0.0}, - {1.0, 1.0, 0 , 0.0, 1.0}, - {1.0, 1.0, 1.0, 0.0, 1.0}, - {1.0, 0 , 0.0, 0.0, 0.0}}; - gy.set_size(5,5); - gy.fill(1); - g.set_size(5,5); - //Test the Backward function - module.Backward(output, gy, g); - REQUIRE(arma::accu(g - delta) == Approx(0).epsilon(1e-4)); + gy.set_size(5,5); + gy.fill(1); + g.set_size(5,5); + + //Test the Backward function + module.Backward(output, gy, g); + REQUIRE(arma::accu(g - delta) == Approx(0).epsilon(1e-4)); } + From 9f4a14c215f5369a8c8f1d0b1010c8998003204d Mon Sep 17 00:00:00 2001 From: Vaibhav Pathak Date: Sun, 2 Apr 2023 20:37:54 +0530 Subject: [PATCH 10/11] Fix Code Style issues --- src/mlpack/methods/ann/layer/serialization.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mlpack/methods/ann/layer/serialization.hpp b/src/mlpack/methods/ann/layer/serialization.hpp index db3e252af4..c8daf52a71 100644 --- a/src/mlpack/methods/ann/layer/serialization.hpp +++ b/src/mlpack/methods/ann/layer/serialization.hpp @@ -69,7 +69,7 @@ CEREAL_REGISTER_TYPE(mlpack::RBFType<__VA_ARGS__>); \ CEREAL_REGISTER_TYPE(mlpack::SoftmaxType<__VA_ARGS__>); \ CEREAL_REGISTER_TYPE(mlpack::SoftminType<__VA_ARGS__>); \ - CEREAL_REGISTER_TYPE(mlpack::HardTanHType<__VA_ARGS__>); \ + CEREAL_REGISTER_TYPE(mlpack::HardTanHType<__VA_ARGS__>); \ CEREAL_REGISTER_MLPACK_LAYERS(arma::mat); From 2d3f021af4b905950eec830097dd0e303cd24b5a Mon Sep 17 00:00:00 2001 From: Vaibhav Pathak Date: Mon, 3 Apr 2023 21:48:32 +0530 Subject: [PATCH 11/11] Updated HISTORY.md --- HISTORY.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/HISTORY.md b/HISTORY.md index 50f3f92efd..a851154847 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,6 +1,8 @@ ### mlpack ?.?.? ###### ????-??-?? + * Adapt HardTanH layer (#3454). + * Adapt Softmin layer for new neural network API (#3437). * Adapt PReLU layer for new neural network API (#3420).