From 1a6edbb64abd18d9feb25253ae4400a6e19d749f Mon Sep 17 00:00:00 2001 From: dakshitagrawal97 Date: Thu, 1 Mar 2018 23:43:05 +0530 Subject: [PATCH 01/11] implement SELU activation function --- .../activation_functions/selu_function.hpp | 167 ++++++++++++++++++ 1 file changed, 167 insertions(+) create mode 100644 src/mlpack/methods/ann/activation_functions/selu_function.hpp diff --git a/src/mlpack/methods/ann/activation_functions/selu_function.hpp b/src/mlpack/methods/ann/activation_functions/selu_function.hpp new file mode 100644 index 0000000000..41f9c5811c --- /dev/null +++ b/src/mlpack/methods/ann/activation_functions/selu_function.hpp @@ -0,0 +1,167 @@ +/** + * @file selu_function.hpp + * @author Dakshit Agrawal + * + * Definition and implementation of the SELU function as introduced by + * Klambauer et. al. in Self Neural Networks. The SELU activation + * function keeps the mean and variance of the input invariant. + * + * For more information, see the following paper. + * + * @code + * @article{Klambauer2017, + * author = {Gunter Klambauer and Thomas Unterthiner and + * Andreas Mayr}, + * title = {Self-Normalizing Neural Networks}, + * journal = {Advances in Neural Information Processing Systems}, + * year = {2017} + * } + * } + * @endcode + * + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ + +#ifndef MLPACK_METHODS_ANN_ACTIVATION_FUNCTIONS_SELU_FUNCTION_HPP +#define MLPACK_METHODS_ANN_ACTIVATION_FUNCTIONS_SELU_FUNCTION_HPP + +#include +#include + +namespace mlpack { +namespace ann /** Artificial Neural Network. */ { + +/** + * The SELU activation function, defined by + * + * @f{eqnarray*}{ + * f(x) &=& \left\{ + * \begin{array}{lr} + * lambda * x & : x > 0 \\ + * lambda * alpha(e^x - 1) & : x \le 0 + * \end{array} + * \right. \\ + * f'(x) &=& \left\{ + * \begin{array}{lr} + * lambda & : x > 0 \\ + * lambda * (y + alpha) & : x \le 0 + * \end{array} + * \right. + * @f} + * + * + * + * NOTE: + * Make sure to use this activation function with normalized inputs and + * weights initialized with Lecun Normal Initialization. + */ + +class SELUFunction +{ + public: + + /** + * Computes the SELU activation function. + * + * @param x Input data. + * @return f(x). + */ + static double Fn(const double x) + { + if (x < DBL_MAX) { + return (x > 0) ? lambda * x : lambda * alpha * (std::exp(x) - 1); + } + return 1.0; + } + + /** + * Computes the SELU activation function using a dense matrix as input. + * + * @param x Input data. + * @param y The resulting output activation. + */ + template + static void Fn(const arma::Mat& x, arma::Mat& y) + { + y = x; + + for (size_t i = 0; i < x.n_elem; i++) + { + y(i) = Fn(x(i)); + } + } + + /** + * Computes the SELU activation function using a 3rd-order + * tensor as input. + * + * @param x Input data. + * @param y The resulting output activation. + */ + template + static void Fn(const arma::Cube& x, arma::Cube& y) + { + y = x; + for (size_t s = 0; s < x.n_slices; s++) + { + Fn(x.slice(s), y.slice(s)); + } + } + + /** + * Computes the first derivative of the SELU activation function. + * + * @param x Input data. + * @return f'(x) + */ + static double Deriv(const double y) + { + return (y > 0) ? lambda : lambda * (y + alpha); + } + + /** + * Computes the first derivatives of the SELU activation function + * using a dense matrix as input. + * + * @param y Input activations. + * @param x The resulting derivatives. + */ + template + static void Deriv(const InputType& y, OutputType& x) + { + x = y; + + for (size_t i = 0; i < y.n_elem; i++) + { + x(i) = Deriv(y(i)); + } + } + + //! Get the non zero alpha. + static double const& Alpha() { return alpha; } + + + //! Get the non zero lambda. + static double const& Lambda() { return lambda; } + + + private: + + // The following default constant values are for input whose mean is 0 and + // variance is 1, i.e. input is normalized. + + constexpr static double alpha = 1.6732632423543774; + + constexpr static double lambda = 1.0507009873554802; + +}; // class SELUFunction + +} // namespace ann +} // namespace mlpack + +#endif + From 05fe9e4c841136e6b132bb44e264037dd6dfd40d Mon Sep 17 00:00:00 2001 From: dakshitagrawal97 Date: Thu, 1 Mar 2018 23:43:52 +0530 Subject: [PATCH 02/11] update CMAKE to include SELU activation function --- src/mlpack/methods/ann/activation_functions/CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/src/mlpack/methods/ann/activation_functions/CMakeLists.txt b/src/mlpack/methods/ann/activation_functions/CMakeLists.txt index cad7606542..7262ea9977 100644 --- a/src/mlpack/methods/ann/activation_functions/CMakeLists.txt +++ b/src/mlpack/methods/ann/activation_functions/CMakeLists.txt @@ -8,6 +8,7 @@ set(SOURCES rectifier_function.hpp softplus_function.hpp swish_function.hpp + selu_function.hpp ) # Add directory name to sources. From edeb238b0eecfebf6b86d66474a14f15036f8f78 Mon Sep 17 00:00:00 2001 From: dakshitagrawal97 Date: Thu, 1 Mar 2018 23:44:34 +0530 Subject: [PATCH 03/11] add tests for SELU activation function --- .../tests/activation_functions_test.cpp | 52 +++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/src/mlpack/tests/activation_functions_test.cpp b/src/mlpack/tests/activation_functions_test.cpp index 1d9e3f4557..93c0eadb2a 100644 --- a/src/mlpack/tests/activation_functions_test.cpp +++ b/src/mlpack/tests/activation_functions_test.cpp @@ -20,6 +20,7 @@ #include #include #include +#include #include #include "test_tools.hpp" @@ -325,6 +326,57 @@ void CheckPReLUGradientCorrect(const arma::colvec input, BOOST_REQUIRE_CLOSE(gradient(0), target(0), 1e-3); } +/* + * Simple SELU activation test to check whether the mean and variance remain + * invariant after passing through the function. + * + * NOTE: the input must be normalized. + */ + +BOOST_AUTO_TEST_CASE(SELUFunctionTest) +{ + arma::mat input = arma::randn(1000, 1); + + arma::mat output; + + SELUFunction::Fn(input, output); + + BOOST_REQUIRE_LE( + arma::as_scalar(arma::abs(arma::mean(input) - arma::mean(output))), 0.01); + + BOOST_REQUIRE_LE( + arma::as_scalar(arma::abs(arma::var(input) - arma::var(output))), 0.01); +} + +/* + * Simple SELU derivative test to check whether the derivatives + * produced by the activation function are correct. + * + */ + +BOOST_AUTO_TEST_CASE(SELUFunctionDerivativeTest) +{ + arma::mat input = arma::ones(1000, 1); + + arma::mat output; + + SELUFunction::Deriv(input, output); + + BOOST_REQUIRE_LE( + arma::as_scalar(arma::abs(arma::mean(output) - SELUFunction::Lambda())), + 10e-6); + + input.fill(-1); + + SELUFunction::Deriv(input, output); + + BOOST_REQUIRE_LE( + arma::as_scalar(arma::abs(arma::mean(output) - + SELUFunction::Lambda()*(SELUFunction::Alpha()-1))), + 10e-6); + +} + /** * Basic test of the tanh function. */ From fa18dc369d4f5df86122b254b8e3f0e560279c4a Mon Sep 17 00:00:00 2001 From: dakshitagrawal97 Date: Thu, 1 Mar 2018 23:57:54 +0530 Subject: [PATCH 04/11] fix style errors --- src/mlpack/methods/ann/activation_functions/selu_function.hpp | 3 --- src/mlpack/tests/activation_functions_test.cpp | 1 - 2 files changed, 4 deletions(-) diff --git a/src/mlpack/methods/ann/activation_functions/selu_function.hpp b/src/mlpack/methods/ann/activation_functions/selu_function.hpp index 41f9c5811c..9c009eecee 100644 --- a/src/mlpack/methods/ann/activation_functions/selu_function.hpp +++ b/src/mlpack/methods/ann/activation_functions/selu_function.hpp @@ -63,7 +63,6 @@ namespace ann /** Artificial Neural Network. */ { class SELUFunction { public: - /** * Computes the SELU activation function. * @@ -150,14 +149,12 @@ class SELUFunction private: - // The following default constant values are for input whose mean is 0 and // variance is 1, i.e. input is normalized. constexpr static double alpha = 1.6732632423543774; constexpr static double lambda = 1.0507009873554802; - }; // class SELUFunction } // namespace ann diff --git a/src/mlpack/tests/activation_functions_test.cpp b/src/mlpack/tests/activation_functions_test.cpp index 93c0eadb2a..4433618038 100644 --- a/src/mlpack/tests/activation_functions_test.cpp +++ b/src/mlpack/tests/activation_functions_test.cpp @@ -374,7 +374,6 @@ BOOST_AUTO_TEST_CASE(SELUFunctionDerivativeTest) arma::as_scalar(arma::abs(arma::mean(output) - SELUFunction::Lambda()*(SELUFunction::Alpha()-1))), 10e-6); - } /** From a8450715e4d3075de4ccc69c9b67b2b27080e065 Mon Sep 17 00:00:00 2001 From: dakshitagrawal97 Date: Sun, 4 Mar 2018 03:23:47 +0530 Subject: [PATCH 05/11] implement SELU in ELU layer, introduce lambda parameter --- src/mlpack/methods/ann/layer/elu.hpp | 116 ++++++++++++++++-- src/mlpack/methods/ann/layer/elu_impl.hpp | 20 ++- .../tests/activation_functions_test.cpp | 70 ++++++++--- 3 files changed, 179 insertions(+), 27 deletions(-) diff --git a/src/mlpack/methods/ann/layer/elu.hpp b/src/mlpack/methods/ann/layer/elu.hpp index 1211ee7154..4afad43a61 100644 --- a/src/mlpack/methods/ann/layer/elu.hpp +++ b/src/mlpack/methods/ann/layer/elu.hpp @@ -1,6 +1,7 @@ /** * @file elu.hpp * @author Vivek Pal + * @author Dakshit Agrawal * * Definition of the ELU activation function as descibed by Djork-Arne Clevert, * Thomas Unterthiner and Sepp Hochreiter. @@ -18,6 +19,29 @@ * } * @endcode * + * Definition of the SELU function as introduced by + * Klambauer et. al. in Self Neural Networks. The SELU activation + * function keeps the mean and variance of the input invariant. + * + * For more information, see the following paper. + * + * @code + * @article{Klambauer2017, + * author = {Gunter Klambauer and Thomas Unterthiner and + * Andreas Mayr}, + * title = {Self-Normalizing Neural Networks}, + * journal = {Advances in Neural Information Processing Systems}, + * year = {2017} + * } + * } + * @endcode + * + * In short, SELU = lambda * ELU, with 'alpha' and 'lambda' fixed for + * normalized inputs. + * + * Hence both ELU and SELU are implemented in the same file, with + * lambda = 1 for ELU function. + * * mlpack is free software; you may redistribute it and/or modify it under the * terms of the 3-clause BSD license. You should have received a copy of the * 3-clause BSD license along with mlpack. If not, see @@ -49,6 +73,56 @@ namespace ann /** Artificial Neural Network. */ { * \right. * @f} * + * For more information, read the following paper: + * + * @code + * @article{Clevert2015, + * author = {Djork{-}Arn{\'{e}} Clevert and Thomas Unterthiner and + * Sepp Hochreiter}, + * title = {Fast and Accurate Deep Network Learning by Exponential Linear + * Units (ELUs)}, + * journal = {CoRR}, + * year = {2015} + * } + * @endcode + * + * + * The SELU activation function is defined by + * + * @f{eqnarray*}{ + * f(x) &=& \left\{ + * \begin{array}{lr} + * lambda * x & : x > 0 \\ + * lambda * alpha(e^x - 1) & : x \le 0 + * \end{array} + * \right. \\ + * f'(x) &=& \left\{ + * \begin{array}{lr} + * lambda & : x > 0 \\ + * lambda * (y + alpha) & : x \le 0 + * \end{array} + * \right. + * @f} + * + * + * For more information, see the following paper. + * + * @code + * @article{Klambauer2017, + * author = {Gunter Klambauer and Thomas Unterthiner and + * Andreas Mayr}, + * title = {Self-Normalizing Neural Networks}, + * journal = {Advances in Neural Information Processing Systems}, + * year = {2017} + * } + * } + * @endcode + * + * NOTE: + * Make sure to use SELU activation function with normalized inputs and + * weights initialized with Lecun Normal Initialization. + * + * * @tparam InputDataType Type of the input data (arma::colvec, arma::mat, * arma::sp_mat or arma::cube). * @tparam OutputDataType Type of the output data (arma::colvec, arma::mat, @@ -62,13 +136,25 @@ class ELU { public: /** - * Create the ELU object using the specified parameters. The non zero + * Create the ELU object. + * + * NOTE: Use this constructor for SELU activation function. + * + */ + + ELU(); + + /** + * Create the ELU object using the specified parameter. The non zero * gradient for negative inputs can be adjusted by specifying the ELU * hyperparameter alpha (alpha > 0). * - * @param alpha Scale parameter for the negative factor (Default alpha = 1.0). + * NOTE: Use this constructor for ELU activation function. + * + * @param alpha Scale parameter for the negative factor. */ - ELU(const double alpha = 1.0); + + ELU(const double alpha); /** * Ordinary feed forward pass of a neural network, evaluating the function @@ -120,20 +206,21 @@ class ELU private: /** - * Computes the ELU function + * Computes the value of activation function. * * @param x Input data. * @return f(x). */ double Fn(const double x) { - if (x < DBL_MAX) - return (x > 0) ? x : alpha * (std::exp(x) - 1); + if (x < DBL_MAX) { + return (x > 0) ? lambda * x : lambda * alpha * (std::exp(x) - 1); + } return 1.0; } /** - * Computes the ELU function using a dense matrix as input. + * Computes the value of activation function using a dense matrix as input. * * @param x Input data. * @param y The resulting output activation. @@ -141,7 +228,7 @@ class ELU template void Fn(const arma::Mat& x, arma::Mat& y) { - y = x; + y.set_size(size(x)); for (size_t i = 0; i < x.n_elem; i++) { @@ -150,18 +237,18 @@ class ELU } /** - * Computes the first derivative of the ELU function. + * Computes the first derivative of the activation function. * * @param x Input data. * @return f'(x) */ double Deriv(const double y) { - return (y > 0) ? 1 : (y + alpha); + return (y > 0) ? lambda : lambda * (y + alpha); } /** - * Computes the first derivative of the ELU function. + * Computes the first derivative of the activation function. * * @param y Input activations. * @param x The resulting derivatives. @@ -188,7 +275,14 @@ class ELU OutputDataType outputParameter; //! ELU Hyperparameter (0 < alpha) + //! SELU parameter fixed to 1.6732632423543774 for normalized inputs. double alpha; + + //! Lambda Parameter used for multiplication of ELU function. + //! For ELU activation function, lambda = 1. + //! For SELU activation function, lambda = 1.0507009873554802 for normalized + //! inputs. + double lambda; }; // class ELU } // namespace ann diff --git a/src/mlpack/methods/ann/layer/elu_impl.hpp b/src/mlpack/methods/ann/layer/elu_impl.hpp index 3cc8246df1..1319a94de8 100644 --- a/src/mlpack/methods/ann/layer/elu_impl.hpp +++ b/src/mlpack/methods/ann/layer/elu_impl.hpp @@ -1,10 +1,15 @@ /** * @file elu_impl.hpp * @author Vivek Pal + * @author Dakshit Agrawal * * Implementation of the ELU activation function as descibed by Djork-Arne * Clevert, Thomas Unterthiner and Sepp Hochreiter. * + * Implementation of the SELU function as introduced by Klambauer et. al. in + * Self Neural Networks. The SELU activation function keeps the mean and + * variance of the input invariant. + * * mlpack is free software; you may redistribute it and/or modify it under the * terms of the 3-clause BSD license. You should have received a copy of the * 3-clause BSD license along with mlpack. If not, see @@ -19,9 +24,21 @@ namespace mlpack { namespace ann /** Artificial Neural Network. */ { +// This constructor is called for SELU activation function. The values of +// alpha and lambda are constant for normalized inputs. +template +ELU::ELU() : + alpha(1.6732632423543774), + lambda(1.0507009873554802) +{ + // Nothing to do here. +} + +// This constructor is called for ELU activation function. The value of lambda +// is fixed and equal to 1. 'alpha' is a hyperparameter. template ELU::ELU( - const double alpha) : alpha(alpha) + const double alpha) : alpha(alpha), lambda(1) { // Nothing to do here. } @@ -51,6 +68,7 @@ void ELU::serialize( const unsigned int /* version */) { ar & BOOST_SERIALIZATION_NVP(alpha); + ar & BOOST_SERIALIZATION_NVP(lambda); } } // namespace ann diff --git a/src/mlpack/tests/activation_functions_test.cpp b/src/mlpack/tests/activation_functions_test.cpp index 4433618038..5079c065d8 100644 --- a/src/mlpack/tests/activation_functions_test.cpp +++ b/src/mlpack/tests/activation_functions_test.cpp @@ -218,7 +218,8 @@ void CheckLeakyReLUDerivativeCorrect(const arma::colvec input, void CheckELUActivationCorrect(const arma::colvec input, const arma::colvec target) { - ELU<> lrf; + // Initialize ELU object with alpha = 1.0. + ELU<> lrf(1.0); // Test the activation function using the entire vector as input. arma::colvec activations; @@ -239,7 +240,8 @@ void CheckELUActivationCorrect(const arma::colvec input, void CheckELUDerivativeCorrect(const arma::colvec input, const arma::colvec target) { - ELU<> lrf; + // Initialize ELU object with alpha = 1.0. + ELU<> lrf(1.0); // Test the calculation of the derivatives using the entire vector as input. arma::colvec derivatives; @@ -328,24 +330,55 @@ void CheckPReLUGradientCorrect(const arma::colvec input, /* * Simple SELU activation test to check whether the mean and variance remain - * invariant after passing through the function. + * invariant after passing normalized inputs through the function. * - * NOTE: the input must be normalized. */ -BOOST_AUTO_TEST_CASE(SELUFunctionTest) +BOOST_AUTO_TEST_CASE(SELUFunctionNormalizedTest) { arma::mat input = arma::randn(1000, 1); arma::mat output; - SELUFunction::Fn(input, output); + // Using alias to specify that SELU activation function has been called. + using SELU = ELU; + + SELU selu; + + selu.Forward(std::move(input), output); BOOST_REQUIRE_LE( - arma::as_scalar(arma::abs(arma::mean(input) - arma::mean(output))), 0.01); + arma::as_scalar(arma::abs(arma::mean(input) - arma::mean(output))), 0.1); BOOST_REQUIRE_LE( - arma::as_scalar(arma::abs(arma::var(input) - arma::var(output))), 0.01); + arma::as_scalar(arma::abs(arma::var(input) - arma::var(output))), 0.1); +} + +/* + * Simple SELU activation test to check whether the mean and variance + * vary significantly after passing unnormalized inputs through the function. + * + */ + +BOOST_AUTO_TEST_CASE(SELUFunctionUnnormalizedTest) +{ + const arma::colvec input("5.96402758 0.9966824 0.99975321 1 \ + 7.76159416 -0.76159416 0.96402758 8"); + + arma::mat output; + + // Using alias to specify that SELU activation function has been called. + using SELU = ELU; + + SELU selu; + + selu.Forward(std::move(input), output); + + BOOST_REQUIRE_GE( + arma::as_scalar(arma::abs(arma::mean(input) - arma::mean(output))), 0.1); + + BOOST_REQUIRE_GE( + arma::as_scalar(arma::abs(arma::var(input) - arma::var(output))), 0.1); } /* @@ -358,22 +391,29 @@ BOOST_AUTO_TEST_CASE(SELUFunctionDerivativeTest) { arma::mat input = arma::ones(1000, 1); - arma::mat output; + arma::mat error = arma::ones(input.n_elem,1); - SELUFunction::Deriv(input, output); + arma::mat derivatives; + + // Using alias to specify that SELU activation function has been called. + using SELU = ELU; + + SELU selu; + + selu.Backward(std::move(input), std::move(error), std::move(derivatives)); BOOST_REQUIRE_LE( - arma::as_scalar(arma::abs(arma::mean(output) - SELUFunction::Lambda())), - 10e-6); + arma::as_scalar(arma::abs(arma::mean(derivatives) - SELUFunction::Lambda())), + 10e-5); input.fill(-1); - SELUFunction::Deriv(input, output); + selu.Backward(std::move(input), std::move(error), std::move(derivatives)); BOOST_REQUIRE_LE( - arma::as_scalar(arma::abs(arma::mean(output) - + arma::as_scalar(arma::abs(arma::mean(derivatives) - SELUFunction::Lambda()*(SELUFunction::Alpha()-1))), - 10e-6); + 10e-5); } /** From 7b8b915c177d93c4f9db2ff1fd30ee27c8822320 Mon Sep 17 00:00:00 2001 From: dakshitagrawal97 Date: Sun, 4 Mar 2018 03:26:44 +0530 Subject: [PATCH 06/11] remove selu_function.hpp --- .../ann/activation_functions/CMakeLists.txt | 3 +- .../activation_functions/selu_function.hpp | 164 ------------------ .../tests/activation_functions_test.cpp | 1 - 3 files changed, 1 insertion(+), 167 deletions(-) delete mode 100644 src/mlpack/methods/ann/activation_functions/selu_function.hpp diff --git a/src/mlpack/methods/ann/activation_functions/CMakeLists.txt b/src/mlpack/methods/ann/activation_functions/CMakeLists.txt index 7262ea9977..a6e514a15e 100644 --- a/src/mlpack/methods/ann/activation_functions/CMakeLists.txt +++ b/src/mlpack/methods/ann/activation_functions/CMakeLists.txt @@ -8,8 +8,7 @@ set(SOURCES rectifier_function.hpp softplus_function.hpp swish_function.hpp - selu_function.hpp -) + ) # Add directory name to sources. set(DIR_SRCS) diff --git a/src/mlpack/methods/ann/activation_functions/selu_function.hpp b/src/mlpack/methods/ann/activation_functions/selu_function.hpp deleted file mode 100644 index 9c009eecee..0000000000 --- a/src/mlpack/methods/ann/activation_functions/selu_function.hpp +++ /dev/null @@ -1,164 +0,0 @@ -/** - * @file selu_function.hpp - * @author Dakshit Agrawal - * - * Definition and implementation of the SELU function as introduced by - * Klambauer et. al. in Self Neural Networks. The SELU activation - * function keeps the mean and variance of the input invariant. - * - * For more information, see the following paper. - * - * @code - * @article{Klambauer2017, - * author = {Gunter Klambauer and Thomas Unterthiner and - * Andreas Mayr}, - * title = {Self-Normalizing Neural Networks}, - * journal = {Advances in Neural Information Processing Systems}, - * year = {2017} - * } - * } - * @endcode - * - * - * mlpack is free software; you may redistribute it and/or modify it under the - * terms of the 3-clause BSD license. You should have received a copy of the - * 3-clause BSD license along with mlpack. If not, see - * http://www.opensource.org/licenses/BSD-3-Clause for more information. - */ - -#ifndef MLPACK_METHODS_ANN_ACTIVATION_FUNCTIONS_SELU_FUNCTION_HPP -#define MLPACK_METHODS_ANN_ACTIVATION_FUNCTIONS_SELU_FUNCTION_HPP - -#include -#include - -namespace mlpack { -namespace ann /** Artificial Neural Network. */ { - -/** - * The SELU activation function, defined by - * - * @f{eqnarray*}{ - * f(x) &=& \left\{ - * \begin{array}{lr} - * lambda * x & : x > 0 \\ - * lambda * alpha(e^x - 1) & : x \le 0 - * \end{array} - * \right. \\ - * f'(x) &=& \left\{ - * \begin{array}{lr} - * lambda & : x > 0 \\ - * lambda * (y + alpha) & : x \le 0 - * \end{array} - * \right. - * @f} - * - * - * - * NOTE: - * Make sure to use this activation function with normalized inputs and - * weights initialized with Lecun Normal Initialization. - */ - -class SELUFunction -{ - public: - /** - * Computes the SELU activation function. - * - * @param x Input data. - * @return f(x). - */ - static double Fn(const double x) - { - if (x < DBL_MAX) { - return (x > 0) ? lambda * x : lambda * alpha * (std::exp(x) - 1); - } - return 1.0; - } - - /** - * Computes the SELU activation function using a dense matrix as input. - * - * @param x Input data. - * @param y The resulting output activation. - */ - template - static void Fn(const arma::Mat& x, arma::Mat& y) - { - y = x; - - for (size_t i = 0; i < x.n_elem; i++) - { - y(i) = Fn(x(i)); - } - } - - /** - * Computes the SELU activation function using a 3rd-order - * tensor as input. - * - * @param x Input data. - * @param y The resulting output activation. - */ - template - static void Fn(const arma::Cube& x, arma::Cube& y) - { - y = x; - for (size_t s = 0; s < x.n_slices; s++) - { - Fn(x.slice(s), y.slice(s)); - } - } - - /** - * Computes the first derivative of the SELU activation function. - * - * @param x Input data. - * @return f'(x) - */ - static double Deriv(const double y) - { - return (y > 0) ? lambda : lambda * (y + alpha); - } - - /** - * Computes the first derivatives of the SELU activation function - * using a dense matrix as input. - * - * @param y Input activations. - * @param x The resulting derivatives. - */ - template - static void Deriv(const InputType& y, OutputType& x) - { - x = y; - - for (size_t i = 0; i < y.n_elem; i++) - { - x(i) = Deriv(y(i)); - } - } - - //! Get the non zero alpha. - static double const& Alpha() { return alpha; } - - - //! Get the non zero lambda. - static double const& Lambda() { return lambda; } - - - private: - // The following default constant values are for input whose mean is 0 and - // variance is 1, i.e. input is normalized. - - constexpr static double alpha = 1.6732632423543774; - - constexpr static double lambda = 1.0507009873554802; -}; // class SELUFunction - -} // namespace ann -} // namespace mlpack - -#endif - diff --git a/src/mlpack/tests/activation_functions_test.cpp b/src/mlpack/tests/activation_functions_test.cpp index 5079c065d8..b12cf1cbfe 100644 --- a/src/mlpack/tests/activation_functions_test.cpp +++ b/src/mlpack/tests/activation_functions_test.cpp @@ -20,7 +20,6 @@ #include #include #include -#include #include #include "test_tools.hpp" From 540df42a98f49ded85f96d73da647c3efe185e31 Mon Sep 17 00:00:00 2001 From: dakshitagrawal97 Date: Sun, 4 Mar 2018 03:40:44 +0530 Subject: [PATCH 07/11] fix activation function test errors --- src/mlpack/methods/ann/layer/elu.hpp | 3 +++ src/mlpack/tests/activation_functions_test.cpp | 18 ++++++++---------- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/src/mlpack/methods/ann/layer/elu.hpp b/src/mlpack/methods/ann/layer/elu.hpp index 4afad43a61..684e9dbc7e 100644 --- a/src/mlpack/methods/ann/layer/elu.hpp +++ b/src/mlpack/methods/ann/layer/elu.hpp @@ -198,6 +198,9 @@ class ELU //! Modify the non zero gradient. double& Alpha() { return alpha; } + //! Get the lambda parameter. + double const& Lambda() const { return lambda; } + /** * Serialize the layer. */ diff --git a/src/mlpack/tests/activation_functions_test.cpp b/src/mlpack/tests/activation_functions_test.cpp index b12cf1cbfe..a013501133 100644 --- a/src/mlpack/tests/activation_functions_test.cpp +++ b/src/mlpack/tests/activation_functions_test.cpp @@ -347,10 +347,10 @@ BOOST_AUTO_TEST_CASE(SELUFunctionNormalizedTest) selu.Forward(std::move(input), output); BOOST_REQUIRE_LE( - arma::as_scalar(arma::abs(arma::mean(input) - arma::mean(output))), 0.1); + arma::as_scalar(arma::abs(arma::mean(input) - arma::mean(output))), 0.1); BOOST_REQUIRE_LE( - arma::as_scalar(arma::abs(arma::var(input) - arma::var(output))), 0.1); + arma::as_scalar(arma::abs(arma::var(input) - arma::var(output))), 0.1); } /* @@ -374,10 +374,10 @@ BOOST_AUTO_TEST_CASE(SELUFunctionUnnormalizedTest) selu.Forward(std::move(input), output); BOOST_REQUIRE_GE( - arma::as_scalar(arma::abs(arma::mean(input) - arma::mean(output))), 0.1); + arma::as_scalar(arma::abs(arma::mean(input) - arma::mean(output))), 0.1); BOOST_REQUIRE_GE( - arma::as_scalar(arma::abs(arma::var(input) - arma::var(output))), 0.1); + arma::as_scalar(arma::abs(arma::var(input) - arma::var(output))), 0.1); } /* @@ -402,17 +402,15 @@ BOOST_AUTO_TEST_CASE(SELUFunctionDerivativeTest) selu.Backward(std::move(input), std::move(error), std::move(derivatives)); BOOST_REQUIRE_LE( - arma::as_scalar(arma::abs(arma::mean(derivatives) - SELUFunction::Lambda())), - 10e-5); + arma::as_scalar(arma::abs(arma::mean(derivatives) - selu.Lambda())), + 10e-5); input.fill(-1); selu.Backward(std::move(input), std::move(error), std::move(derivatives)); - BOOST_REQUIRE_LE( - arma::as_scalar(arma::abs(arma::mean(derivatives) - - SELUFunction::Lambda()*(SELUFunction::Alpha()-1))), - 10e-5); + BOOST_REQUIRE_LE( arma::as_scalar(arma::abs(arma::mean(derivatives) - + selu.Lambda()*(selu.Alpha()-1))), 10e-5); } /** From 2b0a7554b328e6b04f63e54c9b7a68da3ea99abb Mon Sep 17 00:00:00 2001 From: dakshitagrawal97 Date: Sun, 4 Mar 2018 03:46:03 +0530 Subject: [PATCH 08/11] fix minor style errors --- src/mlpack/tests/activation_functions_test.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/mlpack/tests/activation_functions_test.cpp b/src/mlpack/tests/activation_functions_test.cpp index a013501133..543ba0a7dc 100644 --- a/src/mlpack/tests/activation_functions_test.cpp +++ b/src/mlpack/tests/activation_functions_test.cpp @@ -390,7 +390,7 @@ BOOST_AUTO_TEST_CASE(SELUFunctionDerivativeTest) { arma::mat input = arma::ones(1000, 1); - arma::mat error = arma::ones(input.n_elem,1); + arma::mat error = arma::ones(input.n_elem, 1); arma::mat derivatives; @@ -409,7 +409,7 @@ BOOST_AUTO_TEST_CASE(SELUFunctionDerivativeTest) selu.Backward(std::move(input), std::move(error), std::move(derivatives)); - BOOST_REQUIRE_LE( arma::as_scalar(arma::abs(arma::mean(derivatives) - + BOOST_REQUIRE_LE(arma::as_scalar(arma::abs(arma::mean(derivatives) - selu.Lambda()*(selu.Alpha()-1))), 10e-5); } From 4dbcb54e8c547604010f7faf89100d7aadf9dea1 Mon Sep 17 00:00:00 2001 From: Prabhat Date: Sat, 31 Mar 2018 13:39:54 +0530 Subject: [PATCH 09/11] Style change --- .../ann/activation_functions/CMakeLists.txt | 2 +- src/mlpack/methods/ann/layer/elu.hpp | 15 ++++---- .../tests/activation_functions_test.cpp | 36 +++++++------------ 3 files changed, 19 insertions(+), 34 deletions(-) diff --git a/src/mlpack/methods/ann/activation_functions/CMakeLists.txt b/src/mlpack/methods/ann/activation_functions/CMakeLists.txt index a6e514a15e..cad7606542 100644 --- a/src/mlpack/methods/ann/activation_functions/CMakeLists.txt +++ b/src/mlpack/methods/ann/activation_functions/CMakeLists.txt @@ -8,7 +8,7 @@ set(SOURCES rectifier_function.hpp softplus_function.hpp swish_function.hpp - ) +) # Add directory name to sources. set(DIR_SRCS) diff --git a/src/mlpack/methods/ann/layer/elu.hpp b/src/mlpack/methods/ann/layer/elu.hpp index 684e9dbc7e..6cd0bf5e13 100644 --- a/src/mlpack/methods/ann/layer/elu.hpp +++ b/src/mlpack/methods/ann/layer/elu.hpp @@ -104,7 +104,6 @@ namespace ann /** Artificial Neural Network. */ { * \right. * @f} * - * * For more information, see the following paper. * * @code @@ -118,9 +117,8 @@ namespace ann /** Artificial Neural Network. */ { * } * @endcode * - * NOTE: - * Make sure to use SELU activation function with normalized inputs and - * weights initialized with Lecun Normal Initialization. + * @note Make sure to use SELU activation function with normalized inputs and + * weights initialized with Lecun Normal Initialization. * * * @tparam InputDataType Type of the input data (arma::colvec, arma::mat, @@ -149,13 +147,10 @@ class ELU * gradient for negative inputs can be adjusted by specifying the ELU * hyperparameter alpha (alpha > 0). * - * NOTE: Use this constructor for ELU activation function. - * + * @note Use this constructor for ELU activation function. * @param alpha Scale parameter for the negative factor. */ - ELU(const double alpha); - /** * Ordinary feed forward pass of a neural network, evaluating the function * f(x) by propagating the activity forward through f. @@ -216,7 +211,8 @@ class ELU */ double Fn(const double x) { - if (x < DBL_MAX) { + if (x < DBL_MAX) + { return (x > 0) ? lambda * x : lambda * alpha * (std::exp(x) - 1); } return 1.0; @@ -287,6 +283,7 @@ class ELU //! inputs. double lambda; }; // class ELU +using SELU = ELU; } // namespace ann } // namespace mlpack diff --git a/src/mlpack/tests/activation_functions_test.cpp b/src/mlpack/tests/activation_functions_test.cpp index 543ba0a7dc..bd64477a7a 100644 --- a/src/mlpack/tests/activation_functions_test.cpp +++ b/src/mlpack/tests/activation_functions_test.cpp @@ -332,25 +332,21 @@ void CheckPReLUGradientCorrect(const arma::colvec input, * invariant after passing normalized inputs through the function. * */ - BOOST_AUTO_TEST_CASE(SELUFunctionNormalizedTest) { arma::mat input = arma::randn(1000, 1); arma::mat output; - // Using alias to specify that SELU activation function has been called. - using SELU = ELU; - SELU selu; selu.Forward(std::move(input), output); - BOOST_REQUIRE_LE( - arma::as_scalar(arma::abs(arma::mean(input) - arma::mean(output))), 0.1); + BOOST_REQUIRE_LE(arma::as_scalar(arma::abs(arma::mean(input) - + arma::mean(output))), 0.1); - BOOST_REQUIRE_LE( - arma::as_scalar(arma::abs(arma::var(input) - arma::var(output))), 0.1); + BOOST_REQUIRE_LE(arma::as_scalar(arma::abs(arma::var(input) - + arma::var(output))), 0.1); } /* @@ -362,22 +358,19 @@ BOOST_AUTO_TEST_CASE(SELUFunctionNormalizedTest) BOOST_AUTO_TEST_CASE(SELUFunctionUnnormalizedTest) { const arma::colvec input("5.96402758 0.9966824 0.99975321 1 \ - 7.76159416 -0.76159416 0.96402758 8"); + 7.76159416 -0.76159416 0.96402758 8"); arma::mat output; - // Using alias to specify that SELU activation function has been called. - using SELU = ELU; - SELU selu; selu.Forward(std::move(input), output); - BOOST_REQUIRE_GE( - arma::as_scalar(arma::abs(arma::mean(input) - arma::mean(output))), 0.1); + BOOST_REQUIRE_GE(arma::as_scalar(arma::abs(arma::mean(input) - + arma::mean(output))), 0.1); - BOOST_REQUIRE_GE( - arma::as_scalar(arma::abs(arma::var(input) - arma::var(output))), 0.1); + BOOST_REQUIRE_GE(arma::as_scalar(arma::abs(arma::var(input) - + arma::var(output))), 0.1); } /* @@ -385,7 +378,6 @@ BOOST_AUTO_TEST_CASE(SELUFunctionUnnormalizedTest) * produced by the activation function are correct. * */ - BOOST_AUTO_TEST_CASE(SELUFunctionDerivativeTest) { arma::mat input = arma::ones(1000, 1); @@ -394,23 +386,19 @@ BOOST_AUTO_TEST_CASE(SELUFunctionDerivativeTest) arma::mat derivatives; - // Using alias to specify that SELU activation function has been called. - using SELU = ELU; - SELU selu; selu.Backward(std::move(input), std::move(error), std::move(derivatives)); - BOOST_REQUIRE_LE( - arma::as_scalar(arma::abs(arma::mean(derivatives) - selu.Lambda())), - 10e-5); + BOOST_REQUIRE_LE(arma::as_scalar(arma::abs(arma::mean(derivatives) - + selu.Lambda())), 10e-5); input.fill(-1); selu.Backward(std::move(input), std::move(error), std::move(derivatives)); BOOST_REQUIRE_LE(arma::as_scalar(arma::abs(arma::mean(derivatives) - - selu.Lambda()*(selu.Alpha()-1))), 10e-5); + selu.Lambda()*(selu.Alpha()-1))), 10e-5); } /** From b5424b3dce247a8d2293afd31322d4b12b8ffe8f Mon Sep 17 00:00:00 2001 From: Prabhat Date: Thu, 5 Apr 2018 08:47:27 +0530 Subject: [PATCH 10/11] Fix style issues --- src/mlpack/methods/ann/layer/elu.hpp | 61 +++++-------------- .../tests/activation_functions_test.cpp | 3 - 2 files changed, 16 insertions(+), 48 deletions(-) diff --git a/src/mlpack/methods/ann/layer/elu.hpp b/src/mlpack/methods/ann/layer/elu.hpp index 6cd0bf5e13..1bc91a675e 100644 --- a/src/mlpack/methods/ann/layer/elu.hpp +++ b/src/mlpack/methods/ann/layer/elu.hpp @@ -6,36 +6,10 @@ * Definition of the ELU activation function as descibed by Djork-Arne Clevert, * Thomas Unterthiner and Sepp Hochreiter. * - * For more information, read the following paper: - * - * @code - * @article{Clevert2015, - * author = {Djork{-}Arn{\'{e}} Clevert and Thomas Unterthiner and - * Sepp Hochreiter}, - * title = {Fast and Accurate Deep Network Learning by Exponential Linear - * Units (ELUs)}, - * journal = {CoRR}, - * year = {2015} - * } - * @endcode - * * Definition of the SELU function as introduced by * Klambauer et. al. in Self Neural Networks. The SELU activation * function keeps the mean and variance of the input invariant. * - * For more information, see the following paper. - * - * @code - * @article{Klambauer2017, - * author = {Gunter Klambauer and Thomas Unterthiner and - * Andreas Mayr}, - * title = {Self-Normalizing Neural Networks}, - * journal = {Advances in Neural Information Processing Systems}, - * year = {2017} - * } - * } - * @endcode - * * In short, SELU = lambda * ELU, with 'alpha' and 'lambda' fixed for * normalized inputs. * @@ -56,6 +30,21 @@ namespace mlpack { namespace ann /** Artificial Neural Network. */ { /** + * @note Make sure to use SELU activation function with normalized inputs and + * weights initialized with Lecun Normal Initialization. + * + * @tparam InputDataType Type of the input data (arma::colvec, arma::mat, + * arma::sp_mat or arma::cube). + * @tparam OutputDataType Type of the output data (arma::colvec, arma::mat, + * arma::sp_mat or arma::cube). + */ +template < + typename InputDataType = arma::mat, + typename OutputDataType = arma::mat +> +class ELU +{ + /** * The ELU activation function, defined by * * @f{eqnarray*}{ @@ -104,7 +93,7 @@ namespace ann /** Artificial Neural Network. */ { * \right. * @f} * - * For more information, see the following paper. + * For more information, read the following paper: * * @code * @article{Klambauer2017, @@ -114,24 +103,8 @@ namespace ann /** Artificial Neural Network. */ { * journal = {Advances in Neural Information Processing Systems}, * year = {2017} * } - * } * @endcode - * - * @note Make sure to use SELU activation function with normalized inputs and - * weights initialized with Lecun Normal Initialization. - * - * - * @tparam InputDataType Type of the input data (arma::colvec, arma::mat, - * arma::sp_mat or arma::cube). - * @tparam OutputDataType Type of the output data (arma::colvec, arma::mat, - * arma::sp_mat or arma::cube). */ -template < - typename InputDataType = arma::mat, - typename OutputDataType = arma::mat -> -class ELU -{ public: /** * Create the ELU object. @@ -139,9 +112,7 @@ class ELU * NOTE: Use this constructor for SELU activation function. * */ - ELU(); - /** * Create the ELU object using the specified parameter. The non zero * gradient for negative inputs can be adjusted by specifying the ELU diff --git a/src/mlpack/tests/activation_functions_test.cpp b/src/mlpack/tests/activation_functions_test.cpp index bd64477a7a..43f937aa44 100644 --- a/src/mlpack/tests/activation_functions_test.cpp +++ b/src/mlpack/tests/activation_functions_test.cpp @@ -330,7 +330,6 @@ void CheckPReLUGradientCorrect(const arma::colvec input, /* * Simple SELU activation test to check whether the mean and variance remain * invariant after passing normalized inputs through the function. - * */ BOOST_AUTO_TEST_CASE(SELUFunctionNormalizedTest) { @@ -352,9 +351,7 @@ BOOST_AUTO_TEST_CASE(SELUFunctionNormalizedTest) /* * Simple SELU activation test to check whether the mean and variance * vary significantly after passing unnormalized inputs through the function. - * */ - BOOST_AUTO_TEST_CASE(SELUFunctionUnnormalizedTest) { const arma::colvec input("5.96402758 0.9966824 0.99975321 1 \ From d156b8d85edc834faaae16f0fc5717e0da34a152 Mon Sep 17 00:00:00 2001 From: Prabhat Date: Fri, 6 Apr 2018 22:54:58 +0530 Subject: [PATCH 11/11] Fix style issues --- src/mlpack/methods/ann/layer/elu.hpp | 1 + src/mlpack/tests/activation_functions_test.cpp | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/mlpack/methods/ann/layer/elu.hpp b/src/mlpack/methods/ann/layer/elu.hpp index 1bc91a675e..d586514217 100644 --- a/src/mlpack/methods/ann/layer/elu.hpp +++ b/src/mlpack/methods/ann/layer/elu.hpp @@ -254,6 +254,7 @@ class ELU //! inputs. double lambda; }; // class ELU +// Template alias for SELU using ELU class using SELU = ELU; } // namespace ann diff --git a/src/mlpack/tests/activation_functions_test.cpp b/src/mlpack/tests/activation_functions_test.cpp index 43f937aa44..cc2e6ccce1 100644 --- a/src/mlpack/tests/activation_functions_test.cpp +++ b/src/mlpack/tests/activation_functions_test.cpp @@ -388,14 +388,14 @@ BOOST_AUTO_TEST_CASE(SELUFunctionDerivativeTest) selu.Backward(std::move(input), std::move(error), std::move(derivatives)); BOOST_REQUIRE_LE(arma::as_scalar(arma::abs(arma::mean(derivatives) - - selu.Lambda())), 10e-5); + selu.Lambda())), 10e-4); input.fill(-1); selu.Backward(std::move(input), std::move(error), std::move(derivatives)); BOOST_REQUIRE_LE(arma::as_scalar(arma::abs(arma::mean(derivatives) - - selu.Lambda()*(selu.Alpha()-1))), 10e-5); + selu.Lambda() * (selu.Alpha() - 1))), 10e-4); } /**