diff --git a/src/mlpack/methods/ann/layer/elu.hpp b/src/mlpack/methods/ann/layer/elu.hpp index 1211ee7154..d586514217 100644 --- a/src/mlpack/methods/ann/layer/elu.hpp +++ b/src/mlpack/methods/ann/layer/elu.hpp @@ -1,22 +1,20 @@ /** * @file elu.hpp * @author Vivek Pal + * @author Dakshit Agrawal * * Definition of the ELU activation function as descibed by Djork-Arne Clevert, * Thomas Unterthiner and Sepp Hochreiter. * - * For more information, read the following paper: + * Definition of the SELU function as introduced by + * Klambauer et. al. in Self Neural Networks. The SELU activation + * function keeps the mean and variance of the input invariant. * - * @code - * @article{Clevert2015, - * author = {Djork{-}Arn{\'{e}} Clevert and Thomas Unterthiner and - * Sepp Hochreiter}, - * title = {Fast and Accurate Deep Network Learning by Exponential Linear - * Units (ELUs)}, - * journal = {CoRR}, - * year = {2015} - * } - * @endcode + * In short, SELU = lambda * ELU, with 'alpha' and 'lambda' fixed for + * normalized inputs. + * + * Hence both ELU and SELU are implemented in the same file, with + * lambda = 1 for ELU function. * * mlpack is free software; you may redistribute it and/or modify it under the * terms of the 3-clause BSD license. You should have received a copy of the @@ -32,6 +30,21 @@ namespace mlpack { namespace ann /** Artificial Neural Network. */ { /** + * @note Make sure to use SELU activation function with normalized inputs and + * weights initialized with Lecun Normal Initialization. + * + * @tparam InputDataType Type of the input data (arma::colvec, arma::mat, + * arma::sp_mat or arma::cube). + * @tparam OutputDataType Type of the output data (arma::colvec, arma::mat, + * arma::sp_mat or arma::cube). + */ +template < + typename InputDataType = arma::mat, + typename OutputDataType = arma::mat +> +class ELU +{ + /** * The ELU activation function, defined by * * @f{eqnarray*}{ @@ -49,27 +62,66 @@ namespace ann /** Artificial Neural Network. */ { * \right. * @f} * - * @tparam InputDataType Type of the input data (arma::colvec, arma::mat, - * arma::sp_mat or arma::cube). - * @tparam OutputDataType Type of the output data (arma::colvec, arma::mat, - * arma::sp_mat or arma::cube). + * For more information, read the following paper: + * + * @code + * @article{Clevert2015, + * author = {Djork{-}Arn{\'{e}} Clevert and Thomas Unterthiner and + * Sepp Hochreiter}, + * title = {Fast and Accurate Deep Network Learning by Exponential Linear + * Units (ELUs)}, + * journal = {CoRR}, + * year = {2015} + * } + * @endcode + * + * + * The SELU activation function is defined by + * + * @f{eqnarray*}{ + * f(x) &=& \left\{ + * \begin{array}{lr} + * lambda * x & : x > 0 \\ + * lambda * alpha(e^x - 1) & : x \le 0 + * \end{array} + * \right. \\ + * f'(x) &=& \left\{ + * \begin{array}{lr} + * lambda & : x > 0 \\ + * lambda * (y + alpha) & : x \le 0 + * \end{array} + * \right. + * @f} + * + * For more information, read the following paper: + * + * @code + * @article{Klambauer2017, + * author = {Gunter Klambauer and Thomas Unterthiner and + * Andreas Mayr}, + * title = {Self-Normalizing Neural Networks}, + * journal = {Advances in Neural Information Processing Systems}, + * year = {2017} + * } + * @endcode */ -template < - typename InputDataType = arma::mat, - typename OutputDataType = arma::mat -> -class ELU -{ public: /** - * Create the ELU object using the specified parameters. The non zero + * Create the ELU object. + * + * NOTE: Use this constructor for SELU activation function. + * + */ + ELU(); + /** + * Create the ELU object using the specified parameter. The non zero * gradient for negative inputs can be adjusted by specifying the ELU * hyperparameter alpha (alpha > 0). * - * @param alpha Scale parameter for the negative factor (Default alpha = 1.0). + * @note Use this constructor for ELU activation function. + * @param alpha Scale parameter for the negative factor. */ - ELU(const double alpha = 1.0); - + ELU(const double alpha); /** * Ordinary feed forward pass of a neural network, evaluating the function * f(x) by propagating the activity forward through f. @@ -112,6 +164,9 @@ class ELU //! Modify the non zero gradient. double& Alpha() { return alpha; } + //! Get the lambda parameter. + double const& Lambda() const { return lambda; } + /** * Serialize the layer. */ @@ -120,7 +175,7 @@ class ELU private: /** - * Computes the ELU function + * Computes the value of activation function. * * @param x Input data. * @return f(x). @@ -128,12 +183,14 @@ class ELU double Fn(const double x) { if (x < DBL_MAX) - return (x > 0) ? x : alpha * (std::exp(x) - 1); + { + return (x > 0) ? lambda * x : lambda * alpha * (std::exp(x) - 1); + } return 1.0; } /** - * Computes the ELU function using a dense matrix as input. + * Computes the value of activation function using a dense matrix as input. * * @param x Input data. * @param y The resulting output activation. @@ -141,7 +198,7 @@ class ELU template void Fn(const arma::Mat& x, arma::Mat& y) { - y = x; + y.set_size(size(x)); for (size_t i = 0; i < x.n_elem; i++) { @@ -150,18 +207,18 @@ class ELU } /** - * Computes the first derivative of the ELU function. + * Computes the first derivative of the activation function. * * @param x Input data. * @return f'(x) */ double Deriv(const double y) { - return (y > 0) ? 1 : (y + alpha); + return (y > 0) ? lambda : lambda * (y + alpha); } /** - * Computes the first derivative of the ELU function. + * Computes the first derivative of the activation function. * * @param y Input activations. * @param x The resulting derivatives. @@ -188,8 +245,17 @@ class ELU OutputDataType outputParameter; //! ELU Hyperparameter (0 < alpha) + //! SELU parameter fixed to 1.6732632423543774 for normalized inputs. double alpha; + + //! Lambda Parameter used for multiplication of ELU function. + //! For ELU activation function, lambda = 1. + //! For SELU activation function, lambda = 1.0507009873554802 for normalized + //! inputs. + double lambda; }; // class ELU +// Template alias for SELU using ELU class +using SELU = ELU; } // namespace ann } // namespace mlpack diff --git a/src/mlpack/methods/ann/layer/elu_impl.hpp b/src/mlpack/methods/ann/layer/elu_impl.hpp index 3cc8246df1..1319a94de8 100644 --- a/src/mlpack/methods/ann/layer/elu_impl.hpp +++ b/src/mlpack/methods/ann/layer/elu_impl.hpp @@ -1,10 +1,15 @@ /** * @file elu_impl.hpp * @author Vivek Pal + * @author Dakshit Agrawal * * Implementation of the ELU activation function as descibed by Djork-Arne * Clevert, Thomas Unterthiner and Sepp Hochreiter. * + * Implementation of the SELU function as introduced by Klambauer et. al. in + * Self Neural Networks. The SELU activation function keeps the mean and + * variance of the input invariant. + * * mlpack is free software; you may redistribute it and/or modify it under the * terms of the 3-clause BSD license. You should have received a copy of the * 3-clause BSD license along with mlpack. If not, see @@ -19,9 +24,21 @@ namespace mlpack { namespace ann /** Artificial Neural Network. */ { +// This constructor is called for SELU activation function. The values of +// alpha and lambda are constant for normalized inputs. +template +ELU::ELU() : + alpha(1.6732632423543774), + lambda(1.0507009873554802) +{ + // Nothing to do here. +} + +// This constructor is called for ELU activation function. The value of lambda +// is fixed and equal to 1. 'alpha' is a hyperparameter. template ELU::ELU( - const double alpha) : alpha(alpha) + const double alpha) : alpha(alpha), lambda(1) { // Nothing to do here. } @@ -51,6 +68,7 @@ void ELU::serialize( const unsigned int /* version */) { ar & BOOST_SERIALIZATION_NVP(alpha); + ar & BOOST_SERIALIZATION_NVP(lambda); } } // namespace ann diff --git a/src/mlpack/tests/activation_functions_test.cpp b/src/mlpack/tests/activation_functions_test.cpp index 1d9e3f4557..cc2e6ccce1 100644 --- a/src/mlpack/tests/activation_functions_test.cpp +++ b/src/mlpack/tests/activation_functions_test.cpp @@ -217,7 +217,8 @@ void CheckLeakyReLUDerivativeCorrect(const arma::colvec input, void CheckELUActivationCorrect(const arma::colvec input, const arma::colvec target) { - ELU<> lrf; + // Initialize ELU object with alpha = 1.0. + ELU<> lrf(1.0); // Test the activation function using the entire vector as input. arma::colvec activations; @@ -238,7 +239,8 @@ void CheckELUActivationCorrect(const arma::colvec input, void CheckELUDerivativeCorrect(const arma::colvec input, const arma::colvec target) { - ELU<> lrf; + // Initialize ELU object with alpha = 1.0. + ELU<> lrf(1.0); // Test the calculation of the derivatives using the entire vector as input. arma::colvec derivatives; @@ -325,6 +327,77 @@ void CheckPReLUGradientCorrect(const arma::colvec input, BOOST_REQUIRE_CLOSE(gradient(0), target(0), 1e-3); } +/* + * Simple SELU activation test to check whether the mean and variance remain + * invariant after passing normalized inputs through the function. + */ +BOOST_AUTO_TEST_CASE(SELUFunctionNormalizedTest) +{ + arma::mat input = arma::randn(1000, 1); + + arma::mat output; + + SELU selu; + + selu.Forward(std::move(input), output); + + BOOST_REQUIRE_LE(arma::as_scalar(arma::abs(arma::mean(input) - + arma::mean(output))), 0.1); + + BOOST_REQUIRE_LE(arma::as_scalar(arma::abs(arma::var(input) - + arma::var(output))), 0.1); +} + +/* + * Simple SELU activation test to check whether the mean and variance + * vary significantly after passing unnormalized inputs through the function. + */ +BOOST_AUTO_TEST_CASE(SELUFunctionUnnormalizedTest) +{ + const arma::colvec input("5.96402758 0.9966824 0.99975321 1 \ + 7.76159416 -0.76159416 0.96402758 8"); + + arma::mat output; + + SELU selu; + + selu.Forward(std::move(input), output); + + BOOST_REQUIRE_GE(arma::as_scalar(arma::abs(arma::mean(input) - + arma::mean(output))), 0.1); + + BOOST_REQUIRE_GE(arma::as_scalar(arma::abs(arma::var(input) - + arma::var(output))), 0.1); +} + +/* + * Simple SELU derivative test to check whether the derivatives + * produced by the activation function are correct. + * + */ +BOOST_AUTO_TEST_CASE(SELUFunctionDerivativeTest) +{ + arma::mat input = arma::ones(1000, 1); + + arma::mat error = arma::ones(input.n_elem, 1); + + arma::mat derivatives; + + SELU selu; + + selu.Backward(std::move(input), std::move(error), std::move(derivatives)); + + BOOST_REQUIRE_LE(arma::as_scalar(arma::abs(arma::mean(derivatives) - + selu.Lambda())), 10e-4); + + input.fill(-1); + + selu.Backward(std::move(input), std::move(error), std::move(derivatives)); + + BOOST_REQUIRE_LE(arma::as_scalar(arma::abs(arma::mean(derivatives) - + selu.Lambda() * (selu.Alpha() - 1))), 10e-4); +} + /** * Basic test of the tanh function. */