apaam/mlpack
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #1346 from Prabhat-IIT/SELU

Browse Source 
Selu Activation Function.
This commit is contained in:
Marcus Edel
2018-04-07 16:32:52 +02:00
committed by GitHub
parent bb503e2faa d156b8d85e
commit ea634094bb
3 changed files with 192 additions and 35 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/mlpack/methods/ann/layer/elu.hpp
+98 -32
View File
@@ -1,22 +1,20 @@
/**
* @file elu.hpp
* @author Vivek Pal
* @author Dakshit Agrawal
*
* Definition of the ELU activation function as descibed by Djork-Arne Clevert,
* Thomas Unterthiner and Sepp Hochreiter.
*
* For more information, read the following paper:
* Definition of the SELU function as introduced by
* Klambauer et. al. in Self Neural Networks. The SELU activation
* function keeps the mean and variance of the input invariant.
*
* @code
* @article{Clevert2015,
* author = {Djork{-}Arn{\'{e}} Clevert and Thomas Unterthiner and
* Sepp Hochreiter},
* title = {Fast and Accurate Deep Network Learning by Exponential Linear
* Units (ELUs)},
* journal = {CoRR},
* year = {2015}
* }
* @endcode
* In short, SELU = lambda * ELU, with 'alpha' and 'lambda' fixed for
* normalized inputs.
*
* Hence both ELU and SELU are implemented in the same file, with
* lambda = 1 for ELU function.
*
* mlpack is free software; you may redistribute it and/or modify it under the
* terms of the 3-clause BSD license. You should have received a copy of the
@@ -32,6 +30,21 @@ namespace mlpack {
namespace ann /** Artificial Neural Network. */ {
/**
* @note Make sure to use SELU activation function with normalized inputs and
* weights initialized with Lecun Normal Initialization.
*
* @tparam InputDataType Type of the input data (arma::colvec, arma::mat,
* arma::sp_mat or arma::cube).
* @tparam OutputDataType Type of the output data (arma::colvec, arma::mat,
* arma::sp_mat or arma::cube).
*/
template <
typename InputDataType = arma::mat,
typename OutputDataType = arma::mat
>
class ELU
{
/**
* The ELU activation function, defined by
*
* @f{eqnarray*}{
@@ -49,27 +62,66 @@ namespace ann /** Artificial Neural Network. */ {
* \right.
* @f}
*
* @tparam InputDataType Type of the input data (arma::colvec, arma::mat,
* arma::sp_mat or arma::cube).
* @tparam OutputDataType Type of the output data (arma::colvec, arma::mat,
* arma::sp_mat or arma::cube).
* For more information, read the following paper:
*
* @code
* @article{Clevert2015,
* author = {Djork{-}Arn{\'{e}} Clevert and Thomas Unterthiner and
* Sepp Hochreiter},
* title = {Fast and Accurate Deep Network Learning by Exponential Linear
* Units (ELUs)},
* journal = {CoRR},
* year = {2015}
* }
* @endcode
*
*
* The SELU activation function is defined by
*
* @f{eqnarray*}{
* f(x) &=& \left\{
* \begin{array}{lr}
* lambda * x & : x > 0 \\
* lambda * alpha(e^x - 1) & : x \le 0
* \end{array}
* \right. \\
* f'(x) &=& \left\{
* \begin{array}{lr}
* lambda & : x > 0 \\
* lambda * (y + alpha) & : x \le 0
* \end{array}
* \right.
* @f}
*
* For more information, read the following paper:
*
* @code
* @article{Klambauer2017,
* author = {Gunter Klambauer and Thomas Unterthiner and
* Andreas Mayr},
* title = {Self-Normalizing Neural Networks},
* journal = {Advances in Neural Information Processing Systems},
* year = {2017}
* }
* @endcode
*/
template <
typename InputDataType = arma::mat,
typename OutputDataType = arma::mat
>
class ELU
{
public:
/**
* Create the ELU object using the specified parameters. The non zero
* Create the ELU object.
*
* NOTE: Use this constructor for SELU activation function.
*
*/
ELU();
/**
* Create the ELU object using the specified parameter. The non zero
* gradient for negative inputs can be adjusted by specifying the ELU
* hyperparameter alpha (alpha > 0).
*
* @param alpha Scale parameter for the negative factor (Default alpha = 1.0).
* @note Use this constructor for ELU activation function.
* @param alpha Scale parameter for the negative factor.
*/
ELU(const double alpha = 1.0);
ELU(const double alpha);
/**
* Ordinary feed forward pass of a neural network, evaluating the function
* f(x) by propagating the activity forward through f.
@@ -112,6 +164,9 @@ class ELU
//! Modify the non zero gradient.
double& Alpha() { return alpha; }
//! Get the lambda parameter.
double const& Lambda() const { return lambda; }
/**
* Serialize the layer.
*/
@@ -120,7 +175,7 @@ class ELU
private:
/**
* Computes the ELU function
* Computes the value of activation function.
*
* @param x Input data.
* @return f(x).
@@ -128,12 +183,14 @@ class ELU
double Fn(const double x)
{
if (x < DBL_MAX)
return (x > 0) ? x : alpha * (std::exp(x) - 1);
{
return (x > 0) ? lambda * x : lambda * alpha * (std::exp(x) - 1);
}
return 1.0;
}
/**
* Computes the ELU function using a dense matrix as input.
* Computes the value of activation function using a dense matrix as input.
*
* @param x Input data.
* @param y The resulting output activation.
@@ -141,7 +198,7 @@ class ELU
template<typename eT>
void Fn(const arma::Mat<eT>& x, arma::Mat<eT>& y)
{
y = x;
y.set_size(size(x));
for (size_t i = 0; i < x.n_elem; i++)
{
@@ -150,18 +207,18 @@ class ELU
}
/**
* Computes the first derivative of the ELU function.
* Computes the first derivative of the activation function.
*
* @param x Input data.
* @return f'(x)
*/
double Deriv(const double y)
{
return (y > 0) ? 1 : (y + alpha);
return (y > 0) ? lambda : lambda * (y + alpha);
}
/**
* Computes the first derivative of the ELU function.
* Computes the first derivative of the activation function.
*
* @param y Input activations.
* @param x The resulting derivatives.
@@ -188,8 +245,17 @@ class ELU
OutputDataType outputParameter;
//! ELU Hyperparameter (0 < alpha)
//! SELU parameter fixed to 1.6732632423543774 for normalized inputs.
double alpha;
//! Lambda Parameter used for multiplication of ELU function.
//! For ELU activation function, lambda = 1.
//! For SELU activation function, lambda = 1.0507009873554802 for normalized
//! inputs.
double lambda;
}; // class ELU
// Template alias for SELU using ELU class
using SELU = ELU<arma::mat, arma::mat>;
} // namespace ann
} // namespace mlpack
src/mlpack/methods/ann/layer/elu_impl.hpp
+19 -1
View File
@@ -1,10 +1,15 @@
/**
* @file elu_impl.hpp
* @author Vivek Pal
* @author Dakshit Agrawal
*
* Implementation of the ELU activation function as descibed by Djork-Arne
* Clevert, Thomas Unterthiner and Sepp Hochreiter.
*
* Implementation of the SELU function as introduced by Klambauer et. al. in
* Self Neural Networks. The SELU activation function keeps the mean and
* variance of the input invariant.
*
* mlpack is free software; you may redistribute it and/or modify it under the
* terms of the 3-clause BSD license. You should have received a copy of the
* 3-clause BSD license along with mlpack. If not, see
@@ -19,9 +24,21 @@
namespace mlpack {
namespace ann /** Artificial Neural Network. */ {
// This constructor is called for SELU activation function. The values of
// alpha and lambda are constant for normalized inputs.
template<typename InputDataType, typename OutputDataType>
ELU<InputDataType, OutputDataType>::ELU() :
alpha(1.6732632423543774),
lambda(1.0507009873554802)
{
// Nothing to do here.
}
// This constructor is called for ELU activation function. The value of lambda
// is fixed and equal to 1. 'alpha' is a hyperparameter.
template<typename InputDataType, typename OutputDataType>
ELU<InputDataType, OutputDataType>::ELU(
const double alpha) : alpha(alpha)
const double alpha) : alpha(alpha), lambda(1)
{
// Nothing to do here.
}
@@ -51,6 +68,7 @@ void ELU<InputDataType, OutputDataType>::serialize(
const unsigned int /* version */)
{
ar & BOOST_SERIALIZATION_NVP(alpha);
ar & BOOST_SERIALIZATION_NVP(lambda);
}
} // namespace ann
src/mlpack/tests/activation_functions_test.cpp
+75 -2
View File
@@ -217,7 +217,8 @@ void CheckLeakyReLUDerivativeCorrect(const arma::colvec input,
void CheckELUActivationCorrect(const arma::colvec input,
const arma::colvec target)
{
ELU<> lrf;
// Initialize ELU object with alpha = 1.0.
ELU<> lrf(1.0);
// Test the activation function using the entire vector as input.
arma::colvec activations;
@@ -238,7 +239,8 @@ void CheckELUActivationCorrect(const arma::colvec input,
void CheckELUDerivativeCorrect(const arma::colvec input,
const arma::colvec target)
{
ELU<> lrf;
// Initialize ELU object with alpha = 1.0.
ELU<> lrf(1.0);
// Test the calculation of the derivatives using the entire vector as input.
arma::colvec derivatives;
@@ -325,6 +327,77 @@ void CheckPReLUGradientCorrect(const arma::colvec input,
BOOST_REQUIRE_CLOSE(gradient(0), target(0), 1e-3);
}
/*
* Simple SELU activation test to check whether the mean and variance remain
* invariant after passing normalized inputs through the function.
*/
BOOST_AUTO_TEST_CASE(SELUFunctionNormalizedTest)
{
arma::mat input = arma::randn<arma::mat>(1000, 1);
arma::mat output;
SELU selu;
selu.Forward(std::move(input), output);
BOOST_REQUIRE_LE(arma::as_scalar(arma::abs(arma::mean(input) -
arma::mean(output))), 0.1);
BOOST_REQUIRE_LE(arma::as_scalar(arma::abs(arma::var(input) -
arma::var(output))), 0.1);
}
/*
* Simple SELU activation test to check whether the mean and variance
* vary significantly after passing unnormalized inputs through the function.
*/
BOOST_AUTO_TEST_CASE(SELUFunctionUnnormalizedTest)
{
const arma::colvec input("5.96402758 0.9966824 0.99975321 1 \
7.76159416 -0.76159416 0.96402758 8");
arma::mat output;
SELU selu;
selu.Forward(std::move(input), output);
BOOST_REQUIRE_GE(arma::as_scalar(arma::abs(arma::mean(input) -
arma::mean(output))), 0.1);
BOOST_REQUIRE_GE(arma::as_scalar(arma::abs(arma::var(input) -
arma::var(output))), 0.1);
}
/*
* Simple SELU derivative test to check whether the derivatives
* produced by the activation function are correct.
*
*/
BOOST_AUTO_TEST_CASE(SELUFunctionDerivativeTest)
{
arma::mat input = arma::ones<arma::mat>(1000, 1);
arma::mat error = arma::ones<arma::mat>(input.n_elem, 1);
arma::mat derivatives;
SELU selu;
selu.Backward(std::move(input), std::move(error), std::move(derivatives));
BOOST_REQUIRE_LE(arma::as_scalar(arma::abs(arma::mean(derivatives) -
selu.Lambda())), 10e-4);
input.fill(-1);
selu.Backward(std::move(input), std::move(error), std::move(derivatives));
BOOST_REQUIRE_LE(arma::as_scalar(arma::abs(arma::mean(derivatives) -
selu.Lambda() * (selu.Alpha() - 1))), 10e-4);
}
/**
* Basic test of the tanh function.
*/