diff --git a/HISTORY.md b/HISTORY.md index 1e5bc7565d..f63f9890c0 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -2,6 +2,8 @@ ###### ????-??-?? * Updated terminal state for Pendulum environment (#2354). + * Added `EliSH` activation function (#2323). + ### mlpack 3.3.0 ###### 2020-04-07 * Templated return type of `Forward function` of loss functions (#2339). diff --git a/src/mlpack/methods/ann/activation_functions/CMakeLists.txt b/src/mlpack/methods/ann/activation_functions/CMakeLists.txt index ad995034c7..f155de71ed 100644 --- a/src/mlpack/methods/ann/activation_functions/CMakeLists.txt +++ b/src/mlpack/methods/ann/activation_functions/CMakeLists.txt @@ -11,6 +11,7 @@ set(SOURCES mish_function.hpp lisht_function.hpp gelu_function.hpp + elish_function.hpp ) # Add directory name to sources. diff --git a/src/mlpack/methods/ann/activation_functions/elish_function.hpp b/src/mlpack/methods/ann/activation_functions/elish_function.hpp new file mode 100644 index 0000000000..3d13bfd8b7 --- /dev/null +++ b/src/mlpack/methods/ann/activation_functions/elish_function.hpp @@ -0,0 +1,119 @@ +/** + * @file elish_function.hpp + * @author Bisakh Mondal + * + * Definition and implementation of the ELiSH function as described by + * Mina Basirat and Peter M. Roth. + * + * For more information see the following paper + * + * @code + * @misc{Basirat2018, + * title = {The Quest for the Golden Activation Function}, + * author = {Mina Basirat and Peter M. Roth}, + * year = {2018}, + * url = {https://arxiv.org/pdf/1808.00783.pdf}, + * eprint = {1808.00783}, + * archivePrefix = {arXiv}, + * primaryClass = {cs.NE} } + * @endcode + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_METHODS_ANN_ACTIVATION_FUNCTIONS_ELISH_FUNCTION_HPP +#define MLPACK_METHODS_ANN_ACTIVATION_FUNCTIONS_ELISH_FUNCTION_HPP + +#include + +namespace mlpack { +namespace ann /** Artificial Neural Network. */ { + +/** + * The ELiSH function, defined by + * + * @f{eqnarray*}{ + * f(x) &=& \left\{ + * \begin{array}{lr} + * \frac{x}{1+e^{-x}} & : x \ge 0 \\ + * \frac{e^x-1}{1+e^{-x}} & : x < 0 + * \end{array} + * \right \\ + * f'(x) &=& \left\{ + * \begin{array}{lr} + * \frac{1}{1+e^{-x}} + + * \frac{xe^{-x}}{ \left(1+e^{-x}}\right)^2} & : x \ge 0 \\ + * e^x-\frac{2}{1+e^x}+\frac{2}{\left(1+e^x}\right)^2} & : x < 0 + * \end{array} + * \right. + * @f} + */ +class ElishFunction +{ + public: + /** + * Computes the ELiSH function. + * + * @param x Input data. + * @return f(x). + */ + static double Fn(const double x) + { + if (x < 0.0) + return (std::exp(x) - 1) / (1 + std::exp(-x)); + + return x / (1 + std::exp(-x)); + } + + /** + * Computes the ELiSH function. + * + * @param x Input data. + * @param y The resulting output activations. + */ + template + static void Fn(const InputVecType& x, OutputVecType& y) + { + y = ((x < 0.0) % ((arma::exp(x) -1) / (1 + arma::exp(-x)))) + + ((x >= 0.0) % (x / (1 + arma::exp(-x)))); + } + + /** + * Computes the first derivatives of ELiSH function. + * + * @param y Input data. + * @return f'(x). + */ + static double Deriv(const double y) + { + if (y < 0.0) + { + return std::exp(y) - 2 / (1 + std::exp(y)) + + 2 / std::pow(1 + std::exp(y) , 2); + } + + return 1 / (1 + std::exp(-y)) + y * std::exp(-y) / + std::pow(1 + std::exp(-y) , 2); + } + + /** + * Computes the first derivatives of the ELiSH function. + * + * @param y Input activations. + * @param x The resulting derivatives. + */ + template + static void Deriv(const InputVecType& y, OutputVecType& x) + { + x = ((y < 0.0) % (arma::exp(y) - 2 / (1 + arma::exp(y)) + 2 / arma::pow( + 1 + arma::exp(y), 2))) + ((y >= 0.0) % (1 / (1 + arma::exp(-y)) + y % + arma::exp(-y) / arma::pow(1 + arma::exp(-y), 2))); + } +}; // class ElishFunction + +} // namespace ann +} // namespace mlpack + +#endif diff --git a/src/mlpack/methods/ann/layer/base_layer.hpp b/src/mlpack/methods/ann/layer/base_layer.hpp index 6559845d43..c516c7ae3b 100644 --- a/src/mlpack/methods/ann/layer/base_layer.hpp +++ b/src/mlpack/methods/ann/layer/base_layer.hpp @@ -24,6 +24,7 @@ #include #include #include +#include namespace mlpack { namespace ann /** Artificial Neural Network. */ { @@ -38,6 +39,13 @@ namespace ann /** Artificial Neural Network. */ { * - IdentityLayer * - ReLULayer * - TanHLayer + * - SoftplusLayer + * - HardSigmoidLayer + * - SwishLayer + * - MishLayer + * - LiSHTLayer + * - GELULayer + * - ELiSHLayer * * @tparam ActivationFunction Activation function used for the embedding layer. * @tparam InputDataType Type of the input data (arma::colvec, arma::mat, @@ -232,6 +240,17 @@ template < using GELUFunctionLayer = BaseLayer< ActivationFunction, InputDataType, OutputDataType>; +/** + * Standard ELiSH-Layer using the ELiSH activation function. + */ +template < + class ActivationFunction = ElishFunction, + typename InputDataType = arma::mat, + typename OutputDataType = arma::mat +> +using ElishFunctionLayer = BaseLayer< + ActivationFunction, InputDataType, OutputDataType>; + } // namespace ann } // namespace mlpack diff --git a/src/mlpack/tests/activation_functions_test.cpp b/src/mlpack/tests/activation_functions_test.cpp index 78b32af158..a5987995d0 100644 --- a/src/mlpack/tests/activation_functions_test.cpp +++ b/src/mlpack/tests/activation_functions_test.cpp @@ -24,6 +24,7 @@ #include #include #include +#include #include #include "test_tools.hpp" @@ -36,7 +37,7 @@ BOOST_AUTO_TEST_SUITE(ActivationFunctionsTest); // Generate dataset for activation function tests. const arma::colvec activationData("-2 3.2 4.5 -100.2 1 -1 2 0"); -/* +/** * Implementation of the activation function test. * * @param input Input data used for evaluating the activation function. @@ -45,7 +46,8 @@ const arma::colvec activationData("-2 3.2 4.5 -100.2 1 -1 2 0"); * @tparam ActivationFunction Activation function used for the check. */ template -void CheckActivationCorrect(const arma::colvec input, const arma::colvec target) +void CheckActivationCorrect(const arma::colvec input, + const arma::colvec target) { // Test the activation function using a single value as input. for (size_t i = 0; i < target.n_elem; i++) @@ -63,7 +65,7 @@ void CheckActivationCorrect(const arma::colvec input, const arma::colvec target) } } -/* +/** * Implementation of the activation function derivative test. * * @param input Input data used for evaluating the activation function. @@ -72,7 +74,8 @@ void CheckActivationCorrect(const arma::colvec input, const arma::colvec target) * @tparam ActivationFunction Activation function used for the check. */ template -void CheckDerivativeCorrect(const arma::colvec input, const arma::colvec target) +void CheckDerivativeCorrect(const arma::colvec input, + const arma::colvec target) { // Test the calculation of the derivatives using a single value as input. for (size_t i = 0; i < target.n_elem; i++) @@ -90,7 +93,7 @@ void CheckDerivativeCorrect(const arma::colvec input, const arma::colvec target) } } -/* +/** * Implementation of the activation function inverse test. * * @param input Input data used for evaluating the activation function. @@ -119,7 +122,7 @@ void CheckInverseCorrect(const arma::colvec input) } } -/* +/** * Implementation of the HardTanH activation function test. The function is * implemented as a HardTanH Layer in hard_tanh.hpp * @@ -140,11 +143,12 @@ void CheckHardTanHActivationCorrect(const arma::colvec input, } } -/* +/** * Implementation of the HardTanH activation function derivative test. The * derivative is implemented as HardTanH Layer in hard_tanh.hpp * - * @param input Input data used for evaluating the HardTanH activation function. + * @param input Input data used for evaluating the HardTanH activation + * function. * @param target Target data used to evaluate the HardTanH activation. */ void CheckHardTanHDerivativeCorrect(const arma::colvec input, @@ -165,11 +169,12 @@ void CheckHardTanHDerivativeCorrect(const arma::colvec input, } } -/* +/** * Implementation of the LeakyReLU activation function test. The function is * implemented as LeakyReLU layer in the file leaky_relu.hpp * - * @param input Input data used for evaluating the LeakyReLU activation function. + * @param input Input data used for evaluating the LeakyReLU activation + * function. * @param target Target data used to evaluate the LeakyReLU activation. */ void CheckLeakyReLUActivationCorrect(const arma::colvec input, @@ -186,12 +191,13 @@ void CheckLeakyReLUActivationCorrect(const arma::colvec input, } } -/* +/** * Implementation of the LeakyReLU activation function derivative test. * The derivative function is implemented as LeakyReLU layer in the file * leaky_relu_layer.hpp * - * @param input Input data used for evaluating the LeakyReLU activation function. + * @param input Input data used for evaluating the LeakyReLU activation + * function. * @param target Target data used to evaluate the LeakyReLU activation. */ void CheckLeakyReLUDerivativeCorrect(const arma::colvec input, @@ -211,7 +217,7 @@ void CheckLeakyReLUDerivativeCorrect(const arma::colvec input, } } -/* +/** * Implementation of the ELU activation function test. The function is * implemented as ELU layer in the file elu.hpp * @@ -233,7 +239,7 @@ void CheckELUActivationCorrect(const arma::colvec input, } } -/* +/** * Implementation of the ELU activation function derivative test. The function * is implemented as ELU layer in the file elu.hpp * @@ -259,7 +265,7 @@ void CheckELUDerivativeCorrect(const arma::colvec input, } } -/* +/** * Implementation of the PReLU activation function test. The function * is implemented as PReLU layer in the file parametric_relu.hpp. * @@ -281,7 +287,7 @@ void CheckPReLUActivationCorrect(const arma::colvec input, } } -/* +/** * Implementation of the PReLU activation function derivative test. * The function is implemented as PReLU layer in the file * parametric_relu.hpp @@ -307,7 +313,7 @@ void CheckPReLUDerivativeCorrect(const arma::colvec input, } } -/* +/** * Implementation of the PReLU activation function gradient test. * The function is implemented as PReLU layer in the file * parametric_relu.hpp @@ -332,7 +338,7 @@ void CheckPReLUGradientCorrect(const arma::colvec input, BOOST_REQUIRE_CLOSE(gradient(0), target(0), 1e-3); } -/* +/** * Implementation of the Hard Shrink activation function test. The function is * implemented as Hard Shrink layer in the file hardshrink.hpp * @@ -353,12 +359,13 @@ void CheckHardShrinkActivationCorrect(const arma::colvec input, } } -/* +/** * Implementation of the HardShrink activation function derivative test. * The derivative function is implemented as HardShrink layer in the file * hardshrink.hpp * - * @param input Input data used for evaluating the HardShrink activation function. + * @param input Input data used for evaluating the HardShrink activation + * function. * @param target Target data used to evaluate the HardShrink activation. */ void CheckHardShrinkDerivativeCorrect(const arma::colvec input, @@ -378,11 +385,12 @@ void CheckHardShrinkDerivativeCorrect(const arma::colvec input, } } -/* +/** * Implementation of the Soft Shrink activation function test. The function is * implemented as Soft Shrink layer in the file softshrink.hpp. * - * @param input Input data used for evaluating the Soft Shrink activation function. + * @param input Input data used for evaluating the Soft Shrink activation + * function. * @param target Target data used to evaluate the Soft Shrink activation. */ void CheckSoftShrinkActivationCorrect(const arma::colvec input, @@ -399,12 +407,13 @@ void CheckSoftShrinkActivationCorrect(const arma::colvec input, } } -/* +/** * Implementation of the Soft Shrink activation function derivative test. * The derivative function is implemented as Soft Shrink layer in the file * softshrink.hpp * - * @param input Input data used for evaluating the Soft Shrink activation function. + * @param input Input data used for evaluating the Soft Shrink activation + * function. * @param target Target data used to evaluate the Soft Shrink activation. */ void CheckSoftShrinkDerivativeCorrect(const arma::colvec input, @@ -424,7 +433,7 @@ void CheckSoftShrinkDerivativeCorrect(const arma::colvec input, } } -/* +/** * Simple SELU activation test to check whether the mean and variance remain * invariant after passing normalized inputs through the function. */ @@ -445,7 +454,7 @@ BOOST_AUTO_TEST_CASE(SELUFunctionNormalizedTest) arma::var(output))), 0.1); } -/* +/** * Simple SELU activation test to check whether the mean and variance * vary significantly after passing unnormalized inputs through the function. */ @@ -467,7 +476,7 @@ BOOST_AUTO_TEST_CASE(SELUFunctionUnnormalizedTest) arma::var(output))), 0.1); } -/* +/** * Simple SELU derivative test to check whether the derivatives * produced by the activation function are correct. * @@ -577,7 +586,7 @@ BOOST_AUTO_TEST_CASE(LogisticFunctionTest) CheckActivationCorrect(activationData, desiredActivations); CheckDerivativeCorrect(desiredActivations, - desiredDerivatives); + desiredDerivatives); CheckInverseCorrect(activationData); } @@ -594,7 +603,7 @@ BOOST_AUTO_TEST_CASE(SoftsignFunctionTest) CheckActivationCorrect(activationData, desiredActivations); CheckDerivativeCorrect(desiredActivations, - desiredDerivatives); + desiredDerivatives); CheckInverseCorrect(desiredActivations); } @@ -621,7 +630,7 @@ BOOST_AUTO_TEST_CASE(RectifierFunctionTest) CheckActivationCorrect(activationData, desiredActivations); CheckDerivativeCorrect(desiredActivations, - desiredDerivatives); + desiredDerivatives); } /** @@ -684,7 +693,7 @@ BOOST_AUTO_TEST_CASE(SoftplusFunctionTest) CheckActivationCorrect(activationData, desiredActivations); CheckDerivativeCorrect(desiredActivations, - desiredDerivatives); + desiredDerivatives); CheckInverseCorrect(desiredActivations); } @@ -711,8 +720,8 @@ BOOST_AUTO_TEST_CASE(PReLUFunctionTest) BOOST_AUTO_TEST_CASE(CReLUFunctionTest) { const arma::colvec desiredActivations("0 3.2 4.5 0 \ - 1 0 2 0 2 0 0 \ - 100.2 0 1 0 0"); + 1 0 2 0 2 0 0 \ + 100.2 0 1 0 0"); const arma::colvec desiredDerivatives("0 0 0 0 \ 0 0 0 0"); @@ -750,7 +759,7 @@ BOOST_AUTO_TEST_CASE(SwishFunctionTest) CheckActivationCorrect(activationData, desiredActivations); CheckDerivativeCorrect(desiredActivations, - desiredDerivatives); + desiredDerivatives); } /** @@ -768,9 +777,9 @@ BOOST_AUTO_TEST_CASE(HardSigmoidFunctionTest) 0.2"); CheckActivationCorrect(activationData, - desiredActivations); + desiredActivations); CheckDerivativeCorrect(desiredActivations, - desiredDerivatives); + desiredDerivatives); } /** @@ -785,14 +794,14 @@ BOOST_AUTO_TEST_CASE(MishFunctionTest) -0.30340138 1.943959 0"); const arma::colvec desiredDerivatives("0.4382387 1.0159768849 \ - 1.0019108 0.6 \ - 1.0192586 0.40639898 \ - 1.0725079 0.6"); + 1.0019108 0.6 \ + 1.0192586 0.40639898 \ + 1.0725079 0.6"); CheckActivationCorrect(activationData, desiredActivations); CheckDerivativeCorrect(desiredActivations, - desiredDerivatives); + desiredDerivatives); } /** @@ -823,27 +832,18 @@ BOOST_AUTO_TEST_CASE(LiSHTFunctionTest) BOOST_AUTO_TEST_CASE(GELUFunctionTest) { // Calculated using torch.nn.gelu(). - const arma::colvec desiredActivations("-0.04540230591222 \ - 3.1981304348379158 \ - 4.5000 -0.0000 \ - 0.84119199060827676 \ - -0.15880800939172329 \ - 1.954597694087775 \ - 0.0000"); + const arma::colvec desiredActivations("-0.0454023 3.1981304 \ + 4.5 -0.0 0.84119199 \ + -0.158808 1.954597694 0.0"); - const arma::colvec desiredDerivatives("0.46379920685377229 \ - 1.0065302165778773 \ - 1.0000293221871797 \ - 0.5 \ - 1.0351344625840642 \ - 0.37435387859861063 \ - 1.0909840032535403 \ - 0.5"); + const arma::colvec desiredDerivatives("0.4637992 1.0065302 \ + 1.0000293 0.5 1.03513446 \ + 0.37435387 1.090984 0.5"); CheckActivationCorrect(activationData, desiredActivations); CheckDerivativeCorrect(desiredActivations, - desiredDerivatives); + desiredDerivatives); } /** @@ -862,6 +862,26 @@ BOOST_AUTO_TEST_CASE(HardShrinkFunctionTest) } /** + * Basic test of the EliSH function. + */ +BOOST_AUTO_TEST_CASE(ElishFunctionTest) +{ + // Manually-calculated using python-numpy module. + const arma::colvec desiredActivations("-0.10307056 3.0746696 4.4505587 \ + -3.0457406e-44 0.731058578 \ + -0.1700034 1.76159415 0.0 "); + + const arma::colvec desiredDerivatives("0.4033889 1.0856292 \ + 1.03921798 0.5 0.83540389 \ + 0.34725726 1.07378804 0.5"); + + CheckActivationCorrect(activationData, + desiredActivations); + CheckDerivativeCorrect(desiredActivations, + desiredDerivatives); +} + + /** * Basic test of the Soft Shrink function. */ BOOST_AUTO_TEST_CASE(SoftShrinkFunctionTest)