apaam/mlpack
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'master' into patch-2

Browse Source
This commit is contained in:
Nishant Kumar
2020-04-08 12:16:37 +05:30
committed by GitHub
parent dc3fd7b9c0 a129c5069d
commit de55dcdbba
5 changed files with 217 additions and 56 deletions
Download Patch File Download Diff File Expand all files Collapse all files
HISTORY.md
+2
View File
@@ -2,6 +2,8 @@
###### ????-??-??
* Updated terminal state for Pendulum environment (#2354).
* Added `EliSH` activation function (#2323).
### mlpack 3.3.0
###### 2020-04-07
* Templated return type of `Forward function` of loss functions (#2339).
src/mlpack/methods/ann/activation_functions/CMakeLists.txt
+1
View File
@@ -11,6 +11,7 @@ set(SOURCES
mish_function.hpp
lisht_function.hpp
gelu_function.hpp
elish_function.hpp
)
# Add directory name to sources.
src/mlpack/methods/ann/activation_functions/elish_function.hpp
+119
View File
@@ -0,0 +1,119 @@
/**
* @file elish_function.hpp
* @author Bisakh Mondal
*
* Definition and implementation of the ELiSH function as described by
* Mina Basirat and Peter M. Roth.
*
* For more information see the following paper
*
* @code
* @misc{Basirat2018,
* title = {The Quest for the Golden Activation Function},
* author = {Mina Basirat and Peter M. Roth},
* year = {2018},
* url = {https://arxiv.org/pdf/1808.00783.pdf},
* eprint = {1808.00783},
* archivePrefix = {arXiv},
* primaryClass = {cs.NE} }
* @endcode
*
* mlpack is free software; you may redistribute it and/or modify it under the
* terms of the 3-clause BSD license. You should have received a copy of the
* 3-clause BSD license along with mlpack. If not, see
* http://www.opensource.org/licenses/BSD-3-Clause for more information.
*/
#ifndef MLPACK_METHODS_ANN_ACTIVATION_FUNCTIONS_ELISH_FUNCTION_HPP
#define MLPACK_METHODS_ANN_ACTIVATION_FUNCTIONS_ELISH_FUNCTION_HPP
#include <mlpack/prereqs.hpp>
namespace mlpack {
namespace ann /** Artificial Neural Network. */ {
/**
* The ELiSH function, defined by
*
* @f{eqnarray*}{
* f(x) &=& \left\{
* \begin{array}{lr}
* \frac{x}{1+e^{-x}} & : x \ge 0 \\
* \frac{e^x-1}{1+e^{-x}} & : x < 0
* \end{array}
* \right \\
* f'(x) &=& \left\{
* \begin{array}{lr}
* \frac{1}{1+e^{-x}} +
* \frac{xe^{-x}}{ \left(1+e^{-x}}\right)^2} & : x \ge 0 \\
* e^x-\frac{2}{1+e^x}+\frac{2}{\left(1+e^x}\right)^2} & : x < 0
* \end{array}
* \right.
* @f}
*/
class ElishFunction
{
public:
/**
* Computes the ELiSH function.
*
* @param x Input data.
* @return f(x).
*/
static double Fn(const double x)
{
if (x < 0.0)
return (std::exp(x) - 1) / (1 + std::exp(-x));
return x / (1 + std::exp(-x));
}
/**
* Computes the ELiSH function.
*
* @param x Input data.
* @param y The resulting output activations.
*/
template<typename InputVecType, typename OutputVecType>
static void Fn(const InputVecType& x, OutputVecType& y)
{
y = ((x < 0.0) % ((arma::exp(x) -1) / (1 + arma::exp(-x))))
+ ((x >= 0.0) % (x / (1 + arma::exp(-x))));
}
/**
* Computes the first derivatives of ELiSH function.
*
* @param y Input data.
* @return f'(x).
*/
static double Deriv(const double y)
{
if (y < 0.0)
{
return std::exp(y) - 2 / (1 + std::exp(y)) +
2 / std::pow(1 + std::exp(y) , 2);
}
return 1 / (1 + std::exp(-y)) + y * std::exp(-y) /
std::pow(1 + std::exp(-y) , 2);
}
/**
* Computes the first derivatives of the ELiSH function.
*
* @param y Input activations.
* @param x The resulting derivatives.
*/
template<typename InputVecType, typename OutputVecType>
static void Deriv(const InputVecType& y, OutputVecType& x)
{
x = ((y < 0.0) % (arma::exp(y) - 2 / (1 + arma::exp(y)) + 2 / arma::pow(
1 + arma::exp(y), 2))) + ((y >= 0.0) % (1 / (1 + arma::exp(-y)) + y %
arma::exp(-y) / arma::pow(1 + arma::exp(-y), 2)));
}
}; // class ElishFunction
} // namespace ann
} // namespace mlpack
#endif
src/mlpack/methods/ann/layer/base_layer.hpp
+19
View File
@@ -24,6 +24,7 @@
#include <mlpack/methods/ann/activation_functions/mish_function.hpp>
#include <mlpack/methods/ann/activation_functions/lisht_function.hpp>
#include <mlpack/methods/ann/activation_functions/gelu_function.hpp>
#include <mlpack/methods/ann/activation_functions/elish_function.hpp>
namespace mlpack {
namespace ann /** Artificial Neural Network. */ {
@@ -38,6 +39,13 @@ namespace ann /** Artificial Neural Network. */ {
* - IdentityLayer
* - ReLULayer
* - TanHLayer
* - SoftplusLayer
* - HardSigmoidLayer
* - SwishLayer
* - MishLayer
* - LiSHTLayer
* - GELULayer
* - ELiSHLayer
*
* @tparam ActivationFunction Activation function used for the embedding layer.
* @tparam InputDataType Type of the input data (arma::colvec, arma::mat,
@@ -232,6 +240,17 @@ template <
using GELUFunctionLayer = BaseLayer<
ActivationFunction, InputDataType, OutputDataType>;
/**
* Standard ELiSH-Layer using the ELiSH activation function.
*/
template <
class ActivationFunction = ElishFunction,
typename InputDataType = arma::mat,
typename OutputDataType = arma::mat
>
using ElishFunctionLayer = BaseLayer<
ActivationFunction, InputDataType, OutputDataType>;
} // namespace ann
} // namespace mlpack
src/mlpack/tests/activation_functions_test.cpp
+76 -56
View File
@@ -24,6 +24,7 @@
#include <mlpack/methods/ann/activation_functions/mish_function.hpp>
#include <mlpack/methods/ann/activation_functions/lisht_function.hpp>
#include <mlpack/methods/ann/activation_functions/gelu_function.hpp>
#include <mlpack/methods/ann/activation_functions/elish_function.hpp>
#include <boost/test/unit_test.hpp>
#include "test_tools.hpp"
@@ -36,7 +37,7 @@ BOOST_AUTO_TEST_SUITE(ActivationFunctionsTest);
// Generate dataset for activation function tests.
const arma::colvec activationData("-2 3.2 4.5 -100.2 1 -1 2 0");
/*
/**
* Implementation of the activation function test.
*
* @param input Input data used for evaluating the activation function.
@@ -45,7 +46,8 @@ const arma::colvec activationData("-2 3.2 4.5 -100.2 1 -1 2 0");
* @tparam ActivationFunction Activation function used for the check.
*/
template<class ActivationFunction>
void CheckActivationCorrect(const arma::colvec input, const arma::colvec target)
void CheckActivationCorrect(const arma::colvec input,
const arma::colvec target)
{
// Test the activation function using a single value as input.
for (size_t i = 0; i < target.n_elem; i++)
@@ -63,7 +65,7 @@ void CheckActivationCorrect(const arma::colvec input, const arma::colvec target)
}
}
/*
/**
* Implementation of the activation function derivative test.
*
* @param input Input data used for evaluating the activation function.
@@ -72,7 +74,8 @@ void CheckActivationCorrect(const arma::colvec input, const arma::colvec target)
* @tparam ActivationFunction Activation function used for the check.
*/
template<class ActivationFunction>
void CheckDerivativeCorrect(const arma::colvec input, const arma::colvec target)
void CheckDerivativeCorrect(const arma::colvec input,
const arma::colvec target)
{
// Test the calculation of the derivatives using a single value as input.
for (size_t i = 0; i < target.n_elem; i++)
@@ -90,7 +93,7 @@ void CheckDerivativeCorrect(const arma::colvec input, const arma::colvec target)
}
}
/*
/**
* Implementation of the activation function inverse test.
*
* @param input Input data used for evaluating the activation function.
@@ -119,7 +122,7 @@ void CheckInverseCorrect(const arma::colvec input)
}
}
/*
/**
* Implementation of the HardTanH activation function test. The function is
* implemented as a HardTanH Layer in hard_tanh.hpp
*
@@ -140,11 +143,12 @@ void CheckHardTanHActivationCorrect(const arma::colvec input,
}
}
/*
/**
* Implementation of the HardTanH activation function derivative test. The
* derivative is implemented as HardTanH Layer in hard_tanh.hpp
*
* @param input Input data used for evaluating the HardTanH activation function.
* @param input Input data used for evaluating the HardTanH activation
* function.
* @param target Target data used to evaluate the HardTanH activation.
*/
void CheckHardTanHDerivativeCorrect(const arma::colvec input,
@@ -165,11 +169,12 @@ void CheckHardTanHDerivativeCorrect(const arma::colvec input,
}
}
/*
/**
* Implementation of the LeakyReLU activation function test. The function is
* implemented as LeakyReLU layer in the file leaky_relu.hpp
*
* @param input Input data used for evaluating the LeakyReLU activation function.
* @param input Input data used for evaluating the LeakyReLU activation
* function.
* @param target Target data used to evaluate the LeakyReLU activation.
*/
void CheckLeakyReLUActivationCorrect(const arma::colvec input,
@@ -186,12 +191,13 @@ void CheckLeakyReLUActivationCorrect(const arma::colvec input,
}
}
/*
/**
* Implementation of the LeakyReLU activation function derivative test.
* The derivative function is implemented as LeakyReLU layer in the file
* leaky_relu_layer.hpp
*
* @param input Input data used for evaluating the LeakyReLU activation function.
* @param input Input data used for evaluating the LeakyReLU activation
* function.
* @param target Target data used to evaluate the LeakyReLU activation.
*/
void CheckLeakyReLUDerivativeCorrect(const arma::colvec input,
@@ -211,7 +217,7 @@ void CheckLeakyReLUDerivativeCorrect(const arma::colvec input,
}
}
/*
/**
* Implementation of the ELU activation function test. The function is
* implemented as ELU layer in the file elu.hpp
*
@@ -233,7 +239,7 @@ void CheckELUActivationCorrect(const arma::colvec input,
}
}
/*
/**
* Implementation of the ELU activation function derivative test. The function
* is implemented as ELU layer in the file elu.hpp
*
@@ -259,7 +265,7 @@ void CheckELUDerivativeCorrect(const arma::colvec input,
}
}
/*
/**
* Implementation of the PReLU activation function test. The function
* is implemented as PReLU layer in the file parametric_relu.hpp.
*
@@ -281,7 +287,7 @@ void CheckPReLUActivationCorrect(const arma::colvec input,
}
}
/*
/**
* Implementation of the PReLU activation function derivative test.
* The function is implemented as PReLU layer in the file
* parametric_relu.hpp
@@ -307,7 +313,7 @@ void CheckPReLUDerivativeCorrect(const arma::colvec input,
}
}
/*
/**
* Implementation of the PReLU activation function gradient test.
* The function is implemented as PReLU layer in the file
* parametric_relu.hpp
@@ -332,7 +338,7 @@ void CheckPReLUGradientCorrect(const arma::colvec input,
BOOST_REQUIRE_CLOSE(gradient(0), target(0), 1e-3);
}
/*
/**
* Implementation of the Hard Shrink activation function test. The function is
* implemented as Hard Shrink layer in the file hardshrink.hpp
*
@@ -353,12 +359,13 @@ void CheckHardShrinkActivationCorrect(const arma::colvec input,
}
}
/*
/**
* Implementation of the HardShrink activation function derivative test.
* The derivative function is implemented as HardShrink layer in the file
* hardshrink.hpp
*
* @param input Input data used for evaluating the HardShrink activation function.
* @param input Input data used for evaluating the HardShrink activation
* function.
* @param target Target data used to evaluate the HardShrink activation.
*/
void CheckHardShrinkDerivativeCorrect(const arma::colvec input,
@@ -378,11 +385,12 @@ void CheckHardShrinkDerivativeCorrect(const arma::colvec input,
}
}
/*
/**
* Implementation of the Soft Shrink activation function test. The function is
* implemented as Soft Shrink layer in the file softshrink.hpp.
*
* @param input Input data used for evaluating the Soft Shrink activation function.
* @param input Input data used for evaluating the Soft Shrink activation
* function.
* @param target Target data used to evaluate the Soft Shrink activation.
*/
void CheckSoftShrinkActivationCorrect(const arma::colvec input,
@@ -399,12 +407,13 @@ void CheckSoftShrinkActivationCorrect(const arma::colvec input,
}
}
/*
/**
* Implementation of the Soft Shrink activation function derivative test.
* The derivative function is implemented as Soft Shrink layer in the file
* softshrink.hpp
*
* @param input Input data used for evaluating the Soft Shrink activation function.
* @param input Input data used for evaluating the Soft Shrink activation
* function.
* @param target Target data used to evaluate the Soft Shrink activation.
*/
void CheckSoftShrinkDerivativeCorrect(const arma::colvec input,
@@ -424,7 +433,7 @@ void CheckSoftShrinkDerivativeCorrect(const arma::colvec input,
}
}
/*
/**
* Simple SELU activation test to check whether the mean and variance remain
* invariant after passing normalized inputs through the function.
*/
@@ -445,7 +454,7 @@ BOOST_AUTO_TEST_CASE(SELUFunctionNormalizedTest)
arma::var(output))), 0.1);
}
/*
/**
* Simple SELU activation test to check whether the mean and variance
* vary significantly after passing unnormalized inputs through the function.
*/
@@ -467,7 +476,7 @@ BOOST_AUTO_TEST_CASE(SELUFunctionUnnormalizedTest)
arma::var(output))), 0.1);
}
/*
/**
* Simple SELU derivative test to check whether the derivatives
* produced by the activation function are correct.
*
@@ -577,7 +586,7 @@ BOOST_AUTO_TEST_CASE(LogisticFunctionTest)
CheckActivationCorrect<LogisticFunction>(activationData, desiredActivations);
CheckDerivativeCorrect<LogisticFunction>(desiredActivations,
desiredDerivatives);
desiredDerivatives);
CheckInverseCorrect<LogisticFunction>(activationData);
}
@@ -594,7 +603,7 @@ BOOST_AUTO_TEST_CASE(SoftsignFunctionTest)
CheckActivationCorrect<SoftsignFunction>(activationData, desiredActivations);
CheckDerivativeCorrect<SoftsignFunction>(desiredActivations,
desiredDerivatives);
desiredDerivatives);
CheckInverseCorrect<SoftsignFunction>(desiredActivations);
}
@@ -621,7 +630,7 @@ BOOST_AUTO_TEST_CASE(RectifierFunctionTest)
CheckActivationCorrect<RectifierFunction>(activationData, desiredActivations);
CheckDerivativeCorrect<RectifierFunction>(desiredActivations,
desiredDerivatives);
desiredDerivatives);
}
/**
@@ -684,7 +693,7 @@ BOOST_AUTO_TEST_CASE(SoftplusFunctionTest)
CheckActivationCorrect<SoftplusFunction>(activationData, desiredActivations);
CheckDerivativeCorrect<SoftplusFunction>(desiredActivations,
desiredDerivatives);
desiredDerivatives);
CheckInverseCorrect<SoftplusFunction>(desiredActivations);
}
@@ -711,8 +720,8 @@ BOOST_AUTO_TEST_CASE(PReLUFunctionTest)
BOOST_AUTO_TEST_CASE(CReLUFunctionTest)
{
const arma::colvec desiredActivations("0 3.2 4.5 0 \
1 0 2 0 2 0 0 \
100.2 0 1 0 0");
1 0 2 0 2 0 0 \
100.2 0 1 0 0");
const arma::colvec desiredDerivatives("0 0 0 0 \
0 0 0 0");
@@ -750,7 +759,7 @@ BOOST_AUTO_TEST_CASE(SwishFunctionTest)
CheckActivationCorrect<SwishFunction>(activationData, desiredActivations);
CheckDerivativeCorrect<SwishFunction>(desiredActivations,
desiredDerivatives);
desiredDerivatives);
}
/**
@@ -768,9 +777,9 @@ BOOST_AUTO_TEST_CASE(HardSigmoidFunctionTest)
0.2");
CheckActivationCorrect<HardSigmoidFunction>(activationData,
desiredActivations);
desiredActivations);
CheckDerivativeCorrect<HardSigmoidFunction>(desiredActivations,
desiredDerivatives);
desiredDerivatives);
}
/**
@@ -785,14 +794,14 @@ BOOST_AUTO_TEST_CASE(MishFunctionTest)
-0.30340138 1.943959 0");
const arma::colvec desiredDerivatives("0.4382387 1.0159768849 \
1.0019108 0.6 \
1.0192586 0.40639898 \
1.0725079 0.6");
1.0019108 0.6 \
1.0192586 0.40639898 \
1.0725079 0.6");
CheckActivationCorrect<MishFunction>(activationData,
desiredActivations);
CheckDerivativeCorrect<MishFunction>(desiredActivations,
desiredDerivatives);
desiredDerivatives);
}
/**
@@ -823,27 +832,18 @@ BOOST_AUTO_TEST_CASE(LiSHTFunctionTest)
BOOST_AUTO_TEST_CASE(GELUFunctionTest)
{
// Calculated using torch.nn.gelu().
const arma::colvec desiredActivations("-0.04540230591222 \
3.1981304348379158 \
4.5000 -0.0000 \
0.84119199060827676 \
-0.15880800939172329 \
1.954597694087775 \
0.0000");
const arma::colvec desiredActivations("-0.0454023 3.1981304 \
4.5 -0.0 0.84119199 \
-0.158808 1.954597694 0.0");
const arma::colvec desiredDerivatives("0.46379920685377229 \
1.0065302165778773 \
1.0000293221871797 \
0.5 \
1.0351344625840642 \
0.37435387859861063 \
1.0909840032535403 \
0.5");
const arma::colvec desiredDerivatives("0.4637992 1.0065302 \
1.0000293 0.5 1.03513446 \
0.37435387 1.090984 0.5");
CheckActivationCorrect<GELUFunction>(activationData,
desiredActivations);
CheckDerivativeCorrect<GELUFunction>(desiredActivations,
desiredDerivatives);
desiredDerivatives);
}
/**
@@ -862,6 +862,26 @@ BOOST_AUTO_TEST_CASE(HardShrinkFunctionTest)
}
/**
* Basic test of the EliSH function.
*/
BOOST_AUTO_TEST_CASE(ElishFunctionTest)
{
// Manually-calculated using python-numpy module.
const arma::colvec desiredActivations("-0.10307056 3.0746696 4.4505587 \
-3.0457406e-44 0.731058578 \
-0.1700034 1.76159415 0.0 ");
const arma::colvec desiredDerivatives("0.4033889 1.0856292 \
1.03921798 0.5 0.83540389 \
0.34725726 1.07378804 0.5");
CheckActivationCorrect<ElishFunction>(activationData,
desiredActivations);
CheckDerivativeCorrect<ElishFunction>(desiredActivations,
desiredDerivatives);
}
/**
* Basic test of the Soft Shrink function.
*/
BOOST_AUTO_TEST_CASE(SoftShrinkFunctionTest)