From 2f4de23208eda8229a1ea0cc7c1564041b3f0a9c Mon Sep 17 00:00:00 2001 From: mayank raj Date: Wed, 17 Mar 2021 20:51:23 +0530 Subject: [PATCH 1/2] added tanh exponential activation function --- .../ann/activation_functions/CMakeLists.txt | 1 + .../tanh_exponential_function.hpp | 83 +++++++++++++++++++ src/mlpack/methods/ann/layer/base_layer.hpp | 13 +++ .../tests/activation_functions_test.cpp | 16 ++++ 4 files changed, 113 insertions(+) create mode 100644 src/mlpack/methods/ann/activation_functions/tanh_exponential_function.hpp diff --git a/src/mlpack/methods/ann/activation_functions/CMakeLists.txt b/src/mlpack/methods/ann/activation_functions/CMakeLists.txt index d5c0868c1c..1639817716 100644 --- a/src/mlpack/methods/ann/activation_functions/CMakeLists.txt +++ b/src/mlpack/methods/ann/activation_functions/CMakeLists.txt @@ -20,6 +20,7 @@ set(SOURCES poisson1_function.hpp gaussian_function.hpp hard_swish_function.hpp + tanh_exponential_function.hpp ) # Add directory name to sources. diff --git a/src/mlpack/methods/ann/activation_functions/tanh_exponential_function.hpp b/src/mlpack/methods/ann/activation_functions/tanh_exponential_function.hpp new file mode 100644 index 0000000000..8bbc3c3771 --- /dev/null +++ b/src/mlpack/methods/ann/activation_functions/tanh_exponential_function.hpp @@ -0,0 +1,83 @@ +/** + * @file methods/ann/activation_functions/tanh_exponential_function.hpp + * @author Mayank Raj + * + * Definition and implementation of the Tanh exponential function. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef TANH_EXPONENTIAL_FUNCTION_HPP_INCLUDED +#define TANH_EXPONENTIAL_FUNCTION_HPP_INCLUDED + +#include + +namespace mlpack { +namespace ann /** Artificial Neural Network. */ { + +/** + * The TanhExp function, defined by + * + * @f{eqnarray*}{ + * f(x) = x * tanh(e^x)\\ + * f'(x) = tanh(e^x) + x*e^x*(sech(e^x))^2\\ + * @f} + */ + class TanhExpFunction +{ + public: + /** + * Computes the TanhExp function. + * + * @param x Input data. + * @return f(x). + */ + static double Fn(const double x) + { + return x*std::tanh(std::exp(x)); + } + + /** + * Computes the TanhExp function. + * + * @param x Input data. + * @param y The resulting output activation. + */ + template + static void Fn(const InputVecType& x, OutputVecType& y) + { + y = x*arma::tanh(arma::exp(x)); + } + + /** + * Computes the first derivative of the TanhExp function. + * + * @param y Input activation. + * @return f'(x) + */ + static double Deriv(const double y) + { + return std::tanh(std::exp(y)) + + y*std::exp(y)*std::pow(std::sech(std::exp(y)),2); + } + + /** + * Computes the first derivatives of the tanh function. + * + * @param y Input activations. + * @param x The resulting derivatives. + */ + template + static void Deriv(const InputVecType& y, OutputVecType& x) + { + x = arma::tanh(arma::exp(y)) + + y*arma::exp(y)*arma::pow(arma::sech(arma::exp(y)),2); + } +}; // class TanhExpFunction + +} // namespace ann +} // namespace mlpack + +#endif // TANH_EXPONENTIAL_FUNCTION_HPP_INCLUDED diff --git a/src/mlpack/methods/ann/layer/base_layer.hpp b/src/mlpack/methods/ann/layer/base_layer.hpp index ae49f30fe6..a1dcca3e35 100644 --- a/src/mlpack/methods/ann/layer/base_layer.hpp +++ b/src/mlpack/methods/ann/layer/base_layer.hpp @@ -28,6 +28,7 @@ #include #include #include +#include namespace mlpack { namespace ann /** Artificial Neural Network. */ { @@ -52,6 +53,7 @@ namespace ann /** Artificial Neural Network. */ { * - ElliotLayer * - GaussianLayer * - HardSwishLayer + * - TanhExpLayer * * @tparam ActivationFunction Activation function used for the embedding layer. * @tparam InputDataType Type of the input data (arma::colvec, arma::mat, @@ -290,6 +292,17 @@ template < using HardSwishFunctionLayer = BaseLayer< ActivationFunction, InputDataType, OutputDataType>; + /** + * Standard TanhExp-Layer using the TanhExp activation function. + */ +template < + class ActivationFunction = TanhExpFunction, + typename InputDataType = arma::mat, + typename OutputDataType = arma::mat +> +using TanhExpFunctionLayer = BaseLayer< + ActivationFunction, InputDataType, OutputDataType>; + } // namespace ann } // namespace mlpack diff --git a/src/mlpack/tests/activation_functions_test.cpp b/src/mlpack/tests/activation_functions_test.cpp index 85cc1f7608..b1359d60e1 100644 --- a/src/mlpack/tests/activation_functions_test.cpp +++ b/src/mlpack/tests/activation_functions_test.cpp @@ -33,6 +33,7 @@ #include #include #include +#include #include "catch.hpp" @@ -1220,3 +1221,18 @@ TEST_CASE("HardSwishFunctionTest", "[ActivationFunctionsTest]") CheckDerivativeCorrect (desiredActivations, desiredDerivatives); } + +/** + * Basic test of the TanhExp function. + */ +TEST_CASE("TanhExpFunctionTest", "[ActivationFunctionsTest]") +{ + const arma::colvec desiredActivations("-0.26903 0.3.20000 0.4.50000 0.0000 \ + 0.99133 -0.35214 2.0 0.0000"); + + const arma::colvec desiredDerivatives("-0.13126 1.0000 1.0000 0 \ + 1.383 0.029873 1 0.76159"); + + CheckActivationCorrect(activationData, desiredActivations); + CheckDerivativeCorrect(desiredActivations, desiredDerivatives); +} From 6af0387171c5179a14dc7946bc2e6626b978d4e6 Mon Sep 17 00:00:00 2001 From: mayank raj Date: Wed, 17 Mar 2021 21:56:36 +0530 Subject: [PATCH 2/2] minor change --- .../activation_functions/tanh_exponential_function.hpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/mlpack/methods/ann/activation_functions/tanh_exponential_function.hpp b/src/mlpack/methods/ann/activation_functions/tanh_exponential_function.hpp index 8bbc3c3771..42576fc013 100644 --- a/src/mlpack/methods/ann/activation_functions/tanh_exponential_function.hpp +++ b/src/mlpack/methods/ann/activation_functions/tanh_exponential_function.hpp @@ -22,7 +22,7 @@ namespace ann /** Artificial Neural Network. */ { * * @f{eqnarray*}{ * f(x) = x * tanh(e^x)\\ - * f'(x) = tanh(e^x) + x*e^x*(sech(e^x))^2\\ + * f'(x) = tanh(e^x) - x*e^x*(tanh(e^x)^2 - 1)\\ * @f} */ class TanhExpFunction @@ -59,8 +59,8 @@ namespace ann /** Artificial Neural Network. */ { */ static double Deriv(const double y) { - return std::tanh(std::exp(y)) + - y*std::exp(y)*std::pow(std::sech(std::exp(y)),2); + return std::tanh(std::exp(y)) - + y*std::exp(y)*(std::pow(std::tanh(std::exp(y)),2) - 1); } /** @@ -72,8 +72,8 @@ namespace ann /** Artificial Neural Network. */ { template static void Deriv(const InputVecType& y, OutputVecType& x) { - x = arma::tanh(arma::exp(y)) + - y*arma::exp(y)*arma::pow(arma::sech(arma::exp(y)),2); + x = arma::tanh(arma::exp(y)) - + y*arma::exp(y)*(arma::pow(arma::tanh(arma::exp(y)),2) - 1); } }; // class TanhExpFunction