Merge commit 'refs/pull/2880/head' of https://github.com/mlpack/mlpack

src/mlpack/methods/ann/activation_functions/CMakeLists.txt

@@ -20,6 +20,7 @@ set(SOURCES
   poisson1_function.hpp
   gaussian_function.hpp
   hard_swish_function.hpp
+  tanh_exponential_function.hpp
 )
 
 # Add directory name to sources.

src/mlpack/methods/ann/activation_functions/tanh_exponential_function.hpp

@@ -0,0 +1,83 @@
+/**
+ * @file methods/ann/activation_functions/tanh_exponential_function.hpp
+ * @author Mayank Raj
+ *
+ * Definition and implementation of the Tanh exponential  function.
+ *
+ * mlpack is free software; you may redistribute it and/or modify it under the
+ * terms of the 3-clause BSD license.  You should have received a copy of the
+ * 3-clause BSD license along with mlpack.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef TANH_EXPONENTIAL_FUNCTION_HPP_INCLUDED
+#define TANH_EXPONENTIAL_FUNCTION_HPP_INCLUDED
+
+#include <mlpack/prereqs.hpp>
+
+namespace mlpack {
+namespace ann /** Artificial Neural Network. */ {
+
+/**
+ * The TanhExp function, defined by
+ *
+ * @f{eqnarray*}{
+ * f(x) = x * tanh(e^x)\\
+ * f'(x) = tanh(e^x) - x*e^x*(tanh(e^x)^2 - 1)\\
+ * @f}
+ */
+ class TanhExpFunction
+{
+ public:
+  /**
+   * Computes the TanhExp function.
+   *
+   * @param x Input data.
+   * @return f(x).
+   */
+  static double Fn(const double x)
+  {
+    return x*std::tanh(std::exp(x));
+  }
+
+  /**
+   * Computes the TanhExp function.
+   *
+   * @param x Input data.
+   * @param y The resulting output activation.
+   */
+  template<typename InputVecType, typename OutputVecType>
+  static void Fn(const InputVecType& x, OutputVecType& y)
+  {
+    y = x*arma::tanh(arma::exp(x));
+  }
+
+  /**
+   * Computes the first derivative of the TanhExp function.
+   *
+   * @param y Input activation.
+   * @return f'(x)
+   */
+  static double Deriv(const double y)
+  {
+    return std::tanh(std::exp(y)) -
+    y*std::exp(y)*(std::pow(std::tanh(std::exp(y)),2) - 1);
+  }
+
+  /**
+   * Computes the first derivatives of the tanh function.
+   *
+   * @param y Input activations.
+   * @param x The resulting derivatives.
+   */
+  template<typename InputVecType, typename OutputVecType>
+  static void Deriv(const InputVecType& y, OutputVecType& x)
+  {
+    x = arma::tanh(arma::exp(y)) -
+        y*arma::exp(y)*(arma::pow(arma::tanh(arma::exp(y)),2) - 1);
+  }
+}; // class TanhExpFunction
+
+} // namespace ann
+} // namespace mlpack
+
+#endif // TANH_EXPONENTIAL_FUNCTION_HPP_INCLUDED

src/mlpack/methods/ann/layer/base_layer.hpp

@@ -28,6 +28,7 @@
 #include <mlpack/methods/ann/activation_functions/elish_function.hpp>
 #include <mlpack/methods/ann/activation_functions/gaussian_function.hpp>
 #include <mlpack/methods/ann/activation_functions/hard_swish_function.hpp>
+#include <mlpack/methods/ann/activation_functions/tanh_exponential_function.hpp>
 
 namespace mlpack {
 namespace ann /** Artificial Neural Network. */ {
@@ -52,6 +53,7 @@ namespace ann /** Artificial Neural Network. */ {
  *  - ElliotLayer
  *  - GaussianLayer
  *  - HardSwishLayer
+ *  - TanhExpLayer
  *
  * @tparam ActivationFunction Activation function used for the embedding layer.
  * @tparam InputDataType Type of the input data (arma::colvec, arma::mat,
@@ -290,6 +292,17 @@ template <
 using HardSwishFunctionLayer = BaseLayer<
     ActivationFunction, InputDataType, OutputDataType>;
 
+    /**
+ * Standard TanhExp-Layer using the TanhExp activation function.
+ */
+template <
+    class ActivationFunction = TanhExpFunction,
+    typename InputDataType = arma::mat,
+    typename OutputDataType = arma::mat
+>
+using TanhExpFunctionLayer = BaseLayer<
+    ActivationFunction, InputDataType, OutputDataType>;
+
 } // namespace ann
 } // namespace mlpack
 

src/mlpack/tests/activation_functions_test.cpp

@@ -33,6 +33,7 @@
 #include <mlpack/methods/ann/activation_functions/poisson1_function.hpp>
 #include <mlpack/methods/ann/activation_functions/gaussian_function.hpp>
 #include <mlpack/methods/ann/activation_functions/hard_swish_function.hpp>
+#include <mlpack/methods/ann/activation_functions/tanh_exponential_function.hpp>
 
 #include "catch.hpp"
 
@@ -1220,3 +1221,18 @@ TEST_CASE("HardSwishFunctionTest", "[ActivationFunctionsTest]")
   CheckDerivativeCorrect<HardSwishFunction>
       (desiredActivations, desiredDerivatives);
 }
+
+/**
+ * Basic test of the TanhExp function.
+ */
+TEST_CASE("TanhExpFunctionTest", "[ActivationFunctionsTest]")
+{
+  const arma::colvec desiredActivations("-0.26903 0.3.20000 0.4.50000 0.0000 \
+                                         0.99133 -0.35214 2.0 0.0000");
+
+  const arma::colvec desiredDerivatives("-0.13126 1.0000 1.0000 0 \
+                                         1.383 0.029873 1 0.76159");
+
+  CheckActivationCorrect<TanhExpFunction>(activationData, desiredActivations);
+  CheckDerivativeCorrect<TanhExpFunction>(desiredActivations, desiredDerivatives);
+}