Adding gelu activation function

src/mlpack/methods/ann/activation_functions/CMakeLists.txt

@@ -9,6 +9,7 @@ set(SOURCES
   softplus_function.hpp
   swish_function.hpp
   mish_function.hpp
+  gelu_function.hpp
 )
 
 # Add directory name to sources.

src/mlpack/methods/ann/activation_functions/gelu_function.hpp

@@ -0,0 +1,91 @@
+
+/**
+ * @file gelu_function.hpp
+ * @author Himanshu Pathak
+ *
+ * Definition and implementation of the Gaussian Error Linear Unit (GELU)
+ * function.
+ *
+ * mlpack is free software; you may redistribute it and/or modify it under the
+ * terms of the 3-clause BSD license.  You should have received a copy of the
+ * 3-clause BSD license along with mlpack.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef MLPACK_METHODS_ANN_ACTIVATION_FUNCTIONS_GELU_FUNCTION_HPP
+#define MLPACK_METHODS_ANN_ACTIVATION_FUNCTIONS_GELU_FUNCTION_HPP
+# define PI          3.141592653589793238462643383279502884L /* pi */
+
+#include <mlpack/prereqs.hpp>
+
+namespace mlpack {
+namespace ann /** Artificial Neural Network. */ {
+
+/**
+ * The GELU function, defined by
+ *
+ * @f{eqnarray*}{
+ * f(x) = 0.5 * x * {1 + tanh[(2/pi)^(1/2) * (x + 0.044715 * x^3)]} \\
+ * f'(x) = 0.5 * tanh(0.0356774 * x^3) + 0.797885 * x) + 
+ *         (0.0535161x^3 + 0.398942 * x) * 
+ *         sech^2(0.0356774 * x^3+0.797885 * x) + 0.5\\
+ * @f}
+ */
+class GELUFunction
+{
+ public:
+  /**
+   * Computes the GELU function.
+   *
+   * @param x Input data.
+   * @return f(x).
+   */
+  static double Fn(const double x)
+  {
+    return 0.5 * x * (1 + std::tanh(std::sqrt(2/PI)*(x + 0.044715 * std::pow(x, 3))));
+  }
+
+  /**
+   * Computes the tanh function.
+   *
+   * @param x Input data.
+   * @param y The resulting output activation.
+   */
+  template<typename InputVecType, typename OutputVecType>
+  static void Fn(const InputVecType& x, OutputVecType& y)
+  {
+    y = 0.5 * x * (1 + arma::tanh(std::sqrt(2/PI)*(x + 0.044715 * arma::pow(x, 3))));
+  }
+
+  /**
+   * Computes the first derivative of the tanh function.
+   *
+   * @param y Input data.
+   * @return f'(x)
+   */
+  static double Deriv(const double y)
+  {
+    return 0.5 * std::tanh(0.0356774 * std::pow(y, 3) + 0.797885 * y) +
+           (0.0535161 * std::pow(y, 3) + 0.398942 * y) *
+           std::pow(1 / std::cosh(0.0356774 * std::pow(y, 3) + 0.797885 * y), 2) + 0.5;
+  }
+
+  /**
+   * Computes the first derivatives of the tanh function.
+   *
+   * @param y Input data.
+   * @param x The resulting derivatives.
+   */
+  template<typename InputVecType, typename OutputVecType>
+  static void Deriv(const InputVecType& y, OutputVecType& x)
+  {
+    x = 0.5 * arma::tanh(0.0356774 * arma::pow(y, 3) + 0.797885 * y) +
+        (0.0535161 * arma::pow(y, 3) + 0.398942 * y) *
+        arma::pow(1 / arma::cosh(0.0356774 * arma::pow(y, 3) + 0.797885 * y), 2) + 0.5;
+  }
+
+}; // class GELUFunction
+
+} // namespace ann
+} // namespace mlpack
+
+#endif

src/mlpack/methods/ann/layer/base_layer.hpp

@@ -22,6 +22,7 @@
 #include <mlpack/methods/ann/activation_functions/hard_sigmoid_function.hpp>
 #include <mlpack/methods/ann/activation_functions/swish_function.hpp>
 #include <mlpack/methods/ann/activation_functions/mish_function.hpp>
+#include <mlpack/methods/ann/activation_functions/gelu_function.hpp>
 
 namespace mlpack {
 namespace ann /** Artificial Neural Network. */ {
@@ -208,6 +209,17 @@ template <
 using MishFunctionLayer = BaseLayer<
     ActivationFunction, InputDataType, OutputDataType>;
 
+/**
+ * Standard Mish-Layer using the GELU activation function.
+ */
+template <
+    class ActivationFunction = GELUFunction,
+    typename InputDataType = arma::mat,
+    typename OutputDataType = arma::mat
+>
+using GELUFunctionLayer = BaseLayer<
+    ActivationFunction, InputDataType, OutputDataType>;
+
 } // namespace ann
 } // namespace mlpack
 

src/mlpack/tests/activation_functions_test.cpp

@@ -22,6 +22,7 @@
 #include <mlpack/methods/ann/activation_functions/swish_function.hpp>
 #include <mlpack/methods/ann/activation_functions/hard_sigmoid_function.hpp>
 #include <mlpack/methods/ann/activation_functions/mish_function.hpp>
+#include <mlpack/methods/ann/activation_functions/gelu_function.hpp>
 
 #include <boost/test/unit_test.hpp>
 #include "test_tools.hpp"