From 701f4e2103a9d41a8bd140e81bd0ceb892643612 Mon Sep 17 00:00:00 2001
From: himanshupathak21061998 <hpathak336@gmail.com>
Date: Thu, 6 Feb 2020 22:37:31 +0530
Subject: [PATCH] Adding gelu activation function

---
 .../ann/activation_functions/CMakeLists.txt   |  1 +
 .../activation_functions/gelu_function.hpp    | 91 +++++++++++++++++++
 src/mlpack/methods/ann/layer/base_layer.hpp   | 12 +++
 .../tests/activation_functions_test.cpp       |  1 +
 4 files changed, 105 insertions(+)
 create mode 100644 src/mlpack/methods/ann/activation_functions/gelu_function.hpp

diff --git a/src/mlpack/methods/ann/activation_functions/CMakeLists.txt b/src/mlpack/methods/ann/activation_functions/CMakeLists.txt
index 50445dcc47..4cf9e40295 100644
--- a/src/mlpack/methods/ann/activation_functions/CMakeLists.txt
+++ b/src/mlpack/methods/ann/activation_functions/CMakeLists.txt
@@ -9,6 +9,7 @@ set(SOURCES
   softplus_function.hpp
   swish_function.hpp
   mish_function.hpp
+  gelu_function.hpp
 )
 
 # Add directory name to sources.
diff --git a/src/mlpack/methods/ann/activation_functions/gelu_function.hpp b/src/mlpack/methods/ann/activation_functions/gelu_function.hpp
new file mode 100644
index 0000000000..d6c23f9ae4
--- /dev/null
+++ b/src/mlpack/methods/ann/activation_functions/gelu_function.hpp
@@ -0,0 +1,91 @@
+
+/**
+ * @file gelu_function.hpp
+ * @author Himanshu Pathak
+ *
+ * Definition and implementation of the Gaussian Error Linear Unit (GELU)
+ * function.
+ *
+ * mlpack is free software; you may redistribute it and/or modify it under the
+ * terms of the 3-clause BSD license.  You should have received a copy of the
+ * 3-clause BSD license along with mlpack.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef MLPACK_METHODS_ANN_ACTIVATION_FUNCTIONS_GELU_FUNCTION_HPP
+#define MLPACK_METHODS_ANN_ACTIVATION_FUNCTIONS_GELU_FUNCTION_HPP
+# define PI          3.141592653589793238462643383279502884L /* pi */
+
+#include <mlpack/prereqs.hpp>
+
+namespace mlpack {
+namespace ann /** Artificial Neural Network. */ {
+
+/**
+ * The GELU function, defined by
+ *
+ * @f{eqnarray*}{
+ * f(x) = 0.5 * x * {1 + tanh[(2/pi)^(1/2) * (x + 0.044715 * x^3)]} \\
+ * f'(x) = 0.5 * tanh(0.0356774 * x^3) + 0.797885 * x) + 
+ *         (0.0535161x^3 + 0.398942 * x) * 
+ *         sech^2(0.0356774 * x^3+0.797885 * x) + 0.5\\
+ * @f}
+ */
+class GELUFunction
+{
+ public:
+  /**
+   * Computes the GELU function.
+   *
+   * @param x Input data.
+   * @return f(x).
+   */
+  static double Fn(const double x)
+  {
+    return 0.5 * x * (1 + std::tanh(std::sqrt(2/PI)*(x + 0.044715 * std::pow(x, 3))));
+  }
+
+  /**
+   * Computes the tanh function.
+   *
+   * @param x Input data.
+   * @param y The resulting output activation.
+   */
+  template<typename InputVecType, typename OutputVecType>
+  static void Fn(const InputVecType& x, OutputVecType& y)
+  {
+    y = 0.5 * x * (1 + arma::tanh(std::sqrt(2/PI)*(x + 0.044715 * arma::pow(x, 3))));
+  }
+
+  /**
+   * Computes the first derivative of the tanh function.
+   *
+   * @param y Input data.
+   * @return f'(x)
+   */
+  static double Deriv(const double y)
+  {
+    return 0.5 * std::tanh(0.0356774 * std::pow(y, 3) + 0.797885 * y) +
+           (0.0535161 * std::pow(y, 3) + 0.398942 * y) *
+           std::pow(1 / std::cosh(0.0356774 * std::pow(y, 3) + 0.797885 * y), 2) + 0.5;
+  }
+
+  /**
+   * Computes the first derivatives of the tanh function.
+   *
+   * @param y Input data.
+   * @param x The resulting derivatives.
+   */
+  template<typename InputVecType, typename OutputVecType>
+  static void Deriv(const InputVecType& y, OutputVecType& x)
+  {
+    x = 0.5 * arma::tanh(0.0356774 * arma::pow(y, 3) + 0.797885 * y) +
+        (0.0535161 * arma::pow(y, 3) + 0.398942 * y) *
+        arma::pow(1 / arma::cosh(0.0356774 * arma::pow(y, 3) + 0.797885 * y), 2) + 0.5;
+  }
+
+}; // class GELUFunction
+
+} // namespace ann
+} // namespace mlpack
+
+#endif
\ No newline at end of file
diff --git a/src/mlpack/methods/ann/layer/base_layer.hpp b/src/mlpack/methods/ann/layer/base_layer.hpp
index 90ce63a281..d0b53c0a2f 100644
--- a/src/mlpack/methods/ann/layer/base_layer.hpp
+++ b/src/mlpack/methods/ann/layer/base_layer.hpp
@@ -22,6 +22,7 @@
 #include <mlpack/methods/ann/activation_functions/hard_sigmoid_function.hpp>
 #include <mlpack/methods/ann/activation_functions/swish_function.hpp>
 #include <mlpack/methods/ann/activation_functions/mish_function.hpp>
+#include <mlpack/methods/ann/activation_functions/gelu_function.hpp>
 
 namespace mlpack {
 namespace ann /** Artificial Neural Network. */ {
@@ -208,6 +209,17 @@ template <
 using MishFunctionLayer = BaseLayer<
     ActivationFunction, InputDataType, OutputDataType>;
 
+/**
+ * Standard Mish-Layer using the GELU activation function.
+ */
+template <
+    class ActivationFunction = GELUFunction,
+    typename InputDataType = arma::mat,
+    typename OutputDataType = arma::mat
+>
+using GELUFunctionLayer = BaseLayer<
+    ActivationFunction, InputDataType, OutputDataType>;
+
 } // namespace ann
 } // namespace mlpack
 
diff --git a/src/mlpack/tests/activation_functions_test.cpp b/src/mlpack/tests/activation_functions_test.cpp
index 54e9448254..702d179d73 100644
--- a/src/mlpack/tests/activation_functions_test.cpp
+++ b/src/mlpack/tests/activation_functions_test.cpp
@@ -22,6 +22,7 @@
 #include <mlpack/methods/ann/activation_functions/swish_function.hpp>
 #include <mlpack/methods/ann/activation_functions/hard_sigmoid_function.hpp>
 #include <mlpack/methods/ann/activation_functions/mish_function.hpp>
+#include <mlpack/methods/ann/activation_functions/gelu_function.hpp>
 
 #include <boost/test/unit_test.hpp>
 #include "test_tools.hpp"