From 517a3a5e8d1ccfa75fa90e6e4e2239fa0494585b Mon Sep 17 00:00:00 2001
From: Anush V Kini <anushkini@gmail.com>
Date: Sun, 14 Feb 2021 12:54:35 +0530
Subject: [PATCH] Added Hard Swish Function Implementation and Test Skeleton

---
 .../ann/activation_functions/CMakeLists.txt   |   1 +
 .../hard_swish_function.hpp                   | 116 ++++++++++++++++++
 .../tests/activation_functions_test.cpp       |  20 +++
 3 files changed, 137 insertions(+)
 create mode 100644 src/mlpack/methods/ann/activation_functions/hard_swish_function.hpp

diff --git a/src/mlpack/methods/ann/activation_functions/CMakeLists.txt b/src/mlpack/methods/ann/activation_functions/CMakeLists.txt
index fd4e765006..d5c0868c1c 100644
--- a/src/mlpack/methods/ann/activation_functions/CMakeLists.txt
+++ b/src/mlpack/methods/ann/activation_functions/CMakeLists.txt
@@ -19,6 +19,7 @@ set(SOURCES
   multi_quadratic_function.hpp
   poisson1_function.hpp
   gaussian_function.hpp
+  hard_swish_function.hpp
 )
 
 # Add directory name to sources.
diff --git a/src/mlpack/methods/ann/activation_functions/hard_swish_function.hpp b/src/mlpack/methods/ann/activation_functions/hard_swish_function.hpp
new file mode 100644
index 0000000000..d3526da428
--- /dev/null
+++ b/src/mlpack/methods/ann/activation_functions/hard_swish_function.hpp
@@ -0,0 +1,116 @@
+/**
+ * @file methods/ann/activation_functions/hard_swish_function.hpp
+ * @author Anush Kini
+ *
+ * Definition and implementation of the Hard Swish function as described by
+ * Howard A, Sandler M, Chu G, Chen LC, Chen B, Tan M, Wang W, Zhu Y, Pang R,
+ * Vasudevan V and Le QV.
+ * For more information, see the following paper.
+ *
+ * @code
+ * @misc{
+ *   author = {Howard A, Sandler M, Chu G, Chen LC, Chen B, Tan M, Wang W,
+ *            Zhu Y, Pang R, Vasudevan V and Le QV},
+ *   title = {Searching for MobileNetV3},
+ *   year = {2019}
+ * }
+ * @endcode
+ *
+ * mlpack is free software; you may redistribute it and/or modify it under the
+ * terms of the 3-clause BSD license.  You should have received a copy of the
+ * 3-clause BSD license along with mlpack.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef MLPACK_METHODS_ANN_ACTIVATION_FUNCTIONS_HARD_SWISH_FUNCTION_HPP
+#define MLPACK_METHODS_ANN_ACTIVATION_FUNCTIONS_HARD_SWISH_FUNCTION_HPP
+
+#include <mlpack/prereqs.hpp>
+
+namespace mlpack {
+namespace ann /** Artificial Neural Network. */ {
+/**
+ * The Hard Swish function, defined by
+ *
+ * @f{eqnarray*}{
+ *   f(x) &=& \begin{cases}
+ *     0 & x \leq -3\\
+ *     x & x \geq +3\\
+ *     \frac{x * (x + 3)}{6} & otherwise\\
+ *   \end{cases} \\
+ *   f'(x) &=& \begin{cases}
+ *     0 & x \leq -3\\
+ *     1 & x \geq +3\\
+ *     \frac{2x + 3}{6} & otherwise\\
+ *   \end{cases}
+ * @f}
+ */
+class HardSwishFunction
+{
+ public:
+  /**
+   * Computes the Hard Swish function.
+   *
+   * @param x Input data.
+   * @return f(x).
+   */
+  static double Fn(const double x)
+  {
+    double x2 = x + 3.0;
+    x2 = x2 > 0.0 ? x2 : 0.0;
+    x2 = x2 < 6.0 ? x2 : 6.0;
+    x2 = x * x2 / 6.0;
+
+    return x2;
+  }
+
+  /**
+   * Computes the Hard Swish function.
+   *
+   * @param x Input data.
+   * @param y The resulting output activation.
+   */
+  template <typename InputVecType, typename OutputVecType>
+  static void Fn(const InputVecType &x, OutputVecType &y)
+  {
+    y.set_size(size(x));
+
+    for (size_t i = 0; i < x.n_elem; i++)
+      y(i) = Fn(x(i));
+  }
+
+  /**
+   * Computes the first derivative of the Hard Swish function.
+   *
+   * @param y Input data.
+   * @return f'(x).
+   */
+  static double Deriv(const double y)
+  {
+    if (y <= -3)
+      return 0;
+    else if (y >= 3)
+      return 1;
+
+    return (2*y + 3.0)/6.0;
+  }
+
+  /**
+   * Computes the first derivatives of the Hard Swish function.
+   *
+   * @param y Input activations.
+   * @param x The resulting derivatives.
+   */
+  template <typename InputVecType, typename OutputVecType>
+  static void Deriv(const InputVecType &y, OutputVecType &x)
+  {
+    x.set_size(size(y));
+
+    for (size_t i = 0; i < y.n_elem; i++)
+      x(i) = Deriv(y(i));
+  }
+}; // class HardSwishFunction
+
+} // namespace ann
+} // namespace mlpack
+
+#endif
diff --git a/src/mlpack/tests/activation_functions_test.cpp b/src/mlpack/tests/activation_functions_test.cpp
index d88c0a5109..f99f9781e8 100644
--- a/src/mlpack/tests/activation_functions_test.cpp
+++ b/src/mlpack/tests/activation_functions_test.cpp
@@ -1134,3 +1134,23 @@ TEST_CASE("SoftminFunctionTest", "[ActivationFunctionsTest]")
   CheckSoftminDerivativeCorrect(activationData,
                                 desiredDerivatives);
 }
+
+/**
+ * Basic test of the Hard Swish function.
+ */
+TEST_CASE("HardSwishFunctionTest", "[ActivationFunctionsTest]")
+{
+  // Randomly generated data.
+  const arma::colvec activationData("3.6544 -1.9714 -5.2277 1.5448 2.1164");
+
+  // Calculated from torch.nn.Hardswish.
+  const arma::colvec desiredActivations("3.6544 -0.3380 0 1.1701 1.8047");
+
+  // Hand Calculated Values.
+  const arma::colvec desiredDerivatives("1 ");
+
+  CheckSoftminActivationCorrect(activationData,
+                                desiredActivations);
+  CheckSoftminDerivativeCorrect(activationData,
+                                desiredDerivatives);
+}