Added Hard Swish Function Implementation and Test Skeleton

src/mlpack/methods/ann/activation_functions/CMakeLists.txt

@@ -19,6 +19,7 @@ set(SOURCES
   multi_quadratic_function.hpp
   poisson1_function.hpp
   gaussian_function.hpp
+  hard_swish_function.hpp
 )
 
 # Add directory name to sources.

src/mlpack/methods/ann/activation_functions/hard_swish_function.hpp

@@ -0,0 +1,116 @@
+/**
+ * @file methods/ann/activation_functions/hard_swish_function.hpp
+ * @author Anush Kini
+ *
+ * Definition and implementation of the Hard Swish function as described by
+ * Howard A, Sandler M, Chu G, Chen LC, Chen B, Tan M, Wang W, Zhu Y, Pang R,
+ * Vasudevan V and Le QV.
+ * For more information, see the following paper.
+ *
+ * @code
+ * @misc{
+ *   author = {Howard A, Sandler M, Chu G, Chen LC, Chen B, Tan M, Wang W,
+ *            Zhu Y, Pang R, Vasudevan V and Le QV},
+ *   title = {Searching for MobileNetV3},
+ *   year = {2019}
+ * }
+ * @endcode
+ *
+ * mlpack is free software; you may redistribute it and/or modify it under the
+ * terms of the 3-clause BSD license.  You should have received a copy of the
+ * 3-clause BSD license along with mlpack.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef MLPACK_METHODS_ANN_ACTIVATION_FUNCTIONS_HARD_SWISH_FUNCTION_HPP
+#define MLPACK_METHODS_ANN_ACTIVATION_FUNCTIONS_HARD_SWISH_FUNCTION_HPP
+
+#include <mlpack/prereqs.hpp>
+
+namespace mlpack {
+namespace ann /** Artificial Neural Network. */ {
+/**
+ * The Hard Swish function, defined by
+ *
+ * @f{eqnarray*}{
+ *   f(x) &=& \begin{cases}
+ *     0 & x \leq -3\\
+ *     x & x \geq +3\\
+ *     \frac{x * (x + 3)}{6} & otherwise\\
+ *   \end{cases} \\
+ *   f'(x) &=& \begin{cases}
+ *     0 & x \leq -3\\
+ *     1 & x \geq +3\\
+ *     \frac{2x + 3}{6} & otherwise\\
+ *   \end{cases}
+ * @f}
+ */
+class HardSwishFunction
+{
+ public:
+  /**
+   * Computes the Hard Swish function.
+   *
+   * @param x Input data.
+   * @return f(x).
+   */
+  static double Fn(const double x)
+  {
+    double x2 = x + 3.0;
+    x2 = x2 > 0.0 ? x2 : 0.0;
+    x2 = x2 < 6.0 ? x2 : 6.0;
+    x2 = x * x2 / 6.0;
+
+    return x2;
+  }
+
+  /**
+   * Computes the Hard Swish function.
+   *
+   * @param x Input data.
+   * @param y The resulting output activation.
+   */
+  template <typename InputVecType, typename OutputVecType>
+  static void Fn(const InputVecType &x, OutputVecType &y)
+  {
+    y.set_size(size(x));
+
+    for (size_t i = 0; i < x.n_elem; i++)
+      y(i) = Fn(x(i));
+  }
+
+  /**
+   * Computes the first derivative of the Hard Swish function.
+   *
+   * @param y Input data.
+   * @return f'(x).
+   */
+  static double Deriv(const double y)
+  {
+    if (y <= -3)
+      return 0;
+    else if (y >= 3)
+      return 1;
+
+    return (2*y + 3.0)/6.0;
+  }
+
+  /**
+   * Computes the first derivatives of the Hard Swish function.
+   *
+   * @param y Input activations.
+   * @param x The resulting derivatives.
+   */
+  template <typename InputVecType, typename OutputVecType>
+  static void Deriv(const InputVecType &y, OutputVecType &x)
+  {
+    x.set_size(size(y));
+
+    for (size_t i = 0; i < y.n_elem; i++)
+      x(i) = Deriv(y(i));
+  }
+}; // class HardSwishFunction
+
+} // namespace ann
+} // namespace mlpack
+
+#endif

src/mlpack/tests/activation_functions_test.cpp

@@ -1134,3 +1134,23 @@ TEST_CASE("SoftminFunctionTest", "[ActivationFunctionsTest]")
   CheckSoftminDerivativeCorrect(activationData,
                                 desiredDerivatives);
 }
+
+/**
+ * Basic test of the Hard Swish function.
+ */
+TEST_CASE("HardSwishFunctionTest", "[ActivationFunctionsTest]")
+{
+  // Randomly generated data.
+  const arma::colvec activationData("3.6544 -1.9714 -5.2277 1.5448 2.1164");
+
+  // Calculated from torch.nn.Hardswish.
+  const arma::colvec desiredActivations("3.6544 -0.3380 0 1.1701 1.8047");
+
+  // Hand Calculated Values.
+  const arma::colvec desiredDerivatives("1 ");
+
+  CheckSoftminActivationCorrect(activationData,
+                                desiredActivations);
+  CheckSoftminDerivativeCorrect(activationData,
+                                desiredDerivatives);
+}