Merge pull request #548 from awhitesong/HardTanH

Added HardTanH Layer and its tests.
2016-03-08 21:02:30 +01:00
parent d5841b1f72 7de16454b3
commit 3e366ec501
2 changed files with 358 additions and 0 deletions
@@ -0,0 +1,282 @@
+/**
+ * @file hard_tanh_layer.hpp
+ * @author Dhawal Arora
+ *
+ * Implementation of hard_tanh activation function. The function is mentioned below.
+ */
+
+#ifndef __MLPACK_METHODS_ANN_LAYER_HARD_TANH_LAYER_HPP
+#define __MLPACK_METHODS_ANN_LAYER_HARD_TANH_LAYER_HPP
+
+#include <mlpack/core.hpp>
+
+namespace mlpack {
+namespace ann /** Artificial Neural Network. */ {
+
+/**
+ * The Hard Tanh activation function, defined by
+ *
+ * @f{eqnarray*}{
+ * f(x) &=& \left\{
+ *   \begin{array}{lr}
+ *     max & : x > maxValue \\
+ *     min & : x \le minValue \\
+ *     x   & : otherwise
+ *   \end{array}
+ * \right.
+ * f'(x) &=& \left\{
+ *   \begin{array}{lr}
+ *     0 & : x > maxValue \\ 
+ *     0 & : x \le minValue \\
+ *     1 & : otherwise
+ *   \end{array}
+ * \right.
+ * @f}
+ */
+
+template <
+    typename InputDataType = arma::mat,
+    typename OutputDataType = arma::mat
+>
+
+class HardTanHLayer
+{
+ public:
+
+  /**
+   * Constructor. Default maxValue is set to 1 and default minValue is set to -1.
+   *
+   */
+
+  HardTanHLayer(const double maxValue = 1.00, const double minValue = -1.00) : maxValue(maxValue), minValue(minValue)
+  {
+     // Nothing to do here.
+  }
+
+  /**
+   * Ordinary feed forward pass of a neural network, evaluating the function
+   * f(x) by propagating the activity forward through f.
+   * 
+   * @param x Input data used for evaluating the specified function. This is for just one input value. 
+   * @return f(x) The activation value for the input. 
+   */
+
+  double Forward(const double x)
+  {
+    return fn(x);
+  }
+
+  /**
+   * Ordinary feed forward pass of a neural network, evaluating the function
+   * f(x) by propagating the activity forward through f.
+   *
+   * @param input Input data used for evaluating the specified function.
+   * @param output Resulting output activation.
+   */
+  
+  template<typename InputType, typename OutputType>
+  void Forward(const InputType& input, OutputType& output)
+  {
+    fn(input, output);
+  }
+
+  /**
+   * Ordinary feed backward pass of a neural network, calculating the function
+   * f(x) by propagating x backwards through f. Using the results from the feed
+   * forward pass.
+   *
+   * @param input The propagated input activation. This function is for just a single input. 
+   * @param gy The backpropagated error.
+   * @return The calculated gradient.
+   */
+
+  double Backward(const double input,
+                  const double gy)
+  {
+    return gy * deriv(input);
+  }
+
+  /**
+   * Ordinary feed backward pass of a neural network, calculating the function
+   * f(x) by propagating x backwards through f. Using the results from the feed
+   * forward pass.
+   *
+   * @param input The propagated input activation.
+   * @param gy The backpropagated error.
+   * @param g The calculated gradient.
+   */
+  template<typename DataType>
+  void Backward(const DataType& input,
+                const DataType& gy,
+                DataType& g)
+  {
+    DataType derivative;
+    deriv(input, derivative);
+    g = gy % derivative;
+  }
+
+  /**
+   * Ordinary feed backward pass of a neural network, calculating the function
+   * f(x) by propagating x backwards through f. Using the results from the feed
+   * forward pass.
+   *
+   * @param input The propagated input activation.
+   * @param gy The backpropagated error.
+   * @param g The calculated gradient.
+   */
+  template<typename eT>
+  void Backward(const arma::Cube<eT>& input,
+                const arma::Mat<eT>& gy,
+                arma::Cube<eT>& g)
+  {
+    // Generate a cube using the backpropagated error matrix.
+    arma::Cube<eT> mappedError = arma::zeros<arma::cube>(input.n_rows,
+        input.n_cols, input.n_slices);
+
+    for (size_t s = 0, j = 0; s < mappedError.n_slices; s+= gy.n_cols, j++)
+    {
+      for (size_t i = 0; i < gy.n_cols; i++)
+      {
+        arma::Col<eT> temp = gy.col(i).subvec(
+            j * input.n_rows * input.n_cols,
+            (j + 1) * input.n_rows * input.n_cols - 1);
+
+        mappedError.slice(s + i) = arma::Mat<eT>(temp.memptr(),
+            input.n_rows, input.n_cols);
+      }
+    }
+
+    arma::Cube<eT> derivative;
+    deriv(input, derivative);
+    g = mappedError % derivative;
+  }
+
+  //! Get the input parameter.
+  InputDataType const& InputParameter() const { return inputParameter; }
+  //! Modify the input parameter.
+  InputDataType& InputParameter() { return inputParameter; }
+
+  //! Get the output parameter.
+  OutputDataType const& OutputParameter() const { return outputParameter; }
+  //! Modify the output parameter.
+  OutputDataType& OutputParameter() { return outputParameter; }
+
+  //! Get the delta.
+  OutputDataType const& Delta() const { return delta; }
+  //! Modify the delta.
+  OutputDataType& Delta() { return delta; }
+
+  //! Get the Maximum value.
+  double const& MaxValue() const { return maxValue; }
+  //! Modify the Maximum value.
+  double& MaxValue() { return maxValue; }
+
+  //! Get the Minimum value.
+  double const& MinValue() const { return minValue; }
+  //! Modify the Minimum value.
+  double& MinValue() { return minValue; }
+
+  
+  /**
+   * Serialize the layer.
+   */
+  template<typename Archive>
+  void Serialize(Archive& /* ar */, const unsigned int /* version */)
+  {
+    /* Nothing to do here */
+  }
+
+ private:
+
+
+  /**
+   * Computes the HardTanH function.
+   *
+   * @param x Input data.
+   * @return f(x).
+   */
+  double fn(const double x)
+  {
+    if (x > maxValue)
+      return maxValue;
+    else if (x < minValue)
+      return minValue;
+    return x;
+  }
+
+  /**
+   * Computes the HardTanH function using a dense matrix as input.
+   *
+   * @param x Input data.
+   * @param y The resulting output activation.
+   */
+   
+  template<typename eT>
+  void fn(const arma::Mat<eT>& x, arma::Mat<eT>& y)
+  {
+    y = x;
+    y = y.transform( [&](eT val) { return std::min( std::max( val, minValue ), maxValue ); } );
+  }
+
+  /**
+   * Computes the HardTanH function using a 3rd-order tensor as input.
+   *
+   * @param x Input data.
+   * @param y The resulting output activation.
+   */
+  template<typename eT>
+  void fn(const arma::Cube<eT>& x, arma::Cube<eT>& y)
+  {
+    y = x;
+    for (size_t s = 0; s < x.n_slices; s++)
+      fn(x.slice(s), y.slice(s));
+  }
+
+  /**
+   * Computes the first derivative of the HardTanH function.
+   *
+   * @param x Input data.
+   * @return f'(x)
+   */
+  double deriv(const double x)
+  {
+    return (x > maxValue || x < minValue) ? 0 : 1;
+  }
+
+  /**
+   * Computes the first derivative of the HardTanH function.
+   *
+   * @param y Input activations.
+   * @param x The resulting derivatives.
+   */
+  template<typename InputType, typename OutputType>
+  void deriv(const InputType& x, OutputType& y)
+  {
+    y = x;
+
+    for (size_t i = 0; i < x.n_elem; i++)
+      y(i) = deriv(x(i));
+  }
+
+  //! Locally-stored delta object.
+  OutputDataType delta;
+
+  //! Locally-stored input parameter object.
+  InputDataType inputParameter;
+
+  //! Locally-stored output parameter object.
+  OutputDataType outputParameter;
+
+  //! Maximum value for the HardTanH function.
+  double maxValue;
+
+  //! Minimum value for the HardTanH function.
+  double minValue;
+
+
+}; // class HardTanHLayer
+
+} // namespace ann
+} // namespace mlpack
+
+#endif
@@ -21,6 +21,7 @@
 #include <mlpack/methods/ann/layer/linear_layer.hpp>
 #include <mlpack/methods/ann/layer/base_layer.hpp>
 #include <mlpack/methods/ann/layer/binary_classification_layer.hpp>
+#include <mlpack/methods/ann/layer/hard_tanh_layer.hpp>

 #include <boost/test/unit_test.hpp>
 #include "old_boost_test_definitions.hpp"
@@ -120,6 +121,64 @@ void CheckInverseCorrect(const arma::colvec input)
  }
 }

+
+/*
+ * Implementation of the HardTanH activation function test. The function is implemented as a HardTanH Layer
+ * in file hard_tanh_layer.hpp
+ * @param input Input data used for evaluating the HardTanH activation function.
+ * @param target Target data used to evaluate the HardTanH activation.
+ *
+ */
+void CheckHardTanHActivationCorrect(const arma::colvec input, const arma::colvec target)
+{
+  HardTanHLayer<> htf;
+  // Test the activation function using a single value as input.
+  for (size_t i = 0; i < target.n_elem; i++)
+  {
+    BOOST_REQUIRE_CLOSE(htf.Forward(input.at(i)),
+        target.at(i), 1e-3);
+  }
+
+  // Test the activation function using the entire vector as input.
+  arma::colvec activations;
+  htf.Forward(input, activations);
+  for (size_t i = 0; i < activations.n_elem; i++)
+  {
+    BOOST_REQUIRE_CLOSE(activations.at(i), target.at(i), 1e-3);
+  }
+}
+
+/*
+ * Implementation of the HardTanH activation function derivative test. The derivative is implemented in HardTanH Layer
+ * in file hard_tanh_layer.hpp
+ * @param input Input data used for evaluating the HardTanH activation function.
+ * @param target Target data used to evaluate the HardTanH activation.
+ *
+ */
+
+void CheckHardTanHDerivativeCorrect(const arma::colvec input, const arma::colvec target)
+{
+  HardTanHLayer<> htf;
+  // Test the calculation of the derivatives using a single value as input.
+  for (size_t i = 0; i < target.n_elem; i++)
+  {
+    BOOST_REQUIRE_CLOSE(htf.Backward(input.at(i), 1),
+        target.at(i), 1e-3);
+  }
+
+  // Test the calculation of the derivatives using the entire vector as input.
+  arma::colvec derivatives;
+  // This error vector will be set to 1 to get the derivatives.
+  arma::colvec error(input.n_elem);
+  htf.Backward(input, (arma::colvec)error.ones(), derivatives);
+  for (size_t i = 0; i < derivatives.n_elem; i++)
+  {
+    BOOST_REQUIRE_CLOSE(derivatives.at(i), target.at(i), 1e-3);
+  }
+}
+
+
+
 /**
 * Basic test of the tanh function.
 */
@@ -199,4 +258,21 @@ BOOST_AUTO_TEST_CASE(RectifierFunctionTest)
      desiredDerivatives);
 }

+/**
+ * Basic test of the HardTanH function.
+ */
+BOOST_AUTO_TEST_CASE(HardTanHFunctionTest)
+{
+  const arma::colvec desiredActivations("-1 1 1 -1 \
+                                         1 -1 1 0");
+
+  const arma::colvec desiredDerivatives("0 0 0 0 \
+                                         1 1 0 1");
+
+  CheckHardTanHActivationCorrect(activationData, desiredActivations);
+  CheckHardTanHDerivativeCorrect(activationData, desiredDerivatives);
+}
+
+
+
 BOOST_AUTO_TEST_SUITE_END();