Merge pull request #1495 from akhandait/concatenate_layer

[WIP] Add Concatenate layer.

src/mlpack/methods/ann/layer/CMakeLists.txt

@@ -18,6 +18,8 @@ set(SOURCES
   concat_impl.hpp
   concat_performance.hpp
   concat_performance_impl.hpp
+  concatenate.hpp
+  concatenate_impl.hpp
   constant.hpp
   constant_impl.hpp
   convolution.hpp

src/mlpack/methods/ann/layer/concatenate.hpp

@@ -0,0 +1,120 @@
+/**
+ * @file concatenate.hpp
+ * @author Atharva Khandait
+ *
+ * Definition of the Concatenate class that concatenate a constant matrix to
+ * the incoming data.
+ *
+ * mlpack is free software; you may redistribute it and/or modify it under the
+ * terms of the 3-clause BSD license.  You should have received a copy of the
+ * 3-clause BSD license along with mlpack.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef MLPACK_METHODS_ANN_LAYER_CONCATENATE_HPP
+#define MLPACK_METHODS_ANN_LAYER_CONCATENATE_HPP
+
+#include <mlpack/prereqs.hpp>
+#include <mlpack/methods/ann/layer/layer_traits.hpp>
+
+namespace mlpack {
+namespace ann /** Artificial Neural Network. */ {
+
+/**
+ * Implementation of the Concatenate module class. The Concatenate module
+ * concatenates a constant given matrix to the incoming data.
+ * Note: Users need to use the Concat() function to provide the concat matrix.
+ *
+ * @tparam InputDataType Type of the input data (arma::colvec, arma::mat,
+ *         arma::sp_mat or arma::cube).
+ * @tparam OutputDataType Type of the output data (arma::colvec, arma::mat,
+ *         arma::sp_mat or arma::cube).
+ */
+template <
+    typename InputDataType = arma::mat,
+    typename OutputDataType = arma::mat
+>
+class Concatenate
+{
+ public:
+  /**
+   * Create the Concatenate object using the specified number of output units.
+   */
+  Concatenate();
+
+  /**
+   * Ordinary feed forward pass of a neural network, evaluating the function
+   * f(x) by propagating the activity forward through f.
+   *
+   * @param input Input data used for evaluating the specified function.
+   * @param output Resulting output activation.
+   */
+  template<typename eT>
+  void Forward(const arma::Mat<eT>&& input, arma::Mat<eT>&& output);
+
+  /**
+   * Ordinary feed backward pass of a neural network, calculating the function
+   * f(x) by propagating x backwards trough f. Using the results from the feed
+   * forward pass.
+   *
+   * @param input The propagated input activation.
+   * @param gy The backpropagated error.
+   * @param g The calculated gradient.
+   */
+  template<typename eT>
+  void Backward(const arma::Mat<eT>&& /* input */,
+                const arma::Mat<eT>&& gy,
+                arma::Mat<eT>&& g);
+
+  //! Get the parameters.
+  OutputDataType const& Parameters() const { return weights; }
+  //! Modify the parameters.
+  OutputDataType& Parameters() { return weights; }
+
+  //! Get the output parameter.
+  OutputDataType const& OutputParameter() const { return outputParameter; }
+  //! Modify the output parameter.
+  OutputDataType& OutputParameter() { return outputParameter; }
+
+  //! Get the delta.
+  OutputDataType const& Delta() const { return delta; }
+  //! Modify the delta.
+  OutputDataType& Delta() { return delta; }
+
+  //! Get the concat matrix.
+  OutputDataType const& Concat() const { return concat; }
+  //! Modify the delta.
+  OutputDataType& Concat() { return concat; }
+
+  /**
+   * Serialize the layer
+   */
+  template<typename Archive>
+  void serialize(Archive& /* ar */, const unsigned int /* version */)
+  {
+    // Nothing to do here.
+  }
+
+ private:
+  //! Locally-stored number of input rows.
+  size_t inRows;
+
+  //! Locally-stored weight object.
+  OutputDataType weights;
+
+  //! Locally-stored delta object.
+  OutputDataType delta;
+
+  //! Locally-stored output parameter object.
+  OutputDataType outputParameter;
+
+  //! Locally-stored matrix to be concatenated to input.
+  OutputDataType concat;
+}; // class Concatenate
+
+} // namespace ann
+} // namespace mlpack
+
+// Include implementation.
+#include "concatenate_impl.hpp"
+
+#endif

src/mlpack/methods/ann/layer/concatenate_impl.hpp

@@ -0,0 +1,59 @@
+/**
+ * @file concatenate_impl.hpp
+ * @author Atharva Khandait
+ *
+ * Implementation of the Concatenate class that concatenates a constant matrix to
+ * the incoming data.
+ *
+ * mlpack is free software; you may redistribute it and/or modify it under the
+ * terms of the 3-clause BSD license.  You should have received a copy of the
+ * 3-clause BSD license along with mlpack.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef MLPACK_METHODS_ANN_LAYER_CONCATENATE_IMPL_HPP
+#define MLPACK_METHODS_ANN_LAYER_CONCATENATE_IMPL_HPP
+
+// In case it hasn't yet been included.
+#include "concatenate.hpp"
+
+namespace mlpack {
+namespace ann /** Artificial Neural Network. */ {
+
+template<typename InputDataType, typename OutputDataType>
+Concatenate<InputDataType, OutputDataType>::Concatenate()
+{
+  // Nothing to do here.
+}
+
+template<typename InputDataType, typename OutputDataType>
+template<typename eT>
+void Concatenate<InputDataType, OutputDataType>::Forward(
+    const arma::Mat<eT>&& input, arma::Mat<eT>&& output)
+{
+  if (concat.is_empty())
+    Log::Warn << "The concat matrix has not been provided." << std::endl;
+
+  if (input.n_cols != concat.n_cols)
+  {
+    Log::Fatal << "The number of columns of the concat matrix should be equal "
+        << "to the number of columns of input matrix." << std::endl;
+  }
+
+  inRows = input.n_rows;
+  output = arma::join_cols(input, concat);
+}
+
+template<typename InputDataType, typename OutputDataType>
+template<typename eT>
+void Concatenate<InputDataType, OutputDataType>::Backward(
+    const arma::Mat<eT>&& /* input */,
+    const arma::Mat<eT>&& gy,
+    arma::Mat<eT>&& g)
+{
+  g = gy.submat(0, 0, inRows - 1, concat.n_cols - 1);
+}
+
+} // namespace ann
+} // namespace mlpack
+
+#endif

src/mlpack/methods/ann/layer/layer_types.hpp

@@ -21,6 +21,7 @@
 #include <mlpack/methods/ann/layer/batch_norm.hpp>
 #include <mlpack/methods/ann/layer/bilinear_interpolation.hpp>
 #include <mlpack/methods/ann/layer/constant.hpp>
+#include <mlpack/methods/ann/layer/concatenate.hpp>
 #include <mlpack/methods/ann/layer/dropout.hpp>
 #include <mlpack/methods/ann/layer/elu.hpp>
 #include <mlpack/methods/ann/layer/hard_tanh.hpp>
@@ -60,6 +61,7 @@ template<typename InputDataType, typename OutputDataType> class LSTM;
 template<typename InputDataType, typename OutputDataType> class GRU;
 template<typename InputDataType, typename OutputDataType> class FastLSTM;
 template<typename InputDataType, typename OutputDataType> class VRClassReward;
+template<typename InputDataType, typename OutputDataType> class Concatenate;
 
 template<typename InputDataType,
          typename OutputDataType
@@ -152,6 +154,7 @@ using LayerTypes = boost::variant<
     BatchNorm<arma::mat, arma::mat>*,
     BilinearInterpolation<arma::mat, arma::mat>*,
     Concat<arma::mat, arma::mat>*,
+    Concatenate<arma::mat, arma::mat>*,
     ConcatPerformance<NegativeLogLikelihood<arma::mat, arma::mat>,
                       arma::mat, arma::mat>*,
     Constant<arma::mat, arma::mat>*,

src/mlpack/methods/ann/layer/reparametrization_impl.hpp

@@ -81,7 +81,7 @@ void Reparametrization<InputDataType, OutputDataType>::Backward(
   if (includeKl)
   {
     g = join_cols(gy % std::move(gaussianSample) % g + (-1 / stdDev + stdDev)
-        % g * beta, gy + mean * beta);
+        % g * beta, gy + mean * beta / mean.n_cols);
   }
   else
     g = join_cols(gy % std::move(gaussianSample) % g, gy);

src/mlpack/methods/ann/loss_functions/mean_squared_error_impl.hpp

@@ -39,7 +39,7 @@ void MeanSquaredError<InputDataType, OutputDataType>::Backward(
     const TargetType&& target,
     OutputType&& output)
 {
-  output = (input - target);
+  output = 2 * (input - target) / target.n_cols;
 }
 
 template<typename InputDataType, typename OutputDataType>

src/mlpack/tests/ann_layer_test.cpp

@@ -1083,6 +1083,77 @@ BOOST_AUTO_TEST_CASE(GradientConcatLayerTest)
   BOOST_REQUIRE_LE(CheckGradient(function), 1e-4);
 }
 
+/**
+ * Simple concatenate module test.
+ */
+BOOST_AUTO_TEST_CASE(SimpleConcatenateLayerTest)
+{
+  arma::mat input = arma::ones(5, 1);
+  arma::mat output, delta;
+
+  Concatenate<> module;
+  module.Concat() = arma::ones(5, 1) * 0.5;
+
+  // Test the Forward function.
+  module.Forward(std::move(input), std::move(output));
+
+  BOOST_REQUIRE_EQUAL(arma::accu(output), 7.5);
+
+  // Test the Backward function.
+  module.Backward(std::move(input), std::move(output), std::move(delta));
+  BOOST_REQUIRE_EQUAL(arma::accu(delta), 5);
+}
+
+/**
+ * Concatenate layer numerical gradient test.
+ */
+BOOST_AUTO_TEST_CASE(GradientConcatenateLayerTest)
+{
+  // Concatenate function gradient instantiation.
+  struct GradientFunction
+  {
+    GradientFunction()
+    {
+      input = arma::randu(10, 1);
+      target = arma::mat("1");
+
+      model = new FFN<NegativeLogLikelihood<>, NguyenWidrowInitialization>();
+      model->Predictors() = input;
+      model->Responses() = target;
+      model->Add<IdentityLayer<> >();
+      model->Add<Linear<> >(10, 5);
+
+      arma::mat concat = arma::ones(5, 1);
+      concatenate = new Concatenate<>();
+      concatenate->Concat() = concat;
+      model->Add(concatenate);
+
+      model->Add<Linear<> >(10, 5);
+      model->Add<LogSoftMax<> >();
+    }
+
+    ~GradientFunction()
+    {
+      delete model;
+    }
+
+    double Gradient(arma::mat& gradient) const
+    {
+      double error = model->Evaluate(model->Parameters(), 0, 1);
+      model->Gradient(model->Parameters(), 0, gradient, 1);
+      return error;
+    }
+
+    arma::mat& Parameters() { return model->Parameters(); }
+
+    FFN<NegativeLogLikelihood<>, NguyenWidrowInitialization>* model;
+    Concatenate<>* concatenate;
+    arma::mat input, target;
+  } function;
+
+  BOOST_REQUIRE_LE(CheckGradient(function), 1e-4);
+}
+
 /**
  * Simple lookup module test.
  */
@@ -1922,8 +1993,8 @@ BOOST_AUTO_TEST_CASE(GradientReparametrizationLayerBetaTest)
   {
     GradientFunction()
     {
-      input = arma::randu(10, 1);
-      target = arma::mat("1");
+      input = arma::randu(10, 2);
+      target = arma::mat("1 1");
 
       model = new FFN<NegativeLogLikelihood<>, NguyenWidrowInitialization>();
       model->Predictors() = input;

src/mlpack/tests/loss_functions_test.cpp

@@ -107,8 +107,10 @@ BOOST_AUTO_TEST_CASE(SimpleMeanSquaredErrorTest)
 
   // Test the Backward function.
   module.Backward(std::move(input), std::move(target), std::move(output));
-  // We subtract a zero vector, so the output should be equal with the input.
-  CheckMatrices(input, output);
+  // We subtract a zero vector, so according to the used backward formula:
+  // output = 2 * (input - target) / target.n_cols,
+  // output * nofColumns / 2 should be equal to input.
+  CheckMatrices(input, output * output.n_cols / 2);
   BOOST_REQUIRE_EQUAL(output.n_rows, input.n_rows);
   BOOST_REQUIRE_EQUAL(output.n_cols, input.n_cols);
 
@@ -121,7 +123,7 @@ BOOST_AUTO_TEST_CASE(SimpleMeanSquaredErrorTest)
   // Test the Backward function on a single input.
   module.Backward(std::move(input), std::move(target), std::move(output));
   // Test whether the output is negative.
-  BOOST_REQUIRE_EQUAL(arma::accu(output), -1);
+  BOOST_REQUIRE_EQUAL(arma::accu(output), -2);
   BOOST_REQUIRE_EQUAL(output.n_elem, 1);
 }
 
@@ -266,6 +268,49 @@ BOOST_AUTO_TEST_CASE(SimpleEarthMoverDistanceLayerTest)
   BOOST_REQUIRE_EQUAL(output.n_cols, input2.n_cols);
 }
 
+/*
+ * Mean Squared Error numerical gradient test.
+ */
+BOOST_AUTO_TEST_CASE(GradientMeanSquaredErrorTest)
+{
+  // Linear function gradient instantiation.
+  struct GradientFunction
+  {
+    GradientFunction()
+    {
+      input = arma::randu(10, 1);
+      target = arma::randu(2, 1);
+
+      model = new FFN<MeanSquaredError<>, NguyenWidrowInitialization>();
+      model->Predictors() = input;
+      model->Responses() = target;
+      model->Add<IdentityLayer<> >();
+      model->Add<Linear<> >(10, 2);
+      model->Add<SigmoidLayer<> >();
+    }
+
+    ~GradientFunction()
+    {
+      delete model;
+    }
+
+    double Gradient(arma::mat& gradient) const
+    {
+      arma::mat output;
+      double error = model->Evaluate(model->Parameters(), 0, 1);
+      model->Gradient(model->Parameters(), 0, gradient, 1);
+      return error;
+    }
+
+    arma::mat& Parameters() { return model->Parameters(); }
+
+    FFN<MeanSquaredError<>, NguyenWidrowInitialization>* model;
+    arma::mat input, target;
+  } function;
+
+  BOOST_REQUIRE_LE(CheckGradient(function), 1e-4);
+}
+
 /*
  * Reconstruction Loss numerical gradient test.
  */