Merge pull request #1427 from zoq/merge_layer_run

Merge layer run parameter.

src/mlpack/methods/ann/layer/add_merge.hpp

@@ -46,8 +46,9 @@ class AddMerge
    * Create the AddMerge object using the specified parameters.
    *
    * @param model Expose all the network modules.
+   * @param run Call the Forward/Backward method before the output is merged.
    */
-  AddMerge(const bool model = false);
+  AddMerge(const bool model = false, const bool run = true);
 
   //! Destructor to release allocated memory.
   ~AddMerge();
@@ -60,7 +61,7 @@ class AddMerge
    * @param output Resulting output activation.
    */
   template<typename InputType, typename OutputType>
-  void Forward(const InputType&& /* input */, OutputType&& output);
+  void Forward(InputType&& /* input */, OutputType&& output);
 
   /**
    * Ordinary feed backward pass of a neural network, calculating the function
@@ -77,19 +78,16 @@ class AddMerge
                 arma::Mat<eT>&& g);
 
   /*
-   * Add a new module to the model.
+   * Calculate the gradient using the output delta and the input activation.
    *
-   * @param layer The Layer to be added to the model.
+   * @param input The input parameter used for calculating the gradient.
+   * @param error The calculated error.
+   * @param gradient The calculated gradient.
    */
-  void Add(LayerTypes<CustomLayers...> layer) { network.push_back(layer); }
-
-  /*
-   * Add a new module to the model.
-   *
-   * @param layer The Layer to be added to the model.
-   */
-  template<typename LayerType>
-  void Add(const LayerType& layer) { network.push_back(new LayerType(layer)); }
+  template<typename eT>
+  void Gradient(arma::Mat<eT>&& input,
+                arma::Mat<eT>&& error,
+                arma::Mat<eT>&& gradient);
 
   /*
    * Add a new module to the model.
@@ -99,6 +97,13 @@ class AddMerge
   template <class LayerType, class... Args>
   void Add(Args... args) { network.push_back(new LayerType(args...)); }
 
+  /*
+   * Add a new module to the model.
+   *
+   * @param layer The Layer to be added to the model.
+   */
+  void Add(LayerTypes<CustomLayers...> layer) { network.push_back(layer); }
+
   //! Get the input parameter.
   InputDataType const& InputParameter() const { return inputParameter; }
   //! Modify the input parameter.
@@ -125,6 +130,11 @@ class AddMerge
     return empty;
   }
 
+  //! Get the parameters.
+  OutputDataType const& Parameters() const { return weights; }
+  //! Modify the parameters.
+  OutputDataType& Parameters() { return weights; }
+
   /**
    * Serialize the layer.
    */
@@ -135,6 +145,10 @@ class AddMerge
   //! Parameter which indicates if the modules should be exposed.
   bool model;
 
+  //! Parameter which indicates if the Forward/Backward method should be called
+  //! before merging the output.
+  bool run;
+
   //! We need this to know whether we should delete the layer in the destructor.
   bool ownsLayer;
 
@@ -156,11 +170,17 @@ class AddMerge
   //! Locally-stored delta object.
   OutputDataType delta;
 
+  //! Locally-stored gradient object.
+  OutputDataType gradient;
+
   //! Locally-stored input parameter object.
   InputDataType inputParameter;
 
   //! Locally-stored output parameter object.
   OutputDataType outputParameter;
+
+  //! Locally-stored weight object.
+  OutputDataType weights;
 }; // class AddMerge
 
 } // namespace ann

src/mlpack/methods/ann/layer/add_merge_impl.hpp

@@ -16,13 +16,18 @@
 // In case it hasn't yet been included.
 #include "add_merge.hpp"
 
+#include "../visitor/forward_visitor.hpp"
+#include "../visitor/backward_visitor.hpp"
+#include "../visitor/gradient_visitor.hpp"
+
 namespace mlpack {
 namespace ann /** Artificial Neural Network. */ {
 
 template<typename InputDataType, typename OutputDataType,
          typename... CustomLayers>
 AddMerge<InputDataType, OutputDataType, CustomLayers...>::AddMerge(
-    const bool model) : model(model), ownsLayer(!model)
+    const bool model, const bool run) :
+    model(model), run(run), ownsLayer(!model)
 {
   // Nothing to do here.
 }
@@ -42,10 +47,19 @@ template <typename InputDataType, typename OutputDataType,
           typename... CustomLayers>
 template<typename InputType, typename OutputType>
 void AddMerge<InputDataType, OutputDataType, CustomLayers...>::Forward(
-    const InputType&& /* input */, OutputType&& output)
+    InputType&& input, OutputType&& output)
 {
-  output = boost::apply_visitor(outputParameterVisitor, network.front());
+  if (run)
+  {
+    for (size_t i = 0; i < network.size(); ++i)
+    {
+      boost::apply_visitor(ForwardVisitor(std::move(input), std::move(
+          boost::apply_visitor(outputParameterVisitor, network[i]))),
+          network[i]);
+    }
+  }
 
+  output = boost::apply_visitor(outputParameterVisitor, network.front());
   for (size_t i = 1; i < network.size(); ++i)
   {
     output += boost::apply_visitor(outputParameterVisitor, network[i]);
@@ -58,7 +72,41 @@ template<typename eT>
 void AddMerge<InputDataType, OutputDataType, CustomLayers...>::Backward(
     const arma::Mat<eT>&& /* input */, arma::Mat<eT>&& gy, arma::Mat<eT>&& g)
 {
-  g = gy;
+  if (run)
+  {
+    for (size_t i = 0; i < network.size(); ++i)
+    {
+      boost::apply_visitor(BackwardVisitor(std::move(boost::apply_visitor(
+          outputParameterVisitor, network[i])), std::move(gy), std::move(
+          boost::apply_visitor(deltaVisitor, network[i]))), network[i]);
+    }
+
+    g = boost::apply_visitor(deltaVisitor, network[0]);
+    for (size_t i = 1; i < network.size(); ++i)
+    {
+      g += boost::apply_visitor(deltaVisitor, network[i]);
+    }
+  }
+  else
+    g = gy;
+}
+
+template<typename InputDataType, typename OutputDataType,
+         typename... CustomLayers>
+template<typename eT>
+void AddMerge<InputDataType, OutputDataType, CustomLayers...>::Gradient(
+    arma::Mat<eT>&& input,
+    arma::Mat<eT>&& error,
+    arma::Mat<eT>&& /* gradient */ )
+{
+  if (run)
+  {
+    for (size_t i = 0; i < network.size(); ++i)
+    {
+      boost::apply_visitor(GradientVisitor(std::move(input), std::move(error)),
+          network[i]);
+    }
+  }
 }
 
 template<typename InputDataType, typename OutputDataType,

src/mlpack/methods/ann/layer/multiply_merge.hpp

@@ -46,8 +46,9 @@ class MultiplyMerge
    * Create the MultiplyMerge object using the specified parameters.
    *
    * @param model Expose all the network modules.
+   * @param run Call the Forward/Backward method before the output is merged.
    */
-  MultiplyMerge(const bool model = false);
+  MultiplyMerge(const bool model = false, const bool run = true);
 
   //! Destructor to release allocated memory.
   ~MultiplyMerge();
@@ -60,7 +61,7 @@ class MultiplyMerge
    * @param output Resulting output activation.
    */
   template<typename InputType, typename OutputType>
-  void Forward(const InputType&& /* input */, OutputType&& output);
+  void Forward(InputType&& /* input */, OutputType&& output);
 
   /**
    * Ordinary feed backward pass of a neural network, calculating the function
@@ -77,19 +78,16 @@ class MultiplyMerge
                 arma::Mat<eT>&& g);
 
   /*
-   * Add a new module to the model.
+   * Calculate the gradient using the output delta and the input activation.
    *
-   * @param layer The Layer to be added to the model.
+   * @param input The input parameter used for calculating the gradient.
+   * @param error The calculated error.
+   * @param gradient The calculated gradient.
    */
-  void Add(LayerTypes<CustomLayers...> layer) { network.push_back(layer); }
-
-  /*
-   * Add a new module to the model.
-   *
-   * @param layer The Layer to be added to the model.
-   */
-  template<typename LayerType>
-  void Add(const LayerType& layer) { network.push_back(new LayerType(layer)); }
+  template<typename eT>
+  void Gradient(arma::Mat<eT>&& input,
+                arma::Mat<eT>&& error,
+                arma::Mat<eT>&& gradient);
 
   /*
    * Add a new module to the model.
@@ -99,6 +97,13 @@ class MultiplyMerge
   template <class LayerType, class... Args>
   void Add(Args... args) { network.push_back(new LayerType(args...)); }
 
+  /*
+   * Add a new module to the model.
+   *
+   * @param layer The Layer to be added to the model.
+   */
+  void Add(LayerTypes<CustomLayers...> layer) { network.push_back(layer); }
+
   //! Get the input parameter.
   InputDataType const& InputParameter() const { return inputParameter; }
   //! Modify the input parameter.
@@ -114,6 +119,11 @@ class MultiplyMerge
   //! Modify the delta.
   OutputDataType& Delta() { return delta; }
 
+  //! Get the gradient.
+  OutputDataType const& Gradient() const { return gradient; }
+  //! Modify the gradient.
+  OutputDataType& Gradient() { return gradient; }
+
   //! Return the model modules.
   std::vector<LayerTypes<CustomLayers...> >& Model()
   {
@@ -125,6 +135,11 @@ class MultiplyMerge
     return empty;
   }
 
+  //! Get the parameters.
+  OutputDataType const& Parameters() const { return weights; }
+  //! Modify the parameters.
+  OutputDataType& Parameters() { return weights; }
+
   /**
    * Serialize the layer.
    */
@@ -135,6 +150,10 @@ class MultiplyMerge
   //! Parameter which indicates if the modules should be exposed.
   bool model;
 
+  //! Parameter which indicates if the Forward/Backward method should be called
+  //! before merging the output.
+  bool run;
+
   //! We need this to know whether we should delete the layer in the destructor.
   bool ownsLayer;
 
@@ -156,11 +175,17 @@ class MultiplyMerge
   //! Locally-stored delta object.
   OutputDataType delta;
 
+  //! Locally-stored gradient object.
+  OutputDataType gradient;
+
   //! Locally-stored input parameter object.
   InputDataType inputParameter;
 
   //! Locally-stored output parameter object.
   OutputDataType outputParameter;
+
+  //! Locally-stored weight object.
+  OutputDataType weights;
 }; // class MultiplyMerge
 
 } // namespace ann

src/mlpack/methods/ann/layer/multiply_merge_impl.hpp

@@ -16,13 +16,18 @@
 // In case it hasn't yet been included.
 #include "multiply_merge.hpp"
 
+#include "../visitor/forward_visitor.hpp"
+#include "../visitor/backward_visitor.hpp"
+#include "../visitor/gradient_visitor.hpp"
+
 namespace mlpack {
 namespace ann /** Artificial Neural Network. */ {
 
 template<typename InputDataType, typename OutputDataType,
          typename... CustomLayers>
 MultiplyMerge<InputDataType, OutputDataType, CustomLayers...>::MultiplyMerge(
-    const bool model) : model(model), ownsLayer(!model)
+    const bool model, const bool run) :
+    model(model), run(run), ownsLayer(!model)
 {
   // Nothing to do here.
 }
@@ -42,10 +47,19 @@ template <typename InputDataType, typename OutputDataType,
           typename... CustomLayers>
 template<typename InputType, typename OutputType>
 void MultiplyMerge<InputDataType, OutputDataType, CustomLayers...>::Forward(
-    const InputType&& /* input */, OutputType&& output)
+    InputType&& input, OutputType&& output)
 {
-  output = boost::apply_visitor(outputParameterVisitor, network.front());
+  if (run)
+  {
+    for (size_t i = 0; i < network.size(); ++i)
+    {
+      boost::apply_visitor(ForwardVisitor(std::move(input), std::move(
+          boost::apply_visitor(outputParameterVisitor, network[i]))),
+          network[i]);
+    }
+  }
 
+  output = boost::apply_visitor(outputParameterVisitor, network.front());
   for (size_t i = 1; i < network.size(); ++i)
   {
     output %= boost::apply_visitor(outputParameterVisitor, network[i]);
@@ -58,7 +72,41 @@ template<typename eT>
 void MultiplyMerge<InputDataType, OutputDataType, CustomLayers...>::Backward(
     const arma::Mat<eT>&& /* input */, arma::Mat<eT>&& gy, arma::Mat<eT>&& g)
 {
-  g = gy;
+  if (run)
+  {
+    for (size_t i = 0; i < network.size(); ++i)
+    {
+      boost::apply_visitor(BackwardVisitor(std::move(boost::apply_visitor(
+          outputParameterVisitor, network[i])), std::move(gy), std::move(
+          boost::apply_visitor(deltaVisitor, network[i]))), network[i]);
+    }
+
+    g = boost::apply_visitor(deltaVisitor, network[0]);
+    for (size_t i = 1; i < network.size(); ++i)
+    {
+      g += boost::apply_visitor(deltaVisitor, network[i]);
+    }
+  }
+  else
+    g = gy;
+}
+
+template<typename InputDataType, typename OutputDataType,
+         typename... CustomLayers>
+template<typename eT>
+void MultiplyMerge<InputDataType, OutputDataType, CustomLayers...>::Gradient(
+    arma::Mat<eT>&& input,
+    arma::Mat<eT>&& error,
+    arma::Mat<eT>&& /* gradient */ )
+{
+  if (run)
+  {
+    for (size_t i = 0; i < network.size(); ++i)
+    {
+      boost::apply_visitor(GradientVisitor(std::move(input), std::move(error)),
+          network[i]);
+    }
+  }
 }
 
 template<typename InputDataType, typename OutputDataType,

src/mlpack/methods/ann/layer/recurrent_impl.hpp

@@ -76,7 +76,7 @@ Recurrent<InputDataType, OutputDataType, CustomLayers...>::Recurrent(
     ownsLayer(true)
 {
   initialModule = new Sequential<>();
-  mergeModule = new AddMerge<>(false);
+  mergeModule = new AddMerge<>(false, false);
   recurrentModule = new Sequential<>(false);
 
   boost::apply_visitor(AddVisitor<CustomLayers...>(inputModule),
@@ -261,7 +261,7 @@ void Recurrent<InputDataType, OutputDataType, CustomLayers...>::serialize(
   if (Archive::is_loading::value)
   {
     initialModule = new Sequential<>();
-    mergeModule = new AddMerge<>(false);
+    mergeModule = new AddMerge<>(false, false);
     recurrentModule = new Sequential<>(false);
 
     boost::apply_visitor(AddVisitor<CustomLayers...>(inputModule),

src/mlpack/tests/ann_layer_test.cpp

@@ -869,7 +869,7 @@ BOOST_AUTO_TEST_CASE(SimpleAddMergeLayerTest)
 
   for (size_t i = 0; i < 5; ++i)
   {
-    AddMerge<> module;
+    AddMerge<> module(false, false);
     const size_t numMergeModules = math::RandInt(2, 10);
     for (size_t m = 0; m < numMergeModules; ++m)
     {
@@ -877,7 +877,7 @@ BOOST_AUTO_TEST_CASE(SimpleAddMergeLayerTest)
       identityLayer.Forward(std::move(input),
           std::move(identityLayer.OutputParameter()));
 
-      module.Add(identityLayer);
+      module.Add<IdentityLayer<> >(identityLayer);
     }
 
     // Test the Forward function.
@@ -1617,7 +1617,7 @@ BOOST_AUTO_TEST_CASE(SimpleMultiplyMergeLayerTest)
 
   for (size_t i = 0; i < 5; ++i)
   {
-    MultiplyMerge<> module;
+    MultiplyMerge<> module(false, false);
     const size_t numMergeModules = math::RandInt(2, 10);
     for (size_t m = 0; m < numMergeModules; ++m)
     {
@@ -1625,7 +1625,7 @@ BOOST_AUTO_TEST_CASE(SimpleMultiplyMergeLayerTest)
       identityLayer.Forward(std::move(input),
           std::move(identityLayer.OutputParameter()));
 
-      module.Add(identityLayer);
+      module.Add<IdentityLayer<> >(identityLayer);
     }
 
     // Test the Forward function.
@@ -1798,6 +1798,70 @@ BOOST_AUTO_TEST_CASE(GradientLayerNormTest)
   BOOST_REQUIRE_LE(CheckGradient(function), 1e-4);
 }
 
+/**
+ * Test if the AddMerge layer is able to forward the
+ * Forward/Backward/Gradient calls.
+ */
+BOOST_AUTO_TEST_CASE(AddMergeRunTest)
+{
+  arma::mat output, input, delta, error;
+
+  AddMerge<> module(true, true);
+
+  Linear<>* linear = new Linear<>(10, 10);
+  module.Add(linear);
+
+  linear->Parameters().randu();
+  linear->Reset();
+
+  input = arma::zeros(10, 1);
+  module.Forward(std::move(input), std::move(output));
+
+  double parameterSum = arma::accu(linear->Parameters().submat(
+      100, 0, linear->Parameters().n_elem - 1, 0));
+
+  // Test the Backward function.
+  module.Backward(std::move(input), std::move(input), std::move(delta));
+
+  // Clean up before we break,
+  delete linear;
+
+  BOOST_REQUIRE_CLOSE(parameterSum, arma::accu(output), 1e-3);
+  BOOST_REQUIRE_EQUAL(arma::accu(delta), 0);
+}
+
+/**
+ * Test if the MultiplyMerge layer is able to forward the
+ * Forward/Backward/Gradient calls.
+ */
+BOOST_AUTO_TEST_CASE(MultiplyMergeRunTest)
+{
+  arma::mat output, input, delta, error;
+
+  MultiplyMerge<> module(true, true);
+
+  Linear<>* linear = new Linear<>(10, 10);
+  module.Add(linear);
+
+  linear->Parameters().randu();
+  linear->Reset();
+
+  input = arma::zeros(10, 1);
+  module.Forward(std::move(input), std::move(output));
+
+  double parameterSum = arma::accu(linear->Parameters().submat(
+      100, 0, linear->Parameters().n_elem - 1, 0));
+
+  // Test the Backward function.
+  module.Backward(std::move(input), std::move(input), std::move(delta));
+
+  // Clean up before we break,
+  delete linear;
+
+  BOOST_REQUIRE_CLOSE(parameterSum, arma::accu(output), 1e-3);
+  BOOST_REQUIRE_EQUAL(arma::accu(delta), 0);
+}
+
 /**
  * Simple subview module test.
  */