changes acc to suggestions.

src/mlpack/methods/ann/layer/CMakeLists.txt

@@ -7,6 +7,8 @@ set(SOURCES
   adaptive_mean_pooling_impl.hpp
   add.hpp
   add_impl.hpp
+  add_merge.hpp
+  add_merge_impl.hpp
   alpha_dropout.hpp
   alpha_dropout_impl.hpp
   base_layer.hpp
@@ -43,8 +45,6 @@ set(SOURCES
   padding.hpp
   radial_basis_function.hpp
   radial_basis_function_impl.hpp
-  residual.hpp
-  residual_impl.hpp
   serialization.hpp
 )
 

src/mlpack/methods/ann/layer/add_merge.hpp

@@ -0,0 +1,106 @@
+/**
+ * @file methods/ann/layer/add_merge.hpp
+ * @author Shubham Agrawal
+ *
+ * Definition of the AddMerge class, which acts as a addition container.
+ *
+ * mlpack is free software; you may redistribute it and/or modify it under the
+ * terms of the 3-clause BSD license.  You should have received a copy of the
+ * 3-clause BSD license along with mlpack.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef MLPACK_METHODS_ANN_LAYER_ADD_MERGE_HPP
+#define MLPACK_METHODS_ANN_LAYER_ADD_MERGE_HPP
+
+#include "../make_alias.hpp"
+#include "multi_layer.hpp"
+
+namespace mlpack {
+namespace ann {
+
+/**
+ * Implementation of the AddMerge class. The AddMerge class works as a
+ * feed-forward fully connected network container which plugs various layers
+ * together.
+ *
+ * @tparam MatType Matrix representation to accept as input and use for
+ *    computation.
+ */
+template<typename MatType>
+class AddMergeType : public MultiLayer<MatType>
+{
+ public:
+  /**
+   * Create an empty AddMergeType that holds no layers of its own.  Be sure to add
+   * layers with Add() before using!
+   */
+  AddMergeType();
+
+  //! Copy the given AddMergeType.
+  AddMergeType(const AddMergeType& other);
+  //! Take ownership of the layers of the given AddMergeType.
+  AddMergeType(AddMergeType&& other);
+  //! Copy the given AddMergeType.
+  AddMergeType& operator=(const AddMergeType& other);
+  //! Take ownership of the given AddMergeType.
+  AddMergeType& operator=(AddMergeType&& other);
+
+  //! Virtual destructor: delete all held layers.
+  virtual ~AddMergeType()
+  {
+    // Nothing to do here. 
+  }
+
+  //! Create a copy of the AddMergeType (this is safe for polymorphic use).
+  AddMergeType* Clone() const { return new AddMergeType(*this); }
+
+  /**
+   * Ordinary feed forward pass of a neural network, evaluating the function
+   * f(x) by propagating the activity forward through f.
+   *
+   * @param input Input data used for evaluating the specified function.
+   * @param output Resulting output activation.
+   */
+  void Forward(const MatType& input, MatType& output);
+
+  /**
+   * Ordinary feed backward pass of a neural network, using 3rd-order tensors as
+   * input, calculating the function f(x) by propagating x backwards through f.
+   * Using the results from the feed forward pass.
+   *
+   * @param input The propagated input activation.
+   * @param gy The backpropagated error.
+   * @param g The calculated gradient.
+   */
+  void Backward(const MatType& input,
+                const MatType& gy,
+                MatType& g);
+
+  /**
+   * Calculate the gradient using the output delta and the input activation.
+   *
+   * @param input The input parameter used for calculating the gradient.
+   * @param error The calculated error.
+   * @param gradient The calculated gradient.
+   */
+  void Gradient(const MatType& input,
+                const MatType& error,
+                MatType& gradient);
+
+  //! Compute the size of the output given `InputDimensions()`.
+  void ComputeOutputDimensions();
+
+  //! Serialize the AddMergeType.
+  template<typename Archive>
+  void serialize(Archive& ar, const uint32_t /* version */);
+};
+
+typedef AddMergeType<arma::mat> AddMerge;
+
+} // namespace ann
+} // namespace mlpack
+
+// Include implementation.
+#include "add_merge_impl.hpp"
+
+#endif

src/mlpack/methods/ann/layer/add_merge_impl.hpp

@@ -1,46 +1,45 @@
 /**
- * @file methods/ann/layer/residual_impl.hpp
+ * @file methods/ann/layer/add_merge_impl.hpp
  * @author Shubham Agrawal
  *
- * Implementation of the base class for neural network layers that are wrappers
- * around other layers.
+ * Implementation of the AddMerge class, which acts as a addition container.
  *
  * mlpack is free software; you may redistribute it and/or modify it under the
  * terms of the 3-clause BSD license.  You should have received a copy of the
  * 3-clause BSD license along with mlpack.  If not, see
  * http://www.opensource.org/licenses/BSD-3-Clause for more information.
  */
-#ifndef MLPACK_METHODS_ANN_LAYER_RESIDUAL_IMPL_HPP
-#define MLPACK_METHODS_ANN_LAYER_RESIDUAL_IMPL_HPP
+#ifndef MLPACK_METHODS_ANN_LAYER_ADD_MERGE_IMPL_HPP
+#define MLPACK_METHODS_ANN_LAYER_ADD_MERGE_IMPL_HPP
 
-#include "residual.hpp"
+#include "add_merge.hpp"
 
 namespace mlpack {
 namespace ann {
 
 template<typename MatType>
-ResidualType<MatType>::ResidualType() :
+AddMergeType<MatType>::AddMergeType() :
     MultiLayer<MatType>()
 {
   // Nothing to do.
 }
 
 template<typename MatType>
-ResidualType<MatType>::ResidualType(const ResidualType& other) :
+AddMergeType<MatType>::AddMergeType(const AddMergeType& other) :
     MultiLayer<MatType>(other)
 {
   // Nothing to do here.
 }
 
 template<typename MatType>
-ResidualType<MatType>::ResidualType(ResidualType&& other) :
+AddMergeType<MatType>::AddMergeType(AddMergeType&& other) :
     MultiLayer<MatType>(other)
 {
   // Nothing to do here.
 }
 
 template<typename MatType>
-ResidualType<MatType>& ResidualType<MatType>::operator=(const ResidualType& other)
+AddMergeType<MatType>& AddMergeType<MatType>::operator=(const AddMergeType& other)
 {
   if (this != &other)
   {
@@ -51,7 +50,7 @@ ResidualType<MatType>& ResidualType<MatType>::operator=(const ResidualType& othe
 }
 
 template<typename MatType>
-ResidualType<MatType>& ResidualType<MatType>::operator=(ResidualType&& other)
+AddMergeType<MatType>& AddMergeType<MatType>::operator=(AddMergeType&& other)
 {
   if (this != &other)
   {
@@ -62,7 +61,7 @@ ResidualType<MatType>& ResidualType<MatType>::operator=(ResidualType&& other)
 }
 
 template<typename MatType>
-void ResidualType<MatType>::Forward(
+void AddMergeType<MatType>::Forward(
     const MatType& input, MatType& output)
 {
   // Make sure training/testing mode is set right in each layer.
@@ -76,15 +75,16 @@ void ResidualType<MatType>::Forward(
     // Initialize memory for the forward pass (if needed).
     this->InitializeForwardPassMemory(input.n_cols);
 
+    // Forward pass every layer in network with same input.
     for (size_t i = 0; i < this->network.size(); i++)
       this->network[i]->Forward(input, this->layerOutputs[i]);
 
-		// Reduce the outputs to single output.
+		// Reduce the outputs to single output by adding element-wise.
     output.zeros();
-		for (size_t i = 0; i < this->layerOutputs.size(); i++)
-		{
-			output += this->layerOutputs[i];
-		}
+    for (size_t i = 0; i < this->layerOutputs.size(); i++)
+    {
+      output += this->layerOutputs[i];
+    }
   }
   else if (this->network.size() == 1)
   {
@@ -98,7 +98,7 @@ void ResidualType<MatType>::Forward(
 }
 
 template<typename MatType>
-void ResidualType<MatType>::Backward(
+void AddMergeType<MatType>::Backward(
     const MatType& input, const MatType& gy, MatType& g)
 {
   if (this->network.size() > 1)
@@ -107,10 +107,11 @@ void ResidualType<MatType>::Backward(
     this->InitializeBackwardPassMemory(input.n_cols);
 
     g.zeros();
-    for (size_t i = 0; i < this->network.size(); i++) {
+    for (size_t i = 0; i < this->network.size(); i++) 
+    {
       this->network[i]->Backward(this->layerOutputs[i], gy, this->layerDeltas[i]);
-			g += this->layerDeltas[i];
-		}
+      g += this->layerDeltas[i];
+    }
   }
   else if (this->network.size() == 1)
   {
@@ -124,7 +125,7 @@ void ResidualType<MatType>::Backward(
 }
 
 template<typename MatType>
-void ResidualType<MatType>::Gradient(
+void AddMergeType<MatType>::Gradient(
     const MatType& input, const MatType& error, MatType& gradient)
 {
   // We assume gradient has the right size already.
@@ -151,7 +152,7 @@ void ResidualType<MatType>::Gradient(
 }
 
 template<typename MatType>
-void ResidualType<MatType>::ComputeOutputDimensions()
+void AddMergeType<MatType>::ComputeOutputDimensions()
 {
   this->inSize = 0;
   this->totalInputSize = 0;
@@ -175,7 +176,7 @@ void ResidualType<MatType>::ComputeOutputDimensions()
     this->totalOutputSize += layerOutputSize;
   }
 
-	// Compute the output size of the network using reduction rules.
+  // Compute the output size of the network using reduction rules.
   if (this->network.size() == 1) 
   {
     this->outputDimensions = this->network[0]->OutputDimensions();
@@ -197,7 +198,7 @@ void ResidualType<MatType>::ComputeOutputDimensions()
 
 template<typename MatType>
 template<typename Archive>
-void ResidualType<MatType>::serialize(
+void AddMergeType<MatType>::serialize(
     Archive& ar, const uint32_t /* version */)
 {
   ar(cereal::base_class<MultiLayer<MatType>>(this));

src/mlpack/methods/ann/layer/layer_types.hpp

@@ -20,6 +20,7 @@
 #include <mlpack/methods/ann/layer/adaptive_max_pooling.hpp>
 #include <mlpack/methods/ann/layer/adaptive_mean_pooling.hpp>
 #include <mlpack/methods/ann/layer/add.hpp>
+#include <mlpack/methods/ann/layer/add_merge.hpp>
 #include <mlpack/methods/ann/layer/alpha_dropout.hpp>
 #include <mlpack/methods/ann/layer/base_layer.hpp>
 #include <mlpack/methods/ann/layer/concatenate.hpp>
@@ -37,7 +38,6 @@
 #include <mlpack/methods/ann/layer/noisylinear.hpp>
 #include <mlpack/methods/ann/layer/padding.hpp>
 #include <mlpack/methods/ann/layer/radial_basis_function.hpp>
-#include <mlpack/methods/ann/layer/residual.hpp>
 #include <mlpack/methods/ann/layer/softmax.hpp>
 
 // Convolution modes.

src/mlpack/methods/ann/layer/residual.hpp

@@ -1,127 +0,0 @@
-/**
- * @file methods/ann/layer/residual.hpp
- * @author Shubham Agrawal
- *
- * Base class for neural network layers that are wrappers around other layers.
- *
- * mlpack is free software; you may redistribute it and/or modify it under the
- * terms of the 3-clause BSD license.  You should have received a copy of the
- * 3-clause BSD license along with mlpack.  If not, see
- * http://www.opensource.org/licenses/BSD-3-Clause for more information.
- */
-#ifndef MLPACK_METHODS_ANN_LAYER_RESIDUAL_HPP
-#define MLPACK_METHODS_ANN_LAYER_RESIDUAL_HPP
-
-#include "../make_alias.hpp"
-#include "multi_layer.hpp"
-
-namespace mlpack {
-namespace ann {
-
-/**
- * A lambda "map-reduce" is a layer that is a wrapper around other layers.  
- * It passes the input through all of its child layers sequentially, returning
- * the output from reducing the output.
- *
- * @tparam MatType Matrix representation to accept as input and use for
- *    computation.
- */
-template<typename MatType>
-class ResidualType : public MultiLayer<MatType>
-{
- public:
-  /**
-   * Create an empty ResidualType that holds no layers of its own.  Be sure to add
-   * layers with Add() before using!
-   */
-  ResidualType();
-
-  //! Copy the given ResidualType.
-  ResidualType(const ResidualType& other);
-  //! Take ownership of the layers of the given ResidualType.
-  ResidualType(ResidualType&& other);
-  //! Copy the given ResidualType.
-  ResidualType& operator=(const ResidualType& other);
-  //! Take ownership of the given ResidualType.
-  ResidualType& operator=(ResidualType&& other);
-
-  //! Virtual destructor: delete all held layers.
-  virtual ~ResidualType()
-  {
-    // Nothing to do here. 
-  }
-
-  //! Create a copy of the ResidualType (this is safe for polymorphic use).
-  virtual ResidualType* Clone() const { return new ResidualType(*this); }
-
-  /**
-   * Perform a forward pass with the given input data.  `output` is expected to
-   * have the correct size (e.g. number of rows equal to `OutputSize()` of the
-   * last held layer; number of columns equal to `input.n_cols`).
-   *
-   * @param input Input data to pass through the ResidualType.
-   * @param output Matrix to store output in.
-   */
-  virtual void Forward(const MatType& input, MatType& output);
-
-  /**
-   * Perform a backward pass with the given data.  `gy` is expected to be the
-   * propagated error from the subsequent layer (or output), `input` is expected
-   * to be the output from this layer when `Forward()` was called, and `g` will
-   * store the propagated error from this layer (to be passed to the previous
-   * layer as `gy`).
-   *
-   * It is expected that `g` has the correct size already (e.g., number of rows
-   * equal to `OutputSize()` of the previous layer, and number of columns equal
-   * to `input.n_cols`).
-   *
-   * This function is expected to be called for the same input data as
-   * `Forward()` was just called for.
-   *
-   * @param input Output of Forward().
-   * @param gy Propagated error from next layer.
-   * @param g Matrix to store propagated error in for previous layer.
-   */
-  virtual void Backward(const MatType& input,
-                        const MatType& gy,
-                        MatType& g);
-
-  /**
-   * Compute the gradients of each layer.
-   *
-   * This function is expected to be called for the same input data as
-   * `Forward()` and `Backward()` were just called for.  That is, `input` here
-   * should be the same data as `Forward()` was called with.
-   *
-   * `gradient` is expected to have the correct size already (e.g., number of
-   * rows equal to 1, and number of columns equal to `WeightSize()`).
-   *
-   * @param input Original input data provided to Forward().
-   * @param error Error as computed by `Backward()`.
-   * @param gradient Matrix to store the gradients in.
-   */
-  virtual void Gradient(const MatType& input,
-                        const MatType& error,
-                        MatType& gradient);
-
-  /**
-   * Compute the output dimensions of the ResidualType using `InputDimensions()`.
-   * This computes the dimensions of each layer held by the ResidualType, and the
-   * output dimensions are set to the output dimensions of the last layer.
-   */
-  virtual void ComputeOutputDimensions();
-
-  //! Serialize the ResidualType.
-  template<typename Archive>
-  void serialize(Archive& ar, const uint32_t /* version */);
-};
-
-typedef ResidualType<arma::mat> Residual;
-
-} // namespace ann
-} // namespace mlpack
-
-// Include implementation.
-#include "residual_impl.hpp"
-
-#endif

src/mlpack/methods/ann/layer/serialization.hpp

@@ -16,6 +16,7 @@
     CEREAL_REGISTER_TYPE(mlpack::ann::AdaptiveMeanPoolingType<__VA_ARGS__>); \
     CEREAL_REGISTER_TYPE(mlpack::ann::AdaptiveMaxPoolingType<__VA_ARGS__>); \
     CEREAL_REGISTER_TYPE(mlpack::ann::AddType<__VA_ARGS__>); \
+    CEREAL_REGISTER_TYPE(mlpack::ann::AddMergeType<__VA_ARGS__>); \
     CEREAL_REGISTER_TYPE(mlpack::ann::AlphaDropoutType<__VA_ARGS__>); \
     /* Base layers from base_layer.hpp. */ \
     CEREAL_REGISTER_TYPE(mlpack::ann::SigmoidType<__VA_ARGS__>); \
@@ -50,7 +51,6 @@
     CEREAL_REGISTER_TYPE(mlpack::ann::NoisyLinearType<__VA_ARGS__>); \
     CEREAL_REGISTER_TYPE(mlpack::ann::PaddingType<__VA_ARGS__>); \
     CEREAL_REGISTER_TYPE(mlpack::ann::RBFType<__VA_ARGS__>); \
-    CEREAL_REGISTER_TYPE(mlpack::ann::ResidualType<__VA_ARGS__>); \
     CEREAL_REGISTER_TYPE(mlpack::ann::SoftmaxType<__VA_ARGS__>); \
 
 CEREAL_REGISTER_MLPACK_LAYERS(arma::mat);

src/mlpack/tests/ann_layer_test.cpp

@@ -4854,9 +4854,9 @@ TEST_CASE("TransposedConvolutionLayerPaddingTest", "[ANNLayerTest]")
 // }
 
 /**
- * Simple test for Residual layer.
+ * Simple test for AddMerge layer.
  */
-TEST_CASE("ResidualTestCase", "[ANNLayerTest]")
+TEST_CASE("AddMergeTestCase", "[ANNLayerTest]")
 {
   // For rectangular input to pooling layers.
   arma::mat input = arma::mat(28, 1);
@@ -4871,11 +4871,11 @@ TEST_CASE("ResidualTestCase", "[ANNLayerTest]")
   input(14) = input(25) = 8;
   input(15) = input(26) = 9;
 
-  Residual module1;
+  AddMerge module1;
   module1.Add<MeanPooling>(2, 2, 2, 2, false);
   module1.Add<MeanPooling>(2, 2, 2, 2, false);
 
-  Residual module2;
+  AddMerge module2;
   module2.Add<MeanPooling>(2, 2, 2, 2, true);
   module2.Add<MeanPooling>(2, 2, 2, 2, true);