From 07bbc016bb775904d11a2ec7a152e368fc46c4c4 Mon Sep 17 00:00:00 2001 From: Shubham Agrawal Date: Wed, 15 Jun 2022 19:59:22 +0530 Subject: [PATCH] changes acc to suggestions. --- src/mlpack/methods/ann/layer/CMakeLists.txt | 4 +- src/mlpack/methods/ann/layer/add_merge.hpp | 106 +++++++++++++++ .../{residual_impl.hpp => add_merge_impl.hpp} | 51 +++---- src/mlpack/methods/ann/layer/layer_types.hpp | 2 +- src/mlpack/methods/ann/layer/residual.hpp | 127 ------------------ .../methods/ann/layer/serialization.hpp | 2 +- src/mlpack/tests/ann_layer_test.cpp | 8 +- 7 files changed, 140 insertions(+), 160 deletions(-) create mode 100644 src/mlpack/methods/ann/layer/add_merge.hpp rename src/mlpack/methods/ann/layer/{residual_impl.hpp => add_merge_impl.hpp} (78%) delete mode 100644 src/mlpack/methods/ann/layer/residual.hpp diff --git a/src/mlpack/methods/ann/layer/CMakeLists.txt b/src/mlpack/methods/ann/layer/CMakeLists.txt index 63019e606c..78fb292fc7 100644 --- a/src/mlpack/methods/ann/layer/CMakeLists.txt +++ b/src/mlpack/methods/ann/layer/CMakeLists.txt @@ -7,6 +7,8 @@ set(SOURCES adaptive_mean_pooling_impl.hpp add.hpp add_impl.hpp + add_merge.hpp + add_merge_impl.hpp alpha_dropout.hpp alpha_dropout_impl.hpp base_layer.hpp @@ -43,8 +45,6 @@ set(SOURCES padding.hpp radial_basis_function.hpp radial_basis_function_impl.hpp - residual.hpp - residual_impl.hpp serialization.hpp ) diff --git a/src/mlpack/methods/ann/layer/add_merge.hpp b/src/mlpack/methods/ann/layer/add_merge.hpp new file mode 100644 index 0000000000..39048ecbb6 --- /dev/null +++ b/src/mlpack/methods/ann/layer/add_merge.hpp @@ -0,0 +1,106 @@ +/** + * @file methods/ann/layer/add_merge.hpp + * @author Shubham Agrawal + * + * Definition of the AddMerge class, which acts as a addition container. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_METHODS_ANN_LAYER_ADD_MERGE_HPP +#define MLPACK_METHODS_ANN_LAYER_ADD_MERGE_HPP + +#include "../make_alias.hpp" +#include "multi_layer.hpp" + +namespace mlpack { +namespace ann { + +/** + * Implementation of the AddMerge class. The AddMerge class works as a + * feed-forward fully connected network container which plugs various layers + * together. + * + * @tparam MatType Matrix representation to accept as input and use for + * computation. + */ +template +class AddMergeType : public MultiLayer +{ + public: + /** + * Create an empty AddMergeType that holds no layers of its own. Be sure to add + * layers with Add() before using! + */ + AddMergeType(); + + //! Copy the given AddMergeType. + AddMergeType(const AddMergeType& other); + //! Take ownership of the layers of the given AddMergeType. + AddMergeType(AddMergeType&& other); + //! Copy the given AddMergeType. + AddMergeType& operator=(const AddMergeType& other); + //! Take ownership of the given AddMergeType. + AddMergeType& operator=(AddMergeType&& other); + + //! Virtual destructor: delete all held layers. + virtual ~AddMergeType() + { + // Nothing to do here. + } + + //! Create a copy of the AddMergeType (this is safe for polymorphic use). + AddMergeType* Clone() const { return new AddMergeType(*this); } + + /** + * Ordinary feed forward pass of a neural network, evaluating the function + * f(x) by propagating the activity forward through f. + * + * @param input Input data used for evaluating the specified function. + * @param output Resulting output activation. + */ + void Forward(const MatType& input, MatType& output); + + /** + * Ordinary feed backward pass of a neural network, using 3rd-order tensors as + * input, calculating the function f(x) by propagating x backwards through f. + * Using the results from the feed forward pass. + * + * @param input The propagated input activation. + * @param gy The backpropagated error. + * @param g The calculated gradient. + */ + void Backward(const MatType& input, + const MatType& gy, + MatType& g); + + /** + * Calculate the gradient using the output delta and the input activation. + * + * @param input The input parameter used for calculating the gradient. + * @param error The calculated error. + * @param gradient The calculated gradient. + */ + void Gradient(const MatType& input, + const MatType& error, + MatType& gradient); + + //! Compute the size of the output given `InputDimensions()`. + void ComputeOutputDimensions(); + + //! Serialize the AddMergeType. + template + void serialize(Archive& ar, const uint32_t /* version */); +}; + +typedef AddMergeType AddMerge; + +} // namespace ann +} // namespace mlpack + +// Include implementation. +#include "add_merge_impl.hpp" + +#endif diff --git a/src/mlpack/methods/ann/layer/residual_impl.hpp b/src/mlpack/methods/ann/layer/add_merge_impl.hpp similarity index 78% rename from src/mlpack/methods/ann/layer/residual_impl.hpp rename to src/mlpack/methods/ann/layer/add_merge_impl.hpp index bd0a726e67..44d1b689aa 100644 --- a/src/mlpack/methods/ann/layer/residual_impl.hpp +++ b/src/mlpack/methods/ann/layer/add_merge_impl.hpp @@ -1,46 +1,45 @@ /** - * @file methods/ann/layer/residual_impl.hpp + * @file methods/ann/layer/add_merge_impl.hpp * @author Shubham Agrawal * - * Implementation of the base class for neural network layers that are wrappers - * around other layers. + * Implementation of the AddMerge class, which acts as a addition container. * * mlpack is free software; you may redistribute it and/or modify it under the * terms of the 3-clause BSD license. You should have received a copy of the * 3-clause BSD license along with mlpack. If not, see * http://www.opensource.org/licenses/BSD-3-Clause for more information. */ -#ifndef MLPACK_METHODS_ANN_LAYER_RESIDUAL_IMPL_HPP -#define MLPACK_METHODS_ANN_LAYER_RESIDUAL_IMPL_HPP +#ifndef MLPACK_METHODS_ANN_LAYER_ADD_MERGE_IMPL_HPP +#define MLPACK_METHODS_ANN_LAYER_ADD_MERGE_IMPL_HPP -#include "residual.hpp" +#include "add_merge.hpp" namespace mlpack { namespace ann { template -ResidualType::ResidualType() : +AddMergeType::AddMergeType() : MultiLayer() { // Nothing to do. } template -ResidualType::ResidualType(const ResidualType& other) : +AddMergeType::AddMergeType(const AddMergeType& other) : MultiLayer(other) { // Nothing to do here. } template -ResidualType::ResidualType(ResidualType&& other) : +AddMergeType::AddMergeType(AddMergeType&& other) : MultiLayer(other) { // Nothing to do here. } template -ResidualType& ResidualType::operator=(const ResidualType& other) +AddMergeType& AddMergeType::operator=(const AddMergeType& other) { if (this != &other) { @@ -51,7 +50,7 @@ ResidualType& ResidualType::operator=(const ResidualType& othe } template -ResidualType& ResidualType::operator=(ResidualType&& other) +AddMergeType& AddMergeType::operator=(AddMergeType&& other) { if (this != &other) { @@ -62,7 +61,7 @@ ResidualType& ResidualType::operator=(ResidualType&& other) } template -void ResidualType::Forward( +void AddMergeType::Forward( const MatType& input, MatType& output) { // Make sure training/testing mode is set right in each layer. @@ -76,15 +75,16 @@ void ResidualType::Forward( // Initialize memory for the forward pass (if needed). this->InitializeForwardPassMemory(input.n_cols); + // Forward pass every layer in network with same input. for (size_t i = 0; i < this->network.size(); i++) this->network[i]->Forward(input, this->layerOutputs[i]); - // Reduce the outputs to single output. + // Reduce the outputs to single output by adding element-wise. output.zeros(); - for (size_t i = 0; i < this->layerOutputs.size(); i++) - { - output += this->layerOutputs[i]; - } + for (size_t i = 0; i < this->layerOutputs.size(); i++) + { + output += this->layerOutputs[i]; + } } else if (this->network.size() == 1) { @@ -98,7 +98,7 @@ void ResidualType::Forward( } template -void ResidualType::Backward( +void AddMergeType::Backward( const MatType& input, const MatType& gy, MatType& g) { if (this->network.size() > 1) @@ -107,10 +107,11 @@ void ResidualType::Backward( this->InitializeBackwardPassMemory(input.n_cols); g.zeros(); - for (size_t i = 0; i < this->network.size(); i++) { + for (size_t i = 0; i < this->network.size(); i++) + { this->network[i]->Backward(this->layerOutputs[i], gy, this->layerDeltas[i]); - g += this->layerDeltas[i]; - } + g += this->layerDeltas[i]; + } } else if (this->network.size() == 1) { @@ -124,7 +125,7 @@ void ResidualType::Backward( } template -void ResidualType::Gradient( +void AddMergeType::Gradient( const MatType& input, const MatType& error, MatType& gradient) { // We assume gradient has the right size already. @@ -151,7 +152,7 @@ void ResidualType::Gradient( } template -void ResidualType::ComputeOutputDimensions() +void AddMergeType::ComputeOutputDimensions() { this->inSize = 0; this->totalInputSize = 0; @@ -175,7 +176,7 @@ void ResidualType::ComputeOutputDimensions() this->totalOutputSize += layerOutputSize; } - // Compute the output size of the network using reduction rules. + // Compute the output size of the network using reduction rules. if (this->network.size() == 1) { this->outputDimensions = this->network[0]->OutputDimensions(); @@ -197,7 +198,7 @@ void ResidualType::ComputeOutputDimensions() template template -void ResidualType::serialize( +void AddMergeType::serialize( Archive& ar, const uint32_t /* version */) { ar(cereal::base_class>(this)); diff --git a/src/mlpack/methods/ann/layer/layer_types.hpp b/src/mlpack/methods/ann/layer/layer_types.hpp index e5166c73d1..7499b6cf13 100644 --- a/src/mlpack/methods/ann/layer/layer_types.hpp +++ b/src/mlpack/methods/ann/layer/layer_types.hpp @@ -20,6 +20,7 @@ #include #include #include +#include #include #include #include @@ -37,7 +38,6 @@ #include #include #include -#include #include // Convolution modes. diff --git a/src/mlpack/methods/ann/layer/residual.hpp b/src/mlpack/methods/ann/layer/residual.hpp deleted file mode 100644 index 86015df80b..0000000000 --- a/src/mlpack/methods/ann/layer/residual.hpp +++ /dev/null @@ -1,127 +0,0 @@ -/** - * @file methods/ann/layer/residual.hpp - * @author Shubham Agrawal - * - * Base class for neural network layers that are wrappers around other layers. - * - * mlpack is free software; you may redistribute it and/or modify it under the - * terms of the 3-clause BSD license. You should have received a copy of the - * 3-clause BSD license along with mlpack. If not, see - * http://www.opensource.org/licenses/BSD-3-Clause for more information. - */ -#ifndef MLPACK_METHODS_ANN_LAYER_RESIDUAL_HPP -#define MLPACK_METHODS_ANN_LAYER_RESIDUAL_HPP - -#include "../make_alias.hpp" -#include "multi_layer.hpp" - -namespace mlpack { -namespace ann { - -/** - * A lambda "map-reduce" is a layer that is a wrapper around other layers. - * It passes the input through all of its child layers sequentially, returning - * the output from reducing the output. - * - * @tparam MatType Matrix representation to accept as input and use for - * computation. - */ -template -class ResidualType : public MultiLayer -{ - public: - /** - * Create an empty ResidualType that holds no layers of its own. Be sure to add - * layers with Add() before using! - */ - ResidualType(); - - //! Copy the given ResidualType. - ResidualType(const ResidualType& other); - //! Take ownership of the layers of the given ResidualType. - ResidualType(ResidualType&& other); - //! Copy the given ResidualType. - ResidualType& operator=(const ResidualType& other); - //! Take ownership of the given ResidualType. - ResidualType& operator=(ResidualType&& other); - - //! Virtual destructor: delete all held layers. - virtual ~ResidualType() - { - // Nothing to do here. - } - - //! Create a copy of the ResidualType (this is safe for polymorphic use). - virtual ResidualType* Clone() const { return new ResidualType(*this); } - - /** - * Perform a forward pass with the given input data. `output` is expected to - * have the correct size (e.g. number of rows equal to `OutputSize()` of the - * last held layer; number of columns equal to `input.n_cols`). - * - * @param input Input data to pass through the ResidualType. - * @param output Matrix to store output in. - */ - virtual void Forward(const MatType& input, MatType& output); - - /** - * Perform a backward pass with the given data. `gy` is expected to be the - * propagated error from the subsequent layer (or output), `input` is expected - * to be the output from this layer when `Forward()` was called, and `g` will - * store the propagated error from this layer (to be passed to the previous - * layer as `gy`). - * - * It is expected that `g` has the correct size already (e.g., number of rows - * equal to `OutputSize()` of the previous layer, and number of columns equal - * to `input.n_cols`). - * - * This function is expected to be called for the same input data as - * `Forward()` was just called for. - * - * @param input Output of Forward(). - * @param gy Propagated error from next layer. - * @param g Matrix to store propagated error in for previous layer. - */ - virtual void Backward(const MatType& input, - const MatType& gy, - MatType& g); - - /** - * Compute the gradients of each layer. - * - * This function is expected to be called for the same input data as - * `Forward()` and `Backward()` were just called for. That is, `input` here - * should be the same data as `Forward()` was called with. - * - * `gradient` is expected to have the correct size already (e.g., number of - * rows equal to 1, and number of columns equal to `WeightSize()`). - * - * @param input Original input data provided to Forward(). - * @param error Error as computed by `Backward()`. - * @param gradient Matrix to store the gradients in. - */ - virtual void Gradient(const MatType& input, - const MatType& error, - MatType& gradient); - - /** - * Compute the output dimensions of the ResidualType using `InputDimensions()`. - * This computes the dimensions of each layer held by the ResidualType, and the - * output dimensions are set to the output dimensions of the last layer. - */ - virtual void ComputeOutputDimensions(); - - //! Serialize the ResidualType. - template - void serialize(Archive& ar, const uint32_t /* version */); -}; - -typedef ResidualType Residual; - -} // namespace ann -} // namespace mlpack - -// Include implementation. -#include "residual_impl.hpp" - -#endif diff --git a/src/mlpack/methods/ann/layer/serialization.hpp b/src/mlpack/methods/ann/layer/serialization.hpp index c774c8ee51..62cfec7b94 100644 --- a/src/mlpack/methods/ann/layer/serialization.hpp +++ b/src/mlpack/methods/ann/layer/serialization.hpp @@ -16,6 +16,7 @@ CEREAL_REGISTER_TYPE(mlpack::ann::AdaptiveMeanPoolingType<__VA_ARGS__>); \ CEREAL_REGISTER_TYPE(mlpack::ann::AdaptiveMaxPoolingType<__VA_ARGS__>); \ CEREAL_REGISTER_TYPE(mlpack::ann::AddType<__VA_ARGS__>); \ + CEREAL_REGISTER_TYPE(mlpack::ann::AddMergeType<__VA_ARGS__>); \ CEREAL_REGISTER_TYPE(mlpack::ann::AlphaDropoutType<__VA_ARGS__>); \ /* Base layers from base_layer.hpp. */ \ CEREAL_REGISTER_TYPE(mlpack::ann::SigmoidType<__VA_ARGS__>); \ @@ -50,7 +51,6 @@ CEREAL_REGISTER_TYPE(mlpack::ann::NoisyLinearType<__VA_ARGS__>); \ CEREAL_REGISTER_TYPE(mlpack::ann::PaddingType<__VA_ARGS__>); \ CEREAL_REGISTER_TYPE(mlpack::ann::RBFType<__VA_ARGS__>); \ - CEREAL_REGISTER_TYPE(mlpack::ann::ResidualType<__VA_ARGS__>); \ CEREAL_REGISTER_TYPE(mlpack::ann::SoftmaxType<__VA_ARGS__>); \ CEREAL_REGISTER_MLPACK_LAYERS(arma::mat); diff --git a/src/mlpack/tests/ann_layer_test.cpp b/src/mlpack/tests/ann_layer_test.cpp index 45cd89c24c..d421cd0a72 100644 --- a/src/mlpack/tests/ann_layer_test.cpp +++ b/src/mlpack/tests/ann_layer_test.cpp @@ -4854,9 +4854,9 @@ TEST_CASE("TransposedConvolutionLayerPaddingTest", "[ANNLayerTest]") // } /** - * Simple test for Residual layer. + * Simple test for AddMerge layer. */ -TEST_CASE("ResidualTestCase", "[ANNLayerTest]") +TEST_CASE("AddMergeTestCase", "[ANNLayerTest]") { // For rectangular input to pooling layers. arma::mat input = arma::mat(28, 1); @@ -4871,11 +4871,11 @@ TEST_CASE("ResidualTestCase", "[ANNLayerTest]") input(14) = input(25) = 8; input(15) = input(26) = 9; - Residual module1; + AddMerge module1; module1.Add(2, 2, 2, 2, false); module1.Add(2, 2, 2, 2, false); - Residual module2; + AddMerge module2; module2.Add(2, 2, 2, 2, true); module2.Add(2, 2, 2, 2, true);