diff --git a/src/mlpack/methods/ann/CMakeLists.txt b/src/mlpack/methods/ann/CMakeLists.txt index 414dc863f3..3b7ee5b79f 100644 --- a/src/mlpack/methods/ann/CMakeLists.txt +++ b/src/mlpack/methods/ann/CMakeLists.txt @@ -14,7 +14,7 @@ add_subdirectory(layer) add_subdirectory(loss_functions) add_subdirectory(convolution_rules) add_subdirectory(regularizer) -add_subdirectory(reduction_rules) +# add_subdirectory(reduction_rules) # Add directory name to sources. set(DIR_SRCS) diff --git a/src/mlpack/methods/ann/layer/CMakeLists.txt b/src/mlpack/methods/ann/layer/CMakeLists.txt index 44dbbade5b..63019e606c 100644 --- a/src/mlpack/methods/ann/layer/CMakeLists.txt +++ b/src/mlpack/methods/ann/layer/CMakeLists.txt @@ -18,8 +18,6 @@ set(SOURCES dropconnect_impl.hpp dropout.hpp dropout_impl.hpp - lambda_map_reduce.hpp - lambda_map_reduce_impl.hpp layer.hpp layer_types.hpp leaky_relu.hpp @@ -45,6 +43,8 @@ set(SOURCES padding.hpp radial_basis_function.hpp radial_basis_function_impl.hpp + residual.hpp + residual_impl.hpp serialization.hpp ) diff --git a/src/mlpack/methods/ann/layer/lambda_map_reduce.hpp b/src/mlpack/methods/ann/layer/lambda_map_reduce.hpp deleted file mode 100644 index b124719822..0000000000 --- a/src/mlpack/methods/ann/layer/lambda_map_reduce.hpp +++ /dev/null @@ -1,254 +0,0 @@ -/** - * @file methods/ann/layer/lambda_map_reduce.hpp - * @author Shubham Agrawal - * - * Base class for neural network layers that are wrappers around other layers. - * - * mlpack is free software; you may redistribute it and/or modify it under the - * terms of the 3-clause BSD license. You should have received a copy of the - * 3-clause BSD license along with mlpack. If not, see - * http://www.opensource.org/licenses/BSD-3-Clause for more information. - */ -#ifndef MLPACK_METHODS_ANN_LAYER_LAMBDA_MAP_REDUCE_HPP -#define MLPACK_METHODS_ANN_LAYER_LAMBDA_MAP_REDUCE_HPP - -#include "../make_alias.hpp" -#include "layer.hpp" -#include - -namespace mlpack { -namespace ann { - -/** - * A lambda "map-reduce" is a layer that is a wrapper around other layers. - * It passes the input through all of its child layers sequentially, returning - * the output from reducing the output. - * - * It's likely not very useful to use this layer directly; instead, this layer - * is meant as a base class for use by other layers that must store and use - * multiple layers. - * - * @tparam ReductionRuleType Reduction rule to use for the layer. - * @tparam MatType Matrix representation to accept as input and use for - * computation. - */ -template -class LambdaMapReduceType : public Layer -{ - public: - /** - * Create an empty LambdaMapReduceType that holds no layers of its own. Be sure to add - * layers with Add() before using! - */ - LambdaMapReduceType(); - - //! Copy the given LambdaMapReduceType. - LambdaMapReduceType(const LambdaMapReduceType& other); - //! Take ownership of the layers of the given LambdaMapReduceType. - LambdaMapReduceType(LambdaMapReduceType&& other); - //! Copy the given LambdaMapReduceType. - LambdaMapReduceType& operator=(const LambdaMapReduceType& other); - //! Take ownership of the given LambdaMapReduceType. - LambdaMapReduceType& operator=(LambdaMapReduceType&& other); - - //! Virtual destructor: delete all held layers. - virtual ~LambdaMapReduceType() - { - for (size_t i = 0; i < network.size(); ++i) - delete network[i]; - } - - //! Create a copy of the LambdaMapReduceType (this is safe for polymorphic use). - virtual LambdaMapReduceType* Clone() const { return new LambdaMapReduceType(*this); } - - /** - * Perform a forward pass with the given input data. `output` is expected to - * have the correct size (e.g. number of rows equal to `OutputSize()` of the - * last held layer; number of columns equal to `input.n_cols`). - * - * @param input Input data to pass through the LambdaMapReduceType. - * @param output Matrix to store output in. - */ - virtual void Forward(const MatType& input, MatType& output); - - /** - * Perform a backward pass with the given data. `gy` is expected to be the - * propagated error from the subsequent layer (or output), `input` is expected - * to be the output from this layer when `Forward()` was called, and `g` will - * store the propagated error from this layer (to be passed to the previous - * layer as `gy`). - * - * It is expected that `g` has the correct size already (e.g., number of rows - * equal to `OutputSize()` of the previous layer, and number of columns equal - * to `input.n_cols`). - * - * This function is expected to be called for the same input data as - * `Forward()` was just called for. - * - * @param input Output of Forward(). - * @param gy Propagated error from next layer. - * @param g Matrix to store propagated error in for previous layer. - */ - virtual void Backward(const MatType& input, - const MatType& gy, - MatType& g); - - /** - * Compute the gradients of each layer. - * - * This function is expected to be called for the same input data as - * `Forward()` and `Backward()` were just called for. That is, `input` here - * should be the same data as `Forward()` was called with. - * - * `gradient` is expected to have the correct size already (e.g., number of - * rows equal to 1, and number of columns equal to `WeightSize()`). - * - * @param input Original input data provided to Forward(). - * @param error Error as computed by `Backward()`. - * @param gradient Matrix to store the gradients in. - */ - virtual void Gradient(const MatType& input, - const MatType& error, - MatType& gradient); - - /** - * Set the weights of the layer to use the memory given as `weightsPtr`. - */ - virtual void SetWeights(typename MatType::elem_type* weightsPtr); - - /** - * Return the number of weights in the LambdaMapReduceType. This is the sum of the - * number of weights in each layer. - */ - virtual size_t WeightSize() const; - - /** - * Compute the output dimensions of the LambdaMapReduceType using `InputDimensions()`. - * This computes the dimensions of each layer held by the LambdaMapReduceType, and the - * output dimensions are set to the output dimensions of the last layer. - */ - virtual void ComputeOutputDimensions(); - - /** - * Compute the loss that should be added to the objective. - */ - virtual double Loss() const; - - /** - * Add a new module to the model. - * - * @param args The layer parameter. - */ - template - void Add(Args... args) - { - network.push_back(new LayerType(args...)); - layerOutputs.push_back(MatType()); - layerDeltas.push_back(MatType()); - layerGradients.push_back(MatType()); - } - - /** - * Add a new module to the model. - * - * @param layer The Layer to be added to the model. - */ - void Add(Layer* layer) - { - network.push_back(layer); - layerOutputs.push_back(MatType()); - layerDeltas.push_back(MatType()); - layerGradients.push_back(MatType()); - } - - //! Get the network (series of layers) held by this LambdaMapReduceType. - const std::vector*> Network() const - { - return network; - } - //! Modify the network (series of layers) held by this LambdaMapReduceType. Be - //! careful! - std::vector*>& Network() { return network; } - - //! Get the parameters. - MatType const& Parameters() const { return weights; } - //! Modify the parameters. - MatType& Parameters() { return weights; } - - //! Serialize the LambdaMapReduceType. - template - void serialize(Archive& ar, const uint32_t /* version */); - - protected: - /** - * Initialize memory that will be used by each layer for the forward pass, - * assuming that the input will have the given `batchSize`. When `Forward()` - * is called, each internally-held layer will output its results into the - * memory allocated by this function (this is the internal member - * `layerOutputMatrix` and its aliases `layerOutputs`). - */ - void InitializeForwardPassMemory(const size_t batchSize); - - /** - * Initialize memory that will be used by each layer for the backwards pass, - * assuming that the input will have the given `batchSize`. When `Backward()` - * is called, each internally-held layer will output the results of its - * backwards pass into the memory allocated by this function (this is the - * internal member `layerDeltaMatrix` and its aliases `layerDeltas`). - */ - void InitializeBackwardPassMemory(const size_t batchSize); - - /** - * Initialize memory for the gradient pass. This sets the internal aliases - * `layerGradients` appropriately using the memory from the given `gradient`, - * such that each layer will output its gradient (via its `Gradient()` method) - * into the appropriate member of `layerGradients`. - */ - void InitializeGradientPassMemory(MatType& gradient); - - //! The internally-held network. - std::vector*> network; - - // Total number of elements in the input, cached for convenience. - size_t inSize; - // Total number of input elements for *every* layer. - size_t totalInputSize; - // Total number of output elements for *every* layer. - size_t totalOutputSize; - - //! Redction rule for the output of each layer. - ReductionRuleType reductionRule; - - //! Locally-stored weight object. - MatType weights; - - //! This matrix stores all of the outputs of each layer when Forward() is - //! called. See `InitializeForwardPassMemory()`. - MatType layerOutputMatrix; - //! These are aliases of `layerOutputMatrix` for each layer. - std::vector layerOutputs; - - //! This matrix stores all of the backwards pass results of each layer when - //! Backward() is called. See `InitializeBackwardPassMemory()`. - MatType layerDeltaMatrix; - //! These are aliases of `layerDeltaMatrix` for each layer. - std::vector layerDeltas; - - //! Gradient aliases for each layer. Note that this is *only* valid in the - //! context of `Gradient()`! We have it as a class member to avoid - //! reallocating the `MatType`s each call to `Gradient()`. - std::vector layerGradients; -}; - -typedef LambdaMapReduceType< - AddReduction, - arma::mat -> LambdaMapReduce; - -} // namespace ann -} // namespace mlpack - -// Include implementation. -#include "lambda_map_reduce_impl.hpp" - -#endif diff --git a/src/mlpack/methods/ann/layer/lambda_map_reduce_impl.hpp b/src/mlpack/methods/ann/layer/lambda_map_reduce_impl.hpp deleted file mode 100644 index 1bfc84f42f..0000000000 --- a/src/mlpack/methods/ann/layer/lambda_map_reduce_impl.hpp +++ /dev/null @@ -1,405 +0,0 @@ -/** - * @file methods/ann/layer/lambda_map_reduce_impl.hpp - * @author Shubham Agrawal - * - * Implementation of the base class for neural network layers that are wrappers - * around other layers. - * - * mlpack is free software; you may redistribute it and/or modify it under the - * terms of the 3-clause BSD license. You should have received a copy of the - * 3-clause BSD license along with mlpack. If not, see - * http://www.opensource.org/licenses/BSD-3-Clause for more information. - */ -#ifndef MLPACK_METHODS_ANN_LAYER_LAMBDA_MAP_REDUCE_IMPL_HPP -#define MLPACK_METHODS_ANN_LAYER_LAMBDA_MAP_REDUCE_IMPL_HPP - -#include "lambda_map_reduce.hpp" - -namespace mlpack { -namespace ann { - -template -LambdaMapReduceType::LambdaMapReduceType() : - Layer(), - inSize(0), - totalInputSize(0), - totalOutputSize(0) -{ - // Nothing to do. -} - -template -LambdaMapReduceType::LambdaMapReduceType(const LambdaMapReduceType& other) : - Layer(other), - inSize(other.inSize), - totalInputSize(other.totalInputSize), - totalOutputSize(other.totalOutputSize), - reductionRule(other.reductionRule), - layerOutputMatrix(other.layerOutputMatrix), - layerDeltaMatrix(other.layerDeltaMatrix) -{ - // Copy each layer. - for (size_t i = 0; i < other.network.size(); ++i) - network.push_back(other.network[i]->Clone()); - - // Ensure that the aliases for layers during passes have the right size. - layerOutputs.resize(network.size(), MatType()); - layerDeltas.resize(network.size(), MatType()); - layerGradients.resize(network.size(), MatType()); - - // layerOutputs, layerDeltas, and layerGradients will be reset the next time - // Forward(), Backward(), or Gradient() is called. -} - -template -LambdaMapReduceType::LambdaMapReduceType(LambdaMapReduceType&& other) : - Layer(other), - network(std::move(other.network)), - inSize(std::move(other.inSize)), - totalInputSize(std::move(other.totalInputSize)), - totalOutputSize(std::move(other.totalOutputSize)), - reductionRule(std::move(other.reductionRule)), - layerOutputMatrix(std::move(other.layerOutputMatrix)), - layerDeltaMatrix(std::move(other.layerDeltaMatrix)) -{ - // Ensure that the aliases for layers during passes have the right size. - layerOutputs.resize(network.size(), MatType()); - layerDeltas.resize(network.size(), MatType()); - layerGradients.resize(network.size(), MatType()); - - // layerOutputs, layerDeltas, and layerGradients will be reset the next time - // Forward(), Backward(), or Gradient() is called. - - other.layerOutputs.clear(); - other.layerDeltas.clear(); - other.layerGradients.clear(); -} - -template -LambdaMapReduceType& LambdaMapReduceType::operator=(const LambdaMapReduceType& other) -{ - if (this != &other) - { - Layer::operator=(other); - - network.clear(); - layerOutputs.clear(); - layerDeltas.clear(); - layerGradients.clear(); - - inSize = other.inSize; - totalInputSize = other.totalInputSize; - totalOutputSize = other.totalOutputSize; - reductionRule = other.reductionRule; - - layerOutputMatrix = other.layerOutputMatrix; - layerDeltaMatrix = other.layerDeltaMatrix; - - for (size_t i = 0; i < other.network.size(); ++i) - network.push_back(other.network[i]->Clone()); - - // Ensure that the aliases for layers during passes have the right size. - layerOutputs.resize(network.size(), MatType()); - layerDeltas.resize(network.size(), MatType()); - layerGradients.resize(network.size(), MatType()); - } - - return *this; -} - -template -LambdaMapReduceType& LambdaMapReduceType::operator=(LambdaMapReduceType&& other) -{ - if (this != &other) - { - Layer::operator=(other); - - layerOutputs.clear(); - layerDeltas.clear(); - layerGradients.clear(); - - inSize = std::move(other.inSize); - totalInputSize = std::move(other.totalInputSize); - totalOutputSize = std::move(other.totalOutputSize); - reductionRule = std::move(other.reductionRule); - - network = std::move(other.network); - - layerOutputs.resize(network.size(), MatType()); - layerDeltas.resize(network.size(), MatType()); - layerGradients.resize(network.size(), MatType()); - - other.layerOutputs.clear(); - other.layerDeltas.clear(); - other.layerGradients.clear(); - } - - return *this; -} - -template -void LambdaMapReduceType::Forward( - const MatType& input, MatType& output) -{ - // Make sure training/testing mode is set right in each layer. - for (size_t i = 0; i < network.size(); ++i) - network[i]->Training() = this->training; - - // Note that we use `output` for the last layer; layerOutputs is only used for - // intermediate values between layers. - if (network.size() > 1) - { - // Initialize memory for the forward pass (if needed). - InitializeForwardPassMemory(input.n_cols); - - for (size_t i = 0; i < network.size(); i++) - network[i]->Forward(input, layerOutputs[i]); - - // Reduce the outputs to single output. - reductionRule.Reduce(layerOutputs, output); - } - else if (network.size() == 1) - { - network[0]->Forward(input, output); - } - else - { - // Empty network? - output = input; - } -} - -template -void LambdaMapReduceType::Backward( - const MatType& input, const MatType& gy, MatType& g) -{ - if (network.size() > 1) - { - // Initialize memory for the backward pass (if needed). - InitializeBackwardPassMemory(input.n_cols); - - std::vector layerTempDeltas; - // Compute the gy for all layers. - reductionRule.UnReduce(gy, network.size(), layerTempDeltas); - - g.zeros(); - for (size_t i = 0; i < network.size(); i++) { - network[i]->Backward(layerOutputs[i], layerTempDeltas[i], layerDeltas[i]); - g += layerDeltas[i]; - } - } - else if (network.size() == 1) - { - network[0]->Backward(input, gy, g); - } - else - { - // Empty network? - g = gy; - } -} - -template -void LambdaMapReduceType::Gradient( - const MatType& input, const MatType& error, MatType& gradient) -{ - // We assume gradient has the right size already. - - // Pass gradients through each layer. - if (network.size() > 1) - { - // Initialize memory for the gradient pass (if needed). - InitializeGradientPassMemory(gradient); - - std::vector layerTempDeltas; - // Compute the error for all layers. - reductionRule.UnReduce(error, network.size(), layerTempDeltas); - - for (size_t i = 0; i < network.size(); ++i) - { - network[i]->Gradient(input, layerTempDeltas[i], - layerGradients[i]); - } - } - else if (network.size() == 1) - { - network[0]->Gradient(input, error, gradient); - } - else - { - // Nothing to do if the network is empty... there is no gradient. - } -} - -template -void LambdaMapReduceType::SetWeights(typename MatType::elem_type* weightsPtr) -{ - size_t start = 0; - const size_t totalWeightSize = WeightSize(); - for (size_t i = 0; i < network.size(); ++i) - { - const size_t weightSize = network[i]->WeightSize(); - - // Sanity check: ensure we aren't passing memory past the end of the - // parameters. - Log::Assert(start + weightSize <= totalWeightSize, - "FNN::SetLayerMemory(): parameter size does not match total layer " - "weight size!"); - - network[i]->SetWeights(weightsPtr + start); - start += weightSize; - } - - // Technically this check should be unnecessary, but there's nothing wrong - // with a little paranoia... - Log::Assert(start == totalWeightSize, - "FNN::SetLayerMemory(): total layer weight size does not match parameter " - "size!"); - - MakeAlias(weights, weightsPtr, totalWeightSize, 1); -} - -template -size_t LambdaMapReduceType::WeightSize() const -{ - // Sum the weights in each layer. - size_t total = 0; - for (size_t i = 0; i < network.size(); ++i) - total += network[i]->WeightSize(); - return total; -} - -template -void LambdaMapReduceType::ComputeOutputDimensions() -{ - inSize = 0; - totalInputSize = 0; - totalOutputSize = 0; - - // Propagate the input dimensions forward to the output. - if (network.size() == 0) - { - this->outputDimensions = this->inputDimensions; - return; - } - inSize = this->inputDimensions[0]; - for (size_t i = 1; i < this->inputDimensions.size(); ++i) - inSize *= this->inputDimensions[i]; - totalInputSize = network.size() * inSize; - - for (size_t i = 0; i < network.size(); ++i) - { - network[i]->InputDimensions() = this->inputDimensions; - size_t layerOutputSize = network[i]->OutputSize(); - totalOutputSize += layerOutputSize; - } - - // Compute the output size of the network using reduction rules. - this->outputDimensions = reductionRule.ReduceSize(network); -} - -template -double LambdaMapReduceType::Loss() const -{ - double loss = 0.0; - for (size_t i = 0; i < network.size(); ++i) - loss += network[i]->Loss(); - - return loss; -} - -template -template -void LambdaMapReduceType::serialize( - Archive& ar, const uint32_t /* version */) -{ - ar(cereal::base_class>(this)); - - ar(CEREAL_VECTOR_POINTER(network)); - ar(CEREAL_NVP(inSize)); - ar(CEREAL_NVP(totalInputSize)); - ar(CEREAL_NVP(totalOutputSize)); - - if (Archive::is_loading::value) - { - layerOutputMatrix.clear(); - layerDeltaMatrix.clear(); - layerGradients.clear(); - layerOutputs.resize(network.size(), MatType()); - layerDeltas.resize(network.size(), MatType()); - layerGradients.resize(network.size(), MatType()); - } -} - -template -void LambdaMapReduceType::InitializeForwardPassMemory(const size_t batchSize) -{ - // We need to initialize memory to store the output of each layer's Forward() - // call. We'll do this all in one matrix, but, the size of this matrix - // depends on the batch size we are using for computation. We avoid resizing - // layerOutputMatrix down, unless we only need 10% or less of it. - if (batchSize * totalOutputSize > layerOutputMatrix.n_elem || - batchSize * totalOutputSize < - std::floor(0.1 * layerOutputMatrix.n_elem)) - { - // All outputs will be represented by one big block of memory. - layerOutputMatrix = MatType(1, batchSize * totalOutputSize); - } - - // Now, create an alias to the right place for each layer. We assume that - // layerOutputs is already sized correctly (this should be done by Add()). - size_t start = 0; - for (size_t i = 0; i < layerOutputs.size(); ++i) - { - const size_t layerOutputSize = network[i]->OutputSize(); - MakeAlias(layerOutputs[i], layerOutputMatrix.colptr(start), - layerOutputSize, batchSize); - start += batchSize * layerOutputSize; - } -} - -template -void LambdaMapReduceType::InitializeBackwardPassMemory( - const size_t batchSize) -{ - // We need to initialize memory to store the output of each layer's Backward() - // call. We do this similarly to InitializeForwardPassMemory(), but we must - // store a matrix to use as the delta for each layer. - if (batchSize * totalInputSize > layerDeltaMatrix.n_elem || - batchSize * totalInputSize < std::floor(0.1 * layerDeltaMatrix.n_elem)) - { - // All deltas will be represented by one big block of memory. - layerDeltaMatrix = MatType(1, batchSize * totalInputSize); - } - - // Now, create an alias to the right place for each layer. We assume that - // layerDeltas is already sized correctly (this should be done by Add()). - size_t start = 0; - for (size_t i = 0; i < layerDeltas.size(); ++i) - { - size_t layerInputSize = inSize; - MakeAlias(layerDeltas[i], layerDeltaMatrix.colptr(start), layerInputSize, - batchSize); - start += batchSize * layerInputSize; - } -} - -template -void LambdaMapReduceType::InitializeGradientPassMemory(MatType& gradient) -{ - // We need to initialize memory to store the gradients of each layer. To do - // this, we need to know the weight size of each layer. - size_t gradientStart = 0; - for (size_t i = 0; i < network.size(); ++i) - { - const size_t weightSize = network[i]->WeightSize(); - MakeAlias(layerGradients[i], gradient.memptr() + gradientStart, - weightSize, 1); - gradientStart += weightSize; - } -} - -} // namespace ann -} // namespace mlpack - -#endif diff --git a/src/mlpack/methods/ann/layer/layer_types.hpp b/src/mlpack/methods/ann/layer/layer_types.hpp index 0d244572d1..e5166c73d1 100644 --- a/src/mlpack/methods/ann/layer/layer_types.hpp +++ b/src/mlpack/methods/ann/layer/layer_types.hpp @@ -26,7 +26,6 @@ #include #include #include -#include #include #include #include @@ -38,6 +37,7 @@ #include #include #include +#include #include // Convolution modes. @@ -51,9 +51,6 @@ // Loss function modules. #include -// Include Reduction Rules. -#include - // Include definitions for polymorphic serialization. #include diff --git a/src/mlpack/methods/ann/layer/multi_layer_impl.hpp b/src/mlpack/methods/ann/layer/multi_layer_impl.hpp index 39fd13d222..3565213dc0 100644 --- a/src/mlpack/methods/ann/layer/multi_layer_impl.hpp +++ b/src/mlpack/methods/ann/layer/multi_layer_impl.hpp @@ -379,15 +379,9 @@ void MultiLayer::InitializeBackwardPassMemory( for (size_t i = 0; i < layerDeltas.size(); ++i) { size_t layerInputSize = 1; - if (i == 0) - { - for (size_t j = 0; j < this->inputDimensions.size(); ++j) - layerInputSize *= this->inputDimensions[j]; - } - else - { - layerInputSize = network[i - 1]->OutputSize(); - } + for (size_t j = 0; j < network[i]->InputDimensions().size(); ++j) + layerInputSize *= network[i]->InputDimensions()[j]; + MakeAlias(layerDeltas[i], layerDeltaMatrix.colptr(start), layerInputSize, batchSize); start += batchSize * layerInputSize; diff --git a/src/mlpack/methods/ann/layer/residual.hpp b/src/mlpack/methods/ann/layer/residual.hpp new file mode 100644 index 0000000000..86015df80b --- /dev/null +++ b/src/mlpack/methods/ann/layer/residual.hpp @@ -0,0 +1,127 @@ +/** + * @file methods/ann/layer/residual.hpp + * @author Shubham Agrawal + * + * Base class for neural network layers that are wrappers around other layers. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_METHODS_ANN_LAYER_RESIDUAL_HPP +#define MLPACK_METHODS_ANN_LAYER_RESIDUAL_HPP + +#include "../make_alias.hpp" +#include "multi_layer.hpp" + +namespace mlpack { +namespace ann { + +/** + * A lambda "map-reduce" is a layer that is a wrapper around other layers. + * It passes the input through all of its child layers sequentially, returning + * the output from reducing the output. + * + * @tparam MatType Matrix representation to accept as input and use for + * computation. + */ +template +class ResidualType : public MultiLayer +{ + public: + /** + * Create an empty ResidualType that holds no layers of its own. Be sure to add + * layers with Add() before using! + */ + ResidualType(); + + //! Copy the given ResidualType. + ResidualType(const ResidualType& other); + //! Take ownership of the layers of the given ResidualType. + ResidualType(ResidualType&& other); + //! Copy the given ResidualType. + ResidualType& operator=(const ResidualType& other); + //! Take ownership of the given ResidualType. + ResidualType& operator=(ResidualType&& other); + + //! Virtual destructor: delete all held layers. + virtual ~ResidualType() + { + // Nothing to do here. + } + + //! Create a copy of the ResidualType (this is safe for polymorphic use). + virtual ResidualType* Clone() const { return new ResidualType(*this); } + + /** + * Perform a forward pass with the given input data. `output` is expected to + * have the correct size (e.g. number of rows equal to `OutputSize()` of the + * last held layer; number of columns equal to `input.n_cols`). + * + * @param input Input data to pass through the ResidualType. + * @param output Matrix to store output in. + */ + virtual void Forward(const MatType& input, MatType& output); + + /** + * Perform a backward pass with the given data. `gy` is expected to be the + * propagated error from the subsequent layer (or output), `input` is expected + * to be the output from this layer when `Forward()` was called, and `g` will + * store the propagated error from this layer (to be passed to the previous + * layer as `gy`). + * + * It is expected that `g` has the correct size already (e.g., number of rows + * equal to `OutputSize()` of the previous layer, and number of columns equal + * to `input.n_cols`). + * + * This function is expected to be called for the same input data as + * `Forward()` was just called for. + * + * @param input Output of Forward(). + * @param gy Propagated error from next layer. + * @param g Matrix to store propagated error in for previous layer. + */ + virtual void Backward(const MatType& input, + const MatType& gy, + MatType& g); + + /** + * Compute the gradients of each layer. + * + * This function is expected to be called for the same input data as + * `Forward()` and `Backward()` were just called for. That is, `input` here + * should be the same data as `Forward()` was called with. + * + * `gradient` is expected to have the correct size already (e.g., number of + * rows equal to 1, and number of columns equal to `WeightSize()`). + * + * @param input Original input data provided to Forward(). + * @param error Error as computed by `Backward()`. + * @param gradient Matrix to store the gradients in. + */ + virtual void Gradient(const MatType& input, + const MatType& error, + MatType& gradient); + + /** + * Compute the output dimensions of the ResidualType using `InputDimensions()`. + * This computes the dimensions of each layer held by the ResidualType, and the + * output dimensions are set to the output dimensions of the last layer. + */ + virtual void ComputeOutputDimensions(); + + //! Serialize the ResidualType. + template + void serialize(Archive& ar, const uint32_t /* version */); +}; + +typedef ResidualType Residual; + +} // namespace ann +} // namespace mlpack + +// Include implementation. +#include "residual_impl.hpp" + +#endif diff --git a/src/mlpack/methods/ann/layer/residual_impl.hpp b/src/mlpack/methods/ann/layer/residual_impl.hpp new file mode 100644 index 0000000000..bd0a726e67 --- /dev/null +++ b/src/mlpack/methods/ann/layer/residual_impl.hpp @@ -0,0 +1,209 @@ +/** + * @file methods/ann/layer/residual_impl.hpp + * @author Shubham Agrawal + * + * Implementation of the base class for neural network layers that are wrappers + * around other layers. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_METHODS_ANN_LAYER_RESIDUAL_IMPL_HPP +#define MLPACK_METHODS_ANN_LAYER_RESIDUAL_IMPL_HPP + +#include "residual.hpp" + +namespace mlpack { +namespace ann { + +template +ResidualType::ResidualType() : + MultiLayer() +{ + // Nothing to do. +} + +template +ResidualType::ResidualType(const ResidualType& other) : + MultiLayer(other) +{ + // Nothing to do here. +} + +template +ResidualType::ResidualType(ResidualType&& other) : + MultiLayer(other) +{ + // Nothing to do here. +} + +template +ResidualType& ResidualType::operator=(const ResidualType& other) +{ + if (this != &other) + { + MultiLayer::operator=(other); + } + + return *this; +} + +template +ResidualType& ResidualType::operator=(ResidualType&& other) +{ + if (this != &other) + { + MultiLayer::operator=(other); + } + + return *this; +} + +template +void ResidualType::Forward( + const MatType& input, MatType& output) +{ + // Make sure training/testing mode is set right in each layer. + for (size_t i = 0; i < this->network.size(); ++i) + this->network[i]->Training() = this->training; + + // Note that we use `output` for the last layer; layerOutputs is only used for + // intermediate values between layers. + if (this->network.size() > 1) + { + // Initialize memory for the forward pass (if needed). + this->InitializeForwardPassMemory(input.n_cols); + + for (size_t i = 0; i < this->network.size(); i++) + this->network[i]->Forward(input, this->layerOutputs[i]); + + // Reduce the outputs to single output. + output.zeros(); + for (size_t i = 0; i < this->layerOutputs.size(); i++) + { + output += this->layerOutputs[i]; + } + } + else if (this->network.size() == 1) + { + this->network[0]->Forward(input, output); + } + else + { + // Empty network? + output = input; + } +} + +template +void ResidualType::Backward( + const MatType& input, const MatType& gy, MatType& g) +{ + if (this->network.size() > 1) + { + // Initialize memory for the backward pass (if needed). + this->InitializeBackwardPassMemory(input.n_cols); + + g.zeros(); + for (size_t i = 0; i < this->network.size(); i++) { + this->network[i]->Backward(this->layerOutputs[i], gy, this->layerDeltas[i]); + g += this->layerDeltas[i]; + } + } + else if (this->network.size() == 1) + { + this->network[0]->Backward(input, gy, g); + } + else + { + // Empty network? + g = gy; + } +} + +template +void ResidualType::Gradient( + const MatType& input, const MatType& error, MatType& gradient) +{ + // We assume gradient has the right size already. + + // Pass gradients through each layer. + if (this->network.size() > 1) + { + // Initialize memory for the gradient pass (if needed). + this->InitializeGradientPassMemory(gradient); + + for (size_t i = 0; i < this->network.size(); ++i) + { + this->network[i]->Gradient(input, error, this->layerGradients[i]); + } + } + else if (this->network.size() == 1) + { + this->network[0]->Gradient(input, error, gradient); + } + else + { + // Nothing to do if the network is empty... there is no gradient. + } +} + +template +void ResidualType::ComputeOutputDimensions() +{ + this->inSize = 0; + this->totalInputSize = 0; + this->totalOutputSize = 0; + + // Propagate the input dimensions forward to the output. + if (this->network.size() == 0) + { + this->outputDimensions = this->inputDimensions; + return; + } + this->inSize = this->inputDimensions[0]; + for (size_t i = 1; i < this->inputDimensions.size(); ++i) + this->inSize *= this->inputDimensions[i]; + this->totalInputSize = this->network.size() * this->inSize; + + for (size_t i = 0; i < this->network.size(); ++i) + { + this->network[i]->InputDimensions() = this->inputDimensions; + size_t layerOutputSize = this->network[i]->OutputSize(); + this->totalOutputSize += layerOutputSize; + } + + // Compute the output size of the network using reduction rules. + if (this->network.size() == 1) + { + this->outputDimensions = this->network[0]->OutputDimensions(); + return; + } + const std::vector networkSize = this->network[0]->OutputDimensions(); + for (size_t i = 1; i < this->network.size(); i++) + { + if (!(networkSize == this->network[i]->OutputDimensions())) + { + Log::Fatal << "Network size mismatch. (" << networkSize[0] << ", " + << networkSize[1] << ") != (" + << this->network[i]->OutputDimensions()[0] << ", " << this->network[i]->OutputDimensions()[1] + << ")." << std::endl; + } + } + this->outputDimensions = networkSize; +} + +template +template +void ResidualType::serialize( + Archive& ar, const uint32_t /* version */) +{ + ar(cereal::base_class>(this)); +} + +} // namespace ann +} // namespace mlpack + +#endif diff --git a/src/mlpack/methods/ann/layer/serialization.hpp b/src/mlpack/methods/ann/layer/serialization.hpp index d201933664..c774c8ee51 100644 --- a/src/mlpack/methods/ann/layer/serialization.hpp +++ b/src/mlpack/methods/ann/layer/serialization.hpp @@ -39,8 +39,6 @@ __VA_ARGS__>); \ CEREAL_REGISTER_TYPE(mlpack::ann::DropConnectType<__VA_ARGS__>); \ CEREAL_REGISTER_TYPE(mlpack::ann::DropoutType<__VA_ARGS__>); \ - CEREAL_REGISTER_TYPE(mlpack::ann::LambdaMapReduceType< \ - mlpack::ann::AddReduction, __VA_ARGS__>); \ CEREAL_REGISTER_TYPE(mlpack::ann::LeakyReLUType<__VA_ARGS__>); \ CEREAL_REGISTER_TYPE(mlpack::ann::Linear3DType<__VA_ARGS__>); \ CEREAL_REGISTER_TYPE(mlpack::ann::LinearType<__VA_ARGS__>); \ @@ -52,6 +50,7 @@ CEREAL_REGISTER_TYPE(mlpack::ann::NoisyLinearType<__VA_ARGS__>); \ CEREAL_REGISTER_TYPE(mlpack::ann::PaddingType<__VA_ARGS__>); \ CEREAL_REGISTER_TYPE(mlpack::ann::RBFType<__VA_ARGS__>); \ + CEREAL_REGISTER_TYPE(mlpack::ann::ResidualType<__VA_ARGS__>); \ CEREAL_REGISTER_TYPE(mlpack::ann::SoftmaxType<__VA_ARGS__>); \ CEREAL_REGISTER_MLPACK_LAYERS(arma::mat); diff --git a/src/mlpack/tests/ann_layer_test.cpp b/src/mlpack/tests/ann_layer_test.cpp index 48ce8c9f14..45cd89c24c 100644 --- a/src/mlpack/tests/ann_layer_test.cpp +++ b/src/mlpack/tests/ann_layer_test.cpp @@ -4854,9 +4854,9 @@ TEST_CASE("TransposedConvolutionLayerPaddingTest", "[ANNLayerTest]") // } /** - * Simple test for Lambda Map Reduce layer. + * Simple test for Residual layer. */ -TEST_CASE("LambdaMapReduceTestCase", "[ANNLayerTest]") +TEST_CASE("ResidualTestCase", "[ANNLayerTest]") { // For rectangular input to pooling layers. arma::mat input = arma::mat(28, 1); @@ -4871,11 +4871,11 @@ TEST_CASE("LambdaMapReduceTestCase", "[ANNLayerTest]") input(14) = input(25) = 8; input(15) = input(26) = 9; - LambdaMapReduce module1; + Residual module1; module1.Add(2, 2, 2, 2, false); module1.Add(2, 2, 2, 2, false); - LambdaMapReduce module2; + Residual module2; module2.Add(2, 2, 2, 2, true); module2.Add(2, 2, 2, 2, true); @@ -4907,23 +4907,23 @@ TEST_CASE("LambdaMapReduceTestCase", "[ANNLayerTest]") CheckMatrices(output1, result1, 1e-1); CheckMatrices(output2, result2, 1e-1); - arma::mat prev_delta1, prev_delta2; - prev_delta1 << 3.6000 << -0.9000 << arma::endr + arma::mat prevDelta1, prevDelta2; + prevDelta1 << 3.6000 << -0.9000 << arma::endr << 3.6000 << -0.9000 << arma::endr << 3.6000 << -0.9000 << arma::endr << 3.6000 << -0.9000 << arma::endr; - prev_delta2 << 3.6000 << -0.9000 << arma::endr + prevDelta2 << 3.6000 << -0.9000 << arma::endr << 3.6000 << -0.9000 << arma::endr << 3.6000 << -0.9000 << arma::endr; arma::mat delta1, delta2; delta1.set_size(28, 1); delta2.set_size(28, 1); - prev_delta1.reshape(8, 1); - prev_delta2.reshape(6, 1); - module1.Backward(input, prev_delta1, delta1); + prevDelta1.reshape(8, 1); + prevDelta2.reshape(6, 1); + module1.Backward(input, prevDelta1, delta1); REQUIRE(arma::accu(delta1) == Approx(21.6).epsilon(1e-3)); - module2.Backward(input, prev_delta2, delta2); + module2.Backward(input, prevDelta2, delta2); REQUIRE(arma::accu(delta2) == Approx(16.2).epsilon(1e-3)); }