apaam/mlpack
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

changed to residual layer

Browse Source
This commit is contained in:
Shubham Agrawal
2022-06-13 20:43:21 +05:30
parent dd24cc652f
commit ee52187afd
10 changed files with 355 additions and 688 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/mlpack/methods/ann/CMakeLists.txt
+1 -1
View File
@@ -14,7 +14,7 @@ add_subdirectory(layer)
add_subdirectory(loss_functions)
add_subdirectory(convolution_rules)
add_subdirectory(regularizer)
add_subdirectory(reduction_rules)
# add_subdirectory(reduction_rules)
# Add directory name to sources.
set(DIR_SRCS)
src/mlpack/methods/ann/layer/CMakeLists.txt
+2 -2
View File
@@ -18,8 +18,6 @@ set(SOURCES
dropconnect_impl.hpp
dropout.hpp
dropout_impl.hpp
lambda_map_reduce.hpp
lambda_map_reduce_impl.hpp
layer.hpp
layer_types.hpp
leaky_relu.hpp
@@ -45,6 +43,8 @@ set(SOURCES
padding.hpp
radial_basis_function.hpp
radial_basis_function_impl.hpp
residual.hpp
residual_impl.hpp
serialization.hpp
)
src/mlpack/methods/ann/layer/lambda_map_reduce.hpp
-254
View File
@@ -1,254 +0,0 @@
/**
* @file methods/ann/layer/lambda_map_reduce.hpp
* @author Shubham Agrawal
*
* Base class for neural network layers that are wrappers around other layers.
*
* mlpack is free software; you may redistribute it and/or modify it under the
* terms of the 3-clause BSD license. You should have received a copy of the
* 3-clause BSD license along with mlpack. If not, see
* http://www.opensource.org/licenses/BSD-3-Clause for more information.
*/
#ifndef MLPACK_METHODS_ANN_LAYER_LAMBDA_MAP_REDUCE_HPP
#define MLPACK_METHODS_ANN_LAYER_LAMBDA_MAP_REDUCE_HPP
#include "../make_alias.hpp"
#include "layer.hpp"
#include <mlpack/methods/ann/reduction_rules/add_reduction.hpp>
namespace mlpack {
namespace ann {
/**
* A lambda "map-reduce" is a layer that is a wrapper around other layers.
* It passes the input through all of its child layers sequentially, returning
* the output from reducing the output.
*
* It's likely not very useful to use this layer directly; instead, this layer
* is meant as a base class for use by other layers that must store and use
* multiple layers.
*
* @tparam ReductionRuleType Reduction rule to use for the layer.
* @tparam MatType Matrix representation to accept as input and use for
* computation.
*/
template<typename ReductionRuleType, typename MatType>
class LambdaMapReduceType : public Layer<MatType>
{
public:
/**
* Create an empty LambdaMapReduceType that holds no layers of its own. Be sure to add
* layers with Add() before using!
*/
LambdaMapReduceType();
//! Copy the given LambdaMapReduceType.
LambdaMapReduceType(const LambdaMapReduceType& other);
//! Take ownership of the layers of the given LambdaMapReduceType.
LambdaMapReduceType(LambdaMapReduceType&& other);
//! Copy the given LambdaMapReduceType.
LambdaMapReduceType& operator=(const LambdaMapReduceType& other);
//! Take ownership of the given LambdaMapReduceType.
LambdaMapReduceType& operator=(LambdaMapReduceType&& other);
//! Virtual destructor: delete all held layers.
virtual ~LambdaMapReduceType()
{
for (size_t i = 0; i < network.size(); ++i)
delete network[i];
}
//! Create a copy of the LambdaMapReduceType (this is safe for polymorphic use).
virtual LambdaMapReduceType* Clone() const { return new LambdaMapReduceType(*this); }
/**
* Perform a forward pass with the given input data. `output` is expected to
* have the correct size (e.g. number of rows equal to `OutputSize()` of the
* last held layer; number of columns equal to `input.n_cols`).
*
* @param input Input data to pass through the LambdaMapReduceType.
* @param output Matrix to store output in.
*/
virtual void Forward(const MatType& input, MatType& output);
/**
* Perform a backward pass with the given data. `gy` is expected to be the
* propagated error from the subsequent layer (or output), `input` is expected
* to be the output from this layer when `Forward()` was called, and `g` will
* store the propagated error from this layer (to be passed to the previous
* layer as `gy`).
*
* It is expected that `g` has the correct size already (e.g., number of rows
* equal to `OutputSize()` of the previous layer, and number of columns equal
* to `input.n_cols`).
*
* This function is expected to be called for the same input data as
* `Forward()` was just called for.
*
* @param input Output of Forward().
* @param gy Propagated error from next layer.
* @param g Matrix to store propagated error in for previous layer.
*/
virtual void Backward(const MatType& input,
const MatType& gy,
MatType& g);
/**
* Compute the gradients of each layer.
*
* This function is expected to be called for the same input data as
* `Forward()` and `Backward()` were just called for. That is, `input` here
* should be the same data as `Forward()` was called with.
*
* `gradient` is expected to have the correct size already (e.g., number of
* rows equal to 1, and number of columns equal to `WeightSize()`).
*
* @param input Original input data provided to Forward().
* @param error Error as computed by `Backward()`.
* @param gradient Matrix to store the gradients in.
*/
virtual void Gradient(const MatType& input,
const MatType& error,
MatType& gradient);
/**
* Set the weights of the layer to use the memory given as `weightsPtr`.
*/
virtual void SetWeights(typename MatType::elem_type* weightsPtr);
/**
* Return the number of weights in the LambdaMapReduceType. This is the sum of the
* number of weights in each layer.
*/
virtual size_t WeightSize() const;
/**
* Compute the output dimensions of the LambdaMapReduceType using `InputDimensions()`.
* This computes the dimensions of each layer held by the LambdaMapReduceType, and the
* output dimensions are set to the output dimensions of the last layer.
*/
virtual void ComputeOutputDimensions();
/**
* Compute the loss that should be added to the objective.
*/
virtual double Loss() const;
/**
* Add a new module to the model.
*
* @param args The layer parameter.
*/
template <typename LayerType, typename... Args>
void Add(Args... args)
{
network.push_back(new LayerType(args...));
layerOutputs.push_back(MatType());
layerDeltas.push_back(MatType());
layerGradients.push_back(MatType());
}
/**
* Add a new module to the model.
*
* @param layer The Layer to be added to the model.
*/
void Add(Layer<MatType>* layer)
{
network.push_back(layer);
layerOutputs.push_back(MatType());
layerDeltas.push_back(MatType());
layerGradients.push_back(MatType());
}
//! Get the network (series of layers) held by this LambdaMapReduceType.
const std::vector<Layer<MatType>*> Network() const
{
return network;
}
//! Modify the network (series of layers) held by this LambdaMapReduceType. Be
//! careful!
std::vector<Layer<MatType>*>& Network() { return network; }
//! Get the parameters.
MatType const& Parameters() const { return weights; }
//! Modify the parameters.
MatType& Parameters() { return weights; }
//! Serialize the LambdaMapReduceType.
template<typename Archive>
void serialize(Archive& ar, const uint32_t /* version */);
protected:
/**
* Initialize memory that will be used by each layer for the forward pass,
* assuming that the input will have the given `batchSize`. When `Forward()`
* is called, each internally-held layer will output its results into the
* memory allocated by this function (this is the internal member
* `layerOutputMatrix` and its aliases `layerOutputs`).
*/
void InitializeForwardPassMemory(const size_t batchSize);
/**
* Initialize memory that will be used by each layer for the backwards pass,
* assuming that the input will have the given `batchSize`. When `Backward()`
* is called, each internally-held layer will output the results of its
* backwards pass into the memory allocated by this function (this is the
* internal member `layerDeltaMatrix` and its aliases `layerDeltas`).
*/
void InitializeBackwardPassMemory(const size_t batchSize);
/**
* Initialize memory for the gradient pass. This sets the internal aliases
* `layerGradients` appropriately using the memory from the given `gradient`,
* such that each layer will output its gradient (via its `Gradient()` method)
* into the appropriate member of `layerGradients`.
*/
void InitializeGradientPassMemory(MatType& gradient);
//! The internally-held network.
std::vector<Layer<MatType>*> network;
// Total number of elements in the input, cached for convenience.
size_t inSize;
// Total number of input elements for *every* layer.
size_t totalInputSize;
// Total number of output elements for *every* layer.
size_t totalOutputSize;
//! Redction rule for the output of each layer.
ReductionRuleType reductionRule;
//! Locally-stored weight object.
MatType weights;
//! This matrix stores all of the outputs of each layer when Forward() is
//! called. See `InitializeForwardPassMemory()`.
MatType layerOutputMatrix;
//! These are aliases of `layerOutputMatrix` for each layer.
std::vector<MatType> layerOutputs;
//! This matrix stores all of the backwards pass results of each layer when
//! Backward() is called. See `InitializeBackwardPassMemory()`.
MatType layerDeltaMatrix;
//! These are aliases of `layerDeltaMatrix` for each layer.
std::vector<MatType> layerDeltas;
//! Gradient aliases for each layer. Note that this is *only* valid in the
//! context of `Gradient()`! We have it as a class member to avoid
//! reallocating the `MatType`s each call to `Gradient()`.
std::vector<MatType> layerGradients;
};
typedef LambdaMapReduceType<
AddReduction,
arma::mat
> LambdaMapReduce;
} // namespace ann
} // namespace mlpack
// Include implementation.
#include "lambda_map_reduce_impl.hpp"
#endif
src/mlpack/methods/ann/layer/lambda_map_reduce_impl.hpp
-405
View File
@@ -1,405 +0,0 @@
/**
* @file methods/ann/layer/lambda_map_reduce_impl.hpp
* @author Shubham Agrawal
*
* Implementation of the base class for neural network layers that are wrappers
* around other layers.
*
* mlpack is free software; you may redistribute it and/or modify it under the
* terms of the 3-clause BSD license. You should have received a copy of the
* 3-clause BSD license along with mlpack. If not, see
* http://www.opensource.org/licenses/BSD-3-Clause for more information.
*/
#ifndef MLPACK_METHODS_ANN_LAYER_LAMBDA_MAP_REDUCE_IMPL_HPP
#define MLPACK_METHODS_ANN_LAYER_LAMBDA_MAP_REDUCE_IMPL_HPP
#include "lambda_map_reduce.hpp"
namespace mlpack {
namespace ann {
template<typename ReductionRuleType, typename MatType>
LambdaMapReduceType<ReductionRuleType, MatType>::LambdaMapReduceType() :
Layer<MatType>(),
inSize(0),
totalInputSize(0),
totalOutputSize(0)
{
// Nothing to do.
}
template<typename ReductionRuleType, typename MatType>
LambdaMapReduceType<ReductionRuleType, MatType>::LambdaMapReduceType(const LambdaMapReduceType& other) :
Layer<MatType>(other),
inSize(other.inSize),
totalInputSize(other.totalInputSize),
totalOutputSize(other.totalOutputSize),
reductionRule(other.reductionRule),
layerOutputMatrix(other.layerOutputMatrix),
layerDeltaMatrix(other.layerDeltaMatrix)
{
// Copy each layer.
for (size_t i = 0; i < other.network.size(); ++i)
network.push_back(other.network[i]->Clone());
// Ensure that the aliases for layers during passes have the right size.
layerOutputs.resize(network.size(), MatType());
layerDeltas.resize(network.size(), MatType());
layerGradients.resize(network.size(), MatType());
// layerOutputs, layerDeltas, and layerGradients will be reset the next time
// Forward(), Backward(), or Gradient() is called.
}
template<typename ReductionRuleType, typename MatType>
LambdaMapReduceType<ReductionRuleType, MatType>::LambdaMapReduceType(LambdaMapReduceType&& other) :
Layer<MatType>(other),
network(std::move(other.network)),
inSize(std::move(other.inSize)),
totalInputSize(std::move(other.totalInputSize)),
totalOutputSize(std::move(other.totalOutputSize)),
reductionRule(std::move(other.reductionRule)),
layerOutputMatrix(std::move(other.layerOutputMatrix)),
layerDeltaMatrix(std::move(other.layerDeltaMatrix))
{
// Ensure that the aliases for layers during passes have the right size.
layerOutputs.resize(network.size(), MatType());
layerDeltas.resize(network.size(), MatType());
layerGradients.resize(network.size(), MatType());
// layerOutputs, layerDeltas, and layerGradients will be reset the next time
// Forward(), Backward(), or Gradient() is called.
other.layerOutputs.clear();
other.layerDeltas.clear();
other.layerGradients.clear();
}
template<typename ReductionRuleType, typename MatType>
LambdaMapReduceType<ReductionRuleType, MatType>& LambdaMapReduceType<ReductionRuleType, MatType>::operator=(const LambdaMapReduceType& other)
{
if (this != &other)
{
Layer<MatType>::operator=(other);
network.clear();
layerOutputs.clear();
layerDeltas.clear();
layerGradients.clear();
inSize = other.inSize;
totalInputSize = other.totalInputSize;
totalOutputSize = other.totalOutputSize;
reductionRule = other.reductionRule;
layerOutputMatrix = other.layerOutputMatrix;
layerDeltaMatrix = other.layerDeltaMatrix;
for (size_t i = 0; i < other.network.size(); ++i)
network.push_back(other.network[i]->Clone());
// Ensure that the aliases for layers during passes have the right size.
layerOutputs.resize(network.size(), MatType());
layerDeltas.resize(network.size(), MatType());
layerGradients.resize(network.size(), MatType());
}
return *this;
}
template<typename ReductionRuleType, typename MatType>
LambdaMapReduceType<ReductionRuleType, MatType>& LambdaMapReduceType<ReductionRuleType, MatType>::operator=(LambdaMapReduceType&& other)
{
if (this != &other)
{
Layer<MatType>::operator=(other);
layerOutputs.clear();
layerDeltas.clear();
layerGradients.clear();
inSize = std::move(other.inSize);
totalInputSize = std::move(other.totalInputSize);
totalOutputSize = std::move(other.totalOutputSize);
reductionRule = std::move(other.reductionRule);
network = std::move(other.network);
layerOutputs.resize(network.size(), MatType());
layerDeltas.resize(network.size(), MatType());
layerGradients.resize(network.size(), MatType());
other.layerOutputs.clear();
other.layerDeltas.clear();
other.layerGradients.clear();
}
return *this;
}
template<typename ReductionRuleType, typename MatType>
void LambdaMapReduceType<ReductionRuleType, MatType>::Forward(
const MatType& input, MatType& output)
{
// Make sure training/testing mode is set right in each layer.
for (size_t i = 0; i < network.size(); ++i)
network[i]->Training() = this->training;
// Note that we use `output` for the last layer; layerOutputs is only used for
// intermediate values between layers.
if (network.size() > 1)
{
// Initialize memory for the forward pass (if needed).
InitializeForwardPassMemory(input.n_cols);
for (size_t i = 0; i < network.size(); i++)
network[i]->Forward(input, layerOutputs[i]);
// Reduce the outputs to single output.
reductionRule.Reduce(layerOutputs, output);
}
else if (network.size() == 1)
{
network[0]->Forward(input, output);
}
else
{
// Empty network?
output = input;
}
}
template<typename ReductionRuleType, typename MatType>
void LambdaMapReduceType<ReductionRuleType, MatType>::Backward(
const MatType& input, const MatType& gy, MatType& g)
{
if (network.size() > 1)
{
// Initialize memory for the backward pass (if needed).
InitializeBackwardPassMemory(input.n_cols);
std::vector<MatType> layerTempDeltas;
// Compute the gy for all layers.
reductionRule.UnReduce(gy, network.size(), layerTempDeltas);
g.zeros();
for (size_t i = 0; i < network.size(); i++) {
network[i]->Backward(layerOutputs[i], layerTempDeltas[i], layerDeltas[i]);
g += layerDeltas[i];
}
}
else if (network.size() == 1)
{
network[0]->Backward(input, gy, g);
}
else
{
// Empty network?
g = gy;
}
}
template<typename ReductionRuleType, typename MatType>
void LambdaMapReduceType<ReductionRuleType, MatType>::Gradient(
const MatType& input, const MatType& error, MatType& gradient)
{
// We assume gradient has the right size already.
// Pass gradients through each layer.
if (network.size() > 1)
{
// Initialize memory for the gradient pass (if needed).
InitializeGradientPassMemory(gradient);
std::vector<MatType> layerTempDeltas;
// Compute the error for all layers.
reductionRule.UnReduce(error, network.size(), layerTempDeltas);
for (size_t i = 0; i < network.size(); ++i)
{
network[i]->Gradient(input, layerTempDeltas[i],
layerGradients[i]);
}
}
else if (network.size() == 1)
{
network[0]->Gradient(input, error, gradient);
}
else
{
// Nothing to do if the network is empty... there is no gradient.
}
}
template<typename ReductionRuleType, typename MatType>
void LambdaMapReduceType<ReductionRuleType, MatType>::SetWeights(typename MatType::elem_type* weightsPtr)
{
size_t start = 0;
const size_t totalWeightSize = WeightSize();
for (size_t i = 0; i < network.size(); ++i)
{
const size_t weightSize = network[i]->WeightSize();
// Sanity check: ensure we aren't passing memory past the end of the
// parameters.
Log::Assert(start + weightSize <= totalWeightSize,
"FNN::SetLayerMemory(): parameter size does not match total layer "
"weight size!");
network[i]->SetWeights(weightsPtr + start);
start += weightSize;
}
// Technically this check should be unnecessary, but there's nothing wrong
// with a little paranoia...
Log::Assert(start == totalWeightSize,
"FNN::SetLayerMemory(): total layer weight size does not match parameter "
"size!");
MakeAlias(weights, weightsPtr, totalWeightSize, 1);
}
template<typename ReductionRuleType, typename MatType>
size_t LambdaMapReduceType<ReductionRuleType, MatType>::WeightSize() const
{
// Sum the weights in each layer.
size_t total = 0;
for (size_t i = 0; i < network.size(); ++i)
total += network[i]->WeightSize();
return total;
}
template<typename ReductionRuleType, typename MatType>
void LambdaMapReduceType<ReductionRuleType, MatType>::ComputeOutputDimensions()
{
inSize = 0;
totalInputSize = 0;
totalOutputSize = 0;
// Propagate the input dimensions forward to the output.
if (network.size() == 0)
{
this->outputDimensions = this->inputDimensions;
return;
}
inSize = this->inputDimensions[0];
for (size_t i = 1; i < this->inputDimensions.size(); ++i)
inSize *= this->inputDimensions[i];
totalInputSize = network.size() * inSize;
for (size_t i = 0; i < network.size(); ++i)
{
network[i]->InputDimensions() = this->inputDimensions;
size_t layerOutputSize = network[i]->OutputSize();
totalOutputSize += layerOutputSize;
}
// Compute the output size of the network using reduction rules.
this->outputDimensions = reductionRule.ReduceSize(network);
}
template<typename ReductionRuleType, typename MatType>
double LambdaMapReduceType<ReductionRuleType, MatType>::Loss() const
{
double loss = 0.0;
for (size_t i = 0; i < network.size(); ++i)
loss += network[i]->Loss();
return loss;
}
template<typename ReductionRuleType, typename MatType>
template<typename Archive>
void LambdaMapReduceType<ReductionRuleType, MatType>::serialize(
Archive& ar, const uint32_t /* version */)
{
ar(cereal::base_class<Layer<MatType>>(this));
ar(CEREAL_VECTOR_POINTER(network));
ar(CEREAL_NVP(inSize));
ar(CEREAL_NVP(totalInputSize));
ar(CEREAL_NVP(totalOutputSize));
if (Archive::is_loading::value)
{
layerOutputMatrix.clear();
layerDeltaMatrix.clear();
layerGradients.clear();
layerOutputs.resize(network.size(), MatType());
layerDeltas.resize(network.size(), MatType());
layerGradients.resize(network.size(), MatType());
}
}
template<typename ReductionRuleType, typename MatType>
void LambdaMapReduceType<ReductionRuleType, MatType>::InitializeForwardPassMemory(const size_t batchSize)
{
// We need to initialize memory to store the output of each layer's Forward()
// call. We'll do this all in one matrix, but, the size of this matrix
// depends on the batch size we are using for computation. We avoid resizing
// layerOutputMatrix down, unless we only need 10% or less of it.
if (batchSize * totalOutputSize > layerOutputMatrix.n_elem ||
batchSize * totalOutputSize <
std::floor(0.1 * layerOutputMatrix.n_elem))
{
// All outputs will be represented by one big block of memory.
layerOutputMatrix = MatType(1, batchSize * totalOutputSize);
}
// Now, create an alias to the right place for each layer. We assume that
// layerOutputs is already sized correctly (this should be done by Add()).
size_t start = 0;
for (size_t i = 0; i < layerOutputs.size(); ++i)
{
const size_t layerOutputSize = network[i]->OutputSize();
MakeAlias(layerOutputs[i], layerOutputMatrix.colptr(start),
layerOutputSize, batchSize);
start += batchSize * layerOutputSize;
}
}
template<typename ReductionRuleType, typename MatType>
void LambdaMapReduceType<ReductionRuleType, MatType>::InitializeBackwardPassMemory(
const size_t batchSize)
{
// We need to initialize memory to store the output of each layer's Backward()
// call. We do this similarly to InitializeForwardPassMemory(), but we must
// store a matrix to use as the delta for each layer.
if (batchSize * totalInputSize > layerDeltaMatrix.n_elem ||
batchSize * totalInputSize < std::floor(0.1 * layerDeltaMatrix.n_elem))
{
// All deltas will be represented by one big block of memory.
layerDeltaMatrix = MatType(1, batchSize * totalInputSize);
}
// Now, create an alias to the right place for each layer. We assume that
// layerDeltas is already sized correctly (this should be done by Add()).
size_t start = 0;
for (size_t i = 0; i < layerDeltas.size(); ++i)
{
size_t layerInputSize = inSize;
MakeAlias(layerDeltas[i], layerDeltaMatrix.colptr(start), layerInputSize,
batchSize);
start += batchSize * layerInputSize;
}
}
template<typename ReductionRuleType, typename MatType>
void LambdaMapReduceType<ReductionRuleType, MatType>::InitializeGradientPassMemory(MatType& gradient)
{
// We need to initialize memory to store the gradients of each layer. To do
// this, we need to know the weight size of each layer.
size_t gradientStart = 0;
for (size_t i = 0; i < network.size(); ++i)
{
const size_t weightSize = network[i]->WeightSize();
MakeAlias(layerGradients[i], gradient.memptr() + gradientStart,
weightSize, 1);
gradientStart += weightSize;
}
}
} // namespace ann
} // namespace mlpack
#endif
src/mlpack/methods/ann/layer/layer_types.hpp
+1 -4
View File
@@ -26,7 +26,6 @@
#include <mlpack/methods/ann/layer/convolution.hpp>
#include <mlpack/methods/ann/layer/dropconnect.hpp>
#include <mlpack/methods/ann/layer/dropout.hpp>
#include <mlpack/methods/ann/layer/lambda_map_reduce.hpp>
#include <mlpack/methods/ann/layer/leaky_relu.hpp>
#include <mlpack/methods/ann/layer/linear.hpp>
#include <mlpack/methods/ann/layer/linear_no_bias.hpp>
@@ -38,6 +37,7 @@
#include <mlpack/methods/ann/layer/noisylinear.hpp>
#include <mlpack/methods/ann/layer/padding.hpp>
#include <mlpack/methods/ann/layer/radial_basis_function.hpp>
#include <mlpack/methods/ann/layer/residual.hpp>
#include <mlpack/methods/ann/layer/softmax.hpp>
// Convolution modes.
@@ -51,9 +51,6 @@
// Loss function modules.
#include <mlpack/methods/ann/loss_functions/negative_log_likelihood.hpp>
// Include Reduction Rules.
#include <mlpack/methods/ann/reduction_rules/add_reduction.hpp>
// Include definitions for polymorphic serialization.
#include <mlpack/methods/ann/layer/serialization.hpp>
src/mlpack/methods/ann/layer/multi_layer_impl.hpp
+3 -9
View File
@@ -379,15 +379,9 @@ void MultiLayer<MatType>::InitializeBackwardPassMemory(
for (size_t i = 0; i < layerDeltas.size(); ++i)
{
size_t layerInputSize = 1;
if (i == 0)
{
for (size_t j = 0; j < this->inputDimensions.size(); ++j)
layerInputSize *= this->inputDimensions[j];
}
else
{
layerInputSize = network[i - 1]->OutputSize();
}
for (size_t j = 0; j < network[i]->InputDimensions().size(); ++j)
layerInputSize *= network[i]->InputDimensions()[j];
MakeAlias(layerDeltas[i], layerDeltaMatrix.colptr(start), layerInputSize,
batchSize);
start += batchSize * layerInputSize;
src/mlpack/methods/ann/layer/residual.hpp
+127
View File
@@ -0,0 +1,127 @@
/**
* @file methods/ann/layer/residual.hpp
* @author Shubham Agrawal
*
* Base class for neural network layers that are wrappers around other layers.
*
* mlpack is free software; you may redistribute it and/or modify it under the
* terms of the 3-clause BSD license. You should have received a copy of the
* 3-clause BSD license along with mlpack. If not, see
* http://www.opensource.org/licenses/BSD-3-Clause for more information.
*/
#ifndef MLPACK_METHODS_ANN_LAYER_RESIDUAL_HPP
#define MLPACK_METHODS_ANN_LAYER_RESIDUAL_HPP
#include "../make_alias.hpp"
#include "multi_layer.hpp"
namespace mlpack {
namespace ann {
/**
* A lambda "map-reduce" is a layer that is a wrapper around other layers.
* It passes the input through all of its child layers sequentially, returning
* the output from reducing the output.
*
* @tparam MatType Matrix representation to accept as input and use for
* computation.
*/
template<typename MatType>
class ResidualType : public MultiLayer<MatType>
{
public:
/**
* Create an empty ResidualType that holds no layers of its own. Be sure to add
* layers with Add() before using!
*/
ResidualType();
//! Copy the given ResidualType.
ResidualType(const ResidualType& other);
//! Take ownership of the layers of the given ResidualType.
ResidualType(ResidualType&& other);
//! Copy the given ResidualType.
ResidualType& operator=(const ResidualType& other);
//! Take ownership of the given ResidualType.
ResidualType& operator=(ResidualType&& other);
//! Virtual destructor: delete all held layers.
virtual ~ResidualType()
{
// Nothing to do here.
}
//! Create a copy of the ResidualType (this is safe for polymorphic use).
virtual ResidualType* Clone() const { return new ResidualType(*this); }
/**
* Perform a forward pass with the given input data. `output` is expected to
* have the correct size (e.g. number of rows equal to `OutputSize()` of the
* last held layer; number of columns equal to `input.n_cols`).
*
* @param input Input data to pass through the ResidualType.
* @param output Matrix to store output in.
*/
virtual void Forward(const MatType& input, MatType& output);
/**
* Perform a backward pass with the given data. `gy` is expected to be the
* propagated error from the subsequent layer (or output), `input` is expected
* to be the output from this layer when `Forward()` was called, and `g` will
* store the propagated error from this layer (to be passed to the previous
* layer as `gy`).
*
* It is expected that `g` has the correct size already (e.g., number of rows
* equal to `OutputSize()` of the previous layer, and number of columns equal
* to `input.n_cols`).
*
* This function is expected to be called for the same input data as
* `Forward()` was just called for.
*
* @param input Output of Forward().
* @param gy Propagated error from next layer.
* @param g Matrix to store propagated error in for previous layer.
*/
virtual void Backward(const MatType& input,
const MatType& gy,
MatType& g);
/**
* Compute the gradients of each layer.
*
* This function is expected to be called for the same input data as
* `Forward()` and `Backward()` were just called for. That is, `input` here
* should be the same data as `Forward()` was called with.
*
* `gradient` is expected to have the correct size already (e.g., number of
* rows equal to 1, and number of columns equal to `WeightSize()`).
*
* @param input Original input data provided to Forward().
* @param error Error as computed by `Backward()`.
* @param gradient Matrix to store the gradients in.
*/
virtual void Gradient(const MatType& input,
const MatType& error,
MatType& gradient);
/**
* Compute the output dimensions of the ResidualType using `InputDimensions()`.
* This computes the dimensions of each layer held by the ResidualType, and the
* output dimensions are set to the output dimensions of the last layer.
*/
virtual void ComputeOutputDimensions();
//! Serialize the ResidualType.
template<typename Archive>
void serialize(Archive& ar, const uint32_t /* version */);
};
typedef ResidualType<arma::mat> Residual;
} // namespace ann
} // namespace mlpack
// Include implementation.
#include "residual_impl.hpp"
#endif
src/mlpack/methods/ann/layer/residual_impl.hpp
+209
View File
@@ -0,0 +1,209 @@
/**
* @file methods/ann/layer/residual_impl.hpp
* @author Shubham Agrawal
*
* Implementation of the base class for neural network layers that are wrappers
* around other layers.
*
* mlpack is free software; you may redistribute it and/or modify it under the
* terms of the 3-clause BSD license. You should have received a copy of the
* 3-clause BSD license along with mlpack. If not, see
* http://www.opensource.org/licenses/BSD-3-Clause for more information.
*/
#ifndef MLPACK_METHODS_ANN_LAYER_RESIDUAL_IMPL_HPP
#define MLPACK_METHODS_ANN_LAYER_RESIDUAL_IMPL_HPP
#include "residual.hpp"
namespace mlpack {
namespace ann {
template<typename MatType>
ResidualType<MatType>::ResidualType() :
MultiLayer<MatType>()
{
// Nothing to do.
}
template<typename MatType>
ResidualType<MatType>::ResidualType(const ResidualType& other) :
MultiLayer<MatType>(other)
{
// Nothing to do here.
}
template<typename MatType>
ResidualType<MatType>::ResidualType(ResidualType&& other) :
MultiLayer<MatType>(other)
{
// Nothing to do here.
}
template<typename MatType>
ResidualType<MatType>& ResidualType<MatType>::operator=(const ResidualType& other)
{
if (this != &other)
{
MultiLayer<MatType>::operator=(other);
}
return *this;
}
template<typename MatType>
ResidualType<MatType>& ResidualType<MatType>::operator=(ResidualType&& other)
{
if (this != &other)
{
MultiLayer<MatType>::operator=(other);
}
return *this;
}
template<typename MatType>
void ResidualType<MatType>::Forward(
const MatType& input, MatType& output)
{
// Make sure training/testing mode is set right in each layer.
for (size_t i = 0; i < this->network.size(); ++i)
this->network[i]->Training() = this->training;
// Note that we use `output` for the last layer; layerOutputs is only used for
// intermediate values between layers.
if (this->network.size() > 1)
{
// Initialize memory for the forward pass (if needed).
this->InitializeForwardPassMemory(input.n_cols);
for (size_t i = 0; i < this->network.size(); i++)
this->network[i]->Forward(input, this->layerOutputs[i]);
// Reduce the outputs to single output.
output.zeros();
for (size_t i = 0; i < this->layerOutputs.size(); i++)
{
output += this->layerOutputs[i];
}
}
else if (this->network.size() == 1)
{
this->network[0]->Forward(input, output);
}
else
{
// Empty network?
output = input;
}
}
template<typename MatType>
void ResidualType<MatType>::Backward(
const MatType& input, const MatType& gy, MatType& g)
{
if (this->network.size() > 1)
{
// Initialize memory for the backward pass (if needed).
this->InitializeBackwardPassMemory(input.n_cols);
g.zeros();
for (size_t i = 0; i < this->network.size(); i++) {
this->network[i]->Backward(this->layerOutputs[i], gy, this->layerDeltas[i]);
g += this->layerDeltas[i];
}
}
else if (this->network.size() == 1)
{
this->network[0]->Backward(input, gy, g);
}
else
{
// Empty network?
g = gy;
}
}
template<typename MatType>
void ResidualType<MatType>::Gradient(
const MatType& input, const MatType& error, MatType& gradient)
{
// We assume gradient has the right size already.
// Pass gradients through each layer.
if (this->network.size() > 1)
{
// Initialize memory for the gradient pass (if needed).
this->InitializeGradientPassMemory(gradient);
for (size_t i = 0; i < this->network.size(); ++i)
{
this->network[i]->Gradient(input, error, this->layerGradients[i]);
}
}
else if (this->network.size() == 1)
{
this->network[0]->Gradient(input, error, gradient);
}
else
{
// Nothing to do if the network is empty... there is no gradient.
}
}
template<typename MatType>
void ResidualType<MatType>::ComputeOutputDimensions()
{
this->inSize = 0;
this->totalInputSize = 0;
this->totalOutputSize = 0;
// Propagate the input dimensions forward to the output.
if (this->network.size() == 0)
{
this->outputDimensions = this->inputDimensions;
return;
}
this->inSize = this->inputDimensions[0];
for (size_t i = 1; i < this->inputDimensions.size(); ++i)
this->inSize *= this->inputDimensions[i];
this->totalInputSize = this->network.size() * this->inSize;
for (size_t i = 0; i < this->network.size(); ++i)
{
this->network[i]->InputDimensions() = this->inputDimensions;
size_t layerOutputSize = this->network[i]->OutputSize();
this->totalOutputSize += layerOutputSize;
}
// Compute the output size of the network using reduction rules.
if (this->network.size() == 1)
{
this->outputDimensions = this->network[0]->OutputDimensions();
return;
}
const std::vector<size_t> networkSize = this->network[0]->OutputDimensions();
for (size_t i = 1; i < this->network.size(); i++)
{
if (!(networkSize == this->network[i]->OutputDimensions()))
{
Log::Fatal << "Network size mismatch. (" << networkSize[0] << ", "
<< networkSize[1] << ") != ("
<< this->network[i]->OutputDimensions()[0] << ", " << this->network[i]->OutputDimensions()[1]
<< ")." << std::endl;
}
}
this->outputDimensions = networkSize;
}
template<typename MatType>
template<typename Archive>
void ResidualType<MatType>::serialize(
Archive& ar, const uint32_t /* version */)
{
ar(cereal::base_class<MultiLayer<MatType>>(this));
}
} // namespace ann
} // namespace mlpack
#endif
src/mlpack/methods/ann/layer/serialization.hpp
+1 -2
View File
@@ -39,8 +39,6 @@
__VA_ARGS__>); \
CEREAL_REGISTER_TYPE(mlpack::ann::DropConnectType<__VA_ARGS__>); \
CEREAL_REGISTER_TYPE(mlpack::ann::DropoutType<__VA_ARGS__>); \
CEREAL_REGISTER_TYPE(mlpack::ann::LambdaMapReduceType< \
mlpack::ann::AddReduction, __VA_ARGS__>); \
CEREAL_REGISTER_TYPE(mlpack::ann::LeakyReLUType<__VA_ARGS__>); \
CEREAL_REGISTER_TYPE(mlpack::ann::Linear3DType<__VA_ARGS__>); \
CEREAL_REGISTER_TYPE(mlpack::ann::LinearType<__VA_ARGS__>); \
@@ -52,6 +50,7 @@
CEREAL_REGISTER_TYPE(mlpack::ann::NoisyLinearType<__VA_ARGS__>); \
CEREAL_REGISTER_TYPE(mlpack::ann::PaddingType<__VA_ARGS__>); \
CEREAL_REGISTER_TYPE(mlpack::ann::RBFType<__VA_ARGS__>); \
CEREAL_REGISTER_TYPE(mlpack::ann::ResidualType<__VA_ARGS__>); \
CEREAL_REGISTER_TYPE(mlpack::ann::SoftmaxType<__VA_ARGS__>); \
CEREAL_REGISTER_MLPACK_LAYERS(arma::mat);
src/mlpack/tests/ann_layer_test.cpp
+11 -11
View File
@@ -4854,9 +4854,9 @@ TEST_CASE("TransposedConvolutionLayerPaddingTest", "[ANNLayerTest]")
// }
/**
* Simple test for Lambda Map Reduce layer.
* Simple test for Residual layer.
*/
TEST_CASE("LambdaMapReduceTestCase", "[ANNLayerTest]")
TEST_CASE("ResidualTestCase", "[ANNLayerTest]")
{
// For rectangular input to pooling layers.
arma::mat input = arma::mat(28, 1);
@@ -4871,11 +4871,11 @@ TEST_CASE("LambdaMapReduceTestCase", "[ANNLayerTest]")
input(14) = input(25) = 8;
input(15) = input(26) = 9;
LambdaMapReduce module1;
Residual module1;
module1.Add<MeanPooling>(2, 2, 2, 2, false);
module1.Add<MeanPooling>(2, 2, 2, 2, false);
LambdaMapReduce module2;
Residual module2;
module2.Add<MeanPooling>(2, 2, 2, 2, true);
module2.Add<MeanPooling>(2, 2, 2, 2, true);
@@ -4907,23 +4907,23 @@ TEST_CASE("LambdaMapReduceTestCase", "[ANNLayerTest]")
CheckMatrices(output1, result1, 1e-1);
CheckMatrices(output2, result2, 1e-1);
arma::mat prev_delta1, prev_delta2;
prev_delta1 << 3.6000 << -0.9000 << arma::endr
arma::mat prevDelta1, prevDelta2;
prevDelta1 << 3.6000 << -0.9000 << arma::endr
<< 3.6000 << -0.9000 << arma::endr
<< 3.6000 << -0.9000 << arma::endr
<< 3.6000 << -0.9000 << arma::endr;
prev_delta2 << 3.6000 << -0.9000 << arma::endr
prevDelta2 << 3.6000 << -0.9000 << arma::endr
<< 3.6000 << -0.9000 << arma::endr
<< 3.6000 << -0.9000 << arma::endr;
arma::mat delta1, delta2;
delta1.set_size(28, 1);
delta2.set_size(28, 1);
prev_delta1.reshape(8, 1);
prev_delta2.reshape(6, 1);
module1.Backward(input, prev_delta1, delta1);
prevDelta1.reshape(8, 1);
prevDelta2.reshape(6, 1);
module1.Backward(input, prevDelta1, delta1);
REQUIRE(arma::accu(delta1) == Approx(21.6).epsilon(1e-3));
module2.Backward(input, prev_delta2, delta2);
module2.Backward(input, prevDelta2, delta2);
REQUIRE(arma::accu(delta2) == Approx(16.2).epsilon(1e-3));
}