apaam/mlpack
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Refactor Highway (and fix MultiLayer).

Browse Source
This commit is contained in:
Ryan Curtin
2021-08-01 21:42:09 -04:00
parent 0375e7d575
commit 8a4e5b9bdf
6 changed files with 104 additions and 192 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/mlpack/methods/ann/layer/highway.hpp
+5 -91
View File
@@ -21,7 +21,7 @@ namespace mlpack {
namespace ann /** Artificial Neural Network. */ {
/**
* Implementation of the Highway layer. The Highway class can vary its behavior
* Implementation of the Highway layer. The Highway class can vary its behavior
* between that of feed-forward fully connected network container and that
* of a layer which simply passes its inputs through depending on the transform
* gate. Note that the size of the input and output matrices of this class
@@ -54,13 +54,6 @@ class HighwayType : public MultiLayer<InputType, OutputType>
//! Create the HighwayTest object.
HighwayType();
/**
* Create the HighwayTest object.
*
* @param inSize The number of input units.
*/
HighwayType(const size_t inSize);
//! Destroy the Highway object.
~HighwayType();
@@ -102,60 +95,20 @@ class HighwayType : public MultiLayer<InputType, OutputType>
const OutputType& error,
OutputType& gradient);
/**
* Add a new module to the model.
*
* @param args The layer parameter.
*/
template <class LayerType, class... Args>
void Add(Args... args)
{
network.push_back(new LayerType(args...));
networkOwnerships.push_back(true);
}
/**
* Add a new module to the model.
*
* @param layer The Layer to be added to the model.
*/
void Add(Layer<arma::mat, arma::mat>* layer)
{
network.push_back(layer);
networkOwnerships.push_back(false);
}
//! Get the parameters.
OutputType const& Parameters() const { return weights; }
//! Modify the parameters.
OutputType& Parameters() { return weights; }
//! Get the number of input units.
size_t InSize() const { return inSize; }
//! Get the number of trainable weights.
const size_t WeightSize() const
size_t WeightSize() const
{
size_t result = inSize * (inSize + 1);
for (size_t i = 0; i < network.size(); ++i)
result += network[i]->WeightSize();
size_t result = this->totalInputSize * (this->totalInputSize + 1);
for (size_t i = 0; i < this->network.size(); ++i)
result += this->network[i]->WeightSize();
return result;
}
//! Get the output dimensions.
const std::vector<size_t>& OutputDimensions() const
{
// Push the input dimensions through the layers in order to compute the
// output size.
network.front()->InputDimensions() = inputDimensions;
for (size_t i = 1; i < network.size(); ++i)
{
network[i]->InputDimensions() = network[i - 1]->OutputDimensions();
}
return network.back()->OutputDimensions();
}
/**
* Serialize the layer.
*/
@@ -163,33 +116,9 @@ class HighwayType : public MultiLayer<InputType, OutputType>
void serialize(Archive& ar, const uint32_t /* version */);
private:
//! Locally-stored number of input units.
size_t inSize;
//! Parameter which indicates if the modules should be exposed.
bool model;
//! Indicator if we already initialized the model.
bool reset;
//! Locally-stored network modules.
std::vector<Layer<InputType, OutputType>*> network;
//! The list of network modules we are responsible for.
std::vector<bool> networkOwnerships;
//! Locally-stored empty list of modules.
std::vector<Layer<InputType, OutputType>*> empty;
//! Locally-stored weight object.
OutputType weights;
//! Locally-stored delta object.
OutputType delta;
//! Locally-stored gradient object.
OutputType gradient;
//! Weights for transformation of output.
OutputType transformWeight;
@@ -204,21 +133,6 @@ class HighwayType : public MultiLayer<InputType, OutputType>
//! Locally-stored transform gate error.
OutputType transformGateError;
//! Locally-stored input parameter object.
InputType inputParameter;
//! Locally-stored output parameter object.
OutputType outputParameter;
//! The input width.
size_t width;
//! The input height.
size_t height;
//! The normal output without highway network.
OutputType networkOutput;
}; // class HighwayType
// Standard Highway layer.
src/mlpack/methods/ann/layer/highway_impl.hpp
+46 -73
View File
@@ -20,49 +20,31 @@ namespace mlpack {
namespace ann /** Artificial Neural Network. */ {
template<typename InputType, typename OutputType>
HighwayType<InputType, OutputType>::HighwayType() :
inSize(0),
reset(false),
width(0),
height(0)
HighwayType<InputType, OutputType>::HighwayType()
{
// Nothing to do here.
}
template<typename InputType, typename OutputType>
HighwayType<InputType, OutputType>::HighwayType(
const size_t inSize) :
inSize(inSize),
reset(false),
width(0),
height(0)
{
weights.set_size(inSize * inSize + inSize, 1);
// TODO: how do we add the child layers ?? (read paper ...)
}
template<typename InputType, typename OutputType>
HighwayType<InputType, OutputType>::~HighwayType()
{
for (size_t i = 0; i < network.size(); ++i)
{
if (networkOwnerships[i])
delete network[i];
}
}
template<typename InputType, typename OutputType>
void HighwayType<InputType, OutputType>::SetWeights(
typename OutputType::elem_type* weightsPtr)
{
transformWeight = OutputType(weightsPtr, inSize, inSize, false, false);
transformWeight = OutputType(weightsPtr, this->inSize,
this->inSize, false, false);
transformBias = OutputType(weightsPtr + transformWeight.n_elem,
inSize, 1, false, false);
this->inSize, 1, false, false);
size_t start = transformWeight.n_elem + transformBias.n_elem;
for (size_t i = 0; i < network.size(); ++i)
for (size_t i = 0; i < this->network.size(); ++i)
{
network[i]->SetWeights(weightsPtr + start);
start += network[i]->WeightSize();
this->network[i]->SetWeights(weightsPtr + start);
start += this->network[i]->WeightSize();
}
}
@@ -70,17 +52,18 @@ template<typename InputType, typename OutputType>
void HighwayType<InputType, OutputType>::Forward(
const InputType& input, OutputType& output)
{
InitializeForwardPassMemory();
this->InitializeForwardPassMemory(input.n_cols);
network.front()->Forward(input, layerOutputs.front());
this->network.front()->Forward(input, this->layerOutputs.front());
for (size_t i = 1; i < network.size(); ++i)
for (size_t i = 1; i < this->network.size(); ++i)
{
network[i]->Forward(layerOutputs[i - 1], layerOutputs[i]);
this->network[i]->Forward(this->layerOutputs[i - 1], this->layerOutputs[i]);
}
output = network.back()->OutputParameter();
output = this->layerOutputs.back(); // TODO: can this be cleaned up?
// TODO: move to ComputeOutputDimensions()
if (arma::size(output) != arma::size(input))
{
Log::Fatal << "The sizes of the output and input matrices of the Highway"
@@ -90,33 +73,33 @@ void HighwayType<InputType, OutputType>::Forward(
transformGate = transformWeight * input;
transformGate.each_col() += transformBias;
transformGateActivation = 1.0 /(1 + arma::exp(-transformGate));
inputParameter = input;
networkOutput = output; // TODO: what is done with this?
output = (layerOutputs.back() % transformGateActivation) +
output = (this->layerOutputs.back() % transformGateActivation) +
(input % (1 - transformGateActivation));
}
template<typename InputType, typename OutputType>
void HighwayType<InputType, OutputType>::Backward(
const InputType& /* input */,
const InputType& input,
const OutputType& gy,
OutputType& g)
{
InitializeBackwardPassMemory();
this->InitializeBackwardPassMemory(input.n_cols);
OutputType gyTransform = gy % transformGateActivation;
network.back()->Backward(layerOutputs.back(), gyTransform,
layerDeltas.back());
this->network.back()->Backward(this->layerOutputs.back(), gyTransform,
this->layerDeltas.back());
for (size_t i = 2; i < network.size() + 1; ++i)
for (size_t i = 2; i < this->network.size() + 1; ++i)
{
network[network.size() - i]->Backward(layerOutputs[network.size() - i],
layerDeltas[network.size() - i + 1], layerDeltas[network.size() - i]);
this->network[this->network.size() - i]->Backward(
this->layerOutputs[this->network.size() - i],
this->layerDeltas[this->network.size() - i + 1],
this->layerDeltas[this->network.size() - i]);
}
transformGateError = gy % (networkOutput - inputParameter) %
transformGateError = gy % (gy - input) %
transformGateActivation % (1.0 - transformGateActivation);
g = layerDeltas.front() + (transformWeight.t() * transformGateError) +
g = this->layerDeltas.front() + (transformWeight.t() * transformGateError) +
(gy % (1 - transformGateActivation));
}
@@ -126,32 +109,31 @@ void HighwayType<InputType, OutputType>::Gradient(
const OutputType& error,
OutputType& gradient)
{
OutputType errorTransform = error % transformGateActivation;
size_t gradientStart = gradient.n_elem -
network[network.size() - 1].WeightSize();
network.back()->Gradient(
layerOutputs[network.size() - 2],
errorTransform,
OutputType(gradient.colptr(gradientStart), 1,
network[network.size() - 1].WeightSize(), false, true)
);
// Create an alias for the gradient that only refers to the elements in the
// network itself.
OutputType layerGradient(gradient.memptr() + (this->inSize *
(this->inSize + 1)), 1, gradient.n_elem - (this->inSize *
(this->inSize + 1)), false, true);
this->InitializeGradientPassMemory(layerGradient);
for (size_t i = 2; i < network.size(); ++i)
OutputType errorTransform = error % transformGateActivation;
this->network.back()->Gradient(
this->layerOutputs[this->network.size() - 2],
errorTransform,
this->layerGradients[this->network.size() - 1]);
for (size_t i = 2; i < this->network.size(); ++i)
{
gradientStart -= network[network.size() - i]->WeightSize();
network[network.size() - i]->Gradient(
layerOutputs[network.size() - i - 1],
layerDeltas[network.size() - i],
OutputType(gradient.colptr(gradientStart), 1,
network[network.size() - i]->WeightSize(), false, true)
);
this->network[this->network.size() - i]->Gradient(
this->layerOutputs[this->network.size() - i - 1],
this->layerDeltas[this->network.size() - i],
this->layerGradients[this->network.size() - i]);
}
network.front()->Gradient(
this->network.front()->Gradient(
input,
layerDeltas[1],
layerDeltas.front()
);
this->layerDeltas[1],
this->layerGradients.front());
gradient.submat(0, 0, transformWeight.n_elem - 1, 0) = arma::vectorise(
transformGateError * input.t());
@@ -165,15 +147,6 @@ void HighwayType<InputType, OutputType>::serialize(
Archive& ar, const uint32_t /* version */)
{
ar(cereal::base_class<Layer<InputType, OutputType>>(this));
ar(CEREAL_VECTOR_POINTER(network));
// Reset the memory.
if (Archive::is_loading::value)
{
networkOwnerships.clear();
networkOwnerships.resize(network.size(), true);
}
}
} // namespace ann
src/mlpack/methods/ann/layer/layer_types.hpp
+1 -1
View File
@@ -35,7 +35,7 @@
//#include <mlpack/methods/ann/layer/glimpse.hpp>
//#include <mlpack/methods/ann/layer/hardshrink.hpp>
//#include <mlpack/methods/ann/layer/hard_tanh.hpp>
//#include <mlpack/methods/ann/layer/highway.hpp>
#include <mlpack/methods/ann/layer/highway.hpp>
//#include <mlpack/methods/ann/layer/join.hpp>
//#include <mlpack/methods/ann/layer/layer_norm.hpp>
//#include <mlpack/methods/ann/layer/leaky_relu.hpp>
src/mlpack/methods/ann/layer/multi_layer.hpp
+49 -25
View File
@@ -12,6 +12,8 @@
#ifndef MLPACK_METHODS_ANN_LAYER_MULTI_LAYER_HPP
#define MLPACK_METHODS_ANN_LAYER_MULTI_LAYER_HPP
#include "../make_alias.hpp"
namespace mlpack {
namespace ann {
@@ -70,22 +72,19 @@ class MultiLayer : public Layer<InputType, OutputType>
const OutputType& error,
OutputType& gradient)
{
// Pass gradients through each layer, creating an alias for the right
// elements of the gradient.
InitializeGradientPassMemory(gradient);
network.front()->Gradient(input, layerDeltas[1], OutputType(
gradient.memptr(), 1, network.front()->WeightSize(), false, true));
size_t start = network.front()->WeightSize();
// Pass gradients through each layer.
// TODO: do we need to go back to front? I guess not?
network.front()->Gradient(input, layerDeltas[1], layerGradients.front());
for (size_t i = 0; i < network.size() - 1; ++i)
{
network[i]->Gradient(layerOutputs[i - 1], layerDeltas[i + 1], OutputType(
gradient.colptr(start), 1, network[i]->WeightSize(), false, true));
start += network[i]->WeightSize();
network[i]->Gradient(layerOutputs[i - 1], layerDeltas[i + 1],
layerGradients[i]);
}
network.back()->Gradient(layerOutputs[network.size() - 2], error,
OutputType(gradient.colptr(start), 1, network.back()->WeightSize(),
false, true));
layerGradients[network.size() - 1]);
}
virtual void SetWeights(typename OutputType::elem_type* weightsPtr)
@@ -98,7 +97,7 @@ class MultiLayer : public Layer<InputType, OutputType>
}
}
virtual void OutputSize() const
virtual size_t OutputSize() const
{
// Return the output size of the last layer.
return network.back()->OutputSize();
@@ -115,13 +114,15 @@ class MultiLayer : public Layer<InputType, OutputType>
virtual void ComputeOutputDimensions()
{
inSize = 0;
totalInputSize = 0;
totalOutputSize = 0;
// Propagate the input dimensions forward to the output.
network.front()->InputDimensions() = this->inputDimensions;
totalInputSize += std::accumulate(this->inputDimensions.begin(),
inSize = std::accumulate(this->inputDimensions.begin(),
this->inputDimensions.end(), 0);
totalInputSize += inSize;
for (size_t i = 1; i < network.size(); ++i)
{
@@ -162,8 +163,11 @@ class MultiLayer : public Layer<InputType, OutputType>
network.push_back(new LayerType(args...));
layerOutputs.push_back(OutputType());
layerDeltas.push_back(OutputType());
layerGradients.push_back(OutputType());
}
// TODO: handle network ownership?
/*
* Add a new module to the model.
*
@@ -174,6 +178,7 @@ class MultiLayer : public Layer<InputType, OutputType>
network.push_back(layer);
layerOutputs.push_back(OutputType());
layerDeltas.push_back(OutputType());
layerGradients.push_back(OutputType());
}
const std::vector<Layer<InputType, OutputType>*> Network() const { return
@@ -195,6 +200,7 @@ network; }
layerDeltaMatrix.clear();
layerOutputs.resize(network.size(), OutputType());
layerDeltas.resize(network.size(), OutputType());
layerGradients.resize(network.size(), OutputType());
}
}
@@ -219,9 +225,9 @@ network; }
size_t start = 0;
for (size_t i = 0; i < layerOutputs.size(); ++i)
{
const size_t layerOutputSize = network[i].OutputSize();
layerOutputs[i] = OutputType(layerOutputMatrix.colptr(start),
layerOutputSize, batchSize, false, true);
const size_t layerOutputSize = network[i]->OutputSize();
MakeAlias(layerOutputs[i], layerOutputMatrix.colptr(start),
layerOutputSize, batchSize);
start += batchSize * layerOutputSize;
}
}
@@ -229,9 +235,8 @@ network; }
void InitializeBackwardPassMemory(const size_t batchSize)
{
// We need to initialize memory to store the output of each layer's
// Backward() and Gradient() calls. We do this similarly to
// InitializeForwardPassMemory(), but we must store a matrix to use as the
// delta for each layer.
// Backward() call. We do this similarly to InitializeForwardPassMemory(),
// but we must store a matrix to use as the delta for each layer.
if (batchSize * totalInputSize > layerDeltaMatrix.n_elem ||
batchSize * totalInputSize < std::floor(0.1 * layerOutputMatrix.n_elem))
{
@@ -247,22 +252,41 @@ network; }
const size_t layerInputSize = (i == 0) ?
std::accumulate(this->inputDimensions.begin(),
this->inputDimensions.end(), 0) :
network[i - 1].OutputSize();
layerDeltas[i] = OutputType(layerDeltaMatrix.colptr(start),
layerInputSize, batchSize, false, true);
network[i - 1]->OutputSize();
MakeAlias(layerDeltas[i], layerDeltaMatrix.colptr(start), layerInputSize,
batchSize);
start += batchSize * layerInputSize;
}
}
void InitializeGradientPassMemory(OutputType& gradient)
{
// We need to initialize memory to store the gradients of each layer.
// To do this, we need to know the weight size of each layer.
size_t gradientStart = 0;
for (size_t i = 0; i < network.size(); ++i)
{
const size_t weightSize = network[i]->WeightSize();
MakeAlias(layerGradients[i], gradient.colptr(gradientStart), weightSize,
1);
gradientStart += weightSize;
}
}
std::vector<Layer<InputType, OutputType>*> network;
// Total number of elements in the input, cached for convenience.
size_t inSize;
// Total number of input elements for *every* layer.
size_t totalInputSize;
// Total number of output elements for *every* layer.
size_t totalOutputSize;
arma::mat layerOutputMatrix;
std::vector<arma::mat> layerOutputs;
arma::mat layerDeltaMatrix;
std::vector<arma::mat> layerDeltas;
OutputType layerOutputMatrix;
std::vector<OutputType> layerOutputs;
OutputType layerDeltaMatrix;
std::vector<OutputType> layerDeltas;
std::vector<OutputType> layerGradients;
};
} // namespace ann
src/mlpack/methods/ann/layer/serialization.hpp
+1
View File
@@ -126,6 +126,7 @@
CEREAL_REGISTER_TYPE(mlpack::ann::ConcatenateType<__VA_ARGS__>); \
CEREAL_REGISTER_TYPE(mlpack::ann::AddType<__VA_ARGS__>); \
CEREAL_REGISTER_TYPE(mlpack::ann::RBF<__VA_ARGS__>); \
CEREAL_REGISTER_TYPE(mlpack::ann::HighwayType<__VA_ARGS__>); \
// TODO: continue...
src/mlpack/tests/feedforward_network_test.cpp
+2 -2
View File
@@ -560,7 +560,7 @@ TEST_CASE("DropoutNetworkTest", "[FeedForwardNetworkTest]")
/**
* Train the highway network on a larger dataset.
*
*/
TEST_CASE("HighwayNetworkTest", "[FeedForwardNetworkTest]")
{
arma::mat dataset;
@@ -576,7 +576,7 @@ TEST_CASE("HighwayNetworkTest", "[FeedForwardNetworkTest]")
FFN<NegativeLogLikelihood<> > model;
model.Add<Linear>(10);
Highway* highway = new Highway(10, true);
Highway* highway = new Highway();
highway->Add<Linear>(10);
highway->Add<Sigmoid>();
model.Add(highway); // This takes ownership of the memory.