apaam/mlpack
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Fix various bugs in the MultiLayer implementation.

Browse Source
This commit is contained in:
Ryan Curtin
2021-08-07 11:15:14 -04:00
parent 8105bb8583
commit 498afa0ab7
5 changed files with 152 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/mlpack/methods/ann/ffn.hpp
+1
View File
@@ -365,6 +365,7 @@ network.Network(); }
const size_t begin,
const size_t end);
// TODO: this API needs to be changed!
/**
* Perform the backward pass of the data in real batch mode.
*
src/mlpack/methods/ann/ffn_impl.hpp
+15 -8
View File
@@ -268,8 +268,15 @@ void FFN<
// Ensure the network is valid.
CheckNetwork("FFN::Forward()", inputs.n_rows);
results.set_size(network.OutputSize(), inputs.n_cols);
network.Forward(inputs, results, begin, end);
// We must always store a copy of the forward pass in `networkOutputs` in case
// we do a backward pass.
networkOutput.set_size(network.OutputSize(), inputs.n_cols);
network.Forward(inputs, networkOutput, begin, end);
// It's possible the user passed `networkOutputs` as `results`; in this case,
// we don't need to create an alias.
if (&results != &networkOutput)
results = networkOutput;
}
template<typename OutputLayerType,
@@ -298,7 +305,7 @@ double FFN<
// Now compute the gradients.
// The gradient should have the same size as the parameters.
gradients.set_size(parameters.n_rows, parameters.n_cols);
network.Gradient(inputs, networkDelta, gradients);
network.Gradient(inputs, error, gradients);
return res;
}
@@ -348,8 +355,8 @@ double FFN<
// Sanity check: ensure network is valid.
CheckNetwork("FFN::Evaluate()", predictors.n_rows);
// networkOutput will be initialized by network.Forward().
OutputType networkOutput;
// Set networkOutput to the right size if needed, then perform the forward
// pass.
network.Forward(predictors, networkOutput);
return outputLayer.Forward(networkOutput, responses) + network.Loss();
@@ -386,7 +393,7 @@ double FFN<
const size_t begin,
const size_t batchSize)
{
CheckNetwork("FFN::Evaluate()", predictors.n_rows, true, false);
CheckNetwork("FFN::Evaluate()", predictors.n_rows);
// Set networkOutput to the right size if needed, then perform the forward
// pass.
@@ -429,7 +436,7 @@ double FFN<
OutputType& gradient,
const size_t batchSize)
{
CheckNetwork("FFN::EvaluateWithGradient()", predictors.n_rows, true, false);
CheckNetwork("FFN::EvaluateWithGradient()", predictors.n_rows);
// Set networkOutput to the right size if needed, then perform the forward
// pass.
@@ -450,7 +457,7 @@ double FFN<
// Now compute the gradients.
// The gradient should have the same size as the parameters.
gradient.set_size(parameters.n_rows, parameters.n_cols);
network.Gradient(predictors.cols(begin, begin + batchSize - 1), networkDelta,
network.Gradient(predictors.cols(begin, begin + batchSize - 1), error,
gradient);
return obj;
src/mlpack/methods/ann/layer/base_layer.hpp
+1 -1
View File
@@ -68,7 +68,7 @@ class BaseLayer : public Layer<InputType, OutputType>
/**
* Create the BaseLayer object.
*/
BaseLayer()
BaseLayer() : Layer<InputType, OutputType>()
{
// Nothing to do here.
}
src/mlpack/methods/ann/layer/multi_layer.hpp
+10 -4
View File
@@ -26,6 +26,12 @@ template<typename InputType, typename OutputType>
class MultiLayer : public Layer<InputType, OutputType>
{
public:
// TODO: implement these types of things...
MultiLayer();
MultiLayer(const MultiLayer& other);
MultiLayer(MultiLayer&& other);
MultiLayer& operator=(const MultiLayer& other);
MultiLayer& operator=(MultiLayer&& other);
virtual ~MultiLayer()
{
@@ -33,8 +39,6 @@ class MultiLayer : public Layer<InputType, OutputType>
delete network[i];
}
// TODO: implement these types of things...
// MultiLayer(const MultiLayer& other);
virtual MultiLayer* Clone() const { return new MultiLayer(*this); }
@@ -223,6 +227,7 @@ network; }
ar(cereal::base_class<Layer<InputType, OutputType>>(this));
ar(CEREAL_VECTOR_POINTER(network));
ar(CEREAL_NVP(inSize));
ar(CEREAL_NVP(totalInputSize));
ar(CEREAL_NVP(totalOutputSize));
@@ -230,6 +235,7 @@ network; }
{
layerOutputMatrix.clear();
layerDeltaMatrix.clear();
layerGradients.clear();
layerOutputs.resize(network.size(), OutputType());
layerDeltas.resize(network.size(), OutputType());
layerGradients.resize(network.size(), OutputType());
@@ -299,8 +305,8 @@ network; }
for (size_t i = 0; i < network.size(); ++i)
{
const size_t weightSize = network[i]->WeightSize();
MakeAlias(layerGradients[i], gradient.colptr(gradientStart), weightSize,
1);
MakeAlias(layerGradients[i], gradient.memptr() + gradientStart,
weightSize, 1);
gradientStart += weightSize;
}
}
src/mlpack/methods/ann/layer/multi_layer_impl.hpp
+125 -8
View File
@@ -18,11 +18,128 @@
namespace mlpack {
namespace ann {
template<typename InputType, typename OutputType>
MultiLayer<InputType, OutputType>::MultiLayer() :
inSize(0),
totalInputSize(0),
totalOutputSize(0)
{
// Nothing to do.
}
template<typename InputType, typename OutputType>
MultiLayer<InputType, OutputType>::MultiLayer(const MultiLayer& other) :
Layer<InputType, OutputType>(other),
inSize(other.inSize),
totalInputSize(other.totalInputSize),
totalOutputSize(other.totalOutputSize),
layerOutputMatrix(other.layerOutputMatrix),
layerDeltaMatrix(other.layerDeltaMatrix)
{
// Copy each layer.
for (size_t i = 0; i < other.network.size(); ++i)
network.push_back(other.network[i]->Clone());
// Ensure that the aliases for layers during passes have the right size.
layerOutputs.resize(network.size(), OutputType());
layerDeltas.resize(network.size(), OutputType());
layerGradients.resize(network.size(), OutputType());
// layerOutputs, layerDeltas, and layerGradients will be reset the next time
// Forward(), Backward(), or Gradient() is called.
}
template<typename InputType, typename OutputType>
MultiLayer<InputType, OutputType>::MultiLayer(MultiLayer&& other) :
Layer<InputType, OutputType>(other),
network(std::move(other.network)),
inSize(std::move(other.inSize)),
totalInputSize(std::move(other.totalInputSize)),
totalOutputSize(std::move(other.totalOutputSize)),
layerOutputMatrix(std::move(other.layerOutputMatrix)),
layerDeltaMatrix(std::move(other.layerDeltaMatrix))
{
// Ensure that the aliases for layers during passes have the right size.
layerOutputs.resize(network.size(), OutputType());
layerDeltas.resize(network.size(), OutputType());
layerGradients.resize(network.size(), OutputType());
// layerOutputs, layerDeltas, and layerGradients will be reset the next time
// Forward(), Backward(), or Gradient() is called.
other.layerOutputs.clear();
other.layerDeltas.clear();
other.layerGradients.clear();
}
template<typename InputType, typename OutputType>
MultiLayer<InputType, OutputType>&
MultiLayer<InputType, OutputType>::operator=(const MultiLayer& other)
{
if (this != &other)
{
Layer<InputType, OutputType>::operator=(other);
network.clear();
layerOutputs.clear();
layerDeltas.clear();
layerGradients.clear();
inSize = other.inSize;
totalInputSize = other.totalInputSize;
totalOutputSize = other.totalOutputSize;
layerOutputMatrix = other.layerOutputMatrix;
layerDeltaMatrix = other.layerDeltaMatrix;
for (size_t i = 0; i < other.network.size(); ++i)
network.push_back(other.network[i]->Clone());
// Ensure that the aliases for layers during passes have the right size.
layerOutputs.resize(network.size(), OutputType());
layerDeltas.resize(network.size(), OutputType());
layerGradients.resize(network.size(), OutputType());
}
return *this;
}
template<typename InputType, typename OutputType>
MultiLayer<InputType, OutputType>&
MultiLayer<InputType, OutputType>::operator=(MultiLayer&& other)
{
if (this != &other)
{
Layer<InputType, OutputType>::operator=(other);
layerOutputs.clear();
layerDeltas.clear();
layerGradients.clear();
inSize = std::move(other.inSize);
totalInputSize = std::move(other.totalInputSize);
totalOutputSize = std::move(other.totalOutputSize);
// TODO: network ownerships??
network = std::move(other.network);
layerOutputs.resize(network.size(), OutputType());
layerDeltas.resize(network.size(), OutputType());
layerGradients.resize(network.size(), OutputType());
other.layerOutputs.clear();
other.layerDeltas.clear();
other.layerGradients.clear();
}
return *this;
}
template<typename InputType, typename OutputType>
void MultiLayer<InputType, OutputType>::Forward(
const InputType& input, OutputType& output)
{
Forward(input, output, 0, network.size());
Forward(input, output, 0, network.size() - 1);
}
template<typename InputType, typename OutputType>
@@ -38,17 +155,17 @@ void MultiLayer<InputType, OutputType>::Forward(
// Note that we use `output` for the last layer; layerOutputs is only used for
// intermediate values between layers.
if ((end - start) > 1)
if ((end - start) > 0)
{
// Initialize memory for the forward pass (if needed).
InitializeForwardPassMemory(input.n_cols);
network[start]->Forward(input, layerOutputs.front());
for (size_t i = start; i < end - 1; ++i)
network[start]->Forward(input, layerOutputs[start]);
for (size_t i = start + 1; i < end; ++i)
network[i]->Forward(layerOutputs[i - 1], layerOutputs[i]);
network[end]->Forward(layerOutputs.back(), output);
network[end]->Forward(layerOutputs[end - 1], output);
}
else if ((end - start) == 1)
else if ((end - start) == 0 && network.size() > 0)
{
network[start]->Forward(input, output);
}
@@ -71,7 +188,7 @@ void MultiLayer<InputType, OutputType>::Backward(
network.back()->Backward(input, gy, layerDeltas.back());
for (size_t i = network.size() - 2; i > 0; --i)
network[i]->Backward(layerOutputs[i], layerDeltas[i + 1], layerDeltas[i]);
network[0]->Backward(layerOutputs[0], layerDeltas[1], g);
}
else if (network.size() == 1)
{
@@ -99,7 +216,7 @@ void MultiLayer<InputType, OutputType>::Gradient(
network.front()->Gradient(input, layerDeltas[1], layerGradients.front());
for (size_t i = 1; i < network.size() - 1; ++i)
{
network[i]->Gradient(layerOutputs[i], layerDeltas[i + 1],
network[i]->Gradient(layerOutputs[i - 1], layerDeltas[i + 1],
layerGradients[i]);
}
network.back()->Gradient(layerOutputs[network.size() - 2], error,