diff --git a/src/mlpack/methods/ann/ffn.hpp b/src/mlpack/methods/ann/ffn.hpp index b29f79376e..3dc050acb2 100644 --- a/src/mlpack/methods/ann/ffn.hpp +++ b/src/mlpack/methods/ann/ffn.hpp @@ -279,7 +279,6 @@ class FFN * * @param args The layer parameter. */ - //! TODO: This should invalidate cached parameters template void Add(Args... args) { @@ -295,7 +294,6 @@ class FFN * * @param layer The Layer to be added to the model. */ - //! TODO: this should invalidate cached parameters //! TODO: if weights are set in this layer, we should copy them and update our //cached parameters void Add(Layer* layer) @@ -500,8 +498,13 @@ class FFN //! `totalInputSize` and `totalOutputSize` are valid. bool inputDimensionsAreSet; + //! Cached total number of input elements across all layers (for deltaMatrix + //! and layerDeltas). size_t totalInputSize; + //! Cached total number of output elements across all layers (for + //! layerOutputMatrix and layerOutputs). size_t totalOutputSize; + //! Locally-stored output parameter object. This holds the results of //! Forward() for each layer, all in one matrix. OutputType layerOutputMatrix; @@ -513,11 +516,9 @@ class FFN OutputType deltaMatrix; std::vector layerDeltas; - //! Locally-stored gradient object. This holds the results of Gradient() for - //! each layer, all in one matrix. - OutputType gradientMatrix; - //! Aliases to different parts of the gradientOutputMatrix, for convenience. - //! gradientOutputs[i] stores the results of Gradient() for layer i. + //! Aliases to different parts of the gradient, for convenience. + //! gradientOutputs[i] stores the results of Gradient() for layer i. These + //! elements are only valid inside of Gradient(). std::vector layerGradients; }; // class FFN diff --git a/src/mlpack/methods/ann/ffn_impl.hpp b/src/mlpack/methods/ann/ffn_impl.hpp index c03322b083..bc40b4ff05 100644 --- a/src/mlpack/methods/ann/ffn_impl.hpp +++ b/src/mlpack/methods/ann/ffn_impl.hpp @@ -38,7 +38,10 @@ FFN< reset(false), numFunctions(0), deterministic(false), - inputDimensionsAreSet(false) + layerMemoryIsSet(false), + inputDimensionsAreSet(false), + totalInputSize(0), + totalOutputSize(0) { /* Nothing to do here. */ } @@ -62,19 +65,21 @@ FFN< responses(network.responses), numFunctions(network.numFunctions), error(network.error), - deterministic(network.deterministic) + deterministic(network.deterministic), + // These will be set correctly in the first Forward() call. + layerMemoryIsSet(false), + inputDimensionsAreSet(false), + totalInputSize(0), + totalOutputSize(0) { // Build new layers according to source network for (size_t i = 0; i < network.network.size(); ++i) { this->network.push_back(network.network[i]->Clone()); - // TODO: dig into this - // this will need the weights to be set right for each layer... - //ResetUpdate(this->network.back()); } + // Initialize cached matrix vectors to the right size. layerOutputs.resize(this->network.size(), OutputType()); - totalOutputSize = network.totalOutputSize; layerDeltas.resize(this->network.size(), OutputType()); layerGradients.resize(this->network.size(), OutputType()); }; @@ -99,15 +104,18 @@ FFN< responses(std::move(network.responses)), numFunctions(network.numFunctions), error(std::move(network.error)), - deterministic(network.deterministic), - layerOutputMatrix(std::move(network.layerOutputMatrix)), - layerOutputs(std::move(network.layerOutputs)), - deltaMatrix(std::move(network.deltaMatrix)), - layerDeltas(std::move(network.layerDeltas)), - gradientMatrix(std::move(network.gradientMatrix)), - layerGradients(std::move(network.layerGradients)) + deterministic(std::move(network.deterministic)), + // Aliases will not be correct after a std::move(), so we will manually + // reset them. + layerMemoryIsSet(false), + inputDimensionsAreSet(std::move(network.inputDimensionsAreSet)), + totalInputSize(std::move(network.totalInputSize)), + totalOutputSize(std::move(network.totalOutputSize)) { - // Nothing else to do. + // Initialize cached matrix parameters to the right size. + layerOutputs.resize(this->network.size(), OutputType()); + layerDeltas.resize(this->network.size(), OutputType()); + layerGradients.resize(this->network.size(), OutputType()); }; template(optimizer, this->predictors.n_cols); + // Make sure to set all layers in training mode. + for (size_t i = 0; i < network.size(); ++i) + network[i]->Deterministic() = false; + // Train the model. Timer::Start("ffn_optimization"); const double out = optimizer.Optimize(*this, parameters, callbacks...); @@ -392,7 +404,8 @@ double FFN< // error's size will be set correctly by outputLayer.Backward(). outputLayer.Backward(layerOutputs[network.size() - 1], targets, error); - gradients = arma::zeros(parameters.n_rows, parameters.n_cols); + // This does nothing if the gradient's size is already set right. + gradients.zeros(parameters.n_rows, parameters.n_cols); Backward(); Gradient(inputs, gradients); @@ -411,25 +424,71 @@ void FFN< OutputType >::Predict(InputType predictors, OutputType& results, const size_t batchSize) { - // TODO: actually care about batchSize - - // TODO: this may not be needed here - if (parameters.is_empty()) - InitializeWeights(); - if (!deterministic) { deterministic = true; ResetDeterministic(); } + // If there are no weights for the network at all, we need to initialize the + // weights. + if (parameters.is_empty()) + InitializeWeights(); + + // If each layer's memory has not been set with SetWeights(), pass through the + // network and do that. + if (!layerMemoryIsSet) + SetLayerMemory(); + + // Now, pass the input dimensions through the network if needed. + if (network.front()->InputDimensions() != inputDimensions || + !inputDimensionsAreSet) + { + // If the input dimensions are completely unset, then assume our input is + // flat. + if (inputDimensions.size() == 0) + inputDimensions = { predictors.n_rows }; + + totalInputSize = std::accumulate(inputDimensions.begin(), + inputDimensions.end(), 0); + + // TODO: improve this error message... + Log::Assert(totalInputSize == predictors.n_rows, "FFN::Predict(): input " + "size does not match expected size set with InputDimensions()!"); + + network.front()->InputDimensions() = inputDimensions; + totalInputSize += network[0]->OutputSize(); + totalOutputSize = network[0]->OutputSize(); + for (size_t i = 1; i < network.size(); ++i) + { + network[i]->InputDimensions() = network[i - 1]->OutputDimensions(); + const size_t layerOutputSize = network[i]->OutputSize(); + + // If we are not at the last layer, then this output is the input to the + // next layer. + if (i != network.size() - 1) + totalInputSize += layerOutputSize; + + totalOutputSize += layerOutputSize; + } + + inputDimensionsAreSet = true; + } + + // TODO: this is the problem! + // we have to make sure the dimensions are set right here. + results.set_size(network.back()->OutputSize(), predictors.n_cols); - for (size_t i = 0; i < predictors.n_cols; ++i) + for (size_t i = 0; i < predictors.n_cols; i += batchSize) { - InputType predictorAlias(predictors.colptr(i), predictors.n_rows, 1, false, - true); - OutputType resultAlias(results.colptr(i), results.n_rows, 1, false, true); + const size_t effectiveBatchSize = std::min(batchSize, + size_t(predictors.n_cols) - i); + + InputType predictorAlias(predictors.colptr(i), predictors.n_rows, + effectiveBatchSize, false, true); + OutputType resultAlias(results.colptr(i), results.n_rows, + effectiveBatchSize, false, true); Forward(predictorAlias, resultAlias); } @@ -794,16 +853,19 @@ void FFN< layerDeltas.clear(); layerDeltas.resize(network.size(), OutputType()); - gradientMatrix.clear(); layerGradients.clear(); layerGradients.resize(network.size(), OutputType()); deterministic = true; + layerMemoryIsSet = false; + inputDimensionsAreSet = false; - // Reset each layer so its weights are set right. - // TODO: should we just do that in serialize()? -// for (size_t i = 0; i < network.size(); ++i) -// network[i]->Reset(); + // We'll recompute this during the first call to Forward(). + totalInputSize = 0; + totalOutputSize = 0; + + // The weights in parameters will be correctly set for each layer in the + // first call to Forward(). } } @@ -829,13 +891,26 @@ void FFN< std::swap(numFunctions, network.numFunctions); std::swap(error, network.error); std::swap(deterministic, network.deterministic); - std::swap(layerOutputMatrix, network.layerOutputMatrix); + std::swap(inputDimensionsAreSet, network.inputDimensionsAreSet); + std::swap(totalInputSize, network.totalInputSize); std::swap(totalOutputSize, network.totalOutputSize); + std::swap(layerOutputMatrix, network.layerOutputMatrix); std::swap(layerOutputs, network.layerOutputs); std::swap(deltaMatrix, network.deltaMatrix); std::swap(layerDeltas, network.layerDeltas); - std::swap(gradientMatrix, network.gradientMatrix); std::swap(layerGradients, network.layerGradients); + + // std::swap() will not preserve Armadillo aliases correctly, so we will reset + // those. + layerMemoryIsSet = false; + layerOutputs.resize(this->network.size(), OutputType()); + layerDeltas.resize(this->network.size(), OutputType()); + layerGradients.resize(this->network.size(), OutputType()); + + network.layerMemoryIsSet = false; + network.layerOutputs.resize(network.network.size(), OutputType()); + network.layerDeltas.resize(network.network.size(), OutputType()); + network.layerGradients.resize(network.network.size(), OutputType()); }; // Initialize memory to be used for storing the outputs of each layer, if @@ -936,6 +1011,14 @@ void FFN< OutputType >::InitializeGradientPassMemory(OutputType& gradient) { + // Sanity check: make sure the gradient has the right size. + size_t totalWeightSize = 0; + for (size_t i = 0; i < layerGradients.size(); ++i) + totalWeightSize += network[i]->WeightSize(); + // TODO: cache totalWeightSize ... + if (gradient.n_elem != totalWeightSize) + gradient.set_size(totalWeightSize); + // We need to initialize the aliases `layerGradients()` for each layer. size_t start = 0; for (size_t i = 0; i < layerGradients.size(); ++i) diff --git a/src/mlpack/methods/ann/layer/dropconnect.hpp b/src/mlpack/methods/ann/layer/dropconnect.hpp index 6f15268757..a1d0305d0e 100644 --- a/src/mlpack/methods/ann/layer/dropconnect.hpp +++ b/src/mlpack/methods/ann/layer/dropconnect.hpp @@ -125,6 +125,16 @@ class DropConnectType : public Layer this->outputDimensions = baseLayer->OutputDimensions(); } + size_t WeightSize() const + { + return baseLayer->WeightSize(); + } + + void SetWeights(typename OutputType::elem_type* weightsPtr) + { + baseLayer->SetWeights(weightsPtr); + } + /** * Serialize the layer. */ diff --git a/src/mlpack/methods/ann/layer/layer.hpp b/src/mlpack/methods/ann/layer/layer.hpp index 3ead5d684c..0d7d277586 100644 --- a/src/mlpack/methods/ann/layer/layer.hpp +++ b/src/mlpack/methods/ann/layer/layer.hpp @@ -194,7 +194,11 @@ class Layer //! Get the input dimensions. const std::vector& InputDimensions() const { return inputDimensions; } //! Modify the input dimensions. - std::vector& InputDimensions() { return inputDimensions; } + std::vector& InputDimensions() + { + validOutputDimensions = false; + return inputDimensions; + } //! Get the output dimensions. const std::vector& OutputDimensions() diff --git a/src/mlpack/tests/feedforward_network_test.cpp b/src/mlpack/tests/feedforward_network_test.cpp index 4a93912dce..e44c91c575 100644 --- a/src/mlpack/tests/feedforward_network_test.cpp +++ b/src/mlpack/tests/feedforward_network_test.cpp @@ -902,4 +902,5 @@ TEST_CASE("FFNCheckInputShapeTest", "[FeedForwardNetworkTest]") ens::DE opt(200, 1000, 0.6, 0.8, 1e-5); REQUIRE_THROWS_AS(model.Train(trainData, trainLabels, opt), std::logic_error); + // TODO: check expectedMsg ? }