diff --git a/HISTORY.md b/HISTORY.md index e930dba62f..fc580bf68f 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,5 +1,6 @@ ### mlpack ?.?.? ###### ????-??-?? + * Fix Visual Studio compilation issue (#1443). ### mlpack 3.0.2 ###### 2018-06-08 diff --git a/src/mlpack/core/math/make_alias.hpp b/src/mlpack/core/math/make_alias.hpp index 7ed0687141..a83620226f 100644 --- a/src/mlpack/core/math/make_alias.hpp +++ b/src/mlpack/core/math/make_alias.hpp @@ -16,6 +16,19 @@ namespace mlpack { namespace math { +/** + * Make an alias of a dense cube. If strict is true, then the alias cannot be + * resized or pointed at new memory. + */ +template +arma::Cube MakeAlias(arma::Cube& input, + const bool strict = true) +{ + // Use the advanced constructor. + return arma::Cube(input.memptr(), input.n_rows, input.n_cols, + input.n_slices, false, strict); +} + /** * Make an alias of a dense matrix. If strict is true, then the alias cannot be * resized or pointed at new memory. diff --git a/src/mlpack/methods/ann/ffn.hpp b/src/mlpack/methods/ann/ffn.hpp index 46db64b77a..840a1c453b 100644 --- a/src/mlpack/methods/ann/ffn.hpp +++ b/src/mlpack/methods/ann/ffn.hpp @@ -23,6 +23,7 @@ #include "visitor/reset_visitor.hpp" #include "visitor/weight_size_visitor.hpp" #include "visitor/copy_visitor.hpp" +#include "visitor/loss_visitor.hpp" #include "init_rules/network_init.hpp" @@ -371,6 +372,9 @@ class FFN //! Locally-stored output height visitor. OutputHeightVisitor outputHeightVisitor; + //! Locally-stored loss visitor + LossVisitor lossVisitor; + //! Locally-stored reset visitor. ResetVisitor resetVisitor; diff --git a/src/mlpack/methods/ann/ffn_impl.hpp b/src/mlpack/methods/ann/ffn_impl.hpp index c0811f4716..0a00f05466 100644 --- a/src/mlpack/methods/ann/ffn_impl.hpp +++ b/src/mlpack/methods/ann/ffn_impl.hpp @@ -132,6 +132,11 @@ double FFN::Backward( double res = outputLayer.Forward(std::move(boost::apply_visitor( outputParameterVisitor, network.back())), std::move(targets)); + for (size_t i = 0; i < network.size(); ++i) + { + res += boost::apply_visitor(lossVisitor, network[i]); + } + outputLayer.Backward(std::move(boost::apply_visitor(outputParameterVisitor, network.back())), std::move(targets), std::move(error)); @@ -212,6 +217,11 @@ double FFN::Evaluate( std::move(boost::apply_visitor(outputParameterVisitor, network.back())), std::move(responses.cols(begin, begin + batchSize - 1))); + for (size_t i = 0; i < network.size(); ++i) + { + res += boost::apply_visitor(lossVisitor, network[i]); + } + return res; } diff --git a/src/mlpack/methods/ann/gan_impl.hpp b/src/mlpack/methods/ann/gan_impl.hpp index 673cb74b68..1160dba6f9 100644 --- a/src/mlpack/methods/ann/gan_impl.hpp +++ b/src/mlpack/methods/ann/gan_impl.hpp @@ -60,19 +60,21 @@ GAN::GAN( responses.set_size(1, predictors.n_cols); responses.ones(); - discriminator.predictors.set_size(predictors.n_rows, predictors.n_cols + 1); + discriminator.predictors.set_size(predictors.n_rows, + predictors.n_cols + batchSize); discriminator.predictors.cols(0, predictors.n_cols - 1) = predictors; - discriminator.responses.set_size(1, predictors.n_cols + 1); + discriminator.responses.set_size(1, predictors.n_cols + batchSize); discriminator.responses.ones(); - discriminator.responses(predictors.n_cols) = 0; + discriminator.responses.cols(predictors.n_cols, + predictors.n_cols + batchSize - 1) = arma::zeros(1, batchSize); numFunctions = predictors.n_cols; - noise.set_size(noiseDim, 1); + noise.set_size(noiseDim, batchSize); - generator.predictors.set_size(noiseDim, 1); - generator.responses.set_size(predictors.n_rows, 1); + generator.predictors.set_size(noiseDim, batchSize); + generator.responses.set_size(predictors.n_rows, batchSize); } template @@ -98,7 +100,7 @@ void GAN::Reset() generator.Parameters() = arma::mat(parameter.memptr(), genWeights, 1, false, false); discriminator.Parameters() = arma::mat(parameter.memptr() + genWeights, - discWeights, 1 , false, false); + discWeights, 1, false, false); // Initialize the parameters generator networkInit.Initialize(generator.network, parameter); @@ -127,8 +129,10 @@ double GAN::Evaluate( if (!reset) Reset(); - currentInput = this->predictors.unsafe_col(i); - currentTarget = this->responses.unsafe_col(i); + currentInput = arma::mat(predictors.memptr() + (i * predictors.n_rows), + predictors.n_rows, batchSize, false, false); + currentTarget = arma::mat(responses.memptr() + i, 1, batchSize, false, + false); discriminator.Forward(std::move(currentInput)); double res = discriminator.outputLayer.Forward( @@ -139,12 +143,15 @@ double GAN::Evaluate( noise.imbue( [&]() { return noiseFunction();} ); generator.Forward(std::move(noise)); - discriminator.predictors.col(numFunctions) = boost::apply_visitor( - outputParameterVisitor, generator.network.back());; - discriminator.Forward(std::move(discriminator.predictors.col(numFunctions))); - discriminator.responses(numFunctions) = 0; + discriminator.predictors.cols(numFunctions, numFunctions + batchSize - 1) = + boost::apply_visitor(outputParameterVisitor, generator.network.back()); + discriminator.Forward(std::move(discriminator.predictors.cols(numFunctions, + numFunctions + batchSize - 1))); + discriminator.responses.cols(numFunctions, numFunctions + batchSize - 1) = + arma::zeros(1, batchSize); - currentTarget = discriminator.responses.unsafe_col(numFunctions); + currentTarget = arma::mat(discriminator.responses.memptr() + numFunctions, + 1, batchSize, false, false); res += discriminator.outputLayer.Forward( std::move(boost::apply_visitor( outputParameterVisitor, @@ -190,13 +197,13 @@ Gradient(const arma::mat& /*parameters*/, const size_t i, arma::mat& gradient, // Get the gradients of the Discriminator. discriminator.Gradient(discriminator.parameter, i, gradientDiscriminator, batchSize); - noise.imbue( [&]() { return noiseFunction();} ); generator.Forward(std::move(noise)); - discriminator.predictors.col(numFunctions) = boost::apply_visitor( - outputParameterVisitor, generator.network.back()); + discriminator.predictors.cols(numFunctions, numFunctions + batchSize - 1) = + boost::apply_visitor(outputParameterVisitor, generator.network.back()); - discriminator.responses(numFunctions) = 0; + discriminator.responses.cols(numFunctions, numFunctions + batchSize - 1) = + arma::zeros(1, batchSize); discriminator.Gradient(discriminator.parameter, numFunctions, noiseGradientDiscriminator, batchSize); gradientDiscriminator += noiseGradientDiscriminator; @@ -205,7 +212,8 @@ Gradient(const arma::mat& /*parameters*/, const size_t i, arma::mat& gradient, { // Minimize -log(D(G(noise))). // Pass the error from Discriminator to Generator. - discriminator.responses(numFunctions) = 1; + discriminator.responses.cols(numFunctions, numFunctions + batchSize - 1) = + arma::ones(1, batchSize); discriminator.Gradient(discriminator.parameter, numFunctions, noiseGradientDiscriminator, batchSize); generator.error = boost::apply_visitor(deltaVisitor, @@ -213,7 +221,7 @@ Gradient(const arma::mat& /*parameters*/, const size_t i, arma::mat& gradient, generator.Predictors() = noise; generator.ResetGradients(gradientGenerator); - generator.Gradient(generator.parameter, 0, gradientGenerator, noise.n_cols); + generator.Gradient(generator.parameter, 0, gradientGenerator, batchSize); gradientGenerator *= multiplier; } diff --git a/src/mlpack/methods/ann/layer/CMakeLists.txt b/src/mlpack/methods/ann/layer/CMakeLists.txt index 6523499f74..6644a8138b 100644 --- a/src/mlpack/methods/ann/layer/CMakeLists.txt +++ b/src/mlpack/methods/ann/layer/CMakeLists.txt @@ -65,8 +65,6 @@ set(SOURCES multiply_constant_impl.hpp multiply_merge.hpp multiply_merge_impl.hpp - negative_log_likelihood.hpp - negative_log_likelihood_impl.hpp parametric_relu.hpp parametric_relu_impl.hpp recurrent.hpp @@ -75,6 +73,8 @@ set(SOURCES recurrent_attention_impl.hpp reinforce_normal.hpp reinforce_normal_impl.hpp + reparametrization.hpp + reparametrization_impl.hpp select.hpp select_impl.hpp sequential.hpp diff --git a/src/mlpack/methods/ann/layer/add.hpp b/src/mlpack/methods/ann/layer/add.hpp index 1db7e95a3e..47de360eed 100644 --- a/src/mlpack/methods/ann/layer/add.hpp +++ b/src/mlpack/methods/ann/layer/add.hpp @@ -82,11 +82,6 @@ class Add //! Modify the parameters. OutputDataType& Parameters() { return weights; } - //! Get the input parameter. - InputDataType const& InputParameter() const { return inputParameter; } - //! Modify the input parameter. - InputDataType& InputParameter() { return inputParameter; } - //! Get the output parameter. OutputDataType const& OutputParameter() const { return outputParameter; } //! Modify the output parameter. @@ -121,9 +116,6 @@ class Add //! Locally-stored gradient object. OutputDataType gradient; - //! Locally-stored input parameter object. - InputDataType inputParameter; - //! Locally-stored output parameter object. OutputDataType outputParameter; }; // class Add diff --git a/src/mlpack/methods/ann/layer/add_merge.hpp b/src/mlpack/methods/ann/layer/add_merge.hpp index 44aee7bcac..24120645b0 100644 --- a/src/mlpack/methods/ann/layer/add_merge.hpp +++ b/src/mlpack/methods/ann/layer/add_merge.hpp @@ -46,8 +46,9 @@ class AddMerge * Create the AddMerge object using the specified parameters. * * @param model Expose all the network modules. + * @param run Call the Forward/Backward method before the output is merged. */ - AddMerge(const bool model = false); + AddMerge(const bool model = false, const bool run = true); //! Destructor to release allocated memory. ~AddMerge(); @@ -60,7 +61,7 @@ class AddMerge * @param output Resulting output activation. */ template - void Forward(const InputType&& /* input */, OutputType&& output); + void Forward(InputType&& /* input */, OutputType&& output); /** * Ordinary feed backward pass of a neural network, calculating the function @@ -77,19 +78,16 @@ class AddMerge arma::Mat&& g); /* - * Add a new module to the model. + * Calculate the gradient using the output delta and the input activation. * - * @param layer The Layer to be added to the model. + * @param input The input parameter used for calculating the gradient. + * @param error The calculated error. + * @param gradient The calculated gradient. */ - void Add(LayerTypes layer) { network.push_back(layer); } - - /* - * Add a new module to the model. - * - * @param layer The Layer to be added to the model. - */ - template - void Add(const LayerType& layer) { network.push_back(new LayerType(layer)); } + template + void Gradient(arma::Mat&& input, + arma::Mat&& error, + arma::Mat&& gradient); /* * Add a new module to the model. @@ -99,6 +97,13 @@ class AddMerge template void Add(Args... args) { network.push_back(new LayerType(args...)); } + /* + * Add a new module to the model. + * + * @param layer The Layer to be added to the model. + */ + void Add(LayerTypes layer) { network.push_back(layer); } + //! Get the input parameter. InputDataType const& InputParameter() const { return inputParameter; } //! Modify the input parameter. @@ -125,6 +130,11 @@ class AddMerge return empty; } + //! Get the parameters. + OutputDataType const& Parameters() const { return weights; } + //! Modify the parameters. + OutputDataType& Parameters() { return weights; } + /** * Serialize the layer. */ @@ -135,6 +145,10 @@ class AddMerge //! Parameter which indicates if the modules should be exposed. bool model; + //! Parameter which indicates if the Forward/Backward method should be called + //! before merging the output. + bool run; + //! We need this to know whether we should delete the layer in the destructor. bool ownsLayer; @@ -156,11 +170,17 @@ class AddMerge //! Locally-stored delta object. OutputDataType delta; + //! Locally-stored gradient object. + OutputDataType gradient; + //! Locally-stored input parameter object. InputDataType inputParameter; //! Locally-stored output parameter object. OutputDataType outputParameter; + + //! Locally-stored weight object. + OutputDataType weights; }; // class AddMerge } // namespace ann diff --git a/src/mlpack/methods/ann/layer/add_merge_impl.hpp b/src/mlpack/methods/ann/layer/add_merge_impl.hpp index 1e67c32371..71437d6859 100644 --- a/src/mlpack/methods/ann/layer/add_merge_impl.hpp +++ b/src/mlpack/methods/ann/layer/add_merge_impl.hpp @@ -16,13 +16,18 @@ // In case it hasn't yet been included. #include "add_merge.hpp" +#include "../visitor/forward_visitor.hpp" +#include "../visitor/backward_visitor.hpp" +#include "../visitor/gradient_visitor.hpp" + namespace mlpack { namespace ann /** Artificial Neural Network. */ { template AddMerge::AddMerge( - const bool model) : model(model), ownsLayer(!model) + const bool model, const bool run) : + model(model), run(run), ownsLayer(!model) { // Nothing to do here. } @@ -42,10 +47,19 @@ template template void AddMerge::Forward( - const InputType&& /* input */, OutputType&& output) + InputType&& input, OutputType&& output) { - output = boost::apply_visitor(outputParameterVisitor, network.front()); + if (run) + { + for (size_t i = 0; i < network.size(); ++i) + { + boost::apply_visitor(ForwardVisitor(std::move(input), std::move( + boost::apply_visitor(outputParameterVisitor, network[i]))), + network[i]); + } + } + output = boost::apply_visitor(outputParameterVisitor, network.front()); for (size_t i = 1; i < network.size(); ++i) { output += boost::apply_visitor(outputParameterVisitor, network[i]); @@ -58,7 +72,41 @@ template void AddMerge::Backward( const arma::Mat&& /* input */, arma::Mat&& gy, arma::Mat&& g) { - g = gy; + if (run) + { + for (size_t i = 0; i < network.size(); ++i) + { + boost::apply_visitor(BackwardVisitor(std::move(boost::apply_visitor( + outputParameterVisitor, network[i])), std::move(gy), std::move( + boost::apply_visitor(deltaVisitor, network[i]))), network[i]); + } + + g = boost::apply_visitor(deltaVisitor, network[0]); + for (size_t i = 1; i < network.size(); ++i) + { + g += boost::apply_visitor(deltaVisitor, network[i]); + } + } + else + g = gy; +} + +template +template +void AddMerge::Gradient( + arma::Mat&& input, + arma::Mat&& error, + arma::Mat&& /* gradient */ ) +{ + if (run) + { + for (size_t i = 0; i < network.size(); ++i) + { + boost::apply_visitor(GradientVisitor(std::move(input), std::move(error)), + network[i]); + } + } } template&& gy, arma::Mat&& g); - //! Get the input parameter. - InputDataType const& InputParameter() const { return inputParameter; } - //! Modify the input parameter. - InputDataType& InputParameter() { return inputParameter; } - //! Get the output parameter. OutputDataType const& OutputParameter() const { return outputParameter; } //! Modify the output parameter. @@ -134,9 +129,6 @@ class AlphaDropout //! Locally-stored delta object. OutputDataType delta; - //! Locally-stored input parameter object. - InputDataType inputParameter; - //! Locally-stored output parameter object. OutputDataType outputParameter; diff --git a/src/mlpack/methods/ann/layer/atrous_convolution.hpp b/src/mlpack/methods/ann/layer/atrous_convolution.hpp index 79bf1def2d..52281086cc 100644 --- a/src/mlpack/methods/ann/layer/atrous_convolution.hpp +++ b/src/mlpack/methods/ann/layer/atrous_convolution.hpp @@ -130,11 +130,6 @@ class AtrousConvolution //! Modify the parameters. OutputDataType& Parameters() { return weights; } - //! Get the input parameter. - InputDataType const& InputParameter() const { return inputParameter; } - //! Modify the input parameter. - InputDataType& InputParameter() { return inputParameter; } - //! Get the output parameter. OutputDataType const& OutputParameter() const { return outputParameter; } //! Modify the output parameter. @@ -272,12 +267,15 @@ class AtrousConvolution } } - //! Locally-stored number of input units. + //! Locally-stored number of input channels. size_t inSize; - //! Locally-stored number of output units. + //! Locally-stored number of output channels. size_t outSize; + //! Locally-stored number of input units. + size_t batchSize; + //! Locally-stored filter/kernel width. size_t kW; @@ -344,9 +342,6 @@ class AtrousConvolution //! Locally-stored gradient object. OutputDataType gradient; - //! Locally-stored input parameter object. - InputDataType inputParameter; - //! Locally-stored output parameter object. OutputDataType outputParameter; }; // class AtrousConvolution diff --git a/src/mlpack/methods/ann/layer/atrous_convolution_impl.hpp b/src/mlpack/methods/ann/layer/atrous_convolution_impl.hpp index 5593a96f81..bac305fa33 100644 --- a/src/mlpack/methods/ann/layer/atrous_convolution_impl.hpp +++ b/src/mlpack/methods/ann/layer/atrous_convolution_impl.hpp @@ -118,7 +118,9 @@ void AtrousConvolution< OutputDataType >::Forward(const arma::Mat&& input, arma::Mat&& output) { - inputTemp = arma::cube(input.memptr(), inputWidth, inputHeight, inSize); + batchSize = input.n_cols; + inputTemp = arma::cube(const_cast&&>(input).memptr(), + inputWidth, inputHeight, inSize * batchSize, false, false); if (padW != 0 || padH != 0) { @@ -128,30 +130,41 @@ void AtrousConvolution< size_t wConv = ConvOutSize(inputWidth, kW, dW, padW, dilationW); size_t hConv = ConvOutSize(inputHeight, kH, dH, padH, dilationH); - output.set_size(wConv * hConv * outSize, 1); - outputTemp = arma::Cube(output.memptr(), wConv, hConv, outSize, - false, false); + output.set_size(wConv * hConv * outSize, batchSize); + outputTemp = arma::Cube(output.memptr(), wConv, hConv, + outSize * batchSize, false, false); outputTemp.zeros(); - for (size_t outMap = 0, outMapIdx = 0; outMap < outSize; outMap++) + for (size_t outMap = 0, outMapIdx = 0, batchCount = 0; outMap < + outSize * batchSize; outMap++) { + if (outMap != 0 && outMap % outSize == 0) + { + batchCount++; + outMapIdx = 0; + } + for (size_t inMap = 0; inMap < inSize; inMap++, outMapIdx++) { arma::Mat convOutput; + if (padW != 0 || padH != 0) { - ForwardConvolutionRule::Convolution(inputPaddedTemp.slice(inMap), - weight.slice(outMapIdx), convOutput, dW, dH, dilationW, dilationH); + ForwardConvolutionRule::Convolution(inputPaddedTemp.slice(inMap + + batchCount * inSize), weight.slice(outMapIdx), convOutput, dW, dH, + dilationW, dilationH); } else { - ForwardConvolutionRule::Convolution(inputTemp.slice(inMap), - weight.slice(outMapIdx), convOutput, dW, dH, dilationW, dilationH); + ForwardConvolutionRule::Convolution(inputTemp.slice(inMap + + batchCount * inSize), weight.slice(outMapIdx), convOutput, dW, dH, + dilationW, dilationH); } + outputTemp.slice(outMap) += convOutput; } - outputTemp.slice(outMap) += bias(outMap); + outputTemp.slice(outMap) += bias(outMap % outSize); } outputWidth = outputTemp.n_rows; @@ -175,16 +188,23 @@ void AtrousConvolution< >::Backward( const arma::Mat&& /* input */, arma::Mat&& gy, arma::Mat&& g) { - arma::cube mappedError(gy.memptr(), outputWidth, outputHeight, outSize, - false, false); + arma::cube mappedError(gy.memptr(), outputWidth, outputHeight, + outSize * batchSize, false, false); - g.set_size(inputTemp.n_rows * inputTemp.n_cols * inputTemp.n_slices, 1); + g.set_size(inputTemp.n_rows * inputTemp.n_cols * inSize, batchSize); gTemp = arma::Cube(g.memptr(), inputTemp.n_rows, inputTemp.n_cols, inputTemp.n_slices, false, false); gTemp.zeros(); - for (size_t outMap = 0, outMapIdx = 0; outMap < outSize; outMap++) + for (size_t outMap = 0, outMapIdx = 0, batchCount = 0; outMap < + outSize * batchSize; outMap++) { + if (outMap != 0 && outMap % outSize == 0) + { + batchCount++; + outMapIdx = 0; + } + for (size_t inMap = 0; inMap < inSize; inMap++, outMapIdx++) { arma::Mat output, rotatedFilter; @@ -195,14 +215,13 @@ void AtrousConvolution< if (padW != 0 || padH != 0) { - gTemp.slice(inMap) += output.submat(rotatedFilter.n_rows / 2, - rotatedFilter.n_cols / 2, - rotatedFilter.n_rows / 2 + gTemp.n_rows - 1, - rotatedFilter.n_cols / 2 + gTemp.n_cols - 1); + gTemp.slice(inMap + batchCount * inSize) += output.submat(padW, padH, + padW + gTemp.n_rows - 1, + padH + gTemp.n_cols - 1); } else { - gTemp.slice(inMap) += output; + gTemp.slice(inMap + batchCount * inSize) += output; } } } @@ -231,81 +250,75 @@ void AtrousConvolution< if (padW != 0 && padH != 0) { mappedError = arma::cube(error.memptr(), outputWidth / padW, - outputHeight / padH, outSize); + outputHeight / padH, outSize * batchSize, false, false); } else { mappedError = arma::cube(error.memptr(), outputWidth, - outputHeight, outSize); + outputHeight, outSize * batchSize, false, false); } gradient.set_size(weights.n_elem, 1); - gradientTemp = arma::Cube(gradient.memptr(), weight.n_rows, weight.n_cols, - weight.n_slices, false, false); + gradientTemp = arma::Cube(gradient.memptr(), weight.n_rows, + weight.n_cols, weight.n_slices, false, false); gradientTemp.zeros(); - for (size_t outMap = 0, outMapIdx = 0; outMap < outSize; outMap++) + for (size_t outMap = 0, outMapIdx = 0, batchCount = 0; outMap < + outSize * batchSize; outMap++) { - for (size_t inMap = 0, s = outMap; inMap < inSize; inMap++, outMapIdx++, - s += outSize) + if (outMap != 0 && outMap % outSize == 0) { - arma::Cube inputSlices; + batchCount++; + outMapIdx = 0; + } + + for (size_t inMap = 0; inMap < inSize; inMap++, outMapIdx++) + { + arma::Mat inputSlice; if (padW != 0 || padH != 0) { - inputSlices = inputPaddedTemp.slices(inMap, inMap); + inputSlice = inputPaddedTemp.slice(inMap + batchCount * inSize); } else { - inputSlices = inputTemp.slices(inMap, inMap); + inputSlice = inputTemp.slice(inMap + batchCount * inSize); } - arma::Cube deltaSlices = mappedError.slices(outMap, outMap); + arma::Mat deltaSlice = mappedError.slice(outMap); - arma::Cube output, reducedOutput; - GradientConvolutionRule::Convolution(inputSlices, deltaSlices, + arma::Mat output; + GradientConvolutionRule::Convolution(inputSlice, deltaSlice, output, dW, dH, 1, 1); - arma::Mat reducedMat; - reducedOutput = arma::zeros >(kW, kH, output.n_slices); - for (size_t j = 0; j < output.n_slices; j++) + if (dilationH > 1) { - reducedMat = output.slice(j); - if (dilationH > 1) - { - for (size_t i = 1; i < reducedMat.n_cols; i++){ - reducedMat.shed_cols(i, i + dilationH - 2); - } + for (size_t i = 1; i < output.n_cols; i++){ + output.shed_cols(i, i + dilationH - 2); } - if (dilationW > 1) - { - for (size_t i = 1; i < reducedMat.n_rows; i++){ - reducedMat.shed_rows(i, i + dilationW - 2); - } + } + if (dilationW > 1) + { + for (size_t i = 1; i < output.n_rows; i++){ + output.shed_rows(i, i + dilationW - 2); } - reducedOutput.slice(j) = reducedMat; } if ((padW != 0 || padH != 0) && - (gradientTemp.n_rows < reducedOutput.n_rows && - gradientTemp.n_cols < reducedOutput.n_cols)) + (gradientTemp.n_rows < output.n_rows && + gradientTemp.n_cols < output.n_cols)) { - for (size_t i = 0; i < reducedOutput.n_slices; i++) - { - gradientTemp.slice(s) += reducedOutput.slice(i).submat( - reducedOutput.n_rows / 2, reducedOutput.n_cols / 2, - reducedOutput.n_rows / 2 + gradientTemp.n_rows - 1, - reducedOutput.n_cols / 2 + gradientTemp.n_cols - 1); - } + gradientTemp.slice(outMapIdx) += output.submat(padW, padH, + padW + gradientTemp.n_rows - 1, + padH + gradientTemp.n_cols - 1); } else { - for (size_t i = 0; i < reducedOutput.n_slices; i++) - gradientTemp.slice(s) += reducedOutput.slice(i); + gradientTemp.slice(outMapIdx) += output; } } - gradient.submat(weight.n_elem + outMap, 0, weight.n_elem + outMap, 0) = - arma::accu(mappedError.slices(outMap, outMap)); + gradient.submat(weight.n_elem + (outMap % outSize), 0, weight.n_elem + + (outMap % outSize), 0) = arma::accu(mappedError.slice(outMap)); } } @@ -328,6 +341,7 @@ void AtrousConvolution< { ar & BOOST_SERIALIZATION_NVP(inSize); ar & BOOST_SERIALIZATION_NVP(outSize); + ar & BOOST_SERIALIZATION_NVP(batchSize); ar & BOOST_SERIALIZATION_NVP(kW); ar & BOOST_SERIALIZATION_NVP(kH); ar & BOOST_SERIALIZATION_NVP(dW); diff --git a/src/mlpack/methods/ann/layer/base_layer.hpp b/src/mlpack/methods/ann/layer/base_layer.hpp index bcb230f3f4..780df1f1a4 100644 --- a/src/mlpack/methods/ann/layer/base_layer.hpp +++ b/src/mlpack/methods/ann/layer/base_layer.hpp @@ -88,11 +88,6 @@ class BaseLayer g = gy % derivative; } - //! Get the input parameter. - InputDataType const& InputParameter() const { return inputParameter; } - //! Modify the input parameter. - InputDataType& InputParameter() { return inputParameter; } - //! Get the output parameter. OutputDataType const& OutputParameter() const { return outputParameter; } //! Modify the output parameter. @@ -116,9 +111,6 @@ class BaseLayer //! Locally-stored delta object. OutputDataType delta; - //! Locally-stored input parameter object. - InputDataType inputParameter; - //! Locally-stored output parameter object. OutputDataType outputParameter; }; // class BaseLayer diff --git a/src/mlpack/methods/ann/layer/batch_norm.hpp b/src/mlpack/methods/ann/layer/batch_norm.hpp index 662619ef80..9b3932e196 100644 --- a/src/mlpack/methods/ann/layer/batch_norm.hpp +++ b/src/mlpack/methods/ann/layer/batch_norm.hpp @@ -112,11 +112,6 @@ class BatchNorm //! Modify the parameters. OutputDataType& Parameters() { return weights; } - //! Get the input parameter. - InputDataType const& InputParameter() const { return inputParameter; } - //! Modify the input parameter. - InputDataType& InputParameter() { return inputParameter; } - //! Get the output parameter. OutputDataType const& OutputParameter() const { return outputParameter; } //! Modify the output parameter. @@ -186,9 +181,6 @@ class BatchNorm //! Locally-stored delta object. OutputDataType delta; - //! Locally-stored input parameter object. - InputDataType inputParameter; - //! Locally-stored output parameter object. OutputDataType outputParameter; diff --git a/src/mlpack/methods/ann/layer/bilinear_interpolation.hpp b/src/mlpack/methods/ann/layer/bilinear_interpolation.hpp index 11305648b9..9ea1e20bbc 100644 --- a/src/mlpack/methods/ann/layer/bilinear_interpolation.hpp +++ b/src/mlpack/methods/ann/layer/bilinear_interpolation.hpp @@ -1,6 +1,7 @@ /** * @file bilinear_interpolation.hpp - * @author Kris Singh and Shikhar Jaiswal + * @author Kris Singh + * @author Shikhar Jaiswal * * mlpack is free software; you may redistribute it and/or modify it under the * terms of the 3-clause BSD license. You should have received a copy of the @@ -82,11 +83,6 @@ class BilinearInterpolation arma::Mat&& gradient, arma::Mat&& output); - //! Get the input parameter. - InputDataType const& InputParameter() const { return inputParameter; } - //! Modify the input parameter. - InputDataType& InputParameter() { return inputParameter; } - //! Get the output parameter. OutputDataType const& OutputParameter() const { return outputParameter; } //! Modify the output parameter. @@ -114,10 +110,10 @@ class BilinearInterpolation size_t outColSize; //! Locally stored depth of the input. size_t depth; + //! Locally stored number of input points. + size_t batchSize; //! Locally-stored delta object. OutputDataType delta; - //! Locally-stored input parameter object. - InputDataType inputParameter; //! Locally-stored output parameter object. OutputDataType outputParameter; }; // class BilinearInterpolation diff --git a/src/mlpack/methods/ann/layer/bilinear_interpolation_impl.hpp b/src/mlpack/methods/ann/layer/bilinear_interpolation_impl.hpp index b9e728b5e6..bd22e2684a 100644 --- a/src/mlpack/methods/ann/layer/bilinear_interpolation_impl.hpp +++ b/src/mlpack/methods/ann/layer/bilinear_interpolation_impl.hpp @@ -1,6 +1,7 @@ /** * @file bilinear_interpolation_impl.hpp - * @author Kris Singh and Shikhar Jaiswal + * @author Kris Singh + * @author Shikhar Jaiswal * * Implementation of the bilinear interpolation function as an individual layer. * @@ -26,7 +27,8 @@ BilinearInterpolation(): inColSize(0), outRowSize(0), outColSize(0), - depth(0) + depth(0), + batchSize(0) { // Nothing to do here. } @@ -43,7 +45,8 @@ BilinearInterpolation( inColSize(inColSize), outRowSize(outRowSize), outColSize(outColSize), - depth(depth) + depth(depth), + batchSize(0) { // Nothing to do here. } @@ -53,20 +56,22 @@ template void BilinearInterpolation::Forward( const arma::Mat&& input, arma::Mat&& output) { + batchSize = input.n_cols; if (output.is_empty()) - output.set_size(outRowSize * outColSize * depth, 1); + output.set_size(outRowSize * outColSize * depth, batchSize); else { assert(output.n_rows == outRowSize * outColSize * depth); - assert(output.n_cols == 1); + assert(output.n_cols == batchSize); } assert(inRowSize >= 2); assert(inColSize >= 2); - arma::cube inputAsCube(input.memptr(), inRowSize, inColSize, depth); - arma::cube outputAsCube(output.memptr(), outRowSize, outColSize, depth, - false, true); + arma::cube inputAsCube(const_cast&&>(input).memptr(), + inRowSize, inColSize, depth * batchSize, false, false); + arma::cube outputAsCube(output.memptr(), outRowSize, outColSize, + depth * batchSize, false, true); double scaleRow = (double) inRowSize / (double) outRowSize; double scaleCol = (double) inColSize / (double) outColSize; @@ -97,7 +102,7 @@ void BilinearInterpolation::Forward( coeffs[2] = (1 - deltaR) * deltaC; coeffs[3] = deltaR * deltaC; - for (size_t k = 0; k < depth; k++) + for (size_t k = 0; k < depth * batchSize; k++) { outputAsCube(i, j, k) = arma::accu(inputAsCube.slice(k).submat( rOrigin, cOrigin, rOrigin + 1, cOrigin + 1) % coeffs); @@ -114,19 +119,20 @@ void BilinearInterpolation::Backward( arma::Mat&& output) { if (output.is_empty()) - output.set_size(inRowSize * inColSize * depth, 1); + output.set_size(inRowSize * inColSize * depth, batchSize); else { assert(output.n_rows == inRowSize * inColSize * depth); - assert(output.n_cols == 1); + assert(output.n_cols == batchSize); } assert(outRowSize >= 2); assert(outColSize >= 2); - arma::cube gradientAsCube(gradient.memptr(), outRowSize, outColSize, depth); - arma::cube outputAsCube(output.memptr(), inRowSize, inColSize, depth, false, - true); + arma::cube gradientAsCube(gradient.memptr(), outRowSize, outColSize, + depth * batchSize, false, false); + arma::cube outputAsCube(output.memptr(), inRowSize, inColSize, + depth * batchSize, false, true); if (gradient.n_elem == output.n_elem) { @@ -157,7 +163,7 @@ void BilinearInterpolation::Backward( coeffs[2] = (1 - deltaR) * deltaC; coeffs[3] = deltaR * deltaC; - for (size_t k = 0; k < depth; k++) + for (size_t k = 0; k < depth * batchSize; k++) { outputAsCube(i, j, k) = arma::accu(gradientAsCube.slice(k).submat( rOrigin, cOrigin, rOrigin + 1, cOrigin + 1) % coeffs); diff --git a/src/mlpack/methods/ann/layer/concat_performance.hpp b/src/mlpack/methods/ann/layer/concat_performance.hpp index 121bc2a7f1..02ba31a683 100644 --- a/src/mlpack/methods/ann/layer/concat_performance.hpp +++ b/src/mlpack/methods/ann/layer/concat_performance.hpp @@ -72,11 +72,6 @@ class ConcatPerformance const arma::Mat&& target, arma::Mat&& output); - //! Get the input parameter. - InputDataType& InputParameter() const { return inputParameter; } - //! Modify the input parameter. - InputDataType& InputParameter() { return inputParameter; } - //! Get the output parameter. OutputDataType& OutputParameter() const { return outputParameter; } //! Modify the output parameter. @@ -103,9 +98,6 @@ class ConcatPerformance //! Locally-stored delta object. OutputDataType delta; - //! Locally-stored input parameter object. - InputDataType inputParameter; - //! Locally-stored output parameter object. OutputDataType outputParameter; }; // class ConcatPerformance diff --git a/src/mlpack/methods/ann/layer/constant.hpp b/src/mlpack/methods/ann/layer/constant.hpp index 03d5a90144..62ed80faa9 100644 --- a/src/mlpack/methods/ann/layer/constant.hpp +++ b/src/mlpack/methods/ann/layer/constant.hpp @@ -66,11 +66,6 @@ class Constant DataType&& /* gy */, DataType&& g); - //! Get the input parameter. - InputDataType& InputParameter() const { return inputParameter; } - //! Modify the input parameter. - InputDataType& InputParameter() { return inputParameter; } - //! Get the output parameter. OutputDataType& OutputParameter() const { return outputParameter; } //! Modify the output parameter. @@ -100,9 +95,6 @@ class Constant //! Locally-stored delta object. OutputDataType delta; - //! Locally-stored input parameter object. - InputDataType inputParameter; - //! Locally-stored output parameter object. OutputDataType outputParameter; }; // class ConstantLayer diff --git a/src/mlpack/methods/ann/layer/convolution.hpp b/src/mlpack/methods/ann/layer/convolution.hpp index 940581c138..83c2e09457 100644 --- a/src/mlpack/methods/ann/layer/convolution.hpp +++ b/src/mlpack/methods/ann/layer/convolution.hpp @@ -261,12 +261,15 @@ class Convolution } } - //! Locally-stored number of input units. + //! Locally-stored number of input channels. size_t inSize; - //! Locally-stored number of output units. + //! Locally-stored number of output channels. size_t outSize; + //! Locally-stored number of input units. + size_t batchSize; + //! Locally-stored filter/kernel width. size_t kW; diff --git a/src/mlpack/methods/ann/layer/convolution_impl.hpp b/src/mlpack/methods/ann/layer/convolution_impl.hpp index 0b2618c860..e06905b845 100644 --- a/src/mlpack/methods/ann/layer/convolution_impl.hpp +++ b/src/mlpack/methods/ann/layer/convolution_impl.hpp @@ -113,7 +113,9 @@ void Convolution< OutputDataType >::Forward(const arma::Mat&& input, arma::Mat&& output) { - inputTemp = arma::cube(input.memptr(), inputWidth, inputHeight, inSize); + batchSize = input.n_cols; + inputTemp = arma::cube(const_cast&&>(input).memptr(), + inputWidth, inputHeight, inSize * batchSize, false, false); if (padW != 0 || padH != 0) { @@ -123,32 +125,39 @@ void Convolution< size_t wConv = ConvOutSize(inputWidth, kW, dW, padW); size_t hConv = ConvOutSize(inputHeight, kH, dH, padH); - output.set_size(wConv * hConv * outSize, 1); - outputTemp = arma::Cube(output.memptr(), wConv, hConv, outSize, - false, false); + output.set_size(wConv * hConv * outSize, batchSize); + outputTemp = arma::Cube(output.memptr(), wConv, hConv, + outSize * batchSize, false, false); outputTemp.zeros(); - for (size_t outMap = 0, outMapIdx = 0; outMap < outSize; outMap++) + for (size_t outMap = 0, outMapIdx = 0, batchCount = 0; outMap < + outSize * batchSize; outMap++) { + if (outMap != 0 && outMap % outSize == 0) + { + batchCount++; + outMapIdx = 0; + } + for (size_t inMap = 0; inMap < inSize; inMap++, outMapIdx++) { arma::Mat convOutput; if (padW != 0 || padH != 0) { - ForwardConvolutionRule::Convolution(inputPaddedTemp.slice(inMap), - weight.slice(outMapIdx), convOutput, dW, dH); + ForwardConvolutionRule::Convolution(inputPaddedTemp.slice(inMap + + batchCount * inSize), weight.slice(outMapIdx), convOutput, dW, dH); } else { - ForwardConvolutionRule::Convolution(inputTemp.slice(inMap), - weight.slice(outMapIdx), convOutput, dW, dH); + ForwardConvolutionRule::Convolution(inputTemp.slice(inMap + + batchCount * inSize), weight.slice(outMapIdx), convOutput, dW, dH); } outputTemp.slice(outMap) += convOutput; } - outputTemp.slice(outMap) += bias(outMap); + outputTemp.slice(outMap) += bias(outMap % outSize); } outputWidth = outputTemp.n_rows; @@ -172,35 +181,40 @@ void Convolution< >::Backward( const arma::Mat&& /* input */, arma::Mat&& gy, arma::Mat&& g) { - arma::cube mappedError(gy.memptr(), outputWidth, outputHeight, outSize, - false, false); + arma::cube mappedError(gy.memptr(), outputWidth, outputHeight, + outSize * batchSize, false, false); - g.set_size(inputTemp.n_rows * inputTemp.n_cols * inputTemp.n_slices, 1); + g.set_size(inputTemp.n_rows * inputTemp.n_cols * inSize, batchSize); gTemp = arma::Cube(g.memptr(), inputTemp.n_rows, inputTemp.n_cols, inputTemp.n_slices, false, false); gTemp.zeros(); - for (size_t outMap = 0, outMapIdx = 0; outMap < outSize; outMap++) + for (size_t outMap = 0, outMapIdx = 0, batchCount = 0; outMap < + outSize * batchSize; outMap++) { + if (outMap != 0 && outMap % outSize == 0) + { + batchCount++; + outMapIdx = 0; + } + for (size_t inMap = 0; inMap < inSize; inMap++, outMapIdx++) { - arma::Mat rotatedFilter; + arma::Mat output, rotatedFilter; Rotate180(weight.slice(outMapIdx), rotatedFilter); - arma::Mat output; BackwardConvolutionRule::Convolution(mappedError.slice(outMap), rotatedFilter, output, dW, dH); if (padW != 0 || padH != 0) { - gTemp.slice(inMap) += output.submat(rotatedFilter.n_rows / 2, - rotatedFilter.n_cols / 2, - rotatedFilter.n_rows / 2 + gTemp.n_rows - 1, - rotatedFilter.n_cols / 2 + gTemp.n_cols - 1); + gTemp.slice(inMap + batchCount * inSize) += output.submat(padW, padH, + padW + gTemp.n_rows - 1, + padH + gTemp.n_cols - 1); } else { - gTemp.slice(inMap) += output; + gTemp.slice(inMap + batchCount * inSize) += output; } } } @@ -229,61 +243,62 @@ void Convolution< if (padW != 0 && padH != 0) { mappedError = arma::cube(error.memptr(), outputWidth / padW, - outputHeight / padH, outSize); + outputHeight / padH, outSize * batchSize, false, false); } else { mappedError = arma::cube(error.memptr(), outputWidth, - outputHeight, outSize); + outputHeight, outSize * batchSize, false, false); } gradient.set_size(weights.n_elem, 1); - gradientTemp = arma::Cube(gradient.memptr(), weight.n_rows, weight.n_cols, - weight.n_slices, false, false); + gradientTemp = arma::Cube(gradient.memptr(), weight.n_rows, + weight.n_cols, weight.n_slices, false, false); gradientTemp.zeros(); - for (size_t outMap = 0, outMapIdx = 0; outMap < outSize; outMap++) + for (size_t outMap = 0, outMapIdx = 0, batchCount = 0; outMap < + outSize * batchSize; outMap++) { - for (size_t inMap = 0, s = outMap; inMap < inSize; inMap++, outMapIdx++, - s += outSize) + if (outMap != 0 && outMap % outSize == 0) { - arma::Cube inputSlices; + batchCount++; + outMapIdx = 0; + } + + for (size_t inMap = 0; inMap < inSize; inMap++, outMapIdx++) + { + arma::Mat inputSlice; if (padW != 0 || padH != 0) { - inputSlices = inputPaddedTemp.slices(inMap, inMap); + inputSlice = inputPaddedTemp.slice(inMap + batchCount * inSize); } else { - inputSlices = inputTemp.slices(inMap, inMap); + inputSlice = inputTemp.slice(inMap + batchCount * inSize); } - arma::Cube deltaSlices = mappedError.slices(outMap, outMap); + arma::Mat deltaSlice = mappedError.slice(outMap); - arma::Cube output; - GradientConvolutionRule::Convolution(inputSlices, deltaSlices, + arma::Mat output; + GradientConvolutionRule::Convolution(inputSlice, deltaSlice, output, dW, dH); if ((padW != 0 || padH != 0) && (gradientTemp.n_rows < output.n_rows && gradientTemp.n_cols < output.n_cols)) { - for (size_t i = 0; i < output.n_slices; i++) - { - gradientTemp.slice(s) += output.slice(i).submat(output.n_rows / 2, - output.n_cols / 2, - output.n_rows / 2 + gradientTemp.n_rows - 1, - output.n_cols / 2 + gradientTemp.n_cols - 1); - } + gradientTemp.slice(outMapIdx) += output.submat(padW, padH, + padW + gradientTemp.n_rows - 1, + padH + gradientTemp.n_cols - 1); } else { - for (size_t i = 0; i < output.n_slices; i++) - gradientTemp.slice(s) += output.slice(i); + gradientTemp.slice(outMapIdx) += output; } } - gradient.submat(weight.n_elem + outMap, 0, weight.n_elem + outMap, 0) = - arma::accu(mappedError.slices(outMap, outMap)); + gradient.submat(weight.n_elem + (outMap % outSize), 0, weight.n_elem + + (outMap % outSize), 0) = arma::accu(mappedError.slice(outMap)); } } @@ -306,6 +321,7 @@ void Convolution< { ar & BOOST_SERIALIZATION_NVP(inSize); ar & BOOST_SERIALIZATION_NVP(outSize); + ar & BOOST_SERIALIZATION_NVP(batchSize); ar & BOOST_SERIALIZATION_NVP(kW); ar & BOOST_SERIALIZATION_NVP(kH); ar & BOOST_SERIALIZATION_NVP(dW); diff --git a/src/mlpack/methods/ann/layer/dropconnect.hpp b/src/mlpack/methods/ann/layer/dropconnect.hpp index 46bef506fd..1ed6f2ce0c 100644 --- a/src/mlpack/methods/ann/layer/dropconnect.hpp +++ b/src/mlpack/methods/ann/layer/dropconnect.hpp @@ -120,11 +120,6 @@ class DropConnect //! Modify the parameters. OutputDataType& Parameters() { return parameters; } - //! Get the input parameter. - InputDataType const& InputParameter() const { return inputParameter; } - //! Modify the input parameter. - InputDataType& InputParameter() { return inputParameter; } - //! Get the output parameter. OutputDataType const& OutputParameter() const { return outputParameter; } //! Modify the output parameter. @@ -178,9 +173,6 @@ class DropConnect //! Locally-stored gradient object. OutputDataType gradient; - //! Locally-stored input parameter object. - InputDataType inputParameter; - //! Locally-stored output parameter object. OutputDataType outputParameter; diff --git a/src/mlpack/methods/ann/layer/dropout.hpp b/src/mlpack/methods/ann/layer/dropout.hpp index a388820c71..696d92869c 100644 --- a/src/mlpack/methods/ann/layer/dropout.hpp +++ b/src/mlpack/methods/ann/layer/dropout.hpp @@ -80,11 +80,6 @@ class Dropout arma::Mat&& gy, arma::Mat&& g); - //! Get the input parameter. - InputDataType const& InputParameter() const { return inputParameter; } - //! Modify the input parameter. - InputDataType& InputParameter() { return inputParameter; } - //! Get the output parameter. OutputDataType const& OutputParameter() const { return outputParameter; } //! Modify the output parameter. @@ -120,9 +115,6 @@ class Dropout //! Locally-stored delta object. OutputDataType delta; - //! Locally-stored input parameter object. - InputDataType inputParameter; - //! Locally-stored output parameter object. OutputDataType outputParameter; diff --git a/src/mlpack/methods/ann/layer/elu.hpp b/src/mlpack/methods/ann/layer/elu.hpp index 06d6253313..5e273a6dc1 100644 --- a/src/mlpack/methods/ann/layer/elu.hpp +++ b/src/mlpack/methods/ann/layer/elu.hpp @@ -144,11 +144,6 @@ class ELU template void Backward(const DataType&& input, DataType&& gy, DataType&& g); - //! Get the input parameter. - InputDataType const& InputParameter() const { return inputParameter; } - //! Modify the input parameter. - InputDataType& InputParameter() { return inputParameter; } - //! Get the output parameter. OutputDataType const& OutputParameter() const { return outputParameter; } //! Modify the output parameter. @@ -238,9 +233,6 @@ class ELU //! Locally-stored delta object. OutputDataType delta; - //! Locally-stored input parameter object. - InputDataType inputParameter; - //! Locally-stored output parameter object. OutputDataType outputParameter; diff --git a/src/mlpack/methods/ann/layer/fast_lstm.hpp b/src/mlpack/methods/ann/layer/fast_lstm.hpp index e853bb16d2..034bafd744 100644 --- a/src/mlpack/methods/ann/layer/fast_lstm.hpp +++ b/src/mlpack/methods/ann/layer/fast_lstm.hpp @@ -138,11 +138,6 @@ class FastLSTM //! Modify the parameters. OutputDataType& Parameters() { return weights; } - //! Get the input parameter. - InputDataType const& InputParameter() const { return inputParameter; } - //! Modify the input parameter. - InputDataType& InputParameter() { return inputParameter; } - //! Get the output parameter. OutputDataType const& OutputParameter() const { return outputParameter; } //! Modify the output parameter. @@ -254,9 +249,6 @@ class FastLSTM //! Locally-stored gradient object. OutputDataType grad; - //! Locally-stored input parameter object. - InputDataType inputParameter; - //! Locally-stored output parameter object. OutputDataType outputParameter; diff --git a/src/mlpack/methods/ann/layer/flexible_relu.hpp b/src/mlpack/methods/ann/layer/flexible_relu.hpp index a9c837e022..c5e69bc7f8 100644 --- a/src/mlpack/methods/ann/layer/flexible_relu.hpp +++ b/src/mlpack/methods/ann/layer/flexible_relu.hpp @@ -115,11 +115,6 @@ class FlexibleReLU //! Modify the parameters. OutputDataType& Parameters() { return alpha; } - //! Get the input parameter. - InputDataType const& InputParameter() const { return inputParameter; } - //! Modify the input parameter. - InputDataType& InputParameter() { return inputParameter; } - //! Get the output parameter. OutputDataType const& OutputParameter() const { return outputParameter; } //! Modify the output parameter. @@ -150,9 +145,6 @@ class FlexibleReLU //! Locally-stored delta object. OutputDataType delta; - //! Locally-stored input parameter object. - InputDataType inputParameter; - //! Locally-stored output parameter object. OutputDataType outputParameter; diff --git a/src/mlpack/methods/ann/layer/glimpse.hpp b/src/mlpack/methods/ann/layer/glimpse.hpp index 92eaad47e9..357fbce726 100644 --- a/src/mlpack/methods/ann/layer/glimpse.hpp +++ b/src/mlpack/methods/ann/layer/glimpse.hpp @@ -127,11 +127,6 @@ class Glimpse arma::Mat&& gy, arma::Mat&& g); - //! Get the input parameter. - InputDataType& InputParameter() const {return inputParameter; } - //! Modify the input parameter. - InputDataType& InputParameter() { return inputParameter; } - //! Get the output parameter. OutputDataType& OutputParameter() const {return outputParameter; } //! Modify the output parameter. @@ -393,9 +388,6 @@ class Glimpse //! Locally-stored delta object. OutputDataType delta; - //! Locally-stored input parameter object. - InputDataType inputParameter; - //! Locally-stored output parameter object. OutputDataType outputParameter; diff --git a/src/mlpack/methods/ann/layer/gru.hpp b/src/mlpack/methods/ann/layer/gru.hpp index ed2e96fbcd..63fee576be 100644 --- a/src/mlpack/methods/ann/layer/gru.hpp +++ b/src/mlpack/methods/ann/layer/gru.hpp @@ -135,11 +135,6 @@ class GRU //! Modify the parameters. OutputDataType& Parameters() { return weights; } - //! Get the input parameter. - InputDataType const& InputParameter() const { return inputParameter; } - //! Modify the input parameter. - InputDataType& InputParameter() { return inputParameter; } - //! Get the output parameter. OutputDataType const& OutputParameter() const { return outputParameter; } //! Modify the output parameter. @@ -246,9 +241,6 @@ class GRU //! Locally-stored gradient object. OutputDataType gradient; - //! Locally-stored input parameter object. - InputDataType inputParameter; - //! Locally-stored output parameter object. OutputDataType outputParameter; }; // class GRU diff --git a/src/mlpack/methods/ann/layer/hard_tanh.hpp b/src/mlpack/methods/ann/layer/hard_tanh.hpp index 7de2158d6f..8ff75b899f 100644 --- a/src/mlpack/methods/ann/layer/hard_tanh.hpp +++ b/src/mlpack/methods/ann/layer/hard_tanh.hpp @@ -83,11 +83,6 @@ class HardTanH DataType&& gy, DataType&& g); - //! Get the input parameter. - InputDataType const& InputParameter() const { return inputParameter; } - //! Modify the input parameter. - InputDataType& InputParameter() { return inputParameter; } - //! Get the output parameter. OutputDataType const& OutputParameter() const { return outputParameter; } //! Modify the output parameter. @@ -118,9 +113,6 @@ class HardTanH //! Locally-stored delta object. OutputDataType delta; - //! Locally-stored input parameter object. - InputDataType inputParameter; - //! Locally-stored output parameter object. OutputDataType outputParameter; diff --git a/src/mlpack/methods/ann/layer/join.hpp b/src/mlpack/methods/ann/layer/join.hpp index b55c4bca99..2f6ecde25f 100644 --- a/src/mlpack/methods/ann/layer/join.hpp +++ b/src/mlpack/methods/ann/layer/join.hpp @@ -60,11 +60,6 @@ class Join arma::Mat&& gy, arma::Mat&& g); - //! Get the input parameter. - InputDataType const& InputParameter() const { return inputParameter; } - //! Modify the input parameter. - InputDataType& InputParameter() { return inputParameter; } - //! Get the output parameter. OutputDataType const& OutputParameter() const { return outputParameter; } //! Modify the output parameter. @@ -91,9 +86,6 @@ class Join //! Locally-stored delta object. OutputDataType delta; - //! Locally-stored input parameter object. - InputDataType inputParameter; - //! Locally-stored output parameter object. OutputDataType outputParameter; }; // class Join diff --git a/src/mlpack/methods/ann/layer/layer.hpp b/src/mlpack/methods/ann/layer/layer.hpp index e3395a1586..78abe96cf4 100644 --- a/src/mlpack/methods/ann/layer/layer.hpp +++ b/src/mlpack/methods/ann/layer/layer.hpp @@ -29,6 +29,7 @@ #include "fast_lstm.hpp" #include "recurrent.hpp" #include "recurrent_attention.hpp" +#include "reparametrization.hpp" #include "sequential.hpp" #include "subview.hpp" #include "concat.hpp" diff --git a/src/mlpack/methods/ann/layer/layer_norm.hpp b/src/mlpack/methods/ann/layer/layer_norm.hpp index a978f2b022..2c9d532e23 100644 --- a/src/mlpack/methods/ann/layer/layer_norm.hpp +++ b/src/mlpack/methods/ann/layer/layer_norm.hpp @@ -121,11 +121,6 @@ class LayerNorm //! Modify the parameters. OutputDataType& Parameters() { return weights; } - //! Get the input parameter. - InputDataType const& InputParameter() const { return inputParameter; } - //! Modify the input parameter. - InputDataType& InputParameter() { return inputParameter; } - //! Get the output parameter. OutputDataType const& OutputParameter() const { return outputParameter; } //! Modify the output parameter. @@ -181,9 +176,6 @@ class LayerNorm //! Locally-stored delta object. OutputDataType delta; - //! Locally-stored input parameter object. - InputDataType inputParameter; - //! Locally-stored output parameter object. OutputDataType outputParameter; diff --git a/src/mlpack/methods/ann/layer/layer_traits.hpp b/src/mlpack/methods/ann/layer/layer_traits.hpp index 02691f7b7a..f69e9d3c94 100644 --- a/src/mlpack/methods/ann/layer/layer_traits.hpp +++ b/src/mlpack/methods/ann/layer/layer_traits.hpp @@ -104,6 +104,10 @@ HAS_MEM_FUNC(InputHeight, HasInputHeight); // can use with SFINAE to catch when a type has a Rho() function. HAS_MEM_FUNC(Rho, HasRho); +// This gives us a HasLoss type (where U is a function pointer) we +// can use with SFINAE to catch when a type has a Loss() function. +HAS_MEM_FUNC(Loss, HasLoss); + } // namespace ann } // namespace mlpack diff --git a/src/mlpack/methods/ann/layer/layer_types.hpp b/src/mlpack/methods/ann/layer/layer_types.hpp index 3584aeb5bb..c2a422c830 100644 --- a/src/mlpack/methods/ann/layer/layer_types.hpp +++ b/src/mlpack/methods/ann/layer/layer_types.hpp @@ -31,11 +31,11 @@ #include #include #include -#include #include #include #include #include +#include #include #include @@ -44,6 +44,9 @@ #include #include +// Loss function modules. +#include + namespace mlpack { namespace ann { @@ -58,6 +61,11 @@ template class GRU; template class FastLSTM; template class VRClassReward; +template +class Reparametrization; + template*, RecurrentAttention*, ReinforceNormal*, + Reparametrization*, Select*, Sequential*, Subview*, diff --git a/src/mlpack/methods/ann/layer/leaky_relu.hpp b/src/mlpack/methods/ann/layer/leaky_relu.hpp index dfa7af8e92..c577875ce9 100644 --- a/src/mlpack/methods/ann/layer/leaky_relu.hpp +++ b/src/mlpack/methods/ann/layer/leaky_relu.hpp @@ -46,7 +46,7 @@ class LeakyReLU public: /** * Create the LeakyReLU object using the specified parameters. - * The non zero gradient can be adjusted by specifying tha parameter + * The non zero gradient can be adjusted by specifying the parameter * alpha in the range 0 to 1. Default (alpha = 0.03) * * @param alpha Non zero gradient @@ -75,11 +75,6 @@ class LeakyReLU template void Backward(const DataType&& input, DataType&& gy, DataType&& g); - //! Get the input parameter. - InputDataType const& InputParameter() const { return inputParameter; } - //! Modify the input parameter. - InputDataType& InputParameter() { return inputParameter; } - //! Get the output parameter. OutputDataType const& OutputParameter() const { return outputParameter; } //! Modify the output parameter. @@ -157,9 +152,6 @@ class LeakyReLU //! Locally-stored delta object. OutputDataType delta; - //! Locally-stored input parameter object. - InputDataType inputParameter; - //! Locally-stored output parameter object. OutputDataType outputParameter; diff --git a/src/mlpack/methods/ann/layer/linear.hpp b/src/mlpack/methods/ann/layer/linear.hpp index 7de48ece1b..3757533363 100644 --- a/src/mlpack/methods/ann/layer/linear.hpp +++ b/src/mlpack/methods/ann/layer/linear.hpp @@ -129,7 +129,7 @@ class Linear //! Locally-stored weight object. OutputDataType weights; - //! Locally-stored weight paramters. + //! Locally-stored weight parameters. OutputDataType weight; //! Locally-stored bias term parameters. diff --git a/src/mlpack/methods/ann/layer/log_softmax.hpp b/src/mlpack/methods/ann/layer/log_softmax.hpp index 62654907db..613eac4aa3 100644 --- a/src/mlpack/methods/ann/layer/log_softmax.hpp +++ b/src/mlpack/methods/ann/layer/log_softmax.hpp @@ -65,11 +65,6 @@ class LogSoftMax arma::Mat&& gy, arma::Mat&& g); - //! Get the input parameter. - InputDataType& InputParameter() const { return inputParameter; } - //! Modify the input parameter. - InputDataType& InputParameter() { return inputParameter; } - //! Get the output parameter. OutputDataType& OutputParameter() const { return outputParameter; } //! Modify the output parameter. @@ -90,9 +85,6 @@ class LogSoftMax //! Locally-stored delta object. OutputDataType delta; - //! Locally-stored input parameter object. - InputDataType inputParameter; - //! Locally-stored output parameter object. OutputDataType outputParameter; }; // class LogSoftmax diff --git a/src/mlpack/methods/ann/layer/lookup.hpp b/src/mlpack/methods/ann/layer/lookup.hpp index 5c5109cf0e..59e5547cda 100644 --- a/src/mlpack/methods/ann/layer/lookup.hpp +++ b/src/mlpack/methods/ann/layer/lookup.hpp @@ -85,11 +85,6 @@ class Lookup //! Modify the parameters. OutputDataType& Parameters() { return weights; } - //! Get the input parameter. - InputDataType const& InputParameter() const { return inputParameter; } - //! Modify the input parameter. - InputDataType& InputParameter() { return inputParameter; } - //! Get the output parameter. OutputDataType const& OutputParameter() const { return outputParameter; } //! Modify the output parameter. @@ -127,9 +122,6 @@ class Lookup //! Locally-stored gradient object. OutputDataType gradient; - //! Locally-stored input parameter object. - InputDataType inputParameter; - //! Locally-stored output parameter object. OutputDataType outputParameter; }; // class Lookup diff --git a/src/mlpack/methods/ann/layer/lstm.hpp b/src/mlpack/methods/ann/layer/lstm.hpp index cd0f867136..2b4deaeb97 100644 --- a/src/mlpack/methods/ann/layer/lstm.hpp +++ b/src/mlpack/methods/ann/layer/lstm.hpp @@ -136,11 +136,6 @@ class LSTM //! Modify the parameters. OutputDataType& Parameters() { return weights; } - //! Get the input parameter. - InputDataType const& InputParameter() const { return inputParameter; } - //! Modify the input parameter. - InputDataType& InputParameter() { return inputParameter; } - //! Get the output parameter. OutputDataType const& OutputParameter() const { return outputParameter; } //! Modify the output parameter. @@ -206,9 +201,6 @@ class LSTM //! Locally-stored gradient object. OutputDataType grad; - //! Locally-stored input parameter object. - InputDataType inputParameter; - //! Locally-stored output parameter object. OutputDataType outputParameter; diff --git a/src/mlpack/methods/ann/layer/max_pooling.hpp b/src/mlpack/methods/ann/layer/max_pooling.hpp index 0001819959..ef98e2aded 100644 --- a/src/mlpack/methods/ann/layer/max_pooling.hpp +++ b/src/mlpack/methods/ann/layer/max_pooling.hpp @@ -94,11 +94,6 @@ class MaxPooling arma::Mat&& gy, arma::Mat&& g); - //! Get the input parameter. - InputDataType const& InputParameter() const { return inputParameter; } - //! Modify the input parameter. - InputDataType& InputParameter() { return inputParameter; } - //! Get the output parameter. OutputDataType const& OutputParameter() const { return outputParameter; } //! Modify the output parameter. @@ -193,12 +188,15 @@ class MaxPooling } } - //! Locally-stored number of input units. + //! Locally-stored number of input channels. size_t inSize; - //! Locally-stored number of output units. + //! Locally-stored number of output channels. size_t outSize; + //! Locally-stored number of input units. + size_t batchSize; + //! Locally-stored width of the pooling window. size_t kW; @@ -253,9 +251,6 @@ class MaxPooling //! Locally-stored gradient object. OutputDataType gradient; - //! Locally-stored input parameter object. - InputDataType inputParameter; - //! Locally-stored output parameter object. OutputDataType outputParameter; diff --git a/src/mlpack/methods/ann/layer/max_pooling_impl.hpp b/src/mlpack/methods/ann/layer/max_pooling_impl.hpp index 13ff201c51..ff532f1afa 100644 --- a/src/mlpack/methods/ann/layer/max_pooling_impl.hpp +++ b/src/mlpack/methods/ann/layer/max_pooling_impl.hpp @@ -43,6 +43,9 @@ MaxPooling::MaxPooling( inputHeight(0), outputWidth(0), outputHeight(0), + batchSize(0), + inSize(0), + outSize(0), deterministic(false) { // Nothing to do here. @@ -53,8 +56,10 @@ template void MaxPooling::Forward( const arma::Mat&& input, arma::Mat&& output) { - const size_t slices = input.n_elem / (inputWidth * inputHeight); - inputTemp = arma::cube(input.memptr(), inputWidth, inputHeight, slices); + batchSize = input.n_cols; + inSize = input.n_elem / (inputWidth * inputHeight * batchSize); + inputTemp = arma::cube(const_cast&&>(input).memptr(), + inputWidth, inputHeight, batchSize * inSize, false, false); if (floor) { @@ -70,7 +75,7 @@ void MaxPooling::Forward( } outputTemp = arma::zeros >(outputWidth, outputHeight, - slices); + batchSize * inSize); if (!deterministic) { @@ -102,11 +107,12 @@ void MaxPooling::Forward( } } - output = arma::Mat(outputTemp.memptr(), outputTemp.n_elem, 1); + output = arma::Mat(outputTemp.memptr(), outputTemp.n_elem / batchSize, + batchSize); outputWidth = outputTemp.n_rows; outputHeight = outputTemp.n_cols; - outSize = slices; + outSize = batchSize * inSize; } template @@ -115,7 +121,7 @@ void MaxPooling::Backward( const arma::Mat&& /* input */, arma::Mat&& gy, arma::Mat&& g) { arma::cube mappedError = arma::cube(gy.memptr(), outputWidth, - outputHeight, outSize); + outputHeight, outSize, false, false); gTemp = arma::zeros(inputTemp.n_rows, inputTemp.n_cols, inputTemp.n_slices); @@ -128,7 +134,7 @@ void MaxPooling::Backward( poolingIndices.pop_back(); - g = arma::mat(gTemp.memptr(), gTemp.n_elem, 1); + g = arma::mat(gTemp.memptr(), gTemp.n_elem / batchSize, batchSize); } template @@ -141,6 +147,7 @@ void MaxPooling::serialize( ar & BOOST_SERIALIZATION_NVP(kH); ar & BOOST_SERIALIZATION_NVP(dW); ar & BOOST_SERIALIZATION_NVP(dH); + ar & BOOST_SERIALIZATION_NVP(batchSize); } } // namespace ann diff --git a/src/mlpack/methods/ann/layer/mean_pooling.hpp b/src/mlpack/methods/ann/layer/mean_pooling.hpp index 3c77d30964..ee723c8dfa 100644 --- a/src/mlpack/methods/ann/layer/mean_pooling.hpp +++ b/src/mlpack/methods/ann/layer/mean_pooling.hpp @@ -74,11 +74,6 @@ class MeanPooling arma::Mat&& gy, arma::Mat&& g); - //! Get the input parameter. - InputDataType const& InputParameter() const { return inputParameter; } - //! Modify the input parameter. - InputDataType& InputParameter() { return inputParameter; } - //! Get the output parameter. OutputDataType const& OutputParameter() const { return outputParameter; } //! Modify the output parameter. @@ -177,12 +172,15 @@ class MeanPooling } } - //! Locally-stored number of input units. + //! Locally-stored number of input channels. size_t inSize; - //! Locally-stored number of output units. + //! Locally-stored number of output channels. size_t outSize; + //! Locally-stored number of input units. + size_t batchSize; + //! Locally-stored width of the pooling window. size_t kW; @@ -234,9 +232,6 @@ class MeanPooling //! Locally-stored gradient object. OutputDataType gradient; - //! Locally-stored input parameter object. - InputDataType inputParameter; - //! Locally-stored output parameter object. OutputDataType outputParameter; }; // class MeanPooling diff --git a/src/mlpack/methods/ann/layer/mean_pooling_impl.hpp b/src/mlpack/methods/ann/layer/mean_pooling_impl.hpp index 900843e29e..52b2f807df 100644 --- a/src/mlpack/methods/ann/layer/mean_pooling_impl.hpp +++ b/src/mlpack/methods/ann/layer/mean_pooling_impl.hpp @@ -43,7 +43,10 @@ MeanPooling::MeanPooling( reset(false), floor(floor), deterministic(false), - offset(0) + offset(0), + batchSize(0), + inSize(0), + outSize(0) { // Nothing to do here. } @@ -53,8 +56,10 @@ template void MeanPooling::Forward( const arma::Mat&& input, arma::Mat&& output) { - size_t slices = input.n_elem / (inputWidth * inputHeight); - inputTemp = arma::cube(input.memptr(), inputWidth, inputHeight, slices); + batchSize = input.n_cols; + inSize = input.n_elem / (inputWidth * inputHeight * batchSize); + inputTemp = arma::cube(const_cast&&>(input).memptr(), + inputWidth, inputHeight, batchSize * inSize, false, false); if (floor) { @@ -72,16 +77,17 @@ void MeanPooling::Forward( } outputTemp = arma::zeros >(outputWidth, outputHeight, - slices); + batchSize * inSize); for (size_t s = 0; s < inputTemp.n_slices; s++) Pooling(inputTemp.slice(s), outputTemp.slice(s)); - output = arma::Mat(outputTemp.memptr(), outputTemp.n_elem, 1); + output = arma::Mat(outputTemp.memptr(), outputTemp.n_elem / batchSize, + batchSize); outputWidth = outputTemp.n_rows; outputHeight = outputTemp.n_cols; - outSize = slices; + outSize = batchSize * inSize; } template @@ -92,7 +98,7 @@ void MeanPooling::Backward( arma::Mat&& g) { arma::cube mappedError = arma::cube(gy.memptr(), outputWidth, - outputHeight, outSize); + outputHeight, outSize, false, false); gTemp = arma::zeros(inputTemp.n_rows, inputTemp.n_cols, inputTemp.n_slices); @@ -102,7 +108,7 @@ void MeanPooling::Backward( Unpooling(inputTemp.slice(s), mappedError.slice(s), gTemp.slice(s)); } - g = arma::mat(gTemp.memptr(), gTemp.n_elem, 1); + g = arma::mat(gTemp.memptr(), gTemp.n_elem / batchSize, batchSize); } template @@ -115,6 +121,7 @@ void MeanPooling::serialize( ar & BOOST_SERIALIZATION_NVP(kH); ar & BOOST_SERIALIZATION_NVP(dW); ar & BOOST_SERIALIZATION_NVP(dH); + ar & BOOST_SERIALIZATION_NVP(batchSize); } } // namespace ann diff --git a/src/mlpack/methods/ann/layer/multiply_constant.hpp b/src/mlpack/methods/ann/layer/multiply_constant.hpp index c078438889..cdc65659bd 100644 --- a/src/mlpack/methods/ann/layer/multiply_constant.hpp +++ b/src/mlpack/methods/ann/layer/multiply_constant.hpp @@ -60,11 +60,6 @@ class MultiplyConstant template void Backward(const DataType&& /* input */, DataType&& gy, DataType&& g); - //! Get the input parameter. - InputDataType& InputParameter() const { return inputParameter; } - //! Modify the input parameter. - InputDataType& InputParameter() { return inputParameter; } - //! Get the output parameter. OutputDataType& OutputParameter() const { return outputParameter; } //! Modify the output parameter. @@ -88,9 +83,6 @@ class MultiplyConstant //! Locally-stored delta object. OutputDataType delta; - //! Locally-stored input parameter object. - InputDataType inputParameter; - //! Locally-stored output parameter object. OutputDataType outputParameter; }; // class MultiplyConstant diff --git a/src/mlpack/methods/ann/layer/multiply_merge.hpp b/src/mlpack/methods/ann/layer/multiply_merge.hpp index 865f291d76..a247521c73 100644 --- a/src/mlpack/methods/ann/layer/multiply_merge.hpp +++ b/src/mlpack/methods/ann/layer/multiply_merge.hpp @@ -46,8 +46,9 @@ class MultiplyMerge * Create the MultiplyMerge object using the specified parameters. * * @param model Expose all the network modules. + * @param run Call the Forward/Backward method before the output is merged. */ - MultiplyMerge(const bool model = false); + MultiplyMerge(const bool model = false, const bool run = true); //! Destructor to release allocated memory. ~MultiplyMerge(); @@ -60,7 +61,7 @@ class MultiplyMerge * @param output Resulting output activation. */ template - void Forward(const InputType&& /* input */, OutputType&& output); + void Forward(InputType&& /* input */, OutputType&& output); /** * Ordinary feed backward pass of a neural network, calculating the function @@ -77,19 +78,16 @@ class MultiplyMerge arma::Mat&& g); /* - * Add a new module to the model. + * Calculate the gradient using the output delta and the input activation. * - * @param layer The Layer to be added to the model. + * @param input The input parameter used for calculating the gradient. + * @param error The calculated error. + * @param gradient The calculated gradient. */ - void Add(LayerTypes layer) { network.push_back(layer); } - - /* - * Add a new module to the model. - * - * @param layer The Layer to be added to the model. - */ - template - void Add(const LayerType& layer) { network.push_back(new LayerType(layer)); } + template + void Gradient(arma::Mat&& input, + arma::Mat&& error, + arma::Mat&& gradient); /* * Add a new module to the model. @@ -99,10 +97,12 @@ class MultiplyMerge template void Add(Args... args) { network.push_back(new LayerType(args...)); } - //! Get the input parameter. - InputDataType const& InputParameter() const { return inputParameter; } - //! Modify the input parameter. - InputDataType& InputParameter() { return inputParameter; } + /* + * Add a new module to the model. + * + * @param layer The Layer to be added to the model. + */ + void Add(LayerTypes layer) { network.push_back(layer); } //! Get the output parameter. OutputDataType const& OutputParameter() const { return outputParameter; } @@ -114,6 +114,11 @@ class MultiplyMerge //! Modify the delta. OutputDataType& Delta() { return delta; } + //! Get the gradient. + OutputDataType const& Gradient() const { return gradient; } + //! Modify the gradient. + OutputDataType& Gradient() { return gradient; } + //! Return the model modules. std::vector >& Model() { @@ -125,6 +130,11 @@ class MultiplyMerge return empty; } + //! Get the parameters. + OutputDataType const& Parameters() const { return weights; } + //! Modify the parameters. + OutputDataType& Parameters() { return weights; } + /** * Serialize the layer. */ @@ -135,6 +145,10 @@ class MultiplyMerge //! Parameter which indicates if the modules should be exposed. bool model; + //! Parameter which indicates if the Forward/Backward method should be called + //! before merging the output. + bool run; + //! We need this to know whether we should delete the layer in the destructor. bool ownsLayer; @@ -156,11 +170,14 @@ class MultiplyMerge //! Locally-stored delta object. OutputDataType delta; - //! Locally-stored input parameter object. - InputDataType inputParameter; + //! Locally-stored gradient object. + OutputDataType gradient; //! Locally-stored output parameter object. OutputDataType outputParameter; + + //! Locally-stored weight object. + OutputDataType weights; }; // class MultiplyMerge } // namespace ann diff --git a/src/mlpack/methods/ann/layer/multiply_merge_impl.hpp b/src/mlpack/methods/ann/layer/multiply_merge_impl.hpp index 6738bac8c4..19d670d113 100644 --- a/src/mlpack/methods/ann/layer/multiply_merge_impl.hpp +++ b/src/mlpack/methods/ann/layer/multiply_merge_impl.hpp @@ -16,13 +16,18 @@ // In case it hasn't yet been included. #include "multiply_merge.hpp" +#include "../visitor/forward_visitor.hpp" +#include "../visitor/backward_visitor.hpp" +#include "../visitor/gradient_visitor.hpp" + namespace mlpack { namespace ann /** Artificial Neural Network. */ { template MultiplyMerge::MultiplyMerge( - const bool model) : model(model), ownsLayer(!model) + const bool model, const bool run) : + model(model), run(run), ownsLayer(!model) { // Nothing to do here. } @@ -42,10 +47,19 @@ template template void MultiplyMerge::Forward( - const InputType&& /* input */, OutputType&& output) + InputType&& input, OutputType&& output) { - output = boost::apply_visitor(outputParameterVisitor, network.front()); + if (run) + { + for (size_t i = 0; i < network.size(); ++i) + { + boost::apply_visitor(ForwardVisitor(std::move(input), std::move( + boost::apply_visitor(outputParameterVisitor, network[i]))), + network[i]); + } + } + output = boost::apply_visitor(outputParameterVisitor, network.front()); for (size_t i = 1; i < network.size(); ++i) { output %= boost::apply_visitor(outputParameterVisitor, network[i]); @@ -58,7 +72,41 @@ template void MultiplyMerge::Backward( const arma::Mat&& /* input */, arma::Mat&& gy, arma::Mat&& g) { - g = gy; + if (run) + { + for (size_t i = 0; i < network.size(); ++i) + { + boost::apply_visitor(BackwardVisitor(std::move(boost::apply_visitor( + outputParameterVisitor, network[i])), std::move(gy), std::move( + boost::apply_visitor(deltaVisitor, network[i]))), network[i]); + } + + g = boost::apply_visitor(deltaVisitor, network[0]); + for (size_t i = 1; i < network.size(); ++i) + { + g += boost::apply_visitor(deltaVisitor, network[i]); + } + } + else + g = gy; +} + +template +template +void MultiplyMerge::Gradient( + arma::Mat&& input, + arma::Mat&& error, + arma::Mat&& /* gradient */ ) +{ + if (run) + { + for (size_t i = 0; i < network.size(); ++i) + { + boost::apply_visitor(GradientVisitor(std::move(input), std::move(error)), + network[i]); + } + } } template::Recurrent( ownsLayer(true) { initialModule = new Sequential<>(); - mergeModule = new AddMerge<>(false); + mergeModule = new AddMerge<>(false, false); recurrentModule = new Sequential<>(false); boost::apply_visitor(AddVisitor(inputModule), @@ -261,7 +261,7 @@ void Recurrent::serialize( if (Archive::is_loading::value) { initialModule = new Sequential<>(); - mergeModule = new AddMerge<>(false); + mergeModule = new AddMerge<>(false, false); recurrentModule = new Sequential<>(false); boost::apply_visitor(AddVisitor(inputModule), diff --git a/src/mlpack/methods/ann/layer/reinforce_normal.hpp b/src/mlpack/methods/ann/layer/reinforce_normal.hpp index 25e8c09b99..bb0ea2e737 100644 --- a/src/mlpack/methods/ann/layer/reinforce_normal.hpp +++ b/src/mlpack/methods/ann/layer/reinforce_normal.hpp @@ -63,11 +63,6 @@ class ReinforceNormal template void Backward(const DataType&& input, DataType&& /* gy */, DataType&& g); - //! Get the input parameter. - InputDataType& InputParameter() const { return inputParameter; } - //! Modify the input parameter. - InputDataType& InputParameter() { return inputParameter; } - //! Get the output parameter. OutputDataType& OutputParameter() const { return outputParameter; } //! Modify the output parameter. @@ -104,9 +99,6 @@ class ReinforceNormal //! Locally-stored delta object. OutputDataType delta; - //! Locally-stored input parameter object. - InputDataType inputParameter; - //! Locally-stored output parameter object. OutputDataType outputParameter; diff --git a/src/mlpack/methods/ann/layer/reparametrization.hpp b/src/mlpack/methods/ann/layer/reparametrization.hpp new file mode 100644 index 0000000000..ad7b86ae56 --- /dev/null +++ b/src/mlpack/methods/ann/layer/reparametrization.hpp @@ -0,0 +1,162 @@ +/** + * @file reparametrization.hpp + * @author Atharva Khandait + * + * Definition of the Reparametrization layer class which samples from a gaussian + * distribution. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_METHODS_ANN_LAYER_REPARAMETRIZATION_HPP +#define MLPACK_METHODS_ANN_LAYER_REPARAMETRIZATION_HPP + +#include + +#include "layer_types.hpp" +#include "../activation_functions/softplus_function.hpp" + +namespace mlpack { +namespace ann /** Artificial Neural Network. */ { + +/** + * Implementation of the Reparametrization layer class. This layer samples from the + * given parameters of a normal distribution. + * + * @tparam InputDataType Type of the input data (arma::colvec, arma::mat, + * arma::sp_mat or arma::cube). + * @tparam OutputDataType Type of the output data (arma::colvec, arma::mat, + * arma::sp_mat or arma::cube). + */ +template < + typename InputDataType = arma::mat, + typename OutputDataType = arma::mat +> +class Reparametrization +{ + public: + //! Create the Reparametrization object. + Reparametrization(); + + /** + * Create the Reparametrization layer object using the specified sample vector size. + * + * @param layerSize The number of output units. + * @param stochastic Whether we want random sample or constant. + * @param includeKl Whether we want to include KL loss in backward function. + */ + Reparametrization(const size_t latentSize, + const bool stochastic = true, + const bool includeKl = true); + + /** + * Ordinary feed forward pass of a neural network, evaluating the function + * f(x) by propagating the activity forward through f. + * + * @param input Input data used for evaluating the specified function. + * @param output Resulting output activation. + */ + template + void Forward(const arma::Mat&& input, arma::Mat&& output); + + /** + * Ordinary feed backward pass of a neural network, calculating the function + * f(x) by propagating x backwards trough f. Using the results from the feed + * forward pass. + * + * @param input The propagated input activation. + * @param gy The backpropagated error. + * @param g The calculated gradient. + */ + template + void Backward(const arma::Mat&& input, + arma::Mat&& gy, + arma::Mat&& g); + + /** + * Ordinary feed forward pass of a neural network, evaluating + * Kullback–Leibler divergence between a normal distribution + * and the standard normal. + * + * @param input Input data used for evaluating the specified function. + */ + template + double klForward(const InputType&& input); + + /** + * Ordinary feed backward pass of a neural network, evaluating the backward + * pass of Kullback–Leibler divergence. Using the results from the + * KL divergence feed forward pass. + * + * @param output The calculated gradient of KL divergence. + */ + template + void klBackward(OutputType&& output); + + //! Get the output parameter. + OutputDataType const& OutputParameter() const { return outputParameter; } + //! Modify the output parameter. + OutputDataType& OutputParameter() { return outputParameter; } + + //! Get the delta. + OutputDataType const& Delta() const { return delta; } + //! Modify the delta. + OutputDataType& Delta() { return delta; } + + //! Get the output size. + size_t const& OutputSize() const { return latentSize; } + //! Modify the output size. + size_t& OutputSize() { return latentSize; } + + //! Get the KL divergence with standard normal. + double Loss() + { + OutputDataType input = join_cols(stdDev, mean); + return klForward(std::move(input)); + } + + /** + * Serialize the layer + */ + template + void serialize(Archive& ar, const unsigned int /* version */); + + private: + //! Locally-stored number of output units. + size_t latentSize; + + //! If false, sample will be constant. + bool stochastic; + + //! If false, KL error will not be included in Backward function. + bool includeKl; + + //! Locally-stored delta object. + OutputDataType delta; + + //! Locally-stored current gaussian sample. + OutputDataType gaussianSample; + + //! Locally-stored current mean. + OutputDataType mean; + + //! Locally-stored pre standard deviation. + //! After softplus activation gives standard deviation. + OutputDataType preStdDev; + + //! Locally-stored current standard deviation. + OutputDataType stdDev; + + //! Locally-stored output parameter object. + OutputDataType outputParameter; +}; // class Reparametrization + +} // namespace ann +} // namespace mlpack + +// Include implementation. +#include "reparametrization_impl.hpp" + +#endif diff --git a/src/mlpack/methods/ann/layer/reparametrization_impl.hpp b/src/mlpack/methods/ann/layer/reparametrization_impl.hpp new file mode 100644 index 0000000000..68e6defdb7 --- /dev/null +++ b/src/mlpack/methods/ann/layer/reparametrization_impl.hpp @@ -0,0 +1,117 @@ +/** + * @file reparametrization_impl.hpp + * @author Atharva Khandait + * + * Implementation of the Reparametrization layer class which samples from a + * gaussian distribution. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_METHODS_ANN_LAYER_REPARAMETRIZATION_IMPL_HPP +#define MLPACK_METHODS_ANN_LAYER_REPARAMETRIZATION_IMPL_HPP + +// In case it hasn't yet been included. +#include "reparametrization.hpp" + +namespace mlpack { +namespace ann /** Artificial Neural Network. */ { + +template +Reparametrization::Reparametrization() : + latentSize(0), + stochastic(true), + includeKl(true) +{ + // Nothing to do here. +} + +template +Reparametrization::Reparametrization( + const size_t latentSize, + const bool stochastic, + const bool includeKl) : + latentSize(latentSize), + stochastic(stochastic), + includeKl(includeKl) +{ + // Nothing to do here. +} + +template +template +void Reparametrization::Forward( + const arma::Mat&& input, arma::Mat&& output) +{ + if (input.n_rows != 2 * latentSize) + { + Log::Fatal << "The output size of layer before the Reparametrization " + << "layer should be 2 * latent size of the Reparametrization layer!" + << std::endl; + } + + mean = input.submat(latentSize, 0, 2 * latentSize - 1, input.n_cols - 1); + preStdDev = input.submat(0, 0, latentSize - 1, input.n_cols - 1); + + if (stochastic) + gaussianSample = arma::randn >(latentSize, input.n_cols); + else + gaussianSample = arma::ones >(latentSize, input.n_cols) * 0.7; + + SoftplusFunction::Fn(preStdDev, stdDev); + output = mean + stdDev % gaussianSample; +} + +template +template +void Reparametrization::Backward( + const arma::Mat&& /* input */, arma::Mat&& gy, arma::Mat&& g) +{ + SoftplusFunction::Deriv(preStdDev, g); + + if (includeKl) + { + arma::Mat klBack; + klBackward(std::move(klBack)); + g = join_cols(gy % std::move(gaussianSample) % g, gy) + std::move(klBack); + } + else + g = join_cols(gy % std::move(gaussianSample) % g, gy); +} + +template +template +double Reparametrization::klForward( + const InputType&& input) +{ + stdDev = input.submat(0, 0, latentSize - 1, input.n_cols - 1); + mean = input.submat(latentSize, 0, 2 * latentSize - 1, input.n_cols - 1); + + return -0.5 * arma::accu(2 * arma::log(stdDev) - + arma::pow(stdDev, 2) - arma::pow(mean, 2) + 1); +} + +template +template +void Reparametrization::klBackward( + OutputType&& output) +{ + SoftplusFunction::Deriv(preStdDev, output); + output = join_cols((-1 / stdDev + stdDev) % output, mean); +} + +template +template +void Reparametrization::serialize( + Archive& ar, const unsigned int /* version */) +{ + ar & BOOST_SERIALIZATION_NVP(latentSize); + ar & BOOST_SERIALIZATION_NVP(stochastic); +} + +} // namespace ann +} // namespace mlpack + +#endif diff --git a/src/mlpack/methods/ann/layer/select.hpp b/src/mlpack/methods/ann/layer/select.hpp index cd57d0c27b..4ec71b9e16 100644 --- a/src/mlpack/methods/ann/layer/select.hpp +++ b/src/mlpack/methods/ann/layer/select.hpp @@ -64,11 +64,6 @@ class Select arma::Mat&& gy, arma::Mat&& g); - //! Get the input parameter. - InputDataType& InputParameter() const { return inputParameter; } - //! Modify the input parameter. - InputDataType& InputParameter() { return inputParameter; } - //! Get the output parameter. OutputDataType& OutputParameter() const { return outputParameter; } //! Modify the output parameter. @@ -95,9 +90,6 @@ class Select //! Locally-stored delta object. OutputDataType delta; - //! Locally-stored input parameter object. - InputDataType inputParameter; - //! Locally-stored output parameter object. OutputDataType outputParameter; }; // class Select diff --git a/src/mlpack/methods/ann/layer/sequential.hpp b/src/mlpack/methods/ann/layer/sequential.hpp index b04f8e25cb..42a82f8d76 100644 --- a/src/mlpack/methods/ann/layer/sequential.hpp +++ b/src/mlpack/methods/ann/layer/sequential.hpp @@ -124,6 +124,7 @@ class Sequential //! Modify the initial point for the optimization. arma::mat& Parameters() { return parameters; } + //! Get the input parameter. arma::mat const& InputParameter() const { return inputParameter; } //! Modify the input parameter. arma::mat& InputParameter() { return inputParameter; } diff --git a/src/mlpack/methods/ann/layer/subview.hpp b/src/mlpack/methods/ann/layer/subview.hpp index d6a60942c5..f6dda59fb9 100644 --- a/src/mlpack/methods/ann/layer/subview.hpp +++ b/src/mlpack/methods/ann/layer/subview.hpp @@ -119,11 +119,6 @@ class Subview g = gy; } - //! Get the input parameter. - InputDataType const& InputParameter() const { return inputParameter; } - //! Modify the input parameter. - InputDataType& InputParameter() { return inputParameter; } - //! Get the output parameter. OutputDataType const& OutputParameter() const { return outputParameter; } //! Modify the output parameter. @@ -166,9 +161,6 @@ class Subview //! Locally-stored delta object. OutputDataType delta; - //! Locally-stored input parameter object. - InputDataType inputParameter; - //! Locally-stored output parameter object. OutputDataType outputParameter; }; // class Subview diff --git a/src/mlpack/methods/ann/layer/transposed_convolution.hpp b/src/mlpack/methods/ann/layer/transposed_convolution.hpp index fc4b20f07d..8df31aa491 100644 --- a/src/mlpack/methods/ann/layer/transposed_convolution.hpp +++ b/src/mlpack/methods/ann/layer/transposed_convolution.hpp @@ -263,12 +263,15 @@ class TransposedConvolution } } - //! Locally-stored number of input units. + //! Locally-stored number of input channels. size_t inSize; - //! Locally-stored number of output units. + //! Locally-stored number of output channels. size_t outSize; + //! Locally-stored number of input units. + size_t batchSize; + //! Locally-stored filter/kernel width. size_t kW; diff --git a/src/mlpack/methods/ann/layer/transposed_convolution_impl.hpp b/src/mlpack/methods/ann/layer/transposed_convolution_impl.hpp index b667cfbc53..3b0b356dbe 100644 --- a/src/mlpack/methods/ann/layer/transposed_convolution_impl.hpp +++ b/src/mlpack/methods/ann/layer/transposed_convolution_impl.hpp @@ -114,30 +114,39 @@ void TransposedConvolution< OutputDataType >::Forward(const arma::Mat&& input, arma::Mat&& output) { - inputTemp = arma::cube(input.memptr(), inputWidth, inputHeight, inSize); + batchSize = input.n_cols; + inputTemp = arma::cube(const_cast&&>(input).memptr(), + inputWidth, inputHeight, inSize * batchSize, false, false); outputWidth = TransposedConvOutSize(inputWidth, kW, dW, padW); outputHeight = TransposedConvOutSize(inputHeight, kH, dH, padH); - output.set_size(outputWidth * outputHeight * outSize, 1); + output.set_size(outputWidth * outputHeight * outSize, batchSize); outputTemp = arma::Cube(output.memptr(), outputWidth, outputHeight, - outSize, false, false); + outSize * batchSize, false, false); outputTemp.zeros(); - for (size_t outMap = 0, outMapIdx = 0; outMap < outSize; outMap++) + for (size_t outMap = 0, outMapIdx = 0, batchCount = 0; outMap < + outSize * batchSize; outMap++) { + if (outMap != 0 && outMap % outSize == 0) + { + batchCount++; + outMapIdx = 0; + } + for (size_t inMap = 0; inMap < inSize; inMap++, outMapIdx++) { arma::Mat convOutput, rotatedFilter; Rotate180(weight.slice(outMapIdx), rotatedFilter); - BackwardConvolutionRule::Convolution(inputTemp.slice(inMap), - rotatedFilter, convOutput, 1, 1); + BackwardConvolutionRule::Convolution(inputTemp.slice(inMap + + batchCount * inSize), rotatedFilter, convOutput, 1, 1); outputTemp.slice(outMap) += convOutput; } - outputTemp.slice(outMap) += bias(outMap); + outputTemp.slice(outMap) += bias(outMap % outSize); } } @@ -158,16 +167,23 @@ void TransposedConvolution< >::Backward( const arma::Mat&& /* input */, arma::Mat&& gy, arma::Mat&& g) { - arma::cube mappedError(gy.memptr(), outputWidth, outputHeight, outSize, - false, false); - - g.set_size(inputTemp.n_rows * inputTemp.n_cols * inputTemp.n_slices, 1); + arma::cube mappedError(gy.memptr(), outputWidth, outputHeight, + outSize * batchSize, false, false); + g.set_size(inputTemp.n_rows * inputTemp.n_cols * inSize, batchSize); gTemp = arma::Cube(g.memptr(), inputTemp.n_rows, inputTemp.n_cols, inputTemp.n_slices, false, false); + gTemp.zeros(); - for (size_t outMap = 0, outMapIdx = 0; outMap < outSize; outMap++) + for (size_t outMap = 0, outMapIdx = 0, batchCount = 0; outMap < + outSize * batchSize; outMap++) { + if (outMap != 0 && outMap % outSize == 0) + { + batchCount++; + outMapIdx = 0; + } + for (size_t inMap = 0; inMap < inSize; inMap++, outMapIdx++) { arma::Mat output; @@ -175,7 +191,7 @@ void TransposedConvolution< ForwardConvolutionRule::Convolution(mappedError.slice(outMap), weight.slice(outMapIdx), output, 1, 1); - gTemp.slice(inMap) += output; + gTemp.slice(inMap + batchCount * inSize) += output; } } } @@ -200,31 +216,36 @@ void TransposedConvolution< arma::Mat&& gradient) { arma::cube mappedError(error.memptr(), outputWidth, - outputHeight, outSize, false, false); + outputHeight, outSize * batchSize, false, false); gradient.set_size(weights.n_elem, 1); - gradientTemp = arma::Cube(gradient.memptr(), weight.n_rows, weight.n_cols, - weight.n_slices, false, false); + gradientTemp = arma::Cube(gradient.memptr(), weight.n_rows, + weight.n_cols, weight.n_slices, false, false); gradientTemp.zeros(); - for (size_t outMap = 0, outMapIdx = 0; outMap < outSize; outMap++) + for (size_t outMap = 0, outMapIdx = 0, batchCount = 0; outMap < + outSize * batchSize; outMap++) { - for (size_t inMap = 0, s = outMap; inMap < inSize; inMap++, outMapIdx++, - s += outSize) + if (outMap != 0 && outMap % outSize == 0) { - arma::Cube inputSlices, output; - inputSlices = inputTemp.slices(inMap, inMap); - arma::Cube deltaSlices = mappedError.slices(outMap, outMap); - - GradientConvolutionRule::Convolution(deltaSlices, inputSlices, - output, 1, 1); - - for (size_t i = 0; i < output.n_slices; i++) - gradientTemp.slice(s) += output.slice(i); + batchCount++; + outMapIdx = 0; } - gradient.submat(weight.n_elem + outMap, 0, weight.n_elem + outMap, 0) = - arma::accu(mappedError.slices(outMap, outMap)); + for (size_t inMap = 0; inMap < inSize; inMap++, outMapIdx++) + { + arma::Mat inputSlice, output; + inputSlice = inputTemp.slice(inMap + batchCount * inSize); + arma::Mat deltaSlice = mappedError.slice(outMap); + + GradientConvolutionRule::Convolution(deltaSlice, inputSlice, + output, 1, 1); + + gradientTemp.slice(outMapIdx) += output; + } + + gradient.submat(weight.n_elem + (outMap % outSize), 0, weight.n_elem + + (outMap % outSize), 0) = arma::accu(mappedError.slices(outMap, outMap)); } } @@ -247,6 +268,7 @@ void TransposedConvolution< { ar & BOOST_SERIALIZATION_NVP(inSize); ar & BOOST_SERIALIZATION_NVP(outSize); + ar & BOOST_SERIALIZATION_NVP(batchSize); ar & BOOST_SERIALIZATION_NVP(kW); ar & BOOST_SERIALIZATION_NVP(kH); ar & BOOST_SERIALIZATION_NVP(dW); diff --git a/src/mlpack/methods/ann/layer/vr_class_reward.hpp b/src/mlpack/methods/ann/layer/vr_class_reward.hpp index b75e73f436..7a6880f054 100644 --- a/src/mlpack/methods/ann/layer/vr_class_reward.hpp +++ b/src/mlpack/methods/ann/layer/vr_class_reward.hpp @@ -73,11 +73,6 @@ class VRClassReward const TargetType&& target, OutputType&& output); - //! Get the input parameter. - InputDataType& InputParameter() const {return inputParameter; } - //! Modify the input parameter. - InputDataType& InputParameter() { return inputParameter; } - //! Get the output parameter. OutputDataType& OutputParameter() const {return outputParameter; } //! Modify the output parameter. @@ -127,9 +122,6 @@ class VRClassReward //! Locally-stored delta object. OutputDataType delta; - //! Locally-stored input parameter object. - InputDataType inputParameter; - //! Locally-stored output parameter object. OutputDataType outputParameter; diff --git a/src/mlpack/methods/ann/loss_functions/CMakeLists.txt b/src/mlpack/methods/ann/loss_functions/CMakeLists.txt index 786c20bd6d..e372725f95 100644 --- a/src/mlpack/methods/ann/loss_functions/CMakeLists.txt +++ b/src/mlpack/methods/ann/loss_functions/CMakeLists.txt @@ -7,6 +7,8 @@ set(SOURCES kl_divergence_impl.hpp mean_squared_error.hpp mean_squared_error_impl.hpp + negative_log_likelihood.hpp + negative_log_likelihood_impl.hpp sigmoid_cross_entropy_error.hpp sigmoid_cross_entropy_error_impl.hpp ) diff --git a/src/mlpack/methods/ann/loss_functions/cross_entropy_error.hpp b/src/mlpack/methods/ann/loss_functions/cross_entropy_error.hpp index dad8eb8c8e..af9e74d26a 100644 --- a/src/mlpack/methods/ann/loss_functions/cross_entropy_error.hpp +++ b/src/mlpack/methods/ann/loss_functions/cross_entropy_error.hpp @@ -62,21 +62,11 @@ class CrossEntropyError const TargetType&& target, OutputType&& output); - //! Get the input parameter. - InputDataType& InputParameter() const { return inputParameter; } - //! Modify the input parameter. - InputDataType& InputParameter() { return inputParameter; } - //! Get the output parameter. OutputDataType& OutputParameter() const { return outputParameter; } //! Modify the output parameter. OutputDataType& OutputParameter() { return outputParameter; } - //! Get the delta. - OutputDataType& Delta() const { return delta; } - //! Modify the delta. - OutputDataType& Delta() { return delta; } - //! Get the epsilon. double Eps() const { return eps; } //! Modify the epsilon. @@ -89,12 +79,6 @@ class CrossEntropyError void serialize(Archive& ar, const unsigned int /* version */); private: - //! Locally-stored delta object. - OutputDataType delta; - - //! Locally-stored input parameter object. - InputDataType inputParameter; - //! Locally-stored output parameter object. OutputDataType outputParameter; diff --git a/src/mlpack/methods/ann/loss_functions/kl_divergence.hpp b/src/mlpack/methods/ann/loss_functions/kl_divergence.hpp index 3bfe0b7bff..87f32de6f9 100644 --- a/src/mlpack/methods/ann/loss_functions/kl_divergence.hpp +++ b/src/mlpack/methods/ann/loss_functions/kl_divergence.hpp @@ -74,21 +74,11 @@ class KLDivergence const TargetType&& target, OutputType&& output); - //! Get the input parameter. - InputDataType& InputParameter() const { return inputParameter; } - //! Modify the input parameter. - InputDataType& InputParameter() { return inputParameter; } - //! Get the output parameter. OutputDataType& OutputParameter() const { return outputParameter; } //! Modify the output parameter. OutputDataType& OutputParameter() { return outputParameter; } - //! Get the delta. - OutputDataType& Delta() const { return delta; } - //! Modify the delta. - OutputDataType& Delta() { return delta; } - //! Get the value of takeMean. bool TakeMean() const { return takeMean; } //! Modify the value of takeMean. @@ -101,12 +91,6 @@ class KLDivergence void serialize(Archive& ar, const unsigned int /* version */); private: - //! Locally-stored delta object. - OutputDataType delta; - - //! Locally-stored input parameter object. - InputDataType inputParameter; - //! Locally-stored output parameter object. OutputDataType outputParameter; diff --git a/src/mlpack/methods/ann/loss_functions/mean_squared_error.hpp b/src/mlpack/methods/ann/loss_functions/mean_squared_error.hpp index 46e829a8ad..59196a23aa 100644 --- a/src/mlpack/methods/ann/loss_functions/mean_squared_error.hpp +++ b/src/mlpack/methods/ann/loss_functions/mean_squared_error.hpp @@ -59,21 +59,11 @@ class MeanSquaredError const TargetType&& target, OutputType&& output); - //! Get the input parameter. - InputDataType& InputParameter() const { return inputParameter; } - //! Modify the input parameter. - InputDataType& InputParameter() { return inputParameter; } - //! Get the output parameter. OutputDataType& OutputParameter() const { return outputParameter; } //! Modify the output parameter. OutputDataType& OutputParameter() { return outputParameter; } - //! Get the delta. - OutputDataType& Delta() const { return delta; } - //! Modify the delta. - OutputDataType& Delta() { return delta; } - /** * Serialize the layer */ @@ -81,12 +71,6 @@ class MeanSquaredError void serialize(Archive& ar, const unsigned int /* version */); private: - //! Locally-stored delta object. - OutputDataType delta; - - //! Locally-stored input parameter object. - InputDataType inputParameter; - //! Locally-stored output parameter object. OutputDataType outputParameter; }; // class MeanSquaredError diff --git a/src/mlpack/methods/ann/layer/negative_log_likelihood.hpp b/src/mlpack/methods/ann/loss_functions/negative_log_likelihood.hpp similarity index 100% rename from src/mlpack/methods/ann/layer/negative_log_likelihood.hpp rename to src/mlpack/methods/ann/loss_functions/negative_log_likelihood.hpp diff --git a/src/mlpack/methods/ann/layer/negative_log_likelihood_impl.hpp b/src/mlpack/methods/ann/loss_functions/negative_log_likelihood_impl.hpp similarity index 100% rename from src/mlpack/methods/ann/layer/negative_log_likelihood_impl.hpp rename to src/mlpack/methods/ann/loss_functions/negative_log_likelihood_impl.hpp diff --git a/src/mlpack/methods/ann/loss_functions/sigmoid_cross_entropy_error.hpp b/src/mlpack/methods/ann/loss_functions/sigmoid_cross_entropy_error.hpp index 91839f69e9..366bf31082 100644 --- a/src/mlpack/methods/ann/loss_functions/sigmoid_cross_entropy_error.hpp +++ b/src/mlpack/methods/ann/loss_functions/sigmoid_cross_entropy_error.hpp @@ -78,21 +78,11 @@ class SigmoidCrossEntropyError const TargetType&& target, OutputType&& output); - //! Get the input parameter. - InputDataType& InputParameter() const { return inputParameter; } - //! Modify the input parameter. - InputDataType& InputParameter() { return inputParameter; } - //! Get the output parameter. OutputDataType& OutputParameter() const { return outputParameter; } //! Modify the output parameter. OutputDataType& OutputParameter() { return outputParameter; } - //! Get the delta. - OutputDataType& Delta() const { return delta; } - //! Modify the delta. - OutputDataType& Delta() { return delta; } - /** * Serialize the layer. */ @@ -100,12 +90,6 @@ class SigmoidCrossEntropyError void serialize(Archive& ar, const unsigned int /* version */); private: - //! Locally-stored delta object. - OutputDataType delta; - - //! Locally-stored input parameter object. - InputDataType inputParameter; - //! Locally-stored output parameter object. OutputDataType outputParameter; }; // class SigmoidCrossEntropy diff --git a/src/mlpack/methods/ann/visitor/CMakeLists.txt b/src/mlpack/methods/ann/visitor/CMakeLists.txt index 187a2bff1b..d11ddf640d 100644 --- a/src/mlpack/methods/ann/visitor/CMakeLists.txt +++ b/src/mlpack/methods/ann/visitor/CMakeLists.txt @@ -25,6 +25,8 @@ set(SOURCES gradient_zero_visitor_impl.hpp load_output_parameter_visitor.hpp load_output_parameter_visitor_impl.hpp + loss_visitor.hpp + loss_visitor_impl.hpp output_height_visitor.hpp output_height_visitor_impl.hpp output_parameter_visitor.hpp diff --git a/src/mlpack/methods/ann/visitor/loss_visitor.hpp b/src/mlpack/methods/ann/visitor/loss_visitor.hpp new file mode 100644 index 0000000000..31b05e6c56 --- /dev/null +++ b/src/mlpack/methods/ann/visitor/loss_visitor.hpp @@ -0,0 +1,69 @@ +/** + * @file loss_visitor.hpp + * @author Atharva Khandait + * + * This file provides an abstraction for the Loss() function for different + * layers and automatically directs any parameter to the right layer type. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_METHODS_ANN_VISITOR_LOSS_VISITOR_HPP +#define MLPACK_METHODS_ANN_VISITOR_LOSS_VISITOR_HPP + +#include + +#include + +namespace mlpack { +namespace ann { + +/** + * LossVisitor exposes the Loss() method of the given module. + */ +class LossVisitor : public boost::static_visitor +{ + public: + //! Return the Loss. + template + double operator()(LayerType* layer) const; + + private: + //! Return 0 if the module doesn't implement the Loss() or Model() function. + template + typename std::enable_if< + !HasLoss::value && + !HasModelCheck::value, double>::type + LayerLoss(T* layer) const; + + //! Return the output height if the module implements the Loss() function. + template + typename std::enable_if< + HasLoss::value && + !HasModelCheck::value, double>::type + LayerLoss(T* layer) const; + + //! Return the loss if the module implements the Model() function. + template + typename std::enable_if< + !HasLoss::value && + HasModelCheck::value, double>::type + LayerLoss(T* layer) const; + + //! Return the loss if the module implements the Model() or loss() function. + template + typename std::enable_if< + HasLoss::value && + HasModelCheck::value, double>::type + LayerLoss(T* layer) const; +}; + +} // namespace ann +} // namespace mlpack + +// Include implementation. +#include "loss_visitor_impl.hpp" + +#endif diff --git a/src/mlpack/methods/ann/visitor/loss_visitor_impl.hpp b/src/mlpack/methods/ann/visitor/loss_visitor_impl.hpp new file mode 100644 index 0000000000..fa19f63948 --- /dev/null +++ b/src/mlpack/methods/ann/visitor/loss_visitor_impl.hpp @@ -0,0 +1,94 @@ +/** + * @file loss_visitor_impl.hpp + * @author Atharva Khandait + * + * Implementation of the Loss() function layer abstraction. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_METHODS_ANN_VISITOR_LOSS_VISITOR_IMPL_HPP +#define MLPACK_METHODS_ANN_VISITOR_LOSS_VISITOR_IMPL_HPP + +// In case it hasn't been included yet. +#include "loss_visitor.hpp" + +namespace mlpack { +namespace ann { + +//! LossVisitor visitor class. +template +inline double LossVisitor::operator()(LayerType* layer) const +{ + return LayerLoss(layer); +} + +template +inline typename std::enable_if< + !HasLoss::value && + !HasModelCheck::value, double>::type +LossVisitor::LayerLoss(T* /* layer */) const +{ + return 0; +} + +template +inline typename std::enable_if< + HasLoss::value && + !HasModelCheck::value, double>::type +LossVisitor::LayerLoss(T* layer) const +{ + return layer->Loss(); +} + +template +inline typename std::enable_if< + !HasLoss::value && + HasModelCheck::value, double>::type +LossVisitor::LayerLoss(T* layer) const +{ + for (size_t i = 0; i < layer->Model().size(); ++i) + { + double Loss = boost::apply_visitor(LossVisitor(), + layer->Model()[layer->Model().size() - 1 - i]); + + if (Loss != 0) + { + return Loss; + } + } + + return 0; +} + +template +inline typename std::enable_if< + HasLoss::value && + HasModelCheck::value, double>::type +LossVisitor::LayerLoss(T* layer) const +{ + double Loss = layer->Loss(); + + if (Loss == 0) + { + for (size_t i = 0; i < layer->Model().size(); ++i) + { + Loss = boost::apply_visitor(LossVisitor(), + layer->Model()[layer->Model().size() - 1 - i]); + + if (Loss != 0) + { + return Loss; + } + } + } + + return Loss; +} + +} // namespace ann +} // namespace mlpack + +#endif diff --git a/src/mlpack/methods/ann/visitor/output_height_visitor.hpp b/src/mlpack/methods/ann/visitor/output_height_visitor.hpp index b2d968d649..05f9c67e2a 100644 --- a/src/mlpack/methods/ann/visitor/output_height_visitor.hpp +++ b/src/mlpack/methods/ann/visitor/output_height_visitor.hpp @@ -22,7 +22,7 @@ namespace mlpack { namespace ann { /** - * OutputWidthVisitor exposes the OutputHeight() method of the given module. + * OutputHeightVisitor exposes the OutputHeight() method of the given module. */ class OutputHeightVisitor : public boost::static_visitor { @@ -55,7 +55,7 @@ class OutputHeightVisitor : public boost::static_visitor HasModelCheck::value, size_t>::type LayerOutputHeight(T* layer) const; - //! Return the output height if the module implement the Model() or + //! Return the output height if the module implements the Model() or //! InputHeight() function. template typename std::enable_if< diff --git a/src/mlpack/methods/cf/CMakeLists.txt b/src/mlpack/methods/cf/CMakeLists.txt index 01796f29da..98655ab244 100644 --- a/src/mlpack/methods/cf/CMakeLists.txt +++ b/src/mlpack/methods/cf/CMakeLists.txt @@ -3,11 +3,13 @@ set(SOURCES cf.hpp cf_impl.hpp - cf.cpp svd_wrapper.hpp svd_wrapper_impl.hpp ) +add_subdirectory(normalization) +add_subdirectory(decomposition_policies) + # Add directory name to sources. set(DIR_SRCS) foreach(file ${SOURCES}) diff --git a/src/mlpack/methods/cf/cf.cpp b/src/mlpack/methods/cf/cf.cpp deleted file mode 100644 index 40b6229928..0000000000 --- a/src/mlpack/methods/cf/cf.cpp +++ /dev/null @@ -1,267 +0,0 @@ -/** - * @file cf.cpp - * @author Mudit Raj Gupta - * @author Sumedh Ghaisas - * - * Collaborative Filtering. - * - * Implementation of CF class to perform Collaborative Filtering on the - * specified data set. - * - * mlpack is free software; you may redistribute it and/or modify it under the - * terms of the 3-clause BSD license. You should have received a copy of the - * 3-clause BSD license along with mlpack. If not, see - * http://www.opensource.org/licenses/BSD-3-Clause for more information. - */ -#include "cf.hpp" - -#include - -namespace mlpack { -namespace cf { - -// Default CFType constructor. -CFType::CFType(const size_t numUsersForSimilarity, - const size_t rank) : - numUsersForSimilarity(numUsersForSimilarity), - rank(rank) -{ - // Validate neighbourhood size. - if (numUsersForSimilarity < 1) - { - Log::Warn << "CFType::CFType(): neighbourhood size should be > 0 (" - << numUsersForSimilarity << " given). Setting value to 5.\n"; - // Set default value of 5. - this->numUsersForSimilarity = 5; - } -} - -void CFType::GetRecommendations(const size_t numRecs, - arma::Mat& recommendations) -{ - // Generate list of users. Maybe it would be more efficient to pass an empty - // users list, and then have the other overload of GetRecommendations() assume - // that if users is empty, then recommendations should be generated for all - // users? - arma::Col users = arma::linspace >(0, - cleanedData.n_cols - 1, cleanedData.n_cols); - - // Call the main overload for recommendations. - GetRecommendations(numRecs, recommendations, users); -} - -void CFType::GetRecommendations(const size_t numRecs, - arma::Mat& recommendations, - const arma::Col& users) -{ - // We want to avoid calculating the full rating matrix, so we will do nearest - // neighbor search only on the H matrix, using the observation that if the - // rating matrix X = W*H, then d(X.col(i), X.col(j)) = d(W H.col(i), W - // H.col(j)). This can be seen as nearest neighbor search on the H matrix - // with the Mahalanobis distance where M^{-1} = W^T W. So, we'll decompose - // M^{-1} = L L^T (the Cholesky decomposition), and then multiply H by L^T. - // Then we can perform nearest neighbor search. - arma::mat l = arma::chol(w.t() * w); - arma::mat stretchedH = l * h; // Due to the Armadillo API, l is L^T. - - // Now, we will use the decomposed w and h matrices to estimate what the user - // would have rated items as, and then pick the best items. - - // Temporarily store feature vector of queried users. - arma::mat query(stretchedH.n_rows, users.n_elem); - - // Select feature vectors of queried users. - for (size_t i = 0; i < users.n_elem; i++) - query.col(i) = stretchedH.col(users(i)); - - // Temporary storage for neighborhood of the queried users. - arma::Mat neighborhood; - - // Calculate the neighborhood of the queried users. Note that the query user - // is part of the neighborhood---this is intentional. We want to use an - // average of both the query user and the local neighborhood of the query - // user. - // The neighbor search technique should be a template parameter. - neighbor::KNN a(stretchedH); - arma::mat resultingDistances; // Temporary storage. - a.Search(query, numUsersForSimilarity, neighborhood, resultingDistances); - - // Generate recommendations for each query user by finding the maximum numRecs - // elements in the averages matrix. - recommendations.set_size(numRecs, users.n_elem); - arma::mat values(numRecs, users.n_elem); - recommendations.fill(SIZE_MAX); - values.fill(DBL_MAX); - - for (size_t i = 0; i < users.n_elem; i++) - { - // First, calculate average of neighborhood values. - arma::vec averages; - averages.zeros(cleanedData.n_rows); - - for (size_t j = 0; j < neighborhood.n_rows; ++j) - averages += w * h.col(neighborhood(j, i)); - averages /= neighborhood.n_rows; - - // Let's build the list of candidate recomendations for the given user. - // Default candidate: the smallest possible value and invalid item number. - const Candidate def = std::make_pair(-DBL_MAX, cleanedData.n_rows); - std::vector vect(numRecs, def); - typedef std::priority_queue, CandidateCmp> - CandidateList; - CandidateList pqueue(CandidateCmp(), std::move(vect)); - - // Look through the averages column corresponding to the current user. - for (size_t j = 0; j < averages.n_rows; ++j) - { - // Ensure that the user hasn't already rated the item. - if (cleanedData(j, users(i)) != 0.0) - continue; // The user already rated the item. - - // Is the estimated value better than the worst candidate? - if (averages[j] > pqueue.top().first) - { - Candidate c = std::make_pair(averages[j], j); - pqueue.pop(); - pqueue.push(c); - } - } - - for (size_t p = 1; p <= numRecs; p++) - { - recommendations(numRecs - p, i) = pqueue.top().second; - values(numRecs - p, i) = pqueue.top().first; - pqueue.pop(); - } - - // If we were not able to come up with enough recommendations, issue a - // warning. - if (recommendations(numRecs - 1, i) == def.second) - Log::Warn << "Could not provide " << numRecs << " recommendations " - << "for user " << users(i) << " (not enough un-rated items)!" - << std::endl; - } -} - -// Predict the rating for a single user/item combination. -double CFType::Predict(const size_t user, const size_t item) const -{ - // First, we need to find the nearest neighbors of the given user. - // We'll use the same technique as for GetRecommendations(). - - // We want to avoid calculating the full rating matrix, so we will do nearest - // neighbor search only on the H matrix, using the observation that if the - // rating matrix X = W*H, then d(X.col(i), X.col(j)) = d(W H.col(i), W - // H.col(j)). This can be seen as nearest neighbor search on the H matrix - // with the Mahalanobis distance where M^{-1} = W^T W. So, we'll decompose - // M^{-1} = L L^T (the Cholesky decomposition), and then multiply H by L^T. - // Then we can perform nearest neighbor search. - arma::mat l = arma::chol(w.t() * w); - arma::mat stretchedH = l * h; // Due to the Armadillo API, l is L^T. - - // Now, we will use the decomposed w and h matrices to estimate what the user - // would have rated items as, and then pick the best items. - - // Temporarily store feature vector of queried users. - arma::mat query = stretchedH.col(user); - - // Temporary storage for neighborhood of the queried users. - arma::Mat neighborhood; - - // Calculate the neighborhood of the queried users. - // This should be a templatized option. - neighbor::KNN a(stretchedH, neighbor::SINGLE_TREE_MODE); - arma::mat resultingDistances; // Temporary storage. - - a.Search(query, numUsersForSimilarity, neighborhood, resultingDistances); - - double rating = 0; // We'll take the average of neighborhood values. - - for (size_t j = 0; j < neighborhood.n_rows; ++j) - rating += arma::as_scalar(w.row(item) * h.col(neighborhood(j, 0))); - rating /= neighborhood.n_rows; - - return rating; -} - -// Predict the rating for a group of user/item combinations. -void CFType::Predict(const arma::Mat& combinations, - arma::vec& predictions) const -{ - // First, for nearest neighbor search, stretch the H matrix. - arma::mat l = arma::chol(w.t() * w); - arma::mat stretchedH = l * h; // Due to the Armadillo API, l is L^T. - - // Now, we must determine those query indices we need to find the nearest - // neighbors for. This is easiest if we just sort the combinations matrix. - arma::Mat sortedCombinations(combinations.n_rows, - combinations.n_cols); - arma::uvec ordering = arma::sort_index(combinations.row(0).t()); - for (size_t i = 0; i < ordering.n_elem; ++i) - sortedCombinations.col(i) = combinations.col(ordering[i]); - - // Now, we have to get the list of unique users we will be searching for. - arma::Col users = arma::unique(combinations.row(0).t()); - - // Assemble our query matrix from the stretchedH matrix. - arma::mat queries(stretchedH.n_rows, users.n_elem); - for (size_t i = 0; i < queries.n_cols; ++i) - queries.col(i) = stretchedH.col(users[i]); - - // Now calculate the neighborhood of these users. - neighbor::KNN a(stretchedH); - arma::mat distances; - arma::Mat neighborhood; - - a.Search(queries, numUsersForSimilarity, neighborhood, distances); - - // Now that we have the neighborhoods we need, calculate the predictions. - predictions.set_size(combinations.n_cols); - - size_t user = 0; // Cumulative user count, because we are doing it in order. - for (size_t i = 0; i < sortedCombinations.n_cols; ++i) - { - // Could this be made faster by calculating dot products for multiple items - // at once? - double rating = 0.0; - - // Map the combination's user to the user ID used for kNN. - while (users[user] < sortedCombinations(0, i)) - ++user; - - for (size_t j = 0; j < neighborhood.n_rows; ++j) - rating += arma::as_scalar(w.row(sortedCombinations(1, i)) * - h.col(neighborhood(j, user))); - rating /= neighborhood.n_rows; - - predictions(ordering[i]) = rating; - } -} - -void CFType::CleanData(const arma::mat& data, arma::sp_mat& cleanedData) -{ - // Generate list of locations for batch insert constructor for sparse - // matrices. - arma::umat locations(2, data.n_cols); - arma::vec values(data.n_cols); - for (size_t i = 0; i < data.n_cols; ++i) - { - // We have to transpose it because items are rows, and users are columns. - locations(1, i) = ((arma::uword) data(0, i)); - locations(0, i) = ((arma::uword) data(1, i)); - values(i) = data(2, i); - if (values(i) == 0) - Log::Warn << "User rating of 0 ignored for user " << locations(1, i) - << ", item " << locations(0, i) << "." << std::endl; - } - - // Find maximum user and item IDs. - const size_t maxItemID = (size_t) max(locations.row(0)) + 1; - const size_t maxUserID = (size_t) max(locations.row(1)) + 1; - - // Fill sparse matrix. - cleanedData = arma::sp_mat(locations, values, maxItemID, maxUserID); -} - -} // namespace cf -} // namespace mlpack diff --git a/src/mlpack/methods/cf/cf.hpp b/src/mlpack/methods/cf/cf.hpp index 6f0418075e..a3f9f0d4ba 100644 --- a/src/mlpack/methods/cf/cf.hpp +++ b/src/mlpack/methods/cf/cf.hpp @@ -21,6 +21,7 @@ #include #include #include +#include #include #include #include @@ -40,7 +41,7 @@ namespace cf /** Collaborative filtering. **/ { * extern arma::Col users; // users seeking recommendations * arma::Mat recommendations; // Recommendations * - * CFType cf(data); // Default options. + * CFType<> cf(data); // Default options. * * // Generate 10 recommendations for all users. * cf.GetRecommendations(10, recommendations); @@ -56,9 +57,11 @@ namespace cf /** Collaborative filtering. **/ { * are in a matrix that holds doubles, should hold integer (or size_t) values. * The user and item indices are assumed to start at 0. * - * @tparam DecompositionPolicy The algorithm to use to decompose - * the rating matrix (a W and H matrix). + * @tparam NormalizationType The type of normalization performed on raw data. + * Data is normalized before calling Train() method. Predicted rating is + * denormalized before return. */ +template class CFType { public: @@ -66,8 +69,7 @@ class CFType * Initialize the CFType object without performing any factorization. Be sure to * call Train() before calling GetRecommendations() or any other functions! */ - CFType(const size_t numUsersForSimilarity = 5, - const size_t rank = 0); + CFType(const size_t numUsersForSimilarity = 5, const size_t rank = 0); /** * Initialize the CFType object using any decomposition method, immediately @@ -80,6 +82,12 @@ class CFType * where each column corresponds to a (user, item, rating) entry in the * matrix or a sparse matrix representing (user, item) table. * + * @tparam MatType The type of input matrix, which is expected to be either + * arma::mat (table of (user, item, rating)) or arma::sp_mat (sparse + * rating matrix where row is item and column is user). + * @tparam DecompositionPolicy The algorithm to use to decompose + * the rating matrix (a W and H matrix). + * * @param data Data matrix: dense matrix (coordinate lists) * or sparse matrix(cleaned). * @param decomposition Instantiated DecompositionPolicy object. @@ -103,6 +111,9 @@ class CFType * parameters that have already been set for the model (specifically, the rank * parameter), and optionally, using the given DecompositionPolicy. * + * @tparam DecompositionPolicy The algorithm to use to decompose + * the rating matrix (a W and H matrix). + * * @param data Input dataset; dense matrix (coordinate lists). * @param decomposition Instantiated DecompositionPolicy object. * @param maxIterations Maximum number of iterations. @@ -121,6 +132,9 @@ class CFType * parameters that have already been set for the model (specifically, the * rank parameter), and optionally, using the given DecompositionPolicy. * + * @tparam DecompositionPolicy The algorithm to use to decompose + * the rating matrix (a W and H matrix). + * * @param data Input dataset; sparse matrix (user item table). * @param decomposition Instantiated DecompositionPolicy object. * @param maxIterations Maximum number of iterations. @@ -234,6 +248,8 @@ class CFType arma::mat h; //! Cleaned data matrix. arma::sp_mat cleanedData; + //! Data normalization object. + NormalizationType normalization; //! Candidate represents a possible recommendation (value, item). typedef std::pair Candidate; diff --git a/src/mlpack/methods/cf/cf_impl.hpp b/src/mlpack/methods/cf/cf_impl.hpp index 36c1965a82..64a128d417 100644 --- a/src/mlpack/methods/cf/cf_impl.hpp +++ b/src/mlpack/methods/cf/cf_impl.hpp @@ -22,17 +22,35 @@ namespace mlpack { namespace cf { +// Default CF constructor. +template +CFType::CFType(const size_t numUsersForSimilarity, + const size_t rank) : + numUsersForSimilarity(numUsersForSimilarity), + rank(rank) +{ + // Validate neighbourhood size. + if (numUsersForSimilarity < 1) + { + Log::Warn << "CFType::CFType(): neighbourhood size should be > 0 (" + << numUsersForSimilarity << " given). Setting value to 5.\n"; + // Set default value of 5. + this->numUsersForSimilarity = 5; + } +} + /** * Construct the CF object using an instantiated decomposition policy. */ +template template -CFType::CFType(const MatType& data, - DecompositionPolicy& decomposition, - const size_t numUsersForSimilarity, - const size_t rank, - const size_t maxIterations, - const double minResidue, - const bool mit) : +CFType::CFType(const MatType& data, + DecompositionPolicy& decomposition, + const size_t numUsersForSimilarity, + const size_t rank, + const size_t maxIterations, + const double minResidue, + const bool mit) : numUsersForSimilarity(numUsersForSimilarity), rank(rank) { @@ -49,14 +67,18 @@ CFType::CFType(const MatType& data, } // Train when data is given in dense matrix form. +template template -void CFType::Train(const arma::mat& data, - DecompositionPolicy& decomposition, - const size_t maxIterations, - const double minResidue, - const bool mit) +void CFType::Train(const arma::mat& data, + DecompositionPolicy& decomposition, + const size_t maxIterations, + const double minResidue, + const bool mit) { - CleanData(data, cleanedData); + // Make a copy of data before performing normalization. + arma::mat normalizedData(data); + normalization.Normalize(normalizedData); + CleanData(normalizedData, cleanedData); // Check if the user wanted us to choose a rank for them. if (rank == 0) @@ -76,20 +98,24 @@ void CFType::Train(const arma::mat& data, // Decompose the data matrix (which is in coordinate list form) to user and // data matrices. Timer::Start("cf_factorization"); - decomposition.Apply(data, cleanedData, rank, w, + decomposition.Apply(normalizedData, cleanedData, rank, w, h, maxIterations, minResidue, mit); Timer::Stop("cf_factorization"); } // Train when data is given as sparse matrix of user item table. +template template -void CFType::Train(const arma::sp_mat& data, - DecompositionPolicy& decomposition, - const size_t maxIterations, - const double minResidue, - const bool mit) +void CFType::Train(const arma::sp_mat& data, + DecompositionPolicy& decomposition, + const size_t maxIterations, + const double minResidue, + const bool mit) { + // data is not used in the following decomposition.Apply() method, so we only + // need to Normalize cleanedData. cleanedData = data; + normalization.Normalize(cleanedData); // Check if the user wanted us to choose a rank for them. if (rank == 0) @@ -114,9 +140,261 @@ void CFType::Train(const arma::sp_mat& data, Timer::Stop("cf_factorization"); } +template +void CFType::GetRecommendations( + const size_t numRecs, + arma::Mat& recommendations) +{ + // Generate list of users. Maybe it would be more efficient to pass an empty + // users list, and then have the other overload of GetRecommendations() assume + // that if users is empty, then recommendations should be generated for all + // users? + arma::Col users = arma::linspace >(0, + cleanedData.n_cols - 1, cleanedData.n_cols); + + // Call the main overload for recommendations. + GetRecommendations(numRecs, recommendations, users); +} + +template +void CFType::GetRecommendations( + const size_t numRecs, + arma::Mat& recommendations, + const arma::Col& users) +{ + // We want to avoid calculating the full rating matrix, so we will do nearest + // neighbor search only on the H matrix, using the observation that if the + // rating matrix X = W*H, then d(X.col(i), X.col(j)) = d(W H.col(i), W + // H.col(j)). This can be seen as nearest neighbor search on the H matrix + // with the Mahalanobis distance where M^{-1} = W^T W. So, we'll decompose + // M^{-1} = L L^T (the Cholesky decomposition), and then multiply H by L^T. + // Then we can perform nearest neighbor search. + arma::mat l = arma::chol(w.t() * w); + arma::mat stretchedH = l * h; // Due to the Armadillo API, l is L^T. + + // Now, we will use the decomposed w and h matrices to estimate what the user + // would have rated items as, and then pick the best items. + + // Temporarily store feature vector of queried users. + arma::mat query(stretchedH.n_rows, users.n_elem); + + // Select feature vectors of queried users. + for (size_t i = 0; i < users.n_elem; i++) + query.col(i) = stretchedH.col(users(i)); + + // Temporary storage for neighborhood of the queried users. + arma::Mat neighborhood; + + // Calculate the neighborhood of the queried users. Note that the query user + // is part of the neighborhood---this is intentional. We want to use an + // average of both the query user and the local neighborhood of the query + // user. + // The neighbor search technique should be a template parameter. + neighbor::KNN a(stretchedH); + arma::mat resultingDistances; // Temporary storage. + a.Search(query, numUsersForSimilarity, neighborhood, resultingDistances); + + // Generate recommendations for each query user by finding the maximum numRecs + // elements in the averages matrix. + recommendations.set_size(numRecs, users.n_elem); + arma::mat values(numRecs, users.n_elem); + recommendations.fill(SIZE_MAX); + values.fill(DBL_MAX); + + for (size_t i = 0; i < users.n_elem; i++) + { + // First, calculate average of neighborhood values. + arma::vec averages; + averages.zeros(cleanedData.n_rows); + + for (size_t j = 0; j < neighborhood.n_rows; ++j) + averages += w * h.col(neighborhood(j, i)); + averages /= neighborhood.n_rows; + + // Let's build the list of candidate recomendations for the given user. + // Default candidate: the smallest possible value and invalid item number. + const Candidate def = std::make_pair(-DBL_MAX, cleanedData.n_rows); + std::vector vect(numRecs, def); + typedef std::priority_queue, CandidateCmp> + CandidateList; + CandidateList pqueue(CandidateCmp(), std::move(vect)); + + // Look through the averages column corresponding to the current user. + for (size_t j = 0; j < averages.n_rows; ++j) + { + // Ensure that the user hasn't already rated the item. + // The algorithm omits rating of zero. Thus, when normalizing original + // ratings in Normalize(), if normalized rating equals zero, it is set + // to the smallest positive double value. + if (cleanedData(j, users(i)) != 0.0) + continue; // The user already rated the item. + + // Is the estimated value better than the worst candidate? + // Denormalize rating before comparison. + double realRating = normalization.Denormalize(users(i), j, averages[j]); + if (realRating > pqueue.top().first) + { + Candidate c = std::make_pair(realRating, j); + pqueue.pop(); + pqueue.push(c); + } + } + + for (size_t p = 1; p <= numRecs; p++) + { + recommendations(numRecs - p, i) = pqueue.top().second; + values(numRecs - p, i) = pqueue.top().first; + pqueue.pop(); + } + + // If we were not able to come up with enough recommendations, issue a + // warning. + if (recommendations(numRecs - 1, i) == def.second) + Log::Warn << "Could not provide " << numRecs << " recommendations " + << "for user " << users(i) << " (not enough un-rated items)!" + << std::endl; + } +} + +// Predict the rating for a single user/item combination. +template +double CFType::Predict(const size_t user, + const size_t item) const +{ + // First, we need to find the nearest neighbors of the given user. + // We'll use the same technique as for GetRecommendations(). + + // We want to avoid calculating the full rating matrix, so we will do nearest + // neighbor search only on the H matrix, using the observation that if the + // rating matrix X = W*H, then d(X.col(i), X.col(j)) = d(W H.col(i), W + // H.col(j)). This can be seen as nearest neighbor search on the H matrix + // with the Mahalanobis distance where M^{-1} = W^T W. So, we'll decompose + // M^{-1} = L L^T (the Cholesky decomposition), and then multiply H by L^T. + // Then we can perform nearest neighbor search. + arma::mat l = arma::chol(w.t() * w); + arma::mat stretchedH = l * h; // Due to the Armadillo API, l is L^T. + + // Now, we will use the decomposed w and h matrices to estimate what the user + // would have rated items as, and then pick the best items. + + // Temporarily store feature vector of queried users. + arma::mat query = stretchedH.col(user); + + // Temporary storage for neighborhood of the queried users. + arma::Mat neighborhood; + + // Calculate the neighborhood of the queried users. + // This should be a templatized option. + neighbor::KNN a(stretchedH, neighbor::SINGLE_TREE_MODE); + arma::mat resultingDistances; // Temporary storage. + + a.Search(query, numUsersForSimilarity, neighborhood, resultingDistances); + + double rating = 0; // We'll take the average of neighborhood values. + + for (size_t j = 0; j < neighborhood.n_rows; ++j) + rating += arma::as_scalar(w.row(item) * h.col(neighborhood(j, 0))); + rating /= neighborhood.n_rows; + + // Denormalize rating and return. + double realRating = normalization.Denormalize(user, item, rating); + return realRating; +} + +// Predict the rating for a group of user/item combinations. +template +void CFType::Predict(const arma::Mat& combinations, + arma::vec& predictions) const +{ + // First, for nearest neighbor search, stretch the H matrix. + arma::mat l = arma::chol(w.t() * w); + arma::mat stretchedH = l * h; // Due to the Armadillo API, l is L^T. + + // Now, we must determine those query indices we need to find the nearest + // neighbors for. This is easiest if we just sort the combinations matrix. + arma::Mat sortedCombinations(combinations.n_rows, + combinations.n_cols); + arma::uvec ordering = arma::sort_index(combinations.row(0).t()); + for (size_t i = 0; i < ordering.n_elem; ++i) + sortedCombinations.col(i) = combinations.col(ordering[i]); + + // Now, we have to get the list of unique users we will be searching for. + arma::Col users = arma::unique(combinations.row(0).t()); + + // Assemble our query matrix from the stretchedH matrix. + arma::mat queries(stretchedH.n_rows, users.n_elem); + for (size_t i = 0; i < queries.n_cols; ++i) + queries.col(i) = stretchedH.col(users[i]); + + // Now calculate the neighborhood of these users. + neighbor::KNN a(stretchedH); + arma::mat distances; + arma::Mat neighborhood; + + a.Search(queries, numUsersForSimilarity, neighborhood, distances); + + // Now that we have the neighborhoods we need, calculate the predictions. + predictions.set_size(combinations.n_cols); + + size_t user = 0; // Cumulative user count, because we are doing it in order. + for (size_t i = 0; i < sortedCombinations.n_cols; ++i) + { + // Could this be made faster by calculating dot products for multiple items + // at once? + double rating = 0.0; + + // Map the combination's user to the user ID used for kNN. + while (users[user] < sortedCombinations(0, i)) + ++user; + + for (size_t j = 0; j < neighborhood.n_rows; ++j) + rating += arma::as_scalar(w.row(sortedCombinations(1, i)) * + h.col(neighborhood(j, user))); + rating /= neighborhood.n_rows; + + predictions(ordering[i]) = rating; + } + + // Denormalize ratings. + normalization.Denormalize(combinations, predictions); +} + +template +void CFType::CleanData(const arma::mat& data, + arma::sp_mat& cleanedData) +{ + // Generate list of locations for batch insert constructor for sparse + // matrices. + arma::umat locations(2, data.n_cols); + arma::vec values(data.n_cols); + for (size_t i = 0; i < data.n_cols; ++i) + { + // We have to transpose it because items are rows, and users are columns. + locations(1, i) = ((arma::uword) data(0, i)); + locations(0, i) = ((arma::uword) data(1, i)); + values(i) = data(2, i); + + // The algorithm omits rating of zero. Thus, when normalizing original + // ratings in Normalize(), if normalized rating equals zero, it is set + // to the smallest positive double value. + if (values(i) == 0) + Log::Warn << "User rating of 0 ignored for user " << locations(1, i) + << ", item " << locations(0, i) << "." << std::endl; + } + + // Find maximum user and item IDs. + const size_t maxItemID = (size_t) max(locations.row(0)) + 1; + const size_t maxUserID = (size_t) max(locations.row(1)) + 1; + + // Fill sparse matrix. + cleanedData = arma::sp_mat(locations, values, maxItemID, maxUserID); +} + //! Serialize the model. +template template -void CFType::serialize(Archive& ar, const unsigned int /* version */) +void CFType::serialize(Archive& ar, + const unsigned int /* version */) { // This model is simple; just serialize all the members. No special handling // required. @@ -125,6 +403,7 @@ void CFType::serialize(Archive& ar, const unsigned int /* version */) ar & BOOST_SERIALIZATION_NVP(w); ar & BOOST_SERIALIZATION_NVP(h); ar & BOOST_SERIALIZATION_NVP(cleanedData); + ar & BOOST_SERIALIZATION_NVP(normalization); } } // namespace cf diff --git a/src/mlpack/methods/cf/cf_main.cpp b/src/mlpack/methods/cf/cf_main.cpp index 0164da62dc..b2929b3697 100644 --- a/src/mlpack/methods/cf/cf_main.cpp +++ b/src/mlpack/methods/cf/cf_main.cpp @@ -105,8 +105,8 @@ PARAM_DOUBLE_IN("min_residue", "Residue required to terminate the factorization" " (lower values generally mean better fits).", "r", 1e-5); // Load/save a model. -PARAM_MODEL_IN(CFType, "input_model", "Trained CF model to load.", "m"); -PARAM_MODEL_OUT(CFType, "output_model", "Output for trained CF model.", "M"); +PARAM_MODEL_IN(CFType<>, "input_model", "Trained CF model to load.", "m"); +PARAM_MODEL_OUT(CFType<>, "output_model", "Output for trained CF model.", "M"); // Query settings. PARAM_UMATRIX_IN("query", "List of query users for which recommendations should" @@ -120,7 +120,7 @@ PARAM_INT_IN("recommendations", "Number of recommendations to generate for each" PARAM_INT_IN("seed", "Set the random seed (0 uses std::time(NULL)).", "s", 0); -void ComputeRecommendations(CFType* cf, +void ComputeRecommendations(CFType<>* cf, const size_t numRecs, arma::Mat& recommendations) { @@ -146,7 +146,7 @@ void ComputeRecommendations(CFType* cf, } } -void ComputeRMSE(CFType* cf) +void ComputeRMSE(CFType<>* cf) { // Now, compute each test point. arma::mat testData = std::move(CLI::GetParam("test")); @@ -173,7 +173,7 @@ void ComputeRMSE(CFType* cf) Log::Info << "RMSE is " << rmse << "." << endl; } -void PerformAction(CFType* c) +void PerformAction(CFType<>* c) { if (CLI::HasParam("query") || CLI::HasParam("all_user_recommendations")) { @@ -191,7 +191,7 @@ void PerformAction(CFType* c) if (CLI::HasParam("test")) ComputeRMSE(c); - CLI::GetParam("output_model") = c; + CLI::GetParam*>("output_model") = c; } template @@ -202,7 +202,7 @@ void PerformAction(arma::mat& dataset, DecompositionPolicy& decomposition) { const size_t neighborhood = (size_t) CLI::GetParam("neighborhood"); - CFType* c = new CFType(dataset, decomposition, neighborhood, rank, + CFType<>* c = new CFType<>(dataset, decomposition, neighborhood, rank, maxIterations, minResidue, CLI::HasParam("iteration_only_termination")); PerformAction(c); @@ -322,7 +322,7 @@ static void mlpackMain() "test" }, true); // Load an input model. - CFType* c = std::move(CLI::GetParam("input_model")); + CFType<>* c = std::move(CLI::GetParam*>("input_model")); PerformAction(c); } diff --git a/src/mlpack/methods/cf/normalization/CMakeLists.txt b/src/mlpack/methods/cf/normalization/CMakeLists.txt new file mode 100644 index 0000000000..648bf7eca5 --- /dev/null +++ b/src/mlpack/methods/cf/normalization/CMakeLists.txt @@ -0,0 +1,19 @@ +# Define the files we need to compile +# Anything not in this list will not be compiled into mlpack. +set(SOURCES + no_normalization.hpp + overall_mean_normalization.hpp + user_mean_normalization.hpp + item_mean_normalization.hpp + z_score_normalization.hpp + combined_normalization.hpp +) + +# Add directory name to sources. +set(DIR_SRCS) +foreach(file ${SOURCES}) + set(DIR_SRCS ${DIR_SRCS} ${CMAKE_CURRENT_SOURCE_DIR}/${file}) +endforeach() +# Append sources (with directory name) to list of all mlpack sources (used at +# the parent scope). +set(MLPACK_SRCS ${MLPACK_SRCS} ${DIR_SRCS} PARENT_SCOPE) diff --git a/src/mlpack/methods/cf/normalization/combined_normalization.hpp b/src/mlpack/methods/cf/normalization/combined_normalization.hpp new file mode 100644 index 0000000000..bddc76520c --- /dev/null +++ b/src/mlpack/methods/cf/normalization/combined_normalization.hpp @@ -0,0 +1,208 @@ +/** + * @file combined_normalization.hpp + * @author Wenhao Huang + * + * CombinedNormalization is a class template for performing a sequence of data + * normalization methods which are specified by template parameter. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_METHODS_CF_NORMALIZATION_COMBINED_NORMALIZATION_HPP +#define MLPACK_METHODS_CF_NORMALIZATION_COMBINED_NORMALIZATION_HPP + +#include + +namespace mlpack { +namespace cf { + +/** + * This normalization class performs a sequence of normalization methods on + * raw ratings. + * + * An example of how to use CombinedNormalization in CF is shown below: + * + * @code + * extern arma::mat data; // data is a (user, item, rating) table. + * // Users for whom recommendations are generated. + * extern arma::Col users; + * arma::Mat recommendations; // Resulting recommendations. + * + * CFType> cf(data); + * + * // Generate 10 recommendations for all users. + * cf.GetRecommendations(10, recommendations); + * @endcode + */ +template +class CombinedNormalization +{ + public: + using TupleType = std::tuple; + + // Empty constructor. + CombinedNormalization() { } + + /** + * Normalize the data by calling Normalize() in each normalization object. + * + * @param data Input dataset. + */ + template + void Normalize(MatType& data) + { + SequenceNormalize<0>(data); + } + + /** + * Denormalize rating by calling Denormalize() in each normalization object. + * Note that the order of objects calling Denormalize() should be the + * reversed order of objects calling Normalize(). + * + * @param user User ID. + * @param item Item ID. + * @param rating Computed rating before denormalization. + */ + double Denormalize(const size_t user, + const size_t item, + const double rating) const + { + return SequenceDenormalize<0>(user, item, rating); + } + + /** + * Denormalize rating by calling Denormalize() in each normalization object. + * Note that the order of objects calling Denormalize() should be the + * reversed order of objects calling Normalize(). + * + * @param combinations User/Item combinations. + * @param predictions Predicted ratings for each user/item combination. + */ + void Denormalize(const arma::Mat& combinations, + arma::vec& predictions) const + { + SequenceDenormalize<0>(combinations, predictions); + } + + /** + * Return normalizations tuple. + */ + TupleType Normalizations() const + { + return normalizations; + } + + /** + * Serialization. + */ + template + void serialize(Archive& ar, const unsigned int version) + { + SequenceSerialize<0, Archive>(ar, version); + } + + private: + //! A tuple of all normalization objects. + TupleType normalizations; + + //! Unpack normalizations tuple to normalize data. + template< + int I, /* Which normalization in tuple to use */ + typename MatType, + typename = std::enable_if_t<(I < std::tuple_size::value)>> + void SequenceNormalize(MatType& data) + { + std::get(normalizations).Normalize(data); + SequenceNormalize(data); + } + + //! End of tuple unpacking. + template< + int I, /* Which normalization in tuple to use */ + typename MatType, + typename = std::enable_if_t<(I >= std::tuple_size::value)>, + typename = void> + void SequenceNormalize(MatType& /* data */) { } + + //! Unpack normalizations tuple to denormalize. + template< + int I, /* Which normalization in tuple to use */ + typename = std::enable_if_t<(I < std::tuple_size::value)>> + double SequenceDenormalize(const size_t user, + const size_t item, + const double rating) const + { + // The order of denormalization should be the reversed order + // of normalization. + double realRating = SequenceDenormalize(user, item, rating); + realRating = + std::get(normalizations).Denormalize(user, item, realRating); + return realRating; + } + + //! End of tuple unpacking. + template< + int I, /* Which normalization in tuple to use */ + typename = std::enable_if_t<(I >= std::tuple_size::value)>, + typename = void> + double SequenceDenormalize(const size_t /* user */, + const size_t /* item */, + const double rating) const + { + return rating; + } + + //! Unpack normalizations tuple to denormalize. + template< + int I, /* Which normalization in tuple to use */ + typename = std::enable_if_t<(I < std::tuple_size::value)>> + void SequenceDenormalize(const arma::Mat& combinations, + arma::vec& predictions) const + { + // The order of denormalization should be the reversed order + // of normalization. + SequenceDenormalize(combinations, predictions); + std::get(normalizations).Denormalize(combinations, predictions); + } + + //! End of tuple unpacking. + template< + int I, /* Which normalization in tuple to use */ + typename = std::enable_if_t<(I >= std::tuple_size::value)>, + typename = void> + void SequenceDenormalize(const arma::Mat& /* combinations */, + arma::vec& /* predictions */) const { } + + //! Unpack normalizations tuple to serialize. + template< + int I, /* Which normalization in tuple to serialize */ + typename Archive, + typename = std::enable_if_t<(I < std::tuple_size::value)>> + void SequenceSerialize(Archive& ar, const unsigned int version) + { + std::string tagName = "normalization_"; + tagName += std::to_string(I); + ar & boost::serialization::make_nvp( + tagName.c_str(), std::get(normalizations)); + SequenceSerialize(ar, version); + } + + //! End of tuple unpacking. + template< + int I, /* Which normalization in tuple to serialize */ + typename Archive, + typename = std::enable_if_t<(I >= std::tuple_size::value)>, + typename = void> + void SequenceSerialize(Archive& /* ar */, const unsigned int /* version */) + { } +}; + +} // namespace cf +} // namespace mlpack + +#endif diff --git a/src/mlpack/methods/cf/normalization/item_mean_normalization.hpp b/src/mlpack/methods/cf/normalization/item_mean_normalization.hpp new file mode 100644 index 0000000000..c4373975b4 --- /dev/null +++ b/src/mlpack/methods/cf/normalization/item_mean_normalization.hpp @@ -0,0 +1,158 @@ +/** + * @file item_mean_normalization.hpp + * @author Wenhao Huang + * + * This class performs item mean normalization on raw ratings. In another + * word, this class is used to remove global effect of item mean. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_METHODS_CF_NORMALIZATION_ITEM_MEAN_NORMALIZATION_HPP +#define MLPACK_METHODS_CF_NORMALIZATION_ITEM_MEAN_NORMALIZATION_HPP + +#include + +namespace mlpack { +namespace cf { + +/** + * This normalization class performs item mean normalization on raw ratings. + * + * An example of how to use ItemMeanNormalization in CF is shown below: + * + * @code + * extern arma::mat data; // data is a (user, item, rating) table. + * // Users for whom recommendations are generated. + * extern arma::Col users; + * arma::Mat recommendations; // Resulting recommendations. + * + * // Use ItemMeanNormalization as normalization method. + * CFType cf(data); + * + * // Generate 10 recommendations for all users. + * cf.GetRecommendations(10, recommendations); + * @endcode + */ +class ItemMeanNormalization +{ + public: + // Empty constructor. + ItemMeanNormalization() { } + + /** + * Normalize the data by subtracting item mean from each of existing ratings. + * + * @param data Input dataset in the form of coordinate list. + */ + void Normalize(arma::mat& data) + { + const size_t itemNum = arma::max(data.row(1)) + 1; + itemMean = arma::vec(itemNum, arma::fill::zeros); + // Number of ratings for each item. + arma::Row ratingNum(itemNum, arma::fill::zeros); + + // Sum ratings for each item. + data.each_col([&](arma::vec& datapoint) + { + const size_t item = (size_t) datapoint(1); + const double rating = datapoint(2); + itemMean(item) += rating; + ratingNum(item) += 1; + }); + + // Calculate item mean and subtract item mean from ratings. + // Set item mean to 0 if the item has no rating. + for (size_t i = 0; i < itemNum; i++) + { + if (ratingNum(i) != 0) + itemMean(i) /= ratingNum(i); + } + + data.each_col([&](arma::vec& datapoint) + { + const size_t item = (size_t) datapoint(1); + datapoint(2) -= itemMean(item); + // The algorithm omits rating of zero. If normalized rating equals zero, + // it is set to the smallest positive double value. + if (datapoint(2) == 0) + datapoint(2) = std::numeric_limits::min(); + }); + } + + /** + * Normalize the data by subtracting item mean from each of existing ratings. + * + * @param cleanedData Input data as a sparse matrix. + */ + void Normalize(arma::sp_mat& cleanedData) + { + itemMean = arma::vec(arma::mean(cleanedData, 1)); + + arma::sp_mat::iterator it = cleanedData.begin(); + arma::sp_mat::iterator it_end = cleanedData.end(); + for (; it != it_end; it++) + { + *it = *it - itemMean(it.row()); + // The algorithm omits rating of zero. If normalized rating equals zero, + // it is set to the smallest positive double value. + if (*it == 0) + *it = std::numeric_limits::min(); + } + } + + /** + * Denormalize computed rating by adding item mean. + * + * @param user User ID. + * @param item Item ID. + * @param rating Computed rating before denormalization. + */ + double Denormalize(const size_t /* user */, + const size_t item, + const double rating) const + { + return rating + itemMean(item); + } + + /** + * Denormalize computed rating by adding item mean. + * + * @param combinations User/Item combinations. + * @param predictions Predicted ratings for each user/item combination. + */ + void Denormalize(const arma::Mat& combinations, + arma::vec& predictions) const + { + for (size_t i = 0; i < predictions.n_elem; i++) + { + const size_t item = combinations(1, i); + predictions(i) += itemMean(item); + } + } + + /** + * Return item mean. + */ + const arma::vec& Mean() const { return itemMean; } + + /** + * Serialization. + */ + template + void serialize(Archive& ar, const unsigned int /* version */) + { + ar & BOOST_SERIALIZATION_NVP(itemMean); + } + + private: + //! Item mean. + arma::vec itemMean; +}; + +} // namespace cf +} // namespace mlpack + +#endif diff --git a/src/mlpack/methods/cf/normalization/no_normalization.hpp b/src/mlpack/methods/cf/normalization/no_normalization.hpp new file mode 100644 index 0000000000..cd070d4a60 --- /dev/null +++ b/src/mlpack/methods/cf/normalization/no_normalization.hpp @@ -0,0 +1,73 @@ +/** + * @file no_normalization.hpp + * @author Wenhao Huang + * + * This class performs no normalization. It is used as default type of + * normalization for CF class. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_METHODS_CF_NORMALIZATION_NO_NORMALIZATION_HPP +#define MLPACK_METHODS_CF_NORMALIZATION_NO_NORMALIZATION_HPP + +#include + +namespace mlpack { +namespace cf { + +/** + * This normalization class doesn't perform any normalization. It is the default + * normalization type for CF class. + */ +class NoNormalization +{ + public: + // Empty constructor. + NoNormalization() { } + + /** + * Do nothing. + * + * @param data Input dataset. + */ + template + inline void Normalize(const MatType& /* data */) const { } + + /** + * Do nothing. + * + * @param user User ID. + * @param item Item ID. + * @param rating Computed rating before denormalization. + */ + inline double Denormalize(const size_t /* user */, + const size_t /* item */, + const double rating) const + { + return rating; + } + + /** + * Do nothing. + * + * @param combinations User/Item combinations. + * @param predictions Predicted ratings for each user/item combination. + */ + inline void Denormalize(const arma::Mat& /* combinations */, + const arma::vec& /* predictions */) const + { } + + /** + * Serialization. + */ + template + void serialize(Archive& /* ar */, const unsigned int /* version */) { } +}; + +} // namespace cf +} // namespace mlpack + +#endif diff --git a/src/mlpack/methods/cf/normalization/overall_mean_normalization.hpp b/src/mlpack/methods/cf/normalization/overall_mean_normalization.hpp new file mode 100644 index 0000000000..4dab41fbc4 --- /dev/null +++ b/src/mlpack/methods/cf/normalization/overall_mean_normalization.hpp @@ -0,0 +1,144 @@ +/** + * @file overall_mean_normalization.hpp + * @author Wenhao Huang + * + * This class performs overall mean normalization on raw ratings. In another + * word, this class is used to remove global effect of overall mean. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_METHODS_CF_NORMALIZATION_OVERALL_MEAN_NORMALIZATION_HPP +#define MLPACK_METHODS_CF_NORMALIZATION_OVERALL_MEAN_NORMALIZATION_HPP + +#include + +namespace mlpack { +namespace cf { + +/** + * This normalization class performs overall mean normalization on raw ratings. + * + * An example of how to use OverallMeanNormalization in CF is shown below: + * + * @code + * extern arma::mat data; // data is a (user, item, rating) table. + * // Users for whom recommendations are generated. + * extern arma::Col users; + * arma::Mat recommendations; // Resulting recommendations. + * + * // Use OverallMeanNormalization as normalization method. + * CFType cf(data); + * + * // Generate 10 recommendations for all users. + * cf.GetRecommendations(10, recommendations); + * @endcode + */ +class OverallMeanNormalization +{ + public: + // Empty constructor. + OverallMeanNormalization() : mean(0) { } + + /** + * Normalize the data by subtracting the mean of all existing ratings. + * + * @param data Input dataset in the form of coordinate list. + */ + void Normalize(arma::mat& data) + { + mean = arma::mean(data.row(2)); + data.row(2) -= mean; + // The algorithm omits rating of zero. If normalized rating equals zero, + // it is set to the smallest positive double value. + data.row(2).for_each([](double& x) + { + if (x == 0) + x = std::numeric_limits::min(); + }); + } + + /** + * Normalize the data by subtracting the mean of all existing ratings. + * + * @param cleanedData Input data as a sparse matrix. + */ + void Normalize(arma::sp_mat& cleanedData) + { + // Caculate mean of all non zero ratings. + if (cleanedData.n_nonzero != 0) + { + mean = arma::accu(cleanedData) / cleanedData.n_nonzero; + // Subtract mean from all non zero ratings. + arma::sp_mat::iterator it = cleanedData.begin(); + arma::sp_mat::iterator it_end = cleanedData.end(); + for (; it != it_end; it++) + { + *it = *it - mean; + // The algorithm omits rating of zero. If normalized rating equals zero, + // it is set to the smallest positive double value. + if (*it == 0) + *it = std::numeric_limits::min(); + } + } + else + { + mean = 0; + // cleanedData remains the same when mean == 0. + } + } + + /** + * Denormalize computed rating by adding mean. + * + * @param user User ID. + * @param item Item ID. + * @param rating Computed rating before denormalization. + */ + double Denormalize(const size_t /* user */, + const size_t /* item */, + const double rating) const + { + return rating + mean; + } + + /** + * Denormalize computed rating by adding mean. + * + * @param combinations User/Item combinations. + * @param predictions Predicted ratings for each user/item combination. + */ + void Denormalize(const arma::Mat& /* combinations */, + arma::vec& predictions) const + { + predictions += mean; + } + + /** + * Return mean. + */ + double Mean() const + { + return mean; + } + + /** + * Serialization. + */ + template + void serialize(Archive& ar, const unsigned int /* version */) + { + ar & BOOST_SERIALIZATION_NVP(mean); + } + + private: + //! Mean of all existing ratings. + double mean; +}; + +} // namespace cf +} // namespace mlpack + +#endif diff --git a/src/mlpack/methods/cf/normalization/user_mean_normalization.hpp b/src/mlpack/methods/cf/normalization/user_mean_normalization.hpp new file mode 100644 index 0000000000..de1fde589d --- /dev/null +++ b/src/mlpack/methods/cf/normalization/user_mean_normalization.hpp @@ -0,0 +1,158 @@ +/** + * @file user_mean_normalization.hpp + * @author Wenhao Huang + * + * This class performs user mean normalization on raw ratings. In another + * word, this class is used to remove global effect of user mean. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_METHODS_CF_NORMALIZATION_USER_MEAN_NORMALIZATION_HPP +#define MLPACK_METHODS_CF_NORMALIZATION_USER_MEAN_NORMALIZATION_HPP + +#include + +namespace mlpack { +namespace cf { + +/** + * This normalization class performs user mean normalization on raw ratings. + * + * An example of how to use UserMeanNormalization in CF is shown below: + * + * @code + * extern arma::mat data; // data is a (user, item, rating) table. + * // Users for whom recommendations are generated. + * extern arma::Col users; + * arma::Mat recommendations; // Resulting recommendations. + * + * // Use UserMeanNormalization as normalization method. + * CFType cf(data); + * + * // Generate 10 recommendations for all users. + * cf.GetRecommendations(10, recommendations); + * @endcode + */ +class UserMeanNormalization +{ + public: + // Empty constructor. + UserMeanNormalization() { } + + /** + * Normalize the data by subtracting user mean from each of existing ratings. + * + * @param data Input dataset in the form of coordinate list. + */ + void Normalize(arma::mat& data) + { + const size_t userNum = arma::max(data.row(0)) + 1; + userMean = arma::rowvec(userNum, arma::fill::zeros); + // Number of ratings for each user. + arma::Row ratingNum(userNum, arma::fill::zeros); + + // Sum ratings for each user. + data.each_col([&](arma::vec& datapoint) + { + const size_t user = (size_t) datapoint(0); + const double rating = datapoint(2); + userMean(user) += rating; + ratingNum(user) += 1; + }); + + // Calculate user mean and subtract user mean from ratings. + // Set user mean to 0 if the user has no rating. + for (size_t i = 0; i < userNum; i++) + { + if (ratingNum(i) != 0) + userMean(i) /= ratingNum(i); + } + + data.each_col([&](arma::vec& datapoint) + { + const size_t user = (size_t) datapoint(0); + datapoint(2) -= userMean(user); + // The algorithm omits rating of zero. If normalized rating equals zero, + // it is set to the smallest positive double value. + if (datapoint(2) == 0) + datapoint(2) = std::numeric_limits::min(); + }); + } + + /** + * Normalize the data by subtracting user mean from each of existing rating. + * + * @param cleanedData Input data as a sparse matrix. + */ + void Normalize(arma::sp_mat& cleanedData) + { + userMean = arma::rowvec(arma::mean(cleanedData, 0)); + + arma::sp_mat::iterator it = cleanedData.begin(); + arma::sp_mat::iterator it_end = cleanedData.end(); + for (; it != it_end; it++) + { + *it = *it - userMean(it.col()); + // The algorithm omits rating of zero. If normalized rating equals zero, + // it is set to the smallest positive double value. + if (*it == 0) + *it = std::numeric_limits::min(); + } + } + + /** + * Denormalize computed rating by adding user mean. + * + * @param user User ID. + * @param item Item ID. + * @param rating Computed rating before denormalization. + */ + double Denormalize(const size_t user, + const size_t /* item */, + const double rating) const + { + return rating + userMean(user); + } + + /** + * Denormalize computed rating by adding user mean. + * + * @param combinations User/Item combinations. + * @param predictions Predicted ratings for each user/item combination. + */ + void Denormalize(const arma::Mat& combinations, + arma::vec& predictions) const + { + for (size_t i = 0; i < predictions.n_elem; i++) + { + const size_t user = combinations(0, i); + predictions(i) += userMean(user); + } + } + + /** + * Return user mean. + */ + const arma::rowvec& Mean() const { return userMean; } + + /** + * Serialization. + */ + template + void serialize(Archive& ar, const unsigned int /* version */) + { + ar & BOOST_SERIALIZATION_NVP(userMean); + } + + private: + //! User mean. + arma::rowvec userMean; +}; + +} // namespace cf +} // namespace mlpack + +#endif diff --git a/src/mlpack/methods/cf/normalization/z_score_normalization.hpp b/src/mlpack/methods/cf/normalization/z_score_normalization.hpp new file mode 100644 index 0000000000..4ae8b22920 --- /dev/null +++ b/src/mlpack/methods/cf/normalization/z_score_normalization.hpp @@ -0,0 +1,165 @@ +/** + * @file z_score_normalization.hpp + * @author Wenhao Huang + * + * This class performs z-score normalization on raw ratings. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_METHODS_CF_NORMALIZATION_Z_SCORE_NORMALIZATION_HPP +#define MLPACK_METHODS_CF_NORMALIZATION_Z_SCORE_NORMALIZATION_HPP + +#include + +namespace mlpack { +namespace cf { + +/** + * This normalization class performs z-score normalization on raw ratings. + * + * An example of how to use ZScoreNormalization in CF is shown below: + * + * @code + * extern arma::mat data; // data is a (user, item, rating) table. + * // Users for whom recommendations are generated. + * extern arma::Col users; + * arma::Mat recommendations; // Resulting recommendations. + * + * // Use ZScoreNormalization as normalization method. + * CFType cf(data); + * + * // Generate 10 recommendations for all users. + * cf.GetRecommendations(10, recommendations); + * @endcode + */ +class ZScoreNormalization +{ + public: + // Empty constructor. + ZScoreNormalization() : mean(0), stddev(1) { } + + /** + * Normalize the data to zero mean and one standard deviation. + * + * @param data Input dataset in the form of coordinate list. + */ + void Normalize(arma::mat& data) + { + mean = arma::mean(data.row(2)); + stddev = arma::stddev(data.row(2)); + + if (std::fabs(stddev) < 1e-14) + { + Log::Fatal << "Standard deviation of all existing ratings is 0! " + << "This may indicate that all existing ratings are the same." + << std::endl; + } + + data.row(2) = (data.row(2) - mean) / stddev; + // The algorithm omits rating of zero. If normalized rating equals zero, + // it is set to the smallest positive double value. + data.row(2).for_each([](double& x) + { + if (x == 0) + x = std::numeric_limits::min(); + }); + } + + /** + * Normalize the data to zero mean and one standard deviation. + * + * @param cleanedData Input data as a sparse matrix. + */ + void Normalize(arma::sp_mat& cleanedData) + { + // Caculate mean and stdev of all non zero ratings. + arma::vec ratings = arma::nonzeros(cleanedData); + mean = arma::mean(ratings); + stddev = arma::stddev(ratings); + + if (std::fabs(stddev) < 1e-14) + { + Log::Fatal << "Standard deviation of all existing ratings is 0! " + << "This may indicate that all existing ratings are the same." + << std::endl; + } + + // Subtract mean from existing rating and divide it by stddev. + arma::sp_mat::iterator it = cleanedData.begin(); + arma::sp_mat::iterator it_end = cleanedData.end(); + for (; it != it_end; it++) + { + *it = (*it - mean) / stddev; + // The algorithm omits rating of zero. If normalized rating equals zero, + // it is set to the smallest positive double value. + if (*it == 0) + *it = std::numeric_limits::min(); + } + } + + /** + * Denormalize computed rating by adding mean and multiplying stddev. + * + * @param user User ID. + * @param item Item ID. + * @param rating Computed rating before denormalization. + */ + double Denormalize(const size_t /* user */, + const size_t /* item */, + const double rating) const + { + return rating * stddev + mean; + } + + /** + * Denormalize computed rating by adding mean and multiplying stddev. + * + * @param combinations User/Item combinations. + * @param predictions Predicted ratings for each user/item combination. + */ + void Denormalize(const arma::Mat& /* combinations */, + arma::vec& predictions) const + { + predictions = predictions * stddev + mean; + } + + /** + * Return mean. + */ + double Mean() const + { + return mean; + } + + /** + * Return stddev. + */ + double Stddev() const + { + return stddev; + } + + /** + * Serialization. + */ + template + void serialize(Archive& ar, const unsigned int /* version */) + { + ar & BOOST_SERIALIZATION_NVP(mean); + ar & BOOST_SERIALIZATION_NVP(stddev); + } + + private: + //! Mean of all existing ratings. + double mean; + //! Standard deviation of all existing ratings. + double stddev; +}; + +} // namespace cf +} // namespace mlpack + +#endif diff --git a/src/mlpack/methods/radical/radical.cpp b/src/mlpack/methods/radical/radical.cpp index a705e61235..2204bd197c 100644 --- a/src/mlpack/methods/radical/radical.cpp +++ b/src/mlpack/methods/radical/radical.cpp @@ -187,7 +187,7 @@ void mlpack::radical::WhitenFeatureMajorMatrix(const mat& matX, { mat matU, matV; vec s; - svd(matU, s, matV, cov(matX)); + arma::svd(matU, s, matV, cov(matX)); matWhitening = matU * diagmat(1 / sqrt(s)) * trans(matV); matXWhitened = matX * matWhitening; } diff --git a/src/mlpack/tests/CMakeLists.txt b/src/mlpack/tests/CMakeLists.txt index bf6dfa75b1..a32e5407d5 100644 --- a/src/mlpack/tests/CMakeLists.txt +++ b/src/mlpack/tests/CMakeLists.txt @@ -26,6 +26,7 @@ add_executable(mlpack_test cosine_tree_test.cpp cv_test.cpp dbscan_test.cpp + dcgan_test.cpp decision_stump_test.cpp decision_tree_test.cpp det_test.cpp diff --git a/src/mlpack/tests/ann_layer_test.cpp b/src/mlpack/tests/ann_layer_test.cpp index f1754e228a..26b79912e3 100644 --- a/src/mlpack/tests/ann_layer_test.cpp +++ b/src/mlpack/tests/ann_layer_test.cpp @@ -229,7 +229,7 @@ BOOST_AUTO_TEST_CASE(JacobianAddLayerTest) } /** - * Add layer numerically gradient test. + * Add layer numerical gradient test. */ BOOST_AUTO_TEST_CASE(GradientAddLayerTest) { @@ -256,7 +256,6 @@ BOOST_AUTO_TEST_CASE(GradientAddLayerTest) double Gradient(arma::mat& gradient) const { - arma::mat output; double error = model->Evaluate(model->Parameters(), 0, 1); model->Gradient(model->Parameters(), 0, gradient, 1); return error; @@ -535,7 +534,7 @@ BOOST_AUTO_TEST_CASE(JacobianLinearLayerTest) } /** - * Linear layer numerically gradient test. + * Linear layer numerical gradient test. */ BOOST_AUTO_TEST_CASE(GradientLinearLayerTest) { @@ -562,7 +561,6 @@ BOOST_AUTO_TEST_CASE(GradientLinearLayerTest) double Gradient(arma::mat& gradient) const { - arma::mat output; double error = model->Evaluate(model->Parameters(), 0, 1); model->Gradient(model->Parameters(), 0, gradient, 1); return error; @@ -619,7 +617,7 @@ BOOST_AUTO_TEST_CASE(JacobianLinearNoBiasLayerTest) } /** - * LinearNoBias layer numerically gradient test. + * LinearNoBias layer numerical gradient test. */ BOOST_AUTO_TEST_CASE(GradientLinearNoBiasLayerTest) { @@ -646,7 +644,6 @@ BOOST_AUTO_TEST_CASE(GradientLinearNoBiasLayerTest) double Gradient(arma::mat& gradient) const { - arma::mat output; double error = model->Evaluate(model->Parameters(), 0, 1); model->Gradient(model->Parameters(), 0, gradient, 1); return error; @@ -721,7 +718,7 @@ BOOST_AUTO_TEST_CASE(JacobianFlexibleReLULayerTest) } /** - * Flexible ReLU layer numerically gradient test. + * Flexible ReLU layer numerical gradient test. */ BOOST_AUTO_TEST_CASE(GradientFlexibleReLULayerTest) { @@ -750,7 +747,6 @@ BOOST_AUTO_TEST_CASE(GradientFlexibleReLULayerTest) double Gradient(arma::mat& gradient) const { - arma::mat output; double error = model->Evaluate(model->Parameters(), 0, 1); model->Gradient(model->Parameters(), 0, gradient, 1); return error; @@ -869,7 +865,7 @@ BOOST_AUTO_TEST_CASE(SimpleAddMergeLayerTest) for (size_t i = 0; i < 5; ++i) { - AddMerge<> module; + AddMerge<> module(false, false); const size_t numMergeModules = math::RandInt(2, 10); for (size_t m = 0; m < numMergeModules; ++m) { @@ -877,7 +873,7 @@ BOOST_AUTO_TEST_CASE(SimpleAddMergeLayerTest) identityLayer.Forward(std::move(input), std::move(identityLayer.OutputParameter())); - module.Add(identityLayer); + module.Add >(identityLayer); } // Test the Forward function. @@ -957,7 +953,6 @@ BOOST_AUTO_TEST_CASE(GradientLSTMLayerTest) double Gradient(arma::mat& gradient) const { - arma::mat output; double error = model->Evaluate(model->Parameters(), 0, 1); model->Gradient(model->Parameters(), 0, gradient, 1); return error; @@ -1039,7 +1034,6 @@ BOOST_AUTO_TEST_CASE(GradientFastLSTMLayerTest) double Gradient(arma::mat& gradient) const { - arma::mat output; double error = model->Evaluate(model->Parameters(), 0, 1); model->Gradient(model->Parameters(), 0, gradient, 1); return error; @@ -1194,7 +1188,7 @@ BOOST_AUTO_TEST_CASE(SimpleConcatLayerTest) } /** - * Concat layer numerically gradient test. + * Concat layer numerical gradient test. */ BOOST_AUTO_TEST_CASE(GradientConcatLayerTest) { @@ -1225,7 +1219,6 @@ BOOST_AUTO_TEST_CASE(GradientConcatLayerTest) double Gradient(arma::mat& gradient) const { - arma::mat output; double error = model->Evaluate(model->Parameters(), 0, 1); model->Gradient(model->Parameters(), 0, gradient, 1); return error; @@ -1394,7 +1387,7 @@ BOOST_AUTO_TEST_CASE(BatchNormTest) } /** - * BatchNorm layer numerically gradient test. + * BatchNorm layer numerical gradient test. */ BOOST_AUTO_TEST_CASE(GradientBatchNormTest) { @@ -1423,7 +1416,6 @@ BOOST_AUTO_TEST_CASE(GradientBatchNormTest) double Gradient(arma::mat& gradient) const { - arma::mat output; double error = model->Evaluate(model->Parameters(), 0, 256, false); model->Gradient(model->Parameters(), 0, gradient, 256); return error; @@ -1566,7 +1558,7 @@ BOOST_AUTO_TEST_CASE(SimpleTransposedConvolutionLayerTest) } /** - * Transposed Convolution layer numerically gradient test. + * Transposed Convolution layer numerical gradient test. */ BOOST_AUTO_TEST_CASE(GradientTransposedConvolutionLayerTest) { @@ -1592,7 +1584,6 @@ BOOST_AUTO_TEST_CASE(GradientTransposedConvolutionLayerTest) double Gradient(arma::mat& gradient) const { - arma::mat output; double error = model->Evaluate(model->Parameters(), 0, 1); model->Gradient(model->Parameters(), 0, gradient, 1); return error; @@ -1617,7 +1608,7 @@ BOOST_AUTO_TEST_CASE(SimpleMultiplyMergeLayerTest) for (size_t i = 0; i < 5; ++i) { - MultiplyMerge<> module; + MultiplyMerge<> module(false, false); const size_t numMergeModules = math::RandInt(2, 10); for (size_t m = 0; m < numMergeModules; ++m) { @@ -1625,7 +1616,7 @@ BOOST_AUTO_TEST_CASE(SimpleMultiplyMergeLayerTest) identityLayer.Forward(std::move(input), std::move(identityLayer.OutputParameter())); - module.Add(identityLayer); + module.Add >(identityLayer); } // Test the Forward function. @@ -1678,7 +1669,7 @@ BOOST_AUTO_TEST_CASE(SimpleAtrousConvolutionLayerTest) } /** - * Atrous Convolution layer numerically gradient test. + * Atrous Convolution layer numerical gradient test. */ BOOST_AUTO_TEST_CASE(GradientAtrousConvolutionLayerTest) { @@ -1704,7 +1695,6 @@ BOOST_AUTO_TEST_CASE(GradientAtrousConvolutionLayerTest) double Gradient(arma::mat& gradient) const { - arma::mat output; double error = model->Evaluate(model->Parameters(), 0, 1); model->Gradient(model->Parameters(), 0, gradient, 1); return error; @@ -1754,7 +1744,7 @@ BOOST_AUTO_TEST_CASE(LayerNormTest) } /** - * LayerNorm layer numerically gradient test. + * LayerNorm layer numerical gradient test. */ BOOST_AUTO_TEST_CASE(GradientLayerNormTest) { @@ -1783,7 +1773,6 @@ BOOST_AUTO_TEST_CASE(GradientLayerNormTest) double Gradient(arma::mat& gradient) const { - arma::mat output; double error = model->Evaluate(model->Parameters(), 0, 256, false); model->Gradient(model->Parameters(), 0, gradient, 256); return error; @@ -1798,6 +1787,70 @@ BOOST_AUTO_TEST_CASE(GradientLayerNormTest) BOOST_REQUIRE_LE(CheckGradient(function), 1e-4); } +/** + * Test if the AddMerge layer is able to forward the + * Forward/Backward/Gradient calls. + */ +BOOST_AUTO_TEST_CASE(AddMergeRunTest) +{ + arma::mat output, input, delta, error; + + AddMerge<> module(true, true); + + Linear<>* linear = new Linear<>(10, 10); + module.Add(linear); + + linear->Parameters().randu(); + linear->Reset(); + + input = arma::zeros(10, 1); + module.Forward(std::move(input), std::move(output)); + + double parameterSum = arma::accu(linear->Parameters().submat( + 100, 0, linear->Parameters().n_elem - 1, 0)); + + // Test the Backward function. + module.Backward(std::move(input), std::move(input), std::move(delta)); + + // Clean up before we break, + delete linear; + + BOOST_REQUIRE_CLOSE(parameterSum, arma::accu(output), 1e-3); + BOOST_REQUIRE_EQUAL(arma::accu(delta), 0); +} + +/** + * Test if the MultiplyMerge layer is able to forward the + * Forward/Backward/Gradient calls. + */ +BOOST_AUTO_TEST_CASE(MultiplyMergeRunTest) +{ + arma::mat output, input, delta, error; + + MultiplyMerge<> module(true, true); + + Linear<>* linear = new Linear<>(10, 10); + module.Add(linear); + + linear->Parameters().randu(); + linear->Reset(); + + input = arma::zeros(10, 1); + module.Forward(std::move(input), std::move(output)); + + double parameterSum = arma::accu(linear->Parameters().submat( + 100, 0, linear->Parameters().n_elem - 1, 0)); + + // Test the Backward function. + module.Backward(std::move(input), std::move(input), std::move(delta)); + + // Clean up before we break, + delete linear; + + BOOST_REQUIRE_CLOSE(parameterSum, arma::accu(output), 1e-3); + BOOST_REQUIRE_EQUAL(arma::accu(delta), 0); +} + /** * Simple subview module test. */ @@ -1887,4 +1940,122 @@ BOOST_AUTO_TEST_CASE(SubviewBatchTest) CheckMatrices(outputDef, output); } -BOOST_AUTO_TEST_SUITE_END(); +/* + * Simple Reparametrization module test. + */ +BOOST_AUTO_TEST_CASE(SimpleReparametrizationLayerTest) +{ + arma::mat input, output, delta; + Reparametrization<> module(5); + + // Test the Forward function. + input = join_cols(arma::ones(5, 1) * -20, + arma::zeros(5, 1)); + module.Forward(std::move(input), std::move(output)); + BOOST_REQUIRE_LE(arma::accu(output), 1e-5); + + // Test the Backward function. + arma::mat gy = arma::zeros(5, 1); + module.Backward(std::move(input), std::move(gy), std::move(delta)); + BOOST_REQUIRE(arma::accu(delta) != 0); // klBackward will be added. +} + +/** + * Reparametrization module stochastic boolean test. + */ +BOOST_AUTO_TEST_CASE(ReparametrizationLayerStochasticTest) +{ + arma::mat input, outputA, outputB; + Reparametrization<> module(5, false); + + input = join_cols(arma::ones(5, 1), + arma::zeros(5, 1)); + + // Test if two forward passes generate same output. + module.Forward(std::move(input), std::move(outputA)); + module.Forward(std::move(input), std::move(outputB)); + + CheckMatrices(std::move(outputA), std::move(outputB)); +} + +/** + * Reparametrization module includeKl boolean test. + */ +BOOST_AUTO_TEST_CASE(ReparametrizationLayerIncludeKlTest) +{ + arma::mat input, output, gy, delta; + Reparametrization<> module(5, true, false); + + input = join_cols(arma::ones(5, 1), + arma::zeros(5, 1)); + module.Forward(std::move(input), std::move(output)); + gy = arma::zeros(output.n_rows, output.n_cols); + module.Backward(std::move(output), std::move(gy), std::move(delta)); + + BOOST_REQUIRE_EQUAL(arma::accu(std::move(delta)), 0); +} + +/** + * Jacobian Reparametrization module test. + */ +BOOST_AUTO_TEST_CASE(JacobianReparametrizationLayerTest) +{ + for (size_t i = 0; i < 5; i++) + { + const size_t inputElementsHalf = math::RandInt(2, 1000); + + arma::mat input; + input.set_size(inputElementsHalf * 2, 1); + + Reparametrization<> module(inputElementsHalf, false, false); + + double error = JacobianTest(module, input); + BOOST_REQUIRE_LE(error, 1e-5); + } +} + +/** + * Reparametrization layer numerical gradient test. + */ +BOOST_AUTO_TEST_CASE(GradientReparametrizationLayerTest) +{ + // Linear function gradient instantiation. + struct GradientFunction + { + GradientFunction() + { + input = arma::randu(10, 1); + target = arma::mat("1"); + + model = new FFN, NguyenWidrowInitialization>(); + model->Predictors() = input; + model->Responses() = target; + model->Add >(); + model->Add >(10, 6); + model->Add >(3, false); + model->Add >(3, 2); + model->Add >(); + } + + ~GradientFunction() + { + delete model; + } + + double Gradient(arma::mat& gradient) const + { + double error = model->Evaluate(model->Parameters(), 0, 1); + model->Gradient(model->Parameters(), 0, gradient, 1); + return error; + } + + arma::mat& Parameters() { return model->Parameters(); } + + FFN, NguyenWidrowInitialization>* model; + arma::mat input, target; + } function; + + BOOST_REQUIRE_LE(CheckGradient(function), 1e-4); +} + +BOOST_AUTO_TEST_SUITE_END(); \ No newline at end of file diff --git a/src/mlpack/tests/bigbatch_sgd_test.cpp b/src/mlpack/tests/bigbatch_sgd_test.cpp index ca365d61bc..a710563439 100644 --- a/src/mlpack/tests/bigbatch_sgd_test.cpp +++ b/src/mlpack/tests/bigbatch_sgd_test.cpp @@ -84,8 +84,6 @@ void CreateLogisticRegressionTestData(arma::mat& data, */ BOOST_AUTO_TEST_CASE(BBSBBLogisticRegressionTest) { - mlpack::math::RandomSeed(time(NULL)); - arma::mat data, testData, shuffledData; arma::Row responses, testResponses, shuffledResponses; @@ -113,8 +111,6 @@ BOOST_AUTO_TEST_CASE(BBSBBLogisticRegressionTest) */ BOOST_AUTO_TEST_CASE(BBSArmijoLogisticRegressionTest) { - mlpack::math::RandomSeed(time(NULL)); - arma::mat data, testData, shuffledData; arma::Row responses, testResponses, shuffledResponses; diff --git a/src/mlpack/tests/cf_test.cpp b/src/mlpack/tests/cf_test.cpp index 1990b0b029..d991e0ff1b 100644 --- a/src/mlpack/tests/cf_test.cpp +++ b/src/mlpack/tests/cf_test.cpp @@ -18,6 +18,12 @@ #include #include #include +#include +#include +#include +#include +#include +#include #include @@ -36,7 +42,7 @@ using namespace std; * set. Default case. */ template -void GetRecommendationsAllUsers(bool cleanData = true) +void GetRecommendationsAllUsers() { DecompositionPolicy decomposition; // Dummy number of recommendations. @@ -51,16 +57,7 @@ void GetRecommendationsAllUsers(bool cleanData = true) arma::mat dataset; data::Load("GroupLensSmall.csv", dataset); - if (cleanData) - { - // Make data into sparse matrix. - arma::sp_mat cleanedData; - CFType::CleanData(dataset, cleanedData); - - arma::sp_mat dataset; - dataset = cleanedData; - } - CFType c(dataset, decomposition, 5, 5, 70); + CFType<> c(dataset, decomposition, 5, 5, 70); // Generate recommendations when query set is not specified. c.GetRecommendations(numRecs, recommendations); @@ -76,7 +73,7 @@ void GetRecommendationsAllUsers(bool cleanData = true) * Make sure that the recommendations are generated for queried users only. */ template -void GetRecommendationsQueriedUser(bool cleanData = true) +void GetRecommendationsQueriedUser() { DecompositionPolicy decomposition; // Number of users that we will search for recommendations for. @@ -97,17 +94,7 @@ void GetRecommendationsQueriedUser(bool cleanData = true) arma::mat dataset; data::Load("GroupLensSmall.csv", dataset); - if (cleanData) - { - // Make data into sparse matrix. - arma::sp_mat cleanedData; - CFType::CleanData(dataset, cleanedData); - - arma::sp_mat dataset; - dataset = cleanedData; - } - - CFType c(dataset, decomposition, 5, 5, 70); + CFType<> c(dataset, decomposition, 5, 5, 70); // Generate recommendations when query set is specified. c.GetRecommendations(numRecsDefault, recommendations, users); @@ -122,8 +109,9 @@ void GetRecommendationsQueriedUser(bool cleanData = true) /** * Make sure recommendations that are generated are reasonably accurate. */ -template -void RecommendationAccuracy(bool cleanData = true) +template +void RecommendationAccuracy() { DecompositionPolicy decomposition; // Load the GroupLens dataset; then, we will remove some values from it. @@ -164,17 +152,7 @@ void RecommendationAccuracy(bool cleanData = true) } } - if (cleanData) - { - // Make data into sparse matrix. - arma::sp_mat cleanedData; - CFType::CleanData(dataset, cleanedData); - - arma::sp_mat dataset; - dataset = cleanedData; - } - - CFType c(dataset, decomposition, 5, 5, 70); + CFType c(dataset, decomposition, 5, 5, 70); // Obtain 150 recommendations for the users in savedCols, and make sure the // missing item shows up in most of them. First, create the list of users, @@ -223,8 +201,9 @@ void RecommendationAccuracy(bool cleanData = true) } // Make sure that Predict() is returning reasonable results. -template -void CFPredict(bool cleanData = true) +template +void CFPredict(const double rmseBound = 2.0) { DecompositionPolicy decomposition; // Load the GroupLens dataset; then, we will remove some values from it. @@ -265,17 +244,7 @@ void CFPredict(bool cleanData = true) } } - if (cleanData) - { - // Make data into sparse matrix. - arma::sp_mat cleanedData; - CFType::CleanData(dataset, cleanedData); - - arma::sp_mat dataset; - dataset = cleanedData; - } - - CFType c(dataset, decomposition, 5, 5, 70); + CFType c(dataset, decomposition, 5, 5, 70); // Now, for each removed rating, make sure the prediction is... reasonably // accurate. @@ -288,17 +257,17 @@ void CFPredict(bool cleanData = true) totalError += error; } - totalError = std::sqrt(totalError) / savedCols.n_cols; + const double rmse = std::sqrt(totalError / savedCols.n_cols); - // The mean squared error should be less than one. - BOOST_REQUIRE_LT(totalError, 0.6); + // The root mean square error should be less than ?. + BOOST_REQUIRE_LT(rmse, rmseBound); } // Do the same thing as the previous test, but ensure that the ratings we // predict with the batch Predict() are the same as the individual Predict() // calls. template -void BatchPredict(bool cleanData = true) +void BatchPredict() { DecompositionPolicy decomposition; // Load the GroupLens dataset; then, we will remove some values from it. @@ -339,17 +308,7 @@ void BatchPredict(bool cleanData = true) } } - if (cleanData) - { - // Make data into sparse matrix. - arma::sp_mat cleanedData; - CFType::CleanData(dataset, cleanedData); - - arma::sp_mat dataset; - dataset = cleanedData; - } - - CFType c(dataset, decomposition, 5, 5, 70); + CFType<> c(dataset, decomposition, 5, 5, 70); // Get predictions for all user/item pairs we held back. arma::Mat combinations(2, savedCols.n_cols); @@ -378,7 +337,7 @@ void Train(DecompositionPolicy& decomposition) // Generate random data. arma::sp_mat randomData; randomData.sprandu(100, 100, 0.3); - CFType c(randomData, decomposition, 5, 5, 70); + CFType<> c(randomData, decomposition, 5, 5, 70); // Now retrain with data we know about. arma::mat dataset; @@ -420,7 +379,7 @@ void Train(DecompositionPolicy& decomposition) // Make data into sparse matrix. arma::sp_mat cleanedData; - CFType::CleanData(dataset, cleanedData); + CFType<>::CleanData(dataset, cleanedData); // Now retrain. c.Train(dataset, decomposition, 70); @@ -452,7 +411,7 @@ void Train<>(RegSVDPolicy& decomposition) { arma::mat randomData = arma::zeros(100, 100); randomData.diag().ones(); - CFType c(randomData, decomposition, 5, 5, 70); + CFType<> c(randomData, decomposition, 5, 5, 70); // Now retrain with data we know about. arma::mat dataset; @@ -517,11 +476,11 @@ void Train<>(RegSVDPolicy& decomposition) * Make sure we can train a model after using the empty constructor. */ template -void EmptyConstructorTrain(bool cleanData = true) +void EmptyConstructorTrain() { DecompositionPolicy decomposition; // Use default constructor. - CFType c; + CFType<> c; // Now retrain with data we know about. arma::mat dataset; @@ -561,16 +520,6 @@ void EmptyConstructorTrain(bool cleanData = true) } } - if (cleanData) - { - // Make data into sparse matrix. - arma::sp_mat cleanedData; - CFType::CleanData(dataset, cleanedData); - - arma::sp_mat dataset; - dataset = cleanedData; - } - c.Train(dataset, decomposition, 70); // Get predictions for all user/item pairs we held back. @@ -595,7 +544,8 @@ void EmptyConstructorTrain(bool cleanData = true) /** * Ensure we can load and save the CF model. */ -template +template void Serialization() { DecompositionPolicy decomposition; @@ -604,16 +554,16 @@ void Serialization() data::Load("GroupLensSmall.csv", dataset); arma::sp_mat cleanedData; - CFType::CleanData(dataset, cleanedData); + CFType::CleanData(dataset, cleanedData); - CFType c(cleanedData, decomposition, 5, 5, 70); + CFType c(cleanedData, decomposition, 5, 5, 70); arma::sp_mat randomData; randomData.sprandu(100, 100, 0.3); - CFType cXml(randomData, decomposition, 5, 5, 70); - CFType cBinary; - CFType cText(cleanedData, decomposition, 5, 5, 70); + CFType cXml(randomData, decomposition, 5, 5, 70); + CFType cBinary; + CFType cText(cleanedData, decomposition, 5, 5, 70); SerializeObjectAll(c, cXml, cText, cBinary); @@ -690,7 +640,7 @@ BOOST_AUTO_TEST_CASE(CFGetRecommendationsAllUsersRandSVDTest) */ BOOST_AUTO_TEST_CASE(CFGetRecommendationsAllUsersRegSVDTest) { - GetRecommendationsAllUsers(false); + GetRecommendationsAllUsers(); } /** @@ -745,7 +695,7 @@ BOOST_AUTO_TEST_CASE(CFGetRecommendationsQueriedUserRandSVDTest) */ BOOST_AUTO_TEST_CASE(CFGetRecommendationsQueriedUserRegSVDTest) { - GetRecommendationsQueriedUser(false); + GetRecommendationsQueriedUser(); } /** @@ -799,7 +749,7 @@ BOOST_AUTO_TEST_CASE(RecommendationAccuracyRandSVDTest) */ BOOST_AUTO_TEST_CASE(RecommendationAccuracyRegSVDTest) { - RecommendationAccuracy(false); + RecommendationAccuracy(); } /** @@ -841,13 +791,13 @@ BOOST_AUTO_TEST_CASE(RecommendationAccuracySVDIncompleteTest) // Make sure that Predict() is returning reasonable results for randomized SVD. BOOST_AUTO_TEST_CASE(CFPredictRandSVDTest) { - CFPredict(); + CFPredict(4.5); } // Make sure that Predict() is returning reasonable results for regularized SVD. BOOST_AUTO_TEST_CASE(CFPredictRegSVDTest) { - CFPredict(false); + CFPredict(); } // Make sure that Predict() is returning reasonable results for batch SVD. @@ -859,7 +809,7 @@ BOOST_AUTO_TEST_CASE(CFPredictBatchSVDTest) // Make sure that Predict() is returning reasonable results for NMF. BOOST_AUTO_TEST_CASE(CFPredictNMFTest) { - CFPredict(); + CFPredict(3.5); } /** @@ -868,7 +818,7 @@ BOOST_AUTO_TEST_CASE(CFPredictNMFTest) */ BOOST_AUTO_TEST_CASE(CFPredictSVDCompleteTest) { - CFPredict(); + CFPredict(3.5); } /** @@ -877,7 +827,7 @@ BOOST_AUTO_TEST_CASE(CFPredictSVDCompleteTest) */ BOOST_AUTO_TEST_CASE(CFPredictSVDIncompleteTest) { - CFPredict(); + CFPredict(3.5); } // Compare batch Predict() and individual Predict() for randomized SVD. @@ -889,7 +839,7 @@ BOOST_AUTO_TEST_CASE(CFBatchPredictRandSVDTest) // Compare batch Predict() and individual Predict() for regularized SVD. BOOST_AUTO_TEST_CASE(CFBatchPredictRegSVDTest) { - BatchPredict(false); + BatchPredict(); } // Compare batch Predict() and individual Predict() for batch SVD. @@ -993,7 +943,7 @@ BOOST_AUTO_TEST_CASE(EmptyConstructorTrainRandSVDTest) */ BOOST_AUTO_TEST_CASE(EmptyConstructorTrainRegSVDTest) { - EmptyConstructorTrain(false); + EmptyConstructorTrain(); } /** @@ -1072,4 +1022,147 @@ BOOST_AUTO_TEST_CASE(SerializationSVDIncompleteTest) Serialization(); } +/** + * Make sure that Predict() is returning reasonable results for NMF and + * OverallMeanNormalization. + */ +BOOST_AUTO_TEST_CASE(CFPredictOverallMeanNormalization) +{ + CFPredict(); +} + +/** + * Make sure that Predict() is returning reasonable results for NMF and + * UserMeanNormalization. + */ +BOOST_AUTO_TEST_CASE(CFPredictUserMeanNormalization) +{ + CFPredict(); +} + +/** + * Make sure that Predict() is returning reasonable results for NMF and + * ItemMeanNormalization. + */ +BOOST_AUTO_TEST_CASE(CFPredictItemMeanNormalization) +{ + CFPredict(); +} + +/** + * Make sure that Predict() is returning reasonable results for NMF and + * ZScoreNormalization. + */ +BOOST_AUTO_TEST_CASE(CFPredictZScoreNormalization) +{ + CFPredict(); +} + +/** + * Make sure that Predict() is returning reasonable results for NMF and + * CombinedNormalization. + */ +BOOST_AUTO_TEST_CASE(CFPredictCombinedNormalization) +{ + CFPredict>(); +} + +/** + * Make sure recommendations that are generated are reasonably accurate + * for OverallMeanNormalization. + */ +BOOST_AUTO_TEST_CASE(RecommendationAccuracyOverallMeanNormalizationTest) +{ + RecommendationAccuracy(); +} + +/** + * Make sure recommendations that are generated are reasonably accurate + * for UserMeanNormalization. + */ +BOOST_AUTO_TEST_CASE(RecommendationAccuracyUserMeanNormalizationTest) +{ + RecommendationAccuracy(); +} + +/** + * Make sure recommendations that are generated are reasonably accurate + * for ItemMeanNormalization. + */ +BOOST_AUTO_TEST_CASE(RecommendationAccuracyItemMeanNormalizationTest) +{ + RecommendationAccuracy(); +} + +/** + * Make sure recommendations that are generated are reasonably accurate + * for ZScoreNormalization. + */ +BOOST_AUTO_TEST_CASE(RecommendationAccuracyZScoreNormalizationTest) +{ + RecommendationAccuracy(); +} + +/** + * Make sure recommendations that are generated are reasonably accurate + * for CombinedNormalization. + */ +BOOST_AUTO_TEST_CASE(RecommendationAccuracyCombinedNormalizationTest) +{ + RecommendationAccuracy>(); +} + +/** + * Ensure we can load and save the CF model using OverallMeanNormalization. + */ +BOOST_AUTO_TEST_CASE(SerializationOverallMeanNormalizationTest) +{ + Serialization(); +} + +/** + * Ensure we can load and save the CF model using UserMeanNormalization. + */ +BOOST_AUTO_TEST_CASE(SerializationUserMeanNormalizationTest) +{ + Serialization(); +} + +/** + * Ensure we can load and save the CF model using ItemMeanNormalization. + */ +BOOST_AUTO_TEST_CASE(SerializationItemMeanNormalizationTest) +{ + Serialization(); +} + +/** + * Ensure we can load and save the CF model using ZScoreMeanNormalization. + */ +BOOST_AUTO_TEST_CASE(SerializationZScoreNormalizationTest) +{ + Serialization(); +} + +/** + * Ensure we can load and save the CF model using CombinedNormalization. + */ +BOOST_AUTO_TEST_CASE(SerializationCombinedNormalizationTest) +{ + Serialization>(); +} + BOOST_AUTO_TEST_SUITE_END(); diff --git a/src/mlpack/tests/convolutional_network_test.cpp b/src/mlpack/tests/convolutional_network_test.cpp index 831975bb8d..1f7c07078f 100644 --- a/src/mlpack/tests/convolutional_network_test.cpp +++ b/src/mlpack/tests/convolutional_network_test.cpp @@ -74,48 +74,59 @@ BOOST_AUTO_TEST_CASE(VanillaNetworkTest) * | | +-+ | +-+ | +-+ | +-+ | | | * +---+ +---+ +---+ +---+ +---+ +---+ */ - - FFN, RandomInitialization> model; - - model.Add >(1, 8, 5, 5, 1, 1, 0, 0, 28, 28); - model.Add >(); - model.Add >(8, 8, 2, 2); - model.Add >(8, 12, 2, 2); - model.Add >(); - model.Add >(2, 2, 2, 2); - model.Add >(192, 20); - model.Add >(); - model.Add >(20, 10); - model.Add >(); - model.Add >(10, 2); - model.Add >(); - - // Train for only 8 epochs. - RMSProp opt(0.001, 1, 0.88, 1e-8, 8 * nPoints, -1); - - model.Train(X, Y, opt); - - arma::mat predictionTemp; - model.Predict(X, predictionTemp); - arma::mat prediction = arma::zeros(1, predictionTemp.n_cols); - - for (size_t i = 0; i < predictionTemp.n_cols; ++i) + // It isn't guaranteed that the network will converge in the specified number + // of iterations using random weights. If this works 1 of 5 times, I'm fine + // with that. All I want to know is that the network is able to escape from + // local minima and to solve the task. + bool success = false; + for (size_t trial = 0; trial < 5; ++trial) { - prediction(i) = arma::as_scalar(arma::find( - arma::max(predictionTemp.col(i)) == predictionTemp.col(i), 1)) + 1; - } + FFN, RandomInitialization> model; - size_t correct = 0; - for (size_t i = 0; i < X.n_cols; i++) - { - if (prediction(i) == Y(i)) + model.Add >(1, 8, 5, 5, 1, 1, 0, 0, 28, 28); + model.Add >(); + model.Add >(8, 8, 2, 2); + model.Add >(8, 12, 2, 2); + model.Add >(); + model.Add >(2, 2, 2, 2); + model.Add >(192, 20); + model.Add >(); + model.Add >(20, 10); + model.Add >(); + model.Add >(10, 2); + model.Add >(); + + // Train for only 8 epochs. + RMSProp opt(0.001, 1, 0.88, 1e-8, 8 * nPoints, -1); + + model.Train(X, Y, opt); + + arma::mat predictionTemp; + model.Predict(X, predictionTemp); + arma::mat prediction = arma::zeros(1, predictionTemp.n_cols); + + for (size_t i = 0; i < predictionTemp.n_cols; ++i) { - correct++; + prediction(i) = arma::as_scalar(arma::find( + arma::max(predictionTemp.col(i)) == predictionTemp.col(i), 1)) + 1; + } + + size_t correct = 0; + for (size_t i = 0; i < X.n_cols; i++) + { + if (prediction(i) == Y(i)) + correct++; + } + + double classificationError = 1 - double(correct) / X.n_cols; + if (classificationError <= 0.25) + { + success = true; + break; } } - double classificationError = 1 - double(correct) / X.n_cols; - BOOST_REQUIRE_LE(classificationError, 0.25); + BOOST_REQUIRE_EQUAL(success, true); } BOOST_AUTO_TEST_SUITE_END(); diff --git a/src/mlpack/tests/dcgan_test.cpp b/src/mlpack/tests/dcgan_test.cpp new file mode 100644 index 0000000000..1fb94714f3 --- /dev/null +++ b/src/mlpack/tests/dcgan_test.cpp @@ -0,0 +1,280 @@ +/** + * @file dcgan_network_test.cpp + * @author Shikhar Jaiswal + * + * Tests the DCGAN network. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include "test_tools.hpp" + +using namespace mlpack; +using namespace mlpack::ann; +using namespace mlpack::math; +using namespace mlpack::optimization; +using namespace mlpack::regression; +using namespace std::placeholders; + +BOOST_AUTO_TEST_SUITE(DCGANNetworkTest); + +/* + * Tests the DCGAN implementation on the MNIST dataset. + * It's not viable to train on bigger parameters due to time constraints. + * Please refer mlpack/models repository for the tutorial. + */ +BOOST_AUTO_TEST_CASE(DCGANMNISTTest) +{ + size_t dNumKernels = 32; + size_t discriminatorPreTrain = 5; + size_t batchSize = 5; + size_t noiseDim = 100; + size_t generatorUpdateStep = 1; + size_t numSamples = 10; + double stepSize = 0.0003; + double eps = 1e-8; + size_t numEpoches = 1; + double tolerance = 1e-5; + int datasetMaxCols = 10; + bool shuffle = true; + double multiplier = 10; + + Log::Info << std::boolalpha + << " batchSize = " << batchSize << std::endl + << " generatorUpdateStep = " << generatorUpdateStep << std::endl + << " noiseDim = " << noiseDim << std::endl + << " numSamples = " << numSamples << std::endl + << " stepSize = " << stepSize << std::endl + << " numEpoches = " << numEpoches << std::endl + << " tolerance = " << tolerance << std::endl + << " shuffle = " << shuffle << std::endl; + + arma::mat trainData; + trainData.load("mnist_first250_training_4s_and_9s.arm"); + Log::Info << arma::size(trainData) << std::endl; + + if (datasetMaxCols > 0) + trainData = trainData.cols(0, datasetMaxCols - 1); + + size_t numIterations = trainData.n_cols * numEpoches; + numIterations /= batchSize; + + Log::Info << "Dataset loaded (" << trainData.n_rows << ", " + << trainData.n_cols << ")" << std::endl; + Log::Info << trainData.n_rows << "--------" << trainData.n_cols << std::endl; + + // Create the Discriminator network + FFN > discriminator; + discriminator.Add >(1, dNumKernels, 4, 4, 2, 2, 1, 1, 28, 28); + discriminator.Add >(0.2); + discriminator.Add >(dNumKernels, 2 * dNumKernels, 4, 4, 2, 2, + 1, 1, 14, 14); + discriminator.Add >(0.2); + discriminator.Add >(2 * dNumKernels, 4 * dNumKernels, 4, 4, + 2, 2, 1, 1, 7, 7); + discriminator.Add >(0.2); + discriminator.Add >(4 * dNumKernels, 8 * dNumKernels, 4, 4, + 2, 2, 2, 2, 3, 3); + discriminator.Add >(0.2); + discriminator.Add >(8 * dNumKernels, 1, 4, 4, 1, 1, + 1, 1, 2, 2); + discriminator.Add >(); + + // Create the Generator network + FFN > generator; + generator.Add >(noiseDim, 8 * dNumKernels, 2, 2, + 1, 1, 1, 1, 1, 1); + generator.Add >(); + generator.Add >(8 * dNumKernels, 4 * dNumKernels, + 2, 2, 1, 1, 0, 0, 2, 2); + generator.Add >(); + generator.Add >(4 * dNumKernels, 2 * dNumKernels, + 5, 5, 2, 2, 1, 1, 3, 3); + generator.Add >(); + generator.Add >(2 * dNumKernels, dNumKernels, 8, 8, + 1, 1, 1, 1, 7, 7); + generator.Add >(); + generator.Add >(dNumKernels, 1, 15, 15, 1, 1, 1, 1, + 14, 14); + generator.Add >(); + + // Create GAN + GaussianInitialization gaussian(0, 1); + Adam optimizer(stepSize, batchSize, 0.9, 0.999, eps, numIterations, + tolerance, shuffle); + std::function noiseFunction = [] () { + return math::RandNormal(0, 1);}; + GAN >, GaussianInitialization, + std::function > gan(trainData, generator, discriminator, + gaussian, noiseFunction, noiseDim, batchSize, generatorUpdateStep, + discriminatorPreTrain, multiplier); + + Log::Info << "Training..." << std::endl; + gan.Train(optimizer); + + // Generate samples + Log::Info << "Sampling..." << std::endl; + arma::mat noise(noiseDim, 1); + size_t dim = std::sqrt(trainData.n_rows); + arma::mat generatedData(2 * dim, dim * numSamples); + + for (size_t i = 0; i < numSamples; i++) + { + arma::mat samples; + noise.imbue( [&]() { return noiseFunction(); } ); + + generator.Forward(noise, samples); + samples.reshape(dim, dim); + samples = samples.t(); + + generatedData.submat(0, i * dim, dim - 1, i * dim + dim - 1) = samples; + + samples = trainData.col(math::RandInt(0, trainData.n_cols)); + samples.reshape(dim, dim); + samples = samples.t(); + + generatedData.submat(dim, + i * dim, 2 * dim - 1, i * dim + dim - 1) = samples; + } + + Log::Info << "Output generated!" << std::endl; +} + +/* + * Tests the DCGAN implementation on the CelebA dataset. + * It's currently not possible to run this every time due to time constraints. + * Please refer mlpack/models repository for the tutorial. + +BOOST_AUTO_TEST_CASE(DCGANCelebATest) +{ + size_t dNumKernels = 64; + size_t discriminatorPreTrain = 300; + size_t batchSize = 1; + size_t noiseDim = 100; + size_t generatorUpdateStep = 1; + size_t numSamples = 10; + double stepSize = 0.0003; + double eps = 1e-8; + size_t numEpoches = 20; + double tolerance = 1e-5; + int datasetMaxCols = -1; + bool shuffle = true; + double multiplier = 10; + + Log::Info << std::boolalpha + << " batchSize = " << batchSize << std::endl + << " generatorUpdateStep = " << generatorUpdateStep << std::endl + << " noiseDim = " << noiseDim << std::endl + << " numSamples = " << numSamples << std::endl + << " stepSize = " << stepSize << std::endl + << " numEpoches = " << numEpoches << std::endl + << " tolerance = " << tolerance << std::endl + << " shuffle = " << shuffle << std::endl; + + arma::mat trainData; + trainData.load("celeba.csv"); + Log::Info << arma::size(trainData) << std::endl; + + if (datasetMaxCols > 0) + trainData = trainData.cols(0, datasetMaxCols - 1); + + size_t numIterations = trainData.n_cols * numEpoches; + numIterations /= batchSize; + + Log::Info << "Dataset loaded (" << trainData.n_rows << ", " + << trainData.n_cols << ")" << std::endl; + Log::Info << trainData.n_rows << "--------" << trainData.n_cols << std::endl; + + // Create the Discriminator network + FFN > discriminator; + discriminator.Add >(3, dNumKernels, 4, 4, 2, 2, 1, 1, 64, 64); + discriminator.Add >(0.2); + discriminator.Add >(dNumKernels, 2 * dNumKernels, 4, 4, 2, 2, + 1, 1, 32, 32); + discriminator.Add >(0.2); + discriminator.Add >(2 * dNumKernels, 4 * dNumKernels, 4, 4, + 2, 2, 1, 1, 16, 16); + discriminator.Add >(0.2); + discriminator.Add >(4 * dNumKernels, 8 * dNumKernels, 4, 4, + 2, 2, 1, 1, 8, 8); + discriminator.Add >(0.2); + discriminator.Add >(8 * dNumKernels, 1, 4, 4, 1, 1, + 0, 0, 4, 4); + discriminator.Add >(); + + // Create the Generator network + FFN > generator; + generator.Add >(noiseDim, 8 * dNumKernels, 4, 4, + 1, 1, 2, 2, 1, 1); + generator.Add >(); + generator.Add >(8 * dNumKernels, 4 * dNumKernels, + 5, 5, 1, 1, 1, 1, 4, 4); + generator.Add >(); + generator.Add >(4 * dNumKernels, 2 * dNumKernels, + 9, 9, 1, 1, 1, 1, 8, 8); + generator.Add >(); + generator.Add >(2 * dNumKernels, dNumKernels, 17, 17, + 1, 1, 1, 1, 16, 16); + generator.Add >(); + generator.Add >(dNumKernels, 3, 33, 33, 1, 1, 1, 1, + 32, 32); + generator.Add >(); + + // Create GAN + GaussianInitialization gaussian(0, 1); + Adam optimizer(stepSize, batchSize, 0.9, 0.999, eps, numIterations, + tolerance, shuffle); + std::function noiseFunction = [] () { + return math::RandNormal(0, 1);}; + GAN >, GaussianInitialization, + std::function > gan(trainData, generator, discriminator, + gaussian, noiseFunction, noiseDim, batchSize, generatorUpdateStep, + discriminatorPreTrain, multiplier); + + Log::Info << "Training..." << std::endl; + gan.Train(optimizer); + + // Generate samples + Log::Info << "Sampling..." << std::endl; + arma::mat noise(noiseDim, 1); + size_t dim = std::sqrt(trainData.n_rows); + arma::mat generatedData(2 * dim, dim * numSamples); + + for (size_t i = 0; i < numSamples; i++) + { + arma::mat samples; + noise.imbue( [&]() { return noiseFunction(); } ); + + generator.Forward(noise, samples); + samples.reshape(dim, dim); + samples = samples.t(); + + generatedData.submat(0, i * dim, dim - 1, i * dim + dim - 1) = samples; + + samples = trainData.col(math::RandInt(0, trainData.n_cols)); + samples.reshape(dim, dim); + samples = samples.t(); + + generatedData.submat(dim, + i * dim, 2 * dim - 1, i * dim + dim - 1) = samples; + } + + Log::Info << "Output generated!" << std::endl; +} +*/ + +BOOST_AUTO_TEST_SUITE_END(); diff --git a/src/mlpack/tests/gan_test.cpp b/src/mlpack/tests/gan_test.cpp index fcc9710757..a6d47c6f9e 100644 --- a/src/mlpack/tests/gan_test.cpp +++ b/src/mlpack/tests/gan_test.cpp @@ -132,28 +132,26 @@ BOOST_AUTO_TEST_CASE(GANTest) } /* - * Tests the GAN implementation on the O'Reilly Test on the MNIST dataset. - * It's currently not possible to run this every time due to time constraints. + * Tests the GAN implementation of the O'Reilly Test on the MNIST dataset. + * It's not viable to train on bigger parameters due to time constraints. * Please refer mlpack/models repository for the tutorial. - + */ BOOST_AUTO_TEST_CASE(GANMNISTTest) { size_t dNumKernels = 32; - size_t discriminatorPreTrain = 300; - size_t batchSize = 1; + size_t discriminatorPreTrain = 5; + size_t batchSize = 5; size_t noiseDim = 100; size_t generatorUpdateStep = 1; size_t numSamples = 10; double stepSize = 0.0003; double eps = 1e-8; - size_t numEpoches = 10; + size_t numEpoches = 1; double tolerance = 1e-5; - int datasetMaxCols = -1; + int datasetMaxCols = 10; bool shuffle = true; double multiplier = 10; - std::string output_dataset = "output_mnist.csv"; - Log::Info << "output_dataset = '" << output_dataset << "'" << std::endl; Log::Info << std::boolalpha << " batchSize = " << batchSize << std::endl << " generatorUpdateStep = " << generatorUpdateStep << std::endl @@ -221,7 +219,7 @@ BOOST_AUTO_TEST_CASE(GANMNISTTest) // Generate samples Log::Info << "Sampling..." << std::endl; - arma::mat noise(noiseDim, 1); + arma::mat noise(noiseDim, batchSize); size_t dim = std::sqrt(trainData.n_rows); arma::mat generatedData(2 * dim, dim * numSamples); @@ -244,10 +242,7 @@ BOOST_AUTO_TEST_CASE(GANMNISTTest) i * dim, 2 * dim - 1, i * dim + dim - 1) = samples; } - Log::Info << "Saving output to " << output_dataset << "..." << std::endl; - generatedData.save(output_dataset, arma::csv_ascii); - Log::Info << "Output saved!" << std::endl; + Log::Info << "Output generated!" << std::endl; } -*/ BOOST_AUTO_TEST_SUITE_END(); diff --git a/src/mlpack/tests/main_tests/cf_test.cpp b/src/mlpack/tests/main_tests/cf_test.cpp index 3e37776421..55203fe6ba 100644 --- a/src/mlpack/tests/main_tests/cf_test.cpp +++ b/src/mlpack/tests/main_tests/cf_test.cpp @@ -215,7 +215,7 @@ BOOST_AUTO_TEST_CASE(CFModelReuseTest) SetInputParam("query", std::move(query)); SetInputParam("recommendations", recommendations); SetInputParam("input_model", - std::move(CLI::GetParam("output_model"))); + std::move(CLI::GetParam*>("output_model"))); mlpackMain(); @@ -262,7 +262,7 @@ BOOST_AUTO_TEST_CASE(CFRankTest) mlpackMain(); - const CFType* outputModel = CLI::GetParam("output_model"); + const CFType<>* outputModel = CLI::GetParam*>("output_model"); BOOST_REQUIRE_EQUAL(outputModel->Rank(), rank); } @@ -274,7 +274,7 @@ BOOST_AUTO_TEST_CASE(CFMinResidueTest) { mat dataset; data::Load("GroupLensSmall.csv", dataset); - const CFType* outputModel; + const CFType<>* outputModel; // Set a larger min_residue. SetInputParam("min_residue", double(100)); @@ -286,7 +286,7 @@ BOOST_AUTO_TEST_CASE(CFMinResidueTest) mlpack::math::FixedRandomSeed(); mlpackMain(); - outputModel = CLI::GetParam("output_model"); + outputModel = CLI::GetParam*>("output_model"); const mat w1 = outputModel->W(); const mat h1 = outputModel->H(); @@ -302,7 +302,7 @@ BOOST_AUTO_TEST_CASE(CFMinResidueTest) mlpack::math::FixedRandomSeed(); mlpackMain(); - outputModel = CLI::GetParam("output_model"); + outputModel = CLI::GetParam*>("output_model"); const mat w2 = outputModel->W(); const mat h2 = outputModel->H(); @@ -317,7 +317,7 @@ BOOST_AUTO_TEST_CASE(CFIterationOnlyTerminationTest) { mat dataset; data::Load("GroupLensSmall.csv", dataset); - const CFType* outputModel; + const CFType<>* outputModel; // Set iteration_only_termination. SetInputParam("iteration_only_termination", true); @@ -329,7 +329,7 @@ BOOST_AUTO_TEST_CASE(CFIterationOnlyTerminationTest) mlpack::math::FixedRandomSeed(); mlpackMain(); - outputModel = CLI::GetParam("output_model"); + outputModel = CLI::GetParam*>("output_model"); const mat w1 = outputModel->W(); const mat h1 = outputModel->H(); @@ -344,7 +344,7 @@ BOOST_AUTO_TEST_CASE(CFIterationOnlyTerminationTest) mlpack::math::FixedRandomSeed(); mlpackMain(); - outputModel = CLI::GetParam("output_model"); + outputModel = CLI::GetParam*>("output_model"); const mat w2 = outputModel->W(); const mat h2 = outputModel->H(); @@ -359,7 +359,7 @@ BOOST_AUTO_TEST_CASE(CFMaxIterationsTest) { mat dataset; data::Load("GroupLensSmall.csv", dataset); - const CFType* outputModel; + const CFType<>* outputModel; // Set a larger max_iterations. SetInputParam("max_iterations", int(100)); @@ -370,7 +370,7 @@ BOOST_AUTO_TEST_CASE(CFMaxIterationsTest) mlpack::math::FixedRandomSeed(); mlpackMain(); - outputModel = CLI::GetParam("output_model"); + outputModel = CLI::GetParam*>("output_model"); const mat w1 = outputModel->W(); const mat h1 = outputModel->H(); @@ -385,7 +385,7 @@ BOOST_AUTO_TEST_CASE(CFMaxIterationsTest) mlpack::math::FixedRandomSeed(); mlpackMain(); - outputModel = CLI::GetParam("output_model"); + outputModel = CLI::GetParam*>("output_model"); const mat w2 = outputModel->W(); const mat h2 = outputModel->H();