From b2634d50a608eb206d0bcd0feeaa4beb691b7ac6 Mon Sep 17 00:00:00 2001 From: akhandait Date: Fri, 22 Jun 2018 01:02:08 +0530 Subject: [PATCH 1/7] Add reconstruction loss with test. --- .../methods/ann/loss_functions/CMakeLists.txt | 2 + .../loss_functions/reconstruction_loss.hpp | 89 +++++++++ .../reconstruction_loss_impl.hpp | 62 +++++++ src/mlpack/tests/CMakeLists.txt | 1 + src/mlpack/tests/ann_layer_test.cpp | 153 +--------------- src/mlpack/tests/ann_test_tools.hpp | 170 ++++++++++++++++++ src/mlpack/tests/loss_functions_test.cpp | 61 ++++++- 7 files changed, 381 insertions(+), 157 deletions(-) create mode 100644 src/mlpack/methods/ann/loss_functions/reconstruction_loss.hpp create mode 100644 src/mlpack/methods/ann/loss_functions/reconstruction_loss_impl.hpp create mode 100644 src/mlpack/tests/ann_test_tools.hpp diff --git a/src/mlpack/methods/ann/loss_functions/CMakeLists.txt b/src/mlpack/methods/ann/loss_functions/CMakeLists.txt index cce48ed7cb..c332a075b4 100644 --- a/src/mlpack/methods/ann/loss_functions/CMakeLists.txt +++ b/src/mlpack/methods/ann/loss_functions/CMakeLists.txt @@ -11,6 +11,8 @@ set(SOURCES mean_squared_error_impl.hpp negative_log_likelihood.hpp negative_log_likelihood_impl.hpp + reconstruction_loss.hpp + reconstruction_loss_impl.hpp sigmoid_cross_entropy_error.hpp sigmoid_cross_entropy_error_impl.hpp ) diff --git a/src/mlpack/methods/ann/loss_functions/reconstruction_loss.hpp b/src/mlpack/methods/ann/loss_functions/reconstruction_loss.hpp new file mode 100644 index 0000000000..20d79487ac --- /dev/null +++ b/src/mlpack/methods/ann/loss_functions/reconstruction_loss.hpp @@ -0,0 +1,89 @@ +/** + * @file reconstruction_loss.hpp + * @author Atharva Khandait + * + * Definition of the reconstruction loss performance function. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_METHODS_ANN_LOSS_FUNCTION_RECONSTRUCTION_LOSS_HPP +#define MLPACK_METHODS_ANN_LOSS_FUNCTION_RECONSTRUCTION_LOSS_HPP + +#include +#include + +namespace mlpack { +namespace ann /** Artificial Neural Network. */ { + +/** + * The mean squared error performance function measures the network's + * performance according to the mean of squared errors. + * + * @tparam ActivationFunction Activation function used for the embedding layer. + * @tparam InputDataType Type of the input data (arma::colvec, arma::mat, + * arma::sp_mat or arma::cube). + * @tparam OutputDataType Type of the output data (arma::colvec, arma::mat, + * arma::sp_mat or arma::cube). + */ +template < + typename InputDataType = arma::mat, + typename OutputDataType = arma::mat, + typename DistType = NormalDistribution +> +class ReconstructionLoss +{ + public: + /** + * Create the ReconstructionLoss object. + */ + ReconstructionLoss(); + + /* + * Computes the mean squared error function. + * + * @param input Input data used for evaluating the specified function. + * @param output Resulting output activation. + */ + template + double Forward(const InputType&& input, const TargetType&& target); + /** + * Ordinary feed backward pass of a neural network. + * + * @param input The propagated input activation. + * @param target The target vector. + * @param output The calculated error. + */ + template + void Backward(const InputType&& input, + const TargetType&& target, + OutputType&& output); + + //! Get the output parameter. + OutputDataType& OutputParameter() const { return outputParameter; } + //! Modify the output parameter. + OutputDataType& OutputParameter() { return outputParameter; } + + /** + * Serialize the layer + */ + template + void serialize(Archive& ar, const unsigned int /* version */); + + private: + //! Locally-stored distribution object. + DistType* dist; + + //! Locally-stored output parameter object. + OutputDataType outputParameter; +}; // class ReconstructionLoss + +} // namespace ann +} // namespace mlpack + +// Include implementation. +#include "reconstruction_loss_impl.hpp" + +#endif diff --git a/src/mlpack/methods/ann/loss_functions/reconstruction_loss_impl.hpp b/src/mlpack/methods/ann/loss_functions/reconstruction_loss_impl.hpp new file mode 100644 index 0000000000..aeb72df3e4 --- /dev/null +++ b/src/mlpack/methods/ann/loss_functions/reconstruction_loss_impl.hpp @@ -0,0 +1,62 @@ +/** + * @file reconstruction_loss_impl.hpp + * @author Atharva Khandait + * + * Implementation of the reconstruction loss performance function. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_METHODS_ANN_LOSS_FUNCTION_RECONSTRUCTION_LOSS_IMPL_HPP +#define MLPACK_METHODS_ANN_LOSS_FUNCTION_RECONSTRUCTION_LOSS_IMPL_HPP + +// In case it hasn't yet been included. +#include "reconstruction_loss.hpp" + +namespace mlpack { +namespace ann /** Artificial Neural Network. */ { + +template +ReconstructionLoss< + InputDataType, + OutputDataType, + DistType +>::ReconstructionLoss() +{ + // Nothing to do here. +} + +template +template +double ReconstructionLoss::Forward( + const InputType&& input, const TargetType&& target) +{ + dist = new DistType(std::move(input)); + return dist->LogProbability(std::move(target)); +} + +template +template +void ReconstructionLoss::Backward( + const InputType&& /* input */, + const TargetType&& target, + OutputType&& output) +{ + dist->LogProbBackward(std::move(target), std::move(output)); +} + +template +template +void ReconstructionLoss::serialize( + Archive& /* ar */, + const unsigned int /* version */) +{ + // Nothing to do here. +} + +} // namespace ann +} // namespace mlpack + +#endif diff --git a/src/mlpack/tests/CMakeLists.txt b/src/mlpack/tests/CMakeLists.txt index f77ca622c2..e6a875aafd 100644 --- a/src/mlpack/tests/CMakeLists.txt +++ b/src/mlpack/tests/CMakeLists.txt @@ -9,6 +9,7 @@ add_executable(mlpack_test aknn_test.cpp ann_dist_test.cpp ann_layer_test.cpp + ann_test_tools.hpp arma_extend_test.cpp armadillo_svd_test.cpp async_learning_test.cpp diff --git a/src/mlpack/tests/ann_layer_test.cpp b/src/mlpack/tests/ann_layer_test.cpp index 3e34c9d475..440223b40b 100644 --- a/src/mlpack/tests/ann_layer_test.cpp +++ b/src/mlpack/tests/ann_layer_test.cpp @@ -14,8 +14,6 @@ #include #include -#include -#include #include #include #include @@ -24,162 +22,13 @@ #include #include "test_tools.hpp" +#include "ann_test_tools.hpp" using namespace mlpack; using namespace mlpack::ann; BOOST_AUTO_TEST_SUITE(ANNLayerTest); -// Helper function which calls the Reset function of the given module. -template -void ResetFunction( - T& layer, - typename std::enable_if::value>::type* = 0) -{ - layer.Reset(); -} - -template -void ResetFunction( - T& /* layer */, - typename std::enable_if::value>::type* = 0) -{ - /* Nothing to do here */ -} - -// Approximate Jacobian and supposedly-true Jacobian, then compare them -// similarly to before. -template -double JacobianTest(ModuleType& module, - arma::mat& input, - const double minValue = -2, - const double maxValue = -1, - const double perturbation = 1e-6) -{ - arma::mat output, outputA, outputB, jacobianA, jacobianB; - - // Initialize the input matrix. - RandomInitialization init(minValue, maxValue); - init.Initialize(input, input.n_rows, input.n_cols); - - // Initialize the module parameters. - ResetFunction(module); - - // Initialize the jacobian matrix. - module.Forward(std::move(input), std::move(output)); - jacobianA = arma::zeros(input.n_elem, output.n_elem); - - // Share the input paramter matrix. - arma::mat sin = arma::mat(input.memptr(), input.n_rows, input.n_cols, - false, false); - - for (size_t i = 0; i < input.n_elem; ++i) - { - double original = sin(i); - sin(i) = original - perturbation; - module.Forward(std::move(input), std::move(outputA)); - sin(i) = original + perturbation; - module.Forward(std::move(input), std::move(outputB)); - sin(i) = original; - - outputB -= outputA; - outputB /= 2 * perturbation; - jacobianA.row(i) = outputB.t(); - } - - // Initialize the derivative parameter. - arma::mat deriv = arma::zeros(output.n_rows, output.n_cols); - - // Share the derivative parameter. - arma::mat derivTemp = arma::mat(deriv.memptr(), deriv.n_rows, deriv.n_cols, - false, false); - - // Initialize the jacobian matrix. - jacobianB = arma::zeros(input.n_elem, output.n_elem); - - for (size_t i = 0; i < derivTemp.n_elem; ++i) - { - deriv.zeros(); - derivTemp(i) = 1; - - arma::mat delta; - module.Backward(std::move(input), std::move(deriv), std::move(delta)); - - jacobianB.col(i) = delta; - } - - return arma::max(arma::max(arma::abs(jacobianA - jacobianB))); -} - -// Approximate Jacobian and supposedly-true Jacobian, then compare them -// similarly to before. -template -double JacobianPerformanceTest(ModuleType& module, - arma::mat& input, - arma::mat& target, - const double eps = 1e-6) -{ - module.Forward(std::move(input), std::move(target)); - - arma::mat delta; - module.Backward(std::move(input), std::move(target), std::move(delta)); - - arma::mat centralDifference = arma::zeros(delta.n_rows, delta.n_cols); - arma::mat inputTemp = arma::mat(input.memptr(), input.n_rows, input.n_cols, - false, false); - - arma::mat centralDifferenceTemp = arma::mat(centralDifference.memptr(), - centralDifference.n_rows, centralDifference.n_cols, false, false); - - for (size_t i = 0; i < input.n_elem; ++i) - { - inputTemp(i) = inputTemp(i) + eps; - double outputA = module.Forward(std::move(input), std::move(target)); - inputTemp(i) = inputTemp(i) - (2 * eps); - double outputB = module.Forward(std::move(input), std::move(target)); - - centralDifferenceTemp(i) = (outputA - outputB) / (2 * eps); - inputTemp(i) = inputTemp(i) + eps; - } - - return arma::max(arma::max(arma::abs(centralDifference - delta))); -} - -// Simple numerical gradient checker. -template -double CheckGradient(FunctionType& function, const double eps = 1e-7) -{ - // Get gradients for the current parameters. - arma::mat orgGradient, gradient, estGradient; - function.Gradient(orgGradient); - - estGradient = arma::zeros(orgGradient.n_rows, orgGradient.n_cols); - - // Compute numeric approximations to gradient. - for (size_t i = 0; i < orgGradient.n_elem; ++i) - { - double tmp = function.Parameters()(i); - - // Perturb parameter with a positive constant and get costs. - function.Parameters()(i) += eps; - double costPlus = function.Gradient(gradient); - - // Perturb parameter with a negative constant and get costs. - function.Parameters()(i) -= (2 * eps); - double costMinus = function.Gradient(gradient); - - // Restore the parameter value. - function.Parameters()(i) = tmp; - - // Compute numerical gradients using the costs calculated above. - estGradient(i) = (costPlus - costMinus) / (2 * eps); - } - - // Estimate error of gradient. - return arma::norm(orgGradient - estGradient) / - arma::norm(orgGradient + estGradient); -} - /** * Simple add module test. */ diff --git a/src/mlpack/tests/ann_test_tools.hpp b/src/mlpack/tests/ann_test_tools.hpp new file mode 100644 index 0000000000..67f7108cd0 --- /dev/null +++ b/src/mlpack/tests/ann_test_tools.hpp @@ -0,0 +1,170 @@ +/** + * @file ann_test_tools.hpp + * @author Marcus Edel + * + * This file includes some useful functions for ann tests. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_TESTS_ANN_TEST_TOOLS_HPP +#define MLPACK_TESTS_ANN_TEST_TOOLS_HPP + +#include + +using namespace mlpack; +using namespace mlpack::ann; + +// Helper function which calls the Reset function of the given module. +template +void ResetFunction( + T& layer, + typename std::enable_if::value>::type* = 0) +{ + layer.Reset(); +} + +template +void ResetFunction( + T& /* layer */, + typename std::enable_if::value>::type* = 0) +{ + /* Nothing to do here */ +} + +// Approximate Jacobian and supposedly-true Jacobian, then compare them +// similarly to before. +template +double JacobianTest(ModuleType& module, + arma::mat& input, + const double minValue = -2, + const double maxValue = -1, + const double perturbation = 1e-6) +{ + arma::mat output, outputA, outputB, jacobianA, jacobianB; + + // Initialize the input matrix. + RandomInitialization init(minValue, maxValue); + init.Initialize(input, input.n_rows, input.n_cols); + + // Initialize the module parameters. + ResetFunction(module); + + // Initialize the jacobian matrix. + module.Forward(std::move(input), std::move(output)); + jacobianA = arma::zeros(input.n_elem, output.n_elem); + + // Share the input paramter matrix. + arma::mat sin = arma::mat(input.memptr(), input.n_rows, input.n_cols, + false, false); + + for (size_t i = 0; i < input.n_elem; ++i) + { + double original = sin(i); + sin(i) = original - perturbation; + module.Forward(std::move(input), std::move(outputA)); + sin(i) = original + perturbation; + module.Forward(std::move(input), std::move(outputB)); + sin(i) = original; + + outputB -= outputA; + outputB /= 2 * perturbation; + jacobianA.row(i) = outputB.t(); + } + + // Initialize the derivative parameter. + arma::mat deriv = arma::zeros(output.n_rows, output.n_cols); + + // Share the derivative parameter. + arma::mat derivTemp = arma::mat(deriv.memptr(), deriv.n_rows, deriv.n_cols, + false, false); + + // Initialize the jacobian matrix. + jacobianB = arma::zeros(input.n_elem, output.n_elem); + + for (size_t i = 0; i < derivTemp.n_elem; ++i) + { + deriv.zeros(); + derivTemp(i) = 1; + + arma::mat delta; + module.Backward(std::move(input), std::move(deriv), std::move(delta)); + + jacobianB.col(i) = delta; + } + + return arma::max(arma::max(arma::abs(jacobianA - jacobianB))); +} + +// Approximate Jacobian and supposedly-true Jacobian, then compare them +// similarly to before. +template +double JacobianPerformanceTest(ModuleType& module, + arma::mat& input, + arma::mat& target, + const double eps = 1e-6) +{ + module.Forward(std::move(input), std::move(target)); + + arma::mat delta; + module.Backward(std::move(input), std::move(target), std::move(delta)); + + arma::mat centralDifference = arma::zeros(delta.n_rows, delta.n_cols); + arma::mat inputTemp = arma::mat(input.memptr(), input.n_rows, input.n_cols, + false, false); + + arma::mat centralDifferenceTemp = arma::mat(centralDifference.memptr(), + centralDifference.n_rows, centralDifference.n_cols, false, false); + + for (size_t i = 0; i < input.n_elem; ++i) + { + inputTemp(i) = inputTemp(i) + eps; + double outputA = module.Forward(std::move(input), std::move(target)); + inputTemp(i) = inputTemp(i) - (2 * eps); + double outputB = module.Forward(std::move(input), std::move(target)); + + centralDifferenceTemp(i) = (outputA - outputB) / (2 * eps); + inputTemp(i) = inputTemp(i) + eps; + } + + return arma::max(arma::max(arma::abs(centralDifference - delta))); +} + +// Simple numerical gradient checker. +template +double CheckGradient(FunctionType& function, const double eps = 1e-7) +{ + // Get gradients for the current parameters. + arma::mat orgGradient, gradient, estGradient; + function.Gradient(orgGradient); + + estGradient = arma::zeros(orgGradient.n_rows, orgGradient.n_cols); + + // Compute numeric approximations to gradient. + for (size_t i = 0; i < orgGradient.n_elem; ++i) + { + double tmp = function.Parameters()(i); + + // Perturb parameter with a positive constant and get costs. + function.Parameters()(i) += eps; + double costPlus = function.Gradient(gradient); + + // Perturb parameter with a negative constant and get costs. + function.Parameters()(i) -= (2 * eps); + double costMinus = function.Gradient(gradient); + + // Restore the parameter value. + function.Parameters()(i) = tmp; + + // Compute numerical gradients using the costs calculated above. + estGradient(i) = (costPlus - costMinus) / (2 * eps); + } + + // Estimate error of gradient. + return arma::norm(orgGradient - estGradient) / + arma::norm(orgGradient + estGradient); +} + +#endif diff --git a/src/mlpack/tests/loss_functions_test.cpp b/src/mlpack/tests/loss_functions_test.cpp index ae46073786..5f7abbc61f 100644 --- a/src/mlpack/tests/loss_functions_test.cpp +++ b/src/mlpack/tests/loss_functions_test.cpp @@ -2,6 +2,7 @@ * @file loss_functions_test.cpp * @author Dakshit Agrawal * @author Sourabh Varshney + * @author Atharva Khandait * * Tests for loss functions in mlpack::methods::ann:loss_functions. * @@ -10,14 +11,21 @@ * 3-clause BSD license along with mlpack. If not, see * http://www.opensource.org/licenses/BSD-3-Clause for more information. */ -#include +#include + +#include #include +#include #include #include #include +#include +#include +#include #include #include "test_tools.hpp" +#include "ann_test_tools.hpp" using namespace mlpack; using namespace mlpack::ann; @@ -85,7 +93,7 @@ BOOST_AUTO_TEST_CASE(KLDivergenceNoMeanTest) /* * Simple test for the mean squared error performance function. */ -BOOST_AUTO_TEST_CASE(SimpleMeanSquaredErrorLayerTest) +BOOST_AUTO_TEST_CASE(SimpleMeanSquaredErrorTest) { arma::mat input, output, target; MeanSquaredError<> module; @@ -120,7 +128,7 @@ BOOST_AUTO_TEST_CASE(SimpleMeanSquaredErrorLayerTest) /* * Simple test for the cross-entropy error performance function. */ -BOOST_AUTO_TEST_CASE(SimpleCrossEntropyErrorLayerTest) +BOOST_AUTO_TEST_CASE(SimpleCrossEntropyErrorTest) { arma::mat input1, input2, output, target1, target2; CrossEntropyError<> module(1e-6); @@ -161,9 +169,9 @@ BOOST_AUTO_TEST_CASE(SimpleCrossEntropyErrorLayerTest) } /** - * Simple test for the Sigmoid Cross Entropy Layer. + * Simple test for the Sigmoid Cross Entropy performance function. */ -BOOST_AUTO_TEST_CASE(SimpleSigmoidCrossEntropyLayerTest) +BOOST_AUTO_TEST_CASE(SimpleSigmoidCrossEntropyErrorTest) { arma::mat input1, input2, input3, output, target1, target2, target3, expectedOutput; @@ -221,6 +229,7 @@ BOOST_AUTO_TEST_CASE(SimpleSigmoidCrossEntropyLayerTest) } /** +<<<<<<< 9cecc3e18160a4411c1e80831b6773bdbd80b789 * Simple test for the Earth Mover Distance Layer. */ BOOST_AUTO_TEST_CASE(SimpleEarthMoverDistanceLayerTest) @@ -256,6 +265,48 @@ BOOST_AUTO_TEST_CASE(SimpleEarthMoverDistanceLayerTest) BOOST_REQUIRE_SMALL(output(i) - expectedOutput(i), 1e-5); BOOST_REQUIRE_EQUAL(output.n_rows, input2.n_rows); BOOST_REQUIRE_EQUAL(output.n_cols, input2.n_cols); +======= + * Reconstruction Loss numerical gradient test. + */ +BOOST_AUTO_TEST_CASE(GradientReconstructionLossTest) +{ + // Linear function gradient instantiation. + struct GradientFunction + { + GradientFunction() + { + input = arma::randu(10, 1); + target = arma::randu(2, 1); + + model = new FFN, NguyenWidrowInitialization>(); + model->Predictors() = input; + model->Responses() = target; + model->Add >(); + model->Add >(10, 4); + model->Add >(); + } + + ~GradientFunction() + { + delete model; + } + + double Gradient(arma::mat& gradient) const + { + arma::mat output; + double error = model->Evaluate(model->Parameters(), 0, 1); + model->Gradient(model->Parameters(), 0, gradient, 1); + return error; + } + + arma::mat& Parameters() { return model->Parameters(); } + + FFN, NguyenWidrowInitialization>* model; + arma::mat input, target; + } function; + + BOOST_REQUIRE_LE(CheckGradient(function), 1e-4); +>>>>>>> Add reconstruction loss with test. } BOOST_AUTO_TEST_SUITE_END(); From f4180deb9d722cdf83649eb8a50dc29dc5d76a35 Mon Sep 17 00:00:00 2001 From: akhandait Date: Sat, 7 Jul 2018 15:39:42 +0530 Subject: [PATCH 2/7] Remove pointer. --- .../methods/ann/loss_functions/reconstruction_loss.hpp | 2 +- .../ann/loss_functions/reconstruction_loss_impl.hpp | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/mlpack/methods/ann/loss_functions/reconstruction_loss.hpp b/src/mlpack/methods/ann/loss_functions/reconstruction_loss.hpp index 20d79487ac..4f46782b59 100644 --- a/src/mlpack/methods/ann/loss_functions/reconstruction_loss.hpp +++ b/src/mlpack/methods/ann/loss_functions/reconstruction_loss.hpp @@ -74,7 +74,7 @@ class ReconstructionLoss private: //! Locally-stored distribution object. - DistType* dist; + DistType dist; //! Locally-stored output parameter object. OutputDataType outputParameter; diff --git a/src/mlpack/methods/ann/loss_functions/reconstruction_loss_impl.hpp b/src/mlpack/methods/ann/loss_functions/reconstruction_loss_impl.hpp index aeb72df3e4..dc24ec30f5 100644 --- a/src/mlpack/methods/ann/loss_functions/reconstruction_loss_impl.hpp +++ b/src/mlpack/methods/ann/loss_functions/reconstruction_loss_impl.hpp @@ -33,8 +33,8 @@ template double ReconstructionLoss::Forward( const InputType&& input, const TargetType&& target) { - dist = new DistType(std::move(input)); - return dist->LogProbability(std::move(target)); + dist = DistType(std::move(input)); + return -dist.LogProbability(std::move(target)); } template @@ -44,7 +44,8 @@ void ReconstructionLoss::Backward( const TargetType&& target, OutputType&& output) { - dist->LogProbBackward(std::move(target), std::move(output)); + dist.LogProbBackward(std::move(target), std::move(output)); + output = -output; } template From f599f7cf8b012f7edee6e498cf2e6b6ba57ffa83 Mon Sep 17 00:00:00 2001 From: akhandait Date: Sat, 7 Jul 2018 16:19:02 +0530 Subject: [PATCH 3/7] Overload Evaluate() function. --- src/mlpack/methods/ann/ffn.hpp | 9 +++++++++ src/mlpack/methods/ann/ffn_impl.hpp | 27 +++++++++++++++++++++++++++ 2 files changed, 36 insertions(+) diff --git a/src/mlpack/methods/ann/ffn.hpp b/src/mlpack/methods/ann/ffn.hpp index 7c826ae65f..8c9a8356e4 100644 --- a/src/mlpack/methods/ann/ffn.hpp +++ b/src/mlpack/methods/ann/ffn.hpp @@ -134,6 +134,15 @@ class FFN */ void Predict(arma::mat predictors, arma::mat& results); + /** + * Evaluate the feedforward network with the given ppredictors and responses. + * This functions is usually used to monitor progress while training. + * + * @param predictors Input variables. + * @param responses Target outputs for input variables. + */ + double Evaluate(arma::mat predictors, arma::mat responses); + /** * Evaluate the feedforward network with the given parameters. This function * is usually called by the optimizer to train the model. diff --git a/src/mlpack/methods/ann/ffn_impl.hpp b/src/mlpack/methods/ann/ffn_impl.hpp index b180b9b0ec..03cfbae1c1 100644 --- a/src/mlpack/methods/ann/ffn_impl.hpp +++ b/src/mlpack/methods/ann/ffn_impl.hpp @@ -202,6 +202,33 @@ void FFN::Predict( results.col(i) = resultsTemp.col(0); } } +template +double FFN::Evaluate( + arma::mat predictors, arma::mat responses) +{ + if (parameter.is_empty()) + ResetParameters(); + + if (!deterministic) + { + deterministic = true; + ResetDeterministic(); + } + + Forward(std::move(predictors)); + + double res = outputLayer.Forward( + std::move(boost::apply_visitor(outputParameterVisitor, network.back())), + std::move(responses)); + + for (size_t i = 0; i < network.size(); ++i) + { + res += boost::apply_visitor(lossVisitor, network[i]); + } + + return res; +} template From 53f108db54a4a4f2a51e053d3f9baff801d6f40a Mon Sep 17 00:00:00 2001 From: akhandait Date: Sat, 7 Jul 2018 16:52:00 +0530 Subject: [PATCH 4/7] Correct function in Sequential layer, add note, style changes. --- src/mlpack/methods/ann/ffn_impl.hpp | 6 +++--- src/mlpack/methods/ann/layer/sequential.hpp | 3 +++ src/mlpack/methods/ann/layer/sequential_impl.hpp | 15 ++++++++++----- 3 files changed, 16 insertions(+), 8 deletions(-) diff --git a/src/mlpack/methods/ann/ffn_impl.hpp b/src/mlpack/methods/ann/ffn_impl.hpp index 03cfbae1c1..9aca45fa5d 100644 --- a/src/mlpack/methods/ann/ffn_impl.hpp +++ b/src/mlpack/methods/ann/ffn_impl.hpp @@ -202,6 +202,7 @@ void FFN::Predict( results.col(i) = resultsTemp.col(0); } } + template double FFN::Evaluate( @@ -218,9 +219,8 @@ double FFN::Evaluate( Forward(std::move(predictors)); - double res = outputLayer.Forward( - std::move(boost::apply_visitor(outputParameterVisitor, network.back())), - std::move(responses)); + double res = outputLayer.Forward(std::move(boost::apply_visitor( + outputParameterVisitor, network.back())), std::move(responses)); for (size_t i = 0; i < network.size(); ++i) { diff --git a/src/mlpack/methods/ann/layer/sequential.hpp b/src/mlpack/methods/ann/layer/sequential.hpp index 42a82f8d76..0c2cca0dcc 100644 --- a/src/mlpack/methods/ann/layer/sequential.hpp +++ b/src/mlpack/methods/ann/layer/sequential.hpp @@ -34,6 +34,9 @@ namespace ann /** Artificial Neural Network. */ { * feed-forward fully connected network container which plugs various layers * together. * + * Note: If this class is used as the first layer of a network, it should be + * preceded by IdentityLayer<>. + * * @tparam InputDataType Type of the input data (arma::colvec, arma::mat, * arma::sp_mat or arma::cube). * @tparam OutputDataType Type of the output data (arma::colvec, arma::mat, diff --git a/src/mlpack/methods/ann/layer/sequential_impl.hpp b/src/mlpack/methods/ann/layer/sequential_impl.hpp index 448dabb464..96411493bd 100644 --- a/src/mlpack/methods/ann/layer/sequential_impl.hpp +++ b/src/mlpack/methods/ann/layer/sequential_impl.hpp @@ -138,15 +138,20 @@ void Sequential::Gradient( arma::Mat&& error, arma::Mat&& /* gradient */) { - boost::apply_visitor(GradientVisitor(std::move(input), std::move(error)), - network.front()); + boost::apply_visitor(GradientVisitor(std::move(boost::apply_visitor( + outputParameterVisitor, network[network.size() - 2])), std::move(error)), + network.back()); - for (size_t i = 1; i < network.size() - 1; ++i) + for (size_t i = 2; i < network.size(); ++i) { boost::apply_visitor(GradientVisitor(std::move(boost::apply_visitor( - outputParameterVisitor, network[i - 1])), std::move( - boost::apply_visitor(deltaVisitor, network[i + 1]))), network[i]); + outputParameterVisitor, network[network.size() - i - 1])), std::move( + boost::apply_visitor(deltaVisitor, network[network.size() - i + 1]))), + network[network.size() - i]); } + + boost::apply_visitor(GradientVisitor(std::move(input), std::move( + boost::apply_visitor(deltaVisitor, network[1]))), network.front()); } template Date: Mon, 9 Jul 2018 17:07:00 +0530 Subject: [PATCH 5/7] Make suggested changes. --- src/mlpack/tests/loss_functions_test.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/mlpack/tests/loss_functions_test.cpp b/src/mlpack/tests/loss_functions_test.cpp index 5f7abbc61f..8ab5b4cef0 100644 --- a/src/mlpack/tests/loss_functions_test.cpp +++ b/src/mlpack/tests/loss_functions_test.cpp @@ -229,7 +229,6 @@ BOOST_AUTO_TEST_CASE(SimpleSigmoidCrossEntropyErrorTest) } /** -<<<<<<< 9cecc3e18160a4411c1e80831b6773bdbd80b789 * Simple test for the Earth Mover Distance Layer. */ BOOST_AUTO_TEST_CASE(SimpleEarthMoverDistanceLayerTest) @@ -265,7 +264,9 @@ BOOST_AUTO_TEST_CASE(SimpleEarthMoverDistanceLayerTest) BOOST_REQUIRE_SMALL(output(i) - expectedOutput(i), 1e-5); BOOST_REQUIRE_EQUAL(output.n_rows, input2.n_rows); BOOST_REQUIRE_EQUAL(output.n_cols, input2.n_cols); -======= +} + +/* * Reconstruction Loss numerical gradient test. */ BOOST_AUTO_TEST_CASE(GradientReconstructionLossTest) @@ -306,7 +307,6 @@ BOOST_AUTO_TEST_CASE(GradientReconstructionLossTest) } function; BOOST_REQUIRE_LE(CheckGradient(function), 1e-4); ->>>>>>> Add reconstruction loss with test. } BOOST_AUTO_TEST_SUITE_END(); From 23e9b0f1e53c8275c4c36bc28f97a2e34e2e62b3 Mon Sep 17 00:00:00 2001 From: akhandait Date: Tue, 10 Jul 2018 17:33:53 +0530 Subject: [PATCH 6/7] Use average KL loss. --- src/mlpack/methods/ann/layer/reparametrization.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mlpack/methods/ann/layer/reparametrization.hpp b/src/mlpack/methods/ann/layer/reparametrization.hpp index 48451f02b4..3a6acc39fc 100644 --- a/src/mlpack/methods/ann/layer/reparametrization.hpp +++ b/src/mlpack/methods/ann/layer/reparametrization.hpp @@ -116,7 +116,7 @@ class Reparametrization if (!includeKl) return 0; - return -0.5 * beta * arma::accu(2 * arma::log(stdDev) - arma::pow(stdDev, 2) + return -0.5 * arma::accu(2 * arma::log(stdDev) - arma::pow(stdDev, 2) - arma::pow(mean, 2) + 1) / mean.n_cols; } From a2ca441f1fd9237763fa11f2f08f7c4325843231 Mon Sep 17 00:00:00 2001 From: akhandait Date: Thu, 12 Jul 2018 02:11:23 +0530 Subject: [PATCH 7/7] Make suggested changes. --- .../methods/ann/layer/reparametrization.hpp | 2 +- .../ann/loss_functions/cross_entropy_error.hpp | 2 +- .../loss_functions/earth_mover_distance.hpp | 2 +- .../ann/loss_functions/mean_squared_error.hpp | 2 +- .../loss_functions/negative_log_likelihood.hpp | 3 ++- .../ann/loss_functions/reconstruction_loss.hpp | 18 ++++++++++-------- .../reconstruction_loss_impl.hpp | 2 +- .../sigmoid_cross_entropy_error.hpp | 2 +- src/mlpack/tests/loss_functions_test.cpp | 4 ++-- 9 files changed, 20 insertions(+), 17 deletions(-) diff --git a/src/mlpack/methods/ann/layer/reparametrization.hpp b/src/mlpack/methods/ann/layer/reparametrization.hpp index 3a6acc39fc..48451f02b4 100644 --- a/src/mlpack/methods/ann/layer/reparametrization.hpp +++ b/src/mlpack/methods/ann/layer/reparametrization.hpp @@ -116,7 +116,7 @@ class Reparametrization if (!includeKl) return 0; - return -0.5 * arma::accu(2 * arma::log(stdDev) - arma::pow(stdDev, 2) + return -0.5 * beta * arma::accu(2 * arma::log(stdDev) - arma::pow(stdDev, 2) - arma::pow(mean, 2) + 1) / mean.n_cols; } diff --git a/src/mlpack/methods/ann/loss_functions/cross_entropy_error.hpp b/src/mlpack/methods/ann/loss_functions/cross_entropy_error.hpp index 46db741c54..12f3030087 100644 --- a/src/mlpack/methods/ann/loss_functions/cross_entropy_error.hpp +++ b/src/mlpack/methods/ann/loss_functions/cross_entropy_error.hpp @@ -46,7 +46,7 @@ class CrossEntropyError * Computes the cross-entropy function. * * @param input Input data used for evaluating the specified function. - * @param output Resulting output activation. + * @param target The target vector. */ template double Forward(const InputType&& input, const TargetType&& target); diff --git a/src/mlpack/methods/ann/loss_functions/earth_mover_distance.hpp b/src/mlpack/methods/ann/loss_functions/earth_mover_distance.hpp index 2ced5b34f2..75df7c8a03 100644 --- a/src/mlpack/methods/ann/loss_functions/earth_mover_distance.hpp +++ b/src/mlpack/methods/ann/loss_functions/earth_mover_distance.hpp @@ -42,7 +42,7 @@ class EarthMoverDistance * Ordinary feed forward pass of a neural network. * * @param input Input data used for evaluating the specified function. - * @param output Resulting output activation. + * @param target The target vector. */ template double Forward(const InputType&& input, const TargetType&& target); diff --git a/src/mlpack/methods/ann/loss_functions/mean_squared_error.hpp b/src/mlpack/methods/ann/loss_functions/mean_squared_error.hpp index 59196a23aa..dd987b3d22 100644 --- a/src/mlpack/methods/ann/loss_functions/mean_squared_error.hpp +++ b/src/mlpack/methods/ann/loss_functions/mean_squared_error.hpp @@ -43,7 +43,7 @@ class MeanSquaredError * Computes the mean squared error function. * * @param input Input data used for evaluating the specified function. - * @param output Resulting output activation. + * @param target The target vector. */ template double Forward(const InputType&& input, const TargetType&& target); diff --git a/src/mlpack/methods/ann/loss_functions/negative_log_likelihood.hpp b/src/mlpack/methods/ann/loss_functions/negative_log_likelihood.hpp index cd92765d1c..c0362fe36c 100644 --- a/src/mlpack/methods/ann/loss_functions/negative_log_likelihood.hpp +++ b/src/mlpack/methods/ann/loss_functions/negative_log_likelihood.hpp @@ -44,7 +44,8 @@ class NegativeLogLikelihood * Computes the Negative log likelihood. * * @param input Input data used for evaluating the specified function. - * @param output Resulting output activation. + * @param target The target vector, that contains the class index in the range + * between 1 and the number of classes. */ template double Forward(const InputType&& input, TargetType&& target); diff --git a/src/mlpack/methods/ann/loss_functions/reconstruction_loss.hpp b/src/mlpack/methods/ann/loss_functions/reconstruction_loss.hpp index 4f46782b59..98ff7dd07c 100644 --- a/src/mlpack/methods/ann/loss_functions/reconstruction_loss.hpp +++ b/src/mlpack/methods/ann/loss_functions/reconstruction_loss.hpp @@ -13,25 +13,26 @@ #define MLPACK_METHODS_ANN_LOSS_FUNCTION_RECONSTRUCTION_LOSS_HPP #include -#include +#include namespace mlpack { namespace ann /** Artificial Neural Network. */ { /** - * The mean squared error performance function measures the network's - * performance according to the mean of squared errors. + * The reconstruction loss performance function measures the network's + * performance equal to the negative log probability of the target with + * the input distribution. * - * @tparam ActivationFunction Activation function used for the embedding layer. * @tparam InputDataType Type of the input data (arma::colvec, arma::mat, * arma::sp_mat or arma::cube). * @tparam OutputDataType Type of the output data (arma::colvec, arma::mat, * arma::sp_mat or arma::cube). + * @tparam DistType The type of distribution parametrized by the input. */ template < typename InputDataType = arma::mat, typename OutputDataType = arma::mat, - typename DistType = NormalDistribution + typename DistType = BernoulliDistribution > class ReconstructionLoss { @@ -42,18 +43,19 @@ class ReconstructionLoss ReconstructionLoss(); /* - * Computes the mean squared error function. + * Computes the reconstruction loss. * * @param input Input data used for evaluating the specified function. - * @param output Resulting output activation. + * @param target The target matrix. */ template double Forward(const InputType&& input, const TargetType&& target); + /** * Ordinary feed backward pass of a neural network. * * @param input The propagated input activation. - * @param target The target vector. + * @param target The target matrix. * @param output The calculated error. */ template diff --git a/src/mlpack/methods/ann/loss_functions/reconstruction_loss_impl.hpp b/src/mlpack/methods/ann/loss_functions/reconstruction_loss_impl.hpp index dc24ec30f5..73fd57f64d 100644 --- a/src/mlpack/methods/ann/loss_functions/reconstruction_loss_impl.hpp +++ b/src/mlpack/methods/ann/loss_functions/reconstruction_loss_impl.hpp @@ -45,7 +45,7 @@ void ReconstructionLoss::Backward( OutputType&& output) { dist.LogProbBackward(std::move(target), std::move(output)); - output = -output; + output *= -1; } template diff --git a/src/mlpack/methods/ann/loss_functions/sigmoid_cross_entropy_error.hpp b/src/mlpack/methods/ann/loss_functions/sigmoid_cross_entropy_error.hpp index 366bf31082..e15aa998da 100644 --- a/src/mlpack/methods/ann/loss_functions/sigmoid_cross_entropy_error.hpp +++ b/src/mlpack/methods/ann/loss_functions/sigmoid_cross_entropy_error.hpp @@ -61,7 +61,7 @@ class SigmoidCrossEntropyError * Computes the Sigmoid CrossEntropy Error functions. * * @param input Input data used for evaluating the specified function. - * @param output Resulting output activation. + * @param target The target vector. */ template inline double Forward(const InputType&& input, diff --git a/src/mlpack/tests/loss_functions_test.cpp b/src/mlpack/tests/loss_functions_test.cpp index 8ab5b4cef0..13082e9a2c 100644 --- a/src/mlpack/tests/loss_functions_test.cpp +++ b/src/mlpack/tests/loss_functions_test.cpp @@ -283,8 +283,8 @@ BOOST_AUTO_TEST_CASE(GradientReconstructionLossTest) model->Predictors() = input; model->Responses() = target; model->Add >(); - model->Add >(10, 4); - model->Add >(); + model->Add >(10, 2); + model->Add >(); } ~GradientFunction()