diff --git a/src/mlpack/methods/ann/loss_functions/CMakeLists.txt b/src/mlpack/methods/ann/loss_functions/CMakeLists.txt index cce48ed7cb..c332a075b4 100644 --- a/src/mlpack/methods/ann/loss_functions/CMakeLists.txt +++ b/src/mlpack/methods/ann/loss_functions/CMakeLists.txt @@ -11,6 +11,8 @@ set(SOURCES mean_squared_error_impl.hpp negative_log_likelihood.hpp negative_log_likelihood_impl.hpp + reconstruction_loss.hpp + reconstruction_loss_impl.hpp sigmoid_cross_entropy_error.hpp sigmoid_cross_entropy_error_impl.hpp ) diff --git a/src/mlpack/methods/ann/loss_functions/reconstruction_loss.hpp b/src/mlpack/methods/ann/loss_functions/reconstruction_loss.hpp new file mode 100644 index 0000000000..20d79487ac --- /dev/null +++ b/src/mlpack/methods/ann/loss_functions/reconstruction_loss.hpp @@ -0,0 +1,89 @@ +/** + * @file reconstruction_loss.hpp + * @author Atharva Khandait + * + * Definition of the reconstruction loss performance function. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_METHODS_ANN_LOSS_FUNCTION_RECONSTRUCTION_LOSS_HPP +#define MLPACK_METHODS_ANN_LOSS_FUNCTION_RECONSTRUCTION_LOSS_HPP + +#include +#include + +namespace mlpack { +namespace ann /** Artificial Neural Network. */ { + +/** + * The mean squared error performance function measures the network's + * performance according to the mean of squared errors. + * + * @tparam ActivationFunction Activation function used for the embedding layer. + * @tparam InputDataType Type of the input data (arma::colvec, arma::mat, + * arma::sp_mat or arma::cube). + * @tparam OutputDataType Type of the output data (arma::colvec, arma::mat, + * arma::sp_mat or arma::cube). + */ +template < + typename InputDataType = arma::mat, + typename OutputDataType = arma::mat, + typename DistType = NormalDistribution +> +class ReconstructionLoss +{ + public: + /** + * Create the ReconstructionLoss object. + */ + ReconstructionLoss(); + + /* + * Computes the mean squared error function. + * + * @param input Input data used for evaluating the specified function. + * @param output Resulting output activation. + */ + template + double Forward(const InputType&& input, const TargetType&& target); + /** + * Ordinary feed backward pass of a neural network. + * + * @param input The propagated input activation. + * @param target The target vector. + * @param output The calculated error. + */ + template + void Backward(const InputType&& input, + const TargetType&& target, + OutputType&& output); + + //! Get the output parameter. + OutputDataType& OutputParameter() const { return outputParameter; } + //! Modify the output parameter. + OutputDataType& OutputParameter() { return outputParameter; } + + /** + * Serialize the layer + */ + template + void serialize(Archive& ar, const unsigned int /* version */); + + private: + //! Locally-stored distribution object. + DistType* dist; + + //! Locally-stored output parameter object. + OutputDataType outputParameter; +}; // class ReconstructionLoss + +} // namespace ann +} // namespace mlpack + +// Include implementation. +#include "reconstruction_loss_impl.hpp" + +#endif diff --git a/src/mlpack/methods/ann/loss_functions/reconstruction_loss_impl.hpp b/src/mlpack/methods/ann/loss_functions/reconstruction_loss_impl.hpp new file mode 100644 index 0000000000..aeb72df3e4 --- /dev/null +++ b/src/mlpack/methods/ann/loss_functions/reconstruction_loss_impl.hpp @@ -0,0 +1,62 @@ +/** + * @file reconstruction_loss_impl.hpp + * @author Atharva Khandait + * + * Implementation of the reconstruction loss performance function. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_METHODS_ANN_LOSS_FUNCTION_RECONSTRUCTION_LOSS_IMPL_HPP +#define MLPACK_METHODS_ANN_LOSS_FUNCTION_RECONSTRUCTION_LOSS_IMPL_HPP + +// In case it hasn't yet been included. +#include "reconstruction_loss.hpp" + +namespace mlpack { +namespace ann /** Artificial Neural Network. */ { + +template +ReconstructionLoss< + InputDataType, + OutputDataType, + DistType +>::ReconstructionLoss() +{ + // Nothing to do here. +} + +template +template +double ReconstructionLoss::Forward( + const InputType&& input, const TargetType&& target) +{ + dist = new DistType(std::move(input)); + return dist->LogProbability(std::move(target)); +} + +template +template +void ReconstructionLoss::Backward( + const InputType&& /* input */, + const TargetType&& target, + OutputType&& output) +{ + dist->LogProbBackward(std::move(target), std::move(output)); +} + +template +template +void ReconstructionLoss::serialize( + Archive& /* ar */, + const unsigned int /* version */) +{ + // Nothing to do here. +} + +} // namespace ann +} // namespace mlpack + +#endif diff --git a/src/mlpack/tests/CMakeLists.txt b/src/mlpack/tests/CMakeLists.txt index f77ca622c2..e6a875aafd 100644 --- a/src/mlpack/tests/CMakeLists.txt +++ b/src/mlpack/tests/CMakeLists.txt @@ -9,6 +9,7 @@ add_executable(mlpack_test aknn_test.cpp ann_dist_test.cpp ann_layer_test.cpp + ann_test_tools.hpp arma_extend_test.cpp armadillo_svd_test.cpp async_learning_test.cpp diff --git a/src/mlpack/tests/ann_layer_test.cpp b/src/mlpack/tests/ann_layer_test.cpp index 3e34c9d475..440223b40b 100644 --- a/src/mlpack/tests/ann_layer_test.cpp +++ b/src/mlpack/tests/ann_layer_test.cpp @@ -14,8 +14,6 @@ #include #include -#include -#include #include #include #include @@ -24,162 +22,13 @@ #include #include "test_tools.hpp" +#include "ann_test_tools.hpp" using namespace mlpack; using namespace mlpack::ann; BOOST_AUTO_TEST_SUITE(ANNLayerTest); -// Helper function which calls the Reset function of the given module. -template -void ResetFunction( - T& layer, - typename std::enable_if::value>::type* = 0) -{ - layer.Reset(); -} - -template -void ResetFunction( - T& /* layer */, - typename std::enable_if::value>::type* = 0) -{ - /* Nothing to do here */ -} - -// Approximate Jacobian and supposedly-true Jacobian, then compare them -// similarly to before. -template -double JacobianTest(ModuleType& module, - arma::mat& input, - const double minValue = -2, - const double maxValue = -1, - const double perturbation = 1e-6) -{ - arma::mat output, outputA, outputB, jacobianA, jacobianB; - - // Initialize the input matrix. - RandomInitialization init(minValue, maxValue); - init.Initialize(input, input.n_rows, input.n_cols); - - // Initialize the module parameters. - ResetFunction(module); - - // Initialize the jacobian matrix. - module.Forward(std::move(input), std::move(output)); - jacobianA = arma::zeros(input.n_elem, output.n_elem); - - // Share the input paramter matrix. - arma::mat sin = arma::mat(input.memptr(), input.n_rows, input.n_cols, - false, false); - - for (size_t i = 0; i < input.n_elem; ++i) - { - double original = sin(i); - sin(i) = original - perturbation; - module.Forward(std::move(input), std::move(outputA)); - sin(i) = original + perturbation; - module.Forward(std::move(input), std::move(outputB)); - sin(i) = original; - - outputB -= outputA; - outputB /= 2 * perturbation; - jacobianA.row(i) = outputB.t(); - } - - // Initialize the derivative parameter. - arma::mat deriv = arma::zeros(output.n_rows, output.n_cols); - - // Share the derivative parameter. - arma::mat derivTemp = arma::mat(deriv.memptr(), deriv.n_rows, deriv.n_cols, - false, false); - - // Initialize the jacobian matrix. - jacobianB = arma::zeros(input.n_elem, output.n_elem); - - for (size_t i = 0; i < derivTemp.n_elem; ++i) - { - deriv.zeros(); - derivTemp(i) = 1; - - arma::mat delta; - module.Backward(std::move(input), std::move(deriv), std::move(delta)); - - jacobianB.col(i) = delta; - } - - return arma::max(arma::max(arma::abs(jacobianA - jacobianB))); -} - -// Approximate Jacobian and supposedly-true Jacobian, then compare them -// similarly to before. -template -double JacobianPerformanceTest(ModuleType& module, - arma::mat& input, - arma::mat& target, - const double eps = 1e-6) -{ - module.Forward(std::move(input), std::move(target)); - - arma::mat delta; - module.Backward(std::move(input), std::move(target), std::move(delta)); - - arma::mat centralDifference = arma::zeros(delta.n_rows, delta.n_cols); - arma::mat inputTemp = arma::mat(input.memptr(), input.n_rows, input.n_cols, - false, false); - - arma::mat centralDifferenceTemp = arma::mat(centralDifference.memptr(), - centralDifference.n_rows, centralDifference.n_cols, false, false); - - for (size_t i = 0; i < input.n_elem; ++i) - { - inputTemp(i) = inputTemp(i) + eps; - double outputA = module.Forward(std::move(input), std::move(target)); - inputTemp(i) = inputTemp(i) - (2 * eps); - double outputB = module.Forward(std::move(input), std::move(target)); - - centralDifferenceTemp(i) = (outputA - outputB) / (2 * eps); - inputTemp(i) = inputTemp(i) + eps; - } - - return arma::max(arma::max(arma::abs(centralDifference - delta))); -} - -// Simple numerical gradient checker. -template -double CheckGradient(FunctionType& function, const double eps = 1e-7) -{ - // Get gradients for the current parameters. - arma::mat orgGradient, gradient, estGradient; - function.Gradient(orgGradient); - - estGradient = arma::zeros(orgGradient.n_rows, orgGradient.n_cols); - - // Compute numeric approximations to gradient. - for (size_t i = 0; i < orgGradient.n_elem; ++i) - { - double tmp = function.Parameters()(i); - - // Perturb parameter with a positive constant and get costs. - function.Parameters()(i) += eps; - double costPlus = function.Gradient(gradient); - - // Perturb parameter with a negative constant and get costs. - function.Parameters()(i) -= (2 * eps); - double costMinus = function.Gradient(gradient); - - // Restore the parameter value. - function.Parameters()(i) = tmp; - - // Compute numerical gradients using the costs calculated above. - estGradient(i) = (costPlus - costMinus) / (2 * eps); - } - - // Estimate error of gradient. - return arma::norm(orgGradient - estGradient) / - arma::norm(orgGradient + estGradient); -} - /** * Simple add module test. */ diff --git a/src/mlpack/tests/ann_test_tools.hpp b/src/mlpack/tests/ann_test_tools.hpp new file mode 100644 index 0000000000..67f7108cd0 --- /dev/null +++ b/src/mlpack/tests/ann_test_tools.hpp @@ -0,0 +1,170 @@ +/** + * @file ann_test_tools.hpp + * @author Marcus Edel + * + * This file includes some useful functions for ann tests. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_TESTS_ANN_TEST_TOOLS_HPP +#define MLPACK_TESTS_ANN_TEST_TOOLS_HPP + +#include + +using namespace mlpack; +using namespace mlpack::ann; + +// Helper function which calls the Reset function of the given module. +template +void ResetFunction( + T& layer, + typename std::enable_if::value>::type* = 0) +{ + layer.Reset(); +} + +template +void ResetFunction( + T& /* layer */, + typename std::enable_if::value>::type* = 0) +{ + /* Nothing to do here */ +} + +// Approximate Jacobian and supposedly-true Jacobian, then compare them +// similarly to before. +template +double JacobianTest(ModuleType& module, + arma::mat& input, + const double minValue = -2, + const double maxValue = -1, + const double perturbation = 1e-6) +{ + arma::mat output, outputA, outputB, jacobianA, jacobianB; + + // Initialize the input matrix. + RandomInitialization init(minValue, maxValue); + init.Initialize(input, input.n_rows, input.n_cols); + + // Initialize the module parameters. + ResetFunction(module); + + // Initialize the jacobian matrix. + module.Forward(std::move(input), std::move(output)); + jacobianA = arma::zeros(input.n_elem, output.n_elem); + + // Share the input paramter matrix. + arma::mat sin = arma::mat(input.memptr(), input.n_rows, input.n_cols, + false, false); + + for (size_t i = 0; i < input.n_elem; ++i) + { + double original = sin(i); + sin(i) = original - perturbation; + module.Forward(std::move(input), std::move(outputA)); + sin(i) = original + perturbation; + module.Forward(std::move(input), std::move(outputB)); + sin(i) = original; + + outputB -= outputA; + outputB /= 2 * perturbation; + jacobianA.row(i) = outputB.t(); + } + + // Initialize the derivative parameter. + arma::mat deriv = arma::zeros(output.n_rows, output.n_cols); + + // Share the derivative parameter. + arma::mat derivTemp = arma::mat(deriv.memptr(), deriv.n_rows, deriv.n_cols, + false, false); + + // Initialize the jacobian matrix. + jacobianB = arma::zeros(input.n_elem, output.n_elem); + + for (size_t i = 0; i < derivTemp.n_elem; ++i) + { + deriv.zeros(); + derivTemp(i) = 1; + + arma::mat delta; + module.Backward(std::move(input), std::move(deriv), std::move(delta)); + + jacobianB.col(i) = delta; + } + + return arma::max(arma::max(arma::abs(jacobianA - jacobianB))); +} + +// Approximate Jacobian and supposedly-true Jacobian, then compare them +// similarly to before. +template +double JacobianPerformanceTest(ModuleType& module, + arma::mat& input, + arma::mat& target, + const double eps = 1e-6) +{ + module.Forward(std::move(input), std::move(target)); + + arma::mat delta; + module.Backward(std::move(input), std::move(target), std::move(delta)); + + arma::mat centralDifference = arma::zeros(delta.n_rows, delta.n_cols); + arma::mat inputTemp = arma::mat(input.memptr(), input.n_rows, input.n_cols, + false, false); + + arma::mat centralDifferenceTemp = arma::mat(centralDifference.memptr(), + centralDifference.n_rows, centralDifference.n_cols, false, false); + + for (size_t i = 0; i < input.n_elem; ++i) + { + inputTemp(i) = inputTemp(i) + eps; + double outputA = module.Forward(std::move(input), std::move(target)); + inputTemp(i) = inputTemp(i) - (2 * eps); + double outputB = module.Forward(std::move(input), std::move(target)); + + centralDifferenceTemp(i) = (outputA - outputB) / (2 * eps); + inputTemp(i) = inputTemp(i) + eps; + } + + return arma::max(arma::max(arma::abs(centralDifference - delta))); +} + +// Simple numerical gradient checker. +template +double CheckGradient(FunctionType& function, const double eps = 1e-7) +{ + // Get gradients for the current parameters. + arma::mat orgGradient, gradient, estGradient; + function.Gradient(orgGradient); + + estGradient = arma::zeros(orgGradient.n_rows, orgGradient.n_cols); + + // Compute numeric approximations to gradient. + for (size_t i = 0; i < orgGradient.n_elem; ++i) + { + double tmp = function.Parameters()(i); + + // Perturb parameter with a positive constant and get costs. + function.Parameters()(i) += eps; + double costPlus = function.Gradient(gradient); + + // Perturb parameter with a negative constant and get costs. + function.Parameters()(i) -= (2 * eps); + double costMinus = function.Gradient(gradient); + + // Restore the parameter value. + function.Parameters()(i) = tmp; + + // Compute numerical gradients using the costs calculated above. + estGradient(i) = (costPlus - costMinus) / (2 * eps); + } + + // Estimate error of gradient. + return arma::norm(orgGradient - estGradient) / + arma::norm(orgGradient + estGradient); +} + +#endif diff --git a/src/mlpack/tests/loss_functions_test.cpp b/src/mlpack/tests/loss_functions_test.cpp index ae46073786..5f7abbc61f 100644 --- a/src/mlpack/tests/loss_functions_test.cpp +++ b/src/mlpack/tests/loss_functions_test.cpp @@ -2,6 +2,7 @@ * @file loss_functions_test.cpp * @author Dakshit Agrawal * @author Sourabh Varshney + * @author Atharva Khandait * * Tests for loss functions in mlpack::methods::ann:loss_functions. * @@ -10,14 +11,21 @@ * 3-clause BSD license along with mlpack. If not, see * http://www.opensource.org/licenses/BSD-3-Clause for more information. */ -#include +#include + +#include #include +#include #include #include #include +#include +#include +#include #include #include "test_tools.hpp" +#include "ann_test_tools.hpp" using namespace mlpack; using namespace mlpack::ann; @@ -85,7 +93,7 @@ BOOST_AUTO_TEST_CASE(KLDivergenceNoMeanTest) /* * Simple test for the mean squared error performance function. */ -BOOST_AUTO_TEST_CASE(SimpleMeanSquaredErrorLayerTest) +BOOST_AUTO_TEST_CASE(SimpleMeanSquaredErrorTest) { arma::mat input, output, target; MeanSquaredError<> module; @@ -120,7 +128,7 @@ BOOST_AUTO_TEST_CASE(SimpleMeanSquaredErrorLayerTest) /* * Simple test for the cross-entropy error performance function. */ -BOOST_AUTO_TEST_CASE(SimpleCrossEntropyErrorLayerTest) +BOOST_AUTO_TEST_CASE(SimpleCrossEntropyErrorTest) { arma::mat input1, input2, output, target1, target2; CrossEntropyError<> module(1e-6); @@ -161,9 +169,9 @@ BOOST_AUTO_TEST_CASE(SimpleCrossEntropyErrorLayerTest) } /** - * Simple test for the Sigmoid Cross Entropy Layer. + * Simple test for the Sigmoid Cross Entropy performance function. */ -BOOST_AUTO_TEST_CASE(SimpleSigmoidCrossEntropyLayerTest) +BOOST_AUTO_TEST_CASE(SimpleSigmoidCrossEntropyErrorTest) { arma::mat input1, input2, input3, output, target1, target2, target3, expectedOutput; @@ -221,6 +229,7 @@ BOOST_AUTO_TEST_CASE(SimpleSigmoidCrossEntropyLayerTest) } /** +<<<<<<< 9cecc3e18160a4411c1e80831b6773bdbd80b789 * Simple test for the Earth Mover Distance Layer. */ BOOST_AUTO_TEST_CASE(SimpleEarthMoverDistanceLayerTest) @@ -256,6 +265,48 @@ BOOST_AUTO_TEST_CASE(SimpleEarthMoverDistanceLayerTest) BOOST_REQUIRE_SMALL(output(i) - expectedOutput(i), 1e-5); BOOST_REQUIRE_EQUAL(output.n_rows, input2.n_rows); BOOST_REQUIRE_EQUAL(output.n_cols, input2.n_cols); +======= + * Reconstruction Loss numerical gradient test. + */ +BOOST_AUTO_TEST_CASE(GradientReconstructionLossTest) +{ + // Linear function gradient instantiation. + struct GradientFunction + { + GradientFunction() + { + input = arma::randu(10, 1); + target = arma::randu(2, 1); + + model = new FFN, NguyenWidrowInitialization>(); + model->Predictors() = input; + model->Responses() = target; + model->Add >(); + model->Add >(10, 4); + model->Add >(); + } + + ~GradientFunction() + { + delete model; + } + + double Gradient(arma::mat& gradient) const + { + arma::mat output; + double error = model->Evaluate(model->Parameters(), 0, 1); + model->Gradient(model->Parameters(), 0, gradient, 1); + return error; + } + + arma::mat& Parameters() { return model->Parameters(); } + + FFN, NguyenWidrowInitialization>* model; + arma::mat input, target; + } function; + + BOOST_REQUIRE_LE(CheckGradient(function), 1e-4); +>>>>>>> Add reconstruction loss with test. } BOOST_AUTO_TEST_SUITE_END();