diff --git a/HISTORY.md b/HISTORY.md index 6eeffaf5a8..e23250330d 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -74,6 +74,10 @@ * Added `R2 Score` regression metric (#2323). + * Added `poisson negative log likelihood` loss function (#2196). + + * Added `huber` loss function (#2199). + * Added `mean squared logarithmic error` loss function for neural networks (#2210). diff --git a/src/mlpack/methods/ann/loss_functions/CMakeLists.txt b/src/mlpack/methods/ann/loss_functions/CMakeLists.txt index b51d274f29..a4e2cc590c 100644 --- a/src/mlpack/methods/ann/loss_functions/CMakeLists.txt +++ b/src/mlpack/methods/ann/loss_functions/CMakeLists.txt @@ -27,6 +27,8 @@ set(SOURCES negative_log_likelihood_impl.hpp log_cosh_loss.hpp log_cosh_loss_impl.hpp + poisson_nll_loss.hpp + poisson_nll_loss_impl.hpp reconstruction_loss.hpp reconstruction_loss_impl.hpp sigmoid_cross_entropy_error.hpp diff --git a/src/mlpack/methods/ann/loss_functions/poisson_nll_loss.hpp b/src/mlpack/methods/ann/loss_functions/poisson_nll_loss.hpp new file mode 100644 index 0000000000..bdb331f5e5 --- /dev/null +++ b/src/mlpack/methods/ann/loss_functions/poisson_nll_loss.hpp @@ -0,0 +1,165 @@ +/** + * @file methods/ann/loss_functions/poisson_nll_loss.hpp + * @author Mrityunjay Tripathi + * + * Definition of the PoissonNLLLoss class. It is the negative log likelihood of + * the Poisson distribution. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_METHODS_ANN_LOSS_FUNCTIONS_POISSON_NLL_LOSS_HPP +#define MLPACK_METHODS_ANN_LOSS_FUNCTIONS_POISSON_NLL_LOSS_HPP + +#include + +namespace mlpack { +namespace ann /** Artificial Neural Network. */ { + +/** + * Implementation of the Poisson negative log likelihood loss. This loss + * function expects input for each class. It also expects a class index, + * in the range between 1 and the number of classes, as target when calling + * the Forward function. + * + * @tparam InputDataType Type of the input data (arma::colvec, arma::mat, + * arma::sp_mat or arma::cube). + * @tparam OutputDataType Type of the output data (arma::colvec, arma::mat, + * arma::sp_mat or arma::cube). + */ +template < + typename InputDataType = arma::mat, + typename OutputDataType = arma::mat +> +class PoissonNLLLoss +{ + public: + /** + * Create the PoissonNLLLoss object. + * + * @param logInput If true the loss is computed as + * \f$ \exp(input) - target \cdot input \f$, if false then the loss is + * \f$ input - target \cdot \log(input + eps) \f$. + * @param full Boolean value that determines whether to include Stirling's + * approximation term. + * @param eps A small value to prevent 0 in denominators and logarithms. + * @param mean When true, mean loss is computed otherwise total loss. + */ + PoissonNLLLoss(const bool logInput = true, + const bool full = false, + const typename InputDataType::elem_type eps = 1e-08, + const bool mean = true); + + /** + * Computes the Poisson negative log likelihood Loss. + * + * @param input Input data used for evaluating the specified function. + * @param target The target vector, that contains the class index in the range + * between 1 and the number of classes. + */ + template + typename InputDataType::elem_type Forward(const InputType& input, + const TargetType& target); + + /** + * Ordinary feed backward pass of a neural network. The Poisson Negative Log + * Likelihood loss function expects the input for each class. + * It expects a class index, in the range between 1 and the number of classes, + * as target when calling the Forward function. + * + * @param input The propagated input activation. + * @param target The target vector, that contains the class index in the range + * between 1 and the number of classes. + * @param output The calculated error. + */ + template + void Backward(const InputType& input, + const TargetType& target, + OutputType& output); + + //! Get the input parameter. + InputDataType& InputParameter() const { return inputParameter; } + //! Modify the input parameter. + InputDataType& InputParameter() { return inputParameter; } + + //! Get the output parameter. + OutputDataType& OutputParameter() const { return outputParameter; } + //! Modify the output parameter. + OutputDataType& OutputParameter() { return outputParameter; } + + //! Get the value of logInput. logInput is a boolean value that tells if + //! logits are given as input. + bool LogInput() const { return logInput; } + //! Modify the value of logInput. logInput is a boolean value that tells if + //! logits are given as input. + bool& LogInput() { return logInput; } + + //! Get the value of full. full is a boolean value that determines whether to + //! include Stirling's approximation term. + bool Full() const { return full; } + //! Modify the value of full. full is a boolean value that determines whether + //! to include Stirling's approximation term. + bool& Full() { return full; } + + //! Get the value of eps. eps is a small value required to prevent 0 in + //! logarithms and denominators. + typename InputDataType::elem_type Eps() const { return eps; } + //! Modify the value of eps. eps is a small value required to prevent 0 in + //! logarithms and denominators. + typename InputDataType::elem_type& Eps() { return eps; } + + //! Get the value of mean. It's a boolean value that tells if + //! mean of the total loss has to be taken. + bool Mean() const { return mean; } + //! Modify the value of mean. It's a boolean value that tells if + //! mean of the total loss has to be taken. + bool& Mean() { return mean; } + + /** + * Serialize the layer. + */ + template + void serialize(Archive& ar, const unsigned int /* version */); + + private: + //! Check if the probabilities lie in the range [0, 1]. + template + void CheckProbs(const arma::Mat& probs) + { + for (size_t i = 0; i < probs.size(); ++i) + { + if (probs[i] > 1.0 || probs[i] < 0.0) + Log::Fatal << "Probabilities cannot be greater than 1 " + << "or smaller than 0." << std::endl; + } + } + + //! Locally-stored input parameter object. + InputDataType inputParameter; + + //! Locally-stored output parameter object. + OutputDataType outputParameter; + + //! Boolean value that tells if logits are given as input. + bool logInput; + + //! Boolean value that determines whether to include Stirling's + // approximation term. + bool full; + + //! eps is a small value required to prevent 0 in logarithms and denominators. + typename InputDataType::elem_type eps; + + //! Boolean value that tells if mean of the total loss has to be taken. + bool mean; +}; // class PoissonNLLLoss + +} // namespace ann +} // namespace mlpack + +// Include implementation. +#include "poisson_nll_loss_impl.hpp" + +#endif diff --git a/src/mlpack/methods/ann/loss_functions/poisson_nll_loss_impl.hpp b/src/mlpack/methods/ann/loss_functions/poisson_nll_loss_impl.hpp new file mode 100644 index 0000000000..8a69494203 --- /dev/null +++ b/src/mlpack/methods/ann/loss_functions/poisson_nll_loss_impl.hpp @@ -0,0 +1,97 @@ +/** + * @file methods/ann/loss_functions/poisson_nll_loss_impl.hpp + * @author Mrityunjay Tripathi + * + * Implementation of the PoissonNLLLoss class. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_METHODS_ANN_LOSS_FUNCTIONS_POISSON_NLL_LOSS_IMPL_HPP +#define MLPACK_METHODS_ANN_LOSS_FUNCTIONS_POISSON_NLL_LOSS_IMPL_HPP + + +// In case it hasn't yet been included. +#include "poisson_nll_loss.hpp" + +namespace mlpack { +namespace ann /** Artificial Neural Network. */ { + +template +PoissonNLLLoss::PoissonNLLLoss( + const bool logInput, + const bool full, + const typename InputDataType::elem_type eps, + const bool mean): + logInput(logInput), + full(full), + eps(eps), + mean(mean) +{ + Log::Assert(eps >= 0, "Epsilon (eps) must be greater than or equal to zero."); +} + +template +template +typename InputDataType::elem_type +PoissonNLLLoss::Forward( + const InputType& input, + const TargetType& target) +{ + InputType loss(arma::size(input)); + + if (logInput) + loss = arma::exp(input) - target % input; + else + { + CheckProbs(input); + loss = input - target % arma::log(input + eps); + } + + if (full) + { + const auto mask = target > 1.0; + const InputType approx = target % arma::log(target) - target + + 0.5 * arma::log(2 * M_PI * target); + loss.elem(arma::find(mask)) += approx.elem(arma::find(mask)); + } + + return mean ? arma::accu(loss) / loss.n_elem : arma::accu(loss); +} + +template +template +void PoissonNLLLoss::Backward( + const InputType& input, + const TargetType& target, + OutputType& output) +{ + output.set_size(size(input)); + + if (logInput) + output = (arma::exp(input) - target); + else + output = (1 - target / (input + eps)); + + if (mean) + output = output / output.n_elem; +} + +template +template +void PoissonNLLLoss::serialize( + Archive& ar, + const unsigned int /* version */) +{ + ar & BOOST_SERIALIZATION_NVP(logInput); + ar & BOOST_SERIALIZATION_NVP(full); + ar & BOOST_SERIALIZATION_NVP(eps); + ar & BOOST_SERIALIZATION_NVP(mean); +} + +} // namespace ann +} // namespace mlpack + +#endif diff --git a/src/mlpack/tests/loss_functions_test.cpp b/src/mlpack/tests/loss_functions_test.cpp index a750ade85d..180e374402 100644 --- a/src/mlpack/tests/loss_functions_test.cpp +++ b/src/mlpack/tests/loss_functions_test.cpp @@ -16,6 +16,7 @@ #include #include +#include #include #include #include @@ -69,6 +70,73 @@ BOOST_AUTO_TEST_CASE(HuberLossTest) BOOST_REQUIRE_EQUAL(output.n_cols, input.n_cols); } +/** + * Poisson Negative Log Likelihood Loss function test. + */ +BOOST_AUTO_TEST_CASE(PoissonNLLLossTest) +{ + arma::mat input, target, input4, target4; + arma::mat output1, output2, output3, output4; + arma::mat expOutput1, expOutput2, expOutput3, expOutput4; + PoissonNLLLoss<> module1; + PoissonNLLLoss<> module2(true, true, 1e-08, false); + PoissonNLLLoss<> module3(true, true, 1e-08, true); + PoissonNLLLoss<> module4(false, true, 1e-08, true); + + // Test the Forward function on a user generated input. + input = arma::mat("1.0 1.0 1.9 1.6 -1.9 3.7 -1.0 0.5"); + target = arma::mat("1.0 3.0 1.0 2.0 1.0 4.0 2.0 1.0"); + + // Input required for module 4. Probs are in range [0, 1]. + input4 = arma::mat("0.658502 0.445627 0.667651 0.310549 \ + 0.589540 0.052568 0.549769 0.381504 "); + target4 = arma::mat("1.0 3.0 1.0 2.0 1.0 4.0 2.0 1.0"); + + double loss1 = module1.Forward(input, target); + double loss2 = module2.Forward(input, target); + double loss3 = module3.Forward(input, target); + double loss4 = module4.Forward(input4, target4); + BOOST_REQUIRE_CLOSE_FRACTION(loss1, 4.8986, 0.0001); + BOOST_REQUIRE_CLOSE_FRACTION(loss2, 45.4139, 0.0001); + BOOST_REQUIRE_CLOSE_FRACTION(loss3, 5.6767, 0.0001); + BOOST_REQUIRE_CLOSE_FRACTION(loss4, 3.742157, 0.0001); + + // Test the Backward function. + module1.Backward(input, target, output1); + module2.Backward(input, target, output2); + module3.Backward(input, target, output3); + module4.Backward(input4, target4, output4); + + expOutput1 = arma::mat("0.214785 -0.0352148 0.710737 0.369129 \ + -0.106304 4.55591 -0.204015 0.0810902"); + expOutput2 = arma::mat("1.71828 -0.281718 5.68589 2.95303\ + -0.850431 36.4473 -1.63212 0.648721"); + expOutput3 = arma::mat("0.214785 -0.035215 0.710737 0.369129 \ + -0.106304 4.555913 -0.204015 0.081090"); + expOutput4 = arma::mat("-0.064825 -0.716511 -0.062224 -0.680027 \ + -0.087030 -9.386517 -0.329736 -0.202650"); + + BOOST_REQUIRE_EQUAL(output1.n_rows, input.n_rows); + BOOST_REQUIRE_EQUAL(output1.n_cols, input.n_cols); + + BOOST_REQUIRE_EQUAL(output2.n_rows, input.n_rows); + BOOST_REQUIRE_EQUAL(output2.n_cols, input.n_cols); + + BOOST_REQUIRE_EQUAL(output3.n_rows, input.n_rows); + BOOST_REQUIRE_EQUAL(output3.n_cols, input.n_cols); + + BOOST_REQUIRE_EQUAL(output4.n_rows, input4.n_rows); + BOOST_REQUIRE_EQUAL(output4.n_cols, input4.n_cols); + + for (size_t i = 0; i < expOutput1.n_elem; ++i) + { + BOOST_REQUIRE_CLOSE_FRACTION(output1[i], expOutput1[i], 0.0001); + BOOST_REQUIRE_CLOSE_FRACTION(output2[i], expOutput2[i], 0.0001); + BOOST_REQUIRE_CLOSE_FRACTION(output3[i], expOutput3[i], 0.0001); + BOOST_REQUIRE_CLOSE_FRACTION(output4[i], expOutput4[i], 0.0001); + } +} + /** * Simple KL Divergence test. The loss should be zero if input = target. */