Merge PR #2196.

HISTORY.md

@@ -74,6 +74,10 @@
 
   * Added `R2 Score` regression metric (#2323).
 
+  * Added `poisson negative log likelihood` loss function (#2196).
+
+  * Added `huber` loss function (#2199).
+
   * Added `mean squared logarithmic error` loss function for neural networks
     (#2210).
 

src/mlpack/methods/ann/loss_functions/CMakeLists.txt

@@ -27,6 +27,8 @@ set(SOURCES
   negative_log_likelihood_impl.hpp
   log_cosh_loss.hpp
   log_cosh_loss_impl.hpp
+  poisson_nll_loss.hpp
+  poisson_nll_loss_impl.hpp
   reconstruction_loss.hpp
   reconstruction_loss_impl.hpp
   sigmoid_cross_entropy_error.hpp

src/mlpack/methods/ann/loss_functions/poisson_nll_loss.hpp

@@ -0,0 +1,165 @@
+/**
+ * @file methods/ann/loss_functions/poisson_nll_loss.hpp
+ * @author Mrityunjay Tripathi
+ *
+ * Definition of the PoissonNLLLoss class. It is the negative log likelihood of
+ * the Poisson distribution.
+ *
+ * mlpack is free software; you may redistribute it and/or modify it under the
+ * terms of the 3-clause BSD license.  You should have received a copy of the
+ * 3-clause BSD license along with mlpack.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef MLPACK_METHODS_ANN_LOSS_FUNCTIONS_POISSON_NLL_LOSS_HPP
+#define MLPACK_METHODS_ANN_LOSS_FUNCTIONS_POISSON_NLL_LOSS_HPP
+
+#include <mlpack/prereqs.hpp>
+
+namespace mlpack {
+namespace ann /** Artificial Neural Network. */ {
+
+/**
+ * Implementation of the Poisson negative log likelihood loss. This loss
+ * function expects input for each class. It also expects a class index,
+ * in the range between 1 and the number of classes, as target when calling
+ * the Forward function.
+ *
+ * @tparam InputDataType Type of the input data (arma::colvec, arma::mat,
+ *         arma::sp_mat or arma::cube).
+ * @tparam OutputDataType Type of the output data (arma::colvec, arma::mat,
+ *         arma::sp_mat or arma::cube).
+ */
+template <
+  typename InputDataType = arma::mat,
+  typename OutputDataType = arma::mat
+>
+class PoissonNLLLoss
+{
+ public:
+  /**
+   * Create the PoissonNLLLoss object.
+   *
+   * @param logInput If true the loss is computed as
+   *        \f$ \exp(input) - target \cdot input \f$, if false then the loss is
+   *        \f$ input - target \cdot \log(input + eps) \f$.
+   * @param full Boolean value that determines whether to include Stirling's
+   *        approximation term.
+   * @param eps A small value to prevent 0 in denominators and logarithms.
+   * @param mean When true, mean loss is computed otherwise total loss.
+   */
+  PoissonNLLLoss(const bool logInput = true,
+                 const bool full = false,
+                 const typename InputDataType::elem_type eps = 1e-08,
+                 const bool mean = true);
+
+  /**
+   * Computes the Poisson negative log likelihood Loss.
+   *
+   * @param input Input data used for evaluating the specified function.
+   * @param target The target vector, that contains the class index in the range
+   *        between 1 and the number of classes.
+   */
+  template<typename InputType, typename TargetType>
+  typename InputDataType::elem_type Forward(const InputType& input,
+                                            const TargetType& target);
+
+  /**
+   * Ordinary feed backward pass of a neural network. The Poisson Negative Log
+   * Likelihood loss function expects the input for each class.
+   * It expects a class index, in the range between 1 and the number of classes,
+   * as target when calling the Forward function.
+   *
+   * @param input The propagated input activation.
+   * @param target The target vector, that contains the class index in the range
+   *        between 1 and the number of classes.
+   * @param output The calculated error.
+   */
+  template<typename InputType, typename TargetType, typename OutputType>
+  void Backward(const InputType& input,
+                const TargetType& target,
+                OutputType& output);
+
+  //! Get the input parameter.
+  InputDataType& InputParameter() const { return inputParameter; }
+  //! Modify the input parameter.
+  InputDataType& InputParameter() { return inputParameter; }
+
+  //! Get the output parameter.
+  OutputDataType& OutputParameter() const { return outputParameter; }
+  //! Modify the output parameter.
+  OutputDataType& OutputParameter() { return outputParameter; }
+
+  //! Get the value of logInput. logInput is a boolean value that tells if
+  //! logits are given as input.
+  bool LogInput() const { return logInput; }
+  //! Modify the value of logInput. logInput is a boolean value that tells if
+  //! logits are given as input.
+  bool& LogInput() { return logInput; }
+
+  //! Get the value of full. full is a boolean value that determines whether to
+  //! include Stirling's approximation term.
+  bool Full() const { return full; }
+  //! Modify the value of full. full is a boolean value that determines whether
+  //! to include Stirling's approximation term.
+  bool& Full() { return full; }
+
+  //! Get the value of eps. eps is a small value required to prevent 0 in
+  //! logarithms and denominators.
+  typename InputDataType::elem_type Eps() const { return eps; }
+  //! Modify the value of eps. eps is a small value required to prevent 0 in
+  //! logarithms and denominators.
+  typename InputDataType::elem_type& Eps() { return eps; }
+
+  //! Get the value of mean. It's a boolean value that tells if
+  //! mean of the total loss has to be taken.
+  bool Mean() const { return mean; }
+  //! Modify the value of mean. It's a boolean value that tells if
+  //! mean of the total loss has to be taken.
+  bool& Mean() { return mean; }
+
+  /**
+   * Serialize the layer.
+   */
+  template<typename Archive>
+  void serialize(Archive& ar, const unsigned int /* version */);
+
+ private:
+  //! Check if the probabilities lie in the range [0, 1].
+  template<typename eT>
+  void CheckProbs(const arma::Mat<eT>& probs)
+  {
+    for (size_t i = 0; i < probs.size(); ++i)
+    {
+      if (probs[i] > 1.0 || probs[i] < 0.0)
+        Log::Fatal << "Probabilities cannot be greater than 1 "
+                   << "or smaller than 0." << std::endl;
+    }
+  }
+
+  //! Locally-stored input parameter object.
+  InputDataType inputParameter;
+
+  //! Locally-stored output parameter object.
+  OutputDataType outputParameter;
+
+  //! Boolean value that tells if logits are given as input.
+  bool logInput;
+
+  //! Boolean value that determines whether to include Stirling's
+  // approximation term.
+  bool full;
+
+  //! eps is a small value required to prevent 0 in logarithms and denominators.
+  typename InputDataType::elem_type eps;
+
+  //! Boolean value that tells if mean of the total loss has to be taken.
+  bool mean;
+}; // class PoissonNLLLoss
+
+} // namespace ann
+} // namespace mlpack
+
+// Include implementation.
+#include "poisson_nll_loss_impl.hpp"
+
+#endif

src/mlpack/methods/ann/loss_functions/poisson_nll_loss_impl.hpp

@@ -0,0 +1,97 @@
+/**
+ * @file methods/ann/loss_functions/poisson_nll_loss_impl.hpp
+ * @author Mrityunjay Tripathi
+ *
+ * Implementation of the PoissonNLLLoss class.
+ *
+ * mlpack is free software; you may redistribute it and/or modify it under the
+ * terms of the 3-clause BSD license.  You should have received a copy of the
+ * 3-clause BSD license along with mlpack.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef MLPACK_METHODS_ANN_LOSS_FUNCTIONS_POISSON_NLL_LOSS_IMPL_HPP
+#define MLPACK_METHODS_ANN_LOSS_FUNCTIONS_POISSON_NLL_LOSS_IMPL_HPP
+
+
+// In case it hasn't yet been included.
+#include "poisson_nll_loss.hpp"
+
+namespace mlpack {
+namespace ann /** Artificial Neural Network. */ {
+
+template<typename InputDataType, typename OutputDataType>
+PoissonNLLLoss<InputDataType, OutputDataType>::PoissonNLLLoss(
+    const bool logInput,
+    const bool full,
+    const typename InputDataType::elem_type eps,
+    const bool mean):
+    logInput(logInput),
+    full(full),
+    eps(eps),
+    mean(mean)
+{
+  Log::Assert(eps >= 0, "Epsilon (eps) must be greater than or equal to zero.");
+}
+
+template<typename InputDataType, typename OutputDataType>
+template<typename InputType, typename TargetType>
+typename InputDataType::elem_type
+PoissonNLLLoss<InputDataType, OutputDataType>::Forward(
+    const InputType& input,
+    const TargetType& target)
+{
+  InputType loss(arma::size(input));
+
+  if (logInput)
+    loss = arma::exp(input) - target % input;
+  else
+  {
+    CheckProbs(input);
+    loss = input - target % arma::log(input + eps);
+  }
+
+  if (full)
+  {
+    const auto mask = target > 1.0;
+    const InputType approx = target % arma::log(target) - target
+        + 0.5 * arma::log(2 * M_PI * target);
+    loss.elem(arma::find(mask)) += approx.elem(arma::find(mask));
+  }
+
+  return mean ? arma::accu(loss) / loss.n_elem : arma::accu(loss);
+}
+
+template<typename InputDataType, typename OutputDataType>
+template<typename InputType, typename TargetType, typename OutputType>
+void PoissonNLLLoss<InputDataType, OutputDataType>::Backward(
+    const InputType& input,
+    const TargetType& target,
+    OutputType& output)
+{
+  output.set_size(size(input));
+
+  if (logInput)
+    output = (arma::exp(input) - target);
+  else
+    output = (1 - target / (input + eps));
+
+  if (mean)
+    output = output / output.n_elem;
+}
+
+template<typename InputDataType, typename OutputDataType>
+template<typename Archive>
+void PoissonNLLLoss<InputDataType, OutputDataType>::serialize(
+    Archive& ar,
+    const unsigned int /* version */)
+{
+  ar & BOOST_SERIALIZATION_NVP(logInput);
+  ar & BOOST_SERIALIZATION_NVP(full);
+  ar & BOOST_SERIALIZATION_NVP(eps);
+  ar & BOOST_SERIALIZATION_NVP(mean);
+}
+
+} // namespace ann
+} // namespace mlpack
+
+#endif

src/mlpack/tests/loss_functions_test.cpp

@@ -16,6 +16,7 @@
 
 #include <mlpack/methods/ann/layer/layer.hpp>
 #include <mlpack/methods/ann/loss_functions/huber_loss.hpp>
+#include <mlpack/methods/ann/loss_functions/poisson_nll_loss.hpp>
 #include <mlpack/methods/ann/loss_functions/kl_divergence.hpp>
 #include <mlpack/methods/ann/loss_functions/earth_mover_distance.hpp>
 #include <mlpack/methods/ann/loss_functions/mean_squared_error.hpp>
@@ -69,6 +70,73 @@ BOOST_AUTO_TEST_CASE(HuberLossTest)
   BOOST_REQUIRE_EQUAL(output.n_cols, input.n_cols);
 }
 
+/**
+ * Poisson Negative Log Likelihood Loss function test.
+ */
+BOOST_AUTO_TEST_CASE(PoissonNLLLossTest)
+{
+  arma::mat input, target, input4, target4;
+  arma::mat output1, output2, output3, output4;
+  arma::mat expOutput1, expOutput2, expOutput3, expOutput4;
+  PoissonNLLLoss<> module1;
+  PoissonNLLLoss<> module2(true, true, 1e-08, false);
+  PoissonNLLLoss<> module3(true, true, 1e-08, true);
+  PoissonNLLLoss<> module4(false, true, 1e-08, true);
+
+  // Test the Forward function on a user generated input.
+  input = arma::mat("1.0 1.0 1.9 1.6 -1.9 3.7 -1.0 0.5");
+  target = arma::mat("1.0 3.0 1.0 2.0 1.0 4.0 2.0 1.0");
+
+  // Input required for module 4. Probs are in range [0, 1].
+  input4 = arma::mat("0.658502 0.445627 0.667651 0.310549 \
+                      0.589540 0.052568 0.549769 0.381504 ");
+  target4 = arma::mat("1.0 3.0 1.0 2.0 1.0 4.0 2.0 1.0");
+
+  double loss1 = module1.Forward(input, target);
+  double loss2 = module2.Forward(input, target);
+  double loss3 = module3.Forward(input, target);
+  double loss4 = module4.Forward(input4, target4);
+  BOOST_REQUIRE_CLOSE_FRACTION(loss1, 4.8986, 0.0001);
+  BOOST_REQUIRE_CLOSE_FRACTION(loss2, 45.4139, 0.0001);
+  BOOST_REQUIRE_CLOSE_FRACTION(loss3, 5.6767, 0.0001);
+  BOOST_REQUIRE_CLOSE_FRACTION(loss4, 3.742157, 0.0001);
+
+  // Test the Backward function.
+  module1.Backward(input, target, output1);
+  module2.Backward(input, target, output2);
+  module3.Backward(input, target, output3);
+  module4.Backward(input4, target4, output4);
+
+  expOutput1 = arma::mat("0.214785 -0.0352148 0.710737 0.369129 \
+                         -0.106304 4.55591 -0.204015 0.0810902");
+  expOutput2 = arma::mat("1.71828 -0.281718 5.68589 2.95303\
+                         -0.850431 36.4473 -1.63212 0.648721");
+  expOutput3 = arma::mat("0.214785 -0.035215 0.710737 0.369129 \
+                         -0.106304 4.555913 -0.204015 0.081090");
+  expOutput4 = arma::mat("-0.064825 -0.716511 -0.062224 -0.680027 \
+                          -0.087030 -9.386517 -0.329736 -0.202650");
+
+  BOOST_REQUIRE_EQUAL(output1.n_rows, input.n_rows);
+  BOOST_REQUIRE_EQUAL(output1.n_cols, input.n_cols);
+
+  BOOST_REQUIRE_EQUAL(output2.n_rows, input.n_rows);
+  BOOST_REQUIRE_EQUAL(output2.n_cols, input.n_cols);
+
+  BOOST_REQUIRE_EQUAL(output3.n_rows, input.n_rows);
+  BOOST_REQUIRE_EQUAL(output3.n_cols, input.n_cols);
+
+  BOOST_REQUIRE_EQUAL(output4.n_rows, input4.n_rows);
+  BOOST_REQUIRE_EQUAL(output4.n_cols, input4.n_cols);
+
+  for (size_t i = 0; i < expOutput1.n_elem; ++i)
+  {
+    BOOST_REQUIRE_CLOSE_FRACTION(output1[i], expOutput1[i], 0.0001);
+    BOOST_REQUIRE_CLOSE_FRACTION(output2[i], expOutput2[i], 0.0001);
+    BOOST_REQUIRE_CLOSE_FRACTION(output3[i], expOutput3[i], 0.0001);
+    BOOST_REQUIRE_CLOSE_FRACTION(output4[i], expOutput4[i], 0.0001);
+  }
+}
+
 /**
  * Simple KL Divergence test.  The loss should be zero if input = target.
  */