Merge pull request #3003 from RishabhGarg108/sse-loss

SSE Loss for XGBoost

src/mlpack/methods/xgboost/CMakeLists.txt

@@ -0,0 +1,14 @@
+# Define the files we need to compile.
+# Anything not in this list will not be compiled into mlpack.
+set(SOURCES
+  loss_functions/sse_loss.hpp
+)
+
+# Add directory name to sources.
+set(DIR_SRCS)
+foreach(file ${SOURCES})
+  set(DIR_SRCS ${DIR_SRCS} ${CMAKE_CURRENT_SOURCE_DIR}/${file})
+endforeach()
+# Append sources (with directory name) to list of all mlpack sources (used at
+# the parent scope).
+set(MLPACK_SRCS ${MLPACK_SRCS} ${DIR_SRCS} PARENT_SCOPE)

src/mlpack/methods/xgboost/loss_functions/sse_loss.hpp

@@ -0,0 +1,125 @@
+/**
+ * @file methods/xgboost/loss_functions/sse_loss.hpp
+ * @author Rishabh Garg
+ *
+ * The sum of squared error loss class, which is a loss funtion for gradient
+ * xgboost based decision trees.
+ *
+ * mlpack is free software; you may redistribute it and/or modify it under the
+ * terms of the 3-clause BSD license.  You should have received a copy of the
+ * 3-clause BSD license along with mlpack.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef MLPACK_METHODS_XGBOOST_LOSS_FUNCTIONS_SSE_LOSS_HPP
+#define MLPACK_METHODS_XGBOOST_LOSS_FUNCTIONS_SSE_LOSS_HPP
+
+#include <mlpack/prereqs.hpp>
+
+namespace mlpack {
+namespace ensemble {
+
+/**
+ * The SSE (Sum of Squared Errors) loss is a loss function to measure the
+ * quality of prediction of response values present in the node of each
+ * xgboost tree. It is also a good measure to compare the spread of two
+ * distributions. We will try to minimize this value while training.
+ *
+ * Loss = 1 / 2 * (Observed - Predicted)^2
+ */
+class SSELoss
+{
+ public:
+  // Default constructor---No regularization.
+  SSELoss() : alpha(0), lambda(0) { /* Nothing to do. */}
+
+  SSELoss(const double alpha, const double lambda):
+      alpha(alpha), lambda(lambda)
+  {
+    // Nothing to do.
+  }
+
+  /**
+   * Returns the initial predition for gradient boosting.
+   */
+  template<typename VecType>
+  typename VecType::elem_type InitialPrediction(const VecType& values)
+  {
+    // Sanity check for empty vector.
+    if (values.n_elem == 0)
+      return 0;
+
+    return arma::accu(values) / (typename VecType::elem_type) values.n_elem;
+  }
+
+  /**
+   * Returns the output value for the leaf in the tree.
+   */
+  template<typename MatType, typename WeightVecType>
+  double OutputLeafValue(const MatType& /* input */,
+                         const WeightVecType& /* weights */)
+  {
+    return -ApplyL1(arma::accu(gradients)) / (arma::accu(hessians) + lambda);
+  }
+
+  /**
+   * Calculates the gain from begin to end.
+   *
+   * @param begin The begin index to calculate gain.
+   * @param end The end index to calculate gain.
+   */
+  double Evaluate(const size_t begin, const size_t end)
+  {
+    return std::pow(ApplyL1(arma::accu(gradients.subvec(begin, end))), 2) /
+        (arma::accu(hessians.subvec(begin, end)) + lambda);
+  }
+
+  /**
+   * Calculates the gain of the node before splitting. It also initializes the
+   * gradients and hessians used later for finding split.
+   * UseWeights and weights are ignored here. These are just to make the API
+   * consistent.
+   *
+   * @param input This is a 2D matrix. The first row stores the true observed
+   *    values and the second row stores the prediction at the current step
+   *    of boosting.
+   */
+  template<bool UseWeights, typename MatType, typename WeightVecType>
+  double Evaluate(const MatType& input, const WeightVecType& /* weights */)
+  {
+    // Calculate gradients and hessians.
+    gradients = (input.row(1) - input.row(0)).t();
+    hessians = arma::vec(input.n_cols, arma::fill::ones);
+
+    return std::pow(ApplyL1(arma::accu(gradients)), 2) /
+        (arma::accu(hessians) + lambda);
+  }
+ private:
+  //! The L2 regularization parameter.
+  const double lambda;
+  //! The L1 regularization parameter.
+  const double alpha;
+  //! First order gradients.
+  arma::vec gradients;
+  //! Second order gradients (hessians).
+  arma::vec hessians;
+
+  //! Applies the L1 regularization.
+  double ApplyL1(const double sumGradients)
+  {
+    if (sumGradients > alpha)
+    {
+      return sumGradients - alpha;
+    }
+    else if (sumGradients < - alpha)
+    {
+      return sumGradients + alpha;
+    }
+    
+    return 0;
+  }
+};
+
+} // namespace ensemble
+} // namespace mlpack
+
+#endif

src/mlpack/tests/CMakeLists.txt

@@ -129,6 +129,7 @@ add_executable(mlpack_test
   union_find_test.cpp
   vantage_point_tree_test.cpp
   wgan_test.cpp
+  xgboost_test.cpp
   main_tests/adaboost_test.cpp
   main_tests/approx_kfn_test.cpp
   main_tests/bayesian_linear_regression_test.cpp

src/mlpack/tests/xgboost_test.cpp

@@ -0,0 +1,66 @@
+/**
+ * @file tests/xgboost_test.cpp
+ * @author Rishabh Garg
+ *
+ * Tests for the XGBoost class and related classes.
+ *
+ * mlpack is free software; you may redistribute it and/or modify it under the
+ * terms of the 3-clause BSD license.  You should have received a copy of the
+ * 3-clause BSD license along with mlpack.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#include <mlpack/core.hpp>
+#include <mlpack/methods/xgboost/loss_functions/sse_loss.hpp>
+
+#include "catch.hpp"
+#include "serialization.hpp"
+
+using namespace mlpack;
+using namespace mlpack::ensemble;
+
+/**
+ * Test that the initial prediction is calculated correctly for SSE loss.
+ */
+TEST_CASE("SSEInitialPredictionTest", "[XGBTest]")
+{
+  arma::vec values = {1, 3, 2, 2, 5, 6, 9, 11, 8, 8};
+
+  double initPred = 5.5;
+
+  SSELoss Loss;
+  REQUIRE(Loss.InitialPrediction(values) == initPred);
+}
+
+/**
+ * Test that output leaf value is calculated correctly for SSE Loss.
+ */
+TEST_CASE("SSELeafValueTest", "[XGBTest]")
+{
+  arma::mat input = { { 1,   3,   2,   2, 5, 6, 9,    11, 8,   8 },
+                      { 0.5, 1, 2.5, 1.5, 5, 8, 8, 10.75, 9, 9.5 } };
+  arma::vec weights; // dummy weights not used.
+
+  // Actual output leaf value.
+  double leafValue = -0.075;
+
+  SSELoss Loss;
+  (void) Loss.Evaluate<false>(input, weights);
+
+  REQUIRE(Loss.OutputLeafValue(input, weights) == leafValue);
+}
+
+/**
+ * Test that the gain is computed correctly for SSE Loss.
+ */
+TEST_CASE("SSEGainTest", "[XGBTest]")
+{
+  arma::mat input = { { 1,   3,   2,   2, 5, 6, 9,    11, 8,   8 },
+                      { 0.5, 1, 2.5, 1.5, 5, 8, 8, 10.75, 9, 9.5 } };
+  arma::vec weights; // dummy weights not used.
+
+  // Actual gain value.
+  double gain = 0.05625;
+
+  SSELoss Loss;
+  REQUIRE(Loss.Evaluate<false>(input, weights) == gain);
+}