From 873b35e37f38d1d451a8d790956848213a8bc3b9 Mon Sep 17 00:00:00 2001
From: Rishabh Garg <56191449+RishabhGarg108@users.noreply.github.com>
Date: Wed, 30 Jun 2021 17:42:39 +0530
Subject: [PATCH 01/29] Add SSE Loss for xgboost

---
 .../xgboost/loss_functions/sse_loss.hpp       | 100 ++++++++++++++++++
 1 file changed, 100 insertions(+)
 create mode 100644 src/mlpack/methods/xgboost/loss_functions/sse_loss.hpp

diff --git a/src/mlpack/methods/xgboost/loss_functions/sse_loss.hpp b/src/mlpack/methods/xgboost/loss_functions/sse_loss.hpp
new file mode 100644
index 0000000000..f702052537
--- /dev/null
+++ b/src/mlpack/methods/xgboost/loss_functions/sse_loss.hpp
@@ -0,0 +1,100 @@
+/**
+ * @file methods/xgboost/loss_functions/sse_loss.hpp
+ * @author Rishabh Garg
+ *
+ * The sum of squared error loss class, which is a loss funtion for gradient
+ * xgboost based decision trees.
+ *
+ * mlpack is free software; you may redistribute it and/or modify it under the
+ * terms of the 3-clause BSD license.  You should have received a copy of the
+ * 3-clause BSD license along with mlpack.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef MLPACK_METHODS_DECISION_TREE_SSE_LOSS_HPP
+#define MLPACK_METHODS_DECISION_TREE_SSE_LOSS_HPP
+
+#include <mlpack/prereqs.hpp>
+
+namespace mlpack {
+namespace ensemble {
+
+/**
+ * The SSE (Sum of Squared Errors) loss is a loss function to measure the
+ * quality of prediction of response values present in the node of each
+ * xgboost tree. It is also a good measure to compare the spread of two
+ * distributions. We will try to minimize this value while training.
+ *
+ * Loss = 1 / 2 * (Observed - Predicted)^2
+ */
+class SSELoss
+{
+ public:
+  /**
+   * Returns the initial predition for gradient boosting.
+   */
+  template<typename eT>
+  eT InitialPrediction(const arma::Row<eT>& values)
+  {
+    return arma::accu(values) / (eT) values.n_elem;
+  }
+
+  /**
+   * Returns the first order gradient of the loss function with respect to the
+   * values.
+   *
+   * This is primarily used in calculating the residuals and split gain for the
+   * gradient boosted trees.
+   *
+   * @tparam T The type of input data. This can be both a vector or a scalar.
+   * @param observed The true observed values.
+   * @param values The values with respect to which the gradient will be
+   *     calculated.
+   */
+  template<typename T>
+  T Gradients(const T& observed, const T& values)
+  {
+    return - (observed - values);
+  }
+
+  /**
+   * Returns the second order gradient of the loss function with respect to the
+   * values. This is used only for scalars.
+   */
+  template<typename T>
+  T Hessians(const T& /* observed */, const T& /* values */)
+  {
+    return (T) 1;
+  }
+
+  /**
+   * Returns the second order gradient of the loss function with respect to the
+   * values. This is used only for vectors.
+   */
+  template<typename VecType,
+           class = std::enable_if_t<arma::is_Col<VecType>::value ||
+               arma::is_Row<VecType>::value>>
+  VecType Hessians(const VecType& /* observed */, const VecType& values)
+  {
+    VecType h(values.n_elem, 1);
+    return h;
+  }
+
+  /**
+   * Returns the pseudo residuals of the predictions.
+   * This is equal to the negative gradient of the loss function with respect
+   * to the predicted values f.
+   *
+   * @param observed The true observed values.
+   * @param f The prediction at the current step of boosting.
+   */
+  template<typename VecType>
+  VecType Residuals(const VecType& observed, const VecType& f)
+  {
+    return - Gradients(observed, f);
+  }
+}
+
+} // namespace ensemble
+} // namespace mlpack
+
+#endif

From e9cba5a179a82759238aa47782a72feaacb6da45 Mon Sep 17 00:00:00 2001
From: Rishabh Garg <56191449+RishabhGarg108@users.noreply.github.com>
Date: Wed, 30 Jun 2021 17:43:59 +0530
Subject: [PATCH 02/29] Fix name of header guards

---
 src/mlpack/methods/xgboost/loss_functions/sse_loss.hpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/mlpack/methods/xgboost/loss_functions/sse_loss.hpp b/src/mlpack/methods/xgboost/loss_functions/sse_loss.hpp
index f702052537..64b965fbc9 100644
--- a/src/mlpack/methods/xgboost/loss_functions/sse_loss.hpp
+++ b/src/mlpack/methods/xgboost/loss_functions/sse_loss.hpp
@@ -10,8 +10,8 @@
  * 3-clause BSD license along with mlpack.  If not, see
  * http://www.opensource.org/licenses/BSD-3-Clause for more information.
  */
-#ifndef MLPACK_METHODS_DECISION_TREE_SSE_LOSS_HPP
-#define MLPACK_METHODS_DECISION_TREE_SSE_LOSS_HPP
+#ifndef MLPACK_METHODS_XGBOOST_LOSS_FUNCTIONS_SSE_LOSS_HPP
+#define MLPACK_METHODS_XGBOOST_LOSS_FUNCTIONS_SSE_LOSS_HPP
 
 #include <mlpack/prereqs.hpp>
 

From e4b073f3ee7f876580f92e2859b17262a7802dbe Mon Sep 17 00:00:00 2001
From: Rishabh Garg <56191449+RishabhGarg108@users.noreply.github.com>
Date: Wed, 30 Jun 2021 20:46:12 +0530
Subject: [PATCH 03/29] Update template for InitialPrediction so that it can
 take any armadillo vector instead of just arma::Row

---
 src/mlpack/methods/xgboost/loss_functions/sse_loss.hpp | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/mlpack/methods/xgboost/loss_functions/sse_loss.hpp b/src/mlpack/methods/xgboost/loss_functions/sse_loss.hpp
index 64b965fbc9..7ae8b8379b 100644
--- a/src/mlpack/methods/xgboost/loss_functions/sse_loss.hpp
+++ b/src/mlpack/methods/xgboost/loss_functions/sse_loss.hpp
@@ -32,10 +32,10 @@ class SSELoss
   /**
    * Returns the initial predition for gradient boosting.
    */
-  template<typename eT>
-  eT InitialPrediction(const arma::Row<eT>& values)
+  template<typename VecType>
+  VecType::elem_type InitialPrediction(const VecType& values)
   {
-    return arma::accu(values) / (eT) values.n_elem;
+    return arma::accu(values) / (VecType::elem_type) values.n_elem;
   }
 
   /**

From b80ecf2e958d472a0b507632a8160683296a8ba4 Mon Sep 17 00:00:00 2001
From: Rishabh Garg <56191449+RishabhGarg108@users.noreply.github.com>
Date: Wed, 30 Jun 2021 20:50:40 +0530
Subject: [PATCH 04/29] Add missing typename

---
 src/mlpack/methods/xgboost/loss_functions/sse_loss.hpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/mlpack/methods/xgboost/loss_functions/sse_loss.hpp b/src/mlpack/methods/xgboost/loss_functions/sse_loss.hpp
index 7ae8b8379b..a06c7123b8 100644
--- a/src/mlpack/methods/xgboost/loss_functions/sse_loss.hpp
+++ b/src/mlpack/methods/xgboost/loss_functions/sse_loss.hpp
@@ -33,9 +33,9 @@ class SSELoss
    * Returns the initial predition for gradient boosting.
    */
   template<typename VecType>
-  VecType::elem_type InitialPrediction(const VecType& values)
+  typename VecType::elem_type InitialPrediction(const VecType& values)
   {
-    return arma::accu(values) / (VecType::elem_type) values.n_elem;
+    return arma::accu(values) / (typename VecType::elem_type) values.n_elem;
   }
 
   /**

From 61e88e6b3c914807bf23da8e39d239f9102364dd Mon Sep 17 00:00:00 2001
From: Rishabh Garg <56191449+RishabhGarg108@users.noreply.github.com>
Date: Wed, 30 Jun 2021 23:00:50 +0530
Subject: [PATCH 05/29] Add OutputValue and SimilarityScore methods

---
 .../xgboost/loss_functions/sse_loss.hpp       | 27 +++++++++++++++++++
 1 file changed, 27 insertions(+)

diff --git a/src/mlpack/methods/xgboost/loss_functions/sse_loss.hpp b/src/mlpack/methods/xgboost/loss_functions/sse_loss.hpp
index a06c7123b8..fc4f941620 100644
--- a/src/mlpack/methods/xgboost/loss_functions/sse_loss.hpp
+++ b/src/mlpack/methods/xgboost/loss_functions/sse_loss.hpp
@@ -92,6 +92,33 @@ class SSELoss
   {
     return - Gradients(observed, f);
   }
+
+  /**
+   * Returns the output value for the leaf in the tree.
+   */
+  template<typename VecType>
+  typename VecType::elem_type
+  OutputValue(const VecType& gradients, const VecType& hessians,
+              const double lambda)
+  {
+    return - arma::accu(gradients) / (arma::accu(hessians) + lambda);
+  }
+
+  /**
+   * Calculates the similarity score for evaluating the splits.
+   */
+  template<typename VecType>
+  double SimilarityScore(const VecType& observed, const VecType& residuals,
+      const size_t begin, const size_t end, const double lambda)
+  {
+    VecType gradients = Gradients(observed.subvec(begin, end),
+        residuals.subvec(begin, end));
+    VecType hessians = Hessians(observed.subvec(begin, end),
+        residuals.subvec(begin, end));
+
+    return std::pow(arma::accu(gradients), 2) /
+        (arma::accu(hessians) + lambda);
+  }
 }
 
 } // namespace ensemble

From 093c4433f13d54718e893ee3fc81e3ec0f1bdc5f Mon Sep 17 00:00:00 2001
From: Rishabh Garg <56191449+RishabhGarg108@users.noreply.github.com>
Date: Wed, 30 Jun 2021 23:02:53 +0530
Subject: [PATCH 06/29] Fixed filling the hessian vector with 1

---
 src/mlpack/methods/xgboost/loss_functions/sse_loss.hpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/mlpack/methods/xgboost/loss_functions/sse_loss.hpp b/src/mlpack/methods/xgboost/loss_functions/sse_loss.hpp
index fc4f941620..c754e9e7eb 100644
--- a/src/mlpack/methods/xgboost/loss_functions/sse_loss.hpp
+++ b/src/mlpack/methods/xgboost/loss_functions/sse_loss.hpp
@@ -75,7 +75,7 @@ class SSELoss
                arma::is_Row<VecType>::value>>
   VecType Hessians(const VecType& /* observed */, const VecType& values)
   {
-    VecType h(values.n_elem, 1);
+    VecType h(values.n_elem, arma::fill::ones);
     return h;
   }
 

From 72e617bca913076dbbe0948aa1b5d2a7b5e0d448 Mon Sep 17 00:00:00 2001
From: Rishabh Garg <56191449+RishabhGarg108@users.noreply.github.com>
Date: Thu, 1 Jul 2021 21:27:57 +0530
Subject: [PATCH 07/29] Missed semicolon

---
 src/mlpack/methods/xgboost/loss_functions/sse_loss.hpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/mlpack/methods/xgboost/loss_functions/sse_loss.hpp b/src/mlpack/methods/xgboost/loss_functions/sse_loss.hpp
index c754e9e7eb..bae167fffc 100644
--- a/src/mlpack/methods/xgboost/loss_functions/sse_loss.hpp
+++ b/src/mlpack/methods/xgboost/loss_functions/sse_loss.hpp
@@ -119,7 +119,7 @@ class SSELoss
     return std::pow(arma::accu(gradients), 2) /
         (arma::accu(hessians) + lambda);
   }
-}
+};
 
 } // namespace ensemble
 } // namespace mlpack

From f974f4e95ea48b7d1aebe6d5192e1a60f6b8b40f Mon Sep 17 00:00:00 2001
From: Rishabh Garg <56191449+RishabhGarg108@users.noreply.github.com>
Date: Thu, 1 Jul 2021 21:59:16 +0530
Subject: [PATCH 08/29] Start compiling (hopefully)

---
 src/mlpack/methods/xgboost/CMakeLists.txt          | 13 +++++++++++++
 .../methods/xgboost/loss_functions/CMakeLists.txt  | 14 ++++++++++++++
 2 files changed, 27 insertions(+)
 create mode 100644 src/mlpack/methods/xgboost/CMakeLists.txt
 create mode 100644 src/mlpack/methods/xgboost/loss_functions/CMakeLists.txt

diff --git a/src/mlpack/methods/xgboost/CMakeLists.txt b/src/mlpack/methods/xgboost/CMakeLists.txt
new file mode 100644
index 0000000000..63339c806e
--- /dev/null
+++ b/src/mlpack/methods/xgboost/CMakeLists.txt
@@ -0,0 +1,13 @@
+# Define the files we need to compile.
+# Anything not in this list will not be compiled into mlpack.
+set(SOURCES
+)
+
+# Add directory name to sources.
+set(DIR_SRCS)
+foreach(file ${SOURCES})
+  set(DIR_SRCS ${DIR_SRCS} ${CMAKE_CURRENT_SOURCE_DIR}/${file})
+endforeach()
+# Append sources (with directory name) to list of all mlpack sources (used at
+# the parent scope).
+set(MLPACK_SRCS ${MLPACK_SRCS} ${DIR_SRCS} PARENT_SCOPE)
diff --git a/src/mlpack/methods/xgboost/loss_functions/CMakeLists.txt b/src/mlpack/methods/xgboost/loss_functions/CMakeLists.txt
new file mode 100644
index 0000000000..30ffd3867e
--- /dev/null
+++ b/src/mlpack/methods/xgboost/loss_functions/CMakeLists.txt
@@ -0,0 +1,14 @@
+# Define the files we need to compile.
+# Anything not in this list will not be compiled into mlpack.
+set(SOURCES
+  sse_loss.hpp
+)
+
+# Add directory name to sources.
+set(DIR_SRCS)
+foreach(file ${SOURCES})
+  set(DIR_SRCS ${DIR_SRCS} ${CMAKE_CURRENT_SOURCE_DIR}/${file})
+endforeach()
+# Append sources (with directory name) to list of all mlpack sources (used at
+# the parent scope).
+set(MLPACK_SRCS ${MLPACK_SRCS} ${DIR_SRCS} PARENT_SCOPE)

From 44f76f0c96d98a05643badf36a05508bc83ce0ff Mon Sep 17 00:00:00 2001
From: Rishabh Garg <56191449+RishabhGarg108@users.noreply.github.com>
Date: Thu, 1 Jul 2021 21:59:34 +0530
Subject: [PATCH 09/29] write first test

---
 src/mlpack/tests/CMakeLists.txt   |  1 +
 src/mlpack/tests/xgboost_test.cpp | 32 +++++++++++++++++++++++++++++++
 2 files changed, 33 insertions(+)
 create mode 100644 src/mlpack/tests/xgboost_test.cpp

diff --git a/src/mlpack/tests/CMakeLists.txt b/src/mlpack/tests/CMakeLists.txt
index 2119879ab7..3bf2cc0532 100644
--- a/src/mlpack/tests/CMakeLists.txt
+++ b/src/mlpack/tests/CMakeLists.txt
@@ -128,6 +128,7 @@ add_executable(mlpack_test
   union_find_test.cpp
   vantage_point_tree_test.cpp
   wgan_test.cpp
+  xgboost_test.cpp
   main_tests/adaboost_test.cpp
   main_tests/approx_kfn_test.cpp
   main_tests/bayesian_linear_regression_test.cpp
diff --git a/src/mlpack/tests/xgboost_test.cpp b/src/mlpack/tests/xgboost_test.cpp
new file mode 100644
index 0000000000..c34b973399
--- /dev/null
+++ b/src/mlpack/tests/xgboost_test.cpp
@@ -0,0 +1,32 @@
+/**
+ * @file tests/xgboost_test.cpp
+ * @author Rishabh Garg
+ *
+ * Tests for the XGBoost class and related classes.
+ *
+ * mlpack is free software; you may redistribute it and/or modify it under the
+ * terms of the 3-clause BSD license.  You should have received a copy of the
+ * 3-clause BSD license along with mlpack.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#include <mlpack/core.hpp>
+#include <mlpack/methods/xgboost/loss_functions/sse_loss.hpp>
+
+#include "catch.hpp"
+#include "serialization.hpp"
+
+using namespace mlpack;
+using namespace mlpack::ensemble;
+
+/**
+ * Test that the initial prediction is calculated correctly for SSE loss.
+ */
+TEST_CASE("SSEInitialPredictionTest", "[XGBTest]")
+{
+  arma::vec values = {1, 3, 2, 2, 5, 6, 9, 11, 8, 8};
+
+  double initPred = 5.5;
+
+  SSELoss Loss;
+  REQUIRE(Loss.InitialPrediction(values) == initPred);
+}

From b81c84c09389fc1d7c6f8d398c09f6b27c785f0a Mon Sep 17 00:00:00 2001
From: Rishabh Garg <56191449+RishabhGarg108@users.noreply.github.com>
Date: Sat, 3 Jul 2021 16:22:21 +0530
Subject: [PATCH 10/29] moved sse_loss.hpp to parent directory's CMakeLists.txt

---
 src/mlpack/methods/xgboost/CMakeLists.txt          |  1 +
 .../methods/xgboost/loss_functions/CMakeLists.txt  | 14 --------------
 2 files changed, 1 insertion(+), 14 deletions(-)
 delete mode 100644 src/mlpack/methods/xgboost/loss_functions/CMakeLists.txt

diff --git a/src/mlpack/methods/xgboost/CMakeLists.txt b/src/mlpack/methods/xgboost/CMakeLists.txt
index 63339c806e..37be5ee1d1 100644
--- a/src/mlpack/methods/xgboost/CMakeLists.txt
+++ b/src/mlpack/methods/xgboost/CMakeLists.txt
@@ -1,6 +1,7 @@
 # Define the files we need to compile.
 # Anything not in this list will not be compiled into mlpack.
 set(SOURCES
+  loss_functions/sse_loss.hpp
 )
 
 # Add directory name to sources.
diff --git a/src/mlpack/methods/xgboost/loss_functions/CMakeLists.txt b/src/mlpack/methods/xgboost/loss_functions/CMakeLists.txt
deleted file mode 100644
index 30ffd3867e..0000000000
--- a/src/mlpack/methods/xgboost/loss_functions/CMakeLists.txt
+++ /dev/null
@@ -1,14 +0,0 @@
-# Define the files we need to compile.
-# Anything not in this list will not be compiled into mlpack.
-set(SOURCES
-  sse_loss.hpp
-)
-
-# Add directory name to sources.
-set(DIR_SRCS)
-foreach(file ${SOURCES})
-  set(DIR_SRCS ${DIR_SRCS} ${CMAKE_CURRENT_SOURCE_DIR}/${file})
-endforeach()
-# Append sources (with directory name) to list of all mlpack sources (used at
-# the parent scope).
-set(MLPACK_SRCS ${MLPACK_SRCS} ${DIR_SRCS} PARENT_SCOPE)

From d082f9a866fe74f992e9ec7d8612118cc9274907 Mon Sep 17 00:00:00 2001
From: Rishabh Garg <56191449+RishabhGarg108@users.noreply.github.com>
Date: Sat, 3 Jul 2021 16:56:52 +0530
Subject: [PATCH 11/29] Change enable_if statement to use cleaner syntax

---
 src/mlpack/methods/xgboost/loss_functions/sse_loss.hpp | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/src/mlpack/methods/xgboost/loss_functions/sse_loss.hpp b/src/mlpack/methods/xgboost/loss_functions/sse_loss.hpp
index bae167fffc..50e016524c 100644
--- a/src/mlpack/methods/xgboost/loss_functions/sse_loss.hpp
+++ b/src/mlpack/methods/xgboost/loss_functions/sse_loss.hpp
@@ -71,8 +71,7 @@ class SSELoss
    * values. This is used only for vectors.
    */
   template<typename VecType,
-           class = std::enable_if_t<arma::is_Col<VecType>::value ||
-               arma::is_Row<VecType>::value>>
+           class = std::enable_if_t<VecType::is_col || VecType::is_row>>
   VecType Hessians(const VecType& /* observed */, const VecType& values)
   {
     VecType h(values.n_elem, arma::fill::ones);

From de9938ee51fdda645a8c2d0d39422077dc0be5e0 Mon Sep 17 00:00:00 2001
From: Rishabh Garg <56191449+RishabhGarg108@users.noreply.github.com>
Date: Mon, 12 Jul 2021 19:57:14 +0530
Subject: [PATCH 12/29] Removing unrequired overload of Hessian()

---
 src/mlpack/methods/xgboost/loss_functions/sse_loss.hpp | 10 ----------
 1 file changed, 10 deletions(-)

diff --git a/src/mlpack/methods/xgboost/loss_functions/sse_loss.hpp b/src/mlpack/methods/xgboost/loss_functions/sse_loss.hpp
index 50e016524c..a7ffe559a1 100644
--- a/src/mlpack/methods/xgboost/loss_functions/sse_loss.hpp
+++ b/src/mlpack/methods/xgboost/loss_functions/sse_loss.hpp
@@ -56,16 +56,6 @@ class SSELoss
     return - (observed - values);
   }
 
-  /**
-   * Returns the second order gradient of the loss function with respect to the
-   * values. This is used only for scalars.
-   */
-  template<typename T>
-  T Hessians(const T& /* observed */, const T& /* values */)
-  {
-    return (T) 1;
-  }
-
   /**
    * Returns the second order gradient of the loss function with respect to the
    * values. This is used only for vectors.

From 25f8f47cb5063120259f390dfb7b4b964eb030dc Mon Sep 17 00:00:00 2001
From: Rishabh Garg <56191449+RishabhGarg108@users.noreply.github.com>
Date: Mon, 12 Jul 2021 19:57:32 +0530
Subject: [PATCH 13/29] Add more tests

---
 src/mlpack/tests/xgboost_test.cpp | 77 +++++++++++++++++++++++++++++++
 1 file changed, 77 insertions(+)

diff --git a/src/mlpack/tests/xgboost_test.cpp b/src/mlpack/tests/xgboost_test.cpp
index c34b973399..31f8754569 100644
--- a/src/mlpack/tests/xgboost_test.cpp
+++ b/src/mlpack/tests/xgboost_test.cpp
@@ -30,3 +30,80 @@ TEST_CASE("SSEInitialPredictionTest", "[XGBTest]")
   SSELoss Loss;
   REQUIRE(Loss.InitialPrediction(values) == initPred);
 }
+
+/**
+ * Test that gradients are calculated correctly for SSE Loss.
+ */
+TEST_CASE("SSEGradientsTest", "[XGBTest]")
+{
+  arma::vec observed = {1, 3, 2, 2, 5, 6, 9, 11, 8, 8};
+  arma::vec predicted = {0.5, 1, 2.5, 1.5, 5, 8, 8, 10.75, 9, 9.5};
+
+  // Actual gradients.
+  arma::vec gradients = {-0.5, -2, 0.5, -0.5, 0, 2, -1, -0.25, 1, 1.5};
+
+  SSELoss Loss;
+  // Calculated gradients.
+  arma::vec calculatedGradients = Loss.Gradients(observed, predicted);
+
+  for (int i = 0; i < 10; i++)
+    REQUIRE(calculatedGradients[i] == gradients[i]);
+}
+
+/**
+ * Test that hessians are calculated correctly for SSE Loss.
+ */
+TEST_CASE("SSEHessiansTest", "[XGBTest]")
+{
+  arma::vec observed = {1, 3, 2, 2, 5, 6, 9, 11, 8, 8};
+  arma::vec predicted = {0.5, 1, 2.5, 1.5, 5, 8, 8, 10.75, 9, 9.5};
+
+  // Actual hessians.
+  arma::vec hessians = {1, 1, 1, 1, 1, 1, 1, 1, 1, 1};
+
+  SSELoss Loss;
+  // Calculated hessians.
+  arma::vec calculatedHessians = Loss.Hessians(observed, predicted);
+
+  for (int i = 0; i < 10; i++)
+    REQUIRE(calculatedHessians[i] == hessians[i]);
+}
+
+/**
+ * Test that residuals are calculated correctly for SSE Loss.
+ */
+TEST_CASE("SSEResidualsTest", "[XGBTest]")
+{
+  arma::vec observed = {1, 3, 2, 2, 5, 6, 9, 11, 8, 8};
+  arma::vec predicted = {0.5, 1, 2.5, 1.5, 5, 8, 8, 10.75, 9, 9.5};
+
+  // Actual residuals.
+  arma::vec residuals = {0.5, 2, -0.5, 0.5, 0, -2, 1, 0.25, -1, -1.5};
+
+  SSELoss Loss;
+  // Calculated residuals.
+  arma::vec calculatedResiduals = Loss.Residuals(observed, predicted);
+
+  for (int i = 0; i < 10; i++)
+    REQUIRE(calculatedResiduals[i] == residuals[i]);
+}
+
+/**
+ * Test that output value is calculated correctly for SSE Loss.
+ */
+TEST_CASE("SSEOutputValueTest", "[XGBTest]")
+{
+  arma::vec observed = {1, 3, 2, 2, 5, 6, 9, 11, 8, 8};
+  arma::vec predicted = {0.5, 1, 2.5, 1.5, 5, 8, 8, 10.75, 9, 9.5};
+
+  // Actual outut value.
+  double outputValue = -0.075;
+
+  SSELoss Loss;
+  // Calculating gradients and hessians for input to OutputValue().
+  arma::vec gradients = Loss.Gradients(observed, predicted);
+  arma::vec hessians = Loss.Hessians(observed, predicted);
+
+  // Lambda = 0;
+  REQUIRE(Loss.OutputValue(gradients, hessians, 0) == outputValue);
+}

From c3d65e857123870f74fece6747ec5dfedf2910ea Mon Sep 17 00:00:00 2001
From: Rishabh Garg <56191449+RishabhGarg108@users.noreply.github.com>
Date: Mon, 12 Jul 2021 21:36:17 +0530
Subject: [PATCH 14/29] Simplified methods by removing unnecessary indirections
 of functions

---
 src/mlpack/methods/xgboost/loss_functions/sse_loss.hpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/mlpack/methods/xgboost/loss_functions/sse_loss.hpp b/src/mlpack/methods/xgboost/loss_functions/sse_loss.hpp
index a7ffe559a1..ef86cebf35 100644
--- a/src/mlpack/methods/xgboost/loss_functions/sse_loss.hpp
+++ b/src/mlpack/methods/xgboost/loss_functions/sse_loss.hpp
@@ -53,7 +53,7 @@ class SSELoss
   template<typename T>
   T Gradients(const T& observed, const T& values)
   {
-    return - (observed - values);
+    return values - observed;
   }
 
   /**
@@ -79,7 +79,7 @@ class SSELoss
   template<typename VecType>
   VecType Residuals(const VecType& observed, const VecType& f)
   {
-    return - Gradients(observed, f);
+    return observed - f;
   }
 
   /**

From 154768851a848d5ceb6927d96c7f37346ddaaa69 Mon Sep 17 00:00:00 2001
From: Rishabh Garg <56191449+RishabhGarg108@users.noreply.github.com>
Date: Thu, 15 Jul 2021 07:19:53 +0530
Subject: [PATCH 15/29] Update
 src/mlpack/methods/xgboost/loss_functions/sse_loss.hpp

Co-authored-by: Ryan Curtin <ryan@ratml.org>
---
 src/mlpack/methods/xgboost/loss_functions/sse_loss.hpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/mlpack/methods/xgboost/loss_functions/sse_loss.hpp b/src/mlpack/methods/xgboost/loss_functions/sse_loss.hpp
index ef86cebf35..a5ede0ab93 100644
--- a/src/mlpack/methods/xgboost/loss_functions/sse_loss.hpp
+++ b/src/mlpack/methods/xgboost/loss_functions/sse_loss.hpp
@@ -90,7 +90,7 @@ class SSELoss
   OutputValue(const VecType& gradients, const VecType& hessians,
               const double lambda)
   {
-    return - arma::accu(gradients) / (arma::accu(hessians) + lambda);
+    return -arma::accu(gradients) / (arma::accu(hessians) + lambda);
   }
 
   /**

From a1edb432b894526dac4530af45bdfb74a9f3fe45 Mon Sep 17 00:00:00 2001
From: Rishabh Garg <56191449+RishabhGarg108@users.noreply.github.com>
Date: Thu, 15 Jul 2021 06:57:21 +0530
Subject: [PATCH 16/29] Removed unnecessary SFINAE check

---
 src/mlpack/methods/xgboost/loss_functions/sse_loss.hpp | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/src/mlpack/methods/xgboost/loss_functions/sse_loss.hpp b/src/mlpack/methods/xgboost/loss_functions/sse_loss.hpp
index a5ede0ab93..7c6df621f4 100644
--- a/src/mlpack/methods/xgboost/loss_functions/sse_loss.hpp
+++ b/src/mlpack/methods/xgboost/loss_functions/sse_loss.hpp
@@ -58,10 +58,9 @@ class SSELoss
 
   /**
    * Returns the second order gradient of the loss function with respect to the
-   * values. This is used only for vectors.
+   * values.
    */
-  template<typename VecType,
-           class = std::enable_if_t<VecType::is_col || VecType::is_row>>
+  template<typename VecType>
   VecType Hessians(const VecType& /* observed */, const VecType& values)
   {
     VecType h(values.n_elem, arma::fill::ones);

From 6a187c41cd63b21dd952ff6cadc8080e7e445dcb Mon Sep 17 00:00:00 2001
From: Rishabh Garg <56191449+RishabhGarg108@users.noreply.github.com>
Date: Thu, 15 Jul 2021 06:59:50 +0530
Subject: [PATCH 17/29] Add sanity check for empty vector

---
 src/mlpack/methods/xgboost/loss_functions/sse_loss.hpp | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/src/mlpack/methods/xgboost/loss_functions/sse_loss.hpp b/src/mlpack/methods/xgboost/loss_functions/sse_loss.hpp
index 7c6df621f4..1ff28b75d2 100644
--- a/src/mlpack/methods/xgboost/loss_functions/sse_loss.hpp
+++ b/src/mlpack/methods/xgboost/loss_functions/sse_loss.hpp
@@ -35,6 +35,10 @@ class SSELoss
   template<typename VecType>
   typename VecType::elem_type InitialPrediction(const VecType& values)
   {
+    // Sanity check for empty vector.
+    if (values.n_elem == 0)
+      return 0;
+
     return arma::accu(values) / (typename VecType::elem_type) values.n_elem;
   }
 

From a3769b67999b9016f60f6c4e3fcb987aae66a160 Mon Sep 17 00:00:00 2001
From: Rishabh Garg <56191449+RishabhGarg108@users.noreply.github.com>
Date: Thu, 15 Jul 2021 07:17:20 +0530
Subject: [PATCH 18/29] Add L1 and L2 parameter to SSELoss

---
 .../methods/xgboost/loss_functions/sse_loss.hpp  | 16 +++++++++++++---
 1 file changed, 13 insertions(+), 3 deletions(-)

diff --git a/src/mlpack/methods/xgboost/loss_functions/sse_loss.hpp b/src/mlpack/methods/xgboost/loss_functions/sse_loss.hpp
index 1ff28b75d2..b5f8765b8c 100644
--- a/src/mlpack/methods/xgboost/loss_functions/sse_loss.hpp
+++ b/src/mlpack/methods/xgboost/loss_functions/sse_loss.hpp
@@ -29,6 +29,12 @@ namespace ensemble {
 class SSELoss
 {
  public:
+  SSELoss(const double alpha, const double lambda):
+      alpha(alpha), lambda(lambda)
+  {
+    // Nothing to do.
+  }
+
   /**
    * Returns the initial predition for gradient boosting.
    */
@@ -90,8 +96,7 @@ class SSELoss
    */
   template<typename VecType>
   typename VecType::elem_type
-  OutputValue(const VecType& gradients, const VecType& hessians,
-              const double lambda)
+  OutputValue(const VecType& gradients, const VecType& hessians)
   {
     return -arma::accu(gradients) / (arma::accu(hessians) + lambda);
   }
@@ -101,7 +106,7 @@ class SSELoss
    */
   template<typename VecType>
   double SimilarityScore(const VecType& observed, const VecType& residuals,
-      const size_t begin, const size_t end, const double lambda)
+      const size_t begin, const size_t end)
   {
     VecType gradients = Gradients(observed.subvec(begin, end),
         residuals.subvec(begin, end));
@@ -111,6 +116,11 @@ class SSELoss
     return std::pow(arma::accu(gradients), 2) /
         (arma::accu(hessians) + lambda);
   }
+ private:
+  //! The L2 regularization parameter.
+  const double lambda;
+  //! The L1 regularization parameter.
+  const double alpha;
 };
 
 } // namespace ensemble

From 537e3ca44791c17cab30cfda6f1ee6ba6db9d34a Mon Sep 17 00:00:00 2001
From: Rishabh Garg <56191449+RishabhGarg108@users.noreply.github.com>
Date: Thu, 15 Jul 2021 07:27:32 +0530
Subject: [PATCH 19/29] Add default ctor and fix one test

---
 src/mlpack/methods/xgboost/loss_functions/sse_loss.hpp | 3 +++
 src/mlpack/tests/xgboost_test.cpp                      | 2 +-
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/mlpack/methods/xgboost/loss_functions/sse_loss.hpp b/src/mlpack/methods/xgboost/loss_functions/sse_loss.hpp
index b5f8765b8c..62f27a66da 100644
--- a/src/mlpack/methods/xgboost/loss_functions/sse_loss.hpp
+++ b/src/mlpack/methods/xgboost/loss_functions/sse_loss.hpp
@@ -29,6 +29,9 @@ namespace ensemble {
 class SSELoss
 {
  public:
+  // Default constructor---No regularization.
+  SSELoss(): alpha(0), lambda(0) { /* Nothing to do. */}
+
   SSELoss(const double alpha, const double lambda):
       alpha(alpha), lambda(lambda)
   {
diff --git a/src/mlpack/tests/xgboost_test.cpp b/src/mlpack/tests/xgboost_test.cpp
index 31f8754569..fa7ab0fff0 100644
--- a/src/mlpack/tests/xgboost_test.cpp
+++ b/src/mlpack/tests/xgboost_test.cpp
@@ -105,5 +105,5 @@ TEST_CASE("SSEOutputValueTest", "[XGBTest]")
   arma::vec hessians = Loss.Hessians(observed, predicted);
 
   // Lambda = 0;
-  REQUIRE(Loss.OutputValue(gradients, hessians, 0) == outputValue);
+  REQUIRE(Loss.OutputValue(gradients, hessians) == outputValue);
 }

From 83cca54140750bcaef0274f0994f466c8fd5f3b8 Mon Sep 17 00:00:00 2001
From: Rishabh Garg <56191449+RishabhGarg108@users.noreply.github.com>
Date: Thu, 15 Jul 2021 07:47:14 +0530
Subject: [PATCH 20/29] Add L1 regularization

---
 .../xgboost/loss_functions/sse_loss.hpp       | 19 +++++++++++++++++--
 1 file changed, 17 insertions(+), 2 deletions(-)

diff --git a/src/mlpack/methods/xgboost/loss_functions/sse_loss.hpp b/src/mlpack/methods/xgboost/loss_functions/sse_loss.hpp
index 62f27a66da..fc4b9f3558 100644
--- a/src/mlpack/methods/xgboost/loss_functions/sse_loss.hpp
+++ b/src/mlpack/methods/xgboost/loss_functions/sse_loss.hpp
@@ -101,7 +101,7 @@ class SSELoss
   typename VecType::elem_type
   OutputValue(const VecType& gradients, const VecType& hessians)
   {
-    return -arma::accu(gradients) / (arma::accu(hessians) + lambda);
+    return -ApplyL1(arma::accu(gradients)) / (arma::accu(hessians) + lambda);
   }
 
   /**
@@ -116,7 +116,7 @@ class SSELoss
     VecType hessians = Hessians(observed.subvec(begin, end),
         residuals.subvec(begin, end));
 
-    return std::pow(arma::accu(gradients), 2) /
+    return std::pow(ApplyL1(arma::accu(gradients)), 2) /
         (arma::accu(hessians) + lambda);
   }
  private:
@@ -124,6 +124,21 @@ class SSELoss
   const double lambda;
   //! The L1 regularization parameter.
   const double alpha;
+
+  //! Applies the L1 regularization.
+  double ApplyL1(const double sumGradients)
+  {
+    if (sumGradients > alpha)
+    {
+      return sumGradients - alpha;
+    }
+    else if (sumGradients < - alpha)
+    {
+      return sumGradients + alpha;
+    }
+    
+    return 0;
+  }
 };
 
 } // namespace ensemble

From a5fffe3c0bd629e95dd211678e28a6bbb9439ddd Mon Sep 17 00:00:00 2001
From: Rishabh Garg <56191449+RishabhGarg108@users.noreply.github.com>
Date: Thu, 15 Jul 2021 22:07:00 +0530
Subject: [PATCH 21/29] OutputValue => OutputLeafValue

---
 .../methods/xgboost/loss_functions/sse_loss.hpp     |  2 +-
 src/mlpack/tests/xgboost_test.cpp                   | 13 ++++++-------
 2 files changed, 7 insertions(+), 8 deletions(-)

diff --git a/src/mlpack/methods/xgboost/loss_functions/sse_loss.hpp b/src/mlpack/methods/xgboost/loss_functions/sse_loss.hpp
index fc4b9f3558..8b447371b3 100644
--- a/src/mlpack/methods/xgboost/loss_functions/sse_loss.hpp
+++ b/src/mlpack/methods/xgboost/loss_functions/sse_loss.hpp
@@ -99,7 +99,7 @@ class SSELoss
    */
   template<typename VecType>
   typename VecType::elem_type
-  OutputValue(const VecType& gradients, const VecType& hessians)
+  OutputLeafValue(const VecType& gradients, const VecType& hessians)
   {
     return -ApplyL1(arma::accu(gradients)) / (arma::accu(hessians) + lambda);
   }
diff --git a/src/mlpack/tests/xgboost_test.cpp b/src/mlpack/tests/xgboost_test.cpp
index fa7ab0fff0..6a147950fc 100644
--- a/src/mlpack/tests/xgboost_test.cpp
+++ b/src/mlpack/tests/xgboost_test.cpp
@@ -89,21 +89,20 @@ TEST_CASE("SSEResidualsTest", "[XGBTest]")
 }
 
 /**
- * Test that output value is calculated correctly for SSE Loss.
+ * Test that output leaf value is calculated correctly for SSE Loss.
  */
-TEST_CASE("SSEOutputValueTest", "[XGBTest]")
+TEST_CASE("SSELeafValueTest", "[XGBTest]")
 {
   arma::vec observed = {1, 3, 2, 2, 5, 6, 9, 11, 8, 8};
   arma::vec predicted = {0.5, 1, 2.5, 1.5, 5, 8, 8, 10.75, 9, 9.5};
 
-  // Actual outut value.
-  double outputValue = -0.075;
+  // Actual output leaf value.
+  double leafValue = -0.075;
 
   SSELoss Loss;
-  // Calculating gradients and hessians for input to OutputValue().
+  // Calculating gradients and hessians for input to OutputLeafValue().
   arma::vec gradients = Loss.Gradients(observed, predicted);
   arma::vec hessians = Loss.Hessians(observed, predicted);
 
-  // Lambda = 0;
-  REQUIRE(Loss.OutputValue(gradients, hessians) == outputValue);
+  REQUIRE(Loss.OutputLeafValue(gradients, hessians) == leafValue);
 }

From 8b34433e050fd6d70aa4f572566130fcb364e9a2 Mon Sep 17 00:00:00 2001
From: Rishabh Garg <56191449+RishabhGarg108@users.noreply.github.com>
Date: Fri, 16 Jul 2021 00:20:01 +0530
Subject: [PATCH 22/29] Add evaluate method to calculate gain before split

---
 .../xgboost/loss_functions/sse_loss.hpp       | 29 +++++++++++++++++--
 1 file changed, 26 insertions(+), 3 deletions(-)

diff --git a/src/mlpack/methods/xgboost/loss_functions/sse_loss.hpp b/src/mlpack/methods/xgboost/loss_functions/sse_loss.hpp
index 8b447371b3..eff0693951 100644
--- a/src/mlpack/methods/xgboost/loss_functions/sse_loss.hpp
+++ b/src/mlpack/methods/xgboost/loss_functions/sse_loss.hpp
@@ -97,9 +97,7 @@ class SSELoss
   /**
    * Returns the output value for the leaf in the tree.
    */
-  template<typename VecType>
-  typename VecType::elem_type
-  OutputLeafValue(const VecType& gradients, const VecType& hessians)
+  double OutputLeafValue()
   {
     return -ApplyL1(arma::accu(gradients)) / (arma::accu(hessians) + lambda);
   }
@@ -119,11 +117,36 @@ class SSELoss
     return std::pow(ApplyL1(arma::accu(gradients)), 2) /
         (arma::accu(hessians) + lambda);
   }
+
+  /**
+   * Calculates the gain of the node before splitting. It also initializes the
+   * gradients and hessians used later for finding split.
+   * UseWeights and weights are ignored here. These are just to make the API
+   * consistent.
+   *
+   * @param input This is a 2D matrix. The first row stores the true observed
+   *    values and the second row stores the prediction at the current step
+   *    of boosting.
+   */
+  template<bool UseWeights, typename MatType, typename WeightVecType>
+  double Evaluate(const MatType& input, const WeightVecType& /* weights */)
+  {
+    // Calculate gradients and hessians.
+    gradients = input.row(1) - input.row(0);
+    hessians = arma::vec(input.n_cols, arma::fill::ones);
+
+    return std::pow(ApplyL1(arma::accu(gradients)), 2) /
+        (arma::accu(hessians) + lambda);
+  }
  private:
   //! The L2 regularization parameter.
   const double lambda;
   //! The L1 regularization parameter.
   const double alpha;
+  //! First order gradients.
+  arma::vec gradients;
+  //! Second order gradients (hessians).
+  arma::vec hessians;
 
   //! Applies the L1 regularization.
   double ApplyL1(const double sumGradients)

From d8de86b095d6af96e506ae3ee52dc1cd01430e63 Mon Sep 17 00:00:00 2001
From: Rishabh Garg <56191449+RishabhGarg108@users.noreply.github.com>
Date: Fri, 16 Jul 2021 00:33:53 +0530
Subject: [PATCH 23/29] Remove unrequired functions

---
 .../xgboost/loss_functions/sse_loss.hpp       | 56 +--------------
 src/mlpack/tests/xgboost_test.cpp             | 68 ++-----------------
 2 files changed, 8 insertions(+), 116 deletions(-)

diff --git a/src/mlpack/methods/xgboost/loss_functions/sse_loss.hpp b/src/mlpack/methods/xgboost/loss_functions/sse_loss.hpp
index eff0693951..4eba295a37 100644
--- a/src/mlpack/methods/xgboost/loss_functions/sse_loss.hpp
+++ b/src/mlpack/methods/xgboost/loss_functions/sse_loss.hpp
@@ -51,49 +51,6 @@ class SSELoss
     return arma::accu(values) / (typename VecType::elem_type) values.n_elem;
   }
 
-  /**
-   * Returns the first order gradient of the loss function with respect to the
-   * values.
-   *
-   * This is primarily used in calculating the residuals and split gain for the
-   * gradient boosted trees.
-   *
-   * @tparam T The type of input data. This can be both a vector or a scalar.
-   * @param observed The true observed values.
-   * @param values The values with respect to which the gradient will be
-   *     calculated.
-   */
-  template<typename T>
-  T Gradients(const T& observed, const T& values)
-  {
-    return values - observed;
-  }
-
-  /**
-   * Returns the second order gradient of the loss function with respect to the
-   * values.
-   */
-  template<typename VecType>
-  VecType Hessians(const VecType& /* observed */, const VecType& values)
-  {
-    VecType h(values.n_elem, arma::fill::ones);
-    return h;
-  }
-
-  /**
-   * Returns the pseudo residuals of the predictions.
-   * This is equal to the negative gradient of the loss function with respect
-   * to the predicted values f.
-   *
-   * @param observed The true observed values.
-   * @param f The prediction at the current step of boosting.
-   */
-  template<typename VecType>
-  VecType Residuals(const VecType& observed, const VecType& f)
-  {
-    return observed - f;
-  }
-
   /**
    * Returns the output value for the leaf in the tree.
    */
@@ -105,17 +62,10 @@ class SSELoss
   /**
    * Calculates the similarity score for evaluating the splits.
    */
-  template<typename VecType>
-  double SimilarityScore(const VecType& observed, const VecType& residuals,
-      const size_t begin, const size_t end)
+  double SimilarityScore(const size_t begin, const size_t end)
   {
-    VecType gradients = Gradients(observed.subvec(begin, end),
-        residuals.subvec(begin, end));
-    VecType hessians = Hessians(observed.subvec(begin, end),
-        residuals.subvec(begin, end));
-
-    return std::pow(ApplyL1(arma::accu(gradients)), 2) /
-        (arma::accu(hessians) + lambda);
+    return std::pow(ApplyL1(arma::accu(gradients.subvec(begin, end))), 2) /
+        (arma::accu(hessians.subvec(begin, end)) + lambda);
   }
 
   /**
diff --git a/src/mlpack/tests/xgboost_test.cpp b/src/mlpack/tests/xgboost_test.cpp
index 6a147950fc..ed9b900b91 100644
--- a/src/mlpack/tests/xgboost_test.cpp
+++ b/src/mlpack/tests/xgboost_test.cpp
@@ -31,78 +31,20 @@ TEST_CASE("SSEInitialPredictionTest", "[XGBTest]")
   REQUIRE(Loss.InitialPrediction(values) == initPred);
 }
 
-/**
- * Test that gradients are calculated correctly for SSE Loss.
- */
-TEST_CASE("SSEGradientsTest", "[XGBTest]")
-{
-  arma::vec observed = {1, 3, 2, 2, 5, 6, 9, 11, 8, 8};
-  arma::vec predicted = {0.5, 1, 2.5, 1.5, 5, 8, 8, 10.75, 9, 9.5};
-
-  // Actual gradients.
-  arma::vec gradients = {-0.5, -2, 0.5, -0.5, 0, 2, -1, -0.25, 1, 1.5};
-
-  SSELoss Loss;
-  // Calculated gradients.
-  arma::vec calculatedGradients = Loss.Gradients(observed, predicted);
-
-  for (int i = 0; i < 10; i++)
-    REQUIRE(calculatedGradients[i] == gradients[i]);
-}
-
-/**
- * Test that hessians are calculated correctly for SSE Loss.
- */
-TEST_CASE("SSEHessiansTest", "[XGBTest]")
-{
-  arma::vec observed = {1, 3, 2, 2, 5, 6, 9, 11, 8, 8};
-  arma::vec predicted = {0.5, 1, 2.5, 1.5, 5, 8, 8, 10.75, 9, 9.5};
-
-  // Actual hessians.
-  arma::vec hessians = {1, 1, 1, 1, 1, 1, 1, 1, 1, 1};
-
-  SSELoss Loss;
-  // Calculated hessians.
-  arma::vec calculatedHessians = Loss.Hessians(observed, predicted);
-
-  for (int i = 0; i < 10; i++)
-    REQUIRE(calculatedHessians[i] == hessians[i]);
-}
-
-/**
- * Test that residuals are calculated correctly for SSE Loss.
- */
-TEST_CASE("SSEResidualsTest", "[XGBTest]")
-{
-  arma::vec observed = {1, 3, 2, 2, 5, 6, 9, 11, 8, 8};
-  arma::vec predicted = {0.5, 1, 2.5, 1.5, 5, 8, 8, 10.75, 9, 9.5};
-
-  // Actual residuals.
-  arma::vec residuals = {0.5, 2, -0.5, 0.5, 0, -2, 1, 0.25, -1, -1.5};
-
-  SSELoss Loss;
-  // Calculated residuals.
-  arma::vec calculatedResiduals = Loss.Residuals(observed, predicted);
-
-  for (int i = 0; i < 10; i++)
-    REQUIRE(calculatedResiduals[i] == residuals[i]);
-}
-
 /**
  * Test that output leaf value is calculated correctly for SSE Loss.
  */
 TEST_CASE("SSELeafValueTest", "[XGBTest]")
 {
-  arma::vec observed = {1, 3, 2, 2, 5, 6, 9, 11, 8, 8};
-  arma::vec predicted = {0.5, 1, 2.5, 1.5, 5, 8, 8, 10.75, 9, 9.5};
+  arma::mat input = { { 1,   3,   2,   2, 5, 6, 9,    11, 8,   8 },
+                      { 0.5, 1, 2.5, 1.5, 5, 8, 8, 10.75, 9, 9.5 } };
+  arma::vec weights; // dummy weights not used.
 
   // Actual output leaf value.
   double leafValue = -0.075;
 
   SSELoss Loss;
-  // Calculating gradients and hessians for input to OutputLeafValue().
-  arma::vec gradients = Loss.Gradients(observed, predicted);
-  arma::vec hessians = Loss.Hessians(observed, predicted);
+  double gain = Loss.Evaluate<false>(input, weights);
 
-  REQUIRE(Loss.OutputLeafValue(gradients, hessians) == leafValue);
+  REQUIRE(Loss.OutputLeafValue() == leafValue);
 }

From fb8699af16a9375eea769b2521d05cf2e54491d3 Mon Sep 17 00:00:00 2001
From: Rishabh Garg <56191449+RishabhGarg108@users.noreply.github.com>
Date: Fri, 16 Jul 2021 00:36:11 +0530
Subject: [PATCH 24/29] Add method to calculate gain for split

---
 src/mlpack/methods/xgboost/loss_functions/sse_loss.hpp | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/src/mlpack/methods/xgboost/loss_functions/sse_loss.hpp b/src/mlpack/methods/xgboost/loss_functions/sse_loss.hpp
index 4eba295a37..d2f0aa9785 100644
--- a/src/mlpack/methods/xgboost/loss_functions/sse_loss.hpp
+++ b/src/mlpack/methods/xgboost/loss_functions/sse_loss.hpp
@@ -60,9 +60,12 @@ class SSELoss
   }
 
   /**
-   * Calculates the similarity score for evaluating the splits.
+   * Calculates the gain from begin to end.
+   *
+   * @param begin The begin index to calculate gain.
+   * @param end The end index to calculate gain.
    */
-  double SimilarityScore(const size_t begin, const size_t end)
+  double Evaluate(const size_t begin, const size_t end)
   {
     return std::pow(ApplyL1(arma::accu(gradients.subvec(begin, end))), 2) /
         (arma::accu(hessians.subvec(begin, end)) + lambda);

From 1a7c16feb6e5bc3e5eba8249b1294a0fa161d921 Mon Sep 17 00:00:00 2001
From: Rishabh Garg <56191449+RishabhGarg108@users.noreply.github.com>
Date: Fri, 16 Jul 2021 00:40:00 +0530
Subject: [PATCH 25/29] Updates signature of OutputLeafValue to make API
 consistent

---
 src/mlpack/methods/xgboost/loss_functions/sse_loss.hpp | 4 +++-
 src/mlpack/tests/xgboost_test.cpp                      | 2 +-
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/src/mlpack/methods/xgboost/loss_functions/sse_loss.hpp b/src/mlpack/methods/xgboost/loss_functions/sse_loss.hpp
index d2f0aa9785..d777ade217 100644
--- a/src/mlpack/methods/xgboost/loss_functions/sse_loss.hpp
+++ b/src/mlpack/methods/xgboost/loss_functions/sse_loss.hpp
@@ -54,7 +54,9 @@ class SSELoss
   /**
    * Returns the output value for the leaf in the tree.
    */
-  double OutputLeafValue()
+  template<typename MatType, typename WeightVecType>
+  double OutputLeafValue(const MatType& input,
+                         const WeightVecType& /* weights */)
   {
     return -ApplyL1(arma::accu(gradients)) / (arma::accu(hessians) + lambda);
   }
diff --git a/src/mlpack/tests/xgboost_test.cpp b/src/mlpack/tests/xgboost_test.cpp
index ed9b900b91..f3904857ca 100644
--- a/src/mlpack/tests/xgboost_test.cpp
+++ b/src/mlpack/tests/xgboost_test.cpp
@@ -46,5 +46,5 @@ TEST_CASE("SSELeafValueTest", "[XGBTest]")
   SSELoss Loss;
   double gain = Loss.Evaluate<false>(input, weights);
 
-  REQUIRE(Loss.OutputLeafValue() == leafValue);
+  REQUIRE(Loss.OutputLeafValue(input, weights) == leafValue);
 }

From 76d685cd2e19705d4b44d8a455fdfdddf9799ef2 Mon Sep 17 00:00:00 2001
From: Rishabh Garg <56191449+RishabhGarg108@users.noreply.github.com>
Date: Fri, 16 Jul 2021 09:17:57 +0530
Subject: [PATCH 26/29] Fix static analysis error

---
 src/mlpack/methods/xgboost/loss_functions/sse_loss.hpp | 2 +-
 src/mlpack/tests/xgboost_test.cpp                      | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/mlpack/methods/xgboost/loss_functions/sse_loss.hpp b/src/mlpack/methods/xgboost/loss_functions/sse_loss.hpp
index d777ade217..d4ffe22c3c 100644
--- a/src/mlpack/methods/xgboost/loss_functions/sse_loss.hpp
+++ b/src/mlpack/methods/xgboost/loss_functions/sse_loss.hpp
@@ -55,7 +55,7 @@ class SSELoss
    * Returns the output value for the leaf in the tree.
    */
   template<typename MatType, typename WeightVecType>
-  double OutputLeafValue(const MatType& input,
+  double OutputLeafValue(const MatType& /* input */,
                          const WeightVecType& /* weights */)
   {
     return -ApplyL1(arma::accu(gradients)) / (arma::accu(hessians) + lambda);
diff --git a/src/mlpack/tests/xgboost_test.cpp b/src/mlpack/tests/xgboost_test.cpp
index f3904857ca..40c3fdfe11 100644
--- a/src/mlpack/tests/xgboost_test.cpp
+++ b/src/mlpack/tests/xgboost_test.cpp
@@ -44,7 +44,7 @@ TEST_CASE("SSELeafValueTest", "[XGBTest]")
   double leafValue = -0.075;
 
   SSELoss Loss;
-  double gain = Loss.Evaluate<false>(input, weights);
+  double = Loss.Evaluate<false>(input, weights);
 
   REQUIRE(Loss.OutputLeafValue(input, weights) == leafValue);
 }

From 5d68b3481baace687859fe8a7e4025e5eaa5f812 Mon Sep 17 00:00:00 2001
From: Rishabh Garg <56191449+RishabhGarg108@users.noreply.github.com>
Date: Fri, 16 Jul 2021 09:24:54 +0530
Subject: [PATCH 27/29] Add test for gain computation

---
 src/mlpack/tests/xgboost_test.cpp | 18 +++++++++++++++++-
 1 file changed, 17 insertions(+), 1 deletion(-)

diff --git a/src/mlpack/tests/xgboost_test.cpp b/src/mlpack/tests/xgboost_test.cpp
index 40c3fdfe11..7145e34f7e 100644
--- a/src/mlpack/tests/xgboost_test.cpp
+++ b/src/mlpack/tests/xgboost_test.cpp
@@ -44,7 +44,23 @@ TEST_CASE("SSELeafValueTest", "[XGBTest]")
   double leafValue = -0.075;
 
   SSELoss Loss;
-  double = Loss.Evaluate<false>(input, weights);
+  (void) Loss.Evaluate<false>(input, weights);
 
   REQUIRE(Loss.OutputLeafValue(input, weights) == leafValue);
 }
+
+/**
+ * Test that the gain is computed correctly for SSE Loss.
+ */
+TEST_CASE("SSEGainTest", "[XGBTest]")
+{
+  arma::mat input = { { 1,   3,   2,   2, 5, 6, 9,    11, 8,   8 },
+                      { 0.5, 1, 2.5, 1.5, 5, 8, 8, 10.75, 9, 9.5 } };
+  arma::vec weights; // dummy weights not used.
+
+  // Actual gain value.
+  double gain = 0.05625;
+
+  SSELoss Loss;
+  REQUIRE(Loss.Evaluate<false>(input, weights) == gain);
+}

From 8df808bed3c27fa881d87e0783f9a7367280f00f Mon Sep 17 00:00:00 2001
From: Rishabh Garg <56191449+RishabhGarg108@users.noreply.github.com>
Date: Wed, 21 Jul 2021 21:37:40 +0530
Subject: [PATCH 28/29] Fixed implementation bug in SSELoss::Evaluate

---
 src/mlpack/methods/xgboost/loss_functions/sse_loss.hpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/mlpack/methods/xgboost/loss_functions/sse_loss.hpp b/src/mlpack/methods/xgboost/loss_functions/sse_loss.hpp
index d4ffe22c3c..8d2a73a06c 100644
--- a/src/mlpack/methods/xgboost/loss_functions/sse_loss.hpp
+++ b/src/mlpack/methods/xgboost/loss_functions/sse_loss.hpp
@@ -87,7 +87,7 @@ class SSELoss
   double Evaluate(const MatType& input, const WeightVecType& /* weights */)
   {
     // Calculate gradients and hessians.
-    gradients = input.row(1) - input.row(0);
+    gradients = (input.row(1) - input.row(0)).t();
     hessians = arma::vec(input.n_cols, arma::fill::ones);
 
     return std::pow(ApplyL1(arma::accu(gradients)), 2) /

From 46233af41be7be12b75c17f124b39f07dce59f96 Mon Sep 17 00:00:00 2001
From: Rishabh Garg <56191449+RishabhGarg108@users.noreply.github.com>
Date: Fri, 23 Jul 2021 11:05:23 +0530
Subject: [PATCH 29/29] Update
 src/mlpack/methods/xgboost/loss_functions/sse_loss.hpp

Co-authored-by: Ryan Curtin <ryan@ratml.org>
---
 src/mlpack/methods/xgboost/loss_functions/sse_loss.hpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/mlpack/methods/xgboost/loss_functions/sse_loss.hpp b/src/mlpack/methods/xgboost/loss_functions/sse_loss.hpp
index 8d2a73a06c..97b79eb4d1 100644
--- a/src/mlpack/methods/xgboost/loss_functions/sse_loss.hpp
+++ b/src/mlpack/methods/xgboost/loss_functions/sse_loss.hpp
@@ -30,7 +30,7 @@ class SSELoss
 {
  public:
   // Default constructor---No regularization.
-  SSELoss(): alpha(0), lambda(0) { /* Nothing to do. */}
+  SSELoss() : alpha(0), lambda(0) { /* Nothing to do. */}
 
   SSELoss(const double alpha, const double lambda):
       alpha(alpha), lambda(lambda)