From 2f62b3f4ef347558a860128b26a099a5012cc704 Mon Sep 17 00:00:00 2001
From: Abhimanyu Dayal <abhimanyudayal1@gmail.com>
Date: Sat, 22 Jun 2024 01:43:36 +0530
Subject: [PATCH 01/21] Added sse gain to decision tree.

---
 .../methods/decision_tree/decision_tree.hpp   |  1 +
 .../sse_gain.hpp}                             | 19 +++++++++++++------
 2 files changed, 14 insertions(+), 6 deletions(-)
 rename src/mlpack/methods/{xgboost/loss_functions/sse_loss.hpp => decision_tree/sse_gain.hpp} (84%)

diff --git a/src/mlpack/methods/decision_tree/decision_tree.hpp b/src/mlpack/methods/decision_tree/decision_tree.hpp
index af169822b5..44b6ccd475 100644
--- a/src/mlpack/methods/decision_tree/decision_tree.hpp
+++ b/src/mlpack/methods/decision_tree/decision_tree.hpp
@@ -19,6 +19,7 @@
 #include "information_gain.hpp"
 #include "mad_gain.hpp"
 #include "mse_gain.hpp"
+#include "sse_gain.hpp"
 
 #include "best_binary_numeric_split.hpp"
 #include "random_binary_numeric_split.hpp"
diff --git a/src/mlpack/methods/xgboost/loss_functions/sse_loss.hpp b/src/mlpack/methods/decision_tree/sse_gain.hpp
similarity index 84%
rename from src/mlpack/methods/xgboost/loss_functions/sse_loss.hpp
rename to src/mlpack/methods/decision_tree/sse_gain.hpp
index 1029fa06af..663fdd6433 100644
--- a/src/mlpack/methods/xgboost/loss_functions/sse_loss.hpp
+++ b/src/mlpack/methods/decision_tree/sse_gain.hpp
@@ -1,8 +1,8 @@
 /**
- * @file methods/xgboost/loss_functions/sse_loss.hpp
+ * @file methods/decision_tree/sse_gain.hpp
  * @author Rishabh Garg
  *
- * The sum of squared error loss class, which is a loss funtion for gradient
+ * The sum of squared error loss class, which is a loss function for gradient
  * xgboost based decision trees.
  *
  * mlpack is free software; you may redistribute it and/or modify it under the
@@ -25,13 +25,13 @@ namespace mlpack {
  *
  * Loss = 1 / 2 * (Observed - Predicted)^2
  */
-class SSELoss
+class SSEGain
 {
  public:
   // Default constructor---No regularization.
-  SSELoss() : alpha(0), lambda(0) { /* Nothing to do. */}
+  SSEGain() : alpha(0), lambda(0) { /* Nothing to do. */}
 
-  SSELoss(const double alpha, const double lambda):
+  SSEGain(const double alpha, const double lambda):
       alpha(alpha), lambda(lambda)
   {
     // Nothing to do.
@@ -66,8 +66,15 @@ class SSELoss
    * @param begin The begin index to calculate gain.
    * @param end The end index to calculate gain.
    */
-  double Evaluate(const size_t begin, const size_t end)
+  template<bool UseWeights, typename MatType, typename WeightVecType>
+  double Evaluate(const MatType& input, 
+                  const WeightVecType& /* weights */,
+                  const size_t begin, 
+                  const size_t end)
   {
+    gradients = (input.row(1) - input.row(0)).t();
+    hessians = arma::vec(input.n_cols, arma::fill::ones);
+
     return std::pow(ApplyL1(accu(gradients.subvec(begin, end))), 2) /
         (accu(hessians.subvec(begin, end)) + lambda);
   }

From c37bf9dc82dbd5f71aef35ff39fce0329a2b3b6b Mon Sep 17 00:00:00 2001
From: Abhimanyu Dayal <abhimanyudayal1@gmail.com>
Date: Sat, 22 Jun 2024 02:00:55 +0530
Subject: [PATCH 02/21] organised helper functions into dirs in decision tree
 dir.

---
 .../methods/decision_tree/decision_tree.hpp   | 23 +++++++++----------
 .../decision_tree/decision_tree_regressor.hpp | 14 ++++++-----
 .../{ => gain_functions}/gini_gain.hpp        |  2 +-
 .../{ => gain_functions}/information_gain.hpp |  2 +-
 .../{ => gain_functions}/mad_gain.hpp         |  4 ++--
 .../{ => gain_functions}/mse_gain.hpp         |  4 ++--
 .../{ => gain_functions}/sse_gain.hpp         |  2 +-
 .../all_dimension_select.hpp                  |  2 +-
 .../multiple_random_dimension_select.hpp      |  2 +-
 .../random_dimension_select.hpp               |  2 +-
 .../all_categorical_split.hpp                 |  2 +-
 .../all_categorical_split_impl.hpp            |  2 +-
 .../best_binary_numeric_split.hpp             |  4 ++--
 .../best_binary_numeric_split_impl.hpp        |  2 +-
 .../random_binary_numeric_split.hpp           |  2 +-
 .../random_binary_numeric_split_impl.hpp      |  2 +-
 16 files changed, 36 insertions(+), 35 deletions(-)
 rename src/mlpack/methods/decision_tree/{ => gain_functions}/gini_gain.hpp (99%)
 rename src/mlpack/methods/decision_tree/{ => gain_functions}/information_gain.hpp (98%)
 rename src/mlpack/methods/decision_tree/{ => gain_functions}/mad_gain.hpp (97%)
 rename src/mlpack/methods/decision_tree/{ => gain_functions}/mse_gain.hpp (98%)
 rename src/mlpack/methods/decision_tree/{ => gain_functions}/sse_gain.hpp (98%)
 rename src/mlpack/methods/decision_tree/{ => select_functions}/all_dimension_select.hpp (95%)
 rename src/mlpack/methods/decision_tree/{ => select_functions}/multiple_random_dimension_select.hpp (96%)
 rename src/mlpack/methods/decision_tree/{ => select_functions}/random_dimension_select.hpp (95%)
 rename src/mlpack/methods/decision_tree/{ => split_functions}/all_categorical_split.hpp (98%)
 rename src/mlpack/methods/decision_tree/{ => split_functions}/all_categorical_split_impl.hpp (98%)
 rename src/mlpack/methods/decision_tree/{ => split_functions}/best_binary_numeric_split.hpp (98%)
 rename src/mlpack/methods/decision_tree/{ => split_functions}/best_binary_numeric_split_impl.hpp (99%)
 rename src/mlpack/methods/decision_tree/{ => split_functions}/random_binary_numeric_split.hpp (98%)
 rename src/mlpack/methods/decision_tree/{ => split_functions}/random_binary_numeric_split_impl.hpp (98%)

diff --git a/src/mlpack/methods/decision_tree/decision_tree.hpp b/src/mlpack/methods/decision_tree/decision_tree.hpp
index 44b6ccd475..4b92938681 100644
--- a/src/mlpack/methods/decision_tree/decision_tree.hpp
+++ b/src/mlpack/methods/decision_tree/decision_tree.hpp
@@ -15,20 +15,19 @@
 
 #include <mlpack/core.hpp>
 
-#include "gini_gain.hpp"
-#include "information_gain.hpp"
-#include "mad_gain.hpp"
-#include "mse_gain.hpp"
-#include "sse_gain.hpp"
+#include "gain_functions/gini_gain.hpp"
+#include "gain_functions/information_gain.hpp"
+#include "gain_functions/mad_gain.hpp"
+#include "gain_functions/mse_gain.hpp"
+#include "gain_functions/sse_gain.hpp"
 
-#include "best_binary_numeric_split.hpp"
-#include "random_binary_numeric_split.hpp"
+#include "split_functions/best_binary_numeric_split.hpp"
+#include "split_functions/random_binary_numeric_split.hpp"
+#include "split_functions/all_categorical_split.hpp"
 
-#include "all_categorical_split.hpp"
-
-#include "all_dimension_select.hpp"
-#include "random_dimension_select.hpp"
-#include "multiple_random_dimension_select.hpp"
+#include "select_functions/all_dimension_select.hpp"
+#include "select_functions/random_dimension_select.hpp"
+#include "select_functions/multiple_random_dimension_select.hpp"
 
 namespace mlpack {
 
diff --git a/src/mlpack/methods/decision_tree/decision_tree_regressor.hpp b/src/mlpack/methods/decision_tree/decision_tree_regressor.hpp
index aaa9f74f5b..4dd9732f5f 100644
--- a/src/mlpack/methods/decision_tree/decision_tree_regressor.hpp
+++ b/src/mlpack/methods/decision_tree/decision_tree_regressor.hpp
@@ -15,12 +15,14 @@
 
 #include <mlpack/core.hpp>
 
-#include "mad_gain.hpp"
-#include "mse_gain.hpp"
-#include "best_binary_numeric_split.hpp"
-#include "all_categorical_split.hpp"
-#include "random_binary_numeric_split.hpp"
-#include "all_dimension_select.hpp"
+#include "gain_functions/mad_gain.hpp"
+#include "gain_functions/mse_gain.hpp"
+
+#include "split_functions/best_binary_numeric_split.hpp"
+#include "split_functions/all_categorical_split.hpp"
+#include "split_functions/random_binary_numeric_split.hpp"
+
+#include "select_functions/all_dimension_select.hpp"
 
 namespace mlpack {
 
diff --git a/src/mlpack/methods/decision_tree/gini_gain.hpp b/src/mlpack/methods/decision_tree/gain_functions/gini_gain.hpp
similarity index 99%
rename from src/mlpack/methods/decision_tree/gini_gain.hpp
rename to src/mlpack/methods/decision_tree/gain_functions/gini_gain.hpp
index ea7d4a5407..e52a23d3af 100644
--- a/src/mlpack/methods/decision_tree/gini_gain.hpp
+++ b/src/mlpack/methods/decision_tree/gain_functions/gini_gain.hpp
@@ -1,5 +1,5 @@
 /**
- * @file methods/decision_tree/gini_gain.hpp
+ * @file methods/decision_tree/gain_functions/gini_gain.hpp
  * @author Ryan Curtin
  *
  * The GiniGain class, which is a fitness function (FitnessFunction) for
diff --git a/src/mlpack/methods/decision_tree/information_gain.hpp b/src/mlpack/methods/decision_tree/gain_functions/information_gain.hpp
similarity index 98%
rename from src/mlpack/methods/decision_tree/information_gain.hpp
rename to src/mlpack/methods/decision_tree/gain_functions/information_gain.hpp
index 7cf0f1158e..a768886114 100644
--- a/src/mlpack/methods/decision_tree/information_gain.hpp
+++ b/src/mlpack/methods/decision_tree/gain_functions/information_gain.hpp
@@ -1,5 +1,5 @@
 /**
- * @file methods/decision_tree/information_gain.hpp
+ * @file methods/decision_tree/gain_functions/information_gain.hpp
  * @author Ryan Curtin
  *
  * An implementation of information gain, which can be used in place of Gini
diff --git a/src/mlpack/methods/decision_tree/mad_gain.hpp b/src/mlpack/methods/decision_tree/gain_functions/mad_gain.hpp
similarity index 97%
rename from src/mlpack/methods/decision_tree/mad_gain.hpp
rename to src/mlpack/methods/decision_tree/gain_functions/mad_gain.hpp
index 742700fb34..bcf1275e57 100644
--- a/src/mlpack/methods/decision_tree/mad_gain.hpp
+++ b/src/mlpack/methods/decision_tree/gain_functions/mad_gain.hpp
@@ -1,5 +1,5 @@
 /**
- * @file methods/decision_tree/mad_gain.hpp
+ * @file methods/decision_tree/gain_functions/mad_gain.hpp
  * @author Rishabh Garg
  *
  * The mean absolute deviation gain class, a fitness function for regression
@@ -15,7 +15,7 @@ n.
 #define MLPACK_METHODS_DECISION_TREE_MAD_GAIN_HPP
 
 #include <mlpack/prereqs.hpp>
-#include "utils.hpp"
+#include "mlpack/methods/decision_tree/utils.hpp"
 
 namespace mlpack {
 
diff --git a/src/mlpack/methods/decision_tree/mse_gain.hpp b/src/mlpack/methods/decision_tree/gain_functions/mse_gain.hpp
similarity index 98%
rename from src/mlpack/methods/decision_tree/mse_gain.hpp
rename to src/mlpack/methods/decision_tree/gain_functions/mse_gain.hpp
index 8e64a97a01..f81ffe0781 100644
--- a/src/mlpack/methods/decision_tree/mse_gain.hpp
+++ b/src/mlpack/methods/decision_tree/gain_functions/mse_gain.hpp
@@ -1,5 +1,5 @@
 /**
- * @file methods/decision_tree/mse_gain.hpp
+ * @file methods/decision_tree/gain_functions/mse_gain.hpp
  * @author Rishabh Garg
  *
  * The mean squared error gain class, which is a fitness funtion for
@@ -14,7 +14,7 @@
 #define MLPACK_METHODS_DECISION_TREE_MSE_GAIN_HPP
 
 #include <mlpack/prereqs.hpp>
-#include "utils.hpp"
+#include "mlpack/methods/decision_tree/utils.hpp"
 
 namespace mlpack {
 
diff --git a/src/mlpack/methods/decision_tree/sse_gain.hpp b/src/mlpack/methods/decision_tree/gain_functions/sse_gain.hpp
similarity index 98%
rename from src/mlpack/methods/decision_tree/sse_gain.hpp
rename to src/mlpack/methods/decision_tree/gain_functions/sse_gain.hpp
index 663fdd6433..c0351fca7f 100644
--- a/src/mlpack/methods/decision_tree/sse_gain.hpp
+++ b/src/mlpack/methods/decision_tree/gain_functions/sse_gain.hpp
@@ -1,5 +1,5 @@
 /**
- * @file methods/decision_tree/sse_gain.hpp
+ * @file methods/decision_tree/gain_functions/sse_gain.hpp
  * @author Rishabh Garg
  *
  * The sum of squared error loss class, which is a loss function for gradient
diff --git a/src/mlpack/methods/decision_tree/all_dimension_select.hpp b/src/mlpack/methods/decision_tree/select_functions/all_dimension_select.hpp
similarity index 95%
rename from src/mlpack/methods/decision_tree/all_dimension_select.hpp
rename to src/mlpack/methods/decision_tree/select_functions/all_dimension_select.hpp
index 332a439fd8..5c59a6f60c 100644
--- a/src/mlpack/methods/decision_tree/all_dimension_select.hpp
+++ b/src/mlpack/methods/decision_tree/select_functions/all_dimension_select.hpp
@@ -1,5 +1,5 @@
 /**
- * @file methods/decision_tree/all_dimension_select.hpp
+ * @file methods/decision_tree/select_functions/all_dimension_select.hpp
  * @author Ryan Curtin
  *
  * Selects all dimensions for a split.
diff --git a/src/mlpack/methods/decision_tree/multiple_random_dimension_select.hpp b/src/mlpack/methods/decision_tree/select_functions/multiple_random_dimension_select.hpp
similarity index 96%
rename from src/mlpack/methods/decision_tree/multiple_random_dimension_select.hpp
rename to src/mlpack/methods/decision_tree/select_functions/multiple_random_dimension_select.hpp
index 6ce277d258..d0c3fff36a 100644
--- a/src/mlpack/methods/decision_tree/multiple_random_dimension_select.hpp
+++ b/src/mlpack/methods/decision_tree/select_functions/multiple_random_dimension_select.hpp
@@ -1,5 +1,5 @@
 /**
- * @file methods/decision_tree/multiple_random_dimension_select.hpp
+ * @file methods/decision_tree/select_functions/multiple_random_dimension_select.hpp
  * @author Ryan Curtin
  *
  * Select a number of random dimensions to pick from.
diff --git a/src/mlpack/methods/decision_tree/random_dimension_select.hpp b/src/mlpack/methods/decision_tree/select_functions/random_dimension_select.hpp
similarity index 95%
rename from src/mlpack/methods/decision_tree/random_dimension_select.hpp
rename to src/mlpack/methods/decision_tree/select_functions/random_dimension_select.hpp
index fdf024a70a..03a32f5fc2 100644
--- a/src/mlpack/methods/decision_tree/random_dimension_select.hpp
+++ b/src/mlpack/methods/decision_tree/select_functions/random_dimension_select.hpp
@@ -1,5 +1,5 @@
 /**
- * @file methods/decision_tree/random_dimension_select.hpp
+ * @file methods/decision_tree/select_functions/random_dimension_select.hpp
  * @author Ryan Curtin
  *
  * Selects one single random dimension to split on.
diff --git a/src/mlpack/methods/decision_tree/all_categorical_split.hpp b/src/mlpack/methods/decision_tree/split_functions/all_categorical_split.hpp
similarity index 98%
rename from src/mlpack/methods/decision_tree/all_categorical_split.hpp
rename to src/mlpack/methods/decision_tree/split_functions/all_categorical_split.hpp
index 9d432b732c..76ec876d95 100644
--- a/src/mlpack/methods/decision_tree/all_categorical_split.hpp
+++ b/src/mlpack/methods/decision_tree/split_functions/all_categorical_split.hpp
@@ -1,5 +1,5 @@
 /**
- * @file methods/decision_tree/all_categorical_split.hpp
+ * @file methods/decision_tree/split_functions/all_categorical_split.hpp
  * @author Ryan Curtin
  *
  * This file defines a tree splitter that split a categorical feature into all
diff --git a/src/mlpack/methods/decision_tree/all_categorical_split_impl.hpp b/src/mlpack/methods/decision_tree/split_functions/all_categorical_split_impl.hpp
similarity index 98%
rename from src/mlpack/methods/decision_tree/all_categorical_split_impl.hpp
rename to src/mlpack/methods/decision_tree/split_functions/all_categorical_split_impl.hpp
index 3ee94f1818..abb34c05f2 100644
--- a/src/mlpack/methods/decision_tree/all_categorical_split_impl.hpp
+++ b/src/mlpack/methods/decision_tree/split_functions/all_categorical_split_impl.hpp
@@ -1,5 +1,5 @@
 /**
- * @file methods/decision_tree/all_categorical_split_impl.hpp
+ * @file methods/decision_tree/split_functions/all_categorical_split_impl.hpp
  * @author Ryan Curtin
  *
  * Implementation of the AllCategoricalSplit categorical split class.
diff --git a/src/mlpack/methods/decision_tree/best_binary_numeric_split.hpp b/src/mlpack/methods/decision_tree/split_functions/best_binary_numeric_split.hpp
similarity index 98%
rename from src/mlpack/methods/decision_tree/best_binary_numeric_split.hpp
rename to src/mlpack/methods/decision_tree/split_functions/best_binary_numeric_split.hpp
index 34d397c829..6bc5b101f0 100644
--- a/src/mlpack/methods/decision_tree/best_binary_numeric_split.hpp
+++ b/src/mlpack/methods/decision_tree/split_functions/best_binary_numeric_split.hpp
@@ -1,5 +1,5 @@
 /**
- * @file methods/decision_tree/best_binary_numeric_split.hpp
+ * @file methods/decision_tree/split_functions/best_binary_numeric_split.hpp
  * @author Ryan Curtin
  *
  * A tree splitter that finds the best binary numeric split.
@@ -13,7 +13,7 @@
 #define MLPACK_METHODS_DECISION_TREE_BEST_BINARY_NUMERIC_SPLIT_HPP
 
 #include <mlpack/prereqs.hpp>
-#include "mse_gain.hpp"
+#include "gain_functions/mse_gain.hpp"
 
 #include <mlpack/core/util/sfinae_utility.hpp>
 
diff --git a/src/mlpack/methods/decision_tree/best_binary_numeric_split_impl.hpp b/src/mlpack/methods/decision_tree/split_functions/best_binary_numeric_split_impl.hpp
similarity index 99%
rename from src/mlpack/methods/decision_tree/best_binary_numeric_split_impl.hpp
rename to src/mlpack/methods/decision_tree/split_functions/best_binary_numeric_split_impl.hpp
index e8c8647206..1e6fc50b74 100644
--- a/src/mlpack/methods/decision_tree/best_binary_numeric_split_impl.hpp
+++ b/src/mlpack/methods/decision_tree/split_functions/best_binary_numeric_split_impl.hpp
@@ -1,5 +1,5 @@
 /**
- * @file methods/decision_tree/best_binary_numeric_split_impl.hpp
+ * @file methods/decision_tree/split_functions/best_binary_numeric_split_impl.hpp
  * @author Ryan Curtin
  *
  * Implementation of strategy that finds the best binary numeric split.
diff --git a/src/mlpack/methods/decision_tree/random_binary_numeric_split.hpp b/src/mlpack/methods/decision_tree/split_functions/random_binary_numeric_split.hpp
similarity index 98%
rename from src/mlpack/methods/decision_tree/random_binary_numeric_split.hpp
rename to src/mlpack/methods/decision_tree/split_functions/random_binary_numeric_split.hpp
index a92d9d6eef..7173c7ba17 100644
--- a/src/mlpack/methods/decision_tree/random_binary_numeric_split.hpp
+++ b/src/mlpack/methods/decision_tree/split_functions/random_binary_numeric_split.hpp
@@ -1,5 +1,5 @@
 /**
- * @file methods/decision_tree/random_binary_numeric_split.hpp
+ * @file methods/decision_tree/split_functions/random_binary_numeric_split.hpp
  * @author Rishabh Garg
  *
  * A tree splitter that finds a random binary numeric split.
diff --git a/src/mlpack/methods/decision_tree/random_binary_numeric_split_impl.hpp b/src/mlpack/methods/decision_tree/split_functions/random_binary_numeric_split_impl.hpp
similarity index 98%
rename from src/mlpack/methods/decision_tree/random_binary_numeric_split_impl.hpp
rename to src/mlpack/methods/decision_tree/split_functions/random_binary_numeric_split_impl.hpp
index 9139214248..7447b84e7a 100644
--- a/src/mlpack/methods/decision_tree/random_binary_numeric_split_impl.hpp
+++ b/src/mlpack/methods/decision_tree/split_functions/random_binary_numeric_split_impl.hpp
@@ -1,5 +1,5 @@
 /**
- * @file methods/decision_tree/random_binary_numeric_split_impl.hpp
+ * @file methods/decision_tree/split_functions/random_binary_numeric_split_impl.hpp
  * @author Rishabh Garg
  *
  * Implementation of strategy that finds the random binary numeric split.

From 2fa2e824bac21dbb5e5527f6596471aac93b954b Mon Sep 17 00:00:00 2001
From: Abhimanyu Dayal <abhimanyudayal1@gmail.com>
Date: Sat, 22 Jun 2024 17:31:31 +0530
Subject: [PATCH 03/21] minor fix path

---
 .../decision_tree/split_functions/best_binary_numeric_split.hpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/mlpack/methods/decision_tree/split_functions/best_binary_numeric_split.hpp b/src/mlpack/methods/decision_tree/split_functions/best_binary_numeric_split.hpp
index 6bc5b101f0..438a31fbc0 100644
--- a/src/mlpack/methods/decision_tree/split_functions/best_binary_numeric_split.hpp
+++ b/src/mlpack/methods/decision_tree/split_functions/best_binary_numeric_split.hpp
@@ -13,7 +13,7 @@
 #define MLPACK_METHODS_DECISION_TREE_BEST_BINARY_NUMERIC_SPLIT_HPP
 
 #include <mlpack/prereqs.hpp>
-#include "gain_functions/mse_gain.hpp"
+#include <mlpack/methods/decision_tree/gain_functions/mse_gain.hpp>
 
 #include <mlpack/core/util/sfinae_utility.hpp>
 

From 5eba73c0673beb4f053f1f63b3a97ffcd0ba059f Mon Sep 17 00:00:00 2001
From: Abhimanyu Dayal <abhimanyudayal1@gmail.com>
Date: Sat, 22 Jun 2024 18:55:00 +0530
Subject: [PATCH 04/21] added convenience def

---
 src/mlpack/methods/decision_tree/decision_tree.hpp | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/src/mlpack/methods/decision_tree/decision_tree.hpp b/src/mlpack/methods/decision_tree/decision_tree.hpp
index 4b92938681..632767298b 100644
--- a/src/mlpack/methods/decision_tree/decision_tree.hpp
+++ b/src/mlpack/methods/decision_tree/decision_tree.hpp
@@ -611,6 +611,15 @@ typedef DecisionTree<InformationGain,
                      AllCategoricalSplit,
                      AllDimensionSelect,
                      true> ID3DecisionStump;
+
+/**
+ * Convenience typedef for XGBoost trees.
+ */
+typedef DecisionTree<SSEGain,
+                     BestBinaryNumericSplit,
+                     AllCategoricalSplit,
+                     AllDimensionSelect,
+                     true> XGBTree;
 } // namespace mlpack
 
 // Include implementation.

From 0cf8d5e2ce1936f681f030179c2c3921db8a58c3 Mon Sep 17 00:00:00 2001
From: Abhimanyu Dayal <abhimanyudayal1@gmail.com>
Date: Sat, 22 Jun 2024 21:28:34 +0530
Subject: [PATCH 05/21] started work on log_gain

---
 .../decision_tree/gain_functions/log_gain.cpp | 81 +++++++++++++++++++
 1 file changed, 81 insertions(+)
 create mode 100644 src/mlpack/methods/decision_tree/gain_functions/log_gain.cpp

diff --git a/src/mlpack/methods/decision_tree/gain_functions/log_gain.cpp b/src/mlpack/methods/decision_tree/gain_functions/log_gain.cpp
new file mode 100644
index 0000000000..69d47fac22
--- /dev/null
+++ b/src/mlpack/methods/decision_tree/gain_functions/log_gain.cpp
@@ -0,0 +1,81 @@
+/**
+ * @file methods/decision_tree/gain_functions/log_gain.hpp
+ * @author Abhimanyu Dayal
+ *
+ * The logistic gain class, which is a gain function for xgboost.
+ *
+ * mlpack is free software; you may redistribute it and/or modify it under the
+ * terms of the 3-clause BSD license.  You should have received a copy of the
+ * 3-clause BSD license along with mlpack.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef MLPACK_METHODS_XGBOOST_LOSS_FUNCTIONS_LOG_LOSS_HPP
+#define MLPACK_METHODS_XGBOOST_LOSS_FUNCTIONS_LOG_LOSS_HPP
+
+#include <mlpack/prereqs.hpp>
+
+namespace mlpack {
+/**
+ * Logistic loss, also known as log loss or cross-entropy loss, is a loss 
+ * function used in logistic regression to measure the difference between 
+ * the predicted probability of an event and the actual outcome.
+ * 
+ * Log Loss = - (1 / N) * Σ [y_i * log(p_i) + (1 - y_i) * log(1 - p_i)]
+ */
+class LogLoss
+{
+  public:
+  // Default constructor---No regularization.
+  LogLoss() : alpha(0), lambda(0) { /* Nothing to do. */}
+
+  LogLoss(const double alpha, const double lambda):
+      alpha(alpha), lambda(lambda)
+  {
+    // Nothing to do.
+  }
+
+  /**
+   * Returns the initial prediction for gradient boosting.
+   */
+  template<typename VecType>
+  typename VecType::elem_type InitialPrediction(const VecType& values)
+  {
+    // Sanity check for empty vector.
+    if (values.n_elem == 0)
+      return 0;
+
+    // Return the log-odds of the mean of the values.
+    double mean = accu(values) / (typename VecType::elem_type) values.n_elem;
+    return std::log(mean / (1 - mean));
+  }
+
+
+  private:
+  //! The L1 regularization parameter.
+  const double alpha;
+  //! The L2 regularization parameter.
+  const double lambda;
+  //! First order gradients.
+  arma::vec gradients;
+  //! Second order gradients (hessians).
+  arma::vec hessians;
+
+  //! Applies the L1 regularization.
+  double ApplyL1(const double sumGradients)
+  {
+    if (sumGradients > alpha)
+    {
+      return sumGradients - alpha;
+    }
+    else if (sumGradients < - alpha)
+    {
+      return sumGradients + alpha;
+    }
+
+    return 0;
+  }
+
+};
+} // namespace mlpack
+
+#endif

From 95f1e5d101906707367533cf24b46d2c69ed3d39 Mon Sep 17 00:00:00 2001
From: Abhimanyu Dayal <abhimanyudayal1@gmail.com>
Date: Sun, 23 Jun 2024 19:39:15 +0530
Subject: [PATCH 06/21] added convenience include files

---
 .../methods/decision_tree/decision_tree.hpp      | 16 +++-------------
 .../decision_tree/decision_tree_regressor.hpp    | 11 +++--------
 .../gain_functions/gain_functions.hpp            |  6 ++++++
 .../{log_gain.cpp => log_gain.hpp}               |  0
 .../select_functions/select_functions.hpp        |  3 +++
 .../split_functions/split_functions.hpp          |  3 +++
 6 files changed, 18 insertions(+), 21 deletions(-)
 create mode 100644 src/mlpack/methods/decision_tree/gain_functions/gain_functions.hpp
 rename src/mlpack/methods/decision_tree/gain_functions/{log_gain.cpp => log_gain.hpp} (100%)
 create mode 100644 src/mlpack/methods/decision_tree/select_functions/select_functions.hpp
 create mode 100644 src/mlpack/methods/decision_tree/split_functions/split_functions.hpp

diff --git a/src/mlpack/methods/decision_tree/decision_tree.hpp b/src/mlpack/methods/decision_tree/decision_tree.hpp
index 632767298b..73d557ebbb 100644
--- a/src/mlpack/methods/decision_tree/decision_tree.hpp
+++ b/src/mlpack/methods/decision_tree/decision_tree.hpp
@@ -15,19 +15,9 @@
 
 #include <mlpack/core.hpp>
 
-#include "gain_functions/gini_gain.hpp"
-#include "gain_functions/information_gain.hpp"
-#include "gain_functions/mad_gain.hpp"
-#include "gain_functions/mse_gain.hpp"
-#include "gain_functions/sse_gain.hpp"
-
-#include "split_functions/best_binary_numeric_split.hpp"
-#include "split_functions/random_binary_numeric_split.hpp"
-#include "split_functions/all_categorical_split.hpp"
-
-#include "select_functions/all_dimension_select.hpp"
-#include "select_functions/random_dimension_select.hpp"
-#include "select_functions/multiple_random_dimension_select.hpp"
+#include "gain_functions/gain_functions.hpp"
+#include "split_functions/split_functions.hpp"
+#include "select_functions/select_functions.hpp"
 
 namespace mlpack {
 
diff --git a/src/mlpack/methods/decision_tree/decision_tree_regressor.hpp b/src/mlpack/methods/decision_tree/decision_tree_regressor.hpp
index 4dd9732f5f..1c77050f90 100644
--- a/src/mlpack/methods/decision_tree/decision_tree_regressor.hpp
+++ b/src/mlpack/methods/decision_tree/decision_tree_regressor.hpp
@@ -15,14 +15,9 @@
 
 #include <mlpack/core.hpp>
 
-#include "gain_functions/mad_gain.hpp"
-#include "gain_functions/mse_gain.hpp"
-
-#include "split_functions/best_binary_numeric_split.hpp"
-#include "split_functions/all_categorical_split.hpp"
-#include "split_functions/random_binary_numeric_split.hpp"
-
-#include "select_functions/all_dimension_select.hpp"
+#include "gain_functions/gain_functions.hpp"
+#include "split_functions/split_functions.hpp"
+#include "select_functions/select_functions.hpp"
 
 namespace mlpack {
 
diff --git a/src/mlpack/methods/decision_tree/gain_functions/gain_functions.hpp b/src/mlpack/methods/decision_tree/gain_functions/gain_functions.hpp
new file mode 100644
index 0000000000..f94e653de9
--- /dev/null
+++ b/src/mlpack/methods/decision_tree/gain_functions/gain_functions.hpp
@@ -0,0 +1,6 @@
+#include "gini_gain.hpp"
+#include "information_gain.hpp"
+#include "log_gain.hpp"
+#include "mad_gain.hpp"
+#include "mse_gain.hpp"
+#include "sse_gain.hpp"
\ No newline at end of file
diff --git a/src/mlpack/methods/decision_tree/gain_functions/log_gain.cpp b/src/mlpack/methods/decision_tree/gain_functions/log_gain.hpp
similarity index 100%
rename from src/mlpack/methods/decision_tree/gain_functions/log_gain.cpp
rename to src/mlpack/methods/decision_tree/gain_functions/log_gain.hpp
diff --git a/src/mlpack/methods/decision_tree/select_functions/select_functions.hpp b/src/mlpack/methods/decision_tree/select_functions/select_functions.hpp
new file mode 100644
index 0000000000..da857189f1
--- /dev/null
+++ b/src/mlpack/methods/decision_tree/select_functions/select_functions.hpp
@@ -0,0 +1,3 @@
+#include "all_dimension_select.hpp"
+#include "multiple_random_dimension_select.hpp"
+#include "random_dimension_select.hpp"
\ No newline at end of file
diff --git a/src/mlpack/methods/decision_tree/split_functions/split_functions.hpp b/src/mlpack/methods/decision_tree/split_functions/split_functions.hpp
new file mode 100644
index 0000000000..067be112ec
--- /dev/null
+++ b/src/mlpack/methods/decision_tree/split_functions/split_functions.hpp
@@ -0,0 +1,3 @@
+#include "all_categorical_split.hpp"
+#include "best_binary_numeric_split.hpp"
+#include "random_binary_numeric_split.hpp"
\ No newline at end of file

From e75cc2149c467034ef8a504ad39d5448a4cde7d5 Mon Sep 17 00:00:00 2001
From: Abhimanyu Dayal <abhimanyudayal1@gmail.com>
Date: Wed, 26 Jun 2024 23:38:31 +0530
Subject: [PATCH 07/21] shifting log stuff to another PR

---
 .../decision_tree/gain_functions/log_gain.hpp | 81 -------------------
 1 file changed, 81 deletions(-)
 delete mode 100644 src/mlpack/methods/decision_tree/gain_functions/log_gain.hpp

diff --git a/src/mlpack/methods/decision_tree/gain_functions/log_gain.hpp b/src/mlpack/methods/decision_tree/gain_functions/log_gain.hpp
deleted file mode 100644
index 69d47fac22..0000000000
--- a/src/mlpack/methods/decision_tree/gain_functions/log_gain.hpp
+++ /dev/null
@@ -1,81 +0,0 @@
-/**
- * @file methods/decision_tree/gain_functions/log_gain.hpp
- * @author Abhimanyu Dayal
- *
- * The logistic gain class, which is a gain function for xgboost.
- *
- * mlpack is free software; you may redistribute it and/or modify it under the
- * terms of the 3-clause BSD license.  You should have received a copy of the
- * 3-clause BSD license along with mlpack.  If not, see
- * http://www.opensource.org/licenses/BSD-3-Clause for more information.
- */
-#ifndef MLPACK_METHODS_XGBOOST_LOSS_FUNCTIONS_LOG_LOSS_HPP
-#define MLPACK_METHODS_XGBOOST_LOSS_FUNCTIONS_LOG_LOSS_HPP
-
-#include <mlpack/prereqs.hpp>
-
-namespace mlpack {
-/**
- * Logistic loss, also known as log loss or cross-entropy loss, is a loss 
- * function used in logistic regression to measure the difference between 
- * the predicted probability of an event and the actual outcome.
- * 
- * Log Loss = - (1 / N) * Σ [y_i * log(p_i) + (1 - y_i) * log(1 - p_i)]
- */
-class LogLoss
-{
-  public:
-  // Default constructor---No regularization.
-  LogLoss() : alpha(0), lambda(0) { /* Nothing to do. */}
-
-  LogLoss(const double alpha, const double lambda):
-      alpha(alpha), lambda(lambda)
-  {
-    // Nothing to do.
-  }
-
-  /**
-   * Returns the initial prediction for gradient boosting.
-   */
-  template<typename VecType>
-  typename VecType::elem_type InitialPrediction(const VecType& values)
-  {
-    // Sanity check for empty vector.
-    if (values.n_elem == 0)
-      return 0;
-
-    // Return the log-odds of the mean of the values.
-    double mean = accu(values) / (typename VecType::elem_type) values.n_elem;
-    return std::log(mean / (1 - mean));
-  }
-
-
-  private:
-  //! The L1 regularization parameter.
-  const double alpha;
-  //! The L2 regularization parameter.
-  const double lambda;
-  //! First order gradients.
-  arma::vec gradients;
-  //! Second order gradients (hessians).
-  arma::vec hessians;
-
-  //! Applies the L1 regularization.
-  double ApplyL1(const double sumGradients)
-  {
-    if (sumGradients > alpha)
-    {
-      return sumGradients - alpha;
-    }
-    else if (sumGradients < - alpha)
-    {
-      return sumGradients + alpha;
-    }
-
-    return 0;
-  }
-
-};
-} // namespace mlpack
-
-#endif

From 801ba91ce97f0e35a046d352d2cabcf1df5f3d69 Mon Sep 17 00:00:00 2001
From: Abhimanyu Dayal <abhimanyudayal1@gmail.com>
Date: Fri, 28 Jun 2024 18:28:08 +0530
Subject: [PATCH 08/21] minor fix

---
 .../methods/decision_tree/gain_functions/gain_functions.hpp   | 1 -
 src/mlpack/methods/decision_tree/gain_functions/sse_gain.hpp  | 4 ++--
 2 files changed, 2 insertions(+), 3 deletions(-)

diff --git a/src/mlpack/methods/decision_tree/gain_functions/gain_functions.hpp b/src/mlpack/methods/decision_tree/gain_functions/gain_functions.hpp
index f94e653de9..a73fcd19c6 100644
--- a/src/mlpack/methods/decision_tree/gain_functions/gain_functions.hpp
+++ b/src/mlpack/methods/decision_tree/gain_functions/gain_functions.hpp
@@ -1,6 +1,5 @@
 #include "gini_gain.hpp"
 #include "information_gain.hpp"
-#include "log_gain.hpp"
 #include "mad_gain.hpp"
 #include "mse_gain.hpp"
 #include "sse_gain.hpp"
\ No newline at end of file
diff --git a/src/mlpack/methods/decision_tree/gain_functions/sse_gain.hpp b/src/mlpack/methods/decision_tree/gain_functions/sse_gain.hpp
index c0351fca7f..b369128aaa 100644
--- a/src/mlpack/methods/decision_tree/gain_functions/sse_gain.hpp
+++ b/src/mlpack/methods/decision_tree/gain_functions/sse_gain.hpp
@@ -10,8 +10,8 @@
  * 3-clause BSD license along with mlpack.  If not, see
  * http://www.opensource.org/licenses/BSD-3-Clause for more information.
  */
-#ifndef MLPACK_METHODS_XGBOOST_LOSS_FUNCTIONS_SSE_LOSS_HPP
-#define MLPACK_METHODS_XGBOOST_LOSS_FUNCTIONS_SSE_LOSS_HPP
+#ifndef MLPACK_METHODS_DECISION_TREE_SSE_GAIN_HPP
+#define MLPACK_METHODS_DECISION_TREE_SSE_GAIN_HPP
 
 #include <mlpack/prereqs.hpp>
 

From 1d7e4bbc73482c9f47b10de40c042bd44660773f Mon Sep 17 00:00:00 2001
From: Abhimanyu Dayal <abhimanyudayal1@gmail.com>
Date: Fri, 28 Jun 2024 19:00:58 +0530
Subject: [PATCH 09/21] updated path

---
 src/mlpack/tests/xgboost_test.cpp | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/src/mlpack/tests/xgboost_test.cpp b/src/mlpack/tests/xgboost_test.cpp
index 4b3d1d533a..57dbbc53dc 100644
--- a/src/mlpack/tests/xgboost_test.cpp
+++ b/src/mlpack/tests/xgboost_test.cpp
@@ -10,7 +10,7 @@
  * http://www.opensource.org/licenses/BSD-3-Clause for more information.
  */
 #include <mlpack/core.hpp>
-#include <mlpack/methods/xgboost/loss_functions/sse_loss.hpp>
+#include <mlpack/methods/decision_tree/gain_functions/sse_gain.hpp>
 
 #include "catch.hpp"
 #include "serialization.hpp"
@@ -18,7 +18,7 @@
 using namespace mlpack;
 
 /**
- * Test that the initial prediction is calculated correctly for SSE loss.
+ * Test that the initial prediction is calculated correctly for SSE gain.
  */
 TEST_CASE("SSEInitialPredictionTest", "[XGBTest]")
 {
@@ -26,12 +26,12 @@ TEST_CASE("SSEInitialPredictionTest", "[XGBTest]")
 
   double initPred = 5.5;
 
-  SSELoss Loss;
+  SSEGain Loss;
   REQUIRE(Loss.InitialPrediction(values) == initPred);
 }
 
 /**
- * Test that output leaf value is calculated correctly for SSE Loss.
+ * Test that output leaf value is calculated correctly for SSE gain.
  */
 TEST_CASE("SSELeafValueTest", "[XGBTest]")
 {
@@ -42,14 +42,14 @@ TEST_CASE("SSELeafValueTest", "[XGBTest]")
   // Actual output leaf value.
   double leafValue = -0.075;
 
-  SSELoss Loss;
+  SSEGain Loss;
   (void) Loss.Evaluate<false>(input, weights);
 
   REQUIRE(Loss.OutputLeafValue(input, weights) == leafValue);
 }
 
 /**
- * Test that the gain is computed correctly for SSE Loss.
+ * Test that the gain is computed correctly for SSE gain.
  */
 TEST_CASE("SSEGainTest", "[XGBTest]")
 {
@@ -60,6 +60,6 @@ TEST_CASE("SSEGainTest", "[XGBTest]")
   // Actual gain value.
   double gain = 0.05625;
 
-  SSELoss Loss;
+  SSEGain Loss;
   REQUIRE(Loss.Evaluate<false>(input, weights) == gain);
 }

From cbee98dfd6c99ee3d74eaf2432d6be52b4e09b8f Mon Sep 17 00:00:00 2001
From: Abhimanyu Dayal <abhimanyudayal1@gmail.com>
Date: Fri, 28 Jun 2024 20:13:35 +0530
Subject: [PATCH 10/21] minor fix

---
 src/mlpack/methods/decision_tree/gain_functions/mse_gain.hpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/mlpack/methods/decision_tree/gain_functions/mse_gain.hpp b/src/mlpack/methods/decision_tree/gain_functions/mse_gain.hpp
index f81ffe0781..8709b3e9ae 100644
--- a/src/mlpack/methods/decision_tree/gain_functions/mse_gain.hpp
+++ b/src/mlpack/methods/decision_tree/gain_functions/mse_gain.hpp
@@ -14,7 +14,7 @@
 #define MLPACK_METHODS_DECISION_TREE_MSE_GAIN_HPP
 
 #include <mlpack/prereqs.hpp>
-#include "mlpack/methods/decision_tree/utils.hpp"
+#include <mlpack/methods/decision_tree/utils.hpp>
 
 namespace mlpack {
 

From 840dcf4a82d1d66f7db436fe3a3c6e744c251305 Mon Sep 17 00:00:00 2001
From: Abhimanyu Dayal <abhimanyudayal1@gmail.com>
Date: Mon, 1 Jul 2024 13:14:17 +0530
Subject: [PATCH 11/21] resolved some conflicts

---
 .../{ => split_functions}/best_binary_categorical_split.hpp    | 0
 .../best_binary_categorical_split_impl.hpp                     | 2 +-
 .../methods/decision_tree/split_functions/split_functions.hpp  | 3 ++-
 3 files changed, 3 insertions(+), 2 deletions(-)
 rename src/mlpack/methods/decision_tree/{ => split_functions}/best_binary_categorical_split.hpp (100%)
 rename src/mlpack/methods/decision_tree/{ => split_functions}/best_binary_categorical_split_impl.hpp (99%)

diff --git a/src/mlpack/methods/decision_tree/best_binary_categorical_split.hpp b/src/mlpack/methods/decision_tree/split_functions/best_binary_categorical_split.hpp
similarity index 100%
rename from src/mlpack/methods/decision_tree/best_binary_categorical_split.hpp
rename to src/mlpack/methods/decision_tree/split_functions/best_binary_categorical_split.hpp
diff --git a/src/mlpack/methods/decision_tree/best_binary_categorical_split_impl.hpp b/src/mlpack/methods/decision_tree/split_functions/best_binary_categorical_split_impl.hpp
similarity index 99%
rename from src/mlpack/methods/decision_tree/best_binary_categorical_split_impl.hpp
rename to src/mlpack/methods/decision_tree/split_functions/best_binary_categorical_split_impl.hpp
index c5be2c4de2..bd0c8f62c3 100644
--- a/src/mlpack/methods/decision_tree/best_binary_categorical_split_impl.hpp
+++ b/src/mlpack/methods/decision_tree/split_functions/best_binary_categorical_split_impl.hpp
@@ -1,5 +1,5 @@
 /**
- * @file methods/decision_tree/all_categorical_split_impl.hpp
+ * @file methods/decision_tree/split_functions/all_categorical_split_impl.hpp
  * @author Nikolay Apanasov (nikolay@apanasov.org)
  *
  * Implementation of the BestBinaryCategoricalSplit categorical split class.
diff --git a/src/mlpack/methods/decision_tree/split_functions/split_functions.hpp b/src/mlpack/methods/decision_tree/split_functions/split_functions.hpp
index 067be112ec..63d1d5cb5a 100644
--- a/src/mlpack/methods/decision_tree/split_functions/split_functions.hpp
+++ b/src/mlpack/methods/decision_tree/split_functions/split_functions.hpp
@@ -1,3 +1,4 @@
 #include "all_categorical_split.hpp"
 #include "best_binary_numeric_split.hpp"
-#include "random_binary_numeric_split.hpp"
\ No newline at end of file
+#include "random_binary_numeric_split.hpp"
+#include "best_binary_categorical_split.hpp"
\ No newline at end of file

From 45cc092bd15482eef000b3b1692bc7be60d46128 Mon Sep 17 00:00:00 2001
From: Abhimanyu Dayal <abhimanyudayal1@gmail.com>
Date: Mon, 1 Jul 2024 13:22:31 +0530
Subject: [PATCH 12/21] minor fix

---
 src/mlpack/methods/decision_tree/decision_tree.hpp | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/mlpack/methods/decision_tree/decision_tree.hpp b/src/mlpack/methods/decision_tree/decision_tree.hpp
index 373837965e..73d557ebbb 100644
--- a/src/mlpack/methods/decision_tree/decision_tree.hpp
+++ b/src/mlpack/methods/decision_tree/decision_tree.hpp
@@ -15,9 +15,9 @@
 
 #include <mlpack/core.hpp>
 
-#include <split_functions/split_functions.hpp>
-#include <select_functions/select_functions.hpp>
-#include <gain_functions/gain_functions.hpp>
+#include "gain_functions/gain_functions.hpp"
+#include "split_functions/split_functions.hpp"
+#include "select_functions/select_functions.hpp"
 
 namespace mlpack {
 

From 3e932b7c010f7f099b999409431194be876c81dd Mon Sep 17 00:00:00 2001
From: Abhimanyu Dayal <abhimanyudayal1@gmail.com>
Date: Tue, 2 Jul 2024 23:39:18 +0530
Subject: [PATCH 13/21] changed gain_function to fitness_function

---
 src/mlpack/methods/decision_tree/decision_tree.hpp              | 2 +-
 src/mlpack/methods/decision_tree/decision_tree_regressor.hpp    | 2 +-
 .../fitness_functions.hpp}                                      | 0
 .../{gain_functions => fitness_functions}/gini_gain.hpp         | 0
 .../{gain_functions => fitness_functions}/information_gain.hpp  | 0
 .../{gain_functions => fitness_functions}/mad_gain.hpp          | 0
 .../{gain_functions => fitness_functions}/mse_gain.hpp          | 0
 .../{gain_functions => fitness_functions}/sse_gain.hpp          | 0
 8 files changed, 2 insertions(+), 2 deletions(-)
 rename src/mlpack/methods/decision_tree/{gain_functions/gain_functions.hpp => fitness_functions/fitness_functions.hpp} (100%)
 rename src/mlpack/methods/decision_tree/{gain_functions => fitness_functions}/gini_gain.hpp (100%)
 rename src/mlpack/methods/decision_tree/{gain_functions => fitness_functions}/information_gain.hpp (100%)
 rename src/mlpack/methods/decision_tree/{gain_functions => fitness_functions}/mad_gain.hpp (100%)
 rename src/mlpack/methods/decision_tree/{gain_functions => fitness_functions}/mse_gain.hpp (100%)
 rename src/mlpack/methods/decision_tree/{gain_functions => fitness_functions}/sse_gain.hpp (100%)

diff --git a/src/mlpack/methods/decision_tree/decision_tree.hpp b/src/mlpack/methods/decision_tree/decision_tree.hpp
index 73d557ebbb..244cdec24e 100644
--- a/src/mlpack/methods/decision_tree/decision_tree.hpp
+++ b/src/mlpack/methods/decision_tree/decision_tree.hpp
@@ -15,7 +15,7 @@
 
 #include <mlpack/core.hpp>
 
-#include "gain_functions/gain_functions.hpp"
+#include "fitness_functions/fitness_functions.hpp"
 #include "split_functions/split_functions.hpp"
 #include "select_functions/select_functions.hpp"
 
diff --git a/src/mlpack/methods/decision_tree/decision_tree_regressor.hpp b/src/mlpack/methods/decision_tree/decision_tree_regressor.hpp
index be4f6d975c..41ea252475 100644
--- a/src/mlpack/methods/decision_tree/decision_tree_regressor.hpp
+++ b/src/mlpack/methods/decision_tree/decision_tree_regressor.hpp
@@ -15,7 +15,7 @@
 
 #include <mlpack/core.hpp>
 
-#include "gain_functions/gain_functions.hpp"
+#include "fitness_functions/fitness_functions.hpp"
 #include "split_functions/split_functions.hpp"
 #include "select_functions/select_functions.hpp"
 
diff --git a/src/mlpack/methods/decision_tree/gain_functions/gain_functions.hpp b/src/mlpack/methods/decision_tree/fitness_functions/fitness_functions.hpp
similarity index 100%
rename from src/mlpack/methods/decision_tree/gain_functions/gain_functions.hpp
rename to src/mlpack/methods/decision_tree/fitness_functions/fitness_functions.hpp
diff --git a/src/mlpack/methods/decision_tree/gain_functions/gini_gain.hpp b/src/mlpack/methods/decision_tree/fitness_functions/gini_gain.hpp
similarity index 100%
rename from src/mlpack/methods/decision_tree/gain_functions/gini_gain.hpp
rename to src/mlpack/methods/decision_tree/fitness_functions/gini_gain.hpp
diff --git a/src/mlpack/methods/decision_tree/gain_functions/information_gain.hpp b/src/mlpack/methods/decision_tree/fitness_functions/information_gain.hpp
similarity index 100%
rename from src/mlpack/methods/decision_tree/gain_functions/information_gain.hpp
rename to src/mlpack/methods/decision_tree/fitness_functions/information_gain.hpp
diff --git a/src/mlpack/methods/decision_tree/gain_functions/mad_gain.hpp b/src/mlpack/methods/decision_tree/fitness_functions/mad_gain.hpp
similarity index 100%
rename from src/mlpack/methods/decision_tree/gain_functions/mad_gain.hpp
rename to src/mlpack/methods/decision_tree/fitness_functions/mad_gain.hpp
diff --git a/src/mlpack/methods/decision_tree/gain_functions/mse_gain.hpp b/src/mlpack/methods/decision_tree/fitness_functions/mse_gain.hpp
similarity index 100%
rename from src/mlpack/methods/decision_tree/gain_functions/mse_gain.hpp
rename to src/mlpack/methods/decision_tree/fitness_functions/mse_gain.hpp
diff --git a/src/mlpack/methods/decision_tree/gain_functions/sse_gain.hpp b/src/mlpack/methods/decision_tree/fitness_functions/sse_gain.hpp
similarity index 100%
rename from src/mlpack/methods/decision_tree/gain_functions/sse_gain.hpp
rename to src/mlpack/methods/decision_tree/fitness_functions/sse_gain.hpp

From 389bb88db3e6fa395be38af1514a7e4fffa758a2 Mon Sep 17 00:00:00 2001
From: Abhimanyu Dayal <abhimanyudayal1@gmail.com>
Date: Mon, 15 Jul 2024 19:10:54 +0530
Subject: [PATCH 14/21] revert sse_gain

---
 .../loss_functions/sse_loss.hpp}              | 23 +++++++------------
 1 file changed, 8 insertions(+), 15 deletions(-)
 rename src/mlpack/methods/{decision_tree/fitness_functions/sse_gain.hpp => xgboost/loss_functions/sse_loss.hpp} (81%)

diff --git a/src/mlpack/methods/decision_tree/fitness_functions/sse_gain.hpp b/src/mlpack/methods/xgboost/loss_functions/sse_loss.hpp
similarity index 81%
rename from src/mlpack/methods/decision_tree/fitness_functions/sse_gain.hpp
rename to src/mlpack/methods/xgboost/loss_functions/sse_loss.hpp
index b369128aaa..1029fa06af 100644
--- a/src/mlpack/methods/decision_tree/fitness_functions/sse_gain.hpp
+++ b/src/mlpack/methods/xgboost/loss_functions/sse_loss.hpp
@@ -1,8 +1,8 @@
 /**
- * @file methods/decision_tree/gain_functions/sse_gain.hpp
+ * @file methods/xgboost/loss_functions/sse_loss.hpp
  * @author Rishabh Garg
  *
- * The sum of squared error loss class, which is a loss function for gradient
+ * The sum of squared error loss class, which is a loss funtion for gradient
  * xgboost based decision trees.
  *
  * mlpack is free software; you may redistribute it and/or modify it under the
@@ -10,8 +10,8 @@
  * 3-clause BSD license along with mlpack.  If not, see
  * http://www.opensource.org/licenses/BSD-3-Clause for more information.
  */
-#ifndef MLPACK_METHODS_DECISION_TREE_SSE_GAIN_HPP
-#define MLPACK_METHODS_DECISION_TREE_SSE_GAIN_HPP
+#ifndef MLPACK_METHODS_XGBOOST_LOSS_FUNCTIONS_SSE_LOSS_HPP
+#define MLPACK_METHODS_XGBOOST_LOSS_FUNCTIONS_SSE_LOSS_HPP
 
 #include <mlpack/prereqs.hpp>
 
@@ -25,13 +25,13 @@ namespace mlpack {
  *
  * Loss = 1 / 2 * (Observed - Predicted)^2
  */
-class SSEGain
+class SSELoss
 {
  public:
   // Default constructor---No regularization.
-  SSEGain() : alpha(0), lambda(0) { /* Nothing to do. */}
+  SSELoss() : alpha(0), lambda(0) { /* Nothing to do. */}
 
-  SSEGain(const double alpha, const double lambda):
+  SSELoss(const double alpha, const double lambda):
       alpha(alpha), lambda(lambda)
   {
     // Nothing to do.
@@ -66,15 +66,8 @@ class SSEGain
    * @param begin The begin index to calculate gain.
    * @param end The end index to calculate gain.
    */
-  template<bool UseWeights, typename MatType, typename WeightVecType>
-  double Evaluate(const MatType& input, 
-                  const WeightVecType& /* weights */,
-                  const size_t begin, 
-                  const size_t end)
+  double Evaluate(const size_t begin, const size_t end)
   {
-    gradients = (input.row(1) - input.row(0)).t();
-    hessians = arma::vec(input.n_cols, arma::fill::ones);
-
     return std::pow(ApplyL1(accu(gradients.subvec(begin, end))), 2) /
         (accu(hessians.subvec(begin, end)) + lambda);
   }

From d19d9be10b39b7b795705f021483a217c52e9edc Mon Sep 17 00:00:00 2001
From: Abhimanyu Dayal <abhimanyudayal1@gmail.com>
Date: Mon, 15 Jul 2024 19:12:02 +0530
Subject: [PATCH 15/21] revert tests

---
 src/mlpack/tests/xgboost_test.cpp | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/src/mlpack/tests/xgboost_test.cpp b/src/mlpack/tests/xgboost_test.cpp
index 57dbbc53dc..3c02c28b45 100644
--- a/src/mlpack/tests/xgboost_test.cpp
+++ b/src/mlpack/tests/xgboost_test.cpp
@@ -10,7 +10,7 @@
  * http://www.opensource.org/licenses/BSD-3-Clause for more information.
  */
 #include <mlpack/core.hpp>
-#include <mlpack/methods/decision_tree/gain_functions/sse_gain.hpp>
+#include <mlpack/methods/xgboost/loss_functions/sse_loss.hpp>
 
 #include "catch.hpp"
 #include "serialization.hpp"
@@ -18,7 +18,7 @@
 using namespace mlpack;
 
 /**
- * Test that the initial prediction is calculated correctly for SSE gain.
+ * Test that the initial prediction is calculated correctly for SSE loss.
  */
 TEST_CASE("SSEInitialPredictionTest", "[XGBTest]")
 {
@@ -26,12 +26,12 @@ TEST_CASE("SSEInitialPredictionTest", "[XGBTest]")
 
   double initPred = 5.5;
 
-  SSEGain Loss;
+  SSELoss Loss;
   REQUIRE(Loss.InitialPrediction(values) == initPred);
 }
 
 /**
- * Test that output leaf value is calculated correctly for SSE gain.
+ * Test that output leaf value is calculated correctly for SSE Loss.
  */
 TEST_CASE("SSELeafValueTest", "[XGBTest]")
 {
@@ -42,14 +42,14 @@ TEST_CASE("SSELeafValueTest", "[XGBTest]")
   // Actual output leaf value.
   double leafValue = -0.075;
 
-  SSEGain Loss;
+  SSELoss Loss;
   (void) Loss.Evaluate<false>(input, weights);
 
   REQUIRE(Loss.OutputLeafValue(input, weights) == leafValue);
 }
 
 /**
- * Test that the gain is computed correctly for SSE gain.
+ * Test that the gain is computed correctly for SSE Loss.
  */
 TEST_CASE("SSEGainTest", "[XGBTest]")
 {
@@ -60,6 +60,6 @@ TEST_CASE("SSEGainTest", "[XGBTest]")
   // Actual gain value.
   double gain = 0.05625;
 
-  SSEGain Loss;
+  SSELoss Loss;
   REQUIRE(Loss.Evaluate<false>(input, weights) == gain);
-}
+}
\ No newline at end of file

From abbba79f804733a97883fdeccb104b50d907ff4d Mon Sep 17 00:00:00 2001
From: Abhimanyu Dayal <abhimanyudayal1@gmail.com>
Date: Mon, 15 Jul 2024 19:16:19 +0530
Subject: [PATCH 16/21] minor fix

---
 .../methods/decision_tree/fitness_functions/gini_gain.hpp       | 2 +-
 .../decision_tree/fitness_functions/information_gain.hpp        | 2 +-
 src/mlpack/methods/decision_tree/fitness_functions/mad_gain.hpp | 2 +-
 src/mlpack/methods/decision_tree/fitness_functions/mse_gain.hpp | 2 +-
 4 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/mlpack/methods/decision_tree/fitness_functions/gini_gain.hpp b/src/mlpack/methods/decision_tree/fitness_functions/gini_gain.hpp
index e52a23d3af..34f6c4a7fb 100644
--- a/src/mlpack/methods/decision_tree/fitness_functions/gini_gain.hpp
+++ b/src/mlpack/methods/decision_tree/fitness_functions/gini_gain.hpp
@@ -1,5 +1,5 @@
 /**
- * @file methods/decision_tree/gain_functions/gini_gain.hpp
+ * @file methods/decision_tree/fitness_functions/gini_gain.hpp
  * @author Ryan Curtin
  *
  * The GiniGain class, which is a fitness function (FitnessFunction) for
diff --git a/src/mlpack/methods/decision_tree/fitness_functions/information_gain.hpp b/src/mlpack/methods/decision_tree/fitness_functions/information_gain.hpp
index a768886114..9c6719a23a 100644
--- a/src/mlpack/methods/decision_tree/fitness_functions/information_gain.hpp
+++ b/src/mlpack/methods/decision_tree/fitness_functions/information_gain.hpp
@@ -1,5 +1,5 @@
 /**
- * @file methods/decision_tree/gain_functions/information_gain.hpp
+ * @file methods/decision_tree/fitness_functions/information_gain.hpp
  * @author Ryan Curtin
  *
  * An implementation of information gain, which can be used in place of Gini
diff --git a/src/mlpack/methods/decision_tree/fitness_functions/mad_gain.hpp b/src/mlpack/methods/decision_tree/fitness_functions/mad_gain.hpp
index bcf1275e57..908af9b966 100644
--- a/src/mlpack/methods/decision_tree/fitness_functions/mad_gain.hpp
+++ b/src/mlpack/methods/decision_tree/fitness_functions/mad_gain.hpp
@@ -1,5 +1,5 @@
 /**
- * @file methods/decision_tree/gain_functions/mad_gain.hpp
+ * @file methods/decision_tree/fitness_functions/mad_gain.hpp
  * @author Rishabh Garg
  *
  * The mean absolute deviation gain class, a fitness function for regression
diff --git a/src/mlpack/methods/decision_tree/fitness_functions/mse_gain.hpp b/src/mlpack/methods/decision_tree/fitness_functions/mse_gain.hpp
index 8709b3e9ae..77687d2cd1 100644
--- a/src/mlpack/methods/decision_tree/fitness_functions/mse_gain.hpp
+++ b/src/mlpack/methods/decision_tree/fitness_functions/mse_gain.hpp
@@ -1,5 +1,5 @@
 /**
- * @file methods/decision_tree/gain_functions/mse_gain.hpp
+ * @file methods/decision_tree/fitness_functions/mse_gain.hpp
  * @author Rishabh Garg
  *
  * The mean squared error gain class, which is a fitness funtion for

From facd8882ebc29c12af1935d3b686ba04baa8210c Mon Sep 17 00:00:00 2001
From: Abhimanyu Dayal <abhimanyudayal1@gmail.com>
Date: Mon, 15 Jul 2024 19:28:00 +0530
Subject: [PATCH 17/21] minor fix

---
 src/mlpack/tests/xgboost_test.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/mlpack/tests/xgboost_test.cpp b/src/mlpack/tests/xgboost_test.cpp
index 3c02c28b45..4b3d1d533a 100644
--- a/src/mlpack/tests/xgboost_test.cpp
+++ b/src/mlpack/tests/xgboost_test.cpp
@@ -62,4 +62,4 @@ TEST_CASE("SSEGainTest", "[XGBTest]")
 
   SSELoss Loss;
   REQUIRE(Loss.Evaluate<false>(input, weights) == gain);
-}
\ No newline at end of file
+}

From 96946087fdcc603fdf169c8176a2c3f2e4e8f21d Mon Sep 17 00:00:00 2001
From: Abhimanyu Dayal <abhimanyudayal1@gmail.com>
Date: Wed, 17 Jul 2024 19:25:17 +0530
Subject: [PATCH 18/21] minor fix

---
 .../decision_tree/fitness_functions/fitness_functions.hpp      | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/src/mlpack/methods/decision_tree/fitness_functions/fitness_functions.hpp b/src/mlpack/methods/decision_tree/fitness_functions/fitness_functions.hpp
index a73fcd19c6..2ee82c29ca 100644
--- a/src/mlpack/methods/decision_tree/fitness_functions/fitness_functions.hpp
+++ b/src/mlpack/methods/decision_tree/fitness_functions/fitness_functions.hpp
@@ -1,5 +1,4 @@
 #include "gini_gain.hpp"
 #include "information_gain.hpp"
 #include "mad_gain.hpp"
-#include "mse_gain.hpp"
-#include "sse_gain.hpp"
\ No newline at end of file
+#include "mse_gain.hpp"
\ No newline at end of file

From 3943dbeabd7d76f4b80c5f643703261a4d1e6459 Mon Sep 17 00:00:00 2001
From: Abhimanyu Dayal <abhimanyudayal1@gmail.com>
Date: Wed, 17 Jul 2024 19:42:25 +0530
Subject: [PATCH 19/21] minor fix

---
 .../decision_tree/split_functions/best_binary_numeric_split.hpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/mlpack/methods/decision_tree/split_functions/best_binary_numeric_split.hpp b/src/mlpack/methods/decision_tree/split_functions/best_binary_numeric_split.hpp
index 218b4d1408..79c18ee63f 100644
--- a/src/mlpack/methods/decision_tree/split_functions/best_binary_numeric_split.hpp
+++ b/src/mlpack/methods/decision_tree/split_functions/best_binary_numeric_split.hpp
@@ -13,7 +13,7 @@
 #define MLPACK_METHODS_DECISION_TREE_BEST_BINARY_NUMERIC_SPLIT_HPP
 
 #include <mlpack/prereqs.hpp>
-#include <mlpack/methods/decision_tree/gain_functions/mse_gain.hpp>
+#include <mlpack/methods/decision_tree/fitness_functions/mse_gain.hpp>
 
 #include <mlpack/core/util/sfinae_utility.hpp>
 

From c2742c13f15a7b47745d00ec1d1915aea6c35385 Mon Sep 17 00:00:00 2001
From: Abhimanyu Dayal <abhimanyudayal1@gmail.com>
Date: Wed, 17 Jul 2024 19:59:25 +0530
Subject: [PATCH 20/21] minor fix

---
 src/mlpack/methods/decision_tree/decision_tree.hpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/mlpack/methods/decision_tree/decision_tree.hpp b/src/mlpack/methods/decision_tree/decision_tree.hpp
index 244cdec24e..62979e95ac 100644
--- a/src/mlpack/methods/decision_tree/decision_tree.hpp
+++ b/src/mlpack/methods/decision_tree/decision_tree.hpp
@@ -605,7 +605,7 @@ typedef DecisionTree<InformationGain,
 /**
  * Convenience typedef for XGBoost trees.
  */
-typedef DecisionTree<SSEGain,
+typedef DecisionTree<MSEGain,
                      BestBinaryNumericSplit,
                      AllCategoricalSplit,
                      AllDimensionSelect,

From 08f6b3fdff7b32466abe23e2310c607dfa59ad1a Mon Sep 17 00:00:00 2001
From: Abhimanyu Dayal <abhimanyudayal1@gmail.com>
Date: Wed, 17 Jul 2024 20:04:27 +0530
Subject: [PATCH 21/21] removed xgbtree

---
 src/mlpack/methods/decision_tree/decision_tree.hpp | 9 ---------
 1 file changed, 9 deletions(-)

diff --git a/src/mlpack/methods/decision_tree/decision_tree.hpp b/src/mlpack/methods/decision_tree/decision_tree.hpp
index 62979e95ac..eec026c92f 100644
--- a/src/mlpack/methods/decision_tree/decision_tree.hpp
+++ b/src/mlpack/methods/decision_tree/decision_tree.hpp
@@ -601,15 +601,6 @@ typedef DecisionTree<InformationGain,
                      AllCategoricalSplit,
                      AllDimensionSelect,
                      true> ID3DecisionStump;
-
-/**
- * Convenience typedef for XGBoost trees.
- */
-typedef DecisionTree<MSEGain,
-                     BestBinaryNumericSplit,
-                     AllCategoricalSplit,
-                     AllDimensionSelect,
-                     true> XGBTree;
 } // namespace mlpack
 
 // Include implementation.