From 73635050022cc68523467ba563536b610a810c1f Mon Sep 17 00:00:00 2001
From: Xu Kaiqiang <rickllykqxu@gmail.com>
Date: Sat, 6 Apr 2019 16:48:55 +0800
Subject: [PATCH 01/27] change header file to use ID3DecisionStump in
 decision_tree.h as weak learner

---
 src/mlpack/methods/adaboost/adaboost.hpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/mlpack/methods/adaboost/adaboost.hpp b/src/mlpack/methods/adaboost/adaboost.hpp
index 7eb0ea999e..61d74e6c14 100644
--- a/src/mlpack/methods/adaboost/adaboost.hpp
+++ b/src/mlpack/methods/adaboost/adaboost.hpp
@@ -30,7 +30,7 @@
 
 #include <mlpack/prereqs.hpp>
 #include <mlpack/methods/perceptron/perceptron.hpp>
-#include <mlpack/methods/decision_stump/decision_stump.hpp>
+#include <mlpack/methods/decision_tree/decision_tree.hpp>
 
 namespace mlpack {
 namespace adaboost {

From 75712d223459a82f64c3766bf35f24866ed8062d Mon Sep 17 00:00:00 2001
From: Xu Kaiqiang <rickllykqxu@gmail.com>
Date: Sat, 6 Apr 2019 16:50:46 +0800
Subject: [PATCH 02/27] add a comment about initialization.

---
 src/mlpack/methods/adaboost/adaboost_impl.hpp | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/mlpack/methods/adaboost/adaboost_impl.hpp b/src/mlpack/methods/adaboost/adaboost_impl.hpp
index 6a24b6ced4..0095f73cf8 100644
--- a/src/mlpack/methods/adaboost/adaboost_impl.hpp
+++ b/src/mlpack/methods/adaboost/adaboost_impl.hpp
@@ -121,6 +121,8 @@ double AdaBoost<WeakLearnerType, MatType>::Train(
     weights = arma::sum(D);
 
     // Use the existing weak learner to train a new one with new weights.
+    // In fact, the new weak learner just inherits some hyperparameters
+    // from existing weak learner.
     WeakLearnerType w(other, tempData, labels, numClasses, weights);
     w.Classify(tempData, predictedLabels);
 

From d89486897742b9c65c5e47bac1e07d89ddaf9a4f Mon Sep 17 00:00:00 2001
From: Xu Kaiqiang <rickllykqxu@gmail.com>
Date: Sat, 6 Apr 2019 16:51:59 +0800
Subject: [PATCH 03/27] change namespace

---
 src/mlpack/methods/adaboost/adaboost_main.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/mlpack/methods/adaboost/adaboost_main.cpp b/src/mlpack/methods/adaboost/adaboost_main.cpp
index ced3d3c1f2..2dd43d1767 100644
--- a/src/mlpack/methods/adaboost/adaboost_main.cpp
+++ b/src/mlpack/methods/adaboost/adaboost_main.cpp
@@ -42,7 +42,7 @@ using namespace mlpack;
 using namespace std;
 using namespace arma;
 using namespace mlpack::adaboost;
-using namespace mlpack::decision_stump;
+using namespace mlpack::tree;
 using namespace mlpack::perceptron;
 using namespace mlpack::util;
 

From 097294fbac17bed0e0060d19abf5681c2cf92497 Mon Sep 17 00:00:00 2001
From: Xu Kaiqiang <rickllykqxu@gmail.com>
Date: Sat, 6 Apr 2019 16:55:05 +0800
Subject: [PATCH 04/27] replace decision_stump::DecisionStump<> to
 tree::DecisionTree::ID3DecisionStump as weak learner

---
 src/mlpack/methods/adaboost/adaboost_model.cpp | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/src/mlpack/methods/adaboost/adaboost_model.cpp b/src/mlpack/methods/adaboost/adaboost_model.cpp
index e82ebf65a0..869d127563 100644
--- a/src/mlpack/methods/adaboost/adaboost_model.cpp
+++ b/src/mlpack/methods/adaboost/adaboost_model.cpp
@@ -16,7 +16,7 @@ using namespace mlpack;
 using namespace std;
 using namespace arma;
 using namespace mlpack::adaboost;
-using namespace mlpack::decision_stump;
+using namespace mlpack::tree;
 using namespace mlpack::perceptron;
 
 //! Create an empty AdaBoost model.
@@ -47,7 +47,7 @@ AdaBoostModel::AdaBoostModel(const AdaBoostModel& other) :
     mappings(other.mappings),
     weakLearnerType(other.weakLearnerType),
     dsBoost(other.dsBoost == NULL ? NULL :
-        new AdaBoost<DecisionStump<>>(*other.dsBoost)),
+        new AdaBoost<ID3DecisionStump>(*other.dsBoost)),
     pBoost(other.pBoost == NULL ? NULL :
         new AdaBoost<Perceptron<>>(*other.pBoost)),
     dimensionality(other.dimensionality)
@@ -77,7 +77,7 @@ AdaBoostModel& AdaBoostModel::operator=(const AdaBoostModel& other)
 
   delete dsBoost;
   dsBoost = (other.dsBoost == NULL) ? NULL :
-      new AdaBoost<DecisionStump<>>(*other.dsBoost);
+      new AdaBoost<ID3DecisionStump>(*other.dsBoost);
 
   delete pBoost;
   pBoost = (other.pBoost == NULL) ? NULL :
@@ -105,13 +105,13 @@ void AdaBoostModel::Train(const mat& data,
   if (weakLearnerType == WeakLearnerTypes::DECISION_STUMP)
   {
     delete dsBoost;
-
-    DecisionStump<> ds(data, labels, max(labels) + 1);
-    dsBoost = new AdaBoost<DecisionStump<>>(data, labels, numClasses, ds,
+    ID3DecisionStump ds(data, labels, max(labels) + 1);
+    dsBoost = new AdaBoost<ID3DecisionStump>(data, labels, numClasses, ds,
         iterations, tolerance);
   }
   else if (weakLearnerType == WeakLearnerTypes::PERCEPTRON)
   {
+    delete pBoost;
     Perceptron<> p(data, labels, max(labels) + 1);
     pBoost = new AdaBoost<Perceptron<>>(data, labels, numClasses, p, iterations,
         tolerance);

From 5ede1204959b4db8d811084b374c34248ba2335c Mon Sep 17 00:00:00 2001
From: Xu Kaiqiang <rickllykqxu@gmail.com>
Date: Sat, 6 Apr 2019 16:55:30 +0800
Subject: [PATCH 05/27] replace decision_stump::DecisionStump<> to
 tree::DecisionTree::ID3DecisionStump as weak learner

---
 src/mlpack/methods/adaboost/adaboost_model.hpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/mlpack/methods/adaboost/adaboost_model.hpp b/src/mlpack/methods/adaboost/adaboost_model.hpp
index 95ac52b19f..174f230165 100644
--- a/src/mlpack/methods/adaboost/adaboost_model.hpp
+++ b/src/mlpack/methods/adaboost/adaboost_model.hpp
@@ -38,7 +38,7 @@ class AdaBoostModel
   //! The type of weak learner.
   size_t weakLearnerType;
   //! Non-NULL if using decision stumps.
-  AdaBoost<decision_stump::DecisionStump<>>* dsBoost;
+  AdaBoost<tree::ID3DecisionStump>* dsBoost;
   //! Non-NULL if using perceptrons.
   AdaBoost<perceptron::Perceptron<>>* pBoost;
   //! Number of dimensions in training data.
@@ -79,7 +79,7 @@ class AdaBoostModel
   //! Modify the dimensionality of the model.
   size_t& Dimensionality() { return dimensionality; }
 
-  //! Train the model.
+  //! Train the model, treat the data is all of the numeric type.
   void Train(const arma::mat& data,
              const arma::Row<size_t>& labels,
              const size_t numClasses,

From cb378402072e3c989da4e52210093d1227c37d49 Mon Sep 17 00:00:00 2001
From: Xu Kaiqiang <rickllykqxu@gmail.com>
Date: Sat, 6 Apr 2019 16:56:54 +0800
Subject: [PATCH 06/27] comment original test cases for
 Adaboost<DecisionStump<>>

---
 src/mlpack/tests/adaboost_test.cpp | 1194 ++++++++++++++--------------
 1 file changed, 597 insertions(+), 597 deletions(-)

diff --git a/src/mlpack/tests/adaboost_test.cpp b/src/mlpack/tests/adaboost_test.cpp
index 11e22a0295..633138dc33 100644
--- a/src/mlpack/tests/adaboost_test.cpp
+++ b/src/mlpack/tests/adaboost_test.cpp
@@ -19,7 +19,7 @@
 using namespace arma;
 using namespace mlpack;
 using namespace mlpack::adaboost;
-using namespace mlpack::decision_stump;
+using namespace mlpack::tree;
 using namespace mlpack::perceptron;
 
 BOOST_AUTO_TEST_SUITE(AdaBoostTest);
@@ -294,601 +294,601 @@ BOOST_AUTO_TEST_CASE(WeakLearnerErrorNonLinearSepData)
  * checks whether the Hamming loss breaches the upper bound, which is provided
  * by ztAccumulator.  This uses decision stumps as the weak learner.
  */
-BOOST_AUTO_TEST_CASE(HammingLossIris_DS)
-{
-  arma::mat inputData;
-  if (!data::Load("iris.csv", inputData))
-    BOOST_FAIL("Cannot load test dataset iris.csv!");
-
-  arma::Mat<size_t> labels;
-  if (!data::Load("iris_labels.txt", labels))
-    BOOST_FAIL("Cannot load labels for iris_labels.txt");
-
-  // Define your own weak learner, decision stumps in this case.
-  const size_t numClasses = 3;
-  const size_t inpBucketSize = 6;
-  DecisionStump<> ds(inputData, labels.row(0), numClasses, inpBucketSize);
-
-  // Define parameters for AdaBoost.
-  size_t iterations = 50;
-  double tolerance = 1e-10;
-  AdaBoost<DecisionStump<>> a(inputData, labels.row(0), numClasses, ds,
-      iterations, tolerance);
-
-  arma::Row<size_t> predictedLabels;
-  a.Classify(inputData, predictedLabels);
-
-  size_t countError = 0;
-  for (size_t i = 0; i < labels.n_cols; i++)
-    if (labels(i) != predictedLabels(i))
-      countError++;
-  double hammingLoss = (double) countError / labels.n_cols;
-
-  BOOST_REQUIRE_LE(hammingLoss, a.ZtProduct());
-}
-
-/**
- * This test case runs the AdaBoost.mh algorithm on a non-linearly separable
- * dataset.  It checks if the error returned by running a single instance of the
- * weak learner is worse than running the boosted weak learner using adaboost.
- * This is for the weak learner: decision stumps.
- */
-BOOST_AUTO_TEST_CASE(WeakLearnerErrorIris_DS)
-{
-  arma::mat inputData;
-  if (!data::Load("iris.csv", inputData))
-    BOOST_FAIL("Cannot load test dataset iris.csv!");
-
-  arma::Mat<size_t> labels;
-  if (!data::Load("iris_labels.txt", labels))
-    BOOST_FAIL("Cannot load labels for iris_labels.txt");
-
-  // no need to map the labels here
-
-  // Define your own weak learner, decision stumps in this case.
-  const size_t numClasses = 3;
-  const size_t inpBucketSize = 6;
-
-  arma::Row<size_t> dsPrediction(labels.n_cols);
-
-  DecisionStump<> ds(inputData, labels.row(0), numClasses, inpBucketSize);
-  ds.Classify(inputData, dsPrediction);
-
-  size_t countWeakLearnerError = 0;
-  for (size_t i = 0; i < labels.n_cols; i++)
-    if (labels(i) != dsPrediction(i))
-      countWeakLearnerError++;
-  double weakLearnerErrorRate = (double) countWeakLearnerError / labels.n_cols;
-
-  // Define parameters for AdaBoost.
-  size_t iterations = 50;
-  double tolerance = 1e-10;
-
-  AdaBoost<DecisionStump<>> a(inputData, labels.row(0), numClasses, ds,
-      iterations, tolerance);
-
-  arma::Row<size_t> predictedLabels;
-  a.Classify(inputData, predictedLabels);
-
-  size_t countError = 0;
-  for (size_t i = 0; i < labels.n_cols; i++)
-    if (labels(i) != predictedLabels(i))
-      countError++;
-  double error = (double) countError / labels.n_cols;
-
-  BOOST_REQUIRE_LE(error, weakLearnerErrorRate);
-}
-
-/**
- * This test case runs the AdaBoost.mh algorithm on the UCI Vertebral Column
- * dataset.  It checks if the error returned by running a single instance of the
- * weak learner is worse than running the boosted weak learner using adaboost.
- * This is for the weak learner: decision stumps.
- */
-BOOST_AUTO_TEST_CASE(HammingLossBoundVertebralColumn_DS)
-{
-  arma::mat inputData;
-  if (!data::Load("vc2.csv", inputData))
-    BOOST_FAIL("Cannot load test dataset vc2.csv!");
-
-  arma::Mat<size_t> labels;
-  if (!data::Load("vc2_labels.txt", labels))
-    BOOST_FAIL("Cannot load labels for vc2_labels.txt");
-
-  // Define your own weak learner, decision stumps in this case.
-  const size_t numClasses = 3;
-  const size_t inpBucketSize = 6;
-
-  DecisionStump<> ds(inputData, labels.row(0), numClasses, inpBucketSize);
-
-  // Define parameters for AdaBoost.
-  size_t iterations = 50;
-  double tolerance = 1e-10;
-
-  AdaBoost<DecisionStump<>> a(inputData, labels.row(0), numClasses, ds,
-      iterations, tolerance);
-
-  arma::Row<size_t> predictedLabels;
-  a.Classify(inputData, predictedLabels);
-
-  size_t countError = 0;
-  for (size_t i = 0; i < labels.n_cols; i++)
-    if (labels(i) != predictedLabels(i))
-      countError++;
-  double hammingLoss = (double) countError / labels.n_cols;
-
-  BOOST_REQUIRE_LE(hammingLoss, a.ZtProduct());
-}
-
-/**
- * This test case runs the AdaBoost.mh algorithm on the UCI Vertebral Column
- * dataset.  It checks if the error returned by running a single instance of the
- * weak learner is worse than running the boosted weak learner using adaboost.
- * This is for the weak learner: decision stumps.
- */
-BOOST_AUTO_TEST_CASE(WeakLearnerErrorVertebralColumn_DS)
-{
-  arma::mat inputData;
-  if (!data::Load("vc2.csv", inputData))
-    BOOST_FAIL("Cannot load test dataset vc2.csv!");
-
-  arma::Mat<size_t> labels;
-  if (!data::Load("vc2_labels.txt", labels))
-    BOOST_FAIL("Cannot load labels for vc2_labels.txt");
-
-  // Define your own weak learner, decision stumps in this case.
-  const size_t numClasses = 3;
-  const size_t inpBucketSize = 6;
-  arma::Row<size_t> dsPrediction(labels.n_cols);
-
-  DecisionStump<> ds(inputData, labels.row(0), numClasses, inpBucketSize);
-  ds.Classify(inputData, dsPrediction);
-
-  size_t countWeakLearnerError = 0;
-  for (size_t i = 0; i < labels.n_cols; i++)
-    if (labels(i) != dsPrediction(i))
-      countWeakLearnerError++;
-
-  double weakLearnerErrorRate = (double) countWeakLearnerError / labels.n_cols;
-
-  // Define parameters for AdaBoost.
-  size_t iterations = 50;
-  double tolerance = 1e-10;
-  AdaBoost<DecisionStump<>> a(inputData, labels.row(0), numClasses, ds,
-      iterations, tolerance);
-
-  arma::Row<size_t> predictedLabels;
-  a.Classify(inputData, predictedLabels);
-
-  size_t countError = 0;
-  for (size_t i = 0; i < labels.n_cols; i++)
-    if (labels(i) != predictedLabels(i))
-      countError++;
-  double error = (double) countError / labels.n_cols;
-
-  BOOST_REQUIRE_LE(error, weakLearnerErrorRate);
-}
-
-/**
- * This test case runs the AdaBoost.mh algorithm on non-linearly separable
- * dataset.  It checks whether the hamming loss breaches the upperbound, which
- * is provided by ztAccumulator.  This is for the weak learner: decision stumps.
- */
-BOOST_AUTO_TEST_CASE(HammingLossBoundNonLinearSepData_DS)
-{
-  arma::mat inputData;
-  if (!data::Load("train_nonlinsep.txt", inputData))
-    BOOST_FAIL("Cannot load test dataset train_nonlinsep.txt!");
-
-  arma::Mat<size_t> labels;
-  if (!data::Load("train_labels_nonlinsep.txt", labels))
-    BOOST_FAIL("Cannot load labels for train_labels_nonlinsep.txt");
-
-  // Define your own weak learner, decision stumps in this case.
-  const size_t numClasses = 2;
-  const size_t inpBucketSize = 6;
-
-  DecisionStump<> ds(inputData, labels.row(0), numClasses, inpBucketSize);
-
-  // Define parameters for Adaboost.
-  size_t iterations = 50;
-  double tolerance = 1e-10;
-
-  AdaBoost<DecisionStump<> > a(inputData, labels.row(0), numClasses, ds,
-      iterations, tolerance);
-
-  arma::Row<size_t> predictedLabels;
-  a.Classify(inputData, predictedLabels);
-
-  size_t countError = 0;
-  for (size_t i = 0; i < labels.n_cols; i++)
-    if (labels(i) != predictedLabels(i))
-      countError++;
-  double hammingLoss = (double) countError / labels.n_cols;
-
-  BOOST_REQUIRE_LE(hammingLoss, a.ZtProduct());
-}
-
-/**
- * This test case runs the AdaBoost.mh algorithm on a non-linearly separable
- * dataset.  It checks if the error returned by running a single instance of the
- * weak learner is worse than running the boosted weak learner using adaboost.
- * This for the weak learner: decision stumps.
- */
-BOOST_AUTO_TEST_CASE(WeakLearnerErrorNonLinearSepData_DS)
-{
-  arma::mat inputData;
-  if (!data::Load("train_nonlinsep.txt", inputData))
-    BOOST_FAIL("Cannot load test dataset train_nonlinsep.txt!");
-
-  arma::Mat<size_t> labels;
-  if (!data::Load("train_labels_nonlinsep.txt", labels))
-    BOOST_FAIL("Cannot load labels for train_labels_nonlinsep.txt");
-
-  // Define your own weak learner, decision stumps in this case.
-  const size_t numClasses = 2;
-  const size_t inpBucketSize = 3;
-
-  arma::Row<size_t> dsPrediction(labels.n_cols);
-
-  DecisionStump<> ds(inputData, labels.row(0), numClasses, inpBucketSize);
-  ds.Classify(inputData, dsPrediction);
-
-  size_t countWeakLearnerError = 0;
-  for (size_t i = 0; i < labels.n_cols; i++)
-    if (labels(i) != dsPrediction(i))
-      countWeakLearnerError++;
-  double weakLearnerErrorRate = (double) countWeakLearnerError / labels.n_cols;
-
-  // Define parameters for AdaBoost.
-  size_t iterations = 500;
-  double tolerance = 1e-23;
-
-  AdaBoost<DecisionStump<> > a(inputData, labels.row(0), numClasses, ds,
-      iterations, tolerance);
-
-  arma::Row<size_t> predictedLabels;
-  a.Classify(inputData, predictedLabels);
-
-  size_t countError = 0;
-  for (size_t i = 0; i < labels.n_cols; i++)
-    if (labels(i) != predictedLabels(i))
-      countError++;
-  double error = (double) countError / labels.n_cols;
-
-  BOOST_REQUIRE_LE(error, weakLearnerErrorRate);
-}
-
-/**
- * This test case runs the AdaBoost.mh algorithm on the UCI Vertebral Column
- * dataset.  It tests the Classify function and checks for a satisfactory error
- * rate.
- */
-BOOST_AUTO_TEST_CASE(ClassifyTest_VERTEBRALCOL)
-{
-  arma::mat inputData;
-  if (!data::Load("vc2.csv", inputData))
-    BOOST_FAIL("Cannot load test dataset vc2.csv!");
-
-  arma::Mat<size_t> labels;
-  if (!data::Load("vc2_labels.txt", labels))
-    BOOST_FAIL("Cannot load labels for vc2_labels.txt");
-
-  // Define your own weak learner, perceptron in this case.
-  // Run the perceptron for perceptronIter iterations.
-  size_t perceptronIter = 1000;
-
-  arma::mat testData;
-
-  if (!data::Load("vc2_test.csv", testData))
-    BOOST_FAIL("Cannot load test dataset vc2_test.csv!");
-
-  arma::Mat<size_t> trueTestLabels;
-
-  if (!data::Load("vc2_test_labels.txt", trueTestLabels))
-    BOOST_FAIL("Cannot load labels for vc2_test_labels.txt");
-
-  const size_t numClasses = max(labels.row(0)) + 1;
-
-  Row<size_t> perceptronPrediction(labels.n_cols);
-  Perceptron<> p(inputData, labels.row(0), numClasses, perceptronIter);
-  p.Classify(inputData, perceptronPrediction);
-
-  // Define parameters for AdaBoost.
-  size_t iterations = 100;
-  double tolerance = 1e-10;
-  AdaBoost<> a(inputData, labels.row(0), numClasses, p, iterations, tolerance);
-
-  arma::Row<size_t> predictedLabels(testData.n_cols);
-  a.Classify(testData, predictedLabels);
-
-  size_t localError = 0;
-  for (size_t i = 0; i < trueTestLabels.n_cols; i++)
-    if (trueTestLabels(i) != predictedLabels(i))
-      localError++;
-
-  double lError = (double) localError / trueTestLabels.n_cols;
-  BOOST_REQUIRE_LE(lError, 0.30);
-}
-
-/**
- * This test case runs the AdaBoost.mh algorithm on a non linearly separable
- * dataset.  It tests the Classify function and checks for a satisfactory error
- * rate.
- */
-BOOST_AUTO_TEST_CASE(ClassifyTest_NONLINSEP)
-{
-  arma::mat inputData;
-  if (!data::Load("train_nonlinsep.txt", inputData))
-    BOOST_FAIL("Cannot load test dataset train_nonlinsep.txt!");
-
-  arma::Mat<size_t> labels;
-  if (!data::Load("train_labels_nonlinsep.txt", labels))
-    BOOST_FAIL("Cannot load labels for train_labels_nonlinsep.txt");
-
-  // Define your own weak learner; in this test decision stumps are used.
-  const size_t numClasses = 2;
-  const size_t inpBucketSize = 3;
-
-  arma::mat testData;
-
-  if (!data::Load("test_nonlinsep.txt", testData))
-    BOOST_FAIL("Cannot load test dataset test_nonlinsep.txt!");
-
-  arma::Mat<size_t> trueTestLabels;
-
-  if (!data::Load("test_labels_nonlinsep.txt", trueTestLabels))
-    BOOST_FAIL("Cannot load labels for test_labels_nonlinsep.txt");
-
-  arma::Row<size_t> dsPrediction(labels.n_cols);
-
-  DecisionStump<> ds(inputData, labels.row(0), numClasses, inpBucketSize);
-
-  // Define parameters for AdaBoost.
-  size_t iterations = 50;
-  double tolerance = 1e-10;
-  AdaBoost<DecisionStump<> > a(inputData, labels.row(0), numClasses, ds,
-      iterations, tolerance);
-
-  arma::Row<size_t> predictedLabels(testData.n_cols);
-  a.Classify(testData, predictedLabels);
-
-  size_t localError = 0;
-  for (size_t i = 0; i < trueTestLabels.n_cols; i++)
-    if (trueTestLabels(i) != predictedLabels(i))
-      localError++;
-
-  double lError = (double) localError / trueTestLabels.n_cols;
-  BOOST_REQUIRE_LE(lError, 0.30);
-}
-
-/**
- * This test case runs the AdaBoost.mh algorithm on the UCI Iris Dataset.  It
- * trains it on two thirds of the Iris dataset (iris_train.csv), and tests on
- * the remaining third of the dataset (iris_test.csv).  It tests the Classify()
- * function and checks for a satisfactory error rate.
- */
-BOOST_AUTO_TEST_CASE(ClassifyTest_IRIS)
-{
-  arma::mat inputData;
-  if (!data::Load("iris_train.csv", inputData))
-    BOOST_FAIL("Cannot load test dataset iris_train.csv!");
-
-  arma::Mat<size_t> labels;
-  if (!data::Load("iris_train_labels.csv", labels))
-    BOOST_FAIL("Cannot load labels for iris_train_labels.csv");
-  const size_t numClasses = max(labels.row(0)) + 1;
-
-  // Define your own weak learner, perceptron in this case.
-  // Run the perceptron for perceptronIter iterations.
-  size_t perceptronIter = 800;
-
-  Perceptron<> p(inputData, labels.row(0), numClasses, perceptronIter);
-
-  // Define parameters for AdaBoost.
-  size_t iterations = 50;
-  double tolerance = 1e-10;
-  AdaBoost<> a(inputData, labels.row(0), numClasses, p, iterations, tolerance);
-
-  arma::mat testData;
-  if (!data::Load("iris_test.csv", testData))
-    BOOST_FAIL("Cannot load test dataset iris_test.csv!");
-
-  arma::Row<size_t> predictedLabels(testData.n_cols);
-
-  a.Classify(testData, predictedLabels);
-
-  arma::Mat<size_t> trueTestLabels;
-  if (!data::Load("iris_test_labels.csv", trueTestLabels))
-    BOOST_FAIL("Cannot load test dataset iris_test_labels.csv!");
-
-  size_t localError = 0;
-  for (size_t i = 0; i < trueTestLabels.n_cols; i++)
-    if (trueTestLabels(i) != predictedLabels(i))
-      localError++;
-  double lError = (double) localError / labels.n_cols;
-  BOOST_REQUIRE_LE(lError, 0.30);
-}
-
-/**
- * Ensure that the Train() function works like it is supposed to, by building
- * AdaBoost on one dataset and then re-training on another dataset.
- */
-BOOST_AUTO_TEST_CASE(TrainTest)
-{
-  // First train on the iris dataset.
-  arma::mat inputData;
-  if (!data::Load("iris_train.csv", inputData))
-    BOOST_FAIL("Cannot load test dataset iris_train.csv!");
-
-  arma::Mat<size_t> labels;
-  if (!data::Load("iris_train_labels.csv", labels))
-    BOOST_FAIL("Cannot load labels for iris_train_labels.csv");
-
-  const size_t numClasses = max(labels.row(0)) + 1;
-
-  size_t perceptronIter = 800;
-  Perceptron<> p(inputData, labels.row(0), numClasses, perceptronIter);
-
-  // Now train AdaBoost.
-  size_t iterations = 50;
-  double tolerance = 1e-10;
-  AdaBoost<> a(inputData, labels.row(0), numClasses, p, iterations, tolerance);
-
-  // Now load another dataset...
-  if (!data::Load("vc2.csv", inputData))
-    BOOST_FAIL("Cannot load test dataset vc2.csv!");
-  if (!data::Load("vc2_labels.txt", labels))
-    BOOST_FAIL("Cannot load labels for vc2_labels.txt");
-
-  const size_t newNumClasses = max(labels.row(0)) + 1;
-
-  Perceptron<> p2(inputData, labels.row(0), newNumClasses, perceptronIter);
-
-  a.Train(inputData, labels.row(0), newNumClasses, p2, iterations, tolerance);
-
-  // Load test set to see if it trained on vc2 correctly.
-  arma::mat testData;
-  if (!data::Load("vc2_test.csv", testData))
-    BOOST_FAIL("Cannot load test dataset vc2_test.csv!");
-
-  arma::Mat<size_t> trueTestLabels;
-  if (!data::Load("vc2_test_labels.txt", trueTestLabels))
-    BOOST_FAIL("Cannot load labels for vc2_test_labels.txt");
-
-  // Define parameters for AdaBoost.
-  arma::Row<size_t> predictedLabels(testData.n_cols);
-  a.Classify(testData, predictedLabels);
-
-  int localError = 0;
-  for (size_t i = 0; i < trueTestLabels.n_cols; i++)
-    if (trueTestLabels(i) != predictedLabels(i))
-      localError++;
-
-  double lError = (double) localError / trueTestLabels.n_cols;
-
-  BOOST_REQUIRE_LE(lError, 0.30);
-}
-
-BOOST_AUTO_TEST_CASE(PerceptronSerializationTest)
-{
-  // Build an AdaBoost object.
-  mat data = randu<mat>(10, 500);
-  Row<size_t> labels(500);
-  for (size_t i = 0; i < 250; ++i)
-    labels[i] = 0;
-  for (size_t i = 250; i < 500; ++i)
-    labels[i] = 1;
-
-  Perceptron<> p(data, labels, 2, 800);
-  AdaBoost<> ab(data, labels, 2, p, 50, 1e-10);
-
-  // Now create another dataset to train with.
-  mat otherData = randu<mat>(5, 200);
-  Row<size_t> otherLabels(200);
-  for (size_t i = 0; i < 100; ++i)
-    otherLabels[i] = 1;
-  for (size_t i = 100; i < 150; ++i)
-    otherLabels[i] = 0;
-  for (size_t i = 150; i < 200; ++i)
-    otherLabels[i] = 2;
-
-  Perceptron<> p2(otherData, otherLabels, 3, 500);
-  AdaBoost<> abText(otherData, otherLabels, 3, p2, 50, 1e-10);
-
-  AdaBoost<> abXml, abBinary;
-
-  SerializeObjectAll(ab, abXml, abText, abBinary);
-
-  // Now check that the objects are the same.
-  BOOST_REQUIRE_CLOSE(ab.Tolerance(), abXml.Tolerance(), 1e-5);
-  BOOST_REQUIRE_CLOSE(ab.Tolerance(), abText.Tolerance(), 1e-5);
-  BOOST_REQUIRE_CLOSE(ab.Tolerance(), abBinary.Tolerance(), 1e-5);
-
-  BOOST_REQUIRE_CLOSE(ab.ZtProduct(), abXml.ZtProduct(), 1e-5);
-  BOOST_REQUIRE_CLOSE(ab.ZtProduct(), abText.ZtProduct(), 1e-5);
-  BOOST_REQUIRE_CLOSE(ab.ZtProduct(), abBinary.ZtProduct(), 1e-5);
-
-  BOOST_REQUIRE_EQUAL(ab.WeakLearners(), abXml.WeakLearners());
-  BOOST_REQUIRE_EQUAL(ab.WeakLearners(), abText.WeakLearners());
-  BOOST_REQUIRE_EQUAL(ab.WeakLearners(), abBinary.WeakLearners());
-
-  for (size_t i = 0; i < ab.WeakLearners(); ++i)
-  {
-    CheckMatrices(ab.WeakLearner(i).Weights(),
-                  abXml.WeakLearner(i).Weights(),
-                  abText.WeakLearner(i).Weights(),
-                  abBinary.WeakLearner(i).Weights());
-
-    CheckMatrices(ab.WeakLearner(i).Biases(),
-                  abXml.WeakLearner(i).Biases(),
-                  abText.WeakLearner(i).Biases(),
-                  abBinary.WeakLearner(i).Biases());
-  }
-}
-
-BOOST_AUTO_TEST_CASE(DecisionStumpSerializationTest)
-{
-  // Build an AdaBoost object.
-  mat data = randu<mat>(10, 500);
-  Row<size_t> labels(500);
-  for (size_t i = 0; i < 250; ++i)
-    labels[i] = 0;
-  for (size_t i = 250; i < 500; ++i)
-    labels[i] = 1;
-
-  DecisionStump<> p(data, labels, 2, 800);
-  AdaBoost<DecisionStump<>> ab(data, labels, 2, p, 50, 1e-10);
-
-  // Now create another dataset to train with.
-  mat otherData = randu<mat>(5, 200);
-  Row<size_t> otherLabels(200);
-  for (size_t i = 0; i < 100; ++i)
-    otherLabels[i] = 1;
-  for (size_t i = 100; i < 150; ++i)
-    otherLabels[i] = 0;
-  for (size_t i = 150; i < 200; ++i)
-    otherLabels[i] = 2;
-
-  DecisionStump<> p2(otherData, otherLabels, 3, 500);
-  AdaBoost<DecisionStump<>> abText(otherData, otherLabels, 3, p2, 50, 1e-10);
-
-  AdaBoost<DecisionStump<>> abXml, abBinary;
-
-  SerializeObjectAll(ab, abXml, abText, abBinary);
-
-  // Now check that the objects are the same.
-  BOOST_REQUIRE_CLOSE(ab.Tolerance(), abXml.Tolerance(), 1e-5);
-  BOOST_REQUIRE_CLOSE(ab.Tolerance(), abText.Tolerance(), 1e-5);
-  BOOST_REQUIRE_CLOSE(ab.Tolerance(), abBinary.Tolerance(), 1e-5);
-
-  BOOST_REQUIRE_CLOSE(ab.ZtProduct(), abXml.ZtProduct(), 1e-5);
-  BOOST_REQUIRE_CLOSE(ab.ZtProduct(), abText.ZtProduct(), 1e-5);
-  BOOST_REQUIRE_CLOSE(ab.ZtProduct(), abBinary.ZtProduct(), 1e-5);
-
-  BOOST_REQUIRE_EQUAL(ab.WeakLearners(), abXml.WeakLearners());
-  BOOST_REQUIRE_EQUAL(ab.WeakLearners(), abText.WeakLearners());
-  BOOST_REQUIRE_EQUAL(ab.WeakLearners(), abBinary.WeakLearners());
-
-  for (size_t i = 0; i < ab.WeakLearners(); ++i)
-  {
-    BOOST_REQUIRE_EQUAL(ab.WeakLearner(i).SplitDimension(),
-                        abXml.WeakLearner(i).SplitDimension());
-    BOOST_REQUIRE_EQUAL(ab.WeakLearner(i).SplitDimension(),
-                        abText.WeakLearner(i).SplitDimension());
-    BOOST_REQUIRE_EQUAL(ab.WeakLearner(i).SplitDimension(),
-                        abBinary.WeakLearner(i).SplitDimension());
-
-    CheckMatrices(ab.WeakLearner(i).Split(),
-                  abXml.WeakLearner(i).Split(),
-                  abText.WeakLearner(i).Split(),
-                  abBinary.WeakLearner(i).Split());
-
-    CheckMatrices(ab.WeakLearner(i).BinLabels(),
-                  abXml.WeakLearner(i).BinLabels(),
-                  abText.WeakLearner(i).BinLabels(),
-                  abBinary.WeakLearner(i).BinLabels());
-  }
-}
+//BOOST_AUTO_TEST_CASE(HammingLossIris_DS)
+//{
+//  arma::mat inputData;
+//  if (!data::Load("iris.csv", inputData))
+//    BOOST_FAIL("Cannot load test dataset iris.csv!");
+//
+//  arma::Mat<size_t> labels;
+//  if (!data::Load("iris_labels.txt", labels))
+//    BOOST_FAIL("Cannot load labels for iris_labels.txt");
+//
+//  // Define your own weak learner, decision stumps in this case.
+//  const size_t numClasses = 3;
+//  const size_t inpBucketSize = 6;
+//  ID3DecisionStump ds(inputData, labels.row(0), numClasses, inpBucketSize);
+//
+//  // Define parameters for AdaBoost.
+//  size_t iterations = 50;
+//  double tolerance = 1e-10;
+//  AdaBoost<ID3DecisionStump> a(inputData, labels.row(0), numClasses, ds,
+//      iterations, tolerance);
+//
+//  arma::Row<size_t> predictedLabels;
+//  a.Classify(inputData, predictedLabels);
+//
+//  size_t countError = 0;
+//  for (size_t i = 0; i < labels.n_cols; i++)
+//    if (labels(i) != predictedLabels(i))
+//      countError++;
+//  double hammingLoss = (double) countError / labels.n_cols;
+//
+//  BOOST_REQUIRE_LE(hammingLoss, a.ZtProduct());
+//}
+//
+///**
+// * This test case runs the AdaBoost.mh algorithm on a non-linearly separable
+// * dataset.  It checks if the error returned by running a single instance of the
+// * weak learner is worse than running the boosted weak learner using adaboost.
+// * This is for the weak learner: decision stumps.
+// */
+//BOOST_AUTO_TEST_CASE(WeakLearnerErrorIris_DS)
+//{
+//  arma::mat inputData;
+//  if (!data::Load("iris.csv", inputData))
+//    BOOST_FAIL("Cannot load test dataset iris.csv!");
+//
+//  arma::Mat<size_t> labels;
+//  if (!data::Load("iris_labels.txt", labels))
+//    BOOST_FAIL("Cannot load labels for iris_labels.txt");
+//
+//  // no need to map the labels here
+//
+//  // Define your own weak learner, decision stumps in this case.
+//  const size_t numClasses = 3;
+//  const size_t inpBucketSize = 6;
+//
+//  arma::Row<size_t> dsPrediction(labels.n_cols);
+//
+//  ID3DecisionStump ds(inputData, labels.row(0), numClasses, inpBucketSize);
+//  ds.Classify(inputData, dsPrediction);
+//
+//  size_t countWeakLearnerError = 0;
+//  for (size_t i = 0; i < labels.n_cols; i++)
+//    if (labels(i) != dsPrediction(i))
+//      countWeakLearnerError++;
+//  double weakLearnerErrorRate = (double) countWeakLearnerError / labels.n_cols;
+//
+//  // Define parameters for AdaBoost.
+//  size_t iterations = 50;
+//  double tolerance = 1e-10;
+//
+//  AdaBoost<ID3DecisionStump> a(inputData, labels.row(0), numClasses, ds,
+//      iterations, tolerance);
+//
+//  arma::Row<size_t> predictedLabels;
+//  a.Classify(inputData, predictedLabels);
+//
+//  size_t countError = 0;
+//  for (size_t i = 0; i < labels.n_cols; i++)
+//    if (labels(i) != predictedLabels(i))
+//      countError++;
+//  double error = (double) countError / labels.n_cols;
+//
+//  BOOST_REQUIRE_LE(error, weakLearnerErrorRate);
+//}
+//
+///**
+// * This test case runs the AdaBoost.mh algorithm on the UCI Vertebral Column
+// * dataset.  It checks if the error returned by running a single instance of the
+// * weak learner is worse than running the boosted weak learner using adaboost.
+// * This is for the weak learner: decision stumps.
+// */
+//BOOST_AUTO_TEST_CASE(HammingLossBoundVertebralColumn_DS)
+//{
+//  arma::mat inputData;
+//  if (!data::Load("vc2.csv", inputData))
+//    BOOST_FAIL("Cannot load test dataset vc2.csv!");
+//
+//  arma::Mat<size_t> labels;
+//  if (!data::Load("vc2_labels.txt", labels))
+//    BOOST_FAIL("Cannot load labels for vc2_labels.txt");
+//
+//  // Define your own weak learner, decision stumps in this case.
+//  const size_t numClasses = 3;
+//  const size_t inpBucketSize = 6;
+//
+//  ID3DecisionStump ds(inputData, labels.row(0), numClasses, inpBucketSize);
+//
+//  // Define parameters for AdaBoost.
+//  size_t iterations = 50;
+//  double tolerance = 1e-10;
+//
+//  AdaBoost<ID3DecisionStump> a(inputData, labels.row(0), numClasses, ds,
+//      iterations, tolerance);
+//
+//  arma::Row<size_t> predictedLabels;
+//  a.Classify(inputData, predictedLabels);
+//
+//  size_t countError = 0;
+//  for (size_t i = 0; i < labels.n_cols; i++)
+//    if (labels(i) != predictedLabels(i))
+//      countError++;
+//  double hammingLoss = (double) countError / labels.n_cols;
+//
+//  BOOST_REQUIRE_LE(hammingLoss, a.ZtProduct());
+//}
+//
+///**
+// * This test case runs the AdaBoost.mh algorithm on the UCI Vertebral Column
+// * dataset.  It checks if the error returned by running a single instance of the
+// * weak learner is worse than running the boosted weak learner using adaboost.
+// * This is for the weak learner: decision stumps.
+// */
+//BOOST_AUTO_TEST_CASE(WeakLearnerErrorVertebralColumn_DS)
+//{
+//  arma::mat inputData;
+//  if (!data::Load("vc2.csv", inputData))
+//    BOOST_FAIL("Cannot load test dataset vc2.csv!");
+//
+//  arma::Mat<size_t> labels;
+//  if (!data::Load("vc2_labels.txt", labels))
+//    BOOST_FAIL("Cannot load labels for vc2_labels.txt");
+//
+//  // Define your own weak learner, decision stumps in this case.
+//  const size_t numClasses = 3;
+//  const size_t inpBucketSize = 6;
+//  arma::Row<size_t> dsPrediction(labels.n_cols);
+//
+//  ID3DecisionStump ds(inputData, labels.row(0), numClasses, inpBucketSize);
+//  ds.Classify(inputData, dsPrediction);
+//
+//  size_t countWeakLearnerError = 0;
+//  for (size_t i = 0; i < labels.n_cols; i++)
+//    if (labels(i) != dsPrediction(i))
+//      countWeakLearnerError++;
+//
+//  double weakLearnerErrorRate = (double) countWeakLearnerError / labels.n_cols;
+//
+//  // Define parameters for AdaBoost.
+//  size_t iterations = 50;
+//  double tolerance = 1e-10;
+//  AdaBoost<ID3DecisionStump> a(inputData, labels.row(0), numClasses, ds,
+//      iterations, tolerance);
+//
+//  arma::Row<size_t> predictedLabels;
+//  a.Classify(inputData, predictedLabels);
+//
+//  size_t countError = 0;
+//  for (size_t i = 0; i < labels.n_cols; i++)
+//    if (labels(i) != predictedLabels(i))
+//      countError++;
+//  double error = (double) countError / labels.n_cols;
+//
+//  BOOST_REQUIRE_LE(error, weakLearnerErrorRate);
+//}
+//
+///**
+// * This test case runs the AdaBoost.mh algorithm on non-linearly separable
+// * dataset.  It checks whether the hamming loss breaches the upperbound, which
+// * is provided by ztAccumulator.  This is for the weak learner: decision stumps.
+// */
+//BOOST_AUTO_TEST_CASE(HammingLossBoundNonLinearSepData_DS)
+//{
+//  arma::mat inputData;
+//  if (!data::Load("train_nonlinsep.txt", inputData))
+//    BOOST_FAIL("Cannot load test dataset train_nonlinsep.txt!");
+//
+//  arma::Mat<size_t> labels;
+//  if (!data::Load("train_labels_nonlinsep.txt", labels))
+//    BOOST_FAIL("Cannot load labels for train_labels_nonlinsep.txt");
+//
+//  // Define your own weak learner, decision stumps in this case.
+//  const size_t numClasses = 2;
+//  const size_t inpBucketSize = 6;
+//
+//  ID3DecisionStump ds(inputData, labels.row(0), numClasses, inpBucketSize);
+//
+//  // Define parameters for Adaboost.
+//  size_t iterations = 50;
+//  double tolerance = 1e-10;
+//
+//  AdaBoost<ID3DecisionStump > a(inputData, labels.row(0), numClasses, ds,
+//      iterations, tolerance);
+//
+//  arma::Row<size_t> predictedLabels;
+//  a.Classify(inputData, predictedLabels);
+//
+//  size_t countError = 0;
+//  for (size_t i = 0; i < labels.n_cols; i++)
+//    if (labels(i) != predictedLabels(i))
+//      countError++;
+//  double hammingLoss = (double) countError / labels.n_cols;
+//
+//  BOOST_REQUIRE_LE(hammingLoss, a.ZtProduct());
+//}
+//
+///**
+// * This test case runs the AdaBoost.mh algorithm on a non-linearly separable
+// * dataset.  It checks if the error returned by running a single instance of the
+// * weak learner is worse than running the boosted weak learner using adaboost.
+// * This for the weak learner: decision stumps.
+// */
+//BOOST_AUTO_TEST_CASE(WeakLearnerErrorNonLinearSepData_DS)
+//{
+//  arma::mat inputData;
+//  if (!data::Load("train_nonlinsep.txt", inputData))
+//    BOOST_FAIL("Cannot load test dataset train_nonlinsep.txt!");
+//
+//  arma::Mat<size_t> labels;
+//  if (!data::Load("train_labels_nonlinsep.txt", labels))
+//    BOOST_FAIL("Cannot load labels for train_labels_nonlinsep.txt");
+//
+//  // Define your own weak learner, decision stumps in this case.
+//  const size_t numClasses = 2;
+//  const size_t inpBucketSize = 3;
+//
+//  arma::Row<size_t> dsPrediction(labels.n_cols);
+//
+//  ID3DecisionStump ds(inputData, labels.row(0), numClasses, inpBucketSize);
+//  ds.Classify(inputData, dsPrediction);
+//
+//  size_t countWeakLearnerError = 0;
+//  for (size_t i = 0; i < labels.n_cols; i++)
+//    if (labels(i) != dsPrediction(i))
+//      countWeakLearnerError++;
+//  double weakLearnerErrorRate = (double) countWeakLearnerError / labels.n_cols;
+//
+//  // Define parameters for AdaBoost.
+//  size_t iterations = 500;
+//  double tolerance = 1e-23;
+//
+//  AdaBoost<ID3DecisionStump > a(inputData, labels.row(0), numClasses, ds,
+//      iterations, tolerance);
+//
+//  arma::Row<size_t> predictedLabels;
+//  a.Classify(inputData, predictedLabels);
+//
+//  size_t countError = 0;
+//  for (size_t i = 0; i < labels.n_cols; i++)
+//    if (labels(i) != predictedLabels(i))
+//      countError++;
+//  double error = (double) countError / labels.n_cols;
+//
+//  BOOST_REQUIRE_LE(error, weakLearnerErrorRate);
+//}
+//
+///**
+// * This test case runs the AdaBoost.mh algorithm on the UCI Vertebral Column
+// * dataset.  It tests the Classify function and checks for a satisfactory error
+// * rate.
+// */
+//BOOST_AUTO_TEST_CASE(ClassifyTest_VERTEBRALCOL)
+//{
+//  arma::mat inputData;
+//  if (!data::Load("vc2.csv", inputData))
+//    BOOST_FAIL("Cannot load test dataset vc2.csv!");
+//
+//  arma::Mat<size_t> labels;
+//  if (!data::Load("vc2_labels.txt", labels))
+//    BOOST_FAIL("Cannot load labels for vc2_labels.txt");
+//
+//  // Define your own weak learner, perceptron in this case.
+//  // Run the perceptron for perceptronIter iterations.
+//  size_t perceptronIter = 1000;
+//
+//  arma::mat testData;
+//
+//  if (!data::Load("vc2_test.csv", testData))
+//    BOOST_FAIL("Cannot load test dataset vc2_test.csv!");
+//
+//  arma::Mat<size_t> trueTestLabels;
+//
+//  if (!data::Load("vc2_test_labels.txt", trueTestLabels))
+//    BOOST_FAIL("Cannot load labels for vc2_test_labels.txt");
+//
+//  const size_t numClasses = max(labels.row(0)) + 1;
+//
+//  Row<size_t> perceptronPrediction(labels.n_cols);
+//  Perceptron<> p(inputData, labels.row(0), numClasses, perceptronIter);
+//  p.Classify(inputData, perceptronPrediction);
+//
+//  // Define parameters for AdaBoost.
+//  size_t iterations = 100;
+//  double tolerance = 1e-10;
+//  AdaBoost<> a(inputData, labels.row(0), numClasses, p, iterations, tolerance);
+//
+//  arma::Row<size_t> predictedLabels(testData.n_cols);
+//  a.Classify(testData, predictedLabels);
+//
+//  size_t localError = 0;
+//  for (size_t i = 0; i < trueTestLabels.n_cols; i++)
+//    if (trueTestLabels(i) != predictedLabels(i))
+//      localError++;
+//
+//  double lError = (double) localError / trueTestLabels.n_cols;
+//  BOOST_REQUIRE_LE(lError, 0.30);
+//}
+//
+///**
+// * This test case runs the AdaBoost.mh algorithm on a non linearly separable
+// * dataset.  It tests the Classify function and checks for a satisfactory error
+// * rate.
+// */
+//BOOST_AUTO_TEST_CASE(ClassifyTest_NONLINSEP)
+//{
+//  arma::mat inputData;
+//  if (!data::Load("train_nonlinsep.txt", inputData))
+//    BOOST_FAIL("Cannot load test dataset train_nonlinsep.txt!");
+//
+//  arma::Mat<size_t> labels;
+//  if (!data::Load("train_labels_nonlinsep.txt", labels))
+//    BOOST_FAIL("Cannot load labels for train_labels_nonlinsep.txt");
+//
+//  // Define your own weak learner; in this test decision stumps are used.
+//  const size_t numClasses = 2;
+//  const size_t inpBucketSize = 3;
+//
+//  arma::mat testData;
+//
+//  if (!data::Load("test_nonlinsep.txt", testData))
+//    BOOST_FAIL("Cannot load test dataset test_nonlinsep.txt!");
+//
+//  arma::Mat<size_t> trueTestLabels;
+//
+//  if (!data::Load("test_labels_nonlinsep.txt", trueTestLabels))
+//    BOOST_FAIL("Cannot load labels for test_labels_nonlinsep.txt");
+//
+//  arma::Row<size_t> dsPrediction(labels.n_cols);
+//
+//  ID3DecisionStump ds(inputData, labels.row(0), numClasses, inpBucketSize);
+//
+//  // Define parameters for AdaBoost.
+//  size_t iterations = 50;
+//  double tolerance = 1e-10;
+//  AdaBoost<ID3DecisionStump > a(inputData, labels.row(0), numClasses, ds,
+//      iterations, tolerance);
+//
+//  arma::Row<size_t> predictedLabels(testData.n_cols);
+//  a.Classify(testData, predictedLabels);
+//
+//  size_t localError = 0;
+//  for (size_t i = 0; i < trueTestLabels.n_cols; i++)
+//    if (trueTestLabels(i) != predictedLabels(i))
+//      localError++;
+//
+//  double lError = (double) localError / trueTestLabels.n_cols;
+//  BOOST_REQUIRE_LE(lError, 0.30);
+//}
+//
+///**
+// * This test case runs the AdaBoost.mh algorithm on the UCI Iris Dataset.  It
+// * trains it on two thirds of the Iris dataset (iris_train.csv), and tests on
+// * the remaining third of the dataset (iris_test.csv).  It tests the Classify()
+// * function and checks for a satisfactory error rate.
+// */
+//BOOST_AUTO_TEST_CASE(ClassifyTest_IRIS)
+//{
+//  arma::mat inputData;
+//  if (!data::Load("iris_train.csv", inputData))
+//    BOOST_FAIL("Cannot load test dataset iris_train.csv!");
+//
+//  arma::Mat<size_t> labels;
+//  if (!data::Load("iris_train_labels.csv", labels))
+//    BOOST_FAIL("Cannot load labels for iris_train_labels.csv");
+//  const size_t numClasses = max(labels.row(0)) + 1;
+//
+//  // Define your own weak learner, perceptron in this case.
+//  // Run the perceptron for perceptronIter iterations.
+//  size_t perceptronIter = 800;
+//
+//  Perceptron<> p(inputData, labels.row(0), numClasses, perceptronIter);
+//
+//  // Define parameters for AdaBoost.
+//  size_t iterations = 50;
+//  double tolerance = 1e-10;
+//  AdaBoost<> a(inputData, labels.row(0), numClasses, p, iterations, tolerance);
+//
+//  arma::mat testData;
+//  if (!data::Load("iris_test.csv", testData))
+//    BOOST_FAIL("Cannot load test dataset iris_test.csv!");
+//
+//  arma::Row<size_t> predictedLabels(testData.n_cols);
+//
+//  a.Classify(testData, predictedLabels);
+//
+//  arma::Mat<size_t> trueTestLabels;
+//  if (!data::Load("iris_test_labels.csv", trueTestLabels))
+//    BOOST_FAIL("Cannot load test dataset iris_test_labels.csv!");
+//
+//  size_t localError = 0;
+//  for (size_t i = 0; i < trueTestLabels.n_cols; i++)
+//    if (trueTestLabels(i) != predictedLabels(i))
+//      localError++;
+//  double lError = (double) localError / labels.n_cols;
+//  BOOST_REQUIRE_LE(lError, 0.30);
+//}
+//
+///**
+// * Ensure that the Train() function works like it is supposed to, by building
+// * AdaBoost on one dataset and then re-training on another dataset.
+// */
+//BOOST_AUTO_TEST_CASE(TrainTest)
+//{
+//  // First train on the iris dataset.
+//  arma::mat inputData;
+//  if (!data::Load("iris_train.csv", inputData))
+//    BOOST_FAIL("Cannot load test dataset iris_train.csv!");
+//
+//  arma::Mat<size_t> labels;
+//  if (!data::Load("iris_train_labels.csv", labels))
+//    BOOST_FAIL("Cannot load labels for iris_train_labels.csv");
+//
+//  const size_t numClasses = max(labels.row(0)) + 1;
+//
+//  size_t perceptronIter = 800;
+//  Perceptron<> p(inputData, labels.row(0), numClasses, perceptronIter);
+//
+//  // Now train AdaBoost.
+//  size_t iterations = 50;
+//  double tolerance = 1e-10;
+//  AdaBoost<> a(inputData, labels.row(0), numClasses, p, iterations, tolerance);
+//
+//  // Now load another dataset...
+//  if (!data::Load("vc2.csv", inputData))
+//    BOOST_FAIL("Cannot load test dataset vc2.csv!");
+//  if (!data::Load("vc2_labels.txt", labels))
+//    BOOST_FAIL("Cannot load labels for vc2_labels.txt");
+//
+//  const size_t newNumClasses = max(labels.row(0)) + 1;
+//
+//  Perceptron<> p2(inputData, labels.row(0), newNumClasses, perceptronIter);
+//
+//  a.Train(inputData, labels.row(0), newNumClasses, p2, iterations, tolerance);
+//
+//  // Load test set to see if it trained on vc2 correctly.
+//  arma::mat testData;
+//  if (!data::Load("vc2_test.csv", testData))
+//    BOOST_FAIL("Cannot load test dataset vc2_test.csv!");
+//
+//  arma::Mat<size_t> trueTestLabels;
+//  if (!data::Load("vc2_test_labels.txt", trueTestLabels))
+//    BOOST_FAIL("Cannot load labels for vc2_test_labels.txt");
+//
+//  // Define parameters for AdaBoost.
+//  arma::Row<size_t> predictedLabels(testData.n_cols);
+//  a.Classify(testData, predictedLabels);
+//
+//  int localError = 0;
+//  for (size_t i = 0; i < trueTestLabels.n_cols; i++)
+//    if (trueTestLabels(i) != predictedLabels(i))
+//      localError++;
+//
+//  double lError = (double) localError / trueTestLabels.n_cols;
+//
+//  BOOST_REQUIRE_LE(lError, 0.30);
+//}
+//
+//BOOST_AUTO_TEST_CASE(PerceptronSerializationTest)
+//{
+//  // Build an AdaBoost object.
+//  mat data = randu<mat>(10, 500);
+//  Row<size_t> labels(500);
+//  for (size_t i = 0; i < 250; ++i)
+//    labels[i] = 0;
+//  for (size_t i = 250; i < 500; ++i)
+//    labels[i] = 1;
+//
+//  Perceptron<> p(data, labels, 2, 800);
+//  AdaBoost<> ab(data, labels, 2, p, 50, 1e-10);
+//
+//  // Now create another dataset to train with.
+//  mat otherData = randu<mat>(5, 200);
+//  Row<size_t> otherLabels(200);
+//  for (size_t i = 0; i < 100; ++i)
+//    otherLabels[i] = 1;
+//  for (size_t i = 100; i < 150; ++i)
+//    otherLabels[i] = 0;
+//  for (size_t i = 150; i < 200; ++i)
+//    otherLabels[i] = 2;
+//
+//  Perceptron<> p2(otherData, otherLabels, 3, 500);
+//  AdaBoost<> abText(otherData, otherLabels, 3, p2, 50, 1e-10);
+//
+//  AdaBoost<> abXml, abBinary;
+//
+//  SerializeObjectAll(ab, abXml, abText, abBinary);
+//
+//  // Now check that the objects are the same.
+//  BOOST_REQUIRE_CLOSE(ab.Tolerance(), abXml.Tolerance(), 1e-5);
+//  BOOST_REQUIRE_CLOSE(ab.Tolerance(), abText.Tolerance(), 1e-5);
+//  BOOST_REQUIRE_CLOSE(ab.Tolerance(), abBinary.Tolerance(), 1e-5);
+//
+//  BOOST_REQUIRE_CLOSE(ab.ZtProduct(), abXml.ZtProduct(), 1e-5);
+//  BOOST_REQUIRE_CLOSE(ab.ZtProduct(), abText.ZtProduct(), 1e-5);
+//  BOOST_REQUIRE_CLOSE(ab.ZtProduct(), abBinary.ZtProduct(), 1e-5);
+//
+//  BOOST_REQUIRE_EQUAL(ab.WeakLearners(), abXml.WeakLearners());
+//  BOOST_REQUIRE_EQUAL(ab.WeakLearners(), abText.WeakLearners());
+//  BOOST_REQUIRE_EQUAL(ab.WeakLearners(), abBinary.WeakLearners());
+//
+//  for (size_t i = 0; i < ab.WeakLearners(); ++i)
+//  {
+//    CheckMatrices(ab.WeakLearner(i).Weights(),
+//                  abXml.WeakLearner(i).Weights(),
+//                  abText.WeakLearner(i).Weights(),
+//                  abBinary.WeakLearner(i).Weights());
+//
+//    CheckMatrices(ab.WeakLearner(i).Biases(),
+//                  abXml.WeakLearner(i).Biases(),
+//                  abText.WeakLearner(i).Biases(),
+//                  abBinary.WeakLearner(i).Biases());
+//  }
+//}
+//
+//BOOST_AUTO_TEST_CASE(DecisionStumpSerializationTest)
+//{
+//  // Build an AdaBoost object.
+//  mat data = randu<mat>(10, 500);
+//  Row<size_t> labels(500);
+//  for (size_t i = 0; i < 250; ++i)
+//    labels[i] = 0;
+//  for (size_t i = 250; i < 500; ++i)
+//    labels[i] = 1;
+//
+//  DecisionStump<> p(data, labels, 2, 800);
+//  AdaBoost<DecisionStump<>> ab(data, labels, 2, p, 50, 1e-10);
+//
+//  // Now create another dataset to train with.
+//  mat otherData = randu<mat>(5, 200);
+//  Row<size_t> otherLabels(200);
+//  for (size_t i = 0; i < 100; ++i)
+//    otherLabels[i] = 1;
+//  for (size_t i = 100; i < 150; ++i)
+//    otherLabels[i] = 0;
+//  for (size_t i = 150; i < 200; ++i)
+//    otherLabels[i] = 2;
+//
+//  DecisionStump<> p2(otherData, otherLabels, 3, 500);
+//  AdaBoost<DecisionStump<>> abText(otherData, otherLabels, 3, p2, 50, 1e-10);
+//
+//  AdaBoost<DecisionStump<>> abXml, abBinary;
+//
+//  SerializeObjectAll(ab, abXml, abText, abBinary);
+//
+//  // Now check that the objects are the same.
+//  BOOST_REQUIRE_CLOSE(ab.Tolerance(), abXml.Tolerance(), 1e-5);
+//  BOOST_REQUIRE_CLOSE(ab.Tolerance(), abText.Tolerance(), 1e-5);
+//  BOOST_REQUIRE_CLOSE(ab.Tolerance(), abBinary.Tolerance(), 1e-5);
+//
+//  BOOST_REQUIRE_CLOSE(ab.ZtProduct(), abXml.ZtProduct(), 1e-5);
+//  BOOST_REQUIRE_CLOSE(ab.ZtProduct(), abText.ZtProduct(), 1e-5);
+//  BOOST_REQUIRE_CLOSE(ab.ZtProduct(), abBinary.ZtProduct(), 1e-5);
+//
+//  BOOST_REQUIRE_EQUAL(ab.WeakLearners(), abXml.WeakLearners());
+//  BOOST_REQUIRE_EQUAL(ab.WeakLearners(), abText.WeakLearners());
+//  BOOST_REQUIRE_EQUAL(ab.WeakLearners(), abBinary.WeakLearners());
+//
+//  for (size_t i = 0; i < ab.WeakLearners(); ++i)
+//  {
+//    BOOST_REQUIRE_EQUAL(ab.WeakLearner(i).SplitDimension(),
+//                        abXml.WeakLearner(i).SplitDimension());
+//    BOOST_REQUIRE_EQUAL(ab.WeakLearner(i).SplitDimension(),
+//                        abText.WeakLearner(i).SplitDimension());
+//    BOOST_REQUIRE_EQUAL(ab.WeakLearner(i).SplitDimension(),
+//                        abBinary.WeakLearner(i).SplitDimension());
+//
+//    CheckMatrices(ab.WeakLearner(i).Split(),
+//                  abXml.WeakLearner(i).Split(),
+//                  abText.WeakLearner(i).Split(),
+//                  abBinary.WeakLearner(i).Split());
+//
+//    CheckMatrices(ab.WeakLearner(i).BinLabels(),
+//                  abXml.WeakLearner(i).BinLabels(),
+//                  abText.WeakLearner(i).BinLabels(),
+//                  abBinary.WeakLearner(i).BinLabels());
+//  }
+//}
 
 BOOST_AUTO_TEST_SUITE_END();

From 75174c72ff265ebdbde3ba0788451bd02e6ab7b6 Mon Sep 17 00:00:00 2001
From: Xu Kaiqiang <rickllykqxu@gmail.com>
Date: Sat, 6 Apr 2019 16:58:35 +0800
Subject: [PATCH 07/27] add ID3DecisionStump type using existing DecisionTree

---
 .../methods/decision_tree/decision_tree.hpp   | 25 +++++++++++++++++++
 1 file changed, 25 insertions(+)

diff --git a/src/mlpack/methods/decision_tree/decision_tree.hpp b/src/mlpack/methods/decision_tree/decision_tree.hpp
index a839b420cf..a44403c6f8 100644
--- a/src/mlpack/methods/decision_tree/decision_tree.hpp
+++ b/src/mlpack/methods/decision_tree/decision_tree.hpp
@@ -15,6 +15,7 @@
 
 #include <mlpack/prereqs.hpp>
 #include "gini_gain.hpp"
+#include "information_gain.hpp"
 #include "best_binary_numeric_split.hpp"
 #include "all_categorical_split.hpp"
 #include "all_dimension_select.hpp"
@@ -123,6 +124,18 @@ class DecisionTree :
                    typename std::remove_reference<WeightsType>::type>::value>*
                     = 0);
 
+  template<typename MatType, typename LabelsType, typename WeightsType>
+  DecisionTree(const DecisionTree& other,
+               MatType data,
+               const data::DatasetInfo& datasetInfo,
+               LabelsType labels,
+               const size_t numClasses,
+               WeightsType weights,
+               const size_t minimumLeafSize = 10,
+               const double minimumGainSplit = 1e-7,
+               const std::enable_if_t<arma::is_arma_type<
+                       typename std::remove_reference<WeightsType>::type>::value>*
+               = 0);
   /**
    * Construct the decision tree on the given data and labels with weights,
    * assuming that the data is all of the numeric type. Setting minimumLeafSize
@@ -150,6 +163,17 @@ class DecisionTree :
                    typename std::remove_reference<WeightsType>::type>::value>*
                     = 0);
 
+  template<typename MatType, typename LabelsType, typename WeightsType>
+  DecisionTree(const DecisionTree& other,
+               MatType data,
+               LabelsType labels,
+               const size_t numClasses,
+               WeightsType weights,
+               const size_t minimumLeafSize = 10,
+               const double minimumGainSplit = 1e-7,
+               const std::enable_if_t<arma::is_arma_type<
+                       typename std::remove_reference<WeightsType>::type>::value>*
+               = 0);
 
   /**
    * Construct a decision tree without training it.  It will be a leaf node with
@@ -479,6 +503,7 @@ using DecisionStump = DecisionTree<FitnessFunction,
                                    ElemType,
                                    false>;
 
+typedef DecisionTree<InformationGain, BestBinaryNumericSplit, AllCategoricalSplit, AllDimensionSelect, double, true> ID3DecisionStump;
 } // namespace tree
 } // namespace mlpack
 

From 56f32663621d6996b2f871456bac39342546d3ec Mon Sep 17 00:00:00 2001
From: Xu Kaiqiang <rickllykqxu@gmail.com>
Date: Sat, 6 Apr 2019 17:00:22 +0800
Subject: [PATCH 08/27] As a weak learner, so introduce two functions to
 compatible with existing function call.

---
 .../decision_tree/decision_tree_impl.hpp      | 85 +++++++++++++++++++
 1 file changed, 85 insertions(+)

diff --git a/src/mlpack/methods/decision_tree/decision_tree_impl.hpp b/src/mlpack/methods/decision_tree/decision_tree_impl.hpp
index a947f17a64..0007db72d2 100644
--- a/src/mlpack/methods/decision_tree/decision_tree_impl.hpp
+++ b/src/mlpack/methods/decision_tree/decision_tree_impl.hpp
@@ -12,6 +12,8 @@
 #ifndef MLPACK_METHODS_DECISION_TREE_DECISION_TREE_IMPL_HPP
 #define MLPACK_METHODS_DECISION_TREE_DECISION_TREE_IMPL_HPP
 
+#include "decision_tree.hpp"
+
 namespace mlpack {
 namespace tree {
 
@@ -119,6 +121,48 @@ DecisionTree<FitnessFunction,
       tmpWeights, minimumLeafSize, minimumGainSplit);
 }
 
+//! Construct and train with weights.
+template<typename FitnessFunction,
+        template<typename> class NumericSplitType,
+        template<typename> class CategoricalSplitType,
+        typename DimensionSelectionType,
+        typename ElemType,
+        bool NoRecursion>
+template<typename MatType, typename LabelsType, typename WeightsType>
+DecisionTree<FitnessFunction,
+        NumericSplitType,
+        CategoricalSplitType,
+        DimensionSelectionType,
+        ElemType,
+        NoRecursion>::DecisionTree(const DecisionTree& other,
+                                   MatType data,
+                                   const data::DatasetInfo& datasetInfo,
+                                   LabelsType labels,
+                                   const size_t numClasses,
+                                   WeightsType weights,
+                                   const size_t minimumLeafSize,
+                                   const double minimumGainSplit,
+                                   const std::enable_if_t<
+                                           arma::is_arma_type<
+                                                   typename std::remove_reference<
+                                                           WeightsType>::type>::value>*):
+        NumericAuxiliarySplitInfo(other),
+        CategoricalAuxiliarySplitInfo(other)
+{
+  using TrueMatType = typename std::decay<MatType>::type;
+  using TrueLabelsType = typename std::decay<LabelsType>::type;
+  using TrueWeightsType = typename std::decay<WeightsType>::type;
+
+  // Copy or move data.
+  TrueMatType tmpData(std::move(data));
+  TrueLabelsType tmpLabels(std::move(labels));
+  TrueWeightsType tmpWeights(std::move(weights));
+
+  // Pass off work to the weighted Train() method.
+  Train<true>(tmpData, 0, tmpData.n_cols, datasetInfo, tmpLabels, numClasses,
+              tmpWeights, minimumLeafSize, minimumGainSplit);
+}
+
 //! Construct and train with weights.
 template<typename FitnessFunction,
          template<typename> class NumericSplitType,
@@ -157,6 +201,47 @@ DecisionTree<FitnessFunction,
       minimumLeafSize, minimumGainSplit);
 }
 
+//! Construct and train with weights.
+template<typename FitnessFunction,
+        template<typename> class NumericSplitType,
+        template<typename> class CategoricalSplitType,
+        typename DimensionSelectionType,
+        typename ElemType,
+        bool NoRecursion>
+template<typename MatType, typename LabelsType, typename WeightsType>
+DecisionTree<FitnessFunction,
+        NumericSplitType,
+        CategoricalSplitType,
+        DimensionSelectionType,
+        ElemType,
+        NoRecursion>::DecisionTree(const DecisionTree& other,
+                                   MatType data,
+                                   LabelsType labels,
+                                   const size_t numClasses,
+                                   WeightsType weights,
+                                   const size_t minimumLeafSize,
+                                   const double minimumGainSplit,
+                                   const std::enable_if_t<
+                                           arma::is_arma_type<
+                                                   typename std::remove_reference<
+                                                           WeightsType>::type>::value>*):
+        NumericAuxiliarySplitInfo(other),
+        CategoricalAuxiliarySplitInfo(other)  // other info does need to copy
+{
+  using TrueMatType = typename std::decay<MatType>::type;
+  using TrueLabelsType = typename std::decay<LabelsType>::type;
+  using TrueWeightsType = typename std::decay<WeightsType>::type;
+
+  // Copy or move data.
+  TrueMatType tmpData(std::move(data));
+  TrueLabelsType tmpLabels(std::move(labels));
+  TrueWeightsType tmpWeights(std::move(weights));
+
+  // Pass off work to the weighted Train() method.
+  Train<true>(tmpData, 0, tmpData.n_cols, tmpLabels, numClasses, tmpWeights,
+              minimumLeafSize, minimumGainSplit);
+}
+
 //! Construct, don't train.
 template<typename FitnessFunction,
          template<typename> class NumericSplitType,

From c3d860c64fcb3ca69deb07e5671d3799f36f6f86 Mon Sep 17 00:00:00 2001
From: himanshupathak21061998 <hpathak336@gmail.com>
Date: Sun, 22 Dec 2019 00:17:43 +0530
Subject: [PATCH 09/27] Adding parameters and removing one test

---
 .../methods/decision_tree/decision_tree.hpp   |  99 +++++++-------
 .../decision_tree/decision_tree_impl.hpp      |  56 ++++----
 src/mlpack/tests/adaboost_test.cpp            | 121 +++++++++---------
 3 files changed, 143 insertions(+), 133 deletions(-)

diff --git a/src/mlpack/methods/decision_tree/decision_tree.hpp b/src/mlpack/methods/decision_tree/decision_tree.hpp
index 0eff9011c0..29dfa3957a 100644
--- a/src/mlpack/methods/decision_tree/decision_tree.hpp
+++ b/src/mlpack/methods/decision_tree/decision_tree.hpp
@@ -125,32 +125,31 @@ class DecisionTree :
    * @param dimensionSelector Instantiated dimension selection policy.
    */
   template<typename MatType, typename LabelsType, typename WeightsType>
-  DecisionTree(MatType data,
-               const data::DatasetInfo& datasetInfo,
-               LabelsType labels,
-               const size_t numClasses,
-               WeightsType weights,
-               const size_t minimumLeafSize = 10,
-               const double minimumGainSplit = 1e-7,
-               const size_t maximumDepth = 0,
-               DimensionSelectionType dimensionSelector =
-                   DimensionSelectionType(),
-               const std::enable_if_t<arma::is_arma_type<
-                   typename std::remove_reference<WeightsType>::type>::value>*
-                    = 0);
+  DecisionTree(
+      MatType data,
+      const data::DatasetInfo& datasetInfo,
+      LabelsType labels,
+      const size_t numClasses,
+      WeightsType weights,
+      const size_t minimumLeafSize = 10,
+      const double minimumGainSplit = 1e-7,
+      const size_t maximumDepth = 0,
+      DimensionSelectionType dimensionSelector = DimensionSelectionType(),
+      const std::enable_if_t<arma::is_arma_type<
+          typename std::remove_reference<WeightsType>::type>::value>* = 0);
 
   template<typename MatType, typename LabelsType, typename WeightsType>
-  DecisionTree(const DecisionTree& other,
-               MatType data,
-               const data::DatasetInfo& datasetInfo,
-               LabelsType labels,
-               const size_t numClasses,
-               WeightsType weights,
-               const size_t minimumLeafSize = 10,
-               const double minimumGainSplit = 1e-7,
-               const std::enable_if_t<arma::is_arma_type<
-                       typename std::remove_reference<WeightsType>::type>::value>*
-               = 0);
+  DecisionTree(
+      const DecisionTree& other,
+      MatType data,
+      const data::DatasetInfo& datasetInfo,
+      LabelsType labels,
+      const size_t numClasses,
+      WeightsType weights,
+      const size_t minimumLeafSize = 10,
+      const double minimumGainSplit = 1e-7,
+      const std::enable_if_t<arma::is_arma_type<
+          typename std::remove_reference<WeightsType>::type>::value>* = 0);
   /**
    * Construct the decision tree on the given data and labels with weights,
    * assuming that the data is all of the numeric type. Setting minimumLeafSize
@@ -170,30 +169,31 @@ class DecisionTree :
    * @param dimensionSelector Instantiated dimension selection policy.
    */
   template<typename MatType, typename LabelsType, typename WeightsType>
-  DecisionTree(MatType data,
-               LabelsType labels,
-               const size_t numClasses,
-               WeightsType weights,
-               const size_t minimumLeafSize = 10,
-               const double minimumGainSplit = 1e-7,
-               const size_t maximumDepth = 0,
-               DimensionSelectionType dimensionSelector =
-                   DimensionSelectionType(),
-               const std::enable_if_t<arma::is_arma_type<
-                   typename std::remove_reference<WeightsType>::type>::value>*
-                    = 0);
+  DecisionTree(
+      MatType data,
+      LabelsType labels,
+      const size_t numClasses,
+      WeightsType weights,
+      const size_t minimumLeafSize = 10,
+      const double minimumGainSplit = 1e-7,
+      const size_t maximumDepth = 0,
+      DimensionSelectionType dimensionSelector = DimensionSelectionType(),
+      const std::enable_if_t<arma::is_arma_type<
+          typename std::remove_reference<WeightsType>::type>::value>* = 0);
 
   template<typename MatType, typename LabelsType, typename WeightsType>
-  DecisionTree(const DecisionTree& other,
-               MatType data,
-               LabelsType labels,
-               const size_t numClasses,
-               WeightsType weights,
-               const size_t minimumLeafSize = 10,
-               const double minimumGainSplit = 1e-7,
-               const std::enable_if_t<arma::is_arma_type<
-                       typename std::remove_reference<WeightsType>::type>::value>*
-               = 0);
+  DecisionTree(
+      const DecisionTree& other,
+      MatType data,
+      LabelsType labels,
+      const size_t numClasses,
+      WeightsType weights,
+      const size_t minimumLeafSize = 10,
+      const double minimumGainSplit = 1e-7,
+      const size_t maximumDepth = 0,
+      DimensionSelectionType dimensionSelector = DimensionSelectionType(),
+      const std::enable_if_t<arma::is_arma_type<
+          typename std::remove_reference<WeightsType>::type>::value>* = 0);
 
   /**
    * Construct a decision tree without training it.  It will be a leaf node with
@@ -553,7 +553,12 @@ using DecisionStump = DecisionTree<FitnessFunction,
                                    ElemType,
                                    false>;
 
-typedef DecisionTree<InformationGain, BestBinaryNumericSplit, AllCategoricalSplit, AllDimensionSelect, double, true> ID3DecisionStump;
+typedef DecisionTree<InformationGain,
+                     BestBinaryNumericSplit,
+                     AllCategoricalSplit,
+                     AllDimensionSelect,
+                     double,
+                     true> ID3DecisionStump;
 } // namespace tree
 } // namespace mlpack
 
diff --git a/src/mlpack/methods/decision_tree/decision_tree_impl.hpp b/src/mlpack/methods/decision_tree/decision_tree_impl.hpp
index b9faa30060..306375d7c8 100644
--- a/src/mlpack/methods/decision_tree/decision_tree_impl.hpp
+++ b/src/mlpack/methods/decision_tree/decision_tree_impl.hpp
@@ -154,18 +154,19 @@ DecisionTree<FitnessFunction,
         CategoricalSplitType,
         DimensionSelectionType,
         ElemType,
-        NoRecursion>::DecisionTree(const DecisionTree& other,
-                                   MatType data,
-                                   const data::DatasetInfo& datasetInfo,
-                                   LabelsType labels,
-                                   const size_t numClasses,
-                                   WeightsType weights,
-                                   const size_t minimumLeafSize,
-                                   const double minimumGainSplit,
-                                   const std::enable_if_t<
-                                           arma::is_arma_type<
-                                                   typename std::remove_reference<
-                                                           WeightsType>::type>::value>*):
+        NoRecursion>::DecisionTree(
+    const DecisionTree& other,
+    MatType data,
+    const data::DatasetInfo& datasetInfo,
+    LabelsType labels,
+    const size_t numClasses,
+    WeightsType weights,
+    const size_t minimumLeafSize,
+    const double minimumGainSplit,
+    const std::enable_if_t<
+        arma::is_arma_type<
+        typename std::remove_reference<
+        WeightsType>::type>::value>*):
         NumericAuxiliarySplitInfo(other),
         CategoricalAuxiliarySplitInfo(other)
 {
@@ -240,17 +241,19 @@ DecisionTree<FitnessFunction,
         CategoricalSplitType,
         DimensionSelectionType,
         ElemType,
-        NoRecursion>::DecisionTree(const DecisionTree& other,
-                                   MatType data,
-                                   LabelsType labels,
-                                   const size_t numClasses,
-                                   WeightsType weights,
-                                   const size_t minimumLeafSize,
-                                   const double minimumGainSplit,
-                                   const std::enable_if_t<
-                                           arma::is_arma_type<
-                                                   typename std::remove_reference<
-                                                           WeightsType>::type>::value>*):
+        NoRecursion>::DecisionTree(
+    const DecisionTree& other,
+    MatType data,
+    LabelsType labels,
+    const size_t numClasses,
+    WeightsType weights,
+    const size_t minimumLeafSize,
+    const double minimumGainSplit,
+    const size_t maximumDepth,
+    DimensionSelectionType dimensionSelector,
+    const std::enable_if_t<arma::is_arma_type<
+        typename std::remove_reference<
+        WeightsType>::type>::value>*):
         NumericAuxiliarySplitInfo(other),
         CategoricalAuxiliarySplitInfo(other)  // other info does need to copy
 {
@@ -263,9 +266,12 @@ DecisionTree<FitnessFunction,
   TrueLabelsType tmpLabels(std::move(labels));
   TrueWeightsType tmpWeights(std::move(weights));
 
+  // Set the correct dimensionality for the dimension selector.
+  dimensionSelector.Dimensions() = tmpData.n_rows;
+
   // Pass off work to the weighted Train() method.
   Train<true>(tmpData, 0, tmpData.n_cols, tmpLabels, numClasses, tmpWeights,
-              minimumLeafSize, minimumGainSplit);
+              minimumLeafSize, minimumGainSplit, maximumDepth, dimensionSelector);
 }
 
 //! Construct, don't train.
@@ -653,7 +659,7 @@ double DecisionTree<FitnessFunction,
       dimensionSelector);
 }
 
-//! Train on the given data.
+//! Train on the given data, assuming all dimensions are numeric.
 template<typename FitnessFunction,
          template<typename> class NumericSplitType,
          template<typename> class CategoricalSplitType,
diff --git a/src/mlpack/tests/adaboost_test.cpp b/src/mlpack/tests/adaboost_test.cpp
index 4beb091cd0..740b6f191b 100644
--- a/src/mlpack/tests/adaboost_test.cpp
+++ b/src/mlpack/tests/adaboost_test.cpp
@@ -306,7 +306,6 @@ BOOST_AUTO_TEST_CASE(WeakLearnerErrorNonLinearSepData)
  * checks whether the Hamming loss breaches the upper bound, which is provided
  * by ztAccumulator.  This uses decision stumps as the weak learner.
  */
-<<<<<<< HEAD
 BOOST_AUTO_TEST_CASE(HammingLossIris_DS)
 {
   arma::mat inputData;
@@ -845,66 +844,66 @@ BOOST_AUTO_TEST_CASE(PerceptronSerializationTest)
   }
 }
 
-BOOST_AUTO_TEST_CASE(DecisionStumpSerializationTest)
-{
-  // Build an AdaBoost object.
-  mat data = randu<mat>(10, 500);
-  Row<size_t> labels(500);
-  for (size_t i = 0; i < 250; ++i)
-    labels[i] = 0;
-  for (size_t i = 250; i < 500; ++i)
-    labels[i] = 1;
-
-  DecisionStump<> p(data, labels, 2, 800);
-  AdaBoost<DecisionStump<>> ab(data, labels, 2, p, 50, 1e-10);
-
-  // Now create another dataset to train with.
-  mat otherData = randu<mat>(5, 200);
-  Row<size_t> otherLabels(200);
-  for (size_t i = 0; i < 100; ++i)
-    otherLabels[i] = 1;
-  for (size_t i = 100; i < 150; ++i)
-    otherLabels[i] = 0;
-  for (size_t i = 150; i < 200; ++i)
-    otherLabels[i] = 2;
-
-  DecisionStump<> p2(otherData, otherLabels, 3, 500);
-  AdaBoost<DecisionStump<>> abText(otherData, otherLabels, 3, p2, 50, 1e-10);
-
-  AdaBoost<DecisionStump<>> abXml, abBinary;
-
-  SerializeObjectAll(ab, abXml, abText, abBinary);
-
-  // Now check that the objects are the same.
-  BOOST_REQUIRE_CLOSE(ab.Tolerance(), abXml.Tolerance(), 1e-5);
-  BOOST_REQUIRE_CLOSE(ab.Tolerance(), abText.Tolerance(), 1e-5);
-  BOOST_REQUIRE_CLOSE(ab.Tolerance(), abBinary.Tolerance(), 1e-5);
-
-  BOOST_REQUIRE_EQUAL(ab.WeakLearners(), abXml.WeakLearners());
-  BOOST_REQUIRE_EQUAL(ab.WeakLearners(), abText.WeakLearners());
-  BOOST_REQUIRE_EQUAL(ab.WeakLearners(), abBinary.WeakLearners());
-
-  for (size_t i = 0; i < ab.WeakLearners(); ++i)
-  {
-    BOOST_REQUIRE_EQUAL(ab.WeakLearner(i).SplitDimension(),
-                        abXml.WeakLearner(i).SplitDimension());
-    BOOST_REQUIRE_EQUAL(ab.WeakLearner(i).SplitDimension(),
-                        abText.WeakLearner(i).SplitDimension());
-    BOOST_REQUIRE_EQUAL(ab.WeakLearner(i).SplitDimension(),
-                        abBinary.WeakLearner(i).SplitDimension());
-
-    CheckMatrices(ab.WeakLearner(i).Split(),
-                  abXml.WeakLearner(i).Split(),
-                  abText.WeakLearner(i).Split(),
-                  abBinary.WeakLearner(i).Split());
-
-    CheckMatrices(ab.WeakLearner(i).BinLabels(),
-                  abXml.WeakLearner(i).BinLabels(),
-                  abText.WeakLearner(i).BinLabels(),
-                  abBinary.WeakLearner(i).BinLabels());
-  }
-}
-
+//BOOST_AUTO_TEST_CASE(DecisionStumpSerializationTest)
+//{
+//  // Build an AdaBoost object.
+//  mat data = randu<mat>(10, 500);
+//  Row<size_t> labels(500);
+//  for (size_t i = 0; i < 250; ++i)
+//    labels[i] = 0;
+//  for (size_t i = 250; i < 500; ++i)
+//    labels[i] = 1;
+//
+//  DecisionStump<> p(data, labels, 2, 800);
+//  AdaBoost<DecisionStump<>> ab(data, labels, 2, p, 50, 1e-10);
+//
+//  // Now create another dataset to train with.
+//  mat otherData = randu<mat>(5, 200);
+//  Row<size_t> otherLabels(200);
+// for (size_t i = 0; i < 100; ++i)
+//    otherLabels[i] = 1;
+//  for (size_t i = 100; i < 150; ++i)
+//    otherLabels[i] = 0;
+//  for (size_t i = 150; i < 200; ++i)
+//    otherLabels[i] = 2;
+//
+//  DecisionStump<> p2(otherData, otherLabels, 3, 500);
+//  AdaBoost<DecisionStump<>> abText(otherData, otherLabels, 3, p2, 50, 1e-10);
+//
+//  AdaBoost<DecisionStump<>> abXml, abBinary;
+//
+//  SerializeObjectAll(ab, abXml, abText, abBinary);
+//
+//  // Now check that the objects are the same.
+//  BOOST_REQUIRE_CLOSE(ab.Tolerance(), abXml.Tolerance(), 1e-5);
+// BOOST_REQUIRE_CLOSE(ab.Tolerance(), abText.Tolerance(), 1e-5);
+//  BOOST_REQUIRE_CLOSE(ab.Tolerance(), abBinary.Tolerance(), 1e-5);
+//
+//  BOOST_REQUIRE_EQUAL(ab.WeakLearners(), abXml.WeakLearners());
+//  BOOST_REQUIRE_EQUAL(ab.WeakLearners(), abText.WeakLearners());
+//  BOOST_REQUIRE_EQUAL(ab.WeakLearners(), abBinary.WeakLearners());
+//
+//  for (size_t i = 0; i < ab.WeakLearners(); ++i)
+//  {
+//    BOOST_REQUIRE_EQUAL(ab.WeakLearner(i).SplitDimension(),
+//                        abXml.WeakLearner(i).SplitDimension());
+//    BOOST_REQUIRE_EQUAL(ab.WeakLearner(i).SplitDimension(),
+//                        abText.WeakLearner(i).SplitDimension());
+//    BOOST_REQUIRE_EQUAL(ab.WeakLearner(i).SplitDimension(),
+//                        abBinary.WeakLearner(i).SplitDimension());
+//
+//    CheckMatrices(ab.WeakLearner(i).Split(),
+//                  abXml.WeakLearner(i).Split(),
+//                  abText.WeakLearner(i).Split(),
+//                  abBinary.WeakLearner(i).Split());
+//
+//    CheckMatrices(ab.WeakLearner(i).BinLabels(),
+//                  abXml.WeakLearner(i).BinLabels(),
+//                  abText.WeakLearner(i).BinLabels(),
+//                  abBinary.WeakLearner(i).BinLabels());
+//  }
+//}
+//
 //BOOST_AUTO_TEST_CASE(HammingLossIris_DS)
 //{
 //  arma::mat inputData;

From 9fcbe999b4810e1769bc7ad1a738985355e5b901 Mon Sep 17 00:00:00 2001
From: himanshupathak21061998 <hpathak336@gmail.com>
Date: Thu, 9 Jan 2020 22:45:04 +0530
Subject: [PATCH 10/27] Adding documentation

---
 .../methods/decision_tree/decision_tree.hpp   | 36 +++++++++++++++++++
 1 file changed, 36 insertions(+)

diff --git a/src/mlpack/methods/decision_tree/decision_tree.hpp b/src/mlpack/methods/decision_tree/decision_tree.hpp
index 29dfa3957a..8d3f79db60 100644
--- a/src/mlpack/methods/decision_tree/decision_tree.hpp
+++ b/src/mlpack/methods/decision_tree/decision_tree.hpp
@@ -138,6 +138,24 @@ class DecisionTree :
       const std::enable_if_t<arma::is_arma_type<
           typename std::remove_reference<WeightsType>::type>::value>* = 0);
 
+  /**
+   * Take ownership of another decision tree and train on the given data and 
+   * labels with weights, where the data can be both numeric and categorical.
+   * Setting minimumLeafSize and minimumGainSplit too small may cause the
+   * tree to overfit, but setting them too large may cause it to underfit.
+   *
+   * Use std::move if data, labels or weights are no longer needed to avoid
+   * copies.
+   *
+   * @param other Tree to take ownership of.
+   * @param data Dataset to train on.
+   * @param datasetInfo Type information for each dimension of the dataset.
+   * @param labels Labels for each training point.
+   * @param numClasses Number of classes in the dataset.
+   * @param weights The weight list of given label.
+   * @param minimumLeafSize Minimum number of points in each leaf node.
+   * @param minimumGainSplit Minimum gain for the node to split.
+   */
   template<typename MatType, typename LabelsType, typename WeightsType>
   DecisionTree(
       const DecisionTree& other,
@@ -181,6 +199,24 @@ class DecisionTree :
       const std::enable_if_t<arma::is_arma_type<
           typename std::remove_reference<WeightsType>::type>::value>* = 0);
 
+  /**
+   * Take ownership of another decision tree and train on the given data and labels
+   * with weights, assuming that the data is all of the numeric type. Setting 
+   * minimumLeafSize and minimumGainSplit too small may cause the tree to 
+   * overfit, but setting them too large may cause it to underfit.
+   *
+   * Use std::move if data, labels or weights are no longer needed to avoid
+   * copies.
+   * @param other Tree to take ownership of.
+   * @param data Dataset to train on.
+   * @param labels Labels for each training point.
+   * @param numClasses Number of classes in the dataset.
+   * @param weights The Weight list of given labels.
+   * @param minimumLeafSize Minimum number of points in each leaf node.
+   * @param minimumGainSplit Minimum gain for the node to split.
+   * @param maximumDepth Maximum depth for the tree.
+   * @param dimensionSelector Instantiated dimension selection policy.
+   */
   template<typename MatType, typename LabelsType, typename WeightsType>
   DecisionTree(
       const DecisionTree& other,

From 4e0e038b83d5ef4d9ea9090cfd0cba29f394f0fe Mon Sep 17 00:00:00 2001
From: himanshupathak21061998 <hpathak336@gmail.com>
Date: Fri, 10 Jan 2020 04:48:25 +0530
Subject: [PATCH 11/27] Fixing failure

---
 src/mlpack/methods/decision_tree/decision_tree_impl.hpp | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/mlpack/methods/decision_tree/decision_tree_impl.hpp b/src/mlpack/methods/decision_tree/decision_tree_impl.hpp
index 306375d7c8..f5f832eddf 100644
--- a/src/mlpack/methods/decision_tree/decision_tree_impl.hpp
+++ b/src/mlpack/methods/decision_tree/decision_tree_impl.hpp
@@ -88,10 +88,11 @@ DecisionTree<FitnessFunction,
 
   // Set the correct dimensionality for the dimension selector.
   dimensionSelector.Dimensions() = tmpData.n_rows;
+  arma::Row<size_t> passlabels = tmpLabels;
 
   // Pass off work to the Train() method.
   arma::rowvec weights; // Fake weights, not used.
-  Train<false>(tmpData, 0, tmpData.n_cols, tmpLabels, numClasses, weights,
+  Train<false>(tmpData, 0, tmpData.n_cols, passlabels, numClasses, weights,
       minimumLeafSize, minimumGainSplit, maximumDepth, dimensionSelector);
 }
 

From 416fc0acddbbc9342202801c040b469cacae8110 Mon Sep 17 00:00:00 2001
From: himanshupathak21061998 <hpathak336@gmail.com>
Date: Sat, 11 Jan 2020 02:16:16 +0530
Subject: [PATCH 12/27] Adding changes in tests

---
 src/mlpack/tests/adaboost_test.cpp | 734 +++--------------------------
 1 file changed, 64 insertions(+), 670 deletions(-)

diff --git a/src/mlpack/tests/adaboost_test.cpp b/src/mlpack/tests/adaboost_test.cpp
index 740b6f191b..3a90de388d 100644
--- a/src/mlpack/tests/adaboost_test.cpp
+++ b/src/mlpack/tests/adaboost_test.cpp
@@ -319,12 +319,12 @@ BOOST_AUTO_TEST_CASE(HammingLossIris_DS)
   // Define your own weak learner, decision stumps in this case.
   const size_t numClasses = 3;
   const size_t inpBucketSize = 6;
-  DecisionStump<> ds(inputData, labels.row(0), numClasses, inpBucketSize);
+  ID3DecisionStump ds(inputData, labels.row(0), numClasses, inpBucketSize);
 
   // Define parameters for AdaBoost.
   size_t iterations = 50;
   double tolerance = 1e-10;
-  AdaBoost<DecisionStump<>> a(tolerance);
+  AdaBoost<ID3DecisionStump> a(tolerance);
   double ztProduct = a.Train(inputData, labels.row(0), numClasses, ds,
       iterations, tolerance);
 
@@ -366,7 +366,7 @@ BOOST_AUTO_TEST_CASE(WeakLearnerErrorIris_DS)
 
   arma::Row<size_t> dsPrediction(labels.n_cols);
 
-  DecisionStump<> ds(inputData, labels.row(0), numClasses, inpBucketSize);
+  ID3DecisionStump ds(inputData, labels.row(0), numClasses, inpBucketSize);
   ds.Classify(inputData, dsPrediction);
 
   size_t countWeakLearnerError = 0;
@@ -379,7 +379,7 @@ BOOST_AUTO_TEST_CASE(WeakLearnerErrorIris_DS)
   size_t iterations = 50;
   double tolerance = 1e-10;
 
-  AdaBoost<DecisionStump<>> a(inputData, labels.row(0), numClasses, ds,
+  AdaBoost<ID3DecisionStump> a(inputData, labels.row(0), numClasses, ds,
       iterations, tolerance);
 
   arma::Row<size_t> predictedLabels;
@@ -414,13 +414,13 @@ BOOST_AUTO_TEST_CASE(HammingLossBoundVertebralColumn_DS)
   const size_t numClasses = 3;
   const size_t inpBucketSize = 6;
 
-  DecisionStump<> ds(inputData, labels.row(0), numClasses, inpBucketSize);
+  ID3DecisionStump ds(inputData, labels.row(0), numClasses, inpBucketSize);
 
   // Define parameters for AdaBoost.
   size_t iterations = 50;
   double tolerance = 1e-10;
 
-  AdaBoost<DecisionStump<>> a(tolerance);
+  AdaBoost<ID3DecisionStump> a(tolerance);
   double ztProduct = a.Train(inputData, labels.row(0), numClasses, ds,
       iterations, tolerance);
 
@@ -459,7 +459,7 @@ BOOST_AUTO_TEST_CASE(WeakLearnerErrorVertebralColumn_DS)
   const size_t inpBucketSize = 6;
   arma::Row<size_t> dsPrediction(labels.n_cols);
 
-  DecisionStump<> ds(inputData, labels.row(0), numClasses, inpBucketSize);
+  ID3DecisionStump ds(inputData, labels.row(0), numClasses, inpBucketSize);
   ds.Classify(inputData, dsPrediction);
 
   size_t countWeakLearnerError = 0;
@@ -472,7 +472,7 @@ BOOST_AUTO_TEST_CASE(WeakLearnerErrorVertebralColumn_DS)
   // Define parameters for AdaBoost.
   size_t iterations = 50;
   double tolerance = 1e-10;
-  AdaBoost<DecisionStump<>> a(inputData, labels.row(0), numClasses, ds,
+  AdaBoost<ID3DecisionStump> a(inputData, labels.row(0), numClasses, ds,
       iterations, tolerance);
 
   arma::Row<size_t> predictedLabels;
@@ -506,13 +506,13 @@ BOOST_AUTO_TEST_CASE(HammingLossBoundNonLinearSepData_DS)
   const size_t numClasses = 2;
   const size_t inpBucketSize = 6;
 
-  DecisionStump<> ds(inputData, labels.row(0), numClasses, inpBucketSize);
+  ID3DecisionStump ds(inputData, labels.row(0), numClasses, inpBucketSize);
 
   // Define parameters for Adaboost.
   size_t iterations = 50;
   double tolerance = 1e-10;
 
-  AdaBoost<DecisionStump<>> a(tolerance);
+  AdaBoost<ID3DecisionStump> a(tolerance);
   double ztProduct = a.Train(inputData, labels.row(0), numClasses, ds,
       iterations, tolerance);
 
@@ -552,7 +552,7 @@ BOOST_AUTO_TEST_CASE(WeakLearnerErrorNonLinearSepData_DS)
 
   arma::Row<size_t> dsPrediction(labels.n_cols);
 
-  DecisionStump<> ds(inputData, labels.row(0), numClasses, inpBucketSize);
+  ID3DecisionStump ds(inputData, labels.row(0), numClasses, inpBucketSize);
   ds.Classify(inputData, dsPrediction);
 
   size_t countWeakLearnerError = 0;
@@ -565,7 +565,7 @@ BOOST_AUTO_TEST_CASE(WeakLearnerErrorNonLinearSepData_DS)
   size_t iterations = 500;
   double tolerance = 1e-23;
 
-  AdaBoost<DecisionStump<> > a(inputData, labels.row(0), numClasses, ds,
+  AdaBoost<ID3DecisionStump > a(inputData, labels.row(0), numClasses, ds,
       iterations, tolerance);
 
   arma::Row<size_t> predictedLabels;
@@ -663,12 +663,12 @@ BOOST_AUTO_TEST_CASE(ClassifyTest_NONLINSEP)
 
   arma::Row<size_t> dsPrediction(labels.n_cols);
 
-  DecisionStump<> ds(inputData, labels.row(0), numClasses, inpBucketSize);
+  ID3DecisionStump ds(inputData, labels.row(0), numClasses, inpBucketSize);
 
   // Define parameters for AdaBoost.
   size_t iterations = 50;
   double tolerance = 1e-10;
-  AdaBoost<DecisionStump<> > a(inputData, labels.row(0), numClasses, ds,
+  AdaBoost<ID3DecisionStump > a(inputData, labels.row(0), numClasses, ds,
       iterations, tolerance);
 
   arma::Row<size_t> predictedLabels(testData.n_cols);
@@ -844,661 +844,55 @@ BOOST_AUTO_TEST_CASE(PerceptronSerializationTest)
   }
 }
 
-//BOOST_AUTO_TEST_CASE(DecisionStumpSerializationTest)
-//{
-//  // Build an AdaBoost object.
-//  mat data = randu<mat>(10, 500);
-//  Row<size_t> labels(500);
-//  for (size_t i = 0; i < 250; ++i)
-//    labels[i] = 0;
-//  for (size_t i = 250; i < 500; ++i)
-//    labels[i] = 1;
-//
-//  DecisionStump<> p(data, labels, 2, 800);
-//  AdaBoost<DecisionStump<>> ab(data, labels, 2, p, 50, 1e-10);
-//
-//  // Now create another dataset to train with.
-//  mat otherData = randu<mat>(5, 200);
-//  Row<size_t> otherLabels(200);
-// for (size_t i = 0; i < 100; ++i)
-//    otherLabels[i] = 1;
-//  for (size_t i = 100; i < 150; ++i)
-//    otherLabels[i] = 0;
-//  for (size_t i = 150; i < 200; ++i)
-//    otherLabels[i] = 2;
-//
-//  DecisionStump<> p2(otherData, otherLabels, 3, 500);
-//  AdaBoost<DecisionStump<>> abText(otherData, otherLabels, 3, p2, 50, 1e-10);
-//
-//  AdaBoost<DecisionStump<>> abXml, abBinary;
-//
-//  SerializeObjectAll(ab, abXml, abText, abBinary);
-//
-//  // Now check that the objects are the same.
-//  BOOST_REQUIRE_CLOSE(ab.Tolerance(), abXml.Tolerance(), 1e-5);
-// BOOST_REQUIRE_CLOSE(ab.Tolerance(), abText.Tolerance(), 1e-5);
-//  BOOST_REQUIRE_CLOSE(ab.Tolerance(), abBinary.Tolerance(), 1e-5);
-//
-//  BOOST_REQUIRE_EQUAL(ab.WeakLearners(), abXml.WeakLearners());
-//  BOOST_REQUIRE_EQUAL(ab.WeakLearners(), abText.WeakLearners());
-//  BOOST_REQUIRE_EQUAL(ab.WeakLearners(), abBinary.WeakLearners());
-//
-//  for (size_t i = 0; i < ab.WeakLearners(); ++i)
-//  {
-//    BOOST_REQUIRE_EQUAL(ab.WeakLearner(i).SplitDimension(),
-//                        abXml.WeakLearner(i).SplitDimension());
-//    BOOST_REQUIRE_EQUAL(ab.WeakLearner(i).SplitDimension(),
-//                        abText.WeakLearner(i).SplitDimension());
-//    BOOST_REQUIRE_EQUAL(ab.WeakLearner(i).SplitDimension(),
-//                        abBinary.WeakLearner(i).SplitDimension());
-//
-//    CheckMatrices(ab.WeakLearner(i).Split(),
-//                  abXml.WeakLearner(i).Split(),
-//                  abText.WeakLearner(i).Split(),
-//                  abBinary.WeakLearner(i).Split());
-//
-//    CheckMatrices(ab.WeakLearner(i).BinLabels(),
-//                  abXml.WeakLearner(i).BinLabels(),
-//                  abText.WeakLearner(i).BinLabels(),
-//                  abBinary.WeakLearner(i).BinLabels());
-//  }
-//}
-//
-//BOOST_AUTO_TEST_CASE(HammingLossIris_DS)
-//{
-//  arma::mat inputData;
-//  if (!data::Load("iris.csv", inputData))
-//    BOOST_FAIL("Cannot load test dataset iris.csv!");
-//
-//  arma::Mat<size_t> labels;
-//  if (!data::Load("iris_labels.txt", labels))
-//    BOOST_FAIL("Cannot load labels for iris_labels.txt");
-//
-//  // Define your own weak learner, decision stumps in this case.
-//  const size_t numClasses = 3;
-//  const size_t inpBucketSize = 6;
-//  ID3DecisionStump ds(inputData, labels.row(0), numClasses, inpBucketSize);
-//
-//  // Define parameters for AdaBoost.
-//  size_t iterations = 50;
-//  double tolerance = 1e-10;
-//  AdaBoost<ID3DecisionStump> a(inputData, labels.row(0), numClasses, ds,
-//      iterations, tolerance);
-//
-//  arma::Row<size_t> predictedLabels;
-//  a.Classify(inputData, predictedLabels);
-//
-//  size_t countError = 0;
-//  for (size_t i = 0; i < labels.n_cols; i++)
-//    if (labels(i) != predictedLabels(i))
-//      countError++;
-//  double hammingLoss = (double) countError / labels.n_cols;
-//
-//  BOOST_REQUIRE_LE(hammingLoss, a.ZtProduct());
-//}
-//
-///**
-// * This test case runs the AdaBoost.mh algorithm on a non-linearly separable
-// * dataset.  It checks if the error returned by running a single instance of the
-// * weak learner is worse than running the boosted weak learner using adaboost.
-// * This is for the weak learner: decision stumps.
-// */
-//BOOST_AUTO_TEST_CASE(WeakLearnerErrorIris_DS)
-//{
-//  arma::mat inputData;
-//  if (!data::Load("iris.csv", inputData))
-//    BOOST_FAIL("Cannot load test dataset iris.csv!");
-//
-//  arma::Mat<size_t> labels;
-//  if (!data::Load("iris_labels.txt", labels))
-//    BOOST_FAIL("Cannot load labels for iris_labels.txt");
-//
-//  // no need to map the labels here
-//
-//  // Define your own weak learner, decision stumps in this case.
-//  const size_t numClasses = 3;
-//  const size_t inpBucketSize = 6;
-//
-//  arma::Row<size_t> dsPrediction(labels.n_cols);
-//
-//  ID3DecisionStump ds(inputData, labels.row(0), numClasses, inpBucketSize);
-//  ds.Classify(inputData, dsPrediction);
-//
-//  size_t countWeakLearnerError = 0;
-//  for (size_t i = 0; i < labels.n_cols; i++)
-//    if (labels(i) != dsPrediction(i))
-//      countWeakLearnerError++;
-//  double weakLearnerErrorRate = (double) countWeakLearnerError / labels.n_cols;
-//
-//  // Define parameters for AdaBoost.
-//  size_t iterations = 50;
-//  double tolerance = 1e-10;
-//
-//  AdaBoost<ID3DecisionStump> a(inputData, labels.row(0), numClasses, ds,
-//      iterations, tolerance);
-//
-//  arma::Row<size_t> predictedLabels;
-//  a.Classify(inputData, predictedLabels);
-//
-//  size_t countError = 0;
-//  for (size_t i = 0; i < labels.n_cols; i++)
-//    if (labels(i) != predictedLabels(i))
-//      countError++;
-//  double error = (double) countError / labels.n_cols;
-//
-//  BOOST_REQUIRE_LE(error, weakLearnerErrorRate);
-//}
-//
-///**
-// * This test case runs the AdaBoost.mh algorithm on the UCI Vertebral Column
-// * dataset.  It checks if the error returned by running a single instance of the
-// * weak learner is worse than running the boosted weak learner using adaboost.
-// * This is for the weak learner: decision stumps.
-// */
-//BOOST_AUTO_TEST_CASE(HammingLossBoundVertebralColumn_DS)
-//{
-//  arma::mat inputData;
-//  if (!data::Load("vc2.csv", inputData))
-//    BOOST_FAIL("Cannot load test dataset vc2.csv!");
-//
-//  arma::Mat<size_t> labels;
-//  if (!data::Load("vc2_labels.txt", labels))
-//    BOOST_FAIL("Cannot load labels for vc2_labels.txt");
-//
-//  // Define your own weak learner, decision stumps in this case.
-//  const size_t numClasses = 3;
-//  const size_t inpBucketSize = 6;
-//
-//  ID3DecisionStump ds(inputData, labels.row(0), numClasses, inpBucketSize);
-//
-//  // Define parameters for AdaBoost.
-//  size_t iterations = 50;
-//  double tolerance = 1e-10;
-//
-//  AdaBoost<ID3DecisionStump> a(inputData, labels.row(0), numClasses, ds,
-//      iterations, tolerance);
-//
-//  arma::Row<size_t> predictedLabels;
-//  a.Classify(inputData, predictedLabels);
-//
-//  size_t countError = 0;
-//  for (size_t i = 0; i < labels.n_cols; i++)
-//    if (labels(i) != predictedLabels(i))
-//      countError++;
-//  double hammingLoss = (double) countError / labels.n_cols;
-//
-//  BOOST_REQUIRE_LE(hammingLoss, a.ZtProduct());
-//}
-//
-///**
-// * This test case runs the AdaBoost.mh algorithm on the UCI Vertebral Column
-// * dataset.  It checks if the error returned by running a single instance of the
-// * weak learner is worse than running the boosted weak learner using adaboost.
-// * This is for the weak learner: decision stumps.
-// */
-//BOOST_AUTO_TEST_CASE(WeakLearnerErrorVertebralColumn_DS)
-//{
-//  arma::mat inputData;
-//  if (!data::Load("vc2.csv", inputData))
-//    BOOST_FAIL("Cannot load test dataset vc2.csv!");
-//
-//  arma::Mat<size_t> labels;
-//  if (!data::Load("vc2_labels.txt", labels))
-//    BOOST_FAIL("Cannot load labels for vc2_labels.txt");
-//
-//  // Define your own weak learner, decision stumps in this case.
-//  const size_t numClasses = 3;
-//  const size_t inpBucketSize = 6;
-//  arma::Row<size_t> dsPrediction(labels.n_cols);
-//
-//  ID3DecisionStump ds(inputData, labels.row(0), numClasses, inpBucketSize);
-//  ds.Classify(inputData, dsPrediction);
-//
-//  size_t countWeakLearnerError = 0;
-//  for (size_t i = 0; i < labels.n_cols; i++)
-//    if (labels(i) != dsPrediction(i))
-//      countWeakLearnerError++;
-//
-//  double weakLearnerErrorRate = (double) countWeakLearnerError / labels.n_cols;
-//
-//  // Define parameters for AdaBoost.
-//  size_t iterations = 50;
-//  double tolerance = 1e-10;
-//  AdaBoost<ID3DecisionStump> a(inputData, labels.row(0), numClasses, ds,
-//      iterations, tolerance);
-//
-//  arma::Row<size_t> predictedLabels;
-//  a.Classify(inputData, predictedLabels);
-//
-//  size_t countError = 0;
-//  for (size_t i = 0; i < labels.n_cols; i++)
-//    if (labels(i) != predictedLabels(i))
-//      countError++;
-//  double error = (double) countError / labels.n_cols;
-//
-//  BOOST_REQUIRE_LE(error, weakLearnerErrorRate);
-//}
-//
-///**
-// * This test case runs the AdaBoost.mh algorithm on non-linearly separable
-// * dataset.  It checks whether the hamming loss breaches the upperbound, which
-// * is provided by ztAccumulator.  This is for the weak learner: decision stumps.
-// */
-//BOOST_AUTO_TEST_CASE(HammingLossBoundNonLinearSepData_DS)
-//{
-//  arma::mat inputData;
-//  if (!data::Load("train_nonlinsep.txt", inputData))
-//    BOOST_FAIL("Cannot load test dataset train_nonlinsep.txt!");
-//
-//  arma::Mat<size_t> labels;
-//  if (!data::Load("train_labels_nonlinsep.txt", labels))
-//    BOOST_FAIL("Cannot load labels for train_labels_nonlinsep.txt");
-//
-//  // Define your own weak learner, decision stumps in this case.
-//  const size_t numClasses = 2;
-//  const size_t inpBucketSize = 6;
-//
-//  ID3DecisionStump ds(inputData, labels.row(0), numClasses, inpBucketSize);
-//
-//  // Define parameters for Adaboost.
-//  size_t iterations = 50;
-//  double tolerance = 1e-10;
-//
-//  AdaBoost<ID3DecisionStump > a(inputData, labels.row(0), numClasses, ds,
-//      iterations, tolerance);
-//
-//  arma::Row<size_t> predictedLabels;
-//  a.Classify(inputData, predictedLabels);
-//
-//  size_t countError = 0;
-//  for (size_t i = 0; i < labels.n_cols; i++)
-//    if (labels(i) != predictedLabels(i))
-//      countError++;
-//  double hammingLoss = (double) countError / labels.n_cols;
-//
-//  BOOST_REQUIRE_LE(hammingLoss, a.ZtProduct());
-//}
-//
-///**
-// * This test case runs the AdaBoost.mh algorithm on a non-linearly separable
-// * dataset.  It checks if the error returned by running a single instance of the
-// * weak learner is worse than running the boosted weak learner using adaboost.
-// * This for the weak learner: decision stumps.
-// */
-//BOOST_AUTO_TEST_CASE(WeakLearnerErrorNonLinearSepData_DS)
-//{
-//  arma::mat inputData;
-//  if (!data::Load("train_nonlinsep.txt", inputData))
-//    BOOST_FAIL("Cannot load test dataset train_nonlinsep.txt!");
-//
-//  arma::Mat<size_t> labels;
-//  if (!data::Load("train_labels_nonlinsep.txt", labels))
-//    BOOST_FAIL("Cannot load labels for train_labels_nonlinsep.txt");
-//
-//  // Define your own weak learner, decision stumps in this case.
-//  const size_t numClasses = 2;
-//  const size_t inpBucketSize = 3;
-//
-//  arma::Row<size_t> dsPrediction(labels.n_cols);
-//
-//  ID3DecisionStump ds(inputData, labels.row(0), numClasses, inpBucketSize);
-//  ds.Classify(inputData, dsPrediction);
-//
-//  size_t countWeakLearnerError = 0;
-//  for (size_t i = 0; i < labels.n_cols; i++)
-//    if (labels(i) != dsPrediction(i))
-//      countWeakLearnerError++;
-//  double weakLearnerErrorRate = (double) countWeakLearnerError / labels.n_cols;
-//
-//  // Define parameters for AdaBoost.
-//  size_t iterations = 500;
-//  double tolerance = 1e-23;
-//
-//  AdaBoost<ID3DecisionStump > a(inputData, labels.row(0), numClasses, ds,
-//      iterations, tolerance);
-//
-//  arma::Row<size_t> predictedLabels;
-//  a.Classify(inputData, predictedLabels);
-//
-//  size_t countError = 0;
-//  for (size_t i = 0; i < labels.n_cols; i++)
-//    if (labels(i) != predictedLabels(i))
-//      countError++;
-//  double error = (double) countError / labels.n_cols;
-//
-//  BOOST_REQUIRE_LE(error, weakLearnerErrorRate);
-//}
-//
-///**
-// * This test case runs the AdaBoost.mh algorithm on the UCI Vertebral Column
-// * dataset.  It tests the Classify function and checks for a satisfactory error
-// * rate.
-// */
-//BOOST_AUTO_TEST_CASE(ClassifyTest_VERTEBRALCOL)
-//{
-//  arma::mat inputData;
-//  if (!data::Load("vc2.csv", inputData))
-//    BOOST_FAIL("Cannot load test dataset vc2.csv!");
-//
-//  arma::Mat<size_t> labels;
-//  if (!data::Load("vc2_labels.txt", labels))
-//    BOOST_FAIL("Cannot load labels for vc2_labels.txt");
-//
-//  // Define your own weak learner, perceptron in this case.
-//  // Run the perceptron for perceptronIter iterations.
-//  size_t perceptronIter = 1000;
-//
-//  arma::mat testData;
-//
-//  if (!data::Load("vc2_test.csv", testData))
-//    BOOST_FAIL("Cannot load test dataset vc2_test.csv!");
-//
-//  arma::Mat<size_t> trueTestLabels;
-//
-//  if (!data::Load("vc2_test_labels.txt", trueTestLabels))
-//    BOOST_FAIL("Cannot load labels for vc2_test_labels.txt");
-//
-//  const size_t numClasses = max(labels.row(0)) + 1;
-//
-//  Row<size_t> perceptronPrediction(labels.n_cols);
-//  Perceptron<> p(inputData, labels.row(0), numClasses, perceptronIter);
-//  p.Classify(inputData, perceptronPrediction);
-//
-//  // Define parameters for AdaBoost.
-//  size_t iterations = 100;
-//  double tolerance = 1e-10;
-//  AdaBoost<> a(inputData, labels.row(0), numClasses, p, iterations, tolerance);
-//
-//  arma::Row<size_t> predictedLabels(testData.n_cols);
-//  a.Classify(testData, predictedLabels);
-//
-//  size_t localError = 0;
-//  for (size_t i = 0; i < trueTestLabels.n_cols; i++)
-//    if (trueTestLabels(i) != predictedLabels(i))
-//      localError++;
-//
-//  double lError = (double) localError / trueTestLabels.n_cols;
-//  BOOST_REQUIRE_LE(lError, 0.30);
-//}
-//
-///**
-// * This test case runs the AdaBoost.mh algorithm on a non linearly separable
-// * dataset.  It tests the Classify function and checks for a satisfactory error
-// * rate.
-// */
-//BOOST_AUTO_TEST_CASE(ClassifyTest_NONLINSEP)
-//{
-//  arma::mat inputData;
-//  if (!data::Load("train_nonlinsep.txt", inputData))
-//    BOOST_FAIL("Cannot load test dataset train_nonlinsep.txt!");
-//
-//  arma::Mat<size_t> labels;
-//  if (!data::Load("train_labels_nonlinsep.txt", labels))
-//    BOOST_FAIL("Cannot load labels for train_labels_nonlinsep.txt");
-//
-//  // Define your own weak learner; in this test decision stumps are used.
-//  const size_t numClasses = 2;
-//  const size_t inpBucketSize = 3;
-//
-//  arma::mat testData;
-//
-//  if (!data::Load("test_nonlinsep.txt", testData))
-//    BOOST_FAIL("Cannot load test dataset test_nonlinsep.txt!");
-//
-//  arma::Mat<size_t> trueTestLabels;
-//
-//  if (!data::Load("test_labels_nonlinsep.txt", trueTestLabels))
-//    BOOST_FAIL("Cannot load labels for test_labels_nonlinsep.txt");
-//
-//  arma::Row<size_t> dsPrediction(labels.n_cols);
-//
-//  ID3DecisionStump ds(inputData, labels.row(0), numClasses, inpBucketSize);
-//
-//  // Define parameters for AdaBoost.
-//  size_t iterations = 50;
-//  double tolerance = 1e-10;
-//  AdaBoost<ID3DecisionStump > a(inputData, labels.row(0), numClasses, ds,
-//      iterations, tolerance);
-//
-//  arma::Row<size_t> predictedLabels(testData.n_cols);
-//  a.Classify(testData, predictedLabels);
-//
-//  size_t localError = 0;
-//  for (size_t i = 0; i < trueTestLabels.n_cols; i++)
-//    if (trueTestLabels(i) != predictedLabels(i))
-//      localError++;
-//
-//  double lError = (double) localError / trueTestLabels.n_cols;
-//  BOOST_REQUIRE_LE(lError, 0.30);
-//}
-//
-///**
-// * This test case runs the AdaBoost.mh algorithm on the UCI Iris Dataset.  It
-// * trains it on two thirds of the Iris dataset (iris_train.csv), and tests on
-// * the remaining third of the dataset (iris_test.csv).  It tests the Classify()
-// * function and checks for a satisfactory error rate.
-// */
-//BOOST_AUTO_TEST_CASE(ClassifyTest_IRIS)
-//{
-//  arma::mat inputData;
-//  if (!data::Load("iris_train.csv", inputData))
-//    BOOST_FAIL("Cannot load test dataset iris_train.csv!");
-//
-//  arma::Mat<size_t> labels;
-//  if (!data::Load("iris_train_labels.csv", labels))
-//    BOOST_FAIL("Cannot load labels for iris_train_labels.csv");
-//  const size_t numClasses = max(labels.row(0)) + 1;
-//
-//  // Define your own weak learner, perceptron in this case.
-//  // Run the perceptron for perceptronIter iterations.
-//  size_t perceptronIter = 800;
-//
-//  Perceptron<> p(inputData, labels.row(0), numClasses, perceptronIter);
-//
-//  // Define parameters for AdaBoost.
-//  size_t iterations = 50;
-//  double tolerance = 1e-10;
-//  AdaBoost<> a(inputData, labels.row(0), numClasses, p, iterations, tolerance);
-//
-//  arma::mat testData;
-//  if (!data::Load("iris_test.csv", testData))
-//    BOOST_FAIL("Cannot load test dataset iris_test.csv!");
-//
-//  arma::Row<size_t> predictedLabels(testData.n_cols);
-//
-//  a.Classify(testData, predictedLabels);
-//
-//  arma::Mat<size_t> trueTestLabels;
-//  if (!data::Load("iris_test_labels.csv", trueTestLabels))
-//    BOOST_FAIL("Cannot load test dataset iris_test_labels.csv!");
-//
-//  size_t localError = 0;
-//  for (size_t i = 0; i < trueTestLabels.n_cols; i++)
-//    if (trueTestLabels(i) != predictedLabels(i))
-//      localError++;
-//  double lError = (double) localError / labels.n_cols;
-//  BOOST_REQUIRE_LE(lError, 0.30);
-//}
-//
-///**
-// * Ensure that the Train() function works like it is supposed to, by building
-// * AdaBoost on one dataset and then re-training on another dataset.
-// */
-//BOOST_AUTO_TEST_CASE(TrainTest)
-//{
-//  // First train on the iris dataset.
-//  arma::mat inputData;
-//  if (!data::Load("iris_train.csv", inputData))
-//    BOOST_FAIL("Cannot load test dataset iris_train.csv!");
-//
-//  arma::Mat<size_t> labels;
-//  if (!data::Load("iris_train_labels.csv", labels))
-//    BOOST_FAIL("Cannot load labels for iris_train_labels.csv");
-//
-//  const size_t numClasses = max(labels.row(0)) + 1;
-//
-//  size_t perceptronIter = 800;
-//  Perceptron<> p(inputData, labels.row(0), numClasses, perceptronIter);
-//
-//  // Now train AdaBoost.
-//  size_t iterations = 50;
-//  double tolerance = 1e-10;
-//  AdaBoost<> a(inputData, labels.row(0), numClasses, p, iterations, tolerance);
-//
-//  // Now load another dataset...
-//  if (!data::Load("vc2.csv", inputData))
-//    BOOST_FAIL("Cannot load test dataset vc2.csv!");
-//  if (!data::Load("vc2_labels.txt", labels))
-//    BOOST_FAIL("Cannot load labels for vc2_labels.txt");
-//
-//  const size_t newNumClasses = max(labels.row(0)) + 1;
-//
-//  Perceptron<> p2(inputData, labels.row(0), newNumClasses, perceptronIter);
-//
-//  a.Train(inputData, labels.row(0), newNumClasses, p2, iterations, tolerance);
-//
-//  // Load test set to see if it trained on vc2 correctly.
-//  arma::mat testData;
-//  if (!data::Load("vc2_test.csv", testData))
-//    BOOST_FAIL("Cannot load test dataset vc2_test.csv!");
-//
-//  arma::Mat<size_t> trueTestLabels;
-//  if (!data::Load("vc2_test_labels.txt", trueTestLabels))
-//    BOOST_FAIL("Cannot load labels for vc2_test_labels.txt");
-//
-//  // Define parameters for AdaBoost.
-//  arma::Row<size_t> predictedLabels(testData.n_cols);
-//  a.Classify(testData, predictedLabels);
-//
-//  int localError = 0;
-//  for (size_t i = 0; i < trueTestLabels.n_cols; i++)
-//    if (trueTestLabels(i) != predictedLabels(i))
-//      localError++;
-//
-//  double lError = (double) localError / trueTestLabels.n_cols;
-//
-//  BOOST_REQUIRE_LE(lError, 0.30);
-//}
-//
-//BOOST_AUTO_TEST_CASE(PerceptronSerializationTest)
-//{
-//  // Build an AdaBoost object.
-//  mat data = randu<mat>(10, 500);
-//  Row<size_t> labels(500);
-//  for (size_t i = 0; i < 250; ++i)
-//    labels[i] = 0;
-//  for (size_t i = 250; i < 500; ++i)
-//    labels[i] = 1;
-//
-//  Perceptron<> p(data, labels, 2, 800);
-//  AdaBoost<> ab(data, labels, 2, p, 50, 1e-10);
-//
-//  // Now create another dataset to train with.
-//  mat otherData = randu<mat>(5, 200);
-//  Row<size_t> otherLabels(200);
-//  for (size_t i = 0; i < 100; ++i)
-//    otherLabels[i] = 1;
-//  for (size_t i = 100; i < 150; ++i)
-//    otherLabels[i] = 0;
-//  for (size_t i = 150; i < 200; ++i)
-//    otherLabels[i] = 2;
-//
-//  Perceptron<> p2(otherData, otherLabels, 3, 500);
-//  AdaBoost<> abText(otherData, otherLabels, 3, p2, 50, 1e-10);
-//
-//  AdaBoost<> abXml, abBinary;
-//
-//  SerializeObjectAll(ab, abXml, abText, abBinary);
-//
-//  // Now check that the objects are the same.
-//  BOOST_REQUIRE_CLOSE(ab.Tolerance(), abXml.Tolerance(), 1e-5);
-//  BOOST_REQUIRE_CLOSE(ab.Tolerance(), abText.Tolerance(), 1e-5);
-//  BOOST_REQUIRE_CLOSE(ab.Tolerance(), abBinary.Tolerance(), 1e-5);
-//
-//  BOOST_REQUIRE_CLOSE(ab.ZtProduct(), abXml.ZtProduct(), 1e-5);
-//  BOOST_REQUIRE_CLOSE(ab.ZtProduct(), abText.ZtProduct(), 1e-5);
-//  BOOST_REQUIRE_CLOSE(ab.ZtProduct(), abBinary.ZtProduct(), 1e-5);
-//
-//  BOOST_REQUIRE_EQUAL(ab.WeakLearners(), abXml.WeakLearners());
-//  BOOST_REQUIRE_EQUAL(ab.WeakLearners(), abText.WeakLearners());
-//  BOOST_REQUIRE_EQUAL(ab.WeakLearners(), abBinary.WeakLearners());
-//
-//  for (size_t i = 0; i < ab.WeakLearners(); ++i)
-//  {
-//    CheckMatrices(ab.WeakLearner(i).Weights(),
-//                  abXml.WeakLearner(i).Weights(),
-//                  abText.WeakLearner(i).Weights(),
-//                  abBinary.WeakLearner(i).Weights());
-//
-//    CheckMatrices(ab.WeakLearner(i).Biases(),
-//                  abXml.WeakLearner(i).Biases(),
-//                  abText.WeakLearner(i).Biases(),
-//                  abBinary.WeakLearner(i).Biases());
-//  }
-//}
-//
-//BOOST_AUTO_TEST_CASE(DecisionStumpSerializationTest)
-//{
-//  // Build an AdaBoost object.
-//  mat data = randu<mat>(10, 500);
-//  Row<size_t> labels(500);
-//  for (size_t i = 0; i < 250; ++i)
-//    labels[i] = 0;
-//  for (size_t i = 250; i < 500; ++i)
-//    labels[i] = 1;
-//
-//  DecisionStump<> p(data, labels, 2, 800);
-//  AdaBoost<DecisionStump<>> ab(data, labels, 2, p, 50, 1e-10);
-//
-//  // Now create another dataset to train with.
-//  mat otherData = randu<mat>(5, 200);
-//  Row<size_t> otherLabels(200);
-//  for (size_t i = 0; i < 100; ++i)
-//    otherLabels[i] = 1;
-//  for (size_t i = 100; i < 150; ++i)
-//    otherLabels[i] = 0;
-//  for (size_t i = 150; i < 200; ++i)
-//    otherLabels[i] = 2;
-//
-//  DecisionStump<> p2(otherData, otherLabels, 3, 500);
-//  AdaBoost<DecisionStump<>> abText(otherData, otherLabels, 3, p2, 50, 1e-10);
-//
-//  AdaBoost<DecisionStump<>> abXml, abBinary;
-//
-//  SerializeObjectAll(ab, abXml, abText, abBinary);
-//
-//  // Now check that the objects are the same.
-//  BOOST_REQUIRE_CLOSE(ab.Tolerance(), abXml.Tolerance(), 1e-5);
-//  BOOST_REQUIRE_CLOSE(ab.Tolerance(), abText.Tolerance(), 1e-5);
-//  BOOST_REQUIRE_CLOSE(ab.Tolerance(), abBinary.Tolerance(), 1e-5);
-//
-//  BOOST_REQUIRE_CLOSE(ab.ZtProduct(), abXml.ZtProduct(), 1e-5);
-//  BOOST_REQUIRE_CLOSE(ab.ZtProduct(), abText.ZtProduct(), 1e-5);
-//  BOOST_REQUIRE_CLOSE(ab.ZtProduct(), abBinary.ZtProduct(), 1e-5);
-//
-//  BOOST_REQUIRE_EQUAL(ab.WeakLearners(), abXml.WeakLearners());
-//  BOOST_REQUIRE_EQUAL(ab.WeakLearners(), abText.WeakLearners());
-//  BOOST_REQUIRE_EQUAL(ab.WeakLearners(), abBinary.WeakLearners());
-//
-//  for (size_t i = 0; i < ab.WeakLearners(); ++i)
-//  {
-//    BOOST_REQUIRE_EQUAL(ab.WeakLearner(i).SplitDimension(),
-//                        abXml.WeakLearner(i).SplitDimension());
-//    BOOST_REQUIRE_EQUAL(ab.WeakLearner(i).SplitDimension(),
-//                        abText.WeakLearner(i).SplitDimension());
-//    BOOST_REQUIRE_EQUAL(ab.WeakLearner(i).SplitDimension(),
-//                        abBinary.WeakLearner(i).SplitDimension());
-//
-//    CheckMatrices(ab.WeakLearner(i).Split(),
-//                  abXml.WeakLearner(i).Split(),
-//                  abText.WeakLearner(i).Split(),
-//                  abBinary.WeakLearner(i).Split());
-//
-//    CheckMatrices(ab.WeakLearner(i).BinLabels(),
-//                  abXml.WeakLearner(i).BinLabels(),
-//                  abText.WeakLearner(i).BinLabels(),
-//                  abBinary.WeakLearner(i).BinLabels());
-//  }
-//}
+BOOST_AUTO_TEST_CASE(ID3DecisionStumpSerializationTest)
+{
+  // Build an AdaBoost object.
+  mat data = randu<mat>(10, 500);
+  Row<size_t> labels(500);
+  for (size_t i = 0; i < 250; ++i)
+    labels[i] = 0;
+  for (size_t i = 250; i < 500; ++i)
+    labels[i] = 1;
+
+  ID3DecisionStump p(data, labels, 2, 800);
+  AdaBoost<ID3DecisionStump> ab(data, labels, 2, p, 50, 1e-10);
+
+  // Now create another dataset to train with.
+  mat otherData = randu<mat>(5, 200);
+  Row<size_t> otherLabels(200);
+  for (size_t i = 0; i < 100; ++i)
+    otherLabels[i] = 1;
+  for (size_t i = 100; i < 150; ++i)
+    otherLabels[i] = 0;
+  for (size_t i = 150; i < 200; ++i)
+    otherLabels[i] = 2;
+
+  ID3DecisionStump p2(otherData, otherLabels, 3, 500);
+  AdaBoost<ID3DecisionStump> abText(otherData, otherLabels, 3, p2, 50, 1e-10);
+
+  AdaBoost<ID3DecisionStump> abXml, abBinary;
+
+  SerializeObjectAll(ab, abXml, abText, abBinary);
+
+  // Now check that the objects are the same.
+  BOOST_REQUIRE_CLOSE(ab.Tolerance(), abXml.Tolerance(), 1e-5);
+  BOOST_REQUIRE_CLOSE(ab.Tolerance(), abText.Tolerance(), 1e-5);
+  BOOST_REQUIRE_CLOSE(ab.Tolerance(), abBinary.Tolerance(), 1e-5);
+
+  BOOST_REQUIRE_EQUAL(ab.WeakLearners(), abXml.WeakLearners());
+  BOOST_REQUIRE_EQUAL(ab.WeakLearners(), abText.WeakLearners());
+  BOOST_REQUIRE_EQUAL(ab.WeakLearners(), abBinary.WeakLearners());
+
+  for (size_t i = 0; i < ab.WeakLearners(); ++i)
+  {
+    BOOST_REQUIRE_EQUAL(ab.WeakLearner(i).SplitDimension(),
+                        abXml.WeakLearner(i).SplitDimension());
+    BOOST_REQUIRE_EQUAL(ab.WeakLearner(i).SplitDimension(),
+                        abText.WeakLearner(i).SplitDimension());
+    BOOST_REQUIRE_EQUAL(ab.WeakLearner(i).SplitDimension(),
+                        abBinary.WeakLearner(i).SplitDimension());
+
+  }
+}
 
 BOOST_AUTO_TEST_SUITE_END();

From ab3f7a11788d3d5a2b9e1cabd58e0258dc70dcec Mon Sep 17 00:00:00 2001
From: himanshupathak21061998 <hpathak336@gmail.com>
Date: Sat, 11 Jan 2020 02:26:03 +0530
Subject: [PATCH 13/27] Removing styling errors

---
 src/mlpack/methods/decision_tree/decision_tree_impl.hpp | 2 +-
 src/mlpack/tests/adaboost_test.cpp                      | 1 -
 2 files changed, 1 insertion(+), 2 deletions(-)

diff --git a/src/mlpack/methods/decision_tree/decision_tree_impl.hpp b/src/mlpack/methods/decision_tree/decision_tree_impl.hpp
index f5f832eddf..b11a1a8225 100644
--- a/src/mlpack/methods/decision_tree/decision_tree_impl.hpp
+++ b/src/mlpack/methods/decision_tree/decision_tree_impl.hpp
@@ -272,7 +272,7 @@ DecisionTree<FitnessFunction,
 
   // Pass off work to the weighted Train() method.
   Train<true>(tmpData, 0, tmpData.n_cols, tmpLabels, numClasses, tmpWeights,
-              minimumLeafSize, minimumGainSplit, maximumDepth, dimensionSelector);
+      minimumLeafSize, minimumGainSplit, maximumDepth, dimensionSelector);
 }
 
 //! Construct, don't train.
diff --git a/src/mlpack/tests/adaboost_test.cpp b/src/mlpack/tests/adaboost_test.cpp
index 3a90de388d..56eae3ee59 100644
--- a/src/mlpack/tests/adaboost_test.cpp
+++ b/src/mlpack/tests/adaboost_test.cpp
@@ -891,7 +891,6 @@ BOOST_AUTO_TEST_CASE(ID3DecisionStumpSerializationTest)
                         abText.WeakLearner(i).SplitDimension());
     BOOST_REQUIRE_EQUAL(ab.WeakLearner(i).SplitDimension(),
                         abBinary.WeakLearner(i).SplitDimension());
-
   }
 }
 

From 3ff74de96e895782fb7d112eec04528655df324a Mon Sep 17 00:00:00 2001
From: himanshupathak21061998 <hpathak336@gmail.com>
Date: Fri, 17 Jan 2020 13:09:17 +0530
Subject: [PATCH 14/27] Removing info about new weaklearner

---
 src/mlpack/methods/adaboost/adaboost_impl.hpp           | 2 --
 src/mlpack/methods/decision_tree/decision_tree_impl.hpp | 3 +--
 2 files changed, 1 insertion(+), 4 deletions(-)

diff --git a/src/mlpack/methods/adaboost/adaboost_impl.hpp b/src/mlpack/methods/adaboost/adaboost_impl.hpp
index 35514eb476..88929c06c6 100644
--- a/src/mlpack/methods/adaboost/adaboost_impl.hpp
+++ b/src/mlpack/methods/adaboost/adaboost_impl.hpp
@@ -120,8 +120,6 @@ double AdaBoost<WeakLearnerType, MatType>::Train(
     weights = arma::sum(D);
 
     // Use the existing weak learner to train a new one with new weights.
-    // In fact, the new weak learner just inherits some hyperparameters
-    // from existing weak learner.
     WeakLearnerType w(other, tempData, labels, numClasses, weights);
     w.Classify(tempData, predictedLabels);
 
diff --git a/src/mlpack/methods/decision_tree/decision_tree_impl.hpp b/src/mlpack/methods/decision_tree/decision_tree_impl.hpp
index b11a1a8225..7221ee903b 100644
--- a/src/mlpack/methods/decision_tree/decision_tree_impl.hpp
+++ b/src/mlpack/methods/decision_tree/decision_tree_impl.hpp
@@ -88,11 +88,10 @@ DecisionTree<FitnessFunction,
 
   // Set the correct dimensionality for the dimension selector.
   dimensionSelector.Dimensions() = tmpData.n_rows;
-  arma::Row<size_t> passlabels = tmpLabels;
 
   // Pass off work to the Train() method.
   arma::rowvec weights; // Fake weights, not used.
-  Train<false>(tmpData, 0, tmpData.n_cols, passlabels, numClasses, weights,
+  Train<false>(tmpData, 0, tmpData.n_cols, tmpLabels, numClasses, weights,
       minimumLeafSize, minimumGainSplit, maximumDepth, dimensionSelector);
 }
 

From 04e13b562557c3d7c590b4cf2df5f5ba58135019 Mon Sep 17 00:00:00 2001
From: himanshupathak21061998 <hpathak336@gmail.com>
Date: Fri, 24 Jan 2020 03:21:21 +0530
Subject: [PATCH 15/27] Adding changes in tests

---
 src/mlpack/tests/adaboost_test.cpp | 35 ++++++++++++++++++------------
 1 file changed, 21 insertions(+), 14 deletions(-)

diff --git a/src/mlpack/tests/adaboost_test.cpp b/src/mlpack/tests/adaboost_test.cpp
index 56eae3ee59..d4ab0b77bf 100644
--- a/src/mlpack/tests/adaboost_test.cpp
+++ b/src/mlpack/tests/adaboost_test.cpp
@@ -319,13 +319,14 @@ BOOST_AUTO_TEST_CASE(HammingLossIris_DS)
   // Define your own weak learner, decision stumps in this case.
   const size_t numClasses = 3;
   const size_t inpBucketSize = 6;
-  ID3DecisionStump ds(inputData, labels.row(0), numClasses, inpBucketSize);
+  arma::Row<size_t> labelsvec = labels.row(0);
+  ID3DecisionStump ds(inputData, labelsvec, numClasses, inpBucketSize);
 
   // Define parameters for AdaBoost.
   size_t iterations = 50;
   double tolerance = 1e-10;
   AdaBoost<ID3DecisionStump> a(tolerance);
-  double ztProduct = a.Train(inputData, labels.row(0), numClasses, ds,
+  double ztProduct = a.Train(inputData, labelsvec, numClasses, ds,
       iterations, tolerance);
 
   arma::Row<size_t> predictedLabels;
@@ -363,10 +364,11 @@ BOOST_AUTO_TEST_CASE(WeakLearnerErrorIris_DS)
   // Define your own weak learner, decision stumps in this case.
   const size_t numClasses = 3;
   const size_t inpBucketSize = 6;
+  arma::Row<size_t> labelsvec = labels.row(0);
 
   arma::Row<size_t> dsPrediction(labels.n_cols);
 
-  ID3DecisionStump ds(inputData, labels.row(0), numClasses, inpBucketSize);
+  ID3DecisionStump ds(inputData, labelsvec, numClasses, inpBucketSize);
   ds.Classify(inputData, dsPrediction);
 
   size_t countWeakLearnerError = 0;
@@ -379,7 +381,7 @@ BOOST_AUTO_TEST_CASE(WeakLearnerErrorIris_DS)
   size_t iterations = 50;
   double tolerance = 1e-10;
 
-  AdaBoost<ID3DecisionStump> a(inputData, labels.row(0), numClasses, ds,
+  AdaBoost<ID3DecisionStump> a(inputData, labelsvec, numClasses, ds,
       iterations, tolerance);
 
   arma::Row<size_t> predictedLabels;
@@ -413,15 +415,16 @@ BOOST_AUTO_TEST_CASE(HammingLossBoundVertebralColumn_DS)
   // Define your own weak learner, decision stumps in this case.
   const size_t numClasses = 3;
   const size_t inpBucketSize = 6;
+  arma::Row<size_t> labelsvec = labels.row(0);
 
-  ID3DecisionStump ds(inputData, labels.row(0), numClasses, inpBucketSize);
+  ID3DecisionStump ds(inputData, labelsvec, numClasses, inpBucketSize);
 
   // Define parameters for AdaBoost.
   size_t iterations = 50;
   double tolerance = 1e-10;
 
   AdaBoost<ID3DecisionStump> a(tolerance);
-  double ztProduct = a.Train(inputData, labels.row(0), numClasses, ds,
+  double ztProduct = a.Train(inputData, labelsvec, numClasses, ds,
       iterations, tolerance);
 
   arma::Row<size_t> predictedLabels;
@@ -458,8 +461,9 @@ BOOST_AUTO_TEST_CASE(WeakLearnerErrorVertebralColumn_DS)
   const size_t numClasses = 3;
   const size_t inpBucketSize = 6;
   arma::Row<size_t> dsPrediction(labels.n_cols);
+  arma::Row<size_t> labelsvec = labels.row(0);
 
-  ID3DecisionStump ds(inputData, labels.row(0), numClasses, inpBucketSize);
+  ID3DecisionStump ds(inputData, labelsvec, numClasses, inpBucketSize);
   ds.Classify(inputData, dsPrediction);
 
   size_t countWeakLearnerError = 0;
@@ -472,7 +476,7 @@ BOOST_AUTO_TEST_CASE(WeakLearnerErrorVertebralColumn_DS)
   // Define parameters for AdaBoost.
   size_t iterations = 50;
   double tolerance = 1e-10;
-  AdaBoost<ID3DecisionStump> a(inputData, labels.row(0), numClasses, ds,
+  AdaBoost<ID3DecisionStump> a(inputData, labelsvec, numClasses, ds,
       iterations, tolerance);
 
   arma::Row<size_t> predictedLabels;
@@ -505,15 +509,16 @@ BOOST_AUTO_TEST_CASE(HammingLossBoundNonLinearSepData_DS)
   // Define your own weak learner, decision stumps in this case.
   const size_t numClasses = 2;
   const size_t inpBucketSize = 6;
+  arma::Row<size_t> labelsvec = labels.row(0);
 
-  ID3DecisionStump ds(inputData, labels.row(0), numClasses, inpBucketSize);
+  ID3DecisionStump ds(inputData, labelsvec, numClasses, inpBucketSize);
 
   // Define parameters for Adaboost.
   size_t iterations = 50;
   double tolerance = 1e-10;
 
   AdaBoost<ID3DecisionStump> a(tolerance);
-  double ztProduct = a.Train(inputData, labels.row(0), numClasses, ds,
+  double ztProduct = a.Train(inputData, labelsvec, numClasses, ds,
       iterations, tolerance);
 
   arma::Row<size_t> predictedLabels;
@@ -549,10 +554,11 @@ BOOST_AUTO_TEST_CASE(WeakLearnerErrorNonLinearSepData_DS)
   // Define your own weak learner, decision stumps in this case.
   const size_t numClasses = 2;
   const size_t inpBucketSize = 3;
+  arma::Row<size_t> labelsvec = labels.row(0);
 
   arma::Row<size_t> dsPrediction(labels.n_cols);
 
-  ID3DecisionStump ds(inputData, labels.row(0), numClasses, inpBucketSize);
+  ID3DecisionStump ds(inputData, labelsvec, numClasses, inpBucketSize);
   ds.Classify(inputData, dsPrediction);
 
   size_t countWeakLearnerError = 0;
@@ -565,7 +571,7 @@ BOOST_AUTO_TEST_CASE(WeakLearnerErrorNonLinearSepData_DS)
   size_t iterations = 500;
   double tolerance = 1e-23;
 
-  AdaBoost<ID3DecisionStump > a(inputData, labels.row(0), numClasses, ds,
+  AdaBoost<ID3DecisionStump > a(inputData, labelsvec, numClasses, ds,
       iterations, tolerance);
 
   arma::Row<size_t> predictedLabels;
@@ -650,6 +656,7 @@ BOOST_AUTO_TEST_CASE(ClassifyTest_NONLINSEP)
   // Define your own weak learner; in this test decision stumps are used.
   const size_t numClasses = 2;
   const size_t inpBucketSize = 3;
+  arma::Row<size_t> labelsvec = labels.row(0);
 
   arma::mat testData;
 
@@ -663,12 +670,12 @@ BOOST_AUTO_TEST_CASE(ClassifyTest_NONLINSEP)
 
   arma::Row<size_t> dsPrediction(labels.n_cols);
 
-  ID3DecisionStump ds(inputData, labels.row(0), numClasses, inpBucketSize);
+  ID3DecisionStump ds(inputData, labelsvec, numClasses, inpBucketSize);
 
   // Define parameters for AdaBoost.
   size_t iterations = 50;
   double tolerance = 1e-10;
-  AdaBoost<ID3DecisionStump > a(inputData, labels.row(0), numClasses, ds,
+  AdaBoost<ID3DecisionStump > a(inputData, labelsvec, numClasses, ds,
       iterations, tolerance);
 
   arma::Row<size_t> predictedLabels(testData.n_cols);

From ef27c8fdbde87fe6ce81df142e0ff12e37efdfa8 Mon Sep 17 00:00:00 2001
From: himanshupathak21061998 <hpathak336@gmail.com>
Date: Tue, 28 Jan 2020 01:03:21 +0530
Subject: [PATCH 16/27] Adding deprecation warning

---
 HISTORY.md                                           |  4 ++++
 .../methods/decision_stump/decision_stump_main.cpp   |  4 ++++
 src/mlpack/methods/decision_tree/decision_tree.hpp   |  3 +++
 .../methods/decision_tree/decision_tree_impl.hpp     | 12 ++++--------
 4 files changed, 15 insertions(+), 8 deletions(-)

diff --git a/HISTORY.md b/HISTORY.md
index 42353feb55..35389193c0 100644
--- a/HISTORY.md
+++ b/HISTORY.md
@@ -1,5 +1,9 @@
 ### mlpack ?.?.?
 ###### ????-??-??
+
+  * The DecisionStump class has been marked deprecated use DecisionTree
+    class with `NoRecursion=true` or use `ID3DecisionStump` (#2099).
+
   * Add Model() to the FFN class to access individual layers (#2043).
 
   * Update documentation for pip and conda installation packages (#2044).
diff --git a/src/mlpack/methods/decision_stump/decision_stump_main.cpp b/src/mlpack/methods/decision_stump/decision_stump_main.cpp
index 6f8f308afc..33417e9481 100644
--- a/src/mlpack/methods/decision_stump/decision_stump_main.cpp
+++ b/src/mlpack/methods/decision_stump/decision_stump_main.cpp
@@ -125,6 +125,10 @@ static void mlpackMain()
 
   ReportIgnoredParam({{ "test", false }}, "predictions");
 
+  Log::Warn << "The Class DecisionStump is deprecated and will be removed in "
+            << "mlpack 4.0.0. Please use DecisionTree class with maximum_depth "
+            << "option set to 1 (that will produce a stump)." <<std::endl;
+
   // We must either load a model, or train a new stump.
   DSModel* model;
   if (CLI::HasParam("training"))
diff --git a/src/mlpack/methods/decision_tree/decision_tree.hpp b/src/mlpack/methods/decision_tree/decision_tree.hpp
index 8d3f79db60..48f5852b0d 100644
--- a/src/mlpack/methods/decision_tree/decision_tree.hpp
+++ b/src/mlpack/methods/decision_tree/decision_tree.hpp
@@ -589,6 +589,9 @@ using DecisionStump = DecisionTree<FitnessFunction,
                                    ElemType,
                                    false>;
 
+/**
+ * Convenience typedef for decision tree (Three level decision trees).
+ */
 typedef DecisionTree<InformationGain,
                      BestBinaryNumericSplit,
                      AllCategoricalSplit,
diff --git a/src/mlpack/methods/decision_tree/decision_tree_impl.hpp b/src/mlpack/methods/decision_tree/decision_tree_impl.hpp
index 7221ee903b..87221e3d7d 100644
--- a/src/mlpack/methods/decision_tree/decision_tree_impl.hpp
+++ b/src/mlpack/methods/decision_tree/decision_tree_impl.hpp
@@ -118,10 +118,8 @@ DecisionTree<FitnessFunction,
     const double minimumGainSplit,
     const size_t maximumDepth,
     DimensionSelectionType dimensionSelector,
-    const std::enable_if_t<
-        arma::is_arma_type<
-        typename std::remove_reference<
-        WeightsType>::type>::value>*)
+    const std::enable_if_t<arma::is_arma_type<
+        typename std::remove_reference<WeightsType>::type>::value>*)
 {
   using TrueMatType = typename std::decay<MatType>::type;
   using TrueLabelsType = typename std::decay<LabelsType>::type;
@@ -163,10 +161,8 @@ DecisionTree<FitnessFunction,
     WeightsType weights,
     const size_t minimumLeafSize,
     const double minimumGainSplit,
-    const std::enable_if_t<
-        arma::is_arma_type<
-        typename std::remove_reference<
-        WeightsType>::type>::value>*):
+    const std::enable_if_t<arma::is_arma_type<
+        typename std::remove_reference<WeightsType>::type>::value>*):
         NumericAuxiliarySplitInfo(other),
         CategoricalAuxiliarySplitInfo(other)
 {

From 6d57e8e3a19678e24044f1f7952179f26c007865 Mon Sep 17 00:00:00 2001
From: himanshupathak21061998 <hpathak336@gmail.com>
Date: Tue, 28 Jan 2020 02:00:18 +0530
Subject: [PATCH 17/27] Adding mlpack_deprecated in DecisionStump

---
 .../methods/decision_stump/decision_stump.hpp | 39 ++++++++++---------
 1 file changed, 20 insertions(+), 19 deletions(-)

diff --git a/src/mlpack/methods/decision_stump/decision_stump.hpp b/src/mlpack/methods/decision_stump/decision_stump.hpp
index 416bb2bdbf..2cd4eed7d9 100644
--- a/src/mlpack/methods/decision_stump/decision_stump.hpp
+++ b/src/mlpack/methods/decision_stump/decision_stump.hpp
@@ -27,6 +27,7 @@ namespace decision_stump {
  * each bin.  Bin i is specified by the range [split[i], split[i + 1]).  The
  * last bin has range up to \infty (split[i + 1] does not exist in that case).
  * Points that are below the first bin will take the label of the first bin.
+ * Note: This Class has been and should be removed in mlpack 4.0.0.
  *
  * @tparam MatType Type of matrix that is being used (sparse or dense).
  */
@@ -43,10 +44,10 @@ class DecisionStump
    * @param numClasses Number of distinct classes in labels.
    * @param bucketSize Minimum size of bucket when splitting.
    */
-  DecisionStump(const MatType& data,
-                const arma::Row<size_t>& labels,
-                const size_t numClasses,
-                const size_t bucketSize = 10);
+  mlpack_deprecated DecisionStump(const MatType& data,
+                                  const arma::Row<size_t>& labels,
+                                  const size_t numClasses,
+                                  const size_t bucketSize = 10);
 
   /**
    * Alternate constructor which copies the parameters bucketSize and classes
@@ -59,11 +60,11 @@ class DecisionStump
    * @param labels The labels of data.
    * @param weights Weight vector to use while training. For boosting purposes.
    */
-  DecisionStump(const DecisionStump<>& other,
-                const MatType& data,
-                const arma::Row<size_t>& labels,
-                const size_t numClasses,
-                const arma::rowvec& weights);
+  mlpack_deprecated DecisionStump(const DecisionStump<>& other,
+                                  const MatType& data,
+                                  const arma::Row<size_t>& labels,
+                                  const size_t numClasses,
+                                  const arma::rowvec& weights);
 
   /**
    * Create a decision stump without training.  This stump will not be useful
@@ -83,10 +84,10 @@ class DecisionStump
    * @param bucketSize Minimum size of bucket when splitting.
    * @return The final entropy after splitting.
    */
-  double Train(const MatType& data,
-               const arma::Row<size_t>& labels,
-               const size_t numClasses,
-               const size_t bucketSize);
+  mlpack_deprecated double Train(const MatType& data,
+                                 const arma::Row<size_t>& labels,
+                                 const size_t numClasses,
+                                 const size_t bucketSize);
 
   /**
    * Train the decision stump on the given data, with the given weights.  This
@@ -100,11 +101,11 @@ class DecisionStump
    * @param bucketSize Minimum size of bucket when splitting.
    * @return The final entropy after splitting.
    */
-  double Train(const MatType& data,
-               const arma::Row<size_t>& labels,
-               const arma::rowvec& weights,
-               const size_t numClasses,
-               const size_t bucketSize);
+  mlpack_deprecated double Train(const MatType& data,
+                                 const arma::Row<size_t>& labels,
+                                 const arma::rowvec& weights,
+                                 const size_t numClasses,
+                                 const size_t bucketSize);
 
   /**
    * Classification function. After training, classify test, and put the
@@ -114,7 +115,7 @@ class DecisionStump
    * @param predictedLabels Vector to store the predicted classes after
    *     classifying test data.
    */
-  void Classify(const MatType& test, arma::Row<size_t>& predictedLabels);
+  mlpack_deprecated void Classify(const MatType& test, arma::Row<size_t>& predictedLabels);
 
   //! Access the splitting dimension.
   size_t SplitDimension() const { return splitDimension; }

From fbc6f278c71fba975cdb9021d735b7699feadbb6 Mon Sep 17 00:00:00 2001
From: himanshupathak21061998 <hpathak336@gmail.com>
Date: Tue, 28 Jan 2020 02:05:02 +0530
Subject: [PATCH 18/27] Style fix

---
 src/mlpack/methods/decision_stump/decision_stump.hpp | 3 ++-
 src/mlpack/methods/decision_tree/decision_tree.hpp   | 3 ++-
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/src/mlpack/methods/decision_stump/decision_stump.hpp b/src/mlpack/methods/decision_stump/decision_stump.hpp
index 2cd4eed7d9..7f2b495124 100644
--- a/src/mlpack/methods/decision_stump/decision_stump.hpp
+++ b/src/mlpack/methods/decision_stump/decision_stump.hpp
@@ -115,7 +115,8 @@ class DecisionStump
    * @param predictedLabels Vector to store the predicted classes after
    *     classifying test data.
    */
-  mlpack_deprecated void Classify(const MatType& test, arma::Row<size_t>& predictedLabels);
+  mlpack_deprecated void Classify(const MatType& test,
+                                  arma::Row<size_t>& predictedLabels);
 
   //! Access the splitting dimension.
   size_t SplitDimension() const { return splitDimension; }
diff --git a/src/mlpack/methods/decision_tree/decision_tree.hpp b/src/mlpack/methods/decision_tree/decision_tree.hpp
index 48f5852b0d..9b3deef594 100644
--- a/src/mlpack/methods/decision_tree/decision_tree.hpp
+++ b/src/mlpack/methods/decision_tree/decision_tree.hpp
@@ -590,7 +590,8 @@ using DecisionStump = DecisionTree<FitnessFunction,
                                    false>;
 
 /**
- * Convenience typedef for decision tree (Three level decision trees).
+ * Convenience typedef for ID3decision stumps (single level decision trees
+ * made with ID3 algorithm).
  */
 typedef DecisionTree<InformationGain,
                      BestBinaryNumericSplit,

From 701f4e2103a9d41a8bd140e81bd0ceb892643612 Mon Sep 17 00:00:00 2001
From: himanshupathak21061998 <hpathak336@gmail.com>
Date: Thu, 6 Feb 2020 22:37:31 +0530
Subject: [PATCH 19/27] Adding gelu activation function

---
 .../ann/activation_functions/CMakeLists.txt   |  1 +
 .../activation_functions/gelu_function.hpp    | 91 +++++++++++++++++++
 src/mlpack/methods/ann/layer/base_layer.hpp   | 12 +++
 .../tests/activation_functions_test.cpp       |  1 +
 4 files changed, 105 insertions(+)
 create mode 100644 src/mlpack/methods/ann/activation_functions/gelu_function.hpp

diff --git a/src/mlpack/methods/ann/activation_functions/CMakeLists.txt b/src/mlpack/methods/ann/activation_functions/CMakeLists.txt
index 50445dcc47..4cf9e40295 100644
--- a/src/mlpack/methods/ann/activation_functions/CMakeLists.txt
+++ b/src/mlpack/methods/ann/activation_functions/CMakeLists.txt
@@ -9,6 +9,7 @@ set(SOURCES
   softplus_function.hpp
   swish_function.hpp
   mish_function.hpp
+  gelu_function.hpp
 )
 
 # Add directory name to sources.
diff --git a/src/mlpack/methods/ann/activation_functions/gelu_function.hpp b/src/mlpack/methods/ann/activation_functions/gelu_function.hpp
new file mode 100644
index 0000000000..d6c23f9ae4
--- /dev/null
+++ b/src/mlpack/methods/ann/activation_functions/gelu_function.hpp
@@ -0,0 +1,91 @@
+
+/**
+ * @file gelu_function.hpp
+ * @author Himanshu Pathak
+ *
+ * Definition and implementation of the Gaussian Error Linear Unit (GELU)
+ * function.
+ *
+ * mlpack is free software; you may redistribute it and/or modify it under the
+ * terms of the 3-clause BSD license.  You should have received a copy of the
+ * 3-clause BSD license along with mlpack.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef MLPACK_METHODS_ANN_ACTIVATION_FUNCTIONS_GELU_FUNCTION_HPP
+#define MLPACK_METHODS_ANN_ACTIVATION_FUNCTIONS_GELU_FUNCTION_HPP
+# define PI          3.141592653589793238462643383279502884L /* pi */
+
+#include <mlpack/prereqs.hpp>
+
+namespace mlpack {
+namespace ann /** Artificial Neural Network. */ {
+
+/**
+ * The GELU function, defined by
+ *
+ * @f{eqnarray*}{
+ * f(x) = 0.5 * x * {1 + tanh[(2/pi)^(1/2) * (x + 0.044715 * x^3)]} \\
+ * f'(x) = 0.5 * tanh(0.0356774 * x^3) + 0.797885 * x) + 
+ *         (0.0535161x^3 + 0.398942 * x) * 
+ *         sech^2(0.0356774 * x^3+0.797885 * x) + 0.5\\
+ * @f}
+ */
+class GELUFunction
+{
+ public:
+  /**
+   * Computes the GELU function.
+   *
+   * @param x Input data.
+   * @return f(x).
+   */
+  static double Fn(const double x)
+  {
+    return 0.5 * x * (1 + std::tanh(std::sqrt(2/PI)*(x + 0.044715 * std::pow(x, 3))));
+  }
+
+  /**
+   * Computes the tanh function.
+   *
+   * @param x Input data.
+   * @param y The resulting output activation.
+   */
+  template<typename InputVecType, typename OutputVecType>
+  static void Fn(const InputVecType& x, OutputVecType& y)
+  {
+    y = 0.5 * x * (1 + arma::tanh(std::sqrt(2/PI)*(x + 0.044715 * arma::pow(x, 3))));
+  }
+
+  /**
+   * Computes the first derivative of the tanh function.
+   *
+   * @param y Input data.
+   * @return f'(x)
+   */
+  static double Deriv(const double y)
+  {
+    return 0.5 * std::tanh(0.0356774 * std::pow(y, 3) + 0.797885 * y) +
+           (0.0535161 * std::pow(y, 3) + 0.398942 * y) *
+           std::pow(1 / std::cosh(0.0356774 * std::pow(y, 3) + 0.797885 * y), 2) + 0.5;
+  }
+
+  /**
+   * Computes the first derivatives of the tanh function.
+   *
+   * @param y Input data.
+   * @param x The resulting derivatives.
+   */
+  template<typename InputVecType, typename OutputVecType>
+  static void Deriv(const InputVecType& y, OutputVecType& x)
+  {
+    x = 0.5 * arma::tanh(0.0356774 * arma::pow(y, 3) + 0.797885 * y) +
+        (0.0535161 * arma::pow(y, 3) + 0.398942 * y) *
+        arma::pow(1 / arma::cosh(0.0356774 * arma::pow(y, 3) + 0.797885 * y), 2) + 0.5;
+  }
+
+}; // class GELUFunction
+
+} // namespace ann
+} // namespace mlpack
+
+#endif
\ No newline at end of file
diff --git a/src/mlpack/methods/ann/layer/base_layer.hpp b/src/mlpack/methods/ann/layer/base_layer.hpp
index 90ce63a281..d0b53c0a2f 100644
--- a/src/mlpack/methods/ann/layer/base_layer.hpp
+++ b/src/mlpack/methods/ann/layer/base_layer.hpp
@@ -22,6 +22,7 @@
 #include <mlpack/methods/ann/activation_functions/hard_sigmoid_function.hpp>
 #include <mlpack/methods/ann/activation_functions/swish_function.hpp>
 #include <mlpack/methods/ann/activation_functions/mish_function.hpp>
+#include <mlpack/methods/ann/activation_functions/gelu_function.hpp>
 
 namespace mlpack {
 namespace ann /** Artificial Neural Network. */ {
@@ -208,6 +209,17 @@ template <
 using MishFunctionLayer = BaseLayer<
     ActivationFunction, InputDataType, OutputDataType>;
 
+/**
+ * Standard Mish-Layer using the GELU activation function.
+ */
+template <
+    class ActivationFunction = GELUFunction,
+    typename InputDataType = arma::mat,
+    typename OutputDataType = arma::mat
+>
+using GELUFunctionLayer = BaseLayer<
+    ActivationFunction, InputDataType, OutputDataType>;
+
 } // namespace ann
 } // namespace mlpack
 
diff --git a/src/mlpack/tests/activation_functions_test.cpp b/src/mlpack/tests/activation_functions_test.cpp
index 54e9448254..702d179d73 100644
--- a/src/mlpack/tests/activation_functions_test.cpp
+++ b/src/mlpack/tests/activation_functions_test.cpp
@@ -22,6 +22,7 @@
 #include <mlpack/methods/ann/activation_functions/swish_function.hpp>
 #include <mlpack/methods/ann/activation_functions/hard_sigmoid_function.hpp>
 #include <mlpack/methods/ann/activation_functions/mish_function.hpp>
+#include <mlpack/methods/ann/activation_functions/gelu_function.hpp>
 
 #include <boost/test/unit_test.hpp>
 #include "test_tools.hpp"

From 7b687608cf1c4fa0fc1bba9a4372088c3163fa39 Mon Sep 17 00:00:00 2001
From: himanshupathak21061998 <hpathak336@gmail.com>
Date: Thu, 6 Feb 2020 22:56:12 +0530
Subject: [PATCH 20/27] Typo fix

---
 .../ann/activation_functions/gelu_function.hpp   | 16 ++++++++++------
 src/mlpack/methods/ann/layer/base_layer.hpp      |  2 +-
 2 files changed, 11 insertions(+), 7 deletions(-)

diff --git a/src/mlpack/methods/ann/activation_functions/gelu_function.hpp b/src/mlpack/methods/ann/activation_functions/gelu_function.hpp
index d6c23f9ae4..e1a1164fba 100644
--- a/src/mlpack/methods/ann/activation_functions/gelu_function.hpp
+++ b/src/mlpack/methods/ann/activation_functions/gelu_function.hpp
@@ -13,7 +13,7 @@
  */
 #ifndef MLPACK_METHODS_ANN_ACTIVATION_FUNCTIONS_GELU_FUNCTION_HPP
 #define MLPACK_METHODS_ANN_ACTIVATION_FUNCTIONS_GELU_FUNCTION_HPP
-# define PI          3.141592653589793238462643383279502884L /* pi */
+# define PI 3.141592653589793238462643383279502884L /* pi */
 
 #include <mlpack/prereqs.hpp>
 
@@ -41,7 +41,8 @@ class GELUFunction
    */
   static double Fn(const double x)
   {
-    return 0.5 * x * (1 + std::tanh(std::sqrt(2/PI)*(x + 0.044715 * std::pow(x, 3))));
+    return 0.5 * x * (1 + std::tanh(std::sqrt(2 / PI) *
+           (x + 0.044715 * std::pow(x, 3))));
   }
 
   /**
@@ -53,7 +54,8 @@ class GELUFunction
   template<typename InputVecType, typename OutputVecType>
   static void Fn(const InputVecType& x, OutputVecType& y)
   {
-    y = 0.5 * x * (1 + arma::tanh(std::sqrt(2/PI)*(x + 0.044715 * arma::pow(x, 3))));
+    y = 0.5 * x * (1 + arma::tanh(std::sqrt(2 / PI) *
+        (x + 0.044715 * arma::pow(x, 3))));
   }
 
   /**
@@ -66,7 +68,8 @@ class GELUFunction
   {
     return 0.5 * std::tanh(0.0356774 * std::pow(y, 3) + 0.797885 * y) +
            (0.0535161 * std::pow(y, 3) + 0.398942 * y) *
-           std::pow(1 / std::cosh(0.0356774 * std::pow(y, 3) + 0.797885 * y), 2) + 0.5;
+           std::pow(1 / std::cosh(0.0356774 * std::pow(y, 3) +
+           0.797885 * y), 2) + 0.5;
   }
 
   /**
@@ -80,7 +83,8 @@ class GELUFunction
   {
     x = 0.5 * arma::tanh(0.0356774 * arma::pow(y, 3) + 0.797885 * y) +
         (0.0535161 * arma::pow(y, 3) + 0.398942 * y) *
-        arma::pow(1 / arma::cosh(0.0356774 * arma::pow(y, 3) + 0.797885 * y), 2) + 0.5;
+        arma::pow(1 / arma::cosh(0.0356774 * arma::pow(y, 3) +
+        0.797885 * y), 2) + 0.5;
   }
 
 }; // class GELUFunction
@@ -88,4 +92,4 @@ class GELUFunction
 } // namespace ann
 } // namespace mlpack
 
-#endif
\ No newline at end of file
+#endif
diff --git a/src/mlpack/methods/ann/layer/base_layer.hpp b/src/mlpack/methods/ann/layer/base_layer.hpp
index d0b53c0a2f..841d797416 100644
--- a/src/mlpack/methods/ann/layer/base_layer.hpp
+++ b/src/mlpack/methods/ann/layer/base_layer.hpp
@@ -210,7 +210,7 @@ using MishFunctionLayer = BaseLayer<
     ActivationFunction, InputDataType, OutputDataType>;
 
 /**
- * Standard Mish-Layer using the GELU activation function.
+ * Standard GELU-Layer using the GELU activation function.
  */
 template <
     class ActivationFunction = GELUFunction,

From dc9bccc8c557035cfd9a8a0930914fe1e29af9b4 Mon Sep 17 00:00:00 2001
From: himanshupathak21061998 <hpathak336@gmail.com>
Date: Thu, 6 Feb 2020 22:59:18 +0530
Subject: [PATCH 21/27] Typo fix

---
 src/mlpack/methods/ann/activation_functions/gelu_function.hpp | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/mlpack/methods/ann/activation_functions/gelu_function.hpp b/src/mlpack/methods/ann/activation_functions/gelu_function.hpp
index e1a1164fba..22d0024842 100644
--- a/src/mlpack/methods/ann/activation_functions/gelu_function.hpp
+++ b/src/mlpack/methods/ann/activation_functions/gelu_function.hpp
@@ -86,7 +86,6 @@ class GELUFunction
         arma::pow(1 / arma::cosh(0.0356774 * arma::pow(y, 3) +
         0.797885 * y), 2) + 0.5;
   }
-
 }; // class GELUFunction
 
 } // namespace ann

From 598509b56b0d85fdb859e386cb1403c47f28e2a2 Mon Sep 17 00:00:00 2001
From: himanshupathak21061998 <hpathak336@gmail.com>
Date: Sat, 8 Feb 2020 21:21:14 +0530
Subject: [PATCH 22/27] Adding tests

---
 .../activation_functions/gelu_function.hpp    | 10 +++++-----
 .../tests/activation_functions_test.cpp       | 19 +++++++++++++++++++
 2 files changed, 24 insertions(+), 5 deletions(-)

diff --git a/src/mlpack/methods/ann/activation_functions/gelu_function.hpp b/src/mlpack/methods/ann/activation_functions/gelu_function.hpp
index 22d0024842..ae68e2308a 100644
--- a/src/mlpack/methods/ann/activation_functions/gelu_function.hpp
+++ b/src/mlpack/methods/ann/activation_functions/gelu_function.hpp
@@ -46,7 +46,7 @@ class GELUFunction
   }
 
   /**
-   * Computes the tanh function.
+   * Computes the GELU function.
    *
    * @param x Input data.
    * @param y The resulting output activation.
@@ -54,12 +54,12 @@ class GELUFunction
   template<typename InputVecType, typename OutputVecType>
   static void Fn(const InputVecType& x, OutputVecType& y)
   {
-    y = 0.5 * x * (1 + arma::tanh(std::sqrt(2 / PI) *
+    y = 0.5 * x % (1 + arma::tanh(std::sqrt(2 / PI) *
         (x + 0.044715 * arma::pow(x, 3))));
   }
 
   /**
-   * Computes the first derivative of the tanh function.
+   * Computes the first derivative of the GELU function.
    *
    * @param y Input data.
    * @return f'(x)
@@ -73,7 +73,7 @@ class GELUFunction
   }
 
   /**
-   * Computes the first derivatives of the tanh function.
+   * Computes the first derivatives of the GELU function.
    *
    * @param y Input data.
    * @param x The resulting derivatives.
@@ -82,7 +82,7 @@ class GELUFunction
   static void Deriv(const InputVecType& y, OutputVecType& x)
   {
     x = 0.5 * arma::tanh(0.0356774 * arma::pow(y, 3) + 0.797885 * y) +
-        (0.0535161 * arma::pow(y, 3) + 0.398942 * y) *
+        (0.0535161 * arma::pow(y, 3) + 0.398942 * y) %
         arma::pow(1 / arma::cosh(0.0356774 * arma::pow(y, 3) +
         0.797885 * y), 2) + 0.5;
   }
diff --git a/src/mlpack/tests/activation_functions_test.cpp b/src/mlpack/tests/activation_functions_test.cpp
index 702d179d73..5a08671da5 100644
--- a/src/mlpack/tests/activation_functions_test.cpp
+++ b/src/mlpack/tests/activation_functions_test.cpp
@@ -656,4 +656,23 @@ BOOST_AUTO_TEST_CASE(MishFunctionTest)
   CheckDerivativeCorrect<MishFunction>(desiredActivations,
                                         desiredDerivatives);
 }
+
+BOOST_AUTO_TEST_CASE(GELUFunctionTest)
+{
+  // Calculated using torch.nn.gelu().
+  const arma::colvec desiredActivations("-0.04540230591222498 3.1981304348379158 \
+  	                                     4.5000 -0.0000 0.84119199060827676 \
+  	                                     -0.15880800939172329 1.954597694087775 \
+  	                                      0.0000");
+
+  const arma::colvec desiredDerivatives("0.57086073237471269 1.0157887154923151 \
+  	                                     1.0019070441688074 0.59999999999999998 \
+  	                                     1.0129296521663382 0.49788131879136671 \
+  	                                     1.0719101501822128 0.59999999999999998");
+
+  CheckActivationCorrect<GELUFunction>(activationData,
+                                       desiredActivations);
+  CheckDerivativeCorrect<MishFunction>(desiredActivations,
+                                        desiredDerivatives);
+}
 BOOST_AUTO_TEST_SUITE_END();

From ea28b6125ac01bacf7812b288be8d15c60fda64f Mon Sep 17 00:00:00 2001
From: himanshupathak21061998 <hpathak336@gmail.com>
Date: Sat, 8 Feb 2020 21:54:02 +0530
Subject: [PATCH 23/27] Style fix

---
 .../tests/activation_functions_test.cpp       | 25 ++++++++++++-------
 1 file changed, 16 insertions(+), 9 deletions(-)

diff --git a/src/mlpack/tests/activation_functions_test.cpp b/src/mlpack/tests/activation_functions_test.cpp
index 5a08671da5..fc9147c843 100644
--- a/src/mlpack/tests/activation_functions_test.cpp
+++ b/src/mlpack/tests/activation_functions_test.cpp
@@ -660,19 +660,26 @@ BOOST_AUTO_TEST_CASE(MishFunctionTest)
 BOOST_AUTO_TEST_CASE(GELUFunctionTest)
 {
   // Calculated using torch.nn.gelu().
-  const arma::colvec desiredActivations("-0.04540230591222498 3.1981304348379158 \
-  	                                     4.5000 -0.0000 0.84119199060827676 \
-  	                                     -0.15880800939172329 1.954597694087775 \
-  	                                      0.0000");
+  const arma::colvec desiredActivations("-0.04540230591222498 \
+  	                                     3.1981304348379158 \
+  	                                     4.5000 -0.0000 \
+  	                                     0.84119199060827676 \
+  	                                     -0.15880800939172329 \
+  	                                     1.954597694087775 \
+  	                                     0.0000");
 
-  const arma::colvec desiredDerivatives("0.57086073237471269 1.0157887154923151 \
-  	                                     1.0019070441688074 0.59999999999999998 \
-  	                                     1.0129296521663382 0.49788131879136671 \
-  	                                     1.0719101501822128 0.59999999999999998");
+  const arma::colvec desiredDerivatives("0.46379920685377229 \
+  	                                     1.0065302165778773 \
+  	                                     1.0000293221871797 \
+  	                                     0.5 \
+  	                                     1.0351344625840642 \
+  	                                     0.37435387859861063 \
+  	                                     1.0909840032535403 \
+  	                                     0.5");
 
   CheckActivationCorrect<GELUFunction>(activationData,
                                        desiredActivations);
-  CheckDerivativeCorrect<MishFunction>(desiredActivations,
+  CheckDerivativeCorrect<GELUFunction>(desiredActivations,
                                         desiredDerivatives);
 }
 BOOST_AUTO_TEST_SUITE_END();

From 628c6fc83213dfa7685d0e1a744182ad81ad89e5 Mon Sep 17 00:00:00 2001
From: himanshupathak21061998 <hpathak336@gmail.com>
Date: Sat, 8 Feb 2020 22:21:20 +0530
Subject: [PATCH 24/27] Style fix

---
 .../activation_functions/gelu_function.hpp    |  1 -
 .../tests/activation_functions_test.cpp       | 28 +++++++++----------
 2 files changed, 14 insertions(+), 15 deletions(-)

diff --git a/src/mlpack/methods/ann/activation_functions/gelu_function.hpp b/src/mlpack/methods/ann/activation_functions/gelu_function.hpp
index ae68e2308a..e5d4d24f84 100644
--- a/src/mlpack/methods/ann/activation_functions/gelu_function.hpp
+++ b/src/mlpack/methods/ann/activation_functions/gelu_function.hpp
@@ -1,4 +1,3 @@
-
 /**
  * @file gelu_function.hpp
  * @author Himanshu Pathak
diff --git a/src/mlpack/tests/activation_functions_test.cpp b/src/mlpack/tests/activation_functions_test.cpp
index fc9147c843..143f037c89 100644
--- a/src/mlpack/tests/activation_functions_test.cpp
+++ b/src/mlpack/tests/activation_functions_test.cpp
@@ -660,22 +660,22 @@ BOOST_AUTO_TEST_CASE(MishFunctionTest)
 BOOST_AUTO_TEST_CASE(GELUFunctionTest)
 {
   // Calculated using torch.nn.gelu().
-  const arma::colvec desiredActivations("-0.04540230591222498 \
-  	                                     3.1981304348379158 \
-  	                                     4.5000 -0.0000 \
-  	                                     0.84119199060827676 \
-  	                                     -0.15880800939172329 \
-  	                                     1.954597694087775 \
-  	                                     0.0000");
+  const arma::colvec desiredActivations("-0.04540230591222 \
+                                          3.1981304348379158 \
+                                          4.5000 -0.0000 \
+                                          0.84119199060827676 \
+                                          -0.15880800939172329 \
+                                          1.954597694087775 \
+                                          0.0000");
 
   const arma::colvec desiredDerivatives("0.46379920685377229 \
-  	                                     1.0065302165778773 \
-  	                                     1.0000293221871797 \
-  	                                     0.5 \
-  	                                     1.0351344625840642 \
-  	                                     0.37435387859861063 \
-  	                                     1.0909840032535403 \
-  	                                     0.5");
+                                          1.0065302165778773 \
+                                          1.0000293221871797 \
+                                          0.5 \
+                                          1.0351344625840642 \
+                                          0.37435387859861063 \
+                                          1.0909840032535403 \
+                                          0.5");
 
   CheckActivationCorrect<GELUFunction>(activationData,
                                        desiredActivations);

From 05e785c8be76355a57d796aef6cf11d7ada85162 Mon Sep 17 00:00:00 2001
From: Omar Shrit <shrit@lri.fr>
Date: Sat, 8 Feb 2020 23:23:10 +0100
Subject: [PATCH 25/27] Remove execution rights from header files

Signed-off-by: Omar Shrit <shrit@lri.fr>
---
 src/mlpack/core/boost_backport/collections_load_imp.hpp           | 0
 src/mlpack/core/boost_backport/collections_save_imp.hpp           | 0
 src/mlpack/core/boost_backport/vector.hpp                         | 0
 src/mlpack/methods/ann/layer/transposed_convolution.hpp           | 0
 src/mlpack/methods/ann/layer/transposed_convolution_impl.hpp      | 0
 .../reinforcement_learning/environment/double_pole_cart.hpp       | 0
 src/mlpack/tests/ann_layer_test.cpp                               | 0
 src/mlpack/tests/data/test_data_3_1000.csv                        | 0
 8 files changed, 0 insertions(+), 0 deletions(-)
 mode change 100755 => 100644 src/mlpack/core/boost_backport/collections_load_imp.hpp
 mode change 100755 => 100644 src/mlpack/core/boost_backport/collections_save_imp.hpp
 mode change 100755 => 100644 src/mlpack/core/boost_backport/vector.hpp
 mode change 100755 => 100644 src/mlpack/methods/ann/layer/transposed_convolution.hpp
 mode change 100755 => 100644 src/mlpack/methods/ann/layer/transposed_convolution_impl.hpp
 mode change 100755 => 100644 src/mlpack/methods/reinforcement_learning/environment/double_pole_cart.hpp
 mode change 100755 => 100644 src/mlpack/tests/ann_layer_test.cpp
 mode change 100755 => 100644 src/mlpack/tests/data/test_data_3_1000.csv

diff --git a/src/mlpack/core/boost_backport/collections_load_imp.hpp b/src/mlpack/core/boost_backport/collections_load_imp.hpp
old mode 100755
new mode 100644
diff --git a/src/mlpack/core/boost_backport/collections_save_imp.hpp b/src/mlpack/core/boost_backport/collections_save_imp.hpp
old mode 100755
new mode 100644
diff --git a/src/mlpack/core/boost_backport/vector.hpp b/src/mlpack/core/boost_backport/vector.hpp
old mode 100755
new mode 100644
diff --git a/src/mlpack/methods/ann/layer/transposed_convolution.hpp b/src/mlpack/methods/ann/layer/transposed_convolution.hpp
old mode 100755
new mode 100644
diff --git a/src/mlpack/methods/ann/layer/transposed_convolution_impl.hpp b/src/mlpack/methods/ann/layer/transposed_convolution_impl.hpp
old mode 100755
new mode 100644
diff --git a/src/mlpack/methods/reinforcement_learning/environment/double_pole_cart.hpp b/src/mlpack/methods/reinforcement_learning/environment/double_pole_cart.hpp
old mode 100755
new mode 100644
diff --git a/src/mlpack/tests/ann_layer_test.cpp b/src/mlpack/tests/ann_layer_test.cpp
old mode 100755
new mode 100644
diff --git a/src/mlpack/tests/data/test_data_3_1000.csv b/src/mlpack/tests/data/test_data_3_1000.csv
old mode 100755
new mode 100644

From 6ff2f0e8f2f14057b8c66e057c062ee435601cbf Mon Sep 17 00:00:00 2001
From: himanshupathak21061998 <hpathak336@gmail.com>
Date: Mon, 10 Feb 2020 02:39:05 +0530
Subject: [PATCH 26/27] Adding changes in histtory.md

---
 HISTORY.md                                                   | 2 ++
 .../methods/ann/activation_functions/gelu_function.hpp       | 5 ++---
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/HISTORY.md b/HISTORY.md
index 03fdf60de4..9bb3ab077c 100644
--- a/HISTORY.md
+++ b/HISTORY.md
@@ -27,6 +27,8 @@
   
   * Add Mish activation function (#2158).
 
+  * Add GELU activation function (#2183).
+
 ### mlpack 3.2.2
 ###### 2019-11-26
   * Add `valid` and `same` padding option in `Convolution` and `Atrous
diff --git a/src/mlpack/methods/ann/activation_functions/gelu_function.hpp b/src/mlpack/methods/ann/activation_functions/gelu_function.hpp
index e5d4d24f84..de16e5019b 100644
--- a/src/mlpack/methods/ann/activation_functions/gelu_function.hpp
+++ b/src/mlpack/methods/ann/activation_functions/gelu_function.hpp
@@ -12,7 +12,6 @@
  */
 #ifndef MLPACK_METHODS_ANN_ACTIVATION_FUNCTIONS_GELU_FUNCTION_HPP
 #define MLPACK_METHODS_ANN_ACTIVATION_FUNCTIONS_GELU_FUNCTION_HPP
-# define PI 3.141592653589793238462643383279502884L /* pi */
 
 #include <mlpack/prereqs.hpp>
 
@@ -40,7 +39,7 @@ class GELUFunction
    */
   static double Fn(const double x)
   {
-    return 0.5 * x * (1 + std::tanh(std::sqrt(2 / PI) *
+    return 0.5 * x * (1 + std::tanh(std::sqrt(2 / M_PI) *
            (x + 0.044715 * std::pow(x, 3))));
   }
 
@@ -53,7 +52,7 @@ class GELUFunction
   template<typename InputVecType, typename OutputVecType>
   static void Fn(const InputVecType& x, OutputVecType& y)
   {
-    y = 0.5 * x % (1 + arma::tanh(std::sqrt(2 / PI) *
+    y = 0.5 * x % (1 + arma::tanh(std::sqrt(2 / M_PI) *
         (x + 0.044715 * arma::pow(x, 3))));
   }
 

From 9664afef7ab06de22e12c684c808db3682b3e4fb Mon Sep 17 00:00:00 2001
From: Ryan Curtin <ryan@ratml.org>
Date: Thu, 13 Feb 2020 20:45:12 -0500
Subject: [PATCH 27/27] Fix some wording.

---
 HISTORY.md                                                | 5 ++---
 src/mlpack/methods/decision_stump/decision_stump.hpp      | 5 ++++-
 src/mlpack/methods/decision_stump/decision_stump_main.cpp | 7 ++++---
 src/mlpack/methods/decision_tree/decision_tree.hpp        | 4 ++--
 4 files changed, 12 insertions(+), 9 deletions(-)

diff --git a/HISTORY.md b/HISTORY.md
index ff3d9093b4..1e10947b61 100644
--- a/HISTORY.md
+++ b/HISTORY.md
@@ -1,8 +1,7 @@
 ### mlpack ?.?.?
 ###### ????-??-??
-
-  * The DecisionStump class has been marked deprecated use DecisionTree
-    class with `NoRecursion=true` or use `ID3DecisionStump` (#2099).
+  * The DecisionStump class has been marked deprecated; use the `DecisionTree`
+    class with `NoRecursion=true` or use `ID3DecisionStump` instead (#2099).
 
   * Added `probabilities_file` parameter to get the probabilities matrix of
     AdaBoost classifier (#2050).
diff --git a/src/mlpack/methods/decision_stump/decision_stump.hpp b/src/mlpack/methods/decision_stump/decision_stump.hpp
index 7f2b495124..e8863908ab 100644
--- a/src/mlpack/methods/decision_stump/decision_stump.hpp
+++ b/src/mlpack/methods/decision_stump/decision_stump.hpp
@@ -27,7 +27,10 @@ namespace decision_stump {
  * each bin.  Bin i is specified by the range [split[i], split[i + 1]).  The
  * last bin has range up to \infty (split[i + 1] does not exist in that case).
  * Points that are below the first bin will take the label of the first bin.
- * Note: This Class has been and should be removed in mlpack 4.0.0.
+ *
+ * @note
+ * This class has been deprecated and should be removed in mlpack 4.0.0.  Use
+ * `ID3DecisionStump`, found in src/mlpack/methods/decision_tree/, instead.
  *
  * @tparam MatType Type of matrix that is being used (sparse or dense).
  */
diff --git a/src/mlpack/methods/decision_stump/decision_stump_main.cpp b/src/mlpack/methods/decision_stump/decision_stump_main.cpp
index 33417e9481..e9e683a61f 100644
--- a/src/mlpack/methods/decision_stump/decision_stump_main.cpp
+++ b/src/mlpack/methods/decision_stump/decision_stump_main.cpp
@@ -125,9 +125,10 @@ static void mlpackMain()
 
   ReportIgnoredParam({{ "test", false }}, "predictions");
 
-  Log::Warn << "The Class DecisionStump is deprecated and will be removed in "
-            << "mlpack 4.0.0. Please use DecisionTree class with maximum_depth "
-            << "option set to 1 (that will produce a stump)." <<std::endl;
+  Log::Warn << "DecisionStump is deprecated and will be removed in mlpack "
+            << "4.0.0. Please use DecisionTree instead with the maximum tree "
+            << "depth option set to 1 (that will produce a stump)."
+            << std::endl;
 
   // We must either load a model, or train a new stump.
   DSModel* model;
diff --git a/src/mlpack/methods/decision_tree/decision_tree.hpp b/src/mlpack/methods/decision_tree/decision_tree.hpp
index 9b3deef594..5e23332cca 100644
--- a/src/mlpack/methods/decision_tree/decision_tree.hpp
+++ b/src/mlpack/methods/decision_tree/decision_tree.hpp
@@ -590,8 +590,8 @@ using DecisionStump = DecisionTree<FitnessFunction,
                                    false>;
 
 /**
- * Convenience typedef for ID3decision stumps (single level decision trees
- * made with ID3 algorithm).
+ * Convenience typedef for ID3 decision stumps (single level decision trees made
+ * with the ID3 algorithm).
  */
 typedef DecisionTree<InformationGain,
                      BestBinaryNumericSplit,