Merge branch 'master' into Add-Lisht-Function

HISTORY.md

@@ -1,5 +1,8 @@
 ### mlpack ?.?.?
 ###### ????-??-??
+  * The DecisionStump class has been marked deprecated; use the `DecisionTree`
+    class with `NoRecursion=true` or use `ID3DecisionStump` instead (#2099).
+
   * Added `probabilities_file` parameter to get the probabilities matrix of
     AdaBoost classifier (#2050).
 

src/mlpack/methods/adaboost/adaboost.hpp

@@ -30,7 +30,7 @@
 
 #include <mlpack/prereqs.hpp>
 #include <mlpack/methods/perceptron/perceptron.hpp>
-#include <mlpack/methods/decision_stump/decision_stump.hpp>
+#include <mlpack/methods/decision_tree/decision_tree.hpp>
 
 namespace mlpack {
 namespace adaboost {

src/mlpack/methods/adaboost/adaboost_main.cpp

@@ -42,7 +42,7 @@ using namespace mlpack;
 using namespace std;
 using namespace arma;
 using namespace mlpack::adaboost;
-using namespace mlpack::decision_stump;
+using namespace mlpack::tree;
 using namespace mlpack::perceptron;
 using namespace mlpack::util;
 

src/mlpack/methods/adaboost/adaboost_model.cpp

@@ -16,7 +16,7 @@ using namespace mlpack;
 using namespace std;
 using namespace arma;
 using namespace mlpack::adaboost;
-using namespace mlpack::decision_stump;
+using namespace mlpack::tree;
 using namespace mlpack::perceptron;
 
 //! Create an empty AdaBoost model.
@@ -47,7 +47,7 @@ AdaBoostModel::AdaBoostModel(const AdaBoostModel& other) :
     mappings(other.mappings),
     weakLearnerType(other.weakLearnerType),
     dsBoost(other.dsBoost == NULL ? NULL :
-        new AdaBoost<DecisionStump<>>(*other.dsBoost)),
+        new AdaBoost<ID3DecisionStump>(*other.dsBoost)),
     pBoost(other.pBoost == NULL ? NULL :
         new AdaBoost<Perceptron<>>(*other.pBoost)),
     dimensionality(other.dimensionality)
@@ -77,7 +77,7 @@ AdaBoostModel& AdaBoostModel::operator=(const AdaBoostModel& other)
 
   delete dsBoost;
   dsBoost = (other.dsBoost == NULL) ? NULL :
-      new AdaBoost<DecisionStump<>>(*other.dsBoost);
+      new AdaBoost<ID3DecisionStump>(*other.dsBoost);
 
   delete pBoost;
   pBoost = (other.pBoost == NULL) ? NULL :
@@ -105,13 +105,13 @@ void AdaBoostModel::Train(const mat& data,
   if (weakLearnerType == WeakLearnerTypes::DECISION_STUMP)
   {
     delete dsBoost;
-
-    DecisionStump<> ds(data, labels, max(labels) + 1);
-    dsBoost = new AdaBoost<DecisionStump<>>(data, labels, numClasses, ds,
+    ID3DecisionStump ds(data, labels, max(labels) + 1);
+    dsBoost = new AdaBoost<ID3DecisionStump>(data, labels, numClasses, ds,
         iterations, tolerance);
   }
   else if (weakLearnerType == WeakLearnerTypes::PERCEPTRON)
   {
+    delete pBoost;
     Perceptron<> p(data, labels, max(labels) + 1);
     pBoost = new AdaBoost<Perceptron<>>(data, labels, numClasses, p, iterations,
         tolerance);

src/mlpack/methods/adaboost/adaboost_model.hpp

@@ -38,7 +38,7 @@ class AdaBoostModel
   //! The type of weak learner.
   size_t weakLearnerType;
   //! Non-NULL if using decision stumps.
-  AdaBoost<decision_stump::DecisionStump<>>* dsBoost;
+  AdaBoost<tree::ID3DecisionStump>* dsBoost;
   //! Non-NULL if using perceptrons.
   AdaBoost<perceptron::Perceptron<>>* pBoost;
   //! Number of dimensions in training data.
@@ -79,7 +79,7 @@ class AdaBoostModel
   //! Modify the dimensionality of the model.
   size_t& Dimensionality() { return dimensionality; }
 
-  //! Train the model.
+  //! Train the model, treat the data is all of the numeric type.
   void Train(const arma::mat& data,
              const arma::Row<size_t>& labels,
              const size_t numClasses,

src/mlpack/methods/ann/activation_functions/CMakeLists.txt

@@ -10,6 +10,7 @@ set(SOURCES
   swish_function.hpp
   mish_function.hpp
   lisht_function.hpp
+  gelu_function.hpp
 )
 
 # Add directory name to sources.

src/mlpack/methods/ann/activation_functions/gelu_function.hpp

@@ -0,0 +1,92 @@
+/**
+ * @file gelu_function.hpp
+ * @author Himanshu Pathak
+ *
+ * Definition and implementation of the Gaussian Error Linear Unit (GELU)
+ * function.
+ *
+ * mlpack is free software; you may redistribute it and/or modify it under the
+ * terms of the 3-clause BSD license.  You should have received a copy of the
+ * 3-clause BSD license along with mlpack.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef MLPACK_METHODS_ANN_ACTIVATION_FUNCTIONS_GELU_FUNCTION_HPP
+#define MLPACK_METHODS_ANN_ACTIVATION_FUNCTIONS_GELU_FUNCTION_HPP
+
+#include <mlpack/prereqs.hpp>
+
+namespace mlpack {
+namespace ann /** Artificial Neural Network. */ {
+
+/**
+ * The GELU function, defined by
+ *
+ * @f{eqnarray*}{
+ * f(x) = 0.5 * x * {1 + tanh[(2/pi)^(1/2) * (x + 0.044715 * x^3)]} \\
+ * f'(x) = 0.5 * tanh(0.0356774 * x^3) + 0.797885 * x) + 
+ *         (0.0535161x^3 + 0.398942 * x) * 
+ *         sech^2(0.0356774 * x^3+0.797885 * x) + 0.5\\
+ * @f}
+ */
+class GELUFunction
+{
+ public:
+  /**
+   * Computes the GELU function.
+   *
+   * @param x Input data.
+   * @return f(x).
+   */
+  static double Fn(const double x)
+  {
+    return 0.5 * x * (1 + std::tanh(std::sqrt(2 / M_PI) *
+           (x + 0.044715 * std::pow(x, 3))));
+  }
+
+  /**
+   * Computes the GELU function.
+   *
+   * @param x Input data.
+   * @param y The resulting output activation.
+   */
+  template<typename InputVecType, typename OutputVecType>
+  static void Fn(const InputVecType& x, OutputVecType& y)
+  {
+    y = 0.5 * x % (1 + arma::tanh(std::sqrt(2 / M_PI) *
+        (x + 0.044715 * arma::pow(x, 3))));
+  }
+
+  /**
+   * Computes the first derivative of the GELU function.
+   *
+   * @param y Input data.
+   * @return f'(x)
+   */
+  static double Deriv(const double y)
+  {
+    return 0.5 * std::tanh(0.0356774 * std::pow(y, 3) + 0.797885 * y) +
+           (0.0535161 * std::pow(y, 3) + 0.398942 * y) *
+           std::pow(1 / std::cosh(0.0356774 * std::pow(y, 3) +
+           0.797885 * y), 2) + 0.5;
+  }
+
+  /**
+   * Computes the first derivatives of the GELU function.
+   *
+   * @param y Input data.
+   * @param x The resulting derivatives.
+   */
+  template<typename InputVecType, typename OutputVecType>
+  static void Deriv(const InputVecType& y, OutputVecType& x)
+  {
+    x = 0.5 * arma::tanh(0.0356774 * arma::pow(y, 3) + 0.797885 * y) +
+        (0.0535161 * arma::pow(y, 3) + 0.398942 * y) %
+        arma::pow(1 / arma::cosh(0.0356774 * arma::pow(y, 3) +
+        0.797885 * y), 2) + 0.5;
+  }
+}; // class GELUFunction
+
+} // namespace ann
+} // namespace mlpack
+
+#endif

src/mlpack/methods/ann/layer/base_layer.hpp

@@ -23,6 +23,7 @@
 #include <mlpack/methods/ann/activation_functions/swish_function.hpp>
 #include <mlpack/methods/ann/activation_functions/mish_function.hpp>
 #include <mlpack/methods/ann/activation_functions/lisht_function.hpp>
+#include <mlpack/methods/ann/activation_functions/gelu_function.hpp>
 
 namespace mlpack {
 namespace ann /** Artificial Neural Network. */ {
@@ -219,6 +220,18 @@ template <
 >
 using LiSHTFunctionLayer = BaseLayer<
     ActivationFunction, InputDataType, OutputDataType>;
+
+/**
+ * Standard GELU-Layer using the GELU activation function.
+ */
+template <
+    class ActivationFunction = GELUFunction,
+    typename InputDataType = arma::mat,
+    typename OutputDataType = arma::mat
+>
+using GELUFunctionLayer = BaseLayer<
+    ActivationFunction, InputDataType, OutputDataType>;
+
 } // namespace ann
 } // namespace mlpack
 

src/mlpack/methods/decision_stump/decision_stump.hpp

@@ -28,6 +28,10 @@ namespace decision_stump {
  * last bin has range up to \infty (split[i + 1] does not exist in that case).
  * Points that are below the first bin will take the label of the first bin.
  *
+ * @note
+ * This class has been deprecated and should be removed in mlpack 4.0.0.  Use
+ * `ID3DecisionStump`, found in src/mlpack/methods/decision_tree/, instead.
+ *
  * @tparam MatType Type of matrix that is being used (sparse or dense).
  */
 template<typename MatType = arma::mat>
@@ -43,10 +47,10 @@ class DecisionStump
    * @param numClasses Number of distinct classes in labels.
    * @param bucketSize Minimum size of bucket when splitting.
    */
-  DecisionStump(const MatType& data,
-                const arma::Row<size_t>& labels,
-                const size_t numClasses,
-                const size_t bucketSize = 10);
+  mlpack_deprecated DecisionStump(const MatType& data,
+                                  const arma::Row<size_t>& labels,
+                                  const size_t numClasses,
+                                  const size_t bucketSize = 10);
 
   /**
    * Alternate constructor which copies the parameters bucketSize and classes
@@ -59,11 +63,11 @@ class DecisionStump
    * @param labels The labels of data.
    * @param weights Weight vector to use while training. For boosting purposes.
    */
-  DecisionStump(const DecisionStump<>& other,
-                const MatType& data,
-                const arma::Row<size_t>& labels,
-                const size_t numClasses,
-                const arma::rowvec& weights);
+  mlpack_deprecated DecisionStump(const DecisionStump<>& other,
+                                  const MatType& data,
+                                  const arma::Row<size_t>& labels,
+                                  const size_t numClasses,
+                                  const arma::rowvec& weights);
 
   /**
    * Create a decision stump without training.  This stump will not be useful
@@ -83,10 +87,10 @@ class DecisionStump
    * @param bucketSize Minimum size of bucket when splitting.
    * @return The final entropy after splitting.
    */
-  double Train(const MatType& data,
-               const arma::Row<size_t>& labels,
-               const size_t numClasses,
-               const size_t bucketSize);
+  mlpack_deprecated double Train(const MatType& data,
+                                 const arma::Row<size_t>& labels,
+                                 const size_t numClasses,
+                                 const size_t bucketSize);
 
   /**
    * Train the decision stump on the given data, with the given weights.  This
@@ -100,11 +104,11 @@ class DecisionStump
    * @param bucketSize Minimum size of bucket when splitting.
    * @return The final entropy after splitting.
    */
-  double Train(const MatType& data,
-               const arma::Row<size_t>& labels,
-               const arma::rowvec& weights,
-               const size_t numClasses,
-               const size_t bucketSize);
+  mlpack_deprecated double Train(const MatType& data,
+                                 const arma::Row<size_t>& labels,
+                                 const arma::rowvec& weights,
+                                 const size_t numClasses,
+                                 const size_t bucketSize);
 
   /**
    * Classification function. After training, classify test, and put the
@@ -114,7 +118,8 @@ class DecisionStump
    * @param predictedLabels Vector to store the predicted classes after
    *     classifying test data.
    */
-  void Classify(const MatType& test, arma::Row<size_t>& predictedLabels);
+  mlpack_deprecated void Classify(const MatType& test,
+                                  arma::Row<size_t>& predictedLabels);
 
   //! Access the splitting dimension.
   size_t SplitDimension() const { return splitDimension; }

src/mlpack/methods/decision_stump/decision_stump_main.cpp

@@ -125,6 +125,11 @@ static void mlpackMain()
 
   ReportIgnoredParam({{ "test", false }}, "predictions");
 
+  Log::Warn << "DecisionStump is deprecated and will be removed in mlpack "
+            << "4.0.0. Please use DecisionTree instead with the maximum tree "
+            << "depth option set to 1 (that will produce a stump)."
+            << std::endl;
+
   // We must either load a model, or train a new stump.
   DSModel* model;
   if (CLI::HasParam("training"))

src/mlpack/methods/decision_tree/decision_tree.hpp

@@ -15,6 +15,7 @@
 
 #include <mlpack/prereqs.hpp>
 #include "gini_gain.hpp"
+#include "information_gain.hpp"
 #include "best_binary_numeric_split.hpp"
 #include "all_categorical_split.hpp"
 #include "all_dimension_select.hpp"
@@ -124,20 +125,49 @@ class DecisionTree :
    * @param dimensionSelector Instantiated dimension selection policy.
    */
   template<typename MatType, typename LabelsType, typename WeightsType>
-  DecisionTree(MatType data,
-               const data::DatasetInfo& datasetInfo,
-               LabelsType labels,
-               const size_t numClasses,
-               WeightsType weights,
-               const size_t minimumLeafSize = 10,
-               const double minimumGainSplit = 1e-7,
-               const size_t maximumDepth = 0,
-               DimensionSelectionType dimensionSelector =
-                   DimensionSelectionType(),
-               const std::enable_if_t<arma::is_arma_type<
-                   typename std::remove_reference<WeightsType>::type>::value>*
-                    = 0);
+  DecisionTree(
+      MatType data,
+      const data::DatasetInfo& datasetInfo,
+      LabelsType labels,
+      const size_t numClasses,
+      WeightsType weights,
+      const size_t minimumLeafSize = 10,
+      const double minimumGainSplit = 1e-7,
+      const size_t maximumDepth = 0,
+      DimensionSelectionType dimensionSelector = DimensionSelectionType(),
+      const std::enable_if_t<arma::is_arma_type<
+          typename std::remove_reference<WeightsType>::type>::value>* = 0);
 
+  /**
+   * Take ownership of another decision tree and train on the given data and 
+   * labels with weights, where the data can be both numeric and categorical.
+   * Setting minimumLeafSize and minimumGainSplit too small may cause the
+   * tree to overfit, but setting them too large may cause it to underfit.
+   *
+   * Use std::move if data, labels or weights are no longer needed to avoid
+   * copies.
+   *
+   * @param other Tree to take ownership of.
+   * @param data Dataset to train on.
+   * @param datasetInfo Type information for each dimension of the dataset.
+   * @param labels Labels for each training point.
+   * @param numClasses Number of classes in the dataset.
+   * @param weights The weight list of given label.
+   * @param minimumLeafSize Minimum number of points in each leaf node.
+   * @param minimumGainSplit Minimum gain for the node to split.
+   */
+  template<typename MatType, typename LabelsType, typename WeightsType>
+  DecisionTree(
+      const DecisionTree& other,
+      MatType data,
+      const data::DatasetInfo& datasetInfo,
+      LabelsType labels,
+      const size_t numClasses,
+      WeightsType weights,
+      const size_t minimumLeafSize = 10,
+      const double minimumGainSplit = 1e-7,
+      const std::enable_if_t<arma::is_arma_type<
+          typename std::remove_reference<WeightsType>::type>::value>* = 0);
   /**
    * Construct the decision tree on the given data and labels with weights,
    * assuming that the data is all of the numeric type. Setting minimumLeafSize
@@ -157,19 +187,49 @@ class DecisionTree :
    * @param dimensionSelector Instantiated dimension selection policy.
    */
   template<typename MatType, typename LabelsType, typename WeightsType>
-  DecisionTree(MatType data,
-               LabelsType labels,
-               const size_t numClasses,
-               WeightsType weights,
-               const size_t minimumLeafSize = 10,
-               const double minimumGainSplit = 1e-7,
-               const size_t maximumDepth = 0,
-               DimensionSelectionType dimensionSelector =
-                   DimensionSelectionType(),
-               const std::enable_if_t<arma::is_arma_type<
-                   typename std::remove_reference<WeightsType>::type>::value>*
-                    = 0);
+  DecisionTree(
+      MatType data,
+      LabelsType labels,
+      const size_t numClasses,
+      WeightsType weights,
+      const size_t minimumLeafSize = 10,
+      const double minimumGainSplit = 1e-7,
+      const size_t maximumDepth = 0,
+      DimensionSelectionType dimensionSelector = DimensionSelectionType(),
+      const std::enable_if_t<arma::is_arma_type<
+          typename std::remove_reference<WeightsType>::type>::value>* = 0);
 
+  /**
+   * Take ownership of another decision tree and train on the given data and labels
+   * with weights, assuming that the data is all of the numeric type. Setting 
+   * minimumLeafSize and minimumGainSplit too small may cause the tree to 
+   * overfit, but setting them too large may cause it to underfit.
+   *
+   * Use std::move if data, labels or weights are no longer needed to avoid
+   * copies.
+   * @param other Tree to take ownership of.
+   * @param data Dataset to train on.
+   * @param labels Labels for each training point.
+   * @param numClasses Number of classes in the dataset.
+   * @param weights The Weight list of given labels.
+   * @param minimumLeafSize Minimum number of points in each leaf node.
+   * @param minimumGainSplit Minimum gain for the node to split.
+   * @param maximumDepth Maximum depth for the tree.
+   * @param dimensionSelector Instantiated dimension selection policy.
+   */
+  template<typename MatType, typename LabelsType, typename WeightsType>
+  DecisionTree(
+      const DecisionTree& other,
+      MatType data,
+      LabelsType labels,
+      const size_t numClasses,
+      WeightsType weights,
+      const size_t minimumLeafSize = 10,
+      const double minimumGainSplit = 1e-7,
+      const size_t maximumDepth = 0,
+      DimensionSelectionType dimensionSelector = DimensionSelectionType(),
+      const std::enable_if_t<arma::is_arma_type<
+          typename std::remove_reference<WeightsType>::type>::value>* = 0);
 
   /**
    * Construct a decision tree without training it.  It will be a leaf node with
@@ -529,6 +589,16 @@ using DecisionStump = DecisionTree<FitnessFunction,
                                    ElemType,
                                    false>;
 
+/**
+ * Convenience typedef for ID3 decision stumps (single level decision trees made
+ * with the ID3 algorithm).
+ */
+typedef DecisionTree<InformationGain,
+                     BestBinaryNumericSplit,
+                     AllCategoricalSplit,
+                     AllDimensionSelect,
+                     double,
+                     true> ID3DecisionStump;
 } // namespace tree
 } // namespace mlpack
 

src/mlpack/methods/decision_tree/decision_tree_impl.hpp

@@ -12,6 +12,8 @@
 #ifndef MLPACK_METHODS_DECISION_TREE_DECISION_TREE_IMPL_HPP
 #define MLPACK_METHODS_DECISION_TREE_DECISION_TREE_IMPL_HPP
 
+#include "decision_tree.hpp"
+
 namespace mlpack {
 namespace tree {
 
@@ -116,10 +118,8 @@ DecisionTree<FitnessFunction,
     const double minimumGainSplit,
     const size_t maximumDepth,
     DimensionSelectionType dimensionSelector,
-    const std::enable_if_t<
-        arma::is_arma_type<
-        typename std::remove_reference<
-        WeightsType>::type>::value>*)
+    const std::enable_if_t<arma::is_arma_type<
+        typename std::remove_reference<WeightsType>::type>::value>*)
 {
   using TrueMatType = typename std::decay<MatType>::type;
   using TrueLabelsType = typename std::decay<LabelsType>::type;
@@ -139,6 +139,47 @@ DecisionTree<FitnessFunction,
       dimensionSelector);
 }
 
+//! Construct and train with weights.
+template<typename FitnessFunction,
+        template<typename> class NumericSplitType,
+        template<typename> class CategoricalSplitType,
+        typename DimensionSelectionType,
+        typename ElemType,
+        bool NoRecursion>
+template<typename MatType, typename LabelsType, typename WeightsType>
+DecisionTree<FitnessFunction,
+        NumericSplitType,
+        CategoricalSplitType,
+        DimensionSelectionType,
+        ElemType,
+        NoRecursion>::DecisionTree(
+    const DecisionTree& other,
+    MatType data,
+    const data::DatasetInfo& datasetInfo,
+    LabelsType labels,
+    const size_t numClasses,
+    WeightsType weights,
+    const size_t minimumLeafSize,
+    const double minimumGainSplit,
+    const std::enable_if_t<arma::is_arma_type<
+        typename std::remove_reference<WeightsType>::type>::value>*):
+        NumericAuxiliarySplitInfo(other),
+        CategoricalAuxiliarySplitInfo(other)
+{
+  using TrueMatType = typename std::decay<MatType>::type;
+  using TrueLabelsType = typename std::decay<LabelsType>::type;
+  using TrueWeightsType = typename std::decay<WeightsType>::type;
+
+  // Copy or move data.
+  TrueMatType tmpData(std::move(data));
+  TrueLabelsType tmpLabels(std::move(labels));
+  TrueWeightsType tmpWeights(std::move(weights));
+
+  // Pass off work to the weighted Train() method.
+  Train<true>(tmpData, 0, tmpData.n_cols, datasetInfo, tmpLabels, numClasses,
+              tmpWeights, minimumLeafSize, minimumGainSplit);
+}
+
 //! Construct and train with weights.
 template<typename FitnessFunction,
          template<typename> class NumericSplitType,
@@ -183,6 +224,52 @@ DecisionTree<FitnessFunction,
       minimumLeafSize, minimumGainSplit, maximumDepth, dimensionSelector);
 }
 
+//! Construct and train with weights.
+template<typename FitnessFunction,
+        template<typename> class NumericSplitType,
+        template<typename> class CategoricalSplitType,
+        typename DimensionSelectionType,
+        typename ElemType,
+        bool NoRecursion>
+template<typename MatType, typename LabelsType, typename WeightsType>
+DecisionTree<FitnessFunction,
+        NumericSplitType,
+        CategoricalSplitType,
+        DimensionSelectionType,
+        ElemType,
+        NoRecursion>::DecisionTree(
+    const DecisionTree& other,
+    MatType data,
+    LabelsType labels,
+    const size_t numClasses,
+    WeightsType weights,
+    const size_t minimumLeafSize,
+    const double minimumGainSplit,
+    const size_t maximumDepth,
+    DimensionSelectionType dimensionSelector,
+    const std::enable_if_t<arma::is_arma_type<
+        typename std::remove_reference<
+        WeightsType>::type>::value>*):
+        NumericAuxiliarySplitInfo(other),
+        CategoricalAuxiliarySplitInfo(other)  // other info does need to copy
+{
+  using TrueMatType = typename std::decay<MatType>::type;
+  using TrueLabelsType = typename std::decay<LabelsType>::type;
+  using TrueWeightsType = typename std::decay<WeightsType>::type;
+
+  // Copy or move data.
+  TrueMatType tmpData(std::move(data));
+  TrueLabelsType tmpLabels(std::move(labels));
+  TrueWeightsType tmpWeights(std::move(weights));
+
+  // Set the correct dimensionality for the dimension selector.
+  dimensionSelector.Dimensions() = tmpData.n_rows;
+
+  // Pass off work to the weighted Train() method.
+  Train<true>(tmpData, 0, tmpData.n_cols, tmpLabels, numClasses, tmpWeights,
+      minimumLeafSize, minimumGainSplit, maximumDepth, dimensionSelector);
+}
+
 //! Construct, don't train.
 template<typename FitnessFunction,
          template<typename> class NumericSplitType,
@@ -568,7 +655,7 @@ double DecisionTree<FitnessFunction,
       dimensionSelector);
 }
 
-//! Train on the given data.
+//! Train on the given data, assuming all dimensions are numeric.
 template<typename FitnessFunction,
          template<typename> class NumericSplitType,
          template<typename> class CategoricalSplitType,

src/mlpack/tests/activation_functions_test.cpp

@@ -23,6 +23,7 @@
 #include <mlpack/methods/ann/activation_functions/hard_sigmoid_function.hpp>
 #include <mlpack/methods/ann/activation_functions/mish_function.hpp>
 #include <mlpack/methods/ann/activation_functions/lisht_function.hpp>
+#include <mlpack/methods/ann/activation_functions/gelu_function.hpp>
 
 #include <boost/test/unit_test.hpp>
 #include "test_tools.hpp"
@@ -635,6 +636,7 @@ BOOST_AUTO_TEST_CASE(HardSigmoidFunctionTest)
   CheckDerivativeCorrect<HardSigmoidFunction>(desiredActivations,
       desiredDerivatives);
 }
+
 /**
  * Basic test of the Mish function.
  */
@@ -678,4 +680,34 @@ BOOST_AUTO_TEST_CASE(LiSHTFunctionTest)
   CheckDerivativeCorrect<LiSHTFunction>(desiredActivations,
                                        desiredDerivatives);
 }
+
+/**
+ * Basic test of the GELU function.
+ */
+BOOST_AUTO_TEST_CASE(GELUFunctionTest)
+{
+  // Calculated using torch.nn.gelu().
+  const arma::colvec desiredActivations("-0.04540230591222 \
+                                          3.1981304348379158 \
+                                          4.5000 -0.0000 \
+                                          0.84119199060827676 \
+                                          -0.15880800939172329 \
+                                          1.954597694087775 \
+                                          0.0000");
+
+  const arma::colvec desiredDerivatives("0.46379920685377229 \
+                                          1.0065302165778773 \
+                                          1.0000293221871797 \
+                                          0.5 \
+                                          1.0351344625840642 \
+                                          0.37435387859861063 \
+                                          1.0909840032535403 \
+                                          0.5");
+
+  CheckActivationCorrect<GELUFunction>(activationData,
+                                       desiredActivations);
+  CheckDerivativeCorrect<GELUFunction>(desiredActivations,
+                                        desiredDerivatives);
+}
+
 BOOST_AUTO_TEST_SUITE_END();

src/mlpack/tests/adaboost_test.cpp

@@ -19,7 +19,7 @@
 using namespace arma;
 using namespace mlpack;
 using namespace mlpack::adaboost;
-using namespace mlpack::decision_stump;
+using namespace mlpack::tree;
 using namespace mlpack::perceptron;
 
 BOOST_AUTO_TEST_SUITE(AdaBoostTest);
@@ -319,13 +319,14 @@ BOOST_AUTO_TEST_CASE(HammingLossIris_DS)
   // Define your own weak learner, decision stumps in this case.
   const size_t numClasses = 3;
   const size_t inpBucketSize = 6;
-  DecisionStump<> ds(inputData, labels.row(0), numClasses, inpBucketSize);
+  arma::Row<size_t> labelsvec = labels.row(0);
+  ID3DecisionStump ds(inputData, labelsvec, numClasses, inpBucketSize);
 
   // Define parameters for AdaBoost.
   size_t iterations = 50;
   double tolerance = 1e-10;
-  AdaBoost<DecisionStump<>> a(tolerance);
-  double ztProduct = a.Train(inputData, labels.row(0), numClasses, ds,
+  AdaBoost<ID3DecisionStump> a(tolerance);
+  double ztProduct = a.Train(inputData, labelsvec, numClasses, ds,
       iterations, tolerance);
 
   arma::Row<size_t> predictedLabels;
@@ -363,10 +364,11 @@ BOOST_AUTO_TEST_CASE(WeakLearnerErrorIris_DS)
   // Define your own weak learner, decision stumps in this case.
   const size_t numClasses = 3;
   const size_t inpBucketSize = 6;
+  arma::Row<size_t> labelsvec = labels.row(0);
 
   arma::Row<size_t> dsPrediction(labels.n_cols);
 
-  DecisionStump<> ds(inputData, labels.row(0), numClasses, inpBucketSize);
+  ID3DecisionStump ds(inputData, labelsvec, numClasses, inpBucketSize);
   ds.Classify(inputData, dsPrediction);
 
   size_t countWeakLearnerError = 0;
@@ -379,7 +381,7 @@ BOOST_AUTO_TEST_CASE(WeakLearnerErrorIris_DS)
   size_t iterations = 50;
   double tolerance = 1e-10;
 
-  AdaBoost<DecisionStump<>> a(inputData, labels.row(0), numClasses, ds,
+  AdaBoost<ID3DecisionStump> a(inputData, labelsvec, numClasses, ds,
       iterations, tolerance);
 
   arma::Row<size_t> predictedLabels;
@@ -413,15 +415,16 @@ BOOST_AUTO_TEST_CASE(HammingLossBoundVertebralColumn_DS)
   // Define your own weak learner, decision stumps in this case.
   const size_t numClasses = 3;
   const size_t inpBucketSize = 6;
+  arma::Row<size_t> labelsvec = labels.row(0);
 
-  DecisionStump<> ds(inputData, labels.row(0), numClasses, inpBucketSize);
+  ID3DecisionStump ds(inputData, labelsvec, numClasses, inpBucketSize);
 
   // Define parameters for AdaBoost.
   size_t iterations = 50;
   double tolerance = 1e-10;
 
-  AdaBoost<DecisionStump<>> a(tolerance);
-  double ztProduct = a.Train(inputData, labels.row(0), numClasses, ds,
+  AdaBoost<ID3DecisionStump> a(tolerance);
+  double ztProduct = a.Train(inputData, labelsvec, numClasses, ds,
       iterations, tolerance);
 
   arma::Row<size_t> predictedLabels;
@@ -458,8 +461,9 @@ BOOST_AUTO_TEST_CASE(WeakLearnerErrorVertebralColumn_DS)
   const size_t numClasses = 3;
   const size_t inpBucketSize = 6;
   arma::Row<size_t> dsPrediction(labels.n_cols);
+  arma::Row<size_t> labelsvec = labels.row(0);
 
-  DecisionStump<> ds(inputData, labels.row(0), numClasses, inpBucketSize);
+  ID3DecisionStump ds(inputData, labelsvec, numClasses, inpBucketSize);
   ds.Classify(inputData, dsPrediction);
 
   size_t countWeakLearnerError = 0;
@@ -472,7 +476,7 @@ BOOST_AUTO_TEST_CASE(WeakLearnerErrorVertebralColumn_DS)
   // Define parameters for AdaBoost.
   size_t iterations = 50;
   double tolerance = 1e-10;
-  AdaBoost<DecisionStump<>> a(inputData, labels.row(0), numClasses, ds,
+  AdaBoost<ID3DecisionStump> a(inputData, labelsvec, numClasses, ds,
       iterations, tolerance);
 
   arma::Row<size_t> predictedLabels;
@@ -505,15 +509,16 @@ BOOST_AUTO_TEST_CASE(HammingLossBoundNonLinearSepData_DS)
   // Define your own weak learner, decision stumps in this case.
   const size_t numClasses = 2;
   const size_t inpBucketSize = 6;
+  arma::Row<size_t> labelsvec = labels.row(0);
 
-  DecisionStump<> ds(inputData, labels.row(0), numClasses, inpBucketSize);
+  ID3DecisionStump ds(inputData, labelsvec, numClasses, inpBucketSize);
 
   // Define parameters for Adaboost.
   size_t iterations = 50;
   double tolerance = 1e-10;
 
-  AdaBoost<DecisionStump<>> a(tolerance);
-  double ztProduct = a.Train(inputData, labels.row(0), numClasses, ds,
+  AdaBoost<ID3DecisionStump> a(tolerance);
+  double ztProduct = a.Train(inputData, labelsvec, numClasses, ds,
       iterations, tolerance);
 
   arma::Row<size_t> predictedLabels;
@@ -549,10 +554,11 @@ BOOST_AUTO_TEST_CASE(WeakLearnerErrorNonLinearSepData_DS)
   // Define your own weak learner, decision stumps in this case.
   const size_t numClasses = 2;
   const size_t inpBucketSize = 3;
+  arma::Row<size_t> labelsvec = labels.row(0);
 
   arma::Row<size_t> dsPrediction(labels.n_cols);
 
-  DecisionStump<> ds(inputData, labels.row(0), numClasses, inpBucketSize);
+  ID3DecisionStump ds(inputData, labelsvec, numClasses, inpBucketSize);
   ds.Classify(inputData, dsPrediction);
 
   size_t countWeakLearnerError = 0;
@@ -565,7 +571,7 @@ BOOST_AUTO_TEST_CASE(WeakLearnerErrorNonLinearSepData_DS)
   size_t iterations = 500;
   double tolerance = 1e-23;
 
-  AdaBoost<DecisionStump<> > a(inputData, labels.row(0), numClasses, ds,
+  AdaBoost<ID3DecisionStump > a(inputData, labelsvec, numClasses, ds,
       iterations, tolerance);
 
   arma::Row<size_t> predictedLabels;
@@ -671,6 +677,7 @@ BOOST_AUTO_TEST_CASE(ClassifyTest_NONLINSEP)
   // Define your own weak learner; in this test decision stumps are used.
   const size_t numClasses = 2;
   const size_t inpBucketSize = 3;
+  arma::Row<size_t> labelsvec = labels.row(0);
 
   arma::mat testData;
 
@@ -684,12 +691,12 @@ BOOST_AUTO_TEST_CASE(ClassifyTest_NONLINSEP)
 
   arma::Row<size_t> dsPrediction(labels.n_cols);
 
-  DecisionStump<> ds(inputData, labels.row(0), numClasses, inpBucketSize);
+  ID3DecisionStump ds(inputData, labelsvec, numClasses, inpBucketSize);
 
   // Define parameters for AdaBoost.
   size_t iterations = 50;
   double tolerance = 1e-10;
-  AdaBoost<DecisionStump<> > a(inputData, labels.row(0), numClasses, ds,
+  AdaBoost<ID3DecisionStump > a(inputData, labelsvec, numClasses, ds,
       iterations, tolerance);
 
   arma::Row<size_t> predictedLabels1(testData.n_cols),
@@ -907,7 +914,7 @@ BOOST_AUTO_TEST_CASE(PerceptronSerializationTest)
   }
 }
 
-BOOST_AUTO_TEST_CASE(DecisionStumpSerializationTest)
+BOOST_AUTO_TEST_CASE(ID3DecisionStumpSerializationTest)
 {
   // Build an AdaBoost object.
   mat data = randu<mat>(10, 500);
@@ -917,8 +924,8 @@ BOOST_AUTO_TEST_CASE(DecisionStumpSerializationTest)
   for (size_t i = 250; i < 500; ++i)
     labels[i] = 1;
 
-  DecisionStump<> p(data, labels, 2, 800);
-  AdaBoost<DecisionStump<>> ab(data, labels, 2, p, 50, 1e-10);
+  ID3DecisionStump p(data, labels, 2, 800);
+  AdaBoost<ID3DecisionStump> ab(data, labels, 2, p, 50, 1e-10);
 
   // Now create another dataset to train with.
   mat otherData = randu<mat>(5, 200);
@@ -930,10 +937,10 @@ BOOST_AUTO_TEST_CASE(DecisionStumpSerializationTest)
   for (size_t i = 150; i < 200; ++i)
     otherLabels[i] = 2;
 
-  DecisionStump<> p2(otherData, otherLabels, 3, 500);
-  AdaBoost<DecisionStump<>> abText(otherData, otherLabels, 3, p2, 50, 1e-10);
+  ID3DecisionStump p2(otherData, otherLabels, 3, 500);
+  AdaBoost<ID3DecisionStump> abText(otherData, otherLabels, 3, p2, 50, 1e-10);
 
-  AdaBoost<DecisionStump<>> abXml, abBinary;
+  AdaBoost<ID3DecisionStump> abXml, abBinary;
 
   SerializeObjectAll(ab, abXml, abText, abBinary);
 
@@ -954,16 +961,6 @@ BOOST_AUTO_TEST_CASE(DecisionStumpSerializationTest)
                         abText.WeakLearner(i).SplitDimension());
     BOOST_REQUIRE_EQUAL(ab.WeakLearner(i).SplitDimension(),
                         abBinary.WeakLearner(i).SplitDimension());
-
-    CheckMatrices(ab.WeakLearner(i).Split(),
-                  abXml.WeakLearner(i).Split(),
-                  abText.WeakLearner(i).Split(),
-                  abBinary.WeakLearner(i).Split());
-
-    CheckMatrices(ab.WeakLearner(i).BinLabels(),
-                  abXml.WeakLearner(i).BinLabels(),
-                  abText.WeakLearner(i).BinLabels(),
-                  abBinary.WeakLearner(i).BinLabels());
   }
 }