Merge pull request #3014 from RishabhGarg108/refactor-tree

Refactor DecisionTreeRegressor to support XGBoost.

src/mlpack/methods/decision_tree/all_categorical_split.hpp

@@ -91,6 +91,8 @@ class AllCategoricalSplit
    * @param minimumGainSplit Minimum  gain split.
    * @param aux Auxiliary split information, which may be modified on a
    *      successful split.
+   * @param fitnessFunction The FitnessFunction object instance. It it used to
+   *      evaluate the gain for the split.
    */
   template<bool UseWeights, typename VecType, typename ResponsesType,
            typename WeightVecType>
@@ -103,7 +105,8 @@ class AllCategoricalSplit
       const size_t minimumLeafSize,
       const double minimumGainSplit,
       double& splitInfo,
-      AuxiliarySplitInfo& aux);
+      AuxiliarySplitInfo& aux,
+      FitnessFunction& fitnessFunction);
 
   /**
    * Return the number of children in the split.

src/mlpack/methods/decision_tree/all_categorical_split_impl.hpp

@@ -125,7 +125,8 @@ double AllCategoricalSplit<FitnessFunction>::SplitIfBetter(
     const size_t minimumLeafSize,
     const double minimumGainSplit,
     double& splitInfo,
-    AuxiliarySplitInfo& /* aux */)
+    AuxiliarySplitInfo& /* aux */,
+    FitnessFunction& fitnessFunction)
 {
   // Count the number of elements in each potential child.
   const double epsilon = 1e-7; // Tolerance for floating-point errors.
@@ -190,7 +191,7 @@ double AllCategoricalSplit<FitnessFunction>::SplitIfBetter(
     const double childPct = UseWeights ?
         double(childWeightSums[i]) / sumWeight :
         double(counts[i]) / double(data.n_elem);
-    const double childGain = FitnessFunction::template Evaluate<UseWeights>(
+    const double childGain = fitnessFunction.template Evaluate<UseWeights>(
         childResponses[i], childWeights[i]);
 
     overallGain += childPct * childGain;

src/mlpack/methods/decision_tree/best_binary_numeric_split.hpp

@@ -104,6 +104,8 @@ class BestBinaryNumericSplit
    * @param splitInfo Stores split information on a successful split.
    * @param aux Auxiliary split information, which may be modified on a
    *      successful split.
+   * @param fitnessFunction The FitnessFunction object instance. It it used to
+   *      evaluate the gain for the split.
    */
   template<bool UseWeights, typename VecType, typename ResponsesType,
            typename WeightVecType>
@@ -118,7 +120,8 @@ class BestBinaryNumericSplit
       const size_t minimumLeafSize,
       const double minimumGainSplit,
       double& splitInfo,
-      AuxiliarySplitInfo& aux);
+      AuxiliarySplitInfo& aux,
+      FitnessFunction& fitnessFunction);
 
   /**
    * Check if we can split a node.  If we can split a node in a way that
@@ -154,7 +157,8 @@ class BestBinaryNumericSplit
       const size_t minimumLeafSize,
       const double minimumGainSplit,
       double& splitInfo,
-      AuxiliarySplitInfo& /* aux */);
+      AuxiliarySplitInfo& /* aux */,
+      FitnessFunction& fitnessFunction);
 
   /**
    * Returns 2, since the binary split always has two children.

src/mlpack/methods/decision_tree/best_binary_numeric_split_impl.hpp

@@ -200,7 +200,8 @@ BestBinaryNumericSplit<FitnessFunction>::SplitIfBetter(
     const size_t minimumLeafSize,
     const double minimumGainSplit,
     double& splitInfo,
-    AuxiliarySplitInfo& /* aux */)
+    AuxiliarySplitInfo& /* aux */,
+    FitnessFunction& fitnessFunction)
 {
   typedef typename ResponsesType::elem_type RType;
   typedef typename WeightVecType::elem_type WType;
@@ -270,9 +271,9 @@ BestBinaryNumericSplit<FitnessFunction>::SplitIfBetter(
       continue;
 
     // Calculate the gain for the left and right child.
-    const double leftGain = FitnessFunction::template
+    const double leftGain = fitnessFunction.template
         Evaluate<UseWeights>(sortedResponses, sortedWeights, 0, index);
-    const double rightGain = FitnessFunction::template
+    const double rightGain = fitnessFunction.template
         Evaluate<UseWeights>(sortedResponses, sortedWeights, index,
             responses.n_elem);
 
@@ -338,13 +339,12 @@ BestBinaryNumericSplit<FitnessFunction>::SplitIfBetter(
     const size_t minimumLeafSize,
     const double minimumGainSplit,
     double& splitInfo,
-    AuxiliarySplitInfo& /* aux */)
+    AuxiliarySplitInfo& /* aux */,
+    FitnessFunction& fitnessFunction)
 {
   typedef typename ResponsesType::elem_type RType;
   typedef typename WeightVecType::elem_type WType;
 
-  FitnessFunction fitnessFunction;
-
   // First sanity check: if we don't have enough points, we can't split.
   if (data.n_elem < (minimumLeafSize * 2))
     return DBL_MAX;

src/mlpack/methods/decision_tree/decision_tree_regressor.hpp

@@ -276,6 +276,8 @@ class DecisionTreeRegressor :
    * @param minimumGainSplit Minimum gain for the node to split.
    * @param maximumDepth Maximum depth for the tree.
    * @param dimensionSelector Instantiated dimension selection policy.
+   * @param fitnessFunction Instantiated fitnessFunction. It is used to
+   *      evaluate the fitness score for splitting each node.
    * @return The final entropy of decision tree.
    */
   template<typename MatType, typename ResponsesType>
@@ -286,7 +288,8 @@ class DecisionTreeRegressor :
                const double minimumGainSplit = 1e-7,
                const size_t maximumDepth = 0,
                DimensionSelectionType dimensionSelector =
-                   DimensionSelectionType());
+                   DimensionSelectionType(),
+               FitnessFunction fitnessFunction = FitnessFunction());
 
   /**
    * Train the decision tree on the given data, assuming that all dimensions are
@@ -302,6 +305,8 @@ class DecisionTreeRegressor :
    * @param minimumGainSplit Minimum gain for the node to split.
    * @param maximumDepth Maximum depth for the tree.
    * @param dimensionSelector Instantiated dimension selection policy.
+   * @param fitnessFunction Instantiated fitnessFunction. It is used to
+   *      evaluate the fitness score for splitting each node.
    * @return The final entropy of decision tree.
    */
   template<typename MatType, typename ResponsesType>
@@ -311,7 +316,8 @@ class DecisionTreeRegressor :
                const double minimumGainSplit = 1e-7,
                const size_t maximumDepth = 0,
                DimensionSelectionType dimensionSelector =
-                   DimensionSelectionType());
+                   DimensionSelectionType(),
+               FitnessFunction fitnessFunction = FitnessFunction());
 
   /**
    * Train the decision tree on the given weighted data.  This will overwrite
@@ -331,6 +337,8 @@ class DecisionTreeRegressor :
    * @param minimumGainSplit Minimum gain for the node to split.
    * @param maximumDepth Maximum depth for the tree.
    * @param dimensionSelector Instantiated dimension selection policy.
+   * @param fitnessFunction Instantiated fitnessFunction. It is used to
+   *      evaluate the fitness score for splitting each node.
    * @return The final entropy of decision tree.
    */
   template<typename MatType, typename ResponsesType, typename WeightsType>
@@ -343,6 +351,7 @@ class DecisionTreeRegressor :
                const size_t maximumDepth = 0,
                DimensionSelectionType dimensionSelector =
                    DimensionSelectionType(),
+               FitnessFunction fitnessFunction = FitnessFunction(),
                const std::enable_if_t<arma::is_arma_type<typename
                    std::remove_reference<WeightsType>::type>::value>* = 0);
 
@@ -362,6 +371,8 @@ class DecisionTreeRegressor :
    * @param minimumGainSplit Minimum gain for the node to split.
    * @param maximumDepth Maximum depth for the tree.
    * @param dimensionSelector Instantiated dimension selection policy.
+   * @param fitnessFunction Instantiated fitnessFunction. It is used to
+   *      evaluate the fitness score for splitting each node.
    * @return The final entropy of decision tree.
    */
   template<typename MatType, typename ResponsesType, typename WeightsType>
@@ -373,6 +384,7 @@ class DecisionTreeRegressor :
                const size_t maximumDepth = 0,
                DimensionSelectionType dimensionSelector =
                    DimensionSelectionType(),
+               FitnessFunction fitnessFunction = FitnessFunction(),
                const std::enable_if_t<arma::is_arma_type<typename
                    std::remove_reference<WeightsType>::type>::value>* = 0);
 
@@ -455,13 +467,6 @@ class DecisionTreeRegressor :
   typedef typename CategoricalSplit::AuxiliarySplitInfo
       CategoricalAuxiliarySplitInfo;
 
-  /**
-   * Calculate the prediction value for the leaf nodes.
-   */
-  template<bool UseWeights, typename ResponsesType, typename WeightsType>
-  void CalculatePrediction(const ResponsesType& responses,
-                           const WeightsType& weights);
-
   /**
    * Corresponding to the public Train() method, this method is designed for
    * avoiding unnecessary copies during training.  This function is called to
@@ -476,6 +481,8 @@ class DecisionTreeRegressor :
    * @param minimumLeafSize Minimum number of points in each leaf node.
    * @param minimumGainSplit Minimum gain for the node to split.
    * @param maximumDepth Maximum depth for the tree.
+   * @param fitnessFunction Instantiated fitnessFunction. It is used to
+   *      evaluate the fitness score for splitting each node.
    * @return The final entropy of decision tree.
    */
   template<bool UseWeights, typename MatType, typename ResponsesType>
@@ -488,7 +495,8 @@ class DecisionTreeRegressor :
                const size_t minimumLeafSize,
                const double minimumGainSplit,
                const size_t maximumDepth,
-               DimensionSelectionType& dimensionSelector);
+               DimensionSelectionType& dimensionSelector,
+               FitnessFunction fitnessFunction = FitnessFunction());
 
   /**
    * Corresponding to the public Train() method, this method is designed for
@@ -503,6 +511,8 @@ class DecisionTreeRegressor :
    * @param minimumLeafSize Minimum number of points in each leaf node.
    * @param minimumGainSplit Minimum gain for the node to split.
    * @param maximumDepth Maximum depth for the tree.
+   * @param fitnessFunction Instantiated fitnessFunction. It is used to
+   *      evaluate the fitness score for splitting each node.
    * @return The final entropy of decision tree.
    */
   template<bool UseWeights, typename MatType, typename ResponsesType>
@@ -514,7 +524,8 @@ class DecisionTreeRegressor :
                const size_t minimumLeafSize,
                const double minimumGainSplit,
                const size_t maximumDepth,
-               DimensionSelectionType& dimensionSelector);
+               DimensionSelectionType& dimensionSelector,
+               FitnessFunction fitnessFunction = FitnessFunction());
 };
 
 

src/mlpack/methods/decision_tree/decision_tree_regressor_impl.hpp

@@ -432,7 +432,8 @@ double DecisionTreeRegressor<FitnessFunction,
     const size_t minimumLeafSize,
     const double minimumGainSplit,
     const size_t maximumDepth,
-    DimensionSelectionType dimensionSelector)
+    DimensionSelectionType dimensionSelector,
+    FitnessFunction fitnessFunction)
 {
   // Sanity check on data.
   util::CheckSameSizes(data, responses, "DecisionTreeRegressor::Train()");
@@ -451,7 +452,7 @@ double DecisionTreeRegressor<FitnessFunction,
   arma::rowvec weights; // Fake weights, not used.
   return Train<false>(tmpData, 0, tmpData.n_cols, datasetInfo, tmpResponses,
       weights, minimumLeafSize, minimumGainSplit, maximumDepth,
-      dimensionSelector);
+      dimensionSelector, fitnessFunction);
 }
 
 //! Train on the given data, assuming all dimensions are numeric.
@@ -471,7 +472,8 @@ double DecisionTreeRegressor<FitnessFunction,
     const size_t minimumLeafSize,
     const double minimumGainSplit,
     const size_t maximumDepth,
-    DimensionSelectionType dimensionSelector)
+    DimensionSelectionType dimensionSelector,
+    FitnessFunction fitnessFunction)
 {
   // Sanity check on data.
   util::CheckSameSizes(data, responses, "DecisionTreeRegressor::Train()");
@@ -490,7 +492,7 @@ double DecisionTreeRegressor<FitnessFunction,
   arma::rowvec weights; // Fake weights, not used.
   return Train<false>(tmpData, 0, tmpData.n_cols, tmpResponses,
       weights, minimumLeafSize, minimumGainSplit, maximumDepth,
-      dimensionSelector);
+      dimensionSelector, fitnessFunction);
 }
 
 //! Train on the given weighted data.
@@ -513,6 +515,7 @@ double DecisionTreeRegressor<FitnessFunction,
     const double minimumGainSplit,
     const size_t maximumDepth,
     DimensionSelectionType dimensionSelector,
+    FitnessFunction fitnessFunction,
     const std::enable_if_t<
         arma::is_arma_type<
         typename std::remove_reference<
@@ -536,7 +539,7 @@ double DecisionTreeRegressor<FitnessFunction,
   // Pass off work to the Train() method.
   return Train<true>(tmpData, 0, tmpData.n_cols, datasetInfo, tmpResponses,
       tmpWeights, minimumLeafSize, minimumGainSplit, maximumDepth,
-      dimensionSelector);
+      dimensionSelector, fitnessFunction);
 }
 
 //! Train on the given weighted all numeric data.
@@ -558,6 +561,7 @@ double DecisionTreeRegressor<FitnessFunction,
     const double minimumGainSplit,
     const size_t maximumDepth,
     DimensionSelectionType dimensionSelector,
+    FitnessFunction fitnessFunction,
     const std::enable_if_t<
         arma::is_arma_type<
         typename std::remove_reference<
@@ -581,7 +585,7 @@ double DecisionTreeRegressor<FitnessFunction,
   // Pass off work to the Train() method.
   return Train<true>(tmpData, 0, tmpData.n_cols, tmpResponses,
       tmpWeights, minimumLeafSize, minimumGainSplit, maximumDepth,
-      dimensionSelector);
+      dimensionSelector, fitnessFunction);
 }
 
 //! Train on the given data.
@@ -605,7 +609,8 @@ double DecisionTreeRegressor<FitnessFunction,
     const size_t minimumLeafSize,
     const double minimumGainSplit,
     const size_t maximumDepth,
-    DimensionSelectionType& dimensionSelector)
+    DimensionSelectionType& dimensionSelector,
+    FitnessFunction fitnessFunction)
 {
   // Clear children if needed.
   for (size_t i = 0; i < children.size(); ++i)
@@ -617,8 +622,8 @@ double DecisionTreeRegressor<FitnessFunction,
   // in numericAux and categoricalAux (and clear them later if we make no
   // split). The split point is stored in splitPointOrPrediction for all
   // internal nodes of the tree.
-  double bestGain = FitnessFunction::template Evaluate<UseWeights>(
-      responses.subvec(begin, begin + count - 1),
+  double bestGain = fitnessFunction.template Evaluate<UseWeights>(
+      responses.cols(begin, begin + count - 1),
       UseWeights ? weights.subvec(begin, begin + count - 1) : weights);
   size_t bestDim = datasetInfo.Dimensionality(); // This means "no split".
   const size_t end = dimensionSelector.End();
@@ -634,23 +639,25 @@ double DecisionTreeRegressor<FitnessFunction,
         dimGain = CategoricalSplit::template SplitIfBetter<UseWeights>(bestGain,
             data.cols(begin, begin + count - 1).row(i),
             datasetInfo.NumMappings(i),
-            responses.subvec(begin, begin + count - 1),
+            responses.cols(begin, begin + count - 1),
             UseWeights ? weights.subvec(begin, begin + count - 1) : weights,
             minimumLeafSize,
             minimumGainSplit,
             splitPointOrPrediction,
-            *this);
+            *this,
+            fitnessFunction);
       }
       else if (datasetInfo.Type(i) == data::Datatype::numeric)
       {
         dimGain = NumericSplit::template SplitIfBetter<UseWeights>(bestGain,
             data.cols(begin, begin + count - 1).row(i),
-            responses.subvec(begin, begin + count - 1),
+            responses.cols(begin, begin + count - 1),
             UseWeights ? weights.subvec(begin, begin + count - 1) : weights,
             minimumLeafSize,
             minimumGainSplit,
             splitPointOrPrediction,
-            *this);
+            *this,
+            fitnessFunction);
       }
 
       // If the splitter reported that it did not split, move to the next
@@ -755,10 +762,11 @@ double DecisionTreeRegressor<FitnessFunction,
     NumericAuxiliarySplitInfo::operator=(NumericAuxiliarySplitInfo());
     CategoricalAuxiliarySplitInfo::operator=(CategoricalAuxiliarySplitInfo());
 
-    // Calculate prediction label because we are a leaf.
-    CalculatePrediction<UseWeights>(
-        responses.subvec(begin, begin + count - 1),
-        UseWeights ? weights.subvec(begin, begin + count - 1) : weights);
+    // Calculate prediction value because we are a leaf.
+    splitPointOrPrediction =
+        fitnessFunction.template OutputLeafValue<UseWeights>(
+            responses.cols(begin, begin + count - 1),
+            UseWeights ? weights.subvec(begin, begin + count - 1) : weights);
   }
 
   return -bestGain;
@@ -784,7 +792,8 @@ double DecisionTreeRegressor<FitnessFunction,
     const size_t minimumLeafSize,
     const double minimumGainSplit,
     const size_t maximumDepth,
-    DimensionSelectionType& dimensionSelector)
+    DimensionSelectionType& dimensionSelector,
+    FitnessFunction fitnessFunction)
 {
   // Clear children if needed.
   for (size_t i = 0; i < children.size(); ++i)
@@ -798,8 +807,8 @@ double DecisionTreeRegressor<FitnessFunction,
   // the best numeric split auxiliary information in numericAux (and clear it
   // later if we don't make a split). The split point is stored in
   // splitPointOrPrediction for all internal nodes of the tree.
-  double bestGain = FitnessFunction::template Evaluate<UseWeights>(
-      responses.subvec(begin, begin + count - 1),
+  double bestGain = fitnessFunction.template Evaluate<UseWeights>(
+      responses.cols(begin, begin + count - 1),
       UseWeights ? weights.subvec(begin, begin + count - 1) : weights);
   size_t bestDim = data.n_rows; // This means "no split".
 
@@ -818,7 +827,8 @@ double DecisionTreeRegressor<FitnessFunction,
                                     minimumLeafSize,
                                     minimumGainSplit,
                                     splitPointOrPrediction,
-                                    *this);
+                                    *this,
+                                    fitnessFunction);
 
       // If the splitter did not report that it improved, then move to the next
       // dimension.
@@ -907,10 +917,11 @@ double DecisionTreeRegressor<FitnessFunction,
     // We won't be needing these members, so reset them.
     NumericAuxiliarySplitInfo::operator=(NumericAuxiliarySplitInfo());
 
-    // Calculate prediction label because we are a leaf.
-    CalculatePrediction<UseWeights>(
-        responses.subvec(begin, begin + count - 1),
-        UseWeights ? weights.subvec(begin, begin + count - 1) : weights);
+    // Calculate prediction value because we are a leaf.
+    splitPointOrPrediction =
+        fitnessFunction.template OutputLeafValue<UseWeights>(
+            responses.cols(begin, begin + count - 1),
+            UseWeights ? weights.subvec(begin, begin + count - 1) : weights);
   }
 
   return -bestGain;
@@ -965,35 +976,6 @@ void DecisionTreeRegressor<FitnessFunction,
     predictions[i] = Predict(data.col(i));
 }
 
-template<typename FitnessFunction,
-         template<typename> class NumericSplitType,
-         template<typename> class CategoricalSplitType,
-         typename DimensionSelectionType,
-         bool NoRecursion>
-template<bool UseWeights, typename ResponsesType, typename WeightsType>
-void DecisionTreeRegressor<FitnessFunction,
-                             NumericSplitType,
-                             CategoricalSplitType,
-                             DimensionSelectionType,
-                             NoRecursion
->::CalculatePrediction(const ResponsesType& responses,
-                       const WeightsType& weights)
-{
-  if (UseWeights)
-  {
-    double accWeights, weightedSum;
-    WeightedSum(responses, weights, 0, responses.n_elem, accWeights,
-        weightedSum);
-    splitPointOrPrediction = weightedSum / accWeights;
-  }
-  else
-  {
-    double sum;
-    Sum(responses, 0, responses.n_elem, sum);
-    splitPointOrPrediction = sum / responses.n_elem;
-  }
-}
-
 template<typename FitnessFunction,
          template<typename> class NumericSplitType,
          template<typename> class CategoricalSplitType,

src/mlpack/methods/decision_tree/mad_gain.hpp

@@ -98,6 +98,30 @@ class MADGain
 
     return Evaluate<UseWeights>(values, weights, 0, values.n_elem);
   }
+
+  /**
+   * Returns the output value for each leaf node for prediction. The output
+   * value is calculated as the average of all the points in that leaf node.
+   * This calculation is specific to regression trees only.
+   */
+  template<bool UseWeights, typename ResponsesType, typename WeightsType>
+  double OutputLeafValue(const ResponsesType& responses,
+                         const WeightsType& weights)
+  {
+    if (UseWeights)
+    {
+      double accWeights, weightedSum;
+      WeightedSum(responses, weights, 0, responses.n_elem, accWeights,
+          weightedSum);
+      return weightedSum / accWeights;
+    }
+    else
+    {
+      double sum;
+      Sum(responses, 0, responses.n_elem, sum);
+      return sum / responses.n_elem;
+    }
+  }
 };
 
 } // namespace tree

src/mlpack/methods/decision_tree/mse_gain.hpp

@@ -95,6 +95,30 @@ class MSEGain
     return Evaluate<UseWeights>(values, weights, 0, values.n_elem);
   }
 
+  /**
+   * Returns the output value for each leaf node for prediction. The output
+   * value is calculated as the average of all the points in that leaf node.
+   * This calculation is specific to regression trees only.
+   */
+  template<bool UseWeights, typename ResponsesType, typename WeightsType>
+  double OutputLeafValue(const ResponsesType& responses,
+                         const WeightsType& weights)
+  {
+    if (UseWeights)
+    {
+      double accWeights, weightedSum;
+      WeightedSum(responses, weights, 0, responses.n_elem, accWeights,
+          weightedSum);
+      return weightedSum / accWeights;
+    }
+    else
+    {
+      double sum;
+      Sum(responses, 0, responses.n_elem, sum);
+      return sum / responses.n_elem;
+    }
+  }
+
   /**
    * Calculates the  mean squared error gain for the left and right children
    * for the current index.

src/mlpack/methods/decision_tree/random_binary_numeric_split.hpp

@@ -107,6 +107,8 @@ class RandomBinaryNumericSplit
    * @param splitInfo Stores split information on a successful split.
    * @param aux Auxiliary split information, which may be modified on a
    *      successful split.
+   * @param fitnessFunction The FitnessFunction object instance. It it used to
+   *      evaluate the gain for the split.
    * @param splitIfBetterGain When set to true, it will split only when gain is
    *      better than the current best gain. Otherwise, it always makes a
    *      split regardless of gain.
@@ -121,6 +123,7 @@ class RandomBinaryNumericSplit
       const double minimumGainSplit,
       double& splitInfo,
       AuxiliarySplitInfo& aux,
+      FitnessFunction& fitnessFunction,
       const bool splitIfBetterGain = false);
 
   /**

src/mlpack/methods/decision_tree/random_binary_numeric_split_impl.hpp

@@ -149,6 +149,7 @@ double RandomBinaryNumericSplit<FitnessFunction>::SplitIfBetter(
     const double minimumGainSplit,
     double& splitInfo,
     AuxiliarySplitInfo& /* aux */,
+    FitnessFunction& fitnessFunction,
     const bool splitIfBetterGain)
 {
   double bestFoundGain = std::min(bestGain + minimumGainSplit, 0.0);
@@ -230,9 +231,9 @@ double RandomBinaryNumericSplit<FitnessFunction>::SplitIfBetter(
   }
 
   // Calculate the gain for the left and right child.
-  const double leftGain = FitnessFunction::template
+  const double leftGain = fitnessFunction.template
       Evaluate<UseWeights>(leftResponses, leftWeights, 0, leftLeafSize);
-  const double rightGain = FitnessFunction::template
+  const double rightGain = fitnessFunction.template
       Evaluate<UseWeights>(rightResponses, rightWeights, 0, rightLeafSize);
 
   // Calculate the gain at this split point.

src/mlpack/tests/decision_tree_regressor_test.cpp

@@ -88,9 +88,9 @@ TEST_CASE("MSEGainEmptyTest", "[DecisionTreeRegressorTest]")
 {
   arma::rowvec weights = arma::ones<arma::rowvec>(10);
   arma::rowvec responses;
+
   REQUIRE(MSEGain::Evaluate<false>(responses, weights) ==
           Approx(0.0).margin(1e-5));
-
   REQUIRE(MSEGain::Evaluate<true>(responses, weights) ==
           Approx(0.0).margin(1e-5));
 }
@@ -107,6 +107,7 @@ TEST_CASE("MSEGainHandCalculation", "[DecisionTreeRegressorTest]")
   // Hand calculated gain values.
   const double gain = -27.08999;
   const double weightedGain = -27.53960;
+
   REQUIRE(MSEGain::Evaluate<false>(responses, weights) ==
           Approx(gain).margin(1e-5));
   REQUIRE(MSEGain::Evaluate<true>(responses, weights) ==
@@ -142,8 +143,7 @@ TEST_CASE("MADGainNormalTest", "[DecisionTreeRegressorTest")
   theoreticalGain /= (double) responses.n_elem;
 
   // Calculated gain.
-  const double calculatedGain =
-      MADGain::Evaluate<false>(responses, weights);
+  const double calculatedGain = MADGain::Evaluate<false>(responses, weights);
 
   REQUIRE(calculatedGain == Approx(theoreticalGain).margin(1e-5));
 }
@@ -155,9 +155,9 @@ TEST_CASE("MADGainEmptyTest", "[DecisionTreeRegressorTest]")
 {
   arma::rowvec weights = arma::ones<arma::rowvec>(10);
   arma::rowvec responses;
+
   REQUIRE(MADGain::Evaluate<false>(responses, weights) ==
           Approx(0.0).margin(1e-5));
-
   REQUIRE(MADGain::Evaluate<true>(responses, weights) ==
           Approx(0.0).margin(1e-5));
 }
@@ -174,6 +174,7 @@ TEST_CASE("MADGainHandCalculation", "[DecisionTreeRegressorTest]")
   // Hand calculated gain values.
   const double gain = -4.1;
   const double weightedGain = -3.8592;
+
   REQUIRE(MADGain::Evaluate<false>(responses, weights) ==
           Approx(gain).margin(1e-5));
   REQUIRE(MADGain::Evaluate<true>(responses, weights) ==
@@ -203,12 +204,13 @@ TEST_CASE("AllCategoricalSplitSimpleSplitTest_", "[DecisionTreeRegressorTest]")
   AllCategoricalSplit<MSEGain>::AuxiliarySplitInfo aux;
 
   // Call the method to do the splitting.
-  const double bestGain = MSEGain::Evaluate<false>(responses, weights);
+  MSEGain f;
+  const double bestGain = f.Evaluate<false>(responses, weights);
   const double gain = AllCategoricalSplit<MSEGain>::SplitIfBetter<false>(
-      bestGain, predictor, 2, responses, weights, 3, 1e-7, splitInfo, aux);
+      bestGain, predictor, 2, responses, weights, 3, 1e-7, splitInfo, aux, f);
   const double weightedGain =
       AllCategoricalSplit<MSEGain>::SplitIfBetter<true>(bestGain, predictor, 2,
-      responses, weights, 3, 1e-7, splitInfo, aux);
+      responses, weights, 3, 1e-7, splitInfo, aux, f);
 
   // Make sure that a split was made.
   REQUIRE(gain > bestGain);
@@ -234,9 +236,10 @@ TEST_CASE("AllCategoricalSplitMinSamplesTest_", "[DecisionTreeRegressorTest]")
   AllCategoricalSplit<MSEGain>::AuxiliarySplitInfo aux;
 
   // Call the method to do the splitting.
-  const double bestGain = MSEGain::Evaluate<false>(responses, weights);
+  MSEGain f;
+  const double bestGain = f.Evaluate<false>(responses, weights);
   const double gain = AllCategoricalSplit<MSEGain>::SplitIfBetter<false>(
-      bestGain, predictors, 4, responses, weights, 4, 1e-7, splitInfo, aux);
+      bestGain, predictors, 4, responses, weights, 4, 1e-7, splitInfo, aux, f);
 
   // Make sure it's not split.
   REQUIRE(gain == DBL_MAX);
@@ -265,13 +268,14 @@ TEST_CASE("AllCategoricalSplitNoGainTest_", "[DecisionTreeRegressorTest]")
   AllCategoricalSplit<MSEGain>::AuxiliarySplitInfo aux;
 
   // Call the method to do the splitting.
-  const double bestGain = MSEGain::Evaluate<false>(responses, weights);
+  MSEGain f;
+  const double bestGain = f.Evaluate<false>(responses, weights);
   const double gain = AllCategoricalSplit<MSEGain>::SplitIfBetter<false>(
       bestGain, predictors, 10, responses, weights, 10, 1e-7,
-      splitInfo, aux);
+      splitInfo, aux, f);
   const double weightedGain =
       AllCategoricalSplit<MSEGain>::SplitIfBetter<true>(bestGain, predictors,
-      10, responses, weights, 10, 1e-7, splitInfo, aux);
+      10, responses, weights, 10, 1e-7, splitInfo, aux, f);
 
   // Make sure that there was no split.
   REQUIRE(gain == DBL_MAX);
@@ -296,12 +300,13 @@ TEST_CASE("BestBinaryNumericSplitSimpleSplitTest_",
   BestBinaryNumericSplit<MADGain>::AuxiliarySplitInfo aux;
 
   // Call the method to do the splitting.
-  const double bestGain = MADGain::Evaluate<false>(responses, weights);
+  MADGain f;
+  const double bestGain = f.Evaluate<false>(responses, weights);
   const double gain = BestBinaryNumericSplit<MADGain>::SplitIfBetter<false>(
-      bestGain, predictors, responses, weights, 3, 1e-7, splitInfo, aux);
+      bestGain, predictors, responses, weights, 3, 1e-7, splitInfo, aux, f);
   const double weightedGain =
       BestBinaryNumericSplit<MADGain>::SplitIfBetter<true>(bestGain, predictors,
-      responses, weights, 3, 1e-7, splitInfo, aux);
+      responses, weights, 3, 1e-7, splitInfo, aux, f);
 
   // Make sure that a split was made.
   REQUIRE(gain > bestGain);
@@ -332,13 +337,14 @@ TEST_CASE("BestBinaryNumericSplitMinSamplesTest_",
   BestBinaryNumericSplit<MSEGain>::AuxiliarySplitInfo aux;
 
   // Call the method to do the splitting.
-  const double bestGain = MSEGain::Evaluate<false>(responses, weights);
+  MSEGain f;
+  const double bestGain = f.Evaluate<false>(responses, weights);
   const double gain = BestBinaryNumericSplit<MSEGain>::SplitIfBetter<false>(
-      bestGain, predictors, responses, weights, 8, 1e-7, splitInfo, aux);
+      bestGain, predictors, responses, weights, 8, 1e-7, splitInfo, aux, f);
   // This should make no difference because it won't split at all.
   const double weightedGain =
       BestBinaryNumericSplit<MSEGain>::SplitIfBetter<true>(bestGain,
-      predictors, responses, weights, 8, 1e-7, splitInfo, aux);
+      predictors, responses, weights, 8, 1e-7, splitInfo, aux, f);
 
   // Make sure that no split was made.
   REQUIRE(gain == DBL_MAX);
@@ -366,9 +372,10 @@ TEST_CASE("BestBinaryNumericSplitNoGainTest_", "[DecisionTreeRegressorTest]")
   BestBinaryNumericSplit<MSEGain>::AuxiliarySplitInfo aux;
 
   // Call the method to do the splitting.
-  const double bestGain = MSEGain::Evaluate<false>(responses, weights);
+  MSEGain f;
+  const double bestGain = f.Evaluate<false>(responses, weights);
   const double gain = BestBinaryNumericSplit<MSEGain>::SplitIfBetter<false>(
-      bestGain, predictors, responses, weights, 10, 1e-7, splitInfo, aux);
+      bestGain, predictors, responses, weights, 10, 1e-7, splitInfo, aux, f);
 
   // Make sure there was no split.
   REQUIRE(gain == DBL_MAX);
@@ -390,12 +397,13 @@ TEST_CASE("RandomBinaryNumericSplitAlwaysSplit_",
   RandomBinaryNumericSplit<MSEGain>::AuxiliarySplitInfo aux;
 
   // Call the method to do the splitting.
-  const double bestGain = MSEGain::Evaluate<false>(responses, weights);
+  MSEGain f;
+  const double bestGain = f.Evaluate<false>(responses, weights);
   const double gain = RandomBinaryNumericSplit<MSEGain>::SplitIfBetter<false>(
-      bestGain, values, responses, weights, 1, 1e-7, splitInfo, aux);
+      bestGain, values, responses, weights, 1, 1e-7, splitInfo, aux, f);
   const double weightedGain =
       RandomBinaryNumericSplit<MSEGain>::SplitIfBetter<true>(bestGain, values,
-      responses, weights, 1, 1e-7, splitInfo, aux);
+      responses, weights, 1, 1e-7, splitInfo, aux, f);
 
   // Make sure that split was made.
   REQUIRE(gain != DBL_MAX);
@@ -417,13 +425,14 @@ TEST_CASE("RandomBinaryNumericSplitMinSamplesTest_",
   RandomBinaryNumericSplit<MSEGain>::AuxiliarySplitInfo aux;
 
   // Call the method to do the splitting.
-  const double bestGain = MSEGain::Evaluate<false>(responses, weights);
+  MSEGain f;
+  const double bestGain = f.Evaluate<false>(responses, weights);
   const double gain = RandomBinaryNumericSplit<MSEGain>::SplitIfBetter<false>(
-      bestGain, values, responses, weights, 8, 1e-7, splitInfo, aux);
+      bestGain, values, responses, weights, 8, 1e-7, splitInfo, aux, f);
   // This should make no difference because it won't split at all.
   const double weightedGain =
       RandomBinaryNumericSplit<MSEGain>::SplitIfBetter<true>(bestGain, values,
-      responses, weights, 8, 1e-7, splitInfo, aux);
+      responses, weights, 8, 1e-7, splitInfo, aux, f);
 
   // Make sure that no split was made.
   REQUIRE(gain == DBL_MAX);
@@ -451,9 +460,10 @@ TEST_CASE("RandomBinaryNumericSplitNoGainTest_", "[DecisionTreeRegressorTest]")
   RandomBinaryNumericSplit<MSEGain>::AuxiliarySplitInfo aux;
 
   // Call the method to do the splitting.
-  const double bestGain = MSEGain::Evaluate<false>(responses, weights);
+  MSEGain f;
+  const double bestGain = f.Evaluate<false>(responses, weights);
   const double gain = RandomBinaryNumericSplit<MSEGain>::SplitIfBetter<false>(
-      bestGain, values, responses, weights, 10, 1e-7, splitInfo, aux, true);
+      bestGain, values, responses, weights, 10, 1e-7, splitInfo, aux, f, true);
 
   // Make sure there was no split.
   REQUIRE(gain == DBL_MAX);