apaam/mlpack
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Added BestBinaryCategoricalSplit splitter for DecisionTree.

Browse Source
This commit is contained in:
nikolay apanasov
2024-06-27 23:37:12 -07:00
parent 98b5bdf42d
commit c4d0aab14f
16 changed files with 18417 additions and 118 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/mlpack/methods/decision_tree/all_categorical_split.hpp
+7 -5
View File
@@ -103,21 +103,23 @@ class AllCategoricalSplit
const WeightVecType& weights,
const size_t minimumLeafSize,
const double minimumGainSplit,
double& splitInfo,
arma::vec& splitInfo,
AuxiliarySplitInfo& aux,
FitnessFunction& fitnessFunction);
/**
* Return the number of children in the split.
* If a split was found, returns the number of children of the split.
* Otherwise if there was no split, returns zero.
*
* @param splitInfo Auxiliary information for the split.
* @param * (aux) Auxiliary information for the split (Unused).
*/
static size_t NumChildren(const double& splitInfo,
static size_t NumChildren(const arma::vec& splitInfo,
const AuxiliarySplitInfo& /* aux */);
/**
* Calculate the direction a point should percolate to.
* If a split was found, given a point, calculates the index of the child
* it should go to. Otherwise if there was no split, returns SIZE_MAX.
*
* @param point the Point to use.
* @param splitInfo Auxiliary information for the split.
@@ -126,7 +128,7 @@ class AllCategoricalSplit
template<typename ElemType>
static size_t CalculateDirection(
const ElemType& point,
const double& splitInfo,
const arma::vec& splitInfo,
const AuxiliarySplitInfo& /* aux */);
};
src/mlpack/methods/decision_tree/all_categorical_split_impl.hpp
+7 -6
View File
@@ -123,7 +123,7 @@ double AllCategoricalSplit<FitnessFunction>::SplitIfBetter(
const WeightVecType& weights,
const size_t minimumLeafSize,
const double minimumGainSplit,
double& splitInfo,
arma::vec& splitInfo,
AuxiliarySplitInfo& /* aux */,
FitnessFunction& fitnessFunction)
{
@@ -199,7 +199,8 @@ double AllCategoricalSplit<FitnessFunction>::SplitIfBetter(
if (overallGain > bestGain + minimumGainSplit + epsilon)
{
// This is better, so store it in splitInfo and return.
splitInfo = numCategories;
splitInfo.set_size(1);
splitInfo[0] = numCategories;
return overallGain;
}
@@ -209,20 +210,20 @@ double AllCategoricalSplit<FitnessFunction>::SplitIfBetter(
template<typename FitnessFunction>
size_t AllCategoricalSplit<FitnessFunction>::NumChildren(
const double& splitInfo,
const arma::vec& splitInfo,
const AuxiliarySplitInfo& /* aux */)
{
return (size_t) splitInfo;
return splitInfo.n_elem == 0 ? 0 : (size_t) splitInfo[0];
}
template<typename FitnessFunction>
template<typename ElemType>
size_t AllCategoricalSplit<FitnessFunction>::CalculateDirection(
const ElemType& point,
const double& /* splitInfo */,
const arma::vec& splitInfo,
const AuxiliarySplitInfo& /* aux */)
{
return (size_t) point;
return splitInfo.n_elem == 0 ? SIZE_MAX : (size_t) point;
}
} // namespace mlpack
src/mlpack/methods/decision_tree/best_binary_categorical_split.hpp
+299
View File
@@ -0,0 +1,299 @@
/**
* @file methods/decision_tree/best_binary_categorical_split.hpp
* @author Nikolay Apanasov (nikolay@apanasov.org)
*
* A tree splitter that finds the best binary categorical split.
*
* mlpack is free software; you may redistribute it and/or modify it under the
* terms of the 3-clause BSD license. You should have received a copy of the
* 3-clause BSD license along with mlpack. If not, see
* http://www.opensource.org/licenses/BSD-3-Clause for more information.
*/
#ifndef MLPACK_METHODS_DECISION_TREE_BEST_BINARY_CATEGORICAL_SPLIT_HPP
#define MLPACK_METHODS_DECISION_TREE_BEST_BINARY_CATEGORICAL_SPLIT_HPP
#define LEFT 0
#define RIGHT 1
#include <mlpack/prereqs.hpp>
using namespace arma;
/**
* The BestBinaryCategoricalSplit is a splitting function for decision trees
* that will exhaustively search a categorical dimension for the best binary
* split of a variable vₖ. This is a generic splitting strategy and can be
* used for both regression and classification.
*
* In the case of binary outcomes, it shown in CART[4.2] by Breiman et al.
* that if we order the categories by the proportion that fall in class C₁,
* and then split vₖ as if it was a numeric type, then the result is optimal.
* Surprising, but true. In the case of multiple classes, there is no such
* simplification. This method will search through all the 2ʲ possible
* partitions (Gₗ, Gᵣ) of the categories C₀, ..., Cⱼ₋₁, every time assigning
* samples with vₖ ∈ Gₗ to left tree Tₗ and those with vₖ ∈ Gᵣ to right
* tree Tᵣ.
*
* Warning: in the classification setting with multiple outcomes, this
* algorithm is exponential in the number of categories. Therefore
* BestBinaryCategoricalSplit should not be chosen when there are multiple
* classes and many categories.
*
* @book{CART,
* author = {Breiman, L. and Friedman, J. and Olshen, R. and Stone, C.},
* year = {1984},
* title = {Classification and Regression Trees},
* publisher = {Chapman \& Hall}
* }
*
* In the regression setting, the algorithm is similar to the preceding linear-
* time split for the case of binary outcomes. The correctness of the algorithm
* for a quantitative response under l₂ loss is due to Fisher.
*
* @article{Fisher58,
* author = {Fisher, W.},
* year = {1958},
* title = {On Grouping for Maximum Homogeniety},
* journal = {Journal of the American Statistical Association},
* volume = {53},
* pages = {789–798}
* }
*
* @tparam FitnessFunction Fitness function to use to calculate gain.
* categorical variable in the case of binary outcomes or regression.
*/
namespace mlpack {
template<typename FitnessFunction>
class BestBinaryCategoricalSplit
{
public:
// No extra info needed for split.
class AuxiliarySplitInfo { };
// Allow access to the numeric split type.
typedef BestBinaryNumericSplit<FitnessFunction> NumericSplit;
// For calls to the numeric splitter.
typedef typename BestBinaryNumericSplit<FitnessFunction>
::AuxiliarySplitInfo NumericAux;
/**
* Check if we can split a node. If we can split a node in a way that
* improves on bestGain, then we return the improved gain. Otherwise we
* return the value DBL_MAX.
*
* This overload is used only for classification.
*
* @param bestGain Best gain seen so far (we'll only split if we find gain
* better than this).
* @param data The dimension of data points to check for a split in.
* @param numCategories Number of categories in the categorical data.
* @param labels Labels for each point.
* @param numClasses Number of classes in the dataset.
* @param weights Weights associated with labels.
* @param minLeafSize min number of points in a leaf node for
* splitting.
* @param minGainSplit min gain split.
* @param splitInfo Stores split information on a succesful split. A
* vector of size J, where J is the number of categories. splitInfo[k]
* is zero if category k is assigned to the left child, and otherwise
* it is one if assigned to the right.
* @param aux (ignored)
*/
template<bool UseWeights, typename VecType, typename LabelsType,
typename WeightVecType>
static double SplitIfBetter(
const double bestGain,
const VecType& data,
const size_t numCategories,
const LabelsType& labels,
const size_t numClasses,
const WeightVecType& weights,
const size_t minLeafSize,
const double minGainSplit,
arma::vec& splitInfo,
AuxiliarySplitInfo& aux);
/**
* Check if we can split a node. If we can split a node in a way that
* improves on bestGain, then we return the improved gain. Otherwise we
* return the value DBL_MAX.
*
* Overload for regression. As mentioned above, the result of Fisher only
* applies under l₂ loss, and thus this overload is used only for regression
* with MSEGain.
*
* @param bestGain Best gain seen so far (we'll only split if we find gain
* better than this).
* @param data The dimension of data points to check for a split in.
* @param numCategories Number of categories in the categorical data.
* @param responses Responses for each point.
* @param weights Weights associated with responses.
* @param minLeafSize min number of points in a leaf node for
* splitting.
* @param minGainSplit min gain split.
* @param splitInfo Stores split information on a successful split.
*
* @param splitInfo Stores split information on a succesful split. A
* vector of size J, where J is the number of categories. splitInfo[k]
* is zero if category k is assigned to the left child, and otherwise
* it is one if assigned to the right.
* @param aux (ignored)
* @param fitnessFunction The FitnessFunction object instance. It it used to
* evaluate the gain for the split.
*/
template<bool UseWeights, typename VecType, typename ResponsesType,
typename WeightVecType>
static double SplitIfBetter(
const double bestGain,
const VecType& data,
const size_t numCategories,
const ResponsesType& responses,
const WeightVecType& weights,
const size_t minLeafSize,
const double minGainSplit,
arma::vec& splitInfo,
AuxiliarySplitInfo& aux,
FitnessFunction& fitnessFunction);
/**
* In the case that a split was found, returns the number of children
* of the split. Otherwise if there was not split, returns zero. A binary
* split always has two children.
*
* @param splitInfo Auxiliary information for the split. A vector
* of size J, where J is the number of categories. splitInfo[k]
* is zero if category k is assigned to the left child, and otherwise
* it is one if assigned to the right.
* @param * (aux) Auxiliary information for the split (Unused).
*/
static size_t NumChildren(const arma::vec& splitInfo,
const AuxiliarySplitInfo& /* aux */)
{
return splitInfo.n_elem == 0 ? 0 : 2;
}
/**
*
* In the case that a split was found, given a point, calculates
* the index of the child it should go to. Otherwise if there was
* no split, returns SIZE_MAX.
*
* @param point the Point to use.
* @param splitInfo Auxiliary information for the split. A vector
* of size J, where J is the number of categories. splitInfo[k] is
* zero if category Cₖ is assigned to the left child, and otherwise
* it is one if Cₖ is assigned to the right.
* @param * (aux) Auxiliary information for the split (Unused).
*/
template<typename ElemType>
static size_t CalculateDirection(
const ElemType& point,
const arma::vec& splitInfo,
const AuxiliarySplitInfo& /* aux */)
{
return splitInfo.n_elem == 0 ? SIZE_MAX : (size_t) splitInfo[point];
}
private:
/**
* Auxiliary for SplitIfBetter in the multi-class setting. Recursively
* enumerates all partitions (Gₗ, Gᵣ) of categories C₀, ..., Cⱼ₋₁, and
* computes the gain for each one, where samples with vₖ ∈ Gₗ are assigned
* to the left tree Tₗ and those with vₖ ∈ Gᵣ to the right tree Tᵣ.
*
* In the case that a better split is found, bestFoundGain is updated with
* the gain value and splitInfo is updated with the corresponding partition.
*
* @param labels -- Labels for each point.
* @param numClasses -- Number of classes in the dataset.
* @param numCategories Number of categories in the categorical data.
* @param bestFoundGain -- The best gain found thus far. Updated if
* and when a better split is found.
* @param categorySamples -- Map from category Cⱼ to the samples whose
* categorical value for variable vₖ is Cⱼ. Column j is for Cⱼ.
* @param categories -- J dimensional vector used to maintain the
* current partition of the categories.
* @param splitInfo -- Stores split information on a succesful split. A
* vector of size J, where J is the number of categories. splitInfo[k]
* is zero if category k is assigned to the left child, and otherwise
* it is one if assigned to the right.
* @param classCounts -- mx2 matrix, where m = numClasses, used to compute
* the gain with the FitnessFunction. All zero initially.
* @param totalLeft, totalRight -- Number of samples assigned
* to the left and right subtrees respectively. Initialized to zero.
* @param k -- Index of the current category being assigned.
* Initialized value is zero.
*/
template<typename VecType, typename LabelsType>
static bool PartitionSplit(
const VecType& data,
const LabelsType& labels,
const size_t numCategories,
const size_t numClasses,
double& bestFoundGain,
arma::SpMat<short>& categorySamples,
arma::uvec& categories,
arma::vec& splitInfo,
arma::Mat<size_t>& classCounts,
size_t totalLeft = 0,
size_t totalRight = 0,
size_t k = 0);
/**
* Auxiliary for SplitIfBetter in the multi-class setting. Recursively
* enumerates all partitions (Gₗ, Gᵣ) of categories C₀, ..., Cⱼ₋₁, and
* computes the gain for each one, where samples with vₖ ∈ Gₗ are assigned
* to the left tree Tₗ and those with vₖ ∈ Gᵣ to the right tree Tᵣ.
*
* In the case that a better split is found, bestFoundGain is updated with
* the gain value and splitInfo is updated with the corresponding partition.
*
* This overload is used to compute the partition using weights, that is
* when the template variable UseWeights is true.
*
* @param labels -- Labels for each point.
* @param numCategories Number of categories in the categorical data.
* @param numClasses -- Number of classes in the dataset.
* @param weights -- Weights associated with labels.
* @param totalWeight -- Sum of weights.
* @param bestFoundGain -- The best gain found thus far. Updated if
* and when a better split is found.
* @param categorySamples -- Map from category Cⱼ to the samples whose
* categorical value for variable vₖ is Cⱼ. Column j is for Cⱼ.
* @param categories -- J dimensional vector used to maintain the
* current partition of the categories.
* @param splitInfo -- Stores split information on a succesful split. A
* vector of size J, where J is the number of categories. splitInfo[k]
* is zero if category k is assigned to the left child, and otherwise
* it is one if assigned to the right.
* @param classWeightSums -- mx2 matrix, where m = numClasses, used to
* compute the gain with the FitnessFunction. All zero initially.
* @param totalLeftWeight, totalRightWeight -- Weight assigned to the left
* and right subtree respectively. Initialized to zero.
* @param k -- Index of the current category being assigned.
* Initialized value is zero.
*/
template<typename VecType, typename LabelsType, typename WeightVecType>
static bool PartitionSplit(
const VecType& data,
const LabelsType& labels,
const size_t numCategories,
const size_t numClasses,
const WeightVecType& weights,
const double totalWeight,
double& bestFoundGain,
arma::SpMat<short>& categorySamples,
arma::uvec& categories,
arma::vec& splitInfo,
arma::mat& classWeightSums,
double totalLeftWeight = 0.0,
double totalRightWeight = 0.0,
size_t k = 0);
};
} // namespace mlpack
// Include implementation.
#include "best_binary_categorical_split_impl.hpp"
#endif
src/mlpack/methods/decision_tree/best_binary_categorical_split_impl.hpp
+414
View File
@@ -0,0 +1,414 @@
/**
* @file methods/decision_tree/all_categorical_split_impl.hpp
* @author Nikolay Apanasov (nikolay@apanasov.org)
*
* Implementation of the BestBinaryCategoricalSplit categorical split class.
*
* mlpack is free software; you may redistribute it and/or modify it under the
* terms of the 3-clause BSD license. You should have received a copy of the
* 3-clause BSD license along with mlpack. If not, see
* http://www.opensource.org/licenses/BSD-3-Clause for more information.
*/
#ifndef MLPACK_METHODS_DECISION_TREE_BEST_BINARY_CATEGORICAL_SPLIT_IMPL_HPP
#define MLPACK_METHODS_DECISION_TREE_BEST_BINARY_CATEGORICAL_SPLIT_IMPL_HPP
namespace mlpack {
// Overload used in classification.
template<typename FitnessFunction>
template<bool UseWeights, typename VecType, typename LabelsType,
typename WeightVecType>
double BestBinaryCategoricalSplit<FitnessFunction>::SplitIfBetter(
const double bestGain,
const VecType& data,
const size_t numCategories,
const LabelsType& labels,
const size_t numClasses,
const WeightVecType& weights,
const size_t minLeafSize,
const double minGainSplit,
vec& splitInfo,
AuxiliarySplitInfo& aux)
{
const size_t n = data.n_elem;
double bestFoundGain = std::min(bestGain + minGainSplit, 0.0);
bool improved = false;
// We are too small to split.
if (n < (minLeafSize * 2))
return DBL_MAX;
// Binary classification
if (numClasses == 2) {
// Order the categories of variable vₖ by their proportion in class C₁
// and map each categorical vₖ to its categorical rank
umat categoryCounts(numCategories, 2, fill::zeros);
vec categoryP(numCategories);
size_t totalCount;
for (size_t i = 0; i < n; ++i)
++categoryCounts(data[i], labels[i]);
for (size_t i = 0; i < numCategories; ++i)
{
totalCount = (categoryCounts(i, 0) + categoryCounts(i, 1));
categoryP[i] = totalCount == 0 ? 0 : (categoryCounts(i, 1)
/(categoryCounts(i, 0) + categoryCounts(i, 1)));
}
uvec sortedCategories = sort_index(categoryP);
uvec categoryRank(numCategories);
for (size_t i = 0; i < numCategories; i++)
categoryRank[sortedCategories[i]] = i;
uvec transformedData(n);
for (size_t i = 0; i < n; ++i)
transformedData[i] = categoryRank[data[i]];
// Split the transformed vₖ as a numeric type.
bestFoundGain = NumericSplit::template SplitIfBetter<UseWeights>(
bestFoundGain,
transformedData,
labels,
numClasses,
weights,
minLeafSize,
minGainSplit,
splitInfo,
(NumericAux &) aux
);
improved = bestFoundGain != DBL_MAX;
// This split is better: store the set membership in splitInfo
// and return. Thus splitInfo is a vector of size Q, where Q is
// the number of categories, and splitInfo[k] is zero if category
// k is assigned to the left child, and otherwise it is one if k
// is assigned to the right.
if (improved) {
const size_t splitIndex = (size_t) std::floor((double) splitInfo[0]);
splitInfo.set_size(numCategories);
for (size_t c: sortedCategories.subvec(0, splitIndex))
splitInfo[c] = LEFT;
for (size_t c: sortedCategories.subvec(splitIndex+1, numCategories-1))
splitInfo[c] = RIGHT;
}
}
// Multi-class classification -- Brute force search through all the
// 2ʲ possible partitions (Gₗ, Gᵣ) of the categories C₀, ..., Cⱼ,
// assigning samples with vₖ ∈ Gₗ to left tree Tₗ and those with vₖ ∈ Gᵣ
// to right tree Tᵣ.
else
{
// A map from category Cⱼ to the samples whose categorical value
// for variable vₖ is Cⱼ. The jth column corresponds to Cⱼ.
SpMat<short> categorySamples(n, numCategories);
for (size_t i = 0; i < n; ++i)
categorySamples(i, data[i]) = 1;
uvec categories(numCategories);
if (UseWeights)
{
// Weight counts for computing the gain.
double totalWeight = accu(weights);
mat classWeightSums = zeros<mat>(numClasses, 2);
bestFoundGain *= totalWeight;
// Recursively check the gain for all partitions.
improved = PartitionSplit(
data, labels, numCategories, numClasses, weights,
totalWeight, bestFoundGain, categorySamples, categories,
splitInfo, classWeightSums
);
bestFoundGain /= totalWeight;
}
else
{
// Class counts for computing the gain.
Mat<size_t> classCounts = zeros<Mat<size_t>>(numClasses, 2);
bestFoundGain *= n;
// Recursively check the gain for all partitions.
improved = PartitionSplit(
data, labels, numCategories, numClasses, bestFoundGain,
categorySamples, categories, splitInfo, classCounts
);
bestFoundGain /= n;
}
}
return improved? bestFoundGain : DBL_MAX;
}
// Overload used in regression with MSEGain.
template<typename FitnessFunction>
template<bool UseWeights, typename VecType, typename ResponsesType,
typename WeightVecType>
double BestBinaryCategoricalSplit<FitnessFunction>::SplitIfBetter(
const double bestGain,
const VecType& data,
const size_t numCategories,
const ResponsesType& responses,
const WeightVecType& weights,
const size_t minLeafSize,
const double minGainSplit,
vec& splitInfo,
AuxiliarySplitInfo& aux,
FitnessFunction& fitnessFunction)
{
static_assert(std::is_same<FitnessFunction, MSEGain>::value,
"BestBinaryCategoricalSplit: regression FitnessFunction must be MSEGain."
);
const size_t n = data.n_elem;
double bestFoundGain = std::min(bestGain + minGainSplit, 0.0);
// We are too small to split.
if (n < (minLeafSize * 2))
return DBL_MAX;
// Order the categories of variable vₖ by increasing mean
// of the response y. categoryResponse[i, 0] will contain
// the mean response for category Cᵢ.
vec categoryResponse(numCategories, fill::zeros);
uvec categoryCounts(numCategories, fill::zeros);
for (size_t i = 0; i < n; ++i)
{
categoryResponse[data[i]] += responses[i];
++categoryCounts[data[i]];
}
for (size_t i = 0; i < numCategories; ++i)
categoryResponse[i] = categoryCounts[i] == 0 ? 0 :
categoryResponse[i]/categoryCounts[i];
uvec sortedCategories = sort_index(categoryResponse);
uvec categoryRank(numCategories);
for (size_t i = 0; i < numCategories; i++)
categoryRank[sortedCategories[i]] = i;
uvec transformedData(n);
for (size_t i = 0; i < n; ++i)
transformedData[i] = categoryRank[data[i]];
// Split the transformed vₖ as a numeric type.
bestFoundGain = NumericSplit::template SplitIfBetter<UseWeights>(
bestFoundGain,
transformedData,
responses,
weights,
minLeafSize,
minGainSplit,
splitInfo,
(NumericAux &) aux,
fitnessFunction
);
bool improved = bestFoundGain != DBL_MAX;
if (improved)
{
// This split is better: store the set membership in splitInfo
// and return. Thus splitInfo is a vector of size Q, where Q is
// the number of categories, and splitInfo[k] is zero if category
// k is assigned to the left child, and otherwise it is one if k
// is assigned to the right.
size_t splitIndex = (size_t) std::floor((double) splitInfo[0]);
splitInfo.set_size(numCategories + 1);
for (size_t c: sortedCategories.subvec(0, splitIndex))
splitInfo[c] = LEFT;
for (size_t c: sortedCategories.subvec(splitIndex+1, numCategories-1))
splitInfo[c] = RIGHT;
return bestFoundGain;
}
return DBL_MAX;
}
template<typename FitnessFunction>
template<typename VecType, typename LabelsType>
bool BestBinaryCategoricalSplit<FitnessFunction>::PartitionSplit(
const VecType& data,
const LabelsType& labels,
const size_t numCategories,
const size_t numClasses,
double& bestFoundGain,
SpMat<short>& categorySamples,
uvec& categories,
vec& splitInfo,
Mat<size_t>& classCounts,
size_t totalLeft,
size_t totalRight,
size_t k)
{
/* Base case -- We have already found an optimal split. */
if (bestFoundGain >= 0.0)
return false;
if (k == numCategories)
{
/**
* Base case -- Compute the gain for the current partition.
*/
double leftGain, rightGain, gain;
// The gain for children Tₗ and Tᵣ.
leftGain = FitnessFunction::template EvaluatePtr<false>(
classCounts.colptr(LEFT), numClasses, totalLeft
);
rightGain = FitnessFunction::template EvaluatePtr<false>(
classCounts.colptr(RIGHT), numClasses, totalRight
);
// The gain for this split.
gain = double(totalLeft)*leftGain + double(totalRight)*rightGain;
// This is the best split found thus far.
if (gain > bestFoundGain)
{
bestFoundGain = gain;
splitInfo.set_size(numCategories);
for (size_t i = 0; i < numCategories; ++i)
splitInfo[i] = categories[i];
return true;
}
return false;
}
bool improved = false;
uvec samples = find(categorySamples.col(k));
/**
* Compute the gain with category Cₖ ∈ Tₗ
*/
categories[k] = LEFT;
for (auto i : samples)
{
++classCounts(labels[i], LEFT);
}
totalLeft += samples.n_elem;
improved = PartitionSplit(
data, labels, numCategories, numClasses, bestFoundGain, categorySamples,
categories, splitInfo, classCounts, totalLeft, totalRight, k+1
);
/**
* Compute the gain with category Cₖ ∈ Tᵣ
*/
categories[k] = RIGHT;
for (auto i : samples)
{
--classCounts(labels[i], LEFT);
++classCounts(labels[i], RIGHT);
}
totalLeft -= samples.n_elem;
totalRight += samples.n_elem;
improved |= PartitionSplit(
data, labels, numCategories, numClasses, bestFoundGain, categorySamples,
categories, splitInfo, classCounts, totalLeft, totalRight, k+1
);
/* Unassign Cₖ and return whether a better split was found. */
for (auto i : samples)
{
--classCounts(labels[i], RIGHT);
}
totalRight -= samples.n_elem;
return improved;
}
// Overload used with weights.
template<typename FitnessFunction>
template<typename VecType, typename LabelsType, typename WeightVecType>
bool BestBinaryCategoricalSplit<FitnessFunction>::PartitionSplit(
const VecType& data,
const LabelsType& labels,
const size_t numCategories,
const size_t numClasses,
const WeightVecType& weights,
const double totalWeight,
double& bestFoundGain,
SpMat<short>& categorySamples,
uvec& categories,
vec& splitInfo,
mat& classWeightSums,
double totalLeftWeight,
double totalRightWeight,
size_t k)
{
/* Base case -- We have already found an optimal split. */
if (bestFoundGain >= 0.0)
return false;
/**
* Base case -- Compute the gain for the current partition.
*/
if (k == numCategories)
{
double leftGain, rightGain, gain;
// The gain for children Tₗ and Tᵣ.
leftGain = FitnessFunction::template EvaluatePtr<true>(
classWeightSums.colptr(LEFT), numClasses, totalLeftWeight
);
rightGain = FitnessFunction::template EvaluatePtr<true>(
classWeightSums.colptr(RIGHT), numClasses, totalRightWeight
);
// The gain for this split.
gain = (totalLeftWeight * leftGain) + (totalRightWeight * rightGain);
// This is the best split found thus far.
if (gain > bestFoundGain)
{
bestFoundGain = gain;
splitInfo.set_size(numCategories);
for (size_t i = 0; i < numCategories; ++i)
splitInfo[i] = categories[i];
return true;
}
return false;
}
bool improved= false;
uvec samples = find(categorySamples.col(k));
/**
* Compute the gain with category Cₖ ∈ Tₗ
*/
categories[k] = LEFT;
for (auto i : samples)
{
classWeightSums(labels[i], LEFT) += weights[i];
totalLeftWeight += weights[i];
}
improved = PartitionSplit(
data, labels, numCategories, numClasses, weights, totalWeight,
bestFoundGain, categorySamples, categories, splitInfo,
classWeightSums, totalLeftWeight, totalRightWeight, k+1
);
/**
* Compute the gain with category Cₖ ∈ Tᵣ
*/
categories[k] = RIGHT;
for (auto i : samples)
{
classWeightSums(labels[i], LEFT) -= weights[i];
classWeightSums(labels[i], RIGHT) += weights[i];
totalLeftWeight -= weights[i];
totalRightWeight += weights[i];
}
improved |= PartitionSplit(
data, labels, numCategories, numClasses, weights, totalWeight,
bestFoundGain, categorySamples, categories, splitInfo,
classWeightSums, totalLeftWeight, totalRightWeight, k+1
);
/* Unassign Cₖ and return whether a better split was found. */
for (auto i : samples)
{
classWeightSums(labels[i], RIGHT) -= weights[i];
totalRightWeight -= weights[i];
}
return improved;
}
} // namespace mlpack
#endif
src/mlpack/methods/decision_tree/best_binary_numeric_split.hpp
+10 -7
View File
@@ -118,7 +118,7 @@ class BestBinaryNumericSplit
const WeightVecType& weights,
const size_t minimumLeafSize,
const double minimumGainSplit,
double& splitInfo,
arma::vec& splitInfo,
AuxiliarySplitInfo& aux,
FitnessFunction& fitnessFunction);
@@ -155,21 +155,24 @@ class BestBinaryNumericSplit
const WeightVecType& weights,
const size_t minimumLeafSize,
const double minimumGainSplit,
double& splitInfo,
arma::vec& splitInfo,
AuxiliarySplitInfo& /* aux */,
FitnessFunction& fitnessFunction);
/**
* Returns 2, since the binary split always has two children.
* If a split was found, returns the number of children of the split.
* Otherwise returns zero. A binary split always has two children.
*/
static size_t NumChildren(const double& /* splitInfo */,
static size_t NumChildren(const arma::vec& splitInfo,
const AuxiliarySplitInfo& /* aux */)
{
return 2;
return splitInfo.n_elem == 0 ? 0 : 2;
}
/**
* Given a point, calculate which child it should go to (left or right).
* In the case that a split was found, given a point, calculate
* which child it should go to (left or right). Otherwise if
* there was no split, returns SIZE_MAX.
*
* @param point Point to calculate direction of.
* @param splitInfo Auxiliary information for the split.
@@ -178,7 +181,7 @@ class BestBinaryNumericSplit
template<typename ElemType>
static size_t CalculateDirection(
const ElemType& point,
const double& splitInfo,
const arma::vec& splitInfo,
const AuxiliarySplitInfo& /* aux */);
};
src/mlpack/methods/decision_tree/best_binary_numeric_split_impl.hpp
+16 -13
View File
@@ -118,11 +118,9 @@ double BestBinaryNumericSplit<FitnessFunction>::SplitIfBetter(
--classCounts(sortedLabels[index - 1], 1);
++classCounts(sortedLabels[index - 1], 0);
}
// Make sure that the value has changed.
if (data[sortedIndices[index]] == data[sortedIndices[index - 1]])
if (data[sortedIndices[index - 1]] == data[sortedIndices[index]])
continue;
// Calculate the gain for the left and right child. Only use weights if
// needed.
const double leftGain = UseWeights ?
@@ -218,7 +216,7 @@ BestBinaryNumericSplit<FitnessFunction>::SplitIfBetter(
const WeightVecType& weights,
const size_t minimumLeafSize,
const double minimumGainSplit,
double& splitInfo,
arma::vec& splitInfo,
AuxiliarySplitInfo& /* aux */,
FitnessFunction& fitnessFunction)
{
@@ -314,7 +312,8 @@ BestBinaryNumericSplit<FitnessFunction>::SplitIfBetter(
// We can take a shortcut: no split will be better than this, so just
// take this one. The actual split value will be halfway between the
// value at index - 1 and index.
splitInfo = (data[sortedIndices[index - 1]] +
splitInfo.set_size(1);
splitInfo[0] = (data[sortedIndices[index - 1]] +
data[sortedIndices[index]]) / 2.0;
// In some very extreme cases, floating-point inaccuracies can lead to the
@@ -323,14 +322,14 @@ BestBinaryNumericSplit<FitnessFunction>::SplitIfBetter(
// bump it down incrementally.
if (splitInfo == data[sortedIndices[index]])
splitInfo = std::nexttoward(splitInfo, data[sortedIndices[index - 1]]);
return gain;
}
if (gain > bestFoundGain)
{
// We still have a better split.
bestFoundGain = gain;
splitInfo = (data[sortedIndices[index - 1]] +
splitInfo.set_size(1);
splitInfo[0] = (data[sortedIndices[index - 1]] +
data[sortedIndices[index]]) / 2.0;
improved = true;
@@ -371,7 +370,7 @@ BestBinaryNumericSplit<FitnessFunction>::SplitIfBetter(
const WeightVecType& weights,
const size_t minimumLeafSize,
const double minimumGainSplit,
double& splitInfo,
arma::vec& splitInfo,
AuxiliarySplitInfo& /* aux */,
FitnessFunction& fitnessFunction)
{
@@ -475,7 +474,8 @@ BestBinaryNumericSplit<FitnessFunction>::SplitIfBetter(
// We can take a shortcut: no split will be better than this, so just
// take this one. The actual split value will be halfway between the
// value at index - 1 and index.
splitInfo = (data[sortedIndices[index - 1]] +
splitInfo.set_size(1);
splitInfo[0] = (data[sortedIndices[index - 1]] +
data[sortedIndices[index]]) / 2.0;
// In some very extreme cases, floating-point inaccuracies can lead to the
@@ -491,7 +491,8 @@ BestBinaryNumericSplit<FitnessFunction>::SplitIfBetter(
{
// We still have a better split.
bestFoundGain = gain;
splitInfo = (data[sortedIndices[index - 1]] +
splitInfo.set_size(1);
splitInfo[0] = (data[sortedIndices[index - 1]] +
data[sortedIndices[index]]) / 2.0;
improved = true;
@@ -520,10 +521,12 @@ template<typename FitnessFunction>
template<typename ElemType>
size_t BestBinaryNumericSplit<FitnessFunction>::CalculateDirection(
const ElemType& point,
const double& splitInfo,
const arma::vec& splitInfo,
const AuxiliarySplitInfo& /* aux */)
{
if (point <= splitInfo)
if (splitInfo.n_elem == 0)
return SIZE_MAX;
else if (point <= splitInfo[0])
return 0; // Go left.
else
return 1; // Go right.
src/mlpack/methods/decision_tree/decision_tree.hpp
+1
View File
@@ -23,6 +23,7 @@
#include "best_binary_numeric_split.hpp"
#include "random_binary_numeric_split.hpp"
#include "best_binary_categorical_split.hpp"
#include "all_categorical_split.hpp"
#include "all_dimension_select.hpp"
src/mlpack/methods/decision_tree/decision_tree_impl.hpp
+9 -8
View File
@@ -270,6 +270,7 @@ DecisionTree<FitnessFunction,
dimensionType(0),
classProbabilities(numClasses)
{
// TODO @dfs
// Initialize utility vector.
classProbabilities.fill(1.0 / (double) numClasses);
}
@@ -705,9 +706,9 @@ double DecisionTree<FitnessFunction,
// Get the number of children we will have.
size_t numChildren = 0;
if (datasetInfo.Type(bestDim) == data::Datatype::categorical)
numChildren = CategoricalSplit::NumChildren(classProbabilities[0], *this);
numChildren = CategoricalSplit::NumChildren(classProbabilities, *this);
else
numChildren = NumericSplit::NumChildren(classProbabilities[0], *this);
numChildren = NumericSplit::NumChildren(classProbabilities, *this);
// Calculate all child assignments.
arma::Row<size_t> childAssignments(count);
@@ -715,14 +716,14 @@ double DecisionTree<FitnessFunction,
{
for (size_t j = begin; j < begin + count; ++j)
childAssignments[j - begin] = CategoricalSplit::CalculateDirection(
data(bestDim, j), classProbabilities[0], *this);
data(bestDim, j), classProbabilities, *this);
}
else
{
for (size_t j = begin; j < begin + count; ++j)
{
childAssignments[j - begin] = NumericSplit::CalculateDirection(
data(bestDim, j), classProbabilities[0], *this);
data(bestDim, j), classProbabilities, *this);
}
}
@@ -871,7 +872,7 @@ double DecisionTree<FitnessFunction,
{
// We know that the split is numeric.
size_t numChildren =
NumericSplit::NumChildren(classProbabilities[0], *this);
NumericSplit::NumChildren(classProbabilities, *this);
splitDimension = bestDim;
dimensionType = (size_t) data::Datatype::numeric;
@@ -881,7 +882,7 @@ double DecisionTree<FitnessFunction,
for (size_t j = begin; j < begin + count; ++j)
{
childAssignments[j - begin] = NumericSplit::CalculateDirection(
data(bestDim, j), classProbabilities[0], *this);
data(bestDim, j), classProbabilities, *this);
}
// Calculate counts of children in each node.
@@ -1105,10 +1106,10 @@ size_t DecisionTree<FitnessFunction,
{
if ((data::Datatype) dimensionType == data::Datatype::categorical)
return CategoricalSplit::CalculateDirection(point[splitDimension],
classProbabilities[0], *this);
classProbabilities, *this);
else
return NumericSplit::CalculateDirection(point[splitDimension],
classProbabilities[0], *this);
classProbabilities, *this);
}
// Get the number of classes in the tree.
src/mlpack/methods/decision_tree/decision_tree_regressor.hpp
+7 -9
View File
@@ -443,19 +443,17 @@ class DecisionTreeRegressor :
private:
//! The vector of children.
std::vector<DecisionTreeRegressor*> children;
//! The dimension this node splits on.
size_t splitDimension;
//! The type of the dimension that we have split on (only meaningful if this
//! is a non-leaf in a trained tree).
size_t dimensionType;
union
{
//! Stores the split point for internal nodes of the tree.
double splitPoint;
//! Stores the prediction value for leaf nodes of the tree.
//! Stores the prediction value, for leaf nodes of the tree.
double prediction;
//! The dimension of the split, for internal nodes.
size_t splitDimension;
};
//! For internal nodes, the type of the split variable.
size_t dimensionType;
//! For internal nodes, the split information for the splitter.
arma::vec splitInfo;
//! Note that this class will also hold the members of the NumericSplit and
//! CategoricalSplit AuxiliarySplitInfo classes, since it inherits from them.
src/mlpack/methods/decision_tree/decision_tree_regressor_impl.hpp
+33 -42
View File
@@ -28,9 +28,9 @@ DecisionTreeRegressor<FitnessFunction,
CategoricalSplitType,
DimensionSelectionType,
NoRecursion>::DecisionTreeRegressor() :
splitDimension(0),
prediction(0),
dimensionType(0),
splitPoint(0.0)
splitInfo()
{
// Nothing to do here.
}
@@ -53,7 +53,7 @@ DecisionTreeRegressor<FitnessFunction,
const size_t minimumLeafSize,
const double minimumGainSplit,
const size_t maximumDepth,
DimensionSelectionType dimensionSelector)
DimensionSelectionType dimensionSelector) : splitInfo()
{
using TrueMatType = typename std::decay<MatType>::type;
using TrueResponsesType = typename std::decay<ResponsesType>::type;
@@ -89,7 +89,7 @@ DecisionTreeRegressor<FitnessFunction,
const size_t minimumLeafSize,
const double minimumGainSplit,
const size_t maximumDepth,
DimensionSelectionType dimensionSelector)
DimensionSelectionType dimensionSelector) : splitInfo()
{
using TrueMatType = typename std::decay<MatType>::type;
using TrueResponsesType = typename std::decay<ResponsesType>::type;
@@ -129,6 +129,7 @@ DecisionTreeRegressor<FitnessFunction,
DimensionSelectionType dimensionSelector,
const std::enable_if_t<arma::is_arma_type<
typename std::remove_reference<WeightsType>::type>::value>*)
: splitInfo()
{
using TrueMatType = typename std::decay<MatType>::type;
using TrueResponsesType = typename std::decay<ResponsesType>::type;
@@ -169,7 +170,7 @@ DecisionTreeRegressor<FitnessFunction,
const std::enable_if_t<
arma::is_arma_type<
typename std::remove_reference<
WeightsType>::type>::value>*)
WeightsType>::type>::value>*) : splitInfo()
{
using TrueMatType = typename std::decay<MatType>::type;
using TrueResponsesType = typename std::decay<ResponsesType>::type;
@@ -210,7 +211,8 @@ DecisionTreeRegressor<FitnessFunction,
const std::enable_if_t<arma::is_arma_type<
typename std::remove_reference<WeightsType>::type>::value>*):
NumericAuxiliarySplitInfo(other),
CategoricalAuxiliarySplitInfo(other)
CategoricalAuxiliarySplitInfo(other),
splitInfo(std::move(other.splitInfo))
{
using TrueMatType = typename std::decay<MatType>::type;
using TrueResponsesType = typename std::decay<ResponsesType>::type;
@@ -250,7 +252,8 @@ DecisionTreeRegressor<FitnessFunction,
typename std::remove_reference<
WeightsType>::type>::value>*):
NumericAuxiliarySplitInfo(other),
CategoricalAuxiliarySplitInfo(other) // other info does need to copy
CategoricalAuxiliarySplitInfo(other), // other info does need to copy
splitInfo(std::move(other.splitInfo))
{
using TrueMatType = typename std::decay<MatType>::type;
using TrueResponsesType = typename std::decay<ResponsesType>::type;
@@ -284,17 +287,12 @@ DecisionTreeRegressor<FitnessFunction,
const DecisionTreeRegressor& other) :
NumericAuxiliarySplitInfo(other),
CategoricalAuxiliarySplitInfo(other),
splitDimension(other.splitDimension),
prediction(other.prediction),
dimensionType(other.dimensionType)
{
// Copy each child.
for (size_t i = 0; i < other.children.size(); ++i)
children.push_back(new DecisionTreeRegressor(*other.children[i]));
if (children.size() != 0)
splitPoint = other.splitPoint;
else
prediction = other.prediction;
}
//! Take ownership of another tree.
@@ -313,13 +311,10 @@ DecisionTreeRegressor<FitnessFunction,
NumericAuxiliarySplitInfo(std::move(other)),
CategoricalAuxiliarySplitInfo(std::move(other)),
children(std::move(other.children)),
splitDimension(other.splitDimension),
dimensionType(other.dimensionType)
prediction(other.prediction),
dimensionType(other.dimensionType),
splitInfo(other.splitInfo)
{
if (children.size() != 0)
splitPoint = other.splitPoint;
else
prediction = other.prediction;
}
//! Copy another tree.
@@ -353,7 +348,7 @@ DecisionTreeRegressor<FitnessFunction,
dimensionType = other.dimensionType;
if (other.children.size() != 0)
splitPoint = other.splitPoint;
splitInfo = other.splitInfo;
else
prediction = other.prediction;
@@ -400,7 +395,7 @@ DecisionTreeRegressor<FitnessFunction,
dimensionType = other.dimensionType;
if (children.size() != 0)
splitPoint = other.splitPoint;
splitInfo = other.splitInfo;
else
prediction = other.prediction;
@@ -632,9 +627,7 @@ double DecisionTreeRegressor<FitnessFunction,
// Look through the list of dimensions and obtain the gain of the best split.
// We'll cache the best numeric and categorical split auxiliary information
// in numericAux and categoricalAux (and clear them later if we make no
// split). The split point is stored in splitPointOrPrediction for all
// internal nodes of the tree.
// in splitInfo, which is non-empty only for internal nodes of the tree.
double bestGain = fitnessFunction.template Evaluate<UseWeights>(
responses.cols(begin, begin + count - 1),
UseWeights ? weights.subvec(begin, begin + count - 1) : weights);
@@ -656,7 +649,7 @@ double DecisionTreeRegressor<FitnessFunction,
UseWeights ? weights.subvec(begin, begin + count - 1) : weights,
minimumLeafSize,
minimumGainSplit,
splitPoint,
splitInfo,
*this,
fitnessFunction);
}
@@ -668,7 +661,7 @@ double DecisionTreeRegressor<FitnessFunction,
UseWeights ? weights.subvec(begin, begin + count - 1) : weights,
minimumLeafSize,
minimumGainSplit,
splitPoint,
splitInfo,
*this,
fitnessFunction);
}
@@ -697,9 +690,9 @@ double DecisionTreeRegressor<FitnessFunction,
// Get the number of children we will have.
size_t numChildren = 0;
if (datasetInfo.Type(bestDim) == data::Datatype::categorical)
numChildren = CategoricalSplit::NumChildren(splitPoint, *this);
numChildren = CategoricalSplit::NumChildren(splitInfo, *this);
else
numChildren = NumericSplit::NumChildren(splitPoint, *this);
numChildren = NumericSplit::NumChildren(splitInfo, *this);
// Calculate all child assignments.
arma::Row<size_t> childAssignments(count);
@@ -707,14 +700,14 @@ double DecisionTreeRegressor<FitnessFunction,
{
for (size_t j = begin; j < begin + count; ++j)
childAssignments[j - begin] = CategoricalSplit::CalculateDirection(
data(bestDim, j), splitPoint, *this);
data(bestDim, j), splitInfo, *this);
}
else
{
for (size_t j = begin; j < begin + count; ++j)
{
childAssignments[j - begin] = NumericSplit::CalculateDirection(
data(bestDim, j), splitPoint, *this);
data(bestDim, j), splitInfo, *this);
}
}
@@ -814,9 +807,9 @@ double DecisionTreeRegressor<FitnessFunction,
// We won't be using these members, so reset them.
CategoricalAuxiliarySplitInfo::operator=(CategoricalAuxiliarySplitInfo());
// Look through the list of dimensions and obtain the best split. We'll cache
// the best numeric split auxiliary information in numericAux (and clear it
// later if we don't make a split). The split point is stored in
// Look through the list of dimensions and obtain the best split. We'll
// cache the best numeric and categorical split auxiliary information
// in splitInfo, which is non-empty only for internal nodes of the tree.
// splitPointOrPrediction for all internal nodes of the tree.
double bestGain = fitnessFunction.template Evaluate<UseWeights>(
responses.cols(begin, begin + count - 1),
@@ -837,7 +830,7 @@ double DecisionTreeRegressor<FitnessFunction,
weights,
minimumLeafSize,
minimumGainSplit,
splitPoint,
splitInfo,
*this,
fitnessFunction);
@@ -859,7 +852,7 @@ double DecisionTreeRegressor<FitnessFunction,
if (bestDim != data.n_rows)
{
// We know that the split is numeric.
size_t numChildren = NumericSplit::NumChildren(splitPoint, *this);
size_t numChildren = NumericSplit::NumChildren(splitInfo, *this);
splitDimension = bestDim;
dimensionType = (size_t) data::Datatype::numeric;
@@ -869,7 +862,7 @@ double DecisionTreeRegressor<FitnessFunction,
for (size_t j = begin; j < begin + count; ++j)
{
childAssignments[j - begin] = NumericSplit::CalculateDirection(
data(bestDim, j), splitPoint, *this);
data(bestDim, j), splitInfo, *this);
}
// Calculate counts of children in each node.
@@ -1005,10 +998,10 @@ size_t DecisionTreeRegressor<FitnessFunction,
{
if ((data::Datatype) dimensionType == data::Datatype::categorical)
return CategoricalSplit::CalculateDirection(point[splitDimension],
splitPoint, *this);
splitInfo, *this);
else
return NumericSplit::CalculateDirection(point[splitDimension],
splitPoint, *this);
splitInfo, *this);
}
//! Serialize the tree.
@@ -1036,11 +1029,9 @@ void DecisionTreeRegressor<FitnessFunction,
ar(CEREAL_VECTOR_POINTER(children));
// Now serialize the rest of the object.
ar(CEREAL_NVP(splitDimension));
ar(CEREAL_NVP(prediction));
ar(CEREAL_NVP(dimensionType));
ar(CEREAL_NVP(splitPoint));
// Since splitPoint and prediction are a union, we only need to serialize one of them.
ar(CEREAL_NVP(splitPoint));
ar(CEREAL_NVP(splitInfo));
}
//! Return the number of leaves.
src/mlpack/methods/decision_tree/random_binary_numeric_split.hpp
+8 -6
View File
@@ -120,26 +120,28 @@ class RandomBinaryNumericSplit
const WeightVecType& weights,
const size_t minimumLeafSize,
const double minimumGainSplit,
double& splitInfo,
arma::vec& splitInfo,
AuxiliarySplitInfo& aux,
FitnessFunction& fitnessFunction,
const bool splitIfBetterGain = false);
/**
* Returns 2, since the binary split always has two children.
* If a split was found, returns the number of children of the split.
* Otherwise returns zero. A binary split always has two children.
*
* @param splitInfo Auxiliary information for the split.
* @param aux Auxiliary split information, which may be modified on a
* successful split.
*/
static size_t NumChildren(const double& /* splitInfo */,
static size_t NumChildren(const arma::vec& splitInfo,
const AuxiliarySplitInfo& /* aux */)
{
return 2;
return splitInfo.n_elem == 0 ? 0 : 2;
}
/**
* Given a point, calculate which child it should go to (left or right).
* If a split was found, given a point, calculate which child it should
* go to (left or right). Otherwise if there was no split, returns SIZE_MAX.
*
* @param point Point to calculate direction of.
* @param splitInfo Auxiliary information for the split.
@@ -148,7 +150,7 @@ class RandomBinaryNumericSplit
template<typename ElemType>
static size_t CalculateDirection(
const ElemType& point,
const double& splitInfo,
const arma::vec& splitInfo,
const AuxiliarySplitInfo& /* aux */);
};
src/mlpack/methods/decision_tree/random_binary_numeric_split_impl.hpp
+7 -4
View File
@@ -146,7 +146,7 @@ double RandomBinaryNumericSplit<FitnessFunction>::SplitIfBetter(
const WeightVecType& weights,
const size_t minimumLeafSize,
const double minimumGainSplit,
double& splitInfo,
arma::vec& splitInfo,
AuxiliarySplitInfo& /* aux */,
FitnessFunction& fitnessFunction,
const bool splitIfBetterGain)
@@ -245,7 +245,8 @@ double RandomBinaryNumericSplit<FitnessFunction>::SplitIfBetter(
if (gain < bestFoundGain && splitIfBetterGain)
return DBL_MAX;
splitInfo = randomPivot;
splitInfo.set_size(1);
splitInfo[0] = randomPivot;
if (UseWeights)
gain /= totalWeight;
@@ -259,10 +260,12 @@ template<typename FitnessFunction>
template<typename ElemType>
size_t RandomBinaryNumericSplit<FitnessFunction>::CalculateDirection(
const ElemType& point,
const double& splitInfo,
const arma::vec& splitInfo,
const AuxiliarySplitInfo& /* aux */)
{
if (point <= splitInfo)
if (splitInfo.n_elem == 0)
return SIZE_MAX;
else if (point <= splitInfo[0])
return 0; // Go left.
else
return 1; // Go right.
src/mlpack/tests/data/mushroom.data.csv
+8416
View File
File diff suppressed because it is too large Load Diff
src/mlpack/tests/data/mushroom.labels.csv
+8416
View File
File diff suppressed because it is too large Load Diff
src/mlpack/tests/decision_tree_regressor_test.cpp
+285 -12
View File
@@ -193,7 +193,7 @@ TEST_CASE("AllCategoricalSplitSimpleSplitTest_", "[DecisionTreeRegressorTest]")
responses[i + 1] = 100;
}
double splitInfo;
arma::vec splitInfo;
AllCategoricalSplit<MSEGain>::AuxiliarySplitInfo aux;
// Call the method to do the splitting.
@@ -211,7 +211,8 @@ TEST_CASE("AllCategoricalSplitSimpleSplitTest_", "[DecisionTreeRegressorTest]")
REQUIRE(gain == weightedGain);
// Make sure that splitInfo now hold the number of children.
REQUIRE((size_t) splitInfo == 2);
REQUIRE(splitInfo.n_elem == 1);
REQUIRE((size_t) splitInfo[0] == 2);
}
/**
@@ -225,7 +226,7 @@ TEST_CASE("AllCategoricalSplitMinSamplesTest_", "[DecisionTreeRegressorTest]")
arma::rowvec weights(responses.n_elem);
weights.ones();
double splitInfo;
arma::vec splitInfo;
AllCategoricalSplit<MSEGain>::AuxiliarySplitInfo aux;
// Call the method to do the splitting.
@@ -257,7 +258,7 @@ TEST_CASE("AllCategoricalSplitNoGainTest_", "[DecisionTreeRegressorTest]")
responses[i + 2] = 0.5;
}
double splitInfo;
arma::vec splitInfo;
AllCategoricalSplit<MSEGain>::AuxiliarySplitInfo aux;
// Call the method to do the splitting.
@@ -289,7 +290,7 @@ TEST_CASE("BestBinaryNumericSplitSimpleSplitTest_",
arma::rowvec weights(responses.n_elem);
weights.ones();
double splitInfo;
arma::vec splitInfo;
BestBinaryNumericSplit<MADGain>::AuxiliarySplitInfo aux;
// Call the method to do the splitting.
@@ -309,8 +310,9 @@ TEST_CASE("BestBinaryNumericSplitSimpleSplitTest_",
// The class probabilities, for this split, hold the splitting point, which
// should be between 4 and 5.
REQUIRE(splitInfo > 0.4);
REQUIRE(splitInfo < 0.5);
REQUIRE(splitInfo.n_elem == 1);
REQUIRE(splitInfo[0] > 0.4);
REQUIRE(splitInfo[0] < 0.5);
}
/**
@@ -326,7 +328,7 @@ TEST_CASE("BestBinaryNumericSplitMinSamplesTest_",
{ 0.5, 0.5, 0.5, 0.5, 0.5, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0 };
arma::rowvec weights(responses.n_elem);
double splitInfo;
arma::vec splitInfo;
BestBinaryNumericSplit<MSEGain>::AuxiliarySplitInfo aux;
// Call the method to do the splitting.
@@ -361,7 +363,7 @@ TEST_CASE("BestBinaryNumericSplitNoGainTest_", "[DecisionTreeRegressorTest]")
responses[i + 1] = 1.0;
}
double splitInfo;
arma::vec splitInfo;
BestBinaryNumericSplit<MSEGain>::AuxiliarySplitInfo aux;
// Call the method to do the splitting.
@@ -386,7 +388,7 @@ TEST_CASE("RandomBinaryNumericSplitAlwaysSplit_",
arma::rowvec weights;
weights.ones(responses.n_elem);
double splitInfo;
arma::vec splitInfo;
RandomBinaryNumericSplit<MSEGain>::AuxiliarySplitInfo aux;
// Call the method to do the splitting.
@@ -414,7 +416,7 @@ TEST_CASE("RandomBinaryNumericSplitMinSamplesTest_",
arma::rowvec responses("0 0 0 0 0 1 1 1 1 1 1");
arma::rowvec weights(responses.n_elem);
double splitInfo;
arma::vec splitInfo;
RandomBinaryNumericSplit<MSEGain>::AuxiliarySplitInfo aux;
// Call the method to do the splitting.
@@ -449,7 +451,7 @@ TEST_CASE("RandomBinaryNumericSplitNoGainTest_", "[DecisionTreeRegressorTest]")
responses[i + 1] = 1.0;
}
double splitInfo;
arma::vec splitInfo;
RandomBinaryNumericSplit<MSEGain>::AuxiliarySplitInfo aux;
// Call the method to do the splitting.
@@ -462,6 +464,277 @@ TEST_CASE("RandomBinaryNumericSplitNoGainTest_", "[DecisionTreeRegressorTest]")
REQUIRE(gain == DBL_MAX);
}
/**
* Check that the BestBinaryCategoricalSplit will split perfectly
* when this is clearly possible. Category C₂ has response 2.0
* and the rest are zero.
*/
TEST_CASE("BestBinaryCategoricalSplitRegressionTwoPerfectTest",
"[DecisionTreeRegressorTest]")
{
size_t N = 3131;
size_t K = 17;
double EPSILON = 1e-7;
size_t minLeaf = 10;
BestBinaryCategoricalSplit<MSEGain>::AuxiliarySplitInfo aux;
vec splitInfo;
MSEGain gainFn;
vec data = randi<vec>(N, distr_param(0,K-1));
rowvec weights = ones<rowvec>(N);
rowvec response(N);
for (size_t i = 0; i < N; ++i)
response[i] = data[i] == 2 ? 2 : 0;
// Find the best binary split of the data.
double bestGain = MSEGain::Evaluate<false>(response, weights);
double gain = BestBinaryCategoricalSplit<MSEGain>::SplitIfBetter<false>(
bestGain, data, K, response, weights, minLeaf,
EPSILON, splitInfo, aux, gainFn
);
double weightedGain =
BestBinaryCategoricalSplit<MSEGain>::SplitIfBetter<true>(
bestGain, data, K, response, weights, minLeaf,
EPSILON, splitInfo, aux, gainFn
);
// The split into categories [(2), (0, 1, 3, ..., K-1)] would be
// optimal, resulting in two pure children nodes, therefore the gain
// should be zero here. Unity weights means that the gain is the same
// with or without weights.
REQUIRE(gain > bestGain);
REQUIRE(gain == weightedGain);
REQUIRE(gain == Approx(0.0).margin(EPSILON));
// CalculateDirection should now send all of C₂ to the
// one direction and the remaining Cⱼ to the other.
vec class1_data = ones(N) * 2;
vec class1_direction(N);
vec class0_data= randi<vec>(N, distr_param(3, K - 1));
vec class0_direction(N);
for (size_t i = 0; i < N; ++i)
{
class0_direction(i) = BestBinaryCategoricalSplit<MSEGain>
::CalculateDirection(
class0_data(i), splitInfo, aux
);
class1_direction(i) = BestBinaryCategoricalSplit<MSEGain>
::CalculateDirection(
class1_data(i), splitInfo, aux
);
}
REQUIRE((all(class0_direction == LEFT) || all(class0_direction == RIGHT)));
REQUIRE((all(class1_direction == LEFT) || all(class1_direction == RIGHT)));
}
/**
* Check that no split is made when it doesn't get us anything.
*/
TEST_CASE("BestBinaryCategoricalSplitRegressionNoGainTest",
"[DecisionTreeRegressorTest]")
{
size_t N = 300;
size_t K = 10;
double EPSILON = 1e-7;
size_t minLeaf = 10;
BestBinaryCategoricalSplit<MSEGain>::AuxiliarySplitInfo aux;
MSEGain gainFn;
vec splitInfo;
vec data(N);
rowvec response(N);
rowvec weights = ones<rowvec>(N);
for (size_t i = 0; i < N; i += 2)
{
data[i] = i % K;
response[i] = 0;
data[i + 1] = i % K;
response[i + 1] = 1;
}
// Call the method to do the splitting.
double bestGain = MSEGain::Evaluate<false>(response, weights);
double gain = BestBinaryCategoricalSplit<MSEGain>::SplitIfBetter<false>(
bestGain, data, K, response, weights, minLeaf,
EPSILON, splitInfo, aux, gainFn
);
double weightedGain =
BestBinaryCategoricalSplit<MSEGain>::SplitIfBetter<true>(
bestGain, data, K, response, weights, minLeaf,
EPSILON, splitInfo, aux, gainFn
);
// Make sure that there was no split.
REQUIRE(gain == DBL_MAX);
REQUIRE(gain == weightedGain);
REQUIRE(splitInfo.n_elem == 0);
}
/**
* Make sure that BestBinaryCategoricalSplit respects the minimum number
* of samples required to split.
*/
TEST_CASE("BestBinaryCategoricalSplitRegressionMinSamplesTest",
"[DecisionTreeRegressorTest]")
{
size_t K = 4;
double EPSILON = 1e-7;
size_t minLeaf = 8;
BestBinaryCategoricalSplit<MSEGain>::AuxiliarySplitInfo aux;
MSEGain gainFn;
vec splitInfo;
vec data("0 0 0 1 1 1 2 2 2 3 3 3");
vec response("0 0 0 1 1 1 0 0 0 1 1 1");
rowvec weights(response.n_elem, arma::fill::ones);
// Call the method to do the splitting.
double bestGain = MSEGain::Evaluate<false>(response, weights);
double gain = BestBinaryCategoricalSplit<MSEGain>::SplitIfBetter<false>(
bestGain, data, K, response, weights, minLeaf,
EPSILON, splitInfo, aux, gainFn
);
double weightedGain =
BestBinaryCategoricalSplit<MSEGain>::SplitIfBetter<true>(
bestGain, data, K, response, weights, minLeaf,
EPSILON, splitInfo, aux, gainFn
);
// Make sure it's not split.
REQUIRE(gain == DBL_MAX);
REQUIRE(weightedGain == DBL_MAX);
REQUIRE(splitInfo.n_elem == 0);
}
/**
* Test that we can build a decision tree on a simple categorical dataset
* using the BestBinaryCategoricalSplit.
*/
TEST_CASE("BestCategoricalBuildTest_", "[DecisionTreeRegressorTest]")
{
mat d;
rowvec r;
data::DatasetInfo di;
MockCategoricalData(d, r, di);
// Split into a training set and a test set.
mat trainingData = d.cols(0, 1999);
mat testData = d.cols(2000, 3999);
rowvec trainingResponses = r.subvec(0, 1999);
rowvec testResponses = r.subvec(2000, 3999);
size_t minLeaf = 10;
// Build the tree.
DecisionTreeRegressor<
MSEGain, BestBinaryNumericSplit, BestBinaryCategoricalSplit>
tree(trainingData, di, trainingResponses, minLeaf);
// Now evaluate the quality of predictions.
rowvec predictions;
tree.Predict(testData, predictions);
// Make sure we get reasonable rmse.
const double rmse = RMSE(predictions, testResponses);
REQUIRE(predictions.n_elem == testData.n_cols);
REQUIRE(rmse < 1.0);
}
/**
* Test that we can build a decision tree with weights on a simple categorical
* dataset using the BestBinaryCategoricalSplit.
*/
TEST_CASE("BestCategoricalWeightedBuildTest_", "[DecisionTreeRegressorTest]")
{
mat d;
rowvec r;
data::DatasetInfo di;
MockCategoricalData(d, r, di);
// Split into a training set and a test set.
mat trainingData = d.cols(0, 1999);
mat testData = d.cols(2000, 3999);
rowvec trainingResponses = r.subvec(0, 1999);
rowvec testResponses = r.subvec(2000, 3999);
rowvec weights = ones<rowvec>(trainingResponses.n_elem);
size_t minLeaf = 10;
// Build the tree.
DecisionTreeRegressor<
MSEGain, BestBinaryNumericSplit, BestBinaryCategoricalSplit>
tree(trainingData, di, trainingResponses, weights, minLeaf);
// Now evaluate the quality of predictions.
rowvec predictions;
tree.Predict(testData, predictions);
// Make sure we get reasonable rmse.
const double rmse = RMSE(predictions, testResponses);
REQUIRE(predictions.n_elem == testData.n_cols);
REQUIRE(rmse < 1.0);
}
/**
* Test that we can build a decision tree on a simple categorical dataset using
* weights, with low-weight noise added, using the BestBinaryCategoricalSplit.
*/
TEST_CASE("BestCategoricalNoisyWeightedBuildTest_",
"[DecisionTreeRegressorTest]")
{
mat d;
rowvec r;
data::DatasetInfo di;
MockCategoricalData(d, r, di);
// Split into a training set and a test set.
mat trainingData = d.cols(0, 1999);
mat testData = d.cols(2000, 3999);
rowvec trainingResponses = r.subvec(0, 1999);
rowvec testResponses = r.subvec(2000, 3999);
size_t minLeaf = 10;
// Now create random points.
mat randomNoise(5, 2000);
rowvec randomResponses(2000);
for (size_t i = 0; i < 2000; ++i)
{
randomNoise(0, i) = Random();
randomNoise(1, i) = Random(-1, 1);
randomNoise(2, i) = Random();
randomNoise(3, i) = RandInt(0, 2);
randomNoise(4, i) = RandInt(0, 5);
randomResponses[i] = Random(-10, 18);
}
// Generate weights.
rowvec weights(4000);
for (size_t i = 0; i < 2000; ++i)
weights[i] = Random(0.9, 1.0);
for (size_t i = 2000; i < 4000; ++i)
weights[i] = Random(0.0, 0.001);
mat fullData = join_rows(trainingData, randomNoise);
rowvec fullResponses = join_rows(trainingResponses, randomResponses);
// Build the tree.
DecisionTreeRegressor<
MSEGain, BestBinaryNumericSplit, BestBinaryCategoricalSplit>
tree(fullData, di, fullResponses, weights, minLeaf);
// Now evaluate the quality of predictions.
rowvec predictions;
tree.Predict(testData, predictions);
// Make sure we get reasonable rmse.
const double rmse = RMSE(predictions, testResponses);
REQUIRE(predictions.n_elem == testData.n_cols);
REQUIRE(rmse < 1.5);
}
/**
* A basic construction of the decision tree---ensure that we can create the
* tree and that it split at least once.
src/mlpack/tests/decision_tree_test.cpp
+482 -6
View File
@@ -282,16 +282,16 @@ TEST_CASE("BestBinaryNumericSplitSimpleSplitTest", "[DecisionTreeTest]")
arma::rowvec weights(labels.n_elem);
weights.ones();
arma::vec classProbabilities;
arma::vec splitInfo;
BestBinaryNumericSplit<GiniGain>::AuxiliarySplitInfo aux;
// Call the method to do the splitting.
const double bestGain = GiniGain::Evaluate<false>(labels, 2, weights);
const double gain = BestBinaryNumericSplit<GiniGain>::SplitIfBetter<false>(
bestGain, values, labels, 2, weights, 3, 1e-7, classProbabilities, aux);
bestGain, values, labels, 2, weights, 3, 1e-7, splitInfo, aux);
const double weightedGain =
BestBinaryNumericSplit<GiniGain>::SplitIfBetter<true>(bestGain, values,
labels, 2, weights, 3, 1e-7, classProbabilities, aux);
labels, 2, weights, 3, 1e-7, splitInfo, aux);
// Make sure that a split was made.
REQUIRE(gain > bestGain);
@@ -304,9 +304,9 @@ TEST_CASE("BestBinaryNumericSplitSimpleSplitTest", "[DecisionTreeTest]")
// The class probabilities, for this split, hold the splitting point, which
// should be between 4 and 5.
REQUIRE(classProbabilities.n_elem == 1);
REQUIRE(classProbabilities[0] > 0.4);
REQUIRE(classProbabilities[0] < 0.5);
REQUIRE(splitInfo.n_elem == 1);
REQUIRE(splitInfo[0] > 0.4);
REQUIRE(splitInfo[0] < 0.5);
}
/**
@@ -566,6 +566,482 @@ TEST_CASE("AllCategoricalSplitNoGainTest", "[DecisionTreeTest]")
REQUIRE(classProbabilities.n_elem == 0);
}
/**
* Check that the BestBinaryCategoricalSplit will split perfectly
* (in the binary class setting) when this is clearly possible.
* Category C₂ is class one and the rest are class zero.
*/
TEST_CASE("BestBinaryCategoricalSplitBinaryClassTwoPerfectTest",
"[DecisionTreeTest]")
{
size_t N = 3131;
size_t K = 17;
double EPSILON = 1e-7;
size_t numClasses = 2;
size_t minLeaf = 10;
vec splitInfo;
BestBinaryCategoricalSplit<GiniGain>::AuxiliarySplitInfo aux;
vec data = randi<vec>(N, distr_param(0,K-1));
rowvec weights = ones<rowvec>(N);
Row<size_t> labels(N);
for (size_t i = 0; i < N; ++i)
labels[i] = (size_t) data[i] == 2;
// Find the best binary split of the data.
double bestGain = GiniGain::Evaluate<false>(labels, numClasses, weights);
double gain = BestBinaryCategoricalSplit<GiniGain>::SplitIfBetter<false>(
bestGain, data, K, labels, numClasses, weights, minLeaf,
EPSILON, splitInfo, aux
);
double weightedGain = BestBinaryCategoricalSplit<GiniGain>
::SplitIfBetter<true>(
bestGain, data, K, labels, numClasses, weights, minLeaf,
EPSILON, splitInfo, aux
);
// The split into categories [(2), (0, 1, 3, ..., K-1)] would be
// optimal, resulting in two pure children nodes, therefore the gain
// should be zero here. Unity weights means that the gain is the same
// with or without weights.
REQUIRE(gain > bestGain);
REQUIRE(gain == weightedGain);
REQUIRE(gain == Approx(0.0).margin(EPSILON));
// CalculateDirection should now send all of C₂ to the
// one direction and the remaining Cⱼ to the other.
vec class1_data = ones(N) * 2;
vec class1_direction(N);
vec class0_data= randi<vec>(N, distr_param(3, K - 1));
vec class0_direction(N);
for (size_t i = 0; i < N; ++i)
{
class0_direction(i) = BestBinaryCategoricalSplit<GiniGain>
::CalculateDirection(
class0_data(i), splitInfo, aux
);
class1_direction(i) = BestBinaryCategoricalSplit<GiniGain>
::CalculateDirection(
class1_data(i), splitInfo, aux
);
}
REQUIRE((all(class0_direction == LEFT) || all(class0_direction == RIGHT)));
REQUIRE((all(class1_direction == LEFT) || all(class1_direction == RIGHT)));
}
/**
* Check that the BestBinaryCategoricalSplit will split optimally in the
* multi-class setting. We need another test for this case because the
* algorithm for multiple classes is fundamentally different. Labels are
* created by the identity function over categories, that is a sample with
* category Cⱼ has label j. All but four of the samples are category (and label)
* zero, therefore BestBinaryCategoricalSplit should choose to partition
* category C₀ from all the rest of the Cⱼ.
*/
TEST_CASE("BestBinaryCategoricalSplitMultiClassZeroTest", "[DecisionTreeTest]")
{
size_t N = 3131;
size_t K = 5;
double EPSILON = 1e-7;
size_t numClasses = K;
size_t minLeaf = 10;
vec splitInfo;
BestBinaryCategoricalSplit<GiniGain>::AuxiliarySplitInfo aux;
size_t index;
// Initialize data such that it is all category C₂, except for one
// sample from each of the remaining categories. Labels are mapped
// by the identity function. Category Cᵢ -> i.
vec data = zeros(N);
Row<size_t> labels = zeros<Row<size_t>>(N);
rowvec weights = ones<rowvec>(N);
for (size_t category = 1; category < K; ++category)
{
index = randi(distr_param(0, N-1));
data[index] = (double) category;
labels[index] = category;
}
// Find the best binary split of the data.
double bestGain = GiniGain::Evaluate<false>(labels, numClasses, weights);
double gain = BestBinaryCategoricalSplit<GiniGain>::SplitIfBetter<false>(
bestGain, data, K, labels, numClasses, weights, minLeaf,
EPSILON, splitInfo, aux
);
double weightedGain = BestBinaryCategoricalSplit<GiniGain>
::SplitIfBetter<true>(
bestGain, data, K, labels, numClasses, weights, minLeaf,
EPSILON, splitInfo, aux
);
// The split into categories [(0), (1, 2, ..., K-1)] would be
// optimal, resulting in one pure child node of many zeros, and
// one child with K - 1 samples, all of different classes.
double expectedGain = -.00095816;
REQUIRE(gain > bestGain);
REQUIRE(gain == weightedGain);
REQUIRE(gain == Approx(expectedGain).margin(EPSILON));
// CalculateDirection should now send all of C₂ to the
// one direction and the remaining Cⱼ to the other.
vec class0_data = zeros(N);
vec class0_direction(N);
vec classj_data= randi<vec>(N, distr_param(1, K - 1));
vec classj_direction(N);
for (size_t i = 0; i < N; ++i)
{
class0_direction(i) = BestBinaryCategoricalSplit<GiniGain>
::CalculateDirection(
class0_data(i), splitInfo, aux
);
classj_direction(i) = BestBinaryCategoricalSplit<GiniGain>
::CalculateDirection(
classj_data(i), splitInfo, aux
);
}
REQUIRE((all(class0_direction == LEFT) || all(class0_direction == RIGHT)));
REQUIRE((all(classj_direction == LEFT) || all(classj_direction == RIGHT)));
}
/**
* Check that no split is made when it doesn't get us anything
* in the binary classification setting.
*/
TEST_CASE("BestBinaryCategoricalSplitNoGainBinaryTest", "[DecisionTreeTest]")
{
size_t N = 300;
size_t K = 10;
double EPSILON = 1e-7;
size_t numClasses = 2;
size_t minLeaf = 10;
vec splitInfo;
BestBinaryCategoricalSplit<GiniGain>::AuxiliarySplitInfo aux;
vec data(N);
Row<size_t> labels(N);
rowvec weights = ones<rowvec>(N);
for (size_t i = 0; i < N; i += numClasses)
{
data[i] = int(i / numClasses) % 10;
labels[i] = 0;
data[i + 1] = int(i / numClasses) % 10;
labels[i + 1] = 1;
}
// Call the method to do the splitting.
double bestGain = GiniGain::Evaluate<false>(labels, numClasses, weights);
double gain = BestBinaryCategoricalSplit<GiniGain>::SplitIfBetter<false>(
bestGain, data, K, labels, numClasses, weights, minLeaf, EPSILON,
splitInfo, aux);
double weightedGain =
BestBinaryCategoricalSplit<GiniGain>::SplitIfBetter<true>(
bestGain, data, K, labels, numClasses, weights,
minLeaf, EPSILON, splitInfo, aux
);
// Make sure that there was no split.
REQUIRE(gain == DBL_MAX);
REQUIRE(gain == weightedGain);
REQUIRE(splitInfo.n_elem == 0);
}
/**
* Check that no split is made when it doesn't get us anything
* in the multi-class classification setting.
*/
TEST_CASE("BestBinaryCategoricalSplitNoGainMultiTest", "[DecisionTreeTest]")
{
size_t N = 300;
size_t K = 10;
double EPSILON = 1e-7;
size_t numClasses = 5;
size_t minLeaf = 10;
vec splitInfo;
BestBinaryCategoricalSplit<GiniGain>::AuxiliarySplitInfo aux;
vec data(N);
Row<size_t> labels(N);
rowvec weights = ones<rowvec>(N);
for (size_t i = 0; i < N; i += numClasses)
{
data[i] = int(i / numClasses) % 10;
labels[i] = 0;
data[i + 1] = int(i / numClasses) % 10;
labels[i + 1] = 1;
data[i + 2] = int(i / numClasses) % 10;
labels[i + 2] = 2;
data[i + 3] = int(i / numClasses) % 10;
labels[i + 3] = 3;
data[i + 4] = int(i / numClasses) % 10;
labels[i + 4] = 4;
}
// Call the method to do the splitting.
double bestGain = GiniGain::Evaluate<false>(labels, numClasses, weights);
double gain = BestBinaryCategoricalSplit<GiniGain>::SplitIfBetter<false>(
bestGain, data, K, labels, numClasses, weights, minLeaf, EPSILON,
splitInfo, aux);
double weightedGain =
BestBinaryCategoricalSplit<GiniGain>::SplitIfBetter<true>(
bestGain, data, K, labels, numClasses, weights, minLeaf,
EPSILON, splitInfo, aux
);
REQUIRE(gain == DBL_MAX);
REQUIRE(gain == weightedGain);
REQUIRE(splitInfo.n_elem == 0);
}
/**
* Make sure that BestBinaryCategoricalSplit respects the minimum number
* of samples required to split in the binary classification setting.
*/
TEST_CASE("BestBinaryCategoricalSplitMinSamplesBinaryTest", "[DecisionTreeTest]")
{
size_t K = 4;
double EPSILON = 1e-7;
size_t numClasses = 2;
size_t minLeaf = 8;
vec data("0 0 0 1 1 1 2 2 2 3 3 3");
Row<size_t> labels("0 0 0 1 1 1 0 0 0 1 1 1");
rowvec weights(labels.n_elem);
weights.ones();
vec splitInfo;
BestBinaryCategoricalSplit<GiniGain>::AuxiliarySplitInfo aux;
// Call the method to do the splitting.
double bestGain = GiniGain::Evaluate<false>(labels, numClasses, weights);
double gain = BestBinaryCategoricalSplit<GiniGain>::SplitIfBetter<false>(
bestGain, data, K, labels, numClasses, weights, minLeaf, EPSILON,
splitInfo, aux);
// Make sure it's not split.
REQUIRE(gain == DBL_MAX);
REQUIRE(splitInfo.n_elem == 0);
}
/**
* Make sure that BestBinaryCategoricalSplit respects the minimum number
* of samples required to split in the multi-class setting.
*/
TEST_CASE("BestBinaryCategoricalSplitMinSamplesMultiTest", "[DecisionTreeTest]")
{
size_t K = 4;
double EPSILON = 1e-7;
size_t numClasses = 4;
size_t minLeaf = 8;
vec data("0 0 0 1 1 1 2 2 2 3 3 3");
Row<size_t> labels("0 0 0 1 1 1 2 2 2 3 3 3");
rowvec weights(labels.n_elem);
weights.ones();
vec splitInfo;
BestBinaryCategoricalSplit<GiniGain>::AuxiliarySplitInfo aux;
// Call the method to do the splitting.
double bestGain = GiniGain::Evaluate<false>(labels, numClasses, weights);
double gain = BestBinaryCategoricalSplit<GiniGain>::SplitIfBetter<false>(
bestGain, data, K, labels, numClasses, weights, minLeaf, EPSILON,
splitInfo, aux);
// Make sure it's not split.
REQUIRE(gain == DBL_MAX);
REQUIRE(splitInfo.n_elem == 0);
}
/**
* Test that we can build a decision tree on a simple categorical dataset
* (the mushroom dataset from UCI) using the BestBinaryCategoricalSplit
* in a binary classification setting. This dataset can be found at
*
* https://archive.ics.uci.edu/dataset/73/mushroom
*/
TEST_CASE("BestCategoricalBuildBinaryTest", "[DecisionTreeTest]")
{
// Load the categorical UCI mushroom dataset and split
// into a training set and test set.
mat dataset;
Row<size_t> labels;
data::DatasetInfo dataInfo;
data::LoadCSV csv("mushroom.data.csv");
csv.LoadCategoricalCSV(dataset, dataInfo);
data::Load("mushroom.labels.csv", labels, true);
mat trainDataset, testDataset;
Row<size_t> trainLabels, testLabels;
data::Split(dataset, labels,
trainDataset, testDataset, trainLabels, testLabels, 0.3);
// Build the DecisionTree with a BestBinaryCategoricalSplit.
size_t numClasses = 2;
size_t minLeaf = 10;
size_t maxDepth = 4;
double minGainSplit = 10e-7;
DecisionTree<GiniGain, BestBinaryNumericSplit, BestBinaryCategoricalSplit>
tree(trainDataset, dataInfo, trainLabels, numClasses,
minLeaf, minGainSplit, maxDepth
);
// Compute the accuracy of the DecisionTree. It should
// be well over 95%.
Row<size_t> predictions;
tree.Classify(testDataset, predictions);
size_t correct = 0;
for (size_t i = 0; i < testDataset.n_cols; ++i)
if (testLabels[i] == predictions[i])
++correct;
const double correctPct = double(correct) / double(testDataset.n_cols);
REQUIRE(predictions.n_cols == testDataset.n_cols);
REQUIRE(correctPct > 0.95);
}
/**
* Test that we can build a decision tree on a simple categorical dataset
* using the BestBinaryCategoricalSplit in a multi-class setting.
*/
TEST_CASE("BestCategoricalBuildMultiTest", "[DecisionTreeTest]")
{
mat d;
Row<size_t> l;
data::DatasetInfo di;
MockCategoricalData(d, l, di);
// Split into a training set and a test set.
mat trainingData = d.cols(0, 1999);
mat testData = d.cols(2000, 3999);
Row<size_t> trainingLabels = l.subvec(0, 1999);
Row<size_t> testLabels = l.subvec(2000, 3999);
// Build the tree.
DecisionTree<GiniGain, BestBinaryNumericSplit, BestBinaryCategoricalSplit>
tree(trainingData, di, trainingLabels, 5, 10);
// Now evaluate the accuracy of the tree.
Row<size_t> predictions;
tree.Classify(testData, predictions);
size_t correct = 0;
for (size_t i = 0; i < testData.n_cols; ++i)
if (testLabels[i] == predictions[i])
++correct;
// Make sure we got at least 70% accuracy.
const double correctPct = double(correct) / double(testData.n_cols);
REQUIRE(predictions.n_cols == testData.n_cols);
REQUIRE(correctPct > 0.70);
}
/**
* Test that we can build a decision tree with weights on a simple categorical
* dataset using the BestBinaryCategoricalSplit.
*/
TEST_CASE("BestCategoricalBuildTestWithWeight", "[DecisionTreeTest]")
{
mat d;
Row<size_t> l;
data::DatasetInfo di;
MockCategoricalData(d, l, di);
// Split into a training set and a test set.
mat trainingData = d.cols(0, 1999);
mat testData = d.cols(2000, 3999);
Row<size_t> trainingLabels = l.subvec(0, 1999);
Row<size_t> testLabels = l.subvec(2000, 3999);
Row<double> weights = ones<Row<double>>(
trainingLabels.n_elem);
// Build the tree.
DecisionTree<GiniGain, BestBinaryNumericSplit, BestBinaryCategoricalSplit>
tree(trainingData, di, trainingLabels, 5, weights, 10);
// Now evaluate the accuracy of the tree.
Row<size_t> predictions;
tree.Classify(testData, predictions);
REQUIRE(predictions.n_elem == testData.n_cols);
size_t correct = 0;
for (size_t i = 0; i < testData.n_cols; ++i)
if (testLabels[i] == predictions[i])
++correct;
// Make sure we got at least 90% accuracy.
const double correctPct = double(correct) / double(testData.n_cols);
REQUIRE(correctPct > 0.90);
}
/**
* Test that we can build a decision tree on a simple categorical dataset using
* weights, with low-weight noise added, using the BestBinaryCategoricalSplit.
*/
TEST_CASE("BestCategoricalBuildTestWithWeightNoisy", "[DecisionTreeTest]")
{
mat d;
Row<size_t> l;
data::DatasetInfo di;
MockCategoricalData(d, l, di);
// Split into a training set and a test set.
mat trainingData = d.cols(0, 1999);
mat testData = d.cols(2000, 3999);
Row<size_t> trainingLabels = l.subvec(0, 1999);
Row<size_t> testLabels = l.subvec(2000, 3999);
// Now create random points.
mat randomNoise(4, 2000);
Row<size_t> randomLabels(2000);
for (size_t i = 0; i < 2000; ++i)
{
randomNoise(0, i) = Random();
randomNoise(1, i) = Random();
randomNoise(2, i) = RandInt(4);
randomNoise(3, i) = RandInt(2);
randomLabels[i] = RandInt(5);
}
// Generate weights.
rowvec weights(4000);
for (size_t i = 0; i < 2000; ++i)
weights[i] = Random(0.9, 1.0);
for (size_t i = 2000; i < 4000; ++i)
weights[i] = Random(0.0, 0.001);
mat fullData = join_rows(trainingData, randomNoise);
Row<size_t> fullLabels = join_rows(trainingLabels, randomLabels);
// Build the tree.
DecisionTree<GiniGain, BestBinaryNumericSplit, BestBinaryCategoricalSplit>
tree(fullData, di, fullLabels, 5, weights, 10);
// Now evaluate the accuracy of the tree.
Row<size_t> predictions;
tree.Classify(testData, predictions);
REQUIRE(predictions.n_elem == testData.n_cols);
size_t correct = 0;
for (size_t i = 0; i < testData.n_cols; ++i)
if (testLabels[i] == predictions[i])
++correct;
// Make sure we got at least 90% accuracy.
const double correctPct = double(correct) / double(testData.n_cols);
REQUIRE(correctPct > 0.90);
}
/**
* A basic construction of the decision tree---ensure that we can create the
* tree and that it split at least once.