apaam/mlpack
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #2266 from bisakhmondal/Removing-Decision-stump

Browse Source 
Removing decision stump class
This commit is contained in:
Ryan Curtin
2020-12-21 22:03:08 -05:00
committed by GitHub
parent 685de4d5a9 eb8fe0a83e
commit a766ea292e
10 changed files with 1 additions and 1463 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/mlpack/methods/CMakeLists.txt
-1
View File
@@ -10,7 +10,6 @@ set(DIRS
block_krylov_svd
cf
dbscan
decision_stump
decision_tree
det
emst
src/mlpack/methods/adaboost/adaboost.hpp
+1 -1
View File
@@ -71,7 +71,7 @@ namespace adaboost {
* @endcode
*
* For more information on and examples of weak learners, see
* perceptron::Perceptron<> and decision_stump::DecisionStump<>.
* perceptron::Perceptron<> and tree::ID3DecisionStump.
*
* @tparam MatType Data matrix type (i.e. arma::mat or arma::sp_mat).
* @tparam WeakLearnerType Type of weak learner to use.
src/mlpack/methods/decision_stump/CMakeLists.txt
-21
View File
@@ -1,21 +0,0 @@
# Define the files we need to compile.
# Anything not in this list will not be compiled into mlpack.
set(SOURCES
decision_stump.hpp
decision_stump_impl.hpp
)
# Add directory name to sources.
set(DIR_SRCS)
foreach(file ${SOURCES})
set(DIR_SRCS ${DIR_SRCS} ${CMAKE_CURRENT_SOURCE_DIR}/${file})
endforeach()
# Append sources (with directory name) to list of all mlpack sources (used at
# the parent scope).
set(MLPACK_SRCS ${MLPACK_SRCS} ${DIR_SRCS} PARENT_SCOPE)
add_cli_executable(decision_stump)
add_python_binding(decision_stump)
add_julia_binding(decision_stump)
add_go_binding(decision_stump)
add_markdown_docs(decision_stump "cli;python;julia;go" "classification")
src/mlpack/methods/decision_stump/decision_stump.hpp
-239
View File
@@ -1,239 +0,0 @@
/**
* @file methods/decision_stump/decision_stump.hpp
* @author Udit Saxena
*
* Definition of decision stumps.
*
* mlpack is free software; you may redistribute it and/or modify it under the
* terms of the 3-clause BSD license. You should have received a copy of the
* 3-clause BSD license along with mlpack. If not, see
* http://www.opensource.org/licenses/BSD-3-Clause for more information.
*/
#ifndef MLPACK_METHODS_DECISION_STUMP_DECISION_STUMP_HPP
#define MLPACK_METHODS_DECISION_STUMP_DECISION_STUMP_HPP
#include <mlpack/prereqs.hpp>
namespace mlpack {
namespace decision_stump {
/**
* This class implements a decision stump. It constructs a single level
* decision tree, i.e., a decision stump. It uses entropy to decide splitting
* ranges.
*
* The stump is parameterized by a splitting dimension (the dimension on which
* points are split), a vector of bin split values, and a vector of labels for
* each bin. Bin i is specified by the range [split[i], split[i + 1]). The
* last bin has range up to @f$ \infty @f$ (split[i + 1] does not exist in that
* case).
* Points that are below the first bin will take the label of the first bin.
*
* @note
* This class has been deprecated and should be removed in mlpack 4.0.0. Use
* `ID3DecisionStump`, found in src/mlpack/methods/decision_tree/, instead.
*
* @tparam MatType Type of matrix that is being used (sparse or dense).
*/
template<typename MatType = arma::mat>
class DecisionStump
{
public:
/**
* Constructor. Train on the provided data. Generate a decision stump from
* data.
*
* @param data Input, training data.
* @param labels Labels of training data.
* @param numClasses Number of distinct classes in labels.
* @param bucketSize Minimum size of bucket when splitting.
*/
mlpack_deprecated DecisionStump(const MatType& data,
const arma::Row<size_t>& labels,
const size_t numClasses,
const size_t bucketSize = 10);
/**
* Alternate constructor which copies the parameters bucketSize and classes
* from an already initiated decision stump, other. It appropriately sets the
* weight vector.
*
* @param other The other initiated Decision Stump object from
* which we copy the values.
* @param data The data on which to train this object on.
* @param labels The labels of data.
* @param numClasses The number of classes.
* @param weights Weight vector to use while training. For boosting purposes.
*/
mlpack_deprecated DecisionStump(const DecisionStump<>& other,
const MatType& data,
const arma::Row<size_t>& labels,
const size_t numClasses,
const arma::rowvec& weights);
/**
* Create a decision stump without training. This stump will not be useful
* and will always return a class of 0 for anything that is to be classified,
* so it would be a prudent idea to call Train() after using this constructor.
*/
DecisionStump();
/**
* Train the decision stump on the given data. This completely overwrites any
* previous training data, so after training the stump may be completely
* different.
*
* @param data Dataset to train on.
* @param labels Labels for each point in the dataset.
* @param numClasses Number of classes in the dataset.
* @param bucketSize Minimum size of bucket when splitting.
* @return The final entropy after splitting.
*/
mlpack_deprecated double Train(const MatType& data,
const arma::Row<size_t>& labels,
const size_t numClasses,
const size_t bucketSize);
/**
* Train the decision stump on the given data, with the given weights. This
* completely overwrites any previous training data, so after training the
* stump may be completely different.
*
* @param data Dataset to train on.
* @param labels Labels for each point in the dataset.
* @param weights Weights for each point in the dataset.
* @param numClasses Number of classes in the dataset.
* @param bucketSize Minimum size of bucket when splitting.
* @return The final entropy after splitting.
*/
mlpack_deprecated double Train(const MatType& data,
const arma::Row<size_t>& labels,
const arma::rowvec& weights,
const size_t numClasses,
const size_t bucketSize);
/**
* Classification function. After training, classify test, and put the
* predicted classes in predictedLabels.
*
* @param test Testing data or data to classify.
* @param predictedLabels Vector to store the predicted classes after
* classifying test data.
*/
mlpack_deprecated void Classify(const MatType& test,
arma::Row<size_t>& predictedLabels);
//! Access the splitting dimension.
size_t SplitDimension() const { return splitDimension; }
//! Modify the splitting dimension (be careful!).
size_t& SplitDimension() { return splitDimension; }
//! Access the splitting values.
const arma::vec& Split() const { return split; }
//! Modify the splitting values (be careful!).
arma::vec& Split() { return split; }
//! Access the labels for each split bin.
const arma::Col<size_t> BinLabels() const { return binLabels; }
//! Modify the labels for each split bin (be careful!).
arma::Col<size_t>& BinLabels() { return binLabels; }
//! Serialize the decision stump.
template<typename Archive>
void serialize(Archive& ar, const uint32_t /* version */);
private:
//! The number of classes (we must store this for boosting).
size_t numClasses;
//! The minimum number of points in a bucket.
size_t bucketSize;
//! Stores the value of the dimension on which to split.
size_t splitDimension;
//! Stores the splitting values after training.
arma::vec split;
//! Stores the labels for each splitting bin.
arma::Col<size_t> binLabels;
/**
* Sets up dimension as if it were splitting on it and finds entropy when
* splitting on dimension.
*
* @param dimension A row from the training data, which might be a
* candidate for the splitting dimension.
* @tparam UseWeights Whether we need to run a weighted Decision Stump.
*/
template<bool UseWeights, typename VecType>
double SetupSplitDimension(const VecType& dimension,
const arma::Row<size_t>& labels,
const arma::rowvec& weightD);
/**
* After having decided the dimension on which to split, train on that
* dimension.
*
* @tparam dimension dimension is the dimension decided by the constructor
* on which we now train the decision stump.
*/
template<typename VecType>
void TrainOnDim(const VecType& dimension,
const arma::Row<size_t>& labels);
/**
* After the "split" matrix has been set up, merge ranges with identical class
* labels.
*/
void MergeRanges();
/**
* Count the most frequently occurring element in subCols.
*
* @param subCols The vector in which to find the most frequently occurring
* element.
*/
template<typename VecType>
double CountMostFreq(const VecType& subCols);
/**
* Returns 1 if all the values of featureRow are not same.
*
* @param featureRow The dimension which is checked for identical values.
*/
template<typename VecType>
int IsDistinct(const VecType& featureRow);
/**
* Calculate the entropy of the given dimension.
*
* @param labels Corresponding labels of the dimension.
* @param classes Number of classes.
* @param weights Weights for this set of labels.
* @tparam UseWeights If true, the weights in the weight vector will be used
* (otherwise they are ignored).
*/
template<bool UseWeights, typename VecType, typename WeightVecType>
double CalculateEntropy(const VecType& labels,
const WeightVecType& weights);
/**
* Train the decision stump on the given data and labels.
*
* @param data Dataset to train on.
* @param labels Labels for dataset.
* @param weights Weights for this set of labels.
* @tparam UseWeights If true, the weights in the weight vector will be used
* (otherwise they are ignored).
* @return The final entropy after splitting.
*/
template<bool UseWeights>
double Train(const MatType& data,
const arma::Row<size_t>& labels,
const arma::rowvec& weights);
};
} // namespace decision_stump
} // namespace mlpack
#include "decision_stump_impl.hpp"
#endif
src/mlpack/methods/decision_stump/decision_stump_impl.hpp
-518
View File
@@ -1,518 +0,0 @@
/**
* @file methods/decision_stump/decision_stump_impl.hpp
* @author Udit Saxena
*
* Implementation of DecisionStump class.
*
* mlpack is free software; you may redistribute it and/or modify it under the
* terms of the 3-clause BSD license. You should have received a copy of the
* 3-clause BSD license along with mlpack. If not, see
* http://www.opensource.org/licenses/BSD-3-Clause for more information.
*/
#ifndef MLPACK_METHODS_DECISION_STUMP_DECISION_STUMP_IMPL_HPP
#define MLPACK_METHODS_DECISION_STUMP_DECISION_STUMP_IMPL_HPP
// In case it hasn't been included yet.
#include "decision_stump.hpp"
namespace mlpack {
namespace decision_stump {
/**
* Constructor. Train on the provided data. Generate a decision stump from data.
*
* @param data Input, training data.
* @param labels Labels of data.
* @param numClasses Number of distinct classes in labels.
* @param bucketSize Minimum size of bucket when splitting.
*/
template<typename MatType>
DecisionStump<MatType>::DecisionStump(const MatType& data,
const arma::Row<size_t>& labels,
const size_t numClasses,
const size_t bucketSize) :
numClasses(numClasses),
bucketSize(bucketSize)
{
arma::rowvec weights;
Train<false>(data, labels, weights);
}
/**
* Empty constructor.
*/
template<typename MatType>
DecisionStump<MatType>::DecisionStump() :
numClasses(1),
bucketSize(0),
splitDimension(0),
split(1),
binLabels(1)
{
split[0] = DBL_MAX;
binLabels[0] = 0;
}
/**
* Train on the given data and labels.
*/
template<typename MatType>
double DecisionStump<MatType>::Train(const MatType& data,
const arma::Row<size_t>& labels,
const size_t numClasses,
const size_t bucketSize)
{
this->numClasses = numClasses;
this->bucketSize = bucketSize;
// Pass to unweighted training function.
arma::rowvec weights;
return Train<false>(data, labels, weights);
}
/**
* Train the decision stump on the given data, with the given weights. This
* completely overwrites any previous training data, so after training the
* stump may be completely different.
*/
template<typename MatType>
double DecisionStump<MatType>::Train(const MatType& data,
const arma::Row<size_t>& labels,
const arma::rowvec& weights,
const size_t numClasses,
const size_t bucketSize)
{
this->numClasses = numClasses;
this->bucketSize = bucketSize;
// Pass to weighted training function.
return Train<true>(data, labels, weights);
}
/**
* Train the decision stump on the given data and labels.
*
* @param data Dataset to train on.
* @param labels Labels for dataset.
* @param UseWeights Whether we need to run a weighted Decision Stump.
*/
template<typename MatType>
template<bool UseWeights>
double DecisionStump<MatType>::Train(const MatType& data,
const arma::Row<size_t>& labels,
const arma::rowvec& weights)
{
// If classLabels are not all identical, proceed with training.
size_t bestDim = 0;
double entropy;
const double rootEntropy = CalculateEntropy<UseWeights>(labels, weights);
double gain, bestGain = 0.0;
for (size_t i = 0; i < data.n_rows; ++i)
{
// Go through each dimension of the data.
if (IsDistinct(data.row(i)))
{
// For each dimension with non-identical values, treat it as a potential
// splitting dimension and calculate entropy if split on it.
entropy = SetupSplitDimension<UseWeights>(data.row(i), labels, weights);
gain = rootEntropy - entropy;
// Find the dimension with the best entropy so that the gain is
// maximized.
// We are maximizing gain, which is what is returned from
// SetupSplitDimension().
if (gain < bestGain)
{
bestDim = i;
bestGain = gain;
}
}
}
splitDimension = bestDim;
// Once the splitting column/dimension has been decided, train on it.
TrainOnDim(data.row(splitDimension), labels);
return -bestGain;
}
/**
* Classification function. After training, classify test, and put the predicted
* classes in predictedLabels.
*
* @param test Testing data or data to classify.
* @param predictedLabels Vector to store the predicted classes after
* classifying test
*/
template<typename MatType>
void DecisionStump<MatType>::Classify(const MatType& test,
arma::Row<size_t>& predictedLabels)
{
predictedLabels.set_size(test.n_cols);
for (size_t i = 0; i < test.n_cols; ++i)
{
// Determine which bin the test point falls into.
// Assume first that it falls into the first bin, then proceed through the
// bins until it is known which bin it falls into.
size_t bin = 0;
const double val = test(splitDimension, i);
while (bin < split.n_elem - 1)
{
if (val < split(bin + 1))
break;
++bin;
}
predictedLabels(i) = binLabels(bin);
}
}
/**
* Alternate constructor which copies parameters bucketSize and numClasses
* from an already initiated decision stump, other. It appropriately
* sets the Weight vector.
*
* @param other The other initiated Decision Stump object from
* which we copy the values from.
* @param data The data on which to train this object on.
* @param D Weight vector to use while training. For boosting purposes.
* @param labels The labels of data.
* @param UseWeights Whether we need to run a weighted Decision Stump.
*/
template<typename MatType>
DecisionStump<MatType>::DecisionStump(const DecisionStump<>& other,
const MatType& data,
const arma::Row<size_t>& labels,
const size_t numClasses,
const arma::rowvec& weights) :
numClasses(numClasses),
bucketSize(other.bucketSize)
{
Train<true>(data, labels, weights);
}
/**
* Serialize the decision stump.
*/
template<typename MatType>
template<typename Archive>
void DecisionStump<MatType>::serialize(Archive& ar,
const uint32_t /* version */)
{
// This is straightforward; just serialize all of the members of the class.
// None need special handling.
ar(CEREAL_NVP(numClasses));
ar(CEREAL_NVP(bucketSize));
ar(CEREAL_NVP(splitDimension));
ar(CEREAL_NVP(split));
ar(CEREAL_NVP(binLabels));
}
/**
* Sets up dimension as if it were splitting on it and finds entropy when
* splitting on dimension.
*
* @param dimension A row from the training data, which might be a candidate for
* the splitting dimension.
* @param UseWeights Whether we need to run a weighted Decision Stump.
*/
template<typename MatType>
template<bool UseWeights, typename VecType>
double DecisionStump<MatType>::SetupSplitDimension(
const VecType& dimension,
const arma::Row<size_t>& labels,
const arma::rowvec& weights)
{
size_t i, count, begin, end;
double entropy = 0.0;
// Store the indices of the sorted dimension to build a vector of sorted
// labels. This sort is stable.
arma::uvec sortedIndexDim = arma::stable_sort_index(dimension.t());
arma::Row<size_t> sortedLabels(dimension.n_elem);
arma::rowvec sortedWeights(dimension.n_elem);
for (i = 0; i < dimension.n_elem; ++i)
{
sortedLabels(i) = labels(sortedIndexDim(i));
// Apply weights if necessary.
if (UseWeights)
sortedWeights(i) = weights(sortedIndexDim(i));
}
i = 0;
count = 0;
// This splits the sorted data into buckets of size greater than or equal to
// bucketSize.
while (i < sortedLabels.n_elem)
{
count++;
if (i == sortedLabels.n_elem - 1)
{
// If we're at the end, then don't worry about the bucket size; just take
// this as the last bin.
begin = i - count + 1;
end = i;
// Use ratioEl to calculate the ratio of elements in this split.
const double ratioEl = ((double) (end - begin + 1) / sortedLabels.n_elem);
entropy += ratioEl * CalculateEntropy<UseWeights>(
sortedLabels.subvec(begin, end), sortedWeights.subvec(begin, end));
++i;
}
else if (sortedLabels(i) != sortedLabels(i + 1))
{
// If we're not at the last element of sortedLabels, then check whether
// count is less than the current bucket size.
if (count < bucketSize)
{
// If it is, then take the minimum bucket size anyways.
// This is where the inpBucketSize comes into use.
// This makes sure there isn't a bucket for every change in labels.
begin = i - count + 1;
end = begin + bucketSize - 1;
if (end > sortedLabels.n_elem - 1)
end = sortedLabels.n_elem - 1;
}
else
{
// If it is not, then take the bucket size as the value of count.
begin = i - count + 1;
end = i;
}
const double ratioEl = ((double) (end - begin + 1) / sortedLabels.n_elem);
entropy += ratioEl * CalculateEntropy<UseWeights>(
sortedLabels.subvec(begin, end), sortedWeights.subvec(begin, end));
i = end + 1;
count = 0;
}
else
++i;
}
return entropy;
}
/**
* After having decided the dimension on which to split, train on that
* dimension.
*
* @param dimension Dimension is the dimension decided by the constructor on
* which we now train the decision stump.
*/
template<typename MatType>
template<typename VecType>
void DecisionStump<MatType>::TrainOnDim(const VecType& dimension,
const arma::Row<size_t>& labels)
{
size_t i, count, begin, end;
typename MatType::row_type sortedSplitDim = arma::sort(dimension);
arma::uvec sortedSplitIndexDim = arma::stable_sort_index(dimension.t());
arma::Row<size_t> sortedLabels(dimension.n_elem);
sortedLabels.fill(0);
for (i = 0; i < dimension.n_elem; ++i)
sortedLabels(i) = labels(sortedSplitIndexDim(i));
arma::rowvec subCols;
double mostFreq;
i = 0;
count = 0;
while (i < sortedLabels.n_elem)
{
count++;
if (i == sortedLabels.n_elem - 1)
{
begin = i - count + 1;
end = i;
mostFreq = CountMostFreq(sortedLabels.cols(begin, end));
split.resize(split.n_elem + 1);
split(split.n_elem - 1) = sortedSplitDim(begin);
binLabels.resize(binLabels.n_elem + 1);
binLabels(binLabels.n_elem - 1) = mostFreq;
++i;
}
else if (sortedLabels(i) != sortedLabels(i + 1))
{
if (count < bucketSize)
{
// Test for different values of bucketSize, especially extreme cases.
begin = i - count + 1;
end = begin + bucketSize - 1;
if (end > sortedLabels.n_elem - 1)
end = sortedLabels.n_elem - 1;
}
else
{
begin = i - count + 1;
end = i;
}
// Find the most frequent element in subCols so as to assign a label to
// the bucket of subCols.
mostFreq = CountMostFreq(sortedLabels.cols(begin, end));
split.resize(split.n_elem + 1);
split(split.n_elem - 1) = sortedSplitDim(begin);
binLabels.resize(binLabels.n_elem + 1);
binLabels(binLabels.n_elem - 1) = mostFreq;
i = end + 1;
count = 0;
}
else
++i;
}
// Now trim the split matrix so that buckets one after the after which point
// to the same classLabel are merged as one big bucket.
MergeRanges();
}
/**
* After the "split" matrix has been set up, merge ranges with identical class
* labels.
*/
template<typename MatType>
void DecisionStump<MatType>::MergeRanges()
{
for (size_t i = 1; i < split.n_rows; ++i)
{
if (binLabels(i) == binLabels(i - 1))
{
// Remove this row, as it has the same label as the previous bucket.
binLabels.shed_row(i);
split.shed_row(i);
// Go back to previous row.
i--;
}
}
}
template<typename MatType>
template<typename VecType>
double DecisionStump<MatType>::CountMostFreq(const VecType& subCols)
{
// We'll create a map of elements and the number of times that each element is
// seen.
std::map<double, size_t> countMap;
for (size_t i = 0; i < subCols.n_elem; ++i)
{
if (countMap.count(subCols[i]) == 0)
countMap[subCols[i]] = 1;
else
++countMap[subCols[i]];
}
// Now find the maximum value.
typename std::map<double, size_t>::iterator it = countMap.begin();
double mostFreq = it->first;
size_t mostFreqCount = it->second;
while (it != countMap.end())
{
if (it->second >= mostFreqCount)
{
mostFreq = it->first;
mostFreqCount = it->second;
}
++it;
}
return mostFreq;
}
/**
* Returns 1 if all the values of featureRow are not the same.
*
* @param featureRow The dimension which is checked for identical values.
*/
template<typename MatType>
template<typename VecType>
int DecisionStump<MatType>::IsDistinct(const VecType& featureRow)
{
typename VecType::elem_type val = featureRow(0);
for (size_t i = 1; i < featureRow.n_elem; ++i)
if (val != featureRow(i))
return 1;
return 0;
}
/**
* Calculate entropy of dimension.
*
* @param labels Corresponding labels of the dimension.
* @param UseWeights Whether we need to run a weighted Decision Stump.
*/
template<typename MatType>
template<bool UseWeights, typename VecType, typename WeightVecType>
double DecisionStump<MatType>::CalculateEntropy(
const VecType& labels,
const WeightVecType& weights)
{
double entropy = 0.0;
size_t j;
arma::rowvec numElem(numClasses);
numElem.fill(0);
// Variable to accumulate the weight in this subview_row.
double accWeight = 0.0;
// Populate numElem; they are used as helpers to calculate entropy.
if (UseWeights)
{
for (j = 0; j < labels.n_elem; ++j)
{
numElem(labels(j)) += weights(j);
accWeight += weights(j);
}
for (j = 0; j < numClasses; ++j)
{
const double p1 = ((double) numElem(j) / accWeight);
// Instead of using log2(), which is C99 and may not exist on some
// compilers, use std::log(), then use the change-of-base formula to make
// the result correct.
entropy += (p1 == 0) ? 0 : p1 * std::log(p1);
}
}
else
{
for (j = 0; j < labels.n_elem; ++j)
numElem(labels(j))++;
for (j = 0; j < numClasses; ++j)
{
const double p1 = ((double) numElem(j) / labels.n_elem);
// Instead of using log2(), which is C99 and may not exist on some
// compilers, use std::log(), then use the change-of-base formula to make
// the result correct.
entropy += (p1 == 0) ? 0 : p1 * std::log(p1);
}
}
return entropy / std::log(2.0);
}
} // namespace decision_stump
} // namespace mlpack
#endif
src/mlpack/methods/decision_stump/decision_stump_main.cpp
-209
View File
@@ -1,209 +0,0 @@
/**
* @file methods/decision_stump/decision_stump_main.cpp
* @author Udit Saxena
*
* Main executable for the decision stump.
*
* mlpack is free software; you may redistribute it and/or modify it under the
* terms of the 3-clause BSD license. You should have received a copy of the
* 3-clause BSD license along with mlpack. If not, see
* http://www.opensource.org/licenses/BSD-3-Clause for more information.
*/
#include <mlpack/prereqs.hpp>
#include <mlpack/core/util/io.hpp>
#include <mlpack/core/data/normalize_labels.hpp>
#include <mlpack/core/util/mlpack_main.hpp>
#include "decision_stump.hpp"
using namespace mlpack;
using namespace mlpack::decision_stump;
using namespace mlpack::util;
using namespace std;
using namespace arma;
// Program Name.
BINDING_NAME("Decision Stump");
// Short description.
BINDING_SHORT_DESC(
"An implementation of a decision stump, which is a single-level decision "
"tree. Given labeled data, a new decision stump can be trained; or, an "
"existing decision stump can be used to classify points.");
// Long description.
BINDING_LONG_DESC(
"This program implements a decision stump, which is a single-level decision"
" tree. The decision stump will split on one dimension of the input data, "
"and will split into multiple buckets. The dimension and bins are selected"
" by maximizing the information gain of the split. Optionally, the minimum"
" number of training points in each bin can be specified with the " +
PRINT_PARAM_STRING("bucket_size") + " parameter."
"\n\n"
"The decision stump is parameterized by a splitting dimension and a vector "
"of values that denote the splitting values of each bin."
"\n\n"
"This program enables several applications: a decision tree may be trained "
"or loaded, and then that decision tree may be used to classify a given set"
" of test points. The decision tree may also be saved to a file for later "
"usage."
"\n\n"
"To train a decision stump, training data should be passed with the " +
PRINT_PARAM_STRING("training") + " parameter, and their corresponding "
"labels should be passed with the " + PRINT_PARAM_STRING("labels") + " "
"option. Optionally, if " + PRINT_PARAM_STRING("labels") + " is not "
"specified, the labels are assumed to be the last dimension of the "
"training dataset. The " + PRINT_PARAM_STRING("bucket_size") + " "
"parameter controls the minimum number of training points in each decision "
"stump bucket."
"\n\n"
"For classifying a test set, a decision stump may be loaded with the " +
PRINT_PARAM_STRING("input_model") + " parameter (useful for the situation "
"where a stump has already been trained), and a test set may be specified "
"with the " + PRINT_PARAM_STRING("test") + " parameter. The predicted "
"labels can be saved with the " + PRINT_PARAM_STRING("predictions") + " "
"output parameter."
"\n\n"
"Because decision stumps are trained in batch, retraining does not make "
"sense and thus it is not possible to pass both " +
PRINT_PARAM_STRING("training") + " and " +
PRINT_PARAM_STRING("input_model") + "; instead, simply build a new "
"decision stump with the training data."
"\n\n"
"After training, a decision stump can be saved with the " +
PRINT_PARAM_STRING("output_model") + " output parameter. That stump may "
"later be re-used in subsequent calls to this program (or others).");
// See also...
BINDING_SEE_ALSO("Decision tree", "#decision_tree");
BINDING_SEE_ALSO("Decision stumps on Wikipedia",
"https://en.wikipedia.org/wiki/Decision_stump");
BINDING_SEE_ALSO("mlpack::decision_stump::DecisionStump class documentation",
"@doxygen/classmlpack_1_1decision__stump_1_1DecisionStump.html");
// Datasets we might load.
PARAM_MATRIX_IN("training", "The dataset to train on.", "t");
PARAM_UROW_IN("labels", "Labels for the training set. If not specified, the "
"labels are assumed to be the last row of the training data.", "l");
PARAM_MATRIX_IN("test", "A dataset to calculate predictions for.", "T");
// Output.
PARAM_UROW_OUT("predictions", "The output matrix that will hold the "
"predicted labels for the test set.", "p");
/**
* This is the structure that actually saves to disk. We have to save the
* label mappings, too, otherwise everything we load at test time in a future
* run will end up being borked.
*/
struct DSModel
{
//! The mappings.
arma::Col<size_t> mappings;
//! The stump.
DecisionStump<> stump;
//! Serialize the model.
template<typename Archive>
void serialize(Archive& ar, const uint32_t /* version */)
{
ar(CEREAL_NVP(mappings));
ar(CEREAL_NVP(stump));
}
};
// We may load or save a model.
PARAM_MODEL_IN(DSModel, "input_model", "Decision stump model to "
"load.", "m");
PARAM_MODEL_OUT(DSModel, "output_model", "Output decision stump model to save.",
"M");
PARAM_INT_IN("bucket_size", "The minimum number of training points in each "
"decision stump bucket.", "b", 6);
static void mlpackMain()
{
// Check that the parameters are reasonable.
RequireOnlyOnePassed({ "training", "input_model" }, true);
RequireAtLeastOnePassed({ "output_model", "predictions" }, false, "no results"
" will be saved");
RequireParamValue<int>("bucket_size", [](int x) { return x > 0; }, true,
"bucket size must be positive");
ReportIgnoredParam({{ "test", false }}, "predictions");
Log::Warn << "DecisionStump is deprecated and will be removed in mlpack "
<< "4.0.0. Please use DecisionTree instead with the maximum tree "
<< "depth option set to 1 (that will produce a stump)."
<< std::endl;
// We must either load a model, or train a new stump.
DSModel* model;
if (IO::HasParam("training"))
{
model = new DSModel();
mat trainingData = std::move(IO::GetParam<mat>("training"));
// Load labels, if necessary.
Row<size_t> labelsIn;
if (IO::HasParam("labels"))
{
labelsIn = std::move(IO::GetParam<Row<size_t>>("labels"));
}
else
{
// Extract the labels as the last
Log::Info << "Using the last dimension of training set as labels."
<< endl;
labelsIn = arma::conv_to<arma::Row<size_t>>::from(
trainingData.row(trainingData.n_rows - 1));
trainingData.shed_row(trainingData.n_rows - 1);
}
// Normalize the labels.
Row<size_t> labels;
data::NormalizeLabels(labelsIn, labels, model->mappings);
const size_t bucketSize = IO::GetParam<int>("bucket_size");
const size_t classes = labels.max() + 1;
Timer::Start("training");
model->stump.Train(trainingData, labels, classes, bucketSize);
Timer::Stop("training");
}
else
{
model = IO::GetParam<DSModel*>("input_model");
}
// Now, do we need to do any testing?
if (IO::HasParam("test"))
{
// Load the test file.
mat testingData = std::move(IO::GetParam<arma::mat>("test"));
if (testingData.n_rows <= model->stump.SplitDimension())
Log::Fatal << "Test data dimensionality (" << testingData.n_rows << ") "
<< "is too low; the trained stump requires at least "
<< model->stump.SplitDimension() << " dimensions!" << endl;
Row<size_t> predictedLabels(testingData.n_cols);
Timer::Start("testing");
model->stump.Classify(testingData, predictedLabels);
Timer::Stop("testing");
// Denormalize predicted labels, if we want to save them.
if (IO::HasParam("predictions"))
{
Row<size_t> actualLabels;
data::RevertLabels(predictedLabels, model->mappings, actualLabels);
// Save the predicted labels as output.
IO::GetParam<Row<size_t>>("predictions") = std::move(actualLabels);
}
}
// Save the model, if desired.
IO::GetParam<DSModel*>("output_model") = model;
}
src/mlpack/tests/CMakeLists.txt
-2
View File
@@ -26,7 +26,6 @@ add_executable(mlpack_test
cv_test.cpp
dbscan_test.cpp
dcgan_test.cpp
decision_stump_test.cpp
decision_tree_test.cpp
det_test.cpp
distribution_test.cpp
@@ -129,7 +128,6 @@ add_executable(mlpack_test
main_tests/bayesian_linear_regression_test.cpp
main_tests/cf_test.cpp
main_tests/dbscan_test.cpp
main_tests/decision_stump_test.cpp
main_tests/decision_tree_test.cpp
main_tests/det_test.cpp
main_tests/emst_test.cpp
src/mlpack/tests/decision_stump_test.cpp
-412
View File
@@ -1,412 +0,0 @@
/**
* @file tests/decision_stump_test.cpp
* @author Udit Saxena
*
* Tests for DecisionStump class.
*
* mlpack is free software; you may redistribute it and/or modify it under the
* terms of the 3-clause BSD license. You should have received a copy of the
* 3-clause BSD license along with mlpack. If not, see
* http://www.opensource.org/licenses/BSD-3-Clause for more information.
*/
#include <mlpack/core.hpp>
#include <mlpack/methods/decision_stump/decision_stump.hpp>
#include "catch.hpp"
using namespace mlpack;
using namespace mlpack::decision_stump;
using namespace arma;
using namespace mlpack::distribution;
/**
* This tests handles the case wherein only one class exists in the input
* labels. It checks whether the only class supplied was the only class
* predicted.
*/
TEST_CASE("OneClass", "[DecisionStumpTest]")
{
const size_t numClasses = 2;
const size_t inpBucketSize = 6;
mat trainingData;
trainingData = { { 2.4, 3.8, 3.8 },
{ 1, 1, 2 },
{ 1.3, 1.9, 1.3 } };
// No need to normalize labels here.
Mat<size_t> labelsIn;
labelsIn = { 1, 1, 1 };
mat testingData;
testingData = { 2.4, 2.5, 2.6 };
DecisionStump<> ds(trainingData, labelsIn.row(0), numClasses, inpBucketSize);
Row<size_t> predictedLabels;
ds.Classify(testingData, predictedLabels);
for (size_t i = 0; i < predictedLabels.size(); ++i)
REQUIRE(predictedLabels(i) == 1);
}
/**
* This tests whether the entropy is being correctly calculated by checking the
* correct value of the splitting column value. This test is for an
* inpBucketSize of 4 and the correct value of the splitting dimension is 0.
*/
TEST_CASE("CorrectDimensionChosen", "[DecisionStumpTest]")
{
const size_t numClasses = 2;
const size_t inpBucketSize = 4;
// This dataset comes from Chapter 6 of the book "Data Mining: Concepts,
// Models, Methods, and Algorithms" (2nd Edition) by Mehmed Kantardzic. It is
// found on page 176 (and a description of the correct splitting dimension is
// given below that).
mat trainingData;
trainingData = { { 0, 0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 2, 2 },
{ 70, 90, 85, 95, 70, 90, 78, 65, 75, 80, 70, 80, 80, 96 },
{ 1, 1, 0, 0, 0, 1, 0, 1, 0, 1, 1, 0, 0, 0 } };
// No need to normalize labels here.
Mat<size_t> labelsIn;
labelsIn = { 0, 1, 1, 1, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0 };
DecisionStump<> ds(trainingData, labelsIn.row(0), numClasses, inpBucketSize);
// Only need to check the value of the splitting column, no need of
// classification.
REQUIRE(ds.SplitDimension() == 0);
}
/**
* This tests for the classification:
* if testinput < 0 - class 0
* if testinput > 0 - class 1
* An almost perfect split on zero.
*/
TEST_CASE("PerfectSplitOnZero", "[DecisionStumpTest]")
{
const size_t numClasses = 2;
const size_t inpBucketSize = 2;
mat trainingData;
trainingData = { -1, 1, -2, 2, -3, 3 };
// No need to normalize labels here.
Mat<size_t> labelsIn;
labelsIn = { 0, 1, 0, 1, 0, 1 };
mat testingData;
testingData = { -4, 7, -7, -5, 6 };
DecisionStump<> ds(trainingData, labelsIn.row(0), numClasses, inpBucketSize);
Row<size_t> predictedLabels;
ds.Classify(testingData, predictedLabels);
REQUIRE(predictedLabels(0, 0) == 0);
REQUIRE(predictedLabels(0, 1) == 1);
REQUIRE(predictedLabels(0, 2) == 0);
REQUIRE(predictedLabels(0, 3) == 0);
REQUIRE(predictedLabels(0, 4) == 1);
}
/**
* This tests the binning function for the case when a dataset with cardinality
* of input < inpBucketSize is provided.
*/
TEST_CASE("BinningTesting", "[DecisionStumpTest]")
{
const size_t numClasses = 2;
const size_t inpBucketSize = 10;
mat trainingData;
trainingData = { -1, 1, -2, 2, -3, 3, -4 };
// No need to normalize labels here.
Mat<size_t> labelsIn;
labelsIn = { 0, 1, 0, 1, 0, 1, 0 };
mat testingData;
testingData = {5};
DecisionStump<> ds(trainingData, labelsIn.row(0), numClasses, inpBucketSize);
Row<size_t> predictedLabels;
ds.Classify(testingData, predictedLabels);
REQUIRE(predictedLabels(0, 0) == 0);
}
/**
* This is a test for the case when non-overlapping, multiple classes are
* provided. It tests for a perfect split due to the non-overlapping nature of
* the input classes.
*/
TEST_CASE("PerfectMultiClassSplit", "[DecisionStumpTest]")
{
const size_t numClasses = 4;
const size_t inpBucketSize = 3;
mat trainingData;
trainingData = { -8, -7, -6, -5, -4, -3, -2, -1, 0, 1, 2, 3, 4, 5, 6, 7 };
// No need to normalize labels here.
Mat<size_t> labelsIn;
labelsIn = { 0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3 };
mat testingData;
testingData = { -6.1, -2.1, 1.1, 5.1 };
DecisionStump<> ds(trainingData, labelsIn.row(0), numClasses, inpBucketSize);
Row<size_t> predictedLabels;
ds.Classify(testingData, predictedLabels);
REQUIRE(predictedLabels(0, 0) == 0);
REQUIRE(predictedLabels(0, 1) == 1);
REQUIRE(predictedLabels(0, 2) == 2);
REQUIRE(predictedLabels(0, 3) == 3);
}
/**
* This test is for the case when reasonably overlapping, multiple classes are
* provided in the input label set. It tests whether classification takes place
* with a reasonable amount of error due to the overlapping nature of input
* classes.
*/
TEST_CASE("MultiClassSplit", "[DecisionStumpTest]")
{
const size_t numClasses = 3;
const size_t inpBucketSize = 3;
mat trainingData;
trainingData = { -7, -6, -5, -4, -3, -2, -1, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9,
10 };
// No need to normalize labels here.
Mat<size_t> labelsIn;
labelsIn = { 0, 0, 0, 0, 1, 1, 0, 0, 1, 1, 1, 2, 1, 2, 2, 2, 2, 2 };
mat testingData;
testingData = { -6.1, -5.9, -2.1, -0.7, 2.5, 4.7, 7.2, 9.1 };
DecisionStump<> ds(trainingData, labelsIn.row(0), numClasses, inpBucketSize);
Row<size_t> predictedLabels;
ds.Classify(testingData, predictedLabels);
REQUIRE(predictedLabels(0, 0) == 0);
REQUIRE(predictedLabels(0, 1) == 0);
REQUIRE(predictedLabels(0, 2) == 1);
REQUIRE(predictedLabels(0, 3) == 1);
REQUIRE(predictedLabels(0, 4) == 1);
REQUIRE(predictedLabels(0, 5) == 1);
REQUIRE(predictedLabels(0, 6) == 2);
REQUIRE(predictedLabels(0, 7) == 2);
}
/**
* This tests that the decision stump can learn a good split on a dataset with
* four dimensions that have progressing levels of separation.
*/
TEST_CASE("DimensionSelectionTest", "[DecisionStumpTest]")
{
const size_t numClasses = 2;
const size_t inpBucketSize = 2500;
arma::mat dataset(4, 5000);
// The most separable dimension.
GaussianDistribution g1("-5", "1");
GaussianDistribution g2("5", "1");
for (size_t i = 0; i < 2500; ++i)
{
arma::vec tmp = g1.Random();
dataset(1, i) = tmp[0];
}
for (size_t i = 2500; i < 5000; ++i)
{
arma::vec tmp = g2.Random();
dataset(1, i) = tmp[0];
}
g1 = GaussianDistribution("-3", "1");
g2 = GaussianDistribution("3", "1");
for (size_t i = 0; i < 2500; ++i)
{
arma::vec tmp = g1.Random();
dataset(3, i) = tmp[0];
}
for (size_t i = 2500; i < 5000; ++i)
{
arma::vec tmp = g2.Random();
dataset(3, i) = tmp[0];
}
g1 = GaussianDistribution("-1", "1");
g2 = GaussianDistribution("1", "1");
for (size_t i = 0; i < 2500; ++i)
{
arma::vec tmp = g1.Random();
dataset(0, i) = tmp[0];
}
for (size_t i = 2500; i < 5000; ++i)
{
arma::vec tmp = g2.Random();
dataset(0, i) = tmp[0];
}
// Not separable at all.
g1 = GaussianDistribution("0", "1");
g2 = GaussianDistribution("0", "1");
for (size_t i = 0; i < 2500; ++i)
{
arma::vec tmp = g1.Random();
dataset(2, i) = tmp[0];
}
for (size_t i = 2500; i < 5000; ++i)
{
arma::vec tmp = g2.Random();
dataset(2, i) = tmp[0];
}
// Generate the labels.
arma::Row<size_t> labels(5000);
for (size_t i = 0; i < 2500; ++i)
labels[i] = 0;
for (size_t i = 2500; i < 5000; ++i)
labels[i] = 1;
// Now create a decision stump.
DecisionStump<> ds(dataset, labels, numClasses, inpBucketSize);
// Make sure it split on the dimension that is most separable.
REQUIRE(ds.SplitDimension() == 1);
// Make sure every bin below -1 classifies as label 0, and every bin above 1
// classifies as label 1 (What happens in [-1, 1] isn't that big a deal.).
for (size_t i = 0; i < ds.Split().n_elem; ++i)
{
if (ds.Split()[i] <= -3.0)
REQUIRE(ds.BinLabels()[i] == 0);
else if (ds.Split()[i] >= 3.0)
REQUIRE(ds.BinLabels()[i] == 1);
}
}
/**
* Ensure that the default constructor works and that it classifies things as 0
* always.
*/
TEST_CASE("EmptyConstructorTest", "[DecisionStumpTest]")
{
DecisionStump<> d;
arma::mat data = arma::randu<arma::mat>(3, 10);
arma::Row<size_t> labels;
d.Classify(data, labels);
for (size_t i = 0; i < 10; ++i)
REQUIRE(labels[i] == 0);
// Now train on another dataset and make sure something kind of makes sense.
mat trainingData;
trainingData = { -7, -6, -5, -4, -3, -2, -1, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9,
10 };
// No need to normalize labels here.
Mat<size_t> labelsIn;
labelsIn = { 0, 0, 0, 0, 1, 1, 0, 0, 1, 1, 1, 2, 1, 2, 2, 2, 2, 2 };
mat testingData;
testingData = { -6.1, -5.9, -2.1, -0.7, 2.5, 4.7, 7.2, 9.1 };
DecisionStump<> ds(trainingData, labelsIn.row(0), 4, 3);
Row<size_t> predictedLabels(testingData.n_cols);
ds.Classify(testingData, predictedLabels);
REQUIRE(predictedLabels(0, 0) == 0);
REQUIRE(predictedLabels(0, 1) == 0);
REQUIRE(predictedLabels(0, 2) == 1);
REQUIRE(predictedLabels(0, 3) == 1);
REQUIRE(predictedLabels(0, 4) == 1);
REQUIRE(predictedLabels(0, 5) == 1);
REQUIRE(predictedLabels(0, 6) == 2);
REQUIRE(predictedLabels(0, 7) == 2);
}
/**
* Ensure that a matrix holding ints can be trained. The bigger issue here is
* just compilation.
*/
TEST_CASE("IntTest", "[DecisionStumpTest]")
{
// Train on a dataset and make sure something kind of makes sense.
imat trainingData;
trainingData = { -7, -6, -5, -4, -3, -2, -1, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9,
10 };
// No need to normalize labels here.
Mat<size_t> labelsIn;
labelsIn = { 0, 0, 0, 0, 1, 1, 0, 0, 1, 1, 1, 2, 1, 2, 2, 2, 2, 2 };
DecisionStump<arma::imat> ds(trainingData, labelsIn.row(0), 4, 3);
imat testingData;
testingData = { -6, -6, -2, -1, 3, 5, 7, 9 };
arma::Row<size_t> predictedLabels;
ds.Classify(testingData, predictedLabels);
REQUIRE(predictedLabels(0, 0) == 0);
REQUIRE(predictedLabels(0, 1) == 0);
REQUIRE(predictedLabels(0, 2) == 1);
REQUIRE(predictedLabels(0, 3) == 1);
REQUIRE(predictedLabels(0, 4) == 1);
REQUIRE(predictedLabels(0, 5) == 1);
REQUIRE(predictedLabels(0, 6) == 2);
REQUIRE(predictedLabels(0, 7) == 2);
}
/**
* Test that DecisionStump::Train() returns finite gain.
*/
TEST_CASE("DecisionStumpTrainReturnEntropy", "[DecisionStumpTest]")
{
const size_t numClasses = 2;
const size_t inpBucketSize = 2;
mat trainingData;
trainingData = { -1, 1, -2, 2, -3, 3 };
// No need to normalize labels here.
Mat<size_t> labelsIn;
labelsIn = { 0, 1, 0, 1, 0, 1 };
arma::Row<double> weights = arma::ones<arma::Row<double>>(labelsIn.n_elem);
// Train a simple decision stump without weights.
DecisionStump<> ds;
double gain = ds.Train(trainingData, labelsIn.row(0), numClasses,
inpBucketSize);
REQUIRE(std::isfinite(gain) == true);
// Train decision stump with weights.
DecisionStump<> wds;
gain = wds.Train(trainingData, labelsIn.row(0), weights, numClasses,
inpBucketSize);
REQUIRE(std::isfinite(gain) == true);
}
src/mlpack/tests/decision_tree_test.cpp
-22
View File
@@ -773,28 +773,6 @@ TEST_CASE("CategoricalBuildTestWithWeight", "[DecisionTreeTest]")
REQUIRE(correctPct > 0.70);
}
/**
* Make sure that when we ask for a decision stump, we get one.
*/
TEST_CASE("DTDecisionStumpTest", "[DecisionTreeTest]")
{
// Use a random dataset.
arma::mat dataset(10, 1000, arma::fill::randu);
arma::Row<size_t> labels(1000);
for (size_t i = 0; i < 1000; ++i)
labels[i] = i % 3; // 3 classes.
// Build a decision stump.
DecisionTree<GiniGain, BestBinaryNumericSplit, AllCategoricalSplit,
AllDimensionSelect, double, true> stump(dataset, labels, 3, 1);
// Check that it has children.
REQUIRE(stump.NumChildren() == 2);
// Check that its children doesn't have children.
REQUIRE(stump.Child(0).NumChildren() == 0);
REQUIRE(stump.Child(1).NumChildren() == 0);
}
/**
* Test that we can build a decision tree using weighted data (where the
* low-weighted data is random noise), and that the tree still builds correctly
src/mlpack/tests/serialization_test.cpp
-38
View File
@@ -36,7 +36,6 @@
#include <mlpack/methods/naive_bayes/naive_bayes_classifier.hpp>
#include <mlpack/methods/rann/ra_search.hpp>
#include <mlpack/methods/lsh/lsh_search.hpp>
#include <mlpack/methods/decision_stump/decision_stump.hpp>
#include <mlpack/methods/lars/lars.hpp>
#include <mlpack/methods/bayesian_linear_regression/bayesian_linear_regression.hpp>
#include <mlpack/methods/ann/rbm/rbm.hpp>
@@ -52,7 +51,6 @@ using namespace mlpack::perceptron;
using namespace mlpack::regression;
using namespace mlpack::naive_bayes;
using namespace mlpack::neighbor;
using namespace mlpack::decision_stump;
using namespace mlpack::ann;
using namespace arma;
@@ -1087,42 +1085,6 @@ TEST_CASE("LSHTest", "[SerializationTest]")
jsonLsh.SecondHashTable()[i], binaryLsh.SecondHashTable()[i]);
}
// Make sure serialization works for the decision stump.
TEST_CASE("DecisionStumpTest", "[SerializationTest]")
{
// Generate dataset.
arma::mat trainingData = arma::randu<arma::mat>(4, 100);
arma::Row<size_t> labels(100);
for (size_t i = 0; i < 25; ++i)
labels[i] = 0;
for (size_t i = 25; i < 50; ++i)
labels[i] = 3;
for (size_t i = 50; i < 75; ++i)
labels[i] = 1;
for (size_t i = 75; i < 100; ++i)
labels[i] = 2;
DecisionStump<> ds(trainingData, labels, 4, 3);
arma::mat otherData = arma::randu<arma::mat>(3, 100);
arma::Row<size_t> otherLabels = arma::randu<arma::Row<size_t>>(100);
DecisionStump<> xmlDs(otherData, otherLabels, 2, 3);
DecisionStump<> jsonDs;
DecisionStump<> binaryDs(trainingData, labels, 4, 10);
SerializeObjectAll(ds, xmlDs, jsonDs, binaryDs);
// Make sure that everything is the same about the new decision stumps.
REQUIRE(ds.SplitDimension() == xmlDs.SplitDimension());
REQUIRE(ds.SplitDimension() == jsonDs.SplitDimension());
REQUIRE(ds.SplitDimension() == binaryDs.SplitDimension());
CheckMatrices(ds.Split(), xmlDs.Split(), jsonDs.Split(), binaryDs.Split());
CheckMatrices(ds.BinLabels(), xmlDs.BinLabels(), jsonDs.BinLabels(),
binaryDs.BinLabels());
}
// Make sure serialization works for LARS.
TEST_CASE("LARSTest", "[SerializationTest]")
{