From 3812017ba9df058f662b0604b74a504d2be77975 Mon Sep 17 00:00:00 2001 From: Bisakh Mondal Date: Sun, 8 Mar 2020 15:13:23 +0530 Subject: [PATCH 1/9] DStump package removed --- .../methods/decision_stump/CMakeLists.txt | 20 - .../methods/decision_stump/decision_stump.hpp | 237 -------- .../decision_stump/decision_stump_impl.hpp | 518 ------------------ .../decision_stump/decision_stump_main.cpp | 202 ------- 4 files changed, 977 deletions(-) delete mode 100644 src/mlpack/methods/decision_stump/CMakeLists.txt delete mode 100644 src/mlpack/methods/decision_stump/decision_stump.hpp delete mode 100644 src/mlpack/methods/decision_stump/decision_stump_impl.hpp delete mode 100644 src/mlpack/methods/decision_stump/decision_stump_main.cpp diff --git a/src/mlpack/methods/decision_stump/CMakeLists.txt b/src/mlpack/methods/decision_stump/CMakeLists.txt deleted file mode 100644 index 40a1d198ce..0000000000 --- a/src/mlpack/methods/decision_stump/CMakeLists.txt +++ /dev/null @@ -1,20 +0,0 @@ -# Define the files we need to compile. -# Anything not in this list will not be compiled into mlpack. -set(SOURCES - decision_stump.hpp - decision_stump_impl.hpp -) - -# Add directory name to sources. -set(DIR_SRCS) -foreach(file ${SOURCES}) - set(DIR_SRCS ${DIR_SRCS} ${CMAKE_CURRENT_SOURCE_DIR}/${file}) -endforeach() -# Append sources (with directory name) to list of all mlpack sources (used at -# the parent scope). -set(MLPACK_SRCS ${MLPACK_SRCS} ${DIR_SRCS} PARENT_SCOPE) - -add_cli_executable(decision_stump) -add_python_binding(decision_stump) -add_julia_binding(decision_stump) -add_markdown_docs(decision_stump "cli;python;julia" "classification") diff --git a/src/mlpack/methods/decision_stump/decision_stump.hpp b/src/mlpack/methods/decision_stump/decision_stump.hpp deleted file mode 100644 index e8863908ab..0000000000 --- a/src/mlpack/methods/decision_stump/decision_stump.hpp +++ /dev/null @@ -1,237 +0,0 @@ -/** - * @file decision_stump.hpp - * @author Udit Saxena - * - * Definition of decision stumps. - * - * mlpack is free software; you may redistribute it and/or modify it under the - * terms of the 3-clause BSD license. You should have received a copy of the - * 3-clause BSD license along with mlpack. If not, see - * http://www.opensource.org/licenses/BSD-3-Clause for more information. - */ -#ifndef MLPACK_METHODS_DECISION_STUMP_DECISION_STUMP_HPP -#define MLPACK_METHODS_DECISION_STUMP_DECISION_STUMP_HPP - -#include - -namespace mlpack { -namespace decision_stump { - -/** - * This class implements a decision stump. It constructs a single level - * decision tree, i.e., a decision stump. It uses entropy to decide splitting - * ranges. - * - * The stump is parameterized by a splitting dimension (the dimension on which - * points are split), a vector of bin split values, and a vector of labels for - * each bin. Bin i is specified by the range [split[i], split[i + 1]). The - * last bin has range up to \infty (split[i + 1] does not exist in that case). - * Points that are below the first bin will take the label of the first bin. - * - * @note - * This class has been deprecated and should be removed in mlpack 4.0.0. Use - * `ID3DecisionStump`, found in src/mlpack/methods/decision_tree/, instead. - * - * @tparam MatType Type of matrix that is being used (sparse or dense). - */ -template -class DecisionStump -{ - public: - /** - * Constructor. Train on the provided data. Generate a decision stump from - * data. - * - * @param data Input, training data. - * @param labels Labels of training data. - * @param numClasses Number of distinct classes in labels. - * @param bucketSize Minimum size of bucket when splitting. - */ - mlpack_deprecated DecisionStump(const MatType& data, - const arma::Row& labels, - const size_t numClasses, - const size_t bucketSize = 10); - - /** - * Alternate constructor which copies the parameters bucketSize and classes - * from an already initiated decision stump, other. It appropriately sets the - * weight vector. - * - * @param other The other initiated Decision Stump object from - * which we copy the values. - * @param data The data on which to train this object on. - * @param labels The labels of data. - * @param weights Weight vector to use while training. For boosting purposes. - */ - mlpack_deprecated DecisionStump(const DecisionStump<>& other, - const MatType& data, - const arma::Row& labels, - const size_t numClasses, - const arma::rowvec& weights); - - /** - * Create a decision stump without training. This stump will not be useful - * and will always return a class of 0 for anything that is to be classified, - * so it would be a prudent idea to call Train() after using this constructor. - */ - DecisionStump(); - - /** - * Train the decision stump on the given data. This completely overwrites any - * previous training data, so after training the stump may be completely - * different. - * - * @param data Dataset to train on. - * @param labels Labels for each point in the dataset. - * @param numClasses Number of classes in the dataset. - * @param bucketSize Minimum size of bucket when splitting. - * @return The final entropy after splitting. - */ - mlpack_deprecated double Train(const MatType& data, - const arma::Row& labels, - const size_t numClasses, - const size_t bucketSize); - - /** - * Train the decision stump on the given data, with the given weights. This - * completely overwrites any previous training data, so after training the - * stump may be completely different. - * - * @param data Dataset to train on. - * @param labels Labels for each point in the dataset. - * @param weights Weights for each point in the dataset. - * @param numClasses Number of classes in the dataset. - * @param bucketSize Minimum size of bucket when splitting. - * @return The final entropy after splitting. - */ - mlpack_deprecated double Train(const MatType& data, - const arma::Row& labels, - const arma::rowvec& weights, - const size_t numClasses, - const size_t bucketSize); - - /** - * Classification function. After training, classify test, and put the - * predicted classes in predictedLabels. - * - * @param test Testing data or data to classify. - * @param predictedLabels Vector to store the predicted classes after - * classifying test data. - */ - mlpack_deprecated void Classify(const MatType& test, - arma::Row& predictedLabels); - - //! Access the splitting dimension. - size_t SplitDimension() const { return splitDimension; } - //! Modify the splitting dimension (be careful!). - size_t& SplitDimension() { return splitDimension; } - - //! Access the splitting values. - const arma::vec& Split() const { return split; } - //! Modify the splitting values (be careful!). - arma::vec& Split() { return split; } - - //! Access the labels for each split bin. - const arma::Col BinLabels() const { return binLabels; } - //! Modify the labels for each split bin (be careful!). - arma::Col& BinLabels() { return binLabels; } - - //! Serialize the decision stump. - template - void serialize(Archive& ar, const unsigned int /* version */); - - private: - //! The number of classes (we must store this for boosting). - size_t numClasses; - //! The minimum number of points in a bucket. - size_t bucketSize; - - //! Stores the value of the dimension on which to split. - size_t splitDimension; - //! Stores the splitting values after training. - arma::vec split; - //! Stores the labels for each splitting bin. - arma::Col binLabels; - - /** - * Sets up dimension as if it were splitting on it and finds entropy when - * splitting on dimension. - * - * @param dimension A row from the training data, which might be a - * candidate for the splitting dimension. - * @tparam UseWeights Whether we need to run a weighted Decision Stump. - */ - template - double SetupSplitDimension(const VecType& dimension, - const arma::Row& labels, - const arma::rowvec& weightD); - - /** - * After having decided the dimension on which to split, train on that - * dimension. - * - * @tparam dimension dimension is the dimension decided by the constructor - * on which we now train the decision stump. - */ - template - void TrainOnDim(const VecType& dimension, - const arma::Row& labels); - - /** - * After the "split" matrix has been set up, merge ranges with identical class - * labels. - */ - void MergeRanges(); - - /** - * Count the most frequently occurring element in subCols. - * - * @param subCols The vector in which to find the most frequently occurring - * element. - */ - template - double CountMostFreq(const VecType& subCols); - - /** - * Returns 1 if all the values of featureRow are not same. - * - * @param featureRow The dimension which is checked for identical values. - */ - template - int IsDistinct(const VecType& featureRow); - - /** - * Calculate the entropy of the given dimension. - * - * @param labels Corresponding labels of the dimension. - * @param classes Number of classes. - * @param weights Weights for this set of labels. - * @tparam UseWeights If true, the weights in the weight vector will be used - * (otherwise they are ignored). - */ - template - double CalculateEntropy(const VecType& labels, - const WeightVecType& weights); - - /** - * Train the decision stump on the given data and labels. - * - * @param data Dataset to train on. - * @param labels Labels for dataset. - * @param weights Weights for this set of labels. - * @tparam UseWeights If true, the weights in the weight vector will be used - * (otherwise they are ignored). - * @return The final entropy after splitting. - */ - template - double Train(const MatType& data, - const arma::Row& labels, - const arma::rowvec& weights); -}; - -} // namespace decision_stump -} // namespace mlpack - -#include "decision_stump_impl.hpp" - -#endif diff --git a/src/mlpack/methods/decision_stump/decision_stump_impl.hpp b/src/mlpack/methods/decision_stump/decision_stump_impl.hpp deleted file mode 100644 index 73e324e2e1..0000000000 --- a/src/mlpack/methods/decision_stump/decision_stump_impl.hpp +++ /dev/null @@ -1,518 +0,0 @@ -/** - * @file decision_stump_impl.hpp - * @author Udit Saxena - * - * Implementation of DecisionStump class. - * - * mlpack is free software; you may redistribute it and/or modify it under the - * terms of the 3-clause BSD license. You should have received a copy of the - * 3-clause BSD license along with mlpack. If not, see - * http://www.opensource.org/licenses/BSD-3-Clause for more information. - */ -#ifndef MLPACK_METHODS_DECISION_STUMP_DECISION_STUMP_IMPL_HPP -#define MLPACK_METHODS_DECISION_STUMP_DECISION_STUMP_IMPL_HPP - -// In case it hasn't been included yet. -#include "decision_stump.hpp" - -namespace mlpack { -namespace decision_stump { - -/** - * Constructor. Train on the provided data. Generate a decision stump from data. - * - * @param data Input, training data. - * @param labels Labels of data. - * @param numClasses Number of distinct classes in labels. - * @param bucketSize Minimum size of bucket when splitting. - */ -template -DecisionStump::DecisionStump(const MatType& data, - const arma::Row& labels, - const size_t numClasses, - const size_t bucketSize) : - numClasses(numClasses), - bucketSize(bucketSize) -{ - arma::rowvec weights; - Train(data, labels, weights); -} - -/** - * Empty constructor. - */ -template -DecisionStump::DecisionStump() : - numClasses(1), - bucketSize(0), - splitDimension(0), - split(1), - binLabels(1) -{ - split[0] = DBL_MAX; - binLabels[0] = 0; -} - -/** - * Train on the given data and labels. - */ -template -double DecisionStump::Train(const MatType& data, - const arma::Row& labels, - const size_t numClasses, - const size_t bucketSize) -{ - this->numClasses = numClasses; - this->bucketSize = bucketSize; - - // Pass to unweighted training function. - arma::rowvec weights; - return Train(data, labels, weights); -} - -/** - * Train the decision stump on the given data, with the given weights. This - * completely overwrites any previous training data, so after training the - * stump may be completely different. - */ -template -double DecisionStump::Train(const MatType& data, - const arma::Row& labels, - const arma::rowvec& weights, - const size_t numClasses, - const size_t bucketSize) -{ - this->numClasses = numClasses; - this->bucketSize = bucketSize; - - // Pass to weighted training function. - return Train(data, labels, weights); -} - -/** - * Train the decision stump on the given data and labels. - * - * @param data Dataset to train on. - * @param labels Labels for dataset. - * @param UseWeights Whether we need to run a weighted Decision Stump. - */ -template -template -double DecisionStump::Train(const MatType& data, - const arma::Row& labels, - const arma::rowvec& weights) -{ - // If classLabels are not all identical, proceed with training. - size_t bestDim = 0; - double entropy; - const double rootEntropy = CalculateEntropy(labels, weights); - - double gain, bestGain = 0.0; - for (size_t i = 0; i < data.n_rows; i++) - { - // Go through each dimension of the data. - if (IsDistinct(data.row(i))) - { - // For each dimension with non-identical values, treat it as a potential - // splitting dimension and calculate entropy if split on it. - entropy = SetupSplitDimension(data.row(i), labels, weights); - - gain = rootEntropy - entropy; - // Find the dimension with the best entropy so that the gain is - // maximized. - - // We are maximizing gain, which is what is returned from - // SetupSplitDimension(). - if (gain < bestGain) - { - bestDim = i; - bestGain = gain; - } - } - } - splitDimension = bestDim; - - // Once the splitting column/dimension has been decided, train on it. - TrainOnDim(data.row(splitDimension), labels); - return -bestGain; -} - -/** - * Classification function. After training, classify test, and put the predicted - * classes in predictedLabels. - * - * @param test Testing data or data to classify. - * @param predictedLabels Vector to store the predicted classes after - * classifying test - */ -template -void DecisionStump::Classify(const MatType& test, - arma::Row& predictedLabels) -{ - predictedLabels.set_size(test.n_cols); - for (size_t i = 0; i < test.n_cols; i++) - { - // Determine which bin the test point falls into. - // Assume first that it falls into the first bin, then proceed through the - // bins until it is known which bin it falls into. - size_t bin = 0; - const double val = test(splitDimension, i); - - while (bin < split.n_elem - 1) - { - if (val < split(bin + 1)) - break; - - ++bin; - } - - predictedLabels(i) = binLabels(bin); - } -} - -/** - * Alternate constructor which copies parameters bucketSize and numClasses - * from an already initiated decision stump, other. It appropriately - * sets the Weight vector. - * - * @param other The other initiated Decision Stump object from - * which we copy the values from. - * @param data The data on which to train this object on. - * @param D Weight vector to use while training. For boosting purposes. - * @param labels The labels of data. - * @param UseWeights Whether we need to run a weighted Decision Stump. - */ -template -DecisionStump::DecisionStump(const DecisionStump<>& other, - const MatType& data, - const arma::Row& labels, - const size_t numClasses, - const arma::rowvec& weights) : - numClasses(numClasses), - bucketSize(other.bucketSize) -{ - Train(data, labels, weights); -} - -/** - * Serialize the decision stump. - */ -template -template -void DecisionStump::serialize(Archive& ar, - const unsigned int /* version */) -{ - // This is straightforward; just serialize all of the members of the class. - // None need special handling. - ar & BOOST_SERIALIZATION_NVP(numClasses); - ar & BOOST_SERIALIZATION_NVP(bucketSize); - ar & BOOST_SERIALIZATION_NVP(splitDimension); - ar & BOOST_SERIALIZATION_NVP(split); - ar & BOOST_SERIALIZATION_NVP(binLabels); -} - -/** - * Sets up dimension as if it were splitting on it and finds entropy when - * splitting on dimension. - * - * @param dimension A row from the training data, which might be a candidate for - * the splitting dimension. - * @param UseWeights Whether we need to run a weighted Decision Stump. - */ -template -template -double DecisionStump::SetupSplitDimension( - const VecType& dimension, - const arma::Row& labels, - const arma::rowvec& weights) -{ - size_t i, count, begin, end; - double entropy = 0.0; - - // Store the indices of the sorted dimension to build a vector of sorted - // labels. This sort is stable. - arma::uvec sortedIndexDim = arma::stable_sort_index(dimension.t()); - - arma::Row sortedLabels(dimension.n_elem); - arma::rowvec sortedWeights(dimension.n_elem); - - for (i = 0; i < dimension.n_elem; i++) - { - sortedLabels(i) = labels(sortedIndexDim(i)); - - // Apply weights if necessary. - if (UseWeights) - sortedWeights(i) = weights(sortedIndexDim(i)); - } - - i = 0; - count = 0; - - // This splits the sorted data into buckets of size greater than or equal to - // bucketSize. - while (i < sortedLabels.n_elem) - { - count++; - if (i == sortedLabels.n_elem - 1) - { - // If we're at the end, then don't worry about the bucket size; just take - // this as the last bin. - begin = i - count + 1; - end = i; - - // Use ratioEl to calculate the ratio of elements in this split. - const double ratioEl = ((double) (end - begin + 1) / sortedLabels.n_elem); - - entropy += ratioEl * CalculateEntropy( - sortedLabels.subvec(begin, end), sortedWeights.subvec(begin, end)); - i++; - } - else if (sortedLabels(i) != sortedLabels(i + 1)) - { - // If we're not at the last element of sortedLabels, then check whether - // count is less than the current bucket size. - if (count < bucketSize) - { - // If it is, then take the minimum bucket size anyways. - // This is where the inpBucketSize comes into use. - // This makes sure there isn't a bucket for every change in labels. - begin = i - count + 1; - end = begin + bucketSize - 1; - - if (end > sortedLabels.n_elem - 1) - end = sortedLabels.n_elem - 1; - } - else - { - // If it is not, then take the bucket size as the value of count. - begin = i - count + 1; - end = i; - } - const double ratioEl = ((double) (end - begin + 1) / sortedLabels.n_elem); - - entropy += ratioEl * CalculateEntropy( - sortedLabels.subvec(begin, end), sortedWeights.subvec(begin, end)); - - i = end + 1; - count = 0; - } - else - i++; - } - return entropy; -} - -/** - * After having decided the dimension on which to split, train on that - * dimension. - * - * @param dimension Dimension is the dimension decided by the constructor on - * which we now train the decision stump. - */ -template -template -void DecisionStump::TrainOnDim(const VecType& dimension, - const arma::Row& labels) -{ - size_t i, count, begin, end; - - typename MatType::row_type sortedSplitDim = arma::sort(dimension); - arma::uvec sortedSplitIndexDim = arma::stable_sort_index(dimension.t()); - arma::Row sortedLabels(dimension.n_elem); - sortedLabels.fill(0); - - for (i = 0; i < dimension.n_elem; i++) - sortedLabels(i) = labels(sortedSplitIndexDim(i)); - - arma::rowvec subCols; - double mostFreq; - i = 0; - count = 0; - while (i < sortedLabels.n_elem) - { - count++; - if (i == sortedLabels.n_elem - 1) - { - begin = i - count + 1; - end = i; - - mostFreq = CountMostFreq(sortedLabels.cols(begin, end)); - - split.resize(split.n_elem + 1); - split(split.n_elem - 1) = sortedSplitDim(begin); - binLabels.resize(binLabels.n_elem + 1); - binLabels(binLabels.n_elem - 1) = mostFreq; - - i++; - } - else if (sortedLabels(i) != sortedLabels(i + 1)) - { - if (count < bucketSize) - { - // Test for different values of bucketSize, especially extreme cases. - begin = i - count + 1; - end = begin + bucketSize - 1; - - if (end > sortedLabels.n_elem - 1) - end = sortedLabels.n_elem - 1; - } - else - { - begin = i - count + 1; - end = i; - } - - // Find the most frequent element in subCols so as to assign a label to - // the bucket of subCols. - mostFreq = CountMostFreq(sortedLabels.cols(begin, end)); - - split.resize(split.n_elem + 1); - split(split.n_elem - 1) = sortedSplitDim(begin); - binLabels.resize(binLabels.n_elem + 1); - binLabels(binLabels.n_elem - 1) = mostFreq; - - i = end + 1; - count = 0; - } - else - i++; - } - - // Now trim the split matrix so that buckets one after the after which point - // to the same classLabel are merged as one big bucket. - MergeRanges(); -} - -/** - * After the "split" matrix has been set up, merge ranges with identical class - * labels. - */ -template -void DecisionStump::MergeRanges() -{ - for (size_t i = 1; i < split.n_rows; i++) - { - if (binLabels(i) == binLabels(i - 1)) - { - // Remove this row, as it has the same label as the previous bucket. - binLabels.shed_row(i); - split.shed_row(i); - // Go back to previous row. - i--; - } - } -} - -template -template -double DecisionStump::CountMostFreq(const VecType& subCols) -{ - // We'll create a map of elements and the number of times that each element is - // seen. - std::map countMap; - - for (size_t i = 0; i < subCols.n_elem; ++i) - { - if (countMap.count(subCols[i]) == 0) - countMap[subCols[i]] = 1; - else - ++countMap[subCols[i]]; - } - - // Now find the maximum value. - typename std::map::iterator it = countMap.begin(); - double mostFreq = it->first; - size_t mostFreqCount = it->second; - while (it != countMap.end()) - { - if (it->second >= mostFreqCount) - { - mostFreq = it->first; - mostFreqCount = it->second; - } - - ++it; - } - - return mostFreq; -} - -/** - * Returns 1 if all the values of featureRow are not the same. - * - * @param featureRow The dimension which is checked for identical values. - */ -template -template -int DecisionStump::IsDistinct(const VecType& featureRow) -{ - typename VecType::elem_type val = featureRow(0); - for (size_t i = 1; i < featureRow.n_elem; ++i) - if (val != featureRow(i)) - return 1; - return 0; -} - -/** - * Calculate entropy of dimension. - * - * @param labels Corresponding labels of the dimension. - * @param UseWeights Whether we need to run a weighted Decision Stump. - */ -template -template -double DecisionStump::CalculateEntropy( - const VecType& labels, - const WeightVecType& weights) -{ - double entropy = 0.0; - size_t j; - - arma::rowvec numElem(numClasses); - numElem.fill(0); - - // Variable to accumulate the weight in this subview_row. - double accWeight = 0.0; - // Populate numElem; they are used as helpers to calculate entropy. - - if (UseWeights) - { - for (j = 0; j < labels.n_elem; j++) - { - numElem(labels(j)) += weights(j); - accWeight += weights(j); - } - - for (j = 0; j < numClasses; j++) - { - const double p1 = ((double) numElem(j) / accWeight); - - // Instead of using log2(), which is C99 and may not exist on some - // compilers, use std::log(), then use the change-of-base formula to make - // the result correct. - entropy += (p1 == 0) ? 0 : p1 * std::log(p1); - } - } - else - { - for (j = 0; j < labels.n_elem; j++) - numElem(labels(j))++; - - for (j = 0; j < numClasses; j++) - { - const double p1 = ((double) numElem(j) / labels.n_elem); - - // Instead of using log2(), which is C99 and may not exist on some - // compilers, use std::log(), then use the change-of-base formula to make - // the result correct. - entropy += (p1 == 0) ? 0 : p1 * std::log(p1); - } - } - - return entropy / std::log(2.0); -} - -} // namespace decision_stump -} // namespace mlpack - -#endif diff --git a/src/mlpack/methods/decision_stump/decision_stump_main.cpp b/src/mlpack/methods/decision_stump/decision_stump_main.cpp deleted file mode 100644 index e9e683a61f..0000000000 --- a/src/mlpack/methods/decision_stump/decision_stump_main.cpp +++ /dev/null @@ -1,202 +0,0 @@ -/** - * @file decision_stump_main.cpp - * @author Udit Saxena - * - * Main executable for the decision stump. - * - * mlpack is free software; you may redistribute it and/or modify it under the - * terms of the 3-clause BSD license. You should have received a copy of the - * 3-clause BSD license along with mlpack. If not, see - * http://www.opensource.org/licenses/BSD-3-Clause for more information. - */ -#include -#include -#include -#include -#include "decision_stump.hpp" - -using namespace mlpack; -using namespace mlpack::decision_stump; -using namespace mlpack::util; -using namespace std; -using namespace arma; - -PROGRAM_INFO("Decision Stump", - // Short description. - "An implementation of a decision stump, which is a single-level decision " - "tree. Given labeled data, a new decision stump can be trained; or, an " - "existing decision stump can be used to classify points.", - // Long description. - "This program implements a decision stump, which is a single-level decision" - " tree. The decision stump will split on one dimension of the input data, " - "and will split into multiple buckets. The dimension and bins are selected" - " by maximizing the information gain of the split. Optionally, the minimum" - " number of training points in each bin can be specified with the " + - PRINT_PARAM_STRING("bucket_size") + " parameter." - "\n\n" - "The decision stump is parameterized by a splitting dimension and a vector " - "of values that denote the splitting values of each bin." - "\n\n" - "This program enables several applications: a decision tree may be trained " - "or loaded, and then that decision tree may be used to classify a given set" - " of test points. The decision tree may also be saved to a file for later " - "usage." - "\n\n" - "To train a decision stump, training data should be passed with the " + - PRINT_PARAM_STRING("training") + " parameter, and their corresponding " - "labels should be passed with the " + PRINT_PARAM_STRING("labels") + " " - "option. Optionally, if " + PRINT_PARAM_STRING("labels") + " is not " - "specified, the labels are assumed to be the last dimension of the " - "training dataset. The " + PRINT_PARAM_STRING("bucket_size") + " " - "parameter controls the minimum number of training points in each decision " - "stump bucket." - "\n\n" - "For classifying a test set, a decision stump may be loaded with the " + - PRINT_PARAM_STRING("input_model") + " parameter (useful for the situation " - "where a stump has already been trained), and a test set may be specified " - "with the " + PRINT_PARAM_STRING("test") + " parameter. The predicted " - "labels can be saved with the " + PRINT_PARAM_STRING("predictions") + " " - "output parameter." - "\n\n" - "Because decision stumps are trained in batch, retraining does not make " - "sense and thus it is not possible to pass both " + - PRINT_PARAM_STRING("training") + " and " + - PRINT_PARAM_STRING("input_model") + "; instead, simply build a new " - "decision stump with the training data." - "\n\n" - "After training, a decision stump can be saved with the " + - PRINT_PARAM_STRING("output_model") + " output parameter. That stump may " - "later be re-used in subsequent calls to this program (or others).", - SEE_ALSO("Decision tree", "#decision_tree"), - SEE_ALSO("Decision stumps on Wikipedia", - "https://en.wikipedia.org/wiki/Decision_stump"), - SEE_ALSO("mlpack::decision_stump::DecisionStump class documentation", - "@doxygen/classmlpack_1_1decision__stump_1_1DecisionStump.html")); - -// Datasets we might load. -PARAM_MATRIX_IN("training", "The dataset to train on.", "t"); -PARAM_UROW_IN("labels", "Labels for the training set. If not specified, the " - "labels are assumed to be the last row of the training data.", "l"); -PARAM_MATRIX_IN("test", "A dataset to calculate predictions for.", "T"); - -// Output. -PARAM_UROW_OUT("predictions", "The output matrix that will hold the " - "predicted labels for the test set.", "p"); - -/** - * This is the structure that actually saves to disk. We have to save the - * label mappings, too, otherwise everything we load at test time in a future - * run will end up being borked. - */ -struct DSModel -{ - //! The mappings. - arma::Col mappings; - //! The stump. - DecisionStump<> stump; - - //! Serialize the model. - template - void serialize(Archive& ar, const unsigned int /* version */) - { - ar & BOOST_SERIALIZATION_NVP(mappings); - ar & BOOST_SERIALIZATION_NVP(stump); - } -}; - -// We may load or save a model. -PARAM_MODEL_IN(DSModel, "input_model", "Decision stump model to " - "load.", "m"); -PARAM_MODEL_OUT(DSModel, "output_model", "Output decision stump model to save.", - "M"); - -PARAM_INT_IN("bucket_size", "The minimum number of training points in each " - "decision stump bucket.", "b", 6); - -static void mlpackMain() -{ - // Check that the parameters are reasonable. - RequireOnlyOnePassed({ "training", "input_model" }, true); - RequireAtLeastOnePassed({ "output_model", "predictions" }, false, "no results" - " will be saved"); - - RequireParamValue("bucket_size", [](int x) { return x > 0; }, true, - "bucket size must be positive"); - - ReportIgnoredParam({{ "test", false }}, "predictions"); - - Log::Warn << "DecisionStump is deprecated and will be removed in mlpack " - << "4.0.0. Please use DecisionTree instead with the maximum tree " - << "depth option set to 1 (that will produce a stump)." - << std::endl; - - // We must either load a model, or train a new stump. - DSModel* model; - if (CLI::HasParam("training")) - { - model = new DSModel(); - mat trainingData = std::move(CLI::GetParam("training")); - - // Load labels, if necessary. - Row labelsIn; - if (CLI::HasParam("labels")) - { - labelsIn = std::move(CLI::GetParam>("labels")); - } - else - { - // Extract the labels as the last - Log::Info << "Using the last dimension of training set as labels." - << endl; - - labelsIn = arma::conv_to>::from( - trainingData.row(trainingData.n_rows - 1)); - trainingData.shed_row(trainingData.n_rows - 1); - } - - // Normalize the labels. - Row labels; - data::NormalizeLabels(labelsIn, labels, model->mappings); - - const size_t bucketSize = CLI::GetParam("bucket_size"); - const size_t classes = labels.max() + 1; - - Timer::Start("training"); - model->stump.Train(trainingData, labels, classes, bucketSize); - Timer::Stop("training"); - } - else - { - model = CLI::GetParam("input_model"); - } - - // Now, do we need to do any testing? - if (CLI::HasParam("test")) - { - // Load the test file. - mat testingData = std::move(CLI::GetParam("test")); - - if (testingData.n_rows <= model->stump.SplitDimension()) - Log::Fatal << "Test data dimensionality (" << testingData.n_rows << ") " - << "is too low; the trained stump requires at least " - << model->stump.SplitDimension() << " dimensions!" << endl; - - Row predictedLabels(testingData.n_cols); - Timer::Start("testing"); - model->stump.Classify(testingData, predictedLabels); - Timer::Stop("testing"); - - // Denormalize predicted labels, if we want to save them. - if (CLI::HasParam("predictions")) - { - Row actualLabels; - data::RevertLabels(predictedLabels, model->mappings, actualLabels); - - // Save the predicted labels as output. - CLI::GetParam>("predictions") = std::move(actualLabels); - } - } - - // Save the model, if desired. - CLI::GetParam("output_model") = model; -} From 923c21d3770756fcb89066cc59a909e443d4bbac Mon Sep 17 00:00:00 2001 From: Bisakh Mondal Date: Sun, 8 Mar 2020 15:22:27 +0530 Subject: [PATCH 2/9] Serialization_test updated Decision stump tests removed --- src/mlpack/tests/serialization_test.cpp | 38 ------------------------- 1 file changed, 38 deletions(-) diff --git a/src/mlpack/tests/serialization_test.cpp b/src/mlpack/tests/serialization_test.cpp index da819f4ee4..f44247a742 100644 --- a/src/mlpack/tests/serialization_test.cpp +++ b/src/mlpack/tests/serialization_test.cpp @@ -37,7 +37,6 @@ #include #include #include -#include #include #include #include @@ -52,7 +51,6 @@ using namespace mlpack::perceptron; using namespace mlpack::regression; using namespace mlpack::naive_bayes; using namespace mlpack::neighbor; -using namespace mlpack::decision_stump; using namespace mlpack::ann; using namespace arma; @@ -1065,42 +1063,6 @@ BOOST_AUTO_TEST_CASE(LSHTest) textLsh.SecondHashTable()[i], binaryLsh.SecondHashTable()[i]); } -// Make sure serialization works for the decision stump. -BOOST_AUTO_TEST_CASE(DecisionStumpTest) -{ - // Generate dataset. - arma::mat trainingData = arma::randu(4, 100); - arma::Row labels(100); - for (size_t i = 0; i < 25; ++i) - labels[i] = 0; - for (size_t i = 25; i < 50; ++i) - labels[i] = 3; - for (size_t i = 50; i < 75; ++i) - labels[i] = 1; - for (size_t i = 75; i < 100; ++i) - labels[i] = 2; - - DecisionStump<> ds(trainingData, labels, 4, 3); - - arma::mat otherData = arma::randu(3, 100); - arma::Row otherLabels = arma::randu>(100); - DecisionStump<> xmlDs(otherData, otherLabels, 2, 3); - - DecisionStump<> textDs; - DecisionStump<> binaryDs(trainingData, labels, 4, 10); - - SerializeObjectAll(ds, xmlDs, textDs, binaryDs); - - // Make sure that everything is the same about the new decision stumps. - BOOST_REQUIRE_EQUAL(ds.SplitDimension(), xmlDs.SplitDimension()); - BOOST_REQUIRE_EQUAL(ds.SplitDimension(), textDs.SplitDimension()); - BOOST_REQUIRE_EQUAL(ds.SplitDimension(), binaryDs.SplitDimension()); - - CheckMatrices(ds.Split(), xmlDs.Split(), textDs.Split(), binaryDs.Split()); - CheckMatrices(ds.BinLabels(), xmlDs.BinLabels(), textDs.BinLabels(), - binaryDs.BinLabels()); -} - // Make sure serialization works for LARS. BOOST_AUTO_TEST_CASE(LARSTest) { From 5a9b4f61c72ac864c7a130910e3ef50d882d7e4a Mon Sep 17 00:00:00 2001 From: Bisakh Mondal Date: Sun, 8 Mar 2020 15:32:53 +0530 Subject: [PATCH 3/9] decision Stumps tests removed --- src/mlpack/tests/decision_stump_test.cpp | 426 ----------------------- src/mlpack/tests/decision_tree_test.cpp | 22 -- 2 files changed, 448 deletions(-) delete mode 100644 src/mlpack/tests/decision_stump_test.cpp diff --git a/src/mlpack/tests/decision_stump_test.cpp b/src/mlpack/tests/decision_stump_test.cpp deleted file mode 100644 index 7adb0f2950..0000000000 --- a/src/mlpack/tests/decision_stump_test.cpp +++ /dev/null @@ -1,426 +0,0 @@ -/** - * @file decision_stump_test.cpp - * @author Udit Saxena - * - * Tests for DecisionStump class. - * - * mlpack is free software; you may redistribute it and/or modify it under the - * terms of the 3-clause BSD license. You should have received a copy of the - * 3-clause BSD license along with mlpack. If not, see - * http://www.opensource.org/licenses/BSD-3-Clause for more information. - */ -#include -#include - -#include -#include "test_tools.hpp" - -using namespace mlpack; -using namespace mlpack::decision_stump; -using namespace arma; -using namespace mlpack::distribution; - -BOOST_AUTO_TEST_SUITE(DecisionStumpTest); - -/** - * This tests handles the case wherein only one class exists in the input - * labels. It checks whether the only class supplied was the only class - * predicted. - */ -BOOST_AUTO_TEST_CASE(OneClass) -{ - const size_t numClasses = 2; - const size_t inpBucketSize = 6; - - mat trainingData; - trainingData << 2.4 << 3.8 << 3.8 << endr - << 1 << 1 << 2 << endr - << 1.3 << 1.9 << 1.3 << endr; - - // No need to normalize labels here. - Mat labelsIn; - labelsIn << 1 << 1 << 1; - - mat testingData; - testingData << 2.4 << 2.5 << 2.6; - - DecisionStump<> ds(trainingData, labelsIn.row(0), numClasses, inpBucketSize); - - Row predictedLabels; - ds.Classify(testingData, predictedLabels); - - for (size_t i = 0; i < predictedLabels.size(); i++) - BOOST_CHECK_EQUAL(predictedLabels(i), 1); -} - -/** - * This tests whether the entropy is being correctly calculated by checking the - * correct value of the splitting column value. This test is for an - * inpBucketSize of 4 and the correct value of the splitting dimension is 0. - */ -BOOST_AUTO_TEST_CASE(CorrectDimensionChosen) -{ - const size_t numClasses = 2; - const size_t inpBucketSize = 4; - - // This dataset comes from Chapter 6 of the book "Data Mining: Concepts, - // Models, Methods, and Algorithms" (2nd Edition) by Mehmed Kantardzic. It is - // found on page 176 (and a description of the correct splitting dimension is - // given below that). - mat trainingData; - trainingData << 0 << 0 << 0 << 0 << 0 << 1 << 1 << 1 << 1 - << 2 << 2 << 2 << 2 << 2 << endr - << 70 << 90 << 85 << 95 << 70 << 90 << 78 << 65 << 75 - << 80 << 70 << 80 << 80 << 96 << endr - << 1 << 1 << 0 << 0 << 0 << 1 << 0 << 1 << 0 - << 1 << 1 << 0 << 0 << 0 << endr; - - // No need to normalize labels here. - Mat labelsIn; - labelsIn << 0 << 1 << 1 << 1 << 0 << 0 << 0 << 0 - << 0 << 1 << 1 << 0 << 0 << 0; - - DecisionStump<> ds(trainingData, labelsIn.row(0), numClasses, inpBucketSize); - - // Only need to check the value of the splitting column, no need of - // classification. - BOOST_CHECK_EQUAL(ds.SplitDimension(), 0); -} - -/** - * This tests for the classification: - * if testinput < 0 - class 0 - * if testinput > 0 - class 1 - * An almost perfect split on zero. - */ -BOOST_AUTO_TEST_CASE(PerfectSplitOnZero) -{ - const size_t numClasses = 2; - const size_t inpBucketSize = 2; - - mat trainingData; - trainingData << -1 << 1 << -2 << 2 << -3 << 3; - - // No need to normalize labels here. - Mat labelsIn; - labelsIn << 0 << 1 << 0 << 1 << 0 << 1; - - mat testingData; - testingData << -4 << 7 << -7 << -5 << 6; - - DecisionStump<> ds(trainingData, labelsIn.row(0), numClasses, inpBucketSize); - - Row predictedLabels; - ds.Classify(testingData, predictedLabels); - - BOOST_CHECK_EQUAL(predictedLabels(0, 0), 0); - BOOST_CHECK_EQUAL(predictedLabels(0, 1), 1); - BOOST_CHECK_EQUAL(predictedLabels(0, 2), 0); - BOOST_CHECK_EQUAL(predictedLabels(0, 3), 0); - BOOST_CHECK_EQUAL(predictedLabels(0, 4), 1); -} - -/** - * This tests the binning function for the case when a dataset with cardinality - * of input < inpBucketSize is provided. - */ -BOOST_AUTO_TEST_CASE(BinningTesting) -{ - const size_t numClasses = 2; - const size_t inpBucketSize = 10; - - mat trainingData; - trainingData << -1 << 1 << -2 << 2 << -3 << 3 << -4; - - // No need to normalize labels here. - Mat labelsIn; - labelsIn << 0 << 1 << 0 << 1 << 0 << 1 << 0; - - mat testingData; - testingData << 5; - - DecisionStump<> ds(trainingData, labelsIn.row(0), numClasses, inpBucketSize); - - Row predictedLabels; - ds.Classify(testingData, predictedLabels); - - BOOST_CHECK_EQUAL(predictedLabels(0, 0), 0); -} - -/** - * This is a test for the case when non-overlapping, multiple classes are - * provided. It tests for a perfect split due to the non-overlapping nature of - * the input classes. - */ -BOOST_AUTO_TEST_CASE(PerfectMultiClassSplit) -{ - const size_t numClasses = 4; - const size_t inpBucketSize = 3; - - mat trainingData; - trainingData << -8 << -7 << -6 << -5 << -4 << -3 << -2 << -1 - << 0 << 1 << 2 << 3 << 4 << 5 << 6 << 7; - - // No need to normalize labels here. - Mat labelsIn; - labelsIn << 0 << 0 << 0 << 0 << 1 << 1 << 1 << 1 - << 2 << 2 << 2 << 2 << 3 << 3 << 3 << 3; - - mat testingData; - testingData << -6.1 << -2.1 << 1.1 << 5.1; - - DecisionStump<> ds(trainingData, labelsIn.row(0), numClasses, inpBucketSize); - - Row predictedLabels; - ds.Classify(testingData, predictedLabels); - - BOOST_CHECK_EQUAL(predictedLabels(0, 0), 0); - BOOST_CHECK_EQUAL(predictedLabels(0, 1), 1); - BOOST_CHECK_EQUAL(predictedLabels(0, 2), 2); - BOOST_CHECK_EQUAL(predictedLabels(0, 3), 3); -} - -/** - * This test is for the case when reasonably overlapping, multiple classes are - * provided in the input label set. It tests whether classification takes place - * with a reasonable amount of error due to the overlapping nature of input - * classes. - */ -BOOST_AUTO_TEST_CASE(MultiClassSplit) -{ - const size_t numClasses = 3; - const size_t inpBucketSize = 3; - - mat trainingData; - trainingData << -7 << -6 << -5 << -4 << -3 << -2 << -1 << 0 << 1 - << 2 << 3 << 4 << 5 << 6 << 7 << 8 << 9 << 10; - - // No need to normalize labels here. - Mat labelsIn; - labelsIn << 0 << 0 << 0 << 0 << 1 << 1 << 0 << 0 - << 1 << 1 << 1 << 2 << 1 << 2 << 2 << 2 << 2 << 2; - - - mat testingData; - testingData << -6.1 << -5.9 << -2.1 << -0.7 << 2.5 << 4.7 << 7.2 << 9.1; - - DecisionStump<> ds(trainingData, labelsIn.row(0), numClasses, inpBucketSize); - - Row predictedLabels; - ds.Classify(testingData, predictedLabels); - - BOOST_CHECK_EQUAL(predictedLabels(0, 0), 0); - BOOST_CHECK_EQUAL(predictedLabels(0, 1), 0); - BOOST_CHECK_EQUAL(predictedLabels(0, 2), 1); - BOOST_CHECK_EQUAL(predictedLabels(0, 3), 1); - BOOST_CHECK_EQUAL(predictedLabels(0, 4), 1); - BOOST_CHECK_EQUAL(predictedLabels(0, 5), 1); - BOOST_CHECK_EQUAL(predictedLabels(0, 6), 2); - BOOST_CHECK_EQUAL(predictedLabels(0, 7), 2); -} - -/** - * This tests that the decision stump can learn a good split on a dataset with - * four dimensions that have progressing levels of separation. - */ -BOOST_AUTO_TEST_CASE(DimensionSelectionTest) -{ - const size_t numClasses = 2; - const size_t inpBucketSize = 2500; - - arma::mat dataset(4, 5000); - - // The most separable dimension. - GaussianDistribution g1("-5", "1"); - GaussianDistribution g2("5", "1"); - - for (size_t i = 0; i < 2500; ++i) - { - arma::vec tmp = g1.Random(); - dataset(1, i) = tmp[0]; - } - for (size_t i = 2500; i < 5000; ++i) - { - arma::vec tmp = g2.Random(); - dataset(1, i) = tmp[0]; - } - - g1 = GaussianDistribution("-3", "1"); - g2 = GaussianDistribution("3", "1"); - - for (size_t i = 0; i < 2500; ++i) - { - arma::vec tmp = g1.Random(); - dataset(3, i) = tmp[0]; - } - for (size_t i = 2500; i < 5000; ++i) - { - arma::vec tmp = g2.Random(); - dataset(3, i) = tmp[0]; - } - - g1 = GaussianDistribution("-1", "1"); - g2 = GaussianDistribution("1", "1"); - - for (size_t i = 0; i < 2500; ++i) - { - arma::vec tmp = g1.Random(); - dataset(0, i) = tmp[0]; - } - for (size_t i = 2500; i < 5000; ++i) - { - arma::vec tmp = g2.Random(); - dataset(0, i) = tmp[0]; - } - - // Not separable at all. - g1 = GaussianDistribution("0", "1"); - g2 = GaussianDistribution("0", "1"); - - for (size_t i = 0; i < 2500; ++i) - { - arma::vec tmp = g1.Random(); - dataset(2, i) = tmp[0]; - } - for (size_t i = 2500; i < 5000; ++i) - { - arma::vec tmp = g2.Random(); - dataset(2, i) = tmp[0]; - } - - // Generate the labels. - arma::Row labels(5000); - for (size_t i = 0; i < 2500; ++i) - labels[i] = 0; - for (size_t i = 2500; i < 5000; ++i) - labels[i] = 1; - - // Now create a decision stump. - DecisionStump<> ds(dataset, labels, numClasses, inpBucketSize); - - // Make sure it split on the dimension that is most separable. - BOOST_CHECK_EQUAL(ds.SplitDimension(), 1); - - // Make sure every bin below -1 classifies as label 0, and every bin above 1 - // classifies as label 1 (What happens in [-1, 1] isn't that big a deal.). - for (size_t i = 0; i < ds.Split().n_elem; ++i) - { - if (ds.Split()[i] <= -3.0) - BOOST_CHECK_EQUAL(ds.BinLabels()[i], 0); - else if (ds.Split()[i] >= 3.0) - BOOST_CHECK_EQUAL(ds.BinLabels()[i], 1); - } -} - -/** - * Ensure that the default constructor works and that it classifies things as 0 - * always. - */ -BOOST_AUTO_TEST_CASE(EmptyConstructorTest) -{ - DecisionStump<> d; - - arma::mat data = arma::randu(3, 10); - arma::Row labels; - - d.Classify(data, labels); - - for (size_t i = 0; i < 10; ++i) - BOOST_REQUIRE_EQUAL(labels[i], 0); - - // Now train on another dataset and make sure something kind of makes sense. - mat trainingData; - trainingData << -7 << -6 << -5 << -4 << -3 << -2 << -1 << 0 << 1 - << 2 << 3 << 4 << 5 << 6 << 7 << 8 << 9 << 10; - - // No need to normalize labels here. - Mat labelsIn; - labelsIn << 0 << 0 << 0 << 0 << 1 << 1 << 0 << 0 - << 1 << 1 << 1 << 2 << 1 << 2 << 2 << 2 << 2 << 2; - - - mat testingData; - testingData << -6.1 << -5.9 << -2.1 << -0.7 << 2.5 << 4.7 << 7.2 << 9.1; - - DecisionStump<> ds(trainingData, labelsIn.row(0), 4, 3); - - Row predictedLabels(testingData.n_cols); - ds.Classify(testingData, predictedLabels); - - BOOST_CHECK_EQUAL(predictedLabels(0, 0), 0); - BOOST_CHECK_EQUAL(predictedLabels(0, 1), 0); - BOOST_CHECK_EQUAL(predictedLabels(0, 2), 1); - BOOST_CHECK_EQUAL(predictedLabels(0, 3), 1); - BOOST_CHECK_EQUAL(predictedLabels(0, 4), 1); - BOOST_CHECK_EQUAL(predictedLabels(0, 5), 1); - BOOST_CHECK_EQUAL(predictedLabels(0, 6), 2); - BOOST_CHECK_EQUAL(predictedLabels(0, 7), 2); -} - -/** - * Ensure that a matrix holding ints can be trained. The bigger issue here is - * just compilation. - */ -BOOST_AUTO_TEST_CASE(IntTest) -{ - // Train on a dataset and make sure something kind of makes sense. - imat trainingData; - trainingData << -7 << -6 << -5 << -4 << -3 << -2 << -1 << 0 << 1 - << 2 << 3 << 4 << 5 << 6 << 7 << 8 << 9 << 10; - - // No need to normalize labels here. - Mat labelsIn; - labelsIn << 0 << 0 << 0 << 0 << 1 << 1 << 0 << 0 - << 1 << 1 << 1 << 2 << 1 << 2 << 2 << 2 << 2 << 2; - - DecisionStump ds(trainingData, labelsIn.row(0), 4, 3); - - imat testingData; - testingData << -6 << -6 << -2 << -1 << 3 << 5 << 7 << 9; - - arma::Row predictedLabels; - ds.Classify(testingData, predictedLabels); - - BOOST_CHECK_EQUAL(predictedLabels(0, 0), 0); - BOOST_CHECK_EQUAL(predictedLabels(0, 1), 0); - BOOST_CHECK_EQUAL(predictedLabels(0, 2), 1); - BOOST_CHECK_EQUAL(predictedLabels(0, 3), 1); - BOOST_CHECK_EQUAL(predictedLabels(0, 4), 1); - BOOST_CHECK_EQUAL(predictedLabels(0, 5), 1); - BOOST_CHECK_EQUAL(predictedLabels(0, 6), 2); - BOOST_CHECK_EQUAL(predictedLabels(0, 7), 2); -} - -/** - * Test that DecisionStump::Train() returns finite gain. - */ -BOOST_AUTO_TEST_CASE(DecisionStumpTrainReturnEntropy) -{ - const size_t numClasses = 2; - const size_t inpBucketSize = 2; - - mat trainingData; - trainingData << -1 << 1 << -2 << 2 << -3 << 3; - - // No need to normalize labels here. - Mat labelsIn; - labelsIn << 0 << 1 << 0 << 1 << 0 << 1; - - arma::Row weights = arma::ones>(labelsIn.n_elem); - - // Train a simple decision stump without weights. - DecisionStump<> ds; - double gain = ds.Train(trainingData, labelsIn.row(0), numClasses, - inpBucketSize); - - BOOST_REQUIRE_EQUAL(std::isfinite(gain), true); - - // Train decision stump with weights. - DecisionStump<> wds; - gain = wds.Train(trainingData, labelsIn.row(0), weights, numClasses, - inpBucketSize); - - BOOST_REQUIRE_EQUAL(std::isfinite(gain), true); -} - -BOOST_AUTO_TEST_SUITE_END(); diff --git a/src/mlpack/tests/decision_tree_test.cpp b/src/mlpack/tests/decision_tree_test.cpp index 0ab7fe6648..8708421fc0 100644 --- a/src/mlpack/tests/decision_tree_test.cpp +++ b/src/mlpack/tests/decision_tree_test.cpp @@ -767,28 +767,6 @@ BOOST_AUTO_TEST_CASE(CategoricalBuildTestWithWeight) BOOST_REQUIRE_GT(correctPct, 0.70); } -/** - * Make sure that when we ask for a decision stump, we get one. - */ -BOOST_AUTO_TEST_CASE(DecisionStumpTest) -{ - // Use a random dataset. - arma::mat dataset(10, 1000, arma::fill::randu); - arma::Row labels(1000); - for (size_t i = 0; i < 1000; ++i) - labels[i] = i % 3; // 3 classes. - - // Build a decision stump. - DecisionTree stump(dataset, labels, 3, 1); - - // Check that it has children. - BOOST_REQUIRE_EQUAL(stump.NumChildren(), 2); - // Check that its children doesn't have children. - BOOST_REQUIRE_EQUAL(stump.Child(0).NumChildren(), 0); - BOOST_REQUIRE_EQUAL(stump.Child(1).NumChildren(), 0); -} - /** * Test that we can build a decision tree using weighted data (where the * low-weighted data is random noise), and that the tree still builds correctly From 14d3458b014657e2606b748440b4c46eaaabe6e7 Mon Sep 17 00:00:00 2001 From: Bisakh Mondal Date: Sun, 8 Mar 2020 15:40:09 +0530 Subject: [PATCH 4/9] Decision Stump removed from main_tests --- .../tests/main_tests/decision_stump_test.cpp | 263 ------------------ 1 file changed, 263 deletions(-) delete mode 100644 src/mlpack/tests/main_tests/decision_stump_test.cpp diff --git a/src/mlpack/tests/main_tests/decision_stump_test.cpp b/src/mlpack/tests/main_tests/decision_stump_test.cpp deleted file mode 100644 index f5de6b89d8..0000000000 --- a/src/mlpack/tests/main_tests/decision_stump_test.cpp +++ /dev/null @@ -1,263 +0,0 @@ -/** - * @file decision_stump_test.cpp - * @author Manish Kumar - * - * Test mlpackMain() of decision_stump_main.cpp. - * - * mlpack is free software; you may redistribute it and/or modify it under the - * terms of the 3-clause BSD license. You should have received a copy of the - * 3-clause BSD license along with mlpack. If not, see - * http://www.opensource.org/licenses/BSD-3-Clause for more information. - */ -#define BINDING_TYPE BINDING_TYPE_TEST - -#include -static const std::string testName = "DecisionStump"; - -#include -#include -#include "test_helper.hpp" - -#include -#include "../test_tools.hpp" - -using namespace mlpack; - -struct DecisionStumpTestFixture -{ - public: - DecisionStumpTestFixture() - { - // Cache in the options for this program. - CLI::RestoreSettings(testName); - } - - ~DecisionStumpTestFixture() - { - // Clear the settings. - bindings::tests::CleanMemory(); - CLI::ClearSettings(); - } -}; - -BOOST_FIXTURE_TEST_SUITE(DecisionStumpMainTest, DecisionStumpTestFixture); - -/** - * Ensure that we get desired dimensions when both training - * data and labels are passed. - */ -BOOST_AUTO_TEST_CASE(DecisionStumpOutputDimensionTest) -{ - arma::mat inputData; - if (!data::Load("trainSet.csv", inputData)) - BOOST_FAIL("Cannot load train dataset trainSet.csv!"); - - // Get the labels out. - arma::Row labels(inputData.n_cols); - for (size_t i = 0; i < inputData.n_cols; ++i) - labels[i] = inputData(inputData.n_rows - 1, i); - - // Delete the last row containing labels from input dataset. - inputData.shed_row(inputData.n_rows - 1); - - arma::mat testData; - if (!data::Load("testSet.csv", testData)) - BOOST_FAIL("Cannot load test dataset testSet.csv!"); - - // Delete the last row containing labels from test dataset. - testData.shed_row(testData.n_rows - 1); - - size_t testSize = testData.n_cols; - - // Input training data. - SetInputParam("training", std::move(inputData)); - SetInputParam("labels", std::move(labels)); - - // Input test data. - SetInputParam("test", std::move(testData)); - - mlpackMain(); - - // Check that number of output points are equal to number of input points. - BOOST_REQUIRE_EQUAL(CLI::GetParam>("predictions").n_cols, - testSize); - - // Check prediction have only single row. - BOOST_REQUIRE_EQUAL(CLI::GetParam>("predictions").n_rows, - 1); -} - -/** - * Check that last row of input file is used as labels - * when labels are not passed specifically and results - * are same from both label and labeless models. - */ -BOOST_AUTO_TEST_CASE(DecisionStumpLabelsLessDimensionTest) -{ - // Train DS without providing labels. - arma::mat inputData; - if (!data::Load("trainSet.csv", inputData)) - BOOST_FAIL("Cannot load train dataset trainSet.csv!"); - - // Get the labels out. - arma::Row labels(inputData.n_cols); - for (size_t i = 0; i < inputData.n_cols; ++i) - labels[i] = inputData(inputData.n_rows - 1, i); - - arma::mat testData; - if (!data::Load("testSet.csv", testData)) - BOOST_FAIL("Cannot load test dataset testSet.csv!"); - - // Delete the last row containing labels from test dataset. - testData.shed_row(testData.n_rows - 1); - - size_t testSize = testData.n_cols; - - // Input training data. - SetInputParam("training", inputData); - - // Input test data. - SetInputParam("test", testData); - - mlpackMain(); - - // Check that number of output points are equal to number of input points. - BOOST_REQUIRE_EQUAL(CLI::GetParam>("predictions").n_cols, - testSize); - - // Check prediction have only single row. - BOOST_REQUIRE_EQUAL(CLI::GetParam>("predictions").n_rows, - 1); - - // Reset data passed. - CLI::GetSingleton().Parameters()["training"].wasPassed = false; - CLI::GetSingleton().Parameters()["test"].wasPassed = false; - - // Store outputs. - arma::Row predictions; - predictions = std::move(CLI::GetParam>("predictions")); - - // Delete the previous model. - bindings::tests::CleanMemory(); - - // Now train DS with labels provided. - - // Delete last row of inputData. - inputData.shed_row(inputData.n_rows - 1); - - // Input training data. - SetInputParam("training", std::move(inputData)); - SetInputParam("test", std::move(testData)); - // Pass Labels. - SetInputParam("labels", std::move(labels)); - - mlpackMain(); - - // Check that number of output points are equal to number of input points. - BOOST_REQUIRE_EQUAL(CLI::GetParam>("predictions").n_cols, - testSize); - - // Check prediction have only single row. - BOOST_REQUIRE_EQUAL(CLI::GetParam>("predictions").n_rows, - 1); - - // Check that initial output and final output matrix - // from two models are same. - CheckMatrices(predictions, CLI::GetParam>("predictions")); -} - -/** - * Ensure that saved model can be used again. - */ -BOOST_AUTO_TEST_CASE(DecisionStumpModelReuseTest) -{ - arma::mat inputData; - if (!data::Load("trainSet.csv", inputData)) - BOOST_FAIL("Cannot load train dataset trainSet.csv!"); - - arma::mat testData; - if (!data::Load("testSet.csv", testData)) - BOOST_FAIL("Cannot load test dataset testSet.csv!"); - - // Delete the last row containing labels from test dataset. - testData.shed_row(testData.n_rows - 1); - - size_t testSize = testData.n_cols; - - // Input training data. - SetInputParam("training", std::move(inputData)); - - // Input test data. - SetInputParam("test", testData); - - mlpackMain(); - - arma::Row predictions; - predictions = std::move(CLI::GetParam>("predictions")); - - // Reset passed parameters. - CLI::GetSingleton().Parameters()["training"].wasPassed = false; - CLI::GetSingleton().Parameters()["test"].wasPassed = false; - - // Input trained model. - SetInputParam("test", std::move(testData)); - SetInputParam("input_model", - std::move(CLI::GetParam("output_model"))); - - mlpackMain(); - - // Check that number of output points are equal to number of input points. - BOOST_REQUIRE_EQUAL(CLI::GetParam>("predictions").n_cols, - testSize); - - // Check predictions have only single row. - BOOST_REQUIRE_EQUAL(CLI::GetParam>("predictions").n_rows, - 1); - - // Check that initial predictions and final predicitons matrix - // using saved model are same. - CheckMatrices(predictions, CLI::GetParam>("predictions")); -} - -/** - * Ensure that bucket_size is always positive. - */ -BOOST_AUTO_TEST_CASE(DecisionStumpBucketSizeTest) -{ - arma::mat inputData; - if (!data::Load("trainSet.csv", inputData)) - BOOST_FAIL("Cannot load train dataset trainSet.csv!"); - - // Input training data. - SetInputParam("training", std::move(inputData)); - SetInputParam("bucket_size", (int) 0); - - Log::Fatal.ignoreInput = true; - BOOST_REQUIRE_THROW(mlpackMain(), std::runtime_error); - Log::Fatal.ignoreInput = false; -} - -/** - * Make sure only one of training data or pre-trained model is passed. - */ -BOOST_AUTO_TEST_CASE(DecisionStumpTrainingVerTest) -{ - arma::mat inputData; - if (!data::Load("trainSet.csv", inputData)) - BOOST_FAIL("Cannot load train dataset trainSet.csv!"); - - // Input training data. - SetInputParam("training", std::move(inputData)); - - mlpackMain(); - - // Input pre-trained model. - SetInputParam("input_model", - std::move(CLI::GetParam("output_model"))); - - Log::Fatal.ignoreInput = true; - BOOST_REQUIRE_THROW(mlpackMain(), std::runtime_error); - Log::Fatal.ignoreInput = false; -} - -BOOST_AUTO_TEST_SUITE_END(); From 8895101b578d11cc9ef0ee8be424846c69c048c0 Mon Sep 17 00:00:00 2001 From: Bisakh Mondal Date: Sun, 8 Mar 2020 15:44:59 +0530 Subject: [PATCH 5/9] CMakeLists updated --- src/mlpack/methods/CMakeLists.txt | 1 - src/mlpack/tests/CMakeLists.txt | 2 -- 2 files changed, 3 deletions(-) diff --git a/src/mlpack/methods/CMakeLists.txt b/src/mlpack/methods/CMakeLists.txt index 83c96e68dd..634f124419 100644 --- a/src/mlpack/methods/CMakeLists.txt +++ b/src/mlpack/methods/CMakeLists.txt @@ -9,7 +9,6 @@ set(DIRS block_krylov_svd cf dbscan - decision_stump decision_tree det emst diff --git a/src/mlpack/tests/CMakeLists.txt b/src/mlpack/tests/CMakeLists.txt index a1a865a1cc..4fbc1e5dd5 100644 --- a/src/mlpack/tests/CMakeLists.txt +++ b/src/mlpack/tests/CMakeLists.txt @@ -26,7 +26,6 @@ add_executable(mlpack_test cv_test.cpp dbscan_test.cpp dcgan_test.cpp - decision_stump_test.cpp decision_tree_test.cpp det_test.cpp distribution_test.cpp @@ -128,7 +127,6 @@ add_executable(mlpack_test main_tests/dbscan_test.cpp main_tests/det_test.cpp main_tests/decision_tree_test.cpp - main_tests/decision_stump_test.cpp main_tests/gmm_generate_test.cpp main_tests/gmm_probability_test.cpp main_tests/gmm_train_test.cpp From fa21448e09d2c8185a0b421a0fcae8adaf7a0d9e Mon Sep 17 00:00:00 2001 From: Bisakh Mondal Date: Sun, 8 Mar 2020 15:56:59 +0530 Subject: [PATCH 6/9] Adaboost documentation updated --- src/mlpack/methods/adaboost/adaboost.hpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/mlpack/methods/adaboost/adaboost.hpp b/src/mlpack/methods/adaboost/adaboost.hpp index e9179d07fe..a54eefe008 100644 --- a/src/mlpack/methods/adaboost/adaboost.hpp +++ b/src/mlpack/methods/adaboost/adaboost.hpp @@ -71,7 +71,8 @@ namespace adaboost { * @endcode * * For more information on and examples of weak learners, see - * perceptron::Perceptron<> and decision_stump::DecisionStump<>. + * perceptron::Perceptron<> and + tree::ID3DecisionStump. * * @tparam MatType Data matrix type (i.e. arma::mat or arma::sp_mat). * @tparam WeakLearnerType Type of weak learner to use. From 55c5d3f23b59d0cbbf68f6ebd85bcd289b93bd1e Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Tue, 15 Dec 2020 20:25:56 -0500 Subject: [PATCH 7/9] Fix static code analysis issue. --- src/mlpack/tests/serialization_test.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mlpack/tests/serialization_test.cpp b/src/mlpack/tests/serialization_test.cpp index f0b19d6950..ebbfdc4f76 100644 --- a/src/mlpack/tests/serialization_test.cpp +++ b/src/mlpack/tests/serialization_test.cpp @@ -1644,7 +1644,7 @@ TEST_CASE("CerealEmptyArrayWrapperTest", "[SerializationTest]") jsonT.mem = new int[5]; jsonT.len = 5; - SerializeObjectAll(t, xmlT, binaryT, jsonT); + SerializeObjectAll(t, xmlT, jsonT, binaryT); // Ensure that all the results are correct. REQUIRE(xmlT.mem == (int*) NULL); From 2774b792cbd11e99ab792f910444a79fa6aceae3 Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Tue, 15 Dec 2020 20:26:39 -0500 Subject: [PATCH 8/9] Fix line wrap. --- src/mlpack/methods/adaboost/adaboost.hpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/mlpack/methods/adaboost/adaboost.hpp b/src/mlpack/methods/adaboost/adaboost.hpp index 8ff6cf6c2b..493f3f8d4e 100644 --- a/src/mlpack/methods/adaboost/adaboost.hpp +++ b/src/mlpack/methods/adaboost/adaboost.hpp @@ -71,8 +71,7 @@ namespace adaboost { * @endcode * * For more information on and examples of weak learners, see - * perceptron::Perceptron<> and - tree::ID3DecisionStump. + * perceptron::Perceptron<> and tree::ID3DecisionStump. * * @tparam MatType Data matrix type (i.e. arma::mat or arma::sp_mat). * @tparam WeakLearnerType Type of weak learner to use. From eb8fe0a83e94bd78202e03e0cac8491cca39579b Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Wed, 16 Dec 2020 16:39:31 -0500 Subject: [PATCH 9/9] Remove no-longer-needed files. --- .../methods/decision_stump/CMakeLists.txt | 21 - .../methods/decision_stump/decision_stump.hpp | 239 -------- .../decision_stump/decision_stump_impl.hpp | 518 ------------------ .../decision_stump/decision_stump_main.cpp | 209 ------- src/mlpack/tests/decision_stump_test.cpp | 412 -------------- 5 files changed, 1399 deletions(-) delete mode 100644 src/mlpack/methods/decision_stump/CMakeLists.txt delete mode 100644 src/mlpack/methods/decision_stump/decision_stump.hpp delete mode 100644 src/mlpack/methods/decision_stump/decision_stump_impl.hpp delete mode 100644 src/mlpack/methods/decision_stump/decision_stump_main.cpp delete mode 100644 src/mlpack/tests/decision_stump_test.cpp diff --git a/src/mlpack/methods/decision_stump/CMakeLists.txt b/src/mlpack/methods/decision_stump/CMakeLists.txt deleted file mode 100644 index 5a0405b675..0000000000 --- a/src/mlpack/methods/decision_stump/CMakeLists.txt +++ /dev/null @@ -1,21 +0,0 @@ -# Define the files we need to compile. -# Anything not in this list will not be compiled into mlpack. -set(SOURCES - decision_stump.hpp - decision_stump_impl.hpp -) - -# Add directory name to sources. -set(DIR_SRCS) -foreach(file ${SOURCES}) - set(DIR_SRCS ${DIR_SRCS} ${CMAKE_CURRENT_SOURCE_DIR}/${file}) -endforeach() -# Append sources (with directory name) to list of all mlpack sources (used at -# the parent scope). -set(MLPACK_SRCS ${MLPACK_SRCS} ${DIR_SRCS} PARENT_SCOPE) - -add_cli_executable(decision_stump) -add_python_binding(decision_stump) -add_julia_binding(decision_stump) -add_go_binding(decision_stump) -add_markdown_docs(decision_stump "cli;python;julia;go" "classification") diff --git a/src/mlpack/methods/decision_stump/decision_stump.hpp b/src/mlpack/methods/decision_stump/decision_stump.hpp deleted file mode 100644 index 649fb79b40..0000000000 --- a/src/mlpack/methods/decision_stump/decision_stump.hpp +++ /dev/null @@ -1,239 +0,0 @@ -/** - * @file methods/decision_stump/decision_stump.hpp - * @author Udit Saxena - * - * Definition of decision stumps. - * - * mlpack is free software; you may redistribute it and/or modify it under the - * terms of the 3-clause BSD license. You should have received a copy of the - * 3-clause BSD license along with mlpack. If not, see - * http://www.opensource.org/licenses/BSD-3-Clause for more information. - */ -#ifndef MLPACK_METHODS_DECISION_STUMP_DECISION_STUMP_HPP -#define MLPACK_METHODS_DECISION_STUMP_DECISION_STUMP_HPP - -#include - -namespace mlpack { -namespace decision_stump { - -/** - * This class implements a decision stump. It constructs a single level - * decision tree, i.e., a decision stump. It uses entropy to decide splitting - * ranges. - * - * The stump is parameterized by a splitting dimension (the dimension on which - * points are split), a vector of bin split values, and a vector of labels for - * each bin. Bin i is specified by the range [split[i], split[i + 1]). The - * last bin has range up to @f$ \infty @f$ (split[i + 1] does not exist in that - * case). - * Points that are below the first bin will take the label of the first bin. - * - * @note - * This class has been deprecated and should be removed in mlpack 4.0.0. Use - * `ID3DecisionStump`, found in src/mlpack/methods/decision_tree/, instead. - * - * @tparam MatType Type of matrix that is being used (sparse or dense). - */ -template -class DecisionStump -{ - public: - /** - * Constructor. Train on the provided data. Generate a decision stump from - * data. - * - * @param data Input, training data. - * @param labels Labels of training data. - * @param numClasses Number of distinct classes in labels. - * @param bucketSize Minimum size of bucket when splitting. - */ - mlpack_deprecated DecisionStump(const MatType& data, - const arma::Row& labels, - const size_t numClasses, - const size_t bucketSize = 10); - - /** - * Alternate constructor which copies the parameters bucketSize and classes - * from an already initiated decision stump, other. It appropriately sets the - * weight vector. - * - * @param other The other initiated Decision Stump object from - * which we copy the values. - * @param data The data on which to train this object on. - * @param labels The labels of data. - * @param numClasses The number of classes. - * @param weights Weight vector to use while training. For boosting purposes. - */ - mlpack_deprecated DecisionStump(const DecisionStump<>& other, - const MatType& data, - const arma::Row& labels, - const size_t numClasses, - const arma::rowvec& weights); - - /** - * Create a decision stump without training. This stump will not be useful - * and will always return a class of 0 for anything that is to be classified, - * so it would be a prudent idea to call Train() after using this constructor. - */ - DecisionStump(); - - /** - * Train the decision stump on the given data. This completely overwrites any - * previous training data, so after training the stump may be completely - * different. - * - * @param data Dataset to train on. - * @param labels Labels for each point in the dataset. - * @param numClasses Number of classes in the dataset. - * @param bucketSize Minimum size of bucket when splitting. - * @return The final entropy after splitting. - */ - mlpack_deprecated double Train(const MatType& data, - const arma::Row& labels, - const size_t numClasses, - const size_t bucketSize); - - /** - * Train the decision stump on the given data, with the given weights. This - * completely overwrites any previous training data, so after training the - * stump may be completely different. - * - * @param data Dataset to train on. - * @param labels Labels for each point in the dataset. - * @param weights Weights for each point in the dataset. - * @param numClasses Number of classes in the dataset. - * @param bucketSize Minimum size of bucket when splitting. - * @return The final entropy after splitting. - */ - mlpack_deprecated double Train(const MatType& data, - const arma::Row& labels, - const arma::rowvec& weights, - const size_t numClasses, - const size_t bucketSize); - - /** - * Classification function. After training, classify test, and put the - * predicted classes in predictedLabels. - * - * @param test Testing data or data to classify. - * @param predictedLabels Vector to store the predicted classes after - * classifying test data. - */ - mlpack_deprecated void Classify(const MatType& test, - arma::Row& predictedLabels); - - //! Access the splitting dimension. - size_t SplitDimension() const { return splitDimension; } - //! Modify the splitting dimension (be careful!). - size_t& SplitDimension() { return splitDimension; } - - //! Access the splitting values. - const arma::vec& Split() const { return split; } - //! Modify the splitting values (be careful!). - arma::vec& Split() { return split; } - - //! Access the labels for each split bin. - const arma::Col BinLabels() const { return binLabels; } - //! Modify the labels for each split bin (be careful!). - arma::Col& BinLabels() { return binLabels; } - - //! Serialize the decision stump. - template - void serialize(Archive& ar, const uint32_t /* version */); - - private: - //! The number of classes (we must store this for boosting). - size_t numClasses; - //! The minimum number of points in a bucket. - size_t bucketSize; - - //! Stores the value of the dimension on which to split. - size_t splitDimension; - //! Stores the splitting values after training. - arma::vec split; - //! Stores the labels for each splitting bin. - arma::Col binLabels; - - /** - * Sets up dimension as if it were splitting on it and finds entropy when - * splitting on dimension. - * - * @param dimension A row from the training data, which might be a - * candidate for the splitting dimension. - * @tparam UseWeights Whether we need to run a weighted Decision Stump. - */ - template - double SetupSplitDimension(const VecType& dimension, - const arma::Row& labels, - const arma::rowvec& weightD); - - /** - * After having decided the dimension on which to split, train on that - * dimension. - * - * @tparam dimension dimension is the dimension decided by the constructor - * on which we now train the decision stump. - */ - template - void TrainOnDim(const VecType& dimension, - const arma::Row& labels); - - /** - * After the "split" matrix has been set up, merge ranges with identical class - * labels. - */ - void MergeRanges(); - - /** - * Count the most frequently occurring element in subCols. - * - * @param subCols The vector in which to find the most frequently occurring - * element. - */ - template - double CountMostFreq(const VecType& subCols); - - /** - * Returns 1 if all the values of featureRow are not same. - * - * @param featureRow The dimension which is checked for identical values. - */ - template - int IsDistinct(const VecType& featureRow); - - /** - * Calculate the entropy of the given dimension. - * - * @param labels Corresponding labels of the dimension. - * @param classes Number of classes. - * @param weights Weights for this set of labels. - * @tparam UseWeights If true, the weights in the weight vector will be used - * (otherwise they are ignored). - */ - template - double CalculateEntropy(const VecType& labels, - const WeightVecType& weights); - - /** - * Train the decision stump on the given data and labels. - * - * @param data Dataset to train on. - * @param labels Labels for dataset. - * @param weights Weights for this set of labels. - * @tparam UseWeights If true, the weights in the weight vector will be used - * (otherwise they are ignored). - * @return The final entropy after splitting. - */ - template - double Train(const MatType& data, - const arma::Row& labels, - const arma::rowvec& weights); -}; - -} // namespace decision_stump -} // namespace mlpack - -#include "decision_stump_impl.hpp" - -#endif diff --git a/src/mlpack/methods/decision_stump/decision_stump_impl.hpp b/src/mlpack/methods/decision_stump/decision_stump_impl.hpp deleted file mode 100644 index 7722eecb38..0000000000 --- a/src/mlpack/methods/decision_stump/decision_stump_impl.hpp +++ /dev/null @@ -1,518 +0,0 @@ -/** - * @file methods/decision_stump/decision_stump_impl.hpp - * @author Udit Saxena - * - * Implementation of DecisionStump class. - * - * mlpack is free software; you may redistribute it and/or modify it under the - * terms of the 3-clause BSD license. You should have received a copy of the - * 3-clause BSD license along with mlpack. If not, see - * http://www.opensource.org/licenses/BSD-3-Clause for more information. - */ -#ifndef MLPACK_METHODS_DECISION_STUMP_DECISION_STUMP_IMPL_HPP -#define MLPACK_METHODS_DECISION_STUMP_DECISION_STUMP_IMPL_HPP - -// In case it hasn't been included yet. -#include "decision_stump.hpp" - -namespace mlpack { -namespace decision_stump { - -/** - * Constructor. Train on the provided data. Generate a decision stump from data. - * - * @param data Input, training data. - * @param labels Labels of data. - * @param numClasses Number of distinct classes in labels. - * @param bucketSize Minimum size of bucket when splitting. - */ -template -DecisionStump::DecisionStump(const MatType& data, - const arma::Row& labels, - const size_t numClasses, - const size_t bucketSize) : - numClasses(numClasses), - bucketSize(bucketSize) -{ - arma::rowvec weights; - Train(data, labels, weights); -} - -/** - * Empty constructor. - */ -template -DecisionStump::DecisionStump() : - numClasses(1), - bucketSize(0), - splitDimension(0), - split(1), - binLabels(1) -{ - split[0] = DBL_MAX; - binLabels[0] = 0; -} - -/** - * Train on the given data and labels. - */ -template -double DecisionStump::Train(const MatType& data, - const arma::Row& labels, - const size_t numClasses, - const size_t bucketSize) -{ - this->numClasses = numClasses; - this->bucketSize = bucketSize; - - // Pass to unweighted training function. - arma::rowvec weights; - return Train(data, labels, weights); -} - -/** - * Train the decision stump on the given data, with the given weights. This - * completely overwrites any previous training data, so after training the - * stump may be completely different. - */ -template -double DecisionStump::Train(const MatType& data, - const arma::Row& labels, - const arma::rowvec& weights, - const size_t numClasses, - const size_t bucketSize) -{ - this->numClasses = numClasses; - this->bucketSize = bucketSize; - - // Pass to weighted training function. - return Train(data, labels, weights); -} - -/** - * Train the decision stump on the given data and labels. - * - * @param data Dataset to train on. - * @param labels Labels for dataset. - * @param UseWeights Whether we need to run a weighted Decision Stump. - */ -template -template -double DecisionStump::Train(const MatType& data, - const arma::Row& labels, - const arma::rowvec& weights) -{ - // If classLabels are not all identical, proceed with training. - size_t bestDim = 0; - double entropy; - const double rootEntropy = CalculateEntropy(labels, weights); - - double gain, bestGain = 0.0; - for (size_t i = 0; i < data.n_rows; ++i) - { - // Go through each dimension of the data. - if (IsDistinct(data.row(i))) - { - // For each dimension with non-identical values, treat it as a potential - // splitting dimension and calculate entropy if split on it. - entropy = SetupSplitDimension(data.row(i), labels, weights); - - gain = rootEntropy - entropy; - // Find the dimension with the best entropy so that the gain is - // maximized. - - // We are maximizing gain, which is what is returned from - // SetupSplitDimension(). - if (gain < bestGain) - { - bestDim = i; - bestGain = gain; - } - } - } - splitDimension = bestDim; - - // Once the splitting column/dimension has been decided, train on it. - TrainOnDim(data.row(splitDimension), labels); - return -bestGain; -} - -/** - * Classification function. After training, classify test, and put the predicted - * classes in predictedLabels. - * - * @param test Testing data or data to classify. - * @param predictedLabels Vector to store the predicted classes after - * classifying test - */ -template -void DecisionStump::Classify(const MatType& test, - arma::Row& predictedLabels) -{ - predictedLabels.set_size(test.n_cols); - for (size_t i = 0; i < test.n_cols; ++i) - { - // Determine which bin the test point falls into. - // Assume first that it falls into the first bin, then proceed through the - // bins until it is known which bin it falls into. - size_t bin = 0; - const double val = test(splitDimension, i); - - while (bin < split.n_elem - 1) - { - if (val < split(bin + 1)) - break; - - ++bin; - } - - predictedLabels(i) = binLabels(bin); - } -} - -/** - * Alternate constructor which copies parameters bucketSize and numClasses - * from an already initiated decision stump, other. It appropriately - * sets the Weight vector. - * - * @param other The other initiated Decision Stump object from - * which we copy the values from. - * @param data The data on which to train this object on. - * @param D Weight vector to use while training. For boosting purposes. - * @param labels The labels of data. - * @param UseWeights Whether we need to run a weighted Decision Stump. - */ -template -DecisionStump::DecisionStump(const DecisionStump<>& other, - const MatType& data, - const arma::Row& labels, - const size_t numClasses, - const arma::rowvec& weights) : - numClasses(numClasses), - bucketSize(other.bucketSize) -{ - Train(data, labels, weights); -} - -/** - * Serialize the decision stump. - */ -template -template -void DecisionStump::serialize(Archive& ar, - const uint32_t /* version */) -{ - // This is straightforward; just serialize all of the members of the class. - // None need special handling. - ar(CEREAL_NVP(numClasses)); - ar(CEREAL_NVP(bucketSize)); - ar(CEREAL_NVP(splitDimension)); - ar(CEREAL_NVP(split)); - ar(CEREAL_NVP(binLabels)); -} - -/** - * Sets up dimension as if it were splitting on it and finds entropy when - * splitting on dimension. - * - * @param dimension A row from the training data, which might be a candidate for - * the splitting dimension. - * @param UseWeights Whether we need to run a weighted Decision Stump. - */ -template -template -double DecisionStump::SetupSplitDimension( - const VecType& dimension, - const arma::Row& labels, - const arma::rowvec& weights) -{ - size_t i, count, begin, end; - double entropy = 0.0; - - // Store the indices of the sorted dimension to build a vector of sorted - // labels. This sort is stable. - arma::uvec sortedIndexDim = arma::stable_sort_index(dimension.t()); - - arma::Row sortedLabels(dimension.n_elem); - arma::rowvec sortedWeights(dimension.n_elem); - - for (i = 0; i < dimension.n_elem; ++i) - { - sortedLabels(i) = labels(sortedIndexDim(i)); - - // Apply weights if necessary. - if (UseWeights) - sortedWeights(i) = weights(sortedIndexDim(i)); - } - - i = 0; - count = 0; - - // This splits the sorted data into buckets of size greater than or equal to - // bucketSize. - while (i < sortedLabels.n_elem) - { - count++; - if (i == sortedLabels.n_elem - 1) - { - // If we're at the end, then don't worry about the bucket size; just take - // this as the last bin. - begin = i - count + 1; - end = i; - - // Use ratioEl to calculate the ratio of elements in this split. - const double ratioEl = ((double) (end - begin + 1) / sortedLabels.n_elem); - - entropy += ratioEl * CalculateEntropy( - sortedLabels.subvec(begin, end), sortedWeights.subvec(begin, end)); - ++i; - } - else if (sortedLabels(i) != sortedLabels(i + 1)) - { - // If we're not at the last element of sortedLabels, then check whether - // count is less than the current bucket size. - if (count < bucketSize) - { - // If it is, then take the minimum bucket size anyways. - // This is where the inpBucketSize comes into use. - // This makes sure there isn't a bucket for every change in labels. - begin = i - count + 1; - end = begin + bucketSize - 1; - - if (end > sortedLabels.n_elem - 1) - end = sortedLabels.n_elem - 1; - } - else - { - // If it is not, then take the bucket size as the value of count. - begin = i - count + 1; - end = i; - } - const double ratioEl = ((double) (end - begin + 1) / sortedLabels.n_elem); - - entropy += ratioEl * CalculateEntropy( - sortedLabels.subvec(begin, end), sortedWeights.subvec(begin, end)); - - i = end + 1; - count = 0; - } - else - ++i; - } - return entropy; -} - -/** - * After having decided the dimension on which to split, train on that - * dimension. - * - * @param dimension Dimension is the dimension decided by the constructor on - * which we now train the decision stump. - */ -template -template -void DecisionStump::TrainOnDim(const VecType& dimension, - const arma::Row& labels) -{ - size_t i, count, begin, end; - - typename MatType::row_type sortedSplitDim = arma::sort(dimension); - arma::uvec sortedSplitIndexDim = arma::stable_sort_index(dimension.t()); - arma::Row sortedLabels(dimension.n_elem); - sortedLabels.fill(0); - - for (i = 0; i < dimension.n_elem; ++i) - sortedLabels(i) = labels(sortedSplitIndexDim(i)); - - arma::rowvec subCols; - double mostFreq; - i = 0; - count = 0; - while (i < sortedLabels.n_elem) - { - count++; - if (i == sortedLabels.n_elem - 1) - { - begin = i - count + 1; - end = i; - - mostFreq = CountMostFreq(sortedLabels.cols(begin, end)); - - split.resize(split.n_elem + 1); - split(split.n_elem - 1) = sortedSplitDim(begin); - binLabels.resize(binLabels.n_elem + 1); - binLabels(binLabels.n_elem - 1) = mostFreq; - - ++i; - } - else if (sortedLabels(i) != sortedLabels(i + 1)) - { - if (count < bucketSize) - { - // Test for different values of bucketSize, especially extreme cases. - begin = i - count + 1; - end = begin + bucketSize - 1; - - if (end > sortedLabels.n_elem - 1) - end = sortedLabels.n_elem - 1; - } - else - { - begin = i - count + 1; - end = i; - } - - // Find the most frequent element in subCols so as to assign a label to - // the bucket of subCols. - mostFreq = CountMostFreq(sortedLabels.cols(begin, end)); - - split.resize(split.n_elem + 1); - split(split.n_elem - 1) = sortedSplitDim(begin); - binLabels.resize(binLabels.n_elem + 1); - binLabels(binLabels.n_elem - 1) = mostFreq; - - i = end + 1; - count = 0; - } - else - ++i; - } - - // Now trim the split matrix so that buckets one after the after which point - // to the same classLabel are merged as one big bucket. - MergeRanges(); -} - -/** - * After the "split" matrix has been set up, merge ranges with identical class - * labels. - */ -template -void DecisionStump::MergeRanges() -{ - for (size_t i = 1; i < split.n_rows; ++i) - { - if (binLabels(i) == binLabels(i - 1)) - { - // Remove this row, as it has the same label as the previous bucket. - binLabels.shed_row(i); - split.shed_row(i); - // Go back to previous row. - i--; - } - } -} - -template -template -double DecisionStump::CountMostFreq(const VecType& subCols) -{ - // We'll create a map of elements and the number of times that each element is - // seen. - std::map countMap; - - for (size_t i = 0; i < subCols.n_elem; ++i) - { - if (countMap.count(subCols[i]) == 0) - countMap[subCols[i]] = 1; - else - ++countMap[subCols[i]]; - } - - // Now find the maximum value. - typename std::map::iterator it = countMap.begin(); - double mostFreq = it->first; - size_t mostFreqCount = it->second; - while (it != countMap.end()) - { - if (it->second >= mostFreqCount) - { - mostFreq = it->first; - mostFreqCount = it->second; - } - - ++it; - } - - return mostFreq; -} - -/** - * Returns 1 if all the values of featureRow are not the same. - * - * @param featureRow The dimension which is checked for identical values. - */ -template -template -int DecisionStump::IsDistinct(const VecType& featureRow) -{ - typename VecType::elem_type val = featureRow(0); - for (size_t i = 1; i < featureRow.n_elem; ++i) - if (val != featureRow(i)) - return 1; - return 0; -} - -/** - * Calculate entropy of dimension. - * - * @param labels Corresponding labels of the dimension. - * @param UseWeights Whether we need to run a weighted Decision Stump. - */ -template -template -double DecisionStump::CalculateEntropy( - const VecType& labels, - const WeightVecType& weights) -{ - double entropy = 0.0; - size_t j; - - arma::rowvec numElem(numClasses); - numElem.fill(0); - - // Variable to accumulate the weight in this subview_row. - double accWeight = 0.0; - // Populate numElem; they are used as helpers to calculate entropy. - - if (UseWeights) - { - for (j = 0; j < labels.n_elem; ++j) - { - numElem(labels(j)) += weights(j); - accWeight += weights(j); - } - - for (j = 0; j < numClasses; ++j) - { - const double p1 = ((double) numElem(j) / accWeight); - - // Instead of using log2(), which is C99 and may not exist on some - // compilers, use std::log(), then use the change-of-base formula to make - // the result correct. - entropy += (p1 == 0) ? 0 : p1 * std::log(p1); - } - } - else - { - for (j = 0; j < labels.n_elem; ++j) - numElem(labels(j))++; - - for (j = 0; j < numClasses; ++j) - { - const double p1 = ((double) numElem(j) / labels.n_elem); - - // Instead of using log2(), which is C99 and may not exist on some - // compilers, use std::log(), then use the change-of-base formula to make - // the result correct. - entropy += (p1 == 0) ? 0 : p1 * std::log(p1); - } - } - - return entropy / std::log(2.0); -} - -} // namespace decision_stump -} // namespace mlpack - -#endif diff --git a/src/mlpack/methods/decision_stump/decision_stump_main.cpp b/src/mlpack/methods/decision_stump/decision_stump_main.cpp deleted file mode 100644 index 463c680dfd..0000000000 --- a/src/mlpack/methods/decision_stump/decision_stump_main.cpp +++ /dev/null @@ -1,209 +0,0 @@ -/** - * @file methods/decision_stump/decision_stump_main.cpp - * @author Udit Saxena - * - * Main executable for the decision stump. - * - * mlpack is free software; you may redistribute it and/or modify it under the - * terms of the 3-clause BSD license. You should have received a copy of the - * 3-clause BSD license along with mlpack. If not, see - * http://www.opensource.org/licenses/BSD-3-Clause for more information. - */ -#include -#include -#include -#include -#include "decision_stump.hpp" - -using namespace mlpack; -using namespace mlpack::decision_stump; -using namespace mlpack::util; -using namespace std; -using namespace arma; - -// Program Name. -BINDING_NAME("Decision Stump"); - -// Short description. -BINDING_SHORT_DESC( - "An implementation of a decision stump, which is a single-level decision " - "tree. Given labeled data, a new decision stump can be trained; or, an " - "existing decision stump can be used to classify points."); - -// Long description. -BINDING_LONG_DESC( - "This program implements a decision stump, which is a single-level decision" - " tree. The decision stump will split on one dimension of the input data, " - "and will split into multiple buckets. The dimension and bins are selected" - " by maximizing the information gain of the split. Optionally, the minimum" - " number of training points in each bin can be specified with the " + - PRINT_PARAM_STRING("bucket_size") + " parameter." - "\n\n" - "The decision stump is parameterized by a splitting dimension and a vector " - "of values that denote the splitting values of each bin." - "\n\n" - "This program enables several applications: a decision tree may be trained " - "or loaded, and then that decision tree may be used to classify a given set" - " of test points. The decision tree may also be saved to a file for later " - "usage." - "\n\n" - "To train a decision stump, training data should be passed with the " + - PRINT_PARAM_STRING("training") + " parameter, and their corresponding " - "labels should be passed with the " + PRINT_PARAM_STRING("labels") + " " - "option. Optionally, if " + PRINT_PARAM_STRING("labels") + " is not " - "specified, the labels are assumed to be the last dimension of the " - "training dataset. The " + PRINT_PARAM_STRING("bucket_size") + " " - "parameter controls the minimum number of training points in each decision " - "stump bucket." - "\n\n" - "For classifying a test set, a decision stump may be loaded with the " + - PRINT_PARAM_STRING("input_model") + " parameter (useful for the situation " - "where a stump has already been trained), and a test set may be specified " - "with the " + PRINT_PARAM_STRING("test") + " parameter. The predicted " - "labels can be saved with the " + PRINT_PARAM_STRING("predictions") + " " - "output parameter." - "\n\n" - "Because decision stumps are trained in batch, retraining does not make " - "sense and thus it is not possible to pass both " + - PRINT_PARAM_STRING("training") + " and " + - PRINT_PARAM_STRING("input_model") + "; instead, simply build a new " - "decision stump with the training data." - "\n\n" - "After training, a decision stump can be saved with the " + - PRINT_PARAM_STRING("output_model") + " output parameter. That stump may " - "later be re-used in subsequent calls to this program (or others)."); - -// See also... -BINDING_SEE_ALSO("Decision tree", "#decision_tree"); -BINDING_SEE_ALSO("Decision stumps on Wikipedia", - "https://en.wikipedia.org/wiki/Decision_stump"); -BINDING_SEE_ALSO("mlpack::decision_stump::DecisionStump class documentation", - "@doxygen/classmlpack_1_1decision__stump_1_1DecisionStump.html"); - -// Datasets we might load. -PARAM_MATRIX_IN("training", "The dataset to train on.", "t"); -PARAM_UROW_IN("labels", "Labels for the training set. If not specified, the " - "labels are assumed to be the last row of the training data.", "l"); -PARAM_MATRIX_IN("test", "A dataset to calculate predictions for.", "T"); - -// Output. -PARAM_UROW_OUT("predictions", "The output matrix that will hold the " - "predicted labels for the test set.", "p"); - -/** - * This is the structure that actually saves to disk. We have to save the - * label mappings, too, otherwise everything we load at test time in a future - * run will end up being borked. - */ -struct DSModel -{ - //! The mappings. - arma::Col mappings; - //! The stump. - DecisionStump<> stump; - - //! Serialize the model. - template - void serialize(Archive& ar, const uint32_t /* version */) - { - ar(CEREAL_NVP(mappings)); - ar(CEREAL_NVP(stump)); - } -}; - -// We may load or save a model. -PARAM_MODEL_IN(DSModel, "input_model", "Decision stump model to " - "load.", "m"); -PARAM_MODEL_OUT(DSModel, "output_model", "Output decision stump model to save.", - "M"); - -PARAM_INT_IN("bucket_size", "The minimum number of training points in each " - "decision stump bucket.", "b", 6); - -static void mlpackMain() -{ - // Check that the parameters are reasonable. - RequireOnlyOnePassed({ "training", "input_model" }, true); - RequireAtLeastOnePassed({ "output_model", "predictions" }, false, "no results" - " will be saved"); - - RequireParamValue("bucket_size", [](int x) { return x > 0; }, true, - "bucket size must be positive"); - - ReportIgnoredParam({{ "test", false }}, "predictions"); - - Log::Warn << "DecisionStump is deprecated and will be removed in mlpack " - << "4.0.0. Please use DecisionTree instead with the maximum tree " - << "depth option set to 1 (that will produce a stump)." - << std::endl; - - // We must either load a model, or train a new stump. - DSModel* model; - if (IO::HasParam("training")) - { - model = new DSModel(); - mat trainingData = std::move(IO::GetParam("training")); - - // Load labels, if necessary. - Row labelsIn; - if (IO::HasParam("labels")) - { - labelsIn = std::move(IO::GetParam>("labels")); - } - else - { - // Extract the labels as the last - Log::Info << "Using the last dimension of training set as labels." - << endl; - - labelsIn = arma::conv_to>::from( - trainingData.row(trainingData.n_rows - 1)); - trainingData.shed_row(trainingData.n_rows - 1); - } - - // Normalize the labels. - Row labels; - data::NormalizeLabels(labelsIn, labels, model->mappings); - - const size_t bucketSize = IO::GetParam("bucket_size"); - const size_t classes = labels.max() + 1; - - Timer::Start("training"); - model->stump.Train(trainingData, labels, classes, bucketSize); - Timer::Stop("training"); - } - else - { - model = IO::GetParam("input_model"); - } - - // Now, do we need to do any testing? - if (IO::HasParam("test")) - { - // Load the test file. - mat testingData = std::move(IO::GetParam("test")); - - if (testingData.n_rows <= model->stump.SplitDimension()) - Log::Fatal << "Test data dimensionality (" << testingData.n_rows << ") " - << "is too low; the trained stump requires at least " - << model->stump.SplitDimension() << " dimensions!" << endl; - - Row predictedLabels(testingData.n_cols); - Timer::Start("testing"); - model->stump.Classify(testingData, predictedLabels); - Timer::Stop("testing"); - - // Denormalize predicted labels, if we want to save them. - if (IO::HasParam("predictions")) - { - Row actualLabels; - data::RevertLabels(predictedLabels, model->mappings, actualLabels); - - // Save the predicted labels as output. - IO::GetParam>("predictions") = std::move(actualLabels); - } - } - - // Save the model, if desired. - IO::GetParam("output_model") = model; -} diff --git a/src/mlpack/tests/decision_stump_test.cpp b/src/mlpack/tests/decision_stump_test.cpp deleted file mode 100644 index d9d633fe3e..0000000000 --- a/src/mlpack/tests/decision_stump_test.cpp +++ /dev/null @@ -1,412 +0,0 @@ -/** - * @file tests/decision_stump_test.cpp - * @author Udit Saxena - * - * Tests for DecisionStump class. - * - * mlpack is free software; you may redistribute it and/or modify it under the - * terms of the 3-clause BSD license. You should have received a copy of the - * 3-clause BSD license along with mlpack. If not, see - * http://www.opensource.org/licenses/BSD-3-Clause for more information. - */ -#include -#include - -#include "catch.hpp" - -using namespace mlpack; -using namespace mlpack::decision_stump; -using namespace arma; -using namespace mlpack::distribution; - -/** - * This tests handles the case wherein only one class exists in the input - * labels. It checks whether the only class supplied was the only class - * predicted. - */ -TEST_CASE("OneClass", "[DecisionStumpTest]") -{ - const size_t numClasses = 2; - const size_t inpBucketSize = 6; - - mat trainingData; - trainingData = { { 2.4, 3.8, 3.8 }, - { 1, 1, 2 }, - { 1.3, 1.9, 1.3 } }; - - // No need to normalize labels here. - Mat labelsIn; - labelsIn = { 1, 1, 1 }; - - mat testingData; - testingData = { 2.4, 2.5, 2.6 }; - - DecisionStump<> ds(trainingData, labelsIn.row(0), numClasses, inpBucketSize); - - Row predictedLabels; - ds.Classify(testingData, predictedLabels); - - for (size_t i = 0; i < predictedLabels.size(); ++i) - REQUIRE(predictedLabels(i) == 1); -} - -/** - * This tests whether the entropy is being correctly calculated by checking the - * correct value of the splitting column value. This test is for an - * inpBucketSize of 4 and the correct value of the splitting dimension is 0. - */ -TEST_CASE("CorrectDimensionChosen", "[DecisionStumpTest]") -{ - const size_t numClasses = 2; - const size_t inpBucketSize = 4; - - // This dataset comes from Chapter 6 of the book "Data Mining: Concepts, - // Models, Methods, and Algorithms" (2nd Edition) by Mehmed Kantardzic. It is - // found on page 176 (and a description of the correct splitting dimension is - // given below that). - mat trainingData; - trainingData = { { 0, 0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 2, 2 }, - { 70, 90, 85, 95, 70, 90, 78, 65, 75, 80, 70, 80, 80, 96 }, - { 1, 1, 0, 0, 0, 1, 0, 1, 0, 1, 1, 0, 0, 0 } }; - - // No need to normalize labels here. - Mat labelsIn; - labelsIn = { 0, 1, 1, 1, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0 }; - - DecisionStump<> ds(trainingData, labelsIn.row(0), numClasses, inpBucketSize); - - // Only need to check the value of the splitting column, no need of - // classification. - REQUIRE(ds.SplitDimension() == 0); -} - -/** - * This tests for the classification: - * if testinput < 0 - class 0 - * if testinput > 0 - class 1 - * An almost perfect split on zero. - */ -TEST_CASE("PerfectSplitOnZero", "[DecisionStumpTest]") -{ - const size_t numClasses = 2; - const size_t inpBucketSize = 2; - - mat trainingData; - trainingData = { -1, 1, -2, 2, -3, 3 }; - - // No need to normalize labels here. - Mat labelsIn; - labelsIn = { 0, 1, 0, 1, 0, 1 }; - - mat testingData; - testingData = { -4, 7, -7, -5, 6 }; - - DecisionStump<> ds(trainingData, labelsIn.row(0), numClasses, inpBucketSize); - - Row predictedLabels; - ds.Classify(testingData, predictedLabels); - - REQUIRE(predictedLabels(0, 0) == 0); - REQUIRE(predictedLabels(0, 1) == 1); - REQUIRE(predictedLabels(0, 2) == 0); - REQUIRE(predictedLabels(0, 3) == 0); - REQUIRE(predictedLabels(0, 4) == 1); -} - -/** - * This tests the binning function for the case when a dataset with cardinality - * of input < inpBucketSize is provided. - */ -TEST_CASE("BinningTesting", "[DecisionStumpTest]") -{ - const size_t numClasses = 2; - const size_t inpBucketSize = 10; - - mat trainingData; - trainingData = { -1, 1, -2, 2, -3, 3, -4 }; - - // No need to normalize labels here. - Mat labelsIn; - labelsIn = { 0, 1, 0, 1, 0, 1, 0 }; - - mat testingData; - testingData = {5}; - - DecisionStump<> ds(trainingData, labelsIn.row(0), numClasses, inpBucketSize); - - Row predictedLabels; - ds.Classify(testingData, predictedLabels); - - REQUIRE(predictedLabels(0, 0) == 0); -} - -/** - * This is a test for the case when non-overlapping, multiple classes are - * provided. It tests for a perfect split due to the non-overlapping nature of - * the input classes. - */ -TEST_CASE("PerfectMultiClassSplit", "[DecisionStumpTest]") -{ - const size_t numClasses = 4; - const size_t inpBucketSize = 3; - - mat trainingData; - trainingData = { -8, -7, -6, -5, -4, -3, -2, -1, 0, 1, 2, 3, 4, 5, 6, 7 }; - - // No need to normalize labels here. - Mat labelsIn; - labelsIn = { 0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3 }; - - mat testingData; - testingData = { -6.1, -2.1, 1.1, 5.1 }; - - DecisionStump<> ds(trainingData, labelsIn.row(0), numClasses, inpBucketSize); - - Row predictedLabels; - ds.Classify(testingData, predictedLabels); - - REQUIRE(predictedLabels(0, 0) == 0); - REQUIRE(predictedLabels(0, 1) == 1); - REQUIRE(predictedLabels(0, 2) == 2); - REQUIRE(predictedLabels(0, 3) == 3); -} - -/** - * This test is for the case when reasonably overlapping, multiple classes are - * provided in the input label set. It tests whether classification takes place - * with a reasonable amount of error due to the overlapping nature of input - * classes. - */ -TEST_CASE("MultiClassSplit", "[DecisionStumpTest]") -{ - const size_t numClasses = 3; - const size_t inpBucketSize = 3; - - mat trainingData; - trainingData = { -7, -6, -5, -4, -3, -2, -1, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, - 10 }; - - // No need to normalize labels here. - Mat labelsIn; - labelsIn = { 0, 0, 0, 0, 1, 1, 0, 0, 1, 1, 1, 2, 1, 2, 2, 2, 2, 2 }; - - - mat testingData; - testingData = { -6.1, -5.9, -2.1, -0.7, 2.5, 4.7, 7.2, 9.1 }; - - DecisionStump<> ds(trainingData, labelsIn.row(0), numClasses, inpBucketSize); - - Row predictedLabels; - ds.Classify(testingData, predictedLabels); - - REQUIRE(predictedLabels(0, 0) == 0); - REQUIRE(predictedLabels(0, 1) == 0); - REQUIRE(predictedLabels(0, 2) == 1); - REQUIRE(predictedLabels(0, 3) == 1); - REQUIRE(predictedLabels(0, 4) == 1); - REQUIRE(predictedLabels(0, 5) == 1); - REQUIRE(predictedLabels(0, 6) == 2); - REQUIRE(predictedLabels(0, 7) == 2); -} - -/** - * This tests that the decision stump can learn a good split on a dataset with - * four dimensions that have progressing levels of separation. - */ -TEST_CASE("DimensionSelectionTest", "[DecisionStumpTest]") -{ - const size_t numClasses = 2; - const size_t inpBucketSize = 2500; - - arma::mat dataset(4, 5000); - - // The most separable dimension. - GaussianDistribution g1("-5", "1"); - GaussianDistribution g2("5", "1"); - - for (size_t i = 0; i < 2500; ++i) - { - arma::vec tmp = g1.Random(); - dataset(1, i) = tmp[0]; - } - for (size_t i = 2500; i < 5000; ++i) - { - arma::vec tmp = g2.Random(); - dataset(1, i) = tmp[0]; - } - - g1 = GaussianDistribution("-3", "1"); - g2 = GaussianDistribution("3", "1"); - - for (size_t i = 0; i < 2500; ++i) - { - arma::vec tmp = g1.Random(); - dataset(3, i) = tmp[0]; - } - for (size_t i = 2500; i < 5000; ++i) - { - arma::vec tmp = g2.Random(); - dataset(3, i) = tmp[0]; - } - - g1 = GaussianDistribution("-1", "1"); - g2 = GaussianDistribution("1", "1"); - - for (size_t i = 0; i < 2500; ++i) - { - arma::vec tmp = g1.Random(); - dataset(0, i) = tmp[0]; - } - for (size_t i = 2500; i < 5000; ++i) - { - arma::vec tmp = g2.Random(); - dataset(0, i) = tmp[0]; - } - - // Not separable at all. - g1 = GaussianDistribution("0", "1"); - g2 = GaussianDistribution("0", "1"); - - for (size_t i = 0; i < 2500; ++i) - { - arma::vec tmp = g1.Random(); - dataset(2, i) = tmp[0]; - } - for (size_t i = 2500; i < 5000; ++i) - { - arma::vec tmp = g2.Random(); - dataset(2, i) = tmp[0]; - } - - // Generate the labels. - arma::Row labels(5000); - for (size_t i = 0; i < 2500; ++i) - labels[i] = 0; - for (size_t i = 2500; i < 5000; ++i) - labels[i] = 1; - - // Now create a decision stump. - DecisionStump<> ds(dataset, labels, numClasses, inpBucketSize); - - // Make sure it split on the dimension that is most separable. - REQUIRE(ds.SplitDimension() == 1); - - // Make sure every bin below -1 classifies as label 0, and every bin above 1 - // classifies as label 1 (What happens in [-1, 1] isn't that big a deal.). - for (size_t i = 0; i < ds.Split().n_elem; ++i) - { - if (ds.Split()[i] <= -3.0) - REQUIRE(ds.BinLabels()[i] == 0); - else if (ds.Split()[i] >= 3.0) - REQUIRE(ds.BinLabels()[i] == 1); - } -} - -/** - * Ensure that the default constructor works and that it classifies things as 0 - * always. - */ -TEST_CASE("EmptyConstructorTest", "[DecisionStumpTest]") -{ - DecisionStump<> d; - - arma::mat data = arma::randu(3, 10); - arma::Row labels; - - d.Classify(data, labels); - - for (size_t i = 0; i < 10; ++i) - REQUIRE(labels[i] == 0); - - // Now train on another dataset and make sure something kind of makes sense. - mat trainingData; - trainingData = { -7, -6, -5, -4, -3, -2, -1, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, - 10 }; - - // No need to normalize labels here. - Mat labelsIn; - labelsIn = { 0, 0, 0, 0, 1, 1, 0, 0, 1, 1, 1, 2, 1, 2, 2, 2, 2, 2 }; - - - mat testingData; - testingData = { -6.1, -5.9, -2.1, -0.7, 2.5, 4.7, 7.2, 9.1 }; - - DecisionStump<> ds(trainingData, labelsIn.row(0), 4, 3); - - Row predictedLabels(testingData.n_cols); - ds.Classify(testingData, predictedLabels); - - REQUIRE(predictedLabels(0, 0) == 0); - REQUIRE(predictedLabels(0, 1) == 0); - REQUIRE(predictedLabels(0, 2) == 1); - REQUIRE(predictedLabels(0, 3) == 1); - REQUIRE(predictedLabels(0, 4) == 1); - REQUIRE(predictedLabels(0, 5) == 1); - REQUIRE(predictedLabels(0, 6) == 2); - REQUIRE(predictedLabels(0, 7) == 2); -} - -/** - * Ensure that a matrix holding ints can be trained. The bigger issue here is - * just compilation. - */ -TEST_CASE("IntTest", "[DecisionStumpTest]") -{ - // Train on a dataset and make sure something kind of makes sense. - imat trainingData; - trainingData = { -7, -6, -5, -4, -3, -2, -1, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, - 10 }; - - // No need to normalize labels here. - Mat labelsIn; - labelsIn = { 0, 0, 0, 0, 1, 1, 0, 0, 1, 1, 1, 2, 1, 2, 2, 2, 2, 2 }; - - DecisionStump ds(trainingData, labelsIn.row(0), 4, 3); - - imat testingData; - testingData = { -6, -6, -2, -1, 3, 5, 7, 9 }; - - arma::Row predictedLabels; - ds.Classify(testingData, predictedLabels); - - REQUIRE(predictedLabels(0, 0) == 0); - REQUIRE(predictedLabels(0, 1) == 0); - REQUIRE(predictedLabels(0, 2) == 1); - REQUIRE(predictedLabels(0, 3) == 1); - REQUIRE(predictedLabels(0, 4) == 1); - REQUIRE(predictedLabels(0, 5) == 1); - REQUIRE(predictedLabels(0, 6) == 2); - REQUIRE(predictedLabels(0, 7) == 2); -} - -/** - * Test that DecisionStump::Train() returns finite gain. - */ -TEST_CASE("DecisionStumpTrainReturnEntropy", "[DecisionStumpTest]") -{ - const size_t numClasses = 2; - const size_t inpBucketSize = 2; - - mat trainingData; - trainingData = { -1, 1, -2, 2, -3, 3 }; - - // No need to normalize labels here. - Mat labelsIn; - labelsIn = { 0, 1, 0, 1, 0, 1 }; - - arma::Row weights = arma::ones>(labelsIn.n_elem); - - // Train a simple decision stump without weights. - DecisionStump<> ds; - double gain = ds.Train(trainingData, labelsIn.row(0), numClasses, - inpBucketSize); - - REQUIRE(std::isfinite(gain) == true); - - // Train decision stump with weights. - DecisionStump<> wds; - gain = wds.Train(trainingData, labelsIn.row(0), weights, numClasses, - inpBucketSize); - - REQUIRE(std::isfinite(gain) == true); -}