From b1b149282188ca6e1c9eaaf123feafc483aa9b72 Mon Sep 17 00:00:00 2001 From: theJonan Date: Fri, 3 Feb 2017 16:16:06 +0200 Subject: [PATCH 01/39] - First version of path printing. --- src/mlpack/methods/det/det_main.cpp | 173 +++++++++++++++++++++-- src/mlpack/methods/det/dt_utils.hpp | 4 +- src/mlpack/methods/det/dt_utils_impl.hpp | 12 +- src/mlpack/methods/det/dtree.hpp | 14 ++ src/mlpack/methods/det/dtree_impl.hpp | 36 ++++- 5 files changed, 221 insertions(+), 18 deletions(-) diff --git a/src/mlpack/methods/det/det_main.cpp b/src/mlpack/methods/det/det_main.cpp index 7323d9bb01..900e1c779d 100644 --- a/src/mlpack/methods/det/det_main.cpp +++ b/src/mlpack/methods/det/det_main.cpp @@ -57,6 +57,15 @@ PARAM_STRING_OUT("test_set_estimates_file", "The file in which to output the " PARAM_STRING_OUT("vi_file", "The file to output the variable importance values " "for each feature.", "i"); +// Tagging and path printing options +PARAM_STRING_IN("path_format", "The format of path printing - lr|idlr|ldid", + "p", "lr"); + +PARAM_STRING_IN("tag_counters", "The file to output tag counters.", "c", ""); + +PARAM_STRING_OUT("tag_file", "The file to output the tags (and possibly paths) " + " for each sample in the test set.", "g"); + // Parameters for the training algorithm. PARAM_INT_IN("folds", "The number of folds of cross-validation to perform for the " "estimation (0 is LOOCV)", "f", 10); @@ -72,6 +81,45 @@ PARAM_FLAG("volume_regularization", "This flag gives the used the option to use" "penalize low volume leaves.", "R"); */ + +class PathCacher +{ +public: + enum PathFormat + { + FormatLR, + FormatLR_ID, + FormatID_LR + }; + + template + PathCacher(PathFormat fmt, DTree* tree); + + ~PathCacher(); + + template + void Enter(const DTree* node, const DTree* parent); + + template + void Leave(const DTree* node, const DTree* parent); + + const std::string& PathFor(int tag) const; + + size_t NumLeaves() const { return numLeaves; } + +private: + typedef std::list > PathType; + + PathType path; + PathFormat format; + + int numLeaves; + std::string* pathCache; + + std::string BuildString(); +}; + + int main(int argc, char *argv[]) { CLI::ParseCommandLine(argc, argv); @@ -162,17 +210,67 @@ int main(int argc, char *argv[]) { arma::mat testData; data::Load(testFile, testData, true); + if (CLI::HasParam("test_set_estimates_file")) + { + // Compute test set densities. + Timer::Start("det_test_set_estimation"); + arma::rowvec testDensities(testData.n_cols); + for (size_t i = 0; i < testData.n_cols; i++) + testDensities[i] = tree->ComputeValue(testData.unsafe_col(i)); + Timer::Stop("det_test_set_estimation"); - // Compute test set densities. - Timer::Start("det_test_set_estimation"); - arma::rowvec testDensities(testData.n_cols); - for (size_t i = 0; i < testData.n_cols; i++) - testDensities[i] = tree->ComputeValue(testData.unsafe_col(i)); - Timer::Stop("det_test_set_estimation"); + if (CLI::GetParam("test_set_estimates_file") != "") + data::Save(CLI::GetParam("test_set_estimates_file"), + testDensities); + } + + if (CLI::HasParam("tag_file")) + { + const string tagFile = CLI::GetParam("tag_file"); + std::ofstream ofs; + ofs.open(tagFile, std::ofstream::out); + + arma::Row counters; - if (CLI::GetParam("test_set_estimates_file") != "") - data::Save(CLI::GetParam("test_set_estimates_file"), - testDensities); + Timer::Start("det_test_set_tagging"); + if (!ofs.is_open()) + { + Log::Warn << "Unable to open file '" << tagFile + << "' to save tag membership info." + << std::endl; + } + else if (CLI::HasParam("path_format")) + { + PathCacher path(PathCacher::FormatLR, tree); + counters.zeros(path.NumLeaves()); + + for (size_t i = 0; i < testData.n_cols; ++i) + { + int tag = tree->FindBucket(testData.unsafe_col(i)); + counters(tag) += 1; + + ofs << tag << " " << path.PathFor(tag) << std::endl; + } + } + else + { + int numLeaves = tree->TagTree(); + counters.zeros(numLeaves); + + for (size_t i = 0; i < testData.n_cols; ++i) + { + int tag = tree->FindBucket(testData.unsafe_col(i)); + counters(tag) += 1; + ofs << tag << std::endl; + } + } + + if (CLI::GetParam("tag_counters") != "") + data::Save(CLI::GetParam("tag_counters"), counters); + + Timer::Stop("det_test_set_tagging"); + ofs.close(); + } } // Print variable importance. @@ -186,3 +284,60 @@ int main(int argc, char *argv[]) delete tree; } + + +template +PathCacher::PathCacher(PathCacher::PathFormat fmt, DTree* dtree) : format(fmt) +{ + numLeaves = dtree->TagTree(); + pathCache = new std::string [numLeaves]; + assert(!!pathCache); + dtree->EnumerateTree(*this); +} + +PathCacher::~PathCacher() +{ + delete[] pathCache; +} + +template +void PathCacher::Enter(const DTree* node, const DTree* parent) +{ + if (parent == nullptr) + return; + + int tag = node->BucketTag(); + + path.push_back(PathType::value_type(parent->Left() == node, tag)); + if (tag >= 0) + pathCache[tag] = BuildString(); +} + +template +void PathCacher::Leave(const DTree* , const DTree* parent) +{ + if (parent != nullptr) + path.pop_back(); +} + +std::string PathCacher::BuildString() +{ + std::string str(""); + for (PathType::iterator it = path.begin(); it != path.end(); it++) + { + switch (format) + { + case FormatLR: str += it->first ? "L" : "R"; break; + default: + assert(0); + } + } + + return str; +} + +const std::string& PathCacher::PathFor(int tag) const +{ + assert(tag >= 0 && tag < numLeaves ); + return pathCache[tag]; +} diff --git a/src/mlpack/methods/det/dt_utils.hpp b/src/mlpack/methods/det/dt_utils.hpp index e8456006d6..ff1cb10471 100644 --- a/src/mlpack/methods/det/dt_utils.hpp +++ b/src/mlpack/methods/det/dt_utils.hpp @@ -35,8 +35,8 @@ void PrintLeafMembership(DTree* dtree, const MatType& data, const arma::Mat& labels, const size_t numClasses, - const std::string leafClassMembershipFile = ""); - + const std::string& leafClassMembershipFile = ""); + /** * Print the variable importance of each dimension of a density estimation tree. * Optionally, pass the name of a file to print this information to (otherwise diff --git a/src/mlpack/methods/det/dt_utils_impl.hpp b/src/mlpack/methods/det/dt_utils_impl.hpp index e9e1c61401..a9286b58bc 100644 --- a/src/mlpack/methods/det/dt_utils_impl.hpp +++ b/src/mlpack/methods/det/dt_utils_impl.hpp @@ -18,15 +18,15 @@ namespace mlpack { namespace det { -template -void PrintLeafMembership(DTree* dtree, +template +void PrintLeafMembership(DTree* dtree, const MatType& data, const arma::Mat& labels, const size_t numClasses, - const std::string leafClassMembershipFile) + const std::string& leafClassMembershipFile) { // Tag the leaves with numbers. - TagType numLeaves = dtree->TagTree(); + int numLeaves = dtree->TagTree(); arma::Mat table(numLeaves, (numClasses + 1)); table.zeros(); @@ -34,7 +34,7 @@ void PrintLeafMembership(DTree* dtree, for (size_t i = 0; i < data.n_cols; i++) { const typename MatType::vec_type testPoint = data.unsafe_col(i); - const TagType leafTag = dtree->FindBucket(testPoint); + const int leafTag = dtree->FindBucket(testPoint); const size_t label = labels[i]; table(leafTag, label) += 1; } @@ -65,7 +65,7 @@ void PrintLeafMembership(DTree* dtree, return; } - + template void PrintVariableImportance(const DTree* dtree, const std::string viFile) diff --git a/src/mlpack/methods/det/dtree.hpp b/src/mlpack/methods/det/dtree.hpp index d910af7167..7631bee162 100644 --- a/src/mlpack/methods/det/dtree.hpp +++ b/src/mlpack/methods/det/dtree.hpp @@ -161,6 +161,20 @@ class DTree * @param tag Tag for the next leaf; leave at 0 for the initial call. */ TagType TagTree(const TagType& tag = 0); + + + /** + * Traverses all nodes of the tree, including the inner ones. On each node + * two methods of the `enumer` are called: + * + * Enter(DTree* node, DTree* parent); + * Leave(Dtree* node, DTree* parent); + * + * @param walker An instance of custom class, receiver of the enumeration. + */ + template + void EnumerateTree(Walker& walker) const; + /** * Return the tag of the leaf containing the query. This is useful for diff --git a/src/mlpack/methods/det/dtree_impl.hpp b/src/mlpack/methods/det/dtree_impl.hpp index 5625006504..06abf07e6c 100644 --- a/src/mlpack/methods/det/dtree_impl.hpp +++ b/src/mlpack/methods/det/dtree_impl.hpp @@ -760,13 +760,47 @@ TagType DTree::TagTree(const TagType& tag) } } +// Enumerate the nodes of the tree. +template +template +void DTree::EnumerateTree(Walker& walker) const +{ + if (root == 1) + walker.Enter(this, (const DTree*)nullptr); + + if (subtreeLeaves > 1) + { + // walk the left ... + walker.Enter(left, this); + left->EnumerateTree(walker); + walker.Leave(left, this); + + // ... and the right. + walker.Enter(right, this); + right->EnumerateTree(walker); + walker.Leave(right, this); + } + + if (root == 1) + walker.Leave(this, (const DTree*)nullptr); + +} + template TagType DTree::FindBucket(const VecType& query) const { Log::Assert(query.n_elem == maxVals.n_elem); - if (subtreeLeaves == 1) // If we are a leaf... + if (root == 1) // If we are the root... + { + // Check if the query is within range. + if (!WithinRange(query)) + return bucketTag; + } + + // If we are a leaf... + if (subtreeLeaves == 1) { return bucketTag; } From c102205e0db3bed3b625ade008dfa94ad64c84ca Mon Sep 17 00:00:00 2001 From: theJonan Date: Fri, 3 Feb 2017 17:44:58 +0200 Subject: [PATCH 02/39] - Small fix. --- src/mlpack/methods/det/det_main.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mlpack/methods/det/det_main.cpp b/src/mlpack/methods/det/det_main.cpp index 900e1c779d..d214895a02 100644 --- a/src/mlpack/methods/det/det_main.cpp +++ b/src/mlpack/methods/det/det_main.cpp @@ -61,7 +61,7 @@ PARAM_STRING_OUT("vi_file", "The file to output the variable importance values " PARAM_STRING_IN("path_format", "The format of path printing - lr|idlr|ldid", "p", "lr"); -PARAM_STRING_IN("tag_counters", "The file to output tag counters.", "c", ""); +PARAM_STRING_OUT("tag_counters", "The file to output tag counters.", "c"); PARAM_STRING_OUT("tag_file", "The file to output the tags (and possibly paths) " " for each sample in the test set.", "g"); From 94b32766e06427bea96a10580ef93f3b99b467a8 Mon Sep 17 00:00:00 2001 From: theJonan Date: Fri, 3 Feb 2017 18:06:02 +0200 Subject: [PATCH 03/39] - Parallelism added to tag/path computation. --- src/mlpack/methods/det/det_main.cpp | 38 ++++++++++++++++++++++++----- 1 file changed, 32 insertions(+), 6 deletions(-) diff --git a/src/mlpack/methods/det/det_main.cpp b/src/mlpack/methods/det/det_main.cpp index d214895a02..70afb966cd 100644 --- a/src/mlpack/methods/det/det_main.cpp +++ b/src/mlpack/methods/det/det_main.cpp @@ -215,8 +215,18 @@ int main(int argc, char *argv[]) // Compute test set densities. Timer::Start("det_test_set_estimation"); arma::rowvec testDensities(testData.n_cols); + +#ifdef _WIN32 + #pragma omp parallel for default(shared) + for (intmax_t i = 0; i < testData.n_cols; i++) +#else + #pragma omp parallel for default(shared) for (size_t i = 0; i < testData.n_cols; i++) +#endif + { testDensities[i] = tree->ComputeValue(testData.unsafe_col(i)); + } + Timer::Stop("det_test_set_estimation"); if (CLI::GetParam("test_set_estimates_file") != "") @@ -244,12 +254,20 @@ int main(int argc, char *argv[]) PathCacher path(PathCacher::FormatLR, tree); counters.zeros(path.NumLeaves()); - for (size_t i = 0; i < testData.n_cols; ++i) +#ifdef _WIN32 + #pragma omp parallel for default(shared) + for (intmax_t i = 0; i < testData.n_cols; i++) +#else + #pragma omp parallel for default(shared) + for (size_t i = 0; i < testData.n_cols; i++) +#endif { - int tag = tree->FindBucket(testData.unsafe_col(i)); - counters(tag) += 1; + const int tag = tree->FindBucket(testData.unsafe_col(i)); ofs << tag << " " << path.PathFor(tag) << std::endl; + + #pragma omp critical (DTreeCounterUpdate) + counters(tag) += 1; } } else @@ -257,11 +275,19 @@ int main(int argc, char *argv[]) int numLeaves = tree->TagTree(); counters.zeros(numLeaves); - for (size_t i = 0; i < testData.n_cols; ++i) +#ifdef _WIN32 + #pragma omp parallel for default(shared) + for (intmax_t i = 0; i < testData.n_cols; i++) +#else + #pragma omp parallel for default(shared) + for (size_t i = 0; i < testData.n_cols; i++) +#endif { - int tag = tree->FindBucket(testData.unsafe_col(i)); - counters(tag) += 1; + const int tag = tree->FindBucket(testData.unsafe_col(i)); ofs << tag << std::endl; + + #pragma omp critical (DTreeCounterUpdate) + counters(tag) += 1; } } From 0f0546ce002b37be67c9a36e9bcccef8e4edbf28 Mon Sep 17 00:00:00 2001 From: theJonan Date: Fri, 3 Feb 2017 20:04:03 +0200 Subject: [PATCH 04/39] - Parallelism removed on estimations (logN) --- src/mlpack/methods/det/det_main.cpp | 27 ++------------------------- 1 file changed, 2 insertions(+), 25 deletions(-) diff --git a/src/mlpack/methods/det/det_main.cpp b/src/mlpack/methods/det/det_main.cpp index 70afb966cd..c045ae9cbd 100644 --- a/src/mlpack/methods/det/det_main.cpp +++ b/src/mlpack/methods/det/det_main.cpp @@ -216,16 +216,8 @@ int main(int argc, char *argv[]) Timer::Start("det_test_set_estimation"); arma::rowvec testDensities(testData.n_cols); -#ifdef _WIN32 - #pragma omp parallel for default(shared) - for (intmax_t i = 0; i < testData.n_cols; i++) -#else - #pragma omp parallel for default(shared) for (size_t i = 0; i < testData.n_cols; i++) -#endif - { testDensities[i] = tree->ComputeValue(testData.unsafe_col(i)); - } Timer::Stop("det_test_set_estimation"); @@ -254,20 +246,12 @@ int main(int argc, char *argv[]) PathCacher path(PathCacher::FormatLR, tree); counters.zeros(path.NumLeaves()); -#ifdef _WIN32 - #pragma omp parallel for default(shared) - for (intmax_t i = 0; i < testData.n_cols; i++) -#else - #pragma omp parallel for default(shared) for (size_t i = 0; i < testData.n_cols; i++) -#endif { const int tag = tree->FindBucket(testData.unsafe_col(i)); - ofs << tag << " " << path.PathFor(tag) << std::endl; - - #pragma omp critical (DTreeCounterUpdate) counters(tag) += 1; + ofs << tag << " " << path.PathFor(tag) << std::endl; } } else @@ -275,18 +259,11 @@ int main(int argc, char *argv[]) int numLeaves = tree->TagTree(); counters.zeros(numLeaves); -#ifdef _WIN32 - #pragma omp parallel for default(shared) - for (intmax_t i = 0; i < testData.n_cols; i++) -#else - #pragma omp parallel for default(shared) for (size_t i = 0; i < testData.n_cols; i++) -#endif { const int tag = tree->FindBucket(testData.unsafe_col(i)); - ofs << tag << std::endl; - #pragma omp critical (DTreeCounterUpdate) + ofs << tag << std::endl; counters(tag) += 1; } } From c41aa8878b7dbfc3f6160a129dcc6b5866034b12 Mon Sep 17 00:00:00 2001 From: theJonan Date: Tue, 7 Feb 2017 00:03:45 +0200 Subject: [PATCH 05/39] - Fixes and small improvements. --- src/mlpack/methods/det/dt_utils_impl.hpp | 5 +++++ src/mlpack/methods/det/dtree.hpp | 5 +++++ src/mlpack/methods/det/dtree_impl.hpp | 23 ++++++++++++++++++++++- 3 files changed, 32 insertions(+), 1 deletion(-) diff --git a/src/mlpack/methods/det/dt_utils_impl.hpp b/src/mlpack/methods/det/dt_utils_impl.hpp index a9286b58bc..18df1a5312 100644 --- a/src/mlpack/methods/det/dt_utils_impl.hpp +++ b/src/mlpack/methods/det/dt_utils_impl.hpp @@ -116,6 +116,7 @@ DTree* Trainer(MatType& dataset, // Initialize the tree. DTree dtree(dataset); + Timer::Start("tree_growing"); // Prepare to grow the tree... arma::Col oldFromNew(dataset.n_cols); for (size_t i = 0; i < oldFromNew.n_elem; i++) @@ -129,6 +130,7 @@ DTree* Trainer(MatType& dataset, double alpha = dtree.Grow(newDataset, oldFromNew, useVolumeReg, maxLeafSize, minLeafSize); + Timer::Stop("tree_growing"); Log::Info << dtree.SubtreeLeaves() << " leaf nodes in the tree using full " << "dataset; minimum alpha: " << alpha << "." << std::endl; @@ -154,6 +156,8 @@ DTree* Trainer(MatType& dataset, outfile.close(); } + Timer::Start("prunning_sequence"); + // Sequentially prune and save the alpha values and the values of c_t^2 * r_t. std::vector > prunedSequence; while (dtree.SubtreeLeaves() > 1) @@ -175,6 +179,7 @@ DTree* Trainer(MatType& dataset, std::pair treeSeq(oldAlpha, dtree.SubtreeLeavesLogNegError()); prunedSequence.push_back(treeSeq); + Timer::Stop("prunning_sequence"); Log::Info << prunedSequence.size() << " trees in the sequence; maximum alpha:" << " " << oldAlpha << "." << std::endl; diff --git a/src/mlpack/methods/det/dtree.hpp b/src/mlpack/methods/det/dtree.hpp index 7631bee162..a2a2233e1f 100644 --- a/src/mlpack/methods/det/dtree.hpp +++ b/src/mlpack/methods/det/dtree.hpp @@ -57,6 +57,11 @@ class DTree * Create an empty density estimation tree. */ DTree(); + + /** + * Create a copy of an existing tree. + */ + DTree(const DTree& tree); /** * Create a density estimation tree with the given bounds and the given number diff --git a/src/mlpack/methods/det/dtree_impl.hpp b/src/mlpack/methods/det/dtree_impl.hpp index 06abf07e6c..39a8b1e112 100644 --- a/src/mlpack/methods/det/dtree_impl.hpp +++ b/src/mlpack/methods/det/dtree_impl.hpp @@ -67,7 +67,7 @@ namespace details const size_t minLeafSize) { typedef std::pair SplitItem; - arma::vec dimVec = data(dim, arma::span(start, end - 1)); + arma::rowvec dimVec = data(dim, arma::span(start, end - 1)); // We sort these, in-place (it's a copy of the data, anyways). std::sort(dimVec.begin(), dimVec.end()); @@ -165,6 +165,27 @@ DTree::DTree() : right(NULL) { /* Nothing to do. */ } +template +DTree::DTree(const DTree& src) : + start(src.start), + end(src.end), + splitDim(src.splitDim), + splitValue(src.splitValue), + logNegError(src.logNegError), + subtreeLeavesLogNegError(src.subtreeLeavesLogNegError), + subtreeLeaves(src.subtreeLeaves), + root(src.root), + ratio(src.ratio), + logVolume(src.logVolume), + bucketTag(src.bucketTag), + alphaUpper(src.alphaUpper) +{ + if (!!src.left) + left = new DTree(*src.left); + if (!!src.right) + right = new DTree(*src.right); +} + // Root node initializers From 8e1892a5277a17cf8d9b4a0df858bf82ed58eb5f Mon Sep 17 00:00:00 2001 From: theJonan Date: Tue, 7 Feb 2017 02:13:32 +0200 Subject: [PATCH 06/39] - More small fixes. - Optimized the DET model saving. - Added unpruned tree saving. --- src/mlpack/methods/det/det_main.cpp | 9 +++- src/mlpack/methods/det/dt_utils_impl.hpp | 9 +++- src/mlpack/methods/det/dtree.hpp | 8 ++-- src/mlpack/methods/det/dtree_impl.hpp | 55 ++++++++++++++---------- 4 files changed, 50 insertions(+), 31 deletions(-) diff --git a/src/mlpack/methods/det/det_main.cpp b/src/mlpack/methods/det/det_main.cpp index c045ae9cbd..e07af1468b 100644 --- a/src/mlpack/methods/det/det_main.cpp +++ b/src/mlpack/methods/det/det_main.cpp @@ -66,6 +66,9 @@ PARAM_STRING_OUT("tag_counters", "The file to output tag counters.", "c"); PARAM_STRING_OUT("tag_file", "The file to output the tags (and possibly paths) " " for each sample in the test set.", "g"); +PARAM_STRING_OUT("unpruned_tree", "The file to output the unpruned model to.", + "u"); + // Parameters for the training algorithm. PARAM_INT_IN("folds", "The number of folds of cross-validation to perform for the " "estimation (0 is LOOCV)", "f", 10); @@ -180,9 +183,11 @@ int main(int argc, char *argv[]) // Obtain the optimal tree. Timer::Start("det_training"); - tree = Trainer(trainingData, folds, regularization, maxLeafSize, minLeafSize, ""); + tree = Trainer(trainingData, folds, regularization, + maxLeafSize, minLeafSize, "", + CLI::GetParam("unpruned_tree")); Timer::Stop("det_training"); - + // Compute training set estimates, if desired. if (CLI::HasParam("training_set_estimates_file")) { diff --git a/src/mlpack/methods/det/dt_utils_impl.hpp b/src/mlpack/methods/det/dt_utils_impl.hpp index 18df1a5312..24ab262b95 100644 --- a/src/mlpack/methods/det/dt_utils_impl.hpp +++ b/src/mlpack/methods/det/dt_utils_impl.hpp @@ -111,7 +111,8 @@ DTree* Trainer(MatType& dataset, const bool useVolumeReg, const size_t maxLeafSize, const size_t minLeafSize, - const std::string unprunedTreeOutput) + const std::string unprunedTreeOutput, + const std::string unprunedModel) { // Initialize the tree. DTree dtree(dataset); @@ -134,6 +135,12 @@ DTree* Trainer(MatType& dataset, Log::Info << dtree.SubtreeLeaves() << " leaf nodes in the tree using full " << "dataset; minimum alpha: " << alpha << "." << std::endl; + if (unprunedModel != "") + { + Log::Info << "Saving unprunned tree in: " << unprunedModel << std::endl; + data::Save(unprunedModel, "det_model", dtree, false); + } + // Compute densities for the training points in the full tree, if we were // asked for this. if (unprunedTreeOutput != "") diff --git a/src/mlpack/methods/det/dtree.hpp b/src/mlpack/methods/det/dtree.hpp index a2a2233e1f..15012cc1ac 100644 --- a/src/mlpack/methods/det/dtree.hpp +++ b/src/mlpack/methods/det/dtree.hpp @@ -57,11 +57,6 @@ class DTree * Create an empty density estimation tree. */ DTree(); - - /** - * Create a copy of an existing tree. - */ - DTree(const DTree& tree); /** * Create a density estimation tree with the given bounds and the given number @@ -327,6 +322,9 @@ class DTree const size_t splitDim, const ElemType splitValue, arma::Col& oldFromNew) const; + + void FillMinMax(const StatType& mins, + const StatType& maxs); }; diff --git a/src/mlpack/methods/det/dtree_impl.hpp b/src/mlpack/methods/det/dtree_impl.hpp index 39a8b1e112..dc5f8082ec 100644 --- a/src/mlpack/methods/det/dtree_impl.hpp +++ b/src/mlpack/methods/det/dtree_impl.hpp @@ -165,27 +165,6 @@ DTree::DTree() : right(NULL) { /* Nothing to do. */ } -template -DTree::DTree(const DTree& src) : - start(src.start), - end(src.end), - splitDim(src.splitDim), - splitValue(src.splitValue), - logNegError(src.logNegError), - subtreeLeavesLogNegError(src.subtreeLeavesLogNegError), - subtreeLeaves(src.subtreeLeaves), - root(src.root), - ratio(src.ratio), - logVolume(src.logVolume), - bucketTag(src.bucketTag), - alphaUpper(src.alphaUpper) -{ - if (!!src.left) - left = new DTree(*src.left); - if (!!src.right) - right = new DTree(*src.right); -} - // Root node initializers @@ -863,6 +842,29 @@ DTree::ComputeVariableImportance(arma::vec& importances) const } } +template +void DTree::FillMinMax(const StatType& mins, + const StatType& maxs) +{ + if (!root) + { + minVals = mins; + maxVals = maxs; + } + + if (left && right) + { + StatType maxValsL(maxs); + StatType maxValsR(maxs); + StatType minValsL(mins); + StatType minValsR(mins); + + maxValsL[splitDim] = minValsR[splitDim] = splitValue; + left->FillMinMax(minValsL, maxValsL); + right->FillMinMax(minValsR, maxValsR); + } +} + template template void DTree::Serialize(Archive& ar, const unsigned int /* version */) @@ -871,8 +873,6 @@ void DTree::Serialize(Archive& ar, const unsigned int /* versi ar & CreateNVP(start, "start"); ar & CreateNVP(end, "end"); - ar & CreateNVP(maxVals, "maxVals"); - ar & CreateNVP(minVals, "minVals"); ar & CreateNVP(splitDim, "splitDim"); ar & CreateNVP(splitValue, "splitValue"); ar & CreateNVP(logNegError, "logNegError"); @@ -894,5 +894,14 @@ void DTree::Serialize(Archive& ar, const unsigned int /* versi ar & CreateNVP(left, "left"); ar & CreateNVP(right, "right"); + + if (root) + { + ar & CreateNVP(maxVals, "maxVals"); + ar & CreateNVP(minVals, "minVals"); + + if (Archive::is_loading::value && left && right) + FillMinMax(minVals, maxVals); + } } From 4041631b516253dbddd186c30ab30a7ac9f35a01 Mon Sep 17 00:00:00 2001 From: theJonan Date: Tue, 7 Feb 2017 02:17:17 +0200 Subject: [PATCH 07/39] - Minor logging fix. --- src/mlpack/methods/det/dt_utils_impl.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mlpack/methods/det/dt_utils_impl.hpp b/src/mlpack/methods/det/dt_utils_impl.hpp index 24ab262b95..fced4490c1 100644 --- a/src/mlpack/methods/det/dt_utils_impl.hpp +++ b/src/mlpack/methods/det/dt_utils_impl.hpp @@ -137,8 +137,8 @@ DTree* Trainer(MatType& dataset, if (unprunedModel != "") { - Log::Info << "Saving unprunned tree in: " << unprunedModel << std::endl; data::Save(unprunedModel, "det_model", dtree, false); + Log::Info << "Saved unprunned tree in: " << unprunedModel << std::endl; } // Compute densities for the training points in the full tree, if we were From 8010a98b39378e9d6913e236a1634fb2e560ca54 Mon Sep 17 00:00:00 2001 From: theJonan Date: Tue, 7 Feb 2017 11:24:21 +0200 Subject: [PATCH 08/39] - Internal nodes printing enabled. --- src/mlpack/methods/det/det_main.cpp | 31 ++++++++++++++++++++---- src/mlpack/methods/det/dt_utils_impl.hpp | 2 +- src/mlpack/methods/det/dtree.hpp | 4 +-- src/mlpack/methods/det/dtree_impl.hpp | 15 +++++++++--- 4 files changed, 40 insertions(+), 12 deletions(-) diff --git a/src/mlpack/methods/det/det_main.cpp b/src/mlpack/methods/det/det_main.cpp index e07af1468b..e22c35d489 100644 --- a/src/mlpack/methods/det/det_main.cpp +++ b/src/mlpack/methods/det/det_main.cpp @@ -248,7 +248,22 @@ int main(int argc, char *argv[]) } else if (CLI::HasParam("path_format")) { - PathCacher path(PathCacher::FormatLR, tree); + const string pathFormat = CLI::GetParam("path_format"); + PathCacher::PathFormat theFormat; + if (pathFormat == "lr" || pathFormat == "LR") + theFormat = PathCacher::FormatLR; + else if (pathFormat == "lr-id" || pathFormat == "LR-ID") + theFormat = PathCacher::FormatLR_ID; + else if (pathFormat == "id-lr" || pathFormat == "ID-LR") + theFormat = PathCacher::FormatID_LR; + else + { + Log::Warn << "Unknown path format specified: '" << pathFormat + << "'. Valid are: lr | lr-id | id-lr. Defaults to 'lr'" << endl; + theFormat = PathCacher::FormatLR; + } + + PathCacher path(theFormat, tree); counters.zeros(path.NumLeaves()); for (size_t i = 0; i < testData.n_cols; i++) @@ -297,7 +312,7 @@ int main(int argc, char *argv[]) template PathCacher::PathCacher(PathCacher::PathFormat fmt, DTree* dtree) : format(fmt) { - numLeaves = dtree->TagTree(); + numLeaves = dtree->TagTree(0, true); pathCache = new std::string [numLeaves]; assert(!!pathCache); dtree->EnumerateTree(*this); @@ -335,9 +350,15 @@ std::string PathCacher::BuildString() { switch (format) { - case FormatLR: str += it->first ? "L" : "R"; break; - default: - assert(0); + case FormatLR: + str += it->first ? "L" : "R"; + break; + case FormatLR_ID: + str += (it->first ? "L" : "R") + std::to_string(it->second); + break; + case FormatID_LR: + str += std::to_string(it->second) + (it->first ? "L" : "R"); + break; } } diff --git a/src/mlpack/methods/det/dt_utils_impl.hpp b/src/mlpack/methods/det/dt_utils_impl.hpp index fced4490c1..5e5b1e10c9 100644 --- a/src/mlpack/methods/det/dt_utils_impl.hpp +++ b/src/mlpack/methods/det/dt_utils_impl.hpp @@ -190,7 +190,7 @@ DTree* Trainer(MatType& dataset, Log::Info << prunedSequence.size() << " trees in the sequence; maximum alpha:" << " " << oldAlpha << "." << std::endl; - MatType cvData(dataset); + const MatType cvData(dataset); const size_t testSize = dataset.n_cols / folds; arma::vec regularizationConstants(prunedSequence.size()); diff --git a/src/mlpack/methods/det/dtree.hpp b/src/mlpack/methods/det/dtree.hpp index 15012cc1ac..12a724d38d 100644 --- a/src/mlpack/methods/det/dtree.hpp +++ b/src/mlpack/methods/det/dtree.hpp @@ -160,7 +160,7 @@ class DTree * * @param tag Tag for the next leaf; leave at 0 for the initial call. */ - TagType TagTree(const TagType& tag = 0); + TagType TagTree(const TagType& tag = 0, bool internal = false); /** @@ -287,7 +287,7 @@ class DTree //! Return the upper part of the alpha sum. double AlphaUpper() const { return alphaUpper; } //! Return the current bucket's ID, if leaf, or -1 otherwise - TagType BucketTag() const { return subtreeLeaves == 1 ? bucketTag : -1; } + TagType BucketTag() const { return bucketTag; } //! Return the maximum values. const StatType& MaxVals() const { return maxVals; } diff --git a/src/mlpack/methods/det/dtree_impl.hpp b/src/mlpack/methods/det/dtree_impl.hpp index dc5f8082ec..487bee927c 100644 --- a/src/mlpack/methods/det/dtree_impl.hpp +++ b/src/mlpack/methods/det/dtree_impl.hpp @@ -746,7 +746,7 @@ double DTree::ComputeValue(const VecType& query) const // Index the buckets for possible usage later. template -TagType DTree::TagTree(const TagType& tag) +TagType DTree::TagTree(const TagType& tag, bool internal) { if (subtreeLeaves == 1) { @@ -754,10 +754,17 @@ TagType DTree::TagTree(const TagType& tag) bucketTag = tag; return (tag + 1); } - else + + TagType nextTag; + if (internal) { - return right->TagTree(left->TagTree(tag)); + bucketTag = tag; + nextTag = tag + 1; } + else + nextTag = tag; + + return right->TagTree(left->TagTree(nextTag, internal), internal); } // Enumerate the nodes of the tree. @@ -796,7 +803,7 @@ TagType DTree::FindBucket(const VecType& query) const { // Check if the query is within range. if (!WithinRange(query)) - return bucketTag; + return -1; } // If we are a leaf... From 6e9ab983bfd0d87d9192fd1e0e03eb23ea331582 Mon Sep 17 00:00:00 2001 From: theJonan Date: Tue, 7 Feb 2017 15:20:24 +0200 Subject: [PATCH 09/39] Omp shared fix --- src/mlpack/methods/det/dt_utils_impl.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/mlpack/methods/det/dt_utils_impl.hpp b/src/mlpack/methods/det/dt_utils_impl.hpp index 5e5b1e10c9..6de4016070 100644 --- a/src/mlpack/methods/det/dt_utils_impl.hpp +++ b/src/mlpack/methods/det/dt_utils_impl.hpp @@ -202,11 +202,11 @@ DTree* Trainer(MatType& dataset, // implementation. #ifdef _WIN32 #pragma omp parallel for default(none) \ - shared(cvData, prunedSequence, regularizationConstants) + shared(prunedSequence, regularizationConstants) for (intmax_t fold = 0; fold < (intmax_t) folds; fold++) #else #pragma omp parallel for default(none) \ - shared(cvData, prunedSequence, regularizationConstants) + shared(prunedSequence, regularizationConstants) for (size_t fold = 0; fold < folds; fold++) #endif { From 52a626aec5ad79633f1c3a3693dab09b86e5c0bd Mon Sep 17 00:00:00 2001 From: theJonan Date: Tue, 7 Feb 2017 18:10:13 +0200 Subject: [PATCH 10/39] - More improvements. --- src/mlpack/methods/det/det_main.cpp | 29 +++++------- src/mlpack/methods/det/dt_utils.hpp | 3 +- src/mlpack/methods/det/dt_utils_impl.hpp | 60 ++++++++++++++---------- 3 files changed, 47 insertions(+), 45 deletions(-) diff --git a/src/mlpack/methods/det/det_main.cpp b/src/mlpack/methods/det/det_main.cpp index e22c35d489..3e5847a1d5 100644 --- a/src/mlpack/methods/det/det_main.cpp +++ b/src/mlpack/methods/det/det_main.cpp @@ -58,7 +58,7 @@ PARAM_STRING_OUT("vi_file", "The file to output the variable importance values " "for each feature.", "i"); // Tagging and path printing options -PARAM_STRING_IN("path_format", "The format of path printing - lr|idlr|ldid", +PARAM_STRING_IN("path_format", "The format of path printing - lr|id-lr|lr-id", "p", "lr"); PARAM_STRING_OUT("tag_counters", "The file to output tag counters.", "c"); @@ -66,8 +66,11 @@ PARAM_STRING_OUT("tag_counters", "The file to output tag counters.", "c"); PARAM_STRING_OUT("tag_file", "The file to output the tags (and possibly paths) " " for each sample in the test set.", "g"); -PARAM_STRING_OUT("unpruned_tree", "The file to output the unpruned model to.", - "u"); +PARAM_STRING_OUT("unpruned_estimates", "The file to output the estimations from " + "the unpruned tree.", "u"); + +PARAM_FLAG("skip_pruning", "Whether to bypass the pruning process and output " + "the unpruned tree only", "s"); // Parameters for the training algorithm. PARAM_INT_IN("folds", "The number of folds of cross-validation to perform for the " @@ -164,28 +167,18 @@ int main(int argc, char *argv[]) arma::mat trainingData; data::Load(trainSetFile, trainingData, true); - // Cross-validation here. - size_t folds = CLI::GetParam("folds"); - if (folds == 0) - { - folds = trainingData.n_cols; - Log::Info << "Performing leave-one-out cross validation." << endl; - } - else - { - Log::Info << "Performing " << folds << "-fold cross validation." << endl; - } - const bool regularization = false; // const bool regularization = CLI::HasParam("volume_regularization"); const int maxLeafSize = CLI::GetParam("max_leaf_size"); const int minLeafSize = CLI::GetParam("min_leaf_size"); + const bool skipPruning = CLI::HasParam("skip_pruning"); // Obtain the optimal tree. Timer::Start("det_training"); - tree = Trainer(trainingData, folds, regularization, - maxLeafSize, minLeafSize, "", - CLI::GetParam("unpruned_tree")); + tree = Trainer(trainingData, CLI::GetParam("folds"), + regularization, maxLeafSize, minLeafSize, + CLI::GetParam("unpruned_estimates"), + skipPruning); Timer::Stop("det_training"); // Compute training set estimates, if desired. diff --git a/src/mlpack/methods/det/dt_utils.hpp b/src/mlpack/methods/det/dt_utils.hpp index ff1cb10471..c5605190e5 100644 --- a/src/mlpack/methods/det/dt_utils.hpp +++ b/src/mlpack/methods/det/dt_utils.hpp @@ -67,7 +67,8 @@ DTree* Trainer(MatType& dataset, const bool useVolumeReg = false, const size_t maxLeafSize = 10, const size_t minLeafSize = 5, - const std::string unprunedTreeOutput = ""); + const std::string unprunedTreeOutput = "", + const bool skipPruning = false); } // namespace det } // namespace mlpack diff --git a/src/mlpack/methods/det/dt_utils_impl.hpp b/src/mlpack/methods/det/dt_utils_impl.hpp index 6de4016070..2966574c43 100644 --- a/src/mlpack/methods/det/dt_utils_impl.hpp +++ b/src/mlpack/methods/det/dt_utils_impl.hpp @@ -107,15 +107,15 @@ void PrintVariableImportance(const DTree* dtree, // folds. template DTree* Trainer(MatType& dataset, - const size_t folds, + size_t folds, const bool useVolumeReg, const size_t maxLeafSize, const size_t minLeafSize, const std::string unprunedTreeOutput, - const std::string unprunedModel) + const bool skipPruning) { // Initialize the tree. - DTree dtree(dataset); + DTree* dtree = new DTree(dataset); Timer::Start("tree_growing"); // Prepare to grow the tree... @@ -128,19 +128,13 @@ DTree* Trainer(MatType& dataset, // Growing the tree double oldAlpha = 0.0; - double alpha = dtree.Grow(newDataset, oldFromNew, useVolumeReg, maxLeafSize, + double alpha = dtree->Grow(newDataset, oldFromNew, useVolumeReg, maxLeafSize, minLeafSize); Timer::Stop("tree_growing"); - Log::Info << dtree.SubtreeLeaves() << " leaf nodes in the tree using full " + Log::Info << dtree->SubtreeLeaves() << " leaf nodes in the tree using full " << "dataset; minimum alpha: " << alpha << "." << std::endl; - if (unprunedModel != "") - { - data::Save(unprunedModel, "det_model", dtree, false); - Log::Info << "Saved unprunned tree in: " << unprunedModel << std::endl; - } - // Compute densities for the training points in the full tree, if we were // asked for this. if (unprunedTreeOutput != "") @@ -151,7 +145,7 @@ DTree* Trainer(MatType& dataset, for (size_t i = 0; i < dataset.n_cols; ++i) { arma::vec testPoint = dataset.unsafe_col(i); - outfile << dtree.ComputeValue(testPoint) << std::endl; + outfile << dtree->ComputeValue(testPoint) << std::endl; } } else @@ -162,28 +156,41 @@ DTree* Trainer(MatType& dataset, outfile.close(); } + + if (skipPruning) + return dtree; + if (folds == 0) + { + folds = dataset.n_cols; + Log::Info << "Performing leave-one-out cross validation." << std::endl; + } + else + { + Log::Info << "Performing " << folds << "-fold cross validation." << std::endl; + } + Timer::Start("prunning_sequence"); // Sequentially prune and save the alpha values and the values of c_t^2 * r_t. std::vector > prunedSequence; - while (dtree.SubtreeLeaves() > 1) + while (dtree->SubtreeLeaves() > 1) { - std::pair treeSeq(oldAlpha, dtree.SubtreeLeavesLogNegError()); + std::pair treeSeq(oldAlpha, dtree->SubtreeLeavesLogNegError()); prunedSequence.push_back(treeSeq); oldAlpha = alpha; - alpha = dtree.PruneAndUpdate(oldAlpha, dataset.n_cols, useVolumeReg); + alpha = dtree->PruneAndUpdate(oldAlpha, dataset.n_cols, useVolumeReg); // Some sanity checks. It seems that on some datasets, the error does not // increase as the tree is pruned but instead stays the same---hence the // "<=" in the final assert. Log::Assert((alpha < std::numeric_limits::max()) - || (dtree.SubtreeLeaves() == 1)); + || (dtree->SubtreeLeaves() == 1)); Log::Assert(alpha > oldAlpha); - Log::Assert(dtree.SubtreeLeavesLogNegError() <= treeSeq.second); + Log::Assert(dtree->SubtreeLeavesLogNegError() <= treeSeq.second); } - std::pair treeSeq(oldAlpha, dtree.SubtreeLeavesLogNegError()); + std::pair treeSeq(oldAlpha, dtree->SubtreeLeavesLogNegError()); prunedSequence.push_back(treeSeq); Timer::Stop("prunning_sequence"); @@ -305,8 +312,9 @@ DTree* Trainer(MatType& dataset, Log::Info << "Optimal alpha: " << optimalAlpha << "." << std::endl; - // Initialize the tree. - DTree* dtreeOpt = new DTree(dataset); + // Re-Initialize the tree. + delete dtree; + dtree = new DTree(dataset); // Getting ready to grow the tree... for (size_t i = 0; i < oldFromNew.n_elem; i++) @@ -317,28 +325,28 @@ DTree* Trainer(MatType& dataset, // Grow the tree. oldAlpha = -DBL_MAX; - alpha = dtreeOpt->Grow(newDataset, + alpha = dtree->Grow(newDataset, oldFromNew, useVolumeReg, maxLeafSize, minLeafSize); // Prune with optimal alpha. - while ((oldAlpha < optimalAlpha) && (dtreeOpt->SubtreeLeaves() > 1)) + while ((oldAlpha < optimalAlpha) && (dtree->SubtreeLeaves() > 1)) { oldAlpha = alpha; - alpha = dtreeOpt->PruneAndUpdate(oldAlpha, newDataset.n_cols, useVolumeReg); + alpha = dtree->PruneAndUpdate(oldAlpha, newDataset.n_cols, useVolumeReg); // Some sanity checks. Log::Assert((alpha < std::numeric_limits::max()) || - (dtreeOpt->SubtreeLeaves() == 1)); + (dtree->SubtreeLeaves() == 1)); Log::Assert(alpha > oldAlpha); } - Log::Info << dtreeOpt->SubtreeLeaves() << " leaf nodes in the optimally " + Log::Info << dtree->SubtreeLeaves() << " leaf nodes in the optimally " << "pruned tree; optimal alpha: " << oldAlpha << "." << std::endl; - return dtreeOpt; + return dtree; } } // namespace det From ae228e2a2579872f96bafe4c7d39039d7021a017 Mon Sep 17 00:00:00 2001 From: theJonan Date: Tue, 7 Feb 2017 18:19:15 +0200 Subject: [PATCH 11/39] More OMP fixes for folds. --- src/mlpack/methods/det/det_main.cpp | 8 ++++++-- src/mlpack/methods/det/dt_utils_impl.hpp | 9 ++------- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/src/mlpack/methods/det/det_main.cpp b/src/mlpack/methods/det/det_main.cpp index 3e5847a1d5..65603bfc43 100644 --- a/src/mlpack/methods/det/det_main.cpp +++ b/src/mlpack/methods/det/det_main.cpp @@ -172,11 +172,15 @@ int main(int argc, char *argv[]) const int maxLeafSize = CLI::GetParam("max_leaf_size"); const int minLeafSize = CLI::GetParam("min_leaf_size"); const bool skipPruning = CLI::HasParam("skip_pruning"); + size_t folds = CLI::GetParam("folds"); + + if (folds == 0) + folds = trainingData.n_cols; // Obtain the optimal tree. Timer::Start("det_training"); - tree = Trainer(trainingData, CLI::GetParam("folds"), - regularization, maxLeafSize, minLeafSize, + tree = Trainer(trainingData, folds, regularization, + maxLeafSize, minLeafSize, CLI::GetParam("unpruned_estimates"), skipPruning); Timer::Stop("det_training"); diff --git a/src/mlpack/methods/det/dt_utils_impl.hpp b/src/mlpack/methods/det/dt_utils_impl.hpp index 2966574c43..289d15e06d 100644 --- a/src/mlpack/methods/det/dt_utils_impl.hpp +++ b/src/mlpack/methods/det/dt_utils_impl.hpp @@ -107,7 +107,7 @@ void PrintVariableImportance(const DTree* dtree, // folds. template DTree* Trainer(MatType& dataset, - size_t folds, + const size_t folds, const bool useVolumeReg, const size_t maxLeafSize, const size_t minLeafSize, @@ -160,15 +160,10 @@ DTree* Trainer(MatType& dataset, if (skipPruning) return dtree; - if (folds == 0) - { - folds = dataset.n_cols; + if (folds == dataset.n_cols) Log::Info << "Performing leave-one-out cross validation." << std::endl; - } else - { Log::Info << "Performing " << folds << "-fold cross validation." << std::endl; - } Timer::Start("prunning_sequence"); From dbff1b137743fc3bba1eb11ef58962b853580683 Mon Sep 17 00:00:00 2001 From: theJonan Date: Tue, 7 Feb 2017 19:45:08 +0200 Subject: [PATCH 12/39] - New options fixes. --- src/mlpack/methods/det/det_main.cpp | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/mlpack/methods/det/det_main.cpp b/src/mlpack/methods/det/det_main.cpp index 65603bfc43..13fac90246 100644 --- a/src/mlpack/methods/det/det_main.cpp +++ b/src/mlpack/methods/det/det_main.cpp @@ -61,14 +61,14 @@ PARAM_STRING_OUT("vi_file", "The file to output the variable importance values " PARAM_STRING_IN("path_format", "The format of path printing - lr|id-lr|lr-id", "p", "lr"); -PARAM_STRING_OUT("tag_counters", "The file to output tag counters.", "c"); +PARAM_STRING_OUT("tag_counters_file", "The file to output tag counters.", "c"); + +PARAM_STRING_OUT("raw_estimates_file", "The file to output the estimations from " + "the unpruned tree.", "u"); PARAM_STRING_OUT("tag_file", "The file to output the tags (and possibly paths) " " for each sample in the test set.", "g"); -PARAM_STRING_OUT("unpruned_estimates", "The file to output the estimations from " - "the unpruned tree.", "u"); - PARAM_FLAG("skip_pruning", "Whether to bypass the pruning process and output " "the unpruned tree only", "s"); @@ -181,7 +181,7 @@ int main(int argc, char *argv[]) Timer::Start("det_training"); tree = Trainer(trainingData, folds, regularization, maxLeafSize, minLeafSize, - CLI::GetParam("unpruned_estimates"), + CLI::GetParam("raw_estimates_file"), skipPruning); Timer::Stop("det_training"); @@ -285,8 +285,8 @@ int main(int argc, char *argv[]) } } - if (CLI::GetParam("tag_counters") != "") - data::Save(CLI::GetParam("tag_counters"), counters); + if (CLI::GetParam("tag_counters_file") != "") + data::Save(CLI::GetParam("tag_counters_file"), counters); Timer::Stop("det_test_set_tagging"); ofs.close(); From 6dad5abef333aa40a8b1b23cb645420e9d1d4274 Mon Sep 17 00:00:00 2001 From: theJonan Date: Tue, 7 Feb 2017 22:41:59 +0200 Subject: [PATCH 13/39] - More path printing features. --- src/mlpack/methods/det/det_main.cpp | 75 ++++++++++++++++++----------- 1 file changed, 47 insertions(+), 28 deletions(-) diff --git a/src/mlpack/methods/det/det_main.cpp b/src/mlpack/methods/det/det_main.cpp index 13fac90246..0315c8ce7f 100644 --- a/src/mlpack/methods/det/det_main.cpp +++ b/src/mlpack/methods/det/det_main.cpp @@ -101,26 +101,25 @@ public: template PathCacher(PathFormat fmt, DTree* tree); - ~PathCacher(); - template void Enter(const DTree* node, const DTree* parent); template void Leave(const DTree* node, const DTree* parent); - const std::string& PathFor(int tag) const; + const std::string& PathFor(int tag) const; - size_t NumLeaves() const { return numLeaves; } + int ParentOf(int tag) const; -private: - typedef std::list > PathType; + size_t NumNodes() const { return pathCache.size(); } + +protected: + typedef std::list > PathType; + typedef std::vector > PathCacheType; PathType path; PathFormat format; - - int numLeaves; - std::string* pathCache; + PathCacheType pathCache; std::string BuildString(); }; @@ -245,7 +244,9 @@ int main(int argc, char *argv[]) } else if (CLI::HasParam("path_format")) { + const bool reqCounters = CLI::HasParam("tag_counters_file"); const string pathFormat = CLI::GetParam("path_format"); + PathCacher::PathFormat theFormat; if (pathFormat == "lr" || pathFormat == "LR") theFormat = PathCacher::FormatLR; @@ -261,14 +262,30 @@ int main(int argc, char *argv[]) } PathCacher path(theFormat, tree); - counters.zeros(path.NumLeaves()); + counters.zeros(path.NumNodes()); for (size_t i = 0; i < testData.n_cols; i++) { - const int tag = tree->FindBucket(testData.unsafe_col(i)); - - counters(tag) += 1; + int tag = tree->FindBucket(testData.unsafe_col(i)); + ofs << tag << " " << path.PathFor(tag) << std::endl; + for (;tag >= 0 && reqCounters; tag = path.ParentOf(tag)) + counters(tag) += 1; + } + + ofs.close(); + + if (reqCounters) + { + ofs.open(CLI::GetParam("tag_counters_file"), + std::ofstream::out); + + for (size_t j = 0;j < counters.n_elem; ++j) + ofs << j << " " + << counters(j) << " " + << path.PathFor(j) << endl; + + ofs.close(); } } else @@ -283,10 +300,11 @@ int main(int argc, char *argv[]) ofs << tag << std::endl; counters(tag) += 1; } + + if (CLI::HasParam("tag_counters_file")) + data::Save(CLI::GetParam("tag_counters_file"), counters); } - if (CLI::GetParam("tag_counters_file") != "") - data::Save(CLI::GetParam("tag_counters_file"), counters); Timer::Stop("det_test_set_tagging"); ofs.close(); @@ -309,19 +327,14 @@ int main(int argc, char *argv[]) template PathCacher::PathCacher(PathCacher::PathFormat fmt, DTree* dtree) : format(fmt) { - numLeaves = dtree->TagTree(0, true); - pathCache = new std::string [numLeaves]; - assert(!!pathCache); + pathCache.resize(dtree->TagTree(0, true)); + pathCache[0] = PathCacheType::value_type(-1, ""); dtree->EnumerateTree(*this); } -PathCacher::~PathCacher() -{ - delete[] pathCache; -} - template -void PathCacher::Enter(const DTree* node, const DTree* parent) +void PathCacher::Enter(const DTree* node, + const DTree* parent) { if (parent == nullptr) return; @@ -329,8 +342,10 @@ void PathCacher::Enter(const DTree* node, const DTreeBucketTag(); path.push_back(PathType::value_type(parent->Left() == node, tag)); - if (tag >= 0) - pathCache[tag] = BuildString(); + pathCache[tag] = PathCacheType::value_type(parent->BucketTag(), + (node->SubtreeLeaves() > 1) ? + "" : BuildString() + ); } template @@ -362,8 +377,12 @@ std::string PathCacher::BuildString() return str; } +int PathCacher::ParentOf(int tag) const +{ + return pathCache[tag].first; +} + const std::string& PathCacher::PathFor(int tag) const { - assert(tag >= 0 && tag < numLeaves ); - return pathCache[tag]; + return pathCache[tag].second; } From 4412d8ae88b8aff845f61b90f8a17a253c587ff1 Mon Sep 17 00:00:00 2001 From: theJonan Date: Wed, 8 Feb 2017 00:45:51 +0200 Subject: [PATCH 14/39] - After merge fix. --- src/mlpack/methods/det/det_main.cpp | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/mlpack/methods/det/det_main.cpp b/src/mlpack/methods/det/det_main.cpp index 249b037661..b9714cfea7 100644 --- a/src/mlpack/methods/det/det_main.cpp +++ b/src/mlpack/methods/det/det_main.cpp @@ -213,8 +213,7 @@ int main(int argc, char *argv[]) if (CLI::HasParam("test")) { arma::mat testData = std::move(CLI::GetParam("test")); - data::Load(testFile, testData, true); - if (CLI::HasParam("test_set_estimates_file")) + if (CLI::HasParam("test_set_estimates")) { // Compute test set densities. Timer::Start("det_test_set_estimation"); @@ -316,7 +315,7 @@ int main(int argc, char *argv[]) { arma::vec importances; tree->ComputeVariableImportance(importances); - CLI::GetParam("vi") = std::move(importances.t()); + CLI::GetParam("vi") = importances.t(); } // Save the model, if desired. From b01e3d0e7328bcd515215405a84b3430fdbd1255 Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Wed, 12 Jul 2017 14:30:12 -0400 Subject: [PATCH 15/39] Use arma::gmm_diag to fix #215. --- .../methods/gmm/diagonal_constraint.hpp | 3 +- src/mlpack/methods/gmm/em_fit.hpp | 16 +++ src/mlpack/methods/gmm/em_fit_impl.hpp | 66 ++++++++++++ src/mlpack/methods/gmm/gmm_train_main.cpp | 43 +++++++- src/mlpack/tests/gmm_test.cpp | 100 ++++++++++++++++++ 5 files changed, 221 insertions(+), 7 deletions(-) diff --git a/src/mlpack/methods/gmm/diagonal_constraint.hpp b/src/mlpack/methods/gmm/diagonal_constraint.hpp index e0eafc02b7..864e2f5525 100644 --- a/src/mlpack/methods/gmm/diagonal_constraint.hpp +++ b/src/mlpack/methods/gmm/diagonal_constraint.hpp @@ -27,8 +27,7 @@ class DiagonalConstraint static void ApplyConstraint(arma::mat& covariance) { // Save the diagonal only. - arma::vec diagonal = covariance.diag(); - covariance = arma::diagmat(diagonal); + covariance = arma::diagmat(arma::clamp(covariance.diag(), 1e-10, DBL_MAX)); } //! Serialize the constraint (which holds nothing, so, nothing to do). diff --git a/src/mlpack/methods/gmm/em_fit.hpp b/src/mlpack/methods/gmm/em_fit.hpp index 71817f8cab..be70157181 100644 --- a/src/mlpack/methods/gmm/em_fit.hpp +++ b/src/mlpack/methods/gmm/em_fit.hpp @@ -162,6 +162,22 @@ class EMFit dists, const arma::vec& weights) const; + /** + * Use the Armadillo gmm_diag clusterer to train a GMM with diagonal + * covariance. If InitialClusteringType == kmeans::KMeans<>, this will use + * Armadillo's initialization also. + * + * @param observations Data to train on. + * @param dists Distributions to store model in. + * @param weights Prior weights. + * @param useInitialModel If true, the existing model will be used. + */ + void ArmadilloGMMWrapper( + const arma::mat& observations, + std::vector& dists, + arma::vec& weights, + const bool useInitialModel); + //! Maximum iterations of EM algorithm. size_t maxIterations; //! Tolerance for convergence of EM. diff --git a/src/mlpack/methods/gmm/em_fit_impl.hpp b/src/mlpack/methods/gmm/em_fit_impl.hpp index feaa9d4ae1..acf76ed35f 100644 --- a/src/mlpack/methods/gmm/em_fit_impl.hpp +++ b/src/mlpack/methods/gmm/em_fit_impl.hpp @@ -15,6 +15,7 @@ // In case it hasn't been included yet. #include "em_fit.hpp" +#include "diagonal_constraint.hpp" namespace mlpack { namespace gmm { @@ -39,6 +40,14 @@ void EMFit::Estimate( arma::vec& weights, const bool useInitialModel) { + // Shortcut: if the user is using the DiagonalConstraint, then we will call + // out to Armadillo. + if (std::is_same::value) + { + ArmadilloGMMWrapper(observations, dists, weights, useInitialModel); + return; + } + // Only perform initial clustering if the user wanted it. if (!useInitialModel) InitialClustering(observations, dists, weights); @@ -316,6 +325,63 @@ void EMFit::Serialize( ar & CreateNVP(constraint, "constraint"); } +template +void EMFit:: +ArmadilloGMMWrapper(const arma::mat& observations, + std::vector& dists, + arma::vec& weights, + const bool useInitialModel) +{ + arma::gmm_diag g; + + // Warn the user that tolerance isn't used for convergence here. + Log::Warn << "GMM::Train(): tolerance ignored when training GMMs with " + << "DiagonalConstraint." << std::endl; + + // If the initial clustering is the default k-means, we'll just use + // Armadillo's implementation. If mlpack ever changes k-means defaults to use + // something that is reliably quicker than the Lloyd iteration k-means update, + // then this code maybe should be revisited. + if (!std::is_same>::value || + useInitialModel) + { + // Use clusterer to get initial values. + if (!useInitialModel) + InitialClustering(observations, dists, weights); + + // Assemble matrix of means. + arma::mat means(observations.n_rows, dists.size()); + arma::mat covs(observations.n_rows, dists.size()); + for (size_t i = 0; i < dists.size(); ++i) + { + means.col(i) = dists[i].Mean(); + covs.col(i) = dists[i].Covariance().diag(); + } + + g.set_means(std::move(means)); + g.set_dcovs(std::move(covs)); + g.set_hefts(std::move(weights)); + + g.learn(observations, dists.size(), arma::eucl_dist, arma::keep_existing, 0, + maxIterations, 1e-10, false /* no printing */); + } + else + { + // Use Armadillo for the initial clustering. We'll try and match mlpack + // defaults. + g.learn(observations, dists.size(), arma::eucl_dist, arma::random_subset, + 1000, maxIterations, 1e-10, false /* no printing */); + } + + // Extract means, covariances, and weights. + weights = g.hefts; + for (size_t i = 0; i < dists.size(); ++i) + { + dists[i].Mean() = g.means.col(i); + dists[i].Covariance(std::move(arma::diagmat(g.dcovs.col(i)))); + } +} + } // namespace gmm } // namespace mlpack diff --git a/src/mlpack/methods/gmm/gmm_train_main.cpp b/src/mlpack/methods/gmm/gmm_train_main.cpp index 14b09cbc64..2a87dc1ad7 100644 --- a/src/mlpack/methods/gmm/gmm_train_main.cpp +++ b/src/mlpack/methods/gmm/gmm_train_main.cpp @@ -14,6 +14,7 @@ #include "gmm.hpp" #include "no_constraint.hpp" +#include "diagonal_constraint.hpp" #include @@ -42,6 +43,11 @@ PROGRAM_INFO("Gaussian Mixture Model (GMM) Training", "but may also cause non-positive definite covariance matrices, which will " "cause the program to crash." "\n\n" + "The 'diagonal_covariance' flag will cause the learned covariances to be " + "diagonal matrices. This significantly simplifies the model itself and " + "causes training to be faster, but restricts the ability to fit more " + "complex GMMs." + "\n\n" "Optionally, multiple trials may be performed, by specifying the --trials " "option. The model with greatest log-likelihood will be taken."); @@ -59,6 +65,8 @@ PARAM_FLAG("no_force_positive", "Do not force the covariance matrices to be " "positive definite.", "P"); PARAM_INT_IN("max_iterations", "Maximum number of iterations of EM algorithm " "(passing 0 will run until convergence).", "n", 250); +PARAM_FLAG("diagonal_covariance", "Force the covariance of the Gaussians to " + "be diagonal. This can accelerate training time significantly.", "d"); // Parameters for dataset modification. PARAM_DOUBLE_IN("noise", "Variance of zero-mean Gaussian noise to add to data.", @@ -95,6 +103,11 @@ int main(int argc, char* argv[]) "be greater than or equal to 1." << std::endl; } + if (CLI::HasParam("diagonal_covariance") && + CLI::HasParam("no_force_positive")) + Log::Warn << "--no_force_positive ignored because --diagonal_covariance is " + << "specified!" << endl; + if (!CLI::HasParam("output_model")) Log::Warn << "--output_model_file is not specified, so no model will be " << "saved!" << endl; @@ -130,6 +143,7 @@ int main(int argc, char* argv[]) const size_t maxIterations = (size_t) CLI::GetParam("max_iterations"); const double tolerance = CLI::GetParam("tolerance"); const bool forcePositive = !CLI::HasParam("no_force_positive"); + const bool diagonalCovariance = CLI::HasParam("diagonal_covariance"); // This gets a bit weird because we need different types depending on whether // --refined_start is specified. @@ -153,9 +167,18 @@ int main(int argc, char* argv[]) KMeansType k(1000, metric::SquaredEuclideanDistance(), RefinedStart(samplings, percentage)); - // Depending on the value of 'forcePositive', we have to use different - // types. - if (forcePositive) + // Depending on the value of forcePositive and diagonalCovariance, we have + // to use different types. + if (diagonalCovariance) + { + // Compute the parameters of the model using the EM algorithm. + Timer::Start("em"); + EMFit em(maxIterations, tolerance, k); + likelihood = gmm.Train(dataPoints, CLI::GetParam("trials"), false, + em); + Timer::Stop("em"); + } + else if (forcePositive) { // Compute the parameters of the model using the EM algorithm. Timer::Start("em"); @@ -176,8 +199,18 @@ int main(int argc, char* argv[]) } else { - // Depending on the value of forcePositive, we have to use different types. - if (forcePositive) + // Depending on the value of forcePositive and diagonalCovariance, we have + // to use different types. + if (diagonalCovariance) + { + // Compute the parameters of the model using the EM algorithm. + Timer::Start("em"); + EMFit, DiagonalConstraint> em(maxIterations, tolerance); + likelihood = gmm.Train(dataPoints, CLI::GetParam("trials"), false, + em); + Timer::Stop("em"); + } + else if (forcePositive) { // Compute the parameters of the model using the EM algorithm. Timer::Start("em"); diff --git a/src/mlpack/tests/gmm_test.cpp b/src/mlpack/tests/gmm_test.cpp index d3c8cd5c29..b9a836eb7e 100644 --- a/src/mlpack/tests/gmm_test.cpp +++ b/src/mlpack/tests/gmm_test.cpp @@ -760,5 +760,105 @@ BOOST_AUTO_TEST_CASE(UseExistingModelTest) } } +/** + * Make sure we can fit a diagonal GMM reasonably. + */ +BOOST_AUTO_TEST_CASE(DiagonalGMMTrainTest) +{ + Log::Warn.ignoreInput = false; + // We'll have three diagonal-covariance Gaussian distributions from this + // mixture. + distribution::GaussianDistribution d1("0.0 1.0 0.0", "1.0 0.0 0.0;" + "0.0 0.8 0.0;" + "0.0 0.0 1.0"); + distribution::GaussianDistribution d2("2.0 -1.0 5.0", "3.0 0.0 0.0;" + "0.0 1.2 0.0;" + "0.0 0.0 1.3"); + distribution::GaussianDistribution d3("0.0 5.0 -3.0", "2.0 0.0 0.0;" + "0.0 0.3 0.0;" + "0.0 0.0 1.0"); + + // Now we'll generate points and probabilities. 1500 points. Slower than I + // would like... + arma::mat points(3, 5000); + + for (size_t i = 0; i < 5000; i++) + { + double randValue = math::Random(); + + if (randValue <= 0.20) // p(d1) = 0.20 + points.col(i) = d1.Random(); + else if (randValue <= 0.50) // p(d2) = 0.30 + points.col(i) = d2.Random(); + else // p(d3) = 0.50 + points.col(i) = d3.Random(); + } + + // Now train the model. 3 dimensions, 3 components. + GMM g(3, 3); + + g.Train, DiagonalConstraint>>(points, 5); + + // Now check the results. We need to order by weights so that when we do the + // checking, things will be correct. + arma::uvec sortedIndices = sort_index(g.Weights()); + + // First Gaussian (d1). + BOOST_REQUIRE_SMALL(g.Weights()[sortedIndices[0]] - 0.2, 0.1); + + for (size_t i = 0; i < 3; i++) + BOOST_REQUIRE_SMALL((g.Component(sortedIndices[0]).Mean()[i] + - d1.Mean()[i]), 0.4); + + for (size_t row = 0; row < 3; ++row) + { + for (size_t col = 0; col < 3; ++col) + { + const double v = g.Component(sortedIndices[0]).Covariance()(row, col); + if (row == col) + BOOST_REQUIRE_SMALL(v - d1.Covariance()(row, col), 0.5); + else + BOOST_REQUIRE_SMALL(v, 1e-5); + } + } + + // Second Gaussian (d2). + BOOST_REQUIRE_SMALL(g.Weights()[sortedIndices[1]] - 0.3, 0.1); + + for (size_t i = 0; i < 3; i++) + BOOST_REQUIRE_SMALL((g.Component(sortedIndices[1]).Mean()[i] + - d2.Mean()[i]), 0.4); + + for (size_t row = 0; row < 3; ++row) + { + for (size_t col = 0; col < 3; ++col) + { + const double v = g.Component(sortedIndices[1]).Covariance()(row, col); + if (row == col) + BOOST_REQUIRE_SMALL(v - d2.Covariance()(row, col), 0.5); + else + BOOST_REQUIRE_SMALL(v, 1e-5); + } + } + + // Third Gaussian (d3). + BOOST_REQUIRE_SMALL(g.Weights()[sortedIndices[2]] - 0.5, 0.1); + + for (size_t i = 0; i < 3; ++i) + BOOST_REQUIRE_SMALL((g.Component(sortedIndices[2]).Mean()[i] + - d3.Mean()[i]), 0.4); + + for (size_t row = 0; row < 3; ++row) + { + for (size_t col = 0; col < 3; ++col) + { + const double v = g.Component(sortedIndices[2]).Covariance()(row, col); + if (row == col) + BOOST_REQUIRE_SMALL(v - d3.Covariance()(row, col), 0.5); + else + BOOST_REQUIRE_SMALL(v, 1e-5); + } + } +} BOOST_AUTO_TEST_SUITE_END(); From df394893be57e37b7f7d2784089d5faa267c9f0b Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Mon, 17 Jul 2017 15:22:02 -0400 Subject: [PATCH 16/39] Improve implementation---use set_params() and copy weights correctly. --- src/mlpack/methods/gmm/em_fit_impl.hpp | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/mlpack/methods/gmm/em_fit_impl.hpp b/src/mlpack/methods/gmm/em_fit_impl.hpp index acf76ed35f..21cbfc4a65 100644 --- a/src/mlpack/methods/gmm/em_fit_impl.hpp +++ b/src/mlpack/methods/gmm/em_fit_impl.hpp @@ -358,9 +358,8 @@ ArmadilloGMMWrapper(const arma::mat& observations, covs.col(i) = dists[i].Covariance().diag(); } - g.set_means(std::move(means)); - g.set_dcovs(std::move(covs)); - g.set_hefts(std::move(weights)); + g.reset(observations.n_rows, dists.size()); + g.set_params(std::move(means), std::move(covs), weights.t()); g.learn(observations, dists.size(), arma::eucl_dist, arma::keep_existing, 0, maxIterations, 1e-10, false /* no printing */); @@ -369,12 +368,12 @@ ArmadilloGMMWrapper(const arma::mat& observations, { // Use Armadillo for the initial clustering. We'll try and match mlpack // defaults. - g.learn(observations, dists.size(), arma::eucl_dist, arma::random_subset, + g.learn(observations, dists.size(), arma::eucl_dist, arma::static_subset, 1000, maxIterations, 1e-10, false /* no printing */); } // Extract means, covariances, and weights. - weights = g.hefts; + weights = g.hefts.t(); for (size_t i = 0; i < dists.size(); ++i) { dists[i].Mean() = g.means.col(i); From c3219af77f1d7256e3646b70636e5241a408dde3 Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Mon, 18 Sep 2017 13:12:05 -0400 Subject: [PATCH 17/39] Only give the warning if a non-default value is passed. --- src/mlpack/methods/gmm/em_fit_impl.hpp | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/mlpack/methods/gmm/em_fit_impl.hpp b/src/mlpack/methods/gmm/em_fit_impl.hpp index 21cbfc4a65..8070e7d06a 100644 --- a/src/mlpack/methods/gmm/em_fit_impl.hpp +++ b/src/mlpack/methods/gmm/em_fit_impl.hpp @@ -334,9 +334,11 @@ ArmadilloGMMWrapper(const arma::mat& observations, { arma::gmm_diag g; - // Warn the user that tolerance isn't used for convergence here. - Log::Warn << "GMM::Train(): tolerance ignored when training GMMs with " - << "DiagonalConstraint." << std::endl; + // Warn the user that tolerance isn't used for convergence here if they've + // specified a non-default value. + if (tolerance != EMFit().Tolerance()) + Log::Warn << "GMM::Train(): tolerance ignored when training GMMs with " + << "DiagonalConstraint." << std::endl; // If the initial clustering is the default k-means, we'll just use // Armadillo's implementation. If mlpack ever changes k-means defaults to use From 6de1a020af17deff66593633b3ba19e36797f6df Mon Sep 17 00:00:00 2001 From: Kirill Mishchenko Date: Wed, 30 Aug 2017 15:42:06 +0500 Subject: [PATCH 18/39] Add a tutorial for the CV and HPT modules --- doc/guide/cv_and_hpt.hpp | 208 ++++++++++++++++++++++++++++++++++++ doc/tutorials/tutorials.txt | 1 + src/mlpack/core.hpp | 1 + 3 files changed, 210 insertions(+) create mode 100644 doc/guide/cv_and_hpt.hpp diff --git a/doc/guide/cv_and_hpt.hpp b/doc/guide/cv_and_hpt.hpp new file mode 100644 index 0000000000..53e264f55f --- /dev/null +++ b/doc/guide/cv_and_hpt.hpp @@ -0,0 +1,208 @@ +/*! @page cv_and_hpt Cross-Validation and Hyper-Parameter Tuning + +@section intro Introduction +In this tutorial we will see the usage examples of the cross-validation and +hyper-parameter tuning modules. + +@section cv Cross-Validation + +@subsection cv_basic Basic Usage + +Suppose we have some data to train and validate on. +@code + // 100-point 6-dimensional random dataset. + arma::mat data = arma::randu(6, 100); + // Random labels in the [0, 4] interval. + arma::Row labels = + arma::randi>(100, arma::distr_param(0, 4)); + size_t numClasses = 5; +@endcode + +To run 10-fold cross-validation for softmax regression with accuracy as a +metric we can write the following piece of code. +@code + KFoldCV cv(10, data, labels, numClasses); + double lambda = 0.1; + double softmaxAccuracy = cv.Evaluate(lambda); +@endcode +In this example the \c Evaluate method relies on the following \c +SoftmaxRegression constructor: +@code + template + SoftmaxRegression(const arma::mat& data, + const arma::Row& labels, + const size_t numClasses, + const double lambda = 0.0001, + const bool fitIntercept = false, + OptimizerType optimizer = OptimizerType()); +@endcode +which has the parameter \c lambda after three conventional arguments (\c data, +\c labels and \c numClasses). We can skip passing \c fitIntercept and \c +optimizer (as well as \c lambda), since there are the default values. + +In general to cross-validate you need to specify what machine learning algorithm +and metric you are going to use, and then to pass some conventional data-related +parameters into one of the cross-validation constructors and all other +parameters (which are hyper-parameters in many cases) into the \c Evaluate +method. + +@subsection cv_examples More Examples + +In the following example we will cross-validate \c DecisionTree with weights. +@code + // Random weights for every point from the code snippet above. + arma::rowvec weights = arma::randu(1, 100); + + KFoldCV, Accuracy> cv2(10, data, labels, numClasses, weights); + size_t minimumLeafSize = 8; + double weightedDecisionTreeAccuracy = cv2.Evaluate(minimumLeafSize); +@endcode +It relies on the following \c DecisionTree constructor: +@code + template + DecisionTree(MatType&& data, + LabelsType&& labels, + const size_t numClasses, + WeightsType&& weights, + const size_t minimumLeafSize = 10, + const std::enable_if_t::type>::value>* + = 0); +@endcode +\c DecisionTree models can be constructed in multiple other ways. For example, +if you want to use some particular \c DatasetInfo parameter during construction +of \c DecisionTree objects for cross-validation, you can write the following +code. +@code + size_t dimensionality = 6; + data::DatasetInfo datasetInfo(dimensionality); + + KFoldCV, Accuracy> cv3(10, data, datasetInfo, labels, + numClasses); + double decisionTreeWithDIAccuracy = cv3.Evaluate(minimumLeafSize); +@endcode +It relies on the following DecisionTree constructor: +@code + template + DecisionTree(MatType&& data, + const data::DatasetInfo& datasetInfo, + LabelsType&& labels, + const size_t numClasses, + const size_t minimumLeafSize = 10); +@endcode + +\c SimpleCV has the same interface as \c KFoldCV, except it takes as one of its +arguments a proportion (from 0 to 1) of data used as a validation set. For +example, to validate \c LinearRegression with 20\% of training data we can write +the following code. +@code + // Random responses for every point from the code snippet above. + arma::rowvec responses = arma::randu(100); + + SimpleCV cv4(0.2, data, responses); + double lrLambda = 0.05; + double lrMSE = cv4.Evaluate(lrLambda); +@endcode + +The whole list of constructors for a cross-validation class you can find in the +related header file. + +@section hpt Hyper-Parameter Tuning + +@subsection hpt_basic Basic Usage + +The interface of the hyper-parameter tuning module is quite similar to the +interface of the cross-validation module. To construct a \c HyperParameterTuner +object you need to specify what machine learning algorithm, cross-validation +strategy, metric and optimization strategy (\c GridSearch will be used by +default) you are going to use, and then to pass the same arguments as we do for +cross-validation classes. Let's see some examples. + +Suppose we have the following data to train and validate on. +@code + // 100-point 5-dimensional random dataset. + arma::mat data = arma::randu(5, 100); + // Noisy responses retrieved by a random linear transformation of data. + arma::rowvec responses = arma::randu(5) * data + + 0.1 * arma::randn(100); +@endcode + +Then we can use the following code to try to find a good \c lambda value for +\c LinearRegression. + +@code + // Using 80% of data for training and remaining 20% for assessing MSE. + double validationSize = 0.2; + HyperParameterTuner hpt(validationSize, + data, responses); + + // Finding a good value for lambda from the values 0.0, 0.001, 0.01, 0.1, + // and 1.0. + arma::vec lambdas{0.0, 0.001, 0.01, 0.1, 1.0}; + double bestLambda; + std::tie(bestLambda) = hpt.Optimize(lambdas); +@endcode + +In this example we have used GridSearch (the default optimizer) to find a good +value for the \c lambda hyper-parameter. For that we have specified what values +should be tried. + +@subsection hpt_fixed Fixed Arguments + +When some hyper-parameters should not be optimized, you can specify values +for them with the \c Fixed function as in the following example of trying to +find good \c lambda1 and \c lambda2 values for \c LARS. + +@code + HyperParameterTuner hpt2(validationSize, data, + responses); + + bool transposeData = true; + bool useCholesky = false; + arma::vec lambda1Set{0.0, 0.001, 0.01, 0.1, 1.0}; + arma::vec lambda2Set{0.0, 0.002, 0.02, 0.2, 2.0}; + + double bestLambda1, bestLambda2; + std::tie(bestLambda1, bestLambda2) = hpt2.Optimize(Fixed(transposeData), + Fixed(useCholesky), lambda1Set, lambda2Set); +@endcode +Note that we have used the same order of arguments as they appear in the \c LARS +constructor: +@code + LARS(const arma::mat& data, + const arma::rowvec& responses, + const bool transposeData = true, + const bool useCholesky = false, + const double lambda1 = 0.0, + const double lambda2 = 0.0, + const double tolerance = 1e-16); +@endcode + +@subsection hpt_gradient Gradient-Based Optimization + +When we know approximate optimal values (which we can try to find with the +\c GridSearch optimizer) for real-valued hyper-parameters, we can try to tune +them even more with gradient-based optimization. In the following example we +try to optimize the \c lambda1 and \c lambda2 hyper-parameters for \c LARS with +the \c GradientDescent optimizer. +@code + HyperParameterTuner hpt3(validationSize, + data, responses); + + // GradientDescent can be adjusted in the following way. + hpt3.Optimizer().StepSize() = 0.1; + hpt3.Optimizer().Tolerance() = 1e-15; + + // We can set up values used for calculating gradients. + hpt3.RelativeDelta() = 0.01; + hpt3.MinDelta() = 1e-10; + + double initialLambda1 = 0.001; + double initialLambda2 = 0.002; + + double bestGDLambda1, bestGDLambda2; + std::tie(bestGDLambda1, bestGDLambda2) = hpt3.Optimize(Fixed(transposeData), + Fixed(useCholesky), initialLambda1, initialLambda2); +@endcode + +*/ diff --git a/doc/tutorials/tutorials.txt b/doc/tutorials/tutorials.txt index 204bfd1477..2c5c75e4bc 100644 --- a/doc/tutorials/tutorials.txt +++ b/doc/tutorials/tutorials.txt @@ -18,6 +18,7 @@ start. - \ref iodoc - \ref timer - \ref sample + - \ref cv_and_hpt @section method_tut Method-specific Tutorials diff --git a/src/mlpack/core.hpp b/src/mlpack/core.hpp index 43bb812277..85e670fb87 100644 --- a/src/mlpack/core.hpp +++ b/src/mlpack/core.hpp @@ -100,6 +100,7 @@ * - @ref iodoc * - @ref timer * - @ref sample + * - @ref cv_and_hpt * - @ref verinfo * * Tutorials on specific methods are also available. From 10e632cb9c65fe45aafcad3231f88e002bb296ec Mon Sep 17 00:00:00 2001 From: Kirill Mishchenko Date: Sat, 9 Sep 2017 11:08:10 +0500 Subject: [PATCH 19/39] Split into two tutorials --- doc/guide/{cv_and_hpt.hpp => cv.hpp} | 114 ++------------------------- doc/guide/hpt.hpp | 103 ++++++++++++++++++++++++ doc/tutorials/tutorials.txt | 3 +- src/mlpack/core.hpp | 3 +- 4 files changed, 114 insertions(+), 109 deletions(-) rename doc/guide/{cv_and_hpt.hpp => cv.hpp} (50%) create mode 100644 doc/guide/hpt.hpp diff --git a/doc/guide/cv_and_hpt.hpp b/doc/guide/cv.hpp similarity index 50% rename from doc/guide/cv_and_hpt.hpp rename to doc/guide/cv.hpp index 53e264f55f..49296b69d7 100644 --- a/doc/guide/cv_and_hpt.hpp +++ b/doc/guide/cv.hpp @@ -1,12 +1,9 @@ -/*! @page cv_and_hpt Cross-Validation and Hyper-Parameter Tuning +/*! @page cv Cross-Validation -@section intro Introduction -In this tutorial we will see the usage examples of the cross-validation and -hyper-parameter tuning modules. +@section cvintro Introduction +In this tutorial we will see the usage examples of the cross-validation module. -@section cv Cross-Validation - -@subsection cv_basic Basic Usage +@section cvbasic Basic Usage Suppose we have some data to train and validate on. @code @@ -46,7 +43,7 @@ parameters into one of the cross-validation constructors and all other parameters (which are hyper-parameters in many cases) into the \c Evaluate method. -@subsection cv_examples More Examples +@section cvexamples More Examples In the following example we will cross-validate \c DecisionTree with weights. @code @@ -96,7 +93,8 @@ arguments a proportion (from 0 to 1) of data used as a validation set. For example, to validate \c LinearRegression with 20\% of training data we can write the following code. @code - // Random responses for every point from the code snippet above. + // Random responses for every point from the code snippet in the beginning of + // the tutorial. arma::rowvec responses = arma::randu(100); SimpleCV cv4(0.2, data, responses); @@ -107,102 +105,4 @@ the following code. The whole list of constructors for a cross-validation class you can find in the related header file. -@section hpt Hyper-Parameter Tuning - -@subsection hpt_basic Basic Usage - -The interface of the hyper-parameter tuning module is quite similar to the -interface of the cross-validation module. To construct a \c HyperParameterTuner -object you need to specify what machine learning algorithm, cross-validation -strategy, metric and optimization strategy (\c GridSearch will be used by -default) you are going to use, and then to pass the same arguments as we do for -cross-validation classes. Let's see some examples. - -Suppose we have the following data to train and validate on. -@code - // 100-point 5-dimensional random dataset. - arma::mat data = arma::randu(5, 100); - // Noisy responses retrieved by a random linear transformation of data. - arma::rowvec responses = arma::randu(5) * data + - 0.1 * arma::randn(100); -@endcode - -Then we can use the following code to try to find a good \c lambda value for -\c LinearRegression. - -@code - // Using 80% of data for training and remaining 20% for assessing MSE. - double validationSize = 0.2; - HyperParameterTuner hpt(validationSize, - data, responses); - - // Finding a good value for lambda from the values 0.0, 0.001, 0.01, 0.1, - // and 1.0. - arma::vec lambdas{0.0, 0.001, 0.01, 0.1, 1.0}; - double bestLambda; - std::tie(bestLambda) = hpt.Optimize(lambdas); -@endcode - -In this example we have used GridSearch (the default optimizer) to find a good -value for the \c lambda hyper-parameter. For that we have specified what values -should be tried. - -@subsection hpt_fixed Fixed Arguments - -When some hyper-parameters should not be optimized, you can specify values -for them with the \c Fixed function as in the following example of trying to -find good \c lambda1 and \c lambda2 values for \c LARS. - -@code - HyperParameterTuner hpt2(validationSize, data, - responses); - - bool transposeData = true; - bool useCholesky = false; - arma::vec lambda1Set{0.0, 0.001, 0.01, 0.1, 1.0}; - arma::vec lambda2Set{0.0, 0.002, 0.02, 0.2, 2.0}; - - double bestLambda1, bestLambda2; - std::tie(bestLambda1, bestLambda2) = hpt2.Optimize(Fixed(transposeData), - Fixed(useCholesky), lambda1Set, lambda2Set); -@endcode -Note that we have used the same order of arguments as they appear in the \c LARS -constructor: -@code - LARS(const arma::mat& data, - const arma::rowvec& responses, - const bool transposeData = true, - const bool useCholesky = false, - const double lambda1 = 0.0, - const double lambda2 = 0.0, - const double tolerance = 1e-16); -@endcode - -@subsection hpt_gradient Gradient-Based Optimization - -When we know approximate optimal values (which we can try to find with the -\c GridSearch optimizer) for real-valued hyper-parameters, we can try to tune -them even more with gradient-based optimization. In the following example we -try to optimize the \c lambda1 and \c lambda2 hyper-parameters for \c LARS with -the \c GradientDescent optimizer. -@code - HyperParameterTuner hpt3(validationSize, - data, responses); - - // GradientDescent can be adjusted in the following way. - hpt3.Optimizer().StepSize() = 0.1; - hpt3.Optimizer().Tolerance() = 1e-15; - - // We can set up values used for calculating gradients. - hpt3.RelativeDelta() = 0.01; - hpt3.MinDelta() = 1e-10; - - double initialLambda1 = 0.001; - double initialLambda2 = 0.002; - - double bestGDLambda1, bestGDLambda2; - std::tie(bestGDLambda1, bestGDLambda2) = hpt3.Optimize(Fixed(transposeData), - Fixed(useCholesky), initialLambda1, initialLambda2); -@endcode - */ diff --git a/doc/guide/hpt.hpp b/doc/guide/hpt.hpp new file mode 100644 index 0000000000..4ddeaca1e0 --- /dev/null +++ b/doc/guide/hpt.hpp @@ -0,0 +1,103 @@ +/*! @page hpt Hyper-Parameter Tuning + +@section hptintro Introduction +In this tutorial we will see the usage examples of the hyper-parameter tuning +module. + +@section hptbasic Basic Usage + +The interface of the hyper-parameter tuning module is quite similar to the +interface of the @ref cv module. To construct a \c HyperParameterTuner object +you need to specify what machine learning algorithm, cross-validation strategy, +metric and optimization strategy (\c GridSearch will be used by default) you are +going to use, and then to pass the same arguments as we do for cross-validation +classes. Let's see some examples. + +Suppose we have the following data to train and validate on. +@code + // 100-point 5-dimensional random dataset. + arma::mat data = arma::randu(5, 100); + // Noisy responses retrieved by a random linear transformation of data. + arma::rowvec responses = arma::randu(5) * data + + 0.1 * arma::randn(100); +@endcode + +Then we can use the following code to try to find a good \c lambda value for +\c LinearRegression. + +@code + // Using 80% of data for training and remaining 20% for assessing MSE. + double validationSize = 0.2; + HyperParameterTuner hpt(validationSize, + data, responses); + + // Finding a good value for lambda from the values 0.0, 0.001, 0.01, 0.1, + // and 1.0. + arma::vec lambdas{0.0, 0.001, 0.01, 0.1, 1.0}; + double bestLambda; + std::tie(bestLambda) = hpt.Optimize(lambdas); +@endcode + +In this example we have used GridSearch (the default optimizer) to find a good +value for the \c lambda hyper-parameter. For that we have specified what values +should be tried. + +@section hptfixed Fixed Arguments + +When some hyper-parameters should not be optimized, you can specify values +for them with the \c Fixed function as in the following example of trying to +find good \c lambda1 and \c lambda2 values for \c LARS. + +@code + HyperParameterTuner hpt2(validationSize, data, + responses); + + bool transposeData = true; + bool useCholesky = false; + arma::vec lambda1Set{0.0, 0.001, 0.01, 0.1, 1.0}; + arma::vec lambda2Set{0.0, 0.002, 0.02, 0.2, 2.0}; + + double bestLambda1, bestLambda2; + std::tie(bestLambda1, bestLambda2) = hpt2.Optimize(Fixed(transposeData), + Fixed(useCholesky), lambda1Set, lambda2Set); +@endcode +Note that we have used the same order of arguments as they appear in the \c LARS +constructor: +@code + LARS(const arma::mat& data, + const arma::rowvec& responses, + const bool transposeData = true, + const bool useCholesky = false, + const double lambda1 = 0.0, + const double lambda2 = 0.0, + const double tolerance = 1e-16); +@endcode + +@section hptgradient Gradient-Based Optimization + +When we know approximate optimal values (which we can try to find with the +\c GridSearch optimizer) for real-valued hyper-parameters, we can try to tune +them even more with gradient-based optimization. In the following example we +try to optimize the \c lambda1 and \c lambda2 hyper-parameters for \c LARS with +the \c GradientDescent optimizer. +@code + HyperParameterTuner hpt3(validationSize, + data, responses); + + // GradientDescent can be adjusted in the following way. + hpt3.Optimizer().StepSize() = 0.1; + hpt3.Optimizer().Tolerance() = 1e-15; + + // We can set up values used for calculating gradients. + hpt3.RelativeDelta() = 0.01; + hpt3.MinDelta() = 1e-10; + + double initialLambda1 = 0.001; + double initialLambda2 = 0.002; + + double bestGDLambda1, bestGDLambda2; + std::tie(bestGDLambda1, bestGDLambda2) = hpt3.Optimize(Fixed(transposeData), + Fixed(useCholesky), initialLambda1, initialLambda2); +@endcode + +*/ diff --git a/doc/tutorials/tutorials.txt b/doc/tutorials/tutorials.txt index 2c5c75e4bc..e364da6bd9 100644 --- a/doc/tutorials/tutorials.txt +++ b/doc/tutorials/tutorials.txt @@ -18,7 +18,8 @@ start. - \ref iodoc - \ref timer - \ref sample - - \ref cv_and_hpt + - \ref cv + - \ref hpt @section method_tut Method-specific Tutorials diff --git a/src/mlpack/core.hpp b/src/mlpack/core.hpp index 85e670fb87..6f71e6e8ba 100644 --- a/src/mlpack/core.hpp +++ b/src/mlpack/core.hpp @@ -100,7 +100,8 @@ * - @ref iodoc * - @ref timer * - @ref sample - * - @ref cv_and_hpt + * - @ref cv + * - @ref hpt * - @ref verinfo * * Tutorials on specific methods are also available. From 8e5846575a37b6228cacc1a5306bdf8a276950fa Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Mon, 18 Sep 2017 14:19:52 -0400 Subject: [PATCH 20/39] Update tutorials, add more examples and links to other classes. --- doc/guide/cv.hpp | 309 ++++++++++++++++++++++++++++++++++++++++++---- doc/guide/hpt.hpp | 176 ++++++++++++++++++++++---- 2 files changed, 440 insertions(+), 45 deletions(-) diff --git a/doc/guide/cv.hpp b/doc/guide/cv.hpp index 49296b69d7..132cb48c0a 100644 --- a/doc/guide/cv.hpp +++ b/doc/guide/cv.hpp @@ -1,11 +1,50 @@ +namespace mlpack { +namespace cv { + /*! @page cv Cross-Validation @section cvintro Introduction -In this tutorial we will see the usage examples of the cross-validation module. -@section cvbasic Basic Usage +@b mlpack implements cross-validation support for its learning algorithms, for a +variety of performance measures. Cross-validation is useful for determining an +estimate of how well the learner will generalize to un-seen test data. It is a +commonly used part of the data science pipeline. + +In short, given some learner and some performance measure, we wish to get an +average of the performance measure given different splits of the dataset into +training data and validation data. The learner is trained on the training data, +and the performance measure is evaluated on the validation data. + +mlpack currently implements two easy-to-use forms of cross-validation: + + - @b simple @b cross-validation, where we simply desire the performance measure + on a single split of the data into a training set and validation set + + - @b k-fold @b cross-validation, where we split the data k ways and desire the + average performance measure on each of the k splits of the data + +In this tutorial we will see the usage examples and details of the +cross-validation module. Because the cross-validation code is generic and can +be used with any learner and performance measure, any use of the +cross-validation code in mlpack has to be in C++. + +This tutorial is split into the following sections: + + - @ref cvbasic Simple cross-validation examples + - @ref cvbasic_ex_1 10-fold cross-validation on softmax regression + - @ref cvbasic_ex_2 10-fold cross-validation on weighted decision trees + - @ref cvbasic_ex_3 10-fold cross-validation with categorical decision trees + - @ref cvbasic_ex_4 Simple cross-validation for linear regression + - @ref cvbasic_metrics Performance measures + - @ref cvbasic_api The \c KFoldCV and \c SimpleCV classes + - @ref cvbasic_further Further reference + +@section cvbasic Simple cross-validation examples + +@subsection cvbasic_ex_1 10-fold cross-validation on softmax regression + +Suppose we have some data to train and validate on, as defined below: -Suppose we have some data to train and validate on. @code // 100-point 6-dimensional random dataset. arma::mat data = arma::randu(6, 100); @@ -15,15 +54,23 @@ Suppose we have some data to train and validate on. size_t numClasses = 5; @endcode +The code above generates an 100-point random 6-dimensional dataset with 5 +classes. + To run 10-fold cross-validation for softmax regression with accuracy as a -metric we can write the following piece of code. +performance measure, we can write the following piece of code. + @code KFoldCV cv(10, data, labels, numClasses); double lambda = 0.1; double softmaxAccuracy = cv.Evaluate(lambda); @endcode -In this example the \c Evaluate method relies on the following \c -SoftmaxRegression constructor: + +Note that the \c Evaluate method of \c KFoldCV takes any hyperparameters of the +algorithm---that is, anything that is not \c data, \c labels, or \c numClasses. +To be more specific, in this example the \c Evaluate method relies on the +following \ref regression::SoftmaxRegression "SoftmaxRegression" constructor: + @code template SoftmaxRegression(const arma::mat& data, @@ -33,28 +80,42 @@ SoftmaxRegression constructor: const bool fitIntercept = false, OptimizerType optimizer = OptimizerType()); @endcode + which has the parameter \c lambda after three conventional arguments (\c data, \c labels and \c numClasses). We can skip passing \c fitIntercept and \c -optimizer (as well as \c lambda), since there are the default values. +optimizer since there are the default values. (Technically, we don't even need +to pass \c lambda since there is a default value.) In general to cross-validate you need to specify what machine learning algorithm and metric you are going to use, and then to pass some conventional data-related parameters into one of the cross-validation constructors and all other -parameters (which are hyper-parameters in many cases) into the \c Evaluate -method. +parameters (which are generally hyperparameters) into the \c Evaluate method. -@section cvexamples More Examples +@subsection cvbasic_ex_2 10-fold cross-validation on weighted decision trees + +In the following example we will cross-validate +\ref tree::DecisionTree "DecisionTree" with weights. This is very similar to +the previous example, except that we also have instance weights for each point +in the dataset. We can generate weights for the dataset from the previous +example with the code below: -In the following example we will cross-validate \c DecisionTree with weights. @code // Random weights for every point from the code snippet above. arma::rowvec weights = arma::randu(1, 100); +@endcode +Given those weights for each point, we can now perform cross-validation by also +passing the weights to the constructor of \c KFoldCV: + +@code KFoldCV, Accuracy> cv2(10, data, labels, numClasses, weights); size_t minimumLeafSize = 8; double weightedDecisionTreeAccuracy = cv2.Evaluate(minimumLeafSize); @endcode -It relies on the following \c DecisionTree constructor: + +As with the previous example, internally this call to \c cv2.Evaluate() relies +on the following \ref tree::DecisionTree "DecisionTree" constructor: + @code template DecisionTree(MatType&& data, @@ -66,19 +127,26 @@ It relies on the following \c DecisionTree constructor: typename std::remove_reference::type>::value>* = 0); @endcode -\c DecisionTree models can be constructed in multiple other ways. For example, -if you want to use some particular \c DatasetInfo parameter during construction -of \c DecisionTree objects for cross-validation, you can write the following -code. -@code - size_t dimensionality = 6; - data::DatasetInfo datasetInfo(dimensionality); +@subsection cvbasic_ex_3 10-fold cross-validation with categorical decision trees + +\ref tree::DecisionTree "DecisionTree" models can be constructed in multiple +other ways. For example, if we have a dataset with both categorical and +numerical features, we can also perform cross-validation by using the associated +\c data::DatasetInfo object. Thus, given some \c data::DatasetInfo object +called \c datasetInfo (that perhaps was produced by a call to \c data::Load() ), +we can perform k-fold cross-validation in a similar manner to the other +examples: + +@code KFoldCV, Accuracy> cv3(10, data, datasetInfo, labels, numClasses); double decisionTreeWithDIAccuracy = cv3.Evaluate(minimumLeafSize); @endcode -It relies on the following DecisionTree constructor: + +This particular call to \c cv3.Evaluate() relies on the following +\ref tree::DecisionTree "DecisionTree" constructor: + @code template DecisionTree(MatType&& data, @@ -88,10 +156,13 @@ It relies on the following DecisionTree constructor: const size_t minimumLeafSize = 10); @endcode +@subsection cvbasic_ex_4 Simple cross-validation for linear regression + \c SimpleCV has the same interface as \c KFoldCV, except it takes as one of its arguments a proportion (from 0 to 1) of data used as a validation set. For -example, to validate \c LinearRegression with 20\% of training data we can write -the following code. +example, to validate \ref regression::LinearRegression "LinearRegression" with +20\% of the data used in the validation set we can write the following code. + @code // Random responses for every point from the code snippet in the beginning of // the tutorial. @@ -102,7 +173,197 @@ the following code. double lrMSE = cv4.Evaluate(lrLambda); @endcode -The whole list of constructors for a cross-validation class you can find in the -related header file. +@section cvbasic_metrics Performance measures + +The cross-validation classes require a performance measure to be specified. +\b mlpack has a number of performance measures implemented; below is a list: + + - mlpack::cv::Accuracy: a simple measure of accuracy + - mlpack::cv::F1: the F1 score; depends on an averaging strategy + - mlpack::cv::MSE: minimum squared error (for regression problems) + - mlpack::cv::Precision: the precision, for classification problems + - mlpack::cv::Recall: the recall, for classification problems + +In addition, it is not difficult to implement a custom performance measure. A +class following the structure below can be used: + +@code +class CustomMeasure +{ + // + // This evaluates the metric given a trained model and a set of data (with + // labels or responses) to evaluate on. The data parameter will be a type of + // Armadillo matrix, and the labels will be the labels that go with the model. + // + // If you know that your model is a classification model (and thus that + // ResponsesType will be arma::Row), it is ok to replace the + // ResponsesType template parameter with arma::Row. + // + template + static double Evaluate(MLAlgorithm& model, + const DataType& data, + const ResponsesType& labels) + { + // Inside the function you should call model.Predict() and compare the + // values with the labels, in order to get the desired performance measure + // and return it. + } +}; +@endcode + +Once this is implemented, then \c CustomMeasure (or whatever the class is +called) is easy to use as a custom performance measure with \c KFoldCV or +\c SimpleCV. + +@section cvbasic_api The KFoldCV and SimpleCV classes + +This section provides details about the \c KFoldCV and \c SimpleCV classes. +The cross-validation infrastructure is based on heavy amounts of template +metaprogramming, so that any \b mlpack learner and any performance measure can +be used. Both classes have two required template parameters and one optional +parameter: + + - \c MLAlgorithm: the type of learner to be used + - \c Metric: the performance measure to be evaluated + - \c MatType: the type of matrix used to store the data + +In addition, there are two more template parameters, but these are automatically +extracted from the given \c MLAlgorithm class, and users should not need to +specify these parameters. + +The general structure of the \c KFoldCV and \c SimpleCV classes is split into +two parts: + + - The constructor: create the object, and store the data for the \c MLAlgorithm + training. + - The \c Evaluate() function: take any non-data parameters for the + \c MLAlgorithm and calculate the desired performance measure. + +This split is important because it defines the API: all data-related parameters +are passed to the constructor, whereas algorithm hyperparameters are passed to +the \c Evaluate() method. + +@subsection cvbasic_api_constructor The KFoldCV and SimpleCV constructors + +There are six constructors available for \c KFoldCV and \c SimpleCV, each +tailored for a different learning situation. Each is given below for the +\c KFoldCV class, but the same constructors are also available for the +\c SimpleCV class, with the exception that instead of specifying \c k, the +number of folds, the \c SimpleCV class takes a parameter between 0 and 1 +specifying the percentage of the dataset to use as a validation set. + + - `KFoldCV(k, xs, ys)`: this is for unweighted regression applications and + two-class classification applications; \c xs is the dataset and \c ys + are the responses or labels for each point in the dataset. + + - `KFoldCV(k, xs, ys, numClasses)`: this is for unweighted classification + applications; \c xs is the dataset, \c ys are the class labels for each + data point, and \c numClasses is the number of classes in the dataset. + + - `KFoldCV(k, xs, datasetInfo, ys, numClasses)`: this is for unweighted + categorical/numeric classification applications; \c xs is the dataset, + \c datasetInfo is a data::DatasetInfo object that holds the types of + each dimension in the dataset, \c ys are the class labels for each data + point, and \c numClasses is the number of classes in the dataset. + + - `KFoldCV(k, xs, ys, weights)`: this is for weighted regression or + two-class classification applications; \c xs is the dataset, \c ys are + the responses or labels for each point in the dataset, and \c weights + are the weights for each point in the dataset. + + - `KFoldCV(k, xs, ys, numClasses, weights)`: this is for weighted + classification applications; \c xs is the dataset, \c ys are the class + labels for each point in the dataset; \c numClasses is the number of + classes in the dataset, and \c weights holds the weights for each point + in the dataset. + + - `KFoldCV(k, xs, datasetInfo, ys, numClasses, weights)`: this is for + weighted cateogrical/numeric classification applications; \c xs is the + dataset, \c datasetInfo is a data::DatasetInfo object that holds the + types of each dimension in the dataset, \c ys are the class labels for + each data point, \c numClasses is the number of classes in each dataset, + and \c weights holds the weights for each point in the dataset. + +Note that the constructor you should use is the constructor that most closely +matches the constructor of the machine learning algorithm you would like +performance measures of. So, for instance, if you are doing multi-class softmax +regression, you could call the constructor +\c "SoftmaxRegression(xs, ys, numClasses)". Therefore, for \c KFoldCV you would +call the constructor \c "KFoldCV(k, xs, ys, numClasses)" and for \c SimpleCV you +would call the constructor \c "SimpleCV(pct, xs, ys, numClasses)". + +@subsection cvbasic_api_evaluate The Evaluate() function + +The other function that \c KFoldCV and \c SimpleCV have is the function to +actually calculate the performance measure: \c Evaluate(). The \c Evaluate() +function takes any hyperparameters that would follow the data arguments to the +constructor or \c Train() function of the given \c MLAlgorithm. The +\c Evaluate() function takes no more arguments than that, and returns the +desired performance measure on the dataset. + +Therefore, let us suppose that we are interested in cross-validating the +performance of a softmax regression model, and that we have constructed +the appropriate \c KFoldCV object using the code below: + +@code +KFoldCV cv(k, data, labels, numClasses); +@endcode + +The \ref regression::SoftmaxRegression "SoftmaxRegression" class has the +constructor + +@code + template + SoftmaxRegression(const arma::mat& data, + const arma::Row& labels, + const size_t numClasses, + const double lambda = 0.0001, + const bool fitIntercept = false, + OptimizerType optimizer = OptimizerType()); +@endcode + +Note that all parameters are \c numClasses are optional. This means that we can +specify none or any of them in our call to \c Evaluate(). Below is some example +code showing three different ways we can call \c Evaluate() with the \c cv +object from the code snippet above. + +@code +// First, call with all defaults. +double result1 = cv.Evaluate(); + +// Next, call with lambda set to 0.1 and fitIntercept set to true. +double result2 = cv.Evaluate(0.1, true); + +// Lastly, create a custom optimizer to use for optimization, and use a lambda +// value of 0.5 and fit no intercept. +optimization::SGD<> sgd(0.05, 50000); // Step size of 0.05, 50k max iterations. +double result3 = cv.Evaluate(0.5, false, sgd); +@endcode + +The same general idea applies to any \c MLAlgorithm: all hyperparameters must be +passed to the \c Evaluate() function of \c KFoldCV or \c SimpleCV. + +@section cvbasic_further Further references + +For further documentation, please see the associated Doxygen documentation for +each of the relevant classes: + + - mlpack::cv::SimpleCV + - mlpack::cv::KFoldCV + - mlpack::cv::Accuracy + - mlpack::cv::F1 + - mlpack::cv::MSE + - mlpack::cv::Precision + - mlpack::cv::Recall + +If you are interested in implementing a different cross-validation strategy than +k-fold cross-validation or simple cross-validation, take a look at the +implementations of each of those classes to guide your implementation. + +In addition, the @ref hpt "hyperparameter tuner" documentation may also be +relevant. */ + +} // namespace cv +} // namespace mlpack diff --git a/doc/guide/hpt.hpp b/doc/guide/hpt.hpp index 4ddeaca1e0..63ff5644b1 100644 --- a/doc/guide/hpt.hpp +++ b/doc/guide/hpt.hpp @@ -1,17 +1,45 @@ +namespace mlpack { +namespace hpt { + /*! @page hpt Hyper-Parameter Tuning @section hptintro Introduction + +\b mlpack implements a generic hyperparameter tuner that is able to tune both +continuous and discrete parameters of various different algorithms. This is an +important task---the performance of many machine learning algorithms can be +highly dependent on the hyperparameters that are chosen for that algorithm. +(One example: the choice of \f$k\f$ for a \f$k\f$-nearest-neighbors classifier.) + +This hyper-parameter tuner is built on the same general concept as the +cross-validation classes (see the @ref cv "cross-validation tutorial"): given +some machine learning algorithm, some data, some performance measure, and a set +of hyperparameters, attempt to find the hyperparameter set that best optimizes +the performance measure on the given data with the given algorithm. + +\b mlpack's implementation of hyperparameter tuning is flexible, and is built in +a way that supports many algorithms and many optimizers. At the time of this +writing, complex hyperparameter optimization techniques are not available, but +the hyperparameter tuner does support these, should they be implemented in the +future. + In this tutorial we will see the usage examples of the hyper-parameter tuning -module. +module, and also more details about the \c HyperParameterTuner class. @section hptbasic Basic Usage The interface of the hyper-parameter tuning module is quite similar to the -interface of the @ref cv module. To construct a \c HyperParameterTuner object -you need to specify what machine learning algorithm, cross-validation strategy, -metric and optimization strategy (\c GridSearch will be used by default) you are -going to use, and then to pass the same arguments as we do for cross-validation -classes. Let's see some examples. +interface of the @ref cv "cross-validation module". To construct a \c +HyperParameterTuner object you need to specify as template parameters what +machine learning algorithm, cross-validation strategy, performance measure, and +optimization strategy (\ref optimization::GridSearch "GridSearch" will be used by +default) you are going to use. Then, you must pass the same arguments as for +the cross-validation classes: the data and labels (or responses) to use are +given to the constructor, and the possible hyperparameter values are given to +the \c HyperParameterTuner::Optimize() function, which returns the best +algorithm configuration as a \c std::tuple<>. + +Let's see some examples. Suppose we have the following data to train and validate on. @code @@ -22,8 +50,10 @@ Suppose we have the following data to train and validate on. 0.1 * arma::randn(100); @endcode -Then we can use the following code to try to find a good \c lambda value for -\c LinearRegression. +Given the dataset above, we can use the following code to try to find a good \c +lambda value for \ref regression::LinearRegression "LinearRegression". Here we +use \ref cv::SimpleCV "SimpleCV" instead of k-fold cross-validation to save +computation time. @code // Using 80% of data for training and remaining 20% for assessing MSE. @@ -31,29 +61,34 @@ Then we can use the following code to try to find a good \c lambda value for HyperParameterTuner hpt(validationSize, data, responses); - // Finding a good value for lambda from the values 0.0, 0.001, 0.01, 0.1, - // and 1.0. + // Finding a good value for lambda from the discrete set of values 0.0, 0.001, + // 0.01, 0.1, and 1.0. arma::vec lambdas{0.0, 0.001, 0.01, 0.1, 1.0}; double bestLambda; std::tie(bestLambda) = hpt.Optimize(lambdas); @endcode -In this example we have used GridSearch (the default optimizer) to find a good -value for the \c lambda hyper-parameter. For that we have specified what values -should be tried. +In this example we have used \ref optimization::GridSearch "GridSearch" (the +default optimizer) to find a good value for the \c lambda hyper-parameter. For +that we have specified what values should be tried. @section hptfixed Fixed Arguments When some hyper-parameters should not be optimized, you can specify values -for them with the \c Fixed function as in the following example of trying to -find good \c lambda1 and \c lambda2 values for \c LARS. +for them with the \c Fixed() function as in the following example of trying to +find good \c lambda1 and \c lambda2 values for \ref regression::LARS "LARS" +(least-angle regression). @code HyperParameterTuner hpt2(validationSize, data, responses); + // The hyper-parameter tuner should not try to change the transposeData or + // useCholesky parameters. bool transposeData = true; bool useCholesky = false; + + // We wish only to search for the best lambda1 and lambda2 values. arma::vec lambda1Set{0.0, 0.001, 0.01, 0.1, 1.0}; arma::vec lambda2Set{0.0, 0.002, 0.02, 0.2, 2.0}; @@ -61,8 +96,11 @@ find good \c lambda1 and \c lambda2 values for \c LARS. std::tie(bestLambda1, bestLambda2) = hpt2.Optimize(Fixed(transposeData), Fixed(useCholesky), lambda1Set, lambda2Set); @endcode -Note that we have used the same order of arguments as they appear in the \c LARS + +Note that for the call to \c hpt2.Optimize(), we have used the same order of +arguments as they appear in the corresponding \ref regression::LARS "LARS" constructor: + @code LARS(const arma::mat& data, const arma::rowvec& responses, @@ -75,11 +113,16 @@ constructor: @section hptgradient Gradient-Based Optimization -When we know approximate optimal values (which we can try to find with the -\c GridSearch optimizer) for real-valued hyper-parameters, we can try to tune -them even more with gradient-based optimization. In the following example we -try to optimize the \c lambda1 and \c lambda2 hyper-parameters for \c LARS with -the \c GradientDescent optimizer. +In some cases we may wish to optimize a hyperparameter over the space of all +possible real values, instead of providing a grid in which to search. +Alternately, we may know approximately optimal values from a grid search for a +real-valued hyperparameter, but wish to further tune those values. + +In this case, we can use a gradient-based optimizer for hyperparameter search. +In the following example, we try to optimize the \c lambda1 and \c lambda2 +hyper-parameters for \ref regression::LARS "LARS" with the +\ref optimization::GradientDescent "GradientDescent" optimizer. + @code HyperParameterTuner hpt3(validationSize, data, responses); @@ -100,4 +143,95 @@ the \c GradientDescent optimizer. Fixed(useCholesky), initialLambda1, initialLambda2); @endcode +@section hpt_class The HyperParameterTuner class + +The \c HyperParameterTuner class is very similar to the +\ref cv::KFoldCV "KFoldCV" and \ref cv::SimpleCV "SimpleCV" classes (see the +@ref "cross-validation tutorial" for more information on those two classes), but +there are a few important differences. + +First, the \c HyperParameterTuner accepts five different hyperparameters; only +the first two of these are required: + + - \c MLAlgorithm This is the algorithm to be used. + - \c Metric This is the performance measure to be used; see + @ref cvbasic_metrics for more information. + - \c CVType This is the type of cross-validation to be used for evaluating the + performance measure; this should be \ref cv::KFoldCV "KFoldCV" or + \ref cv::SimpleCV "SimpleCV". + - \c OptimizerType This is the type of optimizer to use; it can be + \c GridSearch or a gradient-based optimizer. + - \c MatType This is the type of data matrix to use. The default is + \c arma::mat. This only needs to be changed if you are specifically + using sparse data, or if you want to use a numeric type other than + \c double. + +The last two template parameters are automatically inferred by the +\c HyperParameterTuner and should not need to be manually specified. + +Typically, \ref cv::SimpleCV "SimpleCV" is a good choice for \c CVType because +it takes so much less time to compute than full \ref cv::KFoldCV "KFoldCV"; +however, the disadvantage is that \ref cv::SimpleCV "SimpleCV" might give a +somewhat more noisy estimate of the performance measure on unseen test data. + +The constructor for the \c HyperParameterTuner is called with exactly the same +arguments as the corresponding \c CVType that has been chosen. For more +information on that, please see the +@ref cvbasic_api "cross-validation constructor tutorial". As an example, if we +are using \ref cv::SimpleCV "SimpleCV" and wish to hold out 20\% of the dataset +as a validation set, we might construct a \c HyperParameterTuner like this: + +@code +// We will use LinearRegression as the MLAlgorithm, and MSE as the performance +// measure. Our dataset is 'dataset' and the responses are 'responses'. +HyperParameterTuner hpt(0.2, dataset, + responses); +@endcode + +Next, we must set up the hyperparameters to be optimized. If we are doing a +grid search with the \ref optimization::GridSearch "GridSearch" optimizer (the +default), then we only need to pass a `std::vector` (for non-numeric +hyperparameters) or an `arma::vec` (for numeric hyperparameters) containing all +of the possible choices that we wish to search over. + +For instance, a set of numeric values might be chosen like this, for the +\c lambda parameter (of type \c double): + +@code +arma::vec lambdaSet = arma::vec("0.0 0.1 0.5 1.0"); +@endcode + +Similarly, a set of non-numeric values might be chosen like this, for the +\c intercept parameter: + +@code +std::vector interceptSet = { false, true }; +@endcode + +Once all of these are set up, the \c HyperParameterTuner::Optimize() method may +be called to find the best set of hyperparameters: + +@code +bool intercept; +double lambda; +std::tie(lambda, intercept) = hpt.Optimize(lambdaSet, interceptSet); +@endcode + +Alternately, the \c Fixed() method (detailed in the @ref hptfixed +"Fixed arguments" section) can be used to fix the values of some parameters. + +For continuous optimizers like +\ref optimization::GradientDescent "GradientDescent", a range does not need to +be specified but instead only a single value. See the +\ref hptgradient "Gradient-Based Optimization" section for more details. + +@section hptfurther Further documentation + +For more information on the \c HyperParameterTuner class, see the +mlpack::hpt::HyperParameterTuner class documentation and the +@ref cv "cross-validation tutorial". + */ + +} // namespace hpt +} // namespace mlpack From 20a7d28828ba2c5b7fc5c628e5d6932994f36f19 Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Mon, 18 Sep 2017 14:46:09 -0400 Subject: [PATCH 21/39] Don't use gmm_diag on Visual Studio. --- src/mlpack/methods/gmm/em_fit.hpp | 4 ++++ src/mlpack/methods/gmm/em_fit_impl.hpp | 9 ++++++++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/src/mlpack/methods/gmm/em_fit.hpp b/src/mlpack/methods/gmm/em_fit.hpp index be70157181..3326c57adc 100644 --- a/src/mlpack/methods/gmm/em_fit.hpp +++ b/src/mlpack/methods/gmm/em_fit.hpp @@ -162,6 +162,9 @@ class EMFit dists, const arma::vec& weights) const; + // Armadillo uses uword internally as an OpenMP index type, which crashes + // Visual Studio. + #ifdef _WIN32 /** * Use the Armadillo gmm_diag clusterer to train a GMM with diagonal * covariance. If InitialClusteringType == kmeans::KMeans<>, this will use @@ -177,6 +180,7 @@ class EMFit std::vector& dists, arma::vec& weights, const bool useInitialModel); + #endif //! Maximum iterations of EM algorithm. size_t maxIterations; diff --git a/src/mlpack/methods/gmm/em_fit_impl.hpp b/src/mlpack/methods/gmm/em_fit_impl.hpp index 8070e7d06a..40d7013fbf 100644 --- a/src/mlpack/methods/gmm/em_fit_impl.hpp +++ b/src/mlpack/methods/gmm/em_fit_impl.hpp @@ -41,12 +41,15 @@ void EMFit::Estimate( const bool useInitialModel) { // Shortcut: if the user is using the DiagonalConstraint, then we will call - // out to Armadillo. + // out to Armadillo. But Armadillo uses uword internally as an OpenMP index + // type, which crashes Visual Studio, so don't do this on Windows. + #ifndef _WIN32 if (std::is_same::value) { ArmadilloGMMWrapper(observations, dists, weights, useInitialModel); return; } + #endif // Only perform initial clustering if the user wanted it. if (!useInitialModel) @@ -325,6 +328,9 @@ void EMFit::Serialize( ar & CreateNVP(constraint, "constraint"); } +// Armadillo uses uword internally as an OpenMP index type, which crashes Visual +// Studio. +#ifndef _WIN32 template void EMFit:: ArmadilloGMMWrapper(const arma::mat& observations, @@ -382,6 +388,7 @@ ArmadilloGMMWrapper(const arma::mat& observations, dists[i].Covariance(std::move(arma::diagmat(g.dcovs.col(i)))); } } +#endif } // namespace gmm } // namespace mlpack From 3470e7be1a2714fd1fa530ac07e48405e82b986b Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Mon, 18 Sep 2017 15:45:04 -0400 Subject: [PATCH 22/39] Fix typo. --- src/mlpack/methods/gmm/em_fit.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mlpack/methods/gmm/em_fit.hpp b/src/mlpack/methods/gmm/em_fit.hpp index 3326c57adc..82a646db70 100644 --- a/src/mlpack/methods/gmm/em_fit.hpp +++ b/src/mlpack/methods/gmm/em_fit.hpp @@ -164,7 +164,7 @@ class EMFit // Armadillo uses uword internally as an OpenMP index type, which crashes // Visual Studio. - #ifdef _WIN32 + #ifndef _WIN32 /** * Use the Armadillo gmm_diag clusterer to train a GMM with diagonal * covariance. If InitialClusteringType == kmeans::KMeans<>, this will use From a6065e486d325c270ea4c5009a4f16d1fb22e7d7 Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Mon, 25 Sep 2017 10:43:27 -0400 Subject: [PATCH 23/39] Fix comments from Kirill. --- doc/guide/cv.hpp | 27 ++++++++++++++------------- doc/guide/hpt.hpp | 13 +++++++------ 2 files changed, 21 insertions(+), 19 deletions(-) diff --git a/doc/guide/cv.hpp b/doc/guide/cv.hpp index 132cb48c0a..7d69049b5f 100644 --- a/doc/guide/cv.hpp +++ b/doc/guide/cv.hpp @@ -204,7 +204,7 @@ class CustomMeasure const DataType& data, const ResponsesType& labels) { - // Inside the function you should call model.Predict() and compare the + // Inside the method you should call model.Predict() and compare the // values with the labels, in order to get the desired performance measure // and return it. } @@ -229,14 +229,15 @@ parameter: In addition, there are two more template parameters, but these are automatically extracted from the given \c MLAlgorithm class, and users should not need to -specify these parameters. +specify these parameters except when using an unconventional type like +\c arma::fmat for data points. The general structure of the \c KFoldCV and \c SimpleCV classes is split into two parts: - The constructor: create the object, and store the data for the \c MLAlgorithm training. - - The \c Evaluate() function: take any non-data parameters for the + - The \c Evaluate() method: take any non-data parameters for the \c MLAlgorithm and calculate the desired performance measure. This split is important because it defines the API: all data-related parameters @@ -292,13 +293,13 @@ regression, you could call the constructor call the constructor \c "KFoldCV(k, xs, ys, numClasses)" and for \c SimpleCV you would call the constructor \c "SimpleCV(pct, xs, ys, numClasses)". -@subsection cvbasic_api_evaluate The Evaluate() function +@subsection cvbasic_api_evaluate The Evaluate() method -The other function that \c KFoldCV and \c SimpleCV have is the function to +The other method that \c KFoldCV and \c SimpleCV have is the method to actually calculate the performance measure: \c Evaluate(). The \c Evaluate() -function takes any hyperparameters that would follow the data arguments to the -constructor or \c Train() function of the given \c MLAlgorithm. The -\c Evaluate() function takes no more arguments than that, and returns the +method takes any hyperparameters that would follow the data arguments to the +constructor or \c Train() method of the given \c MLAlgorithm. The +\c Evaluate() method takes no more arguments than that, and returns the desired performance measure on the dataset. Therefore, let us suppose that we are interested in cross-validating the @@ -322,10 +323,10 @@ constructor OptimizerType optimizer = OptimizerType()); @endcode -Note that all parameters are \c numClasses are optional. This means that we can -specify none or any of them in our call to \c Evaluate(). Below is some example -code showing three different ways we can call \c Evaluate() with the \c cv -object from the code snippet above. +Note that all parameters after are \c numClasses are optional. This means that +we can specify none or any of them in our call to \c Evaluate(). Below is some +example code showing three different ways we can call \c Evaluate() with the +\c cv object from the code snippet above. @code // First, call with all defaults. @@ -341,7 +342,7 @@ double result3 = cv.Evaluate(0.5, false, sgd); @endcode The same general idea applies to any \c MLAlgorithm: all hyperparameters must be -passed to the \c Evaluate() function of \c KFoldCV or \c SimpleCV. +passed to the \c Evaluate() method of \c KFoldCV or \c SimpleCV. @section cvbasic_further Further references diff --git a/doc/guide/hpt.hpp b/doc/guide/hpt.hpp index 63ff5644b1..ff6bc8ffa1 100644 --- a/doc/guide/hpt.hpp +++ b/doc/guide/hpt.hpp @@ -36,7 +36,7 @@ optimization strategy (\ref optimization::GridSearch "GridSearch" will be used b default) you are going to use. Then, you must pass the same arguments as for the cross-validation classes: the data and labels (or responses) to use are given to the constructor, and the possible hyperparameter values are given to -the \c HyperParameterTuner::Optimize() function, which returns the best +the \c HyperParameterTuner::Optimize() method, which returns the best algorithm configuration as a \c std::tuple<>. Let's see some examples. @@ -75,7 +75,7 @@ that we have specified what values should be tried. @section hptfixed Fixed Arguments When some hyper-parameters should not be optimized, you can specify values -for them with the \c Fixed() function as in the following example of trying to +for them with the \c Fixed() method as in the following example of trying to find good \c lambda1 and \c lambda2 values for \ref regression::LARS "LARS" (least-angle regression). @@ -115,8 +115,8 @@ constructor: In some cases we may wish to optimize a hyperparameter over the space of all possible real values, instead of providing a grid in which to search. -Alternately, we may know approximately optimal values from a grid search for a -real-valued hyperparameter, but wish to further tune those values. +Alternately, we may know approximately optimal values from a grid search for +real-valued hyperparameters, but wish to further tune those values. In this case, we can use a gradient-based optimizer for hyperparameter search. In the following example, we try to optimize the \c lambda1 and \c lambda2 @@ -151,7 +151,7 @@ The \c HyperParameterTuner class is very similar to the there are a few important differences. First, the \c HyperParameterTuner accepts five different hyperparameters; only -the first two of these are required: +the first three of these are required: - \c MLAlgorithm This is the algorithm to be used. - \c Metric This is the performance measure to be used; see @@ -167,7 +167,8 @@ the first two of these are required: \c double. The last two template parameters are automatically inferred by the -\c HyperParameterTuner and should not need to be manually specified. +\c HyperParameterTuner and should not need to be manually specified, unless an +unconventional data type like \c arma::fmat is being used for data points. Typically, \ref cv::SimpleCV "SimpleCV" is a good choice for \c CVType because it takes so much less time to compute than full \ref cv::KFoldCV "KFoldCV"; From b8ee319818e6213da4b92cb69dab94b956795401 Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Tue, 26 Sep 2017 11:11:07 -0400 Subject: [PATCH 24/39] Refactor words to be more clear. --- doc/guide/cv.hpp | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/doc/guide/cv.hpp b/doc/guide/cv.hpp index 7d69049b5f..4c6f8f9478 100644 --- a/doc/guide/cv.hpp +++ b/doc/guide/cv.hpp @@ -66,10 +66,12 @@ performance measure, we can write the following piece of code. double softmaxAccuracy = cv.Evaluate(lambda); @endcode -Note that the \c Evaluate method of \c KFoldCV takes any hyperparameters of the -algorithm---that is, anything that is not \c data, \c labels, or \c numClasses. -To be more specific, in this example the \c Evaluate method relies on the -following \ref regression::SoftmaxRegression "SoftmaxRegression" constructor: +Note that the \c Evaluate method of \c KFoldCV takes any hyperparameters of an +algorithm---that is, anything that is not \c data, \c labels, \c numClasses, +\c datasetInfo, or \c weights (those last three may not be present for every +algorithm type). To be more specific, in this example the \c Evaluate method +relies on the following \ref regression::SoftmaxRegression "SoftmaxRegression" +constructor: @code template From b558187e3163687da78403e48ad57062f69bcb5d Mon Sep 17 00:00:00 2001 From: theJonan Date: Thu, 28 Sep 2017 18:59:07 +0300 Subject: [PATCH 25/39] - After merge clearings. - training / testing set usable for tagging. - PRINT_PARAM_STRING() utilized more. --- src/mlpack/methods/det/det_main.cpp | 224 ++++++++++++----------- src/mlpack/methods/det/dt_utils_impl.hpp | 2 +- src/mlpack/methods/det/dtree_impl.hpp | 1 + 3 files changed, 122 insertions(+), 105 deletions(-) diff --git a/src/mlpack/methods/det/det_main.cpp b/src/mlpack/methods/det/det_main.cpp index e20d1c1a11..a7159cd121 100644 --- a/src/mlpack/methods/det/det_main.cpp +++ b/src/mlpack/methods/det/det_main.cpp @@ -133,46 +133,60 @@ void mlpackMain() { // Validate input parameters. if (CLI::HasParam("training") && CLI::HasParam("input_model")) - Log::Fatal << "Only one of --training_file (-t) or --input_model_file (-m) " - << "may be specified!" << endl; + Log::Fatal << "Only one of " << PRINT_PARAM_STRING("training") << " or " << + PRINT_PARAM_STRING("input_model") << " may be specified!" << endl; if (!CLI::HasParam("training") && !CLI::HasParam("input_model")) - Log::Fatal << "Neither --training_file (-t) nor --input_model_file (-m) " - << "are specified!" << endl; + Log::Fatal << "Neither " << PRINT_PARAM_STRING("training") << " nor " << + PRINT_PARAM_STRING("input_model") << " are specified!" << endl; + + if (CLI::HasParam("tag_file") && + !CLI::HasParam("training") && !CLI::HasParam("test")) + { + Log::Fatal << "Neither " << PRINT_PARAM_STRING("training") << " nor " << + PRINT_PARAM_STRING("test") << " are specified, but needed when " << + PRINT_PARAM_STRING("tag_file") << " is asked." << endl; + } if (!CLI::HasParam("training")) { if (CLI::HasParam("training_set_estimates")) - Log::Warn << "--training_set_estimates_file (-e) ignored because " - << "--training_file (-t) is not specified." << endl; + Log::Warn << PRINT_PARAM_STRING("training_set_estimates") << + " ignored because " << PRINT_PARAM_STRING("training") << + " is not specified." << endl; if (CLI::HasParam("folds")) - Log::Warn << "--folds (-f) ignored because --training_file (-t) is not " - << "specified." << endl; + Log::Warn << PRINT_PARAM_STRING("folds") << " ignored because " << + PRINT_PARAM_STRING("training") << " is not specified." << endl; if (CLI::HasParam("min_leaf_size")) - Log::Warn << "--min_leaf_size (-l) ignored because --training_file (-t) " - << "is not specified." << endl; + Log::Warn << PRINT_PARAM_STRING("min_leaf_size") << " ignored because " << + PRINT_PARAM_STRING("training") << " is not specified." << endl; if (CLI::HasParam("max_leaf_size")) - Log::Warn << "--max_leaf_size (-L) ignored because --training_file (-t) " - << "is not specified." << endl; + Log::Warn << PRINT_PARAM_STRING("max_leaf_size") << " ignored because " << + PRINT_PARAM_STRING("training") << " is not specified." << endl; } else if (!CLI::HasParam("output_model") && !CLI::HasParam("training_set_estimates") && !CLI::HasParam("vi")) { - Log::Warn << "None of --output_model_file (-M), --training_set_estimates " - << "(-e), or --vi (-i) are specified; no output will be saved!" << endl; + Log::Warn << "None of " << PRINT_PARAM_STRING("output_model") << ", " << + PRINT_PARAM_STRING("training_set_estimates") << ", or " << + PRINT_PARAM_STRING("vi") << " are specified; no output will be saved!" << + endl; } if (!CLI::HasParam("test") && CLI::HasParam("test_set_estimates")) - Log::Warn << "--test_set_estimates_file (-E) ignored because --test_file " - << "(-T) is not specified." << endl; + Log::Warn << PRINT_PARAM_STRING("test_set_estimates") << " ignored " << + "because " << PRINT_PARAM_STRING("test") << " is not specified." << endl; // Are we training a DET or loading from file? DTree* tree; + arma::mat trainingData; + arma::mat testData; + if (CLI::HasParam("training")) { - arma::mat trainingData = std::move(CLI::GetParam("training")); - + trainingData = std::move(CLI::GetParam("training")); + const bool regularization = false; // const bool regularization = CLI::HasParam("volume_regularization"); const int maxLeafSize = CLI::GetParam("max_leaf_size"); @@ -214,7 +228,7 @@ void mlpackMain() // the given file. if (CLI::HasParam("test")) { - arma::mat testData = std::move(CLI::GetParam("test")); + testData = std::move(CLI::GetParam("test")); if (CLI::HasParam("test_set_estimates")) { // Compute test set densities. @@ -229,95 +243,97 @@ void mlpackMain() CLI::GetParam("test_set_estimates") = std::move(testDensities); } - if (CLI::HasParam("tag_file")) + // Print variable importance. + if (CLI::HasParam("vi")) { - const string tagFile = CLI::GetParam("tag_file"); - std::ofstream ofs; - ofs.open(tagFile, std::ofstream::out); - - arma::Row counters; - - Timer::Start("det_test_set_tagging"); - if (!ofs.is_open()) - { - Log::Warn << "Unable to open file '" << tagFile - << "' to save tag membership info." - << std::endl; - } - else if (CLI::HasParam("path_format")) - { - const bool reqCounters = CLI::HasParam("tag_counters_file"); - const string pathFormat = CLI::GetParam("path_format"); - - PathCacher::PathFormat theFormat; - if (pathFormat == "lr" || pathFormat == "LR") - theFormat = PathCacher::FormatLR; - else if (pathFormat == "lr-id" || pathFormat == "LR-ID") - theFormat = PathCacher::FormatLR_ID; - else if (pathFormat == "id-lr" || pathFormat == "ID-LR") - theFormat = PathCacher::FormatID_LR; - else - { - Log::Warn << "Unknown path format specified: '" << pathFormat - << "'. Valid are: lr | lr-id | id-lr. Defaults to 'lr'" << endl; - theFormat = PathCacher::FormatLR; - } - - PathCacher path(theFormat, tree); - counters.zeros(path.NumNodes()); - - for (size_t i = 0; i < testData.n_cols; i++) - { - int tag = tree->FindBucket(testData.unsafe_col(i)); - - ofs << tag << " " << path.PathFor(tag) << std::endl; - for (;tag >= 0 && reqCounters; tag = path.ParentOf(tag)) - counters(tag) += 1; - } - - ofs.close(); - - if (reqCounters) - { - ofs.open(CLI::GetParam("tag_counters_file"), - std::ofstream::out); - - for (size_t j = 0;j < counters.n_elem; ++j) - ofs << j << " " - << counters(j) << " " - << path.PathFor(j) << endl; - - ofs.close(); - } - } - else - { - int numLeaves = tree->TagTree(); - counters.zeros(numLeaves); - - for (size_t i = 0; i < testData.n_cols; i++) - { - const int tag = tree->FindBucket(testData.unsafe_col(i)); - - ofs << tag << std::endl; - counters(tag) += 1; - } - - if (CLI::HasParam("tag_counters_file")) - data::Save(CLI::GetParam("tag_counters_file"), counters); - } - - Timer::Stop("det_test_set_tagging"); - ofs.close(); + arma::vec importances; + tree->ComputeVariableImportance(importances); + CLI::GetParam("vi") = importances.t(); } } - - // Print variable importance. - if (CLI::HasParam("vi")) + + if (CLI::HasParam("tag_file")) { - arma::vec importances; - tree->ComputeVariableImportance(importances); - CLI::GetParam("vi") = importances.t(); + const arma::mat& estimationData = + CLI::HasParam("test") ? testData : trainingData; + const string tagFile = CLI::GetParam("tag_file"); + std::ofstream ofs; + ofs.open(tagFile, std::ofstream::out); + + arma::Row counters; + + Timer::Start("det_test_set_tagging"); + if (!ofs.is_open()) + { + Log::Warn << "Unable to open file '" << tagFile + << "' to save tag membership info." + << std::endl; + } + else if (CLI::HasParam("path_format")) + { + const bool reqCounters = CLI::HasParam("tag_counters_file"); + const string pathFormat = CLI::GetParam("path_format"); + + PathCacher::PathFormat theFormat; + if (pathFormat == "lr" || pathFormat == "LR") + theFormat = PathCacher::FormatLR; + else if (pathFormat == "lr-id" || pathFormat == "LR-ID") + theFormat = PathCacher::FormatLR_ID; + else if (pathFormat == "id-lr" || pathFormat == "ID-LR") + theFormat = PathCacher::FormatID_LR; + else + { + Log::Warn << "Unknown path format specified: '" << pathFormat + << "'. Valid are: lr | lr-id | id-lr. Defaults to 'lr'" << endl; + theFormat = PathCacher::FormatLR; + } + + PathCacher path(theFormat, tree); + counters.zeros(path.NumNodes()); + + for (size_t i = 0; i < estimationData.n_cols; i++) + { + int tag = tree->FindBucket(estimationData.unsafe_col(i)); + + ofs << tag << " " << path.PathFor(tag) << std::endl; + for (;tag >= 0 && reqCounters; tag = path.ParentOf(tag)) + counters(tag) += 1; + } + + ofs.close(); + + if (reqCounters) + { + ofs.open(CLI::GetParam("tag_counters_file"), + std::ofstream::out); + + for (size_t j = 0;j < counters.n_elem; ++j) + ofs << j << " " + << counters(j) << " " + << path.PathFor(j) << endl; + + ofs.close(); + } + } + else + { + int numLeaves = tree->TagTree(); + counters.zeros(numLeaves); + + for (size_t i = 0; i < estimationData.n_cols; i++) + { + const int tag = tree->FindBucket(estimationData.unsafe_col(i)); + + ofs << tag << std::endl; + counters(tag) += 1; + } + + if (CLI::HasParam("tag_counters_file")) + data::Save(CLI::GetParam("tag_counters_file"), counters); + } + + Timer::Stop("det_test_set_tagging"); + ofs.close(); } // Save the model, if desired. diff --git a/src/mlpack/methods/det/dt_utils_impl.hpp b/src/mlpack/methods/det/dt_utils_impl.hpp index 64c0503358..dc376d15a6 100644 --- a/src/mlpack/methods/det/dt_utils_impl.hpp +++ b/src/mlpack/methods/det/dt_utils_impl.hpp @@ -172,7 +172,7 @@ DTree* Trainer(MatType& dataset, while (dtree->SubtreeLeaves() > 1) { std::pair treeSeq(oldAlpha, - dtree.SubtreeLeavesLogNegError()); + dtree->SubtreeLeavesLogNegError()); prunedSequence.push_back(treeSeq); oldAlpha = alpha; alpha = dtree->PruneAndUpdate(oldAlpha, dataset.n_cols, useVolumeReg); diff --git a/src/mlpack/methods/det/dtree_impl.hpp b/src/mlpack/methods/det/dtree_impl.hpp index 4a589e6808..4f50999e9a 100644 --- a/src/mlpack/methods/det/dtree_impl.hpp +++ b/src/mlpack/methods/det/dtree_impl.hpp @@ -1044,6 +1044,7 @@ void DTree::Serialize(Archive& ar, ar & CreateNVP(maxVals, "maxVals"); ar & CreateNVP(minVals, "minVals"); + // This is added in order to reduce (dramatically!) the model file size. if (Archive::is_loading::value && left && right) FillMinMax(minVals, maxVals); } From 59c9c2dd71a5ad983e2b5b3512ffa09ed998896b Mon Sep 17 00:00:00 2001 From: theJonan Date: Tue, 3 Oct 2017 10:41:19 +0300 Subject: [PATCH 26/39] [core] Moved EnumerateTree() from DET to core/tree. [det] Style and documentation fixes. --- src/mlpack/core/tree/CMakeLists.txt | 1 + src/mlpack/core/tree/enumerate_tree.hpp | 61 +++++++++ src/mlpack/methods/det/det_main.cpp | 161 +++++------------------ src/mlpack/methods/det/dt_utils.hpp | 45 ++++++- src/mlpack/methods/det/dt_utils_impl.hpp | 86 ++++++++++-- src/mlpack/methods/det/dtree.hpp | 31 +++-- src/mlpack/methods/det/dtree_impl.hpp | 41 +----- 7 files changed, 238 insertions(+), 188 deletions(-) create mode 100644 src/mlpack/core/tree/enumerate_tree.hpp diff --git a/src/mlpack/core/tree/CMakeLists.txt b/src/mlpack/core/tree/CMakeLists.txt index adc5bae0b2..0b2f01a5f3 100644 --- a/src/mlpack/core/tree/CMakeLists.txt +++ b/src/mlpack/core/tree/CMakeLists.txt @@ -120,6 +120,7 @@ set(SOURCES statistic.hpp traversal_info.hpp tree_traits.hpp + enumerate_tree.hpp ) # add directory name to sources diff --git a/src/mlpack/core/tree/enumerate_tree.hpp b/src/mlpack/core/tree/enumerate_tree.hpp new file mode 100644 index 0000000000..7dcf0e258d --- /dev/null +++ b/src/mlpack/core/tree/enumerate_tree.hpp @@ -0,0 +1,61 @@ +/** + * @file enumerate_tree.hpp + * @author Ivan (Jonan) Georgiev + * + * This file contains function that performs a simple depth-first walk on the tree + * calling `Enter` and `Leave` methods of a provided walker. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_CORE_TREE_ENUMERATE_TREE_HPP +#define MLPACK_CORE_TREE_ENUMERATE_TREE_HPP + +namespace mlpack { +namespace tree /** Trees and tree-building procedures. */ { +namespace enumerate { + + // Actual implementation of the enumeration. The problem is the unified detection + // if we're on the root, because Enter and Leave expect the parent being passed. + template + void EnumerateTreeImpl(TreeType* tree, Walker& walker, bool root) + { + if (root) + walker.Enter(tree, (const TreeType*)nullptr); + + const size_t numChildren = tree->NumChildren(); + for (size_t i = 0; i < numChildren; ++i) + { + TreeType* child = tree->ChildPtr(i); + walker.Enter(child, tree); + EnumerateTreeImpl(child, walker, false); + walker.Leave(child, tree); + } + + if (root) + walker.Leave(tree, (const TreeType*)nullptr); + } +} // namespace enumerate + +/** + * Traverses all nodes of the tree, including the inner ones. On each node + * two methods of the `enumer` are called: + * + * Enter(DTree* node, DTree* parent); + * Leave(Dtree* node, DTree* parent); + * + * @param walker An instance of custom class, receiver of the enumeration. + */ +template +inline void EnumerateTree(TreeType* tree, Walker& walker) +{ + enumerate::EnumerateTreeImpl(tree, walker, true); +} + +} // namespace tree +} // namespace mlpack + + +#endif // MLPACK_CORE_TREE_ENUMERATE_TREE_HPP diff --git a/src/mlpack/methods/det/det_main.cpp b/src/mlpack/methods/det/det_main.cpp index a7159cd121..06dc3b1afe 100644 --- a/src/mlpack/methods/det/det_main.cpp +++ b/src/mlpack/methods/det/det_main.cpp @@ -33,6 +33,11 @@ PROGRAM_INFO("Density Estimation With Density Estimation Trees", " with the " + PRINT_PARAM_STRING("training_set_estimates") + " output " "parameter." "\n\n" + "Enabling path printing for each node outputs strings like `LRLRLR` for " + "each entry in the test set, or training one, if test is not provided. " + "If `lr-id` or `id-lr` is provided, it'll also print the id (tag) of the " + "node (not just leaf!) in tree along the path to the leaf." + "\n\n" "This program also can provide density estimates for a set of test points, " "specified in the " + PRINT_PARAM_STRING("test") + " parameter. The " "density estimation tree used for this task will be the tree that was " @@ -65,12 +70,13 @@ PARAM_MATRIX_OUT("vi", "The output variable importance values for each " PARAM_STRING_IN("path_format", "The format of path printing - lr|id-lr|lr-id", "p", "lr"); -PARAM_STRING_OUT("tag_counters_file", "The file to output tag counters.", "c"); +PARAM_STRING_OUT("tag_counters_file", "The file to output number of points " + "that went to each leaf.", "c"); -PARAM_STRING_OUT("raw_estimates_file", "The file to output the estimations from " - "the unpruned tree.", "u"); +PARAM_STRING_OUT("raw_estimates_file", "The file to output the estimations from" + " the unpruned tree.", "u"); -PARAM_STRING_OUT("tag_file", "The file to output the tags (and possibly paths) " +PARAM_STRING_OUT("tag_file", "The file to output the tags (and possibly paths)" " for each sample in the test set.", "g"); PARAM_FLAG("skip_pruning", "Whether to bypass the pruning process and output " @@ -92,43 +98,6 @@ PARAM_FLAG("volume_regularization", "This flag gives the used the option to use" */ -class PathCacher -{ -public: - enum PathFormat - { - FormatLR, - FormatLR_ID, - FormatID_LR - }; - - template - PathCacher(PathFormat fmt, DTree* tree); - - template - void Enter(const DTree* node, const DTree* parent); - - template - void Leave(const DTree* node, const DTree* parent); - - const std::string& PathFor(int tag) const; - - int ParentOf(int tag) const; - - size_t NumNodes() const { return pathCache.size(); } - -protected: - typedef std::list > PathType; - typedef std::vector > PathCacheType; - - PathType path; - PathFormat format; - PathCacheType pathCache; - - std::string BuildString(); -}; - - void mlpackMain() { // Validate input parameters. @@ -139,7 +108,7 @@ void mlpackMain() if (!CLI::HasParam("training") && !CLI::HasParam("input_model")) Log::Fatal << "Neither " << PRINT_PARAM_STRING("training") << " nor " << PRINT_PARAM_STRING("input_model") << " are specified!" << endl; - + if (CLI::HasParam("tag_file") && !CLI::HasParam("training") && !CLI::HasParam("test")) { @@ -182,29 +151,29 @@ void mlpackMain() DTree* tree; arma::mat trainingData; arma::mat testData; - + if (CLI::HasParam("training")) { trainingData = std::move(CLI::GetParam("training")); - + const bool regularization = false; // const bool regularization = CLI::HasParam("volume_regularization"); const int maxLeafSize = CLI::GetParam("max_leaf_size"); const int minLeafSize = CLI::GetParam("min_leaf_size"); const bool skipPruning = CLI::HasParam("skip_pruning"); size_t folds = CLI::GetParam("folds"); - + if (folds == 0) folds = trainingData.n_cols; // Obtain the optimal tree. Timer::Start("det_training"); - tree = Trainer(trainingData, folds, regularization, + tree = Trainer(trainingData, folds, regularization, maxLeafSize, minLeafSize, CLI::GetParam("raw_estimates_file"), skipPruning); Timer::Stop("det_training"); - + // Compute training set estimates, if desired. if (CLI::HasParam("training_set_estimates")) { @@ -234,15 +203,15 @@ void mlpackMain() // Compute test set densities. Timer::Start("det_test_set_estimation"); arma::rowvec testDensities(testData.n_cols); - + for (size_t i = 0; i < testData.n_cols; i++) testDensities[i] = tree->ComputeValue(testData.unsafe_col(i)); - + Timer::Stop("det_test_set_estimation"); CLI::GetParam("test_set_estimates") = std::move(testDensities); } - + // Print variable importance. if (CLI::HasParam("vi")) { @@ -251,7 +220,7 @@ void mlpackMain() CLI::GetParam("vi") = importances.t(); } } - + if (CLI::HasParam("tag_file")) { const arma::mat& estimationData = @@ -259,7 +228,7 @@ void mlpackMain() const string tagFile = CLI::GetParam("tag_file"); std::ofstream ofs; ofs.open(tagFile, std::ofstream::out); - + arma::Row counters; Timer::Start("det_test_set_tagging"); @@ -284,34 +253,34 @@ void mlpackMain() else { Log::Warn << "Unknown path format specified: '" << pathFormat - << "'. Valid are: lr | lr-id | id-lr. Defaults to 'lr'" << endl; + << "'. Valid are: lr | lr-id | id-lr. Defaults to 'lr'." << endl; theFormat = PathCacher::FormatLR; } - + PathCacher path(theFormat, tree); counters.zeros(path.NumNodes()); - + for (size_t i = 0; i < estimationData.n_cols; i++) { int tag = tree->FindBucket(estimationData.unsafe_col(i)); ofs << tag << " " << path.PathFor(tag) << std::endl; - for (;tag >= 0 && reqCounters; tag = path.ParentOf(tag)) + for (; tag >= 0 && reqCounters; tag = path.ParentOf(tag)) counters(tag) += 1; } - + ofs.close(); - + if (reqCounters) { ofs.open(CLI::GetParam("tag_counters_file"), std::ofstream::out); - - for (size_t j = 0;j < counters.n_elem; ++j) + + for (size_t j = 0; j < counters.n_elem; ++j) ofs << j << " " << counters(j) << " " << path.PathFor(j) << endl; - + ofs.close(); } } @@ -319,15 +288,15 @@ void mlpackMain() { int numLeaves = tree->TagTree(); counters.zeros(numLeaves); - + for (size_t i = 0; i < estimationData.n_cols; i++) { const int tag = tree->FindBucket(estimationData.unsafe_col(i)); - + ofs << tag << std::endl; counters(tag) += 1; } - + if (CLI::HasParam("tag_counters_file")) data::Save(CLI::GetParam("tag_counters_file"), counters); } @@ -344,67 +313,3 @@ void mlpackMain() if (!CLI::HasParam("input_model") && !CLI::HasParam("output_model")) delete tree; } - - -template -PathCacher::PathCacher(PathCacher::PathFormat fmt, DTree* dtree) : format(fmt) -{ - pathCache.resize(dtree->TagTree(0, true)); - pathCache[0] = PathCacheType::value_type(-1, ""); - dtree->EnumerateTree(*this); -} - -template -void PathCacher::Enter(const DTree* node, - const DTree* parent) -{ - if (parent == nullptr) - return; - - int tag = node->BucketTag(); - - path.push_back(PathType::value_type(parent->Left() == node, tag)); - pathCache[tag] = PathCacheType::value_type(parent->BucketTag(), - (node->SubtreeLeaves() > 1) ? - "" : BuildString() - ); -} - -template -void PathCacher::Leave(const DTree* , const DTree* parent) -{ - if (parent != nullptr) - path.pop_back(); -} - -std::string PathCacher::BuildString() -{ - std::string str(""); - for (PathType::iterator it = path.begin(); it != path.end(); it++) - { - switch (format) - { - case FormatLR: - str += it->first ? "L" : "R"; - break; - case FormatLR_ID: - str += (it->first ? "L" : "R") + std::to_string(it->second); - break; - case FormatID_LR: - str += std::to_string(it->second) + (it->first ? "L" : "R"); - break; - } - } - - return str; -} - -int PathCacher::ParentOf(int tag) const -{ - return pathCache[tag].first; -} - -const std::string& PathCacher::PathFor(int tag) const -{ - return pathCache[tag].second; -} diff --git a/src/mlpack/methods/det/dt_utils.hpp b/src/mlpack/methods/det/dt_utils.hpp index 7469b8358b..9fc81c2541 100644 --- a/src/mlpack/methods/det/dt_utils.hpp +++ b/src/mlpack/methods/det/dt_utils.hpp @@ -36,7 +36,7 @@ void PrintLeafMembership(DTree* dtree, const arma::Mat& labels, const size_t numClasses, const std::string& leafClassMembershipFile = ""); - + /** * Print the variable importance of each dimension of a density estimation tree. * Optionally, pass the name of a file to print this information to (otherwise @@ -70,6 +70,49 @@ DTree* Trainer(MatType& dataset, const std::string unprunedTreeOutput = "", const bool skipPruning = false); +/** + * The class responsible for cacheing the path to each node of the tree. Its instance + * is provided to EnumerateTree() utility ONCE and it caches the paths to all the + * leafs and then easily (and quickly) retrieves these paths for each test entry. + */ +class PathCacher +{ +public: + enum PathFormat + { + FormatLR, + FormatLR_ID, + FormatID_LR + }; + + template + PathCacher(PathFormat fmt, DTree* tree); + + template + void Enter(const DTree* node, + const DTree* parent); + + template + void Leave(const DTree* node, + const DTree* parent); + + const std::string& PathFor(int tag) const; + + int ParentOf(int tag) const; + + size_t NumNodes() const { return pathCache.size(); } + +protected: + typedef std::list > PathType; + typedef std::vector > PathCacheType; + + PathType path; + PathFormat format; + PathCacheType pathCache; + + std::string BuildString(); +}; + } // namespace det } // namespace mlpack diff --git a/src/mlpack/methods/det/dt_utils_impl.hpp b/src/mlpack/methods/det/dt_utils_impl.hpp index dc376d15a6..dcd69e9f23 100644 --- a/src/mlpack/methods/det/dt_utils_impl.hpp +++ b/src/mlpack/methods/det/dt_utils_impl.hpp @@ -14,6 +14,7 @@ #define MLPACK_METHODS_DET_DT_UTILS_IMPL_HPP #include "dt_utils.hpp" +#include namespace mlpack { namespace det { @@ -65,7 +66,7 @@ void PrintLeafMembership(DTree* dtree, return; } - + template void PrintVariableImportance(const DTree* dtree, const std::string viFile) @@ -156,17 +157,18 @@ DTree* Trainer(MatType& dataset, outfile.close(); } - + if (skipPruning) return dtree; if (folds == dataset.n_cols) Log::Info << "Performing leave-one-out cross validation." << std::endl; else - Log::Info << "Performing " << folds << "-fold cross validation." << std::endl; - - Timer::Start("prunning_sequence"); - + Log::Info << "Performing " << folds << "-fold cross validation." << + std::endl; + + Timer::Start("pruning_sequence"); + // Sequentially prune and save the alpha values and the values of c_t^2 * r_t. std::vector > prunedSequence; while (dtree->SubtreeLeaves() > 1) @@ -186,10 +188,11 @@ DTree* Trainer(MatType& dataset, Log::Assert(dtree->SubtreeLeavesLogNegError() <= treeSeq.second); } - std::pair treeSeq(oldAlpha, dtree->SubtreeLeavesLogNegError()); + std::pair treeSeq(oldAlpha, + dtree->SubtreeLeavesLogNegError()); prunedSequence.push_back(treeSeq); - Timer::Stop("prunning_sequence"); + Timer::Stop("pruning_sequence"); Log::Info << prunedSequence.size() << " trees in the sequence; maximum alpha:" << " " << oldAlpha << "." << std::endl; @@ -205,7 +208,7 @@ DTree* Trainer(MatType& dataset, // implementation. omp_size_t is the appropriate type according to the // platform. #pragma omp parallel for default(none) \ - shared(cvData, prunedSequence, regularizationConstants) + shared(prunedSequence, regularizationConstants) for (omp_size_t fold = 0; fold < (omp_size_t) folds; fold++) { // Break up data into train and test sets. @@ -340,6 +343,71 @@ DTree* Trainer(MatType& dataset, return dtree; } +template +PathCacher::PathCacher(PathCacher::PathFormat fmt, DTree* dtree) : + format(fmt) +{ + // Here we use TagTree()'s output to determine the number of _nodes_ in the tree). + pathCache.resize(dtree->TagTree(0, true)); + pathCache[0] = PathCacheType::value_type(-1, ""); + tree::EnumerateTree(dtree, *this); +} + +template +void PathCacher::Enter(const DTree* node, + const DTree* parent) +{ + if (parent == nullptr) + return; + + int tag = node->BucketTag(); + + path.push_back(PathType::value_type(parent->Left() == node, tag)); + pathCache[tag] = PathCacheType::value_type(parent->BucketTag(), + (node->SubtreeLeaves() > 1) ? + "" : BuildString()); +} + +template +void PathCacher::Leave(const DTree* , + const DTree* parent) +{ + if (parent != nullptr) + path.pop_back(); +} + +std::string PathCacher::BuildString() +{ + std::string str(""); + for (PathType::iterator it = path.begin(); it != path.end(); it++) + { + switch (format) + { + case FormatLR: + str += it->first ? "L" : "R"; + break; + case FormatLR_ID: + str += (it->first ? "L" : "R") + std::to_string(it->second); + break; + case FormatID_LR: + str += std::to_string(it->second) + (it->first ? "L" : "R"); + break; + } + } + + return str; +} + +int PathCacher::ParentOf(int tag) const +{ + return pathCache[tag].first; +} + +const std::string& PathCacher::PathFor(int tag) const +{ + return pathCache[tag].second; +} + } // namespace det } // namespace mlpack diff --git a/src/mlpack/methods/det/dtree.hpp b/src/mlpack/methods/det/dtree.hpp index ba9196cb89..a167aad6df 100644 --- a/src/mlpack/methods/det/dtree.hpp +++ b/src/mlpack/methods/det/dtree.hpp @@ -189,20 +189,7 @@ class DTree * @param tag Tag for the next leaf; leave at 0 for the initial call. */ TagType TagTree(const TagType& tag = 0, bool internal = false); - - - /** - * Traverses all nodes of the tree, including the inner ones. On each node - * two methods of the `enumer` are called: - * - * Enter(DTree* node, DTree* parent); - * Leave(Dtree* node, DTree* parent); - * - * @param walker An instance of custom class, receiver of the enumeration. - */ - template - void EnumerateTree(Walker& walker) const; - + /** * Return the tag of the leaf containing the query. This is useful for @@ -212,6 +199,7 @@ class DTree */ TagType FindBucket(const VecType& query) const; + /** * Compute the variable importance of each dimension in the learned tree. * @@ -316,6 +304,18 @@ class DTree double AlphaUpper() const { return alphaUpper; } //! Return the current bucket's ID, if leaf, or -1 otherwise TagType BucketTag() const { return bucketTag; } + //! Return the number of children in this node. + size_t NumChildren() const { return !left ? 0 : 2; } + + /** + * Return the specified child (0 will be left, 1 will be right). If the index + * is greater than 1, this will return the right child. + * + * @param child Index of child to return. + */ + DTree& Child(const size_t child) const { return !child ? *left : *right; } + + DTree*& ChildPtr(const size_t child) { return (!child) ? left : right; } //! Return the maximum values. const StatType& MaxVals() const { return maxVals; } @@ -349,10 +349,9 @@ class DTree const size_t splitDim, const ElemType splitValue, arma::Col& oldFromNew) const; - + void FillMinMax(const StatType& mins, const StatType& maxs); - }; } // namespace det diff --git a/src/mlpack/methods/det/dtree_impl.hpp b/src/mlpack/methods/det/dtree_impl.hpp index 4f50999e9a..c5afc9c142 100644 --- a/src/mlpack/methods/det/dtree_impl.hpp +++ b/src/mlpack/methods/det/dtree_impl.hpp @@ -67,7 +67,7 @@ namespace details { typedef std::pair SplitItem; arma::rowvec dimVec = data(dim, arma::span(start, end - 1)); - + // We sort these, in-place (it's a copy of the data, anyways). std::sort(dimVec.begin(), dimVec.end()); @@ -891,7 +891,7 @@ TagType DTree::TagTree(const TagType& tag, bool internal) bucketTag = tag; return (tag + 1); } - + TagType nextTag; if (internal) { @@ -900,36 +900,10 @@ TagType DTree::TagTree(const TagType& tag, bool internal) } else nextTag = tag; - + return right->TagTree(left->TagTree(nextTag, internal), internal); } -// Enumerate the nodes of the tree. -template -template -void DTree::EnumerateTree(Walker& walker) const -{ - if (root == 1) - walker.Enter(this, (const DTree*)nullptr); - - if (subtreeLeaves > 1) - { - // walk the left ... - walker.Enter(left, this); - left->EnumerateTree(walker); - walker.Leave(left, this); - - // ... and the right. - walker.Enter(right, this); - right->EnumerateTree(walker); - walker.Leave(right, this); - } - - if (root == 1) - walker.Leave(this, (const DTree*)nullptr); - -} - template TagType DTree::FindBucket(const VecType& query) const { @@ -941,7 +915,7 @@ TagType DTree::FindBucket(const VecType& query) const if (!WithinRange(query)) return -1; } - + // If we are a leaf... if (subtreeLeaves == 1) { @@ -994,14 +968,14 @@ void DTree::FillMinMax(const StatType& mins, minVals = mins; maxVals = maxs; } - + if (left && right) { StatType maxValsL(maxs); StatType maxValsR(maxs); StatType minValsL(mins); StatType minValsR(mins); - + maxValsL[splitDim] = minValsR[splitDim] = splitValue; left->FillMinMax(minValsL, maxValsL); right->FillMinMax(minValsR, maxValsR); @@ -1038,7 +1012,7 @@ void DTree::Serialize(Archive& ar, ar & CreateNVP(left, "left"); ar & CreateNVP(right, "right"); - + if (root) { ar & CreateNVP(maxVals, "maxVals"); @@ -1049,4 +1023,3 @@ void DTree::Serialize(Archive& ar, FillMinMax(minVals, maxVals); } } - From 191c375a8d733ab68c7ca68bb52386d63eb9bfb9 Mon Sep 17 00:00:00 2001 From: theJonan Date: Tue, 3 Oct 2017 14:10:20 +0300 Subject: [PATCH 27/39] CI fixes. --- src/mlpack/core/tree/enumerate_tree.hpp | 38 +++++++++++++----------- src/mlpack/methods/det/dt_utils.hpp | 4 +-- src/mlpack/methods/det/dt_utils_impl.hpp | 5 ++-- 3 files changed, 25 insertions(+), 22 deletions(-) diff --git a/src/mlpack/core/tree/enumerate_tree.hpp b/src/mlpack/core/tree/enumerate_tree.hpp index 7dcf0e258d..734f48df2d 100644 --- a/src/mlpack/core/tree/enumerate_tree.hpp +++ b/src/mlpack/core/tree/enumerate_tree.hpp @@ -17,26 +17,28 @@ namespace mlpack { namespace tree /** Trees and tree-building procedures. */ { namespace enumerate { - // Actual implementation of the enumeration. The problem is the unified detection - // if we're on the root, because Enter and Leave expect the parent being passed. - template - void EnumerateTreeImpl(TreeType* tree, Walker& walker, bool root) +// Actual implementation of the enumeration. The problem is the unified +// detection if we're on the root, because Enter and Leave expect the +// parent being passed. +template +void EnumerateTreeImpl(TreeType* tree, Walker& walker, bool root) +{ + if (root) + walker.Enter(tree, (const TreeType*)nullptr); + + const size_t numChildren = tree->NumChildren(); + for (size_t i = 0; i < numChildren; ++i) { - if (root) - walker.Enter(tree, (const TreeType*)nullptr); - - const size_t numChildren = tree->NumChildren(); - for (size_t i = 0; i < numChildren; ++i) - { - TreeType* child = tree->ChildPtr(i); - walker.Enter(child, tree); - EnumerateTreeImpl(child, walker, false); - walker.Leave(child, tree); - } - - if (root) - walker.Leave(tree, (const TreeType*)nullptr); + TreeType* child = tree->ChildPtr(i); + walker.Enter(child, tree); + EnumerateTreeImpl(child, walker, false); + walker.Leave(child, tree); } + + if (root) + walker.Leave(tree, (const TreeType*)nullptr); +} + } // namespace enumerate /** diff --git a/src/mlpack/methods/det/dt_utils.hpp b/src/mlpack/methods/det/dt_utils.hpp index 9fc81c2541..8e49597936 100644 --- a/src/mlpack/methods/det/dt_utils.hpp +++ b/src/mlpack/methods/det/dt_utils.hpp @@ -77,7 +77,7 @@ DTree* Trainer(MatType& dataset, */ class PathCacher { -public: + public: enum PathFormat { FormatLR, @@ -102,7 +102,7 @@ public: size_t NumNodes() const { return pathCache.size(); } -protected: + protected: typedef std::list > PathType; typedef std::vector > PathCacheType; diff --git a/src/mlpack/methods/det/dt_utils_impl.hpp b/src/mlpack/methods/det/dt_utils_impl.hpp index dcd69e9f23..b453e8d6df 100644 --- a/src/mlpack/methods/det/dt_utils_impl.hpp +++ b/src/mlpack/methods/det/dt_utils_impl.hpp @@ -347,7 +347,8 @@ template PathCacher::PathCacher(PathCacher::PathFormat fmt, DTree* dtree) : format(fmt) { - // Here we use TagTree()'s output to determine the number of _nodes_ in the tree). + // Here we use TagTree()'s output to determine the + // number of _nodes_ in the tree. pathCache.resize(dtree->TagTree(0, true)); pathCache[0] = PathCacheType::value_type(-1, ""); tree::EnumerateTree(dtree, *this); @@ -411,4 +412,4 @@ const std::string& PathCacher::PathFor(int tag) const } // namespace det } // namespace mlpack -#endif +#endif // MLPACK_METHODS_DET_DT_UTILS_IMPL_HPP From 81e5bf683b6a23be8c571278b05d4fcb15535b20 Mon Sep 17 00:00:00 2001 From: theJonan Date: Tue, 3 Oct 2017 15:36:12 +0300 Subject: [PATCH 28/39] - CI restart commit. --- src/mlpack/core/tree/enumerate_tree.hpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/mlpack/core/tree/enumerate_tree.hpp b/src/mlpack/core/tree/enumerate_tree.hpp index 734f48df2d..a115a7a5e5 100644 --- a/src/mlpack/core/tree/enumerate_tree.hpp +++ b/src/mlpack/core/tree/enumerate_tree.hpp @@ -41,6 +41,7 @@ void EnumerateTreeImpl(TreeType* tree, Walker& walker, bool root) } // namespace enumerate + /** * Traverses all nodes of the tree, including the inner ones. On each node * two methods of the `enumer` are called: From 5294299ee01f630b44ff1625065ec745a1cc79da Mon Sep 17 00:00:00 2001 From: theJonan Date: Tue, 3 Oct 2017 17:04:15 +0300 Subject: [PATCH 29/39] [det] Static analyzer fixes. --- src/mlpack/methods/det/dtree_impl.hpp | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/mlpack/methods/det/dtree_impl.hpp b/src/mlpack/methods/det/dtree_impl.hpp index c5afc9c142..97fb24c607 100644 --- a/src/mlpack/methods/det/dtree_impl.hpp +++ b/src/mlpack/methods/det/dtree_impl.hpp @@ -191,6 +191,9 @@ template DTree& DTree::operator=( const DTree& obj) { + if (this == &obj) + return *this; + // Copy the values from the other tree. start = obj.start; end = obj.end; @@ -213,7 +216,7 @@ DTree& DTree::operator=( // Copy the children. left = ((obj.left == NULL) ? NULL : new DTree(*obj.left)); - left = ((obj.right == NULL) ? NULL : new DTree(*obj.right)); + right = ((obj.right == NULL) ? NULL : new DTree(*obj.right)); return *this; } @@ -258,6 +261,9 @@ template DTree& DTree::operator=( DTree&& obj) { + if (this == &obj) + return *this; + // Move the values from the other tree. start = obj.start; end = obj.end; From 1d7e7940804426e6acddeea2d6f6cc5333e1bb62 Mon Sep 17 00:00:00 2001 From: theJonan Date: Wed, 4 Oct 2017 10:10:20 +0300 Subject: [PATCH 30/39] [det] raw_estimates removed. [det] Style cleanups. --- src/mlpack/core/tree/enumerate_tree.hpp | 4 ++-- src/mlpack/methods/det/det_main.cpp | 10 +++------- src/mlpack/methods/det/dt_utils_impl.hpp | 23 ----------------------- 3 files changed, 5 insertions(+), 32 deletions(-) diff --git a/src/mlpack/core/tree/enumerate_tree.hpp b/src/mlpack/core/tree/enumerate_tree.hpp index a115a7a5e5..8811c8f4c2 100644 --- a/src/mlpack/core/tree/enumerate_tree.hpp +++ b/src/mlpack/core/tree/enumerate_tree.hpp @@ -46,8 +46,8 @@ void EnumerateTreeImpl(TreeType* tree, Walker& walker, bool root) * Traverses all nodes of the tree, including the inner ones. On each node * two methods of the `enumer` are called: * - * Enter(DTree* node, DTree* parent); - * Leave(Dtree* node, DTree* parent); + * Enter(TreeType* node, TreeType* parent); + * Leave(TreeType* node, TreeType* parent); * * @param walker An instance of custom class, receiver of the enumeration. */ diff --git a/src/mlpack/methods/det/det_main.cpp b/src/mlpack/methods/det/det_main.cpp index 06dc3b1afe..cc55fccc34 100644 --- a/src/mlpack/methods/det/det_main.cpp +++ b/src/mlpack/methods/det/det_main.cpp @@ -73,14 +73,11 @@ PARAM_STRING_IN("path_format", "The format of path printing - lr|id-lr|lr-id", PARAM_STRING_OUT("tag_counters_file", "The file to output number of points " "that went to each leaf.", "c"); -PARAM_STRING_OUT("raw_estimates_file", "The file to output the estimations from" - " the unpruned tree.", "u"); - PARAM_STRING_OUT("tag_file", "The file to output the tags (and possibly paths)" " for each sample in the test set.", "g"); PARAM_FLAG("skip_pruning", "Whether to bypass the pruning process and output " - "the unpruned tree only", "s"); + "the unpruned tree only.", "s"); // Parameters for the training algorithm. PARAM_INT_IN("folds", "The number of folds of cross-validation to perform for " @@ -164,13 +161,12 @@ void mlpackMain() size_t folds = CLI::GetParam("folds"); if (folds == 0) - folds = trainingData.n_cols; + folds = trainingData.n_cols; // Obtain the optimal tree. Timer::Start("det_training"); tree = Trainer(trainingData, folds, regularization, maxLeafSize, minLeafSize, - CLI::GetParam("raw_estimates_file"), skipPruning); Timer::Stop("det_training"); @@ -224,7 +220,7 @@ void mlpackMain() if (CLI::HasParam("tag_file")) { const arma::mat& estimationData = - CLI::HasParam("test") ? testData : trainingData; + CLI::HasParam("test") ? testData : trainingData; const string tagFile = CLI::GetParam("tag_file"); std::ofstream ofs; ofs.open(tagFile, std::ofstream::out); diff --git a/src/mlpack/methods/det/dt_utils_impl.hpp b/src/mlpack/methods/det/dt_utils_impl.hpp index b453e8d6df..16aa42e427 100644 --- a/src/mlpack/methods/det/dt_utils_impl.hpp +++ b/src/mlpack/methods/det/dt_utils_impl.hpp @@ -112,7 +112,6 @@ DTree* Trainer(MatType& dataset, const bool useVolumeReg, const size_t maxLeafSize, const size_t minLeafSize, - const std::string unprunedTreeOutput, const bool skipPruning) { // Initialize the tree. @@ -136,28 +135,6 @@ DTree* Trainer(MatType& dataset, Log::Info << dtree->SubtreeLeaves() << " leaf nodes in the tree using full " << "dataset; minimum alpha: " << alpha << "." << std::endl; - // Compute densities for the training points in the full tree, if we were - // asked for this. - if (unprunedTreeOutput != "") - { - std::ofstream outfile(unprunedTreeOutput.c_str()); - if (outfile.good()) - { - for (size_t i = 0; i < dataset.n_cols; ++i) - { - arma::vec testPoint = dataset.unsafe_col(i); - outfile << dtree->ComputeValue(testPoint) << std::endl; - } - } - else - { - Log::Warn << "Can't open '" << unprunedTreeOutput << "' to write computed" - << " densities to." << std::endl; - } - - outfile.close(); - } - if (skipPruning) return dtree; From 6cacb8fbaba44035b55db0d43dae2422bede1b03 Mon Sep 17 00:00:00 2001 From: theJonan Date: Fri, 6 Oct 2017 09:43:23 +0300 Subject: [PATCH 31/39] [det] Minor fix on dtree tagging. --- src/mlpack/methods/det/dtree.hpp | 6 ++++-- src/mlpack/methods/det/dtree_impl.hpp | 10 +++++----- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/src/mlpack/methods/det/dtree.hpp b/src/mlpack/methods/det/dtree.hpp index a167aad6df..7ebcb4aea5 100644 --- a/src/mlpack/methods/det/dtree.hpp +++ b/src/mlpack/methods/det/dtree.hpp @@ -184,11 +184,13 @@ class DTree /** * Index the buckets for possible usage later; this results in every leaf in * the tree having a specific tag (accessible with BucketTag()). This - * function calls itself recursively. + * function calls itself recursively. The tag is incremented with + * `operator++()`, so any `TagType` overriding it will do. * * @param tag Tag for the next leaf; leave at 0 for the initial call. + * @param everyNodde Whether to increment on every node, not just leaves. */ - TagType TagTree(const TagType& tag = 0, bool internal = false); + TagType TagTree(const TagType& tag = 0, bool everyNode = false); /** diff --git a/src/mlpack/methods/det/dtree_impl.hpp b/src/mlpack/methods/det/dtree_impl.hpp index 97fb24c607..cb499dcef7 100644 --- a/src/mlpack/methods/det/dtree_impl.hpp +++ b/src/mlpack/methods/det/dtree_impl.hpp @@ -889,25 +889,25 @@ double DTree::ComputeValue(const VecType& query) const // Index the buckets for possible usage later. template -TagType DTree::TagTree(const TagType& tag, bool internal) +TagType DTree::TagTree(const TagType& tag, bool every) { if (subtreeLeaves == 1) { // Only label leaves. bucketTag = tag; - return (tag + 1); + return ++tag; } TagType nextTag; - if (internal) + if (every) { bucketTag = tag; - nextTag = tag + 1; + nextTag = ++tag; } else nextTag = tag; - return right->TagTree(left->TagTree(nextTag, internal), internal); + return right->TagTree(left->TagTree(nextTag, every), every); } template From bfb550e4c8a8254337bb1581905b7d3091798969 Mon Sep 17 00:00:00 2001 From: theJonan Date: Fri, 6 Oct 2017 17:13:04 +0300 Subject: [PATCH 32/39] [det] Stupid ++tag fix. --- src/mlpack/methods/det/dtree_impl.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/mlpack/methods/det/dtree_impl.hpp b/src/mlpack/methods/det/dtree_impl.hpp index cb499dcef7..686b2804e4 100644 --- a/src/mlpack/methods/det/dtree_impl.hpp +++ b/src/mlpack/methods/det/dtree_impl.hpp @@ -895,14 +895,14 @@ TagType DTree::TagTree(const TagType& tag, bool every) { // Only label leaves. bucketTag = tag; - return ++tag; + return (tag + 1); } TagType nextTag; if (every) { bucketTag = tag; - nextTag = ++tag; + nextTag = (tag + 1); } else nextTag = tag; From f6e4eab9b45592e36acda96228d10910c632129f Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Mon, 9 Oct 2017 10:57:55 -0400 Subject: [PATCH 33/39] Minor style and formatting fixes. --- src/mlpack/methods/det/det_main.cpp | 12 +- src/mlpack/methods/det/dt_utils.hpp | 58 +++-- src/mlpack/methods/det/dt_utils_impl.hpp | 16 +- src/mlpack/methods/det/dtree.hpp | 12 +- src/mlpack/methods/det/dtree_impl.hpp | 294 ++++++++++++----------- 5 files changed, 213 insertions(+), 179 deletions(-) diff --git a/src/mlpack/methods/det/det_main.cpp b/src/mlpack/methods/det/det_main.cpp index cc55fccc34..d9e05eb991 100644 --- a/src/mlpack/methods/det/det_main.cpp +++ b/src/mlpack/methods/det/det_main.cpp @@ -33,10 +33,14 @@ PROGRAM_INFO("Density Estimation With Density Estimation Trees", " with the " + PRINT_PARAM_STRING("training_set_estimates") + " output " "parameter." "\n\n" - "Enabling path printing for each node outputs strings like `LRLRLR` for " - "each entry in the test set, or training one, if test is not provided. " - "If `lr-id` or `id-lr` is provided, it'll also print the id (tag) of the " - "node (not just leaf!) in tree along the path to the leaf." + "Enabling path printing for each node outputs the path from the root node " + "to a leaf for each entry in the test set, or training set (if a test set " + "is not provided). Strings like 'LRLRLR' (indicating that traversal went " + "to the left child, then the right child, then the left child, and so " + "forth) will be output. If 'lr-id' or 'id-lr' are given as the " + + PRINT_PARAM_STRING("path_format") + " parameter, then the ID (tag) of " + "every node along the path will be printed after or before the L or R " + "character indicating the direction of traversal, respectively." "\n\n" "This program also can provide density estimates for a set of test points, " "specified in the " + PRINT_PARAM_STRING("test") + " parameter. The " diff --git a/src/mlpack/methods/det/dt_utils.hpp b/src/mlpack/methods/det/dt_utils.hpp index 8e49597936..3e2c9ec7d4 100644 --- a/src/mlpack/methods/det/dt_utils.hpp +++ b/src/mlpack/methods/det/dt_utils.hpp @@ -71,40 +71,68 @@ DTree* Trainer(MatType& dataset, const bool skipPruning = false); /** - * The class responsible for cacheing the path to each node of the tree. Its instance - * is provided to EnumerateTree() utility ONCE and it caches the paths to all the - * leafs and then easily (and quickly) retrieves these paths for each test entry. + * This class is responsible for caching the path to each node of the tree. Its + * instance is provided to EnumerateTree() utility ONCE and it caches the paths + * to all the leafs and then easily (and quickly) retrieves these paths for each + * test entry. */ class PathCacher { public: + /** + * Possible formats to use for output. + */ enum PathFormat { + //! Print only whether we went left or right. FormatLR, + //! Print the direction, then the tag of the node. FormatLR_ID, + //! Print the tag of the node, then the direction. FormatID_LR }; - template + /** + * Construct a PathCacher object on the given tree with the given format. + * + * @param fmt Format to use for output. + * @param tree Tree to cache paths in. + */ + template PathCacher(PathFormat fmt, DTree* tree); - template - void Enter(const DTree* node, - const DTree* parent); + /** + * Enter a given node. + */ + template + void Enter(const DTree* node, + const DTree* parent); - template - void Leave(const DTree* node, - const DTree* parent); + /** + * Leave the given node. + */ + template + void Leave(const DTree* node, + const DTree* parent); - const std::string& PathFor(int tag) const; + /** + * Return the constructed path for a given tag. + */ + const std::string& PathFor(int tag) const; - int ParentOf(int tag) const; + /** + * Get the parent tag of a given tag. + */ + int ParentOf(int tag) const; - size_t NumNodes() const { return pathCache.size(); } + /** + * Get the number of nodes in the path cache. + */ + size_t NumNodes() const { return pathCache.size(); } protected: - typedef std::list > PathType; - typedef std::vector > PathCacheType; + typedef std::list> PathType; + typedef std::vector> PathCacheType; PathType path; PathFormat format; diff --git a/src/mlpack/methods/det/dt_utils_impl.hpp b/src/mlpack/methods/det/dt_utils_impl.hpp index 16aa42e427..866413245b 100644 --- a/src/mlpack/methods/det/dt_utils_impl.hpp +++ b/src/mlpack/methods/det/dt_utils_impl.hpp @@ -320,9 +320,9 @@ DTree* Trainer(MatType& dataset, return dtree; } -template +template PathCacher::PathCacher(PathCacher::PathFormat fmt, DTree* dtree) : - format(fmt) + format(fmt) { // Here we use TagTree()'s output to determine the // number of _nodes_ in the tree. @@ -331,9 +331,9 @@ PathCacher::PathCacher(PathCacher::PathFormat fmt, DTree* dtree) : tree::EnumerateTree(dtree, *this); } -template -void PathCacher::Enter(const DTree* node, - const DTree* parent) +template +void PathCacher::Enter(const DTree* node, + const DTree* parent) { if (parent == nullptr) return; @@ -346,9 +346,9 @@ void PathCacher::Enter(const DTree* node, "" : BuildString()); } -template -void PathCacher::Leave(const DTree* , - const DTree* parent) +template +void PathCacher::Leave(const DTree* /* node */, + const DTree* parent) { if (parent != nullptr) path.pop_back(); diff --git a/src/mlpack/methods/det/dtree.hpp b/src/mlpack/methods/det/dtree.hpp index 7ebcb4aea5..88acb45990 100644 --- a/src/mlpack/methods/det/dtree.hpp +++ b/src/mlpack/methods/det/dtree.hpp @@ -46,12 +46,12 @@ template StatType; + //! The actual, underlying type we're working with. + typedef typename MatType::elem_type ElemType; + //! The type of vector we are using. + typedef typename MatType::vec_type VecType; + //! The statistic type we are holding. + typedef typename arma::Col StatType; /** * Create an empty density estimation tree. diff --git a/src/mlpack/methods/det/dtree_impl.hpp b/src/mlpack/methods/det/dtree_impl.hpp index 686b2804e4..5dbdad7254 100644 --- a/src/mlpack/methods/det/dtree_impl.hpp +++ b/src/mlpack/methods/det/dtree_impl.hpp @@ -20,134 +20,136 @@ using namespace det; namespace details { - /** - * This one sorts and scand the given per-dimension extract and puts all splits - * in a vector, that can easily be iterated afterwards. General implementation. - */ - template - void ExtractSplits(std::vector>& splitVec, - const MatType& data, - size_t dim, - const size_t start, - const size_t end, - const size_t minLeafSize) - { - static_assert( - std::is_same::value == true, - "The ElemType does not correspond to the matrix's element type."); - typedef std::pair SplitItem; - const typename MatType::row_type dimVec = +/** + * This one sorts and scand the given per-dimension extract and puts all splits + * in a vector, that can easily be iterated afterwards. General implementation. + */ +template +void ExtractSplits(std::vector>& splitVec, + const MatType& data, + size_t dim, + const size_t start, + const size_t end, + const size_t minLeafSize) +{ + static_assert( + std::is_same::value == true, + "The ElemType does not correspond to the matrix's element type."); + + typedef std::pair SplitItem; + const typename MatType::row_type dimVec = arma::sort(data(dim, arma::span(start, end - 1))); - // Ensure the minimum leaf size on both sides. We need to figure out why - // there are spikes if this minLeafSize is enforced here... - for (size_t i = minLeafSize - 1; i < dimVec.n_elem - minLeafSize; ++i) + // Ensure the minimum leaf size on both sides. We need to figure out why there + // are spikes if this minLeafSize is enforced here... + for (size_t i = minLeafSize - 1; i < dimVec.n_elem - minLeafSize; ++i) + { + // This makes sense for real continuous data. This kinda corrupts the data + // and estimation if the data is ordinal. Potentially we can fix that by + // taking into account ordinality later in the min/max update, but then we + // can end-up with a zero-volumed dimension. No good. + const ElemType split = (dimVec[i] + dimVec[i + 1]) / 2.0; + + // Check if we can split here (two points are different) + if (split != dimVec[i]) + splitVec.push_back(SplitItem(split, i + 1)); + } +} + +// Now the custom arma::Mat implementation. +template +void ExtractSplits(std::vector>& splitVec, + const arma::Mat& data, + size_t dim, + const size_t start, + const size_t end, + const size_t minLeafSize) +{ + typedef std::pair SplitItem; + arma::rowvec dimVec = data(dim, arma::span(start, end - 1)); + + // We sort these, in-place (it's a copy of the data, anyways). + std::sort(dimVec.begin(), dimVec.end()); + + for (size_t i = minLeafSize - 1; i < dimVec.n_elem - minLeafSize; ++i) + { + // This makes sense for real continuous data. This kinda corrupts the data + // and estimation if the data is ordinal. Potentially we can fix that by + // taking into account ordinality later in the min/max update, but then we + // can end-up with a zero-volumed dimension. No good. + const ElemType split = (dimVec[i] + dimVec[i + 1]) / 2.0; + + if (split != dimVec[i]) + splitVec.push_back(SplitItem(split, i + 1)); + } +} + +// This the custom, sparse optimized implementation of the same routine. +template +void ExtractSplits(std::vector>& splitVec, + const arma::SpMat& data, + size_t dim, + const size_t start, + const size_t end, + const size_t minLeafSize) +{ + // It's common sense, but we also use it in a check later. + Log::Assert(minLeafSize > 0); + + typedef std::pair SplitItem; + const size_t n_elem = end - start; + + // Construct a vector of values. + const arma::SpRow row = data(dim, arma::span(start, end - 1)); + std::vector valsVec(row.begin(), row.end()); + + // ... and sort it! + std::sort(valsVec.begin(), valsVec.end()); + + // Now iterate over the values, taking account for the over-the-zeroes jump + // and construct the splits vector. + const size_t zeroes = n_elem - valsVec.size(); + ElemType lastVal = -std::numeric_limits::max(); + size_t padding = 0; + + for (size_t i = 0; i < valsVec.size(); ++i) + { + const ElemType newVal = valsVec[i]; + if (lastVal < ElemType(0) && newVal > ElemType(0) && zeroes > 0) { - // This makes sense for real continuous data. This kinda corrupts the - // data and estimation if the data is ordinal. Potentially we can fix - // that by taking into account ordinality later in the min/max update, - // but then we can end-up with a zero-volumed dimension. No good. - const ElemType split = (dimVec[i] + dimVec[i + 1]) / 2.0; + Log::Assert(padding == 0); // We should arrive here once! + + // The minLeafSize > 0 also guarantees we're not entering right at the + // start. + if (i >= minLeafSize && i <= n_elem - minLeafSize) + splitVec.push_back(SplitItem(lastVal / 2.0, i)); + + padding = zeroes; + lastVal = ElemType(0); + } + + // This is the normal case. + if (i + padding >= minLeafSize && i + padding <= n_elem - minLeafSize) + { + // This makes sense for real continuous data. This kinda corrupts the + // data and estimation if the data is ordinal. Potentially we can fix that + // by taking into account ordinality later in the min/max update, but then + // we can end-up with a zero-volumed dimension. No good. + const ElemType split = (lastVal + newVal) / 2.0; // Check if we can split here (two points are different) - if (split != dimVec[i]) - splitVec.push_back(SplitItem(split, i + 1)); + if (split != newVal) + splitVec.push_back(SplitItem(split, i + padding)); } + + lastVal = newVal; } +} - // Now the custom arma::Mat implementation - template - void ExtractSplits(std::vector>& splitVec, - const arma::Mat& data, - size_t dim, - const size_t start, - const size_t end, - const size_t minLeafSize) - { - typedef std::pair SplitItem; - arma::rowvec dimVec = data(dim, arma::span(start, end - 1)); +} // namespace details - // We sort these, in-place (it's a copy of the data, anyways). - std::sort(dimVec.begin(), dimVec.end()); - - for (size_t i = minLeafSize - 1; i < dimVec.n_elem - minLeafSize; ++i) - { - // This makes sense for real continuous data. This kinda corrupts the - // data and estimation if the data is ordinal. Potentially we can fix - // that by taking into account ordinality later in the min/max update, - // but then we can end-up with a zero-volumed dimension. No good. - const ElemType split = (dimVec[i] + dimVec[i + 1]) / 2.0; - - if (split != dimVec[i]) - splitVec.push_back(SplitItem(split, i + 1)); - } - } - - // This the custom, sparse optimized implementation of the same routine. - template - void ExtractSplits(std::vector>& splitVec, - const arma::SpMat& data, - size_t dim, - const size_t start, - const size_t end, - const size_t minLeafSize) - { - // It's common sense, but we also use it in a check later. - Log::Assert(minLeafSize > 0); - - typedef std::pair SplitItem; - const size_t n_elem = end - start; - - // Construct a vector of values. - const arma::SpRow row = data(dim, arma::span(start, end - 1)); - std::vector valsVec(row.begin(), row.end()); - - // ... and sort it! - std::sort(valsVec.begin(), valsVec.end()); - - // Now iterate over the values, taking account for the over-the-zeroes - // jump and construct the splits vector. - const size_t zeroes = n_elem - valsVec.size(); - ElemType lastVal = -std::numeric_limits::max(); - size_t padding = 0; - - for (size_t i = 0; i < valsVec.size(); ++i) - { - const ElemType newVal = valsVec[i]; - if (lastVal < ElemType(0) && newVal > ElemType(0) && zeroes > 0) - { - Log::Assert(padding == 0); // We should arrive here once! - - // The minLeafSize > 0 also guarantees we're not entering right at the - // start. - if (i >= minLeafSize && i <= n_elem - minLeafSize) - splitVec.push_back(SplitItem(lastVal / 2.0, i)); - - padding = zeroes; - lastVal = ElemType(0); - } - - // the normal case - if (i + padding >= minLeafSize && i + padding <= n_elem - minLeafSize) - { - // This makes sense for real continuous data. This kinda corrupts the - // data and estimation if the data is ordinal. Potentially we can fix - // that by taking into account ordinality later in the min/max update, - // but then we can end-up with a zero-volumed dimension. No good. - const ElemType split = (lastVal + newVal) / 2.0; - - // Check if we can split here (two points are different) - if (split != newVal) - splitVec.push_back(SplitItem(split, i + padding)); - } - - lastVal = newVal; - } - } -}; // namespace details - -template +template DTree::DTree() : start(0), end(0), @@ -165,7 +167,7 @@ DTree::DTree() : right(NULL) { /* Nothing to do. */ } -template +template DTree::DTree(const DTree& obj) : start(obj.start), end(obj.end), @@ -187,7 +189,7 @@ DTree::DTree(const DTree& obj) : /* Nothing to do. */ } -template +template DTree& DTree::operator=( const DTree& obj) { @@ -221,7 +223,7 @@ DTree& DTree::operator=( return *this; } -template +template DTree::DTree(DTree&& obj): start(obj.start), end(obj.end), @@ -257,7 +259,7 @@ DTree::DTree(DTree&& obj): obj.right = NULL; } -template +template DTree& DTree::operator=( DTree&& obj) { @@ -308,8 +310,8 @@ DTree& DTree::operator=( } -// Root node initializers -template +// Root node initializers. +template DTree::DTree(const StatType& maxVals, const StatType& minVals, const size_t totalPoints) : @@ -331,7 +333,7 @@ DTree::DTree(const StatType& maxVals, right(NULL) { /* Nothing to do. */ } -template +template DTree::DTree(MatType & data) : start(0), end(data.n_cols), @@ -352,8 +354,8 @@ DTree::DTree(MatType & data) : logNegError = LogNegativeError(data.n_cols); } -// Non-root node initializers -template +// Non-root node initializers. +template DTree::DTree(const StatType& maxVals, const StatType& minVals, const size_t start, @@ -377,7 +379,7 @@ DTree::DTree(const StatType& maxVals, right(NULL) { /* Nothing to do. */ } -template +template DTree::DTree(const StatType& maxVals, const StatType& minVals, const size_t totalPoints, @@ -401,7 +403,7 @@ DTree::DTree(const StatType& maxVals, right(NULL) { /* Nothing to do. */ } -template +template DTree::~DTree() { delete left; @@ -410,7 +412,7 @@ DTree::~DTree() // This function computes the log-l2-negative-error of a given node from the // formula R(t) = log(|t|^2 / (N^2 V_t)). -template +template double DTree::LogNegativeError(const size_t totalPoints) const { // log(-|t|^2 / (N^2 V_t)) = log(-1) + 2 log(|t|) - 2 log(N) - log(V_t). @@ -431,7 +433,7 @@ double DTree::LogNegativeError(const size_t totalPoints) const // This function finds the best split with respect to the L2-error, by trying // all possible splits. The dataset is the full data set but the start and // end are used to obtain the point in this node. -template +template bool DTree::FindSplit(const MatType& data, size_t& splitDim, ElemType& splitValue, @@ -439,7 +441,7 @@ bool DTree::FindSplit(const MatType& data, double& rightError, const size_t minLeafSize) const { - typedef std::pair SplitItem; + typedef std::pair SplitItem; // Ensure the dimensionality of the data is the same as the dimensionality of // the bounding rectangle. @@ -549,7 +551,7 @@ bool DTree::FindSplit(const MatType& data, return splitFound; } -template +template size_t DTree::SplitData(MatType& data, const size_t splitDim, const ElemType splitValue, @@ -583,8 +585,8 @@ size_t DTree::SplitData(MatType& data, return left; } -// Greedily expand the tree -template +// Greedily expand the tree. +template double DTree::Grow(MatType& data, arma::Col& oldFromNew, const bool useVolReg, @@ -733,7 +735,7 @@ double DTree::Grow(MatType& data, } -template +template double DTree::PruneAndUpdate(const double oldAlpha, const size_t points, const bool useVolReg) @@ -848,7 +850,7 @@ double DTree::PruneAndUpdate(const double oldAlpha, // // Future improvement: Open up the range with epsilons on both sides where // epsilon depends on the density near the boundary. -template +template bool DTree::WithinRange(const VecType& query) const { for (size_t i = 0; i < query.n_elem; ++i) @@ -859,7 +861,7 @@ bool DTree::WithinRange(const VecType& query) const } -template +template double DTree::ComputeValue(const VecType& query) const { Log::Assert(query.n_elem == maxVals.n_elem); @@ -888,7 +890,7 @@ double DTree::ComputeValue(const VecType& query) const } // Index the buckets for possible usage later. -template +template TagType DTree::TagTree(const TagType& tag, bool every) { if (subtreeLeaves == 1) @@ -910,7 +912,7 @@ TagType DTree::TagTree(const TagType& tag, bool every) return right->TagTree(left->TagTree(nextTag, every), every); } -template +template TagType DTree::FindBucket(const VecType& query) const { Log::Assert(query.n_elem == maxVals.n_elem); @@ -936,9 +938,9 @@ TagType DTree::FindBucket(const VecType& query) const } } -template -void -DTree::ComputeVariableImportance(arma::vec& importances) const +template +void DTree::ComputeVariableImportance(arma::vec& importances) + const { // Clear and set to right size. importances.zeros(maxVals.n_elem); @@ -965,7 +967,7 @@ DTree::ComputeVariableImportance(arma::vec& importances) const } } -template +template void DTree::FillMinMax(const StatType& mins, const StatType& maxs) { @@ -988,8 +990,8 @@ void DTree::FillMinMax(const StatType& mins, } } -template -template +template +template void DTree::Serialize(Archive& ar, const unsigned int /* version */) { From 36ec078f7de1ff5a3d20a61966147b220951042b Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Mon, 9 Oct 2017 11:11:58 -0400 Subject: [PATCH 34/39] Really minor change to formatting of option documentation. --- src/mlpack/methods/det/det_main.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/mlpack/methods/det/det_main.cpp b/src/mlpack/methods/det/det_main.cpp index d9e05eb991..41568dfbaa 100644 --- a/src/mlpack/methods/det/det_main.cpp +++ b/src/mlpack/methods/det/det_main.cpp @@ -71,10 +71,10 @@ PARAM_MATRIX_OUT("vi", "The output variable importance values for each " "feature.", "i"); // Tagging and path printing options -PARAM_STRING_IN("path_format", "The format of path printing - lr|id-lr|lr-id", - "p", "lr"); +PARAM_STRING_IN("path_format", "The format of path printing: 'lr', 'id-lr', or " + "'lr-id'.", "p", "lr"); -PARAM_STRING_OUT("tag_counters_file", "The file to output number of points " +PARAM_STRING_OUT("tag_counters_file", "The file to output the number of points " "that went to each leaf.", "c"); PARAM_STRING_OUT("tag_file", "The file to output the tags (and possibly paths)" From 07056406620d3932300a9cfda10d6293d9e7f155 Mon Sep 17 00:00:00 2001 From: C0deAi Date: Thu, 5 Oct 2017 15:16:39 -0400 Subject: [PATCH 35/39] Remove dead code, CWE 561 --- src/mlpack/methods/rann/ra_util.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/mlpack/methods/rann/ra_util.cpp b/src/mlpack/methods/rann/ra_util.cpp index 867d5908fe..77be2f4733 100644 --- a/src/mlpack/methods/rann/ra_util.cpp +++ b/src/mlpack/methods/rann/ra_util.cpp @@ -46,7 +46,6 @@ size_t mlpack::neighbor::RAUtil::MinimumSamplesReqd(const size_t n, { if (prob - alpha < 0.001 || ub < lb + 2) { - done = true; break; } else From 7a2dae01253b2c199a2169b938025a50d4e17d2e Mon Sep 17 00:00:00 2001 From: C0deAi Date: Thu, 5 Oct 2017 15:19:35 -0400 Subject: [PATCH 36/39] Remove dead code, CWE 561 --- src/mlpack/methods/rann/ra_util.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/mlpack/methods/rann/ra_util.cpp b/src/mlpack/methods/rann/ra_util.cpp index 77be2f4733..e3a3fcc311 100644 --- a/src/mlpack/methods/rann/ra_util.cpp +++ b/src/mlpack/methods/rann/ra_util.cpp @@ -65,7 +65,6 @@ size_t mlpack::neighbor::RAUtil::MinimumSamplesReqd(const size_t n, } else { - done = true; break; } } From bd4eb30d52354a370d8e348177178350f0841150 Mon Sep 17 00:00:00 2001 From: C0deAi Date: Thu, 5 Oct 2017 15:21:10 -0400 Subject: [PATCH 37/39] Remove dead code, CWE 561 --- src/mlpack/core/util/timers.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/mlpack/core/util/timers.cpp b/src/mlpack/core/util/timers.cpp index 974b3eba40..d05312a791 100644 --- a/src/mlpack/core/util/timers.cpp +++ b/src/mlpack/core/util/timers.cpp @@ -111,7 +111,6 @@ void Timers::PrintTimer(const std::string& timerName) Log::Info << ", "; Log::Info << s.count() << "." << std::setw(1) << (totalDurationMicroSec.count() / 100000) << " secs"; - output = true; } Log::Info << ")"; From 1fd5f323445ddc252d75dfdefeb39a8735a42c15 Mon Sep 17 00:00:00 2001 From: C0deAi Date: Thu, 5 Oct 2017 15:22:56 -0400 Subject: [PATCH 38/39] Tighten condition to prevent possible null pointer dereference --- src/mlpack/core/tree/cosine_tree/cosine_tree.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mlpack/core/tree/cosine_tree/cosine_tree.cpp b/src/mlpack/core/tree/cosine_tree/cosine_tree.cpp index 929ca82b4e..51105ae4d7 100644 --- a/src/mlpack/core/tree/cosine_tree/cosine_tree.cpp +++ b/src/mlpack/core/tree/cosine_tree/cosine_tree.cpp @@ -94,7 +94,7 @@ CosineTree::CosineTree(const arma::mat& dataset, // Initialize Monte Carlo error estimate for comparison. double monteCarloError = root.FrobNormSquared(); - while (monteCarloError > epsilon * root.FrobNormSquared()) + while (treeQueue.top() && (monteCarloError > epsilon * root.FrobNormSquared())) { // Pop node from queue with highest projection error. CosineTree* currentNode; From 8b5765bb3532f70778629425751cfa0927bb2838 Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Mon, 9 Oct 2017 11:00:35 -0400 Subject: [PATCH 39/39] Fix style and static analysis issues. --- src/mlpack/core/tree/cosine_tree/cosine_tree.cpp | 3 ++- src/mlpack/methods/rann/ra_util.cpp | 7 +------ 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/src/mlpack/core/tree/cosine_tree/cosine_tree.cpp b/src/mlpack/core/tree/cosine_tree/cosine_tree.cpp index 51105ae4d7..61159a6001 100644 --- a/src/mlpack/core/tree/cosine_tree/cosine_tree.cpp +++ b/src/mlpack/core/tree/cosine_tree/cosine_tree.cpp @@ -94,7 +94,8 @@ CosineTree::CosineTree(const arma::mat& dataset, // Initialize Monte Carlo error estimate for comparison. double monteCarloError = root.FrobNormSquared(); - while (treeQueue.top() && (monteCarloError > epsilon * root.FrobNormSquared())) + while (treeQueue.top() && + (monteCarloError > epsilon * root.FrobNormSquared())) { // Pop node from queue with highest projection error. CosineTree* currentNode; diff --git a/src/mlpack/methods/rann/ra_util.cpp b/src/mlpack/methods/rann/ra_util.cpp index e3a3fcc311..3788917261 100644 --- a/src/mlpack/methods/rann/ra_util.cpp +++ b/src/mlpack/methods/rann/ra_util.cpp @@ -30,11 +30,6 @@ size_t mlpack::neighbor::RAUtil::MinimumSamplesReqd(const size_t n, double prob; Log::Assert(alpha <= 1.0); - // going through all values of sample sizes - // to find the minimum samples required to satisfy the - // desired bound - bool done = false; - // This performs a binary search on the integer values between 'lb = k' // and 'ub = n' to find the minimum number of samples 'm' required to obtain // the desired success probability 'alpha'. @@ -69,7 +64,7 @@ size_t mlpack::neighbor::RAUtil::MinimumSamplesReqd(const size_t n, } } m = (ub + lb) / 2; - } while (!done); + } while (true); return (std::min(m + 1, n)); }