diff --git a/HISTORY.md b/HISTORY.md index cf4b7e54f4..ed6859fdfd 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,5 +1,7 @@ ### mlpack ?.?.? ###### ????-??-?? + * Add `print_training_accuracy` option to LogisticRegression bindings (#3552). + * Fix `preprocess_split()` call in documentation for `LinearRegression` and `AdaBoost` Python classes (#3563). diff --git a/src/mlpack/core/tree/binary_space_tree/binary_space_tree.hpp b/src/mlpack/core/tree/binary_space_tree/binary_space_tree.hpp index a042ba849d..8289e0179f 100644 --- a/src/mlpack/core/tree/binary_space_tree/binary_space_tree.hpp +++ b/src/mlpack/core/tree/binary_space_tree/binary_space_tree.hpp @@ -504,7 +504,7 @@ class BinarySpaceTree size_t& Count() { return count; } //! Store the center of the bounding region in the given vector. - void Center(arma::vec& center) const { bound.Center(center); } + void Center(arma::Col& center) const { bound.Center(center); } private: /** diff --git a/src/mlpack/core/tree/binary_space_tree/binary_space_tree_impl.hpp b/src/mlpack/core/tree/binary_space_tree/binary_space_tree_impl.hpp index 0a1e1d31db..da5bd3d3ba 100644 --- a/src/mlpack/core/tree/binary_space_tree/binary_space_tree_impl.hpp +++ b/src/mlpack/core/tree/binary_space_tree/binary_space_tree_impl.hpp @@ -909,7 +909,7 @@ void BinarySpaceTree:: splitter, maxLeafSize); // Calculate parent distances for those two nodes. - arma::vec center, leftCenter, rightCenter; + arma::Col center, leftCenter, rightCenter; Center(center); left->Center(leftCenter); right->Center(rightCenter); @@ -977,7 +977,7 @@ SplitNode(std::vector& oldFromNew, oldFromNew, splitter, maxLeafSize); // Calculate parent distances for those two nodes. - arma::vec center, leftCenter, rightCenter; + arma::Col center, leftCenter, rightCenter; Center(center); left->Center(leftCenter); right->Center(rightCenter); diff --git a/src/mlpack/methods/logistic_regression/logistic_regression_main.cpp b/src/mlpack/methods/logistic_regression/logistic_regression_main.cpp index d2a842fa6d..8566980e80 100644 --- a/src/mlpack/methods/logistic_regression/logistic_regression_main.cpp +++ b/src/mlpack/methods/logistic_regression/logistic_regression_main.cpp @@ -101,7 +101,8 @@ BINDING_EXAMPLE( PRINT_MODEL("lr_model") + "', the following command may be used:" "\n\n" + PRINT_CALL("logistic_regression", "training", "data", "labels", "labels", - "lambda", 0.1, "output_model", "lr_model") + + "lambda", 0.1, "output_model", "lr_model", "print_training_accuracy", + true) + "\n\n" "Then, to use that model to predict classes for the dataset '" + PRINT_DATASET("test") + "', storing the output predictions in '" + @@ -153,6 +154,9 @@ PARAM_MATRIX_OUT("probabilities", "If test data is specified, this " PARAM_DOUBLE_IN("decision_boundary", "Decision boundary for prediction; if the " "logistic function for a point is less than the boundary, the class is " "taken to be 0; otherwise, the class is 1.", "d", 0.5); +PARAM_FLAG("print_training_accuracy", "If set, then the accuracy of the model " + "on the training set will be printed (verbose must also be specified).", + "a"); void BINDING_FUNCTION(util::Params& params, util::Timers& timers) { @@ -182,6 +186,12 @@ void BINDING_FUNCTION(util::Params& params, util::Timers& timers) ReportIgnoredParam(params, {{ "test", false }}, "predictions"); ReportIgnoredParam(params, {{ "test", false }}, "probabilities"); + ReportIgnoredParam(params, {{ "training", false }}, "print_training_accuracy"); + + RequireAtLeastOnePassed(params, + { "test", "output_model", "print_training_accuracy" }, false, + "the trained logistic regression model will not be used or saved"); + // Max Iterations needs to be positive. RequireParamValue(params, "max_iterations", [](int x) { return x >= 0; }, true, "max_iterations must be positive or zero"); @@ -325,8 +335,23 @@ void BINDING_FUNCTION(util::Params& params, util::Timers& timers) model->Train(regressors, responses, lbfgsOpt); timers.Stop("logistic_regression_optimization"); } - } + // Did we want training accuracy? + if (params.Has("print_training_accuracy")) + { + timers.Start("lr_prediction"); + arma::Row predictions; + model->Classify(regressors, predictions); + + const size_t correct = arma::accu(predictions == responses); + + Log::Info << correct << " of " << responses.n_elem << " correct on training" + << " set (" << (double(correct) / double(responses.n_elem) * 100) << ")." + << endl; + timers.Stop("lr_prediction"); + } + } + if (params.Has("test")) { const arma::mat& testSet = params.Get("test"); diff --git a/src/mlpack/methods/neighbor_search/neighbor_search.hpp b/src/mlpack/methods/neighbor_search/neighbor_search.hpp index 2e2b9934d8..558a4a300f 100644 --- a/src/mlpack/methods/neighbor_search/neighbor_search.hpp +++ b/src/mlpack/methods/neighbor_search/neighbor_search.hpp @@ -82,6 +82,8 @@ class NeighborSearch public: //! Convenience typedef. typedef TreeType, MatType> Tree; + //! The type of element held in MatType. + typedef typename MatType::elem_type ElemType; /** * Initialize the NeighborSearch object, passing a reference dataset (this is @@ -223,7 +225,7 @@ class NeighborSearch void Search(const MatType& querySet, const size_t k, arma::Mat& neighbors, - arma::mat& distances); + arma::Mat& distances); /** * Given a pre-built query tree, search for the nearest neighbors of each @@ -248,7 +250,7 @@ class NeighborSearch void Search(Tree& queryTree, const size_t k, arma::Mat& neighbors, - arma::mat& distances, + arma::Mat& distances, bool sameSet = false); /** @@ -267,7 +269,7 @@ class NeighborSearch */ void Search(const size_t k, arma::Mat& neighbors, - arma::mat& distances); + arma::Mat& distances); /** * Calculate the average relative error (effective error) between the @@ -284,8 +286,8 @@ class NeighborSearch * query point. * @return Average relative error. */ - static double EffectiveError(arma::mat& foundDistances, - arma::mat& realDistances); + static double EffectiveError(arma::Mat& foundDistances, + arma::Mat& realDistances); /** * Calculate the recall (% of neighbors found) given the list of found diff --git a/src/mlpack/methods/neighbor_search/neighbor_search_impl.hpp b/src/mlpack/methods/neighbor_search/neighbor_search_impl.hpp index 13e42232f2..c528d23f04 100644 --- a/src/mlpack/methods/neighbor_search/neighbor_search_impl.hpp +++ b/src/mlpack/methods/neighbor_search/neighbor_search_impl.hpp @@ -104,7 +104,7 @@ SingleTreeTraversalType>::NeighborSearch(const NeighborSearchMode mode, // Build the tree on the empty dataset, if necessary. if (mode != NAIVE_MODE) { - referenceTree = BuildTree(std::move(arma::mat()), + referenceTree = BuildTree(std::move(MatType()), oldFromNewReferences); referenceSet = &referenceTree->Dataset(); } @@ -255,7 +255,7 @@ NeighborSearch(std::move(arma::mat()), + other.referenceTree = BuildTree(std::move(MatType()), other.oldFromNewReferences); other.referenceSet = &other.referenceTree->Dataset(); other.searchMode = DUAL_TREE_MODE, @@ -365,7 +365,7 @@ DualTreeTraversalType, SingleTreeTraversalType>::Search( const MatType& querySet, const size_t k, arma::Mat& neighbors, - arma::mat& distances) + arma::Mat& distances) { if (k > referenceSet->n_cols) { @@ -386,14 +386,14 @@ DualTreeTraversalType, SingleTreeTraversalType>::Search( // To avoid an extra copy, we will store the neighbors and distances in a // separate matrix. arma::Mat* neighborPtr = &neighbors; - arma::mat* distancePtr = &distances; + arma::Mat* distancePtr = &distances; // Mapping is only necessary if the tree rearranges points. if (TreeTraits::RearrangesDataset) { if (searchMode == DUAL_TREE_MODE) { - distancePtr = new arma::mat; // Query indices need to be mapped. + distancePtr = new arma::Mat; // Query indices need to be mapped. neighborPtr = new arma::Mat; } else if (!oldFromNewReferences.empty()) @@ -570,7 +570,7 @@ DualTreeTraversalType, SingleTreeTraversalType>::Search( Tree& queryTree, const size_t k, arma::Mat& neighbors, - arma::mat& distances, + arma::Mat& distances, bool sameSet) { if (k > referenceSet->n_cols) @@ -648,7 +648,7 @@ void NeighborSearch::Search( const size_t k, arma::Mat& neighbors, - arma::mat& distances) + arma::Mat& distances) { if (k > referenceSet->n_cols) { @@ -670,12 +670,12 @@ DualTreeTraversalType, SingleTreeTraversalType>::Search( scores = 0; arma::Mat* neighborPtr = &neighbors; - arma::mat* distancePtr = &distances; + arma::Mat* distancePtr = &distances; if (!oldFromNewReferences.empty() && TreeTraits::RearrangesDataset) { // We will always need to rearrange in this case. - distancePtr = new arma::mat; + distancePtr = new MatType; neighborPtr = new arma::Mat; } @@ -825,8 +825,8 @@ template class SingleTreeTraversalType> double NeighborSearch::EffectiveError( - arma::mat& foundDistances, - arma::mat& realDistances) + arma::Mat& foundDistances, + arma::Mat& realDistances) { if (foundDistances.n_rows != realDistances.n_rows || foundDistances.n_cols != realDistances.n_cols) diff --git a/src/mlpack/methods/neighbor_search/neighbor_search_rules.hpp b/src/mlpack/methods/neighbor_search/neighbor_search_rules.hpp index 9b7ce53661..02577a4049 100644 --- a/src/mlpack/methods/neighbor_search/neighbor_search_rules.hpp +++ b/src/mlpack/methods/neighbor_search/neighbor_search_rules.hpp @@ -34,6 +34,9 @@ template class NeighborSearchRules { public: + //! The type of element held in MatType. + typedef typename TreeType::Mat::elem_type ElemType; + /** * Construct the NeighborSearchRules object. This is usually done from within * the NeighborSearch class at search time. @@ -60,7 +63,7 @@ class NeighborSearchRules * @param distances Matrix storing distances of neighbors for each query * point. */ - void GetResults(arma::Mat& neighbors, arma::mat& distances); + void GetResults(arma::Mat& neighbors, arma::Mat& distances); /** * Get the distance from the query point to the reference point. diff --git a/src/mlpack/methods/neighbor_search/neighbor_search_rules_impl.hpp b/src/mlpack/methods/neighbor_search/neighbor_search_rules_impl.hpp index 0c1f50d58e..f754427dc2 100644 --- a/src/mlpack/methods/neighbor_search/neighbor_search_rules_impl.hpp +++ b/src/mlpack/methods/neighbor_search/neighbor_search_rules_impl.hpp @@ -61,7 +61,7 @@ NeighborSearchRules::NeighborSearchRules( template void NeighborSearchRules::GetResults( arma::Mat& neighbors, - arma::mat& distances) + arma::Mat& distances) { neighbors.set_size(k, querySet.n_cols); distances.set_size(k, querySet.n_cols); diff --git a/src/mlpack/tests/knn_test.cpp b/src/mlpack/tests/knn_test.cpp index dd302a85df..3f77aba9ca 100644 --- a/src/mlpack/tests/knn_test.cpp +++ b/src/mlpack/tests/knn_test.cpp @@ -16,6 +16,18 @@ using namespace mlpack; +/** + * A couple of handful declarations for float32 testing. + * These will be removed when we refactor the Bounds to accept MatType. + * For now, we will keep the following declarations. + */ +template +using FloatHRectBound = HRectBound; + +template +using FloatKDTree = BinarySpaceTree; + /** * Test that Unmap() works in the dual-tree case (see unmap.hpp). */ @@ -746,6 +758,48 @@ TEST_CASE("KNNSingleTreeVsNaive", "[KNNTest]") } } +/** + * Test the single-tree nearest-neighbors method with the naive method. + * + * The main difference with the above test is that this one loads the reference + * dataset as a float32, and the distances as a float32 as well. + * + * Errors are produced if the results are not identical. + */ +TEST_CASE("KNNSingleTreeVsNaiveF32", "[KNNTest]") +{ + arma::fmat dataset; + + // Hard-coded filename: bad? + // Code duplication: also bad! + if (!data::Load("test_data_3_1000.csv", dataset)) + FAIL("Cannot load test dataset test_data_3_1000.csv!"); + + NeighborSearch knn(dataset, SINGLE_TREE_MODE); + + // Set up computation for naive mode. + NeighborSearch naive(dataset, NAIVE_MODE); + + arma::Mat neighborsTree; + arma::fmat distancesTree; + knn.Search(15, neighborsTree, distancesTree); + + arma::Mat neighborsNaive; + arma::fmat distancesNaive; + naive.Search(15, neighborsNaive, distancesNaive); + + for (size_t i = 0; i < neighborsTree.n_elem; ++i) + { + REQUIRE(neighborsTree[i] ==neighborsNaive[i]); + REQUIRE(distancesTree[i] == Approx(distancesNaive[i]).epsilon(1e-7)); + } +} /** * Test the cover tree single-tree nearest-neighbors method against the naive * method. This uses only a random reference dataset. diff --git a/src/mlpack/tests/main_tests/logistic_regression_test.cpp b/src/mlpack/tests/main_tests/logistic_regression_test.cpp index fee66b221e..d832ca56ad 100644 --- a/src/mlpack/tests/main_tests/logistic_regression_test.cpp +++ b/src/mlpack/tests/main_tests/logistic_regression_test.cpp @@ -627,3 +627,25 @@ TEST_CASE_METHOD(LogisticRegressionTestFixture, "LRDecisionBoundaryTest", // Check that the output changed when the decision boundary moved. REQUIRE(arma::accu(output1 != output2) > 0); } + +/** + * Check that running the binding with print_training_accuracy set to true + * does not crash. + **/ +TEST_CASE_METHOD(LogisticRegressionTestFixture, "LRPrintTrainingAccuracyTest", + "[LogisticRegressionMainTest][BindingTests]") +{ + constexpr int N = 100; + constexpr int D = 5; + + arma::mat trainX = arma::randu(D, N); + arma::Row trainY = arma::randi>(N, + arma::distr_param(0, 1)); + + SetInputParam("training", trainX); + SetInputParam("labels", trainY); + SetInputParam("print_training_accuracy", true); + + // Run the binding with print_training_accuracy set to true. + REQUIRE_NOTHROW(RUN_BINDING()); +}