From 04a970228b869562be78d84a6f69d747fd869a9e Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Mon, 13 Nov 2023 16:17:21 -0500 Subject: [PATCH 01/91] Include workaround for allowing templated versions for serialization. --- .../core/cereal/template_class_version.hpp | 78 +++++++++++++++++++ src/mlpack/prereqs.hpp | 1 + 2 files changed, 79 insertions(+) create mode 100644 src/mlpack/core/cereal/template_class_version.hpp diff --git a/src/mlpack/core/cereal/template_class_version.hpp b/src/mlpack/core/cereal/template_class_version.hpp new file mode 100644 index 0000000000..6789de5e39 --- /dev/null +++ b/src/mlpack/core/cereal/template_class_version.hpp @@ -0,0 +1,78 @@ +/** + * @file core/cereal/template_class_version.hpp + * @author Ryan Curtin + * + * Implementation of CEREAL_TEMPLATE_CLASS_VERSION() macro, useful for + * templatized types where CEREAL_CLASS_VERSION() will not work. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_CORE_CEREAL_TEMPLATE_CLASS_VERSION_HPP +#define MLPACK_CORE_CEREAL_TEMPLATE_CLASS_VERSION_HPP + +#include + +// This useful implementation is adapted from @lubensky on Github: +// https://github.com/uscilab/cereal/issues/319#issuecomment-1512927210 + +#define CEREAL_UNPACK(...) __VA_ARGS__ + +#ifdef MLPACK_HAVE_CXX17 + +// The C++17 version sets `version` as `inline`. +#define CEREAL_TEMPLATE_CLASS_VERSION(ARGS, TYPE, VERSION_NUMBER) \ +namespace cereal { \ +namespace detail { \ +template \ +struct Version \ +{ \ + static std::uint32_t registerVersion() \ + { \ + ::cereal::detail::StaticObject::getInstance().mapping.emplace( \ + std::type_index(typeid(CEREAL_UNPACK TYPE)).hash_code(), \ + CEREAL_UNPACK VERSION_NUMBER); \ + return CEREAL_UNPACK VERSION_NUMBER; \ + } \ + \ + static inline const std::uint32_t version = registerVersion(); \ + \ + CEREAL_UNUSED_FUNCTION \ +}; /* end Version */ \ + \ +} \ +} + +#else + +// Here we cannot use inline variables. +#define CEREAL_TEMPLATE_CLASS_VERSION(ARGS, TYPE, VERSION_NUMBER) \ +namespace cereal { \ +namespace detail { \ +template \ +struct Version \ +{ \ + static const std::uint32_t version; \ + static std::uint32_t registerVersion() \ + { \ + ::cereal::detail::StaticObject::getInstance().mapping.emplace( \ + std::type_index(typeid(CEREAL_UNPACK TYPE)).hash_code(), \ + CEREAL_UNPACK VERSION_NUMBER); \ + return CEREAL_UNPACK VERSION_NUMBER; \ + } \ + \ + CEREAL_UNUSED_FUNCTION \ +}; /* end Version */ \ + \ +template \ +const std::uint32_t Version::version = \ + Version::registerVersion(); \ + \ +} \ +} + +#endif // MLPACK_HAVE_CXX17 + +#endif // TEMPLATE_CLASS_VERSION_HPP diff --git a/src/mlpack/prereqs.hpp b/src/mlpack/prereqs.hpp index 202788013e..990373b48c 100644 --- a/src/mlpack/prereqs.hpp +++ b/src/mlpack/prereqs.hpp @@ -31,6 +31,7 @@ #include #include #include +#include #include // All code should have access to logging. From 4bc1297f963055bb39cc0937467e830f678bfc80 Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Wed, 15 Nov 2023 08:53:36 -0500 Subject: [PATCH 02/91] Correct handling for older cereal versions. --- src/mlpack/core/cereal/template_class_version.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mlpack/core/cereal/template_class_version.hpp b/src/mlpack/core/cereal/template_class_version.hpp index 6789de5e39..ea9e508964 100644 --- a/src/mlpack/core/cereal/template_class_version.hpp +++ b/src/mlpack/core/cereal/template_class_version.hpp @@ -39,7 +39,7 @@ struct Version \ \ static inline const std::uint32_t version = registerVersion(); \ \ - CEREAL_UNUSED_FUNCTION \ + static void unused() { (void) version; } \ }; /* end Version */ \ \ } \ From 39e35a1f79e605ea5fc093cde721c8ba21476f21 Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Wed, 15 Nov 2023 08:55:36 -0500 Subject: [PATCH 03/91] Fix other use of cereal macro not available in older versions. --- src/mlpack/core/cereal/template_class_version.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mlpack/core/cereal/template_class_version.hpp b/src/mlpack/core/cereal/template_class_version.hpp index ea9e508964..bb68af0705 100644 --- a/src/mlpack/core/cereal/template_class_version.hpp +++ b/src/mlpack/core/cereal/template_class_version.hpp @@ -63,7 +63,7 @@ struct Version \ return CEREAL_UNPACK VERSION_NUMBER; \ } \ \ - CEREAL_UNUSED_FUNCTION \ + static void unused() { (void) version; } \ }; /* end Version */ \ \ template \ From 2c8289149b76fd00b6186c5581fe651ac414300d Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Mon, 27 Nov 2023 16:49:24 -0500 Subject: [PATCH 04/91] Fix some compilation warnings. --- src/mlpack/methods/ann/layer/layer.hpp | 4 ++-- src/mlpack/methods/quic_svd/quic_svd_impl.hpp | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/mlpack/methods/ann/layer/layer.hpp b/src/mlpack/methods/ann/layer/layer.hpp index e022e3f387..ad37bb757d 100644 --- a/src/mlpack/methods/ann/layer/layer.hpp +++ b/src/mlpack/methods/ann/layer/layer.hpp @@ -81,7 +81,7 @@ class Layer { /* Nothing to do here */ } //! Copy assignment operator. This is not responsible for copying weights! - virtual Layer& operator=(const Layer& layer) + Layer& operator=(const Layer& layer) { if (&layer != this) { @@ -95,7 +95,7 @@ class Layer } //! Move assignment operator. This is not responsible for moving weights! - virtual Layer& operator=(Layer&& layer) + Layer& operator=(Layer&& layer) { if (&layer != this) { diff --git a/src/mlpack/methods/quic_svd/quic_svd_impl.hpp b/src/mlpack/methods/quic_svd/quic_svd_impl.hpp index eb010c6e16..36d00991c8 100644 --- a/src/mlpack/methods/quic_svd/quic_svd_impl.hpp +++ b/src/mlpack/methods/quic_svd/quic_svd_impl.hpp @@ -29,8 +29,8 @@ inline QUIC_SVD::QUIC_SVD( } inline QUIC_SVD::QUIC_SVD( - const double epsilon, - const double delta) + const double /* epsilon */, + const double /* delta */) { /* Nothing to do here */ } From 727fb58ada25c6735621a1c5ec179fc953e95e78 Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Mon, 27 Nov 2023 16:51:39 -0500 Subject: [PATCH 05/91] Add documentation for NaiveBayesClassifier. --- doc/user/methods/naive_bayes_classifier.md | 310 +++++++++++++++++++++ 1 file changed, 310 insertions(+) create mode 100644 doc/user/methods/naive_bayes_classifier.md diff --git a/doc/user/methods/naive_bayes_classifier.md b/doc/user/methods/naive_bayes_classifier.md new file mode 100644 index 0000000000..ba3b8db766 --- /dev/null +++ b/doc/user/methods/naive_bayes_classifier.md @@ -0,0 +1,310 @@ +## `NaiveBayesClassifier` + +The `NaiveBayesClassifier` implements a trivial Naive Bayes classifier for +numerical data. The class offers standard classification functionality. Naive +Bayes is useful for multi-class classification (i.e. classes are `0`, `1`, `2`, +etc.), and due to its simplicity scales well to large-data scenarios. + +#### Simple usage example: + +```c++ +// Train a Naive Bayes classifier on random data and predict labels: + +// All data and labels are uniform random; 5 dimensional data, 4 classes. +// Replace with a data::Load() call or similar for a real application. +arma::mat dataset(5, 1000, arma::fill::randu); // 1000 points. +arma::Row labels = + arma::randi>(1000, arma::distr_param(0, 3)); +arma::mat testDataset(5, 500, arma::fill::randu); // 500 test points. + +mlpack::NaiveBayesClassifier nbc; // Step 1: create model. +nbc.Train(dataset, labels); // Step 2: train model. +arma::Row predictions; +nbc.Classify(testDataset, predictions); // Step 3: classify points. + +// Print some information about the test predictions. +std::cout << arma::accu(predictions == 2) << " test points classified as class " + << "2." << std::endl; +``` +

More examples...

+ +#### Quick links: + + * [Constructors](#constructors): create `NaiveBayesClassifier` objects. + * [`Train()`](#training): train model. + * [`Classify()`](#classification): classify with a trained model. + * [Other functionality](#other-functionality) for loading, saving, and + inspecting. + * [Examples](#simple-examples) of simple usage and links to detailed example + projects. + * [Template parameters](#advanced-functionality-different-element-types) for + using different element types for a model. + +#### See also: + + * [mlpack classifiers](#mlpack_classifiers) + * [`GaussianDistribution`](#gaussian_distribution) + * [Naive Bayes classifier on Wikipedia](https://en.wikipedia.org/wiki/Naive_Bayes_classifier) + +### Constructors + + * `nbc = NaiveBayesClassifier()` + - Initialize the model without training. + - You will need to call [`Train()`](#training) later to train the model + before calling [`Classify()`](#classification). + +--- + + * `nbc = NaiveBayesClassifier(dimensionality, numClasses, epsilon=1e-10)` + - Initialize model to the given dimensionality and number of classes without + training. + - This is meant to be used with the incremental version of `Train()` that + takes only a single point. + +--- + + * `nbc = NaiveBayesClassifier(data, labels, numClasses, incremental=true, epsilon=1e-10)` + - Train model, optionally specifying whether to do incremental training. + +--- + +#### Constructor Parameters: + + + +| **name** | **type** | **description** | **default** | +|----------|----------|-----------------|-------------| +| `data` | [`arma::mat`](../matrices.md) | [Column-major](../matrices.md) training matrix. | _(N/A)_ | +| `labels` | [`arma::Row`]('../matrices.md') | Training labels, between `0` and `numClasses - 1` (inclusive). Should have length `data.n_cols`. | _(N/A)_ | +| `numClasses` | `size_t` | Number of classes in the dataset. | _(N/A)_ | +| `incremental` | `bool` | If `true`, then the model will not be reset before training, and will use a robust incremental algorithm for variance computation. | `true` | +| `epsilon` | `double` | Initial small value for sample variances, to prevent +underflow (via `log(0)`). | 1e-10 | + +### Training + +If training is not done as part of the constructor call, it can be done with the +`Train()` function: + + * `nbc.Train(data, labels, numClasses, incremental=true, epsilon=1e-10)` + - Train model on the given data, optionally specifying whether to do + incremental training. + - Arguments described in [Constructor Parameters](#constructor_parameters) + table above. + +--- + + * `nbc.Train(point, label)` + - Incrementally train on a single data point with the given label. + - Ensure that the model has the right size and number of classes by using the + appropriate constructor form to set `dimensionality`, or by calling + `Reset()` (see [other functionality](#other_functionality)). + + + +| **name** | **type** | **description** | **default** | +|----------|----------|-----------------|-------------| +| `point` | [`arma::vec`](../matrices.md) | [Column-major](../matrices.md) training point (i.e. one column). | _(N/A)_ | +| `label` | `size_t` | Training label, in range `0` to `numClasses`. | _(N/A)_ | + +***Note***: when performing incremental training, if `data` has a different +dimensionality than the model, or if `numClasses` is different, the model will +be reset. For single-point `Train()`, if `point` has different dimensionality, +an exception will be thrown. + +### Classification + +Once a `NaiveBayesClassifier` model is trained, the `Classify()` member function +can be used to make class predictions for new data. + + * `size_t predictedClass = sr.Classify(point)` + - ***(Single-point)*** + - Classify a single point, returning the predicted class (`0` through + `numClasses - 1`, inclusive). + +--- + + * `sr.Classify(point, prediction, probabilitiesVec)` + - ***(Single-point)*** + - Classify a single point and compute class probabilities. + - The predicted class is stored in `prediction`. + - The probability of class `i` can be accessed with `probabilitiesVec[i]`. + +--- + + * `nbc.Classify(data, predictions)` + - ***(Multi-point)*** + - Classify a set of points. + - The prediction for data point `i` can be accessed with `predictions[i]`. + +--- + + * `nbc.Classify(data, predictions, probabilities)` + - ***(Multi-point)*** + - Classify a set of points and compute class probabilities. + - The prediction for data point `i` can be accessed with `predictions[i]`. + - The probability of class `j` for data point `i` can be accessed with + `probabilities(j, i)`. + +--- + +#### Classification Parameters: + +| **usage** | **name** | **type** | **description** | +|-----------|----------|----------|-----------------| +| _single-point_ | `point` | [`arma::vec`](../matrices.md) | Single point for classification. | +| _single-point_ | `prediction` | `size_t&` | `size_t` to store class prediction into. | +| _single-point_ | `probabilitiesVec` | [`arma::vec&`](../matrices.md) | `arma::vec&` to store class probabilities into; will have length 2. | +|||| +| _multi-point_ | `data` | [`arma::mat`](../matrices.md) | Set of [column-major](../matrices.md) points for classification. | +| _multi-point_ | `predictions` | [`arma::Row&`](../matrices.md) | Vector of `size_t`s to store class prediction into; will be set to length `data.n_cols`. | +| _multi-point_ | `probabilities` | [`arma::mat&`](../matrices.md) | Matrix to store class probabilities into (number of rows will be equal to 2; number of columns will be equal to `data.n_cols`). | + +### Other Functionality + + + + * A `NaiveBayesClassifier` model can be serialized with + [`data::Save()`](../formats.md) and [`data::Load()`](../formats.md). + + * `nbc.Probabilities()` will return a column vector of length `numClasses` + representing the prior probability of each class. + + * `nbc.Means()` will return a matrix with rows equal to the dimensionality of + the model and `numClasses` columns. Column `i` represents the sample mean of + class `i`. + + * `nbc.Variances()` will return a matrix with rows equal to the dimensionality + of the model and `numClasses` columns. The element at row `i` and column `j` + represents the sample variance in dimension `i` of class `j`. + + * `nbc.Reset()` will reset the model to zeros; this is useful before + incremental training. The form + `nbc.Reset(dimensionality, numClasses, epsilon=1e-10)` can + also be used to set the dimensionality and number of classes in the reset + model. + + * `nbc.TrainingPoints()` will return the number of points that the model has + been trained on. When `nbc.Reset()` is called, this is reset to 0. + +### Simple Examples + +See also the [simple usage example](#simple-usage-example) for a trivial usage +of the `NaiveBayesClassifier` class. + +--- + +Train a Naive Bayes classifier incrementally, one point at a time, then compute +accuracy on a test set and save the model to disk. + +```c++ +// See https://datasets.mlpack.org/mnist.train.csv. +arma::mat dataset; +mlpack::data::Load("mnist.train.csv", dataset, true); +// See https://datasets.mlpack.org/mnist.train.labels.csv. +arma::Row labels; +mlpack::data::Load("mnist.train.labels.csv", labels, true); + +mlpack::NaiveBayesClassifier nbc(data.n_rows /* dimensionality */, + 10 /* numClasses */); + +// Iterate over all points in the dataset and call Train() on each point. +for (size_t i = 0; i < data.n_cols; ++i) + nbc.Train(data.col(i), labels[i]); + +// Now compute the accuracy of the fully trained model on a test set. + +// See https://datasets.mlpack.org/mnist.test.csv. +arma::mat testDataset; +mlpack::data::Load("mnist.test.csv", testDataset, true); +// See https://datasets.mlpack.org/mnist.test.labels.csv. +arma::Row testLabels; +mlpack::data::Load("mnist.test.labels.csv", testLabels, true); + +arma::Row predictions; +nbc.Classify(testDataset, predictions); + +const double accuracy = 100.0 * + ((double) arma::accu(predictions == testLabels)) / testLabels.n_elem; +std::cout << "Accuracy of model on test data: " << accuracy << "\%." + << std::endl; + +// Save the model to disk with the name "nbc". +data::Save("nbc_model.bin", "nbc", nbc, true); +``` + +--- + +Load a saved Naive Bayes classifier and print some information about it. + +```c++ +NaiveBayesClassifier nbc; + +// Load the model named "nbc" from "nbc_model.bin". +data::Load("nbc_model.bin", "nbc", nbc, true); + +// Print information about the model. +std::cout << "The dimensionality of the model in nbc_model.bin is " + << nbc.Means().n_rows << "." << std::endl; +std::cout << "The number of classes in the model is " + << nbc.Probabilities().n_elem << "." << std::endl; +std::cout << "The prior probabilities of each class are: " + << nbc.Probabilities().t(); + +// Compute the class probabilities of a random point. +arma::vec randomPoint(nbc.Means().n_rows, arma::fill::randu); + +size_t prediction; +arma::vec probabilities; +nbc.Classify(randomPoint, prediction, probabilities); + +std::cout << "Random point class prediction: " << prediction << "." + << std::endl; +std::cout << "Random point class probabilities: " << probabilities.t(); +``` + +### Advanced Functionality: Different Element Types + +The `NaiveBayesClassifier` class has one template parameter that can be used to +control the element type of the model. The full signature of the class is: + +```c++ +NaiveBayesClassifier +``` + +`ModelMatType` specifies the type of matrix used for training data and internal +representation of model parameters. Any matrix type that implements the +Armadillo API can be used. + +Note that the `Train()` and `Classify()` functions themselves are templatized +and can allow any matrix type that has the same element type. So, for instance, +a `NaiveBayesClassifier` can accept an `arma::sp_mat` for training. + +The example below trains a Naive Bayes model on sparse 32-bit floating point +data, but uses dense 32-bit floating point matrices to store the model itself. + +```c++ +// Create random, sparse 100-dimensional data, with 3 classes. +arma::sp_fmat dataset; +dataset.sprandu(100, 5000, 0.3); +arma::Row labels = + arma::randi>(5000, arma::distr_param(0, 2)); + +mlpack::NaiveBayesClassifier sr(dataset, labels, 3); + +// Now classify a test point. +arma::sp_fvec point; +point.sprandu(100, 1, 0.3); + +size_t prediction; +arma::fvec probabilitiesVec; +sr.Classify(point, prediction, probabilitiesVec); + +std::cout << "Prediction for random test point: " << prediction << "." + << std::endl; +std::cout << "Class probabilities for random test point: " + << probabilitiesVec.t(); +``` From 0947848c2dd86cddf83c3e1b7fb9b72d18e524ff Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Mon, 27 Nov 2023 16:51:55 -0500 Subject: [PATCH 06/91] Update NaiveBayesClassifier class to match documentation, and add appropriate tests. --- .../naive_bayes/naive_bayes_classifier.hpp | 54 +++++++- .../naive_bayes_classifier_impl.hpp | 128 ++++++++++++++---- src/mlpack/tests/nbc_test.cpp | 101 ++++++++++++++ 3 files changed, 254 insertions(+), 29 deletions(-) diff --git a/src/mlpack/methods/naive_bayes/naive_bayes_classifier.hpp b/src/mlpack/methods/naive_bayes/naive_bayes_classifier.hpp index dc621303cc..c11887aa04 100644 --- a/src/mlpack/methods/naive_bayes/naive_bayes_classifier.hpp +++ b/src/mlpack/methods/naive_bayes/naive_bayes_classifier.hpp @@ -77,7 +77,8 @@ class NaiveBayesClassifier * @param incrementalVariance If true, an incremental algorithm is used to * calculate the variance; this can prevent loss of precision in some * cases, but will be somewhat slower to calculate. - * @param epsilon Small value to prevent log of zero. + * @param epsilon Small initialization value for variances to prevent log of + * zero. */ template NaiveBayesClassifier(const MatType& data, @@ -109,7 +110,7 @@ class NaiveBayesClassifier * * @param data The dataset to train on. * @param labels The labels for the dataset. - * @param numClasses The numbe of classes in the dataset. + * @param numClasses The number of classes in the dataset. * @param incremental Whether or not to use the incremental algorithm for * training. */ @@ -119,6 +120,32 @@ class NaiveBayesClassifier const size_t numClasses, const bool incremental = true); + /** + * Train the Naive Bayes classifier on the given dataset. If the incremental + * algorithm is used, the current model is used as a starting point (this is + * the default). If the incremental algorithm is not used, then the current + * model is ignored and the new model will be trained only on the given data. + * Note that even if the incremental algorithm is not used, the data must have + * the same dimensionality and number of classes that the model was + * initialized with. If you want to change the dimensionality or number of + * classes, either re-initialize or call Means(), Variances(), and + * Probabilities() individually to set them to the right size. + * + * @param data The dataset to train on. + * @param labels The labels for the dataset. + * @param numClasses The number of classes in the dataset. + * @param incremental Whether or not to use the incremental algorithm for + * training. + * @param epsilon Small reinitialization value for variances to prevent log of + * zero (ignored if incremental is true). + */ + template + void Train(const MatType& data, + const arma::Row& labels, + const size_t numClasses, + const bool incremental, + const double epsilon); + /** * Train the Naive Bayes classifier on the given point. This will use the * incremental algorithm for updating the model parameters. The data must be @@ -198,6 +225,21 @@ class NaiveBayesClassifier arma::Row& predictions, ProbabilitiesMatType& probabilities) const; + /** + * Reset the model to zeros, keeping the model's current dimensionality and + * number of classes. + */ + void Reset(); + + /** + * Reset the model to zeros, with a new dimensionality and number of classes. + * The value epsilon specifies an initial very small value for the variances, + * to prevent log(0) issues. + */ + void Reset(const size_t dimensionality, + const size_t numClasses, + const double epsilon = 1e-10); + //! Get the sample means for each class. const ModelMatType& Means() const { return means; } //! Modify the sample means for each class. @@ -213,9 +255,12 @@ class NaiveBayesClassifier //! Modify the prior probabilities for each class. ModelMatType& Probabilities() { return probabilities; } + //! Get the number of points the model has been trained on so far. + size_t TrainingPoints() const { return trainingPoints; } + //! Serialize the classifier. template - void serialize(Archive& ar, const uint32_t /* version */); + void serialize(Archive& ar, const uint32_t version); private: //! Sample mean for each class. @@ -244,6 +289,9 @@ class NaiveBayesClassifier } // namespace mlpack +CEREAL_TEMPLATE_CLASS_VERSION((typename MatType), + (mlpack::NaiveBayesClassifier), (1)); + // Include implementation. #include "naive_bayes_classifier_impl.hpp" diff --git a/src/mlpack/methods/naive_bayes/naive_bayes_classifier_impl.hpp b/src/mlpack/methods/naive_bayes/naive_bayes_classifier_impl.hpp index 14f220ddc6..5c2eeaa7ed 100644 --- a/src/mlpack/methods/naive_bayes/naive_bayes_classifier_impl.hpp +++ b/src/mlpack/methods/naive_bayes/naive_bayes_classifier_impl.hpp @@ -84,30 +84,14 @@ void NaiveBayesClassifier::Train( "NaiveBayesClassifier: element type of given data must match the element " "type of the model!"); - // Do we need to resize the model? - if (probabilities.n_elem != numClasses) - { - // Perform training, after initializing the model to 0 (that is, if Train() - // won't do that for us, which it won't if we're using the incremental - // algorithm). - if (incremental) - { - probabilities.zeros(numClasses); - means.zeros(data.n_rows, numClasses); - variances.zeros(data.n_rows, numClasses); - } - else - { - probabilities.set_size(numClasses); - means.set_size(data.n_rows, numClasses); - variances.set_size(data.n_rows, numClasses); - } - } - // Calculate the class probabilities as well as the sample mean and variance // for each of the features with respect to each of the labels. if (incremental) { + // Do we need to resize the model? + if (probabilities.n_elem != numClasses || data.n_rows != means.n_rows) + Reset(data.n_rows, numClasses); + // Use incremental algorithm. // Fist, de-normalize probabilities. probabilities *= trainingPoints; @@ -117,7 +101,7 @@ void NaiveBayesClassifier::Train( const size_t label = labels[j]; ++probabilities[label]; - arma::vec delta = data.col(j) - means.col(label); + arma::Col delta = data.col(j) - means.col(label); means.col(label) += delta / probabilities[label]; variances.col(label) += delta % (data.col(j) - means.col(label)); } @@ -131,9 +115,9 @@ void NaiveBayesClassifier::Train( else { // Set all parameters to zero. - probabilities.zeros(); - means.zeros(); - variances.zeros(); + probabilities.zeros(numClasses); + means.zeros(data.n_rows, numClasses); + variances.zeros(data.n_rows, numClasses); // Don't use incremental algorithm. This is a two-pass algorithm. It is // possible to calculate the means and variances using a faster one-pass @@ -174,6 +158,19 @@ void NaiveBayesClassifier::Train( trainingPoints += data.n_cols; } +template +template +void NaiveBayesClassifier::Train( + const MatType& data, + const arma::Row& labels, + const size_t numClasses, + const bool incremental, + const double epsilon) +{ + this->epsilon = epsilon; + Train(data, labels, numClasses, incremental); +} + template template void NaiveBayesClassifier::Train(const VecType& point, @@ -183,11 +180,20 @@ void NaiveBayesClassifier::Train(const VecType& point, "NaiveBayesClassifier: element type of given data must match the element " "type of the model!"); + if (point.n_elem != means.n_rows) + { + std::ostringstream oss; + oss << "NaiveBayesClassifier::Train(): given point has dimensionality " + << point.n_elem << ", but model has dimensionality " << means.n_rows + << "!"; + throw std::invalid_argument(oss.str()); + } + // We must use the incremental algorithm here. probabilities *= trainingPoints; probabilities[label]++; - arma::vec delta = point - means.col(label); + arma::Col delta = point - means.col(label); means.col(label) += delta / probabilities[label]; if (probabilities[label] > 2) variances.col(label) *= (probabilities[label] - 2); @@ -237,6 +243,15 @@ size_t NaiveBayesClassifier::Classify(const VecType& point) const "NaiveBayesClassifier: element type of given data must match the element " "type of the model!"); + if (point.n_elem != means.n_rows) + { + std::ostringstream oss; + oss << "NaiveBayesClassifier::Classify(): given point has dimensionality " + << point.n_elem << ", but model has dimensionality " << means.n_rows + << "!"; + throw std::invalid_argument(oss.str()); + } + // Find the label(class) with max log likelihood. ModelMatType logLikelihoods; LogLikelihood(point, logLikelihoods); @@ -261,6 +276,15 @@ void NaiveBayesClassifier::Classify( "NaiveBayesClassifier: element type of given data must match the element " "type of the model!"); + if (point.n_elem != means.n_rows) + { + std::ostringstream oss; + oss << "NaiveBayesClassifier::Classify(): given point has dimensionality " + << point.n_elem << ", but model has dimensionality " << means.n_rows + << "!"; + throw std::invalid_argument(oss.str()); + } + // log(Prob(Y|X)) = Log(Prob(X|Y)) + Log(Prob(Y)) - Log(Prob(X)); // But LogLikelihood() gives us the unnormalized log likelihood which is // Log(Prob(X|Y)) + Log(Prob(Y)) so we need to subtract the normalization @@ -290,6 +314,15 @@ void NaiveBayesClassifier::Classify( "NaiveBayesClassifier: element type of given data must match the element " "type of the model!"); + if (data.n_rows != means.n_rows) + { + std::ostringstream oss; + oss << "NaiveBayesClassifier::Classify(): given data has dimensionality " + << data.n_rows << ", but model has dimensionality " << means.n_rows + << "!"; + throw std::invalid_argument(oss.str()); + } + predictions.set_size(data.n_cols); ModelMatType logLikelihoods; @@ -318,6 +351,15 @@ void NaiveBayesClassifier::Classify( "NaiveBayesClassifier: element type of given data must match the element " "type of the model!"); + if (data.n_rows != means.n_rows) + { + std::ostringstream oss; + oss << "NaiveBayesClassifier::Classify(): given data has dimensionality " + << data.n_rows << ", but model has dimensionality " << means.n_rows + << "!"; + throw std::invalid_argument(oss.str()); + } + predictions.set_size(data.n_cols); ModelMatType logLikelihoods; @@ -346,15 +388,49 @@ void NaiveBayesClassifier::Classify( } } +template +void NaiveBayesClassifier::Reset() +{ + means.zeros(); + probabilities.zeros(); + variances.fill(epsilon); + trainingPoints = 0; +} + +template +void NaiveBayesClassifier::Reset(const size_t dimensionality, + const size_t numClasses, + const double epsilon) +{ + this->epsilon = epsilon; + + probabilities.zeros(numClasses); + means.zeros(dimensionality, numClasses); + variances.zeros(dimensionality, numClasses); + trainingPoints = 0; +} + template template void NaiveBayesClassifier::serialize( Archive& ar, - const uint32_t /* version */) + const uint32_t version) { ar(CEREAL_NVP(means)); ar(CEREAL_NVP(variances)); ar(CEREAL_NVP(probabilities)); + + if (cereal::is_loading() && version == 0) + { + // Old versions did not serialize the trainingPoints or epsilon members. + trainingPoints = 0; + epsilon = 1e-10; + } + else + { + ar(CEREAL_NVP(trainingPoints)); + ar(CEREAL_NVP(epsilon)); + } } } // namespace mlpack diff --git a/src/mlpack/tests/nbc_test.cpp b/src/mlpack/tests/nbc_test.cpp index e080903e4d..56df32b762 100644 --- a/src/mlpack/tests/nbc_test.cpp +++ b/src/mlpack/tests/nbc_test.cpp @@ -397,3 +397,104 @@ TEST_CASE("NaiveBayesClassifierHighDimensionsTest", "[NBCTest]") for (size_t i = 0; i < calcVec.n_cols; ++i) REQUIRE(calcVec(i) == testLabels(i)); } + +/** + * Test that we can reset the model. + */ +TEST_CASE("NBCResetTest", "[NBCTest]") +{ + const char* trainFilename = "trainSet.csv"; + + arma::mat trainData; + if (!data::Load(trainFilename, trainData)) + FAIL("Cannot load dataset"); + + // Get the labels out. + arma::Row labels(trainData.n_cols); + for (size_t i = 0; i < trainData.n_cols; ++i) + labels[i] = trainData(trainData.n_rows - 1, i); + trainData.shed_row(trainData.n_rows - 1); + + NaiveBayesClassifier<> nbc1, nbc2; + + nbc1.Train(trainData, labels, 2); + nbc2.Train(trainData, labels, 2); + + REQUIRE(approx_equal(nbc1.Probabilities(), nbc2.Probabilities(), "absdiff", + 1e-5)); + REQUIRE(approx_equal(nbc1.Means(), nbc2.Means(), "absdiff", 1e-5)); + REQUIRE(approx_equal(nbc1.Variances(), nbc2.Variances(), "absdiff", 1e-5)); + + // Now reset one model but train the other. Modify the training set slightly. + trainData += 0.1 * arma::randu(trainData.n_rows, trainData.n_cols); + + nbc1.Train(trainData, labels, 2); + nbc2.Reset(); + nbc2.Train(trainData, labels, 2); + + REQUIRE(!approx_equal(nbc1.Probabilities(), nbc2.Probabilities(), "absdiff", + 1e-5)); + REQUIRE(!approx_equal(nbc1.Means(), nbc2.Means(), "absdiff", 1e-5)); + REQUIRE(!approx_equal(nbc1.Variances(), nbc2.Variances(), "absdiff", 1e-5)); +} + +/** + * Test that we can use a model for incremental point-by-point training. + */ +TEMPLATE_TEST_CASE("NBCIncrementalTest", "[NBCTest]", arma::fmat, arma::mat) +{ + typedef TestType MatType; + + const char* trainFilename = "trainSet.csv"; + + MatType trainData; + if (!data::Load(trainFilename, trainData)) + FAIL("Cannot load dataset"); + + // Get the labels out. + arma::Row labels(trainData.n_cols); + for (size_t i = 0; i < trainData.n_cols; ++i) + labels[i] = trainData(trainData.n_rows - 1, i); + trainData.shed_row(trainData.n_rows - 1); + + NaiveBayesClassifier nbc(trainData.n_rows, classes); + + for (size_t i = 0; i < trainData.n_cols; ++i) + { + nbc.Train(trainData.col(i), labels[i]); + } + + // We don't care what the result is, we are more concerned with the fact that + // training worked at all. + REQUIRE(nbc.Probabilities().n_elem == 2); + REQUIRE(nbc.Means().n_rows == trainData.n_rows); + REQUIRE(nbc.Means().n_cols == 2); + REQUIRE(nbc.Variances().n_rows == trainData.n_rows); + REQUIRE(nbc.Variances().n_cols == 2); +} + +/** + * Test that we can train on sparse data with different internal model types. + */ +TEMPLATE_TEST_CASE("NBCModelMatTypeTest", "[NBCTest]", float, double) +{ + typedef TestType ElemType; + + NaiveBayesClassifier> nbc; + + // Create random data; 5000 points in 4 classes. + arma::SpMat data; + data.sprandu(100, 5000, 0.2); + arma::Row labels = + arma::randi>(5000, arma::distr_param(0, 3)); + + nbc.Train(data, labels, 4); + + // We don't care what the result is, we are more concerned with the fact that + // training worked at all. + REQUIRE(nbc.Probabilities().n_elem == 4); + REQUIRE(nbc.Means().n_rows == data.n_rows); + REQUIRE(nbc.Means().n_cols == 4); + REQUIRE(nbc.Variances().n_rows == data.n_rows); + REQUIRE(nbc.Variances().n_cols == 4); +} From b9ba26b4f92e4fa7c0b6d86f0982596ef8e7ab39 Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Mon, 27 Nov 2023 17:05:41 -0500 Subject: [PATCH 07/91] Fix example code. --- doc/user/methods/naive_bayes_classifier.md | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/doc/user/methods/naive_bayes_classifier.md b/doc/user/methods/naive_bayes_classifier.md index ba3b8db766..61f8a578de 100644 --- a/doc/user/methods/naive_bayes_classifier.md +++ b/doc/user/methods/naive_bayes_classifier.md @@ -18,7 +18,7 @@ arma::Row labels = arma::mat testDataset(5, 500, arma::fill::randu); // 500 test points. mlpack::NaiveBayesClassifier nbc; // Step 1: create model. -nbc.Train(dataset, labels); // Step 2: train model. +nbc.Train(dataset, labels, 4); // Step 2: train model. arma::Row predictions; nbc.Classify(testDataset, predictions); // Step 3: classify points. @@ -208,12 +208,12 @@ mlpack::data::Load("mnist.train.csv", dataset, true); arma::Row labels; mlpack::data::Load("mnist.train.labels.csv", labels, true); -mlpack::NaiveBayesClassifier nbc(data.n_rows /* dimensionality */, +mlpack::NaiveBayesClassifier nbc(dataset.n_rows /* dimensionality */, 10 /* numClasses */); // Iterate over all points in the dataset and call Train() on each point. -for (size_t i = 0; i < data.n_cols; ++i) - nbc.Train(data.col(i), labels[i]); +for (size_t i = 0; i < dataset.n_cols; ++i) + nbc.Train(dataset.col(i), labels[i]); // Now compute the accuracy of the fully trained model on a test set. @@ -233,7 +233,7 @@ std::cout << "Accuracy of model on test data: " << accuracy << "\%." << std::endl; // Save the model to disk with the name "nbc". -data::Save("nbc_model.bin", "nbc", nbc, true); +mlpack::data::Save("nbc_model.bin", "nbc", nbc, true); ``` --- @@ -241,10 +241,10 @@ data::Save("nbc_model.bin", "nbc", nbc, true); Load a saved Naive Bayes classifier and print some information about it. ```c++ -NaiveBayesClassifier nbc; +mlpack::NaiveBayesClassifier nbc; // Load the model named "nbc" from "nbc_model.bin". -data::Load("nbc_model.bin", "nbc", nbc, true); +mlpack::data::Load("nbc_model.bin", "nbc", nbc, true); // Print information about the model. std::cout << "The dimensionality of the model in nbc_model.bin is " @@ -255,7 +255,9 @@ std::cout << "The prior probabilities of each class are: " << nbc.Probabilities().t(); // Compute the class probabilities of a random point. -arma::vec randomPoint(nbc.Means().n_rows, arma::fill::randu); +// For our random point, we'll use one of the means plus some noise. +arma::vec randomPoint = nbc.Means().col(2) + + 10.0 * arma::randu(nbc.Means().n_rows); size_t prediction; arma::vec probabilities; From 48b85e8646e58c1a935bffb789def76752b91402 Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Mon, 27 Nov 2023 17:05:55 -0500 Subject: [PATCH 08/91] Make sure variances always have epsilon, regardless of training mode. --- .../naive_bayes/naive_bayes_classifier_impl.hpp | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/src/mlpack/methods/naive_bayes/naive_bayes_classifier_impl.hpp b/src/mlpack/methods/naive_bayes/naive_bayes_classifier_impl.hpp index 5c2eeaa7ed..88c291d4fa 100644 --- a/src/mlpack/methods/naive_bayes/naive_bayes_classifier_impl.hpp +++ b/src/mlpack/methods/naive_bayes/naive_bayes_classifier_impl.hpp @@ -46,7 +46,8 @@ NaiveBayesClassifier::NaiveBayesClassifier( { probabilities.zeros(numClasses); means.zeros(data.n_rows, numClasses); - variances.zeros(data.n_rows, numClasses); + variances.set_size(data.n_rows, numClasses); + variances.fill(epsilon); } else { @@ -69,7 +70,8 @@ NaiveBayesClassifier::NaiveBayesClassifier( // Initialize model to 0. probabilities.zeros(numClasses); means.zeros(dimensionality, numClasses); - variances.zeros(dimensionality, numClasses); + variances.set_size(dimensionality, numClasses); + variances.fill(epsilon); } template @@ -294,8 +296,8 @@ void NaiveBayesClassifier::Classify( // To prevent underflow in log of sum of exp of x operation (where x is a // small negative value), we use logsumexp(x - max(x)) + max(x). - const double maxValue = arma::max(logLikelihoods); - const double logProbX = log(arma::accu(exp(logLikelihoods - maxValue))) + + const ElemType maxValue = logLikelihoods.max(); + const ElemType logProbX = log(arma::accu(exp(logLikelihoods - maxValue))) + maxValue; probabilities = exp(logLikelihoods - logProbX); // log(exp(value)) == value. @@ -406,7 +408,8 @@ void NaiveBayesClassifier::Reset(const size_t dimensionality, probabilities.zeros(numClasses); means.zeros(dimensionality, numClasses); - variances.zeros(dimensionality, numClasses); + variances.set_size(dimensionality, numClasses); + variances.fill(epsilon); trainingPoints = 0; } From 4878ade588ba886e9d363ec4649a6d7e943435e8 Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Mon, 27 Nov 2023 17:13:24 -0500 Subject: [PATCH 09/91] Point out Epsilon() method. --- doc/user/methods/naive_bayes_classifier.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/doc/user/methods/naive_bayes_classifier.md b/doc/user/methods/naive_bayes_classifier.md index 61f8a578de..f85751fc8b 100644 --- a/doc/user/methods/naive_bayes_classifier.md +++ b/doc/user/methods/naive_bayes_classifier.md @@ -84,6 +84,11 @@ std::cout << arma::accu(predictions == 2) << " test points classified as class " | `epsilon` | `double` | Initial small value for sample variances, to prevent underflow (via `log(0)`). | 1e-10 | +As an alternative to passing the `epsilon` parameter, it can be set with the +standalone `Epsilon()` method: `nbc.Epsilon() = eps;` will set the value of +`epsilon` to `eps` for the next time non-incremental `Train()` or `Reset()` is +called. + ### Training If training is not done as part of the constructor call, it can be done with the From 308be4fa9503ba7d2faa6289291ac027003b5a7a Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Mon, 27 Nov 2023 17:16:39 -0500 Subject: [PATCH 10/91] Add a caveat about sparse model types. --- doc/user/methods/naive_bayes_classifier.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/doc/user/methods/naive_bayes_classifier.md b/doc/user/methods/naive_bayes_classifier.md index f85751fc8b..6970a8abd6 100644 --- a/doc/user/methods/naive_bayes_classifier.md +++ b/doc/user/methods/naive_bayes_classifier.md @@ -315,3 +315,6 @@ std::cout << "Prediction for random test point: " << prediction << "." std::cout << "Class probabilities for random test point: " << probabilitiesVec.t(); ``` + +***Note:*** dense objects should be used for `ModelMatType`, since in general +the mean and sample variance of sparse data is dense. From 4682706a99c921a90911b618dcca0a1337a95983 Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Mon, 27 Nov 2023 17:22:31 -0500 Subject: [PATCH 11/91] Some additional cleanups and fixes. --- doc/user/methods/naive_bayes_classifier.md | 26 ++++++++++++++-------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/doc/user/methods/naive_bayes_classifier.md b/doc/user/methods/naive_bayes_classifier.md index 6970a8abd6..2cb975d028 100644 --- a/doc/user/methods/naive_bayes_classifier.md +++ b/doc/user/methods/naive_bayes_classifier.md @@ -81,8 +81,7 @@ std::cout << arma::accu(predictions == 2) << " test points classified as class " | `labels` | [`arma::Row`]('../matrices.md') | Training labels, between `0` and `numClasses - 1` (inclusive). Should have length `data.n_cols`. | _(N/A)_ | | `numClasses` | `size_t` | Number of classes in the dataset. | _(N/A)_ | | `incremental` | `bool` | If `true`, then the model will not be reset before training, and will use a robust incremental algorithm for variance computation. | `true` | -| `epsilon` | `double` | Initial small value for sample variances, to prevent -underflow (via `log(0)`). | 1e-10 | +| `epsilon` | `double` | Initial small value for sample variances, to prevent underflow (via `log(0)`). | 1e-10 | As an alternative to passing the `epsilon` parameter, it can be set with the standalone `Epsilon()` method: `nbc.Epsilon() = eps;` will set the value of @@ -230,11 +229,17 @@ arma::Row testLabels; mlpack::data::Load("mnist.test.labels.csv", testLabels, true); arma::Row predictions; +nbc.Classify(dataset, predictions); +const double trainAccuracy = 100.0 * + ((double) arma::accu(predictions == labels)) / labels.n_elem; +std::cout << "Accuracy of model on training data: " << trainAccuracy << "\%." + << std::endl; + nbc.Classify(testDataset, predictions); -const double accuracy = 100.0 * +const double testAccuracy = 100.0 * ((double) arma::accu(predictions == testLabels)) / testLabels.n_elem; -std::cout << "Accuracy of model on test data: " << accuracy << "\%." +std::cout << "Accuracy of model on test data: " << testAccuracy << "\%." << std::endl; // Save the model to disk with the name "nbc". @@ -256,6 +261,8 @@ std::cout << "The dimensionality of the model in nbc_model.bin is " << nbc.Means().n_rows << "." << std::endl; std::cout << "The number of classes in the model is " << nbc.Probabilities().n_elem << "." << std::endl; +std::cout << "The model was trained on " << nbc.TrainingPoints() << " points." + << std::endl; std::cout << "The prior probabilities of each class are: " << nbc.Probabilities().t(); @@ -283,12 +290,13 @@ NaiveBayesClassifier ``` `ModelMatType` specifies the type of matrix used for training data and internal -representation of model parameters. Any matrix type that implements the -Armadillo API can be used. +representation of model parameters. -Note that the `Train()` and `Classify()` functions themselves are templatized -and can allow any matrix type that has the same element type. So, for instance, -a `NaiveBayesClassifier` can accept an `arma::sp_mat` for training. + * Any matrix type that implements the Armadillo API can be used. + + * `Train()` and `Classify()` functions themselves are templatized and can allow + any matrix type that has the same element type. So, for instance, a + `NaiveBayesClassifier` can accept an `arma::sp_mat` for training. The example below trains a Naive Bayes model on sparse 32-bit floating point data, but uses dense 32-bit floating point matrices to store the model itself. From 599a630656bcb68a9a6fc2fa10c5f5056d857732 Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Mon, 27 Nov 2023 17:23:24 -0500 Subject: [PATCH 12/91] Fix object names. --- doc/user/methods/naive_bayes_classifier.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/doc/user/methods/naive_bayes_classifier.md b/doc/user/methods/naive_bayes_classifier.md index 2cb975d028..c53d5d415e 100644 --- a/doc/user/methods/naive_bayes_classifier.md +++ b/doc/user/methods/naive_bayes_classifier.md @@ -124,14 +124,14 @@ an exception will be thrown. Once a `NaiveBayesClassifier` model is trained, the `Classify()` member function can be used to make class predictions for new data. - * `size_t predictedClass = sr.Classify(point)` + * `size_t predictedClass = nbc.Classify(point)` - ***(Single-point)*** - Classify a single point, returning the predicted class (`0` through `numClasses - 1`, inclusive). --- - * `sr.Classify(point, prediction, probabilitiesVec)` + * `nbc.Classify(point, prediction, probabilitiesVec)` - ***(Single-point)*** - Classify a single point and compute class probabilities. - The predicted class is stored in `prediction`. @@ -308,7 +308,7 @@ dataset.sprandu(100, 5000, 0.3); arma::Row labels = arma::randi>(5000, arma::distr_param(0, 2)); -mlpack::NaiveBayesClassifier sr(dataset, labels, 3); +mlpack::NaiveBayesClassifier nbc(dataset, labels, 3); // Now classify a test point. arma::sp_fvec point; @@ -316,7 +316,7 @@ point.sprandu(100, 1, 0.3); size_t prediction; arma::fvec probabilitiesVec; -sr.Classify(point, prediction, probabilitiesVec); +nbc.Classify(point, prediction, probabilitiesVec); std::cout << "Prediction for random test point: " << prediction << "." << std::endl; From c1b76943916837a0819411f08871c2efe055ee57 Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Wed, 29 Nov 2023 11:11:13 -0500 Subject: [PATCH 13/91] Implement new Train() overloads and tests for them. --- .../dists/regression_distribution_impl.hpp | 4 +- .../linear_regression/linear_regression.hpp | 116 +++++++++++++++++- .../linear_regression_impl.hpp | 57 ++++++++- src/mlpack/tests/linear_regression_test.cpp | 55 +++++++++ 4 files changed, 224 insertions(+), 8 deletions(-) diff --git a/src/mlpack/core/dists/regression_distribution_impl.hpp b/src/mlpack/core/dists/regression_distribution_impl.hpp index 328f0c4ab2..0634549e8b 100644 --- a/src/mlpack/core/dists/regression_distribution_impl.hpp +++ b/src/mlpack/core/dists/regression_distribution_impl.hpp @@ -63,8 +63,8 @@ inline double RegressionDistribution::Probability( const arma::vec& observation) const { arma::rowvec fitted; - rf.Predict(observation.rows(1, observation.n_rows-1), fitted); - return err.Probability(observation(0)-fitted.t()); + rf.Predict(observation.rows(1, observation.n_rows - 1), fitted); + return err.Probability(observation(0) - fitted.t()); } inline void RegressionDistribution::Predict(const arma::mat& points, diff --git a/src/mlpack/methods/linear_regression/linear_regression.hpp b/src/mlpack/methods/linear_regression/linear_regression.hpp index c2d7ce2e2b..7fe0bff132 100644 --- a/src/mlpack/methods/linear_regression/linear_regression.hpp +++ b/src/mlpack/methods/linear_regression/linear_regression.hpp @@ -71,14 +71,90 @@ class LinearRegression * regularization parameter lambda, call Lambda() or set a different value in * the constructor. * + * This version of `Train()` is deprecated and will be removed in mlpack + * 5.0.0. Use the version of `Train()` that specifies `lambda` before + * `intercept` instead. + * * @param predictors X, the matrix of data points to train the model on. * @param responses y, the responses to the data points. * @param intercept Whether or not to fit an intercept term. * @return The least squares error after training. */ + mlpack_deprecated /** Will be removed in mlpack 5.0.0. */ double Train(const arma::mat& predictors, const arma::rowvec& responses, - const bool intercept = true); + const bool intercept); + + /** + * Train the LinearRegression model on the given data and weights. Careful! + * This will completely ignore and overwrite the existing model. This + * particular implementation does not have an incremental training algorithm. + * To set the regularization parameter lambda, call Lambda() or set a + * different value in the constructor. + * + * This version of `Train()` is deprecated and will be removed in mlpack + * 5.0.0. Use the version of `Train()` that specifies `lambda` before + * `intercept` instead. + * + * @param predictors X, the matrix of data points to train the model on. + * @param responses y, the responses to the data points. + * @param weights Observation weights (for boosting). + * @param intercept Whether or not to fit an intercept term. + * @return The least squares error after training. + */ + mlpack_deprecated /** Will be removed in mlpack 5.0.0. */ + double Train(const arma::mat& predictors, + const arma::rowvec& responses, + const arma::rowvec& weights, + const bool intercept); + + /** + * Train the LinearRegression model on the given data and weights. Careful! + * This will completely ignore and overwrite the existing model. This + * particular implementation does not have an incremental training algorithm. + * To set the regularization parameter lambda, call Lambda() or set a + * different value in the constructor. + * + * @param predictors X, the matrix of data points to train the model on. + * @param responses y, the responses to the data points. + * @return The least squares error after training. + */ + double Train(const arma::mat& predictors, + const arma::rowvec& responses); + + /** + * Train the LinearRegression model on the given data and weights. Careful! + * This will completely ignore and overwrite the existing model. This + * particular implementation does not have an incremental training algorithm. + * To set the regularization parameter lambda, call Lambda() or set a + * different value in the constructor. + * + * @param predictors X, the matrix of data points to train the model on. + * @param responses y, the responses to the data points. + * @param lambda L2 regularization penalty parameter to use. + * @return The least squares error after training. + */ + double Train(const arma::mat& predictors, + const arma::rowvec& responses, + const double lambda); + + /** + * Train the LinearRegression model on the given data and weights. Careful! + * This will completely ignore and overwrite the existing model. This + * particular implementation does not have an incremental training algorithm. + * To set the regularization parameter lambda, call Lambda() or set a + * different value in the constructor. + * + * @param predictors X, the matrix of data points to train the model on. + * @param responses y, the responses to the data points. + * @param lambda L2 regularization penalty parameter to use. + * @param intercept Whether or not to fit an intercept term. + * @return The least squares error after training. + */ + double Train(const arma::mat& predictors, + const arma::rowvec& responses, + const double lambda, + const bool intercept); /** * Train the LinearRegression model on the given data and weights. Careful! @@ -90,13 +166,49 @@ class LinearRegression * @param predictors X, the matrix of data points to train the model on. * @param responses y, the responses to the data points. * @param weights Observation weights (for boosting). + * @return The least squares error after training. + */ + double Train(const arma::mat& predictors, + const arma::rowvec& responses, + const arma::rowvec& weights); + + /** + * Train the LinearRegression model on the given data and weights. Careful! + * This will completely ignore and overwrite the existing model. This + * particular implementation does not have an incremental training algorithm. + * To set the regularization parameter lambda, call Lambda() or set a + * different value in the constructor. + * + * @param predictors X, the matrix of data points to train the model on. + * @param responses y, the responses to the data points. + * @param weights Observation weights (for boosting). + * @param lambda L2 regularization penalty parameter to use. + * @return The least squares error after training. + */ + double Train(const arma::mat& predictors, + const arma::rowvec& responses, + const arma::rowvec& weights, + const double lambda); + + /** + * Train the LinearRegression model on the given data and weights. Careful! + * This will completely ignore and overwrite the existing model. This + * particular implementation does not have an incremental training algorithm. + * To set the regularization parameter lambda, call Lambda() or set a + * different value in the constructor. + * + * @param predictors X, the matrix of data points to train the model on. + * @param responses y, the responses to the data points. + * @param weights Observation weights (for boosting). + * @param lambda L2 regularization penalty parameter to use. * @param intercept Whether or not to fit an intercept term. * @return The least squares error after training. */ double Train(const arma::mat& predictors, const arma::rowvec& responses, const arma::rowvec& weights, - const bool intercept = true); + const double lambda, + const bool intercept); /** * Calculate y_i for each data point in points. diff --git a/src/mlpack/methods/linear_regression/linear_regression_impl.hpp b/src/mlpack/methods/linear_regression/linear_regression_impl.hpp index fe79f2ed55..3980b9d881 100644 --- a/src/mlpack/methods/linear_regression/linear_regression_impl.hpp +++ b/src/mlpack/methods/linear_regression/linear_regression_impl.hpp @@ -37,18 +37,67 @@ inline LinearRegression::LinearRegression( Train(predictors, responses, weights, intercept); } +mlpack_deprecated /** Will be removed in mlpack 5.0.0. */ inline double LinearRegression::Train(const arma::mat& predictors, const arma::rowvec& responses, const bool intercept) { - return Train(predictors, responses, arma::rowvec(), intercept); + return Train(predictors, responses, arma::rowvec(), this->lambda, intercept); } +mlpack_deprecated /** Will be removed in mlpack 5.0.0. */ inline double LinearRegression::Train(const arma::mat& predictors, const arma::rowvec& responses, const arma::rowvec& weights, const bool intercept) { + return Train(predictors, responses, weights, this->lambda, intercept); +} + +inline double LinearRegression::Train(const arma::mat& predictors, + const arma::rowvec& responses) +{ + return Train(predictors, responses, arma::rowvec(), this->lambda, + this->intercept); +} + +inline double LinearRegression::Train(const arma::mat& predictors, + const arma::rowvec& responses, + const double lambda) +{ + return Train(predictors, responses, arma::rowvec(), lambda, this->intercept); +} + +inline double LinearRegression::Train(const arma::mat& predictors, + const arma::rowvec& responses, + const double lambda, + const bool intercept) +{ + return Train(predictors, responses, arma::rowvec(), lambda, intercept); +} + +inline double LinearRegression::Train(const arma::mat& predictors, + const arma::rowvec& responses, + const arma::rowvec& weights) +{ + return Train(predictors, responses, weights, this->lambda, this->intercept); +} + +inline double LinearRegression::Train(const arma::mat& predictors, + const arma::rowvec& responses, + const arma::rowvec& weights, + const double lambda) +{ + return Train(predictors, responses, weights, lambda, this->intercept); +} + +inline double LinearRegression::Train(const arma::mat& predictors, + const arma::rowvec& responses, + const arma::rowvec& weights, + const double lambda, + const bool intercept) +{ + this->lambda = lambda; this->intercept = intercept; /* @@ -106,7 +155,7 @@ inline void LinearRegression::Predict( // Prevent underflow. const size_t labels = (parameters.n_rows == 0) ? size_t(0) : size_t(parameters.n_rows - 1); - util::CheckSameDimensionality(points, labels, "LinearRegression::Predict()", + util::CheckSameDimensionality(points, labels, "LinearRegression::Predict()", "points"); // Get the predictions, but this ignores the intercept value // (parameters[0]). @@ -119,7 +168,7 @@ inline void LinearRegression::Predict( { // We want to be sure we have the correct number of dimensions in // the dataset. - util::CheckSameDimensionality(points, parameters, + util::CheckSameDimensionality(points, parameters, "LinearRegression::Predict()", "points"); predictions = arma::trans(parameters) * points; } @@ -131,7 +180,7 @@ inline double LinearRegression::ComputeError( { // Sanity check on data. util::CheckSameSizes(predictors, responses, "LinearRegression::Train()"); - + // Get the number of columns and rows of the dataset. const size_t nCols = predictors.n_cols; const size_t nRows = predictors.n_rows; diff --git a/src/mlpack/tests/linear_regression_test.cpp b/src/mlpack/tests/linear_regression_test.cpp index 8bbf9dc7d6..bd12dc18c8 100644 --- a/src/mlpack/tests/linear_regression_test.cpp +++ b/src/mlpack/tests/linear_regression_test.cpp @@ -265,3 +265,58 @@ TEST_CASE("LinearRegressionTrainReturnObjective", "[LinearRegressionTest]") REQUIRE(std::isfinite(error) == true); } + +/** + * Make sure all versions of Train() work correctly. + */ +TEST_CASE("LinearRegressionAllTrainVersionsTest", "[LinearRegressionTest]") +{ + // The data doesn't really matter for this test; mostly we want to make sure + // that all the Train() variants work properly. + arma::mat predictors; + predictors = { { 0, 1, 2, 4, 8, 16 }, + { 16, 8, 4, 2, 1, 0 } }; + arma::rowvec responses = "0 2 4 3 8 8"; + arma::rowvec weights = "1.0 1.1 1.2 0.8 0.9 1.0"; + + LinearRegression lr1, lr2, lr3, lr4, lr5, lr6; + + lr1.Train(predictors, responses); + lr2.Train(predictors, responses, 0.1); + lr3.Train(predictors, responses, 0.2, false); + lr4.Train(predictors, responses, weights); + lr5.Train(predictors, responses, weights, 0.3); + lr6.Train(predictors, responses, weights, 0.4, false); + + // We don't care about the specifics of the trained model, but we want to just + // make sure everything appears to be correct from the sizes and + // hyperparameters. + REQUIRE(lr1.Lambda() == Approx(0.0).margin(1e-10)); + REQUIRE(lr1.Intercept() == true); + REQUIRE(lr1.Parameters().n_elem == 3); + + REQUIRE(lr2.Lambda() == Approx(0.1).margin(1e-10)); + REQUIRE(lr2.Intercept() == true); + REQUIRE(lr2.Parameters().n_elem == 3); + + REQUIRE(lr3.Lambda() == Approx(0.2).margin(1e-10)); + REQUIRE(lr3.Intercept() == false); + REQUIRE(lr3.Parameters().n_elem == 2); + + REQUIRE(lr4.Lambda() == Approx(0.0).margin(1e-10)); + REQUIRE(lr4.Intercept() == true); + REQUIRE(lr4.Parameters().n_elem == 3); + + REQUIRE(lr5.Lambda() == Approx(0.3).margin(1e-10)); + REQUIRE(lr5.Intercept() == true); + REQUIRE(lr5.Parameters().n_elem == 3); + + REQUIRE(lr6.Lambda() == Approx(0.4).margin(1e-10)); + REQUIRE(lr6.Intercept() == false); + REQUIRE(lr6.Parameters().n_elem == 2); + + // We can also check that the weighted model is different from the unweighted + // model. + REQUIRE(!arma::approx_equal(lr1.Parameters(), lr4.Parameters(), "absdiff", + 1e-5)); +} From 2acea2db66aaab9725abcdcac389b8376cc3dc04 Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Wed, 29 Nov 2023 11:11:25 -0500 Subject: [PATCH 14/91] Fix bug in Naive Bayes test. --- src/mlpack/tests/nbc_test.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mlpack/tests/nbc_test.cpp b/src/mlpack/tests/nbc_test.cpp index 56df32b762..626594f266 100644 --- a/src/mlpack/tests/nbc_test.cpp +++ b/src/mlpack/tests/nbc_test.cpp @@ -457,7 +457,7 @@ TEMPLATE_TEST_CASE("NBCIncrementalTest", "[NBCTest]", arma::fmat, arma::mat) labels[i] = trainData(trainData.n_rows - 1, i); trainData.shed_row(trainData.n_rows - 1); - NaiveBayesClassifier nbc(trainData.n_rows, classes); + NaiveBayesClassifier nbc(trainData.n_rows, 2); for (size_t i = 0; i < trainData.n_cols; ++i) { From b741ee30b1dfc89c7487f7883e1452502755b3c7 Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Wed, 29 Nov 2023 11:11:31 -0500 Subject: [PATCH 15/91] Remove fixed TODO. --- doc/user/methods/naive_bayes_classifier.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/doc/user/methods/naive_bayes_classifier.md b/doc/user/methods/naive_bayes_classifier.md index c53d5d415e..bb4395b97b 100644 --- a/doc/user/methods/naive_bayes_classifier.md +++ b/doc/user/methods/naive_bayes_classifier.md @@ -107,8 +107,6 @@ If training is not done as part of the constructor call, it can be done with the appropriate constructor form to set `dimensionality`, or by calling `Reset()` (see [other functionality](#other_functionality)). - - | **name** | **type** | **description** | **default** | |----------|----------|-----------------|-------------| | `point` | [`arma::vec`](../matrices.md) | [Column-major](../matrices.md) training point (i.e. one column). | _(N/A)_ | From 09edc48b24c64a830edd0e947022313e75bb2e19 Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Wed, 29 Nov 2023 11:20:29 -0500 Subject: [PATCH 16/91] Add single-point Predict() to LinearRegression and test it. --- .../linear_regression/linear_regression.hpp | 7 +++++ .../linear_regression_impl.hpp | 28 ++++++++++++++++++- src/mlpack/tests/linear_regression_test.cpp | 25 +++++++++++++++++ 3 files changed, 59 insertions(+), 1 deletion(-) diff --git a/src/mlpack/methods/linear_regression/linear_regression.hpp b/src/mlpack/methods/linear_regression/linear_regression.hpp index 7fe0bff132..e3221ab089 100644 --- a/src/mlpack/methods/linear_regression/linear_regression.hpp +++ b/src/mlpack/methods/linear_regression/linear_regression.hpp @@ -210,6 +210,13 @@ class LinearRegression const double lambda, const bool intercept); + /** + * Calculate y_i for a single data point. + * + * @param point the data point to calculate with. + */ + double Predict(const arma::vec& point) const; + /** * Calculate y_i for each data point in points. * diff --git a/src/mlpack/methods/linear_regression/linear_regression_impl.hpp b/src/mlpack/methods/linear_regression/linear_regression_impl.hpp index 3980b9d881..45ac0b7211 100644 --- a/src/mlpack/methods/linear_regression/linear_regression_impl.hpp +++ b/src/mlpack/methods/linear_regression/linear_regression_impl.hpp @@ -34,7 +34,7 @@ inline LinearRegression::LinearRegression( lambda(lambda), intercept(intercept) { - Train(predictors, responses, weights, intercept); + Train(predictors, responses, weights, lambda, intercept); } mlpack_deprecated /** Will be removed in mlpack 5.0.0. */ @@ -144,6 +144,32 @@ inline double LinearRegression::Train(const arma::mat& predictors, return ComputeError(predictors, responses); } +inline double LinearRegression::Predict(const arma::vec& point) const +{ + if (intercept) + { + // We want to be sure we have the correct number of dimensions in the + // dataset. + // Prevent underflow. + const size_t labels = (parameters.n_rows == 0) ? size_t(0) : + size_t(parameters.n_rows - 1); + util::CheckSameDimensionality(point, labels, "LinearRegression::Predict()", + "point"); + + return dot(parameters.subvec(1, parameters.n_elem - 1).t(), point) + + parameters(0); + } + else + { + // We want to be sure we have the correct number of dimensions in + // the dataset. + util::CheckSameDimensionality(point, parameters, + "LinearRegression::Predict()", "point"); + + return dot(parameters.t(), point); + } +} + inline void LinearRegression::Predict( const arma::mat& points, arma::rowvec& predictions) const diff --git a/src/mlpack/tests/linear_regression_test.cpp b/src/mlpack/tests/linear_regression_test.cpp index bd12dc18c8..1dad701986 100644 --- a/src/mlpack/tests/linear_regression_test.cpp +++ b/src/mlpack/tests/linear_regression_test.cpp @@ -320,3 +320,28 @@ TEST_CASE("LinearRegressionAllTrainVersionsTest", "[LinearRegressionTest]") REQUIRE(!arma::approx_equal(lr1.Parameters(), lr4.Parameters(), "absdiff", 1e-5)); } + +/** + * Ensure that single-point Predict() returns the same results as multi-point + * Predict(). + */ +TEST_CASE("LinearRegressionSinglePointPredictTest", "[LinearRegressionTest]") +{ + arma::mat predictors; + predictors = { { 0, 1, 2, 4, 8, 16 }, + { 16, 8, 4, 2, 1, 0 } }; + arma::rowvec responses = "0 2 4 3 8 8"; + + LinearRegression lr(predictors, responses, 0.1, true); + + // Compute predictions for test points in batch. + arma::rowvec predictions; + lr.Predict(predictors, predictions); + + // Now compute each prediction individually. + for (size_t i = 0; i < predictors.n_cols; ++i) + { + const double prediction = lr.Predict(predictors.col(i)); + REQUIRE(prediction == Approx(predictions[i])); + } +} From d32087c0464fbffd49674d2104746227bf1b8fb1 Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Wed, 29 Nov 2023 11:38:46 -0500 Subject: [PATCH 17/91] Templatize LinearRegression. --- .../linear_regression/linear_regression.hpp | 87 +++++----- .../linear_regression_impl.hpp | 161 ++++++++++++------ 2 files changed, 160 insertions(+), 88 deletions(-) diff --git a/src/mlpack/methods/linear_regression/linear_regression.hpp b/src/mlpack/methods/linear_regression/linear_regression.hpp index e3221ab089..b56f379471 100644 --- a/src/mlpack/methods/linear_regression/linear_regression.hpp +++ b/src/mlpack/methods/linear_regression/linear_regression.hpp @@ -26,9 +26,12 @@ namespace mlpack { * Optionally, this class can perform ridge regression, if the lambda parameter * is set to a number greater than zero. */ +template class LinearRegression { public: + typedef typename ModelMatType::elem_type ElemType; + /** * Creates the model. * @@ -37,8 +40,9 @@ class LinearRegression * @param lambda Regularization constant for ridge regression. * @param intercept Whether or not to include an intercept term. */ - LinearRegression(const arma::mat& predictors, - const arma::rowvec& responses, + template + LinearRegression(const MatType& predictors, + const ResponsesType& responses, const double lambda = 0, const bool intercept = true); @@ -51,9 +55,10 @@ class LinearRegression * @param lambda Regularization constant for ridge regression. * @param intercept Whether or not to include an intercept term. */ - LinearRegression(const arma::mat& predictors, - const arma::rowvec& responses, - const arma::rowvec& weights, + template + LinearRegression(const MatType& predictors, + const ResponsesType& responses, + const ResponsesType& weights, const double lambda = 0, const bool intercept = true); @@ -119,8 +124,9 @@ class LinearRegression * @param responses y, the responses to the data points. * @return The least squares error after training. */ - double Train(const arma::mat& predictors, - const arma::rowvec& responses); + template + ElemType Train(const MatType& predictors, + const ResponsesType& responses); /** * Train the LinearRegression model on the given data and weights. Careful! @@ -134,9 +140,10 @@ class LinearRegression * @param lambda L2 regularization penalty parameter to use. * @return The least squares error after training. */ - double Train(const arma::mat& predictors, - const arma::rowvec& responses, - const double lambda); + template + ElemType Train(const MatType& predictors, + const ResponsesType& responses, + const double lambda); /** * Train the LinearRegression model on the given data and weights. Careful! @@ -151,10 +158,11 @@ class LinearRegression * @param intercept Whether or not to fit an intercept term. * @return The least squares error after training. */ - double Train(const arma::mat& predictors, - const arma::rowvec& responses, - const double lambda, - const bool intercept); + template + ElemType Train(const MatType& predictors, + const ResponsesType& responses, + const double lambda, + const bool intercept); /** * Train the LinearRegression model on the given data and weights. Careful! @@ -168,9 +176,10 @@ class LinearRegression * @param weights Observation weights (for boosting). * @return The least squares error after training. */ - double Train(const arma::mat& predictors, - const arma::rowvec& responses, - const arma::rowvec& weights); + template + ElemType Train(const MatType& predictors, + const ResponsesType& responses, + const ResponsesType& weights); /** * Train the LinearRegression model on the given data and weights. Careful! @@ -185,10 +194,11 @@ class LinearRegression * @param lambda L2 regularization penalty parameter to use. * @return The least squares error after training. */ - double Train(const arma::mat& predictors, - const arma::rowvec& responses, - const arma::rowvec& weights, - const double lambda); + template + ElemType Train(const MatType& predictors, + const ResponsesType& responses, + const ResponsesType& weights, + const double lambda); /** * Train the LinearRegression model on the given data and weights. Careful! @@ -204,18 +214,20 @@ class LinearRegression * @param intercept Whether or not to fit an intercept term. * @return The least squares error after training. */ - double Train(const arma::mat& predictors, - const arma::rowvec& responses, - const arma::rowvec& weights, - const double lambda, - const bool intercept); + template + ElemType Train(const MatType& predictors, + const ResponsesType& responses, + const ResponsesType& weights, + const double lambda, + const bool intercept); /** * Calculate y_i for a single data point. * * @param point the data point to calculate with. */ - double Predict(const arma::vec& point) const; + template + ElemType Predict(const VecType& point) const; /** * Calculate y_i for each data point in points. @@ -223,7 +235,8 @@ class LinearRegression * @param points the data points to calculate with. * @param predictions y, will contain calculated values on completion. */ - void Predict(const arma::mat& points, arma::rowvec& predictions) const; + template + void Predict(const MatType& points, ResponsesType& predictions) const; /** * Calculate the L2 squared error on the given predictors and responses using @@ -242,13 +255,14 @@ class LinearRegression * @param points Matrix of predictors (X). * @param responses Transposed vector of responses (y^T). */ - double ComputeError(const arma::mat& points, - const arma::rowvec& responses) const; + template + ElemType ComputeError(const MatType& points, + const ResponsesType& responses) const; //! Return the parameters (the b vector). - const arma::vec& Parameters() const { return parameters; } + const ModelMatType& Parameters() const { return parameters; } //! Modify the parameters (the b vector). - arma::vec& Parameters() { return parameters; } + ModelMatType& Parameters() { return parameters; } //! Return the Tikhonov regularization parameter for ridge regression. double Lambda() const { return lambda; } @@ -262,19 +276,14 @@ class LinearRegression * Serialize the model. */ template - void serialize(Archive& ar, const uint32_t /* version */) - { - ar(CEREAL_NVP(parameters)); - ar(CEREAL_NVP(lambda)); - ar(CEREAL_NVP(intercept)); - } + void serialize(Archive& ar, const uint32_t version); private: /** * The calculated B. * Initialized and filled by constructor to hold the least squares solution. */ - arma::vec parameters; + ModelMatType parameters; /** * The Tikhonov regularization parameter for ridge regression (0 for linear diff --git a/src/mlpack/methods/linear_regression/linear_regression_impl.hpp b/src/mlpack/methods/linear_regression/linear_regression_impl.hpp index 45ac0b7211..14180895fc 100644 --- a/src/mlpack/methods/linear_regression/linear_regression_impl.hpp +++ b/src/mlpack/methods/linear_regression/linear_regression_impl.hpp @@ -17,18 +17,22 @@ namespace mlpack { -inline LinearRegression::LinearRegression( - const arma::mat& predictors, - const arma::rowvec& responses, +template +template +inline LinearRegression::LinearRegression( + const MatType& predictors, + const ResponsesType& responses, const double lambda, const bool intercept) : - LinearRegression(predictors, responses, arma::rowvec(), lambda, intercept) + LinearRegression(predictors, responses, ResponsesType(), lambda, intercept) { /* Nothing to do. */ } -inline LinearRegression::LinearRegression( - const arma::mat& predictors, - const arma::rowvec& responses, - const arma::rowvec& weights, +template +template +inline LinearRegression::LinearRegression( + const MatType& predictors, + const ResponsesType& responses, + const ResponsesType& weights, const double lambda, const bool intercept) : lambda(lambda), @@ -37,63 +41,91 @@ inline LinearRegression::LinearRegression( Train(predictors, responses, weights, lambda, intercept); } +template mlpack_deprecated /** Will be removed in mlpack 5.0.0. */ -inline double LinearRegression::Train(const arma::mat& predictors, - const arma::rowvec& responses, - const bool intercept) +inline double LinearRegression::Train( + const arma::mat& predictors, + const arma::rowvec& responses, + const bool intercept) { return Train(predictors, responses, arma::rowvec(), this->lambda, intercept); } +template mlpack_deprecated /** Will be removed in mlpack 5.0.0. */ -inline double LinearRegression::Train(const arma::mat& predictors, - const arma::rowvec& responses, - const arma::rowvec& weights, - const bool intercept) +inline double LinearRegression::Train( + const arma::mat& predictors, + const arma::rowvec& responses, + const arma::rowvec& weights, + const bool intercept) { return Train(predictors, responses, weights, this->lambda, intercept); } -inline double LinearRegression::Train(const arma::mat& predictors, - const arma::rowvec& responses) +template +template +inline +typename LinearRegression::ElemType +LinearRegression::Train(const MatType& predictors, + const ResponsesType& responses) { - return Train(predictors, responses, arma::rowvec(), this->lambda, + return Train(predictors, responses, ResponsesType(), this->lambda, this->intercept); } -inline double LinearRegression::Train(const arma::mat& predictors, - const arma::rowvec& responses, +template +template +inline +typename LinearRegression::ElemType +LinearRegression::Train(const MatType& predictors, + const ResponsesType& responses, const double lambda) { - return Train(predictors, responses, arma::rowvec(), lambda, this->intercept); + return Train(predictors, responses, ResponsesType(), lambda, this->intercept); } -inline double LinearRegression::Train(const arma::mat& predictors, - const arma::rowvec& responses, +template +template +inline +typename LinearRegression::ElemType +LinearRegression::Train(const MatType& predictors, + const ResponsesType& responses, const double lambda, const bool intercept) { - return Train(predictors, responses, arma::rowvec(), lambda, intercept); + return Train(predictors, responses, ResponsesType(), lambda, intercept); } -inline double LinearRegression::Train(const arma::mat& predictors, - const arma::rowvec& responses, - const arma::rowvec& weights) +template +template +inline +typename LinearRegression::ElemType +LinearRegression::Train(const MatType& predictors, + const ResponsesType& responses, + const ResponsesType& weights) { return Train(predictors, responses, weights, this->lambda, this->intercept); } -inline double LinearRegression::Train(const arma::mat& predictors, - const arma::rowvec& responses, - const arma::rowvec& weights, +template +template +inline +typename LinearRegression::ElemType +LinearRegression::Train(const MatType& predictors, + const ResponsesType& responses, + const ResponsesType& weights, const double lambda) { return Train(predictors, responses, weights, lambda, this->intercept); } -inline double LinearRegression::Train(const arma::mat& predictors, - const arma::rowvec& responses, - const arma::rowvec& weights, +template +template +inline +typename LinearRegression::ElemType +LinearRegression::Train(const MatType& predictors, + const ResponsesType& responses, + const ResponsesType& weights, const double lambda, const bool intercept) { @@ -115,15 +147,16 @@ inline double LinearRegression::Train(const arma::mat& predictors, const size_t nCols = predictors.n_cols; - arma::mat p = predictors; - arma::rowvec r = responses; + // TODO: avoid copy if possible. + MatType p = predictors; + MatType r = responses; // Here we add the row of ones to the predictors. // The intercept is not penalized. Add an "all ones" row to design and set // intercept = false to get a penalized intercept. if (intercept) { - p.insert_rows(0, arma::ones(1, nCols)); + p.insert_rows(0, arma::ones(1, nCols)); } if (weights.n_elem > 0) @@ -137,14 +170,18 @@ inline double LinearRegression::Train(const arma::mat& predictors, // Then we'll use Armadillo to solve it. // The total runtime of this should be O(d^2 N) + O(d^3) + O(dN). // (assuming the SVD is used to solve it) - arma::mat cov = p * p.t() + - lambda * arma::eye(p.n_rows, p.n_rows); + MatType cov = p * p.t() + + lambda * arma::eye(p.n_rows, p.n_rows); parameters = arma::solve(cov, p * r.t()); return ComputeError(predictors, responses); } -inline double LinearRegression::Predict(const arma::vec& point) const +template +template +inline +typename LinearRegression::ElemType +LinearRegression::Predict(const VecType& point) const { if (intercept) { @@ -170,9 +207,11 @@ inline double LinearRegression::Predict(const arma::vec& point) const } } -inline void LinearRegression::Predict( - const arma::mat& points, - arma::rowvec& predictions) const +template +template +inline void LinearRegression::Predict( + const MatType& points, + ResponsesType& predictions) const { if (intercept) { @@ -200,9 +239,12 @@ inline void LinearRegression::Predict( } } -inline double LinearRegression::ComputeError( - const arma::mat& predictors, - const arma::rowvec& responses) const +template +template +inline typename LinearRegression::ElemType +LinearRegression::ComputeError( + const MatType& predictors, + const ResponsesType& responses) const { // Sanity check on data. util::CheckSameSizes(predictors, responses, "LinearRegression::Train()"); @@ -213,7 +255,7 @@ inline double LinearRegression::ComputeError( // Calculate the differences between actual responses and predicted responses. // We must also add the intercept (parameters(0)) to the predictions. - arma::rowvec temp; + ResponsesType temp; if (intercept) { // Ensure that we have the correct number of dimensions in the dataset. @@ -223,7 +265,7 @@ inline double LinearRegression::ComputeError( "training file." << std::endl; } temp = responses - (parameters(0) + - arma::trans(parameters.subvec(1, parameters.n_elem - 1)) * predictors); + parameters.subvec(1, parameters.n_elem - 1).t() * predictors); } else { @@ -233,13 +275,34 @@ inline double LinearRegression::ComputeError( Log::Fatal << "The test data must have the same number of columns as the " "training file." << std::endl; } - temp = responses - arma::trans(parameters) * predictors; + temp = responses - parameters.t() * predictors; } - const double cost = arma::dot(temp, temp) / nCols; + const ElemType cost = dot(temp, temp) / nCols; return cost; } +template +template +void LinearRegression::serialize(Archive& ar, + const uint32_t version) +{ + if (cereal::is_loading() && version == 0) + { + // Old versions always represented `parameters` as an arma::rowvec. + arma::rowvec parametersTmp; + ar(cereal::make_nvp("parameters", parametersTmp)); + parameters = arma::conv_to::from(parametersTmp); + } + else + { + ar(CEREAL_NVP(parameters)); + } + + ar(CEREAL_NVP(lambda)); + ar(CEREAL_NVP(intercept)); +} + } // namespace mlpack #endif From 20fbb6abf0fbd5ff51b8264c2f74435ffd67afdd Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Thu, 30 Nov 2023 10:36:57 -0500 Subject: [PATCH 18/91] Adapt code to new LinearRegression<> template parameter. --- .../core/dists/regression_distribution.hpp | 6 +- .../dists/regression_distribution_impl.hpp | 4 +- .../linear_regression_main.cpp | 12 +-- .../linear_regression_predict_main.cpp | 6 +- .../linear_regression_train_main.cpp | 7 +- .../tests/bayesian_linear_regression_test.cpp | 2 +- src/mlpack/tests/cv_test.cpp | 26 +++--- src/mlpack/tests/linear_regression_test.cpp | 93 ++++++++++++------- .../linear_regression_predict_test.cpp | 4 +- .../main_tests/linear_regression_test.cpp | 4 +- .../linear_regression_train_test.cpp | 6 +- 11 files changed, 95 insertions(+), 75 deletions(-) diff --git a/src/mlpack/core/dists/regression_distribution.hpp b/src/mlpack/core/dists/regression_distribution.hpp index 252e483d6d..9a3e0b8e4a 100644 --- a/src/mlpack/core/dists/regression_distribution.hpp +++ b/src/mlpack/core/dists/regression_distribution.hpp @@ -31,7 +31,7 @@ class RegressionDistribution { private: //! Regression function for representing conditional mean. - LinearRegression rf; + LinearRegression<> rf; //! Error distribution. GaussianDistribution err; @@ -81,9 +81,9 @@ class RegressionDistribution } //! Return regression function. - const LinearRegression& Rf() const { return rf; } + const LinearRegression<>& Rf() const { return rf; } //! Modify regression function. - LinearRegression& Rf() { return rf; } + LinearRegression<>& Rf() { return rf; } //! Return error distribution. const GaussianDistribution& Err() const { return err; } diff --git a/src/mlpack/core/dists/regression_distribution_impl.hpp b/src/mlpack/core/dists/regression_distribution_impl.hpp index 0634549e8b..1aa270d5d1 100644 --- a/src/mlpack/core/dists/regression_distribution_impl.hpp +++ b/src/mlpack/core/dists/regression_distribution_impl.hpp @@ -24,7 +24,7 @@ namespace mlpack { */ inline void RegressionDistribution::Train(const arma::mat& observations) { - LinearRegression lr(observations.rows(1, observations.n_rows - 1), + LinearRegression<> lr(observations.rows(1, observations.n_rows - 1), arma::rowvec(observations.row(0)), 0, true); rf = lr; arma::rowvec fitted; @@ -46,7 +46,7 @@ inline void RegressionDistribution::Train(const arma::mat& observations, inline void RegressionDistribution::Train(const arma::mat& observations, const arma::rowvec& weights) { - LinearRegression lr(observations.rows(1, observations.n_rows - 1), + LinearRegression<> lr(observations.rows(1, observations.n_rows - 1), arma::rowvec(observations.row(0)), weights, 0, true); rf = lr; arma::rowvec fitted; diff --git a/src/mlpack/methods/linear_regression/linear_regression_main.cpp b/src/mlpack/methods/linear_regression/linear_regression_main.cpp index e59ba2543e..034f70de59 100644 --- a/src/mlpack/methods/linear_regression/linear_regression_main.cpp +++ b/src/mlpack/methods/linear_regression/linear_regression_main.cpp @@ -95,9 +95,9 @@ PARAM_ROW_IN("training_responses", "Optional vector containing y " "(responses). If not given, the responses are assumed to be the last row " "of the input file.", "r"); -PARAM_MODEL_IN(LinearRegression, "input_model", "Existing LinearRegression " +PARAM_MODEL_IN(LinearRegression<>, "input_model", "Existing LinearRegression " "model to use.", "m"); -PARAM_MODEL_OUT(LinearRegression, "output_model", "Output LinearRegression " +PARAM_MODEL_OUT(LinearRegression<>, "output_model", "Output LinearRegression " "model.", "M"); PARAM_MATRIX_IN("test", "Matrix containing X' (test regressors).", "T"); @@ -120,7 +120,7 @@ void BINDING_FUNCTION(util::Params& params, util::Timers& timer) mat regressors; rowvec responses; - LinearRegression* lr; + LinearRegression<>* lr; const bool computeModel = !params.Has("input_model"); const bool computePrediction = params.Has("test"); @@ -172,14 +172,14 @@ void BINDING_FUNCTION(util::Params& params, util::Timers& timer) } timer.Start("regression"); - lr = new LinearRegression(regressors, responses, lambda); + lr = new LinearRegression<>(regressors, responses, lambda); timer.Stop("regression"); } else { // A model file was passed in, so load it. timer.Start("load_model"); - lr = params.Get("input_model"); + lr = params.Get*>("input_model"); timer.Stop("load_model"); } @@ -221,5 +221,5 @@ void BINDING_FUNCTION(util::Params& params, util::Timers& timer) } // Save the model if needed. - params.Get("output_model") = lr; + params.Get*>("output_model") = lr; } diff --git a/src/mlpack/methods/linear_regression/linear_regression_predict_main.cpp b/src/mlpack/methods/linear_regression/linear_regression_predict_main.cpp index 19302dd60f..4c7c0adaae 100644 --- a/src/mlpack/methods/linear_regression/linear_regression_predict_main.cpp +++ b/src/mlpack/methods/linear_regression/linear_regression_predict_main.cpp @@ -37,8 +37,8 @@ BINDING_LONG_DESC(""); BINDING_EXAMPLE( CALL_METHOD("model", "predict", "test", "X_test")); -PARAM_MODEL_IN_REQ(LinearRegression, "input_model", "Existing LinearRegression " - "model to use.", "m"); +PARAM_MODEL_IN_REQ(LinearRegression<>, "input_model", "Existing " + "LinearRegression model to use.", "m"); PARAM_MATRIX_IN_REQ("test", "Matrix containing X' (test regressors).", "T"); @@ -50,7 +50,7 @@ void BINDING_FUNCTION(util::Params& params, util::Timers& timer) { // A model file was passed in, so load it. timer.Start("load_model"); - LinearRegression* lr = params.Get("input_model"); + LinearRegression<>* lr = params.Get*>("input_model"); timer.Stop("load_model"); // Cache the output of GetPrintable before we std::move() the test diff --git a/src/mlpack/methods/linear_regression/linear_regression_train_main.cpp b/src/mlpack/methods/linear_regression/linear_regression_train_main.cpp index 161f131b8d..e0b3eb2258 100644 --- a/src/mlpack/methods/linear_regression/linear_regression_train_main.cpp +++ b/src/mlpack/methods/linear_regression/linear_regression_train_main.cpp @@ -61,7 +61,7 @@ PARAM_ROW_IN("training_responses", "Optional vector containing y " "(responses). If not given, the responses are assumed to be the last row " "of the input file.", "r"); -PARAM_MODEL_OUT(LinearRegression, "output_model", "Output LinearRegression " +PARAM_MODEL_OUT(LinearRegression<>, "output_model", "Output LinearRegression " "model.", "M"); PARAM_DOUBLE_IN("lambda", "Tikhonov regularization for ridge regression. If 0," @@ -108,9 +108,10 @@ void BINDING_FUNCTION(util::Params& params, util::Timers& timer) Log::Fatal << "Regressors and Responses must have the same number of data points!" << endl; timer.Start("regression"); - LinearRegression* lr = new LinearRegression(regressors, responses, lambda); + LinearRegression<>* lr = new LinearRegression<>(regressors, responses, + lambda); timer.Stop("regression"); // Save the model if needed. - params.Get("output_model") = lr; + params.Get*>("output_model") = lr; } diff --git a/src/mlpack/tests/bayesian_linear_regression_test.cpp b/src/mlpack/tests/bayesian_linear_regression_test.cpp index 4f6bfb43c9..faed9745a3 100644 --- a/src/mlpack/tests/bayesian_linear_regression_test.cpp +++ b/src/mlpack/tests/bayesian_linear_regression_test.cpp @@ -174,7 +174,7 @@ TEST_CASE("EqualtoRidge", "[BayesianLinearRegressionTest]") BayesianLinearRegression blr(false, false); blr.Train(matX, y); - LinearRegression ridge(matX, y, blr.Alpha() / blr.Beta(), false); + LinearRegression<> ridge(matX, y, blr.Alpha() / blr.Beta(), false); blr.Predict(matX, blrPred); ridge.Predict(matX, ridgePred); diff --git a/src/mlpack/tests/cv_test.cpp b/src/mlpack/tests/cv_test.cpp index 3185046598..74ab62b822 100644 --- a/src/mlpack/tests/cv_test.cpp +++ b/src/mlpack/tests/cv_test.cpp @@ -207,7 +207,7 @@ TEST_CASE("MSETest", "[CVTest]") arma::mat trainingData("0 1"); arma::rowvec trainingResponses("-1 0"); - LinearRegression lr(trainingData, trainingResponses); + LinearRegression<> lr(trainingData, trainingResponses); // Making three responses that differ from the correct ones by 0, 1, and 2 // respectively @@ -229,7 +229,7 @@ TEST_CASE("R2ScoreTest", "[CVTest]") arma::mat trainingData("0 1"); arma::rowvec trainingResponses("-1 0"); - LinearRegression lr(trainingData, trainingResponses); + LinearRegression<> lr(trainingData, trainingResponses); // Making five responses that are the output of regression function f(x) // with some responses having a slight deviation of 0.005. @@ -256,7 +256,7 @@ TEST_CASE("AdjR2ScoreTest", "[CVTest]") arma::rowvec Y; Y = { 3, 5, 7, 9, 11, 13 }; - LinearRegression lr(X, Y); + LinearRegression<> lr(X, Y); // Theoretically Adjusted R squared should be equal 1 double expAdjR2 = 1; @@ -309,7 +309,7 @@ void CheckPredictionsType() */ TEST_CASE("PredictionsTypeTest", "[CVTest]") { - CheckPredictionsType(); + CheckPredictionsType, arma::rowvec>(); // CheckPredictionsType, arma::mat>(); CheckPredictionsType, arma::Row>(); @@ -328,7 +328,7 @@ TEST_CASE("PredictionsTypeTest", "[CVTest]") */ TEST_CASE("SupportsWeightsTest", "[CVTest]") { - static_assert(MetaInfoExtractor::SupportsWeights, + static_assert(MetaInfoExtractor>::SupportsWeights, "Value should be true"); static_assert(MetaInfoExtractor>::SupportsWeights, "Value should be true"); @@ -360,7 +360,7 @@ void CheckWeightsType() */ TEST_CASE("WeightsTypeTest", "[CVTest]") { - CheckWeightsType(); + CheckWeightsType, arma::rowvec>(); CheckWeightsType, arma::rowvec>(); CheckWeightsType, arma::Row, arma::mat, arma::Row, arma::Row>(); @@ -374,7 +374,7 @@ TEST_CASE("TakesDatasetInfoTest", "[CVTest]") { static_assert(MetaInfoExtractor>::TakesDatasetInfo, "Value should be true"); - static_assert(!MetaInfoExtractor::TakesDatasetInfo, + static_assert(!MetaInfoExtractor>::TakesDatasetInfo, "Value should be false"); static_assert(!MetaInfoExtractor::TakesDatasetInfo, "Value should be false"); @@ -390,7 +390,7 @@ TEST_CASE("TakesNumClassesTest", "[CVTest]") "Value should be true"); static_assert(MetaInfoExtractor::TakesNumClasses, "Value should be true"); - static_assert(!MetaInfoExtractor::TakesNumClasses, + static_assert(!MetaInfoExtractor>::TakesNumClasses, "Value should be false"); static_assert(!MetaInfoExtractor::TakesNumClasses, "Value should be false"); @@ -425,7 +425,7 @@ TEST_CASE("SimpleCVMSETest", "[CVTest]") double expectedMSE = (0 * 0 + 1 * 1 + 2 * 2) / 3.0; - SimpleCV cv(0.6, data, responses); + SimpleCV, MSE> cv(0.6, data, responses); REQUIRE(cv.Evaluate() == Approx(expectedMSE).epsilon(1e-7)); @@ -438,7 +438,7 @@ TEST_CASE("SimpleCVMSETest", "[CVTest]") arma::rowvec weights = arma::join_rows(arma::zeros(noiseData.n_cols).t(), arma::ones(data.n_cols).t()); - SimpleCV weightedCV(0.3, allData, allResponces, + SimpleCV, MSE> weightedCV(0.3, allData, allResponces, weights); REQUIRE(weightedCV.Evaluate() == Approx(expectedMSE).epsilon(1e-7)); @@ -446,7 +446,7 @@ TEST_CASE("SimpleCVMSETest", "[CVTest]") arma::rowvec weights2 = arma::join_rows(arma::zeros(noiseData.n_cols - 1).t(), arma::ones(data.n_cols + 1).t()); - SimpleCV weightedCV2(0.3, allData, allResponces, + SimpleCV, MSE> weightedCV2(0.3, allData, allResponces, weights2); REQUIRE(std::abs(weightedCV2.Evaluate() - expectedMSE) > 1e-5); @@ -543,7 +543,7 @@ TEST_CASE("KFoldCVMSETest", "[CVTest]") arma::rowvec responses("0 1 1 3"); // 2-fold cross-validation, no shuffling. - KFoldCV cv(2, data, responses, false); + KFoldCV, MSE> cv(2, data, responses, false); // In each of two validation tests the MSE value should be the same. double expectedMSE = @@ -620,7 +620,7 @@ TEST_CASE("KFoldCVWithWeightedLRTest", "[CVTest]") arma::rowvec responses("1 2 30 40"); arma::rowvec weights("1 1 0 0"); - KFoldCV cv(2, arma::join_rows(data, data), + KFoldCV, MSE> cv(2, arma::join_rows(data, data), arma::join_rows(responses, responses), arma::join_rows(weights, weights), false); cv.Evaluate(); diff --git a/src/mlpack/tests/linear_regression_test.cpp b/src/mlpack/tests/linear_regression_test.cpp index 1dad701986..fd8e210ad7 100644 --- a/src/mlpack/tests/linear_regression_test.cpp +++ b/src/mlpack/tests/linear_regression_test.cpp @@ -21,25 +21,30 @@ using namespace mlpack; * Creates two 10x3 random matrices and one 10x1 "results" matrix. * Finds B in y=BX with one matrix, then predicts against the other. */ -TEST_CASE("LinearRegressionTestCase", "[LinearRegressionTest]") +TEMPLATE_TEST_CASE("LinearRegressionTestCase", "[LinearRegressionTest]", + arma::fmat, arma::mat) { + typedef TestType MatType; + typedef arma::Row RowType; + typedef arma::Col ColType; + // Predictors and points are 10x3 matrices. - arma::mat predictors(3, 10); - arma::mat points(3, 10); + MatType predictors(3, 10); + MatType points(3, 10); // Responses is the "correct" value for each point in predictors and points. - arma::rowvec responses(10); + RowType responses(10); // The values we get back when we predict for points. - arma::rowvec predictions(10); + RowType predictions(10); // We'll randomly select some coefficients for the linear response. - arma::vec coeffs; + ColType coeffs; coeffs.randu(4); // Now generate each point. for (size_t row = 0; row < 3; row++) - predictors.row(row) = arma::linspace(0, 9, 10); + predictors.row(row) = arma::linspace(0, 9, 10); points = predictors; @@ -57,7 +62,7 @@ TEST_CASE("LinearRegressionTestCase", "[LinearRegressionTest]") dot(coeffs.rows(1, 3), arma::ones(3) * elem); // Initialize and predict. - LinearRegression lr(predictors, responses); + LinearRegression lr(predictors, responses); lr.Predict(points, predictions); // Output result and verify we have less than 5% error from "correct" value @@ -69,16 +74,20 @@ TEST_CASE("LinearRegressionTestCase", "[LinearRegressionTest]") /** * Check the functionality of ComputeError(). */ -TEST_CASE("ComputeErrorTest", "[LinearRegressionTest]") +TEMPLATE_TEST_CASE("ComputeErrorTest", "[LinearRegressionTest]", arma::fmat, + arma::mat) { - arma::mat predictors; + typedef TestType MatType; + typedef arma::Row RowType; + + MatType predictors; predictors = { { 0, 1, 2, 4, 8, 16 }, { 16, 8, 4, 2, 1, 0 } }; - arma::rowvec responses = "0 2 4 3 8 8"; + RowType responses = "0 2 4 3 8 8"; // http://www.mlpack.org/trac/ticket/298 // This dataset gives a cost of 1.189500337 (as calculated in Octave). - LinearRegression lr(predictors, responses); + LinearRegression lr(predictors, responses); REQUIRE(lr.ComputeError(predictors, responses) == Approx(1.189500337).epsilon(1e-5)); @@ -95,7 +104,7 @@ TEST_CASE("ComputeErrorPerfectFitTest", "[LinearRegressionTest]") { 0, 1, 2, 2, 2, 6 } }; arma::rowvec responses = "0 2 4 3 8 8"; - LinearRegression lr(predictors, responses); + LinearRegression<> lr(predictors, responses); REQUIRE(lr.ComputeError(predictors, responses) == Approx(0.0).margin(1e-25)); } @@ -116,7 +125,7 @@ TEST_CASE("RidgeRegressionTest", "[LinearRegressionTest]") // invertible. If ridge regression is not working correctly, then the matrix // will not be invertible and the test should segfault (or something else // ugly). - LinearRegression lr(data, responses, 0.0001); + LinearRegression<> lr(data, responses, 0.0001); // Now just make sure that it predicts some more zeros. arma::rowvec predictedResponses; @@ -167,7 +176,7 @@ TEST_CASE("RidgeRegressionTestCase", "[LinearRegressionTest]") dot(coeffs.rows(1, 3), arma::ones(3) * elem); // Initialize and predict with very small lambda. - LinearRegression lr(predictors, responses, 0.001); + LinearRegression<> lr(predictors, responses, 0.001); lr.Predict(points, predictions); // Output result and verify we have less than 5% error from "correct" value @@ -186,8 +195,8 @@ TEST_CASE("LinearRegressionTrainTest", "[LinearRegressionTest]") arma::mat dataset = arma::randu(5, 1000); arma::rowvec responses = arma::randu(1000); - LinearRegression lr(dataset, responses, 0.3); - LinearRegression lrTrain; + LinearRegression<> lr(dataset, responses, 0.3); + LinearRegression<> lrTrain; lrTrain.Lambda() = 0.3; lrTrain.Train(dataset, responses); @@ -209,8 +218,8 @@ TEST_CASE("LinearRegressionTest", "[LinearRegressionTest]") arma::rowvec responses; responses.randn(800); - LinearRegression lr(data, responses, 0.05); // Train the model. - LinearRegression xmlLr, jsonLr, binaryLr; + LinearRegression<> lr(data, responses, 0.05); // Train the model. + LinearRegression<> xmlLr, jsonLr, binaryLr; SerializeObjectAll(lr, xmlLr, jsonLr, binaryLr); @@ -260,7 +269,7 @@ TEST_CASE("LinearRegressionTrainReturnObjective", "[LinearRegressionTest]") dot(coeffs.rows(1, 3), arma::ones(3) * elem); // Initialize and predict. - LinearRegression lr; + LinearRegression<> lr; double error = lr.Train(predictors, responses); REQUIRE(std::isfinite(error) == true); @@ -269,24 +278,28 @@ TEST_CASE("LinearRegressionTrainReturnObjective", "[LinearRegressionTest]") /** * Make sure all versions of Train() work correctly. */ -TEST_CASE("LinearRegressionAllTrainVersionsTest", "[LinearRegressionTest]") +TEMPLATE_TEST_CASE("LinearRegressionAllTrainVersionsTest", + "[LinearRegressionTest]", arma::fmat, arma::mat) { + typedef TestType MatType; + typedef arma::Row RowType; + // The data doesn't really matter for this test; mostly we want to make sure // that all the Train() variants work properly. - arma::mat predictors; + MatType predictors; predictors = { { 0, 1, 2, 4, 8, 16 }, { 16, 8, 4, 2, 1, 0 } }; - arma::rowvec responses = "0 2 4 3 8 8"; - arma::rowvec weights = "1.0 1.1 1.2 0.8 0.9 1.0"; + RowType responses = "0 2 4 3 8 8"; + RowType weights = "1.0 1.1 1.2 0.8 0.9 1.0"; - LinearRegression lr1, lr2, lr3, lr4, lr5, lr6; + LinearRegression lr1, lr2, lr3, lr4, lr5, lr6; - lr1.Train(predictors, responses); - lr2.Train(predictors, responses, 0.1); - lr3.Train(predictors, responses, 0.2, false); - lr4.Train(predictors, responses, weights); - lr5.Train(predictors, responses, weights, 0.3); - lr6.Train(predictors, responses, weights, 0.4, false); + (void) lr1.Train(predictors, responses); + (void) lr2.Train(predictors, responses, 0.1); + (void) lr3.Train(predictors, responses, 0.2, false); + (void) lr4.Train(predictors, responses, weights); + (void) lr5.Train(predictors, responses, weights, 0.3); + (void) lr6.Train(predictors, responses, weights, 0.4, false); // We don't care about the specifics of the trained model, but we want to just // make sure everything appears to be correct from the sizes and @@ -325,17 +338,21 @@ TEST_CASE("LinearRegressionAllTrainVersionsTest", "[LinearRegressionTest]") * Ensure that single-point Predict() returns the same results as multi-point * Predict(). */ -TEST_CASE("LinearRegressionSinglePointPredictTest", "[LinearRegressionTest]") +TEMPLATE_TEST_CASE("LinearRegressionSinglePointPredictTest", + "[LinearRegressionTest]", arma::fmat, arma::mat) { - arma::mat predictors; + typedef TestType MatType; + typedef arma::Row RowType; + + MatType predictors; predictors = { { 0, 1, 2, 4, 8, 16 }, { 16, 8, 4, 2, 1, 0 } }; - arma::rowvec responses = "0 2 4 3 8 8"; + RowType responses = "0 2 4 3 8 8"; - LinearRegression lr(predictors, responses, 0.1, true); + LinearRegression lr(predictors, responses, 0.1, true); // Compute predictions for test points in batch. - arma::rowvec predictions; + RowType predictions; lr.Predict(predictors, predictions); // Now compute each prediction individually. @@ -345,3 +362,7 @@ TEST_CASE("LinearRegressionSinglePointPredictTest", "[LinearRegressionTest]") REQUIRE(prediction == Approx(predictions[i])); } } + +// Make sure training on submatrices and subvectors works. + +// Make sure we can train on sparse data. diff --git a/src/mlpack/tests/main_tests/linear_regression_predict_test.cpp b/src/mlpack/tests/main_tests/linear_regression_predict_test.cpp index fff634d9a6..e8a0b3c961 100644 --- a/src/mlpack/tests/main_tests/linear_regression_predict_test.cpp +++ b/src/mlpack/tests/main_tests/linear_regression_predict_test.cpp @@ -37,7 +37,7 @@ TEST_CASE_METHOD(LRPredictTestFixture, "LRPredictWrongDimOfDataTest1t", arma::rowvec trainY = arma::randu(N); arma::mat testX = arma::randu(D - 1, M); // Wrong dimensionality. - LinearRegression* model = new LinearRegression(); + LinearRegression<>* model = new LinearRegression<>(); model->Train(trainX, trainY); SetInputParam("input_model", std::move(model)); @@ -60,7 +60,7 @@ TEST_CASE_METHOD(LRPredictTestFixture, "LRPredictPredictionSizeCheck", arma::rowvec trainY = arma::randu(N); arma::mat testX = arma::randu(D, M); - LinearRegression* model = new LinearRegression(); + LinearRegression<>* model = new LinearRegression<>(); model->Train(trainX, trainY); SetInputParam("input_model", std::move(model)); diff --git a/src/mlpack/tests/main_tests/linear_regression_test.cpp b/src/mlpack/tests/main_tests/linear_regression_test.cpp index 92edf0c8e5..542a59993f 100644 --- a/src/mlpack/tests/main_tests/linear_regression_test.cpp +++ b/src/mlpack/tests/main_tests/linear_regression_test.cpp @@ -118,7 +118,7 @@ TEST_CASE_METHOD(LRTestFixture, "LRModelReload", RUN_BINDING(); - LinearRegression* model = params.Get("output_model"); + LinearRegression<>* model = params.Get*>("output_model"); const arma::rowvec testY1 = params.Get("output_predictions"); ResetSettings(); @@ -191,7 +191,7 @@ TEST_CASE_METHOD(LRTestFixture, "LRWrongDimOfDataTest2", RUN_BINDING(); - LinearRegression* model = params.Get("output_model"); + LinearRegression<>* model = params.Get*>("output_model"); ResetSettings(); diff --git a/src/mlpack/tests/main_tests/linear_regression_train_test.cpp b/src/mlpack/tests/main_tests/linear_regression_train_test.cpp index fe3b435b2b..88c49ec68a 100644 --- a/src/mlpack/tests/main_tests/linear_regression_train_test.cpp +++ b/src/mlpack/tests/main_tests/linear_regression_train_test.cpp @@ -44,8 +44,7 @@ TEST_CASE_METHOD(LRFitTestFixture, "LRFitDifferentLambdas", // The first solution. RUN_BINDING(); arma::rowvec preds1; - params.Get("output_model")->Predict(testX, - preds1); + params.Get*>("output_model")->Predict(testX, preds1); const double testY1 = preds1(0); ResetSettings(); @@ -57,8 +56,7 @@ TEST_CASE_METHOD(LRFitTestFixture, "LRFitDifferentLambdas", // The second solution. RUN_BINDING(); arma::rowvec preds2; - params.Get("output_model")->Predict(testX, - preds2); + params.Get*>("output_model")->Predict(testX, preds2); const double testY2 = preds2(0); // Second solution has stronger regularization, From b64797a05e5257f1cff3090881da25205fb131f0 Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Thu, 30 Nov 2023 10:37:28 -0500 Subject: [PATCH 19/91] Fix LinearRegression for MetaInfoExtractor. --- src/mlpack/core/util/arma_traits.hpp | 20 ++++ .../linear_regression/linear_regression.hpp | 108 +++++++++++++++--- .../linear_regression_impl.hpp | 79 +++++++++---- 3 files changed, 169 insertions(+), 38 deletions(-) diff --git a/src/mlpack/core/util/arma_traits.hpp b/src/mlpack/core/util/arma_traits.hpp index a755ce9efa..bf8f8109e0 100644 --- a/src/mlpack/core/util/arma_traits.hpp +++ b/src/mlpack/core/util/arma_traits.hpp @@ -111,4 +111,24 @@ struct IsVector > #endif +// Get the column vector type corresponding to a given MatType. + +template +struct GetColType +{ + typedef MatType type; // Not sure... +}; + +template +struct GetColType> +{ + typedef arma::Col type; +}; + +template +struct GetColType> +{ + typedef arma::SpCol type; +}; + #endif diff --git a/src/mlpack/methods/linear_regression/linear_regression.hpp b/src/mlpack/methods/linear_regression/linear_regression.hpp index b56f379471..8a875aaf6a 100644 --- a/src/mlpack/methods/linear_regression/linear_regression.hpp +++ b/src/mlpack/methods/linear_regression/linear_regression.hpp @@ -30,6 +30,7 @@ template class LinearRegression { public: + typedef typename GetColType::type ModelColType; typedef typename ModelMatType::elem_type ElemType; /** @@ -40,7 +41,11 @@ class LinearRegression * @param lambda Regularization constant for ridge regression. * @param intercept Whether or not to include an intercept term. */ - template + template::value + >::type> LinearRegression(const MatType& predictors, const ResponsesType& responses, const double lambda = 0, @@ -55,10 +60,18 @@ class LinearRegression * @param lambda Regularization constant for ridge regression. * @param intercept Whether or not to include an intercept term. */ - template + template::value + >::type, + typename = typename std::enable_if< + std::is_same::value + >::type> LinearRegression(const MatType& predictors, const ResponsesType& responses, - const ResponsesType& weights, + const WeightsType& weights, const double lambda = 0, const bool intercept = true); @@ -113,6 +126,15 @@ class LinearRegression const arma::rowvec& weights, const bool intercept); + /** + * Train the LinearRegression model. This is a dummy overload so that + * MetaInfoExtractor can properly detect that LinearRegression is a regression + * method. + */ + template + ElemType Train(const MatType& predictors, + const arma::rowvec& responses); + /** * Train the LinearRegression model on the given data and weights. Careful! * This will completely ignore and overwrite the existing model. This @@ -124,7 +146,15 @@ class LinearRegression * @param responses y, the responses to the data points. * @return The least squares error after training. */ - template + template::value + >::type, + typename = typename std::enable_if< + !std::is_same::value + >::type> ElemType Train(const MatType& predictors, const ResponsesType& responses); @@ -140,7 +170,12 @@ class LinearRegression * @param lambda L2 regularization penalty parameter to use. * @return The least squares error after training. */ - template + template::value + >::type> ElemType Train(const MatType& predictors, const ResponsesType& responses, const double lambda); @@ -158,12 +193,27 @@ class LinearRegression * @param intercept Whether or not to fit an intercept term. * @return The least squares error after training. */ - template + template::value + >::type> ElemType Train(const MatType& predictors, const ResponsesType& responses, const double lambda, const bool intercept); + /** + * Train the LinearRegression model. This is a dummy overload so that + * MetaInfoExtractor can properly detect that LinearRegression is a regression + * method. + */ + template + ElemType Train(const MatType& predictors, + const arma::rowvec& responses, + const arma::rowvec& weights); + /** * Train the LinearRegression model on the given data and weights. Careful! * This will completely ignore and overwrite the existing model. This @@ -176,10 +226,22 @@ class LinearRegression * @param weights Observation weights (for boosting). * @return The least squares error after training. */ - template + template::value + >::type, + typename = typename std::enable_if< + !std::is_same::value || + !std::is_same::value + >::type, + typename = typename std::enable_if< + std::is_same::value + >::type> ElemType Train(const MatType& predictors, const ResponsesType& responses, - const ResponsesType& weights); + const WeightsType& weights); /** * Train the LinearRegression model on the given data and weights. Careful! @@ -194,10 +256,18 @@ class LinearRegression * @param lambda L2 regularization penalty parameter to use. * @return The least squares error after training. */ - template + template::value + >::type, + typename = typename std::enable_if< + std::is_same::value + >::type> ElemType Train(const MatType& predictors, const ResponsesType& responses, - const ResponsesType& weights, + const WeightsType& weights, const double lambda); /** @@ -214,10 +284,18 @@ class LinearRegression * @param intercept Whether or not to fit an intercept term. * @return The least squares error after training. */ - template + template::value + >::type, + typename = typename std::enable_if< + std::is_same::value + >::type> ElemType Train(const MatType& predictors, const ResponsesType& responses, - const ResponsesType& weights, + const WeightsType& weights, const double lambda, const bool intercept); @@ -260,9 +338,9 @@ class LinearRegression const ResponsesType& responses) const; //! Return the parameters (the b vector). - const ModelMatType& Parameters() const { return parameters; } + const ModelColType& Parameters() const { return parameters; } //! Modify the parameters (the b vector). - ModelMatType& Parameters() { return parameters; } + ModelColType& Parameters() { return parameters; } //! Return the Tikhonov regularization parameter for ridge regression. double Lambda() const { return lambda; } @@ -283,7 +361,7 @@ class LinearRegression * The calculated B. * Initialized and filled by constructor to hold the least squares solution. */ - ModelMatType parameters; + ModelColType parameters; /** * The Tikhonov regularization parameter for ridge regression (0 for linear diff --git a/src/mlpack/methods/linear_regression/linear_regression_impl.hpp b/src/mlpack/methods/linear_regression/linear_regression_impl.hpp index 14180895fc..9fa7db57c8 100644 --- a/src/mlpack/methods/linear_regression/linear_regression_impl.hpp +++ b/src/mlpack/methods/linear_regression/linear_regression_impl.hpp @@ -18,7 +18,7 @@ namespace mlpack { template -template +template inline LinearRegression::LinearRegression( const MatType& predictors, const ResponsesType& responses, @@ -28,11 +28,14 @@ inline LinearRegression::LinearRegression( { /* Nothing to do. */ } template -template +template inline LinearRegression::LinearRegression( const MatType& predictors, const ResponsesType& responses, - const ResponsesType& weights, + const WeightsType& weights, const double lambda, const bool intercept) : lambda(lambda), @@ -63,7 +66,18 @@ inline double LinearRegression::Train( } template -template +template +inline +typename LinearRegression::ElemType +LinearRegression::Train(const MatType& predictors, + const arma::rowvec& responses) +{ + return Train(predictors, responses, arma::rowvec(), this->lambda, + this->intercept); +} + +template +template inline typename LinearRegression::ElemType LinearRegression::Train(const MatType& predictors, @@ -74,7 +88,7 @@ LinearRegression::Train(const MatType& predictors, } template -template +template inline typename LinearRegression::ElemType LinearRegression::Train(const MatType& predictors, @@ -85,7 +99,7 @@ LinearRegression::Train(const MatType& predictors, } template -template +template inline typename LinearRegression::ElemType LinearRegression::Train(const MatType& predictors, @@ -97,35 +111,55 @@ LinearRegression::Train(const MatType& predictors, } template -template +template inline typename LinearRegression::ElemType LinearRegression::Train(const MatType& predictors, - const ResponsesType& responses, - const ResponsesType& weights) + const arma::rowvec& responses, + const arma::rowvec& weights) { return Train(predictors, responses, weights, this->lambda, this->intercept); } template -template +template inline typename LinearRegression::ElemType LinearRegression::Train(const MatType& predictors, const ResponsesType& responses, - const ResponsesType& weights, + const WeightsType& weights) +{ + return Train(predictors, responses, weights, this->lambda, this->intercept); +} + +template +template +inline +typename LinearRegression::ElemType +LinearRegression::Train(const MatType& predictors, + const ResponsesType& responses, + const WeightsType& weights, const double lambda) { return Train(predictors, responses, weights, lambda, this->intercept); } template -template +template inline typename LinearRegression::ElemType LinearRegression::Train(const MatType& predictors, const ResponsesType& responses, - const ResponsesType& weights, + const WeightsType& weights, const double lambda, const bool intercept) { @@ -148,15 +182,15 @@ LinearRegression::Train(const MatType& predictors, const size_t nCols = predictors.n_cols; // TODO: avoid copy if possible. - MatType p = predictors; - MatType r = responses; + arma::Mat p = predictors; + arma::Row r = responses; // Here we add the row of ones to the predictors. // The intercept is not penalized. Add an "all ones" row to design and set // intercept = false to get a penalized intercept. if (intercept) { - p.insert_rows(0, arma::ones(1, nCols)); + p.insert_rows(0, arma::ones>(1, nCols)); } if (weights.n_elem > 0) @@ -170,8 +204,8 @@ LinearRegression::Train(const MatType& predictors, // Then we'll use Armadillo to solve it. // The total runtime of this should be O(d^2 N) + O(d^3) + O(dN). // (assuming the SVD is used to solve it) - MatType cov = p * p.t() + - lambda * arma::eye(p.n_rows, p.n_rows); + arma::Mat cov = p * p.t() + + ((ElemType) lambda) * arma::eye>(p.n_rows, p.n_rows); parameters = arma::solve(cov, p * r.t()); return ComputeError(predictors, responses); @@ -224,8 +258,7 @@ inline void LinearRegression::Predict( "points"); // Get the predictions, but this ignores the intercept value // (parameters[0]). - predictions = arma::trans(parameters.subvec(1, parameters.n_elem - 1)) - * points; + predictions = parameters.subvec(1, parameters.n_elem - 1).t() * points; // Now add the intercept. predictions += parameters(0); } @@ -289,10 +322,10 @@ void LinearRegression::serialize(Archive& ar, { if (cereal::is_loading() && version == 0) { - // Old versions always represented `parameters` as an arma::rowvec. - arma::rowvec parametersTmp; + // Old versions represented `parameters` as an arma::vec. + arma::vec parametersTmp; ar(cereal::make_nvp("parameters", parametersTmp)); - parameters = arma::conv_to::from(parametersTmp); + parameters = arma::conv_to::from(parametersTmp); } else { From 30dd6fb3ff6f09373b0285974cb80d42809c325b Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Thu, 30 Nov 2023 11:17:49 -0500 Subject: [PATCH 20/91] Add linear regression documentation (fully tested). --- doc/user/methods/linear_regression.md | 285 ++++++++++++++++++++++++++ 1 file changed, 285 insertions(+) create mode 100644 doc/user/methods/linear_regression.md diff --git a/doc/user/methods/linear_regression.md b/doc/user/methods/linear_regression.md new file mode 100644 index 0000000000..57d6a02595 --- /dev/null +++ b/doc/user/methods/linear_regression.md @@ -0,0 +1,285 @@ +## `LinearRegression` + +The `LinearRegression` class implements a standard L2-regularized linear +regression model for numerical data, trained by direct decomposition of the +training data. The class offers configurable functionality and template +parameters to control the data type used for storing the model. + +#### Simple usage example: + +```c++ +// Train a linear regression model on random numeric data and make predictions. + +// All data and responses are uniform random; this uses 10 dimensional data. +// Replace with a data::Load() call or similar for a real application. +arma::mat dataset(10, 1000, arma::fill::randu); // 1000 points. +arma::rowvec responses = arma::randn(1000); +arma::mat testDataset(10, 500, arma::fill::randu); // 500 test points. + +mlpack::LinearRegression lr; // Step 1: create model. +lr.Train(dataset, responses); // Step 2: train model. +arma::rowvec predictions; +lr.Predict(testDataset, predictions); // Step 3: use model to predict. + +// Print some information about the test predictions. +std::cout << arma::accu(predictions > 0.7) << " test points predicted to have" + << " responses greater than 0.7." << std::endl; +std::cout << arma::accu(predictions < 0) << " test points predicted to have " + << "negative responses." << std::endl; +``` +

More examples...

+ +#### Quick links: + + * [Constructors](#constructors): create `LinearRegression` objects. + * [`Train()`](#training): train model. + * [`Predict()`](#prediction): predict with a trained model. + * [Other functionality](#other-functionality) for loading, saving, and + inspecting. + * [Examples](#simple-examples) of simple usage and links to detailed example + projects. + * [Template parameters](#advanced-functionality-different-element-types) for + using different element types for a model. + +#### See also: + + * [mlpack regression techniques](#mlpack_regression_techniques) + * [Linear Regression on Wikipedia](https://en.wikipedia.org/wiki/Linear_regression) + +### Constructors + + * `lr = LinearRegression()` + - Initialize the model without training. + - You will need to call [`Train()`](#training) later to train the model + before calling [`Predict()`](#prediction). + +--- + + * `lr = LinearRegression(data, responses, lambda=0.0, intercept=true)` + * `lr = LinearRegression(data, responses, weights, lambda=0.0, intercept=true)` + - Train model, optionally with instance weights. + +--- + +#### Constructor Parameters: + + + +| **name** | **type** | **description** | **default** | +|----------|----------|-----------------|-------------| +| `data` | [`arma::mat`](../matrices.md) | [Column-major](../matrices.md) training matrix. | _(N/A)_ | +| `responses` | [`arma::rowvec`](../matrices.md) | Training responses (e.g. values to predict). Should have length `data.n_cols`. | _(N/A)_ | +| `weights` | [`arma::rowvec`](../matrices.md) | Weights for each training point. Should have length `data.n_cols`. | _(N/A)_ | +| `lambda` | `double` | L2 regularization penalty parameter. | `0.0` | +| `intercept` | `bool` | Whether to fit an intercept term in the model. | `bool` | + +As an alternative to passing `lambda`, it can be set with the standalone +`Lambda()` method: `lr.Lambda() = l;` will set the value of `lambda` to `l` for +the next time `Train()` is called. + +***Note***: setting `lambda` too small may cause the model to overfit; however, +setting it too large may cause the model to underfit. [Automatic hyperparameter +tuning](#hyperparameter-tuner) can be used to find a good value of `lambda` +instead of a manual setting. + +### Training + +If training is not done as part of the constructor call, it can be done with the +`Train()` function: + + + * `lr.Train(data, responses, lambda=0.0, intercept=true)` + * `lr.Train(data, responses, weights, lambda=0.0, intercept=true)` + - Train model on the given data, optionally with instance weights. + +--- + +Types of each argument are the same as in the table for constructors +[above](#constructor-parameters). + +***Notes:*** + + * Training is not incremental. A second call to `Train()` will retrain the + model from scratch. + + * `Train()` returns the mean squared error (MSE) of the model on the training + set as a `double`. + +### Prediction + +Once a `LinearRegression` model is trained, the `Predict()` member function +can be used to make predictions for new data. + + * `double predictedValue = lr.Predict(point)` + - ***(Single-point)*** + - Make a prediction for a single point, returning the predicted value. + +--- + + * `lr.Predict(data, predictions)` + - ***(Multi-point)*** + - Make predictions for a set of points. + - The prediction for data point `i` can be accessed with `predictions[i]`. + +--- + +#### Prediction Parameters: + +| **usage** | **name** | **type** | **description** | +|-----------|----------|----------|-----------------| +| _single-point_ | `point` | [`arma::vec`](../matrices.md) | Single point for prediction. | +|||| +| _multi-point_ | `data` | [`arma::mat`](../matrices.md) | Set of [column-major](../matrices.md) points for classification. | +| _multi-point_ | `predictions` | [`arma::rowvec&`](../matrices.md) | Vector of `double`s to store predictions into. Will be set to length `data.n_cols`. | + +### Other Functionality + + + + * A `LinearRegression` model can be serialized with + [`data::Save()`](../formats.md) and [`data::Load()`](../formats.md). + + * `lr.Intercept()` will return a `bool` indicating whether the model was + trained with an intercept term. + + * `lr.Parameters()` will return an `arma::vec&` with the model parameters. + This will have length equal to the dimensionality of the model if + `lr.Intercept()` is `false`, and length equal to the dimensionality of the + model plus one if `lr.Intercept()` is `true`. If an intercept was fitted, + the intercept term is the first element of `lr.Parameters()`. + + * `lr.ComputeError(data, responses)` will return a `double` containing the mean + squared error (MSE) of the model on `data`, given that the true responses are + `responses`. + +### Simple Examples + +See also the [simple usage example](#simple-usage-example) for a trivial usage +of the `LinearRegression` class. + +--- + +Train a linear regression model in the constructor on weighted data, compute the +objective function with `ComputeError()`, and save the model. + +```c++ +// See https://datasets.mlpack.org/admission_predict.csv. +arma::mat data; +mlpack::data::Load("admission_predict.csv", data, true); + +// See https://datasets.mlpack.org/admission_predict.responses.csv. +arma::rowvec responses; +mlpack::data::Load("admission_predict.responses.csv", responses, true); + +// Generate random instance weights for each point, in the range 0.5 to 1.5. +arma::rowvec weights(data.n_cols, arma::fill::randu); +weights += 0.5; + +// Train a linear regression model, fitting an intercept term and using an L2 +// regularization parameter of 0.3. +mlpack::LinearRegression lr(data, responses, weights, 0.3, true); + +// Now compute the MSE on the training set. +std::cout << "MSE on the training set: " << lr.ComputeError(data, responses) + << "." << std::endl; + +// Finally, save the model with the name "lr". +mlpack::data::Save("lr_model.bin", "lr", lr, true); +``` + +--- + +Load a saved linear regression model and print some information about it, then +make some predictions individually for random points. + +``` +mlpack::LinearRegression lr; + +// Load the model named "lr" from "lr_model.bin". +mlpack::data::Load("lr_model.bin", "lr", lr, true); + +// Print some information about the model. +const size_t dimensionality = + (lr.Intercept() ? (lr.Parameters().n_elem - 1) : lr.Parameters().n_elem); + +std::cout << "Information on the LinearRegression model in 'lr_model.bin':" + << std::endl; +std::cout << " - Model has intercept: " + << (lr.Intercept() ? std::string("yes") : std::string("no")) << "." + << std::endl; +if (lr.Intercept()) +{ + std::cout << " - Intercept weight: " << lr.Parameters()[0] << "." + << std::endl; +} +std::cout << " - Model dimensionality: " << dimensionality << "." << std::endl; +std::cout << " - Lambda value: " << lr.Lambda() << "." << std::endl; +std::cout << std::endl; + +// Now make a prediction for three random points. +for (size_t t = 0; t < 3; ++t) +{ + arma::vec randomPoint(dimensionality, arma::fill::randu); + const double prediction = lr.Predict(randomPoint); + + std::cout << "Prediction for random point " << t << ": " << prediction << "." + << std::endl; +} +``` + +--- + +### Advanced Functionality: Different Element Types + +The `LinearRegression` class has one template parameter that can be used to +control the element type of the model. The full signature of the class is: + +```c++ +LinearRegression +``` + +`ModelMatType` specifies the type of matrix used for the internal representation +of model parameters. Any matrix type that implements the Armadillo API can be +used. + +Note that the `Train()` and `Predict()` functions themselves are templatized and +can allow any matrix type that has the same element type. So, for instance, a +`LinearRegression` can accept an `arma::sp_mat` for training. + +The example below trains a linear regression model on sparse 32-bit floating +point data, but uses a dense 32-bit floating point vector to store the model +itself. + +```c++ +// Create random, sparse 100-dimensional data. +arma::sp_fmat dataset; +dataset.sprandu(100, 5000, 0.3); + +// Generate noisy responses from random data. +arma::fvec trueWeights(100, arma::fill::randu); +arma::frowvec responses = trueWeights.t() * dataset + + 0.01 * arma::randu(5000) /* noise term */; + +mlpack::LinearRegression lr; +lr.Lambda() = 0.01; + +lr.Train(dataset, responses); + +// Compute the MSE on the training set and a random test set. +arma::sp_fmat testDataset; +testDataset.sprandu(100, 1000, 0.3); + +arma::frowvec testResponses = trueWeights.t() * testDataset + + 0.01 * arma::randu(1000) /* noise term */; + +std::cout << "MSE on training set: " + << lr.ComputeError(dataset, responses) << "." << std::endl; +std::cout << "MSE on test set: " + << lr.ComputeError(testDataset, testResponses) << "." << std::endl; +``` + +***Note:*** dense objects should be used for `ModelMatType`, since in general an +L2-regularized linear regression model will not be sparse. From 5de75373770f289c91966e4d6978cdc4e2d61307 Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Thu, 30 Nov 2023 11:17:58 -0500 Subject: [PATCH 21/91] Handle sparse inputs a little better. --- src/mlpack/methods/linear_regression/linear_regression_impl.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mlpack/methods/linear_regression/linear_regression_impl.hpp b/src/mlpack/methods/linear_regression/linear_regression_impl.hpp index 9fa7db57c8..0952c04fbc 100644 --- a/src/mlpack/methods/linear_regression/linear_regression_impl.hpp +++ b/src/mlpack/methods/linear_regression/linear_regression_impl.hpp @@ -182,7 +182,7 @@ LinearRegression::Train(const MatType& predictors, const size_t nCols = predictors.n_cols; // TODO: avoid copy if possible. - arma::Mat p = predictors; + arma::Mat p = arma::conv_to>::from(predictors); arma::Row r = responses; // Here we add the row of ones to the predictors. From abae46cbffbcab4ec89ea766cfdf00d52dacc524 Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Thu, 30 Nov 2023 11:31:11 -0500 Subject: [PATCH 22/91] Remove TODO that is fixed. --- doc/user/methods/linear_regression.md | 1 - 1 file changed, 1 deletion(-) diff --git a/doc/user/methods/linear_regression.md b/doc/user/methods/linear_regression.md index 57d6a02595..583d127062 100644 --- a/doc/user/methods/linear_regression.md +++ b/doc/user/methods/linear_regression.md @@ -90,7 +90,6 @@ instead of a manual setting. If training is not done as part of the constructor call, it can be done with the `Train()` function: - * `lr.Train(data, responses, lambda=0.0, intercept=true)` * `lr.Train(data, responses, weights, lambda=0.0, intercept=true)` - Train model on the given data, optionally with instance weights. From 3ef93bc29b11ff25654ae9abf5d255814068cd28 Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Thu, 30 Nov 2023 12:57:36 -0500 Subject: [PATCH 23/91] Add first step towards LARS documentation. --- doc/user/methods/lars.md | 136 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 136 insertions(+) create mode 100644 doc/user/methods/lars.md diff --git a/doc/user/methods/lars.md b/doc/user/methods/lars.md new file mode 100644 index 0000000000..a4828acaa7 --- /dev/null +++ b/doc/user/methods/lars.md @@ -0,0 +1,136 @@ +## `LARS` + + +#### Simple usage example: + +```c++ +``` +

More examples...

+ +#### Quick links: + + * [Constructors](#constructors): create `LinearRegression` objects. + * [`Train()`](#training): train model. + * [`Predict()`](#prediction): predict with a trained model. + * [Other functionality](#other-functionality) for loading, saving, and + inspecting. + * [Examples](#simple-examples) of simple usage and links to detailed example + projects. + * [Template parameters](#advanced-functionality-different-element-types) for + using different element types for a model. + +#### See also: + + * [`LinearRegression`](#linear_regression) + * [mlpack regression techniques](#mlpack_regression_techniques) + * [Least-angle Regression on Wikipedia](https://en.wikipedia.org/wiki/Least-angle_regression) + +### Constructors + +--- + +#### Constructor Parameters: + + + +| **name** | **type** | **description** | **default** | +|----------|----------|-----------------|-------------| +| `data` | [`arma::mat`](../matrices.md) | [Column-major](../matrices.md) training matrix. | _(N/A)_ | +| `responses` | [`arma::rowvec`](../matrices.md) | Training responses (e.g. values to predict). Should have length `data.n_cols`. | _(N/A)_ | +| `weights` | [`arma::rowvec`](../matrices.md) | Weights for each training point. Should have length `data.n_cols`. | _(N/A)_ | +| `lambda` | `double` | L2 regularization penalty parameter. | `0.0` | +| `intercept` | `bool` | Whether to fit an intercept term in the model. | `bool` | + +### Training + +If training is not done as part of the constructor call, it can be done with the +`Train()` function: + +--- + +Types of each argument are the same as in the table for constructors +[above](#constructor-parameters). + +***Notes:*** + +### Prediction + +Once a `LARS` model is trained, the `Predict()` member function +can be used to make predictions for new data. + + * `double predictedValue = lars.Predict(point)` + - ***(Single-point)*** + - Make a prediction for a single point, returning the predicted value. + +--- + + * `lars.Predict(data, predictions)` + - ***(Multi-point)*** + - Make predictions for a set of points. + - The prediction for data point `i` can be accessed with `predictions[i]`. + +--- + +#### Prediction Parameters: + +| **usage** | **name** | **type** | **description** | +|-----------|----------|----------|-----------------| +| _single-point_ | `point` | [`arma::vec`](../matrices.md) | Single point for prediction. | +|||| +| _multi-point_ | `data` | [`arma::mat`](../matrices.md) | Set of [column-major](../matrices.md) points for classification. | +| _multi-point_ | `predictions` | [`arma::rowvec&`](../matrices.md) | Vector of `double`s to store predictions into. Will be set to length `data.n_cols`. | + +### Other Functionality + + + + * A `LARS` model can be serialized with + [`data::Save()`](../formats.md) and [`data::Load()`](../formats.md). + + + * `lr.Intercept()` will return a `bool` indicating whether the model was + trained with an intercept term. + + * `lr.Parameters()` will return an `arma::vec&` with the model parameters. + This will have length equal to the dimensionality of the model if + `lr.Intercept()` is `false`, and length equal to the dimensionality of the + model plus one if `lr.Intercept()` is `true`. If an intercept was fitted, + the intercept term is the first element of `lr.Parameters()`. + + * `lr.ComputeError(data, responses)` will return a `double` containing the mean + squared error (MSE) of the model on `data`, given that the true responses are + `responses`. + +### Simple Examples + +See also the [simple usage example](#simple-usage-example) for a trivial usage +of the `LARS` class. + +--- + +--- + +### Advanced Functionality: Different Element Types + +The `LARS` class has one template parameter that can be used to +control the element type of the model. The full signature of the class is: + +```c++ +LARS +``` + +`ModelMatType` specifies the type of matrix used for the internal representation +of model parameters. Any matrix type that implements the Armadillo API can be +used. + +Note that the `Train()` and `Predict()` functions themselves are templatized and +can allow any matrix type that has the same element type. So, for instance, a +`LARS` can accept an `arma::sp_mat` for training. + +The example below TODO + +```c++ +``` From 4005f5e91e5788d1914289053558407e8031ffd6 Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Thu, 30 Nov 2023 16:57:59 -0500 Subject: [PATCH 24/91] Add first pass at LARS documentation. --- doc/user/methods/lars.md | 344 +++++++++++++++++++++++++++++++++++++-- 1 file changed, 327 insertions(+), 17 deletions(-) diff --git a/doc/user/methods/lars.md b/doc/user/methods/lars.md index a4828acaa7..40f9ec6d38 100644 --- a/doc/user/methods/lars.md +++ b/doc/user/methods/lars.md @@ -1,19 +1,44 @@ ## `LARS` +The `LARS` class implements the least-angle regression (LARS) algorithm for +L1-penalized and L2-penalized regression. `LARS` can also solve the LASSO +(least absolute shrinkage and selection operator) problem. The LARS algorithm +is a *path* algorithm, and thus will recover solutions for *all* L1 penalty +parameters greater than or equal to the given L1 penalty parameter. #### Simple usage example: ```c++ +// Train a LARS model on random numeric data and make predictions. + +// All data and responses are uniform random; this uses 10 dimensional data. +// Replace with a data::Load() call or similar for a real application. +arma::mat dataset(10, 1000, arma::fill::randu); // 1000 points. +arma::rowvec responses = arma::randn(1000); +arma::mat testDataset(10, 500, arma::fill::randu); // 500 test points. + +mlpack::LARS lars(true, 0.1 /* L1 penalty */); // Step 1: create model. +lars.Train(dataset, responses); // Step 2: train model. +arma::rowvec predictions; +lars.Predict(testDataset, predictions); // Step 3: use model to predict. + +// Print some information about the test predictions. +std::cout << arma::accu(predictions > 0.7) << " test points predicted to have" + << " responses greater than 0.7." << std::endl; +std::cout << arma::accu(predictions < 0) << " test points predicted to have " + << "negative responses." << std::endl; ```

More examples...

#### Quick links: - * [Constructors](#constructors): create `LinearRegression` objects. + * [Constructors](#constructors): create `LARS` objects. * [`Train()`](#training): train model. * [`Predict()`](#prediction): predict with a trained model. * [Other functionality](#other-functionality) for loading, saving, and inspecting. + * [The LARS path](#the-lars-path): use models from the LARS path with different + L1 penalty values. * [Examples](#simple-examples) of simple usage and links to detailed example projects. * [Template parameters](#advanced-functionality-different-element-types) for @@ -27,6 +52,31 @@ ### Constructors + * `lars = LARS(useCholesky=false, lambda1=0.0, lambda2=0.0, tolerance=1e-16, fitIntercept=true, normalizeData=true)` + - Initialize the model without training. + - You will need to call [`Train()`](#training) later to train the model + before calling [`Predict()`](#prediction). + +--- + + + * `lars = LARS(data, responses, transposeData=true, useCholesky=true, lambda1=0.0, lambda2=0.0, tolerance=1e-16, fitIntercept=true, normalizeData=true)` + - Train model on the given data and responses, using the given settings for + hyperparameters. + +--- + + * `lars = LARS(data, responses, transposeData, useCholesky, gramMatrix, lambda1=0.0, lambda2=0.0, tolerance=1e-16, fitIntercept=true, normalizeData=true)` + - *(Advanced constructor)*. + - Train model on the given data and responses, using a precomputed Gram + matrix (`gramMatrix`, equivalent to `data * data.t()`). + - Using a precomputed Gram matrix can save time, if it has already been + computed. + - ***Note:*** any precomputed Gram matrix must also match the settings of + `fitIntercept` and `normalizeData`; so, if both are `true`, then + `gramMatrix` must be computed on mean-centered data whose features are + normalized to have unit variance. + --- #### Constructor Parameters: @@ -38,17 +88,65 @@ | **name** | **type** | **description** | **default** | |----------|----------|-----------------|-------------| -| `data` | [`arma::mat`](../matrices.md) | [Column-major](../matrices.md) training matrix. | _(N/A)_ | +| `data` | [`arma::mat`](../matrices.md) | Training matrix. | _(N/A)_ | | `responses` | [`arma::rowvec`](../matrices.md) | Training responses (e.g. values to predict). Should have length `data.n_cols`. | _(N/A)_ | -| `weights` | [`arma::rowvec`](../matrices.md) | Weights for each training point. Should have length `data.n_cols`. | _(N/A)_ | -| `lambda` | `double` | L2 regularization penalty parameter. | `0.0` | -| `intercept` | `bool` | Whether to fit an intercept term in the model. | `bool` | +| `transposeData` | `bool` | Should be set to true if `data` is [column-major](../matrices.md). Passing row-major data can avoid a transpose operation. | `true` | +| `useCholesky` | `bool` | If `true`, use the Cholesky decomposition of the Gram +matrix to solve linear systems (as opposed to the full Gram matrix). | `true` | +| `gramMatrix` | [`arma::mat`](../matrices.md) | Precomputed Gram matrix of `data` (i.e. `data * data.t()` for column-major data). | _(N/A)_ | +| `lambda1` | `double` | L1 regularization penalty parameter. | `0.0` | +| `lambda2` | `double` | L2 regularization penalty parameter. | `0.0` | +| `tolerance` | `double` | Tolerance on feature correlations for convergence. | `1e-16` | +| `fitIntercept` | `bool` | If `true`, an intercept term will be included in the model. | `true` | +| `normalizeData` | `bool` | If `true`, data will be normalized before fitting +the model. | `true` | + +***Notes:*** + + - The `lambda1` parameter implicitly controls the sparsity of the model; for + more sparse models (i.e. fewer nonzero weights), specify a larger `lambda1`. + + - Specifying a too-small `lambda1` or `lambda2` value may cause the model to + overfit; however, setting it too large may cause the model to underfit. + Because LARS is a path algorithm, the + [`SelectBeta()`](#other_functionality) functions can be used to select models + with different values of `lambda1`. For tuning `lambda2`, + [Automatic hyperparameter tuning](#hyperparameter-tuner) can be used. + + - `fitIntercept` and `normalizeData` are recommended to be set as `true`, in + accordance with the original LARS algorithm. `false` can be used for + `fitIntercept` if the features and responses are already mean-centered, and + `false` can also be used for `normalizeData` if the features are already + unit-variance. Using `false` for either option can provide a small amount of + speedup. + + - `useCholesky` should generally be set to `true` and in most situations will + result in faster training. ### Training If training is not done as part of the constructor call, it can be done with the `Train()` function: + + + + * `lars.Train(data, responses, transposeData=true, useCholesky=true, lambda1=0.0, lambda2=0.0, tolerance=1e-16, fitIntercept=true, normalizeData=true)` + - Train the model on the given data. + +--- + + * `lars.Train(data, responses, transposeData, useCholesky, gramMatrix, lambda1=0.0, lambda2=0.0, tolerance=1e-16, fitIntercept=true, normalizeData=true)` + - *(Advanced training.)* + - Train model on the given data and responses, using a precomputed Gram + matrix (`gramMatrix`, equivalent to `data * data.t()`). + - Using a precomputed Gram matrix can save time, if it has already been + computed. + - ***Note:*** any precomputed Gram matrix must also match the settings of + `fitIntercept` and `normalizeData`; so, if both are `true`, then + `gramMatrix` must be computed on mean-centered data whose features are + normalized to have unit variance. + --- Types of each argument are the same as in the table for constructors @@ -56,11 +154,19 @@ Types of each argument are the same as in the table for constructors ***Notes:*** + * Training is not incremental. A second call to `Train()` will retrain the + model from scratch. + + * `Train()` returns the mean squared error (MSE) of the model on the training + set as a `double`. + ### Prediction Once a `LARS` model is trained, the `Predict()` member function can be used to make predictions for new data. + + * `double predictedValue = lars.Predict(point)` - ***(Single-point)*** - Make a prediction for a single point, returning the predicted value. @@ -90,19 +196,59 @@ can be used to make predictions for new data. * A `LARS` model can be serialized with [`data::Save()`](../formats.md) and [`data::Load()`](../formats.md). - - * `lr.Intercept()` will return a `bool` indicating whether the model was - trained with an intercept term. + * `lars.Beta()` will return an `arma::vec` with the model parameters. This + will have length equal to the dimensionality of the model. Note that + `lars.Beta()` can be changed to a different model on the LARS path using the + [`lars.SelectBeta()` method](#the-lars-path). - * `lr.Parameters()` will return an `arma::vec&` with the model parameters. - This will have length equal to the dimensionality of the model if - `lr.Intercept()` is `false`, and length equal to the dimensionality of the - model plus one if `lr.Intercept()` is `true`. If an intercept was fitted, - the intercept term is the first element of `lr.Parameters()`. + * `lars.Intercept()` will return a `double` representing the fitted intercept + term, if `lars.FitIntercept()` is `true`. - * `lr.ComputeError(data, responses)` will return a `double` containing the mean - squared error (MSE) of the model on `data`, given that the true responses are - `responses`. + * `lars.ActiveSet()` will return a `std::vector&` containing the + indices of nonzero dimensions in the model parameters (`lars.Beta()`). + + * `lars.ComputeError(data, responses, rowMajor=false)` will return a `double` + containing the mean squared error (MSE) of the model on `data`, given that + the true responses are `responses`. + +### The LARS Path + +LARS is a *path* (or stepwise) algorithm, meaning it adds one feature at a time +to the model. This in turn means that when we train a LARS model with +`lambda1` set to `l`, we also recover every possible LARS model on the same data +with a `lambda1` greater than `l`. + +The `LARS` class provides a way to access all of the models on the path, and +switch between them for prediction purposes: + + * `lars.BetaPath()` returns a `std::vector&` containing each set of + model weights on the LARS path. + + * `lars.InterceptPath()` returns a `std::vector&` containing each + intercept value on the LARS path. These values are only meaningful if + `lars.FitIntercept()` is `true`. + + * `lars.LambdaPath()` returns a `std::vector&` containing each + `lambda1` value that is associated with each element in `lars.BetaPath()` and + `lars.InterceptPath()`. That is, `lars.LambdaPath(i)` is the `lambda1` value + corresponding to the model defined by `lars.BetaPath(i)` and + `lars.InterceptPath(i)`. + + + * `lars.SelectBeta(lambda1)` will set the model weights (`lars.ActiveSet()`, + `lars.Beta()` and `lars.Intercept()`) to the path location with L1 penalty + `lambda1`. This is equivalent to calling `lars.Train(data, responses, + transposeData, useCholesky, lambda1)`---but much more efficient! + + + * `lars.SelectedLambda()` returns the currently selected L1 regularization + penalty parameter. + + * For any value `lambda1` between `lars.LambdaPath(i)` and `lars.LambdaPath(i + + 1)`, the corresponding model is a linear interpolation between + `lars.BetaPath(i)` and `lars.BetaPath(i + 1)` (and `lars.InterceptPath(i)` + and `lars.InterceptPath(i + 1)`). This exact linear interpolation is what is + computed by `lars.SelectBeta(lambda1)`. ### Simple Examples @@ -111,8 +257,141 @@ of the `LARS` class. --- +Train a LARS model in the constructor, and print the MSE on training and +test data for each set of weights in the path. + +```c++ +// See https://datasets.mlpack.org/wave_energy_farm_100.csv. +arma::mat data; +mlpack::data::Load("wave_energy_farm_100.csv", data, true); + +// Split the last row off: it is the responses. +arma::rowvec responses = data.row(data.n_rows - 1); +data.shed_rows(data.n_rows - 1); + +// Split into a training and test dataset. 20% of the data is held out as a +// test set. +arma::mat trainingData, testData; +arma::rowvec trainingResponses, testResponses; +data::Split(data, responses, trainingData, trainingResponses, testData, + testResponses, 0.2); + +// Train a LARS model with lambda1 = 1e-5 and lambda2 = 1e-6. +mlpack::LARS lars(trainingData, trainingResponses, true, true, 1e-5, 1e-6); + +// Iterate over all the models in the path. +const size_t pathLength = lars.BetaPath().size(); +for (size_t i = 0; i < pathLength; ++i) +{ + // Use the i'th model in the path. + lars.SelectBeta(lars.BetaPath(i)); + + std::cout << "L1 penalty parameter: " << lars.SelectedLambda() << std::endl; + std::cout << " MSE on training set: " + << lars.ComputeError(trainingData, trainingResponses) << "." << std::endl; + std::cout << " MSE on test set: " + << lars.ComputeError(testData, testResponses) << "." << std::endl; +} +``` + --- +Train a LARS model, print predictions for a random point, and save to a file. + +```c++ +// See https://datasets.mlpack.org/admission_predict.csv. +arma::mat data; +mlpack::data::Load("admission_predict.csv", data, true); + +// See https://datasets.mlpack.org/admission_predict.responses.csv. +arma::rowvec responses; +mlpack::data::Load("admission_predict.responses.csv", responses, true); + +// Train a LARS model with only L2 regularization. +mlpack::LARS lars(data, responses, true, true, 0.0, 0.1 /* lambda2 */); + +// Predict on a random point. +arma::vec point = arma::randu(data.n_rows); +const double prediction = lars.Predict(point); + +std::cout << "Prediction on random point: " << prediction << "." << std::endl; + +// Save the model to "lars_model.bin" with the name "lars". +mlpack::data::Save("lars_model.bin", "lars", lars, true); +``` + +--- + +Load a LARS model from disk and print some information about it. + +```c++ +// This assumes a model named "lars" has previously been saved to +// "lars_model.bin". +mlpack::LARS lars; +mlpack::data::Load("lars_model.bin", "lars", lars, true); + +if (lars.Beta().n_elem) +{ + std::cout << "lars_model.bin contains an untrained LARS model." << std::endl; +} +else +{ + std::cout << "Information on the LARS model in lars_model.bin:" << std::endl; + + std::cout << " - Model dimensionality: " << lars.Beta().n_elem << "." + << std::endl; + std::cout << " - Has intercept: " + << (lars.FitIntercept() ? std::string("yes") : std::string("no")) << "." + << std::endl; + std::cout << " - Current L1 regularization penalty parameter value: " + << lars.SelectedLambda() << "." << std::endl; + std::cout << " - L2 regularization penalty parameter: " << lars.Lambda2() + << "." << std::endl; + std::cout << " - Number of nonzero elements in model: " + << lars.ActiveSet().size() << "." << std::endl; + std::cout << " - Number of models in LARS path: " << lars.BetaPath().size() + << "." << std::endl; + std::cout << " - Model weight for dimension 0: " << lars.Beta()[0] << "." + << std::endl; + + if (lars.FitIntercept()) + { + std::cout << " - Intercept value: " << lars.Intercept() << "." << std::endl; + } +} +``` + +--- + +Train several models with different L2 regularization penalty parameters, using +a precomputed Gram matrix. + +```c++ +// See https://datasets.mlpack.org/admission_predict.csv. +arma::mat data; +mlpack::data::Load("admission_predict.csv", data, true); + +// See https://datasets.mlpack.org/admission_predict.responses.csv. +arma::rowvec responses; +mlpack::data::Load("admission_predict.responses.csv", responses, true); + +// Precompute Gram matrix. +arma::mat gramMatrix = data * data.t(); + +std::vector lambda2Values = { 0.0001, 0.001, 0.01, 0.1, 1.0 }; +for (lambda2 : lambda2Values) +{ + // Build a LARS model using the precomputed Gram matrix. We did not normalize + // or center the data before computing the Gram matrix, so we have to set + // fitIntercept and normalizeData accordingly. + LARS lars(data, responses, true, true, gramMatrix, 0.01, lambda2, 1e-16, + false, false); + + std::cout << "MSE with L2 penalty " << lambda2 << ": " + << lars.ComputeError(data, responses) << "." << std::endl; +} +``` + ### Advanced Functionality: Different Element Types The `LARS` class has one template parameter that can be used to @@ -130,7 +409,38 @@ Note that the `Train()` and `Predict()` functions themselves are templatized and can allow any matrix type that has the same element type. So, for instance, a `LARS` can accept an `arma::sp_mat` for training. -The example below TODO +The example below trains a LARS model on sparse 32-bit precision data, using +`arma::sp_mat` to store the model parameters. ```c++ +// Create random, sparse 1000-dimensional data. +arma::sp_fmat dataset; +dataset.sprandu(1000, 5000, 0.1); + +// Generate noisy responses from random data. +arma::fvec trueWeights(1000, arma::fill::randu); +arma::frowvec responses = trueWeights.t() * dataset + + 0.01 * arma::randu(5000) /* noise term */; + +mlpack::LARS lars; +lars.Lambda1() = 0.1; +lars.Lambda2() = 0.01; + +lars.Train(dataset, responses); + +// Compute the MSE on the training set and a random test set. +arma::sp_fmat testDataset; +testDataset.sprandu(1000, 2500, 0.3); + +arma::frowvec testResponses = trueWeights.t() * testDataset + + 0.01 * arma::randu(2500) /* noise term */; + +std::cout << "MSE on training set: " + << lars.ComputeError(dataset, responses) << "." << std::endl; +std::cout << "MSE on test set: " + << lars.ComputeError(testDataset, testResponses) << "." << std::endl; ``` + +***Note:*** it is generally only more efficient to use a sparse type (e.g. +`arma::sp_mat`) for `ModelMatType` when the L1 regularization parameter is set +such that a highly sparse model is produced. From ffa41b231f59a1d81481f49df427c0cd8aca1766 Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Thu, 30 Nov 2023 17:26:46 -0500 Subject: [PATCH 25/91] Add some additional tests for linear regression. --- .../linear_regression_impl.hpp | 14 +++-- src/mlpack/tests/linear_regression_test.cpp | 57 +++++++++++++++++++ 2 files changed, 66 insertions(+), 5 deletions(-) diff --git a/src/mlpack/methods/linear_regression/linear_regression_impl.hpp b/src/mlpack/methods/linear_regression/linear_regression_impl.hpp index 0952c04fbc..2504b3cf06 100644 --- a/src/mlpack/methods/linear_regression/linear_regression_impl.hpp +++ b/src/mlpack/methods/linear_regression/linear_regression_impl.hpp @@ -24,7 +24,8 @@ inline LinearRegression::LinearRegression( const ResponsesType& responses, const double lambda, const bool intercept) : - LinearRegression(predictors, responses, ResponsesType(), lambda, intercept) + LinearRegression(predictors, responses, + arma::Row(), lambda, intercept) { /* Nothing to do. */ } template @@ -83,7 +84,8 @@ typename LinearRegression::ElemType LinearRegression::Train(const MatType& predictors, const ResponsesType& responses) { - return Train(predictors, responses, ResponsesType(), this->lambda, + return Train(predictors, responses, + arma::Row(), this->lambda, this->intercept); } @@ -95,7 +97,8 @@ LinearRegression::Train(const MatType& predictors, const ResponsesType& responses, const double lambda) { - return Train(predictors, responses, ResponsesType(), lambda, this->intercept); + return Train(predictors, responses, + arma::Row(), lambda, this->intercept); } template @@ -107,7 +110,8 @@ LinearRegression::Train(const MatType& predictors, const double lambda, const bool intercept) { - return Train(predictors, responses, ResponsesType(), lambda, intercept); + return Train(predictors, responses, + arma::Row(), lambda, intercept); } template @@ -288,7 +292,7 @@ LinearRegression::ComputeError( // Calculate the differences between actual responses and predicted responses. // We must also add the intercept (parameters(0)) to the predictions. - ResponsesType temp; + arma::Row temp; if (intercept) { // Ensure that we have the correct number of dimensions in the dataset. diff --git a/src/mlpack/tests/linear_regression_test.cpp b/src/mlpack/tests/linear_regression_test.cpp index fd8e210ad7..1706babe80 100644 --- a/src/mlpack/tests/linear_regression_test.cpp +++ b/src/mlpack/tests/linear_regression_test.cpp @@ -364,5 +364,62 @@ TEMPLATE_TEST_CASE("LinearRegressionSinglePointPredictTest", } // Make sure training on submatrices and subvectors works. +TEST_CASE("LinearRegressionSubmatrixTrainingTest", "[LinearRegressionTest]") +{ + // The quality of the model doesn't matter---mostly this is a compilation + // test. + arma::mat predictors(100, 1000, arma::fill::randu); + arma::rowvec responses(1000, arma::fill::randu); + arma::rowvec weights(1000, arma::fill::randu); + + LinearRegression<> lr1(predictors.cols(0, 499), responses.subvec(0, 499)); + LinearRegression<> lr2; + lr2.Train(predictors.cols(0, 499), responses.subvec(0, 499)); + LinearRegression<> lr3(predictors.cols(0, 499), responses.subvec(0, 499), + weights.subvec(0, 499)); + LinearRegression<> lr4; + lr4.Train(predictors.cols(0, 499), responses.subvec(0, 499), + weights.subvec(0, 499)); + + REQUIRE(lr1.Parameters().n_elem == 101); + REQUIRE(lr2.Parameters().n_elem == 101); + REQUIRE(lr3.Parameters().n_elem == 101); + REQUIRE(lr4.Parameters().n_elem == 101); + + arma::rowvec predictions; + + lr1.Predict(predictors.cols(500, 999), predictions); + REQUIRE(predictions.n_cols == 500); + + lr2.Predict(predictors.cols(500, 999), predictions); + REQUIRE(predictions.n_cols == 500); + + lr3.Predict(predictors.cols(500, 999), predictions); + REQUIRE(predictions.n_cols == 500); + + lr4.Predict(predictors.cols(500, 999), predictions); + REQUIRE(predictions.n_cols == 500); +} // Make sure we can train on sparse data. +TEST_CASE("LinearRegressionSparseTrainingTest", "[LinearRegressionTest]") +{ + // For this test the quality of the model doesn't matter---mostly this is a + // compilation test, but we check that the sizes of the returned predictions + // and the sizes of the learned models are correct. + + // Generate sparse random data. + arma::sp_mat data; + data.sprandu(100, 5000, 0.3); + + arma::rowvec responses(5000, arma::fill::randu); + + LinearRegression<> lr(data, responses); + + REQUIRE(lr.Parameters().n_elem == 101); + + arma::rowvec predictions; + lr.Predict(data, predictions); + + REQUIRE(predictions.n_elem == 5000); +} From 807876a7d8629c6b980e75265f8f3a02b6476737 Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Mon, 4 Dec 2023 11:25:32 -0500 Subject: [PATCH 26/91] Update documentation. --- doc/user/methods/lars.md | 25 +++++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/doc/user/methods/lars.md b/doc/user/methods/lars.md index 40f9ec6d38..f0621a8457 100644 --- a/doc/user/methods/lars.md +++ b/doc/user/methods/lars.md @@ -101,6 +101,26 @@ matrix to solve linear systems (as opposed to the full Gram matrix). | `true` | | `normalizeData` | `bool` | If `true`, data will be normalized before fitting the model. | `true` | +As an alternative to passing hyperparameters, each hyperparameter can be set +with a standalone method. The following functions can be used before calling +`Train()` to set hyperparameters: + + * `lars.RowMajor() = rowMajor;` will set whether the data is given in row-major + form to `rowMajor`. + * `lars.UseCholesky() = useChol;` will set whether or not the Cholesky + decomposition will be used during training to `useChol`. + * `lars.Lambda1() = lambda1;` will set the L1 regularization penalty parameter + to `lambda1`. + * `lars.Lambda2() = lambda2;` will set the L2 regularization penalty parameter + to `lambda2`. + * `lars.Tolerance() = tol;` will set the convergence tolerance to `tol`. + * `lars.FitIntercept(fitIntercept);` will set whether an intercept will be fit + to `fitIntercept`. If an external Gram matrix has been specified, this will + throw an exception. + * `lars.NormalizeData(normalizeData);` will set whether data should be + normalized to `normalizeData`. If an external Gram matrix has been + specified, this will throw an exception. + ***Notes:*** - The `lambda1` parameter implicitly controls the sparsity of the model; for @@ -202,7 +222,7 @@ can be used to make predictions for new data. [`lars.SelectBeta()` method](#the-lars-path). * `lars.Intercept()` will return a `double` representing the fitted intercept - term, if `lars.FitIntercept()` is `true`. + term, or 0 if `lars.FitIntercept()` is `false`. * `lars.ActiveSet()` will return a `std::vector&` containing the indices of nonzero dimensions in the model parameters (`lars.Beta()`). @@ -238,7 +258,8 @@ switch between them for prediction purposes: * `lars.SelectBeta(lambda1)` will set the model weights (`lars.ActiveSet()`, `lars.Beta()` and `lars.Intercept()`) to the path location with L1 penalty `lambda1`. This is equivalent to calling `lars.Train(data, responses, - transposeData, useCholesky, lambda1)`---but much more efficient! + transposeData, useCholesky, lambda1)`---but much more efficient! `lambda1` + cannot be greater than `lars.Lambda1()`, or an exception will be thrown. * `lars.SelectedLambda()` returns the currently selected L1 regularization From 0c4d0f984683439e076eac11076cb20f31a19376 Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Mon, 4 Dec 2023 13:27:26 -0500 Subject: [PATCH 27/91] Refactor for Train() overloads, constructor overloads, and add support for SelectBeta(). --- src/mlpack/methods/lars/lars.hpp | 164 +++++++++++++- src/mlpack/methods/lars/lars_impl.hpp | 315 ++++++++++++++++++++++++-- 2 files changed, 452 insertions(+), 27 deletions(-) diff --git a/src/mlpack/methods/lars/lars.hpp b/src/mlpack/methods/lars/lars.hpp index 712d917cfc..b744a80a1e 100644 --- a/src/mlpack/methods/lars/lars.hpp +++ b/src/mlpack/methods/lars/lars.hpp @@ -134,6 +134,7 @@ class LARS * @param normalizeData If true, normalize all features to have unit variance * for training. */ + mlpack_deprecated LARS(const bool useCholesky, const arma::mat& gramMatrix, const double lambda1 = 0.0, @@ -248,6 +249,7 @@ class LARS * @param transposeData Set to false if the data is row-major. * @return minimum cost error(||y-beta*X||2 is used to calculate error). */ + mlpack_deprecated double Train(const arma::mat& data, const arma::rowvec& responses, arma::vec& beta, @@ -261,6 +263,9 @@ class LARS * necessary (i.e., you want to pass in a row-major matrix), pass 'false' for * the transposeData parameter. * + * All of the different overloads below are needed until C++17 is the minimum + * required standard (then std::optional could be used). + * * @param data Input data. * @param responses A vector of targets. * @param transposeData Should be true if the input data is column-major and @@ -271,6 +276,127 @@ class LARS const arma::rowvec& responses, const bool transposeData = true); + double Train(const arma::mat& data, + const arma::rowvec& responses, + const bool transposeData, + const bool useCholesky); + + double Train(const arma::mat& data, + const arma::rowvec& responses, + const bool transposeData, + const bool useCholesky, + const double lambda1); + + double Train(const arma::mat& data, + const arma::rowvec& responses, + const bool transposeData, + const bool useCholesky, + const double lambda1, + const double lambda2); + + double Train(const arma::mat& data, + const arma::rowvec& responses, + const bool transposeData, + const bool useCholesky, + const double lambda1, + const double lambda2, + const double tolerance); + + double Train(const arma::mat& data, + const arma::rowvec& responses, + const bool transposeData, + const bool useCholesky, + const double lambda1, + const double lambda2, + const double tolerance, + const bool fitIntercept); + + double Train(const arma::mat& data, + const arma::rowvec& responses, + const bool transposeData, + const bool useCholesky, + const double lambda1, + const double lambda2, + const double tolerance, + const bool fitIntercept, + const bool normalizeData); + + /** + * Run LARS with a precomputed Gram matrix. The input matrix (like all mlpack + * matrices) should be column-major -- each column is an observation and each + * row is a dimension. However, because LARS is more efficient on a row-major + * matrix, this method will (internally) transpose the matrix. If this + * transposition is not necessary (i.e., you want to pass in a row-major + * matrix), pass 'false' for the transposeData parameter. + * + * All of the different overloads below are needed until C++17 is the minimum + * required standard (then std::optional could be used). + * + * @param data Input data. + * @param responses A vector of targets. + * @param transposeData Should be true if the input data is column-major and + * false otherwise. + * @return minimum cost error(||y-beta*X||2 is used to calculate error). + */ + double Train(const arma::mat& data, + const arma::rowvec& responses, + const bool transposeData, + const bool useCholesky, + const arma::mat& gramMatrix); + + double Train(const arma::mat& data, + const arma::rowvec& responses, + const bool transposeData, + const bool useCholesky, + const arma::mat& gramMatrix, + const double lambda1); + + double Train(const arma::mat& data, + const arma::rowvec& responses, + const bool transposeData, + const bool useCholesky, + const arma::mat& gramMatrix, + const double lambda1, + const double lambda2); + + double Train(const arma::mat& data, + const arma::rowvec& responses, + const bool transposeData, + const bool useCholesky, + const arma::mat& gramMatrix, + const double lambda1, + const double lambda2, + const double tolerance); + + double Train(const arma::mat& data, + const arma::rowvec& responses, + const bool transposeData, + const bool useCholesky, + const arma::mat& gramMatrix, + const double lambda1, + const double lambda2, + const double tolerance, + const bool fitIntercept); + + double Train(const arma::mat& data, + const arma::rowvec& responses, + const bool transposeData, + const bool useCholesky, + const arma::mat& gramMatrix, + const double lambda1, + const double lambda2, + const double tolerance, + const bool fitIntercept, + const bool normalizeData); + + /** + * Predict y_i for the given data point. + * + * @param point The data point to regress on. + * @return Predicted value for y_i for `point`. + */ + double Predict(const arma::vec& point) const; + /** * Predict y_i for each data point in the given data matrix using the * currently-trained LARS model. @@ -337,8 +463,8 @@ class LARS { if (matGram != &matGramInternal) { - throw std::invalid_argument("LARS::NormalizeData(): cannot change value " - "when an external Gram matrix was specified!"); + throw std::invalid_argument("LARS::NormalizeData(): cannot change value" + " when an external Gram matrix was specified!"); } normalizeData = newNormalizeData; @@ -354,18 +480,33 @@ class LARS const std::vector& BetaPath() const { return betaPath; } //! Access the solution coefficients - const arma::vec& Beta() const { return betaPath.back(); } + const arma::vec& Beta() const + { + return (selectedIndex < betaPath.size()) ? + betaPath[selectedIndex] : selectedBeta; + } //! Access the set of values for lambda1 after each iteration; the solution is //! the last element. const std::vector& LambdaPath() const { return lambdaPath; } //! Return the intercept (if fitted, otherwise 0). - double Intercept() const { return interceptPath.back(); } + double Intercept() const + { + return (selectedIndex < interceptPath.size()) ? + interceptPath[selectedIndex] : selectedIntercept; + } //! Return the intercept path (the intercept for every model). const std::vector& InterceptPath() const { return interceptPath; } + //! Set the model to use the given lambda1 value in the path. + void SelectBeta(const double lambda1); + + //! Get the L1 penalty parameter corresponding to the currently selected + //! model. + double SelectedLambda1() const { return selectedLambda1; } + //! Access the upper triangular cholesky factor. const arma::mat& MatUtriCholFactor() const { return matUtriCholFactor; } @@ -436,6 +577,21 @@ class LARS //! Active set of dimensions. std::vector activeSet; + //! Selected lambda1 value for Predict(). + double selectedLambda1; + + //! Index of selected beta (if selectedLambda1 is in lambdaPath). + size_t selectedIndex; + + //! Selected beta, if selectedLambda1 is not in lambdaPath. + arma::vec selectedBeta; + + //! Selected intercept, if selectedLambda1 is not in lambdaPath. + double selectedIntercept; + + //! Might be needed to compute the intercept for other lambda values. + double offsetY; + //! Active set membership indicator (for each dimension). std::vector isActive; diff --git a/src/mlpack/methods/lars/lars_impl.hpp b/src/mlpack/methods/lars/lars_impl.hpp index c55278b368..81a08d1232 100644 --- a/src/mlpack/methods/lars/lars_impl.hpp +++ b/src/mlpack/methods/lars/lars_impl.hpp @@ -32,7 +32,11 @@ inline LARS::LARS( lambda2(lambda2), tolerance(tolerance), fitIntercept(fitIntercept), - normalizeData(normalizeData) + normalizeData(normalizeData), + selectedLambda1(lambda1), + selectedIndex(0), + selectedIntercept(0.0), + offsetY(0.0) { /* Nothing left to do. */ } inline LARS::LARS( @@ -51,7 +55,11 @@ inline LARS::LARS( lambda2(lambda2), tolerance(tolerance), fitIntercept(fitIntercept), - normalizeData(normalizeData) + normalizeData(normalizeData), + selectedLambda1(lambda1), + selectedIndex(0), + selectedIntercept(0.0), + offsetY(0.0) { /* Nothing left to do */ } inline LARS::LARS( @@ -104,6 +112,11 @@ inline LARS::LARS(const LARS& other) : lambdaPath(other.lambdaPath), interceptPath(other.interceptPath), activeSet(other.activeSet), + selectedLambda1(other.selectedLambda1), + selectedIndex(other.selectedIndex), + selectedBeta(other.selectedBeta), + selectedIntercept(other.selectedIntercept), + offsetY(other.offsetY), isActive(other.isActive), ignoreSet(other.ignoreSet), isIgnored(other.isIgnored) @@ -129,6 +142,11 @@ inline LARS::LARS(LARS&& other) : lambdaPath(std::move(other.lambdaPath)), interceptPath(std::move(other.interceptPath)), activeSet(std::move(other.activeSet)), + selectedLambda1(std::move(other.selectedLambda1)), + selectedIndex(std::move(other.selectedIndex)), + selectedBeta(std::move(other.selectedBeta)), + selectedIntercept(std::move(other.selectedIntercept)), + offsetY(std::move(other.offsetY)), isActive(std::move(other.isActive)), ignoreSet(std::move(other.ignoreSet)), isIgnored(std::move(other.isIgnored)) @@ -158,6 +176,11 @@ inline LARS& LARS::operator=(const LARS& other) lambdaPath = other.lambdaPath; interceptPath = other.interceptPath; activeSet = other.activeSet; + selectedLambda1 = other.selectedLambda1; + selectedIndex = other.selectedIndex; + selectedBeta = other.selectedBeta; + selectedIntercept = other.selectedIntercept; + offsetY = other.offsetY; isActive = other.isActive; ignoreSet = other.ignoreSet; isIgnored = other.isIgnored; @@ -185,6 +208,11 @@ inline LARS& LARS::operator=(LARS&& other) betaPath = std::move(other.betaPath); lambdaPath = std::move(other.lambdaPath); interceptPath = std::move(other.interceptPath); + selectedLambda1 = std::move(other.selectedLambda1); + selectedIndex = std::move(other.selectedIndex); + selectedBeta = std::move(other.selectedBeta); + selectedIntercept = std::move(other.selectedIntercept); + offsetY = std::move(other.offsetY); activeSet = std::move(other.activeSet); isActive = std::move(other.isActive); ignoreSet = std::move(other.ignoreSet); @@ -192,11 +220,178 @@ inline LARS& LARS::operator=(LARS&& other) return *this; } +mlpack_deprecated inline double LARS::Train(const arma::mat& matX, const arma::rowvec& y, arma::vec& beta, const bool transposeData) { + const double result = Train(matX, y, transposeData); + beta = betaPath.back(); + return result; +} + +inline double LARS::Train(const arma::mat& data, + const arma::rowvec& responses, + const bool transposeData) +{ + return Train(data, responses, transposeData, this->useCholesky, this->lambda1, + this->lambda2, this->tolerance, this->fitIntercept, this->normalizeData); +} + +inline double LARS::Train(const arma::mat& data, + const arma::rowvec& responses, + const bool transposeData, + const bool useCholesky) +{ + return Train(data, responses, transposeData, useCholesky, this->lambda1, + this->lambda2, this->tolerance, this->fitIntercept, this->normalizeData); +} + +inline double LARS::Train(const arma::mat& data, + const arma::rowvec& responses, + const bool transposeData, + const bool useCholesky, + const double lambda1) +{ + return Train(data, responses, transposeData, useCholesky, lambda1, + this->lambda2, this->tolerance, this->fitIntercept, this->normalizeData); +} + +inline double LARS::Train(const arma::mat& data, + const arma::rowvec& responses, + const bool transposeData, + const bool useCholesky, + const double lambda1, + const double lambda2) +{ + return Train(data, responses, transposeData, useCholesky, lambda1, lambda2, + this->tolerance, this->fitIntercept, this->normalizeData); +} + +inline double LARS::Train(const arma::mat& data, + const arma::rowvec& responses, + const bool transposeData, + const bool useCholesky, + const double lambda1, + const double lambda2, + const double tolerance) +{ + return Train(data, responses, transposeData, useCholesky, lambda1, lambda2, + tolerance, this->fitIntercept, this->normalizeData); +} + +inline double LARS::Train(const arma::mat& data, + const arma::rowvec& responses, + const bool transposeData, + const bool useCholesky, + const double lambda1, + const double lambda2, + const double tolerance, + const bool fitIntercept) +{ + return Train(data, responses, transposeData, useCholesky, lambda1, lambda2, + tolerance, fitIntercept, this->normalizeData); +} + +inline double LARS::Train(const arma::mat& data, + const arma::rowvec& responses, + const bool transposeData, + const bool useCholesky, + const arma::mat& gramMatrix) +{ + return Train(data, responses, transposeData, useCholesky, gramMatrix, + this->lambda1, this->lambda2, this->tolerance, this->fitIntercept, + this->normalizeData); +} + +inline double LARS::Train(const arma::mat& data, + const arma::rowvec& responses, + const bool transposeData, + const bool useCholesky, + const arma::mat& gramMatrix, + const double lambda1) +{ + return Train(data, responses, transposeData, useCholesky, gramMatrix, lambda1, + this->lambda2, this->tolerance, this->fitIntercept, this->normalizeData); +} + +inline double LARS::Train(const arma::mat& data, + const arma::rowvec& responses, + const bool transposeData, + const bool useCholesky, + const arma::mat& gramMatrix, + const double lambda1, + const double lambda2) +{ + return Train(data, responses, transposeData, useCholesky, gramMatrix, lambda1, + lambda2, this->tolerance, this->fitIntercept, this->normalizeData); +} + +inline double LARS::Train(const arma::mat& data, + const arma::rowvec& responses, + const bool transposeData, + const bool useCholesky, + const arma::mat& gramMatrix, + const double lambda1, + const double lambda2, + const double tolerance) +{ + return Train(data, responses, transposeData, useCholesky, gramMatrix, lambda1, + lambda2, tolerance, this->fitIntercept, this->normalizeData); +} + +inline double LARS::Train(const arma::mat& data, + const arma::rowvec& responses, + const bool transposeData, + const bool useCholesky, + const arma::mat& gramMatrix, + const double lambda1, + const double lambda2, + const double tolerance, + const bool fitIntercept) +{ + return Train(data, responses, transposeData, useCholesky, gramMatrix, lambda1, + lambda2, fitIntercept, tolerance, this->normalizeData); +} + +inline double LARS::Train(const arma::mat& data, + const arma::rowvec& responses, + const bool transposeData, + const bool useCholesky, + const arma::mat& gramMatrix, + const double lambda1, + const double lambda2, + const double tolerance, + const bool fitIntercept, + const bool normalizeData) +{ + // Set Gram matrix. + matGramInternal.clear(); + matGram = &gramMatrix; + + return Train(data, responses, transposeData, useCholesky, lambda1, lambda2, + tolerance, fitIntercept, normalizeData); +} + +inline double LARS::Train(const arma::mat& matX, + const arma::rowvec& y, + const bool transposeData, + const bool useCholesky, + const double lambda1, + const double lambda2, + const double tolerance, + const bool fitIntercept, + const bool normalizeData) +{ + // Update hyperparameter settings. + this->useCholesky = useCholesky; + this->lambda1 = lambda1; + this->lambda2 = lambda2; + this->tolerance = tolerance; + this->fitIntercept = fitIntercept; + this->normalizeData = normalizeData; + // Clear any previous solution information. betaPath.clear(); lambdaPath.clear(); @@ -205,6 +400,7 @@ inline double LARS::Train(const arma::mat& matX, ignoreSet.clear(); isIgnored.clear(); matUtriCholFactor.reset(); + selectedBeta.clear(); // Update values in case lambda1 or lambda2 changed. lasso = (lambda1 != 0); @@ -223,7 +419,7 @@ inline double LARS::Train(const arma::mat& matX, (fitIntercept) ? yCentered : y; arma::vec offsetX; // used only if fitting an intercept - double offsetY = 0.0; // used only if fitting an intercept + this->offsetY = 0.0; // used only if fitting an intercept arma::vec stdX; // used only if normalizing if (transposeData) @@ -278,8 +474,8 @@ inline double LARS::Train(const arma::mat& matX, if (fitIntercept) { - offsetY = arma::mean(y); - yCentered = y - offsetY; + this->offsetY = arma::mean(y); + yCentered = y - this->offsetY; } // Compute X' * y. @@ -293,7 +489,7 @@ inline double LARS::Train(const arma::mat& matX, isIgnored.resize(dataRef.n_cols, false); // Initialize yHat and beta. - beta = arma::zeros(dataRef.n_cols); + arma::vec beta = arma::zeros(dataRef.n_cols); arma::vec yHat = arma::zeros(dataRef.n_rows); arma::vec yHatDirection(dataRef.n_rows); @@ -322,7 +518,7 @@ inline double LARS::Train(const arma::mat& matX, lambdaPath[0] = lambda1; if (fitIntercept) - interceptPath.push_back(offsetY - arma::dot(offsetX, betaPath[0])); + interceptPath.push_back(this->offsetY - arma::dot(offsetX, betaPath[0])); else interceptPath.push_back(0.0); @@ -647,7 +843,7 @@ inline double LARS::Train(const arma::mat& matX, { interceptPath.clear(); for (size_t i = 0; i < betaPath.size(); ++i) - interceptPath.push_back(offsetY - arma::dot(offsetX, betaPath[i])); + interceptPath.push_back(this->offsetY - arma::dot(offsetX, betaPath[i])); } else { @@ -655,33 +851,97 @@ inline double LARS::Train(const arma::mat& matX, interceptPath.resize(betaPath.size(), 0.0); } - // Unfortunate copy... - beta = betaPath.back(); + // Make the model we use point to the last element in the path after + // interpolation. + selectedLambda1 = lambda1; + selectedIndex = betaPath.size() - 1; return ComputeError(matX, y, !transposeData); } -inline double LARS::Train(const arma::mat& data, - const arma::rowvec& responses, - const bool transposeData) -{ - arma::vec beta; - return Train(data, responses, beta, transposeData); -} - inline void LARS::Predict(const arma::mat& points, arma::rowvec& predictions, const bool rowMajor) const { // We really only need to store beta internally... if (rowMajor && !fitIntercept) - predictions = trans(points * betaPath.back()); + predictions = trans(points * Beta()); else if (rowMajor) - predictions = trans(points * betaPath.back()) + interceptPath.back(); + predictions = trans(points * Beta()) + Intercept(); else if (fitIntercept) - predictions = betaPath.back().t() * points + interceptPath.back(); + predictions = Beta().t() * points + Intercept(); else - predictions = betaPath.back().t() * points; + predictions = Beta().t() * points; +} + +inline void LARS::SelectBeta(const double selLambda1) +{ + if (selLambda1 < lambda1) + { + std::ostringstream oss; + oss << "LARS::SelectBeta(): given lambda1 value (" << selLambda1 << ") " + << "cannot be less than model's Lambda1() value (" << lambda1 + << ")!"; + throw std::invalid_argument(oss.str()); + } + else if (betaPath.size() == 0) + { + throw std::runtime_error("LARS::SelectBeta(): model must be trained " + "before calling SelectBeta()!"); + } + + this->selectedLambda1 = selLambda1; + selectedBeta.clear(); + + // Find which lambda values we are interpolating between. lambdaPath is in + // reverse order (due to the fact that LARS is a stepwise algorithm), so the + // largest lambdas come first. + size_t i = 0; + while (i < lambdaPath.size()) + { + if (selLambda1 == lambdaPath[i]) + { + // If it's an exact match, no interpolation is necessary, and we can + // directly use the element from the path. + selectedIndex = i; + return; + } + else if (selLambda1 > lambdaPath[i]) + { + // It's not an exact match, but lambdaPath[i] is the first lambda element + // that is smaller than the desired lambda. + break; + } + + ++i; + } + + // In the case where selLambda1 is larger than the largest lambda we have a + // model for, we can interpolate between the zero vector and the first model. + if (i == 0) + { + const double interp = selLambda1 / lambdaPath[0]; + selectedIndex = betaPath.size(); + + selectedLambda1 = interp * lambdaPath[0]; + selectedBeta = interp * betaPath[0]; + // Computing the intercept differs just a little bit from what's expected, + // because we have to account for the offsetY term, which is not zero even + // for a zero model. + selectedIntercept = (1 - interp) * this->offsetY + + interp * interceptPath[0]; + } + else + { + const double interp = (lambdaPath[i - 1] - selLambda1) / + (lambdaPath[i - 1] - lambdaPath[i]); + selectedIndex = betaPath.size(); + + selectedLambda1 = (1 - interp) * lambdaPath[i - 1] + interp * lambdaPath[i]; + selectedBeta = (1 - interp) * betaPath[i - 1] + interp * betaPath[i]; + selectedIntercept = (1 - interp) * interceptPath[i - 1] + + interp * interceptPath[i]; + } } // Private functions. @@ -861,7 +1121,7 @@ inline double LARS::ComputeError(const arma::mat& matX, * Serialize the LARS model. */ template -void LARS::serialize(Archive& ar, const uint32_t /* version */) +void LARS::serialize(Archive& ar, const uint32_t version) { // If we're loading, we have to use the internal storage. if (cereal::is_loading()) @@ -891,6 +1151,15 @@ void LARS::serialize(Archive& ar, const uint32_t /* version */) ar(CEREAL_NVP(isActive)); ar(CEREAL_NVP(ignoreSet)); ar(CEREAL_NVP(isIgnored)); + + if (version > 0) + { + ar(CEREAL_NVP(selectedLambda1)); + ar(CEREAL_NVP(selectedIndex)); + ar(CEREAL_NVP(selectedBeta)); + ar(CEREAL_NVP(selectedIntercept)); + ar(CEREAL_NVP(offsetY)); + } } } // namespace mlpack From 0008e0f71585d8d52bb9e868665aeb552f0900d3 Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Tue, 5 Dec 2023 23:20:36 -0500 Subject: [PATCH 28/91] Fix some minor bugs and clarify documentation about external Gram matrices. --- doc/user/methods/lars.md | 8 ++++++-- src/mlpack/methods/lars/lars_impl.hpp | 27 ++++++++++++++++++++------- 2 files changed, 26 insertions(+), 9 deletions(-) diff --git a/doc/user/methods/lars.md b/doc/user/methods/lars.md index f0621a8457..475c9e41f4 100644 --- a/doc/user/methods/lars.md +++ b/doc/user/methods/lars.md @@ -75,7 +75,9 @@ std::cout << arma::accu(predictions < 0) << " test points predicted to have " - ***Note:*** any precomputed Gram matrix must also match the settings of `fitIntercept` and `normalizeData`; so, if both are `true`, then `gramMatrix` must be computed on mean-centered data whose features are - normalized to have unit variance. + normalized to have unit variance. In addition, if `lambda2 > 0`, then + it is expected that `lambda2` is added to each element on the diagonal of + `gramMatrix`. --- @@ -165,7 +167,9 @@ If training is not done as part of the constructor call, it can be done with the - ***Note:*** any precomputed Gram matrix must also match the settings of `fitIntercept` and `normalizeData`; so, if both are `true`, then `gramMatrix` must be computed on mean-centered data whose features are - normalized to have unit variance. + normalized to have unit variance. In addition, if `lambda2 > 0`, then + it is expected that `lambda2` is added to each element on the diagonal of + `gramMatrix`. --- diff --git a/src/mlpack/methods/lars/lars_impl.hpp b/src/mlpack/methods/lars/lars_impl.hpp index 81a08d1232..73f8de0432 100644 --- a/src/mlpack/methods/lars/lars_impl.hpp +++ b/src/mlpack/methods/lars/lars_impl.hpp @@ -88,8 +88,19 @@ inline LARS::LARS( const double tolerance, const bool fitIntercept, const bool normalizeData) : - LARS(useCholesky, gramMatrix, lambda1, lambda2, tolerance, fitIntercept, - normalizeData) + matGram(&gramMatrix), + useCholesky(useCholesky), + lasso((lambda1 != 0)), + lambda1(lambda1), + elasticNet((lambda1 != 0) && (lambda2 != 0)), + lambda2(lambda2), + tolerance(tolerance), + fitIntercept(fitIntercept), + normalizeData(normalizeData), + selectedLambda1(lambda1), + selectedIndex(0), + selectedIntercept(0.0), + offsetY(0.0) { Train(data, responses, transposeData); } @@ -352,7 +363,7 @@ inline double LARS::Train(const arma::mat& data, const bool fitIntercept) { return Train(data, responses, transposeData, useCholesky, gramMatrix, lambda1, - lambda2, fitIntercept, tolerance, this->normalizeData); + lambda2, tolerance, fitIntercept, this->normalizeData); } inline double LARS::Train(const arma::mat& data, @@ -569,8 +580,8 @@ inline double LARS::Train(const arma::mat& matX, if (maxCorr < tolerance) break; - if ((matGram != &matGramInternal) && ((maxActiveCorr - minActiveCorr) > - 100 * std::numeric_limits::epsilon())) + if ((matGram != &matGramInternal) && + ((maxActiveCorr - minActiveCorr) / maxActiveCorr) > 1e-10) { // Construct the error message to match the user's settings. std::ostringstream oss; @@ -584,8 +595,10 @@ inline double LARS::Train(const arma::mat& matX, oss << "unit-variance (normalized) "; else oss << "non-normalized "; - oss << "data!"; - oss << std::endl; + oss << "data"; + if (lambda2 > 0.0) + oss << " with lambda2 = " << lambda2 << " added to the diagonal"; + oss << "!"; throw std::runtime_error(oss.str()); } From 06cb6ce74a8b6e27f9b3592c3bfee1571cafc754 Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Tue, 5 Dec 2023 23:25:28 -0500 Subject: [PATCH 29/91] Don't use deprecated functionality. --- .../local_coordinate_coding/lcc_impl.hpp | 7 +- .../sparse_coding/sparse_coding_impl.hpp | 7 +- src/mlpack/tests/lars_test.cpp | 105 ++++++++---------- src/mlpack/tests/serialization_test.cpp | 6 +- 4 files changed, 56 insertions(+), 69 deletions(-) diff --git a/src/mlpack/methods/local_coordinate_coding/lcc_impl.hpp b/src/mlpack/methods/local_coordinate_coding/lcc_impl.hpp index 5857758b2b..cb96bf22a9 100644 --- a/src/mlpack/methods/local_coordinate_coding/lcc_impl.hpp +++ b/src/mlpack/methods/local_coordinate_coding/lcc_impl.hpp @@ -141,14 +141,15 @@ inline void LocalCoordinateCoding::Encode(const arma::mat& data, bool useCholesky = false; // Normalization and fitting and intercept are disabled. - LARS lars(useCholesky, dictGramTD, 0.5 * lambda, 0, - 1e-16 /* default tolerance */, false, false); + LARS lars(useCholesky, 0.5 * lambda, 0, 1e-16 /* default tolerance */, + false, false); // Run LARS for this point, by making an alias of the point and passing // that. arma::vec beta = codes.unsafe_col(i); arma::rowvec responses = data.unsafe_col(i).t(); - lars.Train(dictPrime, responses, beta, false); + lars.Train(dictPrime, responses, false, useCholesky, dictGramTD); + beta = lars.Beta(); beta %= invW; // Remember, beta is an alias of codes.col(i). } } diff --git a/src/mlpack/methods/sparse_coding/sparse_coding_impl.hpp b/src/mlpack/methods/sparse_coding/sparse_coding_impl.hpp index 826a572606..e90c0fff3e 100644 --- a/src/mlpack/methods/sparse_coding/sparse_coding_impl.hpp +++ b/src/mlpack/methods/sparse_coding/sparse_coding_impl.hpp @@ -71,15 +71,16 @@ inline void SparseCoding::Encode(const arma::mat& data, bool useCholesky = true; // Intercept fitting and data normalization is disabled. - LARS lars(useCholesky, matGram, lambda1, lambda2, - 1e-16 /* default tolerance */, false, false); + LARS lars(useCholesky, lambda1, lambda2, 1e-16 /* default tolerance */, + false, false); // Create an alias of the code (using the same memory), and then LARS will // place the result directly into that; then we will not need to have an // extra copy. arma::vec code = codes.unsafe_col(i); arma::rowvec responses = data.unsafe_col(i).t(); - lars.Train(dictionary, responses, code, false); + lars.Train(dictionary, responses, false, useCholesky, matGram); + code = lars.Beta(); } } diff --git a/src/mlpack/tests/lars_test.cpp b/src/mlpack/tests/lars_test.cpp index b3a08dcb8c..121ed45c00 100644 --- a/src/mlpack/tests/lars_test.cpp +++ b/src/mlpack/tests/lars_test.cpp @@ -72,8 +72,8 @@ void LassoTest(size_t nPoints, size_t nDims, bool elasticNet, bool useCholesky, LARS lars(useCholesky, lambda1, lambda2); lars.FitIntercept(fitIntercept); lars.NormalizeData(normalizeData); - arma::vec betaOpt; - lars.Train(X, y, betaOpt); + lars.Train(X, y); + arma::vec betaOpt = lars.Beta(); if (fitIntercept) { @@ -151,12 +151,11 @@ TEST_CASE("CholeskySingularityTest", "[LARSTest]") LARS lars(true, lambda1, 0.0); lars.FitIntercept(false); lars.NormalizeData(false); - arma::vec betaOpt; - lars.Train(X, y, betaOpt); + lars.Train(X, y); - arma::vec errCorr = (X * X.t()) * betaOpt - X * y.t(); + arma::vec errCorr = (X * X.t()) * lars.Beta() - X * y.t(); - LARSVerifyCorrectness(betaOpt, errCorr, lambda1); + LARSVerifyCorrectness(lars.Beta(), errCorr, lambda1); } } @@ -179,12 +178,11 @@ TEST_CASE("NoCholeskySingularityTest", "[LARSTest]") LARS lars(false, lambda1, 0.0); lars.FitIntercept(false); lars.NormalizeData(false); - arma::vec betaOpt; - lars.Train(X, y, betaOpt); + lars.Train(X, y); - arma::vec errCorr = (X * X.t()) * betaOpt - X * y.t(); + arma::vec errCorr = (X * X.t()) * lars.Beta() - X * y.t(); - LARSVerifyCorrectness(betaOpt, errCorr, lambda1); + LARSVerifyCorrectness(lars.Beta(), errCorr, lambda1); } } @@ -208,12 +206,11 @@ TEST_CASE("PredictTest", "[LARSTest]") LARS lars(useCholesky, lambda1, lambda2); lars.FitIntercept(false); lars.NormalizeData(false); - arma::vec betaOpt; - lars.Train(X, y, betaOpt); + lars.Train(X, y); // Calculate what the actual error should be with these regression // parameters. - arma::vec betaOptPred = (X * X.t()) * betaOpt; + arma::vec betaOptPred = (X * X.t()) * lars.Beta(); arma::rowvec predictions; lars.Predict(X, predictions); arma::vec adjPred = X * predictions.t(); @@ -242,8 +239,7 @@ TEST_CASE("PredictRowMajorTest", "[LARSTest]") LARS lars(false, 0, 0); lars.FitIntercept(false); lars.NormalizeData(false); - arma::vec betaOpt; - lars.Train(X, y, betaOpt); + lars.Train(X, y); // Get both row-major and column-major predictions. Make sure they are the // same. @@ -278,16 +274,15 @@ TEST_CASE("LARSRetrainTest", "[LARSTest]") LARS lars(false, 0.1, 0.1); lars.FitIntercept(false); lars.NormalizeData(false); - arma::vec betaOpt; - lars.Train(origX, origY, betaOpt); + lars.Train(origX, origY); // Now train on new data. - lars.Train(newX, newY, betaOpt); + lars.Train(newX, newY); arma::vec errCorr = (newX * trans(newX) + 0.1 * - arma::eye(75, 75)) * betaOpt - newX * newY.t(); + arma::eye(75, 75)) * lars.Beta() - newX * newY.t(); - LARSVerifyCorrectness(betaOpt, errCorr, 0.1); + LARSVerifyCorrectness(lars.Beta(), errCorr, 0.1); } /** @@ -307,16 +302,15 @@ TEST_CASE("RetrainCholeskyTest", "[LARSTest]") LARS lars(true, 0.1, 0.1); lars.FitIntercept(false); lars.NormalizeData(false); - arma::vec betaOpt; - lars.Train(origX, origY, betaOpt); + lars.Train(origX, origY); // Now train on new data. - lars.Train(newX, newY, betaOpt); + lars.Train(newX, newY); arma::vec errCorr = (newX * trans(newX) + 0.1 * - arma::eye(75, 75)) * betaOpt - newX * newY.t(); + arma::eye(75, 75)) * lars.Beta() - newX * newY.t(); - LARSVerifyCorrectness(betaOpt, errCorr, 0.1); + LARSVerifyCorrectness(lars.Beta(), errCorr, 0.1); } /** @@ -331,8 +325,8 @@ TEST_CASE("TrainingAndAccessingBetaTest", "[LARSTest]") GenerateProblem(X, y, 1000, 100); LARS lars1; - arma::vec beta; - lars1.Train(X, y, beta); + lars1.Train(X, y); + arma::vec beta = lars1.Beta(); LARS lars2; lars2.Train(X, y); @@ -354,8 +348,8 @@ TEST_CASE("TrainingConstructorWithDefaultsTest", "[LARSTest]") GenerateProblem(X, y, 1000, 100); LARS lars1; - arma::vec beta; - lars1.Train(X, y, beta); + lars1.Train(X, y); + arma::vec beta = lars1.Beta(); LARS lars2(X, y); @@ -381,8 +375,8 @@ TEST_CASE("TrainingConstructorWithNonDefaultsTest", "[LARSTest]") double lambda2 = 0.4; LARS lars1(useCholesky, lambda1, lambda2); - arma::vec beta; - lars1.Train(X, y, beta); + lars1.Train(X, y); + arma::vec beta = lars1.Beta(); LARS lars2(X, y, transposeData, useCholesky, lambda1, lambda2); @@ -411,29 +405,25 @@ TEST_CASE("LARSTrainReturnCorrelation", "[LARSTest]") // Test with Cholesky decomposition and with lasso. LARS lars1(true, lambda1, 0.0); - arma::vec betaOpt1; - double error = lars1.Train(X, y, betaOpt1); + double error = lars1.Train(X, y); REQUIRE(std::isfinite(error) == true); // Test without Cholesky decomposition and with lasso. LARS lars2(false, lambda1, 0.0); - arma::vec betaOpt2; - error = lars2.Train(X, y, betaOpt2); + error = lars2.Train(X, y); REQUIRE(std::isfinite(error) == true); // Test with Cholesky decomposition and with elasticnet. LARS lars3(true, lambda1, lambda2); - arma::vec betaOpt3; - error = lars3.Train(X, y, betaOpt3); + error = lars3.Train(X, y); REQUIRE(std::isfinite(error) == true); // Test without Cholesky decomposition and with elasticnet. LARS lars4(false, lambda1, lambda2); - arma::vec betaOpt4; - error = lars4.Train(X, y, betaOpt4); + error = lars4.Train(X, y); REQUIRE(std::isfinite(error) == true); } @@ -457,8 +447,7 @@ TEST_CASE("LARSTestComputeError", "[LARSTest]") LARS lars1(true, 0.1, 0.0); lars1.FitIntercept(false); lars1.NormalizeData(false); - arma::vec betaOpt1; - double train1 = lars1.Train(X, y, betaOpt1); + double train1 = lars1.Train(X, y); double cost = lars1.ComputeError(X, y); REQUIRE(cost <= 1); @@ -552,14 +541,13 @@ TEST_CASE("PredictFitInterceptTest", "[LARSTest]") LARS lars(useCholesky, lambda1, lambda2); lars.FitIntercept(true); lars.NormalizeData(false); - arma::vec betaOpt; - lars.Train(X, y, betaOpt); + lars.Train(X, y); const double intercept = arma::mean(y) - - arma::dot(arma::mean(X, 1), betaOpt); + arma::dot(arma::mean(X, 1), lars.Beta()); // Calculate what the actual error should be with these regression // parameters. - arma::vec betaOptPred = X.t() * betaOpt + intercept; + arma::vec betaOptPred = X.t() * lars.Beta() + intercept; arma::rowvec predictions; lars.Predict(X, predictions); arma::vec adjPred = predictions.t(); @@ -598,12 +586,11 @@ TEST_CASE("PredictNormalizeDataTest", "[LARSTest]") LARS lars(useCholesky, lambda1, lambda2); lars.FitIntercept(false); lars.NormalizeData(true); - arma::vec betaOpt; - lars.Train(X, y, betaOpt); + lars.Train(X, y); // Calculate what the actual error should be with these regression // parameters. - arma::vec betaOptPred = (X * X.t()) * betaOpt; + arma::vec betaOptPred = (X * X.t()) * lars.Beta(); arma::rowvec predictions; lars.Predict(X, predictions); arma::vec adjPred = X * predictions.t(); @@ -642,14 +629,13 @@ TEST_CASE("PredictFitInterceptNormalizeDataTest", "[LARSTest]") LARS lars(useCholesky, lambda1, lambda2); lars.FitIntercept(true); lars.NormalizeData(true); - arma::vec betaOpt; - lars.Train(X, y, betaOpt); + lars.Train(X, y); const double intercept = arma::mean(y) - - arma::dot(arma::mean(X, 1), betaOpt); + arma::dot(arma::mean(X, 1), lars.Beta()); // Calculate what the actual error should be with these regression // parameters. - arma::vec betaOptPred = X.t() * betaOpt + intercept; + arma::vec betaOptPred = X.t() * lars.Beta() + intercept; arma::rowvec predictions; lars.Predict(X, predictions); arma::vec adjPred = predictions.t(); @@ -715,7 +701,6 @@ TEST_CASE("LARSTestKKT", "[LARSTest]") arma::field F; F.load("lars_kkt.bin"); - arma::vec beta; bool useCholesky = true; LARS lars(useCholesky, 1.0, 0.0); @@ -730,19 +715,19 @@ TEST_CASE("LARSTestKKT", "[LARSTest]") lars.FitIntercept(false); lars.NormalizeData(false); - lars.Train(X, y, beta, false); - CheckKKT(beta, X, y, 1.0); + lars.Train(X, y, false); + CheckKKT(lars.Beta(), X, y, 1.0); // Now try when we fit an intercept too. lars.FitIntercept(true); lars.NormalizeData(false); - lars.Train(X, y, beta, false); + lars.Train(X, y, false); // Now mean-center data before the check. X.each_row() -= xMean; y -= yMean; - CheckKKT(beta, X, y, 1.0); + CheckKKT(lars.Beta(), X, y, 1.0); X.each_row() += xMean; y += yMean; @@ -750,9 +735,10 @@ TEST_CASE("LARSTestKKT", "[LARSTest]") // Now try when we normalize the data. lars.FitIntercept(false); lars.NormalizeData(true); - lars.Train(X, y, beta, false); + lars.Train(X, y, false); X.each_row() /= xStds; + arma::vec beta = lars.Beta(); beta %= xStds.t(); CheckKKT(beta, X, y, 1.0); @@ -761,7 +747,8 @@ TEST_CASE("LARSTestKKT", "[LARSTest]") lars.FitIntercept(true); lars.NormalizeData(true); - lars.Train(X, y, beta, false); + lars.Train(X, y, false); + beta = lars.Beta(); X.each_row() -= xMean; X.each_row() /= xStds; diff --git a/src/mlpack/tests/serialization_test.cpp b/src/mlpack/tests/serialization_test.cpp index da4f87a6d7..b0d12db35e 100644 --- a/src/mlpack/tests/serialization_test.cpp +++ b/src/mlpack/tests/serialization_test.cpp @@ -1062,8 +1062,7 @@ TEST_CASE("LARSTest", "[SerializationTest]") arma::rowvec y = beta.t() * X; LARS lars(true, 0.1, 0.1); - arma::vec betaOpt; - lars.Train(X, y, betaOpt); + lars.Train(X, y); // Now, serialize. LARS xmlLars(false, 0.5, 0.0), binaryLars(true, 1.0, 0.0), @@ -1073,8 +1072,7 @@ TEST_CASE("LARSTest", "[SerializationTest]") arma::mat jsonX = arma::randn(25, 150); arma::vec jsonBeta = arma::randn(25, 1); arma::rowvec jsonY = jsonBeta.t() * jsonX; - arma::vec jsonBetaOpt; - jsonLars.Train(jsonX, jsonY, jsonBetaOpt); + jsonLars.Train(jsonX, jsonY); SerializeObjectAll(lars, xmlLars, binaryLars, jsonLars); From e810bdb94d5ee623221bd8d088ddaa76af120a58 Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Tue, 5 Dec 2023 23:25:40 -0500 Subject: [PATCH 30/91] Test all the new constructor and Train() variants, as well as SelectBeta(). --- src/mlpack/tests/lars_test.cpp | 343 +++++++++++++++++++++++++++++++++ 1 file changed, 343 insertions(+) diff --git a/src/mlpack/tests/lars_test.cpp b/src/mlpack/tests/lars_test.cpp index 121ed45c00..c9b64a1663 100644 --- a/src/mlpack/tests/lars_test.cpp +++ b/src/mlpack/tests/lars_test.cpp @@ -758,3 +758,346 @@ TEST_CASE("LARSTestKKT", "[LARSTest]") CheckKKT(beta, X, y, 1.0); } } + +// Check that all variants of constructors appear to work. +TEST_CASE("LARSConstructorVariantTest", "[LARSTest]") +{ + // The results of the training are not all that important here; the more + // important thing is just that all the overloads compile properly. We do + // some basic sanity checks on the trained model nonetheless. + arma::mat X; + arma::rowvec y; + + GenerateProblem(X, y, 1000, 100); + arma::mat Xt = X.t(); + + const arma::vec xMean = arma::mean(X, 1); + arma::vec xStds = arma::stddev(X, 0, 1); + xStds.replace(0.0, 1.0); + const double yMean = arma::mean(y); + + arma::mat centeredX = X.each_col() - xMean; + arma::rowvec centeredY = y - yMean; + + arma::mat centeredUnitX = centeredX.each_col() / xStds; + + arma::mat matGram = X * X.t(); + arma::mat centeredUnitMatGram = centeredUnitX * centeredUnitX.t(); + + LARS l1; + LARS l2(false, 0.1, 0.2, 1e-15, false, false); + LARS l3(X, y); + LARS l4(Xt, y, false); + LARS l5(Xt, y, false, false); + LARS l6(Xt, y, false, false, 0.1); + LARS l7(X, y, true, false, 0.11, 0.01); + LARS l8(X, y, true, false, 0.12, 0.02, 1e-8); + LARS l9(centeredX, centeredY, true, false, 0.13, 0.03, 1e-7, false); + LARS l10(centeredUnitX, centeredY, true, false, 0.14, 0.04, 1e-6, false, + false); + + REQUIRE(l1.BetaPath().size() == 0); + + REQUIRE(l2.BetaPath().size() == 0); + REQUIRE(l2.UseCholesky() == false); + REQUIRE(l2.Lambda1() == Approx(0.1)); + REQUIRE(l2.Lambda2() == Approx(0.2)); + REQUIRE(l2.Tolerance() == Approx(1e-15)); + REQUIRE(l2.FitIntercept() == false); + REQUIRE(l2.NormalizeData() == false); + + REQUIRE(l3.Beta().n_elem == X.n_rows); + + REQUIRE(l4.Beta().n_elem == X.n_rows); + + REQUIRE(l5.Beta().n_elem == X.n_rows); + REQUIRE(l5.UseCholesky() == false); + + REQUIRE(l6.Beta().n_elem == X.n_rows); + REQUIRE(l6.UseCholesky() == false); + REQUIRE(l6.Lambda1() == Approx(0.1)); + + REQUIRE(l7.Beta().n_elem == X.n_rows); + REQUIRE(l7.UseCholesky() == false); + REQUIRE(l7.Lambda1() == Approx(0.11)); + REQUIRE(l7.Lambda2() == Approx(0.01)); + + REQUIRE(l8.Beta().n_elem == X.n_rows); + REQUIRE(l8.UseCholesky() == false); + REQUIRE(l8.Lambda1() == Approx(0.12)); + REQUIRE(l8.Lambda2() == Approx(0.02)); + REQUIRE(l8.Tolerance() == Approx(1e-8)); + + REQUIRE(l9.Beta().n_elem == X.n_rows); + REQUIRE(l9.UseCholesky() == false); + REQUIRE(l9.Lambda1() == Approx(0.13)); + REQUIRE(l9.Lambda2() == Approx(0.03)); + REQUIRE(l9.Tolerance() == Approx(1e-7)); + REQUIRE(l9.FitIntercept() == false); + + REQUIRE(l10.Beta().n_elem == X.n_rows); + REQUIRE(l10.UseCholesky() == false); + REQUIRE(l10.Lambda1() == Approx(0.14)); + REQUIRE(l10.Lambda2() == Approx(0.04)); + REQUIRE(l10.Tolerance() == Approx(1e-6)); + REQUIRE(l10.FitIntercept() == false); + REQUIRE(l10.NormalizeData() == false); + + // Now check constructors where we specify the Gram matrix. + + const size_t dim = centeredUnitMatGram.n_rows; + LARS l11(X, y, true, false, centeredUnitMatGram); + LARS l12(Xt, y, false, false, centeredUnitMatGram, 0.1); + + // If lambda2 > 0, then we have to adjust the Gram matrix to account for that. + arma::mat centeredUnitMatGramL13 = centeredUnitMatGram + + 0.01 * arma::eye(dim, dim); + LARS l13(Xt, y, false, false, centeredUnitMatGramL13, 0.11, 0.01); + + arma::mat centeredUnitMatGramL14 = centeredUnitMatGram + + 0.02 * arma::eye(dim, dim); + LARS l14(Xt, y, false, false, centeredUnitMatGramL14, 0.12, 0.02, 1e-15); + + arma::mat centeredUnitMatGramL15 = centeredUnitMatGram + + 0.03 * arma::eye(dim, dim); + LARS l15(centeredX, centeredY, true, false, centeredUnitMatGramL15, 0.13, + 0.03, 1e-14, false); + + arma::mat centeredUnitMatGramL16 = centeredUnitMatGram + + 0.04 * arma::eye(dim, dim); + LARS l16(centeredUnitX, centeredY, true, false, centeredUnitMatGramL16, 0.14, + 0.04, 1e-13, false, false); + + REQUIRE(l11.Beta().n_elem == X.n_rows); + REQUIRE(l11.UseCholesky() == false); + + REQUIRE(l12.Beta().n_elem == X.n_rows); + REQUIRE(l12.UseCholesky() == false); + REQUIRE(l12.Lambda1() == Approx(0.1)); + + REQUIRE(l13.Beta().n_elem == X.n_rows); + REQUIRE(l13.UseCholesky() == false); + REQUIRE(l13.Lambda1() == Approx(0.11)); + REQUIRE(l13.Lambda2() == Approx(0.01)); + + REQUIRE(l14.Beta().n_elem == X.n_rows); + REQUIRE(l14.UseCholesky() == false); + REQUIRE(l14.Lambda1() == Approx(0.12)); + REQUIRE(l14.Lambda2() == Approx(0.02)); + REQUIRE(l14.Tolerance() == Approx(1e-15)); + + REQUIRE(l15.Beta().n_elem == X.n_rows); + REQUIRE(l15.UseCholesky() == false); + REQUIRE(l15.Lambda1() == Approx(0.13)); + REQUIRE(l15.Lambda2() == Approx(0.03)); + REQUIRE(l15.Tolerance() == Approx(1e-14)); + REQUIRE(l15.FitIntercept() == false); + + REQUIRE(l16.Beta().n_elem == X.n_rows); + REQUIRE(l16.UseCholesky() == false); + REQUIRE(l16.Lambda1() == Approx(0.14)); + REQUIRE(l16.Lambda2() == Approx(0.04)); + REQUIRE(l16.Tolerance() == Approx(1e-13)); + REQUIRE(l16.FitIntercept() == false); + REQUIRE(l16.NormalizeData() == false); +} + +// Check that all variants of Train() appear to work. +TEST_CASE("LARSTrainVariantTest", "[LARSTest]") +{ + // The results of the training are not all that important here; the more + // important thing is just that all the overloads compile properly. We do + // some basic sanity checks on the trained model nonetheless. + arma::mat X; + arma::rowvec y; + + GenerateProblem(X, y, 1000, 5); + arma::mat Xt = X.t(); + + const arma::vec xMean = arma::mean(X, 1); + arma::vec xStds = arma::stddev(X, 0, 1); + xStds.replace(0.0, 1.0); + const double yMean = arma::mean(y); + + arma::mat centeredX = X.each_col() - xMean; + arma::rowvec centeredY = y - yMean; + + arma::mat centeredUnitX = centeredX.each_col() / xStds; + + arma::mat matGram = X * X.t(); + arma::mat centeredUnitMatGram = centeredUnitX * centeredUnitX.t(); + + LARS l1, l2, l3, l4, l5, l6, l7, l8, l9, l10, l11, l12, l13, l14; + + l1.Train(X, y); + l2.Train(Xt, y, false); + l3.Train(Xt, y, false, false); + l4.Train(Xt, y, false, false, 0.1); + l5.Train(Xt, y, false, false, 0.11, 0.01); + l6.Train(Xt, y, false, false, 0.12, 0.02, 1e-15); + l7.Train(centeredX, centeredY, true, false, 0.13, 0.03, 1e-14, false); + l8.Train(centeredX, centeredY, true, false, 0.14, 0.04, 1e-13, false, false); + + l9.Train(Xt, y, false, false, centeredUnitMatGram); + l10.Train(X, y, true, false, centeredUnitMatGram, 0.15); + + // If lambda2 > 0, then we have to adjust the Gram matrix. + const size_t dim = centeredUnitMatGram.n_rows; + arma::mat centeredUnitMatGramL11 = centeredUnitMatGram + + 0.05 * arma::eye(dim, dim); + l11.Train(X, y, true, false, centeredUnitMatGramL11, 0.16, 0.05); + + arma::mat centeredUnitMatGramL12 = centeredUnitMatGram + + 0.06 * arma::eye(dim, dim); + l12.Train(X, y, true, false, centeredUnitMatGramL12, 0.17, 0.06, 1e-12); + + arma::mat centeredUnitMatGramL13 = centeredUnitMatGram + + 0.07 * arma::eye(dim, dim); + l13.Train(centeredX, centeredY, true, false, centeredUnitMatGramL13, 0.18, + 0.07, 1e-11, false); + + arma::mat centeredUnitMatGramL14 = centeredUnitMatGram + + 0.08 * arma::eye(dim, dim); + l14.Train(centeredUnitX, centeredY, true, false, centeredUnitMatGramL14, 0.19, + 0.08, 1e-10, false, false); + + REQUIRE(l1.Beta().n_elem == X.n_rows); + + REQUIRE(l2.Beta().n_elem == X.n_rows); + + REQUIRE(l3.Beta().n_elem == X.n_rows); + REQUIRE(l3.UseCholesky() == false); + + REQUIRE(l4.Beta().n_elem == X.n_rows); + REQUIRE(l4.UseCholesky() == false); + REQUIRE(l4.Lambda1() == Approx(0.1)); + + REQUIRE(l5.Beta().n_elem == X.n_rows); + REQUIRE(l5.UseCholesky() == false); + REQUIRE(l5.Lambda1() == Approx(0.11)); + REQUIRE(l5.Lambda2() == Approx(0.01)); + + REQUIRE(l6.Beta().n_elem == X.n_rows); + REQUIRE(l6.UseCholesky() == false); + REQUIRE(l6.Lambda1() == Approx(0.12)); + REQUIRE(l6.Lambda2() == Approx(0.02)); + REQUIRE(l6.Tolerance() == Approx(1e-15)); + + REQUIRE(l7.Beta().n_elem == X.n_rows); + REQUIRE(l7.UseCholesky() == false); + REQUIRE(l7.Lambda1() == Approx(0.13)); + REQUIRE(l7.Lambda2() == Approx(0.03)); + REQUIRE(l7.Tolerance() == Approx(1e-14)); + REQUIRE(l7.FitIntercept() == false); + + REQUIRE(l8.Beta().n_elem == X.n_rows); + REQUIRE(l8.UseCholesky() == false); + REQUIRE(l8.Lambda1() == Approx(0.14)); + REQUIRE(l8.Lambda2() == Approx(0.04)); + REQUIRE(l8.Tolerance() == Approx(1e-13)); + REQUIRE(l8.FitIntercept() == false); + REQUIRE(l8.NormalizeData() == false); + + REQUIRE(l9.Beta().n_elem == X.n_rows); + REQUIRE(l9.UseCholesky() == false); + + REQUIRE(l10.Beta().n_elem == X.n_rows); + REQUIRE(l10.UseCholesky() == false); + REQUIRE(l10.Lambda1() == Approx(0.15)); + + REQUIRE(l11.Beta().n_elem == X.n_rows); + REQUIRE(l11.UseCholesky() == false); + REQUIRE(l11.Lambda1() == Approx(0.16)); + REQUIRE(l11.Lambda2() == Approx(0.05)); + + REQUIRE(l12.Beta().n_elem == X.n_rows); + REQUIRE(l12.UseCholesky() == false); + REQUIRE(l12.Lambda1() == Approx(0.17)); + REQUIRE(l12.Lambda2() == Approx(0.06)); + REQUIRE(l12.Tolerance() == Approx(1e-12)); + + REQUIRE(l13.Beta().n_elem == X.n_rows); + REQUIRE(l13.UseCholesky() == false); + REQUIRE(l13.Lambda1() == Approx(0.18)); + REQUIRE(l13.Lambda2() == Approx(0.07)); + REQUIRE(l13.Tolerance() == Approx(1e-11)); + REQUIRE(l13.FitIntercept() == false); + + REQUIRE(l14.Beta().n_elem == X.n_rows); + REQUIRE(l14.UseCholesky() == false); + REQUIRE(l14.Lambda1() == Approx(0.19)); + REQUIRE(l14.Lambda2() == Approx(0.08)); + REQUIRE(l14.Tolerance() == Approx(1e-10)); + REQUIRE(l14.FitIntercept() == false); + REQUIRE(l14.NormalizeData() == false); +} + +// Ensure that SelectBeta() works correctly. +TEST_CASE("LARSSelectBetaTest", "[LARSTest]") +{ + // Train a model on a randomly generated problem. Then, we will iterate + // through different selected lambda values, ensuring that the error on the + // training set is monotonically increasing. + arma::mat X; + arma::rowvec y; + + GenerateProblem(X, y, 1000, 100); + + LARS lars(X, y); + + // Ensure that the solution with no regularization is fully dense. + REQUIRE(lars.ActiveSet().size() == X.n_rows); + REQUIRE(lars.Beta().n_elem == X.n_rows); + + // Now step through numerous different lambda values. + double lastError = std::numeric_limits::max(); + for (int i = 5; i >= -5; i -= 0.1) + { + const double selLambda1 = std::pow(10.0, (double) i); + lars.SelectBeta(selLambda1); + + REQUIRE(lars.Beta().n_elem == X.n_rows); + REQUIRE(lars.SelectedLambda1() == Approx(selLambda1)); + REQUIRE(arma::accu(lars.Beta() != 0.0) == lars.ActiveSet().size()); + const double newError = lars.ComputeError(X, y); + REQUIRE(newError <= lastError); + lastError = newError; + } + + // Lastly, step through values corresponding to the lambda path exactly. + // Here we can just check that we are looking at the right model via + // Intercept() and Beta(). + for (size_t i = 0; i < lars.LambdaPath().size(); ++i) + { + lars.SelectBeta(lars.LambdaPath()[i]); + + REQUIRE(lars.SelectedLambda1() == Approx(lars.LambdaPath()[i])); + REQUIRE(lars.Intercept() == Approx(lars.InterceptPath()[i])); + REQUIRE(arma::approx_equal(lars.Beta(), lars.BetaPath()[i], "absdiff", + 1e-5)); + } +} + +// Test that SelectBeta() throws an error when the model is not trained. +TEST_CASE("LARSSelectBetaUntrainedModelTest", "[LARSTest]") +{ + LARS lars; + + REQUIRE_THROWS_AS(lars.SelectBeta(0.01), std::runtime_error); +} + +// Test that SelectBeta() throws an exception when an invalid new lambda1 value +// is specified. +TEST_CASE("LARSSelectBetaInvalidLambda1Test", "[LARSTest]") +{ + arma::mat X; + arma::rowvec y; + + GenerateProblem(X, y, 1000, 100); + + LARS lars; + lars.Lambda1() = 1.0; + lars.Train(X, y); + + REQUIRE_THROWS_AS(lars.SelectBeta(0.1), std::runtime_error); +} From bf678e919334462b3f1896971dcb9cc89e07011c Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Tue, 5 Dec 2023 23:44:02 -0500 Subject: [PATCH 31/91] Fix minor test errors. --- src/mlpack/tests/lars_test.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/mlpack/tests/lars_test.cpp b/src/mlpack/tests/lars_test.cpp index c9b64a1663..4b3375d536 100644 --- a/src/mlpack/tests/lars_test.cpp +++ b/src/mlpack/tests/lars_test.cpp @@ -1051,7 +1051,7 @@ TEST_CASE("LARSSelectBetaTest", "[LARSTest]") // Now step through numerous different lambda values. double lastError = std::numeric_limits::max(); - for (int i = 5; i >= -5; i -= 0.1) + for (double i = 5.0; i >= -5.0; i -= 0.1) { const double selLambda1 = std::pow(10.0, (double) i); lars.SelectBeta(selLambda1); @@ -1099,5 +1099,5 @@ TEST_CASE("LARSSelectBetaInvalidLambda1Test", "[LARSTest]") lars.Lambda1() = 1.0; lars.Train(X, y); - REQUIRE_THROWS_AS(lars.SelectBeta(0.1), std::runtime_error); + REQUIRE_THROWS_AS(lars.SelectBeta(0.1), std::invalid_argument); } From 3b9d4f365fe7a6e00944ebc474c6465e05deb860 Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Tue, 5 Dec 2023 23:44:16 -0500 Subject: [PATCH 32/91] Don't use deprecated functionality. --- src/mlpack/methods/lars/lars_main.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/mlpack/methods/lars/lars_main.cpp b/src/mlpack/methods/lars/lars_main.cpp index 47aa635e77..1cf135f1aa 100644 --- a/src/mlpack/methods/lars/lars_main.cpp +++ b/src/mlpack/methods/lars/lars_main.cpp @@ -176,10 +176,9 @@ void BINDING_FUNCTION(util::Params& params, util::Timers& timers) Log::Fatal << "Number of responses must be equal to number of rows of X!" << endl; - vec beta; arma::rowvec y = std::move(matY); timers.Start("lars_regression"); - lars->Train(matX, y, beta, false /* do not transpose */); + lars->Train(matX, y, false /* do not transpose */); timers.Stop("lars_regression"); } else // We must have --input_model_file. From ed52fd833fc21a38d33ec6fcb401731f3859dc89 Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Tue, 5 Dec 2023 23:44:56 -0500 Subject: [PATCH 33/91] Move implementations to lars_impl.hpp, and set activeSet in SelectBeta(). --- src/mlpack/methods/lars/lars.hpp | 56 ++++--------------- src/mlpack/methods/lars/lars_impl.hpp | 77 +++++++++++++++++++++++++++ 2 files changed, 87 insertions(+), 46 deletions(-) diff --git a/src/mlpack/methods/lars/lars.hpp b/src/mlpack/methods/lars/lars.hpp index b744a80a1e..10d3a783eb 100644 --- a/src/mlpack/methods/lars/lars.hpp +++ b/src/mlpack/methods/lars/lars.hpp @@ -433,69 +433,29 @@ class LARS //! Get whether or not to fit an intercept. bool FitIntercept() const { return fitIntercept; } //! Modify whether or not to fit an intercept. - void FitIntercept(const bool newFitIntercept) - { - // If we are storing a Gram matrix internally, but now will be normalizing - // data, then the Gram matrix we have computed is incorrect and needs to be - // recomputed. - if (fitIntercept != newFitIntercept) - { - if (matGram != &matGramInternal) - { - throw std::invalid_argument("LARS::FitIntercept(): cannot change value " - "when an external Gram matrix was specified!"); - } - - fitIntercept = newFitIntercept; - matGramInternal.clear(); - } - } + void FitIntercept(const bool newFitIntercept); //! Get whether or not to normalize data during training. bool NormalizeData() const { return normalizeData; } //! Modify whether or not to normalize data during training. - void NormalizeData(const bool newNormalizeData) - { - // If we are storing a Gram matrix internally, but now will be normalizing - // data, then the Gram matrix we have computed is incorrect and needs to be - // recomputed. - if (normalizeData != newNormalizeData) - { - if (matGram != &matGramInternal) - { - throw std::invalid_argument("LARS::NormalizeData(): cannot change value" - " when an external Gram matrix was specified!"); - } + void NormalizeData(const bool newNormalizeData); - normalizeData = newNormalizeData; - matGramInternal.clear(); - } - } - - //! Access the set of active dimensions. - const std::vector& ActiveSet() const { return activeSet; } + //! Access the set of active dimensions in the currently selected model. + const std::vector& ActiveSet() const; //! Access the set of coefficients after each iteration; the solution is the //! last element. const std::vector& BetaPath() const { return betaPath; } //! Access the solution coefficients - const arma::vec& Beta() const - { - return (selectedIndex < betaPath.size()) ? - betaPath[selectedIndex] : selectedBeta; - } + const arma::vec& Beta() const; //! Access the set of values for lambda1 after each iteration; the solution is //! the last element. const std::vector& LambdaPath() const { return lambdaPath; } //! Return the intercept (if fitted, otherwise 0). - double Intercept() const - { - return (selectedIndex < interceptPath.size()) ? - interceptPath[selectedIndex] : selectedIntercept; - } + double Intercept() const; //! Return the intercept path (the intercept for every model). const std::vector& InterceptPath() const { return interceptPath; } @@ -589,6 +549,10 @@ class LARS //! Selected intercept, if selectedLambda1 is not in lambdaPath. double selectedIntercept; + //! Selected active set of dimensions, if selectedLambda1 is not the last + //! element in the path. + std::vector selectedActiveSet; + //! Might be needed to compute the intercept for other lambda values. double offsetY; diff --git a/src/mlpack/methods/lars/lars_impl.hpp b/src/mlpack/methods/lars/lars_impl.hpp index 73f8de0432..1be637cf71 100644 --- a/src/mlpack/methods/lars/lars_impl.hpp +++ b/src/mlpack/methods/lars/lars_impl.hpp @@ -127,6 +127,7 @@ inline LARS::LARS(const LARS& other) : selectedIndex(other.selectedIndex), selectedBeta(other.selectedBeta), selectedIntercept(other.selectedIntercept), + selectedActiveSet(other.selectedActiveSet), offsetY(other.offsetY), isActive(other.isActive), ignoreSet(other.ignoreSet), @@ -157,6 +158,7 @@ inline LARS::LARS(LARS&& other) : selectedIndex(std::move(other.selectedIndex)), selectedBeta(std::move(other.selectedBeta)), selectedIntercept(std::move(other.selectedIntercept)), + selectedActiveSet(std::move(other.selectedActiveSet)), offsetY(std::move(other.offsetY)), isActive(std::move(other.isActive)), ignoreSet(std::move(other.ignoreSet)), @@ -191,6 +193,7 @@ inline LARS& LARS::operator=(const LARS& other) selectedIndex = other.selectedIndex; selectedBeta = other.selectedBeta; selectedIntercept = other.selectedIntercept; + selectedActiveSet = other.selectedActiveSet; offsetY = other.offsetY; isActive = other.isActive; ignoreSet = other.ignoreSet; @@ -223,6 +226,7 @@ inline LARS& LARS::operator=(LARS&& other) selectedIndex = std::move(other.selectedIndex); selectedBeta = std::move(other.selectedBeta); selectedIntercept = std::move(other.selectedIntercept); + selectedActiveSet = std::move(other.selectedActiveSet); offsetY = std::move(other.offsetY); activeSet = std::move(other.activeSet); isActive = std::move(other.isActive); @@ -887,6 +891,66 @@ inline void LARS::Predict(const arma::mat& points, predictions = Beta().t() * points; } +inline void LARS::FitIntercept(const bool newFitIntercept) +{ + // If we are storing a Gram matrix internally, but now will be normalizing + // data, then the Gram matrix we have computed is incorrect and needs to be + // recomputed. + if (fitIntercept != newFitIntercept) + { + if (matGram != &matGramInternal) + { + throw std::invalid_argument("LARS::FitIntercept(): cannot change value " + "when an external Gram matrix was specified!"); + } + + fitIntercept = newFitIntercept; + matGramInternal.clear(); + } +} + +inline void LARS::NormalizeData(const bool newNormalizeData) +{ + // If we are storing a Gram matrix internally, but now will be normalizing + // data, then the Gram matrix we have computed is incorrect and needs to be + // recomputed. + if (normalizeData != newNormalizeData) + { + if (matGram != &matGramInternal) + { + throw std::invalid_argument("LARS::NormalizeData(): cannot change value" + " when an external Gram matrix was specified!"); + } + + normalizeData = newNormalizeData; + matGramInternal.clear(); + } +} + +inline const std::vector& LARS::ActiveSet() const +{ + if (selectedIndex != (betaPath.size() - 1)) + return selectedActiveSet; + else + return activeSet; +} + +inline const arma::vec& LARS::Beta() const +{ + if (selectedIndex < betaPath.size()) + return betaPath[selectedIndex]; + else + return selectedBeta; +} + +inline double LARS::Intercept() const +{ + if (selectedIndex < betaPath.size()) + return interceptPath[selectedIndex]; + else + return selectedIntercept; +} + inline void LARS::SelectBeta(const double selLambda1) { if (selLambda1 < lambda1) @@ -917,6 +981,14 @@ inline void LARS::SelectBeta(const double selLambda1) // If it's an exact match, no interpolation is necessary, and we can // directly use the element from the path. selectedIndex = i; + + // However, we may need to compute the active set. + if (i != lambdaPath.size() - 1) + { + selectedActiveSet = arma::conv_to>::from( + arma::find(betaPath[i] != 0)); + } + return; } else if (selLambda1 > lambdaPath[i]) @@ -955,6 +1027,10 @@ inline void LARS::SelectBeta(const double selLambda1) selectedIntercept = (1 - interp) * interceptPath[i - 1] + interp * interceptPath[i]; } + + // Compute the active set of variables. + selectedActiveSet = arma::conv_to>::from( + arma::find(selectedBeta != 0)); } // Private functions. @@ -1171,6 +1247,7 @@ void LARS::serialize(Archive& ar, const uint32_t version) ar(CEREAL_NVP(selectedIndex)); ar(CEREAL_NVP(selectedBeta)); ar(CEREAL_NVP(selectedIntercept)); + ar(CEREAL_NVP(selectedActiveSet)); ar(CEREAL_NVP(offsetY)); } } From 6a1676994b678a9330d30fc5186917236fb7656c Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Fri, 8 Dec 2023 08:37:01 -0500 Subject: [PATCH 34/91] Template LARS to allow selecting the type to represent the model as. --- src/mlpack/core/util/arma_traits.hpp | 18 + src/mlpack/methods/lars/lars.hpp | 404 +++++++----- src/mlpack/methods/lars/lars_impl.hpp | 592 ++++++++++++------ src/mlpack/methods/lars/lars_main.cpp | 12 +- .../local_coordinate_coding/lcc_impl.hpp | 2 +- .../sparse_coding/sparse_coding_impl.hpp | 2 +- src/mlpack/tests/cv_test.cpp | 4 +- src/mlpack/tests/hpt_test.cpp | 22 +- src/mlpack/tests/lars_test.cpp | 386 +++++++----- src/mlpack/tests/serialization_test.cpp | 4 +- 10 files changed, 929 insertions(+), 517 deletions(-) diff --git a/src/mlpack/core/util/arma_traits.hpp b/src/mlpack/core/util/arma_traits.hpp index bf8f8109e0..81fd9408e6 100644 --- a/src/mlpack/core/util/arma_traits.hpp +++ b/src/mlpack/core/util/arma_traits.hpp @@ -131,4 +131,22 @@ struct GetColType> typedef arma::SpCol type; }; +template +struct GetDenseMatType +{ + typedef MatType type; // Not sure... +}; + +template +struct GetDenseMatType> +{ + typedef arma::Mat type; +}; + +template +struct GetDenseMatType> +{ + typedef arma::Mat type; +}; + #endif diff --git a/src/mlpack/methods/lars/lars.hpp b/src/mlpack/methods/lars/lars.hpp index 10d3a783eb..2e6ec7346d 100644 --- a/src/mlpack/methods/lars/lars.hpp +++ b/src/mlpack/methods/lars/lars.hpp @@ -28,9 +28,6 @@ namespace mlpack { -// beta is the estimator -// yHat is the prediction from the current estimator - /** * An implementation of LARS, a stage-wise homotopy-based algorithm for * l1-regularized linear regression (LASSO) and l1+l2 regularized linear @@ -85,9 +82,14 @@ namespace mlpack { * } * @endcode */ +template class LARS { public: + typedef typename GetColType::type ModelColType; + typedef typename GetDenseMatType::type DenseMatType; + typedef typename ModelMatType::elem_type ElemType; + /** * Set the parameters to LARS. Both lambda1 and lambda2 default to 0. * @@ -105,9 +107,9 @@ class LARS * for training. */ LARS(const bool useCholesky = false, - const double lambda1 = 0.0, - const double lambda2 = 0.0, - const double tolerance = 1e-16, + const ElemType lambda1 = 0.0, + const ElemType lambda2 = 0.0, + const ElemType tolerance = 1e-16, const bool fitIntercept = true, const bool normalizeData = true); @@ -161,13 +163,18 @@ class LARS * @param normalizeData If true, normalize all features to have unit variance * for training. */ - LARS(const arma::mat& data, - const arma::rowvec& responses, - const bool transposeData = true, + template::value + >::type> + LARS(const MatType& data, + const ResponsesType& responses, + bool transposeData = true, const bool useCholesky = false, - const double lambda1 = 0.0, - const double lambda2 = 0.0, - const double tolerance = 1e-16, + const ElemType lambda1 = 0.0, + const ElemType lambda2 = 0.0, + const ElemType tolerance = 1e-16, const bool fitIntercept = true, const bool normalizeData = true); @@ -195,14 +202,19 @@ class LARS * @param normalizeData If true, normalize all features to have unit variance * for training. */ - LARS(const arma::mat& data, - const arma::rowvec& responses, + template::value + >::type> + LARS(const MatType& data, + const ResponsesType& responses, const bool transposeData, const bool useCholesky, - const arma::mat& gramMatrix, - const double lambda1 = 0.0, - const double lambda2 = 0.0, - const double tolerance = 1e-16, + const DenseMatType& gramMatrix, + const ElemType lambda1 = 0.0, + const ElemType lambda2 = 0.0, + const ElemType tolerance = 1e-16, const bool fitIntercept = true, const bool normalizeData = true); @@ -272,54 +284,107 @@ class LARS * false otherwise. * @return minimum cost error(||y-beta*X||2 is used to calculate error). */ - double Train(const arma::mat& data, - const arma::rowvec& responses, - const bool transposeData = true); - double Train(const arma::mat& data, - const arma::rowvec& responses, - const bool transposeData, - const bool useCholesky); + // Dummy overload so MetaInfoExtractor can properly detect that LARS is a + // regression method. + template + ElemType Train(const MatType& data, + const arma::rowvec& responses, + const bool transposeData = true); - double Train(const arma::mat& data, - const arma::rowvec& responses, - const bool transposeData, - const bool useCholesky, - const double lambda1); + template::value + >::type, + typename = typename std::enable_if< + !std::is_same::value + >::type> + ElemType Train(const MatType& data, + const ResponsesType& responses, + const bool transposeData = true); - double Train(const arma::mat& data, - const arma::rowvec& responses, - const bool transposeData, - const bool useCholesky, - const double lambda1, - const double lambda2); + template::value + >::type> + ElemType Train(const MatType& data, + const ResponsesType& responses, + const bool transposeData, + const bool useCholesky); - double Train(const arma::mat& data, - const arma::rowvec& responses, - const bool transposeData, - const bool useCholesky, - const double lambda1, - const double lambda2, - const double tolerance); + template::value + >::type> + ElemType Train(const MatType& data, + const ResponsesType& responses, + const bool transposeData, + const bool useCholesky, + const ElemType lambda1); - double Train(const arma::mat& data, - const arma::rowvec& responses, - const bool transposeData, - const bool useCholesky, - const double lambda1, - const double lambda2, - const double tolerance, - const bool fitIntercept); + template::value + >::type> + ElemType Train(const MatType& data, + const ResponsesType& responses, + const bool transposeData, + const bool useCholesky, + const ElemType lambda1, + const ElemType lambda2); - double Train(const arma::mat& data, - const arma::rowvec& responses, - const bool transposeData, - const bool useCholesky, - const double lambda1, - const double lambda2, - const double tolerance, - const bool fitIntercept, - const bool normalizeData); + template::value + >::type> + ElemType Train(const MatType& data, + const ResponsesType& responses, + const bool transposeData, + const bool useCholesky, + const ElemType lambda1, + const ElemType lambda2, + const ElemType tolerance); + + template::value + >::type> + ElemType Train(const MatType& data, + const ResponsesType& responses, + const bool transposeData, + const bool useCholesky, + const ElemType lambda1, + const ElemType lambda2, + const ElemType tolerance, + const bool fitIntercept); + + template::value + >::type> + ElemType Train(const MatType& data, + const ResponsesType& responses, + const bool transposeData, + const bool useCholesky, + const ElemType lambda1, + const ElemType lambda2, + const ElemType tolerance, + const bool fitIntercept, + const bool normalizeData); /** * Run LARS with a precomputed Gram matrix. The input matrix (like all mlpack @@ -338,56 +403,92 @@ class LARS * false otherwise. * @return minimum cost error(||y-beta*X||2 is used to calculate error). */ - double Train(const arma::mat& data, - const arma::rowvec& responses, - const bool transposeData, - const bool useCholesky, - const arma::mat& gramMatrix); + template::value + >::type> + ElemType Train(const MatType& data, + const ResponsesType& responses, + const bool transposeData, + const bool useCholesky, + const DenseMatType& gramMatrix); - double Train(const arma::mat& data, - const arma::rowvec& responses, - const bool transposeData, - const bool useCholesky, - const arma::mat& gramMatrix, - const double lambda1); + template::value + >::type> + ElemType Train(const MatType& data, + const ResponsesType& responses, + const bool transposeData, + const bool useCholesky, + const DenseMatType& gramMatrix, + const ElemType lambda1); - double Train(const arma::mat& data, - const arma::rowvec& responses, - const bool transposeData, - const bool useCholesky, - const arma::mat& gramMatrix, - const double lambda1, - const double lambda2); + template::value + >::type> + ElemType Train(const MatType& data, + const ResponsesType& responses, + const bool transposeData, + const bool useCholesky, + const DenseMatType& gramMatrix, + const ElemType lambda1, + const ElemType lambda2); - double Train(const arma::mat& data, - const arma::rowvec& responses, - const bool transposeData, - const bool useCholesky, - const arma::mat& gramMatrix, - const double lambda1, - const double lambda2, - const double tolerance); + template::value + >::type> + ElemType Train(const MatType& data, + const ResponsesType& responses, + const bool transposeData, + const bool useCholesky, + const DenseMatType& gramMatrix, + const ElemType lambda1, + const ElemType lambda2, + const ElemType tolerance); - double Train(const arma::mat& data, - const arma::rowvec& responses, - const bool transposeData, - const bool useCholesky, - const arma::mat& gramMatrix, - const double lambda1, - const double lambda2, - const double tolerance, - const bool fitIntercept); + template::value + >::type> + ElemType Train(const MatType& data, + const ResponsesType& responses, + const bool transposeData, + const bool useCholesky, + const DenseMatType& gramMatrix, + const ElemType lambda1, + const ElemType lambda2, + const ElemType tolerance, + const bool fitIntercept); - double Train(const arma::mat& data, - const arma::rowvec& responses, - const bool transposeData, - const bool useCholesky, - const arma::mat& gramMatrix, - const double lambda1, - const double lambda2, - const double tolerance, - const bool fitIntercept, - const bool normalizeData); + template::value + >::type> + ElemType Train(const MatType& data, + const ResponsesType& responses, + const bool transposeData, + const bool useCholesky, + const DenseMatType& gramMatrix, + const ElemType lambda1, + const ElemType lambda2, + const ElemType tolerance, + const bool fitIntercept, + const bool normalizeData); /** * Predict y_i for the given data point. @@ -395,7 +496,8 @@ class LARS * @param point The data point to regress on. * @return Predicted value for y_i for `point`. */ - double Predict(const arma::vec& point) const; + template + ElemType Predict(const VecType& point) const; /** * Predict y_i for each data point in the given data matrix using the @@ -406,19 +508,20 @@ class LARS * @param rowMajor Should be true if the data points matrix is row-major and * false otherwise. */ - void Predict(const arma::mat& points, - arma::rowvec& predictions, + template + void Predict(const MatType& points, + ResponsesType& predictions, const bool rowMajor = false) const; //! Get the L1 regularization coefficient. - double Lambda1() const { return lambda1; } + ElemType Lambda1() const { return lambda1; } //! Modify the L1 regularization coefficient. - double& Lambda1() { return lambda1; } + ElemType& Lambda1() { return lambda1; } //! Get the L2 regularization coefficient. - double Lambda2() const { return lambda2; } + ElemType Lambda2() const { return lambda2; } //! Modify the L2 regularization coefficient. - double& Lambda2() { return lambda2; } + ElemType& Lambda2() { return lambda2; } //! Get whether to use the Cholesky decomposition. bool UseCholesky() const { return useCholesky; } @@ -426,9 +529,9 @@ class LARS bool& UseCholesky() { return useCholesky; } //! Get the tolerance for maximum correlation during training. - double Tolerance() const { return tolerance; } + ElemType Tolerance() const { return tolerance; } //! Modify the tolerance for maximum correlation during training. - double& Tolerance() { return tolerance; } + ElemType& Tolerance() { return tolerance; } //! Get whether or not to fit an intercept. bool FitIntercept() const { return fitIntercept; } @@ -445,30 +548,30 @@ class LARS //! Access the set of coefficients after each iteration; the solution is the //! last element. - const std::vector& BetaPath() const { return betaPath; } + const std::vector& BetaPath() const { return betaPath; } //! Access the solution coefficients - const arma::vec& Beta() const; + const ModelColType& Beta() const; //! Access the set of values for lambda1 after each iteration; the solution is //! the last element. - const std::vector& LambdaPath() const { return lambdaPath; } + const std::vector& LambdaPath() const { return lambdaPath; } //! Return the intercept (if fitted, otherwise 0). - double Intercept() const; + ElemType Intercept() const; //! Return the intercept path (the intercept for every model). - const std::vector& InterceptPath() const { return interceptPath; } + const std::vector& InterceptPath() const { return interceptPath; } //! Set the model to use the given lambda1 value in the path. - void SelectBeta(const double lambda1); + void SelectBeta(const ElemType lambda1); //! Get the L1 penalty parameter corresponding to the currently selected //! model. - double SelectedLambda1() const { return selectedLambda1; } + ElemType SelectedLambda1() const { return selectedLambda1; } //! Access the upper triangular cholesky factor. - const arma::mat& MatUtriCholFactor() const { return matUtriCholFactor; } + const DenseMatType& MatUtriCholFactor() const { return matUtriCholFactor; } /** * Serialize the LARS model. @@ -488,19 +591,20 @@ class LARS * false otherwise. * @return The minimum cost error. */ - double ComputeError(const arma::mat& matX, - const arma::rowvec& y, - const bool rowMajor = false); + template + ElemType ComputeError(const MatType& matX, + const ResponsesType& y, + const bool rowMajor = false); private: //! Gram matrix. - arma::mat matGramInternal; + DenseMatType matGramInternal; //! Pointer to the Gram matrix we will use. - const arma::mat* matGram; + const DenseMatType* matGram; //! Upper triangular cholesky factor; initially 0x0 matrix. - arma::mat matUtriCholFactor; + DenseMatType matUtriCholFactor; //! Whether or not to use Cholesky decomposition when solving linear system. bool useCholesky; @@ -508,15 +612,15 @@ class LARS //! True if this is the LASSO problem. bool lasso; //! Regularization parameter for l1 penalty. - double lambda1; + ElemType lambda1; //! True if this is the elastic net problem. bool elasticNet; //! Regularization parameter for l2 penalty. - double lambda2; + ElemType lambda2; //! Tolerance for main loop. - double tolerance; + ElemType tolerance; //! Whether or not to fit an intercept. bool fitIntercept; @@ -526,35 +630,35 @@ class LARS bool normalizeData; //! Solution path. - std::vector betaPath; + std::vector betaPath; //! Value of lambda_1 for each solution in solution path. - std::vector lambdaPath; + std::vector lambdaPath; //! Intercept (only if fitIntercept is true). - std::vector interceptPath; + std::vector interceptPath; //! Active set of dimensions. std::vector activeSet; //! Selected lambda1 value for Predict(). - double selectedLambda1; + ElemType selectedLambda1; //! Index of selected beta (if selectedLambda1 is in lambdaPath). size_t selectedIndex; //! Selected beta, if selectedLambda1 is not in lambdaPath. - arma::vec selectedBeta; + ModelColType selectedBeta; //! Selected intercept, if selectedLambda1 is not in lambdaPath. - double selectedIntercept; + ElemType selectedIntercept; //! Selected active set of dimensions, if selectedLambda1 is not the last //! element in the path. std::vector selectedActiveSet; //! Might be needed to compute the intercept for other lambda values. - double offsetY; + ElemType offsetY; //! Active set membership indicator (for each dimension). std::vector isActive; @@ -588,21 +692,25 @@ class LARS */ void Ignore(const size_t varInd); - // compute "equiangular" direction in output space - void ComputeYHatDirection(const arma::mat& matX, - const arma::vec& betaDirection, - arma::vec& yHatDirection); + // Compute "equiangular" direction in output space. + template + void ComputeYHatDirection(const MatType& matX, + const VecType& betaDirection, + VecType& yHatDirection); - // interpolate to compute last solution vector + // Interpolate to compute last solution vector. void InterpolateBeta(); - void CholeskyInsert(const arma::vec& newX, const arma::mat& X); + template + void CholeskyInsert(const VecType& newX, const MatType& X); - void CholeskyInsert(double sqNormNewX, const arma::vec& newGramCol); + template + void CholeskyInsert(ElemType sqNormNewX, const VecType& newGramCol); - void GivensRotate(const arma::vec::fixed<2>& x, - arma::vec::fixed<2>& rotatedX, - arma::mat& G); + template + void GivensRotate(const typename arma::Col::fixed<2>& x, + typename arma::Col::fixed<2>& rotatedX, + MatType& G); void CholeskyDelete(const size_t colToKill); }; diff --git a/src/mlpack/methods/lars/lars_impl.hpp b/src/mlpack/methods/lars/lars_impl.hpp index 1be637cf71..cb1a45fbe8 100644 --- a/src/mlpack/methods/lars/lars_impl.hpp +++ b/src/mlpack/methods/lars/lars_impl.hpp @@ -17,11 +17,12 @@ namespace mlpack { -inline LARS::LARS( +template +inline LARS::LARS( const bool useCholesky, - const double lambda1, - const double lambda2, - const double tolerance, + const typename LARS::ElemType lambda1, + const typename LARS::ElemType lambda2, + const typename LARS::ElemType tolerance, const bool fitIntercept, const bool normalizeData) : matGram(&matGramInternal), @@ -39,7 +40,9 @@ inline LARS::LARS( offsetY(0.0) { /* Nothing left to do. */ } -inline LARS::LARS( +template +mlpack_deprecated +inline LARS::LARS( const bool useCholesky, const arma::mat& gramMatrix, const double lambda1, @@ -62,14 +65,16 @@ inline LARS::LARS( offsetY(0.0) { /* Nothing left to do */ } -inline LARS::LARS( - const arma::mat& data, - const arma::rowvec& responses, +template +template +inline LARS::LARS( + const MatType& data, + const ResponsesType& responses, const bool transposeData, const bool useCholesky, - const double lambda1, - const double lambda2, - const double tolerance, + const typename LARS::ElemType lambda1, + const typename LARS::ElemType lambda2, + const typename LARS::ElemType tolerance, const bool fitIntercept, const bool normalizeData) : LARS(useCholesky, lambda1, lambda2, tolerance, fitIntercept, normalizeData) @@ -77,15 +82,17 @@ inline LARS::LARS( Train(data, responses, transposeData); } -inline LARS::LARS( - const arma::mat& data, - const arma::rowvec& responses, +template +template +inline LARS::LARS( + const MatType& data, + const ResponsesType& responses, const bool transposeData, const bool useCholesky, - const arma::mat& gramMatrix, - const double lambda1, - const double lambda2, - const double tolerance, + const typename LARS::DenseMatType& gramMatrix, + const typename LARS::ElemType lambda1, + const typename LARS::ElemType lambda2, + const typename LARS::ElemType tolerance, const bool fitIntercept, const bool normalizeData) : matGram(&gramMatrix), @@ -106,7 +113,8 @@ inline LARS::LARS( } // Copy Constructor. -inline LARS::LARS(const LARS& other) : +template +inline LARS::LARS(const LARS& other) : matGramInternal(other.matGramInternal), matGram(other.matGram != &other.matGramInternal ? other.matGram : &matGramInternal), @@ -137,7 +145,8 @@ inline LARS::LARS(const LARS& other) : } // Move constructor. -inline LARS::LARS(LARS&& other) : +template +inline LARS::LARS(LARS&& other) : matGramInternal(std::move(other.matGramInternal)), matGram(other.matGram != &other.matGramInternal ? other.matGram : &matGramInternal), @@ -168,7 +177,9 @@ inline LARS::LARS(LARS&& other) : } // Copy operator. -inline LARS& LARS::operator=(const LARS& other) +template +inline LARS& LARS::operator=( + const LARS& other) { if (&other == this) return *this; @@ -202,7 +213,9 @@ inline LARS& LARS::operator=(const LARS& other) } // Move Operator. -inline LARS& LARS::operator=(LARS&& other) +template +inline LARS& LARS::operator=( + LARS&& other) { if (&other == this) return *this; @@ -235,18 +248,23 @@ inline LARS& LARS::operator=(LARS&& other) return *this; } +template mlpack_deprecated -inline double LARS::Train(const arma::mat& matX, - const arma::rowvec& y, - arma::vec& beta, - const bool transposeData) +inline double LARS::Train(const arma::mat& matX, + const arma::rowvec& y, + arma::vec& beta, + const bool transposeData) { const double result = Train(matX, y, transposeData); beta = betaPath.back(); return result; } -inline double LARS::Train(const arma::mat& data, +// Dummy overload for MetaInfoExtractor. +template +template +inline typename LARS::ElemType +LARS::Train(const MatType& data, const arma::rowvec& responses, const bool transposeData) { @@ -254,8 +272,22 @@ inline double LARS::Train(const arma::mat& data, this->lambda2, this->tolerance, this->fitIntercept, this->normalizeData); } -inline double LARS::Train(const arma::mat& data, - const arma::rowvec& responses, +template +template +inline typename LARS::ElemType +LARS::Train(const MatType& data, + const ResponsesType& responses, + const bool transposeData) +{ + return Train(data, responses, transposeData, this->useCholesky, this->lambda1, + this->lambda2, this->tolerance, this->fitIntercept, this->normalizeData); +} + +template +template +inline typename LARS::ElemType +LARS::Train(const MatType& data, + const ResponsesType& responses, const bool transposeData, const bool useCholesky) { @@ -263,123 +295,159 @@ inline double LARS::Train(const arma::mat& data, this->lambda2, this->tolerance, this->fitIntercept, this->normalizeData); } -inline double LARS::Train(const arma::mat& data, - const arma::rowvec& responses, +template +template +inline typename LARS::ElemType +LARS::Train(const MatType& data, + const ResponsesType& responses, const bool transposeData, const bool useCholesky, - const double lambda1) + const typename LARS::ElemType lambda1) { return Train(data, responses, transposeData, useCholesky, lambda1, this->lambda2, this->tolerance, this->fitIntercept, this->normalizeData); } -inline double LARS::Train(const arma::mat& data, - const arma::rowvec& responses, +template +template +inline typename LARS::ElemType +LARS::Train(const MatType& data, + const ResponsesType& responses, const bool transposeData, const bool useCholesky, - const double lambda1, - const double lambda2) + const typename LARS::ElemType lambda1, + const typename LARS::ElemType lambda2) { return Train(data, responses, transposeData, useCholesky, lambda1, lambda2, this->tolerance, this->fitIntercept, this->normalizeData); } -inline double LARS::Train(const arma::mat& data, - const arma::rowvec& responses, +template +template +inline typename LARS::ElemType +LARS::Train(const MatType& data, + const ResponsesType& responses, const bool transposeData, const bool useCholesky, - const double lambda1, - const double lambda2, - const double tolerance) + const typename LARS::ElemType lambda1, + const typename LARS::ElemType lambda2, + const typename LARS::ElemType tolerance) { return Train(data, responses, transposeData, useCholesky, lambda1, lambda2, tolerance, this->fitIntercept, this->normalizeData); } -inline double LARS::Train(const arma::mat& data, - const arma::rowvec& responses, +template +template +inline typename LARS::ElemType +LARS::Train(const MatType& data, + const ResponsesType& responses, const bool transposeData, const bool useCholesky, - const double lambda1, - const double lambda2, - const double tolerance, + const typename LARS::ElemType lambda1, + const typename LARS::ElemType lambda2, + const typename LARS::ElemType tolerance, const bool fitIntercept) { return Train(data, responses, transposeData, useCholesky, lambda1, lambda2, tolerance, fitIntercept, this->normalizeData); } -inline double LARS::Train(const arma::mat& data, - const arma::rowvec& responses, - const bool transposeData, - const bool useCholesky, - const arma::mat& gramMatrix) +template +template +inline typename LARS::ElemType +LARS::Train( + const MatType& data, + const ResponsesType& responses, + const bool transposeData, + const bool useCholesky, + const typename LARS::DenseMatType& gramMatrix) { return Train(data, responses, transposeData, useCholesky, gramMatrix, this->lambda1, this->lambda2, this->tolerance, this->fitIntercept, this->normalizeData); } -inline double LARS::Train(const arma::mat& data, - const arma::rowvec& responses, - const bool transposeData, - const bool useCholesky, - const arma::mat& gramMatrix, - const double lambda1) +template +template +inline typename LARS::ElemType +LARS::Train( + const MatType& data, + const ResponsesType& responses, + const bool transposeData, + const bool useCholesky, + const typename LARS::DenseMatType& gramMatrix, + const typename LARS::ElemType lambda1) { return Train(data, responses, transposeData, useCholesky, gramMatrix, lambda1, this->lambda2, this->tolerance, this->fitIntercept, this->normalizeData); } -inline double LARS::Train(const arma::mat& data, - const arma::rowvec& responses, - const bool transposeData, - const bool useCholesky, - const arma::mat& gramMatrix, - const double lambda1, - const double lambda2) +template +template +inline typename LARS::ElemType +LARS::Train( + const MatType& data, + const ResponsesType& responses, + const bool transposeData, + const bool useCholesky, + const typename LARS::DenseMatType& gramMatrix, + const typename LARS::ElemType lambda1, + const typename LARS::ElemType lambda2) { return Train(data, responses, transposeData, useCholesky, gramMatrix, lambda1, lambda2, this->tolerance, this->fitIntercept, this->normalizeData); } -inline double LARS::Train(const arma::mat& data, - const arma::rowvec& responses, - const bool transposeData, - const bool useCholesky, - const arma::mat& gramMatrix, - const double lambda1, - const double lambda2, - const double tolerance) +template +template +inline typename LARS::ElemType +LARS::Train( + const MatType& data, + const ResponsesType& responses, + const bool transposeData, + const bool useCholesky, + const typename LARS::DenseMatType& gramMatrix, + const typename LARS::ElemType lambda1, + const typename LARS::ElemType lambda2, + const typename LARS::ElemType tolerance) { return Train(data, responses, transposeData, useCholesky, gramMatrix, lambda1, lambda2, tolerance, this->fitIntercept, this->normalizeData); } -inline double LARS::Train(const arma::mat& data, - const arma::rowvec& responses, - const bool transposeData, - const bool useCholesky, - const arma::mat& gramMatrix, - const double lambda1, - const double lambda2, - const double tolerance, - const bool fitIntercept) +template +template +inline typename LARS::ElemType +LARS::Train( + const MatType& data, + const ResponsesType& responses, + const bool transposeData, + const bool useCholesky, + const typename LARS::DenseMatType& gramMatrix, + const typename LARS::ElemType lambda1, + const typename LARS::ElemType lambda2, + const typename LARS::ElemType tolerance, + const bool fitIntercept) { return Train(data, responses, transposeData, useCholesky, gramMatrix, lambda1, lambda2, tolerance, fitIntercept, this->normalizeData); } -inline double LARS::Train(const arma::mat& data, - const arma::rowvec& responses, - const bool transposeData, - const bool useCholesky, - const arma::mat& gramMatrix, - const double lambda1, - const double lambda2, - const double tolerance, - const bool fitIntercept, - const bool normalizeData) +template +template +inline typename LARS::ElemType +LARS::Train( + const MatType& data, + const ResponsesType& responses, + const bool transposeData, + const bool useCholesky, + const typename LARS::DenseMatType& gramMatrix, + const typename LARS::ElemType lambda1, + const typename LARS::ElemType lambda2, + const typename LARS::ElemType tolerance, + const bool fitIntercept, + const bool normalizeData) { // Set Gram matrix. matGramInternal.clear(); @@ -389,13 +457,16 @@ inline double LARS::Train(const arma::mat& data, tolerance, fitIntercept, normalizeData); } -inline double LARS::Train(const arma::mat& matX, - const arma::rowvec& y, +template +template +inline typename LARS::ElemType +LARS::Train(const MatType& matX, + const ResponsesType& y, const bool transposeData, const bool useCholesky, - const double lambda1, - const double lambda2, - const double tolerance, + const typename LARS::ElemType lambda1, + const typename LARS::ElemType lambda2, + const typename LARS::ElemType tolerance, const bool fitIntercept, const bool normalizeData) { @@ -422,20 +493,20 @@ inline double LARS::Train(const arma::mat& matX, elasticNet = (lambda1 != 0 && lambda2 != 0); // This matrix may end up holding the transpose -- if necessary. - arma::mat dataTrans; + MatType dataTrans; // This vector may hold zero-centered responses, if necessary. - arma::rowvec yCentered; + ResponsesType yCentered; // dataRef is row-major. We can reuse the given matX, but only if we don't // need to do any transformations to it. - const arma::mat& dataRef = + const MatType& dataRef = (transposeData || fitIntercept || normalizeData) ? dataTrans : matX; - const arma::rowvec& yRef = + const ResponsesType& yRef = (fitIntercept) ? yCentered : y; - arma::vec offsetX; // used only if fitting an intercept + arma::Col offsetX; // used only if fitting an intercept this->offsetY = 0.0; // used only if fitting an intercept - arma::vec stdX; // used only if normalizing + arma::Col stdX; // used only if normalizing if (transposeData) { @@ -494,7 +565,7 @@ inline double LARS::Train(const arma::mat& matX, } // Compute X' * y. - arma::vec vecXTy = trans(yRef * dataRef); + arma::Col vecXTy = trans(yRef * dataRef); // Set up active set variables. In the beginning, the active set has size 0 // (all dimensions are inactive). @@ -504,15 +575,15 @@ inline double LARS::Train(const arma::mat& matX, isIgnored.resize(dataRef.n_cols, false); // Initialize yHat and beta. - arma::vec beta = arma::zeros(dataRef.n_cols); - arma::vec yHat = arma::zeros(dataRef.n_rows); - arma::vec yHatDirection(dataRef.n_rows); + arma::Col beta(dataRef.n_cols, arma::fill::zeros); + arma::Col yHat(dataRef.n_rows, arma::fill::zeros); + arma::Col yHatDirection(dataRef.n_rows, arma::fill::none); bool lassocond = false; // Compute the initial maximum correlation among all dimensions. - arma::vec corr = vecXTy; - double maxCorr = 0; + arma::Col corr = vecXTy; + ElemType maxCorr = 0; size_t changeInd = 0; size_t lassocondInd = dataRef.n_cols; for (size_t i = 0; i < vecXTy.n_elem; ++i) @@ -548,7 +619,10 @@ inline double LARS::Train(const arma::mat& matX, matGramInternal = trans(dataRef) * dataRef; if (elasticNet && !useCholesky) - matGramInternal += lambda2 * arma::eye(dataRef.n_cols, dataRef.n_cols); + { + matGramInternal += lambda2 * + arma::eye(dataRef.n_cols, dataRef.n_cols); + } } // Main loop. @@ -557,8 +631,8 @@ inline double LARS::Train(const arma::mat& matX, { // Compute the maximum correlation among inactive dimensions. maxCorr = 0; - double maxActiveCorr = 0; - double minActiveCorr = DBL_MAX; + ElemType maxActiveCorr = 0; + ElemType minActiveCorr = DBL_MAX; for (size_t i = 0; i < dataRef.n_cols; ++i) { if ((!isActive[i]) && (!isIgnored[i]) && (fabs(corr(i)) > maxCorr)) @@ -584,9 +658,12 @@ inline double LARS::Train(const arma::mat& matX, if (maxCorr < tolerance) break; + // Floats require a really large tolerance for this condition. + const ElemType tol = (std::is_same::value) ? 1e-10 : 0.01; if ((matGram != &matGramInternal) && - ((maxActiveCorr - minActiveCorr) / maxActiveCorr) > 1e-10) + ((maxActiveCorr - minActiveCorr) / maxActiveCorr) > tol) { + std::cout << ((maxActiveCorr - minActiveCorr) / maxActiveCorr) << "\n"; // Construct the error message to match the user's settings. std::ostringstream oss; oss << "LARS::Train(): correlation conditions violated; check that your " @@ -618,7 +695,8 @@ inline double LARS::Train(const arma::mat& matX, // newGramCol[i] = dot(matX.col(activeSet[i]), matX.col(changeInd)); // } // This is equivalent to the above 5 lines. - arma::vec newGramCol = matGram->elem(changeInd * dataRef.n_cols + + arma::Col newGramCol = matGram->elem( + changeInd * dataRef.n_cols + arma::conv_to::from(activeSet)); CholeskyInsert((*matGram)(changeInd, changeInd), newGramCol); @@ -627,20 +705,20 @@ inline double LARS::Train(const arma::mat& matX, } // Compute signs of correlations. - arma::vec s = arma::vec(activeSet.size()); + arma::Col s(activeSet.size()); for (size_t i = 0; i < activeSet.size(); ++i) s(i) = corr(activeSet[i]) / fabs(corr(activeSet[i])); // Compute the "equiangular" direction in parameter space (betaDirection). // We use quotes because in the case of non-unit norm variables, this need // not be equiangular. - arma::vec unnormalizedBetaDirection; - double normalization; - arma::vec betaDirection; + arma::Col unnormalizedBetaDirection; + ElemType normalization; + arma::Col betaDirection; if (useCholesky) { // Check for singularity. - const double lastUtriElement = matUtriCholFactor( + const ElemType lastUtriElement = matUtriCholFactor( matUtriCholFactor.n_cols - 1, matUtriCholFactor.n_rows - 1); if (std::abs(lastUtriElement) > tolerance) { @@ -686,16 +764,16 @@ inline double LARS::Train(const arma::mat& matX, } else { - arma::mat matGramActive = arma::mat(activeSet.size(), activeSet.size()); + MatType matGramActive(activeSet.size(), activeSet.size()); for (size_t i = 0; i < activeSet.size(); ++i) for (size_t j = 0; j < activeSet.size(); ++j) matGramActive(i, j) = (*matGram)(activeSet[i], activeSet[j]); // Check for singularity. - arma::mat matS = s * arma::ones(1, activeSet.size()); + MatType matS = s * arma::ones(1, activeSet.size()); const bool solvedOk = solve(unnormalizedBetaDirection, matGramActive % trans(matS) % matS, - arma::ones(activeSet.size(), 1)); + arma::ones(activeSet.size(), 1)); if (solvedOk) { // Ok, no singularity. @@ -716,11 +794,11 @@ inline double LARS::Train(const arma::mat& matX, // need to take a step with the previous beta direction towards the next // variable we will add. s = s.subvec(0, activeSet.size() - 1); // Drop last element. - matS = s * arma::ones(1, activeSet.size()); + matS = s * arma::ones(1, activeSet.size()); // This worked last iteration, so there can't be a singularity. solve(unnormalizedBetaDirection, matGramActive % trans(matS) % matS, - arma::ones(activeSet.size(), 1)); + arma::ones(activeSet.size(), 1)); normalization = 1.0 / sqrt(sum(unnormalizedBetaDirection)); betaDirection = normalization * unnormalizedBetaDirection % s; } @@ -729,7 +807,7 @@ inline double LARS::Train(const arma::mat& matX, // compute "equiangular" direction in output space ComputeYHatDirection(dataRef, betaDirection, yHatDirection); - double gamma = maxCorr / normalization; + ElemType gamma = maxCorr / normalization; // If not all variables are active. if ((activeSet.size() + ignoreSet.size()) < dataRef.n_cols) @@ -740,9 +818,9 @@ inline double LARS::Train(const arma::mat& matX, if (isActive[ind] || isIgnored[ind]) continue; - const double dirCorr = dot(dataRef.col(ind), yHatDirection); - const double val1 = (maxCorr - corr(ind)) / (normalization - dirCorr); - const double val2 = (maxCorr + corr(ind)) / (normalization + dirCorr); + const ElemType dirCorr = dot(dataRef.col(ind), yHatDirection); + const ElemType val1 = (maxCorr - corr(ind)) / (normalization - dirCorr); + const ElemType val2 = (maxCorr + corr(ind)) / (normalization + dirCorr); // If we kicked out a feature due to the LASSO modification last // iteration, then we do not allow relaxation of the step size to 0 for @@ -769,12 +847,12 @@ inline double LARS::Train(const arma::mat& matX, { lassocond = false; lassocondInd = dataRef.n_cols; - double lassoboundOnGamma = DBL_MAX; + ElemType lassoboundOnGamma = DBL_MAX; size_t activeIndToKickOut = -1; for (size_t i = 0; i < activeSet.size(); ++i) { - double val = -beta(activeSet[i]) / betaDirection(i); + ElemType val = -beta(activeSet[i]) / betaDirection(i); if ((val > 0) && (val < lassoboundOnGamma)) { lassoboundOnGamma = val; @@ -824,7 +902,7 @@ inline double LARS::Train(const arma::mat& matX, if (elasticNet) corr -= lambda2 * beta; - double curLambda = 0; + ElemType curLambda = 0; for (size_t i = 0; i < activeSet.size(); ++i) curLambda += fabs(corr(activeSet[i])); @@ -876,11 +954,23 @@ inline double LARS::Train(const arma::mat& matX, return ComputeError(matX, y, !transposeData); } -inline void LARS::Predict(const arma::mat& points, - arma::rowvec& predictions, - const bool rowMajor) const +template +template +inline typename LARS::ElemType LARS::Predict( + const VecType& point) const +{ + if (!fitIntercept) + return Beta().t() * point; + else + return Beta().t() * point + Intercept(); +} + +template +template +inline void LARS::Predict(const MatType& points, + ResponsesType& predictions, + const bool rowMajor) const { - // We really only need to store beta internally... if (rowMajor && !fitIntercept) predictions = trans(points * Beta()); else if (rowMajor) @@ -891,7 +981,8 @@ inline void LARS::Predict(const arma::mat& points, predictions = Beta().t() * points; } -inline void LARS::FitIntercept(const bool newFitIntercept) +template +inline void LARS::FitIntercept(const bool newFitIntercept) { // If we are storing a Gram matrix internally, but now will be normalizing // data, then the Gram matrix we have computed is incorrect and needs to be @@ -909,7 +1000,8 @@ inline void LARS::FitIntercept(const bool newFitIntercept) } } -inline void LARS::NormalizeData(const bool newNormalizeData) +template +inline void LARS::NormalizeData(const bool newNormalizeData) { // If we are storing a Gram matrix internally, but now will be normalizing // data, then the Gram matrix we have computed is incorrect and needs to be @@ -927,7 +1019,8 @@ inline void LARS::NormalizeData(const bool newNormalizeData) } } -inline const std::vector& LARS::ActiveSet() const +template +inline const std::vector& LARS::ActiveSet() const { if (selectedIndex != (betaPath.size() - 1)) return selectedActiveSet; @@ -935,7 +1028,9 @@ inline const std::vector& LARS::ActiveSet() const return activeSet; } -inline const arma::vec& LARS::Beta() const +template +inline const typename LARS::ModelColType& +LARS::Beta() const { if (selectedIndex < betaPath.size()) return betaPath[selectedIndex]; @@ -943,7 +1038,9 @@ inline const arma::vec& LARS::Beta() const return selectedBeta; } -inline double LARS::Intercept() const +template +inline typename LARS::ElemType +LARS::Intercept() const { if (selectedIndex < betaPath.size()) return interceptPath[selectedIndex]; @@ -951,7 +1048,9 @@ inline double LARS::Intercept() const return selectedIntercept; } -inline void LARS::SelectBeta(const double selLambda1) +template +inline void LARS::SelectBeta( + const typename LARS::ElemType selLambda1) { if (selLambda1 < lambda1) { @@ -1005,7 +1104,7 @@ inline void LARS::SelectBeta(const double selLambda1) // model for, we can interpolate between the zero vector and the first model. if (i == 0) { - const double interp = selLambda1 / lambdaPath[0]; + const ElemType interp = selLambda1 / lambdaPath[0]; selectedIndex = betaPath.size(); selectedLambda1 = interp * lambdaPath[0]; @@ -1016,9 +1115,17 @@ inline void LARS::SelectBeta(const double selLambda1) selectedIntercept = (1 - interp) * this->offsetY + interp * interceptPath[0]; } + else if (i == betaPath.size()) + { + // It's possible that we fit the model perfectly with some lambda1 value + // less than this->lambda1. In that case, the interpolated solution is just + // the last solution. + selectedIndex = betaPath.size() - 1; + return; + } else { - const double interp = (lambdaPath[i - 1] - selLambda1) / + const ElemType interp = (lambdaPath[i - 1] - selLambda1) / (lambdaPath[i - 1] - lambdaPath[i]); selectedIndex = betaPath.size(); @@ -1034,41 +1141,48 @@ inline void LARS::SelectBeta(const double selLambda1) } // Private functions. -inline void LARS::Deactivate(const size_t activeVarInd) +template +inline void LARS::Deactivate(const size_t activeVarInd) { isActive[activeSet[activeVarInd]] = false; activeSet.erase(activeSet.begin() + activeVarInd); } -inline void LARS::Activate(const size_t varInd) +template +inline void LARS::Activate(const size_t varInd) { isActive[varInd] = true; activeSet.push_back(varInd); } -inline void LARS::Ignore(const size_t varInd) +template +inline void LARS::Ignore(const size_t varInd) { isIgnored[varInd] = true; ignoreSet.push_back(varInd); } -inline void LARS::ComputeYHatDirection(const arma::mat& matX, - const arma::vec& betaDirection, - arma::vec& yHatDirection) +template +template +inline void LARS::ComputeYHatDirection( + const MatType& matX, + const VecType& betaDirection, + VecType& yHatDirection) { yHatDirection.fill(0); for (size_t i = 0; i < activeSet.size(); ++i) yHatDirection += betaDirection(i) * matX.col(activeSet[i]); } -inline void LARS::InterpolateBeta() +template +inline void LARS::InterpolateBeta() { const size_t pathLength = betaPath.size(); // interpolate beta and stop - double ultimateLambda = lambdaPath[pathLength - 1]; - double penultimateLambda = lambdaPath[pathLength - 2]; - double interp = (penultimateLambda - lambda1) + ElemType ultimateLambda = lambdaPath[pathLength - 1]; + ElemType penultimateLambda = lambdaPath[pathLength - 2]; + ElemType interp = (penultimateLambda - lambda1) / (penultimateLambda - ultimateLambda); betaPath[pathLength - 1] = (1 - interp) * (betaPath[pathLength - 2]) @@ -1077,12 +1191,14 @@ inline void LARS::InterpolateBeta() lambdaPath[pathLength - 1] = lambda1; } -inline void LARS::CholeskyInsert(const arma::vec& newX, - const arma::mat& X) +template +template +inline void LARS::CholeskyInsert(const VecType& newX, + const MatType& X) { if (matUtriCholFactor.n_rows == 0) { - matUtriCholFactor = arma::mat(1, 1); + matUtriCholFactor.set_size(1, 1); if (elasticNet) matUtriCholFactor(0, 0) = sqrt(dot(newX, newX) + lambda2); @@ -1091,19 +1207,22 @@ inline void LARS::CholeskyInsert(const arma::vec& newX, } else { - arma::vec newGramCol = trans(X) * newX; + VecType newGramCol = trans(X) * newX; CholeskyInsert(dot(newX, newX), newGramCol); } } -inline void LARS::CholeskyInsert(double sqNormNewX, - const arma::vec& newGramCol) +template +template +inline void LARS::CholeskyInsert( + typename LARS::ElemType sqNormNewX, + const VecType& newGramCol) { int n = matUtriCholFactor.n_rows; if (n == 0) { - matUtriCholFactor = arma::mat(1, 1); + matUtriCholFactor.set_size(1, 1); if (elasticNet) matUtriCholFactor(0, 0) = sqrt(sqNormNewX + lambda2); @@ -1112,13 +1231,13 @@ inline void LARS::CholeskyInsert(double sqNormNewX, } else { - arma::mat matNewR = arma::mat(n + 1, n + 1); + DenseMatType matNewR(n + 1, n + 1); if (elasticNet) sqNormNewX += lambda2; - arma::vec matUtriCholFactork = solve(trimatl(trans(matUtriCholFactor)), - newGramCol); + arma::Col matUtriCholFactork = + solve(trimatl(trans(matUtriCholFactor)), newGramCol); matNewR(arma::span(0, n - 1), arma::span(0, n - 1)) = matUtriCholFactor; matNewR(arma::span(0, n - 1), n) = matUtriCholFactork; @@ -1126,39 +1245,46 @@ inline void LARS::CholeskyInsert(double sqNormNewX, matNewR(n, n) = sqrt(sqNormNewX - dot(matUtriCholFactork, matUtriCholFactork)); - matUtriCholFactor = matNewR; + matUtriCholFactor = std::move(matNewR); } } -inline void LARS::GivensRotate(const arma::vec::fixed<2>& x, - arma::vec::fixed<2>& rotatedX, - arma::mat& matG) +template +template +inline void LARS::GivensRotate( + const typename arma::Col< + typename LARS::ElemType + >::fixed<2>& x, + typename arma::Col< + typename LARS::ElemType + >::fixed<2>& rotatedX, + MatType& matG) { if (x(1) == 0) { - matG = arma::eye(2, 2); + matG = arma::eye(2, 2); rotatedX = x; } else { - double r = norm(x, 2); - matG = arma::mat(2, 2); + ElemType r = norm(x, 2); + matG.set_size(2, 2); - double scaledX1 = x(0) / r; - double scaledX2 = x(1) / r; + ElemType scaledX1 = x(0) / r; + ElemType scaledX2 = x(1) / r; matG(0, 0) = scaledX1; matG(1, 0) = -scaledX2; matG(0, 1) = scaledX2; matG(1, 1) = scaledX1; - rotatedX = arma::vec(2); rotatedX(0) = r; rotatedX(1) = 0; } } -inline void LARS::CholeskyDelete(const size_t colToKill) +template +inline void LARS::CholeskyDelete(const size_t colToKill) { size_t n = matUtriCholFactor.n_rows; @@ -1174,8 +1300,8 @@ inline void LARS::CholeskyDelete(const size_t colToKill) for (size_t k = colToKill; k < n; ++k) { - arma::mat matG; - arma::vec::fixed<2> rotatedVec; + DenseMatType matG; + typename arma::Col::fixed<2> rotatedVec; GivensRotate(matUtriCholFactor(arma::span(k, k + 1), k), rotatedVec, matG); matUtriCholFactor(arma::span(k, k + 1), k) = rotatedVec; @@ -1191,8 +1317,11 @@ inline void LARS::CholeskyDelete(const size_t colToKill) } } -inline double LARS::ComputeError(const arma::mat& matX, - const arma::rowvec& y, +template +template +inline typename LARS::ElemType +LARS::ComputeError(const MatType& matX, + const ResponsesType& y, const bool rowMajor) { if (rowMajor) @@ -1209,14 +1338,25 @@ inline double LARS::ComputeError(const arma::mat& matX, /** * Serialize the LARS model. */ +template template -void LARS::serialize(Archive& ar, const uint32_t version) +void LARS::serialize(Archive& ar, const uint32_t version) { // If we're loading, we have to use the internal storage. if (cereal::is_loading()) { matGram = &matGramInternal; - ar(CEREAL_NVP(matGramInternal)); + if (version == 0) + { + // Older versions stored matGramInternal as type arma::mat. + arma::mat matGramInternalTmp; + ar(cereal::make_nvp("matGramInternal", matGramInternalTmp)); + matGramInternal = arma::conv_to::from(matGramInternalTmp); + } + else + { + ar(CEREAL_NVP(matGramInternal)); + } } else { @@ -1224,22 +1364,75 @@ void LARS::serialize(Archive& ar, const uint32_t version) (const_cast(*matGram)))); } - ar(CEREAL_NVP(matUtriCholFactor)); - ar(CEREAL_NVP(useCholesky)); - ar(CEREAL_NVP(lasso)); - ar(CEREAL_NVP(lambda1)); - ar(CEREAL_NVP(elasticNet)); - ar(CEREAL_NVP(lambda2)); - ar(CEREAL_NVP(tolerance)); - ar(CEREAL_NVP(fitIntercept)); - ar(CEREAL_NVP(normalizeData)); - ar(CEREAL_NVP(betaPath)); - ar(CEREAL_NVP(lambdaPath)); - ar(CEREAL_NVP(interceptPath)); - ar(CEREAL_NVP(activeSet)); - ar(CEREAL_NVP(isActive)); - ar(CEREAL_NVP(ignoreSet)); - ar(CEREAL_NVP(isIgnored)); + if (cereal::is_loading() && version == 0) + { + // Older versions stored matUtriCholFactor as type arma::mat, and other + // elements as type double. This version loads everything as + // double/arma::mat and converts as needed. + arma::mat matUtriCholFactorTmp; + ar(cereal::make_nvp("matUtriCholFactor", matUtriCholFactorTmp)); + matUtriCholFactor = arma::conv_to::from(matUtriCholFactorTmp); + + ar(CEREAL_NVP(useCholesky)); + ar(CEREAL_NVP(lasso)); + + double tmp; + ar(cereal::make_nvp("lambda1", tmp)); + lambda1 = tmp; + + ar(CEREAL_NVP(elasticNet)); + + ar(cereal::make_nvp("lambda2", tmp)); + lambda2 = tmp; + + ar(cereal::make_nvp("tolerance", tmp)); + tolerance = tmp; + + ar(CEREAL_NVP(fitIntercept)); + ar(CEREAL_NVP(normalizeData)); + + std::vector betaPathTmp; + ar(cereal::make_nvp("betaPath", betaPathTmp)); + betaPath.resize(betaPathTmp.size()); + for (size_t i = 0; i < betaPathTmp.size(); ++i) + betaPath[i] = arma::conv_to::from(betaPathTmp[i]); + + std::vector lambdaPathTmp; + ar(cereal::make_nvp("lambdaPath", lambdaPathTmp)); + lambdaPath.resize(lambdaPathTmp.size()); + for (size_t i = 0; i < lambdaPathTmp.size(); ++i) + lambdaPath[i] = (ElemType) lambdaPathTmp[i]; + + std::vector interceptPathTmp; + ar(cereal::make_nvp("interceptPath", interceptPathTmp)); + interceptPath.resize(interceptPathTmp.size()); + for (size_t i = 0; i < interceptPathTmp.size(); ++i) + interceptPath[i] = (ElemType) interceptPathTmp[i]; + + ar(CEREAL_NVP(activeSet)); + ar(CEREAL_NVP(isActive)); + ar(CEREAL_NVP(ignoreSet)); + ar(CEREAL_NVP(isIgnored)); + } + else + { + ar(CEREAL_NVP(matUtriCholFactor)); + ar(CEREAL_NVP(useCholesky)); + ar(CEREAL_NVP(lasso)); + ar(CEREAL_NVP(lambda1)); + ar(CEREAL_NVP(elasticNet)); + ar(CEREAL_NVP(lambda2)); + ar(CEREAL_NVP(tolerance)); + ar(CEREAL_NVP(fitIntercept)); + ar(CEREAL_NVP(normalizeData)); + ar(CEREAL_NVP(betaPath)); + ar(CEREAL_NVP(lambdaPath)); + ar(CEREAL_NVP(interceptPath)); + ar(CEREAL_NVP(activeSet)); + ar(CEREAL_NVP(isActive)); + ar(CEREAL_NVP(ignoreSet)); + ar(CEREAL_NVP(isIgnored)); + } if (version > 0) { @@ -1250,6 +1443,15 @@ void LARS::serialize(Archive& ar, const uint32_t version) ar(CEREAL_NVP(selectedActiveSet)); ar(CEREAL_NVP(offsetY)); } + else if (cereal::is_loading()) + { + selectedLambda1 = lambdaPath.back(); + selectedIndex = betaPath.size() - 1; + selectedBeta.clear(); + selectedIntercept = 0.0; + selectedActiveSet.clear(); + offsetY = 0.0; + } } } // namespace mlpack diff --git a/src/mlpack/methods/lars/lars_main.cpp b/src/mlpack/methods/lars/lars_main.cpp index 1cf135f1aa..969be3af1d 100644 --- a/src/mlpack/methods/lars/lars_main.cpp +++ b/src/mlpack/methods/lars/lars_main.cpp @@ -107,8 +107,8 @@ BINDING_SEE_ALSO("LARS C++ class documentation", PARAM_TMATRIX_IN("input", "Matrix of covariates (X).", "i"); PARAM_MATRIX_IN("responses", "Matrix of responses/observations (y).", "r"); -PARAM_MODEL_IN(LARS, "input_model", "Trained LARS model to use.", "m"); -PARAM_MODEL_OUT(LARS, "output_model", "Output LARS model.", "M"); +PARAM_MODEL_IN(LARS<>, "input_model", "Trained LARS model to use.", "m"); +PARAM_MODEL_OUT(LARS<>, "output_model", "Output LARS model.", "M"); PARAM_TMATRIX_IN("test", "Matrix containing points to regress on (test " "points).", "t"); @@ -149,11 +149,11 @@ void BINDING_FUNCTION(util::Params& params, util::Timers& timers) false, "no results will be saved"); ReportIgnoredParam(params, {{ "test", true }}, "output_predictions"); - LARS* lars; + LARS<>* lars; if (params.Has("input")) { // Initialize the object. - lars = new LARS(useCholesky, lambda1, lambda2); + lars = new LARS<>(useCholesky, lambda1, lambda2); lars->FitIntercept(!noIntercept); lars->NormalizeData(!noNormalize); @@ -183,7 +183,7 @@ void BINDING_FUNCTION(util::Params& params, util::Timers& timers) } else // We must have --input_model_file. { - lars = params.Get("input_model"); + lars = params.Get*>("input_model"); } if (params.Has("test")) @@ -207,5 +207,5 @@ void BINDING_FUNCTION(util::Params& params, util::Timers& timers) params.Get("output_predictions") = predictions.t(); } - params.Get("output_model") = lars; + params.Get*>("output_model") = lars; } diff --git a/src/mlpack/methods/local_coordinate_coding/lcc_impl.hpp b/src/mlpack/methods/local_coordinate_coding/lcc_impl.hpp index cb96bf22a9..6189b192c9 100644 --- a/src/mlpack/methods/local_coordinate_coding/lcc_impl.hpp +++ b/src/mlpack/methods/local_coordinate_coding/lcc_impl.hpp @@ -141,7 +141,7 @@ inline void LocalCoordinateCoding::Encode(const arma::mat& data, bool useCholesky = false; // Normalization and fitting and intercept are disabled. - LARS lars(useCholesky, 0.5 * lambda, 0, 1e-16 /* default tolerance */, + LARS<> lars(useCholesky, 0.5 * lambda, 0, 1e-16 /* default tolerance */, false, false); // Run LARS for this point, by making an alias of the point and passing diff --git a/src/mlpack/methods/sparse_coding/sparse_coding_impl.hpp b/src/mlpack/methods/sparse_coding/sparse_coding_impl.hpp index e90c0fff3e..3346674a60 100644 --- a/src/mlpack/methods/sparse_coding/sparse_coding_impl.hpp +++ b/src/mlpack/methods/sparse_coding/sparse_coding_impl.hpp @@ -71,7 +71,7 @@ inline void SparseCoding::Encode(const arma::mat& data, bool useCholesky = true; // Intercept fitting and data normalization is disabled. - LARS lars(useCholesky, lambda1, lambda2, 1e-16 /* default tolerance */, + LARS<> lars(useCholesky, lambda1, lambda2, 1e-16 /* default tolerance */, false, false); // Create an alias of the code (using the same memory), and then LARS will diff --git a/src/mlpack/tests/cv_test.cpp b/src/mlpack/tests/cv_test.cpp index 74ab62b822..c2f6e5bc4f 100644 --- a/src/mlpack/tests/cv_test.cpp +++ b/src/mlpack/tests/cv_test.cpp @@ -335,7 +335,7 @@ TEST_CASE("SupportsWeightsTest", "[CVTest]") static_assert(MetaInfoExtractor, arma::mat, arma::urowvec, arma::Row>::SupportsWeights, "Value should be true"); - static_assert(!MetaInfoExtractor::SupportsWeights, + static_assert(!MetaInfoExtractor>::SupportsWeights, "Value should be false"); static_assert(!MetaInfoExtractor>::SupportsWeights, "Value should be false"); @@ -392,7 +392,7 @@ TEST_CASE("TakesNumClassesTest", "[CVTest]") "Value should be true"); static_assert(!MetaInfoExtractor>::TakesNumClasses, "Value should be false"); - static_assert(!MetaInfoExtractor::TakesNumClasses, + static_assert(!MetaInfoExtractor>::TakesNumClasses, "Value should be false"); } diff --git a/src/mlpack/tests/hpt_test.cpp b/src/mlpack/tests/hpt_test.cpp index fc8d79561c..6d57059f11 100644 --- a/src/mlpack/tests/hpt_test.cpp +++ b/src/mlpack/tests/hpt_test.cpp @@ -29,7 +29,7 @@ TEST_CASE("CVFunctionTest", "[HPTTest]") arma::vec beta = arma::randn(5, 1); arma::rowvec ys = beta.t() * xs + 0.1 * arma::randn(1, 100); - SimpleCV cv(0.2, xs, ys); + SimpleCV, MSE> cv(0.2, xs, ys); bool transposeData = true; bool useCholesky = false; @@ -41,7 +41,7 @@ TEST_CASE("CVFunctionTest", "[HPTTest]") FixedArg fixedUseCholesky{useCholesky}; FixedArg fixedLambda1{lambda2}; - CVFunction, FixedArg> + CVFunction, 4, FixedArg, FixedArg> cvFun(cv, datasetInfo, 0.0, 0.0, fixedUseCholesky, fixedLambda1); double expected = cv.Evaluate(transposeData, useCholesky, lambda1, lambda2); @@ -64,7 +64,7 @@ TEST_CASE("CVFunctionCategoricalTest", "[HPTTest]") arma::vec beta = arma::randn(5, 1); arma::rowvec ys = beta.t() * xs + 0.1 * arma::randn(1, 100); - SimpleCV cv(0.2, xs, ys); + SimpleCV, MSE> cv(0.2, xs, ys); bool transposeData = true; bool useCholesky = false; @@ -78,7 +78,7 @@ TEST_CASE("CVFunctionCategoricalTest", "[HPTTest]") FixedArg fixedUseCholesky{useCholesky}; FixedArg fixedLambda1{lambda2}; - CVFunction, FixedArg> + CVFunction, 4, FixedArg, FixedArg> cvFun(cv, datasetInfo, 0.0, 0.0, fixedUseCholesky, fixedLambda1); double expected = cv.Evaluate(transposeData, useCholesky, lambda1, lambda2); @@ -137,7 +137,7 @@ TEST_CASE("CVFunctionGradientTest", "[HPTTest]") double b = -1.5; double c = 2.5; double d = 3.0; - QuadraticFunction lf(a, b, c, d); + QuadraticFunction> lf(a, b, c, d); // All values are numeric. IncrementPolicy policy(true); @@ -145,7 +145,7 @@ TEST_CASE("CVFunctionGradientTest", "[HPTTest]") double relativeDelta = 0.01; double minDelta = 0.001; - CVFunction cvFun(lf, datasetInfo, relativeDelta, + CVFunction, 3> cvFun(lf, datasetInfo, relativeDelta, minDelta); double x = 0.0; @@ -202,7 +202,7 @@ void FindLARSBestLambdas(arma::mat& xs, double& bestLambda2, double& bestObjective) { - SimpleCV cv(validationSize, xs, ys); + SimpleCV, MSE> cv(validationSize, xs, ys); bestObjective = std::numeric_limits::max(); @@ -250,8 +250,8 @@ TEST_CASE("GridSearchTest", "[HPTTest]") for (double lambda2 : lambda2Set) datasetInfo.MapString(lambda2, 1); - SimpleCV cv(validationSize, xs, ys); - CVFunction, FixedArg> + SimpleCV, MSE> cv(validationSize, xs, ys); + CVFunction, 4, FixedArg, FixedArg> cvFun(cv, datasetInfo, 0.0, 0.0, {transposeData}, {useCholesky}); ens::GridSearch optimizer; @@ -297,7 +297,7 @@ TEST_CASE("HPTTest", "[HPTTest]") expectedObjective); double actualLambda1, actualLambda2; - HyperParameterTuner + HyperParameterTuner, MSE, SimpleCV, GridSearch> hpt(validationSize, xs, ys); std::tie(actualLambda1, actualLambda2) = hpt.Optimize(Fixed(transposeData), Fixed(useCholesky), lambda1Set, lambda2Set); @@ -368,7 +368,7 @@ TEST_CASE("HPTGradientDescentTest", "[HPTTest]") // We pass LARS just because some ML algorithm should be passed. We pass MSE // to tell HyperParameterTuner that the objective function (QuadraticFunction) // should be minimized. - HyperParameterTuner, MSE, QuadraticFunction, GradientDescent> hpt(a, b, c, d, xMin, yMin, zMin); // Setting GradientDescent to find more close solution to the optimal one. diff --git a/src/mlpack/tests/lars_test.cpp b/src/mlpack/tests/lars_test.cpp index 4b3375d536..c60b3eb0a9 100644 --- a/src/mlpack/tests/lars_test.cpp +++ b/src/mlpack/tests/lars_test.cpp @@ -17,24 +17,32 @@ using namespace mlpack; +template void GenerateProblem( - arma::mat& X, arma::rowvec& y, size_t nPoints, size_t nDims) + MatType& X, ResponsesType& y, size_t nPoints, size_t nDims) { - X = arma::randn(nDims, nPoints); - arma::vec beta = arma::randn(nDims, 1); + X = arma::randn(nDims, nPoints); + arma::Col beta = + arma::randn>(nDims); y = beta.t() * X; } -void LARSVerifyCorrectness(arma::vec beta, arma::vec errCorr, double lambda) +template +void LARSVerifyCorrectness(const VecType& beta, + const VecType& errCorr, + ElemType lambda) { size_t nDims = beta.n_elem; - const double tol = 1e-10; + + // floats require a much larger tolerance. + const ElemType tol = (std::is_same::value) ? 1e-10 : 1e-3; + for (size_t j = 0; j < nDims; ++j) { if (beta(j) == 0) { // Make sure that |errCorr(j)| <= lambda. - REQUIRE(std::max(fabs(errCorr(j)) - lambda, 0.0) == + REQUIRE(std::max(fabs(errCorr(j)) - lambda, (ElemType) 0.0) == Approx(0.0).margin(tol)); } else if (beta(j) < 0) @@ -50,30 +58,33 @@ void LARSVerifyCorrectness(arma::vec beta, arma::vec errCorr, double lambda) } } +template void LassoTest(size_t nPoints, size_t nDims, bool elasticNet, bool useCholesky, bool fitIntercept, bool normalizeData) { - arma::mat X; - arma::rowvec y; + typedef typename MatType::elem_type ElemType; + + MatType X; + arma::Row y; for (size_t i = 0; i < 100; ++i) { GenerateProblem(X, y, nPoints, nDims); // Armadillo's median is broken, so... - arma::vec sortedAbsCorr = sort(abs(X * y.t())); - double lambda1 = sortedAbsCorr(nDims / 2); - double lambda2; + arma::Col sortedAbsCorr = sort(abs(X * y.t())); + ElemType lambda1 = sortedAbsCorr(nDims / 2); + ElemType lambda2; if (elasticNet) lambda2 = lambda1 / 2; else lambda2 = 0; - LARS lars(useCholesky, lambda1, lambda2); + LARS lars(useCholesky, lambda1, lambda2); lars.FitIntercept(fitIntercept); lars.NormalizeData(normalizeData); lars.Train(X, y); - arma::vec betaOpt = lars.Beta(); + arma::Col betaOpt = lars.Beta(); if (fitIntercept) { @@ -83,51 +94,53 @@ void LassoTest(size_t nPoints, size_t nDims, bool elasticNet, bool useCholesky, if (normalizeData) { - arma::vec stds = arma::stddev(X, 0, 1); + arma::Col stds = arma::stddev(X, 0, 1); stds.replace(0.0, 1.0); X.each_col() /= stds; betaOpt %= stds; // recover solution in normalized space } - arma::vec errCorr = (X * trans(X) + lambda2 * - arma::eye(nDims, nDims)) * betaOpt - X * y.t(); + arma::Col errCorr = (X * trans(X) + lambda2 * + arma::eye(nDims, nDims)) * betaOpt - X * y.t(); LARSVerifyCorrectness(betaOpt, errCorr, lambda1); } } -TEST_CASE("LARSTestLassoCholesky", "[LARSTest]") +TEMPLATE_TEST_CASE("LARSTestLassoCholesky", "[LARSTest]", arma::fmat, arma::mat) { - LassoTest(100, 10, false, true, false, false); - LassoTest(100, 10, false, true, true, false); - LassoTest(100, 10, false, true, false, true); - LassoTest(100, 10, false, true, true, true); + LassoTest(100, 10, false, true, false, false); + LassoTest(100, 10, false, true, true, false); + LassoTest(100, 10, false, true, false, true); + LassoTest(100, 10, false, true, true, true); } -TEST_CASE("LARSTestLassoGram", "[LARSTest]") +TEMPLATE_TEST_CASE("LARSTestLassoGram", "[LARSTest]", arma::fmat, arma::mat) { - LassoTest(100, 10, false, false, false, false); - LassoTest(100, 10, false, false, true, false); - LassoTest(100, 10, false, false, false, true); - LassoTest(100, 10, false, false, true, true); + LassoTest(100, 10, false, false, false, false); + LassoTest(100, 10, false, false, true, false); + LassoTest(100, 10, false, false, false, true); + LassoTest(100, 10, false, false, true, true); } -TEST_CASE("LARSTestElasticNetCholesky", "[LARSTest]") +TEMPLATE_TEST_CASE("LARSTestElasticNetCholesky", "[LARSTest]", arma::fmat, + arma::mat) { - LassoTest(100, 10, true, true, false, false); - LassoTest(100, 10, true, true, true, false); - LassoTest(100, 10, true, true, false, true); - LassoTest(100, 10, true, true, true, true); + LassoTest(100, 10, true, true, false, false); + LassoTest(100, 10, true, true, true, false); + LassoTest(100, 10, true, true, false, true); + LassoTest(100, 10, true, true, true, true); } -TEST_CASE("LARSTestElasticNetGram", "[LARSTest]") +TEMPLATE_TEST_CASE("LARSTestElasticNetGram", "[LARSTest]", arma::fmat, + arma::mat) { - LassoTest(100, 10, true, false, false, false); - LassoTest(100, 10, true, false, true, false); - LassoTest(100, 10, true, false, false, true); - LassoTest(100, 10, true, false, true, true); + LassoTest(100, 10, true, false, false, false); + LassoTest(100, 10, true, false, true, false); + LassoTest(100, 10, true, false, false, true); + LassoTest(100, 10, true, false, true, true); } // Ensure that LARS doesn't crash when the data has linearly dependent features @@ -148,7 +161,7 @@ TEST_CASE("CholeskySingularityTest", "[LARSTest]") // Test for a couple values of lambda1. for (double lambda1 = 0.0; lambda1 < 1.0; lambda1 += 0.1) { - LARS lars(true, lambda1, 0.0); + LARS<> lars(true, lambda1, 0.0); lars.FitIntercept(false); lars.NormalizeData(false); lars.Train(X, y); @@ -175,7 +188,7 @@ TEST_CASE("NoCholeskySingularityTest", "[LARSTest]") // Test for a couple values of lambda1. for (double lambda1 = 0.0; lambda1 < 1.0; lambda1 += 0.1) { - LARS lars(false, lambda1, 0.0); + LARS<> lars(false, lambda1, 0.0); lars.FitIntercept(false); lars.NormalizeData(false); lars.Train(X, y); @@ -187,33 +200,36 @@ TEST_CASE("NoCholeskySingularityTest", "[LARSTest]") } // Make sure that Predict() provides reasonable enough solutions. -TEST_CASE("PredictTest", "[LARSTest]") +TEMPLATE_TEST_CASE("PredictTest", "[LARSTest]", arma::fmat, arma::mat) { + typedef TestType MatType; + typedef typename MatType::elem_type ElemType; + for (size_t i = 0; i < 2; ++i) { // Run with both true and false. bool useCholesky = bool(i); - arma::mat X; - arma::rowvec y; + MatType X; + arma::Row y; GenerateProblem(X, y, 1000, 100); - for (double lambda1 = 0.0; lambda1 < 1.0; lambda1 += 0.2) + for (ElemType lambda1 = 0.0; lambda1 < 1.0; lambda1 += 0.2) { - for (double lambda2 = 0.0; lambda2 < 1.0; lambda2 += 0.2) + for (ElemType lambda2 = 0.0; lambda2 < 1.0; lambda2 += 0.2) { - LARS lars(useCholesky, lambda1, lambda2); + LARS lars(useCholesky, lambda1, lambda2); lars.FitIntercept(false); lars.NormalizeData(false); lars.Train(X, y); // Calculate what the actual error should be with these regression // parameters. - arma::vec betaOptPred = (X * X.t()) * lars.Beta(); - arma::rowvec predictions; + arma::Col betaOptPred = (X * X.t()) * lars.Beta(); + arma::Row predictions; lars.Predict(X, predictions); - arma::vec adjPred = X * predictions.t(); + arma::Col adjPred = X * predictions.t(); REQUIRE(predictions.n_elem == 1000); for (size_t i = 0; i < betaOptPred.n_elem; ++i) @@ -236,7 +252,7 @@ TEST_CASE("PredictRowMajorTest", "[LARSTest]") // Set lambdas to 0. - LARS lars(false, 0, 0); + LARS<> lars(false, 0, 0); lars.FitIntercept(false); lars.NormalizeData(false); lars.Train(X, y); @@ -271,7 +287,7 @@ TEST_CASE("LARSRetrainTest", "[LARSTest]") arma::rowvec newY; GenerateProblem(newX, newY, 750, 75); - LARS lars(false, 0.1, 0.1); + LARS<> lars(false, 0.1, 0.1); lars.FitIntercept(false); lars.NormalizeData(false); lars.Train(origX, origY); @@ -299,7 +315,7 @@ TEST_CASE("RetrainCholeskyTest", "[LARSTest]") arma::rowvec newY; GenerateProblem(newX, newY, 750, 75); - LARS lars(true, 0.1, 0.1); + LARS<> lars(true, 0.1, 0.1); lars.FitIntercept(false); lars.NormalizeData(false); lars.Train(origX, origY); @@ -324,11 +340,11 @@ TEST_CASE("TrainingAndAccessingBetaTest", "[LARSTest]") GenerateProblem(X, y, 1000, 100); - LARS lars1; + LARS<> lars1; lars1.Train(X, y); arma::vec beta = lars1.Beta(); - LARS lars2; + LARS<> lars2; lars2.Train(X, y); REQUIRE(beta.n_elem == lars2.Beta().n_elem); @@ -347,11 +363,11 @@ TEST_CASE("TrainingConstructorWithDefaultsTest", "[LARSTest]") GenerateProblem(X, y, 1000, 100); - LARS lars1; + LARS<> lars1; lars1.Train(X, y); arma::vec beta = lars1.Beta(); - LARS lars2(X, y); + LARS<> lars2(X, y); REQUIRE(beta.n_elem == lars2.Beta().n_elem); for (size_t i = 0; i < beta.n_elem; ++i) @@ -374,11 +390,11 @@ TEST_CASE("TrainingConstructorWithNonDefaultsTest", "[LARSTest]") double lambda1 = 0.2; double lambda2 = 0.4; - LARS lars1(useCholesky, lambda1, lambda2); + LARS<> lars1(useCholesky, lambda1, lambda2); lars1.Train(X, y); arma::vec beta = lars1.Beta(); - LARS lars2(X, y, transposeData, useCholesky, lambda1, lambda2); + LARS<> lars2(X, y, transposeData, useCholesky, lambda1, lambda2); REQUIRE(beta.n_elem == lars2.Beta().n_elem); for (size_t i = 0; i < beta.n_elem; ++i) @@ -404,25 +420,25 @@ TEST_CASE("LARSTrainReturnCorrelation", "[LARSTest]") double lambda2 = 0.1; // Test with Cholesky decomposition and with lasso. - LARS lars1(true, lambda1, 0.0); + LARS<> lars1(true, lambda1, 0.0); double error = lars1.Train(X, y); REQUIRE(std::isfinite(error) == true); // Test without Cholesky decomposition and with lasso. - LARS lars2(false, lambda1, 0.0); + LARS<> lars2(false, lambda1, 0.0); error = lars2.Train(X, y); REQUIRE(std::isfinite(error) == true); // Test with Cholesky decomposition and with elasticnet. - LARS lars3(true, lambda1, lambda2); + LARS<> lars3(true, lambda1, lambda2); error = lars3.Train(X, y); REQUIRE(std::isfinite(error) == true); // Test without Cholesky decomposition and with elasticnet. - LARS lars4(false, lambda1, lambda2); + LARS<> lars4(false, lambda1, lambda2); error = lars4.Train(X, y); REQUIRE(std::isfinite(error) == true); @@ -432,23 +448,26 @@ TEST_CASE("LARSTrainReturnCorrelation", "[LARSTest]") * Test that LARS::ComputeError() returns error value less than 1 * and greater than 0. */ -TEST_CASE("LARSTestComputeError", "[LARSTest]") +TEMPLATE_TEST_CASE("LARSTestComputeError", "[LARSTest]", arma::fmat, arma::mat) { - arma::mat X; - arma::mat Y; + typedef TestType MatType; + typedef typename MatType::elem_type ElemType; + + MatType X; + MatType Y; if (!data::Load("lars_dependent_x.csv", X)) FAIL("Cannot load dataset lars_dependent_x.csv"); if (!data::Load("lars_dependent_y.csv", Y)) FAIL("Cannot load dataset lars_dependent_y.csv"); - arma::rowvec y = Y.row(0); + arma::Row y = Y.row(0); - LARS lars1(true, 0.1, 0.0); + LARS lars1(true, 0.1, 0.0); lars1.FitIntercept(false); lars1.NormalizeData(false); - double train1 = lars1.Train(X, y); - double cost = lars1.ComputeError(X, y); + ElemType train1 = lars1.Train(X, y); + ElemType cost = lars1.ComputeError(X, y); REQUIRE(cost <= 1); REQUIRE(cost >= 0); @@ -472,9 +491,9 @@ TEST_CASE("LARSCopyConstructorTest", "[LARSTest]") // Check if the copy is accessible even after deleting the pointer to the // object. - LARS* glm1 = new LARS(false, .1, .1); + LARS<>* glm1 = new LARS<>(false, .1, .1); arma::rowvec predictions, predictionsFromCopiedModel; - std::vector models; + std::vector> models; glm1->Train(features, targets); glm1->Predict(features, predictions); models.emplace_back(*glm1); // Call the copy constructor. @@ -486,13 +505,13 @@ TEST_CASE("LARSCopyConstructorTest", "[LARSTest]") REQUIRE_NOTHROW(models[0].Train(features, targets)); // Check if we can train the copied model. - LARS glm2(false, 0.1, 0.1); + LARS<> glm2(false, 0.1, 0.1); models.emplace_back(glm2); // Call the copy constructor. REQUIRE_NOTHROW(glm2.Train(features, targets)); REQUIRE_NOTHROW(models[1].Train(features, targets)); // Create a copy using assignment operator. - LARS glm3 = glm2; + LARS<> glm3 = glm2; models[1].Predict(features, predictions); glm3.Predict(features, predictionsFromCopiedModel); // The output of both models should be the same. @@ -510,8 +529,8 @@ TEST_CASE("LARSFitInterceptTest", "[LARSTest]") arma::mat centeredFeatures = features.each_col() - arma::mean(features, 1); arma::rowvec centeredResponses = responses - arma::mean(responses); - LARS l1(features, responses, true, true, 0.001, 0.001, 1e-16, true, false); - LARS l2(centeredFeatures, centeredResponses, true, true, 0.001, 0.001, 1e-16, false, false); + LARS<> l1(features, responses, true, true, 0.001, 0.001, 1e-16, true, false); + LARS<> l2(centeredFeatures, centeredResponses, true, true, 0.001, 0.001, 1e-16, false, false); // The weights learned should be the same. REQUIRE(l1.Beta().n_elem == l2.Beta().n_elem); @@ -538,7 +557,7 @@ TEST_CASE("PredictFitInterceptTest", "[LARSTest]") { for (double lambda2 = 0.0; lambda2 < 1.0; lambda2 += 0.2) { - LARS lars(useCholesky, lambda1, lambda2); + LARS<> lars(useCholesky, lambda1, lambda2); lars.FitIntercept(true); lars.NormalizeData(false); lars.Train(X, y); @@ -583,7 +602,7 @@ TEST_CASE("PredictNormalizeDataTest", "[LARSTest]") { for (double lambda2 = 0.0; lambda2 < 1.0; lambda2 += 0.2) { - LARS lars(useCholesky, lambda1, lambda2); + LARS<> lars(useCholesky, lambda1, lambda2); lars.FitIntercept(false); lars.NormalizeData(true); lars.Train(X, y); @@ -626,7 +645,7 @@ TEST_CASE("PredictFitInterceptNormalizeDataTest", "[LARSTest]") { for (double lambda2 = 0.0; lambda2 < 1.0; lambda2 += 0.2) { - LARS lars(useCholesky, lambda1, lambda2); + LARS<> lars(useCholesky, lambda1, lambda2); lars.FitIntercept(true); lars.NormalizeData(true); lars.Train(X, y); @@ -702,7 +721,7 @@ TEST_CASE("LARSTestKKT", "[LARSTest]") F.load("lars_kkt.bin"); bool useCholesky = true; - LARS lars(useCholesky, 1.0, 0.0); + LARS<> lars(useCholesky, 1.0, 0.0); for (size_t i = 0; i < F.n_cols; i++) { @@ -760,41 +779,45 @@ TEST_CASE("LARSTestKKT", "[LARSTest]") } // Check that all variants of constructors appear to work. -TEST_CASE("LARSConstructorVariantTest", "[LARSTest]") +TEMPLATE_TEST_CASE("LARSConstructorVariantTest", "[LARSTest]", arma::fmat, + arma::mat) { + typedef TestType MatType; + typedef typename MatType::elem_type ElemType; + // The results of the training are not all that important here; the more // important thing is just that all the overloads compile properly. We do // some basic sanity checks on the trained model nonetheless. - arma::mat X; - arma::rowvec y; + MatType X; + arma::Row y; GenerateProblem(X, y, 1000, 100); - arma::mat Xt = X.t(); + MatType Xt = X.t(); - const arma::vec xMean = arma::mean(X, 1); - arma::vec xStds = arma::stddev(X, 0, 1); + const arma::Col xMean = arma::mean(X, 1); + arma::Col xStds = arma::stddev(X, 0, 1); xStds.replace(0.0, 1.0); - const double yMean = arma::mean(y); + const ElemType yMean = arma::mean(y); - arma::mat centeredX = X.each_col() - xMean; - arma::rowvec centeredY = y - yMean; + MatType centeredX = X.each_col() - xMean; + arma::Row centeredY = y - yMean; - arma::mat centeredUnitX = centeredX.each_col() / xStds; + MatType centeredUnitX = centeredX.each_col() / xStds; - arma::mat matGram = X * X.t(); - arma::mat centeredUnitMatGram = centeredUnitX * centeredUnitX.t(); + MatType matGram = X * X.t(); + MatType centeredUnitMatGram = centeredUnitX * centeredUnitX.t(); - LARS l1; - LARS l2(false, 0.1, 0.2, 1e-15, false, false); - LARS l3(X, y); - LARS l4(Xt, y, false); - LARS l5(Xt, y, false, false); - LARS l6(Xt, y, false, false, 0.1); - LARS l7(X, y, true, false, 0.11, 0.01); - LARS l8(X, y, true, false, 0.12, 0.02, 1e-8); - LARS l9(centeredX, centeredY, true, false, 0.13, 0.03, 1e-7, false); - LARS l10(centeredUnitX, centeredY, true, false, 0.14, 0.04, 1e-6, false, - false); + LARS l1; + LARS l2(false, 0.1, 0.2, 1e-15, false, false); + LARS l3(X, y); + LARS l4(Xt, y, false); + LARS l5(Xt, y, false, false); + LARS l6(Xt, y, false, false, 0.1); + LARS l7(X, y, true, false, 0.11, 0.01); + LARS l8(X, y, true, false, 0.12, 0.02, 1e-8); + LARS l9(centeredX, centeredY, true, false, 0.13, 0.03, 1e-7, false); + LARS l10(centeredUnitX, centeredY, true, false, 0.14, 0.04, 1e-6, + false, false); REQUIRE(l1.BetaPath().size() == 0); @@ -846,27 +869,28 @@ TEST_CASE("LARSConstructorVariantTest", "[LARSTest]") // Now check constructors where we specify the Gram matrix. const size_t dim = centeredUnitMatGram.n_rows; - LARS l11(X, y, true, false, centeredUnitMatGram); - LARS l12(Xt, y, false, false, centeredUnitMatGram, 0.1); + LARS l11(X, y, true, false, centeredUnitMatGram); + LARS l12(Xt, y, false, false, centeredUnitMatGram, 0.1); // If lambda2 > 0, then we have to adjust the Gram matrix to account for that. - arma::mat centeredUnitMatGramL13 = centeredUnitMatGram + - 0.01 * arma::eye(dim, dim); - LARS l13(Xt, y, false, false, centeredUnitMatGramL13, 0.11, 0.01); + MatType centeredUnitMatGramL13 = centeredUnitMatGram + + 0.01 * arma::eye(dim, dim); + LARS l13(Xt, y, false, false, centeredUnitMatGramL13, 0.11, 0.01); - arma::mat centeredUnitMatGramL14 = centeredUnitMatGram + - 0.02 * arma::eye(dim, dim); - LARS l14(Xt, y, false, false, centeredUnitMatGramL14, 0.12, 0.02, 1e-15); + MatType centeredUnitMatGramL14 = centeredUnitMatGram + + 0.02 * arma::eye(dim, dim); + LARS l14(Xt, y, false, false, centeredUnitMatGramL14, 0.12, 0.02, + 1e-15); - arma::mat centeredUnitMatGramL15 = centeredUnitMatGram + - 0.03 * arma::eye(dim, dim); - LARS l15(centeredX, centeredY, true, false, centeredUnitMatGramL15, 0.13, - 0.03, 1e-14, false); + MatType centeredUnitMatGramL15 = centeredUnitMatGram + + 0.03 * arma::eye(dim, dim); + LARS l15(centeredX, centeredY, true, false, centeredUnitMatGramL15, + 0.13, 0.03, 1e-14, false); - arma::mat centeredUnitMatGramL16 = centeredUnitMatGram + - 0.04 * arma::eye(dim, dim); - LARS l16(centeredUnitX, centeredY, true, false, centeredUnitMatGramL16, 0.14, - 0.04, 1e-13, false, false); + MatType centeredUnitMatGramL16 = centeredUnitMatGram + + 0.04 * arma::eye(dim, dim); + LARS l16(centeredUnitX, centeredY, true, false, + centeredUnitMatGramL16, 0.14, 0.04, 1e-13, false, false); REQUIRE(l11.Beta().n_elem == X.n_rows); REQUIRE(l11.UseCholesky() == false); @@ -903,31 +927,34 @@ TEST_CASE("LARSConstructorVariantTest", "[LARSTest]") } // Check that all variants of Train() appear to work. -TEST_CASE("LARSTrainVariantTest", "[LARSTest]") +TEMPLATE_TEST_CASE("LARSTrainVariantTest", "[LARSTest]", arma::fmat, arma::mat) { + typedef TestType MatType; + typedef typename MatType::elem_type ElemType; + // The results of the training are not all that important here; the more // important thing is just that all the overloads compile properly. We do // some basic sanity checks on the trained model nonetheless. - arma::mat X; - arma::rowvec y; + MatType X; + arma::Row y; GenerateProblem(X, y, 1000, 5); - arma::mat Xt = X.t(); + MatType Xt = X.t(); - const arma::vec xMean = arma::mean(X, 1); - arma::vec xStds = arma::stddev(X, 0, 1); + const arma::Col xMean = arma::mean(X, 1); + arma::Col xStds = arma::stddev(X, 0, 1); xStds.replace(0.0, 1.0); - const double yMean = arma::mean(y); + const ElemType yMean = arma::mean(y); - arma::mat centeredX = X.each_col() - xMean; - arma::rowvec centeredY = y - yMean; + MatType centeredX = X.each_col() - xMean; + arma::Row centeredY = y - yMean; - arma::mat centeredUnitX = centeredX.each_col() / xStds; + MatType centeredUnitX = centeredX.each_col() / xStds; - arma::mat matGram = X * X.t(); - arma::mat centeredUnitMatGram = centeredUnitX * centeredUnitX.t(); + MatType matGram = X * X.t(); + MatType centeredUnitMatGram = centeredUnitX * centeredUnitX.t(); - LARS l1, l2, l3, l4, l5, l6, l7, l8, l9, l10, l11, l12, l13, l14; + LARS l1, l2, l3, l4, l5, l6, l7, l8, l9, l10, l11, l12, l13, l14; l1.Train(X, y); l2.Train(Xt, y, false); @@ -943,21 +970,21 @@ TEST_CASE("LARSTrainVariantTest", "[LARSTest]") // If lambda2 > 0, then we have to adjust the Gram matrix. const size_t dim = centeredUnitMatGram.n_rows; - arma::mat centeredUnitMatGramL11 = centeredUnitMatGram + - 0.05 * arma::eye(dim, dim); + MatType centeredUnitMatGramL11 = centeredUnitMatGram + + 0.05 * arma::eye(dim, dim); l11.Train(X, y, true, false, centeredUnitMatGramL11, 0.16, 0.05); - arma::mat centeredUnitMatGramL12 = centeredUnitMatGram + - 0.06 * arma::eye(dim, dim); + MatType centeredUnitMatGramL12 = centeredUnitMatGram + + 0.06 * arma::eye(dim, dim); l12.Train(X, y, true, false, centeredUnitMatGramL12, 0.17, 0.06, 1e-12); - arma::mat centeredUnitMatGramL13 = centeredUnitMatGram + - 0.07 * arma::eye(dim, dim); + MatType centeredUnitMatGramL13 = centeredUnitMatGram + + 0.07 * arma::eye(dim, dim); l13.Train(centeredX, centeredY, true, false, centeredUnitMatGramL13, 0.18, 0.07, 1e-11, false); - arma::mat centeredUnitMatGramL14 = centeredUnitMatGram + - 0.08 * arma::eye(dim, dim); + MatType centeredUnitMatGramL14 = centeredUnitMatGram + + 0.08 * arma::eye(dim, dim); l14.Train(centeredUnitX, centeredY, true, false, centeredUnitMatGramL14, 0.19, 0.08, 1e-10, false, false); @@ -1033,33 +1060,38 @@ TEST_CASE("LARSTrainVariantTest", "[LARSTest]") } // Ensure that SelectBeta() works correctly. -TEST_CASE("LARSSelectBetaTest", "[LARSTest]") +TEMPLATE_TEST_CASE("LARSSelectBetaTest", "[LARSTest]", arma::fmat, arma::mat) { + typedef TestType MatType; + typedef typename MatType::elem_type ElemType; + + const ElemType tol = (std::is_same::value) ? 1e-5 : 1e-3; + // Train a model on a randomly generated problem. Then, we will iterate // through different selected lambda values, ensuring that the error on the // training set is monotonically increasing. - arma::mat X; - arma::rowvec y; + MatType X; + arma::Row y; GenerateProblem(X, y, 1000, 100); - LARS lars(X, y); + LARS lars(X, y); // Ensure that the solution with no regularization is fully dense. REQUIRE(lars.ActiveSet().size() == X.n_rows); REQUIRE(lars.Beta().n_elem == X.n_rows); // Now step through numerous different lambda values. - double lastError = std::numeric_limits::max(); - for (double i = 5.0; i >= -5.0; i -= 0.1) + ElemType lastError = std::numeric_limits::max(); + for (ElemType i = 5.0; i >= -5.0; i -= 0.1) { - const double selLambda1 = std::pow(10.0, (double) i); + const ElemType selLambda1 = std::pow(10.0, (ElemType) i); lars.SelectBeta(selLambda1); REQUIRE(lars.Beta().n_elem == X.n_rows); - REQUIRE(lars.SelectedLambda1() == Approx(selLambda1)); + REQUIRE(lars.SelectedLambda1() == Approx(selLambda1).margin(tol)); REQUIRE(arma::accu(lars.Beta() != 0.0) == lars.ActiveSet().size()); - const double newError = lars.ComputeError(X, y); + const ElemType newError = lars.ComputeError(X, y); REQUIRE(newError <= lastError); lastError = newError; } @@ -1071,17 +1103,17 @@ TEST_CASE("LARSSelectBetaTest", "[LARSTest]") { lars.SelectBeta(lars.LambdaPath()[i]); - REQUIRE(lars.SelectedLambda1() == Approx(lars.LambdaPath()[i])); - REQUIRE(lars.Intercept() == Approx(lars.InterceptPath()[i])); + REQUIRE(lars.SelectedLambda1() == Approx(lars.LambdaPath()[i]).margin(tol)); + REQUIRE(lars.Intercept() == Approx(lars.InterceptPath()[i]).margin(tol)); REQUIRE(arma::approx_equal(lars.Beta(), lars.BetaPath()[i], "absdiff", - 1e-5)); + tol)); } } // Test that SelectBeta() throws an error when the model is not trained. TEST_CASE("LARSSelectBetaUntrainedModelTest", "[LARSTest]") { - LARS lars; + LARS<> lars; REQUIRE_THROWS_AS(lars.SelectBeta(0.01), std::runtime_error); } @@ -1095,9 +1127,61 @@ TEST_CASE("LARSSelectBetaInvalidLambda1Test", "[LARSTest]") GenerateProblem(X, y, 1000, 100); - LARS lars; + LARS<> lars; lars.Lambda1() = 1.0; lars.Train(X, y); REQUIRE_THROWS_AS(lars.SelectBeta(0.1), std::invalid_argument); } + +// Test that we can train a sparse model on dense data. +TEMPLATE_TEST_CASE("LARSSparseModelDenseData", "[LARSTest]", float, double) +{ + typedef TestType eT; + + // 10k-dimensional data. + arma::Mat data(10000, 1000, arma::fill::randu); + + arma::SpCol betaSp; + betaSp.sprandu(10000, 1, 0.1); + arma::Col beta = betaSp + arma::randu>(10000) * 0.01; + + // Create slightly noisy responses. + arma::Row responses = beta.t() * data + + 0.02 * arma::randu>(1000); + + LARS> lars1(data, responses, true, true, 0.5, 0.01); + LARS> lars2(true, 0.01, 1e-4); + lars2.Train(data, responses); + + // Make sure we at least approximately recovered the solution vector. + REQUIRE(lars1.Beta().n_elem == 10000); + REQUIRE(lars2.Beta().n_elem == 10000); + + REQUIRE((arma::norm(lars1.Beta() - beta, 2) / lars1.Beta().n_elem) < 0.01); + REQUIRE((arma::norm(lars2.Beta() - beta, 2) / lars2.Beta().n_elem) < 0.01); + + // Make some predictions and ensure they are approximately correct. + arma::Row responses1, responses2; + lars1.Predict(data, responses1); + lars2.Predict(data, responses2); + + REQUIRE(responses1.n_elem == responses.n_elem); + REQUIRE(responses2.n_elem == responses.n_elem); + + REQUIRE((arma::accu(arma::abs(responses - responses1)) / responses.n_elem) + < 0.1); + REQUIRE((arma::accu(arma::abs(responses - responses2)) / responses.n_elem) + < 0.1); + + // Make sure ComputeError returns something reasonable. This actually could + // be quite large because the responses tend to be large. + REQUIRE((lars1.ComputeError(data, responses) / responses.n_elem) < 100000); + REQUIRE((lars2.ComputeError(data, responses) / responses.n_elem) < 100000); + + REQUIRE(lars1.ActiveSet().size() < 10000); + REQUIRE(lars1.ActiveSet().size() > 0); + + REQUIRE(lars2.ActiveSet().size() < 10000); + REQUIRE(lars2.ActiveSet().size() > 0); +} diff --git a/src/mlpack/tests/serialization_test.cpp b/src/mlpack/tests/serialization_test.cpp index b0d12db35e..cb42706071 100644 --- a/src/mlpack/tests/serialization_test.cpp +++ b/src/mlpack/tests/serialization_test.cpp @@ -1061,11 +1061,11 @@ TEST_CASE("LARSTest", "[SerializationTest]") arma::vec beta = arma::randn(75, 1); arma::rowvec y = beta.t() * X; - LARS lars(true, 0.1, 0.1); + LARS<> lars(true, 0.1, 0.1); lars.Train(X, y); // Now, serialize. - LARS xmlLars(false, 0.5, 0.0), binaryLars(true, 1.0, 0.0), + LARS<> xmlLars(false, 0.5, 0.0), binaryLars(true, 1.0, 0.0), jsonLars(false, 0.1, 0.1); // Train jsonLars. From 53a108f5ea65c755a8a3f32f0d0e03178740a4bf Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Fri, 8 Dec 2023 09:05:32 -0500 Subject: [PATCH 35/91] Fix errors in examples. --- doc/user/methods/lars.md | 76 +++++++++++++++++++++------------------- 1 file changed, 40 insertions(+), 36 deletions(-) diff --git a/doc/user/methods/lars.md b/doc/user/methods/lars.md index 475c9e41f4..04f652a4b6 100644 --- a/doc/user/methods/lars.md +++ b/doc/user/methods/lars.md @@ -181,8 +181,8 @@ Types of each argument are the same as in the table for constructors * Training is not incremental. A second call to `Train()` will retrain the model from scratch. - * `Train()` returns the mean squared error (MSE) of the model on the training - set as a `double`. + * `Train()` returns the squared error (loss) of the model on the training set + as a `double`. To obtain the MSE, divide by the number of training points. ### Prediction @@ -232,8 +232,9 @@ can be used to make predictions for new data. indices of nonzero dimensions in the model parameters (`lars.Beta()`). * `lars.ComputeError(data, responses, rowMajor=false)` will return a `double` - containing the mean squared error (MSE) of the model on `data`, given that - the true responses are `responses`. + containing the squared error of the model on `data`, given that + the true responses are `responses`. To obtain the MSE, divide by the number + of points in `data`. ### The LARS Path @@ -254,9 +255,9 @@ switch between them for prediction purposes: * `lars.LambdaPath()` returns a `std::vector&` containing each `lambda1` value that is associated with each element in `lars.BetaPath()` and - `lars.InterceptPath()`. That is, `lars.LambdaPath(i)` is the `lambda1` value - corresponding to the model defined by `lars.BetaPath(i)` and - `lars.InterceptPath(i)`. + `lars.InterceptPath()`. That is, `lars.LambdaPath()[i]` is the `lambda1` + value corresponding to the model defined by `lars.BetaPath()[i]` and + `lars.InterceptPath()[i]`. * `lars.SelectBeta(lambda1)` will set the model weights (`lars.ActiveSet()`, @@ -266,14 +267,14 @@ switch between them for prediction purposes: cannot be greater than `lars.Lambda1()`, or an exception will be thrown. - * `lars.SelectedLambda()` returns the currently selected L1 regularization + * `lars.SelectedLambda1()` returns the currently selected L1 regularization penalty parameter. * For any value `lambda1` between `lars.LambdaPath(i)` and `lars.LambdaPath(i + 1)`, the corresponding model is a linear interpolation between - `lars.BetaPath(i)` and `lars.BetaPath(i + 1)` (and `lars.InterceptPath(i)` - and `lars.InterceptPath(i + 1)`). This exact linear interpolation is what is - computed by `lars.SelectBeta(lambda1)`. + `lars.BetaPath()[i]` and `lars.BetaPath()[i + 1]` (and + `lars.InterceptPath()[i]` and `lars.InterceptPath()[i + 1]`). This exact + linear interpolation is what is computed by `lars.SelectBeta(lambda1)`. ### Simple Examples @@ -290,15 +291,17 @@ test data for each set of weights in the path. arma::mat data; mlpack::data::Load("wave_energy_farm_100.csv", data, true); -// Split the last row off: it is the responses. +// Split the last row off: it is the responses. Also, normalize the responses +// to [0, 1]. arma::rowvec responses = data.row(data.n_rows - 1); -data.shed_rows(data.n_rows - 1); +responses /= responses.max(); +data.shed_row(data.n_rows - 1); // Split into a training and test dataset. 20% of the data is held out as a // test set. arma::mat trainingData, testData; arma::rowvec trainingResponses, testResponses; -data::Split(data, responses, trainingData, trainingResponses, testData, +mlpack::data::Split(data, responses, trainingData, testData, trainingResponses, testResponses, 0.2); // Train a LARS model with lambda1 = 1e-5 and lambda2 = 1e-6. @@ -309,13 +312,15 @@ const size_t pathLength = lars.BetaPath().size(); for (size_t i = 0; i < pathLength; ++i) { // Use the i'th model in the path. - lars.SelectBeta(lars.BetaPath(i)); + lars.SelectBeta(lars.LambdaPath()[i]); - std::cout << "L1 penalty parameter: " << lars.SelectedLambda() << std::endl; - std::cout << " MSE on training set: " - << lars.ComputeError(trainingData, trainingResponses) << "." << std::endl; - std::cout << " MSE on test set: " - << lars.ComputeError(testData, testResponses) << "." << std::endl; + const double trainMSE = lars.ComputeError(trainingData, trainingResponses) / + trainingData.n_cols; + const double testMSE = lars.ComputeError(testData, testResponses) / + testData.n_cols; + std::cout << "L1 penalty parameter: " << lars.SelectedLambda1() << std::endl; + std::cout << " MSE on training set: " << trainMSE << "." << std::endl; + std::cout << " MSE on test set: " << testMSE << "." << std::endl; } ``` @@ -355,7 +360,7 @@ Load a LARS model from disk and print some information about it. mlpack::LARS lars; mlpack::data::Load("lars_model.bin", "lars", lars, true); -if (lars.Beta().n_elem) +if (lars.BetaPath().size() == 0) { std::cout << "lars_model.bin contains an untrained LARS model." << std::endl; } @@ -369,7 +374,7 @@ else << (lars.FitIntercept() ? std::string("yes") : std::string("no")) << "." << std::endl; std::cout << " - Current L1 regularization penalty parameter value: " - << lars.SelectedLambda() << "." << std::endl; + << lars.SelectedLambda1() << "." << std::endl; std::cout << " - L2 regularization penalty parameter: " << lars.Lambda2() << "." << std::endl; std::cout << " - Number of nonzero elements in model: " @@ -403,17 +408,17 @@ mlpack::data::Load("admission_predict.responses.csv", responses, true); // Precompute Gram matrix. arma::mat gramMatrix = data * data.t(); -std::vector lambda2Values = { 0.0001, 0.001, 0.01, 0.1, 1.0 }; -for (lambda2 : lambda2Values) +std::vector lambda2Values = { 0.01, 0.1, 1.0, 10.0, 100.0 }; +for (double lambda2 : lambda2Values) { // Build a LARS model using the precomputed Gram matrix. We did not normalize // or center the data before computing the Gram matrix, so we have to set // fitIntercept and normalizeData accordingly. - LARS lars(data, responses, true, true, gramMatrix, 0.01, lambda2, 1e-16, - false, false); + mlpack::LARS lars(data, responses, true, true, gramMatrix, 0.01, lambda2, + 1e-16, false, false); std::cout << "MSE with L2 penalty " << lambda2 << ": " - << lars.ComputeError(data, responses) << "." << std::endl; + << (lars.ComputeError(data, responses) / data.n_cols) << "." << std::endl; } ``` @@ -439,8 +444,7 @@ The example below trains a LARS model on sparse 32-bit precision data, using ```c++ // Create random, sparse 1000-dimensional data. -arma::sp_fmat dataset; -dataset.sprandu(1000, 5000, 0.1); +arma::fmat dataset(1000, 5000, arma::fill::randu); // Generate noisy responses from random data. arma::fvec trueWeights(1000, arma::fill::randu); @@ -454,16 +458,16 @@ lars.Lambda2() = 0.01; lars.Train(dataset, responses); // Compute the MSE on the training set and a random test set. -arma::sp_fmat testDataset; -testDataset.sprandu(1000, 2500, 0.3); - +arma::fmat testDataset(1000, 2500, arma::fill::randu); arma::frowvec testResponses = trueWeights.t() * testDataset + 0.01 * arma::randu(2500) /* noise term */; -std::cout << "MSE on training set: " - << lars.ComputeError(dataset, responses) << "." << std::endl; -std::cout << "MSE on test set: " - << lars.ComputeError(testDataset, testResponses) << "." << std::endl; +const float trainMSE = lars.ComputeError(dataset, responses) / dataset.n_cols; +const float testMSE = lars.ComputeError(testDataset, testResponses) / + testDataset.n_cols; + +std::cout << "MSE on training set: " << trainMSE << "." << std::endl; +std::cout << "MSE on test set: " << testMSE << "." << std::endl; ``` ***Note:*** it is generally only more efficient to use a sparse type (e.g. From bbf3002aa7e17ff510baf60b2d9cb5e0447a0761 Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Fri, 8 Dec 2023 09:06:00 -0500 Subject: [PATCH 36/91] Test single-point Predict() and fix bug. --- src/mlpack/methods/lars/lars_impl.hpp | 4 ++-- src/mlpack/tests/lars_test.cpp | 19 +++++++++++++++++++ 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/src/mlpack/methods/lars/lars_impl.hpp b/src/mlpack/methods/lars/lars_impl.hpp index cb1a45fbe8..44e78bd159 100644 --- a/src/mlpack/methods/lars/lars_impl.hpp +++ b/src/mlpack/methods/lars/lars_impl.hpp @@ -960,9 +960,9 @@ inline typename LARS::ElemType LARS::Predict( const VecType& point) const { if (!fitIntercept) - return Beta().t() * point; + return arma::dot(Beta(), point); else - return Beta().t() * point + Intercept(); + return arma::dot(Beta(), point) + Intercept(); } template diff --git a/src/mlpack/tests/lars_test.cpp b/src/mlpack/tests/lars_test.cpp index c60b3eb0a9..11f29706ff 100644 --- a/src/mlpack/tests/lars_test.cpp +++ b/src/mlpack/tests/lars_test.cpp @@ -239,6 +239,25 @@ TEMPLATE_TEST_CASE("PredictTest", "[LARSTest]", arma::fmat, arma::mat) else REQUIRE(adjPred[i] == Approx(betaOptPred[i]).epsilon(1e-7)); } + + // Now check with single-point Predict(). + for (size_t i = 0; i < X.n_cols; ++i) + { + // Pass different types into Predict() to test templating support. + const ElemType pred1 = lars.Predict(X.col(i)); + const ElemType pred2 = lars.Predict(X.unsafe_col(i)); + + if (std::abs(betaOptPred[i]) < 1e-5) + { + REQUIRE(pred1 == Approx(0.0).margin(1e-5)); + REQUIRE(pred2 == Approx(0.0).margin(1e-5)); + } + else + { + REQUIRE(pred1 == Approx(betaOptPred[i]).epsilon(1e-7)); + REQUIRE(pred2 == Approx(betaOptPred[i]).epsilon(1e-7)); + } + } } } } From bef302286ef999e79d8ab49f5e4a547443ebdb58 Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Fri, 8 Dec 2023 09:06:14 -0500 Subject: [PATCH 37/91] Use the correct model in ComputeError() and consider the intercept too. --- src/mlpack/methods/lars/lars_impl.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/mlpack/methods/lars/lars_impl.hpp b/src/mlpack/methods/lars/lars_impl.hpp index 44e78bd159..594c647a2a 100644 --- a/src/mlpack/methods/lars/lars_impl.hpp +++ b/src/mlpack/methods/lars/lars_impl.hpp @@ -1326,12 +1326,12 @@ LARS::ComputeError(const MatType& matX, { if (rowMajor) { - return arma::accu(arma::pow(y - trans(matX * betaPath.back()), 2.0)); + return arma::accu(arma::pow(y - trans(matX * Beta()) - Intercept(), 2.0)); } else { - return arma::accu(arma::pow(y - betaPath.back().t() * matX, 2.0)); + return arma::accu(arma::pow(y - Beta().t() * matX - Intercept(), 2.0)); } } From fa1673b9baf2c26d2ecef8c4b595300f4da81270 Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Fri, 8 Dec 2023 09:15:41 -0500 Subject: [PATCH 38/91] Clean up errors in some tests and make them take less long. --- src/mlpack/tests/lars_test.cpp | 69 ++++++++++++++++++++-------------- 1 file changed, 41 insertions(+), 28 deletions(-) diff --git a/src/mlpack/tests/lars_test.cpp b/src/mlpack/tests/lars_test.cpp index 11f29706ff..0e50d2a9e6 100644 --- a/src/mlpack/tests/lars_test.cpp +++ b/src/mlpack/tests/lars_test.cpp @@ -231,32 +231,42 @@ TEMPLATE_TEST_CASE("PredictTest", "[LARSTest]", arma::fmat, arma::mat) lars.Predict(X, predictions); arma::Col adjPred = X * predictions.t(); + const ElemType tol = (std::is_same::value) ? 1e-7 : + 1e-4; + REQUIRE(predictions.n_elem == 1000); for (size_t i = 0; i < betaOptPred.n_elem; ++i) { if (std::abs(betaOptPred[i]) < 1e-5) REQUIRE(adjPred[i] == Approx(0.0).margin(1e-5)); else - REQUIRE(adjPred[i] == Approx(betaOptPred[i]).epsilon(1e-7)); + REQUIRE(adjPred[i] == Approx(betaOptPred[i]).epsilon(tol)); } - // Now check with single-point Predict(). + // Now check with single-point Predict(), in two ways: we will pass + // different types into Predict() to test templating support. for (size_t i = 0; i < X.n_cols; ++i) - { - // Pass different types into Predict() to test templating support. - const ElemType pred1 = lars.Predict(X.col(i)); - const ElemType pred2 = lars.Predict(X.unsafe_col(i)); + predictions[i] = lars.Predict(X.col(i)); + adjPred = X * predictions.t(); + for (size_t i = 0; i < betaOptPred.n_elem; ++i) + { if (std::abs(betaOptPred[i]) < 1e-5) - { - REQUIRE(pred1 == Approx(0.0).margin(1e-5)); - REQUIRE(pred2 == Approx(0.0).margin(1e-5)); - } + REQUIRE(adjPred[i] == Approx(0.0).margin(1e-5)); else - { - REQUIRE(pred1 == Approx(betaOptPred[i]).epsilon(1e-7)); - REQUIRE(pred2 == Approx(betaOptPred[i]).epsilon(1e-7)); - } + REQUIRE(adjPred[i] == Approx(betaOptPred[i]).epsilon(tol)); + } + + for (size_t i = 0; i < X.n_cols; ++i) + predictions[i] = lars.Predict(X.unsafe_col(i)); + + adjPred = X * predictions.t(); + for (size_t i = 0; i < betaOptPred.n_elem; ++i) + { + if (std::abs(betaOptPred[i]) < 1e-5) + REQUIRE(adjPred[i] == Approx(0.0).margin(1e-5)); + else + REQUIRE(adjPred[i] == Approx(betaOptPred[i]).epsilon(tol)); } } } @@ -1093,6 +1103,8 @@ TEMPLATE_TEST_CASE("LARSSelectBetaTest", "[LARSTest]", arma::fmat, arma::mat) arma::Row y; GenerateProblem(X, y, 1000, 100); + // Add some noise. + y += 0.2 * arma::randu>(y.n_elem); LARS lars(X, y); @@ -1102,6 +1114,8 @@ TEMPLATE_TEST_CASE("LARSSelectBetaTest", "[LARSTest]", arma::fmat, arma::mat) // Now step through numerous different lambda values. ElemType lastError = std::numeric_limits::max(); + const ElemType errorTol = (std::is_same::value) ? 1e-10 : + 1e-5; for (ElemType i = 5.0; i >= -5.0; i -= 0.1) { const ElemType selLambda1 = std::pow(10.0, (ElemType) i); @@ -1111,7 +1125,7 @@ TEMPLATE_TEST_CASE("LARSSelectBetaTest", "[LARSTest]", arma::fmat, arma::mat) REQUIRE(lars.SelectedLambda1() == Approx(selLambda1).margin(tol)); REQUIRE(arma::accu(lars.Beta() != 0.0) == lars.ActiveSet().size()); const ElemType newError = lars.ComputeError(X, y); - REQUIRE(newError <= lastError); + REQUIRE(newError <= lastError + errorTol); lastError = newError; } @@ -1158,24 +1172,24 @@ TEMPLATE_TEST_CASE("LARSSparseModelDenseData", "[LARSTest]", float, double) { typedef TestType eT; - // 10k-dimensional data. - arma::Mat data(10000, 1000, arma::fill::randu); + // 1k-dimensional data. + arma::Mat data(1000, 500, arma::fill::randu); arma::SpCol betaSp; - betaSp.sprandu(10000, 1, 0.1); - arma::Col beta = betaSp + arma::randu>(10000) * 0.01; + betaSp.sprandu(1000, 1, 0.1); + arma::Col beta = betaSp + arma::randu>(1000) * 0.01; // Create slightly noisy responses. arma::Row responses = beta.t() * data + - 0.02 * arma::randu>(1000); + 0.02 * arma::randu>(500); LARS> lars1(data, responses, true, true, 0.5, 0.01); LARS> lars2(true, 0.01, 1e-4); lars2.Train(data, responses); // Make sure we at least approximately recovered the solution vector. - REQUIRE(lars1.Beta().n_elem == 10000); - REQUIRE(lars2.Beta().n_elem == 10000); + REQUIRE(lars1.Beta().n_elem == 1000); + REQUIRE(lars2.Beta().n_elem == 1000); REQUIRE((arma::norm(lars1.Beta() - beta, 2) / lars1.Beta().n_elem) < 0.01); REQUIRE((arma::norm(lars2.Beta() - beta, 2) / lars2.Beta().n_elem) < 0.01); @@ -1193,14 +1207,13 @@ TEMPLATE_TEST_CASE("LARSSparseModelDenseData", "[LARSTest]", float, double) REQUIRE((arma::accu(arma::abs(responses - responses2)) / responses.n_elem) < 0.1); - // Make sure ComputeError returns something reasonable. This actually could - // be quite large because the responses tend to be large. - REQUIRE((lars1.ComputeError(data, responses) / responses.n_elem) < 100000); - REQUIRE((lars2.ComputeError(data, responses) / responses.n_elem) < 100000); + // Make sure ComputeError returns something reasonable. + REQUIRE((lars1.ComputeError(data, responses) / responses.n_elem) < 1); + REQUIRE((lars2.ComputeError(data, responses) / responses.n_elem) < 1); - REQUIRE(lars1.ActiveSet().size() < 10000); + REQUIRE(lars1.ActiveSet().size() < 1000); REQUIRE(lars1.ActiveSet().size() > 0); - REQUIRE(lars2.ActiveSet().size() < 10000); + REQUIRE(lars2.ActiveSet().size() < 1000); REQUIRE(lars2.ActiveSet().size() > 0); } From 632d7abd9c5912a17936cd673f1d0a839d8cd0ec Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Fri, 8 Dec 2023 09:50:37 -0500 Subject: [PATCH 39/91] Change transposeData parameter to colMajor for clarity. --- doc/user/methods/lars.md | 28 ++++----- src/mlpack/methods/lars/lars.hpp | 82 ++++++++++++------------- src/mlpack/methods/lars/lars_impl.hpp | 87 +++++++++++++-------------- src/mlpack/tests/lars_test.cpp | 4 +- 4 files changed, 95 insertions(+), 106 deletions(-) diff --git a/doc/user/methods/lars.md b/doc/user/methods/lars.md index 04f652a4b6..1362e90c8b 100644 --- a/doc/user/methods/lars.md +++ b/doc/user/methods/lars.md @@ -1,10 +1,10 @@ ## `LARS` The `LARS` class implements the least-angle regression (LARS) algorithm for -L1-penalized and L2-penalized regression. `LARS` can also solve the LASSO -(least absolute shrinkage and selection operator) problem. The LARS algorithm -is a *path* algorithm, and thus will recover solutions for *all* L1 penalty -parameters greater than or equal to the given L1 penalty parameter. +L1-penalized and L2-penalized linear regression. `LARS` can also solve the +LASSO (least absolute shrinkage and selection operator) problem. The LARS +algorithm is a *path* algorithm, and thus will recover solutions for *all* L1 +penalty parameters greater than or equal to the given L1 penalty parameter. #### Simple usage example: @@ -59,14 +59,13 @@ std::cout << arma::accu(predictions < 0) << " test points predicted to have " --- - - * `lars = LARS(data, responses, transposeData=true, useCholesky=true, lambda1=0.0, lambda2=0.0, tolerance=1e-16, fitIntercept=true, normalizeData=true)` + * `lars = LARS(data, responses, colMajor=true, useCholesky=true, lambda1=0.0, lambda2=0.0, tolerance=1e-16, fitIntercept=true, normalizeData=true)` - Train model on the given data and responses, using the given settings for hyperparameters. --- - * `lars = LARS(data, responses, transposeData, useCholesky, gramMatrix, lambda1=0.0, lambda2=0.0, tolerance=1e-16, fitIntercept=true, normalizeData=true)` + * `lars = LARS(data, responses, colMajor, useCholesky, gramMatrix, lambda1=0.0, lambda2=0.0, tolerance=1e-16, fitIntercept=true, normalizeData=true)` - *(Advanced constructor)*. - Train model on the given data and responses, using a precomputed Gram matrix (`gramMatrix`, equivalent to `data * data.t()`). @@ -92,7 +91,7 @@ std::cout << arma::accu(predictions < 0) << " test points predicted to have " |----------|----------|-----------------|-------------| | `data` | [`arma::mat`](../matrices.md) | Training matrix. | _(N/A)_ | | `responses` | [`arma::rowvec`](../matrices.md) | Training responses (e.g. values to predict). Should have length `data.n_cols`. | _(N/A)_ | -| `transposeData` | `bool` | Should be set to true if `data` is [column-major](../matrices.md). Passing row-major data can avoid a transpose operation. | `true` | +| `colMajor` | `bool` | Should be set to `true` if `data` is [column-major](../matrices.md). Passing row-major data can avoid a transpose operation. | `false` | | `useCholesky` | `bool` | If `true`, use the Cholesky decomposition of the Gram matrix to solve linear systems (as opposed to the full Gram matrix). | `true` | | `gramMatrix` | [`arma::mat`](../matrices.md) | Precomputed Gram matrix of `data` (i.e. `data * data.t()` for column-major data). | _(N/A)_ | @@ -100,15 +99,12 @@ matrix to solve linear systems (as opposed to the full Gram matrix). | `true` | | `lambda2` | `double` | L2 regularization penalty parameter. | `0.0` | | `tolerance` | `double` | Tolerance on feature correlations for convergence. | `1e-16` | | `fitIntercept` | `bool` | If `true`, an intercept term will be included in the model. | `true` | -| `normalizeData` | `bool` | If `true`, data will be normalized before fitting -the model. | `true` | +| `normalizeData` | `bool` | If `true`, data will be normalized before fitting the model. | `true` | As an alternative to passing hyperparameters, each hyperparameter can be set with a standalone method. The following functions can be used before calling `Train()` to set hyperparameters: - * `lars.RowMajor() = rowMajor;` will set whether the data is given in row-major - form to `rowMajor`. * `lars.UseCholesky() = useChol;` will set whether or not the Cholesky decomposition will be used during training to `useChol`. * `lars.Lambda1() = lambda1;` will set the L1 regularization penalty parameter @@ -153,12 +149,12 @@ If training is not done as part of the constructor call, it can be done with the - * `lars.Train(data, responses, transposeData=true, useCholesky=true, lambda1=0.0, lambda2=0.0, tolerance=1e-16, fitIntercept=true, normalizeData=true)` + * `lars.Train(data, responses, colMajor=true, useCholesky=true, lambda1=0.0, lambda2=0.0, tolerance=1e-16, fitIntercept=true, normalizeData=true)` - Train the model on the given data. --- - * `lars.Train(data, responses, transposeData, useCholesky, gramMatrix, lambda1=0.0, lambda2=0.0, tolerance=1e-16, fitIntercept=true, normalizeData=true)` + * `lars.Train(data, responses, colMajor, useCholesky, gramMatrix, lambda1=0.0, lambda2=0.0, tolerance=1e-16, fitIntercept=true, normalizeData=true)` - *(Advanced training.)* - Train model on the given data and responses, using a precomputed Gram matrix (`gramMatrix`, equivalent to `data * data.t()`). @@ -231,7 +227,7 @@ can be used to make predictions for new data. * `lars.ActiveSet()` will return a `std::vector&` containing the indices of nonzero dimensions in the model parameters (`lars.Beta()`). - * `lars.ComputeError(data, responses, rowMajor=false)` will return a `double` + * `lars.ComputeError(data, responses, colMajor=false)` will return a `double` containing the squared error of the model on `data`, given that the true responses are `responses`. To obtain the MSE, divide by the number of points in `data`. @@ -263,7 +259,7 @@ switch between them for prediction purposes: * `lars.SelectBeta(lambda1)` will set the model weights (`lars.ActiveSet()`, `lars.Beta()` and `lars.Intercept()`) to the path location with L1 penalty `lambda1`. This is equivalent to calling `lars.Train(data, responses, - transposeData, useCholesky, lambda1)`---but much more efficient! `lambda1` + colMajor, useCholesky, lambda1)`---but much more efficient! `lambda1` cannot be greater than `lars.Lambda1()`, or an exception will be thrown. diff --git a/src/mlpack/methods/lars/lars.hpp b/src/mlpack/methods/lars/lars.hpp index 2e6ec7346d..a7c0949856 100644 --- a/src/mlpack/methods/lars/lars.hpp +++ b/src/mlpack/methods/lars/lars.hpp @@ -151,8 +151,8 @@ class LARS * * @param data Input data. * @param responses A vector of targets. - * @param transposeData Should be true if the input data is column-major and - * false otherwise. + * @param colMajor Should be true if the input data is column-major. Passing + * row-major data can avoid a transpose operation. * @param useCholesky Whether or not to use Cholesky decomposition when * solving linear system (as opposed to using the full Gram matrix). * @param lambda1 Regularization parameter for l1-norm penalty. @@ -170,7 +170,7 @@ class LARS >::type> LARS(const MatType& data, const ResponsesType& responses, - bool transposeData = true, + bool colMajor = true, const bool useCholesky = false, const ElemType lambda1 = 0.0, const ElemType lambda2 = 0.0, @@ -189,8 +189,8 @@ class LARS * * @param data Input data. * @param responses A vector of targets. - * @param transposeData Should be true if the input data is column-major and - * false otherwise. + * @param colMajor Should be true if the input data is column-major. Passing + * row-major data can avoid a transpose operation. * @param useCholesky Whether or not to use Cholesky decomposition when * solving linear system (as opposed to using the full Gram matrix). * @param gramMatrix Gram matrix. @@ -209,7 +209,7 @@ class LARS >::type> LARS(const MatType& data, const ResponsesType& responses, - const bool transposeData, + const bool colMajor, const bool useCholesky, const DenseMatType& gramMatrix, const ElemType lambda1 = 0.0, @@ -251,37 +251,38 @@ class LARS * column-major -- each column is an observation and each row is a dimension. * However, because LARS is more efficient on a row-major matrix, this method * will (internally) transpose the matrix. If this transposition is not - * necessary (i.e., you want to pass in a row-major matrix), pass 'false' for - * the transposeData parameter. + * necessary (i.e., you want to pass in a row-major matrix), pass `false` for + * the `colMajor` parameter. * - * @param data Column-major input data (or row-major input data if rowMajor = - * true). + * @param data Column-major input data (or row-major input data if colMajor = + * false). * @param responses A vector of targets. * @param beta Vector to store the solution (the coefficients) in. - * @param transposeData Set to false if the data is row-major. + * @param colMajor Should be true if the input data is column-major. Passing + * row-major data can avoid a transpose operation. * @return minimum cost error(||y-beta*X||2 is used to calculate error). */ mlpack_deprecated double Train(const arma::mat& data, const arma::rowvec& responses, arma::vec& beta, - const bool transposeData = true); + const bool colMajor = true); /** * Run LARS. The input matrix (like all mlpack matrices) should be * column-major -- each column is an observation and each row is a dimension. * However, because LARS is more efficient on a row-major matrix, this method * will (internally) transpose the matrix. If this transposition is not - * necessary (i.e., you want to pass in a row-major matrix), pass 'false' for - * the transposeData parameter. + * necessary (i.e., you want to pass in a row-major matrix), pass `false` for + * the `colMajor` parameter. * * All of the different overloads below are needed until C++17 is the minimum * required standard (then std::optional could be used). * * @param data Input data. * @param responses A vector of targets. - * @param transposeData Should be true if the input data is column-major and - * false otherwise. + * @param colMajor Should be true if the input data is column-major. Passing + * row-major data can avoid a transpose operation. * @return minimum cost error(||y-beta*X||2 is used to calculate error). */ @@ -290,7 +291,7 @@ class LARS template ElemType Train(const MatType& data, const arma::rowvec& responses, - const bool transposeData = true); + const bool colMajor = true); template::type> ElemType Train(const MatType& data, const ResponsesType& responses, - const bool transposeData = true); + const bool colMajor = true); template::type> ElemType Train(const MatType& data, const ResponsesType& responses, - const bool transposeData, + const bool colMajor, const bool useCholesky); template::type> ElemType Train(const MatType& data, const ResponsesType& responses, - const bool transposeData, + const bool colMajor, const bool useCholesky, const ElemType lambda1); @@ -336,7 +337,7 @@ class LARS >::type> ElemType Train(const MatType& data, const ResponsesType& responses, - const bool transposeData, + const bool colMajor, const bool useCholesky, const ElemType lambda1, const ElemType lambda2); @@ -349,7 +350,7 @@ class LARS >::type> ElemType Train(const MatType& data, const ResponsesType& responses, - const bool transposeData, + const bool colMajor, const bool useCholesky, const ElemType lambda1, const ElemType lambda2, @@ -363,7 +364,7 @@ class LARS >::type> ElemType Train(const MatType& data, const ResponsesType& responses, - const bool transposeData, + const bool colMajor, const bool useCholesky, const ElemType lambda1, const ElemType lambda2, @@ -378,7 +379,7 @@ class LARS >::type> ElemType Train(const MatType& data, const ResponsesType& responses, - const bool transposeData, + const bool colMajor, const bool useCholesky, const ElemType lambda1, const ElemType lambda2, @@ -392,15 +393,15 @@ class LARS * row is a dimension. However, because LARS is more efficient on a row-major * matrix, this method will (internally) transpose the matrix. If this * transposition is not necessary (i.e., you want to pass in a row-major - * matrix), pass 'false' for the transposeData parameter. + * matrix), pass `false` for the `colMajor` parameter. * * All of the different overloads below are needed until C++17 is the minimum * required standard (then std::optional could be used). * * @param data Input data. * @param responses A vector of targets. - * @param transposeData Should be true if the input data is column-major and - * false otherwise. + * @param colMajor Should be true if the input data is column-major. Passing + * row-major data can avoid a transpose operation. * @return minimum cost error(||y-beta*X||2 is used to calculate error). */ template::type> ElemType Train(const MatType& data, const ResponsesType& responses, - const bool transposeData, + const bool colMajor, const bool useCholesky, const DenseMatType& gramMatrix); @@ -423,7 +424,7 @@ class LARS >::type> ElemType Train(const MatType& data, const ResponsesType& responses, - const bool transposeData, + const bool colMajor, const bool useCholesky, const DenseMatType& gramMatrix, const ElemType lambda1); @@ -436,7 +437,7 @@ class LARS >::type> ElemType Train(const MatType& data, const ResponsesType& responses, - const bool transposeData, + const bool colMajor, const bool useCholesky, const DenseMatType& gramMatrix, const ElemType lambda1, @@ -450,7 +451,7 @@ class LARS >::type> ElemType Train(const MatType& data, const ResponsesType& responses, - const bool transposeData, + const bool colMajor, const bool useCholesky, const DenseMatType& gramMatrix, const ElemType lambda1, @@ -465,7 +466,7 @@ class LARS >::type> ElemType Train(const MatType& data, const ResponsesType& responses, - const bool transposeData, + const bool colMajor, const bool useCholesky, const DenseMatType& gramMatrix, const ElemType lambda1, @@ -481,7 +482,7 @@ class LARS >::type> ElemType Train(const MatType& data, const ResponsesType& responses, - const bool transposeData, + const bool colMajor, const bool useCholesky, const DenseMatType& gramMatrix, const ElemType lambda1, @@ -505,13 +506,13 @@ class LARS * * @param points The data points to regress on. * @param predictions y, which will contained calculated values on completion. - * @param rowMajor Should be true if the data points matrix is row-major and - * false otherwise. + * @param colMajor Should be true if the input data is column-major. Passing + * row-major data can avoid a transpose operation. */ template void Predict(const MatType& points, ResponsesType& predictions, - const bool rowMajor = false) const; + const bool colMajor = true) const; //! Get the L1 regularization coefficient. ElemType Lambda1() const { return lambda1; } @@ -584,17 +585,16 @@ class LARS * currently-trained LARS model. Only ||y-beta*X||2 is used to calculate * cost error. * - * @param matX Column-major input data (or row-major input data if rowMajor = - * true). + * @param matX Column-major input data (or row-major input data if colMajor = + * false). * @param y responses A vector of targets. - * @param rowMajor Should be true if the data points matrix is row-major and - * false otherwise. + * @param colMajor Should be true if the data points matrix is column-major. * @return The minimum cost error. */ template ElemType ComputeError(const MatType& matX, const ResponsesType& y, - const bool rowMajor = false); + const bool colMajor = true); private: //! Gram matrix. diff --git a/src/mlpack/methods/lars/lars_impl.hpp b/src/mlpack/methods/lars/lars_impl.hpp index 594c647a2a..1e8db7eeb8 100644 --- a/src/mlpack/methods/lars/lars_impl.hpp +++ b/src/mlpack/methods/lars/lars_impl.hpp @@ -70,7 +70,7 @@ template inline LARS::LARS( const MatType& data, const ResponsesType& responses, - const bool transposeData, + const bool colMajor, const bool useCholesky, const typename LARS::ElemType lambda1, const typename LARS::ElemType lambda2, @@ -79,7 +79,7 @@ inline LARS::LARS( const bool normalizeData) : LARS(useCholesky, lambda1, lambda2, tolerance, fitIntercept, normalizeData) { - Train(data, responses, transposeData); + Train(data, responses, colMajor); } template @@ -87,7 +87,7 @@ template inline LARS::LARS( const MatType& data, const ResponsesType& responses, - const bool transposeData, + const bool colMajor, const bool useCholesky, const typename LARS::DenseMatType& gramMatrix, const typename LARS::ElemType lambda1, @@ -109,7 +109,7 @@ inline LARS::LARS( selectedIntercept(0.0), offsetY(0.0) { - Train(data, responses, transposeData); + Train(data, responses, colMajor); } // Copy Constructor. @@ -253,9 +253,9 @@ mlpack_deprecated inline double LARS::Train(const arma::mat& matX, const arma::rowvec& y, arma::vec& beta, - const bool transposeData) + const bool colMajor) { - const double result = Train(matX, y, transposeData); + const double result = Train(matX, y, colMajor); beta = betaPath.back(); return result; } @@ -266,9 +266,9 @@ template inline typename LARS::ElemType LARS::Train(const MatType& data, const arma::rowvec& responses, - const bool transposeData) + const bool colMajor) { - return Train(data, responses, transposeData, this->useCholesky, this->lambda1, + return Train(data, responses, colMajor, this->useCholesky, this->lambda1, this->lambda2, this->tolerance, this->fitIntercept, this->normalizeData); } @@ -277,9 +277,9 @@ template inline typename LARS::ElemType LARS::Train(const MatType& data, const ResponsesType& responses, - const bool transposeData) + const bool colMajor) { - return Train(data, responses, transposeData, this->useCholesky, this->lambda1, + return Train(data, responses, colMajor, this->useCholesky, this->lambda1, this->lambda2, this->tolerance, this->fitIntercept, this->normalizeData); } @@ -288,10 +288,10 @@ template inline typename LARS::ElemType LARS::Train(const MatType& data, const ResponsesType& responses, - const bool transposeData, + const bool colMajor, const bool useCholesky) { - return Train(data, responses, transposeData, useCholesky, this->lambda1, + return Train(data, responses, colMajor, useCholesky, this->lambda1, this->lambda2, this->tolerance, this->fitIntercept, this->normalizeData); } @@ -300,11 +300,11 @@ template inline typename LARS::ElemType LARS::Train(const MatType& data, const ResponsesType& responses, - const bool transposeData, + const bool colMajor, const bool useCholesky, const typename LARS::ElemType lambda1) { - return Train(data, responses, transposeData, useCholesky, lambda1, + return Train(data, responses, colMajor, useCholesky, lambda1, this->lambda2, this->tolerance, this->fitIntercept, this->normalizeData); } @@ -313,12 +313,12 @@ template inline typename LARS::ElemType LARS::Train(const MatType& data, const ResponsesType& responses, - const bool transposeData, + const bool colMajor, const bool useCholesky, const typename LARS::ElemType lambda1, const typename LARS::ElemType lambda2) { - return Train(data, responses, transposeData, useCholesky, lambda1, lambda2, + return Train(data, responses, colMajor, useCholesky, lambda1, lambda2, this->tolerance, this->fitIntercept, this->normalizeData); } @@ -327,13 +327,13 @@ template inline typename LARS::ElemType LARS::Train(const MatType& data, const ResponsesType& responses, - const bool transposeData, + const bool colMajor, const bool useCholesky, const typename LARS::ElemType lambda1, const typename LARS::ElemType lambda2, const typename LARS::ElemType tolerance) { - return Train(data, responses, transposeData, useCholesky, lambda1, lambda2, + return Train(data, responses, colMajor, useCholesky, lambda1, lambda2, tolerance, this->fitIntercept, this->normalizeData); } @@ -342,14 +342,14 @@ template inline typename LARS::ElemType LARS::Train(const MatType& data, const ResponsesType& responses, - const bool transposeData, + const bool colMajor, const bool useCholesky, const typename LARS::ElemType lambda1, const typename LARS::ElemType lambda2, const typename LARS::ElemType tolerance, const bool fitIntercept) { - return Train(data, responses, transposeData, useCholesky, lambda1, lambda2, + return Train(data, responses, colMajor, useCholesky, lambda1, lambda2, tolerance, fitIntercept, this->normalizeData); } @@ -359,11 +359,11 @@ inline typename LARS::ElemType LARS::Train( const MatType& data, const ResponsesType& responses, - const bool transposeData, + const bool colMajor, const bool useCholesky, const typename LARS::DenseMatType& gramMatrix) { - return Train(data, responses, transposeData, useCholesky, gramMatrix, + return Train(data, responses, colMajor, useCholesky, gramMatrix, this->lambda1, this->lambda2, this->tolerance, this->fitIntercept, this->normalizeData); } @@ -374,12 +374,12 @@ inline typename LARS::ElemType LARS::Train( const MatType& data, const ResponsesType& responses, - const bool transposeData, + const bool colMajor, const bool useCholesky, const typename LARS::DenseMatType& gramMatrix, const typename LARS::ElemType lambda1) { - return Train(data, responses, transposeData, useCholesky, gramMatrix, lambda1, + return Train(data, responses, colMajor, useCholesky, gramMatrix, lambda1, this->lambda2, this->tolerance, this->fitIntercept, this->normalizeData); } @@ -389,13 +389,13 @@ inline typename LARS::ElemType LARS::Train( const MatType& data, const ResponsesType& responses, - const bool transposeData, + const bool colMajor, const bool useCholesky, const typename LARS::DenseMatType& gramMatrix, const typename LARS::ElemType lambda1, const typename LARS::ElemType lambda2) { - return Train(data, responses, transposeData, useCholesky, gramMatrix, lambda1, + return Train(data, responses, colMajor, useCholesky, gramMatrix, lambda1, lambda2, this->tolerance, this->fitIntercept, this->normalizeData); } @@ -405,14 +405,14 @@ inline typename LARS::ElemType LARS::Train( const MatType& data, const ResponsesType& responses, - const bool transposeData, + const bool colMajor, const bool useCholesky, const typename LARS::DenseMatType& gramMatrix, const typename LARS::ElemType lambda1, const typename LARS::ElemType lambda2, const typename LARS::ElemType tolerance) { - return Train(data, responses, transposeData, useCholesky, gramMatrix, lambda1, + return Train(data, responses, colMajor, useCholesky, gramMatrix, lambda1, lambda2, tolerance, this->fitIntercept, this->normalizeData); } @@ -422,7 +422,7 @@ inline typename LARS::ElemType LARS::Train( const MatType& data, const ResponsesType& responses, - const bool transposeData, + const bool colMajor, const bool useCholesky, const typename LARS::DenseMatType& gramMatrix, const typename LARS::ElemType lambda1, @@ -430,7 +430,7 @@ LARS::Train( const typename LARS::ElemType tolerance, const bool fitIntercept) { - return Train(data, responses, transposeData, useCholesky, gramMatrix, lambda1, + return Train(data, responses, colMajor, useCholesky, gramMatrix, lambda1, lambda2, tolerance, fitIntercept, this->normalizeData); } @@ -440,7 +440,7 @@ inline typename LARS::ElemType LARS::Train( const MatType& data, const ResponsesType& responses, - const bool transposeData, + const bool colMajor, const bool useCholesky, const typename LARS::DenseMatType& gramMatrix, const typename LARS::ElemType lambda1, @@ -453,7 +453,7 @@ LARS::Train( matGramInternal.clear(); matGram = &gramMatrix; - return Train(data, responses, transposeData, useCholesky, lambda1, lambda2, + return Train(data, responses, colMajor, useCholesky, lambda1, lambda2, tolerance, fitIntercept, normalizeData); } @@ -462,7 +462,7 @@ template inline typename LARS::ElemType LARS::Train(const MatType& matX, const ResponsesType& y, - const bool transposeData, + const bool colMajor, const bool useCholesky, const typename LARS::ElemType lambda1, const typename LARS::ElemType lambda2, @@ -500,7 +500,7 @@ LARS::Train(const MatType& matX, // dataRef is row-major. We can reuse the given matX, but only if we don't // need to do any transformations to it. const MatType& dataRef = - (transposeData || fitIntercept || normalizeData) ? dataTrans : matX; + (colMajor || fitIntercept || normalizeData) ? dataTrans : matX; const ResponsesType& yRef = (fitIntercept) ? yCentered : y; @@ -508,7 +508,7 @@ LARS::Train(const MatType& matX, this->offsetY = 0.0; // used only if fitting an intercept arma::Col stdX; // used only if normalizing - if (transposeData) + if (colMajor) { if (fitIntercept) { @@ -951,7 +951,7 @@ LARS::Train(const MatType& matX, selectedLambda1 = lambda1; selectedIndex = betaPath.size() - 1; - return ComputeError(matX, y, !transposeData); + return ComputeError(matX, y, colMajor); } template @@ -969,11 +969,11 @@ template template inline void LARS::Predict(const MatType& points, ResponsesType& predictions, - const bool rowMajor) const + const bool colMajor) const { - if (rowMajor && !fitIntercept) + if (!colMajor && !fitIntercept) predictions = trans(points * Beta()); - else if (rowMajor) + else if (!colMajor) predictions = trans(points * Beta()) + Intercept(); else if (fitIntercept) predictions = Beta().t() * points + Intercept(); @@ -1322,17 +1322,12 @@ template inline typename LARS::ElemType LARS::ComputeError(const MatType& matX, const ResponsesType& y, - const bool rowMajor) + const bool colMajor) { - if (rowMajor) - { + if (!colMajor) return arma::accu(arma::pow(y - trans(matX * Beta()) - Intercept(), 2.0)); - } - else - { return arma::accu(arma::pow(y - Beta().t() * matX - Intercept(), 2.0)); - } } /** diff --git a/src/mlpack/tests/lars_test.cpp b/src/mlpack/tests/lars_test.cpp index 0e50d2a9e6..23703b0823 100644 --- a/src/mlpack/tests/lars_test.cpp +++ b/src/mlpack/tests/lars_test.cpp @@ -291,7 +291,7 @@ TEST_CASE("PredictRowMajorTest", "[LARSTest]") arma::rowvec rowMajorPred, colMajorPred; lars.Predict(X, colMajorPred); - lars.Predict(X.t(), rowMajorPred, true); + lars.Predict(X.t(), rowMajorPred, false); REQUIRE(colMajorPred.n_elem == rowMajorPred.n_elem); for (size_t i = 0; i < colMajorPred.n_elem; ++i) @@ -1103,8 +1103,6 @@ TEMPLATE_TEST_CASE("LARSSelectBetaTest", "[LARSTest]", arma::fmat, arma::mat) arma::Row y; GenerateProblem(X, y, 1000, 100); - // Add some noise. - y += 0.2 * arma::randu>(y.n_elem); LARS lars(X, y); From b0f0d9a39a8ed7e555d6322d90d136ea03a29b78 Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Fri, 8 Dec 2023 09:59:08 -0500 Subject: [PATCH 40/91] Fix minor issues found in the documentation. --- doc/user/methods/lars.md | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/doc/user/methods/lars.md b/doc/user/methods/lars.md index 1362e90c8b..5b06664389 100644 --- a/doc/user/methods/lars.md +++ b/doc/user/methods/lars.md @@ -92,8 +92,7 @@ std::cout << arma::accu(predictions < 0) << " test points predicted to have " | `data` | [`arma::mat`](../matrices.md) | Training matrix. | _(N/A)_ | | `responses` | [`arma::rowvec`](../matrices.md) | Training responses (e.g. values to predict). Should have length `data.n_cols`. | _(N/A)_ | | `colMajor` | `bool` | Should be set to `true` if `data` is [column-major](../matrices.md). Passing row-major data can avoid a transpose operation. | `false` | -| `useCholesky` | `bool` | If `true`, use the Cholesky decomposition of the Gram -matrix to solve linear systems (as opposed to the full Gram matrix). | `true` | +| `useCholesky` | `bool` | If `true`, use the Cholesky decomposition of the Gram matrix to solve linear systems (as opposed to the full Gram matrix). | `false` | | `gramMatrix` | [`arma::mat`](../matrices.md) | Precomputed Gram matrix of `data` (i.e. `data * data.t()` for column-major data). | _(N/A)_ | | `lambda1` | `double` | L1 regularization penalty parameter. | `0.0` | | `lambda2` | `double` | L2 regularization penalty parameter. | `0.0` | @@ -193,7 +192,7 @@ can be used to make predictions for new data. --- - * `lars.Predict(data, predictions)` + * `lars.Predict(data, predictions, colMajor=true)` - ***(Multi-point)*** - Make predictions for a set of points. - The prediction for data point `i` can be accessed with `predictions[i]`. @@ -208,7 +207,7 @@ can be used to make predictions for new data. |||| | _multi-point_ | `data` | [`arma::mat`](../matrices.md) | Set of [column-major](../matrices.md) points for classification. | | _multi-point_ | `predictions` | [`arma::rowvec&`](../matrices.md) | Vector of `double`s to store predictions into. Will be set to length `data.n_cols`. | - +| _multi-point_ | `colMajor` | `bool` | Should be set to `true` if `data` is [column-major](../matrices.md). Passing row-major data can avoid a transpose operation. (Default `true`.) | ### Other Functionality @@ -227,7 +226,7 @@ can be used to make predictions for new data. * `lars.ActiveSet()` will return a `std::vector&` containing the indices of nonzero dimensions in the model parameters (`lars.Beta()`). - * `lars.ComputeError(data, responses, colMajor=false)` will return a `double` + * `lars.ComputeError(data, responses, colMajor=true)` will return a `double` containing the squared error of the model on `data`, given that the true responses are `responses`. To obtain the MSE, divide by the number of points in `data`. @@ -260,15 +259,15 @@ switch between them for prediction purposes: `lars.Beta()` and `lars.Intercept()`) to the path location with L1 penalty `lambda1`. This is equivalent to calling `lars.Train(data, responses, colMajor, useCholesky, lambda1)`---but much more efficient! `lambda1` - cannot be greater than `lars.Lambda1()`, or an exception will be thrown. + cannot be less than `lars.Lambda1()`, or an exception will be thrown. * `lars.SelectedLambda1()` returns the currently selected L1 regularization penalty parameter. - * For any value `lambda1` between `lars.LambdaPath(i)` and `lars.LambdaPath(i + - 1)`, the corresponding model is a linear interpolation between - `lars.BetaPath()[i]` and `lars.BetaPath()[i + 1]` (and + * For any value `lambda1` between `lars.LambdaPath()[i]` and + `lars.LambdaPath()[i + 1]`, the corresponding model is a linear interpolation + between `lars.BetaPath()[i]` and `lars.BetaPath()[i + 1]` (and `lars.InterceptPath()[i]` and `lars.InterceptPath()[i + 1]`). This exact linear interpolation is what is computed by `lars.SelectBeta(lambda1)`. @@ -310,6 +309,8 @@ for (size_t i = 0; i < pathLength; ++i) // Use the i'th model in the path. lars.SelectBeta(lars.LambdaPath()[i]); + // ComputeError() returns the total loss, which we need to divide by the + // number of points to get the MSE. const double trainMSE = lars.ComputeError(trainingData, trainingResponses) / trainingData.n_cols; const double testMSE = lars.ComputeError(testData, testResponses) / @@ -433,10 +434,10 @@ used. Note that the `Train()` and `Predict()` functions themselves are templatized and can allow any matrix type that has the same element type. So, for instance, a -`LARS` can accept an `arma::sp_mat` for training. +`LARS` can accept an `arma::mat` for training. -The example below trains a LARS model on sparse 32-bit precision data, using -`arma::sp_mat` to store the model parameters. +The example below trains a LARS model on 32-bit precision data, using +`arma::sp_fmat` to store the model parameters. ```c++ // Create random, sparse 1000-dimensional data. From f7b5377c47bb9301760ad6ebe8203df88204bfc6 Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Fri, 8 Dec 2023 10:01:51 -0500 Subject: [PATCH 41/91] Fix some tiny other issues. --- doc/user/methods/lars.md | 1 + doc/user/methods/linear_regression.md | 1 + 2 files changed, 2 insertions(+) diff --git a/doc/user/methods/lars.md b/doc/user/methods/lars.md index 5b06664389..2f34e3c8c2 100644 --- a/doc/user/methods/lars.md +++ b/doc/user/methods/lars.md @@ -208,6 +208,7 @@ can be used to make predictions for new data. | _multi-point_ | `data` | [`arma::mat`](../matrices.md) | Set of [column-major](../matrices.md) points for classification. | | _multi-point_ | `predictions` | [`arma::rowvec&`](../matrices.md) | Vector of `double`s to store predictions into. Will be set to length `data.n_cols`. | | _multi-point_ | `colMajor` | `bool` | Should be set to `true` if `data` is [column-major](../matrices.md). Passing row-major data can avoid a transpose operation. (Default `true`.) | + ### Other Functionality diff --git a/doc/user/methods/linear_regression.md b/doc/user/methods/linear_regression.md index 583d127062..0b6d102a4d 100644 --- a/doc/user/methods/linear_regression.md +++ b/doc/user/methods/linear_regression.md @@ -44,6 +44,7 @@ std::cout << arma::accu(predictions < 0) << " test points predicted to have " #### See also: * [mlpack regression techniques](#mlpack_regression_techniques) + * [`LARS`](lars.md) * [Linear Regression on Wikipedia](https://en.wikipedia.org/wiki/Linear_regression) ### Constructors From 257182edd64586f4631ff9d08fb48a256776427d Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Fri, 8 Dec 2023 10:08:55 -0500 Subject: [PATCH 42/91] Fix serialization versions. --- src/mlpack/methods/lars/lars.hpp | 3 +++ src/mlpack/methods/linear_regression/linear_regression.hpp | 3 +++ 2 files changed, 6 insertions(+) diff --git a/src/mlpack/methods/lars/lars.hpp b/src/mlpack/methods/lars/lars.hpp index a7c0949856..9aa8d122cd 100644 --- a/src/mlpack/methods/lars/lars.hpp +++ b/src/mlpack/methods/lars/lars.hpp @@ -717,6 +717,9 @@ class LARS } // namespace mlpack +CEREAL_TEMPLATE_CLASS_VERSION((typename ModelMatType), + (mlpack::LARS), (1)); + // Include implementation of serialize(). #include "lars_impl.hpp" diff --git a/src/mlpack/methods/linear_regression/linear_regression.hpp b/src/mlpack/methods/linear_regression/linear_regression.hpp index 8a875aaf6a..bd9cd3ad47 100644 --- a/src/mlpack/methods/linear_regression/linear_regression.hpp +++ b/src/mlpack/methods/linear_regression/linear_regression.hpp @@ -375,6 +375,9 @@ class LinearRegression } // namespace mlpack +CEREAL_TEMPLATE_CLASS_VERSION((typename ModelMatType), + (mlpack::LinearRegression), (1)); + // Include implementation. #include "linear_regression_impl.hpp" From ce6722ff5b146501ec5aa844f5ba3edbfc341902 Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Fri, 8 Dec 2023 17:12:00 -0500 Subject: [PATCH 43/91] Add HoeffdingTree documentation first pass. --- doc/user/methods/hoeffding_tree.md | 610 +++++++++++++++++++++++++++++ 1 file changed, 610 insertions(+) create mode 100644 doc/user/methods/hoeffding_tree.md diff --git a/doc/user/methods/hoeffding_tree.md b/doc/user/methods/hoeffding_tree.md new file mode 100644 index 0000000000..9b7e4b3afd --- /dev/null +++ b/doc/user/methods/hoeffding_tree.md @@ -0,0 +1,610 @@ +## `HoeffdingTree` + +The `HoeffdingTree` class implements a streaming (or incremental) decision tree +classifier that supports numerical and categorical features, by default using +Gini impurity to choose which feature to split on. The class offers several +template parameters and several runtime options that can be used to control the +behavior of the tree. + +Hoeffding trees are useful for classifying points with _discrete labels_ (i.e. +`0`, `1`, `2`). + +#### Simple usage example: + +```c++ +// Train a Hoeffding tree on random numeric data; predict labels on test data: + +// All data and labels are uniform random; 10 dimensional data, 5 classes. +// Replace with a data::Load() call or similar for a real application. +arma::mat dataset(10, 1000, arma::fill::randu); // 1000 points. +arma::Row labels = + arma::randi>(1000, arma::distr_param(0, 4)); +arma::mat testDataset(10, 500, arma::fill::randu); // 500 test points. + +mlpack::HoeffdingTree tree; // Step 1: create model. +tree.Train(dataset, labels, 5); // Step 2a: train model (batch). +tree.Train(dataset.col(0), labels[0]); // Step 2b: train model (incremental). +arma::Row predictions; +tree.Classify(testDataset, predictions); // Step 3: classify points. + +// Print some information about the test predictions. +std::cout << arma::accu(predictions == 2) << " test points classified as class " + << "2." << std::endl; +``` +

More examples...

+ +#### Quick links: + + * [Constructors](#constructors): create `HoeffdingTree` objects. + * [`Train()`](#training): train model. + * [`Classify()`](#classification): classify with a trained model. + * [Other functionality](#other-functionality) for loading, saving, and + inspecting. + * [Examples](#simple-examples) of simple usage and links to detailed example + projects. + * [Template parameters](#advanced-functionality-template-parameters) for custom + behavior. + +#### See also: + + * [`DecisionTree`](#decision_tree) + * [Random forests](#random_forests) + * [mlpack classifiers](#mlpack_classifiers) + * [Incremental decision tree on Wikipedia](https://en.wikipedia.org/wiki/Incremental_decision_tree) + * [Mining High-Speed Data Streams (pdf)](https://dl.acm.org/doi/pdf/10.1145/347090.347107) + +### Constructors + + * `tree = HoeffdingTree()` + - Initialize tree without training. + - You will need to call [`Train()`](#training) later to train the tree before + calling [`Classify()`](#classification). + +--- + + * `tree = HoeffdingTree(numClasses)` + * `tree = HoeffdingTree(numClasses, successProbability=0.95, maxSamples=0, checkInterval=100, minSamples=100)` + - Initialize tree for incremental training on numerical-only data. + - The single-point [`Train()`](#training) function can be used to train + incrementally. + +--- + + * `tree = HoeffdingTree(datasetInfo, numClasses)` + * `tree = HoeffdingTree(datasetInfo, numClasses, successProbability=0.95, maxSamples=0, checkInterval=100, minSamples=100)` + - Initialize tree for incremental training on mixed categorical data. + - The single-point [`Train()`](#training) function can be used to train + incrementally. + +--- + + * `tree = HoeffdingTree(data, labels, numClasses) + * `tree = HoeffdingTree(data, labels, numClasses, batchTraining=true, successProbability=0.95, maxSamples=0, checkInterval=100, minSamples=100)` + - Train non-incrementally on numerical-only data. + TODO: should this actually be numerical-only? + +--- + + * `tree = HoeffdingTree(data, datasetInfo, labels, numClasses) + * `tree = HoeffdingTree(data, datasetInfo, labels, numClasses, batchTraining=true, successProbability=0.95, maxSamples=0, checkInterval=100, minSamples=100)` + - Train non-incrementally on mixed categorical data. + +--- + +#### Constructor Parameters: + + + +| **name** | **type** | **description** | **default** | +|----------|----------|-----------------|-------------| +| `data` | [`arma::mat`](../matrices.md) | [Column-major](../matrices.md) training matrix. | _(N/A)_ | +| `datasetInfo` | [`data::DatasetInfo`](../../tutorials/datasetmapper.md) | Dataset information, specifying type information for each dimension. | _(N/A)_ | +| `labels` | [`arma::Row`]('../matrices.md') | Training labels, between `0` and `numClasses - 1` (inclusive). Should have length `data.n_cols`. | _(N/A)_ | +| `numClasses` | `size_t` | Number of classes in the dataset. | _(N/A)_ | +| `batchTraining` | `bool` | If `true`, a batch training algorithm is used, instead of the usual incremental algorithm. This is generally more efficient for larger datasets. | `true` | +| `successProbability` | `double` | Probability of success required for Hoeffding bound before a node split can happen. | `0.95` | +| `maxSamples` | `size_t` | Maximum number of samples before a node split is forced. `0` means no limit. | `0` | +| `checkInterval` | `size_t` | Number of samples required before each split check. Higher values check less often, which is more efficient, but may not split a node as early as possible. | `100` | +| `minSamples` | `size_t` | Minimum number of samples for a node to see before a split is allowed. | `100` | + + * Setting `successProbability` higher than the default means that the Hoeffding + tree is less likely (and will take more samples) to split a node. This can + result in a smaller tree. + +***Note:*** different types can be used for `data` (e.g., `arma::fmat`, `arma::sp_mat`). + +### Training + +If training is not done as part of the constructor call, it can be done with one +of the following versions of the `Train()` member function: + + * `tree.Train(point, label)` + - Streaming (incremental) training: train on a single data point. + +--- + + + + * `tree.Train(data, labels, numClasses)` + * `tree.Train(data, labels, numClasses, batchTraining=true, successProbability=0.95, maxSamples=0, checkInterval=100, minSamples=100)` + - Train on numerical-only data. + +--- + + + + * `tree.Train(data, datasetInfo, labels, numClasses)` + * `tree.Train(data, datasetInfo, labels, numClasses, batchTraining=true, successProbability=0.95, maxSamples=0, checkInterval=100, minSamples=100)` + - Train on mixed categorical data. + +--- + +Types of each argument are the same as in the table for constructors +[above](#constructor-parameters). + +***Notes***: + + * Training is incremental. Successive calls to `Train()` will train the + Hoeffding tree further. To reset the tree, call + [`Reset()`](#other-functionality). + +### Classification + +Once a `DecisionTree` is trained, the `Classify()` member function can be used +to make class predictions for new data. + + * `size_t predictedClass = tree.Classify(point)` + - ***(Single-point)*** + - Classify a single point, returning the predicted class. + +--- + + + + * `tree.Classify(point, prediction, probabilitiesVec)` + - ***(Single-point)*** + - Classify a single point and compute class probabilities. + - The predicted class is stored in `prediction`. + - The probability of class `i` can be accessed with `probabilitiesVec[i]`. + +--- + + * `tree.Classify(data, predictions)` + - ***(Multi-point)*** + - Classify a set of points. + - The prediction for data point `i` can be accessed with `predictions[i]`. + +--- + + + + * `tree.Classify(data, predictions, probabilities)` + - ***(Multi-point)*** + - Classify a set of points and compute class probabilities for each point. + - The prediction for data point `i` can be accessed with `predictions[i]`. + - The probability of class `j` for data point `i` can be accessed with + `probabilities(j, i)`. + +--- + +#### Classification Parameters: + +| **usage** | **name** | **type** | **description** | +|-----------|----------|----------|-----------------| +| _single-point_ | `point` | [`arma::vec`](../matrices.md) | Single point for classification. | +| _single-point_ | `prediction` | `size_t&` | `size_t` to store class prediction into. | +| _single-point_ | `probabilitiesVec` | [`arma::vec&`](../matrices.md) | `arma::vec&` to store class probabilities into. Will be set to length `numClasses`. | +|||| +| _multi-point_ | `data` | [`arma::mat`](../matrices.md) | Set of [column-major](../matrices.md) points for classification. | +| _multi-point_ | `predictions` | [`arma::Row&`](../matrices.md) | Vector of `size_t`s to store class prediction into. Will be set to length `data.n_cols`. | +| _multi-point_ | `probabilities` | [`arma::mat&`](../matrices.md) | Matrix to store class probabilities into (number of rows will be equal to number of classes, number of columns will be equal to `data.n_cols`). | + +***Note:*** different types can be used for `data` and `point` (e.g. +`arma::fmat`, `arma::sp_mat`, `arma::sp_vec`, etc.). However, the element type +that is used should be the same type that was used for training. + +### Other Functionality + + + + * A `HoeffdingTree` can be serialized with [`data::Save()`](../formats.md) and + [`data::Load()`](../formats.md). + + * `tree.NumChildren()` will return a `size_t` indicating the number of children + in the node `tree`. + + * `tree.NumDescendants()` will return a `size_t` indicating the total number of + descendant nodes of the tree. + + * `tree.Child(i)` will return a `HoeffdingTree` object representing the `i`th + child of the node `tree`. + + * `tree.SplitDimension()` returns a `size_t` indicating which dimension the + node `tree` splits on. + + + + * `tree.NumClasses()` returns a `size_t` indicating the number of classes the + tree was trained on. + +For complete functionality, the [source +code](/src/mlpack/methods/hoeffding_trees/hoeffding_tree.hpp) can be consulted. +Each method is fully documented. + +### Simple Examples + +See also the [simple usage example](#simple-usage-example) for a trivial use of +`HoeffdingTree`. + +--- + +Train a Hoeffding tree incrementally on mixed categorical data: + +```c++ +// Load a categorical dataset. +arma::mat dataset; +mlpack::data::DatasetInfo info; +// See https://datasets.mlpack.org/covertype.train.arff. +mlpack::data::Load("covertype.train.arff", dataset, info, true); + +arma::Row labels; +// See https://datasets.mlpack.org/covertype.train.labels.csv. +mlpack::data::Load("covertype.train.labels.csv", labels, true); + +// Create the tree. +mlpack::HoeffdingTree tree(info, 7 /* classes */); + +// Train on each point in the given dataset. +for (size_t i = 0; i < dataset.n_cols; ++i) + tree.Train(dataset.col(i), labels[i]); + +// Load categorical test data. +arma::mat testDataset; +// See https://datasets.mlpack.org/covertype.test.arff. +mlpack::data::Load("covertype.test.arff", testDataset, info, true); + +// Predict class of first test point. +const size_t firstPrediction = tree.Classify(testDataset.col(0)); +std::cout << "Predicted class of first test point is " << firstPrediction << "." + << std::endl; + +// Predict class and probabilities of second test point. +size_t secondPrediction; +arma::vec secondProbabilities; +tree.Classify(testDataset.col(1), secondPrediction, secondProbabilities); +std::cout << "Class probabilities of second test point: " << + secondProbabilities.t(); +``` + +--- + +Train a Hoeffding tree on blocks of a dataset, print accuracy measures on a test +set during training, and save the model to disk. + +```c++ +// Load a categorical dataset. +arma::mat dataset; +mlpack::data::DatasetInfo info; +// See https://datasets.mlpack.org/covertype.train.arff. +mlpack::data::Load("covertype.train.arff", dataset, info, true); + +arma::Row labels; +// See https://datasets.mlpack.org/covertype.train.labels.csv. +mlpack::data::Load("covertype.train.labels.csv", labels, true); + +// Also load test data. + +// See https://datasets.mlpack.org/covertype.test.arff. +arma::mat testDataset; +mlpack::data::Load("covertype.test.arff", testDataset, info, true); + +// See https://datasets.mlpack.org/covertype.test.labels.arff. +arma::Row testLabels; +mlpack::data::Load("covertype.test.labels.csv", testLabels, true); + +// Create the tree with custom parameters. +mlpack::HoeffdingTree tree; +tree.SuccessProbability() = 0.99; +tree.CheckInterval() = 500; + +// Now iterate over 10k-point chunks in the dataset. +for (size_t start = 0; start < data.n_cols; start += 10000) +{ + size_t end = std::min(start + 9999, data.n_cols - 1); + + tree.Train(dataset.cols(start, end), info, labels.subvec(start, end)); + + // Compute accuracy on the test set. + arma::Row predictions; + tree.Predict(testDataset, predictions); + const double accuracy = 100.0 * arma::accu(predictions == testLabels) / + testLabels.n_elem; + + std::cout << "Accuracy after " << (end + 1) << " points: " << accuracy + << "\%." << std::endl; +} + +// Save the fully trained tree in `tree.bin` with name `tree`. +mlpack::data::Save("tree.bin", "tree", tree, true); +``` + +--- + +Load a tree and print some information about it. + +```c++ +mlpack::HoeffdingTree tree; +// This call assumes a tree called "tree" has already been saved to `tree.bin` +// with `data::Save()`. +mlpack::data::Load("tree.bin", "tree", tree, true); + +if (tree.NumChildren() > 0) +{ + std::cout << "The split dimension of the root node of the tree in `tree.bin` " + << "is dimension " << tree.SplitDimension() << "." << std::endl; +} +else +{ + std::cout << "The tree in `tree.bin` is a leaf (it has no children)." + << std::endl; +} +``` + +--- + +### Advanced Functionality: Template Parameters + +#### Using different element types. + +`HoeffdingTree`'s constructors, `Train()`, and `Classify()` functions support +any data type, so long as it supports the Armadillo matrix API. So, for +instance, learning can be done on single-precision floating-point data: + +```c++ +// 1000 random points in 10 dimensions. +arma::fmat dataset(10, 1000, arma::fill::randu); +// Random labels for each point, totaling 5 classes. +arma::Row labels = + arma::randi>(1000, arma::distr_param(0, 4)); + +// Train in the constructor. +mlpack::HoeffdingTree tree(dataset, labels, 5); + +// Create test data (500 points). +arma::fmat testDataset(10, 500, arma::fill::randu); +arma::Row predictions; +tree.Classify(testDataset, predictions); +// Now `predictions` holds predictions for the test dataset. + +// Print some information about the test predictions. +std::cout << arma::accu(predictions == 2) << " test points classified as class " + << "2." << std::endl; +``` + +--- + +#### Fully custom behavior. + +The `HoeffdingTree` class also supports several template parameters, which can +be used for custom behavior during learning. The full signature of the class is +as follows: + +```c++ +HoeffdingTree +``` + + * `FitnessFunction`: the measure of goodness to use when deciding on tree + splits + * `NumericSplitType`: the strategy used for finding splits on numeric data + dimensions + * `CategoricalSplitType`: the strategy used for finding splits on categorical + data dimensions + +Below, details are given for the requirements of each of these template types. + +--- + +#### `FitnessFunction` + + * Specifies the fitness function to use when learning a decision tree. + * The `GiniImpurity` _(default)_ and `HoeffdingInformationGain` classes are + available for drop-in usage. + * A custom class must implement two functions: + +```c++ +// You can use this as a starting point for implementation. +class CustomFitnessFunction +{ + // Return the range (difference between maximum and minimum gain values). + double Range(const size_t numClasses); + + // Compute the gain for the given split candidates represented in the matrix + // `counts`. `counts` is a matrix with `numChildren` columns and `numClasses` + // rows, containing the number of points for each class held by each child. + // + // Note that the gain returned should be the gain for *all* child nodes (e.g. + // all columns of `counts`). + double Evaluate(const arma::Mat& counts); +}; +``` + +--- + +#### `NumericSplitType` + + * Specifies the strategy to be used during training when splitting a numeric + feature. + * The `HoeffdingDoubleNumericSplit` _(default)_ class is available for drop-in + usage and discretizes the given numeric data into a default of 10 bins. This + expects `double` to be the type of the input data. + * The `HoeffdingFloatNumericSplit` class is available for drop-in usage and + operates similarly to `HoeffdingDoubleNumericSplit`, but expects `float` to + be the type of the input data. + * The `BinaryNumericSplit` class is available for drop-in usage and splits + numeric features in two in the way that maximizes gain. This split type is + more computationally expensive during training. + * A custom class must take a [`FitnessFunction`](#fitness-function) as a + template parameter, implement several functions, and have an internal + structure `SplitInfo` that is used at classification time: + +```c++ +// The job of this class is to track sufficient statistics of training data, +// returning gain information if a split were to happen according to this +// class's split strategy. +// +// For details, consult the HoeffdingNumericSplit and BinaryNumericSplit class +// implementations. +template +class CustomNumericSplit +{ + public: + // Create the split object with the given number of classes. + CustomNumericSplit(const size_t numClasses); + + // Create the split from another split object with the given number of + // classes. + CustomNumericSplit(const size_t numClasses, const CustomNumericSplit& other); + + // Train on the given value with the given label. + // Note that the type used here must match the element type of the training + // data (so, e.g., if you plan to use `arma::fmat`, use `float` instead of + // `double`). + void Train(double value, const size_t label); + + // Given the points seen so far, evaluate the fitness function, returning the + // gain if a split were to occur. If this `NumericSplitType` class could + // provide multiple possible splits, also return the second best fitness + // value. (If not, set secondBestFitness to 0.) + void EvaluateFitnessFunction(double& bestFitness, double& secondBestFitness); + + // Return the number of children that would be created if a split were to + // occur. (For example, if this class implements a binary split, this should + // return 2.) + size_t NumChildren() const; + + // Given that a split should happen, return the majority classes of the + // children and an initialized SplitInfo object. + // + // childMajorities should be set to have length equal to the number of + // children that this strategy splits into, and the i'th element should be the + // majority class label of the i'th child after splitting. + void Split(arma::Col& childMajorities, SplitInfo& splitInfo); + + // Return the current majority class of points seen so far. + size_t MajorityClass() const; + // Return the probability of the majority class given the points seen so far. + double MajorityProbability() const; + + // Serialize (load/save) the split object using cereal. + template + void serialize(Archive& ar, const uint32_t version); + + // The SplitInfo class should implement two functions. It is used at + // prediction time, after a split has occurred, and should contain the + // information necessary to classify a point. + // + // The SplitInfo class must implement two methods; one for classification and + // one for serialization. + class SplitInfo + { + public: + // Given that the point in the split dimension has the value `value`, return + // the index of the child that the traversal should go to. + template + size_t CalculateDirection(const eT& value) const; + + // Serialize the split (load/save) using cereal. + template + void serialize(Archive& ar, const uint32_t version); + }; +}; +``` + +--- + +#### `CategoricalSplitType` + + * Specifies the strategy to be used during training when splitting a + categorical feature. + * The `HoeffdingCategoricalSplit` _(default)_ is available for drop-in usage + and splits all categories into their own node. + * A custom class must take a [`FitnessFunction`](#fitness-function) as a + template parameter, implement several functions, and have an internal + structure `SplitInfo` that is used at classification time: + +```c++ +template +class CustomCategoricalSplit +{ + public: + // Create the split object with the given number of classes. The dimension + // that this object tracks has `numCategories` possible category values. + CustomCategoricalSplit(const size_t numCategories, const size_t numClasses); + + // Create the split object from another split object with the given number of + // classes. The dimension that this object tracks has `numCategories` + // possible category values. + CustomCategoricalSplit(const size_t numCategories, const size_t numClasses, + const CustomCategoricalSplit& other); + + // Train on the given value with the given label. + // Note that the type used here must match the element type of the training + // data (so, e.g., if you plan to use `arma::fmat`, use `float` instead of + // `double`). + void Train(double value, const size_t label); + + // Given the points seen so far, evaluate the fitness function, returning the + // gain if a split were to occur. If this `NumericSplitType` class could + // provide multiple possible splits, also return the second best fitness + // value. (If not, set secondBestFitness to 0.) + void EvaluateFitnessFunction(double& bestFitness, double& secondBestFitness); + + // Return the number of children that would be created if a split were to + // occur. (For example, if this class implements a binary split, this should + // return 2.) + size_t NumChildren() const; + + // Given that a split should happen, return the majority classes of the + // children and an initialized SplitInfo object. + // + // childMajorities should be set to have length equal to the number of + // children that this strategy splits into, and the i'th element should be the + // majority class label of the i'th child after splitting. + void Split(arma::Col& childMajorities, SplitInfo& splitInfo); + + // Return the current majority class of points seen so far. + size_t MajorityClass() const; + // Return the probability of the majority class given the points seen so far. + double MajorityProbability() const; + + // Serialize (load/save) the split object using cereal. + template + void serialize(Archive& ar, const uint32_t version); + + // The SplitInfo class should implement two functions. It is used at + // prediction time, after a split has occurred, and should contain the + // information necessary to classify a point. + // + // The SplitInfo class must implement two methods; one for classification and + // one for serialization. + class SplitInfo + { + public: + // Given that the point in the split dimension has the value `value`, return + // the index of the child that the traversal should go to. + template + size_t CalculateDirection(const eT& value) const; + + // Serialize the split (load/save) using cereal. + template + void serialize(Archive& ar, const uint32_t version); + }; +}; +``` From 081a04817964951e0f8fa86383450db7e9e7e3e2 Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Mon, 11 Dec 2023 08:58:15 -0500 Subject: [PATCH 44/91] Add a comment to the class description. --- doc/user/methods/hoeffding_tree.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/doc/user/methods/hoeffding_tree.md b/doc/user/methods/hoeffding_tree.md index 9b7e4b3afd..2d7afe0a19 100644 --- a/doc/user/methods/hoeffding_tree.md +++ b/doc/user/methods/hoeffding_tree.md @@ -540,6 +540,11 @@ class CustomNumericSplit structure `SplitInfo` that is used at classification time: ```c++ +// The job of this class is to track sufficient statistics of training data, +// returning gain information if a split were to happen according to this +// class's split strategy. +// +// For details, consult the HoeffdingCategoricalSplit class implementation. template class CustomCategoricalSplit { From 8fd1ae3b30ee9f562ad996c9328cf4640ce70769 Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Mon, 11 Dec 2023 11:53:41 -0500 Subject: [PATCH 45/91] Update code to match documentation. --- .../hoeffding_numeric_split.hpp | 4 + .../hoeffding_trees/hoeffding_tree.hpp | 308 ++++++++++--- .../hoeffding_trees/hoeffding_tree_impl.hpp | 428 ++++++++++++++++-- .../hoeffding_tree_model_impl.hpp | 12 +- 4 files changed, 641 insertions(+), 111 deletions(-) diff --git a/src/mlpack/methods/hoeffding_trees/hoeffding_numeric_split.hpp b/src/mlpack/methods/hoeffding_trees/hoeffding_numeric_split.hpp index 07532bee0f..17c6d36f2b 100644 --- a/src/mlpack/methods/hoeffding_trees/hoeffding_numeric_split.hpp +++ b/src/mlpack/methods/hoeffding_trees/hoeffding_numeric_split.hpp @@ -146,6 +146,10 @@ template using HoeffdingDoubleNumericSplit = HoeffdingNumericSplit; +template +using HoeffdingFloatNumericSplit = HoeffdingNumericSplit; + } // namespace mlpack // Include implementation. diff --git a/src/mlpack/methods/hoeffding_trees/hoeffding_tree.hpp b/src/mlpack/methods/hoeffding_trees/hoeffding_tree.hpp index 52e67f4555..5a33818507 100644 --- a/src/mlpack/methods/hoeffding_trees/hoeffding_tree.hpp +++ b/src/mlpack/methods/hoeffding_trees/hoeffding_tree.hpp @@ -72,6 +72,119 @@ class HoeffdingTree //! Allow access to the categorical split type. typedef CategoricalSplitType CategoricalSplit; + /** + * Construct a Hoeffding tree with no data and no information. Be sure to + * call Train() before trying to use the tree. + */ + HoeffdingTree(); + + /** + * Construct the Hoeffding tree with the given parameters for training on + * numerical data, but training on no data. The dimensionMappings parameter + * is only used if it is desired that this node does not create its own + * dimensionMappings object (for instance, if this is a child of another node + * in the tree). + * + * @param numClasses Number of classes in the dataset. + * @param successProbability Probability of success required in Hoeffding + * bound before a split can happen. + * @param maxSamples Maximum number of samples before a split is forced. + * @param checkInterval Number of samples required before each split check. + * @param minSamples If the node has seen this many points or fewer, no split + * will be allowed. + * @param categoricalSplitIn Optional instantiated categorical split object. + * @param numericSplitIn Optional instantiated numeric split object. + * @param dimensionMappings Mappings from dimension indices to positions in + * numeric and categorical split vectors. If left NULL, a new one will + * be created. + */ + HoeffdingTree(const size_t dimensionality, + const size_t numClasses, + const double successProbability = 0.95, + const size_t maxSamples = 0, + const size_t checkInterval = 100, + const size_t minSamples = 100, + const CategoricalSplitType& categoricalSplitIn + = CategoricalSplitType(0, 0), + const NumericSplitType& numericSplitIn = + NumericSplitType(0), + std::unordered_map>* + dimensionMappings = NULL); + + /** + * Construct the Hoeffding tree with the given parameters, but training on no + * data. The dimensionMappings parameter is only used if it is desired that + * this node does not create its own dimensionMappings object (for instance, + * if this is a child of another node in the tree). + * + * @param datasetInfo Information on the dataset (types of each feature). + * @param numClasses Number of classes in the dataset. + * @param successProbability Probability of success required in Hoeffding + * bound before a split can happen. + * @param maxSamples Maximum number of samples before a split is forced. + * @param checkInterval Number of samples required before each split check. + * @param minSamples If the node has seen this many points or fewer, no split + * will be allowed. + * @param categoricalSplitIn Optional instantiated categorical split object. + * @param numericSplitIn Optional instantiated numeric split object. + * @param dimensionMappings Mappings from dimension indices to positions in + * numeric and categorical split vectors. If left NULL, a new one will + * be created. + * @param copyDatasetInfo If true, then a copy of the datasetInfo will be + * made. + */ + HoeffdingTree(const data::DatasetInfo& datasetInfo, + const size_t numClasses, + const double successProbability = 0.95, + const size_t maxSamples = 0, + const size_t checkInterval = 100, + const size_t minSamples = 100, + const CategoricalSplitType& categoricalSplitIn + = CategoricalSplitType(0, 0), + const NumericSplitType& numericSplitIn = + NumericSplitType(0), + std::unordered_map>* + dimensionMappings = NULL, + const bool copyDatasetInfo = true); + + /** + * Construct the Hoeffding tree with the given parameters and given training + * data, where the training data contains only numerical features. The tree + * may be trained either in batch mode (which looks at all points before + * splitting, and propagates these points to the created children for further + * training), or in streaming mode, where each point is only considered once. + * (In general, batch mode will give better-performing trees, but will have + * higher memory and runtime costs for the same dataset.) + * + * @param data Dataset to train on. + * @param labels Labels of each point in the dataset. + * @param numClasses Number of classes in the dataset. + * @param batchTraining Whether or not to train in batch. + * @param successProbability Probability of success required in Hoeffding + * bounds before a split can happen. + * @param maxSamples Maximum number of samples before a split is forced (0 + * never forces a split); ignored in batch training mode. + * @param checkInterval Number of samples required before each split; ignored + * in batch training mode. + * @param minSamples If the node has seen this many points or fewer, no split + * will be allowed. + * @param categoricalSplitIn Optional instantiated categorical split object. + * @param numericSplitIn Optional instantiated numeric split object. + */ + template + HoeffdingTree(const MatType& data, + const arma::Row& labels, + const size_t numClasses, + const bool batchTraining = true, + const double successProbability = 0.95, + const size_t maxSamples = 0, + const size_t checkInterval = 100, + const size_t minSamples = 100, + const CategoricalSplitType& categoricalSplitIn + = CategoricalSplitType(0, 0), + const NumericSplitType& numericSplitIn = + NumericSplitType(0)); + /** * Construct the Hoeffding tree with the given parameters and given training * data. The tree may be trained either in batch mode (which looks at all @@ -111,48 +224,6 @@ class HoeffdingTree const NumericSplitType& numericSplitIn = NumericSplitType(0)); - /** - * Construct the Hoeffding tree with the given parameters, but training on no - * data. The dimensionMappings parameter is only used if it is desired that - * this node does not create its own dimensionMappings object (for instance, - * if this is a child of another node in the tree). - * - * @param numClasses Number of classes in the dataset. - * @param datasetInfo Information on the dataset (types of each feature). - * @param successProbability Probability of success required in Hoeffding - * bound before a split can happen. - * @param maxSamples Maximum number of samples before a split is forced. - * @param checkInterval Number of samples required before each split check. - * @param minSamples If the node has seen this many points or fewer, no split - * will be allowed. - * @param dimensionMappings Mappings from dimension indices to positions in - * numeric and categorical split vectors. If left NULL, a new one will - * be created. - * @param copyDatasetInfo If true, then a copy of the datasetInfo will be - * made. - * @param categoricalSplitIn Optional instantiated categorical split object. - * @param numericSplitIn Optional instantiated numeric split object. - */ - HoeffdingTree(const data::DatasetInfo& datasetInfo, - const size_t numClasses, - const double successProbability = 0.95, - const size_t maxSamples = 0, - const size_t checkInterval = 100, - const size_t minSamples = 100, - const CategoricalSplitType& categoricalSplitIn - = CategoricalSplitType(0, 0), - const NumericSplitType& numericSplitIn = - NumericSplitType(0), - std::unordered_map>* - dimensionMappings = NULL, - const bool copyDatasetInfo = true); - - /** - * Construct a Hoeffding tree with no data and no information. Be sure to - * call Train() before trying to use the tree. - */ - HoeffdingTree(); - /** * Copy another tree (warning: this will duplicate the tree entirely, and may * use a lot of memory. Make sure it's what you want before you do it). @@ -194,44 +265,136 @@ class HoeffdingTree * * Note that the tree will be automatically reset if the dimensionality of * `data` does not match the dimensionality that the tree was currently - * trained with. The tree will also be reset if `numClasses` is passed. + * trained with. The tree will also be reset if `numClasses` is passed and + * differs from the existing setting. * * @param data Data points to train on. * @param labels Labels of data points. - * @param batchTraining If true, perform training in batch. - * @param resetTree If true, reset the tree to an empty tree before training. * @param numClasses The number of classes in `labels`. Passing this will * reset the tree. If not given and `resetTree` is `true`, then the * number of classes will be computed from `labels`. + * @param batchTraining If true, perform training in batch. + * @param successProbability Probability of success required in Hoeffding + * bounds before a split can happen. + * @param maxSamples Maximum number of samples before a split is forced (0 + * never forces a split); ignored in batch training mode. + * @param checkInterval Number of samples required before each split; ignored + * in batch training mode. + * @param minSamples If the node has seen this many points or fewer, no split + * will be allowed. */ + // Many overloads needed here until we have std::optional. template void Train(const MatType& data, const arma::Row& labels, - const bool batchTraining = true, - const bool resetTree = false, - const size_t numClasses = 0); + const size_t numClasses = 0, + const bool batchTraining = true); + + template + void Train(const MatType& data, + const arma::Row& labels, + const size_t numClasses, + const bool batchTraining, + const double successProbability); + + template + void Train(const MatType& data, + const arma::Row& labels, + const size_t numClasses, + const bool batchTraining, + const double successProbability, + const size_t maxSamples); + + template + void Train(const MatType& data, + const arma::Row& labels, + const size_t numClasses, + const bool batchTraining, + const double successProbability, + const size_t maxSamples, + const size_t checkInterval); + + template + void Train(const MatType& data, + const arma::Row& labels, + const size_t numClasses, + const bool batchTraining, + const double successProbability, + const size_t maxSamples, + const size_t checkInterval, + const size_t minSamples); /** * Train on a set of points, either in streaming mode or in batch mode, with - * the given labels and the given `DatasetInfo`. This will reset the tree. - * This only needs to be called when the `DatasetInfo` has changed---if you - * are training incrementally but have already passed the DatasetInfo once, - * use the overload of `Train()` that does not take a `DatasetInfo` and make - * sure `resetTree` is set to `false`. + * the given labels. If `resetTree` is set to `true`, then reset the state of + * the tree to an empty tree before training. + * + * Note that the tree will be automatically reset if the dimensionality of + * `data` does not match the dimensionality that the tree was currently + * trained with. The tree will also be reset if `numClasses` is passed and + * differs from the existing setting. * * @param data Data points to train on. - * @param info DatasetInfo object with information about each dimension. + * @param datasetInfo Information on the dataset (types of each feature). * @param labels Labels of data points. + * @param numClasses The number of classes in `labels`. Passing this will + * reset the tree. If not given and `resetTree` is `true`, then the + * number of classes will be computed from `labels`. * @param batchTraining If true, perform training in batch. - * @param numClasses Number of classes in `labels`. If not specified, it is - * computed from `labels`. + * @param successProbability Probability of success required in Hoeffding + * bounds before a split can happen. + * @param maxSamples Maximum number of samples before a split is forced (0 + * never forces a split); ignored in batch training mode. + * @param checkInterval Number of samples required before each split; ignored + * in batch training mode. + * @param minSamples If the node has seen this many points or fewer, no split + * will be allowed. */ + // Many overloads needed here until we have std::optional. template void Train(const MatType& data, const data::DatasetInfo& info, const arma::Row& labels, - const bool batchTraining = true, - const size_t numClasses = 0); + const size_t numClasses = 0, + const bool batchTraining = true); + + template + void Train(const MatType& data, + const data::DatasetInfo& info, + const arma::Row& labels, + const size_t numClasses, + const bool batchTraining, + const double successProbability); + + template + void Train(const MatType& data, + const data::DatasetInfo& info, + const arma::Row& labels, + const size_t numClasses, + const bool batchTraining, + const double successProbability, + const size_t maxSamples); + + template + void Train(const MatType& data, + const data::DatasetInfo& info, + const arma::Row& labels, + const size_t numClasses, + const bool batchTraining, + const double successProbability, + const size_t maxSamples, + const size_t checkInterval); + + template + void Train(const MatType& data, + const data::DatasetInfo& info, + const arma::Row& labels, + const size_t numClasses, + const bool batchTraining, + const double successProbability, + const size_t maxSamples, + const size_t checkInterval, + const size_t minSamples); /** * Train on a single point in streaming mode, with the given label. The tree @@ -291,6 +454,12 @@ class HoeffdingTree //! Modify the number of samples before a split check is performed. void CheckInterval(const size_t checkInterval); + //! Get the number of points seen so far. + size_t NumSamples() const { return numSamples; } + + //! Get the number of classes the tree is trained on. + size_t NumClasses() const { return numClasses; } + /** * Given a point and that this node is not a leaf, calculate the index of the * child node this point would go towards. This method is primarily used by @@ -301,6 +470,9 @@ class HoeffdingTree template size_t CalculateDirection(const VecType& point) const; + //! Get the size of the Hoeffding Tree. + size_t NumDescendants() const; + /** * Classify the given point, using this node and the entire (sub)tree beneath * it. The predicted label is returned. @@ -311,9 +483,6 @@ class HoeffdingTree template size_t Classify(const VecType& point) const; - //! Get the size of the Hoeffding Tree. - size_t NumDescendants() const; - /** * Classify the given point and also return an estimate of the probability * that the prediction is correct. (This estimate is simply the probability @@ -360,6 +529,23 @@ class HoeffdingTree */ void CreateChildren(); + /** + * Reset the tree, keeping the number of classes and dimension information + * intact. + */ + void Reset(); + + /** + * Reset the tree, setting a new number of classes and dimensionality. This + * assumes all dimensions are numeric. + */ + void Reset(const size_t dimensionality, const size_t numClasses); + + /** + * Reset the tree, setting a new number of classes and a new datasetInfo. + */ + void Reset(const data::DatasetInfo& datasetInfo, const size_t numClasses); + //! Serialize the split. template void serialize(Archive& ar, const uint32_t /* version */); diff --git a/src/mlpack/methods/hoeffding_trees/hoeffding_tree_impl.hpp b/src/mlpack/methods/hoeffding_trees/hoeffding_tree_impl.hpp index 5540fc38a7..16f94bbccd 100644 --- a/src/mlpack/methods/hoeffding_trees/hoeffding_tree_impl.hpp +++ b/src/mlpack/methods/hoeffding_trees/hoeffding_tree_impl.hpp @@ -14,38 +14,64 @@ // In case it hasn't been included yet. #include "hoeffding_tree.hpp" -#include namespace mlpack { template class NumericSplitType, template class CategoricalSplitType> -template HoeffdingTree< FitnessFunction, NumericSplitType, CategoricalSplitType ->::HoeffdingTree(const MatType& data, - const data::DatasetInfo& datasetInfoIn, - const arma::Row& labels, +>::HoeffdingTree() : + dimensionMappings( + new std::unordered_map>()), + ownsMappings(true), + numSamples(0), + numClasses(0), + maxSamples(size_t(-1)), + checkInterval(100), + minSamples(100), + datasetInfo(new data::DatasetInfo()), + ownsInfo(true), + successProbability(0.95), + splitDimension(size_t(-1)), + majorityClass(0), + majorityProbability(0.0), + categoricalSplit(0), + numericSplit() +{ + // Nothing to do. +} + +template class NumericSplitType, + template class CategoricalSplitType> +HoeffdingTree< + FitnessFunction, + NumericSplitType, + CategoricalSplitType +>::HoeffdingTree(const size_t dimensionality, const size_t numClasses, - const bool batchTraining, const double successProbability, const size_t maxSamples, const size_t checkInterval, const size_t minSamples, const CategoricalSplitType& categoricalSplitIn, - const NumericSplitType& numericSplitIn) : - dimensionMappings(NULL), - ownsMappings(false), + const NumericSplitType& numericSplitIn, + std::unordered_map>* + dimensionMappingsIn) : + dimensionMappings((dimensionMappingsIn != NULL) ? dimensionMappingsIn : + new std::unordered_map>()), + ownsMappings(dimensionMappingsIn == NULL), numSamples(0), numClasses(numClasses), maxSamples((maxSamples == 0) ? size_t(-1) : maxSamples), checkInterval(checkInterval), minSamples(minSamples), - datasetInfo(new data::DatasetInfo(datasetInfoIn)), + datasetInfo(new data::DatasetInfo(dimensionality)), ownsInfo(true), successProbability(successProbability), splitDimension(size_t(-1)), @@ -54,11 +80,20 @@ HoeffdingTree< categoricalSplit(0), numericSplit() { - // Reset the tree. - ResetTree(categoricalSplitIn, numericSplitIn); - - // Now train. - Train(data, labels, batchTraining); + // Do we need to generate the mappings too? + if (ownsMappings) + { + ResetTree(categoricalSplitIn, numericSplitIn); + } + else + { + // All dimensions are numeric. + for (size_t i = 0; i < datasetInfo->Dimensionality(); ++i) + { + numericSplits.push_back(NumericSplitType(numClasses, + numericSplitIn)); + } + } } template class NumericSplitType, template class CategoricalSplitType> +template HoeffdingTree< FitnessFunction, NumericSplitType, CategoricalSplitType ->::HoeffdingTree() : - dimensionMappings( - new std::unordered_map>()), - ownsMappings(true), +>::HoeffdingTree(const MatType& data, + const arma::Row& labels, + const size_t numClasses, + const bool batchTraining, + const double successProbability, + const size_t maxSamples, + const size_t checkInterval, + const size_t minSamples, + const CategoricalSplitType& + categoricalSplitIn, + const NumericSplitType& numericSplitIn) : + dimensionMappings(NULL), + ownsMappings(false), numSamples(0), - numClasses(0), - maxSamples(size_t(-1)), - checkInterval(100), - minSamples(100), - datasetInfo(new data::DatasetInfo()), + numClasses(numClasses), + maxSamples((maxSamples == 0) ? size_t(-1) : maxSamples), + checkInterval(checkInterval), + minSamples(minSamples), + datasetInfo(new data::DatasetInfo(data.n_rows)), ownsInfo(true), - successProbability(0.95), + successProbability(successProbability), splitDimension(size_t(-1)), majorityClass(0), majorityProbability(0.0), categoricalSplit(0), numericSplit() { - // Nothing to do. + // Reset the tree. + ResetTree(categoricalSplitIn, numericSplitIn); + + // Now train. + Train(data, labels, numClasses, batchTraining); +} + +template class NumericSplitType, + template class CategoricalSplitType> +template +HoeffdingTree< + FitnessFunction, + NumericSplitType, + CategoricalSplitType +>::HoeffdingTree(const MatType& data, + const data::DatasetInfo& datasetInfoIn, + const arma::Row& labels, + const size_t numClasses, + const bool batchTraining, + const double successProbability, + const size_t maxSamples, + const size_t checkInterval, + const size_t minSamples, + const CategoricalSplitType& + categoricalSplitIn, + const NumericSplitType& numericSplitIn) : + dimensionMappings(NULL), + ownsMappings(false), + numSamples(0), + numClasses(numClasses), + maxSamples((maxSamples == 0) ? size_t(-1) : maxSamples), + checkInterval(checkInterval), + minSamples(minSamples), + datasetInfo(new data::DatasetInfo(datasetInfoIn)), + ownsInfo(true), + successProbability(successProbability), + splitDimension(size_t(-1)), + majorityClass(0), + majorityProbability(0.0), + categoricalSplit(0), + numericSplit() +{ + // Reset the tree. + ResetTree(categoricalSplitIn, numericSplitIn); + + // Now train. + Train(data, labels, numClasses, batchTraining); } // Copy constructor. @@ -336,7 +428,6 @@ HoeffdingTree:: delete children[i]; } -//! Train on a set of points. template class NumericSplitType, template class CategoricalSplitType> @@ -347,14 +438,96 @@ void HoeffdingTree< CategoricalSplitType >::Train(const MatType& data, const arma::Row& labels, - const bool batchTraining, - const bool resetTree, - const size_t numClassesIn) + const size_t numClasses, + const bool batchTraining) { + Train(data, labels, numClasses, batchTraining, this->successProbability, + this->maxSamples, this->checkInterval, this->minSamples); +} + +template class NumericSplitType, + template class CategoricalSplitType> +template +void HoeffdingTree< + FitnessFunction, + NumericSplitType, + CategoricalSplitType +>::Train(const MatType& data, + const arma::Row& labels, + const size_t numClasses, + const bool batchTraining, + const double successProbability) +{ + Train(data, labels, numClasses, batchTraining, successProbability, + this->maxSamples, this->checkInterval, this->minSamples); +} + +template class NumericSplitType, + template class CategoricalSplitType> +template +void HoeffdingTree< + FitnessFunction, + NumericSplitType, + CategoricalSplitType +>::Train(const MatType& data, + const arma::Row& labels, + const size_t numClasses, + const bool batchTraining, + const double successProbability, + const size_t maxSamples) +{ + Train(data, labels, numClasses, batchTraining, successProbability, maxSamples, + this->checkInterval, this->minSamples); +} + +template class NumericSplitType, + template class CategoricalSplitType> +template +void HoeffdingTree< + FitnessFunction, + NumericSplitType, + CategoricalSplitType +>::Train(const MatType& data, + const arma::Row& labels, + const size_t numClasses, + const bool batchTraining, + const double successProbability, + const size_t maxSamples, + const size_t checkInterval) +{ + Train(data, labels, numClasses, batchTraining, successProbability, maxSamples, + checkInterval, this->minSamples); +} + +template class NumericSplitType, + template class CategoricalSplitType> +template +void HoeffdingTree< + FitnessFunction, + NumericSplitType, + CategoricalSplitType +>::Train(const MatType& data, + const arma::Row& labels, + const size_t numClasses, + const bool batchTraining, + const double successProbability, + const size_t maxSamples, + const size_t checkInterval, + const size_t minSamples) +{ + this->successProbability = successProbability; + this->maxSamples = maxSamples; + this->checkInterval = checkInterval; + this->minSamples = minSamples; + // We need to reset the tree either if the user asked for it, or if they // passed data whose dimensionality is different than our datasetInfo object. - if (resetTree || data.n_rows != datasetInfo->Dimensionality() || - numClassesIn != 0) + if (data.n_rows != datasetInfo->Dimensionality() || + (numClasses != 0 && numClasses != this->numClasses)) { // Create a new datasetInfo, which assumes that all features are numeric. if (ownsInfo) @@ -363,7 +536,14 @@ void HoeffdingTree< ownsInfo = true; // Set the number of classes correctly. - numClasses = (numClassesIn != 0) ? numClassesIn : arma::max(labels) + 1; + if (numClasses != 0) + this->numClasses = numClasses; + + if (this->numClasses == 0) + { + throw std::invalid_argument("HoeffdingTree::Train(): must specify number " + "of classes!"); + } ResetTree(); } @@ -371,7 +551,6 @@ void HoeffdingTree< TrainInternal(data, labels, batchTraining); } -//! Train on a set of points. template class NumericSplitType, template class CategoricalSplitType> @@ -383,21 +562,122 @@ void HoeffdingTree< >::Train(const MatType& data, const data::DatasetInfo& info, const arma::Row& labels, - const bool batchTraining, - const size_t numClassesIn) + const size_t numClasses, + const bool batchTraining) { - // Take over new DatasetInfo. - if (ownsInfo) - delete datasetInfo; - datasetInfo = &info; - ownsInfo = false; + Train(data, info, labels, numClasses, batchTraining, this->successProbability, + this->maxSamples, this->checkInterval, this->minSamples); +} - // Set the number of classes correctly. - numClasses = (numClassesIn != 0) ? numClassesIn : arma::max(labels) + 1; +template class NumericSplitType, + template class CategoricalSplitType> +template +void HoeffdingTree< + FitnessFunction, + NumericSplitType, + CategoricalSplitType +>::Train(const MatType& data, + const data::DatasetInfo& info, + const arma::Row& labels, + const size_t numClasses, + const bool batchTraining, + const double successProbability) +{ + Train(data, info, labels, numClasses, batchTraining, successProbability, + this->maxSamples, this->checkInterval, this->minSamples); +} - ResetTree(); +template class NumericSplitType, + template class CategoricalSplitType> +template +void HoeffdingTree< + FitnessFunction, + NumericSplitType, + CategoricalSplitType +>::Train(const MatType& data, + const data::DatasetInfo& info, + const arma::Row& labels, + const size_t numClasses, + const bool batchTraining, + const double successProbability, + const size_t maxSamples) +{ + Train(data, info, labels, numClasses, batchTraining, successProbability, + maxSamples, this->checkInterval, this->minSamples); +} + +template class NumericSplitType, + template class CategoricalSplitType> +template +void HoeffdingTree< + FitnessFunction, + NumericSplitType, + CategoricalSplitType +>::Train(const MatType& data, + const data::DatasetInfo& info, + const arma::Row& labels, + const size_t numClasses, + const bool batchTraining, + const double successProbability, + const size_t maxSamples, + const size_t checkInterval) +{ + Train(data, info, labels, numClasses, batchTraining, successProbability, + maxSamples, checkInterval, this->minSamples); +} + +template class NumericSplitType, + template class CategoricalSplitType> +template +void HoeffdingTree< + FitnessFunction, + NumericSplitType, + CategoricalSplitType +>::Train(const MatType& data, + const data::DatasetInfo& info, + const arma::Row& labels, + const size_t numClasses, + const bool batchTraining, + const double successProbability, + const size_t maxSamples, + const size_t checkInterval, + const size_t minSamples) +{ + this->successProbability = successProbability; + this->maxSamples = maxSamples; + this->checkInterval = checkInterval; + this->minSamples = minSamples; + + // We need to reset the tree either if the user asked for it, or if they + // passed data whose dimensionality is different than our datasetInfo object. + if (data.n_rows != datasetInfo->Dimensionality() || + (numClasses != 0 && numClasses != this->numClasses)) + { + // Set the number of classes correctly. + if (numClasses != 0) + this->numClasses = numClasses; + + if (this->numClasses == 0) + { + throw std::invalid_argument("HoeffdingTree::Train(): must specify number " + "of classes!"); + } + + Reset(info, this->numClasses); + } + else if (datasetInfo != &info) + { + // Take over new DatasetInfo. + if (ownsInfo) + delete datasetInfo; + datasetInfo = &info; + ownsInfo = false; + } - // Now train. TrainInternal(data, labels, batchTraining); } @@ -812,6 +1092,62 @@ void HoeffdingTree< categoricalSplits.clear(); } +template< + typename FitnessFunction, + template class NumericSplitType, + template class CategoricalSplitType +> +void HoeffdingTree< + FitnessFunction, + NumericSplitType, + CategoricalSplitType +>::Reset() +{ + ResetTree(); +} + +template< + typename FitnessFunction, + template class NumericSplitType, + template class CategoricalSplitType +> +void HoeffdingTree< + FitnessFunction, + NumericSplitType, + CategoricalSplitType +>::Reset(const size_t dimensionality, const size_t numClasses) +{ + if (ownsInfo) + delete datasetInfo; + datasetInfo = new data::DatasetInfo(dimensionality); // All features numeric. + ownsInfo = true; + + this->numClasses = numClasses; + + ResetTree(); +} + +template< + typename FitnessFunction, + template class NumericSplitType, + template class CategoricalSplitType +> +void HoeffdingTree< + FitnessFunction, + NumericSplitType, + CategoricalSplitType +>::Reset(const data::DatasetInfo& info, const size_t numClasses) +{ + if (ownsInfo) + delete datasetInfo; + datasetInfo = &info; + ownsInfo = false; + + this->numClasses = numClasses; + + ResetTree(); +} + template< typename FitnessFunction, template class NumericSplitType, @@ -1000,7 +1336,7 @@ void HoeffdingTree< // unfortunately, instead, we'll just extract the non-contiguous // submatrix. MatType childData = data.cols(indices[i].subvec(0, counts[i] - 1)); - children[i]->Train(childData, childLabels, true); + children[i]->Train(childData, childLabels, numClasses, true); } } } diff --git a/src/mlpack/methods/hoeffding_trees/hoeffding_tree_model_impl.hpp b/src/mlpack/methods/hoeffding_trees/hoeffding_tree_model_impl.hpp index 55b02bf409..2394b963a1 100644 --- a/src/mlpack/methods/hoeffding_trees/hoeffding_tree_model_impl.hpp +++ b/src/mlpack/methods/hoeffding_trees/hoeffding_tree_model_impl.hpp @@ -198,19 +198,23 @@ inline void HoeffdingTreeModel::Train(const arma::mat& dataset, switch (type) { case GINI_HOEFFDING: - giniHoeffdingTree->Train(dataset, labels, batchTraining); + giniHoeffdingTree->Train(dataset, labels, giniHoeffdingTree->NumClasses(), + batchTraining); break; case GINI_BINARY: - giniBinaryTree->Train(dataset, labels, batchTraining); + giniBinaryTree->Train(dataset, labels, giniBinaryTree->NumClasses(), + batchTraining); break; case INFO_HOEFFDING: - infoHoeffdingTree->Train(dataset, labels, batchTraining); + infoHoeffdingTree->Train(dataset, labels, infoHoeffdingTree->NumClasses(), + batchTraining); break; case INFO_BINARY: - infoBinaryTree->Train(dataset, labels, batchTraining); + infoBinaryTree->Train(dataset, labels, infoBinaryTree->NumClasses(), + batchTraining); break; } } From 2a5de809c9d5974da582962cfbf99ef872f895a5 Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Mon, 11 Dec 2023 11:53:54 -0500 Subject: [PATCH 46/91] Some additional documentation updates. --- doc/user/methods/hoeffding_tree.md | 149 +++++++++++++++++++++++------ 1 file changed, 118 insertions(+), 31 deletions(-) diff --git a/doc/user/methods/hoeffding_tree.md b/doc/user/methods/hoeffding_tree.md index 2d7afe0a19..452f0066e1 100644 --- a/doc/user/methods/hoeffding_tree.md +++ b/doc/user/methods/hoeffding_tree.md @@ -6,8 +6,8 @@ Gini impurity to choose which feature to split on. The class offers several template parameters and several runtime options that can be used to control the behavior of the tree. -Hoeffding trees are useful for classifying points with _discrete labels_ (i.e. -`0`, `1`, `2`). +Hoeffding trees (also known as "Very Fast Decision Trees" or VFDTs) are useful +for classifying points with _discrete labels_ (i.e. `0`, `1`, `2`). #### Simple usage example: @@ -57,13 +57,13 @@ std::cout << arma::accu(predictions == 2) << " test points classified as class " * `tree = HoeffdingTree()` - Initialize tree without training. - - You will need to call [`Train()`](#training) later to train the tree before - calling [`Classify()`](#classification). + - You will need to call the batch version of [`Train()`](#training) later to + train the tree before calling [`Classify()`](#classification). --- - * `tree = HoeffdingTree(numClasses)` - * `tree = HoeffdingTree(numClasses, successProbability=0.95, maxSamples=0, checkInterval=100, minSamples=100)` + * `tree = HoeffdingTree(dimensionality, numClasses)` + * `tree = HoeffdingTree(dimensionality, numClasses, successProbability=0.95, maxSamples=0, checkInterval=100, minSamples=100)` - Initialize tree for incremental training on numerical-only data. - The single-point [`Train()`](#training) function can be used to train incrementally. @@ -105,6 +105,7 @@ std::cout << arma::accu(predictions == 2) << " test points classified as class " | `data` | [`arma::mat`](../matrices.md) | [Column-major](../matrices.md) training matrix. | _(N/A)_ | | `datasetInfo` | [`data::DatasetInfo`](../../tutorials/datasetmapper.md) | Dataset information, specifying type information for each dimension. | _(N/A)_ | | `labels` | [`arma::Row`]('../matrices.md') | Training labels, between `0` and `numClasses - 1` (inclusive). Should have length `data.n_cols`. | _(N/A)_ | +| `dimensionality` | `size_t` | When using on numeric-only data, this specifies the number of dimensions in the data. | _(N/A)_ | | `numClasses` | `size_t` | Number of classes in the dataset. | _(N/A)_ | | `batchTraining` | `bool` | If `true`, a batch training algorithm is used, instead of the usual incremental algorithm. This is generally more efficient for larger datasets. | `true` | | `successProbability` | `double` | Probability of success required for Hoeffding bound before a node split can happen. | `0.95` | @@ -125,22 +126,31 @@ of the following versions of the `Train()` member function: * `tree.Train(point, label)` - Streaming (incremental) training: train on a single data point. + - The number of classes and dataset information must have already been + specified by a previous constructor, `Train()`, or `Reset()` call. --- - - + * `tree.Train(data, labels)` * `tree.Train(data, labels, numClasses)` * `tree.Train(data, labels, numClasses, batchTraining=true, successProbability=0.95, maxSamples=0, checkInterval=100, minSamples=100)` - - Train on numerical-only data. + - Train on the given data. + - If the data is mixed categorical, then `datasetInfo` should have already + been passed via a previous constructor, `Train()`, or `Reset()` call. + - `numClasses` does not need to be specified if it has been specified in an + earlier constructor or `Train()` call. --- - - + * `tree.Train(data, datasetInfo, labels)` * `tree.Train(data, datasetInfo, labels, numClasses)` * `tree.Train(data, datasetInfo, labels, numClasses, batchTraining=true, successProbability=0.95, maxSamples=0, checkInterval=100, minSamples=100)` - Train on mixed categorical data. + - The previous overload (without `datasetInfo`) can be used instead if + `datasetInfo` has already been passed in a previous constructor, `Train()`, + or `Reset()` call, and has not changed. + - `numClasses` does not need to be specified if it has been specified in an + earlier constructor or `Train()` call. --- @@ -164,13 +174,11 @@ to make class predictions for new data. --- - - - * `tree.Classify(point, prediction, probabilitiesVec)` + * `tree.Classify(point, prediction, probability)` - ***(Single-point)*** - Classify a single point and compute class probabilities. - The predicted class is stored in `prediction`. - - The probability of class `i` can be accessed with `probabilitiesVec[i]`. + - The probability of class `i` is stored in `probability`. --- @@ -181,14 +189,12 @@ to make class predictions for new data. --- - - * `tree.Classify(data, predictions, probabilities)` - ***(Multi-point)*** - Classify a set of points and compute class probabilities for each point. - The prediction for data point `i` can be accessed with `predictions[i]`. - - The probability of class `j` for data point `i` can be accessed with - `probabilities(j, i)`. + - The probability of class `predictions[i]` for data point `i` can be + accessed with `probabilities[i]`. --- @@ -198,11 +204,11 @@ to make class predictions for new data. |-----------|----------|----------|-----------------| | _single-point_ | `point` | [`arma::vec`](../matrices.md) | Single point for classification. | | _single-point_ | `prediction` | `size_t&` | `size_t` to store class prediction into. | -| _single-point_ | `probabilitiesVec` | [`arma::vec&`](../matrices.md) | `arma::vec&` to store class probabilities into. Will be set to length `numClasses`. | +| _single-point_ | `probability` | `double&` | `double` to store predicted class probability into. | |||| | _multi-point_ | `data` | [`arma::mat`](../matrices.md) | Set of [column-major](../matrices.md) points for classification. | | _multi-point_ | `predictions` | [`arma::Row&`](../matrices.md) | Vector of `size_t`s to store class prediction into. Will be set to length `data.n_cols`. | -| _multi-point_ | `probabilities` | [`arma::mat&`](../matrices.md) | Matrix to store class probabilities into (number of rows will be equal to number of classes, number of columns will be equal to `data.n_cols`). | +| _multi-point_ | `probabilities` | [`arma::rowvec&`](../matrices.md) | Vector to store probability of predicted class in for each point. Will be set to length `data.n_cols`. | ***Note:*** different types can be used for `data` and `point` (e.g. `arma::fmat`, `arma::sp_mat`, `arma::sp_vec`, etc.). However, the element type @@ -227,11 +233,23 @@ that is used should be the same type that was used for training. * `tree.SplitDimension()` returns a `size_t` indicating which dimension the node `tree` splits on. - + * `tree.NumSamples()` returns a `size_t` indicating the number of points seen + so far by `tree`, if `tree` has not yet split. If `tree` has split (i.e. if + `tree.NumChildren() > 0`), then what is returned is the number of points seen + up until the split occurred. * `tree.NumClasses()` returns a `size_t` indicating the number of classes the tree was trained on. + * `tree.Reset()` will reset the tree to an empty tree, and: + - `tree.Reset()` will leave the number of classes and dataset information + (e.g. `datasetInfo`) intact. + - `tree.Reset(dimensionality, numClasses)` will set the number of classes to + `numClasses` and set the dimensionality of the data to `dimensionality`, + assuming all dimensions are numeric. + - `tree.Reset(datasetInfo, numClasses)` will set the number of classes to + `numClasses` and set the dataset information to `datasetInfo`. + For complete functionality, the [source code](/src/mlpack/methods/hoeffding_trees/hoeffding_tree.hpp) can be consulted. Each method is fully documented. @@ -357,6 +375,66 @@ else --- +Train a tree, reset a tree, and train again. + +```c++ +// See the following files: +// - https://datasets.mlpack.org/covertype.train.arff. +// - https://datasets.mlpack.org/covertype.train.labels.csv. +// - https://datasets.mlpack.org/covertype.test.arff. +// - https://datasets.mlpack.org/covertype.test.labels.arff. + +arma::mat dataset, testDataset; +arma::Row labels, testLabels; +mlpack::data::DatasetInfo info; + +mlpack::data::Load("covertype.train.arff", dataset, info, true); +mlpack::data::Load("covertype.train.labels.csv", labels, true); +mlpack::data::Load("covertype.test.arff", testDataset, info, true); +mlpack::data::Load("covertype.test.labels.csv", testLabels, true); + +// Create a tree, and train on the training data. +mlpack::HoeffdingTree tree(info, 7 /* number of classes */, 0.98); +tree.MinSamples() = 500; +tree.CheckInterval() = 500; + +tree.Train(dataset, labels); + +// Print accuracy on the training and test set. +arma::Row predictions, testPredictions; +tree.Classify(dataset, predictions); +tree.Classify(testDataset, testPredictions); + +double trainAcc = (100.0 * arma::accu(predictions == labels)) / labels.n_elem; +double testAcc = (100.0 * arma::accu(testPredictions == testLabels)) / + testLabels.n_elem; + +std::cout << "When trained on the training data:" << std::endl; +std::cout << " - Training set accuracy: " << trainAcc << "\%." << std::endl; +std::cout << " - Test set accuracy: " << testAcc << "\%." << std::endl; + +// Now reset the tree, and train on the test set instead. +// The dataset info and number of classes has not changed, so we can just call +// Reset() with no arguments. +tree.Reset(); +tree.Train(testDataset, testLabels); + +// Print accuracy on the training and test set, now that we have trained on the +// test set. +tree.Classify(dataset, predictions); +tree.Classify(testDataset, testPredictions); + +trainAcc = (100.0 * arma::accu(predictions == labels)) / labels.n_elem; +testAcc = (100.0 * arma::accu(testPredictions == testLabels)) / + testLabels.n_elem; + +std::cout << "When trained on the test data:" << std::endl; +std::cout << " - Training set accuracy: " << trainAcc << "\%." << std::endl; +std::cout << " - Test set accuracy: " << testAcc << "\%." << std::endl; +``` + +--- + ### Advanced Functionality: Template Parameters #### Using different element types. @@ -441,19 +519,24 @@ class CustomFitnessFunction * Specifies the strategy to be used during training when splitting a numeric feature. - * The `HoeffdingDoubleNumericSplit` _(default)_ class is available for drop-in - usage and discretizes the given numeric data into a default of 10 bins. This - expects `double` to be the type of the input data. - * The `HoeffdingFloatNumericSplit` class is available for drop-in usage and - operates similarly to `HoeffdingDoubleNumericSplit`, but expects `float` to - be the type of the input data. - * The `BinaryNumericSplit` class is available for drop-in usage and splits - numeric features in two in the way that maximizes gain. This split type is - more computationally expensive during training. + + * Several options are already implemented and available for drop-in usage. + - The `HoeffdingDoubleNumericSplit` _(default)_ class discretizes the given + numeric data into a default of 10 bins. This expects `double` to be the + type of the input data. + - The `HoeffdingFloatNumericSplit` class operates similarly to + `HoeffdingDoubleNumericSplit`, but expects `float` to be the type of the + input data. + - The `BinaryNumericSplit` class splits numeric features in two in the way + that maximizes gain. This split type is more computationally expensive + during training. + * A custom class must take a [`FitnessFunction`](#fitness-function) as a template parameter, implement several functions, and have an internal structure `SplitInfo` that is used at classification time: +TODO: note about constructor arguments + ```c++ // The job of this class is to track sufficient statistics of training data, // returning gain information if a split were to happen according to this @@ -533,12 +616,16 @@ class CustomNumericSplit * Specifies the strategy to be used during training when splitting a categorical feature. + * The `HoeffdingCategoricalSplit` _(default)_ is available for drop-in usage and splits all categories into their own node. + * A custom class must take a [`FitnessFunction`](#fitness-function) as a template parameter, implement several functions, and have an internal structure `SplitInfo` that is used at classification time: +TODO: note about constructor arguments + ```c++ // The job of this class is to track sufficient statistics of training data, // returning gain information if a split were to happen according to this From 7b4e21d23e1fb6f933ca5269f98d5a0261d0c51d Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Mon, 11 Dec 2023 11:54:04 -0500 Subject: [PATCH 47/91] Update tests and add tests for new functionality. --- src/mlpack/tests/hoeffding_tree_test.cpp | 323 ++++++++++++++++++++++- 1 file changed, 319 insertions(+), 4 deletions(-) diff --git a/src/mlpack/tests/hoeffding_tree_test.cpp b/src/mlpack/tests/hoeffding_tree_test.cpp index 0d5434426e..8a9deb10ff 100644 --- a/src/mlpack/tests/hoeffding_tree_test.cpp +++ b/src/mlpack/tests/hoeffding_tree_test.cpp @@ -1485,7 +1485,7 @@ TEST_CASE("HoeffdingTreeEmptyConstructorTrainTest", "[HoeffdingTreeTest]") HoeffdingTree<> ht; // Just ensure that we can train without throwing an exception. - REQUIRE_NOTHROW(ht.Train(data, labels)); + REQUIRE_NOTHROW(ht.Train(data, labels, 2)); // Now, create a categorical dataset and retrain. arma::mat data2 = arma::mat(4, 3000); @@ -1514,9 +1514,324 @@ TEST_CASE("HoeffdingTreeEmptyConstructorTrainTest", "[HoeffdingTreeTest]") } // Ensure we can train without throwing an exception. - REQUIRE_NOTHROW(ht.Train(data2, info, labels2)); + REQUIRE_NOTHROW(ht.Train(data2, info, labels2, 3)); // Train while specifying the number of classes. - REQUIRE_NOTHROW(ht.Train(data, labels, false, true, 2)); - REQUIRE_NOTHROW(ht.Train(data2, info, labels2, false, 3)); + REQUIRE_NOTHROW(ht.Train(data, labels, 2, false, true)); + REQUIRE_NOTHROW(ht.Train(data2, info, labels2, 3, false)); +} + +// Test all numeric Train() variants. +TEST_CASE("HoeffdingTreeNumericTrainVariantTest", "[HoeffdingTreeTest]") +{ + // Generate data. + arma::mat data(5, 1000, arma::fill::randu); + // Generate labels. + arma::Row labels(1000); + for (size_t i = 0; i < 500; ++i) + labels[i] = 0; + for (size_t i = 500; i < 1000; ++i) + labels[i] = 1; + + // Create trees. + HoeffdingTree<> htEmpty, ht1(5, 2), ht2(5, 2), ht3(5, 2), ht4(5, 2), + ht5(5, 2), ht6(5, 2), ht7(5, 2); + + htEmpty.Train(data, labels, 2); + ht1.Train(data, labels); + ht2.Train(data, labels, 2); + ht3.Train(data, labels, 2, true); + ht4.Train(data, labels, 2, false, 0.96); + ht5.Train(data, labels, 2, false, 0.97, 150); + ht6.Train(data, labels, 2, false, 0.98, 160, 110); + ht7.Train(data, labels, 2, false, 0.99, 170, 120, 110); + + REQUIRE(htEmpty.NumClasses() == 2); + REQUIRE(htEmpty.NumSamples() == data.n_cols); + + REQUIRE(ht1.NumClasses() == 2); + REQUIRE(ht1.NumSamples() == data.n_cols); + + REQUIRE(ht2.NumClasses() == 2); + REQUIRE(ht2.NumSamples() == data.n_cols); + + REQUIRE(ht3.NumClasses() == 2); + REQUIRE(ht3.NumSamples() == data.n_cols); + + REQUIRE(ht4.NumClasses() == 2); + REQUIRE(ht4.NumSamples() > 0); + REQUIRE(ht4.SuccessProbability() == 0.96); + + REQUIRE(ht5.NumClasses() == 2); + REQUIRE(ht5.NumSamples() > 0); + REQUIRE(ht5.SuccessProbability() == 0.97); + REQUIRE(ht5.MaxSamples() == 150); + + REQUIRE(ht6.NumClasses() == 2); + REQUIRE(ht6.NumSamples() > 0); + REQUIRE(ht6.SuccessProbability() == 0.98); + REQUIRE(ht6.MaxSamples() == 160); + REQUIRE(ht6.CheckInterval() == 110); + + REQUIRE(ht7.NumClasses() == 2); + REQUIRE(ht7.NumSamples() > 0); + REQUIRE(ht7.SuccessProbability() == 0.99); + REQUIRE(ht7.MaxSamples() == 170); + REQUIRE(ht7.CheckInterval() == 120); + REQUIRE(ht7.MinSamples() == 110); +} + +// Test all categorical Train() variants. +TEST_CASE("HoeffdingTreeCategoricalTrainVariantTest", "[HoeffdingTreeTest]") +{ + // Generate data. + arma::mat data(4, 9000); + arma::Row labels(9000); + data::DatasetInfo info(4); // All features are numeric, except the fourth. + info.MapString("0", 3); + for (size_t i = 0; i < 9000; i += 3) + { + data(0, i) = Random(); + data(1, i) = Random(); + data(2, i) = Random(); + data(3, i) = 0.0; + labels[i] = 0; + + data(0, i + 1) = Random(); + data(1, i + 1) = Random() - 1.0; + data(2, i + 1) = Random() + 0.5; + data(3, i + 1) = 0.0; + labels[i + 1] = 2; + + data(0, i + 2) = Random(); + data(1, i + 2) = Random() + 1.0; + data(2, i + 2) = Random() + 0.8; + data(3, i + 2) = 0.0; + labels[i + 2] = 1; + } + + // Create trees. + HoeffdingTree<> htEmpty, ht1(info, 3), ht2(info, 3), ht3(info, 3), + ht4(info, 3), ht5(info, 3), ht6(info, 3), ht7(info, 3); + + htEmpty.Train(data, info, labels, 3); + ht1.Train(data, info, labels); + ht2.Train(data, info, labels, 3); + ht3.Train(data, info, labels, 3, true); + ht4.Train(data, info, labels, 3, false, 0.96); + ht5.Train(data, info, labels, 3, false, 0.97, 150); + ht6.Train(data, info, labels, 3, false, 0.98, 160, 110); + ht7.Train(data, info, labels, 3, false, 0.99, 170, 120, 110); + + REQUIRE(htEmpty.NumClasses() == 3); + REQUIRE(htEmpty.NumSamples() == data.n_cols); + + REQUIRE(ht1.NumClasses() == 3); + REQUIRE(ht1.NumSamples() == data.n_cols); + + REQUIRE(ht2.NumClasses() == 3); + REQUIRE(ht2.NumSamples() == data.n_cols); + + REQUIRE(ht3.NumClasses() == 3); + REQUIRE(ht3.NumSamples() == data.n_cols); + + REQUIRE(ht4.NumClasses() == 3); + REQUIRE(ht4.NumSamples() > 0); + REQUIRE(ht4.SuccessProbability() == 0.96); + + REQUIRE(ht5.NumClasses() == 3); + REQUIRE(ht5.NumSamples() > 0); + REQUIRE(ht5.SuccessProbability() == 0.97); + REQUIRE(ht5.MaxSamples() == 150); + + REQUIRE(ht6.NumClasses() == 3); + REQUIRE(ht6.NumSamples() > 0); + REQUIRE(ht6.SuccessProbability() == 0.98); + REQUIRE(ht6.MaxSamples() == 160); + REQUIRE(ht6.CheckInterval() == 110); + + REQUIRE(ht7.NumClasses() == 3); + REQUIRE(ht7.NumSamples() > 0); + REQUIRE(ht7.SuccessProbability() == 0.99); + REQUIRE(ht7.MaxSamples() == 170); + REQUIRE(ht7.CheckInterval() == 120); + REQUIRE(ht7.MinSamples() == 110); +} + +// Test overloads of Reset(). (Also test NumSamples().) +TEST_CASE("HoeffdingTreeResetTests", "[HoeffdingTreeTest]") +{ + // Generate data. + arma::mat data(5, 1000, arma::fill::randu); + // Generate labels. + arma::Row labels(1000); + for (size_t i = 0; i < 500; ++i) + labels[i] = 0; + for (size_t i = 500; i < 1000; ++i) + labels[i] = 1; + + // Train the tree. + HoeffdingTree<> ht(data, labels, 2); + + REQUIRE(ht.NumSamples() > 0); + REQUIRE(ht.NumClasses() == 2); + + // Reset the tree, changing nothing. + ht.Reset(); + REQUIRE(ht.NumSamples() == 0); + REQUIRE(ht.NumChildren() == 0); + + ht.Train(data, labels); + REQUIRE(ht.NumSamples() > 0); + REQUIRE(ht.NumClasses() == 2); + + // Reset the tree, changing the dimensionality and number of classes. + ht.Reset(10, 3); + REQUIRE(ht.NumSamples() == 0); + REQUIRE(ht.NumChildren() == 0); + + data = arma::randu(10, 1000); + for(size_t i = 750; i < 1000; ++i) + labels[i] = 2; + + ht.Train(data, labels); + REQUIRE(ht.NumSamples() > 0); + REQUIRE(ht.NumClasses() == 3); + + // Reset the tree to work on categorical data with a different number of + // classes. + data::DatasetInfo info(10); + info.MapString("0", 3); + data.row(9).fill(0.0); + for (size_t i = 0; i < 250; ++i) + labels[i] = 3; + + ht.Reset(info, 4); + REQUIRE(ht.NumSamples() == 0); + REQUIRE(ht.NumChildren() == 0); + + ht.Train(data, info, labels); + + REQUIRE(ht.NumSamples() > 0); + REQUIRE(ht.NumClasses() == 4); +} + +// Test that we can learn on floating-point data. +TEST_CASE("HoeffdingTreeNumericFloatDataTest", "[HoeffdingTreeTest]") +{ + // We need to create a dataset with some amount of complexity, that must be + // split in a handful of ways to accurately classify the data. An expanding + // spiral should do the trick here. We'll make the spiral in two dimensions. + // The label will change as the index increases. + arma::fmat spiralDataset(2, 10000); + for (size_t i = 0; i < 10000; ++i) + { + // One circle every 20000 samples. Plus some noise. + const float magnitude = 2.0 + (float(i) / 20000.0) + 0.5 * Random(); + const float angle = (i % 20000) * (2 * M_PI) + Random(); + + const float x = magnitude * cos(angle); + const float y = magnitude * sin(angle); + + spiralDataset(0, i) = x; + spiralDataset(1, i) = y; + } + + arma::Row labels(10000); + for (size_t i = 0; i < 2000; ++i) + labels[i] = 1; + for (size_t i = 2000; i < 4000; ++i) + labels[i] = 3; + for (size_t i = 4000; i < 6000; ++i) + labels[i] = 2; + for (size_t i = 6000; i < 8000; ++i) + labels[i] = 0; + for (size_t i = 8000; i < 10000; ++i) + labels[i] = 4; + + // Now shuffle the dataset. + arma::uvec indices = arma::shuffle(arma::linspace(0, 9999, + 10000)); + arma::fmat d(2, 10000); + arma::Row l(10000); + for (size_t i = 0; i < 10000; ++i) + { + d.col(i) = spiralDataset.col(indices[i]); + l[i] = labels[indices[i]]; + } + + // Split into a training set and a test set. + arma::fmat trainingData = d.cols(0, 4999); + arma::fmat testData = d.cols(5000, 9999); + arma::Row trainingLabels = l.subvec(0, 4999); + arma::Row testLabels = l.subvec(5000, 9999); + + data::DatasetInfo info(2); + + // Now build two decision trees; one in batch mode, and one in streaming mode. + // We need to set the confidence pretty high so that the streaming tree isn't + // able to have enough samples to build to the same leaves. + HoeffdingTree + batchTree(trainingData, info, trainingLabels, 5, true, 0.99999999); + HoeffdingTree + streamTree(trainingData, info, trainingLabels, 5, false, 0.99999999); + + // Ensure that the performance of the batch tree is better. + size_t batchCorrect = 0; + size_t streamCorrect = 0; + for (size_t i = 0; i < 5000; ++i) + { + size_t streamLabel = streamTree.Classify(testData.col(i)); + size_t batchLabel = batchTree.Classify(testData.col(i)); + + if (streamLabel == testLabels[i]) + ++streamCorrect; + if (batchLabel == testLabels[i]) + ++batchCorrect; + } + + // The batch tree must be a bit better than the stream tree. But not too + // much, since the accuracy is already going to be very high. + REQUIRE(batchCorrect >= streamCorrect); +} + +// Test that we can learn on categorical floating-point data. +TEST_CASE("HoeffdingTreeCategoricalFloatDataTest", "[HoeffdingTreeTest]") +{ + // Generate data. + arma::fmat dataset(4, 3000, arma::fill::randu); + arma::Row labels(3000); + for (size_t i = 0; i < 3000; i += 3) + { + labels[i] = 0; + + labels[i + 1] = 2; + dataset.col(i + 1) += 2.0; + + labels[i + 2] = 1; + dataset.col(i + 2) -= 2.0; + } + + data::DatasetInfo info(4); // All features are numeric, except the fourth. + info.MapString("0", 3); + dataset.row(3).fill(0.0); + + // Now build two decision trees; one in batch mode, and one in streaming mode. + HoeffdingTree + batchTree(dataset, info, labels, 3, true); + HoeffdingTree + streamTree(dataset, info, labels, 3, false); + + // Make sure that we trained successfully. + arma::Row batchPredictions, streamPredictions; + batchTree.Classify(dataset, batchPredictions); + streamTree.Classify(dataset, streamPredictions); + + REQUIRE(batchPredictions.n_elem == labels.n_elem); + REQUIRE(streamPredictions.n_elem == labels.n_elem); + + // Make sure that the accuracy is at least reasonable. (This is a very loose + // test.) + REQUIRE(arma::accu(batchPredictions == labels) > (labels.n_elem / 2)); + REQUIRE(arma::accu(streamPredictions == labels) > (labels.n_elem / 2)); } From eb4b78ad56b7f01bc0f1e907bc716ec2c0bec84d Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Mon, 11 Dec 2023 12:29:54 -0500 Subject: [PATCH 48/91] Clean up documentation. --- doc/user/methods/hoeffding_tree.md | 72 +++++++++++++++++++++--------- 1 file changed, 52 insertions(+), 20 deletions(-) diff --git a/doc/user/methods/hoeffding_tree.md b/doc/user/methods/hoeffding_tree.md index 452f0066e1..ed8adc9055 100644 --- a/doc/user/methods/hoeffding_tree.md +++ b/doc/user/methods/hoeffding_tree.md @@ -80,14 +80,17 @@ std::cout << arma::accu(predictions == 2) << " test points classified as class " * `tree = HoeffdingTree(data, labels, numClasses) * `tree = HoeffdingTree(data, labels, numClasses, batchTraining=true, successProbability=0.95, maxSamples=0, checkInterval=100, minSamples=100)` - - Train non-incrementally on numerical-only data. - TODO: should this actually be numerical-only? + - Train non-incrementally on the given data. + - The tree will be reset if `numClasses` or the data's dimensionality does + not match the current settings of the tree. --- * `tree = HoeffdingTree(data, datasetInfo, labels, numClasses) * `tree = HoeffdingTree(data, datasetInfo, labels, numClasses, batchTraining=true, successProbability=0.95, maxSamples=0, checkInterval=100, minSamples=100)` - Train non-incrementally on mixed categorical data. + - The tree will be reset if `numClasses` or `datasetInfo` does not match the + current settings of the tree. --- @@ -113,11 +116,28 @@ std::cout << arma::accu(predictions == 2) << " test points classified as class " | `checkInterval` | `size_t` | Number of samples required before each split check. Higher values check less often, which is more efficient, but may not split a node as early as possible. | `100` | | `minSamples` | `size_t` | Minimum number of samples for a node to see before a split is allowed. | `100` | +As an alternative to passing hyperparameters, these can be set with a +standalone method. The following functions can be used before calling +`Train()`: + + * `tree.SuccessProbability(successProbability);` will set the required success + probability for splitting to `successProbability`. + * `tree.MaxSamples(maxSamples);` will set the maximum number of samples before + a split to `maxSamples`. + * `tree.CheckInterval(checkInterval);` will set the number of samples between + split checks to `checkInterval`. + * `tree.MinSamples(minSamples);` will set the minimum number of samples before + a split to `minSamples`. + +***Notes:*** + * Setting `successProbability` higher than the default means that the Hoeffding tree is less likely (and will take more samples) to split a node. This can result in a smaller tree. -***Note:*** different types can be used for `data` (e.g., `arma::fmat`, `arma::sp_mat`). + * Different types can be used for `data` (e.g., `arma::fmat`, `arma::sp_mat`). + See [template parameters](#advanced-functionality-template-parameters) for + using different `NumericSplitType`s that accept different element types. ### Training @@ -288,15 +308,15 @@ mlpack::data::Load("covertype.test.arff", testDataset, info, true); // Predict class of first test point. const size_t firstPrediction = tree.Classify(testDataset.col(0)); -std::cout << "Predicted class of first test point is " << firstPrediction << "." +std::cout << "First test point has predicted class " << firstPrediction << "." << std::endl; // Predict class and probabilities of second test point. size_t secondPrediction; -arma::vec secondProbabilities; -tree.Classify(testDataset.col(1), secondPrediction, secondProbabilities); -std::cout << "Class probabilities of second test point: " << - secondProbabilities.t(); +double secondProbability; +tree.Classify(testDataset.col(1), secondPrediction, secondProbability); +std::cout << "Second test point has predicted class " << secondPrediction + << " with probability " << secondProbability << "." << std::endl; ``` --- @@ -326,20 +346,20 @@ arma::Row testLabels; mlpack::data::Load("covertype.test.labels.csv", testLabels, true); // Create the tree with custom parameters. -mlpack::HoeffdingTree tree; -tree.SuccessProbability() = 0.99; -tree.CheckInterval() = 500; +mlpack::HoeffdingTree tree(info, 7 /* number of classes */); +tree.SuccessProbability(0.99); +tree.CheckInterval(500); // Now iterate over 10k-point chunks in the dataset. -for (size_t start = 0; start < data.n_cols; start += 10000) +for (size_t start = 0; start < dataset.n_cols; start += 10000) { - size_t end = std::min(start + 9999, data.n_cols - 1); + size_t end = std::min(start + 9999, (size_t) dataset.n_cols - 1); tree.Train(dataset.cols(start, end), info, labels.subvec(start, end)); // Compute accuracy on the test set. arma::Row predictions; - tree.Predict(testDataset, predictions); + tree.Classify(testDataset, predictions); const double accuracy = 100.0 * arma::accu(predictions == testLabels) / testLabels.n_elem; @@ -395,8 +415,8 @@ mlpack::data::Load("covertype.test.labels.csv", testLabels, true); // Create a tree, and train on the training data. mlpack::HoeffdingTree tree(info, 7 /* number of classes */, 0.98); -tree.MinSamples() = 500; -tree.CheckInterval() = 500; +tree.MinSamples(500); +tree.CheckInterval(500); tree.Train(dataset, labels); @@ -531,12 +551,18 @@ class CustomFitnessFunction that maximizes gain. This split type is more computationally expensive during training. + * If a non-default `NumericSplitType` is specified, the following constructor + forms can be used to pass constructed `NumericSplitType`s to the + `HoeffdingTree` to use as copy-constructed templates during splitting: + - `HoeffdingTree(dimensionality, numClasses, successProbability, maxSamples, checkInterval, minSamples, categoricalSplit, numericSplit)` + - `HoeffdingTree(datasetInfo, numClasses, successProbability, maxSamples, checkInterval, minSamples, categoricalSplit, numericSplit)` + - `HoeffdingTree(data, labels, numClasses, successProbability, maxSamples, checkInterval, minSamples, categoricalSplit, numericSplit)` + - `HoeffdingTree(data, datasetInfo, labels, numClasses, successProbability, maxSamples, checkInterval, minSamples, categoricalSplit, numericSplit)` + * A custom class must take a [`FitnessFunction`](#fitness-function) as a template parameter, implement several functions, and have an internal structure `SplitInfo` that is used at classification time: -TODO: note about constructor arguments - ```c++ // The job of this class is to track sufficient statistics of training data, // returning gain information if a split were to happen according to this @@ -620,12 +646,18 @@ class CustomNumericSplit * The `HoeffdingCategoricalSplit` _(default)_ is available for drop-in usage and splits all categories into their own node. + * If a non-default `CategoricalSplitType` is specified, the following + constructor forms can be used to pass constructed `CategoricalSplitType`s to + the `HoeffdingTree` to use as copy-constructed templates during splitting: + - `HoeffdingTree(dimensionality, numClasses, successProbability, maxSamples, checkInterval, minSamples, categoricalSplit, numericSplit)` + - `HoeffdingTree(datasetInfo, numClasses, successProbability, maxSamples, checkInterval, minSamples, categoricalSplit, numericSplit)` + - `HoeffdingTree(data, labels, numClasses, successProbability, maxSamples, checkInterval, minSamples, categoricalSplit, numericSplit)` + - `HoeffdingTree(data, datasetInfo, labels, numClasses, successProbability, maxSamples, checkInterval, minSamples, categoricalSplit, numericSplit)` + * A custom class must take a [`FitnessFunction`](#fitness-function) as a template parameter, implement several functions, and have an internal structure `SplitInfo` that is used at classification time: -TODO: note about constructor arguments - ```c++ // The job of this class is to track sufficient statistics of training data, // returning gain information if a split were to happen according to this From 85371f37472d1326be57b96e851b899d6ad05c01 Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Mon, 11 Dec 2023 12:30:10 -0500 Subject: [PATCH 49/91] Bugfix for subview datatypes. --- .../methods/hoeffding_trees/hoeffding_tree_impl.hpp | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/src/mlpack/methods/hoeffding_trees/hoeffding_tree_impl.hpp b/src/mlpack/methods/hoeffding_trees/hoeffding_tree_impl.hpp index 16f94bbccd..fd5f997da4 100644 --- a/src/mlpack/methods/hoeffding_trees/hoeffding_tree_impl.hpp +++ b/src/mlpack/methods/hoeffding_trees/hoeffding_tree_impl.hpp @@ -1335,7 +1335,18 @@ void HoeffdingTree< // Train(), since the col() function is not provided. So, // unfortunately, instead, we'll just extract the non-contiguous // submatrix. - MatType childData = data.cols(indices[i].subvec(0, counts[i] - 1)); + // + // I'd rather be able to use: + // + // arma::Mat childData = + // data.cols(indices[i].subvec(0, counts[i] - 1)); + // + // but this isn't currently supported by Armadillo. + arma::Mat childData(data.n_rows, + counts[i]); + for (size_t j = 0; j < counts[i]; ++j) + childData.col(j) = data.col(indices[i][j]); + children[i]->Train(childData, childLabels, numClasses, true); } } From b54f7544ed62c6ce149f3ac6d4ac9344f0211e55 Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Mon, 11 Dec 2023 12:33:13 -0500 Subject: [PATCH 50/91] Fix typo. --- doc/user/methods/hoeffding_tree.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/user/methods/hoeffding_tree.md b/doc/user/methods/hoeffding_tree.md index ed8adc9055..003e2bd3af 100644 --- a/doc/user/methods/hoeffding_tree.md +++ b/doc/user/methods/hoeffding_tree.md @@ -78,7 +78,7 @@ std::cout << arma::accu(predictions == 2) << " test points classified as class " --- - * `tree = HoeffdingTree(data, labels, numClasses) + * `tree = HoeffdingTree(data, labels, numClasses)` * `tree = HoeffdingTree(data, labels, numClasses, batchTraining=true, successProbability=0.95, maxSamples=0, checkInterval=100, minSamples=100)` - Train non-incrementally on the given data. - The tree will be reset if `numClasses` or the data's dimensionality does From 25ba022503bfe28b9f03b406cff2a4d6572fe764 Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Mon, 11 Dec 2023 13:17:09 -0500 Subject: [PATCH 51/91] Link to fully-working examples from examples repository. --- doc/user/methods/linear_regression.md | 6 ++++++ doc/user/methods/naive_bayes_classifier.md | 6 ++++++ 2 files changed, 12 insertions(+) diff --git a/doc/user/methods/linear_regression.md b/doc/user/methods/linear_regression.md index 0b6d102a4d..84b81a337f 100644 --- a/doc/user/methods/linear_regression.md +++ b/doc/user/methods/linear_regression.md @@ -232,6 +232,12 @@ for (size_t t = 0; t < 3; ++t) --- +See also the following fully-working examples: + + - [Salary prediction with `LinearRegression`](https://github.com/mlpack/examples/blob/master/salary_prediction_with_linear_regression/salary-prediction-linear-regression-cpp.ipynb) + - [Avocado price prediction with `LinearRegression`](https://github.com/mlpack/examples/blob/master/avocado_price_prediction_with_linear_regression/avocado_price_prediction_with_lr_cpp.ipynb) + - [California housing price prediction with `LinearRegression`](https://github.com/mlpack/examples/blob/master/california_housing_price_prediction_with_linear_regression/california_housing_price_prediction_with_lr_cpp.ipynb) + ### Advanced Functionality: Different Element Types The `LinearRegression` class has one template parameter that can be used to diff --git a/doc/user/methods/naive_bayes_classifier.md b/doc/user/methods/naive_bayes_classifier.md index bb4395b97b..3dc56041d2 100644 --- a/doc/user/methods/naive_bayes_classifier.md +++ b/doc/user/methods/naive_bayes_classifier.md @@ -278,6 +278,12 @@ std::cout << "Random point class prediction: " << prediction << "." std::cout << "Random point class probabilities: " << probabilities.t(); ``` +--- + +See also the following fully-working examples: + + - [Microchip QA Classification using `NaiveBayesClassifier`](https://github.com/mlpack/examples/blob/master/microchip_quality_control_naive_bayes/microchip-quality-control-naive-bayes-cpp.ipynb) + ### Advanced Functionality: Different Element Types The `NaiveBayesClassifier` class has one template parameter that can be used to From 0b02c1aae1424cee518850361ef7e3c39246f656 Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Mon, 11 Dec 2023 15:12:52 -0500 Subject: [PATCH 52/91] Don't use deprecated function. --- .../methods/logistic_regression/logistic_regression_main.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mlpack/methods/logistic_regression/logistic_regression_main.cpp b/src/mlpack/methods/logistic_regression/logistic_regression_main.cpp index 8566980e80..3ec5310868 100644 --- a/src/mlpack/methods/logistic_regression/logistic_regression_main.cpp +++ b/src/mlpack/methods/logistic_regression/logistic_regression_main.cpp @@ -386,7 +386,7 @@ void BINDING_FUNCTION(util::Params& params, util::Timers& timers) Log::Info << "Calculating class probabilities of points in '" << params.GetPrintable("test") << "'." << endl; arma::mat probabilities; - model->Classify(testSet, probabilities); + model->Classify(testSet, predictions, probabilities); if (params.Has("probabilities")) params.Get("probabilities") = std::move(probabilities); From d4c335e7b3422ea1ddd758f0aebfb539e226e688 Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Mon, 11 Dec 2023 17:18:35 -0500 Subject: [PATCH 53/91] Add first pass of linear SVM documentation. --- doc/user/methods/linear_svm.md | 371 +++++++++++++++++++++++++++++++++ 1 file changed, 371 insertions(+) create mode 100644 doc/user/methods/linear_svm.md diff --git a/doc/user/methods/linear_svm.md b/doc/user/methods/linear_svm.md new file mode 100644 index 0000000000..e4a7c81064 --- /dev/null +++ b/doc/user/methods/linear_svm.md @@ -0,0 +1,371 @@ +## `LinearSVM` + +The `LinearSVM` class implements an L2-regularized support vector machine for +numerical data, with training done using any ensmallen optimizer. The class +offers standard classification functionality. Linear SVM is useful for +multi-class classification (i.e. classes are `0`, `1`, `2`, etc.). + +#### Simple usage example: + +```c++ +// Train a linear SVM classifier on random data and predict labels: + +// All data and labels are uniform random; 5 dimensional data, 4 classes. +// Replace with a data::Load() call or similar for a real application. +arma::mat dataset(5, 1000, arma::fill::randu); // 1000 points. +arma::Row labels = + arma::randi>(1000, arma::distr_param(0, 3)); +arma::mat testDataset(5, 500, arma::fill::randu); // 500 test points. + +mlpack::LinearSVM svm; // Step 1: create model. +svm.Train(dataset, labels, 4); // Step 2: train model. +arma::Row predictions; +svm.Classify(testDataset, predictions); // Step 3: classify points. + +// Print some information about the test predictions. +std::cout << arma::accu(predictions == 1) << " test points classified as class " + << "1." << std::endl; +``` +

More examples...

+ +#### Quick links: + + * [Constructors](#constructors): create `LinearSVM` objects. + * [`Train()`](#training): train model. + * [`Classify()`](#classification): classify with a trained model. + * [Other functionality](#other-functionality) for loading, saving, and + inspecting. + * [Examples](#simple-examples) of simple usage and links to detailed example + projects. + * [Template parameters](#advanced-functionality-different-element-types) for + using different element types for a model. + +#### See also: + + * [mlpack classifiers](#mlpack_classifiers) + * [`GaussianDistribution`](#gaussian_distribution) + * [Naive Bayes classifier on Wikipedia](https://en.wikipedia.org/wiki/Naive_Bayes_classifier) + +### Constructors + + * `svm = NaiveBayesClassifier(dimensionality)` + * `svm = NaiveBayesClassifier(dimensionality, numClasses, lambda=0.0001, delta=1.0, fitIntercept=false)` + - Initialize the model without training. + - You will need to call [`Train()`](#training) later to train the model + before calling [`Classify()`](#classification). + +--- + + * `svm = LinearSVM(data, labels, numClasses, lambda=0.0001, delta=1.0, fitIntercept=false, [callbacks...])` + - Train model, optionally specifying ensmallen callbacks for use during + optimization. + +--- + + * `svm = LinearSVM(data, labels, numClasses, optimizer, lambda=0.0001, delta=1.0, fitIntercept=false, [callbacks...])` + - Train model with a custom ensmallen optimizer, optionally specifying + callbacks for use during optimization. + +--- + +#### Constructor Parameters: + + + +| **name** | **type** | **description** | **default** | +|----------|----------|-----------------|-------------| +| `data` | [`arma::mat`](../matrices.md) | [Column-major](../matrices.md) training matrix. | _(N/A)_ | +| `labels` | [`arma::Row`]('../matrices.md') | Training labels, between `0` and `numClasses - 1` (inclusive). Should have length `data.n_cols`. | _(N/A)_ | +| `dimensionality` | `size_t` | Dimension of input data (if data is not specified). Should be equal to `data.n_rows`. | _(N/A)_ | +| `numClasses` | `size_t` | Number of classes in the dataset. | _(N/A)_ | +| `optimizer` | [any ensmallen optimizer](https://www.ensmallen.org) | Instantiated ensmallen optimizer for [differentiable functions](https://www.ensmallen.org/docs.html#differentiable-functions) or [differentiable separable functions](https://www.ensmallen.org/docs.html#differentiable-separable-functions). | `ens::L_BFGS()` | +| `lambda` | `double` | L2 regularization penalty parameter. Must be nonnegative. | `0.0` | +| `delta` | `double` | Margin of difference between correct class and other classes. | `1.0` | +| `fitIntercept` | `bool` | If `true`, then an intercept term is fitted to the model. | `false` | +| `callbacks...` | [any set of ensmallen callbacks](https://www.ensmallen.org/docs.html#callback-documentation) | Optional callbacks for the ensmallen optimizer, such as e.g. `ens::ProgressBar()`, `ens::Report()`, or others. | _(N/A)_ | +As an alternative to passing the `epsilon` parameter, it can be set with the +standalone `Epsilon()` method: `nbc.Epsilon() = eps;` will set the value of +`epsilon` to `eps` for the next time non-incremental `Train()` or `Reset()` is +called. + +As an alternative to passing `lambda`, `delta`, or `fitIntercept`, these can be +set with a standalone method. The following functions can be used before +calling `Train()`: + + * `svm.Lambda() = lambda;` will set the L2 regularization penalty parameter to + `lambda`. + * `svm.Delta() = delta;` will set the margin of difference to `delta`. + * `svm.FitIntercept() = fitIntercept;` will set whether the model fits an + intercept to `fitIntercept`. + +### Training + +If training is not done as part of the constructor call, it can be done with the +`Train()` function: + + * `svm.Train(data, labels, numClasses, lambda=0.0001, delta=1.0, fitIntercept=false, [callbacks...]) + - Train model on the given data, optionally specifying ensmallen callbacks + for use during optimization. + +--- + + * `svm.Train(data, labels, numClasses, optimizer, lambda=0.0001, delta=1.0, + fitIntercept=false, [callbacks...]) + - Train model on the given data with a custom ensmallen optimizer, optionally + specifying callbacks for use during optimization. + +--- + +Types of each argument are the same as in the table for constructors +[above](#constructor-parameters). + +***Note:*** Training is not incremental. Successive calls to `Train()` will +train entirely new models. + +### Classification + +Once a `LinearSVM` model is trained, the `Classify()` member function +can be used to make class predictions for new data. + + * `size_t predictedClass = svm.Classify(point)` + - ***(Single-point)*** + - Classify a single point, returning the predicted class (`0` through + `numClasses - 1`, inclusive). + +--- + + * `svm.Classify(point, prediction, probabilitiesVec)` + - ***(Single-point)*** + - Classify a single point and compute class probabilities. + - The predicted class is stored in `prediction`. + - The probability of class `i` can be accessed with `probabilitiesVec[i]`. + +--- + + * `svm.Classify(data, predictions)` + - ***(Multi-point)*** + - Classify a set of points. + - The prediction for data point `i` can be accessed with `predictions[i]`. + +--- + + * `svm.Classify(data, predictions, probabilities)` + - ***(Multi-point)*** + - Classify a set of points and compute class probabilities. + - The prediction for data point `i` can be accessed with `predictions[i]`. + - The probability of class `j` for data point `i` can be accessed with + `probabilities(j, i)`. + +--- + +#### Classification Parameters: + +| **usage** | **name** | **type** | **description** | +|-----------|----------|----------|-----------------| +| _single-point_ | `point` | [`arma::vec`](../matrices.md) | Single point for classification. | +| _single-point_ | `prediction` | `size_t&` | `size_t` to store class prediction into. | +| _single-point_ | `probabilitiesVec` | [`arma::vec&`](../matrices.md) | `arma::vec&` to store class probabilities into; will have length 2. | +|||| +| _multi-point_ | `data` | [`arma::mat`](../matrices.md) | Set of [column-major](../matrices.md) points for classification. | +| _multi-point_ | `predictions` | [`arma::Row&`](../matrices.md) | Vector of `size_t`s to store class prediction into; will be set to length `data.n_cols`. | +| _multi-point_ | `probabilities` | [`arma::mat&`](../matrices.md) | Matrix to store class probabilities into (number of rows will be equal to 2; number of columns will be equal to `data.n_cols`). | + +### Other Functionality + + + + * A `LinearSVM` model can be serialized with + [`data::Save()`](../formats.md) and [`data::Load()`](../formats.md). + + * `svm.Parameters()` will return the parameters of the model as an `arma::mat` + with either `data.n_rows` rows (if `FitIntercept()` is `false`) or + `data.n_rows + 1` rows (if `FitIntercept()` is `true`), and `numClasses` + columns. The weight for dimension `i` for class `j` can be accessed with + `svm.Parameters()(i, j)`. If `FitIntercept()` is `true`, the last row of + `svm.Parameters()` represents the bias parameters for each class. + + * `svm.FeatureSize()` will return the number of features in the model. This is + equivalent to `data.n_rows` when the model was trained. + + * `svm.ComputeAccuracy(data, labels)` will return the accuracy of the model on + the given `data` with the given `labels`. The returned accuracy is between 0 + and 100. + +### Simple Examples + +See also the [simple usage example](#simple-usage-example) for a trivial usage +of the `LinearSVM` class. + +--- + +Train a linear SVM using a custom SGD-like optimizer with callbacks. + +```c++ +// See https://datasets.mlpack.org/satellite.train.csv. +arma::mat dataset; +mlpack::data::Load("satellite.train.csv", dataset, true); +// See https://datasets.mlpack.org/satellite.train.labels.csv. +arma::Row labels; +mlpack::data::Load("satellite.train.labels.csv", labels, true); + +mlpack::LinearSVM svm; +svm.Lambda() = 0.1; + +// Create AMSGrad optimizer with custom step size and batch size. +ens::AMSGrad optimizer(0.01 /* step size */, 16 /* batch size */); +optimizer.MaxIterations() = 100 * dataset.n_cols; // Allow 100 epochs. + +// Print a progress bar and an optimization report when training is finished. +svm.Train(dataset, labels, optimizer, ens::ProgressBar(), ens::Report()); + +// Now predict on test labels and compute accuracy. + +// See https://datasets.mlpack.org/satellite.test.csv. +arma::mat testDataset; +mlpack::data::Load("satellite.test.csv", testDataset, true); +// See https://datasets.mlpack.org/satellite.test.labels.csv. +arma::Row testLabels; +mlpack::data::Load("satellite.test.labels.csv", testLabels, true); + +std::cout << std::endl; +std::cout << "Accuracy on training set: " + << svm.ComputeAccuracy(dataset, labels) << "\%." << std::endl; +std::cout << "Accuracy on test set: " + << svm.ComputeAccuracy(testDataset, testLabels) << "\%." << std::endl; +``` + +--- + +Train a linear SVM with SGD and save the model every epoch using a [custom +ensmallen callback](https://www.ensmallen.org/docs.html#custom-callbacks): + +```c++ +// This callback saves the model into "model-.bin" after every epoch. +class ModelCheckpoint +{ + public: + ModelCheckpoint(mlpack::LogisticRegression<>& model) : model(model) { } + + template + bool EndEpoch(OptimizerType& /* optimizer */, + FunctionType& /* function */, const MatType& /* coordinates */, + const size_t epoch, const double /* objective */) + { + const std::string filename = "model-" + std::to_string(epoch) + ".bin"; mlpack::data::Save(filename, "svm", model, true); + return false; // Do not terminate the optimization. + } + + private: + mlpack::LinearSVM<>& model; +}; +``` + +With that callback available, the code to train the model is below: + +```c++ +// See https://datasets.mlpack.org/satellite.train.csv. +arma::mat dataset; +mlpack::data::Load("satellite.train.csv", dataset, true); +// See https://datasets.mlpack.org/satellite.train.labels.csv. +arma::Row labels; +mlpack::data::Load("satellite.train.labels.csv", labels, true); + +mlpack::LinearSVM lr; + +// Create AdaDelta optimizer with a small step size and batch size of 1. +ens::AdaDelta adaDelta(0.001, 1); +adaDelta.MaxIterations() = 100 * dataset.n_cols; // 100 epochs maximum. + +// Use the custom callback and an L2 penalty parameter of 0.01, with default +// delta and fitting an intercept. +svm.Train(dataset, labels, adaDelta, 0.01, 1.0, true, ModelCheckpoint(lr), + ens::ProgressBar()); + +// Now files like model-1.bin, model-2.bin, etc. should be saved on disk. +``` + +--- + +Load a linear SVM from disk and print some information about it. + +```c++ +mlpack::LinearSVM svm; +// This assumes that a model called "svm" has been saved to the file +// "model-1.bin" (as in the previous example). +mlpack::data::Load("model-1.bin", "svm", svm, true); + +// Print the dimensionality of the model and some other statistics. +std::cout << "The dimensionality of the model in model-1.bin is " + << svm.FeatureSize() << "." << std::endl; +if (svm.FitIntercept()) +{ + std::cout << "Intercept values for each class: " << std::endl; + for (size_t i = 0; i < svm.Parameters().n_cols; ++i) + { + std::cout << " - Class " << i << ": " + << svm.Parameters()(svm.Parameters().n_rows - 1, i) << "." << std::endl; + } +} +else +{ + std::cout << "The model does not have an intercept fitted." << std::endl; +} + +std::cout << "The L2 regularization penalty parameter is: " << svm.Lambda() + << "." << std::endl; + +std::cout << "Weights for the first dimension are: " + << svm.Parameters().row(0) << "." << std::endl; +``` + +--- + +### Advanced Functionality: Different Element Types + +The `LinearSVM` class has one template parameter that can be used to +control the element type of the model. The full signature of the class is: + +```c++ +NaiveBayesClassifier +``` + +`ModelMatType` specifies the type of matrix used for training data and internal +representation of model parameters. + + * Any matrix type that implements the Armadillo API can be used. + + * `Train()` and `Classify()` functions themselves are templatized and can allow + any matrix type that has the same element type. So, for instance, a + `LinearSVM` can accept an `arma::sp_mat` for training. + +The example below trains a linear SVM on sparse 32-bit floating point +data, but uses dense 32-bit floating point matrices to store the model itself. + +```c++ +// Create random, sparse 100-dimensional data, with 3 classes. +arma::sp_fmat dataset; +dataset.sprandu(100, 5000, 0.3); +arma::Row labels = + arma::randi>(5000, arma::distr_param(0, 2)); + +mlpack::LinearSVM svm(dataset, labels, 3); + +// Now classify a test point. +arma::sp_fvec point; +point.sprandu(100, 1, 0.3); + +size_t prediction; +arma::fvec probabilitiesVec; +svm.Classify(point, prediction, probabilitiesVec); + +std::cout << "Prediction for random test point: " << prediction << "." + << std::endl; +std::cout << "Class probabilities for random test point: " + << probabilitiesVec.t(); +``` + +***Note:*** dense objects should be used for `ModelMatType`, since in general +L2-regularized models are fully dense. From 0ce1000d75ec53c138fd9aa400edf9385b7ba8e3 Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Mon, 20 Nov 2023 16:02:27 -0500 Subject: [PATCH 54/91] Add detectors for ensmallen optimizers and callbacks. --- src/mlpack/core.hpp | 1 + src/mlpack/core/util/ens_traits.hpp | 99 +++++++++++++++++++++++++ src/mlpack/core/util/sfinae_utility.hpp | 2 +- 3 files changed, 101 insertions(+), 1 deletion(-) create mode 100644 src/mlpack/core/util/ens_traits.hpp diff --git a/src/mlpack/core.hpp b/src/mlpack/core.hpp index f016df2f88..80e1e9b9b7 100644 --- a/src/mlpack/core.hpp +++ b/src/mlpack/core.hpp @@ -36,6 +36,7 @@ // Now the core mlpack classes. #include +#include #include #include #include diff --git a/src/mlpack/core/util/ens_traits.hpp b/src/mlpack/core/util/ens_traits.hpp new file mode 100644 index 0000000000..e8789f052f --- /dev/null +++ b/src/mlpack/core/util/ens_traits.hpp @@ -0,0 +1,99 @@ +/** + * @file core/util/ens_traits.hpp + * @author Ryan Curtin + * + * This file contains utilities for SFINAE on ensmallen types (optimizers and + * callbacks). + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_CORE_UTIL_ENS_TRAITS +#define MLPACK_CORE_UTIL_ENS_TRAITS + +#include "sfinae_utility.hpp" + +namespace mlpack { + +HAS_MEM_FUNC(Optimize, HasOptimize); + +// Utility struct for IsEnsOptimizer (below). By default returns false. (See +// specialization below for the real logic.) +template +struct IsEnsOptimizerInternal +{ + constexpr static bool value = false; +}; + +/** + * If the given argument is an ensmallen-compatible optimizer for the given + * MatType and FunctionType (e.g. if it has an `Optimize()` function that can + * handle the given `FunctionType` and `MatType`), then the `value` member will + * be `true`. + */ +template +struct IsEnsOptimizer +{ + // Dispatch to IsEnsOptimizerInternal, which will filter out when + // OptimizerType is a non-class. + constexpr static bool value = IsEnsOptimizerInternal< + OptimizerType, + FunctionType, + MatType, + std::is_class::value + >::value; +}; + +// Logic for detecting ensmallen optimizers when OptimizerType is a class. +template +struct IsEnsOptimizerInternal +{ + // If OptimizerType is a reference type, then forming the types below will + // fail. So we need to strip the reference (and the const for good measure). + typedef typename std::remove_cv< + typename std::remove_reference::type>::type + SafeOptimizerType; + + using OptimizeElemReturnForm = + typename MatType::elem_type(SafeOptimizerType::*)(FunctionType&, + MatType&); + + using OptimizeVoidReturnForm = + void(SafeOptimizerType::*)(FunctionType&, MatType&); + + constexpr static bool value = + HasOptimize::value || + HasOptimize::value; +}; + +/** + * If the given template parameter pack could all be valid ensmallen callbacks + * (i.e. they are all classes), then the `value` member will be `true`. + * + * This is not a perfect check, but it is sufficient to differentiate from + * hyperparameters. + */ +template +struct IsEnsCallbackTypes; + +template +struct IsEnsCallbackTypes +{ + constexpr static bool value = std::is_class::value && + IsEnsCallbackTypes::value; +}; + +template<> +struct IsEnsCallbackTypes<> +{ + constexpr static bool value = true; +}; + +} // namespace mlpack + +#endif diff --git a/src/mlpack/core/util/sfinae_utility.hpp b/src/mlpack/core/util/sfinae_utility.hpp index 71df637529..848fa515b7 100644 --- a/src/mlpack/core/util/sfinae_utility.hpp +++ b/src/mlpack/core/util/sfinae_utility.hpp @@ -132,7 +132,7 @@ struct NAME \ < \ T, \ sig, \ - std::integral_constant::value> \ + std::integral_constant::value> \ > : std::true_type {}; /** From b62119b240d5c952f54feebe625bd70baf75f610 Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Thu, 14 Dec 2023 10:23:33 -0500 Subject: [PATCH 55/91] Revert "Don't use deprecated function." (Pushed this to the wrong branch.) This reverts commit 0b02c1aae1424cee518850361ef7e3c39246f656. --- .../methods/logistic_regression/logistic_regression_main.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mlpack/methods/logistic_regression/logistic_regression_main.cpp b/src/mlpack/methods/logistic_regression/logistic_regression_main.cpp index 3ec5310868..8566980e80 100644 --- a/src/mlpack/methods/logistic_regression/logistic_regression_main.cpp +++ b/src/mlpack/methods/logistic_regression/logistic_regression_main.cpp @@ -386,7 +386,7 @@ void BINDING_FUNCTION(util::Params& params, util::Timers& timers) Log::Info << "Calculating class probabilities of points in '" << params.GetPrintable("test") << "'." << endl; arma::mat probabilities; - model->Classify(testSet, predictions, probabilities); + model->Classify(testSet, probabilities); if (params.Has("probabilities")) params.Get("probabilities") = std::move(probabilities); From f3cc2328d1525b4e919932eb5a3603c4ab760508 Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Thu, 14 Dec 2023 10:34:58 -0500 Subject: [PATCH 56/91] Fix some minor issues with constructors. --- doc/user/methods/linear_svm.md | 31 +++++++++++++++++++++---------- 1 file changed, 21 insertions(+), 10 deletions(-) diff --git a/doc/user/methods/linear_svm.md b/doc/user/methods/linear_svm.md index e4a7c81064..cead0b493e 100644 --- a/doc/user/methods/linear_svm.md +++ b/doc/user/methods/linear_svm.md @@ -48,12 +48,21 @@ std::cout << arma::accu(predictions == 1) << " test points classified as class " ### Constructors - * `svm = NaiveBayesClassifier(dimensionality)` - * `svm = NaiveBayesClassifier(dimensionality, numClasses, lambda=0.0001, delta=1.0, fitIntercept=false)` - - Initialize the model without training. + * `svm = LinearSVM()` + * `svm = LinearSVM(lambda=0.0001, delta=1.0, fitIntercept=false)` + - Initialize the parameters of the model without training. - You will need to call [`Train()`](#training) later to train the model before calling [`Classify()`](#classification). +--- + + * `svm = LinearSVM(dimensionality, numClasses, lambda=0.0001, delta=1.0, fitIntercept=false)` + - Initialize the model without training, to default weights. + - [`Classify()`](#classification) can immediately be called and + `Parameters()` returns valid weights, but the model is otherwise untrained. + - The model should be trained with [`Train()`](#training) before calling + [`Classify()`](#classification). + --- * `svm = LinearSVM(data, labels, numClasses, lambda=0.0001, delta=1.0, fitIntercept=false, [callbacks...])` @@ -106,16 +115,18 @@ calling `Train()`: If training is not done as part of the constructor call, it can be done with the `Train()` function: - * `svm.Train(data, labels, numClasses, lambda=0.0001, delta=1.0, fitIntercept=false, [callbacks...]) - - Train model on the given data, optionally specifying ensmallen callbacks - for use during optimization. + * `svm.Train(data, labels, numClasses, [callbacks...])` + * `svm.Train(data, labels, numClasses, optimizer, [callbacks...])` + - Train model without changing any hyperparameters, optionally using a custom + ensmallen optimizer and specifying callbacks for use during optimization. --- - * `svm.Train(data, labels, numClasses, optimizer, lambda=0.0001, delta=1.0, - fitIntercept=false, [callbacks...]) - - Train model on the given data with a custom ensmallen optimizer, optionally - specifying callbacks for use during optimization. + * `svm.Train(data, labels, numClasses, lambda=0.0001, delta=1.0, fitIntercept=false, [callbacks...]) + * `svm.Train(data, labels, numClasses, optimizer, lambda=0.0001, delta=1.0, fitIntercept=false, [callbacks...]) + - Train model on the given data, specifying hyperparameters and optionally + also a custom ensmallen optimizer and callbacks for use during + optimization. --- From c09c3fec4e211302ab571010f570daf94e778081 Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Thu, 14 Dec 2023 10:35:23 -0500 Subject: [PATCH 57/91] Fix bug in IsEnsCallbacks: detect references correctly too. --- src/mlpack/core/util/ens_traits.hpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/mlpack/core/util/ens_traits.hpp b/src/mlpack/core/util/ens_traits.hpp index e8789f052f..79c1635c3c 100644 --- a/src/mlpack/core/util/ens_traits.hpp +++ b/src/mlpack/core/util/ens_traits.hpp @@ -84,8 +84,10 @@ struct IsEnsCallbackTypes; template struct IsEnsCallbackTypes { - constexpr static bool value = std::is_class::value && - IsEnsCallbackTypes::value; + constexpr static bool value = + std::is_class::type + >::type>::value && IsEnsCallbackTypes::value; }; template<> From 5afa57f4e1389b1681dd3450522ff3d0bf75d354 Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Thu, 14 Dec 2023 10:36:05 -0500 Subject: [PATCH 58/91] Adapt functions to match new interfaces that are documented. --- src/mlpack/methods/linear_svm/linear_svm.hpp | 299 ++++++++++++++---- .../methods/linear_svm/linear_svm_impl.hpp | 232 +++++++++++--- .../methods/linear_svm/linear_svm_main.cpp | 2 +- src/mlpack/tests/linear_svm_test.cpp | 39 ++- 4 files changed, 456 insertions(+), 116 deletions(-) diff --git a/src/mlpack/methods/linear_svm/linear_svm.hpp b/src/mlpack/methods/linear_svm/linear_svm.hpp index cb2e431b43..0d70c1ec2c 100644 --- a/src/mlpack/methods/linear_svm/linear_svm.hpp +++ b/src/mlpack/methods/linear_svm/linear_svm.hpp @@ -79,12 +79,66 @@ template class LinearSVM { public: + /** + * Initialize the Linear SVM without performing training. Default + * value of lambda is 0.0001. Be sure to use Train() before calling + * Classify() or ComputeAccuracy(), otherwise the results may be meaningless. + * + * @param lambda L2-regularization constant. + * @param delta Margin of difference between correct class and other classes. + * @param fitIntercept add intercept term or not. + */ + LinearSVM(const double lambda = 0.0001, + const double delta = 1.0, + const bool fitIntercept = false); + + /** + * Initialize the Linear SVM without performing training. Default + * value of lambda is 0.0001. Be sure to use Train() before calling + * Classify() or ComputeAccuracy(), otherwise the results may be meaningless. + * + * @param inputSize Size of the input feature vector. + * @param numClasses Number of classes for classification. + * @param lambda L2-regularization constant. + * @param delta Margin of difference between correct class and other classes. + * @param fitIntercept add intercept term or not. + */ + LinearSVM(const size_t inputSize, + const size_t numClasses, + const double lambda = 0.0001, + const double delta = 1.0, + const bool fitIntercept = false); + + /** + * Initialize the Linear SVM without performing training. Default + * value of lambda is 0.0001. Be sure to use Train() before calling + * Classify() or ComputeAccuracy(), otherwise the results may be meaningless. + * + * This constructor is deprecated; if you want to specify a custom optimizer, + * use the constructor with the optimizer after `numClasses`. The constructor + * will be removed in mlpack 5.0.0. + * + * @param numClasses Number of classes for classification. + * @param lambda L2-regularization constant. + * @param delta Margin of difference between correct class and other classes. + * @param fitIntercept add intercept term or not. + */ + mlpack_deprecated + LinearSVM(const size_t numClasses, + const double lambda = 0.0001, + const double delta = 1.0, + const bool fitIntercept = false); + /** * Construct the LinearSVM class with the provided data and labels. * This will train the model. Optionally, the parameter 'lambda' can be * passed, which controls the amount of L2-regularization in the objective * function. By default, the model takes a small value. * + * This constructor is deprecated; if you want to specify a custom optimizer, + * use the constructor with the optimizer after `numClasses`. The constructor + * will be removed in mlpack 5.0.0. + * * @tparam OptimizerType Desired differentiable separable optimizer * @tparam CallbackTypes Types of callback functions. * @param data Input training features. Each column associate with one sample @@ -97,7 +151,15 @@ class LinearSVM * @param callbacks Callback functions. * See https://www.ensmallen.org/docs.html#callback-documentation. */ - template + template , arma::mat + >::value>::type, + typename = typename std::enable_if::value>::type> + mlpack_deprecated /** To be removed in mlpack 5.0.0. **/ LinearSVM(const MatType& data, const arma::Row& labels, const size_t numClasses, @@ -113,6 +175,10 @@ class LinearSVM * passed, which controls the amount of L2-regularization in the objective * function. By default, the model takes a small value. * + * This constructor is deprecated; if you want to specify a custom optimizer, + * use the constructor with the optimizer after `numClasses`. The constructor + * will be removed in mlpack 5.0.0. + * * @tparam OptimizerType Desired differentiable separable optimizer * @param data Input training features. Each column associate with one sample * @param labels Labels associated with the feature data. @@ -122,45 +188,190 @@ class LinearSVM * @param fitIntercept add intercept term or not. * @param optimizer Desired optimizer. */ - template + template , arma::mat + >::value>::type> + mlpack_deprecated /** To be removed in mlpack 5.0.0. **/ LinearSVM(const MatType& data, const arma::Row& labels, - const size_t numClasses = 2, + const size_t numClasses, + const double lambda, + const double delta, + const bool fitIntercept, + OptimizerType optimizer); + + /** + * Construct the LinearSVM class with the provided data and labels. + * This will train the model. Optionally, hyperparameters can be passed. + * + * @tparam OptimizerType Desired differentiable separable optimizer + * @param data Input training features. Each column associate with one sample + * @param labels Labels associated with the feature data. + * @param numClasses Number of classes for classification. + * @param lambda L2-regularization constant. + * @param delta Margin of difference between correct class and other classes. + * @param fitIntercept add intercept term or not. + * @param callbacks Callback Functions. + * See https://www.ensmallen.org/docs.html#callback-documentation. + */ + template::value>::type> + LinearSVM(const MatType& data, + const arma::Row& labels, + const size_t numClasses, const double lambda = 0.0001, const double delta = 1.0, const bool fitIntercept = false, - OptimizerType optimizer = OptimizerType()); + CallbackTypes&&... callbacks); /** - * Initialize the Linear SVM without performing training. Default - * value of lambda is 0.0001. Be sure to use Train() before calling - * Classify() or ComputeAccuracy(), otherwise the results may be meaningless. + * Construct the LinearSVM class with the provided data and labels. This will + * train the model with the given custom optimizer. Optionally, + * hyperparameters can be passed. * - * @param inputSize Size of the input feature vector. + * @tparam OptimizerType Desired differentiable separable optimizer + * @param data Input training features. Each column associate with one sample + * @param labels Labels associated with the feature data. * @param numClasses Number of classes for classification. * @param lambda L2-regularization constant. * @param delta Margin of difference between correct class and other classes. * @param fitIntercept add intercept term or not. + * @param callbacks Callback Functions. + * See https://www.ensmallen.org/docs.html#callback-documentation. */ - LinearSVM(const size_t inputSize, - const size_t numClasses = 0, + template::value>::type> + LinearSVM(const MatType& data, + const arma::Row& labels, + const size_t numClasses, + OptimizerType& optimizer, const double lambda = 0.0001, const double delta = 1.0, - const bool fitIntercept = false); + const bool fitIntercept = false, + CallbackTypes&&... callbacks); + /** - * Initialize the Linear SVM without performing training. Default - * value of lambda is 0.0001. Be sure to use Train() before calling - * Classify() or ComputeAccuracy(), otherwise the results may be meaningless. + * Train the Linear SVM with the given training data. * + * @param data Input training features. Each column associate with one sample. + * @param labels Labels associated with the feature data. * @param numClasses Number of classes for classification. * @param lambda L2-regularization constant. * @param delta Margin of difference between correct class and other classes. * @param fitIntercept add intercept term or not. + * @param callbacks Callback Functions. + * See https://www.ensmallen.org/docs.html#callback-documentation. + * @return Objective value of the final point. */ - LinearSVM(const size_t numClasses = 0, - const double lambda = 0.0001, - const double delta = 1.0, - const bool fitIntercept = false); + // Many overloads are necessary because we don't yet require C++17, which + // would give std::optional support. + template::value>::type> + double Train(const MatType& data, + const arma::Row& labels, + const size_t numClasses, + CallbackTypes&&... callbackTypes); + + double Train(const MatType& data, + const arma::Row& labels, + const size_t numClasses, + const double lambda); + + double Train(const MatType& data, + const arma::Row& labels, + const size_t numClasses, + const double lambda, + const double delta); + + template ::value>::type> + double Train(const MatType& data, + const arma::Row& labels, + const size_t numClasses, + const double lambda, + const double delta, + const bool fitIntercept, + CallbackTypes&&... callbacks); + + /** + * Train the Linear SVM with the given training data using a custom ensmallen + * optimizer. + * + * @tparam OptimizerType Desired optimizer. + * @param data Input training features. Each column associate with one sample. + * @param labels Labels associated with the feature data. + * @param numClasses Number of classes for classification. + * @param optimizer Desired optimizer. + * @param lambda L2-regularization constant. + * @param delta Margin of difference between correct class and other classes. + * @param fitIntercept add intercept term or not. + * @param callbacks Callback Functions. + * See https://www.ensmallen.org/docs.html#callback-documentation. + * @return Objective value of the final point. + */ + // Many overloads are necessary because we don't yet require C++17, which + // would give std::optional support. + template , arma::mat + >::value>::type, + typename = typename std::enable_if::value>::type> + double Train(const MatType& data, + const arma::Row& labels, + const size_t numClasses, + OptimizerType optimizer, + CallbackTypes&&... callbacks); + + template , arma::mat + >::value>::type> + double Train(const MatType& data, + const arma::Row& labels, + const size_t numClasses, + OptimizerType optimizer, + const double lambda); + + template , arma::mat + >::value>::type> + double Train(const MatType& data, + const arma::Row& labels, + const size_t numClasses, + OptimizerType optimizer, + const double lambda, + const double delta); + + template , arma::mat + >::value>::type, + typename = typename std::enable_if::value>::type> + double Train(const MatType& data, + const arma::Row& labels, + const size_t numClasses, + OptimizerType optimizer, + const double lambda, + const double delta, + const bool fitIntercept, + CallbackTypes&&... callbacks); /** * Classify the given points, returning the predicted labels for each point. @@ -195,6 +406,7 @@ class LinearSVM * @param data Matrix of data points to be classified. * @param scores Class scores for each point. */ + mlpack_deprecated void Classify(const MatType& data, arma::mat& scores) const; @@ -209,6 +421,20 @@ class LinearSVM template size_t Classify(const VecType& point) const; + /** + * Classify the given point. The predicted class label is stored in `label`, + * and the probability of each class is stored in `probabilities`.. + * + * @param point Point to be classified. + * @param label size_t to store predicted label into. + * @param probabilities Vector to store class probabilities into. + * @return Predicted class label of the point. + */ + template + void Classify(const VecType& point, + size_t& label, + arma::rowvec& probabilities) const; + /** * Computes accuracy of the learned model given the feature data and the * labels associated with each data point. Predictions are made using the @@ -221,43 +447,6 @@ class LinearSVM double ComputeAccuracy(const MatType& testData, const arma::Row& testLabels) const; - /** - * Train the Linear SVM with the given training data. - * - * @tparam OptimizerType Desired optimizer. - * @tparam CallbackTypes Types of Callback Functions. - * @param data Input training features. Each column associate with one sample. - * @param labels Labels associated with the feature data. - * @param numClasses Number of classes for classification. - * @param optimizer Desired optimizer. - * @param callbacks Callback Functions. - * See https://www.ensmallen.org/docs.html#callback-documentation. - * @return Objective value of the final point. - */ - template - double Train(const MatType& data, - const arma::Row& labels, - const size_t numClasses, - OptimizerType optimizer, - CallbackTypes&&... callbacks); - - /** - * Train the Linear SVM with the given training data. - * - * @tparam OptimizerType Desired optimizer. - * @param data Input training features. Each column associate with one sample. - * @param labels Labels associated with the feature data. - * @param numClasses Number of classes for classification. - * @param optimizer Desired optimizer. - * @return Objective value of the final point. - */ - template - double Train(const MatType& data, - const arma::Row& labels, - const size_t numClasses = 2, - OptimizerType optimizer = OptimizerType()); - - //! Sets the number of classes. size_t& NumClasses() { return numClasses; } //! Gets the number of classes. diff --git a/src/mlpack/methods/linear_svm/linear_svm_impl.hpp b/src/mlpack/methods/linear_svm/linear_svm_impl.hpp index dda79e87d4..f5b1f02b54 100644 --- a/src/mlpack/methods/linear_svm/linear_svm_impl.hpp +++ b/src/mlpack/methods/linear_svm/linear_svm_impl.hpp @@ -18,7 +18,52 @@ namespace mlpack { template -template +LinearSVM::LinearSVM( + const double lambda, + const double delta, + const bool fitIntercept) : + lambda(lambda), + delta(delta), + fitIntercept(fitIntercept) +{ + // No training to do here. +} + +template +mlpack_deprecated /** Will be removed in mlpack 5.0.0. **/ +LinearSVM::LinearSVM( + const size_t inputSize, + const size_t numClasses, + const double lambda, + const double delta, + const bool fitIntercept) : + numClasses(numClasses), + lambda(lambda), + delta(delta), + fitIntercept(fitIntercept) +{ + LinearSVMFunction::InitializeWeights(parameters, inputSize, + numClasses, fitIntercept); +} + +template +mlpack_deprecated /** Will be removed in mlpack 5.0.0. **/ +LinearSVM::LinearSVM( + const size_t numClasses, + const double lambda, + const double delta, + const bool fitIntercept) : + numClasses(numClasses), + lambda(lambda), + delta(delta), + fitIntercept(fitIntercept) +{ + // No training to do here. +} + +template +template +mlpack_deprecated /** Will be removed in mlpack 5.0.0. **/ LinearSVM::LinearSVM( const MatType& data, const arma::Row& labels, @@ -33,11 +78,13 @@ LinearSVM::LinearSVM( delta(delta), fitIntercept(fitIntercept) { - Train(data, labels, numClasses, optimizer, callbacks...); + Train(data, labels, numClasses, optimizer, + std::forward(callbacks)...); } template -template +template +mlpack_deprecated /** Will be removed in mlpack 5.0.0. **/ LinearSVM::LinearSVM( const MatType& data, const arma::Row& labels, @@ -55,37 +102,99 @@ LinearSVM::LinearSVM( } template +template LinearSVM::LinearSVM( - const size_t inputSize, + const MatType& data, + const arma::Row& labels, const size_t numClasses, const double lambda, const double delta, - const bool fitIntercept) : + const bool fitIntercept, + CallbackTypes&&... callbacks) : numClasses(numClasses), lambda(lambda), delta(delta), fitIntercept(fitIntercept) { - LinearSVMFunction::InitializeWeights(parameters, inputSize, - numClasses, fitIntercept); + // By default we use L-BFGS. + ens::L_BFGS optimizer; + Train(data, labels, numClasses, optimizer, + std::forward(callbacks)...); } template +template LinearSVM::LinearSVM( + const MatType& data, + const arma::Row& labels, const size_t numClasses, + OptimizerType& optimizer, const double lambda, const double delta, - const bool fitIntercept) : + const bool fitIntercept, + CallbackTypes&&... callbacks) : numClasses(numClasses), lambda(lambda), delta(delta), fitIntercept(fitIntercept) { - // No training to do here. + Train(data, labels, numClasses, optimizer, + std::forward(callbacks)...); } template -template +template +double LinearSVM::Train( + const MatType& data, + const arma::Row& labels, + const size_t numClasses, + CallbackTypes&&... callbacks) +{ + return Train(data, labels, numClasses, this->lambda, this->delta, + this->fitIntercept, std::forward(callbacks)...); +} + +template +double LinearSVM::Train( + const MatType& data, + const arma::Row& labels, + const size_t numClasses, + const double lambda) +{ + return Train(data, labels, numClasses, lambda, this->delta, + this->fitIntercept); +} + +template +double LinearSVM::Train( + const MatType& data, + const arma::Row& labels, + const size_t numClasses, + const double lambda, + const double delta) +{ + return Train(data, labels, numClasses, lambda, delta, this->fitIntercept); +} + +template +template +double LinearSVM::Train( + const MatType& data, + const arma::Row& labels, + const size_t numClasses, + const double lambda, + const double delta, + const bool fitIntercept, + CallbackTypes&&... callbacks) +{ + // By default, train with L-BFGS. + ens::L_BFGS lbfgs; + return Train(data, labels, numClasses, lbfgs, lambda, delta, fitIntercept, + std::forward(callbacks)...); +} + +template +template double LinearSVM::Train( const MatType& data, const arma::Row& labels, @@ -93,6 +202,54 @@ double LinearSVM::Train( OptimizerType optimizer, CallbackTypes&&... callbacks) { + return Train(data, labels, numClasses, optimizer, this->lambda, this->delta, + this->fitIntercept, std::forward(callbacks)...); +} + +template +template +double LinearSVM::Train( + const MatType& data, + const arma::Row& labels, + const size_t numClasses, + OptimizerType optimizer, + const double lambda) +{ + return Train(data, labels, numClasses, optimizer, lambda, this->delta, + this->fitIntercept); +} + +template +template +double LinearSVM::Train( + const MatType& data, + const arma::Row& labels, + const size_t numClasses, + OptimizerType optimizer, + const double lambda, + const double delta) +{ + return Train(data, labels, numClasses, optimizer, lambda, delta, + this->fitIntercept); +} + +template +template +double LinearSVM::Train( + const MatType& data, + const arma::Row& labels, + const size_t numClasses, + OptimizerType optimizer, + const double lambda, + const double delta, + const bool fitIntercept, + CallbackTypes&&... callbacks) +{ + this->numClasses = numClasses; + this->lambda = lambda; + this->delta = delta; + this->fitIntercept = fitIntercept; + if (numClasses <= 1) { throw std::invalid_argument("LinearSVM dataset has 0 number of classes!"); @@ -112,33 +269,6 @@ double LinearSVM::Train( return out; } -template -template -double LinearSVM::Train( - const MatType& data, - const arma::Row& labels, - const size_t numClasses, - OptimizerType optimizer) -{ - if (numClasses <= 1) - { - throw std::invalid_argument("LinearSVM dataset has 0 number of classes!"); - } - - LinearSVMFunction svm(data, labels, numClasses, lambda, delta, - fitIntercept); - if (parameters.is_empty()) - parameters = svm.InitialPoint(); - - // Train the model. - const double out = optimizer.Optimize(svm, parameters); - - Log::Info << "LinearSVM::LinearSVM(): final objective of " - << "trained model is " << out << "." << std::endl; - - return out; -} - template void LinearSVM::Classify( const MatType& data, @@ -154,7 +284,18 @@ void LinearSVM::Classify( arma::Row& labels, arma::mat& scores) const { - Classify(data, scores); + util::CheckSameDimensionality(data, FeatureSize(), "LinearSVM::Classify()"); + + if (fitIntercept) + { + scores = parameters.rows(0, parameters.n_rows - 2).t() * data + + arma::repmat(parameters.row(parameters.n_rows - 1).t(), 1, + data.n_cols); + } + else + { + scores = parameters.t() * data; + } // Prepare necessary data. labels.zeros(data.n_cols); @@ -164,6 +305,7 @@ void LinearSVM::Classify( } template +mlpack_deprecated void LinearSVM::Classify( const MatType& data, arma::mat& scores) const @@ -191,6 +333,18 @@ size_t LinearSVM::Classify(const VecType& point) const return size_t(label(0)); } +template +template +void LinearSVM::Classify( + const VecType& point, + size_t& label, + arma::rowvec& probabilities) const +{ + arma::Row labelRow(1); + Classify(point, labelRow, probabilities); + label = labelRow[0]; +} + template double LinearSVM::ComputeAccuracy( const MatType& testData, @@ -208,7 +362,7 @@ double LinearSVM::ComputeAccuracy( count++; // Return the accuracy. - return (double) count / labels.n_elem; + return (double) 100.0 * count / labels.n_elem; } } // namespace mlpack diff --git a/src/mlpack/methods/linear_svm/linear_svm_main.cpp b/src/mlpack/methods/linear_svm/linear_svm_main.cpp index 5aa0477a5e..2ec7dceb4c 100644 --- a/src/mlpack/methods/linear_svm/linear_svm_main.cpp +++ b/src/mlpack/methods/linear_svm/linear_svm_main.cpp @@ -405,7 +405,7 @@ void BINDING_FUNCTION(util::Params& params, util::Timers& timers) Log::Info << "Calculating class probabilities of points in " << testOutput << "." << endl; arma::mat probabilities; - model->svm.Classify(testSet, probabilities); + model->svm.Classify(testSet, predictions, probabilities); params.Get("probabilities") = std::move(probabilities); } diff --git a/src/mlpack/tests/linear_svm_test.cpp b/src/mlpack/tests/linear_svm_test.cpp index ba9054d2b3..26baf5cc5b 100644 --- a/src/mlpack/tests/linear_svm_test.cpp +++ b/src/mlpack/tests/linear_svm_test.cpp @@ -486,7 +486,7 @@ TEST_CASE("LinearSVMLBFGSSimpleTest", "[LinearSVMTest]") // Compare training accuracy to 1. const double acc = lsvm.ComputeAccuracy(dataset, labels); - REQUIRE(acc == Approx(1.0).epsilon(0.005)); + REQUIRE(acc == Approx(100.0).epsilon(0.005)); } /** @@ -514,12 +514,12 @@ TEST_CASE("LinearSVMGradientDescentSimpleTest", "[LinearSVMTest]") // Create a linear svm object using custom gradient descent optimizer. ens::GradientDescent optimizer(stepSize, maxIterations, tolerance); - LinearSVM lsvm(dataset, labels, numClasses, lambda, - delta, false, optimizer); + LinearSVM lsvm(dataset, labels, numClasses, optimizer, lambda, + delta, false); // Compare training accuracy to 1. const double acc = lsvm.ComputeAccuracy(dataset, labels); - REQUIRE(acc == Approx(1.0).epsilon(0.005)); + REQUIRE(acc == Approx(100.0).epsilon(0.005)); } /** @@ -632,8 +632,7 @@ TEST_CASE("LinearSVMFitIntercept", "[LinearSVMTest]") } // Now train a svm object on it. - LinearSVM svm(data, labels, numClasses, lambda, - delta, true, ens::L_BFGS()); + LinearSVM svm(data, labels, numClasses, lambda, delta, true); // Ensure that the error is close to zero. const double acc = svm.ComputeAccuracy(data, labels); @@ -775,12 +774,12 @@ TEST_CASE("LinearSVMPSGDSimpleTest", "[LinearSVMTest]") ens::ParallelSGD optimizer(0, std::ceil((float) dataset.n_cols / omp_get_max_threads()), 1e-5, true, decayPolicy); - LinearSVM lsvm(dataset, labels, numClasses, lambda, - delta, false, optimizer); + LinearSVM lsvm(dataset, labels, numClasses, optimizer, lambda, + delta, false); // Compare training accuracy to 1. const double acc = lsvm.ComputeAccuracy(dataset, labels); - REQUIRE(acc == Approx(1.0).epsilon(1e-2)); + REQUIRE(acc == Approx(100.0).epsilon(1e-2)); } /** @@ -824,13 +823,13 @@ TEST_CASE("LinearSVMParallelSGDTwoClasses", "[LinearSVMTest]") // Train linear svm object using Parallel SGD optimizer. // The threadShareSize is chosen such that each function gets optimized. - ens::ParallelSGD optimizer(0, + ens::ParallelSGD optimizer(100000, std::ceil((float) data.n_cols / omp_get_max_threads()), 1e-5, true, decayPolicy); - LinearSVM lsvm(data, labels, numClasses, lambda, - delta, false, optimizer); + LinearSVM lsvm(data, labels, numClasses, optimizer, lambda, + delta, false); - // Compare training accuracy to 1. + // Compare training accuracy to 100. const double acc = lsvm.ComputeAccuracy(data, labels); // Create test dataset. @@ -849,8 +848,8 @@ TEST_CASE("LinearSVMParallelSGDTwoClasses", "[LinearSVMTest]") const double testAcc = lsvm.ComputeAccuracy(data, labels); // Larger tolerance is sometimes needed. - if (testAcc == Approx(1.0).epsilon(0.02) && - acc == Approx(1.0).epsilon(0.02)) + if (testAcc == Approx(100.0).epsilon(0.02) && + acc == Approx(100.0).epsilon(0.02)) { success = true; break; @@ -876,10 +875,8 @@ TEST_CASE("LinearSVMSparseLBFGSTest", "[LinearSVMTest]") for (size_t i = 0; i < 800; ++i) labels[i] = RandInt(0, 2); - LinearSVM lr(denseDataset, labels, 2, 0.3, 1, - false, ens::L_BFGS()); - LinearSVM lrSparse(dataset, labels, 2, 0.3, 1, - false, ens::L_BFGS()); + LinearSVM lr(denseDataset, labels, 2, 0.3, 1, false); + LinearSVM lrSparse(dataset, labels, 2, 0.3, 1, false); REQUIRE(lr.Parameters().n_elem == lrSparse.Parameters().n_elem); for (size_t i = 0; i < lr.Parameters().n_elem; ++i) @@ -1188,8 +1185,8 @@ TEST_CASE("LinearSVMCallbackTest", "[LinearSVMTest]") CallbackTestFunction cb; ens::L_BFGS opt; - LinearSVM lsvm(dataset, labels, numClasses, lambda, - delta, false, opt, cb); + LinearSVM lsvm(dataset, labels, numClasses, lambda, delta, + false, cb); REQUIRE(cb.calledEndOptimization == true); } From f255930ba45830359d33e0275e0d6754165e32d7 Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Thu, 14 Dec 2023 13:52:20 -0500 Subject: [PATCH 59/91] Test new overloads of Train() and constructor; adapt for sparse tests; fix some bugs with SGD-type optimizers. --- doc/user/methods/linear_svm.md | 3 +- src/mlpack/core/util/arma_traits.hpp | 96 +++++ src/mlpack/methods/linear_svm/linear_svm.hpp | 277 +++++++------- .../linear_svm/linear_svm_function.hpp | 64 ++-- .../linear_svm/linear_svm_function_impl.hpp | 162 ++++---- .../methods/linear_svm/linear_svm_impl.hpp | 193 +++++----- src/mlpack/tests/linear_svm_test.cpp | 345 ++++++++++++++++-- 7 files changed, 774 insertions(+), 366 deletions(-) diff --git a/doc/user/methods/linear_svm.md b/doc/user/methods/linear_svm.md index cead0b493e..dd41da3a30 100644 --- a/doc/user/methods/linear_svm.md +++ b/doc/user/methods/linear_svm.md @@ -199,7 +199,8 @@ can be used to make class predictions for new data. `svm.Parameters()` represents the bias parameters for each class. * `svm.FeatureSize()` will return the number of features in the model. This is - equivalent to `data.n_rows` when the model was trained. + equivalent to `data.n_rows` when the model was trained. The output is only + valid if the model has been trained. * `svm.ComputeAccuracy(data, labels)` will return the accuracy of the model on the given `data` with the given `labels`. The returned accuracy is between 0 diff --git a/src/mlpack/core/util/arma_traits.hpp b/src/mlpack/core/util/arma_traits.hpp index a755ce9efa..35ed3e035a 100644 --- a/src/mlpack/core/util/arma_traits.hpp +++ b/src/mlpack/core/util/arma_traits.hpp @@ -111,4 +111,100 @@ struct IsVector > #endif +// Get the row vector type corresponding to a given MatType. + +template +struct GetRowType +{ + typedef MatType type; // Not sure... +}; + +template +struct GetRowType> +{ + typedef arma::Row type; +}; + +template +struct GetRowType> +{ + typedef arma::SpRow type; +}; + +// Get the column vector type corresponding to a given MatType. + +template +struct GetColType +{ + typedef MatType type; // Not sure... +}; + +template +struct GetColType> +{ + typedef arma::Col type; +}; + +template +struct GetColType> +{ + typedef arma::SpCol type; +}; + +// Get the dense row vector type corresponding to a given MatType. + +template +struct GetDenseRowType +{ + typedef typename GetRowType::type type; +}; + +template +struct GetDenseRowType> +{ + typedef arma::Row type; +}; + +// Get the dense column vector type corresponding to a given MatType. + +template +struct GetDenseColType +{ + typedef typename GetColType::type type; +}; + +template +struct GetDenseColType> +{ + typedef arma::Col type; +}; + +// Get the dense matrix type corresponding to a given MatType. + +template +struct GetDenseMatType +{ + typedef MatType type; +}; + +template +struct GetDenseMatType> +{ + typedef arma::Mat type; +}; + +// Get the sparse matrix type corresponding to a given MatType. + +template +struct GetSparseMatType +{ + typedef arma::SpMat type; +}; + +template +struct GetSparseMatType> +{ + typedef arma::SpMat type; +}; + #endif diff --git a/src/mlpack/methods/linear_svm/linear_svm.hpp b/src/mlpack/methods/linear_svm/linear_svm.hpp index 0d70c1ec2c..9dbf1a0c70 100644 --- a/src/mlpack/methods/linear_svm/linear_svm.hpp +++ b/src/mlpack/methods/linear_svm/linear_svm.hpp @@ -73,62 +73,44 @@ namespace mlpack { * lsvm.Classify(test_data, predictions); * @endcode * - * @tparam MatType Type of data matrix. + * @tparam ModelMatType Type of data matrix to use to store model parameters. */ -template +template class LinearSVM { public: - /** - * Initialize the Linear SVM without performing training. Default - * value of lambda is 0.0001. Be sure to use Train() before calling - * Classify() or ComputeAccuracy(), otherwise the results may be meaningless. - * - * @param lambda L2-regularization constant. - * @param delta Margin of difference between correct class and other classes. - * @param fitIntercept add intercept term or not. - */ - LinearSVM(const double lambda = 0.0001, - const double delta = 1.0, - const bool fitIntercept = false); + typedef typename ModelMatType::elem_type ElemType; + typedef typename GetDenseMatType::type DenseMatType; + typedef typename GetDenseColType::type DenseColType; /** * Initialize the Linear SVM without performing training. Default * value of lambda is 0.0001. Be sure to use Train() before calling * Classify() or ComputeAccuracy(), otherwise the results may be meaningless. * - * @param inputSize Size of the input feature vector. + * @param lambda L2-regularization constant. + * @param delta Margin of difference between correct class and other classes. + * @param fitIntercept add intercept term or not. + */ + LinearSVM(); + + /** + * Initialize the Linear SVM without performing training. Default + * value of lambda is 0.0001. Be sure to use Train() before calling + * Classify() or ComputeAccuracy(), otherwise the results may be meaningless. + * + * @param dimensionality Size of the input feature vector. * @param numClasses Number of classes for classification. * @param lambda L2-regularization constant. * @param delta Margin of difference between correct class and other classes. * @param fitIntercept add intercept term or not. */ - LinearSVM(const size_t inputSize, + LinearSVM(const size_t dimensionality, const size_t numClasses, const double lambda = 0.0001, const double delta = 1.0, const bool fitIntercept = false); - /** - * Initialize the Linear SVM without performing training. Default - * value of lambda is 0.0001. Be sure to use Train() before calling - * Classify() or ComputeAccuracy(), otherwise the results may be meaningless. - * - * This constructor is deprecated; if you want to specify a custom optimizer, - * use the constructor with the optimizer after `numClasses`. The constructor - * will be removed in mlpack 5.0.0. - * - * @param numClasses Number of classes for classification. - * @param lambda L2-regularization constant. - * @param delta Margin of difference between correct class and other classes. - * @param fitIntercept add intercept term or not. - */ - mlpack_deprecated - LinearSVM(const size_t numClasses, - const double lambda = 0.0001, - const double delta = 1.0, - const bool fitIntercept = false); - /** * Construct the LinearSVM class with the provided data and labels. * This will train the model. Optionally, the parameter 'lambda' can be @@ -151,16 +133,18 @@ class LinearSVM * @param callbacks Callback functions. * See https://www.ensmallen.org/docs.html#callback-documentation. */ - template , arma::mat - >::value>::type, - typename = typename std::enable_if::value>::type> + template, + ModelMatType + >::value>::type, + typename = typename std::enable_if::value>::type> mlpack_deprecated /** To be removed in mlpack 5.0.0. **/ - LinearSVM(const MatType& data, + LinearSVM(const arma::mat& data, const arma::Row& labels, const size_t numClasses, const double lambda, @@ -188,12 +172,14 @@ class LinearSVM * @param fitIntercept add intercept term or not. * @param optimizer Desired optimizer. */ - template , arma::mat - >::value>::type> + template, + ModelMatType + >::value>::type> mlpack_deprecated /** To be removed in mlpack 5.0.0. **/ - LinearSVM(const MatType& data, + LinearSVM(const arma::mat& data, const arma::Row& labels, const size_t numClasses, const double lambda, @@ -215,7 +201,8 @@ class LinearSVM * @param callbacks Callback Functions. * See https://www.ensmallen.org/docs.html#callback-documentation. */ - template::value>::type> @@ -242,7 +229,8 @@ class LinearSVM * @param callbacks Callback Functions. * See https://www.ensmallen.org/docs.html#callback-documentation. */ - template::value>::type> - double Train(const MatType& data, - const arma::Row& labels, - const size_t numClasses, - CallbackTypes&&... callbackTypes); + ElemType Train(const MatType& data, + const arma::Row& labels, + const size_t numClasses, + CallbackTypes&&... callbackTypes); - double Train(const MatType& data, - const arma::Row& labels, - const size_t numClasses, - const double lambda); + template + ElemType Train(const MatType& data, + const arma::Row& labels, + const size_t numClasses, + const double lambda); - double Train(const MatType& data, - const arma::Row& labels, - const size_t numClasses, - const double lambda, - const double delta); + template + ElemType Train(const MatType& data, + const arma::Row& labels, + const size_t numClasses, + const double lambda, + const double delta); - template ::value>::type> - double Train(const MatType& data, - const arma::Row& labels, - const size_t numClasses, - const double lambda, - const double delta, - const bool fitIntercept, - CallbackTypes&&... callbacks); + template::value>::type> + ElemType Train(const MatType& data, + const arma::Row& labels, + const size_t numClasses, + const double lambda, + const double delta, + const bool fitIntercept, + CallbackTypes&&... callbacks); /** * Train the Linear SVM with the given training data using a custom ensmallen @@ -321,57 +313,69 @@ class LinearSVM */ // Many overloads are necessary because we don't yet require C++17, which // would give std::optional support. - template , arma::mat - >::value>::type, - typename = typename std::enable_if::value>::type> - double Train(const MatType& data, - const arma::Row& labels, - const size_t numClasses, - OptimizerType optimizer, - CallbackTypes&&... callbacks); + template, + ModelMatType + >::value>::type, + typename = typename std::enable_if::value>::type> + ElemType Train(const MatType& data, + const arma::Row& labels, + const size_t numClasses, + OptimizerType optimizer, + CallbackTypes&&... callbacks); - template , arma::mat - >::value>::type> - double Train(const MatType& data, - const arma::Row& labels, - const size_t numClasses, - OptimizerType optimizer, - const double lambda); + template, + ModelMatType + >::value>::type> + ElemType Train(const MatType& data, + const arma::Row& labels, + const size_t numClasses, + OptimizerType optimizer, + const double lambda); - template , arma::mat - >::value>::type> - double Train(const MatType& data, - const arma::Row& labels, - const size_t numClasses, - OptimizerType optimizer, - const double lambda, - const double delta); + template, + ModelMatType + >::value>::type> + ElemType Train(const MatType& data, + const arma::Row& labels, + const size_t numClasses, + OptimizerType optimizer, + const double lambda, + const double delta); - template , arma::mat - >::value>::type, - typename = typename std::enable_if::value>::type> - double Train(const MatType& data, - const arma::Row& labels, - const size_t numClasses, - OptimizerType optimizer, - const double lambda, - const double delta, - const bool fitIntercept, - CallbackTypes&&... callbacks); + template, + ModelMatType + >::value>::type, + typename = typename std::enable_if::value>::type> + ElemType Train(const MatType& data, + const arma::Row& labels, + const size_t numClasses, + OptimizerType optimizer, + const double lambda, + const double delta, + const bool fitIntercept, + CallbackTypes&&... callbacks); /** * Classify the given points, returning the predicted labels for each point. @@ -382,6 +386,7 @@ class LinearSVM * @param data Set of points to classify. * @param labels Predicted labels for each point. */ + template void Classify(const MatType& data, arma::Row& labels) const; @@ -396,9 +401,10 @@ class LinearSVM * @param labels Predicted labels for each point. * @param scores Class probabilities for each point. */ + template void Classify(const MatType& data, arma::Row& labels, - arma::mat& scores) const; + DenseMatType& scores) const; /** * Classify the given points, returning class scores for each point. @@ -407,7 +413,7 @@ class LinearSVM * @param scores Class scores for each point. */ mlpack_deprecated - void Classify(const MatType& data, + void Classify(const arma::mat& data, arma::mat& scores) const; /** @@ -433,7 +439,7 @@ class LinearSVM template void Classify(const VecType& point, size_t& label, - arma::rowvec& probabilities) const; + DenseColType& probabilities) const; /** * Computes accuracy of the learned model given the feature data and the @@ -444,6 +450,7 @@ class LinearSVM * @param testLabels Vector of labels associated with the data. * @return Accuracy of the model. */ + template double ComputeAccuracy(const MatType& testData, const arma::Row& testLabels) const; @@ -466,9 +473,9 @@ class LinearSVM bool& FitIntercept() { return fitIntercept; } //! Set the model parameters. - arma::mat& Parameters() { return parameters; } + ModelMatType& Parameters() { return parameters; } //! Get the model parameters. - const arma::mat& Parameters() const { return parameters; } + const ModelMatType& Parameters() const { return parameters; } //! Gets the features size of the training data size_t FeatureSize() const @@ -479,17 +486,11 @@ class LinearSVM * Serialize the LinearSVM model. */ template - void serialize(Archive& ar, const uint32_t /* version */) - { - ar(CEREAL_NVP(parameters)); - ar(CEREAL_NVP(numClasses)); - ar(CEREAL_NVP(lambda)); - ar(CEREAL_NVP(fitIntercept)); - } + void serialize(Archive& ar, const uint32_t /* version */); private: //! Parameters after optimization. - arma::mat parameters; + ModelMatType parameters; //! Number of classes. size_t numClasses; //! L2-Regularization constant. diff --git a/src/mlpack/methods/linear_svm/linear_svm_function.hpp b/src/mlpack/methods/linear_svm/linear_svm_function.hpp index 8274efb9fd..05a5910aa5 100644 --- a/src/mlpack/methods/linear_svm/linear_svm_function.hpp +++ b/src/mlpack/methods/linear_svm/linear_svm_function.hpp @@ -23,10 +23,16 @@ namespace mlpack { * This is used by various ensmallen optimizers to train the linear * SVM model. */ -template +template class LinearSVMFunction { public: + typedef typename ParametersType::elem_type ElemType; + typedef typename GetDenseMatType::type DenseMatType; + typedef typename GetSparseMatType::type SparseMatType; + typedef typename GetDenseColType::type DenseColType; + typedef typename GetDenseRowType::type DenseRowType; + /** * Construct the Linear SVM objective function with given parameters. * @@ -58,7 +64,7 @@ class LinearSVMFunction * @param numClasses Number of classes for classification. * @param fitIntercept If true, an intercept is fitted. */ - static void InitializeWeights(arma::mat& weights, + static void InitializeWeights(ParametersType& weights, const size_t featureSize, const size_t numClasses, const bool fitIntercept = false); @@ -67,10 +73,10 @@ class LinearSVMFunction * Constructs the ground truth label matrix with the passed labels. * * @param labels Labels associated with the training data. - * @param groundTruth Pointer to arma::mat which stores the computed matrix. + * @param groundTruth Reference to sparse matrix to stores the result. */ void GetGroundTruthMatrix(const arma::Row& labels, - arma::sp_mat& groundTruth); + SparseMatType& groundTruth) const; /** * Evaluate the hinge loss function for all the datapoints @@ -78,7 +84,7 @@ class LinearSVMFunction * @param parameters The parameters of the SVM. * @return The value of the loss function for the entire dataset. */ - double Evaluate(const arma::mat& parameters); + ElemType Evaluate(const ParametersType& parameters) const; /** * Evaluate the hinge loss function on the specified datapoints. @@ -89,9 +95,9 @@ class LinearSVMFunction * @param batchSize Size of batch to process. * @return The value of the loss function for the given parameters. */ - double Evaluate(const arma::mat& parameters, - const size_t firstId, - const size_t batchSize = 1); + ElemType Evaluate(const ParametersType& parameters, + const size_t firstId, + const size_t batchSize = 1) const; /** * Evaluate the gradient of the hinge loss function following the @@ -101,9 +107,9 @@ class LinearSVMFunction * @param parameters The parameters of the SVM. * @param gradient Linear matrix to output the gradient into. */ - template - void Gradient(const arma::mat& parameters, - GradType& gradient); + template + void Gradient(const ParametersType& parameters, + GradType& gradient) const; /** * Evaluate the gradient of the hinge loss function, following @@ -115,11 +121,11 @@ class LinearSVMFunction * @param gradient Linear matrix to output the gradient into. * @param batchSize Size of the batch to process. */ - template - void Gradient(const arma::mat& parameters, + template + void Gradient(const ParametersType& parameters, const size_t firstId, GradType& gradient, - const size_t batchSize = 1); + const size_t batchSize = 1) const; /** * Evaluate the gradient of the hinge loss function, following @@ -132,9 +138,9 @@ class LinearSVMFunction * @param gradient Linear matrix to output the gradient into. * @return The value of the loss function at the given parameters. */ - template - double EvaluateWithGradient(const arma::mat& parameters, - GradType& gradient) const; + template + ElemType EvaluateWithGradient(const ParametersType& parameters, + GradType& gradient) const; /** * Evaluate the gradient of the hinge loss function, following @@ -150,21 +156,21 @@ class LinearSVMFunction * @param batchSize Size of the batch to process. * @return The value of the loss function at the given parameters. */ - template - double EvaluateWithGradient(const arma::mat& parameters, - const size_t firstId, - GradType& gradient, - const size_t batchSize = 1) const; + template + ElemType EvaluateWithGradient(const ParametersType& parameters, + const size_t firstId, + GradType& gradient, + const size_t batchSize = 1) const; //! Return the initial point for the optimization. - const arma::mat& InitialPoint() const { return initialPoint; } + const ParametersType& InitialPoint() const { return initialPoint; } //! Modify the initial point for the optimization. - arma::mat& InitialPoint() { return initialPoint; } + ParametersType& InitialPoint() { return initialPoint; } //! Get the dataset. - const arma::sp_mat& Dataset() const { return dataset; } + const SparseMatType& Dataset() const { return dataset; } //! Modify the dataset. - arma::sp_mat& Dataset() { return dataset; } + SparseMatType& Dataset() { return dataset; } //! Sets the regularization parameter. double& Lambda() { return lambda; } @@ -179,12 +185,12 @@ class LinearSVMFunction private: //! The initial point, from which to start the optimization. - arma::mat initialPoint; + ParametersType initialPoint; //! Label matrix for provided data - arma::sp_mat groundTruth; + SparseMatType groundTruth; - //! The datapoints for training. + //! The datapoints for training. This will be an alias until Shuffle(). MatType dataset; //! Number of Classes. diff --git a/src/mlpack/methods/linear_svm/linear_svm_function_impl.hpp b/src/mlpack/methods/linear_svm/linear_svm_function_impl.hpp index 8c36f24597..277593b06f 100644 --- a/src/mlpack/methods/linear_svm/linear_svm_function_impl.hpp +++ b/src/mlpack/methods/linear_svm/linear_svm_function_impl.hpp @@ -22,8 +22,8 @@ namespace mlpack { -template -LinearSVMFunction::LinearSVMFunction( +template +LinearSVMFunction::LinearSVMFunction( const MatType& dataset, const arma::Row& labels, const size_t numClasses, @@ -48,9 +48,9 @@ LinearSVMFunction::LinearSVMFunction( * normal distribution. The weights cannot be initialized to zero, as that will * lead to each class output being the same. */ -template -void LinearSVMFunction::InitializeWeights( - arma::mat &weights, +template +void LinearSVMFunction::InitializeWeights( + ParametersType& weights, const size_t featureSize, const size_t numClasses, const bool fitIntercept) @@ -69,10 +69,11 @@ void LinearSVMFunction::InitializeWeights( * labels. The output is in the form of a matrix, which leads to simpler * calculations in the Evaluate() and Gradient() methods. */ -template -void LinearSVMFunction::GetGroundTruthMatrix( +template +void LinearSVMFunction::GetGroundTruthMatrix( const arma::Row& labels, - arma::sp_mat& groundTruth) + typename LinearSVMFunction::SparseMatType& + groundTruth) const { // Calculate the ground truth matrix according to the labels passed. The // ground truth matrix is a matrix of dimensions 'numClasses * numExamples', @@ -95,19 +96,19 @@ void LinearSVMFunction::GetGroundTruthMatrix( } // All entries are '1'. - arma::vec values; + DenseColType values; values.ones(labels.n_elem); // Calculate the matrix. - groundTruth = arma::sp_mat(rowPointers, colPointers, values, numClasses, - labels.n_elem); + groundTruth = SparseMatType(rowPointers, colPointers, values, numClasses, + labels.n_elem); } /** * Shuffle the data. */ -template -void LinearSVMFunction::Shuffle() +template +void LinearSVMFunction::Shuffle() { // Determine new ordering. arma::uvec ordering = arma::shuffle(arma::linspace(0, @@ -124,26 +125,27 @@ void LinearSVMFunction::Shuffle() reverseOrdering[ordering[i]] = i; arma::umat newLocations(2, groundTruth.n_nonzero); - arma::vec values(groundTruth.n_nonzero); - arma::sp_mat::const_iterator it = groundTruth.begin(); + DenseColType values(groundTruth.n_nonzero); + typename SparseMatType::const_iterator it = groundTruth.begin(); size_t loc = 0; while (it != groundTruth.end()) { - newLocations(0, loc) = reverseOrdering(it.col()); - newLocations(1, loc) = it.row(); + newLocations(0, loc) = it.row(); + newLocations(1, loc) = reverseOrdering(it.col()); values(loc) = (*it); ++it; ++loc; } - groundTruth = arma::sp_mat(newLocations, values, groundTruth.n_rows, - groundTruth.n_cols); + groundTruth = SparseMatType(newLocations, values, groundTruth.n_rows, + groundTruth.n_cols); } -template -double LinearSVMFunction::Evaluate( - const arma::mat& parameters) +template +typename LinearSVMFunction::ElemType +LinearSVMFunction::Evaluate( + const ParametersType& parameters) const { // The objective function is the hinge loss function and it is // calculated over all the training examples. @@ -151,10 +153,10 @@ double LinearSVMFunction::Evaluate( // Calculate the loss and regularization terms. // L_i = Σ_i Σ_m max(0, Δ + (w_m x_i + b_m) - (w_{y_i} x_i + b_{y_i})) // where (m != y_i) - double loss, regularization; + ElemType loss, regularization; // Scores for each class are evaluated. - arma::mat scores; + DenseMatType scores; // Check intercept condition. if (!fitIntercept) @@ -178,7 +180,7 @@ double LinearSVMFunction::Evaluate( // - Adding the margin parameter `delta`. // - Removing the `delta` parameter from correct class label in each // column. - arma::mat margin = scores - (arma::repmat(arma::ones(numClasses).t() + DenseMatType margin = scores - (arma::repmat(arma::ones(numClasses).t() * (scores % groundTruth), numClasses, 1)) + delta - (delta * groundTruth); @@ -191,19 +193,20 @@ double LinearSVMFunction::Evaluate( return loss + regularization; } -template -double LinearSVMFunction::Evaluate( - const arma::mat& parameters, +template +typename LinearSVMFunction::ElemType +LinearSVMFunction::Evaluate( + const ParametersType& parameters, const size_t firstId, - const size_t batchSize) + const size_t batchSize) const { const size_t lastId = firstId + batchSize - 1; // Calculate the loss and regularization terms. - double loss, regularization, cost; + ElemType loss, regularization, cost; // Scores for each class are evaluated. - arma::mat scores; + DenseMatType scores; // Check intercept condition. if (!fitIntercept) @@ -218,7 +221,7 @@ double LinearSVMFunction::Evaluate( dataset.n_cols); } - arma::mat margin = scores - (arma::repmat(arma::ones(numClasses).t() + DenseMatType margin = scores - (arma::repmat(arma::ones(numClasses).t() * (scores % groundTruth.cols(firstId, lastId)), numClasses, 1)) + delta - (delta * groundTruth.cols(firstId, lastId)); @@ -233,11 +236,11 @@ double LinearSVMFunction::Evaluate( return cost; } -template -template -void LinearSVMFunction::Gradient( - const arma::mat& parameters, - GradType& gradient) +template +template +void LinearSVMFunction::Gradient( + const ParametersType& parameters, + GradType& gradient) const { // The objective is to minimize the loss, which is evaluated as the sum // of all the positive elements of `margin` matrix. @@ -245,7 +248,7 @@ void LinearSVMFunction::Gradient( // Also, we need to increase the score of the correct class. // Scores for each class are evaluated. - arma::mat scores; + DenseMatType scores; if (!fitIntercept) { @@ -258,16 +261,16 @@ void LinearSVMFunction::Gradient( dataset.n_cols); } - arma::mat margin = scores - (arma::repmat(arma::ones(numClasses).t() + DenseMatType margin = scores - (arma::repmat(arma::ones(numClasses).t() * (scores % groundTruth), numClasses, 1)) + delta - (delta * groundTruth); // An element of `mask` matrix holds `1` corresponding to // each positive element of `margin` matrix. - arma::mat mask = margin.for_each([](arma::mat::elem_type& val) + DenseMatType mask = margin.for_each([](arma::mat::elem_type& val) { val = (val > 0) ? 1: 0; }); - arma::mat difference = groundTruth + DenseMatType difference = groundTruth % (-arma::repmat(arma::sum(mask), numClasses, 1)) + mask; // The gradient is evaluated as follows: @@ -288,7 +291,7 @@ void LinearSVMFunction::Gradient( gradient.submat(0, 0, parameters.n_rows - 2, parameters.n_cols - 1) = dataset * difference.t(); gradient.row(parameters.n_rows - 1) = - arma::ones(dataset.n_cols) * difference.t(); + arma::ones(dataset.n_cols) * difference.t(); } gradient /= dataset.n_cols; @@ -297,18 +300,18 @@ void LinearSVMFunction::Gradient( gradient += lambda * parameters; } -template -template -void LinearSVMFunction::Gradient( - const arma::mat& parameters, +template +template +void LinearSVMFunction::Gradient( + const ParametersType& parameters, const size_t firstId, GradType& gradient, - const size_t batchSize) + const size_t batchSize) const { const size_t lastId = firstId + batchSize - 1; // Scores for each class are evaluated. - arma::mat scores; + DenseMatType scores; // Check intercept condition. if (!fitIntercept) @@ -322,16 +325,16 @@ void LinearSVMFunction::Gradient( + arma::repmat(parameters.row(dataset.n_rows).t(), 1, batchSize); } - arma::mat margin = scores - (arma::repmat(arma::ones(numClasses).t() + DenseMatType margin = scores - (arma::repmat(arma::ones(numClasses).t() * (scores % groundTruth.cols(firstId, lastId)), numClasses, 1)) + delta - (delta * groundTruth.cols(firstId, lastId)); // For each sample, find the total number of classes where // ( margin > 0 ). - arma::mat mask = margin.for_each([](arma::mat::elem_type& val) + DenseMatType mask = margin.for_each([](arma::mat::elem_type& val) { val = (val > 0) ? 1: 0; }); - arma::mat difference = groundTruth.cols(firstId, lastId) + DenseMatType difference = groundTruth.cols(firstId, lastId) % (-arma::repmat(arma::sum(mask), numClasses, 1)) + mask; // Check intercept condition @@ -345,7 +348,7 @@ void LinearSVMFunction::Gradient( gradient.submat(0, 0, parameters.n_rows - 2, parameters.n_cols - 1) = dataset.cols(firstId, lastId) * difference.t(); gradient.row(parameters.n_rows - 1) = - arma::ones(batchSize) * difference.t(); + arma::ones(batchSize) * difference.t(); } gradient /= batchSize; @@ -354,16 +357,17 @@ void LinearSVMFunction::Gradient( gradient += lambda * parameters; } -template -template -double LinearSVMFunction::EvaluateWithGradient( - const arma::mat& parameters, +template +template +typename LinearSVMFunction::ElemType +LinearSVMFunction::EvaluateWithGradient( + const ParametersType& parameters, GradType& gradient) const { - double loss, regularization, cost; + ElemType loss, regularization, cost; // Scores for each class are evaluated. - arma::mat scores; + DenseMatType scores; if (!fitIntercept) { @@ -376,16 +380,16 @@ double LinearSVMFunction::EvaluateWithGradient( dataset.n_cols); } - arma::mat margin = scores - (arma::repmat(arma::ones(numClasses).t() - * (scores % groundTruth), numClasses, 1)) + delta - - (delta * groundTruth); + DenseMatType margin = scores - (arma::repmat( + arma::ones(numClasses).t() * (scores % groundTruth), + numClasses, 1)) + delta - (delta * groundTruth); // For each sample, find the total number of classes where // ( margin > 0 ). - arma::mat mask = margin.for_each([](arma::mat::elem_type& val) + DenseMatType mask = margin.for_each([](ElemType& val) { val = (val > 0) ? 1: 0; }); - arma::mat difference = groundTruth + DenseMatType difference = groundTruth % (-arma::repmat(arma::sum(mask), numClasses, 1)) + mask; // Check intercept condition @@ -399,7 +403,7 @@ double LinearSVMFunction::EvaluateWithGradient( gradient.submat(0, 0, parameters.n_rows - 2, parameters.n_cols - 1) = dataset * difference.t(); gradient.row(parameters.n_rows - 1) = - arma::ones(dataset.n_cols) * difference.t(); + arma::ones(dataset.n_cols) * difference.t(); } gradient /= dataset.n_cols; @@ -418,10 +422,11 @@ double LinearSVMFunction::EvaluateWithGradient( return cost; } -template -template -double LinearSVMFunction::EvaluateWithGradient( - const arma::mat& parameters, +template +template +typename LinearSVMFunction::ElemType +LinearSVMFunction::EvaluateWithGradient( + const ParametersType& parameters, const size_t firstId, GradType& gradient, const size_t batchSize) const @@ -429,10 +434,10 @@ double LinearSVMFunction::EvaluateWithGradient( const size_t lastId = firstId + batchSize - 1; // Calculate the loss and regularization terms. - double loss, regularization, cost; + ElemType loss, regularization, cost; // Scores for each class are evaluated. - arma::mat scores; + DenseMatType scores; // Check intercept condition. if (!fitIntercept) @@ -443,19 +448,20 @@ double LinearSVMFunction::EvaluateWithGradient( { scores = parameters.rows(0, dataset.n_rows - 1).t() * dataset.cols(firstId, lastId) - + arma::repmat(parameters.row(dataset.n_rows).t(), 1, dataset.n_cols); + + arma::repmat(parameters.row(dataset.n_rows).t(), 1, + (lastId - firstId + 1)); } - arma::mat margin = scores - (arma::repmat(arma::ones(numClasses).t() + DenseMatType margin = scores - (arma::repmat(arma::ones(numClasses).t() * (scores % groundTruth.cols(firstId, lastId)), numClasses, 1)) + delta - (delta * groundTruth.cols(firstId, lastId)); // For each sample, find the total number of classes where // ( margin > 0 ). - arma::mat mask = margin.for_each([](arma::mat::elem_type& val) + DenseMatType mask = margin.for_each([](arma::mat::elem_type& val) { val = (val > 0) ? 1: 0; }); - arma::mat difference = groundTruth.cols(firstId, lastId) + DenseMatType difference = groundTruth.cols(firstId, lastId) % (-arma::repmat(arma::sum(mask), numClasses, 1)) + mask; // Check intercept condition @@ -469,7 +475,7 @@ double LinearSVMFunction::EvaluateWithGradient( gradient.submat(0, 0, parameters.n_rows - 2, parameters.n_cols - 1) = dataset.cols(firstId, lastId) * difference.t(); gradient.row(parameters.n_rows - 1) = - arma::ones(batchSize) * difference.t(); + arma::ones(batchSize) * difference.t(); } gradient /= batchSize; @@ -479,7 +485,7 @@ double LinearSVMFunction::EvaluateWithGradient( gradient += lambda * parameters; // The Hinge Loss Function - loss = arma::accu(arma::clamp(margin.cols(firstId, lastId), 0.0, DBL_MAX)); + loss = arma::accu(arma::clamp(margin, 0.0, DBL_MAX)); loss /= batchSize; // Adding the regularization term. @@ -489,8 +495,8 @@ double LinearSVMFunction::EvaluateWithGradient( return cost; } -template -size_t LinearSVMFunction::NumFunctions() const +template +size_t LinearSVMFunction::NumFunctions() const { // The number of points in the dataset is the number of functions, as this // is a data dependent function. diff --git a/src/mlpack/methods/linear_svm/linear_svm_impl.hpp b/src/mlpack/methods/linear_svm/linear_svm_impl.hpp index f5b1f02b54..3d3865ca1e 100644 --- a/src/mlpack/methods/linear_svm/linear_svm_impl.hpp +++ b/src/mlpack/methods/linear_svm/linear_svm_impl.hpp @@ -17,22 +17,18 @@ namespace mlpack { -template -LinearSVM::LinearSVM( - const double lambda, - const double delta, - const bool fitIntercept) : - lambda(lambda), - delta(delta), - fitIntercept(fitIntercept) +template +LinearSVM::LinearSVM() : + lambda(0.0001), + delta(1.0), + fitIntercept(false) { // No training to do here. } -template -mlpack_deprecated /** Will be removed in mlpack 5.0.0. **/ -LinearSVM::LinearSVM( - const size_t inputSize, +template +LinearSVM::LinearSVM( + const size_t dimensionality, const size_t numClasses, const double lambda, const double delta, @@ -42,30 +38,16 @@ LinearSVM::LinearSVM( delta(delta), fitIntercept(fitIntercept) { - LinearSVMFunction::InitializeWeights(parameters, inputSize, - numClasses, fitIntercept); + LinearSVMFunction::InitializeWeights( + parameters, dimensionality, numClasses, fitIntercept); } -template +template +template mlpack_deprecated /** Will be removed in mlpack 5.0.0. **/ -LinearSVM::LinearSVM( - const size_t numClasses, - const double lambda, - const double delta, - const bool fitIntercept) : - numClasses(numClasses), - lambda(lambda), - delta(delta), - fitIntercept(fitIntercept) -{ - // No training to do here. -} - -template -template -mlpack_deprecated /** Will be removed in mlpack 5.0.0. **/ -LinearSVM::LinearSVM( - const MatType& data, +LinearSVM::LinearSVM( + const arma::mat& data, const arma::Row& labels, const size_t numClasses, const double lambda, @@ -82,11 +64,11 @@ LinearSVM::LinearSVM( std::forward(callbacks)...); } -template -template +template +template mlpack_deprecated /** Will be removed in mlpack 5.0.0. **/ -LinearSVM::LinearSVM( - const MatType& data, +LinearSVM::LinearSVM( + const arma::mat& data, const arma::Row& labels, const size_t numClasses, const double lambda, @@ -101,9 +83,9 @@ LinearSVM::LinearSVM( Train(data, labels, numClasses, optimizer); } -template -template -LinearSVM::LinearSVM( +template +template +LinearSVM::LinearSVM( const MatType& data, const arma::Row& labels, const size_t numClasses, @@ -122,9 +104,12 @@ LinearSVM::LinearSVM( std::forward(callbacks)...); } -template -template -LinearSVM::LinearSVM( +template +template +LinearSVM::LinearSVM( const MatType& data, const arma::Row& labels, const size_t numClasses, @@ -142,9 +127,9 @@ LinearSVM::LinearSVM( std::forward(callbacks)...); } -template -template -double LinearSVM::Train( +template +template +typename LinearSVM::ElemType LinearSVM::Train( const MatType& data, const arma::Row& labels, const size_t numClasses, @@ -154,8 +139,9 @@ double LinearSVM::Train( this->fitIntercept, std::forward(callbacks)...); } -template -double LinearSVM::Train( +template +template +typename LinearSVM::ElemType LinearSVM::Train( const MatType& data, const arma::Row& labels, const size_t numClasses, @@ -165,8 +151,9 @@ double LinearSVM::Train( this->fitIntercept); } -template -double LinearSVM::Train( +template +template +typename LinearSVM::ElemType LinearSVM::Train( const MatType& data, const arma::Row& labels, const size_t numClasses, @@ -176,9 +163,9 @@ double LinearSVM::Train( return Train(data, labels, numClasses, lambda, delta, this->fitIntercept); } -template -template -double LinearSVM::Train( +template +template +typename LinearSVM::ElemType LinearSVM::Train( const MatType& data, const arma::Row& labels, const size_t numClasses, @@ -193,9 +180,12 @@ double LinearSVM::Train( std::forward(callbacks)...); } -template -template -double LinearSVM::Train( +template +template +typename LinearSVM::ElemType LinearSVM::Train( const MatType& data, const arma::Row& labels, const size_t numClasses, @@ -206,9 +196,9 @@ double LinearSVM::Train( this->fitIntercept, std::forward(callbacks)...); } -template -template -double LinearSVM::Train( +template +template +typename LinearSVM::ElemType LinearSVM::Train( const MatType& data, const arma::Row& labels, const size_t numClasses, @@ -219,9 +209,9 @@ double LinearSVM::Train( this->fitIntercept); } -template -template -double LinearSVM::Train( +template +template +typename LinearSVM::ElemType LinearSVM::Train( const MatType& data, const arma::Row& labels, const size_t numClasses, @@ -233,9 +223,12 @@ double LinearSVM::Train( this->fitIntercept); } -template -template -double LinearSVM::Train( +template +template +typename LinearSVM::ElemType LinearSVM::Train( const MatType& data, const arma::Row& labels, const size_t numClasses, @@ -255,9 +248,13 @@ double LinearSVM::Train( throw std::invalid_argument("LinearSVM dataset has 0 number of classes!"); } - LinearSVMFunction svm(data, labels, numClasses, lambda, delta, - fitIntercept); - if (parameters.is_empty()) + LinearSVMFunction svm(data, labels, numClasses, lambda, + delta, fitIntercept); + const bool needNewParameters = (parameters.is_empty() || + (fitIntercept && (parameters.n_rows != data.n_rows + 1)) || + (!fitIntercept && (parameters.n_rows != data.n_rows)) || + (parameters.n_cols != numClasses)); + if (needNewParameters) parameters = svm.InitialPoint(); // Train the model. @@ -269,20 +266,22 @@ double LinearSVM::Train( return out; } -template -void LinearSVM::Classify( +template +template +void LinearSVM::Classify( const MatType& data, arma::Row& labels) const { - arma::mat scores; + DenseMatType scores; Classify(data, labels, scores); } -template -void LinearSVM::Classify( +template +template +void LinearSVM::Classify( const MatType& data, arma::Row& labels, - arma::mat& scores) const + typename LinearSVM::DenseMatType& scores) const { util::CheckSameDimensionality(data, FeatureSize(), "LinearSVM::Classify()"); @@ -304,10 +303,10 @@ void LinearSVM::Classify( arma::index_max(scores)); } -template +template mlpack_deprecated -void LinearSVM::Classify( - const MatType& data, +void LinearSVM::Classify( + const arma::mat& data, arma::mat& scores) const { util::CheckSameDimensionality(data, FeatureSize(), "LinearSVM::Classify()"); @@ -324,29 +323,30 @@ void LinearSVM::Classify( } } -template -template -size_t LinearSVM::Classify(const VecType& point) const +template +template +size_t LinearSVM::Classify(const VecType& point) const { arma::Row label(1); Classify(point, label); return size_t(label(0)); } -template -template -void LinearSVM::Classify( +template +template +void LinearSVM::Classify( const VecType& point, size_t& label, - arma::rowvec& probabilities) const + typename LinearSVM::DenseColType& probabilities) const { arma::Row labelRow(1); Classify(point, labelRow, probabilities); label = labelRow[0]; } -template -double LinearSVM::ComputeAccuracy( +template +template +double LinearSVM::ComputeAccuracy( const MatType& testData, const arma::Row& testLabels) const { @@ -365,6 +365,27 @@ double LinearSVM::ComputeAccuracy( return (double) 100.0 * count / labels.n_elem; } +template +template +void LinearSVM::serialize(Archive& ar, const uint32_t version) +{ + // Old versions used `arma::mat` for the type of `parameters`. + if (cereal::is_loading() && version == 0) + { + arma::mat parametersTmp; + ar(cereal::make_nvp("parameters", parametersTmp)); + parameters = arma::conv_to::from(parametersTmp); + } + else + { + ar(CEREAL_NVP(parameters)); + } + + ar(CEREAL_NVP(numClasses)); + ar(CEREAL_NVP(lambda)); + ar(CEREAL_NVP(fitIntercept)); +} + } // namespace mlpack #endif // MLPACK_METHODS_LINEAR_SVM_LINEAR_SVM_IMPL_HPP diff --git a/src/mlpack/tests/linear_svm_test.cpp b/src/mlpack/tests/linear_svm_test.cpp index 26baf5cc5b..0239c745ef 100644 --- a/src/mlpack/tests/linear_svm_test.cpp +++ b/src/mlpack/tests/linear_svm_test.cpp @@ -862,21 +862,28 @@ TEST_CASE("LinearSVMParallelSGDTwoClasses", "[LinearSVMTest]") #endif /** - * Test sparse and dense linear svm and make sure they both work the + * Test sparse and dense linear svm training and make sure they both work the * same using the L-BFGS optimizer. */ -TEST_CASE("LinearSVMSparseLBFGSTest", "[LinearSVMTest]") +TEMPLATE_TEST_CASE("LinearSVMSparseLBFGSTest", "[LinearSVMTest]", float, double) { + typedef TestType ElemType; + typedef typename arma::SpMat SparseMatType; + typedef typename arma::Mat MatType; + // Create a random dataset. - arma::sp_mat dataset; + SparseMatType dataset; dataset.sprandu(10, 800, 0.3); - arma::mat denseDataset(dataset); + MatType denseDataset(dataset); arma::Row labels(800); for (size_t i = 0; i < 800; ++i) labels[i] = RandInt(0, 2); - LinearSVM lr(denseDataset, labels, 2, 0.3, 1, false); - LinearSVM lrSparse(dataset, labels, 2, 0.3, 1, false); + LinearSVM<> lr(denseDataset, labels, 2, 0.3, 1, false); + LinearSVM<> lrSparse(dataset, labels, 2, 0.3, 1, false); + + // Make the initial points the same. + lrSparse.Parameters() = lr.Parameters(); REQUIRE(lr.Parameters().n_elem == lrSparse.Parameters().n_elem); for (size_t i = 0; i < lr.Parameters().n_elem; ++i) @@ -888,10 +895,15 @@ TEST_CASE("LinearSVMSparseLBFGSTest", "[LinearSVMTest]") /** * Test training of linear svm for multiple classes on a complex gaussian - * dataset using L-BFGS optimizer. + * dataset using L-BFGS optimizer, with different types. */ -TEST_CASE("LinearSVMLBFGSMultipleClasses", "[LinearSVMTest]") +TEMPLATE_TEST_CASE("LinearSVMLBFGSMultipleClasses", "[LinearSVMTest]", float, + double) { + typedef TestType ElemType; + typedef typename arma::Mat MatType; + typedef typename arma::Col VecType; + const size_t points = 1000; const size_t inputSize = 5; const size_t numClasses = 5; @@ -905,7 +917,7 @@ TEST_CASE("LinearSVMLBFGSMultipleClasses", "[LinearSVMTest]") GaussianDistribution g4(arma::vec("4.0 1.0 1.0 2.0 7.0"), identity); GaussianDistribution g5(arma::vec("1.0 0.0 1.0 8.0 3.0"), identity); - arma::mat data(inputSize, points); + MatType data(inputSize, points); arma::Row labels(points); // This loop can be removed when ensmallen PR #136 is merged into a version @@ -918,32 +930,32 @@ TEST_CASE("LinearSVMLBFGSMultipleClasses", "[LinearSVMTest]") { for (size_t i = 0; i < points / 5; ++i) { - data.col(i) = g1.Random(); + data.col(i) = arma::conv_to::from(g1.Random()); labels(i) = 0; } for (size_t i = points / 5; i < (2 * points) / 5; ++i) { - data.col(i) = g2.Random(); + data.col(i) = arma::conv_to::from(g2.Random()); labels(i) = 1; } for (size_t i = (2 * points) / 5; i < (3 * points) / 5; ++i) { - data.col(i) = g3.Random(); + data.col(i) = arma::conv_to::from(g3.Random()); labels(i) = 2; } for (size_t i = (3 * points) / 5; i < (4 * points) / 5; ++i) { - data.col(i) = g4.Random(); + data.col(i) = arma::conv_to::from(g4.Random()); labels(i) = 3; } for (size_t i = (4 * points) / 5; i < points; ++i) { - data.col(i) = g5.Random(); + data.col(i) = arma::conv_to::from(g5.Random()); labels(i) = 4; } // Train linear svm object using L-BFGS optimizer. - LinearSVM lsvm(data, labels, numClasses, lambda); + LinearSVM lsvm(data, labels, numClasses, lambda); // Compare training accuracy to 1. const double acc = lsvm.ComputeAccuracy(data, labels); @@ -953,27 +965,27 @@ TEST_CASE("LinearSVMLBFGSMultipleClasses", "[LinearSVMTest]") // Create test dataset. for (size_t i = 0; i < points / 5; ++i) { - data.col(i) = g1.Random(); + data.col(i) = arma::conv_to::from(g1.Random()); labels(i) = 0; } for (size_t i = points / 5; i < (2 * points) / 5; ++i) { - data.col(i) = g2.Random(); + data.col(i) = arma::conv_to::from(g2.Random()); labels(i) = 1; } for (size_t i = (2 * points) / 5; i < (3 * points) / 5; ++i) { - data.col(i) = g3.Random(); + data.col(i) = arma::conv_to::from(g3.Random()); labels(i) = 2; } for (size_t i = (3 * points) / 5; i < (4 * points) / 5; ++i) { - data.col(i) = g4.Random(); + data.col(i) = arma::conv_to::from(g4.Random()); labels(i) = 3; } for (size_t i = (4 * points) / 5; i < points; ++i) { - data.col(i) = g5.Random(); + data.col(i) = arma::conv_to::from(g5.Random()); labels(i) = 4; } @@ -992,8 +1004,13 @@ TEST_CASE("LinearSVMLBFGSMultipleClasses", "[LinearSVMTest]") /** * Testing single point classification (Classify()). */ -TEST_CASE("LinearSVMClassifySinglePointTest", "[LinearSVMTest]") +TEMPLATE_TEST_CASE("LinearSVMClassifySinglePointTest", "[LinearSVMTest]", float, + double) { + typedef TestType ElemType; + typedef typename arma::Mat MatType; + typedef typename arma::Col VecType; + const size_t points = 500; const size_t inputSize = 5; const size_t numClasses = 5; @@ -1007,70 +1024,79 @@ TEST_CASE("LinearSVMClassifySinglePointTest", "[LinearSVMTest]") GaussianDistribution g4(arma::vec("4.0 1.0 1.0 2.0 7.0"), identity); GaussianDistribution g5(arma::vec("1.0 0.0 1.0 8.0 3.0"), identity); - arma::mat data(inputSize, points); + MatType data(inputSize, points); arma::Row labels(points); for (size_t i = 0; i < points / 5; ++i) { - data.col(i) = g1.Random(); + data.col(i) = arma::conv_to::from(g1.Random()); labels(i) = 0; } for (size_t i = points / 5; i < (2 * points) / 5; ++i) { - data.col(i) = g2.Random(); + data.col(i) = arma::conv_to::from(g2.Random()); labels(i) = 1; } for (size_t i = (2 * points) / 5; i < (3 * points) / 5; ++i) { - data.col(i) = g3.Random(); + data.col(i) = arma::conv_to::from(g3.Random()); labels(i) = 2; } for (size_t i = (3 * points) / 5; i < (4 * points) / 5; ++i) { - data.col(i) = g4.Random(); + data.col(i) = arma::conv_to::from(g4.Random()); labels(i) = 3; } for (size_t i = (4 * points) / 5; i < points; ++i) { - data.col(i) = g5.Random(); + data.col(i) = arma::conv_to::from(g5.Random()); labels(i) = 4; } // Train linear svm object. - LinearSVM lsvm(data, labels, numClasses, lambda); + LinearSVM lsvm(data, labels, numClasses, lambda); // Create test dataset. for (size_t i = 0; i < points / 5; ++i) { - data.col(i) = g1.Random(); + data.col(i) = arma::conv_to::from(g1.Random()); labels(i) = 0; } for (size_t i = points / 5; i < (2 * points) / 5; ++i) { - data.col(i) = g2.Random(); + data.col(i) = arma::conv_to::from(g2.Random()); labels(i) = 1; } for (size_t i = (2 * points) / 5; i < (3 * points) / 5; ++i) { - data.col(i) = g3.Random(); + data.col(i) = arma::conv_to::from(g3.Random()); labels(i) = 2; } for (size_t i = (3 * points) / 5; i < (4 * points) / 5; ++i) { - data.col(i) = g4.Random(); + data.col(i) = arma::conv_to::from(g4.Random()); labels(i) = 3; } for (size_t i = (4 * points) / 5; i < points; ++i) { - data.col(i) = g5.Random(); + data.col(i) = arma::conv_to::from(g5.Random()); labels(i) = 4; } - lsvm.Classify(data, labels); + MatType scores; + lsvm.Classify(data, labels, scores); for (size_t i = 0; i < data.n_cols; ++i) { REQUIRE(lsvm.Classify(data.col(i)) == labels(i)); + + size_t prediction; + VecType scoresVec; + lsvm.Classify(data.col(i), prediction, scoresVec); + + REQUIRE(prediction == labels(i)); + REQUIRE(scoresVec.n_elem == scores.n_rows); + REQUIRE(arma::approx_equal(scoresVec, scores.col(i), "absdiff", 1e-5)); } } @@ -1190,3 +1216,254 @@ TEST_CASE("LinearSVMCallbackTest", "[LinearSVMTest]") REQUIRE(cb.calledEndOptimization == true); } + +// Test all variants of LinearSVM constructors. +TEMPLATE_TEST_CASE("LinearSVMConstructorVariantTest", "[LinearSVMTest]", + arma::fmat, arma::mat) +{ + typedef TestType MatType; + + // Create some random data. The results here do not matter all that much; + // this is more of a test that all constructor variants successfully compile + // and produce models at all. + MatType dataset(10, 800, arma::fill::randu); + arma::Row labels(800); + for (size_t i = 0; i < 800; ++i) + labels[i] = RandInt(0, 2); + + LinearSVM<> lsvm1; + LinearSVM<> lsvm2(10, 2); + LinearSVM<> lsvm3(10, 2, 0.0002, 1.1, true); + LinearSVM<> lsvm4(dataset, labels, 2); + LinearSVM<> lsvm5(dataset, labels, 2, 0.0003, 1.2, true); + LinearSVM<> lsvm6(dataset, labels, 2, 0.0004, 1.3, true, + CallbackTestFunction()); + LinearSVM<> lsvm7(dataset, labels, 2, 0.0005, 1.4, true, + CallbackTestFunction(), ens::TimerStop(1.0)); + + ens::StandardSGD sgd; + LinearSVM<> lsvm8(dataset, labels, 2, sgd); + LinearSVM<> lsvm9(dataset, labels, 2, sgd, 0.0006, 1.5, true); + LinearSVM<> lsvm10(dataset, labels, 2, sgd, 0.0007, 1.6, true, + CallbackTestFunction()); + LinearSVM<> lsvm11(dataset, labels, 2, sgd, 0.0008, 1.7, true, + CallbackTestFunction(), ens::TimerStop(1.0)); + + // Check that the variants that did not train have reasonable values. + REQUIRE(lsvm1.Lambda() == Approx(0.0001)); + REQUIRE(lsvm1.Delta() == Approx(1.0)); + REQUIRE(lsvm1.FitIntercept() == false); + + REQUIRE(lsvm2.Lambda() == Approx(0.0001)); + REQUIRE(lsvm2.Delta() == Approx(1.0)); + REQUIRE(lsvm2.FitIntercept() == false); + REQUIRE(lsvm2.NumClasses() == 2); + + REQUIRE(lsvm3.Lambda() == Approx(0.0002)); + REQUIRE(lsvm3.Delta() == Approx(1.1)); + REQUIRE(lsvm3.FitIntercept() == true); + REQUIRE(lsvm3.NumClasses() == 2); + + // Now check that the variants that trained have reasonable models of the + // right size. + REQUIRE(lsvm4.Lambda() == Approx(0.0001)); + REQUIRE(lsvm4.Delta() == Approx(1.0)); + REQUIRE(lsvm4.FitIntercept() == false); + REQUIRE(lsvm4.FeatureSize() == 10); + REQUIRE(lsvm4.NumClasses() == 2); + + REQUIRE(lsvm5.Lambda() == Approx(0.0003)); + REQUIRE(lsvm5.Delta() == Approx(1.2)); + REQUIRE(lsvm5.FitIntercept() == true); + REQUIRE(lsvm5.FeatureSize() == 10); + REQUIRE(lsvm5.NumClasses() == 2); + + REQUIRE(lsvm6.Lambda() == Approx(0.0004)); + REQUIRE(lsvm6.Delta() == Approx(1.3)); + REQUIRE(lsvm6.FitIntercept() == true); + REQUIRE(lsvm6.FeatureSize() == 10); + REQUIRE(lsvm6.NumClasses() == 2); + + REQUIRE(lsvm7.Lambda() == Approx(0.0005)); + REQUIRE(lsvm7.Delta() == Approx(1.4)); + REQUIRE(lsvm7.FitIntercept() == true); + REQUIRE(lsvm7.FeatureSize() == 10); + REQUIRE(lsvm7.NumClasses() == 2); + + REQUIRE(lsvm8.Lambda() == Approx(0.0001)); + REQUIRE(lsvm8.Delta() == Approx(1.0)); + REQUIRE(lsvm8.FitIntercept() == false); + REQUIRE(lsvm8.FeatureSize() == 10); + REQUIRE(lsvm8.NumClasses() == 2); + + REQUIRE(lsvm9.Lambda() == Approx(0.0006)); + REQUIRE(lsvm9.Delta() == Approx(1.5)); + REQUIRE(lsvm9.FitIntercept() == true); + REQUIRE(lsvm9.FeatureSize() == 10); + REQUIRE(lsvm9.NumClasses() == 2); + + REQUIRE(lsvm10.Lambda() == Approx(0.0007)); + REQUIRE(lsvm10.Delta() == Approx(1.6)); + REQUIRE(lsvm10.FitIntercept() == true); + REQUIRE(lsvm10.FeatureSize() == 10); + REQUIRE(lsvm10.NumClasses() == 2); + + REQUIRE(lsvm11.Lambda() == Approx(0.0008)); + REQUIRE(lsvm11.Delta() == Approx(1.7)); + REQUIRE(lsvm11.FitIntercept() == true); + REQUIRE(lsvm11.FeatureSize() == 10); + REQUIRE(lsvm11.NumClasses() == 2); +} + +// Test all variants of LinearSVM Train() functions. +TEMPLATE_TEST_CASE("LinearSVMTrainVariantTest", "[LinearSVMTest]", arma::fmat, + arma::mat) +{ + typedef TestType MatType; + + // Create some random data. The results here do not matter all that much; + // this is more of a test that all constructor variants successfully compile + // and produce models at all. + MatType dataset(10, 800, arma::fill::randu); + arma::Row labels(800); + for (size_t i = 0; i < 800; ++i) + labels[i] = RandInt(0, 2); + + LinearSVM<> lsvm1(10, 2, 1.2, 0.5, true); + LinearSVM<> lsvm2(10, 2, 1.2, 0.5, true); + LinearSVM<> lsvm3(10, 2, 1.2, 0.5, true); + LinearSVM<> lsvm4(10, 2, 1.2, 0.5, true); + LinearSVM<> lsvm5(10, 2, 1.2, 0.5, true); + LinearSVM<> lsvm6(10, 2, 1.2, 0.5, true); + LinearSVM<> lsvm7(10, 2, 1.2, 0.5, true); + LinearSVM<> lsvm8(10, 2, 1.2, 0.5, true); + LinearSVM<> lsvm9(10, 2, 1.2, 0.5, true); + LinearSVM<> lsvm10(10, 2, 1.2, 0.5, true); + LinearSVM<> lsvm11(10, 2, 1.2, 0.5, true); + LinearSVM<> lsvm12(10, 2, 1.2, 0.5, true); + LinearSVM<> lsvm13(10, 2, 1.2, 0.5, true); + LinearSVM<> lsvm14(10, 2, 1.2, 0.5, true); + LinearSVM<> lsvm15(10, 2, 1.2, 0.5, true); + LinearSVM<> lsvm16(10, 2, 1.2, 0.5, true); + + lsvm1.Train(dataset, labels, 2); + lsvm2.Train(dataset, labels, 2, CallbackTestFunction()); + lsvm3.Train(dataset, labels, 2, CallbackTestFunction(), ens::TimerStop(1.0)); + lsvm4.Train(dataset, labels, 2, 0.0002); + lsvm5.Train(dataset, labels, 2, 0.0003, 1.1); + lsvm6.Train(dataset, labels, 2, 0.0004, 1.2, false); + lsvm7.Train(dataset, labels, 2, 0.0005, 1.3, true, CallbackTestFunction()); + lsvm8.Train(dataset, labels, 2, 0.0006, 1.4, false, CallbackTestFunction(), + ens::TimerStop(1.0)); + + ens::Adam adam; + lsvm9.Train(dataset, labels, 2, adam); + lsvm10.Train(dataset, labels, 2, adam, CallbackTestFunction()); + lsvm11.Train(dataset, labels, 2, adam, CallbackTestFunction(), + ens::TimerStop(1.0)); + lsvm12.Train(dataset, labels, 2, adam, 0.0007); + lsvm13.Train(dataset, labels, 2, adam, 0.0008, 1.5); + lsvm14.Train(dataset, labels, 2, adam, 0.0009, 1.6, false); + lsvm15.Train(dataset, labels, 2, adam, 0.001, 1.7, true, + CallbackTestFunction()); + lsvm16.Train(dataset, labels, 2, adam, 0.0011, 1.8, false, + CallbackTestFunction(), ens::TimerStop(1.0)); + + // Check that all hyperparameters are set as expected, and that the model has + // the correct size. + REQUIRE(lsvm1.Lambda() == Approx(1.2)); + REQUIRE(lsvm1.Delta() == Approx(0.5)); + REQUIRE(lsvm1.FitIntercept() == true); + REQUIRE(lsvm1.FeatureSize() == 10); + REQUIRE(lsvm1.NumClasses() == 2); + + REQUIRE(lsvm2.Lambda() == Approx(1.2)); + REQUIRE(lsvm2.Delta() == Approx(0.5)); + REQUIRE(lsvm2.FitIntercept() == true); + REQUIRE(lsvm2.FeatureSize() == 10); + REQUIRE(lsvm2.NumClasses() == 2); + + REQUIRE(lsvm3.Lambda() == Approx(1.2)); + REQUIRE(lsvm3.Delta() == Approx(0.5)); + REQUIRE(lsvm3.FitIntercept() == true); + REQUIRE(lsvm3.FeatureSize() == 10); + REQUIRE(lsvm3.NumClasses() == 2); + + REQUIRE(lsvm4.Lambda() == Approx(0.0002)); + REQUIRE(lsvm4.Delta() == Approx(0.5)); + REQUIRE(lsvm4.FitIntercept() == true); + REQUIRE(lsvm4.FeatureSize() == 10); + REQUIRE(lsvm4.NumClasses() == 2); + + REQUIRE(lsvm5.Lambda() == Approx(0.0003)); + REQUIRE(lsvm5.Delta() == Approx(1.1)); + REQUIRE(lsvm5.FitIntercept() == true); + REQUIRE(lsvm5.FeatureSize() == 10); + REQUIRE(lsvm5.NumClasses() == 2); + + REQUIRE(lsvm6.Lambda() == Approx(0.0004)); + REQUIRE(lsvm6.Delta() == Approx(1.2)); + REQUIRE(lsvm6.FitIntercept() == false); + REQUIRE(lsvm6.FeatureSize() == 10); + REQUIRE(lsvm6.NumClasses() == 2); + + REQUIRE(lsvm7.Lambda() == Approx(0.0005)); + REQUIRE(lsvm7.Delta() == Approx(1.3)); + REQUIRE(lsvm7.FitIntercept() == true); + REQUIRE(lsvm7.FeatureSize() == 10); + REQUIRE(lsvm7.NumClasses() == 2); + + REQUIRE(lsvm8.Lambda() == Approx(0.0006)); + REQUIRE(lsvm8.Delta() == Approx(1.4)); + REQUIRE(lsvm8.FitIntercept() == false); + REQUIRE(lsvm8.FeatureSize() == 10); + REQUIRE(lsvm8.NumClasses() == 2); + + REQUIRE(lsvm9.Lambda() == Approx(1.2)); + REQUIRE(lsvm9.Delta() == Approx(0.5)); + REQUIRE(lsvm9.FitIntercept() == true); + REQUIRE(lsvm9.FeatureSize() == 10); + REQUIRE(lsvm9.NumClasses() == 2); + + REQUIRE(lsvm10.Lambda() == Approx(1.2)); + REQUIRE(lsvm10.Delta() == Approx(0.5)); + REQUIRE(lsvm10.FitIntercept() == true); + REQUIRE(lsvm10.FeatureSize() == 10); + REQUIRE(lsvm10.NumClasses() == 2); + + REQUIRE(lsvm11.Lambda() == Approx(1.2)); + REQUIRE(lsvm11.Delta() == Approx(0.5)); + REQUIRE(lsvm11.FitIntercept() == true); + REQUIRE(lsvm11.FeatureSize() == 10); + REQUIRE(lsvm11.NumClasses() == 2); + + REQUIRE(lsvm12.Lambda() == Approx(0.0007)); + REQUIRE(lsvm12.Delta() == Approx(0.5)); + REQUIRE(lsvm12.FitIntercept() == true); + REQUIRE(lsvm12.FeatureSize() == 10); + REQUIRE(lsvm12.NumClasses() == 2); + + REQUIRE(lsvm13.Lambda() == Approx(0.0008)); + REQUIRE(lsvm13.Delta() == Approx(1.5)); + REQUIRE(lsvm13.FitIntercept() == true); + REQUIRE(lsvm13.FeatureSize() == 10); + REQUIRE(lsvm13.NumClasses() == 2); + + REQUIRE(lsvm14.Lambda() == Approx(0.0009)); + REQUIRE(lsvm14.Delta() == Approx(1.6)); + REQUIRE(lsvm14.FitIntercept() == false); + REQUIRE(lsvm14.FeatureSize() == 10); + REQUIRE(lsvm14.NumClasses() == 2); + + REQUIRE(lsvm15.Lambda() == Approx(0.001)); + REQUIRE(lsvm15.Delta() == Approx(1.7)); + REQUIRE(lsvm15.FitIntercept() == true); + REQUIRE(lsvm15.FeatureSize() == 10); + REQUIRE(lsvm15.NumClasses() == 2); + + REQUIRE(lsvm16.Lambda() == Approx(0.0011)); + REQUIRE(lsvm16.Delta() == Approx(1.8)); + REQUIRE(lsvm16.FitIntercept() == false); + REQUIRE(lsvm16.FeatureSize() == 10); + REQUIRE(lsvm16.NumClasses() == 2); +} From c357506fdda29c74467e5813dc97e7fb001c79ee Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Thu, 14 Dec 2023 14:06:06 -0500 Subject: [PATCH 60/91] Fix minor bugs in examples. --- doc/user/methods/linear_svm.md | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/doc/user/methods/linear_svm.md b/doc/user/methods/linear_svm.md index dd41da3a30..3eeb638aab 100644 --- a/doc/user/methods/linear_svm.md +++ b/doc/user/methods/linear_svm.md @@ -49,7 +49,6 @@ std::cout << arma::accu(predictions == 1) << " test points classified as class " ### Constructors * `svm = LinearSVM()` - * `svm = LinearSVM(lambda=0.0001, delta=1.0, fitIntercept=false)` - Initialize the parameters of the model without training. - You will need to call [`Train()`](#training) later to train the model before calling [`Classify()`](#classification). @@ -231,7 +230,7 @@ ens::AMSGrad optimizer(0.01 /* step size */, 16 /* batch size */); optimizer.MaxIterations() = 100 * dataset.n_cols; // Allow 100 epochs. // Print a progress bar and an optimization report when training is finished. -svm.Train(dataset, labels, optimizer, ens::ProgressBar(), ens::Report()); +svm.Train(dataset, labels, 2, optimizer, ens::ProgressBar(), ens::Report()); // Now predict on test labels and compute accuracy. @@ -259,12 +258,14 @@ ensmallen callback](https://www.ensmallen.org/docs.html#custom-callbacks): class ModelCheckpoint { public: - ModelCheckpoint(mlpack::LogisticRegression<>& model) : model(model) { } + ModelCheckpoint(mlpack::LinearSVM<>& model) : model(model) { } template bool EndEpoch(OptimizerType& /* optimizer */, - FunctionType& /* function */, const MatType& /* coordinates */, - const size_t epoch, const double /* objective */) + FunctionType& /* function */, + const MatType& /* coordinates */, + const size_t epoch, + const double /* objective */) { const std::string filename = "model-" + std::to_string(epoch) + ".bin"; mlpack::data::Save(filename, "svm", model, true); return false; // Do not terminate the optimization. @@ -285,7 +286,7 @@ mlpack::data::Load("satellite.train.csv", dataset, true); arma::Row labels; mlpack::data::Load("satellite.train.labels.csv", labels, true); -mlpack::LinearSVM lr; +mlpack::LinearSVM svm; // Create AdaDelta optimizer with a small step size and batch size of 1. ens::AdaDelta adaDelta(0.001, 1); @@ -293,7 +294,7 @@ adaDelta.MaxIterations() = 100 * dataset.n_cols; // 100 epochs maximum. // Use the custom callback and an L2 penalty parameter of 0.01, with default // delta and fitting an intercept. -svm.Train(dataset, labels, adaDelta, 0.01, 1.0, true, ModelCheckpoint(lr), +svm.Train(dataset, labels, 2, adaDelta, 0.01, 1.0, true, ModelCheckpoint(svm), ens::ProgressBar()); // Now files like model-1.bin, model-2.bin, etc. should be saved on disk. @@ -330,7 +331,7 @@ std::cout << "The L2 regularization penalty parameter is: " << svm.Lambda() << "." << std::endl; std::cout << "Weights for the first dimension are: " - << svm.Parameters().row(0) << "." << std::endl; + << svm.Parameters().row(0); ``` --- From 4b75c5ad1f875f6d21817c9326ba9b750da67379 Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Thu, 14 Dec 2023 14:11:59 -0500 Subject: [PATCH 61/91] Fix minor problems found with rendered documentation. --- doc/user/methods/linear_svm.md | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/doc/user/methods/linear_svm.md b/doc/user/methods/linear_svm.md index 3eeb638aab..68f3c6e736 100644 --- a/doc/user/methods/linear_svm.md +++ b/doc/user/methods/linear_svm.md @@ -1,9 +1,9 @@ ## `LinearSVM` The `LinearSVM` class implements an L2-regularized support vector machine for -numerical data, with training done using any ensmallen optimizer. The class -offers standard classification functionality. Linear SVM is useful for -multi-class classification (i.e. classes are `0`, `1`, `2`, etc.). +numerical data that can train using any ensmallen optimizer. The class offers +standard classification functionality. Linear SVM is useful for multi-class +classification (i.e. classes are `0`, `1`, `2`, etc.). #### Simple usage example: @@ -94,10 +94,6 @@ std::cout << arma::accu(predictions == 1) << " test points classified as class " | `delta` | `double` | Margin of difference between correct class and other classes. | `1.0` | | `fitIntercept` | `bool` | If `true`, then an intercept term is fitted to the model. | `false` | | `callbacks...` | [any set of ensmallen callbacks](https://www.ensmallen.org/docs.html#callback-documentation) | Optional callbacks for the ensmallen optimizer, such as e.g. `ens::ProgressBar()`, `ens::Report()`, or others. | _(N/A)_ | -As an alternative to passing the `epsilon` parameter, it can be set with the -standalone `Epsilon()` method: `nbc.Epsilon() = eps;` will set the value of -`epsilon` to `eps` for the next time non-incremental `Train()` or `Reset()` is -called. As an alternative to passing `lambda`, `delta`, or `fitIntercept`, these can be set with a standalone method. The following functions can be used before @@ -114,15 +110,15 @@ calling `Train()`: If training is not done as part of the constructor call, it can be done with the `Train()` function: - * `svm.Train(data, labels, numClasses, [callbacks...])` + * `svm.Train(data, labels, numClasses, [callbacks...])` * `svm.Train(data, labels, numClasses, optimizer, [callbacks...])` - Train model without changing any hyperparameters, optionally using a custom ensmallen optimizer and specifying callbacks for use during optimization. --- - * `svm.Train(data, labels, numClasses, lambda=0.0001, delta=1.0, fitIntercept=false, [callbacks...]) - * `svm.Train(data, labels, numClasses, optimizer, lambda=0.0001, delta=1.0, fitIntercept=false, [callbacks...]) + * `svm.Train(data, labels, numClasses, lambda=0.0001, delta=1.0, fitIntercept=false, [callbacks...])` + * `svm.Train(data, labels, numClasses, optimizer, lambda=0.0001, delta=1.0, fitIntercept=false, [callbacks...])` - Train model on the given data, specifying hyperparameters and optionally also a custom ensmallen optimizer and callbacks for use during optimization. @@ -267,7 +263,8 @@ class ModelCheckpoint const size_t epoch, const double /* objective */) { - const std::string filename = "model-" + std::to_string(epoch) + ".bin"; mlpack::data::Save(filename, "svm", model, true); + const std::string filename = "model-" + std::to_string(epoch) + ".bin"; + mlpack::data::Save(filename, "svm", model, true); return false; // Do not terminate the optimization. } @@ -342,7 +339,7 @@ The `LinearSVM` class has one template parameter that can be used to control the element type of the model. The full signature of the class is: ```c++ -NaiveBayesClassifier +LinearSVM ``` `ModelMatType` specifies the type of matrix used for training data and internal From f39c22e91fd4462369e14c5f5eae0d2bedc44dee Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Thu, 14 Dec 2023 14:56:37 -0500 Subject: [PATCH 62/91] Slight relaxation of correlation conditions. --- src/mlpack/methods/lars/lars_impl.hpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/mlpack/methods/lars/lars_impl.hpp b/src/mlpack/methods/lars/lars_impl.hpp index 1e8db7eeb8..6ba607a47c 100644 --- a/src/mlpack/methods/lars/lars_impl.hpp +++ b/src/mlpack/methods/lars/lars_impl.hpp @@ -659,11 +659,10 @@ LARS::Train(const MatType& matX, break; // Floats require a really large tolerance for this condition. - const ElemType tol = (std::is_same::value) ? 1e-10 : 0.01; + const ElemType tol = (std::is_same::value) ? 1e-8 : 0.01; if ((matGram != &matGramInternal) && ((maxActiveCorr - minActiveCorr) / maxActiveCorr) > tol) { - std::cout << ((maxActiveCorr - minActiveCorr) / maxActiveCorr) << "\n"; // Construct the error message to match the user's settings. std::ostringstream oss; oss << "LARS::Train(): correlation conditions violated; check that your " From 9c0ec1d399986fe3eaa50979b1a45e8b513d39c6 Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Thu, 14 Dec 2023 15:58:34 -0500 Subject: [PATCH 63/91] Add first attempt at Bayesian linear regression documentation. --- .../methods/bayesian_linear_regression.md | 331 ++++++++++++++++++ 1 file changed, 331 insertions(+) create mode 100644 doc/user/methods/bayesian_linear_regression.md diff --git a/doc/user/methods/bayesian_linear_regression.md b/doc/user/methods/bayesian_linear_regression.md new file mode 100644 index 0000000000..237e37c4b5 --- /dev/null +++ b/doc/user/methods/bayesian_linear_regression.md @@ -0,0 +1,331 @@ +## `BayesianLinearRegression` + +The `BayesianLinearRegression` class implements a Bayesian ridge regression +model for numerical data that optimally tunes the regularization strength to the +given data. The class offers configurable functionality and template parameters +to control the data type used for storing the model. + +#### Simple usage example: + +```c++ +// Train a linear regression model on random data and make predictions. + +// All data and responses are uniform random; this uses 10 dimensional data. +// Replace with a data::Load() call or similar for a real application. +arma::mat dataset(10, 1000, arma::fill::randu); // 1000 points. +arma::rowvec responses = arma::randn(1000); +arma::mat testDataset(10, 500, arma::fill::randu); // 500 test points. + +mlpack::BayesianLinearRegression blr; // Step 1: create model. +blr.Train(dataset, responses); // Step 2: train model. +arma::rowvec predictions; +blr.Predict(testDataset, predictions); // Step 3: use model to predict. + +// Print some information about the test predictions. +std::cout << arma::accu(predictions > 0.6) << " test points predicted to have" + << " responses greater than 0.6." << std::endl; +std::cout << arma::accu(predictions < 0) << " test points predicted to have " + << "negative responses." << std::endl; +``` +

More examples...

+ +#### Quick links: + + * [Constructors](#constructors): create `BayesianLinearRegression` objects. + * [`Train()`](#training): train model. + * [`Predict()`](#prediction): predict with a trained model. + * [Other functionality](#other-functionality) for loading, saving, and + inspecting. + * [Examples](#simple-examples) of simple usage and links to detailed example + projects. + * [Template parameters](#advanced-functionality-different-element-types) for + using different element types for a model. + +#### See also: + + * [mlpack regression techniques](#mlpack_regression_techniques) + * [`LinearRegression`](linear_regression.md) + * [`LARS`](lars.md) + * [Bayesian linear regression on Wikipedia](https://en.wikipedia.org/wiki/Bayesian_linear_regression) + +### Constructors + + * `blr = BayesianLinearRegression(centerData=true, scaleData=false, maxIterations=50, tolerance=1e-4)` + - Initialize the model without training. + - You will need to call [`Train()`](#training) later to train the model + before calling [`Predict()`](#prediction). + +--- + + + * `blr = BayesianLinearRegression(data, responses)` + * `blr = BayesianLinearRegression(data, responses, centerData=true, scaleData=false, maxIterations=50, tolerance=1e-4)` + - Train model on the given data. + +--- + +#### Constructor Parameters: + + + +| **name** | **type** | **description** | **default** | +|----------|----------|-----------------|-------------| +| `data` | [`arma::mat`](../matrices.md) | [Column-major](../matrices.md) training matrix. | _(N/A)_ | +| `responses` | [`arma::rowvec`](../matrices.md) | Training responses (e.g. values to predict). Should have length `data.n_cols`. | _(N/A)_ | +| `centerData` | `bool` | Whether to center the data before learning. | `true` | +| `scaleData` | `bool` | Whether to scale the data to unit variance before learning. | `false` | +| `maxIterations` | `size_t` | Maximum number of iterations for convergence. | `50` | +| `tolerance` | `double` | Tolerance for convergence of the model. | `1e-4` | + +As an alternative to passing `centerData`, `scaleData`, `maxIterations`, or +`tolerance`, they can each be set or accessed with standalone methods: + + * `blr.CenterData() = centerData;` will set whether to center the data before + learning to `centerData`. + * `blr.ScaleData() = scaleData;` will set whether to scale the data to unit + variance before learning to `scaleData`. + * `blr.MaxIterations() = maxIterations;` will set the maximum number of + iterations to `maxIterations`. + * `blr.Tolerance() = tolerance;` will set the tolerance for convergence to + `tolerance`. + +### Training + +If training is not done as part of the constructor call, it can be done with the +`Train()` function: + + * `blr.Train(data, responses, centerData=true, scaleData=false, maxIterations=50, tolerance=1e-4)` + - Train model on the given data. + +--- + +Types of each argument are the same as in the table for constructors +[above](#constructor-parameters). + +***Notes:*** + + * Training is not incremental. A second call to `Train()` will retrain the + model from scratch. + + * `Train()` returns the root mean squared error (RMSE) of the model on the + training set as a `double`. + +### Prediction + +Once a `LinearRegression` model is trained, the `Predict()` member function +can be used to make predictions for new data. + + * `double predictedValue = blr.Predict(point)` + - ***(Single-point)*** + - Make a prediction for a single point, returning the predicted value. + +--- + * `blr.Predict(point, prediction, stddev)` + - ***(Single-point)*** + - Make a prediction for a single point, storing the predicted value in + `prediction` and the standard deviation of the prediction in `stddev`. + +--- + + * `blr.Predict(data, predictions)` + - ***(Multi-point)*** + - Make predictions for a set of points. + - The prediction for data point `i` can be accessed with `predictions[i]`. + +--- + + * `blr.Predict(data, predictions, stddevs)` + - ***(Multi-point)*** + - Make predictions for a set of points and compute standard deviations of + predictions. + - The prediction for data point `i` can be accessed with `predictions[i]`. + - The standard deviation of the prediction for data point `i` can be accessed + with `stddevs[i]`. + +--- + +#### Prediction Parameters: + +| **usage** | **name** | **type** | **description** | +|-----------|----------|----------|-----------------| +| _single-point_ | `point` | [`arma::vec`](../matrices.md) | Single point for prediction. | +| _single-point_ | `prediction` | `double&` | `double` to store predicted value into. | +| _single-point_ | `stddev` | `double&` | `double` to store standard deviation of predicted value into. | +|||| +| _multi-point_ | `data` | [`arma::mat`](../matrices.md) | Set of [column-major](../matrices.md) points for classification. | +| _multi-point_ | `predictions` | [`arma::rowvec&`](../matrices.md) | Vector of `double`s to store predictions into. Will be set to length `data.n_cols`. | +| _multi-point_ | `stddevs` | [`arma::rowvec&`](../matrices.md) | Vector of `double`s to store standard deviations of predictions into. Will be set to length `data.n_cols`. | + +### Other Functionality + + + + * A `BayesianLinearRegression` model can be serialized with + [`data::Save()`](../formats.md) and [`data::Load()`](../formats.md). + + * After training is complete, the following methods can be used to inspect the + model: + - `blr.Omega()` returns the weights of the trained model as an + `const arma::vec&` of length `data.n_rows`. The weight for the `i`th + dimension can be accessed with `blr.Omega()[i]`. + + - `blr.Alpha()` returns the precision (or inverse variance) of the Gaussian + prior of the model as a `double`. + + - `blr.Beta()` returns the precision (or inverse variance) of the model as a + `double`. + + - `blr.Variance()` returns the estimated variance as a `double`. + + - `blr.DataOffset()` returns a `const arma::vec&` containing the mean values + of the training data in each dimension. The vector has length + `data.n_rows`. The result is only meaningful if `centerData` is `true`. + + - `blr.DataScale()` returns a `const arma::vec&` containing the standard + deviations of the training data in each dimension. The vector has length + `data.n_rows`. The result is only meaningful if `scaleData` is `true`. + + - `blr.ResponsesOffset()` returns the mean value of the training responses as + a `double`. This is the intercept of the model. + + * `blr.RMSE(data, responses)` returns a `double` containing the RMSE (root mean + squared error) of the model on the given `data` and `responses`. + +### Simple Examples + +See also the [simple usage example](#simple-usage-example) for a trivial usage +of the `BayesianLinearRegression` class. + +--- + +Train a Bayesian linear regression model in the constructor on weighted data, +compute the RMSE with `RMSE()`, and save the model. + +```c++ +// See https://datasets.mlpack.org/admission_predict.csv. +arma::mat data; +mlpack::data::Load("admission_predict.csv", data, true); + +// See https://datasets.mlpack.org/admission_predict.responses.csv. +arma::rowvec responses; +mlpack::data::Load("admission_predict.responses.csv", responses, true); + +// Generate random instance weights for each point, in the range 0.5 to 1.5. +arma::rowvec weights(data.n_cols, arma::fill::randu); +weights += 0.5; + +// Train Bayesian linear regression model. The data will be both centered and +// scaled to have unit variance. +mlpack::BayesianLinearRegression blr(data, responses, true, true); + +// Now compute the RMSE on the training set. +std::cout << "RMSE on the training set: " << blr.RMSE(data, responses) + << "." << std::endl; + +// Finally, save the model with the name "blr". +mlpack::data::Save("blr_model.bin", "blr", blr, true); +``` + +--- + +Load a saved Bayesian linear regression model and print some information about +it, then make some predictions individually for random points. + +``` +mlpack::BayesianLinearRegression blr; + +// Load the model named "blr" from "lr_model.bin". +mlpack::data::Load("blr_model.bin", "blr", blr, true); + +// Print some information about the model. +const size_t dimensionality = blr.Omega().n_elem; +if (dimensionality == 0) +{ + std::cout << "The model in `blr_model.bin` has not been trained." + << std::endl; + return 0; +} + +std::cout << "Information on the BayesianLinearRegression model in " + << "'blr_model.bin':" << std::endl; +std::cout << " - Data was centered when training: " + << (blr.CenterData() ? std::string("yes") : std::string("no")) << "." + << std::endl; +std::cout << " - Data was scaled to unit variance when training: " + << (blr.ScaleData() ? std::string("yes") : std::string("no")) << "." + << std::endl; +std::cout << " - Model intercept: " << blr.ResponsesOffset() << "." + << std::endl; +std::cout << " - Precision of Gaussian prior: " << blr.Alpha() << "." + << std::endl; +std::cout << " - Precision of model: " << blr.Beta() << "." << std::endl; + +// Now make a prediction for three random points. +for (size_t t = 0; t < 3; ++t) +{ + arma::vec randomPoint(dimensionality, arma::fill::randu); + double prediction, stddev; + blr.Predict(randomPoint, prediction, stddev); + + std::cout << "Prediction for random point " << t << ": " << prediction + << " +/- " << stddev << "." << std::endl; +} +``` + +--- + +### Advanced Functionality: Different Element Types + +The `BayesianLinearRegression` class has one template parameter that can be used +to control the element type of the model. The full signature of the class is: + +```c++ +BayesianLinearRegression +``` + +`ModelMatType` specifies the type of matrix used for the internal representation +of model parameters. Any matrix type that implements the Armadillo API can be +used. + +Note that the `Train()` and `Predict()` functions themselves are templatized and +can allow any matrix type that has the same element type. So, for instance, a +`BayesianLinearRegression` can accept an `arma::sp_mat` for training. + +The example below trains a Bayesian linear regression model on sparse 32-bit +floating point data, but uses a dense 32-bit floating point vector to store the +model itself. + +```c++ +// Create random, sparse 100-dimensional data. +arma::sp_fmat dataset; +dataset.sprandu(100, 5000, 0.3); + +// Generate noisy responses from random data. +arma::fvec trueWeights(100, arma::fill::randu); +arma::frowvec responses = trueWeights.t() * dataset + + 0.01 * arma::randu(5000) /* noise term */; + +mlpack::BayesianLinearRegression blr; +blr.ScaleData() = true; +blr.MaxIterations() = 75; + +blr.Train(dataset, responses); + +// Compute the RMSE on the training set and a random test set. +arma::sp_fmat testDataset; +testDataset.sprandu(100, 1000, 0.3); + +arma::frowvec testResponses = trueWeights.t() * testDataset + + 0.01 * arma::randu(1000) /* noise term */; + +std::cout << "RMSE on training set: " + << blr.RMSE(dataset, responses) << "." << std::endl; +std::cout << "RMSE on test set: " + << blr.RMSE(testDataset, testResponses) << "." << std::endl; +``` + +***Note:*** dense objects should be used for `ModelMatType`, since in general +`BayesianLinearRegression` will produce models that are not sparse. From e4153c6c13adf8e358bce585965fba5054949601 Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Thu, 14 Dec 2023 17:09:23 -0500 Subject: [PATCH 64/91] Bump serialization version of LinearSVM. --- src/mlpack/methods/linear_svm/linear_svm.hpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/mlpack/methods/linear_svm/linear_svm.hpp b/src/mlpack/methods/linear_svm/linear_svm.hpp index 9dbf1a0c70..acc0b85cb5 100644 --- a/src/mlpack/methods/linear_svm/linear_svm.hpp +++ b/src/mlpack/methods/linear_svm/linear_svm.hpp @@ -503,6 +503,9 @@ class LinearSVM } // namespace mlpack +CEREAL_TEMPLATE_CLASS_VERSION((typename ModelMatType), + (mlpack::LinearSVM), (1)); + // Include implementation. #include "linear_svm_impl.hpp" From 79864e142e6244dfda2e1d6763a93d8bcf0887b7 Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Thu, 14 Dec 2023 17:40:19 -0500 Subject: [PATCH 65/91] Add functions to match documentation. --- .../bayesian_linear_regression.hpp | 85 ++++++++++++- .../bayesian_linear_regression_impl.hpp | 103 +++++++++++++-- .../tests/bayesian_linear_regression_test.cpp | 120 ++++++++++++++++++ 3 files changed, 295 insertions(+), 13 deletions(-) diff --git a/src/mlpack/methods/bayesian_linear_regression/bayesian_linear_regression.hpp b/src/mlpack/methods/bayesian_linear_regression/bayesian_linear_regression.hpp index a9d004e82d..1a063b10fa 100644 --- a/src/mlpack/methods/bayesian_linear_regression/bayesian_linear_regression.hpp +++ b/src/mlpack/methods/bayesian_linear_regression/bayesian_linear_regression.hpp @@ -99,9 +99,9 @@ class BayesianLinearRegression { public: /** - * Set the parameters of Bayesian Ridge regression object. The - * regularization parameter is automatically set to its optimal value by - * maximization of the marginal likelihood. + * Set the parameters of Bayesian Ridge regression object. The regularization + * parameter will be automatically set to its optimal value by maximization of + * the marginal likelihood when training is done. * * @param centerData Whether or not center the data according to the * examples. @@ -116,6 +116,29 @@ class BayesianLinearRegression const size_t maxIterations = 50, const double tolerance = 1e-4); + /** + * Create the BayesianLinearRegression object and train the model. The + * regularization parameter is automatically set to its optimal value by + * maximization of the maginal likelihood. + * + * @param data Column-major input data, dim(P, N). + * @param responses A vector of targets, dim(N). + * @param centerData Whether or not center the data according to the + * examples. + * @param scaleData Whether or not scale the data according to the + * standard deviation of each feature. + * @param maxIterations Maximum number of iterations for convergency. + * @param tolerance Level from which the solution is considered sufficientlly + * stable. + * @return Root mean squared error. + */ + BayesianLinearRegression(const arma::mat& data, + const arma::rowvec& responses, + const bool centerData = true, + const bool scaleData = false, + const size_t maxIterations = 50, + const double tolerance = 1e-4); + /** * Run BayesianLinearRegression. The input matrix (like all mlpack matrices) * should be column-major -- each column is an observation and each row is a @@ -123,18 +146,72 @@ class BayesianLinearRegression * * @param data Column-major input data, dim(P, N). * @param responses A vector of targets, dim(N). + * @param centerData Whether or not center the data according to the + * examples. + * @param scaleData Whether or not scale the data according to the + * standard deviation of each feature. + * @param maxIterations Maximum number of iterations for convergency. + * @param tolerance Level from which the solution is considered sufficientlly + * stable. * @return Root mean squared error. */ + // Many overloads necessary here until std::optional is available with C++17. double Train(const arma::mat& data, const arma::rowvec& responses); + double Train(const arma::mat& data, + const arma::rowvec& responses, + const bool centerData); + + double Train(const arma::mat& data, + const arma::rowvec& responses, + const bool centerData, + const bool scaleData); + + double Train(const arma::mat& data, + const arma::rowvec& responses, + const bool centerData, + const bool scaleData, + const size_t maxIterations); + + double Train(const arma::mat& data, + const arma::rowvec& responses, + const bool centerData, + const bool scaleData, + const size_t maxIterations, + const double tolerance); + + /** + * Predict \f$y\f$ for a single data point \f$x\f$ using the currently-trained + * Bayesian ridge regression model. + * + * @param point The data point to apply the model to. + * @return Prediction for the `point`. + */ + template + double Predict(const VecType& point) const; + + /** + * Predict \f$y\f$ for a single data point \f$x\f$ using the currently-trained + * Bayesian ridge regression model, storing the prediction in `prediction` and + * the standard deviation of the prediction in `stddev`. + * + * @param point The data point to apply the model to. + * @param prediction `double` to store the prediction into. + * @param stddev `double` to store the standard deviation of the prediction + * into. + */ + template + void Predict(const VecType& point, + double& prediction, + double& stddev) const; + /** * Predict \f$y_{i}\f$ for each data point in the given data matrix using the * currently-trained Bayesian Ridge model. * * @param points The data points to apply the model. * @param predictions y, Contains the predicted values on completion. - * @return Root mean squared error computed on the train set. */ void Predict(const arma::mat& points, arma::rowvec& predictions) const; diff --git a/src/mlpack/methods/bayesian_linear_regression/bayesian_linear_regression_impl.hpp b/src/mlpack/methods/bayesian_linear_regression/bayesian_linear_regression_impl.hpp index c09b1dfdc6..6f326d3014 100644 --- a/src/mlpack/methods/bayesian_linear_regression/bayesian_linear_regression_impl.hpp +++ b/src/mlpack/methods/bayesian_linear_regression/bayesian_linear_regression_impl.hpp @@ -29,11 +29,79 @@ inline BayesianLinearRegression::BayesianLinearRegression( alpha(0.0), beta(0.0), gamma(0.0) -{/* Nothing to do */} +{ /* Nothing to do */ } -inline double BayesianLinearRegression::Train(const arma::mat& data, - const arma::rowvec& responses) +inline BayesianLinearRegression::BayesianLinearRegression( + const arma::mat& data, + const arma::rowvec& responses, + const bool centerData, + const bool scaleData, + const size_t maxIterations, + const double tolerance) : + centerData(centerData), + scaleData(scaleData), + maxIterations(maxIterations), + tolerance(tolerance), + responsesOffset(0.0), + alpha(0.0), + beta(0.0), + gamma(0.0) { + // Train the model. + Train(data, responses); +} + +inline double BayesianLinearRegression::Train( + const arma::mat& data, + const arma::rowvec& responses) +{ + return Train(data, responses, this->centerData, this->scaleData, + this->maxIterations, this->tolerance); +} + +inline double BayesianLinearRegression::Train( + const arma::mat& data, + const arma::rowvec& responses, + const bool centerData) +{ + return Train(data, responses, centerData, this->scaleData, + this->maxIterations, this->tolerance); +} + +inline double BayesianLinearRegression::Train( + const arma::mat& data, + const arma::rowvec& responses, + const bool centerData, + const bool scaleData) +{ + return Train(data, responses, centerData, scaleData, this->maxIterations, + this->tolerance); +} + +inline double BayesianLinearRegression::Train( + const arma::mat& data, + const arma::rowvec& responses, + const bool centerData, + const bool scaleData, + const size_t maxIterations) +{ + return Train(data, responses, centerData, scaleData, maxIterations, + this->tolerance); +} + +inline double BayesianLinearRegression::Train( + const arma::mat& data, + const arma::rowvec& responses, + const bool centerData, + const bool scaleData, + const size_t maxIterations, + const double tolerance) +{ + this->centerData = centerData; + this->scaleData = scaleData; + this->maxIterations = maxIterations; + this->tolerance = tolerance; + arma::mat phi; arma::rowvec t; arma::colvec eigVal; @@ -87,6 +155,28 @@ inline double BayesianLinearRegression::Train(const arma::mat& data, return RMSE(data, responses); } +template +inline double BayesianLinearRegression::Predict(const VecType& point) const +{ + // Center and scale the point before applying the model. + arma::mat centeredPoint; + CenterScaleDataPred(point, centeredPoint); + return arma::dot(omega, centeredPoint) + responsesOffset; +} + +template +inline void BayesianLinearRegression::Predict(const VecType& point, + double& prediction, + double& stddev) const +{ + // Center and scale the point before applying the model. + arma::mat centeredPoint; + CenterScaleDataPred(point, centeredPoint); + prediction = arma::dot(omega, centeredPoint) + responsesOffset; + stddev = std::sqrt(Variance() + + arma::accu(centeredPoint % (matCovariance * centeredPoint))); +} + inline void BayesianLinearRegression::Predict(const arma::mat& points, arma::rowvec& predictions) const { @@ -131,7 +221,6 @@ inline double BayesianLinearRegression::CenterScaleData( responses.n_elem, false, true); } - else if (centerData && !scaleData) { dataOffset = mean(data, 1); @@ -139,7 +228,6 @@ inline double BayesianLinearRegression::CenterScaleData( dataProc = data.each_col() - dataOffset; responsesProc = responses - responsesOffset; } - else if (!centerData && scaleData) { dataScale = stddev(data, 0, 1); @@ -148,7 +236,6 @@ inline double BayesianLinearRegression::CenterScaleData( responses.n_elem, false, true); } - else { dataOffset = mean(data, 1); @@ -157,6 +244,7 @@ inline double BayesianLinearRegression::CenterScaleData( dataProc = (data.each_col() - dataOffset).each_col() / dataScale; responsesProc = responses - responsesOffset; } + return responsesOffset; } @@ -169,17 +257,14 @@ inline void BayesianLinearRegression::CenterScaleDataPred( dataProc = arma::mat(const_cast(data.memptr()), data.n_rows, data.n_cols, false, true); } - else if (centerData && !scaleData) { dataProc = data.each_col() - dataOffset; } - else if (!centerData && scaleData) { dataProc = data.each_col() / dataScale; } - else { dataProc = (data.each_col() - dataOffset).each_col() / dataScale; diff --git a/src/mlpack/tests/bayesian_linear_regression_test.cpp b/src/mlpack/tests/bayesian_linear_regression_test.cpp index 4f6bfb43c9..0ebd17bf51 100644 --- a/src/mlpack/tests/bayesian_linear_regression_test.cpp +++ b/src/mlpack/tests/bayesian_linear_regression_test.cpp @@ -156,6 +156,15 @@ TEST_CASE("PredictiveUncertainties", "[BayesianLinearRegressionTest]") for (size_t i = 0; i < matX.n_cols; i++) REQUIRE(std[i] > estStd); + // Also make single-point predictions. + for (size_t i = 0; i < matX.n_cols; ++i) + { + double prediction, stddev; + estimator.Predict(matX.col(i), prediction, stddev); + REQUIRE(prediction == Approx(responses[i])); + REQUIRE(stddev > estStd); + } + // Check that the estimated variance is close to 1. REQUIRE(estStd == Approx(1).epsilon(0.3)); } @@ -187,9 +196,120 @@ TEST_CASE("EqualtoRidge", "[BayesianLinearRegressionTest]") for (size_t i = 0; i < y.size(); ++i) REQUIRE(blrPred[i] == Approx(ridgePred[i]).epsilon(1)); + // Also make single-point predictions. + for (size_t i = 0; i < y.n_elem; ++i) + REQUIRE(blr.Predict(matX.col(i)) == Approx(ridgePred[i]).epsilon(1)); + // Exit once a test case has completed. break; } REQUIRE(trial <= 3); } + +// Check that all constructor variants work. +TEMPLATE_TEST_CASE("BayesianLinearRegressionConstructorVariantTest", + "[BayesianLinearRegressionTest]", arma::mat) +{ + typedef TestType MatType; + + MatType matX; + arma::Row y; + size_t nDims = 5, nPoints = 100; + GenerateProblem(matX, y, nPoints, nDims, 0.5); + + // The important thing here is that all of the inputs to the constructor are + // properly parsed. The actual details of the models that are learned are + // less important and are checked by other tests. + BayesianLinearRegression blr1; + BayesianLinearRegression blr2(false); + BayesianLinearRegression blr3(false, true); + BayesianLinearRegression blr4(false, true, 100); + BayesianLinearRegression blr5(false, true, 110, 1e-3); + BayesianLinearRegression blr6(matX, y); + BayesianLinearRegression blr7(matX, y, false); + BayesianLinearRegression blr8(matX, y, false, true); + BayesianLinearRegression blr9(matX, y, false, true, 120); + BayesianLinearRegression blr10(matX, y, false, true, 130, 1e-2); + + // Check that the hyperparameters as reported by the model are correct. + REQUIRE(blr1.Omega().n_elem == 0); + + REQUIRE(blr2.Omega().n_elem == 0); + REQUIRE(blr2.CenterData() == false); + + REQUIRE(blr3.Omega().n_elem == 0); + REQUIRE(blr3.CenterData() == false); + REQUIRE(blr3.ScaleData() == true); + + REQUIRE(blr4.Omega().n_elem == 0); + REQUIRE(blr4.CenterData() == false); + REQUIRE(blr4.ScaleData() == true); + REQUIRE(blr4.MaxIterations() == 100); + + REQUIRE(blr5.Omega().n_elem == 0); + REQUIRE(blr5.CenterData() == false); + REQUIRE(blr5.ScaleData() == true); + REQUIRE(blr5.MaxIterations() == 110); + REQUIRE(blr5.Tolerance() == 1e-3); + + REQUIRE(blr6.Omega().n_elem == matX.n_rows); + + REQUIRE(blr7.Omega().n_elem == matX.n_rows); + REQUIRE(blr7.CenterData() == false); + + REQUIRE(blr8.Omega().n_elem == matX.n_rows); + REQUIRE(blr8.CenterData() == false); + REQUIRE(blr8.ScaleData() == true); + + REQUIRE(blr9.Omega().n_elem == matX.n_rows); + REQUIRE(blr9.CenterData() == false); + REQUIRE(blr9.ScaleData() == true); + REQUIRE(blr9.MaxIterations() == 120); + + REQUIRE(blr10.Omega().n_elem == matX.n_rows); + REQUIRE(blr10.CenterData() == false); + REQUIRE(blr10.ScaleData() == true); + REQUIRE(blr10.MaxIterations() == 130); + REQUIRE(blr10.Tolerance() == 1e-2); +} + +// Test that all Train() variants work. +TEMPLATE_TEST_CASE("BayesianLinearRegressionTrainVariantTest", + "[BayesianLinearRegressionTest]", arma::mat) +{ + typedef TestType MatType; + + MatType matX; + arma::Row y; + size_t nDims = 5, nPoints = 100; + GenerateProblem(matX, y, nPoints, nDims, 0.5); + + BayesianLinearRegression blr1, blr2, blr3, blr4, blr5; + + blr1.Train(matX, y); + blr2.Train(matX, y, false); + blr3.Train(matX, y, false, true); + blr4.Train(matX, y, false, true, 100); + blr5.Train(matX, y, false, true, 110, 1e-3); + + REQUIRE(blr1.Omega().n_elem == matX.n_rows); + + REQUIRE(blr2.Omega().n_elem == matX.n_rows); + REQUIRE(blr2.CenterData() == false); + + REQUIRE(blr3.Omega().n_elem == matX.n_rows); + REQUIRE(blr3.CenterData() == false); + REQUIRE(blr3.ScaleData() == true); + + REQUIRE(blr4.Omega().n_elem == matX.n_rows); + REQUIRE(blr4.CenterData() == false); + REQUIRE(blr4.ScaleData() == true); + REQUIRE(blr4.MaxIterations() == 100); + + REQUIRE(blr5.Omega().n_elem == matX.n_rows); + REQUIRE(blr5.CenterData() == false); + REQUIRE(blr5.ScaleData() == true); + REQUIRE(blr5.MaxIterations() == 110); + REQUIRE(blr5.Tolerance() == 1e-3); +} From 91e630aa32d33b662e3c87c7ca1f81cc508dd77a Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Fri, 15 Dec 2023 09:10:49 -0500 Subject: [PATCH 66/91] Add template parameter for matrix type to BayesianLinearRegression. --- src/mlpack/core/util/arma_traits.hpp | 82 +++++ .../bayesian_linear_regression.hpp | 168 +++++++--- .../bayesian_linear_regression_impl.hpp | 311 +++++++++++++----- .../bayesian_linear_regression_main.cpp | 12 +- .../tests/bayesian_linear_regression_test.cpp | 36 +- .../bayesian_linear_regression_test.cpp | 14 +- src/mlpack/tests/serialization_test.cpp | 4 +- 7 files changed, 455 insertions(+), 172 deletions(-) diff --git a/src/mlpack/core/util/arma_traits.hpp b/src/mlpack/core/util/arma_traits.hpp index a755ce9efa..5f956a3307 100644 --- a/src/mlpack/core/util/arma_traits.hpp +++ b/src/mlpack/core/util/arma_traits.hpp @@ -111,4 +111,86 @@ struct IsVector > #endif +// Get the row vector type corresponding to a given MatType. + +template +struct GetRowType +{ + typedef arma::Row type; +}; + +template +struct GetRowType> +{ + typedef arma::Row type; +}; + +template +struct GetRowType> +{ + typedef arma::SpRow type; +}; + +// Get the column vector type corresponding to a given MatType. + +template +struct GetColType +{ + typedef arma::Row type; +}; + +template +struct GetColType> +{ + typedef arma::Col type; +}; + +template +struct GetColType> +{ + typedef arma::SpCol type; +}; + +// Get the dense row vector type corresponding to a given MatType. + +template +struct GetDenseRowType +{ + typedef typename GetRowType::type type; +}; + +template +struct GetDenseRowType> +{ + typedef arma::Row type; +}; + +// Get the dense column vector type corresponding to a given MatType. + +template +struct GetDenseColType +{ + typedef typename GetColType::type type; +}; + +template +struct GetDenseColType> +{ + typedef arma::Col type; +}; + +// Get the dense matrix type corresponding to a given MatType. + +template +struct GetDenseMatType +{ + typedef arma::Mat type; +}; + +template +struct GetDenseMatType> +{ + typedef arma::Mat type; +}; + #endif diff --git a/src/mlpack/methods/bayesian_linear_regression/bayesian_linear_regression.hpp b/src/mlpack/methods/bayesian_linear_regression/bayesian_linear_regression.hpp index 1a063b10fa..958ce41417 100644 --- a/src/mlpack/methods/bayesian_linear_regression/bayesian_linear_regression.hpp +++ b/src/mlpack/methods/bayesian_linear_regression/bayesian_linear_regression.hpp @@ -95,9 +95,14 @@ namespace mlpack { * estimator.Predict(xTest, responses, stds) * @endcode */ +template class BayesianLinearRegression { public: + typedef typename ModelMatType::elem_type ElemType; + typedef typename GetDenseColType::type DenseVecType; + typedef typename GetDenseRowType::type DenseRowType; + /** * Set the parameters of Bayesian Ridge regression object. The regularization * parameter will be automatically set to its optimal value by maximization of @@ -132,8 +137,13 @@ class BayesianLinearRegression * stable. * @return Root mean squared error. */ - BayesianLinearRegression(const arma::mat& data, - const arma::rowvec& responses, + template::value + >::type> + BayesianLinearRegression(const MatType& data, + const ResponsesType& responses, const bool centerData = true, const bool scaleData = false, const size_t maxIterations = 50, @@ -156,30 +166,69 @@ class BayesianLinearRegression * @return Root mean squared error. */ // Many overloads necessary here until std::optional is available with C++17. - double Train(const arma::mat& data, - const arma::rowvec& responses); + // The first overload is also necessary to avoid confusing the hyperparameter + // tuner, so that this can be correctly detected as a regression algorithm. + template + ElemType Train(const MatType& data, + const arma::rowvec& responses); - double Train(const arma::mat& data, - const arma::rowvec& responses, - const bool centerData); + template::value + >::type, + typename = typename std::enable_if< + !std::is_same::value + >::type> + ElemType Train(const MatType& data, + const ResponsesType& responses); - double Train(const arma::mat& data, - const arma::rowvec& responses, - const bool centerData, - const bool scaleData); + template::value + >::type> + ElemType Train(const MatType& data, + const ResponsesType& responses, + const bool centerData); - double Train(const arma::mat& data, - const arma::rowvec& responses, - const bool centerData, - const bool scaleData, - const size_t maxIterations); + template::value + >::type> + ElemType Train(const MatType& data, + const ResponsesType& responses, + const bool centerData, + const bool scaleData); - double Train(const arma::mat& data, - const arma::rowvec& responses, - const bool centerData, - const bool scaleData, - const size_t maxIterations, - const double tolerance); + template::value + >::type> + ElemType Train(const MatType& data, + const ResponsesType& responses, + const bool centerData, + const bool scaleData, + const size_t maxIterations); + + template::value + >::type> + ElemType Train(const MatType& data, + const ResponsesType& responses, + const bool centerData, + const bool scaleData, + const size_t maxIterations, + const double tolerance); /** * Predict \f$y\f$ for a single data point \f$x\f$ using the currently-trained @@ -189,7 +238,7 @@ class BayesianLinearRegression * @return Prediction for the `point`. */ template - double Predict(const VecType& point) const; + ElemType Predict(const VecType& point) const; /** * Predict \f$y\f$ for a single data point \f$x\f$ using the currently-trained @@ -203,8 +252,8 @@ class BayesianLinearRegression */ template void Predict(const VecType& point, - double& prediction, - double& stddev) const; + ElemType& prediction, + ElemType& stddev) const; /** * Predict \f$y_{i}\f$ for each data point in the given data matrix using the @@ -213,8 +262,13 @@ class BayesianLinearRegression * @param points The data points to apply the model. * @param predictions y, Contains the predicted values on completion. */ - void Predict(const arma::mat& points, - arma::rowvec& predictions) const; + template::value + >::type> + void Predict(const MatType& points, + ResponsesType& predictions) const; /** * Predict \f$y_{i}\f$ and the standard deviation of the predictive posterior @@ -226,9 +280,14 @@ class BayesianLinearRegression * completion. * @param std Standard deviations of the predictions. */ - void Predict(const arma::mat& points, - arma::rowvec& predictions, - arma::rowvec& std) const; + template::value + >::type> + void Predict(const MatType& points, + ResponsesType& predictions, + ResponsesType& std) const; /** * Compute the Root Mean Square Error between the predictions returned by the @@ -238,8 +297,13 @@ class BayesianLinearRegression * @param responses A vector of targets. * @return Root mean squared error. **/ - double RMSE(const arma::mat& data, - const arma::rowvec& responses) const; + template::value + >::type> + ElemType RMSE(const MatType& data, + const ResponsesType& responses) const; /** * Get the solution vector. @@ -276,7 +340,7 @@ class BayesianLinearRegression * * @return responsesOffset */ - const arma::colvec& DataOffset() const { return dataOffset; } + const DenseVecType& DataOffset() const { return dataOffset; } /** * Get the vector of standard deviations computed on the features over the @@ -284,14 +348,14 @@ class BayesianLinearRegression * * @return dataOffset */ - const arma::colvec& DataScale() const { return dataScale; } + const DenseVecType& DataScale() const { return dataScale; } /** * Get the mean value of the train responses. * * @return responsesOffset */ - double ResponsesOffset() const { return responsesOffset; } + ElemType ResponsesOffset() const { return responsesOffset; } //! Get whether the data will be centered during training. bool CenterData() const { return centerData; } @@ -335,28 +399,28 @@ class BayesianLinearRegression double tolerance; //! Mean vector computed over the points. - arma::colvec dataOffset; + DenseVecType dataOffset; //! Std vector computed over the points. - arma::colvec dataScale; + DenseVecType dataScale; //! Mean of the response vector computed over the points. - double responsesOffset; + ElemType responsesOffset; //! Precision of the prior pdf (gaussian). - double alpha; + ElemType alpha; //! Noise inverse variance. - double beta; + ElemType beta; //! Effective number of parameters. - double gamma; + ElemType gamma; //! Solution vector. - arma::colvec omega; + DenseVecType omega; //! Covariance matrix of the solution vector omega. - arma::mat matCovariance; + ModelMatType matCovariance; /** * Center and scale the data accordind to centerData and scaleData. @@ -368,19 +432,23 @@ class BayesianLinearRegression * @param responsesProc Responses processed, dim(N). * @return reponsesOffset Mean of responses. */ - double CenterScaleData(const arma::mat& data, - const arma::rowvec& responses, - arma::mat& dataProc, - arma::rowvec& responsesProc); + template + double CenterScaleData(const MatType& data, + const ResponsesType& responses, + MatType& dataProc, + ResponsesType& responsesProc); /** - * Center and scale the points before prediction. + * Center and scale the points before prediction. This should only be called + * if centerData or scaleData is true; if neither is true, then `dataProc` + * will be unmodified. * * @param data Design matrix in column-major format, dim(P, N). * @param dataProc Data processed, dim(P, N). */ - void CenterScaleDataPred(const arma::mat& data, - arma::mat& dataProc) const; + template + void CenterScaleDataPred(const MatType& data, + OutMatType& dataProc) const; }; } // namespace mlpack diff --git a/src/mlpack/methods/bayesian_linear_regression/bayesian_linear_regression_impl.hpp b/src/mlpack/methods/bayesian_linear_regression/bayesian_linear_regression_impl.hpp index 6f326d3014..f24f70fa25 100644 --- a/src/mlpack/methods/bayesian_linear_regression/bayesian_linear_regression_impl.hpp +++ b/src/mlpack/methods/bayesian_linear_regression/bayesian_linear_regression_impl.hpp @@ -16,7 +16,8 @@ namespace mlpack { -inline BayesianLinearRegression::BayesianLinearRegression( +template +inline BayesianLinearRegression::BayesianLinearRegression( const bool centerData, const bool scaleData, const size_t maxIterations, @@ -31,9 +32,11 @@ inline BayesianLinearRegression::BayesianLinearRegression( gamma(0.0) { /* Nothing to do */ } -inline BayesianLinearRegression::BayesianLinearRegression( - const arma::mat& data, - const arma::rowvec& responses, +template +template +inline BayesianLinearRegression::BayesianLinearRegression( + const MatType& data, + const ResponsesType& responses, const bool centerData, const bool scaleData, const size_t maxIterations, @@ -51,26 +54,50 @@ inline BayesianLinearRegression::BayesianLinearRegression( Train(data, responses); } -inline double BayesianLinearRegression::Train( - const arma::mat& data, +template +template +inline +typename BayesianLinearRegression::ElemType +BayesianLinearRegression::Train( + const MatType& data, const arma::rowvec& responses) { return Train(data, responses, this->centerData, this->scaleData, this->maxIterations, this->tolerance); } -inline double BayesianLinearRegression::Train( - const arma::mat& data, - const arma::rowvec& responses, +template +template +inline +typename BayesianLinearRegression::ElemType +BayesianLinearRegression::Train( + const MatType& data, + const ResponsesType& responses) +{ + return Train(data, responses, this->centerData, this->scaleData, + this->maxIterations, this->tolerance); +} + +template +template +inline +typename BayesianLinearRegression::ElemType +BayesianLinearRegression::Train( + const MatType& data, + const ResponsesType& responses, const bool centerData) { return Train(data, responses, centerData, this->scaleData, this->maxIterations, this->tolerance); } -inline double BayesianLinearRegression::Train( - const arma::mat& data, - const arma::rowvec& responses, +template +template +inline +typename BayesianLinearRegression::ElemType +BayesianLinearRegression::Train( + const MatType& data, + const ResponsesType& responses, const bool centerData, const bool scaleData) { @@ -78,9 +105,13 @@ inline double BayesianLinearRegression::Train( this->tolerance); } -inline double BayesianLinearRegression::Train( - const arma::mat& data, - const arma::rowvec& responses, +template +template +inline +typename BayesianLinearRegression::ElemType +BayesianLinearRegression::Train( + const MatType& data, + const ResponsesType& responses, const bool centerData, const bool scaleData, const size_t maxIterations) @@ -89,9 +120,13 @@ inline double BayesianLinearRegression::Train( this->tolerance); } -inline double BayesianLinearRegression::Train( - const arma::mat& data, - const arma::rowvec& responses, +template +template +inline +typename BayesianLinearRegression::ElemType +BayesianLinearRegression::Train( + const MatType& data, + const ResponsesType& responses, const bool centerData, const bool scaleData, const size_t maxIterations, @@ -102,10 +137,10 @@ inline double BayesianLinearRegression::Train( this->maxIterations = maxIterations; this->tolerance = tolerance; - arma::mat phi; - arma::rowvec t; - arma::colvec eigVal; - arma::mat eigVec; + ModelMatType phi; + DenseRowType t; + DenseVecType eigVal; + ModelMatType eigVec; // Preprocess the data. Center and scale. responsesOffset = CenterScaleData(data, responses, phi, t); @@ -117,20 +152,20 @@ inline double BayesianLinearRegression::Train( } // Compute this quantities once and for all. - const arma::mat eigVecInv = inv(eigVec); - const arma::colvec eigVecInvPhitT = eigVecInv * phi * t.t(); + const ModelMatType eigVecInv = inv(eigVec); + const DenseVecType eigVecInvPhitT = eigVecInv * phi * t.t(); // Initialize the hyperparameters and begin with an infinitely broad prior. alpha = 1e-6; beta = 1 / (var(t, 1) * 0.1); unsigned short i = 0; - double crit = 1.0; + ElemType crit = 1.0; - while ((crit > tolerance) && (i < maxIterations)) + while (((double) crit > tolerance) && (i < maxIterations)) { - double deltaAlpha = -alpha; - double deltaBeta = -beta; + ElemType deltaAlpha = -alpha; + ElemType deltaBeta = -beta; // Update the solution. omega = eigVec * diagmat(1 / (eigVal + (alpha / beta))) * eigVecInvPhitT; @@ -140,7 +175,7 @@ inline double BayesianLinearRegression::Train( alpha = gamma / dot(omega, omega); // Update beta. - const arma::rowvec temp = t - omega.t() * phi; + const DenseRowType temp = t - omega.t() * phi; beta = (data.n_cols - gamma) / dot(temp, temp); // Compute the stopping criterion. @@ -155,71 +190,131 @@ inline double BayesianLinearRegression::Train( return RMSE(data, responses); } +template template -inline double BayesianLinearRegression::Predict(const VecType& point) const +inline +typename BayesianLinearRegression::ElemType +BayesianLinearRegression::Predict(const VecType& point) const { - // Center and scale the point before applying the model. - arma::mat centeredPoint; - CenterScaleDataPred(point, centeredPoint); - return arma::dot(omega, centeredPoint) + responsesOffset; + // Center and scale the point before applying the model, if needed. + if (!centerData && !scaleData) + return arma::dot(omega, point) + responsesOffset; + else if (centerData && !scaleData) + return arma::dot(omega, point - dataOffset) + responsesOffset; + else if (!centerData && scaleData) + return arma::dot(omega, point / dataScale) + responsesOffset; + else + return arma::dot(omega, (point - dataOffset) / dataScale) + responsesOffset; } +template template -inline void BayesianLinearRegression::Predict(const VecType& point, - double& prediction, - double& stddev) const +inline void BayesianLinearRegression::Predict( + const VecType& point, + typename BayesianLinearRegression::ElemType& prediction, + typename BayesianLinearRegression::ElemType& stddev) const { - // Center and scale the point before applying the model. - arma::mat centeredPoint; - CenterScaleDataPred(point, centeredPoint); - prediction = arma::dot(omega, centeredPoint) + responsesOffset; - stddev = std::sqrt(Variance() + - arma::accu(centeredPoint % (matCovariance * centeredPoint))); + prediction = Predict(point); + + ElemType inner; + if (!centerData && !scaleData) + { + stddev = std::sqrt(Variance() + + arma::accu(point % (matCovariance * point))); + inner = arma::accu(point % (matCovariance * point)); + } + else if (centerData && !scaleData) + { + inner = arma::accu((point - dataOffset) % + (matCovariance * (point - dataOffset))); + } + else if (!centerData && scaleData) + { + inner = arma::accu((point / dataScale) % + (matCovariance * (point / dataScale))); + } + else + { + inner = arma::accu(((point - dataOffset) / dataScale) % + (matCovariance * ((point - dataOffset) / dataScale))); + } + + stddev = std::sqrt(Variance() + inner); } -inline void BayesianLinearRegression::Predict(const arma::mat& points, - arma::rowvec& predictions) const +template +template +inline void BayesianLinearRegression::Predict( + const MatType& points, + ResponsesType& predictions) const { - // Center and scale the points before applying the model. - arma::mat matX; - CenterScaleDataPred(points, matX); - predictions = omega.t() * matX + responsesOffset; + if (!centerData && !scaleData) + { + predictions = omega.t() * points + responsesOffset; + } + else + { + // Center and scale the points before applying the model. + arma::Mat pointsProc; + CenterScaleDataPred(points, pointsProc); + + predictions = omega.t() * pointsProc + responsesOffset; + } } -inline void BayesianLinearRegression::Predict(const arma::mat& points, - arma::rowvec& predictions, - arma::rowvec& std) const +template +template +inline void BayesianLinearRegression::Predict( + const MatType& points, + ResponsesType& predictions, + ResponsesType& std) const { - // Center and scale the points before applying the model. - arma::mat matX; - CenterScaleDataPred(points, matX); - predictions = omega.t() * matX + responsesOffset; - // Compute the standard deviation for each point. - std = sqrt(Variance() + sum(matX % (matCovariance * matX), 0)); + if (!centerData && !scaleData) + { + Predict(points, predictions); + std = arma::sqrt(Variance() + arma::sum(points % + (matCovariance * points), 0)); + } + else + { + // Center or scale data. + arma::Mat pointsProc; + CenterScaleDataPred(points, pointsProc); + + predictions = omega.t() * pointsProc + responsesOffset; + std = arma::sqrt(Variance() + arma::sum(pointsProc % + (matCovariance * pointsProc), 0)); + } } -inline double BayesianLinearRegression::RMSE( - const arma::mat& data, - const arma::rowvec& responses) const +template +template +inline +typename BayesianLinearRegression::ElemType +BayesianLinearRegression::RMSE( + const MatType& data, + const ResponsesType& responses) const { - arma::rowvec predictions; + typename GetDenseRowType::type predictions; Predict(data, predictions); return sqrt(mean(square(responses - predictions))); } -inline double BayesianLinearRegression::CenterScaleData( - const arma::mat& data, - const arma::rowvec& responses, - arma::mat& dataProc, - arma::rowvec& responsesProc) +template +template +inline double BayesianLinearRegression::CenterScaleData( + const MatType& data, + const ResponsesType& responses, + MatType& dataProc, + ResponsesType& responsesProc) { if (!centerData && !scaleData) { - dataProc = arma::mat(const_cast(data.memptr()), data.n_rows, + dataProc = MatType(const_cast(data.memptr()), data.n_rows, data.n_cols, false, true); - responsesProc = arma::rowvec(const_cast(responses.memptr()), - responses.n_elem, false, - true); + responsesProc = ResponsesType(const_cast(responses.memptr()), + responses.n_elem, false, + true); } else if (centerData && !scaleData) { @@ -232,9 +327,9 @@ inline double BayesianLinearRegression::CenterScaleData( { dataScale = stddev(data, 0, 1); dataProc = data.each_col() / dataScale; - responsesProc = arma::rowvec(const_cast(responses.memptr()), - responses.n_elem, false, - true); + responsesProc = ResponsesType(const_cast(responses.memptr()), + responses.n_elem, false, + true); } else { @@ -248,14 +343,15 @@ inline double BayesianLinearRegression::CenterScaleData( return responsesOffset; } -inline void BayesianLinearRegression::CenterScaleDataPred( - const arma::mat& data, - arma::mat& dataProc) const +template +template +inline void BayesianLinearRegression::CenterScaleDataPred( + const MatType& data, + OutMatType& dataProc) const { if (!centerData && !scaleData) { - dataProc = arma::mat(const_cast(data.memptr()), data.n_rows, - data.n_cols, false, true); + return; // Don't modify dataProc. } else if (centerData && !scaleData) { @@ -274,22 +370,59 @@ inline void BayesianLinearRegression::CenterScaleDataPred( /** * Serialize the Bayesian linear regression model. */ +template template -void BayesianLinearRegression::serialize(Archive& ar, - const uint32_t /* version */) +void BayesianLinearRegression::serialize(Archive& ar, + const uint32_t version) { ar(CEREAL_NVP(centerData)); ar(CEREAL_NVP(scaleData)); ar(CEREAL_NVP(maxIterations)); ar(CEREAL_NVP(tolerance)); - ar(CEREAL_NVP(dataOffset)); - ar(CEREAL_NVP(dataScale)); - ar(CEREAL_NVP(responsesOffset)); - ar(CEREAL_NVP(alpha)); - ar(CEREAL_NVP(beta)); - ar(CEREAL_NVP(gamma)); - ar(CEREAL_NVP(omega)); - ar(CEREAL_NVP(matCovariance)); + + // In older versions, dataOffset and dataScale were of type arma::colvec, + // responsesOffset, alpha, beta, and gamma were of type double, omega was of + // type arma::colvec, and matCovariance was of type arma::mat. + if (cereal::is_loading() && version == 0) + { + arma::colvec colvecTmp; + ar(cereal::make_nvp("dataOffset", colvecTmp)); + dataOffset = arma::conv_to::from(colvecTmp); + + ar(cereal::make_nvp("dataScale", colvecTmp)); + dataScale = arma::conv_to::from(colvecTmp); + + double dblTmp; + ar(cereal::make_nvp("responsesOffset", dblTmp)); + responsesOffset = (ElemType) dblTmp; + + ar(cereal::make_nvp("alpha", dblTmp)); + alpha = (ElemType) dblTmp; + + ar(cereal::make_nvp("beta", dblTmp)); + beta = (ElemType) dblTmp; + + ar(cereal::make_nvp("gamma", dblTmp)); + gamma = (ElemType) dblTmp; + + ar(cereal::make_nvp("omega", colvecTmp)); + omega = arma::conv_to::from(colvecTmp); + + arma::mat matTmp; + ar(cereal::make_nvp("matCovariance", matTmp)); + matCovariance = arma::conv_to::from(matCovariance); + } + else + { + ar(CEREAL_NVP(dataOffset)); + ar(CEREAL_NVP(dataScale)); + ar(CEREAL_NVP(responsesOffset)); + ar(CEREAL_NVP(alpha)); + ar(CEREAL_NVP(beta)); + ar(CEREAL_NVP(gamma)); + ar(CEREAL_NVP(omega)); + ar(CEREAL_NVP(matCovariance)); + } } } // namespace mlpack diff --git a/src/mlpack/methods/bayesian_linear_regression/bayesian_linear_regression_main.cpp b/src/mlpack/methods/bayesian_linear_regression/bayesian_linear_regression_main.cpp index 16be8aaeeb..426342791b 100644 --- a/src/mlpack/methods/bayesian_linear_regression/bayesian_linear_regression_main.cpp +++ b/src/mlpack/methods/bayesian_linear_regression/bayesian_linear_regression_main.cpp @@ -109,10 +109,10 @@ PARAM_MATRIX_IN("input", "Matrix of covariates (X).", "i"); PARAM_ROW_IN("responses", "Matrix of responses/observations (y).", "r"); -PARAM_MODEL_IN(BayesianLinearRegression, "input_model", "Trained " +PARAM_MODEL_IN(BayesianLinearRegression<>, "input_model", "Trained " "BayesianLinearRegression model to use.", "m"); -PARAM_MODEL_OUT(BayesianLinearRegression, "output_model", "Output " +PARAM_MODEL_OUT(BayesianLinearRegression<>, "output_model", "Output " "BayesianLinearRegression model.", "M"); PARAM_MATRIX_IN("test", "Matrix containing points to regress on (test " @@ -149,12 +149,12 @@ void BINDING_FUNCTION(util::Params& params, util::Timers& timers) // Ignore out_predictions unless test is specified. ReportIgnoredParam(params, {{"test", false}}, "predictions"); - BayesianLinearRegression* bayesLinReg; + BayesianLinearRegression<>* bayesLinReg; if (params.Has("input")) { Log::Info << "Input given; model will be trained." << std::endl; // Initialize the object. - bayesLinReg = new BayesianLinearRegression(center, scale); + bayesLinReg = new BayesianLinearRegression<>(center, scale); // Load covariates. mat matX = std::move(params.Get("input")); @@ -180,7 +180,7 @@ void BINDING_FUNCTION(util::Params& params, util::Timers& timers) } else // We must have --input_model_file. { - bayesLinReg = params.Get("input_model"); + bayesLinReg = params.Get*>("input_model"); } if (params.Has("test")) @@ -209,5 +209,5 @@ void BINDING_FUNCTION(util::Params& params, util::Timers& timers) params.Get("predictions") = std::move(predictions); } - params.Get("output_model") = bayesLinReg; + params.Get*>("output_model") = bayesLinReg; } diff --git a/src/mlpack/tests/bayesian_linear_regression_test.cpp b/src/mlpack/tests/bayesian_linear_regression_test.cpp index 0ebd17bf51..cf4eb92fb2 100644 --- a/src/mlpack/tests/bayesian_linear_regression_test.cpp +++ b/src/mlpack/tests/bayesian_linear_regression_test.cpp @@ -42,7 +42,7 @@ TEST_CASE("BayesianLinearRegressionRegressionTest", GenerateProblem(matX, y, 200, 10); // Instanciate and train the estimator. - BayesianLinearRegression estimator(true); + BayesianLinearRegression<> estimator(true); estimator.Train(matX, y); estimator.Predict(matX, predictions); @@ -63,7 +63,7 @@ TEST_CASE("TestCenter0ScaleData0", "[BayesianLinearRegressionTest]") GenerateProblem(matX, y, nPoints, nDims, 0.5); - BayesianLinearRegression estimator(false, false); + BayesianLinearRegression<> estimator(false, false); estimator.Train(matX, y); @@ -85,7 +85,7 @@ TEST_CASE("TestCenterDataTrueScaleDataTrue", "[BayesianLinearRegressionTest]") size_t nDims = 5, nPoints = 100; GenerateProblem(matX, y, nPoints, nDims, 0.5); - BayesianLinearRegression estimator(true, true); + BayesianLinearRegression<> estimator(true, true); estimator.Train(matX, y); arma::colvec xMean = arma::mean(matX, 1); @@ -108,7 +108,7 @@ TEST_CASE("OptionsMakeModelDifferent", "[BayesianLinearRegressionTest]") size_t nDims = 10, nPoints = 100; GenerateProblem(matX, y, nPoints, nDims, 0.5); - BayesianLinearRegression blr(false, false), blrC(true, false), + BayesianLinearRegression<> blr(false, false), blrC(true, false), blrCS(true, true); blr.Train(matX, y); @@ -133,7 +133,7 @@ TEST_CASE("SingularMatix", "[BayesianLinearRegressionTest]") // Now the first and the second rows are indentical. matX.row(1) = matX.row(0); - BayesianLinearRegression estimator; + BayesianLinearRegression<> estimator; estimator.Train(matX, y); } @@ -146,7 +146,7 @@ TEST_CASE("PredictiveUncertainties", "[BayesianLinearRegressionTest]") GenerateProblem(matX, y, 100, 10, 1); - BayesianLinearRegression estimator(true, true); + BayesianLinearRegression<> estimator(true, true); estimator.Train(matX, y); arma::rowvec responses, std; @@ -180,7 +180,7 @@ TEST_CASE("EqualtoRidge", "[BayesianLinearRegressionTest]") { GenerateProblem(matX, y, 100, 10, 1); - BayesianLinearRegression blr(false, false); + BayesianLinearRegression<> blr(false, false); blr.Train(matX, y); LinearRegression ridge(matX, y, blr.Alpha() / blr.Beta(), false); @@ -221,16 +221,16 @@ TEMPLATE_TEST_CASE("BayesianLinearRegressionConstructorVariantTest", // The important thing here is that all of the inputs to the constructor are // properly parsed. The actual details of the models that are learned are // less important and are checked by other tests. - BayesianLinearRegression blr1; - BayesianLinearRegression blr2(false); - BayesianLinearRegression blr3(false, true); - BayesianLinearRegression blr4(false, true, 100); - BayesianLinearRegression blr5(false, true, 110, 1e-3); - BayesianLinearRegression blr6(matX, y); - BayesianLinearRegression blr7(matX, y, false); - BayesianLinearRegression blr8(matX, y, false, true); - BayesianLinearRegression blr9(matX, y, false, true, 120); - BayesianLinearRegression blr10(matX, y, false, true, 130, 1e-2); + BayesianLinearRegression<> blr1; + BayesianLinearRegression<> blr2(false); + BayesianLinearRegression<> blr3(false, true); + BayesianLinearRegression<> blr4(false, true, 100); + BayesianLinearRegression<> blr5(false, true, 110, 1e-3); + BayesianLinearRegression<> blr6(matX, y); + BayesianLinearRegression<> blr7(matX, y, false); + BayesianLinearRegression<> blr8(matX, y, false, true); + BayesianLinearRegression<> blr9(matX, y, false, true, 120); + BayesianLinearRegression<> blr10(matX, y, false, true, 130, 1e-2); // Check that the hyperparameters as reported by the model are correct. REQUIRE(blr1.Omega().n_elem == 0); @@ -285,7 +285,7 @@ TEMPLATE_TEST_CASE("BayesianLinearRegressionTrainVariantTest", size_t nDims = 5, nPoints = 100; GenerateProblem(matX, y, nPoints, nDims, 0.5); - BayesianLinearRegression blr1, blr2, blr3, blr4, blr5; + BayesianLinearRegression<> blr1, blr2, blr3, blr4, blr5; blr1.Train(matX, y); blr2.Train(matX, y, false); diff --git a/src/mlpack/tests/main_tests/bayesian_linear_regression_test.cpp b/src/mlpack/tests/main_tests/bayesian_linear_regression_test.cpp index 2dbde86249..ed5b690852 100644 --- a/src/mlpack/tests/main_tests/bayesian_linear_regression_test.cpp +++ b/src/mlpack/tests/main_tests/bayesian_linear_regression_test.cpp @@ -41,8 +41,8 @@ TEST_CASE_METHOD(BRTestFixture, RUN_BINDING(); - BayesianLinearRegression* estimator = - params.Get("output_model"); + BayesianLinearRegression<>* estimator = + params.Get*>("output_model"); REQUIRE(estimator->DataOffset().n_elem == 0); REQUIRE(estimator->DataScale().n_elem == 0); @@ -61,7 +61,7 @@ TEST_CASE_METHOD(BRTestFixture, const arma::rowvec omega = arma::randu(m); arma::rowvec y = omega * matX; - BayesianLinearRegression model; + BayesianLinearRegression<> model; model.Train(matX, y); arma::rowvec responses; @@ -72,8 +72,8 @@ TEST_CASE_METHOD(BRTestFixture, RUN_BINDING(); - BayesianLinearRegression* mOut = - params.Get("output_model"); + BayesianLinearRegression<>* mOut = + params.Get*>("output_model"); ResetSettings(); @@ -101,7 +101,7 @@ TEST_CASE_METHOD(BRTestFixture, const arma::rowvec omega = arma::randu(m); arma::rowvec y = omega * matX; - BayesianLinearRegression model; + BayesianLinearRegression<> model; model.Train(matX, y); arma::rowvec responses; @@ -121,7 +121,7 @@ TEST_CASE_METHOD(BRTestFixture, // An error should occur. SetInputParam("input", std::move(matX)); SetInputParam("input_model", - params.Get("output_model")); + params.Get*>("output_model")); SetInputParam("test", std::move(matXtest)); REQUIRE_THROWS_AS(RUN_BINDING(), std::runtime_error); diff --git a/src/mlpack/tests/serialization_test.cpp b/src/mlpack/tests/serialization_test.cpp index da4f87a6d7..cd2a25a7d8 100644 --- a/src/mlpack/tests/serialization_test.cpp +++ b/src/mlpack/tests/serialization_test.cpp @@ -1552,12 +1552,12 @@ TEST_CASE("BayesianLinearRegressionTest", "[SerializationTest]") arma::vec omega = arma::randn(75, 1); arma::rowvec y = omega.t() * matX; - BayesianLinearRegression blr(false, false); + BayesianLinearRegression<> blr(false, false); blr.Train(matX, y); arma::vec omegaOpt = blr.Omega(); // Now, serialize. - BayesianLinearRegression xmlBlr(false, false), binaryBlr(false, false), + BayesianLinearRegression<> xmlBlr(false, false), binaryBlr(false, false), textBlr(false, false); SerializeObjectAll(blr, xmlBlr, binaryBlr, textBlr); From c7eea92d6c35fac69d9169c4caed5de4556d84fa Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Fri, 15 Dec 2023 09:38:47 -0500 Subject: [PATCH 67/91] Bump serialization version. --- .../bayesian_linear_regression/bayesian_linear_regression.hpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/mlpack/methods/bayesian_linear_regression/bayesian_linear_regression.hpp b/src/mlpack/methods/bayesian_linear_regression/bayesian_linear_regression.hpp index 958ce41417..5511448730 100644 --- a/src/mlpack/methods/bayesian_linear_regression/bayesian_linear_regression.hpp +++ b/src/mlpack/methods/bayesian_linear_regression/bayesian_linear_regression.hpp @@ -453,6 +453,9 @@ class BayesianLinearRegression } // namespace mlpack +CEREAL_TEMPLATE_CLASS_VERSION((typename ModelMatType), + (mlpack::BayesianLinearRegression), (1)); + // Include implementation of serialize. #include "bayesian_linear_regression_impl.hpp" From 926aea63235ed5a5e14b5b75cbe36390714cb81e Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Fri, 15 Dec 2023 09:39:51 -0500 Subject: [PATCH 68/91] Fix problematic examples; remove bits about sparse support. --- .../methods/bayesian_linear_regression.md | 23 ++++++------------- 1 file changed, 7 insertions(+), 16 deletions(-) diff --git a/doc/user/methods/bayesian_linear_regression.md b/doc/user/methods/bayesian_linear_regression.md index 237e37c4b5..6f9b9e7ef0 100644 --- a/doc/user/methods/bayesian_linear_regression.md +++ b/doc/user/methods/bayesian_linear_regression.md @@ -8,7 +8,7 @@ to control the data type used for storing the model. #### Simple usage example: ```c++ -// Train a linear regression model on random data and make predictions. +// Train a Bayesian linear regression model on random data and make predictions. // All data and responses are uniform random; this uses 10 dimensional data. // Replace with a data::Load() call or similar for a real application. @@ -288,20 +288,15 @@ BayesianLinearRegression `ModelMatType` specifies the type of matrix used for the internal representation of model parameters. Any matrix type that implements the Armadillo API can be -used. +used; however, the matrix should be dense, as in general +`BayesianLinearRegression` will produce models that are not sparse. -Note that the `Train()` and `Predict()` functions themselves are templatized and -can allow any matrix type that has the same element type. So, for instance, a -`BayesianLinearRegression` can accept an `arma::sp_mat` for training. - -The example below trains a Bayesian linear regression model on sparse 32-bit -floating point data, but uses a dense 32-bit floating point vector to store the -model itself. +The example below trains a Bayesian linear regression model on 32-bit floating +point data. ```c++ // Create random, sparse 100-dimensional data. -arma::sp_fmat dataset; -dataset.sprandu(100, 5000, 0.3); +arma::fmat dataset(100, 5000, arma::fill::randu); // Generate noisy responses from random data. arma::fvec trueWeights(100, arma::fill::randu); @@ -315,8 +310,7 @@ blr.MaxIterations() = 75; blr.Train(dataset, responses); // Compute the RMSE on the training set and a random test set. -arma::sp_fmat testDataset; -testDataset.sprandu(100, 1000, 0.3); +arma::fmat testDataset(100, 1000, arma::fill::randu); arma::frowvec testResponses = trueWeights.t() * testDataset + 0.01 * arma::randu(1000) /* noise term */; @@ -326,6 +320,3 @@ std::cout << "RMSE on training set: " std::cout << "RMSE on test set: " << blr.RMSE(testDataset, testResponses) << "." << std::endl; ``` - -***Note:*** dense objects should be used for `ModelMatType`, since in general -`BayesianLinearRegression` will produce models that are not sparse. From 36da8c07a3460b537355e6e5b3513d2aa6c215c5 Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Fri, 15 Dec 2023 09:47:56 -0500 Subject: [PATCH 69/91] Minor cleanup. --- doc/user/methods/bayesian_linear_regression.md | 4 ---- 1 file changed, 4 deletions(-) diff --git a/doc/user/methods/bayesian_linear_regression.md b/doc/user/methods/bayesian_linear_regression.md index 6f9b9e7ef0..906ac9a533 100644 --- a/doc/user/methods/bayesian_linear_regression.md +++ b/doc/user/methods/bayesian_linear_regression.md @@ -57,7 +57,6 @@ std::cout << arma::accu(predictions < 0) << " test points predicted to have " --- - * `blr = BayesianLinearRegression(data, responses)` * `blr = BayesianLinearRegression(data, responses, centerData=true, scaleData=false, maxIterations=50, tolerance=1e-4)` - Train model on the given data. @@ -98,9 +97,6 @@ If training is not done as part of the constructor call, it can be done with the `Train()` function: * `blr.Train(data, responses, centerData=true, scaleData=false, maxIterations=50, tolerance=1e-4)` - - Train model on the given data. - ---- Types of each argument are the same as in the table for constructors [above](#constructor-parameters). From 0b96ee2cf42eccf27642a20e334c108b404c99ed Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Tue, 19 Dec 2023 15:58:49 -0500 Subject: [PATCH 70/91] Relax error condition for floating-point matrices. --- src/mlpack/tests/lars_test.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mlpack/tests/lars_test.cpp b/src/mlpack/tests/lars_test.cpp index 23703b0823..c9c4d57e90 100644 --- a/src/mlpack/tests/lars_test.cpp +++ b/src/mlpack/tests/lars_test.cpp @@ -1113,7 +1113,7 @@ TEMPLATE_TEST_CASE("LARSSelectBetaTest", "[LARSTest]", arma::fmat, arma::mat) // Now step through numerous different lambda values. ElemType lastError = std::numeric_limits::max(); const ElemType errorTol = (std::is_same::value) ? 1e-10 : - 1e-5; + 1e-3; for (ElemType i = 5.0; i >= -5.0; i -= 0.1) { const ElemType selLambda1 = std::pow(10.0, (ElemType) i); From b74b8bc47617e16b4f77826fc61d8bea1f8e05d6 Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Tue, 19 Dec 2023 21:06:41 +0000 Subject: [PATCH 71/91] Detail fitness functions a little more. --- doc/user/methods/hoeffding_tree.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/doc/user/methods/hoeffding_tree.md b/doc/user/methods/hoeffding_tree.md index 003e2bd3af..17689d8933 100644 --- a/doc/user/methods/hoeffding_tree.md +++ b/doc/user/methods/hoeffding_tree.md @@ -514,6 +514,11 @@ Below, details are given for the requirements of each of these template types. * Specifies the fitness function to use when learning a decision tree. * The `GiniImpurity` _(default)_ and `HoeffdingInformationGain` classes are available for drop-in usage. + * `GiniImpurity` uses the [Gini impurity](https://en.wikipedia.org/wiki/Decision_tree_learning#Gini_impurity), + which measures the probability of a randomly labeled random element in the + split nodes being correctly labeled. + * `HoeffdingInformationGain` uses the [information gain](https://en.wikipedia.org/wiki/Decision_tree_learning#Information_gain), + which is based on the information-theoretic entropy of the possible splits. * A custom class must implement two functions: ```c++ From 621ade2616b708e3c578c53527d7ce2c8e3c6fa0 Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Tue, 19 Dec 2023 21:11:45 +0000 Subject: [PATCH 72/91] Fix return type of optimizer call. --- src/mlpack/methods/linear_svm/linear_svm_impl.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mlpack/methods/linear_svm/linear_svm_impl.hpp b/src/mlpack/methods/linear_svm/linear_svm_impl.hpp index 3d3865ca1e..c3046e7546 100644 --- a/src/mlpack/methods/linear_svm/linear_svm_impl.hpp +++ b/src/mlpack/methods/linear_svm/linear_svm_impl.hpp @@ -258,7 +258,7 @@ typename LinearSVM::ElemType LinearSVM::Train( parameters = svm.InitialPoint(); // Train the model. - const double out = optimizer.Optimize(svm, parameters, callbacks...); + const ElemType out = optimizer.Optimize(svm, parameters, callbacks...); Log::Info << "LinearSVM::LinearSVM(): final objective of " << "trained model is " << out << "." << std::endl; From 0696cf15c3f32ca61b0361f7d66050f5784903af Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Tue, 19 Dec 2023 21:25:07 +0000 Subject: [PATCH 73/91] Avoid unintentional casting. --- .../methods/linear_svm/linear_svm_function.hpp | 3 ++- .../methods/linear_svm/linear_svm_function_impl.hpp | 12 ++++++++---- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/src/mlpack/methods/linear_svm/linear_svm_function.hpp b/src/mlpack/methods/linear_svm/linear_svm_function.hpp index 05a5910aa5..20ef0cc24b 100644 --- a/src/mlpack/methods/linear_svm/linear_svm_function.hpp +++ b/src/mlpack/methods/linear_svm/linear_svm_function.hpp @@ -190,7 +190,8 @@ class LinearSVMFunction //! Label matrix for provided data SparseMatType groundTruth; - //! The datapoints for training. This will be an alias until Shuffle(). + //! The datapoints for training. This will be an alias until Shuffle() is + //! called. MatType dataset; //! Number of Classes. diff --git a/src/mlpack/methods/linear_svm/linear_svm_function_impl.hpp b/src/mlpack/methods/linear_svm/linear_svm_function_impl.hpp index 277593b06f..4a6df4232a 100644 --- a/src/mlpack/methods/linear_svm/linear_svm_function_impl.hpp +++ b/src/mlpack/methods/linear_svm/linear_svm_function_impl.hpp @@ -188,7 +188,8 @@ LinearSVMFunction::Evaluate( loss = arma::accu(arma::clamp(margin, 0.0, DBL_MAX)) / dataset.n_cols; // Adding the regularization term. - regularization = 0.5 * lambda * arma::dot(parameters, parameters); + constexpr ElemType half = ((ElemType) 0.5); + regularization = half * lambda * arma::dot(parameters, parameters); return loss + regularization; } @@ -230,7 +231,8 @@ LinearSVMFunction::Evaluate( loss /= batchSize; // Adding the regularization term. - regularization = 0.5 * lambda * arma::dot(parameters, parameters); + constexpr ElemType half = ((ElemType) 0.5); + regularization = half * lambda * arma::dot(parameters, parameters); cost = loss + regularization; return cost; @@ -416,7 +418,8 @@ LinearSVMFunction::EvaluateWithGradient( loss /= dataset.n_cols; // Adding the regularization term. - regularization = 0.5 * lambda * arma::dot(parameters, parameters); + constexpr ElemType half = ((ElemType) 0.5); + regularization = half * lambda * arma::dot(parameters, parameters); cost = loss + regularization; return cost; @@ -489,7 +492,8 @@ LinearSVMFunction::EvaluateWithGradient( loss /= batchSize; // Adding the regularization term. - regularization = 0.5 * lambda * arma::dot(parameters, parameters); + constexpr ElemType half = ((ElemType) 0.5); + regularization = half * lambda * arma::dot(parameters, parameters); cost = loss + regularization; return cost; From 2a12d4a461bccf140090300d2f336fadf53fb349 Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Tue, 19 Dec 2023 21:28:32 +0000 Subject: [PATCH 74/91] Avoid unintentional casts to double. --- .../bayesian_linear_regression_impl.hpp | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/mlpack/methods/bayesian_linear_regression/bayesian_linear_regression_impl.hpp b/src/mlpack/methods/bayesian_linear_regression/bayesian_linear_regression_impl.hpp index f24f70fa25..40762cd501 100644 --- a/src/mlpack/methods/bayesian_linear_regression/bayesian_linear_regression_impl.hpp +++ b/src/mlpack/methods/bayesian_linear_regression/bayesian_linear_regression_impl.hpp @@ -156,11 +156,11 @@ BayesianLinearRegression::Train( const DenseVecType eigVecInvPhitT = eigVecInv * phi * t.t(); // Initialize the hyperparameters and begin with an infinitely broad prior. - alpha = 1e-6; - beta = 1 / (var(t, 1) * 0.1); + alpha = ((ElemType) 1e-6); + beta = ((ElemType) 1 / (var(t, 1) * 0.1)); unsigned short i = 0; - ElemType crit = 1.0; + ElemType crit = ((ElemType) 1.0); while (((double) crit > tolerance) && (i < maxIterations)) { @@ -185,7 +185,8 @@ BayesianLinearRegression::Train( i++; } // Compute the covariance matrix for the uncertainties later. - matCovariance = eigVec * diagmat(1 / (beta * eigVal + alpha)) * eigVecInv; + matCovariance = eigVec * diagmat(((ElemType) 1) / (beta * eigVal + alpha)) * + eigVecInv; return RMSE(data, responses); } From e92ff9455a9615b95df059167d15a0f93f68847e Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Tue, 19 Dec 2023 16:53:34 -0500 Subject: [PATCH 75/91] Fix build failure. --- src/mlpack/methods/lars/lars.hpp | 4 ++-- src/mlpack/methods/lars/lars_impl.hpp | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/mlpack/methods/lars/lars.hpp b/src/mlpack/methods/lars/lars.hpp index 9aa8d122cd..dbde79254c 100644 --- a/src/mlpack/methods/lars/lars.hpp +++ b/src/mlpack/methods/lars/lars.hpp @@ -708,8 +708,8 @@ class LARS void CholeskyInsert(ElemType sqNormNewX, const VecType& newGramCol); template - void GivensRotate(const typename arma::Col::fixed<2>& x, - typename arma::Col::fixed<2>& rotatedX, + void GivensRotate(const typename arma::Col::template fixed<2>& x, + typename arma::Col::template fixed<2>& rotatedX, MatType& G); void CholeskyDelete(const size_t colToKill); diff --git a/src/mlpack/methods/lars/lars_impl.hpp b/src/mlpack/methods/lars/lars_impl.hpp index 6ba607a47c..ee9bb7514f 100644 --- a/src/mlpack/methods/lars/lars_impl.hpp +++ b/src/mlpack/methods/lars/lars_impl.hpp @@ -1253,10 +1253,10 @@ template inline void LARS::GivensRotate( const typename arma::Col< typename LARS::ElemType - >::fixed<2>& x, + >::template fixed<2>& x, typename arma::Col< typename LARS::ElemType - >::fixed<2>& rotatedX, + >::template fixed<2>& rotatedX, MatType& matG) { if (x(1) == 0) From d8cdc5ef129e0089ff2938f448af484d2ed0f102 Mon Sep 17 00:00:00 2001 From: Omar Shrit Date: Wed, 20 Dec 2023 23:45:27 +0100 Subject: [PATCH 76/91] Fix the dependency on cube and mat from arma Signed-off-by: Omar Shrit --- .../rectifier_function.hpp | 24 +++++-------------- 1 file changed, 6 insertions(+), 18 deletions(-) diff --git a/src/mlpack/methods/ann/activation_functions/rectifier_function.hpp b/src/mlpack/methods/ann/activation_functions/rectifier_function.hpp index 4fc2edf11c..f444122dad 100644 --- a/src/mlpack/methods/ann/activation_functions/rectifier_function.hpp +++ b/src/mlpack/methods/ann/activation_functions/rectifier_function.hpp @@ -56,29 +56,17 @@ class RectifierFunction } /** - * Computes the rectifier function using a dense matrix as input. + * Computes the rectifier function using a 2nd /3rd-order tensor as input. * * @param x Input data. * @param y The resulting output activation. */ - template - static void Fn(const arma::Mat& x, arma::Mat& y) + template + static void Fn(const MatType& x, MatType& y) { - y.zeros(x.n_rows, x.n_cols); - y = arma::max(y, x); - } - - /** - * Computes the rectifier function using a 3rd-order tensor as input. - * - * @param x Input data. - * @param y The resulting output activation. - */ - template - static void Fn(const arma::Cube& x, arma::Cube& y) - { - y.zeros(x.n_rows, x.n_cols, x.n_slices); - y = arma::max(y, x); + y.set_size(size(x)); + y.zeros(); + y = max(y, x); } /** From fff4c87e797121610b42b2b8f83dff70b7f4543c Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Thu, 21 Dec 2023 08:56:11 -0500 Subject: [PATCH 77/91] Fix python test duplicate filename error. --- src/mlpack/bindings/python/CMakeLists.txt | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/mlpack/bindings/python/CMakeLists.txt b/src/mlpack/bindings/python/CMakeLists.txt index e443a68232..2668c6422d 100644 --- a/src/mlpack/bindings/python/CMakeLists.txt +++ b/src/mlpack/bindings/python/CMakeLists.txt @@ -158,11 +158,16 @@ add_custom_command(TARGET python_copy PRE_BUILD ${CMAKE_CURRENT_SOURCE_DIR}/setup_readme.md ${CMAKE_BINARY_DIR}/src/mlpack/bindings/python/) -# Copy all mlpack headers for inclusion in the package. +# Copy all mlpack headers for inclusion in the package, but remove the bindings/ +# and tests/ directories as they should not be included. add_custom_command(TARGET python_copy PRE_BUILD COMMAND ${CMAKE_COMMAND} ARGS -E copy_directory ${CMAKE_SOURCE_DIR}/src/ ${CMAKE_BINARY_DIR}/src/mlpack/bindings/python/mlpack/include/) +add_custom_command(TARGET python_copy PRE_BUILD + COMMAND ${CMAKE_COMMAND} ARGS -E rm -r + ${CMAKE_BINARY_DIR}/src/mlpack/bindings/python/mlpack/include/mlpack/bindings/ + ${CMAKE_BINARY_DIR}/src/mlpack/bindings/python/mlpack/include/mlpack/tests/) # Generate pkgconfig file for easy use of included headers. add_custom_target(python_pkgconfig From a65153795b2e163fcd3144212807437aea73683d Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Thu, 21 Dec 2023 11:29:02 -0500 Subject: [PATCH 78/91] Fix build failure (attempt 2). --- src/mlpack/methods/lars/lars_impl.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mlpack/methods/lars/lars_impl.hpp b/src/mlpack/methods/lars/lars_impl.hpp index ee9bb7514f..513bc51308 100644 --- a/src/mlpack/methods/lars/lars_impl.hpp +++ b/src/mlpack/methods/lars/lars_impl.hpp @@ -1300,7 +1300,7 @@ inline void LARS::CholeskyDelete(const size_t colToKill) for (size_t k = colToKill; k < n; ++k) { DenseMatType matG; - typename arma::Col::fixed<2> rotatedVec; + typename arma::Col::template fixed<2> rotatedVec; GivensRotate(matUtriCholFactor(arma::span(k, k + 1), k), rotatedVec, matG); matUtriCholFactor(arma::span(k, k + 1), k) = rotatedVec; From d9f26303b77a26f6655ddaac732a7c6218db6d63 Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Thu, 21 Dec 2023 11:29:38 -0500 Subject: [PATCH 79/91] Relax tolerance for floating-point tests. --- src/mlpack/tests/lars_test.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mlpack/tests/lars_test.cpp b/src/mlpack/tests/lars_test.cpp index c9c4d57e90..3dba356fc4 100644 --- a/src/mlpack/tests/lars_test.cpp +++ b/src/mlpack/tests/lars_test.cpp @@ -232,7 +232,7 @@ TEMPLATE_TEST_CASE("PredictTest", "[LARSTest]", arma::fmat, arma::mat) arma::Col adjPred = X * predictions.t(); const ElemType tol = (std::is_same::value) ? 1e-7 : - 1e-4; + 1e-3; REQUIRE(predictions.n_elem == 1000); for (size_t i = 0; i < betaOptPred.n_elem; ++i) From f5a33d3a8ba011d10549ced622e982a30303c42b Mon Sep 17 00:00:00 2001 From: Omar Shrit Date: Thu, 21 Dec 2023 18:30:46 +0100 Subject: [PATCH 80/91] Disable macro expansion and force the compiler to use ADL Signed-off-by: Omar Shrit --- .../methods/ann/activation_functions/rectifier_function.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mlpack/methods/ann/activation_functions/rectifier_function.hpp b/src/mlpack/methods/ann/activation_functions/rectifier_function.hpp index f444122dad..0a372e23d5 100644 --- a/src/mlpack/methods/ann/activation_functions/rectifier_function.hpp +++ b/src/mlpack/methods/ann/activation_functions/rectifier_function.hpp @@ -22,7 +22,7 @@ */ #ifndef MLPACK_METHODS_ANN_ACTIVATION_FUNCTIONS_RECTIFIER_FUNCTION_HPP #define MLPACK_METHODS_ANN_ACTIVATION_FUNCTIONS_RECTIFIER_FUNCTION_HPP - +#undef max #include #include From 27099654308b89f5cf4cc8d8156ba73f65740c53 Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Sat, 23 Dec 2023 16:20:26 -0500 Subject: [PATCH 81/91] Relax tolerance even more, because sometimes the newError value is quite large. --- src/mlpack/tests/lars_test.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/mlpack/tests/lars_test.cpp b/src/mlpack/tests/lars_test.cpp index 3dba356fc4..3f115ca8b0 100644 --- a/src/mlpack/tests/lars_test.cpp +++ b/src/mlpack/tests/lars_test.cpp @@ -1112,8 +1112,8 @@ TEMPLATE_TEST_CASE("LARSSelectBetaTest", "[LARSTest]", arma::fmat, arma::mat) // Now step through numerous different lambda values. ElemType lastError = std::numeric_limits::max(); - const ElemType errorTol = (std::is_same::value) ? 1e-10 : - 1e-3; + const ElemType errorTol = (std::is_same::value) ? 1e-8 : + 0.05; for (ElemType i = 5.0; i >= -5.0; i -= 0.1) { const ElemType selLambda1 = std::pow(10.0, (ElemType) i); From 4dfbc142a8be981c1df93cb8a29aeeaec82e341d Mon Sep 17 00:00:00 2001 From: Arun Date: Tue, 26 Dec 2023 22:17:16 -0800 Subject: [PATCH 82/91] Use GITHUB_OUTPUT envvar instead of set-output command as the latter is deprecated --- .github/workflows/main.yml | 133 +++++++++++++++------------- .github/workflows/update-catch.yaml | 8 +- .github/workflows/update-cli11.yaml | 10 +-- 3 files changed, 78 insertions(+), 73 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 9ca13d02a7..97bb2e27f8 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -24,76 +24,76 @@ jobs: r_bindings: ${{ steps.mlpack_version.outputs.mlpack_r_package }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v3 - - name: Extract mlpack version - id: mlpack_version - run: | - MLPACK_VERSION_MAJOR=$(grep -i ".*#define MLPACK_VERSION_MAJOR.*" src/mlpack/core/util/version.hpp | grep -o "[0-9]*") - MLPACK_VERSION_MINOR=$(grep -i ".*#define MLPACK_VERSION_MINOR.*" src/mlpack/core/util/version.hpp | grep -o "[0-9]*") - MLPACK_VERSION_PATCH=$(grep -i ".*#define MLPACK_VERSION_PATCH.*" src/mlpack/core/util/version.hpp | grep -o "[0-9]*") - MLPACK_VERSION_VALUE=${MLPACK_VERSION_MAJOR}.${MLPACK_VERSION_MINOR}.${MLPACK_VERSION_PATCH} - echo ::set-output name=mlpack_r_package::$(echo mlpack_"$MLPACK_VERSION_VALUE".tar.gz) + - name: Extract mlpack version + id: mlpack_version + run: | + MLPACK_VERSION_MAJOR=$(grep -i ".*#define MLPACK_VERSION_MAJOR.*" src/mlpack/core/util/version.hpp | grep -o "[0-9]*") + MLPACK_VERSION_MINOR=$(grep -i ".*#define MLPACK_VERSION_MINOR.*" src/mlpack/core/util/version.hpp | grep -o "[0-9]*") + MLPACK_VERSION_PATCH=$(grep -i ".*#define MLPACK_VERSION_PATCH.*" src/mlpack/core/util/version.hpp | grep -o "[0-9]*") + MLPACK_VERSION_VALUE=${MLPACK_VERSION_MAJOR}.${MLPACK_VERSION_MINOR}.${MLPACK_VERSION_PATCH} + echo "mlpack_r_package=$(echo mlpack_"$MLPACK_VERSION_VALUE".tar.gz)" >> $GITHUB_OUTPUT - # Setup Pandoc - - uses: r-lib/actions/setup-pandoc@v2 + # Setup Pandoc + - uses: r-lib/actions/setup-pandoc@v2 - # Setup R actions - - uses: r-lib/actions/setup-r@v2 + # Setup R actions + - uses: r-lib/actions/setup-r@v2 - - name: Query dependencies - run: | - cp src/mlpack/bindings/R/mlpack/DESCRIPTION.in DESCRIPTION - Rscript -e "install.packages('remotes')" -e "saveRDS(remotes::dev_package_deps(dependencies = TRUE), 'depends.Rds')" + - name: Query dependencies + run: | + cp src/mlpack/bindings/R/mlpack/DESCRIPTION.in DESCRIPTION + Rscript -e "install.packages('remotes')" -e "saveRDS(remotes::dev_package_deps(dependencies = TRUE), 'depends.Rds')" - - name: Cache R packages - if: runner.os != 'Windows' && runner.os != 'macOS' - uses: actions/cache@v3 - with: - path: ${{ env.R_LIBS_USER }} - key: ${{ runner.os }}-r-release-${{ hashFiles('depends.Rds') }} - restore-keys: ${{ runner.os }}-r-release- + - name: Cache R packages + if: runner.os != 'Windows' && runner.os != 'macOS' + uses: actions/cache@v3 + with: + path: ${{ env.R_LIBS_USER }} + key: ${{ runner.os }}-r-release-${{ hashFiles('depends.Rds') }} + restore-keys: ${{ runner.os }}-r-release- - - name: Install Build Dependencies - run: | - sudo apt-get update - # We don't install cereal via apt, because the Debian packagers - # split the rapidjson dependency into a separate package. We will - # bundle the cereal sources with the R package, so we want them to - # be exactly the upstream sources (with rapidjson included). - sudo apt-get install -y --allow-unauthenticated libopenblas-dev liblapack-dev g++ libensmallen-dev libhdf5-dev libarmadillo-dev libcurl4-openssl-dev - wget https://github.com/USCiLab/cereal/archive/refs/tags/v1.3.2.tar.gz - tar -xvzpf v1.3.2.tar.gz - # These directives cause warnings on CRAN: - # https://github.com/USCiLab/cereal/blob/master/include/cereal/external/base64.hpp#L28-L31 - # The command below comments them out. - sed -i 's|#pragma|// #pragma|' cereal-1.3.2/include/cereal/external/base64.hpp + - name: Install Build Dependencies + run: | + sudo apt-get update + # We don't install cereal via apt, because the Debian packagers + # split the rapidjson dependency into a separate package. We will + # bundle the cereal sources with the R package, so we want them to + # be exactly the upstream sources (with rapidjson included). + sudo apt-get install -y --allow-unauthenticated libopenblas-dev liblapack-dev g++ libensmallen-dev libhdf5-dev libarmadillo-dev libcurl4-openssl-dev + wget https://github.com/USCiLab/cereal/archive/refs/tags/v1.3.2.tar.gz + tar -xvzpf v1.3.2.tar.gz + # These directives cause warnings on CRAN: + # https://github.com/USCiLab/cereal/blob/master/include/cereal/external/base64.hpp#L28-L31 + # The command below comments them out. + sed -i 's|#pragma|// #pragma|' cereal-1.3.2/include/cereal/external/base64.hpp - - name: Install R-bindings dependencies - run: | - remotes::install_deps(dependencies = TRUE) - remotes::install_cran("roxygen2") - remotes::install_cran("pkgbuild") - shell: Rscript {0} + - name: Install R-bindings dependencies + run: | + remotes::install_deps(dependencies = TRUE) + remotes::install_cran("roxygen2") + remotes::install_cran("pkgbuild") + shell: Rscript {0} - - name: CMake - run: | - mkdir build - cd build && cmake -DDEBUG=OFF -DPROFILE=OFF -DBUILD_CLI_EXECUTABLES=OFF -DBUILD_PYTHON_BINDINGS=OFF -DBUILD_JULIA_BINDINGS=OFF -DBUILD_GO_BINDINGS=OFF -DBUILD_R_BINDINGS=ON -DDOWNLOAD_DEPENDENCIES=ON -DBUILD_TESTS=ON -DCEREAL_INCLUDE_DIR=../cereal-1.3.2/include/ .. + - name: CMake + run: | + mkdir build + cd build && cmake -DDEBUG=OFF -DPROFILE=OFF -DBUILD_CLI_EXECUTABLES=OFF -DBUILD_PYTHON_BINDINGS=OFF -DBUILD_JULIA_BINDINGS=OFF -DBUILD_GO_BINDINGS=OFF -DBUILD_R_BINDINGS=ON -DDOWNLOAD_DEPENDENCIES=ON -DBUILD_TESTS=ON -DCEREAL_INCLUDE_DIR=../cereal-1.3.2/include/ .. - - name: Build - run: | - cd build && make + - name: Build + run: | + cd build && make - - name: Run tests via ctest - run: | - cd build && CTEST_OUTPUT_ON_FAILURE=1 ctest -T Test . + - name: Run tests via ctest + run: | + cd build && CTEST_OUTPUT_ON_FAILURE=1 ctest -T Test . - - name: Upload R packages - uses: actions/upload-artifact@v3 - with: - name: mlpack_r_tarball - path: build/src/mlpack/bindings/R/${{ steps.mlpack_version.outputs.mlpack_r_package }} + - name: Upload R packages + uses: actions/upload-artifact@v3 + with: + name: mlpack_r_tarball + path: build/src/mlpack/bindings/R/${{ steps.mlpack_version.outputs.mlpack_r_package }} R-CMD-check: needs: jobR @@ -106,9 +106,14 @@ jobs: fail-fast: false matrix: config: - - {os: windows-latest, r: 'release', name: 'Windows R'} - - {os: macOS-latest, r: 'release', name: 'macOS R'} - - {os: ubuntu-latest, r: 'devel', http-user-agent: 'release', name: 'Linux R'} + - { os: windows-latest, r: "release", name: "Windows R" } + - { os: macOS-latest, r: "release", name: "macOS R" } + - { + os: ubuntu-latest, + r: "devel", + http-user-agent: "release", + name: "Linux R", + } env: MAKEFLAGS: "-j 2" @@ -146,8 +151,8 @@ jobs: - name: Install check dependencies if: runner.os != 'Windows' && runner.os != 'macOS' run: | - sudo apt-get update - sudo apt-get install -y --allow-unauthenticated libcurl4-openssl-dev + sudo apt-get update + sudo apt-get install -y --allow-unauthenticated libcurl4-openssl-dev - name: Install dependencies run: | diff --git a/.github/workflows/update-catch.yaml b/.github/workflows/update-catch.yaml index eb628a0d1c..ab9efdae8c 100644 --- a/.github/workflows/update-catch.yaml +++ b/.github/workflows/update-catch.yaml @@ -6,7 +6,7 @@ name: Update Catch on: workflow_dispatch: schedule: - - cron: '0 10 1/16 * *' + - cron: "0 10 1/16 * *" permissions: contents: read @@ -16,7 +16,7 @@ jobs: contents: write # for peter-evans/create-pull-request to create branch pull-requests: write # for peter-evans/create-pull-request to create a PR if: ${{ false }} -# if: ${{ github.repository == 'mlpack/mlpack' }} + # if: ${{ github.repository == 'mlpack/mlpack' }} runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 @@ -26,7 +26,7 @@ jobs: # Ping version information upstream. CATCH_RELEASE_JSON=$(curl -sL https://api.github.com/repos/catchorg/Catch2/releases/latest) CATCH_RELEASE_VERSION=$(jq -r ".tag_name" <<< "$CATCH_RELEASE_JSON" | tr -d v) - echo ::set-output name=release_tag::$(echo $CATCH_RELEASE_VERSION) + echo "release_tag=$(echo $CATCH_RELEASE_VERSION)" >> $GITHUB_OUTPUT # Extract out version information from git repository. CATCH_VERSION_MAJOR=$(grep -i ".*#define CATCH_VERSION_MAJOR.*" src/mlpack/tests/catch.hpp | grep -o "[0-9]*") CATCH_VERSION_MINOR=$(grep -i ".*#define CATCH_VERSION_MINOR.*" src/mlpack/tests/catch.hpp | grep -o "[0-9]*") @@ -34,7 +34,7 @@ jobs: # Combine values to match release tag information. CATCH_VERSION_VALUE=${CATCH_VERSION_MAJOR}.${CATCH_VERSION_MINOR}.${CATCH_VERSION_PATCH} # Set the current release tag. - echo ::set-output name=current_tag::$(echo $CATCH_VERSION_VALUE) + echo "current_tag=$(echo $CATCH_VERSION_VALUE)" >> $GITHUB_OUTPUT - name: Update Catch if: steps.catch-header.outputs.current_tag != steps.catch-header.outputs.release_tag diff --git a/.github/workflows/update-cli11.yaml b/.github/workflows/update-cli11.yaml index 21d3f917ed..c0c282f1cf 100644 --- a/.github/workflows/update-cli11.yaml +++ b/.github/workflows/update-cli11.yaml @@ -2,15 +2,15 @@ name: Update CLI11 on: workflow_dispatch: schedule: - - cron: '0 10 1/16 * *' + - cron: "0 10 1/16 * *" permissions: contents: read jobs: updateCLI11: permissions: - contents: write # for peter-evans/create-pull-request to create branch - pull-requests: write # for peter-evans/create-pull-request to create a PR + contents: write # for peter-evans/create-pull-request to create branch + pull-requests: write # for peter-evans/create-pull-request to create a PR if: ${{ github.repository == 'mlpack/mlpack' }} runs-on: ubuntu-latest steps: @@ -21,11 +21,11 @@ jobs: # Ping version information upstream. CLI11_RELEASE_JSON=$(curl -sL https://api.github.com/repos/CLIUtils/CLI11/releases/latest) CLI11_RELEASE_VERSION=$(jq -r ".tag_name" <<< "$CLI11_RELEASE_JSON" | tr -d v) - echo ::set-output name=release_tag::$(echo $CLI11_RELEASE_VERSION) + echo "release_tag=$(echo $CLI11_RELEASE_VERSION)" >> $GITHUB_OUTPUT # Extract out version information from git repository. CLI11_VERSION_VALUE=$(grep -i ".*#define CLI11_VERSION.*" src/mlpack/bindings/cli/third_party/CLI/CLI11.hpp | grep -Po "(\d+\.)+\d+") # Set the current release tag. - echo ::set-output name=current_tag::$(echo $CLI11_VERSION_VALUE) + echo "current_tag=$(echo $CLI11_VERSION_VALUE)" >> $GITHUB_OUTPUT - name: Update CLI11 if: steps.cli11-header.outputs.current_tag != steps.cli11-header.outputs.release_tag From 2b4262682e276161e9855b2e58a16f4d5b3aaf2b Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Sun, 31 Dec 2023 13:26:50 -0500 Subject: [PATCH 83/91] Adjust tolerances a little bit. --- src/mlpack/methods/lars/lars_impl.hpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/mlpack/methods/lars/lars_impl.hpp b/src/mlpack/methods/lars/lars_impl.hpp index 513bc51308..1893b50a63 100644 --- a/src/mlpack/methods/lars/lars_impl.hpp +++ b/src/mlpack/methods/lars/lars_impl.hpp @@ -659,11 +659,12 @@ LARS::Train(const MatType& matX, break; // Floats require a really large tolerance for this condition. - const ElemType tol = (std::is_same::value) ? 1e-8 : 0.01; + const ElemType tol = (std::is_same::value) ? 1e-6 : 0.01; if ((matGram != &matGramInternal) && ((maxActiveCorr - minActiveCorr) / maxActiveCorr) > tol) { // Construct the error message to match the user's settings. + std::cout << "maxActiveCorr: " << maxActiveCorr << " minActiveCorr: " << minActiveCorr << "; result " << ((maxActiveCorr - minActiveCorr) / maxActiveCorr) << "; tol " << tol << "\n"; std::ostringstream oss; oss << "LARS::Train(): correlation conditions violated; check that your " << "given Gram matrix is properly computed on "; From 0eb51d3e91f6459ed24b2a2dda1b48caedc69037 Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Sun, 31 Dec 2023 13:27:06 -0500 Subject: [PATCH 84/91] Fix compilation warnings. --- src/mlpack/bindings/python/mlpack/io_util.hpp | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/mlpack/bindings/python/mlpack/io_util.hpp b/src/mlpack/bindings/python/mlpack/io_util.hpp index 8c2ad26da0..4b9f956419 100644 --- a/src/mlpack/bindings/python/mlpack/io_util.hpp +++ b/src/mlpack/bindings/python/mlpack/io_util.hpp @@ -22,16 +22,14 @@ namespace util { // Utility functions to correctly handle transposed Armadillo matrices. template inline void TransposeIfNeeded( - const std::string& identifier, - T& value, - bool transpose) + T& /* value */, + bool /* transpose */) { // No transpose needed for non-matrices. return; } inline void TransposeIfNeeded( - const std::string& identifier, arma::mat& value, bool transpose) { @@ -59,7 +57,7 @@ inline void SetParam(util::Params& params, T& value, bool transpose = false) { - TransposeIfNeeded(identifier, value, transpose); + TransposeIfNeeded(value, transpose); params.Get(identifier) = std::move(value); } From bccc337756bcf6186f8a7951fa06474cf45e13bb Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Sun, 31 Dec 2023 13:27:36 -0500 Subject: [PATCH 85/91] Use the stripped name of the C++ class instead of the full name, so that we don't include template parameters. --- .../bindings/python/print_wrapper_py.cpp | 170 ++++++++++-------- 1 file changed, 95 insertions(+), 75 deletions(-) diff --git a/src/mlpack/bindings/python/print_wrapper_py.cpp b/src/mlpack/bindings/python/print_wrapper_py.cpp index 3ed5bc2976..b35d7553e7 100644 --- a/src/mlpack/bindings/python/print_wrapper_py.cpp +++ b/src/mlpack/bindings/python/print_wrapper_py.cpp @@ -13,6 +13,7 @@ #include "print_wrapper_py.hpp" #include "get_arma_type.hpp" #include "wrapper_functions.hpp" +#include "strip_type.hpp" #include using namespace mlpack::util; @@ -42,18 +43,18 @@ void PrintWrapperPY(const std::string& category, // keeps track of indentation at each point. int indent = 0; - for(string i: methods) + for (string i: methods) { params[i] = IO::Parameters(groupName + "_" + i); } - if(category == "regression") + if (category == "regression") { cout << "class BaseEstimator:" << endl; cout << " pass" << endl; cout << endl; } - else if(category == "classification") + else if (category == "classification") { cout << "class BaseEstimator:" << endl; cout << " pass" << endl; @@ -64,7 +65,7 @@ void PrintWrapperPY(const std::string& category, } // Import different mlpack programs that are to be wrapped. - for(size_t i = 0; i < methods.size(); i++) + for (size_t i = 0; i < methods.size(); i++) { cout << "from mlpack." << groupName << "_" << methods[i] << " "; cout << "import " << groupName << "_" << methods[i] << endl; @@ -72,19 +73,19 @@ void PrintWrapperPY(const std::string& category, // Try importing scikit-learn, only for classification and // regression. - if(category == "regression") + if (category == "regression") { cout << "try:" << endl; cout << " from sklearn.base import BaseEstimator" << endl; } - else if(category == "classification") + else if (category == "classification") { cout << "try: " << endl; cout << " from sklearn.base import BaseEstimator, ClassifierMixin"; cout << endl; } - if(category == "regression" || category == "classification") + if (category == "regression" || category == "classification") { cout << "except:" << endl; cout << " pass" << endl; @@ -93,30 +94,34 @@ void PrintWrapperPY(const std::string& category, // Check and store if every parameter is serializable, // a hyperparameter and boolean. - for(map::iterator i=params.begin(); - i!=params.end(); i++) + for (map::iterator i = params.begin(); + i != params.end(); i++) { map methodParams = i->second.Parameters(); - for(map::iterator itr=methodParams.begin(); - itr!=methodParams.end(); itr++) + for (map::iterator itr = methodParams.begin(); + itr != methodParams.end(); itr++) { + // Get the sanitized name of the class. + string strippedName, unused1, unused2; + StripType(itr->second.cppType, strippedName, unused1, unused2); + // Checking for serializability. bool isSerial; i->second.functionMap[itr->second.tname]["IsSerializable"]( itr->second, NULL, (void*)& isSerial); - if(isSerial) - serializable.insert(itr->second.cppType); + if (isSerial) + serializable.insert(strippedName); // Checking for hyperparameter. bool isHyperParam = false; size_t foundArma = itr->second.cppType.find("arma"); - if(itr->second.input && foundArma == string::npos && !isSerial) + if (itr->second.input && foundArma == string::npos && !isSerial) isHyperParam = true; hyperParams[itr->first] = isHyperParam; // Checking for boolean. - if(itr->second.cppType == "bool") + if (itr->second.cppType == "bool") isBool[itr->first] = true; else isBool[itr->first] = false; @@ -126,9 +131,9 @@ void PrintWrapperPY(const std::string& category, string className = GetClassName(groupName); // print class. - if(category == "regression") + if (category == "regression") cout << "class " << className << "(BaseEstimator)" << ":" << endl; - else if(category == "classification") + else if (category == "classification") { cout << "class " << className << "(BaseEstimator, ClassifierMixin)" << ":" << endl; @@ -140,15 +145,15 @@ void PrintWrapperPY(const std::string& category, indent += 2; cout << string(indent, ' ') << "def __init__(self," << endl; - for(auto itr=hyperParams.begin(); itr!=hyperParams.end(); itr++) + for (auto itr = hyperParams.begin(); itr != hyperParams.end(); itr++) { // indent + 13, here 13 -> def __init__( - if(itr->second && isBool[itr->first]) + if (itr->second && isBool[itr->first]) { cout << string(indent+13, ' ') << GetValidName(itr->first) << " = False," << endl; } - else if(itr->second && !isBool[itr->first]) + else if (itr->second && !isBool[itr->first]) { cout << string(indent+13, ' ') << GetValidName(itr->first) << " = None," << endl; @@ -161,10 +166,10 @@ void PrintWrapperPY(const std::string& category, // storing given arguments in attributes. indent += 2; cout << string(indent, ' ') << "# serializable attributes." << endl; - if(serializable.size() == 0) + if (serializable.size() == 0) cout << string(indent, ' ') << "# None" << endl; - for(auto itr=serializable.begin(); itr!=serializable.end(); itr++) + for (auto itr = serializable.begin(); itr != serializable.end(); itr++) { cout << string(indent, ' '); cout << "self._" << *itr << " = None" << endl; @@ -173,9 +178,9 @@ void PrintWrapperPY(const std::string& category, cout << endl; cout << string(indent, ' ') << "# hyper-parameters." << endl; - for(auto itr=hyperParams.begin(); itr!=hyperParams.end(); itr++) + for (auto itr = hyperParams.begin(); itr != hyperParams.end(); itr++) { - if(itr->second) + if (itr->second) { string validName = GetValidName(itr->first); cout << string(indent, ' '); @@ -187,7 +192,7 @@ void PrintWrapperPY(const std::string& category, indent -= 2; // print all method definitions. - for(string methodName: methods) + for (string methodName: methods) { // print method name. cout << string(indent, ' ') << "def " << GetMappedName(methodName); @@ -205,39 +210,44 @@ void PrintWrapperPY(const std::string& category, int numMatrixInputs = 0; int numVectorInputs = 0; - if(category == "regression" || category == "classification") + if (category == "regression" || category == "classification") { - for(map::iterator itr=methodParams.begin(); - itr!=methodParams.end(); itr++) + for (map::iterator itr = methodParams.begin(); + itr != methodParams.end(); itr++) { // Throw error if there are more than one matrix input params, // or more than one vector input params. - if(numMatrixInputs > 1) - Log::Fatal << "More than one matrix input parameters for " << - methodName << "(" << GetMappedName(methodName) << ") method!" << - endl; - if(numVectorInputs > 1) - Log::Fatal << "More than one vector input parameters for " << + if (numMatrixInputs > 1) + { + Log::Fatal << "More than one matrix input parameter for " << methodName << "(" << GetMappedName(methodName) << ") method!" << endl; + } - if(itr->second.input) + if (numVectorInputs > 1) + { + Log::Fatal << "More than one vector input parameter for " << + methodName << "(" << GetMappedName(methodName) << ") method!" << + endl; + } + + if (itr->second.input) { // If this is a matrix parameter. - if(itr->second.cppType == "arma::mat" || - itr->second.cppType == - "std::tuple" || - itr->second.cppType == "arma::Mat") + if (itr->second.cppType == "arma::mat" || + itr->second.cppType == + "std::tuple" || + itr->second.cppType == "arma::Mat") { numMatrixInputs++; mapToScikitNames[itr->first] = "X"; invMapToScikitNames["X"] = itr->first; } // If this is a vector parameter. - else if(itr->second.cppType == "arma::vec" || - itr->second.cppType == "arma::rowvec" || - itr->second.cppType == "arma::Row" || - itr->second.cppType == "arma::Col") + else if (itr->second.cppType == "arma::vec" || + itr->second.cppType == "arma::rowvec" || + itr->second.cppType == "arma::Row" || + itr->second.cppType == "arma::Col") { numVectorInputs++; mapToScikitNames[itr->first] = "y"; @@ -248,13 +258,13 @@ void PrintWrapperPY(const std::string& category, } // Now we print the matrix and vector params. - if(category == "classification" || category == "regression") + if (category == "classification" || category == "regression") { bool hasX = false; bool hasy = false; // First print X, if it is present. - if(invMapToScikitNames.find("X") != invMapToScikitNames.end()) + if (invMapToScikitNames.find("X") != invMapToScikitNames.end()) { cout << string(indent + addIndent, ' '); cout << "X = None," << endl; @@ -262,25 +272,25 @@ void PrintWrapperPY(const std::string& category, } // Now print y, if it is present. - if(invMapToScikitNames.find("y") != invMapToScikitNames.end()) + if (invMapToScikitNames.find("y") != invMapToScikitNames.end()) { cout << string(indent + addIndent, ' '); cout << "y = None," << endl; hasy = true; } - // Now print actual names, to make it work for - // both mapped and actual names. + // Now print actual names, to make it work for both mapped and actual + // names. // Now print actual name for X. - if(hasX) + if (hasX) { cout << string(indent + addIndent, ' '); cout << GetValidName(invMapToScikitNames["X"]) << " = None," << endl; } // Now print actual name for y. - if(hasy) + if (hasy) { cout << string(indent + addIndent, ' '); cout << GetValidName(invMapToScikitNames["y"]) << " = None," << endl; @@ -295,10 +305,10 @@ void PrintWrapperPY(const std::string& category, string logicString = ""; indent += 2; - if(hasX) + if (hasX) { string realName = GetValidName(invMapToScikitNames["X"]); - + cout << string(indent, ' ') << "if X is not None and "; cout << realName << " is None:" << endl; @@ -312,7 +322,7 @@ void PrintWrapperPY(const std::string& category, cout << endl; } - if(hasy) + if (hasy) { string realName = GetValidName(invMapToScikitNames["y"]); @@ -332,13 +342,17 @@ void PrintWrapperPY(const std::string& category, else { // print input parameters. - for(map::iterator itr=methodParams.begin(); - itr!=methodParams.end(); itr++) + for (map::iterator itr = methodParams.begin(); + itr != methodParams.end(); itr++) { string validName = GetValidName(itr->first); - if(itr->second.input && serializable.find(itr->second.cppType) == - serializable.end() && !hyperParams[itr->first]) + // Get valid stripped name. + string strippedName, unused1, unused2; + StripType(itr->second.cppType, strippedName, unused1, unused2); + + if (itr->second.input && serializable.find(strippedName) == + serializable.end() && !hyperParams[itr->first]) { cout << string(indent + addIndent, ' '); cout << validName << " = None," << endl; @@ -357,8 +371,8 @@ void PrintWrapperPY(const std::string& category, int count = 0; // just for reference. // first pass through the parameters and print all required parameters. - for(map::iterator itr=methodParams.begin(); - itr!=methodParams.end(); itr++) + for (map::iterator itr = methodParams.begin(); + itr != methodParams.end(); itr++) { if(itr->second.input && itr->second.required) { @@ -367,9 +381,11 @@ void PrintWrapperPY(const std::string& category, cout << string(indent + addIndent, ' '); cout << validName << " = "; - if(serializable.find(itr->second.cppType) != serializable.end()) - cout << "self._" << itr->second.cppType << "," << endl; - else if(hyperParams[itr->first]) + string strippedName, unused1, unused2; + StripType(itr->second.cppType, strippedName, unused1, unused2); + if (serializable.find(strippedName) != serializable.end()) + cout << "self._" << strippedName << "," << endl; + else if (hyperParams[itr->first]) cout << "self." << validName << "," << endl; else cout << validName << "," << endl; @@ -379,20 +395,22 @@ void PrintWrapperPY(const std::string& category, } // Now print all non-required parameters. - for(map::iterator itr=methodParams.begin(); - itr!=methodParams.end(); itr++) + for (map::iterator itr = methodParams.begin(); + itr != methodParams.end(); itr++) { - if(itr->second.input && !itr->second.required) + if (itr->second.input && !itr->second.required) { string validName = GetValidName(itr->first); - if(count != 0) + if (count != 0) cout << string(indent + addIndent, ' '); cout << validName << " = "; - if(serializable.find(itr->second.cppType) != serializable.end()) - cout << "self._" << itr->second.cppType << "," << endl; - else if(hyperParams[itr->first]) + string strippedName, unused1, unused2; + StripType(itr->second.cppType, strippedName, unused1, unused2); + if (serializable.find(strippedName) != serializable.end()) + cout << "self._" << strippedName << "," << endl; + else if (hyperParams[itr->first]) cout << "self." << validName << "," << endl; else cout << validName << "," << endl; @@ -409,15 +427,17 @@ void PrintWrapperPY(const std::string& category, string returnString = string(indent, ' ') + "return "; bool outputsOnlySerial = true; - for(map::iterator itr=methodParams.begin(); - itr!=methodParams.end(); itr++) + for (map::iterator itr = methodParams.begin(); + itr != methodParams.end(); itr++) { - if(!itr->second.input) + if (!itr->second.input) { - if(serializable.find(itr->second.cppType) != serializable.end()) + string strippedName, unused1, unused2; + StripType(itr->second.cppType, strippedName, unused1, unused2); + if (serializable.find(strippedName) != serializable.end()) { cout << string(indent, ' '); - cout << "self._" << itr->second.cppType << " = out[\"" << + cout << "self._" << strippedName << " = out[\"" << itr->first << "\"]" << endl; } else @@ -430,7 +450,7 @@ void PrintWrapperPY(const std::string& category, cout << endl; // return somethings. - if(outputsOnlySerial) + if (outputsOnlySerial) cout << returnString + "self" << endl; else cout << returnString.substr(0, returnString.size() - 2) << endl; From d01266617b3d3dd31e54cc3076bca83331290782 Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Wed, 3 Jan 2024 12:34:34 -0500 Subject: [PATCH 86/91] Reduce random failures for DecisionTreeRegressor tests by allowing multiple trials. --- .../tests/decision_tree_regressor_test.cpp | 244 +++++++++--------- src/mlpack/tests/test_function_tools.hpp | 25 +- 2 files changed, 135 insertions(+), 134 deletions(-) diff --git a/src/mlpack/tests/decision_tree_regressor_test.cpp b/src/mlpack/tests/decision_tree_regressor_test.cpp index af1a0932bb..fa7b361d1c 100644 --- a/src/mlpack/tests/decision_tree_regressor_test.cpp +++ b/src/mlpack/tests/decision_tree_regressor_test.cpp @@ -924,84 +924,61 @@ TEST_CASE("CategoricalMADGainWeightedBuildTest", "[DecisionTreeRegressorTest]") // Make sure we get reasonable rmse. const double rmse = RMSE(predictions, testResponses); - REQUIRE(rmse < 1.05); -} - -/** - * Test that the decision tree generalizes reasonably. - */ -TEST_CASE("SimpleGeneralizationTest_", "[DecisionTreeRegressorTest]") -{ - // Loading data. - data::DatasetInfo info; - arma::mat trainData, testData; - arma::rowvec trainResponses, testResponses; - LoadBostonHousingDataset(trainData, testData, trainResponses, testResponses, - info); - arma::rowvec weights = arma::ones(trainResponses.n_elem); - - // Build decision tree. - DecisionTreeRegressor<> d(trainData, info, trainResponses); - DecisionTreeRegressor<> wd(trainData, info, trainResponses, weights); - - // Get the predicted test responses. - arma::rowvec predictions; - d.Predict(testData, predictions); - - REQUIRE(predictions.n_elem == testData.n_cols); - - // Figure out rmse. - double rmse = RMSE(predictions, testResponses); - REQUIRE(rmse < 6.1); - - // Reset the predictions. - predictions.zeros(); - wd.Predict(testData, predictions); - - REQUIRE(predictions.n_elem == testData.n_cols); - - // Figure out rmse. - rmse = RMSE(predictions, testResponses); - REQUIRE(rmse < 6.1); + REQUIRE(rmse < 1.25); } /** * Test that the decision tree generalizes reasonably when built on float data. */ -TEST_CASE("SimpleGeneralizationFMatTest_", "[DecisionTreeRegressorTest]") +TEMPLATE_TEST_CASE("SimpleGeneralizationTest_", + "[DecisionTreeRegressorTest]", float, double) { - // Loading data. - data::DatasetInfo info; - arma::fmat trainData, testData; - arma::rowvec trainLabels, testLabels; - LoadBostonHousingDataset(trainData, testData, trainLabels, testLabels, info); + typedef TestType ElemType; - // Initialize an all-ones weight matrix. - arma::rowvec weights(trainLabels.n_cols, arma::fill::ones); + // Allow three trials. + bool success = false; + for (size_t trial = 0; trial < 3; ++trial) + { + // Loading data. + data::DatasetInfo info; + arma::Mat trainData, testData; + arma::Row trainResponses, testResponses; + LoadBostonHousingDataset(trainData, testData, trainResponses, testResponses, + info); - // Build decision tree. - DecisionTreeRegressor<> d(trainData, trainLabels); - DecisionTreeRegressor<> wd(trainData, trainLabels, weights); + // Initialize an all-ones weight matrix. + arma::rowvec weights(trainResponses.n_cols, arma::fill::ones); - // Get the predicted test labels. - arma::rowvec predictions; - d.Predict(testData, predictions); + // Build decision tree. + DecisionTreeRegressor<> d(trainData, trainResponses); + DecisionTreeRegressor<> wd(trainData, trainResponses, weights); - REQUIRE(predictions.n_elem == testData.n_cols); + // Get the predicted test labels. + arma::Row predictions; + d.Predict(testData, predictions); - // Figure out the rmse. - double rmse = RMSE(predictions, testLabels); - REQUIRE(rmse < 6.0); + REQUIRE(predictions.n_elem == testData.n_cols); - // Reset the prediction. - predictions.zeros(); - wd.Predict(testData, predictions); + // Figure out the rmse. + ElemType rmse = RMSE(predictions, testResponses); - REQUIRE(predictions.n_elem == testData.n_cols); + // Reset the prediction. + predictions.zeros(); + wd.Predict(testData, predictions); - // Figure out the rmse. - double wdrmse = RMSE(predictions, testLabels); - REQUIRE(wdrmse < 6.0); + REQUIRE(predictions.n_elem == testData.n_cols); + + // Figure out the rmse. + ElemType wdrmse = RMSE(predictions, testResponses); + + if (rmse <= 6.2 && wdrmse <= 6.2) + { + success = true; + break; + } + } + + REQUIRE(success == true); } /** @@ -1011,42 +988,53 @@ TEST_CASE("SimpleGeneralizationFMatTest_", "[DecisionTreeRegressorTest]") */ TEST_CASE("WeightedDecisionTreeTest_", "[DecisionTreeRegressorTest]") { - // Loading data. - data::DatasetInfo info; - arma::mat trainData, testData; - arma::rowvec trainResponses, testResponses; - LoadBostonHousingDataset(trainData, testData, trainResponses, testResponses, - info); + // Allow three trials for success. + bool success = false; + for (size_t trial = 0; trial < 3; ++trial) + { + // Loading data. + data::DatasetInfo info; + arma::mat trainData, testData; + arma::rowvec trainResponses, testResponses; + LoadBostonHousingDataset(trainData, testData, trainResponses, testResponses, + info); - // Add some noise. - arma::mat noise(trainData.n_rows, 100, arma::fill::randu); - arma::rowvec noiseResponses(100); - for (size_t i = 0; i < noiseResponses.n_elem; ++i) - noiseResponses[i] = 15 + Random(0, 10); // Random response. + // Add some noise. + arma::mat noise(trainData.n_rows, 100, arma::fill::randu); + arma::rowvec noiseResponses(100); + for (size_t i = 0; i < noiseResponses.n_elem; ++i) + noiseResponses[i] = 15 + Random(0, 10); // Random response. - // Concatenate data matrices. - arma::mat data = arma::join_rows(trainData, noise); - arma::rowvec fullResponses = arma::join_rows(trainResponses, noiseResponses); + // Concatenate data matrices. + arma::mat data = arma::join_rows(trainData, noise); + arma::rowvec fullResponses = arma::join_rows(trainResponses, + noiseResponses); - // Now set weights. - arma::rowvec weights(trainData.n_cols + 100); - for (size_t i = 0; i < trainData.n_cols; ++i) - weights[i] = Random(0.9, 1.0); - for (size_t i = trainData.n_cols; i < trainData.n_cols + 100; ++i) - weights[i] = Random(0.0, 0.01); // Low weights for false points. + // Set weights. + arma::rowvec weights(trainData.n_cols + 100); + for (size_t i = 0; i < trainData.n_cols; ++i) + weights[i] = Random(0.9, 1.0); + for (size_t i = trainData.n_cols; i < trainData.n_cols + 100; ++i) + weights[i] = Random(0.0, 0.01); // Low weights for false points. - // Now build the decision tree. - DecisionTreeRegressor<> d(data, fullResponses, weights); + // Now build the decision tree. + DecisionTreeRegressor<> d(data, fullResponses, weights); - // Now we can check that we get good performance on the test set. - arma::rowvec predictions; - d.Predict(testData, predictions); + // Now we can check that we get good performance on the test set. + arma::rowvec predictions; + d.Predict(testData, predictions); - REQUIRE(predictions.n_elem == testData.n_cols); + REQUIRE(predictions.n_elem == testData.n_cols); - // Figure out the rmse. - double rmse = RMSE(predictions, testResponses); - REQUIRE(rmse < 6.0); + // Figure out the rmse. + double rmse = RMSE(predictions, testResponses); + if (rmse < 6.2) + { + success = true; + break; + } + } + REQUIRE(success == true); } /** @@ -1056,40 +1044,52 @@ TEST_CASE("WeightedDecisionTreeTest_", "[DecisionTreeRegressorTest]") */ TEST_CASE("WeightedDecisionTreeMADGainTest", "[DecisionTreeRegressorTest]") { - // Loading data. - data::DatasetInfo info; - arma::mat trainData, testData; - arma::rowvec trainResponses, testResponses; - LoadBostonHousingDataset(trainData, testData, trainResponses, testResponses, - info); + // Allow multiple trials, if needed. + bool success = false; + for (size_t trial = 0; trial < 5; ++trial) + { + // Loading data. + data::DatasetInfo info; + arma::mat trainData, testData; + arma::rowvec trainResponses, testResponses; + LoadBostonHousingDataset(trainData, testData, trainResponses, testResponses, + info); - // Add some noise. - arma::mat noise(trainData.n_rows, 100, arma::fill::randu); - arma::rowvec noiseResponses(100); - for (size_t i = 0; i < noiseResponses.n_elem; ++i) - noiseResponses[i] = 15 + Random(0, 10); // Random response. + // Add some noise. + arma::mat noise(trainData.n_rows, 100, arma::fill::randu); + arma::rowvec noiseResponses(100); + for (size_t i = 0; i < noiseResponses.n_elem; ++i) + noiseResponses[i] = 15 + Random(0, 10); // Random response. - // Concatenate data matrices. - arma::mat data = arma::join_rows(trainData, noise); - arma::rowvec fullResponses = arma::join_rows(trainResponses, noiseResponses); + // Concatenate data matrices. + arma::mat data = arma::join_rows(trainData, noise); + arma::rowvec fullResponses = arma::join_rows(trainResponses, + noiseResponses); - // Now set weights. - arma::rowvec weights(trainData.n_cols + 100); - for (size_t i = 0; i < trainData.n_cols; ++i) - weights[i] = Random(0.9, 1.0); - for (size_t i = trainData.n_cols; i < trainData.n_cols + 100; ++i) - weights[i] = Random(0.0, 0.01); // Low weights for false points. + // Now set weights. + arma::rowvec weights(trainData.n_cols + 100); + for (size_t i = 0; i < trainData.n_cols; ++i) + weights[i] = Random(0.9, 1.0); + for (size_t i = trainData.n_cols; i < trainData.n_cols + 100; ++i) + weights[i] = Random(0.0, 0.01); // Low weights for false points. - // Now build the decision tree using MADGain. - DecisionTreeRegressor d(data, fullResponses, weights); + // Now build the decision tree using MADGain. + DecisionTreeRegressor d(data, fullResponses, weights); - // Now we can check that we get good performance on the test set. - arma::rowvec predictions; - d.Predict(testData, predictions); + // Now we can check that we get good performance on the test set. + arma::rowvec predictions; + d.Predict(testData, predictions); - REQUIRE(predictions.n_elem == testData.n_cols); + REQUIRE(predictions.n_elem == testData.n_cols); - // Figure out the rmse. - double rmse = RMSE(predictions, testResponses); - REQUIRE(rmse < 6.5); + // Figure out the rmse. + double rmse = RMSE(predictions, testResponses); + if (rmse < 6.6) + { + success = true; + break; + } + } + + REQUIRE(success == true); } diff --git a/src/mlpack/tests/test_function_tools.hpp b/src/mlpack/tests/test_function_tools.hpp index 3bf3d983cb..f06ca36bab 100644 --- a/src/mlpack/tests/test_function_tools.hpp +++ b/src/mlpack/tests/test_function_tools.hpp @@ -30,11 +30,11 @@ using namespace mlpack; * @param shuffledResponses Matrix object to store the shuffled responses into. */ inline void LogisticRegressionTestData(arma::mat& data, - arma::mat& testData, - arma::mat& shuffledData, - arma::Row& responses, - arma::Row& testResponses, - arma::Row& shuffledResponses) + arma::mat& testData, + arma::mat& shuffledData, + arma::Row& responses, + arma::Row& testResponses, + arma::Row& shuffledResponses) { // Generate a two-Gaussian dataset. GaussianDistribution g1(arma::vec("1.0 1.0 1.0"), arma::eye(3, 3)); @@ -79,15 +79,15 @@ inline void LogisticRegressionTestData(arma::mat& data, } } -template +template void LoadBostonHousingDataset(MatType& trainData, MatType& testData, - arma::rowvec& trainResponses, - arma::rowvec& testResponses, + ResponsesType& trainResponses, + ResponsesType& testResponses, data::DatasetInfo& info) { MatType dataset; - arma::rowvec responses; + ResponsesType responses; // Defining categorical deimensions. info.SetDimensionality(13); @@ -103,10 +103,11 @@ void LoadBostonHousingDataset(MatType& trainData, trainResponses, testResponses, 0.3); } -inline double RMSE(const arma::Row& predictions, - const arma::Row& trueResponses) +template +inline ElemType RMSE(const arma::Row& predictions, + const arma::Row& trueResponses) { - double mse = arma::accu(arma::square(predictions - trueResponses)) / + ElemType mse = arma::accu(arma::square(predictions - trueResponses)) / predictions.n_elem; return sqrt(mse); } From 68f59cf3c98c3919a904af51e326536e8ef7560d Mon Sep 17 00:00:00 2001 From: Omar Shrit Date: Wed, 3 Jan 2024 18:45:52 +0100 Subject: [PATCH 87/91] Adding functions to differentiate arma::max and coot::max Signed-off-by: Omar Shrit --- src/mlpack/core/math/mat_redef.hpp | 53 +++++++++++++++++++ src/mlpack/core/math/math.hpp | 1 + .../rectifier_function.hpp | 5 +- 3 files changed, 57 insertions(+), 2 deletions(-) create mode 100644 src/mlpack/core/math/mat_redef.hpp diff --git a/src/mlpack/core/math/mat_redef.hpp b/src/mlpack/core/math/mat_redef.hpp new file mode 100644 index 0000000000..c5ffebc431 --- /dev/null +++ b/src/mlpack/core/math/mat_redef.hpp @@ -0,0 +1,53 @@ +/** + * @file core/math/mat_redef.hpp + * + * A shim around arma and coot functions to avoid confusion with standard + * library functions. This is necessary as we are using ADL to allow the + * compiler to deduce to which library functions belings without the need + * for namespace. This is mostly needed for MSVC compiiler, gcc seems to + * pass without an issue. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + * + * Author: Omar Shrit + * Author: Ryan Curtin + */ +#ifndef MLPACK_CORE_MAT_REDEF_HPP +#define MLPACK_CORE_MAT_REDEF_HPP + +namespace mlpack { + +template +inline arma::Mat SafeMax(const arma::Mat& A, const arma::Mat& B) +{ + return arma::max(A, B); +} + +template +inline arma::Mat SafeMin(const arma::Mat& A, const arma::Mat& B) +{ + return arma::min(A, B); +} + +#ifdef MLPACK_HAS_COOT + +template +inline coot::Mat SafeMax(const coot::Mat& A, const coot::Mat& B) +{ + return coot::max(A, B); +} + +template +inline coot::Mat SafeMin(const coot::Mat& A, const coot::Mat& B) +{ + return coot::min(A, B); +} + +#endif + +} + +#endif diff --git a/src/mlpack/core/math/math.hpp b/src/mlpack/core/math/math.hpp index 31a3a56d76..d0bf0ec90a 100644 --- a/src/mlpack/core/math/math.hpp +++ b/src/mlpack/core/math/math.hpp @@ -19,6 +19,7 @@ #include "lin_alg.hpp" #include "log_add.hpp" #include "make_alias.hpp" +#include "mat_redef.hpp" #include "multiply_slices.hpp" #include "quantile.hpp" #include "random_basis.hpp" diff --git a/src/mlpack/methods/ann/activation_functions/rectifier_function.hpp b/src/mlpack/methods/ann/activation_functions/rectifier_function.hpp index 0a372e23d5..54ebcf9cc3 100644 --- a/src/mlpack/methods/ann/activation_functions/rectifier_function.hpp +++ b/src/mlpack/methods/ann/activation_functions/rectifier_function.hpp @@ -22,8 +22,9 @@ */ #ifndef MLPACK_METHODS_ANN_ACTIVATION_FUNCTIONS_RECTIFIER_FUNCTION_HPP #define MLPACK_METHODS_ANN_ACTIVATION_FUNCTIONS_RECTIFIER_FUNCTION_HPP -#undef max + #include +#include #include namespace mlpack { @@ -66,7 +67,7 @@ class RectifierFunction { y.set_size(size(x)); y.zeros(); - y = max(y, x); + y = SafeMax(y, x); } /** From 073da1e56ed9809f5bcbe01bada0020e6e642899 Mon Sep 17 00:00:00 2001 From: Omar Shrit Date: Thu, 4 Jan 2024 18:38:49 +0100 Subject: [PATCH 88/91] Update src/mlpack/core/math/mat_redef.hpp Co-authored-by: Ryan Curtin --- src/mlpack/core/math/mat_redef.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mlpack/core/math/mat_redef.hpp b/src/mlpack/core/math/mat_redef.hpp index c5ffebc431..808088ff54 100644 --- a/src/mlpack/core/math/mat_redef.hpp +++ b/src/mlpack/core/math/mat_redef.hpp @@ -3,7 +3,7 @@ * * A shim around arma and coot functions to avoid confusion with standard * library functions. This is necessary as we are using ADL to allow the - * compiler to deduce to which library functions belings without the need + * compiler to deduce to which library functions belong without the need * for namespace. This is mostly needed for MSVC compiiler, gcc seems to * pass without an issue. * From c919cb3fe56b1e03b0aed2fd87469a83741fe787 Mon Sep 17 00:00:00 2001 From: Omar Shrit Date: Thu, 4 Jan 2024 19:04:50 +0100 Subject: [PATCH 89/91] Apply the suggestions Signed-off-by: Omar Shrit --- src/mlpack/base.hpp | 2 ++ src/mlpack/core/math/math.hpp | 2 +- src/mlpack/core/math/{mat_redef.hpp => safe_linalg.hpp} | 0 .../methods/ann/activation_functions/rectifier_function.hpp | 1 - 4 files changed, 3 insertions(+), 2 deletions(-) rename src/mlpack/core/math/{mat_redef.hpp => safe_linalg.hpp} (100%) diff --git a/src/mlpack/base.hpp b/src/mlpack/base.hpp index 860b5f0b81..00ea768bff 100644 --- a/src/mlpack/base.hpp +++ b/src/mlpack/base.hpp @@ -103,6 +103,8 @@ // Now include Armadillo through the special mlpack extensions. #include #include +// Include local armadillo safe linear algebra functions. +#include // On Visual Studio, disable C4519 (default arguments for function templates) // since it's by default an error, which doesn't even make any sense because diff --git a/src/mlpack/core/math/math.hpp b/src/mlpack/core/math/math.hpp index d0bf0ec90a..49a41be30b 100644 --- a/src/mlpack/core/math/math.hpp +++ b/src/mlpack/core/math/math.hpp @@ -19,7 +19,7 @@ #include "lin_alg.hpp" #include "log_add.hpp" #include "make_alias.hpp" -#include "mat_redef.hpp" +#include "safe_linalg.hpp" #include "multiply_slices.hpp" #include "quantile.hpp" #include "random_basis.hpp" diff --git a/src/mlpack/core/math/mat_redef.hpp b/src/mlpack/core/math/safe_linalg.hpp similarity index 100% rename from src/mlpack/core/math/mat_redef.hpp rename to src/mlpack/core/math/safe_linalg.hpp diff --git a/src/mlpack/methods/ann/activation_functions/rectifier_function.hpp b/src/mlpack/methods/ann/activation_functions/rectifier_function.hpp index 54ebcf9cc3..e96d6044bf 100644 --- a/src/mlpack/methods/ann/activation_functions/rectifier_function.hpp +++ b/src/mlpack/methods/ann/activation_functions/rectifier_function.hpp @@ -24,7 +24,6 @@ #define MLPACK_METHODS_ANN_ACTIVATION_FUNCTIONS_RECTIFIER_FUNCTION_HPP #include -#include #include namespace mlpack { From 550541df0f553dc40175d647cc005534cd4fbedc Mon Sep 17 00:00:00 2001 From: Omar Shrit Date: Thu, 4 Jan 2024 19:10:23 +0100 Subject: [PATCH 90/91] Fix the author name Signed-off-by: Omar Shrit --- src/mlpack/core/math/safe_linalg.hpp | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/src/mlpack/core/math/safe_linalg.hpp b/src/mlpack/core/math/safe_linalg.hpp index 808088ff54..f243968e7b 100644 --- a/src/mlpack/core/math/safe_linalg.hpp +++ b/src/mlpack/core/math/safe_linalg.hpp @@ -1,19 +1,18 @@ /** - * @file core/math/mat_redef.hpp + * @file core/math/safe_linalg.hpp + * @author Omar Shrit + * @author Ryan Curtin * * A shim around arma and coot functions to avoid confusion with standard * library functions. This is necessary as we are using ADL to allow the - * compiler to deduce to which library functions belong without the need - * for namespace. This is mostly needed for MSVC compiiler, gcc seems to + * compiler to deduce which library the functions belong without the need + * for namespace. This is mostly needed for MSVC compiler, gcc seems to * pass without an issue. * * mlpack is free software; you may redistribute it and/or modify it under the * terms of the 3-clause BSD license. You should have received a copy of the * 3-clause BSD license along with mlpack. If not, see * http://www.opensource.org/licenses/BSD-3-Clause for more information. - * - * Author: Omar Shrit - * Author: Ryan Curtin */ #ifndef MLPACK_CORE_MAT_REDEF_HPP #define MLPACK_CORE_MAT_REDEF_HPP From c194fb73f816db91f0d552d2ba4a9e53cc91513d Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Fri, 12 Jan 2024 11:20:59 -0500 Subject: [PATCH 91/91] Clarify weights vs. instance weights. --- .../linear_regression/linear_regression.hpp | 52 +++++++++---------- 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/src/mlpack/methods/linear_regression/linear_regression.hpp b/src/mlpack/methods/linear_regression/linear_regression.hpp index bd9cd3ad47..55847ccdab 100644 --- a/src/mlpack/methods/linear_regression/linear_regression.hpp +++ b/src/mlpack/methods/linear_regression/linear_regression.hpp @@ -52,11 +52,11 @@ class LinearRegression const bool intercept = true); /** - * Creates the model with weighted learning. + * Creates the model with instance-weighted learning. * * @param predictors X, matrix of data points. * @param responses y, the measured data for each point in X. - * @param weights Observation weights (for boosting). + * @param weights Instance weights (for boosting). * @param lambda Regularization constant for ridge regression. * @param intercept Whether or not to include an intercept term. */ @@ -104,11 +104,11 @@ class LinearRegression const bool intercept); /** - * Train the LinearRegression model on the given data and weights. Careful! - * This will completely ignore and overwrite the existing model. This - * particular implementation does not have an incremental training algorithm. - * To set the regularization parameter lambda, call Lambda() or set a - * different value in the constructor. + * Train the LinearRegression model on the given data and instance weights. + * Careful! This will completely ignore and overwrite the existing model. + * This particular implementation does not have an incremental training + * algorithm. To set the regularization parameter lambda, call Lambda() or + * set a different value in the constructor. * * This version of `Train()` is deprecated and will be removed in mlpack * 5.0.0. Use the version of `Train()` that specifies `lambda` before @@ -116,7 +116,7 @@ class LinearRegression * * @param predictors X, the matrix of data points to train the model on. * @param responses y, the responses to the data points. - * @param weights Observation weights (for boosting). + * @param weights Instance weights (for boosting). * @param intercept Whether or not to fit an intercept term. * @return The least squares error after training. */ @@ -215,15 +215,15 @@ class LinearRegression const arma::rowvec& weights); /** - * Train the LinearRegression model on the given data and weights. Careful! - * This will completely ignore and overwrite the existing model. This - * particular implementation does not have an incremental training algorithm. - * To set the regularization parameter lambda, call Lambda() or set a - * different value in the constructor. + * Train the LinearRegression model on the given data and instance weights. + * Careful! This will completely ignore and overwrite the existing model. + * This particular implementation does not have an incremental training + * algorithm. To set the regularization parameter lambda, call Lambda() or + * set a different value in the constructor. * * @param predictors X, the matrix of data points to train the model on. * @param responses y, the responses to the data points. - * @param weights Observation weights (for boosting). + * @param weights Instance weights (for boosting). * @return The least squares error after training. */ template