From 60a386de04ec1762d723b1cfc0c221d902f36b53 Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Mon, 23 Oct 2023 14:33:43 -0400 Subject: [PATCH 01/47] Add first pass of AdaBoost documentation. --- doc/user/methods/adaboost.md | 470 +++++++++++++++++++++++++++++++++++ 1 file changed, 470 insertions(+) create mode 100644 doc/user/methods/adaboost.md diff --git a/doc/user/methods/adaboost.md b/doc/user/methods/adaboost.md new file mode 100644 index 0000000000..a20c6bd5a3 --- /dev/null +++ b/doc/user/methods/adaboost.md @@ -0,0 +1,470 @@ +## `AdaBoost` + +The `AdaBoost` class implements the 'adaptive boosting' classifier AdaBoost.MH. +This classifier is an ensemble of weak learners. The `AdaBoost` class offers +control over the weak learners and other behavior via template parameters. By +default, the `Perceptron` class is used as a weak learner. + +`AdaBoost` is useful for classifying points with _discrete labels_ (i.e. `0`, +`1`, `2`). + +#### Basic usage example excerpt: + +```c++ +AdaBoost ab; // Step 1: construct object. +ab.Train(data, labels, 3); // Step 2: train model. +ab.Classify(testData, testPredictions); // Step 3: use model to classify. +``` + +#### Quick links: + + * [Constructors](#constructors): create `AdaBoost` objects. + * [`Train()`](#training): train model. + * [`Classify()`](#classify): classify with a trained model. + * [Other functionality](#other-functionality) for loading, saving, and + inspecting. + * [Examples](#simple-examples) of simple usage and links to detailed example + projects. + * [Template parameters](#advanced-functionality-template-parameters) for custom + behavior. + * [Advanced template examples](#advanced-functionality-examples) of use with + custom template parameters. + +#### See also: + + * [mlpack classifiers](#mlpack_classifiers) + * [`Perceptron`](#perceptron) + * [`DecisionTree`](#decision_tree) + * [AdaBoost on Wikipedia](https://en.wikipedia.org/wiki/AdaBoost) + * [AdaBoost.MH paper (pdf)](https://dl.acm.org/doi/pdf/10.1145/279943.279960) + +### Constructors + +Construct an `AdaBoost` object using one of the constructors below. Defaults +and types are detailed in the [Constructor Parameters](#constructor-parameters) +section below. + +#### Forms: + + * `AdaBoost()` + * `AdaBoost(tolerance)` + - **Initialize model without training.** + - You will need to call [`Train()`](#training) later to train the tree before + calling [`Classify()`](#classification). + +--- + + + * `AdaBoost(data, labels, numClasses)` + * `AdaBoost(data, labels, numClasses, maxIterations, tolerance)` + - **Train model using default weak learner parameters.** + - If hyperparameters are not specified, default values are used. + - `labels` should be a vector of length `data.n_cols`, containing values from + `0` to `numClasses - 1` (inclusive). + +--- + + * `AdaBoost(data, labels, numClasses, weakLearner)` + * `AdaBoost(data, labels, numClasses, weakLearner, maxIterations, tolerance)` + - **Train model with custom weak learner parameters.** + - The given `weakLearner` does not need to be trained; any hyperparameter + settings in `weakLearner` are used for training each AdaBoost weak + learner (see the [simple examples](#simple-examples)). + - If hyperparameters are not specified, default values are used. + - `labels` should be a vector of length `data.n_cols`, containing values from + `0` to `numClasses - 1` (inclusive). + +--- + + + +--- + +#### Constructor Parameters: + + + +| **name** | **type** | **description** | **default** | +|----------|----------|-----------------|-------------| +| `data` | [`arma::mat`](../matrices.md) | [Column-major](../matrices.md) training matrix. | _(N/A)_ | +| `labels` | [`arma::Row`]('../matrices.md') | Training labels, between `0` and `numClasses - 1` (inclusive). Should have length `data.n_cols`. | _(N/A)_ | +| `numClasses` | `size_t` | Number of classes in the dataset. | _(N/A)_ | +| `weakLearner` | `Perceptron` | An initialized weak learner whose +hyperparameters will be used as settings for weak learners during training. | +_(N/A)_ | +| `maxIterations` | `size_t` | Maximum number of iterations of AdaBoost.MH to use. This is the maximum number of weak learners to train. (0 means no limit, and weak learners will be trained until the tolerance is met.) | `100` | +| `tolerance` | `double` | When the weighted residual (`r_t`) of the model goes +below `tolerance`, training will terminate and no more weak learners will be +added. | `1e-6` | + +As an alternative to passing hyperparameters, each hyperparameter can be set +with a standalone method. For an instance of `AdaBoost` named `ab`, the +following functions can be used before calling `Train()` to set hyperparameters: + + + + * `ab.MaxIterations() = maxIter;` will set the maximum number of weak learners + during training to `maxIter`. + * `ab.Tolerance() = tol;` will set the tolerance to `tol`. + + + +***Note:*** different types of weak learners can be used than +[`Perceptron`](#perceptron), by changing the [`WeakLearnerType` template +parameter](#advanced-functionality-template-parameters). + +### Training + +If training is not done as part of the constructor call, it can be done with one +of the versions of the `Train()` member function. For an instance of `AdaBoost` +named `ab`, the following functions for training are available: + + + + * `ab.Train(data, labels, numClasses)` + * `ab.Train(data, labels, numClasses, maxIterations, tolerance)` + - **Train model using default weak learner parameters.** + - If hyperparameters are not specified, default values are used. + - `labels` should be a vector of length `data.n_cols`, containing values from + `0` to `numClasses - 1` (inclusive). + +--- + + * `ab.Train(data, labels, numClasses, weakLearner)` + * `ab.Train(data, labels, numClasses, weakLearner, maxIterations, tolerance)` + - **Train model with custom weak learner parameters.** + - The given `weakLearner` does not need to be trained; any hyperparameter + settings in `weakLearner` are used for training each AdaBoost weak learner + (see the [simple examples](#simple-examples)). + - If hyperparameters are not specified, default values are used. + - `labels` should be a vector of length `data.n_cols`, containing values from + `0` to `numClasses - 1` (inclusive). + +--- + + + +--- + +Types of each argument are the same as in the table for constructors +[above](#constructor-parameters). + +***Note***: training is not incremental. A second call to `Train()` will +retrain the AdaBoost model from scratch. + +### Classification + +Once an `AdaBoost` model is trained, the `Classify()` member function can be +used to make class predictions for new data. Defaults and types are detailed in +the [Classification Parameters](#classification-parameters) section below. + +#### Forms: + + + + * `size_t predictedClass = ab.Classify(point)` + - ***(Single-point)*** + - Classify a single point, returning the predicted class. + +--- + + * `ab.Classify(point, prediction, probabilities_vec)` + - ***(Single-point)*** + - Classify a single point and compute class probabilities. + - The predicted class is stored in `prediction`. + - The class probabilities are stored in `probabilities_vec`, which is set to + length `numClasses`. + - The probability of class `i` can be accessed with `probabilities_vec[i]`. + +--- + + * `ab.Classify(data, predictions)` + - ***(Multi-point)*** + - Classify a set of points. + - The predicted class of each point is stored in `predictions`, which is set + to length `data.n_cols`. + - The prediction for data point `i` can be accessed with `predictions[i]`. + +--- + + * `ab.Classify(data, predictions, probabilities)` + - ***(Multi-point)*** + - Classify a set of points and compute class probabilities for each point. + - The predicted class of each point is stored in `predictions`, which is set + to length `data.n_cols`. + - The prediction for data point `i` can be accessed with `predictions[i]`. + - The class probabilities for each point are stored in `probabilities`, which + is set to size `numClasses` by `data.n_cols`. + - The probability of class `j` for data point `i` can be accessed with + `probabilities(j, i)`. + +--- + +#### Classification Parameters: + +| **usage** | **name** | **type** | **description** | +|-----------|----------|----------|-----------------| +| _single-point_ | `point` | [`arma::vec`](../matrices.md) | Single point for classification. | +| _single-point_ | `prediction` | `size_t&` | `size_t` to store class prediction into. | +| _single-point_ | `probabilities_vec` | [`arma::vec&`](../matrices.md) | `arma::vec&` to store class probabilities into. | +|||| +| _multi-point_ | `data` | [`arma::mat`](../matrices.md) | Set of [column-major](../matrices.md) points for classification. | +| _multi-point_ | `predictions` | [`arma::Row&`](../matrices.md) | Vector of `size_t`s to store class prediction into. | +| _multi-point_ | `probabilities` | [`arma::mat&`](../matrices.md) | Matrix to store class probabilities into (number of rows will be equal to number of classes). | + +### Other Functionality + + + + * An `AdaBoost` model can be serialized with [`data::Save()`](../formats.md) + and [`data::Load()`](../formats.md). + + * `ab.NumClasses()` will return a `size_t` indicating the number of classes the + model was trained on. + + * `ab.WeakLearners()` will return a `size_t` indicating the number of weak + learners that the model currently contains. + + * `ab.Alpha(i)` will return the weight of weak learner `i`. + + * `ab.WeakLearner(i)` will return the `i`th weak learner. + +### Simple Examples + +Train an AdaBoost model on random data and predict labels on a random test set. + +```c++ +// 1000 random points in 10 dimensions. +arma::mat dataset(10, 1000, arma::fill::randu); +// Random labels for each point, totaling 5 classes. +arma::Row labels = + arma::randi>(1000, arma::distr_param(0, 4)); + +// Train in the constructor. +AdaBoost<> ab(dataset, labels, 5); + +// Create test data (500 points). +arma::mat testDataset(10, 500, arma::fill::randu); +arma::Row predictions; +ab.Classify(testDataset, predictions); +// Now `predictions` holds predictions for the test dataset. + +// Print some information about the test predictions. +std::cout << arma::accu(predictions == 3) << " test points classified as class " + << "3." << std::endl; +``` + +--- + +Train an AdaBoost model using the hyperparameters from an existing weak learner. + +```c++ +// See https://datasets.mlpack.org/iris.csv. +arma::mat dataset; +data::Load("iris.csv", dataset, true); +// See https://datasets.mlpack.org/iris.labels.csv. +arma::Row labels; +data::Load("iris_labels.csv", dataset, true); + +// Create a weak learner with the desired hyperparameters. +Perceptron<> p; +p.MaxIterations() = 500; // We'll use a custom maximum number of iterations. + +AdaBoost<> ab; +ab.Train(dataset, labels, 3, p); + +// Now predict the label of a point and the probabilities of each class. +size_t prediction; +arma::rowvec probabilities; +ab.Classify(dataset.col(10), prediction, probabilities); + +std::cout << "Point 11 is predicted to have class " << prediction << "." + << std::endl; +std::cout << "Probabilities of each class: " << probabilities; +``` + +--- + +Before training an AdaBoost model, set hyperparameters individually. Save the +trained model to disk. + +```c++ +// See https://datasets.mlpack.org/iris.csv. +arma::mat dataset; +data::Load("iris.csv", dataset, true); +// See https://datasets.mlpack.org/iris.labels.csv. +arma::Row labels; +data::Load("iris_labels.csv", dataset, true); + +AdaBoost<> ab; +ab.MaxIterations() = 50; // Use at most 50 weak learners. +ab.Tolerance() = 1e-4; // Set a custom tolerance for convergence. + +// Now train, using the hyperparameters specified above. +ab.Train(dataset, labels, 3); + +// Save the model to `adaboost_model.bin`. +data::Save("adaboost_model.bin", "adaboost_model", ab, true); +``` + +--- + +Load an AdaBoost model and print some information about it. + +```c++ +// Load a saved model named "adaboost_model" from `adaboost_model.bin`. +AdaBoost<> ab; +data::Load("adaboost_model.bin", "adaboost_model", ab, true); + +std::cout << "Details about the model in `adaboost_model.bin`:" << std::endl; +std::cout << " - Trained on " << ab.NumClasses() << " classes." << std::endl; +std::cout << " - Tolerance used for training: " << ab.Tolerance() << "." + << std::endl; +std::cout << " - Number of perceptron weak learners in model" + << ab.WeakLearners() << "." << std::endl; + +// Print some details about the first weak learner, if available. The weak +// learner type is `Perceptron<>`. +if (ab.WeakLearners() > 0) +{ + std::cout << " - Weight of first perceptron weak learner: " << ab.Alpha(0) + << "." << std::endl; + std::cout << " - Biases of first perceptron learner: " + << ab.WeakLearner(0).Biases().t(); +} +``` + +--- + +See also the following fully-working examples: + + - [Graduate admission classification with `AdaBoost`](https://github.com/mlpack/examples/blob/master/graduate_admission_classification_with_Adaboost/graduate-admission-classification-with-adaboost-cpp.ipynb) + +### Advanced Functionality: Template Parameters + +The `AdaBoost` class has two template parameters that can be used for custom +behavior. The full signature of the class is: + +```c++ +AdaBoost +``` + +#### `WeakLearnerType` + + + + * Specifies the weak learner to use when constructing an AdaBoost model. + * The default `WeakLearnerType` is [`Perceptron<>`](#perceptron). + * The `ID3DecisionStump` class (a custom variant of + [`DecisionTree`](#decision_tree)) is available for drop-in usage as a weak + learner. + * Any custom variant of `Perceptron<>` or `DecisionTree<>` can be used; e.g., + `Perceptron`. + * A custom class must implement the following functions for training and + classification. Note that this is the same API as mlpack's classifiers that + support instance weights for learning, and so any mlpack classifier + supporting instance weights can also be used. + +```c++ +class CustomWeakLearner +{ + public: + // Train the model with the given hyperparameters. + // + // * `MatType` will be an Armadillo-like matrix type (typically `arma::mat`); + // this is the same type as the `MatType` template parameter for the + // `AdaBoost` class. + // + // * `data` and `labels` are the same dataset passed to the `Train()` method + // of `AdaBoost`. + // + // * `weights` contains instance weights for each point (column) of `data`. + // + // Note: there is no restriction on the number or types of hyperparameters + // that can be used, but, they do need default arguments. The example here + // includes two. + template + void Train(const MatType& data, + const arma::Row& labels, + const size_t numClasses, + const arma::rowvec& weights, + const size_t hyperparameterA = 10, + const double hyperparameterB = 0.1); + + // Classify the given point. `VecType` will be an Armadillo-like type that is + // a vector that represents a single point. + template + size_t Classify(const VecType& point); + + // Classify the given points in `data`, storing the predicted classifications + // in `predictions`. + template + void Classify(const MatType& data, arma::Row& predictions); +}; +``` + +#### `MatType` + + * Specifies the matrix type to use for data when learning a model (or + predicting with one). + * By default, `MatType` is `arma::mat` (dense 64-bit precision matrix). + * `arma::fmat` or `arma::sp_mat` could also be used. + +### Advanced Functionality Examples + +Train an AdaBoost model using decision stumps as the weak learner (use a +different `WeakLearnerType`). + +```c++ +// 1000 random points in 10 dimensions. +arma::mat dataset(10, 1000, arma::fill::randu); +// Random labels for each point, totaling 5 classes. +arma::Row labels = + arma::randi>(1000, arma::distr_param(0, 4)); + +// Train in the constructor. +// Note that we specify decision stumps as the weak learner type. +AdaBoost ab(dataset, labels, 5); + +// Create test data (500 points). +arma::mat testDataset(10, 500, arma::fill::randu); +arma::Row predictions; +ab.Classify(testDataset, predictions); +// Now `predictions` holds predictions for the test dataset. + +// Print some information about the test predictions. +std::cout << arma::accu(predictions == 3) << " test points classified as class " + << "3." << std::endl; +``` + +--- + +Train an AdaBoost model on 32-bit floating-point precision data (use a different +`MatType`). + +```c++ +// 1000 random points in 10 dimensions, using 32-bit precision (float). +arma::fmat dataset(10, 1000, arma::fill::randu); +// Random labels for each point, totaling 5 classes. +arma::Row labels = + arma::randi>(1000, arma::distr_param(0, 4)); + +// Train in the constructor, using floating-point data. +// (TODO: do we have to explicitly write MatType?) +AdaBoost<> ab(dataset, labels, 5); + +// Create test data (500 points). +arma::fmat testDataset(10, 500, arma::fill::randu); +arma::Row predictions; +ab.Classify(testDataset, predictions); +// Now `predictions` holds predictions for the test dataset. + +// Print some information about the test predictions. +std::cout << arma::accu(predictions == 3) << " test points classified as class " + << "3." << std::endl; +``` From 13da8cb7ce686712b9ec75153c14721fd5849bdb Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Mon, 23 Oct 2023 18:32:17 -0400 Subject: [PATCH 02/47] First draft of Perceptron documentation. --- doc/user/methods/perceptron.md | 441 +++++++++++++++++++++++++++++++++ 1 file changed, 441 insertions(+) create mode 100644 doc/user/methods/perceptron.md diff --git a/doc/user/methods/perceptron.md b/doc/user/methods/perceptron.md new file mode 100644 index 0000000000..51b3313b99 --- /dev/null +++ b/doc/user/methods/perceptron.md @@ -0,0 +1,441 @@ +## `Perceptron` + +The `Perceptron` class implements the simple perceptron classifier originally +implemented by Frank Rosenblatt in 1958. The perceptron is a linear classifier, +and can be understood as a trivial neural network with one neuron that uses the +step function as an activation function. mlpack's implementation of the +`Perceptron` class also offers several template parameters that can be used to +control the behavior of the perceptron. + +Perceptrons are useful for classifying points with _discrete labels_ (i.e., `0`, +`1`, `1`). Because they are simple classifiers, they are also useful as _weak +learners_ for the [`AdaBoost`](#adaboost) boosting classifier. + + +#### Basic usage example excerpt: + +```c++ +Perceptron p; // Step 1: construct object. +p.Train(data, labels, 3); // Step 2: train model. +p.Classify(test_data, test_predictions); // Step 3: use model to classify. +``` + +#### Quick links: + + * [Constructors](#constructors): create `DecisionTree` objects. + * [`Train()`](#training): train model. + * [`Classify()`](#classification): classify with a trained model. + * [Other functionality](#other-functionality) for loading, saving, and + inspecting. + * [Examples](#simple-examples) of simple usage and links to detailed example + projects. + * [Template parameters](#advanced-functionality-template-parameters) for custom + behavior. + * [Advanced template examples](#advanced-functionality-examples) of use with + custom template parameters. + +#### See also: + + * [`NaiveBayesClassifier`](#naive_bayes_classifier), another simple classifier + + * [`AdaBoost`](#adaboost) + * [`FFN`](#ffn) + * [mlpack classifiers](#mlpack_classifiers) + * [Perceptron on Wikipedia](https://en.wikipedia.org/wiki/Perceptron) + +### Constructors + +Construct a `Perceptron` object using one of the constructors below. Defaults +and types are detailed in the [Constructor Parameters](#constructor-parameters) +section below. + +#### Forms: + + + * `Perceptron()` + * `Perceptron(numClasses)` + * `Perceptron(numClasses, dimensionality, maxIterations)` + - **Initialize perceptron without training.** + - Unless `dimensionality` is specified, you will need to call + [`Train()`](#training) later to train the perceptron before calling + [`Classify()`](#classification). + - If specified, `dimensionality` indicates the number of dimensions that the + model will be trained on, and the model will be initialized (to all zeros). + +--- + + * `Perceptron(data, labels, numClasses)` + * `Perceptron(data, labels, numClasses, weights)` + * `Perceptron(data, labels, numClasses, maxIterations)` + * `Perceptron(data, labels, numClasses, weights, maxIterations)` + - **Train the perceptron (optionally with instance weights).** + - If hyperparameters are not specified here, default values are used. + - `labels` should be a vector of length `data.n_cols`, containing values from + `0` to `numClasses - 1` (inclusive). + - If specified, `weights` should be a vector of length `data.n_cols`, + containing instance weights for each point in `data`. + +--- + + * `Perceptron(other_perceptron, data, labels, numClasses, weights)` + - **Train the perceptron using hyperparameters from another perceptron and + weighted data**. + - `labels` should be a vector of length `data.n_cols`, containing values from + `0` to `numClasses - 1` (inclusive). + - `weights` should be a vector of length `data.n_cols`, containing instance + weights for each point in `data`. + +--- + +#### Constructor Parameters: + + + +| **name** | **type** | **description** | **default** | +|----------|----------|-----------------|-------------| +| `data` | [`arma::mat`](../matrices.md) | [Column-major](../matrices.md) training matrix. | _(N/A)_ | +| `datasetInfo` | [`data::DatasetInfo`](../../tutorials/datasetmapper.md) | Dataset information, specifying type information for each dimension. | _(N/A)_ | +| `labels` | [`arma::Row`]('../matrices.md') | Training labels, between `0` and `numClasses - 1` (inclusive). Should have length `data.n_cols`. | _(N/A)_ | +| `weights` | [`arma::rowvec`]('../matrices.md') | Weights for each training point. Should have length `data.n_cols`. | _(N/A)_ | +| `numClasses` | `size_t` | Number of classes in the dataset. | _(N/A)_ | +| `dimensionality` | `size_t` | Dimensionality of data (only needed if an +initialized but untrained model is desired). | _(N/A)_ | +| `maxIterations` | `size_t` | Maximum number of iterations during training. | `1000` | + +### Training + +If training is not done as part of the constructor call, it can be done with one +of the versions of the `Train()` member function. For an instance of +`Perceptron` named `p`, the following functions for training are available: + + + + * `p.Train(data, labels, numClasses)` + * `p.Train(data, labels, numClasses, maxIterations)` + - **Train the perceptron on unweighted data.** + - If hyperparameters are not specified here, and have not been otherwise set, + default values will be used. + - `labels` should be a vector of length `data.n_cols`, containing values from + `0` to `numClasses - 1` (inclusive). + +--- + + * `p.Train(data, labels, numClasses, weights)` + * `p.Train(data, labels, numClasses, weights, maxIterations)` + - **Train the perceptron on data with instance weights.** + - If hyperparameters are not specified here, and have not been otherwise set, + default values will be used. + - `labels` should be a vector of length `data.n_cols`, containing values from + `0` to `numClasses - 1` (inclusive). + - `weights` should be a vector of length `data.n_cols`, containing instance + weights for each point in `data`. + +--- + +Types of each argument are the same as in the table for constructors +[above](#constructor-parameters). + +***Note***: training is incremental. Successive calls to `Train()` will not +reinitialize the model. To reinitialize the model, call `Reset()` (see [Other +Functionality](#other-functionality)). + +### Classification + +Once a `Perceptron` is trained, the `Classify()` member function can be used to +make class predictions for new data. Defaults and types are detailed in the +[Classification Parameters](#classification-parameters) section below. + +#### Forms: + + * `size_t predictedClass = tree.Classify(point)` + - ***(Single-point)*** + - Classify a single point, returning the predicted class. + +--- + + * `tree.Classify(data, predictions)` + - ***(Multi-point)*** + - Classify a set of points. + - The predicted classes of each point is stored in `predictions`, which is + set to length `data.n_cols`. + - The prediction for data point `i` can be accessed with `predictions[i]`. + +--- + +***Note***: perceptrons do not provide any measure resembling probabilities +during classification, and thus a version of `Classify()` that computes class +probabilities is not available. + +#### Classification Parameters: + +| **usage** | **name** | **type** | **description** | +|-----------|----------|----------|-----------------| +| _single-point_ | `point` | [`arma::vec`](../matrices.md) | Single point for classification. | +|||| +| _multi-point_ | `data` | [`arma::mat`](../matrices.md) | Set of [column-major](../matrices.md) points for classification. | +| _multi-point_ | `predictions` | [`arma::Row&`](../matrices.md) | Vector of `size_t`s to store class prediction into. | + +### Other Functionality + + + + * A `DecisionTree` can be serialized with [`data::Save()`](../formats.md) and + [`data::Load()`](../formats.md). + + * `p.NumClasses()` will return a `size_t` indicating the number of classes the + perceptron was trained on. + + * `p.Biases()` will return an `arma::vec` with the biases of the model (each + element corresponds to the bias for a class). + + * `p.Weights()` will return an `arma::mat` with the weights of the model (each + column corresponds to the weights for one class label). + + + + * `p.Reset()` will re-initialize the weights and biases of the model. + +For complete functionality, the [source +code](/src/mlpack/methods/perceptron/perceptron.hpp) can be consulted. Each +method is fully documented. + +### Simple Examples + +Train a perceptron on random numeric data and predict labels on a test set. + +```c++ +// 1000 random points in 10 dimensions. +arma::mat dataset(10, 1000, arma::fill::randu); +// Random labels for each point, totaling 5 classes. +arma::Row labels = + arma::randi>(1000, arma::distr_param(0, 4)); + +// Train in the constructor. +Perceptron<> p(dataset, labels, 5); + +// Create test data (500 points). +arma::mat testDataset(10, 500, arma::fill::randu); +arma::Row predictions; +tree.Classify(testDataset, predictions); +// Now `predictions` holds predictions for the test dataset. + +// Print some information about the test predictions. +std::cout << arma::accu(predictions == 1) << " test points classified as class " + << "1." << std::endl; +``` + +--- + +Train a perceptron multiple times, incrementally, with custom hyperparameters. + +```c++ +// See https://datasets.mlpack.org/iris.csv. +arma::mat dataset; +data::Load("iris.csv", dataset, true); +// See https://datasets.mlpack.org/iris.labels.csv. +arma::Row labels; +data::Load("iris.labels.csv", dataset, true); + +// Create a Perceptron object. +Perceptron<> p; +// Set the maximum number of iterations to 100. (This can also be done in the +// constructor.) +p.MaxIterations() = 100; + +// Train the model for up to 100 iterations. +p.Train(data, labels, 3); + +// Now, compute and print accuracy on the training set. +arma::Row predictions; +p.Classify(data, predictions); +std::cout << "Training set accuracy after 100 iterations: " + << (100.0 * double(arma::accu(labels == predictions)) / labels.n_elem) + << "\%." << std::endl; + +// Train for another 250 iterations and compute training set accuracy again. +p.Train(data, labels, 3, 250); +p.Classify(data, predictions); +std::cout << "Training set accuracy after 350 iterations: " + << (100.0 * double(arma::accu(labels == predictions)) / labels.n_elem) + << "\%." << std::endl; +``` + +--- + +Load a saved perceptron from disk and print information about it. + +```c++ +Perceptron<> p; +// This call assumes a perceptron called "p" has already been saved to +// `perceptron.bin` with `data::Save()`. +data::Load("perceptron.bin", "p", p, true); + +if (p.NumClasses() > 0) +{ + std::cout << "The perceptron in `perceptron.bin` was trained on " + << p.NumClasses() << " classes." << std::endl; + std::cout << "The dimensionality of the perceptron model is " + << p.Weights().n_rows << "." << std::endl; + std::cout << "The bias weights for each class are:" << std::endl; + for (size_t i = 0; i < p.NumClasses(); ++i) + std::cout << " - Class " << i << ": " << p.Biases()[i] << std::endl; +} +else +{ + std::cout << "The perceptron in `perceptron.bin` has not been trained." + << std::endl; +} +``` + +--- + +### Advanced Functionality: Template Parameters + +The `Perceptron` class also supports several template parameters, which can be +used for custom behavior. The full signature of the class is as follows: + +```c++ +Perceptron +``` + + * `LearnPolicy`: the strategy used to learn the weights during training. + * `WeightInitializationPolicy`: the way that weights are initialized before + training. + * `MatType`: specifies the type of matrix used for learning and internal + representation of weights and biases. + +#### `LearnPolicy` + + * Specifies the step to be taken when a point is misclassified. + * The `SimpleWeightUpdate` class is available, and is the default. + * A custom class must implement only one function: + +```c++ +// You can use this as a starting point for implementation. +class CustomLearnPolicy +{ + // Update the weights and biases in the `weights` matrix and the `biases` + // vector given that the model currently classified `trainingPoint` as having + // the label `incorrectClass`, when in reality it has the label + // `correctClass`. If `instanceWeight` is given, it specifies the instance + // weight for the given `trainingPoint`. + // + // `VecType` will be an Armadillo-like vector type. It will be a column from + // the training data matrix (`data`) given to `Train()` or to the constructor. + template + void UpdateWeights(const VecType& trainingPoint, + arma::mat& weights, + arma::vec& biases, + const size_t incorrectClass, + const size_t correctClass, + const double instanceWeight = 1.0); +}; +``` + +#### `WeightInitializationPolicy` + + * Specifies how the weights matrix and biases vector should be initialized when + the `Perceptron` object is created, or when `Reset()` is called. + * The `ZeroInitialization` _(default)_ and `RandomPerceptronInitialization` + classes are available for drop-in usage. + * `RandomPerceptronInitialization` will initialize weights and biases using a + uniform random distribution between 0 and 1. + * A custom class must implement only one function: + +```c++ +// You can use this as a starting point for implementation. +class CustomWeightInitializationPolicy +{ + // Initialize the `weights` matrix and `biases` vector, given that the model + // will have dimensionality of `numFeatures` (that is, the training data + // matrix will have `numFeatures` rows), and the training data has + // `numClasses` classes. + // + // The initialized `weights` matrix should have `numFeatures` rows and + // `numClasses` columns, and the initialized `biases` vector should have + // `numClasses` elements. + // + // `eT` specifies the element type of the weights and biases; it may be + // `double`, `float`, or another floating-point type. + template + inline static void Initialize(arma::Mat& weights, + arma::Col& biases, + const size_t numFeatures, + const size_t numClasses) + { + weights.randu(numFeatures, numClasses); + biases.randu(numClasses); + } +}; +``` + +#### `MatType` + + * Specifies the matrix type to use for data when learning a perceptron. + * By default, `MatType` is `arma::mat` (dense 64-bit precision matrix). + * Any matrix type implementing the Armadillo API will work; so, for instance, + `arma::fmat` or `arma::sp_mat` can be used. + +### Advanced Functionality Examples + +Train a `Perceptron` with random initialization, instead of zero initialization +of weights. + +```c++ +// 1000 random points in 10 dimensions. +arma::mat dataset(10, 1000, arma::fill::randu); +// Random labels for each point, totaling 5 classes. +arma::Row labels = + arma::randi>(1000, arma::distr_param(0, 4)); + +// Train in the constructor. Weights will be initialized randomly. +Perceptron p(dataset, + labels, 5); + +// Create test data (500 points). +arma::mat testDataset(10, 500, arma::fill::randu); +arma::Row predictions; +tree.Classify(testDataset, predictions); +// Now `predictions` holds predictions for the test dataset. + +// Print some information about the test predictions. +std::cout << arma::accu(predictions == 1) << " test points classified as class " + << "1." << std::endl; +``` + +--- + +Train a `Perceptron` on sparse 32-bit floating point data. + +```c++ + +// 1000 sparse random points in 100 dimensions, with 1% nonzero elements. +arma::sp_fmat dataset; +dataset.sprandu(100, 1000, 0.01); +// Random labels for each point, totaling 5 classes. +arma::Row labels = + arma::randi>(1000, arma::distr_param(0, 4)); + +// Train in the constructor. +Perceptron<> p(dataset, labels, 5); + +// Create test data (500 points). +arma::sp_fmat testDataset; +testDataset.sprandu(100, 500, 0.01); +arma::Row predictions; +tree.Classify(testDataset, predictions); +// Now `predictions` holds predictions for the test dataset. + +// Print some information about the test predictions. +std::cout << arma::accu(predictions == 1) << " test points classified as class " + << "1." << std::endl; +``` + +--- From 82080994b396dc3571b7e396701ee450ea795d44 Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Wed, 1 Nov 2023 15:19:55 -0400 Subject: [PATCH 03/47] Incremental steps on AdaBoost documentation. --- doc/user/methods/adaboost.md | 16 ++++++++++++---- src/mlpack/methods/adaboost/adaboost.hpp | 8 ++++---- src/mlpack/methods/adaboost/adaboost_impl.hpp | 10 +++++----- 3 files changed, 21 insertions(+), 13 deletions(-) diff --git a/doc/user/methods/adaboost.md b/doc/user/methods/adaboost.md index a20c6bd5a3..a9e8210b81 100644 --- a/doc/user/methods/adaboost.md +++ b/doc/user/methods/adaboost.md @@ -128,7 +128,8 @@ named `ab`, the following functions for training are available: * `ab.Train(data, labels, numClasses)` * `ab.Train(data, labels, numClasses, maxIterations, tolerance)` - **Train model using default weak learner parameters.** - - If hyperparameters are not specified, default values are used. + - If hyperparameters are not specified here, and have not been otherwise set, + default values are used. - `labels` should be a vector of length `data.n_cols`, containing values from `0` to `numClasses - 1` (inclusive). @@ -141,7 +142,8 @@ named `ab`, the following functions for training are available: settings in `weakLearner` are used for training each AdaBoost weak learner (see the [simple examples](#simple-examples)). - - If hyperparameters are not specified, default values are used. + - If hyperparameters for AdaBoost are not specified, and have not been + otherwise set, default values are used. - `labels` should be a vector of length `data.n_cols`, containing values from `0` to `numClasses - 1` (inclusive). @@ -234,6 +236,10 @@ the [Classification Parameters](#classification-parameters) section below. * `ab.WeakLearner(i)` will return the `i`th weak learner. +For complete functionality, the [source +code](/src/mlpack/methods/adaboost/adaboost.hpp) can be consulted. Each method +is fully documented. + ### Simple Examples Train an AdaBoost model on random data and predict labels on a random test set. @@ -269,7 +275,7 @@ arma::mat dataset; data::Load("iris.csv", dataset, true); // See https://datasets.mlpack.org/iris.labels.csv. arma::Row labels; -data::Load("iris_labels.csv", dataset, true); +data::Load("iris.labels.csv", dataset, true); // Create a weak learner with the desired hyperparameters. Perceptron<> p; @@ -371,6 +377,7 @@ AdaBoost supporting instance weights can also be used. ```c++ +// You can use this as a starting point for implementation. class CustomWeakLearner { public: @@ -413,7 +420,8 @@ class CustomWeakLearner * Specifies the matrix type to use for data when learning a model (or predicting with one). * By default, `MatType` is `arma::mat` (dense 64-bit precision matrix). - * `arma::fmat` or `arma::sp_mat` could also be used. + * Any matrix type implementing the Armadillo API will work; so, for instance, + `arma::fmat` or `arma::sp_mat` can also be used. ### Advanced Functionality Examples diff --git a/src/mlpack/methods/adaboost/adaboost.hpp b/src/mlpack/methods/adaboost/adaboost.hpp index 9bf94aab79..49a7729455 100644 --- a/src/mlpack/methods/adaboost/adaboost.hpp +++ b/src/mlpack/methods/adaboost/adaboost.hpp @@ -89,7 +89,7 @@ class AdaBoost * @param data Input data. * @param labels Corresponding labels. * @param numClasses The number of classes. - * @param iterations Number of boosting rounds. + * @param maxIterations Number of boosting rounds. * @param tolerance The tolerance for change in values of rt. * @param other Weak learner that has already been initialized. */ @@ -97,7 +97,7 @@ class AdaBoost const arma::Row& labels, const size_t numClasses, const WeakLearnerType& other, - const size_t iterations = 100, + const size_t maxIterations = 100, const double tolerance = 1e-6); /** @@ -138,7 +138,7 @@ class AdaBoost * @param labels Labels for each point in the dataset. * @param numClasses The number of classes. * @param learner Learner to use for training. - * @param iterations Number of boosting rounds. + * @param maxIterations Number of boosting rounds. * @param tolerance The tolerance for change in values of rt. * @return The upper bound for training error. */ @@ -146,7 +146,7 @@ class AdaBoost const arma::Row& labels, const size_t numClasses, const WeakLearnerType& learner, - const size_t iterations = 100, + const size_t maxIterations = 100, const double tolerance = 1e-6); /** diff --git a/src/mlpack/methods/adaboost/adaboost_impl.hpp b/src/mlpack/methods/adaboost/adaboost_impl.hpp index ffa5a4bddd..b07f33bf13 100644 --- a/src/mlpack/methods/adaboost/adaboost_impl.hpp +++ b/src/mlpack/methods/adaboost/adaboost_impl.hpp @@ -35,7 +35,7 @@ namespace mlpack { * * @param data Input data * @param labels Corresponding labels - * @param iterations Number of boosting rounds + * @param maxIterations Number of boosting rounds * @param tol Tolerance for termination of Adaboost.MH. * @param other Weak Learner, which has been initialized already. */ @@ -45,10 +45,10 @@ AdaBoost::AdaBoost( const arma::Row& labels, const size_t numClasses, const WeakLearnerType& other, - const size_t iterations, + const size_t maxIterations, const double tol) { - Train(data, labels, numClasses, other, iterations, tol); + Train(data, labels, numClasses, other, maxIterations, tol); } // Empty constructor. @@ -67,7 +67,7 @@ double AdaBoost::Train( const arma::Row& labels, const size_t numClasses, const WeakLearnerType& other, - const size_t iterations, + const size_t maxIterations, const double tolerance) { // Clear information from previous runs. @@ -105,7 +105,7 @@ double AdaBoost::Train( arma::Row finalH(predictedLabels.n_cols); // Now, start the boosting rounds. - for (size_t i = 0; i < iterations; ++i) + for (size_t i = 0; i < maxIterations; ++i) { // Initialized to zero in every round. rt is used for calculation of // alphat; it is the weighted error. From a920b912dc017ca3054476e9716175d1a4874e8f Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Wed, 8 Nov 2023 11:15:21 -0500 Subject: [PATCH 04/47] Finish perceptron documentation; apply significant cleanups. --- doc/user/methods/perceptron.md | 137 +++++++++++++-------------------- 1 file changed, 55 insertions(+), 82 deletions(-) diff --git a/doc/user/methods/perceptron.md b/doc/user/methods/perceptron.md index 51b3313b99..75b497186b 100644 --- a/doc/user/methods/perceptron.md +++ b/doc/user/methods/perceptron.md @@ -8,17 +8,32 @@ step function as an activation function. mlpack's implementation of the control the behavior of the perceptron. Perceptrons are useful for classifying points with _discrete labels_ (i.e., `0`, -`1`, `1`). Because they are simple classifiers, they are also useful as _weak +`1`). Because they are simple classifiers, they are also useful as _weak learners_ for the [`AdaBoost`](#adaboost) boosting classifier. -#### Basic usage example excerpt: +#### Simple usage example: ```c++ -Perceptron p; // Step 1: construct object. -p.Train(data, labels, 3); // Step 2: train model. -p.Classify(test_data, test_predictions); // Step 3: use model to classify. +// Train a perceptron on random numeric data and predict labels on test data: + +// All data and labels are uniform random; 10 dimensional data, 5 classes. +// Replace with a data::Load() call or similar for a real application. +arma::mat dataset(10, 1000, arma::fill::randu); +arma::Row labels = + arma::randi>(1000, arma::distr_param(0, 4)); +arma::mat testDataset(10, 500, arma::fill::randu); // 500 test points. + +Perceptron<> p; // Step 1: create model. +p.Train(dataset, labels, 5); // Step 2: train model. +arma::Row predictions; +tree.Classify(testDataset, predictions); // Step 3: classify points. + +// Print some information about the test predictions. +std::cout << arma::accu(predictions == 1) << " test points classified as class " + << "1." << std::endl; ``` +

More examples...

#### Quick links: @@ -52,38 +67,23 @@ section below. #### Forms: - * `Perceptron()` - * `Perceptron(numClasses)` - * `Perceptron(numClasses, dimensionality, maxIterations)` - - **Initialize perceptron without training.** - - Unless `dimensionality` is specified, you will need to call - [`Train()`](#training) later to train the perceptron before calling - [`Classify()`](#classification). - - If specified, `dimensionality` indicates the number of dimensions that the - model will be trained on, and the model will be initialized (to all zeros). + * `p = Perceptron()` + - Initialize perceptron without training. + - You will need to call [`Train()`](#training) later to train the perceptron + before calling [`Classify()`](#classification). --- - * `Perceptron(data, labels, numClasses)` - * `Perceptron(data, labels, numClasses, weights)` - * `Perceptron(data, labels, numClasses, maxIterations)` - * `Perceptron(data, labels, numClasses, weights, maxIterations)` - - **Train the perceptron (optionally with instance weights).** - - If hyperparameters are not specified here, default values are used. - - `labels` should be a vector of length `data.n_cols`, containing values from - `0` to `numClasses - 1` (inclusive). - - If specified, `weights` should be a vector of length `data.n_cols`, - containing instance weights for each point in `data`. + * `p = Perceptron(numClasses, dimensionality, maxIterations=1000)` + - Initialize perceptron with all-zero weights and biases. + - `Classify()` can immediately be used; training is not required with this + form. --- - * `Perceptron(other_perceptron, data, labels, numClasses, weights)` - - **Train the perceptron using hyperparameters from another perceptron and - weighted data**. - - `labels` should be a vector of length `data.n_cols`, containing values from - `0` to `numClasses - 1` (inclusive). - - `weights` should be a vector of length `data.n_cols`, containing instance - weights for each point in `data`. + * `p = Perceptron(data, labels, numClasses, maxIterations=1000)` + * `p = Perceptron(data, labels, numClasses, weights, maxIterations=1000)` + - Train the perceptron (optionally with instance weights). --- @@ -103,37 +103,23 @@ section below. | `labels` | [`arma::Row`]('../matrices.md') | Training labels, between `0` and `numClasses - 1` (inclusive). Should have length `data.n_cols`. | _(N/A)_ | | `weights` | [`arma::rowvec`]('../matrices.md') | Weights for each training point. Should have length `data.n_cols`. | _(N/A)_ | | `numClasses` | `size_t` | Number of classes in the dataset. | _(N/A)_ | -| `dimensionality` | `size_t` | Dimensionality of data (only needed if an -initialized but untrained model is desired). | _(N/A)_ | +| `dimensionality` | `size_t` | Dimensionality of data (only used if an initialized but untrained model is desired). | _(N/A)_ | | `maxIterations` | `size_t` | Maximum number of iterations during training. | `1000` | ### Training If training is not done as part of the constructor call, it can be done with one -of the versions of the `Train()` member function. For an instance of -`Perceptron` named `p`, the following functions for training are available: +of the following versions of the `Train()` member function: - * `p.Train(data, labels, numClasses)` - * `p.Train(data, labels, numClasses, maxIterations)` - - **Train the perceptron on unweighted data.** - - If hyperparameters are not specified here, and have not been otherwise set, - default values will be used. - - `labels` should be a vector of length `data.n_cols`, containing values from - `0` to `numClasses - 1` (inclusive). + * `p.Train(data, labels, numClasses, maxIterations=1000)` + - Train the perceptron on unweighted data. --- - * `p.Train(data, labels, numClasses, weights)` - * `p.Train(data, labels, numClasses, weights, maxIterations)` - - **Train the perceptron on data with instance weights.** - - If hyperparameters are not specified here, and have not been otherwise set, - default values will be used. - - `labels` should be a vector of length `data.n_cols`, containing values from - `0` to `numClasses - 1` (inclusive). - - `weights` should be a vector of length `data.n_cols`, containing instance - weights for each point in `data`. + * `p.Train(data, labels, numClasses, weights, maxIterations=1000)` + - Train the perceptron on data with instance weights. --- @@ -141,8 +127,9 @@ Types of each argument are the same as in the table for constructors [above](#constructor-parameters). ***Note***: training is incremental. Successive calls to `Train()` will not -reinitialize the model. To reinitialize the model, call `Reset()` (see [Other -Functionality](#other-functionality)). +reinitialize the model, unless the given data has different dimensionality or +`numClasses` is different. To reinitialize the model, call `Reset()` (see +[Other Functionality](#other-functionality)). ### Classification @@ -161,8 +148,6 @@ make class predictions for new data. Defaults and types are detailed in the * `tree.Classify(data, predictions)` - ***(Multi-point)*** - Classify a set of points. - - The predicted classes of each point is stored in `predictions`, which is - set to length `data.n_cols`. - The prediction for data point `i` can be accessed with `predictions[i]`. --- @@ -178,13 +163,13 @@ probabilities is not available. | _single-point_ | `point` | [`arma::vec`](../matrices.md) | Single point for classification. | |||| | _multi-point_ | `data` | [`arma::mat`](../matrices.md) | Set of [column-major](../matrices.md) points for classification. | -| _multi-point_ | `predictions` | [`arma::Row&`](../matrices.md) | Vector of `size_t`s to store class prediction into. | +| _multi-point_ | `predictions` | [`arma::Row&`](../matrices.md) | Vector of `size_t`s to store class prediction into. Will be set to length `data.n_cols`. | ### Other Functionality - * A `DecisionTree` can be serialized with [`data::Save()`](../formats.md) and + * A `Perceptron` can be serialized with [`data::Save()`](../formats.md) and [`data::Load()`](../formats.md). * `p.NumClasses()` will return a `size_t` indicating the number of classes the @@ -206,28 +191,8 @@ method is fully documented. ### Simple Examples -Train a perceptron on random numeric data and predict labels on a test set. - -```c++ -// 1000 random points in 10 dimensions. -arma::mat dataset(10, 1000, arma::fill::randu); -// Random labels for each point, totaling 5 classes. -arma::Row labels = - arma::randi>(1000, arma::distr_param(0, 4)); - -// Train in the constructor. -Perceptron<> p(dataset, labels, 5); - -// Create test data (500 points). -arma::mat testDataset(10, 500, arma::fill::randu); -arma::Row predictions; -tree.Classify(testDataset, predictions); -// Now `predictions` holds predictions for the test dataset. - -// Print some information about the test predictions. -std::cout << arma::accu(predictions == 1) << " test points classified as class " - << "1." << std::endl; -``` +See also the [simple usage example](#simple-usage-example) for a trivial use of +`Perceptron`. --- @@ -311,6 +276,8 @@ Perceptron + // + // `eT` is the element type of the Perceptron (e.g. `float`, `double`). + template void UpdateWeights(const VecType& trainingPoint, - arma::mat& weights, - arma::vec& biases, + arma::Mat& weights, + arma::Col& biases, const size_t incorrectClass, const size_t correctClass, const double instanceWeight = 1.0); }; ``` +--- + #### `WeightInitializationPolicy` * Specifies how the weights matrix and biases vector should be initialized when @@ -376,6 +347,8 @@ class CustomWeightInitializationPolicy }; ``` +--- + #### `MatType` * Specifies the matrix type to use for data when learning a perceptron. From 452cd34002d47fd1107aa82329614c82073e8411 Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Wed, 8 Nov 2023 11:15:39 -0500 Subject: [PATCH 05/47] Allow both arma::uword and size_t to size checks. --- src/mlpack/core/util/size_checks.hpp | 51 ++++++++++++++++------------ 1 file changed, 30 insertions(+), 21 deletions(-) diff --git a/src/mlpack/core/util/size_checks.hpp b/src/mlpack/core/util/size_checks.hpp index f5125a591b..7f62a0b450 100644 --- a/src/mlpack/core/util/size_checks.hpp +++ b/src/mlpack/core/util/size_checks.hpp @@ -32,13 +32,16 @@ namespace util { * before size-check. Default is false. */ template -inline void CheckSameSizes(const DataType& data, - const LabelsType& label, - const std::string& callerDescription, - const std::string& addInfo = "labels", - const bool& isDataTranspose = false, - const bool& isLabelTranspose = false) -{ +inline void CheckSameSizes( + const DataType& data, + const LabelsType& label, + const std::string& callerDescription, + const std::string& addInfo = "labels", + const bool& isDataTranspose = false, + const bool& isLabelTranspose = false, + const typename std::enable_if< + !std::is_integral::value>::type* = 0) +{ const size_t dataPoints = (isDataTranspose == true) ? data.n_rows : data.n_cols; const size_t labelPoints = (isLabelTranspose == true) ? label.n_rows : label.n_cols; @@ -56,11 +59,13 @@ inline void CheckSameSizes(const DataType& data, * An overload of CheckSameSizes() where the size to be checked is known * previously. The second parameter is of type unsigned int. */ -template -inline void CheckSameSizes(const DataType& data, - const size_t& size, - const std::string& callerDescription, - const std::string& addInfo = "labels") +template +inline void CheckSameSizes( + const DataType& data, + const SizeType& size, + const std::string& callerDescription, + const std::string& addInfo = "labels", + const typename std::enable_if::value>::type* = 0) { if (data.n_cols != size) { @@ -84,10 +89,12 @@ inline void CheckSameSizes(const DataType& data, * is "dataset"; for example, "weights" could also be used. */ template -inline void CheckSameDimensionality(const DataType& data, - const DimType& dimension, - const std::string& callerDescription, - const std::string& addInfo = "dataset") +inline void CheckSameDimensionality( + const DataType& data, + const DimType& dimension, + const std::string& callerDescription, + const std::string& addInfo = "dataset", + const typename std::enable_if::value>::type* = 0) { if (data.n_rows != dimension.n_rows) { @@ -104,11 +111,13 @@ inline void CheckSameDimensionality(const DataType& data, * An overload of CheckSameDimensionality() where the dimension to be checked * is known second param is unsigned long int. */ -template -inline void CheckSameDimensionality(const DataType& data, - const size_t& dimension, - const std::string& callerDescription, - const std::string& addInfo = "dataset") +template +inline void CheckSameDimensionality( + const DataType& data, + const DimType& dimension, + const std::string& callerDescription, + const std::string& addInfo = "dataset", + const typename std::enable_if::value>::type* = 0) { if (data.n_rows != dimension) { From 099172d665bd5a167703c9f24f2e9ea5275be953 Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Wed, 8 Nov 2023 11:16:12 -0500 Subject: [PATCH 06/47] Deprecate constructor used by AdaBoost only. --- src/mlpack/methods/perceptron/perceptron.hpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/mlpack/methods/perceptron/perceptron.hpp b/src/mlpack/methods/perceptron/perceptron.hpp index 2f8dfa5cc6..37b7ab2b39 100644 --- a/src/mlpack/methods/perceptron/perceptron.hpp +++ b/src/mlpack/methods/perceptron/perceptron.hpp @@ -101,6 +101,7 @@ class Perceptron * @param instanceWeights Weight vector to use while training. For boosting * purposes. */ + mlpack_deprecated /* was previously only used by AdaBoost */ Perceptron(const Perceptron& other, const MatType& data, const arma::Row& labels, From 72c2b11df0bb377909f2d8ddfcaa1239feca1235 Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Wed, 8 Nov 2023 11:17:35 -0500 Subject: [PATCH 07/47] Add missing Reset() function. --- src/mlpack/methods/perceptron/perceptron.hpp | 17 ++++++++++++++++- .../methods/perceptron/perceptron_impl.hpp | 13 +++++++++++++ 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/src/mlpack/methods/perceptron/perceptron.hpp b/src/mlpack/methods/perceptron/perceptron.hpp index 37b7ab2b39..229f7d745a 100644 --- a/src/mlpack/methods/perceptron/perceptron.hpp +++ b/src/mlpack/methods/perceptron/perceptron.hpp @@ -128,6 +128,15 @@ class Perceptron const size_t numClasses, const arma::rowvec& instanceWeights = arma::rowvec()); + /** + * After training, use the weights matrix to classify `point`, and return the + * predicted class. + * + * @param point Test point to classify. + */ + template + size_t Classify(const VecType& point) const; + /** * Classification function. After training, use the weights matrix to * classify test, and put the predicted classes in predictedLabels. @@ -136,7 +145,13 @@ class Perceptron * @param predictedLabels Vector to store the predicted classes after * classifying test. */ - void Classify(const MatType& test, arma::Row& predictedLabels); + void Classify(const MatType& test, arma::Row& predictedLabels) const; + + /** + * Reset the model, so that the next call to `Train()` will not be + * incremental. + */ + void Reset(); /** * Serialize the perceptron. diff --git a/src/mlpack/methods/perceptron/perceptron_impl.hpp b/src/mlpack/methods/perceptron/perceptron_impl.hpp index 1f1ab92719..e796379cb8 100644 --- a/src/mlpack/methods/perceptron/perceptron_impl.hpp +++ b/src/mlpack/methods/perceptron/perceptron_impl.hpp @@ -218,6 +218,19 @@ void Perceptron::Train( } } +/** + * Reset the model, so that the next call to `Train()` will not be + * incremental. + */ +template +void Perceptron::Reset() +{ + weights.clear(); + biases.clear(); +} + //! Serialize the perceptron. template Date: Wed, 8 Nov 2023 11:18:05 -0500 Subject: [PATCH 08/47] Add additional Train() overloads, and split functionality as needed. Generalize to any MatType. --- src/mlpack/methods/perceptron/perceptron.hpp | 93 +++++++- .../methods/perceptron/perceptron_impl.hpp | 200 +++++++++++++++--- 2 files changed, 260 insertions(+), 33 deletions(-) diff --git a/src/mlpack/methods/perceptron/perceptron.hpp b/src/mlpack/methods/perceptron/perceptron.hpp index 229f7d745a..6268146b70 100644 --- a/src/mlpack/methods/perceptron/perceptron.hpp +++ b/src/mlpack/methods/perceptron/perceptron.hpp @@ -34,6 +34,9 @@ template& labels, + const size_t numClasses); + + /** + * Train the perceptron on the given data for up to the given maximum number + * of iterations. A single iteration corresponds to a single pass through the + * data, so if you want to pass through the dataset only once, set + * `maxIterations` to 1. + * + * After calling this overload, `MaxIterations()` will return whatever + * `maxIterations` was given to this function. + * + * This training does not reset the model weights, so you can call Train() on + * multiple datasets sequentially. + * + * @param data Dataset on which training should be performed. + * @param labels Labels of the dataset. + * @param numClasses Number of classes in the data. + * @param maxIterations Maximum number of iterations for training. + */ + void Train(const MatType& data, + const arma::Row& labels, + const size_t numClasses, + const size_t maxIterations); + /** * Train the perceptron on the given data for up to the maximum number of * iterations (specified in the constructor or through MaxIterations()). A @@ -126,7 +168,32 @@ class Perceptron void Train(const MatType& data, const arma::Row& labels, const size_t numClasses, - const arma::rowvec& instanceWeights = arma::rowvec()); + const arma::rowvec& instanceWeights); + + /** + * Train the perceptron on the given data for up to the given maximum number + * of iterations. A single iteration corresponds to a single pass through the + * data, so if you want to pass through the dataset only once, set + * `maxIterations` to 1. + * + * After calling this overload, `MaxIterations()` will return whatever + * `maxIterations` was given to this function. + * + * This training does not reset the model weights, so you can call Train() on + * multiple datasets sequentially. + * + * @param data Dataset on which training should be performed. + * @param labels Labels of the dataset. + * @param numClasses Number of classes in the data. + * @param instanceWeights Cost matrix. Stores the cost of mispredicting + * instances. This is useful for boosting. + * @param maxIterations Maximum number of iterations for training. + */ + void Train(const MatType& data, + const arma::Row& labels, + const size_t numClasses, + const arma::rowvec& instanceWeights, + const size_t maxIterations); /** * After training, use the weights matrix to classify `point`, and return the @@ -168,16 +235,28 @@ class Perceptron size_t NumClasses() const { return weights.n_cols; } //! Get the weight matrix. - const arma::mat& Weights() const { return weights; } + const arma::Mat& Weights() const { return weights; } //! Modify the weight matrix. You had better know what you are doing! - arma::mat& Weights() { return weights; } + arma::Mat& Weights() { return weights; } //! Get the biases. - const arma::vec& Biases() const { return biases; } + const arma::Col& Biases() const { return biases; } //! Modify the biases. You had better know what you are doing! - arma::vec& Biases() { return biases; } + arma::Col& Biases() { return biases; } private: + /** + * Internal training function; this assumes that maxIterations has been set. + * + * If `HasWeights` is `false`, then `instanceWeights` is ignored (and may be + * left empty). + */ + template + void TrainInternal(const MatType& data, + const arma::Row& labels, + const size_t numClasses, + const arma::rowvec& instanceWeights = arma::rowvec()); + //! The maximum number of iterations during training. size_t maxIterations; @@ -187,10 +266,10 @@ class Perceptron * the weights for one dimension of the input data. The biases are held in a * separate vector. */ - arma::mat weights; + arma::Mat weights; //! The biases for each class. - arma::vec biases; + arma::Col biases; }; } // namespace mlpack diff --git a/src/mlpack/methods/perceptron/perceptron_impl.hpp b/src/mlpack/methods/perceptron/perceptron_impl.hpp index e796379cb8..0bc081d619 100644 --- a/src/mlpack/methods/perceptron/perceptron_impl.hpp +++ b/src/mlpack/methods/perceptron/perceptron_impl.hpp @@ -58,7 +58,7 @@ Perceptron::Perceptron( maxIterations(maxIterations) { // Start training. - Train(data, labels, numClasses); + TrainInternal(data, labels, numClasses); } /** @@ -84,7 +84,7 @@ Perceptron::Perceptron( maxIterations(maxIterations) { // Start training. - Train(data, labels, numClasses, instanceWeights); + TrainInternal(data, labels, numClasses, instanceWeights); } /** @@ -111,37 +111,66 @@ Perceptron::Perceptron( const arma::rowvec& instanceWeights) : maxIterations(other.maxIterations) { - Train(data, labels, numClasses, instanceWeights); + TrainInternal(data, labels, numClasses, instanceWeights); } /** - * Classification function. After training, use the weights matrix to classify - * test, and put the predicted classes in predictedLabels. + * Train the perceptron on the given data for up to the maximum number of + * iterations (specified in the constructor or through MaxIterations()). A + * single iteration corresponds to a single pass through the data, so if you + * want to pass through the dataset only once, set MaxIterations() to 1. * - * @param test Testing data or data to classify. - * @param predictedLabels Vector to store the predicted classes after - * classifying test. + * This training does not reset the model weights, so you can call Train() on + * multiple datasets sequentially. + * + * @param data Dataset on which training should be performed. + * @param labels Labels of the dataset. + * @param numClasses Number of classes in the data. */ template< typename LearnPolicy, typename WeightInitializationPolicy, typename MatType > -void Perceptron::Classify( - const MatType& test, - arma::Row& predictedLabels) +void Perceptron::Train( + const MatType& data, + const arma::Row& labels, + const size_t numClasses) { - arma::vec tempLabelMat; - arma::uword maxIndex = 0; - predictedLabels.set_size(test.n_cols); + TrainInternal(data, labels, numClasses); +} - // Could probably be faster if done in batch. - for (size_t i = 0; i < test.n_cols; ++i) - { - tempLabelMat = weights.t() * test.col(i) + biases; - tempLabelMat.max(maxIndex); - predictedLabels(i) = maxIndex; - } +/** + * Train the perceptron on the given data for up to the given maximum number + * of iterations. A single iteration corresponds to a single pass through the + * data, so if you want to pass through the dataset only once, set + * `maxIterations` to 1. + * + * After calling this overload, `MaxIterations()` will return whatever + * `maxIterations` was given to this function. + * + * This training does not reset the model weights, so you can call Train() on + * multiple datasets sequentially. + * + * @param data Dataset on which training should be performed. + * @param labels Labels of the dataset. + * @param numClasses Number of classes in the data. + * @param maxIterations Maximum number of iterations for training. + */ +template< + typename LearnPolicy, + typename WeightInitializationPolicy, + typename MatType +> +void Perceptron::Train( + const MatType& data, + const arma::Row& labels, + const size_t numClasses, + const size_t maxIterations) +{ + // Set the maximum number of iterations and call unweighted Train(). + this->maxIterations = maxIterations; + TrainInternal(data, labels, numClasses); } /** @@ -163,9 +192,70 @@ void Perceptron::Train( const arma::Row& labels, const size_t numClasses, const arma::rowvec& instanceWeights) +{ + TrainInternal(data, labels, numClasses, instanceWeights); +} + +/** + * Train the perceptron on the given data for up to the given maximum number + * of iterations. A single iteration corresponds to a single pass through the + * data, so if you want to pass through the dataset only once, set + * `maxIterations` to 1. + * + * After calling this overload, `MaxIterations()` will return whatever + * `maxIterations` was given to this function. + * + * This training does not reset the model weights, so you can call Train() on + * multiple datasets sequentially. + * + * @param data Dataset on which training should be performed. + * @param labels Labels of the dataset. + * @param numClasses Number of classes in the data. + * @param instanceWeights Cost matrix. Stores the cost of mispredicting + * instances. This is useful for boosting. + * @param maxIterations Maximum number of iterations for training. + */ +template< + typename LearnPolicy, + typename WeightInitializationPolicy, + typename MatType +> +void Perceptron::Train( + const MatType& data, + const arma::Row& labels, + const size_t numClasses, + const arma::rowvec& instanceWeights, + const size_t maxIterations) +{ + // Set the maximum number of iterations and call weighted training. + this->maxIterations = maxIterations; + TrainInternal(data, labels, numClasses, instanceWeights); +} + +/** + * Training function. It trains on trainData using the cost matrix + * instanceWeights. + * + * @param data Data to train on. + * @param labels Labels of data. + * @param instanceWeights Cost matrix. Stores the cost of mispredicting + * instances. This is useful for boosting. + */ +template< + typename LearnPolicy, + typename WeightInitializationPolicy, + typename MatType +> +template +void Perceptron< + LearnPolicy, WeightInitializationPolicy, MatType +>::TrainInternal(const MatType& data, + const arma::Row& labels, + const size_t numClasses, + const arma::rowvec& instanceWeights) { // Do we need to resize the weights? - if (weights.n_elem != numClasses) + if (weights.n_cols != numClasses || weights.n_rows != data.n_rows) { WeightInitializationPolicy wip; wip.Initialize(weights, biases, data.n_rows, numClasses); @@ -175,12 +265,10 @@ void Perceptron::Train( bool converged = false; size_t tempLabel; arma::uword maxIndexRow = 0, maxIndexCol = 0; - arma::mat tempLabelMat; + arma::Mat tempLabelMat; LearnPolicy LP; - const bool hasWeights = (instanceWeights.n_elem > 0); - while ((i < maxIterations) && (!converged)) { // This outer loop is for each iteration, and we use the 'converged' @@ -207,7 +295,7 @@ void Perceptron::Train( // Send maxIndexRow for knowing which weight to update, send j to know // the value of the vector to update it with. Send tempLabel to know // the correct class. - if (hasWeights) + if (HasWeights) LP.UpdateWeights(data.col(j), weights, biases, maxIndexRow, tempLabel, instanceWeights(j)); else @@ -218,6 +306,66 @@ void Perceptron::Train( } } +/** + * After training, use the weights matrix to classify `point`, and return the + * predicted class. + * + * @param point Test point to classify. + */ +template< + typename LearnPolicy, + typename WeightInitializationPolicy, + typename MatType +> +template +size_t Perceptron::Classify( + const VecType& point) const +{ + util::CheckSameDimensionality(point, weights.n_rows, "Perceptron::Classify()", + "point"); + + arma::Col tempLabelVec; + arma::uword maxIndex = 0; + + tempLabelVec = weights.t() * point + biases; + tempLabelVec.max(maxIndex); + + return size_t(maxIndex); +} + +/** + * Classification function. After training, use the weights matrix to classify + * test, and put the predicted classes in predictedLabels. + * + * @param test Testing data or data to classify. + * @param predictedLabels Vector to store the predicted classes after + * classifying test. + */ +template< + typename LearnPolicy, + typename WeightInitializationPolicy, + typename MatType +> +void Perceptron::Classify( + const MatType& test, + arma::Row& predictedLabels) const +{ + util::CheckSameDimensionality(test, weights.n_rows, "Perceptron::Classify()", + "points"); + + arma::Col tempLabelMat; + arma::uword maxIndex = 0; + predictedLabels.set_size(test.n_cols); + + // Could probably be faster if done in batch. + for (size_t i = 0; i < test.n_cols; ++i) + { + tempLabelMat = weights.t() * test.col(i) + biases; + tempLabelMat.max(maxIndex); + predictedLabels(i) = maxIndex; + } +} + /** * Reset the model, so that the next call to `Train()` will not be * incremental. From 37f479a49a7f83739a35b7dcc16a2ad865c184d6 Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Wed, 8 Nov 2023 11:18:16 -0500 Subject: [PATCH 09/47] Generalize to any MatType. --- .../perceptron/initialization_methods/random_init.hpp | 5 +++-- .../methods/perceptron/initialization_methods/zero_init.hpp | 5 +++-- .../perceptron/learning_policies/simple_weight_update.hpp | 6 +++--- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/src/mlpack/methods/perceptron/initialization_methods/random_init.hpp b/src/mlpack/methods/perceptron/initialization_methods/random_init.hpp index 5b8caa29ac..a4c78a25b3 100644 --- a/src/mlpack/methods/perceptron/initialization_methods/random_init.hpp +++ b/src/mlpack/methods/perceptron/initialization_methods/random_init.hpp @@ -25,8 +25,9 @@ class RandomPerceptronInitialization public: RandomPerceptronInitialization() { } - inline static void Initialize(arma::mat& weights, - arma::vec& biases, + template + inline static void Initialize(arma::Mat& weights, + arma::Col& biases, const size_t numFeatures, const size_t numClasses) { diff --git a/src/mlpack/methods/perceptron/initialization_methods/zero_init.hpp b/src/mlpack/methods/perceptron/initialization_methods/zero_init.hpp index 595559d0aa..084b57a221 100644 --- a/src/mlpack/methods/perceptron/initialization_methods/zero_init.hpp +++ b/src/mlpack/methods/perceptron/initialization_methods/zero_init.hpp @@ -24,8 +24,9 @@ class ZeroInitialization public: ZeroInitialization() { } - inline static void Initialize(arma::mat& weights, - arma::vec& biases, + template + inline static void Initialize(arma::Mat& weights, + arma::Col& biases, const size_t numFeatures, const size_t numClasses) { diff --git a/src/mlpack/methods/perceptron/learning_policies/simple_weight_update.hpp b/src/mlpack/methods/perceptron/learning_policies/simple_weight_update.hpp index f9dfb2e53b..2bffb3946c 100644 --- a/src/mlpack/methods/perceptron/learning_policies/simple_weight_update.hpp +++ b/src/mlpack/methods/perceptron/learning_policies/simple_weight_update.hpp @@ -45,10 +45,10 @@ class SimpleWeightUpdate * @param instanceWeight Weight to be given to this particular point during * training (this is useful for boosting). */ - template + template void UpdateWeights(const VecType& trainingPoint, - arma::mat& weights, - arma::vec& biases, + arma::Mat& weights, + arma::Col& biases, const size_t incorrectClass, const size_t correctClass, const double instanceWeight = 1.0) From 39c0bdebeb17ea1dceef4ff03b47ea3401cc9840 Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Wed, 8 Nov 2023 11:18:27 -0500 Subject: [PATCH 10/47] Add tests for new Perceptron functionality. --- src/mlpack/tests/perceptron_test.cpp | 183 ++++++++++++++++++++------- 1 file changed, 139 insertions(+), 44 deletions(-) diff --git a/src/mlpack/tests/perceptron_test.cpp b/src/mlpack/tests/perceptron_test.cpp index f49dfa2b54..631584bff2 100644 --- a/src/mlpack/tests/perceptron_test.cpp +++ b/src/mlpack/tests/perceptron_test.cpp @@ -42,20 +42,20 @@ TEST_CASE("SimpleWeightUpdateWeights", "[PerceptronTest]") wip.UpdateWeights(trainingPoint, weights, biases, incorrectClass, correctClass); - CHECK(weights(0, 0) == -1); - CHECK(weights(1, 0) == 0); - CHECK(weights(2, 0) == 1); - CHECK(weights(3, 0) == 2); - CHECK(weights(4, 0) == 3); + REQUIRE(weights(0, 0) == -1); + REQUIRE(weights(1, 0) == 0); + REQUIRE(weights(2, 0) == 1); + REQUIRE(weights(3, 0) == 2); + REQUIRE(weights(4, 0) == 3); - CHECK(weights(0, 2) == 7); - CHECK(weights(1, 2) == 8); - CHECK(weights(2, 2) == 9); - CHECK(weights(3, 2) == 10); - CHECK(weights(4, 2) == 11); + REQUIRE(weights(0, 2) == 7); + REQUIRE(weights(1, 2) == 8); + REQUIRE(weights(2, 2) == 9); + REQUIRE(weights(3, 2) == 10); + REQUIRE(weights(4, 2) == 11); - CHECK(biases(0) == 1); - CHECK(biases(2) == 8); + REQUIRE(biases(0) == 1); + REQUIRE(biases(2) == 8); } /** @@ -85,20 +85,20 @@ TEST_CASE("SimpleWeightUpdateInstanceWeight", "[PerceptronTest]") wip.UpdateWeights(trainingPoint, weights, biases, incorrectClass, correctClass, instanceWeight); - CHECK(weights(0, 0) == -3); - CHECK(weights(1, 0) == -4); - CHECK(weights(2, 0) == -5); - CHECK(weights(3, 0) == -6); - CHECK(weights(4, 0) == -7); + REQUIRE(weights(0, 0) == -3); + REQUIRE(weights(1, 0) == -4); + REQUIRE(weights(2, 0) == -5); + REQUIRE(weights(3, 0) == -6); + REQUIRE(weights(4, 0) == -7); - CHECK(weights(0, 2) == 9); - CHECK(weights(1, 2) == 12); - CHECK(weights(2, 2) == 15); - CHECK(weights(3, 2) == 18); - CHECK(weights(4, 2) == 21); + REQUIRE(weights(0, 2) == 9); + REQUIRE(weights(1, 2) == 12); + REQUIRE(weights(2, 2) == 15); + REQUIRE(weights(3, 2) == 18); + REQUIRE(weights(4, 2) == 21); - CHECK(biases(0) == -1); - CHECK(biases(2) == 10); + REQUIRE(biases(0) == -1); + REQUIRE(biases(2) == 10); } /** @@ -120,10 +120,21 @@ TEST_CASE("And", "[PerceptronTest]") Row predictedLabels; p.Classify(testData, predictedLabels); - CHECK(predictedLabels(0, 0) == 0); - CHECK(predictedLabels(0, 1) == 0); - CHECK(predictedLabels(0, 2) == 1); - CHECK(predictedLabels(0, 3) == 0); + REQUIRE(predictedLabels(0) == 0); + REQUIRE(predictedLabels(1) == 0); + REQUIRE(predictedLabels(2) == 1); + REQUIRE(predictedLabels(3) == 0); + + // Test single-point classify too. + predictedLabels(0) = p.Classify(testData.col(0)); + predictedLabels(1) = p.Classify(testData.col(1)); + predictedLabels(2) = p.Classify(testData.col(2)); + predictedLabels(3) = p.Classify(testData.col(3)); + + REQUIRE(predictedLabels(0) == 0); + REQUIRE(predictedLabels(1) == 0); + REQUIRE(predictedLabels(2) == 1); + REQUIRE(predictedLabels(3) == 0); } /** @@ -146,10 +157,10 @@ TEST_CASE("Or", "[PerceptronTest]") Row predictedLabels; p.Classify(testData, predictedLabels); - CHECK(predictedLabels(0, 0) == 1); - CHECK(predictedLabels(0, 1) == 1); - CHECK(predictedLabels(0, 2) == 1); - CHECK(predictedLabels(0, 3) == 0); + REQUIRE(predictedLabels(0, 0) == 1); + REQUIRE(predictedLabels(0, 1) == 1); + REQUIRE(predictedLabels(0, 2) == 1); + REQUIRE(predictedLabels(0, 3) == 0); } /** @@ -174,7 +185,7 @@ TEST_CASE("Random3", "[PerceptronTest]") p.Classify(testData, predictedLabels); for (size_t i = 0; i < predictedLabels.n_cols; ++i) - CHECK(predictedLabels(0, i) == 0); + REQUIRE(predictedLabels(0, i) == 0); } /** @@ -198,35 +209,40 @@ TEST_CASE("TwoPoints", "[PerceptronTest]") Row predictedLabels; p.Classify(testData, predictedLabels); - CHECK(predictedLabels(0, 0) == 0); - CHECK(predictedLabels(0, 1) == 1); + REQUIRE(predictedLabels(0, 0) == 0); + REQUIRE(predictedLabels(0, 1) == 1); } /** * This tests the convergence of the perceptron on a dataset which has a - * non-linearly separable dataset. + * non-linearly separable dataset. We test on multiple element types to ensure + * that MatType can be set correctly. */ -TEST_CASE("NonLinearlySeparableDataset", "[PerceptronTest]") +TEMPLATE_TEST_CASE("NonLinearlySeparableDataset", "[PerceptronTest]", float, + double) { - mat trainData; + typedef TestType eT; + + Mat trainData; trainData = { { 1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8 }, { 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2 } }; Mat labels; labels = { 0, 0, 0, 1, 0, 1, 1, 1, 0, 0, 0, 1, 0, 1, 1, 1 }; - Perceptron<> p(trainData, labels.row(0), 2, 1000); + Perceptron> p(trainData, + labels.row(0), 2, 1000); - mat testData; + Mat testData; testData = { { 3, 4, 5, 6 }, { 3, 2.3, 1.7, 1.5 } }; Row predictedLabels; p.Classify(testData, predictedLabels); - CHECK(predictedLabels(0, 0) == 0); - CHECK(predictedLabels(0, 1) == 0); - CHECK(predictedLabels(0, 2) == 1); - CHECK(predictedLabels(0, 3) == 1); + REQUIRE(predictedLabels(0, 0) == 0); + REQUIRE(predictedLabels(0, 1) == 0); + REQUIRE(predictedLabels(0, 2) == 1); + REQUIRE(predictedLabels(0, 3) == 1); } TEST_CASE("SecondaryConstructor", "[PerceptronTest]") @@ -266,3 +282,82 @@ TEST_CASE("InstanceWeightsConstructor", "[PerceptronTest]") REQUIRE(p.Weights().n_elem > 0); } + +/** + * This tests that incremental training can be stopped with `Reset()`. + */ +TEST_CASE("IncrementalTrainingTest", "[PerceptronTest]") +{ + mat trainData = randu(10, 100); + for (size_t i = 0; i < 50; ++i) + trainData.col(i) -= 0.5; + for (size_t i = 50; i < 100; ++i) + trainData.col(i) += 0.5; + Row labels(100); + labels.subvec(0, 49).zeros(); + labels.subvec(50, 99).ones(); + + Perceptron<> p1, p2; + // This should result in the same model, because the default initialization is + // zeros. + p1.Train(trainData, labels, 2, 1); + p2.Train(trainData, labels, 2, 1); + + REQUIRE(approx_equal(p1.Weights(), p2.Weights(), "absdiff", 1e-5)); + REQUIRE(approx_equal(p1.Biases(), p2.Biases(), "absdiff", 1e-5)); + + // Resetting and retraining p2 should result in the same model. + p2.Reset(); + p2.Train(trainData, labels, 2); + + REQUIRE(approx_equal(p1.Weights(), p2.Weights(), "absdiff", 1e-5)); + REQUIRE(approx_equal(p1.Biases(), p2.Biases(), "absdiff", 1e-5)); + + // Training p1 again should result in a different model. + p1.Train(trainData, labels, 2); + + REQUIRE(!approx_equal(p1.Weights(), p2.Weights(), "absdiff", 1e-5)); + REQUIRE(!approx_equal(p1.Biases(), p2.Biases(), "absdiff", 1e-5)); +} + +// Test all forms of Train(). +TEST_CASE("TrainFormTest", "[PerceptronTest]") +{ + mat trainData; + trainData = { { 1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8 }, + { 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2 } }; + + Row labels; + labels = { 0, 0, 0, 1, 0, 1, 1, 1, 0, 0, 0, 1, 0, 1, 1, 1 }; + + rowvec weights(labels.n_elem); + weights.ones(); + + Perceptron<> p1(trainData, labels, 2); + Perceptron<> p2(trainData, labels, 2, weights); + Perceptron<> p3, p4, p5, p6; + p3.Train(trainData, labels, 2); + p4.Train(trainData, labels, 2, 50); + p5.Train(trainData, labels, 2, weights); + p6.Train(trainData, labels, 2, weights, 50); + + mat testData; + testData = { { 3, 4, 5, 6 }, + { 3, 2.3, 1.7, 1.5 } }; + Row predictions1, predictions2, predictions3, predictions4, + predictions5, predictions6; + Row trueLabels = { 0, 0, 1, 1 }; + p1.Classify(testData, predictions1); + p2.Classify(testData, predictions2); + p3.Classify(testData, predictions3); + p4.Classify(testData, predictions4); + p5.Classify(testData, predictions5); + p6.Classify(testData, predictions6); + + REQUIRE(all(predictions1 == trueLabels)); + REQUIRE(all(predictions2 == trueLabels)); + REQUIRE(all(predictions3 == trueLabels)); + REQUIRE(all(predictions4 == trueLabels)); + REQUIRE(all(predictions5 == trueLabels)); + REQUIRE(all(predictions6 == trueLabels)); +} From b7e759d3a1dfef3a300152a3424af28b97da212d Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Wed, 8 Nov 2023 11:18:36 -0500 Subject: [PATCH 11/47] Fix clarity of dimensionality check. --- .../methods/linear_regression/linear_regression_impl.hpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/mlpack/methods/linear_regression/linear_regression_impl.hpp b/src/mlpack/methods/linear_regression/linear_regression_impl.hpp index fe79f2ed55..4d8265ae2a 100644 --- a/src/mlpack/methods/linear_regression/linear_regression_impl.hpp +++ b/src/mlpack/methods/linear_regression/linear_regression_impl.hpp @@ -104,10 +104,10 @@ inline void LinearRegression::Predict( // We want to be sure we have the correct number of dimensions in the // dataset. // Prevent underflow. - const size_t labels = (parameters.n_rows == 0) ? size_t(0) : + const size_t dimensionality = (parameters.n_rows == 0) ? size_t(0) : size_t(parameters.n_rows - 1); - util::CheckSameDimensionality(points, labels, "LinearRegression::Predict()", - "points"); + util::CheckSameDimensionality(points, dimensionality, + "LinearRegression::Predict()", "points"); // Get the predictions, but this ignores the intercept value // (parameters[0]). predictions = arma::trans(parameters.subvec(1, parameters.n_elem - 1)) From 26c5d600b73c196959d6f255e20a40599e039618 Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Wed, 8 Nov 2023 16:32:48 -0500 Subject: [PATCH 12/47] Perceptrons are multi-class. --- doc/user/methods/perceptron.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/user/methods/perceptron.md b/doc/user/methods/perceptron.md index 75b497186b..bd95a69cf3 100644 --- a/doc/user/methods/perceptron.md +++ b/doc/user/methods/perceptron.md @@ -8,7 +8,7 @@ step function as an activation function. mlpack's implementation of the control the behavior of the perceptron. Perceptrons are useful for classifying points with _discrete labels_ (i.e., `0`, -`1`). Because they are simple classifiers, they are also useful as _weak +`1`, `2`). Because they are simple classifiers, they are also useful as _weak learners_ for the [`AdaBoost`](#adaboost) boosting classifier. From 97b55607390d2c6c88c45a989d353e62f1ac95af Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Wed, 8 Nov 2023 16:55:59 -0500 Subject: [PATCH 13/47] Some additional cleanups and fixes. --- doc/user/methods/perceptron.md | 28 ++++++++++++++++------------ 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/doc/user/methods/perceptron.md b/doc/user/methods/perceptron.md index bd95a69cf3..e4743a69f8 100644 --- a/doc/user/methods/perceptron.md +++ b/doc/user/methods/perceptron.md @@ -66,7 +66,6 @@ section below. #### Forms: - * `p = Perceptron()` - Initialize perceptron without training. - You will need to call [`Train()`](#training) later to train the perceptron @@ -104,15 +103,13 @@ section below. | `weights` | [`arma::rowvec`]('../matrices.md') | Weights for each training point. Should have length `data.n_cols`. | _(N/A)_ | | `numClasses` | `size_t` | Number of classes in the dataset. | _(N/A)_ | | `dimensionality` | `size_t` | Dimensionality of data (only used if an initialized but untrained model is desired). | _(N/A)_ | -| `maxIterations` | `size_t` | Maximum number of iterations during training. | `1000` | +| `maxIterations` | `size_t` | Maximum number of iterations during training. Can also be set with `MaxIterations()`. | `1000` | ### Training If training is not done as part of the constructor call, it can be done with one of the following versions of the `Train()` member function: - - * `p.Train(data, labels, numClasses, maxIterations=1000)` - Train the perceptron on unweighted data. @@ -126,10 +123,15 @@ of the following versions of the `Train()` member function: Types of each argument are the same as in the table for constructors [above](#constructor-parameters). -***Note***: training is incremental. Successive calls to `Train()` will not -reinitialize the model, unless the given data has different dimensionality or -`numClasses` is different. To reinitialize the model, call `Reset()` (see -[Other Functionality](#other-functionality)). +***Notes***: + + * Training is incremental. Successive calls to `Train()` will not reinitialize + the model, unless the given data has different dimensionality or `numClasses` + is different. To reinitialize the model, call `Reset()` (see + [Other Functionality](#other-functionality)). + + * If `maxIterations` is not passed, but has been set in the constructor or with + `MaxIterations()`, the previous setting will be used. ### Classification @@ -139,13 +141,13 @@ make class predictions for new data. Defaults and types are detailed in the #### Forms: - * `size_t predictedClass = tree.Classify(point)` + * `size_t predictedClass = p.Classify(point)` - ***(Single-point)*** - Classify a single point, returning the predicted class. --- - * `tree.Classify(data, predictions)` + * `p.Classify(data, predictions)` - ***(Multi-point)*** - Classify a set of points. - The prediction for data point `i` can be accessed with `predictions[i]`. @@ -181,10 +183,12 @@ probabilities is not available. * `p.Weights()` will return an `arma::mat` with the weights of the model (each column corresponds to the weights for one class label). - - * `p.Reset()` will re-initialize the weights and biases of the model. + * `p.MaxIterations()` can be used to get or set the maximum number of + iterations for training; e.g., `p.MaxIterations() = 500` will set the maximum + number of iterations to 500. + For complete functionality, the [source code](/src/mlpack/methods/perceptron/perceptron.hpp) can be consulted. Each method is fully documented. From 68f5acda9421cfe580575030c668d9c97d042958 Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Mon, 13 Nov 2023 14:18:06 -0500 Subject: [PATCH 14/47] Slight cleanup to standalone functions. --- doc/user/methods/perceptron.md | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/doc/user/methods/perceptron.md b/doc/user/methods/perceptron.md index e4743a69f8..51d1dca40f 100644 --- a/doc/user/methods/perceptron.md +++ b/doc/user/methods/perceptron.md @@ -105,6 +105,13 @@ section below. | `dimensionality` | `size_t` | Dimensionality of data (only used if an initialized but untrained model is desired). | _(N/A)_ | | `maxIterations` | `size_t` | Maximum number of iterations during training. Can also be set with `MaxIterations()`. | `1000` | +As an alternative to passing `maxIterations`, it can be set with a standalone +method. The following function can be used before calling `Train()` to set +the maximum number of iterations: + + * `p.MaxIterations() = maxIter;` will set the maximum number of iterations + during training to `maxIter`. + ### Training If training is not done as part of the constructor call, it can be done with one @@ -185,10 +192,6 @@ probabilities is not available. * `p.Reset()` will re-initialize the weights and biases of the model. - * `p.MaxIterations()` can be used to get or set the maximum number of - iterations for training; e.g., `p.MaxIterations() = 500` will set the maximum - number of iterations to 500. - For complete functionality, the [source code](/src/mlpack/methods/perceptron/perceptron.hpp) can be consulted. Each method is fully documented. From cd3c6da4f1fc55d0c6b05be7927f029d77bd5282 Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Mon, 13 Nov 2023 16:10:51 -0500 Subject: [PATCH 15/47] Clean up AdaBoost documentation; move example to top; remove unneeded text. --- doc/user/methods/adaboost.md | 140 ++++++++++++++++------------------- 1 file changed, 64 insertions(+), 76 deletions(-) diff --git a/doc/user/methods/adaboost.md b/doc/user/methods/adaboost.md index a9e8210b81..4fc79c73da 100644 --- a/doc/user/methods/adaboost.md +++ b/doc/user/methods/adaboost.md @@ -8,13 +8,30 @@ default, the `Perceptron` class is used as a weak learner. `AdaBoost` is useful for classifying points with _discrete labels_ (i.e. `0`, `1`, `2`). -#### Basic usage example excerpt: +#### Simple usage example: + +Train an AdaBoost model on random data and predict labels on a random test set. ```c++ -AdaBoost ab; // Step 1: construct object. -ab.Train(data, labels, 3); // Step 2: train model. -ab.Classify(testData, testPredictions); // Step 3: use model to classify. +// Train an AdaBoost model on random data and predict labels on test data: + +// All data and labels are uniform random; 10 dimensional data, 5 classes. +// Replace with a data::Load() call or similar for a real application. +arma::mat dataset(10, 1000, arma::fill::randu); // 1000 points. +arma::Row labels = + arma::randi>(1000, arma::distr_param(0, 4)); +arma::mat testDataset(10, 500, arma::fill::randu); // 500 test points. + +AdaBoost<> ab; // Step 1: create model. +ab.Train(dataset, labels, 5); // Step 2: train model. +arma::Row predictions; +ab.Classify(testDataset, predictions); // Step 3: classify points. + +// Print some information about the test predictions. +std::cout << arma::accu(predictions == 3) << " test points classified as class " + << "3." << std::endl; ``` +

More examples...

#### Quick links: @@ -40,44 +57,39 @@ ab.Classify(testData, testPredictions); // Step 3: use model to classify. ### Constructors -Construct an `AdaBoost` object using one of the constructors below. Defaults -and types are detailed in the [Constructor Parameters](#constructor-parameters) -section below. - -#### Forms: - - * `AdaBoost()` - * `AdaBoost(tolerance)` - - **Initialize model without training.** + * `ab = AdaBoost(tolerance=1e-6)` + - Initialize model without training. - You will need to call [`Train()`](#training) later to train the tree before calling [`Classify()`](#classification). --- - * `AdaBoost(data, labels, numClasses)` - * `AdaBoost(data, labels, numClasses, maxIterations, tolerance)` - - **Train model using default weak learner parameters.** - - If hyperparameters are not specified, default values are used. - - `labels` should be a vector of length `data.n_cols`, containing values from - `0` to `numClasses - 1` (inclusive). + * `ab = AdaBoost(data, labels, numClasses, maxIterations=100, tolerance=1e-6)` + - Train model using default weak learner hyperparameters. --- - * `AdaBoost(data, labels, numClasses, weakLearner)` - * `AdaBoost(data, labels, numClasses, weakLearner, maxIterations, tolerance)` - - **Train model with custom weak learner parameters.** + * `ab = AdaBoost(data, labels, numClasses, weakLearner, maxIterations=100, tolerance=1e-6)` + - Train model with custom weak learner parameters. - The given `weakLearner` does not need to be trained; any hyperparameter settings in `weakLearner` are used for training each AdaBoost weak learner (see the [simple examples](#simple-examples)). - - If hyperparameters are not specified, default values are used. - - `labels` should be a vector of length `data.n_cols`, containing values from - `0` to `numClasses - 1` (inclusive). --- - + + + * `ab = AdaBoost(data, labels, numClasses, maxIterations=100, tolerance=1e-6, [weak + learner hyperparameters...])` + - Train model with custom weak learner hyperparameters. + - Hyperparameters for the weak learner are any arguments to the weak + learner's `Train()` function that come after `numClasses` or `weights`. + - The only hyperparameter for the default weak learner (`Perceptron`) is + `maxIterations`. + - See [examples of this constructor in use](#simple-examples). --- @@ -102,8 +114,8 @@ below `tolerance`, training will terminate and no more weak learners will be added. | `1e-6` | As an alternative to passing hyperparameters, each hyperparameter can be set -with a standalone method. For an instance of `AdaBoost` named `ab`, the -following functions can be used before calling `Train()` to set hyperparameters: +with a standalone method. The following functions can be used before calling +`Train()` to set hyperparameters: @@ -125,39 +137,43 @@ named `ab`, the following functions for training are available: - * `ab.Train(data, labels, numClasses)` - * `ab.Train(data, labels, numClasses, maxIterations, tolerance)` - - **Train model using default weak learner parameters.** - - If hyperparameters are not specified here, and have not been otherwise set, - default values are used. - - `labels` should be a vector of length `data.n_cols`, containing values from - `0` to `numClasses - 1` (inclusive). + * `ab.Train(data, labels, numClasses, maxIterations=100, tolerance=1e-6)` + - Train model using default weak learner parameters. --- - * `ab.Train(data, labels, numClasses, weakLearner)` - * `ab.Train(data, labels, numClasses, weakLearner, maxIterations, tolerance)` - - **Train model with custom weak learner parameters.** + * `ab.Train(data, labels, numClasses, weakLearner, maxIterations=100, tolerance=1e-6)` + - Train model with custom weak learner parameters. - The given `weakLearner` does not need to be trained; any hyperparameter settings in `weakLearner` are used for training each AdaBoost weak learner (see the [simple examples](#simple-examples)). - - If hyperparameters for AdaBoost are not specified, and have not been - otherwise set, default values are used. - - `labels` should be a vector of length `data.n_cols`, containing values from - `0` to `numClasses - 1` (inclusive). --- + * `ab.Train(data, labels, numClasses, maxIterations=100, tolerance=1e-6, [weak learner hyperparameters...])` + - Train model with custom weak learner parameters. + - Hyperparameters for the weak learner are any arguments to the weak + learner's `Train()` function that come after `numClasses` or `weights`. + - The only hyperparameter for the default weak learner (`Perceptron`) is + `maxIterations`. + - See [examples of this form in use](#simple-examples). + --- Types of each argument are the same as in the table for constructors [above](#constructor-parameters). -***Note***: training is not incremental. A second call to `Train()` will -retrain the AdaBoost model from scratch. +***Notes***: + + * Training is not incremental. A second call to `Train()` will retrain the + AdaBoost model from scratch. + + * `Train()` returns a `double` indicating an upper bound on the training error + (specifically, the product of _Zt_ values, as described in the paper). ### Classification @@ -179,8 +195,6 @@ the [Classification Parameters](#classification-parameters) section below. - ***(Single-point)*** - Classify a single point and compute class probabilities. - The predicted class is stored in `prediction`. - - The class probabilities are stored in `probabilities_vec`, which is set to - length `numClasses`. - The probability of class `i` can be accessed with `probabilities_vec[i]`. --- @@ -188,8 +202,6 @@ the [Classification Parameters](#classification-parameters) section below. * `ab.Classify(data, predictions)` - ***(Multi-point)*** - Classify a set of points. - - The predicted class of each point is stored in `predictions`, which is set - to length `data.n_cols`. - The prediction for data point `i` can be accessed with `predictions[i]`. --- @@ -197,11 +209,7 @@ the [Classification Parameters](#classification-parameters) section below. * `ab.Classify(data, predictions, probabilities)` - ***(Multi-point)*** - Classify a set of points and compute class probabilities for each point. - - The predicted class of each point is stored in `predictions`, which is set - to length `data.n_cols`. - The prediction for data point `i` can be accessed with `predictions[i]`. - - The class probabilities for each point are stored in `probabilities`, which - is set to size `numClasses` by `data.n_cols`. - The probability of class `j` for data point `i` can be accessed with `probabilities(j, i)`. @@ -216,8 +224,8 @@ the [Classification Parameters](#classification-parameters) section below. | _single-point_ | `probabilities_vec` | [`arma::vec&`](../matrices.md) | `arma::vec&` to store class probabilities into. | |||| | _multi-point_ | `data` | [`arma::mat`](../matrices.md) | Set of [column-major](../matrices.md) points for classification. | -| _multi-point_ | `predictions` | [`arma::Row&`](../matrices.md) | Vector of `size_t`s to store class prediction into. | -| _multi-point_ | `probabilities` | [`arma::mat&`](../matrices.md) | Matrix to store class probabilities into (number of rows will be equal to number of classes). | +| _multi-point_ | `predictions` | [`arma::Row&`](../matrices.md) | Vector of `size_t`s to store class prediction into; will be set to length `data.n_cols`. | +| _multi-point_ | `probabilities` | [`arma::mat&`](../matrices.md) | Matrix to store class probabilities into (number of rows will be equal to number of classes; number of columns will be equal to `data.n_cols`). | ### Other Functionality @@ -242,28 +250,8 @@ is fully documented. ### Simple Examples -Train an AdaBoost model on random data and predict labels on a random test set. - -```c++ -// 1000 random points in 10 dimensions. -arma::mat dataset(10, 1000, arma::fill::randu); -// Random labels for each point, totaling 5 classes. -arma::Row labels = - arma::randi>(1000, arma::distr_param(0, 4)); - -// Train in the constructor. -AdaBoost<> ab(dataset, labels, 5); - -// Create test data (500 points). -arma::mat testDataset(10, 500, arma::fill::randu); -arma::Row predictions; -ab.Classify(testDataset, predictions); -// Now `predictions` holds predictions for the test dataset. - -// Print some information about the test predictions. -std::cout << arma::accu(predictions == 3) << " test points classified as class " - << "3." << std::endl; -``` +See also the [simple usage example](#simple-usage-example) for a trivial usage +of the `AdaBoost` class. --- From 1f050b35b3ff8eb5160bc0dd95cb8e34af9a058d Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Mon, 13 Nov 2023 16:12:20 -0500 Subject: [PATCH 16/47] Allow templatized weights for DecisionTree and subclasses. --- src/mlpack/methods/decision_tree/decision_tree.hpp | 8 ++++---- src/mlpack/methods/decision_tree/decision_tree_impl.hpp | 8 ++++---- src/mlpack/methods/decision_tree/information_gain.hpp | 4 ++-- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/src/mlpack/methods/decision_tree/decision_tree.hpp b/src/mlpack/methods/decision_tree/decision_tree.hpp index e7bb0031d1..af169822b5 100644 --- a/src/mlpack/methods/decision_tree/decision_tree.hpp +++ b/src/mlpack/methods/decision_tree/decision_tree.hpp @@ -547,14 +547,14 @@ class DecisionTree : * @param maximumDepth Maximum depth for the tree. * @return The final entropy of decision tree. */ - template + template double Train(MatType& data, const size_t begin, const size_t count, const data::DatasetInfo& datasetInfo, arma::Row& labels, const size_t numClasses, - arma::rowvec& weights, + WeightsType& weights, const size_t minimumLeafSize, const double minimumGainSplit, const size_t maximumDepth, @@ -576,13 +576,13 @@ class DecisionTree : * @param maximumDepth Maximum depth for the tree. * @return The final entropy of decision tree. */ - template + template double Train(MatType& data, const size_t begin, const size_t count, arma::Row& labels, const size_t numClasses, - arma::rowvec& weights, + WeightsType& weights, const size_t minimumLeafSize, const double minimumGainSplit, const size_t maximumDepth, diff --git a/src/mlpack/methods/decision_tree/decision_tree_impl.hpp b/src/mlpack/methods/decision_tree/decision_tree_impl.hpp index 7e1d1362d5..a439f1aa1d 100644 --- a/src/mlpack/methods/decision_tree/decision_tree_impl.hpp +++ b/src/mlpack/methods/decision_tree/decision_tree_impl.hpp @@ -614,7 +614,7 @@ template class CategoricalSplitType, typename DimensionSelectionType, bool NoRecursion> -template +template double DecisionTree& labels, const size_t numClasses, - arma::rowvec& weights, + WeightsType& weights, const size_t minimumLeafSize, const double minimumGainSplit, const size_t maximumDepth, @@ -798,7 +798,7 @@ template class CategoricalSplitType, typename DimensionSelectionType, bool NoRecursion> -template +template double DecisionTree& labels, const size_t numClasses, - arma::rowvec& weights, + WeightsType& weights, const size_t minimumLeafSize, const double minimumGainSplit, const size_t maximumDepth, diff --git a/src/mlpack/methods/decision_tree/information_gain.hpp b/src/mlpack/methods/decision_tree/information_gain.hpp index e07ea7eca2..7cf0f1158e 100644 --- a/src/mlpack/methods/decision_tree/information_gain.hpp +++ b/src/mlpack/methods/decision_tree/information_gain.hpp @@ -55,10 +55,10 @@ class InformationGain * @param numClasses Number of classes in the dataset. * @param weights Weights associated with labels. */ - template + template static double Evaluate(const arma::Row& labels, const size_t numClasses, - const arma::Row& weights) + const WeightsType& weights) { // Edge case: if there are no elements, the gain is zero. if (labels.n_elem == 0) From 1bfb3c9d7715bd28e4dbe11f21e68d3207c45c53 Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Mon, 13 Nov 2023 16:15:52 -0500 Subject: [PATCH 17/47] Allow perceptrons to take arbitrary weight types. --- .../simple_weight_update.hpp | 2 +- src/mlpack/methods/perceptron/perceptron.hpp | 16 +++++++---- .../methods/perceptron/perceptron_impl.hpp | 28 +++++++++++++------ 3 files changed, 31 insertions(+), 15 deletions(-) diff --git a/src/mlpack/methods/perceptron/learning_policies/simple_weight_update.hpp b/src/mlpack/methods/perceptron/learning_policies/simple_weight_update.hpp index 2bffb3946c..435b3625a3 100644 --- a/src/mlpack/methods/perceptron/learning_policies/simple_weight_update.hpp +++ b/src/mlpack/methods/perceptron/learning_policies/simple_weight_update.hpp @@ -51,7 +51,7 @@ class SimpleWeightUpdate arma::Col& biases, const size_t incorrectClass, const size_t correctClass, - const double instanceWeight = 1.0) + const eT instanceWeight = 1.0) { weights.col(incorrectClass) -= instanceWeight * trainingPoint; biases(incorrectClass) -= instanceWeight; diff --git a/src/mlpack/methods/perceptron/perceptron.hpp b/src/mlpack/methods/perceptron/perceptron.hpp index 6268146b70..1703b6e4d7 100644 --- a/src/mlpack/methods/perceptron/perceptron.hpp +++ b/src/mlpack/methods/perceptron/perceptron.hpp @@ -86,11 +86,14 @@ class Perceptron * @param maxIterations Maximum number of iterations for the perceptron * learning algorithm. */ + template Perceptron(const MatType& data, const arma::Row& labels, const size_t numClasses, - const arma::rowvec& instanceWeights, - const size_t maxIterations = 1000); + const WeightsType& instanceWeights, + const size_t maxIterations = 1000, + const typename std::enable_if< + arma::is_arma_type::value>::type* = 0); /** * Alternate constructor which copies parameters from an already initiated @@ -104,12 +107,15 @@ class Perceptron * @param instanceWeights Weight vector to use while training. For boosting * purposes. */ + template mlpack_deprecated /* was previously only used by AdaBoost */ Perceptron(const Perceptron& other, const MatType& data, const arma::Row& labels, const size_t numClasses, - const arma::rowvec& instanceWeights); + const WeightsType& instanceWeights, + const typename std::enable_if< + arma::is_arma_type::value>::type* = 0); /** * Train the perceptron on the given data for up to the maximum number of @@ -251,11 +257,11 @@ class Perceptron * If `HasWeights` is `false`, then `instanceWeights` is ignored (and may be * left empty). */ - template + template void TrainInternal(const MatType& data, const arma::Row& labels, const size_t numClasses, - const arma::rowvec& instanceWeights = arma::rowvec()); + const WeightsType& instanceWeights = WeightsType()); //! The maximum number of iterations during training. size_t maxIterations; diff --git a/src/mlpack/methods/perceptron/perceptron_impl.hpp b/src/mlpack/methods/perceptron/perceptron_impl.hpp index 0bc081d619..8e2e9c0b49 100644 --- a/src/mlpack/methods/perceptron/perceptron_impl.hpp +++ b/src/mlpack/methods/perceptron/perceptron_impl.hpp @@ -58,7 +58,8 @@ Perceptron::Perceptron( maxIterations(maxIterations) { // Start training. - TrainInternal(data, labels, numClasses); + TrainInternal>(data, labels, + numClasses); } /** @@ -75,12 +76,15 @@ template< typename WeightInitializationPolicy, typename MatType > +template Perceptron::Perceptron( const MatType& data, const arma::Row& labels, const size_t numClasses, - const arma::rowvec& instanceWeights, - const size_t maxIterations) : + const WeightsType& instanceWeights, + const size_t maxIterations, + const typename std::enable_if< + arma::is_arma_type::value>::type*) : maxIterations(maxIterations) { // Start training. @@ -103,12 +107,16 @@ template< typename WeightInitializationPolicy, typename MatType > +template +mlpack_deprecated Perceptron::Perceptron( const Perceptron& other, const MatType& data, const arma::Row& labels, const size_t numClasses, - const arma::rowvec& instanceWeights) : + const WeightsType& instanceWeights, + const typename std::enable_if< + arma::is_arma_type::value>::type*) : maxIterations(other.maxIterations) { TrainInternal(data, labels, numClasses, instanceWeights); @@ -137,7 +145,8 @@ void Perceptron::Train( const arma::Row& labels, const size_t numClasses) { - TrainInternal(data, labels, numClasses); + TrainInternal>(data, labels, + numClasses); } /** @@ -170,7 +179,8 @@ void Perceptron::Train( { // Set the maximum number of iterations and call unweighted Train(). this->maxIterations = maxIterations; - TrainInternal(data, labels, numClasses); + TrainInternal>(data, labels, + numClasses); } /** @@ -246,13 +256,13 @@ template< typename WeightInitializationPolicy, typename MatType > -template +template void Perceptron< LearnPolicy, WeightInitializationPolicy, MatType >::TrainInternal(const MatType& data, const arma::Row& labels, const size_t numClasses, - const arma::rowvec& instanceWeights) + const WeightsType& instanceWeights) { // Do we need to resize the weights? if (weights.n_cols != numClasses || weights.n_rows != data.n_rows) @@ -297,7 +307,7 @@ void Perceptron< // the correct class. if (HasWeights) LP.UpdateWeights(data.col(j), weights, biases, maxIndexRow, tempLabel, - instanceWeights(j)); + (typename MatType::elem_type) instanceWeights(j)); else LP.UpdateWeights(data.col(j), weights, biases, maxIndexRow, tempLabel); From 4b81e0b620d9870cbcf21b47e3710e56ee1558a7 Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Mon, 13 Nov 2023 16:17:21 -0500 Subject: [PATCH 18/47] Include workaround for allowing templated versions for serialization. --- .../core/cereal/template_class_version.hpp | 78 +++++++++++++++++++ src/mlpack/prereqs.hpp | 1 + 2 files changed, 79 insertions(+) create mode 100644 src/mlpack/core/cereal/template_class_version.hpp diff --git a/src/mlpack/core/cereal/template_class_version.hpp b/src/mlpack/core/cereal/template_class_version.hpp new file mode 100644 index 0000000000..6789de5e39 --- /dev/null +++ b/src/mlpack/core/cereal/template_class_version.hpp @@ -0,0 +1,78 @@ +/** + * @file core/cereal/template_class_version.hpp + * @author Ryan Curtin + * + * Implementation of CEREAL_TEMPLATE_CLASS_VERSION() macro, useful for + * templatized types where CEREAL_CLASS_VERSION() will not work. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_CORE_CEREAL_TEMPLATE_CLASS_VERSION_HPP +#define MLPACK_CORE_CEREAL_TEMPLATE_CLASS_VERSION_HPP + +#include + +// This useful implementation is adapted from @lubensky on Github: +// https://github.com/uscilab/cereal/issues/319#issuecomment-1512927210 + +#define CEREAL_UNPACK(...) __VA_ARGS__ + +#ifdef MLPACK_HAVE_CXX17 + +// The C++17 version sets `version` as `inline`. +#define CEREAL_TEMPLATE_CLASS_VERSION(ARGS, TYPE, VERSION_NUMBER) \ +namespace cereal { \ +namespace detail { \ +template \ +struct Version \ +{ \ + static std::uint32_t registerVersion() \ + { \ + ::cereal::detail::StaticObject::getInstance().mapping.emplace( \ + std::type_index(typeid(CEREAL_UNPACK TYPE)).hash_code(), \ + CEREAL_UNPACK VERSION_NUMBER); \ + return CEREAL_UNPACK VERSION_NUMBER; \ + } \ + \ + static inline const std::uint32_t version = registerVersion(); \ + \ + CEREAL_UNUSED_FUNCTION \ +}; /* end Version */ \ + \ +} \ +} + +#else + +// Here we cannot use inline variables. +#define CEREAL_TEMPLATE_CLASS_VERSION(ARGS, TYPE, VERSION_NUMBER) \ +namespace cereal { \ +namespace detail { \ +template \ +struct Version \ +{ \ + static const std::uint32_t version; \ + static std::uint32_t registerVersion() \ + { \ + ::cereal::detail::StaticObject::getInstance().mapping.emplace( \ + std::type_index(typeid(CEREAL_UNPACK TYPE)).hash_code(), \ + CEREAL_UNPACK VERSION_NUMBER); \ + return CEREAL_UNPACK VERSION_NUMBER; \ + } \ + \ + CEREAL_UNUSED_FUNCTION \ +}; /* end Version */ \ + \ +template \ +const std::uint32_t Version::version = \ + Version::registerVersion(); \ + \ +} \ +} + +#endif // MLPACK_HAVE_CXX17 + +#endif // TEMPLATE_CLASS_VERSION_HPP diff --git a/src/mlpack/prereqs.hpp b/src/mlpack/prereqs.hpp index 202788013e..990373b48c 100644 --- a/src/mlpack/prereqs.hpp +++ b/src/mlpack/prereqs.hpp @@ -31,6 +31,7 @@ #include #include #include +#include #include // All code should have access to logging. From 0f27f3704601226bce447fcf8b6baad89fef76d9 Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Mon, 13 Nov 2023 16:18:36 -0500 Subject: [PATCH 19/47] Implement CheckMatrices() for floats. --- src/mlpack/tests/serialization.cpp | 51 ++++++++++++++++++++++++++---- src/mlpack/tests/serialization.hpp | 5 +++ 2 files changed, 49 insertions(+), 7 deletions(-) diff --git a/src/mlpack/tests/serialization.cpp b/src/mlpack/tests/serialization.cpp index 0631be7b73..8fca67354d 100644 --- a/src/mlpack/tests/serialization.cpp +++ b/src/mlpack/tests/serialization.cpp @@ -14,7 +14,7 @@ namespace mlpack { -// Utility function to check the equality of two Armadillo matrices. +// Utility function to check the equality of four Armadillo matrices. void CheckMatrices(const arma::mat& x, const arma::mat& xmlX, const arma::mat& jsonX, @@ -39,15 +39,52 @@ void CheckMatrices(const arma::mat& x, const double val = x[i]; if (val == 0.0) { - REQUIRE(xmlX[i] == Approx(0.0).margin(1e-6 / 100)); - REQUIRE(jsonX[i] == Approx(0.0).margin(1e-6 / 100)); - REQUIRE(binaryX[i] == Approx(0.0).margin(1e-6 / 100)); + REQUIRE(xmlX[i] == Approx(0.0).margin(1e-8)); + REQUIRE(jsonX[i] == Approx(0.0).margin(1e-8)); + REQUIRE(binaryX[i] == Approx(0.0).margin(1e-8)); } else { - REQUIRE(val == Approx(xmlX[i]).epsilon(1e-6 / 100)); - REQUIRE(val == Approx(jsonX[i]).epsilon(1e-6 / 100)); - REQUIRE(val == Approx(binaryX[i]).epsilon(1e-6 / 100)); + REQUIRE(val == Approx(xmlX[i]).epsilon(1e-8)); + REQUIRE(val == Approx(jsonX[i]).epsilon(1e-8)); + REQUIRE(val == Approx(binaryX[i]).epsilon(1e-8)); + } + } +} + +void CheckMatrices(const arma::fmat& x, + const arma::fmat& xmlX, + const arma::fmat& jsonX, + const arma::fmat& binaryX) +{ + // First check dimensions. + REQUIRE(x.n_rows == xmlX.n_rows); + REQUIRE(x.n_rows == jsonX.n_rows); + REQUIRE(x.n_rows == binaryX.n_rows); + + REQUIRE(x.n_cols == xmlX.n_cols); + REQUIRE(x.n_cols == jsonX.n_cols); + REQUIRE(x.n_cols == binaryX.n_cols); + + REQUIRE(x.n_elem == xmlX.n_elem); + REQUIRE(x.n_elem == jsonX.n_elem); + REQUIRE(x.n_elem == binaryX.n_elem); + + // Now check elements. + for (size_t i = 0; i < x.n_elem; ++i) + { + const float val = x[i]; + if (val == 0.0) + { + REQUIRE(xmlX[i] == Approx(0.0).margin(1e-6)); + REQUIRE(jsonX[i] == Approx(0.0).margin(1e-6)); + REQUIRE(binaryX[i] == Approx(0.0).margin(1e-6)); + } + else + { + REQUIRE(val == Approx(xmlX[i]).epsilon(1e-6)); + REQUIRE(val == Approx(jsonX[i]).epsilon(1e-6)); + REQUIRE(val == Approx(binaryX[i]).epsilon(1e-6)); } } } diff --git a/src/mlpack/tests/serialization.hpp b/src/mlpack/tests/serialization.hpp index 361523f701..38cb6de9dd 100644 --- a/src/mlpack/tests/serialization.hpp +++ b/src/mlpack/tests/serialization.hpp @@ -219,6 +219,11 @@ void CheckMatrices(const arma::mat& x, const arma::mat& jsonX, const arma::mat& binaryX); +void CheckMatrices(const arma::fmat& x, + const arma::fmat& xmlX, + const arma::fmat& jsonX, + const arma::fmat& binaryX); + void CheckMatrices(const arma::Mat& x, const arma::Mat& xmlX, const arma::Mat& jsonX, From 228b237c0dcdc6ca2ddb73f9fc24e23644ad9255 Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Mon, 13 Nov 2023 16:18:49 -0500 Subject: [PATCH 20/47] Remove debugging output. --- src/mlpack/tests/ann/layer/parametric_relu.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/mlpack/tests/ann/layer/parametric_relu.cpp b/src/mlpack/tests/ann/layer/parametric_relu.cpp index 43904d304d..8c3ab562a1 100644 --- a/src/mlpack/tests/ann/layer/parametric_relu.cpp +++ b/src/mlpack/tests/ann/layer/parametric_relu.cpp @@ -130,8 +130,6 @@ TEST_CASE("PReLUIntegrationTest", "[ANNLayerTest]") double msreTrain = ComputeMSRE(predictions, trainLabels); model.Predict(testData, predictions); double msreTest = ComputeMSRE(predictions, testLabels); - std::cout << "train: " << msreTrain << "\n"; - std::cout << "test: " << msreTest << "\n"; double relativeMSRE = std::abs((msreTest - msreTrain) / msreTrain); if (relativeMSRE <= 0.35) From 1b010ceb79ff0d787544bea6312dd6c513191329 Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Mon, 13 Nov 2023 16:19:40 -0500 Subject: [PATCH 21/47] Update AdaBoost implementation: add Train() overloads, add Classify() overloads, add constructors, fix serialization in reverse-compatible way. --- src/mlpack/methods/adaboost/adaboost.hpp | 190 ++++++-- src/mlpack/methods/adaboost/adaboost_impl.hpp | 443 +++++++++++++----- 2 files changed, 494 insertions(+), 139 deletions(-) diff --git a/src/mlpack/methods/adaboost/adaboost.hpp b/src/mlpack/methods/adaboost/adaboost.hpp index 49a7729455..54ec99fb77 100644 --- a/src/mlpack/methods/adaboost/adaboost.hpp +++ b/src/mlpack/methods/adaboost/adaboost.hpp @@ -80,6 +80,35 @@ template, class AdaBoost { public: + typedef typename MatType::elem_type ElemType; + + /** + * Create the AdaBoost object without training. Be sure to call Train() + * before calling Classify()! + */ + AdaBoost(const ElemType tolerance = 1e-6); + + /** + * Construct an AdaBoost model. Any extra parameters are used as + * hyperparameters for the weak learner. These should be the last arguments + * to the weak learner's constructor or `Train()` function (i.e. anything + * after `numClasses` or `weights`). + * + * @param data Input data. + * @param labels Corresponding labels. + * @param numClasses The number of classes. + * @param maxIterations Number of boosting rounds. + * @param tolerance The tolerance for change in values of rt. + * @param weakLearnerParams... Any hyperparameters for the weak learner. + */ + template + AdaBoost(const MatType& data, + const arma::Row& labels, + const size_t numClasses, + const size_t maxIterations = 100, + const ElemType tolerance = 1e-6, + WeakLearnerArgs&... weakLearnerArgs); + /** * Constructor. This runs the AdaBoost.MH algorithm to provide a trained * boosting model. This constructor takes an already-initialized weak @@ -98,18 +127,17 @@ class AdaBoost const size_t numClasses, const WeakLearnerType& other, const size_t maxIterations = 100, - const double tolerance = 1e-6); + const ElemType tolerance = 1e-6); - /** - * Create the AdaBoost object without training. Be sure to call Train() - * before calling Classify()! - */ - AdaBoost(const double tolerance = 1e-6); + //! Get the maximum number of weak learners allowed in the model. + size_t MaxIterations() const { return maxIterations; } + //! Modify the maximum number of weak learners allowed in the model. + size_t& MaxIterations() { return maxIterations; } //! Get the tolerance for stopping the optimization during training. - double Tolerance() const { return tolerance; } + ElemType Tolerance() const { return tolerance; } //! Modify the tolerance for stopping the optimization during training. - double& Tolerance() { return tolerance; } + ElemType& Tolerance() { return tolerance; } //! Get the number of classes this model is trained on. size_t NumClasses() const { return numClasses; } @@ -118,9 +146,9 @@ class AdaBoost size_t WeakLearners() const { return alpha.size(); } //! Get the weights for the given weak learner. - double Alpha(const size_t i) const { return alpha[i]; } + ElemType Alpha(const size_t i) const { return alpha[i]; } //! Modify the weight for the given weak learner (be careful!). - double& Alpha(const size_t i) { return alpha[i]; } + ElemType& Alpha(const size_t i) { return alpha[i]; } //! Get the given weak learner. const WeakLearnerType& WeakLearner(const size_t i) const { return wl[i]; } @@ -134,6 +162,10 @@ class AdaBoost * completely overwrite any model that has already been trained with this * object. * + * Default values are not used for `maxIterations` and `tolerance`; instead, + * multiple overloads are allowed; this is because we want to use the existing + * setting internal to the class, if one is not specified. + * * @param data Dataset to train on. * @param labels Labels for each point in the dataset. * @param numClasses The number of classes. @@ -142,12 +174,103 @@ class AdaBoost * @param tolerance The tolerance for change in values of rt. * @return The upper bound for training error. */ - double Train(const MatType& data, - const arma::Row& labels, - const size_t numClasses, - const WeakLearnerType& learner, - const size_t maxIterations = 100, - const double tolerance = 1e-6); + template + ElemType Train( + const MatType& data, + const arma::Row& labels, + const size_t numClasses, + const WeakLearnerInType& learner, + // Necessary to distinguish from other overloads. + const typename std::enable_if< + std::is_same::value>::type* = 0); + + template + ElemType Train( + const MatType& data, + const arma::Row& labels, + const size_t numClasses, + const WeakLearnerInType& learner, + const size_t maxIterations, + // Necessary to distinguish from other overloads. + const typename std::enable_if< + std::is_same::value>::type* = 0); + + template + ElemType Train( + const MatType& data, + const arma::Row& labels, + const size_t numClasses, + const WeakLearnerInType& learner, + const size_t maxIterations, + const double tolerance, + // Necessary to distinguish from other overloads. + const typename std::enable_if< + std::is_same::value>::type* = 0); + + /** + * Train AdaBoost on the given dataset, using the given parameters. The last + * parameters are the hyperparameters to use for the weak learners; these are + * all the arguments to `WeakLearnerType::Train()` after `numClasses` and + * `weights`. + * + * Default values are not used for `maxIterations` and `tolerance`; instead, + * multiple overloads are allowed; this is because we want to use the existing + * setting internal to the class, if one is not specified. + * + * @param data Dataset to train on. + * @param labels Labels for each point in the dataset. + * @param numClasses The number of classes in the dataset. + * @param maxIterations Number of boosting rounds. + * @param tolerance The tolerance for change in values of rt. + * @param weakLearnerArgs Hyperparameters to use for each weak learner. + * @return The upper bound for training error. + */ + ElemType Train(const MatType& data, + const arma::Row& labels, + const size_t numClasses); + + ElemType Train(const MatType& data, + const arma::Row& labels, + const size_t numClasses, + const size_t maxIterations); + + template + ElemType Train(const MatType& data, + const arma::Row& labels, + const size_t numClasses, + const size_t maxIterations, + const double tolerance, + WeakLearnerArgs&&... weakLearnerArgs); + + /** + * Classify the given test point. + * + * @param point Test point. + */ + template + size_t Classify(const VecType& point) const; + + /** + * Classify the given test point and compute class probabilities. + * + * @param point Test point. + * @param prediction Will be filled with the predicted class of `point`. + * @param probabilities Will be filled with the class probabilities. + */ + template + void Classify(const VecType& point, + size_t& prediction, + arma::Row& probabilities) const; + + /** + * Classify the given test points. + * + * @param test Testing data. + * @param predictedLabels Vector in which the predicted labels of the test + * set will be stored. + */ + void Classify(const MatType& test, + arma::Row& predictedLabels) const; /** * Classify the given test points. @@ -160,17 +283,7 @@ class AdaBoost */ void Classify(const MatType& test, arma::Row& predictedLabels, - arma::mat& probabilities); - - /** - * Classify the given test points. - * - * @param test Testing data. - * @param predictedLabels Vector in which the predicted labels of the test - * set will be stored. - */ - void Classify(const MatType& test, - arma::Row& predictedLabels); + arma::Mat& probabilities) const; /** * Serialize the AdaBoost model. @@ -179,19 +292,36 @@ class AdaBoost void serialize(Archive& ar, const uint32_t /* version */); private: + /** + * Internal utility training function. `wl` is not used if + * `UseExistingWeakLearner` is false. `weakLearnerArgs` are not used if + * `UseExistingWeakLearner` is true. + */ + template + ElemType TrainInternal(const MatType& data, + const arma::Row& labels, + const size_t numClasses, + const WeakLearnerType& wl, + WeakLearnerArgs&&... weakLearnerArgs); + //! The number of classes in the model. size_t numClasses; - // The tolerance for change in rt and when to stop. - double tolerance; + //! The maximum number of weak learners allowed in the model. + size_t maxIterations; + //! The tolerance for change in rt and when to stop. + ElemType tolerance; //! The vector of weak learners. std::vector wl; //! The weights corresponding to each weak learner. - std::vector alpha; + std::vector alpha; }; // class AdaBoost } // namespace mlpack +CEREAL_TEMPLATE_CLASS_VERSION((typename WeakLearnerType, typename MatType), + (mlpack::AdaBoost), (1)); + // Include implementation. #include "adaboost_impl.hpp" diff --git a/src/mlpack/methods/adaboost/adaboost_impl.hpp b/src/mlpack/methods/adaboost/adaboost_impl.hpp index b07f33bf13..db97d97230 100644 --- a/src/mlpack/methods/adaboost/adaboost_impl.hpp +++ b/src/mlpack/methods/adaboost/adaboost_impl.hpp @@ -30,8 +30,17 @@ namespace mlpack { +// Empty constructor. +template +AdaBoost::AdaBoost(const ElemType tolerance) : + numClasses(0), + tolerance(tolerance) +{ + // Nothing to do. +} + /** - * Constructor. Currently runs the AdaBoost.MH algorithm. + * Constructor. Runs the AdaBoost.MH algorithm. * * @param data Input data * @param labels Corresponding labels @@ -46,29 +55,325 @@ AdaBoost::AdaBoost( const size_t numClasses, const WeakLearnerType& other, const size_t maxIterations, - const double tol) + const typename MatType::elem_type tol) : + maxIterations(maxIterations), + tolerance(tol) { - Train(data, labels, numClasses, other, maxIterations, tol); + (void) TrainInternal(data, labels, numClasses, other); } -// Empty constructor. +/** + * Constructor. Runs the AdaBoost.MH algorithm. + * + * @param data Input data + * @param labels Corresponding labels + * @param maxIterations Number of boosting rounds + * @param tol Tolerance for termination of Adaboost.MH. + * @param other Weak Learner, which has been initialized already. + */ template -AdaBoost::AdaBoost(const double tolerance) : - numClasses(0), - tolerance(tolerance) +template +AdaBoost::AdaBoost( + const MatType& data, + const arma::Row& labels, + const size_t numClasses, + const size_t maxIterations, + const typename MatType::elem_type tol, + WeakLearnerArgs&... weakLearnerArgs) : + maxIterations(maxIterations), + tolerance(tol) { - // Nothing to do. + WeakLearnerType other; // Will not be used. + (void) TrainInternal(data, labels, numClasses, other, + weakLearnerArgs...); +} + +// Train AdaBoost with a given weak learner. +template +template +typename MatType::elem_type AdaBoost::Train( + const MatType& data, + const arma::Row& labels, + const size_t numClasses, + const WeakLearnerInType& other, + const typename std::enable_if< + std::is_same::value>::type*) +{ + return TrainInternal(data, labels, numClasses, other); +} + +// Train AdaBoost with a given weak learner, and set the maximum number of +// iterations. +template +template +typename MatType::elem_type AdaBoost::Train( + const MatType& data, + const arma::Row& labels, + const size_t numClasses, + const WeakLearnerInType& other, + const size_t maxIterations, + const typename std::enable_if< + std::is_same::value>::type*) +{ + this->maxIterations = maxIterations; + return TrainInternal(data, labels, numClasses, other); +} + +// Train AdaBoost with a given weak learner, and set the maximum number of +// iterations and tolerance. +template +template +typename MatType::elem_type AdaBoost::Train( + const MatType& data, + const arma::Row& labels, + const size_t numClasses, + const WeakLearnerInType& other, + const size_t maxIterations, + const double tolerance, + const typename std::enable_if< + std::is_same::value>::type*) +{ + this->maxIterations = maxIterations; + this->tolerance = tolerance; + return TrainInternal(data, labels, numClasses, other); } // Train AdaBoost. template -double AdaBoost::Train( +typename MatType::elem_type AdaBoost::Train( + const MatType& data, + const arma::Row& labels, + const size_t numClasses) +{ + WeakLearnerType other; // Will not be used. + return TrainInternal(data, labels, numClasses, other); +} + +// Train AdaBoost, and set the maximum number of iterations. +template +typename MatType::elem_type AdaBoost::Train( + const MatType& data, + const arma::Row& labels, + const size_t numClasses, + const size_t maxIterations) +{ + this->maxIterations = maxIterations; + + WeakLearnerType other; // Will not be used. + return TrainInternal(data, labels, numClasses, other); +} +// Train AdaBoost. +template +template +typename MatType::elem_type AdaBoost::Train( + const MatType& data, + const arma::Row& labels, + const size_t numClasses, + const size_t maxIterations, + const double tolerance, + WeakLearnerArgs&&... weakLearnerArgs) +{ + this->maxIterations = maxIterations; + this->tolerance = tolerance; + WeakLearnerType other; // Will not be used. + return TrainInternal(data, labels, numClasses, other, + weakLearnerArgs...); +} + +// Classify the given test point. +template +template +size_t AdaBoost::Classify(const VecType& point) const +{ + arma::Row probabilities; + size_t prediction; + Classify(point, prediction, probabilities); + + return prediction; +} + +// Classify the given test point and return class probabilities. +template +template +void AdaBoost::Classify( + const VecType& point, + size_t& prediction, + arma::Row& probabilities) const +{ + probabilities.zeros(numClasses); + for (size_t i = 0; i < wl.size(); ++i) + { + prediction = wl[i].Classify(point); + probabilities(prediction) += alpha[i]; + } + + arma::uword maxIndex = 0; + probabilities /= arma::accu(probabilities); + probabilities.max(maxIndex); + prediction = (size_t) maxIndex; +} + +// Classify the given test points. +template +void AdaBoost::Classify( + const MatType& test, + arma::Row& predictedLabels) const +{ + arma::Row tempPredictedLabels(test.n_cols); + arma::Mat probabilities; + + Classify(test, predictedLabels, probabilities); +} + +// Classify the given test points. +template +void AdaBoost::Classify( + const MatType& test, + arma::Row& predictedLabels, + arma::Mat& probabilities) const +{ + probabilities.zeros(numClasses, test.n_cols); + predictedLabels.set_size(test.n_cols); + + for (size_t i = 0; i < wl.size(); ++i) + { + wl[i].Classify(test, predictedLabels); + + for (size_t j = 0; j < predictedLabels.n_cols; ++j) + probabilities(predictedLabels(j), j) += alpha[i]; + } + + arma::uword maxIndex = 0; + + for (size_t i = 0; i < predictedLabels.n_cols; ++i) + { + probabilities.col(i) /= arma::accu(probabilities.col(i)); + probabilities.col(i).max(maxIndex); + predictedLabels(i) = maxIndex; + } +} + +/** + * Serialize the AdaBoost model. + */ +template +template +void AdaBoost::serialize(Archive& ar, + const uint32_t version) +{ + // Between version 0 and 1, the maxIterations member was added, and `alpha` + // was switched to type arma::Row instead of arma::rowvec. These + // require a little bit of special handling when loading older versions. + if (cereal::is_loading() && version == 0) + { + // This is the legacy version. + ar(CEREAL_NVP(numClasses)); + ar(CEREAL_NVP(tolerance)); + ar(CEREAL_NVP(alpha)); + + // In earlier versions, `alpha` was a vector of doubles---but it might not + // be now. + if (std::is_same::value) + { + ar(CEREAL_NVP(alpha)); // The easy case. + } + else + { + arma::rowvec alphaTmp; + // Avoid CEREAL_NVP so we can specify a custom name. + ar(cereal::make_nvp("alpha", alphaTmp)); + alpha.clear(); + alpha.resize(alphaTmp.size()); + for (size_t i = 0; i < alphaTmp.size(); ++i) + alpha[i] = (ElemType) alphaTmp[i]; + } + + // Now serialize each weak learner. + ar(CEREAL_NVP(wl)); + + // Attempt to set maxIterations to something reasonable. + maxIterations = std::max((size_t) 100, alpha.size()); + } + else + { + // This is the current version. + // (Once there is a major version bump, we should make this version 0.) + ar(CEREAL_NVP(numClasses)); + ar(CEREAL_NVP(tolerance)); + ar(CEREAL_NVP(maxIterations)); + ar(CEREAL_NVP(alpha)); + + // Now serialize each weak learner. + ar(CEREAL_NVP(wl)); + } +} + +template< + bool UseExistingWeakLearner, + typename MatType, + typename WeightsType, + typename WeakLearnerType, + typename... WeakLearnerArgs +> +struct WeakLearnerTrainer +{ + static WeakLearnerType Train( + const MatType& data, + const arma::Row& labels, + const size_t numClasses, + const WeightsType& weights, + const WeakLearnerType& wl, + WeakLearnerArgs&&... /* weakLearnerArgs */) + { + // Use the existing weak learner to train a new one with new weights. + // API requirement: there is a constructor with this signature: + // + // WeakLearnerType(const WeakLearnerType&, + // MatType& data, + // LabelsType& labels, + // const size_t numClasses, + // WeightsType& weights) + // + // This trains the new WeakLearnerType using the hyperparameters from the + // given WeakLearnerType. + return WeakLearnerType(wl, data, labels, numClasses, weights); + } +}; + +template< + typename MatType, + typename WeightsType, + typename WeakLearnerType, + typename... WeakLearnerArgs +> +struct WeakLearnerTrainer< + false, MatType, WeightsType, WeakLearnerType, WeakLearnerArgs... +> +{ + static WeakLearnerType Train( + const MatType& data, + const arma::Row& labels, + const size_t numClasses, + const WeightsType& weights, + const WeakLearnerType& /* wl */, + WeakLearnerArgs&&... weakLearnerArgs) + { + // When UseExistingWeakLearner is false, we use the given hyperparameters. + // (This is the preferred approach that supports more types of weak + // learners.) + return WeakLearnerType(data, labels, numClasses, weights, + weakLearnerArgs...); + } +}; + +template +template +typename MatType::elem_type AdaBoost::TrainInternal( const MatType& data, const arma::Row& labels, const size_t numClasses, const WeakLearnerType& other, - const size_t maxIterations, - const double tolerance) + WeakLearnerArgs&&... weakLearnerArgs) { // Clear information from previous runs. wl.clear(); @@ -79,9 +384,9 @@ double AdaBoost::Train( // crt is the cumulative rt value for terminating the optimization when rt is // changing by less than the tolerance. - double rt, crt = 0.0, alphat = 0.0, zt; + ElemType rt, crt = 0.0, alphat = 0.0, zt; - double ztProduct = 1.0; + ElemType ztProduct = 1.0; // To be used for prediction by the weak learner. arma::Row predictedLabels(labels.n_cols); @@ -90,16 +395,16 @@ double AdaBoost::Train( MatType tempData(data); // This matrix is a helper matrix used to calculate the final hypothesis. - arma::mat sumFinalH = arma::zeros(numClasses, - predictedLabels.n_cols); + MatType sumFinalH(numClasses, predictedLabels.n_cols); + sumFinalH.zeros(); // Load the initial weights into a 2-D matrix. - const double initWeight = 1.0 / double(data.n_cols * numClasses); - arma::mat D(numClasses, data.n_cols); + const ElemType initWeight = 1.0 / ElemType(data.n_cols * numClasses); + MatType D(numClasses, data.n_cols); D.fill(initWeight); // Weights are stored in this row vector. - arma::rowvec weights(predictedLabels.n_cols); + arma::Row weights(predictedLabels.n_cols); // This is the final hypothesis. arma::Row finalH(predictedLabels.n_cols); @@ -118,31 +423,17 @@ double AdaBoost::Train( // Build the weight vectors. weights = arma::sum(D); - // Use the existing weak learner to train a new one with new weights. - // API requirement: there is a constructor with this signature: - // - // WeakLearnerType(const WeakLearnerType&, - // MatType& data, - // LabelsType& labels, - // const size_t numClasses, - // WeightsType& weights) - // - // This trains the new WeakLearnerType using the hyperparameters from the - // given WeakLearnerType. + // This is split into a separate function, so that we can still call + // AdaBoost::Train() with extra hyperparameters, even when the weak learner + // type does not support the special constructor that takes another weak + // learner. + WeakLearnerType w = WeakLearnerTrainer< + UseExistingWeakLearner, MatType, arma::Row, WeakLearnerType, + WeakLearnerArgs... + >::Train(tempData, labels, numClasses, weights, other, weakLearnerArgs...); - WeakLearnerType w(other, tempData, labels, numClasses, weights); - // There is a bug with Adaboost! It will not use the specified - // hyperparameters for the decision tree because they are not properly - // passed to the new weak learners! (And: it's a hard bug, because the - // decision tree itself doesn't even store the hyperparameters it was - // trained with!) - - // DecisionTree(DecisionTree&, MatType&, LabelsType&, size_t, WeightsType&, double = 0.0, double = 0.0, ...); w.Classify(tempData, predictedLabels); - // Now from predictedLabels, build ht, the weak hypothesis - // buildClassificationMatrix(ht, predictedLabels); - // Now, calculate alpha(t) using ht. for (size_t j = 0; j < D.n_cols; ++j) // instead of D, ht { @@ -166,7 +457,7 @@ double AdaBoost::Train( crt = rt; - // Our goal is to find alphat which mizimizes or approximately minimizes the + // Our goal is to find alphat which minimizes or approximately minimizes the // value of Z as a function of alpha. alphat = 0.5 * log((1 + rt) / (1 - rt)); @@ -176,7 +467,7 @@ double AdaBoost::Train( // Now start modifying the weights. for (size_t j = 0; j < D.n_cols; ++j) { - const double expo = exp(alphat); + const ElemType expo = exp(alphat); if (predictedLabels(j) == labels(j)) { for (size_t k = 0; k < D.n_rows; ++k) @@ -216,76 +507,10 @@ double AdaBoost::Train( // Accumulate the value of zt for the Hamming loss bound. ztProduct *= zt; } + return ztProduct; } -/** - * Classify the given test points. - */ -template -void AdaBoost::Classify( - const MatType& test, - arma::Row& predictedLabels) -{ - arma::Row tempPredictedLabels(test.n_cols); - arma::mat probabilities; - - Classify(test, predictedLabels, probabilities); -} - -/** - * Classify the given test points. - */ -template -void AdaBoost::Classify( - const MatType& test, - arma::Row& predictedLabels, - arma::mat& probabilities) -{ - arma::Row tempPredictedLabels(test.n_cols); - - probabilities.zeros(numClasses, test.n_cols); - predictedLabels.set_size(test.n_cols); - - for (size_t i = 0; i < wl.size(); ++i) - { - wl[i].Classify(test, tempPredictedLabels); - - for (size_t j = 0; j < tempPredictedLabels.n_cols; ++j) - probabilities(tempPredictedLabels(j), j) += alpha[i]; - } - - arma::uword maxIndex = 0; - - for (size_t i = 0; i < predictedLabels.n_cols; ++i) - { - probabilities.col(i) /= arma::accu(probabilities.col(i)); - probabilities.col(i).max(maxIndex); - predictedLabels(i) = maxIndex; - } -} - -/** - * Serialize the AdaBoost model. - */ -template -template -void AdaBoost::serialize(Archive& ar, - const uint32_t /* version */) -{ - ar(CEREAL_NVP(numClasses)); - ar(CEREAL_NVP(tolerance)); - ar(CEREAL_NVP(alpha)); - - // Now serialize each weak learner. - if (cereal::is_loading()) - { - wl.clear(); - wl.resize(alpha.size()); - } - ar(CEREAL_NVP(wl)); -} - } // namespace mlpack #endif From 4e0e9cfcdf5911c11ad63614c6d50e0d7a60ccc8 Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Mon, 13 Nov 2023 16:20:01 -0500 Subject: [PATCH 22/47] Update tests. --- src/mlpack/tests/adaboost_test.cpp | 672 ++++++++++++++++++++--------- 1 file changed, 460 insertions(+), 212 deletions(-) diff --git a/src/mlpack/tests/adaboost_test.cpp b/src/mlpack/tests/adaboost_test.cpp index 457b7617dd..208de91900 100644 --- a/src/mlpack/tests/adaboost_test.cpp +++ b/src/mlpack/tests/adaboost_test.cpp @@ -19,19 +19,24 @@ using namespace arma; using namespace mlpack; +// TODO: adapt to non-deprecated calls + /** * This test case runs the AdaBoost.mh algorithm on the UCI Iris dataset. It * checks whether the hamming loss breaches the upperbound, which is provided by * ztAccumulator. */ -TEST_CASE("HammingLossBoundIris", "[AdaBoostTest]") +TEMPLATE_TEST_CASE("HammingLossBoundIris", "[AdaBoostTest]", mat, fmat) { - arma::mat inputData; + typedef TestType MatType; + typedef typename MatType::elem_type eT; + + MatType inputData; if (!data::Load("iris.csv", inputData)) FAIL("Cannot load test dataset iris.csv!"); - arma::Mat labels; + Mat labels; if (!data::Load("iris_labels.txt", labels)) FAIL("Cannot load labels for iris iris_labels.txt"); @@ -41,20 +46,22 @@ TEST_CASE("HammingLossBoundIris", "[AdaBoostTest]") // Run the perceptron for perceptronIter iterations. int perceptronIter = 400; - Perceptron<> p(inputData, labels.row(0), numClasses, perceptronIter); + typedef Perceptron + PerceptronType; + PerceptronType p(inputData, labels.row(0), numClasses, perceptronIter); // Define parameters for AdaBoost. size_t iterations = 100; - double tolerance = 1e-10; - AdaBoost<> a(tolerance); - double ztProduct = a.Train(inputData, labels.row(0), numClasses, p, - iterations, tolerance); + eT tolerance = 1e-10; + AdaBoost a(tolerance); + eT ztProduct = a.Train(inputData, labels.row(0), numClasses, p, iterations, + tolerance); - arma::Row predictedLabels; + Row predictedLabels; a.Classify(inputData, predictedLabels); - size_t countError = arma::accu(labels != predictedLabels); - double hammingLoss = (double) countError / labels.n_cols; + size_t countError = accu(labels != predictedLabels); + eT hammingLoss = (eT) countError / labels.n_cols; // Check that ztProduct is finite. REQUIRE(std::isfinite(ztProduct) == true); @@ -66,14 +73,17 @@ TEST_CASE("HammingLossBoundIris", "[AdaBoostTest]") * checks if the error returned by running a single instance of the weak learner * close to that of the boosted weak learner using adaboost. */ -TEST_CASE("WeakLearnerErrorIris", "[AdaBoostTest]") +TEMPLATE_TEST_CASE("WeakLearnerErrorIris", "[AdaBoostTest]", mat, fmat) { - arma::mat inputData; + typedef TestType MatType; + typedef typename MatType::elem_type eT; + + MatType inputData; if (!data::Load("iris.csv", inputData)) FAIL("Cannot load test dataset iris.csv!"); - arma::Mat labels; + Mat labels; if (!data::Load("iris_labels.txt", labels)) FAIL("Cannot load labels for iris iris_labels.txt"); @@ -84,23 +94,26 @@ TEST_CASE("WeakLearnerErrorIris", "[AdaBoostTest]") // Run the perceptron for perceptronIter iterations. int perceptronIter = 400; - arma::Row perceptronPrediction(labels.n_cols); - Perceptron<> p(inputData, labels.row(0), numClasses, perceptronIter); + Row perceptronPrediction(labels.n_cols); + typedef Perceptron + PerceptronType; + PerceptronType p(inputData, labels.row(0), numClasses, perceptronIter); p.Classify(inputData, perceptronPrediction); - size_t countWeakLearnerError = arma::accu(labels != perceptronPrediction); - double weakLearnerErrorRate = (double) countWeakLearnerError / labels.n_cols; + size_t countWeakLearnerError = accu(labels != perceptronPrediction); + eT weakLearnerErrorRate = (eT) countWeakLearnerError / labels.n_cols; // Define parameters for AdaBoost. size_t iterations = 100; - double tolerance = 1e-10; - AdaBoost<> a(inputData, labels.row(0), numClasses, p, iterations, tolerance); + eT tolerance = 1e-10; + AdaBoost a(inputData, labels.row(0), numClasses, p, + iterations, tolerance); - arma::Row predictedLabels; + Row predictedLabels; a.Classify(inputData, predictedLabels); - size_t countError = arma::accu(labels != predictedLabels);; - double error = (double) countError / labels.n_cols; + size_t countError = accu(labels != predictedLabels);; + eT error = (eT) countError / labels.n_cols; REQUIRE(error <= weakLearnerErrorRate + 0.03); } @@ -110,13 +123,17 @@ TEST_CASE("WeakLearnerErrorIris", "[AdaBoostTest]") * dataset. It checks whether the hamming loss breaches the upperbound, which * is provided by ztAccumulator. */ -TEST_CASE("HammingLossBoundVertebralColumn", "[AdaBoostTest]") +TEMPLATE_TEST_CASE("HammingLossBoundVertebralColumn", "[AdaBoostTest]", mat, + fmat) { - arma::mat inputData; + typedef TestType MatType; + typedef typename MatType::elem_type eT; + + MatType inputData; if (!data::Load("vc2.csv", inputData)) FAIL("Cannot load test dataset vc2.csv!"); - arma::Mat labels; + Mat labels; if (!data::Load("vc2_labels.txt", labels)) FAIL("Cannot load labels for vc2_labels.txt"); @@ -125,20 +142,22 @@ TEST_CASE("HammingLossBoundVertebralColumn", "[AdaBoostTest]") // Define your own weak learner, perceptron in this case. // Run the perceptron for perceptronIter iterations. size_t perceptronIter = 800; - Perceptron<> p(inputData, labels.row(0), numClasses, perceptronIter); + typedef Perceptron + PerceptronType; + PerceptronType p(inputData, labels.row(0), numClasses, perceptronIter); // Define parameters for AdaBoost. size_t iterations = 50; - double tolerance = 1e-10; - AdaBoost<> a(tolerance); - double ztProduct = a.Train(inputData, labels.row(0), numClasses, p, - iterations, tolerance); + eT tolerance = 1e-10; + AdaBoost a(tolerance); + eT ztProduct = a.Train(inputData, labels.row(0), numClasses, p, iterations, + tolerance); - arma::Row predictedLabels; + Row predictedLabels; a.Classify(inputData, predictedLabels); - size_t countError = arma::accu(labels != predictedLabels); - double hammingLoss = (double) countError / labels.n_cols; + size_t countError = accu(labels != predictedLabels); + eT hammingLoss = (eT) countError / labels.n_cols; // Check that ztProduct is finite. REQUIRE(std::isfinite(ztProduct) == true); @@ -150,13 +169,17 @@ TEST_CASE("HammingLossBoundVertebralColumn", "[AdaBoostTest]") * dataset. It checks if the error returned by running a single instance of the * weak learner is close to that of a boosted weak learner using adaboost. */ -TEST_CASE("WeakLearnerErrorVertebralColumn", "[AdaBoostTest]") +TEMPLATE_TEST_CASE("WeakLearnerErrorVertebralColumn", "[AdaBoostTest]", mat, + fmat) { - arma::mat inputData; + typedef TestType MatType; + typedef typename MatType::elem_type eT; + + MatType inputData; if (!data::Load("vc2.csv", inputData)) FAIL("Cannot load test dataset vc2.csv!"); - arma::Mat labels; + Mat labels; if (!data::Load("vc2_labels.txt", labels)) FAIL("Cannot load labels for vc2_labels.txt"); @@ -167,22 +190,25 @@ TEST_CASE("WeakLearnerErrorVertebralColumn", "[AdaBoostTest]") size_t perceptronIter = 800; Row perceptronPrediction(labels.n_cols); - Perceptron<> p(inputData, labels.row(0), numClasses, perceptronIter); + typedef Perceptron + PerceptronType; + PerceptronType p(inputData, labels.row(0), numClasses, perceptronIter); p.Classify(inputData, perceptronPrediction); - size_t countWeakLearnerError = arma::accu(labels != perceptronPrediction); - double weakLearnerErrorRate = (double) countWeakLearnerError / labels.n_cols; + size_t countWeakLearnerError = accu(labels != perceptronPrediction); + eT weakLearnerErrorRate = (eT) countWeakLearnerError / labels.n_cols; // Define parameters for AdaBoost. size_t iterations = 50; - double tolerance = 1e-10; - AdaBoost<> a(inputData, labels.row(0), numClasses, p, iterations, tolerance); + eT tolerance = 1e-10; + AdaBoost a(inputData, labels.row(0), numClasses, p, + iterations, tolerance); - arma::Row predictedLabels; + Row predictedLabels; a.Classify(inputData, predictedLabels); - size_t countError = arma::accu(labels != predictedLabels); - double error = (double) countError / labels.n_cols; + size_t countError = accu(labels != predictedLabels); + eT error = (eT) countError / labels.n_cols; REQUIRE(error <= weakLearnerErrorRate + 0.03); } @@ -192,13 +218,17 @@ TEST_CASE("WeakLearnerErrorVertebralColumn", "[AdaBoostTest]") * dataset. It checks whether the hamming loss breaches the upperbound, which * is provided by ztAccumulator. */ -TEST_CASE("HammingLossBoundNonLinearSepData", "[AdaBoostTest]") +TEMPLATE_TEST_CASE("HammingLossBoundNonLinearSepData", "[AdaBoostTest]", mat, + fmat) { - arma::mat inputData; + typedef TestType MatType; + typedef typename MatType::elem_type eT; + + MatType inputData; if (!data::Load("train_nonlinsep.txt", inputData)) FAIL("Cannot load test dataset train_nonlinsep.txt!"); - arma::Mat labels; + Mat labels; if (!data::Load("train_labels_nonlinsep.txt", labels)) FAIL("Cannot load labels for train_labels_nonlinsep.txt"); @@ -207,20 +237,22 @@ TEST_CASE("HammingLossBoundNonLinearSepData", "[AdaBoostTest]") // Define your own weak learner, perceptron in this case. // Run the perceptron for perceptronIter iterations. size_t perceptronIter = 800; - Perceptron<> p(inputData, labels.row(0), numClasses, perceptronIter); + typedef Perceptron + PerceptronType; + PerceptronType p(inputData, labels.row(0), numClasses, perceptronIter); // Define parameters for AdaBoost. size_t iterations = 50; - double tolerance = 1e-10; - AdaBoost<> a(tolerance); - double ztProduct = a.Train(inputData, labels.row(0), numClasses, p, - iterations, tolerance); + eT tolerance = 1e-10; + AdaBoost a(tolerance); + eT ztProduct = a.Train(inputData, labels.row(0), numClasses, p, iterations, + tolerance); - arma::Row predictedLabels; + Row predictedLabels; a.Classify(inputData, predictedLabels); - size_t countError = arma::accu(labels == predictedLabels); - double hammingLoss = (double) countError / labels.n_cols; + size_t countError = accu(labels == predictedLabels); + eT hammingLoss = (eT) countError / labels.n_cols; // Check that ztProduct is finite. REQUIRE(std::isfinite(ztProduct) <= true); @@ -232,13 +264,17 @@ TEST_CASE("HammingLossBoundNonLinearSepData", "[AdaBoostTest]") * dataset. It checks if the error returned by running a single instance of the * weak learner is close to that of a boosted weak learner using AdaBoost. */ -TEST_CASE("WeakLearnerErrorNonLinearSepData", "[AdaBoostTest]") +TEMPLATE_TEST_CASE("WeakLearnerErrorNonLinearSepData", "[AdaBoostTest]", mat, + fmat) { - arma::mat inputData; + typedef TestType MatType; + typedef typename MatType::elem_type eT; + + MatType inputData; if (!data::Load("train_nonlinsep.txt", inputData)) FAIL("Cannot load test dataset train_nonlinsep.txt!"); - arma::Mat labels; + Mat labels; if (!data::Load("train_labels_nonlinsep.txt", labels)) FAIL("Cannot load labels for train_labels_nonlinsep.txt"); @@ -249,22 +285,25 @@ TEST_CASE("WeakLearnerErrorNonLinearSepData", "[AdaBoostTest]") size_t perceptronIter = 800; Row perceptronPrediction(labels.n_cols); - Perceptron<> p(inputData, labels.row(0), numClasses, perceptronIter); + typedef Perceptron + PerceptronType; + PerceptronType p(inputData, labels.row(0), numClasses, perceptronIter); p.Classify(inputData, perceptronPrediction); - size_t countWeakLearnerError = arma::accu(labels != perceptronPrediction); - double weakLearnerErrorRate = (double) countWeakLearnerError / labels.n_cols; + size_t countWeakLearnerError = accu(labels != perceptronPrediction); + eT weakLearnerErrorRate = (eT) countWeakLearnerError / labels.n_cols; // Define parameters for AdaBoost. size_t iterations = 50; - double tolerance = 1e-10; - AdaBoost<> a(inputData, labels.row(0), numClasses, p, iterations, tolerance); + eT tolerance = 1e-10; + AdaBoost a(inputData, labels.row(0), numClasses, p, + iterations, tolerance); - arma::Row predictedLabels; + Row predictedLabels; a.Classify(inputData, predictedLabels); - size_t countError = arma::accu(labels != predictedLabels); - double error = (double) countError / labels.n_cols; + size_t countError = accu(labels != predictedLabels); + eT error = (eT) countError / labels.n_cols; REQUIRE(error <= weakLearnerErrorRate + 0.03); } @@ -274,34 +313,37 @@ TEST_CASE("WeakLearnerErrorNonLinearSepData", "[AdaBoostTest]") * checks whether the Hamming loss breaches the upper bound, which is provided * by ztAccumulator. This uses decision stumps as the weak learner. */ -TEST_CASE("HammingLossIris_DS", "[AdaBoostTest]") +TEMPLATE_TEST_CASE("HammingLossIris_DS", "[AdaBoostTest]", mat, fmat) { - arma::mat inputData; + typedef TestType MatType; + typedef typename MatType::elem_type eT; + + MatType inputData; if (!data::Load("iris.csv", inputData)) FAIL("Cannot load test dataset iris.csv!"); - arma::Mat labels; + Mat labels; if (!data::Load("iris_labels.txt", labels)) FAIL("Cannot load labels for iris_labels.txt"); // Define your own weak learner, decision stumps in this case. const size_t numClasses = 3; const size_t inpBucketSize = 6; - arma::Row labelsvec = labels.row(0); + Row labelsvec = labels.row(0); ID3DecisionStump ds(inputData, labelsvec, numClasses, inpBucketSize); // Define parameters for AdaBoost. size_t iterations = 50; - double tolerance = 1e-10; - AdaBoost a(tolerance); - double ztProduct = a.Train(inputData, labelsvec, numClasses, ds, - iterations, tolerance); + eT tolerance = 1e-10; + AdaBoost a(tolerance); + eT ztProduct = a.Train(inputData, labelsvec, numClasses, ds, iterations, + tolerance); - arma::Row predictedLabels; + Row predictedLabels; a.Classify(inputData, predictedLabels); - size_t countError = arma::accu(labels != predictedLabels); - double hammingLoss = (double) countError / labels.n_cols; + size_t countError = accu(labels != predictedLabels); + eT hammingLoss = (eT) countError / labels.n_cols; // Check that ztProduct is finite. REQUIRE(std::isfinite(ztProduct) == true); @@ -314,13 +356,16 @@ TEST_CASE("HammingLossIris_DS", "[AdaBoostTest]") * weak learner is close to that of a boosted weak learner using adaboost. * This is for the weak learner: decision stumps. */ -TEST_CASE("WeakLearnerErrorIris_DS", "[AdaBoostTest]") +TEMPLATE_TEST_CASE("WeakLearnerErrorIris_DS", "[AdaBoostTest]", mat, fmat) { - arma::mat inputData; + typedef TestType MatType; + typedef typename MatType::elem_type eT; + + MatType inputData; if (!data::Load("iris.csv", inputData)) FAIL("Cannot load test dataset iris.csv!"); - arma::Mat labels; + Mat labels; if (!data::Load("iris_labels.txt", labels)) FAIL("Cannot load labels for iris_labels.txt"); @@ -329,28 +374,28 @@ TEST_CASE("WeakLearnerErrorIris_DS", "[AdaBoostTest]") // Define your own weak learner, decision stumps in this case. const size_t numClasses = 3; const size_t inpBucketSize = 6; - arma::Row labelsvec = labels.row(0); + Row labelsvec = labels.row(0); - arma::Row dsPrediction(labels.n_cols); + Row dsPrediction(labels.n_cols); ID3DecisionStump ds(inputData, labelsvec, numClasses, inpBucketSize); ds.Classify(inputData, dsPrediction); - size_t countWeakLearnerError = arma::accu(labels != dsPrediction); - double weakLearnerErrorRate = (double) countWeakLearnerError / labels.n_cols; + size_t countWeakLearnerError = accu(labels != dsPrediction); + eT weakLearnerErrorRate = (eT) countWeakLearnerError / labels.n_cols; // Define parameters for AdaBoost. size_t iterations = 50; - double tolerance = 1e-10; + eT tolerance = 1e-10; - AdaBoost a(inputData, labelsvec, numClasses, ds, + AdaBoost a(inputData, labelsvec, numClasses, ds, iterations, tolerance); - arma::Row predictedLabels; + Row predictedLabels; a.Classify(inputData, predictedLabels); - size_t countError = arma::accu(labels != predictedLabels); - double error = (double) countError / labels.n_cols; + size_t countError = accu(labels != predictedLabels); + eT error = (eT) countError / labels.n_cols; REQUIRE(error <= weakLearnerErrorRate + 0.03); } @@ -361,36 +406,40 @@ TEST_CASE("WeakLearnerErrorIris_DS", "[AdaBoostTest]") * weak learner is close to that of a boosted weak learner using adaboost. * This is for the weak learner: decision stumps. */ -TEST_CASE("HammingLossBoundVertebralColumn_DS", "[AdaBoostTest]") +TEMPLATE_TEST_CASE("HammingLossBoundVertebralColumn_DS", "[AdaBoostTest]", mat, + fmat) { - arma::mat inputData; + typedef TestType MatType; + typedef typename MatType::elem_type eT; + + MatType inputData; if (!data::Load("vc2.csv", inputData)) FAIL("Cannot load test dataset vc2.csv!"); - arma::Mat labels; + Mat labels; if (!data::Load("vc2_labels.txt", labels)) FAIL("Cannot load labels for vc2_labels.txt"); // Define your own weak learner, decision stumps in this case. const size_t numClasses = 3; const size_t inpBucketSize = 6; - arma::Row labelsvec = labels.row(0); + Row labelsvec = labels.row(0); ID3DecisionStump ds(inputData, labelsvec, numClasses, inpBucketSize); // Define parameters for AdaBoost. size_t iterations = 50; - double tolerance = 1e-10; + eT tolerance = 1e-10; - AdaBoost a(tolerance); - double ztProduct = a.Train(inputData, labelsvec, numClasses, ds, - iterations, tolerance); + AdaBoost a(tolerance); + eT ztProduct = a.Train(inputData, labelsvec, numClasses, ds, iterations, + tolerance); - arma::Row predictedLabels; + Row predictedLabels; a.Classify(inputData, predictedLabels); - size_t countError = arma::accu(labels != predictedLabels); - double hammingLoss = (double) countError / labels.n_cols; + size_t countError = accu(labels != predictedLabels); + eT hammingLoss = (eT) countError / labels.n_cols; // Check that ztProduct is finite. REQUIRE(std::isfinite(ztProduct) == true); @@ -403,39 +452,43 @@ TEST_CASE("HammingLossBoundVertebralColumn_DS", "[AdaBoostTest]") * weak learner is close to that of a boosted weak learner using adaboost. * This is for the weak learner: decision stumps. */ -TEST_CASE("WeakLearnerErrorVertebralColumn_DS", "[AdaBoostTest]") +TEMPLATE_TEST_CASE("WeakLearnerErrorVertebralColumn_DS", "[AdaBoostTest]", mat, + fmat) { - arma::mat inputData; + typedef TestType MatType; + typedef typename MatType::elem_type eT; + + MatType inputData; if (!data::Load("vc2.csv", inputData)) FAIL("Cannot load test dataset vc2.csv!"); - arma::Mat labels; + Mat labels; if (!data::Load("vc2_labels.txt", labels)) FAIL("Cannot load labels for vc2_labels.txt"); // Define your own weak learner, decision stumps in this case. const size_t numClasses = 3; const size_t inpBucketSize = 6; - arma::Row dsPrediction(labels.n_cols); - arma::Row labelsvec = labels.row(0); + Row dsPrediction(labels.n_cols); + Row labelsvec = labels.row(0); ID3DecisionStump ds(inputData, labelsvec, numClasses, inpBucketSize); ds.Classify(inputData, dsPrediction); - size_t countWeakLearnerError = arma::accu(labels != dsPrediction); - double weakLearnerErrorRate = (double) countWeakLearnerError / labels.n_cols; + size_t countWeakLearnerError = accu(labels != dsPrediction); + eT weakLearnerErrorRate = (eT) countWeakLearnerError / labels.n_cols; // Define parameters for AdaBoost. size_t iterations = 50; - double tolerance = 1e-10; - AdaBoost a(inputData, labelsvec, numClasses, ds, + eT tolerance = 1e-10; + AdaBoost a(inputData, labelsvec, numClasses, ds, iterations, tolerance); - arma::Row predictedLabels; + Row predictedLabels; a.Classify(inputData, predictedLabels); - size_t countError = arma::accu(labels != predictedLabels); - double error = (double) countError / labels.n_cols; + size_t countError = accu(labels != predictedLabels); + eT error = (eT) countError / labels.n_cols; REQUIRE(error <= weakLearnerErrorRate + 0.03); } @@ -445,36 +498,40 @@ TEST_CASE("WeakLearnerErrorVertebralColumn_DS", "[AdaBoostTest]") * dataset. It checks whether the hamming loss breaches the upperbound, which * is provided by ztAccumulator. This is for the weak learner: decision stumps. */ -TEST_CASE("HammingLossBoundNonLinearSepData_DS", "[AdaBoostTest]") +TEMPLATE_TEST_CASE("HammingLossBoundNonLinearSepData_DS", "[AdaBoostTest]", mat, + fmat) { - arma::mat inputData; + typedef TestType MatType; + typedef typename MatType::elem_type eT; + + MatType inputData; if (!data::Load("train_nonlinsep.txt", inputData)) FAIL("Cannot load test dataset train_nonlinsep.txt!"); - arma::Mat labels; + Mat labels; if (!data::Load("train_labels_nonlinsep.txt", labels)) FAIL("Cannot load labels for train_labels_nonlinsep.txt"); // Define your own weak learner, decision stumps in this case. const size_t numClasses = 2; const size_t inpBucketSize = 6; - arma::Row labelsvec = labels.row(0); + Row labelsvec = labels.row(0); ID3DecisionStump ds(inputData, labelsvec, numClasses, inpBucketSize); // Define parameters for Adaboost. size_t iterations = 50; - double tolerance = 1e-10; + eT tolerance = 1e-10; - AdaBoost a(tolerance); - double ztProduct = a.Train(inputData, labelsvec, numClasses, ds, - iterations, tolerance); + AdaBoost a(tolerance); + eT ztProduct = a.Train(inputData, labelsvec, numClasses, ds, iterations, + tolerance); - arma::Row predictedLabels; + Row predictedLabels; a.Classify(inputData, predictedLabels); - size_t countError = arma::accu(labels != predictedLabels); - double hammingLoss = (double) countError / labels.n_cols; + size_t countError = accu(labels != predictedLabels); + eT hammingLoss = (eT) countError / labels.n_cols; // Check that ztProduct is finite. REQUIRE(std::isfinite(ztProduct) == true); @@ -487,41 +544,45 @@ TEST_CASE("HammingLossBoundNonLinearSepData_DS", "[AdaBoostTest]") * weak learner is close to that of a boosted weak learner using adaboost. * This for the weak learner: decision stumps. */ -TEST_CASE("WeakLearnerErrorNonLinearSepData_DS", "[AdaBoostTest]") +TEMPLATE_TEST_CASE("WeakLearnerErrorNonLinearSepData_DS", "[AdaBoostTest]", mat, + fmat) { - arma::mat inputData; + typedef TestType MatType; + typedef typename MatType::elem_type eT; + + MatType inputData; if (!data::Load("train_nonlinsep.txt", inputData)) FAIL("Cannot load test dataset train_nonlinsep.txt!"); - arma::Mat labels; + Mat labels; if (!data::Load("train_labels_nonlinsep.txt", labels)) FAIL("Cannot load labels for train_labels_nonlinsep.txt"); // Define your own weak learner, decision stumps in this case. const size_t numClasses = 2; const size_t inpBucketSize = 3; - arma::Row labelsvec = labels.row(0); + Row labelsvec = labels.row(0); - arma::Row dsPrediction(labels.n_cols); + Row dsPrediction(labels.n_cols); ID3DecisionStump ds(inputData, labelsvec, numClasses, inpBucketSize); ds.Classify(inputData, dsPrediction); - size_t countWeakLearnerError = arma::accu(labels != dsPrediction); - double weakLearnerErrorRate = (double) countWeakLearnerError / labels.n_cols; + size_t countWeakLearnerError = accu(labels != dsPrediction); + eT weakLearnerErrorRate = (eT) countWeakLearnerError / labels.n_cols; // Define parameters for AdaBoost. size_t iterations = 500; - double tolerance = 1e-23; + eT tolerance = 1e-23; - AdaBoost a(inputData, labelsvec, numClasses, ds, + AdaBoost a(inputData, labelsvec, numClasses, ds, iterations, tolerance); - arma::Row predictedLabels; + Row predictedLabels; a.Classify(inputData, predictedLabels); - size_t countError = arma::accu(labels != predictedLabels); - double error = (double) countError / labels.n_cols; + size_t countError = accu(labels != predictedLabels); + eT error = (eT) countError / labels.n_cols; REQUIRE(error <= weakLearnerErrorRate + 0.03); } @@ -531,13 +592,16 @@ TEST_CASE("WeakLearnerErrorNonLinearSepData_DS", "[AdaBoostTest]") * dataset. It tests the Classify function and checks for a satisfactory error * rate. */ -TEST_CASE("ClassifyTest_VERTEBRALCOL", "[AdaBoostTest]") +TEMPLATE_TEST_CASE("ClassifyTest_VERTEBRALCOL", "[AdaBoostTest]", mat, fmat) { - arma::mat inputData; + typedef TestType MatType; + typedef typename MatType::elem_type eT; + + MatType inputData; if (!data::Load("vc2.csv", inputData)) FAIL("Cannot load test dataset vc2.csv!"); - arma::Mat labels; + Mat labels; if (!data::Load("vc2_labels.txt", labels)) FAIL("Cannot load labels for vc2_labels.txt"); @@ -545,12 +609,11 @@ TEST_CASE("ClassifyTest_VERTEBRALCOL", "[AdaBoostTest]") // Run the perceptron for perceptronIter iterations. size_t perceptronIter = 1000; - arma::mat testData; - + MatType testData; if (!data::Load("vc2_test.csv", testData)) FAIL("Cannot load test dataset vc2_test.csv!"); - arma::Mat trueTestLabels; + Mat trueTestLabels; if (!data::Load("vc2_test_labels.txt", trueTestLabels)) FAIL("Cannot load labels for vc2_test_labels.txt"); @@ -558,17 +621,20 @@ TEST_CASE("ClassifyTest_VERTEBRALCOL", "[AdaBoostTest]") const size_t numClasses = max(labels.row(0)) + 1; Row perceptronPrediction(labels.n_cols); - Perceptron<> p(inputData, labels.row(0), numClasses, perceptronIter); + typedef Perceptron + PerceptronType; + PerceptronType p(inputData, labels.row(0), numClasses, perceptronIter); p.Classify(inputData, perceptronPrediction); // Define parameters for AdaBoost. size_t iterations = 100; - double tolerance = 1e-10; - AdaBoost<> a(inputData, labels.row(0), numClasses, p, iterations, tolerance); + eT tolerance = 1e-10; + AdaBoost a(inputData, labels.row(0), numClasses, p, + iterations, tolerance); - arma::Row predictedLabels1(testData.n_cols), - predictedLabels2(testData.n_cols); - arma::mat probabilities; + Row predictedLabels1(testData.n_cols), + predictedLabels2(testData.n_cols); + MatType probabilities; a.Classify(testData, predictedLabels1); a.Classify(testData, predictedLabels2, probabilities); @@ -579,19 +645,19 @@ TEST_CASE("ClassifyTest_VERTEBRALCOL", "[AdaBoostTest]") for (size_t i = 0; i < predictedLabels1.n_cols; ++i) REQUIRE(predictedLabels1[i] == predictedLabels2[i]); - arma::colvec pRow; - arma::uword maxIndex = 0; + Col pRow; + uword maxIndex = 0; for (size_t i = 0; i < predictedLabels1.n_cols; ++i) { pRow = probabilities.unsafe_col(i); pRow.max(maxIndex); REQUIRE(predictedLabels1(i) == maxIndex); - REQUIRE(arma::accu(probabilities.col(i)) == Approx(1).epsilon(1e-7)); + REQUIRE(accu(probabilities.col(i)) == Approx(1)); } - size_t localError = arma::accu(trueTestLabels != predictedLabels1); - double lError = (double) localError / trueTestLabels.n_cols; + size_t localError = accu(trueTestLabels != predictedLabels1); + eT lError = (eT) localError / trueTestLabels.n_cols; REQUIRE(lError <= 0.30); } @@ -600,44 +666,46 @@ TEST_CASE("ClassifyTest_VERTEBRALCOL", "[AdaBoostTest]") * dataset. It tests the Classify function and checks for a satisfactory error * rate. */ -TEST_CASE("ClassifyTest_NONLINSEP", "[AdaBoostTest]") +TEMPLATE_TEST_CASE("ClassifyTest_NONLINSEP", "[AdaBoostTest]", mat, fmat) { - arma::mat inputData; + typedef TestType MatType; + typedef typename MatType::elem_type eT; + + MatType inputData; if (!data::Load("train_nonlinsep.txt", inputData)) FAIL("Cannot load test dataset train_nonlinsep.txt!"); - arma::Mat labels; + Mat labels; if (!data::Load("train_labels_nonlinsep.txt", labels)) FAIL("Cannot load labels for train_labels_nonlinsep.txt"); // Define your own weak learner; in this test decision stumps are used. const size_t numClasses = 2; const size_t inpBucketSize = 3; - arma::Row labelsvec = labels.row(0); + Row labelsvec = labels.row(0); - arma::mat testData; + MatType testData; if (!data::Load("test_nonlinsep.txt", testData)) FAIL("Cannot load test dataset test_nonlinsep.txt!"); - arma::Mat trueTestLabels; - + Mat trueTestLabels; if (!data::Load("test_labels_nonlinsep.txt", trueTestLabels)) FAIL("Cannot load labels for test_labels_nonlinsep.txt"); - arma::Row dsPrediction(labels.n_cols); + Row dsPrediction(labels.n_cols); ID3DecisionStump ds(inputData, labelsvec, numClasses, inpBucketSize); // Define parameters for AdaBoost. size_t iterations = 50; - double tolerance = 1e-10; - AdaBoost a(inputData, labelsvec, numClasses, ds, + eT tolerance = 1e-10; + AdaBoost a(inputData, labelsvec, numClasses, ds, iterations, tolerance); - arma::Row predictedLabels1(testData.n_cols), - predictedLabels2(testData.n_cols); - arma::mat probabilities; + Row predictedLabels1(testData.n_cols), + predictedLabels2(testData.n_cols); + MatType probabilities; a.Classify(testData, predictedLabels1); a.Classify(testData, predictedLabels2, probabilities); @@ -647,19 +715,19 @@ TEST_CASE("ClassifyTest_NONLINSEP", "[AdaBoostTest]") for (size_t i = 0; i < predictedLabels1.n_cols; ++i) REQUIRE(predictedLabels1[i] == predictedLabels2[i]); - arma::colvec pRow; - arma::uword maxIndex = 0; + Col pRow; + uword maxIndex = 0; for (size_t i = 0; i < predictedLabels1.n_cols; ++i) { pRow = probabilities.unsafe_col(i); pRow.max(maxIndex); REQUIRE(predictedLabels1(i) == maxIndex); - REQUIRE(arma::accu(probabilities.col(i)) == Approx(1).epsilon(1e-7)); + REQUIRE(accu(probabilities.col(i)) == Approx(1).epsilon(1e-7)); } - size_t localError = arma::accu(trueTestLabels != predictedLabels1); - double lError = (double) localError / trueTestLabels.n_cols; + size_t localError = accu(trueTestLabels != predictedLabels1); + eT lError = (eT) localError / trueTestLabels.n_cols; REQUIRE(lError <= 0.30); } @@ -669,13 +737,16 @@ TEST_CASE("ClassifyTest_NONLINSEP", "[AdaBoostTest]") * the remaining third of the dataset (iris_test.csv). It tests the Classify() * function and checks for a satisfactory error rate. */ -TEST_CASE("ClassifyTest_IRIS", "[AdaBoostTest]") +TEMPLATE_TEST_CASE("ClassifyTest_IRIS", "[AdaBoostTest]", mat, fmat) { - arma::mat inputData; + typedef TestType MatType; + typedef typename MatType::elem_type eT; + + MatType inputData; if (!data::Load("iris_train.csv", inputData)) FAIL("Cannot load test dataset iris_train.csv!"); - arma::Mat labels; + Mat labels; if (!data::Load("iris_train_labels.csv", labels)) FAIL("Cannot load labels for iris_train_labels.csv"); const size_t numClasses = max(labels.row(0)) + 1; @@ -684,27 +755,30 @@ TEST_CASE("ClassifyTest_IRIS", "[AdaBoostTest]") // Run the perceptron for perceptronIter iterations. size_t perceptronIter = 800; - Perceptron<> p(inputData, labels.row(0), numClasses, perceptronIter); + typedef Perceptron + PerceptronType; + PerceptronType p(inputData, labels.row(0), numClasses, perceptronIter); // Define parameters for AdaBoost. size_t iterations = 50; - double tolerance = 1e-10; - AdaBoost<> a(inputData, labels.row(0), numClasses, p, iterations, tolerance); + eT tolerance = 1e-10; + AdaBoost a(inputData, labels.row(0), numClasses, p, + iterations, tolerance); - arma::mat testData; + MatType testData; if (!data::Load("iris_test.csv", testData)) FAIL("Cannot load test dataset iris_test.csv!"); - arma::Row predictedLabels(testData.n_cols); + Row predictedLabels(testData.n_cols); a.Classify(testData, predictedLabels); - arma::Mat trueTestLabels; + Mat trueTestLabels; if (!data::Load("iris_test_labels.csv", trueTestLabels)) FAIL("Cannot load test dataset iris_test_labels.csv!"); - arma::Row predictedLabels1(testData.n_cols), - predictedLabels2(testData.n_cols); - arma::mat probabilities; + Row predictedLabels1(testData.n_cols), + predictedLabels2(testData.n_cols); + MatType probabilities; a.Classify(testData, predictedLabels1); a.Classify(testData, predictedLabels2, probabilities); @@ -714,8 +788,8 @@ TEST_CASE("ClassifyTest_IRIS", "[AdaBoostTest]") for (size_t i = 0; i < predictedLabels1.n_cols; ++i) REQUIRE(predictedLabels1[i] == predictedLabels2[i]); - arma::colvec pRow; - arma::uword maxIndex = 0; + Col pRow; + uword maxIndex = 0; for (size_t i = 0; i < predictedLabels1.n_cols; ++i) { @@ -726,7 +800,7 @@ TEST_CASE("ClassifyTest_IRIS", "[AdaBoostTest]") } size_t localError = arma::accu(trueTestLabels != predictedLabels1); - double lError = (double) localError / labels.n_cols; + eT lError = (eT) localError / labels.n_cols; REQUIRE(lError <= 0.30); } @@ -734,26 +808,32 @@ TEST_CASE("ClassifyTest_IRIS", "[AdaBoostTest]") * Ensure that the Train() function works like it is supposed to, by building * AdaBoost on one dataset and then re-training on another dataset. */ -TEST_CASE("TrainTest", "[AdaBoostTest]") +TEMPLATE_TEST_CASE("TrainTest", "[AdaBoostTest]", mat, fmat) { + typedef TestType MatType; + typedef typename MatType::elem_type eT; + // First train on the iris dataset. - arma::mat inputData; + MatType inputData; if (!data::Load("iris_train.csv", inputData)) FAIL("Cannot load test dataset iris_train.csv!"); - arma::Mat labels; + Mat labels; if (!data::Load("iris_train_labels.csv", labels)) FAIL("Cannot load labels for iris_train_labels.csv"); const size_t numClasses = max(labels.row(0)) + 1; size_t perceptronIter = 800; - Perceptron<> p(inputData, labels.row(0), numClasses, perceptronIter); + typedef Perceptron + PerceptronType; + PerceptronType p(inputData, labels.row(0), numClasses, perceptronIter); // Now train AdaBoost. size_t iterations = 50; - double tolerance = 1e-10; - AdaBoost<> a(inputData, labels.row(0), numClasses, p, iterations, tolerance); + eT tolerance = 1e-10; + AdaBoost a(inputData, labels.row(0), numClasses, p, + iterations, tolerance); // Now load another dataset... if (!data::Load("vc2.csv", inputData)) @@ -763,44 +843,48 @@ TEST_CASE("TrainTest", "[AdaBoostTest]") const size_t newNumClasses = max(labels.row(0)) + 1; - Perceptron<> p2(inputData, labels.row(0), newNumClasses, perceptronIter); + PerceptronType p2(inputData, labels.row(0), newNumClasses, perceptronIter); a.Train(inputData, labels.row(0), newNumClasses, p2, iterations, tolerance); // Load test set to see if it trained on vc2 correctly. - arma::mat testData; + MatType testData; if (!data::Load("vc2_test.csv", testData)) FAIL("Cannot load test dataset vc2_test.csv!"); - arma::Mat trueTestLabels; + Mat trueTestLabels; if (!data::Load("vc2_test_labels.txt", trueTestLabels)) FAIL("Cannot load labels for vc2_test_labels.txt"); // Define parameters for AdaBoost. - arma::Row predictedLabels(testData.n_cols); + Row predictedLabels(testData.n_cols); a.Classify(testData, predictedLabels); - int localError = arma::accu(trueTestLabels != predictedLabels); - double lError = (double) localError / trueTestLabels.n_cols; + int localError = accu(trueTestLabels != predictedLabels); + eT lError = (eT) localError / trueTestLabels.n_cols; REQUIRE(lError <= 0.30); } -TEST_CASE("PerceptronSerializationTest", "[AdaBoostTest]") +TEMPLATE_TEST_CASE("PerceptronSerializationTest", "[AdaBoostTest]", fmat, mat) { + typedef TestType MatType; + // Build an AdaBoost object. - mat data = randu(10, 500); + MatType data = randu(10, 500); Row labels(500); for (size_t i = 0; i < 250; ++i) labels[i] = 0; for (size_t i = 250; i < 500; ++i) labels[i] = 1; - Perceptron<> p(data, labels, 2, 800); - AdaBoost<> ab(data, labels, 2, p, 50, 1e-10); + typedef Perceptron + PerceptronType; + PerceptronType p(data, labels, 2, 800); + AdaBoost ab(data, labels, 2, p, 50, 1e-10); // Now create another dataset to train with. - mat otherData = randu(5, 200); + MatType otherData = randu(5, 200); Row otherLabels(200); for (size_t i = 0; i < 100; ++i) otherLabels[i] = 1; @@ -809,10 +893,11 @@ TEST_CASE("PerceptronSerializationTest", "[AdaBoostTest]") for (size_t i = 150; i < 200; ++i) otherLabels[i] = 2; - Perceptron<> p2(otherData, otherLabels, 3, 500); - AdaBoost<> abText(otherData, otherLabels, 3, p2, 50, 1e-10); + PerceptronType p2(otherData, otherLabels, 3, 500); + AdaBoost abText(otherData, otherLabels, 3, p2, 50, + 1e-10); - AdaBoost<> abXml, abBinary; + AdaBoost abXml, abBinary; SerializeObjectAll(ab, abXml, abText, abBinary); @@ -839,10 +924,13 @@ TEST_CASE("PerceptronSerializationTest", "[AdaBoostTest]") } } -TEST_CASE("ID3DecisionStumpSerializationTest", "[AdaBoostTest]") +TEMPLATE_TEST_CASE("ID3DecisionStumpSerializationTest", "[AdaBoostTest]", mat, + fmat) { + typedef TestType MatType; + // Build an AdaBoost object. - mat data = randu(10, 500); + MatType data = randu(10, 500); Row labels(500); for (size_t i = 0; i < 250; ++i) labels[i] = 0; @@ -850,10 +938,10 @@ TEST_CASE("ID3DecisionStumpSerializationTest", "[AdaBoostTest]") labels[i] = 1; ID3DecisionStump p(data, labels, 2, 800); - AdaBoost ab(data, labels, 2, p, 50, 1e-10); + AdaBoost ab(data, labels, 2, p, 50, 1e-10); // Now create another dataset to train with. - mat otherData = randu(5, 200); + MatType otherData = randu(5, 200); Row otherLabels(200); for (size_t i = 0; i < 100; ++i) otherLabels[i] = 1; @@ -863,9 +951,10 @@ TEST_CASE("ID3DecisionStumpSerializationTest", "[AdaBoostTest]") otherLabels[i] = 2; ID3DecisionStump p2(otherData, otherLabels, 3, 500); - AdaBoost abText(otherData, otherLabels, 3, p2, 50, 1e-10); + AdaBoost abText(otherData, otherLabels, 3, p2, 50, + 1e-10); - AdaBoost abXml, abBinary; + AdaBoost abXml, abBinary; SerializeObjectAll(ab, abXml, abText, abBinary); @@ -888,3 +977,162 @@ TEST_CASE("ID3DecisionStumpSerializationTest", "[AdaBoostTest]") abBinary.WeakLearner(i).SplitDimension()); } } + +TEMPLATE_TEST_CASE("AdaBoostSinglePointClassify", "[AdaBoostTest]", mat, fmat) +{ + typedef TestType MatType; + + // Create random data. + MatType data = randu(10, 100); + // Create random labels. + Row labels = randi>(100, distr_param(0, 3)); + + // Train a model. + typedef Perceptron + PerceptronType; + AdaBoost ab(data, labels, 4); + + // Ensure that we can get single-point classifications. + for (size_t i = 0; i < 100; ++i) + { + const size_t prediction = ab.Classify(data.col(i)); + + REQUIRE(prediction <= 3); + } +} + +TEMPLATE_TEST_CASE("AdaBoostSinglePointClassifyWithProbs", "[AdaBoostTest]", + mat, fmat) +{ + typedef TestType MatType; + typedef typename MatType::elem_type eT; + + // Create random data. + MatType data = randu(10, 100); + // Create random labels. + Row labels = randi>(100, distr_param(0, 3)); + + // Train a model. + typedef Perceptron + PerceptronType; + AdaBoost ab(data, labels, 4); + + // Ensure that we can get single-point classifications. + for (size_t i = 0; i < 100; ++i) + { + size_t prediction; + Row probabilities; + ab.Classify(data.col(i), prediction, probabilities); + + REQUIRE(prediction <= 3); + REQUIRE(accu(probabilities) == Approx((eT) 1.0)); + } +} + +// Make sure that everything works when we use the constructor that takes extra +// hyperparameters. +TEMPLATE_TEST_CASE("AdaBoostParamsConstructor", "[AdaBoostTest]", fmat, mat) +{ + typedef TestType MatType; + typedef typename MatType::elem_type ElemType; + + MatType inputData; + if (!data::Load("iris.csv", inputData)) + FAIL("Cannot load test dataset iris.csv!"); + + Mat labels; + if (!data::Load("iris_labels.txt", labels)) + FAIL("Cannot load labels for iris iris_labels.txt"); + + const size_t numClasses = max(labels.row(0)) + 1; + + // Create two AdaBoost models. One does not allow the perceptron to train for + // more than one iteration, and therefore should get less accuracy than the + // one we let train in full. + typedef Perceptron + PerceptronType; + + AdaBoost a1(inputData, labels, numClasses, 2, 1e-6, + 1 /* perceptron max iterations */); + AdaBoost a2(inputData, labels, numClasses, 2, 1e-6, + 100 /* perceptron max iterations */); + + // Make sure test data performance is better for a2. + Row predictions1, predictions2; + a1.Classify(inputData, predictions1); + a2.Classify(inputData, predictions2); + + const size_t correct1 = accu(predictions1 == labels); + const size_t correct2 = accu(predictions2 == labels); + + REQUIRE(correct2 > 0); + REQUIRE(correct2 >= correct1); +} + +// Ensure that all Train() overloads work correctly. +TEMPLATE_TEST_CASE("AdaBoostTrainOverloads", "[AdaBoostTest]", fmat, mat) +{ + typedef TestType MatType; + typedef typename MatType::elem_type ElemType; + + // Create random data. + MatType data = randu(10, 100); + // Create random labels. + Row labels = randi>(100, distr_param(0, 3)); + + typedef Perceptron + PerceptronType; + PerceptronType p; // For versions that take an initialized weak learner. + p.MaxIterations() = 150; + AdaBoost a1, a2, a3, a4, a5, a6, a7; + a1.MaxIterations() = 75; + a4.MaxIterations() = 65; + a1.Tolerance() = 1e-4; + a2.Tolerance() = 1e-5; + a4.Tolerance() = 2e-4; + a5.Tolerance() = 2e-5; + + a1.Train(data, labels, 4, p); + a2.Train(data, labels, 4, p, 10); + a3.Train(data, labels, 4, p, 50, 1e-3); + a4.Train(data, labels, 4); + a5.Train(data, labels, 4, 15); + a6.Train(data, labels, 4, 55, 1e-3); + a7.Train(data, labels, 4, 60, 2e-3, 100); + + // Make sure hyperparameters were set correctly, where appropriate. + REQUIRE(a1.MaxIterations() == 75); + REQUIRE(a2.MaxIterations() == 10); + REQUIRE(a3.MaxIterations() == 50); + REQUIRE(a4.MaxIterations() == 65); + REQUIRE(a5.MaxIterations() == 15); + REQUIRE(a6.MaxIterations() == 55); + REQUIRE(a7.MaxIterations() == 60); + + REQUIRE(a1.Tolerance() == Approx(1e-4)); + REQUIRE(a2.Tolerance() == Approx(1e-5)); + REQUIRE(a3.Tolerance() == Approx(1e-3)); + REQUIRE(a4.Tolerance() == Approx(2e-4)); + REQUIRE(a5.Tolerance() == Approx(2e-5)); + REQUIRE(a6.Tolerance() == Approx(1e-3)); + REQUIRE(a7.Tolerance() == Approx(2e-3)); + + // Make sure anything at all was trained. + REQUIRE(a1.WeakLearners() > 0); + REQUIRE(a2.WeakLearners() > 0); + REQUIRE(a3.WeakLearners() > 0); + REQUIRE(a4.WeakLearners() > 0); + REQUIRE(a5.WeakLearners() > 0); + REQUIRE(a6.WeakLearners() > 0); + REQUIRE(a7.WeakLearners() > 0); + + // Make sure the maximum number of iterations in the perceptron was set + // properly. + REQUIRE(a1.WeakLearner(0).MaxIterations() == 150); + REQUIRE(a2.WeakLearner(0).MaxIterations() == 150); + REQUIRE(a3.WeakLearner(0).MaxIterations() == 150); + REQUIRE(a4.WeakLearner(0).MaxIterations() == 1000); + REQUIRE(a5.WeakLearner(0).MaxIterations() == 1000); + REQUIRE(a6.WeakLearner(0).MaxIterations() == 1000); + REQUIRE(a7.WeakLearner(0).MaxIterations() == 100); +} From 23fb0bbe34fb78d3104aa90e7d73131e19022ee7 Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Mon, 13 Nov 2023 16:53:11 -0500 Subject: [PATCH 23/47] Remove functions that will be deprecated; remove finished TODOs. --- doc/user/methods/adaboost.md | 29 ----------------------------- 1 file changed, 29 deletions(-) diff --git a/doc/user/methods/adaboost.md b/doc/user/methods/adaboost.md index 4fc79c73da..01ac225304 100644 --- a/doc/user/methods/adaboost.md +++ b/doc/user/methods/adaboost.md @@ -63,24 +63,12 @@ std::cout << arma::accu(predictions == 3) << " test points classified as class " calling [`Classify()`](#classification). --- - * `ab = AdaBoost(data, labels, numClasses, maxIterations=100, tolerance=1e-6)` - Train model using default weak learner hyperparameters. --- - * `ab = AdaBoost(data, labels, numClasses, weakLearner, maxIterations=100, tolerance=1e-6)` - - Train model with custom weak learner parameters. - - The given `weakLearner` does not need to be trained; any hyperparameter - settings in `weakLearner` are used for training each AdaBoost weak - learner (see the [simple examples](#simple-examples)). - ---- - - - * `ab = AdaBoost(data, labels, numClasses, maxIterations=100, tolerance=1e-6, [weak learner hyperparameters...])` - Train model with custom weak learner hyperparameters. @@ -117,8 +105,6 @@ As an alternative to passing hyperparameters, each hyperparameter can be set with a standalone method. The following functions can be used before calling `Train()` to set hyperparameters: - - * `ab.MaxIterations() = maxIter;` will set the maximum number of weak learners during training to `maxIter`. * `ab.Tolerance() = tol;` will set the tolerance to `tol`. @@ -135,24 +121,11 @@ If training is not done as part of the constructor call, it can be done with one of the versions of the `Train()` member function. For an instance of `AdaBoost` named `ab`, the following functions for training are available: - - * `ab.Train(data, labels, numClasses, maxIterations=100, tolerance=1e-6)` - Train model using default weak learner parameters. --- - * `ab.Train(data, labels, numClasses, weakLearner, maxIterations=100, tolerance=1e-6)` - - Train model with custom weak learner parameters. - - The given `weakLearner` does not need to be trained; any hyperparameter - settings in `weakLearner` are used for training each AdaBoost weak learner - (see the [simple examples](#simple-examples)). - ---- - - - * `ab.Train(data, labels, numClasses, maxIterations=100, tolerance=1e-6, [weak learner hyperparameters...])` - Train model with custom weak learner parameters. - Hyperparameters for the weak learner are any arguments to the weak @@ -183,8 +156,6 @@ the [Classification Parameters](#classification-parameters) section below. #### Forms: - - * `size_t predictedClass = ab.Classify(point)` - ***(Single-point)*** - Classify a single point, returning the predicted class. From 585496bb8bb764ef1d9969dc1a453351e54b33c2 Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Mon, 13 Nov 2023 16:53:44 -0500 Subject: [PATCH 24/47] Use universal references in constructor. --- src/mlpack/methods/adaboost/adaboost.hpp | 2 +- src/mlpack/methods/adaboost/adaboost_impl.hpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/mlpack/methods/adaboost/adaboost.hpp b/src/mlpack/methods/adaboost/adaboost.hpp index 54ec99fb77..7bbb5cbbf1 100644 --- a/src/mlpack/methods/adaboost/adaboost.hpp +++ b/src/mlpack/methods/adaboost/adaboost.hpp @@ -107,7 +107,7 @@ class AdaBoost const size_t numClasses, const size_t maxIterations = 100, const ElemType tolerance = 1e-6, - WeakLearnerArgs&... weakLearnerArgs); + WeakLearnerArgs&&... weakLearnerArgs); /** * Constructor. This runs the AdaBoost.MH algorithm to provide a trained diff --git a/src/mlpack/methods/adaboost/adaboost_impl.hpp b/src/mlpack/methods/adaboost/adaboost_impl.hpp index db97d97230..499da6006f 100644 --- a/src/mlpack/methods/adaboost/adaboost_impl.hpp +++ b/src/mlpack/methods/adaboost/adaboost_impl.hpp @@ -79,7 +79,7 @@ AdaBoost::AdaBoost( const size_t numClasses, const size_t maxIterations, const typename MatType::elem_type tol, - WeakLearnerArgs&... weakLearnerArgs) : + WeakLearnerArgs&&... weakLearnerArgs) : maxIterations(maxIterations), tolerance(tol) { From 1e5375edb16f66c4b7b98567849104c32cd94675 Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Mon, 13 Nov 2023 16:54:00 -0500 Subject: [PATCH 25/47] Deprecate versions where a weak learner is passed. --- src/mlpack/methods/adaboost/adaboost.hpp | 12 ++++++++++-- src/mlpack/methods/adaboost/adaboost_impl.hpp | 7 +++++-- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/src/mlpack/methods/adaboost/adaboost.hpp b/src/mlpack/methods/adaboost/adaboost.hpp index 7bbb5cbbf1..72d21d6031 100644 --- a/src/mlpack/methods/adaboost/adaboost.hpp +++ b/src/mlpack/methods/adaboost/adaboost.hpp @@ -122,12 +122,17 @@ class AdaBoost * @param tolerance The tolerance for change in values of rt. * @param other Weak learner that has already been initialized. */ + template + mlpack_deprecated /* to be removed in mlpack 5.0.0 */ AdaBoost(const MatType& data, const arma::Row& labels, const size_t numClasses, - const WeakLearnerType& other, + const WeakLearnerInType& other, const size_t maxIterations = 100, - const ElemType tolerance = 1e-6); + const ElemType tolerance = 1e-6, + const typename std::enable_if< + std::is_same::value + >::type* = 0); //! Get the maximum number of weak learners allowed in the model. size_t MaxIterations() const { return maxIterations; } @@ -175,6 +180,7 @@ class AdaBoost * @return The upper bound for training error. */ template + mlpack_deprecated /* to be removed in mlpack 5.0.0 */ ElemType Train( const MatType& data, const arma::Row& labels, @@ -185,6 +191,7 @@ class AdaBoost std::is_same::value>::type* = 0); template + mlpack_deprecated /* to be removed in mlpack 5.0.0 */ ElemType Train( const MatType& data, const arma::Row& labels, @@ -196,6 +203,7 @@ class AdaBoost std::is_same::value>::type* = 0); template + mlpack_deprecated /* to be removed in mlpack 5.0.0 */ ElemType Train( const MatType& data, const arma::Row& labels, diff --git a/src/mlpack/methods/adaboost/adaboost_impl.hpp b/src/mlpack/methods/adaboost/adaboost_impl.hpp index 499da6006f..595dc8956a 100644 --- a/src/mlpack/methods/adaboost/adaboost_impl.hpp +++ b/src/mlpack/methods/adaboost/adaboost_impl.hpp @@ -49,13 +49,16 @@ AdaBoost::AdaBoost(const ElemType tolerance) : * @param other Weak Learner, which has been initialized already. */ template +template AdaBoost::AdaBoost( const MatType& data, const arma::Row& labels, const size_t numClasses, - const WeakLearnerType& other, + const WeakLearnerInType& other, const size_t maxIterations, - const typename MatType::elem_type tol) : + const typename MatType::elem_type tol, + const typename std::enable_if< + std::is_same::value>::type*) : maxIterations(maxIterations), tolerance(tol) { From 27b9bb335ec8eb5a3a9f6d8d01ccfb9b76fea60f Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Mon, 13 Nov 2023 16:55:20 -0500 Subject: [PATCH 26/47] Don't use deprecated functions in tests; use new versions. --- src/mlpack/tests/adaboost_test.cpp | 142 +++++++++++------------------ 1 file changed, 55 insertions(+), 87 deletions(-) diff --git a/src/mlpack/tests/adaboost_test.cpp b/src/mlpack/tests/adaboost_test.cpp index 208de91900..8bbbd044e8 100644 --- a/src/mlpack/tests/adaboost_test.cpp +++ b/src/mlpack/tests/adaboost_test.cpp @@ -19,8 +19,6 @@ using namespace arma; using namespace mlpack; -// TODO: adapt to non-deprecated calls - /** * This test case runs the AdaBoost.mh algorithm on the UCI Iris dataset. It * checks whether the hamming loss breaches the upperbound, which is provided by @@ -46,16 +44,14 @@ TEMPLATE_TEST_CASE("HammingLossBoundIris", "[AdaBoostTest]", mat, fmat) // Run the perceptron for perceptronIter iterations. int perceptronIter = 400; - typedef Perceptron - PerceptronType; - PerceptronType p(inputData, labels.row(0), numClasses, perceptronIter); - // Define parameters for AdaBoost. size_t iterations = 100; - eT tolerance = 1e-10; - AdaBoost a(tolerance); - eT ztProduct = a.Train(inputData, labels.row(0), numClasses, p, iterations, - tolerance); + eT tolerance = 2e-10; + typedef Perceptron + PerceptronType; + AdaBoost a; + eT ztProduct = a.Train(inputData, labels.row(0), numClasses, iterations, + tolerance, perceptronIter); Row predictedLabels; a.Classify(inputData, predictedLabels); @@ -79,12 +75,10 @@ TEMPLATE_TEST_CASE("WeakLearnerErrorIris", "[AdaBoostTest]", mat, fmat) typedef typename MatType::elem_type eT; MatType inputData; - if (!data::Load("iris.csv", inputData)) FAIL("Cannot load test dataset iris.csv!"); Mat labels; - if (!data::Load("iris_labels.txt", labels)) FAIL("Cannot load labels for iris iris_labels.txt"); @@ -106,13 +100,13 @@ TEMPLATE_TEST_CASE("WeakLearnerErrorIris", "[AdaBoostTest]", mat, fmat) // Define parameters for AdaBoost. size_t iterations = 100; eT tolerance = 1e-10; - AdaBoost a(inputData, labels.row(0), numClasses, p, - iterations, tolerance); + AdaBoost a(inputData, labels.row(0), numClasses, + iterations, tolerance, perceptronIter); Row predictedLabels; a.Classify(inputData, predictedLabels); - size_t countError = accu(labels != predictedLabels);; + size_t countError = accu(labels != predictedLabels); eT error = (eT) countError / labels.n_cols; REQUIRE(error <= weakLearnerErrorRate + 0.03); @@ -150,8 +144,8 @@ TEMPLATE_TEST_CASE("HammingLossBoundVertebralColumn", "[AdaBoostTest]", mat, size_t iterations = 50; eT tolerance = 1e-10; AdaBoost a(tolerance); - eT ztProduct = a.Train(inputData, labels.row(0), numClasses, p, iterations, - tolerance); + eT ztProduct = a.Train(inputData, labels.row(0), numClasses, iterations, + tolerance, perceptronIter); Row predictedLabels; a.Classify(inputData, predictedLabels); @@ -201,8 +195,8 @@ TEMPLATE_TEST_CASE("WeakLearnerErrorVertebralColumn", "[AdaBoostTest]", mat, // Define parameters for AdaBoost. size_t iterations = 50; eT tolerance = 1e-10; - AdaBoost a(inputData, labels.row(0), numClasses, p, - iterations, tolerance); + AdaBoost a(inputData, labels.row(0), numClasses, + iterations, tolerance, perceptronIter); Row predictedLabels; a.Classify(inputData, predictedLabels); @@ -245,8 +239,8 @@ TEMPLATE_TEST_CASE("HammingLossBoundNonLinearSepData", "[AdaBoostTest]", mat, size_t iterations = 50; eT tolerance = 1e-10; AdaBoost a(tolerance); - eT ztProduct = a.Train(inputData, labels.row(0), numClasses, p, iterations, - tolerance); + eT ztProduct = a.Train(inputData, labels.row(0), numClasses, iterations, + tolerance, perceptronIter); Row predictedLabels; a.Classify(inputData, predictedLabels); @@ -296,8 +290,8 @@ TEMPLATE_TEST_CASE("WeakLearnerErrorNonLinearSepData", "[AdaBoostTest]", mat, // Define parameters for AdaBoost. size_t iterations = 50; eT tolerance = 1e-10; - AdaBoost a(inputData, labels.row(0), numClasses, p, - iterations, tolerance); + AdaBoost a(inputData, labels.row(0), numClasses, + iterations, tolerance, perceptronIter); Row predictedLabels; a.Classify(inputData, predictedLabels); @@ -330,14 +324,13 @@ TEMPLATE_TEST_CASE("HammingLossIris_DS", "[AdaBoostTest]", mat, fmat) const size_t numClasses = 3; const size_t inpBucketSize = 6; Row labelsvec = labels.row(0); - ID3DecisionStump ds(inputData, labelsvec, numClasses, inpBucketSize); // Define parameters for AdaBoost. size_t iterations = 50; eT tolerance = 1e-10; AdaBoost a(tolerance); - eT ztProduct = a.Train(inputData, labelsvec, numClasses, ds, iterations, - tolerance); + eT ztProduct = a.Train(inputData, labelsvec, numClasses, iterations, + tolerance, inpBucketSize); Row predictedLabels; a.Classify(inputData, predictedLabels); @@ -388,8 +381,8 @@ TEMPLATE_TEST_CASE("WeakLearnerErrorIris_DS", "[AdaBoostTest]", mat, fmat) size_t iterations = 50; eT tolerance = 1e-10; - AdaBoost a(inputData, labelsvec, numClasses, ds, - iterations, tolerance); + AdaBoost a(inputData, labelsvec, numClasses, + iterations, tolerance, inpBucketSize); Row predictedLabels; a.Classify(inputData, predictedLabels); @@ -432,8 +425,8 @@ TEMPLATE_TEST_CASE("HammingLossBoundVertebralColumn_DS", "[AdaBoostTest]", mat, eT tolerance = 1e-10; AdaBoost a(tolerance); - eT ztProduct = a.Train(inputData, labelsvec, numClasses, ds, iterations, - tolerance); + eT ztProduct = a.Train(inputData, labelsvec, numClasses, iterations, + tolerance, inpBucketSize); Row predictedLabels; a.Classify(inputData, predictedLabels); @@ -481,8 +474,8 @@ TEMPLATE_TEST_CASE("WeakLearnerErrorVertebralColumn_DS", "[AdaBoostTest]", mat, // Define parameters for AdaBoost. size_t iterations = 50; eT tolerance = 1e-10; - AdaBoost a(inputData, labelsvec, numClasses, ds, - iterations, tolerance); + AdaBoost a(inputData, labelsvec, numClasses, + iterations, tolerance, inpBucketSize); Row predictedLabels; a.Classify(inputData, predictedLabels); @@ -524,8 +517,8 @@ TEMPLATE_TEST_CASE("HammingLossBoundNonLinearSepData_DS", "[AdaBoostTest]", mat, eT tolerance = 1e-10; AdaBoost a(tolerance); - eT ztProduct = a.Train(inputData, labelsvec, numClasses, ds, iterations, - tolerance); + eT ztProduct = a.Train(inputData, labelsvec, numClasses, iterations, + tolerance, inpBucketSize); Row predictedLabels; a.Classify(inputData, predictedLabels); @@ -575,8 +568,8 @@ TEMPLATE_TEST_CASE("WeakLearnerErrorNonLinearSepData_DS", "[AdaBoostTest]", mat, size_t iterations = 500; eT tolerance = 1e-23; - AdaBoost a(inputData, labelsvec, numClasses, ds, - iterations, tolerance); + AdaBoost a(inputData, labelsvec, numClasses, + iterations, tolerance, inpBucketSize); Row predictedLabels; a.Classify(inputData, predictedLabels); @@ -629,8 +622,8 @@ TEMPLATE_TEST_CASE("ClassifyTest_VERTEBRALCOL", "[AdaBoostTest]", mat, fmat) // Define parameters for AdaBoost. size_t iterations = 100; eT tolerance = 1e-10; - AdaBoost a(inputData, labels.row(0), numClasses, p, - iterations, tolerance); + AdaBoost a(inputData, labels.row(0), numClasses, + iterations, tolerance, perceptronIter); Row predictedLabels1(testData.n_cols), predictedLabels2(testData.n_cols); @@ -700,8 +693,8 @@ TEMPLATE_TEST_CASE("ClassifyTest_NONLINSEP", "[AdaBoostTest]", mat, fmat) // Define parameters for AdaBoost. size_t iterations = 50; eT tolerance = 1e-10; - AdaBoost a(inputData, labelsvec, numClasses, ds, - iterations, tolerance); + AdaBoost a(inputData, labelsvec, numClasses, + iterations, tolerance, inpBucketSize); Row predictedLabels1(testData.n_cols), predictedLabels2(testData.n_cols); @@ -762,8 +755,8 @@ TEMPLATE_TEST_CASE("ClassifyTest_IRIS", "[AdaBoostTest]", mat, fmat) // Define parameters for AdaBoost. size_t iterations = 50; eT tolerance = 1e-10; - AdaBoost a(inputData, labels.row(0), numClasses, p, - iterations, tolerance); + AdaBoost a(inputData, labels.row(0), numClasses, + iterations, tolerance, perceptronIter); MatType testData; if (!data::Load("iris_test.csv", testData)) @@ -832,8 +825,8 @@ TEMPLATE_TEST_CASE("TrainTest", "[AdaBoostTest]", mat, fmat) // Now train AdaBoost. size_t iterations = 50; eT tolerance = 1e-10; - AdaBoost a(inputData, labels.row(0), numClasses, p, - iterations, tolerance); + AdaBoost a(inputData, labels.row(0), numClasses, + iterations, tolerance, perceptronIter); // Now load another dataset... if (!data::Load("vc2.csv", inputData)) @@ -845,7 +838,8 @@ TEMPLATE_TEST_CASE("TrainTest", "[AdaBoostTest]", mat, fmat) PerceptronType p2(inputData, labels.row(0), newNumClasses, perceptronIter); - a.Train(inputData, labels.row(0), newNumClasses, p2, iterations, tolerance); + a.Train(inputData, labels.row(0), newNumClasses, iterations, tolerance, + perceptronIter); // Load test set to see if it trained on vc2 correctly. MatType testData; @@ -860,7 +854,7 @@ TEMPLATE_TEST_CASE("TrainTest", "[AdaBoostTest]", mat, fmat) Row predictedLabels(testData.n_cols); a.Classify(testData, predictedLabels); - int localError = accu(trueTestLabels != predictedLabels); + size_t localError = accu(trueTestLabels != predictedLabels); eT lError = (eT) localError / trueTestLabels.n_cols; REQUIRE(lError <= 0.30); @@ -880,8 +874,7 @@ TEMPLATE_TEST_CASE("PerceptronSerializationTest", "[AdaBoostTest]", fmat, mat) typedef Perceptron PerceptronType; - PerceptronType p(data, labels, 2, 800); - AdaBoost ab(data, labels, 2, p, 50, 1e-10); + AdaBoost ab(data, labels, 2, 50, 1e-10, 800); // Now create another dataset to train with. MatType otherData = randu(5, 200); @@ -893,9 +886,8 @@ TEMPLATE_TEST_CASE("PerceptronSerializationTest", "[AdaBoostTest]", fmat, mat) for (size_t i = 150; i < 200; ++i) otherLabels[i] = 2; - PerceptronType p2(otherData, otherLabels, 3, 500); - AdaBoost abText(otherData, otherLabels, 3, p2, 50, - 1e-10); + AdaBoost abText(otherData, otherLabels, 3, 50, 1e-10, + 500); AdaBoost abXml, abBinary; @@ -937,8 +929,7 @@ TEMPLATE_TEST_CASE("ID3DecisionStumpSerializationTest", "[AdaBoostTest]", mat, for (size_t i = 250; i < 500; ++i) labels[i] = 1; - ID3DecisionStump p(data, labels, 2, 800); - AdaBoost ab(data, labels, 2, p, 50, 1e-10); + AdaBoost ab(data, labels, 2, 50, 1e-10, 40); // Now create another dataset to train with. MatType otherData = randu(5, 200); @@ -950,9 +941,8 @@ TEMPLATE_TEST_CASE("ID3DecisionStumpSerializationTest", "[AdaBoostTest]", mat, for (size_t i = 150; i < 200; ++i) otherLabels[i] = 2; - ID3DecisionStump p2(otherData, otherLabels, 3, 500); - AdaBoost abText(otherData, otherLabels, 3, p2, 50, - 1e-10); + AdaBoost abText(otherData, otherLabels, 3, 50, + 1e-10, 25); AdaBoost abXml, abBinary; @@ -970,7 +960,7 @@ TEMPLATE_TEST_CASE("ID3DecisionStumpSerializationTest", "[AdaBoostTest]", mat, for (size_t i = 0; i < ab.WeakLearners(); ++i) { REQUIRE(ab.WeakLearner(i).SplitDimension() == - abXml.WeakLearner(i).SplitDimension()); + abXml.WeakLearner(i).SplitDimension()); REQUIRE(ab.WeakLearner(i).SplitDimension() == abText.WeakLearner(i).SplitDimension()); REQUIRE(ab.WeakLearner(i).SplitDimension() == @@ -1034,7 +1024,6 @@ TEMPLATE_TEST_CASE("AdaBoostSinglePointClassifyWithProbs", "[AdaBoostTest]", TEMPLATE_TEST_CASE("AdaBoostParamsConstructor", "[AdaBoostTest]", fmat, mat) { typedef TestType MatType; - typedef typename MatType::elem_type ElemType; MatType inputData; if (!data::Load("iris.csv", inputData)) @@ -1073,7 +1062,6 @@ TEMPLATE_TEST_CASE("AdaBoostParamsConstructor", "[AdaBoostTest]", fmat, mat) TEMPLATE_TEST_CASE("AdaBoostTrainOverloads", "[AdaBoostTest]", fmat, mat) { typedef TestType MatType; - typedef typename MatType::elem_type ElemType; // Create random data. MatType data = randu(10, 100); @@ -1082,49 +1070,32 @@ TEMPLATE_TEST_CASE("AdaBoostTrainOverloads", "[AdaBoostTest]", fmat, mat) typedef Perceptron PerceptronType; - PerceptronType p; // For versions that take an initialized weak learner. - p.MaxIterations() = 150; - AdaBoost a1, a2, a3, a4, a5, a6, a7; - a1.MaxIterations() = 75; - a4.MaxIterations() = 65; - a1.Tolerance() = 1e-4; - a2.Tolerance() = 1e-5; - a4.Tolerance() = 2e-4; - a5.Tolerance() = 2e-5; + AdaBoost a1, a2, a3, a4; + a1.MaxIterations() = 65; + a1.Tolerance() = 2e-4; + a2.Tolerance() = 2e-5; - a1.Train(data, labels, 4, p); - a2.Train(data, labels, 4, p, 10); - a3.Train(data, labels, 4, p, 50, 1e-3); - a4.Train(data, labels, 4); - a5.Train(data, labels, 4, 15); - a6.Train(data, labels, 4, 55, 1e-3); - a7.Train(data, labels, 4, 60, 2e-3, 100); + a1.Train(data, labels, 4); + a2.Train(data, labels, 4, 15); + a3.Train(data, labels, 4, 55, 1e-3); + a4.Train(data, labels, 4, 60, 2e-3, 100); // Make sure hyperparameters were set correctly, where appropriate. REQUIRE(a1.MaxIterations() == 75); REQUIRE(a2.MaxIterations() == 10); REQUIRE(a3.MaxIterations() == 50); REQUIRE(a4.MaxIterations() == 65); - REQUIRE(a5.MaxIterations() == 15); - REQUIRE(a6.MaxIterations() == 55); - REQUIRE(a7.MaxIterations() == 60); REQUIRE(a1.Tolerance() == Approx(1e-4)); REQUIRE(a2.Tolerance() == Approx(1e-5)); REQUIRE(a3.Tolerance() == Approx(1e-3)); REQUIRE(a4.Tolerance() == Approx(2e-4)); - REQUIRE(a5.Tolerance() == Approx(2e-5)); - REQUIRE(a6.Tolerance() == Approx(1e-3)); - REQUIRE(a7.Tolerance() == Approx(2e-3)); // Make sure anything at all was trained. REQUIRE(a1.WeakLearners() > 0); REQUIRE(a2.WeakLearners() > 0); REQUIRE(a3.WeakLearners() > 0); REQUIRE(a4.WeakLearners() > 0); - REQUIRE(a5.WeakLearners() > 0); - REQUIRE(a6.WeakLearners() > 0); - REQUIRE(a7.WeakLearners() > 0); // Make sure the maximum number of iterations in the perceptron was set // properly. @@ -1132,7 +1103,4 @@ TEMPLATE_TEST_CASE("AdaBoostTrainOverloads", "[AdaBoostTest]", fmat, mat) REQUIRE(a2.WeakLearner(0).MaxIterations() == 150); REQUIRE(a3.WeakLearner(0).MaxIterations() == 150); REQUIRE(a4.WeakLearner(0).MaxIterations() == 1000); - REQUIRE(a5.WeakLearner(0).MaxIterations() == 1000); - REQUIRE(a6.WeakLearner(0).MaxIterations() == 1000); - REQUIRE(a7.WeakLearner(0).MaxIterations() == 100); } From 6989814119e4bd3093b9ba0a6a2ae5e156377ae5 Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Mon, 13 Nov 2023 16:55:49 -0500 Subject: [PATCH 27/47] Don't use deprecated functions in bindings. --- src/mlpack/methods/adaboost/adaboost_model_impl.hpp | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/mlpack/methods/adaboost/adaboost_model_impl.hpp b/src/mlpack/methods/adaboost/adaboost_model_impl.hpp index e12dc39694..7a99e13f36 100644 --- a/src/mlpack/methods/adaboost/adaboost_model_impl.hpp +++ b/src/mlpack/methods/adaboost/adaboost_model_impl.hpp @@ -124,15 +124,13 @@ inline void AdaBoostModel::Train(const arma::mat& data, if (weakLearnerType == WeakLearnerTypes::DECISION_STUMP) { delete dsBoost; - ID3DecisionStump ds(data, labels, max(labels) + 1); - dsBoost = new AdaBoost(data, labels, numClasses, ds, + dsBoost = new AdaBoost(data, labels, numClasses, iterations, tolerance); } else if (weakLearnerType == WeakLearnerTypes::PERCEPTRON) { delete pBoost; - Perceptron<> p(data, labels, max(labels) + 1); - pBoost = new AdaBoost>(data, labels, numClasses, p, iterations, + pBoost = new AdaBoost>(data, labels, numClasses, iterations, tolerance); } } From a388e56294e9c0ca49a41c4ce05c2b7039677bad Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Mon, 13 Nov 2023 17:02:02 -0500 Subject: [PATCH 28/47] Fix test values. --- src/mlpack/tests/adaboost_test.cpp | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/src/mlpack/tests/adaboost_test.cpp b/src/mlpack/tests/adaboost_test.cpp index 8bbbd044e8..6c4e3abe94 100644 --- a/src/mlpack/tests/adaboost_test.cpp +++ b/src/mlpack/tests/adaboost_test.cpp @@ -1081,15 +1081,15 @@ TEMPLATE_TEST_CASE("AdaBoostTrainOverloads", "[AdaBoostTest]", fmat, mat) a4.Train(data, labels, 4, 60, 2e-3, 100); // Make sure hyperparameters were set correctly, where appropriate. - REQUIRE(a1.MaxIterations() == 75); - REQUIRE(a2.MaxIterations() == 10); - REQUIRE(a3.MaxIterations() == 50); - REQUIRE(a4.MaxIterations() == 65); + REQUIRE(a1.MaxIterations() == 65); + REQUIRE(a2.MaxIterations() == 15); + REQUIRE(a3.MaxIterations() == 55); + REQUIRE(a4.MaxIterations() == 60); - REQUIRE(a1.Tolerance() == Approx(1e-4)); - REQUIRE(a2.Tolerance() == Approx(1e-5)); + REQUIRE(a1.Tolerance() == Approx(2e-4)); + REQUIRE(a2.Tolerance() == Approx(2e-5)); REQUIRE(a3.Tolerance() == Approx(1e-3)); - REQUIRE(a4.Tolerance() == Approx(2e-4)); + REQUIRE(a4.Tolerance() == Approx(2e-3)); // Make sure anything at all was trained. REQUIRE(a1.WeakLearners() > 0); @@ -1099,8 +1099,8 @@ TEMPLATE_TEST_CASE("AdaBoostTrainOverloads", "[AdaBoostTest]", fmat, mat) // Make sure the maximum number of iterations in the perceptron was set // properly. - REQUIRE(a1.WeakLearner(0).MaxIterations() == 150); - REQUIRE(a2.WeakLearner(0).MaxIterations() == 150); - REQUIRE(a3.WeakLearner(0).MaxIterations() == 150); - REQUIRE(a4.WeakLearner(0).MaxIterations() == 1000); + REQUIRE(a1.WeakLearner(0).MaxIterations() == 1000); + REQUIRE(a2.WeakLearner(0).MaxIterations() == 1000); + REQUIRE(a3.WeakLearner(0).MaxIterations() == 1000); + REQUIRE(a4.WeakLearner(0).MaxIterations() == 100); } From 1250bd8ee62a7dc74fea46f3e303802786406a8f Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Mon, 13 Nov 2023 17:06:12 -0500 Subject: [PATCH 29/47] Fix small rendering issues. --- doc/user/methods/adaboost.md | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/doc/user/methods/adaboost.md b/doc/user/methods/adaboost.md index 01ac225304..1380fb676b 100644 --- a/doc/user/methods/adaboost.md +++ b/doc/user/methods/adaboost.md @@ -69,8 +69,7 @@ std::cout << arma::accu(predictions == 3) << " test points classified as class " --- - * `ab = AdaBoost(data, labels, numClasses, maxIterations=100, tolerance=1e-6, [weak - learner hyperparameters...])` + * `ab = AdaBoost(data, labels, numClasses, maxIterations=100, tolerance=1e-6, _[weak learner hyperparameters...]_)` - Train model with custom weak learner hyperparameters. - Hyperparameters for the weak learner are any arguments to the weak learner's `Train()` function that come after `numClasses` or `weights`. @@ -97,9 +96,7 @@ std::cout << arma::accu(predictions == 3) << " test points classified as class " hyperparameters will be used as settings for weak learners during training. | _(N/A)_ | | `maxIterations` | `size_t` | Maximum number of iterations of AdaBoost.MH to use. This is the maximum number of weak learners to train. (0 means no limit, and weak learners will be trained until the tolerance is met.) | `100` | -| `tolerance` | `double` | When the weighted residual (`r_t`) of the model goes -below `tolerance`, training will terminate and no more weak learners will be -added. | `1e-6` | +| `tolerance` | `double` | When the weighted residual (`r_t`) of the model goes below `tolerance`, training will terminate and no more weak learners will be added. | `1e-6` | As an alternative to passing hyperparameters, each hyperparameter can be set with a standalone method. The following functions can be used before calling From 7769b58e87cf72ed324940a29c300ea7e2446f73 Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Mon, 13 Nov 2023 17:07:59 -0500 Subject: [PATCH 30/47] Add quick descriptions of template types. --- doc/user/methods/adaboost.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/doc/user/methods/adaboost.md b/doc/user/methods/adaboost.md index 1380fb676b..4016131eb9 100644 --- a/doc/user/methods/adaboost.md +++ b/doc/user/methods/adaboost.md @@ -316,6 +316,12 @@ behavior. The full signature of the class is: AdaBoost ``` + * `WeakLearnerType`: the weak classifier to ensemble in the AdaBoost model. + * `MatType`: specifies the type of matrix used for learning and internal + representation of model parameters. + +--- + #### `WeakLearnerType` From 7480582e307357f2445d79d22f7922525197b5ef Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Mon, 13 Nov 2023 17:08:49 -0500 Subject: [PATCH 31/47] Add a separator. --- doc/user/methods/adaboost.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/user/methods/adaboost.md b/doc/user/methods/adaboost.md index 4016131eb9..3f273ec8d8 100644 --- a/doc/user/methods/adaboost.md +++ b/doc/user/methods/adaboost.md @@ -377,6 +377,8 @@ class CustomWeakLearner }; ``` +--- + #### `MatType` * Specifies the matrix type to use for data when learning a model (or From dd91888b57183505bb5c690665684c4b756c7c9b Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Wed, 15 Nov 2023 08:53:36 -0500 Subject: [PATCH 32/47] Correct handling for older cereal versions. --- src/mlpack/core/cereal/template_class_version.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mlpack/core/cereal/template_class_version.hpp b/src/mlpack/core/cereal/template_class_version.hpp index 6789de5e39..ea9e508964 100644 --- a/src/mlpack/core/cereal/template_class_version.hpp +++ b/src/mlpack/core/cereal/template_class_version.hpp @@ -39,7 +39,7 @@ struct Version \ \ static inline const std::uint32_t version = registerVersion(); \ \ - CEREAL_UNUSED_FUNCTION \ + static void unused() { (void) version; } \ }; /* end Version */ \ \ } \ From 1c82d8bac78ba5edae0406e9fd43e18cd8e69223 Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Wed, 15 Nov 2023 08:54:18 -0500 Subject: [PATCH 33/47] Update HISTORY.md. --- HISTORY.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/HISTORY.md b/HISTORY.md index 1a1b26c19b..3fc2ff8006 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -11,6 +11,8 @@ * Fix setting number of classes correctly in `SoftmaxRegression::Train()` (#3553). + * Allow passing weak learner hyperparameters directly to AdaBoost (#3560). + ### mlpack 4.2.1 ###### 2023-09-05 * Reinforcement Learning: Gaussian noise (#3515). From bcd51d2f1e03b0ddca0df20b9f6a9ebe457e73d3 Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Wed, 15 Nov 2023 08:55:36 -0500 Subject: [PATCH 34/47] Fix other use of cereal macro not available in older versions. --- src/mlpack/core/cereal/template_class_version.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mlpack/core/cereal/template_class_version.hpp b/src/mlpack/core/cereal/template_class_version.hpp index ea9e508964..bb68af0705 100644 --- a/src/mlpack/core/cereal/template_class_version.hpp +++ b/src/mlpack/core/cereal/template_class_version.hpp @@ -63,7 +63,7 @@ struct Version \ return CEREAL_UNPACK VERSION_NUMBER; \ } \ \ - CEREAL_UNUSED_FUNCTION \ + static void unused() { (void) version; } \ }; /* end Version */ \ \ template \ From 652aabea19cb54acef70ea3f2957313ed7fda27d Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Mon, 13 Nov 2023 18:39:55 -0500 Subject: [PATCH 35/47] Fix compilation issues and make sure examples work. --- doc/user/methods/adaboost.md | 40 +++++++++++++++++++++------------- doc/user/methods/perceptron.md | 26 +++++++++++----------- 2 files changed, 38 insertions(+), 28 deletions(-) diff --git a/doc/user/methods/adaboost.md b/doc/user/methods/adaboost.md index 3f273ec8d8..31192ec338 100644 --- a/doc/user/methods/adaboost.md +++ b/doc/user/methods/adaboost.md @@ -22,7 +22,7 @@ arma::Row labels = arma::randi>(1000, arma::distr_param(0, 4)); arma::mat testDataset(10, 500, arma::fill::randu); // 500 test points. -AdaBoost<> ab; // Step 1: create model. +AdaBoost ab; // Step 1: create model. ab.Train(dataset, labels, 5); // Step 2: train model. arma::Row predictions; ab.Classify(testDataset, predictions); // Step 3: classify points. @@ -231,14 +231,14 @@ arma::mat dataset; data::Load("iris.csv", dataset, true); // See https://datasets.mlpack.org/iris.labels.csv. arma::Row labels; -data::Load("iris.labels.csv", dataset, true); +data::Load("iris.labels.csv", labels, true); -// Create a weak learner with the desired hyperparameters. -Perceptron<> p; -p.MaxIterations() = 500; // We'll use a custom maximum number of iterations. - -AdaBoost<> ab; -ab.Train(dataset, labels, 3, p); +AdaBoost ab; +// Train with a custom number of perceptron iterations, and custom AdaBoost +// parameters. +ab.Train(dataset, labels, 3, 75 /* maximum number of weak learners */, + 1e-6 /* tolerance for AdaBoost convergence */, + 100 /* maximum number of perceptron iterations */); // Now predict the label of a point and the probabilities of each class. size_t prediction; @@ -261,9 +261,9 @@ arma::mat dataset; data::Load("iris.csv", dataset, true); // See https://datasets.mlpack.org/iris.labels.csv. arma::Row labels; -data::Load("iris_labels.csv", dataset, true); +data::Load("iris.labels.csv", dataset, true); -AdaBoost<> ab; +AdaBoost ab; ab.MaxIterations() = 50; // Use at most 50 weak learners. ab.Tolerance() = 1e-4; // Set a custom tolerance for convergence. @@ -280,7 +280,7 @@ Load an AdaBoost model and print some information about it. ```c++ // Load a saved model named "adaboost_model" from `adaboost_model.bin`. -AdaBoost<> ab; +AdaBoost ab; data::Load("adaboost_model.bin", "adaboost_model", ab, true); std::cout << "Details about the model in `adaboost_model.bin`:" << std::endl; @@ -400,8 +400,16 @@ arma::Row labels = arma::randi>(1000, arma::distr_param(0, 4)); // Train in the constructor. -// Note that we specify decision stumps as the weak learner type. -AdaBoost ab(dataset, labels, 5); +// Note that we specify decision stumps as the weak learner type, and pass +// hyperparameters for the decision stump (these could be omitted). See the +// DecisionTree documentation for more details on the ID3DecisionStump-specific +// hyperparameters. +AdaBoost ab(dataset, labels, 5, + 25 /* maximum number of decision stumps */, + 1e-6 /* tolerance for convergence of AdaBoost */, + /** Hyperparameters specific to ID3DecisionStump: **/ + 10 /* minimum number of points in each leaf of the decision stump */, + 1e-5 /* minimum gain for splitting the root node of the decision stump */); // Create test data (500 points). arma::mat testDataset(10, 500, arma::fill::randu); @@ -427,8 +435,10 @@ arma::Row labels = arma::randi>(1000, arma::distr_param(0, 4)); // Train in the constructor, using floating-point data. -// (TODO: do we have to explicitly write MatType?) -AdaBoost<> ab(dataset, labels, 5); +// The weak learner type is now a floating-point Perceptron. +typedef Perceptron + PerceptronType; +AdaBoost ab(dataset, labels, 5); // Create test data (500 points). arma::fmat testDataset(10, 500, arma::fill::randu); diff --git a/doc/user/methods/perceptron.md b/doc/user/methods/perceptron.md index 51d1dca40f..3e36b134f1 100644 --- a/doc/user/methods/perceptron.md +++ b/doc/user/methods/perceptron.md @@ -24,10 +24,10 @@ arma::Row labels = arma::randi>(1000, arma::distr_param(0, 4)); arma::mat testDataset(10, 500, arma::fill::randu); // 500 test points. -Perceptron<> p; // Step 1: create model. -p.Train(dataset, labels, 5); // Step 2: train model. +Perceptron p; // Step 1: create model. +p.Train(dataset, labels, 5); // Step 2: train model. arma::Row predictions; -tree.Classify(testDataset, predictions); // Step 3: classify points. +p.Classify(testDataset, predictions); // Step 3: classify points. // Print some information about the test predictions. std::cout << arma::accu(predictions == 1) << " test points classified as class " @@ -211,27 +211,27 @@ arma::mat dataset; data::Load("iris.csv", dataset, true); // See https://datasets.mlpack.org/iris.labels.csv. arma::Row labels; -data::Load("iris.labels.csv", dataset, true); +data::Load("iris.labels.csv", labels, true); // Create a Perceptron object. -Perceptron<> p; +Perceptron p; // Set the maximum number of iterations to 100. (This can also be done in the // constructor.) p.MaxIterations() = 100; // Train the model for up to 100 iterations. -p.Train(data, labels, 3); +p.Train(dataset, labels, 3); // Now, compute and print accuracy on the training set. arma::Row predictions; -p.Classify(data, predictions); +p.Classify(dataset, predictions); std::cout << "Training set accuracy after 100 iterations: " << (100.0 * double(arma::accu(labels == predictions)) / labels.n_elem) << "\%." << std::endl; // Train for another 250 iterations and compute training set accuracy again. -p.Train(data, labels, 3, 250); -p.Classify(data, predictions); +p.Train(dataset, labels, 3, 250); +p.Classify(dataset, predictions); std::cout << "Training set accuracy after 350 iterations: " << (100.0 * double(arma::accu(labels == predictions)) / labels.n_elem) << "\%." << std::endl; @@ -242,7 +242,7 @@ std::cout << "Training set accuracy after 350 iterations: " Load a saved perceptron from disk and print information about it. ```c++ -Perceptron<> p; +Perceptron p; // This call assumes a perceptron called "p" has already been saved to // `perceptron.bin` with `data::Save()`. data::Load("perceptron.bin", "p", p, true); @@ -382,7 +382,7 @@ Perceptron p(dataset, // Create test data (500 points). arma::mat testDataset(10, 500, arma::fill::randu); arma::Row predictions; -tree.Classify(testDataset, predictions); +p.Classify(testDataset, predictions); // Now `predictions` holds predictions for the test dataset. // Print some information about the test predictions. @@ -404,13 +404,13 @@ arma::Row labels = arma::randi>(1000, arma::distr_param(0, 4)); // Train in the constructor. -Perceptron<> p(dataset, labels, 5); +Perceptron p(dataset, labels, 5); // Create test data (500 points). arma::sp_fmat testDataset; testDataset.sprandu(100, 500, 0.01); arma::Row predictions; -tree.Classify(testDataset, predictions); +p.Classify(testDataset, predictions); // Now `predictions` holds predictions for the test dataset. // Print some information about the test predictions. From fdeaade0f24a5c458df25612191da0db11032759 Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Wed, 18 Oct 2023 20:29:52 -0400 Subject: [PATCH 36/47] Standardize capitalization in h3 headers. --- doc/user/methods/decision_tree.md | 4 ++-- doc/user/methods/decision_tree_regressor.md | 4 ++-- doc/user/methods/random_forest.md | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/doc/user/methods/decision_tree.md b/doc/user/methods/decision_tree.md index 8147dbfd40..da85ce3be8 100644 --- a/doc/user/methods/decision_tree.md +++ b/doc/user/methods/decision_tree.md @@ -217,7 +217,7 @@ to make class predictions for new data. Defaults and types are detailed in the `arma::fmat`, `arma::sp_mat`, `arma::sp_vec`, etc.). However, the element type that is used should be the same type that was used for training. -### Other functionality +### Other Functionality @@ -240,7 +240,7 @@ For complete functionality, the [source code](/src/mlpack/methods/decision_tree/decision_tree.hpp) can be consulted. Each method is fully documented. -### Simple examples +### Simple Examples Train a decision tree on random numeric data and predict labels on a test set: diff --git a/doc/user/methods/decision_tree_regressor.md b/doc/user/methods/decision_tree_regressor.md index f9de527d89..e6c5c98756 100644 --- a/doc/user/methods/decision_tree_regressor.md +++ b/doc/user/methods/decision_tree_regressor.md @@ -191,7 +191,7 @@ in the [Prediction Parameters](#prediction-parameters) section below. `arma::fmat`, `arma::sp_mat`, `arma::sp_vec`, etc.). However, the element type that is used should be the same type that was used for training. -### Other functionality +### Other Functionality @@ -214,7 +214,7 @@ For complete functionality, the [source code](/src/mlpack/methods/decision_tree/decision_tree_regressor.hpp) can be consulted. Each method is fully documented. -### Simple examples +### Simple Examples Train a decision tree regressor on random numeric data and make predictions on a test set: diff --git a/doc/user/methods/random_forest.md b/doc/user/methods/random_forest.md index 4e5267ea1d..f9646cb773 100644 --- a/doc/user/methods/random_forest.md +++ b/doc/user/methods/random_forest.md @@ -268,7 +268,7 @@ to make class predictions for new data. Defaults and types are detailed in the `arma::fmat`, `arma::sp_mat`, `arma::sp_vec`, etc.). However, the element type that is used should be the same type that was used for training. -### Other functionality +### Other Functionality @@ -285,7 +285,7 @@ For complete functionality, the [source code](/src/mlpack/methods/random_forest/random_forest.hpp) can be consulted. Each method is fully documented. -### Simple examples +### Simple Examples Train a random forest on random numeric data and predict labels on a test set: From dc7d684910e67e8ac2a9d44d74366d941f7e0378 Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Wed, 18 Oct 2023 20:37:02 -0400 Subject: [PATCH 37/47] No need to specify the number of classes. --- doc/user/methods/decision_tree.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/user/methods/decision_tree.md b/doc/user/methods/decision_tree.md index da85ce3be8..1b25361a0a 100644 --- a/doc/user/methods/decision_tree.md +++ b/doc/user/methods/decision_tree.md @@ -12,7 +12,7 @@ Decision trees are useful for classifying points with _discrete labels_ (i.e. #### Basic usage example excerpt: ```c++ -DecisionTree tree(3); // Step 1: construct object. +DecisionTree tree; // Step 1: construct object. tree.Train(data, labels, 3); // Step 2: train model. tree.Classify(test_data, test_predictions); // Step 3: use model to classify. ``` From 92432d9ec8321ba3fa38a12519153220b77addd0 Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Wed, 18 Oct 2023 20:38:03 -0400 Subject: [PATCH 38/47] Match naming to the rest of mlpack. --- doc/user/methods/decision_tree.md | 6 +++--- doc/user/methods/decision_tree_regressor.md | 6 +++--- doc/user/methods/random_forest.md | 6 +++--- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/doc/user/methods/decision_tree.md b/doc/user/methods/decision_tree.md index 1b25361a0a..7860f632bb 100644 --- a/doc/user/methods/decision_tree.md +++ b/doc/user/methods/decision_tree.md @@ -12,9 +12,9 @@ Decision trees are useful for classifying points with _discrete labels_ (i.e. #### Basic usage example excerpt: ```c++ -DecisionTree tree; // Step 1: construct object. -tree.Train(data, labels, 3); // Step 2: train model. -tree.Classify(test_data, test_predictions); // Step 3: use model to classify. +DecisionTree tree; // Step 1: construct object. +tree.Train(data, labels, 3); // Step 2: train model. +tree.Classify(testData, testPredictions); // Step 3: use model to classify. ``` #### Quick links: diff --git a/doc/user/methods/decision_tree_regressor.md b/doc/user/methods/decision_tree_regressor.md index e6c5c98756..43411cd233 100644 --- a/doc/user/methods/decision_tree_regressor.md +++ b/doc/user/methods/decision_tree_regressor.md @@ -13,9 +13,9 @@ _continuous values_ (`0.3`, `1.2`, etc.). For predicting _discrete labels_ #### Basic usage example excerpt: ```c++ -DecisionTreeRegressor tree; // Step 1: construct object. -tree.Train(data, responses, 3); // Step 2: train model. -tree.Predict(test_data, test_predictions); // Step 3: predict values with model. +DecisionTreeRegressor tree; // Step 1: construct object. +tree.Train(data, responses, 3); // Step 2: train model. +tree.Predict(testData, testPredictions); // Step 3: predict values with model. ``` #### Quick links: diff --git a/doc/user/methods/random_forest.md b/doc/user/methods/random_forest.md index f9646cb773..fbb8f244d6 100644 --- a/doc/user/methods/random_forest.md +++ b/doc/user/methods/random_forest.md @@ -16,9 +16,9 @@ values_). #### Basic usage example excerpt: ```c++ -RandomForest rf; // Step 1: construct object. -rf.Train(data, labels, 3); // Step 2: train model. -rf.Classify(test_data, test_predictions); // Step 3: use model to classify. +RandomForest rf; // Step 1: construct object. +rf.Train(data, labels, 3); // Step 2: train model. +rf.Classify(testData, testPredictions); // Step 3: use model to classify. ``` #### Quick links: From 32dac3f0d90e69e8f6f95d4914b9af7fe52b9235 Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Thu, 19 Oct 2023 16:36:43 -0400 Subject: [PATCH 39/47] Fix name of parameter. --- doc/user/methods/decision_tree.md | 5 ++--- doc/user/methods/random_forest.md | 4 ++-- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/doc/user/methods/decision_tree.md b/doc/user/methods/decision_tree.md index 7860f632bb..5e4dca6011 100644 --- a/doc/user/methods/decision_tree.md +++ b/doc/user/methods/decision_tree.md @@ -160,7 +160,6 @@ Once a `DecisionTree` is trained, the `Classify()` member function can be used to make class predictions for new data. Defaults and types are detailed in the [Classification Parameters](#classification-parameters) section below. - #### Forms: * `size_t predictedClass = tree.Classify(point)` @@ -174,7 +173,7 @@ to make class predictions for new data. Defaults and types are detailed in the - Classify a single point and compute class probabilities. - The predicted class is stored in `prediction`. - The class probabilities are stored in `probabilities_vec`, which is set to - length `num_classes`. + length `numClasses`. - The probability of class `i` can be accessed with `probabilities_vec[i]`. --- @@ -195,7 +194,7 @@ to make class predictions for new data. Defaults and types are detailed in the set to length `data.n_cols`. - The prediction for data point `i` can be accessed with `predictions[i]`. - The class probabilities for each point are stored in `probabilities`, - which is set to size `num_classes` by `data.n_cols`. + which is set to size `numClasses` by `data.n_cols`. - The probability of class `j` for data point `i` can be accessed with `probabilities(j, i)`. diff --git a/doc/user/methods/random_forest.md b/doc/user/methods/random_forest.md index fbb8f244d6..110b392b6a 100644 --- a/doc/user/methods/random_forest.md +++ b/doc/user/methods/random_forest.md @@ -225,7 +225,7 @@ to make class predictions for new data. Defaults and types are detailed in the - Classify a single point and compute class probabilities. - The predicted class is stored in `prediction`. - The class probabilities are stored in `probabilities_vec`, which is set to - length `num_classes`. + length `numClasses`. - The probability of class `i` can be accessed with `probabilities_vec[i]`. --- @@ -246,7 +246,7 @@ to make class predictions for new data. Defaults and types are detailed in the set to length `data.n_cols`. - The prediction for data point `i` can be accessed with `predictions[i]`. - The class probabilities for each point are stored in `probabilities`, - which is set to size `num_classes` by `data.n_cols`. + which is set to size `numClasses` by `data.n_cols`. - The probability of class `j` for data point `i` can be accessed with `probabilities(j, i)`. From ab404c973397c9f2752558a0eb15669d23cd866a Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Thu, 19 Oct 2023 16:38:09 -0400 Subject: [PATCH 40/47] Fix plurality. --- doc/user/methods/decision_tree.md | 8 ++++---- doc/user/methods/random_forest.md | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/doc/user/methods/decision_tree.md b/doc/user/methods/decision_tree.md index 5e4dca6011..ec1070f17e 100644 --- a/doc/user/methods/decision_tree.md +++ b/doc/user/methods/decision_tree.md @@ -181,8 +181,8 @@ to make class predictions for new data. Defaults and types are detailed in the * `tree.Classify(data, predictions)` - ***(Multi-point)*** - Classify a set of points. - - The predicted classes of each point is stored in `predictions`, which is - set to length `data.n_cols`. + - The predicted class of each point is stored in `predictions`, which is set + to length `data.n_cols`. - The prediction for data point `i` can be accessed with `predictions[i]`. --- @@ -190,8 +190,8 @@ to make class predictions for new data. Defaults and types are detailed in the * `tree.Classify(data, predictions, probabilities)` - ***(Multi-point)*** - Classify a set of points and compute class probabilities for each point. - - The predicted classes of each point is stored in `predictions`, which is - set to length `data.n_cols`. + - The predicted class of each point is stored in `predictions`, which is set + to length `data.n_cols`. - The prediction for data point `i` can be accessed with `predictions[i]`. - The class probabilities for each point are stored in `probabilities`, which is set to size `numClasses` by `data.n_cols`. diff --git a/doc/user/methods/random_forest.md b/doc/user/methods/random_forest.md index 110b392b6a..27343d3938 100644 --- a/doc/user/methods/random_forest.md +++ b/doc/user/methods/random_forest.md @@ -233,8 +233,8 @@ to make class predictions for new data. Defaults and types are detailed in the * `rf.Classify(data, predictions)` - ***(Multi-point)*** - Classify a set of points. - - The predicted classes of each point is stored in `predictions`, which is - set to length `data.n_cols`. + - The predicted class of each point is stored in `predictions`, which is set + to length `data.n_cols`. - The prediction for data point `i` can be accessed with `predictions[i]`. --- @@ -242,8 +242,8 @@ to make class predictions for new data. Defaults and types are detailed in the * `rf.Classify(data, predictions, probabilities)` - ***(Multi-point)*** - Classify a set of points and compute class probabilities for each point. - - The predicted classes of each point is stored in `predictions`, which is - set to length `data.n_cols`. + - The predicted class of each point is stored in `predictions`, which is set + to length `data.n_cols`. - The prediction for data point `i` can be accessed with `predictions[i]`. - The class probabilities for each point are stored in `probabilities`, which is set to size `numClasses` by `data.n_cols`. From 084f5d756daa4a0f65d4c77664cb528785e19642 Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Tue, 7 Nov 2023 17:28:33 -0500 Subject: [PATCH 41/47] Handle feedback about documentation. --- doc/user/methods/decision_tree.md | 137 ++++------- doc/user/methods/decision_tree_regressor.md | 122 ++++------ doc/user/methods/random_forest.md | 237 ++++++++------------ 3 files changed, 184 insertions(+), 312 deletions(-) diff --git a/doc/user/methods/decision_tree.md b/doc/user/methods/decision_tree.md index ec1070f17e..95c572004a 100644 --- a/doc/user/methods/decision_tree.md +++ b/doc/user/methods/decision_tree.md @@ -9,13 +9,28 @@ Decision trees are useful for classifying points with _discrete labels_ (i.e. `0`, `1`, `2`). For predicting _continuous values_ (regression), see [`DecisionTreeRegressor`](#decision_tree_regressor). -#### Basic usage example excerpt: +#### Simple usage example: ```c++ -DecisionTree tree; // Step 1: construct object. -tree.Train(data, labels, 3); // Step 2: train model. -tree.Classify(testData, testPredictions); // Step 3: use model to classify. +// Train a decision tree on random numeric data and predict labels on test data: + +// All data and labels are uniform random; 10 dimensional data, 5 classes. +// Replace with a data::Load() call or similar for a real application. +arma::mat dataset(10, 1000, arma::fill::randu); // 1000 points. +arma::Row labels = + arma::randi>(1000, arma::distr_param(0, 4)); +arma::mat testDataset(10, 500, arma::fill::randu); // 500 test points. + +DecisionTree<> tree; // Step 1: create model. +tree.Train(dataset, labels, 5); // Step 2: train model. +arma::Row predictions; +tree.Classify(testDataset, predictions); // Step 3: classify points. + +// Print some information about the test predictions. +std::cout << arma::accu(predictions == 2) << " test points classified as class " + << "2." << std::endl; ``` +

More examples...

#### Quick links: @@ -39,43 +54,22 @@ tree.Classify(testData, testPredictions); // Step 3: use model to classify. ### Constructors -Construct a `DecisionTree` object using one of the constructors below. Defaults -and types are detailed in the [Constructor Parameters](#constructor-parameters) -section below. - -#### Forms: - - * `DecisionTree()` - * `DecisionTree(numClasses)` - - **Initialize tree without training.** + * `tree = DecisionTree()` + - Initialize tree without training. - You will need to call [`Train()`](#training) later to train the tree before calling [`Classify()`](#classification). --- - * `DecisionTree(data, labels, numClasses)` - * `DecisionTree(data, labels, numClasses, weights)` - * `DecisionTree(data, labels, numClasses, minLeafSize, minGainSplit, maxDepth)` - * `DecisionTree(data, labels, numClasses, weights, minLeafSize, minGainSplit, maxDepth)` - - **Train on numerical-only data (optionally with instance weights).** - - If hyperparameters are not specified, default values are used. - - `labels` should be a vector of length `data.n_cols`, containing values from - `0` to `numClasses - 1` (inclusive). - - If specified, `weights` should be a vector of length `data.n_cols`, - containing instance weights for each point in `data`. + * `tree = DecisionTree(data, labels, numClasses, minLeafSize=10, minGainSplit=1e-7, maxDepth=0)` + * `tree = DecisionTree(data, labels, numClasses, weights, minLeafSize=10, minGainSplit=1e-7, maxDepth=0)` + - Train on numerical-only data (optionally with instance weights). --- - * `DecisionTree(data, datasetInfo, labels, numClasses)` - * `DecisionTree(data, datasetInfo, labels, numClasses, weights)` - * `DecisionTree(data, datasetInfo, labels, numClasses, minLeafSize, minGainSplit, maxDepth)` - * `DecisionTree(data, datasetInfo, labels, numClasses, weights, minLeafSize, minGainSplit, maxDepth)` - - **Train on mixed categorical data (optionally with instance weights).** - - If hyperparameters are not specified, default values are used. - - `labels` should be a vector of length `data.n_cols`, containing values from - `0` to `numClasses - 1` (inclusive). - - If specified, `weights` should be a vector of length `data.n_cols`, - containing instance weights for each point in `data`. + * `tree = DecisionTree(data, datasetInfo, labels, numClasses, minLeafSize=10, minGainSplit=1e-7, maxDepth=0)` + * `tree = DecisionTree(data, datasetInfo, labels, numClasses, weights, minLeafSize=10, minGainSplit=1e-7, maxDepth=0)` + - Train on mixed categorical data (optionally with instance weights). --- @@ -116,43 +110,32 @@ If training is not done as part of the constructor call, it can be done with one of the versions of the `Train()` member function. For an instance of `DecisionTree` named `tree`, the following functions for training are available: - * `tree.Train(data, labels, numClasses)` - * `tree.Train(data, labels, numClasses, weights)` - * `tree.Train(data, labels, numClasses, minLeafSize, minGainSplit, maxDepth)` - * `tree.Train(data, labels, numClasses, weights, minLeafSize, minGainSplit, maxDepth)` - - **Train on numerical-only data (optionally with instance weights).** - - If hyperparameters are not specified, default values are used. - - `labels` should be a vector of length `data.n_cols`, containing values from - `0` to `numClasses - 1` (inclusive). - - If specified, `weights` should be a vector of length `data.n_cols`, - containing instance weights for each point in `data`. + * `tree.Train(data, labels, numClasses, minLeafSize=10, minGainSplit=1e-7, maxDepth=0)` + * `tree.Train(data, labels, numClasses, weights, minLeafSize=10, minGainSplit=1e-7, maxDepth=0)` + - Train on numerical-only data (optionally with instance weights). - Returns a `double` with the final gain of the tree (the Gini gain, unless a different [`FitnessFunction` template parameter](#fully-custom-behavior) is specified. --- - * `tree.Train(data, datasetInfo, labels, numClasses)` - * `tree.Train(data, datasetInfo, labels, numClasses, weights)` - * `tree.Train(data, datasetInfo, labels, numClasses, minLeafSize, minGainSplit, maxDepth)` - * `tree.Train(data, datasetInfo, labels, numClasses, weights, minLeafSize, minGainSplit, maxDepth)` - - **Train on mixed categorical data (optionally with instance weights).** - - If hyperparameters are not specified, default values are used. - - `labels` should be a vector of length `data.n_cols`, containing values from - `0` to `numClasses - 1` (inclusive). - - If specified, `weights` should be a vector of length `data.n_cols`, - containing instance weights for each point in `data`. - - Returns a `double` with the final gain of the tree (the Gini gain, unless a - different [`FitnessFunction` template parameter](#fully-custom-behavior) is - specified. + * `tree.Train(data, datasetInfo, labels, numClasses, minLeafSize=10, minGainSplit=1e-7, maxDepth=0)` + * `tree.Train(data, datasetInfo, labels, numClasses, weights, minLeafSize=10, minGainSplit=1e-7, maxDepth=0)` + - Train on mixed categorical data (optionally with instance weights). --- Types of each argument are the same as in the table for constructors [above](#constructor-parameters). -***Note***: training is not incremental. A second call to `Train()` will -retrain the decision tree from scratch. +***Notes***: + + * Training is not incremental. A second call to `Train()` will retrain the + decision tree from scratch. + + * `Train()` returns a `double` with the final gain of the tree (the Gini gain, + unless a different + [`FitnessFunction` template parameter](#fully-custom-behavior) is specified. ### Classification @@ -172,8 +155,6 @@ to make class predictions for new data. Defaults and types are detailed in the - ***(Single-point)*** - Classify a single point and compute class probabilities. - The predicted class is stored in `prediction`. - - The class probabilities are stored in `probabilities_vec`, which is set to - length `numClasses`. - The probability of class `i` can be accessed with `probabilities_vec[i]`. --- @@ -181,8 +162,6 @@ to make class predictions for new data. Defaults and types are detailed in the * `tree.Classify(data, predictions)` - ***(Multi-point)*** - Classify a set of points. - - The predicted class of each point is stored in `predictions`, which is set - to length `data.n_cols`. - The prediction for data point `i` can be accessed with `predictions[i]`. --- @@ -190,11 +169,7 @@ to make class predictions for new data. Defaults and types are detailed in the * `tree.Classify(data, predictions, probabilities)` - ***(Multi-point)*** - Classify a set of points and compute class probabilities for each point. - - The predicted class of each point is stored in `predictions`, which is set - to length `data.n_cols`. - The prediction for data point `i` can be accessed with `predictions[i]`. - - The class probabilities for each point are stored in `probabilities`, - which is set to size `numClasses` by `data.n_cols`. - The probability of class `j` for data point `i` can be accessed with `probabilities(j, i)`. @@ -206,11 +181,11 @@ to make class predictions for new data. Defaults and types are detailed in the |-----------|----------|----------|-----------------| | _single-point_ | `point` | [`arma::vec`](../matrices.md) | Single point for classification. | | _single-point_ | `prediction` | `size_t&` | `size_t` to store class prediction into. | -| _single-point_ | `probabilities_vec` | [`arma::vec&`](../matrices.md) | `arma::vec&` to store class probabilities into. | +| _single-point_ | `probabilities_vec` | [`arma::vec&`](../matrices.md) | `arma::vec&` to store class probabilities into. Will be set to length `numClasses`. | |||| | _multi-point_ | `data` | [`arma::mat`](../matrices.md) | Set of [column-major](../matrices.md) points for classification. | -| _multi-point_ | `predictions` | [`arma::Row&`](../matrices.md) | Vector of `size_t`s to store class prediction into. | -| _multi-point_ | `probabilities` | [`arma::mat&`](../matrices.md) | Matrix to store class probabilities into (number of rows will be equal to number of classes). | +| _multi-point_ | `predictions` | [`arma::Row&`](../matrices.md) | Vector of `size_t`s to store class prediction into. Will be set to length `data.n_cols`. | +| _multi-point_ | `probabilities` | [`arma::mat&`](../matrices.md) | Matrix to store class probabilities into (number of rows will be equal to number of classes, number of columns will be equal to `data.n_cols`). | ***Note:*** different types can be used for `data` and `point` (e.g. `arma::fmat`, `arma::sp_mat`, `arma::sp_vec`, etc.). However, the element type @@ -241,28 +216,8 @@ Each method is fully documented. ### Simple Examples -Train a decision tree on random numeric data and predict labels on a test set: - -```c++ -// 1000 random points in 10 dimensions. -arma::mat dataset(10, 1000, arma::fill::randu); -// Random labels for each point, totaling 5 classes. -arma::Row labels = - arma::randi>(1000, arma::distr_param(0, 4)); - -// Train in the constructor. -DecisionTree<> tree(dataset, labels, 5); - -// Create test data (500 points). -arma::mat testDataset(10, 500, arma::fill::randu); -arma::Row predictions; -tree.Classify(testDataset, predictions); -// Now `predictions` holds predictions for the test dataset. - -// Print some information about the test predictions. -std::cout << arma::accu(predictions == 2) << " test points classified as class " - << "2." << std::endl; -``` +See also the [simple usage example](#simple-usage-example) for a trivial use of +`DecisionTree`. --- diff --git a/doc/user/methods/decision_tree_regressor.md b/doc/user/methods/decision_tree_regressor.md index 43411cd233..552c7f84cc 100644 --- a/doc/user/methods/decision_tree_regressor.md +++ b/doc/user/methods/decision_tree_regressor.md @@ -10,13 +10,32 @@ The `DecisionTreeRegressor` class is useful for regressions; i.e., predicting _continuous values_ (`0.3`, `1.2`, etc.). For predicting _discrete labels_ (classification), see [`DecisionTree`](#decision_tree). -#### Basic usage example excerpt: +#### Simple usage example: + +Train a decision tree regressor on random numeric data and make predictions on a +test set: ```c++ -DecisionTreeRegressor tree; // Step 1: construct object. -tree.Train(data, responses, 3); // Step 2: train model. -tree.Predict(testData, testPredictions); // Step 3: predict values with model. +// Train a decision tree regressor on random numeric data and make predictions. + +// All data and responses are uniform random; this uses 10 dimensional data. +// Replace with a data::Load() call or similar for a real application. +arma::mat dataset(10, 1000, arma::fill::randu); // 1000 points. +arma::rowvec responses = arma::randn(1000); +arma::mat testDataset(10, 500, arma::fill::randu); // 500 test points. + +DecisionTreeRegressor<> tree; // Step 1: create tree. +tree.Train(dataset, responses); // Step 2: train model. +arma::rowvec predictions; +tree.Predict(testDataset, predictions); // Step 3: use model to predict. + +// Print some information about the test predictions. +std::cout << arma::accu(predictions > 0.7) << " test points predicted to have" + << " responses greater than 0.7." << std::endl; +std::cout << arma::accu(predictions < 0) << " test points predicted to have " + << "negative responses." << std::endl; ``` +

More examples...

#### Quick links: @@ -47,36 +66,22 @@ Parameters](#constructor-parameters) section below. #### Forms: - * `DecisionTreeRegressor()` - - **Initialize tree without training.** + * `tree = DecisionTreeRegressor()` + - Initialize tree without training. - You will need to call [`Train()`](#training) later to train the tree before calling [`Predict()`](#prediction). --- - * `DecisionTreeRegressor(data, responses)` - * `DecisionTreeRegressor(data, responses, weights)` - * `DecisionTreeRegressor(data, responses, minLeafSize, minGainSplit, maxDepth)` - * `DecisionTreeRegressor(data, responses, weights, minLeafSize, minGainSplit, maxDepth)` - - **Train on numerical-only data (optionally with instance weights).** - - If hyperparameters are not specified, default values are used. - - `responses` should be a vector of length `data.n_cols`, containing - continuous real values corresponding to the response for each data point. - - If specified, `weights` should be a vector of length `data.n_cols`, - containing instance weights for each point in `data`. + * `tree = DecisionTreeRegressor(data, responses, minLeafSize=10, minGainSplit=1e-7, maxDepth=0)` + * `tree = DecisionTreeRegressor(data, responses, weights, minLeafSize=10, minGainSplit=1e-7, maxDepth=0)` + - Train on numerical-only data (optionally with instance weights). --- - * `DecisionTreeRegressor(data, datasetInfo, responses)` - * `DecisionTreeRegressor(data, datasetInfo, responses, weights)` - * `DecisionTreeRegressor(data, datasetInfo, responses, minLeafSize, minGainSplit, maxDepth)` - * `DecisionTreeRegressor(data, datasetInfo, responses, weights, minLeafSize, minGainSplit, maxDepth)` - - **Train on mixed categorical data (optionally with instance weights).** - - If hyperparameters are not specified, default values are used. - - `responses` should be a vector of length `data.n_cols`, containing - continuous real values corresponding to the response for each data point. - - If specified, `weights` should be a vector of length `data.n_cols`, - containing instance weights for each point in `data`. + * `tree = DecisionTreeRegressor(data, datasetInfo, responses, minLeafSize=10, minGainSplit=1e-7, maxDepth=0)` + * `tree = DecisionTreeRegressor(data, datasetInfo, responses, weights, minLeafSize=10, minGainSplit=1e-7, maxDepth=0)` + - Train on mixed categorical data (optionally with instance weights). --- @@ -114,22 +119,11 @@ Parameters](#constructor-parameters) section below. ### Training If training is not done as a part of the constructor call, it can be done with -one of the versions of the `Train()` member function. For an instance of -`DecisionTree` named `tree`, the following functions for training are available: +one of the following versions of the `Train()` member function: - * `tree.Train(data, responses)` - * `tree.Train(data, responses, weights)` - * `tree.Train(data, responses, minLeafSize, minGainSplit, maxDepth)` - * `tree.Train(data, responses, weights, minLeafSize, minGainSplit, maxDepth)` - - **Train on numerical-only data (optionally with instance weights).** - - If hyperparameters are not specified, default values are used. - - `responses` should be a vector of length `data.n_cols`, containing - continuous real values corresponding to the response for each data point. - - If specified, `weights` should be a vector of length `data.n_cols`, - containing instance weights for each point in `data`. - - Returns a `double` with the final gain of the tree (the Gini gain, unless a - different [`FitnessFunction` template parameter](#fully-custom-behavior) is - specified. + * `tree.Train(data, responses, minLeafSize=10, minGainSplit=1e-7, maxDepth=0)` + * `tree.Train(data, responses, weights, minLeafSize=10, minGainSplit=1e-7, maxDepth=0)` + - Train on numerical-only data (optionally with instance weights). --- @@ -137,23 +131,21 @@ one of the versions of the `Train()` member function. For an instance of * `tree.Train(data, datasetInfo, responses, weights)` * `tree.Train(data, datasetInfo, responses, minLeafSize, minGainSplit, maxDepth)` * `tree.Train(data, datasetInfo, responses, weights, minLeafSize, minGainSplit, maxDepth)` - - **Train on mixed categorical data (optionally with instance weights).** - - If hyperparameters are not specified, default values are used. - - `responses` should be a vector of length `data.n_cols`, containing - continuous real values corresponding to the response for each data point. - - If specified, `weights` should be a vector of length `data.n_cols`, - containing instance weights for each point in `data`. - - Returns a `double` with the final gain of the tree (the Gini gain, unless a - different [`FitnessFunction` template parameter](#fully-custom-behavior) is - specified. + - Train on mixed categorical data (optionally with instance weights). --- Types of each argument are the same as in the table for constructors [above](#constructor-parameters). -***Note***: training is not incremental. A second call to `Train()` will -retrain the decision tree from scratch. +***Notes***: + + * Training is not incremental. A second call to `Train()` will retrain the + decision tree from scratch. + + * `Train()` returns a `double` with the final gain of the tree (the Gini gain, + unless a different + [`FitnessFunction` template parameter](#fully-custom-behavior) is specified. ### Prediction @@ -216,30 +208,8 @@ consulted. Each method is fully documented. ### Simple Examples -Train a decision tree regressor on random numeric data and make predictions on a -test set: - -```c++ -// 1000 random points in 10 dimensions. -arma::mat dataset(10, 1000, arma::fill::randu); -// Random responses, normally distributed, for each point. -arma::rowvec responses = arma::randn(1000); - -// Train in the constructor. -DecisionTreeRegressor<> tree(dataset, responses); - -// Create test data (500 points). -arma::mat testDataset(10, 500, arma::fill::randu); -arma::rowvec predictions; -tree.Predict(testDataset, predictions); -// Now `predictions` holds predictions for the test dataset. - -// Print some information about the test predictions. -std::cout << arma::accu(predictions > 0.7) << " test points predicted to have" - << " responses greater than 0.7." << std::endl; -std::cout << arma::accu(predictions < 0) << " test points predicted to have " - << "negative responses." << std::endl; -``` +See also the [simple usage example](#simple-usage-example) for a trivial use of +`DecisionTreeRegressor`. --- diff --git a/doc/user/methods/random_forest.md b/doc/user/methods/random_forest.md index 27343d3938..cb64611465 100644 --- a/doc/user/methods/random_forest.md +++ b/doc/user/methods/random_forest.md @@ -2,28 +2,48 @@ The `RandomForest` class implements a parallelized random forest classifier that supports numerical and categorical features, by default using Gini gain to -choose which feature to split on in each tree. The class offers several -template parameters and several runtime options that can be used to control the -behavior of the forest. +choose which feature to split on in each tree. - -Random forests are a collection of decision trees that give -better performance than a single decision tree. They are useful for classifying -points with _discrete labels_ (i.e. `0`, `1`, `2`). This implementation of the +Random forests are a collection of decision trees that give better performance +than a single decision tree. They are useful for classifying points with +_discrete labels_ (i.e. `0`, `1`, `2`). This implementation of the `RandomForest` class is not for regression (i.e. predicting _continuous values_). -#### Basic usage example excerpt: +mlpack's `RandomForest` class offers configurability via template parameters and +runtime parameters. This is used to provide the additional API-compatible +`ExtraTrees` class. To use `ExtraTrees`, simply replace `RandomForest` with +`ExtraTrees` in any of the documentation below. ([More +information...](#fully-custom-behavior)) + + + +#### Simple usage example: ```c++ -RandomForest rf; // Step 1: construct object. -rf.Train(data, labels, 3); // Step 2: train model. -rf.Classify(testData, testPredictions); // Step 3: use model to classify. +// Train a random forest on random numeric data and predict labels on test data: + +// All data and labels are uniform random; 10 dimensional data, 5 classes. +// Replace with a data::Load() call or similar for a real application. +arma::mat dataset(10, 1000, arma::fill::randu); // 1000 points. +arma::Row labels = + arma::randi>(1000, arma::distr_param(0, 4)); +arma::mat testDataset(10, 500, arma::fill::randu); // 500 test points. + +RandomForest<> rf; // Step 1: create model. +rf.Train(dataset, labels, 5, 10); // Step 2: train model. +arma::Row predictions; +rf.Classify(testData, predictions); // Step 3: classify points. +// You can also use `ExtraTrees<>` instead of `RandomForest<>`! + +// Print some information about the test predictions. +std::cout << arma::accu(predictions == 3) << " test points classified as class " + << "3." << std::endl; ``` +

More examples...

#### Quick links: - * [Variants](#variants): alternate behavior of the `RandomForest` class * [Constructors](#constructors): create `RandomForest` objects. * [`Train()`](#training): train model. * [`Classify()`](#classification): classify with a trained model. @@ -43,70 +63,24 @@ rf.Classify(testData, testPredictions); // Step 3: use model to classify. * [Decision tree on Wikipedia](https://en.wikipedia.org/wiki/Decision_tree) * [Leo Breiman's Random Forests page](https://www.stat.berkeley.edu/~breiman/RandomForests/cc_home.htm) -### Variants - -mlpack provides a few variants of the random forest classifier, using the -[fully custom behavior](#fully-custom-behavior) of the `RandomForest` class. In -the documentation below, the following types can be used as drop-in -replacements: - - * `RandomForest` - - This is an implementation of Breiman's seminal random forest algorithm - ([website](https://www.stat.berkeley.edu/~breiman/RandomForests/cc_home.htm), - [paper pdf](https://www.stat.berkeley.edu/~breiman/randomforest2001.pdf)). - - The [`DecisionTree`](#decision_tree) class is used for each individual - decision tree. - - When training each individual decision tree, bootstrapping is used to - compute the samples given to each tree for training. - - * `ExtraTrees` - - This is an implementation of the Extremely Randomized Trees algorithm - ([paper pdf](https://citeseerx.ist.psu.edu/document?repid=rep1&type=pdf&doi=336a165c17c9c56160d332b9f4a2b403fccbdbfb)). - - When training an `ExtraTrees` model, each individual decision tree chooses - splits for numeric data randomly. - - Training an `ExtraTrees` model is generally much faster than - `RandomForest`, but the accuracy of the `ExtraTrees` model will be lower. - - To use `ExtraTrees`, simply replace `RandomForest` with `ExtraTrees` in - the documentation below. - ### Constructors -Construct a `RandomForest` object using one of the constructors below. Defaults -and types are detailed in the [Constructor Parameters](#constructor-parameters) -section below. - -#### Forms: - - * `RandomForest()` - - **Initialize the random forest without training.** + * `rf = RandomForest()` + - Initialize the random forest without training. - You will need to call [`Train()`](#training) later to train the tree before calling [`Classify()`](#classification). --- - * `RandomForest(data, labels, numClasses)` - * `RandomForest(data, labels, numClasses, weights)` - * `RandomForest(data, labels, numClasses, numTrees, minLeafSize, minGainSplit, maxDepth)` - * `RandomForest(data, labels, numClasses, weights, numTrees, minLeafSize, minGainSplit, maxDepth)` - - **Train on numerical-only data (optionally with instance weights).** - - If hyperparameters are not specified, default values are used. - - `labels` should be a vector of length `data.n_cols`, containing values from - `0` to `numClasses - 1` (inclusive). - - If specified, `weights` should be a vector of length `data.n_cols`, - containing instance weights for each point in `data`. + * `rf = RandomForest(data, labels, numClasses, numTrees=20, minLeafSize=1, minGainSplit=1e-7, maxDepth=0)` + * `rf = RandomForest(data, labels, numClasses, weights, numTrees=20, minLeafSize=1, minGainSplit=1e-7, maxDepth=0)` + - Train on numerical-only data (optionally with instance weights). --- - * `RandomForest(data, datasetInfo, labels, numClasses)` - * `RandomForest(data, datasetInfo, labels, numClasses, weights)` - * `RandomForest(data, datasetInfo, labels, numClasses, numTrees, minLeafSize, minGainSplit, maxDepth)` - * `RandomForest(data, datasetInfo, labels, numClasses, weights, numTrees, minLeafSize, minGainSplit, maxDepth)` - - **Train on mixed categorical data (optionally with instance weights).** - - If hyperparameters are not specified, default values are used. - - `labels` should be a vector of length `data.n_cols`, containing values from - `0` to `numClasses - 1` (inclusive). - - If specified, `weights` should be a vector of length `data.n_cols`, - containing instance weights for each point in `data`. + * `rf = RandomForest(data, info, labels, numClasses, numTrees=20, minLeafSize=1, minGainSplit=1e-7, maxDepth=0)` + * `rf = RandomForest(data, info, labels, numClasses, weights, numTrees=20, minLeafSize=1, minGainSplit=1e-7, maxDepth=0)` + - Train on mixed categorical data (optionally with instance weights). --- @@ -122,15 +96,16 @@ section below. | **name** | **type** | **description** | **default** | |----------|----------|-----------------|-------------| | `data` | [`arma::mat`](../matrices.md) | [Column-major](../matrices.md) training matrix. | _(N/A)_ | -| `datasetInfo` | [`data::DatasetInfo`](../../tutorials/datasetmapper.md) | Dataset information, specifying type information for each dimension. | _(N/A)_ | +| `info` | [`data::DatasetInfo`](../../tutorials/datasetmapper.md) | Dataset information, specifying type information for each dimension. | _(N/A)_ | | `labels` | [`arma::Row`]('../matrices.md') | Training labels, between `0` and `numClasses - 1` (inclusive). Should have length `data.n_cols`. | _(N/A)_ | | `numClasses` | `size_t` | Number of classes in the dataset. | _(N/A)_ | -| `weights` | [`arma::rowvec`]('../matrices.md') | Weights for each training point. Should have length `data.n_cols`. | _(N/A)_ | +| `weights` | [`arma::rowvec`]('../matrices.md') | Instance weights for each training point. Should have length `data.n_cols`. | _(N/A)_ | | `numTrees` | `size_t` | Number of trees to train in the random forest. | `20` | | `minLeafSize` | `size_t` | Minimum number of points in each leaf node of each decision tree. | `1` | | `minGainSplit` | `double` | Minimum gain for a node to split in each decision tree. | `1e-7` | | `maxDepth` | `size_t` | Maximum depth for each decision tree. (0 means no limit.) | `0` | +| `warmStart` | `bool` | (Only available in `Train()`.) If true, training adds `numTrees` trees to the random forest. If `false`, an entirely new random forest will be created. | `false` | * If OpenMP is enabled, one thread will be used to train each of the `numTrees` trees in the random forest. The computational effort @@ -152,64 +127,41 @@ section below. ### Training If training is not done as part of the constructor call, it can be done with one -of the versions of the `Train()` member function. For an instance of -`RandomForest` named `rf`, the following functions for training are available: +of the following versions of the `Train()` member function: - * `rf.Train(data, labels, numClasses)` - * `rf.Train(data, labels, numClasses, weights)` - * `rf.Train(data, labels, numClasses, numTrees, minLeafSize, minGainSplit, maxDepth)` - * `rf.Train(data, labels, numClasses, weights, numTrees, minLeafSize, minGainSplit, maxDepth)` - * `rf.Train(data, labels, numClasses, numTrees, minLeafSize, minGainSplit, maxDepth, warmStart)` - * `rf.Train(data, labels, numClasses, weights, numTrees, minLeafSize, minGainSplit, maxDepth, warmStart)` - - **Train on numerical-only data (optionally with instance weights).** - - If hyperparameters are not specified, default values are used. - - `labels` should be a vector of length `data.n_cols`, containing values from - `0` to `numClasses - 1` (inclusive). - - If specified, `weights` should be a vector of length `data.n_cols`, - containing instance weights for each point in `data`. + * `rf.Train(data, labels, numClasses, numTrees=20, minLeafSize=1, minGainSplit=1e-7, maxDepth=0, warmStart=false)` + * `rf.Train(data, labels, numClasses, weights, numTrees=20, minLeafSize=1, minGainSplit=1e-7, maxDepth=0, warmStart=false)` + - Train on numerical-only data (optionally with instance weights). - Returns a `double` with the average gain of each tree in the random forest. By default, this is the Gini gain, unless a different [`FitnessFunction` template parameter](#fully-custom-behavior) is specified. - - If the optional `warmStart` parameter is set to `true`, then the `Train()` - call will simply add `numTrees` new trees to the existing random forest. - Otherwise, a new random forest will be trained. --- - * `rf.Train(data, datasetInfo, labels, numClasses)` - * `rf.Train(data, datasetInfo, labels, numClasses, weights)` - * `rf.Train(data, datasetInfo, labels, numClasses, numTrees, minLeafSize, minGainSplit, maxDepth)` - * `rf.Train(data, datasetInfo, labels, numClasses, weights, numTrees, minLeafSize, minGainSplit, maxDepth)` - * `rf.Train(data, datasetInfo, labels, numClasses, numTrees, minLeafSize, minGainSplit, maxDepth, warmStart)` - * `rf.Train(data, datasetInfo, labels, numClasses, weights, numTrees, minLeafSize, minGainSplit, maxDepth, warmStart)` - - **Train on mixed categorical data (optionally with instance weights).** - - If hyperparameters are not specified, default values are used. - - `labels` should be a vector of length `data.n_cols`, containing values from - `0` to `numClasses - 1` (inclusive). - - If specified, `weights` should be a vector of length `data.n_cols`, - containing instance weights for each point in `data`. - - Returns a `double` with the average gain of each tree in the random forest. - By default, this is the Gini gain, unless a different - [`FitnessFunction` template parameter](#fully-custom-behavior) is - specified. - - If the optional `warmStart` parameter is set to `true`, then the `Train()` - call will simply add `numTrees` new trees to the existing random forest. - Otherwise, a new random forest will be trained. + * `rf.Train(data, info, labels, numClasses, numTrees=20, minLeafSize=1, minGainSplit=1e-7, maxDepth=0, warmStart=false)` + * `rf.Train(data, info, labels, numClasses, weights, numTrees=20, minLeafSize=1, minGainSplit=1e-7, maxDepth=0, warmStart=false)` + - Train on mixed categorical data (optionally with instance weights). --- Types of each argument are the same as in the table for constructors [above](#constructor-parameters). -The `warmStart` option, which allows incremental training (i.e. additional -training on top of an existing model) is of type `bool` and defaults to `false`. -This option is not available in the [constructors](#constructors). +**Notes**: + + * The `warmStart` option, which allows incremental training (i.e. additional + training on top of an existing model) is of type `bool` and defaults to + `false`. This option is not available in the [constructors](#constructors). + + * `Train()` returns a `double` with the average gain of each tree in the random + forest. By default, this is the Gini gain, unless a different + [`FitnessFunction` template parameter](#fully-custom-behavior) is specified. ### Classification Once a `RandomForest` is trained, the `Classify()` member function can be used -to make class predictions for new data. Defaults and types are detailed in the +to make class predictions for new data. Parameters are detailed in the [Classification Parameters](#classification-parameters) section below. #### Forms: @@ -224,8 +176,6 @@ to make class predictions for new data. Defaults and types are detailed in the - ***(Single-point)*** - Classify a single point and compute class probabilities. - The predicted class is stored in `prediction`. - - The class probabilities are stored in `probabilities_vec`, which is set to - length `numClasses`. - The probability of class `i` can be accessed with `probabilities_vec[i]`. --- @@ -233,8 +183,6 @@ to make class predictions for new data. Defaults and types are detailed in the * `rf.Classify(data, predictions)` - ***(Multi-point)*** - Classify a set of points. - - The predicted class of each point is stored in `predictions`, which is set - to length `data.n_cols`. - The prediction for data point `i` can be accessed with `predictions[i]`. --- @@ -242,11 +190,7 @@ to make class predictions for new data. Defaults and types are detailed in the * `rf.Classify(data, predictions, probabilities)` - ***(Multi-point)*** - Classify a set of points and compute class probabilities for each point. - - The predicted class of each point is stored in `predictions`, which is set - to length `data.n_cols`. - The prediction for data point `i` can be accessed with `predictions[i]`. - - The class probabilities for each point are stored in `probabilities`, - which is set to size `numClasses` by `data.n_cols`. - The probability of class `j` for data point `i` can be accessed with `probabilities(j, i)`. @@ -258,11 +202,11 @@ to make class predictions for new data. Defaults and types are detailed in the |-----------|----------|----------|-----------------| | _single-point_ | `point` | [`arma::vec`](../matrices.md) | Single point for classification. | | _single-point_ | `prediction` | `size_t&` | `size_t` to store class prediction into. | -| _single-point_ | `probabilities_vec` | [`arma::vec&`](../matrices.md) | `arma::vec&` to store class probabilities into. | +| _single-point_ | `probabilities_vec` | [`arma::vec&`](../matrices.md) | `arma::vec&` to store class probabilities into. Will be set to length `numClasses`. | |||| | _multi-point_ | `data` | [`arma::mat`](../matrices.md) | Set of [column-major](../matrices.md) points for classification. | -| _multi-point_ | `predictions` | [`arma::Row&`](../matrices.md) | Vector of `size_t`s to store class prediction into. | -| _multi-point_ | `probabilities` | [`arma::mat&`](../matrices.md) | Matrix to store class probabilities into (number of rows will be equal to number of classes). | +| _multi-point_ | `predictions` | [`arma::Row&`](../matrices.md) | Vector of `size_t`s to store class prediction into. Will be set to length `data.n_cols`. | +| _multi-point_ | `probabilities` | [`arma::mat&`](../matrices.md) | Matrix to store class probabilities into (number of rows will be equal to number of classes, number of columns will be equal to `data.n_cols`). | ***Note:*** different types can be used for `data` and `point` (e.g. `arma::fmat`, `arma::sp_mat`, `arma::sp_vec`, etc.). However, the element type @@ -287,28 +231,8 @@ Each method is fully documented. ### Simple Examples -Train a random forest on random numeric data and predict labels on a test set: - -```c++ -// 1000 random points in 10 dimensions. -arma::mat dataset(10, 1000, arma::fill::randu); -// Random labels for each point, totaling 5 classes. -arma::Row labels = - arma::randi>(1000, arma::distr_param(0, 4)); - -// Train in the constructor, using 10 trees in the forest. -RandomForest<> rf(dataset, labels, 5, 10); - -// Create test data (500 points). -arma::mat testDataset(10, 500, arma::fill::randu); -arma::Row predictions; -rf.Classify(testDataset, predictions); -// Now `predictions` holds predictions for the test dataset. - -// Print some information about the test predictions. -std::cout << arma::accu(predictions == 3) << " test points classified as class " - << "3." << std::endl; -``` +See also the [simple usage example](#simple-usage-example) for a trivial use of +`RandomForest`. --- @@ -491,10 +415,33 @@ std::cout << arma::accu(predictions == 0) << " test points classified as class " #### Fully custom behavior. -The `RandomForest<>` class also supports several template parameters, which can -be used for custom behavior during learning. This flexibility is used to -provide [API-compatible variants of `RandomForest`](#variants). The full -signature of the class is as follows: +mlpack provides a few variants of the random forest classifier, using the +template parameters of the `RandomForest` class. The following types can be +used as drop-in replacements throughout this documentation page: + + * `RandomForest` + - This is an implementation of Breiman's seminal random forest algorithm + ([website](https://www.stat.berkeley.edu/~breiman/RandomForests/cc_home.htm), + [paper pdf](https://www.stat.berkeley.edu/~breiman/randomforest2001.pdf)). + - The [`DecisionTree`](#decision_tree) class is used for each individual + decision tree. + - When training each individual decision tree, bootstrapping is used to + compute the samples given to each tree for training. + + * `ExtraTrees` + - This is an implementation of the Extremely Randomized Trees algorithm + ([paper pdf](https://citeseerx.ist.psu.edu/document?repid=rep1&type=pdf&doi=336a165c17c9c56160d332b9f4a2b403fccbdbfb)). + - When training an `ExtraTrees` model, each individual decision tree chooses + splits for numeric data randomly. + - Training an `ExtraTrees` model is generally much faster than + `RandomForest`, but the accuracy of the `ExtraTrees` model will be lower. + - To use `ExtraTrees`, simply replace `RandomForest` with `ExtraTrees` in + the documentation below. + +--- + +Fully custom classes can also be used to control the behavior of the +`RandomForest` class. The full signature of the class is as follows: ```c++ RandomForest Date: Tue, 7 Nov 2023 17:33:51 -0500 Subject: [PATCH 42/47] Oops, remove redundant bullet point. --- doc/user/methods/decision_tree.md | 3 --- 1 file changed, 3 deletions(-) diff --git a/doc/user/methods/decision_tree.md b/doc/user/methods/decision_tree.md index 95c572004a..085657aeaa 100644 --- a/doc/user/methods/decision_tree.md +++ b/doc/user/methods/decision_tree.md @@ -113,9 +113,6 @@ of the versions of the `Train()` member function. For an instance of * `tree.Train(data, labels, numClasses, minLeafSize=10, minGainSplit=1e-7, maxDepth=0)` * `tree.Train(data, labels, numClasses, weights, minLeafSize=10, minGainSplit=1e-7, maxDepth=0)` - Train on numerical-only data (optionally with instance weights). - - Returns a `double` with the final gain of the tree (the Gini gain, unless a - different [`FitnessFunction` template parameter](#fully-custom-behavior) is - specified. --- From 471dbeafe6b92275cd15d60a50038d5432746d81 Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Wed, 15 Nov 2023 11:26:19 -0500 Subject: [PATCH 43/47] Use C++17 and fully qualify mlpack namespace in examples. --- doc/user/methods/decision_tree.md | 15 +++++++-------- doc/user/methods/decision_tree_regressor.md | 10 +++++----- doc/user/methods/random_forest.md | 16 ++++++++-------- 3 files changed, 20 insertions(+), 21 deletions(-) diff --git a/doc/user/methods/decision_tree.md b/doc/user/methods/decision_tree.md index 085657aeaa..680f50d1a6 100644 --- a/doc/user/methods/decision_tree.md +++ b/doc/user/methods/decision_tree.md @@ -21,7 +21,7 @@ arma::Row labels = arma::randi>(1000, arma::distr_param(0, 4)); arma::mat testDataset(10, 500, arma::fill::randu); // 500 test points. -DecisionTree<> tree; // Step 1: create model. +mlpack::DecisionTree tree; // Step 1: create model. tree.Train(dataset, labels, 5); // Step 2: train model. arma::Row predictions; tree.Classify(testDataset, predictions); // Step 3: classify points. @@ -107,8 +107,7 @@ std::cout << arma::accu(predictions == 2) << " test points classified as class " ### Training If training is not done as part of the constructor call, it can be done with one -of the versions of the `Train()` member function. For an instance of -`DecisionTree` named `tree`, the following functions for training are available: +of the following versions of the `Train()` member function: * `tree.Train(data, labels, numClasses, minLeafSize=10, minGainSplit=1e-7, maxDepth=0)` * `tree.Train(data, labels, numClasses, weights, minLeafSize=10, minGainSplit=1e-7, maxDepth=0)` @@ -232,7 +231,7 @@ arma::Row labels; data::Load("covertype.train.labels.csv", labels, true); // Create the tree. -DecisionTree<> tree; +mlpack::DecisionTree tree; // Train on the given dataset, specifying a minimum leaf size of 5. tree.Train(dataset, info, labels, 7 /* classes */, 5 /* minimum leaf size */); @@ -259,7 +258,7 @@ std::cout << "Class probabilities of second test point: " << Load a tree and print some information about it. ```c++ -DecisionTree<> tree; +mlpack::DecisionTree tree; // This call assumes a tree called "tree" has already been saved to `tree.bin` // with `data::Save()`. data::Load("tree.bin", "tree", tree, true); @@ -298,7 +297,7 @@ arma::Row labels = arma::randi>(1000, arma::distr_param(0, 4)); // Train in the constructor. -DecisionTree<> tree(dataset, labels, 5); +mlpack::DecisionTree tree(dataset, labels, 5); // Create test data (500 points). arma::fmat testDataset(10, 500, arma::fill::randu); @@ -315,7 +314,7 @@ std::cout << arma::accu(predictions == 2) << " test points classified as class " #### Fully custom behavior. -The `DecisionTree<>` class also supports several template parameters, which can +The `DecisionTree` class also supports several template parameters, which can be used for custom behavior during learning. The full signature of the class is as follows: @@ -549,7 +548,7 @@ class CustomCategoricalSplit - By default each random subset is of size `sqrt(d)` where `d` is the number of dimensions in the data. - If constructed as `MultipleRandomDimensionSelect(n)` and passed to the - constructor of `DecisionTree<>` or the `Train()` function, each random + constructor of `DecisionTree` or the `Train()` function, each random subset will be of size `n`. * Each `DecisionTree` [constructor](#constructors) and each version of the [`Train()`](#training) function optionally accept an instantiated diff --git a/doc/user/methods/decision_tree_regressor.md b/doc/user/methods/decision_tree_regressor.md index 552c7f84cc..f8263c6821 100644 --- a/doc/user/methods/decision_tree_regressor.md +++ b/doc/user/methods/decision_tree_regressor.md @@ -24,7 +24,7 @@ arma::mat dataset(10, 1000, arma::fill::randu); // 1000 points. arma::rowvec responses = arma::randn(1000); arma::mat testDataset(10, 500, arma::fill::randu); // 500 test points. -DecisionTreeRegressor<> tree; // Step 1: create tree. +mlpack::DecisionTreeRegressor tree; // Step 1: create tree. tree.Train(dataset, responses); // Step 2: train model. arma::rowvec predictions; tree.Predict(testDataset, predictions); // Step 3: use model to predict. @@ -233,7 +233,7 @@ data::Split(data, responses, trainData, testData, trainResponses, testResponses, 0.2); // Create the tree. -DecisionTreeRegressor<> tree; +mlpack::DecisionTreeRegressor tree; // Train on the given dataset, specifying a minimum gain of 1e-6 and keeping the // default minimum leaf size. const double mse = tree.Train(trainData, info, trainResponses, @@ -261,7 +261,7 @@ std::cout << "Average error on test set: " << testAverageError << "." Load a tree and print some information about it. ```c++ -DecisionTreeRegressor<> tree; +mlpack::DecisionTreeRegressor tree; // This call assumes a tree called "tree" has already been saved to `tree.bin` // with `data::Save()`. data::Load("tree.bin", "tree", tree, true); @@ -296,7 +296,7 @@ arma::fmat dataset(10, 1000, arma::fill::randu); arma::frowvec responses = arma::randn(1000); // Train in the constructor. -DecisionTreeRegressor<> tree(dataset, responses, 5); +mlpack::DecisionTreeRegressor tree(dataset, responses, 5); // Create test data (500 points). arma::fmat testDataset(10, 500, arma::fill::randu); @@ -313,7 +313,7 @@ std::cout << arma::accu(predictions > 1) << " test points predicted to have " #### Fully custom behavior. -The `DecisionTreeRegressor<>` class also supports several template parameters, +The `DecisionTreeRegressor` class also supports several template parameters, which can be used for custom behavior during learning. The full signature of the class is as follows: diff --git a/doc/user/methods/random_forest.md b/doc/user/methods/random_forest.md index cb64611465..b66c57eb30 100644 --- a/doc/user/methods/random_forest.md +++ b/doc/user/methods/random_forest.md @@ -30,11 +30,11 @@ arma::Row labels = arma::randi>(1000, arma::distr_param(0, 4)); arma::mat testDataset(10, 500, arma::fill::randu); // 500 test points. -RandomForest<> rf; // Step 1: create model. +mlpack::RandomForest rf; // Step 1: create model. rf.Train(dataset, labels, 5, 10); // Step 2: train model. arma::Row predictions; rf.Classify(testData, predictions); // Step 3: classify points. -// You can also use `ExtraTrees<>` instead of `RandomForest<>`! +// You can also use `ExtraTrees` instead of `RandomForest`! // Print some information about the test predictions. std::cout << arma::accu(predictions == 3) << " test points classified as class " @@ -250,7 +250,7 @@ arma::Row labels; data::Load("covertype.train.labels.csv", labels, true); // Create the random forest. -RandomForest<> rf; +mlpack::RandomForest rf; // Train 10 trees on the given dataset, with a minimum leaf size of 3. rf.Train(dataset, info, labels, 7 /* classes */, 10 /* trees */, 3 /* minimum leaf size */); @@ -289,7 +289,7 @@ std::cout << "After training 20 trees, test set accuracy is " << accuracy Load a random forest and print some information about it. ```c++ -RandomForest<> rf; +mlpack::RandomForest rf; // This call assumes a random forest called "rf" has already been saved to // `rf.bin` with `data::Save()`. data::Load("rf.bin", "rf", rf, true); @@ -325,7 +325,7 @@ data::Load("covertype.test.arff", testDataset, info, true); data::Load("covertype.test.labels.csv", testLabels, true); // Create the random forest. -RandomForest<> rf; +mlpack::RandomForest rf; // Train 20 trees on the given dataset, with a minimum leaf size of 5. rf.Train(dataset, info, labels, 7 /* classes */, 20 /* trees */, 5 /* minimum leaf size */); @@ -362,7 +362,7 @@ arma::Row labels = // Train in the constructor, using 10 trees in the forest. // Note that `ExtraTrees` has exactly the same API as `RandomForest`. -ExtraTrees<> rf(dataset, labels, 5, 10); +mlpack::ExtraTrees rf(dataset, labels, 5, 10); // Create a single test point. arma::vec testPoint(10, arma::fill::randu); @@ -398,7 +398,7 @@ arma::Row labels = arma::randi>(1000, arma::distr_param(0, 4)); // Train in the constructor. -RandomForest<> rf(dataset, labels, 5); +mlpack::RandomForest rf(dataset, labels, 5); // Create test data (500 points). arma::fmat testDataset(10, 500, arma::fill::randu); @@ -520,7 +520,7 @@ class CustomFitnessFunction - By default each random subset is of size `sqrt(d)` where `d` is the number of dimensions in the data. - If constructed as `MultipleRandomDimensionSelect(n)` and passed to the - constructor of `RandomForest<>` or the `Train()` function, each random + constructor of `RandomForest` or the `Train()` function, each random subset will be of size `n`. * Each `RandomForest` [constructor](#constructors) and each version of the [`Train()`](#training) function optionally accept an instantiated From c62a6dcaa9de69b1a9151f76e9bb6c23deacb5d6 Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Wed, 15 Nov 2023 11:29:11 -0500 Subject: [PATCH 44/47] Fully qualify namespace for AdaBoost and Perceptron. --- doc/user/methods/adaboost.md | 12 ++++++------ doc/user/methods/perceptron.md | 12 ++++++------ 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/doc/user/methods/adaboost.md b/doc/user/methods/adaboost.md index 31192ec338..a85eaf8631 100644 --- a/doc/user/methods/adaboost.md +++ b/doc/user/methods/adaboost.md @@ -22,7 +22,7 @@ arma::Row labels = arma::randi>(1000, arma::distr_param(0, 4)); arma::mat testDataset(10, 500, arma::fill::randu); // 500 test points. -AdaBoost ab; // Step 1: create model. +mlpack::AdaBoost ab; // Step 1: create model. ab.Train(dataset, labels, 5); // Step 2: train model. arma::Row predictions; ab.Classify(testDataset, predictions); // Step 3: classify points. @@ -233,7 +233,7 @@ data::Load("iris.csv", dataset, true); arma::Row labels; data::Load("iris.labels.csv", labels, true); -AdaBoost ab; +mlpack::AdaBoost ab; // Train with a custom number of perceptron iterations, and custom AdaBoost // parameters. ab.Train(dataset, labels, 3, 75 /* maximum number of weak learners */, @@ -263,7 +263,7 @@ data::Load("iris.csv", dataset, true); arma::Row labels; data::Load("iris.labels.csv", dataset, true); -AdaBoost ab; +mlpack::AdaBoost ab; ab.MaxIterations() = 50; // Use at most 50 weak learners. ab.Tolerance() = 1e-4; // Set a custom tolerance for convergence. @@ -280,7 +280,7 @@ Load an AdaBoost model and print some information about it. ```c++ // Load a saved model named "adaboost_model" from `adaboost_model.bin`. -AdaBoost ab; +mlpack::AdaBoost ab; data::Load("adaboost_model.bin", "adaboost_model", ab, true); std::cout << "Details about the model in `adaboost_model.bin`:" << std::endl; @@ -404,7 +404,7 @@ arma::Row labels = // hyperparameters for the decision stump (these could be omitted). See the // DecisionTree documentation for more details on the ID3DecisionStump-specific // hyperparameters. -AdaBoost ab(dataset, labels, 5, +mlpack::AdaBoost ab(dataset, labels, 5, 25 /* maximum number of decision stumps */, 1e-6 /* tolerance for convergence of AdaBoost */, /** Hyperparameters specific to ID3DecisionStump: **/ @@ -438,7 +438,7 @@ arma::Row labels = // The weak learner type is now a floating-point Perceptron. typedef Perceptron PerceptronType; -AdaBoost ab(dataset, labels, 5); +mlpack::AdaBoost ab(dataset, labels, 5); // Create test data (500 points). arma::fmat testDataset(10, 500, arma::fill::randu); diff --git a/doc/user/methods/perceptron.md b/doc/user/methods/perceptron.md index 3e36b134f1..63269f676e 100644 --- a/doc/user/methods/perceptron.md +++ b/doc/user/methods/perceptron.md @@ -24,7 +24,7 @@ arma::Row labels = arma::randi>(1000, arma::distr_param(0, 4)); arma::mat testDataset(10, 500, arma::fill::randu); // 500 test points. -Perceptron p; // Step 1: create model. +mlpack::Perceptron p; // Step 1: create model. p.Train(dataset, labels, 5); // Step 2: train model. arma::Row predictions; p.Classify(testDataset, predictions); // Step 3: classify points. @@ -214,7 +214,7 @@ arma::Row labels; data::Load("iris.labels.csv", labels, true); // Create a Perceptron object. -Perceptron p; +mlpack::Perceptron p; // Set the maximum number of iterations to 100. (This can also be done in the // constructor.) p.MaxIterations() = 100; @@ -242,7 +242,7 @@ std::cout << "Training set accuracy after 350 iterations: " Load a saved perceptron from disk and print information about it. ```c++ -Perceptron p; +mlpack::Perceptron p; // This call assumes a perceptron called "p" has already been saved to // `perceptron.bin` with `data::Save()`. data::Load("perceptron.bin", "p", p, true); @@ -376,8 +376,8 @@ arma::Row labels = arma::randi>(1000, arma::distr_param(0, 4)); // Train in the constructor. Weights will be initialized randomly. -Perceptron p(dataset, - labels, 5); +mlpack::Perceptron p( + dataset, labels, 5); // Create test data (500 points). arma::mat testDataset(10, 500, arma::fill::randu); @@ -404,7 +404,7 @@ arma::Row labels = arma::randi>(1000, arma::distr_param(0, 4)); // Train in the constructor. -Perceptron p(dataset, labels, 5); +mlpack::Perceptron p(dataset, labels, 5); // Create test data (500 points). arma::sp_fmat testDataset; From 1c78d3efa2ff4798733bab9e0cbe8bc716023689 Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Wed, 15 Nov 2023 11:44:24 -0500 Subject: [PATCH 45/47] Some additional cleanups and shortening. --- doc/user/methods/adaboost.md | 5 +---- doc/user/methods/perceptron.md | 5 +---- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/doc/user/methods/adaboost.md b/doc/user/methods/adaboost.md index a85eaf8631..fc8f108839 100644 --- a/doc/user/methods/adaboost.md +++ b/doc/user/methods/adaboost.md @@ -148,10 +148,7 @@ Types of each argument are the same as in the table for constructors ### Classification Once an `AdaBoost` model is trained, the `Classify()` member function can be -used to make class predictions for new data. Defaults and types are detailed in -the [Classification Parameters](#classification-parameters) section below. - -#### Forms: +used to make class predictions for new data. * `size_t predictedClass = ab.Classify(point)` - ***(Single-point)*** diff --git a/doc/user/methods/perceptron.md b/doc/user/methods/perceptron.md index 63269f676e..22101fc9f9 100644 --- a/doc/user/methods/perceptron.md +++ b/doc/user/methods/perceptron.md @@ -143,10 +143,7 @@ Types of each argument are the same as in the table for constructors ### Classification Once a `Perceptron` is trained, the `Classify()` member function can be used to -make class predictions for new data. Defaults and types are detailed in the -[Classification Parameters](#classification-parameters) section below. - -#### Forms: +make class predictions for new data. * `size_t predictedClass = p.Classify(point)` - ***(Single-point)*** From edafa00ea42ef9f2c33d6b6ccf77b4868c1b9eb0 Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Wed, 15 Nov 2023 11:50:00 -0500 Subject: [PATCH 46/47] Fix capitalization to match the rest of the documentation. --- doc/user/methods/adaboost.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/user/methods/adaboost.md b/doc/user/methods/adaboost.md index fc8f108839..ec52f54d8b 100644 --- a/doc/user/methods/adaboost.md +++ b/doc/user/methods/adaboost.md @@ -156,11 +156,11 @@ used to make class predictions for new data. --- - * `ab.Classify(point, prediction, probabilities_vec)` + * `ab.Classify(point, prediction, probabilitiesVec)` - ***(Single-point)*** - Classify a single point and compute class probabilities. - The predicted class is stored in `prediction`. - - The probability of class `i` can be accessed with `probabilities_vec[i]`. + - The probability of class `i` can be accessed with `probabilitiesVec[i]`. --- @@ -186,7 +186,7 @@ used to make class predictions for new data. |-----------|----------|----------|-----------------| | _single-point_ | `point` | [`arma::vec`](../matrices.md) | Single point for classification. | | _single-point_ | `prediction` | `size_t&` | `size_t` to store class prediction into. | -| _single-point_ | `probabilities_vec` | [`arma::vec&`](../matrices.md) | `arma::vec&` to store class probabilities into. | +| _single-point_ | `probabilitiesVec` | [`arma::vec&`](../matrices.md) | `arma::vec&` to store class probabilities into. | |||| | _multi-point_ | `data` | [`arma::mat`](../matrices.md) | Set of [column-major](../matrices.md) points for classification. | | _multi-point_ | `predictions` | [`arma::Row&`](../matrices.md) | Vector of `size_t`s to store class prediction into; will be set to length `data.n_cols`. | From 9b8797711c194ccb6457254bac287c6501f6b449 Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Wed, 22 Nov 2023 14:03:15 -0500 Subject: [PATCH 47/47] Fully qualify data:: uses so that using namespace mlpack isn't needed. --- doc/user/methods/adaboost.md | 12 ++++++------ doc/user/methods/perceptron.md | 6 +++--- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/doc/user/methods/adaboost.md b/doc/user/methods/adaboost.md index ec52f54d8b..7366034cf0 100644 --- a/doc/user/methods/adaboost.md +++ b/doc/user/methods/adaboost.md @@ -225,10 +225,10 @@ Train an AdaBoost model using the hyperparameters from an existing weak learner. ```c++ // See https://datasets.mlpack.org/iris.csv. arma::mat dataset; -data::Load("iris.csv", dataset, true); +mlpack::data::Load("iris.csv", dataset, true); // See https://datasets.mlpack.org/iris.labels.csv. arma::Row labels; -data::Load("iris.labels.csv", labels, true); +mlpack::data::Load("iris.labels.csv", labels, true); mlpack::AdaBoost ab; // Train with a custom number of perceptron iterations, and custom AdaBoost @@ -255,10 +255,10 @@ trained model to disk. ```c++ // See https://datasets.mlpack.org/iris.csv. arma::mat dataset; -data::Load("iris.csv", dataset, true); +mlpack::data::Load("iris.csv", dataset, true); // See https://datasets.mlpack.org/iris.labels.csv. arma::Row labels; -data::Load("iris.labels.csv", dataset, true); +mlpack::data::Load("iris.labels.csv", dataset, true); mlpack::AdaBoost ab; ab.MaxIterations() = 50; // Use at most 50 weak learners. @@ -268,7 +268,7 @@ ab.Tolerance() = 1e-4; // Set a custom tolerance for convergence. ab.Train(dataset, labels, 3); // Save the model to `adaboost_model.bin`. -data::Save("adaboost_model.bin", "adaboost_model", ab, true); +mlpack::data::Save("adaboost_model.bin", "adaboost_model", ab, true); ``` --- @@ -278,7 +278,7 @@ Load an AdaBoost model and print some information about it. ```c++ // Load a saved model named "adaboost_model" from `adaboost_model.bin`. mlpack::AdaBoost ab; -data::Load("adaboost_model.bin", "adaboost_model", ab, true); +mlpack::data::Load("adaboost_model.bin", "adaboost_model", ab, true); std::cout << "Details about the model in `adaboost_model.bin`:" << std::endl; std::cout << " - Trained on " << ab.NumClasses() << " classes." << std::endl; diff --git a/doc/user/methods/perceptron.md b/doc/user/methods/perceptron.md index 22101fc9f9..679ef0bcda 100644 --- a/doc/user/methods/perceptron.md +++ b/doc/user/methods/perceptron.md @@ -205,10 +205,10 @@ Train a perceptron multiple times, incrementally, with custom hyperparameters. ```c++ // See https://datasets.mlpack.org/iris.csv. arma::mat dataset; -data::Load("iris.csv", dataset, true); +mlpack::data::Load("iris.csv", dataset, true); // See https://datasets.mlpack.org/iris.labels.csv. arma::Row labels; -data::Load("iris.labels.csv", labels, true); +mlpack::data::Load("iris.labels.csv", labels, true); // Create a Perceptron object. mlpack::Perceptron p; @@ -242,7 +242,7 @@ Load a saved perceptron from disk and print information about it. mlpack::Perceptron p; // This call assumes a perceptron called "p" has already been saved to // `perceptron.bin` with `data::Save()`. -data::Load("perceptron.bin", "p", p, true); +mlpack::data::Load("perceptron.bin", "p", p, true); if (p.NumClasses() > 0) {