diff --git a/HISTORY.md b/HISTORY.md index 84e5b415e2..cf4b7e54f4 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -22,6 +22,8 @@ * Fix inconsistent use of the "input" parameter to the Backward method in ANNs (#3551). + * Allow passing weak learner hyperparameters directly to AdaBoost (#3560). + ### mlpack 4.2.1 ###### 2023-09-05 * Reinforcement Learning: Gaussian noise (#3515). diff --git a/doc/user/methods/adaboost.md b/doc/user/methods/adaboost.md new file mode 100644 index 0000000000..7366034cf0 --- /dev/null +++ b/doc/user/methods/adaboost.md @@ -0,0 +1,449 @@ +## `AdaBoost` + +The `AdaBoost` class implements the 'adaptive boosting' classifier AdaBoost.MH. +This classifier is an ensemble of weak learners. The `AdaBoost` class offers +control over the weak learners and other behavior via template parameters. By +default, the `Perceptron` class is used as a weak learner. + +`AdaBoost` is useful for classifying points with _discrete labels_ (i.e. `0`, +`1`, `2`). + +#### Simple usage example: + +Train an AdaBoost model on random data and predict labels on a random test set. + +```c++ +// Train an AdaBoost model on random data and predict labels on test data: + +// All data and labels are uniform random; 10 dimensional data, 5 classes. +// Replace with a data::Load() call or similar for a real application. +arma::mat dataset(10, 1000, arma::fill::randu); // 1000 points. +arma::Row labels = + arma::randi>(1000, arma::distr_param(0, 4)); +arma::mat testDataset(10, 500, arma::fill::randu); // 500 test points. + +mlpack::AdaBoost ab; // Step 1: create model. +ab.Train(dataset, labels, 5); // Step 2: train model. +arma::Row predictions; +ab.Classify(testDataset, predictions); // Step 3: classify points. + +// Print some information about the test predictions. +std::cout << arma::accu(predictions == 3) << " test points classified as class " + << "3." << std::endl; +``` +

More examples...

+ +#### Quick links: + + * [Constructors](#constructors): create `AdaBoost` objects. + * [`Train()`](#training): train model. + * [`Classify()`](#classify): classify with a trained model. + * [Other functionality](#other-functionality) for loading, saving, and + inspecting. + * [Examples](#simple-examples) of simple usage and links to detailed example + projects. + * [Template parameters](#advanced-functionality-template-parameters) for custom + behavior. + * [Advanced template examples](#advanced-functionality-examples) of use with + custom template parameters. + +#### See also: + + * [mlpack classifiers](#mlpack_classifiers) + * [`Perceptron`](#perceptron) + * [`DecisionTree`](#decision_tree) + * [AdaBoost on Wikipedia](https://en.wikipedia.org/wiki/AdaBoost) + * [AdaBoost.MH paper (pdf)](https://dl.acm.org/doi/pdf/10.1145/279943.279960) + +### Constructors + + * `ab = AdaBoost(tolerance=1e-6)` + - Initialize model without training. + - You will need to call [`Train()`](#training) later to train the tree before + calling [`Classify()`](#classification). + +--- + + * `ab = AdaBoost(data, labels, numClasses, maxIterations=100, tolerance=1e-6)` + - Train model using default weak learner hyperparameters. + +--- + + * `ab = AdaBoost(data, labels, numClasses, maxIterations=100, tolerance=1e-6, _[weak learner hyperparameters...]_)` + - Train model with custom weak learner hyperparameters. + - Hyperparameters for the weak learner are any arguments to the weak + learner's `Train()` function that come after `numClasses` or `weights`. + - The only hyperparameter for the default weak learner (`Perceptron`) is + `maxIterations`. + - See [examples of this constructor in use](#simple-examples). + +--- + +#### Constructor Parameters: + + + +| **name** | **type** | **description** | **default** | +|----------|----------|-----------------|-------------| +| `data` | [`arma::mat`](../matrices.md) | [Column-major](../matrices.md) training matrix. | _(N/A)_ | +| `labels` | [`arma::Row`]('../matrices.md') | Training labels, between `0` and `numClasses - 1` (inclusive). Should have length `data.n_cols`. | _(N/A)_ | +| `numClasses` | `size_t` | Number of classes in the dataset. | _(N/A)_ | +| `weakLearner` | `Perceptron` | An initialized weak learner whose +hyperparameters will be used as settings for weak learners during training. | +_(N/A)_ | +| `maxIterations` | `size_t` | Maximum number of iterations of AdaBoost.MH to use. This is the maximum number of weak learners to train. (0 means no limit, and weak learners will be trained until the tolerance is met.) | `100` | +| `tolerance` | `double` | When the weighted residual (`r_t`) of the model goes below `tolerance`, training will terminate and no more weak learners will be added. | `1e-6` | + +As an alternative to passing hyperparameters, each hyperparameter can be set +with a standalone method. The following functions can be used before calling +`Train()` to set hyperparameters: + + * `ab.MaxIterations() = maxIter;` will set the maximum number of weak learners + during training to `maxIter`. + * `ab.Tolerance() = tol;` will set the tolerance to `tol`. + + + +***Note:*** different types of weak learners can be used than +[`Perceptron`](#perceptron), by changing the [`WeakLearnerType` template +parameter](#advanced-functionality-template-parameters). + +### Training + +If training is not done as part of the constructor call, it can be done with one +of the versions of the `Train()` member function. For an instance of `AdaBoost` +named `ab`, the following functions for training are available: + + * `ab.Train(data, labels, numClasses, maxIterations=100, tolerance=1e-6)` + - Train model using default weak learner parameters. + +--- + + * `ab.Train(data, labels, numClasses, maxIterations=100, tolerance=1e-6, [weak learner hyperparameters...])` + - Train model with custom weak learner parameters. + - Hyperparameters for the weak learner are any arguments to the weak + learner's `Train()` function that come after `numClasses` or `weights`. + - The only hyperparameter for the default weak learner (`Perceptron`) is + `maxIterations`. + - See [examples of this form in use](#simple-examples). + +--- + +Types of each argument are the same as in the table for constructors +[above](#constructor-parameters). + +***Notes***: + + * Training is not incremental. A second call to `Train()` will retrain the + AdaBoost model from scratch. + + * `Train()` returns a `double` indicating an upper bound on the training error + (specifically, the product of _Zt_ values, as described in the paper). + +### Classification + +Once an `AdaBoost` model is trained, the `Classify()` member function can be +used to make class predictions for new data. + + * `size_t predictedClass = ab.Classify(point)` + - ***(Single-point)*** + - Classify a single point, returning the predicted class. + +--- + + * `ab.Classify(point, prediction, probabilitiesVec)` + - ***(Single-point)*** + - Classify a single point and compute class probabilities. + - The predicted class is stored in `prediction`. + - The probability of class `i` can be accessed with `probabilitiesVec[i]`. + +--- + + * `ab.Classify(data, predictions)` + - ***(Multi-point)*** + - Classify a set of points. + - The prediction for data point `i` can be accessed with `predictions[i]`. + +--- + + * `ab.Classify(data, predictions, probabilities)` + - ***(Multi-point)*** + - Classify a set of points and compute class probabilities for each point. + - The prediction for data point `i` can be accessed with `predictions[i]`. + - The probability of class `j` for data point `i` can be accessed with + `probabilities(j, i)`. + +--- + +#### Classification Parameters: + +| **usage** | **name** | **type** | **description** | +|-----------|----------|----------|-----------------| +| _single-point_ | `point` | [`arma::vec`](../matrices.md) | Single point for classification. | +| _single-point_ | `prediction` | `size_t&` | `size_t` to store class prediction into. | +| _single-point_ | `probabilitiesVec` | [`arma::vec&`](../matrices.md) | `arma::vec&` to store class probabilities into. | +|||| +| _multi-point_ | `data` | [`arma::mat`](../matrices.md) | Set of [column-major](../matrices.md) points for classification. | +| _multi-point_ | `predictions` | [`arma::Row&`](../matrices.md) | Vector of `size_t`s to store class prediction into; will be set to length `data.n_cols`. | +| _multi-point_ | `probabilities` | [`arma::mat&`](../matrices.md) | Matrix to store class probabilities into (number of rows will be equal to number of classes; number of columns will be equal to `data.n_cols`). | + +### Other Functionality + + + + * An `AdaBoost` model can be serialized with [`data::Save()`](../formats.md) + and [`data::Load()`](../formats.md). + + * `ab.NumClasses()` will return a `size_t` indicating the number of classes the + model was trained on. + + * `ab.WeakLearners()` will return a `size_t` indicating the number of weak + learners that the model currently contains. + + * `ab.Alpha(i)` will return the weight of weak learner `i`. + + * `ab.WeakLearner(i)` will return the `i`th weak learner. + +For complete functionality, the [source +code](/src/mlpack/methods/adaboost/adaboost.hpp) can be consulted. Each method +is fully documented. + +### Simple Examples + +See also the [simple usage example](#simple-usage-example) for a trivial usage +of the `AdaBoost` class. + +--- + +Train an AdaBoost model using the hyperparameters from an existing weak learner. + +```c++ +// See https://datasets.mlpack.org/iris.csv. +arma::mat dataset; +mlpack::data::Load("iris.csv", dataset, true); +// See https://datasets.mlpack.org/iris.labels.csv. +arma::Row labels; +mlpack::data::Load("iris.labels.csv", labels, true); + +mlpack::AdaBoost ab; +// Train with a custom number of perceptron iterations, and custom AdaBoost +// parameters. +ab.Train(dataset, labels, 3, 75 /* maximum number of weak learners */, + 1e-6 /* tolerance for AdaBoost convergence */, + 100 /* maximum number of perceptron iterations */); + +// Now predict the label of a point and the probabilities of each class. +size_t prediction; +arma::rowvec probabilities; +ab.Classify(dataset.col(10), prediction, probabilities); + +std::cout << "Point 11 is predicted to have class " << prediction << "." + << std::endl; +std::cout << "Probabilities of each class: " << probabilities; +``` + +--- + +Before training an AdaBoost model, set hyperparameters individually. Save the +trained model to disk. + +```c++ +// See https://datasets.mlpack.org/iris.csv. +arma::mat dataset; +mlpack::data::Load("iris.csv", dataset, true); +// See https://datasets.mlpack.org/iris.labels.csv. +arma::Row labels; +mlpack::data::Load("iris.labels.csv", dataset, true); + +mlpack::AdaBoost ab; +ab.MaxIterations() = 50; // Use at most 50 weak learners. +ab.Tolerance() = 1e-4; // Set a custom tolerance for convergence. + +// Now train, using the hyperparameters specified above. +ab.Train(dataset, labels, 3); + +// Save the model to `adaboost_model.bin`. +mlpack::data::Save("adaboost_model.bin", "adaboost_model", ab, true); +``` + +--- + +Load an AdaBoost model and print some information about it. + +```c++ +// Load a saved model named "adaboost_model" from `adaboost_model.bin`. +mlpack::AdaBoost ab; +mlpack::data::Load("adaboost_model.bin", "adaboost_model", ab, true); + +std::cout << "Details about the model in `adaboost_model.bin`:" << std::endl; +std::cout << " - Trained on " << ab.NumClasses() << " classes." << std::endl; +std::cout << " - Tolerance used for training: " << ab.Tolerance() << "." + << std::endl; +std::cout << " - Number of perceptron weak learners in model" + << ab.WeakLearners() << "." << std::endl; + +// Print some details about the first weak learner, if available. The weak +// learner type is `Perceptron<>`. +if (ab.WeakLearners() > 0) +{ + std::cout << " - Weight of first perceptron weak learner: " << ab.Alpha(0) + << "." << std::endl; + std::cout << " - Biases of first perceptron learner: " + << ab.WeakLearner(0).Biases().t(); +} +``` + +--- + +See also the following fully-working examples: + + - [Graduate admission classification with `AdaBoost`](https://github.com/mlpack/examples/blob/master/graduate_admission_classification_with_Adaboost/graduate-admission-classification-with-adaboost-cpp.ipynb) + +### Advanced Functionality: Template Parameters + +The `AdaBoost` class has two template parameters that can be used for custom +behavior. The full signature of the class is: + +```c++ +AdaBoost +``` + + * `WeakLearnerType`: the weak classifier to ensemble in the AdaBoost model. + * `MatType`: specifies the type of matrix used for learning and internal + representation of model parameters. + +--- + +#### `WeakLearnerType` + + + + * Specifies the weak learner to use when constructing an AdaBoost model. + * The default `WeakLearnerType` is [`Perceptron<>`](#perceptron). + * The `ID3DecisionStump` class (a custom variant of + [`DecisionTree`](#decision_tree)) is available for drop-in usage as a weak + learner. + * Any custom variant of `Perceptron<>` or `DecisionTree<>` can be used; e.g., + `Perceptron`. + * A custom class must implement the following functions for training and + classification. Note that this is the same API as mlpack's classifiers that + support instance weights for learning, and so any mlpack classifier + supporting instance weights can also be used. + +```c++ +// You can use this as a starting point for implementation. +class CustomWeakLearner +{ + public: + // Train the model with the given hyperparameters. + // + // * `MatType` will be an Armadillo-like matrix type (typically `arma::mat`); + // this is the same type as the `MatType` template parameter for the + // `AdaBoost` class. + // + // * `data` and `labels` are the same dataset passed to the `Train()` method + // of `AdaBoost`. + // + // * `weights` contains instance weights for each point (column) of `data`. + // + // Note: there is no restriction on the number or types of hyperparameters + // that can be used, but, they do need default arguments. The example here + // includes two. + template + void Train(const MatType& data, + const arma::Row& labels, + const size_t numClasses, + const arma::rowvec& weights, + const size_t hyperparameterA = 10, + const double hyperparameterB = 0.1); + + // Classify the given point. `VecType` will be an Armadillo-like type that is + // a vector that represents a single point. + template + size_t Classify(const VecType& point); + + // Classify the given points in `data`, storing the predicted classifications + // in `predictions`. + template + void Classify(const MatType& data, arma::Row& predictions); +}; +``` + +--- + +#### `MatType` + + * Specifies the matrix type to use for data when learning a model (or + predicting with one). + * By default, `MatType` is `arma::mat` (dense 64-bit precision matrix). + * Any matrix type implementing the Armadillo API will work; so, for instance, + `arma::fmat` or `arma::sp_mat` can also be used. + +### Advanced Functionality Examples + +Train an AdaBoost model using decision stumps as the weak learner (use a +different `WeakLearnerType`). + +```c++ +// 1000 random points in 10 dimensions. +arma::mat dataset(10, 1000, arma::fill::randu); +// Random labels for each point, totaling 5 classes. +arma::Row labels = + arma::randi>(1000, arma::distr_param(0, 4)); + +// Train in the constructor. +// Note that we specify decision stumps as the weak learner type, and pass +// hyperparameters for the decision stump (these could be omitted). See the +// DecisionTree documentation for more details on the ID3DecisionStump-specific +// hyperparameters. +mlpack::AdaBoost ab(dataset, labels, 5, + 25 /* maximum number of decision stumps */, + 1e-6 /* tolerance for convergence of AdaBoost */, + /** Hyperparameters specific to ID3DecisionStump: **/ + 10 /* minimum number of points in each leaf of the decision stump */, + 1e-5 /* minimum gain for splitting the root node of the decision stump */); + +// Create test data (500 points). +arma::mat testDataset(10, 500, arma::fill::randu); +arma::Row predictions; +ab.Classify(testDataset, predictions); +// Now `predictions` holds predictions for the test dataset. + +// Print some information about the test predictions. +std::cout << arma::accu(predictions == 3) << " test points classified as class " + << "3." << std::endl; +``` + +--- + +Train an AdaBoost model on 32-bit floating-point precision data (use a different +`MatType`). + +```c++ +// 1000 random points in 10 dimensions, using 32-bit precision (float). +arma::fmat dataset(10, 1000, arma::fill::randu); +// Random labels for each point, totaling 5 classes. +arma::Row labels = + arma::randi>(1000, arma::distr_param(0, 4)); + +// Train in the constructor, using floating-point data. +// The weak learner type is now a floating-point Perceptron. +typedef Perceptron + PerceptronType; +mlpack::AdaBoost ab(dataset, labels, 5); + +// Create test data (500 points). +arma::fmat testDataset(10, 500, arma::fill::randu); +arma::Row predictions; +ab.Classify(testDataset, predictions); +// Now `predictions` holds predictions for the test dataset. + +// Print some information about the test predictions. +std::cout << arma::accu(predictions == 3) << " test points classified as class " + << "3." << std::endl; +``` diff --git a/doc/user/methods/perceptron.md b/doc/user/methods/perceptron.md new file mode 100644 index 0000000000..679ef0bcda --- /dev/null +++ b/doc/user/methods/perceptron.md @@ -0,0 +1,418 @@ +## `Perceptron` + +The `Perceptron` class implements the simple perceptron classifier originally +implemented by Frank Rosenblatt in 1958. The perceptron is a linear classifier, +and can be understood as a trivial neural network with one neuron that uses the +step function as an activation function. mlpack's implementation of the +`Perceptron` class also offers several template parameters that can be used to +control the behavior of the perceptron. + +Perceptrons are useful for classifying points with _discrete labels_ (i.e., `0`, +`1`, `2`). Because they are simple classifiers, they are also useful as _weak +learners_ for the [`AdaBoost`](#adaboost) boosting classifier. + + +#### Simple usage example: + +```c++ +// Train a perceptron on random numeric data and predict labels on test data: + +// All data and labels are uniform random; 10 dimensional data, 5 classes. +// Replace with a data::Load() call or similar for a real application. +arma::mat dataset(10, 1000, arma::fill::randu); +arma::Row labels = + arma::randi>(1000, arma::distr_param(0, 4)); +arma::mat testDataset(10, 500, arma::fill::randu); // 500 test points. + +mlpack::Perceptron p; // Step 1: create model. +p.Train(dataset, labels, 5); // Step 2: train model. +arma::Row predictions; +p.Classify(testDataset, predictions); // Step 3: classify points. + +// Print some information about the test predictions. +std::cout << arma::accu(predictions == 1) << " test points classified as class " + << "1." << std::endl; +``` +

More examples...

+ +#### Quick links: + + * [Constructors](#constructors): create `DecisionTree` objects. + * [`Train()`](#training): train model. + * [`Classify()`](#classification): classify with a trained model. + * [Other functionality](#other-functionality) for loading, saving, and + inspecting. + * [Examples](#simple-examples) of simple usage and links to detailed example + projects. + * [Template parameters](#advanced-functionality-template-parameters) for custom + behavior. + * [Advanced template examples](#advanced-functionality-examples) of use with + custom template parameters. + +#### See also: + + * [`NaiveBayesClassifier`](#naive_bayes_classifier), another simple classifier + + * [`AdaBoost`](#adaboost) + * [`FFN`](#ffn) + * [mlpack classifiers](#mlpack_classifiers) + * [Perceptron on Wikipedia](https://en.wikipedia.org/wiki/Perceptron) + +### Constructors + +Construct a `Perceptron` object using one of the constructors below. Defaults +and types are detailed in the [Constructor Parameters](#constructor-parameters) +section below. + +#### Forms: + + * `p = Perceptron()` + - Initialize perceptron without training. + - You will need to call [`Train()`](#training) later to train the perceptron + before calling [`Classify()`](#classification). + +--- + + * `p = Perceptron(numClasses, dimensionality, maxIterations=1000)` + - Initialize perceptron with all-zero weights and biases. + - `Classify()` can immediately be used; training is not required with this + form. + +--- + + * `p = Perceptron(data, labels, numClasses, maxIterations=1000)` + * `p = Perceptron(data, labels, numClasses, weights, maxIterations=1000)` + - Train the perceptron (optionally with instance weights). + +--- + +#### Constructor Parameters: + + + +| **name** | **type** | **description** | **default** | +|----------|----------|-----------------|-------------| +| `data` | [`arma::mat`](../matrices.md) | [Column-major](../matrices.md) training matrix. | _(N/A)_ | +| `datasetInfo` | [`data::DatasetInfo`](../../tutorials/datasetmapper.md) | Dataset information, specifying type information for each dimension. | _(N/A)_ | +| `labels` | [`arma::Row`]('../matrices.md') | Training labels, between `0` and `numClasses - 1` (inclusive). Should have length `data.n_cols`. | _(N/A)_ | +| `weights` | [`arma::rowvec`]('../matrices.md') | Weights for each training point. Should have length `data.n_cols`. | _(N/A)_ | +| `numClasses` | `size_t` | Number of classes in the dataset. | _(N/A)_ | +| `dimensionality` | `size_t` | Dimensionality of data (only used if an initialized but untrained model is desired). | _(N/A)_ | +| `maxIterations` | `size_t` | Maximum number of iterations during training. Can also be set with `MaxIterations()`. | `1000` | + +As an alternative to passing `maxIterations`, it can be set with a standalone +method. The following function can be used before calling `Train()` to set +the maximum number of iterations: + + * `p.MaxIterations() = maxIter;` will set the maximum number of iterations + during training to `maxIter`. + +### Training + +If training is not done as part of the constructor call, it can be done with one +of the following versions of the `Train()` member function: + + * `p.Train(data, labels, numClasses, maxIterations=1000)` + - Train the perceptron on unweighted data. + +--- + + * `p.Train(data, labels, numClasses, weights, maxIterations=1000)` + - Train the perceptron on data with instance weights. + +--- + +Types of each argument are the same as in the table for constructors +[above](#constructor-parameters). + +***Notes***: + + * Training is incremental. Successive calls to `Train()` will not reinitialize + the model, unless the given data has different dimensionality or `numClasses` + is different. To reinitialize the model, call `Reset()` (see + [Other Functionality](#other-functionality)). + + * If `maxIterations` is not passed, but has been set in the constructor or with + `MaxIterations()`, the previous setting will be used. + +### Classification + +Once a `Perceptron` is trained, the `Classify()` member function can be used to +make class predictions for new data. + + * `size_t predictedClass = p.Classify(point)` + - ***(Single-point)*** + - Classify a single point, returning the predicted class. + +--- + + * `p.Classify(data, predictions)` + - ***(Multi-point)*** + - Classify a set of points. + - The prediction for data point `i` can be accessed with `predictions[i]`. + +--- + +***Note***: perceptrons do not provide any measure resembling probabilities +during classification, and thus a version of `Classify()` that computes class +probabilities is not available. + +#### Classification Parameters: + +| **usage** | **name** | **type** | **description** | +|-----------|----------|----------|-----------------| +| _single-point_ | `point` | [`arma::vec`](../matrices.md) | Single point for classification. | +|||| +| _multi-point_ | `data` | [`arma::mat`](../matrices.md) | Set of [column-major](../matrices.md) points for classification. | +| _multi-point_ | `predictions` | [`arma::Row&`](../matrices.md) | Vector of `size_t`s to store class prediction into. Will be set to length `data.n_cols`. | + +### Other Functionality + + + + * A `Perceptron` can be serialized with [`data::Save()`](../formats.md) and + [`data::Load()`](../formats.md). + + * `p.NumClasses()` will return a `size_t` indicating the number of classes the + perceptron was trained on. + + * `p.Biases()` will return an `arma::vec` with the biases of the model (each + element corresponds to the bias for a class). + + * `p.Weights()` will return an `arma::mat` with the weights of the model (each + column corresponds to the weights for one class label). + + * `p.Reset()` will re-initialize the weights and biases of the model. + +For complete functionality, the [source +code](/src/mlpack/methods/perceptron/perceptron.hpp) can be consulted. Each +method is fully documented. + +### Simple Examples + +See also the [simple usage example](#simple-usage-example) for a trivial use of +`Perceptron`. + +--- + +Train a perceptron multiple times, incrementally, with custom hyperparameters. + +```c++ +// See https://datasets.mlpack.org/iris.csv. +arma::mat dataset; +mlpack::data::Load("iris.csv", dataset, true); +// See https://datasets.mlpack.org/iris.labels.csv. +arma::Row labels; +mlpack::data::Load("iris.labels.csv", labels, true); + +// Create a Perceptron object. +mlpack::Perceptron p; +// Set the maximum number of iterations to 100. (This can also be done in the +// constructor.) +p.MaxIterations() = 100; + +// Train the model for up to 100 iterations. +p.Train(dataset, labels, 3); + +// Now, compute and print accuracy on the training set. +arma::Row predictions; +p.Classify(dataset, predictions); +std::cout << "Training set accuracy after 100 iterations: " + << (100.0 * double(arma::accu(labels == predictions)) / labels.n_elem) + << "\%." << std::endl; + +// Train for another 250 iterations and compute training set accuracy again. +p.Train(dataset, labels, 3, 250); +p.Classify(dataset, predictions); +std::cout << "Training set accuracy after 350 iterations: " + << (100.0 * double(arma::accu(labels == predictions)) / labels.n_elem) + << "\%." << std::endl; +``` + +--- + +Load a saved perceptron from disk and print information about it. + +```c++ +mlpack::Perceptron p; +// This call assumes a perceptron called "p" has already been saved to +// `perceptron.bin` with `data::Save()`. +mlpack::data::Load("perceptron.bin", "p", p, true); + +if (p.NumClasses() > 0) +{ + std::cout << "The perceptron in `perceptron.bin` was trained on " + << p.NumClasses() << " classes." << std::endl; + std::cout << "The dimensionality of the perceptron model is " + << p.Weights().n_rows << "." << std::endl; + std::cout << "The bias weights for each class are:" << std::endl; + for (size_t i = 0; i < p.NumClasses(); ++i) + std::cout << " - Class " << i << ": " << p.Biases()[i] << std::endl; +} +else +{ + std::cout << "The perceptron in `perceptron.bin` has not been trained." + << std::endl; +} +``` + +--- + +### Advanced Functionality: Template Parameters + +The `Perceptron` class also supports several template parameters, which can be +used for custom behavior. The full signature of the class is as follows: + +```c++ +Perceptron +``` + + * `LearnPolicy`: the strategy used to learn the weights during training. + * `WeightInitializationPolicy`: the way that weights are initialized before + training. + * `MatType`: specifies the type of matrix used for learning and internal + representation of weights and biases. + +--- + +#### `LearnPolicy` + + * Specifies the step to be taken when a point is misclassified. + * The `SimpleWeightUpdate` class is available, and is the default. + * A custom class must implement only one function: + +```c++ +// You can use this as a starting point for implementation. +class CustomLearnPolicy +{ + // Update the weights and biases in the `weights` matrix and the `biases` + // vector given that the model currently classified `trainingPoint` as having + // the label `incorrectClass`, when in reality it has the label + // `correctClass`. If `instanceWeight` is given, it specifies the instance + // weight for the given `trainingPoint`. + // + // `VecType` will be an Armadillo-like vector type. It will be a column from + // the training data matrix (`data`) given to `Train()` or to the constructor. + // + // `eT` is the element type of the Perceptron (e.g. `float`, `double`). + template + void UpdateWeights(const VecType& trainingPoint, + arma::Mat& weights, + arma::Col& biases, + const size_t incorrectClass, + const size_t correctClass, + const double instanceWeight = 1.0); +}; +``` + +--- + +#### `WeightInitializationPolicy` + + * Specifies how the weights matrix and biases vector should be initialized when + the `Perceptron` object is created, or when `Reset()` is called. + * The `ZeroInitialization` _(default)_ and `RandomPerceptronInitialization` + classes are available for drop-in usage. + * `RandomPerceptronInitialization` will initialize weights and biases using a + uniform random distribution between 0 and 1. + * A custom class must implement only one function: + +```c++ +// You can use this as a starting point for implementation. +class CustomWeightInitializationPolicy +{ + // Initialize the `weights` matrix and `biases` vector, given that the model + // will have dimensionality of `numFeatures` (that is, the training data + // matrix will have `numFeatures` rows), and the training data has + // `numClasses` classes. + // + // The initialized `weights` matrix should have `numFeatures` rows and + // `numClasses` columns, and the initialized `biases` vector should have + // `numClasses` elements. + // + // `eT` specifies the element type of the weights and biases; it may be + // `double`, `float`, or another floating-point type. + template + inline static void Initialize(arma::Mat& weights, + arma::Col& biases, + const size_t numFeatures, + const size_t numClasses) + { + weights.randu(numFeatures, numClasses); + biases.randu(numClasses); + } +}; +``` + +--- + +#### `MatType` + + * Specifies the matrix type to use for data when learning a perceptron. + * By default, `MatType` is `arma::mat` (dense 64-bit precision matrix). + * Any matrix type implementing the Armadillo API will work; so, for instance, + `arma::fmat` or `arma::sp_mat` can be used. + +### Advanced Functionality Examples + +Train a `Perceptron` with random initialization, instead of zero initialization +of weights. + +```c++ +// 1000 random points in 10 dimensions. +arma::mat dataset(10, 1000, arma::fill::randu); +// Random labels for each point, totaling 5 classes. +arma::Row labels = + arma::randi>(1000, arma::distr_param(0, 4)); + +// Train in the constructor. Weights will be initialized randomly. +mlpack::Perceptron p( + dataset, labels, 5); + +// Create test data (500 points). +arma::mat testDataset(10, 500, arma::fill::randu); +arma::Row predictions; +p.Classify(testDataset, predictions); +// Now `predictions` holds predictions for the test dataset. + +// Print some information about the test predictions. +std::cout << arma::accu(predictions == 1) << " test points classified as class " + << "1." << std::endl; +``` + +--- + +Train a `Perceptron` on sparse 32-bit floating point data. + +```c++ + +// 1000 sparse random points in 100 dimensions, with 1% nonzero elements. +arma::sp_fmat dataset; +dataset.sprandu(100, 1000, 0.01); +// Random labels for each point, totaling 5 classes. +arma::Row labels = + arma::randi>(1000, arma::distr_param(0, 4)); + +// Train in the constructor. +mlpack::Perceptron p(dataset, labels, 5); + +// Create test data (500 points). +arma::sp_fmat testDataset; +testDataset.sprandu(100, 500, 0.01); +arma::Row predictions; +p.Classify(testDataset, predictions); +// Now `predictions` holds predictions for the test dataset. + +// Print some information about the test predictions. +std::cout << arma::accu(predictions == 1) << " test points classified as class " + << "1." << std::endl; +``` + +--- diff --git a/src/mlpack/core/cereal/template_class_version.hpp b/src/mlpack/core/cereal/template_class_version.hpp new file mode 100644 index 0000000000..bb68af0705 --- /dev/null +++ b/src/mlpack/core/cereal/template_class_version.hpp @@ -0,0 +1,78 @@ +/** + * @file core/cereal/template_class_version.hpp + * @author Ryan Curtin + * + * Implementation of CEREAL_TEMPLATE_CLASS_VERSION() macro, useful for + * templatized types where CEREAL_CLASS_VERSION() will not work. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_CORE_CEREAL_TEMPLATE_CLASS_VERSION_HPP +#define MLPACK_CORE_CEREAL_TEMPLATE_CLASS_VERSION_HPP + +#include + +// This useful implementation is adapted from @lubensky on Github: +// https://github.com/uscilab/cereal/issues/319#issuecomment-1512927210 + +#define CEREAL_UNPACK(...) __VA_ARGS__ + +#ifdef MLPACK_HAVE_CXX17 + +// The C++17 version sets `version` as `inline`. +#define CEREAL_TEMPLATE_CLASS_VERSION(ARGS, TYPE, VERSION_NUMBER) \ +namespace cereal { \ +namespace detail { \ +template \ +struct Version \ +{ \ + static std::uint32_t registerVersion() \ + { \ + ::cereal::detail::StaticObject::getInstance().mapping.emplace( \ + std::type_index(typeid(CEREAL_UNPACK TYPE)).hash_code(), \ + CEREAL_UNPACK VERSION_NUMBER); \ + return CEREAL_UNPACK VERSION_NUMBER; \ + } \ + \ + static inline const std::uint32_t version = registerVersion(); \ + \ + static void unused() { (void) version; } \ +}; /* end Version */ \ + \ +} \ +} + +#else + +// Here we cannot use inline variables. +#define CEREAL_TEMPLATE_CLASS_VERSION(ARGS, TYPE, VERSION_NUMBER) \ +namespace cereal { \ +namespace detail { \ +template \ +struct Version \ +{ \ + static const std::uint32_t version; \ + static std::uint32_t registerVersion() \ + { \ + ::cereal::detail::StaticObject::getInstance().mapping.emplace( \ + std::type_index(typeid(CEREAL_UNPACK TYPE)).hash_code(), \ + CEREAL_UNPACK VERSION_NUMBER); \ + return CEREAL_UNPACK VERSION_NUMBER; \ + } \ + \ + static void unused() { (void) version; } \ +}; /* end Version */ \ + \ +template \ +const std::uint32_t Version::version = \ + Version::registerVersion(); \ + \ +} \ +} + +#endif // MLPACK_HAVE_CXX17 + +#endif // TEMPLATE_CLASS_VERSION_HPP diff --git a/src/mlpack/core/util/size_checks.hpp b/src/mlpack/core/util/size_checks.hpp index f5125a591b..7f62a0b450 100644 --- a/src/mlpack/core/util/size_checks.hpp +++ b/src/mlpack/core/util/size_checks.hpp @@ -32,13 +32,16 @@ namespace util { * before size-check. Default is false. */ template -inline void CheckSameSizes(const DataType& data, - const LabelsType& label, - const std::string& callerDescription, - const std::string& addInfo = "labels", - const bool& isDataTranspose = false, - const bool& isLabelTranspose = false) -{ +inline void CheckSameSizes( + const DataType& data, + const LabelsType& label, + const std::string& callerDescription, + const std::string& addInfo = "labels", + const bool& isDataTranspose = false, + const bool& isLabelTranspose = false, + const typename std::enable_if< + !std::is_integral::value>::type* = 0) +{ const size_t dataPoints = (isDataTranspose == true) ? data.n_rows : data.n_cols; const size_t labelPoints = (isLabelTranspose == true) ? label.n_rows : label.n_cols; @@ -56,11 +59,13 @@ inline void CheckSameSizes(const DataType& data, * An overload of CheckSameSizes() where the size to be checked is known * previously. The second parameter is of type unsigned int. */ -template -inline void CheckSameSizes(const DataType& data, - const size_t& size, - const std::string& callerDescription, - const std::string& addInfo = "labels") +template +inline void CheckSameSizes( + const DataType& data, + const SizeType& size, + const std::string& callerDescription, + const std::string& addInfo = "labels", + const typename std::enable_if::value>::type* = 0) { if (data.n_cols != size) { @@ -84,10 +89,12 @@ inline void CheckSameSizes(const DataType& data, * is "dataset"; for example, "weights" could also be used. */ template -inline void CheckSameDimensionality(const DataType& data, - const DimType& dimension, - const std::string& callerDescription, - const std::string& addInfo = "dataset") +inline void CheckSameDimensionality( + const DataType& data, + const DimType& dimension, + const std::string& callerDescription, + const std::string& addInfo = "dataset", + const typename std::enable_if::value>::type* = 0) { if (data.n_rows != dimension.n_rows) { @@ -104,11 +111,13 @@ inline void CheckSameDimensionality(const DataType& data, * An overload of CheckSameDimensionality() where the dimension to be checked * is known second param is unsigned long int. */ -template -inline void CheckSameDimensionality(const DataType& data, - const size_t& dimension, - const std::string& callerDescription, - const std::string& addInfo = "dataset") +template +inline void CheckSameDimensionality( + const DataType& data, + const DimType& dimension, + const std::string& callerDescription, + const std::string& addInfo = "dataset", + const typename std::enable_if::value>::type* = 0) { if (data.n_rows != dimension) { diff --git a/src/mlpack/methods/adaboost/adaboost.hpp b/src/mlpack/methods/adaboost/adaboost.hpp index 9bf94aab79..72d21d6031 100644 --- a/src/mlpack/methods/adaboost/adaboost.hpp +++ b/src/mlpack/methods/adaboost/adaboost.hpp @@ -80,6 +80,35 @@ template, class AdaBoost { public: + typedef typename MatType::elem_type ElemType; + + /** + * Create the AdaBoost object without training. Be sure to call Train() + * before calling Classify()! + */ + AdaBoost(const ElemType tolerance = 1e-6); + + /** + * Construct an AdaBoost model. Any extra parameters are used as + * hyperparameters for the weak learner. These should be the last arguments + * to the weak learner's constructor or `Train()` function (i.e. anything + * after `numClasses` or `weights`). + * + * @param data Input data. + * @param labels Corresponding labels. + * @param numClasses The number of classes. + * @param maxIterations Number of boosting rounds. + * @param tolerance The tolerance for change in values of rt. + * @param weakLearnerParams... Any hyperparameters for the weak learner. + */ + template + AdaBoost(const MatType& data, + const arma::Row& labels, + const size_t numClasses, + const size_t maxIterations = 100, + const ElemType tolerance = 1e-6, + WeakLearnerArgs&&... weakLearnerArgs); + /** * Constructor. This runs the AdaBoost.MH algorithm to provide a trained * boosting model. This constructor takes an already-initialized weak @@ -89,27 +118,31 @@ class AdaBoost * @param data Input data. * @param labels Corresponding labels. * @param numClasses The number of classes. - * @param iterations Number of boosting rounds. + * @param maxIterations Number of boosting rounds. * @param tolerance The tolerance for change in values of rt. * @param other Weak learner that has already been initialized. */ + template + mlpack_deprecated /* to be removed in mlpack 5.0.0 */ AdaBoost(const MatType& data, const arma::Row& labels, const size_t numClasses, - const WeakLearnerType& other, - const size_t iterations = 100, - const double tolerance = 1e-6); + const WeakLearnerInType& other, + const size_t maxIterations = 100, + const ElemType tolerance = 1e-6, + const typename std::enable_if< + std::is_same::value + >::type* = 0); - /** - * Create the AdaBoost object without training. Be sure to call Train() - * before calling Classify()! - */ - AdaBoost(const double tolerance = 1e-6); + //! Get the maximum number of weak learners allowed in the model. + size_t MaxIterations() const { return maxIterations; } + //! Modify the maximum number of weak learners allowed in the model. + size_t& MaxIterations() { return maxIterations; } //! Get the tolerance for stopping the optimization during training. - double Tolerance() const { return tolerance; } + ElemType Tolerance() const { return tolerance; } //! Modify the tolerance for stopping the optimization during training. - double& Tolerance() { return tolerance; } + ElemType& Tolerance() { return tolerance; } //! Get the number of classes this model is trained on. size_t NumClasses() const { return numClasses; } @@ -118,9 +151,9 @@ class AdaBoost size_t WeakLearners() const { return alpha.size(); } //! Get the weights for the given weak learner. - double Alpha(const size_t i) const { return alpha[i]; } + ElemType Alpha(const size_t i) const { return alpha[i]; } //! Modify the weight for the given weak learner (be careful!). - double& Alpha(const size_t i) { return alpha[i]; } + ElemType& Alpha(const size_t i) { return alpha[i]; } //! Get the given weak learner. const WeakLearnerType& WeakLearner(const size_t i) const { return wl[i]; } @@ -134,20 +167,118 @@ class AdaBoost * completely overwrite any model that has already been trained with this * object. * + * Default values are not used for `maxIterations` and `tolerance`; instead, + * multiple overloads are allowed; this is because we want to use the existing + * setting internal to the class, if one is not specified. + * * @param data Dataset to train on. * @param labels Labels for each point in the dataset. * @param numClasses The number of classes. * @param learner Learner to use for training. - * @param iterations Number of boosting rounds. + * @param maxIterations Number of boosting rounds. * @param tolerance The tolerance for change in values of rt. * @return The upper bound for training error. */ - double Train(const MatType& data, - const arma::Row& labels, - const size_t numClasses, - const WeakLearnerType& learner, - const size_t iterations = 100, - const double tolerance = 1e-6); + template + mlpack_deprecated /* to be removed in mlpack 5.0.0 */ + ElemType Train( + const MatType& data, + const arma::Row& labels, + const size_t numClasses, + const WeakLearnerInType& learner, + // Necessary to distinguish from other overloads. + const typename std::enable_if< + std::is_same::value>::type* = 0); + + template + mlpack_deprecated /* to be removed in mlpack 5.0.0 */ + ElemType Train( + const MatType& data, + const arma::Row& labels, + const size_t numClasses, + const WeakLearnerInType& learner, + const size_t maxIterations, + // Necessary to distinguish from other overloads. + const typename std::enable_if< + std::is_same::value>::type* = 0); + + template + mlpack_deprecated /* to be removed in mlpack 5.0.0 */ + ElemType Train( + const MatType& data, + const arma::Row& labels, + const size_t numClasses, + const WeakLearnerInType& learner, + const size_t maxIterations, + const double tolerance, + // Necessary to distinguish from other overloads. + const typename std::enable_if< + std::is_same::value>::type* = 0); + + /** + * Train AdaBoost on the given dataset, using the given parameters. The last + * parameters are the hyperparameters to use for the weak learners; these are + * all the arguments to `WeakLearnerType::Train()` after `numClasses` and + * `weights`. + * + * Default values are not used for `maxIterations` and `tolerance`; instead, + * multiple overloads are allowed; this is because we want to use the existing + * setting internal to the class, if one is not specified. + * + * @param data Dataset to train on. + * @param labels Labels for each point in the dataset. + * @param numClasses The number of classes in the dataset. + * @param maxIterations Number of boosting rounds. + * @param tolerance The tolerance for change in values of rt. + * @param weakLearnerArgs Hyperparameters to use for each weak learner. + * @return The upper bound for training error. + */ + ElemType Train(const MatType& data, + const arma::Row& labels, + const size_t numClasses); + + ElemType Train(const MatType& data, + const arma::Row& labels, + const size_t numClasses, + const size_t maxIterations); + + template + ElemType Train(const MatType& data, + const arma::Row& labels, + const size_t numClasses, + const size_t maxIterations, + const double tolerance, + WeakLearnerArgs&&... weakLearnerArgs); + + /** + * Classify the given test point. + * + * @param point Test point. + */ + template + size_t Classify(const VecType& point) const; + + /** + * Classify the given test point and compute class probabilities. + * + * @param point Test point. + * @param prediction Will be filled with the predicted class of `point`. + * @param probabilities Will be filled with the class probabilities. + */ + template + void Classify(const VecType& point, + size_t& prediction, + arma::Row& probabilities) const; + + /** + * Classify the given test points. + * + * @param test Testing data. + * @param predictedLabels Vector in which the predicted labels of the test + * set will be stored. + */ + void Classify(const MatType& test, + arma::Row& predictedLabels) const; /** * Classify the given test points. @@ -160,17 +291,7 @@ class AdaBoost */ void Classify(const MatType& test, arma::Row& predictedLabels, - arma::mat& probabilities); - - /** - * Classify the given test points. - * - * @param test Testing data. - * @param predictedLabels Vector in which the predicted labels of the test - * set will be stored. - */ - void Classify(const MatType& test, - arma::Row& predictedLabels); + arma::Mat& probabilities) const; /** * Serialize the AdaBoost model. @@ -179,19 +300,36 @@ class AdaBoost void serialize(Archive& ar, const uint32_t /* version */); private: + /** + * Internal utility training function. `wl` is not used if + * `UseExistingWeakLearner` is false. `weakLearnerArgs` are not used if + * `UseExistingWeakLearner` is true. + */ + template + ElemType TrainInternal(const MatType& data, + const arma::Row& labels, + const size_t numClasses, + const WeakLearnerType& wl, + WeakLearnerArgs&&... weakLearnerArgs); + //! The number of classes in the model. size_t numClasses; - // The tolerance for change in rt and when to stop. - double tolerance; + //! The maximum number of weak learners allowed in the model. + size_t maxIterations; + //! The tolerance for change in rt and when to stop. + ElemType tolerance; //! The vector of weak learners. std::vector wl; //! The weights corresponding to each weak learner. - std::vector alpha; + std::vector alpha; }; // class AdaBoost } // namespace mlpack +CEREAL_TEMPLATE_CLASS_VERSION((typename WeakLearnerType, typename MatType), + (mlpack::AdaBoost), (1)); + // Include implementation. #include "adaboost_impl.hpp" diff --git a/src/mlpack/methods/adaboost/adaboost_impl.hpp b/src/mlpack/methods/adaboost/adaboost_impl.hpp index ffa5a4bddd..595dc8956a 100644 --- a/src/mlpack/methods/adaboost/adaboost_impl.hpp +++ b/src/mlpack/methods/adaboost/adaboost_impl.hpp @@ -30,45 +30,353 @@ namespace mlpack { -/** - * Constructor. Currently runs the AdaBoost.MH algorithm. - * - * @param data Input data - * @param labels Corresponding labels - * @param iterations Number of boosting rounds - * @param tol Tolerance for termination of Adaboost.MH. - * @param other Weak Learner, which has been initialized already. - */ -template -AdaBoost::AdaBoost( - const MatType& data, - const arma::Row& labels, - const size_t numClasses, - const WeakLearnerType& other, - const size_t iterations, - const double tol) -{ - Train(data, labels, numClasses, other, iterations, tol); -} - // Empty constructor. template -AdaBoost::AdaBoost(const double tolerance) : +AdaBoost::AdaBoost(const ElemType tolerance) : numClasses(0), tolerance(tolerance) { // Nothing to do. } +/** + * Constructor. Runs the AdaBoost.MH algorithm. + * + * @param data Input data + * @param labels Corresponding labels + * @param maxIterations Number of boosting rounds + * @param tol Tolerance for termination of Adaboost.MH. + * @param other Weak Learner, which has been initialized already. + */ +template +template +AdaBoost::AdaBoost( + const MatType& data, + const arma::Row& labels, + const size_t numClasses, + const WeakLearnerInType& other, + const size_t maxIterations, + const typename MatType::elem_type tol, + const typename std::enable_if< + std::is_same::value>::type*) : + maxIterations(maxIterations), + tolerance(tol) +{ + (void) TrainInternal(data, labels, numClasses, other); +} + +/** + * Constructor. Runs the AdaBoost.MH algorithm. + * + * @param data Input data + * @param labels Corresponding labels + * @param maxIterations Number of boosting rounds + * @param tol Tolerance for termination of Adaboost.MH. + * @param other Weak Learner, which has been initialized already. + */ +template +template +AdaBoost::AdaBoost( + const MatType& data, + const arma::Row& labels, + const size_t numClasses, + const size_t maxIterations, + const typename MatType::elem_type tol, + WeakLearnerArgs&&... weakLearnerArgs) : + maxIterations(maxIterations), + tolerance(tol) +{ + WeakLearnerType other; // Will not be used. + (void) TrainInternal(data, labels, numClasses, other, + weakLearnerArgs...); +} + +// Train AdaBoost with a given weak learner. +template +template +typename MatType::elem_type AdaBoost::Train( + const MatType& data, + const arma::Row& labels, + const size_t numClasses, + const WeakLearnerInType& other, + const typename std::enable_if< + std::is_same::value>::type*) +{ + return TrainInternal(data, labels, numClasses, other); +} + +// Train AdaBoost with a given weak learner, and set the maximum number of +// iterations. +template +template +typename MatType::elem_type AdaBoost::Train( + const MatType& data, + const arma::Row& labels, + const size_t numClasses, + const WeakLearnerInType& other, + const size_t maxIterations, + const typename std::enable_if< + std::is_same::value>::type*) +{ + this->maxIterations = maxIterations; + return TrainInternal(data, labels, numClasses, other); +} + +// Train AdaBoost with a given weak learner, and set the maximum number of +// iterations and tolerance. +template +template +typename MatType::elem_type AdaBoost::Train( + const MatType& data, + const arma::Row& labels, + const size_t numClasses, + const WeakLearnerInType& other, + const size_t maxIterations, + const double tolerance, + const typename std::enable_if< + std::is_same::value>::type*) +{ + this->maxIterations = maxIterations; + this->tolerance = tolerance; + return TrainInternal(data, labels, numClasses, other); +} + // Train AdaBoost. template -double AdaBoost::Train( +typename MatType::elem_type AdaBoost::Train( + const MatType& data, + const arma::Row& labels, + const size_t numClasses) +{ + WeakLearnerType other; // Will not be used. + return TrainInternal(data, labels, numClasses, other); +} + +// Train AdaBoost, and set the maximum number of iterations. +template +typename MatType::elem_type AdaBoost::Train( + const MatType& data, + const arma::Row& labels, + const size_t numClasses, + const size_t maxIterations) +{ + this->maxIterations = maxIterations; + + WeakLearnerType other; // Will not be used. + return TrainInternal(data, labels, numClasses, other); +} +// Train AdaBoost. +template +template +typename MatType::elem_type AdaBoost::Train( + const MatType& data, + const arma::Row& labels, + const size_t numClasses, + const size_t maxIterations, + const double tolerance, + WeakLearnerArgs&&... weakLearnerArgs) +{ + this->maxIterations = maxIterations; + this->tolerance = tolerance; + WeakLearnerType other; // Will not be used. + return TrainInternal(data, labels, numClasses, other, + weakLearnerArgs...); +} + +// Classify the given test point. +template +template +size_t AdaBoost::Classify(const VecType& point) const +{ + arma::Row probabilities; + size_t prediction; + Classify(point, prediction, probabilities); + + return prediction; +} + +// Classify the given test point and return class probabilities. +template +template +void AdaBoost::Classify( + const VecType& point, + size_t& prediction, + arma::Row& probabilities) const +{ + probabilities.zeros(numClasses); + for (size_t i = 0; i < wl.size(); ++i) + { + prediction = wl[i].Classify(point); + probabilities(prediction) += alpha[i]; + } + + arma::uword maxIndex = 0; + probabilities /= arma::accu(probabilities); + probabilities.max(maxIndex); + prediction = (size_t) maxIndex; +} + +// Classify the given test points. +template +void AdaBoost::Classify( + const MatType& test, + arma::Row& predictedLabels) const +{ + arma::Row tempPredictedLabels(test.n_cols); + arma::Mat probabilities; + + Classify(test, predictedLabels, probabilities); +} + +// Classify the given test points. +template +void AdaBoost::Classify( + const MatType& test, + arma::Row& predictedLabels, + arma::Mat& probabilities) const +{ + probabilities.zeros(numClasses, test.n_cols); + predictedLabels.set_size(test.n_cols); + + for (size_t i = 0; i < wl.size(); ++i) + { + wl[i].Classify(test, predictedLabels); + + for (size_t j = 0; j < predictedLabels.n_cols; ++j) + probabilities(predictedLabels(j), j) += alpha[i]; + } + + arma::uword maxIndex = 0; + + for (size_t i = 0; i < predictedLabels.n_cols; ++i) + { + probabilities.col(i) /= arma::accu(probabilities.col(i)); + probabilities.col(i).max(maxIndex); + predictedLabels(i) = maxIndex; + } +} + +/** + * Serialize the AdaBoost model. + */ +template +template +void AdaBoost::serialize(Archive& ar, + const uint32_t version) +{ + // Between version 0 and 1, the maxIterations member was added, and `alpha` + // was switched to type arma::Row instead of arma::rowvec. These + // require a little bit of special handling when loading older versions. + if (cereal::is_loading() && version == 0) + { + // This is the legacy version. + ar(CEREAL_NVP(numClasses)); + ar(CEREAL_NVP(tolerance)); + ar(CEREAL_NVP(alpha)); + + // In earlier versions, `alpha` was a vector of doubles---but it might not + // be now. + if (std::is_same::value) + { + ar(CEREAL_NVP(alpha)); // The easy case. + } + else + { + arma::rowvec alphaTmp; + // Avoid CEREAL_NVP so we can specify a custom name. + ar(cereal::make_nvp("alpha", alphaTmp)); + alpha.clear(); + alpha.resize(alphaTmp.size()); + for (size_t i = 0; i < alphaTmp.size(); ++i) + alpha[i] = (ElemType) alphaTmp[i]; + } + + // Now serialize each weak learner. + ar(CEREAL_NVP(wl)); + + // Attempt to set maxIterations to something reasonable. + maxIterations = std::max((size_t) 100, alpha.size()); + } + else + { + // This is the current version. + // (Once there is a major version bump, we should make this version 0.) + ar(CEREAL_NVP(numClasses)); + ar(CEREAL_NVP(tolerance)); + ar(CEREAL_NVP(maxIterations)); + ar(CEREAL_NVP(alpha)); + + // Now serialize each weak learner. + ar(CEREAL_NVP(wl)); + } +} + +template< + bool UseExistingWeakLearner, + typename MatType, + typename WeightsType, + typename WeakLearnerType, + typename... WeakLearnerArgs +> +struct WeakLearnerTrainer +{ + static WeakLearnerType Train( + const MatType& data, + const arma::Row& labels, + const size_t numClasses, + const WeightsType& weights, + const WeakLearnerType& wl, + WeakLearnerArgs&&... /* weakLearnerArgs */) + { + // Use the existing weak learner to train a new one with new weights. + // API requirement: there is a constructor with this signature: + // + // WeakLearnerType(const WeakLearnerType&, + // MatType& data, + // LabelsType& labels, + // const size_t numClasses, + // WeightsType& weights) + // + // This trains the new WeakLearnerType using the hyperparameters from the + // given WeakLearnerType. + return WeakLearnerType(wl, data, labels, numClasses, weights); + } +}; + +template< + typename MatType, + typename WeightsType, + typename WeakLearnerType, + typename... WeakLearnerArgs +> +struct WeakLearnerTrainer< + false, MatType, WeightsType, WeakLearnerType, WeakLearnerArgs... +> +{ + static WeakLearnerType Train( + const MatType& data, + const arma::Row& labels, + const size_t numClasses, + const WeightsType& weights, + const WeakLearnerType& /* wl */, + WeakLearnerArgs&&... weakLearnerArgs) + { + // When UseExistingWeakLearner is false, we use the given hyperparameters. + // (This is the preferred approach that supports more types of weak + // learners.) + return WeakLearnerType(data, labels, numClasses, weights, + weakLearnerArgs...); + } +}; + +template +template +typename MatType::elem_type AdaBoost::TrainInternal( const MatType& data, const arma::Row& labels, const size_t numClasses, const WeakLearnerType& other, - const size_t iterations, - const double tolerance) + WeakLearnerArgs&&... weakLearnerArgs) { // Clear information from previous runs. wl.clear(); @@ -79,9 +387,9 @@ double AdaBoost::Train( // crt is the cumulative rt value for terminating the optimization when rt is // changing by less than the tolerance. - double rt, crt = 0.0, alphat = 0.0, zt; + ElemType rt, crt = 0.0, alphat = 0.0, zt; - double ztProduct = 1.0; + ElemType ztProduct = 1.0; // To be used for prediction by the weak learner. arma::Row predictedLabels(labels.n_cols); @@ -90,22 +398,22 @@ double AdaBoost::Train( MatType tempData(data); // This matrix is a helper matrix used to calculate the final hypothesis. - arma::mat sumFinalH = arma::zeros(numClasses, - predictedLabels.n_cols); + MatType sumFinalH(numClasses, predictedLabels.n_cols); + sumFinalH.zeros(); // Load the initial weights into a 2-D matrix. - const double initWeight = 1.0 / double(data.n_cols * numClasses); - arma::mat D(numClasses, data.n_cols); + const ElemType initWeight = 1.0 / ElemType(data.n_cols * numClasses); + MatType D(numClasses, data.n_cols); D.fill(initWeight); // Weights are stored in this row vector. - arma::rowvec weights(predictedLabels.n_cols); + arma::Row weights(predictedLabels.n_cols); // This is the final hypothesis. arma::Row finalH(predictedLabels.n_cols); // Now, start the boosting rounds. - for (size_t i = 0; i < iterations; ++i) + for (size_t i = 0; i < maxIterations; ++i) { // Initialized to zero in every round. rt is used for calculation of // alphat; it is the weighted error. @@ -118,31 +426,17 @@ double AdaBoost::Train( // Build the weight vectors. weights = arma::sum(D); - // Use the existing weak learner to train a new one with new weights. - // API requirement: there is a constructor with this signature: - // - // WeakLearnerType(const WeakLearnerType&, - // MatType& data, - // LabelsType& labels, - // const size_t numClasses, - // WeightsType& weights) - // - // This trains the new WeakLearnerType using the hyperparameters from the - // given WeakLearnerType. + // This is split into a separate function, so that we can still call + // AdaBoost::Train() with extra hyperparameters, even when the weak learner + // type does not support the special constructor that takes another weak + // learner. + WeakLearnerType w = WeakLearnerTrainer< + UseExistingWeakLearner, MatType, arma::Row, WeakLearnerType, + WeakLearnerArgs... + >::Train(tempData, labels, numClasses, weights, other, weakLearnerArgs...); - WeakLearnerType w(other, tempData, labels, numClasses, weights); - // There is a bug with Adaboost! It will not use the specified - // hyperparameters for the decision tree because they are not properly - // passed to the new weak learners! (And: it's a hard bug, because the - // decision tree itself doesn't even store the hyperparameters it was - // trained with!) - - // DecisionTree(DecisionTree&, MatType&, LabelsType&, size_t, WeightsType&, double = 0.0, double = 0.0, ...); w.Classify(tempData, predictedLabels); - // Now from predictedLabels, build ht, the weak hypothesis - // buildClassificationMatrix(ht, predictedLabels); - // Now, calculate alpha(t) using ht. for (size_t j = 0; j < D.n_cols; ++j) // instead of D, ht { @@ -166,7 +460,7 @@ double AdaBoost::Train( crt = rt; - // Our goal is to find alphat which mizimizes or approximately minimizes the + // Our goal is to find alphat which minimizes or approximately minimizes the // value of Z as a function of alpha. alphat = 0.5 * log((1 + rt) / (1 - rt)); @@ -176,7 +470,7 @@ double AdaBoost::Train( // Now start modifying the weights. for (size_t j = 0; j < D.n_cols; ++j) { - const double expo = exp(alphat); + const ElemType expo = exp(alphat); if (predictedLabels(j) == labels(j)) { for (size_t k = 0; k < D.n_rows; ++k) @@ -216,76 +510,10 @@ double AdaBoost::Train( // Accumulate the value of zt for the Hamming loss bound. ztProduct *= zt; } + return ztProduct; } -/** - * Classify the given test points. - */ -template -void AdaBoost::Classify( - const MatType& test, - arma::Row& predictedLabels) -{ - arma::Row tempPredictedLabels(test.n_cols); - arma::mat probabilities; - - Classify(test, predictedLabels, probabilities); -} - -/** - * Classify the given test points. - */ -template -void AdaBoost::Classify( - const MatType& test, - arma::Row& predictedLabels, - arma::mat& probabilities) -{ - arma::Row tempPredictedLabels(test.n_cols); - - probabilities.zeros(numClasses, test.n_cols); - predictedLabels.set_size(test.n_cols); - - for (size_t i = 0; i < wl.size(); ++i) - { - wl[i].Classify(test, tempPredictedLabels); - - for (size_t j = 0; j < tempPredictedLabels.n_cols; ++j) - probabilities(tempPredictedLabels(j), j) += alpha[i]; - } - - arma::uword maxIndex = 0; - - for (size_t i = 0; i < predictedLabels.n_cols; ++i) - { - probabilities.col(i) /= arma::accu(probabilities.col(i)); - probabilities.col(i).max(maxIndex); - predictedLabels(i) = maxIndex; - } -} - -/** - * Serialize the AdaBoost model. - */ -template -template -void AdaBoost::serialize(Archive& ar, - const uint32_t /* version */) -{ - ar(CEREAL_NVP(numClasses)); - ar(CEREAL_NVP(tolerance)); - ar(CEREAL_NVP(alpha)); - - // Now serialize each weak learner. - if (cereal::is_loading()) - { - wl.clear(); - wl.resize(alpha.size()); - } - ar(CEREAL_NVP(wl)); -} - } // namespace mlpack #endif diff --git a/src/mlpack/methods/adaboost/adaboost_model_impl.hpp b/src/mlpack/methods/adaboost/adaboost_model_impl.hpp index e12dc39694..7a99e13f36 100644 --- a/src/mlpack/methods/adaboost/adaboost_model_impl.hpp +++ b/src/mlpack/methods/adaboost/adaboost_model_impl.hpp @@ -124,15 +124,13 @@ inline void AdaBoostModel::Train(const arma::mat& data, if (weakLearnerType == WeakLearnerTypes::DECISION_STUMP) { delete dsBoost; - ID3DecisionStump ds(data, labels, max(labels) + 1); - dsBoost = new AdaBoost(data, labels, numClasses, ds, + dsBoost = new AdaBoost(data, labels, numClasses, iterations, tolerance); } else if (weakLearnerType == WeakLearnerTypes::PERCEPTRON) { delete pBoost; - Perceptron<> p(data, labels, max(labels) + 1); - pBoost = new AdaBoost>(data, labels, numClasses, p, iterations, + pBoost = new AdaBoost>(data, labels, numClasses, iterations, tolerance); } } diff --git a/src/mlpack/methods/decision_tree/decision_tree.hpp b/src/mlpack/methods/decision_tree/decision_tree.hpp index e7bb0031d1..af169822b5 100644 --- a/src/mlpack/methods/decision_tree/decision_tree.hpp +++ b/src/mlpack/methods/decision_tree/decision_tree.hpp @@ -547,14 +547,14 @@ class DecisionTree : * @param maximumDepth Maximum depth for the tree. * @return The final entropy of decision tree. */ - template + template double Train(MatType& data, const size_t begin, const size_t count, const data::DatasetInfo& datasetInfo, arma::Row& labels, const size_t numClasses, - arma::rowvec& weights, + WeightsType& weights, const size_t minimumLeafSize, const double minimumGainSplit, const size_t maximumDepth, @@ -576,13 +576,13 @@ class DecisionTree : * @param maximumDepth Maximum depth for the tree. * @return The final entropy of decision tree. */ - template + template double Train(MatType& data, const size_t begin, const size_t count, arma::Row& labels, const size_t numClasses, - arma::rowvec& weights, + WeightsType& weights, const size_t minimumLeafSize, const double minimumGainSplit, const size_t maximumDepth, diff --git a/src/mlpack/methods/decision_tree/decision_tree_impl.hpp b/src/mlpack/methods/decision_tree/decision_tree_impl.hpp index 7e1d1362d5..a439f1aa1d 100644 --- a/src/mlpack/methods/decision_tree/decision_tree_impl.hpp +++ b/src/mlpack/methods/decision_tree/decision_tree_impl.hpp @@ -614,7 +614,7 @@ template class CategoricalSplitType, typename DimensionSelectionType, bool NoRecursion> -template +template double DecisionTree& labels, const size_t numClasses, - arma::rowvec& weights, + WeightsType& weights, const size_t minimumLeafSize, const double minimumGainSplit, const size_t maximumDepth, @@ -798,7 +798,7 @@ template class CategoricalSplitType, typename DimensionSelectionType, bool NoRecursion> -template +template double DecisionTree& labels, const size_t numClasses, - arma::rowvec& weights, + WeightsType& weights, const size_t minimumLeafSize, const double minimumGainSplit, const size_t maximumDepth, diff --git a/src/mlpack/methods/decision_tree/information_gain.hpp b/src/mlpack/methods/decision_tree/information_gain.hpp index e07ea7eca2..7cf0f1158e 100644 --- a/src/mlpack/methods/decision_tree/information_gain.hpp +++ b/src/mlpack/methods/decision_tree/information_gain.hpp @@ -55,10 +55,10 @@ class InformationGain * @param numClasses Number of classes in the dataset. * @param weights Weights associated with labels. */ - template + template static double Evaluate(const arma::Row& labels, const size_t numClasses, - const arma::Row& weights) + const WeightsType& weights) { // Edge case: if there are no elements, the gain is zero. if (labels.n_elem == 0) diff --git a/src/mlpack/methods/linear_regression/linear_regression_impl.hpp b/src/mlpack/methods/linear_regression/linear_regression_impl.hpp index fe79f2ed55..4d8265ae2a 100644 --- a/src/mlpack/methods/linear_regression/linear_regression_impl.hpp +++ b/src/mlpack/methods/linear_regression/linear_regression_impl.hpp @@ -104,10 +104,10 @@ inline void LinearRegression::Predict( // We want to be sure we have the correct number of dimensions in the // dataset. // Prevent underflow. - const size_t labels = (parameters.n_rows == 0) ? size_t(0) : + const size_t dimensionality = (parameters.n_rows == 0) ? size_t(0) : size_t(parameters.n_rows - 1); - util::CheckSameDimensionality(points, labels, "LinearRegression::Predict()", - "points"); + util::CheckSameDimensionality(points, dimensionality, + "LinearRegression::Predict()", "points"); // Get the predictions, but this ignores the intercept value // (parameters[0]). predictions = arma::trans(parameters.subvec(1, parameters.n_elem - 1)) diff --git a/src/mlpack/methods/perceptron/initialization_methods/random_init.hpp b/src/mlpack/methods/perceptron/initialization_methods/random_init.hpp index 5b8caa29ac..a4c78a25b3 100644 --- a/src/mlpack/methods/perceptron/initialization_methods/random_init.hpp +++ b/src/mlpack/methods/perceptron/initialization_methods/random_init.hpp @@ -25,8 +25,9 @@ class RandomPerceptronInitialization public: RandomPerceptronInitialization() { } - inline static void Initialize(arma::mat& weights, - arma::vec& biases, + template + inline static void Initialize(arma::Mat& weights, + arma::Col& biases, const size_t numFeatures, const size_t numClasses) { diff --git a/src/mlpack/methods/perceptron/initialization_methods/zero_init.hpp b/src/mlpack/methods/perceptron/initialization_methods/zero_init.hpp index 595559d0aa..084b57a221 100644 --- a/src/mlpack/methods/perceptron/initialization_methods/zero_init.hpp +++ b/src/mlpack/methods/perceptron/initialization_methods/zero_init.hpp @@ -24,8 +24,9 @@ class ZeroInitialization public: ZeroInitialization() { } - inline static void Initialize(arma::mat& weights, - arma::vec& biases, + template + inline static void Initialize(arma::Mat& weights, + arma::Col& biases, const size_t numFeatures, const size_t numClasses) { diff --git a/src/mlpack/methods/perceptron/learning_policies/simple_weight_update.hpp b/src/mlpack/methods/perceptron/learning_policies/simple_weight_update.hpp index f9dfb2e53b..435b3625a3 100644 --- a/src/mlpack/methods/perceptron/learning_policies/simple_weight_update.hpp +++ b/src/mlpack/methods/perceptron/learning_policies/simple_weight_update.hpp @@ -45,13 +45,13 @@ class SimpleWeightUpdate * @param instanceWeight Weight to be given to this particular point during * training (this is useful for boosting). */ - template + template void UpdateWeights(const VecType& trainingPoint, - arma::mat& weights, - arma::vec& biases, + arma::Mat& weights, + arma::Col& biases, const size_t incorrectClass, const size_t correctClass, - const double instanceWeight = 1.0) + const eT instanceWeight = 1.0) { weights.col(incorrectClass) -= instanceWeight * trainingPoint; biases(incorrectClass) -= instanceWeight; diff --git a/src/mlpack/methods/perceptron/perceptron.hpp b/src/mlpack/methods/perceptron/perceptron.hpp index 2f8dfa5cc6..1703b6e4d7 100644 --- a/src/mlpack/methods/perceptron/perceptron.hpp +++ b/src/mlpack/methods/perceptron/perceptron.hpp @@ -34,6 +34,9 @@ template Perceptron(const MatType& data, const arma::Row& labels, const size_t numClasses, - const arma::rowvec& instanceWeights, - const size_t maxIterations = 1000); + const WeightsType& instanceWeights, + const size_t maxIterations = 1000, + const typename std::enable_if< + arma::is_arma_type::value>::type* = 0); /** * Alternate constructor which copies parameters from an already initiated @@ -101,11 +107,54 @@ class Perceptron * @param instanceWeights Weight vector to use while training. For boosting * purposes. */ + template + mlpack_deprecated /* was previously only used by AdaBoost */ Perceptron(const Perceptron& other, const MatType& data, const arma::Row& labels, const size_t numClasses, - const arma::rowvec& instanceWeights); + const WeightsType& instanceWeights, + const typename std::enable_if< + arma::is_arma_type::value>::type* = 0); + + /** + * Train the perceptron on the given data for up to the maximum number of + * iterations (specified in the constructor or through MaxIterations()). A + * single iteration corresponds to a single pass through the data, so if you + * want to pass through the dataset only once, set MaxIterations() to 1. + * + * This training does not reset the model weights, so you can call Train() on + * multiple datasets sequentially. + * + * @param data Dataset on which training should be performed. + * @param labels Labels of the dataset. + * @param numClasses Number of classes in the data. + */ + void Train(const MatType& data, + const arma::Row& labels, + const size_t numClasses); + + /** + * Train the perceptron on the given data for up to the given maximum number + * of iterations. A single iteration corresponds to a single pass through the + * data, so if you want to pass through the dataset only once, set + * `maxIterations` to 1. + * + * After calling this overload, `MaxIterations()` will return whatever + * `maxIterations` was given to this function. + * + * This training does not reset the model weights, so you can call Train() on + * multiple datasets sequentially. + * + * @param data Dataset on which training should be performed. + * @param labels Labels of the dataset. + * @param numClasses Number of classes in the data. + * @param maxIterations Maximum number of iterations for training. + */ + void Train(const MatType& data, + const arma::Row& labels, + const size_t numClasses, + const size_t maxIterations); /** * Train the perceptron on the given data for up to the maximum number of @@ -125,7 +174,41 @@ class Perceptron void Train(const MatType& data, const arma::Row& labels, const size_t numClasses, - const arma::rowvec& instanceWeights = arma::rowvec()); + const arma::rowvec& instanceWeights); + + /** + * Train the perceptron on the given data for up to the given maximum number + * of iterations. A single iteration corresponds to a single pass through the + * data, so if you want to pass through the dataset only once, set + * `maxIterations` to 1. + * + * After calling this overload, `MaxIterations()` will return whatever + * `maxIterations` was given to this function. + * + * This training does not reset the model weights, so you can call Train() on + * multiple datasets sequentially. + * + * @param data Dataset on which training should be performed. + * @param labels Labels of the dataset. + * @param numClasses Number of classes in the data. + * @param instanceWeights Cost matrix. Stores the cost of mispredicting + * instances. This is useful for boosting. + * @param maxIterations Maximum number of iterations for training. + */ + void Train(const MatType& data, + const arma::Row& labels, + const size_t numClasses, + const arma::rowvec& instanceWeights, + const size_t maxIterations); + + /** + * After training, use the weights matrix to classify `point`, and return the + * predicted class. + * + * @param point Test point to classify. + */ + template + size_t Classify(const VecType& point) const; /** * Classification function. After training, use the weights matrix to @@ -135,7 +218,13 @@ class Perceptron * @param predictedLabels Vector to store the predicted classes after * classifying test. */ - void Classify(const MatType& test, arma::Row& predictedLabels); + void Classify(const MatType& test, arma::Row& predictedLabels) const; + + /** + * Reset the model, so that the next call to `Train()` will not be + * incremental. + */ + void Reset(); /** * Serialize the perceptron. @@ -152,16 +241,28 @@ class Perceptron size_t NumClasses() const { return weights.n_cols; } //! Get the weight matrix. - const arma::mat& Weights() const { return weights; } + const arma::Mat& Weights() const { return weights; } //! Modify the weight matrix. You had better know what you are doing! - arma::mat& Weights() { return weights; } + arma::Mat& Weights() { return weights; } //! Get the biases. - const arma::vec& Biases() const { return biases; } + const arma::Col& Biases() const { return biases; } //! Modify the biases. You had better know what you are doing! - arma::vec& Biases() { return biases; } + arma::Col& Biases() { return biases; } private: + /** + * Internal training function; this assumes that maxIterations has been set. + * + * If `HasWeights` is `false`, then `instanceWeights` is ignored (and may be + * left empty). + */ + template + void TrainInternal(const MatType& data, + const arma::Row& labels, + const size_t numClasses, + const WeightsType& instanceWeights = WeightsType()); + //! The maximum number of iterations during training. size_t maxIterations; @@ -171,10 +272,10 @@ class Perceptron * the weights for one dimension of the input data. The biases are held in a * separate vector. */ - arma::mat weights; + arma::Mat weights; //! The biases for each class. - arma::vec biases; + arma::Col biases; }; } // namespace mlpack diff --git a/src/mlpack/methods/perceptron/perceptron_impl.hpp b/src/mlpack/methods/perceptron/perceptron_impl.hpp index 1f1ab92719..8e2e9c0b49 100644 --- a/src/mlpack/methods/perceptron/perceptron_impl.hpp +++ b/src/mlpack/methods/perceptron/perceptron_impl.hpp @@ -58,7 +58,8 @@ Perceptron::Perceptron( maxIterations(maxIterations) { // Start training. - Train(data, labels, numClasses); + TrainInternal>(data, labels, + numClasses); } /** @@ -75,16 +76,19 @@ template< typename WeightInitializationPolicy, typename MatType > +template Perceptron::Perceptron( const MatType& data, const arma::Row& labels, const size_t numClasses, - const arma::rowvec& instanceWeights, - const size_t maxIterations) : + const WeightsType& instanceWeights, + const size_t maxIterations, + const typename std::enable_if< + arma::is_arma_type::value>::type*) : maxIterations(maxIterations) { // Start training. - Train(data, labels, numClasses, instanceWeights); + TrainInternal(data, labels, numClasses, instanceWeights); } /** @@ -103,45 +107,80 @@ template< typename WeightInitializationPolicy, typename MatType > +template +mlpack_deprecated Perceptron::Perceptron( const Perceptron& other, const MatType& data, const arma::Row& labels, const size_t numClasses, - const arma::rowvec& instanceWeights) : + const WeightsType& instanceWeights, + const typename std::enable_if< + arma::is_arma_type::value>::type*) : maxIterations(other.maxIterations) { - Train(data, labels, numClasses, instanceWeights); + TrainInternal(data, labels, numClasses, instanceWeights); } /** - * Classification function. After training, use the weights matrix to classify - * test, and put the predicted classes in predictedLabels. + * Train the perceptron on the given data for up to the maximum number of + * iterations (specified in the constructor or through MaxIterations()). A + * single iteration corresponds to a single pass through the data, so if you + * want to pass through the dataset only once, set MaxIterations() to 1. * - * @param test Testing data or data to classify. - * @param predictedLabels Vector to store the predicted classes after - * classifying test. + * This training does not reset the model weights, so you can call Train() on + * multiple datasets sequentially. + * + * @param data Dataset on which training should be performed. + * @param labels Labels of the dataset. + * @param numClasses Number of classes in the data. */ template< typename LearnPolicy, typename WeightInitializationPolicy, typename MatType > -void Perceptron::Classify( - const MatType& test, - arma::Row& predictedLabels) +void Perceptron::Train( + const MatType& data, + const arma::Row& labels, + const size_t numClasses) { - arma::vec tempLabelMat; - arma::uword maxIndex = 0; - predictedLabels.set_size(test.n_cols); + TrainInternal>(data, labels, + numClasses); +} - // Could probably be faster if done in batch. - for (size_t i = 0; i < test.n_cols; ++i) - { - tempLabelMat = weights.t() * test.col(i) + biases; - tempLabelMat.max(maxIndex); - predictedLabels(i) = maxIndex; - } +/** + * Train the perceptron on the given data for up to the given maximum number + * of iterations. A single iteration corresponds to a single pass through the + * data, so if you want to pass through the dataset only once, set + * `maxIterations` to 1. + * + * After calling this overload, `MaxIterations()` will return whatever + * `maxIterations` was given to this function. + * + * This training does not reset the model weights, so you can call Train() on + * multiple datasets sequentially. + * + * @param data Dataset on which training should be performed. + * @param labels Labels of the dataset. + * @param numClasses Number of classes in the data. + * @param maxIterations Maximum number of iterations for training. + */ +template< + typename LearnPolicy, + typename WeightInitializationPolicy, + typename MatType +> +void Perceptron::Train( + const MatType& data, + const arma::Row& labels, + const size_t numClasses, + const size_t maxIterations) +{ + // Set the maximum number of iterations and call unweighted Train(). + this->maxIterations = maxIterations; + TrainInternal>(data, labels, + numClasses); } /** @@ -163,9 +202,70 @@ void Perceptron::Train( const arma::Row& labels, const size_t numClasses, const arma::rowvec& instanceWeights) +{ + TrainInternal(data, labels, numClasses, instanceWeights); +} + +/** + * Train the perceptron on the given data for up to the given maximum number + * of iterations. A single iteration corresponds to a single pass through the + * data, so if you want to pass through the dataset only once, set + * `maxIterations` to 1. + * + * After calling this overload, `MaxIterations()` will return whatever + * `maxIterations` was given to this function. + * + * This training does not reset the model weights, so you can call Train() on + * multiple datasets sequentially. + * + * @param data Dataset on which training should be performed. + * @param labels Labels of the dataset. + * @param numClasses Number of classes in the data. + * @param instanceWeights Cost matrix. Stores the cost of mispredicting + * instances. This is useful for boosting. + * @param maxIterations Maximum number of iterations for training. + */ +template< + typename LearnPolicy, + typename WeightInitializationPolicy, + typename MatType +> +void Perceptron::Train( + const MatType& data, + const arma::Row& labels, + const size_t numClasses, + const arma::rowvec& instanceWeights, + const size_t maxIterations) +{ + // Set the maximum number of iterations and call weighted training. + this->maxIterations = maxIterations; + TrainInternal(data, labels, numClasses, instanceWeights); +} + +/** + * Training function. It trains on trainData using the cost matrix + * instanceWeights. + * + * @param data Data to train on. + * @param labels Labels of data. + * @param instanceWeights Cost matrix. Stores the cost of mispredicting + * instances. This is useful for boosting. + */ +template< + typename LearnPolicy, + typename WeightInitializationPolicy, + typename MatType +> +template +void Perceptron< + LearnPolicy, WeightInitializationPolicy, MatType +>::TrainInternal(const MatType& data, + const arma::Row& labels, + const size_t numClasses, + const WeightsType& instanceWeights) { // Do we need to resize the weights? - if (weights.n_elem != numClasses) + if (weights.n_cols != numClasses || weights.n_rows != data.n_rows) { WeightInitializationPolicy wip; wip.Initialize(weights, biases, data.n_rows, numClasses); @@ -175,12 +275,10 @@ void Perceptron::Train( bool converged = false; size_t tempLabel; arma::uword maxIndexRow = 0, maxIndexCol = 0; - arma::mat tempLabelMat; + arma::Mat tempLabelMat; LearnPolicy LP; - const bool hasWeights = (instanceWeights.n_elem > 0); - while ((i < maxIterations) && (!converged)) { // This outer loop is for each iteration, and we use the 'converged' @@ -207,9 +305,9 @@ void Perceptron::Train( // Send maxIndexRow for knowing which weight to update, send j to know // the value of the vector to update it with. Send tempLabel to know // the correct class. - if (hasWeights) + if (HasWeights) LP.UpdateWeights(data.col(j), weights, biases, maxIndexRow, tempLabel, - instanceWeights(j)); + (typename MatType::elem_type) instanceWeights(j)); else LP.UpdateWeights(data.col(j), weights, biases, maxIndexRow, tempLabel); @@ -218,6 +316,79 @@ void Perceptron::Train( } } +/** + * After training, use the weights matrix to classify `point`, and return the + * predicted class. + * + * @param point Test point to classify. + */ +template< + typename LearnPolicy, + typename WeightInitializationPolicy, + typename MatType +> +template +size_t Perceptron::Classify( + const VecType& point) const +{ + util::CheckSameDimensionality(point, weights.n_rows, "Perceptron::Classify()", + "point"); + + arma::Col tempLabelVec; + arma::uword maxIndex = 0; + + tempLabelVec = weights.t() * point + biases; + tempLabelVec.max(maxIndex); + + return size_t(maxIndex); +} + +/** + * Classification function. After training, use the weights matrix to classify + * test, and put the predicted classes in predictedLabels. + * + * @param test Testing data or data to classify. + * @param predictedLabels Vector to store the predicted classes after + * classifying test. + */ +template< + typename LearnPolicy, + typename WeightInitializationPolicy, + typename MatType +> +void Perceptron::Classify( + const MatType& test, + arma::Row& predictedLabels) const +{ + util::CheckSameDimensionality(test, weights.n_rows, "Perceptron::Classify()", + "points"); + + arma::Col tempLabelMat; + arma::uword maxIndex = 0; + predictedLabels.set_size(test.n_cols); + + // Could probably be faster if done in batch. + for (size_t i = 0; i < test.n_cols; ++i) + { + tempLabelMat = weights.t() * test.col(i) + biases; + tempLabelMat.max(maxIndex); + predictedLabels(i) = maxIndex; + } +} + +/** + * Reset the model, so that the next call to `Train()` will not be + * incremental. + */ +template +void Perceptron::Reset() +{ + weights.clear(); + biases.clear(); +} + //! Serialize the perceptron. template #include #include +#include #include // All code should have access to logging. diff --git a/src/mlpack/tests/adaboost_test.cpp b/src/mlpack/tests/adaboost_test.cpp index 457b7617dd..6c4e3abe94 100644 --- a/src/mlpack/tests/adaboost_test.cpp +++ b/src/mlpack/tests/adaboost_test.cpp @@ -24,14 +24,17 @@ using namespace mlpack; * checks whether the hamming loss breaches the upperbound, which is provided by * ztAccumulator. */ -TEST_CASE("HammingLossBoundIris", "[AdaBoostTest]") +TEMPLATE_TEST_CASE("HammingLossBoundIris", "[AdaBoostTest]", mat, fmat) { - arma::mat inputData; + typedef TestType MatType; + typedef typename MatType::elem_type eT; + + MatType inputData; if (!data::Load("iris.csv", inputData)) FAIL("Cannot load test dataset iris.csv!"); - arma::Mat labels; + Mat labels; if (!data::Load("iris_labels.txt", labels)) FAIL("Cannot load labels for iris iris_labels.txt"); @@ -41,20 +44,20 @@ TEST_CASE("HammingLossBoundIris", "[AdaBoostTest]") // Run the perceptron for perceptronIter iterations. int perceptronIter = 400; - Perceptron<> p(inputData, labels.row(0), numClasses, perceptronIter); - // Define parameters for AdaBoost. size_t iterations = 100; - double tolerance = 1e-10; - AdaBoost<> a(tolerance); - double ztProduct = a.Train(inputData, labels.row(0), numClasses, p, - iterations, tolerance); + eT tolerance = 2e-10; + typedef Perceptron + PerceptronType; + AdaBoost a; + eT ztProduct = a.Train(inputData, labels.row(0), numClasses, iterations, + tolerance, perceptronIter); - arma::Row predictedLabels; + Row predictedLabels; a.Classify(inputData, predictedLabels); - size_t countError = arma::accu(labels != predictedLabels); - double hammingLoss = (double) countError / labels.n_cols; + size_t countError = accu(labels != predictedLabels); + eT hammingLoss = (eT) countError / labels.n_cols; // Check that ztProduct is finite. REQUIRE(std::isfinite(ztProduct) == true); @@ -66,15 +69,16 @@ TEST_CASE("HammingLossBoundIris", "[AdaBoostTest]") * checks if the error returned by running a single instance of the weak learner * close to that of the boosted weak learner using adaboost. */ -TEST_CASE("WeakLearnerErrorIris", "[AdaBoostTest]") +TEMPLATE_TEST_CASE("WeakLearnerErrorIris", "[AdaBoostTest]", mat, fmat) { - arma::mat inputData; + typedef TestType MatType; + typedef typename MatType::elem_type eT; + MatType inputData; if (!data::Load("iris.csv", inputData)) FAIL("Cannot load test dataset iris.csv!"); - arma::Mat labels; - + Mat labels; if (!data::Load("iris_labels.txt", labels)) FAIL("Cannot load labels for iris iris_labels.txt"); @@ -84,23 +88,26 @@ TEST_CASE("WeakLearnerErrorIris", "[AdaBoostTest]") // Run the perceptron for perceptronIter iterations. int perceptronIter = 400; - arma::Row perceptronPrediction(labels.n_cols); - Perceptron<> p(inputData, labels.row(0), numClasses, perceptronIter); + Row perceptronPrediction(labels.n_cols); + typedef Perceptron + PerceptronType; + PerceptronType p(inputData, labels.row(0), numClasses, perceptronIter); p.Classify(inputData, perceptronPrediction); - size_t countWeakLearnerError = arma::accu(labels != perceptronPrediction); - double weakLearnerErrorRate = (double) countWeakLearnerError / labels.n_cols; + size_t countWeakLearnerError = accu(labels != perceptronPrediction); + eT weakLearnerErrorRate = (eT) countWeakLearnerError / labels.n_cols; // Define parameters for AdaBoost. size_t iterations = 100; - double tolerance = 1e-10; - AdaBoost<> a(inputData, labels.row(0), numClasses, p, iterations, tolerance); + eT tolerance = 1e-10; + AdaBoost a(inputData, labels.row(0), numClasses, + iterations, tolerance, perceptronIter); - arma::Row predictedLabels; + Row predictedLabels; a.Classify(inputData, predictedLabels); - size_t countError = arma::accu(labels != predictedLabels);; - double error = (double) countError / labels.n_cols; + size_t countError = accu(labels != predictedLabels); + eT error = (eT) countError / labels.n_cols; REQUIRE(error <= weakLearnerErrorRate + 0.03); } @@ -110,13 +117,17 @@ TEST_CASE("WeakLearnerErrorIris", "[AdaBoostTest]") * dataset. It checks whether the hamming loss breaches the upperbound, which * is provided by ztAccumulator. */ -TEST_CASE("HammingLossBoundVertebralColumn", "[AdaBoostTest]") +TEMPLATE_TEST_CASE("HammingLossBoundVertebralColumn", "[AdaBoostTest]", mat, + fmat) { - arma::mat inputData; + typedef TestType MatType; + typedef typename MatType::elem_type eT; + + MatType inputData; if (!data::Load("vc2.csv", inputData)) FAIL("Cannot load test dataset vc2.csv!"); - arma::Mat labels; + Mat labels; if (!data::Load("vc2_labels.txt", labels)) FAIL("Cannot load labels for vc2_labels.txt"); @@ -125,20 +136,22 @@ TEST_CASE("HammingLossBoundVertebralColumn", "[AdaBoostTest]") // Define your own weak learner, perceptron in this case. // Run the perceptron for perceptronIter iterations. size_t perceptronIter = 800; - Perceptron<> p(inputData, labels.row(0), numClasses, perceptronIter); + typedef Perceptron + PerceptronType; + PerceptronType p(inputData, labels.row(0), numClasses, perceptronIter); // Define parameters for AdaBoost. size_t iterations = 50; - double tolerance = 1e-10; - AdaBoost<> a(tolerance); - double ztProduct = a.Train(inputData, labels.row(0), numClasses, p, - iterations, tolerance); + eT tolerance = 1e-10; + AdaBoost a(tolerance); + eT ztProduct = a.Train(inputData, labels.row(0), numClasses, iterations, + tolerance, perceptronIter); - arma::Row predictedLabels; + Row predictedLabels; a.Classify(inputData, predictedLabels); - size_t countError = arma::accu(labels != predictedLabels); - double hammingLoss = (double) countError / labels.n_cols; + size_t countError = accu(labels != predictedLabels); + eT hammingLoss = (eT) countError / labels.n_cols; // Check that ztProduct is finite. REQUIRE(std::isfinite(ztProduct) == true); @@ -150,13 +163,17 @@ TEST_CASE("HammingLossBoundVertebralColumn", "[AdaBoostTest]") * dataset. It checks if the error returned by running a single instance of the * weak learner is close to that of a boosted weak learner using adaboost. */ -TEST_CASE("WeakLearnerErrorVertebralColumn", "[AdaBoostTest]") +TEMPLATE_TEST_CASE("WeakLearnerErrorVertebralColumn", "[AdaBoostTest]", mat, + fmat) { - arma::mat inputData; + typedef TestType MatType; + typedef typename MatType::elem_type eT; + + MatType inputData; if (!data::Load("vc2.csv", inputData)) FAIL("Cannot load test dataset vc2.csv!"); - arma::Mat labels; + Mat labels; if (!data::Load("vc2_labels.txt", labels)) FAIL("Cannot load labels for vc2_labels.txt"); @@ -167,22 +184,25 @@ TEST_CASE("WeakLearnerErrorVertebralColumn", "[AdaBoostTest]") size_t perceptronIter = 800; Row perceptronPrediction(labels.n_cols); - Perceptron<> p(inputData, labels.row(0), numClasses, perceptronIter); + typedef Perceptron + PerceptronType; + PerceptronType p(inputData, labels.row(0), numClasses, perceptronIter); p.Classify(inputData, perceptronPrediction); - size_t countWeakLearnerError = arma::accu(labels != perceptronPrediction); - double weakLearnerErrorRate = (double) countWeakLearnerError / labels.n_cols; + size_t countWeakLearnerError = accu(labels != perceptronPrediction); + eT weakLearnerErrorRate = (eT) countWeakLearnerError / labels.n_cols; // Define parameters for AdaBoost. size_t iterations = 50; - double tolerance = 1e-10; - AdaBoost<> a(inputData, labels.row(0), numClasses, p, iterations, tolerance); + eT tolerance = 1e-10; + AdaBoost a(inputData, labels.row(0), numClasses, + iterations, tolerance, perceptronIter); - arma::Row predictedLabels; + Row predictedLabels; a.Classify(inputData, predictedLabels); - size_t countError = arma::accu(labels != predictedLabels); - double error = (double) countError / labels.n_cols; + size_t countError = accu(labels != predictedLabels); + eT error = (eT) countError / labels.n_cols; REQUIRE(error <= weakLearnerErrorRate + 0.03); } @@ -192,13 +212,17 @@ TEST_CASE("WeakLearnerErrorVertebralColumn", "[AdaBoostTest]") * dataset. It checks whether the hamming loss breaches the upperbound, which * is provided by ztAccumulator. */ -TEST_CASE("HammingLossBoundNonLinearSepData", "[AdaBoostTest]") +TEMPLATE_TEST_CASE("HammingLossBoundNonLinearSepData", "[AdaBoostTest]", mat, + fmat) { - arma::mat inputData; + typedef TestType MatType; + typedef typename MatType::elem_type eT; + + MatType inputData; if (!data::Load("train_nonlinsep.txt", inputData)) FAIL("Cannot load test dataset train_nonlinsep.txt!"); - arma::Mat labels; + Mat labels; if (!data::Load("train_labels_nonlinsep.txt", labels)) FAIL("Cannot load labels for train_labels_nonlinsep.txt"); @@ -207,20 +231,22 @@ TEST_CASE("HammingLossBoundNonLinearSepData", "[AdaBoostTest]") // Define your own weak learner, perceptron in this case. // Run the perceptron for perceptronIter iterations. size_t perceptronIter = 800; - Perceptron<> p(inputData, labels.row(0), numClasses, perceptronIter); + typedef Perceptron + PerceptronType; + PerceptronType p(inputData, labels.row(0), numClasses, perceptronIter); // Define parameters for AdaBoost. size_t iterations = 50; - double tolerance = 1e-10; - AdaBoost<> a(tolerance); - double ztProduct = a.Train(inputData, labels.row(0), numClasses, p, - iterations, tolerance); + eT tolerance = 1e-10; + AdaBoost a(tolerance); + eT ztProduct = a.Train(inputData, labels.row(0), numClasses, iterations, + tolerance, perceptronIter); - arma::Row predictedLabels; + Row predictedLabels; a.Classify(inputData, predictedLabels); - size_t countError = arma::accu(labels == predictedLabels); - double hammingLoss = (double) countError / labels.n_cols; + size_t countError = accu(labels == predictedLabels); + eT hammingLoss = (eT) countError / labels.n_cols; // Check that ztProduct is finite. REQUIRE(std::isfinite(ztProduct) <= true); @@ -232,13 +258,17 @@ TEST_CASE("HammingLossBoundNonLinearSepData", "[AdaBoostTest]") * dataset. It checks if the error returned by running a single instance of the * weak learner is close to that of a boosted weak learner using AdaBoost. */ -TEST_CASE("WeakLearnerErrorNonLinearSepData", "[AdaBoostTest]") +TEMPLATE_TEST_CASE("WeakLearnerErrorNonLinearSepData", "[AdaBoostTest]", mat, + fmat) { - arma::mat inputData; + typedef TestType MatType; + typedef typename MatType::elem_type eT; + + MatType inputData; if (!data::Load("train_nonlinsep.txt", inputData)) FAIL("Cannot load test dataset train_nonlinsep.txt!"); - arma::Mat labels; + Mat labels; if (!data::Load("train_labels_nonlinsep.txt", labels)) FAIL("Cannot load labels for train_labels_nonlinsep.txt"); @@ -249,22 +279,25 @@ TEST_CASE("WeakLearnerErrorNonLinearSepData", "[AdaBoostTest]") size_t perceptronIter = 800; Row perceptronPrediction(labels.n_cols); - Perceptron<> p(inputData, labels.row(0), numClasses, perceptronIter); + typedef Perceptron + PerceptronType; + PerceptronType p(inputData, labels.row(0), numClasses, perceptronIter); p.Classify(inputData, perceptronPrediction); - size_t countWeakLearnerError = arma::accu(labels != perceptronPrediction); - double weakLearnerErrorRate = (double) countWeakLearnerError / labels.n_cols; + size_t countWeakLearnerError = accu(labels != perceptronPrediction); + eT weakLearnerErrorRate = (eT) countWeakLearnerError / labels.n_cols; // Define parameters for AdaBoost. size_t iterations = 50; - double tolerance = 1e-10; - AdaBoost<> a(inputData, labels.row(0), numClasses, p, iterations, tolerance); + eT tolerance = 1e-10; + AdaBoost a(inputData, labels.row(0), numClasses, + iterations, tolerance, perceptronIter); - arma::Row predictedLabels; + Row predictedLabels; a.Classify(inputData, predictedLabels); - size_t countError = arma::accu(labels != predictedLabels); - double error = (double) countError / labels.n_cols; + size_t countError = accu(labels != predictedLabels); + eT error = (eT) countError / labels.n_cols; REQUIRE(error <= weakLearnerErrorRate + 0.03); } @@ -274,34 +307,36 @@ TEST_CASE("WeakLearnerErrorNonLinearSepData", "[AdaBoostTest]") * checks whether the Hamming loss breaches the upper bound, which is provided * by ztAccumulator. This uses decision stumps as the weak learner. */ -TEST_CASE("HammingLossIris_DS", "[AdaBoostTest]") +TEMPLATE_TEST_CASE("HammingLossIris_DS", "[AdaBoostTest]", mat, fmat) { - arma::mat inputData; + typedef TestType MatType; + typedef typename MatType::elem_type eT; + + MatType inputData; if (!data::Load("iris.csv", inputData)) FAIL("Cannot load test dataset iris.csv!"); - arma::Mat labels; + Mat labels; if (!data::Load("iris_labels.txt", labels)) FAIL("Cannot load labels for iris_labels.txt"); // Define your own weak learner, decision stumps in this case. const size_t numClasses = 3; const size_t inpBucketSize = 6; - arma::Row labelsvec = labels.row(0); - ID3DecisionStump ds(inputData, labelsvec, numClasses, inpBucketSize); + Row labelsvec = labels.row(0); // Define parameters for AdaBoost. size_t iterations = 50; - double tolerance = 1e-10; - AdaBoost a(tolerance); - double ztProduct = a.Train(inputData, labelsvec, numClasses, ds, - iterations, tolerance); + eT tolerance = 1e-10; + AdaBoost a(tolerance); + eT ztProduct = a.Train(inputData, labelsvec, numClasses, iterations, + tolerance, inpBucketSize); - arma::Row predictedLabels; + Row predictedLabels; a.Classify(inputData, predictedLabels); - size_t countError = arma::accu(labels != predictedLabels); - double hammingLoss = (double) countError / labels.n_cols; + size_t countError = accu(labels != predictedLabels); + eT hammingLoss = (eT) countError / labels.n_cols; // Check that ztProduct is finite. REQUIRE(std::isfinite(ztProduct) == true); @@ -314,13 +349,16 @@ TEST_CASE("HammingLossIris_DS", "[AdaBoostTest]") * weak learner is close to that of a boosted weak learner using adaboost. * This is for the weak learner: decision stumps. */ -TEST_CASE("WeakLearnerErrorIris_DS", "[AdaBoostTest]") +TEMPLATE_TEST_CASE("WeakLearnerErrorIris_DS", "[AdaBoostTest]", mat, fmat) { - arma::mat inputData; + typedef TestType MatType; + typedef typename MatType::elem_type eT; + + MatType inputData; if (!data::Load("iris.csv", inputData)) FAIL("Cannot load test dataset iris.csv!"); - arma::Mat labels; + Mat labels; if (!data::Load("iris_labels.txt", labels)) FAIL("Cannot load labels for iris_labels.txt"); @@ -329,28 +367,28 @@ TEST_CASE("WeakLearnerErrorIris_DS", "[AdaBoostTest]") // Define your own weak learner, decision stumps in this case. const size_t numClasses = 3; const size_t inpBucketSize = 6; - arma::Row labelsvec = labels.row(0); + Row labelsvec = labels.row(0); - arma::Row dsPrediction(labels.n_cols); + Row dsPrediction(labels.n_cols); ID3DecisionStump ds(inputData, labelsvec, numClasses, inpBucketSize); ds.Classify(inputData, dsPrediction); - size_t countWeakLearnerError = arma::accu(labels != dsPrediction); - double weakLearnerErrorRate = (double) countWeakLearnerError / labels.n_cols; + size_t countWeakLearnerError = accu(labels != dsPrediction); + eT weakLearnerErrorRate = (eT) countWeakLearnerError / labels.n_cols; // Define parameters for AdaBoost. size_t iterations = 50; - double tolerance = 1e-10; + eT tolerance = 1e-10; - AdaBoost a(inputData, labelsvec, numClasses, ds, - iterations, tolerance); + AdaBoost a(inputData, labelsvec, numClasses, + iterations, tolerance, inpBucketSize); - arma::Row predictedLabels; + Row predictedLabels; a.Classify(inputData, predictedLabels); - size_t countError = arma::accu(labels != predictedLabels); - double error = (double) countError / labels.n_cols; + size_t countError = accu(labels != predictedLabels); + eT error = (eT) countError / labels.n_cols; REQUIRE(error <= weakLearnerErrorRate + 0.03); } @@ -361,36 +399,40 @@ TEST_CASE("WeakLearnerErrorIris_DS", "[AdaBoostTest]") * weak learner is close to that of a boosted weak learner using adaboost. * This is for the weak learner: decision stumps. */ -TEST_CASE("HammingLossBoundVertebralColumn_DS", "[AdaBoostTest]") +TEMPLATE_TEST_CASE("HammingLossBoundVertebralColumn_DS", "[AdaBoostTest]", mat, + fmat) { - arma::mat inputData; + typedef TestType MatType; + typedef typename MatType::elem_type eT; + + MatType inputData; if (!data::Load("vc2.csv", inputData)) FAIL("Cannot load test dataset vc2.csv!"); - arma::Mat labels; + Mat labels; if (!data::Load("vc2_labels.txt", labels)) FAIL("Cannot load labels for vc2_labels.txt"); // Define your own weak learner, decision stumps in this case. const size_t numClasses = 3; const size_t inpBucketSize = 6; - arma::Row labelsvec = labels.row(0); + Row labelsvec = labels.row(0); ID3DecisionStump ds(inputData, labelsvec, numClasses, inpBucketSize); // Define parameters for AdaBoost. size_t iterations = 50; - double tolerance = 1e-10; + eT tolerance = 1e-10; - AdaBoost a(tolerance); - double ztProduct = a.Train(inputData, labelsvec, numClasses, ds, - iterations, tolerance); + AdaBoost a(tolerance); + eT ztProduct = a.Train(inputData, labelsvec, numClasses, iterations, + tolerance, inpBucketSize); - arma::Row predictedLabels; + Row predictedLabels; a.Classify(inputData, predictedLabels); - size_t countError = arma::accu(labels != predictedLabels); - double hammingLoss = (double) countError / labels.n_cols; + size_t countError = accu(labels != predictedLabels); + eT hammingLoss = (eT) countError / labels.n_cols; // Check that ztProduct is finite. REQUIRE(std::isfinite(ztProduct) == true); @@ -403,39 +445,43 @@ TEST_CASE("HammingLossBoundVertebralColumn_DS", "[AdaBoostTest]") * weak learner is close to that of a boosted weak learner using adaboost. * This is for the weak learner: decision stumps. */ -TEST_CASE("WeakLearnerErrorVertebralColumn_DS", "[AdaBoostTest]") +TEMPLATE_TEST_CASE("WeakLearnerErrorVertebralColumn_DS", "[AdaBoostTest]", mat, + fmat) { - arma::mat inputData; + typedef TestType MatType; + typedef typename MatType::elem_type eT; + + MatType inputData; if (!data::Load("vc2.csv", inputData)) FAIL("Cannot load test dataset vc2.csv!"); - arma::Mat labels; + Mat labels; if (!data::Load("vc2_labels.txt", labels)) FAIL("Cannot load labels for vc2_labels.txt"); // Define your own weak learner, decision stumps in this case. const size_t numClasses = 3; const size_t inpBucketSize = 6; - arma::Row dsPrediction(labels.n_cols); - arma::Row labelsvec = labels.row(0); + Row dsPrediction(labels.n_cols); + Row labelsvec = labels.row(0); ID3DecisionStump ds(inputData, labelsvec, numClasses, inpBucketSize); ds.Classify(inputData, dsPrediction); - size_t countWeakLearnerError = arma::accu(labels != dsPrediction); - double weakLearnerErrorRate = (double) countWeakLearnerError / labels.n_cols; + size_t countWeakLearnerError = accu(labels != dsPrediction); + eT weakLearnerErrorRate = (eT) countWeakLearnerError / labels.n_cols; // Define parameters for AdaBoost. size_t iterations = 50; - double tolerance = 1e-10; - AdaBoost a(inputData, labelsvec, numClasses, ds, - iterations, tolerance); + eT tolerance = 1e-10; + AdaBoost a(inputData, labelsvec, numClasses, + iterations, tolerance, inpBucketSize); - arma::Row predictedLabels; + Row predictedLabels; a.Classify(inputData, predictedLabels); - size_t countError = arma::accu(labels != predictedLabels); - double error = (double) countError / labels.n_cols; + size_t countError = accu(labels != predictedLabels); + eT error = (eT) countError / labels.n_cols; REQUIRE(error <= weakLearnerErrorRate + 0.03); } @@ -445,36 +491,40 @@ TEST_CASE("WeakLearnerErrorVertebralColumn_DS", "[AdaBoostTest]") * dataset. It checks whether the hamming loss breaches the upperbound, which * is provided by ztAccumulator. This is for the weak learner: decision stumps. */ -TEST_CASE("HammingLossBoundNonLinearSepData_DS", "[AdaBoostTest]") +TEMPLATE_TEST_CASE("HammingLossBoundNonLinearSepData_DS", "[AdaBoostTest]", mat, + fmat) { - arma::mat inputData; + typedef TestType MatType; + typedef typename MatType::elem_type eT; + + MatType inputData; if (!data::Load("train_nonlinsep.txt", inputData)) FAIL("Cannot load test dataset train_nonlinsep.txt!"); - arma::Mat labels; + Mat labels; if (!data::Load("train_labels_nonlinsep.txt", labels)) FAIL("Cannot load labels for train_labels_nonlinsep.txt"); // Define your own weak learner, decision stumps in this case. const size_t numClasses = 2; const size_t inpBucketSize = 6; - arma::Row labelsvec = labels.row(0); + Row labelsvec = labels.row(0); ID3DecisionStump ds(inputData, labelsvec, numClasses, inpBucketSize); // Define parameters for Adaboost. size_t iterations = 50; - double tolerance = 1e-10; + eT tolerance = 1e-10; - AdaBoost a(tolerance); - double ztProduct = a.Train(inputData, labelsvec, numClasses, ds, - iterations, tolerance); + AdaBoost a(tolerance); + eT ztProduct = a.Train(inputData, labelsvec, numClasses, iterations, + tolerance, inpBucketSize); - arma::Row predictedLabels; + Row predictedLabels; a.Classify(inputData, predictedLabels); - size_t countError = arma::accu(labels != predictedLabels); - double hammingLoss = (double) countError / labels.n_cols; + size_t countError = accu(labels != predictedLabels); + eT hammingLoss = (eT) countError / labels.n_cols; // Check that ztProduct is finite. REQUIRE(std::isfinite(ztProduct) == true); @@ -487,41 +537,45 @@ TEST_CASE("HammingLossBoundNonLinearSepData_DS", "[AdaBoostTest]") * weak learner is close to that of a boosted weak learner using adaboost. * This for the weak learner: decision stumps. */ -TEST_CASE("WeakLearnerErrorNonLinearSepData_DS", "[AdaBoostTest]") +TEMPLATE_TEST_CASE("WeakLearnerErrorNonLinearSepData_DS", "[AdaBoostTest]", mat, + fmat) { - arma::mat inputData; + typedef TestType MatType; + typedef typename MatType::elem_type eT; + + MatType inputData; if (!data::Load("train_nonlinsep.txt", inputData)) FAIL("Cannot load test dataset train_nonlinsep.txt!"); - arma::Mat labels; + Mat labels; if (!data::Load("train_labels_nonlinsep.txt", labels)) FAIL("Cannot load labels for train_labels_nonlinsep.txt"); // Define your own weak learner, decision stumps in this case. const size_t numClasses = 2; const size_t inpBucketSize = 3; - arma::Row labelsvec = labels.row(0); + Row labelsvec = labels.row(0); - arma::Row dsPrediction(labels.n_cols); + Row dsPrediction(labels.n_cols); ID3DecisionStump ds(inputData, labelsvec, numClasses, inpBucketSize); ds.Classify(inputData, dsPrediction); - size_t countWeakLearnerError = arma::accu(labels != dsPrediction); - double weakLearnerErrorRate = (double) countWeakLearnerError / labels.n_cols; + size_t countWeakLearnerError = accu(labels != dsPrediction); + eT weakLearnerErrorRate = (eT) countWeakLearnerError / labels.n_cols; // Define parameters for AdaBoost. size_t iterations = 500; - double tolerance = 1e-23; + eT tolerance = 1e-23; - AdaBoost a(inputData, labelsvec, numClasses, ds, - iterations, tolerance); + AdaBoost a(inputData, labelsvec, numClasses, + iterations, tolerance, inpBucketSize); - arma::Row predictedLabels; + Row predictedLabels; a.Classify(inputData, predictedLabels); - size_t countError = arma::accu(labels != predictedLabels); - double error = (double) countError / labels.n_cols; + size_t countError = accu(labels != predictedLabels); + eT error = (eT) countError / labels.n_cols; REQUIRE(error <= weakLearnerErrorRate + 0.03); } @@ -531,13 +585,16 @@ TEST_CASE("WeakLearnerErrorNonLinearSepData_DS", "[AdaBoostTest]") * dataset. It tests the Classify function and checks for a satisfactory error * rate. */ -TEST_CASE("ClassifyTest_VERTEBRALCOL", "[AdaBoostTest]") +TEMPLATE_TEST_CASE("ClassifyTest_VERTEBRALCOL", "[AdaBoostTest]", mat, fmat) { - arma::mat inputData; + typedef TestType MatType; + typedef typename MatType::elem_type eT; + + MatType inputData; if (!data::Load("vc2.csv", inputData)) FAIL("Cannot load test dataset vc2.csv!"); - arma::Mat labels; + Mat labels; if (!data::Load("vc2_labels.txt", labels)) FAIL("Cannot load labels for vc2_labels.txt"); @@ -545,12 +602,11 @@ TEST_CASE("ClassifyTest_VERTEBRALCOL", "[AdaBoostTest]") // Run the perceptron for perceptronIter iterations. size_t perceptronIter = 1000; - arma::mat testData; - + MatType testData; if (!data::Load("vc2_test.csv", testData)) FAIL("Cannot load test dataset vc2_test.csv!"); - arma::Mat trueTestLabels; + Mat trueTestLabels; if (!data::Load("vc2_test_labels.txt", trueTestLabels)) FAIL("Cannot load labels for vc2_test_labels.txt"); @@ -558,17 +614,20 @@ TEST_CASE("ClassifyTest_VERTEBRALCOL", "[AdaBoostTest]") const size_t numClasses = max(labels.row(0)) + 1; Row perceptronPrediction(labels.n_cols); - Perceptron<> p(inputData, labels.row(0), numClasses, perceptronIter); + typedef Perceptron + PerceptronType; + PerceptronType p(inputData, labels.row(0), numClasses, perceptronIter); p.Classify(inputData, perceptronPrediction); // Define parameters for AdaBoost. size_t iterations = 100; - double tolerance = 1e-10; - AdaBoost<> a(inputData, labels.row(0), numClasses, p, iterations, tolerance); + eT tolerance = 1e-10; + AdaBoost a(inputData, labels.row(0), numClasses, + iterations, tolerance, perceptronIter); - arma::Row predictedLabels1(testData.n_cols), - predictedLabels2(testData.n_cols); - arma::mat probabilities; + Row predictedLabels1(testData.n_cols), + predictedLabels2(testData.n_cols); + MatType probabilities; a.Classify(testData, predictedLabels1); a.Classify(testData, predictedLabels2, probabilities); @@ -579,19 +638,19 @@ TEST_CASE("ClassifyTest_VERTEBRALCOL", "[AdaBoostTest]") for (size_t i = 0; i < predictedLabels1.n_cols; ++i) REQUIRE(predictedLabels1[i] == predictedLabels2[i]); - arma::colvec pRow; - arma::uword maxIndex = 0; + Col pRow; + uword maxIndex = 0; for (size_t i = 0; i < predictedLabels1.n_cols; ++i) { pRow = probabilities.unsafe_col(i); pRow.max(maxIndex); REQUIRE(predictedLabels1(i) == maxIndex); - REQUIRE(arma::accu(probabilities.col(i)) == Approx(1).epsilon(1e-7)); + REQUIRE(accu(probabilities.col(i)) == Approx(1)); } - size_t localError = arma::accu(trueTestLabels != predictedLabels1); - double lError = (double) localError / trueTestLabels.n_cols; + size_t localError = accu(trueTestLabels != predictedLabels1); + eT lError = (eT) localError / trueTestLabels.n_cols; REQUIRE(lError <= 0.30); } @@ -600,44 +659,46 @@ TEST_CASE("ClassifyTest_VERTEBRALCOL", "[AdaBoostTest]") * dataset. It tests the Classify function and checks for a satisfactory error * rate. */ -TEST_CASE("ClassifyTest_NONLINSEP", "[AdaBoostTest]") +TEMPLATE_TEST_CASE("ClassifyTest_NONLINSEP", "[AdaBoostTest]", mat, fmat) { - arma::mat inputData; + typedef TestType MatType; + typedef typename MatType::elem_type eT; + + MatType inputData; if (!data::Load("train_nonlinsep.txt", inputData)) FAIL("Cannot load test dataset train_nonlinsep.txt!"); - arma::Mat labels; + Mat labels; if (!data::Load("train_labels_nonlinsep.txt", labels)) FAIL("Cannot load labels for train_labels_nonlinsep.txt"); // Define your own weak learner; in this test decision stumps are used. const size_t numClasses = 2; const size_t inpBucketSize = 3; - arma::Row labelsvec = labels.row(0); + Row labelsvec = labels.row(0); - arma::mat testData; + MatType testData; if (!data::Load("test_nonlinsep.txt", testData)) FAIL("Cannot load test dataset test_nonlinsep.txt!"); - arma::Mat trueTestLabels; - + Mat trueTestLabels; if (!data::Load("test_labels_nonlinsep.txt", trueTestLabels)) FAIL("Cannot load labels for test_labels_nonlinsep.txt"); - arma::Row dsPrediction(labels.n_cols); + Row dsPrediction(labels.n_cols); ID3DecisionStump ds(inputData, labelsvec, numClasses, inpBucketSize); // Define parameters for AdaBoost. size_t iterations = 50; - double tolerance = 1e-10; - AdaBoost a(inputData, labelsvec, numClasses, ds, - iterations, tolerance); + eT tolerance = 1e-10; + AdaBoost a(inputData, labelsvec, numClasses, + iterations, tolerance, inpBucketSize); - arma::Row predictedLabels1(testData.n_cols), - predictedLabels2(testData.n_cols); - arma::mat probabilities; + Row predictedLabels1(testData.n_cols), + predictedLabels2(testData.n_cols); + MatType probabilities; a.Classify(testData, predictedLabels1); a.Classify(testData, predictedLabels2, probabilities); @@ -647,19 +708,19 @@ TEST_CASE("ClassifyTest_NONLINSEP", "[AdaBoostTest]") for (size_t i = 0; i < predictedLabels1.n_cols; ++i) REQUIRE(predictedLabels1[i] == predictedLabels2[i]); - arma::colvec pRow; - arma::uword maxIndex = 0; + Col pRow; + uword maxIndex = 0; for (size_t i = 0; i < predictedLabels1.n_cols; ++i) { pRow = probabilities.unsafe_col(i); pRow.max(maxIndex); REQUIRE(predictedLabels1(i) == maxIndex); - REQUIRE(arma::accu(probabilities.col(i)) == Approx(1).epsilon(1e-7)); + REQUIRE(accu(probabilities.col(i)) == Approx(1).epsilon(1e-7)); } - size_t localError = arma::accu(trueTestLabels != predictedLabels1); - double lError = (double) localError / trueTestLabels.n_cols; + size_t localError = accu(trueTestLabels != predictedLabels1); + eT lError = (eT) localError / trueTestLabels.n_cols; REQUIRE(lError <= 0.30); } @@ -669,13 +730,16 @@ TEST_CASE("ClassifyTest_NONLINSEP", "[AdaBoostTest]") * the remaining third of the dataset (iris_test.csv). It tests the Classify() * function and checks for a satisfactory error rate. */ -TEST_CASE("ClassifyTest_IRIS", "[AdaBoostTest]") +TEMPLATE_TEST_CASE("ClassifyTest_IRIS", "[AdaBoostTest]", mat, fmat) { - arma::mat inputData; + typedef TestType MatType; + typedef typename MatType::elem_type eT; + + MatType inputData; if (!data::Load("iris_train.csv", inputData)) FAIL("Cannot load test dataset iris_train.csv!"); - arma::Mat labels; + Mat labels; if (!data::Load("iris_train_labels.csv", labels)) FAIL("Cannot load labels for iris_train_labels.csv"); const size_t numClasses = max(labels.row(0)) + 1; @@ -684,27 +748,30 @@ TEST_CASE("ClassifyTest_IRIS", "[AdaBoostTest]") // Run the perceptron for perceptronIter iterations. size_t perceptronIter = 800; - Perceptron<> p(inputData, labels.row(0), numClasses, perceptronIter); + typedef Perceptron + PerceptronType; + PerceptronType p(inputData, labels.row(0), numClasses, perceptronIter); // Define parameters for AdaBoost. size_t iterations = 50; - double tolerance = 1e-10; - AdaBoost<> a(inputData, labels.row(0), numClasses, p, iterations, tolerance); + eT tolerance = 1e-10; + AdaBoost a(inputData, labels.row(0), numClasses, + iterations, tolerance, perceptronIter); - arma::mat testData; + MatType testData; if (!data::Load("iris_test.csv", testData)) FAIL("Cannot load test dataset iris_test.csv!"); - arma::Row predictedLabels(testData.n_cols); + Row predictedLabels(testData.n_cols); a.Classify(testData, predictedLabels); - arma::Mat trueTestLabels; + Mat trueTestLabels; if (!data::Load("iris_test_labels.csv", trueTestLabels)) FAIL("Cannot load test dataset iris_test_labels.csv!"); - arma::Row predictedLabels1(testData.n_cols), - predictedLabels2(testData.n_cols); - arma::mat probabilities; + Row predictedLabels1(testData.n_cols), + predictedLabels2(testData.n_cols); + MatType probabilities; a.Classify(testData, predictedLabels1); a.Classify(testData, predictedLabels2, probabilities); @@ -714,8 +781,8 @@ TEST_CASE("ClassifyTest_IRIS", "[AdaBoostTest]") for (size_t i = 0; i < predictedLabels1.n_cols; ++i) REQUIRE(predictedLabels1[i] == predictedLabels2[i]); - arma::colvec pRow; - arma::uword maxIndex = 0; + Col pRow; + uword maxIndex = 0; for (size_t i = 0; i < predictedLabels1.n_cols; ++i) { @@ -726,7 +793,7 @@ TEST_CASE("ClassifyTest_IRIS", "[AdaBoostTest]") } size_t localError = arma::accu(trueTestLabels != predictedLabels1); - double lError = (double) localError / labels.n_cols; + eT lError = (eT) localError / labels.n_cols; REQUIRE(lError <= 0.30); } @@ -734,26 +801,32 @@ TEST_CASE("ClassifyTest_IRIS", "[AdaBoostTest]") * Ensure that the Train() function works like it is supposed to, by building * AdaBoost on one dataset and then re-training on another dataset. */ -TEST_CASE("TrainTest", "[AdaBoostTest]") +TEMPLATE_TEST_CASE("TrainTest", "[AdaBoostTest]", mat, fmat) { + typedef TestType MatType; + typedef typename MatType::elem_type eT; + // First train on the iris dataset. - arma::mat inputData; + MatType inputData; if (!data::Load("iris_train.csv", inputData)) FAIL("Cannot load test dataset iris_train.csv!"); - arma::Mat labels; + Mat labels; if (!data::Load("iris_train_labels.csv", labels)) FAIL("Cannot load labels for iris_train_labels.csv"); const size_t numClasses = max(labels.row(0)) + 1; size_t perceptronIter = 800; - Perceptron<> p(inputData, labels.row(0), numClasses, perceptronIter); + typedef Perceptron + PerceptronType; + PerceptronType p(inputData, labels.row(0), numClasses, perceptronIter); // Now train AdaBoost. size_t iterations = 50; - double tolerance = 1e-10; - AdaBoost<> a(inputData, labels.row(0), numClasses, p, iterations, tolerance); + eT tolerance = 1e-10; + AdaBoost a(inputData, labels.row(0), numClasses, + iterations, tolerance, perceptronIter); // Now load another dataset... if (!data::Load("vc2.csv", inputData)) @@ -763,44 +836,48 @@ TEST_CASE("TrainTest", "[AdaBoostTest]") const size_t newNumClasses = max(labels.row(0)) + 1; - Perceptron<> p2(inputData, labels.row(0), newNumClasses, perceptronIter); + PerceptronType p2(inputData, labels.row(0), newNumClasses, perceptronIter); - a.Train(inputData, labels.row(0), newNumClasses, p2, iterations, tolerance); + a.Train(inputData, labels.row(0), newNumClasses, iterations, tolerance, + perceptronIter); // Load test set to see if it trained on vc2 correctly. - arma::mat testData; + MatType testData; if (!data::Load("vc2_test.csv", testData)) FAIL("Cannot load test dataset vc2_test.csv!"); - arma::Mat trueTestLabels; + Mat trueTestLabels; if (!data::Load("vc2_test_labels.txt", trueTestLabels)) FAIL("Cannot load labels for vc2_test_labels.txt"); // Define parameters for AdaBoost. - arma::Row predictedLabels(testData.n_cols); + Row predictedLabels(testData.n_cols); a.Classify(testData, predictedLabels); - int localError = arma::accu(trueTestLabels != predictedLabels); - double lError = (double) localError / trueTestLabels.n_cols; + size_t localError = accu(trueTestLabels != predictedLabels); + eT lError = (eT) localError / trueTestLabels.n_cols; REQUIRE(lError <= 0.30); } -TEST_CASE("PerceptronSerializationTest", "[AdaBoostTest]") +TEMPLATE_TEST_CASE("PerceptronSerializationTest", "[AdaBoostTest]", fmat, mat) { + typedef TestType MatType; + // Build an AdaBoost object. - mat data = randu(10, 500); + MatType data = randu(10, 500); Row labels(500); for (size_t i = 0; i < 250; ++i) labels[i] = 0; for (size_t i = 250; i < 500; ++i) labels[i] = 1; - Perceptron<> p(data, labels, 2, 800); - AdaBoost<> ab(data, labels, 2, p, 50, 1e-10); + typedef Perceptron + PerceptronType; + AdaBoost ab(data, labels, 2, 50, 1e-10, 800); // Now create another dataset to train with. - mat otherData = randu(5, 200); + MatType otherData = randu(5, 200); Row otherLabels(200); for (size_t i = 0; i < 100; ++i) otherLabels[i] = 1; @@ -809,10 +886,10 @@ TEST_CASE("PerceptronSerializationTest", "[AdaBoostTest]") for (size_t i = 150; i < 200; ++i) otherLabels[i] = 2; - Perceptron<> p2(otherData, otherLabels, 3, 500); - AdaBoost<> abText(otherData, otherLabels, 3, p2, 50, 1e-10); + AdaBoost abText(otherData, otherLabels, 3, 50, 1e-10, + 500); - AdaBoost<> abXml, abBinary; + AdaBoost abXml, abBinary; SerializeObjectAll(ab, abXml, abText, abBinary); @@ -839,21 +916,23 @@ TEST_CASE("PerceptronSerializationTest", "[AdaBoostTest]") } } -TEST_CASE("ID3DecisionStumpSerializationTest", "[AdaBoostTest]") +TEMPLATE_TEST_CASE("ID3DecisionStumpSerializationTest", "[AdaBoostTest]", mat, + fmat) { + typedef TestType MatType; + // Build an AdaBoost object. - mat data = randu(10, 500); + MatType data = randu(10, 500); Row labels(500); for (size_t i = 0; i < 250; ++i) labels[i] = 0; for (size_t i = 250; i < 500; ++i) labels[i] = 1; - ID3DecisionStump p(data, labels, 2, 800); - AdaBoost ab(data, labels, 2, p, 50, 1e-10); + AdaBoost ab(data, labels, 2, 50, 1e-10, 40); // Now create another dataset to train with. - mat otherData = randu(5, 200); + MatType otherData = randu(5, 200); Row otherLabels(200); for (size_t i = 0; i < 100; ++i) otherLabels[i] = 1; @@ -862,10 +941,10 @@ TEST_CASE("ID3DecisionStumpSerializationTest", "[AdaBoostTest]") for (size_t i = 150; i < 200; ++i) otherLabels[i] = 2; - ID3DecisionStump p2(otherData, otherLabels, 3, 500); - AdaBoost abText(otherData, otherLabels, 3, p2, 50, 1e-10); + AdaBoost abText(otherData, otherLabels, 3, 50, + 1e-10, 25); - AdaBoost abXml, abBinary; + AdaBoost abXml, abBinary; SerializeObjectAll(ab, abXml, abText, abBinary); @@ -881,10 +960,147 @@ TEST_CASE("ID3DecisionStumpSerializationTest", "[AdaBoostTest]") for (size_t i = 0; i < ab.WeakLearners(); ++i) { REQUIRE(ab.WeakLearner(i).SplitDimension() == - abXml.WeakLearner(i).SplitDimension()); + abXml.WeakLearner(i).SplitDimension()); REQUIRE(ab.WeakLearner(i).SplitDimension() == abText.WeakLearner(i).SplitDimension()); REQUIRE(ab.WeakLearner(i).SplitDimension() == abBinary.WeakLearner(i).SplitDimension()); } } + +TEMPLATE_TEST_CASE("AdaBoostSinglePointClassify", "[AdaBoostTest]", mat, fmat) +{ + typedef TestType MatType; + + // Create random data. + MatType data = randu(10, 100); + // Create random labels. + Row labels = randi>(100, distr_param(0, 3)); + + // Train a model. + typedef Perceptron + PerceptronType; + AdaBoost ab(data, labels, 4); + + // Ensure that we can get single-point classifications. + for (size_t i = 0; i < 100; ++i) + { + const size_t prediction = ab.Classify(data.col(i)); + + REQUIRE(prediction <= 3); + } +} + +TEMPLATE_TEST_CASE("AdaBoostSinglePointClassifyWithProbs", "[AdaBoostTest]", + mat, fmat) +{ + typedef TestType MatType; + typedef typename MatType::elem_type eT; + + // Create random data. + MatType data = randu(10, 100); + // Create random labels. + Row labels = randi>(100, distr_param(0, 3)); + + // Train a model. + typedef Perceptron + PerceptronType; + AdaBoost ab(data, labels, 4); + + // Ensure that we can get single-point classifications. + for (size_t i = 0; i < 100; ++i) + { + size_t prediction; + Row probabilities; + ab.Classify(data.col(i), prediction, probabilities); + + REQUIRE(prediction <= 3); + REQUIRE(accu(probabilities) == Approx((eT) 1.0)); + } +} + +// Make sure that everything works when we use the constructor that takes extra +// hyperparameters. +TEMPLATE_TEST_CASE("AdaBoostParamsConstructor", "[AdaBoostTest]", fmat, mat) +{ + typedef TestType MatType; + + MatType inputData; + if (!data::Load("iris.csv", inputData)) + FAIL("Cannot load test dataset iris.csv!"); + + Mat labels; + if (!data::Load("iris_labels.txt", labels)) + FAIL("Cannot load labels for iris iris_labels.txt"); + + const size_t numClasses = max(labels.row(0)) + 1; + + // Create two AdaBoost models. One does not allow the perceptron to train for + // more than one iteration, and therefore should get less accuracy than the + // one we let train in full. + typedef Perceptron + PerceptronType; + + AdaBoost a1(inputData, labels, numClasses, 2, 1e-6, + 1 /* perceptron max iterations */); + AdaBoost a2(inputData, labels, numClasses, 2, 1e-6, + 100 /* perceptron max iterations */); + + // Make sure test data performance is better for a2. + Row predictions1, predictions2; + a1.Classify(inputData, predictions1); + a2.Classify(inputData, predictions2); + + const size_t correct1 = accu(predictions1 == labels); + const size_t correct2 = accu(predictions2 == labels); + + REQUIRE(correct2 > 0); + REQUIRE(correct2 >= correct1); +} + +// Ensure that all Train() overloads work correctly. +TEMPLATE_TEST_CASE("AdaBoostTrainOverloads", "[AdaBoostTest]", fmat, mat) +{ + typedef TestType MatType; + + // Create random data. + MatType data = randu(10, 100); + // Create random labels. + Row labels = randi>(100, distr_param(0, 3)); + + typedef Perceptron + PerceptronType; + AdaBoost a1, a2, a3, a4; + a1.MaxIterations() = 65; + a1.Tolerance() = 2e-4; + a2.Tolerance() = 2e-5; + + a1.Train(data, labels, 4); + a2.Train(data, labels, 4, 15); + a3.Train(data, labels, 4, 55, 1e-3); + a4.Train(data, labels, 4, 60, 2e-3, 100); + + // Make sure hyperparameters were set correctly, where appropriate. + REQUIRE(a1.MaxIterations() == 65); + REQUIRE(a2.MaxIterations() == 15); + REQUIRE(a3.MaxIterations() == 55); + REQUIRE(a4.MaxIterations() == 60); + + REQUIRE(a1.Tolerance() == Approx(2e-4)); + REQUIRE(a2.Tolerance() == Approx(2e-5)); + REQUIRE(a3.Tolerance() == Approx(1e-3)); + REQUIRE(a4.Tolerance() == Approx(2e-3)); + + // Make sure anything at all was trained. + REQUIRE(a1.WeakLearners() > 0); + REQUIRE(a2.WeakLearners() > 0); + REQUIRE(a3.WeakLearners() > 0); + REQUIRE(a4.WeakLearners() > 0); + + // Make sure the maximum number of iterations in the perceptron was set + // properly. + REQUIRE(a1.WeakLearner(0).MaxIterations() == 1000); + REQUIRE(a2.WeakLearner(0).MaxIterations() == 1000); + REQUIRE(a3.WeakLearner(0).MaxIterations() == 1000); + REQUIRE(a4.WeakLearner(0).MaxIterations() == 100); +} diff --git a/src/mlpack/tests/ann/layer/parametric_relu.cpp b/src/mlpack/tests/ann/layer/parametric_relu.cpp index 3d67c7c48d..479245d0eb 100644 --- a/src/mlpack/tests/ann/layer/parametric_relu.cpp +++ b/src/mlpack/tests/ann/layer/parametric_relu.cpp @@ -132,8 +132,6 @@ TEST_CASE("PReLUIntegrationTest", "[ANNLayerTest]") double msreTrain = ComputeMSRE(predictions, trainLabels); model.Predict(testData, predictions); double msreTest = ComputeMSRE(predictions, testLabels); - std::cout << "train: " << msreTrain << "\n"; - std::cout << "test: " << msreTest << "\n"; double relativeMSRE = std::abs((msreTest - msreTrain) / msreTrain); if (relativeMSRE <= 0.35) diff --git a/src/mlpack/tests/perceptron_test.cpp b/src/mlpack/tests/perceptron_test.cpp index f49dfa2b54..631584bff2 100644 --- a/src/mlpack/tests/perceptron_test.cpp +++ b/src/mlpack/tests/perceptron_test.cpp @@ -42,20 +42,20 @@ TEST_CASE("SimpleWeightUpdateWeights", "[PerceptronTest]") wip.UpdateWeights(trainingPoint, weights, biases, incorrectClass, correctClass); - CHECK(weights(0, 0) == -1); - CHECK(weights(1, 0) == 0); - CHECK(weights(2, 0) == 1); - CHECK(weights(3, 0) == 2); - CHECK(weights(4, 0) == 3); + REQUIRE(weights(0, 0) == -1); + REQUIRE(weights(1, 0) == 0); + REQUIRE(weights(2, 0) == 1); + REQUIRE(weights(3, 0) == 2); + REQUIRE(weights(4, 0) == 3); - CHECK(weights(0, 2) == 7); - CHECK(weights(1, 2) == 8); - CHECK(weights(2, 2) == 9); - CHECK(weights(3, 2) == 10); - CHECK(weights(4, 2) == 11); + REQUIRE(weights(0, 2) == 7); + REQUIRE(weights(1, 2) == 8); + REQUIRE(weights(2, 2) == 9); + REQUIRE(weights(3, 2) == 10); + REQUIRE(weights(4, 2) == 11); - CHECK(biases(0) == 1); - CHECK(biases(2) == 8); + REQUIRE(biases(0) == 1); + REQUIRE(biases(2) == 8); } /** @@ -85,20 +85,20 @@ TEST_CASE("SimpleWeightUpdateInstanceWeight", "[PerceptronTest]") wip.UpdateWeights(trainingPoint, weights, biases, incorrectClass, correctClass, instanceWeight); - CHECK(weights(0, 0) == -3); - CHECK(weights(1, 0) == -4); - CHECK(weights(2, 0) == -5); - CHECK(weights(3, 0) == -6); - CHECK(weights(4, 0) == -7); + REQUIRE(weights(0, 0) == -3); + REQUIRE(weights(1, 0) == -4); + REQUIRE(weights(2, 0) == -5); + REQUIRE(weights(3, 0) == -6); + REQUIRE(weights(4, 0) == -7); - CHECK(weights(0, 2) == 9); - CHECK(weights(1, 2) == 12); - CHECK(weights(2, 2) == 15); - CHECK(weights(3, 2) == 18); - CHECK(weights(4, 2) == 21); + REQUIRE(weights(0, 2) == 9); + REQUIRE(weights(1, 2) == 12); + REQUIRE(weights(2, 2) == 15); + REQUIRE(weights(3, 2) == 18); + REQUIRE(weights(4, 2) == 21); - CHECK(biases(0) == -1); - CHECK(biases(2) == 10); + REQUIRE(biases(0) == -1); + REQUIRE(biases(2) == 10); } /** @@ -120,10 +120,21 @@ TEST_CASE("And", "[PerceptronTest]") Row predictedLabels; p.Classify(testData, predictedLabels); - CHECK(predictedLabels(0, 0) == 0); - CHECK(predictedLabels(0, 1) == 0); - CHECK(predictedLabels(0, 2) == 1); - CHECK(predictedLabels(0, 3) == 0); + REQUIRE(predictedLabels(0) == 0); + REQUIRE(predictedLabels(1) == 0); + REQUIRE(predictedLabels(2) == 1); + REQUIRE(predictedLabels(3) == 0); + + // Test single-point classify too. + predictedLabels(0) = p.Classify(testData.col(0)); + predictedLabels(1) = p.Classify(testData.col(1)); + predictedLabels(2) = p.Classify(testData.col(2)); + predictedLabels(3) = p.Classify(testData.col(3)); + + REQUIRE(predictedLabels(0) == 0); + REQUIRE(predictedLabels(1) == 0); + REQUIRE(predictedLabels(2) == 1); + REQUIRE(predictedLabels(3) == 0); } /** @@ -146,10 +157,10 @@ TEST_CASE("Or", "[PerceptronTest]") Row predictedLabels; p.Classify(testData, predictedLabels); - CHECK(predictedLabels(0, 0) == 1); - CHECK(predictedLabels(0, 1) == 1); - CHECK(predictedLabels(0, 2) == 1); - CHECK(predictedLabels(0, 3) == 0); + REQUIRE(predictedLabels(0, 0) == 1); + REQUIRE(predictedLabels(0, 1) == 1); + REQUIRE(predictedLabels(0, 2) == 1); + REQUIRE(predictedLabels(0, 3) == 0); } /** @@ -174,7 +185,7 @@ TEST_CASE("Random3", "[PerceptronTest]") p.Classify(testData, predictedLabels); for (size_t i = 0; i < predictedLabels.n_cols; ++i) - CHECK(predictedLabels(0, i) == 0); + REQUIRE(predictedLabels(0, i) == 0); } /** @@ -198,35 +209,40 @@ TEST_CASE("TwoPoints", "[PerceptronTest]") Row predictedLabels; p.Classify(testData, predictedLabels); - CHECK(predictedLabels(0, 0) == 0); - CHECK(predictedLabels(0, 1) == 1); + REQUIRE(predictedLabels(0, 0) == 0); + REQUIRE(predictedLabels(0, 1) == 1); } /** * This tests the convergence of the perceptron on a dataset which has a - * non-linearly separable dataset. + * non-linearly separable dataset. We test on multiple element types to ensure + * that MatType can be set correctly. */ -TEST_CASE("NonLinearlySeparableDataset", "[PerceptronTest]") +TEMPLATE_TEST_CASE("NonLinearlySeparableDataset", "[PerceptronTest]", float, + double) { - mat trainData; + typedef TestType eT; + + Mat trainData; trainData = { { 1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8 }, { 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2 } }; Mat labels; labels = { 0, 0, 0, 1, 0, 1, 1, 1, 0, 0, 0, 1, 0, 1, 1, 1 }; - Perceptron<> p(trainData, labels.row(0), 2, 1000); + Perceptron> p(trainData, + labels.row(0), 2, 1000); - mat testData; + Mat testData; testData = { { 3, 4, 5, 6 }, { 3, 2.3, 1.7, 1.5 } }; Row predictedLabels; p.Classify(testData, predictedLabels); - CHECK(predictedLabels(0, 0) == 0); - CHECK(predictedLabels(0, 1) == 0); - CHECK(predictedLabels(0, 2) == 1); - CHECK(predictedLabels(0, 3) == 1); + REQUIRE(predictedLabels(0, 0) == 0); + REQUIRE(predictedLabels(0, 1) == 0); + REQUIRE(predictedLabels(0, 2) == 1); + REQUIRE(predictedLabels(0, 3) == 1); } TEST_CASE("SecondaryConstructor", "[PerceptronTest]") @@ -266,3 +282,82 @@ TEST_CASE("InstanceWeightsConstructor", "[PerceptronTest]") REQUIRE(p.Weights().n_elem > 0); } + +/** + * This tests that incremental training can be stopped with `Reset()`. + */ +TEST_CASE("IncrementalTrainingTest", "[PerceptronTest]") +{ + mat trainData = randu(10, 100); + for (size_t i = 0; i < 50; ++i) + trainData.col(i) -= 0.5; + for (size_t i = 50; i < 100; ++i) + trainData.col(i) += 0.5; + Row labels(100); + labels.subvec(0, 49).zeros(); + labels.subvec(50, 99).ones(); + + Perceptron<> p1, p2; + // This should result in the same model, because the default initialization is + // zeros. + p1.Train(trainData, labels, 2, 1); + p2.Train(trainData, labels, 2, 1); + + REQUIRE(approx_equal(p1.Weights(), p2.Weights(), "absdiff", 1e-5)); + REQUIRE(approx_equal(p1.Biases(), p2.Biases(), "absdiff", 1e-5)); + + // Resetting and retraining p2 should result in the same model. + p2.Reset(); + p2.Train(trainData, labels, 2); + + REQUIRE(approx_equal(p1.Weights(), p2.Weights(), "absdiff", 1e-5)); + REQUIRE(approx_equal(p1.Biases(), p2.Biases(), "absdiff", 1e-5)); + + // Training p1 again should result in a different model. + p1.Train(trainData, labels, 2); + + REQUIRE(!approx_equal(p1.Weights(), p2.Weights(), "absdiff", 1e-5)); + REQUIRE(!approx_equal(p1.Biases(), p2.Biases(), "absdiff", 1e-5)); +} + +// Test all forms of Train(). +TEST_CASE("TrainFormTest", "[PerceptronTest]") +{ + mat trainData; + trainData = { { 1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8 }, + { 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2 } }; + + Row labels; + labels = { 0, 0, 0, 1, 0, 1, 1, 1, 0, 0, 0, 1, 0, 1, 1, 1 }; + + rowvec weights(labels.n_elem); + weights.ones(); + + Perceptron<> p1(trainData, labels, 2); + Perceptron<> p2(trainData, labels, 2, weights); + Perceptron<> p3, p4, p5, p6; + p3.Train(trainData, labels, 2); + p4.Train(trainData, labels, 2, 50); + p5.Train(trainData, labels, 2, weights); + p6.Train(trainData, labels, 2, weights, 50); + + mat testData; + testData = { { 3, 4, 5, 6 }, + { 3, 2.3, 1.7, 1.5 } }; + Row predictions1, predictions2, predictions3, predictions4, + predictions5, predictions6; + Row trueLabels = { 0, 0, 1, 1 }; + p1.Classify(testData, predictions1); + p2.Classify(testData, predictions2); + p3.Classify(testData, predictions3); + p4.Classify(testData, predictions4); + p5.Classify(testData, predictions5); + p6.Classify(testData, predictions6); + + REQUIRE(all(predictions1 == trueLabels)); + REQUIRE(all(predictions2 == trueLabels)); + REQUIRE(all(predictions3 == trueLabels)); + REQUIRE(all(predictions4 == trueLabels)); + REQUIRE(all(predictions5 == trueLabels)); + REQUIRE(all(predictions6 == trueLabels)); +} diff --git a/src/mlpack/tests/serialization.cpp b/src/mlpack/tests/serialization.cpp index 0631be7b73..8fca67354d 100644 --- a/src/mlpack/tests/serialization.cpp +++ b/src/mlpack/tests/serialization.cpp @@ -14,7 +14,7 @@ namespace mlpack { -// Utility function to check the equality of two Armadillo matrices. +// Utility function to check the equality of four Armadillo matrices. void CheckMatrices(const arma::mat& x, const arma::mat& xmlX, const arma::mat& jsonX, @@ -39,15 +39,52 @@ void CheckMatrices(const arma::mat& x, const double val = x[i]; if (val == 0.0) { - REQUIRE(xmlX[i] == Approx(0.0).margin(1e-6 / 100)); - REQUIRE(jsonX[i] == Approx(0.0).margin(1e-6 / 100)); - REQUIRE(binaryX[i] == Approx(0.0).margin(1e-6 / 100)); + REQUIRE(xmlX[i] == Approx(0.0).margin(1e-8)); + REQUIRE(jsonX[i] == Approx(0.0).margin(1e-8)); + REQUIRE(binaryX[i] == Approx(0.0).margin(1e-8)); } else { - REQUIRE(val == Approx(xmlX[i]).epsilon(1e-6 / 100)); - REQUIRE(val == Approx(jsonX[i]).epsilon(1e-6 / 100)); - REQUIRE(val == Approx(binaryX[i]).epsilon(1e-6 / 100)); + REQUIRE(val == Approx(xmlX[i]).epsilon(1e-8)); + REQUIRE(val == Approx(jsonX[i]).epsilon(1e-8)); + REQUIRE(val == Approx(binaryX[i]).epsilon(1e-8)); + } + } +} + +void CheckMatrices(const arma::fmat& x, + const arma::fmat& xmlX, + const arma::fmat& jsonX, + const arma::fmat& binaryX) +{ + // First check dimensions. + REQUIRE(x.n_rows == xmlX.n_rows); + REQUIRE(x.n_rows == jsonX.n_rows); + REQUIRE(x.n_rows == binaryX.n_rows); + + REQUIRE(x.n_cols == xmlX.n_cols); + REQUIRE(x.n_cols == jsonX.n_cols); + REQUIRE(x.n_cols == binaryX.n_cols); + + REQUIRE(x.n_elem == xmlX.n_elem); + REQUIRE(x.n_elem == jsonX.n_elem); + REQUIRE(x.n_elem == binaryX.n_elem); + + // Now check elements. + for (size_t i = 0; i < x.n_elem; ++i) + { + const float val = x[i]; + if (val == 0.0) + { + REQUIRE(xmlX[i] == Approx(0.0).margin(1e-6)); + REQUIRE(jsonX[i] == Approx(0.0).margin(1e-6)); + REQUIRE(binaryX[i] == Approx(0.0).margin(1e-6)); + } + else + { + REQUIRE(val == Approx(xmlX[i]).epsilon(1e-6)); + REQUIRE(val == Approx(jsonX[i]).epsilon(1e-6)); + REQUIRE(val == Approx(binaryX[i]).epsilon(1e-6)); } } } diff --git a/src/mlpack/tests/serialization.hpp b/src/mlpack/tests/serialization.hpp index 361523f701..38cb6de9dd 100644 --- a/src/mlpack/tests/serialization.hpp +++ b/src/mlpack/tests/serialization.hpp @@ -219,6 +219,11 @@ void CheckMatrices(const arma::mat& x, const arma::mat& jsonX, const arma::mat& binaryX); +void CheckMatrices(const arma::fmat& x, + const arma::fmat& xmlX, + const arma::fmat& jsonX, + const arma::fmat& binaryX); + void CheckMatrices(const arma::Mat& x, const arma::Mat& xmlX, const arma::Mat& jsonX,