diff --git a/doc/citation.md b/doc/citation.md new file mode 100644 index 0000000000..fc23af8746 --- /dev/null +++ b/doc/citation.md @@ -0,0 +1,52 @@ +# Citation details + +If you use mlpack in your research or software, please cite mlpack using the +citation for +[the paper below](https://joss.theoj.org/papers/10.21105/joss.05026): + + * ***mlpack 4: a fast, header-only C++ machine learning library***. R.R. + Curtin, M. Edel, O. Shrit, S. Agrawal, S. Basak, J.J. Balamuta, R. + Birmingham, K. Dutt, D. Eddelbuettel, R. Garg, S. Jaiswal, A. Kaushik, S. + Kim, A. Mukherjee, N.G. Sai, N. Sharma, Y.S. Parihar, R. Swain, C. Sanderson. + _Journal of Open Source Software_ 8:82, p. 5026, 2023. + +In BibTeX format: + +``` +@article{mlpack2023, + title = {mlpack 4: a fast, header-only C++ machine learning library}, + author = {Ryan R. Curtin and Marcus Edel and Omar Shrit and + Shubham Agrawal and Suryoday Basak and James J. Balamuta and + Ryan Birmingham and Kartik Dutt and Dirk Eddelbuettel and + Rishabh Garg and Shikhar Jaiswal and Aakash Kaushik and + Sangyeon Kim and Anjishnu Mukherjee and Nanubala Gnana Sai and + Nippun Sharma and Yashwant Singh Parihar and Roshan Swain and + Conrad Sanderson}, + journal = {Journal of Open Source Software}, + volume = {8}, + number = {82}, + pages = {5026}, + year = {2023}, + doi = {10.21105/joss.05026}, + url = {https://doi.org/10.21105/joss.05026} +} +``` + +Citations are beneficial for the growth and improvement of mlpack. + +### Older papers + +See also the following older papers concerning mlpack, its community, and its +internal design. + + * [mlpack 3: a fast, flexible machine learning + library](https://joss.theoj.org/papers/10.21105/joss.00726) (2018) + + * [mlpack open-source machine learning library and + community](http://kurg.org/pub/pdf/2018mlossmlpack.pdf) (2018) + + * [Designing and building the mlpack open-source machine learning + library](https://arxiv.org/abs/1708.05279) (2017) + + * [mlpack: a scalable C++ machine learning + library](https://www.jmlr.org/papers/volume14/curtin13a/curtin13a.pdf) (2013) diff --git a/doc/developer/version.md b/doc/developer/version.md deleted file mode 100644 index c303f2a624..0000000000 --- a/doc/developer/version.md +++ /dev/null @@ -1,25 +0,0 @@ -# mlpack versions in code - -mlpack provides a couple of convenience macros and functions to get the version -of mlpack. More information (and straightforward code) can be found in -`src/mlpack/core/util/version.hpp`. - -The following three macros provide major, minor, and patch versions of mlpack -(i.e. for `mlpack-x.y.z`, `x` is the major version, `y` is the minor version, -and `z` is the patch version): - -```c++ -MLPACK_VERSION_MAJOR -MLPACK_VERSION_MINOR -MLPACK_VERSION_PATCH -``` - -In addition, the function `mlpack::util::GetVersion()` returns the mlpack -version as a string (for instance, `"mlpack 1.0.8"`). - -## mlpack command-line program versions - -Each mlpack command-line program supports the `--version` (or `-V`) option, -which will print the version of mlpack used. If the version is not an official -release but instead from git, the version will be `mlpack git` (and will have a -git revision SHA appended to `git`). diff --git a/doc/index.md b/doc/index.md new file mode 100644 index 0000000000..1b43b3fc1b --- /dev/null +++ b/doc/index.md @@ -0,0 +1,156 @@ +# Documentation for mlpack + +## A fast, flexible machine learning library + +mlpack is an intuitive, fast, and flexible header-only C++ machine learning +library with bindings to other languages. It aims to provide fast, lightweight +implementations of both common and cutting-edge machine learning algorithms. + +mlpack's lightweight C++ implementation makes it ideal for deployment, and it +can also be used for interactive prototyping via C++ notebooks (these can be +seen in action on mlpack's [homepage](https://www.mlpack.org/)). + +In addition to its [powerful C++ interface](quickstart/cpp.md), mlpack also +provides [command-line programs](quickstart/cli.md), and bindings to the +[Python](quickstart/python.md), [R](quickstart/r.md), +[Julia](quickstart/julia.md), and [Go](quickstart/go.md) languages. + +_If you use mlpack, please [cite the software](citation.md)._ + +## mlpack basics + +Installing mlpack can be done using the +[instructions in the README](README.md#3-installing-and-using-mlpack-in-c); +or the [Windows build guide](user/build_windows.md). +The following basic guides are *highly recommended* before using mlpack. + + * ***First steps***: + - [mlpack C++ quickstart](quickstart/cpp.md): create a couple simple C++ + programs that use mlpack + - [Sample Windows mlpack C++ application](user/sample_ml_app.md): create a + working mlpack Windows program using Visual Studio + + * ***Basics of matrices and data in mlpack***: + - [Matrices and data in mlpack](user/matrices.md) + - [Loading and saving mlpack objects](user/load_save.md) + + * ***Reference for mlpack core classes***: + - [Core mlpack documentation](user/core.md) + + * ***Using mlpack natively with our extensions in Python, R, CLI, Julia, and Go***: + - [Links to quickstarts and references](#bindings-to-other-languages) + +## mlpack algorithm documentation + +Documentation for each machine learning algorithm that mlpack implements is +detailed in the sections below. + + * [Classification algorithms](#classification-algorithms): classify points as + discrete labels (`0`, `1`, `2`, ...). + * [Regression algorithms](#regression-algorithms): predict continuous values. + * [Clustering algorithms](#clustering-algorithms): group points into clusters. + * [Geometric algorithms](#geometric-algorithms): computations based on distance + metrics (nearest neighbors, kernel density estimation, etc.). + * [Preprocessing utilities](#preprocessing-utilities): prepare data for machine + learning algorithms. + * [Transformations](#transformations): transform data from one space to + another (principal components analysis, etc.). + * [Modeling utilities](#modeling-utilities): cross-validation, hyperparameter + tuning, etc. + +### Classification algorithms + +Classify points as discrete labels (`0`, `1`, `2`, ...). + + * [`AdaBoost`](user/methods/adaboost.md): Adaptive Boosting + * [`DecisionTree`](user/methods/decision_tree.md): ID3-style decision tree + classifier + * [`LogisticRegression`](user/methods/logistic_regression.md): L2-regularized + logistic regression (two-class only) + * [`NaiveBayesClassifier`](user/methods/naive_bayes_classifier.md): simple + multi-class naive Bayes classifier + * [`Perceptron`](user/methods/perceptron.md): simple Perceptron classifier + * [`SoftmaxRegression`](user/methods/softmax_regression.md): L2-regularized + softmax regression (i.e. multi-class logistic regression) + +### Regression algorithms + +Predict continuous values. + + * [`DecisionTreeRegressor`](user/methods/decision_tree_regressor.md): ID3-style + decision tree regressor + * [`LARS`](user/methods/lars.md): Least Angle Regression (LARS), L1-regularized + and L2-regularized + * [`LinearRegression`](user/methods/linear_regression.md): L2-regularized + linear regression (ridge regression) + +### Clustering algorithms + +Group points into clusters. + + + +### Geometric algorithms + +Computations based on distance metrics. + + + +### Preprocessing utilities + +Prepare data for machine learning algorithms. + + + +### Transformations + +Transform data from one space to another. + + + +### Modeling utilities + +Cross-validation, hyperparameter tuning, etc. + + + +## Bindings to other languages + +mlpack's bindings to other languages have less complete functionality than +mlpack in C++, but almost all the same algorithms are available. + +| ***Python*** | -- | [quickstart](quickstart/python.md) | -- | [reference](https://www.mlpack.org/doc/python_documentation.html) | +| ***Julia*** | -- | [quickstart](quickstart/julia.md) | -- | [reference](https://www.mlpack.org/doc/julia_documentation.html) | +| ***R*** | -- | [quickstart](quickstart/r.md) | -- | [reference](https://www.mlpack.org/doc/r_documentation.html) +| ***Command-line programs*** | -- | [quickstart](quickstart/cli.md) | -- | [reference](https://www.mlpack.org/doc/cli_documentation.html) | +| ***Go*** | -- | [quickstart](quickstart/go.md) | -- | [reference](https://www.mlpack.org/doc/go_documentation.html) | + +## Examples and further documentation + + * [mlpack examples repository](https://github.com/mlpack/examples/): numerous + fully-working example applications of mlpack, in C++ and other languages. + * [mlpack models repository](https://github.com/mlpack/models/): complex models + in C++ built with mlpack + +For additional documentation beyond what is covered in all the resources above, +the source code should be consulted. Each method is fully documented. + +## Developer documentation + +Throughout the codebase, mlpack uses some common template parameter policies. +These are documented below. + + * [The `ElemType` policy](developer/elemtype.md): element types for data + * [The `MetricType` policy](developer/metrics.md): distance metrics + * [The `KernelType` policy](developer/kernels.md): kernel functions + * [The `TreeType` policy](developer/trees.md): space trees (ball trees, + KD-trees, etc.) + +In addition, the following documentation may be useful when developing bindings +for other languages: + + * [Timers](developer/timer.md): timing parts of bindings + * [Writing an mlpack binding](developer/iodoc.md): simple examples of mlpack + bindings + * [Automatic bindings](developer/bindings.md): details on mlpack's automatic + binding generator system. diff --git a/doc/user/core.md b/doc/user/core.md new file mode 100644 index 0000000000..4b930c7d9a --- /dev/null +++ b/doc/user/core.md @@ -0,0 +1,598 @@ +# mlpack core class documentation + +Underlying the implementations of [mlpack's machine learning +algorithms](index.md#mlpack-algorithm-documentation) are mlpack core support +classes, each of which are documented on this page. + + * [Core math utilities](#core-math-utilities): utility classes for mathematical + purposes + * [Distributions](#distributions): probability distributions + * [Metrics](#metrics): distance metrics for geometric algorithms + * [Kernels](#kernels): Mercer kernels for kernel-based algorithms + +## Core math utilities + +Utilities in the `mlpack::math::` namespace are meant to provide additional +mathematical support on top of Armadillo. + + * [`math::Range`](#mathrange): simple mathematical range (i.e. `[0, 3]`) + +--- + +### `math::Range` + +The `math::Range` class represents a simple mathematical range (i.e. `[0, 3]`), +with the bounds represented as `double`s. + +--- + +#### Constructors + + * `r = math::Range()` + - Construct an empty range. + + * `r = math::Range(p)` + - Construct the range `[p, p]`. + + * `r = math::Range(lo, hi)` + - Construct the range `[lo, hi]`. + +--- + +#### Accessing and modifying range properties + + * `r.Lo()` and `r.Hi()` return the lower and upper bounds of the range as + `double`s. + - A range is considered empty if `r.Lo() > r.Hi()`. + - These can be used to modify the bounds, e.g., `r.Lo() = 3.0`. + + * `r.Width()` returns the span of the range (i.e. `r.Hi() - r.Lo()`) as a + `double`. + + * `r.Mid()` returns the midpoint of the range as a `double`. + +--- + +#### Working with ranges + + * Given two ranges `r1` and `r2`, + - `r1 | r2` returns the union of the ranges, + - `r1 |= r2` expands `r1` to include the range `r2`, + - `r1 & r2` returns the intersection of the ranges (possibly an empty range), + - `r1 &= r2` shrinks `r1` to the intersection of `r1` and `r2`, + - `r1 == r2` returns `true` if the two ranges are strictly equal (i.e. lower + and upper bounds are equal), + - `r1 != r2` returns `true` if the two ranges are not strictly equal, + - `r1 < r2` returns `true` if `r1.Hi() < r2.Lo()`, + - `r1 > r2` returns `true` if `r1.Lo() > r2.Hi()`, and + - `r1.Contains(r2)` returns `true` if the ranges overlap at all. + + * Given a range `r` and a `double` scalar `d`, + - `r * d` returns a new range `[d * r.Lo(), d * r.Hi()]`, + - `r *= d` scales `r.Lo()` and `r.Hi()` by `d`, and + - `r.Contains(d)` returns `true` if `d` is contained in the range. + +--- + + * To use ranges with different element types (e.g. `float`), use the type + `math::RangeType` or similar. + +--- + +Example: + +```c++ +mlpack::math::Range r1(5.0, 6.0); // [5, 6] +mlpack::math::Range r2(7.0, 8.0); // [7, 8] + +mlpack::math::Range r3 = r1 | r2; // [5, 8] +mlpack::math::Range r4 = r1 & r2; // empty range + +bool b1 = r1.Contains(r2); // false +bool b2 = r1.Contains(5.5); // true +bool b3 = r1.Contains(r3); // true +bool b4 = r3.Contains(r4); // false + +// Create a range of `float`s and a range of `int`s. +mlpack::math::RangeType r5(1.0f, 1.5f); // [1.0, 1.5] +mlpack::math::RangeType r6(3, 4); // [3, 4] +``` + +--- + +`math::Range` is used by: + + * [`RangeSearch`](range_search.md) + * [mlpack trees](#trees) + +--- + +## Distributions + +mlpack has support for a number of different distributions, each supporting the +same API. These can be used with, for instance, the [`HMM`](hmm.md) class. + + * [`DiscreteDistribution`](#discretedistribution): multidimensional categorical + distribution (generalized Bernoulli distribution) + * [`GaussianDistribution`](#gaussiandistribution): multidimensional Gaussian + distribution + +### `DiscreteDistribution` + +`DiscreteDistribution` represents a multidimensional categorical distribution +(or generalized Bernoulli distribution) where integer-valued vectors (e.g. +`[0, 3, 4]`) are associated with specific probabilities in each dimension. + +*Example:* a 3-dimensional `DiscreteDistribution` will have a specific +probability value associated with each integer value in each dimension. So, for +the vector `[0, 3, 4]`, `P(0)` in dimension 0 could be, e.g., `0.3`, `P(3)` in +dimension 1 could be, e.g., `0.4`, and `P(4)` in dimension 2 could be, e.g., +`0.6`. Then, `P([0, 3, 4])` would be `0.3 * 0.4 * 0.6 = 0.072`. + +--- + +#### Constructors + + * `d = DiscreteDistribution(numObservations)` + - Create a one-dimensional discrete distribution with `numObservations` + different observations in the one and only dimension. `numObservations` is + of type `size_t`. + + * `d = DiscreteDistribution(numObservationsVec)` + - Create a multidimensional discrete distribution with + `numObservationsVec.n_elem` dimensions and `numObservationsVec[i]` + different observations in dimension `i`. + - `numObservationsVec` is of type `arma::Col`. + + * `d = DiscreteDistribution(probabilities)` + - Create a multidimensional discrete distribution with the given + probabilities. + - `probabilities` should have type `std::vector`, and + `probabilities.size()` should be equal to the dimensionality of the + distribution. + - `probabilities[i]` is a vector such that `probabilities[i][j]` contains the + probability of `j` in dimension `i`. + +--- + +#### Access and modify properties of distribution + + * `d.Dimensionality()` returns a `size_t` indicating the number of dimensions + in the multidimensional discrete distribution. + + * `d.Probabilities(i)` returns an `arma::vec&` containing the probabilities of + each observation in dimension `i`. + - `d.Probabilities(i)[j]` is the probability of `j` in dimension `i`. + - This can be used to modify probabilities: `d.Probabilities(0)[1] = 0.7` + sets the probability of observing the value `1` in dimension `0` to `0.7`. + - *Note:* when setting probabilities manually, be sure that the sum of + probabilities in a dimension is 1! + +--- + +#### Compute probabilities of points + + * `d.Probability(observation)` returns the probability of the given + observation as a `double`. + - `observation` should be an `arma::vec` of size `d.Dimensionality()`. + - `observation[i]` should take integer values between `0` and + `d.Probabilities(i).n_elem - 1`. + + * `d.Probability(observations, probabilities)` computes the probabilities of + many observations. + - `observations` should be an `arma::mat` with number of rows equal to + `d.Dimensionality()`; `observations.n_cols` is the number of observations. + - `probabilities` will be set to size `observations.n_cols`. + - `probabilities[i]` will be set to `d.Probability(observations.col(i))`. + + * `d.LogProbability(observation)` returns the log-probability of the given + observation as a `double`. + + * `d.LogProbability(observations, probabilities)` computes the + log-probabilities of many observations. + +--- + +#### Sample from the distribution + + * `d.Random()` returns an `arma::vec` with a random sample from the + multidimensional discrete distribution. + +--- + +#### Fit the distribution to observations + + * `d.Train(observations)` + - Fit the distribution to the given observations. + - `observations` should be an `arma::mat` with number of rows equal to + `d.Dimensionality()`; `observations.n_cols` is the number of observations. + - `observations(j, i)` should be an integer value between `0` and the number + of observations for dimension `i`. + + * `d.Train(observations, observationProbabilities)` + - Fit the distribution to the given observations, as above, but also provide + probabilities that each observation is from this distribution. + - `observationProbabilities` should be an `arma::vec` of length + `observations.n_cols`. + - `observationProbabilities[i]` should be equal to the probability that + `observations.col(i)` is from `d`. + +--- + +*Example usage:* + +```c++ +// Create a single-dimension Bernoulli distribution: P([0]) = 0.3, P([1]) = 0.7. +mlpack::DiscreteDistribution bernoulli(2); +bernoulli.Probabilities(0)[0] = 0.3; +bernoulli.Probabilities(0)[1] = 0.7; + +const double p1 = bernoulli.Probability(arma::vec("0")); // p1 = 0.3. +const double p2 = bernoulli.Probability(arma::vec("1")); // p2 = 0.7. + +// Create a 3-dimensional discrete distribution by specifying the probabilities +// manually. +arma::vec probDim0 = arma::vec("0.1 0.3 0.5 0.1"); // 4 possible values. +arma::vec probDim1 = arma::vec("0.7 0.3"); // 2 possible values. +arma::vec probDim2 = arma::vec("0.4 0.4 0.2"); // 3 possible values. +std::vector probs { probDim0, probDim1, probDim2 }; +mlpack::DiscreteDistribution d(probs); + +arma::vec obs("2 0 1"); +const double p3 = d.Probability(obs); // p3 = 0.5 * 0.7 * 0.4 = 0.14. + +// Estimate a 10-dimensional discrete distribution. +// Each dimension takes values between 0 and 9. +arma::mat observations = arma::randi(10, 1000, + arma::distr_param(0, 9)); + +// Create a distribution with 10 observations in each of the 10 dimensions. +mlpack::DiscreteDistribution d2( + arma::Col("10 10 10 10 10 10 10 10 10 10")); +d2.Train(observations); + +// Compute the probabilities of each point. +arma::vec probabilities; +d2.Probability(observations, probabilities); +std::cout << "Average probability: " << arma::mean(probabilities) << "." + << std::endl; +``` + +--- + +### `GaussianDistribution` + +`GaussianDistribution` is a standard multivariate Gaussian distribution with +parameterized mean and covariance. + +--- + +#### Constructors + + * `g = GaussianDistribution(dimensionality)` + - Create the distribution with the given dimensionality. + - The distribution will have a zero mean and unit diagonal covariance matrix. + + * `g = GaussianDistribution(mean, covariance)` + - Create the distribution with the given mean and covariance. + - `mean` is of type `arma::vec` and should have length equal to the + dimensionality of the distribution. + - `covariance` is of type `arma::mat`, and should be symmetric and square, + with rows and columns equal to the dimensionality of the distribution. + +--- + +#### Access and modify properties of distribution + + * `g.Dimensionality()` returns the dimensionality of the distribution as a + `size_t`. + + * `g.Mean()` returns an `arma::vec&` holding the mean of the distribution. + This can be modified. + + * `g.Covariance()` returns a `const arma::mat&` holding the covariance of the + distribution. To set a new covariance, use `g.Covariance(newCov)` or + `g.Covariance(std::move(newCov))`. + + * `g.InvCov()` returns a `const arma::mat&` holding the precomputed inverse of + the covariance. + + * `g.LogDetCov()` returns a `double` holding the log-determinant of the + covariance. + +--- + +#### Compute probabilities of points + + * `g.Probability(observation)` returns the probability of the given + observation as a `double`. + - `observation` should be an `arma::vec` of size `d.Dimensionality()`. + + * `g.Probability(observations, probabilities)` computes the probabilities of + many observations. + - `observations` should be an `arma::mat` with number of rows equal to + `d.Dimensionality()`; `observations.n_cols` is the number of observations. + - `probabilities` will be set to size `observations.n_cols`. + - `probabilities[i]` will be set to `g.Probability(observations.col(i))`. + + * `g.LogProbability(observation)` returns the log-probability of the given + observation as a `double`. + + * `g.LogProbability(observations, probabilities)` computes the + log-probabilities of many observations. + +--- + +#### Sample from the distribution + + * `g.Random()` returns an `arma::vec` with a random sample from the + multidimensional discrete distribution. + +--- + +#### Fit the distribution to observations + + * `g.Train(observations)` + - Fit the distribution to the given observations. + - `observations` should be an `arma::mat` with number of rows equal to + `d.Dimensionality()`; `observations.n_cols` is the number of observations. + + * `g.Train(observations, observationProbabilities)` + - Fit the distribution to the given observations, as above, but also provide + probabilities that each observation is from this distribution. + - `observationProbabilities` should be an `arma::vec` of length + `observations.n_cols`. + - `observationProbabilities[i]` should be equal to the probability that + `observations.col(i)` is from `d`. + +--- + +*Example usage:* + +```c++ +// Create a Gaussian distribution in 3 dimensions with zero mean and unit +// covariance. +mlpack::GaussianDistribution g(3); + +// Compute the probability of the point [0, 0.5, 0.25]. +const double p = g.Probability(arma::vec("0 0.5 0.25")); + +// Modify the mean in dimension 0. +g.Mean()[0] = 0.5; + +// Set a random covariance. +arma::mat newCov(3, 3, arma::fill::randu); +newCov *= newCov.t(); // Ensure covariance is positive semidefinite. +g.Covariance(std::move(newCov)); // Set new covariance. + +// Compute the probability of the same point [0, 0.5, 0.25]. +const double p2 = g.Probability(arma::vec("0 0.5 0.25")); + +// Create a Gaussian distribution that is estimated from random samples in 50 +// dimensions. +arma::mat samples(50, 10000, arma::fill::randn); // Normally distributed. + +mlpack::GaussianDistribution g2(50); +g2.Train(samples); + +// Compute the probability of all of the samples. +arma::vec probabilities; +g2.Probability(samples, probabilities); + +std::cout << "Average probability is: " << arma::mean(probabilities) << "." + << std::endl; +``` + +## Metrics + +mlpack includes a number of distance metrics for its distance-based techniques. +These all implement the [same API](../developer/metrics.md), providing one +`Evaluate()` method, and can be used with a variety of different techniques, +including: + + + + * [`NeighborSearch`](neighbor_search.md) + * [`RangeSearch`](range_search.md) + * [`LMNN`](lmnn.md) + * [`EMST`](emst.md) + * [`NCA`](nca.md) + * [`RANN`](rann.md) + * [`KMeans`](kmeans.md) + +Supported metrics: + + * [`LMetric`](#lmetric): generalized L-metric/Lp-metric, including + Manhattan/Euclidean/Chebyshev distances + * [Implement a custom metric](../developer/metrics.md) + +### `LMetric` + +The `LMetric` template class implements a [generalized +L-metric](https://en.wikipedia.org/wiki/Lp_space#Definition) +(L1-metric, L2-metric, etc.). The class has two template parameters: + +```c++ +LMetric +``` + + * `Power` is an `int` representing the type of the metric; e.g., `2` would + represent the L2-metric (Euclidean distance). + - `Power` must be `1` or greater. + - If `Power` is `INT_MAX`, the metric is the L-infinity distance (Chebyshev + distance). + + * `TakeRoot` is a `bool` (default `true`) indicating whether the root of the + distance should be taken. + - If set to `false`, the metric will no longer satisfy the triangle + inequality. + +--- + +Several convenient typedefs are available: + + * `ManhattanDistance` (defined as `LMetric<1>`) + * `EuclideanDistance` (defined as `LMetric<2>`) + * `SquaredEuclideanDistance` (defined as `LMetric<2, false>`) + * `ChebyshevDistance` (defined as `LMetric`) + +--- + +The static `Evaluate()` method can be used to compute the distance between two +vectors. + +*Note:* The vectors given to `Evaluate()` can have any type so long as the type +implements the Armadillo API (e.g. `arma::fvec`, `arma::sp_fvec`, etc.). + +--- + +*Example usage:* + +```c++ +// Create two vectors: [0, 1.0, 5.0] and [1.0, 3.0, 5.0]. +arma::vec a("0.0 1.0 5.0"); +arma::vec b("1.0 3.0 5.0"); + +const double d1 = mlpack::ManhattanDistance::Evaluate(a, b); // d1 = 3.0 +const double d2 = mlpack::EuclideanDistance::Evaluate(a, b); // d2 = 2.24 +const double d3 = mlpack::SquaredEuclideanDistance::Evaluate(a, b); // d3 = 5.0 +const double d4 = mlpack::ChebyshevDistance::Evaluate(a, b); // d4 = 2.0 +const double d5 = mlpack::LMetric<4>::Evaluate(a, b); // d5 = 2.03 +const double d6 = mlpack::LMetric<3, false>::Evaluate(a, b); // d6 = 9.0 + +std::cout << "Manhattan distance: " << d1 << "." << std::endl; +std::cout << "Euclidean distance: " << d2 << "." << std::endl; +std::cout << "Squared Euclidean distance: " << d3 << "." << std::endl; +std::cout << "Chebyshev distance: " << d4 << "." << std::endl; +std::cout << "L4-distance: " << d5 << "." << std::endl; +std::cout << "Cubed L3-distance: " << d6 << "." << std::endl; + +// Compute the distance between two random 10-dimensional vectors in a matrix. +arma::mat m(10, 100, arma::fill::randu); + +const double d7 = mlpack::EuclideanDistance::Evaluate(m.col(0), m.col(7)); + +std::cout << std::endl; +std::cout << "Distance between two random vectors: " << d7 << "." << std::endl; +std::cout << std::endl; + +// Compute the distance between two 32-bit precision `float` vectors. +arma::fvec fa("0.0 1.0 5.0"); +arma::fvec fb("1.0 3.0 5.0"); + +const double d8 = mlpack::EuclideanDistance::Evaluate(fa, fb); // d8 = 2.236 + +std::cout << "Euclidean distance (fvec): " << d8 << "." << std::endl; +``` + +## Kernels + +mlpack includes a number of Mercer kernels for its kernel-based techniques. +These all implement the [same API](../developer/kernels.md), providing one +`Evaluate()` method, and can be used with a variety of different techniques, +including: + + + + * [`KDE`](kde.md) + * [`MeanShift`](mean_shift.md) + * [`KernelPCA`](kernel_pca.md) + * [`FastMKS`](fastmks.md) + * [`NystroemMethod`](nystroem_method.md) + +Supported kernels: + + * [`GaussianKernel`](#gaussiankernel): standard Gaussian/radial basis + function/RBF kernel + * [Implement a custom kernel](../developer/kernels.md) + +### `GaussianKernel` + +The `GaussianKernel` class implements the standard [Gaussian +kernel](https://en.wikipedia.org/wiki/Radial_basis_function_kernel) (also called +the _radial basis function kernel_ or _RBF kernel_). + +The Gaussian kernel is defined as: +`k(x1, x2) = exp(-|| x1 - x2 ||^2 / (2 * bw^2))` +where `bw` is the bandwidth parameter of the kernel. + +--- + +#### Constructors and properties + + * `g = GaussianKernel(bw=1.0)` + - Create a `GaussianKernel` with the given bandwidth `bw`. + + * `g.Bandwidth()` returns the bandwidth of the kernel as a `double`. + - To set the bandwidth, use `g.Bandwidth(newBandwidth)`. + +--- + +#### Kernel evaluation + + * `g.Evaluate(x1, x2)` + - Compute the kernel value between two vectors `x1` and `x2`. + - `x1` and `x2` should be vector types that implement the Armadillo API + (e.g., `arma::vec`). + + * `g.Evaluate(distance)` + - Compute the kernel value between two vectors, given that the distance + between those two vectors (`distance`) is already known. + - `distance` should have type `double`. + +--- + +#### Other utilities + + * `g.Gradient(distance)` + - Compute the (one-dimensional) gradient of the kernel function with respect + to the distance between two points, evaluated at `distance`. + + * `g.Normalizer(dimensionality)` + - Return the [normalizing + constant](https://en.wikipedia.org/wiki/Radial_basis_function_kernel) of + the Gaussian kernel for points in the given dimensionality as a `double`. + +--- + +*Example usage:* + +```c++ +// Create a Gaussian kernel with default bandwidth. +mlpack::GaussianKernel g; + +// Create a Gaussian kernel with bandwidth 5.0. +mlpack::GaussianKernel g2(5.0); + +// Evaluate the kernel value between two 3-dimensional points. +arma::vec x1("0.5 1.0 1.5"); +arma::vec x2("1.5 1.0 0.5"); +const double k1 = g.Evaluate(x1, x2); +const double k2 = g2.Evaluate(x1, x2); +std::cout << "Kernel values: " << k1 << " (bw=1.0), " << k2 << " (bw=5.0)." + << std::endl; + +// Evaluate the kernel value when the distance between two points is already +// computed. +const double distance = 1.5; +const double k3 = g.Evaluate(distance); + +// Change the bandwidth of the kernel to 2.5. +g.Bandwidth(2.5); +const double k4 = g.Evaluate(x1, x2); +std::cout << "Kernel value with bw=2.5: " << k4 << "." << std::endl; + +// Evaluate the kernel value between x1 and all points in a random matrix. +arma::mat r(3, 100, arma::fill::randu); +arma::vec kernelValues(100); +for (size_t i = 0; i < r.n_cols; ++i) + kernelValues[i] = g.Evaluate(x1, r.col(i)); +std::cout << "Average kernel value for random points: " + << arma::mean(kernelValues) << "." << std::endl; + +// Compute the kernel value between two 32-bit floating-point vectors. +arma::fvec fx1("0.5 1.0 1.5"); +arma::fvec fx2("1.5 1.0 0.5"); +const double k5 = g.Evaluate(fx1, fx2); +const double k6 = g2.Evaluate(fx1, fx2); +``` diff --git a/doc/user/load_save.md b/doc/user/load_save.md new file mode 100644 index 0000000000..c74083f982 --- /dev/null +++ b/doc/user/load_save.md @@ -0,0 +1,758 @@ +# Loading and saving mlpack objects + +mlpack provides the `data::Load()` and `data::Save()` functions to load and save +[Armadillo matrices](matrices.md) (e.g. numeric and categorical datasets) and +any mlpack object via the [cereal](https://uscilab.github.io/cereal/) +serialization toolkit. A number of other utilities related to loading and +saving data and objects are also available. + + * [Numeric data](#numeric-data) + * [Mixed categorical data](#mixed-categorical-data) + - [`data::DatasetInfo`](#datadatasetinfo) + - [Loading categorical data](#loading-categorical-data) + * [Image data](#image-data) + - [`data::ImageInfo`](#dataimageinfo) + - [Loading images](#loading-images) + * [mlpack objects](#mlpack-objects): load or save any mlpack object + * [Normalizing labels](#normalizing-labels): convert labels to ranges required + by mlpack classifiers + * [Formats](#formats): supported formats for each load/save variant + +## Numeric data + +Numeric data or general numeric matrices can be loaded or saved with the +following functions. + + - `data::Load(filename, matrix, fatal=false, transpose=true, format=FileType::AutoDetect)` + - `data::Save(filename, matrix, fatal=false, transpose=true, format=FileType::AutoDetect)` + * `filename` is a `std::string` with a path to the file to be loaded. + + * By default the format is auto-detected based on the file extension, but can + be explicitly specified with `format`; see [Formats](#formats). + + * `matrix` is an `arma::mat&`, `arma::Mat&`, or similar (e.g., a + reference to an Armadillo object that data will be loaded into or saved + from). + + * If `fatal` is `true`, a `std::runtime_error` will be thrown on failure. + + * If `transpose` is `true`, then for plaintext formats (CSV/TSV/ASCII), the + matrix will be transposed on save. (Keep this `true` if you want a + column-major matrix to be saved with points as rows and dimensions as + columns; that is generally what is desired.) + + * A `bool` is returned indicating whether the operation was successful. + +--- + +Example usage: + +```c++ +// See https://datasets.mlpack.org/satellite.train.csv. +arma::mat dataset; +mlpack::data::Load("satellite.train.csv", dataset, true); + +// See https://datasets.mlpack.org/satellite.train.labels.csv. +arma::Row labels; +mlpack::data::Load("satellite.train.labels.csv", labels, true); + +// Print information about the data. +std::cout << "The data in 'satellite.train.csv' has: " << std::endl; +std::cout << " - " << dataset.n_cols << " points." << std::endl; +std::cout << " - " << dataset.n_rows << " dimensions." << std::endl; + +std::cout << "The labels in 'satellite.train.labels.csv' have: " << std::endl; +std::cout << " - " << labels.n_elem << " labels." << std::endl; +std::cout << " - A maximum label of " << labels.max() << "." << std::endl; +std::cout << " - A minimum label of " << labels.min() << "." << std::endl; + +// Modify and save the data. Add 2 to the data and drop the last column. +dataset += 2; +dataset.shed_col(dataset.n_cols - 1); +labels.shed_col(labels.n_cols - 1); + +mlpack::data::Save("satellite.train.mod.csv", dataset); +mlpack::data::Save("satellite.train.labels.mod.csv", labels); +``` + +--- + +## Mixed categorical data + +Some mlpack techniques support mixed categorical data, e.g., data where some +dimensions take only categorical values (e.g. `0`, `1`, `2`, etc.). When using +mlpack, string data and other non-numerical data must be mapped to categorical +values and represented as part of an `arma::mat`. Category information is +stored in an auxiliary `data::DatasetInfo` object. + +### `data::DatasetInfo` + + + +mlpack represents categorical data via the use of the auxiliary +`data::DatasetInfo` object, which stores information about which dimensions are +numeric or categorical and allows conversion from the original category values +to the numeric values used to represent those categories. + +--- + +#### Constructors + + - `info = data::DatasetInfo()` + * Create an empty `data::DatasetInfo` object. + * Use this constructor if you intend to populate the `data::DatasetInfo` via + a `data::Load()` call. + + - `info = data::DatasetInfo(dimensionality)` + * Create a `data::DatasetInfo` object with the given dimensionality + * All dimensions are assumed to be numeric (not categorical). + +--- + +#### Accessing and setting properties + + - `info.Type(d)` + * Get the type (categorical or numeric) of dimension `d`. + * Returns a `data::Datatype`, either `data::Datatype::numeric` or + `data::Datatype::categorical`. + * Calling `info.Type(d) = t` will set a dimension to type `t`, but this + should only be done before `info` is used with `data::Load()` or + `data::Save()`. + + - `info.NumMappings(d)` + * Get the number of categories in dimension `d` as a `size_t`. + * Returns `0` if dimension `d` is numeric. + + - `info.Dimensionality()` + * Return the dimensionality of the object as a `size_t`. + +--- + +#### Map to and from numeric values + + - `info.MapString(value, d)` + * Given `value` (a `std::string`), return the `double` representing the + categorical mapping (an integer value) of `value` in dimension `d`. + * If a mapping for `value` does not exist in dimension `d`, a new mapping is + created, and `info.NumMappings(d)` is increased by one. + * If dimension `d` is numeric and `value` cannot be parsed as a numeric + value, then dimension `d` is changed to categorical and a new mapping is + returned. + + - `info.UnmapString(mappedValue, d)` + * Given `mappedValue` (a `size_t`), return the `std::string` containing the + original category that mapped to the value `mappedValue` in dimension `d`. + * If dimension `d` is not categorical, a `std::invalid_argument` is thrown. + +--- + +### Loading categorical data + +With a `data::DatasetInfo` object, categorical data can be loaded: + + - `data::Load(filename, matrix, info, fatal=false, transpose=true)` + * `filename` is a `std::string` with a path to the file to be loaded. + + * The format is auto-detected based on the extension of the filename and the + contents of the file: + - `.csv`, `.tsv`, or `.txt` for CSV/TSV (tab-separated)/ASCII + (space-separated) + - `.arff` for [ARFF](https://www.cs.waikato.ac.nz/~ml/weka/arff.html) + + * `matrix` is an `arma::mat&`, `arma::Mat&`, or similar (e.g., a + reference to an Armadillo object that data will be loaded into or saved + from). + + * `info` is a `data::DatasetInfo&` object. This will be populated with the + category information of the file when loading, and used to unmap values + when saving. + + * If `fatal` is `true`, a `std::runtime_error` will be thrown on failure. + + * If `transpose` is `true`, then for plaintext formats (CSV/TSV/ASCII), the + matrix will be transposed on save. (Keep this `true` if you want a + column-major matrix to be saved with points as rows and dimensions as + columns; that is generally what is desired.) + + * A `bool` is returned indicating whether the operation was successful. + +Saving should be performed with the [numeric](#numeric-data) `data::Load()` +variant. + +--- + +Example usage to load and manipulate an ARFF file. + +```c++ +// Load a categorical dataset. +arma::mat dataset; +mlpack::data::DatasetInfo info; +// See https://datasets.mlpack.org/covertype.train.arff. +mlpack::data::Load("covertype.train.arff", dataset, info, true); + +arma::Row labels; +// See https://datasets.mlpack.org/covertype.train.labels.csv. +mlpack::data::Load("covertype.train.labels.csv", labels, true); + +// Print information about the data. +std::cout << "The data in 'covertype.train.arff' has: " << std::endl; +std::cout << " - " << dataset.n_cols << " points." << std::endl; +std::cout << " - " << info.Dimensionality() << " dimensions." << std::endl; + +// Print information about each dimension. +for (size_t d = 0; d < info.Dimensionality(); ++d) +{ + if (info.Type(d) == mlpack::data::Datatype::categorical) + { + std::cout << " - Dimension " << d << " is categorical with " + << info.NumMappings(d) << " categories." << std::endl; + } + else + { + std::cout << " - Dimension " << d << " is numeric." << std::endl; + } +} + +// Modify the 5th point. Increment any numeric values, and set any categorical +// values to the string "hooray!". +for (size_t d = 0; d < info.Dimensionality(); ++d) +{ + if (info.Type(d) == mlpack::data::Datatype::categorical) + { + // This will create a new mapping if the string "hooray!" does not already + // exist as a category for dimension d.. + dataset(d, 4) = info.MapString("hooray!", d); + } + else + { + dataset(d, 4) += 1.0; + } +} +``` + +--- + +Example usage to manually create a `data::DatasetInfo` object. + +```c++ +// This will manually create the following data matrix (shown as it would appear +// in a CSV): +// +// 1, TRUE, "good", 7.0, 4 +// 2, FALSE, "good", 5.6, 3 +// 3, FALSE, "bad", 6.1, 4 +// 4, TRUE, "bad", 6.1, 1 +// 5, TRUE, "unknown", 6.3, 0 +// 6, FALSE, "unknown", 5.1, 2 +// +// Although the last dimension is numeric, we will take it as a categorical +// dimension. + +arma::mat dataset(5, 6); // 6 data points in 5 dimensions. +mlpack::data::DatasetInfo info(5); + +// Set types of dimensions. By default they are numeric so we only set +// categorical dimensions. +info.Type(1) = mlpack::data::Datatype::categorical; +info.Type(2) = mlpack::data::Datatype::categorical; +info.Type(4) = mlpack::data::Datatype::categorical; + +// The first dimension is numeric. +dataset(0, 0) = 1; +dataset(0, 1) = 2; +dataset(0, 2) = 3; +dataset(0, 3) = 4; +dataset(0, 4) = 5; +dataset(0, 5) = 6; + +// The second dimension is categorical. +dataset(1, 0) = info.MapString("TRUE", 1); +dataset(1, 1) = info.MapString("FALSE", 1); +dataset(1, 2) = info.MapString("FALSE", 1); +dataset(1, 3) = info.MapString("TRUE", 1); +dataset(1, 4) = info.MapString("TRUE", 1); +dataset(1, 5) = info.MapString("FALSE", 1); + +// The third dimension is categorical. +dataset(2, 0) = info.MapString("good", 2); +dataset(2, 1) = info.MapString("good", 2); +dataset(2, 2) = info.MapString("bad", 2); +dataset(2, 3) = info.MapString("bad", 2); +dataset(2, 4) = info.MapString("unknown", 2); +dataset(2, 5) = info.MapString("unknown", 2); + +// The fourth dimension is numeric. +dataset(3, 0) = 7.0; +dataset(3, 1) = 5.6; +dataset(3, 2) = 6.1; +dataset(3, 3) = 6.1; +dataset(3, 4) = 6.3; +dataset(3, 5) = 5.1; + +// The fifth dimension is categorical. Note that `info` will choose to assign +// category values in the order they are seen, even if the category can be +// parsed as a number. So, here, the value '4' will be assigned category '0', +// since it is seen first. +dataset(4, 0) = info.MapString("4", 4); +dataset(4, 1) = info.MapString("3", 4); +dataset(4, 2) = info.MapString("4", 4); +dataset(4, 3) = info.MapString("1", 4); +dataset(4, 4) = info.MapString("0", 4); +dataset(4, 5) = info.MapString("2", 4); + +// Print the dataset with mapped categories. +dataset.print("Dataset with mapped categories"); + +// Print the mappings for the third dimension. +std::cout << "Mappings for dimension 3: " << std::endl; +for (size_t i = 0; i < info.NumMappings(2); ++i) +{ + std::cout << " - \"" << info.UnmapString(i, 2) << "\" maps to " << i << "." + << std::endl; +} + +// Now `dataset` is ready for use with an mlpack algorithm that supports +// categorical data. +``` + +--- + +## Image data + +If the STB image library is available on the system (`stb_image.h` and +`stb_image_write.h` must be available on the compiler's include search path), +then mlpack will define the `MLPACK_HAS_STB` macro, and support for loading +individual images or sets of images will be available. + +Supported formats for loading are `jpg`, `png`, `tga`, `bmp`, `psd`, `gif`, `hdr`, `pic`, and `pnm`. + +Supported formats for saving are `jpg`, `png`, `tga`, `bmp`, and `hdr`. + +When loading images, each image is represented as a flattened single column +vector in a data matrix; each row of the resulting vector will correspond to a +single pixel value in a single channel. An auxiliary `data::ImageInfo` class is +used to store information about the images. + +### `data::ImageInfo` + +The `data::ImageInfo` class contains the metadata of the images. + +--- + +#### Constructors + + - `info = data::ImageInfo()` + * Create a `data::ImageInfo` object with no data. + * Use this constructor if you intend to populate the `data::ImageInfo` via a + `data::Load()` call. + + - `info = data::ImageInfo(width, height, channels)` + * Create a `data::ImageInfo` object with the given image specifications. + * `width` and `height` are specified as pixels. + +--- + +#### Accessing and modifying image metadata + + - `info.Quality() = q` will set the compression quality (e.g. for saving JPEGs) + to `q`. + * `q` should take values between `0` and `100`. + * The quality value is ignored unless calling `data::Save()` with `info`. + + - Calling `info.Channels() = 1` before loading will cause images to be loaded + in grayscale. + + - Metadata stored in the `data::ImageInfo` can be accessed with the following + members: + * `info.Width()` returns the image width in pixels. + * `info.Height()` returns the image height in pixels. + * `info.Channels()` returns the number of color channels in the image. + * `info.Quality()` returns the compression quality that will be used to save + images (between 0 and 100). + +--- + +### Loading images + +With a `data::ImageInfo` object, image data can be loaded or saved, handling +either one or multiple images at a time: + + + + - `data::Load(filename, matrix, info, fatal=false)` + * Load a ***single image*** from `filename` into `matrix`. + - Format is chosen by extension (e.g. `image.png` will load as PNG). + + * `matrix` will have one column representing the image as a flattened vector. + + * `info` will be populated with information from the image in `filename`. + + * If `fatal` is `true`, a `std::runtime_error` will be thrown upon load + failure. + + * Returns a `bool` indicating the success of the operation. + +--- + + - `data::Load(files, matrix, info, fatal=false)` + * Load ***multiple images*** from `files` into `matrix`. + - `files` is of type `std::vector` and should contain the list + of images to be loaded. + - `matrix` will have `files.size()` columns, each representing the + corresponding image as a flattened vector. + + * `info` will be populated with information from the images in `files`. + + * If `fatal` is `true`, a `std::runtime_error` will be thrown if any files + fail to load. + + * Returns a `bool` indicating the success of the operation. + +--- + + - `data::Save(filename, matrix, info, fatal=false)` + * Save a ***single image*** from `matrix` into the file `filename`. + - Format is chosen by extension (e.g. `image.png` will save as PNG). + + * `matrix` is expected to have only one column representing the image as a + flattened vector. + + * If `fatal` is `true`, a `std::runtime_error` will be thrown in the event of + save failure. + + * Returns a `bool` indicating the success of the operation. + +--- + + - `data::Save(files, matrix, info, fatal=false)` + * Save ***multiple images*** from `matrix` into `files`. + - `files` is of type `std::vector` and should contain the list + of files to save to. + - The format of each file is chosen by extension (e.g. `image.png` will + save as PNG); it is allowed for filenames in `files` to have different + extensions. + + * `matrix` is expected to have `files.size()` columns representing images as + flattened vectors. + + * If `fatal` is `true`, a `std::runtime_error` will be thrown if any images + fail to save. + + * Returns a `bool` indicating the success of the operation. + +--- + +Images are flattened along rows, with channel values interleaved, starting from +the top left. Thus, the value of the pixel at position `(x, y)` in channel `c` +will be contained in element/row `y * (width * channels) + x * (channels) + c` +of the flattened vector. + +Pixels take values between 0 and 255. + +--- + +Example of loading and saving a single image: + +```c++ +// See https://www.mlpack.org/static/img/numfocus-logo.png. +mlpack::data::ImageInfo info; +arma::mat matrix; +mlpack::data::Load("numfocus-logo.png", matrix, info, true); + +// `matrix` should now contain one column. + +// Print information about the image. +std::cout << "Information about the image in 'numfocus-logo.png': " + << std::endl; +std::cout << " - " << info.Width() << " pixels in width." << std::endl; +std::cout << " - " << info.Height() << " pixels in height." << std::endl; +std::cout << " - " << info.Channels() << " color channels." << std::endl; + +std::cout << "Value at pixel (x=3, y=4) in the first channel: "; +const size_t index = (4 * info.Width() * info.Channels()) + + (3 * info.Channels()); +std::cout << matrix[index] << "." << std::endl; + +// Increment each pixel value, but make sure they are still within the bounds. +matrix += 1; +matrix = arma::clamp(matrix, 0, 255); + +mlpack::data::Save("numfocus-logo-mod.png", matrix, info); +``` + +--- + +Example of loading and saving multiple images: + +```c++ +// Load some favicons from websites associated with mlpack. +std::vector images; +// See the following files: +// - https://datasets.mlpack.org/images/mlpack-favicon.png +// - https://datasets.mlpack.org/images/ensmallen-favicon.png +// - https://datasets.mlpack.org/images/armadillo-favicon.png +// - https://datasets.mlpack.org/images/bandicoot-favicon.png +images.push_back("mlpack-favicon.png"); +images.push_back("ensmallen-favicon.png"); +images.push_back("armadillo-favicon.png"); +images.push_back("bandicoot-favicon.png"); + +mlpack::data::ImageInfo info; +info.Channels(1); // Force loading in grayscale. + +arma::mat matrix; +mlpack::data::Load(images, matrix, info, true); + +// Print information about what we loaded. +std::cout << "Loaded " << matrix.n_cols << " images. Images are of size " + << info.Width() << " x " << info.Height() << " with " << info.Channels() + << " color channel." << std::endl; + +// Invert images. +matrix = (255.0 - matrix); + +// Save as compressed JPEGs with low quality. +info.Quality() = 75; +std::vector outImages; +outImages.push_back("mlpack-favicon-inv.jpeg"); +outImages.push_back("ensmallen-favicon-inv.jpeg"); +outImages.push_back("armadillo-favicon-inv.jpeg"); +outImages.push_back("bandicoot-favicon-inv.jpeg"); + +mlpack::data::Save(outImages, matrix, info); +``` + +## mlpack objects + +All mlpack objects can be saved with `data::Save()` and loaded with +`data::Load()`. Serialization is performed using the +[cereal](https://uscilab.github.io/cereal/) serialization toolkit. +Each object must be given a logical name. + + - `data::Load(filename, name, object, fatal=false, format=data::format::autodetect)` + - `data::Save(filename, name, object, fatal=false, format=data::format::autodetect)` + * Load/save `object` to/from `filename` with the logical name `name`. + + * If `fatal` is `true`, a `std::runtime_error` will be thrown in the event of + load or save failure. + + * The format is autodetected based on extension (`.bin`, `.json`, or `.xml`), + but can be manually specified: + - `data::format::binary`: binary blob (smallest and fastest). No checks; + assumes all data is correct. + - `data::format::json`: JSON. + - `data::format::xml`: XML (largest and slowest). + + * For JSON and XML types, when loading, `name` must match the name used to + save the object. + + * Returns a `bool` indicating the success of the operation. + +***Note:*** when loading an object that was saved as a binary blob, the C++ type +of the object must be ***exactly the same*** (including template parameters) as +the type used to save the object. If not, undefined behavior will occur---most +likely a crash. + +--- + +Simple example: create a `math::Range` object, then save and load it. + +```c++ +mlpack::math::Range r(3.0, 6.0); + +// Save the Range to 'range.bin', using the name "range". +mlpack::data::Save("range.bin", "range", r, true); + +// Load the range into a new object. +mlpack::math::Range r2; +mlpack::data::Load("range.bin", "range", r2, true); + +std::cout << "Loaded range: [" << r2.Lo() << ", " << r2.Hi() << "]." + << std::endl; + +// Modify and save the range as JSON. +r2.Lo() = 4.0; +mlpack::data::Save("range.json", "range", r2, true); + +// Now 'range.json' will contain the following: +// +// { +// "range": { +// "cereal_class_version": 0, +// "hi": 6.0, +// "lo": 4.0 +// } +// } +``` + +--- + +## Normalizing labels + +mlpack classifiers and other algorithms require labels to be in the range `0` to +`numClasses - 1`. A vector of labels with arbitrary (`size_t`) values can be +normalized to the required range with the `NormalizeLabels()` function. + +--- + + * `data::NormalizeLabels(labelsIn, labelsOut, mappings)` + - Map vector `labelsIn` into the range `0` to `numClasses - 1`, storing as + `labelsOut` (of type `arma::Row`). + * `numClasses` is automatically detected using the number of unique values + in `labelsIn`. + + - The column vector `mappings` will be filled with the reverse mappings to + convert back to the old labels; this can be used by `RevertLabels()`. + + - `mappings[i]` contains the original class label for the mapped label `i`. + +--- + + * `data::RevertLabels(labelsIn, mappings, labelsOut)` + - Unmap normalized labels `labelsIn` using `mappings` into `labelsOut`. + + - Performs the reverse operation of `NormalizeLabels()`; `mappings` should + be the same vector output by `NormalizeLabels()`. + +--- + +Simple example: convert labels into `0`, `1`, `2`, learn a model, then convert +predictions back to the original label values. + +```c++ +// Create a random dataset with 5 points in 10 dimensions. +arma::mat dataset(10, 5, arma::fill::randu); + +// Manually assemble labels vector: [3, 7, 3, 3, 5] +arma::Row labels = { 3, 7, 3, 3, 5 }; + +// Note that these labels are not in the range `0` to `2`, and thus cannot be +// used directly by mlpack classifiers! +// We will map them to that range using NormalizeLabels(). +arma::Row mappedLabels; +arma::Col mappings; +mlpack::data::NormalizeLabels(labels, mappedLabels, mappings); +const size_t numClasses = mappedLabels.max() + 1; + +// Print the mapped values: +// [3, 7, 3, 3, 5] maps to [0, 1, 0, 0, 2]. +// The `mappings` vector will be [3, 7, 5]. +std::cout << "Original labels: " << labels; +std::cout << "Mapped labels: " << mappedLabels; +std::cout << "Mappings: " << mappings; + +// Learn a model with the mapped labels. +mlpack::DecisionTree d(dataset, mappedLabels, numClasses, 1 /* leaf size */); + +// Make predictions on the training dataset. +arma::Row mappedPredictions; +d.Classify(dataset, mappedPredictions); + +// The predictions use mapped labels (0, 1, 2), which we will need to map back +// to the original labels using RevertLabels(). +arma::Row predictions; +mlpack::data::RevertLabels(mappedPredictions, mappings, predictions); + +// Print the predictions before and after unmapping. +// The mapped predictions will take values 0, 1, or 2; the predictions will take +// values 3, 7, or 5 (like the original data). +std::cout << "Mapped predictions: " << mappedPredictions; +std::cout << "Predictions: " << predictions; +``` + +## Formats + +mlpack's `data::Load()` and `data::Save()` functions support a variety of +different formats in different contexts. + +--- + +#### [Numeric data](#numeric-data) + +By default, load/save format is ***autodetected***, but can be manually +specified with the `format` parameter using one of the options below: + + - `FileType::AutoDetect` (default): auto-detects the format as one of the + formats below using the extension of the filename and inspecting the file + contents. + + - `FileType::CSVASCII` (autodetect extensions `.csv`, `.tsv`): CSV format + with no header. + + - `FileType::RawASCII` (autodetect extensions `.csv`, `.txt`): + space-separated values or tab-separated values (TSV) with no header. + + - `FileType::ArmaASCII` (autodetect extension `.txt`): space-separated + values as saved by Armadillo with the + [`arma_ascii`](https://arma.sourceforge.net/docs.html#save_load_mat) + format. + + - `FileType::CoordASCII` (not autodetected, must be manually specified): + coordinate list format for sparse data (see + [`coord_ascii`](https://arma.sourceforge.net/docs.html#save_load_mat)). + + - `FileType::ArmaBinary` (autodetect extension `.bin`): Armadillo's + efficient binary matrix format + ([`arma_binary`](https://arma.sourceforge.net/docs.html#save_load_mat)). + + - `FileType::HDF5Binary` (autodetect extensions `.h5`, `.hdf5`, `.hdf`, + `.he5`): [HDF5](https://en.wikipedia.org/wiki/Hierarchical_Data_Format) + binary format; only available if Armadillo is configured with + [HDF5 support](https://arma.sourceforge.net/docs.html#config_hpp). + + - `FileType::RawBinary` (autodetect extension `.bin`): packed binary data + with no header and no size information; data will be loaded as a single + column vector _(not recommended)_. + + - `FileType::PGMBinary` (autodetect extension `.pgm`): PGM image format + +***Notes:*** + + - ASCII formats (`CSVASCII`, `RawASCII`, `ArmaASCII`) are human-readable but + large; to reduce dataset size, consider a binary format such as + `ArmaBinary` or `HDF5Binary`. + - Sparse data (`arma::sp_mat`, `arma::sp_fmat`, etc.) should be saved in a + binary format (`ArmaBinary` or `HDF5Binary`) or as a coordinate list + (`CoordASCII`). + +--- + +#### [Mixed categorical data](#mixed-categorical-data) + +The format of mixed categorical data is detected automatically based on the +file extension and inspecting the file contents: + + - `.csv`, `.txt`, or `.tsv` indicates CSV/TSV/ASCII format + - `.arff` indicates [ARFF](https://www.cs.waikato.ac.nz/~ml/weka/arff.html) + +--- + +#### [Image data](#image-data) + +The format of images are detected automatically based on the file extension. + + - The following formats are supported for loading: `.jpg`, `.jpeg`, `.png`, + `.tga`, `.bmp`, `.psd`, `.gif`, `.hdr`, `.pic`, `.pnm` + + - The following formats are supported for saving: `.jpg`, `.png`, `.tga`, + `.bmp`, `.hdr` + +--- + +#### [mlpack objects](#mlpack-objects) + +By default, load/save format for mlpack objects is autodetected, but can be +manually specified with the `format` parameter using one of the options below: + + - `format::autodetect` (default): auto-detects the format as one of the + formats below using the extension of the filename + - `format::json` (autodetect extension `.json`) + - `format::xml` (autodetect extension `.xml`) + - `format::binary` (autodetect extension `.bin`) + +***Notes:*** + + - `format::json` (`.json`) and `format::xml` (`.xml`) produce human-readable + files, but they may be quite large. + - `format::binary` (`.bin`) is recommended for the sake of size; objects in + binary format may be an order of magnitude or more smaller than JSON! diff --git a/doc/user/matrices.md b/doc/user/matrices.md index d2159f77d5..b9c8135d6f 100644 --- a/doc/user/matrices.md +++ b/doc/user/matrices.md @@ -1,50 +1,95 @@ # Matrices in mlpack -mlpack uses Armadillo matrices for matrix support. Armadillo is a fast C++ -matrix library which makes use of advanced template techniques to provide the -fastest possible matrix operations. +mlpack uses Armadillo matrices for linear algebra support. Armadillo is a fast +C++ matrix library which uses advanced template metaprogramming techniques to +provide the fastest possible linear algebra operations. -Documentation on Armadillo can be found on [the Armadillo +

Armadillo logo

+ +Detailed documentation on Armadillo can be found on [the Armadillo website](http://arma.sourceforge.net/docs.html). Nonetheless, there are a few further caveats for mlpack Armadillo usage. -## Column-major matrices + * [An Armadillo primer](#an-armadillo-primer) + * [Representing data in mlpack](#representing-data-in-mlpack) + * [Loading data](#loading-data) + * [Loading and using categorical data](#loading-and-using-categorical-data) + * [Alternate matrix types](#alternate-matrix-types) + * [Adapting from other toolkits (Eigen, etc.)](#adapting-from-other-toolkits-eigen-etc) -Armadillo matrices are stored in a column-major format; this means that on disk, -each column is located in contiguous memory. +## An Armadillo primer + +The Armadillo syntax is straightforward and is aimed at ease-of-use and +readability. To give a flavor of what a linear algebra program using Armadillo +looks like, see the trivial (contrived) program below that performs some basic +matrix operations. + +```c++ +// Create a 10x15 matrix with random elements. +arma::mat m(10, 15, arma::fill::randu); + +std::cout << "Size of m: " << m.n_rows << " x " << m.n_cols << "." << std::endl; + +// Sum all elements in the matrix. +const double sumVal = arma::accu(m); +std::cout << "Sum of all elements: " << sumVal << "." << std::endl; + +// Sum the elements in each column. +arma::rowvec sums = arma::sum(m, 0); +std::cout << "Sums in each column: " << sums; + +// Add 1 to all elements. +m += 1; + +// Subtract sums from each row. +m.each_row() -= sums; + +// Print an individual element. +std::cout << "m(3, 4) is: " << m(3, 4) << "." << std::endl; +``` + +For more information on Armadillo, see the following resources: + + * [Armadillo documentation](https://arma.sourceforge.net/docs.html) + * [Armadillo example + program](https://arma.sourceforge.net/docs.html#example_prog) + * [Armadillo/MATLAB syntax conversion + table](https://arma.sourceforge.net/docs.html#syntax) + +## Representing data in mlpack + +Armadillo matrices, unlike numpy and some other toolkits, store data in a +***column-major*** format. This means that each column is located in contiguous +memory; i.e., `x(0, 0)` is adjacent to `x(1, 0)` in memory. This means that, for the vast majority of machine learning methods, it is faster -to store observations as columns and dimensions as rows. This is counter to -most standard machine learning texts! +to store ***observations as columns*** and ***dimensions as rows***. This is +counter to most standard machine learning texts! It also has some implications +for linear algebra operations; for instance, computing the Gram matrix of a +matrix `X` is typically expressed as `X^T X`, but when using column-major +matrices, the expression must be `X X^T`. -Major implications of this are for linear algebra. For instance, the covariance -of a matrix is typically +In general, the following Armadillo types are commonly used inside mlpack: -``` -C = X^T X -``` + * `arma::mat`: datasets and general-purpose matrices + * `arma::Row`: integer response data, e.g., labels for classification + datasets + * `arma::rowvec`: floating-point response data, e.g., responses for regression + datasets + * `arma::vec`: general-purpose column vectors + * `arma::sp_mat`, `arma::fmat`: alternate types for representing data; see + [Alternate matrix types](#alternate-matrix-types) -but for a column-wise matrix, it is +## Loading data -``` -C = X X^T -``` +mlpack provides two simple functions for loading and saving data matrices in a +column-major form: -and this is very important to keep in mind! If your mlpack code is not working, -this may be a factor in why. + * `data::Load(filename, matrix, fatal=false, transpose=true, type=FileType::AutoDetect)` ([full documentation](load_save.md#numeric_data)) + * `data::Save(filename, matrix, fatal=false, transpose=true, type=FileType::AutoDetect)` ([full documentation](load_save.md#numeric_data)) -## Loading matrices - -mlpack provides a `data::Load()` and `data::Save()` function, which should be -used instead of Armadillo's loading and saving functions. - -Most machine learning data is stored in row-major format; a CSV, for example, -will generally have one observation per line and each column will correspond to -a dimension. - -The `data::Load()` and `data::Save()` functions transpose the matrix upon -loading, meaning that the following CSV: +As an example, consider the following CSV file: ```sh $ cat data.csv @@ -63,6 +108,282 @@ $ cat data.csv 2,4,4,2,0 ``` -is actually loaded with 5 rows and 13 columns, not 13 rows and 5 columns like -the CSV is written. More information on mlpack's loading functionality can be -found in [the formats tutorial](formats.md). +The following program will load the data, print information about it, and save a +modified dataset to disk. + +```c++ +// Load data from `data.csv` into `m`. Throw an exception on failure (i.e. set +// `fatal` to `true`). +arma::mat m; +mlpack::data::Load("data.csv", m, true); + +// Since mlpack uses column-major data, +// +// - each column corresponds to a data point! +// - each row corresponds to a dimension! +// +std::cout << "The matrix in 'data.csv' has: " << std::endl; +std::cout << " - " << m.n_cols << " points." << std::endl; +std::cout << " - " << m.n_rows << " dimensions." << std::endl; + +std::cout << "The second point in the dataset: " << std::endl; +std::cout << m.col(1).t(); + +// Now modify the matrix and save to a different format (space-separated +// values). +m += 3; +mlpack::data::Save("data-mod.txt", m); +``` + +Although Armadillo does provide a `.load()` and `.save()` member function for +matrices, the `data::Load()` and `data::Save()` functions offer additional +flexibility, and ensure that data is saved and loaded in a column-major format. + +## Loading and using categorical data + +Some mlpack techniques support mixed categorical data, e.g., data where some +dimensions take only categorical values (e.g. `0`, `1`, `2`, etc.). String data +and other non-numerical data can be represented as categorical values, and +mlpack has support to load mixed categorical data: + + * The `data::DatasetInfo` auxiliary class stores information about whether each + dimension is numeric or categorical. ([full + documentation](load_save.md#dataset_info)) + * `data::Load(filename, matrix, info, fatal=false, transpose=true)` ([full + documentation](load_save.md#load_categorical)) + +For example, consider the following CSV file that contains strings: + +```sh +$ cat mixed_string_data.csv +3,"hello",3,"f",0 +3,"goodbye",4,"f",0 +3,"goodbye",4,"e",0 +3,"hello",4,"d",0 +3,"hello",4,"d",0 +2,"hello",4,"d",0 +2,"hello",4,"d",0 +3,"goodbye",3,"f",0 +3,"goodbye",4,"f",0 +3,"hello",4,"f",0 +3,"hello",4,"c",0 +3,"hello",4,"f",0 +2,"hello",4,"c",0 +``` + +The following program will load the data file, print information about +categorical dimensions, and prepare the data for use with an mlpack algorithm +that supports mixed categorical data. + +```c++ +// Load data from `mixed_string_data.csv` into `m`. Throw an exception on +// failure (i.e. set `fatal` to `true`). This populates the `info` object. +arma::mat m; +mlpack::data::DatasetInfo info; +mlpack::data::Load("mixed_string_data.csv", m, info, true); + +// Print information about the data. +std::cout << "The matrix in 'mixed_string_data.csv' has: " << std::endl; +std::cout << " - " << m.n_cols << " points." << std::endl; +std::cout << " - " << info.Dimensionality() << " dimensions." << std::endl; + +// Print which dimensions are categorical. +for (size_t d = 0; d < info.Dimensionality(); ++d) +{ + if (info.Type(d) == mlpack::data::Datatype::categorical) + { + std::cout << " - Dimension " << d << " is categorical with " + << info.NumMappings(d) << " distinct categories." << std::endl; + } +} + +// Modify the third point to be 4,"wonderful",1,"c",0. +// Note that we manually map the string values; MapString() returns the category +// for a given value. +m(0, 2) = 4; +m(1, 2) = info.MapString("wonderful", 1); // Create new third category. +m(2, 2) = 1; +m(3, 2) = info.MapString("c", 1); +m(4, 2) = 0; + +// `m` can now be used with any mlpack algorithm that supports categorical data. +``` + +Not every mlpack method supports categorical data. Below are the list of +methods that do have categorical data support: + + * [`DecisionTree`](decision_tree.md) + * [`DecisionTreeRegressor`](decision_tree_regressor.md) + * [`RandomForest`](random_forest.md) + * [`HoeffdingTree`](hoeffding_tree.md) + +## Alternate matrix types + +mlpack's documentation focuses on the use of the `arma::mat`, `arma::vec`, and +`arma::rowvec` types, with an underlying `double` numeric type, e.g., 64-bit +floating-point. But, many mlpack algorithms and support utilities have support +for alternate matrix types and element types: + + * Many methods, such as + [`LogisticRegression`](logistic_regression.md#advanced-functionality-different-element-types), + allow specifying the matrix types as a template parameter. + + * Some methods, such as + [`DecisionTree`](decision_tree.md#using-different-element-types), + accept different matrix types to `Train()`, `Classify()`, or `Predict()`, + without needing to specify an explicit template parameter. + + * In general, any matrix type that supports the Armadillo API can be used; this + includes: + - Single-precision floating point matrices (`arma::fmat`, `arma::frowvec`, + `arma::fvec`) + - Sparse matrices (`arma::sp_mat`, `arma::sp_fmat`) + - GPU matrices via [Bandicoot](https://coot.sourceforge.io) (`coot::mat`, + `coot::fmat`) --- ***(note: support is under development and still + experimental)*** + +--- + +A simple example of using single-precision floating point data to train an +[`AdaBoost`](adaboost.md) model is below. + +```c++ +// 1000 random points in 10 dimensions, using 32-bit precision (float). +arma::fmat dataset(10, 1000, arma::fill::randu); +// Random labels for each point, totaling 5 classes. +arma::Row labels = + arma::randi>(1000, arma::distr_param(0, 4)); + +// Train in the constructor, using floating-point data. +// The weak learner type is now a floating-point Perceptron. +typedef mlpack::Perceptron< + mlpack::SimpleWeightUpdate, + mlpack::ZeroInitialization, + arma::fmat> PerceptronType; +mlpack::AdaBoost ab(dataset, labels, 5); + +// Create test data (500 points). +arma::fmat testDataset(10, 500, arma::fill::randu); +arma::Row predictions; +ab.Classify(testDataset, predictions); +// Now `predictions` holds predictions for the test dataset. + +// Print some information about the test predictions. +std::cout << arma::accu(predictions == 3) << " test points classified as class " + << "3." << std::endl; +``` + +--- + +A simple example of using sparse 32-bit floating point data to train a +[`LogisticRegression`](logistic_regression.md) model is below. + +```c++ +// Create random, sparse 100-dimensional data. +arma::sp_fmat dataset; +dataset.sprandu(100, 5000, 0.3); +arma::Row labels = + arma::randi>(5000, arma::distr_param(0, 1)); + +// Train with L2 regularization penalty parameter of 0.1. +mlpack::LogisticRegression lr(dataset, labels, 0.1); + +// Now classify a test point. +arma::sp_fvec point; +point.sprandu(100, 1, 0.3); + +size_t prediction; +arma::fvec probabilitiesVec; +lr.Classify(point, prediction, probabilitiesVec); + +std::cout << "Prediction for random test point: " << prediction << "." + << std::endl; +std::cout << "Class probabilities for random test point: " + << probabilitiesVec.t(); +``` + + + +## Adapting from other toolkits (Eigen, etc.) + +In general, C++ linear algebra toolkits store data in a column-major +representation, and transitioning between toolkits is a matter of getting access +to the underlying memory. + +--- + +Copy an [Eigen](https://eigen.tuxfamily.org) matrix into an Armadillo matrix. + +```c++ +// Note: this will only work if the Eigen matrix is stored in column-major +// order. See https://eigen.tuxfamily.org/dox/group__TopicStorageOrders.html +// for more details. +Eigen::MatrixXd m; +const size_t rows = 10; +const size_t cols = 20; +m.setRandom(rows, cols); // 10x20 random matrix. + +// Copy into an Armadillo matrix. +arma::mat mCopy(&m(0, 0), rows, cols); +``` + +--- + +Copy an [XTensor](https://xtensor.readthedocs.io/en/latest/) matrix into an +Armadillo matrix. + +```c++ +// Note: this will only work correctly if the layout_type of the XTensor matrix +// is column-major (i.e. xt::layout_type::column_major). See +// https://xtensor.readthedocs.io/en/latest/container.html for more details. +const size_t rows = 10; +const size_t cols = 20; + +// Create a random 10 x 20 matrix with normally distributed values. +// Note that we must ensure that the matrix is laid out in column-major form. +xt::xarray m = + xt::random::randn({ rows, cols }); + +// Copy into an Armadillo matrix. +arma::mat mCopy(m.data(), rows, cols); +``` + +--- + +Make an Armadillo matrix that is an alias of an +[Eigen](https://eigen.tuxfamily.org) matrix. Note that changes to the Eigen +matrix will be reflected in the Armadillo matrix, and vice versa. + +*If the Eigen matrix is deallocated, the Armadillo matrix will become invalid. +Be careful! [More details +here](https://arma.sourceforge.net/docs.html#adv_constructors_mat).* + +```c++ +// Note: this will only work if the Eigen matrix is stored in column-major +// order. See https://eigen.tuxfamily.org/dox/group__TopicStorageOrders.html +// for more details. +Eigen::MatrixXd m; +const size_t rows = 10; +const size_t cols = 20; +m.setRandom(rows, cols); // 10x20 random matrix. + +// Make an Armadillo matrix that is an alias of the Eigen matrix. This avoids +// the copy, but is potentially dangerous: be careful that the Eigen matrix is +// not deleted while the Armadillo matrix is in use! +// +// See https://arma.sourceforge.net/docs.html#adv_constructors_mat +arma::mat mAlias(&m(0, 0), rows, cols, false, true); +``` + +--- + +Copy an Armadillo matrix to an Eigen matrix. + +```c++ +const size_t rows = 10; +const size_t cols = 20; +arma::mat m(10, 20, arma::fill::randu); + +// Construct the Eigen matrix by using a map of the Armadillo memory. +Eigen::MatrixXd eigenM(Eigen::Map(m.memptr(), rows, cols)); +``` diff --git a/src/mlpack/core/kernels/epanechnikov_kernel.hpp b/src/mlpack/core/kernels/epanechnikov_kernel.hpp index 97904160fb..e841ea67fb 100644 --- a/src/mlpack/core/kernels/epanechnikov_kernel.hpp +++ b/src/mlpack/core/kernels/epanechnikov_kernel.hpp @@ -63,24 +63,6 @@ class EpanechnikovKernel */ inline double Gradient(const double distance) const; - /** - * Evaluate the Gradient of Epanechnikov kernel - * given that the squared distance between the two - * input points is known. - */ - inline double GradientForSquaredDistance(const double distanceSquared) const; - /** - * Obtains the convolution integral [integral of K(||x-a||) K(||b-x||) dx] - * for the two vectors. - * - * @tparam VecType Type of vector (arma::vec, arma::spvec should be expected). - * @param a First vector. - * @param b Second vector. - * @return the convolution integral value. - */ - template - double ConvolutionIntegral(const VecTypeA& a, const VecTypeB& b); - /** * Compute the normalizer of this Epanechnikov kernel for the given dimension. * diff --git a/src/mlpack/core/kernels/epanechnikov_kernel_impl.hpp b/src/mlpack/core/kernels/epanechnikov_kernel_impl.hpp index bd0432bd66..4fe12f6c6d 100644 --- a/src/mlpack/core/kernels/epanechnikov_kernel_impl.hpp +++ b/src/mlpack/core/kernels/epanechnikov_kernel_impl.hpp @@ -28,50 +28,6 @@ inline double EpanechnikovKernel::Evaluate(const VecTypeA& a, const VecTypeB& b) * inverseBandwidthSquared); } -/** - * Obtains the convolution integral [integral of K(||x-a||) K(||b-x||) dx] - * for the two vectors. - * - * @tparam VecTypeA Type of first vector (arma::vec, arma::sp_vec should be - * expected). - * @tparam VecTypeB Type of second vector (arma::vec, arma::sp_vec). - * @param a First vector. - * @param b Second vector. - * @return the convolution integral value. - */ -template -inline double EpanechnikovKernel::ConvolutionIntegral(const VecTypeA& a, - const VecTypeB& b) -{ - double distance = sqrt(SquaredEuclideanDistance::Evaluate(a, b)); - if (distance >= 2.0 * bandwidth) - return 0.0; - - double volumeSquared = std::pow(Normalizer(a.n_rows), 2.0); - - switch (a.n_rows) - { - case 1: - return 1.0 / volumeSquared * - (16.0 / 15.0 * bandwidth - 4.0 * distance * distance / - (3.0 * bandwidth) + 2.0 * distance * distance * distance / - (3.0 * bandwidth * bandwidth) - - std::pow(distance, 5.0) / (30.0 * std::pow(bandwidth, 4.0))); - case 2: - return 1.0 / volumeSquared * - ((2.0 / 3.0 * bandwidth * bandwidth - distance * distance) * - asin(sqrt(1.0 - std::pow(distance / (2.0 * bandwidth), 2.0))) + - sqrt(4.0 * bandwidth * bandwidth - distance * distance) * - (distance / 6.0 + 2.0 / 9.0 * distance * - std::pow(distance / bandwidth, 2.0) - distance / 72.0 * - std::pow(distance / bandwidth, 4.0))); - default: - Log::Fatal << "EpanechnikovKernel::ConvolutionIntegral(): dimension " - << a.n_rows << " not supported."; - return -1.0; // This line will not execute. - } -} - /** * Compute the normalizer of this Epanechnikov kernel for the given dimension. * @@ -113,29 +69,6 @@ inline double EpanechnikovKernel::Gradient(const double distance) const } } -/** - * Evaluate gradient of the kernel not for two points - * but for a numerical value. - */ -inline double EpanechnikovKernel::GradientForSquaredDistance( - const double distanceSquared) const -{ - double bandwidthSquared = bandwidth * bandwidth; - if (distanceSquared < bandwidthSquared) - { - return -1 * inverseBandwidthSquared; - } - else if (distanceSquared > bandwidthSquared && - distanceSquared >= 0) - { - return 0; - } - else - { - // The gradient doesn't exist. - return arma::datum::nan; - } -} //! Serialize the kernel. template void EpanechnikovKernel::serialize(Archive& ar, diff --git a/src/mlpack/core/kernels/example_kernel.hpp b/src/mlpack/core/kernels/example_kernel.hpp index a5dcfa7502..6b8e2a2981 100644 --- a/src/mlpack/core/kernels/example_kernel.hpp +++ b/src/mlpack/core/kernels/example_kernel.hpp @@ -89,24 +89,6 @@ class ExampleKernel template void serialize(Archive& /* ar */, const uint32_t /* version */) { } - /** - * Obtains the convolution integral [integral K(||x-a||)K(||b-x||)dx] - * for the two vectors. In this case, because - * our simple example kernel has no internal parameters, we can declare the - * function static. For a more complex example which cannot be declared - * static, see the GaussianKernel, which stores an internal parameter. - * - * @tparam VecTypeA Type of first vector (arma::vec, arma::sp_vec should be - * expected). - * @tparam VecTypeB Type of second vector (arma::vec, arma::sp_vec). - * @param * (a) First vector. - * @param * (b) Second vector. - * @return the convolution integral value. - */ - template - static double ConvolutionIntegral(const VecTypeA& /* a */, - const VecTypeB& /* b */) { return 0; } - /** * Obtains the normalizing volume for the kernel with dimension $dimension$. * In this case, because our simple example kernel has no internal parameters, diff --git a/src/mlpack/core/kernels/gaussian_kernel.hpp b/src/mlpack/core/kernels/gaussian_kernel.hpp index 2d9b3d98cc..1c07e29939 100644 --- a/src/mlpack/core/kernels/gaussian_kernel.hpp +++ b/src/mlpack/core/kernels/gaussian_kernel.hpp @@ -92,18 +92,6 @@ class GaussianKernel return 2 * t * gamma * exp(gamma * std::pow(t, 2.0)); } - /** - * Evaluation of the gradient of Gaussian kernel - * given the squared distance between two points. - * - * @param t The squared distance between the two points - * @return K(t) using the bandwidth (@f$\mu@f$) specified in the - * constructor. - */ - double GradientForSquaredDistance(const double t) const { - return gamma * exp(gamma * t); - } - /** * Obtain the normalization constant of the Gaussian kernel. * @@ -115,21 +103,6 @@ class GaussianKernel return pow(sqrt(2.0 * M_PI) * bandwidth, (double) dimension); } - /** - * Obtain a convolution integral of the Gaussian kernel. - * - * @param a First vector. - * @param b Second vector. - * @return The convolution integral. - */ - template - double ConvolutionIntegral(const VecTypeA& a, const VecTypeB& b) - { - return Evaluate(sqrt(SquaredEuclideanDistance::Evaluate(a, b) / - 2.0)) / (Normalizer(a.n_rows) * pow(2.0, (double) a.n_rows / 2.0)); - } - - //! Get the bandwidth. double Bandwidth() const { return bandwidth; } diff --git a/src/mlpack/core/kernels/spherical_kernel.hpp b/src/mlpack/core/kernels/spherical_kernel.hpp index ab0ca6fb70..9012fcce1e 100644 --- a/src/mlpack/core/kernels/spherical_kernel.hpp +++ b/src/mlpack/core/kernels/spherical_kernel.hpp @@ -44,41 +44,7 @@ class SphericalKernel return (SquaredEuclideanDistance::Evaluate(a, b) <= bandwidthSquared) ? 1.0 : 0.0; } - /** - * Obtains the convolution integral [integral K(||x-a||)K(||b-x||)dx] - * for the two vectors. - * - * @tparam VecTypeA Type of first vector (arma::vec, arma::sp_vec should be - * expected). - * @tparam VecTypeB Type of second vector. - * @param a First vector. - * @param b Second vector. - * @return The convolution integral value. - */ - template - double ConvolutionIntegral(const VecTypeA& a, const VecTypeB& b) const - { - double distance = sqrt(SquaredEuclideanDistance::Evaluate(a, b)); - if (distance >= 2.0 * bandwidth) - { - return 0.0; - } - double volumeSquared = pow(Normalizer(a.n_rows), 2.0); - switch (a.n_rows) - { - case 1: - return 1.0 / volumeSquared * (2.0 * bandwidth - distance); - case 2: - return 1.0 / volumeSquared * - (2.0 * bandwidth * bandwidth * acos(distance/(2.0 * bandwidth)) - - distance / 4.0 * sqrt(4.0*bandwidth*bandwidth-distance*distance)); - default: - Log::Fatal << "The spherical kernel does not support convolution\ - integrals above dimension two, yet..." << std::endl; - return -1.0; - } - } double Normalizer(size_t dimension) const { return pow(bandwidth, (double) dimension) * pow(M_PI, dimension / 2.0) / @@ -94,6 +60,7 @@ class SphericalKernel { return (t <= bandwidth) ? 1.0 : 0.0; } + double Gradient(double t) { return t == bandwidth ? arma::datum::nan : 0.0; diff --git a/src/mlpack/tests/kernel_test.cpp b/src/mlpack/tests/kernel_test.cpp index 969b67877c..7d0927793e 100644 --- a/src/mlpack/tests/kernel_test.cpp +++ b/src/mlpack/tests/kernel_test.cpp @@ -286,13 +286,6 @@ TEST_CASE("GaussianKernelTest", "[KernelTest]") REQUIRE(gk.Normalizer(2) == Approx(1.5707963267948963).epsilon(1e-7)); REQUIRE(gk.Normalizer(3) == Approx(1.9687012432153019).epsilon(1e-7)); REQUIRE(gk.Normalizer(4) == Approx(2.4674011002723386).epsilon(1e-7)); - /* check the convolution integral */ - REQUIRE(gk.ConvolutionIntegral(a, b) == - Approx(0.024304474038457577).epsilon(1e-7)); - REQUIRE(gk.ConvolutionIntegral(a, c) == - Approx(0.024304474038457577).epsilon(1e-7)); - REQUIRE(gk.ConvolutionIntegral(b, c) == - Approx(0.024304474038457577).epsilon(1e-7)); } TEST_CASE("GaussianKernelSerializationTest", "[KernelTest]") @@ -329,11 +322,6 @@ TEST_CASE("SphericalKernelTest", "[KernelTest]") REQUIRE(sk.Normalizer(2) == Approx(0.78539816339744828).epsilon(1e-7)); REQUIRE(sk.Normalizer(3) == Approx(0.52359877559829893).epsilon(1e-7)); REQUIRE(sk.Normalizer(4) == Approx(0.30842513753404244).epsilon(1e-7)); - /* check the convolution integral */ - REQUIRE(sk.ConvolutionIntegral(a, b) == Approx(0.0).epsilon(1e-7)); - REQUIRE(sk.ConvolutionIntegral(a, c) == Approx(0.0).epsilon(1e-7)); - REQUIRE(sk.ConvolutionIntegral(b, c) == - Approx(1.0021155029652784).epsilon(1e-7)); } TEST_CASE("EpanechnikovKernelTest", "[KernelTest]") @@ -356,11 +344,6 @@ TEST_CASE("EpanechnikovKernelTest", "[KernelTest]") REQUIRE(ek.Normalizer(2) == Approx(0.39269908169872414).epsilon(1e-7)); REQUIRE(ek.Normalizer(3) == Approx(0.20943951023931956).epsilon(1e-7)); REQUIRE(ek.Normalizer(4) == Approx(0.10280837917801415).epsilon(1e-7)); - /* check the convolution integral */ - REQUIRE(ek.ConvolutionIntegral(a, b) == Approx(0.0).epsilon(1e-7)); - REQUIRE(ek.ConvolutionIntegral(a, c) == Approx(0.0).epsilon(1e-7)); - REQUIRE(ek.ConvolutionIntegral(b, c) == - Approx(1.5263455690698258).epsilon(1e-7)); } TEST_CASE("PolynomialKernelTest", "[KernelTest]")