Merge remote-tracking branch 'origin/master' into main-docs
This commit is contained in:
Ryan Curtin
+69
-64
@@ -24,76 +24,76 @@ jobs:
|
||||
r_bindings: ${{ steps.mlpack_version.outputs.mlpack_r_package }}
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
- uses: actions/checkout@v3
|
||||
|
||||
- name: Extract mlpack version
|
||||
id: mlpack_version
|
||||
run: |
|
||||
MLPACK_VERSION_MAJOR=$(grep -i ".*#define MLPACK_VERSION_MAJOR.*" src/mlpack/core/util/version.hpp | grep -o "[0-9]*")
|
||||
MLPACK_VERSION_MINOR=$(grep -i ".*#define MLPACK_VERSION_MINOR.*" src/mlpack/core/util/version.hpp | grep -o "[0-9]*")
|
||||
MLPACK_VERSION_PATCH=$(grep -i ".*#define MLPACK_VERSION_PATCH.*" src/mlpack/core/util/version.hpp | grep -o "[0-9]*")
|
||||
MLPACK_VERSION_VALUE=${MLPACK_VERSION_MAJOR}.${MLPACK_VERSION_MINOR}.${MLPACK_VERSION_PATCH}
|
||||
echo ::set-output name=mlpack_r_package::$(echo mlpack_"$MLPACK_VERSION_VALUE".tar.gz)
|
||||
- name: Extract mlpack version
|
||||
id: mlpack_version
|
||||
run: |
|
||||
MLPACK_VERSION_MAJOR=$(grep -i ".*#define MLPACK_VERSION_MAJOR.*" src/mlpack/core/util/version.hpp | grep -o "[0-9]*")
|
||||
MLPACK_VERSION_MINOR=$(grep -i ".*#define MLPACK_VERSION_MINOR.*" src/mlpack/core/util/version.hpp | grep -o "[0-9]*")
|
||||
MLPACK_VERSION_PATCH=$(grep -i ".*#define MLPACK_VERSION_PATCH.*" src/mlpack/core/util/version.hpp | grep -o "[0-9]*")
|
||||
MLPACK_VERSION_VALUE=${MLPACK_VERSION_MAJOR}.${MLPACK_VERSION_MINOR}.${MLPACK_VERSION_PATCH}
|
||||
echo "mlpack_r_package=$(echo mlpack_"$MLPACK_VERSION_VALUE".tar.gz)" >> $GITHUB_OUTPUT
|
||||
|
||||
# Setup Pandoc
|
||||
- uses: r-lib/actions/setup-pandoc@v2
|
||||
# Setup Pandoc
|
||||
- uses: r-lib/actions/setup-pandoc@v2
|
||||
|
||||
# Setup R actions
|
||||
- uses: r-lib/actions/setup-r@v2
|
||||
# Setup R actions
|
||||
- uses: r-lib/actions/setup-r@v2
|
||||
|
||||
- name: Query dependencies
|
||||
run: |
|
||||
cp src/mlpack/bindings/R/mlpack/DESCRIPTION.in DESCRIPTION
|
||||
Rscript -e "install.packages('remotes')" -e "saveRDS(remotes::dev_package_deps(dependencies = TRUE), 'depends.Rds')"
|
||||
- name: Query dependencies
|
||||
run: |
|
||||
cp src/mlpack/bindings/R/mlpack/DESCRIPTION.in DESCRIPTION
|
||||
Rscript -e "install.packages('remotes')" -e "saveRDS(remotes::dev_package_deps(dependencies = TRUE), 'depends.Rds')"
|
||||
|
||||
- name: Cache R packages
|
||||
if: runner.os != 'Windows' && runner.os != 'macOS'
|
||||
uses: actions/cache@v3
|
||||
with:
|
||||
path: ${{ env.R_LIBS_USER }}
|
||||
key: ${{ runner.os }}-r-release-${{ hashFiles('depends.Rds') }}
|
||||
restore-keys: ${{ runner.os }}-r-release-
|
||||
- name: Cache R packages
|
||||
if: runner.os != 'Windows' && runner.os != 'macOS'
|
||||
uses: actions/cache@v3
|
||||
with:
|
||||
path: ${{ env.R_LIBS_USER }}
|
||||
key: ${{ runner.os }}-r-release-${{ hashFiles('depends.Rds') }}
|
||||
restore-keys: ${{ runner.os }}-r-release-
|
||||
|
||||
- name: Install Build Dependencies
|
||||
run: |
|
||||
sudo apt-get update
|
||||
# We don't install cereal via apt, because the Debian packagers
|
||||
# split the rapidjson dependency into a separate package. We will
|
||||
# bundle the cereal sources with the R package, so we want them to
|
||||
# be exactly the upstream sources (with rapidjson included).
|
||||
sudo apt-get install -y --allow-unauthenticated libopenblas-dev liblapack-dev g++ libensmallen-dev libhdf5-dev libarmadillo-dev libcurl4-openssl-dev
|
||||
wget https://github.com/USCiLab/cereal/archive/refs/tags/v1.3.2.tar.gz
|
||||
tar -xvzpf v1.3.2.tar.gz
|
||||
# These directives cause warnings on CRAN:
|
||||
# https://github.com/USCiLab/cereal/blob/master/include/cereal/external/base64.hpp#L28-L31
|
||||
# The command below comments them out.
|
||||
sed -i 's|#pragma|// #pragma|' cereal-1.3.2/include/cereal/external/base64.hpp
|
||||
- name: Install Build Dependencies
|
||||
run: |
|
||||
sudo apt-get update
|
||||
# We don't install cereal via apt, because the Debian packagers
|
||||
# split the rapidjson dependency into a separate package. We will
|
||||
# bundle the cereal sources with the R package, so we want them to
|
||||
# be exactly the upstream sources (with rapidjson included).
|
||||
sudo apt-get install -y --allow-unauthenticated libopenblas-dev liblapack-dev g++ libensmallen-dev libhdf5-dev libarmadillo-dev libcurl4-openssl-dev
|
||||
wget https://github.com/USCiLab/cereal/archive/refs/tags/v1.3.2.tar.gz
|
||||
tar -xvzpf v1.3.2.tar.gz
|
||||
# These directives cause warnings on CRAN:
|
||||
# https://github.com/USCiLab/cereal/blob/master/include/cereal/external/base64.hpp#L28-L31
|
||||
# The command below comments them out.
|
||||
sed -i 's|#pragma|// #pragma|' cereal-1.3.2/include/cereal/external/base64.hpp
|
||||
|
||||
- name: Install R-bindings dependencies
|
||||
run: |
|
||||
remotes::install_deps(dependencies = TRUE)
|
||||
remotes::install_cran("roxygen2")
|
||||
remotes::install_cran("pkgbuild")
|
||||
shell: Rscript {0}
|
||||
- name: Install R-bindings dependencies
|
||||
run: |
|
||||
remotes::install_deps(dependencies = TRUE)
|
||||
remotes::install_cran("roxygen2")
|
||||
remotes::install_cran("pkgbuild")
|
||||
shell: Rscript {0}
|
||||
|
||||
- name: CMake
|
||||
run: |
|
||||
mkdir build
|
||||
cd build && cmake -DDEBUG=OFF -DPROFILE=OFF -DBUILD_CLI_EXECUTABLES=OFF -DBUILD_PYTHON_BINDINGS=OFF -DBUILD_JULIA_BINDINGS=OFF -DBUILD_GO_BINDINGS=OFF -DBUILD_R_BINDINGS=ON -DDOWNLOAD_DEPENDENCIES=ON -DBUILD_TESTS=ON -DCEREAL_INCLUDE_DIR=../cereal-1.3.2/include/ ..
|
||||
- name: CMake
|
||||
run: |
|
||||
mkdir build
|
||||
cd build && cmake -DDEBUG=OFF -DPROFILE=OFF -DBUILD_CLI_EXECUTABLES=OFF -DBUILD_PYTHON_BINDINGS=OFF -DBUILD_JULIA_BINDINGS=OFF -DBUILD_GO_BINDINGS=OFF -DBUILD_R_BINDINGS=ON -DDOWNLOAD_DEPENDENCIES=ON -DBUILD_TESTS=ON -DCEREAL_INCLUDE_DIR=../cereal-1.3.2/include/ ..
|
||||
|
||||
- name: Build
|
||||
run: |
|
||||
cd build && make
|
||||
- name: Build
|
||||
run: |
|
||||
cd build && make
|
||||
|
||||
- name: Run tests via ctest
|
||||
run: |
|
||||
cd build && CTEST_OUTPUT_ON_FAILURE=1 ctest -T Test .
|
||||
- name: Run tests via ctest
|
||||
run: |
|
||||
cd build && CTEST_OUTPUT_ON_FAILURE=1 ctest -T Test .
|
||||
|
||||
- name: Upload R packages
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: mlpack_r_tarball
|
||||
path: build/src/mlpack/bindings/R/${{ steps.mlpack_version.outputs.mlpack_r_package }}
|
||||
- name: Upload R packages
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: mlpack_r_tarball
|
||||
path: build/src/mlpack/bindings/R/${{ steps.mlpack_version.outputs.mlpack_r_package }}
|
||||
|
||||
R-CMD-check:
|
||||
needs: jobR
|
||||
@@ -106,9 +106,14 @@ jobs:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
config:
|
||||
- {os: windows-latest, r: 'release', name: 'Windows R'}
|
||||
- {os: macOS-latest, r: 'release', name: 'macOS R'}
|
||||
- {os: ubuntu-latest, r: 'devel', http-user-agent: 'release', name: 'Linux R'}
|
||||
- { os: windows-latest, r: "release", name: "Windows R" }
|
||||
- { os: macOS-latest, r: "release", name: "macOS R" }
|
||||
- {
|
||||
os: ubuntu-latest,
|
||||
r: "devel",
|
||||
http-user-agent: "release",
|
||||
name: "Linux R",
|
||||
}
|
||||
|
||||
env:
|
||||
MAKEFLAGS: "-j 2"
|
||||
@@ -146,8 +151,8 @@ jobs:
|
||||
- name: Install check dependencies
|
||||
if: runner.os != 'Windows' && runner.os != 'macOS'
|
||||
run: |
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y --allow-unauthenticated libcurl4-openssl-dev
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y --allow-unauthenticated libcurl4-openssl-dev
|
||||
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
|
||||
@@ -6,7 +6,7 @@ name: Update Catch
|
||||
on:
|
||||
workflow_dispatch:
|
||||
schedule:
|
||||
- cron: '0 10 1/16 * *'
|
||||
- cron: "0 10 1/16 * *"
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
@@ -16,7 +16,7 @@ jobs:
|
||||
contents: write # for peter-evans/create-pull-request to create branch
|
||||
pull-requests: write # for peter-evans/create-pull-request to create a PR
|
||||
if: ${{ false }}
|
||||
# if: ${{ github.repository == 'mlpack/mlpack' }}
|
||||
# if: ${{ github.repository == 'mlpack/mlpack' }}
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
@@ -26,7 +26,7 @@ jobs:
|
||||
# Ping version information upstream.
|
||||
CATCH_RELEASE_JSON=$(curl -sL https://api.github.com/repos/catchorg/Catch2/releases/latest)
|
||||
CATCH_RELEASE_VERSION=$(jq -r ".tag_name" <<< "$CATCH_RELEASE_JSON" | tr -d v)
|
||||
echo ::set-output name=release_tag::$(echo $CATCH_RELEASE_VERSION)
|
||||
echo "release_tag=$(echo $CATCH_RELEASE_VERSION)" >> $GITHUB_OUTPUT
|
||||
# Extract out version information from git repository.
|
||||
CATCH_VERSION_MAJOR=$(grep -i ".*#define CATCH_VERSION_MAJOR.*" src/mlpack/tests/catch.hpp | grep -o "[0-9]*")
|
||||
CATCH_VERSION_MINOR=$(grep -i ".*#define CATCH_VERSION_MINOR.*" src/mlpack/tests/catch.hpp | grep -o "[0-9]*")
|
||||
@@ -34,7 +34,7 @@ jobs:
|
||||
# Combine values to match release tag information.
|
||||
CATCH_VERSION_VALUE=${CATCH_VERSION_MAJOR}.${CATCH_VERSION_MINOR}.${CATCH_VERSION_PATCH}
|
||||
# Set the current release tag.
|
||||
echo ::set-output name=current_tag::$(echo $CATCH_VERSION_VALUE)
|
||||
echo "current_tag=$(echo $CATCH_VERSION_VALUE)" >> $GITHUB_OUTPUT
|
||||
|
||||
- name: Update Catch
|
||||
if: steps.catch-header.outputs.current_tag != steps.catch-header.outputs.release_tag
|
||||
|
||||
@@ -2,15 +2,15 @@ name: Update CLI11
|
||||
on:
|
||||
workflow_dispatch:
|
||||
schedule:
|
||||
- cron: '0 10 1/16 * *'
|
||||
- cron: "0 10 1/16 * *"
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
updateCLI11:
|
||||
permissions:
|
||||
contents: write # for peter-evans/create-pull-request to create branch
|
||||
pull-requests: write # for peter-evans/create-pull-request to create a PR
|
||||
contents: write # for peter-evans/create-pull-request to create branch
|
||||
pull-requests: write # for peter-evans/create-pull-request to create a PR
|
||||
if: ${{ github.repository == 'mlpack/mlpack' }}
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
@@ -21,11 +21,11 @@ jobs:
|
||||
# Ping version information upstream.
|
||||
CLI11_RELEASE_JSON=$(curl -sL https://api.github.com/repos/CLIUtils/CLI11/releases/latest)
|
||||
CLI11_RELEASE_VERSION=$(jq -r ".tag_name" <<< "$CLI11_RELEASE_JSON" | tr -d v)
|
||||
echo ::set-output name=release_tag::$(echo $CLI11_RELEASE_VERSION)
|
||||
echo "release_tag=$(echo $CLI11_RELEASE_VERSION)" >> $GITHUB_OUTPUT
|
||||
# Extract out version information from git repository.
|
||||
CLI11_VERSION_VALUE=$(grep -i ".*#define CLI11_VERSION.*" src/mlpack/bindings/cli/third_party/CLI/CLI11.hpp | grep -Po "(\d+\.)+\d+")
|
||||
# Set the current release tag.
|
||||
echo ::set-output name=current_tag::$(echo $CLI11_VERSION_VALUE)
|
||||
echo "current_tag=$(echo $CLI11_VERSION_VALUE)" >> $GITHUB_OUTPUT
|
||||
|
||||
- name: Update CLI11
|
||||
if: steps.cli11-header.outputs.current_tag != steps.cli11-header.outputs.release_tag
|
||||
|
||||
@@ -0,0 +1,318 @@
|
||||
## `BayesianLinearRegression`
|
||||
|
||||
The `BayesianLinearRegression` class implements a Bayesian ridge regression
|
||||
model for numerical data that optimally tunes the regularization strength to the
|
||||
given data. The class offers configurable functionality and template parameters
|
||||
to control the data type used for storing the model.
|
||||
|
||||
#### Simple usage example:
|
||||
|
||||
```c++
|
||||
// Train a Bayesian linear regression model on random data and make predictions.
|
||||
|
||||
// All data and responses are uniform random; this uses 10 dimensional data.
|
||||
// Replace with a data::Load() call or similar for a real application.
|
||||
arma::mat dataset(10, 1000, arma::fill::randu); // 1000 points.
|
||||
arma::rowvec responses = arma::randn<arma::rowvec>(1000);
|
||||
arma::mat testDataset(10, 500, arma::fill::randu); // 500 test points.
|
||||
|
||||
mlpack::BayesianLinearRegression blr; // Step 1: create model.
|
||||
blr.Train(dataset, responses); // Step 2: train model.
|
||||
arma::rowvec predictions;
|
||||
blr.Predict(testDataset, predictions); // Step 3: use model to predict.
|
||||
|
||||
// Print some information about the test predictions.
|
||||
std::cout << arma::accu(predictions > 0.6) << " test points predicted to have"
|
||||
<< " responses greater than 0.6." << std::endl;
|
||||
std::cout << arma::accu(predictions < 0) << " test points predicted to have "
|
||||
<< "negative responses." << std::endl;
|
||||
```
|
||||
<p style="text-align: center; font-size: 85%"><a href="#simple-examples">More examples...</a></p>
|
||||
|
||||
#### Quick links:
|
||||
|
||||
* [Constructors](#constructors): create `BayesianLinearRegression` objects.
|
||||
* [`Train()`](#training): train model.
|
||||
* [`Predict()`](#prediction): predict with a trained model.
|
||||
* [Other functionality](#other-functionality) for loading, saving, and
|
||||
inspecting.
|
||||
* [Examples](#simple-examples) of simple usage and links to detailed example
|
||||
projects.
|
||||
* [Template parameters](#advanced-functionality-different-element-types) for
|
||||
using different element types for a model.
|
||||
|
||||
#### See also:
|
||||
|
||||
* [mlpack regression techniques](#mlpack_regression_techniques) <!-- TODO: fix link! -->
|
||||
* [`LinearRegression`](linear_regression.md) <!-- TODO: fix link -->
|
||||
* [`LARS`](lars.md) <!-- TODO: fix link -->
|
||||
* [Bayesian linear regression on Wikipedia](https://en.wikipedia.org/wiki/Bayesian_linear_regression)
|
||||
|
||||
### Constructors
|
||||
|
||||
* `blr = BayesianLinearRegression(centerData=true, scaleData=false, maxIterations=50, tolerance=1e-4)`
|
||||
- Initialize the model without training.
|
||||
- You will need to call [`Train()`](#training) later to train the model
|
||||
before calling [`Predict()`](#prediction).
|
||||
|
||||
---
|
||||
|
||||
* `blr = BayesianLinearRegression(data, responses)`
|
||||
* `blr = BayesianLinearRegression(data, responses, centerData=true, scaleData=false, maxIterations=50, tolerance=1e-4)`
|
||||
- Train model on the given data.
|
||||
|
||||
---
|
||||
|
||||
#### Constructor Parameters:
|
||||
|
||||
<!-- TODOs for table below:
|
||||
* better link for column-major matrices
|
||||
* update matrices.md to include a section on labels and NormalizeLabels()
|
||||
-->
|
||||
|
||||
| **name** | **type** | **description** | **default** |
|
||||
|----------|----------|-----------------|-------------|
|
||||
| `data` | [`arma::mat`](../matrices.md) | [Column-major](../matrices.md) training matrix. | _(N/A)_ |
|
||||
| `responses` | [`arma::rowvec`](../matrices.md) | Training responses (e.g. values to predict). Should have length `data.n_cols`. | _(N/A)_ |
|
||||
| `centerData` | `bool` | Whether to center the data before learning. | `true` |
|
||||
| `scaleData` | `bool` | Whether to scale the data to unit variance before learning. | `false` |
|
||||
| `maxIterations` | `size_t` | Maximum number of iterations for convergence. | `50` |
|
||||
| `tolerance` | `double` | Tolerance for convergence of the model. | `1e-4` |
|
||||
|
||||
As an alternative to passing `centerData`, `scaleData`, `maxIterations`, or
|
||||
`tolerance`, they can each be set or accessed with standalone methods:
|
||||
|
||||
* `blr.CenterData() = centerData;` will set whether to center the data before
|
||||
learning to `centerData`.
|
||||
* `blr.ScaleData() = scaleData;` will set whether to scale the data to unit
|
||||
variance before learning to `scaleData`.
|
||||
* `blr.MaxIterations() = maxIterations;` will set the maximum number of
|
||||
iterations to `maxIterations`.
|
||||
* `blr.Tolerance() = tolerance;` will set the tolerance for convergence to
|
||||
`tolerance`.
|
||||
|
||||
### Training
|
||||
|
||||
If training is not done as part of the constructor call, it can be done with the
|
||||
`Train()` function:
|
||||
|
||||
* `blr.Train(data, responses, centerData=true, scaleData=false, maxIterations=50, tolerance=1e-4)`
|
||||
|
||||
Types of each argument are the same as in the table for constructors
|
||||
[above](#constructor-parameters).
|
||||
|
||||
***Notes:***
|
||||
|
||||
* Training is not incremental. A second call to `Train()` will retrain the
|
||||
model from scratch.
|
||||
|
||||
* `Train()` returns the root mean squared error (RMSE) of the model on the
|
||||
training set as a `double`. <!-- TODO: check this -->
|
||||
|
||||
### Prediction
|
||||
|
||||
Once a `LinearRegression` model is trained, the `Predict()` member function
|
||||
can be used to make predictions for new data.
|
||||
|
||||
* `double predictedValue = blr.Predict(point)`
|
||||
- ***(Single-point)***
|
||||
- Make a prediction for a single point, returning the predicted value.
|
||||
|
||||
---
|
||||
* `blr.Predict(point, prediction, stddev)`
|
||||
- ***(Single-point)***
|
||||
- Make a prediction for a single point, storing the predicted value in
|
||||
`prediction` and the standard deviation of the prediction in `stddev`.
|
||||
|
||||
---
|
||||
|
||||
* `blr.Predict(data, predictions)`
|
||||
- ***(Multi-point)***
|
||||
- Make predictions for a set of points.
|
||||
- The prediction for data point `i` can be accessed with `predictions[i]`.
|
||||
|
||||
---
|
||||
|
||||
* `blr.Predict(data, predictions, stddevs)`
|
||||
- ***(Multi-point)***
|
||||
- Make predictions for a set of points and compute standard deviations of
|
||||
predictions.
|
||||
- The prediction for data point `i` can be accessed with `predictions[i]`.
|
||||
- The standard deviation of the prediction for data point `i` can be accessed
|
||||
with `stddevs[i]`.
|
||||
|
||||
---
|
||||
|
||||
#### Prediction Parameters:
|
||||
|
||||
| **usage** | **name** | **type** | **description** |
|
||||
|-----------|----------|----------|-----------------|
|
||||
| _single-point_ | `point` | [`arma::vec`](../matrices.md) | Single point for prediction. |
|
||||
| _single-point_ | `prediction` | `double&` | `double` to store predicted value into. |
|
||||
| _single-point_ | `stddev` | `double&` | `double` to store standard deviation of predicted value into. |
|
||||
||||
|
||||
| _multi-point_ | `data` | [`arma::mat`](../matrices.md) | Set of [column-major](../matrices.md) points for classification. |
|
||||
| _multi-point_ | `predictions` | [`arma::rowvec&`](../matrices.md) | Vector of `double`s to store predictions into. Will be set to length `data.n_cols`. |
|
||||
| _multi-point_ | `stddevs` | [`arma::rowvec&`](../matrices.md) | Vector of `double`s to store standard deviations of predictions into. Will be set to length `data.n_cols`. |
|
||||
|
||||
### Other Functionality
|
||||
|
||||
<!-- TODO: we should point directly to the documentation of those functions -->
|
||||
|
||||
* A `BayesianLinearRegression` model can be serialized with
|
||||
[`data::Save()`](../formats.md) and [`data::Load()`](../formats.md).
|
||||
|
||||
* After training is complete, the following methods can be used to inspect the
|
||||
model:
|
||||
- `blr.Omega()` returns the weights of the trained model as an
|
||||
`const arma::vec&` of length `data.n_rows`. The weight for the `i`th
|
||||
dimension can be accessed with `blr.Omega()[i]`.
|
||||
|
||||
- `blr.Alpha()` returns the precision (or inverse variance) of the Gaussian
|
||||
prior of the model as a `double`.
|
||||
|
||||
- `blr.Beta()` returns the precision (or inverse variance) of the model as a
|
||||
`double`.
|
||||
|
||||
- `blr.Variance()` returns the estimated variance as a `double`.
|
||||
|
||||
- `blr.DataOffset()` returns a `const arma::vec&` containing the mean values
|
||||
of the training data in each dimension. The vector has length
|
||||
`data.n_rows`. The result is only meaningful if `centerData` is `true`.
|
||||
|
||||
- `blr.DataScale()` returns a `const arma::vec&` containing the standard
|
||||
deviations of the training data in each dimension. The vector has length
|
||||
`data.n_rows`. The result is only meaningful if `scaleData` is `true`.
|
||||
|
||||
- `blr.ResponsesOffset()` returns the mean value of the training responses as
|
||||
a `double`. This is the intercept of the model.
|
||||
|
||||
* `blr.RMSE(data, responses)` returns a `double` containing the RMSE (root mean
|
||||
squared error) of the model on the given `data` and `responses`.
|
||||
|
||||
### Simple Examples
|
||||
|
||||
See also the [simple usage example](#simple-usage-example) for a trivial usage
|
||||
of the `BayesianLinearRegression` class.
|
||||
|
||||
---
|
||||
|
||||
Train a Bayesian linear regression model in the constructor on weighted data,
|
||||
compute the RMSE with `RMSE()`, and save the model.
|
||||
|
||||
```c++
|
||||
// See https://datasets.mlpack.org/admission_predict.csv.
|
||||
arma::mat data;
|
||||
mlpack::data::Load("admission_predict.csv", data, true);
|
||||
|
||||
// See https://datasets.mlpack.org/admission_predict.responses.csv.
|
||||
arma::rowvec responses;
|
||||
mlpack::data::Load("admission_predict.responses.csv", responses, true);
|
||||
|
||||
// Generate random instance weights for each point, in the range 0.5 to 1.5.
|
||||
arma::rowvec weights(data.n_cols, arma::fill::randu);
|
||||
weights += 0.5;
|
||||
|
||||
// Train Bayesian linear regression model. The data will be both centered and
|
||||
// scaled to have unit variance.
|
||||
mlpack::BayesianLinearRegression blr(data, responses, true, true);
|
||||
|
||||
// Now compute the RMSE on the training set.
|
||||
std::cout << "RMSE on the training set: " << blr.RMSE(data, responses)
|
||||
<< "." << std::endl;
|
||||
|
||||
// Finally, save the model with the name "blr".
|
||||
mlpack::data::Save("blr_model.bin", "blr", blr, true);
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
Load a saved Bayesian linear regression model and print some information about
|
||||
it, then make some predictions individually for random points.
|
||||
|
||||
```
|
||||
mlpack::BayesianLinearRegression blr;
|
||||
|
||||
// Load the model named "blr" from "lr_model.bin".
|
||||
mlpack::data::Load("blr_model.bin", "blr", blr, true);
|
||||
|
||||
// Print some information about the model.
|
||||
const size_t dimensionality = blr.Omega().n_elem;
|
||||
if (dimensionality == 0)
|
||||
{
|
||||
std::cout << "The model in `blr_model.bin` has not been trained."
|
||||
<< std::endl;
|
||||
return 0;
|
||||
}
|
||||
|
||||
std::cout << "Information on the BayesianLinearRegression model in "
|
||||
<< "'blr_model.bin':" << std::endl;
|
||||
std::cout << " - Data was centered when training: "
|
||||
<< (blr.CenterData() ? std::string("yes") : std::string("no")) << "."
|
||||
<< std::endl;
|
||||
std::cout << " - Data was scaled to unit variance when training: "
|
||||
<< (blr.ScaleData() ? std::string("yes") : std::string("no")) << "."
|
||||
<< std::endl;
|
||||
std::cout << " - Model intercept: " << blr.ResponsesOffset() << "."
|
||||
<< std::endl;
|
||||
std::cout << " - Precision of Gaussian prior: " << blr.Alpha() << "."
|
||||
<< std::endl;
|
||||
std::cout << " - Precision of model: " << blr.Beta() << "." << std::endl;
|
||||
|
||||
// Now make a prediction for three random points.
|
||||
for (size_t t = 0; t < 3; ++t)
|
||||
{
|
||||
arma::vec randomPoint(dimensionality, arma::fill::randu);
|
||||
double prediction, stddev;
|
||||
blr.Predict(randomPoint, prediction, stddev);
|
||||
|
||||
std::cout << "Prediction for random point " << t << ": " << prediction
|
||||
<< " +/- " << stddev << "." << std::endl;
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Advanced Functionality: Different Element Types
|
||||
|
||||
The `BayesianLinearRegression` class has one template parameter that can be used
|
||||
to control the element type of the model. The full signature of the class is:
|
||||
|
||||
```c++
|
||||
BayesianLinearRegression<ModelMatType>
|
||||
```
|
||||
|
||||
`ModelMatType` specifies the type of matrix used for the internal representation
|
||||
of model parameters. Any matrix type that implements the Armadillo API can be
|
||||
used; however, the matrix should be dense, as in general
|
||||
`BayesianLinearRegression` will produce models that are not sparse.
|
||||
|
||||
The example below trains a Bayesian linear regression model on 32-bit floating
|
||||
point data.
|
||||
|
||||
```c++
|
||||
// Create random, sparse 100-dimensional data.
|
||||
arma::fmat dataset(100, 5000, arma::fill::randu);
|
||||
|
||||
// Generate noisy responses from random data.
|
||||
arma::fvec trueWeights(100, arma::fill::randu);
|
||||
arma::frowvec responses = trueWeights.t() * dataset +
|
||||
0.01 * arma::randu<arma::frowvec>(5000) /* noise term */;
|
||||
|
||||
mlpack::BayesianLinearRegression<arma::fmat> blr;
|
||||
blr.ScaleData() = true;
|
||||
blr.MaxIterations() = 75;
|
||||
|
||||
blr.Train(dataset, responses);
|
||||
|
||||
// Compute the RMSE on the training set and a random test set.
|
||||
arma::fmat testDataset(100, 1000, arma::fill::randu);
|
||||
|
||||
arma::frowvec testResponses = trueWeights.t() * testDataset +
|
||||
0.01 * arma::randu<arma::frowvec>(1000) /* noise term */;
|
||||
|
||||
std::cout << "RMSE on training set: "
|
||||
<< blr.RMSE(dataset, responses) << "." << std::endl;
|
||||
std::cout << "RMSE on test set: "
|
||||
<< blr.RMSE(testDataset, testResponses) << "." << std::endl;
|
||||
```
|
||||
@@ -0,0 +1,739 @@
|
||||
## `HoeffdingTree`
|
||||
|
||||
The `HoeffdingTree` class implements a streaming (or incremental) decision tree
|
||||
classifier that supports numerical and categorical features, by default using
|
||||
Gini impurity to choose which feature to split on. The class offers several
|
||||
template parameters and several runtime options that can be used to control the
|
||||
behavior of the tree.
|
||||
|
||||
Hoeffding trees (also known as "Very Fast Decision Trees" or VFDTs) are useful
|
||||
for classifying points with _discrete labels_ (i.e. `0`, `1`, `2`).
|
||||
|
||||
#### Simple usage example:
|
||||
|
||||
```c++
|
||||
// Train a Hoeffding tree on random numeric data; predict labels on test data:
|
||||
|
||||
// All data and labels are uniform random; 10 dimensional data, 5 classes.
|
||||
// Replace with a data::Load() call or similar for a real application.
|
||||
arma::mat dataset(10, 1000, arma::fill::randu); // 1000 points.
|
||||
arma::Row<size_t> labels =
|
||||
arma::randi<arma::Row<size_t>>(1000, arma::distr_param(0, 4));
|
||||
arma::mat testDataset(10, 500, arma::fill::randu); // 500 test points.
|
||||
|
||||
mlpack::HoeffdingTree tree; // Step 1: create model.
|
||||
tree.Train(dataset, labels, 5); // Step 2a: train model (batch).
|
||||
tree.Train(dataset.col(0), labels[0]); // Step 2b: train model (incremental).
|
||||
arma::Row<size_t> predictions;
|
||||
tree.Classify(testDataset, predictions); // Step 3: classify points.
|
||||
|
||||
// Print some information about the test predictions.
|
||||
std::cout << arma::accu(predictions == 2) << " test points classified as class "
|
||||
<< "2." << std::endl;
|
||||
```
|
||||
<p style="text-align: center; font-size: 85%"><a href="#simple-examples">More examples...</a></p>
|
||||
|
||||
#### Quick links:
|
||||
|
||||
* [Constructors](#constructors): create `HoeffdingTree` objects.
|
||||
* [`Train()`](#training): train model.
|
||||
* [`Classify()`](#classification): classify with a trained model.
|
||||
* [Other functionality](#other-functionality) for loading, saving, and
|
||||
inspecting.
|
||||
* [Examples](#simple-examples) of simple usage and links to detailed example
|
||||
projects.
|
||||
* [Template parameters](#advanced-functionality-template-parameters) for custom
|
||||
behavior.
|
||||
|
||||
#### See also:
|
||||
|
||||
* [`DecisionTree`](#decision_tree) <!-- TODO: fix link! -->
|
||||
* [Random forests](#random_forests) <!-- TODO: fix link! -->
|
||||
* [mlpack classifiers](#mlpack_classifiers) <!-- TODO: fix link! -->
|
||||
* [Incremental decision tree on Wikipedia](https://en.wikipedia.org/wiki/Incremental_decision_tree)
|
||||
* [Mining High-Speed Data Streams (pdf)](https://dl.acm.org/doi/pdf/10.1145/347090.347107)
|
||||
|
||||
### Constructors
|
||||
|
||||
* `tree = HoeffdingTree()`
|
||||
- Initialize tree without training.
|
||||
- You will need to call the batch version of [`Train()`](#training) later to
|
||||
train the tree before calling [`Classify()`](#classification).
|
||||
|
||||
---
|
||||
|
||||
* `tree = HoeffdingTree(dimensionality, numClasses)`
|
||||
* `tree = HoeffdingTree(dimensionality, numClasses, successProbability=0.95, maxSamples=0, checkInterval=100, minSamples=100)`
|
||||
- Initialize tree for incremental training on numerical-only data.
|
||||
- The single-point [`Train()`](#training) function can be used to train
|
||||
incrementally.
|
||||
|
||||
---
|
||||
|
||||
* `tree = HoeffdingTree(datasetInfo, numClasses)`
|
||||
* `tree = HoeffdingTree(datasetInfo, numClasses, successProbability=0.95, maxSamples=0, checkInterval=100, minSamples=100)`
|
||||
- Initialize tree for incremental training on mixed categorical data.
|
||||
- The single-point [`Train()`](#training) function can be used to train
|
||||
incrementally.
|
||||
|
||||
---
|
||||
|
||||
* `tree = HoeffdingTree(data, labels, numClasses)`
|
||||
* `tree = HoeffdingTree(data, labels, numClasses, batchTraining=true, successProbability=0.95, maxSamples=0, checkInterval=100, minSamples=100)`
|
||||
- Train non-incrementally on the given data.
|
||||
- The tree will be reset if `numClasses` or the data's dimensionality does
|
||||
not match the current settings of the tree.
|
||||
|
||||
---
|
||||
|
||||
* `tree = HoeffdingTree(data, datasetInfo, labels, numClasses)
|
||||
* `tree = HoeffdingTree(data, datasetInfo, labels, numClasses, batchTraining=true, successProbability=0.95, maxSamples=0, checkInterval=100, minSamples=100)`
|
||||
- Train non-incrementally on mixed categorical data.
|
||||
- The tree will be reset if `numClasses` or `datasetInfo` does not match the
|
||||
current settings of the tree.
|
||||
|
||||
---
|
||||
|
||||
#### Constructor Parameters:
|
||||
|
||||
<!-- TODOs for table below:
|
||||
* better link for column-major matrices
|
||||
* better link for working with categorical data in straightforward terms
|
||||
* update matrices.md to include a section on labels and NormalizeLabels()
|
||||
* add a bit about instance weights in matrices.md
|
||||
-->
|
||||
|
||||
| **name** | **type** | **description** | **default** |
|
||||
|----------|----------|-----------------|-------------|
|
||||
| `data` | [`arma::mat`](../matrices.md) | [Column-major](../matrices.md) training matrix. | _(N/A)_ |
|
||||
| `datasetInfo` | [`data::DatasetInfo`](../../tutorials/datasetmapper.md) | Dataset information, specifying type information for each dimension. | _(N/A)_ |
|
||||
| `labels` | [`arma::Row<size_t>`]('../matrices.md') | Training labels, between `0` and `numClasses - 1` (inclusive). Should have length `data.n_cols`. | _(N/A)_ |
|
||||
| `dimensionality` | `size_t` | When using on numeric-only data, this specifies the number of dimensions in the data. | _(N/A)_ |
|
||||
| `numClasses` | `size_t` | Number of classes in the dataset. | _(N/A)_ |
|
||||
| `batchTraining` | `bool` | If `true`, a batch training algorithm is used, instead of the usual incremental algorithm. This is generally more efficient for larger datasets. | `true` |
|
||||
| `successProbability` | `double` | Probability of success required for Hoeffding bound before a node split can happen. | `0.95` |
|
||||
| `maxSamples` | `size_t` | Maximum number of samples before a node split is forced. `0` means no limit. | `0` |
|
||||
| `checkInterval` | `size_t` | Number of samples required before each split check. Higher values check less often, which is more efficient, but may not split a node as early as possible. | `100` |
|
||||
| `minSamples` | `size_t` | Minimum number of samples for a node to see before a split is allowed. | `100` |
|
||||
|
||||
As an alternative to passing hyperparameters, these can be set with a
|
||||
standalone method. The following functions can be used before calling
|
||||
`Train()`:
|
||||
|
||||
* `tree.SuccessProbability(successProbability);` will set the required success
|
||||
probability for splitting to `successProbability`.
|
||||
* `tree.MaxSamples(maxSamples);` will set the maximum number of samples before
|
||||
a split to `maxSamples`.
|
||||
* `tree.CheckInterval(checkInterval);` will set the number of samples between
|
||||
split checks to `checkInterval`.
|
||||
* `tree.MinSamples(minSamples);` will set the minimum number of samples before
|
||||
a split to `minSamples`.
|
||||
|
||||
***Notes:***
|
||||
|
||||
* Setting `successProbability` higher than the default means that the Hoeffding
|
||||
tree is less likely (and will take more samples) to split a node. This can
|
||||
result in a smaller tree.
|
||||
|
||||
* Different types can be used for `data` (e.g., `arma::fmat`, `arma::sp_mat`).
|
||||
See [template parameters](#advanced-functionality-template-parameters) for
|
||||
using different `NumericSplitType`s that accept different element types.
|
||||
|
||||
### Training
|
||||
|
||||
If training is not done as part of the constructor call, it can be done with one
|
||||
of the following versions of the `Train()` member function:
|
||||
|
||||
* `tree.Train(point, label)`
|
||||
- Streaming (incremental) training: train on a single data point.
|
||||
- The number of classes and dataset information must have already been
|
||||
specified by a previous constructor, `Train()`, or `Reset()` call.
|
||||
|
||||
---
|
||||
|
||||
* `tree.Train(data, labels)`
|
||||
* `tree.Train(data, labels, numClasses)`
|
||||
* `tree.Train(data, labels, numClasses, batchTraining=true, successProbability=0.95, maxSamples=0, checkInterval=100, minSamples=100)`
|
||||
- Train on the given data.
|
||||
- If the data is mixed categorical, then `datasetInfo` should have already
|
||||
been passed via a previous constructor, `Train()`, or `Reset()` call.
|
||||
- `numClasses` does not need to be specified if it has been specified in an
|
||||
earlier constructor or `Train()` call.
|
||||
|
||||
---
|
||||
|
||||
* `tree.Train(data, datasetInfo, labels)`
|
||||
* `tree.Train(data, datasetInfo, labels, numClasses)`
|
||||
* `tree.Train(data, datasetInfo, labels, numClasses, batchTraining=true, successProbability=0.95, maxSamples=0, checkInterval=100, minSamples=100)`
|
||||
- Train on mixed categorical data.
|
||||
- The previous overload (without `datasetInfo`) can be used instead if
|
||||
`datasetInfo` has already been passed in a previous constructor, `Train()`,
|
||||
or `Reset()` call, and has not changed.
|
||||
- `numClasses` does not need to be specified if it has been specified in an
|
||||
earlier constructor or `Train()` call.
|
||||
|
||||
---
|
||||
|
||||
Types of each argument are the same as in the table for constructors
|
||||
[above](#constructor-parameters).
|
||||
|
||||
***Notes***:
|
||||
|
||||
* Training is incremental. Successive calls to `Train()` will train the
|
||||
Hoeffding tree further. To reset the tree, call
|
||||
[`Reset()`](#other-functionality).
|
||||
|
||||
### Classification
|
||||
|
||||
Once a `DecisionTree` is trained, the `Classify()` member function can be used
|
||||
to make class predictions for new data.
|
||||
|
||||
* `size_t predictedClass = tree.Classify(point)`
|
||||
- ***(Single-point)***
|
||||
- Classify a single point, returning the predicted class.
|
||||
|
||||
---
|
||||
|
||||
* `tree.Classify(point, prediction, probability)`
|
||||
- ***(Single-point)***
|
||||
- Classify a single point and compute class probabilities.
|
||||
- The predicted class is stored in `prediction`.
|
||||
- The probability of class `i` is stored in `probability`.
|
||||
|
||||
---
|
||||
|
||||
* `tree.Classify(data, predictions)`
|
||||
- ***(Multi-point)***
|
||||
- Classify a set of points.
|
||||
- The prediction for data point `i` can be accessed with `predictions[i]`.
|
||||
|
||||
---
|
||||
|
||||
* `tree.Classify(data, predictions, probabilities)`
|
||||
- ***(Multi-point)***
|
||||
- Classify a set of points and compute class probabilities for each point.
|
||||
- The prediction for data point `i` can be accessed with `predictions[i]`.
|
||||
- The probability of class `predictions[i]` for data point `i` can be
|
||||
accessed with `probabilities[i]`.
|
||||
|
||||
---
|
||||
|
||||
#### Classification Parameters:
|
||||
|
||||
| **usage** | **name** | **type** | **description** |
|
||||
|-----------|----------|----------|-----------------|
|
||||
| _single-point_ | `point` | [`arma::vec`](../matrices.md) | Single point for classification. |
|
||||
| _single-point_ | `prediction` | `size_t&` | `size_t` to store class prediction into. |
|
||||
| _single-point_ | `probability` | `double&` | `double` to store predicted class probability into. |
|
||||
||||
|
||||
| _multi-point_ | `data` | [`arma::mat`](../matrices.md) | Set of [column-major](../matrices.md) points for classification. |
|
||||
| _multi-point_ | `predictions` | [`arma::Row<size_t>&`](../matrices.md) | Vector of `size_t`s to store class prediction into. Will be set to length `data.n_cols`. |
|
||||
| _multi-point_ | `probabilities` | [`arma::rowvec&`](../matrices.md) | Vector to store probability of predicted class in for each point. Will be set to length `data.n_cols`. |
|
||||
|
||||
***Note:*** different types can be used for `data` and `point` (e.g.
|
||||
`arma::fmat`, `arma::sp_mat`, `arma::sp_vec`, etc.). However, the element type
|
||||
that is used should be the same type that was used for training.
|
||||
|
||||
### Other Functionality
|
||||
|
||||
<!-- TODO: we should point directly to the documentation of those functions -->
|
||||
|
||||
* A `HoeffdingTree` can be serialized with [`data::Save()`](../formats.md) and
|
||||
[`data::Load()`](../formats.md).
|
||||
|
||||
* `tree.NumChildren()` will return a `size_t` indicating the number of children
|
||||
in the node `tree`.
|
||||
|
||||
* `tree.NumDescendants()` will return a `size_t` indicating the total number of
|
||||
descendant nodes of the tree.
|
||||
|
||||
* `tree.Child(i)` will return a `HoeffdingTree` object representing the `i`th
|
||||
child of the node `tree`.
|
||||
|
||||
* `tree.SplitDimension()` returns a `size_t` indicating which dimension the
|
||||
node `tree` splits on.
|
||||
|
||||
* `tree.NumSamples()` returns a `size_t` indicating the number of points seen
|
||||
so far by `tree`, if `tree` has not yet split. If `tree` has split (i.e. if
|
||||
`tree.NumChildren() > 0`), then what is returned is the number of points seen
|
||||
up until the split occurred.
|
||||
|
||||
* `tree.NumClasses()` returns a `size_t` indicating the number of classes the
|
||||
tree was trained on.
|
||||
|
||||
* `tree.Reset()` will reset the tree to an empty tree, and:
|
||||
- `tree.Reset()` will leave the number of classes and dataset information
|
||||
(e.g. `datasetInfo`) intact.
|
||||
- `tree.Reset(dimensionality, numClasses)` will set the number of classes to
|
||||
`numClasses` and set the dimensionality of the data to `dimensionality`,
|
||||
assuming all dimensions are numeric.
|
||||
- `tree.Reset(datasetInfo, numClasses)` will set the number of classes to
|
||||
`numClasses` and set the dataset information to `datasetInfo`.
|
||||
|
||||
For complete functionality, the [source
|
||||
code](/src/mlpack/methods/hoeffding_trees/hoeffding_tree.hpp) can be consulted.
|
||||
Each method is fully documented.
|
||||
|
||||
### Simple Examples
|
||||
|
||||
See also the [simple usage example](#simple-usage-example) for a trivial use of
|
||||
`HoeffdingTree`.
|
||||
|
||||
---
|
||||
|
||||
Train a Hoeffding tree incrementally on mixed categorical data:
|
||||
|
||||
```c++
|
||||
// Load a categorical dataset.
|
||||
arma::mat dataset;
|
||||
mlpack::data::DatasetInfo info;
|
||||
// See https://datasets.mlpack.org/covertype.train.arff.
|
||||
mlpack::data::Load("covertype.train.arff", dataset, info, true);
|
||||
|
||||
arma::Row<size_t> labels;
|
||||
// See https://datasets.mlpack.org/covertype.train.labels.csv.
|
||||
mlpack::data::Load("covertype.train.labels.csv", labels, true);
|
||||
|
||||
// Create the tree.
|
||||
mlpack::HoeffdingTree tree(info, 7 /* classes */);
|
||||
|
||||
// Train on each point in the given dataset.
|
||||
for (size_t i = 0; i < dataset.n_cols; ++i)
|
||||
tree.Train(dataset.col(i), labels[i]);
|
||||
|
||||
// Load categorical test data.
|
||||
arma::mat testDataset;
|
||||
// See https://datasets.mlpack.org/covertype.test.arff.
|
||||
mlpack::data::Load("covertype.test.arff", testDataset, info, true);
|
||||
|
||||
// Predict class of first test point.
|
||||
const size_t firstPrediction = tree.Classify(testDataset.col(0));
|
||||
std::cout << "First test point has predicted class " << firstPrediction << "."
|
||||
<< std::endl;
|
||||
|
||||
// Predict class and probabilities of second test point.
|
||||
size_t secondPrediction;
|
||||
double secondProbability;
|
||||
tree.Classify(testDataset.col(1), secondPrediction, secondProbability);
|
||||
std::cout << "Second test point has predicted class " << secondPrediction
|
||||
<< " with probability " << secondProbability << "." << std::endl;
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
Train a Hoeffding tree on blocks of a dataset, print accuracy measures on a test
|
||||
set during training, and save the model to disk.
|
||||
|
||||
```c++
|
||||
// Load a categorical dataset.
|
||||
arma::mat dataset;
|
||||
mlpack::data::DatasetInfo info;
|
||||
// See https://datasets.mlpack.org/covertype.train.arff.
|
||||
mlpack::data::Load("covertype.train.arff", dataset, info, true);
|
||||
|
||||
arma::Row<size_t> labels;
|
||||
// See https://datasets.mlpack.org/covertype.train.labels.csv.
|
||||
mlpack::data::Load("covertype.train.labels.csv", labels, true);
|
||||
|
||||
// Also load test data.
|
||||
|
||||
// See https://datasets.mlpack.org/covertype.test.arff.
|
||||
arma::mat testDataset;
|
||||
mlpack::data::Load("covertype.test.arff", testDataset, info, true);
|
||||
|
||||
// See https://datasets.mlpack.org/covertype.test.labels.arff.
|
||||
arma::Row<size_t> testLabels;
|
||||
mlpack::data::Load("covertype.test.labels.csv", testLabels, true);
|
||||
|
||||
// Create the tree with custom parameters.
|
||||
mlpack::HoeffdingTree tree(info, 7 /* number of classes */);
|
||||
tree.SuccessProbability(0.99);
|
||||
tree.CheckInterval(500);
|
||||
|
||||
// Now iterate over 10k-point chunks in the dataset.
|
||||
for (size_t start = 0; start < dataset.n_cols; start += 10000)
|
||||
{
|
||||
size_t end = std::min(start + 9999, (size_t) dataset.n_cols - 1);
|
||||
|
||||
tree.Train(dataset.cols(start, end), info, labels.subvec(start, end));
|
||||
|
||||
// Compute accuracy on the test set.
|
||||
arma::Row<size_t> predictions;
|
||||
tree.Classify(testDataset, predictions);
|
||||
const double accuracy = 100.0 * arma::accu(predictions == testLabels) /
|
||||
testLabels.n_elem;
|
||||
|
||||
std::cout << "Accuracy after " << (end + 1) << " points: " << accuracy
|
||||
<< "\%." << std::endl;
|
||||
}
|
||||
|
||||
// Save the fully trained tree in `tree.bin` with name `tree`.
|
||||
mlpack::data::Save("tree.bin", "tree", tree, true);
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
Load a tree and print some information about it.
|
||||
|
||||
```c++
|
||||
mlpack::HoeffdingTree tree;
|
||||
// This call assumes a tree called "tree" has already been saved to `tree.bin`
|
||||
// with `data::Save()`.
|
||||
mlpack::data::Load("tree.bin", "tree", tree, true);
|
||||
|
||||
if (tree.NumChildren() > 0)
|
||||
{
|
||||
std::cout << "The split dimension of the root node of the tree in `tree.bin` "
|
||||
<< "is dimension " << tree.SplitDimension() << "." << std::endl;
|
||||
}
|
||||
else
|
||||
{
|
||||
std::cout << "The tree in `tree.bin` is a leaf (it has no children)."
|
||||
<< std::endl;
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
Train a tree, reset a tree, and train again.
|
||||
|
||||
```c++
|
||||
// See the following files:
|
||||
// - https://datasets.mlpack.org/covertype.train.arff.
|
||||
// - https://datasets.mlpack.org/covertype.train.labels.csv.
|
||||
// - https://datasets.mlpack.org/covertype.test.arff.
|
||||
// - https://datasets.mlpack.org/covertype.test.labels.arff.
|
||||
|
||||
arma::mat dataset, testDataset;
|
||||
arma::Row<size_t> labels, testLabels;
|
||||
mlpack::data::DatasetInfo info;
|
||||
|
||||
mlpack::data::Load("covertype.train.arff", dataset, info, true);
|
||||
mlpack::data::Load("covertype.train.labels.csv", labels, true);
|
||||
mlpack::data::Load("covertype.test.arff", testDataset, info, true);
|
||||
mlpack::data::Load("covertype.test.labels.csv", testLabels, true);
|
||||
|
||||
// Create a tree, and train on the training data.
|
||||
mlpack::HoeffdingTree tree(info, 7 /* number of classes */, 0.98);
|
||||
tree.MinSamples(500);
|
||||
tree.CheckInterval(500);
|
||||
|
||||
tree.Train(dataset, labels);
|
||||
|
||||
// Print accuracy on the training and test set.
|
||||
arma::Row<size_t> predictions, testPredictions;
|
||||
tree.Classify(dataset, predictions);
|
||||
tree.Classify(testDataset, testPredictions);
|
||||
|
||||
double trainAcc = (100.0 * arma::accu(predictions == labels)) / labels.n_elem;
|
||||
double testAcc = (100.0 * arma::accu(testPredictions == testLabels)) /
|
||||
testLabels.n_elem;
|
||||
|
||||
std::cout << "When trained on the training data:" << std::endl;
|
||||
std::cout << " - Training set accuracy: " << trainAcc << "\%." << std::endl;
|
||||
std::cout << " - Test set accuracy: " << testAcc << "\%." << std::endl;
|
||||
|
||||
// Now reset the tree, and train on the test set instead.
|
||||
// The dataset info and number of classes has not changed, so we can just call
|
||||
// Reset() with no arguments.
|
||||
tree.Reset();
|
||||
tree.Train(testDataset, testLabels);
|
||||
|
||||
// Print accuracy on the training and test set, now that we have trained on the
|
||||
// test set.
|
||||
tree.Classify(dataset, predictions);
|
||||
tree.Classify(testDataset, testPredictions);
|
||||
|
||||
trainAcc = (100.0 * arma::accu(predictions == labels)) / labels.n_elem;
|
||||
testAcc = (100.0 * arma::accu(testPredictions == testLabels)) /
|
||||
testLabels.n_elem;
|
||||
|
||||
std::cout << "When trained on the test data:" << std::endl;
|
||||
std::cout << " - Training set accuracy: " << trainAcc << "\%." << std::endl;
|
||||
std::cout << " - Test set accuracy: " << testAcc << "\%." << std::endl;
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Advanced Functionality: Template Parameters
|
||||
|
||||
#### Using different element types.
|
||||
|
||||
`HoeffdingTree`'s constructors, `Train()`, and `Classify()` functions support
|
||||
any data type, so long as it supports the Armadillo matrix API. So, for
|
||||
instance, learning can be done on single-precision floating-point data:
|
||||
|
||||
```c++
|
||||
// 1000 random points in 10 dimensions.
|
||||
arma::fmat dataset(10, 1000, arma::fill::randu);
|
||||
// Random labels for each point, totaling 5 classes.
|
||||
arma::Row<size_t> labels =
|
||||
arma::randi<arma::Row<size_t>>(1000, arma::distr_param(0, 4));
|
||||
|
||||
// Train in the constructor.
|
||||
mlpack::HoeffdingTree tree(dataset, labels, 5);
|
||||
|
||||
// Create test data (500 points).
|
||||
arma::fmat testDataset(10, 500, arma::fill::randu);
|
||||
arma::Row<size_t> predictions;
|
||||
tree.Classify(testDataset, predictions);
|
||||
// Now `predictions` holds predictions for the test dataset.
|
||||
|
||||
// Print some information about the test predictions.
|
||||
std::cout << arma::accu(predictions == 2) << " test points classified as class "
|
||||
<< "2." << std::endl;
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
#### Fully custom behavior.
|
||||
|
||||
The `HoeffdingTree` class also supports several template parameters, which can
|
||||
be used for custom behavior during learning. The full signature of the class is
|
||||
as follows:
|
||||
|
||||
```c++
|
||||
HoeffdingTree<FitnessFunction,
|
||||
NumericSplitType,
|
||||
CategoricalSplitType>
|
||||
```
|
||||
|
||||
* `FitnessFunction`: the measure of goodness to use when deciding on tree
|
||||
splits
|
||||
* `NumericSplitType`: the strategy used for finding splits on numeric data
|
||||
dimensions
|
||||
* `CategoricalSplitType`: the strategy used for finding splits on categorical
|
||||
data dimensions
|
||||
|
||||
Below, details are given for the requirements of each of these template types.
|
||||
|
||||
---
|
||||
|
||||
#### `FitnessFunction`
|
||||
|
||||
* Specifies the fitness function to use when learning a decision tree.
|
||||
* The `GiniImpurity` _(default)_ and `HoeffdingInformationGain` classes are
|
||||
available for drop-in usage.
|
||||
* `GiniImpurity` uses the [Gini impurity](https://en.wikipedia.org/wiki/Decision_tree_learning#Gini_impurity),
|
||||
which measures the probability of a randomly labeled random element in the
|
||||
split nodes being correctly labeled.
|
||||
* `HoeffdingInformationGain` uses the [information gain](https://en.wikipedia.org/wiki/Decision_tree_learning#Information_gain),
|
||||
which is based on the information-theoretic entropy of the possible splits.
|
||||
* A custom class must implement two functions:
|
||||
|
||||
```c++
|
||||
// You can use this as a starting point for implementation.
|
||||
class CustomFitnessFunction
|
||||
{
|
||||
// Return the range (difference between maximum and minimum gain values).
|
||||
double Range(const size_t numClasses);
|
||||
|
||||
// Compute the gain for the given split candidates represented in the matrix
|
||||
// `counts`. `counts` is a matrix with `numChildren` columns and `numClasses`
|
||||
// rows, containing the number of points for each class held by each child.
|
||||
//
|
||||
// Note that the gain returned should be the gain for *all* child nodes (e.g.
|
||||
// all columns of `counts`).
|
||||
double Evaluate(const arma::Mat<size_t>& counts);
|
||||
};
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
#### `NumericSplitType`
|
||||
|
||||
* Specifies the strategy to be used during training when splitting a numeric
|
||||
feature.
|
||||
|
||||
* Several options are already implemented and available for drop-in usage.
|
||||
- The `HoeffdingDoubleNumericSplit` _(default)_ class discretizes the given
|
||||
numeric data into a default of 10 bins. This expects `double` to be the
|
||||
type of the input data.
|
||||
- The `HoeffdingFloatNumericSplit` class operates similarly to
|
||||
`HoeffdingDoubleNumericSplit`, but expects `float` to be the type of the
|
||||
input data.
|
||||
- The `BinaryNumericSplit` class splits numeric features in two in the way
|
||||
that maximizes gain. This split type is more computationally expensive
|
||||
during training.
|
||||
|
||||
* If a non-default `NumericSplitType` is specified, the following constructor
|
||||
forms can be used to pass constructed `NumericSplitType`s to the
|
||||
`HoeffdingTree` to use as copy-constructed templates during splitting:
|
||||
- `HoeffdingTree(dimensionality, numClasses, successProbability, maxSamples, checkInterval, minSamples, categoricalSplit, numericSplit)`
|
||||
- `HoeffdingTree(datasetInfo, numClasses, successProbability, maxSamples, checkInterval, minSamples, categoricalSplit, numericSplit)`
|
||||
- `HoeffdingTree(data, labels, numClasses, successProbability, maxSamples, checkInterval, minSamples, categoricalSplit, numericSplit)`
|
||||
- `HoeffdingTree(data, datasetInfo, labels, numClasses, successProbability, maxSamples, checkInterval, minSamples, categoricalSplit, numericSplit)`
|
||||
|
||||
* A custom class must take a [`FitnessFunction`](#fitness-function) as a
|
||||
template parameter, implement several functions, and have an internal
|
||||
structure `SplitInfo` that is used at classification time:
|
||||
|
||||
```c++
|
||||
// The job of this class is to track sufficient statistics of training data,
|
||||
// returning gain information if a split were to happen according to this
|
||||
// class's split strategy.
|
||||
//
|
||||
// For details, consult the HoeffdingNumericSplit and BinaryNumericSplit class
|
||||
// implementations.
|
||||
template<typename FitnessFunction>
|
||||
class CustomNumericSplit
|
||||
{
|
||||
public:
|
||||
// Create the split object with the given number of classes.
|
||||
CustomNumericSplit(const size_t numClasses);
|
||||
|
||||
// Create the split from another split object with the given number of
|
||||
// classes.
|
||||
CustomNumericSplit(const size_t numClasses, const CustomNumericSplit& other);
|
||||
|
||||
// Train on the given value with the given label.
|
||||
// Note that the type used here must match the element type of the training
|
||||
// data (so, e.g., if you plan to use `arma::fmat`, use `float` instead of
|
||||
// `double`).
|
||||
void Train(double value, const size_t label);
|
||||
|
||||
// Given the points seen so far, evaluate the fitness function, returning the
|
||||
// gain if a split were to occur. If this `NumericSplitType` class could
|
||||
// provide multiple possible splits, also return the second best fitness
|
||||
// value. (If not, set secondBestFitness to 0.)
|
||||
void EvaluateFitnessFunction(double& bestFitness, double& secondBestFitness);
|
||||
|
||||
// Return the number of children that would be created if a split were to
|
||||
// occur. (For example, if this class implements a binary split, this should
|
||||
// return 2.)
|
||||
size_t NumChildren() const;
|
||||
|
||||
// Given that a split should happen, return the majority classes of the
|
||||
// children and an initialized SplitInfo object.
|
||||
//
|
||||
// childMajorities should be set to have length equal to the number of
|
||||
// children that this strategy splits into, and the i'th element should be the
|
||||
// majority class label of the i'th child after splitting.
|
||||
void Split(arma::Col<size_t>& childMajorities, SplitInfo& splitInfo);
|
||||
|
||||
// Return the current majority class of points seen so far.
|
||||
size_t MajorityClass() const;
|
||||
// Return the probability of the majority class given the points seen so far.
|
||||
double MajorityProbability() const;
|
||||
|
||||
// Serialize (load/save) the split object using cereal.
|
||||
template<typename Archive>
|
||||
void serialize(Archive& ar, const uint32_t version);
|
||||
|
||||
// The SplitInfo class should implement two functions. It is used at
|
||||
// prediction time, after a split has occurred, and should contain the
|
||||
// information necessary to classify a point.
|
||||
//
|
||||
// The SplitInfo class must implement two methods; one for classification and
|
||||
// one for serialization.
|
||||
class SplitInfo
|
||||
{
|
||||
public:
|
||||
// Given that the point in the split dimension has the value `value`, return
|
||||
// the index of the child that the traversal should go to.
|
||||
template<typename eT>
|
||||
size_t CalculateDirection(const eT& value) const;
|
||||
|
||||
// Serialize the split (load/save) using cereal.
|
||||
template<typename Archive>
|
||||
void serialize(Archive& ar, const uint32_t version);
|
||||
};
|
||||
};
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
#### `CategoricalSplitType`
|
||||
|
||||
* Specifies the strategy to be used during training when splitting a
|
||||
categorical feature.
|
||||
|
||||
* The `HoeffdingCategoricalSplit` _(default)_ is available for drop-in usage
|
||||
and splits all categories into their own node.
|
||||
|
||||
* If a non-default `CategoricalSplitType` is specified, the following
|
||||
constructor forms can be used to pass constructed `CategoricalSplitType`s to
|
||||
the `HoeffdingTree` to use as copy-constructed templates during splitting:
|
||||
- `HoeffdingTree(dimensionality, numClasses, successProbability, maxSamples, checkInterval, minSamples, categoricalSplit, numericSplit)`
|
||||
- `HoeffdingTree(datasetInfo, numClasses, successProbability, maxSamples, checkInterval, minSamples, categoricalSplit, numericSplit)`
|
||||
- `HoeffdingTree(data, labels, numClasses, successProbability, maxSamples, checkInterval, minSamples, categoricalSplit, numericSplit)`
|
||||
- `HoeffdingTree(data, datasetInfo, labels, numClasses, successProbability, maxSamples, checkInterval, minSamples, categoricalSplit, numericSplit)`
|
||||
|
||||
* A custom class must take a [`FitnessFunction`](#fitness-function) as a
|
||||
template parameter, implement several functions, and have an internal
|
||||
structure `SplitInfo` that is used at classification time:
|
||||
|
||||
```c++
|
||||
// The job of this class is to track sufficient statistics of training data,
|
||||
// returning gain information if a split were to happen according to this
|
||||
// class's split strategy.
|
||||
//
|
||||
// For details, consult the HoeffdingCategoricalSplit class implementation.
|
||||
template<typename FitnessFunction>
|
||||
class CustomCategoricalSplit
|
||||
{
|
||||
public:
|
||||
// Create the split object with the given number of classes. The dimension
|
||||
// that this object tracks has `numCategories` possible category values.
|
||||
CustomCategoricalSplit(const size_t numCategories, const size_t numClasses);
|
||||
|
||||
// Create the split object from another split object with the given number of
|
||||
// classes. The dimension that this object tracks has `numCategories`
|
||||
// possible category values.
|
||||
CustomCategoricalSplit(const size_t numCategories, const size_t numClasses,
|
||||
const CustomCategoricalSplit& other);
|
||||
|
||||
// Train on the given value with the given label.
|
||||
// Note that the type used here must match the element type of the training
|
||||
// data (so, e.g., if you plan to use `arma::fmat`, use `float` instead of
|
||||
// `double`).
|
||||
void Train(double value, const size_t label);
|
||||
|
||||
// Given the points seen so far, evaluate the fitness function, returning the
|
||||
// gain if a split were to occur. If this `NumericSplitType` class could
|
||||
// provide multiple possible splits, also return the second best fitness
|
||||
// value. (If not, set secondBestFitness to 0.)
|
||||
void EvaluateFitnessFunction(double& bestFitness, double& secondBestFitness);
|
||||
|
||||
// Return the number of children that would be created if a split were to
|
||||
// occur. (For example, if this class implements a binary split, this should
|
||||
// return 2.)
|
||||
size_t NumChildren() const;
|
||||
|
||||
// Given that a split should happen, return the majority classes of the
|
||||
// children and an initialized SplitInfo object.
|
||||
//
|
||||
// childMajorities should be set to have length equal to the number of
|
||||
// children that this strategy splits into, and the i'th element should be the
|
||||
// majority class label of the i'th child after splitting.
|
||||
void Split(arma::Col<size_t>& childMajorities, SplitInfo& splitInfo);
|
||||
|
||||
// Return the current majority class of points seen so far.
|
||||
size_t MajorityClass() const;
|
||||
// Return the probability of the majority class given the points seen so far.
|
||||
double MajorityProbability() const;
|
||||
|
||||
// Serialize (load/save) the split object using cereal.
|
||||
template<typename Archive>
|
||||
void serialize(Archive& ar, const uint32_t version);
|
||||
|
||||
// The SplitInfo class should implement two functions. It is used at
|
||||
// prediction time, after a split has occurred, and should contain the
|
||||
// information necessary to classify a point.
|
||||
//
|
||||
// The SplitInfo class must implement two methods; one for classification and
|
||||
// one for serialization.
|
||||
class SplitInfo
|
||||
{
|
||||
public:
|
||||
// Given that the point in the split dimension has the value `value`, return
|
||||
// the index of the child that the traversal should go to.
|
||||
template<typename eT>
|
||||
size_t CalculateDirection(const eT& value) const;
|
||||
|
||||
// Serialize the split (load/save) using cereal.
|
||||
template<typename Archive>
|
||||
void serialize(Archive& ar, const uint32_t version);
|
||||
};
|
||||
};
|
||||
```
|
||||
@@ -0,0 +1,473 @@
|
||||
## `LARS`
|
||||
|
||||
The `LARS` class implements the least-angle regression (LARS) algorithm for
|
||||
L1-penalized and L2-penalized linear regression. `LARS` can also solve the
|
||||
LASSO (least absolute shrinkage and selection operator) problem. The LARS
|
||||
algorithm is a *path* algorithm, and thus will recover solutions for *all* L1
|
||||
penalty parameters greater than or equal to the given L1 penalty parameter.
|
||||
|
||||
#### Simple usage example:
|
||||
|
||||
```c++
|
||||
// Train a LARS model on random numeric data and make predictions.
|
||||
|
||||
// All data and responses are uniform random; this uses 10 dimensional data.
|
||||
// Replace with a data::Load() call or similar for a real application.
|
||||
arma::mat dataset(10, 1000, arma::fill::randu); // 1000 points.
|
||||
arma::rowvec responses = arma::randn<arma::rowvec>(1000);
|
||||
arma::mat testDataset(10, 500, arma::fill::randu); // 500 test points.
|
||||
|
||||
mlpack::LARS lars(true, 0.1 /* L1 penalty */); // Step 1: create model.
|
||||
lars.Train(dataset, responses); // Step 2: train model.
|
||||
arma::rowvec predictions;
|
||||
lars.Predict(testDataset, predictions); // Step 3: use model to predict.
|
||||
|
||||
// Print some information about the test predictions.
|
||||
std::cout << arma::accu(predictions > 0.7) << " test points predicted to have"
|
||||
<< " responses greater than 0.7." << std::endl;
|
||||
std::cout << arma::accu(predictions < 0) << " test points predicted to have "
|
||||
<< "negative responses." << std::endl;
|
||||
```
|
||||
<p style="text-align: center; font-size: 85%"><a href="#simple-examples">More examples...</a></p>
|
||||
|
||||
#### Quick links:
|
||||
|
||||
* [Constructors](#constructors): create `LARS` objects.
|
||||
* [`Train()`](#training): train model.
|
||||
* [`Predict()`](#prediction): predict with a trained model.
|
||||
* [Other functionality](#other-functionality) for loading, saving, and
|
||||
inspecting.
|
||||
* [The LARS path](#the-lars-path): use models from the LARS path with different
|
||||
L1 penalty values.
|
||||
* [Examples](#simple-examples) of simple usage and links to detailed example
|
||||
projects.
|
||||
* [Template parameters](#advanced-functionality-different-element-types) for
|
||||
using different element types for a model.
|
||||
|
||||
#### See also:
|
||||
|
||||
* [`LinearRegression`](#linear_regression) <!-- TODO: fix link! -->
|
||||
* [mlpack regression techniques](#mlpack_regression_techniques) <!-- TODO: fix link! -->
|
||||
* [Least-angle Regression on Wikipedia](https://en.wikipedia.org/wiki/Least-angle_regression)
|
||||
|
||||
### Constructors
|
||||
|
||||
* `lars = LARS(useCholesky=false, lambda1=0.0, lambda2=0.0, tolerance=1e-16, fitIntercept=true, normalizeData=true)`
|
||||
- Initialize the model without training.
|
||||
- You will need to call [`Train()`](#training) later to train the model
|
||||
before calling [`Predict()`](#prediction).
|
||||
|
||||
---
|
||||
|
||||
* `lars = LARS(data, responses, colMajor=true, useCholesky=true, lambda1=0.0, lambda2=0.0, tolerance=1e-16, fitIntercept=true, normalizeData=true)`
|
||||
- Train model on the given data and responses, using the given settings for
|
||||
hyperparameters.
|
||||
|
||||
---
|
||||
|
||||
* `lars = LARS(data, responses, colMajor, useCholesky, gramMatrix, lambda1=0.0, lambda2=0.0, tolerance=1e-16, fitIntercept=true, normalizeData=true)`
|
||||
- *(Advanced constructor)*.
|
||||
- Train model on the given data and responses, using a precomputed Gram
|
||||
matrix (`gramMatrix`, equivalent to `data * data.t()`).
|
||||
- Using a precomputed Gram matrix can save time, if it has already been
|
||||
computed.
|
||||
- ***Note:*** any precomputed Gram matrix must also match the settings of
|
||||
`fitIntercept` and `normalizeData`; so, if both are `true`, then
|
||||
`gramMatrix` must be computed on mean-centered data whose features are
|
||||
normalized to have unit variance. In addition, if `lambda2 > 0`, then
|
||||
it is expected that `lambda2` is added to each element on the diagonal of
|
||||
`gramMatrix`.
|
||||
|
||||
---
|
||||
|
||||
#### Constructor Parameters:
|
||||
|
||||
<!-- TODOs for table below:
|
||||
* better link for column-major matrices
|
||||
* update matrices.md to include a section on labels and NormalizeLabels()
|
||||
-->
|
||||
|
||||
| **name** | **type** | **description** | **default** |
|
||||
|----------|----------|-----------------|-------------|
|
||||
| `data` | [`arma::mat`](../matrices.md) | Training matrix. | _(N/A)_ |
|
||||
| `responses` | [`arma::rowvec`](../matrices.md) | Training responses (e.g. values to predict). Should have length `data.n_cols`. | _(N/A)_ |
|
||||
| `colMajor` | `bool` | Should be set to `true` if `data` is [column-major](../matrices.md). Passing row-major data can avoid a transpose operation. | `false` |
|
||||
| `useCholesky` | `bool` | If `true`, use the Cholesky decomposition of the Gram matrix to solve linear systems (as opposed to the full Gram matrix). | `false` |
|
||||
| `gramMatrix` | [`arma::mat`](../matrices.md) | Precomputed Gram matrix of `data` (i.e. `data * data.t()` for column-major data). | _(N/A)_ |
|
||||
| `lambda1` | `double` | L1 regularization penalty parameter. | `0.0` |
|
||||
| `lambda2` | `double` | L2 regularization penalty parameter. | `0.0` |
|
||||
| `tolerance` | `double` | Tolerance on feature correlations for convergence. | `1e-16` |
|
||||
| `fitIntercept` | `bool` | If `true`, an intercept term will be included in the model. | `true` |
|
||||
| `normalizeData` | `bool` | If `true`, data will be normalized before fitting the model. | `true` |
|
||||
|
||||
As an alternative to passing hyperparameters, each hyperparameter can be set
|
||||
with a standalone method. The following functions can be used before calling
|
||||
`Train()` to set hyperparameters:
|
||||
|
||||
* `lars.UseCholesky() = useChol;` will set whether or not the Cholesky
|
||||
decomposition will be used during training to `useChol`.
|
||||
* `lars.Lambda1() = lambda1;` will set the L1 regularization penalty parameter
|
||||
to `lambda1`.
|
||||
* `lars.Lambda2() = lambda2;` will set the L2 regularization penalty parameter
|
||||
to `lambda2`.
|
||||
* `lars.Tolerance() = tol;` will set the convergence tolerance to `tol`.
|
||||
* `lars.FitIntercept(fitIntercept);` will set whether an intercept will be fit
|
||||
to `fitIntercept`. If an external Gram matrix has been specified, this will
|
||||
throw an exception.
|
||||
* `lars.NormalizeData(normalizeData);` will set whether data should be
|
||||
normalized to `normalizeData`. If an external Gram matrix has been
|
||||
specified, this will throw an exception.
|
||||
|
||||
***Notes:***
|
||||
|
||||
- The `lambda1` parameter implicitly controls the sparsity of the model; for
|
||||
more sparse models (i.e. fewer nonzero weights), specify a larger `lambda1`.
|
||||
|
||||
- Specifying a too-small `lambda1` or `lambda2` value may cause the model to
|
||||
overfit; however, setting it too large may cause the model to underfit.
|
||||
Because LARS is a path algorithm, the
|
||||
[`SelectBeta()`](#other_functionality) functions can be used to select models
|
||||
with different values of `lambda1`. For tuning `lambda2`,
|
||||
[Automatic hyperparameter tuning](#hyperparameter-tuner) can be used.
|
||||
|
||||
- `fitIntercept` and `normalizeData` are recommended to be set as `true`, in
|
||||
accordance with the original LARS algorithm. `false` can be used for
|
||||
`fitIntercept` if the features and responses are already mean-centered, and
|
||||
`false` can also be used for `normalizeData` if the features are already
|
||||
unit-variance. Using `false` for either option can provide a small amount of
|
||||
speedup.
|
||||
|
||||
- `useCholesky` should generally be set to `true` and in most situations will
|
||||
result in faster training.
|
||||
|
||||
### Training
|
||||
|
||||
If training is not done as part of the constructor call, it can be done with the
|
||||
`Train()` function:
|
||||
|
||||
<!-- TODO: deprecate beta version -->
|
||||
<!-- TODO: implement hyperparameters versions -->
|
||||
|
||||
* `lars.Train(data, responses, colMajor=true, useCholesky=true, lambda1=0.0, lambda2=0.0, tolerance=1e-16, fitIntercept=true, normalizeData=true)`
|
||||
- Train the model on the given data.
|
||||
|
||||
---
|
||||
|
||||
* `lars.Train(data, responses, colMajor, useCholesky, gramMatrix, lambda1=0.0, lambda2=0.0, tolerance=1e-16, fitIntercept=true, normalizeData=true)`
|
||||
- *(Advanced training.)*
|
||||
- Train model on the given data and responses, using a precomputed Gram
|
||||
matrix (`gramMatrix`, equivalent to `data * data.t()`).
|
||||
- Using a precomputed Gram matrix can save time, if it has already been
|
||||
computed.
|
||||
- ***Note:*** any precomputed Gram matrix must also match the settings of
|
||||
`fitIntercept` and `normalizeData`; so, if both are `true`, then
|
||||
`gramMatrix` must be computed on mean-centered data whose features are
|
||||
normalized to have unit variance. In addition, if `lambda2 > 0`, then
|
||||
it is expected that `lambda2` is added to each element on the diagonal of
|
||||
`gramMatrix`.
|
||||
|
||||
---
|
||||
|
||||
Types of each argument are the same as in the table for constructors
|
||||
[above](#constructor-parameters).
|
||||
|
||||
***Notes:***
|
||||
|
||||
* Training is not incremental. A second call to `Train()` will retrain the
|
||||
model from scratch.
|
||||
|
||||
* `Train()` returns the squared error (loss) of the model on the training set
|
||||
as a `double`. To obtain the MSE, divide by the number of training points.
|
||||
|
||||
### Prediction
|
||||
|
||||
Once a `LARS` model is trained, the `Predict()` member function
|
||||
can be used to make predictions for new data.
|
||||
|
||||
<!-- TODO: implement single-point version -->
|
||||
|
||||
* `double predictedValue = lars.Predict(point)`
|
||||
- ***(Single-point)***
|
||||
- Make a prediction for a single point, returning the predicted value.
|
||||
|
||||
---
|
||||
|
||||
* `lars.Predict(data, predictions, colMajor=true)`
|
||||
- ***(Multi-point)***
|
||||
- Make predictions for a set of points.
|
||||
- The prediction for data point `i` can be accessed with `predictions[i]`.
|
||||
|
||||
---
|
||||
|
||||
#### Prediction Parameters:
|
||||
|
||||
| **usage** | **name** | **type** | **description** |
|
||||
|-----------|----------|----------|-----------------|
|
||||
| _single-point_ | `point` | [`arma::vec`](../matrices.md) | Single point for prediction. |
|
||||
||||
|
||||
| _multi-point_ | `data` | [`arma::mat`](../matrices.md) | Set of [column-major](../matrices.md) points for classification. |
|
||||
| _multi-point_ | `predictions` | [`arma::rowvec&`](../matrices.md) | Vector of `double`s to store predictions into. Will be set to length `data.n_cols`. |
|
||||
| _multi-point_ | `colMajor` | `bool` | Should be set to `true` if `data` is [column-major](../matrices.md). Passing row-major data can avoid a transpose operation. (Default `true`.) |
|
||||
|
||||
### Other Functionality
|
||||
|
||||
<!-- TODO: we should point directly to the documentation of those functions -->
|
||||
|
||||
* A `LARS` model can be serialized with
|
||||
[`data::Save()`](../formats.md) and [`data::Load()`](../formats.md).
|
||||
|
||||
* `lars.Beta()` will return an `arma::vec` with the model parameters. This
|
||||
will have length equal to the dimensionality of the model. Note that
|
||||
`lars.Beta()` can be changed to a different model on the LARS path using the
|
||||
[`lars.SelectBeta()` method](#the-lars-path).
|
||||
|
||||
* `lars.Intercept()` will return a `double` representing the fitted intercept
|
||||
term, or 0 if `lars.FitIntercept()` is `false`.
|
||||
|
||||
* `lars.ActiveSet()` will return a `std::vector<size_t>&` containing the
|
||||
indices of nonzero dimensions in the model parameters (`lars.Beta()`).
|
||||
|
||||
* `lars.ComputeError(data, responses, colMajor=true)` will return a `double`
|
||||
containing the squared error of the model on `data`, given that
|
||||
the true responses are `responses`. To obtain the MSE, divide by the number
|
||||
of points in `data`.
|
||||
|
||||
### The LARS Path
|
||||
|
||||
LARS is a *path* (or stepwise) algorithm, meaning it adds one feature at a time
|
||||
to the model. This in turn means that when we train a LARS model with
|
||||
`lambda1` set to `l`, we also recover every possible LARS model on the same data
|
||||
with a `lambda1` greater than `l`.
|
||||
|
||||
The `LARS` class provides a way to access all of the models on the path, and
|
||||
switch between them for prediction purposes:
|
||||
|
||||
* `lars.BetaPath()` returns a `std::vector<arma::vec>&` containing each set of
|
||||
model weights on the LARS path.
|
||||
|
||||
* `lars.InterceptPath()` returns a `std::vector<double>&` containing each
|
||||
intercept value on the LARS path. These values are only meaningful if
|
||||
`lars.FitIntercept()` is `true`.
|
||||
|
||||
* `lars.LambdaPath()` returns a `std::vector<double>&` containing each
|
||||
`lambda1` value that is associated with each element in `lars.BetaPath()` and
|
||||
`lars.InterceptPath()`. That is, `lars.LambdaPath()[i]` is the `lambda1`
|
||||
value corresponding to the model defined by `lars.BetaPath()[i]` and
|
||||
`lars.InterceptPath()[i]`.
|
||||
|
||||
<!-- TODO: implement -->
|
||||
* `lars.SelectBeta(lambda1)` will set the model weights (`lars.ActiveSet()`,
|
||||
`lars.Beta()` and `lars.Intercept()`) to the path location with L1 penalty
|
||||
`lambda1`. This is equivalent to calling `lars.Train(data, responses,
|
||||
colMajor, useCholesky, lambda1)`---but much more efficient! `lambda1`
|
||||
cannot be less than `lars.Lambda1()`, or an exception will be thrown.
|
||||
|
||||
<!-- TODO: implement -->
|
||||
* `lars.SelectedLambda1()` returns the currently selected L1 regularization
|
||||
penalty parameter.
|
||||
|
||||
* For any value `lambda1` between `lars.LambdaPath()[i]` and
|
||||
`lars.LambdaPath()[i + 1]`, the corresponding model is a linear interpolation
|
||||
between `lars.BetaPath()[i]` and `lars.BetaPath()[i + 1]` (and
|
||||
`lars.InterceptPath()[i]` and `lars.InterceptPath()[i + 1]`). This exact
|
||||
linear interpolation is what is computed by `lars.SelectBeta(lambda1)`.
|
||||
|
||||
### Simple Examples
|
||||
|
||||
See also the [simple usage example](#simple-usage-example) for a trivial usage
|
||||
of the `LARS` class.
|
||||
|
||||
---
|
||||
|
||||
Train a LARS model in the constructor, and print the MSE on training and
|
||||
test data for each set of weights in the path.
|
||||
|
||||
```c++
|
||||
// See https://datasets.mlpack.org/wave_energy_farm_100.csv.
|
||||
arma::mat data;
|
||||
mlpack::data::Load("wave_energy_farm_100.csv", data, true);
|
||||
|
||||
// Split the last row off: it is the responses. Also, normalize the responses
|
||||
// to [0, 1].
|
||||
arma::rowvec responses = data.row(data.n_rows - 1);
|
||||
responses /= responses.max();
|
||||
data.shed_row(data.n_rows - 1);
|
||||
|
||||
// Split into a training and test dataset. 20% of the data is held out as a
|
||||
// test set.
|
||||
arma::mat trainingData, testData;
|
||||
arma::rowvec trainingResponses, testResponses;
|
||||
mlpack::data::Split(data, responses, trainingData, testData, trainingResponses,
|
||||
testResponses, 0.2);
|
||||
|
||||
// Train a LARS model with lambda1 = 1e-5 and lambda2 = 1e-6.
|
||||
mlpack::LARS lars(trainingData, trainingResponses, true, true, 1e-5, 1e-6);
|
||||
|
||||
// Iterate over all the models in the path.
|
||||
const size_t pathLength = lars.BetaPath().size();
|
||||
for (size_t i = 0; i < pathLength; ++i)
|
||||
{
|
||||
// Use the i'th model in the path.
|
||||
lars.SelectBeta(lars.LambdaPath()[i]);
|
||||
|
||||
// ComputeError() returns the total loss, which we need to divide by the
|
||||
// number of points to get the MSE.
|
||||
const double trainMSE = lars.ComputeError(trainingData, trainingResponses) /
|
||||
trainingData.n_cols;
|
||||
const double testMSE = lars.ComputeError(testData, testResponses) /
|
||||
testData.n_cols;
|
||||
std::cout << "L1 penalty parameter: " << lars.SelectedLambda1() << std::endl;
|
||||
std::cout << " MSE on training set: " << trainMSE << "." << std::endl;
|
||||
std::cout << " MSE on test set: " << testMSE << "." << std::endl;
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
Train a LARS model, print predictions for a random point, and save to a file.
|
||||
|
||||
```c++
|
||||
// See https://datasets.mlpack.org/admission_predict.csv.
|
||||
arma::mat data;
|
||||
mlpack::data::Load("admission_predict.csv", data, true);
|
||||
|
||||
// See https://datasets.mlpack.org/admission_predict.responses.csv.
|
||||
arma::rowvec responses;
|
||||
mlpack::data::Load("admission_predict.responses.csv", responses, true);
|
||||
|
||||
// Train a LARS model with only L2 regularization.
|
||||
mlpack::LARS lars(data, responses, true, true, 0.0, 0.1 /* lambda2 */);
|
||||
|
||||
// Predict on a random point.
|
||||
arma::vec point = arma::randu<arma::vec>(data.n_rows);
|
||||
const double prediction = lars.Predict(point);
|
||||
|
||||
std::cout << "Prediction on random point: " << prediction << "." << std::endl;
|
||||
|
||||
// Save the model to "lars_model.bin" with the name "lars".
|
||||
mlpack::data::Save("lars_model.bin", "lars", lars, true);
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
Load a LARS model from disk and print some information about it.
|
||||
|
||||
```c++
|
||||
// This assumes a model named "lars" has previously been saved to
|
||||
// "lars_model.bin".
|
||||
mlpack::LARS lars;
|
||||
mlpack::data::Load("lars_model.bin", "lars", lars, true);
|
||||
|
||||
if (lars.BetaPath().size() == 0)
|
||||
{
|
||||
std::cout << "lars_model.bin contains an untrained LARS model." << std::endl;
|
||||
}
|
||||
else
|
||||
{
|
||||
std::cout << "Information on the LARS model in lars_model.bin:" << std::endl;
|
||||
|
||||
std::cout << " - Model dimensionality: " << lars.Beta().n_elem << "."
|
||||
<< std::endl;
|
||||
std::cout << " - Has intercept: "
|
||||
<< (lars.FitIntercept() ? std::string("yes") : std::string("no")) << "."
|
||||
<< std::endl;
|
||||
std::cout << " - Current L1 regularization penalty parameter value: "
|
||||
<< lars.SelectedLambda1() << "." << std::endl;
|
||||
std::cout << " - L2 regularization penalty parameter: " << lars.Lambda2()
|
||||
<< "." << std::endl;
|
||||
std::cout << " - Number of nonzero elements in model: "
|
||||
<< lars.ActiveSet().size() << "." << std::endl;
|
||||
std::cout << " - Number of models in LARS path: " << lars.BetaPath().size()
|
||||
<< "." << std::endl;
|
||||
std::cout << " - Model weight for dimension 0: " << lars.Beta()[0] << "."
|
||||
<< std::endl;
|
||||
|
||||
if (lars.FitIntercept())
|
||||
{
|
||||
std::cout << " - Intercept value: " << lars.Intercept() << "." << std::endl;
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
Train several models with different L2 regularization penalty parameters, using
|
||||
a precomputed Gram matrix.
|
||||
|
||||
```c++
|
||||
// See https://datasets.mlpack.org/admission_predict.csv.
|
||||
arma::mat data;
|
||||
mlpack::data::Load("admission_predict.csv", data, true);
|
||||
|
||||
// See https://datasets.mlpack.org/admission_predict.responses.csv.
|
||||
arma::rowvec responses;
|
||||
mlpack::data::Load("admission_predict.responses.csv", responses, true);
|
||||
|
||||
// Precompute Gram matrix.
|
||||
arma::mat gramMatrix = data * data.t();
|
||||
|
||||
std::vector<double> lambda2Values = { 0.01, 0.1, 1.0, 10.0, 100.0 };
|
||||
for (double lambda2 : lambda2Values)
|
||||
{
|
||||
// Build a LARS model using the precomputed Gram matrix. We did not normalize
|
||||
// or center the data before computing the Gram matrix, so we have to set
|
||||
// fitIntercept and normalizeData accordingly.
|
||||
mlpack::LARS lars(data, responses, true, true, gramMatrix, 0.01, lambda2,
|
||||
1e-16, false, false);
|
||||
|
||||
std::cout << "MSE with L2 penalty " << lambda2 << ": "
|
||||
<< (lars.ComputeError(data, responses) / data.n_cols) << "." << std::endl;
|
||||
}
|
||||
```
|
||||
|
||||
### Advanced Functionality: Different Element Types
|
||||
|
||||
The `LARS` class has one template parameter that can be used to
|
||||
control the element type of the model. The full signature of the class is:
|
||||
|
||||
```c++
|
||||
LARS<ModelMatType>
|
||||
```
|
||||
|
||||
`ModelMatType` specifies the type of matrix used for the internal representation
|
||||
of model parameters. Any matrix type that implements the Armadillo API can be
|
||||
used.
|
||||
|
||||
Note that the `Train()` and `Predict()` functions themselves are templatized and
|
||||
can allow any matrix type that has the same element type. So, for instance, a
|
||||
`LARS<arma::sp_mat>` can accept an `arma::mat` for training.
|
||||
|
||||
The example below trains a LARS model on 32-bit precision data, using
|
||||
`arma::sp_fmat` to store the model parameters.
|
||||
|
||||
```c++
|
||||
// Create random, sparse 1000-dimensional data.
|
||||
arma::fmat dataset(1000, 5000, arma::fill::randu);
|
||||
|
||||
// Generate noisy responses from random data.
|
||||
arma::fvec trueWeights(1000, arma::fill::randu);
|
||||
arma::frowvec responses = trueWeights.t() * dataset +
|
||||
0.01 * arma::randu<arma::frowvec>(5000) /* noise term */;
|
||||
|
||||
mlpack::LARS<arma::sp_fmat> lars;
|
||||
lars.Lambda1() = 0.1;
|
||||
lars.Lambda2() = 0.01;
|
||||
|
||||
lars.Train(dataset, responses);
|
||||
|
||||
// Compute the MSE on the training set and a random test set.
|
||||
arma::fmat testDataset(1000, 2500, arma::fill::randu);
|
||||
arma::frowvec testResponses = trueWeights.t() * testDataset +
|
||||
0.01 * arma::randu<arma::frowvec>(2500) /* noise term */;
|
||||
|
||||
const float trainMSE = lars.ComputeError(dataset, responses) / dataset.n_cols;
|
||||
const float testMSE = lars.ComputeError(testDataset, testResponses) /
|
||||
testDataset.n_cols;
|
||||
|
||||
std::cout << "MSE on training set: " << trainMSE << "." << std::endl;
|
||||
std::cout << "MSE on test set: " << testMSE << "." << std::endl;
|
||||
```
|
||||
|
||||
***Note:*** it is generally only more efficient to use a sparse type (e.g.
|
||||
`arma::sp_mat`) for `ModelMatType` when the L1 regularization parameter is set
|
||||
such that a highly sparse model is produced.
|
||||
@@ -0,0 +1,291 @@
|
||||
## `LinearRegression`
|
||||
|
||||
The `LinearRegression` class implements a standard L2-regularized linear
|
||||
regression model for numerical data, trained by direct decomposition of the
|
||||
training data. The class offers configurable functionality and template
|
||||
parameters to control the data type used for storing the model.
|
||||
|
||||
#### Simple usage example:
|
||||
|
||||
```c++
|
||||
// Train a linear regression model on random numeric data and make predictions.
|
||||
|
||||
// All data and responses are uniform random; this uses 10 dimensional data.
|
||||
// Replace with a data::Load() call or similar for a real application.
|
||||
arma::mat dataset(10, 1000, arma::fill::randu); // 1000 points.
|
||||
arma::rowvec responses = arma::randn<arma::rowvec>(1000);
|
||||
arma::mat testDataset(10, 500, arma::fill::randu); // 500 test points.
|
||||
|
||||
mlpack::LinearRegression lr; // Step 1: create model.
|
||||
lr.Train(dataset, responses); // Step 2: train model.
|
||||
arma::rowvec predictions;
|
||||
lr.Predict(testDataset, predictions); // Step 3: use model to predict.
|
||||
|
||||
// Print some information about the test predictions.
|
||||
std::cout << arma::accu(predictions > 0.7) << " test points predicted to have"
|
||||
<< " responses greater than 0.7." << std::endl;
|
||||
std::cout << arma::accu(predictions < 0) << " test points predicted to have "
|
||||
<< "negative responses." << std::endl;
|
||||
```
|
||||
<p style="text-align: center; font-size: 85%"><a href="#simple-examples">More examples...</a></p>
|
||||
|
||||
#### Quick links:
|
||||
|
||||
* [Constructors](#constructors): create `LinearRegression` objects.
|
||||
* [`Train()`](#training): train model.
|
||||
* [`Predict()`](#prediction): predict with a trained model.
|
||||
* [Other functionality](#other-functionality) for loading, saving, and
|
||||
inspecting.
|
||||
* [Examples](#simple-examples) of simple usage and links to detailed example
|
||||
projects.
|
||||
* [Template parameters](#advanced-functionality-different-element-types) for
|
||||
using different element types for a model.
|
||||
|
||||
#### See also:
|
||||
|
||||
* [mlpack regression techniques](#mlpack_regression_techniques) <!-- TODO: fix link! -->
|
||||
* [`LARS`](lars.md) <!-- TODO: fix link -->
|
||||
* [Linear Regression on Wikipedia](https://en.wikipedia.org/wiki/Linear_regression)
|
||||
|
||||
### Constructors
|
||||
|
||||
* `lr = LinearRegression()`
|
||||
- Initialize the model without training.
|
||||
- You will need to call [`Train()`](#training) later to train the model
|
||||
before calling [`Predict()`](#prediction).
|
||||
|
||||
---
|
||||
|
||||
* `lr = LinearRegression(data, responses, lambda=0.0, intercept=true)`
|
||||
* `lr = LinearRegression(data, responses, weights, lambda=0.0, intercept=true)`
|
||||
- Train model, optionally with instance weights.
|
||||
|
||||
---
|
||||
|
||||
#### Constructor Parameters:
|
||||
|
||||
<!-- TODOs for table below:
|
||||
* better link for column-major matrices
|
||||
* update matrices.md to include a section on labels and NormalizeLabels()
|
||||
-->
|
||||
|
||||
| **name** | **type** | **description** | **default** |
|
||||
|----------|----------|-----------------|-------------|
|
||||
| `data` | [`arma::mat`](../matrices.md) | [Column-major](../matrices.md) training matrix. | _(N/A)_ |
|
||||
| `responses` | [`arma::rowvec`](../matrices.md) | Training responses (e.g. values to predict). Should have length `data.n_cols`. | _(N/A)_ |
|
||||
| `weights` | [`arma::rowvec`](../matrices.md) | Weights for each training point. Should have length `data.n_cols`. | _(N/A)_ |
|
||||
| `lambda` | `double` | L2 regularization penalty parameter. | `0.0` |
|
||||
| `intercept` | `bool` | Whether to fit an intercept term in the model. | `bool` |
|
||||
|
||||
As an alternative to passing `lambda`, it can be set with the standalone
|
||||
`Lambda()` method: `lr.Lambda() = l;` will set the value of `lambda` to `l` for
|
||||
the next time `Train()` is called.
|
||||
|
||||
***Note***: setting `lambda` too small may cause the model to overfit; however,
|
||||
setting it too large may cause the model to underfit. [Automatic hyperparameter
|
||||
tuning](#hyperparameter-tuner) can be used to find a good value of `lambda`
|
||||
instead of a manual setting.
|
||||
|
||||
### Training
|
||||
|
||||
If training is not done as part of the constructor call, it can be done with the
|
||||
`Train()` function:
|
||||
|
||||
* `lr.Train(data, responses, lambda=0.0, intercept=true)`
|
||||
* `lr.Train(data, responses, weights, lambda=0.0, intercept=true)`
|
||||
- Train model on the given data, optionally with instance weights.
|
||||
|
||||
---
|
||||
|
||||
Types of each argument are the same as in the table for constructors
|
||||
[above](#constructor-parameters).
|
||||
|
||||
***Notes:***
|
||||
|
||||
* Training is not incremental. A second call to `Train()` will retrain the
|
||||
model from scratch.
|
||||
|
||||
* `Train()` returns the mean squared error (MSE) of the model on the training
|
||||
set as a `double`.
|
||||
|
||||
### Prediction
|
||||
|
||||
Once a `LinearRegression` model is trained, the `Predict()` member function
|
||||
can be used to make predictions for new data.
|
||||
|
||||
* `double predictedValue = lr.Predict(point)`
|
||||
- ***(Single-point)***
|
||||
- Make a prediction for a single point, returning the predicted value.
|
||||
|
||||
---
|
||||
|
||||
* `lr.Predict(data, predictions)`
|
||||
- ***(Multi-point)***
|
||||
- Make predictions for a set of points.
|
||||
- The prediction for data point `i` can be accessed with `predictions[i]`.
|
||||
|
||||
---
|
||||
|
||||
#### Prediction Parameters:
|
||||
|
||||
| **usage** | **name** | **type** | **description** |
|
||||
|-----------|----------|----------|-----------------|
|
||||
| _single-point_ | `point` | [`arma::vec`](../matrices.md) | Single point for prediction. |
|
||||
||||
|
||||
| _multi-point_ | `data` | [`arma::mat`](../matrices.md) | Set of [column-major](../matrices.md) points for classification. |
|
||||
| _multi-point_ | `predictions` | [`arma::rowvec&`](../matrices.md) | Vector of `double`s to store predictions into. Will be set to length `data.n_cols`. |
|
||||
|
||||
### Other Functionality
|
||||
|
||||
<!-- TODO: we should point directly to the documentation of those functions -->
|
||||
|
||||
* A `LinearRegression` model can be serialized with
|
||||
[`data::Save()`](../formats.md) and [`data::Load()`](../formats.md).
|
||||
|
||||
* `lr.Intercept()` will return a `bool` indicating whether the model was
|
||||
trained with an intercept term.
|
||||
|
||||
* `lr.Parameters()` will return an `arma::vec&` with the model parameters.
|
||||
This will have length equal to the dimensionality of the model if
|
||||
`lr.Intercept()` is `false`, and length equal to the dimensionality of the
|
||||
model plus one if `lr.Intercept()` is `true`. If an intercept was fitted,
|
||||
the intercept term is the first element of `lr.Parameters()`.
|
||||
|
||||
* `lr.ComputeError(data, responses)` will return a `double` containing the mean
|
||||
squared error (MSE) of the model on `data`, given that the true responses are
|
||||
`responses`.
|
||||
|
||||
### Simple Examples
|
||||
|
||||
See also the [simple usage example](#simple-usage-example) for a trivial usage
|
||||
of the `LinearRegression` class.
|
||||
|
||||
---
|
||||
|
||||
Train a linear regression model in the constructor on weighted data, compute the
|
||||
objective function with `ComputeError()`, and save the model.
|
||||
|
||||
```c++
|
||||
// See https://datasets.mlpack.org/admission_predict.csv.
|
||||
arma::mat data;
|
||||
mlpack::data::Load("admission_predict.csv", data, true);
|
||||
|
||||
// See https://datasets.mlpack.org/admission_predict.responses.csv.
|
||||
arma::rowvec responses;
|
||||
mlpack::data::Load("admission_predict.responses.csv", responses, true);
|
||||
|
||||
// Generate random instance weights for each point, in the range 0.5 to 1.5.
|
||||
arma::rowvec weights(data.n_cols, arma::fill::randu);
|
||||
weights += 0.5;
|
||||
|
||||
// Train a linear regression model, fitting an intercept term and using an L2
|
||||
// regularization parameter of 0.3.
|
||||
mlpack::LinearRegression lr(data, responses, weights, 0.3, true);
|
||||
|
||||
// Now compute the MSE on the training set.
|
||||
std::cout << "MSE on the training set: " << lr.ComputeError(data, responses)
|
||||
<< "." << std::endl;
|
||||
|
||||
// Finally, save the model with the name "lr".
|
||||
mlpack::data::Save("lr_model.bin", "lr", lr, true);
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
Load a saved linear regression model and print some information about it, then
|
||||
make some predictions individually for random points.
|
||||
|
||||
```
|
||||
mlpack::LinearRegression lr;
|
||||
|
||||
// Load the model named "lr" from "lr_model.bin".
|
||||
mlpack::data::Load("lr_model.bin", "lr", lr, true);
|
||||
|
||||
// Print some information about the model.
|
||||
const size_t dimensionality =
|
||||
(lr.Intercept() ? (lr.Parameters().n_elem - 1) : lr.Parameters().n_elem);
|
||||
|
||||
std::cout << "Information on the LinearRegression model in 'lr_model.bin':"
|
||||
<< std::endl;
|
||||
std::cout << " - Model has intercept: "
|
||||
<< (lr.Intercept() ? std::string("yes") : std::string("no")) << "."
|
||||
<< std::endl;
|
||||
if (lr.Intercept())
|
||||
{
|
||||
std::cout << " - Intercept weight: " << lr.Parameters()[0] << "."
|
||||
<< std::endl;
|
||||
}
|
||||
std::cout << " - Model dimensionality: " << dimensionality << "." << std::endl;
|
||||
std::cout << " - Lambda value: " << lr.Lambda() << "." << std::endl;
|
||||
std::cout << std::endl;
|
||||
|
||||
// Now make a prediction for three random points.
|
||||
for (size_t t = 0; t < 3; ++t)
|
||||
{
|
||||
arma::vec randomPoint(dimensionality, arma::fill::randu);
|
||||
const double prediction = lr.Predict(randomPoint);
|
||||
|
||||
std::cout << "Prediction for random point " << t << ": " << prediction << "."
|
||||
<< std::endl;
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
See also the following fully-working examples:
|
||||
|
||||
- [Salary prediction with `LinearRegression`](https://github.com/mlpack/examples/blob/master/salary_prediction_with_linear_regression/salary-prediction-linear-regression-cpp.ipynb)
|
||||
- [Avocado price prediction with `LinearRegression`](https://github.com/mlpack/examples/blob/master/avocado_price_prediction_with_linear_regression/avocado_price_prediction_with_lr_cpp.ipynb)
|
||||
- [California housing price prediction with `LinearRegression`](https://github.com/mlpack/examples/blob/master/california_housing_price_prediction_with_linear_regression/california_housing_price_prediction_with_lr_cpp.ipynb)
|
||||
|
||||
### Advanced Functionality: Different Element Types
|
||||
|
||||
The `LinearRegression` class has one template parameter that can be used to
|
||||
control the element type of the model. The full signature of the class is:
|
||||
|
||||
```c++
|
||||
LinearRegression<ModelMatType>
|
||||
```
|
||||
|
||||
`ModelMatType` specifies the type of matrix used for the internal representation
|
||||
of model parameters. Any matrix type that implements the Armadillo API can be
|
||||
used.
|
||||
|
||||
Note that the `Train()` and `Predict()` functions themselves are templatized and
|
||||
can allow any matrix type that has the same element type. So, for instance, a
|
||||
`LinearRegression<arma::mat>` can accept an `arma::sp_mat` for training.
|
||||
|
||||
The example below trains a linear regression model on sparse 32-bit floating
|
||||
point data, but uses a dense 32-bit floating point vector to store the model
|
||||
itself.
|
||||
|
||||
```c++
|
||||
// Create random, sparse 100-dimensional data.
|
||||
arma::sp_fmat dataset;
|
||||
dataset.sprandu(100, 5000, 0.3);
|
||||
|
||||
// Generate noisy responses from random data.
|
||||
arma::fvec trueWeights(100, arma::fill::randu);
|
||||
arma::frowvec responses = trueWeights.t() * dataset +
|
||||
0.01 * arma::randu<arma::frowvec>(5000) /* noise term */;
|
||||
|
||||
mlpack::LinearRegression<arma::fmat> lr;
|
||||
lr.Lambda() = 0.01;
|
||||
|
||||
lr.Train(dataset, responses);
|
||||
|
||||
// Compute the MSE on the training set and a random test set.
|
||||
arma::sp_fmat testDataset;
|
||||
testDataset.sprandu(100, 1000, 0.3);
|
||||
|
||||
arma::frowvec testResponses = trueWeights.t() * testDataset +
|
||||
0.01 * arma::randu<arma::frowvec>(1000) /* noise term */;
|
||||
|
||||
std::cout << "MSE on training set: "
|
||||
<< lr.ComputeError(dataset, responses) << "." << std::endl;
|
||||
std::cout << "MSE on test set: "
|
||||
<< lr.ComputeError(testDataset, testResponses) << "." << std::endl;
|
||||
```
|
||||
|
||||
***Note:*** dense objects should be used for `ModelMatType`, since in general an
|
||||
L2-regularized linear regression model will not be sparse.
|
||||
@@ -0,0 +1,381 @@
|
||||
## `LinearSVM`
|
||||
|
||||
The `LinearSVM` class implements an L2-regularized support vector machine for
|
||||
numerical data that can train using any ensmallen optimizer. The class offers
|
||||
standard classification functionality. Linear SVM is useful for multi-class
|
||||
classification (i.e. classes are `0`, `1`, `2`, etc.).
|
||||
|
||||
#### Simple usage example:
|
||||
|
||||
```c++
|
||||
// Train a linear SVM classifier on random data and predict labels:
|
||||
|
||||
// All data and labels are uniform random; 5 dimensional data, 4 classes.
|
||||
// Replace with a data::Load() call or similar for a real application.
|
||||
arma::mat dataset(5, 1000, arma::fill::randu); // 1000 points.
|
||||
arma::Row<size_t> labels =
|
||||
arma::randi<arma::Row<size_t>>(1000, arma::distr_param(0, 3));
|
||||
arma::mat testDataset(5, 500, arma::fill::randu); // 500 test points.
|
||||
|
||||
mlpack::LinearSVM svm; // Step 1: create model.
|
||||
svm.Train(dataset, labels, 4); // Step 2: train model.
|
||||
arma::Row<size_t> predictions;
|
||||
svm.Classify(testDataset, predictions); // Step 3: classify points.
|
||||
|
||||
// Print some information about the test predictions.
|
||||
std::cout << arma::accu(predictions == 1) << " test points classified as class "
|
||||
<< "1." << std::endl;
|
||||
```
|
||||
<p style="text-align: center; font-size: 85%"><a href="#simple-examples">More examples...</a></p>
|
||||
|
||||
#### Quick links:
|
||||
|
||||
* [Constructors](#constructors): create `LinearSVM` objects.
|
||||
* [`Train()`](#training): train model.
|
||||
* [`Classify()`](#classification): classify with a trained model.
|
||||
* [Other functionality](#other-functionality) for loading, saving, and
|
||||
inspecting.
|
||||
* [Examples](#simple-examples) of simple usage and links to detailed example
|
||||
projects.
|
||||
* [Template parameters](#advanced-functionality-different-element-types) for
|
||||
using different element types for a model.
|
||||
|
||||
#### See also:
|
||||
|
||||
* [mlpack classifiers](#mlpack_classifiers) <!-- TODO: fix link! -->
|
||||
* [`GaussianDistribution`](#gaussian_distribution) <!-- TODO: fix link! -->
|
||||
* [Naive Bayes classifier on Wikipedia](https://en.wikipedia.org/wiki/Naive_Bayes_classifier)
|
||||
|
||||
### Constructors
|
||||
|
||||
* `svm = LinearSVM()`
|
||||
- Initialize the parameters of the model without training.
|
||||
- You will need to call [`Train()`](#training) later to train the model
|
||||
before calling [`Classify()`](#classification).
|
||||
|
||||
---
|
||||
|
||||
* `svm = LinearSVM(dimensionality, numClasses, lambda=0.0001, delta=1.0, fitIntercept=false)`
|
||||
- Initialize the model without training, to default weights.
|
||||
- [`Classify()`](#classification) can immediately be called and
|
||||
`Parameters()` returns valid weights, but the model is otherwise untrained.
|
||||
- The model should be trained with [`Train()`](#training) before calling
|
||||
[`Classify()`](#classification).
|
||||
|
||||
---
|
||||
|
||||
* `svm = LinearSVM(data, labels, numClasses, lambda=0.0001, delta=1.0, fitIntercept=false, [callbacks...])`
|
||||
- Train model, optionally specifying ensmallen callbacks for use during
|
||||
optimization.
|
||||
|
||||
---
|
||||
|
||||
* `svm = LinearSVM(data, labels, numClasses, optimizer, lambda=0.0001, delta=1.0, fitIntercept=false, [callbacks...])`
|
||||
- Train model with a custom ensmallen optimizer, optionally specifying
|
||||
callbacks for use during optimization.
|
||||
|
||||
---
|
||||
|
||||
#### Constructor Parameters:
|
||||
|
||||
<!-- TODOs for table below:
|
||||
* better link for column-major matrices
|
||||
* update matrices.md to include a section on labels and NormalizeLabels()
|
||||
-->
|
||||
|
||||
| **name** | **type** | **description** | **default** |
|
||||
|----------|----------|-----------------|-------------|
|
||||
| `data` | [`arma::mat`](../matrices.md) | [Column-major](../matrices.md) training matrix. | _(N/A)_ |
|
||||
| `labels` | [`arma::Row<size_t>`]('../matrices.md') | Training labels, between `0` and `numClasses - 1` (inclusive). Should have length `data.n_cols`. | _(N/A)_ |
|
||||
| `dimensionality` | `size_t` | Dimension of input data (if data is not specified). Should be equal to `data.n_rows`. | _(N/A)_ |
|
||||
| `numClasses` | `size_t` | Number of classes in the dataset. | _(N/A)_ |
|
||||
| `optimizer` | [any ensmallen optimizer](https://www.ensmallen.org) | Instantiated ensmallen optimizer for [differentiable functions](https://www.ensmallen.org/docs.html#differentiable-functions) or [differentiable separable functions](https://www.ensmallen.org/docs.html#differentiable-separable-functions). | `ens::L_BFGS()` |
|
||||
| `lambda` | `double` | L2 regularization penalty parameter. Must be nonnegative. | `0.0` |
|
||||
| `delta` | `double` | Margin of difference between correct class and other classes. | `1.0` |
|
||||
| `fitIntercept` | `bool` | If `true`, then an intercept term is fitted to the model. | `false` |
|
||||
| `callbacks...` | [any set of ensmallen callbacks](https://www.ensmallen.org/docs.html#callback-documentation) | Optional callbacks for the ensmallen optimizer, such as e.g. `ens::ProgressBar()`, `ens::Report()`, or others. | _(N/A)_ |
|
||||
|
||||
As an alternative to passing `lambda`, `delta`, or `fitIntercept`, these can be
|
||||
set with a standalone method. The following functions can be used before
|
||||
calling `Train()`:
|
||||
|
||||
* `svm.Lambda() = lambda;` will set the L2 regularization penalty parameter to
|
||||
`lambda`.
|
||||
* `svm.Delta() = delta;` will set the margin of difference to `delta`.
|
||||
* `svm.FitIntercept() = fitIntercept;` will set whether the model fits an
|
||||
intercept to `fitIntercept`.
|
||||
|
||||
### Training
|
||||
|
||||
If training is not done as part of the constructor call, it can be done with the
|
||||
`Train()` function:
|
||||
|
||||
* `svm.Train(data, labels, numClasses, [callbacks...])`
|
||||
* `svm.Train(data, labels, numClasses, optimizer, [callbacks...])`
|
||||
- Train model without changing any hyperparameters, optionally using a custom
|
||||
ensmallen optimizer and specifying callbacks for use during optimization.
|
||||
|
||||
---
|
||||
|
||||
* `svm.Train(data, labels, numClasses, lambda=0.0001, delta=1.0, fitIntercept=false, [callbacks...])`
|
||||
* `svm.Train(data, labels, numClasses, optimizer, lambda=0.0001, delta=1.0, fitIntercept=false, [callbacks...])`
|
||||
- Train model on the given data, specifying hyperparameters and optionally
|
||||
also a custom ensmallen optimizer and callbacks for use during
|
||||
optimization.
|
||||
|
||||
---
|
||||
|
||||
Types of each argument are the same as in the table for constructors
|
||||
[above](#constructor-parameters).
|
||||
|
||||
***Note:*** Training is not incremental. Successive calls to `Train()` will
|
||||
train entirely new models.
|
||||
|
||||
### Classification
|
||||
|
||||
Once a `LinearSVM` model is trained, the `Classify()` member function
|
||||
can be used to make class predictions for new data.
|
||||
|
||||
* `size_t predictedClass = svm.Classify(point)`
|
||||
- ***(Single-point)***
|
||||
- Classify a single point, returning the predicted class (`0` through
|
||||
`numClasses - 1`, inclusive).
|
||||
|
||||
---
|
||||
|
||||
* `svm.Classify(point, prediction, probabilitiesVec)`
|
||||
- ***(Single-point)***
|
||||
- Classify a single point and compute class probabilities.
|
||||
- The predicted class is stored in `prediction`.
|
||||
- The probability of class `i` can be accessed with `probabilitiesVec[i]`.
|
||||
|
||||
---
|
||||
|
||||
* `svm.Classify(data, predictions)`
|
||||
- ***(Multi-point)***
|
||||
- Classify a set of points.
|
||||
- The prediction for data point `i` can be accessed with `predictions[i]`.
|
||||
|
||||
---
|
||||
|
||||
* `svm.Classify(data, predictions, probabilities)`
|
||||
- ***(Multi-point)***
|
||||
- Classify a set of points and compute class probabilities.
|
||||
- The prediction for data point `i` can be accessed with `predictions[i]`.
|
||||
- The probability of class `j` for data point `i` can be accessed with
|
||||
`probabilities(j, i)`.
|
||||
|
||||
---
|
||||
|
||||
#### Classification Parameters:
|
||||
|
||||
| **usage** | **name** | **type** | **description** |
|
||||
|-----------|----------|----------|-----------------|
|
||||
| _single-point_ | `point` | [`arma::vec`](../matrices.md) | Single point for classification. |
|
||||
| _single-point_ | `prediction` | `size_t&` | `size_t` to store class prediction into. |
|
||||
| _single-point_ | `probabilitiesVec` | [`arma::vec&`](../matrices.md) | `arma::vec&` to store class probabilities into; will have length 2. |
|
||||
||||
|
||||
| _multi-point_ | `data` | [`arma::mat`](../matrices.md) | Set of [column-major](../matrices.md) points for classification. |
|
||||
| _multi-point_ | `predictions` | [`arma::Row<size_t>&`](../matrices.md) | Vector of `size_t`s to store class prediction into; will be set to length `data.n_cols`. |
|
||||
| _multi-point_ | `probabilities` | [`arma::mat&`](../matrices.md) | Matrix to store class probabilities into (number of rows will be equal to 2; number of columns will be equal to `data.n_cols`). |
|
||||
|
||||
### Other Functionality
|
||||
|
||||
<!-- TODO: we should point directly to the documentation of those functions -->
|
||||
|
||||
* A `LinearSVM` model can be serialized with
|
||||
[`data::Save()`](../formats.md) and [`data::Load()`](../formats.md).
|
||||
|
||||
* `svm.Parameters()` will return the parameters of the model as an `arma::mat`
|
||||
with either `data.n_rows` rows (if `FitIntercept()` is `false`) or
|
||||
`data.n_rows + 1` rows (if `FitIntercept()` is `true`), and `numClasses`
|
||||
columns. The weight for dimension `i` for class `j` can be accessed with
|
||||
`svm.Parameters()(i, j)`. If `FitIntercept()` is `true`, the last row of
|
||||
`svm.Parameters()` represents the bias parameters for each class.
|
||||
|
||||
* `svm.FeatureSize()` will return the number of features in the model. This is
|
||||
equivalent to `data.n_rows` when the model was trained. The output is only
|
||||
valid if the model has been trained.
|
||||
|
||||
* `svm.ComputeAccuracy(data, labels)` will return the accuracy of the model on
|
||||
the given `data` with the given `labels`. The returned accuracy is between 0
|
||||
and 100.
|
||||
|
||||
### Simple Examples
|
||||
|
||||
See also the [simple usage example](#simple-usage-example) for a trivial usage
|
||||
of the `LinearSVM` class.
|
||||
|
||||
---
|
||||
|
||||
Train a linear SVM using a custom SGD-like optimizer with callbacks.
|
||||
|
||||
```c++
|
||||
// See https://datasets.mlpack.org/satellite.train.csv.
|
||||
arma::mat dataset;
|
||||
mlpack::data::Load("satellite.train.csv", dataset, true);
|
||||
// See https://datasets.mlpack.org/satellite.train.labels.csv.
|
||||
arma::Row<size_t> labels;
|
||||
mlpack::data::Load("satellite.train.labels.csv", labels, true);
|
||||
|
||||
mlpack::LinearSVM svm;
|
||||
svm.Lambda() = 0.1;
|
||||
|
||||
// Create AMSGrad optimizer with custom step size and batch size.
|
||||
ens::AMSGrad optimizer(0.01 /* step size */, 16 /* batch size */);
|
||||
optimizer.MaxIterations() = 100 * dataset.n_cols; // Allow 100 epochs.
|
||||
|
||||
// Print a progress bar and an optimization report when training is finished.
|
||||
svm.Train(dataset, labels, 2, optimizer, ens::ProgressBar(), ens::Report());
|
||||
|
||||
// Now predict on test labels and compute accuracy.
|
||||
|
||||
// See https://datasets.mlpack.org/satellite.test.csv.
|
||||
arma::mat testDataset;
|
||||
mlpack::data::Load("satellite.test.csv", testDataset, true);
|
||||
// See https://datasets.mlpack.org/satellite.test.labels.csv.
|
||||
arma::Row<size_t> testLabels;
|
||||
mlpack::data::Load("satellite.test.labels.csv", testLabels, true);
|
||||
|
||||
std::cout << std::endl;
|
||||
std::cout << "Accuracy on training set: "
|
||||
<< svm.ComputeAccuracy(dataset, labels) << "\%." << std::endl;
|
||||
std::cout << "Accuracy on test set: "
|
||||
<< svm.ComputeAccuracy(testDataset, testLabels) << "\%." << std::endl;
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
Train a linear SVM with SGD and save the model every epoch using a [custom
|
||||
ensmallen callback](https://www.ensmallen.org/docs.html#custom-callbacks):
|
||||
|
||||
```c++
|
||||
// This callback saves the model into "model-<epoch>.bin" after every epoch.
|
||||
class ModelCheckpoint
|
||||
{
|
||||
public:
|
||||
ModelCheckpoint(mlpack::LinearSVM<>& model) : model(model) { }
|
||||
|
||||
template<typename OptimizerType, typename FunctionType, typename MatType>
|
||||
bool EndEpoch(OptimizerType& /* optimizer */,
|
||||
FunctionType& /* function */,
|
||||
const MatType& /* coordinates */,
|
||||
const size_t epoch,
|
||||
const double /* objective */)
|
||||
{
|
||||
const std::string filename = "model-" + std::to_string(epoch) + ".bin";
|
||||
mlpack::data::Save(filename, "svm", model, true);
|
||||
return false; // Do not terminate the optimization.
|
||||
}
|
||||
|
||||
private:
|
||||
mlpack::LinearSVM<>& model;
|
||||
};
|
||||
```
|
||||
|
||||
With that callback available, the code to train the model is below:
|
||||
|
||||
```c++
|
||||
// See https://datasets.mlpack.org/satellite.train.csv.
|
||||
arma::mat dataset;
|
||||
mlpack::data::Load("satellite.train.csv", dataset, true);
|
||||
// See https://datasets.mlpack.org/satellite.train.labels.csv.
|
||||
arma::Row<size_t> labels;
|
||||
mlpack::data::Load("satellite.train.labels.csv", labels, true);
|
||||
|
||||
mlpack::LinearSVM svm;
|
||||
|
||||
// Create AdaDelta optimizer with a small step size and batch size of 1.
|
||||
ens::AdaDelta adaDelta(0.001, 1);
|
||||
adaDelta.MaxIterations() = 100 * dataset.n_cols; // 100 epochs maximum.
|
||||
|
||||
// Use the custom callback and an L2 penalty parameter of 0.01, with default
|
||||
// delta and fitting an intercept.
|
||||
svm.Train(dataset, labels, 2, adaDelta, 0.01, 1.0, true, ModelCheckpoint(svm),
|
||||
ens::ProgressBar());
|
||||
|
||||
// Now files like model-1.bin, model-2.bin, etc. should be saved on disk.
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
Load a linear SVM from disk and print some information about it.
|
||||
|
||||
```c++
|
||||
mlpack::LinearSVM svm;
|
||||
// This assumes that a model called "svm" has been saved to the file
|
||||
// "model-1.bin" (as in the previous example).
|
||||
mlpack::data::Load("model-1.bin", "svm", svm, true);
|
||||
|
||||
// Print the dimensionality of the model and some other statistics.
|
||||
std::cout << "The dimensionality of the model in model-1.bin is "
|
||||
<< svm.FeatureSize() << "." << std::endl;
|
||||
if (svm.FitIntercept())
|
||||
{
|
||||
std::cout << "Intercept values for each class: " << std::endl;
|
||||
for (size_t i = 0; i < svm.Parameters().n_cols; ++i)
|
||||
{
|
||||
std::cout << " - Class " << i << ": "
|
||||
<< svm.Parameters()(svm.Parameters().n_rows - 1, i) << "." << std::endl;
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
std::cout << "The model does not have an intercept fitted." << std::endl;
|
||||
}
|
||||
|
||||
std::cout << "The L2 regularization penalty parameter is: " << svm.Lambda()
|
||||
<< "." << std::endl;
|
||||
|
||||
std::cout << "Weights for the first dimension are: "
|
||||
<< svm.Parameters().row(0);
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Advanced Functionality: Different Element Types
|
||||
|
||||
The `LinearSVM` class has one template parameter that can be used to
|
||||
control the element type of the model. The full signature of the class is:
|
||||
|
||||
```c++
|
||||
LinearSVM<ModelMatType>
|
||||
```
|
||||
|
||||
`ModelMatType` specifies the type of matrix used for training data and internal
|
||||
representation of model parameters.
|
||||
|
||||
* Any matrix type that implements the Armadillo API can be used.
|
||||
|
||||
* `Train()` and `Classify()` functions themselves are templatized and can allow
|
||||
any matrix type that has the same element type. So, for instance, a
|
||||
`LinearSVM<arma::mat>` can accept an `arma::sp_mat` for training.
|
||||
|
||||
The example below trains a linear SVM on sparse 32-bit floating point
|
||||
data, but uses dense 32-bit floating point matrices to store the model itself.
|
||||
|
||||
```c++
|
||||
// Create random, sparse 100-dimensional data, with 3 classes.
|
||||
arma::sp_fmat dataset;
|
||||
dataset.sprandu(100, 5000, 0.3);
|
||||
arma::Row<size_t> labels =
|
||||
arma::randi<arma::Row<size_t>>(5000, arma::distr_param(0, 2));
|
||||
|
||||
mlpack::LinearSVM<arma::fmat> svm(dataset, labels, 3);
|
||||
|
||||
// Now classify a test point.
|
||||
arma::sp_fvec point;
|
||||
point.sprandu(100, 1, 0.3);
|
||||
|
||||
size_t prediction;
|
||||
arma::fvec probabilitiesVec;
|
||||
svm.Classify(point, prediction, probabilitiesVec);
|
||||
|
||||
std::cout << "Prediction for random test point: " << prediction << "."
|
||||
<< std::endl;
|
||||
std::cout << "Class probabilities for random test point: "
|
||||
<< probabilitiesVec.t();
|
||||
```
|
||||
|
||||
***Note:*** dense objects should be used for `ModelMatType`, since in general
|
||||
L2-regularized models are fully dense.
|
||||
@@ -0,0 +1,332 @@
|
||||
## `NaiveBayesClassifier`
|
||||
|
||||
The `NaiveBayesClassifier` implements a trivial Naive Bayes classifier for
|
||||
numerical data. The class offers standard classification functionality. Naive
|
||||
Bayes is useful for multi-class classification (i.e. classes are `0`, `1`, `2`,
|
||||
etc.), and due to its simplicity scales well to large-data scenarios.
|
||||
|
||||
#### Simple usage example:
|
||||
|
||||
```c++
|
||||
// Train a Naive Bayes classifier on random data and predict labels:
|
||||
|
||||
// All data and labels are uniform random; 5 dimensional data, 4 classes.
|
||||
// Replace with a data::Load() call or similar for a real application.
|
||||
arma::mat dataset(5, 1000, arma::fill::randu); // 1000 points.
|
||||
arma::Row<size_t> labels =
|
||||
arma::randi<arma::Row<size_t>>(1000, arma::distr_param(0, 3));
|
||||
arma::mat testDataset(5, 500, arma::fill::randu); // 500 test points.
|
||||
|
||||
mlpack::NaiveBayesClassifier nbc; // Step 1: create model.
|
||||
nbc.Train(dataset, labels, 4); // Step 2: train model.
|
||||
arma::Row<size_t> predictions;
|
||||
nbc.Classify(testDataset, predictions); // Step 3: classify points.
|
||||
|
||||
// Print some information about the test predictions.
|
||||
std::cout << arma::accu(predictions == 2) << " test points classified as class "
|
||||
<< "2." << std::endl;
|
||||
```
|
||||
<p style="text-align: center; font-size: 85%"><a href="#simple-examples">More examples...</a></p>
|
||||
|
||||
#### Quick links:
|
||||
|
||||
* [Constructors](#constructors): create `NaiveBayesClassifier` objects.
|
||||
* [`Train()`](#training): train model.
|
||||
* [`Classify()`](#classification): classify with a trained model.
|
||||
* [Other functionality](#other-functionality) for loading, saving, and
|
||||
inspecting.
|
||||
* [Examples](#simple-examples) of simple usage and links to detailed example
|
||||
projects.
|
||||
* [Template parameters](#advanced-functionality-different-element-types) for
|
||||
using different element types for a model.
|
||||
|
||||
#### See also:
|
||||
|
||||
* [mlpack classifiers](#mlpack_classifiers) <!-- TODO: fix link! -->
|
||||
* [`GaussianDistribution`](#gaussian_distribution) <!-- TODO: fix link! -->
|
||||
* [Naive Bayes classifier on Wikipedia](https://en.wikipedia.org/wiki/Naive_Bayes_classifier)
|
||||
|
||||
### Constructors
|
||||
|
||||
* `nbc = NaiveBayesClassifier()`
|
||||
- Initialize the model without training.
|
||||
- You will need to call [`Train()`](#training) later to train the model
|
||||
before calling [`Classify()`](#classification).
|
||||
|
||||
---
|
||||
|
||||
* `nbc = NaiveBayesClassifier(dimensionality, numClasses, epsilon=1e-10)`
|
||||
- Initialize model to the given dimensionality and number of classes without
|
||||
training.
|
||||
- This is meant to be used with the incremental version of `Train()` that
|
||||
takes only a single point.
|
||||
|
||||
---
|
||||
|
||||
* `nbc = NaiveBayesClassifier(data, labels, numClasses, incremental=true, epsilon=1e-10)`
|
||||
- Train model, optionally specifying whether to do incremental training.
|
||||
|
||||
---
|
||||
|
||||
#### Constructor Parameters:
|
||||
|
||||
<!-- TODOs for table below:
|
||||
* better link for column-major matrices
|
||||
* update matrices.md to include a section on labels and NormalizeLabels()
|
||||
-->
|
||||
|
||||
| **name** | **type** | **description** | **default** |
|
||||
|----------|----------|-----------------|-------------|
|
||||
| `data` | [`arma::mat`](../matrices.md) | [Column-major](../matrices.md) training matrix. | _(N/A)_ |
|
||||
| `labels` | [`arma::Row<size_t>`]('../matrices.md') | Training labels, between `0` and `numClasses - 1` (inclusive). Should have length `data.n_cols`. | _(N/A)_ |
|
||||
| `numClasses` | `size_t` | Number of classes in the dataset. | _(N/A)_ |
|
||||
| `incremental` | `bool` | If `true`, then the model will not be reset before training, and will use a robust incremental algorithm for variance computation. | `true` |
|
||||
| `epsilon` | `double` | Initial small value for sample variances, to prevent underflow (via `log(0)`). | 1e-10 |
|
||||
|
||||
As an alternative to passing the `epsilon` parameter, it can be set with the
|
||||
standalone `Epsilon()` method: `nbc.Epsilon() = eps;` will set the value of
|
||||
`epsilon` to `eps` for the next time non-incremental `Train()` or `Reset()` is
|
||||
called.
|
||||
|
||||
### Training
|
||||
|
||||
If training is not done as part of the constructor call, it can be done with the
|
||||
`Train()` function:
|
||||
|
||||
* `nbc.Train(data, labels, numClasses, incremental=true, epsilon=1e-10)`
|
||||
- Train model on the given data, optionally specifying whether to do
|
||||
incremental training.
|
||||
- Arguments described in [Constructor Parameters](#constructor_parameters)
|
||||
table above.
|
||||
|
||||
---
|
||||
|
||||
* `nbc.Train(point, label)`
|
||||
- Incrementally train on a single data point with the given label.
|
||||
- Ensure that the model has the right size and number of classes by using the
|
||||
appropriate constructor form to set `dimensionality`, or by calling
|
||||
`Reset()` (see [other functionality](#other_functionality)).
|
||||
|
||||
| **name** | **type** | **description** | **default** |
|
||||
|----------|----------|-----------------|-------------|
|
||||
| `point` | [`arma::vec`](../matrices.md) | [Column-major](../matrices.md) training point (i.e. one column). | _(N/A)_ |
|
||||
| `label` | `size_t` | Training label, in range `0` to `numClasses`. | _(N/A)_ |
|
||||
|
||||
***Note***: when performing incremental training, if `data` has a different
|
||||
dimensionality than the model, or if `numClasses` is different, the model will
|
||||
be reset. For single-point `Train()`, if `point` has different dimensionality,
|
||||
an exception will be thrown.
|
||||
|
||||
### Classification
|
||||
|
||||
Once a `NaiveBayesClassifier` model is trained, the `Classify()` member function
|
||||
can be used to make class predictions for new data.
|
||||
|
||||
* `size_t predictedClass = nbc.Classify(point)`
|
||||
- ***(Single-point)***
|
||||
- Classify a single point, returning the predicted class (`0` through
|
||||
`numClasses - 1`, inclusive).
|
||||
|
||||
---
|
||||
|
||||
* `nbc.Classify(point, prediction, probabilitiesVec)`
|
||||
- ***(Single-point)***
|
||||
- Classify a single point and compute class probabilities.
|
||||
- The predicted class is stored in `prediction`.
|
||||
- The probability of class `i` can be accessed with `probabilitiesVec[i]`.
|
||||
|
||||
---
|
||||
|
||||
* `nbc.Classify(data, predictions)`
|
||||
- ***(Multi-point)***
|
||||
- Classify a set of points.
|
||||
- The prediction for data point `i` can be accessed with `predictions[i]`.
|
||||
|
||||
---
|
||||
|
||||
* `nbc.Classify(data, predictions, probabilities)`
|
||||
- ***(Multi-point)***
|
||||
- Classify a set of points and compute class probabilities.
|
||||
- The prediction for data point `i` can be accessed with `predictions[i]`.
|
||||
- The probability of class `j` for data point `i` can be accessed with
|
||||
`probabilities(j, i)`.
|
||||
|
||||
---
|
||||
|
||||
#### Classification Parameters:
|
||||
|
||||
| **usage** | **name** | **type** | **description** |
|
||||
|-----------|----------|----------|-----------------|
|
||||
| _single-point_ | `point` | [`arma::vec`](../matrices.md) | Single point for classification. |
|
||||
| _single-point_ | `prediction` | `size_t&` | `size_t` to store class prediction into. |
|
||||
| _single-point_ | `probabilitiesVec` | [`arma::vec&`](../matrices.md) | `arma::vec&` to store class probabilities into; will have length 2. |
|
||||
||||
|
||||
| _multi-point_ | `data` | [`arma::mat`](../matrices.md) | Set of [column-major](../matrices.md) points for classification. |
|
||||
| _multi-point_ | `predictions` | [`arma::Row<size_t>&`](../matrices.md) | Vector of `size_t`s to store class prediction into; will be set to length `data.n_cols`. |
|
||||
| _multi-point_ | `probabilities` | [`arma::mat&`](../matrices.md) | Matrix to store class probabilities into (number of rows will be equal to 2; number of columns will be equal to `data.n_cols`). |
|
||||
|
||||
### Other Functionality
|
||||
|
||||
<!-- TODO: we should point directly to the documentation of those functions -->
|
||||
|
||||
* A `NaiveBayesClassifier` model can be serialized with
|
||||
[`data::Save()`](../formats.md) and [`data::Load()`](../formats.md).
|
||||
|
||||
* `nbc.Probabilities()` will return a column vector of length `numClasses`
|
||||
representing the prior probability of each class.
|
||||
|
||||
* `nbc.Means()` will return a matrix with rows equal to the dimensionality of
|
||||
the model and `numClasses` columns. Column `i` represents the sample mean of
|
||||
class `i`.
|
||||
|
||||
* `nbc.Variances()` will return a matrix with rows equal to the dimensionality
|
||||
of the model and `numClasses` columns. The element at row `i` and column `j`
|
||||
represents the sample variance in dimension `i` of class `j`.
|
||||
|
||||
* `nbc.Reset()` will reset the model to zeros; this is useful before
|
||||
incremental training. The form
|
||||
`nbc.Reset(dimensionality, numClasses, epsilon=1e-10)` can
|
||||
also be used to set the dimensionality and number of classes in the reset
|
||||
model.
|
||||
|
||||
* `nbc.TrainingPoints()` will return the number of points that the model has
|
||||
been trained on. When `nbc.Reset()` is called, this is reset to 0.
|
||||
|
||||
### Simple Examples
|
||||
|
||||
See also the [simple usage example](#simple-usage-example) for a trivial usage
|
||||
of the `NaiveBayesClassifier` class.
|
||||
|
||||
---
|
||||
|
||||
Train a Naive Bayes classifier incrementally, one point at a time, then compute
|
||||
accuracy on a test set and save the model to disk.
|
||||
|
||||
```c++
|
||||
// See https://datasets.mlpack.org/mnist.train.csv.
|
||||
arma::mat dataset;
|
||||
mlpack::data::Load("mnist.train.csv", dataset, true);
|
||||
// See https://datasets.mlpack.org/mnist.train.labels.csv.
|
||||
arma::Row<size_t> labels;
|
||||
mlpack::data::Load("mnist.train.labels.csv", labels, true);
|
||||
|
||||
mlpack::NaiveBayesClassifier nbc(dataset.n_rows /* dimensionality */,
|
||||
10 /* numClasses */);
|
||||
|
||||
// Iterate over all points in the dataset and call Train() on each point.
|
||||
for (size_t i = 0; i < dataset.n_cols; ++i)
|
||||
nbc.Train(dataset.col(i), labels[i]);
|
||||
|
||||
// Now compute the accuracy of the fully trained model on a test set.
|
||||
|
||||
// See https://datasets.mlpack.org/mnist.test.csv.
|
||||
arma::mat testDataset;
|
||||
mlpack::data::Load("mnist.test.csv", testDataset, true);
|
||||
// See https://datasets.mlpack.org/mnist.test.labels.csv.
|
||||
arma::Row<size_t> testLabels;
|
||||
mlpack::data::Load("mnist.test.labels.csv", testLabels, true);
|
||||
|
||||
arma::Row<size_t> predictions;
|
||||
nbc.Classify(dataset, predictions);
|
||||
const double trainAccuracy = 100.0 *
|
||||
((double) arma::accu(predictions == labels)) / labels.n_elem;
|
||||
std::cout << "Accuracy of model on training data: " << trainAccuracy << "\%."
|
||||
<< std::endl;
|
||||
|
||||
nbc.Classify(testDataset, predictions);
|
||||
|
||||
const double testAccuracy = 100.0 *
|
||||
((double) arma::accu(predictions == testLabels)) / testLabels.n_elem;
|
||||
std::cout << "Accuracy of model on test data: " << testAccuracy << "\%."
|
||||
<< std::endl;
|
||||
|
||||
// Save the model to disk with the name "nbc".
|
||||
mlpack::data::Save("nbc_model.bin", "nbc", nbc, true);
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
Load a saved Naive Bayes classifier and print some information about it.
|
||||
|
||||
```c++
|
||||
mlpack::NaiveBayesClassifier nbc;
|
||||
|
||||
// Load the model named "nbc" from "nbc_model.bin".
|
||||
mlpack::data::Load("nbc_model.bin", "nbc", nbc, true);
|
||||
|
||||
// Print information about the model.
|
||||
std::cout << "The dimensionality of the model in nbc_model.bin is "
|
||||
<< nbc.Means().n_rows << "." << std::endl;
|
||||
std::cout << "The number of classes in the model is "
|
||||
<< nbc.Probabilities().n_elem << "." << std::endl;
|
||||
std::cout << "The model was trained on " << nbc.TrainingPoints() << " points."
|
||||
<< std::endl;
|
||||
std::cout << "The prior probabilities of each class are: "
|
||||
<< nbc.Probabilities().t();
|
||||
|
||||
// Compute the class probabilities of a random point.
|
||||
// For our random point, we'll use one of the means plus some noise.
|
||||
arma::vec randomPoint = nbc.Means().col(2) +
|
||||
10.0 * arma::randu<arma::vec>(nbc.Means().n_rows);
|
||||
|
||||
size_t prediction;
|
||||
arma::vec probabilities;
|
||||
nbc.Classify(randomPoint, prediction, probabilities);
|
||||
|
||||
std::cout << "Random point class prediction: " << prediction << "."
|
||||
<< std::endl;
|
||||
std::cout << "Random point class probabilities: " << probabilities.t();
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
See also the following fully-working examples:
|
||||
|
||||
- [Microchip QA Classification using `NaiveBayesClassifier`](https://github.com/mlpack/examples/blob/master/microchip_quality_control_naive_bayes/microchip-quality-control-naive-bayes-cpp.ipynb)
|
||||
|
||||
### Advanced Functionality: Different Element Types
|
||||
|
||||
The `NaiveBayesClassifier` class has one template parameter that can be used to
|
||||
control the element type of the model. The full signature of the class is:
|
||||
|
||||
```c++
|
||||
NaiveBayesClassifier<ModelMatType>
|
||||
```
|
||||
|
||||
`ModelMatType` specifies the type of matrix used for training data and internal
|
||||
representation of model parameters.
|
||||
|
||||
* Any matrix type that implements the Armadillo API can be used.
|
||||
|
||||
* `Train()` and `Classify()` functions themselves are templatized and can allow
|
||||
any matrix type that has the same element type. So, for instance, a
|
||||
`NaiveBayesClassifier<arma::mat>` can accept an `arma::sp_mat` for training.
|
||||
|
||||
The example below trains a Naive Bayes model on sparse 32-bit floating point
|
||||
data, but uses dense 32-bit floating point matrices to store the model itself.
|
||||
|
||||
```c++
|
||||
// Create random, sparse 100-dimensional data, with 3 classes.
|
||||
arma::sp_fmat dataset;
|
||||
dataset.sprandu(100, 5000, 0.3);
|
||||
arma::Row<size_t> labels =
|
||||
arma::randi<arma::Row<size_t>>(5000, arma::distr_param(0, 2));
|
||||
|
||||
mlpack::NaiveBayesClassifier<arma::fmat> nbc(dataset, labels, 3);
|
||||
|
||||
// Now classify a test point.
|
||||
arma::sp_fvec point;
|
||||
point.sprandu(100, 1, 0.3);
|
||||
|
||||
size_t prediction;
|
||||
arma::fvec probabilitiesVec;
|
||||
nbc.Classify(point, prediction, probabilitiesVec);
|
||||
|
||||
std::cout << "Prediction for random test point: " << prediction << "."
|
||||
<< std::endl;
|
||||
std::cout << "Class probabilities for random test point: "
|
||||
<< probabilitiesVec.t();
|
||||
```
|
||||
|
||||
***Note:*** dense objects should be used for `ModelMatType`, since in general
|
||||
the mean and sample variance of sparse data is dense.
|
||||
@@ -103,6 +103,8 @@
|
||||
// Now include Armadillo through the special mlpack extensions.
|
||||
#include <mlpack/core/arma_extend/arma_extend.hpp>
|
||||
#include <mlpack/core/util/arma_traits.hpp>
|
||||
// Include local armadillo safe linear algebra functions.
|
||||
#include <mlpack/core/math/safe_linalg.hpp>
|
||||
|
||||
// On Visual Studio, disable C4519 (default arguments for function templates)
|
||||
// since it's by default an error, which doesn't even make any sense because
|
||||
|
||||
@@ -158,11 +158,16 @@ add_custom_command(TARGET python_copy PRE_BUILD
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/setup_readme.md
|
||||
${CMAKE_BINARY_DIR}/src/mlpack/bindings/python/)
|
||||
|
||||
# Copy all mlpack headers for inclusion in the package.
|
||||
# Copy all mlpack headers for inclusion in the package, but remove the bindings/
|
||||
# and tests/ directories as they should not be included.
|
||||
add_custom_command(TARGET python_copy PRE_BUILD
|
||||
COMMAND ${CMAKE_COMMAND} ARGS -E copy_directory
|
||||
${CMAKE_SOURCE_DIR}/src/
|
||||
${CMAKE_BINARY_DIR}/src/mlpack/bindings/python/mlpack/include/)
|
||||
add_custom_command(TARGET python_copy PRE_BUILD
|
||||
COMMAND ${CMAKE_COMMAND} ARGS -E rm -r
|
||||
${CMAKE_BINARY_DIR}/src/mlpack/bindings/python/mlpack/include/mlpack/bindings/
|
||||
${CMAKE_BINARY_DIR}/src/mlpack/bindings/python/mlpack/include/mlpack/tests/)
|
||||
|
||||
# Generate pkgconfig file for easy use of included headers.
|
||||
add_custom_target(python_pkgconfig
|
||||
|
||||
@@ -22,16 +22,14 @@ namespace util {
|
||||
// Utility functions to correctly handle transposed Armadillo matrices.
|
||||
template<typename T>
|
||||
inline void TransposeIfNeeded(
|
||||
const std::string& identifier,
|
||||
T& value,
|
||||
bool transpose)
|
||||
T& /* value */,
|
||||
bool /* transpose */)
|
||||
{
|
||||
// No transpose needed for non-matrices.
|
||||
return;
|
||||
}
|
||||
|
||||
inline void TransposeIfNeeded(
|
||||
const std::string& identifier,
|
||||
arma::mat& value,
|
||||
bool transpose)
|
||||
{
|
||||
@@ -59,7 +57,7 @@ inline void SetParam(util::Params& params,
|
||||
T& value,
|
||||
bool transpose = false)
|
||||
{
|
||||
TransposeIfNeeded(identifier, value, transpose);
|
||||
TransposeIfNeeded(value, transpose);
|
||||
params.Get<T>(identifier) = std::move(value);
|
||||
}
|
||||
|
||||
|
||||
@@ -13,6 +13,7 @@
|
||||
#include "print_wrapper_py.hpp"
|
||||
#include "get_arma_type.hpp"
|
||||
#include "wrapper_functions.hpp"
|
||||
#include "strip_type.hpp"
|
||||
#include <mlpack/core/util/io.hpp>
|
||||
|
||||
using namespace mlpack::util;
|
||||
@@ -42,18 +43,18 @@ void PrintWrapperPY(const std::string& category,
|
||||
// keeps track of indentation at each point.
|
||||
int indent = 0;
|
||||
|
||||
for(string i: methods)
|
||||
for (string i: methods)
|
||||
{
|
||||
params[i] = IO::Parameters(groupName + "_" + i);
|
||||
}
|
||||
|
||||
if(category == "regression")
|
||||
if (category == "regression")
|
||||
{
|
||||
cout << "class BaseEstimator:" << endl;
|
||||
cout << " pass" << endl;
|
||||
cout << endl;
|
||||
}
|
||||
else if(category == "classification")
|
||||
else if (category == "classification")
|
||||
{
|
||||
cout << "class BaseEstimator:" << endl;
|
||||
cout << " pass" << endl;
|
||||
@@ -64,7 +65,7 @@ void PrintWrapperPY(const std::string& category,
|
||||
}
|
||||
|
||||
// Import different mlpack programs that are to be wrapped.
|
||||
for(size_t i = 0; i < methods.size(); i++)
|
||||
for (size_t i = 0; i < methods.size(); i++)
|
||||
{
|
||||
cout << "from mlpack." << groupName << "_" << methods[i] << " ";
|
||||
cout << "import " << groupName << "_" << methods[i] << endl;
|
||||
@@ -72,19 +73,19 @@ void PrintWrapperPY(const std::string& category,
|
||||
|
||||
// Try importing scikit-learn, only for classification and
|
||||
// regression.
|
||||
if(category == "regression")
|
||||
if (category == "regression")
|
||||
{
|
||||
cout << "try:" << endl;
|
||||
cout << " from sklearn.base import BaseEstimator" << endl;
|
||||
}
|
||||
else if(category == "classification")
|
||||
else if (category == "classification")
|
||||
{
|
||||
cout << "try: " << endl;
|
||||
cout << " from sklearn.base import BaseEstimator, ClassifierMixin";
|
||||
cout << endl;
|
||||
}
|
||||
|
||||
if(category == "regression" || category == "classification")
|
||||
if (category == "regression" || category == "classification")
|
||||
{
|
||||
cout << "except:" << endl;
|
||||
cout << " pass" << endl;
|
||||
@@ -93,30 +94,34 @@ void PrintWrapperPY(const std::string& category,
|
||||
|
||||
// Check and store if every parameter is serializable,
|
||||
// a hyperparameter and boolean.
|
||||
for(map<string, Params>::iterator i=params.begin();
|
||||
i!=params.end(); i++)
|
||||
for (map<string, Params>::iterator i = params.begin();
|
||||
i != params.end(); i++)
|
||||
{
|
||||
map<string, ParamData> methodParams = i->second.Parameters();
|
||||
|
||||
for(map<string, ParamData>::iterator itr=methodParams.begin();
|
||||
itr!=methodParams.end(); itr++)
|
||||
for (map<string, ParamData>::iterator itr = methodParams.begin();
|
||||
itr != methodParams.end(); itr++)
|
||||
{
|
||||
// Get the sanitized name of the class.
|
||||
string strippedName, unused1, unused2;
|
||||
StripType(itr->second.cppType, strippedName, unused1, unused2);
|
||||
|
||||
// Checking for serializability.
|
||||
bool isSerial;
|
||||
i->second.functionMap[itr->second.tname]["IsSerializable"](
|
||||
itr->second, NULL, (void*)& isSerial);
|
||||
if(isSerial)
|
||||
serializable.insert(itr->second.cppType);
|
||||
if (isSerial)
|
||||
serializable.insert(strippedName);
|
||||
|
||||
// Checking for hyperparameter.
|
||||
bool isHyperParam = false;
|
||||
size_t foundArma = itr->second.cppType.find("arma");
|
||||
if(itr->second.input && foundArma == string::npos && !isSerial)
|
||||
if (itr->second.input && foundArma == string::npos && !isSerial)
|
||||
isHyperParam = true;
|
||||
hyperParams[itr->first] = isHyperParam;
|
||||
|
||||
// Checking for boolean.
|
||||
if(itr->second.cppType == "bool")
|
||||
if (itr->second.cppType == "bool")
|
||||
isBool[itr->first] = true;
|
||||
else
|
||||
isBool[itr->first] = false;
|
||||
@@ -126,9 +131,9 @@ void PrintWrapperPY(const std::string& category,
|
||||
string className = GetClassName(groupName);
|
||||
|
||||
// print class.
|
||||
if(category == "regression")
|
||||
if (category == "regression")
|
||||
cout << "class " << className << "(BaseEstimator)" << ":" << endl;
|
||||
else if(category == "classification")
|
||||
else if (category == "classification")
|
||||
{
|
||||
cout << "class " << className << "(BaseEstimator, ClassifierMixin)" <<
|
||||
":" << endl;
|
||||
@@ -140,15 +145,15 @@ void PrintWrapperPY(const std::string& category,
|
||||
indent += 2;
|
||||
cout << string(indent, ' ') << "def __init__(self," << endl;
|
||||
|
||||
for(auto itr=hyperParams.begin(); itr!=hyperParams.end(); itr++)
|
||||
for (auto itr = hyperParams.begin(); itr != hyperParams.end(); itr++)
|
||||
{
|
||||
// indent + 13, here 13 -> def __init__(
|
||||
if(itr->second && isBool[itr->first])
|
||||
if (itr->second && isBool[itr->first])
|
||||
{
|
||||
cout << string(indent+13, ' ') << GetValidName(itr->first)
|
||||
<< " = False," << endl;
|
||||
}
|
||||
else if(itr->second && !isBool[itr->first])
|
||||
else if (itr->second && !isBool[itr->first])
|
||||
{
|
||||
cout << string(indent+13, ' ') << GetValidName(itr->first)
|
||||
<< " = None," << endl;
|
||||
@@ -161,10 +166,10 @@ void PrintWrapperPY(const std::string& category,
|
||||
// storing given arguments in attributes.
|
||||
indent += 2;
|
||||
cout << string(indent, ' ') << "# serializable attributes." << endl;
|
||||
if(serializable.size() == 0)
|
||||
if (serializable.size() == 0)
|
||||
cout << string(indent, ' ') << "# None" << endl;
|
||||
|
||||
for(auto itr=serializable.begin(); itr!=serializable.end(); itr++)
|
||||
for (auto itr = serializable.begin(); itr != serializable.end(); itr++)
|
||||
{
|
||||
cout << string(indent, ' ');
|
||||
cout << "self._" << *itr << " = None" << endl;
|
||||
@@ -173,9 +178,9 @@ void PrintWrapperPY(const std::string& category,
|
||||
cout << endl;
|
||||
cout << string(indent, ' ') << "# hyper-parameters." << endl;
|
||||
|
||||
for(auto itr=hyperParams.begin(); itr!=hyperParams.end(); itr++)
|
||||
for (auto itr = hyperParams.begin(); itr != hyperParams.end(); itr++)
|
||||
{
|
||||
if(itr->second)
|
||||
if (itr->second)
|
||||
{
|
||||
string validName = GetValidName(itr->first);
|
||||
cout << string(indent, ' ');
|
||||
@@ -187,7 +192,7 @@ void PrintWrapperPY(const std::string& category,
|
||||
indent -= 2;
|
||||
|
||||
// print all method definitions.
|
||||
for(string methodName: methods)
|
||||
for (string methodName: methods)
|
||||
{
|
||||
// print method name.
|
||||
cout << string(indent, ' ') << "def " << GetMappedName(methodName);
|
||||
@@ -205,39 +210,44 @@ void PrintWrapperPY(const std::string& category,
|
||||
int numMatrixInputs = 0;
|
||||
int numVectorInputs = 0;
|
||||
|
||||
if(category == "regression" || category == "classification")
|
||||
if (category == "regression" || category == "classification")
|
||||
{
|
||||
for(map<string, ParamData>::iterator itr=methodParams.begin();
|
||||
itr!=methodParams.end(); itr++)
|
||||
for (map<string, ParamData>::iterator itr = methodParams.begin();
|
||||
itr != methodParams.end(); itr++)
|
||||
{
|
||||
// Throw error if there are more than one matrix input params,
|
||||
// or more than one vector input params.
|
||||
if(numMatrixInputs > 1)
|
||||
Log::Fatal << "More than one matrix input parameters for " <<
|
||||
methodName << "(" << GetMappedName(methodName) << ") method!" <<
|
||||
endl;
|
||||
if(numVectorInputs > 1)
|
||||
Log::Fatal << "More than one vector input parameters for " <<
|
||||
if (numMatrixInputs > 1)
|
||||
{
|
||||
Log::Fatal << "More than one matrix input parameter for " <<
|
||||
methodName << "(" << GetMappedName(methodName) << ") method!" <<
|
||||
endl;
|
||||
}
|
||||
|
||||
if(itr->second.input)
|
||||
if (numVectorInputs > 1)
|
||||
{
|
||||
Log::Fatal << "More than one vector input parameter for " <<
|
||||
methodName << "(" << GetMappedName(methodName) << ") method!" <<
|
||||
endl;
|
||||
}
|
||||
|
||||
if (itr->second.input)
|
||||
{
|
||||
// If this is a matrix parameter.
|
||||
if(itr->second.cppType == "arma::mat" ||
|
||||
itr->second.cppType ==
|
||||
"std::tuple<mlpack::data::DatasetInfo, arma::mat>" ||
|
||||
itr->second.cppType == "arma::Mat<size_t>")
|
||||
if (itr->second.cppType == "arma::mat" ||
|
||||
itr->second.cppType ==
|
||||
"std::tuple<mlpack::data::DatasetInfo, arma::mat>" ||
|
||||
itr->second.cppType == "arma::Mat<size_t>")
|
||||
{
|
||||
numMatrixInputs++;
|
||||
mapToScikitNames[itr->first] = "X";
|
||||
invMapToScikitNames["X"] = itr->first;
|
||||
}
|
||||
// If this is a vector parameter.
|
||||
else if(itr->second.cppType == "arma::vec" ||
|
||||
itr->second.cppType == "arma::rowvec" ||
|
||||
itr->second.cppType == "arma::Row<size_t>" ||
|
||||
itr->second.cppType == "arma::Col<size_t>")
|
||||
else if (itr->second.cppType == "arma::vec" ||
|
||||
itr->second.cppType == "arma::rowvec" ||
|
||||
itr->second.cppType == "arma::Row<size_t>" ||
|
||||
itr->second.cppType == "arma::Col<size_t>")
|
||||
{
|
||||
numVectorInputs++;
|
||||
mapToScikitNames[itr->first] = "y";
|
||||
@@ -248,13 +258,13 @@ void PrintWrapperPY(const std::string& category,
|
||||
}
|
||||
|
||||
// Now we print the matrix and vector params.
|
||||
if(category == "classification" || category == "regression")
|
||||
if (category == "classification" || category == "regression")
|
||||
{
|
||||
bool hasX = false;
|
||||
bool hasy = false;
|
||||
|
||||
// First print X, if it is present.
|
||||
if(invMapToScikitNames.find("X") != invMapToScikitNames.end())
|
||||
if (invMapToScikitNames.find("X") != invMapToScikitNames.end())
|
||||
{
|
||||
cout << string(indent + addIndent, ' ');
|
||||
cout << "X = None," << endl;
|
||||
@@ -262,25 +272,25 @@ void PrintWrapperPY(const std::string& category,
|
||||
}
|
||||
|
||||
// Now print y, if it is present.
|
||||
if(invMapToScikitNames.find("y") != invMapToScikitNames.end())
|
||||
if (invMapToScikitNames.find("y") != invMapToScikitNames.end())
|
||||
{
|
||||
cout << string(indent + addIndent, ' ');
|
||||
cout << "y = None," << endl;
|
||||
hasy = true;
|
||||
}
|
||||
|
||||
// Now print actual names, to make it work for
|
||||
// both mapped and actual names.
|
||||
// Now print actual names, to make it work for both mapped and actual
|
||||
// names.
|
||||
|
||||
// Now print actual name for X.
|
||||
if(hasX)
|
||||
if (hasX)
|
||||
{
|
||||
cout << string(indent + addIndent, ' ');
|
||||
cout << GetValidName(invMapToScikitNames["X"]) << " = None," << endl;
|
||||
}
|
||||
|
||||
// Now print actual name for y.
|
||||
if(hasy)
|
||||
if (hasy)
|
||||
{
|
||||
cout << string(indent + addIndent, ' ');
|
||||
cout << GetValidName(invMapToScikitNames["y"]) << " = None," << endl;
|
||||
@@ -295,10 +305,10 @@ void PrintWrapperPY(const std::string& category,
|
||||
string logicString = "";
|
||||
indent += 2;
|
||||
|
||||
if(hasX)
|
||||
if (hasX)
|
||||
{
|
||||
string realName = GetValidName(invMapToScikitNames["X"]);
|
||||
|
||||
|
||||
cout << string(indent, ' ') << "if X is not None and ";
|
||||
cout << realName << " is None:" << endl;
|
||||
|
||||
@@ -312,7 +322,7 @@ void PrintWrapperPY(const std::string& category,
|
||||
cout << endl;
|
||||
}
|
||||
|
||||
if(hasy)
|
||||
if (hasy)
|
||||
{
|
||||
string realName = GetValidName(invMapToScikitNames["y"]);
|
||||
|
||||
@@ -332,13 +342,17 @@ void PrintWrapperPY(const std::string& category,
|
||||
else
|
||||
{
|
||||
// print input parameters.
|
||||
for(map<string, ParamData>::iterator itr=methodParams.begin();
|
||||
itr!=methodParams.end(); itr++)
|
||||
for (map<string, ParamData>::iterator itr = methodParams.begin();
|
||||
itr != methodParams.end(); itr++)
|
||||
{
|
||||
string validName = GetValidName(itr->first);
|
||||
|
||||
if(itr->second.input && serializable.find(itr->second.cppType) ==
|
||||
serializable.end() && !hyperParams[itr->first])
|
||||
// Get valid stripped name.
|
||||
string strippedName, unused1, unused2;
|
||||
StripType(itr->second.cppType, strippedName, unused1, unused2);
|
||||
|
||||
if (itr->second.input && serializable.find(strippedName) ==
|
||||
serializable.end() && !hyperParams[itr->first])
|
||||
{
|
||||
cout << string(indent + addIndent, ' ');
|
||||
cout << validName << " = None," << endl;
|
||||
@@ -357,8 +371,8 @@ void PrintWrapperPY(const std::string& category,
|
||||
int count = 0; // just for reference.
|
||||
|
||||
// first pass through the parameters and print all required parameters.
|
||||
for(map<string, ParamData>::iterator itr=methodParams.begin();
|
||||
itr!=methodParams.end(); itr++)
|
||||
for (map<string, ParamData>::iterator itr = methodParams.begin();
|
||||
itr != methodParams.end(); itr++)
|
||||
{
|
||||
if(itr->second.input && itr->second.required)
|
||||
{
|
||||
@@ -367,9 +381,11 @@ void PrintWrapperPY(const std::string& category,
|
||||
cout << string(indent + addIndent, ' ');
|
||||
cout << validName << " = ";
|
||||
|
||||
if(serializable.find(itr->second.cppType) != serializable.end())
|
||||
cout << "self._" << itr->second.cppType << "," << endl;
|
||||
else if(hyperParams[itr->first])
|
||||
string strippedName, unused1, unused2;
|
||||
StripType(itr->second.cppType, strippedName, unused1, unused2);
|
||||
if (serializable.find(strippedName) != serializable.end())
|
||||
cout << "self._" << strippedName << "," << endl;
|
||||
else if (hyperParams[itr->first])
|
||||
cout << "self." << validName << "," << endl;
|
||||
else
|
||||
cout << validName << "," << endl;
|
||||
@@ -379,20 +395,22 @@ void PrintWrapperPY(const std::string& category,
|
||||
}
|
||||
|
||||
// Now print all non-required parameters.
|
||||
for(map<string, ParamData>::iterator itr=methodParams.begin();
|
||||
itr!=methodParams.end(); itr++)
|
||||
for (map<string, ParamData>::iterator itr = methodParams.begin();
|
||||
itr != methodParams.end(); itr++)
|
||||
{
|
||||
if(itr->second.input && !itr->second.required)
|
||||
if (itr->second.input && !itr->second.required)
|
||||
{
|
||||
string validName = GetValidName(itr->first);
|
||||
|
||||
if(count != 0)
|
||||
if (count != 0)
|
||||
cout << string(indent + addIndent, ' ');
|
||||
cout << validName << " = ";
|
||||
|
||||
if(serializable.find(itr->second.cppType) != serializable.end())
|
||||
cout << "self._" << itr->second.cppType << "," << endl;
|
||||
else if(hyperParams[itr->first])
|
||||
string strippedName, unused1, unused2;
|
||||
StripType(itr->second.cppType, strippedName, unused1, unused2);
|
||||
if (serializable.find(strippedName) != serializable.end())
|
||||
cout << "self._" << strippedName << "," << endl;
|
||||
else if (hyperParams[itr->first])
|
||||
cout << "self." << validName << "," << endl;
|
||||
else
|
||||
cout << validName << "," << endl;
|
||||
@@ -409,15 +427,17 @@ void PrintWrapperPY(const std::string& category,
|
||||
string returnString = string(indent, ' ') + "return ";
|
||||
bool outputsOnlySerial = true;
|
||||
|
||||
for(map<string, ParamData>::iterator itr=methodParams.begin();
|
||||
itr!=methodParams.end(); itr++)
|
||||
for (map<string, ParamData>::iterator itr = methodParams.begin();
|
||||
itr != methodParams.end(); itr++)
|
||||
{
|
||||
if(!itr->second.input)
|
||||
if (!itr->second.input)
|
||||
{
|
||||
if(serializable.find(itr->second.cppType) != serializable.end())
|
||||
string strippedName, unused1, unused2;
|
||||
StripType(itr->second.cppType, strippedName, unused1, unused2);
|
||||
if (serializable.find(strippedName) != serializable.end())
|
||||
{
|
||||
cout << string(indent, ' ');
|
||||
cout << "self._" << itr->second.cppType << " = out[\"" <<
|
||||
cout << "self._" << strippedName << " = out[\"" <<
|
||||
itr->first << "\"]" << endl;
|
||||
}
|
||||
else
|
||||
@@ -430,7 +450,7 @@ void PrintWrapperPY(const std::string& category,
|
||||
cout << endl;
|
||||
|
||||
// return somethings.
|
||||
if(outputsOnlySerial)
|
||||
if (outputsOnlySerial)
|
||||
cout << returnString + "self" << endl;
|
||||
else
|
||||
cout << returnString.substr(0, returnString.size() - 2) << endl;
|
||||
|
||||
@@ -31,7 +31,7 @@ class RegressionDistribution
|
||||
{
|
||||
private:
|
||||
//! Regression function for representing conditional mean.
|
||||
LinearRegression rf;
|
||||
LinearRegression<> rf;
|
||||
//! Error distribution.
|
||||
GaussianDistribution err;
|
||||
|
||||
@@ -81,9 +81,9 @@ class RegressionDistribution
|
||||
}
|
||||
|
||||
//! Return regression function.
|
||||
const LinearRegression& Rf() const { return rf; }
|
||||
const LinearRegression<>& Rf() const { return rf; }
|
||||
//! Modify regression function.
|
||||
LinearRegression& Rf() { return rf; }
|
||||
LinearRegression<>& Rf() { return rf; }
|
||||
|
||||
//! Return error distribution.
|
||||
const GaussianDistribution& Err() const { return err; }
|
||||
|
||||
@@ -24,7 +24,7 @@ namespace mlpack {
|
||||
*/
|
||||
inline void RegressionDistribution::Train(const arma::mat& observations)
|
||||
{
|
||||
LinearRegression lr(observations.rows(1, observations.n_rows - 1),
|
||||
LinearRegression<> lr(observations.rows(1, observations.n_rows - 1),
|
||||
arma::rowvec(observations.row(0)), 0, true);
|
||||
rf = lr;
|
||||
arma::rowvec fitted;
|
||||
@@ -46,7 +46,7 @@ inline void RegressionDistribution::Train(const arma::mat& observations,
|
||||
inline void RegressionDistribution::Train(const arma::mat& observations,
|
||||
const arma::rowvec& weights)
|
||||
{
|
||||
LinearRegression lr(observations.rows(1, observations.n_rows - 1),
|
||||
LinearRegression<> lr(observations.rows(1, observations.n_rows - 1),
|
||||
arma::rowvec(observations.row(0)), weights, 0, true);
|
||||
rf = lr;
|
||||
arma::rowvec fitted;
|
||||
@@ -63,8 +63,8 @@ inline double RegressionDistribution::Probability(
|
||||
const arma::vec& observation) const
|
||||
{
|
||||
arma::rowvec fitted;
|
||||
rf.Predict(observation.rows(1, observation.n_rows-1), fitted);
|
||||
return err.Probability(observation(0)-fitted.t());
|
||||
rf.Predict(observation.rows(1, observation.n_rows - 1), fitted);
|
||||
return err.Probability(observation(0) - fitted.t());
|
||||
}
|
||||
|
||||
inline void RegressionDistribution::Predict(const arma::mat& points,
|
||||
|
||||
@@ -19,6 +19,7 @@
|
||||
#include "lin_alg.hpp"
|
||||
#include "log_add.hpp"
|
||||
#include "make_alias.hpp"
|
||||
#include "safe_linalg.hpp"
|
||||
#include "multiply_slices.hpp"
|
||||
#include "quantile.hpp"
|
||||
#include "random_basis.hpp"
|
||||
|
||||
@@ -0,0 +1,52 @@
|
||||
/**
|
||||
* @file core/math/safe_linalg.hpp
|
||||
* @author Omar Shrit
|
||||
* @author Ryan Curtin
|
||||
*
|
||||
* A shim around arma and coot functions to avoid confusion with standard
|
||||
* library functions. This is necessary as we are using ADL to allow the
|
||||
* compiler to deduce which library the functions belong without the need
|
||||
* for namespace. This is mostly needed for MSVC compiler, gcc seems to
|
||||
* pass without an issue.
|
||||
*
|
||||
* mlpack is free software; you may redistribute it and/or modify it under the
|
||||
* terms of the 3-clause BSD license. You should have received a copy of the
|
||||
* 3-clause BSD license along with mlpack. If not, see
|
||||
* http://www.opensource.org/licenses/BSD-3-Clause for more information.
|
||||
*/
|
||||
#ifndef MLPACK_CORE_MAT_REDEF_HPP
|
||||
#define MLPACK_CORE_MAT_REDEF_HPP
|
||||
|
||||
namespace mlpack {
|
||||
|
||||
template<typename eT>
|
||||
inline arma::Mat<eT> SafeMax(const arma::Mat<eT>& A, const arma::Mat<eT>& B)
|
||||
{
|
||||
return arma::max(A, B);
|
||||
}
|
||||
|
||||
template<typename eT>
|
||||
inline arma::Mat<eT> SafeMin(const arma::Mat<eT>& A, const arma::Mat<eT>& B)
|
||||
{
|
||||
return arma::min(A, B);
|
||||
}
|
||||
|
||||
#ifdef MLPACK_HAS_COOT
|
||||
|
||||
template<typename eT>
|
||||
inline coot::Mat<eT> SafeMax(const coot::Mat<eT>& A, const coot::Mat<eT>& B)
|
||||
{
|
||||
return coot::max(A, B);
|
||||
}
|
||||
|
||||
template<typename eT>
|
||||
inline coot::Mat<eT> SafeMin(const coot::Mat<eT>& A, const coot::Mat<eT>& B)
|
||||
{
|
||||
return coot::min(A, B);
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
}
|
||||
|
||||
#endif
|
||||
@@ -116,7 +116,7 @@ struct IsVector<arma::subview_row<eT> >
|
||||
template<typename MatType>
|
||||
struct GetRowType
|
||||
{
|
||||
typedef MatType type; // Not sure...
|
||||
typedef arma::Row<typename MatType::elem_type> type;
|
||||
};
|
||||
|
||||
template<typename eT>
|
||||
@@ -136,7 +136,7 @@ struct GetRowType<arma::SpMat<eT>>
|
||||
template<typename MatType>
|
||||
struct GetColType
|
||||
{
|
||||
typedef MatType type; // Not sure...
|
||||
typedef arma::Row<typename MatType::elem_type> type;
|
||||
};
|
||||
|
||||
template<typename eT>
|
||||
@@ -184,7 +184,7 @@ struct GetDenseColType<arma::SpMat<eT>>
|
||||
template<typename MatType>
|
||||
struct GetDenseMatType
|
||||
{
|
||||
typedef MatType type;
|
||||
typedef arma::Mat<typename MatType::elem_type> type;
|
||||
};
|
||||
|
||||
template<typename eT>
|
||||
|
||||
@@ -84,8 +84,10 @@ struct IsEnsCallbackTypes;
|
||||
template<typename CallbackType, typename... CallbackTypes>
|
||||
struct IsEnsCallbackTypes<CallbackType, CallbackTypes...>
|
||||
{
|
||||
constexpr static bool value = std::is_class<CallbackType>::value &&
|
||||
IsEnsCallbackTypes<CallbackTypes...>::value;
|
||||
constexpr static bool value =
|
||||
std::is_class<typename std::remove_cv<
|
||||
typename std::remove_reference<CallbackType>::type
|
||||
>::type>::value && IsEnsCallbackTypes<CallbackTypes...>::value;
|
||||
};
|
||||
|
||||
template<>
|
||||
|
||||
@@ -56,29 +56,17 @@ class RectifierFunction
|
||||
}
|
||||
|
||||
/**
|
||||
* Computes the rectifier function using a dense matrix as input.
|
||||
* Computes the rectifier function using a 2nd /3rd-order tensor as input.
|
||||
*
|
||||
* @param x Input data.
|
||||
* @param y The resulting output activation.
|
||||
*/
|
||||
template<typename eT>
|
||||
static void Fn(const arma::Mat<eT>& x, arma::Mat<eT>& y)
|
||||
template<typename MatType>
|
||||
static void Fn(const MatType& x, MatType& y)
|
||||
{
|
||||
y.zeros(x.n_rows, x.n_cols);
|
||||
y = arma::max(y, x);
|
||||
}
|
||||
|
||||
/**
|
||||
* Computes the rectifier function using a 3rd-order tensor as input.
|
||||
*
|
||||
* @param x Input data.
|
||||
* @param y The resulting output activation.
|
||||
*/
|
||||
template<typename eT>
|
||||
static void Fn(const arma::Cube<eT>& x, arma::Cube<eT>& y)
|
||||
{
|
||||
y.zeros(x.n_rows, x.n_cols, x.n_slices);
|
||||
y = arma::max(y, x);
|
||||
y.set_size(size(x));
|
||||
y.zeros();
|
||||
y = SafeMax(y, x);
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@@ -81,7 +81,7 @@ class Layer
|
||||
{ /* Nothing to do here */ }
|
||||
|
||||
//! Copy assignment operator. This is not responsible for copying weights!
|
||||
virtual Layer& operator=(const Layer& layer)
|
||||
Layer& operator=(const Layer& layer)
|
||||
{
|
||||
if (&layer != this)
|
||||
{
|
||||
@@ -95,7 +95,7 @@ class Layer
|
||||
}
|
||||
|
||||
//! Move assignment operator. This is not responsible for moving weights!
|
||||
virtual Layer& operator=(Layer&& layer)
|
||||
Layer& operator=(Layer&& layer)
|
||||
{
|
||||
if (&layer != this)
|
||||
{
|
||||
|
||||
@@ -95,13 +95,18 @@ namespace mlpack {
|
||||
* estimator.Predict(xTest, responses, stds)
|
||||
* @endcode
|
||||
*/
|
||||
template<typename ModelMatType = arma::mat>
|
||||
class BayesianLinearRegression
|
||||
{
|
||||
public:
|
||||
typedef typename ModelMatType::elem_type ElemType;
|
||||
typedef typename GetDenseColType<ModelMatType>::type DenseVecType;
|
||||
typedef typename GetDenseRowType<ModelMatType>::type DenseRowType;
|
||||
|
||||
/**
|
||||
* Set the parameters of Bayesian Ridge regression object. The
|
||||
* regularization parameter is automatically set to its optimal value by
|
||||
* maximization of the marginal likelihood.
|
||||
* Set the parameters of Bayesian Ridge regression object. The regularization
|
||||
* parameter will be automatically set to its optimal value by maximization of
|
||||
* the marginal likelihood when training is done.
|
||||
*
|
||||
* @param centerData Whether or not center the data according to the
|
||||
* examples.
|
||||
@@ -116,6 +121,34 @@ class BayesianLinearRegression
|
||||
const size_t maxIterations = 50,
|
||||
const double tolerance = 1e-4);
|
||||
|
||||
/**
|
||||
* Create the BayesianLinearRegression object and train the model. The
|
||||
* regularization parameter is automatically set to its optimal value by
|
||||
* maximization of the maginal likelihood.
|
||||
*
|
||||
* @param data Column-major input data, dim(P, N).
|
||||
* @param responses A vector of targets, dim(N).
|
||||
* @param centerData Whether or not center the data according to the
|
||||
* examples.
|
||||
* @param scaleData Whether or not scale the data according to the
|
||||
* standard deviation of each feature.
|
||||
* @param maxIterations Maximum number of iterations for convergency.
|
||||
* @param tolerance Level from which the solution is considered sufficientlly
|
||||
* stable.
|
||||
* @return Root mean squared error.
|
||||
*/
|
||||
template<typename MatType,
|
||||
typename ResponsesType,
|
||||
typename = typename std::enable_if<
|
||||
std::is_same<typename ResponsesType::elem_type, ElemType>::value
|
||||
>::type>
|
||||
BayesianLinearRegression(const MatType& data,
|
||||
const ResponsesType& responses,
|
||||
const bool centerData = true,
|
||||
const bool scaleData = false,
|
||||
const size_t maxIterations = 50,
|
||||
const double tolerance = 1e-4);
|
||||
|
||||
/**
|
||||
* Run BayesianLinearRegression. The input matrix (like all mlpack matrices)
|
||||
* should be column-major -- each column is an observation and each row is a
|
||||
@@ -123,10 +156,104 @@ class BayesianLinearRegression
|
||||
*
|
||||
* @param data Column-major input data, dim(P, N).
|
||||
* @param responses A vector of targets, dim(N).
|
||||
* @param centerData Whether or not center the data according to the
|
||||
* examples.
|
||||
* @param scaleData Whether or not scale the data according to the
|
||||
* standard deviation of each feature.
|
||||
* @param maxIterations Maximum number of iterations for convergency.
|
||||
* @param tolerance Level from which the solution is considered sufficientlly
|
||||
* stable.
|
||||
* @return Root mean squared error.
|
||||
*/
|
||||
double Train(const arma::mat& data,
|
||||
const arma::rowvec& responses);
|
||||
// Many overloads necessary here until std::optional is available with C++17.
|
||||
// The first overload is also necessary to avoid confusing the hyperparameter
|
||||
// tuner, so that this can be correctly detected as a regression algorithm.
|
||||
template<typename MatType>
|
||||
ElemType Train(const MatType& data,
|
||||
const arma::rowvec& responses);
|
||||
|
||||
template<typename MatType,
|
||||
typename ResponsesType,
|
||||
typename = void, /* so MetaInfoExtractor does not get confused */
|
||||
typename = typename std::enable_if<
|
||||
std::is_same<typename ResponsesType::elem_type, ElemType>::value
|
||||
>::type,
|
||||
typename = typename std::enable_if<
|
||||
!std::is_same<ResponsesType, arma::rowvec>::value
|
||||
>::type>
|
||||
ElemType Train(const MatType& data,
|
||||
const ResponsesType& responses);
|
||||
|
||||
template<typename MatType,
|
||||
typename ResponsesType,
|
||||
typename = void, /* so MetaInfoExtractor does not get confused */
|
||||
typename = typename std::enable_if<
|
||||
std::is_same<typename ResponsesType::elem_type, ElemType>::value
|
||||
>::type>
|
||||
ElemType Train(const MatType& data,
|
||||
const ResponsesType& responses,
|
||||
const bool centerData);
|
||||
|
||||
template<typename MatType,
|
||||
typename ResponsesType,
|
||||
typename = void, /* so MetaInfoExtractor does not get confused */
|
||||
typename = typename std::enable_if<
|
||||
std::is_same<typename ResponsesType::elem_type, ElemType>::value
|
||||
>::type>
|
||||
ElemType Train(const MatType& data,
|
||||
const ResponsesType& responses,
|
||||
const bool centerData,
|
||||
const bool scaleData);
|
||||
|
||||
template<typename MatType,
|
||||
typename ResponsesType,
|
||||
typename = void, /* so MetaInfoExtractor does not get confused */
|
||||
typename = typename std::enable_if<
|
||||
std::is_same<typename ResponsesType::elem_type, ElemType>::value
|
||||
>::type>
|
||||
ElemType Train(const MatType& data,
|
||||
const ResponsesType& responses,
|
||||
const bool centerData,
|
||||
const bool scaleData,
|
||||
const size_t maxIterations);
|
||||
|
||||
template<typename MatType,
|
||||
typename ResponsesType,
|
||||
typename = void, /* so MetaInfoExtractor does not get confused */
|
||||
typename = typename std::enable_if<
|
||||
std::is_same<typename ResponsesType::elem_type, ElemType>::value
|
||||
>::type>
|
||||
ElemType Train(const MatType& data,
|
||||
const ResponsesType& responses,
|
||||
const bool centerData,
|
||||
const bool scaleData,
|
||||
const size_t maxIterations,
|
||||
const double tolerance);
|
||||
|
||||
/**
|
||||
* Predict \f$y\f$ for a single data point \f$x\f$ using the currently-trained
|
||||
* Bayesian ridge regression model.
|
||||
*
|
||||
* @param point The data point to apply the model to.
|
||||
* @return Prediction for the `point`.
|
||||
*/
|
||||
template<typename VecType>
|
||||
ElemType Predict(const VecType& point) const;
|
||||
|
||||
/**
|
||||
* Predict \f$y\f$ for a single data point \f$x\f$ using the currently-trained
|
||||
* Bayesian ridge regression model, storing the prediction in `prediction` and
|
||||
* the standard deviation of the prediction in `stddev`.
|
||||
*
|
||||
* @param point The data point to apply the model to.
|
||||
* @param prediction `double` to store the prediction into.
|
||||
* @param stddev `double` to store the standard deviation of the prediction
|
||||
* into.
|
||||
*/
|
||||
template<typename VecType>
|
||||
void Predict(const VecType& point,
|
||||
ElemType& prediction,
|
||||
ElemType& stddev) const;
|
||||
|
||||
/**
|
||||
* Predict \f$y_{i}\f$ for each data point in the given data matrix using the
|
||||
@@ -134,10 +261,14 @@ class BayesianLinearRegression
|
||||
*
|
||||
* @param points The data points to apply the model.
|
||||
* @param predictions y, Contains the predicted values on completion.
|
||||
* @return Root mean squared error computed on the train set.
|
||||
*/
|
||||
void Predict(const arma::mat& points,
|
||||
arma::rowvec& predictions) const;
|
||||
template<typename MatType,
|
||||
typename ResponsesType,
|
||||
typename = typename std::enable_if<
|
||||
std::is_same<typename ResponsesType::elem_type, ElemType>::value
|
||||
>::type>
|
||||
void Predict(const MatType& points,
|
||||
ResponsesType& predictions) const;
|
||||
|
||||
/**
|
||||
* Predict \f$y_{i}\f$ and the standard deviation of the predictive posterior
|
||||
@@ -149,9 +280,14 @@ class BayesianLinearRegression
|
||||
* completion.
|
||||
* @param std Standard deviations of the predictions.
|
||||
*/
|
||||
void Predict(const arma::mat& points,
|
||||
arma::rowvec& predictions,
|
||||
arma::rowvec& std) const;
|
||||
template<typename MatType,
|
||||
typename ResponsesType,
|
||||
typename = typename std::enable_if<
|
||||
std::is_same<typename ResponsesType::elem_type, ElemType>::value
|
||||
>::type>
|
||||
void Predict(const MatType& points,
|
||||
ResponsesType& predictions,
|
||||
ResponsesType& std) const;
|
||||
|
||||
/**
|
||||
* Compute the Root Mean Square Error between the predictions returned by the
|
||||
@@ -161,8 +297,13 @@ class BayesianLinearRegression
|
||||
* @param responses A vector of targets.
|
||||
* @return Root mean squared error.
|
||||
**/
|
||||
double RMSE(const arma::mat& data,
|
||||
const arma::rowvec& responses) const;
|
||||
template<typename MatType,
|
||||
typename ResponsesType,
|
||||
typename = typename std::enable_if<
|
||||
std::is_same<typename ResponsesType::elem_type, ElemType>::value
|
||||
>::type>
|
||||
ElemType RMSE(const MatType& data,
|
||||
const ResponsesType& responses) const;
|
||||
|
||||
/**
|
||||
* Get the solution vector.
|
||||
@@ -199,7 +340,7 @@ class BayesianLinearRegression
|
||||
*
|
||||
* @return responsesOffset
|
||||
*/
|
||||
const arma::colvec& DataOffset() const { return dataOffset; }
|
||||
const DenseVecType& DataOffset() const { return dataOffset; }
|
||||
|
||||
/**
|
||||
* Get the vector of standard deviations computed on the features over the
|
||||
@@ -207,14 +348,14 @@ class BayesianLinearRegression
|
||||
*
|
||||
* @return dataOffset
|
||||
*/
|
||||
const arma::colvec& DataScale() const { return dataScale; }
|
||||
const DenseVecType& DataScale() const { return dataScale; }
|
||||
|
||||
/**
|
||||
* Get the mean value of the train responses.
|
||||
*
|
||||
* @return responsesOffset
|
||||
*/
|
||||
double ResponsesOffset() const { return responsesOffset; }
|
||||
ElemType ResponsesOffset() const { return responsesOffset; }
|
||||
|
||||
//! Get whether the data will be centered during training.
|
||||
bool CenterData() const { return centerData; }
|
||||
@@ -258,28 +399,28 @@ class BayesianLinearRegression
|
||||
double tolerance;
|
||||
|
||||
//! Mean vector computed over the points.
|
||||
arma::colvec dataOffset;
|
||||
DenseVecType dataOffset;
|
||||
|
||||
//! Std vector computed over the points.
|
||||
arma::colvec dataScale;
|
||||
DenseVecType dataScale;
|
||||
|
||||
//! Mean of the response vector computed over the points.
|
||||
double responsesOffset;
|
||||
ElemType responsesOffset;
|
||||
|
||||
//! Precision of the prior pdf (gaussian).
|
||||
double alpha;
|
||||
ElemType alpha;
|
||||
|
||||
//! Noise inverse variance.
|
||||
double beta;
|
||||
ElemType beta;
|
||||
|
||||
//! Effective number of parameters.
|
||||
double gamma;
|
||||
ElemType gamma;
|
||||
|
||||
//! Solution vector.
|
||||
arma::colvec omega;
|
||||
DenseVecType omega;
|
||||
|
||||
//! Covariance matrix of the solution vector omega.
|
||||
arma::mat matCovariance;
|
||||
ModelMatType matCovariance;
|
||||
|
||||
/**
|
||||
* Center and scale the data accordind to centerData and scaleData.
|
||||
@@ -291,23 +432,30 @@ class BayesianLinearRegression
|
||||
* @param responsesProc Responses processed, dim(N).
|
||||
* @return reponsesOffset Mean of responses.
|
||||
*/
|
||||
double CenterScaleData(const arma::mat& data,
|
||||
const arma::rowvec& responses,
|
||||
arma::mat& dataProc,
|
||||
arma::rowvec& responsesProc);
|
||||
template<typename MatType, typename ResponsesType>
|
||||
double CenterScaleData(const MatType& data,
|
||||
const ResponsesType& responses,
|
||||
MatType& dataProc,
|
||||
ResponsesType& responsesProc);
|
||||
|
||||
/**
|
||||
* Center and scale the points before prediction.
|
||||
* Center and scale the points before prediction. This should only be called
|
||||
* if centerData or scaleData is true; if neither is true, then `dataProc`
|
||||
* will be unmodified.
|
||||
*
|
||||
* @param data Design matrix in column-major format, dim(P, N).
|
||||
* @param dataProc Data processed, dim(P, N).
|
||||
*/
|
||||
void CenterScaleDataPred(const arma::mat& data,
|
||||
arma::mat& dataProc) const;
|
||||
template<typename MatType, typename OutMatType>
|
||||
void CenterScaleDataPred(const MatType& data,
|
||||
OutMatType& dataProc) const;
|
||||
};
|
||||
|
||||
} // namespace mlpack
|
||||
|
||||
CEREAL_TEMPLATE_CLASS_VERSION((typename ModelMatType),
|
||||
(mlpack::BayesianLinearRegression<ModelMatType>), (1));
|
||||
|
||||
// Include implementation of serialize.
|
||||
#include "bayesian_linear_regression_impl.hpp"
|
||||
|
||||
|
||||
+289
-70
@@ -16,7 +16,8 @@
|
||||
|
||||
namespace mlpack {
|
||||
|
||||
inline BayesianLinearRegression::BayesianLinearRegression(
|
||||
template<typename ModelMatType>
|
||||
inline BayesianLinearRegression<ModelMatType>::BayesianLinearRegression(
|
||||
const bool centerData,
|
||||
const bool scaleData,
|
||||
const size_t maxIterations,
|
||||
@@ -29,15 +30,117 @@ inline BayesianLinearRegression::BayesianLinearRegression(
|
||||
alpha(0.0),
|
||||
beta(0.0),
|
||||
gamma(0.0)
|
||||
{/* Nothing to do */}
|
||||
{ /* Nothing to do */ }
|
||||
|
||||
inline double BayesianLinearRegression::Train(const arma::mat& data,
|
||||
const arma::rowvec& responses)
|
||||
template<typename ModelMatType>
|
||||
template<typename MatType, typename ResponsesType, typename>
|
||||
inline BayesianLinearRegression<ModelMatType>::BayesianLinearRegression(
|
||||
const MatType& data,
|
||||
const ResponsesType& responses,
|
||||
const bool centerData,
|
||||
const bool scaleData,
|
||||
const size_t maxIterations,
|
||||
const double tolerance) :
|
||||
centerData(centerData),
|
||||
scaleData(scaleData),
|
||||
maxIterations(maxIterations),
|
||||
tolerance(tolerance),
|
||||
responsesOffset(0.0),
|
||||
alpha(0.0),
|
||||
beta(0.0),
|
||||
gamma(0.0)
|
||||
{
|
||||
arma::mat phi;
|
||||
arma::rowvec t;
|
||||
arma::colvec eigVal;
|
||||
arma::mat eigVec;
|
||||
// Train the model.
|
||||
Train(data, responses);
|
||||
}
|
||||
|
||||
template<typename ModelMatType>
|
||||
template<typename MatType>
|
||||
inline
|
||||
typename BayesianLinearRegression<ModelMatType>::ElemType
|
||||
BayesianLinearRegression<ModelMatType>::Train(
|
||||
const MatType& data,
|
||||
const arma::rowvec& responses)
|
||||
{
|
||||
return Train(data, responses, this->centerData, this->scaleData,
|
||||
this->maxIterations, this->tolerance);
|
||||
}
|
||||
|
||||
template<typename ModelMatType>
|
||||
template<typename MatType, typename ResponsesType, typename, typename, typename>
|
||||
inline
|
||||
typename BayesianLinearRegression<ModelMatType>::ElemType
|
||||
BayesianLinearRegression<ModelMatType>::Train(
|
||||
const MatType& data,
|
||||
const ResponsesType& responses)
|
||||
{
|
||||
return Train(data, responses, this->centerData, this->scaleData,
|
||||
this->maxIterations, this->tolerance);
|
||||
}
|
||||
|
||||
template<typename ModelMatType>
|
||||
template<typename MatType, typename ResponsesType, typename, typename>
|
||||
inline
|
||||
typename BayesianLinearRegression<ModelMatType>::ElemType
|
||||
BayesianLinearRegression<ModelMatType>::Train(
|
||||
const MatType& data,
|
||||
const ResponsesType& responses,
|
||||
const bool centerData)
|
||||
{
|
||||
return Train(data, responses, centerData, this->scaleData,
|
||||
this->maxIterations, this->tolerance);
|
||||
}
|
||||
|
||||
template<typename ModelMatType>
|
||||
template<typename MatType, typename ResponsesType, typename, typename>
|
||||
inline
|
||||
typename BayesianLinearRegression<ModelMatType>::ElemType
|
||||
BayesianLinearRegression<ModelMatType>::Train(
|
||||
const MatType& data,
|
||||
const ResponsesType& responses,
|
||||
const bool centerData,
|
||||
const bool scaleData)
|
||||
{
|
||||
return Train(data, responses, centerData, scaleData, this->maxIterations,
|
||||
this->tolerance);
|
||||
}
|
||||
|
||||
template<typename ModelMatType>
|
||||
template<typename MatType, typename ResponsesType, typename, typename>
|
||||
inline
|
||||
typename BayesianLinearRegression<ModelMatType>::ElemType
|
||||
BayesianLinearRegression<ModelMatType>::Train(
|
||||
const MatType& data,
|
||||
const ResponsesType& responses,
|
||||
const bool centerData,
|
||||
const bool scaleData,
|
||||
const size_t maxIterations)
|
||||
{
|
||||
return Train(data, responses, centerData, scaleData, maxIterations,
|
||||
this->tolerance);
|
||||
}
|
||||
|
||||
template<typename ModelMatType>
|
||||
template<typename MatType, typename ResponsesType, typename, typename>
|
||||
inline
|
||||
typename BayesianLinearRegression<ModelMatType>::ElemType
|
||||
BayesianLinearRegression<ModelMatType>::Train(
|
||||
const MatType& data,
|
||||
const ResponsesType& responses,
|
||||
const bool centerData,
|
||||
const bool scaleData,
|
||||
const size_t maxIterations,
|
||||
const double tolerance)
|
||||
{
|
||||
this->centerData = centerData;
|
||||
this->scaleData = scaleData;
|
||||
this->maxIterations = maxIterations;
|
||||
this->tolerance = tolerance;
|
||||
|
||||
ModelMatType phi;
|
||||
DenseRowType t;
|
||||
DenseVecType eigVal;
|
||||
ModelMatType eigVec;
|
||||
|
||||
// Preprocess the data. Center and scale.
|
||||
responsesOffset = CenterScaleData(data, responses, phi, t);
|
||||
@@ -49,20 +152,20 @@ inline double BayesianLinearRegression::Train(const arma::mat& data,
|
||||
}
|
||||
|
||||
// Compute this quantities once and for all.
|
||||
const arma::mat eigVecInv = inv(eigVec);
|
||||
const arma::colvec eigVecInvPhitT = eigVecInv * phi * t.t();
|
||||
const ModelMatType eigVecInv = inv(eigVec);
|
||||
const DenseVecType eigVecInvPhitT = eigVecInv * phi * t.t();
|
||||
|
||||
// Initialize the hyperparameters and begin with an infinitely broad prior.
|
||||
alpha = 1e-6;
|
||||
beta = 1 / (var(t, 1) * 0.1);
|
||||
alpha = ((ElemType) 1e-6);
|
||||
beta = ((ElemType) 1 / (var(t, 1) * 0.1));
|
||||
|
||||
unsigned short i = 0;
|
||||
double crit = 1.0;
|
||||
ElemType crit = ((ElemType) 1.0);
|
||||
|
||||
while ((crit > tolerance) && (i < maxIterations))
|
||||
while (((double) crit > tolerance) && (i < maxIterations))
|
||||
{
|
||||
double deltaAlpha = -alpha;
|
||||
double deltaBeta = -beta;
|
||||
ElemType deltaAlpha = -alpha;
|
||||
ElemType deltaBeta = -beta;
|
||||
|
||||
// Update the solution.
|
||||
omega = eigVec * diagmat(1 / (eigVal + (alpha / beta))) * eigVecInvPhitT;
|
||||
@@ -72,7 +175,7 @@ inline double BayesianLinearRegression::Train(const arma::mat& data,
|
||||
alpha = gamma / dot(omega, omega);
|
||||
|
||||
// Update beta.
|
||||
const arma::rowvec temp = t - omega.t() * phi;
|
||||
const DenseRowType temp = t - omega.t() * phi;
|
||||
beta = (data.n_cols - gamma) / dot(temp, temp);
|
||||
|
||||
// Compute the stopping criterion.
|
||||
@@ -82,56 +185,138 @@ inline double BayesianLinearRegression::Train(const arma::mat& data,
|
||||
i++;
|
||||
}
|
||||
// Compute the covariance matrix for the uncertainties later.
|
||||
matCovariance = eigVec * diagmat(1 / (beta * eigVal + alpha)) * eigVecInv;
|
||||
matCovariance = eigVec * diagmat(((ElemType) 1) / (beta * eigVal + alpha)) *
|
||||
eigVecInv;
|
||||
|
||||
return RMSE(data, responses);
|
||||
}
|
||||
|
||||
inline void BayesianLinearRegression::Predict(const arma::mat& points,
|
||||
arma::rowvec& predictions) const
|
||||
template<typename ModelMatType>
|
||||
template<typename VecType>
|
||||
inline
|
||||
typename BayesianLinearRegression<ModelMatType>::ElemType
|
||||
BayesianLinearRegression<ModelMatType>::Predict(const VecType& point) const
|
||||
{
|
||||
// Center and scale the points before applying the model.
|
||||
arma::mat matX;
|
||||
CenterScaleDataPred(points, matX);
|
||||
predictions = omega.t() * matX + responsesOffset;
|
||||
// Center and scale the point before applying the model, if needed.
|
||||
if (!centerData && !scaleData)
|
||||
return arma::dot(omega, point) + responsesOffset;
|
||||
else if (centerData && !scaleData)
|
||||
return arma::dot(omega, point - dataOffset) + responsesOffset;
|
||||
else if (!centerData && scaleData)
|
||||
return arma::dot(omega, point / dataScale) + responsesOffset;
|
||||
else
|
||||
return arma::dot(omega, (point - dataOffset) / dataScale) + responsesOffset;
|
||||
}
|
||||
|
||||
inline void BayesianLinearRegression::Predict(const arma::mat& points,
|
||||
arma::rowvec& predictions,
|
||||
arma::rowvec& std) const
|
||||
template<typename ModelMatType>
|
||||
template<typename VecType>
|
||||
inline void BayesianLinearRegression<ModelMatType>::Predict(
|
||||
const VecType& point,
|
||||
typename BayesianLinearRegression<ModelMatType>::ElemType& prediction,
|
||||
typename BayesianLinearRegression<ModelMatType>::ElemType& stddev) const
|
||||
{
|
||||
// Center and scale the points before applying the model.
|
||||
arma::mat matX;
|
||||
CenterScaleDataPred(points, matX);
|
||||
predictions = omega.t() * matX + responsesOffset;
|
||||
// Compute the standard deviation for each point.
|
||||
std = sqrt(Variance() + sum(matX % (matCovariance * matX), 0));
|
||||
prediction = Predict(point);
|
||||
|
||||
ElemType inner;
|
||||
if (!centerData && !scaleData)
|
||||
{
|
||||
stddev = std::sqrt(Variance() +
|
||||
arma::accu(point % (matCovariance * point)));
|
||||
inner = arma::accu(point % (matCovariance * point));
|
||||
}
|
||||
else if (centerData && !scaleData)
|
||||
{
|
||||
inner = arma::accu((point - dataOffset) %
|
||||
(matCovariance * (point - dataOffset)));
|
||||
}
|
||||
else if (!centerData && scaleData)
|
||||
{
|
||||
inner = arma::accu((point / dataScale) %
|
||||
(matCovariance * (point / dataScale)));
|
||||
}
|
||||
else
|
||||
{
|
||||
inner = arma::accu(((point - dataOffset) / dataScale) %
|
||||
(matCovariance * ((point - dataOffset) / dataScale)));
|
||||
}
|
||||
|
||||
stddev = std::sqrt(Variance() + inner);
|
||||
}
|
||||
|
||||
inline double BayesianLinearRegression::RMSE(
|
||||
const arma::mat& data,
|
||||
const arma::rowvec& responses) const
|
||||
template<typename ModelMatType>
|
||||
template<typename MatType, typename ResponsesType, typename>
|
||||
inline void BayesianLinearRegression<ModelMatType>::Predict(
|
||||
const MatType& points,
|
||||
ResponsesType& predictions) const
|
||||
{
|
||||
arma::rowvec predictions;
|
||||
if (!centerData && !scaleData)
|
||||
{
|
||||
predictions = omega.t() * points + responsesOffset;
|
||||
}
|
||||
else
|
||||
{
|
||||
// Center and scale the points before applying the model.
|
||||
arma::Mat<ElemType> pointsProc;
|
||||
CenterScaleDataPred(points, pointsProc);
|
||||
|
||||
predictions = omega.t() * pointsProc + responsesOffset;
|
||||
}
|
||||
}
|
||||
|
||||
template<typename ModelMatType>
|
||||
template<typename MatType, typename ResponsesType, typename>
|
||||
inline void BayesianLinearRegression<ModelMatType>::Predict(
|
||||
const MatType& points,
|
||||
ResponsesType& predictions,
|
||||
ResponsesType& std) const
|
||||
{
|
||||
if (!centerData && !scaleData)
|
||||
{
|
||||
Predict(points, predictions);
|
||||
std = arma::sqrt(Variance() + arma::sum(points %
|
||||
(matCovariance * points), 0));
|
||||
}
|
||||
else
|
||||
{
|
||||
// Center or scale data.
|
||||
arma::Mat<ElemType> pointsProc;
|
||||
CenterScaleDataPred(points, pointsProc);
|
||||
|
||||
predictions = omega.t() * pointsProc + responsesOffset;
|
||||
std = arma::sqrt(Variance() + arma::sum(pointsProc %
|
||||
(matCovariance * pointsProc), 0));
|
||||
}
|
||||
}
|
||||
|
||||
template<typename ModelMatType>
|
||||
template<typename MatType, typename ResponsesType, typename>
|
||||
inline
|
||||
typename BayesianLinearRegression<ModelMatType>::ElemType
|
||||
BayesianLinearRegression<ModelMatType>::RMSE(
|
||||
const MatType& data,
|
||||
const ResponsesType& responses) const
|
||||
{
|
||||
typename GetDenseRowType<ResponsesType>::type predictions;
|
||||
Predict(data, predictions);
|
||||
return sqrt(mean(square(responses - predictions)));
|
||||
}
|
||||
|
||||
inline double BayesianLinearRegression::CenterScaleData(
|
||||
const arma::mat& data,
|
||||
const arma::rowvec& responses,
|
||||
arma::mat& dataProc,
|
||||
arma::rowvec& responsesProc)
|
||||
template<typename ModelMatType>
|
||||
template<typename MatType, typename ResponsesType>
|
||||
inline double BayesianLinearRegression<ModelMatType>::CenterScaleData(
|
||||
const MatType& data,
|
||||
const ResponsesType& responses,
|
||||
MatType& dataProc,
|
||||
ResponsesType& responsesProc)
|
||||
{
|
||||
if (!centerData && !scaleData)
|
||||
{
|
||||
dataProc = arma::mat(const_cast<double*>(data.memptr()), data.n_rows,
|
||||
dataProc = MatType(const_cast<ElemType*>(data.memptr()), data.n_rows,
|
||||
data.n_cols, false, true);
|
||||
responsesProc = arma::rowvec(const_cast<double*>(responses.memptr()),
|
||||
responses.n_elem, false,
|
||||
true);
|
||||
responsesProc = ResponsesType(const_cast<ElemType*>(responses.memptr()),
|
||||
responses.n_elem, false,
|
||||
true);
|
||||
}
|
||||
|
||||
else if (centerData && !scaleData)
|
||||
{
|
||||
dataOffset = mean(data, 1);
|
||||
@@ -139,16 +324,14 @@ inline double BayesianLinearRegression::CenterScaleData(
|
||||
dataProc = data.each_col() - dataOffset;
|
||||
responsesProc = responses - responsesOffset;
|
||||
}
|
||||
|
||||
else if (!centerData && scaleData)
|
||||
{
|
||||
dataScale = stddev(data, 0, 1);
|
||||
dataProc = data.each_col() / dataScale;
|
||||
responsesProc = arma::rowvec(const_cast<double*>(responses.memptr()),
|
||||
responses.n_elem, false,
|
||||
true);
|
||||
responsesProc = ResponsesType(const_cast<ElemType*>(responses.memptr()),
|
||||
responses.n_elem, false,
|
||||
true);
|
||||
}
|
||||
|
||||
else
|
||||
{
|
||||
dataOffset = mean(data, 1);
|
||||
@@ -157,29 +340,28 @@ inline double BayesianLinearRegression::CenterScaleData(
|
||||
dataProc = (data.each_col() - dataOffset).each_col() / dataScale;
|
||||
responsesProc = responses - responsesOffset;
|
||||
}
|
||||
|
||||
return responsesOffset;
|
||||
}
|
||||
|
||||
inline void BayesianLinearRegression::CenterScaleDataPred(
|
||||
const arma::mat& data,
|
||||
arma::mat& dataProc) const
|
||||
template<typename ModelMatType>
|
||||
template<typename MatType, typename OutMatType>
|
||||
inline void BayesianLinearRegression<ModelMatType>::CenterScaleDataPred(
|
||||
const MatType& data,
|
||||
OutMatType& dataProc) const
|
||||
{
|
||||
if (!centerData && !scaleData)
|
||||
{
|
||||
dataProc = arma::mat(const_cast<double*>(data.memptr()), data.n_rows,
|
||||
data.n_cols, false, true);
|
||||
return; // Don't modify dataProc.
|
||||
}
|
||||
|
||||
else if (centerData && !scaleData)
|
||||
{
|
||||
dataProc = data.each_col() - dataOffset;
|
||||
}
|
||||
|
||||
else if (!centerData && scaleData)
|
||||
{
|
||||
dataProc = data.each_col() / dataScale;
|
||||
}
|
||||
|
||||
else
|
||||
{
|
||||
dataProc = (data.each_col() - dataOffset).each_col() / dataScale;
|
||||
@@ -189,22 +371,59 @@ inline void BayesianLinearRegression::CenterScaleDataPred(
|
||||
/**
|
||||
* Serialize the Bayesian linear regression model.
|
||||
*/
|
||||
template<typename ModelMatType>
|
||||
template<typename Archive>
|
||||
void BayesianLinearRegression::serialize(Archive& ar,
|
||||
const uint32_t /* version */)
|
||||
void BayesianLinearRegression<ModelMatType>::serialize(Archive& ar,
|
||||
const uint32_t version)
|
||||
{
|
||||
ar(CEREAL_NVP(centerData));
|
||||
ar(CEREAL_NVP(scaleData));
|
||||
ar(CEREAL_NVP(maxIterations));
|
||||
ar(CEREAL_NVP(tolerance));
|
||||
ar(CEREAL_NVP(dataOffset));
|
||||
ar(CEREAL_NVP(dataScale));
|
||||
ar(CEREAL_NVP(responsesOffset));
|
||||
ar(CEREAL_NVP(alpha));
|
||||
ar(CEREAL_NVP(beta));
|
||||
ar(CEREAL_NVP(gamma));
|
||||
ar(CEREAL_NVP(omega));
|
||||
ar(CEREAL_NVP(matCovariance));
|
||||
|
||||
// In older versions, dataOffset and dataScale were of type arma::colvec,
|
||||
// responsesOffset, alpha, beta, and gamma were of type double, omega was of
|
||||
// type arma::colvec, and matCovariance was of type arma::mat.
|
||||
if (cereal::is_loading<Archive>() && version == 0)
|
||||
{
|
||||
arma::colvec colvecTmp;
|
||||
ar(cereal::make_nvp("dataOffset", colvecTmp));
|
||||
dataOffset = arma::conv_to<DenseVecType>::from(colvecTmp);
|
||||
|
||||
ar(cereal::make_nvp("dataScale", colvecTmp));
|
||||
dataScale = arma::conv_to<DenseVecType>::from(colvecTmp);
|
||||
|
||||
double dblTmp;
|
||||
ar(cereal::make_nvp("responsesOffset", dblTmp));
|
||||
responsesOffset = (ElemType) dblTmp;
|
||||
|
||||
ar(cereal::make_nvp("alpha", dblTmp));
|
||||
alpha = (ElemType) dblTmp;
|
||||
|
||||
ar(cereal::make_nvp("beta", dblTmp));
|
||||
beta = (ElemType) dblTmp;
|
||||
|
||||
ar(cereal::make_nvp("gamma", dblTmp));
|
||||
gamma = (ElemType) dblTmp;
|
||||
|
||||
ar(cereal::make_nvp("omega", colvecTmp));
|
||||
omega = arma::conv_to<DenseVecType>::from(colvecTmp);
|
||||
|
||||
arma::mat matTmp;
|
||||
ar(cereal::make_nvp("matCovariance", matTmp));
|
||||
matCovariance = arma::conv_to<ModelMatType>::from(matCovariance);
|
||||
}
|
||||
else
|
||||
{
|
||||
ar(CEREAL_NVP(dataOffset));
|
||||
ar(CEREAL_NVP(dataScale));
|
||||
ar(CEREAL_NVP(responsesOffset));
|
||||
ar(CEREAL_NVP(alpha));
|
||||
ar(CEREAL_NVP(beta));
|
||||
ar(CEREAL_NVP(gamma));
|
||||
ar(CEREAL_NVP(omega));
|
||||
ar(CEREAL_NVP(matCovariance));
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace mlpack
|
||||
|
||||
@@ -109,10 +109,10 @@ PARAM_MATRIX_IN("input", "Matrix of covariates (X).", "i");
|
||||
|
||||
PARAM_ROW_IN("responses", "Matrix of responses/observations (y).", "r");
|
||||
|
||||
PARAM_MODEL_IN(BayesianLinearRegression, "input_model", "Trained "
|
||||
PARAM_MODEL_IN(BayesianLinearRegression<>, "input_model", "Trained "
|
||||
"BayesianLinearRegression model to use.", "m");
|
||||
|
||||
PARAM_MODEL_OUT(BayesianLinearRegression, "output_model", "Output "
|
||||
PARAM_MODEL_OUT(BayesianLinearRegression<>, "output_model", "Output "
|
||||
"BayesianLinearRegression model.", "M");
|
||||
|
||||
PARAM_MATRIX_IN("test", "Matrix containing points to regress on (test "
|
||||
@@ -149,12 +149,12 @@ void BINDING_FUNCTION(util::Params& params, util::Timers& timers)
|
||||
// Ignore out_predictions unless test is specified.
|
||||
ReportIgnoredParam(params, {{"test", false}}, "predictions");
|
||||
|
||||
BayesianLinearRegression* bayesLinReg;
|
||||
BayesianLinearRegression<>* bayesLinReg;
|
||||
if (params.Has("input"))
|
||||
{
|
||||
Log::Info << "Input given; model will be trained." << std::endl;
|
||||
// Initialize the object.
|
||||
bayesLinReg = new BayesianLinearRegression(center, scale);
|
||||
bayesLinReg = new BayesianLinearRegression<>(center, scale);
|
||||
|
||||
// Load covariates.
|
||||
mat matX = std::move(params.Get<arma::mat>("input"));
|
||||
@@ -180,7 +180,7 @@ void BINDING_FUNCTION(util::Params& params, util::Timers& timers)
|
||||
}
|
||||
else // We must have --input_model_file.
|
||||
{
|
||||
bayesLinReg = params.Get<BayesianLinearRegression*>("input_model");
|
||||
bayesLinReg = params.Get<BayesianLinearRegression<>*>("input_model");
|
||||
}
|
||||
|
||||
if (params.Has("test"))
|
||||
@@ -209,5 +209,5 @@ void BINDING_FUNCTION(util::Params& params, util::Timers& timers)
|
||||
params.Get<arma::mat>("predictions") = std::move(predictions);
|
||||
}
|
||||
|
||||
params.Get<BayesianLinearRegression*>("output_model") = bayesLinReg;
|
||||
params.Get<BayesianLinearRegression<>*>("output_model") = bayesLinReg;
|
||||
}
|
||||
|
||||
@@ -146,6 +146,10 @@ template<typename FitnessFunction>
|
||||
using HoeffdingDoubleNumericSplit = HoeffdingNumericSplit<FitnessFunction,
|
||||
double>;
|
||||
|
||||
template<typename FitnessFunction>
|
||||
using HoeffdingFloatNumericSplit = HoeffdingNumericSplit<FitnessFunction,
|
||||
float>;
|
||||
|
||||
} // namespace mlpack
|
||||
|
||||
// Include implementation.
|
||||
|
||||
@@ -72,6 +72,119 @@ class HoeffdingTree
|
||||
//! Allow access to the categorical split type.
|
||||
typedef CategoricalSplitType<FitnessFunction> CategoricalSplit;
|
||||
|
||||
/**
|
||||
* Construct a Hoeffding tree with no data and no information. Be sure to
|
||||
* call Train() before trying to use the tree.
|
||||
*/
|
||||
HoeffdingTree();
|
||||
|
||||
/**
|
||||
* Construct the Hoeffding tree with the given parameters for training on
|
||||
* numerical data, but training on no data. The dimensionMappings parameter
|
||||
* is only used if it is desired that this node does not create its own
|
||||
* dimensionMappings object (for instance, if this is a child of another node
|
||||
* in the tree).
|
||||
*
|
||||
* @param numClasses Number of classes in the dataset.
|
||||
* @param successProbability Probability of success required in Hoeffding
|
||||
* bound before a split can happen.
|
||||
* @param maxSamples Maximum number of samples before a split is forced.
|
||||
* @param checkInterval Number of samples required before each split check.
|
||||
* @param minSamples If the node has seen this many points or fewer, no split
|
||||
* will be allowed.
|
||||
* @param categoricalSplitIn Optional instantiated categorical split object.
|
||||
* @param numericSplitIn Optional instantiated numeric split object.
|
||||
* @param dimensionMappings Mappings from dimension indices to positions in
|
||||
* numeric and categorical split vectors. If left NULL, a new one will
|
||||
* be created.
|
||||
*/
|
||||
HoeffdingTree(const size_t dimensionality,
|
||||
const size_t numClasses,
|
||||
const double successProbability = 0.95,
|
||||
const size_t maxSamples = 0,
|
||||
const size_t checkInterval = 100,
|
||||
const size_t minSamples = 100,
|
||||
const CategoricalSplitType<FitnessFunction>& categoricalSplitIn
|
||||
= CategoricalSplitType<FitnessFunction>(0, 0),
|
||||
const NumericSplitType<FitnessFunction>& numericSplitIn =
|
||||
NumericSplitType<FitnessFunction>(0),
|
||||
std::unordered_map<size_t, std::pair<size_t, size_t>>*
|
||||
dimensionMappings = NULL);
|
||||
|
||||
/**
|
||||
* Construct the Hoeffding tree with the given parameters, but training on no
|
||||
* data. The dimensionMappings parameter is only used if it is desired that
|
||||
* this node does not create its own dimensionMappings object (for instance,
|
||||
* if this is a child of another node in the tree).
|
||||
*
|
||||
* @param datasetInfo Information on the dataset (types of each feature).
|
||||
* @param numClasses Number of classes in the dataset.
|
||||
* @param successProbability Probability of success required in Hoeffding
|
||||
* bound before a split can happen.
|
||||
* @param maxSamples Maximum number of samples before a split is forced.
|
||||
* @param checkInterval Number of samples required before each split check.
|
||||
* @param minSamples If the node has seen this many points or fewer, no split
|
||||
* will be allowed.
|
||||
* @param categoricalSplitIn Optional instantiated categorical split object.
|
||||
* @param numericSplitIn Optional instantiated numeric split object.
|
||||
* @param dimensionMappings Mappings from dimension indices to positions in
|
||||
* numeric and categorical split vectors. If left NULL, a new one will
|
||||
* be created.
|
||||
* @param copyDatasetInfo If true, then a copy of the datasetInfo will be
|
||||
* made.
|
||||
*/
|
||||
HoeffdingTree(const data::DatasetInfo& datasetInfo,
|
||||
const size_t numClasses,
|
||||
const double successProbability = 0.95,
|
||||
const size_t maxSamples = 0,
|
||||
const size_t checkInterval = 100,
|
||||
const size_t minSamples = 100,
|
||||
const CategoricalSplitType<FitnessFunction>& categoricalSplitIn
|
||||
= CategoricalSplitType<FitnessFunction>(0, 0),
|
||||
const NumericSplitType<FitnessFunction>& numericSplitIn =
|
||||
NumericSplitType<FitnessFunction>(0),
|
||||
std::unordered_map<size_t, std::pair<size_t, size_t>>*
|
||||
dimensionMappings = NULL,
|
||||
const bool copyDatasetInfo = true);
|
||||
|
||||
/**
|
||||
* Construct the Hoeffding tree with the given parameters and given training
|
||||
* data, where the training data contains only numerical features. The tree
|
||||
* may be trained either in batch mode (which looks at all points before
|
||||
* splitting, and propagates these points to the created children for further
|
||||
* training), or in streaming mode, where each point is only considered once.
|
||||
* (In general, batch mode will give better-performing trees, but will have
|
||||
* higher memory and runtime costs for the same dataset.)
|
||||
*
|
||||
* @param data Dataset to train on.
|
||||
* @param labels Labels of each point in the dataset.
|
||||
* @param numClasses Number of classes in the dataset.
|
||||
* @param batchTraining Whether or not to train in batch.
|
||||
* @param successProbability Probability of success required in Hoeffding
|
||||
* bounds before a split can happen.
|
||||
* @param maxSamples Maximum number of samples before a split is forced (0
|
||||
* never forces a split); ignored in batch training mode.
|
||||
* @param checkInterval Number of samples required before each split; ignored
|
||||
* in batch training mode.
|
||||
* @param minSamples If the node has seen this many points or fewer, no split
|
||||
* will be allowed.
|
||||
* @param categoricalSplitIn Optional instantiated categorical split object.
|
||||
* @param numericSplitIn Optional instantiated numeric split object.
|
||||
*/
|
||||
template<typename MatType>
|
||||
HoeffdingTree(const MatType& data,
|
||||
const arma::Row<size_t>& labels,
|
||||
const size_t numClasses,
|
||||
const bool batchTraining = true,
|
||||
const double successProbability = 0.95,
|
||||
const size_t maxSamples = 0,
|
||||
const size_t checkInterval = 100,
|
||||
const size_t minSamples = 100,
|
||||
const CategoricalSplitType<FitnessFunction>& categoricalSplitIn
|
||||
= CategoricalSplitType<FitnessFunction>(0, 0),
|
||||
const NumericSplitType<FitnessFunction>& numericSplitIn =
|
||||
NumericSplitType<FitnessFunction>(0));
|
||||
|
||||
/**
|
||||
* Construct the Hoeffding tree with the given parameters and given training
|
||||
* data. The tree may be trained either in batch mode (which looks at all
|
||||
@@ -111,48 +224,6 @@ class HoeffdingTree
|
||||
const NumericSplitType<FitnessFunction>& numericSplitIn =
|
||||
NumericSplitType<FitnessFunction>(0));
|
||||
|
||||
/**
|
||||
* Construct the Hoeffding tree with the given parameters, but training on no
|
||||
* data. The dimensionMappings parameter is only used if it is desired that
|
||||
* this node does not create its own dimensionMappings object (for instance,
|
||||
* if this is a child of another node in the tree).
|
||||
*
|
||||
* @param numClasses Number of classes in the dataset.
|
||||
* @param datasetInfo Information on the dataset (types of each feature).
|
||||
* @param successProbability Probability of success required in Hoeffding
|
||||
* bound before a split can happen.
|
||||
* @param maxSamples Maximum number of samples before a split is forced.
|
||||
* @param checkInterval Number of samples required before each split check.
|
||||
* @param minSamples If the node has seen this many points or fewer, no split
|
||||
* will be allowed.
|
||||
* @param dimensionMappings Mappings from dimension indices to positions in
|
||||
* numeric and categorical split vectors. If left NULL, a new one will
|
||||
* be created.
|
||||
* @param copyDatasetInfo If true, then a copy of the datasetInfo will be
|
||||
* made.
|
||||
* @param categoricalSplitIn Optional instantiated categorical split object.
|
||||
* @param numericSplitIn Optional instantiated numeric split object.
|
||||
*/
|
||||
HoeffdingTree(const data::DatasetInfo& datasetInfo,
|
||||
const size_t numClasses,
|
||||
const double successProbability = 0.95,
|
||||
const size_t maxSamples = 0,
|
||||
const size_t checkInterval = 100,
|
||||
const size_t minSamples = 100,
|
||||
const CategoricalSplitType<FitnessFunction>& categoricalSplitIn
|
||||
= CategoricalSplitType<FitnessFunction>(0, 0),
|
||||
const NumericSplitType<FitnessFunction>& numericSplitIn =
|
||||
NumericSplitType<FitnessFunction>(0),
|
||||
std::unordered_map<size_t, std::pair<size_t, size_t>>*
|
||||
dimensionMappings = NULL,
|
||||
const bool copyDatasetInfo = true);
|
||||
|
||||
/**
|
||||
* Construct a Hoeffding tree with no data and no information. Be sure to
|
||||
* call Train() before trying to use the tree.
|
||||
*/
|
||||
HoeffdingTree();
|
||||
|
||||
/**
|
||||
* Copy another tree (warning: this will duplicate the tree entirely, and may
|
||||
* use a lot of memory. Make sure it's what you want before you do it).
|
||||
@@ -194,44 +265,136 @@ class HoeffdingTree
|
||||
*
|
||||
* Note that the tree will be automatically reset if the dimensionality of
|
||||
* `data` does not match the dimensionality that the tree was currently
|
||||
* trained with. The tree will also be reset if `numClasses` is passed.
|
||||
* trained with. The tree will also be reset if `numClasses` is passed and
|
||||
* differs from the existing setting.
|
||||
*
|
||||
* @param data Data points to train on.
|
||||
* @param labels Labels of data points.
|
||||
* @param batchTraining If true, perform training in batch.
|
||||
* @param resetTree If true, reset the tree to an empty tree before training.
|
||||
* @param numClasses The number of classes in `labels`. Passing this will
|
||||
* reset the tree. If not given and `resetTree` is `true`, then the
|
||||
* number of classes will be computed from `labels`.
|
||||
* @param batchTraining If true, perform training in batch.
|
||||
* @param successProbability Probability of success required in Hoeffding
|
||||
* bounds before a split can happen.
|
||||
* @param maxSamples Maximum number of samples before a split is forced (0
|
||||
* never forces a split); ignored in batch training mode.
|
||||
* @param checkInterval Number of samples required before each split; ignored
|
||||
* in batch training mode.
|
||||
* @param minSamples If the node has seen this many points or fewer, no split
|
||||
* will be allowed.
|
||||
*/
|
||||
// Many overloads needed here until we have std::optional.
|
||||
template<typename MatType>
|
||||
void Train(const MatType& data,
|
||||
const arma::Row<size_t>& labels,
|
||||
const bool batchTraining = true,
|
||||
const bool resetTree = false,
|
||||
const size_t numClasses = 0);
|
||||
const size_t numClasses = 0,
|
||||
const bool batchTraining = true);
|
||||
|
||||
template<typename MatType>
|
||||
void Train(const MatType& data,
|
||||
const arma::Row<size_t>& labels,
|
||||
const size_t numClasses,
|
||||
const bool batchTraining,
|
||||
const double successProbability);
|
||||
|
||||
template<typename MatType>
|
||||
void Train(const MatType& data,
|
||||
const arma::Row<size_t>& labels,
|
||||
const size_t numClasses,
|
||||
const bool batchTraining,
|
||||
const double successProbability,
|
||||
const size_t maxSamples);
|
||||
|
||||
template<typename MatType>
|
||||
void Train(const MatType& data,
|
||||
const arma::Row<size_t>& labels,
|
||||
const size_t numClasses,
|
||||
const bool batchTraining,
|
||||
const double successProbability,
|
||||
const size_t maxSamples,
|
||||
const size_t checkInterval);
|
||||
|
||||
template<typename MatType>
|
||||
void Train(const MatType& data,
|
||||
const arma::Row<size_t>& labels,
|
||||
const size_t numClasses,
|
||||
const bool batchTraining,
|
||||
const double successProbability,
|
||||
const size_t maxSamples,
|
||||
const size_t checkInterval,
|
||||
const size_t minSamples);
|
||||
|
||||
/**
|
||||
* Train on a set of points, either in streaming mode or in batch mode, with
|
||||
* the given labels and the given `DatasetInfo`. This will reset the tree.
|
||||
* This only needs to be called when the `DatasetInfo` has changed---if you
|
||||
* are training incrementally but have already passed the DatasetInfo once,
|
||||
* use the overload of `Train()` that does not take a `DatasetInfo` and make
|
||||
* sure `resetTree` is set to `false`.
|
||||
* the given labels. If `resetTree` is set to `true`, then reset the state of
|
||||
* the tree to an empty tree before training.
|
||||
*
|
||||
* Note that the tree will be automatically reset if the dimensionality of
|
||||
* `data` does not match the dimensionality that the tree was currently
|
||||
* trained with. The tree will also be reset if `numClasses` is passed and
|
||||
* differs from the existing setting.
|
||||
*
|
||||
* @param data Data points to train on.
|
||||
* @param info DatasetInfo object with information about each dimension.
|
||||
* @param datasetInfo Information on the dataset (types of each feature).
|
||||
* @param labels Labels of data points.
|
||||
* @param numClasses The number of classes in `labels`. Passing this will
|
||||
* reset the tree. If not given and `resetTree` is `true`, then the
|
||||
* number of classes will be computed from `labels`.
|
||||
* @param batchTraining If true, perform training in batch.
|
||||
* @param numClasses Number of classes in `labels`. If not specified, it is
|
||||
* computed from `labels`.
|
||||
* @param successProbability Probability of success required in Hoeffding
|
||||
* bounds before a split can happen.
|
||||
* @param maxSamples Maximum number of samples before a split is forced (0
|
||||
* never forces a split); ignored in batch training mode.
|
||||
* @param checkInterval Number of samples required before each split; ignored
|
||||
* in batch training mode.
|
||||
* @param minSamples If the node has seen this many points or fewer, no split
|
||||
* will be allowed.
|
||||
*/
|
||||
// Many overloads needed here until we have std::optional.
|
||||
template<typename MatType>
|
||||
void Train(const MatType& data,
|
||||
const data::DatasetInfo& info,
|
||||
const arma::Row<size_t>& labels,
|
||||
const bool batchTraining = true,
|
||||
const size_t numClasses = 0);
|
||||
const size_t numClasses = 0,
|
||||
const bool batchTraining = true);
|
||||
|
||||
template<typename MatType>
|
||||
void Train(const MatType& data,
|
||||
const data::DatasetInfo& info,
|
||||
const arma::Row<size_t>& labels,
|
||||
const size_t numClasses,
|
||||
const bool batchTraining,
|
||||
const double successProbability);
|
||||
|
||||
template<typename MatType>
|
||||
void Train(const MatType& data,
|
||||
const data::DatasetInfo& info,
|
||||
const arma::Row<size_t>& labels,
|
||||
const size_t numClasses,
|
||||
const bool batchTraining,
|
||||
const double successProbability,
|
||||
const size_t maxSamples);
|
||||
|
||||
template<typename MatType>
|
||||
void Train(const MatType& data,
|
||||
const data::DatasetInfo& info,
|
||||
const arma::Row<size_t>& labels,
|
||||
const size_t numClasses,
|
||||
const bool batchTraining,
|
||||
const double successProbability,
|
||||
const size_t maxSamples,
|
||||
const size_t checkInterval);
|
||||
|
||||
template<typename MatType>
|
||||
void Train(const MatType& data,
|
||||
const data::DatasetInfo& info,
|
||||
const arma::Row<size_t>& labels,
|
||||
const size_t numClasses,
|
||||
const bool batchTraining,
|
||||
const double successProbability,
|
||||
const size_t maxSamples,
|
||||
const size_t checkInterval,
|
||||
const size_t minSamples);
|
||||
|
||||
/**
|
||||
* Train on a single point in streaming mode, with the given label. The tree
|
||||
@@ -291,6 +454,12 @@ class HoeffdingTree
|
||||
//! Modify the number of samples before a split check is performed.
|
||||
void CheckInterval(const size_t checkInterval);
|
||||
|
||||
//! Get the number of points seen so far.
|
||||
size_t NumSamples() const { return numSamples; }
|
||||
|
||||
//! Get the number of classes the tree is trained on.
|
||||
size_t NumClasses() const { return numClasses; }
|
||||
|
||||
/**
|
||||
* Given a point and that this node is not a leaf, calculate the index of the
|
||||
* child node this point would go towards. This method is primarily used by
|
||||
@@ -301,6 +470,9 @@ class HoeffdingTree
|
||||
template<typename VecType>
|
||||
size_t CalculateDirection(const VecType& point) const;
|
||||
|
||||
//! Get the size of the Hoeffding Tree.
|
||||
size_t NumDescendants() const;
|
||||
|
||||
/**
|
||||
* Classify the given point, using this node and the entire (sub)tree beneath
|
||||
* it. The predicted label is returned.
|
||||
@@ -311,9 +483,6 @@ class HoeffdingTree
|
||||
template<typename VecType>
|
||||
size_t Classify(const VecType& point) const;
|
||||
|
||||
//! Get the size of the Hoeffding Tree.
|
||||
size_t NumDescendants() const;
|
||||
|
||||
/**
|
||||
* Classify the given point and also return an estimate of the probability
|
||||
* that the prediction is correct. (This estimate is simply the probability
|
||||
@@ -360,6 +529,23 @@ class HoeffdingTree
|
||||
*/
|
||||
void CreateChildren();
|
||||
|
||||
/**
|
||||
* Reset the tree, keeping the number of classes and dimension information
|
||||
* intact.
|
||||
*/
|
||||
void Reset();
|
||||
|
||||
/**
|
||||
* Reset the tree, setting a new number of classes and dimensionality. This
|
||||
* assumes all dimensions are numeric.
|
||||
*/
|
||||
void Reset(const size_t dimensionality, const size_t numClasses);
|
||||
|
||||
/**
|
||||
* Reset the tree, setting a new number of classes and a new datasetInfo.
|
||||
*/
|
||||
void Reset(const data::DatasetInfo& datasetInfo, const size_t numClasses);
|
||||
|
||||
//! Serialize the split.
|
||||
template<typename Archive>
|
||||
void serialize(Archive& ar, const uint32_t /* version */);
|
||||
|
||||
@@ -14,38 +14,64 @@
|
||||
|
||||
// In case it hasn't been included yet.
|
||||
#include "hoeffding_tree.hpp"
|
||||
#include <stack>
|
||||
|
||||
namespace mlpack {
|
||||
|
||||
template<typename FitnessFunction,
|
||||
template<typename> class NumericSplitType,
|
||||
template<typename> class CategoricalSplitType>
|
||||
template<typename MatType>
|
||||
HoeffdingTree<
|
||||
FitnessFunction,
|
||||
NumericSplitType,
|
||||
CategoricalSplitType
|
||||
>::HoeffdingTree(const MatType& data,
|
||||
const data::DatasetInfo& datasetInfoIn,
|
||||
const arma::Row<size_t>& labels,
|
||||
>::HoeffdingTree() :
|
||||
dimensionMappings(
|
||||
new std::unordered_map<size_t, std::pair<size_t, size_t>>()),
|
||||
ownsMappings(true),
|
||||
numSamples(0),
|
||||
numClasses(0),
|
||||
maxSamples(size_t(-1)),
|
||||
checkInterval(100),
|
||||
minSamples(100),
|
||||
datasetInfo(new data::DatasetInfo()),
|
||||
ownsInfo(true),
|
||||
successProbability(0.95),
|
||||
splitDimension(size_t(-1)),
|
||||
majorityClass(0),
|
||||
majorityProbability(0.0),
|
||||
categoricalSplit(0),
|
||||
numericSplit()
|
||||
{
|
||||
// Nothing to do.
|
||||
}
|
||||
|
||||
template<typename FitnessFunction,
|
||||
template<typename> class NumericSplitType,
|
||||
template<typename> class CategoricalSplitType>
|
||||
HoeffdingTree<
|
||||
FitnessFunction,
|
||||
NumericSplitType,
|
||||
CategoricalSplitType
|
||||
>::HoeffdingTree(const size_t dimensionality,
|
||||
const size_t numClasses,
|
||||
const bool batchTraining,
|
||||
const double successProbability,
|
||||
const size_t maxSamples,
|
||||
const size_t checkInterval,
|
||||
const size_t minSamples,
|
||||
const CategoricalSplitType<FitnessFunction>&
|
||||
categoricalSplitIn,
|
||||
const NumericSplitType<FitnessFunction>& numericSplitIn) :
|
||||
dimensionMappings(NULL),
|
||||
ownsMappings(false),
|
||||
const NumericSplitType<FitnessFunction>& numericSplitIn,
|
||||
std::unordered_map<size_t, std::pair<size_t, size_t>>*
|
||||
dimensionMappingsIn) :
|
||||
dimensionMappings((dimensionMappingsIn != NULL) ? dimensionMappingsIn :
|
||||
new std::unordered_map<size_t, std::pair<size_t, size_t>>()),
|
||||
ownsMappings(dimensionMappingsIn == NULL),
|
||||
numSamples(0),
|
||||
numClasses(numClasses),
|
||||
maxSamples((maxSamples == 0) ? size_t(-1) : maxSamples),
|
||||
checkInterval(checkInterval),
|
||||
minSamples(minSamples),
|
||||
datasetInfo(new data::DatasetInfo(datasetInfoIn)),
|
||||
datasetInfo(new data::DatasetInfo(dimensionality)),
|
||||
ownsInfo(true),
|
||||
successProbability(successProbability),
|
||||
splitDimension(size_t(-1)),
|
||||
@@ -54,11 +80,20 @@ HoeffdingTree<
|
||||
categoricalSplit(0),
|
||||
numericSplit()
|
||||
{
|
||||
// Reset the tree.
|
||||
ResetTree(categoricalSplitIn, numericSplitIn);
|
||||
|
||||
// Now train.
|
||||
Train(data, labels, batchTraining);
|
||||
// Do we need to generate the mappings too?
|
||||
if (ownsMappings)
|
||||
{
|
||||
ResetTree(categoricalSplitIn, numericSplitIn);
|
||||
}
|
||||
else
|
||||
{
|
||||
// All dimensions are numeric.
|
||||
for (size_t i = 0; i < datasetInfo->Dimensionality(); ++i)
|
||||
{
|
||||
numericSplits.push_back(NumericSplitType<FitnessFunction>(numClasses,
|
||||
numericSplitIn));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
template<typename FitnessFunction,
|
||||
@@ -124,29 +159,86 @@ HoeffdingTree<
|
||||
template<typename FitnessFunction,
|
||||
template<typename> class NumericSplitType,
|
||||
template<typename> class CategoricalSplitType>
|
||||
template<typename MatType>
|
||||
HoeffdingTree<
|
||||
FitnessFunction,
|
||||
NumericSplitType,
|
||||
CategoricalSplitType
|
||||
>::HoeffdingTree() :
|
||||
dimensionMappings(
|
||||
new std::unordered_map<size_t, std::pair<size_t, size_t>>()),
|
||||
ownsMappings(true),
|
||||
>::HoeffdingTree(const MatType& data,
|
||||
const arma::Row<size_t>& labels,
|
||||
const size_t numClasses,
|
||||
const bool batchTraining,
|
||||
const double successProbability,
|
||||
const size_t maxSamples,
|
||||
const size_t checkInterval,
|
||||
const size_t minSamples,
|
||||
const CategoricalSplitType<FitnessFunction>&
|
||||
categoricalSplitIn,
|
||||
const NumericSplitType<FitnessFunction>& numericSplitIn) :
|
||||
dimensionMappings(NULL),
|
||||
ownsMappings(false),
|
||||
numSamples(0),
|
||||
numClasses(0),
|
||||
maxSamples(size_t(-1)),
|
||||
checkInterval(100),
|
||||
minSamples(100),
|
||||
datasetInfo(new data::DatasetInfo()),
|
||||
numClasses(numClasses),
|
||||
maxSamples((maxSamples == 0) ? size_t(-1) : maxSamples),
|
||||
checkInterval(checkInterval),
|
||||
minSamples(minSamples),
|
||||
datasetInfo(new data::DatasetInfo(data.n_rows)),
|
||||
ownsInfo(true),
|
||||
successProbability(0.95),
|
||||
successProbability(successProbability),
|
||||
splitDimension(size_t(-1)),
|
||||
majorityClass(0),
|
||||
majorityProbability(0.0),
|
||||
categoricalSplit(0),
|
||||
numericSplit()
|
||||
{
|
||||
// Nothing to do.
|
||||
// Reset the tree.
|
||||
ResetTree(categoricalSplitIn, numericSplitIn);
|
||||
|
||||
// Now train.
|
||||
Train(data, labels, numClasses, batchTraining);
|
||||
}
|
||||
|
||||
template<typename FitnessFunction,
|
||||
template<typename> class NumericSplitType,
|
||||
template<typename> class CategoricalSplitType>
|
||||
template<typename MatType>
|
||||
HoeffdingTree<
|
||||
FitnessFunction,
|
||||
NumericSplitType,
|
||||
CategoricalSplitType
|
||||
>::HoeffdingTree(const MatType& data,
|
||||
const data::DatasetInfo& datasetInfoIn,
|
||||
const arma::Row<size_t>& labels,
|
||||
const size_t numClasses,
|
||||
const bool batchTraining,
|
||||
const double successProbability,
|
||||
const size_t maxSamples,
|
||||
const size_t checkInterval,
|
||||
const size_t minSamples,
|
||||
const CategoricalSplitType<FitnessFunction>&
|
||||
categoricalSplitIn,
|
||||
const NumericSplitType<FitnessFunction>& numericSplitIn) :
|
||||
dimensionMappings(NULL),
|
||||
ownsMappings(false),
|
||||
numSamples(0),
|
||||
numClasses(numClasses),
|
||||
maxSamples((maxSamples == 0) ? size_t(-1) : maxSamples),
|
||||
checkInterval(checkInterval),
|
||||
minSamples(minSamples),
|
||||
datasetInfo(new data::DatasetInfo(datasetInfoIn)),
|
||||
ownsInfo(true),
|
||||
successProbability(successProbability),
|
||||
splitDimension(size_t(-1)),
|
||||
majorityClass(0),
|
||||
majorityProbability(0.0),
|
||||
categoricalSplit(0),
|
||||
numericSplit()
|
||||
{
|
||||
// Reset the tree.
|
||||
ResetTree(categoricalSplitIn, numericSplitIn);
|
||||
|
||||
// Now train.
|
||||
Train(data, labels, numClasses, batchTraining);
|
||||
}
|
||||
|
||||
// Copy constructor.
|
||||
@@ -336,7 +428,6 @@ HoeffdingTree<FitnessFunction, NumericSplitType, CategoricalSplitType>::
|
||||
delete children[i];
|
||||
}
|
||||
|
||||
//! Train on a set of points.
|
||||
template<typename FitnessFunction,
|
||||
template<typename> class NumericSplitType,
|
||||
template<typename> class CategoricalSplitType>
|
||||
@@ -347,14 +438,96 @@ void HoeffdingTree<
|
||||
CategoricalSplitType
|
||||
>::Train(const MatType& data,
|
||||
const arma::Row<size_t>& labels,
|
||||
const bool batchTraining,
|
||||
const bool resetTree,
|
||||
const size_t numClassesIn)
|
||||
const size_t numClasses,
|
||||
const bool batchTraining)
|
||||
{
|
||||
Train(data, labels, numClasses, batchTraining, this->successProbability,
|
||||
this->maxSamples, this->checkInterval, this->minSamples);
|
||||
}
|
||||
|
||||
template<typename FitnessFunction,
|
||||
template<typename> class NumericSplitType,
|
||||
template<typename> class CategoricalSplitType>
|
||||
template<typename MatType>
|
||||
void HoeffdingTree<
|
||||
FitnessFunction,
|
||||
NumericSplitType,
|
||||
CategoricalSplitType
|
||||
>::Train(const MatType& data,
|
||||
const arma::Row<size_t>& labels,
|
||||
const size_t numClasses,
|
||||
const bool batchTraining,
|
||||
const double successProbability)
|
||||
{
|
||||
Train(data, labels, numClasses, batchTraining, successProbability,
|
||||
this->maxSamples, this->checkInterval, this->minSamples);
|
||||
}
|
||||
|
||||
template<typename FitnessFunction,
|
||||
template<typename> class NumericSplitType,
|
||||
template<typename> class CategoricalSplitType>
|
||||
template<typename MatType>
|
||||
void HoeffdingTree<
|
||||
FitnessFunction,
|
||||
NumericSplitType,
|
||||
CategoricalSplitType
|
||||
>::Train(const MatType& data,
|
||||
const arma::Row<size_t>& labels,
|
||||
const size_t numClasses,
|
||||
const bool batchTraining,
|
||||
const double successProbability,
|
||||
const size_t maxSamples)
|
||||
{
|
||||
Train(data, labels, numClasses, batchTraining, successProbability, maxSamples,
|
||||
this->checkInterval, this->minSamples);
|
||||
}
|
||||
|
||||
template<typename FitnessFunction,
|
||||
template<typename> class NumericSplitType,
|
||||
template<typename> class CategoricalSplitType>
|
||||
template<typename MatType>
|
||||
void HoeffdingTree<
|
||||
FitnessFunction,
|
||||
NumericSplitType,
|
||||
CategoricalSplitType
|
||||
>::Train(const MatType& data,
|
||||
const arma::Row<size_t>& labels,
|
||||
const size_t numClasses,
|
||||
const bool batchTraining,
|
||||
const double successProbability,
|
||||
const size_t maxSamples,
|
||||
const size_t checkInterval)
|
||||
{
|
||||
Train(data, labels, numClasses, batchTraining, successProbability, maxSamples,
|
||||
checkInterval, this->minSamples);
|
||||
}
|
||||
|
||||
template<typename FitnessFunction,
|
||||
template<typename> class NumericSplitType,
|
||||
template<typename> class CategoricalSplitType>
|
||||
template<typename MatType>
|
||||
void HoeffdingTree<
|
||||
FitnessFunction,
|
||||
NumericSplitType,
|
||||
CategoricalSplitType
|
||||
>::Train(const MatType& data,
|
||||
const arma::Row<size_t>& labels,
|
||||
const size_t numClasses,
|
||||
const bool batchTraining,
|
||||
const double successProbability,
|
||||
const size_t maxSamples,
|
||||
const size_t checkInterval,
|
||||
const size_t minSamples)
|
||||
{
|
||||
this->successProbability = successProbability;
|
||||
this->maxSamples = maxSamples;
|
||||
this->checkInterval = checkInterval;
|
||||
this->minSamples = minSamples;
|
||||
|
||||
// We need to reset the tree either if the user asked for it, or if they
|
||||
// passed data whose dimensionality is different than our datasetInfo object.
|
||||
if (resetTree || data.n_rows != datasetInfo->Dimensionality() ||
|
||||
numClassesIn != 0)
|
||||
if (data.n_rows != datasetInfo->Dimensionality() ||
|
||||
(numClasses != 0 && numClasses != this->numClasses))
|
||||
{
|
||||
// Create a new datasetInfo, which assumes that all features are numeric.
|
||||
if (ownsInfo)
|
||||
@@ -363,7 +536,14 @@ void HoeffdingTree<
|
||||
ownsInfo = true;
|
||||
|
||||
// Set the number of classes correctly.
|
||||
numClasses = (numClassesIn != 0) ? numClassesIn : arma::max(labels) + 1;
|
||||
if (numClasses != 0)
|
||||
this->numClasses = numClasses;
|
||||
|
||||
if (this->numClasses == 0)
|
||||
{
|
||||
throw std::invalid_argument("HoeffdingTree::Train(): must specify number "
|
||||
"of classes!");
|
||||
}
|
||||
|
||||
ResetTree();
|
||||
}
|
||||
@@ -371,7 +551,6 @@ void HoeffdingTree<
|
||||
TrainInternal(data, labels, batchTraining);
|
||||
}
|
||||
|
||||
//! Train on a set of points.
|
||||
template<typename FitnessFunction,
|
||||
template<typename> class NumericSplitType,
|
||||
template<typename> class CategoricalSplitType>
|
||||
@@ -383,21 +562,122 @@ void HoeffdingTree<
|
||||
>::Train(const MatType& data,
|
||||
const data::DatasetInfo& info,
|
||||
const arma::Row<size_t>& labels,
|
||||
const bool batchTraining,
|
||||
const size_t numClassesIn)
|
||||
const size_t numClasses,
|
||||
const bool batchTraining)
|
||||
{
|
||||
// Take over new DatasetInfo.
|
||||
if (ownsInfo)
|
||||
delete datasetInfo;
|
||||
datasetInfo = &info;
|
||||
ownsInfo = false;
|
||||
Train(data, info, labels, numClasses, batchTraining, this->successProbability,
|
||||
this->maxSamples, this->checkInterval, this->minSamples);
|
||||
}
|
||||
|
||||
// Set the number of classes correctly.
|
||||
numClasses = (numClassesIn != 0) ? numClassesIn : arma::max(labels) + 1;
|
||||
template<typename FitnessFunction,
|
||||
template<typename> class NumericSplitType,
|
||||
template<typename> class CategoricalSplitType>
|
||||
template<typename MatType>
|
||||
void HoeffdingTree<
|
||||
FitnessFunction,
|
||||
NumericSplitType,
|
||||
CategoricalSplitType
|
||||
>::Train(const MatType& data,
|
||||
const data::DatasetInfo& info,
|
||||
const arma::Row<size_t>& labels,
|
||||
const size_t numClasses,
|
||||
const bool batchTraining,
|
||||
const double successProbability)
|
||||
{
|
||||
Train(data, info, labels, numClasses, batchTraining, successProbability,
|
||||
this->maxSamples, this->checkInterval, this->minSamples);
|
||||
}
|
||||
|
||||
ResetTree();
|
||||
template<typename FitnessFunction,
|
||||
template<typename> class NumericSplitType,
|
||||
template<typename> class CategoricalSplitType>
|
||||
template<typename MatType>
|
||||
void HoeffdingTree<
|
||||
FitnessFunction,
|
||||
NumericSplitType,
|
||||
CategoricalSplitType
|
||||
>::Train(const MatType& data,
|
||||
const data::DatasetInfo& info,
|
||||
const arma::Row<size_t>& labels,
|
||||
const size_t numClasses,
|
||||
const bool batchTraining,
|
||||
const double successProbability,
|
||||
const size_t maxSamples)
|
||||
{
|
||||
Train(data, info, labels, numClasses, batchTraining, successProbability,
|
||||
maxSamples, this->checkInterval, this->minSamples);
|
||||
}
|
||||
|
||||
template<typename FitnessFunction,
|
||||
template<typename> class NumericSplitType,
|
||||
template<typename> class CategoricalSplitType>
|
||||
template<typename MatType>
|
||||
void HoeffdingTree<
|
||||
FitnessFunction,
|
||||
NumericSplitType,
|
||||
CategoricalSplitType
|
||||
>::Train(const MatType& data,
|
||||
const data::DatasetInfo& info,
|
||||
const arma::Row<size_t>& labels,
|
||||
const size_t numClasses,
|
||||
const bool batchTraining,
|
||||
const double successProbability,
|
||||
const size_t maxSamples,
|
||||
const size_t checkInterval)
|
||||
{
|
||||
Train(data, info, labels, numClasses, batchTraining, successProbability,
|
||||
maxSamples, checkInterval, this->minSamples);
|
||||
}
|
||||
|
||||
template<typename FitnessFunction,
|
||||
template<typename> class NumericSplitType,
|
||||
template<typename> class CategoricalSplitType>
|
||||
template<typename MatType>
|
||||
void HoeffdingTree<
|
||||
FitnessFunction,
|
||||
NumericSplitType,
|
||||
CategoricalSplitType
|
||||
>::Train(const MatType& data,
|
||||
const data::DatasetInfo& info,
|
||||
const arma::Row<size_t>& labels,
|
||||
const size_t numClasses,
|
||||
const bool batchTraining,
|
||||
const double successProbability,
|
||||
const size_t maxSamples,
|
||||
const size_t checkInterval,
|
||||
const size_t minSamples)
|
||||
{
|
||||
this->successProbability = successProbability;
|
||||
this->maxSamples = maxSamples;
|
||||
this->checkInterval = checkInterval;
|
||||
this->minSamples = minSamples;
|
||||
|
||||
// We need to reset the tree either if the user asked for it, or if they
|
||||
// passed data whose dimensionality is different than our datasetInfo object.
|
||||
if (data.n_rows != datasetInfo->Dimensionality() ||
|
||||
(numClasses != 0 && numClasses != this->numClasses))
|
||||
{
|
||||
// Set the number of classes correctly.
|
||||
if (numClasses != 0)
|
||||
this->numClasses = numClasses;
|
||||
|
||||
if (this->numClasses == 0)
|
||||
{
|
||||
throw std::invalid_argument("HoeffdingTree::Train(): must specify number "
|
||||
"of classes!");
|
||||
}
|
||||
|
||||
Reset(info, this->numClasses);
|
||||
}
|
||||
else if (datasetInfo != &info)
|
||||
{
|
||||
// Take over new DatasetInfo.
|
||||
if (ownsInfo)
|
||||
delete datasetInfo;
|
||||
datasetInfo = &info;
|
||||
ownsInfo = false;
|
||||
}
|
||||
|
||||
// Now train.
|
||||
TrainInternal(data, labels, batchTraining);
|
||||
}
|
||||
|
||||
@@ -812,6 +1092,62 @@ void HoeffdingTree<
|
||||
categoricalSplits.clear();
|
||||
}
|
||||
|
||||
template<
|
||||
typename FitnessFunction,
|
||||
template<typename> class NumericSplitType,
|
||||
template<typename> class CategoricalSplitType
|
||||
>
|
||||
void HoeffdingTree<
|
||||
FitnessFunction,
|
||||
NumericSplitType,
|
||||
CategoricalSplitType
|
||||
>::Reset()
|
||||
{
|
||||
ResetTree();
|
||||
}
|
||||
|
||||
template<
|
||||
typename FitnessFunction,
|
||||
template<typename> class NumericSplitType,
|
||||
template<typename> class CategoricalSplitType
|
||||
>
|
||||
void HoeffdingTree<
|
||||
FitnessFunction,
|
||||
NumericSplitType,
|
||||
CategoricalSplitType
|
||||
>::Reset(const size_t dimensionality, const size_t numClasses)
|
||||
{
|
||||
if (ownsInfo)
|
||||
delete datasetInfo;
|
||||
datasetInfo = new data::DatasetInfo(dimensionality); // All features numeric.
|
||||
ownsInfo = true;
|
||||
|
||||
this->numClasses = numClasses;
|
||||
|
||||
ResetTree();
|
||||
}
|
||||
|
||||
template<
|
||||
typename FitnessFunction,
|
||||
template<typename> class NumericSplitType,
|
||||
template<typename> class CategoricalSplitType
|
||||
>
|
||||
void HoeffdingTree<
|
||||
FitnessFunction,
|
||||
NumericSplitType,
|
||||
CategoricalSplitType
|
||||
>::Reset(const data::DatasetInfo& info, const size_t numClasses)
|
||||
{
|
||||
if (ownsInfo)
|
||||
delete datasetInfo;
|
||||
datasetInfo = &info;
|
||||
ownsInfo = false;
|
||||
|
||||
this->numClasses = numClasses;
|
||||
|
||||
ResetTree();
|
||||
}
|
||||
|
||||
template<
|
||||
typename FitnessFunction,
|
||||
template<typename> class NumericSplitType,
|
||||
@@ -999,8 +1335,19 @@ void HoeffdingTree<
|
||||
// Train(), since the col() function is not provided. So,
|
||||
// unfortunately, instead, we'll just extract the non-contiguous
|
||||
// submatrix.
|
||||
MatType childData = data.cols(indices[i].subvec(0, counts[i] - 1));
|
||||
children[i]->Train(childData, childLabels, true);
|
||||
//
|
||||
// I'd rather be able to use:
|
||||
//
|
||||
// arma::Mat<typename MatType::elem_type> childData =
|
||||
// data.cols(indices[i].subvec(0, counts[i] - 1));
|
||||
//
|
||||
// but this isn't currently supported by Armadillo.
|
||||
arma::Mat<typename MatType::elem_type> childData(data.n_rows,
|
||||
counts[i]);
|
||||
for (size_t j = 0; j < counts[i]; ++j)
|
||||
childData.col(j) = data.col(indices[i][j]);
|
||||
|
||||
children[i]->Train(childData, childLabels, numClasses, true);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -198,19 +198,23 @@ inline void HoeffdingTreeModel::Train(const arma::mat& dataset,
|
||||
switch (type)
|
||||
{
|
||||
case GINI_HOEFFDING:
|
||||
giniHoeffdingTree->Train(dataset, labels, batchTraining);
|
||||
giniHoeffdingTree->Train(dataset, labels, giniHoeffdingTree->NumClasses(),
|
||||
batchTraining);
|
||||
break;
|
||||
|
||||
case GINI_BINARY:
|
||||
giniBinaryTree->Train(dataset, labels, batchTraining);
|
||||
giniBinaryTree->Train(dataset, labels, giniBinaryTree->NumClasses(),
|
||||
batchTraining);
|
||||
break;
|
||||
|
||||
case INFO_HOEFFDING:
|
||||
infoHoeffdingTree->Train(dataset, labels, batchTraining);
|
||||
infoHoeffdingTree->Train(dataset, labels, infoHoeffdingTree->NumClasses(),
|
||||
batchTraining);
|
||||
break;
|
||||
|
||||
case INFO_BINARY:
|
||||
infoBinaryTree->Train(dataset, labels, batchTraining);
|
||||
infoBinaryTree->Train(dataset, labels, infoBinaryTree->NumClasses(),
|
||||
batchTraining);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
+346
-115
@@ -28,9 +28,6 @@
|
||||
|
||||
namespace mlpack {
|
||||
|
||||
// beta is the estimator
|
||||
// yHat is the prediction from the current estimator
|
||||
|
||||
/**
|
||||
* An implementation of LARS, a stage-wise homotopy-based algorithm for
|
||||
* l1-regularized linear regression (LASSO) and l1+l2 regularized linear
|
||||
@@ -85,9 +82,14 @@ namespace mlpack {
|
||||
* }
|
||||
* @endcode
|
||||
*/
|
||||
template<typename ModelMatType = arma::mat>
|
||||
class LARS
|
||||
{
|
||||
public:
|
||||
typedef typename GetColType<ModelMatType>::type ModelColType;
|
||||
typedef typename GetDenseMatType<ModelMatType>::type DenseMatType;
|
||||
typedef typename ModelMatType::elem_type ElemType;
|
||||
|
||||
/**
|
||||
* Set the parameters to LARS. Both lambda1 and lambda2 default to 0.
|
||||
*
|
||||
@@ -105,9 +107,9 @@ class LARS
|
||||
* for training.
|
||||
*/
|
||||
LARS(const bool useCholesky = false,
|
||||
const double lambda1 = 0.0,
|
||||
const double lambda2 = 0.0,
|
||||
const double tolerance = 1e-16,
|
||||
const ElemType lambda1 = 0.0,
|
||||
const ElemType lambda2 = 0.0,
|
||||
const ElemType tolerance = 1e-16,
|
||||
const bool fitIntercept = true,
|
||||
const bool normalizeData = true);
|
||||
|
||||
@@ -134,6 +136,7 @@ class LARS
|
||||
* @param normalizeData If true, normalize all features to have unit variance
|
||||
* for training.
|
||||
*/
|
||||
mlpack_deprecated
|
||||
LARS(const bool useCholesky,
|
||||
const arma::mat& gramMatrix,
|
||||
const double lambda1 = 0.0,
|
||||
@@ -148,8 +151,8 @@ class LARS
|
||||
*
|
||||
* @param data Input data.
|
||||
* @param responses A vector of targets.
|
||||
* @param transposeData Should be true if the input data is column-major and
|
||||
* false otherwise.
|
||||
* @param colMajor Should be true if the input data is column-major. Passing
|
||||
* row-major data can avoid a transpose operation.
|
||||
* @param useCholesky Whether or not to use Cholesky decomposition when
|
||||
* solving linear system (as opposed to using the full Gram matrix).
|
||||
* @param lambda1 Regularization parameter for l1-norm penalty.
|
||||
@@ -160,13 +163,18 @@ class LARS
|
||||
* @param normalizeData If true, normalize all features to have unit variance
|
||||
* for training.
|
||||
*/
|
||||
LARS(const arma::mat& data,
|
||||
const arma::rowvec& responses,
|
||||
const bool transposeData = true,
|
||||
template<typename MatType,
|
||||
typename ResponsesType,
|
||||
typename = typename std::enable_if<
|
||||
std::is_same<typename ResponsesType::elem_type, ElemType>::value
|
||||
>::type>
|
||||
LARS(const MatType& data,
|
||||
const ResponsesType& responses,
|
||||
bool colMajor = true,
|
||||
const bool useCholesky = false,
|
||||
const double lambda1 = 0.0,
|
||||
const double lambda2 = 0.0,
|
||||
const double tolerance = 1e-16,
|
||||
const ElemType lambda1 = 0.0,
|
||||
const ElemType lambda2 = 0.0,
|
||||
const ElemType tolerance = 1e-16,
|
||||
const bool fitIntercept = true,
|
||||
const bool normalizeData = true);
|
||||
|
||||
@@ -181,8 +189,8 @@ class LARS
|
||||
*
|
||||
* @param data Input data.
|
||||
* @param responses A vector of targets.
|
||||
* @param transposeData Should be true if the input data is column-major and
|
||||
* false otherwise.
|
||||
* @param colMajor Should be true if the input data is column-major. Passing
|
||||
* row-major data can avoid a transpose operation.
|
||||
* @param useCholesky Whether or not to use Cholesky decomposition when
|
||||
* solving linear system (as opposed to using the full Gram matrix).
|
||||
* @param gramMatrix Gram matrix.
|
||||
@@ -194,14 +202,19 @@ class LARS
|
||||
* @param normalizeData If true, normalize all features to have unit variance
|
||||
* for training.
|
||||
*/
|
||||
LARS(const arma::mat& data,
|
||||
const arma::rowvec& responses,
|
||||
const bool transposeData,
|
||||
template<typename MatType,
|
||||
typename ResponsesType,
|
||||
typename = typename std::enable_if<
|
||||
std::is_same<typename ResponsesType::elem_type, ElemType>::value
|
||||
>::type>
|
||||
LARS(const MatType& data,
|
||||
const ResponsesType& responses,
|
||||
const bool colMajor,
|
||||
const bool useCholesky,
|
||||
const arma::mat& gramMatrix,
|
||||
const double lambda1 = 0.0,
|
||||
const double lambda2 = 0.0,
|
||||
const double tolerance = 1e-16,
|
||||
const DenseMatType& gramMatrix,
|
||||
const ElemType lambda1 = 0.0,
|
||||
const ElemType lambda2 = 0.0,
|
||||
const ElemType tolerance = 1e-16,
|
||||
const bool fitIntercept = true,
|
||||
const bool normalizeData = true);
|
||||
|
||||
@@ -238,38 +251,254 @@ class LARS
|
||||
* column-major -- each column is an observation and each row is a dimension.
|
||||
* However, because LARS is more efficient on a row-major matrix, this method
|
||||
* will (internally) transpose the matrix. If this transposition is not
|
||||
* necessary (i.e., you want to pass in a row-major matrix), pass 'false' for
|
||||
* the transposeData parameter.
|
||||
* necessary (i.e., you want to pass in a row-major matrix), pass `false` for
|
||||
* the `colMajor` parameter.
|
||||
*
|
||||
* @param data Column-major input data (or row-major input data if rowMajor =
|
||||
* true).
|
||||
* @param data Column-major input data (or row-major input data if colMajor =
|
||||
* false).
|
||||
* @param responses A vector of targets.
|
||||
* @param beta Vector to store the solution (the coefficients) in.
|
||||
* @param transposeData Set to false if the data is row-major.
|
||||
* @param colMajor Should be true if the input data is column-major. Passing
|
||||
* row-major data can avoid a transpose operation.
|
||||
* @return minimum cost error(||y-beta*X||2 is used to calculate error).
|
||||
*/
|
||||
mlpack_deprecated
|
||||
double Train(const arma::mat& data,
|
||||
const arma::rowvec& responses,
|
||||
arma::vec& beta,
|
||||
const bool transposeData = true);
|
||||
const bool colMajor = true);
|
||||
|
||||
/**
|
||||
* Run LARS. The input matrix (like all mlpack matrices) should be
|
||||
* column-major -- each column is an observation and each row is a dimension.
|
||||
* However, because LARS is more efficient on a row-major matrix, this method
|
||||
* will (internally) transpose the matrix. If this transposition is not
|
||||
* necessary (i.e., you want to pass in a row-major matrix), pass 'false' for
|
||||
* the transposeData parameter.
|
||||
* necessary (i.e., you want to pass in a row-major matrix), pass `false` for
|
||||
* the `colMajor` parameter.
|
||||
*
|
||||
* All of the different overloads below are needed until C++17 is the minimum
|
||||
* required standard (then std::optional could be used).
|
||||
*
|
||||
* @param data Input data.
|
||||
* @param responses A vector of targets.
|
||||
* @param transposeData Should be true if the input data is column-major and
|
||||
* false otherwise.
|
||||
* @param colMajor Should be true if the input data is column-major. Passing
|
||||
* row-major data can avoid a transpose operation.
|
||||
* @return minimum cost error(||y-beta*X||2 is used to calculate error).
|
||||
*/
|
||||
double Train(const arma::mat& data,
|
||||
const arma::rowvec& responses,
|
||||
const bool transposeData = true);
|
||||
|
||||
// Dummy overload so MetaInfoExtractor can properly detect that LARS is a
|
||||
// regression method.
|
||||
template<typename MatType>
|
||||
ElemType Train(const MatType& data,
|
||||
const arma::rowvec& responses,
|
||||
const bool colMajor = true);
|
||||
|
||||
template<typename MatType,
|
||||
typename ResponsesType,
|
||||
typename = void, /* so MetaInfoExtractor does not get confused */
|
||||
typename = typename std::enable_if<
|
||||
std::is_same<typename ResponsesType::elem_type, ElemType>::value
|
||||
>::type,
|
||||
typename = typename std::enable_if<
|
||||
!std::is_same<ResponsesType, arma::rowvec>::value
|
||||
>::type>
|
||||
ElemType Train(const MatType& data,
|
||||
const ResponsesType& responses,
|
||||
const bool colMajor = true);
|
||||
|
||||
template<typename MatType,
|
||||
typename ResponsesType,
|
||||
typename = void, /* so MetaInfoExtractor does not get confused */
|
||||
typename = typename std::enable_if<
|
||||
std::is_same<typename ResponsesType::elem_type, ElemType>::value
|
||||
>::type>
|
||||
ElemType Train(const MatType& data,
|
||||
const ResponsesType& responses,
|
||||
const bool colMajor,
|
||||
const bool useCholesky);
|
||||
|
||||
template<typename MatType,
|
||||
typename ResponsesType,
|
||||
typename = void, /* so MetaInfoExtractor does not get confused */
|
||||
typename = typename std::enable_if<
|
||||
std::is_same<typename ResponsesType::elem_type, ElemType>::value
|
||||
>::type>
|
||||
ElemType Train(const MatType& data,
|
||||
const ResponsesType& responses,
|
||||
const bool colMajor,
|
||||
const bool useCholesky,
|
||||
const ElemType lambda1);
|
||||
|
||||
template<typename MatType,
|
||||
typename ResponsesType,
|
||||
typename = void, /* so MetaInfoExtractor does not get confused */
|
||||
typename = typename std::enable_if<
|
||||
std::is_same<typename ResponsesType::elem_type, ElemType>::value
|
||||
>::type>
|
||||
ElemType Train(const MatType& data,
|
||||
const ResponsesType& responses,
|
||||
const bool colMajor,
|
||||
const bool useCholesky,
|
||||
const ElemType lambda1,
|
||||
const ElemType lambda2);
|
||||
|
||||
template<typename MatType,
|
||||
typename ResponsesType,
|
||||
typename = void, /* so MetaInfoExtractor does not get confused */
|
||||
typename = typename std::enable_if<
|
||||
std::is_same<typename ResponsesType::elem_type, ElemType>::value
|
||||
>::type>
|
||||
ElemType Train(const MatType& data,
|
||||
const ResponsesType& responses,
|
||||
const bool colMajor,
|
||||
const bool useCholesky,
|
||||
const ElemType lambda1,
|
||||
const ElemType lambda2,
|
||||
const ElemType tolerance);
|
||||
|
||||
template<typename MatType,
|
||||
typename ResponsesType,
|
||||
typename = void, /* so MetaInfoExtractor does not get confused */
|
||||
typename = typename std::enable_if<
|
||||
std::is_same<typename ResponsesType::elem_type, ElemType>::value
|
||||
>::type>
|
||||
ElemType Train(const MatType& data,
|
||||
const ResponsesType& responses,
|
||||
const bool colMajor,
|
||||
const bool useCholesky,
|
||||
const ElemType lambda1,
|
||||
const ElemType lambda2,
|
||||
const ElemType tolerance,
|
||||
const bool fitIntercept);
|
||||
|
||||
template<typename MatType,
|
||||
typename ResponsesType,
|
||||
typename = void, /* so MetaInfoExtractor does not get confused */
|
||||
typename = typename std::enable_if<
|
||||
std::is_same<typename ResponsesType::elem_type, ElemType>::value
|
||||
>::type>
|
||||
ElemType Train(const MatType& data,
|
||||
const ResponsesType& responses,
|
||||
const bool colMajor,
|
||||
const bool useCholesky,
|
||||
const ElemType lambda1,
|
||||
const ElemType lambda2,
|
||||
const ElemType tolerance,
|
||||
const bool fitIntercept,
|
||||
const bool normalizeData);
|
||||
|
||||
/**
|
||||
* Run LARS with a precomputed Gram matrix. The input matrix (like all mlpack
|
||||
* matrices) should be column-major -- each column is an observation and each
|
||||
* row is a dimension. However, because LARS is more efficient on a row-major
|
||||
* matrix, this method will (internally) transpose the matrix. If this
|
||||
* transposition is not necessary (i.e., you want to pass in a row-major
|
||||
* matrix), pass `false` for the `colMajor` parameter.
|
||||
*
|
||||
* All of the different overloads below are needed until C++17 is the minimum
|
||||
* required standard (then std::optional could be used).
|
||||
*
|
||||
* @param data Input data.
|
||||
* @param responses A vector of targets.
|
||||
* @param colMajor Should be true if the input data is column-major. Passing
|
||||
* row-major data can avoid a transpose operation.
|
||||
* @return minimum cost error(||y-beta*X||2 is used to calculate error).
|
||||
*/
|
||||
template<typename MatType,
|
||||
typename ResponsesType,
|
||||
typename = void, /* so MetaInfoExtractor does not get confused */
|
||||
typename = typename std::enable_if<
|
||||
std::is_same<typename ResponsesType::elem_type, ElemType>::value
|
||||
>::type>
|
||||
ElemType Train(const MatType& data,
|
||||
const ResponsesType& responses,
|
||||
const bool colMajor,
|
||||
const bool useCholesky,
|
||||
const DenseMatType& gramMatrix);
|
||||
|
||||
template<typename MatType,
|
||||
typename ResponsesType,
|
||||
typename = void, /* so MetaInfoExtractor does not get confused */
|
||||
typename = typename std::enable_if<
|
||||
std::is_same<typename ResponsesType::elem_type, ElemType>::value
|
||||
>::type>
|
||||
ElemType Train(const MatType& data,
|
||||
const ResponsesType& responses,
|
||||
const bool colMajor,
|
||||
const bool useCholesky,
|
||||
const DenseMatType& gramMatrix,
|
||||
const ElemType lambda1);
|
||||
|
||||
template<typename MatType,
|
||||
typename ResponsesType,
|
||||
typename = void, /* so MetaInfoExtractor does not get confused */
|
||||
typename = typename std::enable_if<
|
||||
std::is_same<typename ResponsesType::elem_type, ElemType>::value
|
||||
>::type>
|
||||
ElemType Train(const MatType& data,
|
||||
const ResponsesType& responses,
|
||||
const bool colMajor,
|
||||
const bool useCholesky,
|
||||
const DenseMatType& gramMatrix,
|
||||
const ElemType lambda1,
|
||||
const ElemType lambda2);
|
||||
|
||||
template<typename MatType,
|
||||
typename ResponsesType,
|
||||
typename = void, /* so MetaInfoExtractor does not get confused */
|
||||
typename = typename std::enable_if<
|
||||
std::is_same<typename ResponsesType::elem_type, ElemType>::value
|
||||
>::type>
|
||||
ElemType Train(const MatType& data,
|
||||
const ResponsesType& responses,
|
||||
const bool colMajor,
|
||||
const bool useCholesky,
|
||||
const DenseMatType& gramMatrix,
|
||||
const ElemType lambda1,
|
||||
const ElemType lambda2,
|
||||
const ElemType tolerance);
|
||||
|
||||
template<typename MatType,
|
||||
typename ResponsesType,
|
||||
typename = void, /* so MetaInfoExtractor does not get confused */
|
||||
typename = typename std::enable_if<
|
||||
std::is_same<typename ResponsesType::elem_type, ElemType>::value
|
||||
>::type>
|
||||
ElemType Train(const MatType& data,
|
||||
const ResponsesType& responses,
|
||||
const bool colMajor,
|
||||
const bool useCholesky,
|
||||
const DenseMatType& gramMatrix,
|
||||
const ElemType lambda1,
|
||||
const ElemType lambda2,
|
||||
const ElemType tolerance,
|
||||
const bool fitIntercept);
|
||||
|
||||
template<typename MatType,
|
||||
typename ResponsesType,
|
||||
typename = void, /* so MetaInfoExtractor does not get confused */
|
||||
typename = typename std::enable_if<
|
||||
std::is_same<typename ResponsesType::elem_type, ElemType>::value
|
||||
>::type>
|
||||
ElemType Train(const MatType& data,
|
||||
const ResponsesType& responses,
|
||||
const bool colMajor,
|
||||
const bool useCholesky,
|
||||
const DenseMatType& gramMatrix,
|
||||
const ElemType lambda1,
|
||||
const ElemType lambda2,
|
||||
const ElemType tolerance,
|
||||
const bool fitIntercept,
|
||||
const bool normalizeData);
|
||||
|
||||
/**
|
||||
* Predict y_i for the given data point.
|
||||
*
|
||||
* @param point The data point to regress on.
|
||||
* @return Predicted value for y_i for `point`.
|
||||
*/
|
||||
template<typename VecType>
|
||||
ElemType Predict(const VecType& point) const;
|
||||
|
||||
/**
|
||||
* Predict y_i for each data point in the given data matrix using the
|
||||
@@ -277,22 +506,23 @@ class LARS
|
||||
*
|
||||
* @param points The data points to regress on.
|
||||
* @param predictions y, which will contained calculated values on completion.
|
||||
* @param rowMajor Should be true if the data points matrix is row-major and
|
||||
* false otherwise.
|
||||
* @param colMajor Should be true if the input data is column-major. Passing
|
||||
* row-major data can avoid a transpose operation.
|
||||
*/
|
||||
void Predict(const arma::mat& points,
|
||||
arma::rowvec& predictions,
|
||||
const bool rowMajor = false) const;
|
||||
template<typename MatType, typename ResponsesType>
|
||||
void Predict(const MatType& points,
|
||||
ResponsesType& predictions,
|
||||
const bool colMajor = true) const;
|
||||
|
||||
//! Get the L1 regularization coefficient.
|
||||
double Lambda1() const { return lambda1; }
|
||||
ElemType Lambda1() const { return lambda1; }
|
||||
//! Modify the L1 regularization coefficient.
|
||||
double& Lambda1() { return lambda1; }
|
||||
ElemType& Lambda1() { return lambda1; }
|
||||
|
||||
//! Get the L2 regularization coefficient.
|
||||
double Lambda2() const { return lambda2; }
|
||||
ElemType Lambda2() const { return lambda2; }
|
||||
//! Modify the L2 regularization coefficient.
|
||||
double& Lambda2() { return lambda2; }
|
||||
ElemType& Lambda2() { return lambda2; }
|
||||
|
||||
//! Get whether to use the Cholesky decomposition.
|
||||
bool UseCholesky() const { return useCholesky; }
|
||||
@@ -300,74 +530,49 @@ class LARS
|
||||
bool& UseCholesky() { return useCholesky; }
|
||||
|
||||
//! Get the tolerance for maximum correlation during training.
|
||||
double Tolerance() const { return tolerance; }
|
||||
ElemType Tolerance() const { return tolerance; }
|
||||
//! Modify the tolerance for maximum correlation during training.
|
||||
double& Tolerance() { return tolerance; }
|
||||
ElemType& Tolerance() { return tolerance; }
|
||||
|
||||
//! Get whether or not to fit an intercept.
|
||||
bool FitIntercept() const { return fitIntercept; }
|
||||
//! Modify whether or not to fit an intercept.
|
||||
void FitIntercept(const bool newFitIntercept)
|
||||
{
|
||||
// If we are storing a Gram matrix internally, but now will be normalizing
|
||||
// data, then the Gram matrix we have computed is incorrect and needs to be
|
||||
// recomputed.
|
||||
if (fitIntercept != newFitIntercept)
|
||||
{
|
||||
if (matGram != &matGramInternal)
|
||||
{
|
||||
throw std::invalid_argument("LARS::FitIntercept(): cannot change value "
|
||||
"when an external Gram matrix was specified!");
|
||||
}
|
||||
|
||||
fitIntercept = newFitIntercept;
|
||||
matGramInternal.clear();
|
||||
}
|
||||
}
|
||||
void FitIntercept(const bool newFitIntercept);
|
||||
|
||||
//! Get whether or not to normalize data during training.
|
||||
bool NormalizeData() const { return normalizeData; }
|
||||
//! Modify whether or not to normalize data during training.
|
||||
void NormalizeData(const bool newNormalizeData)
|
||||
{
|
||||
// If we are storing a Gram matrix internally, but now will be normalizing
|
||||
// data, then the Gram matrix we have computed is incorrect and needs to be
|
||||
// recomputed.
|
||||
if (normalizeData != newNormalizeData)
|
||||
{
|
||||
if (matGram != &matGramInternal)
|
||||
{
|
||||
throw std::invalid_argument("LARS::NormalizeData(): cannot change value "
|
||||
"when an external Gram matrix was specified!");
|
||||
}
|
||||
void NormalizeData(const bool newNormalizeData);
|
||||
|
||||
normalizeData = newNormalizeData;
|
||||
matGramInternal.clear();
|
||||
}
|
||||
}
|
||||
|
||||
//! Access the set of active dimensions.
|
||||
const std::vector<size_t>& ActiveSet() const { return activeSet; }
|
||||
//! Access the set of active dimensions in the currently selected model.
|
||||
const std::vector<size_t>& ActiveSet() const;
|
||||
|
||||
//! Access the set of coefficients after each iteration; the solution is the
|
||||
//! last element.
|
||||
const std::vector<arma::vec>& BetaPath() const { return betaPath; }
|
||||
const std::vector<ModelColType>& BetaPath() const { return betaPath; }
|
||||
|
||||
//! Access the solution coefficients
|
||||
const arma::vec& Beta() const { return betaPath.back(); }
|
||||
const ModelColType& Beta() const;
|
||||
|
||||
//! Access the set of values for lambda1 after each iteration; the solution is
|
||||
//! the last element.
|
||||
const std::vector<double>& LambdaPath() const { return lambdaPath; }
|
||||
const std::vector<ElemType>& LambdaPath() const { return lambdaPath; }
|
||||
|
||||
//! Return the intercept (if fitted, otherwise 0).
|
||||
double Intercept() const { return interceptPath.back(); }
|
||||
ElemType Intercept() const;
|
||||
|
||||
//! Return the intercept path (the intercept for every model).
|
||||
const std::vector<double>& InterceptPath() const { return interceptPath; }
|
||||
const std::vector<ElemType>& InterceptPath() const { return interceptPath; }
|
||||
|
||||
//! Set the model to use the given lambda1 value in the path.
|
||||
void SelectBeta(const ElemType lambda1);
|
||||
|
||||
//! Get the L1 penalty parameter corresponding to the currently selected
|
||||
//! model.
|
||||
ElemType SelectedLambda1() const { return selectedLambda1; }
|
||||
|
||||
//! Access the upper triangular cholesky factor.
|
||||
const arma::mat& MatUtriCholFactor() const { return matUtriCholFactor; }
|
||||
const DenseMatType& MatUtriCholFactor() const { return matUtriCholFactor; }
|
||||
|
||||
/**
|
||||
* Serialize the LARS model.
|
||||
@@ -380,26 +585,26 @@ class LARS
|
||||
* currently-trained LARS model. Only ||y-beta*X||2 is used to calculate
|
||||
* cost error.
|
||||
*
|
||||
* @param matX Column-major input data (or row-major input data if rowMajor =
|
||||
* true).
|
||||
* @param matX Column-major input data (or row-major input data if colMajor =
|
||||
* false).
|
||||
* @param y responses A vector of targets.
|
||||
* @param rowMajor Should be true if the data points matrix is row-major and
|
||||
* false otherwise.
|
||||
* @param colMajor Should be true if the data points matrix is column-major.
|
||||
* @return The minimum cost error.
|
||||
*/
|
||||
double ComputeError(const arma::mat& matX,
|
||||
const arma::rowvec& y,
|
||||
const bool rowMajor = false);
|
||||
template<typename MatType, typename ResponsesType>
|
||||
ElemType ComputeError(const MatType& matX,
|
||||
const ResponsesType& y,
|
||||
const bool colMajor = true);
|
||||
|
||||
private:
|
||||
//! Gram matrix.
|
||||
arma::mat matGramInternal;
|
||||
DenseMatType matGramInternal;
|
||||
|
||||
//! Pointer to the Gram matrix we will use.
|
||||
const arma::mat* matGram;
|
||||
const DenseMatType* matGram;
|
||||
|
||||
//! Upper triangular cholesky factor; initially 0x0 matrix.
|
||||
arma::mat matUtriCholFactor;
|
||||
DenseMatType matUtriCholFactor;
|
||||
|
||||
//! Whether or not to use Cholesky decomposition when solving linear system.
|
||||
bool useCholesky;
|
||||
@@ -407,15 +612,15 @@ class LARS
|
||||
//! True if this is the LASSO problem.
|
||||
bool lasso;
|
||||
//! Regularization parameter for l1 penalty.
|
||||
double lambda1;
|
||||
ElemType lambda1;
|
||||
|
||||
//! True if this is the elastic net problem.
|
||||
bool elasticNet;
|
||||
//! Regularization parameter for l2 penalty.
|
||||
double lambda2;
|
||||
ElemType lambda2;
|
||||
|
||||
//! Tolerance for main loop.
|
||||
double tolerance;
|
||||
ElemType tolerance;
|
||||
|
||||
//! Whether or not to fit an intercept.
|
||||
bool fitIntercept;
|
||||
@@ -425,17 +630,36 @@ class LARS
|
||||
bool normalizeData;
|
||||
|
||||
//! Solution path.
|
||||
std::vector<arma::vec> betaPath;
|
||||
std::vector<ModelColType> betaPath;
|
||||
|
||||
//! Value of lambda_1 for each solution in solution path.
|
||||
std::vector<double> lambdaPath;
|
||||
std::vector<ElemType> lambdaPath;
|
||||
|
||||
//! Intercept (only if fitIntercept is true).
|
||||
std::vector<double> interceptPath;
|
||||
std::vector<ElemType> interceptPath;
|
||||
|
||||
//! Active set of dimensions.
|
||||
std::vector<size_t> activeSet;
|
||||
|
||||
//! Selected lambda1 value for Predict().
|
||||
ElemType selectedLambda1;
|
||||
|
||||
//! Index of selected beta (if selectedLambda1 is in lambdaPath).
|
||||
size_t selectedIndex;
|
||||
|
||||
//! Selected beta, if selectedLambda1 is not in lambdaPath.
|
||||
ModelColType selectedBeta;
|
||||
|
||||
//! Selected intercept, if selectedLambda1 is not in lambdaPath.
|
||||
ElemType selectedIntercept;
|
||||
|
||||
//! Selected active set of dimensions, if selectedLambda1 is not the last
|
||||
//! element in the path.
|
||||
std::vector<size_t> selectedActiveSet;
|
||||
|
||||
//! Might be needed to compute the intercept for other lambda values.
|
||||
ElemType offsetY;
|
||||
|
||||
//! Active set membership indicator (for each dimension).
|
||||
std::vector<bool> isActive;
|
||||
|
||||
@@ -468,27 +692,34 @@ class LARS
|
||||
*/
|
||||
void Ignore(const size_t varInd);
|
||||
|
||||
// compute "equiangular" direction in output space
|
||||
void ComputeYHatDirection(const arma::mat& matX,
|
||||
const arma::vec& betaDirection,
|
||||
arma::vec& yHatDirection);
|
||||
// Compute "equiangular" direction in output space.
|
||||
template<typename MatType, typename VecType>
|
||||
void ComputeYHatDirection(const MatType& matX,
|
||||
const VecType& betaDirection,
|
||||
VecType& yHatDirection);
|
||||
|
||||
// interpolate to compute last solution vector
|
||||
// Interpolate to compute last solution vector.
|
||||
void InterpolateBeta();
|
||||
|
||||
void CholeskyInsert(const arma::vec& newX, const arma::mat& X);
|
||||
template<typename VecType, typename MatType>
|
||||
void CholeskyInsert(const VecType& newX, const MatType& X);
|
||||
|
||||
void CholeskyInsert(double sqNormNewX, const arma::vec& newGramCol);
|
||||
template<typename VecType>
|
||||
void CholeskyInsert(ElemType sqNormNewX, const VecType& newGramCol);
|
||||
|
||||
void GivensRotate(const arma::vec::fixed<2>& x,
|
||||
arma::vec::fixed<2>& rotatedX,
|
||||
arma::mat& G);
|
||||
template<typename MatType>
|
||||
void GivensRotate(const typename arma::Col<ElemType>::template fixed<2>& x,
|
||||
typename arma::Col<ElemType>::template fixed<2>& rotatedX,
|
||||
MatType& G);
|
||||
|
||||
void CholeskyDelete(const size_t colToKill);
|
||||
};
|
||||
|
||||
} // namespace mlpack
|
||||
|
||||
CEREAL_TEMPLATE_CLASS_VERSION((typename ModelMatType),
|
||||
(mlpack::LARS<ModelMatType>), (1));
|
||||
|
||||
// Include implementation of serialize().
|
||||
#include "lars_impl.hpp"
|
||||
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -107,8 +107,8 @@ BINDING_SEE_ALSO("LARS C++ class documentation",
|
||||
PARAM_TMATRIX_IN("input", "Matrix of covariates (X).", "i");
|
||||
PARAM_MATRIX_IN("responses", "Matrix of responses/observations (y).", "r");
|
||||
|
||||
PARAM_MODEL_IN(LARS, "input_model", "Trained LARS model to use.", "m");
|
||||
PARAM_MODEL_OUT(LARS, "output_model", "Output LARS model.", "M");
|
||||
PARAM_MODEL_IN(LARS<>, "input_model", "Trained LARS model to use.", "m");
|
||||
PARAM_MODEL_OUT(LARS<>, "output_model", "Output LARS model.", "M");
|
||||
|
||||
PARAM_TMATRIX_IN("test", "Matrix containing points to regress on (test "
|
||||
"points).", "t");
|
||||
@@ -149,11 +149,11 @@ void BINDING_FUNCTION(util::Params& params, util::Timers& timers)
|
||||
false, "no results will be saved");
|
||||
ReportIgnoredParam(params, {{ "test", true }}, "output_predictions");
|
||||
|
||||
LARS* lars;
|
||||
LARS<>* lars;
|
||||
if (params.Has("input"))
|
||||
{
|
||||
// Initialize the object.
|
||||
lars = new LARS(useCholesky, lambda1, lambda2);
|
||||
lars = new LARS<>(useCholesky, lambda1, lambda2);
|
||||
lars->FitIntercept(!noIntercept);
|
||||
lars->NormalizeData(!noNormalize);
|
||||
|
||||
@@ -176,15 +176,14 @@ void BINDING_FUNCTION(util::Params& params, util::Timers& timers)
|
||||
Log::Fatal << "Number of responses must be equal to number of rows of X!"
|
||||
<< endl;
|
||||
|
||||
vec beta;
|
||||
arma::rowvec y = std::move(matY);
|
||||
timers.Start("lars_regression");
|
||||
lars->Train(matX, y, beta, false /* do not transpose */);
|
||||
lars->Train(matX, y, false /* do not transpose */);
|
||||
timers.Stop("lars_regression");
|
||||
}
|
||||
else // We must have --input_model_file.
|
||||
{
|
||||
lars = params.Get<LARS*>("input_model");
|
||||
lars = params.Get<LARS<>*>("input_model");
|
||||
}
|
||||
|
||||
if (params.Has("test"))
|
||||
@@ -208,5 +207,5 @@ void BINDING_FUNCTION(util::Params& params, util::Timers& timers)
|
||||
params.Get<arma::mat>("output_predictions") = predictions.t();
|
||||
}
|
||||
|
||||
params.Get<LARS*>("output_model") = lars;
|
||||
params.Get<LARS<>*>("output_model") = lars;
|
||||
}
|
||||
|
||||
@@ -26,9 +26,13 @@ namespace mlpack {
|
||||
* Optionally, this class can perform ridge regression, if the lambda parameter
|
||||
* is set to a number greater than zero.
|
||||
*/
|
||||
template<typename ModelMatType = arma::mat>
|
||||
class LinearRegression
|
||||
{
|
||||
public:
|
||||
typedef typename GetColType<ModelMatType>::type ModelColType;
|
||||
typedef typename ModelMatType::elem_type ElemType;
|
||||
|
||||
/**
|
||||
* Creates the model.
|
||||
*
|
||||
@@ -37,23 +41,37 @@ class LinearRegression
|
||||
* @param lambda Regularization constant for ridge regression.
|
||||
* @param intercept Whether or not to include an intercept term.
|
||||
*/
|
||||
LinearRegression(const arma::mat& predictors,
|
||||
const arma::rowvec& responses,
|
||||
template<typename MatType,
|
||||
typename ResponsesType,
|
||||
typename = typename std::enable_if<
|
||||
std::is_same<typename ResponsesType::elem_type, ElemType>::value
|
||||
>::type>
|
||||
LinearRegression(const MatType& predictors,
|
||||
const ResponsesType& responses,
|
||||
const double lambda = 0,
|
||||
const bool intercept = true);
|
||||
|
||||
/**
|
||||
* Creates the model with weighted learning.
|
||||
* Creates the model with instance-weighted learning.
|
||||
*
|
||||
* @param predictors X, matrix of data points.
|
||||
* @param responses y, the measured data for each point in X.
|
||||
* @param weights Observation weights (for boosting).
|
||||
* @param weights Instance weights (for boosting).
|
||||
* @param lambda Regularization constant for ridge regression.
|
||||
* @param intercept Whether or not to include an intercept term.
|
||||
*/
|
||||
LinearRegression(const arma::mat& predictors,
|
||||
const arma::rowvec& responses,
|
||||
const arma::rowvec& weights,
|
||||
template<typename MatType,
|
||||
typename ResponsesType,
|
||||
typename WeightsType,
|
||||
typename = typename std::enable_if<
|
||||
std::is_same<typename ResponsesType::elem_type, ElemType>::value
|
||||
>::type,
|
||||
typename = typename std::enable_if<
|
||||
std::is_same<typename WeightsType::elem_type, ElemType>::value
|
||||
>::type>
|
||||
LinearRegression(const MatType& predictors,
|
||||
const ResponsesType& responses,
|
||||
const WeightsType& weights,
|
||||
const double lambda = 0,
|
||||
const bool intercept = true);
|
||||
|
||||
@@ -71,14 +89,51 @@ class LinearRegression
|
||||
* regularization parameter lambda, call Lambda() or set a different value in
|
||||
* the constructor.
|
||||
*
|
||||
* This version of `Train()` is deprecated and will be removed in mlpack
|
||||
* 5.0.0. Use the version of `Train()` that specifies `lambda` before
|
||||
* `intercept` instead.
|
||||
*
|
||||
* @param predictors X, the matrix of data points to train the model on.
|
||||
* @param responses y, the responses to the data points.
|
||||
* @param intercept Whether or not to fit an intercept term.
|
||||
* @return The least squares error after training.
|
||||
*/
|
||||
mlpack_deprecated /** Will be removed in mlpack 5.0.0. */
|
||||
double Train(const arma::mat& predictors,
|
||||
const arma::rowvec& responses,
|
||||
const bool intercept = true);
|
||||
const bool intercept);
|
||||
|
||||
/**
|
||||
* Train the LinearRegression model on the given data and instance weights.
|
||||
* Careful! This will completely ignore and overwrite the existing model.
|
||||
* This particular implementation does not have an incremental training
|
||||
* algorithm. To set the regularization parameter lambda, call Lambda() or
|
||||
* set a different value in the constructor.
|
||||
*
|
||||
* This version of `Train()` is deprecated and will be removed in mlpack
|
||||
* 5.0.0. Use the version of `Train()` that specifies `lambda` before
|
||||
* `intercept` instead.
|
||||
*
|
||||
* @param predictors X, the matrix of data points to train the model on.
|
||||
* @param responses y, the responses to the data points.
|
||||
* @param weights Instance weights (for boosting).
|
||||
* @param intercept Whether or not to fit an intercept term.
|
||||
* @return The least squares error after training.
|
||||
*/
|
||||
mlpack_deprecated /** Will be removed in mlpack 5.0.0. */
|
||||
double Train(const arma::mat& predictors,
|
||||
const arma::rowvec& responses,
|
||||
const arma::rowvec& weights,
|
||||
const bool intercept);
|
||||
|
||||
/**
|
||||
* Train the LinearRegression model. This is a dummy overload so that
|
||||
* MetaInfoExtractor can properly detect that LinearRegression is a regression
|
||||
* method.
|
||||
*/
|
||||
template<typename MatType>
|
||||
ElemType Train(const MatType& predictors,
|
||||
const arma::rowvec& responses);
|
||||
|
||||
/**
|
||||
* Train the LinearRegression model on the given data and weights. Careful!
|
||||
@@ -89,14 +144,168 @@ class LinearRegression
|
||||
*
|
||||
* @param predictors X, the matrix of data points to train the model on.
|
||||
* @param responses y, the responses to the data points.
|
||||
* @param weights Observation weights (for boosting).
|
||||
* @return The least squares error after training.
|
||||
*/
|
||||
template<typename MatType,
|
||||
typename ResponsesType,
|
||||
typename = void, /* so MetaInfoExtractor does not get confused */
|
||||
typename = typename std::enable_if<
|
||||
std::is_same<typename ResponsesType::elem_type, ElemType>::value
|
||||
>::type,
|
||||
typename = typename std::enable_if<
|
||||
!std::is_same<ResponsesType, arma::rowvec>::value
|
||||
>::type>
|
||||
ElemType Train(const MatType& predictors,
|
||||
const ResponsesType& responses);
|
||||
|
||||
/**
|
||||
* Train the LinearRegression model on the given data and weights. Careful!
|
||||
* This will completely ignore and overwrite the existing model. This
|
||||
* particular implementation does not have an incremental training algorithm.
|
||||
* To set the regularization parameter lambda, call Lambda() or set a
|
||||
* different value in the constructor.
|
||||
*
|
||||
* @param predictors X, the matrix of data points to train the model on.
|
||||
* @param responses y, the responses to the data points.
|
||||
* @param lambda L2 regularization penalty parameter to use.
|
||||
* @return The least squares error after training.
|
||||
*/
|
||||
template<typename MatType,
|
||||
typename ResponsesType,
|
||||
typename = void, /* so MetaInfoExtractor does not get confused */
|
||||
typename = typename std::enable_if<
|
||||
std::is_same<typename ResponsesType::elem_type, ElemType>::value
|
||||
>::type>
|
||||
ElemType Train(const MatType& predictors,
|
||||
const ResponsesType& responses,
|
||||
const double lambda);
|
||||
|
||||
/**
|
||||
* Train the LinearRegression model on the given data and weights. Careful!
|
||||
* This will completely ignore and overwrite the existing model. This
|
||||
* particular implementation does not have an incremental training algorithm.
|
||||
* To set the regularization parameter lambda, call Lambda() or set a
|
||||
* different value in the constructor.
|
||||
*
|
||||
* @param predictors X, the matrix of data points to train the model on.
|
||||
* @param responses y, the responses to the data points.
|
||||
* @param lambda L2 regularization penalty parameter to use.
|
||||
* @param intercept Whether or not to fit an intercept term.
|
||||
* @return The least squares error after training.
|
||||
*/
|
||||
double Train(const arma::mat& predictors,
|
||||
const arma::rowvec& responses,
|
||||
const arma::rowvec& weights,
|
||||
const bool intercept = true);
|
||||
template<typename MatType,
|
||||
typename ResponsesType,
|
||||
typename = void, /* so MetaInfoExtractor does not get confused */
|
||||
typename = typename std::enable_if<
|
||||
std::is_same<typename ResponsesType::elem_type, ElemType>::value
|
||||
>::type>
|
||||
ElemType Train(const MatType& predictors,
|
||||
const ResponsesType& responses,
|
||||
const double lambda,
|
||||
const bool intercept);
|
||||
|
||||
/**
|
||||
* Train the LinearRegression model. This is a dummy overload so that
|
||||
* MetaInfoExtractor can properly detect that LinearRegression is a regression
|
||||
* method.
|
||||
*/
|
||||
template<typename MatType>
|
||||
ElemType Train(const MatType& predictors,
|
||||
const arma::rowvec& responses,
|
||||
const arma::rowvec& weights);
|
||||
|
||||
/**
|
||||
* Train the LinearRegression model on the given data and instance weights.
|
||||
* Careful! This will completely ignore and overwrite the existing model.
|
||||
* This particular implementation does not have an incremental training
|
||||
* algorithm. To set the regularization parameter lambda, call Lambda() or
|
||||
* set a different value in the constructor.
|
||||
*
|
||||
* @param predictors X, the matrix of data points to train the model on.
|
||||
* @param responses y, the responses to the data points.
|
||||
* @param weights Instance weights (for boosting).
|
||||
* @return The least squares error after training.
|
||||
*/
|
||||
template<typename MatType,
|
||||
typename ResponsesType,
|
||||
typename WeightsType,
|
||||
typename = typename std::enable_if<
|
||||
std::is_same<typename ResponsesType::elem_type, ElemType>::value
|
||||
>::type,
|
||||
typename = typename std::enable_if<
|
||||
!std::is_same<ResponsesType, arma::rowvec>::value ||
|
||||
!std::is_same<WeightsType, arma::rowvec>::value
|
||||
>::type,
|
||||
typename = typename std::enable_if<
|
||||
std::is_same<typename WeightsType::elem_type, ElemType>::value
|
||||
>::type>
|
||||
ElemType Train(const MatType& predictors,
|
||||
const ResponsesType& responses,
|
||||
const WeightsType& weights);
|
||||
|
||||
/**
|
||||
* Train the LinearRegression model on the given data and instance weights.
|
||||
* Careful! This will completely ignore and overwrite the existing model.
|
||||
* This particular implementation does not have an incremental training
|
||||
* algorithm. To set the regularization parameter lambda, call Lambda() or
|
||||
* set a different value in the constructor.
|
||||
*
|
||||
* @param predictors X, the matrix of data points to train the model on.
|
||||
* @param responses y, the responses to the data points.
|
||||
* @param weights Instance weights (for boosting).
|
||||
* @param lambda L2 regularization penalty parameter to use.
|
||||
* @return The least squares error after training.
|
||||
*/
|
||||
template<typename MatType,
|
||||
typename ResponsesType,
|
||||
typename WeightsType,
|
||||
typename = typename std::enable_if<
|
||||
std::is_same<typename ResponsesType::elem_type, ElemType>::value
|
||||
>::type,
|
||||
typename = typename std::enable_if<
|
||||
std::is_same<typename WeightsType::elem_type, ElemType>::value
|
||||
>::type>
|
||||
ElemType Train(const MatType& predictors,
|
||||
const ResponsesType& responses,
|
||||
const WeightsType& weights,
|
||||
const double lambda);
|
||||
|
||||
/**
|
||||
* Train the LinearRegression model on the given data and instance weights.
|
||||
* Careful! This will completely ignore and overwrite the existing model.
|
||||
* This particular implementation does not have an incremental training
|
||||
* algorithm. To set the regularization parameter lambda, call Lambda() or
|
||||
* set a different value in the constructor.
|
||||
*
|
||||
* @param predictors X, the matrix of data points to train the model on.
|
||||
* @param responses y, the responses to the data points.
|
||||
* @param weights Instance weights (for boosting).
|
||||
* @param lambda L2 regularization penalty parameter to use.
|
||||
* @param intercept Whether or not to fit an intercept term.
|
||||
* @return The least squares error after training.
|
||||
*/
|
||||
template<typename MatType,
|
||||
typename ResponsesType,
|
||||
typename WeightsType,
|
||||
typename = typename std::enable_if<
|
||||
std::is_same<typename ResponsesType::elem_type, ElemType>::value
|
||||
>::type,
|
||||
typename = typename std::enable_if<
|
||||
std::is_same<typename WeightsType::elem_type, ElemType>::value
|
||||
>::type>
|
||||
ElemType Train(const MatType& predictors,
|
||||
const ResponsesType& responses,
|
||||
const WeightsType& weights,
|
||||
const double lambda,
|
||||
const bool intercept);
|
||||
|
||||
/**
|
||||
* Calculate y_i for a single data point.
|
||||
*
|
||||
* @param point the data point to calculate with.
|
||||
*/
|
||||
template<typename VecType>
|
||||
ElemType Predict(const VecType& point) const;
|
||||
|
||||
/**
|
||||
* Calculate y_i for each data point in points.
|
||||
@@ -104,7 +313,8 @@ class LinearRegression
|
||||
* @param points the data points to calculate with.
|
||||
* @param predictions y, will contain calculated values on completion.
|
||||
*/
|
||||
void Predict(const arma::mat& points, arma::rowvec& predictions) const;
|
||||
template<typename MatType, typename ResponsesType>
|
||||
void Predict(const MatType& points, ResponsesType& predictions) const;
|
||||
|
||||
/**
|
||||
* Calculate the L2 squared error on the given predictors and responses using
|
||||
@@ -123,13 +333,14 @@ class LinearRegression
|
||||
* @param points Matrix of predictors (X).
|
||||
* @param responses Transposed vector of responses (y^T).
|
||||
*/
|
||||
double ComputeError(const arma::mat& points,
|
||||
const arma::rowvec& responses) const;
|
||||
template<typename MatType, typename ResponsesType>
|
||||
ElemType ComputeError(const MatType& points,
|
||||
const ResponsesType& responses) const;
|
||||
|
||||
//! Return the parameters (the b vector).
|
||||
const arma::vec& Parameters() const { return parameters; }
|
||||
const ModelColType& Parameters() const { return parameters; }
|
||||
//! Modify the parameters (the b vector).
|
||||
arma::vec& Parameters() { return parameters; }
|
||||
ModelColType& Parameters() { return parameters; }
|
||||
|
||||
//! Return the Tikhonov regularization parameter for ridge regression.
|
||||
double Lambda() const { return lambda; }
|
||||
@@ -143,19 +354,14 @@ class LinearRegression
|
||||
* Serialize the model.
|
||||
*/
|
||||
template<typename Archive>
|
||||
void serialize(Archive& ar, const uint32_t /* version */)
|
||||
{
|
||||
ar(CEREAL_NVP(parameters));
|
||||
ar(CEREAL_NVP(lambda));
|
||||
ar(CEREAL_NVP(intercept));
|
||||
}
|
||||
void serialize(Archive& ar, const uint32_t version);
|
||||
|
||||
private:
|
||||
/**
|
||||
* The calculated B.
|
||||
* Initialized and filled by constructor to hold the least squares solution.
|
||||
*/
|
||||
arma::vec parameters;
|
||||
ModelColType parameters;
|
||||
|
||||
/**
|
||||
* The Tikhonov regularization parameter for ridge regression (0 for linear
|
||||
@@ -169,6 +375,9 @@ class LinearRegression
|
||||
|
||||
} // namespace mlpack
|
||||
|
||||
CEREAL_TEMPLATE_CLASS_VERSION((typename ModelMatType),
|
||||
(mlpack::LinearRegression<ModelMatType>), (1));
|
||||
|
||||
// Include implementation.
|
||||
#include "linear_regression_impl.hpp"
|
||||
|
||||
|
||||
@@ -17,38 +17,157 @@
|
||||
|
||||
namespace mlpack {
|
||||
|
||||
inline LinearRegression::LinearRegression(
|
||||
const arma::mat& predictors,
|
||||
const arma::rowvec& responses,
|
||||
template<typename ModelMatType>
|
||||
template<typename MatType, typename ResponsesType, typename>
|
||||
inline LinearRegression<ModelMatType>::LinearRegression(
|
||||
const MatType& predictors,
|
||||
const ResponsesType& responses,
|
||||
const double lambda,
|
||||
const bool intercept) :
|
||||
LinearRegression(predictors, responses, arma::rowvec(), lambda, intercept)
|
||||
LinearRegression(predictors, responses,
|
||||
arma::Row<typename ResponsesType::elem_type>(), lambda, intercept)
|
||||
{ /* Nothing to do. */ }
|
||||
|
||||
inline LinearRegression::LinearRegression(
|
||||
const arma::mat& predictors,
|
||||
const arma::rowvec& responses,
|
||||
const arma::rowvec& weights,
|
||||
template<typename ModelMatType>
|
||||
template<typename MatType,
|
||||
typename ResponsesType,
|
||||
typename WeightsType,
|
||||
typename, typename>
|
||||
inline LinearRegression<ModelMatType>::LinearRegression(
|
||||
const MatType& predictors,
|
||||
const ResponsesType& responses,
|
||||
const WeightsType& weights,
|
||||
const double lambda,
|
||||
const bool intercept) :
|
||||
lambda(lambda),
|
||||
intercept(intercept)
|
||||
{
|
||||
Train(predictors, responses, weights, intercept);
|
||||
Train(predictors, responses, weights, lambda, intercept);
|
||||
}
|
||||
|
||||
inline double LinearRegression::Train(const arma::mat& predictors,
|
||||
const arma::rowvec& responses,
|
||||
const bool intercept)
|
||||
template<typename ModelMatType>
|
||||
mlpack_deprecated /** Will be removed in mlpack 5.0.0. */
|
||||
inline double LinearRegression<ModelMatType>::Train(
|
||||
const arma::mat& predictors,
|
||||
const arma::rowvec& responses,
|
||||
const bool intercept)
|
||||
{
|
||||
return Train(predictors, responses, arma::rowvec(), intercept);
|
||||
return Train(predictors, responses, arma::rowvec(), this->lambda, intercept);
|
||||
}
|
||||
|
||||
inline double LinearRegression::Train(const arma::mat& predictors,
|
||||
const arma::rowvec& responses,
|
||||
const arma::rowvec& weights,
|
||||
template<typename ModelMatType>
|
||||
mlpack_deprecated /** Will be removed in mlpack 5.0.0. */
|
||||
inline double LinearRegression<ModelMatType>::Train(
|
||||
const arma::mat& predictors,
|
||||
const arma::rowvec& responses,
|
||||
const arma::rowvec& weights,
|
||||
const bool intercept)
|
||||
{
|
||||
return Train(predictors, responses, weights, this->lambda, intercept);
|
||||
}
|
||||
|
||||
template<typename ModelMatType>
|
||||
template<typename MatType>
|
||||
inline
|
||||
typename LinearRegression<ModelMatType>::ElemType
|
||||
LinearRegression<ModelMatType>::Train(const MatType& predictors,
|
||||
const arma::rowvec& responses)
|
||||
{
|
||||
return Train(predictors, responses, arma::rowvec(), this->lambda,
|
||||
this->intercept);
|
||||
}
|
||||
|
||||
template<typename ModelMatType>
|
||||
template<typename MatType, typename ResponsesType, typename, typename, typename>
|
||||
inline
|
||||
typename LinearRegression<ModelMatType>::ElemType
|
||||
LinearRegression<ModelMatType>::Train(const MatType& predictors,
|
||||
const ResponsesType& responses)
|
||||
{
|
||||
return Train(predictors, responses,
|
||||
arma::Row<typename ResponsesType::elem_type>(), this->lambda,
|
||||
this->intercept);
|
||||
}
|
||||
|
||||
template<typename ModelMatType>
|
||||
template<typename MatType, typename ResponsesType, typename, typename>
|
||||
inline
|
||||
typename LinearRegression<ModelMatType>::ElemType
|
||||
LinearRegression<ModelMatType>::Train(const MatType& predictors,
|
||||
const ResponsesType& responses,
|
||||
const double lambda)
|
||||
{
|
||||
return Train(predictors, responses,
|
||||
arma::Row<typename ResponsesType::elem_type>(), lambda, this->intercept);
|
||||
}
|
||||
|
||||
template<typename ModelMatType>
|
||||
template<typename MatType, typename ResponsesType, typename, typename>
|
||||
inline
|
||||
typename LinearRegression<ModelMatType>::ElemType
|
||||
LinearRegression<ModelMatType>::Train(const MatType& predictors,
|
||||
const ResponsesType& responses,
|
||||
const double lambda,
|
||||
const bool intercept)
|
||||
{
|
||||
return Train(predictors, responses,
|
||||
arma::Row<typename ResponsesType::elem_type>(), lambda, intercept);
|
||||
}
|
||||
|
||||
template<typename ModelMatType>
|
||||
template<typename MatType>
|
||||
inline
|
||||
typename LinearRegression<ModelMatType>::ElemType
|
||||
LinearRegression<ModelMatType>::Train(const MatType& predictors,
|
||||
const arma::rowvec& responses,
|
||||
const arma::rowvec& weights)
|
||||
{
|
||||
return Train(predictors, responses, weights, this->lambda, this->intercept);
|
||||
}
|
||||
|
||||
template<typename ModelMatType>
|
||||
template<typename MatType,
|
||||
typename ResponsesType,
|
||||
typename WeightsType,
|
||||
typename, typename, typename>
|
||||
inline
|
||||
typename LinearRegression<ModelMatType>::ElemType
|
||||
LinearRegression<ModelMatType>::Train(const MatType& predictors,
|
||||
const ResponsesType& responses,
|
||||
const WeightsType& weights)
|
||||
{
|
||||
return Train(predictors, responses, weights, this->lambda, this->intercept);
|
||||
}
|
||||
|
||||
template<typename ModelMatType>
|
||||
template<typename MatType,
|
||||
typename ResponsesType,
|
||||
typename WeightsType,
|
||||
typename, typename>
|
||||
inline
|
||||
typename LinearRegression<ModelMatType>::ElemType
|
||||
LinearRegression<ModelMatType>::Train(const MatType& predictors,
|
||||
const ResponsesType& responses,
|
||||
const WeightsType& weights,
|
||||
const double lambda)
|
||||
{
|
||||
return Train(predictors, responses, weights, lambda, this->intercept);
|
||||
}
|
||||
|
||||
template<typename ModelMatType>
|
||||
template<typename MatType,
|
||||
typename ResponsesType,
|
||||
typename WeightsType,
|
||||
typename, typename>
|
||||
inline
|
||||
typename LinearRegression<ModelMatType>::ElemType
|
||||
LinearRegression<ModelMatType>::Train(const MatType& predictors,
|
||||
const ResponsesType& responses,
|
||||
const WeightsType& weights,
|
||||
const double lambda,
|
||||
const bool intercept)
|
||||
{
|
||||
this->lambda = lambda;
|
||||
this->intercept = intercept;
|
||||
|
||||
/*
|
||||
@@ -66,15 +185,16 @@ inline double LinearRegression::Train(const arma::mat& predictors,
|
||||
|
||||
const size_t nCols = predictors.n_cols;
|
||||
|
||||
arma::mat p = predictors;
|
||||
arma::rowvec r = responses;
|
||||
// TODO: avoid copy if possible.
|
||||
arma::Mat<ElemType> p = arma::conv_to<arma::Mat<ElemType>>::from(predictors);
|
||||
arma::Row<ElemType> r = responses;
|
||||
|
||||
// Here we add the row of ones to the predictors.
|
||||
// The intercept is not penalized. Add an "all ones" row to design and set
|
||||
// intercept = false to get a penalized intercept.
|
||||
if (intercept)
|
||||
{
|
||||
p.insert_rows(0, arma::ones<arma::mat>(1, nCols));
|
||||
p.insert_rows(0, arma::ones<arma::Mat<ElemType>>(1, nCols));
|
||||
}
|
||||
|
||||
if (weights.n_elem > 0)
|
||||
@@ -88,16 +208,48 @@ inline double LinearRegression::Train(const arma::mat& predictors,
|
||||
// Then we'll use Armadillo to solve it.
|
||||
// The total runtime of this should be O(d^2 N) + O(d^3) + O(dN).
|
||||
// (assuming the SVD is used to solve it)
|
||||
arma::mat cov = p * p.t() +
|
||||
lambda * arma::eye<arma::mat>(p.n_rows, p.n_rows);
|
||||
arma::Mat<ElemType> cov = p * p.t() +
|
||||
((ElemType) lambda) * arma::eye<arma::Mat<ElemType>>(p.n_rows, p.n_rows);
|
||||
|
||||
parameters = arma::solve(cov, p * r.t());
|
||||
return ComputeError(predictors, responses);
|
||||
}
|
||||
|
||||
inline void LinearRegression::Predict(
|
||||
const arma::mat& points,
|
||||
arma::rowvec& predictions) const
|
||||
template<typename ModelMatType>
|
||||
template<typename VecType>
|
||||
inline
|
||||
typename LinearRegression<ModelMatType>::ElemType
|
||||
LinearRegression<ModelMatType>::Predict(const VecType& point) const
|
||||
{
|
||||
if (intercept)
|
||||
{
|
||||
// We want to be sure we have the correct number of dimensions in the
|
||||
// dataset.
|
||||
// Prevent underflow.
|
||||
const size_t dimensionality = (parameters.n_rows == 0) ? size_t(0) :
|
||||
size_t(parameters.n_rows - 1);
|
||||
util::CheckSameDimensionality(point, dimensionality,
|
||||
"LinearRegression::Predict()", "point");
|
||||
|
||||
return dot(parameters.subvec(1, parameters.n_elem - 1).t(), point) +
|
||||
parameters(0);
|
||||
}
|
||||
else
|
||||
{
|
||||
// We want to be sure we have the correct number of dimensions in
|
||||
// the dataset.
|
||||
util::CheckSameDimensionality(point, parameters,
|
||||
"LinearRegression::Predict()", "point");
|
||||
|
||||
return dot(parameters.t(), point);
|
||||
}
|
||||
}
|
||||
|
||||
template<typename ModelMatType>
|
||||
template<typename MatType, typename ResponsesType>
|
||||
inline void LinearRegression<ModelMatType>::Predict(
|
||||
const MatType& points,
|
||||
ResponsesType& predictions) const
|
||||
{
|
||||
if (intercept)
|
||||
{
|
||||
@@ -108,10 +260,10 @@ inline void LinearRegression::Predict(
|
||||
size_t(parameters.n_rows - 1);
|
||||
util::CheckSameDimensionality(points, dimensionality,
|
||||
"LinearRegression::Predict()", "points");
|
||||
|
||||
// Get the predictions, but this ignores the intercept value
|
||||
// (parameters[0]).
|
||||
predictions = arma::trans(parameters.subvec(1, parameters.n_elem - 1))
|
||||
* points;
|
||||
predictions = parameters.subvec(1, parameters.n_elem - 1).t() * points;
|
||||
// Now add the intercept.
|
||||
predictions += parameters(0);
|
||||
}
|
||||
@@ -119,26 +271,29 @@ inline void LinearRegression::Predict(
|
||||
{
|
||||
// We want to be sure we have the correct number of dimensions in
|
||||
// the dataset.
|
||||
util::CheckSameDimensionality(points, parameters,
|
||||
util::CheckSameDimensionality(points, parameters,
|
||||
"LinearRegression::Predict()", "points");
|
||||
predictions = arma::trans(parameters) * points;
|
||||
}
|
||||
}
|
||||
|
||||
inline double LinearRegression::ComputeError(
|
||||
const arma::mat& predictors,
|
||||
const arma::rowvec& responses) const
|
||||
template<typename ModelMatType>
|
||||
template<typename MatType, typename ResponsesType>
|
||||
inline typename LinearRegression<ModelMatType>::ElemType
|
||||
LinearRegression<ModelMatType>::ComputeError(
|
||||
const MatType& predictors,
|
||||
const ResponsesType& responses) const
|
||||
{
|
||||
// Sanity check on data.
|
||||
util::CheckSameSizes(predictors, responses, "LinearRegression::Train()");
|
||||
|
||||
|
||||
// Get the number of columns and rows of the dataset.
|
||||
const size_t nCols = predictors.n_cols;
|
||||
const size_t nRows = predictors.n_rows;
|
||||
|
||||
// Calculate the differences between actual responses and predicted responses.
|
||||
// We must also add the intercept (parameters(0)) to the predictions.
|
||||
arma::rowvec temp;
|
||||
arma::Row<typename ResponsesType::elem_type> temp;
|
||||
if (intercept)
|
||||
{
|
||||
// Ensure that we have the correct number of dimensions in the dataset.
|
||||
@@ -148,7 +303,7 @@ inline double LinearRegression::ComputeError(
|
||||
"training file." << std::endl;
|
||||
}
|
||||
temp = responses - (parameters(0) +
|
||||
arma::trans(parameters.subvec(1, parameters.n_elem - 1)) * predictors);
|
||||
parameters.subvec(1, parameters.n_elem - 1).t() * predictors);
|
||||
}
|
||||
else
|
||||
{
|
||||
@@ -158,13 +313,34 @@ inline double LinearRegression::ComputeError(
|
||||
Log::Fatal << "The test data must have the same number of columns as the "
|
||||
"training file." << std::endl;
|
||||
}
|
||||
temp = responses - arma::trans(parameters) * predictors;
|
||||
temp = responses - parameters.t() * predictors;
|
||||
}
|
||||
const double cost = arma::dot(temp, temp) / nCols;
|
||||
const ElemType cost = dot(temp, temp) / nCols;
|
||||
|
||||
return cost;
|
||||
}
|
||||
|
||||
template<typename ModelMatType>
|
||||
template<typename Archive>
|
||||
void LinearRegression<ModelMatType>::serialize(Archive& ar,
|
||||
const uint32_t version)
|
||||
{
|
||||
if (cereal::is_loading<Archive>() && version == 0)
|
||||
{
|
||||
// Old versions represented `parameters` as an arma::vec.
|
||||
arma::vec parametersTmp;
|
||||
ar(cereal::make_nvp("parameters", parametersTmp));
|
||||
parameters = arma::conv_to<ModelColType>::from(parametersTmp);
|
||||
}
|
||||
else
|
||||
{
|
||||
ar(CEREAL_NVP(parameters));
|
||||
}
|
||||
|
||||
ar(CEREAL_NVP(lambda));
|
||||
ar(CEREAL_NVP(intercept));
|
||||
}
|
||||
|
||||
} // namespace mlpack
|
||||
|
||||
#endif
|
||||
|
||||
@@ -95,9 +95,9 @@ PARAM_ROW_IN("training_responses", "Optional vector containing y "
|
||||
"(responses). If not given, the responses are assumed to be the last row "
|
||||
"of the input file.", "r");
|
||||
|
||||
PARAM_MODEL_IN(LinearRegression, "input_model", "Existing LinearRegression "
|
||||
PARAM_MODEL_IN(LinearRegression<>, "input_model", "Existing LinearRegression "
|
||||
"model to use.", "m");
|
||||
PARAM_MODEL_OUT(LinearRegression, "output_model", "Output LinearRegression "
|
||||
PARAM_MODEL_OUT(LinearRegression<>, "output_model", "Output LinearRegression "
|
||||
"model.", "M");
|
||||
|
||||
PARAM_MATRIX_IN("test", "Matrix containing X' (test regressors).", "T");
|
||||
@@ -120,7 +120,7 @@ void BINDING_FUNCTION(util::Params& params, util::Timers& timer)
|
||||
mat regressors;
|
||||
rowvec responses;
|
||||
|
||||
LinearRegression* lr;
|
||||
LinearRegression<>* lr;
|
||||
|
||||
const bool computeModel = !params.Has("input_model");
|
||||
const bool computePrediction = params.Has("test");
|
||||
@@ -172,14 +172,14 @@ void BINDING_FUNCTION(util::Params& params, util::Timers& timer)
|
||||
}
|
||||
|
||||
timer.Start("regression");
|
||||
lr = new LinearRegression(regressors, responses, lambda);
|
||||
lr = new LinearRegression<>(regressors, responses, lambda);
|
||||
timer.Stop("regression");
|
||||
}
|
||||
else
|
||||
{
|
||||
// A model file was passed in, so load it.
|
||||
timer.Start("load_model");
|
||||
lr = params.Get<LinearRegression*>("input_model");
|
||||
lr = params.Get<LinearRegression<>*>("input_model");
|
||||
timer.Stop("load_model");
|
||||
}
|
||||
|
||||
@@ -221,5 +221,5 @@ void BINDING_FUNCTION(util::Params& params, util::Timers& timer)
|
||||
}
|
||||
|
||||
// Save the model if needed.
|
||||
params.Get<LinearRegression*>("output_model") = lr;
|
||||
params.Get<LinearRegression<>*>("output_model") = lr;
|
||||
}
|
||||
|
||||
@@ -37,8 +37,8 @@ BINDING_LONG_DESC("");
|
||||
BINDING_EXAMPLE(
|
||||
CALL_METHOD("model", "predict", "test", "X_test"));
|
||||
|
||||
PARAM_MODEL_IN_REQ(LinearRegression, "input_model", "Existing LinearRegression "
|
||||
"model to use.", "m");
|
||||
PARAM_MODEL_IN_REQ(LinearRegression<>, "input_model", "Existing "
|
||||
"LinearRegression model to use.", "m");
|
||||
|
||||
PARAM_MATRIX_IN_REQ("test", "Matrix containing X' (test regressors).", "T");
|
||||
|
||||
@@ -50,7 +50,7 @@ void BINDING_FUNCTION(util::Params& params, util::Timers& timer)
|
||||
{
|
||||
// A model file was passed in, so load it.
|
||||
timer.Start("load_model");
|
||||
LinearRegression* lr = params.Get<LinearRegression*>("input_model");
|
||||
LinearRegression<>* lr = params.Get<LinearRegression<>*>("input_model");
|
||||
timer.Stop("load_model");
|
||||
|
||||
// Cache the output of GetPrintable before we std::move() the test
|
||||
|
||||
@@ -61,7 +61,7 @@ PARAM_ROW_IN("training_responses", "Optional vector containing y "
|
||||
"(responses). If not given, the responses are assumed to be the last row "
|
||||
"of the input file.", "r");
|
||||
|
||||
PARAM_MODEL_OUT(LinearRegression, "output_model", "Output LinearRegression "
|
||||
PARAM_MODEL_OUT(LinearRegression<>, "output_model", "Output LinearRegression "
|
||||
"model.", "M");
|
||||
|
||||
PARAM_DOUBLE_IN("lambda", "Tikhonov regularization for ridge regression. If 0,"
|
||||
@@ -108,9 +108,10 @@ void BINDING_FUNCTION(util::Params& params, util::Timers& timer)
|
||||
Log::Fatal << "Regressors and Responses must have the same number of data points!" << endl;
|
||||
|
||||
timer.Start("regression");
|
||||
LinearRegression* lr = new LinearRegression(regressors, responses, lambda);
|
||||
LinearRegression<>* lr = new LinearRegression<>(regressors, responses,
|
||||
lambda);
|
||||
timer.Stop("regression");
|
||||
|
||||
// Save the model if needed.
|
||||
params.Get<LinearRegression*>("output_model") = lr;
|
||||
params.Get<LinearRegression<>*>("output_model") = lr;
|
||||
}
|
||||
|
||||
@@ -73,18 +73,54 @@ namespace mlpack {
|
||||
* lsvm.Classify(test_data, predictions);
|
||||
* @endcode
|
||||
*
|
||||
* @tparam MatType Type of data matrix.
|
||||
* @tparam ModelMatType Type of data matrix to use to store model parameters.
|
||||
*/
|
||||
template <typename MatType = arma::mat>
|
||||
template<typename ModelMatType = arma::mat>
|
||||
class LinearSVM
|
||||
{
|
||||
public:
|
||||
typedef typename ModelMatType::elem_type ElemType;
|
||||
typedef typename GetDenseMatType<ModelMatType>::type DenseMatType;
|
||||
typedef typename GetDenseColType<ModelMatType>::type DenseColType;
|
||||
|
||||
/**
|
||||
* Initialize the Linear SVM without performing training. Default
|
||||
* value of lambda is 0.0001. Be sure to use Train() before calling
|
||||
* Classify() or ComputeAccuracy(), otherwise the results may be meaningless.
|
||||
*
|
||||
* @param lambda L2-regularization constant.
|
||||
* @param delta Margin of difference between correct class and other classes.
|
||||
* @param fitIntercept add intercept term or not.
|
||||
*/
|
||||
LinearSVM();
|
||||
|
||||
/**
|
||||
* Initialize the Linear SVM without performing training. Default
|
||||
* value of lambda is 0.0001. Be sure to use Train() before calling
|
||||
* Classify() or ComputeAccuracy(), otherwise the results may be meaningless.
|
||||
*
|
||||
* @param dimensionality Size of the input feature vector.
|
||||
* @param numClasses Number of classes for classification.
|
||||
* @param lambda L2-regularization constant.
|
||||
* @param delta Margin of difference between correct class and other classes.
|
||||
* @param fitIntercept add intercept term or not.
|
||||
*/
|
||||
LinearSVM(const size_t dimensionality,
|
||||
const size_t numClasses,
|
||||
const double lambda = 0.0001,
|
||||
const double delta = 1.0,
|
||||
const bool fitIntercept = false);
|
||||
|
||||
/**
|
||||
* Construct the LinearSVM class with the provided data and labels.
|
||||
* This will train the model. Optionally, the parameter 'lambda' can be
|
||||
* passed, which controls the amount of L2-regularization in the objective
|
||||
* function. By default, the model takes a small value.
|
||||
*
|
||||
* This constructor is deprecated; if you want to specify a custom optimizer,
|
||||
* use the constructor with the optimizer after `numClasses`. The constructor
|
||||
* will be removed in mlpack 5.0.0.
|
||||
*
|
||||
* @tparam OptimizerType Desired differentiable separable optimizer
|
||||
* @tparam CallbackTypes Types of callback functions.
|
||||
* @param data Input training features. Each column associate with one sample
|
||||
@@ -97,8 +133,18 @@ class LinearSVM
|
||||
* @param callbacks Callback functions.
|
||||
* See https://www.ensmallen.org/docs.html#callback-documentation.
|
||||
*/
|
||||
template <typename OptimizerType, typename... CallbackTypes>
|
||||
LinearSVM(const MatType& data,
|
||||
template<typename OptimizerType,
|
||||
typename... CallbackTypes,
|
||||
typename = typename std::enable_if<IsEnsOptimizer<
|
||||
OptimizerType,
|
||||
LinearSVMFunction<arma::mat, ModelMatType>,
|
||||
ModelMatType
|
||||
>::value>::type,
|
||||
typename = typename std::enable_if<IsEnsCallbackTypes<
|
||||
CallbackTypes...
|
||||
>::value>::type>
|
||||
mlpack_deprecated /** To be removed in mlpack 5.0.0. **/
|
||||
LinearSVM(const arma::mat& data,
|
||||
const arma::Row<size_t>& labels,
|
||||
const size_t numClasses,
|
||||
const double lambda,
|
||||
@@ -113,6 +159,10 @@ class LinearSVM
|
||||
* passed, which controls the amount of L2-regularization in the objective
|
||||
* function. By default, the model takes a small value.
|
||||
*
|
||||
* This constructor is deprecated; if you want to specify a custom optimizer,
|
||||
* use the constructor with the optimizer after `numClasses`. The constructor
|
||||
* will be removed in mlpack 5.0.0.
|
||||
*
|
||||
* @tparam OptimizerType Desired differentiable separable optimizer
|
||||
* @param data Input training features. Each column associate with one sample
|
||||
* @param labels Labels associated with the feature data.
|
||||
@@ -122,45 +172,210 @@ class LinearSVM
|
||||
* @param fitIntercept add intercept term or not.
|
||||
* @param optimizer Desired optimizer.
|
||||
*/
|
||||
template <typename OptimizerType = ens::L_BFGS>
|
||||
template<typename OptimizerType = ens::L_BFGS,
|
||||
typename = typename std::enable_if<IsEnsOptimizer<
|
||||
OptimizerType,
|
||||
LinearSVMFunction<arma::mat, ModelMatType>,
|
||||
ModelMatType
|
||||
>::value>::type>
|
||||
mlpack_deprecated /** To be removed in mlpack 5.0.0. **/
|
||||
LinearSVM(const arma::mat& data,
|
||||
const arma::Row<size_t>& labels,
|
||||
const size_t numClasses,
|
||||
const double lambda,
|
||||
const double delta,
|
||||
const bool fitIntercept,
|
||||
OptimizerType optimizer);
|
||||
|
||||
/**
|
||||
* Construct the LinearSVM class with the provided data and labels.
|
||||
* This will train the model. Optionally, hyperparameters can be passed.
|
||||
*
|
||||
* @tparam OptimizerType Desired differentiable separable optimizer
|
||||
* @param data Input training features. Each column associate with one sample
|
||||
* @param labels Labels associated with the feature data.
|
||||
* @param numClasses Number of classes for classification.
|
||||
* @param lambda L2-regularization constant.
|
||||
* @param delta Margin of difference between correct class and other classes.
|
||||
* @param fitIntercept add intercept term or not.
|
||||
* @param callbacks Callback Functions.
|
||||
* See https://www.ensmallen.org/docs.html#callback-documentation.
|
||||
*/
|
||||
template<typename MatType,
|
||||
typename... CallbackTypes,
|
||||
typename = typename std::enable_if<IsEnsCallbackTypes<
|
||||
CallbackTypes...
|
||||
>::value>::type>
|
||||
LinearSVM(const MatType& data,
|
||||
const arma::Row<size_t>& labels,
|
||||
const size_t numClasses = 2,
|
||||
const size_t numClasses,
|
||||
const double lambda = 0.0001,
|
||||
const double delta = 1.0,
|
||||
const bool fitIntercept = false,
|
||||
OptimizerType optimizer = OptimizerType());
|
||||
CallbackTypes&&... callbacks);
|
||||
|
||||
/**
|
||||
* Initialize the Linear SVM without performing training. Default
|
||||
* value of lambda is 0.0001. Be sure to use Train() before calling
|
||||
* Classify() or ComputeAccuracy(), otherwise the results may be meaningless.
|
||||
* Construct the LinearSVM class with the provided data and labels. This will
|
||||
* train the model with the given custom optimizer. Optionally,
|
||||
* hyperparameters can be passed.
|
||||
*
|
||||
* @param inputSize Size of the input feature vector.
|
||||
* @tparam OptimizerType Desired differentiable separable optimizer
|
||||
* @param data Input training features. Each column associate with one sample
|
||||
* @param labels Labels associated with the feature data.
|
||||
* @param numClasses Number of classes for classification.
|
||||
* @param lambda L2-regularization constant.
|
||||
* @param delta Margin of difference between correct class and other classes.
|
||||
* @param fitIntercept add intercept term or not.
|
||||
* @param callbacks Callback Functions.
|
||||
* See https://www.ensmallen.org/docs.html#callback-documentation.
|
||||
*/
|
||||
LinearSVM(const size_t inputSize,
|
||||
const size_t numClasses = 0,
|
||||
template<typename MatType,
|
||||
typename OptimizerType = ens::L_BFGS,
|
||||
typename... CallbackTypes,
|
||||
typename = typename std::enable_if<IsEnsCallbackTypes<
|
||||
CallbackTypes...
|
||||
>::value>::type>
|
||||
LinearSVM(const MatType& data,
|
||||
const arma::Row<size_t>& labels,
|
||||
const size_t numClasses,
|
||||
OptimizerType& optimizer,
|
||||
const double lambda = 0.0001,
|
||||
const double delta = 1.0,
|
||||
const bool fitIntercept = false);
|
||||
const bool fitIntercept = false,
|
||||
CallbackTypes&&... callbacks);
|
||||
|
||||
/**
|
||||
* Initialize the Linear SVM without performing training. Default
|
||||
* value of lambda is 0.0001. Be sure to use Train() before calling
|
||||
* Classify() or ComputeAccuracy(), otherwise the results may be meaningless.
|
||||
* Train the Linear SVM with the given training data.
|
||||
*
|
||||
* @param data Input training features. Each column associate with one sample.
|
||||
* @param labels Labels associated with the feature data.
|
||||
* @param numClasses Number of classes for classification.
|
||||
* @param lambda L2-regularization constant.
|
||||
* @param delta Margin of difference between correct class and other classes.
|
||||
* @param fitIntercept add intercept term or not.
|
||||
* @param callbacks Callback Functions.
|
||||
* See https://www.ensmallen.org/docs.html#callback-documentation.
|
||||
* @return Objective value of the final point.
|
||||
*/
|
||||
LinearSVM(const size_t numClasses = 0,
|
||||
const double lambda = 0.0001,
|
||||
const double delta = 1.0,
|
||||
const bool fitIntercept = false);
|
||||
// Many overloads are necessary because we don't yet require C++17, which
|
||||
// would give std::optional support.
|
||||
template<typename MatType,
|
||||
typename... CallbackTypes,
|
||||
typename = typename std::enable_if<IsEnsCallbackTypes<
|
||||
CallbackTypes...
|
||||
>::value>::type>
|
||||
ElemType Train(const MatType& data,
|
||||
const arma::Row<size_t>& labels,
|
||||
const size_t numClasses,
|
||||
CallbackTypes&&... callbackTypes);
|
||||
|
||||
template<typename MatType>
|
||||
ElemType Train(const MatType& data,
|
||||
const arma::Row<size_t>& labels,
|
||||
const size_t numClasses,
|
||||
const double lambda);
|
||||
|
||||
template<typename MatType>
|
||||
ElemType Train(const MatType& data,
|
||||
const arma::Row<size_t>& labels,
|
||||
const size_t numClasses,
|
||||
const double lambda,
|
||||
const double delta);
|
||||
|
||||
template<typename MatType,
|
||||
typename... CallbackTypes,
|
||||
typename = typename std::enable_if<IsEnsCallbackTypes<
|
||||
CallbackTypes...
|
||||
>::value>::type>
|
||||
ElemType Train(const MatType& data,
|
||||
const arma::Row<size_t>& labels,
|
||||
const size_t numClasses,
|
||||
const double lambda,
|
||||
const double delta,
|
||||
const bool fitIntercept,
|
||||
CallbackTypes&&... callbacks);
|
||||
|
||||
/**
|
||||
* Train the Linear SVM with the given training data using a custom ensmallen
|
||||
* optimizer.
|
||||
*
|
||||
* @tparam OptimizerType Desired optimizer.
|
||||
* @param data Input training features. Each column associate with one sample.
|
||||
* @param labels Labels associated with the feature data.
|
||||
* @param numClasses Number of classes for classification.
|
||||
* @param optimizer Desired optimizer.
|
||||
* @param lambda L2-regularization constant.
|
||||
* @param delta Margin of difference between correct class and other classes.
|
||||
* @param fitIntercept add intercept term or not.
|
||||
* @param callbacks Callback Functions.
|
||||
* See https://www.ensmallen.org/docs.html#callback-documentation.
|
||||
* @return Objective value of the final point.
|
||||
*/
|
||||
// Many overloads are necessary because we don't yet require C++17, which
|
||||
// would give std::optional support.
|
||||
template<typename MatType,
|
||||
typename OptimizerType = ens::L_BFGS,
|
||||
typename... CallbackTypes,
|
||||
typename = typename std::enable_if<IsEnsOptimizer<
|
||||
OptimizerType,
|
||||
LinearSVMFunction<MatType, ModelMatType>,
|
||||
ModelMatType
|
||||
>::value>::type,
|
||||
typename = typename std::enable_if<IsEnsCallbackTypes<
|
||||
CallbackTypes...
|
||||
>::value>::type>
|
||||
ElemType Train(const MatType& data,
|
||||
const arma::Row<size_t>& labels,
|
||||
const size_t numClasses,
|
||||
OptimizerType optimizer,
|
||||
CallbackTypes&&... callbacks);
|
||||
|
||||
template<typename MatType,
|
||||
typename OptimizerType = ens::L_BFGS,
|
||||
typename = typename std::enable_if<IsEnsOptimizer<
|
||||
OptimizerType,
|
||||
LinearSVMFunction<MatType, ModelMatType>,
|
||||
ModelMatType
|
||||
>::value>::type>
|
||||
ElemType Train(const MatType& data,
|
||||
const arma::Row<size_t>& labels,
|
||||
const size_t numClasses,
|
||||
OptimizerType optimizer,
|
||||
const double lambda);
|
||||
|
||||
template<typename MatType,
|
||||
typename OptimizerType = ens::L_BFGS,
|
||||
typename = typename std::enable_if<IsEnsOptimizer<
|
||||
OptimizerType,
|
||||
LinearSVMFunction<MatType, ModelMatType>,
|
||||
ModelMatType
|
||||
>::value>::type>
|
||||
ElemType Train(const MatType& data,
|
||||
const arma::Row<size_t>& labels,
|
||||
const size_t numClasses,
|
||||
OptimizerType optimizer,
|
||||
const double lambda,
|
||||
const double delta);
|
||||
|
||||
template<typename MatType,
|
||||
typename OptimizerType = ens::L_BFGS,
|
||||
typename... CallbackTypes,
|
||||
typename = typename std::enable_if<IsEnsOptimizer<
|
||||
OptimizerType,
|
||||
LinearSVMFunction<MatType, ModelMatType>,
|
||||
ModelMatType
|
||||
>::value>::type,
|
||||
typename = typename std::enable_if<IsEnsCallbackTypes<
|
||||
CallbackTypes...
|
||||
>::value>::type>
|
||||
ElemType Train(const MatType& data,
|
||||
const arma::Row<size_t>& labels,
|
||||
const size_t numClasses,
|
||||
OptimizerType optimizer,
|
||||
const double lambda,
|
||||
const double delta,
|
||||
const bool fitIntercept,
|
||||
CallbackTypes&&... callbacks);
|
||||
|
||||
/**
|
||||
* Classify the given points, returning the predicted labels for each point.
|
||||
@@ -171,6 +386,7 @@ class LinearSVM
|
||||
* @param data Set of points to classify.
|
||||
* @param labels Predicted labels for each point.
|
||||
*/
|
||||
template<typename MatType>
|
||||
void Classify(const MatType& data,
|
||||
arma::Row<size_t>& labels) const;
|
||||
|
||||
@@ -185,9 +401,10 @@ class LinearSVM
|
||||
* @param labels Predicted labels for each point.
|
||||
* @param scores Class probabilities for each point.
|
||||
*/
|
||||
template<typename MatType>
|
||||
void Classify(const MatType& data,
|
||||
arma::Row<size_t>& labels,
|
||||
arma::mat& scores) const;
|
||||
DenseMatType& scores) const;
|
||||
|
||||
/**
|
||||
* Classify the given points, returning class scores for each point.
|
||||
@@ -195,7 +412,8 @@ class LinearSVM
|
||||
* @param data Matrix of data points to be classified.
|
||||
* @param scores Class scores for each point.
|
||||
*/
|
||||
void Classify(const MatType& data,
|
||||
mlpack_deprecated
|
||||
void Classify(const arma::mat& data,
|
||||
arma::mat& scores) const;
|
||||
|
||||
/**
|
||||
@@ -209,6 +427,20 @@ class LinearSVM
|
||||
template<typename VecType>
|
||||
size_t Classify(const VecType& point) const;
|
||||
|
||||
/**
|
||||
* Classify the given point. The predicted class label is stored in `label`,
|
||||
* and the probability of each class is stored in `probabilities`..
|
||||
*
|
||||
* @param point Point to be classified.
|
||||
* @param label size_t to store predicted label into.
|
||||
* @param probabilities Vector to store class probabilities into.
|
||||
* @return Predicted class label of the point.
|
||||
*/
|
||||
template<typename VecType>
|
||||
void Classify(const VecType& point,
|
||||
size_t& label,
|
||||
DenseColType& probabilities) const;
|
||||
|
||||
/**
|
||||
* Computes accuracy of the learned model given the feature data and the
|
||||
* labels associated with each data point. Predictions are made using the
|
||||
@@ -218,46 +450,10 @@ class LinearSVM
|
||||
* @param testLabels Vector of labels associated with the data.
|
||||
* @return Accuracy of the model.
|
||||
*/
|
||||
template<typename MatType>
|
||||
double ComputeAccuracy(const MatType& testData,
|
||||
const arma::Row<size_t>& testLabels) const;
|
||||
|
||||
/**
|
||||
* Train the Linear SVM with the given training data.
|
||||
*
|
||||
* @tparam OptimizerType Desired optimizer.
|
||||
* @tparam CallbackTypes Types of Callback Functions.
|
||||
* @param data Input training features. Each column associate with one sample.
|
||||
* @param labels Labels associated with the feature data.
|
||||
* @param numClasses Number of classes for classification.
|
||||
* @param optimizer Desired optimizer.
|
||||
* @param callbacks Callback Functions.
|
||||
* See https://www.ensmallen.org/docs.html#callback-documentation.
|
||||
* @return Objective value of the final point.
|
||||
*/
|
||||
template <typename OptimizerType, typename... CallbackTypes>
|
||||
double Train(const MatType& data,
|
||||
const arma::Row<size_t>& labels,
|
||||
const size_t numClasses,
|
||||
OptimizerType optimizer,
|
||||
CallbackTypes&&... callbacks);
|
||||
|
||||
/**
|
||||
* Train the Linear SVM with the given training data.
|
||||
*
|
||||
* @tparam OptimizerType Desired optimizer.
|
||||
* @param data Input training features. Each column associate with one sample.
|
||||
* @param labels Labels associated with the feature data.
|
||||
* @param numClasses Number of classes for classification.
|
||||
* @param optimizer Desired optimizer.
|
||||
* @return Objective value of the final point.
|
||||
*/
|
||||
template <typename OptimizerType = ens::L_BFGS>
|
||||
double Train(const MatType& data,
|
||||
const arma::Row<size_t>& labels,
|
||||
const size_t numClasses = 2,
|
||||
OptimizerType optimizer = OptimizerType());
|
||||
|
||||
|
||||
//! Sets the number of classes.
|
||||
size_t& NumClasses() { return numClasses; }
|
||||
//! Gets the number of classes.
|
||||
@@ -277,9 +473,9 @@ class LinearSVM
|
||||
bool& FitIntercept() { return fitIntercept; }
|
||||
|
||||
//! Set the model parameters.
|
||||
arma::mat& Parameters() { return parameters; }
|
||||
ModelMatType& Parameters() { return parameters; }
|
||||
//! Get the model parameters.
|
||||
const arma::mat& Parameters() const { return parameters; }
|
||||
const ModelMatType& Parameters() const { return parameters; }
|
||||
|
||||
//! Gets the features size of the training data
|
||||
size_t FeatureSize() const
|
||||
@@ -290,17 +486,11 @@ class LinearSVM
|
||||
* Serialize the LinearSVM model.
|
||||
*/
|
||||
template<typename Archive>
|
||||
void serialize(Archive& ar, const uint32_t /* version */)
|
||||
{
|
||||
ar(CEREAL_NVP(parameters));
|
||||
ar(CEREAL_NVP(numClasses));
|
||||
ar(CEREAL_NVP(lambda));
|
||||
ar(CEREAL_NVP(fitIntercept));
|
||||
}
|
||||
void serialize(Archive& ar, const uint32_t /* version */);
|
||||
|
||||
private:
|
||||
//! Parameters after optimization.
|
||||
arma::mat parameters;
|
||||
ModelMatType parameters;
|
||||
//! Number of classes.
|
||||
size_t numClasses;
|
||||
//! L2-Regularization constant.
|
||||
@@ -313,6 +503,9 @@ class LinearSVM
|
||||
|
||||
} // namespace mlpack
|
||||
|
||||
CEREAL_TEMPLATE_CLASS_VERSION((typename ModelMatType),
|
||||
(mlpack::LinearSVM<ModelMatType>), (1));
|
||||
|
||||
// Include implementation.
|
||||
#include "linear_svm_impl.hpp"
|
||||
|
||||
|
||||
@@ -23,10 +23,16 @@ namespace mlpack {
|
||||
* This is used by various ensmallen optimizers to train the linear
|
||||
* SVM model.
|
||||
*/
|
||||
template <typename MatType = arma::mat>
|
||||
template<typename MatType = arma::mat, typename ParametersType = arma::mat>
|
||||
class LinearSVMFunction
|
||||
{
|
||||
public:
|
||||
typedef typename ParametersType::elem_type ElemType;
|
||||
typedef typename GetDenseMatType<ParametersType>::type DenseMatType;
|
||||
typedef typename GetSparseMatType<ParametersType>::type SparseMatType;
|
||||
typedef typename GetDenseColType<SparseMatType>::type DenseColType;
|
||||
typedef typename GetDenseRowType<ParametersType>::type DenseRowType;
|
||||
|
||||
/**
|
||||
* Construct the Linear SVM objective function with given parameters.
|
||||
*
|
||||
@@ -58,7 +64,7 @@ class LinearSVMFunction
|
||||
* @param numClasses Number of classes for classification.
|
||||
* @param fitIntercept If true, an intercept is fitted.
|
||||
*/
|
||||
static void InitializeWeights(arma::mat& weights,
|
||||
static void InitializeWeights(ParametersType& weights,
|
||||
const size_t featureSize,
|
||||
const size_t numClasses,
|
||||
const bool fitIntercept = false);
|
||||
@@ -67,10 +73,10 @@ class LinearSVMFunction
|
||||
* Constructs the ground truth label matrix with the passed labels.
|
||||
*
|
||||
* @param labels Labels associated with the training data.
|
||||
* @param groundTruth Pointer to arma::mat which stores the computed matrix.
|
||||
* @param groundTruth Reference to sparse matrix to stores the result.
|
||||
*/
|
||||
void GetGroundTruthMatrix(const arma::Row<size_t>& labels,
|
||||
arma::sp_mat& groundTruth);
|
||||
SparseMatType& groundTruth) const;
|
||||
|
||||
/**
|
||||
* Evaluate the hinge loss function for all the datapoints
|
||||
@@ -78,7 +84,7 @@ class LinearSVMFunction
|
||||
* @param parameters The parameters of the SVM.
|
||||
* @return The value of the loss function for the entire dataset.
|
||||
*/
|
||||
double Evaluate(const arma::mat& parameters);
|
||||
ElemType Evaluate(const ParametersType& parameters) const;
|
||||
|
||||
/**
|
||||
* Evaluate the hinge loss function on the specified datapoints.
|
||||
@@ -89,9 +95,9 @@ class LinearSVMFunction
|
||||
* @param batchSize Size of batch to process.
|
||||
* @return The value of the loss function for the given parameters.
|
||||
*/
|
||||
double Evaluate(const arma::mat& parameters,
|
||||
const size_t firstId,
|
||||
const size_t batchSize = 1);
|
||||
ElemType Evaluate(const ParametersType& parameters,
|
||||
const size_t firstId,
|
||||
const size_t batchSize = 1) const;
|
||||
|
||||
/**
|
||||
* Evaluate the gradient of the hinge loss function following the
|
||||
@@ -101,9 +107,9 @@ class LinearSVMFunction
|
||||
* @param parameters The parameters of the SVM.
|
||||
* @param gradient Linear matrix to output the gradient into.
|
||||
*/
|
||||
template <typename GradType>
|
||||
void Gradient(const arma::mat& parameters,
|
||||
GradType& gradient);
|
||||
template<typename GradType>
|
||||
void Gradient(const ParametersType& parameters,
|
||||
GradType& gradient) const;
|
||||
|
||||
/**
|
||||
* Evaluate the gradient of the hinge loss function, following
|
||||
@@ -115,11 +121,11 @@ class LinearSVMFunction
|
||||
* @param gradient Linear matrix to output the gradient into.
|
||||
* @param batchSize Size of the batch to process.
|
||||
*/
|
||||
template <typename GradType>
|
||||
void Gradient(const arma::mat& parameters,
|
||||
template<typename GradType>
|
||||
void Gradient(const ParametersType& parameters,
|
||||
const size_t firstId,
|
||||
GradType& gradient,
|
||||
const size_t batchSize = 1);
|
||||
const size_t batchSize = 1) const;
|
||||
|
||||
/**
|
||||
* Evaluate the gradient of the hinge loss function, following
|
||||
@@ -132,9 +138,9 @@ class LinearSVMFunction
|
||||
* @param gradient Linear matrix to output the gradient into.
|
||||
* @return The value of the loss function at the given parameters.
|
||||
*/
|
||||
template <typename GradType>
|
||||
double EvaluateWithGradient(const arma::mat& parameters,
|
||||
GradType& gradient) const;
|
||||
template<typename GradType>
|
||||
ElemType EvaluateWithGradient(const ParametersType& parameters,
|
||||
GradType& gradient) const;
|
||||
|
||||
/**
|
||||
* Evaluate the gradient of the hinge loss function, following
|
||||
@@ -150,21 +156,21 @@ class LinearSVMFunction
|
||||
* @param batchSize Size of the batch to process.
|
||||
* @return The value of the loss function at the given parameters.
|
||||
*/
|
||||
template <typename GradType>
|
||||
double EvaluateWithGradient(const arma::mat& parameters,
|
||||
const size_t firstId,
|
||||
GradType& gradient,
|
||||
const size_t batchSize = 1) const;
|
||||
template<typename GradType>
|
||||
ElemType EvaluateWithGradient(const ParametersType& parameters,
|
||||
const size_t firstId,
|
||||
GradType& gradient,
|
||||
const size_t batchSize = 1) const;
|
||||
|
||||
//! Return the initial point for the optimization.
|
||||
const arma::mat& InitialPoint() const { return initialPoint; }
|
||||
const ParametersType& InitialPoint() const { return initialPoint; }
|
||||
//! Modify the initial point for the optimization.
|
||||
arma::mat& InitialPoint() { return initialPoint; }
|
||||
ParametersType& InitialPoint() { return initialPoint; }
|
||||
|
||||
//! Get the dataset.
|
||||
const arma::sp_mat& Dataset() const { return dataset; }
|
||||
const SparseMatType& Dataset() const { return dataset; }
|
||||
//! Modify the dataset.
|
||||
arma::sp_mat& Dataset() { return dataset; }
|
||||
SparseMatType& Dataset() { return dataset; }
|
||||
|
||||
//! Sets the regularization parameter.
|
||||
double& Lambda() { return lambda; }
|
||||
@@ -179,12 +185,13 @@ class LinearSVMFunction
|
||||
|
||||
private:
|
||||
//! The initial point, from which to start the optimization.
|
||||
arma::mat initialPoint;
|
||||
ParametersType initialPoint;
|
||||
|
||||
//! Label matrix for provided data
|
||||
arma::sp_mat groundTruth;
|
||||
SparseMatType groundTruth;
|
||||
|
||||
//! The datapoints for training.
|
||||
//! The datapoints for training. This will be an alias until Shuffle() is
|
||||
//! called.
|
||||
MatType dataset;
|
||||
|
||||
//! Number of Classes.
|
||||
|
||||
@@ -22,8 +22,8 @@
|
||||
|
||||
namespace mlpack {
|
||||
|
||||
template <typename MatType>
|
||||
LinearSVMFunction<MatType>::LinearSVMFunction(
|
||||
template<typename MatType, typename ParametersType>
|
||||
LinearSVMFunction<MatType, ParametersType>::LinearSVMFunction(
|
||||
const MatType& dataset,
|
||||
const arma::Row<size_t>& labels,
|
||||
const size_t numClasses,
|
||||
@@ -48,9 +48,9 @@ LinearSVMFunction<MatType>::LinearSVMFunction(
|
||||
* normal distribution. The weights cannot be initialized to zero, as that will
|
||||
* lead to each class output being the same.
|
||||
*/
|
||||
template <typename MatType>
|
||||
void LinearSVMFunction<MatType>::InitializeWeights(
|
||||
arma::mat &weights,
|
||||
template<typename MatType, typename ParametersType>
|
||||
void LinearSVMFunction<MatType, ParametersType>::InitializeWeights(
|
||||
ParametersType& weights,
|
||||
const size_t featureSize,
|
||||
const size_t numClasses,
|
||||
const bool fitIntercept)
|
||||
@@ -69,10 +69,11 @@ void LinearSVMFunction<MatType>::InitializeWeights(
|
||||
* labels. The output is in the form of a matrix, which leads to simpler
|
||||
* calculations in the Evaluate() and Gradient() methods.
|
||||
*/
|
||||
template <typename MatType>
|
||||
void LinearSVMFunction<MatType>::GetGroundTruthMatrix(
|
||||
template<typename MatType, typename ParametersType>
|
||||
void LinearSVMFunction<MatType, ParametersType>::GetGroundTruthMatrix(
|
||||
const arma::Row<size_t>& labels,
|
||||
arma::sp_mat& groundTruth)
|
||||
typename LinearSVMFunction<MatType, ParametersType>::SparseMatType&
|
||||
groundTruth) const
|
||||
{
|
||||
// Calculate the ground truth matrix according to the labels passed. The
|
||||
// ground truth matrix is a matrix of dimensions 'numClasses * numExamples',
|
||||
@@ -95,19 +96,19 @@ void LinearSVMFunction<MatType>::GetGroundTruthMatrix(
|
||||
}
|
||||
|
||||
// All entries are '1'.
|
||||
arma::vec values;
|
||||
DenseColType values;
|
||||
values.ones(labels.n_elem);
|
||||
|
||||
// Calculate the matrix.
|
||||
groundTruth = arma::sp_mat(rowPointers, colPointers, values, numClasses,
|
||||
labels.n_elem);
|
||||
groundTruth = SparseMatType(rowPointers, colPointers, values, numClasses,
|
||||
labels.n_elem);
|
||||
}
|
||||
|
||||
/**
|
||||
* Shuffle the data.
|
||||
*/
|
||||
template <typename MatType>
|
||||
void LinearSVMFunction<MatType>::Shuffle()
|
||||
template<typename MatType, typename ParametersType>
|
||||
void LinearSVMFunction<MatType, ParametersType>::Shuffle()
|
||||
{
|
||||
// Determine new ordering.
|
||||
arma::uvec ordering = arma::shuffle(arma::linspace<arma::uvec>(0,
|
||||
@@ -124,26 +125,27 @@ void LinearSVMFunction<MatType>::Shuffle()
|
||||
reverseOrdering[ordering[i]] = i;
|
||||
|
||||
arma::umat newLocations(2, groundTruth.n_nonzero);
|
||||
arma::vec values(groundTruth.n_nonzero);
|
||||
arma::sp_mat::const_iterator it = groundTruth.begin();
|
||||
DenseColType values(groundTruth.n_nonzero);
|
||||
typename SparseMatType::const_iterator it = groundTruth.begin();
|
||||
size_t loc = 0;
|
||||
while (it != groundTruth.end())
|
||||
{
|
||||
newLocations(0, loc) = reverseOrdering(it.col());
|
||||
newLocations(1, loc) = it.row();
|
||||
newLocations(0, loc) = it.row();
|
||||
newLocations(1, loc) = reverseOrdering(it.col());
|
||||
values(loc) = (*it);
|
||||
|
||||
++it;
|
||||
++loc;
|
||||
}
|
||||
|
||||
groundTruth = arma::sp_mat(newLocations, values, groundTruth.n_rows,
|
||||
groundTruth.n_cols);
|
||||
groundTruth = SparseMatType(newLocations, values, groundTruth.n_rows,
|
||||
groundTruth.n_cols);
|
||||
}
|
||||
|
||||
template <typename MatType>
|
||||
double LinearSVMFunction<MatType>::Evaluate(
|
||||
const arma::mat& parameters)
|
||||
template<typename MatType, typename ParametersType>
|
||||
typename LinearSVMFunction<MatType, ParametersType>::ElemType
|
||||
LinearSVMFunction<MatType, ParametersType>::Evaluate(
|
||||
const ParametersType& parameters) const
|
||||
{
|
||||
// The objective function is the hinge loss function and it is
|
||||
// calculated over all the training examples.
|
||||
@@ -151,10 +153,10 @@ double LinearSVMFunction<MatType>::Evaluate(
|
||||
// Calculate the loss and regularization terms.
|
||||
// L_i = Σ_i Σ_m max(0, Δ + (w_m x_i + b_m) - (w_{y_i} x_i + b_{y_i}))
|
||||
// where (m != y_i)
|
||||
double loss, regularization;
|
||||
ElemType loss, regularization;
|
||||
|
||||
// Scores for each class are evaluated.
|
||||
arma::mat scores;
|
||||
DenseMatType scores;
|
||||
|
||||
// Check intercept condition.
|
||||
if (!fitIntercept)
|
||||
@@ -178,7 +180,7 @@ double LinearSVMFunction<MatType>::Evaluate(
|
||||
// - Adding the margin parameter `delta`.
|
||||
// - Removing the `delta` parameter from correct class label in each
|
||||
// column.
|
||||
arma::mat margin = scores - (arma::repmat(arma::ones(numClasses).t()
|
||||
DenseMatType margin = scores - (arma::repmat(arma::ones(numClasses).t()
|
||||
* (scores % groundTruth), numClasses, 1)) + delta
|
||||
- (delta * groundTruth);
|
||||
|
||||
@@ -186,24 +188,26 @@ double LinearSVMFunction<MatType>::Evaluate(
|
||||
loss = arma::accu(arma::clamp(margin, 0.0, DBL_MAX)) / dataset.n_cols;
|
||||
|
||||
// Adding the regularization term.
|
||||
regularization = 0.5 * lambda * arma::dot(parameters, parameters);
|
||||
constexpr ElemType half = ((ElemType) 0.5);
|
||||
regularization = half * lambda * arma::dot(parameters, parameters);
|
||||
|
||||
return loss + regularization;
|
||||
}
|
||||
|
||||
template <typename MatType>
|
||||
double LinearSVMFunction<MatType>::Evaluate(
|
||||
const arma::mat& parameters,
|
||||
template<typename MatType, typename ParametersType>
|
||||
typename LinearSVMFunction<MatType, ParametersType>::ElemType
|
||||
LinearSVMFunction<MatType, ParametersType>::Evaluate(
|
||||
const ParametersType& parameters,
|
||||
const size_t firstId,
|
||||
const size_t batchSize)
|
||||
const size_t batchSize) const
|
||||
{
|
||||
const size_t lastId = firstId + batchSize - 1;
|
||||
|
||||
// Calculate the loss and regularization terms.
|
||||
double loss, regularization, cost;
|
||||
ElemType loss, regularization, cost;
|
||||
|
||||
// Scores for each class are evaluated.
|
||||
arma::mat scores;
|
||||
DenseMatType scores;
|
||||
|
||||
// Check intercept condition.
|
||||
if (!fitIntercept)
|
||||
@@ -218,7 +222,7 @@ double LinearSVMFunction<MatType>::Evaluate(
|
||||
dataset.n_cols);
|
||||
}
|
||||
|
||||
arma::mat margin = scores - (arma::repmat(arma::ones(numClasses).t()
|
||||
DenseMatType margin = scores - (arma::repmat(arma::ones(numClasses).t()
|
||||
* (scores % groundTruth.cols(firstId, lastId)), numClasses, 1))
|
||||
+ delta - (delta * groundTruth.cols(firstId, lastId));
|
||||
|
||||
@@ -227,17 +231,18 @@ double LinearSVMFunction<MatType>::Evaluate(
|
||||
loss /= batchSize;
|
||||
|
||||
// Adding the regularization term.
|
||||
regularization = 0.5 * lambda * arma::dot(parameters, parameters);
|
||||
constexpr ElemType half = ((ElemType) 0.5);
|
||||
regularization = half * lambda * arma::dot(parameters, parameters);
|
||||
|
||||
cost = loss + regularization;
|
||||
return cost;
|
||||
}
|
||||
|
||||
template <typename MatType>
|
||||
template <typename GradType>
|
||||
void LinearSVMFunction<MatType>::Gradient(
|
||||
const arma::mat& parameters,
|
||||
GradType& gradient)
|
||||
template<typename MatType, typename ParametersType>
|
||||
template<typename GradType>
|
||||
void LinearSVMFunction<MatType, ParametersType>::Gradient(
|
||||
const ParametersType& parameters,
|
||||
GradType& gradient) const
|
||||
{
|
||||
// The objective is to minimize the loss, which is evaluated as the sum
|
||||
// of all the positive elements of `margin` matrix.
|
||||
@@ -245,7 +250,7 @@ void LinearSVMFunction<MatType>::Gradient(
|
||||
// Also, we need to increase the score of the correct class.
|
||||
|
||||
// Scores for each class are evaluated.
|
||||
arma::mat scores;
|
||||
DenseMatType scores;
|
||||
|
||||
if (!fitIntercept)
|
||||
{
|
||||
@@ -258,16 +263,16 @@ void LinearSVMFunction<MatType>::Gradient(
|
||||
dataset.n_cols);
|
||||
}
|
||||
|
||||
arma::mat margin = scores - (arma::repmat(arma::ones(numClasses).t()
|
||||
DenseMatType margin = scores - (arma::repmat(arma::ones(numClasses).t()
|
||||
* (scores % groundTruth), numClasses, 1)) + delta
|
||||
- (delta * groundTruth);
|
||||
|
||||
// An element of `mask` matrix holds `1` corresponding to
|
||||
// each positive element of `margin` matrix.
|
||||
arma::mat mask = margin.for_each([](arma::mat::elem_type& val)
|
||||
DenseMatType mask = margin.for_each([](arma::mat::elem_type& val)
|
||||
{ val = (val > 0) ? 1: 0; });
|
||||
|
||||
arma::mat difference = groundTruth
|
||||
DenseMatType difference = groundTruth
|
||||
% (-arma::repmat(arma::sum(mask), numClasses, 1)) + mask;
|
||||
|
||||
// The gradient is evaluated as follows:
|
||||
@@ -288,7 +293,7 @@ void LinearSVMFunction<MatType>::Gradient(
|
||||
gradient.submat(0, 0, parameters.n_rows - 2, parameters.n_cols - 1) =
|
||||
dataset * difference.t();
|
||||
gradient.row(parameters.n_rows - 1) =
|
||||
arma::ones<arma::rowvec>(dataset.n_cols) * difference.t();
|
||||
arma::ones<DenseRowType>(dataset.n_cols) * difference.t();
|
||||
}
|
||||
|
||||
gradient /= dataset.n_cols;
|
||||
@@ -297,18 +302,18 @@ void LinearSVMFunction<MatType>::Gradient(
|
||||
gradient += lambda * parameters;
|
||||
}
|
||||
|
||||
template <typename MatType>
|
||||
template <typename GradType>
|
||||
void LinearSVMFunction<MatType>::Gradient(
|
||||
const arma::mat& parameters,
|
||||
template<typename MatType, typename ParametersType>
|
||||
template<typename GradType>
|
||||
void LinearSVMFunction<MatType, ParametersType>::Gradient(
|
||||
const ParametersType& parameters,
|
||||
const size_t firstId,
|
||||
GradType& gradient,
|
||||
const size_t batchSize)
|
||||
const size_t batchSize) const
|
||||
{
|
||||
const size_t lastId = firstId + batchSize - 1;
|
||||
|
||||
// Scores for each class are evaluated.
|
||||
arma::mat scores;
|
||||
DenseMatType scores;
|
||||
|
||||
// Check intercept condition.
|
||||
if (!fitIntercept)
|
||||
@@ -322,16 +327,16 @@ void LinearSVMFunction<MatType>::Gradient(
|
||||
+ arma::repmat(parameters.row(dataset.n_rows).t(), 1, batchSize);
|
||||
}
|
||||
|
||||
arma::mat margin = scores - (arma::repmat(arma::ones(numClasses).t()
|
||||
DenseMatType margin = scores - (arma::repmat(arma::ones(numClasses).t()
|
||||
* (scores % groundTruth.cols(firstId, lastId)), numClasses, 1))
|
||||
+ delta - (delta * groundTruth.cols(firstId, lastId));
|
||||
|
||||
// For each sample, find the total number of classes where
|
||||
// ( margin > 0 ).
|
||||
arma::mat mask = margin.for_each([](arma::mat::elem_type& val)
|
||||
DenseMatType mask = margin.for_each([](arma::mat::elem_type& val)
|
||||
{ val = (val > 0) ? 1: 0; });
|
||||
|
||||
arma::mat difference = groundTruth.cols(firstId, lastId)
|
||||
DenseMatType difference = groundTruth.cols(firstId, lastId)
|
||||
% (-arma::repmat(arma::sum(mask), numClasses, 1)) + mask;
|
||||
|
||||
// Check intercept condition
|
||||
@@ -345,7 +350,7 @@ void LinearSVMFunction<MatType>::Gradient(
|
||||
gradient.submat(0, 0, parameters.n_rows - 2, parameters.n_cols - 1) =
|
||||
dataset.cols(firstId, lastId) * difference.t();
|
||||
gradient.row(parameters.n_rows - 1) =
|
||||
arma::ones<arma::rowvec>(batchSize) * difference.t();
|
||||
arma::ones<DenseRowType>(batchSize) * difference.t();
|
||||
}
|
||||
|
||||
gradient /= batchSize;
|
||||
@@ -354,16 +359,17 @@ void LinearSVMFunction<MatType>::Gradient(
|
||||
gradient += lambda * parameters;
|
||||
}
|
||||
|
||||
template <typename MatType>
|
||||
template <typename GradType>
|
||||
double LinearSVMFunction<MatType>::EvaluateWithGradient(
|
||||
const arma::mat& parameters,
|
||||
template<typename MatType, typename ParametersType>
|
||||
template<typename GradType>
|
||||
typename LinearSVMFunction<MatType, ParametersType>::ElemType
|
||||
LinearSVMFunction<MatType, ParametersType>::EvaluateWithGradient(
|
||||
const ParametersType& parameters,
|
||||
GradType& gradient) const
|
||||
{
|
||||
double loss, regularization, cost;
|
||||
ElemType loss, regularization, cost;
|
||||
|
||||
// Scores for each class are evaluated.
|
||||
arma::mat scores;
|
||||
DenseMatType scores;
|
||||
|
||||
if (!fitIntercept)
|
||||
{
|
||||
@@ -376,16 +382,16 @@ double LinearSVMFunction<MatType>::EvaluateWithGradient(
|
||||
dataset.n_cols);
|
||||
}
|
||||
|
||||
arma::mat margin = scores - (arma::repmat(arma::ones(numClasses).t()
|
||||
* (scores % groundTruth), numClasses, 1)) + delta
|
||||
- (delta * groundTruth);
|
||||
DenseMatType margin = scores - (arma::repmat(
|
||||
arma::ones<DenseColType>(numClasses).t() * (scores % groundTruth),
|
||||
numClasses, 1)) + delta - (delta * groundTruth);
|
||||
|
||||
// For each sample, find the total number of classes where
|
||||
// ( margin > 0 ).
|
||||
arma::mat mask = margin.for_each([](arma::mat::elem_type& val)
|
||||
DenseMatType mask = margin.for_each([](ElemType& val)
|
||||
{ val = (val > 0) ? 1: 0; });
|
||||
|
||||
arma::mat difference = groundTruth
|
||||
DenseMatType difference = groundTruth
|
||||
% (-arma::repmat(arma::sum(mask), numClasses, 1)) + mask;
|
||||
|
||||
// Check intercept condition
|
||||
@@ -399,7 +405,7 @@ double LinearSVMFunction<MatType>::EvaluateWithGradient(
|
||||
gradient.submat(0, 0, parameters.n_rows - 2, parameters.n_cols - 1) =
|
||||
dataset * difference.t();
|
||||
gradient.row(parameters.n_rows - 1) =
|
||||
arma::ones<arma::rowvec>(dataset.n_cols) * difference.t();
|
||||
arma::ones<DenseRowType>(dataset.n_cols) * difference.t();
|
||||
}
|
||||
|
||||
gradient /= dataset.n_cols;
|
||||
@@ -412,16 +418,18 @@ double LinearSVMFunction<MatType>::EvaluateWithGradient(
|
||||
loss /= dataset.n_cols;
|
||||
|
||||
// Adding the regularization term.
|
||||
regularization = 0.5 * lambda * arma::dot(parameters, parameters);
|
||||
constexpr ElemType half = ((ElemType) 0.5);
|
||||
regularization = half * lambda * arma::dot(parameters, parameters);
|
||||
|
||||
cost = loss + regularization;
|
||||
return cost;
|
||||
}
|
||||
|
||||
template <typename MatType>
|
||||
template <typename GradType>
|
||||
double LinearSVMFunction<MatType>::EvaluateWithGradient(
|
||||
const arma::mat& parameters,
|
||||
template<typename MatType, typename ParametersType>
|
||||
template<typename GradType>
|
||||
typename LinearSVMFunction<MatType, ParametersType>::ElemType
|
||||
LinearSVMFunction<MatType, ParametersType>::EvaluateWithGradient(
|
||||
const ParametersType& parameters,
|
||||
const size_t firstId,
|
||||
GradType& gradient,
|
||||
const size_t batchSize) const
|
||||
@@ -429,10 +437,10 @@ double LinearSVMFunction<MatType>::EvaluateWithGradient(
|
||||
const size_t lastId = firstId + batchSize - 1;
|
||||
|
||||
// Calculate the loss and regularization terms.
|
||||
double loss, regularization, cost;
|
||||
ElemType loss, regularization, cost;
|
||||
|
||||
// Scores for each class are evaluated.
|
||||
arma::mat scores;
|
||||
DenseMatType scores;
|
||||
|
||||
// Check intercept condition.
|
||||
if (!fitIntercept)
|
||||
@@ -443,19 +451,20 @@ double LinearSVMFunction<MatType>::EvaluateWithGradient(
|
||||
{
|
||||
scores = parameters.rows(0, dataset.n_rows - 1).t()
|
||||
* dataset.cols(firstId, lastId)
|
||||
+ arma::repmat(parameters.row(dataset.n_rows).t(), 1, dataset.n_cols);
|
||||
+ arma::repmat(parameters.row(dataset.n_rows).t(), 1,
|
||||
(lastId - firstId + 1));
|
||||
}
|
||||
|
||||
arma::mat margin = scores - (arma::repmat(arma::ones(numClasses).t()
|
||||
DenseMatType margin = scores - (arma::repmat(arma::ones(numClasses).t()
|
||||
* (scores % groundTruth.cols(firstId, lastId)), numClasses, 1))
|
||||
+ delta - (delta * groundTruth.cols(firstId, lastId));
|
||||
|
||||
// For each sample, find the total number of classes where
|
||||
// ( margin > 0 ).
|
||||
arma::mat mask = margin.for_each([](arma::mat::elem_type& val)
|
||||
DenseMatType mask = margin.for_each([](arma::mat::elem_type& val)
|
||||
{ val = (val > 0) ? 1: 0; });
|
||||
|
||||
arma::mat difference = groundTruth.cols(firstId, lastId)
|
||||
DenseMatType difference = groundTruth.cols(firstId, lastId)
|
||||
% (-arma::repmat(arma::sum(mask), numClasses, 1)) + mask;
|
||||
|
||||
// Check intercept condition
|
||||
@@ -469,7 +478,7 @@ double LinearSVMFunction<MatType>::EvaluateWithGradient(
|
||||
gradient.submat(0, 0, parameters.n_rows - 2, parameters.n_cols - 1) =
|
||||
dataset.cols(firstId, lastId) * difference.t();
|
||||
gradient.row(parameters.n_rows - 1) =
|
||||
arma::ones<arma::rowvec>(batchSize) * difference.t();
|
||||
arma::ones<DenseRowType>(batchSize) * difference.t();
|
||||
}
|
||||
|
||||
gradient /= batchSize;
|
||||
@@ -479,18 +488,19 @@ double LinearSVMFunction<MatType>::EvaluateWithGradient(
|
||||
gradient += lambda * parameters;
|
||||
|
||||
// The Hinge Loss Function
|
||||
loss = arma::accu(arma::clamp(margin.cols(firstId, lastId), 0.0, DBL_MAX));
|
||||
loss = arma::accu(arma::clamp(margin, 0.0, DBL_MAX));
|
||||
loss /= batchSize;
|
||||
|
||||
// Adding the regularization term.
|
||||
regularization = 0.5 * lambda * arma::dot(parameters, parameters);
|
||||
constexpr ElemType half = ((ElemType) 0.5);
|
||||
regularization = half * lambda * arma::dot(parameters, parameters);
|
||||
|
||||
cost = loss + regularization;
|
||||
return cost;
|
||||
}
|
||||
|
||||
template <typename MatType>
|
||||
size_t LinearSVMFunction<MatType>::NumFunctions() const
|
||||
template<typename MatType, typename ParametersType>
|
||||
size_t LinearSVMFunction<MatType, ParametersType>::NumFunctions() const
|
||||
{
|
||||
// The number of points in the dataset is the number of functions, as this
|
||||
// is a data dependent function.
|
||||
|
||||
@@ -17,10 +17,37 @@
|
||||
|
||||
namespace mlpack {
|
||||
|
||||
template <typename MatType>
|
||||
template <typename OptimizerType, typename... CallbackTypes>
|
||||
LinearSVM<MatType>::LinearSVM(
|
||||
const MatType& data,
|
||||
template<typename ModelMatType>
|
||||
LinearSVM<ModelMatType>::LinearSVM() :
|
||||
lambda(0.0001),
|
||||
delta(1.0),
|
||||
fitIntercept(false)
|
||||
{
|
||||
// No training to do here.
|
||||
}
|
||||
|
||||
template<typename ModelMatType>
|
||||
LinearSVM<ModelMatType>::LinearSVM(
|
||||
const size_t dimensionality,
|
||||
const size_t numClasses,
|
||||
const double lambda,
|
||||
const double delta,
|
||||
const bool fitIntercept) :
|
||||
numClasses(numClasses),
|
||||
lambda(lambda),
|
||||
delta(delta),
|
||||
fitIntercept(fitIntercept)
|
||||
{
|
||||
LinearSVMFunction<ModelMatType /* fake, does not matter for this call */,
|
||||
ModelMatType>::InitializeWeights(
|
||||
parameters, dimensionality, numClasses, fitIntercept);
|
||||
}
|
||||
|
||||
template<typename ModelMatType>
|
||||
template<typename OptimizerType, typename... CallbackTypes, typename, typename>
|
||||
mlpack_deprecated /** Will be removed in mlpack 5.0.0. **/
|
||||
LinearSVM<ModelMatType>::LinearSVM(
|
||||
const arma::mat& data,
|
||||
const arma::Row<size_t>& labels,
|
||||
const size_t numClasses,
|
||||
const double lambda,
|
||||
@@ -33,13 +60,15 @@ LinearSVM<MatType>::LinearSVM(
|
||||
delta(delta),
|
||||
fitIntercept(fitIntercept)
|
||||
{
|
||||
Train(data, labels, numClasses, optimizer, callbacks...);
|
||||
Train(data, labels, numClasses, optimizer,
|
||||
std::forward<CallbackTypes>(callbacks)...);
|
||||
}
|
||||
|
||||
template <typename MatType>
|
||||
template <typename OptimizerType>
|
||||
LinearSVM<MatType>::LinearSVM(
|
||||
const MatType& data,
|
||||
template<typename ModelMatType>
|
||||
template<typename OptimizerType, typename>
|
||||
mlpack_deprecated /** Will be removed in mlpack 5.0.0. **/
|
||||
LinearSVM<ModelMatType>::LinearSVM(
|
||||
const arma::mat& data,
|
||||
const arma::Row<size_t>& labels,
|
||||
const size_t numClasses,
|
||||
const double lambda,
|
||||
@@ -54,84 +83,182 @@ LinearSVM<MatType>::LinearSVM(
|
||||
Train(data, labels, numClasses, optimizer);
|
||||
}
|
||||
|
||||
template <typename MatType>
|
||||
LinearSVM<MatType>::LinearSVM(
|
||||
const size_t inputSize,
|
||||
template<typename ModelMatType>
|
||||
template<typename MatType, typename... CallbackTypes, typename>
|
||||
LinearSVM<ModelMatType>::LinearSVM(
|
||||
const MatType& data,
|
||||
const arma::Row<size_t>& labels,
|
||||
const size_t numClasses,
|
||||
const double lambda,
|
||||
const double delta,
|
||||
const bool fitIntercept) :
|
||||
const bool fitIntercept,
|
||||
CallbackTypes&&... callbacks) :
|
||||
numClasses(numClasses),
|
||||
lambda(lambda),
|
||||
delta(delta),
|
||||
fitIntercept(fitIntercept)
|
||||
{
|
||||
LinearSVMFunction<MatType>::InitializeWeights(parameters, inputSize,
|
||||
numClasses, fitIntercept);
|
||||
// By default we use L-BFGS.
|
||||
ens::L_BFGS optimizer;
|
||||
Train(data, labels, numClasses, optimizer,
|
||||
std::forward<CallbackTypes>(callbacks)...);
|
||||
}
|
||||
|
||||
template <typename MatType>
|
||||
LinearSVM<MatType>::LinearSVM(
|
||||
template<typename ModelMatType>
|
||||
template<typename MatType,
|
||||
typename OptimizerType,
|
||||
typename... CallbackTypes,
|
||||
typename>
|
||||
LinearSVM<ModelMatType>::LinearSVM(
|
||||
const MatType& data,
|
||||
const arma::Row<size_t>& labels,
|
||||
const size_t numClasses,
|
||||
OptimizerType& optimizer,
|
||||
const double lambda,
|
||||
const double delta,
|
||||
const bool fitIntercept) :
|
||||
const bool fitIntercept,
|
||||
CallbackTypes&&... callbacks) :
|
||||
numClasses(numClasses),
|
||||
lambda(lambda),
|
||||
delta(delta),
|
||||
fitIntercept(fitIntercept)
|
||||
{
|
||||
// No training to do here.
|
||||
Train(data, labels, numClasses, optimizer,
|
||||
std::forward<CallbackTypes>(callbacks)...);
|
||||
}
|
||||
|
||||
template <typename MatType>
|
||||
template <typename OptimizerType, typename... CallbackTypes>
|
||||
double LinearSVM<MatType>::Train(
|
||||
template<typename ModelMatType>
|
||||
template<typename MatType, typename... CallbackTypes, typename>
|
||||
typename LinearSVM<ModelMatType>::ElemType LinearSVM<ModelMatType>::Train(
|
||||
const MatType& data,
|
||||
const arma::Row<size_t>& labels,
|
||||
const size_t numClasses,
|
||||
CallbackTypes&&... callbacks)
|
||||
{
|
||||
return Train(data, labels, numClasses, this->lambda, this->delta,
|
||||
this->fitIntercept, std::forward<CallbackTypes>(callbacks)...);
|
||||
}
|
||||
|
||||
template<typename ModelMatType>
|
||||
template<typename MatType>
|
||||
typename LinearSVM<ModelMatType>::ElemType LinearSVM<ModelMatType>::Train(
|
||||
const MatType& data,
|
||||
const arma::Row<size_t>& labels,
|
||||
const size_t numClasses,
|
||||
const double lambda)
|
||||
{
|
||||
return Train(data, labels, numClasses, lambda, this->delta,
|
||||
this->fitIntercept);
|
||||
}
|
||||
|
||||
template<typename ModelMatType>
|
||||
template<typename MatType>
|
||||
typename LinearSVM<ModelMatType>::ElemType LinearSVM<ModelMatType>::Train(
|
||||
const MatType& data,
|
||||
const arma::Row<size_t>& labels,
|
||||
const size_t numClasses,
|
||||
const double lambda,
|
||||
const double delta)
|
||||
{
|
||||
return Train(data, labels, numClasses, lambda, delta, this->fitIntercept);
|
||||
}
|
||||
|
||||
template<typename ModelMatType>
|
||||
template<typename MatType, typename... CallbackTypes, typename>
|
||||
typename LinearSVM<ModelMatType>::ElemType LinearSVM<ModelMatType>::Train(
|
||||
const MatType& data,
|
||||
const arma::Row<size_t>& labels,
|
||||
const size_t numClasses,
|
||||
const double lambda,
|
||||
const double delta,
|
||||
const bool fitIntercept,
|
||||
CallbackTypes&&... callbacks)
|
||||
{
|
||||
// By default, train with L-BFGS.
|
||||
ens::L_BFGS lbfgs;
|
||||
return Train(data, labels, numClasses, lbfgs, lambda, delta, fitIntercept,
|
||||
std::forward<CallbackTypes>(callbacks)...);
|
||||
}
|
||||
|
||||
template<typename ModelMatType>
|
||||
template<typename MatType,
|
||||
typename OptimizerType,
|
||||
typename... CallbackTypes,
|
||||
typename, typename>
|
||||
typename LinearSVM<ModelMatType>::ElemType LinearSVM<ModelMatType>::Train(
|
||||
const MatType& data,
|
||||
const arma::Row<size_t>& labels,
|
||||
const size_t numClasses,
|
||||
OptimizerType optimizer,
|
||||
CallbackTypes&&... callbacks)
|
||||
{
|
||||
if (numClasses <= 1)
|
||||
{
|
||||
throw std::invalid_argument("LinearSVM dataset has 0 number of classes!");
|
||||
}
|
||||
|
||||
LinearSVMFunction<MatType> svm(data, labels, numClasses, lambda, delta,
|
||||
fitIntercept);
|
||||
if (parameters.is_empty())
|
||||
parameters = svm.InitialPoint();
|
||||
|
||||
// Train the model.
|
||||
const double out = optimizer.Optimize(svm, parameters, callbacks...);
|
||||
|
||||
Log::Info << "LinearSVM::LinearSVM(): final objective of "
|
||||
<< "trained model is " << out << "." << std::endl;
|
||||
|
||||
return out;
|
||||
return Train(data, labels, numClasses, optimizer, this->lambda, this->delta,
|
||||
this->fitIntercept, std::forward<CallbackTypes>(callbacks)...);
|
||||
}
|
||||
|
||||
template <typename MatType>
|
||||
template <typename OptimizerType>
|
||||
double LinearSVM<MatType>::Train(
|
||||
template<typename ModelMatType>
|
||||
template<typename MatType, typename OptimizerType, typename>
|
||||
typename LinearSVM<ModelMatType>::ElemType LinearSVM<ModelMatType>::Train(
|
||||
const MatType& data,
|
||||
const arma::Row<size_t>& labels,
|
||||
const size_t numClasses,
|
||||
OptimizerType optimizer)
|
||||
OptimizerType optimizer,
|
||||
const double lambda)
|
||||
{
|
||||
return Train(data, labels, numClasses, optimizer, lambda, this->delta,
|
||||
this->fitIntercept);
|
||||
}
|
||||
|
||||
template<typename ModelMatType>
|
||||
template<typename MatType, typename OptimizerType, typename>
|
||||
typename LinearSVM<ModelMatType>::ElemType LinearSVM<ModelMatType>::Train(
|
||||
const MatType& data,
|
||||
const arma::Row<size_t>& labels,
|
||||
const size_t numClasses,
|
||||
OptimizerType optimizer,
|
||||
const double lambda,
|
||||
const double delta)
|
||||
{
|
||||
return Train(data, labels, numClasses, optimizer, lambda, delta,
|
||||
this->fitIntercept);
|
||||
}
|
||||
|
||||
template<typename ModelMatType>
|
||||
template<typename MatType,
|
||||
typename OptimizerType,
|
||||
typename... CallbackTypes,
|
||||
typename, typename>
|
||||
typename LinearSVM<ModelMatType>::ElemType LinearSVM<ModelMatType>::Train(
|
||||
const MatType& data,
|
||||
const arma::Row<size_t>& labels,
|
||||
const size_t numClasses,
|
||||
OptimizerType optimizer,
|
||||
const double lambda,
|
||||
const double delta,
|
||||
const bool fitIntercept,
|
||||
CallbackTypes&&... callbacks)
|
||||
{
|
||||
this->numClasses = numClasses;
|
||||
this->lambda = lambda;
|
||||
this->delta = delta;
|
||||
this->fitIntercept = fitIntercept;
|
||||
|
||||
if (numClasses <= 1)
|
||||
{
|
||||
throw std::invalid_argument("LinearSVM dataset has 0 number of classes!");
|
||||
}
|
||||
|
||||
LinearSVMFunction<MatType> svm(data, labels, numClasses, lambda, delta,
|
||||
fitIntercept);
|
||||
if (parameters.is_empty())
|
||||
LinearSVMFunction<MatType, ModelMatType> svm(data, labels, numClasses, lambda,
|
||||
delta, fitIntercept);
|
||||
const bool needNewParameters = (parameters.is_empty() ||
|
||||
(fitIntercept && (parameters.n_rows != data.n_rows + 1)) ||
|
||||
(!fitIntercept && (parameters.n_rows != data.n_rows)) ||
|
||||
(parameters.n_cols != numClasses));
|
||||
if (needNewParameters)
|
||||
parameters = svm.InitialPoint();
|
||||
|
||||
// Train the model.
|
||||
const double out = optimizer.Optimize(svm, parameters);
|
||||
const ElemType out = optimizer.Optimize(svm, parameters, callbacks...);
|
||||
|
||||
Log::Info << "LinearSVM::LinearSVM(): final objective of "
|
||||
<< "trained model is " << out << "." << std::endl;
|
||||
@@ -139,22 +266,35 @@ double LinearSVM<MatType>::Train(
|
||||
return out;
|
||||
}
|
||||
|
||||
template <typename MatType>
|
||||
void LinearSVM<MatType>::Classify(
|
||||
template<typename ModelMatType>
|
||||
template<typename MatType>
|
||||
void LinearSVM<ModelMatType>::Classify(
|
||||
const MatType& data,
|
||||
arma::Row<size_t>& labels) const
|
||||
{
|
||||
arma::mat scores;
|
||||
DenseMatType scores;
|
||||
Classify(data, labels, scores);
|
||||
}
|
||||
|
||||
template <typename MatType>
|
||||
void LinearSVM<MatType>::Classify(
|
||||
template<typename ModelMatType>
|
||||
template<typename MatType>
|
||||
void LinearSVM<ModelMatType>::Classify(
|
||||
const MatType& data,
|
||||
arma::Row<size_t>& labels,
|
||||
arma::mat& scores) const
|
||||
typename LinearSVM<ModelMatType>::DenseMatType& scores) const
|
||||
{
|
||||
Classify(data, scores);
|
||||
util::CheckSameDimensionality(data, FeatureSize(), "LinearSVM::Classify()");
|
||||
|
||||
if (fitIntercept)
|
||||
{
|
||||
scores = parameters.rows(0, parameters.n_rows - 2).t() * data
|
||||
+ arma::repmat(parameters.row(parameters.n_rows - 1).t(), 1,
|
||||
data.n_cols);
|
||||
}
|
||||
else
|
||||
{
|
||||
scores = parameters.t() * data;
|
||||
}
|
||||
|
||||
// Prepare necessary data.
|
||||
labels.zeros(data.n_cols);
|
||||
@@ -163,9 +303,10 @@ void LinearSVM<MatType>::Classify(
|
||||
arma::index_max(scores));
|
||||
}
|
||||
|
||||
template <typename MatType>
|
||||
void LinearSVM<MatType>::Classify(
|
||||
const MatType& data,
|
||||
template<typename ModelMatType>
|
||||
mlpack_deprecated
|
||||
void LinearSVM<ModelMatType>::Classify(
|
||||
const arma::mat& data,
|
||||
arma::mat& scores) const
|
||||
{
|
||||
util::CheckSameDimensionality(data, FeatureSize(), "LinearSVM::Classify()");
|
||||
@@ -182,17 +323,30 @@ void LinearSVM<MatType>::Classify(
|
||||
}
|
||||
}
|
||||
|
||||
template <typename MatType>
|
||||
template <typename VecType>
|
||||
size_t LinearSVM<MatType>::Classify(const VecType& point) const
|
||||
template<typename ModelMatType>
|
||||
template<typename VecType>
|
||||
size_t LinearSVM<ModelMatType>::Classify(const VecType& point) const
|
||||
{
|
||||
arma::Row<size_t> label(1);
|
||||
Classify(point, label);
|
||||
return size_t(label(0));
|
||||
}
|
||||
|
||||
template <typename MatType>
|
||||
double LinearSVM<MatType>::ComputeAccuracy(
|
||||
template<typename ModelMatType>
|
||||
template<typename VecType>
|
||||
void LinearSVM<ModelMatType>::Classify(
|
||||
const VecType& point,
|
||||
size_t& label,
|
||||
typename LinearSVM<ModelMatType>::DenseColType& probabilities) const
|
||||
{
|
||||
arma::Row<size_t> labelRow(1);
|
||||
Classify(point, labelRow, probabilities);
|
||||
label = labelRow[0];
|
||||
}
|
||||
|
||||
template<typename ModelMatType>
|
||||
template<typename MatType>
|
||||
double LinearSVM<ModelMatType>::ComputeAccuracy(
|
||||
const MatType& testData,
|
||||
const arma::Row<size_t>& testLabels) const
|
||||
{
|
||||
@@ -208,7 +362,28 @@ double LinearSVM<MatType>::ComputeAccuracy(
|
||||
count++;
|
||||
|
||||
// Return the accuracy.
|
||||
return (double) count / labels.n_elem;
|
||||
return (double) 100.0 * count / labels.n_elem;
|
||||
}
|
||||
|
||||
template<typename ModelMatType>
|
||||
template<typename Archive>
|
||||
void LinearSVM<ModelMatType>::serialize(Archive& ar, const uint32_t version)
|
||||
{
|
||||
// Old versions used `arma::mat` for the type of `parameters`.
|
||||
if (cereal::is_loading<Archive>() && version == 0)
|
||||
{
|
||||
arma::mat parametersTmp;
|
||||
ar(cereal::make_nvp("parameters", parametersTmp));
|
||||
parameters = arma::conv_to<arma::mat>::from(parametersTmp);
|
||||
}
|
||||
else
|
||||
{
|
||||
ar(CEREAL_NVP(parameters));
|
||||
}
|
||||
|
||||
ar(CEREAL_NVP(numClasses));
|
||||
ar(CEREAL_NVP(lambda));
|
||||
ar(CEREAL_NVP(fitIntercept));
|
||||
}
|
||||
|
||||
} // namespace mlpack
|
||||
|
||||
@@ -405,7 +405,7 @@ void BINDING_FUNCTION(util::Params& params, util::Timers& timers)
|
||||
Log::Info << "Calculating class probabilities of points in " << testOutput
|
||||
<< "." << endl;
|
||||
arma::mat probabilities;
|
||||
model->svm.Classify(testSet, probabilities);
|
||||
model->svm.Classify(testSet, predictions, probabilities);
|
||||
params.Get<arma::mat>("probabilities") = std::move(probabilities);
|
||||
}
|
||||
|
||||
|
||||
@@ -141,14 +141,15 @@ inline void LocalCoordinateCoding::Encode(const arma::mat& data,
|
||||
|
||||
bool useCholesky = false;
|
||||
// Normalization and fitting and intercept are disabled.
|
||||
LARS lars(useCholesky, dictGramTD, 0.5 * lambda, 0,
|
||||
1e-16 /* default tolerance */, false, false);
|
||||
LARS<> lars(useCholesky, 0.5 * lambda, 0, 1e-16 /* default tolerance */,
|
||||
false, false);
|
||||
|
||||
// Run LARS for this point, by making an alias of the point and passing
|
||||
// that.
|
||||
arma::vec beta = codes.unsafe_col(i);
|
||||
arma::rowvec responses = data.unsafe_col(i).t();
|
||||
lars.Train(dictPrime, responses, beta, false);
|
||||
lars.Train(dictPrime, responses, false, useCholesky, dictGramTD);
|
||||
beta = lars.Beta();
|
||||
beta %= invW; // Remember, beta is an alias of codes.col(i).
|
||||
}
|
||||
}
|
||||
|
||||
@@ -77,7 +77,8 @@ class NaiveBayesClassifier
|
||||
* @param incrementalVariance If true, an incremental algorithm is used to
|
||||
* calculate the variance; this can prevent loss of precision in some
|
||||
* cases, but will be somewhat slower to calculate.
|
||||
* @param epsilon Small value to prevent log of zero.
|
||||
* @param epsilon Small initialization value for variances to prevent log of
|
||||
* zero.
|
||||
*/
|
||||
template<typename MatType>
|
||||
NaiveBayesClassifier(const MatType& data,
|
||||
@@ -109,7 +110,7 @@ class NaiveBayesClassifier
|
||||
*
|
||||
* @param data The dataset to train on.
|
||||
* @param labels The labels for the dataset.
|
||||
* @param numClasses The numbe of classes in the dataset.
|
||||
* @param numClasses The number of classes in the dataset.
|
||||
* @param incremental Whether or not to use the incremental algorithm for
|
||||
* training.
|
||||
*/
|
||||
@@ -119,6 +120,32 @@ class NaiveBayesClassifier
|
||||
const size_t numClasses,
|
||||
const bool incremental = true);
|
||||
|
||||
/**
|
||||
* Train the Naive Bayes classifier on the given dataset. If the incremental
|
||||
* algorithm is used, the current model is used as a starting point (this is
|
||||
* the default). If the incremental algorithm is not used, then the current
|
||||
* model is ignored and the new model will be trained only on the given data.
|
||||
* Note that even if the incremental algorithm is not used, the data must have
|
||||
* the same dimensionality and number of classes that the model was
|
||||
* initialized with. If you want to change the dimensionality or number of
|
||||
* classes, either re-initialize or call Means(), Variances(), and
|
||||
* Probabilities() individually to set them to the right size.
|
||||
*
|
||||
* @param data The dataset to train on.
|
||||
* @param labels The labels for the dataset.
|
||||
* @param numClasses The number of classes in the dataset.
|
||||
* @param incremental Whether or not to use the incremental algorithm for
|
||||
* training.
|
||||
* @param epsilon Small reinitialization value for variances to prevent log of
|
||||
* zero (ignored if incremental is true).
|
||||
*/
|
||||
template<typename MatType>
|
||||
void Train(const MatType& data,
|
||||
const arma::Row<size_t>& labels,
|
||||
const size_t numClasses,
|
||||
const bool incremental,
|
||||
const double epsilon);
|
||||
|
||||
/**
|
||||
* Train the Naive Bayes classifier on the given point. This will use the
|
||||
* incremental algorithm for updating the model parameters. The data must be
|
||||
@@ -198,6 +225,21 @@ class NaiveBayesClassifier
|
||||
arma::Row<size_t>& predictions,
|
||||
ProbabilitiesMatType& probabilities) const;
|
||||
|
||||
/**
|
||||
* Reset the model to zeros, keeping the model's current dimensionality and
|
||||
* number of classes.
|
||||
*/
|
||||
void Reset();
|
||||
|
||||
/**
|
||||
* Reset the model to zeros, with a new dimensionality and number of classes.
|
||||
* The value epsilon specifies an initial very small value for the variances,
|
||||
* to prevent log(0) issues.
|
||||
*/
|
||||
void Reset(const size_t dimensionality,
|
||||
const size_t numClasses,
|
||||
const double epsilon = 1e-10);
|
||||
|
||||
//! Get the sample means for each class.
|
||||
const ModelMatType& Means() const { return means; }
|
||||
//! Modify the sample means for each class.
|
||||
@@ -213,9 +255,12 @@ class NaiveBayesClassifier
|
||||
//! Modify the prior probabilities for each class.
|
||||
ModelMatType& Probabilities() { return probabilities; }
|
||||
|
||||
//! Get the number of points the model has been trained on so far.
|
||||
size_t TrainingPoints() const { return trainingPoints; }
|
||||
|
||||
//! Serialize the classifier.
|
||||
template<typename Archive>
|
||||
void serialize(Archive& ar, const uint32_t /* version */);
|
||||
void serialize(Archive& ar, const uint32_t version);
|
||||
|
||||
private:
|
||||
//! Sample mean for each class.
|
||||
@@ -244,6 +289,9 @@ class NaiveBayesClassifier
|
||||
|
||||
} // namespace mlpack
|
||||
|
||||
CEREAL_TEMPLATE_CLASS_VERSION((typename MatType),
|
||||
(mlpack::NaiveBayesClassifier<MatType>), (1));
|
||||
|
||||
// Include implementation.
|
||||
#include "naive_bayes_classifier_impl.hpp"
|
||||
|
||||
|
||||
@@ -46,7 +46,8 @@ NaiveBayesClassifier<ModelMatType>::NaiveBayesClassifier(
|
||||
{
|
||||
probabilities.zeros(numClasses);
|
||||
means.zeros(data.n_rows, numClasses);
|
||||
variances.zeros(data.n_rows, numClasses);
|
||||
variances.set_size(data.n_rows, numClasses);
|
||||
variances.fill(epsilon);
|
||||
}
|
||||
else
|
||||
{
|
||||
@@ -69,7 +70,8 @@ NaiveBayesClassifier<ModelMatType>::NaiveBayesClassifier(
|
||||
// Initialize model to 0.
|
||||
probabilities.zeros(numClasses);
|
||||
means.zeros(dimensionality, numClasses);
|
||||
variances.zeros(dimensionality, numClasses);
|
||||
variances.set_size(dimensionality, numClasses);
|
||||
variances.fill(epsilon);
|
||||
}
|
||||
|
||||
template<typename ModelMatType>
|
||||
@@ -84,30 +86,14 @@ void NaiveBayesClassifier<ModelMatType>::Train(
|
||||
"NaiveBayesClassifier: element type of given data must match the element "
|
||||
"type of the model!");
|
||||
|
||||
// Do we need to resize the model?
|
||||
if (probabilities.n_elem != numClasses)
|
||||
{
|
||||
// Perform training, after initializing the model to 0 (that is, if Train()
|
||||
// won't do that for us, which it won't if we're using the incremental
|
||||
// algorithm).
|
||||
if (incremental)
|
||||
{
|
||||
probabilities.zeros(numClasses);
|
||||
means.zeros(data.n_rows, numClasses);
|
||||
variances.zeros(data.n_rows, numClasses);
|
||||
}
|
||||
else
|
||||
{
|
||||
probabilities.set_size(numClasses);
|
||||
means.set_size(data.n_rows, numClasses);
|
||||
variances.set_size(data.n_rows, numClasses);
|
||||
}
|
||||
}
|
||||
|
||||
// Calculate the class probabilities as well as the sample mean and variance
|
||||
// for each of the features with respect to each of the labels.
|
||||
if (incremental)
|
||||
{
|
||||
// Do we need to resize the model?
|
||||
if (probabilities.n_elem != numClasses || data.n_rows != means.n_rows)
|
||||
Reset(data.n_rows, numClasses);
|
||||
|
||||
// Use incremental algorithm.
|
||||
// Fist, de-normalize probabilities.
|
||||
probabilities *= trainingPoints;
|
||||
@@ -117,7 +103,7 @@ void NaiveBayesClassifier<ModelMatType>::Train(
|
||||
const size_t label = labels[j];
|
||||
++probabilities[label];
|
||||
|
||||
arma::vec delta = data.col(j) - means.col(label);
|
||||
arma::Col<ElemType> delta = data.col(j) - means.col(label);
|
||||
means.col(label) += delta / probabilities[label];
|
||||
variances.col(label) += delta % (data.col(j) - means.col(label));
|
||||
}
|
||||
@@ -131,9 +117,9 @@ void NaiveBayesClassifier<ModelMatType>::Train(
|
||||
else
|
||||
{
|
||||
// Set all parameters to zero.
|
||||
probabilities.zeros();
|
||||
means.zeros();
|
||||
variances.zeros();
|
||||
probabilities.zeros(numClasses);
|
||||
means.zeros(data.n_rows, numClasses);
|
||||
variances.zeros(data.n_rows, numClasses);
|
||||
|
||||
// Don't use incremental algorithm. This is a two-pass algorithm. It is
|
||||
// possible to calculate the means and variances using a faster one-pass
|
||||
@@ -174,6 +160,19 @@ void NaiveBayesClassifier<ModelMatType>::Train(
|
||||
trainingPoints += data.n_cols;
|
||||
}
|
||||
|
||||
template<typename ModelMatType>
|
||||
template<typename MatType>
|
||||
void NaiveBayesClassifier<ModelMatType>::Train(
|
||||
const MatType& data,
|
||||
const arma::Row<size_t>& labels,
|
||||
const size_t numClasses,
|
||||
const bool incremental,
|
||||
const double epsilon)
|
||||
{
|
||||
this->epsilon = epsilon;
|
||||
Train(data, labels, numClasses, incremental);
|
||||
}
|
||||
|
||||
template<typename ModelMatType>
|
||||
template<typename VecType>
|
||||
void NaiveBayesClassifier<ModelMatType>::Train(const VecType& point,
|
||||
@@ -183,11 +182,20 @@ void NaiveBayesClassifier<ModelMatType>::Train(const VecType& point,
|
||||
"NaiveBayesClassifier: element type of given data must match the element "
|
||||
"type of the model!");
|
||||
|
||||
if (point.n_elem != means.n_rows)
|
||||
{
|
||||
std::ostringstream oss;
|
||||
oss << "NaiveBayesClassifier::Train(): given point has dimensionality "
|
||||
<< point.n_elem << ", but model has dimensionality " << means.n_rows
|
||||
<< "!";
|
||||
throw std::invalid_argument(oss.str());
|
||||
}
|
||||
|
||||
// We must use the incremental algorithm here.
|
||||
probabilities *= trainingPoints;
|
||||
probabilities[label]++;
|
||||
|
||||
arma::vec delta = point - means.col(label);
|
||||
arma::Col<ElemType> delta = point - means.col(label);
|
||||
means.col(label) += delta / probabilities[label];
|
||||
if (probabilities[label] > 2)
|
||||
variances.col(label) *= (probabilities[label] - 2);
|
||||
@@ -237,6 +245,15 @@ size_t NaiveBayesClassifier<ModelMatType>::Classify(const VecType& point) const
|
||||
"NaiveBayesClassifier: element type of given data must match the element "
|
||||
"type of the model!");
|
||||
|
||||
if (point.n_elem != means.n_rows)
|
||||
{
|
||||
std::ostringstream oss;
|
||||
oss << "NaiveBayesClassifier::Classify(): given point has dimensionality "
|
||||
<< point.n_elem << ", but model has dimensionality " << means.n_rows
|
||||
<< "!";
|
||||
throw std::invalid_argument(oss.str());
|
||||
}
|
||||
|
||||
// Find the label(class) with max log likelihood.
|
||||
ModelMatType logLikelihoods;
|
||||
LogLikelihood(point, logLikelihoods);
|
||||
@@ -261,6 +278,15 @@ void NaiveBayesClassifier<ModelMatType>::Classify(
|
||||
"NaiveBayesClassifier: element type of given data must match the element "
|
||||
"type of the model!");
|
||||
|
||||
if (point.n_elem != means.n_rows)
|
||||
{
|
||||
std::ostringstream oss;
|
||||
oss << "NaiveBayesClassifier::Classify(): given point has dimensionality "
|
||||
<< point.n_elem << ", but model has dimensionality " << means.n_rows
|
||||
<< "!";
|
||||
throw std::invalid_argument(oss.str());
|
||||
}
|
||||
|
||||
// log(Prob(Y|X)) = Log(Prob(X|Y)) + Log(Prob(Y)) - Log(Prob(X));
|
||||
// But LogLikelihood() gives us the unnormalized log likelihood which is
|
||||
// Log(Prob(X|Y)) + Log(Prob(Y)) so we need to subtract the normalization
|
||||
@@ -270,8 +296,8 @@ void NaiveBayesClassifier<ModelMatType>::Classify(
|
||||
|
||||
// To prevent underflow in log of sum of exp of x operation (where x is a
|
||||
// small negative value), we use logsumexp(x - max(x)) + max(x).
|
||||
const double maxValue = arma::max(logLikelihoods);
|
||||
const double logProbX = log(arma::accu(exp(logLikelihoods - maxValue))) +
|
||||
const ElemType maxValue = logLikelihoods.max();
|
||||
const ElemType logProbX = log(arma::accu(exp(logLikelihoods - maxValue))) +
|
||||
maxValue;
|
||||
probabilities = exp(logLikelihoods - logProbX); // log(exp(value)) == value.
|
||||
|
||||
@@ -290,6 +316,15 @@ void NaiveBayesClassifier<ModelMatType>::Classify(
|
||||
"NaiveBayesClassifier: element type of given data must match the element "
|
||||
"type of the model!");
|
||||
|
||||
if (data.n_rows != means.n_rows)
|
||||
{
|
||||
std::ostringstream oss;
|
||||
oss << "NaiveBayesClassifier::Classify(): given data has dimensionality "
|
||||
<< data.n_rows << ", but model has dimensionality " << means.n_rows
|
||||
<< "!";
|
||||
throw std::invalid_argument(oss.str());
|
||||
}
|
||||
|
||||
predictions.set_size(data.n_cols);
|
||||
|
||||
ModelMatType logLikelihoods;
|
||||
@@ -318,6 +353,15 @@ void NaiveBayesClassifier<ModelMatType>::Classify(
|
||||
"NaiveBayesClassifier: element type of given data must match the element "
|
||||
"type of the model!");
|
||||
|
||||
if (data.n_rows != means.n_rows)
|
||||
{
|
||||
std::ostringstream oss;
|
||||
oss << "NaiveBayesClassifier::Classify(): given data has dimensionality "
|
||||
<< data.n_rows << ", but model has dimensionality " << means.n_rows
|
||||
<< "!";
|
||||
throw std::invalid_argument(oss.str());
|
||||
}
|
||||
|
||||
predictions.set_size(data.n_cols);
|
||||
|
||||
ModelMatType logLikelihoods;
|
||||
@@ -346,15 +390,50 @@ void NaiveBayesClassifier<ModelMatType>::Classify(
|
||||
}
|
||||
}
|
||||
|
||||
template<typename ModelMatType>
|
||||
void NaiveBayesClassifier<ModelMatType>::Reset()
|
||||
{
|
||||
means.zeros();
|
||||
probabilities.zeros();
|
||||
variances.fill(epsilon);
|
||||
trainingPoints = 0;
|
||||
}
|
||||
|
||||
template<typename ModelMatType>
|
||||
void NaiveBayesClassifier<ModelMatType>::Reset(const size_t dimensionality,
|
||||
const size_t numClasses,
|
||||
const double epsilon)
|
||||
{
|
||||
this->epsilon = epsilon;
|
||||
|
||||
probabilities.zeros(numClasses);
|
||||
means.zeros(dimensionality, numClasses);
|
||||
variances.set_size(dimensionality, numClasses);
|
||||
variances.fill(epsilon);
|
||||
trainingPoints = 0;
|
||||
}
|
||||
|
||||
template<typename ModelMatType>
|
||||
template<typename Archive>
|
||||
void NaiveBayesClassifier<ModelMatType>::serialize(
|
||||
Archive& ar,
|
||||
const uint32_t /* version */)
|
||||
const uint32_t version)
|
||||
{
|
||||
ar(CEREAL_NVP(means));
|
||||
ar(CEREAL_NVP(variances));
|
||||
ar(CEREAL_NVP(probabilities));
|
||||
|
||||
if (cereal::is_loading<Archive>() && version == 0)
|
||||
{
|
||||
// Old versions did not serialize the trainingPoints or epsilon members.
|
||||
trainingPoints = 0;
|
||||
epsilon = 1e-10;
|
||||
}
|
||||
else
|
||||
{
|
||||
ar(CEREAL_NVP(trainingPoints));
|
||||
ar(CEREAL_NVP(epsilon));
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace mlpack
|
||||
|
||||
@@ -29,8 +29,8 @@ inline QUIC_SVD::QUIC_SVD(
|
||||
}
|
||||
|
||||
inline QUIC_SVD::QUIC_SVD(
|
||||
const double epsilon,
|
||||
const double delta)
|
||||
const double /* epsilon */,
|
||||
const double /* delta */)
|
||||
{
|
||||
/* Nothing to do here */
|
||||
}
|
||||
|
||||
@@ -71,15 +71,16 @@ inline void SparseCoding::Encode(const arma::mat& data,
|
||||
|
||||
bool useCholesky = true;
|
||||
// Intercept fitting and data normalization is disabled.
|
||||
LARS lars(useCholesky, matGram, lambda1, lambda2,
|
||||
1e-16 /* default tolerance */, false, false);
|
||||
LARS<> lars(useCholesky, lambda1, lambda2, 1e-16 /* default tolerance */,
|
||||
false, false);
|
||||
|
||||
// Create an alias of the code (using the same memory), and then LARS will
|
||||
// place the result directly into that; then we will not need to have an
|
||||
// extra copy.
|
||||
arma::vec code = codes.unsafe_col(i);
|
||||
arma::rowvec responses = data.unsafe_col(i).t();
|
||||
lars.Train(dictionary, responses, code, false);
|
||||
lars.Train(dictionary, responses, false, useCholesky, matGram);
|
||||
code = lars.Beta();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -42,7 +42,7 @@ TEST_CASE("BayesianLinearRegressionRegressionTest",
|
||||
GenerateProblem(matX, y, 200, 10);
|
||||
|
||||
// Instanciate and train the estimator.
|
||||
BayesianLinearRegression estimator(true);
|
||||
BayesianLinearRegression<> estimator(true);
|
||||
estimator.Train(matX, y);
|
||||
estimator.Predict(matX, predictions);
|
||||
|
||||
@@ -63,7 +63,7 @@ TEST_CASE("TestCenter0ScaleData0", "[BayesianLinearRegressionTest]")
|
||||
|
||||
GenerateProblem(matX, y, nPoints, nDims, 0.5);
|
||||
|
||||
BayesianLinearRegression estimator(false, false);
|
||||
BayesianLinearRegression<> estimator(false, false);
|
||||
|
||||
estimator.Train(matX, y);
|
||||
|
||||
@@ -85,7 +85,7 @@ TEST_CASE("TestCenterDataTrueScaleDataTrue", "[BayesianLinearRegressionTest]")
|
||||
size_t nDims = 5, nPoints = 100;
|
||||
GenerateProblem(matX, y, nPoints, nDims, 0.5);
|
||||
|
||||
BayesianLinearRegression estimator(true, true);
|
||||
BayesianLinearRegression<> estimator(true, true);
|
||||
estimator.Train(matX, y);
|
||||
|
||||
arma::colvec xMean = arma::mean(matX, 1);
|
||||
@@ -108,7 +108,7 @@ TEST_CASE("OptionsMakeModelDifferent", "[BayesianLinearRegressionTest]")
|
||||
size_t nDims = 10, nPoints = 100;
|
||||
GenerateProblem(matX, y, nPoints, nDims, 0.5);
|
||||
|
||||
BayesianLinearRegression blr(false, false), blrC(true, false),
|
||||
BayesianLinearRegression<> blr(false, false), blrC(true, false),
|
||||
blrCS(true, true);
|
||||
|
||||
blr.Train(matX, y);
|
||||
@@ -133,7 +133,7 @@ TEST_CASE("SingularMatix", "[BayesianLinearRegressionTest]")
|
||||
// Now the first and the second rows are indentical.
|
||||
matX.row(1) = matX.row(0);
|
||||
|
||||
BayesianLinearRegression estimator;
|
||||
BayesianLinearRegression<> estimator;
|
||||
estimator.Train(matX, y);
|
||||
}
|
||||
|
||||
@@ -146,7 +146,7 @@ TEST_CASE("PredictiveUncertainties", "[BayesianLinearRegressionTest]")
|
||||
|
||||
GenerateProblem(matX, y, 100, 10, 1);
|
||||
|
||||
BayesianLinearRegression estimator(true, true);
|
||||
BayesianLinearRegression<> estimator(true, true);
|
||||
estimator.Train(matX, y);
|
||||
|
||||
arma::rowvec responses, std;
|
||||
@@ -156,6 +156,15 @@ TEST_CASE("PredictiveUncertainties", "[BayesianLinearRegressionTest]")
|
||||
for (size_t i = 0; i < matX.n_cols; i++)
|
||||
REQUIRE(std[i] > estStd);
|
||||
|
||||
// Also make single-point predictions.
|
||||
for (size_t i = 0; i < matX.n_cols; ++i)
|
||||
{
|
||||
double prediction, stddev;
|
||||
estimator.Predict(matX.col(i), prediction, stddev);
|
||||
REQUIRE(prediction == Approx(responses[i]));
|
||||
REQUIRE(stddev > estStd);
|
||||
}
|
||||
|
||||
// Check that the estimated variance is close to 1.
|
||||
REQUIRE(estStd == Approx(1).epsilon(0.3));
|
||||
}
|
||||
@@ -171,10 +180,10 @@ TEST_CASE("EqualtoRidge", "[BayesianLinearRegressionTest]")
|
||||
{
|
||||
GenerateProblem(matX, y, 100, 10, 1);
|
||||
|
||||
BayesianLinearRegression blr(false, false);
|
||||
BayesianLinearRegression<> blr(false, false);
|
||||
blr.Train(matX, y);
|
||||
|
||||
LinearRegression ridge(matX, y, blr.Alpha() / blr.Beta(), false);
|
||||
LinearRegression<> ridge(matX, y, blr.Alpha() / blr.Beta(), false);
|
||||
|
||||
blr.Predict(matX, blrPred);
|
||||
ridge.Predict(matX, ridgePred);
|
||||
@@ -187,9 +196,120 @@ TEST_CASE("EqualtoRidge", "[BayesianLinearRegressionTest]")
|
||||
for (size_t i = 0; i < y.size(); ++i)
|
||||
REQUIRE(blrPred[i] == Approx(ridgePred[i]).epsilon(1));
|
||||
|
||||
// Also make single-point predictions.
|
||||
for (size_t i = 0; i < y.n_elem; ++i)
|
||||
REQUIRE(blr.Predict(matX.col(i)) == Approx(ridgePred[i]).epsilon(1));
|
||||
|
||||
// Exit once a test case has completed.
|
||||
break;
|
||||
}
|
||||
|
||||
REQUIRE(trial <= 3);
|
||||
}
|
||||
|
||||
// Check that all constructor variants work.
|
||||
TEMPLATE_TEST_CASE("BayesianLinearRegressionConstructorVariantTest",
|
||||
"[BayesianLinearRegressionTest]", arma::mat)
|
||||
{
|
||||
typedef TestType MatType;
|
||||
|
||||
MatType matX;
|
||||
arma::Row<typename MatType::elem_type> y;
|
||||
size_t nDims = 5, nPoints = 100;
|
||||
GenerateProblem(matX, y, nPoints, nDims, 0.5);
|
||||
|
||||
// The important thing here is that all of the inputs to the constructor are
|
||||
// properly parsed. The actual details of the models that are learned are
|
||||
// less important and are checked by other tests.
|
||||
BayesianLinearRegression<> blr1;
|
||||
BayesianLinearRegression<> blr2(false);
|
||||
BayesianLinearRegression<> blr3(false, true);
|
||||
BayesianLinearRegression<> blr4(false, true, 100);
|
||||
BayesianLinearRegression<> blr5(false, true, 110, 1e-3);
|
||||
BayesianLinearRegression<> blr6(matX, y);
|
||||
BayesianLinearRegression<> blr7(matX, y, false);
|
||||
BayesianLinearRegression<> blr8(matX, y, false, true);
|
||||
BayesianLinearRegression<> blr9(matX, y, false, true, 120);
|
||||
BayesianLinearRegression<> blr10(matX, y, false, true, 130, 1e-2);
|
||||
|
||||
// Check that the hyperparameters as reported by the model are correct.
|
||||
REQUIRE(blr1.Omega().n_elem == 0);
|
||||
|
||||
REQUIRE(blr2.Omega().n_elem == 0);
|
||||
REQUIRE(blr2.CenterData() == false);
|
||||
|
||||
REQUIRE(blr3.Omega().n_elem == 0);
|
||||
REQUIRE(blr3.CenterData() == false);
|
||||
REQUIRE(blr3.ScaleData() == true);
|
||||
|
||||
REQUIRE(blr4.Omega().n_elem == 0);
|
||||
REQUIRE(blr4.CenterData() == false);
|
||||
REQUIRE(blr4.ScaleData() == true);
|
||||
REQUIRE(blr4.MaxIterations() == 100);
|
||||
|
||||
REQUIRE(blr5.Omega().n_elem == 0);
|
||||
REQUIRE(blr5.CenterData() == false);
|
||||
REQUIRE(blr5.ScaleData() == true);
|
||||
REQUIRE(blr5.MaxIterations() == 110);
|
||||
REQUIRE(blr5.Tolerance() == 1e-3);
|
||||
|
||||
REQUIRE(blr6.Omega().n_elem == matX.n_rows);
|
||||
|
||||
REQUIRE(blr7.Omega().n_elem == matX.n_rows);
|
||||
REQUIRE(blr7.CenterData() == false);
|
||||
|
||||
REQUIRE(blr8.Omega().n_elem == matX.n_rows);
|
||||
REQUIRE(blr8.CenterData() == false);
|
||||
REQUIRE(blr8.ScaleData() == true);
|
||||
|
||||
REQUIRE(blr9.Omega().n_elem == matX.n_rows);
|
||||
REQUIRE(blr9.CenterData() == false);
|
||||
REQUIRE(blr9.ScaleData() == true);
|
||||
REQUIRE(blr9.MaxIterations() == 120);
|
||||
|
||||
REQUIRE(blr10.Omega().n_elem == matX.n_rows);
|
||||
REQUIRE(blr10.CenterData() == false);
|
||||
REQUIRE(blr10.ScaleData() == true);
|
||||
REQUIRE(blr10.MaxIterations() == 130);
|
||||
REQUIRE(blr10.Tolerance() == 1e-2);
|
||||
}
|
||||
|
||||
// Test that all Train() variants work.
|
||||
TEMPLATE_TEST_CASE("BayesianLinearRegressionTrainVariantTest",
|
||||
"[BayesianLinearRegressionTest]", arma::mat)
|
||||
{
|
||||
typedef TestType MatType;
|
||||
|
||||
MatType matX;
|
||||
arma::Row<typename MatType::elem_type> y;
|
||||
size_t nDims = 5, nPoints = 100;
|
||||
GenerateProblem(matX, y, nPoints, nDims, 0.5);
|
||||
|
||||
BayesianLinearRegression<> blr1, blr2, blr3, blr4, blr5;
|
||||
|
||||
blr1.Train(matX, y);
|
||||
blr2.Train(matX, y, false);
|
||||
blr3.Train(matX, y, false, true);
|
||||
blr4.Train(matX, y, false, true, 100);
|
||||
blr5.Train(matX, y, false, true, 110, 1e-3);
|
||||
|
||||
REQUIRE(blr1.Omega().n_elem == matX.n_rows);
|
||||
|
||||
REQUIRE(blr2.Omega().n_elem == matX.n_rows);
|
||||
REQUIRE(blr2.CenterData() == false);
|
||||
|
||||
REQUIRE(blr3.Omega().n_elem == matX.n_rows);
|
||||
REQUIRE(blr3.CenterData() == false);
|
||||
REQUIRE(blr3.ScaleData() == true);
|
||||
|
||||
REQUIRE(blr4.Omega().n_elem == matX.n_rows);
|
||||
REQUIRE(blr4.CenterData() == false);
|
||||
REQUIRE(blr4.ScaleData() == true);
|
||||
REQUIRE(blr4.MaxIterations() == 100);
|
||||
|
||||
REQUIRE(blr5.Omega().n_elem == matX.n_rows);
|
||||
REQUIRE(blr5.CenterData() == false);
|
||||
REQUIRE(blr5.ScaleData() == true);
|
||||
REQUIRE(blr5.MaxIterations() == 110);
|
||||
REQUIRE(blr5.Tolerance() == 1e-3);
|
||||
}
|
||||
|
||||
@@ -207,7 +207,7 @@ TEST_CASE("MSETest", "[CVTest]")
|
||||
arma::mat trainingData("0 1");
|
||||
arma::rowvec trainingResponses("-1 0");
|
||||
|
||||
LinearRegression lr(trainingData, trainingResponses);
|
||||
LinearRegression<> lr(trainingData, trainingResponses);
|
||||
|
||||
// Making three responses that differ from the correct ones by 0, 1, and 2
|
||||
// respectively
|
||||
@@ -229,7 +229,7 @@ TEST_CASE("R2ScoreTest", "[CVTest]")
|
||||
arma::mat trainingData("0 1");
|
||||
arma::rowvec trainingResponses("-1 0");
|
||||
|
||||
LinearRegression lr(trainingData, trainingResponses);
|
||||
LinearRegression<> lr(trainingData, trainingResponses);
|
||||
|
||||
// Making five responses that are the output of regression function f(x)
|
||||
// with some responses having a slight deviation of 0.005.
|
||||
@@ -256,7 +256,7 @@ TEST_CASE("AdjR2ScoreTest", "[CVTest]")
|
||||
arma::rowvec Y;
|
||||
Y = { 3, 5, 7, 9, 11, 13 };
|
||||
|
||||
LinearRegression lr(X, Y);
|
||||
LinearRegression<> lr(X, Y);
|
||||
|
||||
// Theoretically Adjusted R squared should be equal 1
|
||||
double expAdjR2 = 1;
|
||||
@@ -309,7 +309,7 @@ void CheckPredictionsType()
|
||||
*/
|
||||
TEST_CASE("PredictionsTypeTest", "[CVTest]")
|
||||
{
|
||||
CheckPredictionsType<LinearRegression, arma::rowvec>();
|
||||
CheckPredictionsType<LinearRegression<>, arma::rowvec>();
|
||||
// CheckPredictionsType<FFN<>, arma::mat>();
|
||||
|
||||
CheckPredictionsType<LogisticRegression<>, arma::Row<size_t>>();
|
||||
@@ -328,14 +328,14 @@ TEST_CASE("PredictionsTypeTest", "[CVTest]")
|
||||
*/
|
||||
TEST_CASE("SupportsWeightsTest", "[CVTest]")
|
||||
{
|
||||
static_assert(MetaInfoExtractor<LinearRegression>::SupportsWeights,
|
||||
static_assert(MetaInfoExtractor<LinearRegression<>>::SupportsWeights,
|
||||
"Value should be true");
|
||||
static_assert(MetaInfoExtractor<DecisionTree<>>::SupportsWeights,
|
||||
"Value should be true");
|
||||
static_assert(MetaInfoExtractor<DecisionTree<>, arma::mat, arma::urowvec,
|
||||
arma::Row<float>>::SupportsWeights, "Value should be true");
|
||||
|
||||
static_assert(!MetaInfoExtractor<LARS>::SupportsWeights,
|
||||
static_assert(!MetaInfoExtractor<LARS<>>::SupportsWeights,
|
||||
"Value should be false");
|
||||
static_assert(!MetaInfoExtractor<LogisticRegression<>>::SupportsWeights,
|
||||
"Value should be false");
|
||||
@@ -360,7 +360,7 @@ void CheckWeightsType()
|
||||
*/
|
||||
TEST_CASE("WeightsTypeTest", "[CVTest]")
|
||||
{
|
||||
CheckWeightsType<LinearRegression, arma::rowvec>();
|
||||
CheckWeightsType<LinearRegression<>, arma::rowvec>();
|
||||
CheckWeightsType<DecisionTree<>, arma::rowvec>();
|
||||
CheckWeightsType<DecisionTree<>, arma::Row<float>, arma::mat,
|
||||
arma::Row<size_t>, arma::Row<float>>();
|
||||
@@ -374,7 +374,7 @@ TEST_CASE("TakesDatasetInfoTest", "[CVTest]")
|
||||
{
|
||||
static_assert(MetaInfoExtractor<DecisionTree<>>::TakesDatasetInfo,
|
||||
"Value should be true");
|
||||
static_assert(!MetaInfoExtractor<LinearRegression>::TakesDatasetInfo,
|
||||
static_assert(!MetaInfoExtractor<LinearRegression<>>::TakesDatasetInfo,
|
||||
"Value should be false");
|
||||
static_assert(!MetaInfoExtractor<SoftmaxRegression<>>::TakesDatasetInfo,
|
||||
"Value should be false");
|
||||
@@ -390,9 +390,9 @@ TEST_CASE("TakesNumClassesTest", "[CVTest]")
|
||||
"Value should be true");
|
||||
static_assert(MetaInfoExtractor<SoftmaxRegression<>>::TakesNumClasses,
|
||||
"Value should be true");
|
||||
static_assert(!MetaInfoExtractor<LinearRegression>::TakesNumClasses,
|
||||
static_assert(!MetaInfoExtractor<LinearRegression<>>::TakesNumClasses,
|
||||
"Value should be false");
|
||||
static_assert(!MetaInfoExtractor<LARS>::TakesNumClasses,
|
||||
static_assert(!MetaInfoExtractor<LARS<>>::TakesNumClasses,
|
||||
"Value should be false");
|
||||
}
|
||||
|
||||
@@ -425,7 +425,7 @@ TEST_CASE("SimpleCVMSETest", "[CVTest]")
|
||||
|
||||
double expectedMSE = (0 * 0 + 1 * 1 + 2 * 2) / 3.0;
|
||||
|
||||
SimpleCV<LinearRegression, MSE> cv(0.6, data, responses);
|
||||
SimpleCV<LinearRegression<>, MSE> cv(0.6, data, responses);
|
||||
|
||||
REQUIRE(cv.Evaluate() == Approx(expectedMSE).epsilon(1e-7));
|
||||
|
||||
@@ -438,7 +438,7 @@ TEST_CASE("SimpleCVMSETest", "[CVTest]")
|
||||
arma::rowvec weights = arma::join_rows(arma::zeros(noiseData.n_cols).t(),
|
||||
arma::ones(data.n_cols).t());
|
||||
|
||||
SimpleCV<LinearRegression, MSE> weightedCV(0.3, allData, allResponces,
|
||||
SimpleCV<LinearRegression<>, MSE> weightedCV(0.3, allData, allResponces,
|
||||
weights);
|
||||
|
||||
REQUIRE(weightedCV.Evaluate() == Approx(expectedMSE).epsilon(1e-7));
|
||||
@@ -446,7 +446,7 @@ TEST_CASE("SimpleCVMSETest", "[CVTest]")
|
||||
arma::rowvec weights2 = arma::join_rows(arma::zeros(noiseData.n_cols - 1).t(),
|
||||
arma::ones(data.n_cols + 1).t());
|
||||
|
||||
SimpleCV<LinearRegression, MSE> weightedCV2(0.3, allData, allResponces,
|
||||
SimpleCV<LinearRegression<>, MSE> weightedCV2(0.3, allData, allResponces,
|
||||
weights2);
|
||||
|
||||
REQUIRE(std::abs(weightedCV2.Evaluate() - expectedMSE) > 1e-5);
|
||||
@@ -543,7 +543,7 @@ TEST_CASE("KFoldCVMSETest", "[CVTest]")
|
||||
arma::rowvec responses("0 1 1 3");
|
||||
|
||||
// 2-fold cross-validation, no shuffling.
|
||||
KFoldCV<LinearRegression, MSE> cv(2, data, responses, false);
|
||||
KFoldCV<LinearRegression<>, MSE> cv(2, data, responses, false);
|
||||
|
||||
// In each of two validation tests the MSE value should be the same.
|
||||
double expectedMSE =
|
||||
@@ -620,7 +620,7 @@ TEST_CASE("KFoldCVWithWeightedLRTest", "[CVTest]")
|
||||
arma::rowvec responses("1 2 30 40");
|
||||
arma::rowvec weights("1 1 0 0");
|
||||
|
||||
KFoldCV<LinearRegression, MSE> cv(2, arma::join_rows(data, data),
|
||||
KFoldCV<LinearRegression<>, MSE> cv(2, arma::join_rows(data, data),
|
||||
arma::join_rows(responses, responses), arma::join_rows(weights, weights),
|
||||
false);
|
||||
cv.Evaluate();
|
||||
|
||||
@@ -924,84 +924,61 @@ TEST_CASE("CategoricalMADGainWeightedBuildTest", "[DecisionTreeRegressorTest]")
|
||||
|
||||
// Make sure we get reasonable rmse.
|
||||
const double rmse = RMSE(predictions, testResponses);
|
||||
REQUIRE(rmse < 1.05);
|
||||
}
|
||||
|
||||
/**
|
||||
* Test that the decision tree generalizes reasonably.
|
||||
*/
|
||||
TEST_CASE("SimpleGeneralizationTest_", "[DecisionTreeRegressorTest]")
|
||||
{
|
||||
// Loading data.
|
||||
data::DatasetInfo info;
|
||||
arma::mat trainData, testData;
|
||||
arma::rowvec trainResponses, testResponses;
|
||||
LoadBostonHousingDataset(trainData, testData, trainResponses, testResponses,
|
||||
info);
|
||||
arma::rowvec weights = arma::ones<arma::rowvec>(trainResponses.n_elem);
|
||||
|
||||
// Build decision tree.
|
||||
DecisionTreeRegressor<> d(trainData, info, trainResponses);
|
||||
DecisionTreeRegressor<> wd(trainData, info, trainResponses, weights);
|
||||
|
||||
// Get the predicted test responses.
|
||||
arma::rowvec predictions;
|
||||
d.Predict(testData, predictions);
|
||||
|
||||
REQUIRE(predictions.n_elem == testData.n_cols);
|
||||
|
||||
// Figure out rmse.
|
||||
double rmse = RMSE(predictions, testResponses);
|
||||
REQUIRE(rmse < 6.1);
|
||||
|
||||
// Reset the predictions.
|
||||
predictions.zeros();
|
||||
wd.Predict(testData, predictions);
|
||||
|
||||
REQUIRE(predictions.n_elem == testData.n_cols);
|
||||
|
||||
// Figure out rmse.
|
||||
rmse = RMSE(predictions, testResponses);
|
||||
REQUIRE(rmse < 6.1);
|
||||
REQUIRE(rmse < 1.25);
|
||||
}
|
||||
|
||||
/**
|
||||
* Test that the decision tree generalizes reasonably when built on float data.
|
||||
*/
|
||||
TEST_CASE("SimpleGeneralizationFMatTest_", "[DecisionTreeRegressorTest]")
|
||||
TEMPLATE_TEST_CASE("SimpleGeneralizationTest_",
|
||||
"[DecisionTreeRegressorTest]", float, double)
|
||||
{
|
||||
// Loading data.
|
||||
data::DatasetInfo info;
|
||||
arma::fmat trainData, testData;
|
||||
arma::rowvec trainLabels, testLabels;
|
||||
LoadBostonHousingDataset(trainData, testData, trainLabels, testLabels, info);
|
||||
typedef TestType ElemType;
|
||||
|
||||
// Initialize an all-ones weight matrix.
|
||||
arma::rowvec weights(trainLabels.n_cols, arma::fill::ones);
|
||||
// Allow three trials.
|
||||
bool success = false;
|
||||
for (size_t trial = 0; trial < 3; ++trial)
|
||||
{
|
||||
// Loading data.
|
||||
data::DatasetInfo info;
|
||||
arma::Mat<ElemType> trainData, testData;
|
||||
arma::Row<ElemType> trainResponses, testResponses;
|
||||
LoadBostonHousingDataset(trainData, testData, trainResponses, testResponses,
|
||||
info);
|
||||
|
||||
// Build decision tree.
|
||||
DecisionTreeRegressor<> d(trainData, trainLabels);
|
||||
DecisionTreeRegressor<> wd(trainData, trainLabels, weights);
|
||||
// Initialize an all-ones weight matrix.
|
||||
arma::rowvec weights(trainResponses.n_cols, arma::fill::ones);
|
||||
|
||||
// Get the predicted test labels.
|
||||
arma::rowvec predictions;
|
||||
d.Predict(testData, predictions);
|
||||
// Build decision tree.
|
||||
DecisionTreeRegressor<> d(trainData, trainResponses);
|
||||
DecisionTreeRegressor<> wd(trainData, trainResponses, weights);
|
||||
|
||||
REQUIRE(predictions.n_elem == testData.n_cols);
|
||||
// Get the predicted test labels.
|
||||
arma::Row<ElemType> predictions;
|
||||
d.Predict(testData, predictions);
|
||||
|
||||
// Figure out the rmse.
|
||||
double rmse = RMSE(predictions, testLabels);
|
||||
REQUIRE(rmse < 6.0);
|
||||
REQUIRE(predictions.n_elem == testData.n_cols);
|
||||
|
||||
// Reset the prediction.
|
||||
predictions.zeros();
|
||||
wd.Predict(testData, predictions);
|
||||
// Figure out the rmse.
|
||||
ElemType rmse = RMSE(predictions, testResponses);
|
||||
|
||||
REQUIRE(predictions.n_elem == testData.n_cols);
|
||||
// Reset the prediction.
|
||||
predictions.zeros();
|
||||
wd.Predict(testData, predictions);
|
||||
|
||||
// Figure out the rmse.
|
||||
double wdrmse = RMSE(predictions, testLabels);
|
||||
REQUIRE(wdrmse < 6.0);
|
||||
REQUIRE(predictions.n_elem == testData.n_cols);
|
||||
|
||||
// Figure out the rmse.
|
||||
ElemType wdrmse = RMSE(predictions, testResponses);
|
||||
|
||||
if (rmse <= 6.2 && wdrmse <= 6.2)
|
||||
{
|
||||
success = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
REQUIRE(success == true);
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -1011,42 +988,53 @@ TEST_CASE("SimpleGeneralizationFMatTest_", "[DecisionTreeRegressorTest]")
|
||||
*/
|
||||
TEST_CASE("WeightedDecisionTreeTest_", "[DecisionTreeRegressorTest]")
|
||||
{
|
||||
// Loading data.
|
||||
data::DatasetInfo info;
|
||||
arma::mat trainData, testData;
|
||||
arma::rowvec trainResponses, testResponses;
|
||||
LoadBostonHousingDataset(trainData, testData, trainResponses, testResponses,
|
||||
info);
|
||||
// Allow three trials for success.
|
||||
bool success = false;
|
||||
for (size_t trial = 0; trial < 3; ++trial)
|
||||
{
|
||||
// Loading data.
|
||||
data::DatasetInfo info;
|
||||
arma::mat trainData, testData;
|
||||
arma::rowvec trainResponses, testResponses;
|
||||
LoadBostonHousingDataset(trainData, testData, trainResponses, testResponses,
|
||||
info);
|
||||
|
||||
// Add some noise.
|
||||
arma::mat noise(trainData.n_rows, 100, arma::fill::randu);
|
||||
arma::rowvec noiseResponses(100);
|
||||
for (size_t i = 0; i < noiseResponses.n_elem; ++i)
|
||||
noiseResponses[i] = 15 + Random(0, 10); // Random response.
|
||||
// Add some noise.
|
||||
arma::mat noise(trainData.n_rows, 100, arma::fill::randu);
|
||||
arma::rowvec noiseResponses(100);
|
||||
for (size_t i = 0; i < noiseResponses.n_elem; ++i)
|
||||
noiseResponses[i] = 15 + Random(0, 10); // Random response.
|
||||
|
||||
// Concatenate data matrices.
|
||||
arma::mat data = arma::join_rows(trainData, noise);
|
||||
arma::rowvec fullResponses = arma::join_rows(trainResponses, noiseResponses);
|
||||
// Concatenate data matrices.
|
||||
arma::mat data = arma::join_rows(trainData, noise);
|
||||
arma::rowvec fullResponses = arma::join_rows(trainResponses,
|
||||
noiseResponses);
|
||||
|
||||
// Now set weights.
|
||||
arma::rowvec weights(trainData.n_cols + 100);
|
||||
for (size_t i = 0; i < trainData.n_cols; ++i)
|
||||
weights[i] = Random(0.9, 1.0);
|
||||
for (size_t i = trainData.n_cols; i < trainData.n_cols + 100; ++i)
|
||||
weights[i] = Random(0.0, 0.01); // Low weights for false points.
|
||||
// Set weights.
|
||||
arma::rowvec weights(trainData.n_cols + 100);
|
||||
for (size_t i = 0; i < trainData.n_cols; ++i)
|
||||
weights[i] = Random(0.9, 1.0);
|
||||
for (size_t i = trainData.n_cols; i < trainData.n_cols + 100; ++i)
|
||||
weights[i] = Random(0.0, 0.01); // Low weights for false points.
|
||||
|
||||
// Now build the decision tree.
|
||||
DecisionTreeRegressor<> d(data, fullResponses, weights);
|
||||
// Now build the decision tree.
|
||||
DecisionTreeRegressor<> d(data, fullResponses, weights);
|
||||
|
||||
// Now we can check that we get good performance on the test set.
|
||||
arma::rowvec predictions;
|
||||
d.Predict(testData, predictions);
|
||||
// Now we can check that we get good performance on the test set.
|
||||
arma::rowvec predictions;
|
||||
d.Predict(testData, predictions);
|
||||
|
||||
REQUIRE(predictions.n_elem == testData.n_cols);
|
||||
REQUIRE(predictions.n_elem == testData.n_cols);
|
||||
|
||||
// Figure out the rmse.
|
||||
double rmse = RMSE(predictions, testResponses);
|
||||
REQUIRE(rmse < 6.0);
|
||||
// Figure out the rmse.
|
||||
double rmse = RMSE(predictions, testResponses);
|
||||
if (rmse < 6.2)
|
||||
{
|
||||
success = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
REQUIRE(success == true);
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -1056,40 +1044,52 @@ TEST_CASE("WeightedDecisionTreeTest_", "[DecisionTreeRegressorTest]")
|
||||
*/
|
||||
TEST_CASE("WeightedDecisionTreeMADGainTest", "[DecisionTreeRegressorTest]")
|
||||
{
|
||||
// Loading data.
|
||||
data::DatasetInfo info;
|
||||
arma::mat trainData, testData;
|
||||
arma::rowvec trainResponses, testResponses;
|
||||
LoadBostonHousingDataset(trainData, testData, trainResponses, testResponses,
|
||||
info);
|
||||
// Allow multiple trials, if needed.
|
||||
bool success = false;
|
||||
for (size_t trial = 0; trial < 5; ++trial)
|
||||
{
|
||||
// Loading data.
|
||||
data::DatasetInfo info;
|
||||
arma::mat trainData, testData;
|
||||
arma::rowvec trainResponses, testResponses;
|
||||
LoadBostonHousingDataset(trainData, testData, trainResponses, testResponses,
|
||||
info);
|
||||
|
||||
// Add some noise.
|
||||
arma::mat noise(trainData.n_rows, 100, arma::fill::randu);
|
||||
arma::rowvec noiseResponses(100);
|
||||
for (size_t i = 0; i < noiseResponses.n_elem; ++i)
|
||||
noiseResponses[i] = 15 + Random(0, 10); // Random response.
|
||||
// Add some noise.
|
||||
arma::mat noise(trainData.n_rows, 100, arma::fill::randu);
|
||||
arma::rowvec noiseResponses(100);
|
||||
for (size_t i = 0; i < noiseResponses.n_elem; ++i)
|
||||
noiseResponses[i] = 15 + Random(0, 10); // Random response.
|
||||
|
||||
// Concatenate data matrices.
|
||||
arma::mat data = arma::join_rows(trainData, noise);
|
||||
arma::rowvec fullResponses = arma::join_rows(trainResponses, noiseResponses);
|
||||
// Concatenate data matrices.
|
||||
arma::mat data = arma::join_rows(trainData, noise);
|
||||
arma::rowvec fullResponses = arma::join_rows(trainResponses,
|
||||
noiseResponses);
|
||||
|
||||
// Now set weights.
|
||||
arma::rowvec weights(trainData.n_cols + 100);
|
||||
for (size_t i = 0; i < trainData.n_cols; ++i)
|
||||
weights[i] = Random(0.9, 1.0);
|
||||
for (size_t i = trainData.n_cols; i < trainData.n_cols + 100; ++i)
|
||||
weights[i] = Random(0.0, 0.01); // Low weights for false points.
|
||||
// Now set weights.
|
||||
arma::rowvec weights(trainData.n_cols + 100);
|
||||
for (size_t i = 0; i < trainData.n_cols; ++i)
|
||||
weights[i] = Random(0.9, 1.0);
|
||||
for (size_t i = trainData.n_cols; i < trainData.n_cols + 100; ++i)
|
||||
weights[i] = Random(0.0, 0.01); // Low weights for false points.
|
||||
|
||||
// Now build the decision tree using MADGain.
|
||||
DecisionTreeRegressor<MADGain> d(data, fullResponses, weights);
|
||||
// Now build the decision tree using MADGain.
|
||||
DecisionTreeRegressor<MADGain> d(data, fullResponses, weights);
|
||||
|
||||
// Now we can check that we get good performance on the test set.
|
||||
arma::rowvec predictions;
|
||||
d.Predict(testData, predictions);
|
||||
// Now we can check that we get good performance on the test set.
|
||||
arma::rowvec predictions;
|
||||
d.Predict(testData, predictions);
|
||||
|
||||
REQUIRE(predictions.n_elem == testData.n_cols);
|
||||
REQUIRE(predictions.n_elem == testData.n_cols);
|
||||
|
||||
// Figure out the rmse.
|
||||
double rmse = RMSE(predictions, testResponses);
|
||||
REQUIRE(rmse < 6.5);
|
||||
// Figure out the rmse.
|
||||
double rmse = RMSE(predictions, testResponses);
|
||||
if (rmse < 6.6)
|
||||
{
|
||||
success = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
REQUIRE(success == true);
|
||||
}
|
||||
|
||||
@@ -1485,7 +1485,7 @@ TEST_CASE("HoeffdingTreeEmptyConstructorTrainTest", "[HoeffdingTreeTest]")
|
||||
HoeffdingTree<> ht;
|
||||
|
||||
// Just ensure that we can train without throwing an exception.
|
||||
REQUIRE_NOTHROW(ht.Train(data, labels));
|
||||
REQUIRE_NOTHROW(ht.Train(data, labels, 2));
|
||||
|
||||
// Now, create a categorical dataset and retrain.
|
||||
arma::mat data2 = arma::mat(4, 3000);
|
||||
@@ -1514,9 +1514,324 @@ TEST_CASE("HoeffdingTreeEmptyConstructorTrainTest", "[HoeffdingTreeTest]")
|
||||
}
|
||||
|
||||
// Ensure we can train without throwing an exception.
|
||||
REQUIRE_NOTHROW(ht.Train(data2, info, labels2));
|
||||
REQUIRE_NOTHROW(ht.Train(data2, info, labels2, 3));
|
||||
|
||||
// Train while specifying the number of classes.
|
||||
REQUIRE_NOTHROW(ht.Train(data, labels, false, true, 2));
|
||||
REQUIRE_NOTHROW(ht.Train(data2, info, labels2, false, 3));
|
||||
REQUIRE_NOTHROW(ht.Train(data, labels, 2, false, true));
|
||||
REQUIRE_NOTHROW(ht.Train(data2, info, labels2, 3, false));
|
||||
}
|
||||
|
||||
// Test all numeric Train() variants.
|
||||
TEST_CASE("HoeffdingTreeNumericTrainVariantTest", "[HoeffdingTreeTest]")
|
||||
{
|
||||
// Generate data.
|
||||
arma::mat data(5, 1000, arma::fill::randu);
|
||||
// Generate labels.
|
||||
arma::Row<size_t> labels(1000);
|
||||
for (size_t i = 0; i < 500; ++i)
|
||||
labels[i] = 0;
|
||||
for (size_t i = 500; i < 1000; ++i)
|
||||
labels[i] = 1;
|
||||
|
||||
// Create trees.
|
||||
HoeffdingTree<> htEmpty, ht1(5, 2), ht2(5, 2), ht3(5, 2), ht4(5, 2),
|
||||
ht5(5, 2), ht6(5, 2), ht7(5, 2);
|
||||
|
||||
htEmpty.Train(data, labels, 2);
|
||||
ht1.Train(data, labels);
|
||||
ht2.Train(data, labels, 2);
|
||||
ht3.Train(data, labels, 2, true);
|
||||
ht4.Train(data, labels, 2, false, 0.96);
|
||||
ht5.Train(data, labels, 2, false, 0.97, 150);
|
||||
ht6.Train(data, labels, 2, false, 0.98, 160, 110);
|
||||
ht7.Train(data, labels, 2, false, 0.99, 170, 120, 110);
|
||||
|
||||
REQUIRE(htEmpty.NumClasses() == 2);
|
||||
REQUIRE(htEmpty.NumSamples() == data.n_cols);
|
||||
|
||||
REQUIRE(ht1.NumClasses() == 2);
|
||||
REQUIRE(ht1.NumSamples() == data.n_cols);
|
||||
|
||||
REQUIRE(ht2.NumClasses() == 2);
|
||||
REQUIRE(ht2.NumSamples() == data.n_cols);
|
||||
|
||||
REQUIRE(ht3.NumClasses() == 2);
|
||||
REQUIRE(ht3.NumSamples() == data.n_cols);
|
||||
|
||||
REQUIRE(ht4.NumClasses() == 2);
|
||||
REQUIRE(ht4.NumSamples() > 0);
|
||||
REQUIRE(ht4.SuccessProbability() == 0.96);
|
||||
|
||||
REQUIRE(ht5.NumClasses() == 2);
|
||||
REQUIRE(ht5.NumSamples() > 0);
|
||||
REQUIRE(ht5.SuccessProbability() == 0.97);
|
||||
REQUIRE(ht5.MaxSamples() == 150);
|
||||
|
||||
REQUIRE(ht6.NumClasses() == 2);
|
||||
REQUIRE(ht6.NumSamples() > 0);
|
||||
REQUIRE(ht6.SuccessProbability() == 0.98);
|
||||
REQUIRE(ht6.MaxSamples() == 160);
|
||||
REQUIRE(ht6.CheckInterval() == 110);
|
||||
|
||||
REQUIRE(ht7.NumClasses() == 2);
|
||||
REQUIRE(ht7.NumSamples() > 0);
|
||||
REQUIRE(ht7.SuccessProbability() == 0.99);
|
||||
REQUIRE(ht7.MaxSamples() == 170);
|
||||
REQUIRE(ht7.CheckInterval() == 120);
|
||||
REQUIRE(ht7.MinSamples() == 110);
|
||||
}
|
||||
|
||||
// Test all categorical Train() variants.
|
||||
TEST_CASE("HoeffdingTreeCategoricalTrainVariantTest", "[HoeffdingTreeTest]")
|
||||
{
|
||||
// Generate data.
|
||||
arma::mat data(4, 9000);
|
||||
arma::Row<size_t> labels(9000);
|
||||
data::DatasetInfo info(4); // All features are numeric, except the fourth.
|
||||
info.MapString<double>("0", 3);
|
||||
for (size_t i = 0; i < 9000; i += 3)
|
||||
{
|
||||
data(0, i) = Random();
|
||||
data(1, i) = Random();
|
||||
data(2, i) = Random();
|
||||
data(3, i) = 0.0;
|
||||
labels[i] = 0;
|
||||
|
||||
data(0, i + 1) = Random();
|
||||
data(1, i + 1) = Random() - 1.0;
|
||||
data(2, i + 1) = Random() + 0.5;
|
||||
data(3, i + 1) = 0.0;
|
||||
labels[i + 1] = 2;
|
||||
|
||||
data(0, i + 2) = Random();
|
||||
data(1, i + 2) = Random() + 1.0;
|
||||
data(2, i + 2) = Random() + 0.8;
|
||||
data(3, i + 2) = 0.0;
|
||||
labels[i + 2] = 1;
|
||||
}
|
||||
|
||||
// Create trees.
|
||||
HoeffdingTree<> htEmpty, ht1(info, 3), ht2(info, 3), ht3(info, 3),
|
||||
ht4(info, 3), ht5(info, 3), ht6(info, 3), ht7(info, 3);
|
||||
|
||||
htEmpty.Train(data, info, labels, 3);
|
||||
ht1.Train(data, info, labels);
|
||||
ht2.Train(data, info, labels, 3);
|
||||
ht3.Train(data, info, labels, 3, true);
|
||||
ht4.Train(data, info, labels, 3, false, 0.96);
|
||||
ht5.Train(data, info, labels, 3, false, 0.97, 150);
|
||||
ht6.Train(data, info, labels, 3, false, 0.98, 160, 110);
|
||||
ht7.Train(data, info, labels, 3, false, 0.99, 170, 120, 110);
|
||||
|
||||
REQUIRE(htEmpty.NumClasses() == 3);
|
||||
REQUIRE(htEmpty.NumSamples() == data.n_cols);
|
||||
|
||||
REQUIRE(ht1.NumClasses() == 3);
|
||||
REQUIRE(ht1.NumSamples() == data.n_cols);
|
||||
|
||||
REQUIRE(ht2.NumClasses() == 3);
|
||||
REQUIRE(ht2.NumSamples() == data.n_cols);
|
||||
|
||||
REQUIRE(ht3.NumClasses() == 3);
|
||||
REQUIRE(ht3.NumSamples() == data.n_cols);
|
||||
|
||||
REQUIRE(ht4.NumClasses() == 3);
|
||||
REQUIRE(ht4.NumSamples() > 0);
|
||||
REQUIRE(ht4.SuccessProbability() == 0.96);
|
||||
|
||||
REQUIRE(ht5.NumClasses() == 3);
|
||||
REQUIRE(ht5.NumSamples() > 0);
|
||||
REQUIRE(ht5.SuccessProbability() == 0.97);
|
||||
REQUIRE(ht5.MaxSamples() == 150);
|
||||
|
||||
REQUIRE(ht6.NumClasses() == 3);
|
||||
REQUIRE(ht6.NumSamples() > 0);
|
||||
REQUIRE(ht6.SuccessProbability() == 0.98);
|
||||
REQUIRE(ht6.MaxSamples() == 160);
|
||||
REQUIRE(ht6.CheckInterval() == 110);
|
||||
|
||||
REQUIRE(ht7.NumClasses() == 3);
|
||||
REQUIRE(ht7.NumSamples() > 0);
|
||||
REQUIRE(ht7.SuccessProbability() == 0.99);
|
||||
REQUIRE(ht7.MaxSamples() == 170);
|
||||
REQUIRE(ht7.CheckInterval() == 120);
|
||||
REQUIRE(ht7.MinSamples() == 110);
|
||||
}
|
||||
|
||||
// Test overloads of Reset(). (Also test NumSamples().)
|
||||
TEST_CASE("HoeffdingTreeResetTests", "[HoeffdingTreeTest]")
|
||||
{
|
||||
// Generate data.
|
||||
arma::mat data(5, 1000, arma::fill::randu);
|
||||
// Generate labels.
|
||||
arma::Row<size_t> labels(1000);
|
||||
for (size_t i = 0; i < 500; ++i)
|
||||
labels[i] = 0;
|
||||
for (size_t i = 500; i < 1000; ++i)
|
||||
labels[i] = 1;
|
||||
|
||||
// Train the tree.
|
||||
HoeffdingTree<> ht(data, labels, 2);
|
||||
|
||||
REQUIRE(ht.NumSamples() > 0);
|
||||
REQUIRE(ht.NumClasses() == 2);
|
||||
|
||||
// Reset the tree, changing nothing.
|
||||
ht.Reset();
|
||||
REQUIRE(ht.NumSamples() == 0);
|
||||
REQUIRE(ht.NumChildren() == 0);
|
||||
|
||||
ht.Train(data, labels);
|
||||
REQUIRE(ht.NumSamples() > 0);
|
||||
REQUIRE(ht.NumClasses() == 2);
|
||||
|
||||
// Reset the tree, changing the dimensionality and number of classes.
|
||||
ht.Reset(10, 3);
|
||||
REQUIRE(ht.NumSamples() == 0);
|
||||
REQUIRE(ht.NumChildren() == 0);
|
||||
|
||||
data = arma::randu<arma::mat>(10, 1000);
|
||||
for(size_t i = 750; i < 1000; ++i)
|
||||
labels[i] = 2;
|
||||
|
||||
ht.Train(data, labels);
|
||||
REQUIRE(ht.NumSamples() > 0);
|
||||
REQUIRE(ht.NumClasses() == 3);
|
||||
|
||||
// Reset the tree to work on categorical data with a different number of
|
||||
// classes.
|
||||
data::DatasetInfo info(10);
|
||||
info.MapString<double>("0", 3);
|
||||
data.row(9).fill(0.0);
|
||||
for (size_t i = 0; i < 250; ++i)
|
||||
labels[i] = 3;
|
||||
|
||||
ht.Reset(info, 4);
|
||||
REQUIRE(ht.NumSamples() == 0);
|
||||
REQUIRE(ht.NumChildren() == 0);
|
||||
|
||||
ht.Train(data, info, labels);
|
||||
|
||||
REQUIRE(ht.NumSamples() > 0);
|
||||
REQUIRE(ht.NumClasses() == 4);
|
||||
}
|
||||
|
||||
// Test that we can learn on floating-point data.
|
||||
TEST_CASE("HoeffdingTreeNumericFloatDataTest", "[HoeffdingTreeTest]")
|
||||
{
|
||||
// We need to create a dataset with some amount of complexity, that must be
|
||||
// split in a handful of ways to accurately classify the data. An expanding
|
||||
// spiral should do the trick here. We'll make the spiral in two dimensions.
|
||||
// The label will change as the index increases.
|
||||
arma::fmat spiralDataset(2, 10000);
|
||||
for (size_t i = 0; i < 10000; ++i)
|
||||
{
|
||||
// One circle every 20000 samples. Plus some noise.
|
||||
const float magnitude = 2.0 + (float(i) / 20000.0) + 0.5 * Random();
|
||||
const float angle = (i % 20000) * (2 * M_PI) + Random();
|
||||
|
||||
const float x = magnitude * cos(angle);
|
||||
const float y = magnitude * sin(angle);
|
||||
|
||||
spiralDataset(0, i) = x;
|
||||
spiralDataset(1, i) = y;
|
||||
}
|
||||
|
||||
arma::Row<size_t> labels(10000);
|
||||
for (size_t i = 0; i < 2000; ++i)
|
||||
labels[i] = 1;
|
||||
for (size_t i = 2000; i < 4000; ++i)
|
||||
labels[i] = 3;
|
||||
for (size_t i = 4000; i < 6000; ++i)
|
||||
labels[i] = 2;
|
||||
for (size_t i = 6000; i < 8000; ++i)
|
||||
labels[i] = 0;
|
||||
for (size_t i = 8000; i < 10000; ++i)
|
||||
labels[i] = 4;
|
||||
|
||||
// Now shuffle the dataset.
|
||||
arma::uvec indices = arma::shuffle(arma::linspace<arma::uvec>(0, 9999,
|
||||
10000));
|
||||
arma::fmat d(2, 10000);
|
||||
arma::Row<size_t> l(10000);
|
||||
for (size_t i = 0; i < 10000; ++i)
|
||||
{
|
||||
d.col(i) = spiralDataset.col(indices[i]);
|
||||
l[i] = labels[indices[i]];
|
||||
}
|
||||
|
||||
// Split into a training set and a test set.
|
||||
arma::fmat trainingData = d.cols(0, 4999);
|
||||
arma::fmat testData = d.cols(5000, 9999);
|
||||
arma::Row<size_t> trainingLabels = l.subvec(0, 4999);
|
||||
arma::Row<size_t> testLabels = l.subvec(5000, 9999);
|
||||
|
||||
data::DatasetInfo info(2);
|
||||
|
||||
// Now build two decision trees; one in batch mode, and one in streaming mode.
|
||||
// We need to set the confidence pretty high so that the streaming tree isn't
|
||||
// able to have enough samples to build to the same leaves.
|
||||
HoeffdingTree<GiniImpurity, HoeffdingFloatNumericSplit>
|
||||
batchTree(trainingData, info, trainingLabels, 5, true, 0.99999999);
|
||||
HoeffdingTree<GiniImpurity, HoeffdingFloatNumericSplit>
|
||||
streamTree(trainingData, info, trainingLabels, 5, false, 0.99999999);
|
||||
|
||||
// Ensure that the performance of the batch tree is better.
|
||||
size_t batchCorrect = 0;
|
||||
size_t streamCorrect = 0;
|
||||
for (size_t i = 0; i < 5000; ++i)
|
||||
{
|
||||
size_t streamLabel = streamTree.Classify(testData.col(i));
|
||||
size_t batchLabel = batchTree.Classify(testData.col(i));
|
||||
|
||||
if (streamLabel == testLabels[i])
|
||||
++streamCorrect;
|
||||
if (batchLabel == testLabels[i])
|
||||
++batchCorrect;
|
||||
}
|
||||
|
||||
// The batch tree must be a bit better than the stream tree. But not too
|
||||
// much, since the accuracy is already going to be very high.
|
||||
REQUIRE(batchCorrect >= streamCorrect);
|
||||
}
|
||||
|
||||
// Test that we can learn on categorical floating-point data.
|
||||
TEST_CASE("HoeffdingTreeCategoricalFloatDataTest", "[HoeffdingTreeTest]")
|
||||
{
|
||||
// Generate data.
|
||||
arma::fmat dataset(4, 3000, arma::fill::randu);
|
||||
arma::Row<size_t> labels(3000);
|
||||
for (size_t i = 0; i < 3000; i += 3)
|
||||
{
|
||||
labels[i] = 0;
|
||||
|
||||
labels[i + 1] = 2;
|
||||
dataset.col(i + 1) += 2.0;
|
||||
|
||||
labels[i + 2] = 1;
|
||||
dataset.col(i + 2) -= 2.0;
|
||||
}
|
||||
|
||||
data::DatasetInfo info(4); // All features are numeric, except the fourth.
|
||||
info.MapString<double>("0", 3);
|
||||
dataset.row(3).fill(0.0);
|
||||
|
||||
// Now build two decision trees; one in batch mode, and one in streaming mode.
|
||||
HoeffdingTree<GiniImpurity, HoeffdingFloatNumericSplit>
|
||||
batchTree(dataset, info, labels, 3, true);
|
||||
HoeffdingTree<GiniImpurity, HoeffdingFloatNumericSplit>
|
||||
streamTree(dataset, info, labels, 3, false);
|
||||
|
||||
// Make sure that we trained successfully.
|
||||
arma::Row<size_t> batchPredictions, streamPredictions;
|
||||
batchTree.Classify(dataset, batchPredictions);
|
||||
streamTree.Classify(dataset, streamPredictions);
|
||||
|
||||
REQUIRE(batchPredictions.n_elem == labels.n_elem);
|
||||
REQUIRE(streamPredictions.n_elem == labels.n_elem);
|
||||
|
||||
// Make sure that the accuracy is at least reasonable. (This is a very loose
|
||||
// test.)
|
||||
REQUIRE(arma::accu(batchPredictions == labels) > (labels.n_elem / 2));
|
||||
REQUIRE(arma::accu(streamPredictions == labels) > (labels.n_elem / 2));
|
||||
}
|
||||
|
||||
@@ -29,7 +29,7 @@ TEST_CASE("CVFunctionTest", "[HPTTest]")
|
||||
arma::vec beta = arma::randn(5, 1);
|
||||
arma::rowvec ys = beta.t() * xs + 0.1 * arma::randn(1, 100);
|
||||
|
||||
SimpleCV<LARS, MSE> cv(0.2, xs, ys);
|
||||
SimpleCV<LARS<>, MSE> cv(0.2, xs, ys);
|
||||
|
||||
bool transposeData = true;
|
||||
bool useCholesky = false;
|
||||
@@ -41,7 +41,7 @@ TEST_CASE("CVFunctionTest", "[HPTTest]")
|
||||
|
||||
FixedArg<bool, 1> fixedUseCholesky{useCholesky};
|
||||
FixedArg<double, 3> fixedLambda1{lambda2};
|
||||
CVFunction<decltype(cv), LARS, 4, FixedArg<bool, 1>, FixedArg<double, 3>>
|
||||
CVFunction<decltype(cv), LARS<>, 4, FixedArg<bool, 1>, FixedArg<double, 3>>
|
||||
cvFun(cv, datasetInfo, 0.0, 0.0, fixedUseCholesky, fixedLambda1);
|
||||
|
||||
double expected = cv.Evaluate(transposeData, useCholesky, lambda1, lambda2);
|
||||
@@ -64,7 +64,7 @@ TEST_CASE("CVFunctionCategoricalTest", "[HPTTest]")
|
||||
arma::vec beta = arma::randn(5, 1);
|
||||
arma::rowvec ys = beta.t() * xs + 0.1 * arma::randn(1, 100);
|
||||
|
||||
SimpleCV<LARS, MSE> cv(0.2, xs, ys);
|
||||
SimpleCV<LARS<>, MSE> cv(0.2, xs, ys);
|
||||
|
||||
bool transposeData = true;
|
||||
bool useCholesky = false;
|
||||
@@ -78,7 +78,7 @@ TEST_CASE("CVFunctionCategoricalTest", "[HPTTest]")
|
||||
|
||||
FixedArg<bool, 1> fixedUseCholesky{useCholesky};
|
||||
FixedArg<double, 3> fixedLambda1{lambda2};
|
||||
CVFunction<decltype(cv), LARS, 4, FixedArg<bool, 1>, FixedArg<double, 3>>
|
||||
CVFunction<decltype(cv), LARS<>, 4, FixedArg<bool, 1>, FixedArg<double, 3>>
|
||||
cvFun(cv, datasetInfo, 0.0, 0.0, fixedUseCholesky, fixedLambda1);
|
||||
|
||||
double expected = cv.Evaluate(transposeData, useCholesky, lambda1, lambda2);
|
||||
@@ -137,7 +137,7 @@ TEST_CASE("CVFunctionGradientTest", "[HPTTest]")
|
||||
double b = -1.5;
|
||||
double c = 2.5;
|
||||
double d = 3.0;
|
||||
QuadraticFunction<LARS> lf(a, b, c, d);
|
||||
QuadraticFunction<LARS<>> lf(a, b, c, d);
|
||||
|
||||
// All values are numeric.
|
||||
IncrementPolicy policy(true);
|
||||
@@ -145,7 +145,7 @@ TEST_CASE("CVFunctionGradientTest", "[HPTTest]")
|
||||
|
||||
double relativeDelta = 0.01;
|
||||
double minDelta = 0.001;
|
||||
CVFunction<decltype(lf), LARS, 3> cvFun(lf, datasetInfo, relativeDelta,
|
||||
CVFunction<decltype(lf), LARS<>, 3> cvFun(lf, datasetInfo, relativeDelta,
|
||||
minDelta);
|
||||
|
||||
double x = 0.0;
|
||||
@@ -202,7 +202,7 @@ void FindLARSBestLambdas(arma::mat& xs,
|
||||
double& bestLambda2,
|
||||
double& bestObjective)
|
||||
{
|
||||
SimpleCV<LARS, MSE> cv(validationSize, xs, ys);
|
||||
SimpleCV<LARS<>, MSE> cv(validationSize, xs, ys);
|
||||
|
||||
bestObjective = std::numeric_limits<double>::max();
|
||||
|
||||
@@ -250,8 +250,8 @@ TEST_CASE("GridSearchTest", "[HPTTest]")
|
||||
for (double lambda2 : lambda2Set)
|
||||
datasetInfo.MapString<size_t>(lambda2, 1);
|
||||
|
||||
SimpleCV<LARS, MSE> cv(validationSize, xs, ys);
|
||||
CVFunction<decltype(cv), LARS, 4, FixedArg<bool, 0>, FixedArg<bool, 1>>
|
||||
SimpleCV<LARS<>, MSE> cv(validationSize, xs, ys);
|
||||
CVFunction<decltype(cv), LARS<>, 4, FixedArg<bool, 0>, FixedArg<bool, 1>>
|
||||
cvFun(cv, datasetInfo, 0.0, 0.0, {transposeData}, {useCholesky});
|
||||
|
||||
ens::GridSearch optimizer;
|
||||
@@ -297,7 +297,7 @@ TEST_CASE("HPTTest", "[HPTTest]")
|
||||
expectedObjective);
|
||||
|
||||
double actualLambda1, actualLambda2;
|
||||
HyperParameterTuner<LARS, MSE, SimpleCV, GridSearch>
|
||||
HyperParameterTuner<LARS<>, MSE, SimpleCV, GridSearch>
|
||||
hpt(validationSize, xs, ys);
|
||||
std::tie(actualLambda1, actualLambda2) = hpt.Optimize(Fixed(transposeData),
|
||||
Fixed(useCholesky), lambda1Set, lambda2Set);
|
||||
@@ -368,7 +368,7 @@ TEST_CASE("HPTGradientDescentTest", "[HPTTest]")
|
||||
// We pass LARS just because some ML algorithm should be passed. We pass MSE
|
||||
// to tell HyperParameterTuner that the objective function (QuadraticFunction)
|
||||
// should be minimized.
|
||||
HyperParameterTuner<LARS, MSE, QuadraticFunction,
|
||||
HyperParameterTuner<LARS<>, MSE, QuadraticFunction,
|
||||
GradientDescent> hpt(a, b, c, d, xMin, yMin, zMin);
|
||||
|
||||
// Setting GradientDescent to find more close solution to the optimal one.
|
||||
|
||||
+579
-135
File diff suppressed because it is too large
Load Diff
@@ -21,25 +21,30 @@ using namespace mlpack;
|
||||
* Creates two 10x3 random matrices and one 10x1 "results" matrix.
|
||||
* Finds B in y=BX with one matrix, then predicts against the other.
|
||||
*/
|
||||
TEST_CASE("LinearRegressionTestCase", "[LinearRegressionTest]")
|
||||
TEMPLATE_TEST_CASE("LinearRegressionTestCase", "[LinearRegressionTest]",
|
||||
arma::fmat, arma::mat)
|
||||
{
|
||||
typedef TestType MatType;
|
||||
typedef arma::Row<typename MatType::elem_type> RowType;
|
||||
typedef arma::Col<typename MatType::elem_type> ColType;
|
||||
|
||||
// Predictors and points are 10x3 matrices.
|
||||
arma::mat predictors(3, 10);
|
||||
arma::mat points(3, 10);
|
||||
MatType predictors(3, 10);
|
||||
MatType points(3, 10);
|
||||
|
||||
// Responses is the "correct" value for each point in predictors and points.
|
||||
arma::rowvec responses(10);
|
||||
RowType responses(10);
|
||||
|
||||
// The values we get back when we predict for points.
|
||||
arma::rowvec predictions(10);
|
||||
RowType predictions(10);
|
||||
|
||||
// We'll randomly select some coefficients for the linear response.
|
||||
arma::vec coeffs;
|
||||
ColType coeffs;
|
||||
coeffs.randu(4);
|
||||
|
||||
// Now generate each point.
|
||||
for (size_t row = 0; row < 3; row++)
|
||||
predictors.row(row) = arma::linspace<arma::rowvec>(0, 9, 10);
|
||||
predictors.row(row) = arma::linspace<RowType>(0, 9, 10);
|
||||
|
||||
points = predictors;
|
||||
|
||||
@@ -57,7 +62,7 @@ TEST_CASE("LinearRegressionTestCase", "[LinearRegressionTest]")
|
||||
dot(coeffs.rows(1, 3), arma::ones<arma::rowvec>(3) * elem);
|
||||
|
||||
// Initialize and predict.
|
||||
LinearRegression lr(predictors, responses);
|
||||
LinearRegression<MatType> lr(predictors, responses);
|
||||
lr.Predict(points, predictions);
|
||||
|
||||
// Output result and verify we have less than 5% error from "correct" value
|
||||
@@ -69,16 +74,20 @@ TEST_CASE("LinearRegressionTestCase", "[LinearRegressionTest]")
|
||||
/**
|
||||
* Check the functionality of ComputeError().
|
||||
*/
|
||||
TEST_CASE("ComputeErrorTest", "[LinearRegressionTest]")
|
||||
TEMPLATE_TEST_CASE("ComputeErrorTest", "[LinearRegressionTest]", arma::fmat,
|
||||
arma::mat)
|
||||
{
|
||||
arma::mat predictors;
|
||||
typedef TestType MatType;
|
||||
typedef arma::Row<typename MatType::elem_type> RowType;
|
||||
|
||||
MatType predictors;
|
||||
predictors = { { 0, 1, 2, 4, 8, 16 },
|
||||
{ 16, 8, 4, 2, 1, 0 } };
|
||||
arma::rowvec responses = "0 2 4 3 8 8";
|
||||
RowType responses = "0 2 4 3 8 8";
|
||||
|
||||
// http://www.mlpack.org/trac/ticket/298
|
||||
// This dataset gives a cost of 1.189500337 (as calculated in Octave).
|
||||
LinearRegression lr(predictors, responses);
|
||||
LinearRegression<MatType> lr(predictors, responses);
|
||||
|
||||
REQUIRE(lr.ComputeError(predictors, responses) ==
|
||||
Approx(1.189500337).epsilon(1e-5));
|
||||
@@ -95,7 +104,7 @@ TEST_CASE("ComputeErrorPerfectFitTest", "[LinearRegressionTest]")
|
||||
{ 0, 1, 2, 2, 2, 6 } };
|
||||
arma::rowvec responses = "0 2 4 3 8 8";
|
||||
|
||||
LinearRegression lr(predictors, responses);
|
||||
LinearRegression<> lr(predictors, responses);
|
||||
|
||||
REQUIRE(lr.ComputeError(predictors, responses) == Approx(0.0).margin(1e-25));
|
||||
}
|
||||
@@ -116,7 +125,7 @@ TEST_CASE("RidgeRegressionTest", "[LinearRegressionTest]")
|
||||
// invertible. If ridge regression is not working correctly, then the matrix
|
||||
// will not be invertible and the test should segfault (or something else
|
||||
// ugly).
|
||||
LinearRegression lr(data, responses, 0.0001);
|
||||
LinearRegression<> lr(data, responses, 0.0001);
|
||||
|
||||
// Now just make sure that it predicts some more zeros.
|
||||
arma::rowvec predictedResponses;
|
||||
@@ -167,7 +176,7 @@ TEST_CASE("RidgeRegressionTestCase", "[LinearRegressionTest]")
|
||||
dot(coeffs.rows(1, 3), arma::ones<arma::rowvec>(3) * elem);
|
||||
|
||||
// Initialize and predict with very small lambda.
|
||||
LinearRegression lr(predictors, responses, 0.001);
|
||||
LinearRegression<> lr(predictors, responses, 0.001);
|
||||
lr.Predict(points, predictions);
|
||||
|
||||
// Output result and verify we have less than 5% error from "correct" value
|
||||
@@ -186,8 +195,8 @@ TEST_CASE("LinearRegressionTrainTest", "[LinearRegressionTest]")
|
||||
arma::mat dataset = arma::randu<arma::mat>(5, 1000);
|
||||
arma::rowvec responses = arma::randu<arma::rowvec>(1000);
|
||||
|
||||
LinearRegression lr(dataset, responses, 0.3);
|
||||
LinearRegression lrTrain;
|
||||
LinearRegression<> lr(dataset, responses, 0.3);
|
||||
LinearRegression<> lrTrain;
|
||||
lrTrain.Lambda() = 0.3;
|
||||
|
||||
lrTrain.Train(dataset, responses);
|
||||
@@ -209,8 +218,8 @@ TEST_CASE("LinearRegressionTest", "[LinearRegressionTest]")
|
||||
arma::rowvec responses;
|
||||
responses.randn(800);
|
||||
|
||||
LinearRegression lr(data, responses, 0.05); // Train the model.
|
||||
LinearRegression xmlLr, jsonLr, binaryLr;
|
||||
LinearRegression<> lr(data, responses, 0.05); // Train the model.
|
||||
LinearRegression<> xmlLr, jsonLr, binaryLr;
|
||||
|
||||
SerializeObjectAll(lr, xmlLr, jsonLr, binaryLr);
|
||||
|
||||
@@ -260,8 +269,157 @@ TEST_CASE("LinearRegressionTrainReturnObjective", "[LinearRegressionTest]")
|
||||
dot(coeffs.rows(1, 3), arma::ones<arma::rowvec>(3) * elem);
|
||||
|
||||
// Initialize and predict.
|
||||
LinearRegression lr;
|
||||
LinearRegression<> lr;
|
||||
double error = lr.Train(predictors, responses);
|
||||
|
||||
REQUIRE(std::isfinite(error) == true);
|
||||
}
|
||||
|
||||
/**
|
||||
* Make sure all versions of Train() work correctly.
|
||||
*/
|
||||
TEMPLATE_TEST_CASE("LinearRegressionAllTrainVersionsTest",
|
||||
"[LinearRegressionTest]", arma::fmat, arma::mat)
|
||||
{
|
||||
typedef TestType MatType;
|
||||
typedef arma::Row<typename MatType::elem_type> RowType;
|
||||
|
||||
// The data doesn't really matter for this test; mostly we want to make sure
|
||||
// that all the Train() variants work properly.
|
||||
MatType predictors;
|
||||
predictors = { { 0, 1, 2, 4, 8, 16 },
|
||||
{ 16, 8, 4, 2, 1, 0 } };
|
||||
RowType responses = "0 2 4 3 8 8";
|
||||
RowType weights = "1.0 1.1 1.2 0.8 0.9 1.0";
|
||||
|
||||
LinearRegression<MatType> lr1, lr2, lr3, lr4, lr5, lr6;
|
||||
|
||||
(void) lr1.Train(predictors, responses);
|
||||
(void) lr2.Train(predictors, responses, 0.1);
|
||||
(void) lr3.Train(predictors, responses, 0.2, false);
|
||||
(void) lr4.Train(predictors, responses, weights);
|
||||
(void) lr5.Train(predictors, responses, weights, 0.3);
|
||||
(void) lr6.Train(predictors, responses, weights, 0.4, false);
|
||||
|
||||
// We don't care about the specifics of the trained model, but we want to just
|
||||
// make sure everything appears to be correct from the sizes and
|
||||
// hyperparameters.
|
||||
REQUIRE(lr1.Lambda() == Approx(0.0).margin(1e-10));
|
||||
REQUIRE(lr1.Intercept() == true);
|
||||
REQUIRE(lr1.Parameters().n_elem == 3);
|
||||
|
||||
REQUIRE(lr2.Lambda() == Approx(0.1).margin(1e-10));
|
||||
REQUIRE(lr2.Intercept() == true);
|
||||
REQUIRE(lr2.Parameters().n_elem == 3);
|
||||
|
||||
REQUIRE(lr3.Lambda() == Approx(0.2).margin(1e-10));
|
||||
REQUIRE(lr3.Intercept() == false);
|
||||
REQUIRE(lr3.Parameters().n_elem == 2);
|
||||
|
||||
REQUIRE(lr4.Lambda() == Approx(0.0).margin(1e-10));
|
||||
REQUIRE(lr4.Intercept() == true);
|
||||
REQUIRE(lr4.Parameters().n_elem == 3);
|
||||
|
||||
REQUIRE(lr5.Lambda() == Approx(0.3).margin(1e-10));
|
||||
REQUIRE(lr5.Intercept() == true);
|
||||
REQUIRE(lr5.Parameters().n_elem == 3);
|
||||
|
||||
REQUIRE(lr6.Lambda() == Approx(0.4).margin(1e-10));
|
||||
REQUIRE(lr6.Intercept() == false);
|
||||
REQUIRE(lr6.Parameters().n_elem == 2);
|
||||
|
||||
// We can also check that the weighted model is different from the unweighted
|
||||
// model.
|
||||
REQUIRE(!arma::approx_equal(lr1.Parameters(), lr4.Parameters(), "absdiff",
|
||||
1e-5));
|
||||
}
|
||||
|
||||
/**
|
||||
* Ensure that single-point Predict() returns the same results as multi-point
|
||||
* Predict().
|
||||
*/
|
||||
TEMPLATE_TEST_CASE("LinearRegressionSinglePointPredictTest",
|
||||
"[LinearRegressionTest]", arma::fmat, arma::mat)
|
||||
{
|
||||
typedef TestType MatType;
|
||||
typedef arma::Row<typename MatType::elem_type> RowType;
|
||||
|
||||
MatType predictors;
|
||||
predictors = { { 0, 1, 2, 4, 8, 16 },
|
||||
{ 16, 8, 4, 2, 1, 0 } };
|
||||
RowType responses = "0 2 4 3 8 8";
|
||||
|
||||
LinearRegression<MatType> lr(predictors, responses, 0.1, true);
|
||||
|
||||
// Compute predictions for test points in batch.
|
||||
RowType predictions;
|
||||
lr.Predict(predictors, predictions);
|
||||
|
||||
// Now compute each prediction individually.
|
||||
for (size_t i = 0; i < predictors.n_cols; ++i)
|
||||
{
|
||||
const double prediction = lr.Predict(predictors.col(i));
|
||||
REQUIRE(prediction == Approx(predictions[i]));
|
||||
}
|
||||
}
|
||||
|
||||
// Make sure training on submatrices and subvectors works.
|
||||
TEST_CASE("LinearRegressionSubmatrixTrainingTest", "[LinearRegressionTest]")
|
||||
{
|
||||
// The quality of the model doesn't matter---mostly this is a compilation
|
||||
// test.
|
||||
arma::mat predictors(100, 1000, arma::fill::randu);
|
||||
arma::rowvec responses(1000, arma::fill::randu);
|
||||
arma::rowvec weights(1000, arma::fill::randu);
|
||||
|
||||
LinearRegression<> lr1(predictors.cols(0, 499), responses.subvec(0, 499));
|
||||
LinearRegression<> lr2;
|
||||
lr2.Train(predictors.cols(0, 499), responses.subvec(0, 499));
|
||||
LinearRegression<> lr3(predictors.cols(0, 499), responses.subvec(0, 499),
|
||||
weights.subvec(0, 499));
|
||||
LinearRegression<> lr4;
|
||||
lr4.Train(predictors.cols(0, 499), responses.subvec(0, 499),
|
||||
weights.subvec(0, 499));
|
||||
|
||||
REQUIRE(lr1.Parameters().n_elem == 101);
|
||||
REQUIRE(lr2.Parameters().n_elem == 101);
|
||||
REQUIRE(lr3.Parameters().n_elem == 101);
|
||||
REQUIRE(lr4.Parameters().n_elem == 101);
|
||||
|
||||
arma::rowvec predictions;
|
||||
|
||||
lr1.Predict(predictors.cols(500, 999), predictions);
|
||||
REQUIRE(predictions.n_cols == 500);
|
||||
|
||||
lr2.Predict(predictors.cols(500, 999), predictions);
|
||||
REQUIRE(predictions.n_cols == 500);
|
||||
|
||||
lr3.Predict(predictors.cols(500, 999), predictions);
|
||||
REQUIRE(predictions.n_cols == 500);
|
||||
|
||||
lr4.Predict(predictors.cols(500, 999), predictions);
|
||||
REQUIRE(predictions.n_cols == 500);
|
||||
}
|
||||
|
||||
// Make sure we can train on sparse data.
|
||||
TEST_CASE("LinearRegressionSparseTrainingTest", "[LinearRegressionTest]")
|
||||
{
|
||||
// For this test the quality of the model doesn't matter---mostly this is a
|
||||
// compilation test, but we check that the sizes of the returned predictions
|
||||
// and the sizes of the learned models are correct.
|
||||
|
||||
// Generate sparse random data.
|
||||
arma::sp_mat data;
|
||||
data.sprandu(100, 5000, 0.3);
|
||||
|
||||
arma::rowvec responses(5000, arma::fill::randu);
|
||||
|
||||
LinearRegression<> lr(data, responses);
|
||||
|
||||
REQUIRE(lr.Parameters().n_elem == 101);
|
||||
|
||||
arma::rowvec predictions;
|
||||
lr.Predict(data, predictions);
|
||||
|
||||
REQUIRE(predictions.n_elem == 5000);
|
||||
}
|
||||
|
||||
@@ -486,7 +486,7 @@ TEST_CASE("LinearSVMLBFGSSimpleTest", "[LinearSVMTest]")
|
||||
|
||||
// Compare training accuracy to 1.
|
||||
const double acc = lsvm.ComputeAccuracy(dataset, labels);
|
||||
REQUIRE(acc == Approx(1.0).epsilon(0.005));
|
||||
REQUIRE(acc == Approx(100.0).epsilon(0.005));
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -514,12 +514,12 @@ TEST_CASE("LinearSVMGradientDescentSimpleTest", "[LinearSVMTest]")
|
||||
|
||||
// Create a linear svm object using custom gradient descent optimizer.
|
||||
ens::GradientDescent optimizer(stepSize, maxIterations, tolerance);
|
||||
LinearSVM<arma::mat> lsvm(dataset, labels, numClasses, lambda,
|
||||
delta, false, optimizer);
|
||||
LinearSVM<arma::mat> lsvm(dataset, labels, numClasses, optimizer, lambda,
|
||||
delta, false);
|
||||
|
||||
// Compare training accuracy to 1.
|
||||
const double acc = lsvm.ComputeAccuracy(dataset, labels);
|
||||
REQUIRE(acc == Approx(1.0).epsilon(0.005));
|
||||
REQUIRE(acc == Approx(100.0).epsilon(0.005));
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -632,8 +632,7 @@ TEST_CASE("LinearSVMFitIntercept", "[LinearSVMTest]")
|
||||
}
|
||||
|
||||
// Now train a svm object on it.
|
||||
LinearSVM<arma::mat> svm(data, labels, numClasses, lambda,
|
||||
delta, true, ens::L_BFGS());
|
||||
LinearSVM<arma::mat> svm(data, labels, numClasses, lambda, delta, true);
|
||||
|
||||
// Ensure that the error is close to zero.
|
||||
const double acc = svm.ComputeAccuracy(data, labels);
|
||||
@@ -775,12 +774,12 @@ TEST_CASE("LinearSVMPSGDSimpleTest", "[LinearSVMTest]")
|
||||
ens::ParallelSGD<ens::ConstantStep> optimizer(0,
|
||||
std::ceil((float) dataset.n_cols / omp_get_max_threads()),
|
||||
1e-5, true, decayPolicy);
|
||||
LinearSVM<arma::mat> lsvm(dataset, labels, numClasses, lambda,
|
||||
delta, false, optimizer);
|
||||
LinearSVM<arma::mat> lsvm(dataset, labels, numClasses, optimizer, lambda,
|
||||
delta, false);
|
||||
|
||||
// Compare training accuracy to 1.
|
||||
const double acc = lsvm.ComputeAccuracy(dataset, labels);
|
||||
REQUIRE(acc == Approx(1.0).epsilon(1e-2));
|
||||
REQUIRE(acc == Approx(100.0).epsilon(1e-2));
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -824,13 +823,13 @@ TEST_CASE("LinearSVMParallelSGDTwoClasses", "[LinearSVMTest]")
|
||||
|
||||
// Train linear svm object using Parallel SGD optimizer.
|
||||
// The threadShareSize is chosen such that each function gets optimized.
|
||||
ens::ParallelSGD<ens::ConstantStep> optimizer(0,
|
||||
ens::ParallelSGD<ens::ConstantStep> optimizer(100000,
|
||||
std::ceil((float) data.n_cols / omp_get_max_threads()),
|
||||
1e-5, true, decayPolicy);
|
||||
LinearSVM<arma::mat> lsvm(data, labels, numClasses, lambda,
|
||||
delta, false, optimizer);
|
||||
LinearSVM<arma::mat> lsvm(data, labels, numClasses, optimizer, lambda,
|
||||
delta, false);
|
||||
|
||||
// Compare training accuracy to 1.
|
||||
// Compare training accuracy to 100.
|
||||
const double acc = lsvm.ComputeAccuracy(data, labels);
|
||||
|
||||
// Create test dataset.
|
||||
@@ -849,8 +848,8 @@ TEST_CASE("LinearSVMParallelSGDTwoClasses", "[LinearSVMTest]")
|
||||
const double testAcc = lsvm.ComputeAccuracy(data, labels);
|
||||
|
||||
// Larger tolerance is sometimes needed.
|
||||
if (testAcc == Approx(1.0).epsilon(0.02) &&
|
||||
acc == Approx(1.0).epsilon(0.02))
|
||||
if (testAcc == Approx(100.0).epsilon(0.02) &&
|
||||
acc == Approx(100.0).epsilon(0.02))
|
||||
{
|
||||
success = true;
|
||||
break;
|
||||
@@ -863,23 +862,28 @@ TEST_CASE("LinearSVMParallelSGDTwoClasses", "[LinearSVMTest]")
|
||||
#endif
|
||||
|
||||
/**
|
||||
* Test sparse and dense linear svm and make sure they both work the
|
||||
* Test sparse and dense linear svm training and make sure they both work the
|
||||
* same using the L-BFGS optimizer.
|
||||
*/
|
||||
TEST_CASE("LinearSVMSparseLBFGSTest", "[LinearSVMTest]")
|
||||
TEMPLATE_TEST_CASE("LinearSVMSparseLBFGSTest", "[LinearSVMTest]", float, double)
|
||||
{
|
||||
typedef TestType ElemType;
|
||||
typedef typename arma::SpMat<ElemType> SparseMatType;
|
||||
typedef typename arma::Mat<ElemType> MatType;
|
||||
|
||||
// Create a random dataset.
|
||||
arma::sp_mat dataset;
|
||||
SparseMatType dataset;
|
||||
dataset.sprandu(10, 800, 0.3);
|
||||
arma::mat denseDataset(dataset);
|
||||
MatType denseDataset(dataset);
|
||||
arma::Row<size_t> labels(800);
|
||||
for (size_t i = 0; i < 800; ++i)
|
||||
labels[i] = RandInt(0, 2);
|
||||
|
||||
LinearSVM<arma::mat> lr(denseDataset, labels, 2, 0.3, 1,
|
||||
false, ens::L_BFGS());
|
||||
LinearSVM<arma::sp_mat> lrSparse(dataset, labels, 2, 0.3, 1,
|
||||
false, ens::L_BFGS());
|
||||
LinearSVM<> lr(denseDataset, labels, 2, 0.3, 1, false);
|
||||
LinearSVM<> lrSparse(dataset, labels, 2, 0.3, 1, false);
|
||||
|
||||
// Make the initial points the same.
|
||||
lrSparse.Parameters() = lr.Parameters();
|
||||
|
||||
REQUIRE(lr.Parameters().n_elem == lrSparse.Parameters().n_elem);
|
||||
for (size_t i = 0; i < lr.Parameters().n_elem; ++i)
|
||||
@@ -891,10 +895,15 @@ TEST_CASE("LinearSVMSparseLBFGSTest", "[LinearSVMTest]")
|
||||
|
||||
/**
|
||||
* Test training of linear svm for multiple classes on a complex gaussian
|
||||
* dataset using L-BFGS optimizer.
|
||||
* dataset using L-BFGS optimizer, with different types.
|
||||
*/
|
||||
TEST_CASE("LinearSVMLBFGSMultipleClasses", "[LinearSVMTest]")
|
||||
TEMPLATE_TEST_CASE("LinearSVMLBFGSMultipleClasses", "[LinearSVMTest]", float,
|
||||
double)
|
||||
{
|
||||
typedef TestType ElemType;
|
||||
typedef typename arma::Mat<ElemType> MatType;
|
||||
typedef typename arma::Col<ElemType> VecType;
|
||||
|
||||
const size_t points = 1000;
|
||||
const size_t inputSize = 5;
|
||||
const size_t numClasses = 5;
|
||||
@@ -908,7 +917,7 @@ TEST_CASE("LinearSVMLBFGSMultipleClasses", "[LinearSVMTest]")
|
||||
GaussianDistribution g4(arma::vec("4.0 1.0 1.0 2.0 7.0"), identity);
|
||||
GaussianDistribution g5(arma::vec("1.0 0.0 1.0 8.0 3.0"), identity);
|
||||
|
||||
arma::mat data(inputSize, points);
|
||||
MatType data(inputSize, points);
|
||||
arma::Row<size_t> labels(points);
|
||||
|
||||
// This loop can be removed when ensmallen PR #136 is merged into a version
|
||||
@@ -921,32 +930,32 @@ TEST_CASE("LinearSVMLBFGSMultipleClasses", "[LinearSVMTest]")
|
||||
{
|
||||
for (size_t i = 0; i < points / 5; ++i)
|
||||
{
|
||||
data.col(i) = g1.Random();
|
||||
data.col(i) = arma::conv_to<VecType>::from(g1.Random());
|
||||
labels(i) = 0;
|
||||
}
|
||||
for (size_t i = points / 5; i < (2 * points) / 5; ++i)
|
||||
{
|
||||
data.col(i) = g2.Random();
|
||||
data.col(i) = arma::conv_to<VecType>::from(g2.Random());
|
||||
labels(i) = 1;
|
||||
}
|
||||
for (size_t i = (2 * points) / 5; i < (3 * points) / 5; ++i)
|
||||
{
|
||||
data.col(i) = g3.Random();
|
||||
data.col(i) = arma::conv_to<VecType>::from(g3.Random());
|
||||
labels(i) = 2;
|
||||
}
|
||||
for (size_t i = (3 * points) / 5; i < (4 * points) / 5; ++i)
|
||||
{
|
||||
data.col(i) = g4.Random();
|
||||
data.col(i) = arma::conv_to<VecType>::from(g4.Random());
|
||||
labels(i) = 3;
|
||||
}
|
||||
for (size_t i = (4 * points) / 5; i < points; ++i)
|
||||
{
|
||||
data.col(i) = g5.Random();
|
||||
data.col(i) = arma::conv_to<VecType>::from(g5.Random());
|
||||
labels(i) = 4;
|
||||
}
|
||||
|
||||
// Train linear svm object using L-BFGS optimizer.
|
||||
LinearSVM<arma::mat> lsvm(data, labels, numClasses, lambda);
|
||||
LinearSVM<MatType> lsvm(data, labels, numClasses, lambda);
|
||||
|
||||
// Compare training accuracy to 1.
|
||||
const double acc = lsvm.ComputeAccuracy(data, labels);
|
||||
@@ -956,27 +965,27 @@ TEST_CASE("LinearSVMLBFGSMultipleClasses", "[LinearSVMTest]")
|
||||
// Create test dataset.
|
||||
for (size_t i = 0; i < points / 5; ++i)
|
||||
{
|
||||
data.col(i) = g1.Random();
|
||||
data.col(i) = arma::conv_to<VecType>::from(g1.Random());
|
||||
labels(i) = 0;
|
||||
}
|
||||
for (size_t i = points / 5; i < (2 * points) / 5; ++i)
|
||||
{
|
||||
data.col(i) = g2.Random();
|
||||
data.col(i) = arma::conv_to<VecType>::from(g2.Random());
|
||||
labels(i) = 1;
|
||||
}
|
||||
for (size_t i = (2 * points) / 5; i < (3 * points) / 5; ++i)
|
||||
{
|
||||
data.col(i) = g3.Random();
|
||||
data.col(i) = arma::conv_to<VecType>::from(g3.Random());
|
||||
labels(i) = 2;
|
||||
}
|
||||
for (size_t i = (3 * points) / 5; i < (4 * points) / 5; ++i)
|
||||
{
|
||||
data.col(i) = g4.Random();
|
||||
data.col(i) = arma::conv_to<VecType>::from(g4.Random());
|
||||
labels(i) = 3;
|
||||
}
|
||||
for (size_t i = (4 * points) / 5; i < points; ++i)
|
||||
{
|
||||
data.col(i) = g5.Random();
|
||||
data.col(i) = arma::conv_to<VecType>::from(g5.Random());
|
||||
labels(i) = 4;
|
||||
}
|
||||
|
||||
@@ -995,8 +1004,13 @@ TEST_CASE("LinearSVMLBFGSMultipleClasses", "[LinearSVMTest]")
|
||||
/**
|
||||
* Testing single point classification (Classify()).
|
||||
*/
|
||||
TEST_CASE("LinearSVMClassifySinglePointTest", "[LinearSVMTest]")
|
||||
TEMPLATE_TEST_CASE("LinearSVMClassifySinglePointTest", "[LinearSVMTest]", float,
|
||||
double)
|
||||
{
|
||||
typedef TestType ElemType;
|
||||
typedef typename arma::Mat<ElemType> MatType;
|
||||
typedef typename arma::Col<ElemType> VecType;
|
||||
|
||||
const size_t points = 500;
|
||||
const size_t inputSize = 5;
|
||||
const size_t numClasses = 5;
|
||||
@@ -1010,70 +1024,79 @@ TEST_CASE("LinearSVMClassifySinglePointTest", "[LinearSVMTest]")
|
||||
GaussianDistribution g4(arma::vec("4.0 1.0 1.0 2.0 7.0"), identity);
|
||||
GaussianDistribution g5(arma::vec("1.0 0.0 1.0 8.0 3.0"), identity);
|
||||
|
||||
arma::mat data(inputSize, points);
|
||||
MatType data(inputSize, points);
|
||||
arma::Row<size_t> labels(points);
|
||||
|
||||
for (size_t i = 0; i < points / 5; ++i)
|
||||
{
|
||||
data.col(i) = g1.Random();
|
||||
data.col(i) = arma::conv_to<VecType>::from(g1.Random());
|
||||
labels(i) = 0;
|
||||
}
|
||||
for (size_t i = points / 5; i < (2 * points) / 5; ++i)
|
||||
{
|
||||
data.col(i) = g2.Random();
|
||||
data.col(i) = arma::conv_to<VecType>::from(g2.Random());
|
||||
labels(i) = 1;
|
||||
}
|
||||
for (size_t i = (2 * points) / 5; i < (3 * points) / 5; ++i)
|
||||
{
|
||||
data.col(i) = g3.Random();
|
||||
data.col(i) = arma::conv_to<VecType>::from(g3.Random());
|
||||
labels(i) = 2;
|
||||
}
|
||||
for (size_t i = (3 * points) / 5; i < (4 * points) / 5; ++i)
|
||||
{
|
||||
data.col(i) = g4.Random();
|
||||
data.col(i) = arma::conv_to<VecType>::from(g4.Random());
|
||||
labels(i) = 3;
|
||||
}
|
||||
for (size_t i = (4 * points) / 5; i < points; ++i)
|
||||
{
|
||||
data.col(i) = g5.Random();
|
||||
data.col(i) = arma::conv_to<VecType>::from(g5.Random());
|
||||
labels(i) = 4;
|
||||
}
|
||||
|
||||
// Train linear svm object.
|
||||
LinearSVM<arma::mat> lsvm(data, labels, numClasses, lambda);
|
||||
LinearSVM<MatType> lsvm(data, labels, numClasses, lambda);
|
||||
|
||||
// Create test dataset.
|
||||
for (size_t i = 0; i < points / 5; ++i)
|
||||
{
|
||||
data.col(i) = g1.Random();
|
||||
data.col(i) = arma::conv_to<VecType>::from(g1.Random());
|
||||
labels(i) = 0;
|
||||
}
|
||||
for (size_t i = points / 5; i < (2 * points) / 5; ++i)
|
||||
{
|
||||
data.col(i) = g2.Random();
|
||||
data.col(i) = arma::conv_to<VecType>::from(g2.Random());
|
||||
labels(i) = 1;
|
||||
}
|
||||
for (size_t i = (2 * points) / 5; i < (3 * points) / 5; ++i)
|
||||
{
|
||||
data.col(i) = g3.Random();
|
||||
data.col(i) = arma::conv_to<VecType>::from(g3.Random());
|
||||
labels(i) = 2;
|
||||
}
|
||||
for (size_t i = (3 * points) / 5; i < (4 * points) / 5; ++i)
|
||||
{
|
||||
data.col(i) = g4.Random();
|
||||
data.col(i) = arma::conv_to<VecType>::from(g4.Random());
|
||||
labels(i) = 3;
|
||||
}
|
||||
for (size_t i = (4 * points) / 5; i < points; ++i)
|
||||
{
|
||||
data.col(i) = g5.Random();
|
||||
data.col(i) = arma::conv_to<VecType>::from(g5.Random());
|
||||
labels(i) = 4;
|
||||
}
|
||||
|
||||
lsvm.Classify(data, labels);
|
||||
MatType scores;
|
||||
lsvm.Classify(data, labels, scores);
|
||||
|
||||
for (size_t i = 0; i < data.n_cols; ++i)
|
||||
{
|
||||
REQUIRE(lsvm.Classify(data.col(i)) == labels(i));
|
||||
|
||||
size_t prediction;
|
||||
VecType scoresVec;
|
||||
lsvm.Classify(data.col(i), prediction, scoresVec);
|
||||
|
||||
REQUIRE(prediction == labels(i));
|
||||
REQUIRE(scoresVec.n_elem == scores.n_rows);
|
||||
REQUIRE(arma::approx_equal(scoresVec, scores.col(i), "absdiff", 1e-5));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1188,8 +1211,259 @@ TEST_CASE("LinearSVMCallbackTest", "[LinearSVMTest]")
|
||||
CallbackTestFunction cb;
|
||||
|
||||
ens::L_BFGS opt;
|
||||
LinearSVM<arma::mat> lsvm(dataset, labels, numClasses, lambda,
|
||||
delta, false, opt, cb);
|
||||
LinearSVM<arma::mat> lsvm(dataset, labels, numClasses, lambda, delta,
|
||||
false, cb);
|
||||
|
||||
REQUIRE(cb.calledEndOptimization == true);
|
||||
}
|
||||
|
||||
// Test all variants of LinearSVM constructors.
|
||||
TEMPLATE_TEST_CASE("LinearSVMConstructorVariantTest", "[LinearSVMTest]",
|
||||
arma::fmat, arma::mat)
|
||||
{
|
||||
typedef TestType MatType;
|
||||
|
||||
// Create some random data. The results here do not matter all that much;
|
||||
// this is more of a test that all constructor variants successfully compile
|
||||
// and produce models at all.
|
||||
MatType dataset(10, 800, arma::fill::randu);
|
||||
arma::Row<size_t> labels(800);
|
||||
for (size_t i = 0; i < 800; ++i)
|
||||
labels[i] = RandInt(0, 2);
|
||||
|
||||
LinearSVM<> lsvm1;
|
||||
LinearSVM<> lsvm2(10, 2);
|
||||
LinearSVM<> lsvm3(10, 2, 0.0002, 1.1, true);
|
||||
LinearSVM<> lsvm4(dataset, labels, 2);
|
||||
LinearSVM<> lsvm5(dataset, labels, 2, 0.0003, 1.2, true);
|
||||
LinearSVM<> lsvm6(dataset, labels, 2, 0.0004, 1.3, true,
|
||||
CallbackTestFunction());
|
||||
LinearSVM<> lsvm7(dataset, labels, 2, 0.0005, 1.4, true,
|
||||
CallbackTestFunction(), ens::TimerStop(1.0));
|
||||
|
||||
ens::StandardSGD sgd;
|
||||
LinearSVM<> lsvm8(dataset, labels, 2, sgd);
|
||||
LinearSVM<> lsvm9(dataset, labels, 2, sgd, 0.0006, 1.5, true);
|
||||
LinearSVM<> lsvm10(dataset, labels, 2, sgd, 0.0007, 1.6, true,
|
||||
CallbackTestFunction());
|
||||
LinearSVM<> lsvm11(dataset, labels, 2, sgd, 0.0008, 1.7, true,
|
||||
CallbackTestFunction(), ens::TimerStop(1.0));
|
||||
|
||||
// Check that the variants that did not train have reasonable values.
|
||||
REQUIRE(lsvm1.Lambda() == Approx(0.0001));
|
||||
REQUIRE(lsvm1.Delta() == Approx(1.0));
|
||||
REQUIRE(lsvm1.FitIntercept() == false);
|
||||
|
||||
REQUIRE(lsvm2.Lambda() == Approx(0.0001));
|
||||
REQUIRE(lsvm2.Delta() == Approx(1.0));
|
||||
REQUIRE(lsvm2.FitIntercept() == false);
|
||||
REQUIRE(lsvm2.NumClasses() == 2);
|
||||
|
||||
REQUIRE(lsvm3.Lambda() == Approx(0.0002));
|
||||
REQUIRE(lsvm3.Delta() == Approx(1.1));
|
||||
REQUIRE(lsvm3.FitIntercept() == true);
|
||||
REQUIRE(lsvm3.NumClasses() == 2);
|
||||
|
||||
// Now check that the variants that trained have reasonable models of the
|
||||
// right size.
|
||||
REQUIRE(lsvm4.Lambda() == Approx(0.0001));
|
||||
REQUIRE(lsvm4.Delta() == Approx(1.0));
|
||||
REQUIRE(lsvm4.FitIntercept() == false);
|
||||
REQUIRE(lsvm4.FeatureSize() == 10);
|
||||
REQUIRE(lsvm4.NumClasses() == 2);
|
||||
|
||||
REQUIRE(lsvm5.Lambda() == Approx(0.0003));
|
||||
REQUIRE(lsvm5.Delta() == Approx(1.2));
|
||||
REQUIRE(lsvm5.FitIntercept() == true);
|
||||
REQUIRE(lsvm5.FeatureSize() == 10);
|
||||
REQUIRE(lsvm5.NumClasses() == 2);
|
||||
|
||||
REQUIRE(lsvm6.Lambda() == Approx(0.0004));
|
||||
REQUIRE(lsvm6.Delta() == Approx(1.3));
|
||||
REQUIRE(lsvm6.FitIntercept() == true);
|
||||
REQUIRE(lsvm6.FeatureSize() == 10);
|
||||
REQUIRE(lsvm6.NumClasses() == 2);
|
||||
|
||||
REQUIRE(lsvm7.Lambda() == Approx(0.0005));
|
||||
REQUIRE(lsvm7.Delta() == Approx(1.4));
|
||||
REQUIRE(lsvm7.FitIntercept() == true);
|
||||
REQUIRE(lsvm7.FeatureSize() == 10);
|
||||
REQUIRE(lsvm7.NumClasses() == 2);
|
||||
|
||||
REQUIRE(lsvm8.Lambda() == Approx(0.0001));
|
||||
REQUIRE(lsvm8.Delta() == Approx(1.0));
|
||||
REQUIRE(lsvm8.FitIntercept() == false);
|
||||
REQUIRE(lsvm8.FeatureSize() == 10);
|
||||
REQUIRE(lsvm8.NumClasses() == 2);
|
||||
|
||||
REQUIRE(lsvm9.Lambda() == Approx(0.0006));
|
||||
REQUIRE(lsvm9.Delta() == Approx(1.5));
|
||||
REQUIRE(lsvm9.FitIntercept() == true);
|
||||
REQUIRE(lsvm9.FeatureSize() == 10);
|
||||
REQUIRE(lsvm9.NumClasses() == 2);
|
||||
|
||||
REQUIRE(lsvm10.Lambda() == Approx(0.0007));
|
||||
REQUIRE(lsvm10.Delta() == Approx(1.6));
|
||||
REQUIRE(lsvm10.FitIntercept() == true);
|
||||
REQUIRE(lsvm10.FeatureSize() == 10);
|
||||
REQUIRE(lsvm10.NumClasses() == 2);
|
||||
|
||||
REQUIRE(lsvm11.Lambda() == Approx(0.0008));
|
||||
REQUIRE(lsvm11.Delta() == Approx(1.7));
|
||||
REQUIRE(lsvm11.FitIntercept() == true);
|
||||
REQUIRE(lsvm11.FeatureSize() == 10);
|
||||
REQUIRE(lsvm11.NumClasses() == 2);
|
||||
}
|
||||
|
||||
// Test all variants of LinearSVM Train() functions.
|
||||
TEMPLATE_TEST_CASE("LinearSVMTrainVariantTest", "[LinearSVMTest]", arma::fmat,
|
||||
arma::mat)
|
||||
{
|
||||
typedef TestType MatType;
|
||||
|
||||
// Create some random data. The results here do not matter all that much;
|
||||
// this is more of a test that all constructor variants successfully compile
|
||||
// and produce models at all.
|
||||
MatType dataset(10, 800, arma::fill::randu);
|
||||
arma::Row<size_t> labels(800);
|
||||
for (size_t i = 0; i < 800; ++i)
|
||||
labels[i] = RandInt(0, 2);
|
||||
|
||||
LinearSVM<> lsvm1(10, 2, 1.2, 0.5, true);
|
||||
LinearSVM<> lsvm2(10, 2, 1.2, 0.5, true);
|
||||
LinearSVM<> lsvm3(10, 2, 1.2, 0.5, true);
|
||||
LinearSVM<> lsvm4(10, 2, 1.2, 0.5, true);
|
||||
LinearSVM<> lsvm5(10, 2, 1.2, 0.5, true);
|
||||
LinearSVM<> lsvm6(10, 2, 1.2, 0.5, true);
|
||||
LinearSVM<> lsvm7(10, 2, 1.2, 0.5, true);
|
||||
LinearSVM<> lsvm8(10, 2, 1.2, 0.5, true);
|
||||
LinearSVM<> lsvm9(10, 2, 1.2, 0.5, true);
|
||||
LinearSVM<> lsvm10(10, 2, 1.2, 0.5, true);
|
||||
LinearSVM<> lsvm11(10, 2, 1.2, 0.5, true);
|
||||
LinearSVM<> lsvm12(10, 2, 1.2, 0.5, true);
|
||||
LinearSVM<> lsvm13(10, 2, 1.2, 0.5, true);
|
||||
LinearSVM<> lsvm14(10, 2, 1.2, 0.5, true);
|
||||
LinearSVM<> lsvm15(10, 2, 1.2, 0.5, true);
|
||||
LinearSVM<> lsvm16(10, 2, 1.2, 0.5, true);
|
||||
|
||||
lsvm1.Train(dataset, labels, 2);
|
||||
lsvm2.Train(dataset, labels, 2, CallbackTestFunction());
|
||||
lsvm3.Train(dataset, labels, 2, CallbackTestFunction(), ens::TimerStop(1.0));
|
||||
lsvm4.Train(dataset, labels, 2, 0.0002);
|
||||
lsvm5.Train(dataset, labels, 2, 0.0003, 1.1);
|
||||
lsvm6.Train(dataset, labels, 2, 0.0004, 1.2, false);
|
||||
lsvm7.Train(dataset, labels, 2, 0.0005, 1.3, true, CallbackTestFunction());
|
||||
lsvm8.Train(dataset, labels, 2, 0.0006, 1.4, false, CallbackTestFunction(),
|
||||
ens::TimerStop(1.0));
|
||||
|
||||
ens::Adam adam;
|
||||
lsvm9.Train(dataset, labels, 2, adam);
|
||||
lsvm10.Train(dataset, labels, 2, adam, CallbackTestFunction());
|
||||
lsvm11.Train(dataset, labels, 2, adam, CallbackTestFunction(),
|
||||
ens::TimerStop(1.0));
|
||||
lsvm12.Train(dataset, labels, 2, adam, 0.0007);
|
||||
lsvm13.Train(dataset, labels, 2, adam, 0.0008, 1.5);
|
||||
lsvm14.Train(dataset, labels, 2, adam, 0.0009, 1.6, false);
|
||||
lsvm15.Train(dataset, labels, 2, adam, 0.001, 1.7, true,
|
||||
CallbackTestFunction());
|
||||
lsvm16.Train(dataset, labels, 2, adam, 0.0011, 1.8, false,
|
||||
CallbackTestFunction(), ens::TimerStop(1.0));
|
||||
|
||||
// Check that all hyperparameters are set as expected, and that the model has
|
||||
// the correct size.
|
||||
REQUIRE(lsvm1.Lambda() == Approx(1.2));
|
||||
REQUIRE(lsvm1.Delta() == Approx(0.5));
|
||||
REQUIRE(lsvm1.FitIntercept() == true);
|
||||
REQUIRE(lsvm1.FeatureSize() == 10);
|
||||
REQUIRE(lsvm1.NumClasses() == 2);
|
||||
|
||||
REQUIRE(lsvm2.Lambda() == Approx(1.2));
|
||||
REQUIRE(lsvm2.Delta() == Approx(0.5));
|
||||
REQUIRE(lsvm2.FitIntercept() == true);
|
||||
REQUIRE(lsvm2.FeatureSize() == 10);
|
||||
REQUIRE(lsvm2.NumClasses() == 2);
|
||||
|
||||
REQUIRE(lsvm3.Lambda() == Approx(1.2));
|
||||
REQUIRE(lsvm3.Delta() == Approx(0.5));
|
||||
REQUIRE(lsvm3.FitIntercept() == true);
|
||||
REQUIRE(lsvm3.FeatureSize() == 10);
|
||||
REQUIRE(lsvm3.NumClasses() == 2);
|
||||
|
||||
REQUIRE(lsvm4.Lambda() == Approx(0.0002));
|
||||
REQUIRE(lsvm4.Delta() == Approx(0.5));
|
||||
REQUIRE(lsvm4.FitIntercept() == true);
|
||||
REQUIRE(lsvm4.FeatureSize() == 10);
|
||||
REQUIRE(lsvm4.NumClasses() == 2);
|
||||
|
||||
REQUIRE(lsvm5.Lambda() == Approx(0.0003));
|
||||
REQUIRE(lsvm5.Delta() == Approx(1.1));
|
||||
REQUIRE(lsvm5.FitIntercept() == true);
|
||||
REQUIRE(lsvm5.FeatureSize() == 10);
|
||||
REQUIRE(lsvm5.NumClasses() == 2);
|
||||
|
||||
REQUIRE(lsvm6.Lambda() == Approx(0.0004));
|
||||
REQUIRE(lsvm6.Delta() == Approx(1.2));
|
||||
REQUIRE(lsvm6.FitIntercept() == false);
|
||||
REQUIRE(lsvm6.FeatureSize() == 10);
|
||||
REQUIRE(lsvm6.NumClasses() == 2);
|
||||
|
||||
REQUIRE(lsvm7.Lambda() == Approx(0.0005));
|
||||
REQUIRE(lsvm7.Delta() == Approx(1.3));
|
||||
REQUIRE(lsvm7.FitIntercept() == true);
|
||||
REQUIRE(lsvm7.FeatureSize() == 10);
|
||||
REQUIRE(lsvm7.NumClasses() == 2);
|
||||
|
||||
REQUIRE(lsvm8.Lambda() == Approx(0.0006));
|
||||
REQUIRE(lsvm8.Delta() == Approx(1.4));
|
||||
REQUIRE(lsvm8.FitIntercept() == false);
|
||||
REQUIRE(lsvm8.FeatureSize() == 10);
|
||||
REQUIRE(lsvm8.NumClasses() == 2);
|
||||
|
||||
REQUIRE(lsvm9.Lambda() == Approx(1.2));
|
||||
REQUIRE(lsvm9.Delta() == Approx(0.5));
|
||||
REQUIRE(lsvm9.FitIntercept() == true);
|
||||
REQUIRE(lsvm9.FeatureSize() == 10);
|
||||
REQUIRE(lsvm9.NumClasses() == 2);
|
||||
|
||||
REQUIRE(lsvm10.Lambda() == Approx(1.2));
|
||||
REQUIRE(lsvm10.Delta() == Approx(0.5));
|
||||
REQUIRE(lsvm10.FitIntercept() == true);
|
||||
REQUIRE(lsvm10.FeatureSize() == 10);
|
||||
REQUIRE(lsvm10.NumClasses() == 2);
|
||||
|
||||
REQUIRE(lsvm11.Lambda() == Approx(1.2));
|
||||
REQUIRE(lsvm11.Delta() == Approx(0.5));
|
||||
REQUIRE(lsvm11.FitIntercept() == true);
|
||||
REQUIRE(lsvm11.FeatureSize() == 10);
|
||||
REQUIRE(lsvm11.NumClasses() == 2);
|
||||
|
||||
REQUIRE(lsvm12.Lambda() == Approx(0.0007));
|
||||
REQUIRE(lsvm12.Delta() == Approx(0.5));
|
||||
REQUIRE(lsvm12.FitIntercept() == true);
|
||||
REQUIRE(lsvm12.FeatureSize() == 10);
|
||||
REQUIRE(lsvm12.NumClasses() == 2);
|
||||
|
||||
REQUIRE(lsvm13.Lambda() == Approx(0.0008));
|
||||
REQUIRE(lsvm13.Delta() == Approx(1.5));
|
||||
REQUIRE(lsvm13.FitIntercept() == true);
|
||||
REQUIRE(lsvm13.FeatureSize() == 10);
|
||||
REQUIRE(lsvm13.NumClasses() == 2);
|
||||
|
||||
REQUIRE(lsvm14.Lambda() == Approx(0.0009));
|
||||
REQUIRE(lsvm14.Delta() == Approx(1.6));
|
||||
REQUIRE(lsvm14.FitIntercept() == false);
|
||||
REQUIRE(lsvm14.FeatureSize() == 10);
|
||||
REQUIRE(lsvm14.NumClasses() == 2);
|
||||
|
||||
REQUIRE(lsvm15.Lambda() == Approx(0.001));
|
||||
REQUIRE(lsvm15.Delta() == Approx(1.7));
|
||||
REQUIRE(lsvm15.FitIntercept() == true);
|
||||
REQUIRE(lsvm15.FeatureSize() == 10);
|
||||
REQUIRE(lsvm15.NumClasses() == 2);
|
||||
|
||||
REQUIRE(lsvm16.Lambda() == Approx(0.0011));
|
||||
REQUIRE(lsvm16.Delta() == Approx(1.8));
|
||||
REQUIRE(lsvm16.FitIntercept() == false);
|
||||
REQUIRE(lsvm16.FeatureSize() == 10);
|
||||
REQUIRE(lsvm16.NumClasses() == 2);
|
||||
}
|
||||
|
||||
@@ -41,8 +41,8 @@ TEST_CASE_METHOD(BRTestFixture,
|
||||
|
||||
RUN_BINDING();
|
||||
|
||||
BayesianLinearRegression* estimator =
|
||||
params.Get<BayesianLinearRegression*>("output_model");
|
||||
BayesianLinearRegression<>* estimator =
|
||||
params.Get<BayesianLinearRegression<>*>("output_model");
|
||||
|
||||
REQUIRE(estimator->DataOffset().n_elem == 0);
|
||||
REQUIRE(estimator->DataScale().n_elem == 0);
|
||||
@@ -61,7 +61,7 @@ TEST_CASE_METHOD(BRTestFixture,
|
||||
const arma::rowvec omega = arma::randu<arma::rowvec>(m);
|
||||
arma::rowvec y = omega * matX;
|
||||
|
||||
BayesianLinearRegression model;
|
||||
BayesianLinearRegression<> model;
|
||||
model.Train(matX, y);
|
||||
|
||||
arma::rowvec responses;
|
||||
@@ -72,8 +72,8 @@ TEST_CASE_METHOD(BRTestFixture,
|
||||
|
||||
RUN_BINDING();
|
||||
|
||||
BayesianLinearRegression* mOut =
|
||||
params.Get<BayesianLinearRegression*>("output_model");
|
||||
BayesianLinearRegression<>* mOut =
|
||||
params.Get<BayesianLinearRegression<>*>("output_model");
|
||||
|
||||
ResetSettings();
|
||||
|
||||
@@ -101,7 +101,7 @@ TEST_CASE_METHOD(BRTestFixture,
|
||||
const arma::rowvec omega = arma::randu<arma::rowvec>(m);
|
||||
arma::rowvec y = omega * matX;
|
||||
|
||||
BayesianLinearRegression model;
|
||||
BayesianLinearRegression<> model;
|
||||
model.Train(matX, y);
|
||||
|
||||
arma::rowvec responses;
|
||||
@@ -121,7 +121,7 @@ TEST_CASE_METHOD(BRTestFixture,
|
||||
// An error should occur.
|
||||
SetInputParam("input", std::move(matX));
|
||||
SetInputParam("input_model",
|
||||
params.Get<BayesianLinearRegression*>("output_model"));
|
||||
params.Get<BayesianLinearRegression<>*>("output_model"));
|
||||
SetInputParam("test", std::move(matXtest));
|
||||
|
||||
REQUIRE_THROWS_AS(RUN_BINDING(), std::runtime_error);
|
||||
|
||||
@@ -37,7 +37,7 @@ TEST_CASE_METHOD(LRPredictTestFixture, "LRPredictWrongDimOfDataTest1t",
|
||||
arma::rowvec trainY = arma::randu<arma::rowvec>(N);
|
||||
arma::mat testX = arma::randu<arma::mat>(D - 1, M); // Wrong dimensionality.
|
||||
|
||||
LinearRegression* model = new LinearRegression();
|
||||
LinearRegression<>* model = new LinearRegression<>();
|
||||
model->Train(trainX, trainY);
|
||||
|
||||
SetInputParam("input_model", std::move(model));
|
||||
@@ -60,7 +60,7 @@ TEST_CASE_METHOD(LRPredictTestFixture, "LRPredictPredictionSizeCheck",
|
||||
arma::rowvec trainY = arma::randu<arma::rowvec>(N);
|
||||
arma::mat testX = arma::randu<arma::mat>(D, M);
|
||||
|
||||
LinearRegression* model = new LinearRegression();
|
||||
LinearRegression<>* model = new LinearRegression<>();
|
||||
model->Train(trainX, trainY);
|
||||
|
||||
SetInputParam("input_model", std::move(model));
|
||||
|
||||
@@ -118,7 +118,7 @@ TEST_CASE_METHOD(LRTestFixture, "LRModelReload",
|
||||
|
||||
RUN_BINDING();
|
||||
|
||||
LinearRegression* model = params.Get<LinearRegression*>("output_model");
|
||||
LinearRegression<>* model = params.Get<LinearRegression<>*>("output_model");
|
||||
const arma::rowvec testY1 = params.Get<arma::rowvec>("output_predictions");
|
||||
|
||||
ResetSettings();
|
||||
@@ -191,7 +191,7 @@ TEST_CASE_METHOD(LRTestFixture, "LRWrongDimOfDataTest2",
|
||||
|
||||
RUN_BINDING();
|
||||
|
||||
LinearRegression* model = params.Get<LinearRegression*>("output_model");
|
||||
LinearRegression<>* model = params.Get<LinearRegression<>*>("output_model");
|
||||
|
||||
ResetSettings();
|
||||
|
||||
|
||||
@@ -44,8 +44,7 @@ TEST_CASE_METHOD(LRFitTestFixture, "LRFitDifferentLambdas",
|
||||
// The first solution.
|
||||
RUN_BINDING();
|
||||
arma::rowvec preds1;
|
||||
params.Get<LinearRegression*>("output_model")->Predict(testX,
|
||||
preds1);
|
||||
params.Get<LinearRegression<>*>("output_model")->Predict(testX, preds1);
|
||||
const double testY1 = preds1(0);
|
||||
|
||||
ResetSettings();
|
||||
@@ -57,8 +56,7 @@ TEST_CASE_METHOD(LRFitTestFixture, "LRFitDifferentLambdas",
|
||||
// The second solution.
|
||||
RUN_BINDING();
|
||||
arma::rowvec preds2;
|
||||
params.Get<LinearRegression*>("output_model")->Predict(testX,
|
||||
preds2);
|
||||
params.Get<LinearRegression<>*>("output_model")->Predict(testX, preds2);
|
||||
const double testY2 = preds2(0);
|
||||
|
||||
// Second solution has stronger regularization,
|
||||
|
||||
@@ -397,3 +397,104 @@ TEST_CASE("NaiveBayesClassifierHighDimensionsTest", "[NBCTest]")
|
||||
for (size_t i = 0; i < calcVec.n_cols; ++i)
|
||||
REQUIRE(calcVec(i) == testLabels(i));
|
||||
}
|
||||
|
||||
/**
|
||||
* Test that we can reset the model.
|
||||
*/
|
||||
TEST_CASE("NBCResetTest", "[NBCTest]")
|
||||
{
|
||||
const char* trainFilename = "trainSet.csv";
|
||||
|
||||
arma::mat trainData;
|
||||
if (!data::Load(trainFilename, trainData))
|
||||
FAIL("Cannot load dataset");
|
||||
|
||||
// Get the labels out.
|
||||
arma::Row<size_t> labels(trainData.n_cols);
|
||||
for (size_t i = 0; i < trainData.n_cols; ++i)
|
||||
labels[i] = trainData(trainData.n_rows - 1, i);
|
||||
trainData.shed_row(trainData.n_rows - 1);
|
||||
|
||||
NaiveBayesClassifier<> nbc1, nbc2;
|
||||
|
||||
nbc1.Train(trainData, labels, 2);
|
||||
nbc2.Train(trainData, labels, 2);
|
||||
|
||||
REQUIRE(approx_equal(nbc1.Probabilities(), nbc2.Probabilities(), "absdiff",
|
||||
1e-5));
|
||||
REQUIRE(approx_equal(nbc1.Means(), nbc2.Means(), "absdiff", 1e-5));
|
||||
REQUIRE(approx_equal(nbc1.Variances(), nbc2.Variances(), "absdiff", 1e-5));
|
||||
|
||||
// Now reset one model but train the other. Modify the training set slightly.
|
||||
trainData += 0.1 * arma::randu<arma::mat>(trainData.n_rows, trainData.n_cols);
|
||||
|
||||
nbc1.Train(trainData, labels, 2);
|
||||
nbc2.Reset();
|
||||
nbc2.Train(trainData, labels, 2);
|
||||
|
||||
REQUIRE(!approx_equal(nbc1.Probabilities(), nbc2.Probabilities(), "absdiff",
|
||||
1e-5));
|
||||
REQUIRE(!approx_equal(nbc1.Means(), nbc2.Means(), "absdiff", 1e-5));
|
||||
REQUIRE(!approx_equal(nbc1.Variances(), nbc2.Variances(), "absdiff", 1e-5));
|
||||
}
|
||||
|
||||
/**
|
||||
* Test that we can use a model for incremental point-by-point training.
|
||||
*/
|
||||
TEMPLATE_TEST_CASE("NBCIncrementalTest", "[NBCTest]", arma::fmat, arma::mat)
|
||||
{
|
||||
typedef TestType MatType;
|
||||
|
||||
const char* trainFilename = "trainSet.csv";
|
||||
|
||||
MatType trainData;
|
||||
if (!data::Load(trainFilename, trainData))
|
||||
FAIL("Cannot load dataset");
|
||||
|
||||
// Get the labels out.
|
||||
arma::Row<size_t> labels(trainData.n_cols);
|
||||
for (size_t i = 0; i < trainData.n_cols; ++i)
|
||||
labels[i] = trainData(trainData.n_rows - 1, i);
|
||||
trainData.shed_row(trainData.n_rows - 1);
|
||||
|
||||
NaiveBayesClassifier<MatType> nbc(trainData.n_rows, 2);
|
||||
|
||||
for (size_t i = 0; i < trainData.n_cols; ++i)
|
||||
{
|
||||
nbc.Train(trainData.col(i), labels[i]);
|
||||
}
|
||||
|
||||
// We don't care what the result is, we are more concerned with the fact that
|
||||
// training worked at all.
|
||||
REQUIRE(nbc.Probabilities().n_elem == 2);
|
||||
REQUIRE(nbc.Means().n_rows == trainData.n_rows);
|
||||
REQUIRE(nbc.Means().n_cols == 2);
|
||||
REQUIRE(nbc.Variances().n_rows == trainData.n_rows);
|
||||
REQUIRE(nbc.Variances().n_cols == 2);
|
||||
}
|
||||
|
||||
/**
|
||||
* Test that we can train on sparse data with different internal model types.
|
||||
*/
|
||||
TEMPLATE_TEST_CASE("NBCModelMatTypeTest", "[NBCTest]", float, double)
|
||||
{
|
||||
typedef TestType ElemType;
|
||||
|
||||
NaiveBayesClassifier<arma::Mat<ElemType>> nbc;
|
||||
|
||||
// Create random data; 5000 points in 4 classes.
|
||||
arma::SpMat<ElemType> data;
|
||||
data.sprandu(100, 5000, 0.2);
|
||||
arma::Row<size_t> labels =
|
||||
arma::randi<arma::Row<size_t>>(5000, arma::distr_param(0, 3));
|
||||
|
||||
nbc.Train(data, labels, 4);
|
||||
|
||||
// We don't care what the result is, we are more concerned with the fact that
|
||||
// training worked at all.
|
||||
REQUIRE(nbc.Probabilities().n_elem == 4);
|
||||
REQUIRE(nbc.Means().n_rows == data.n_rows);
|
||||
REQUIRE(nbc.Means().n_cols == 4);
|
||||
REQUIRE(nbc.Variances().n_rows == data.n_rows);
|
||||
REQUIRE(nbc.Variances().n_cols == 4);
|
||||
}
|
||||
|
||||
@@ -1061,20 +1061,18 @@ TEST_CASE("LARSTest", "[SerializationTest]")
|
||||
arma::vec beta = arma::randn(75, 1);
|
||||
arma::rowvec y = beta.t() * X;
|
||||
|
||||
LARS lars(true, 0.1, 0.1);
|
||||
arma::vec betaOpt;
|
||||
lars.Train(X, y, betaOpt);
|
||||
LARS<> lars(true, 0.1, 0.1);
|
||||
lars.Train(X, y);
|
||||
|
||||
// Now, serialize.
|
||||
LARS xmlLars(false, 0.5, 0.0), binaryLars(true, 1.0, 0.0),
|
||||
LARS<> xmlLars(false, 0.5, 0.0), binaryLars(true, 1.0, 0.0),
|
||||
jsonLars(false, 0.1, 0.1);
|
||||
|
||||
// Train jsonLars.
|
||||
arma::mat jsonX = arma::randn(25, 150);
|
||||
arma::vec jsonBeta = arma::randn(25, 1);
|
||||
arma::rowvec jsonY = jsonBeta.t() * jsonX;
|
||||
arma::vec jsonBetaOpt;
|
||||
jsonLars.Train(jsonX, jsonY, jsonBetaOpt);
|
||||
jsonLars.Train(jsonX, jsonY);
|
||||
|
||||
SerializeObjectAll(lars, xmlLars, binaryLars, jsonLars);
|
||||
|
||||
@@ -1552,12 +1550,12 @@ TEST_CASE("BayesianLinearRegressionTest", "[SerializationTest]")
|
||||
arma::vec omega = arma::randn(75, 1);
|
||||
arma::rowvec y = omega.t() * matX;
|
||||
|
||||
BayesianLinearRegression blr(false, false);
|
||||
BayesianLinearRegression<> blr(false, false);
|
||||
blr.Train(matX, y);
|
||||
arma::vec omegaOpt = blr.Omega();
|
||||
|
||||
// Now, serialize.
|
||||
BayesianLinearRegression xmlBlr(false, false), binaryBlr(false, false),
|
||||
BayesianLinearRegression<> xmlBlr(false, false), binaryBlr(false, false),
|
||||
textBlr(false, false);
|
||||
|
||||
SerializeObjectAll(blr, xmlBlr, binaryBlr, textBlr);
|
||||
|
||||
@@ -30,11 +30,11 @@ using namespace mlpack;
|
||||
* @param shuffledResponses Matrix object to store the shuffled responses into.
|
||||
*/
|
||||
inline void LogisticRegressionTestData(arma::mat& data,
|
||||
arma::mat& testData,
|
||||
arma::mat& shuffledData,
|
||||
arma::Row<size_t>& responses,
|
||||
arma::Row<size_t>& testResponses,
|
||||
arma::Row<size_t>& shuffledResponses)
|
||||
arma::mat& testData,
|
||||
arma::mat& shuffledData,
|
||||
arma::Row<size_t>& responses,
|
||||
arma::Row<size_t>& testResponses,
|
||||
arma::Row<size_t>& shuffledResponses)
|
||||
{
|
||||
// Generate a two-Gaussian dataset.
|
||||
GaussianDistribution g1(arma::vec("1.0 1.0 1.0"), arma::eye<arma::mat>(3, 3));
|
||||
@@ -79,15 +79,15 @@ inline void LogisticRegressionTestData(arma::mat& data,
|
||||
}
|
||||
}
|
||||
|
||||
template<typename MatType>
|
||||
template<typename MatType, typename ResponsesType>
|
||||
void LoadBostonHousingDataset(MatType& trainData,
|
||||
MatType& testData,
|
||||
arma::rowvec& trainResponses,
|
||||
arma::rowvec& testResponses,
|
||||
ResponsesType& trainResponses,
|
||||
ResponsesType& testResponses,
|
||||
data::DatasetInfo& info)
|
||||
{
|
||||
MatType dataset;
|
||||
arma::rowvec responses;
|
||||
ResponsesType responses;
|
||||
|
||||
// Defining categorical deimensions.
|
||||
info.SetDimensionality(13);
|
||||
@@ -103,10 +103,11 @@ void LoadBostonHousingDataset(MatType& trainData,
|
||||
trainResponses, testResponses, 0.3);
|
||||
}
|
||||
|
||||
inline double RMSE(const arma::Row<double>& predictions,
|
||||
const arma::Row<double>& trueResponses)
|
||||
template<typename ElemType>
|
||||
inline ElemType RMSE(const arma::Row<ElemType>& predictions,
|
||||
const arma::Row<ElemType>& trueResponses)
|
||||
{
|
||||
double mse = arma::accu(arma::square(predictions - trueResponses)) /
|
||||
ElemType mse = arma::accu(arma::square(predictions - trueResponses)) /
|
||||
predictions.n_elem;
|
||||
return sqrt(mse);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user