From 7f1a6566eaf47e573e880d00ea64738b2d17ee56 Mon Sep 17 00:00:00 2001 From: Abhimanyu Dayal Date: Wed, 27 Mar 2024 01:54:20 +0530 Subject: [PATCH 1/4] Made minor changes to documentation - bindings.md --- doc/developer/bindings.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/doc/developer/bindings.md b/doc/developer/bindings.md index dd63c0b62f..9b35ba90c7 100644 --- a/doc/developer/bindings.md +++ b/doc/developer/bindings.md @@ -664,7 +664,7 @@ parameter is required, and whether the parameter is an input or output parameter. Then as arguments to the macros, the name, description, and sometimes the single-character alias and the default value of the parameter. -To give a flavor of how these definitions look, the definition +To give an idea of how these definitions look, the definition ```c++ PARAM_STRING_IN("algorithm", "The algorithm to use: 'svd' or 'blah'.", "a"); @@ -724,7 +724,7 @@ And for input parameters, the parameter may also be required: See the source documentation for each macro to read further details. Note also that each possible combination of `IN`, `OUT`, and `REQ` is not -available---output options cannot be required, and some combinations simply have +available - output options cannot be required, and some combinations simply have not been added because they have not been needed. The `PARAM_MODEL_IN()` and `PARAM_MODEL_OUT()` macros are used to serialize @@ -777,8 +777,8 @@ PARAM_MODEL_OUT(LinearRegression, "output_model", "The randomly generated " "linear regression output model.", "M"); ``` -Note that even the parameter documentation strings must be a little be agnostic -to the binding type, because the command-line interface is so different than the +Note that even the parameter documentation strings must be a little agnostic +to the binding type, because the command-line interface is so different from the Python interface to the user. ### Using `Params` in a `BINDING_FUNCTION()` function @@ -815,7 +815,7 @@ To access a string that a user passed in to the `string` parameter, the following code could be used: ```c++ -const std::string& str = params.Has("string"); +const std::string& str = params.Get("string"); ``` Matrix types are accessed in the same way: @@ -943,7 +943,7 @@ populating them with the correct options for the given binding, then calls `BINDING_FUNCTION()` with those instantiated objects. In order to do this, each parameter and the program documentation must make -themselves known to the IO singleton. This is accomplished by having the @c +themselves known to the IO singleton. This is accomplished by having the `BINDING_USER_NAME()`, `BINDING_SHORT_DESC()`, `BINDING_LONG_DESC()`, `BINDING_EXAMPLE()`, `BINDING_SEE_ALSO()` and `PARAM_*()` macros declare global variables that, in their constructors, register themselves with the `IO` From ac8ff38ad3b158e6c01dfa4282d44b979b641e65 Mon Sep 17 00:00:00 2001 From: Abhimanyu Dayal <92638241+TirelessClock@users.noreply.github.com> Date: Wed, 27 Mar 2024 08:53:16 +0530 Subject: [PATCH 2/4] Update bindings.md --- doc/developer/bindings.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/developer/bindings.md b/doc/developer/bindings.md index 9b35ba90c7..fb885a78f5 100644 --- a/doc/developer/bindings.md +++ b/doc/developer/bindings.md @@ -664,7 +664,7 @@ parameter is required, and whether the parameter is an input or output parameter. Then as arguments to the macros, the name, description, and sometimes the single-character alias and the default value of the parameter. -To give an idea of how these definitions look, the definition +To give a flavor of how these definitions look, the definition ```c++ PARAM_STRING_IN("algorithm", "The algorithm to use: 'svd' or 'blah'.", "a"); @@ -724,7 +724,7 @@ And for input parameters, the parameter may also be required: See the source documentation for each macro to read further details. Note also that each possible combination of `IN`, `OUT`, and `REQ` is not -available - output options cannot be required, and some combinations simply have +available---output options cannot be required, and some combinations simply have not been added because they have not been needed. The `PARAM_MODEL_IN()` and `PARAM_MODEL_OUT()` macros are used to serialize From 4909a41a3e1b2ad0c666b45b26c548a17d8fcc91 Mon Sep 17 00:00:00 2001 From: Abhimanyu Dayal Date: Thu, 28 Mar 2024 23:56:49 +0530 Subject: [PATCH 3/4] gradient boosting binding --- .../methods/grad_boosting/grad_boosting.hpp | 243 ++++++++++++++++++ .../grad_boosting/grad_boosting_impl.hpp | 163 ++++++++++++ .../grad_boosting/grad_boosting_main.cpp | 212 +++++++++++++++ .../grad_boosting/grad_boosting_model.hpp | 125 +++++++++ .../grad_boosting_model_impl.hpp | 132 ++++++++++ 5 files changed, 875 insertions(+) create mode 100644 src/mlpack/methods/grad_boosting/grad_boosting.hpp create mode 100644 src/mlpack/methods/grad_boosting/grad_boosting_impl.hpp create mode 100644 src/mlpack/methods/grad_boosting/grad_boosting_main.cpp create mode 100644 src/mlpack/methods/grad_boosting/grad_boosting_model.hpp create mode 100644 src/mlpack/methods/grad_boosting/grad_boosting_model_impl.hpp diff --git a/src/mlpack/methods/grad_boosting/grad_boosting.hpp b/src/mlpack/methods/grad_boosting/grad_boosting.hpp new file mode 100644 index 0000000000..056c170018 --- /dev/null +++ b/src/mlpack/methods/grad_boosting/grad_boosting.hpp @@ -0,0 +1,243 @@ +/** + * @file methods/grad_boosting/grad_boosting.hpp + * @author Abhimanyu Dayal + * + * Gradient Boosting class. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ + +#ifndef MLPACK_METHODS_GRADBOOSTING_GRADBOOSTING_HPP +#define MLPACK_METHODS_GRADBOOSTING_GRADBOOSTING_HPP + +#include +#include + +namespace mlpack { +/** + * The Gradient Boosting class. Gradient Boosting is a boosting algorithm, meaning that it + * combines an ensemble of weak learners to produce a strong learner. + * + * Gradient Boosting is generally implemented using Decision Trees, or more specifically + * Decision Stumps i.e. weak learner Decision Trees with low depth. + * + * @tparam MatType Data matrix type (i.e. arma::mat or arma::sp_mat). + */ + +template, + typename MatType = arma::mat> +class GradBoosting { + public: + + /** + * Constructor for creating GradBoosting without training. + * Be sure to call Train() before calling Classify() + */ + GradBoosting(); + + /** + * Constructor for a GradBoosting model. Any extra parameters are used as + * hyperparameters for the weak learner. These should be the last arguments + * to the weak learner's constructor or `Train()` function (i.e. anything + * after `numClasses` or `weights`). + * + * @param data Input data. + * @param labels Corresponding labels. + * @param numClasses The number of classes. + * @param num_models Number of weak learners. + * @param weakLearnerParams... Any hyperparameters for the weak learner. + */ + template + GradBoosting( + const MatType& data, + const arma::Row& labels, + const size_t numClasses, + const size_t num_models = 10, + WeakLearnerArgs&&... weakLearnerArgs + ); + + /** + * Constructor takes an already-initialized weak learner; all other + * weak learners will learn with the same parameters as the given + * weak learner. + * + * @param data Input data. + * @param labels Corresponding labels. + * @param numClasses The number of classes. + * @param num_models Number of weak learners. + * @param other Weak learner that has already been initialized. + */ + template + GradBoosting ( + const MatType& data, + const arma::Row& labels, + const size_t numClasses, + const size_t num_models = 10, + const WeakLearnerInType& other, + const typename std::enable_if< + std::is_same::value + >::type* = 0 + ); + + //! Get the number of classes this model is trained on. + size_t NumClasses() const { return numClasses; } + + //! Get the number of weak learners . + size_t NumModels() const { return num_models; } + + //! Get the weights for the given weak learner. + ElemType Alpha(const size_t i) const { return alpha[i]; } + + //! Modify the weight for the given weak learner (be careful!). + ElemType& Alpha(const size_t i) { return alpha[i]; } + + //! Get the given weak learner. + const WeakLearnerType& WeakLearner(const size_t i) const { return wl[i]; } + + //! Modify the given weak learner (be careful!). + WeakLearnerType& WeakLearner(const size_t i) { return wl[i]; } + + /** + * Train GradBoosting on the given dataset. This method takes an initialized + * WeakLearnerType; the parameters for this weak learner will be used to train + * each of the weak learners during GradBoosting training. Note that this will + * completely overwrite any model that has already been trained with this + * object. + * + * Default values are not used for `num_models`; instead, it is used to specify + * the number of weak learners (models) to train during gradient boosting. + * + * @param data Dataset to train on. + * @param labels Labels for each point in the dataset. + * @param numClasses The number of classes. + * @param learner Learner to use for training. + * @param num_models Number of weak learners (models) to train. + */ + template + void Train( + const MatType& data, + const arma::Row& labels, + const size_t numClasses, + const WeakLearnerInType& learner, + const size_t num_models + ); + + /** + * Train Gradient Boosting on the given dataset, using the given parameters. + * The last parameters are the hyperparameters to use for the weak learners; + * these are all the arguments to `WeakLearnerType::Train()` after `numClasses` + * and `weights`. + * + * Default values are not used for `num_models`; instead, it is used to specify + * the number of weak learners (models) to train during gradient boosting. + * + * @param data Dataset to train on. + * @param labels Labels for each point in the dataset. + * @param numClasses The number of classes in the dataset. + * @param num_models Number of boosting rounds. + * @param weakLearnerArgs Hyperparameters to use for each weak learner. + */ + ElemType Train( + const MatType& data, + const arma::Row& labels, + const size_t numClasses, + const size_t num_models, + const typename std::enable_if< + std::is_same::value>::type* = 0 + ); + + template + ElemType Train( + const MatType& data, + const arma::Row& labels, + const size_t numClasses, + const size_t num_models, + WeakLearnerArgs&&... weakLearnerArgs + ); + + /** + * Classify the given test point. + * + * @param point Test point. + */ + template + size_t Classify(const VecType& point) const; + + /** + * Classify the given test point and compute class probabilities. + * + * @param point Test point. + * @param prediction Will be filled with the predicted class of `point`. + * @param probabilities Will be filled with the class probabilities. + */ + template + void Classify(const VecType& point, + size_t& prediction, + arma::Row& probabilities) const; + + /** + * Classify the given test points. + * + * @param test Testing data. + * @param predictedLabels Vector in which the predicted labels of the test + * set will be stored. + */ + void Classify(const MatType& test, + arma::Row& predictedLabels) const; + + /** + * Classify the given test points. + * + * @param test Testing data. + * @param predictedLabels Vector in which the predicted labels of the test + * set will be stored. + * @param probabilities matrix to store the predicted class probabilities for + * each point in the test set. + */ + void Classify(const MatType& test, + arma::Row& predictedLabels, + arma::Mat& probabilities) const; + + + /** + * Serialize the GradBoosting model. + */ + template + void serialize(Archive& ar, const uint32_t /* version */); + + private: + /** + * Internal utility training function. `wl` is not used if + * `UseExistingWeakLearner` is false. `weakLearnerArgs` are not used if + * `UseExistingWeakLearner` is true. + */ + template + ElemType TrainInternal(const MatType& data, + const arma::Row& labels, + const size_t numClasses, + const WeakLearnerType& wl, + WeakLearnerArgs&&... weakLearnerArgs); + + //! The number of classes in the model. + size_t numClasses; + //! The number of weak learners in the model. + size_t num_models; + + //! The vector of weak learners. + std::vector wl; + //! The weights corresponding to each weak learner. + std::vector alpha; +}; + +} + +CEREAL_TEMPLATE_CLASS_VERSION((typename WeakLearnerType, typename MatType), + (mlpack::GradBoosting), (1)); + +// Include implementation. +#include "grad_boosting_impl.hpp" + +#endif diff --git a/src/mlpack/methods/grad_boosting/grad_boosting_impl.hpp b/src/mlpack/methods/grad_boosting/grad_boosting_impl.hpp new file mode 100644 index 0000000000..1a8a672362 --- /dev/null +++ b/src/mlpack/methods/grad_boosting/grad_boosting_impl.hpp @@ -0,0 +1,163 @@ +/* + * @file methods/grad_boosting/grad_boosting_impl.hpp + * @author Abhimanyu Dayal + * + * Implementation of the Gradient Boosting class. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_METHODS_GRADBOOSTING_GRADBOOSTING_IMPL_HPP +#define MLPACK_METHODS_GRADBOOSTING_GRADBOOSTING_IMPL_HPP + +#include "grad_boosting.hpp" + +namespace mlpack { + +// Empty constructor. +template +GradBoosting::GradBoosting() : + numClasses(0), + num_models(0) +{ +// Nothing to do. +} + +/** + * Constructor. + * + * @param data Input data + * @param labels Corresponding labels + * @param num_models Number of weak learners + * @param other Weak Learner, which has been initialized already. + */ +template +template +GradBoosting::GradBoosting( + const MatType& data, + const arma::Row& labels, + const size_t numClasses, + const size_t num_models, + const WeakLearnerInType& other, + const typename std::enable_if< + std::is_same::value>::type*) : + num_models(num_models) +{ + (void) TrainInternal(data, labels, numClasses, other); +} + +/** + * Constructor. + * + * @param data Input data + * @param labels Corresponding labels + * @param num_models Number of weak learners + * @param other Weak Learner, which has been initialized already. + */ +template +template +GradBoosting::GradBoosting( + const MatType& data, + const arma::Row& labels, + const size_t numClasses, + const size_t num_models, + WeakLearnerArgs&&... weakLearnerArgs) : + num_models(num_models) +{ + WeakLearnerType other; // Will not be used. + (void) TrainInternal(data, labels, numClasses, other, + weakLearnerArgs...); +} + +// Train GradBoosting with a given weak learner. +template +template +typename MatType::elem_type GradBoosting::Train( + const MatType& data, + const arma::Row& labels, + const size_t numClasses, + const size_t num_models, + const typename std::enable_if< + std::is_same::value>::type* = 0 + ) +{ + WeakLearnerType other; // Will not be used. + return TrainInternal(data, labels, numClasses, other); +} + +template +template +typename MatType::elem_type GradBoosting::Train( + const MatType& data, + const arma::Row& labels, + const size_t numClasses, + const WeakLearnerInType& learner, + const size_t num_models + ) +{ + return TrainInternal(data, labels, numClasses, learner); +} + + +template +template +typename MatType::elem_type GradBoosting::Train( + const MatType& data, + const arma::Row& labels, + const size_t numClasses, + const size_t num_models, + WeakLearnerArgs&&... weakLearnerArgs + ) +{ + WeakLearnerType other; // Will not be used. + return TrainInternal(data, labels, numClasses, other, + weakLearnerArgs...); +} + +// Classify the given test point. +template +template +size_t GradBoosting::Classify(const VecType& point) const +{ + arma::Row probabilities; + size_t prediction; + Classify(point, prediction, probabilities); + + return prediction; +} + +template +template +void GradientBoosting::Classify( + const VecType& point, + size_t& prediction, + arma::Row& probabilities) const +{ + probabilities.zeros(numClasses); + + // Aggregate predictions of each weak learner. + for (size_t i = 0; i < numModels; ++i) + { + // Predict the residual using the weak learner. + typename MatType::elem_type residual; + wl[i].Predict(point, residual); + + // Add the prediction to the ensemble. + probabilities += residual; + } + + // Convert the ensemble predictions to probabilities. + probabilities -= min(probabilities); + probabilities /= accu(probabilities); + + // Determine the class with maximum probability as the final prediction. + arma::uword maxIndex = 0; + probabilities.max(maxIndex); + prediction = (size_t) maxIndex; +} + + + +} \ No newline at end of file diff --git a/src/mlpack/methods/grad_boosting/grad_boosting_main.cpp b/src/mlpack/methods/grad_boosting/grad_boosting_main.cpp new file mode 100644 index 0000000000..e54956feba --- /dev/null +++ b/src/mlpack/methods/grad_boosting/grad_boosting_main.cpp @@ -0,0 +1,212 @@ +/** + * @file methods/grad_boosting/grad_boosting_main.cpp + * @author Abhimanyu Dayal + * + * A program to implement gradient boosting model. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#include + +#undef BINDING_NAME +#define BINDING_NAME grad_boosting + +#include +#include "grad_boosting.hpp" + +using namespace std; +using namespace mlpack; +using namespace mlpack::data; +using namespace mlpack::util; + +// Program Name. +BINDING_USER_NAME("Gradient Boosting"); + +// Short description. +BINDING_SHORT_DESC ( + "An implementation of Gradient Boosting for classification. " + "A gradient boosting model can be trained and saved; or an " + "existing gradient boosting model can be used for classification on new points." +); + +// Long description. +BINDING_LONG_DESC ( + "This program implements Gradient Boosting algorithm. It uses " + "weak learners (primarily decision stumps), and trains them " + "sequentially, such that future learners are trained to detect the " + "errors (or gradients) of previous learners. The results obtained from " + "all of these learners are subsequently aggregated to give a final result." + "Weak learners (i.e. models which predict nearly random outcomes) " + "are used here because they are individually very fast to train and predict." + "\n\n" + "This program allows training of a Gradient Boosting model, and then application of" + " that model to a test dataset. To train a model, a dataset must be passed" + " with the " + PRINT_PARAM_STRING("training") + " option. Labels can be " + "given with the " + PRINT_PARAM_STRING("labels") + " option; if no labels " + "are specified, the labels will be assumed to be the last column of the " + "input dataset. Alternately, a Gradient Boosting model may be loaded with the " + + PRINT_PARAM_STRING("input_model") + " option." + "\n\n" + "Once a model is trained or loaded, it may be used to provide class " + "predictions for a given test dataset. A test dataset may be specified " + "with the " + PRINT_PARAM_STRING("test") + " parameter. The predicted " + "classes for each point in the test dataset are output to the " + + PRINT_PARAM_STRING("predictions") + " output parameter. The Gradient Boosting " + "model itself is output to the " + PRINT_PARAM_STRING("output_model") + + " output parameter." + "\n\n" +); + +// Example. +BINDING_EXAMPLE( + "For example, to run Gradient Boosting on an input dataset " + + PRINT_DATASET("data") + " with labels " + PRINT_DATASET("labels") + + "storing the trained model in " + PRINT_MODEL("model") + + ", one could use the following command: \n\n" + + PRINT_CALL("Gradient Boosting", "training", "data", "labels", "labels", + "output_model", "model") + "\n\n" + "Similarly, an already-trained model in " + PRINT_MODEL("model") + " can" + " be used to provide class predictions from test data " + + PRINT_DATASET("test_data") + " and store the output in " + + PRINT_DATASET("predictions") + " with the following command: " + "\n\n" + + PRINT_CALL("Gradient Boosting", "input_model", "model", "test", "test_data", + "predictions", "predictions")); + +// See also... +BINDING_SEE_ALSO("Gradient Boosting on Wikipedia", "https://en.wikipedia.org/wiki/" + "Gradient_boosting"); +BINDING_SEE_ALSO("Greedy Function Approximation: A Gradient Boosting Machine", + "https://jerryfriedman.su.domains/ftp/trebst.pdf"); +BINDING_SEE_ALSO("Decision Stump", "#decision_stump"); +BINDING_SEE_ALSO("mlpack::grad_boosting::GradientBoosting C++ class documentation", + "@src/mlpack/methods/grad_boosting/grad_boosting.hpp"); + +// Input for training. +PARAM_MATRIX_IN("training", "Dataset for training Gradient Boosting.", "t"); +PARAM_UROW_IN("labels", "Labels for the training set.", "l"); + +// Classification options. +PARAM_MATRIX_IN("test", "Test dataset.", "T"); +PARAM_UROW_OUT("predictions", "Predicted labels for the test set.", "P"); +PARAM_MATRIX_OUT("probabilities", "Predicted class probabilities for each " + "point in the test set.", "p"); + +// Training parameter options. +PARAM_INT_IN("num_learners", "Number of weak learners to use", "n", 1); + +// Loading/saving of a model. +PARAM_MODEL_IN(GradBoostingModel, "input_model", "Input Gradient Boosting model.", "m"); +PARAM_MODEL_OUT(GradBoostingModel, "output_model", "Output trained Gradient Boosting model.", + "M"); + +void BINDING_FUNCTION(util::Params& params, util::Timers& timers) { + + // Checking the inputs and issuing relevant warnings/errors. + + // The user cannot specify both a training file and an input model file. + RequireOnlyOnePassed(params, { "training", "input_model" }); + + // --labels can't be specified without --training. + ReportIgnoredParam(params, {{ "training", false }}, "labels"); + + // --predictions can't be specified without --test. + ReportIgnoredParam(params, {{ "test", false }}, "predictions"); + + // Training parameters are ignored if no training file is given. + ReportIgnoredParam(params, {{ "training", false }}, "num_learners"); + + // If the user gave an input model but no test set, issue a warning. + if (params.Has("input_model")) { + RequireAtLeastOnePassed(params, { "test" }, false, "no task will be performed"); + } + + // If the user doesn't specify output_model, output or prediction, issue a warning. + RequireAtLeastOnePassed(params, { "output_model", "output", "predictions" }, + false, "no results will be saved"); + + // If training new model. + if (params.Has("training")) { + + mat trainingData = std::move(params.Get("training")); + m = new GradBoostingModel(); + + // Load labels. + arma::Row labelsIn; + if (params.Has("labels")) { + labelsIn = std::move(params.Get>("labels")); + } + + // Extract the labels as the last dimension of the training data. + else { + Log::Info << "Using the last dimension of training set as labels." << endl; + labelsIn = ConvTo>::From( + trainingData.row(trainingData.n_rows - 1) + ); + trainingData.shed_row(trainingData.n_rows - 1); + } + + // Helpers for normalizing the labels. + Row labels; + + // Normalize the labels. + data::NormalizeLabels(labelsIn, labels, m->Mappings()); + + // Get other training parameters. + const int num_learners = params.Get("num_learners"); + + // Count number of classes in labels + const size_t numClasses = m->Mappings().n_elem; + Log::Info << numClasses << " classes in dataset." << endl; + + // Start training + timers.Start("grad_boosting_training"); + m->Train(trainingData, labels, numClasses, iterations, tolerance); + timers.Stop("grad_boosting_training"); + } + + // We have a specified input model. + else { + m = params.Get("input_model"); + } + + // Perform classification on test data, if desired. + if (params.Has("test")) { + mat testingData = std::move(params.Get("test")); + + if (testingData.n_rows != m->Dimensionality()) + Log::Fatal << "Test data dimensionality (" << testingData.n_rows << ") " + << "must be the same as the model dimensionality (" + << m->Dimensionality() << ")!" << endl; + + Row predictedLabels(testingData.n_cols); + mat probabilities; + + if (params.Has("probabilities")) { + timers.Start("grad_boosting_classification"); + m->Classify(testingData, predictedLabels, probabilities); + timers.Stop("grad_boosting_classification"); + } + else { + timers.Start("grad_boosting_classification"); + m->Classify(testingData, predictedLabels); + timers.Stop("grad_boosting_classification"); + } + + Row results; + data::RevertLabels(predictedLabels, m->Mappings(), results); + + // Save the predicted labels. + if (params.Has("output")) + params.Get>("output") = results; + if (params.Has("predictions")) + params.Get>("predictions") = std::move(results); + if (params.Has("probabilities")) + params.Get("probabilities") = std::move(probabilities); + } + + params.Get("output_model") = m; +} diff --git a/src/mlpack/methods/grad_boosting/grad_boosting_model.hpp b/src/mlpack/methods/grad_boosting/grad_boosting_model.hpp new file mode 100644 index 0000000000..654c196b9a --- /dev/null +++ b/src/mlpack/methods/grad_boosting/grad_boosting_model.hpp @@ -0,0 +1,125 @@ +/** + * @file methods/grad_boosting/grad_boosting_model.hpp + * @author Abhimanyu Dayal + * + * A serializable Gradient Boosting model, used by the Gradient Boosting binding. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ + +#ifndef MLPACK_METHODS_GRADBOOSTING_GRADBOOSTING_MODEL_HPP +#define MLPACK_METHODS_GRADBOOSTING_GRADBOOSTING_MODEL_HPP + +#include + +// Use forward declaration instead of include to accelerate compilation. +class GradBoosting; + +namespace mlpack { + +/** + * The model to save to disk. + */ +class GradBoostingModel { + public: + enum WeakLearnerTypes { + DECISION_STUMP + }; + + private: + + //! The mappings for the labels. + arma::Col mappings; + + //! The type of weak learner. + size_t weakLearnerType; + + //! Non-NULL if using decision stumps. + GradBoosting* dsBoost; + + //! Number of dimensions in training data. + size_t dimensionality; + + public: + //! Create an empty AdaBoost model. + GradBoostingModel(); + + //! Create the AdaBoost model with the given mappings and type. + GradBoostingModel(const arma::Col& mappings, + const size_t weakLearnerType); + + //! Copy constructor. + GradBoostingModel(const GradBoostingModel& other); + + //! Move constructor. + GradBoostingModel(GradBoostingModel&& other); + + //! Copy assignment operator. + GradBoostingModel& operator=(const GradBoostingModel& other); + + //! Move assignment operator. + GradBoostingModel& operator=(GradBoostingModel&& other); + + //! Clean up memory. + ~GradBoostingModel(); + + //! Get the mappings. + const arma::Col& Mappings() const { return mappings; } + //! Modify the mappings. + arma::Col& Mappings() { return mappings; } + + //! Get the weak learner type. + size_t WeakLearnerType() const { return weakLearnerType; } + //! Modify the weak learner type. + size_t& WeakLearnerType() { return weakLearnerType; } + + //! Get the dimensionality of the model. + size_t Dimensionality() const { return dimensionality; } + //! Modify the dimensionality of the model. + size_t& Dimensionality() { return dimensionality; } + + //! Train the model, treat the data is all of the numeric type. + void Train( + const arma::mat& data, + const arma::Row& labels, + const size_t numClasses, + const size_t num_models + ); + + //! Classify test points. + void Classify( + const arma::mat& testData, + arma::Row& predictions + ); + + //! Classify test points. + void Classify( + const arma::mat& testData, + arma::Row& predictions, + arma::mat& probabilities + ); + + //! Serialize the model. + template + void serialize(Archive& ar, const uint32_t /* version */) { + if (cereal::is_loading()) { + delete dsBoost; + dsBoost = NULL; + } + + ar(CEREAL_NVP(mappings)); + ar(CEREAL_NVP(weakLearnerType)); + ar(CEREAL_POINTER(dsBoost)); + ar(CEREAL_NVP(dimensionality)); + } +}; + +} + +// Include implementation. +#include "grad_boosting_model_impl.hpp" + +#endif diff --git a/src/mlpack/methods/grad_boosting/grad_boosting_model_impl.hpp b/src/mlpack/methods/grad_boosting/grad_boosting_model_impl.hpp new file mode 100644 index 0000000000..c19f87d365 --- /dev/null +++ b/src/mlpack/methods/grad_boosting/grad_boosting_model_impl.hpp @@ -0,0 +1,132 @@ +/** + * @file methods/grad_boosting/grad_boosting_model_impl.hpp + * @author Abhimanyu Dayal + * + * A serializable Gradient Boosting model, used by the main program. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_METHODS_GRADBOOSTING_GRADBOOSTING_MODEL_IMPL_HPP +#define MLPACK_METHODS_GRADBOOSTING_GRADBOOSTING_MODEL_IMPL_HPP + +#include "grad_boosting.hpp" +#include "grad_boosting_model.hpp" + +namespace mlpack { + +//! Create an empty GradBoosting model. +inline GradBoostingModel::GradBoostingModel() : + weakLearnerType(0), + dsBoost(NULL), + dimensionality(0) +{ + // Nothing to do. +} + +//! Create the GradBoosting model with the given mappings and type. +inline GradBoostingModel::GradBoostingModel( + const arma::Col& mappings, + const size_t weakLearnerType) : + mappings(mappings), + weakLearnerType(weakLearnerType), + dsBoost(NULL), + dimensionality(0) +{ + // Nothing to do. +} + +//! Copy constructor. +inline GradBoostingModel::GradBoostingModel(const GradBoostingModel& other) : + mappings(other.mappings), + weakLearnerType(other.weakLearnerType), + dsBoost(other.dsBoost == nullptr ? nullptr : + new GradBoosting(*other.dsBoost)), + dimensionality(other.dimensionality) +{ + // Nothing to do. +} + +//! Move constructor. +inline GradBoostingModel::GradBoostingModel(GradBoostingModel&& other) : + mappings(std::move(other.mappings)), + weakLearnerType(other.weakLearnerType), + dsBoost(other.dsBoost), + dimensionality(other.dimensionality) +{ + other.weakLearnerType = 0; + other.dsBoost = NULL; + other.dimensionality = 0; +} + +//! Copy assignment operator. +inline GradBoostingModel& GradBoostingModel::operator=(const GradBoostingModel& other) +{ + if (this != &other) + { + mappings = other.mappings; + weakLearnerType = other.weakLearnerType; + + delete dsBoost; + dsBoost = (other.dsBoost == NULL) ? NULL : + new GradBoosting(*other.dsBoost); + + dimensionality = other.dimensionality; + } + return *this; +} + +//! Move assignment operator. +inline GradBoostingModel& GradBoostingModel::operator=(GradBoostingModel&& other) +{ + if (this != &other) + { + mappings = std::move(other.mappings); + weakLearnerType = other.weakLearnerType; + + dsBoost = other.dsBoost; + other.dsBoost = nullptr; + + dimensionality = other.dimensionality; + } + return *this; +} + +inline GradBoostingModel::~GradBoostingModel() +{ + delete dsBoost; + delete pBoost; +} + +//! Train the model. +inline void GradBoostingModel::Train(const arma::mat& data, + const arma::Row& labels, + const size_t numClasses, + const size_t num_models) +{ + dimensionality = data.n_rows; + delete dsBoost; + dsBoost = new GradBoosting(data, labels, numClasses, + num_models); +} + +//! Classify test points. +inline void GradBoostingModel::Classify(const arma::mat& testData, + arma::Row& predictions, + arma::mat& probabilities) +{ + dsBoost->Classify(testData, predictions, probabilities); +} + +//! Classify test points. +inline void GradBoostingModel::Classify(const arma::mat& testData, + arma::Row& predictions) +{ + dsBoost->Classify(testData, predictions); +} + +} // namespace mlpack + +#endif From 82b4f60f4405fc0cea6e002517f4819910e1daaf Mon Sep 17 00:00:00 2001 From: Abhimanyu Dayal <92638241+TirelessClock@users.noreply.github.com> Date: Wed, 27 Mar 2024 08:53:16 +0530 Subject: [PATCH 4/4] documentation changes --- .../methods/grad_boosting/grad_boosting.hpp | 243 ------------------ .../grad_boosting/grad_boosting_impl.hpp | 163 ------------ .../grad_boosting/grad_boosting_main.cpp | 212 --------------- .../grad_boosting/grad_boosting_model.hpp | 125 --------- .../grad_boosting_model_impl.hpp | 132 ---------- 5 files changed, 875 deletions(-) delete mode 100644 src/mlpack/methods/grad_boosting/grad_boosting.hpp delete mode 100644 src/mlpack/methods/grad_boosting/grad_boosting_impl.hpp delete mode 100644 src/mlpack/methods/grad_boosting/grad_boosting_main.cpp delete mode 100644 src/mlpack/methods/grad_boosting/grad_boosting_model.hpp delete mode 100644 src/mlpack/methods/grad_boosting/grad_boosting_model_impl.hpp diff --git a/src/mlpack/methods/grad_boosting/grad_boosting.hpp b/src/mlpack/methods/grad_boosting/grad_boosting.hpp deleted file mode 100644 index 056c170018..0000000000 --- a/src/mlpack/methods/grad_boosting/grad_boosting.hpp +++ /dev/null @@ -1,243 +0,0 @@ -/** - * @file methods/grad_boosting/grad_boosting.hpp - * @author Abhimanyu Dayal - * - * Gradient Boosting class. - * - * mlpack is free software; you may redistribute it and/or modify it under the - * terms of the 3-clause BSD license. You should have received a copy of the - * 3-clause BSD license along with mlpack. If not, see - * http://www.opensource.org/licenses/BSD-3-Clause for more information. - */ - -#ifndef MLPACK_METHODS_GRADBOOSTING_GRADBOOSTING_HPP -#define MLPACK_METHODS_GRADBOOSTING_GRADBOOSTING_HPP - -#include -#include - -namespace mlpack { -/** - * The Gradient Boosting class. Gradient Boosting is a boosting algorithm, meaning that it - * combines an ensemble of weak learners to produce a strong learner. - * - * Gradient Boosting is generally implemented using Decision Trees, or more specifically - * Decision Stumps i.e. weak learner Decision Trees with low depth. - * - * @tparam MatType Data matrix type (i.e. arma::mat or arma::sp_mat). - */ - -template, - typename MatType = arma::mat> -class GradBoosting { - public: - - /** - * Constructor for creating GradBoosting without training. - * Be sure to call Train() before calling Classify() - */ - GradBoosting(); - - /** - * Constructor for a GradBoosting model. Any extra parameters are used as - * hyperparameters for the weak learner. These should be the last arguments - * to the weak learner's constructor or `Train()` function (i.e. anything - * after `numClasses` or `weights`). - * - * @param data Input data. - * @param labels Corresponding labels. - * @param numClasses The number of classes. - * @param num_models Number of weak learners. - * @param weakLearnerParams... Any hyperparameters for the weak learner. - */ - template - GradBoosting( - const MatType& data, - const arma::Row& labels, - const size_t numClasses, - const size_t num_models = 10, - WeakLearnerArgs&&... weakLearnerArgs - ); - - /** - * Constructor takes an already-initialized weak learner; all other - * weak learners will learn with the same parameters as the given - * weak learner. - * - * @param data Input data. - * @param labels Corresponding labels. - * @param numClasses The number of classes. - * @param num_models Number of weak learners. - * @param other Weak learner that has already been initialized. - */ - template - GradBoosting ( - const MatType& data, - const arma::Row& labels, - const size_t numClasses, - const size_t num_models = 10, - const WeakLearnerInType& other, - const typename std::enable_if< - std::is_same::value - >::type* = 0 - ); - - //! Get the number of classes this model is trained on. - size_t NumClasses() const { return numClasses; } - - //! Get the number of weak learners . - size_t NumModels() const { return num_models; } - - //! Get the weights for the given weak learner. - ElemType Alpha(const size_t i) const { return alpha[i]; } - - //! Modify the weight for the given weak learner (be careful!). - ElemType& Alpha(const size_t i) { return alpha[i]; } - - //! Get the given weak learner. - const WeakLearnerType& WeakLearner(const size_t i) const { return wl[i]; } - - //! Modify the given weak learner (be careful!). - WeakLearnerType& WeakLearner(const size_t i) { return wl[i]; } - - /** - * Train GradBoosting on the given dataset. This method takes an initialized - * WeakLearnerType; the parameters for this weak learner will be used to train - * each of the weak learners during GradBoosting training. Note that this will - * completely overwrite any model that has already been trained with this - * object. - * - * Default values are not used for `num_models`; instead, it is used to specify - * the number of weak learners (models) to train during gradient boosting. - * - * @param data Dataset to train on. - * @param labels Labels for each point in the dataset. - * @param numClasses The number of classes. - * @param learner Learner to use for training. - * @param num_models Number of weak learners (models) to train. - */ - template - void Train( - const MatType& data, - const arma::Row& labels, - const size_t numClasses, - const WeakLearnerInType& learner, - const size_t num_models - ); - - /** - * Train Gradient Boosting on the given dataset, using the given parameters. - * The last parameters are the hyperparameters to use for the weak learners; - * these are all the arguments to `WeakLearnerType::Train()` after `numClasses` - * and `weights`. - * - * Default values are not used for `num_models`; instead, it is used to specify - * the number of weak learners (models) to train during gradient boosting. - * - * @param data Dataset to train on. - * @param labels Labels for each point in the dataset. - * @param numClasses The number of classes in the dataset. - * @param num_models Number of boosting rounds. - * @param weakLearnerArgs Hyperparameters to use for each weak learner. - */ - ElemType Train( - const MatType& data, - const arma::Row& labels, - const size_t numClasses, - const size_t num_models, - const typename std::enable_if< - std::is_same::value>::type* = 0 - ); - - template - ElemType Train( - const MatType& data, - const arma::Row& labels, - const size_t numClasses, - const size_t num_models, - WeakLearnerArgs&&... weakLearnerArgs - ); - - /** - * Classify the given test point. - * - * @param point Test point. - */ - template - size_t Classify(const VecType& point) const; - - /** - * Classify the given test point and compute class probabilities. - * - * @param point Test point. - * @param prediction Will be filled with the predicted class of `point`. - * @param probabilities Will be filled with the class probabilities. - */ - template - void Classify(const VecType& point, - size_t& prediction, - arma::Row& probabilities) const; - - /** - * Classify the given test points. - * - * @param test Testing data. - * @param predictedLabels Vector in which the predicted labels of the test - * set will be stored. - */ - void Classify(const MatType& test, - arma::Row& predictedLabels) const; - - /** - * Classify the given test points. - * - * @param test Testing data. - * @param predictedLabels Vector in which the predicted labels of the test - * set will be stored. - * @param probabilities matrix to store the predicted class probabilities for - * each point in the test set. - */ - void Classify(const MatType& test, - arma::Row& predictedLabels, - arma::Mat& probabilities) const; - - - /** - * Serialize the GradBoosting model. - */ - template - void serialize(Archive& ar, const uint32_t /* version */); - - private: - /** - * Internal utility training function. `wl` is not used if - * `UseExistingWeakLearner` is false. `weakLearnerArgs` are not used if - * `UseExistingWeakLearner` is true. - */ - template - ElemType TrainInternal(const MatType& data, - const arma::Row& labels, - const size_t numClasses, - const WeakLearnerType& wl, - WeakLearnerArgs&&... weakLearnerArgs); - - //! The number of classes in the model. - size_t numClasses; - //! The number of weak learners in the model. - size_t num_models; - - //! The vector of weak learners. - std::vector wl; - //! The weights corresponding to each weak learner. - std::vector alpha; -}; - -} - -CEREAL_TEMPLATE_CLASS_VERSION((typename WeakLearnerType, typename MatType), - (mlpack::GradBoosting), (1)); - -// Include implementation. -#include "grad_boosting_impl.hpp" - -#endif diff --git a/src/mlpack/methods/grad_boosting/grad_boosting_impl.hpp b/src/mlpack/methods/grad_boosting/grad_boosting_impl.hpp deleted file mode 100644 index 1a8a672362..0000000000 --- a/src/mlpack/methods/grad_boosting/grad_boosting_impl.hpp +++ /dev/null @@ -1,163 +0,0 @@ -/* - * @file methods/grad_boosting/grad_boosting_impl.hpp - * @author Abhimanyu Dayal - * - * Implementation of the Gradient Boosting class. - * - * mlpack is free software; you may redistribute it and/or modify it under the - * terms of the 3-clause BSD license. You should have received a copy of the - * 3-clause BSD license along with mlpack. If not, see - * http://www.opensource.org/licenses/BSD-3-Clause for more information. - */ -#ifndef MLPACK_METHODS_GRADBOOSTING_GRADBOOSTING_IMPL_HPP -#define MLPACK_METHODS_GRADBOOSTING_GRADBOOSTING_IMPL_HPP - -#include "grad_boosting.hpp" - -namespace mlpack { - -// Empty constructor. -template -GradBoosting::GradBoosting() : - numClasses(0), - num_models(0) -{ -// Nothing to do. -} - -/** - * Constructor. - * - * @param data Input data - * @param labels Corresponding labels - * @param num_models Number of weak learners - * @param other Weak Learner, which has been initialized already. - */ -template -template -GradBoosting::GradBoosting( - const MatType& data, - const arma::Row& labels, - const size_t numClasses, - const size_t num_models, - const WeakLearnerInType& other, - const typename std::enable_if< - std::is_same::value>::type*) : - num_models(num_models) -{ - (void) TrainInternal(data, labels, numClasses, other); -} - -/** - * Constructor. - * - * @param data Input data - * @param labels Corresponding labels - * @param num_models Number of weak learners - * @param other Weak Learner, which has been initialized already. - */ -template -template -GradBoosting::GradBoosting( - const MatType& data, - const arma::Row& labels, - const size_t numClasses, - const size_t num_models, - WeakLearnerArgs&&... weakLearnerArgs) : - num_models(num_models) -{ - WeakLearnerType other; // Will not be used. - (void) TrainInternal(data, labels, numClasses, other, - weakLearnerArgs...); -} - -// Train GradBoosting with a given weak learner. -template -template -typename MatType::elem_type GradBoosting::Train( - const MatType& data, - const arma::Row& labels, - const size_t numClasses, - const size_t num_models, - const typename std::enable_if< - std::is_same::value>::type* = 0 - ) -{ - WeakLearnerType other; // Will not be used. - return TrainInternal(data, labels, numClasses, other); -} - -template -template -typename MatType::elem_type GradBoosting::Train( - const MatType& data, - const arma::Row& labels, - const size_t numClasses, - const WeakLearnerInType& learner, - const size_t num_models - ) -{ - return TrainInternal(data, labels, numClasses, learner); -} - - -template -template -typename MatType::elem_type GradBoosting::Train( - const MatType& data, - const arma::Row& labels, - const size_t numClasses, - const size_t num_models, - WeakLearnerArgs&&... weakLearnerArgs - ) -{ - WeakLearnerType other; // Will not be used. - return TrainInternal(data, labels, numClasses, other, - weakLearnerArgs...); -} - -// Classify the given test point. -template -template -size_t GradBoosting::Classify(const VecType& point) const -{ - arma::Row probabilities; - size_t prediction; - Classify(point, prediction, probabilities); - - return prediction; -} - -template -template -void GradientBoosting::Classify( - const VecType& point, - size_t& prediction, - arma::Row& probabilities) const -{ - probabilities.zeros(numClasses); - - // Aggregate predictions of each weak learner. - for (size_t i = 0; i < numModels; ++i) - { - // Predict the residual using the weak learner. - typename MatType::elem_type residual; - wl[i].Predict(point, residual); - - // Add the prediction to the ensemble. - probabilities += residual; - } - - // Convert the ensemble predictions to probabilities. - probabilities -= min(probabilities); - probabilities /= accu(probabilities); - - // Determine the class with maximum probability as the final prediction. - arma::uword maxIndex = 0; - probabilities.max(maxIndex); - prediction = (size_t) maxIndex; -} - - - -} \ No newline at end of file diff --git a/src/mlpack/methods/grad_boosting/grad_boosting_main.cpp b/src/mlpack/methods/grad_boosting/grad_boosting_main.cpp deleted file mode 100644 index e54956feba..0000000000 --- a/src/mlpack/methods/grad_boosting/grad_boosting_main.cpp +++ /dev/null @@ -1,212 +0,0 @@ -/** - * @file methods/grad_boosting/grad_boosting_main.cpp - * @author Abhimanyu Dayal - * - * A program to implement gradient boosting model. - * - * mlpack is free software; you may redistribute it and/or modify it under the - * terms of the 3-clause BSD license. You should have received a copy of the - * 3-clause BSD license along with mlpack. If not, see - * http://www.opensource.org/licenses/BSD-3-Clause for more information. - */ -#include - -#undef BINDING_NAME -#define BINDING_NAME grad_boosting - -#include -#include "grad_boosting.hpp" - -using namespace std; -using namespace mlpack; -using namespace mlpack::data; -using namespace mlpack::util; - -// Program Name. -BINDING_USER_NAME("Gradient Boosting"); - -// Short description. -BINDING_SHORT_DESC ( - "An implementation of Gradient Boosting for classification. " - "A gradient boosting model can be trained and saved; or an " - "existing gradient boosting model can be used for classification on new points." -); - -// Long description. -BINDING_LONG_DESC ( - "This program implements Gradient Boosting algorithm. It uses " - "weak learners (primarily decision stumps), and trains them " - "sequentially, such that future learners are trained to detect the " - "errors (or gradients) of previous learners. The results obtained from " - "all of these learners are subsequently aggregated to give a final result." - "Weak learners (i.e. models which predict nearly random outcomes) " - "are used here because they are individually very fast to train and predict." - "\n\n" - "This program allows training of a Gradient Boosting model, and then application of" - " that model to a test dataset. To train a model, a dataset must be passed" - " with the " + PRINT_PARAM_STRING("training") + " option. Labels can be " - "given with the " + PRINT_PARAM_STRING("labels") + " option; if no labels " - "are specified, the labels will be assumed to be the last column of the " - "input dataset. Alternately, a Gradient Boosting model may be loaded with the " + - PRINT_PARAM_STRING("input_model") + " option." - "\n\n" - "Once a model is trained or loaded, it may be used to provide class " - "predictions for a given test dataset. A test dataset may be specified " - "with the " + PRINT_PARAM_STRING("test") + " parameter. The predicted " - "classes for each point in the test dataset are output to the " + - PRINT_PARAM_STRING("predictions") + " output parameter. The Gradient Boosting " - "model itself is output to the " + PRINT_PARAM_STRING("output_model") + - " output parameter." - "\n\n" -); - -// Example. -BINDING_EXAMPLE( - "For example, to run Gradient Boosting on an input dataset " + - PRINT_DATASET("data") + " with labels " + PRINT_DATASET("labels") + - "storing the trained model in " + PRINT_MODEL("model") + - ", one could use the following command: \n\n" + - PRINT_CALL("Gradient Boosting", "training", "data", "labels", "labels", - "output_model", "model") + "\n\n" - "Similarly, an already-trained model in " + PRINT_MODEL("model") + " can" - " be used to provide class predictions from test data " + - PRINT_DATASET("test_data") + " and store the output in " + - PRINT_DATASET("predictions") + " with the following command: " - "\n\n" + - PRINT_CALL("Gradient Boosting", "input_model", "model", "test", "test_data", - "predictions", "predictions")); - -// See also... -BINDING_SEE_ALSO("Gradient Boosting on Wikipedia", "https://en.wikipedia.org/wiki/" - "Gradient_boosting"); -BINDING_SEE_ALSO("Greedy Function Approximation: A Gradient Boosting Machine", - "https://jerryfriedman.su.domains/ftp/trebst.pdf"); -BINDING_SEE_ALSO("Decision Stump", "#decision_stump"); -BINDING_SEE_ALSO("mlpack::grad_boosting::GradientBoosting C++ class documentation", - "@src/mlpack/methods/grad_boosting/grad_boosting.hpp"); - -// Input for training. -PARAM_MATRIX_IN("training", "Dataset for training Gradient Boosting.", "t"); -PARAM_UROW_IN("labels", "Labels for the training set.", "l"); - -// Classification options. -PARAM_MATRIX_IN("test", "Test dataset.", "T"); -PARAM_UROW_OUT("predictions", "Predicted labels for the test set.", "P"); -PARAM_MATRIX_OUT("probabilities", "Predicted class probabilities for each " - "point in the test set.", "p"); - -// Training parameter options. -PARAM_INT_IN("num_learners", "Number of weak learners to use", "n", 1); - -// Loading/saving of a model. -PARAM_MODEL_IN(GradBoostingModel, "input_model", "Input Gradient Boosting model.", "m"); -PARAM_MODEL_OUT(GradBoostingModel, "output_model", "Output trained Gradient Boosting model.", - "M"); - -void BINDING_FUNCTION(util::Params& params, util::Timers& timers) { - - // Checking the inputs and issuing relevant warnings/errors. - - // The user cannot specify both a training file and an input model file. - RequireOnlyOnePassed(params, { "training", "input_model" }); - - // --labels can't be specified without --training. - ReportIgnoredParam(params, {{ "training", false }}, "labels"); - - // --predictions can't be specified without --test. - ReportIgnoredParam(params, {{ "test", false }}, "predictions"); - - // Training parameters are ignored if no training file is given. - ReportIgnoredParam(params, {{ "training", false }}, "num_learners"); - - // If the user gave an input model but no test set, issue a warning. - if (params.Has("input_model")) { - RequireAtLeastOnePassed(params, { "test" }, false, "no task will be performed"); - } - - // If the user doesn't specify output_model, output or prediction, issue a warning. - RequireAtLeastOnePassed(params, { "output_model", "output", "predictions" }, - false, "no results will be saved"); - - // If training new model. - if (params.Has("training")) { - - mat trainingData = std::move(params.Get("training")); - m = new GradBoostingModel(); - - // Load labels. - arma::Row labelsIn; - if (params.Has("labels")) { - labelsIn = std::move(params.Get>("labels")); - } - - // Extract the labels as the last dimension of the training data. - else { - Log::Info << "Using the last dimension of training set as labels." << endl; - labelsIn = ConvTo>::From( - trainingData.row(trainingData.n_rows - 1) - ); - trainingData.shed_row(trainingData.n_rows - 1); - } - - // Helpers for normalizing the labels. - Row labels; - - // Normalize the labels. - data::NormalizeLabels(labelsIn, labels, m->Mappings()); - - // Get other training parameters. - const int num_learners = params.Get("num_learners"); - - // Count number of classes in labels - const size_t numClasses = m->Mappings().n_elem; - Log::Info << numClasses << " classes in dataset." << endl; - - // Start training - timers.Start("grad_boosting_training"); - m->Train(trainingData, labels, numClasses, iterations, tolerance); - timers.Stop("grad_boosting_training"); - } - - // We have a specified input model. - else { - m = params.Get("input_model"); - } - - // Perform classification on test data, if desired. - if (params.Has("test")) { - mat testingData = std::move(params.Get("test")); - - if (testingData.n_rows != m->Dimensionality()) - Log::Fatal << "Test data dimensionality (" << testingData.n_rows << ") " - << "must be the same as the model dimensionality (" - << m->Dimensionality() << ")!" << endl; - - Row predictedLabels(testingData.n_cols); - mat probabilities; - - if (params.Has("probabilities")) { - timers.Start("grad_boosting_classification"); - m->Classify(testingData, predictedLabels, probabilities); - timers.Stop("grad_boosting_classification"); - } - else { - timers.Start("grad_boosting_classification"); - m->Classify(testingData, predictedLabels); - timers.Stop("grad_boosting_classification"); - } - - Row results; - data::RevertLabels(predictedLabels, m->Mappings(), results); - - // Save the predicted labels. - if (params.Has("output")) - params.Get>("output") = results; - if (params.Has("predictions")) - params.Get>("predictions") = std::move(results); - if (params.Has("probabilities")) - params.Get("probabilities") = std::move(probabilities); - } - - params.Get("output_model") = m; -} diff --git a/src/mlpack/methods/grad_boosting/grad_boosting_model.hpp b/src/mlpack/methods/grad_boosting/grad_boosting_model.hpp deleted file mode 100644 index 654c196b9a..0000000000 --- a/src/mlpack/methods/grad_boosting/grad_boosting_model.hpp +++ /dev/null @@ -1,125 +0,0 @@ -/** - * @file methods/grad_boosting/grad_boosting_model.hpp - * @author Abhimanyu Dayal - * - * A serializable Gradient Boosting model, used by the Gradient Boosting binding. - * - * mlpack is free software; you may redistribute it and/or modify it under the - * terms of the 3-clause BSD license. You should have received a copy of the - * 3-clause BSD license along with mlpack. If not, see - * http://www.opensource.org/licenses/BSD-3-Clause for more information. - */ - -#ifndef MLPACK_METHODS_GRADBOOSTING_GRADBOOSTING_MODEL_HPP -#define MLPACK_METHODS_GRADBOOSTING_GRADBOOSTING_MODEL_HPP - -#include - -// Use forward declaration instead of include to accelerate compilation. -class GradBoosting; - -namespace mlpack { - -/** - * The model to save to disk. - */ -class GradBoostingModel { - public: - enum WeakLearnerTypes { - DECISION_STUMP - }; - - private: - - //! The mappings for the labels. - arma::Col mappings; - - //! The type of weak learner. - size_t weakLearnerType; - - //! Non-NULL if using decision stumps. - GradBoosting* dsBoost; - - //! Number of dimensions in training data. - size_t dimensionality; - - public: - //! Create an empty AdaBoost model. - GradBoostingModel(); - - //! Create the AdaBoost model with the given mappings and type. - GradBoostingModel(const arma::Col& mappings, - const size_t weakLearnerType); - - //! Copy constructor. - GradBoostingModel(const GradBoostingModel& other); - - //! Move constructor. - GradBoostingModel(GradBoostingModel&& other); - - //! Copy assignment operator. - GradBoostingModel& operator=(const GradBoostingModel& other); - - //! Move assignment operator. - GradBoostingModel& operator=(GradBoostingModel&& other); - - //! Clean up memory. - ~GradBoostingModel(); - - //! Get the mappings. - const arma::Col& Mappings() const { return mappings; } - //! Modify the mappings. - arma::Col& Mappings() { return mappings; } - - //! Get the weak learner type. - size_t WeakLearnerType() const { return weakLearnerType; } - //! Modify the weak learner type. - size_t& WeakLearnerType() { return weakLearnerType; } - - //! Get the dimensionality of the model. - size_t Dimensionality() const { return dimensionality; } - //! Modify the dimensionality of the model. - size_t& Dimensionality() { return dimensionality; } - - //! Train the model, treat the data is all of the numeric type. - void Train( - const arma::mat& data, - const arma::Row& labels, - const size_t numClasses, - const size_t num_models - ); - - //! Classify test points. - void Classify( - const arma::mat& testData, - arma::Row& predictions - ); - - //! Classify test points. - void Classify( - const arma::mat& testData, - arma::Row& predictions, - arma::mat& probabilities - ); - - //! Serialize the model. - template - void serialize(Archive& ar, const uint32_t /* version */) { - if (cereal::is_loading()) { - delete dsBoost; - dsBoost = NULL; - } - - ar(CEREAL_NVP(mappings)); - ar(CEREAL_NVP(weakLearnerType)); - ar(CEREAL_POINTER(dsBoost)); - ar(CEREAL_NVP(dimensionality)); - } -}; - -} - -// Include implementation. -#include "grad_boosting_model_impl.hpp" - -#endif diff --git a/src/mlpack/methods/grad_boosting/grad_boosting_model_impl.hpp b/src/mlpack/methods/grad_boosting/grad_boosting_model_impl.hpp deleted file mode 100644 index c19f87d365..0000000000 --- a/src/mlpack/methods/grad_boosting/grad_boosting_model_impl.hpp +++ /dev/null @@ -1,132 +0,0 @@ -/** - * @file methods/grad_boosting/grad_boosting_model_impl.hpp - * @author Abhimanyu Dayal - * - * A serializable Gradient Boosting model, used by the main program. - * - * mlpack is free software; you may redistribute it and/or modify it under the - * terms of the 3-clause BSD license. You should have received a copy of the - * 3-clause BSD license along with mlpack. If not, see - * http://www.opensource.org/licenses/BSD-3-Clause for more information. - */ -#ifndef MLPACK_METHODS_GRADBOOSTING_GRADBOOSTING_MODEL_IMPL_HPP -#define MLPACK_METHODS_GRADBOOSTING_GRADBOOSTING_MODEL_IMPL_HPP - -#include "grad_boosting.hpp" -#include "grad_boosting_model.hpp" - -namespace mlpack { - -//! Create an empty GradBoosting model. -inline GradBoostingModel::GradBoostingModel() : - weakLearnerType(0), - dsBoost(NULL), - dimensionality(0) -{ - // Nothing to do. -} - -//! Create the GradBoosting model with the given mappings and type. -inline GradBoostingModel::GradBoostingModel( - const arma::Col& mappings, - const size_t weakLearnerType) : - mappings(mappings), - weakLearnerType(weakLearnerType), - dsBoost(NULL), - dimensionality(0) -{ - // Nothing to do. -} - -//! Copy constructor. -inline GradBoostingModel::GradBoostingModel(const GradBoostingModel& other) : - mappings(other.mappings), - weakLearnerType(other.weakLearnerType), - dsBoost(other.dsBoost == nullptr ? nullptr : - new GradBoosting(*other.dsBoost)), - dimensionality(other.dimensionality) -{ - // Nothing to do. -} - -//! Move constructor. -inline GradBoostingModel::GradBoostingModel(GradBoostingModel&& other) : - mappings(std::move(other.mappings)), - weakLearnerType(other.weakLearnerType), - dsBoost(other.dsBoost), - dimensionality(other.dimensionality) -{ - other.weakLearnerType = 0; - other.dsBoost = NULL; - other.dimensionality = 0; -} - -//! Copy assignment operator. -inline GradBoostingModel& GradBoostingModel::operator=(const GradBoostingModel& other) -{ - if (this != &other) - { - mappings = other.mappings; - weakLearnerType = other.weakLearnerType; - - delete dsBoost; - dsBoost = (other.dsBoost == NULL) ? NULL : - new GradBoosting(*other.dsBoost); - - dimensionality = other.dimensionality; - } - return *this; -} - -//! Move assignment operator. -inline GradBoostingModel& GradBoostingModel::operator=(GradBoostingModel&& other) -{ - if (this != &other) - { - mappings = std::move(other.mappings); - weakLearnerType = other.weakLearnerType; - - dsBoost = other.dsBoost; - other.dsBoost = nullptr; - - dimensionality = other.dimensionality; - } - return *this; -} - -inline GradBoostingModel::~GradBoostingModel() -{ - delete dsBoost; - delete pBoost; -} - -//! Train the model. -inline void GradBoostingModel::Train(const arma::mat& data, - const arma::Row& labels, - const size_t numClasses, - const size_t num_models) -{ - dimensionality = data.n_rows; - delete dsBoost; - dsBoost = new GradBoosting(data, labels, numClasses, - num_models); -} - -//! Classify test points. -inline void GradBoostingModel::Classify(const arma::mat& testData, - arma::Row& predictions, - arma::mat& probabilities) -{ - dsBoost->Classify(testData, predictions, probabilities); -} - -//! Classify test points. -inline void GradBoostingModel::Classify(const arma::mat& testData, - arma::Row& predictions) -{ - dsBoost->Classify(testData, predictions); -} - -} // namespace mlpack - -#endif