From 5fa02af32eafca1f75fd50d3d7f3f50748f1b39c Mon Sep 17 00:00:00 2001 From: Marcus Edel Date: Mon, 20 Mar 2017 23:01:37 +0100 Subject: [PATCH 01/84] Add update and decay policy to the minibatch sgd class. --- .../decay_policies/CMakeLists.txt | 10 +++ .../minibatch_sgd/decay_policies/no_decay.hpp | 54 ++++++++++++++++ .../minibatch_sgd/minibatch_sgd.hpp | 38 +++++++++-- .../minibatch_sgd/minibatch_sgd_impl.hpp | 63 +++++++++++++------ 4 files changed, 142 insertions(+), 23 deletions(-) create mode 100644 src/mlpack/core/optimizers/minibatch_sgd/decay_policies/CMakeLists.txt create mode 100644 src/mlpack/core/optimizers/minibatch_sgd/decay_policies/no_decay.hpp diff --git a/src/mlpack/core/optimizers/minibatch_sgd/decay_policies/CMakeLists.txt b/src/mlpack/core/optimizers/minibatch_sgd/decay_policies/CMakeLists.txt new file mode 100644 index 0000000000..740bdf77ce --- /dev/null +++ b/src/mlpack/core/optimizers/minibatch_sgd/decay_policies/CMakeLists.txt @@ -0,0 +1,10 @@ +set(SOURCES + no_decay.hpp +) + +set(DIR_SRCS) +foreach(file ${SOURCES}) + set(DIR_SRCS ${DIR_SRCS} ${CMAKE_CURRENT_SOURCE_DIR}/${file}) +endforeach() + +set(MLPACK_SRCS ${MLPACK_SRCS} ${DIR_SRCS} PARENT_SCOPE) diff --git a/src/mlpack/core/optimizers/minibatch_sgd/decay_policies/no_decay.hpp b/src/mlpack/core/optimizers/minibatch_sgd/decay_policies/no_decay.hpp new file mode 100644 index 0000000000..9bfdb90900 --- /dev/null +++ b/src/mlpack/core/optimizers/minibatch_sgd/decay_policies/no_decay.hpp @@ -0,0 +1,54 @@ +/** + * @file no_decay.hpp + * @author Marcus Edel + * + * Definition of the policy type for the decay class. + * + * You should define your own decay update that looks like NoDecay. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ + +#ifndef MLPACK_CORE_OPTIMIZERS_MINIBATCH_SGD_DECAY_POLICIES_NO_DECAY_HPP +#define MLPACK_CORE_OPTIMIZERS_MINIBATCH_SGD_DECAY_POLICIES_NO_DECAY_HPP + +namespace mlpack { +namespace optimization { + +/** + * Definition of the NoDecay class. Use this as a template for your own. + */ +class NoDecay +{ + public: + /** + * This constructor is called before the first iteration. + * + * @param node Node which this corresponds to. + */ + NoDecay() { } + + /** + * This function is called in each iteration after the policy update. + * + * @param stepSize The stepSize to be adjusted. + * @param epoch The current epoch. + * @param batch The current batch. + * @param iterate Function parameters. + */ + void Update(double& /* stepSize */, + const size_t /* epoch */, + const size_t /* batch */, + const arma::mat& /* iterate */) + { + // Nothing to do here. + } +}; + +} // namespace optimization +} // namespace mlpack + +#endif // MLPACK_CORE_OPTIMIZERS_MINIBATCH_SGD_DECAY_POLICIES_NO_DECAY_HPP \ No newline at end of file diff --git a/src/mlpack/core/optimizers/minibatch_sgd/minibatch_sgd.hpp b/src/mlpack/core/optimizers/minibatch_sgd/minibatch_sgd.hpp index e562b4c6fd..9cb367e65e 100644 --- a/src/mlpack/core/optimizers/minibatch_sgd/minibatch_sgd.hpp +++ b/src/mlpack/core/optimizers/minibatch_sgd/minibatch_sgd.hpp @@ -13,6 +13,8 @@ #define MLPACK_CORE_OPTIMIZERS_MINIBATCH_SGD_MINIBATCH_SGD_HPP #include +#include +#include namespace mlpack { namespace optimization { @@ -69,9 +71,22 @@ namespace optimization { * * @tparam DecomposableFunctionType Decomposable objective function type to be * minimized. + * @tparam update Update policy used during the iterative update process. + * By default the vanilla update policy + * (see mlpack::optimization::VanillaUpdate) is used. + * @tparam UpdatePolicyType Update policy used during the iterative update + * process. By default the vanilla update policy + * (see mlpack::optimization::VanillaUpdate) is used. + * @tparam DecayPolicyType Decay policy used during the iterative update + * process to adjust the step size. By default the step size isn't going to + * be adjusted. */ -template -class MiniBatchSGD +template< + typename DecomposableFunctionType, + typename UpdatePolicyType = VanillaUpdate, + typename DecayPolicyType = NoDecay +> +class MiniBatchSGDType { public: /** @@ -89,13 +104,18 @@ class MiniBatchSGD * @param tolerance Maximum absolute tolerance to terminate algorithm. * @param shuffle If true, the mini-batch order is shuffled; otherwise, each * mini-batch is visited in linear order. + * @param updatePolicy Instantiated update policy used to adjust the given + * parameters. + * @param decayPolicy Instantiated decay policy used to adjust the step size. */ - MiniBatchSGD(DecomposableFunctionType& function, + MiniBatchSGDType(DecomposableFunctionType& function, const size_t batchSize = 1000, const double stepSize = 0.01, const size_t maxIterations = 100000, const double tolerance = 1e-5, - const bool shuffle = true); + const bool shuffle = true, + const UpdatePolicyType& updatePolicy = UpdatePolicyType(), + const DecayPolicyType& decayPolicy = DecayPolicyType()); /** * Optimize the given function using mini-batch SGD. The given starting point @@ -156,8 +176,18 @@ class MiniBatchSGD //! Controls whether or not the individual functions are shuffled when //! iterating. bool shuffle; + + //! The update policy used to update the parameters in each iteration. + UpdatePolicyType updatePolicy; + + //! The decay policy used to update the parameters in each iteration. + DecayPolicyType decayPolicy; }; +template +using MiniBatchSGD = MiniBatchSGDType< + DecomposableFunctionType, VanillaUpdate, NoDecay>; + } // namespace optimization } // namespace mlpack diff --git a/src/mlpack/core/optimizers/minibatch_sgd/minibatch_sgd_impl.hpp b/src/mlpack/core/optimizers/minibatch_sgd/minibatch_sgd_impl.hpp index 5b95b415d3..6f0a696dee 100644 --- a/src/mlpack/core/optimizers/minibatch_sgd/minibatch_sgd_impl.hpp +++ b/src/mlpack/core/optimizers/minibatch_sgd/minibatch_sgd_impl.hpp @@ -18,25 +18,44 @@ namespace mlpack { namespace optimization { -template -MiniBatchSGD::MiniBatchSGD( - DecomposableFunctionType& function, - const size_t batchSize, - const double stepSize, - const size_t maxIterations, - const double tolerance, - const bool shuffle) : - function(function), - batchSize(batchSize), - stepSize(stepSize), - maxIterations(maxIterations), - tolerance(tolerance), - shuffle(shuffle) +template< + typename DecomposableFunctionType, + typename UpdatePolicyType, + typename DecayPolicyType +> +MiniBatchSGDType< + DecomposableFunctionType, + UpdatePolicyType, + DecayPolicyType +>::MiniBatchSGDType(DecomposableFunctionType& function, + const size_t batchSize, + const double stepSize, + const size_t maxIterations, + const double tolerance, + const bool shuffle, + const UpdatePolicyType& updatePolicy, + const DecayPolicyType& decayPolicy) : + function(function), + batchSize(batchSize), + stepSize(stepSize), + maxIterations(maxIterations), + tolerance(tolerance), + shuffle(shuffle), + updatePolicy(updatePolicy), + decayPolicy(decayPolicy) { /* Nothing to do. */ } //! Optimize the function (minimize). -template -double MiniBatchSGD::Optimize(arma::mat& iterate) +template< + typename DecomposableFunctionType, + typename UpdatePolicyType, + typename DecayPolicyType +> +double MiniBatchSGDType< + DecomposableFunctionType, + UpdatePolicyType, + DecayPolicyType +>::Optimize(arma::mat& iterate) { // Find the number of functions. const size_t numFunctions = function.NumFunctions(); @@ -59,6 +78,9 @@ double MiniBatchSGD::Optimize(arma::mat& iterate) for (size_t i = 0; i < numFunctions; ++i) overallObjective += function.Evaluate(iterate, i); + // Initialize the update policy. + updatePolicy.Initialize(iterate.n_rows,iterate.n_cols); + // Now iterate! arma::mat gradient(iterate.n_rows, iterate.n_cols); for (size_t i = 1; i != maxIterations; ++i, ++currentBatch) @@ -108,7 +130,7 @@ double MiniBatchSGD::Optimize(arma::mat& iterate) } // Now update the iterate. - iterate -= (stepSize / batchSize) * gradient; + updatePolicy.Update(iterate, stepSize / batchSize, gradient); // Add that to the overall objective function. for (size_t j = 0; j < batchSize; ++j) @@ -130,18 +152,21 @@ double MiniBatchSGD::Optimize(arma::mat& iterate) if (lastBatchSize > 0) { // Now update the iterate. - iterate -= (stepSize / lastBatchSize) * gradient; + updatePolicy.Update(iterate, stepSize / lastBatchSize, gradient); } else { // Now update the iterate. - iterate -= stepSize * gradient; + updatePolicy.Update(iterate, stepSize, gradient); } // Add that to the overall objective function. for (size_t j = 0; j < lastBatchSize; ++j) overallObjective += function.Evaluate(iterate, offset + j); } + + // Now update the learning rate if requested by the user. + decayPolicy.Update(stepSize, i - 1, currentBatch, iterate); } Log::Info << "Mini-batch SGD: maximum iterations (" << maxIterations << ") " From f980e5515eeec779484c70b973fa22cebf93c405 Mon Sep 17 00:00:00 2001 From: Marcus Edel Date: Tue, 21 Mar 2017 22:28:09 +0100 Subject: [PATCH 02/84] Add implementation of Stochastic Gradient Descent with Restarts. --- .../core/optimizers/sgdr/CMakeLists.txt | 13 + .../core/optimizers/sgdr/cyclical_decay.hpp | 138 +++++++++ src/mlpack/core/optimizers/sgdr/sgdr.hpp | 265 ++++++++++++++++++ src/mlpack/core/optimizers/sgdr/sgdr_impl.hpp | 191 +++++++++++++ .../optimizers/sgdr/snapshot_ensembles.hpp | 176 ++++++++++++ 5 files changed, 783 insertions(+) create mode 100644 src/mlpack/core/optimizers/sgdr/CMakeLists.txt create mode 100644 src/mlpack/core/optimizers/sgdr/cyclical_decay.hpp create mode 100644 src/mlpack/core/optimizers/sgdr/sgdr.hpp create mode 100644 src/mlpack/core/optimizers/sgdr/sgdr_impl.hpp create mode 100644 src/mlpack/core/optimizers/sgdr/snapshot_ensembles.hpp diff --git a/src/mlpack/core/optimizers/sgdr/CMakeLists.txt b/src/mlpack/core/optimizers/sgdr/CMakeLists.txt new file mode 100644 index 0000000000..cdb858beda --- /dev/null +++ b/src/mlpack/core/optimizers/sgdr/CMakeLists.txt @@ -0,0 +1,13 @@ +set(SOURCES + cyclical_decay.hpp + sgdr.hpp + sgdr_impl.hpp + snapshot_ensembles.hpp +) + +set(DIR_SRCS) +foreach(file ${SOURCES}) + set(DIR_SRCS ${DIR_SRCS} ${CMAKE_CURRENT_SOURCE_DIR}/${file}) +endforeach() + +set(MLPACK_SRCS ${MLPACK_SRCS} ${DIR_SRCS} PARENT_SCOPE) diff --git a/src/mlpack/core/optimizers/sgdr/cyclical_decay.hpp b/src/mlpack/core/optimizers/sgdr/cyclical_decay.hpp new file mode 100644 index 0000000000..ae8003ff5c --- /dev/null +++ b/src/mlpack/core/optimizers/sgdr/cyclical_decay.hpp @@ -0,0 +1,138 @@ +/** + * @file cyclical_decay.hpp + * @author Marcus Edel + * + * Definition of the warm restart technique (SGDR) described in: + * "SGDR: Stochastic Gradient Descent with Warm Restarts" by + * I. Loshchilov et al. + * + * You should define your own decay update that looks like EmptyDecay. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ + +#ifndef MLPACK_CORE_OPTIMIZERS_SGDR_CYCLICAL_DECAY_HPP +#define MLPACK_CORE_OPTIMIZERS_SGDR_CYCLICAL_DECAY_HPP + +namespace mlpack { +namespace optimization { + +/** + * Simulate a new warm-started run/restart once a number of epochs are + * performed. Importantly, the restarts are not performed from scratch but + * emulated by increasing the step size while the old step size value of as an + * initial parameter. + * + * For more information, please refer to: + * + * @code + * @article{Loshchilov2016, + * title = {Learning representations by back-propagating errors}, + * author = {Ilya Loshchilov and Frank Hutter}, + * title = {{SGDR:} Stochastic Gradient Descent with Restarts}, + * journal = {CoRR}, + * year = {2016} + * } + * @endcode + */ +class CyclicalDecay +{ + public: + /** + * Construct the CyclicalDecay technique a restart method, where the + * step size decays after each batch and peridically resets to its initial + * value. + * + * @param epochRestart Initial epoch where decay is applied. + * @param multFactor Factor to increase the number of epochs before a restart. + * @param stepSize Initial step size for each restart. + * @param batchSize Size of each mini-batch. + * @param numFunctions The number of separable functions (the number of + * predictor points). + */ + CyclicalDecay(const size_t epochRestart, + const double multFactor, + const double stepSize, + const size_t batchSize, + const size_t numFunctions) : + epochRestart(epochRestart), + multFactor(multFactor), + constStepSize(stepSize), + nextRestart(epochRestart), + batchRestart(0), + epochBatches(numFunctions / (double) batchSize) + { /* Nothing to do here */ } + + /** + * This function is called in each iteration after the policy update. + * + * @param stepSize The stepSize to be adjusted. + * @param epoch The current epoch. + * @param batch The current batch. + * @param iterate Function parameters. + */ + void Update(double& stepSize, + const size_t epoch, + const size_t /* batch */, + const arma::mat& /* iterate */) + { + // Time to adjust the step size. + if (epoch >= epochRestart) + { + // n_t = n_min^i + 0.5(n_max^i - n_min^i)(1 + cos(T_cur/T_i * pi)). + stepSize = 0.5 * constStepSize * (1 + cos((batchRestart / epochBatches) + * M_PI)); + + // Keep track of the number of batches since the last restart. + batchRestart++; + } + + // Time to restart. + if (epoch > nextRestart) + { + batchRestart = 0; + + // Adjust the period of restarts. + epochRestart *= multFactor; + + // Update the time for the next restart. + nextRestart += epochRestart; + } + } + + //! Get the step size. + double StepSize() const { return constStepSize; } + //! Modify the step size. + double& StepSize() { return constStepSize; } + + //! Get the restart fraction. + double EpochBatches() const { return epochBatches; } + //! Modify the restart fraction. + double& EpochBatches() { return epochBatches; } + private: + //! Epoch where decay is applied. + size_t epochRestart; + + //! Parameter to increase the number of epochs before a restart. + double multFactor; + + //! The step size for each example. + double constStepSize; + + //! Locally-stored restart time. + size_t nextRestart; + + //! Locally-stored number of batches since the last restart. + size_t batchRestart; + + //! Locally-stored restart fraction. + double epochBatches; +}; + +} // namespace optimization +} // namespace mlpack + +#endif // MLPACK_CORE_OPTIMIZERS_SGDR_CYCLICAL_DECAY_HPP \ No newline at end of file diff --git a/src/mlpack/core/optimizers/sgdr/sgdr.hpp b/src/mlpack/core/optimizers/sgdr/sgdr.hpp new file mode 100644 index 0000000000..6d02017974 --- /dev/null +++ b/src/mlpack/core/optimizers/sgdr/sgdr.hpp @@ -0,0 +1,265 @@ +/** + * @file sgdr.hpp + * @author Marcus Edel + * + * Definition of the Stochastic Gradient Descent with Restarts (SGDR) as + * described in: "SGDR: Stochastic Gradient Descent with Warm Restarts" by + * I. Loshchilov et al and the Snapshot ensembles technique described in: + * "Snapshot ensembles: Train 1, get m for free" by G. Huang et al. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_CORE_OPTIMIZERS_SGDR_SGDR_HPP +#define MLPACK_CORE_OPTIMIZERS_SGDR_SGDR_HPP + +#include + +#include +#include +#include +#include + +namespace mlpack { +namespace optimization { + +/** + * This class is based on Mini-batch Stochastic Gradient Descent class and + * simulates a new warm-started run/restart once a number of epochs are + * performed this class also implements the Snapshot ensembles technique. + * + * For more information, please refer to: + * + * @code + * @article{Loshchilov2016, + * title = {{SGDR:} Stochastic Gradient Descent with Restarts}, + * author = {Ilya Loshchilov and Frank Hutter}, + * journal = {CoRR}, + * year = {2016} + * } + * @endcode + * + * @code + * @inproceedings{Huang2017, + * title = {Snapshot ensembles: Train 1, get m for free}, + * author = {Gao Huang, Yixuan Li, Geoff Pleiss, Zhuang Liu, + * John E. Hopcroft, and Kilian Q. Weinberger}, + * booktitle = {Proceedings of the International Conference on Learning + * Representations (ICLR)}, + * year = {2017} + * } + * @endcode + * + * @tparam DecomposableFunctionType Decomposable objective function type to be + * minimized. + * @tparam UpdatePolicyType Update policy used during the iterative update + * process. By default the vanilla update policy + * (see mlpack::optimization::VanillaUpdate) is used. + * @tparam DecayPolicyType Decay policy used during the iterative update + * process to adjust the step size (CyclicalDecay or SnapshotEnsembles). + */ +template< + typename DecomposableFunctionType, + typename UpdatePolicyType = MomentumUpdate, + typename DecayPolicyType = CyclicalDecay +> +class SGDR +{ + public: + //! Convenience typedef for the internal optimizer construction. + using OptimizerType = MiniBatchSGDType< + DecomposableFunctionType, UpdatePolicyType, DecayPolicyType>; + + /** + * Construct the SGDR optimizer with snapshot ensembles with the given + * function and parameters. The defaults here are not necessarily good for + * the given problem, so it is suggested that the values used be tailored for + * the task at hand. The maximum number of iterations refers to the maximum + * number of mini-batches that are processed. + * + * @param epochRestart Initial epoch where decay is applied. + * @param function Function to be optimized (minimized). + * @param batchSize Size of each mini-batch. + * @param stepSize Step size for each iteration. + * @param maxIterations Maximum number of iterations allowed (0 means no + * limit). + * @param tolerance Maximum absolute tolerance to terminate algorithm. + * @param shuffle If true, the mini-batch order is shuffled; otherwise, each + * mini-batch is visited in linear order. + * @param snapshots Maximum number of snapshots. + * @param updatePolicy Instantiated update policy used to adjust the given + * parameters. + */ + template + SGDR(DecomposableFunctionType& function, + const size_t epochRestart = 50, + const double multFactor = 2.0, + const size_t batchSize = 1000, + const double stepSize = 0.01, + const size_t maxIterations = 100000, + const double tolerance = 1e-5, + const bool shuffle = true, + const size_t snapshots = 5, + const UpdatePolicyType& updatePolicy = UpdatePolicyType(), + const typename std::enable_if_t::value>* junk = 0); + + /** + * Construct the SGDR optimizer with the given function and + * parameters. The defaults here are not necessarily good for the given + * problem, so it is suggested that the values used be tailored for the task + * at hand. The maximum number of iterations refers to the maximum number of + * mini-batches that are processed. + * + * @param epochRestart Initial epoch where decay is applied. + * @param function Function to be optimized (minimized). + * @param batchSize Size of each mini-batch. + * @param stepSize Step size for each iteration. + * @param maxIterations Maximum number of iterations allowed (0 means no + * limit). + * @param tolerance Maximum absolute tolerance to terminate algorithm. + * @param shuffle If true, the mini-batch order is shuffled; otherwise, each + * mini-batch is visited in linear order. + * @param updatePolicy Instantiated update policy used to adjust the given + * parameters. + */ + template + SGDR(DecomposableFunctionType& function, + const size_t epochRestart = 50, + const double multFactor = 2.0, + const size_t batchSize = 1000, + const double stepSize = 0.01, + const size_t maxIterations = 100000, + const double tolerance = 1e-5, + const bool shuffle = true, + const UpdatePolicyType& updatePolicy = UpdatePolicyType(), + const typename std::enable_if_t::value>* junk = 0); + + /** + * Optimize the given function using SGDR. The given starting point + * will be modified to store the finishing point of the algorithm, and the + * final objective value is returned. + * + * @param iterate Starting point (will be modified). + * @param accumulate Accumulate the snapshot parameter (default true). + * @return Objective value of the final point. + */ + template + double Optimize(arma::mat& iterate, + const bool accumulate = true, + const typename std::enable_if_t::value>* junk = 0); + + /** + * Optimize the given function using SGDR. The given starting point + * will be modified to store the finishing point of the algorithm, and the + * final objective value is returned. + * + * @param iterate Starting point (will be modified). + * @return Objective value of the final point. + */ + template + double Optimize(arma::mat& iterate, + const typename std::enable_if_t::value>* junk = 0); + + //! Get the instantiated function to be optimized. + const DecomposableFunctionType& Function() const + { + return optimizer.Function(); + } + + //! Modify the instantiated function. + DecomposableFunctionType& Function() { return optimizer.Function(); } + + //! Get the batch size. + size_t BatchSize() const { return optimizer.BatchSize(); } + //! Modify the batch size. + size_t& BatchSize() { return optimizer.BatchSize(); } + + //! Get the step size. + double StepSize() const { return optimizer.StepSize(); } + //! Modify the step size. + double& StepSize() { return optimizer.StepSize(); } + + //! Get the maximum number of iterations (0 indicates no limit). + size_t MaxIterations() const { return optimizer.MaxIterations(); } + //! Modify the maximum number of iterations (0 indicates no limit). + size_t& MaxIterations() { return optimizer.MaxIterations(); } + + //! Get the tolerance for termination. + double Tolerance() const { return optimizer.Tolerance(); } + //! Modify the tolerance for termination. + double& Tolerance() { return optimizer.Tolerance(); } + + //! Get whether or not the individual functions are shuffled. + bool Shuffle() const { return optimizer.Shuffle(); } + //! Modify whether or not the individual functions are shuffled. + bool& Shuffle() { return optimizer.Shuffle(); } + + //! Get the snapshots. + template + typename std::enable_if< + std::is_same::value, + std::vector >::type + Snapshots() const { return decayPolicy.Snapshots(); } + + //! Modify the snapshots. + template + typename std::enable_if< + std::is_same::value, + std::vector& >::type + Snapshots() { return decayPolicy.snapshots(); } + + //! Get the snapshots. + std::vector Snapshots() const { return junk; } + //! Modify the snapshots. + std::vector& Snapshots() { return junk; } + + private: + //! The instantiated function. + DecomposableFunctionType& function; + + //! The size of each mini-batch. + size_t batchSize; + + //! The maximum number of allowed iterations. + size_t maxIterations; + + //! The tolerance for termination. + double tolerance; + + //! Controls whether or not the individual functions are shuffled when + //! iterating. + bool shuffle; + + //! The decay method used to update the step size in each iteration. + DecayPolicyType decayPolicy; + + //! Locally-stored optimizer instance. + OptimizerType optimizer; + + //! Locally-stored empty snapshots, necessary to provide an output if another + //! decay policy than SnapshotEnsembles is used. + std::vector junk; +}; + +// Convenience typedef. + +/** + * Stochastic Gradient Descent with Restarts and snapshot ensembles. + */ +template +using SnapshotSGDR = SGDR< + DecomposableFunctionType, MomentumUpdate,SnapshotEnsembles>; + +} // namespace optimization +} // namespace mlpack + +// Include implementation. +#include "sgdr_impl.hpp" + +#endif diff --git a/src/mlpack/core/optimizers/sgdr/sgdr_impl.hpp b/src/mlpack/core/optimizers/sgdr/sgdr_impl.hpp new file mode 100644 index 0000000000..6f8add364f --- /dev/null +++ b/src/mlpack/core/optimizers/sgdr/sgdr_impl.hpp @@ -0,0 +1,191 @@ +/** + * @file sgdr_impl.hpp + * @author Marcus Edel + * + * Implementation of SGDR method. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_CORE_OPTIMIZERS_SGDR_SGDR_IMPL_HPP +#define MLPACK_CORE_OPTIMIZERS_SGDR_SGDR_IMPL_HPP + +// In case it hasn't been included yet. +#include "sgdr.hpp" + +namespace mlpack { +namespace optimization { + +template< + typename DecomposableFunctionType, + typename UpdatePolicyType, + typename DecayPolicyType +> +template +SGDR< + DecomposableFunctionType, + UpdatePolicyType, + DecayPolicyType +>::SGDR(DecomposableFunctionType& function, + const size_t epochRestart, + const double multFactor, + const size_t batchSize, + const double stepSize, + const size_t maxIterations, + const double tolerance, + const bool shuffle, + const size_t snapshots, + const UpdatePolicyType& updatePolicy, + const typename std::enable_if_t::value>* /* junk */) : + function(function), + batchSize(batchSize), + decayPolicy(SnapshotEnsembles(epochRestart, + multFactor, + stepSize, + batchSize, + function.NumFunctions(), + maxIterations, + snapshots)), + optimizer(OptimizerType(function, + batchSize, + stepSize, + maxIterations, + tolerance, + shuffle, + updatePolicy, + decayPolicy)) +{ + /* Nothing to do here */ +} + +template< + typename DecomposableFunctionType, + typename UpdatePolicyType, + typename DecayPolicyType +> +template +SGDR< + DecomposableFunctionType, + UpdatePolicyType, + DecayPolicyType +>::SGDR(DecomposableFunctionType& function, + const size_t epochRestart, + const double multFactor, + const size_t batchSize, + const double stepSize, + const size_t maxIterations, + const double tolerance, + const bool shuffle, + const UpdatePolicyType& updatePolicy, + const typename std::enable_if_t::value>* /* junk */) : + function(function), + batchSize(batchSize), + decayPolicy(CyclicalDecay(epochRestart, + multFactor, + stepSize, + batchSize, + function.NumFunctions())), + optimizer(OptimizerType(function, + batchSize, + stepSize, + maxIterations, + tolerance, + shuffle, + updatePolicy, + decayPolicy)) +{ + /* Nothing to do here */ +} + +template< + typename DecomposableFunctionType, + typename UpdatePolicyType, + typename DecayPolicyType +> +template +double SGDR< + DecomposableFunctionType, + UpdatePolicyType, + DecayPolicyType +>::Optimize(arma::mat& iterate, + const bool accumulate, + const typename std::enable_if_t::value>* /* junk */) +{ + // If a user changed the step size he hasn't update the step size of the + // cyclical decay instantiation, so we have to do here. + if (optimizer.StepSize() != decayPolicy.StepSize()) + { + decayPolicy.StepSize() = optimizer.StepSize(); + } + + // If a user changed the batch size we have to update the restart fraction + // of the cyclical decay instantiation. + if (optimizer.BatchSize() != batchSize) + { + batchSize = optimizer.BatchSize(); + decayPolicy.EpochBatches() = function.NumFunctions() / + double(batchSize); + } + + double overallObjective = optimizer.Optimize(iterate); + + // Accumulate snapshots. + if (accumulate) + { + for (size_t i = 0; i < decayPolicy.Snapshots().size(); ++i) + { + iterate += decayPolicy.Snapshots()[i]; + } + iterate /= (decayPolicy.Snapshots().size() + 1); + + // Calculate final objective. + overallObjective = 0; + for (size_t i = 0; i < function.NumFunctions(); ++i) + overallObjective += function.Evaluate(iterate, i); + } + + return overallObjective; +} + +template< + typename DecomposableFunctionType, + typename UpdatePolicyType, + typename DecayPolicyType +> +template +double SGDR< + DecomposableFunctionType, + UpdatePolicyType, + DecayPolicyType +>::Optimize(arma::mat& iterate, + const typename std::enable_if_t::value>* /* junk */) +{ + // If a user changed the step size he hasn't update the step size of the + // cyclical decay instantiation, so we have to do here. + if (optimizer.StepSize() != decayPolicy.StepSize()) + { + decayPolicy.StepSize() = optimizer.StepSize(); + } + + // If a user changed the batch size we have to update the restart fraction + // of the cyclical decay instantiation. + if (optimizer.BatchSize() != batchSize) + { + batchSize = optimizer.BatchSize(); + decayPolicy.EpochBatches() = function.NumFunctions() / + double(batchSize); + } + + return optimizer.Optimize(iterate); +} + +} // namespace optimization +} // namespace mlpack + +#endif diff --git a/src/mlpack/core/optimizers/sgdr/snapshot_ensembles.hpp b/src/mlpack/core/optimizers/sgdr/snapshot_ensembles.hpp new file mode 100644 index 0000000000..ac4de679f2 --- /dev/null +++ b/src/mlpack/core/optimizers/sgdr/snapshot_ensembles.hpp @@ -0,0 +1,176 @@ +/** + * @file snapshot_ensembles.hpp + * @author Marcus Edel + * + * Definition of the Snapshot ensembles technique described in: + * "Snapshot ensembles: Train 1, get m for free" by G. Huang et al. + * + * You should define your own decay update that looks like EmptyDecay. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ + +#ifndef MLPACK_CORE_OPTIMIZERS_SGDR_SNAPSHOT_ENSEMBLES_HPP +#define MLPACK_CORE_OPTIMIZERS_SGDR_SNAPSHOT_ENSEMBLES_HPP + +namespace mlpack { +namespace optimization { + +/** + * Simulate a new warm-started run/restart once a number of epochs are + * performed. Importantly, the restarts are not performed from scratch but + * emulated by increasing the step size while the old step size value of as an + * initial parameter. + * + * For more information, please refer to: + * + * @code + * @inproceedings{Huang2017, + * title = {Snapshot ensembles: Train 1, get m for free}, + * author = {Gao Huang, Yixuan Li, Geoff Pleiss, Zhuang Liu, + * John E. Hopcroft, and Kilian Q. Weinberger}, + * booktitle = {Proceedings of the International Conference on Learning + * Representations (ICLR)}, + * year = {2017} + * } + * @endcode + */ +class SnapshotEnsembles +{ + public: + /** + * Construct the CyclicalDecay technique a restart method, where the + * step size decays after each batch and peridically resets to its initial + * value. + * + * @param epochRestart Initial epoch where decay is applied. + * @param multFactor Factor to increase the number of epochs before a restart. + * @param stepSize Initial step size for each restart. + * @param batchSize Size of each mini-batch. + * @param numFunctions The number of separable functions (the number of + * predictor points). + * @param maxIterations Maximum number of iterations allowed (0 means no + * limit). + * @param snapshots Maximum number of snapshots. + */ + SnapshotEnsembles(const size_t epochRestart, + const double multFactor, + const double stepSize, + const size_t numFunctions, + const size_t batchSize, + const size_t maxIterations, + const size_t snapshots) : + epochRestart(epochRestart), + multFactor(multFactor), + constStepSize(stepSize), + nextRestart(epochRestart), + batchRestart(0), + epochBatches(numFunctions / (double) batchSize) + { + snapshotEpochs = 0; + for (size_t i = 0, er = epochRestart, nr = nextRestart; + i < maxIterations; ++i) + { + if (i > nr) + { + er *= multFactor; + nr += er; + snapshotEpochs++; + } + } + + snapshotEpochs = epochRestart * std::pow(multFactor, + snapshotEpochs - snapshots + 1); + } + + /** + * This function is called in each iteration after the policy update. + * + * @param stepSize The stepSize to be adjusted. + * @param epoch The current epoch. + * @param batch The current batch. + * @param iterate Function parameters. + */ + void Update(double& stepSize, + const size_t epoch, + const size_t /* batch */, + const arma::mat& iterate) + { + // Time to adjust the step size. + if (epoch >= epochRestart) + { + // n_t = n_min^i + 0.5(n_max^i - n_min^i)(1 + cos(T_cur/T_i * pi)). + stepSize = 0.5 * constStepSize * (1 + cos((batchRestart / epochBatches) + * M_PI)); + + // Keep track of the number of batches since the last restart. + batchRestart++; + } + + // Time to restart. + if (epoch > nextRestart) + { + batchRestart = 0; + + // Adjust the period of restarts. + epochRestart *= multFactor; + + // Create a new snapshot. + if (epochRestart >= snapshotEpochs) + { + snapshots.push_back(iterate); + } + + // Update the time for the next restart. + nextRestart += epochRestart; + } + } + + //! Get the step size. + double StepSize() const { return constStepSize; } + //! Modify the step size. + double& StepSize() { return constStepSize; } + + //! Get the restart fraction. + double EpochBatches() const { return epochBatches; } + //! Modify the restart fraction. + double& EpochBatches() { return epochBatches; } + + //! Get the snapshots. + std::vector Snapshots() const { return snapshots; } + //! Modify the snapshots. + std::vector& Snapshots() { return snapshots; } + + private: + //! Epoch where decay is applied. + size_t epochRestart; + + //! Parameter to increase the number of epochs before a restart. + double multFactor; + + //! The step size for each example. + double constStepSize; + + //! Locally-stored restart time. + size_t nextRestart; + + //! Locally-stored number of batches since the last restart. + size_t batchRestart; + + //! Locally-stored restart fraction. + double epochBatches; + + //! Epochs where a new snapshot is created. + size_t snapshotEpochs; + + //! Locally-stored parameter snapshots. + std::vector snapshots; +}; + +} // namespace optimization +} // namespace mlpack + +#endif // MLPACK_CORE_OPTIMIZERS_SGDR_CYCLICAL_DECAY_HPP \ No newline at end of file From d835785dc45595553a4a6286aa2cca49c385d5b0 Mon Sep 17 00:00:00 2001 From: Tiramisu 1993 Date: Sat, 18 Mar 2017 15:29:55 +0800 Subject: [PATCH 03/84] make decision tree support weighted traning --- .../decision_tree/all_categorical_split.hpp | 3 +- .../all_categorical_split_impl.hpp | 23 +- .../best_binary_numeric_split.hpp | 3 +- .../best_binary_numeric_split_impl.hpp | 20 +- .../methods/decision_tree/decision_tree.hpp | 49 +- .../decision_tree/decision_tree_impl.hpp | 81 +- .../decision_tree/decision_tree_main.cpp | 17 +- .../methods/decision_tree/gini_gain.hpp | 46 +- .../decision_tree/information_gain.hpp | 57 +- src/mlpack/tests/decision_tree_test.cpp | 1128 ++++++++++------- 10 files changed, 922 insertions(+), 505 deletions(-) diff --git a/src/mlpack/methods/decision_tree/all_categorical_split.hpp b/src/mlpack/methods/decision_tree/all_categorical_split.hpp index cb09eee30e..56c66fa625 100644 --- a/src/mlpack/methods/decision_tree/all_categorical_split.hpp +++ b/src/mlpack/methods/decision_tree/all_categorical_split.hpp @@ -52,7 +52,7 @@ class AllCategoricalSplit * @param aux Auxiliary split information, which may be modified on a * successful split. */ - template + template static double SplitIfBetter( const double bestGain, const VecType& data, @@ -60,6 +60,7 @@ class AllCategoricalSplit const arma::Row& labels, const size_t numClasses, const size_t minimumLeafSize, + const WeightVecType& weights, arma::Col& classProbabilities, AuxiliarySplitInfo& aux); diff --git a/src/mlpack/methods/decision_tree/all_categorical_split_impl.hpp b/src/mlpack/methods/decision_tree/all_categorical_split_impl.hpp index 0e364fb7c6..1f95f1443e 100644 --- a/src/mlpack/methods/decision_tree/all_categorical_split_impl.hpp +++ b/src/mlpack/methods/decision_tree/all_categorical_split_impl.hpp @@ -16,7 +16,7 @@ namespace mlpack { namespace tree { template -template +template double AllCategoricalSplit::SplitIfBetter( const double bestGain, const VecType& data, @@ -24,6 +24,7 @@ double AllCategoricalSplit::SplitIfBetter( const arma::Row& labels, const size_t numClasses, const size_t minimumLeafSize, + const WeightVecType& weights, arma::Col& classProbabilities, AuxiliarySplitInfo& /* aux */) { @@ -43,14 +44,28 @@ double AllCategoricalSplit::SplitIfBetter( // that would be assigned to each child. arma::uvec childPositions(numCategories, arma::fill::zeros); std::vector> childLabels(numCategories); + std::vector> childWeights(numCategories); for (size_t i = 0; i < numCategories; ++i) + { + // Labels and weights should have same length. childLabels[i].zeros(counts[i]); + childWeights[i].zeros(counts[i]); + } // Extract labels for each child. for (size_t i = 0; i < data.n_elem; ++i) { const size_t category = (size_t) data[i]; - childLabels[category][childPositions[category]++] = labels[i]; + + if (UseWeights) + { + childLabels[category][childPositions[category]] = labels[i]; + childWeights[category][childPositions[category]++] = weights[i] ? weights[i] : 0; + } + else + { + childLabels[category][childPositions[category]++] = labels[i]; + } } double overallGain = 0.0; @@ -58,8 +73,8 @@ double AllCategoricalSplit::SplitIfBetter( { // Calculate the gain of this child. const double childPct = double(counts[i]) / double(data.n_elem); - const double childGain = FitnessFunction::Evaluate(childLabels[i], - numClasses); + const double childGain = FitnessFunction::template Evaluate(childLabels[i], + numClasses, childWeights[i]); overallGain += childPct * childGain; } diff --git a/src/mlpack/methods/decision_tree/best_binary_numeric_split.hpp b/src/mlpack/methods/decision_tree/best_binary_numeric_split.hpp index 791eace4f4..b81dc0cbcc 100644 --- a/src/mlpack/methods/decision_tree/best_binary_numeric_split.hpp +++ b/src/mlpack/methods/decision_tree/best_binary_numeric_split.hpp @@ -50,13 +50,14 @@ class BestBinaryNumericSplit * @param aux Auxiliary split information, which may be modified on a * successful split. */ - template + template static double SplitIfBetter( const double bestGain, const VecType& data, const arma::Row& labels, const size_t numClasses, const size_t minimumLeafSize, + const WeightVecType& weights, arma::Col& classProbabilities, AuxiliarySplitInfo& aux); diff --git a/src/mlpack/methods/decision_tree/best_binary_numeric_split_impl.hpp b/src/mlpack/methods/decision_tree/best_binary_numeric_split_impl.hpp index 8a57a8c7e9..1e573e909d 100644 --- a/src/mlpack/methods/decision_tree/best_binary_numeric_split_impl.hpp +++ b/src/mlpack/methods/decision_tree/best_binary_numeric_split_impl.hpp @@ -16,13 +16,14 @@ namespace mlpack { namespace tree { template -template +template double BestBinaryNumericSplit::SplitIfBetter( const double bestGain, const VecType& data, const arma::Row& labels, const size_t numClasses, const size_t minimumLeafSize, + const WeightVecType& weights, arma::Col& classProbabilities, AuxiliarySplitInfo& /* aux */) { @@ -33,8 +34,16 @@ double BestBinaryNumericSplit::SplitIfBetter( // Next, sort the data. arma::uvec sortedIndices = arma::sort_index(data); arma::Row sortedLabels(labels.n_elem); + arma::Row sortedWeights(labels.n_elem); + sortedWeights.zeros(); for (size_t i = 0; i < sortedLabels.n_elem; ++i) sortedLabels[sortedIndices[i]] = labels[i]; + if (UseWeights) + { + // The weights must keep the same order of labels + for (size_t i = 0; i < sortedLabels.n_elem; ++i) + sortedWeights[sortedIndices[i]] = weights[i]; + } // Loop through all possible split points, choosing the best one. Also, force // a minimum leaf size of 1 (empty children don't make sense). @@ -47,10 +56,11 @@ double BestBinaryNumericSplit::SplitIfBetter( continue; // Calculate the gain for the left and right child. - const double leftGain = FitnessFunction::Evaluate(sortedLabels.subvec(0, - index - 1), numClasses); - const double rightGain = FitnessFunction::Evaluate(sortedLabels.subvec( - index, sortedLabels.n_elem - 1), numClasses); + const double leftGain = FitnessFunction::template Evaluate(sortedLabels.subvec(0, + index - 1), numClasses, sortedWeights.subvec(0, index - 1)); + const double rightGain = FitnessFunction::template Evaluate(sortedLabels.subvec( + index, sortedLabels.n_elem - 1), numClasses, + sortedWeights.subvec(index, sortedLabels.n_elem - 1)); // Calculate the fraction of points in the left and right children. const double leftRatio = double(index) / double(sortedLabels.n_elem); diff --git a/src/mlpack/methods/decision_tree/decision_tree.hpp b/src/mlpack/methods/decision_tree/decision_tree.hpp index 3d7622f1c0..fca2ba6c5c 100644 --- a/src/mlpack/methods/decision_tree/decision_tree.hpp +++ b/src/mlpack/methods/decision_tree/decision_tree.hpp @@ -81,6 +81,47 @@ class DecisionTree : const size_t numClasses, const size_t minimumLeafSize = 10); + /** + * Construct the decision tree on the given data and labels with weight, where the data + * can be both numeric and categorical. Setting minimumLeafSize too small may + * cause the tree to overfit, but setting it too large may cause it to + * underfit. + * + * @param data Dataset to train on. + * @param datasetInfo Type information for each dimension of the dataset. + * @param labels Labels for each training point. + * @param numClasses Number of classes in the dataset. + * @param weights The weight list of given label. + * @param minimumLeafSize Minimum number of points in each leaf node. + */ + template + DecisionTree(const MatType& data, + const data::DatasetInfo& datasetInfo, + const arma::Row& labels, + const size_t numClasses, + const arma::rowvec& weights, + const size_t minimumLeafSize = 10); + + /** + * Construct the decision tree on the given data and labels with weight, assuming that the + * data is all of the numeric type. Setting minimumLeafSize too small may + * cause the tree to overfit, but setting it too large may cause it to + * underfit. + * + * @param data Dataset to train on. + * @param labels Labels for each training point. + * @param numClasses Number of classes in the dataset. + * @param weights The Weight list of given labels. + * @param minimumLeafSize Minimum number of points in each leaf node. + */ + template + DecisionTree(const MatType& data, + const arma::Row& labels, + const size_t numClasses, + const arma::rowvec& weights, + const size_t minimumLeafSize = 10); + + /** * Construct a decision tree without training it. It will be a leaf node with * equal probabilities for each class. @@ -134,13 +175,15 @@ class DecisionTree : * @param datasetInfo Type information for each dimension. * @param labels Labels for each training point. * @param numClasses Number of classes in the dataset. + * @param weights Weights of all the labels * @param minimumLeafSize Minimum number of points in each leaf node. */ - template + template void Train(const MatType& data, const data::DatasetInfo& datasetInfo, const arma::Row& labels, const size_t numClasses, + const arma::rowvec& weights, const size_t minimumLeafSize = 10); /** @@ -152,12 +195,14 @@ class DecisionTree : * @param data Dataset to train on. * @param labels Labels for each training point. * @param numClasses Number of classes in the dataset. + * @param weights Weights of all the labels * @param minimumLeafSize Minimum number of points in each leaf node. */ - template + template void Train(const MatType& data, const arma::Row& labels, const size_t numClasses, + const arma::rowvec& weights, const size_t minimumLeafSize = 10); /** diff --git a/src/mlpack/methods/decision_tree/decision_tree_impl.hpp b/src/mlpack/methods/decision_tree/decision_tree_impl.hpp index 49b9cea084..39819d2884 100644 --- a/src/mlpack/methods/decision_tree/decision_tree_impl.hpp +++ b/src/mlpack/methods/decision_tree/decision_tree_impl.hpp @@ -15,7 +15,7 @@ namespace mlpack { namespace tree { -//! Construct and train. +//! Construct and train without weight. template class NumericSplitType, template class CategoricalSplitType, @@ -32,11 +32,13 @@ DecisionTree(data, datasetInfo, labels, numClasses, weights, minimumLeafSize); } -//! Construct and train. +//! Construct and train without weight. template class NumericSplitType, template class CategoricalSplitType, @@ -52,10 +54,55 @@ DecisionTree(data, labels, numClasses, weights, minimumLeafSize); } +//! Construct and train without weight +template class NumericSplitType, + template class CategoricalSplitType, + typename ElemType, + bool NoRecursion> +template +DecisionTree::DecisionTree(const MatType& data, + const data::DatasetInfo& datasetInfo, + const arma::Row& labels, + const size_t numClasses, + const arma::rowvec& weights, + const size_t minimumLeafSize) +{ + // Pass off work to the weighted Train() method. + Train(data, datasetInfo, labels, numClasses, weights, minimumLeafSize); +} + +//! Construct and train without weight +template class NumericSplitType, + template class CategoricalSplitType, + typename ElemType, + bool NoRecursion> +template +DecisionTree::DecisionTree(const MatType& data, + const arma::Row& labels, + const size_t numClasses, + const arma::rowvec& weights, + const size_t minimumLeafSize) +{ + // Pass off work to the weighted Train() method. + Train(data, labels, numClasses, weights, minimumLeafSize); + } + //! Construct, don't train. template class NumericSplitType, @@ -216,7 +263,7 @@ template class CategoricalSplitType, typename ElemType, bool NoRecursion> -template +template void DecisionTree& labels, const size_t numClasses, + const arma::rowvec& weights, const size_t minimumLeafSize) { // Clear children if needed. @@ -237,18 +285,18 @@ void DecisionTree(labels, numClasses, weights); size_t bestDim = datasetInfo.Dimensionality(); // This means "no split". for (size_t i = 0; i < datasetInfo.Dimensionality(); ++i) { double dimGain = -DBL_MAX; if (datasetInfo.Type(i) == data::Datatype::categorical) - dimGain = CategoricalSplit::SplitIfBetter(bestGain, data.row(i), - datasetInfo.NumMappings(i), labels, numClasses, minimumLeafSize, - classProbabilities, *this); + dimGain = CategoricalSplit::template SplitIfBetter(bestGain, data.row(i), + datasetInfo.NumMappings(i), labels, numClasses, minimumLeafSize, weights, + classProbabilities, *this); else if (datasetInfo.Type(i) == data::Datatype::numeric) - dimGain = NumericSplit::SplitIfBetter(bestGain, data.row(i), labels, - numClasses, minimumLeafSize, classProbabilities, *this); + dimGain = NumericSplit::template SplitIfBetter(bestGain, data.row(i), labels, + numClasses, minimumLeafSize, weights, classProbabilities, *this); // Was there an improvement? If so mark that it's the new best dimension. if (dimGain > bestGain) @@ -337,7 +385,7 @@ template class CategoricalSplitType, typename ElemType, bool NoRecursion> -template +template void DecisionTree::Train(const MatType& data, const arma::Row& labels, const size_t numClasses, + const arma::rowvec& weights, const size_t minimumLeafSize) { // Clear children if needed. @@ -360,13 +409,13 @@ void DecisionTree(labels, numClasses, weights); size_t bestDim = data.n_rows; // This means "no split". for (size_t i = 0; i < data.n_rows; ++i) { - double dimGain = NumericSplitType::SplitIfBetter(bestGain, - data.row(i), labels, numClasses, minimumLeafSize, classProbabilities, - *this); + double dimGain = NumericSplitType::template SplitIfBetter(bestGain, + data.row(i), labels, numClasses, minimumLeafSize, weights, classProbabilities, + *this); if (dimGain > bestGain) { diff --git a/src/mlpack/methods/decision_tree/decision_tree_main.cpp b/src/mlpack/methods/decision_tree/decision_tree_main.cpp index 2b8885bcc8..b2b51a36b1 100644 --- a/src/mlpack/methods/decision_tree/decision_tree_main.cpp +++ b/src/mlpack/methods/decision_tree/decision_tree_main.cpp @@ -48,7 +48,8 @@ PROGRAM_INFO("Decision tree", PARAM_MATRIX_IN("training", "Matrix of training points.", "t"); PARAM_UROW_IN("labels", "Training labels.", "l"); PARAM_MATRIX_IN("test", "Matrix of test points.", "T"); -PARAM_UROW_IN("test_labels", "Test point labels, if accuracy calculation " +PARAM_MATRIX_IN("weights", "The weight of labels", "w"); +PARAM_UMATRIX_IN("test_labels", "Test point labels, if accuracy calculation " "is desired.", "L"); // Training parameters. @@ -149,8 +150,18 @@ int main(int argc, char** argv) // Now build the tree. const size_t minLeafSize = (size_t) CLI::GetParam("minimum_leaf_size"); - model.tree = DecisionTree<>(dataset, labels, numClasses, - minLeafSize); + // Create decision tree with weighted labels. + if (CLI::HasParam("weights")) + { + const arma::Row weights= std::move(CLI::GetParam>("weights")); + model.tree = DecisionTree<>(dataset, labels.row(0), numClasses, + weights, minLeafSize); + } + + else + { + model.tree = DecisionTree<>(dataset, labels.row(0), numClasses, minLeafSize); + } // Do we need to print training error? if (CLI::HasParam("print_training_error")) diff --git a/src/mlpack/methods/decision_tree/gini_gain.hpp b/src/mlpack/methods/decision_tree/gini_gain.hpp index c1f08da786..c311052ec1 100644 --- a/src/mlpack/methods/decision_tree/gini_gain.hpp +++ b/src/mlpack/methods/decision_tree/gini_gain.hpp @@ -33,26 +33,54 @@ class GiniGain * * @param labels Set of labels to evaluate Gini impurity on. * @param numClasses Number of classes in the dataset. + * @param weights Weight of labels. */ - template + template static double Evaluate(const RowType& labels, - const size_t numClasses) + const size_t numClasses, + const WeightVecType& weights) { // Corner case: if there are no elements, the impurity is zero. if (labels.n_elem == 0) return 0.0; - arma::Col counts(numClasses); - counts.zeros(); - for (size_t i = 0; i < labels.n_elem; ++i) - counts[labels[i]]++; + // Count the number of elements in each class. + arma::Col counts(numClasses); + counts.zeros(); // Calculate the Gini impurity of the un-split node. double impurity = 0.0; - for (size_t i = 0; i < numClasses; ++i) + + if (UseWeights) { - const double f = ((double) counts[i] / (double) labels.n_elem); - impurity += f * (1.0 - f); + + // sum all the weights up + double accWeights = 0.0; + + for (size_t i=0; i < labels.n_elem; ++i) + { + // We just plus one if it's 'no weighted label' and plus 'weight' + // if the label had correspond label. + counts[labels[i]] += weights[i]; + accWeights += weights[i]; + } + + for (size_t i = 0; i < numClasses; ++i) + { + const double f = ((double) counts[i] / (double) accWeights); + impurity += f * (1.0 - f); + } + } + else + { + for (size_t i = 0; i < labels.n_elem; ++i) + counts[labels[i]]++; + + for (size_t i = 0; i < numClasses; ++i) + { + const double f = ((double) counts[i] / (double) labels.n_elem); + impurity += f * (1.0 - f); + } } return -impurity; diff --git a/src/mlpack/methods/decision_tree/information_gain.hpp b/src/mlpack/methods/decision_tree/information_gain.hpp index 2dbf814ddd..9eae6e1158 100644 --- a/src/mlpack/methods/decision_tree/information_gain.hpp +++ b/src/mlpack/methods/decision_tree/information_gain.hpp @@ -31,26 +31,53 @@ class InformationGain * @param labels Labels of the dataset. * @param numClasses Number of classes in the dataset. */ + template static double Evaluate(const arma::Row& labels, - const size_t numClasses) + const size_t numClasses, + const arma::Row& weights) { - // Edge case: if there are no elements, the gain is zero. - if (labels.n_elem == 0) - return 0.0; - - // Count the number of elements in each class. - arma::Col counts(numClasses); - counts.zeros(); - for (size_t i = 0; i < labels.n_elem; ++i) - counts[labels[i]]++; - + // Edge case: if there are no elements, the gain is zero. + if (labels.n_elem == 0) + return 0.0; + // Calculate the information gain. double gain = 0.0; - for (size_t i = 0; i < numClasses; ++i) + + // Count the number of elements in each class. + arma::Col counts(numClasses); + counts.zeros(); + + if (UseWeights) { - const double f = ((double) counts[i] / (double) labels.n_elem); - if (f > 0.0) - gain += f * std::log2(f); + // sum all the weights up + double accWeights = 0.0; + + for (size_t i=0; i < labels.n_elem; ++i) + { + // We just plus one if it's 'no weighted label' and plus 'weight' + // if the label had correspond label. + counts[labels[i]] += weights[i]; + accWeights += weights[i]; + } + + for (size_t i = 0; i < numClasses; ++i) + { + const double f = ((double) counts[i] / (double) accWeights); + if (f > 0.0) + gain += f * std::log2(f); + } + } + else + { + for (size_t i = 0; i < labels.n_elem; ++i) + counts[labels[i]]++; + + for (size_t i = 0; i < numClasses; ++i) + { + const double f = ((double) counts[i] / (double) labels.n_elem); + if (f > 0.0) + gain += f * std::log2(f); + } } return gain; diff --git a/src/mlpack/tests/decision_tree_test.cpp b/src/mlpack/tests/decision_tree_test.cpp index 8ba84c81e2..e5644c167b 100644 --- a/src/mlpack/tests/decision_tree_test.cpp +++ b/src/mlpack/tests/decision_tree_test.cpp @@ -22,444 +22,7 @@ using namespace mlpack; using namespace mlpack::tree; using namespace mlpack::distribution; -BOOST_AUTO_TEST_SUITE(DecisionTreeTest); - -/** - * Make sure the Gini gain is zero when the labels are perfect. - */ -BOOST_AUTO_TEST_CASE(GiniGainPerfectTest) -{ - arma::Row labels; - labels.zeros(10); - - // Test that it's perfect regardless of number of classes. - for (size_t c = 1; c < 10; ++c) - BOOST_REQUIRE_SMALL(GiniGain::Evaluate(labels, c), 1e-5); -} - -/** - * Make sure the Gini gain is -0.5 when the class split between two classes - * is even. - */ -BOOST_AUTO_TEST_CASE(GiniGainEvenSplitTest) -{ - arma::Row labels(10); - for (size_t i = 0; i < 5; ++i) - labels[i] = 0; - for (size_t i = 5; i < 10; ++i) - labels[i] = 1; - - // Test that it's -0.5 regardless of the number of classes. - for (size_t c = 2; c < 10; ++c) - BOOST_REQUIRE_CLOSE(GiniGain::Evaluate(labels, c), -0.5, 1e-5); -} - -/** - * The Gini gain of an empty vector is 0. - */ -BOOST_AUTO_TEST_CASE(GiniGainEmptyTest) -{ - // Test across some numbers of classes. - arma::Row labels; - for (size_t c = 1; c < 10; ++c) - BOOST_REQUIRE_SMALL(GiniGain::Evaluate(labels, c), 1e-5); -} - -/** - * The Gini gain is -(1 - 1/k) for k classes evenly split. - */ -BOOST_AUTO_TEST_CASE(GiniGainEvenSplitManyClassTest) -{ - // Try with many different classes. - for (size_t c = 2; c < 30; ++c) - { - arma::Row labels(c); - for (size_t i = 0; i < c; ++i) - labels[i] = i; - - // Calculate Gini gain and make sure it is correct. - BOOST_REQUIRE_CLOSE(GiniGain::Evaluate(labels, c), -(1.0 - 1.0 / c), 1e-5); - } -} - -/** - * The Gini gain should not be sensitive to the number of points. - */ -BOOST_AUTO_TEST_CASE(GiniGainManyPoints) -{ - for (size_t i = 1; i < 20; ++i) - { - const size_t numPoints = 100 * i; - arma::Row labels(numPoints); - for (size_t j = 0; j < numPoints / 2; ++j) - labels[j] = 0; - for (size_t j = numPoints / 2; j < numPoints; ++j) - labels[j] = 1; - - BOOST_REQUIRE_CLOSE(GiniGain::Evaluate(labels, 2), -0.5, 1e-5); - } -} - -/** - * The information gain should be zero when the labels are perfect. - */ -BOOST_AUTO_TEST_CASE(InformationGainPerfectTest) -{ - arma::Row labels; - labels.zeros(10); - - // Test that it's perfect regardless of number of classes. - for (size_t c = 1; c < 10; ++c) - BOOST_REQUIRE_SMALL(InformationGain::Evaluate(labels, c), 1e-5); -} - -/** - * If we have an even split, the information gain should be -1. - */ -BOOST_AUTO_TEST_CASE(InformationGainEvenSplitTest) -{ - arma::Row labels(10); - for (size_t i = 0; i < 5; ++i) - labels[i] = 0; - for (size_t i = 5; i < 10; ++i) - labels[i] = 1; - - // Test that it's -1 regardless of the number of classes. - for (size_t c = 2; c < 10; ++c) - BOOST_REQUIRE_CLOSE(InformationGain::Evaluate(labels, c), -1.0, 1e-5); -} - -/** - * The information gain of an empty vector is 0. - */ -BOOST_AUTO_TEST_CASE(InformationGainEmptyTest) -{ - arma::Row labels; - for (size_t c = 1; c < 10; ++c) - BOOST_REQUIRE_SMALL(InformationGain::Evaluate(labels, c), 1e-5); -} - -/** - * The information gain is log2(1/k) when splitting equal classes. - */ -BOOST_AUTO_TEST_CASE(InformationGainEvenSplitManyClassTest) -{ - // Try with many different numbers of classes. - for (size_t c = 2; c < 30; ++c) - { - arma::Row labels(c); - for (size_t i = 0; i < c; ++i) - labels[i] = i; - - // Calculate information gain and make sure it is correct. - BOOST_REQUIRE_CLOSE(InformationGain::Evaluate(labels, c), - std::log2(1.0 / c), 1e-5); - } -} - -/** - * The information gain should not be sensitive to the number of points. - */ -BOOST_AUTO_TEST_CASE(InformationGainManyPoints) -{ - for (size_t i = 1; i < 20; ++i) - { - const size_t numPoints = 100 * i; - arma::Row labels(numPoints); - for (size_t j = 0; j < numPoints / 2; ++j) - labels[j] = 0; - for (size_t j = numPoints / 2; j < numPoints; ++j) - labels[j] = 1; - - BOOST_REQUIRE_CLOSE(InformationGain::Evaluate(labels, 2), -1.0, 1e-5); - } -} - -/** - * Check that the BestBinaryNumericSplit will split on an obviously splittable - * dimension. - */ -BOOST_AUTO_TEST_CASE(BestBinaryNumericSplitSimpleSplitTest) -{ - arma::vec values("0.0 0.1 0.2 0.3 0.4 0.5 0.6 0.7 0.8 0.9 1.0"); - arma::Row labels("0 0 0 0 0 1 1 1 1 1 1"); - - arma::vec classProbabilities; - BestBinaryNumericSplit::template AuxiliarySplitInfo aux; - - // Call the method to do the splitting. - const double bestGain = GiniGain::Evaluate(labels, 2); - const double gain = BestBinaryNumericSplit::SplitIfBetter(bestGain, - values, labels, 2, 3, classProbabilities, aux); - - // Make sure that a split was made. - BOOST_REQUIRE_GT(gain, bestGain); - - // The split is perfect, so we should be able to accomplish a gain of 0. - BOOST_REQUIRE_SMALL(gain, 1e-5); - - // The class probabilities, for this split, hold the splitting point, which - // should be between 4 and 5. - BOOST_REQUIRE_EQUAL(classProbabilities.n_elem, 1); - BOOST_REQUIRE_GT(classProbabilities[0], 0.4); - BOOST_REQUIRE_LT(classProbabilities[0], 0.5); -} - -/** - * Check that the BestBinaryNumericSplit won't split if not enough points are - * given. - */ -BOOST_AUTO_TEST_CASE(BestBinaryNumericSplitMinSamplesTest) -{ - arma::vec values("0.0 0.1 0.2 0.3 0.4 0.5 0.6 0.7 0.8 0.9 1.0"); - arma::Row labels("0 0 0 0 0 1 1 1 1 1 1"); - - arma::vec classProbabilities; - BestBinaryNumericSplit::template AuxiliarySplitInfo aux; - - // Call the method to do the splitting. - const double bestGain = GiniGain::Evaluate(labels, 2); - const double gain = BestBinaryNumericSplit::SplitIfBetter(bestGain, - values, labels, 2, 8, classProbabilities, aux); - - // Make sure that no split was made. - BOOST_REQUIRE_EQUAL(gain, bestGain); - BOOST_REQUIRE_EQUAL(classProbabilities.n_elem, 0); -} - -/** - * Check that the BestBinaryNumericSplit doesn't split a dimension that gives no - * gain. - */ -BOOST_AUTO_TEST_CASE(BestBinaryNumericSplitNoGainTest) -{ - arma::vec values(100); - arma::Row labels(100); - for (size_t i = 0; i < 100; i += 2) - { - values[i] = i; - labels[i] = 0; - values[i + 1] = i; - labels[i + 1] = 1; - } - - arma::vec classProbabilities; - BestBinaryNumericSplit::template AuxiliarySplitInfo aux; - - // Call the method to do the splitting. - const double bestGain = GiniGain::Evaluate(labels, 2); - const double gain = BestBinaryNumericSplit::SplitIfBetter(bestGain, - values, labels, 2, 10, classProbabilities, aux); - - // Make sure there was no split. - BOOST_REQUIRE_EQUAL(gain, bestGain); - BOOST_REQUIRE_EQUAL(classProbabilities.n_elem, 0); -} - -/** - * Check that the AllCategoricalSplit will split when the split is obviously - * better. - */ -BOOST_AUTO_TEST_CASE(AllCategoricalSplitSimpleSplitTest) -{ - arma::vec values("0 0 0 1 1 1 2 2 2 3 3 3"); - arma::Row labels("0 0 0 2 2 2 1 1 1 2 2 2"); - - arma::vec classProbabilities; - AllCategoricalSplit::template AuxiliarySplitInfo aux; - - // Call the method to do the splitting. - const double bestGain = GiniGain::Evaluate(labels, 3); - const double gain = AllCategoricalSplit::SplitIfBetter(bestGain, - values, 4, labels, 3, 3, classProbabilities, aux); - - // Make sure that a split was made. - BOOST_REQUIRE_GT(gain, bestGain); - - // Since the split is perfect, make sure the new gain is 0. - BOOST_REQUIRE_SMALL(gain, 1e-5); - - // Make sure the class probabilities now hold the number of children. - BOOST_REQUIRE_EQUAL(classProbabilities.n_elem, 1); - BOOST_REQUIRE_EQUAL((size_t) classProbabilities[0], 4); -} - -/** - * Make sure that AllCategoricalSplit respects the minimum number of samples - * required to split. - */ -BOOST_AUTO_TEST_CASE(AllCategoricalSplitMinSamplesTest) -{ - arma::vec values("0 0 0 1 1 1 2 2 2 3 3 3"); - arma::Row labels("0 0 0 2 2 2 1 1 1 2 2 2"); - - arma::vec classProbabilities; - AllCategoricalSplit::template AuxiliarySplitInfo aux; - - // Call the method to do the splitting. - const double bestGain = GiniGain::Evaluate(labels, 3); - const double gain = AllCategoricalSplit::SplitIfBetter(bestGain, - values, 4, labels, 3, 4, classProbabilities, aux); - - // Make sure it's not split. - BOOST_REQUIRE_EQUAL(gain, bestGain); - BOOST_REQUIRE_EQUAL(classProbabilities.n_elem, 0); -} - -/** - * Check that no split is made when it doesn't get us anything. - */ -BOOST_AUTO_TEST_CASE(AllCategoricalSplitNoGainTest) -{ - arma::vec values(300); - arma::Row labels(300); - for (size_t i = 0; i < 300; i += 3) - { - values[i] = (i / 3) % 10; - labels[i] = 0; - values[i + 1] = (i / 3) % 10; - labels[i + 1] = 1; - values[i + 2] = (i / 3) % 10; - labels[i + 2] = 2; - } - - arma::vec classProbabilities; - AllCategoricalSplit::template AuxiliarySplitInfo aux; - - // Call the method to do the splitting. - const double bestGain = GiniGain::Evaluate(labels, 3); - const double gain = AllCategoricalSplit::SplitIfBetter(bestGain, - values, 10, labels, 3, 10, classProbabilities, aux); - - // Make sure that there was no split. - BOOST_REQUIRE_EQUAL(gain, bestGain); - BOOST_REQUIRE_EQUAL(classProbabilities.n_elem, 0); -} - -/** - * A basic construction of the decision tree---ensure that we can create the - * tree and that it split at least once. - */ -BOOST_AUTO_TEST_CASE(BasicConstructionTest) -{ - arma::mat dataset(10, 1000, arma::fill::randu); - arma::Row labels(1000); - for (size_t i = 0; i < 1000; ++i) - labels[i] = i % 3; // 3 classes. - - // Use default parameters. - DecisionTree<> d(dataset, labels, 3, 50); - - // Now require that we have some children. - BOOST_REQUIRE_GT(d.NumChildren(), 0); -} - -/** - * Construct the decision tree on numeric data only and see that we can fit it - * exactly and achieve perfect performance on the training set. - */ -BOOST_AUTO_TEST_CASE(PerfectTrainingSet) -{ - // Completely random dataset with no structure. - arma::mat dataset(10, 1000, arma::fill::randu); - arma::Row labels(1000); - for (size_t i = 0; i < 1000; ++i) - labels[i] = i % 3; // 3 classes. - - DecisionTree<> d(dataset, labels, 3, 1); // Minimum leaf size of 1. - - // Make sure that we can get perfect accuracy on the training set. - for (size_t i = 0; i < 1000; ++i) - { - size_t prediction; - arma::vec probabilities; - d.Classify(dataset.col(i), prediction, probabilities); - - BOOST_REQUIRE_EQUAL(prediction, labels[i]); - BOOST_REQUIRE_EQUAL(probabilities.n_elem, 3); - for (size_t j = 0; j < 3; ++j) - { - if (labels[i] == j) - BOOST_REQUIRE_CLOSE(probabilities[j], 1.0, 1e-5); - else - BOOST_REQUIRE_SMALL(probabilities[j], 1e-5); - } - } -} - -/** - * Make sure class probabilities are computed correctly in the root node. - */ -BOOST_AUTO_TEST_CASE(ClassProbabilityTest) -{ - arma::mat dataset(5, 100, arma::fill::randu); - arma::Row labels(100); - for (size_t i = 0; i < 100; i += 2) - { - labels[i] = 0; - labels[i + 1] = 1; - } - - // Create a decision tree that can't split. - DecisionTree<> d(dataset, labels, 2, 1000); - - BOOST_REQUIRE_EQUAL(d.NumChildren(), 0); - - // Estimate a point's probabilities. - arma::vec probabilities; - size_t prediction; - d.Classify(dataset.col(0), prediction, probabilities); - - BOOST_REQUIRE_EQUAL(probabilities.n_elem, 2); - BOOST_REQUIRE_CLOSE(probabilities[0], 0.5, 1e-5); - BOOST_REQUIRE_CLOSE(probabilities[1], 0.5, 1e-5); -} - -/** - * Test that the decision tree generalizes reasonably. - */ -BOOST_AUTO_TEST_CASE(SimpleGeneralizationTest) -{ - arma::mat inputData; - if (!data::Load("vc2.csv", inputData)) - BOOST_FAIL("Cannot load test dataset vc2.csv!"); - - arma::Mat labels; - if (!data::Load("vc2_labels.txt", labels)) - BOOST_FAIL("Cannot load labels for vc2_labels.txt"); - - // Build decision tree. - DecisionTree<> d(inputData, labels, 3, 10); // Leaf size of 10. - - // Load testing data. - arma::mat testData; - if (!data::Load("vc2_test.csv", testData)) - BOOST_FAIL("Cannot load test dataset vc2_test.csv!"); - - arma::Mat trueTestLabels; - if (!data::Load("vc2_test_labels.txt", trueTestLabels)) - BOOST_FAIL("Cannot load labels for vc2_test_labels.txt"); - - // Get the predicted test labels. - arma::Row predictions; - d.Classify(testData, predictions); - - BOOST_REQUIRE_EQUAL(predictions.n_elem, testData.n_cols); - - // Figure out the accuracy. - double correct = 0.0; - for (size_t i = 0; i < predictions.n_elem; ++i) - if (predictions[i] == trueTestLabels[i]) - ++correct; - correct /= predictions.n_elem; - - BOOST_REQUIRE_GT(correct, 0.75); -} - -/** - * Test that we can build a decision tree on a simple categorical dataset. - */ -BOOST_AUTO_TEST_CASE(CategoricalBuildTest) +void MockCategoricalData(arma::mat& d, arma::Row& l, data::DatasetInfo& datasetInfo) { // We'll build a spiral dataset plus two noisy categorical features. We need // to build the distributions for the categorical features (they'll be @@ -530,22 +93,22 @@ BOOST_AUTO_TEST_CASE(CategoricalBuildTest) } // Now create the dataset info. - data::DatasetInfo di(4); - di.Type(2) = data::Datatype::categorical; - di.Type(3) = data::Datatype::categorical; + datasetInfo = data::DatasetInfo(4); + datasetInfo.Type(2) = data::Datatype::categorical; + datasetInfo.Type(3) = data::Datatype::categorical; // Set mappings. - di.MapString("0", 2); - di.MapString("1", 2); - di.MapString("2", 2); - di.MapString("3", 2); - di.MapString("0", 3); - di.MapString("1", 3); + datasetInfo.MapString("0", 2); + datasetInfo.MapString("1", 2); + datasetInfo.MapString("2", 2); + datasetInfo.MapString("3", 2); + datasetInfo.MapString("0", 3); + datasetInfo.MapString("1", 3); // Now shuffle the dataset. arma::uvec indices = arma::shuffle(arma::linspace(0, 9999, 10000)); - arma::mat d(4, 10000); - arma::Row l(10000); + d = arma::mat(4, 10000); + l = arma::Row(10000); for (size_t i = 0; i < 10000; ++i) { d.col(i) = spiralDataset.col(indices[i]); @@ -557,6 +120,637 @@ BOOST_AUTO_TEST_CASE(CategoricalBuildTest) arma::mat testData = d.cols(5000, 9999); arma::Row trainingLabels = l.subvec(0, 4999); arma::Row testLabels = l.subvec(5000, 9999); +} + +BOOST_AUTO_TEST_SUITE(DecisionTreeTest); + +/** + * Make sure the Gini gain is zero when the labels are perfect. + */ +BOOST_AUTO_TEST_CASE(GiniGainPerfectTest) +{ + arma::rowvec weights(10); + arma::Row labels; + labels.zeros(10); + + // Test that it's perfect regardless of number of classes. + for (size_t c = 1; c < 10; ++c) + BOOST_REQUIRE_SMALL(GiniGain::Evaluate(labels, c, weights), 1e-5); +} + +/** + * Make sure the Gini gain is -0.5 when the class split between two classes + * is even. + */ +BOOST_AUTO_TEST_CASE(GiniGainEvenSplitTest) +{ + arma::rowvec weights = arma::ones(10); + arma::Row labels(10); + for (size_t i = 0; i < 5; ++i) + labels[i] = 0; + for (size_t i = 5; i < 10; ++i) + labels[i] = 1; + + // Test that it's -0.5 regardless of the number of classes. + for (size_t c = 2; c < 10; ++c) + { + BOOST_REQUIRE_CLOSE(GiniGain::Evaluate(labels, c, weights), -0.5, 1e-5); + double weightedGain = GiniGain::Evaluate(labels, c, weights); + + // The weighted gain should stay the same with unweight one + BOOST_REQUIRE_EQUAL(GiniGain::Evaluate(labels, c, weights), weightedGain); + } +} + +/** + * The Gini gain of an empty vector is 0. + */ +BOOST_AUTO_TEST_CASE(GiniGainEmptyTest) +{ + arma::rowvec weights = arma::ones(10); + // Test across some numbers of classes. + arma::Row labels; + for (size_t c = 1; c < 10; ++c) + BOOST_REQUIRE_SMALL(GiniGain::Evaluate(labels, c, weights), 1e-5); + + for (size_t c = 1; c < 10; ++c) + BOOST_REQUIRE_SMALL(GiniGain::Evaluate(labels, c, weights), 1e-5); +} + +/** + * The Gini gain is -(1 - 1/k) for k classes evenly split. + */ +BOOST_AUTO_TEST_CASE(GiniGainEvenSplitManyClassTest) +{ + // Try with many different classes. + for (size_t c = 2; c < 30; ++c) + { + arma::Row labels(c); + arma::rowvec weights(c); + for (size_t i = 0; i < c; ++i) + { + labels[i] = i; + weights[i] = 1; + } + + + // Calculate Gini gain and make sure it is correct. + BOOST_REQUIRE_CLOSE(GiniGain::Evaluate(labels, c, weights), -(1.0 - 1.0 / c), 1e-5); + BOOST_REQUIRE_CLOSE(GiniGain::Evaluate(labels, c, weights), -(1.0 - 1.0 / c), 1e-5); + } +} + +/** + * The Gini gain should not be sensitive to the number of points. + */ +BOOST_AUTO_TEST_CASE(GiniGainManyPoints) +{ + for (size_t i = 1; i < 20; ++i) + { + const size_t numPoints = 100 * i; + arma::rowvec weights(numPoints); + weights.ones(); + arma::Row labels(numPoints); + for (size_t j = 0; j < numPoints / 2; ++j) + labels[j] = 0; + for (size_t j = numPoints / 2; j < numPoints; ++j) + labels[j] = 1; + + BOOST_REQUIRE_CLOSE(GiniGain::Evaluate(labels, 2, weights), -0.5, 1e-5); + BOOST_REQUIRE_CLOSE(GiniGain::Evaluate(labels, 2, weights), -0.5, 1e-5); + } +} + +/** + * To make sure the Gini gain can been cacluate proporately with weight. + */ + BOOST_AUTO_TEST_CASE(GiniGainWithWeight) + { + arma::Row labels(10); + arma::rowvec weights(10); + for (size_t i = 0; i < 5; ++i) + { + labels[i] = 0; + weights[i] = 0.3; + } + for (size_t i = 5; i < 10; ++i) + { + labels[i] = 1; + weights[i] = 0.7; + } + + BOOST_REQUIRE_CLOSE(GiniGain::Evaluate(labels, 2, weights), -0.42, 1e-5); + } + +/** + * The information gain should be zero when the labels are perfect. + */ +BOOST_AUTO_TEST_CASE(InformationGainPerfectTest) +{ + arma::rowvec weights; + arma::Row labels; + labels.zeros(10); + + // Test that it's perfect regardless of number of classes. + for (size_t c = 1; c < 10; ++c) + BOOST_REQUIRE_SMALL(InformationGain::Evaluate(labels, c, weights), 1e-5); +} + +/** + * If we have an even split, the information gain should be -1. + */ +BOOST_AUTO_TEST_CASE(InformationGainEvenSplitTest) +{ + arma::Row labels(10); + arma::rowvec weights(10); + weights.ones(); + for (size_t i = 0; i < 5; ++i) + labels[i] = 0; + for (size_t i = 5; i < 10; ++i) + labels[i] = 1; + + // Test that it's -1 regardless of the number of classes. + for (size_t c = 2; c < 10; ++c) + { + // weighted and unweighted result should make no difference. + BOOST_REQUIRE_CLOSE(InformationGain::Evaluate(labels, c, weights), -1.0, 1e-5); + BOOST_REQUIRE_CLOSE(InformationGain::Evaluate(labels, c, weights), -1.0, 1e-5); + } +} + +/** + * The information gain of an empty vector is 0. + */ +BOOST_AUTO_TEST_CASE(InformationGainEmptyTest) +{ + arma::Row labels; + arma::rowvec weights = arma::ones(10); + for (size_t c = 1; c < 10; ++c) + { + BOOST_REQUIRE_SMALL(InformationGain::Evaluate(labels, c, weights), 1e-5); + BOOST_REQUIRE_SMALL(InformationGain::Evaluate(labels, c, weights), 1e-5); + } +} + +/** + * The information gain is log2(1/k) when splitting equal classes. + */ +BOOST_AUTO_TEST_CASE(InformationGainEvenSplitManyClassTest) +{ + arma::rowvec weights; + // Try with many different numbers of classes. + for (size_t c = 2; c < 30; ++c) + { + arma::Row labels(c); + for (size_t i = 0; i < c; ++i) + labels[i] = i; + + // Calculate information gain and make sure it is correct. + BOOST_REQUIRE_CLOSE(InformationGain::Evaluate(labels, c, weights), + std::log2(1.0 / c), 1e-5); + } +} + +/** + * Test the information gain with weighted labels + */ +BOOST_AUTO_TEST_CASE(InformationWithWeight) +{ + arma::Row labels(10); + arma::rowvec weights("1 1 1 1 1 0 0 0 0 0"); + for (size_t i = 0; i < 5; ++i) + labels[i] = 0; + for (size_t i = 5; i < 10; ++i) + labels[i] = 1; + + // Zero is not a good result as gain, but we just need to prove cacluation works. + BOOST_REQUIRE_CLOSE(InformationGain::Evaluate(labels, 2, weights), 0, 1e-5); + +} + + +/** + * The information gain should not be sensitive to the number of points. + */ +BOOST_AUTO_TEST_CASE(InformationGainManyPoints) +{ + for (size_t i = 1; i < 20; ++i) + { + const size_t numPoints = 100 * i; + arma::Row labels(numPoints); + arma::rowvec weights = arma::ones(numPoints); + for (size_t j = 0; j < numPoints / 2; ++j) + labels[j] = 0; + for (size_t j = numPoints / 2; j < numPoints; ++j) + labels[j] = 1; + + BOOST_REQUIRE_CLOSE(InformationGain::Evaluate(labels, 2, weights), -1.0, 1e-5); + // It should make no difference between weighted and no weight labels. + BOOST_REQUIRE_CLOSE(InformationGain::Evaluate(labels, 2, weights), -1.0, 1e-5); + } +} + +/** + * Check that the BestBinaryNumericSplit will split on an obviously splittable + * dimension. + */ +BOOST_AUTO_TEST_CASE(BestBinaryNumericSplitSimpleSplitTest) +{ + arma::vec values("0.0 0.1 0.2 0.3 0.4 0.5 0.6 0.7 0.8 0.9 1.0"); + arma::Row labels("0 0 0 0 0 1 1 1 1 1 1"); + arma::rowvec weights(labels.n_elem); + weights.ones(); + + arma::vec classProbabilities; + BestBinaryNumericSplit::template AuxiliarySplitInfo aux; + + // Call the method to do the splitting. + const double bestGain = GiniGain::Evaluate(labels, 2, weights); + const double gain = BestBinaryNumericSplit::SplitIfBetter(bestGain, + values, labels, 2, 3, weights, classProbabilities, aux); + const double weightedGain = BestBinaryNumericSplit::SplitIfBetter(bestGain, + values, labels, 2, 3, weights, classProbabilities, aux); + + // Make sure that a split was made. + BOOST_REQUIRE_GT(gain, bestGain); + + // Make sure weight works and make no different with no weighted one + BOOST_REQUIRE_EQUAL(gain, weightedGain); + + // The split is perfect, so we should be able to accomplish a gain of 0. + BOOST_REQUIRE_SMALL(gain, 1e-5); + + // The class probabilities, for this split, hold the splitting point, which + // should be between 4 and 5. + BOOST_REQUIRE_EQUAL(classProbabilities.n_elem, 1); + BOOST_REQUIRE_GT(classProbabilities[0], 0.4); + BOOST_REQUIRE_LT(classProbabilities[0], 0.5); +} + +/** + * Check that the BestBinaryNumericSplit won't split if not enough points are + * given. + */ +BOOST_AUTO_TEST_CASE(BestBinaryNumericSplitMinSamplesTest) +{ + arma::vec values("0.0 0.1 0.2 0.3 0.4 0.5 0.6 0.7 0.8 0.9 1.0"); + arma::Row labels("0 0 0 0 0 1 1 1 1 1 1"); + arma::rowvec weights(labels.n_elem); + + arma::vec classProbabilities; + BestBinaryNumericSplit::template AuxiliarySplitInfo aux; + + // Call the method to do the splitting. + const double bestGain = GiniGain::Evaluate(labels, 2, weights); + const double gain = BestBinaryNumericSplit::SplitIfBetter(bestGain, + values, labels, 2, 8, weights, classProbabilities, aux); + // This should make no difference because it won't split at all. + const double weightedGain = BestBinaryNumericSplit::SplitIfBetter(bestGain, + values, labels, 2, 8, weights, classProbabilities, aux); + + // Make sure that no split was made. + BOOST_REQUIRE_EQUAL(gain, bestGain); + BOOST_REQUIRE_EQUAL(gain, weightedGain); + BOOST_REQUIRE_EQUAL(classProbabilities.n_elem, 0); +} + +/** + * Check that the BestBinaryNumericSplit doesn't split a dimension that gives no + * gain. + */ +BOOST_AUTO_TEST_CASE(BestBinaryNumericSplitNoGainTest) +{ + arma::vec values(100); + arma::Row labels(100); + arma::rowvec weights; + for (size_t i = 0; i < 100; i += 2) + { + values[i] = i; + labels[i] = 0; + values[i + 1] = i; + labels[i + 1] = 1; + } + + arma::vec classProbabilities; + BestBinaryNumericSplit::template AuxiliarySplitInfo aux; + + // Call the method to do the splitting. + const double bestGain = GiniGain::Evaluate(labels, 2, weights); + const double gain = BestBinaryNumericSplit::SplitIfBetter(bestGain, + values, labels, 2, 10, weights, classProbabilities, aux); + + // Make sure there was no split. + BOOST_REQUIRE_EQUAL(gain, bestGain); + BOOST_REQUIRE_EQUAL(classProbabilities.n_elem, 0); +} + +/** + * Check that the AllCategoricalSplit will split when the split is obviously + * better. + */ +BOOST_AUTO_TEST_CASE(AllCategoricalSplitSimpleSplitTest) +{ + arma::vec values("0 0 0 1 1 1 2 2 2 3 3 3"); + arma::Row labels("0 0 0 2 2 2 1 1 1 2 2 2"); + arma::rowvec weights(labels.n_elem); + weights.ones(); + + arma::vec classProbabilities; + AllCategoricalSplit::template AuxiliarySplitInfo aux; + + // Call the method to do the splitting. + const double bestGain = GiniGain::Evaluate(labels, 3, weights); + const double gain = AllCategoricalSplit::SplitIfBetter(bestGain, + values, 4, labels, 3, 3, weights, classProbabilities, aux); + const double weightedGain = AllCategoricalSplit::SplitIfBetter(bestGain, + values, 4, labels, 3, 3, weights, classProbabilities, aux); + + // Make sure that a split was made. + BOOST_REQUIRE_GT(gain, bestGain); + + // Since the split is perfect, make sure the new gain is 0. + BOOST_REQUIRE_SMALL(gain, 1e-5); + + BOOST_REQUIRE_EQUAL(gain, weightedGain); + + // Make sure the class probabilities now hold the number of children. + BOOST_REQUIRE_EQUAL(classProbabilities.n_elem, 1); + BOOST_REQUIRE_EQUAL((size_t) classProbabilities[0], 4); +} + +/** + * Make sure that AllCategoricalSplit respects the minimum number of samples + * required to split. + */ +BOOST_AUTO_TEST_CASE(AllCategoricalSplitMinSamplesTest) +{ + arma::vec values("0 0 0 1 1 1 2 2 2 3 3 3"); + arma::Row labels("0 0 0 2 2 2 1 1 1 2 2 2"); + arma::rowvec weights(labels.n_elem); + weights.ones(); + + arma::vec classProbabilities; + AllCategoricalSplit::template AuxiliarySplitInfo aux; + + // Call the method to do the splitting. + const double bestGain = GiniGain::Evaluate(labels, 3, weights); + const double gain = AllCategoricalSplit::SplitIfBetter(bestGain, + values, 4, labels, 3, 4, weights, classProbabilities, aux); + + // Make sure it's not split. + BOOST_REQUIRE_EQUAL(gain, bestGain); + BOOST_REQUIRE_EQUAL(classProbabilities.n_elem, 0); +} + +/** + * Check that no split is made when it doesn't get us anything. + */ +BOOST_AUTO_TEST_CASE(AllCategoricalSplitNoGainTest) +{ + arma::vec values(300); + arma::Row labels(300); + arma::rowvec weights = arma::ones(300); + + for (size_t i = 0; i < 300; i += 3) + { + values[i] = (i / 3) % 10; + labels[i] = 0; + values[i + 1] = (i / 3) % 10; + labels[i + 1] = 1; + values[i + 2] = (i / 3) % 10; + labels[i + 2] = 2; + } + + arma::vec classProbabilities; + AllCategoricalSplit::template AuxiliarySplitInfo aux; + + // Call the method to do the splitting. + const double bestGain = GiniGain::Evaluate(labels, 3, weights); + const double gain = AllCategoricalSplit::SplitIfBetter(bestGain, + values, 10, labels, 3, 10, weights, classProbabilities, aux); + const double weightedGain = AllCategoricalSplit::SplitIfBetter(bestGain, + values, 10, labels, 3, 10, weights, classProbabilities, aux); + + // Make sure that there was no split. + BOOST_REQUIRE_EQUAL(gain, bestGain); + BOOST_REQUIRE_EQUAL(gain, weightedGain); + BOOST_REQUIRE_EQUAL(classProbabilities.n_elem, 0); +} + +/** + * A basic construction of the decision tree---ensure that we can create the + * tree and that it split at least once. + */ +BOOST_AUTO_TEST_CASE(BasicConstructionTest) +{ + arma::mat dataset(10, 1000, arma::fill::randu); + arma::Row labels(1000); + + for (size_t i = 0; i < 1000; ++i) + labels[i] = i % 3; // 3 classes. + + // Use default parameters. + DecisionTree<> d(dataset, labels, 3, 50); + + // Now require that we have some children. + BOOST_REQUIRE_GT(d.NumChildren(), 0); +} + +/** + * Construct a tree with weighted labels. + */ +BOOST_AUTO_TEST_CASE(BasicConstructionTestWithWeight) +{ + arma::mat dataset(10, 1000, arma::fill::randu); + arma::Row labels(1000); + arma::rowvec weights(labels.n_elem); + weights.ones(); + + for (size_t i = 0; i < 1000; ++i) + labels[i] = i % 3; // 3 classes. + + // Use default parameters. + DecisionTree<> wd(dataset, labels, 3, weights, 50); + DecisionTree<> d(dataset, labels, 3, 50); + + // Now require that we have some children. + BOOST_REQUIRE_GT(wd.NumChildren(), 0); + BOOST_REQUIRE_EQUAL(wd.NumChildren(), d.NumChildren()); +} + +/** + * Construct the decision tree on numeric data only and see that we can fit it + * exactly and achieve perfect performance on the training set. + */ +BOOST_AUTO_TEST_CASE(PerfectTrainingSet) +{ + // Completely random dataset with no structure. + arma::mat dataset(10, 1000, arma::fill::randu); + arma::Row labels(1000); + for (size_t i = 0; i < 1000; ++i) + labels[i] = i % 3; // 3 classes. + arma::rowvec weights(labels.n_elem); + weights.ones(); + + DecisionTree<> d(dataset, labels, 3, 1); // Minimum leaf size of 1. + + // Make sure that we can get perfect accuracy on the training set. + for (size_t i = 0; i < 1000; ++i) + { + size_t prediction; + arma::vec probabilities; + d.Classify(dataset.col(i), prediction, probabilities); + + BOOST_REQUIRE_EQUAL(prediction, labels[i]); + BOOST_REQUIRE_EQUAL(probabilities.n_elem, 3); + for (size_t j = 0; j < 3; ++j) + { + if (labels[i] == j) + BOOST_REQUIRE_CLOSE(probabilities[j], 1.0, 1e-5); + else + BOOST_REQUIRE_SMALL(probabilities[j], 1e-5); + } + } +} + +/** + * onstruct the decision tree with weighted labels + */ +BOOST_AUTO_TEST_CASE(PerfectTrainingSetWithWeight) +{ + // Completely random dataset with no structure. + arma::mat dataset(10, 1000, arma::fill::randu); + arma::Row labels(1000); + for (size_t i = 0; i < 1000; ++i) + labels[i] = i % 3; // 3 classes. + arma::rowvec weights(labels.n_elem); + weights.ones(); + + DecisionTree<> d(dataset, labels, 3, weights, 1); // Minimum leaf size of 1. + + // This part of code is dupliacte with no weighted one. + for (size_t i = 0; i < 1000; ++i) + { + size_t prediction; + arma::vec probabilities; + d.Classify(dataset.col(i), prediction, probabilities); + + BOOST_REQUIRE_EQUAL(prediction, labels[i]); + BOOST_REQUIRE_EQUAL(probabilities.n_elem, 3); + for (size_t j = 0; j < 3; ++j) + { + if (labels[i] == j) + BOOST_REQUIRE_CLOSE(probabilities[j], 1.0, 1e-5); + else + BOOST_REQUIRE_SMALL(probabilities[j], 1e-5); + } + } +} + + +/** + * Make sure class probabilities are computed correctly in the root node. + */ +BOOST_AUTO_TEST_CASE(ClassProbabilityTest) +{ + arma::mat dataset(5, 100, arma::fill::randu); + arma::Row labels(100); + for (size_t i = 0; i < 100; i += 2) + { + labels[i] = 0; + labels[i + 1] = 1; + } + + // Create a decision tree that can't split. + DecisionTree<> d(dataset, labels, 2, 1000); + + BOOST_REQUIRE_EQUAL(d.NumChildren(), 0); + + // Estimate a point's probabilities. + arma::vec probabilities; + size_t prediction; + d.Classify(dataset.col(0), prediction, probabilities); + + BOOST_REQUIRE_EQUAL(probabilities.n_elem, 2); + BOOST_REQUIRE_CLOSE(probabilities[0], 0.5, 1e-5); + BOOST_REQUIRE_CLOSE(probabilities[1], 0.5, 1e-5); +} + +/** + * Test that the decision tree generalizes reasonably. + */ +BOOST_AUTO_TEST_CASE(SimpleGeneralizationTest) +{ + arma::mat inputData; + if (!data::Load("vc2.csv", inputData)) + BOOST_FAIL("Cannot load test dataset vc2.csv!"); + + arma::Mat labels; + if (!data::Load("vc2_labels.txt", labels)) + BOOST_FAIL("Cannot load labels for vc2_labels.txt"); + + // init a weight martix + arma::mat weights = arma::ones>(labels.n_rows, labels.n_cols); + + // Build decision tree. + DecisionTree<> d(inputData, labels, 3, 10); // Leaf size of 10. + DecisionTree<> wd(inputData, labels, 3, weights, 10); // Leaf size of 10 + + // Load testing data. + arma::mat testData; + if (!data::Load("vc2_test.csv", testData)) + BOOST_FAIL("Cannot load test dataset vc2_test.csv!"); + + arma::Mat trueTestLabels; + if (!data::Load("vc2_test_labels.txt", trueTestLabels)) + BOOST_FAIL("Cannot load labels for vc2_test_labels.txt"); + + // Get the predicted test labels. + arma::Row predictions; + d.Classify(testData, predictions); + + BOOST_REQUIRE_EQUAL(predictions.n_elem, testData.n_cols); + + // Figure out the accuracy. + double correct = 0.0; + for (size_t i = 0; i < predictions.n_elem; ++i) + if (predictions[i] == trueTestLabels[i]) + ++correct; + correct /= predictions.n_elem; + + BOOST_REQUIRE_GT(correct, 0.75); + + // reset the prediction + predictions.zeros(); + wd.Classify(testData, predictions); + + BOOST_REQUIRE_EQUAL(predictions.n_elem, testData.n_cols); + + // Figure out the accuracy. + double wdcorrect = 0.0; + for (size_t i = 0; i < predictions.n_elem; ++i) + if (predictions[i] == trueTestLabels[i]) + ++wdcorrect; + wdcorrect /= predictions.n_elem; + + BOOST_REQUIRE_GT(wdcorrect, 0.75); +} + +/** + * Test that we can build a decision tree on a simple categorical dataset. + */ +BOOST_AUTO_TEST_CASE(CategoricalBuildTest) +{ + arma::mat d; + arma::Row l; + data::DatasetInfo di; + MockCategoricalData(d, l, di); + + // Split into a training set and a test set. + arma::mat trainingData = d.cols(0, 4999); + arma::mat testData = d.cols(5000, 9999); + arma::Row trainingLabels = l.subvec(0, 4999); + arma::Row testLabels = l.subvec(5000, 9999); // Build the tree. DecisionTree<> tree(trainingData, di, trainingLabels, 5, 10); @@ -576,6 +770,42 @@ BOOST_AUTO_TEST_CASE(CategoricalBuildTest) BOOST_REQUIRE_GT(correctPct, 0.70); } +/** + * Test that we can build a decision tree with weighted on a simple categorical dataset. + */ +BOOST_AUTO_TEST_CASE(CategoricalBuildTestWithWeight) +{ + arma::mat d; + arma::Row l; + data::DatasetInfo di; + MockCategoricalData(d, l, di); + + // Split into a training set and a test set. + arma::mat trainingData = d.cols(0, 4999); + arma::mat testData = d.cols(5000, 9999); + arma::Row trainingLabels = l.subvec(0, 4999); + arma::Row testLabels = l.subvec(5000, 9999); + + arma::Row weights = arma::ones>(trainingLabels.n_elem); + + // Build the tree. + DecisionTree<> tree(trainingData, di, trainingLabels, 5, weights, 10); + + // Now evaluate the accuracy of the tree. + arma::Row predictions; + tree.Classify(testData, predictions); + + BOOST_REQUIRE_EQUAL(predictions.n_elem, testData.n_cols); + size_t correct = 0; + for (size_t i = 0; i < testData.n_cols; ++i) + if (testLabels[i] == predictions[i]) + ++correct; + + // Make sure we got at least 70% accuracy. + const double correctPct = double(correct) / double(testData.n_cols); + BOOST_REQUIRE_GT(correctPct, 0.70); +} + /** * Make sure that when we ask for a decision stump, we get one. */ From 9e5d788c7f7fedf09e52fb781a59503918d1a13f Mon Sep 17 00:00:00 2001 From: Marcus Edel Date: Mon, 24 Apr 2017 13:14:42 +0200 Subject: [PATCH 04/84] Remove unnecessary parameters from the policy update function. --- .../minibatch_sgd/decay_policies/no_decay.hpp | 16 ++++------ .../minibatch_sgd/minibatch_sgd.hpp | 24 ++++++++++---- .../minibatch_sgd/minibatch_sgd_impl.hpp | 32 +++++++++---------- .../core/optimizers/sgdr/cyclical_decay.hpp | 22 +++++++------ src/mlpack/core/optimizers/sgdr/sgdr.hpp | 21 ++++-------- .../optimizers/sgdr/snapshot_ensembles.hpp | 22 +++++++------ 6 files changed, 72 insertions(+), 65 deletions(-) diff --git a/src/mlpack/core/optimizers/minibatch_sgd/decay_policies/no_decay.hpp b/src/mlpack/core/optimizers/minibatch_sgd/decay_policies/no_decay.hpp index 9bfdb90900..787a967420 100644 --- a/src/mlpack/core/optimizers/minibatch_sgd/decay_policies/no_decay.hpp +++ b/src/mlpack/core/optimizers/minibatch_sgd/decay_policies/no_decay.hpp @@ -34,15 +34,13 @@ class NoDecay /** * This function is called in each iteration after the policy update. * - * @param stepSize The stepSize to be adjusted. - * @param epoch The current epoch. - * @param batch The current batch. - * @param iterate Function parameters. - */ - void Update(double& /* stepSize */, - const size_t /* epoch */, - const size_t /* batch */, - const arma::mat& /* iterate */) + * @param iterate Parameters that minimize the function. + * @param stepSize Step size to be used for the given iteration. + * @param gradient The gradient matrix. + */ + void Update(arma::mat& /* iterate */, + const double /* stepSize */, + const arma::mat& /* gradient */) { // Nothing to do here. } diff --git a/src/mlpack/core/optimizers/minibatch_sgd/minibatch_sgd.hpp b/src/mlpack/core/optimizers/minibatch_sgd/minibatch_sgd.hpp index 9cb367e65e..1eba57f522 100644 --- a/src/mlpack/core/optimizers/minibatch_sgd/minibatch_sgd.hpp +++ b/src/mlpack/core/optimizers/minibatch_sgd/minibatch_sgd.hpp @@ -109,13 +109,13 @@ class MiniBatchSGDType * @param decayPolicy Instantiated decay policy used to adjust the step size. */ MiniBatchSGDType(DecomposableFunctionType& function, - const size_t batchSize = 1000, - const double stepSize = 0.01, - const size_t maxIterations = 100000, - const double tolerance = 1e-5, - const bool shuffle = true, - const UpdatePolicyType& updatePolicy = UpdatePolicyType(), - const DecayPolicyType& decayPolicy = DecayPolicyType()); + const size_t batchSize = 1000, + const double stepSize = 0.01, + const size_t maxIterations = 100000, + const double tolerance = 1e-5, + const bool shuffle = true, + const UpdatePolicyType& updatePolicy = UpdatePolicyType(), + const DecayPolicyType& decayPolicy = DecayPolicyType()); /** * Optimize the given function using mini-batch SGD. The given starting point @@ -157,6 +157,16 @@ class MiniBatchSGDType //! Modify whether or not the individual functions are shuffled. bool& Shuffle() { return shuffle; } + //! Get the update policy. + UpdatePolicyType UpdatePolicy() const { return updatePolicy; } + //! Modify the update policy. + UpdatePolicyType& UpdatePolicy() { return updatePolicy; } + + //! Get the decay policy. + DecayPolicyType DecayPolicy() const { return decayPolicy; } + //! Modify the decay policy. + DecayPolicyType& DecayPolicy() { return decayPolicy; } + private: //! The instantiated function. DecomposableFunctionType& function; diff --git a/src/mlpack/core/optimizers/minibatch_sgd/minibatch_sgd_impl.hpp b/src/mlpack/core/optimizers/minibatch_sgd/minibatch_sgd_impl.hpp index 6f0a696dee..620f9bcf06 100644 --- a/src/mlpack/core/optimizers/minibatch_sgd/minibatch_sgd_impl.hpp +++ b/src/mlpack/core/optimizers/minibatch_sgd/minibatch_sgd_impl.hpp @@ -28,21 +28,21 @@ MiniBatchSGDType< UpdatePolicyType, DecayPolicyType >::MiniBatchSGDType(DecomposableFunctionType& function, - const size_t batchSize, - const double stepSize, - const size_t maxIterations, - const double tolerance, - const bool shuffle, - const UpdatePolicyType& updatePolicy, - const DecayPolicyType& decayPolicy) : - function(function), - batchSize(batchSize), - stepSize(stepSize), - maxIterations(maxIterations), - tolerance(tolerance), - shuffle(shuffle), - updatePolicy(updatePolicy), - decayPolicy(decayPolicy) + const size_t batchSize, + const double stepSize, + const size_t maxIterations, + const double tolerance, + const bool shuffle, + const UpdatePolicyType& updatePolicy, + const DecayPolicyType& decayPolicy) : + function(function), + batchSize(batchSize), + stepSize(stepSize), + maxIterations(maxIterations), + tolerance(tolerance), + shuffle(shuffle), + updatePolicy(updatePolicy), + decayPolicy(decayPolicy) { /* Nothing to do. */ } //! Optimize the function (minimize). @@ -166,7 +166,7 @@ double MiniBatchSGDType< } // Now update the learning rate if requested by the user. - decayPolicy.Update(stepSize, i - 1, currentBatch, iterate); + decayPolicy.Update(iterate, stepSize, gradient); } Log::Info << "Mini-batch SGD: maximum iterations (" << maxIterations << ") " diff --git a/src/mlpack/core/optimizers/sgdr/cyclical_decay.hpp b/src/mlpack/core/optimizers/sgdr/cyclical_decay.hpp index ae8003ff5c..9cd9e437c5 100644 --- a/src/mlpack/core/optimizers/sgdr/cyclical_decay.hpp +++ b/src/mlpack/core/optimizers/sgdr/cyclical_decay.hpp @@ -63,21 +63,20 @@ class CyclicalDecay constStepSize(stepSize), nextRestart(epochRestart), batchRestart(0), - epochBatches(numFunctions / (double) batchSize) + epochBatches(numFunctions / (double) batchSize), + epoch(0) { /* Nothing to do here */ } /** * This function is called in each iteration after the policy update. * - * @param stepSize The stepSize to be adjusted. - * @param epoch The current epoch. - * @param batch The current batch. - * @param iterate Function parameters. + * @param iterate Parameters that minimize the function. + * @param stepSize Step size to be used for the given iteration. + * @param gradient The gradient matrix. */ - void Update(double& stepSize, - const size_t epoch, - const size_t /* batch */, - const arma::mat& /* iterate */) + void Update(arma::mat& /* iterate */, + double& stepSize, + const arma::mat& /* gradient */) { // Time to adjust the step size. if (epoch >= epochRestart) @@ -101,6 +100,8 @@ class CyclicalDecay // Update the time for the next restart. nextRestart += epochRestart; } + + epoch++; } //! Get the step size. @@ -130,6 +131,9 @@ class CyclicalDecay //! Locally-stored restart fraction. double epochBatches; + + //! Locally-stored epoch. + size_t epoch; }; } // namespace optimization diff --git a/src/mlpack/core/optimizers/sgdr/sgdr.hpp b/src/mlpack/core/optimizers/sgdr/sgdr.hpp index 6d02017974..1bd762bd12 100644 --- a/src/mlpack/core/optimizers/sgdr/sgdr.hpp +++ b/src/mlpack/core/optimizers/sgdr/sgdr.hpp @@ -89,6 +89,7 @@ class SGDR * @param shuffle If true, the mini-batch order is shuffled; otherwise, each * mini-batch is visited in linear order. * @param snapshots Maximum number of snapshots. + * @param accumulate Accumulate the snapshot parameter (default true). * @param updatePolicy Instantiated update policy used to adjust the given * parameters. */ @@ -102,6 +103,7 @@ class SGDR const double tolerance = 1e-5, const bool shuffle = true, const size_t snapshots = 5, + const bool accumulate = true, const UpdatePolicyType& updatePolicy = UpdatePolicyType(), const typename std::enable_if_t::value>* junk = 0); @@ -149,7 +151,6 @@ class SGDR */ template double Optimize(arma::mat& iterate, - const bool accumulate = true, const typename std::enable_if_t::value>* junk = 0); @@ -205,14 +206,14 @@ class SGDR typename std::enable_if< std::is_same::value, std::vector >::type - Snapshots() const { return decayPolicy.Snapshots(); } + Snapshots() const { return optimizer.DecayPolicy().Snapshots(); } //! Modify the snapshots. template typename std::enable_if< std::is_same::value, std::vector& >::type - Snapshots() { return decayPolicy.snapshots(); } + Snapshots() { return optimizer.DecayPolicy().Snapshots(); } //! Get the snapshots. std::vector Snapshots() const { return junk; } @@ -226,18 +227,8 @@ class SGDR //! The size of each mini-batch. size_t batchSize; - //! The maximum number of allowed iterations. - size_t maxIterations; - - //! The tolerance for termination. - double tolerance; - - //! Controls whether or not the individual functions are shuffled when - //! iterating. - bool shuffle; - - //! The decay method used to update the step size in each iteration. - DecayPolicyType decayPolicy; + //! Whether or not to accumulate the snapshots. + bool accumulate; //! Locally-stored optimizer instance. OptimizerType optimizer; diff --git a/src/mlpack/core/optimizers/sgdr/snapshot_ensembles.hpp b/src/mlpack/core/optimizers/sgdr/snapshot_ensembles.hpp index ac4de679f2..7c9a66c408 100644 --- a/src/mlpack/core/optimizers/sgdr/snapshot_ensembles.hpp +++ b/src/mlpack/core/optimizers/sgdr/snapshot_ensembles.hpp @@ -68,7 +68,8 @@ class SnapshotEnsembles constStepSize(stepSize), nextRestart(epochRestart), batchRestart(0), - epochBatches(numFunctions / (double) batchSize) + epochBatches(numFunctions / (double) batchSize), + epoch(0) { snapshotEpochs = 0; for (size_t i = 0, er = epochRestart, nr = nextRestart; @@ -89,15 +90,13 @@ class SnapshotEnsembles /** * This function is called in each iteration after the policy update. * - * @param stepSize The stepSize to be adjusted. - * @param epoch The current epoch. - * @param batch The current batch. - * @param iterate Function parameters. + * @param iterate Parameters that minimize the function. + * @param stepSize Step size to be used for the given iteration. + * @param gradient The gradient matrix. */ - void Update(double& stepSize, - const size_t epoch, - const size_t /* batch */, - const arma::mat& iterate) + void Update(arma::mat& iterate, + double& stepSize, + const arma::mat& /* gradient */) { // Time to adjust the step size. if (epoch >= epochRestart) @@ -127,6 +126,8 @@ class SnapshotEnsembles // Update the time for the next restart. nextRestart += epochRestart; } + + epoch++; } //! Get the step size. @@ -163,6 +164,9 @@ class SnapshotEnsembles //! Locally-stored restart fraction. double epochBatches; + //! Locally-stored epoch. + size_t epoch; + //! Epochs where a new snapshot is created. size_t snapshotEpochs; From b842a54ba8710375db238efd5248eb8b0ee94e8e Mon Sep 17 00:00:00 2001 From: Marcus Edel Date: Mon, 24 Apr 2017 13:16:04 +0200 Subject: [PATCH 05/84] Use policy instantiation from optimizer object. --- src/mlpack/core/optimizers/sgdr/sgdr_impl.hpp | 50 ++++++++++--------- 1 file changed, 26 insertions(+), 24 deletions(-) diff --git a/src/mlpack/core/optimizers/sgdr/sgdr_impl.hpp b/src/mlpack/core/optimizers/sgdr/sgdr_impl.hpp index 6f8add364f..78d884afc1 100644 --- a/src/mlpack/core/optimizers/sgdr/sgdr_impl.hpp +++ b/src/mlpack/core/optimizers/sgdr/sgdr_impl.hpp @@ -37,18 +37,13 @@ SGDR< const double tolerance, const bool shuffle, const size_t snapshots, + const bool accumulate, const UpdatePolicyType& updatePolicy, const typename std::enable_if_t::value>* /* junk */) : function(function), batchSize(batchSize), - decayPolicy(SnapshotEnsembles(epochRestart, - multFactor, - stepSize, - batchSize, - function.NumFunctions(), - maxIterations, - snapshots)), + accumulate(accumulate), optimizer(OptimizerType(function, batchSize, stepSize, @@ -56,7 +51,14 @@ SGDR< tolerance, shuffle, updatePolicy, - decayPolicy)) + SnapshotEnsembles( + epochRestart, + multFactor, + stepSize, + batchSize, + function.NumFunctions(), + maxIterations, + snapshots))) { /* Nothing to do here */ } @@ -84,11 +86,7 @@ SGDR< PolicyType, CyclicalDecay>::value>* /* junk */) : function(function), batchSize(batchSize), - decayPolicy(CyclicalDecay(epochRestart, - multFactor, - stepSize, - batchSize, - function.NumFunctions())), + accumulate(true), optimizer(OptimizerType(function, batchSize, stepSize, @@ -96,7 +94,12 @@ SGDR< tolerance, shuffle, updatePolicy, - decayPolicy)) + CyclicalDecay( + epochRestart, + multFactor, + stepSize, + batchSize, + function.NumFunctions()))) { /* Nothing to do here */ } @@ -112,15 +115,14 @@ double SGDR< UpdatePolicyType, DecayPolicyType >::Optimize(arma::mat& iterate, - const bool accumulate, const typename std::enable_if_t::value>* /* junk */) { // If a user changed the step size he hasn't update the step size of the // cyclical decay instantiation, so we have to do here. - if (optimizer.StepSize() != decayPolicy.StepSize()) + if (optimizer.StepSize() != optimizer.DecayPolicy().StepSize()) { - decayPolicy.StepSize() = optimizer.StepSize(); + optimizer.DecayPolicy().StepSize() = optimizer.StepSize(); } // If a user changed the batch size we have to update the restart fraction @@ -128,7 +130,7 @@ double SGDR< if (optimizer.BatchSize() != batchSize) { batchSize = optimizer.BatchSize(); - decayPolicy.EpochBatches() = function.NumFunctions() / + optimizer.DecayPolicy().EpochBatches() = function.NumFunctions() / double(batchSize); } @@ -137,11 +139,11 @@ double SGDR< // Accumulate snapshots. if (accumulate) { - for (size_t i = 0; i < decayPolicy.Snapshots().size(); ++i) + for (size_t i = 0; i < optimizer.DecayPolicy().Snapshots().size(); ++i) { - iterate += decayPolicy.Snapshots()[i]; + iterate += optimizer.DecayPolicy().Snapshots()[i]; } - iterate /= (decayPolicy.Snapshots().size() + 1); + iterate /= (optimizer.DecayPolicy().Snapshots().size() + 1); // Calculate final objective. overallObjective = 0; @@ -168,9 +170,9 @@ double SGDR< { // If a user changed the step size he hasn't update the step size of the // cyclical decay instantiation, so we have to do here. - if (optimizer.StepSize() != decayPolicy.StepSize()) + if (optimizer.StepSize() != optimizer.DecayPolicy().StepSize()) { - decayPolicy.StepSize() = optimizer.StepSize(); + optimizer.DecayPolicy().StepSize() = optimizer.StepSize(); } // If a user changed the batch size we have to update the restart fraction @@ -178,7 +180,7 @@ double SGDR< if (optimizer.BatchSize() != batchSize) { batchSize = optimizer.BatchSize(); - decayPolicy.EpochBatches() = function.NumFunctions() / + optimizer.DecayPolicy().EpochBatches() = function.NumFunctions() / double(batchSize); } From 191782d04647ce2f066b4e7fc901d025b5eb32bd Mon Sep 17 00:00:00 2001 From: Marcus Edel Date: Mon, 24 Apr 2017 13:17:02 +0200 Subject: [PATCH 06/84] Build sgdr and snapshot ensembles test. --- src/mlpack/tests/CMakeLists.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/mlpack/tests/CMakeLists.txt b/src/mlpack/tests/CMakeLists.txt index 2d298c9046..a49b9bb541 100644 --- a/src/mlpack/tests/CMakeLists.txt +++ b/src/mlpack/tests/CMakeLists.txt @@ -78,6 +78,8 @@ add_executable(mlpack_test sa_test.cpp sdp_primal_dual_test.cpp sgd_test.cpp + sgdr_test.cpp + snapshot_ensembles.cpp serialization.hpp serialization.cpp serialization_test.cpp From c28ad218b994cca4dfa57a467a400efb7879ed53 Mon Sep 17 00:00:00 2001 From: Marcus Edel Date: Tue, 25 Apr 2017 13:07:11 +0200 Subject: [PATCH 07/84] Add test source files. --- src/mlpack/tests/sgdr_test.cpp | 131 +++++++++++++++++++++++ src/mlpack/tests/snapshot_ensembles.cpp | 135 ++++++++++++++++++++++++ 2 files changed, 266 insertions(+) create mode 100644 src/mlpack/tests/sgdr_test.cpp create mode 100644 src/mlpack/tests/snapshot_ensembles.cpp diff --git a/src/mlpack/tests/sgdr_test.cpp b/src/mlpack/tests/sgdr_test.cpp new file mode 100644 index 0000000000..f8190e05dd --- /dev/null +++ b/src/mlpack/tests/sgdr_test.cpp @@ -0,0 +1,131 @@ +/** + * @file sgdr_test.cpp + * @author Marcus Edel + * + * Test file for SGDR. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#include +#include +#include +#include + +#include +#include "test_tools.hpp" + +using namespace std; +using namespace arma; +using namespace mlpack; +using namespace mlpack::optimization; + +using namespace mlpack::distribution; +using namespace mlpack::regression; + +BOOST_AUTO_TEST_SUITE(SGDRTest); + +/* + * Test that the step size resets after a specified number of epochs. + */ +BOOST_AUTO_TEST_CASE(CyclicalResetTest) +{ + const double stepSize = 0.5; + arma::mat iterate; + + // Now run cyclical decay policy with a couple of multiplicators and initial + // restarts. + for (size_t restart = 5; restart < 100; restart += 10) + { + for (size_t mult = 2; mult < 5; ++mult) + { + double epochStepSize = stepSize; + + CyclicalDecay cyclicalDecay(restart, double(mult), stepSize, 10, 1000); + + // Create all restart epochs. + arma::Col nextRestart(1000 / 10 / mult); + nextRestart(0) = restart; + for (size_t j = 1; j < nextRestart.n_elem; ++j) + nextRestart(j) = nextRestart(j - 1) * mult; + + for (size_t i = 0; i < 1000; ++i) + { + cyclicalDecay.Update(iterate, epochStepSize, iterate); + if (i <= restart || arma::accu(arma::find(nextRestart == i)) > 0) + { + BOOST_CHECK_EQUAL(epochStepSize, stepSize); + } + } + } + } +} + +/** + * Run SGDR on logistic regression and make sure the results are acceptable. + */ +BOOST_AUTO_TEST_CASE(LogisticRegressionTest) +{ + // Generate a two-Gaussian dataset. + GaussianDistribution g1(arma::vec("1.0 1.0 1.0"), arma::eye(3, 3)); + GaussianDistribution g2(arma::vec("9.0 9.0 9.0"), arma::eye(3, 3)); + + arma::mat data(3, 1000); + arma::Row responses(1000); + for (size_t i = 0; i < 500; ++i) + { + data.col(i) = g1.Random(); + responses[i] = 0; + } + for (size_t i = 500; i < 1000; ++i) + { + data.col(i) = g2.Random(); + responses[i] = 1; + } + + // Shuffle the dataset. + arma::uvec indices = arma::shuffle(arma::linspace(0, + data.n_cols - 1, data.n_cols)); + arma::mat shuffledData(3, 1000); + arma::Row shuffledResponses(1000); + for (size_t i = 0; i < data.n_cols; ++i) + { + shuffledData.col(i) = data.col(indices[i]); + shuffledResponses[i] = responses[indices[i]]; + } + + // Create a test set. + arma::mat testData(3, 1000); + arma::Row testResponses(1000); + for (size_t i = 0; i < 500; ++i) + { + testData.col(i) = g1.Random(); + testResponses[i] = 0; + } + for (size_t i = 500; i < 1000; ++i) + { + testData.col(i) = g2.Random(); + testResponses[i] = 1; + } + + // Now run SGDR with a couple of batch sizes. + for (size_t batchSize = 5; batchSize < 50; batchSize += 5) + { + LogisticRegression<> lr(shuffledData.n_rows, 0.5); + + LogisticRegressionFunction<> lrf(shuffledData, shuffledResponses, 0.5); + SGDR > sgdr(lrf, 50, 2.0, batchSize); + lr.Train(sgdr); + + // Ensure that the error is close to zero. + const double acc = lr.ComputeAccuracy(data, responses); + BOOST_REQUIRE_CLOSE(acc, 100.0, 0.3); // 0.3% error tolerance. + + const double testAcc = lr.ComputeAccuracy(testData, testResponses); + BOOST_REQUIRE_CLOSE(testAcc, 100.0, 0.6); // 0.6% error tolerance. + } +} + +BOOST_AUTO_TEST_SUITE_END(); diff --git a/src/mlpack/tests/snapshot_ensembles.cpp b/src/mlpack/tests/snapshot_ensembles.cpp new file mode 100644 index 0000000000..cd0f04e361 --- /dev/null +++ b/src/mlpack/tests/snapshot_ensembles.cpp @@ -0,0 +1,135 @@ +/** + * @file snapshot_ensembles.cpp + * @author Marcus Edel + * + * Test file for SGDR with snapshot ensembles. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#include +#include +#include +#include + +#include +#include "test_tools.hpp" + +using namespace std; +using namespace arma; +using namespace mlpack; +using namespace mlpack::optimization; + +using namespace mlpack::distribution; +using namespace mlpack::regression; + +BOOST_AUTO_TEST_SUITE(SnapshotEnsemblesTest); + +/* + * Test that the step size resets after a specified number of epochs. + */ +BOOST_AUTO_TEST_CASE(SnapshotEnsemblesResetTest) +{ + const double stepSize = 0.5; + arma::mat iterate; + + // Now run cyclical decay policy with a couple of multiplicators and initial + // restarts. + for (size_t restart = 5; restart < 100; restart += 10) + { + for (size_t mult = 2; mult < 5; ++mult) + { + double epochStepSize = stepSize; + + SnapshotEnsembles snapshotEnsembles(restart, double(mult), stepSize, + 10, 1000, 1000, 2); + + // Create all restart epochs. + arma::Col nextRestart(1000 / 10 / mult); + nextRestart(0) = restart; + for (size_t j = 1; j < nextRestart.n_elem; ++j) + nextRestart(j) = nextRestart(j - 1) * mult; + + for (size_t i = 0; i < 1000; ++i) + { + snapshotEnsembles.Update(iterate, epochStepSize, iterate); + if (i <= restart || arma::accu(arma::find(nextRestart == i)) > 0) + { + BOOST_CHECK_EQUAL(epochStepSize, stepSize); + } + } + + BOOST_CHECK_EQUAL(snapshotEnsembles.Snapshots().size(), 2); + } + } +} + +/** + * Run SGDR with snapshot ensembles on logistic regression and make sure the + * results are acceptable. + */ +BOOST_AUTO_TEST_CASE(LogisticRegressionTest) +{ + // Generate a two-Gaussian dataset. + GaussianDistribution g1(arma::vec("1.0 1.0 1.0"), arma::eye(3, 3)); + GaussianDistribution g2(arma::vec("9.0 9.0 9.0"), arma::eye(3, 3)); + + arma::mat data(3, 1000); + arma::Row responses(1000); + for (size_t i = 0; i < 500; ++i) + { + data.col(i) = g1.Random(); + responses[i] = 0; + } + for (size_t i = 500; i < 1000; ++i) + { + data.col(i) = g2.Random(); + responses[i] = 1; + } + + // Shuffle the dataset. + arma::uvec indices = arma::shuffle(arma::linspace(0, + data.n_cols - 1, data.n_cols)); + arma::mat shuffledData(3, 1000); + arma::Row shuffledResponses(1000); + for (size_t i = 0; i < data.n_cols; ++i) + { + shuffledData.col(i) = data.col(indices[i]); + shuffledResponses[i] = responses[indices[i]]; + } + + // Create a test set. + arma::mat testData(3, 1000); + arma::Row testResponses(1000); + for (size_t i = 0; i < 500; ++i) + { + testData.col(i) = g1.Random(); + testResponses[i] = 0; + } + for (size_t i = 500; i < 1000; ++i) + { + testData.col(i) = g2.Random(); + testResponses[i] = 1; + } + + // Now run SGDR with snapshot ensembles on a couple of batch sizes. + for (size_t batchSize = 5; batchSize < 50; batchSize += 5) + { + LogisticRegression<> lr(shuffledData.n_rows, 0.5); + + LogisticRegressionFunction<> lrf(shuffledData, shuffledResponses, 0.5); + SnapshotSGDR > sgdr(lrf, 50, 2.0, batchSize); + lr.Train(sgdr); + + // Ensure that the error is close to zero. + const double acc = lr.ComputeAccuracy(data, responses); + BOOST_REQUIRE_CLOSE(acc, 100.0, 0.3); // 0.3% error tolerance. + + const double testAcc = lr.ComputeAccuracy(testData, testResponses); + BOOST_REQUIRE_CLOSE(testAcc, 100.0, 0.6); // 0.6% error tolerance. + } +} + +BOOST_AUTO_TEST_SUITE_END(); From a1cb41af63c969b833da1c1e156c2c839cfd18e1 Mon Sep 17 00:00:00 2001 From: Marcus Edel Date: Mon, 1 May 2017 18:08:25 +0200 Subject: [PATCH 08/84] Remove misleading comments. --- .../core/optimizers/minibatch_sgd/decay_policies/no_decay.hpp | 4 +--- src/mlpack/core/optimizers/sgdr/cyclical_decay.hpp | 2 -- src/mlpack/core/optimizers/sgdr/snapshot_ensembles.hpp | 2 -- 3 files changed, 1 insertion(+), 7 deletions(-) diff --git a/src/mlpack/core/optimizers/minibatch_sgd/decay_policies/no_decay.hpp b/src/mlpack/core/optimizers/minibatch_sgd/decay_policies/no_decay.hpp index 787a967420..3c96a37fe8 100644 --- a/src/mlpack/core/optimizers/minibatch_sgd/decay_policies/no_decay.hpp +++ b/src/mlpack/core/optimizers/minibatch_sgd/decay_policies/no_decay.hpp @@ -26,8 +26,6 @@ class NoDecay public: /** * This constructor is called before the first iteration. - * - * @param node Node which this corresponds to. */ NoDecay() { } @@ -39,7 +37,7 @@ class NoDecay * @param gradient The gradient matrix. */ void Update(arma::mat& /* iterate */, - const double /* stepSize */, + double& /* stepSize */, const arma::mat& /* gradient */) { // Nothing to do here. diff --git a/src/mlpack/core/optimizers/sgdr/cyclical_decay.hpp b/src/mlpack/core/optimizers/sgdr/cyclical_decay.hpp index 9cd9e437c5..482e66cc0e 100644 --- a/src/mlpack/core/optimizers/sgdr/cyclical_decay.hpp +++ b/src/mlpack/core/optimizers/sgdr/cyclical_decay.hpp @@ -6,8 +6,6 @@ * "SGDR: Stochastic Gradient Descent with Warm Restarts" by * I. Loshchilov et al. * - * You should define your own decay update that looks like EmptyDecay. - * * mlpack is free software; you may redistribute it and/or modify it under the * terms of the 3-clause BSD license. You should have received a copy of the * 3-clause BSD license along with mlpack. If not, see diff --git a/src/mlpack/core/optimizers/sgdr/snapshot_ensembles.hpp b/src/mlpack/core/optimizers/sgdr/snapshot_ensembles.hpp index 7c9a66c408..12a50612dc 100644 --- a/src/mlpack/core/optimizers/sgdr/snapshot_ensembles.hpp +++ b/src/mlpack/core/optimizers/sgdr/snapshot_ensembles.hpp @@ -5,8 +5,6 @@ * Definition of the Snapshot ensembles technique described in: * "Snapshot ensembles: Train 1, get m for free" by G. Huang et al. * - * You should define your own decay update that looks like EmptyDecay. - * * mlpack is free software; you may redistribute it and/or modify it under the * terms of the 3-clause BSD license. You should have received a copy of the * 3-clause BSD license along with mlpack. If not, see From 2f9049c2b2b3a43a67b66f5875c78769acd76d72 Mon Sep 17 00:00:00 2001 From: Marcus Edel Date: Mon, 8 May 2017 17:36:15 +0200 Subject: [PATCH 09/84] Split SGDR and SnapshotSGDR into two seperate classes. --- .../core/optimizers/sgdr/CMakeLists.txt | 2 + .../core/optimizers/sgdr/cyclical_decay.hpp | 1 + src/mlpack/core/optimizers/sgdr/sgdr.hpp | 125 ++------------- src/mlpack/core/optimizers/sgdr/sgdr_impl.hpp | 146 ++---------------- src/mlpack/tests/snapshot_ensembles.cpp | 2 +- 5 files changed, 28 insertions(+), 248 deletions(-) diff --git a/src/mlpack/core/optimizers/sgdr/CMakeLists.txt b/src/mlpack/core/optimizers/sgdr/CMakeLists.txt index cdb858beda..5c3e185bff 100644 --- a/src/mlpack/core/optimizers/sgdr/CMakeLists.txt +++ b/src/mlpack/core/optimizers/sgdr/CMakeLists.txt @@ -3,6 +3,8 @@ set(SOURCES sgdr.hpp sgdr_impl.hpp snapshot_ensembles.hpp + snapshot_sgdr.hpp + snapshot_sgdr_impl.hpp ) set(DIR_SRCS) diff --git a/src/mlpack/core/optimizers/sgdr/cyclical_decay.hpp b/src/mlpack/core/optimizers/sgdr/cyclical_decay.hpp index 482e66cc0e..6aeca8f324 100644 --- a/src/mlpack/core/optimizers/sgdr/cyclical_decay.hpp +++ b/src/mlpack/core/optimizers/sgdr/cyclical_decay.hpp @@ -111,6 +111,7 @@ class CyclicalDecay double EpochBatches() const { return epochBatches; } //! Modify the restart fraction. double& EpochBatches() { return epochBatches; } + private: //! Epoch where decay is applied. size_t epochRestart; diff --git a/src/mlpack/core/optimizers/sgdr/sgdr.hpp b/src/mlpack/core/optimizers/sgdr/sgdr.hpp index 1bd762bd12..19bbfaea80 100644 --- a/src/mlpack/core/optimizers/sgdr/sgdr.hpp +++ b/src/mlpack/core/optimizers/sgdr/sgdr.hpp @@ -4,8 +4,7 @@ * * Definition of the Stochastic Gradient Descent with Restarts (SGDR) as * described in: "SGDR: Stochastic Gradient Descent with Warm Restarts" by - * I. Loshchilov et al and the Snapshot ensembles technique described in: - * "Snapshot ensembles: Train 1, get m for free" by G. Huang et al. + * I. Loshchilov et al. * * mlpack is free software; you may redistribute it and/or modify it under the * terms of the 3-clause BSD license. You should have received a copy of the @@ -19,8 +18,7 @@ #include #include -#include -#include +#include "cyclical_decay.hpp" namespace mlpack { namespace optimization { @@ -28,7 +26,7 @@ namespace optimization { /** * This class is based on Mini-batch Stochastic Gradient Descent class and * simulates a new warm-started run/restart once a number of epochs are - * performed this class also implements the Snapshot ensembles technique. + * performed. * * For more information, please refer to: * @@ -41,72 +39,22 @@ namespace optimization { * } * @endcode * - * @code - * @inproceedings{Huang2017, - * title = {Snapshot ensembles: Train 1, get m for free}, - * author = {Gao Huang, Yixuan Li, Geoff Pleiss, Zhuang Liu, - * John E. Hopcroft, and Kilian Q. Weinberger}, - * booktitle = {Proceedings of the International Conference on Learning - * Representations (ICLR)}, - * year = {2017} - * } - * @endcode - * * @tparam DecomposableFunctionType Decomposable objective function type to be * minimized. * @tparam UpdatePolicyType Update policy used during the iterative update - * process. By default the vanilla update policy - * (see mlpack::optimization::VanillaUpdate) is used. - * @tparam DecayPolicyType Decay policy used during the iterative update - * process to adjust the step size (CyclicalDecay or SnapshotEnsembles). + * process. By default the momentum update policy + * (see mlpack::optimization::MomentumUpdate) is used. */ template< typename DecomposableFunctionType, - typename UpdatePolicyType = MomentumUpdate, - typename DecayPolicyType = CyclicalDecay + typename UpdatePolicyType = MomentumUpdate > class SGDR { public: //! Convenience typedef for the internal optimizer construction. using OptimizerType = MiniBatchSGDType< - DecomposableFunctionType, UpdatePolicyType, DecayPolicyType>; - - /** - * Construct the SGDR optimizer with snapshot ensembles with the given - * function and parameters. The defaults here are not necessarily good for - * the given problem, so it is suggested that the values used be tailored for - * the task at hand. The maximum number of iterations refers to the maximum - * number of mini-batches that are processed. - * - * @param epochRestart Initial epoch where decay is applied. - * @param function Function to be optimized (minimized). - * @param batchSize Size of each mini-batch. - * @param stepSize Step size for each iteration. - * @param maxIterations Maximum number of iterations allowed (0 means no - * limit). - * @param tolerance Maximum absolute tolerance to terminate algorithm. - * @param shuffle If true, the mini-batch order is shuffled; otherwise, each - * mini-batch is visited in linear order. - * @param snapshots Maximum number of snapshots. - * @param accumulate Accumulate the snapshot parameter (default true). - * @param updatePolicy Instantiated update policy used to adjust the given - * parameters. - */ - template - SGDR(DecomposableFunctionType& function, - const size_t epochRestart = 50, - const double multFactor = 2.0, - const size_t batchSize = 1000, - const double stepSize = 0.01, - const size_t maxIterations = 100000, - const double tolerance = 1e-5, - const bool shuffle = true, - const size_t snapshots = 5, - const bool accumulate = true, - const UpdatePolicyType& updatePolicy = UpdatePolicyType(), - const typename std::enable_if_t::value>* junk = 0); + DecomposableFunctionType, UpdatePolicyType, CyclicalDecay>; /** * Construct the SGDR optimizer with the given function and @@ -127,7 +75,6 @@ class SGDR * @param updatePolicy Instantiated update policy used to adjust the given * parameters. */ - template SGDR(DecomposableFunctionType& function, const size_t epochRestart = 50, const double multFactor = 2.0, @@ -136,23 +83,7 @@ class SGDR const size_t maxIterations = 100000, const double tolerance = 1e-5, const bool shuffle = true, - const UpdatePolicyType& updatePolicy = UpdatePolicyType(), - const typename std::enable_if_t::value>* junk = 0); - - /** - * Optimize the given function using SGDR. The given starting point - * will be modified to store the finishing point of the algorithm, and the - * final objective value is returned. - * - * @param iterate Starting point (will be modified). - * @param accumulate Accumulate the snapshot parameter (default true). - * @return Objective value of the final point. - */ - template - double Optimize(arma::mat& iterate, - const typename std::enable_if_t::value>* junk = 0); + const UpdatePolicyType& updatePolicy = UpdatePolicyType()); /** * Optimize the given function using SGDR. The given starting point @@ -162,10 +93,7 @@ class SGDR * @param iterate Starting point (will be modified). * @return Objective value of the final point. */ - template - double Optimize(arma::mat& iterate, - const typename std::enable_if_t::value>* junk = 0); + double Optimize(arma::mat& iterate); //! Get the instantiated function to be optimized. const DecomposableFunctionType& Function() const @@ -201,25 +129,6 @@ class SGDR //! Modify whether or not the individual functions are shuffled. bool& Shuffle() { return optimizer.Shuffle(); } - //! Get the snapshots. - template - typename std::enable_if< - std::is_same::value, - std::vector >::type - Snapshots() const { return optimizer.DecayPolicy().Snapshots(); } - - //! Modify the snapshots. - template - typename std::enable_if< - std::is_same::value, - std::vector& >::type - Snapshots() { return optimizer.DecayPolicy().Snapshots(); } - - //! Get the snapshots. - std::vector Snapshots() const { return junk; } - //! Modify the snapshots. - std::vector& Snapshots() { return junk; } - private: //! The instantiated function. DecomposableFunctionType& function; @@ -227,26 +136,10 @@ class SGDR //! The size of each mini-batch. size_t batchSize; - //! Whether or not to accumulate the snapshots. - bool accumulate; - //! Locally-stored optimizer instance. OptimizerType optimizer; - - //! Locally-stored empty snapshots, necessary to provide an output if another - //! decay policy than SnapshotEnsembles is used. - std::vector junk; }; -// Convenience typedef. - -/** - * Stochastic Gradient Descent with Restarts and snapshot ensembles. - */ -template -using SnapshotSGDR = SGDR< - DecomposableFunctionType, MomentumUpdate,SnapshotEnsembles>; - } // namespace optimization } // namespace mlpack diff --git a/src/mlpack/core/optimizers/sgdr/sgdr_impl.hpp b/src/mlpack/core/optimizers/sgdr/sgdr_impl.hpp index 78d884afc1..ce530f7e07 100644 --- a/src/mlpack/core/optimizers/sgdr/sgdr_impl.hpp +++ b/src/mlpack/core/optimizers/sgdr/sgdr_impl.hpp @@ -18,75 +18,19 @@ namespace mlpack { namespace optimization { -template< - typename DecomposableFunctionType, - typename UpdatePolicyType, - typename DecayPolicyType -> -template -SGDR< - DecomposableFunctionType, - UpdatePolicyType, - DecayPolicyType ->::SGDR(DecomposableFunctionType& function, - const size_t epochRestart, - const double multFactor, - const size_t batchSize, - const double stepSize, - const size_t maxIterations, - const double tolerance, - const bool shuffle, - const size_t snapshots, - const bool accumulate, - const UpdatePolicyType& updatePolicy, - const typename std::enable_if_t::value>* /* junk */) : +template +SGDR::SGDR( + DecomposableFunctionType& function, + const size_t epochRestart, + const double multFactor, + const size_t batchSize, + const double stepSize, + const size_t maxIterations, + const double tolerance, + const bool shuffle, + const UpdatePolicyType& updatePolicy) : function(function), batchSize(batchSize), - accumulate(accumulate), - optimizer(OptimizerType(function, - batchSize, - stepSize, - maxIterations, - tolerance, - shuffle, - updatePolicy, - SnapshotEnsembles( - epochRestart, - multFactor, - stepSize, - batchSize, - function.NumFunctions(), - maxIterations, - snapshots))) -{ - /* Nothing to do here */ -} - -template< - typename DecomposableFunctionType, - typename UpdatePolicyType, - typename DecayPolicyType -> -template -SGDR< - DecomposableFunctionType, - UpdatePolicyType, - DecayPolicyType ->::SGDR(DecomposableFunctionType& function, - const size_t epochRestart, - const double multFactor, - const size_t batchSize, - const double stepSize, - const size_t maxIterations, - const double tolerance, - const bool shuffle, - const UpdatePolicyType& updatePolicy, - const typename std::enable_if_t::value>* /* junk */) : - function(function), - batchSize(batchSize), - accumulate(true), optimizer(OptimizerType(function, batchSize, stepSize, @@ -104,72 +48,12 @@ SGDR< /* Nothing to do here */ } -template< - typename DecomposableFunctionType, - typename UpdatePolicyType, - typename DecayPolicyType -> -template -double SGDR< - DecomposableFunctionType, - UpdatePolicyType, - DecayPolicyType ->::Optimize(arma::mat& iterate, - const typename std::enable_if_t::value>* /* junk */) +template +double SGDR::Optimize( + arma::mat& iterate) { // If a user changed the step size he hasn't update the step size of the - // cyclical decay instantiation, so we have to do here. - if (optimizer.StepSize() != optimizer.DecayPolicy().StepSize()) - { - optimizer.DecayPolicy().StepSize() = optimizer.StepSize(); - } - - // If a user changed the batch size we have to update the restart fraction - // of the cyclical decay instantiation. - if (optimizer.BatchSize() != batchSize) - { - batchSize = optimizer.BatchSize(); - optimizer.DecayPolicy().EpochBatches() = function.NumFunctions() / - double(batchSize); - } - - double overallObjective = optimizer.Optimize(iterate); - - // Accumulate snapshots. - if (accumulate) - { - for (size_t i = 0; i < optimizer.DecayPolicy().Snapshots().size(); ++i) - { - iterate += optimizer.DecayPolicy().Snapshots()[i]; - } - iterate /= (optimizer.DecayPolicy().Snapshots().size() + 1); - - // Calculate final objective. - overallObjective = 0; - for (size_t i = 0; i < function.NumFunctions(); ++i) - overallObjective += function.Evaluate(iterate, i); - } - - return overallObjective; -} - -template< - typename DecomposableFunctionType, - typename UpdatePolicyType, - typename DecayPolicyType -> -template -double SGDR< - DecomposableFunctionType, - UpdatePolicyType, - DecayPolicyType ->::Optimize(arma::mat& iterate, - const typename std::enable_if_t::value>* /* junk */) -{ - // If a user changed the step size he hasn't update the step size of the - // cyclical decay instantiation, so we have to do here. + // cyclical decay instantiation, so we have to do it here. if (optimizer.StepSize() != optimizer.DecayPolicy().StepSize()) { optimizer.DecayPolicy().StepSize() = optimizer.StepSize(); diff --git a/src/mlpack/tests/snapshot_ensembles.cpp b/src/mlpack/tests/snapshot_ensembles.cpp index cd0f04e361..04a3650cd1 100644 --- a/src/mlpack/tests/snapshot_ensembles.cpp +++ b/src/mlpack/tests/snapshot_ensembles.cpp @@ -11,7 +11,7 @@ */ #include #include -#include +#include #include #include From 76e1456ceaa215d40a94a0e6651fa8de8c8e8f71 Mon Sep 17 00:00:00 2001 From: Marcus Edel Date: Mon, 8 May 2017 17:42:02 +0200 Subject: [PATCH 10/84] Add arXiv url for the SGDR and snapshot SGDR implementation. --- src/mlpack/core/optimizers/sgdr/cyclical_decay.hpp | 3 ++- src/mlpack/core/optimizers/sgdr/sgdr.hpp | 3 ++- src/mlpack/core/optimizers/sgdr/snapshot_ensembles.hpp | 3 ++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/mlpack/core/optimizers/sgdr/cyclical_decay.hpp b/src/mlpack/core/optimizers/sgdr/cyclical_decay.hpp index 6aeca8f324..67e0b14055 100644 --- a/src/mlpack/core/optimizers/sgdr/cyclical_decay.hpp +++ b/src/mlpack/core/optimizers/sgdr/cyclical_decay.hpp @@ -32,7 +32,8 @@ namespace optimization { * author = {Ilya Loshchilov and Frank Hutter}, * title = {{SGDR:} Stochastic Gradient Descent with Restarts}, * journal = {CoRR}, - * year = {2016} + * year = {2016}, + * url = {https://arxiv.org/abs/1608.03983} * } * @endcode */ diff --git a/src/mlpack/core/optimizers/sgdr/sgdr.hpp b/src/mlpack/core/optimizers/sgdr/sgdr.hpp index 19bbfaea80..50070a56b7 100644 --- a/src/mlpack/core/optimizers/sgdr/sgdr.hpp +++ b/src/mlpack/core/optimizers/sgdr/sgdr.hpp @@ -35,7 +35,8 @@ namespace optimization { * title = {{SGDR:} Stochastic Gradient Descent with Restarts}, * author = {Ilya Loshchilov and Frank Hutter}, * journal = {CoRR}, - * year = {2016} + * year = {2016}, + * url = {https://arxiv.org/abs/1608.03983} * } * @endcode * diff --git a/src/mlpack/core/optimizers/sgdr/snapshot_ensembles.hpp b/src/mlpack/core/optimizers/sgdr/snapshot_ensembles.hpp index 12a50612dc..a3c9052a9a 100644 --- a/src/mlpack/core/optimizers/sgdr/snapshot_ensembles.hpp +++ b/src/mlpack/core/optimizers/sgdr/snapshot_ensembles.hpp @@ -32,7 +32,8 @@ namespace optimization { * John E. Hopcroft, and Kilian Q. Weinberger}, * booktitle = {Proceedings of the International Conference on Learning * Representations (ICLR)}, - * year = {2017} + * year = {2017}, + * url = {https://arxiv.org/abs/1704.00109} * } * @endcode */ From b275f266327e042bc6c2695c42fdf1c3c244a15d Mon Sep 17 00:00:00 2001 From: Marcus Edel Date: Mon, 8 May 2017 18:43:23 +0200 Subject: [PATCH 11/84] Add snapshot SGDR definition and implementation. --- .../core/optimizers/sgdr/snapshot_sgdr.hpp | 182 ++++++++++++++++++ .../optimizers/sgdr/snapshot_sgdr_impl.hpp | 99 ++++++++++ 2 files changed, 281 insertions(+) create mode 100644 src/mlpack/core/optimizers/sgdr/snapshot_sgdr.hpp create mode 100644 src/mlpack/core/optimizers/sgdr/snapshot_sgdr_impl.hpp diff --git a/src/mlpack/core/optimizers/sgdr/snapshot_sgdr.hpp b/src/mlpack/core/optimizers/sgdr/snapshot_sgdr.hpp new file mode 100644 index 0000000000..7d2030123e --- /dev/null +++ b/src/mlpack/core/optimizers/sgdr/snapshot_sgdr.hpp @@ -0,0 +1,182 @@ +/** + * @file snapshots_sgdr.hpp + * @author Marcus Edel + * + * Definition of the Stochastic Gradient Descent with Restarts (SGDR) as + * described in: "SGDR: Stochastic Gradient Descent with Warm Restarts" by + * I. Loshchilov et al and the Snapshot ensembles technique described in: + * "Snapshot ensembles: Train 1, get m for free" by G. Huang et al. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_CORE_OPTIMIZERS_SGDR_SNAPSHOT_SGDR_HPP +#define MLPACK_CORE_OPTIMIZERS_SGDR_SNAPSHOT_SGDR_HPP + +#include + +#include +#include +#include "snapshot_ensembles.hpp" + +namespace mlpack { +namespace optimization { + +/** + * This class is based on Mini-batch Stochastic Gradient Descent class and + * simulates a new warm-started run/restart once a number of epochs are + * performed using the Snapshot ensembles technique. + * + * For more information, please refer to: + * + * @code + * @article{Loshchilov2016, + * title = {{SGDR:} Stochastic Gradient Descent with Restarts}, + * author = {Ilya Loshchilov and Frank Hutter}, + * journal = {CoRR}, + * year = {2016}, + * url = {https://arxiv.org/abs/1608.03983} + * } + * @endcode + * + * @code + * @inproceedings{Huang2017, + * title = {Snapshot ensembles: Train 1, get m for free}, + * author = {Gao Huang, Yixuan Li, Geoff Pleiss, Zhuang Liu, + * John E. Hopcroft, and Kilian Q. Weinberger}, + * booktitle = {Proceedings of the International Conference on Learning + * Representations (ICLR)}, + * year = {2017}, + * url = {https://arxiv.org/abs/1704.00109} + * } + * @endcode + * + * @tparam DecomposableFunctionType Decomposable objective function type to be + * minimized. + * @tparam UpdatePolicyType Update policy used during the iterative update + * process. By default the momentum update policy + * (see mlpack::optimization::MomentumUpdate) is used. + */ +template< + typename DecomposableFunctionType, + typename UpdatePolicyType = MomentumUpdate +> +class SnapshotSGDR +{ + public: + //! Convenience typedef for the internal optimizer construction. + using OptimizerType = MiniBatchSGDType< + DecomposableFunctionType, UpdatePolicyType, SnapshotEnsembles>; + + /** + * Construct the SnapshotSGDR optimizer with snapshot ensembles with the given + * function and parameters. The defaults here are not necessarily good for + * the given problem, so it is suggested that the values used be tailored for + * the task at hand. The maximum number of iterations refers to the maximum + * number of mini-batches that are processed. + * + * @param epochRestart Initial epoch where decay is applied. + * @param function Function to be optimized (minimized). + * @param batchSize Size of each mini-batch. + * @param stepSize Step size for each iteration. + * @param maxIterations Maximum number of iterations allowed (0 means no + * limit). + * @param tolerance Maximum absolute tolerance to terminate algorithm. + * @param shuffle If true, the mini-batch order is shuffled; otherwise, each + * mini-batch is visited in linear order. + * @param snapshots Maximum number of snapshots. + * @param accumulate Accumulate the snapshot parameter (default true). + * @param updatePolicy Instantiated update policy used to adjust the given + * parameters. + */ + SnapshotSGDR(DecomposableFunctionType& function, + const size_t epochRestart = 50, + const double multFactor = 2.0, + const size_t batchSize = 1000, + const double stepSize = 0.01, + const size_t maxIterations = 100000, + const double tolerance = 1e-5, + const bool shuffle = true, + const size_t snapshots = 5, + const bool accumulate = true, + const UpdatePolicyType& updatePolicy = UpdatePolicyType()); + + /** + * Optimize the given function using SGDR. The given starting point + * will be modified to store the finishing point of the algorithm, and the + * final objective value is returned. + * + * @param iterate Starting point (will be modified). + * @param accumulate Accumulate the snapshot parameter (default true). + * @return Objective value of the final point. + */ + double Optimize(arma::mat& iterate); + + //! Get the instantiated function to be optimized. + const DecomposableFunctionType& Function() const + { + return optimizer.Function(); + } + + //! Modify the instantiated function. + DecomposableFunctionType& Function() { return optimizer.Function(); } + + //! Get the batch size. + size_t BatchSize() const { return optimizer.BatchSize(); } + //! Modify the batch size. + size_t& BatchSize() { return optimizer.BatchSize(); } + + //! Get the step size. + double StepSize() const { return optimizer.StepSize(); } + //! Modify the step size. + double& StepSize() { return optimizer.StepSize(); } + + //! Get the maximum number of iterations (0 indicates no limit). + size_t MaxIterations() const { return optimizer.MaxIterations(); } + //! Modify the maximum number of iterations (0 indicates no limit). + size_t& MaxIterations() { return optimizer.MaxIterations(); } + + //! Get the tolerance for termination. + double Tolerance() const { return optimizer.Tolerance(); } + //! Modify the tolerance for termination. + double& Tolerance() { return optimizer.Tolerance(); } + + //! Get whether or not the individual functions are shuffled. + bool Shuffle() const { return optimizer.Shuffle(); } + //! Modify whether or not the individual functions are shuffled. + bool& Shuffle() { return optimizer.Shuffle(); } + + //! Get the snapshots. + std::vector Snapshots() const + { + return optimizer.DecayPolicy().Snapshots(); + } + //! Modify the snapshots. + std::vector& Snapshots() + { + return optimizer.DecayPolicy().Snapshots(); + } + + private: + //! The instantiated function. + DecomposableFunctionType& function; + + //! The size of each mini-batch. + size_t batchSize; + + //! Whether or not to accumulate the snapshots. + bool accumulate; + + //! Locally-stored optimizer instance. + OptimizerType optimizer; +}; + +} // namespace optimization +} // namespace mlpack + +// Include implementation. +#include "snapshot_sgdr_impl.hpp" + +#endif diff --git a/src/mlpack/core/optimizers/sgdr/snapshot_sgdr_impl.hpp b/src/mlpack/core/optimizers/sgdr/snapshot_sgdr_impl.hpp new file mode 100644 index 0000000000..c58d78627b --- /dev/null +++ b/src/mlpack/core/optimizers/sgdr/snapshot_sgdr_impl.hpp @@ -0,0 +1,99 @@ +/** + * @file snapshots_sgdr_impl.hpp + * @author Marcus Edel + * + * Implementation of SGDR method using snapshots ensembles. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_CORE_OPTIMIZERS_SGDR_SNAPSHOT_SGDR_IMPL_HPP +#define MLPACK_CORE_OPTIMIZERS_SGDR_SNAPSHOT_SGDR_IMPL_HPP + +// In case it hasn't been included yet. +#include "snapshot_sgdr.hpp" + +namespace mlpack { +namespace optimization { + +template +SnapshotSGDR::SnapshotSGDR( + DecomposableFunctionType& function, + const size_t epochRestart, + const double multFactor, + const size_t batchSize, + const double stepSize, + const size_t maxIterations, + const double tolerance, + const bool shuffle, + const size_t snapshots, + const bool accumulate, + const UpdatePolicyType& updatePolicy) : + function(function), + batchSize(batchSize), + accumulate(accumulate), + optimizer(OptimizerType(function, + batchSize, + stepSize, + maxIterations, + tolerance, + shuffle, + updatePolicy, + SnapshotEnsembles( + epochRestart, + multFactor, + stepSize, + batchSize, + function.NumFunctions(), + maxIterations, + snapshots))) +{ + /* Nothing to do here */ +} + +template +double SnapshotSGDR::Optimize( + arma::mat& iterate) +{ + // If a user changed the step size he hasn't update the step size of the + // cyclical decay instantiation, so we have to do here. + if (optimizer.StepSize() != optimizer.DecayPolicy().StepSize()) + { + optimizer.DecayPolicy().StepSize() = optimizer.StepSize(); + } + + // If a user changed the batch size we have to update the restart fraction + // of the cyclical decay instantiation. + if (optimizer.BatchSize() != batchSize) + { + batchSize = optimizer.BatchSize(); + optimizer.DecayPolicy().EpochBatches() = function.NumFunctions() / + double(batchSize); + } + + double overallObjective = optimizer.Optimize(iterate); + + // Accumulate snapshots. + if (accumulate) + { + for (size_t i = 0; i < optimizer.DecayPolicy().Snapshots().size(); ++i) + { + iterate += optimizer.DecayPolicy().Snapshots()[i]; + } + iterate /= (optimizer.DecayPolicy().Snapshots().size() + 1); + + // Calculate final objective. + overallObjective = 0; + for (size_t i = 0; i < function.NumFunctions(); ++i) + overallObjective += function.Evaluate(iterate, i); + } + + return overallObjective; +} + +} // namespace optimization +} // namespace mlpack + +#endif From 9585037b58d0e5ebb24d353a0dd297e16da6d491 Mon Sep 17 00:00:00 2001 From: Shangtong Zhang Date: Wed, 17 May 2017 23:15:10 -0600 Subject: [PATCH 12/84] Add random replay for DQN --- .../environment/cart_pole.hpp | 5 +- .../environment/mountain_car.hpp | 5 +- .../replay/CMakeLists.txt | 14 ++ .../replay/random_replay.hpp | 142 ++++++++++++++++++ src/mlpack/tests/CMakeLists.txt | 2 +- ...onment_test.cpp => rl_components_test.cpp} | 43 +++++- 6 files changed, 206 insertions(+), 5 deletions(-) create mode 100644 src/mlpack/methods/reinforcement_learning/replay/CMakeLists.txt create mode 100644 src/mlpack/methods/reinforcement_learning/replay/random_replay.hpp rename src/mlpack/tests/{rl_environment_test.cpp => rl_components_test.cpp} (53%) diff --git a/src/mlpack/methods/reinforcement_learning/environment/cart_pole.hpp b/src/mlpack/methods/reinforcement_learning/environment/cart_pole.hpp index b7040aa4d1..20d7ea95c9 100644 --- a/src/mlpack/methods/reinforcement_learning/environment/cart_pole.hpp +++ b/src/mlpack/methods/reinforcement_learning/environment/cart_pole.hpp @@ -37,7 +37,7 @@ class CartPole /** * Construct a state instance. */ - State() : data(4) + State() : data(dimension) { /* Nothing to do here. */ } /** @@ -74,6 +74,9 @@ class CartPole //! Encode the state to a column vector. const arma::colvec& Encode() const { return data; } + //! Dimension of the encoded state. + static constexpr size_t dimension = 4; + private: //! Locally-stored (position, velocity, angle, angular velocity). arma::colvec data; diff --git a/src/mlpack/methods/reinforcement_learning/environment/mountain_car.hpp b/src/mlpack/methods/reinforcement_learning/environment/mountain_car.hpp index c5826009a7..1c26642cc9 100644 --- a/src/mlpack/methods/reinforcement_learning/environment/mountain_car.hpp +++ b/src/mlpack/methods/reinforcement_learning/environment/mountain_car.hpp @@ -38,7 +38,7 @@ class MountainCar /** * Construct a state instance. */ - State(): data(2, arma::fill::zeros) + State(): data(dimension, arma::fill::zeros) { /* Nothing to do here. */ } /** @@ -65,6 +65,9 @@ class MountainCar //! Encode the state to a column vector. const arma::colvec& Encode() const { return data; } + //! Dimension of the encoded state. + static constexpr size_t dimension = 2; + private: //! Locally-stored velocity and position vector. arma::colvec data; diff --git a/src/mlpack/methods/reinforcement_learning/replay/CMakeLists.txt b/src/mlpack/methods/reinforcement_learning/replay/CMakeLists.txt new file mode 100644 index 0000000000..03ff3a5720 --- /dev/null +++ b/src/mlpack/methods/reinforcement_learning/replay/CMakeLists.txt @@ -0,0 +1,14 @@ +# Define the files we need to compile +# Anything not in this list will not be compiled into mlpack. +set(SOURCES + random_replay.hpp +) + +# Add directory name to sources. +set(DIR_SRCS) +foreach(file ${SOURCES}) + set(DIR_SRCS ${DIR_SRCS} ${CMAKE_CURRENT_SOURCE_DIR}/${file}) +endforeach() +# Append sources (with directory name) to list of all mlpack sources (used at +# the parent scope). +set(MLPACK_SRCS ${MLPACK_SRCS} ${DIR_SRCS} PARENT_SCOPE) diff --git a/src/mlpack/methods/reinforcement_learning/replay/random_replay.hpp b/src/mlpack/methods/reinforcement_learning/replay/random_replay.hpp new file mode 100644 index 0000000000..045411330a --- /dev/null +++ b/src/mlpack/methods/reinforcement_learning/replay/random_replay.hpp @@ -0,0 +1,142 @@ +/** + * @file random_replay.hpp + * @author Shangtong Zhang + * + * This file is an implementation of random experience replay. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_METHODS_RL_REPLAY_RANDOM_REPLAY_HPP +#define MLPACK_METHODS_RL_REPLAY_RANDOM_REPLAY_HPP + +#include + +namespace mlpack { +namespace rl { + +/** + * Implementation of random experience replay. + * + * @tparam EnvironmentType Desired task. + */ +template +class RandomReplay +{ + public: + using ActionType = typename EnvironmentType::Action; + using StateType = typename EnvironmentType::State; + + /** + * Construct an instance of random experience replay class. + * + * @param batchSize # of examples returned at each sample. + * @param capacity Total memory size in terms of # of examples. + * @param dimension The dimension of an encoded state. + */ + RandomReplay(size_t batchSize, + size_t capacity, + size_t dimension = StateType::dimension) : + batchSize(batchSize), + capacity(capacity), + position(0), + states(dimension, 0), + nextStates(dimension, 0) + { /* Nothing to do here. */ } + + /** + * Store the given experience. + * + * @param state Given state. + * @param action Given action. + * @param reward Given reward. + * @param nextState Given next state. + * @param isEnd Whether next state is terminal state. + */ + void Store(const StateType& state, ActionType action, + double reward, const StateType& nextState, bool isEnd) + { + if (isTerminal.n_elem < capacity) + { + states.insert_cols(position, 1); + actions.insert_rows(position, 1); + rewards.insert_rows(position, 1); + nextStates.insert_cols(position, 1); + isTerminal.insert_rows(position, 1); + } + states.col(position) = state.Encode(); + actions(position) = action; + rewards(position) = reward; + nextStates.col(position) = nextState.Encode(); + isTerminal(position) = isEnd; + position++; + position %= capacity; + } + + /** + * Sample some experiences. + * + * @param sampledStates Sampled encoded states. + * @param sampledActions Sampled actions. + * @param sampledRewards Sampled rewards. + * @param sampledNextStates Sampled encoded next states. + * @param isTerminal Indicate whether corresponding next state is terminal state. + */ + void Sample(arma::mat& sampledStates, + arma::icolvec& sampledActions, + arma::colvec& sampledRewards, + arma::mat& sampledNextStates, + arma::icolvec& isTerminal) + { + size_t upperBound = this->isTerminal.n_elem == capacity ? capacity : position; + arma::uvec sampledIndices = + arma::randi(batchSize, arma::distr_param(0, upperBound - 1)); + sampledStates = states.cols(sampledIndices); + sampledActions = actions.elem(sampledIndices); + sampledRewards = rewards.elem(sampledIndices); + sampledNextStates = nextStates.cols(sampledIndices); + isTerminal = this->isTerminal.elem(sampledIndices); + } + + /** + * Get the # of transitions in the memory. + * + * @return Actual memory size + */ + size_t Size() + { + return isTerminal.n_elem == capacity ? capacity : position; + } + + private: + //! Locally-stored # of examples of each sample. + size_t batchSize; + + //! Locally-stored total memory limit. + size_t capacity; + + //! Indicate the position to store new transition. + size_t position; + + //! Locally-stored encoded previous states. + arma::mat states; + + //! Locally-stored previous actions. + arma::icolvec actions; + + //! Locally-stored previous rewards. + arma::colvec rewards; + + //! Locally-stored encoded previous next states. + arma::mat nextStates; + + //! Locally-stored termination information of previous experience. + arma::icolvec isTerminal; +}; + +} // namespace rl +} // namespace mlpack + +#endif diff --git a/src/mlpack/tests/CMakeLists.txt b/src/mlpack/tests/CMakeLists.txt index f96a25d39e..609a7b3385 100644 --- a/src/mlpack/tests/CMakeLists.txt +++ b/src/mlpack/tests/CMakeLists.txt @@ -76,7 +76,7 @@ add_executable(mlpack_test recurrent_network_test.cpp rectangle_tree_test.cpp regularized_svd_test.cpp - rl_environment_test.cpp + rl_components_test.cpp rmsprop_test.cpp sa_test.cpp sdp_primal_dual_test.cpp diff --git a/src/mlpack/tests/rl_environment_test.cpp b/src/mlpack/tests/rl_components_test.cpp similarity index 53% rename from src/mlpack/tests/rl_environment_test.cpp rename to src/mlpack/tests/rl_components_test.cpp index 57d80e3f34..7966c80871 100644 --- a/src/mlpack/tests/rl_environment_test.cpp +++ b/src/mlpack/tests/rl_components_test.cpp @@ -2,7 +2,7 @@ * @file rl_environment_test.hpp * @author Shangtong Zhang * - * Basic test for the reinforcement learning task environment. + * Basic test for the components of reinforcement learning algorithms. * * mlpack is free software; you may redistribute it and/or modify it under the * terms of the 3-clause BSD license. You should have received a copy of the @@ -14,6 +14,7 @@ #include #include +#include #include #include "test_tools.hpp" @@ -21,7 +22,7 @@ using namespace mlpack; using namespace mlpack::rl; -BOOST_AUTO_TEST_SUITE(RLEnvironmentTest) +BOOST_AUTO_TEST_SUITE(RLComponentsTest) /** * Constructs a MountainCar instance and check if the main rountine works as @@ -57,4 +58,42 @@ BOOST_AUTO_TEST_CASE(SimpleCartPoleTest) BOOST_REQUIRE_EQUAL(2, CartPole::Action::size); } +/** + * Compare two matrix + */ +bool Equal(const arma::mat& m1, const arma::mat& m2) +{ + return arma::mean(arma::mean(arma::abs(m1 - m2))) < 1e-5; +} + +/** + * Construct a random replay instance and check if it works as + * it should be. + */ +BOOST_AUTO_TEST_CASE(RandomReplayTest) +{ + RandomReplay replay(1, 1); + MountainCar env; + MountainCar::State state = env.InitialSample(); + MountainCar::Action action = MountainCar::Action::forward; + MountainCar::State nextState; + double reward = env.Sample(state, action, nextState); + for (size_t i = 0; i < 4; ++i) + { + replay.Store(state, action, reward, nextState, env.IsTerminal(nextState)); + } + arma::mat sampledState; + arma::icolvec sampledAction; + arma::colvec sampledReward; + arma::mat sampledNextState; + arma::icolvec sampledTerminal; + replay.Sample(sampledState, sampledAction, sampledReward, sampledNextState, sampledTerminal); + BOOST_REQUIRE(Equal(state.Encode(), sampledState)); + BOOST_REQUIRE_EQUAL(action, arma::as_scalar(sampledAction)); + BOOST_REQUIRE_CLOSE(reward, arma::as_scalar(sampledReward), 1e-5); + BOOST_REQUIRE(Equal(nextState.Encode(), sampledNextState)); + BOOST_REQUIRE_EQUAL(false, arma::as_scalar(sampledTerminal)); + BOOST_REQUIRE_EQUAL(1, replay.Size()); +} + BOOST_AUTO_TEST_SUITE_END() From 947b71addf9cde6a43f7a620ffafab7d7af014be Mon Sep 17 00:00:00 2001 From: Marcus Edel Date: Sat, 20 May 2017 18:38:47 +0200 Subject: [PATCH 13/84] Re-run the Rank10Test test if the reconstructed kernel matrix is singular. --- src/mlpack/tests/nystroem_method_test.cpp | 47 +++++++++++++++-------- 1 file changed, 32 insertions(+), 15 deletions(-) diff --git a/src/mlpack/tests/nystroem_method_test.cpp b/src/mlpack/tests/nystroem_method_test.cpp index 2c505768e1..1e76ee2438 100644 --- a/src/mlpack/tests/nystroem_method_test.cpp +++ b/src/mlpack/tests/nystroem_method_test.cpp @@ -93,28 +93,45 @@ BOOST_AUTO_TEST_CASE(Rank10Test) LinearKernel lk; arma::mat kernel = dataMod.t() * dataMod; - // Now use the linear kernel to get a Nystroem approximation; try this several - // times. - double normalizedFroAverage = 0.0; - for (size_t trial = 0; trial < 20; ++trial) + size_t successes = 0; + for (size_t testTrial = 0; testTrial < 5; ++testTrial) { - LinearKernel lk; - NystroemMethod nm(dataMod, lk, 10); + // Now use the linear kernel to get a Nystroem approximation; try this several + // times. + double normalizedFroAverage = 0.0; + for (size_t trial = 0; trial < 20; ++trial) + { + while(true) + { + LinearKernel lk; + NystroemMethod nm(dataMod, lk, 10); - arma::mat g; - nm.Apply(g); + arma::mat g; + nm.Apply(g); - arma::mat approximation = g * g.t(); + arma::mat approximation = g * g.t(); - // Check the normalized Frobenius norm. - const double normalizedFro = arma::norm(kernel - approximation, "fro") / - arma::norm(kernel, "fro"); + // Check the normalized Frobenius norm. + const double normalizedFro = arma::norm(kernel - approximation, "fro"); - normalizedFroAverage += normalizedFro; + // Sometimes K' is singular. Unlucky. + if (normalizedFro != normalizedFro) + continue; + + normalizedFroAverage += (normalizedFro / arma::norm(kernel, "fro")); + break; + } + } + + normalizedFroAverage /= 20; + if (std::abs(normalizedFroAverage) <= 1e-3) + { + ++successes; + break; + } } - normalizedFroAverage /= 20; - BOOST_REQUIRE_SMALL(normalizedFroAverage, 1e-3); + BOOST_REQUIRE_GE(successes, 1); } /** From 70f8c7db182894eb33c693a1ecaef230c59449ec Mon Sep 17 00:00:00 2001 From: Shangtong Zhang Date: Sat, 20 May 2017 22:44:30 -0600 Subject: [PATCH 14/84] Preallocation for memory buffer --- .../replay/random_replay.hpp | 66 ++++++++++++------- src/mlpack/tests/rl_components_test.cpp | 12 +--- 2 files changed, 46 insertions(+), 32 deletions(-) diff --git a/src/mlpack/methods/reinforcement_learning/replay/random_replay.hpp b/src/mlpack/methods/reinforcement_learning/replay/random_replay.hpp index 045411330a..696dcde10b 100644 --- a/src/mlpack/methods/reinforcement_learning/replay/random_replay.hpp +++ b/src/mlpack/methods/reinforcement_learning/replay/random_replay.hpp @@ -20,6 +20,23 @@ namespace rl { /** * Implementation of random experience replay. * + * At each time step, interactions between the agent and the + * environment will be saved to a memory buffer. When necessary, + * we can simply sample previous experiences from the buffer to + * train the agent. Typically this would be a random sample and + * the memory will be a First-In-First-Out buffer. + * + * For more information, see the following. + * + * @code + * @phdthesis {lin1993reinforcement, + * title = {Reinforcement learning for robots using neural networks}, + * author = {Lin, Long-Ji}, + * year = {1993}, + * school = {Fujitsu Laboratories Ltd} + * } + * @endcode + * * @tparam EnvironmentType Desired task. */ template @@ -36,14 +53,18 @@ class RandomReplay * @param capacity Total memory size in terms of # of examples. * @param dimension The dimension of an encoded state. */ - RandomReplay(size_t batchSize, - size_t capacity, - size_t dimension = StateType::dimension) : + RandomReplay(const size_t batchSize, + const size_t capacity, + const size_t dimension = StateType::dimension) : batchSize(batchSize), capacity(capacity), position(0), - states(dimension, 0), - nextStates(dimension, 0) + states(dimension, capacity), + actions(dimension, capacity), + rewards(dimension, capacity), + nextStates(dimension, capacity), + isTerminal(dimension, capacity), + full(false) { /* Nothing to do here. */ } /** @@ -55,24 +76,22 @@ class RandomReplay * @param nextState Given next state. * @param isEnd Whether next state is terminal state. */ - void Store(const StateType& state, ActionType action, - double reward, const StateType& nextState, bool isEnd) + void Store(const StateType& state, + ActionType action, + double reward, + const StateType& nextState, + bool isEnd) { - if (isTerminal.n_elem < capacity) - { - states.insert_cols(position, 1); - actions.insert_rows(position, 1); - rewards.insert_rows(position, 1); - nextStates.insert_cols(position, 1); - isTerminal.insert_rows(position, 1); - } states.col(position) = state.Encode(); actions(position) = action; rewards(position) = reward; nextStates.col(position) = nextState.Encode(); isTerminal(position) = isEnd; position++; - position %= capacity; + if (position == capacity) { + full = true; + position = 0; + } } /** @@ -90,7 +109,7 @@ class RandomReplay arma::mat& sampledNextStates, arma::icolvec& isTerminal) { - size_t upperBound = this->isTerminal.n_elem == capacity ? capacity : position; + size_t upperBound = full ? capacity : position; arma::uvec sampledIndices = arma::randi(batchSize, arma::distr_param(0, upperBound - 1)); sampledStates = states.cols(sampledIndices); @@ -101,17 +120,17 @@ class RandomReplay } /** - * Get the # of transitions in the memory. + * Get the number of transitions in the memory. * - * @return Actual memory size + * @return Actual used memory size */ - size_t Size() + const size_t& Size() { - return isTerminal.n_elem == capacity ? capacity : position; + return full ? capacity : position; } private: - //! Locally-stored # of examples of each sample. + //! Locally-stored number of examples of each sample. size_t batchSize; //! Locally-stored total memory limit. @@ -134,6 +153,9 @@ class RandomReplay //! Locally-stored termination information of previous experience. arma::icolvec isTerminal; + + //! Locally-stored indicator that whether the memory is full or not + bool full; }; } // namespace rl diff --git a/src/mlpack/tests/rl_components_test.cpp b/src/mlpack/tests/rl_components_test.cpp index 7966c80871..5c775b97d0 100644 --- a/src/mlpack/tests/rl_components_test.cpp +++ b/src/mlpack/tests/rl_components_test.cpp @@ -58,14 +58,6 @@ BOOST_AUTO_TEST_CASE(SimpleCartPoleTest) BOOST_REQUIRE_EQUAL(2, CartPole::Action::size); } -/** - * Compare two matrix - */ -bool Equal(const arma::mat& m1, const arma::mat& m2) -{ - return arma::mean(arma::mean(arma::abs(m1 - m2))) < 1e-5; -} - /** * Construct a random replay instance and check if it works as * it should be. @@ -88,10 +80,10 @@ BOOST_AUTO_TEST_CASE(RandomReplayTest) arma::mat sampledNextState; arma::icolvec sampledTerminal; replay.Sample(sampledState, sampledAction, sampledReward, sampledNextState, sampledTerminal); - BOOST_REQUIRE(Equal(state.Encode(), sampledState)); + CheckMatrices(state.Encode(), sampledState); BOOST_REQUIRE_EQUAL(action, arma::as_scalar(sampledAction)); BOOST_REQUIRE_CLOSE(reward, arma::as_scalar(sampledReward), 1e-5); - BOOST_REQUIRE(Equal(nextState.Encode(), sampledNextState)); + CheckMatrices(nextState.Encode(), sampledNextState); BOOST_REQUIRE_EQUAL(false, arma::as_scalar(sampledTerminal)); BOOST_REQUIRE_EQUAL(1, replay.Size()); } From ad8a6fb93bfc5bd687e17de61417a9544e0d6c20 Mon Sep 17 00:00:00 2001 From: Shangtong Zhang Date: Sat, 20 May 2017 22:45:57 -0600 Subject: [PATCH 15/84] Minor code style fix --- .../methods/reinforcement_learning/replay/random_replay.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/mlpack/methods/reinforcement_learning/replay/random_replay.hpp b/src/mlpack/methods/reinforcement_learning/replay/random_replay.hpp index 696dcde10b..3f3c32d6fd 100644 --- a/src/mlpack/methods/reinforcement_learning/replay/random_replay.hpp +++ b/src/mlpack/methods/reinforcement_learning/replay/random_replay.hpp @@ -49,8 +49,8 @@ class RandomReplay /** * Construct an instance of random experience replay class. * - * @param batchSize # of examples returned at each sample. - * @param capacity Total memory size in terms of # of examples. + * @param batchSize Number of examples returned at each sample. + * @param capacity Total memory size in terms of number of examples. * @param dimension The dimension of an encoded state. */ RandomReplay(const size_t batchSize, From 17afcd71318357d2ac86055a79e300e8e2304b38 Mon Sep 17 00:00:00 2001 From: Shangtong Zhang Date: Sat, 20 May 2017 22:47:20 -0600 Subject: [PATCH 16/84] Minor code style fix --- .../methods/reinforcement_learning/replay/random_replay.hpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/mlpack/methods/reinforcement_learning/replay/random_replay.hpp b/src/mlpack/methods/reinforcement_learning/replay/random_replay.hpp index 3f3c32d6fd..6c09207fe9 100644 --- a/src/mlpack/methods/reinforcement_learning/replay/random_replay.hpp +++ b/src/mlpack/methods/reinforcement_learning/replay/random_replay.hpp @@ -88,7 +88,8 @@ class RandomReplay nextStates.col(position) = nextState.Encode(); isTerminal(position) = isEnd; position++; - if (position == capacity) { + if (position == capacity) + { full = true; position = 0; } From e1e961ab17ded5c1d982a8c5b48469bd6e350b56 Mon Sep 17 00:00:00 2001 From: Marcus Edel Date: Sun, 21 May 2017 13:04:00 +0200 Subject: [PATCH 17/84] Fix the correct index if the first approximation of the kernel matrix fails. --- src/mlpack/tests/nystroem_method_test.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mlpack/tests/nystroem_method_test.cpp b/src/mlpack/tests/nystroem_method_test.cpp index 1e76ee2438..6f3db50426 100644 --- a/src/mlpack/tests/nystroem_method_test.cpp +++ b/src/mlpack/tests/nystroem_method_test.cpp @@ -170,7 +170,7 @@ BOOST_AUTO_TEST_CASE(GermanTest) { // We will repeat each trial 20 times. double avgError = 0.0; - for (size_t z = 0; z < 20; ++z) + for (size_t z = 1; z < 21; ++z) { NystroemMethod > nm(dataset, gk, size_t((double((trial + 1) * 2) / 100.0) * dataset.n_cols)); From 0b7e2b8c42ded8ca09263e1114ef31d2752725c4 Mon Sep 17 00:00:00 2001 From: Shangtong Zhang Date: Sun, 21 May 2017 15:04:48 -0600 Subject: [PATCH 18/84] Fix a bug and update the test case for random replay --- .../replay/random_replay.hpp | 6 ++--- src/mlpack/tests/rl_components_test.cpp | 23 +++++++++++++++---- 2 files changed, 21 insertions(+), 8 deletions(-) diff --git a/src/mlpack/methods/reinforcement_learning/replay/random_replay.hpp b/src/mlpack/methods/reinforcement_learning/replay/random_replay.hpp index 6c09207fe9..a6e89b06f7 100644 --- a/src/mlpack/methods/reinforcement_learning/replay/random_replay.hpp +++ b/src/mlpack/methods/reinforcement_learning/replay/random_replay.hpp @@ -60,10 +60,10 @@ class RandomReplay capacity(capacity), position(0), states(dimension, capacity), - actions(dimension, capacity), - rewards(dimension, capacity), + actions(capacity), + rewards(capacity), nextStates(dimension, capacity), - isTerminal(dimension, capacity), + isTerminal(capacity), full(false) { /* Nothing to do here. */ } diff --git a/src/mlpack/tests/rl_components_test.cpp b/src/mlpack/tests/rl_components_test.cpp index 5c775b97d0..31a9ef5991 100644 --- a/src/mlpack/tests/rl_components_test.cpp +++ b/src/mlpack/tests/rl_components_test.cpp @@ -64,21 +64,20 @@ BOOST_AUTO_TEST_CASE(SimpleCartPoleTest) */ BOOST_AUTO_TEST_CASE(RandomReplayTest) { - RandomReplay replay(1, 1); + RandomReplay replay(1, 3); MountainCar env; MountainCar::State state = env.InitialSample(); MountainCar::Action action = MountainCar::Action::forward; MountainCar::State nextState; double reward = env.Sample(state, action, nextState); - for (size_t i = 0; i < 4; ++i) - { - replay.Store(state, action, reward, nextState, env.IsTerminal(nextState)); - } + replay.Store(state, action, reward, nextState, env.IsTerminal(nextState)); arma::mat sampledState; arma::icolvec sampledAction; arma::colvec sampledReward; arma::mat sampledNextState; arma::icolvec sampledTerminal; + + //! So far there should be only one record in the memory replay.Sample(sampledState, sampledAction, sampledReward, sampledNextState, sampledTerminal); CheckMatrices(state.Encode(), sampledState); BOOST_REQUIRE_EQUAL(action, arma::as_scalar(sampledAction)); @@ -86,6 +85,20 @@ BOOST_AUTO_TEST_CASE(RandomReplayTest) CheckMatrices(nextState.Encode(), sampledNextState); BOOST_REQUIRE_EQUAL(false, arma::as_scalar(sampledTerminal)); BOOST_REQUIRE_EQUAL(1, replay.Size()); + + //! Overwrite the memory with a nonsense record + for (size_t i = 0; i < 5; ++i) { + replay.Store(nextState, action, reward, state, true); + } + BOOST_REQUIRE_EQUAL(3, replay.Size()); + + //! Sample several times, the original record shouldn't appear + for (size_t i = 0; i < 30; ++i) { + replay.Sample(sampledState, sampledAction, sampledReward, sampledNextState, sampledTerminal); + CheckMatrices(state.Encode(), sampledNextState); + CheckMatrices(nextState.Encode(), sampledState); + BOOST_REQUIRE_EQUAL(true, arma::as_scalar(sampledTerminal)); + } } BOOST_AUTO_TEST_SUITE_END() From 7aa6a0c495ae4088c6228024e9e9de7e55e5c229 Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Sun, 21 May 2017 19:30:47 -0400 Subject: [PATCH 19/84] Reduce number of points to accelerate test. --- src/mlpack/tests/fastmks_test.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/mlpack/tests/fastmks_test.cpp b/src/mlpack/tests/fastmks_test.cpp index e9f5cf9a47..5b1d495930 100644 --- a/src/mlpack/tests/fastmks_test.cpp +++ b/src/mlpack/tests/fastmks_test.cpp @@ -67,7 +67,7 @@ BOOST_AUTO_TEST_CASE(DualTreeVsNaive) { // First create a random dataset. arma::mat data; - data.randn(10, 5000); + data.randn(10, 2000); LinearKernel lk; // Now run FastMKS naively. @@ -101,7 +101,7 @@ BOOST_AUTO_TEST_CASE(DualTreeVsSingleTree) { // First create a random dataset. arma::mat data; - data.randu(8, 5000); + data.randu(8, 2000); PolynomialKernel pk(5.0, 2.5); FastMKS single(data, pk, true); From e065f829c284620a1f280d07be03df6de11dca46 Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Sun, 21 May 2017 21:48:15 -0400 Subject: [PATCH 20/84] Accelerate a couple of tests. --- src/mlpack/tests/minibatch_sgd_test.cpp | 20 ++++++++++---------- src/mlpack/tests/svd_batch_test.cpp | 2 +- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/src/mlpack/tests/minibatch_sgd_test.cpp b/src/mlpack/tests/minibatch_sgd_test.cpp index 22d65abf57..ee08c73d77 100644 --- a/src/mlpack/tests/minibatch_sgd_test.cpp +++ b/src/mlpack/tests/minibatch_sgd_test.cpp @@ -80,14 +80,14 @@ BOOST_AUTO_TEST_CASE(LogisticRegressionTest) GaussianDistribution g1(arma::vec("1.0 1.0 1.0"), arma::eye(3, 3)); GaussianDistribution g2(arma::vec("9.0 9.0 9.0"), arma::eye(3, 3)); - arma::mat data(3, 1000); - arma::Row responses(1000); - for (size_t i = 0; i < 500; ++i) + arma::mat data(3, 500); + arma::Row responses(500); + for (size_t i = 0; i < 250; ++i) { data.col(i) = g1.Random(); responses[i] = 0; } - for (size_t i = 500; i < 1000; ++i) + for (size_t i = 250; i < 500; ++i) { data.col(i) = g2.Random(); responses[i] = 1; @@ -96,8 +96,8 @@ BOOST_AUTO_TEST_CASE(LogisticRegressionTest) // Shuffle the dataset. arma::uvec indices = arma::shuffle(arma::linspace(0, data.n_cols - 1, data.n_cols)); - arma::mat shuffledData(3, 1000); - arma::Row shuffledResponses(1000); + arma::mat shuffledData(3, 500); + arma::Row shuffledResponses(500); for (size_t i = 0; i < data.n_cols; ++i) { shuffledData.col(i) = data.col(indices[i]); @@ -105,14 +105,14 @@ BOOST_AUTO_TEST_CASE(LogisticRegressionTest) } // Create a test set. - arma::mat testData(3, 1000); - arma::Row testResponses(1000); - for (size_t i = 0; i < 500; ++i) + arma::mat testData(3, 500); + arma::Row testResponses(500); + for (size_t i = 0; i < 250; ++i) { testData.col(i) = g1.Random(); testResponses[i] = 0; } - for (size_t i = 500; i < 1000; ++i) + for (size_t i = 250; i < 500; ++i) { testData.col(i) = g2.Random(); testResponses[i] = 1; diff --git a/src/mlpack/tests/svd_batch_test.cpp b/src/mlpack/tests/svd_batch_test.cpp index fee96ec7d5..dd9e66dc26 100644 --- a/src/mlpack/tests/svd_batch_test.cpp +++ b/src/mlpack/tests/svd_batch_test.cpp @@ -91,7 +91,7 @@ BOOST_AUTO_TEST_CASE(SVDBatchMomentumTest) // Create the initial matrices. SpecificRandomInitialization sri(cleanedData.n_rows, 2, cleanedData.n_cols); - ValidationRMSETermination vrt(cleanedData, 2000); + ValidationRMSETermination vrt(cleanedData, 100); AMF, SpecificRandomInitialization, SVDBatchLearning> amf1(vrt, sri, SVDBatchLearning(0.0009, 0, 0, 0)); From 85a81663aebb39f7c0e61146434440e385cb46bf Mon Sep 17 00:00:00 2001 From: Kirill Mishchenko Date: Fri, 19 May 2017 08:43:04 +0500 Subject: [PATCH 21/84] Change the Predict interface in LinearRegression Make the predictions argument column-major in the method Predict. --- .../linear_regression/linear_regression.cpp | 14 +++++++++++--- .../linear_regression/linear_regression.hpp | 11 ++++++++++- .../linear_regression/linear_regression_main.cpp | 2 +- src/mlpack/tests/linear_regression_test.cpp | 6 +++--- 4 files changed, 25 insertions(+), 8 deletions(-) diff --git a/src/mlpack/methods/linear_regression/linear_regression.cpp b/src/mlpack/methods/linear_regression/linear_regression.cpp index cf38e43ca5..0bcfb70098 100644 --- a/src/mlpack/methods/linear_regression/linear_regression.cpp +++ b/src/mlpack/methods/linear_regression/linear_regression.cpp @@ -101,6 +101,14 @@ void LinearRegression::Train(const arma::mat& predictors, void LinearRegression::Predict(const arma::mat& points, arma::vec& predictions) const +{ + arma::rowvec rowPredictions; + Predict(points, rowPredictions); + predictions = arma::trans(rowPredictions); +} + +void LinearRegression::Predict(const arma::mat& points, + arma::rowvec& predictions) const { if (intercept) { @@ -109,8 +117,8 @@ void LinearRegression::Predict(const arma::mat& points, arma::vec& predictions) Log::Assert(points.n_rows == parameters.n_rows - 1); // Get the predictions, but this ignores the intercept value // (parameters[0]). - predictions = arma::trans(arma::trans(parameters.subvec(1, - parameters.n_elem - 1)) * points); + predictions = arma::trans(parameters.subvec(1, parameters.n_elem - 1)) + * points; // Now add the intercept. predictions += parameters(0); } @@ -118,7 +126,7 @@ void LinearRegression::Predict(const arma::mat& points, arma::vec& predictions) { // We want to be sure we have the correct number of dimensions in the dataset. Log::Assert(points.n_rows == parameters.n_rows); - predictions = arma::trans(arma::trans(parameters) * points); + predictions = arma::trans(parameters) * points; } } diff --git a/src/mlpack/methods/linear_regression/linear_regression.hpp b/src/mlpack/methods/linear_regression/linear_regression.hpp index 7fc9c0b421..7d489eaa3f 100644 --- a/src/mlpack/methods/linear_regression/linear_regression.hpp +++ b/src/mlpack/methods/linear_regression/linear_regression.hpp @@ -78,7 +78,16 @@ class LinearRegression * @param points the data points to calculate with. * @param predictions y, will contain calculated values on completion. */ - void Predict(const arma::mat& points, arma::vec& predictions) const; + mlpack_deprecated void Predict(const arma::mat& points, + arma::vec& predictions) const; + + /** + * Calculate y_i for each data point in points. + * + * @param points the data points to calculate with. + * @param predictions y, will contain calculated values on completion. + */ + void Predict(const arma::mat& points, arma::rowvec& predictions) const; /** * Calculate the L2 squared error on the given predictors and responses using diff --git a/src/mlpack/methods/linear_regression/linear_regression_main.cpp b/src/mlpack/methods/linear_regression/linear_regression_main.cpp index 8e9b355eab..105aa094ec 100644 --- a/src/mlpack/methods/linear_regression/linear_regression_main.cpp +++ b/src/mlpack/methods/linear_regression/linear_regression_main.cpp @@ -183,7 +183,7 @@ int main(int argc, char* argv[]) } // Perform the predictions using our model. - vec predictions; + rowvec predictions; Timer::Start("prediction"); lr.Predict(points, predictions); Timer::Stop("prediction"); diff --git a/src/mlpack/tests/linear_regression_test.cpp b/src/mlpack/tests/linear_regression_test.cpp index eba31c42b6..409626ec31 100644 --- a/src/mlpack/tests/linear_regression_test.cpp +++ b/src/mlpack/tests/linear_regression_test.cpp @@ -33,7 +33,7 @@ BOOST_AUTO_TEST_CASE(LinearRegressionTestCase) arma::vec responses(10); // The values we get back when we predict for points. - arma::vec predictions(10); + arma::rowvec predictions(10); // We'll randomly select some coefficients for the linear response. arma::vec coeffs; @@ -121,7 +121,7 @@ BOOST_AUTO_TEST_CASE(RidgeRegressionTest) LinearRegression lr(data, responses, 0.0001); // Now just make sure that it predicts some more zeros. - arma::vec predictedResponses; + arma::rowvec predictedResponses; lr.Predict(data, predictedResponses); for (size_t i = 0; i < 5000; ++i) @@ -143,7 +143,7 @@ BOOST_AUTO_TEST_CASE(RidgeRegressionTestCase) arma::vec responses(10); // The values we get back when we predict for points. - arma::vec predictions(10); + arma::rowvec predictions(10); // We'll randomly select some coefficients for the linear response. arma::vec coeffs; From 8fbebacd17dc9f18bb79223141ed69e3e63ff103 Mon Sep 17 00:00:00 2001 From: Kirill Mishchenko Date: Fri, 19 May 2017 16:41:03 +0500 Subject: [PATCH 22/84] Change constructor and Train signatures This commit makes responses column-major in Linear Regression, as well as separates constructors and Train methods for learning with weights and without them. --- .../linear_regression/linear_regression.cpp | 57 ++++++++--- .../linear_regression/linear_regression.hpp | 94 +++++++++++++++++-- .../linear_regression_main.cpp | 6 +- src/mlpack/tests/linear_regression_test.cpp | 12 +-- 4 files changed, 142 insertions(+), 27 deletions(-) diff --git a/src/mlpack/methods/linear_regression/linear_regression.cpp b/src/mlpack/methods/linear_regression/linear_regression.cpp index 0bcfb70098..6a3976a291 100644 --- a/src/mlpack/methods/linear_regression/linear_regression.cpp +++ b/src/mlpack/methods/linear_regression/linear_regression.cpp @@ -21,10 +21,25 @@ LinearRegression::LinearRegression(const arma::mat& predictors, const double lambda, const bool intercept, const arma::vec& weights) : + LinearRegression(predictors, responses.t(), weights.t(), lambda, intercept) +{} + +LinearRegression::LinearRegression(const arma::mat& predictors, + const arma::rowvec& responses, + const double lambda, + const bool intercept) : + LinearRegression(predictors, responses, arma::rowvec(), lambda, intercept) +{} + +LinearRegression::LinearRegression(const arma::mat& predictors, + const arma::rowvec& responses, + const arma::rowvec& weights, + const double lambda, + const bool intercept) : lambda(lambda), intercept(intercept) { - Train(predictors, responses, intercept, weights); + Train(predictors, responses, weights, intercept); } LinearRegression::LinearRegression(const LinearRegression& linearRegression) : @@ -36,6 +51,21 @@ void LinearRegression::Train(const arma::mat& predictors, const arma::vec& responses, const bool intercept, const arma::vec& weights) +{ + Train(predictors, responses.t(), weights.t(), intercept); +} + +void LinearRegression::Train(const arma::mat& predictors, + const arma::rowvec& responses, + const bool intercept) +{ + Train(predictors, responses, arma::rowvec(), intercept); +} + +void LinearRegression::Train(const arma::mat& predictors, + const arma::rowvec& responses, + const arma::rowvec& weights, + const bool intercept) { this->intercept = intercept; @@ -51,7 +81,7 @@ void LinearRegression::Train(const arma::mat& predictors, const size_t nCols = predictors.n_cols; arma::mat p = predictors; - arma::vec r = responses; + arma::rowvec r = responses; // Here we add the row of ones to the predictors. // The intercept is not penalized. Add an "all ones" row to design and set @@ -88,14 +118,12 @@ void LinearRegression::Train(const arma::mat& predictors, // B = Q^T * responses * R^-1 // If lambda > 0, then we must add a bunch of empty responses. if (lambda == 0.0) - { - arma::solve(parameters, R, arma::trans(Q) * r); - } + arma::solve(parameters, R, arma::trans(r * Q)); else { // Copy responses into larger vector. - r.insert_rows(nCols,p.n_cols - nCols); - arma::solve(parameters, R, arma::trans(Q) * r); + r.insert_cols(nCols,p.n_cols - nCols); + arma::solve(parameters, R, arma::trans(r * Q)); } } @@ -134,6 +162,13 @@ void LinearRegression::Predict(const arma::mat& points, //! Compute the L2 squared error on the given predictors and responses. double LinearRegression::ComputeError(const arma::mat& predictors, const arma::vec& responses) const +{ + arma::rowvec rowResponses = responses.t(); + return ComputeError(predictors, rowResponses); +} + +double LinearRegression::ComputeError(const arma::mat& predictors, + const arma::rowvec& responses) const { // Get the number of columns and rows of the dataset. const size_t nCols = predictors.n_cols; @@ -141,7 +176,7 @@ double LinearRegression::ComputeError(const arma::mat& predictors, // Calculate the differences between actual responses and predicted responses. // We must also add the intercept (parameters(0)) to the predictions. - arma::vec temp; + arma::rowvec temp; if (intercept) { // Ensure that we have the correct number of dimensions in the dataset. @@ -150,8 +185,8 @@ double LinearRegression::ComputeError(const arma::mat& predictors, Log::Fatal << "The test data must have the same number of columns as the " "training file." << std::endl; } - temp = responses - arma::trans( (arma::trans(parameters.subvec(1, - parameters.n_elem - 1)) * predictors) + parameters(0)); + temp = responses - (parameters(0) + + arma::trans(parameters.subvec(1, parameters.n_elem - 1)) * predictors); } else { @@ -161,7 +196,7 @@ double LinearRegression::ComputeError(const arma::mat& predictors, Log::Fatal << "The test data must have the same number of columns as the " "training file." << std::endl; } - temp = responses - arma::trans((arma::trans(parameters) * predictors)); + temp = responses - arma::trans(parameters) * predictors; } const double cost = arma::dot(temp, temp) / nCols; diff --git a/src/mlpack/methods/linear_regression/linear_regression.hpp b/src/mlpack/methods/linear_regression/linear_regression.hpp index 7d489eaa3f..b8fa6ca78b 100644 --- a/src/mlpack/methods/linear_regression/linear_regression.hpp +++ b/src/mlpack/methods/linear_regression/linear_regression.hpp @@ -35,11 +35,39 @@ class LinearRegression * @param intercept Whether or not to include an intercept term. * @param weights Observation weights (for boosting). */ + mlpack_deprecated LinearRegression(const arma::mat& predictors, + const arma::vec& responses, + const double lambda = 0, + const bool intercept = true, + const arma::vec& weights = arma::vec()); + + /** + * Creates the model. + * + * @param predictors X, matrix of data points. + * @param responses y, the measured data for each point in X. + * @param lambda Regularization constant for ridge regression. + * @param intercept Whether or not to include an intercept term. + */ LinearRegression(const arma::mat& predictors, - const arma::vec& responses, + const arma::rowvec& responses, const double lambda = 0, - const bool intercept = true, - const arma::vec& weights = arma::vec()); + const bool intercept = true); + + /** + * Creates the model with weighted learning. + * + * @param predictors X, matrix of data points. + * @param responses y, the measured data for each point in X. + * @param weights Observation weights (for boosting). + * @param lambda Regularization constant for ridge regression. + * @param intercept Whether or not to include an intercept term. + */ + LinearRegression(const arma::mat& predictors, + const arma::rowvec& responses, + const arma::rowvec& weights, + const double lambda = 0, + const bool intercept = true); /** * Copy constructor. @@ -67,10 +95,42 @@ class LinearRegression * @param intercept Whether or not to fit an intercept term. * @param weights Observation weights (for boosting). */ + mlpack_deprecated void Train(const arma::mat& predictors, + const arma::vec& responses, + const bool intercept = true, + const arma::vec& weights = arma::vec()); + + /** + * Train the LinearRegression model on the given data. Careful! This will + * completely ignore and overwrite the existing model. This particular + * implementation does not have an incremental training algorithm. To set the + * regularization parameter lambda, call Lambda() or set a different value in + * the constructor. + * + * @param predictors X, the matrix of data points to train the model on. + * @param responses y, the responses to the data points. + * @param intercept Whether or not to fit an intercept term. + */ void Train(const arma::mat& predictors, - const arma::vec& responses, - const bool intercept = true, - const arma::vec& weights = arma::vec()); + const arma::rowvec& responses, + const bool intercept = true); + + /** + * Train the LinearRegression model on the given data and weights. Careful! + * This will completely ignore and overwrite the existing model. This + * particular implementation does not have an incremental training algorithm. + * To set the regularization parameter lambda, call Lambda() or set a + * different value in the constructor. + * + * @param predictors X, the matrix of data points to train the model on. + * @param responses y, the responses to the data points. + * @param intercept Whether or not to fit an intercept term. + * @param weights Observation weights (for boosting). + */ + void Train(const arma::mat& predictors, + const arma::rowvec& responses, + const arma::rowvec& weights, + const bool intercept = true); /** * Calculate y_i for each data point in points. @@ -106,8 +166,28 @@ class LinearRegression * @param points Matrix of predictors (X). * @param responses Vector of responses (y). */ + mlpack_deprecated double ComputeError(const arma::mat& points, + const arma::vec& responses) const; + + /** + * Calculate the L2 squared error on the given predictors and responses using + * this linear regression model. This calculation returns + * + * \f[ + * (1 / n) * \| y - X B \|^2_2 + * \f] + * + * where \f$ y \f$ is the responses vector, \f$ X \f$ is the matrix of + * predictors, and \f$ B \f$ is the parameters of the trained linear + * regression model. + * + * As this number decreases to 0, the linear regression fit is better. + * + * @param points Matrix of predictors (X). + * @param responses Transposed vector of responses (y^T). + */ double ComputeError(const arma::mat& points, - const arma::vec& responses) const; + const arma::rowvec& responses) const; //! Return the parameters (the b vector). const arma::vec& Parameters() const { return parameters; } diff --git a/src/mlpack/methods/linear_regression/linear_regression_main.cpp b/src/mlpack/methods/linear_regression/linear_regression_main.cpp index 105aa094ec..95709d4417 100644 --- a/src/mlpack/methods/linear_regression/linear_regression_main.cpp +++ b/src/mlpack/methods/linear_regression/linear_regression_main.cpp @@ -69,7 +69,7 @@ int main(int argc, char* argv[]) << "(-T) is not specified." << endl; mat regressors; - vec responses; + rowvec responses; LinearRegression lr; lr.Lambda() = lambda; @@ -140,10 +140,10 @@ int main(int argc, char* argv[]) { // The initial predictors for y, Nx1. Timer::Start("load_responses"); - responses = std::move(CLI::GetParam("training_responses")); + responses = CLI::GetParam("training_responses").t(); Timer::Stop("load_responses"); - if (responses.n_rows != regressors.n_cols) + if (responses.n_cols != regressors.n_cols) Log::Fatal << "The responses must have the same number of rows as the " "training file." << endl; } diff --git a/src/mlpack/tests/linear_regression_test.cpp b/src/mlpack/tests/linear_regression_test.cpp index 409626ec31..74b42a0d6f 100644 --- a/src/mlpack/tests/linear_regression_test.cpp +++ b/src/mlpack/tests/linear_regression_test.cpp @@ -30,7 +30,7 @@ BOOST_AUTO_TEST_CASE(LinearRegressionTestCase) arma::mat points(3, 10); // Responses is the "correct" value for each point in predictors and points. - arma::vec responses(10); + arma::rowvec responses(10); // The values we get back when we predict for points. arma::rowvec predictions(10); @@ -76,7 +76,7 @@ BOOST_AUTO_TEST_CASE(ComputeErrorTest) arma::mat predictors; predictors << 0 << 1 << 2 << 4 << 8 << 16 << arma::endr << 16 << 8 << 4 << 2 << 1 << 0 << arma::endr; - arma::vec responses = "0 2 4 3 8 8"; + arma::rowvec responses = "0 2 4 3 8 8"; // http://www.mlpack.org/trac/ticket/298 // This dataset gives a cost of 1.189500337 (as calculated in Octave). @@ -95,7 +95,7 @@ BOOST_AUTO_TEST_CASE(ComputeErrorPerfectFitTest) arma::mat predictors; predictors << 0 << 1 << 2 << 1 << 6 << 2 << arma::endr << 0 << 1 << 2 << 2 << 2 << 6 << arma::endr; - arma::vec responses = "0 2 4 3 8 8"; + arma::rowvec responses = "0 2 4 3 8 8"; LinearRegression lr(predictors, responses); @@ -111,7 +111,7 @@ BOOST_AUTO_TEST_CASE(RidgeRegressionTest) // Create empty dataset. arma::mat data; data.zeros(10, 5000); // 10-dimensional, 5000 points. - arma::vec responses; + arma::rowvec responses; responses.zeros(5000); // 5000 points. // Any lambda greater than 0 works to make the predictors covariance matrix @@ -140,7 +140,7 @@ BOOST_AUTO_TEST_CASE(RidgeRegressionTestCase) arma::mat points(3, 10); // Responses is the "correct" value for each point in predictors and points. - arma::vec responses(10); + arma::rowvec responses(10); // The values we get back when we predict for points. arma::rowvec predictions(10); @@ -186,7 +186,7 @@ BOOST_AUTO_TEST_CASE(LinearRegressionTrainTest) { // Random dataset. arma::mat dataset = arma::randu(5, 1000); - arma::vec responses = arma::randu(1000); + arma::rowvec responses = arma::randu(1000); LinearRegression lr(dataset, responses, 0.3); LinearRegression lrTrain; From 433ff8e9ad70519bcb12c17db069de221553c531 Mon Sep 17 00:00:00 2001 From: Kirill Mishchenko Date: Mon, 22 May 2017 08:42:00 +0500 Subject: [PATCH 23/84] Change the Predict interface in LARS Make the predictions argument column-major in the method Predict. --- src/mlpack/methods/lars/lars.cpp | 13 +++++++++++-- src/mlpack/methods/lars/lars.hpp | 15 ++++++++++++++- src/mlpack/methods/lars/lars_main.cpp | 6 +++--- src/mlpack/tests/lars_test.cpp | 6 +++--- 4 files changed, 31 insertions(+), 9 deletions(-) diff --git a/src/mlpack/methods/lars/lars.cpp b/src/mlpack/methods/lars/lars.cpp index 1d892bb8a1..45480c175d 100644 --- a/src/mlpack/methods/lars/lars.cpp +++ b/src/mlpack/methods/lars/lars.cpp @@ -386,12 +386,21 @@ void LARS::Train(const arma::mat& data, void LARS::Predict(const arma::mat& points, arma::vec& predictions, const bool rowMajor) const +{ + arma::rowvec rowPredictions; + Predict(points, rowPredictions, rowMajor); + predictions = rowPredictions.t(); +} + +void LARS::Predict(const arma::mat& points, + arma::rowvec& predictions, + const bool rowMajor) const { // We really only need to store beta internally... if (rowMajor) - predictions = points * betaPath.back(); + predictions = trans(points * betaPath.back()); else - predictions = (betaPath.back().t() * points).t(); + predictions = betaPath.back().t() * points; } // Private functions. diff --git a/src/mlpack/methods/lars/lars.hpp b/src/mlpack/methods/lars/lars.hpp index 1925e02797..1d09b56398 100644 --- a/src/mlpack/methods/lars/lars.hpp +++ b/src/mlpack/methods/lars/lars.hpp @@ -215,8 +215,21 @@ class LARS * @param points The data points to regress on. * @param predictions y, which will contained calculated values on completion. */ + mlpack_deprecated void Predict(const arma::mat& points, + arma::vec& predictions, + const bool rowMajor = false) const; + + /** + * Predict y_i for each data point in the given data matrix using the + * currently-trained LARS model. + * + * @param points The data points to regress on. + * @param predictions y, which will contained calculated values on completion. + * @param rowMajor Should be true if the data points matrix is row-major and + * false otherwise. + */ void Predict(const arma::mat& points, - arma::vec& predictions, + arma::rowvec& predictions, const bool rowMajor = false) const; //! Access the set of active dimensions. diff --git a/src/mlpack/methods/lars/lars_main.cpp b/src/mlpack/methods/lars/lars_main.cpp index 1a2b3b70c1..743e1abe81 100644 --- a/src/mlpack/methods/lars/lars_main.cpp +++ b/src/mlpack/methods/lars/lars_main.cpp @@ -159,12 +159,12 @@ int main(int argc, char* argv[]) << "is not equal to the dimensionality of the model (" << lars.BetaPath().back().n_elem << ")!" << endl; - arma::vec predictions; + arma::rowvec predictions; lars.Predict(testPoints.t(), predictions, false); - // Save test predictions. One per line, so, don't transpose on save. + // Save test predictions (one per line). if (CLI::HasParam("output_predictions")) - CLI::GetParam("output_predictions") = std::move(predictions); + CLI::GetParam("output_predictions") = predictions.t(); } if (CLI::HasParam("output_model")) diff --git a/src/mlpack/tests/lars_test.cpp b/src/mlpack/tests/lars_test.cpp index b33b33e9a5..395fdaf173 100644 --- a/src/mlpack/tests/lars_test.cpp +++ b/src/mlpack/tests/lars_test.cpp @@ -180,9 +180,9 @@ BOOST_AUTO_TEST_CASE(PredictTest) // Calculate what the actual error should be with these regression // parameters. arma::vec betaOptPred = (X * X.t()) * betaOpt; - arma::vec predictions; + arma::rowvec predictions; lars.Predict(X, predictions); - arma::vec adjPred = X * predictions; + arma::vec adjPred = X * predictions.t(); BOOST_REQUIRE_EQUAL(predictions.n_elem, 1000); for (size_t i = 0; i < betaOptPred.n_elem; ++i) @@ -211,7 +211,7 @@ BOOST_AUTO_TEST_CASE(PredictRowMajorTest) // Get both row-major and column-major predictions. Make sure they are the // same. - arma::vec rowMajorPred, colMajorPred; + arma::rowvec rowMajorPred, colMajorPred; lars.Predict(X, colMajorPred); lars.Predict(X.t(), rowMajorPred, true); From a688853091866287a6e39e09158da10e915f01e6 Mon Sep 17 00:00:00 2001 From: Kirill Mishchenko Date: Mon, 22 May 2017 10:46:06 +0500 Subject: [PATCH 24/84] Change constructor and Train signatures in LARS This commit makes responses column-major. --- src/mlpack/methods/lars/lars.cpp | 51 ++++++++++++++- src/mlpack/methods/lars/lars.hpp | 92 +++++++++++++++++++++++++-- src/mlpack/methods/lars/lars_main.cpp | 7 +- src/mlpack/tests/lars_test.cpp | 40 ++++++------ 4 files changed, 160 insertions(+), 30 deletions(-) diff --git a/src/mlpack/methods/lars/lars.cpp b/src/mlpack/methods/lars/lars.cpp index 45480c175d..721d7563d9 100644 --- a/src/mlpack/methods/lars/lars.cpp +++ b/src/mlpack/methods/lars/lars.cpp @@ -58,7 +58,8 @@ LARS::LARS(const arma::mat& data, lambda2(lambda2), tolerance(tolerance) { - Train(data, responses, transposeData); + arma::rowvec rowResponses = responses.t(); + Train(data, rowResponses, transposeData); } LARS::LARS(const arma::mat& data, @@ -76,6 +77,32 @@ LARS::LARS(const arma::mat& data, elasticNet((lambda1 != 0) && (lambda2 != 0)), lambda2(lambda2), tolerance(tolerance) +{ + arma::rowvec rowResponses = responses.t(); + Train(data, rowResponses, transposeData); +} + +LARS::LARS(const arma::mat& data, + const arma::rowvec& responses, + const bool transposeData, + const bool useCholesky, + const double lambda1, + const double lambda2, + const double tolerance) : + LARS(useCholesky, lambda1, lambda2, tolerance) +{ + Train(data, responses, transposeData); +} + +LARS::LARS(const arma::mat& data, + const arma::rowvec& responses, + const bool transposeData, + const bool useCholesky, + const arma::mat& gramMatrix, + const double lambda1, + const double lambda2, + const double tolerance) : + LARS(useCholesky, gramMatrix, lambda1, lambda2, tolerance) { Train(data, responses, transposeData); } @@ -84,6 +111,24 @@ void LARS::Train(const arma::mat& matX, const arma::vec& y, arma::vec& beta, const bool transposeData) +{ + arma::rowvec rowY = y.t(); + Train(matX, rowY, beta, transposeData); +} + +void LARS::Train(const arma::mat& data, + const arma::vec& responses, + const bool transposeData) +{ + arma::rowvec rowResponses = responses.t(); + arma::vec beta; + Train(data, rowResponses, beta, transposeData); +} + +void LARS::Train(const arma::mat& matX, + const arma::rowvec& y, + arma::vec& beta, + const bool transposeData) { Timer::Start("lars_regression"); @@ -104,7 +149,7 @@ void LARS::Train(const arma::mat& matX, dataTrans = trans(matX); // Compute X' * y. - arma::vec vecXTy = trans(dataRef) * y; + arma::vec vecXTy = trans(y * dataRef); // Set up active set variables. In the beginning, the active set has size 0 // (all dimensions are inactive). @@ -376,7 +421,7 @@ void LARS::Train(const arma::mat& matX, } void LARS::Train(const arma::mat& data, - const arma::vec& responses, + const arma::rowvec& responses, const bool transposeData) { arma::vec beta; diff --git a/src/mlpack/methods/lars/lars.hpp b/src/mlpack/methods/lars/lars.hpp index 1d09b56398..baba45f4f6 100644 --- a/src/mlpack/methods/lars/lars.hpp +++ b/src/mlpack/methods/lars/lars.hpp @@ -122,6 +122,54 @@ class LARS const double lambda2 = 0.0, const double tolerance = 1e-16); + /** + * Set the parameters to LARS and run training. Both lambda1 and lambda2 + * are set by default to 0. + * + * @param data Input data. + * @param responses A vector of targets. + * @param transposeData Should be true if the input data is column-major and + * false otherwise. + * @param useCholesky Whether or not to use Cholesky decomposition when + * solving linear system (as opposed to using the full Gram matrix). + * @param lambda1 Regularization parameter for l1-norm penalty. + * @param lambda2 Regularization parameter for l2-norm penalty. + * @param tolerance Run until the maximum correlation of elements in (X^T y) + * is less than this. + */ + mlpack_deprecated LARS(const arma::mat& data, + const arma::vec& responses, + const bool transposeData = true, + const bool useCholesky = false, + const double lambda1 = 0.0, + const double lambda2 = 0.0, + const double tolerance = 1e-16); + + /** + * Set the parameters to LARS, pass in a precalculated Gram matrix, and run + * training. Both lambda1 and lambda2 are set by default to 0. + * + * @param data Input data. + * @param responses A vector of targets. + * @param transposeData Should be true if the input data is column-major and + * false otherwise. + * @param useCholesky Whether or not to use Cholesky decomposition when + * solving linear system (as opposed to using the full Gram matrix). + * @param gramMatrix Gram matrix. + * @param lambda1 Regularization parameter for l1-norm penalty. + * @param lambda2 Regularization parameter for l2-norm penalty. + * @param tolerance Run until the maximum correlation of elements in (X^T y) + * is less than this. + */ + mlpack_deprecated LARS(const arma::mat& data, + const arma::vec& responses, + const bool transposeData, + const bool useCholesky, + const arma::mat& gramMatrix, + const double lambda1 = 0.0, + const double lambda2 = 0.0, + const double tolerance = 1e-16); + /** * Set the parameters to LARS and run training. Both lambda1 and lambda2 * are set by default to 0. @@ -138,7 +186,7 @@ class LARS * is less than this. */ LARS(const arma::mat& data, - const arma::vec& responses, + const arma::rowvec& responses, const bool transposeData = true, const bool useCholesky = false, const double lambda1 = 0.0, @@ -162,7 +210,7 @@ class LARS * is less than this. */ LARS(const arma::mat& data, - const arma::vec& responses, + const arma::rowvec& responses, const bool transposeData, const bool useCholesky, const arma::mat& gramMatrix, @@ -170,6 +218,42 @@ class LARS const double lambda2 = 0.0, const double tolerance = 1e-16); + /** + * Run LARS. The input matrix (like all mlpack matrices) should be + * column-major -- each column is an observation and each row is a dimension. + * However, because LARS is more efficient on a row-major matrix, this method + * will (internally) transpose the matrix. If this transposition is not + * necessary (i.e., you want to pass in a row-major matrix), pass 'false' for + * the transposeData parameter. + * + * @param data Column-major input data (or row-major input data if rowMajor = + * true). + * @param responses A vector of targets. + * @param beta Vector to store the solution (the coefficients) in. + * @param transposeData Set to false if the data is row-major. + */ + mlpack_deprecated void Train(const arma::mat& data, + const arma::vec& responses, + arma::vec& beta, + const bool transposeData = true); + + /** + * Run LARS. The input matrix (like all mlpack matrices) should be + * column-major -- each column is an observation and each row is a dimension. + * However, because LARS is more efficient on a row-major matrix, this method + * will (internally) transpose the matrix. If this transposition is not + * necessary (i.e., you want to pass in a row-major matrix), pass 'false' for + * the transposeData parameter. + * + * @param data Input data. + * @param responses A vector of targets. + * @param transposeData Should be true if the input data is column-major and + * false otherwise. + */ + mlpack_deprecated void Train(const arma::mat& data, + const arma::vec& responses, + const bool transposeData = true); + /** * Run LARS. The input matrix (like all mlpack matrices) should be * column-major -- each column is an observation and each row is a dimension. @@ -185,7 +269,7 @@ class LARS * @param transposeData Set to false if the data is row-major. */ void Train(const arma::mat& data, - const arma::vec& responses, + const arma::rowvec& responses, arma::vec& beta, const bool transposeData = true); @@ -203,7 +287,7 @@ class LARS * false otherwise. */ void Train(const arma::mat& data, - const arma::vec& responses, + const arma::rowvec& responses, const bool transposeData = true); /** diff --git a/src/mlpack/methods/lars/lars_main.cpp b/src/mlpack/methods/lars/lars_main.cpp index 743e1abe81..4d9f90c87c 100644 --- a/src/mlpack/methods/lars/lars_main.cpp +++ b/src/mlpack/methods/lars/lars_main.cpp @@ -128,9 +128,9 @@ int main(int argc, char* argv[]) mat matY = std::move(CLI::GetParam("responses")); // Make sure y is oriented the right way. - if (matY.n_rows == 1) + if (matY.n_cols == 1) matY = trans(matY); - if (matY.n_cols > 1) + if (matY.n_rows > 1) Log::Fatal << "Only one column or row allowed in responses file!" << endl; if (matY.n_elem != matX.n_rows) @@ -138,7 +138,8 @@ int main(int argc, char* argv[]) << endl; vec beta; - lars.Train(matX, matY.unsafe_col(0), beta, false /* do not transpose */); + arma::rowvec y = std::move(matY); + lars.Train(matX, y, beta, false /* do not transpose */); } else // We must have --input_model_file. { diff --git a/src/mlpack/tests/lars_test.cpp b/src/mlpack/tests/lars_test.cpp index 395fdaf173..b0d2a19ae2 100644 --- a/src/mlpack/tests/lars_test.cpp +++ b/src/mlpack/tests/lars_test.cpp @@ -23,11 +23,11 @@ using namespace mlpack::regression; BOOST_AUTO_TEST_SUITE(LARSTest); -void GenerateProblem(arma::mat& X, arma::vec& y, size_t nPoints, size_t nDims) +void GenerateProblem(arma::mat& X, arma::rowvec& y, size_t nPoints, size_t nDims) { X = arma::randn(nDims, nPoints); arma::vec beta = arma::randn(nDims, 1); - y = trans(X) * beta; + y = beta.t() * X; } void LARSVerifyCorrectness(arma::vec beta, arma::vec errCorr, double lambda) @@ -57,14 +57,14 @@ void LARSVerifyCorrectness(arma::vec beta, arma::vec errCorr, double lambda) void LassoTest(size_t nPoints, size_t nDims, bool elasticNet, bool useCholesky) { arma::mat X; - arma::vec y; + arma::rowvec y; for (size_t i = 0; i < 100; i++) { GenerateProblem(X, y, nPoints, nDims); // Armadillo's median is broken, so... - arma::vec sortedAbsCorr = sort(abs(X * y)); + arma::vec sortedAbsCorr = sort(abs(X * y.t())); double lambda1 = sortedAbsCorr(nDims / 2); double lambda2; if (elasticNet) @@ -78,7 +78,7 @@ void LassoTest(size_t nPoints, size_t nDims, bool elasticNet, bool useCholesky) lars.Train(X, y, betaOpt); arma::vec errCorr = (X * trans(X) + lambda2 * - arma::eye(nDims, nDims)) * betaOpt - X * y; + arma::eye(nDims, nDims)) * betaOpt - X * y.t(); LARSVerifyCorrectness(betaOpt, errCorr, lambda1); } @@ -116,7 +116,7 @@ BOOST_AUTO_TEST_CASE(CholeskySingularityTest) data::Load("lars_dependent_x.csv", X); data::Load("lars_dependent_y.csv", Y); - arma::vec y = Y.row(0).t(); + arma::rowvec y = Y.row(0); // Test for a couple values of lambda1. for (double lambda1 = 0.0; lambda1 < 1.0; lambda1 += 0.1) @@ -125,7 +125,7 @@ BOOST_AUTO_TEST_CASE(CholeskySingularityTest) arma::vec betaOpt; lars.Train(X, y, betaOpt); - arma::vec errCorr = (X * X.t()) * betaOpt - X * y; + arma::vec errCorr = (X * X.t()) * betaOpt - X * y.t(); LARSVerifyCorrectness(betaOpt, errCorr, lambda1); } @@ -140,7 +140,7 @@ BOOST_AUTO_TEST_CASE(NoCholeskySingularityTest) data::Load("lars_dependent_x.csv", X); data::Load("lars_dependent_y.csv", Y); - arma::vec y = Y.row(0).t(); + arma::rowvec y = Y.row(0); // Test for a couple values of lambda1. for (double lambda1 = 0.0; lambda1 < 1.0; lambda1 += 0.1) @@ -149,7 +149,7 @@ BOOST_AUTO_TEST_CASE(NoCholeskySingularityTest) arma::vec betaOpt; lars.Train(X, y, betaOpt); - arma::vec errCorr = (X * X.t()) * betaOpt - X * y; + arma::vec errCorr = (X * X.t()) * betaOpt - X * y.t(); // #373: this test fails on i386 only sometimes. // LARSVerifyCorrectness(betaOpt, errCorr, lambda1); @@ -165,7 +165,7 @@ BOOST_AUTO_TEST_CASE(PredictTest) bool useCholesky = bool(i); arma::mat X; - arma::vec y; + arma::rowvec y; GenerateProblem(X, y, 1000, 100); @@ -200,7 +200,7 @@ BOOST_AUTO_TEST_CASE(PredictTest) BOOST_AUTO_TEST_CASE(PredictRowMajorTest) { arma::mat X; - arma::vec y; + arma::rowvec y; GenerateProblem(X, y, 1000, 100); // Set lambdas to 0. @@ -232,11 +232,11 @@ BOOST_AUTO_TEST_CASE(PredictRowMajorTest) BOOST_AUTO_TEST_CASE(RetrainTest) { arma::mat origX; - arma::vec origY; + arma::rowvec origY; GenerateProblem(origX, origY, 1000, 50); arma::mat newX; - arma::vec newY; + arma::rowvec newY; GenerateProblem(newX, newY, 750, 75); LARS lars(false, 0.1, 0.1); @@ -247,7 +247,7 @@ BOOST_AUTO_TEST_CASE(RetrainTest) lars.Train(newX, newY, betaOpt); arma::vec errCorr = (newX * trans(newX) + 0.1 * - arma::eye(75, 75)) * betaOpt - newX * newY; + arma::eye(75, 75)) * betaOpt - newX * newY.t(); LARSVerifyCorrectness(betaOpt, errCorr, 0.1); } @@ -259,11 +259,11 @@ BOOST_AUTO_TEST_CASE(RetrainTest) BOOST_AUTO_TEST_CASE(RetrainCholeskyTest) { arma::mat origX; - arma::vec origY; + arma::rowvec origY; GenerateProblem(origX, origY, 1000, 50); arma::mat newX; - arma::vec newY; + arma::rowvec newY; GenerateProblem(newX, newY, 750, 75); LARS lars(true, 0.1, 0.1); @@ -274,7 +274,7 @@ BOOST_AUTO_TEST_CASE(RetrainCholeskyTest) lars.Train(newX, newY, betaOpt); arma::vec errCorr = (newX * trans(newX) + 0.1 * - arma::eye(75, 75)) * betaOpt - newX * newY; + arma::eye(75, 75)) * betaOpt - newX * newY.t(); LARSVerifyCorrectness(betaOpt, errCorr, 0.1); } @@ -286,7 +286,7 @@ BOOST_AUTO_TEST_CASE(RetrainCholeskyTest) BOOST_AUTO_TEST_CASE(TrainingAndAccessingBetaTest) { arma::mat X; - arma::vec y; + arma::rowvec y; GenerateProblem(X, y, 1000, 100); @@ -309,7 +309,7 @@ BOOST_AUTO_TEST_CASE(TrainingAndAccessingBetaTest) BOOST_AUTO_TEST_CASE(TrainingConstructorWithDefaultsTest) { arma::mat X; - arma::vec y; + arma::rowvec y; GenerateProblem(X, y, 1000, 100); @@ -331,7 +331,7 @@ BOOST_AUTO_TEST_CASE(TrainingConstructorWithDefaultsTest) BOOST_AUTO_TEST_CASE(TrainingConstructorWithNonDefaultsTest) { arma::mat X; - arma::vec y; + arma::rowvec y; GenerateProblem(X, y, 1000, 100); From 5e17a153ea28fabd677d0cec94d01e7144b11f88 Mon Sep 17 00:00:00 2001 From: Kirill Mishchenko Date: Mon, 22 May 2017 18:12:46 +0500 Subject: [PATCH 25/84] Fix errors and warnings --- .../core/dists/regression_distribution.cpp | 20 ++++++++++--------- .../core/dists/regression_distribution.hpp | 7 ++++--- .../methods/local_coordinate_coding/lcc.cpp | 3 ++- .../methods/sparse_coding/sparse_coding.cpp | 3 ++- src/mlpack/tests/serialization_test.cpp | 8 ++++---- 5 files changed, 23 insertions(+), 18 deletions(-) diff --git a/src/mlpack/core/dists/regression_distribution.cpp b/src/mlpack/core/dists/regression_distribution.cpp index e306bc8096..e61721b6e0 100644 --- a/src/mlpack/core/dists/regression_distribution.cpp +++ b/src/mlpack/core/dists/regression_distribution.cpp @@ -23,11 +23,11 @@ using namespace mlpack::distribution; void RegressionDistribution::Train(const arma::mat& observations) { regression::LinearRegression lr(observations.rows(1, observations.n_rows - 1), - (observations.row(0)).t(), 0, true); + arma::rowvec(observations.row(0)), 0, true); rf = lr; - arma::vec fitted; + arma::rowvec fitted; lr.Predict(observations.rows(1, observations.n_rows - 1), fitted); - err.Train(observations.row(0) - fitted.t()); + err.Train(observations.row(0) - fitted); } /** @@ -39,11 +39,11 @@ void RegressionDistribution::Train(const arma::mat& observations, const arma::vec& weights) { regression::LinearRegression lr(observations.rows(1, observations.n_rows - 1), - (observations.row(0)).t(), 0, true, weights); + arma::rowvec(observations.row(0)), weights, 0, true); rf = lr; - arma::vec fitted; + arma::rowvec fitted; lr.Predict(observations.rows(1, observations.n_rows - 1), fitted); - err.Train(observations.row(0) - fitted.t(), weights); + err.Train(observations.row(0) - fitted, weights); } /** @@ -53,13 +53,15 @@ void RegressionDistribution::Train(const arma::mat& observations, */ double RegressionDistribution::Probability(const arma::vec& observation) const { - arma::vec fitted; + arma::rowvec fitted; rf.Predict(observation.rows(1, observation.n_rows-1), fitted); - return err.Probability(observation(0)-fitted); + return err.Probability(observation(0)-fitted.t()); } void RegressionDistribution::Predict(const arma::mat& points, arma::vec& predictions) const { - rf.Predict(points, predictions); + arma::rowvec rowPredictions; + rf.Predict(points, rowPredictions); + predictions = rowPredictions.t(); } diff --git a/src/mlpack/core/dists/regression_distribution.hpp b/src/mlpack/core/dists/regression_distribution.hpp index d38a2a21e2..66e075444d 100644 --- a/src/mlpack/core/dists/regression_distribution.hpp +++ b/src/mlpack/core/dists/regression_distribution.hpp @@ -49,12 +49,13 @@ class RegressionDistribution * @param responses Vector of responses (y). */ RegressionDistribution(const arma::mat& predictors, - const arma::vec& responses) : - rf(regression::LinearRegression(predictors, responses)) + const arma::vec& responses) { + arma::rowvec rowResponses = responses.t(); + rf.Train(predictors, rowResponses); err = GaussianDistribution(1); arma::mat cov(1, 1); - cov(0, 0) = rf.ComputeError(predictors, responses); + cov(0, 0) = rf.ComputeError(predictors, rowResponses); err.Covariance(std::move(cov)); } diff --git a/src/mlpack/methods/local_coordinate_coding/lcc.cpp b/src/mlpack/methods/local_coordinate_coding/lcc.cpp index 8c420d8413..b066d4c7d5 100644 --- a/src/mlpack/methods/local_coordinate_coding/lcc.cpp +++ b/src/mlpack/methods/local_coordinate_coding/lcc.cpp @@ -57,7 +57,8 @@ void LocalCoordinateCoding::Encode(const arma::mat& data, arma::mat& codes) // Run LARS for this point, by making an alias of the point and passing // that. arma::vec beta = codes.unsafe_col(i); - lars.Train(dictPrime, data.unsafe_col(i), beta, false); + arma::rowvec responses = data.unsafe_col(i).t(); + lars.Train(dictPrime, responses, beta, false); beta %= invW; // Remember, beta is an alias of codes.col(i). } } diff --git a/src/mlpack/methods/sparse_coding/sparse_coding.cpp b/src/mlpack/methods/sparse_coding/sparse_coding.cpp index c7f1d67aa2..8b8a62be07 100644 --- a/src/mlpack/methods/sparse_coding/sparse_coding.cpp +++ b/src/mlpack/methods/sparse_coding/sparse_coding.cpp @@ -54,7 +54,8 @@ void SparseCoding::Encode(const arma::mat& data, arma::mat& codes) // place the result directly into that; then we will not need to have an // extra copy. arma::vec code = codes.unsafe_col(i); - lars.Train(dictionary, data.unsafe_col(i), code, false); + arma::rowvec responses = data.unsafe_col(i).t(); + lars.Train(dictionary, responses, code, false); } } diff --git a/src/mlpack/tests/serialization_test.cpp b/src/mlpack/tests/serialization_test.cpp index 2d92f636f1..cc38c764f9 100644 --- a/src/mlpack/tests/serialization_test.cpp +++ b/src/mlpack/tests/serialization_test.cpp @@ -274,7 +274,7 @@ BOOST_AUTO_TEST_CASE(LinearRegressionTest) // Generate some random data. mat data; data.randn(15, 800); - vec responses; + rowvec responses; responses.randn(800, 1); LinearRegression lr(data, responses, 0.05); // Train the model. @@ -1288,7 +1288,7 @@ BOOST_AUTO_TEST_CASE(LARSTest) // Create a dataset. arma::mat X = arma::randn(75, 250); arma::vec beta = arma::randn(75, 1); - arma::vec y = trans(X) * beta; + arma::rowvec y = beta.t() * X; LARS lars(true, 0.1, 0.1); arma::vec betaOpt; @@ -1301,14 +1301,14 @@ BOOST_AUTO_TEST_CASE(LARSTest) // Train textLars. arma::mat textX = arma::randn(25, 150); arma::vec textBeta = arma::randn(25, 1); - arma::vec textY = trans(textX) * textBeta; + arma::rowvec textY = textBeta.t() * textX; arma::vec textBetaOpt; textLars.Train(textX, textY, textBetaOpt); SerializeObjectAll(lars, xmlLars, binaryLars, textLars); // Now, check that predictions are the same. - arma::vec pred, xmlPred, textPred, binaryPred; + arma::rowvec pred, xmlPred, textPred, binaryPred; lars.Predict(X, pred); xmlLars.Predict(X, xmlPred); textLars.Predict(X, textPred); From 188187f46aee5493258f348762fc74908327de77 Mon Sep 17 00:00:00 2001 From: Shangtong Zhang Date: Tue, 23 May 2017 11:57:37 -0600 Subject: [PATCH 26/84] Minor style fix --- .../reinforcement_learning/replay/random_replay.hpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/mlpack/methods/reinforcement_learning/replay/random_replay.hpp b/src/mlpack/methods/reinforcement_learning/replay/random_replay.hpp index a6e89b06f7..29bc16d1d6 100644 --- a/src/mlpack/methods/reinforcement_learning/replay/random_replay.hpp +++ b/src/mlpack/methods/reinforcement_learning/replay/random_replay.hpp @@ -29,11 +29,11 @@ namespace rl { * For more information, see the following. * * @code - * @phdthesis {lin1993reinforcement, - * title = {Reinforcement learning for robots using neural networks}, - * author = {Lin, Long-Ji}, - * year = {1993}, - * school = {Fujitsu Laboratories Ltd} + * @phdthesis{lin1993reinforcement, + * title = {Reinforcement learning for robots using neural networks}, + * author = {Lin, Long-Ji}, + * year = {1993}, + * school = {Fujitsu Laboratories Ltd} * } * @endcode * From 5c456fd9c6bdcca1d9e7de8d3326aa6aabde6fbd Mon Sep 17 00:00:00 2001 From: Kirill Mishchenko Date: Wed, 24 May 2017 07:16:52 +0500 Subject: [PATCH 27/84] Change the definition of training_responses Change the definition of training_responses for mlpack_linear_regression. --- .../methods/linear_regression/linear_regression_main.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/mlpack/methods/linear_regression/linear_regression_main.cpp b/src/mlpack/methods/linear_regression/linear_regression_main.cpp index 95709d4417..72bfff58b5 100644 --- a/src/mlpack/methods/linear_regression/linear_regression_main.cpp +++ b/src/mlpack/methods/linear_regression/linear_regression_main.cpp @@ -39,7 +39,7 @@ PROGRAM_INFO("Simple Linear Regression and Prediction", PARAM_MATRIX_IN("training", "Matrix containing training set X (regressors).", "t"); -PARAM_COL_IN("training_responses", "Optional vector containing y " +PARAM_ROW_IN("training_responses", "Optional vector containing y " "(responses). If not given, the responses are assumed to be the last row " "of the input file.", "r"); @@ -140,7 +140,7 @@ int main(int argc, char* argv[]) { // The initial predictors for y, Nx1. Timer::Start("load_responses"); - responses = CLI::GetParam("training_responses").t(); + responses = CLI::GetParam("training_responses"); Timer::Stop("load_responses"); if (responses.n_cols != regressors.n_cols) From cc23c359a77670a115da68a13f32441e6e727d5b Mon Sep 17 00:00:00 2001 From: Kirill Mishchenko Date: Wed, 24 May 2017 08:11:45 +0500 Subject: [PATCH 28/84] Make predictions and responses column-major Make predictions and responses column-major in RegressionDistribution. --- .../core/dists/regression_distribution.cpp | 16 +++++++- .../core/dists/regression_distribution.hpp | 40 ++++++++++++++++--- src/mlpack/tests/serialization_test.cpp | 4 +- 3 files changed, 50 insertions(+), 10 deletions(-) diff --git a/src/mlpack/core/dists/regression_distribution.cpp b/src/mlpack/core/dists/regression_distribution.cpp index e61721b6e0..50d186b838 100644 --- a/src/mlpack/core/dists/regression_distribution.cpp +++ b/src/mlpack/core/dists/regression_distribution.cpp @@ -37,13 +37,19 @@ void RegressionDistribution::Train(const arma::mat& observations) */ void RegressionDistribution::Train(const arma::mat& observations, const arma::vec& weights) +{ + Train(observations, arma::rowvec(weights.t())); +} + +void RegressionDistribution::Train(const arma::mat& observations, + const arma::rowvec& weights) { regression::LinearRegression lr(observations.rows(1, observations.n_rows - 1), arma::rowvec(observations.row(0)), weights, 0, true); rf = lr; arma::rowvec fitted; lr.Predict(observations.rows(1, observations.n_rows - 1), fitted); - err.Train(observations.row(0) - fitted, weights); + err.Train(observations.row(0) - fitted, weights.t()); } /** @@ -62,6 +68,12 @@ void RegressionDistribution::Predict(const arma::mat& points, arma::vec& predictions) const { arma::rowvec rowPredictions; - rf.Predict(points, rowPredictions); + Predict(points, rowPredictions); predictions = rowPredictions.t(); } + +void RegressionDistribution::Predict(const arma::mat& points, + arma::rowvec& predictions) const +{ + rf.Predict(points, predictions); +} diff --git a/src/mlpack/core/dists/regression_distribution.hpp b/src/mlpack/core/dists/regression_distribution.hpp index 66e075444d..13b299e0e5 100644 --- a/src/mlpack/core/dists/regression_distribution.hpp +++ b/src/mlpack/core/dists/regression_distribution.hpp @@ -41,6 +41,18 @@ class RegressionDistribution */ RegressionDistribution() { /* nothing to do */ } + /** + * Create a Conditional Gaussian distribution with conditional mean function + * obtained by running RegressionFunction on predictors, responses. + * + * @param predictors Matrix of predictors (X). + * @param responses Vector of responses (y). + */ + mlpack_deprecated RegressionDistribution(const arma::mat& predictors, + const arma::vec& responses) : + RegressionDistribution(predictors, arma::rowvec(responses.t())) + {} + /** * Create a Conditional Gaussian distribution with conditional mean function * obtained by running RegressionFunction on predictors, responses. @@ -49,13 +61,12 @@ class RegressionDistribution * @param responses Vector of responses (y). */ RegressionDistribution(const arma::mat& predictors, - const arma::vec& responses) + const arma::rowvec& responses) { - arma::rowvec rowResponses = responses.t(); - rf.Train(predictors, rowResponses); + rf.Train(predictors, responses); err = GaussianDistribution(1); arma::mat cov(1, 1); - cov(0, 0) = rf.ComputeError(predictors, rowResponses); + cov(0, 0) = rf.ComputeError(predictors, responses); err.Covariance(std::move(cov)); } @@ -91,7 +102,15 @@ class RegressionDistribution * * @param weights probability that given observation is from distribution */ - void Train(const arma::mat& observations, const arma::vec& weights); + mlpack_deprecated void Train(const arma::mat& observations, + const arma::vec& weights); + + /** + * Estimate parameters using provided observation weights + * + * @param weights probability that given observation is from distribution + */ + void Train(const arma::mat& observations, const arma::rowvec& weights); /** * Evaluate probability density function of given observation @@ -115,7 +134,16 @@ class RegressionDistribution * @param points the data points to calculate with. * @param predictions y, will contain calculated values on completion. */ - void Predict(const arma::mat& points, arma::vec& predictions) const; + mlpack_deprecated void Predict(const arma::mat& points, + arma::vec& predictions) const; + + /** + * Calculate y_i for each data point in points. + * + * @param points the data points to calculate with. + * @param predictions y, will contain calculated values on completion. + */ + void Predict(const arma::mat& points, arma::rowvec& predictions) const; //! Return the parameters (the b vector). const arma::vec& Parameters() const { return rf.Parameters(); } diff --git a/src/mlpack/tests/serialization_test.cpp b/src/mlpack/tests/serialization_test.cpp index cc38c764f9..21ec438908 100644 --- a/src/mlpack/tests/serialization_test.cpp +++ b/src/mlpack/tests/serialization_test.cpp @@ -295,8 +295,8 @@ BOOST_AUTO_TEST_CASE(RegressionDistributionTest) // Generate some random data. mat data; data.randn(15, 800); - vec responses; - responses.randn(800, 1); + rowvec responses; + responses.randn(800); RegressionDistribution rd(data, responses); RegressionDistribution xmlRd, textRd, binaryRd; From 2fd5333b663eb51db60117c66b2f6fdc53f76eb0 Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Wed, 24 May 2017 17:16:26 -0400 Subject: [PATCH 29/84] Fix places where weighted learning was not handled. --- .../all_categorical_split_impl.hpp | 27 +++++- .../best_binary_numeric_split_impl.hpp | 49 +++++++--- .../methods/decision_tree/decision_tree.hpp | 5 +- .../decision_tree/decision_tree_impl.hpp | 94 +++++++++++++++---- .../methods/decision_tree/gini_gain.hpp | 8 +- 5 files changed, 141 insertions(+), 42 deletions(-) diff --git a/src/mlpack/methods/decision_tree/all_categorical_split_impl.hpp b/src/mlpack/methods/decision_tree/all_categorical_split_impl.hpp index 1f95f1443e..1cd90298b8 100644 --- a/src/mlpack/methods/decision_tree/all_categorical_split_impl.hpp +++ b/src/mlpack/methods/decision_tree/all_categorical_split_impl.hpp @@ -30,11 +30,25 @@ double AllCategoricalSplit::SplitIfBetter( { // Count the number of elements in each potential child. const double epsilon = 1e-7; // Tolerance for floating-point errors. - arma::Col counts(numCategories); - counts.zeros(); + arma::Col counts(numCategories, arma::fill::zeros); + + // If we are using weighted training, learn the weights for each child too. + arma::vec childWeightSums; + double sumWeight = 0.0; + if (UseWeights) + childWeightSums.zeros(numCategories); + for (size_t i = 0; i < data.n_elem; ++i) + { counts[(size_t) data[i]]++; + if (UseWeights) + { + childWeightSums[(size_t) data[i]] += weights[i]; + sumWeight += weights[i]; + } + } + // If each child will have the minimum number of points in it, we can split. // Otherwise we can't. if (arma::min(counts) < minimumLeafSize) @@ -49,7 +63,8 @@ double AllCategoricalSplit::SplitIfBetter( { // Labels and weights should have same length. childLabels[i].zeros(counts[i]); - childWeights[i].zeros(counts[i]); + if (UseWeights) + childWeights[i].zeros(counts[i]); } // Extract labels for each child. @@ -60,7 +75,7 @@ double AllCategoricalSplit::SplitIfBetter( if (UseWeights) { childLabels[category][childPositions[category]] = labels[i]; - childWeights[category][childPositions[category]++] = weights[i] ? weights[i] : 0; + childWeights[category][childPositions[category]++] = weights[i]; } else { @@ -72,7 +87,9 @@ double AllCategoricalSplit::SplitIfBetter( for (size_t i = 0; i < counts.n_elem; ++i) { // Calculate the gain of this child. - const double childPct = double(counts[i]) / double(data.n_elem); + const double childPct = UseWeights ? + double(childWeightSums[i]) / sumWeight : + double(counts[i]) / double(data.n_elem); const double childGain = FitnessFunction::template Evaluate(childLabels[i], numClasses, childWeights[i]); diff --git a/src/mlpack/methods/decision_tree/best_binary_numeric_split_impl.hpp b/src/mlpack/methods/decision_tree/best_binary_numeric_split_impl.hpp index 1e573e909d..feb21020e5 100644 --- a/src/mlpack/methods/decision_tree/best_binary_numeric_split_impl.hpp +++ b/src/mlpack/methods/decision_tree/best_binary_numeric_split_impl.hpp @@ -34,12 +34,14 @@ double BestBinaryNumericSplit::SplitIfBetter( // Next, sort the data. arma::uvec sortedIndices = arma::sort_index(data); arma::Row sortedLabels(labels.n_elem); - arma::Row sortedWeights(labels.n_elem); - sortedWeights.zeros(); + arma::Row sortedWeights; for (size_t i = 0; i < sortedLabels.n_elem; ++i) sortedLabels[sortedIndices[i]] = labels[i]; + + // Only initialize if we are using weights. if (UseWeights) { + sortedWeights.set_size(sortedLabels.n_elem); // The weights must keep the same order of labels for (size_t i = 0; i < sortedLabels.n_elem; ++i) sortedWeights[sortedIndices[i]] = weights[i]; @@ -55,19 +57,40 @@ double BestBinaryNumericSplit::SplitIfBetter( if (data[sortedIndices[index]] == data[sortedIndices[index - 1]]) continue; - // Calculate the gain for the left and right child. - const double leftGain = FitnessFunction::template Evaluate(sortedLabels.subvec(0, - index - 1), numClasses, sortedWeights.subvec(0, index - 1)); - const double rightGain = FitnessFunction::template Evaluate(sortedLabels.subvec( - index, sortedLabels.n_elem - 1), numClasses, - sortedWeights.subvec(index, sortedLabels.n_elem - 1)); + // Calculate the gain for the left and right child. Only use weights if + // needed. + const double leftGain = UseWeights ? + FitnessFunction::template Evaluate(sortedLabels.subvec(0, + index - 1), numClasses, sortedWeights.subvec(0, index - 1)) : + FitnessFunction::template Evaluate(sortedLabels.subvec(0, + index - 1), numClasses, sortedWeights /* ignored */); + const double rightGain = UseWeights ? + FitnessFunction::template Evaluate(sortedLabels.subvec(index, + sortedLabels.n_elem - 1), numClasses, sortedWeights.subvec(index, + sortedLabels.n_elem - 1)) : + FitnessFunction::template Evaluate(sortedLabels.subvec(index, + sortedLabels.n_elem - 1), numClasses, sortedWeights /* ignored */); - // Calculate the fraction of points in the left and right children. - const double leftRatio = double(index) / double(sortedLabels.n_elem); - const double rightRatio = 1.0 - leftRatio; + double gain; + if (UseWeights) + { + const double leftWeights = arma::accu(sortedWeights.subvec(0, index - 1)); + const double rightWeights = arma::accu(sortedWeights.subvec(index, + sortedWeights.n_elem - 1)); + const double fullWeight = leftWeights + rightWeights; - // Calculate the gain at this split point. - const double gain = leftRatio * leftGain + rightRatio * rightGain; + gain = (leftWeights / fullWeight) * leftGain + + (rightWeights / fullWeight) * rightGain; + } + else + { + // Calculate the fraction of points in the left and right children. + const double leftRatio = double(index) / double(sortedLabels.n_elem); + const double rightRatio = 1.0 - leftRatio; + + // Calculate the gain at this split point. + gain = leftRatio * leftGain + rightRatio * rightGain; + } // Corner case: is this the best possible split? if (gain == 0.0) diff --git a/src/mlpack/methods/decision_tree/decision_tree.hpp b/src/mlpack/methods/decision_tree/decision_tree.hpp index fca2ba6c5c..535c99953f 100644 --- a/src/mlpack/methods/decision_tree/decision_tree.hpp +++ b/src/mlpack/methods/decision_tree/decision_tree.hpp @@ -306,9 +306,10 @@ class DecisionTree : /** * Calculate the class probabilities of the given labels. */ - template + template void CalculateClassProbabilities(const RowType& labels, - const size_t numClasses); + const size_t numClasses, + const WeightsRowType& weights); }; /** diff --git a/src/mlpack/methods/decision_tree/decision_tree_impl.hpp b/src/mlpack/methods/decision_tree/decision_tree_impl.hpp index 39819d2884..608a0a4f3e 100644 --- a/src/mlpack/methods/decision_tree/decision_tree_impl.hpp +++ b/src/mlpack/methods/decision_tree/decision_tree_impl.hpp @@ -292,8 +292,8 @@ void DecisionTree(bestGain, data.row(i), - datasetInfo.NumMappings(i), labels, numClasses, minimumLeafSize, weights, - classProbabilities, *this); + datasetInfo.NumMappings(i), labels, numClasses, minimumLeafSize, weights, + classProbabilities, *this); else if (datasetInfo.Type(i) == data::Datatype::numeric) dimGain = NumericSplit::template SplitIfBetter(bestGain, data.row(i), labels, numClasses, minimumLeafSize, weights, classProbabilities, *this); @@ -350,22 +350,43 @@ void DecisionTree childLabels(childCounts[i]); size_t currentCol = 0; + + // Initialize weights if needed. + arma::rowvec childWeights; + if (UseWeights) + childWeights.set_size(childCounts[i]); + for (size_t j = 0; j < data.n_cols; ++j) { if (childAssignments[j] == i) { childPoints.col(currentCol) = data.col(j); childLabels[currentCol++] = labels[j]; + + if (UseWeights) + childWeights[currentCol - 1] = weights[j]; } } - // Now build the child recursively. - if (NoRecursion) - children.push_back(new DecisionTree(childPoints, datasetInfo, - childLabels, numClasses, childPoints.n_cols)); + // Now build the child recursively, with or without weights. + if (UseWeights) + { + if (NoRecursion) + children.push_back(new DecisionTree(childPoints, datasetInfo, + childLabels, numClasses, childWeights, childPoints.n_cols)); + else + children.push_back(new DecisionTree(childPoints, datasetInfo, + childLabels, numClasses, childWeights, minimumLeafSize)); + } else - children.push_back(new DecisionTree(childPoints, datasetInfo, - childLabels, numClasses, minimumLeafSize)); + { + if (NoRecursion) + children.push_back(new DecisionTree(childPoints, datasetInfo, + childLabels, numClasses, childPoints.n_cols)); + else + children.push_back(new DecisionTree(childPoints, datasetInfo, + childLabels, numClasses, minimumLeafSize)); + } } } else @@ -375,7 +396,7 @@ void DecisionTree(labels, numClasses, weights); } } @@ -453,6 +474,12 @@ void DecisionTree childLabels(childCounts[i]); + + // Initialize weights if necessary. + arma::rowvec childWeights; + if (UseWeights) + childWeights.set_size(childCounts[i]); + size_t currentCol = 0; for (size_t j = 0; j < data.n_cols; ++j) { @@ -460,16 +487,31 @@ void DecisionTree(labels, numClasses, weights); } } @@ -665,21 +707,33 @@ template class CategoricalSplitType, typename ElemType, bool NoRecursion> -template +template void DecisionTree::CalculateClassProbabilities( const RowType& labels, - const size_t numClasses) + const size_t numClasses, + const WeightsRowType& weights) { classProbabilities.zeros(numClasses); + double sumWeights = 0.0; for (size_t i = 0; i < labels.n_elem; ++i) - classProbabilities[labels[i]]++; + { + if (UseWeights) + { + classProbabilities[labels[i]] += weights[i]; + sumWeights += weights[i]; + } + else + { + classProbabilities[labels[i]]++; + } + } // Now normalize into probabilities. - classProbabilities /= labels.n_elem; + classProbabilities /= UseWeights ? sumWeights : labels.n_elem; arma::uword maxIndex; classProbabilities.max(maxIndex); dimensionTypeOrMajorityClass = (size_t) maxIndex; diff --git a/src/mlpack/methods/decision_tree/gini_gain.hpp b/src/mlpack/methods/decision_tree/gini_gain.hpp index c311052ec1..f3d2e0a479 100644 --- a/src/mlpack/methods/decision_tree/gini_gain.hpp +++ b/src/mlpack/methods/decision_tree/gini_gain.hpp @@ -45,7 +45,7 @@ class GiniGain return 0.0; // Count the number of elements in each class. - arma::Col counts(numClasses); + arma::vec counts(numClasses); counts.zeros(); // Calculate the Gini impurity of the un-split node. @@ -57,7 +57,7 @@ class GiniGain // sum all the weights up double accWeights = 0.0; - for (size_t i=0; i < labels.n_elem; ++i) + for (size_t i = 0; i < labels.n_elem; ++i) { // We just plus one if it's 'no weighted label' and plus 'weight' // if the label had correspond label. @@ -65,6 +65,10 @@ class GiniGain accWeights += weights[i]; } + // Catch edge case: if there are no weights, the impurity is zero. + if (accWeights == 0.0) + return 0.0; + for (size_t i = 0; i < numClasses; ++i) { const double f = ((double) counts[i] / (double) accWeights); From 8a6e86a06f04e82b745844bebed722bfcf05e326 Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Thu, 25 May 2017 10:56:35 -0400 Subject: [PATCH 30/84] Fix incorrect ordering bug. --- .../decision_tree/best_binary_numeric_split_impl.hpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/mlpack/methods/decision_tree/best_binary_numeric_split_impl.hpp b/src/mlpack/methods/decision_tree/best_binary_numeric_split_impl.hpp index feb21020e5..88537c035b 100644 --- a/src/mlpack/methods/decision_tree/best_binary_numeric_split_impl.hpp +++ b/src/mlpack/methods/decision_tree/best_binary_numeric_split_impl.hpp @@ -34,9 +34,9 @@ double BestBinaryNumericSplit::SplitIfBetter( // Next, sort the data. arma::uvec sortedIndices = arma::sort_index(data); arma::Row sortedLabels(labels.n_elem); - arma::Row sortedWeights; + arma::rowvec sortedWeights; for (size_t i = 0; i < sortedLabels.n_elem; ++i) - sortedLabels[sortedIndices[i]] = labels[i]; + sortedLabels[i] = labels[sortedIndices[i]]; // Only initialize if we are using weights. if (UseWeights) @@ -44,7 +44,7 @@ double BestBinaryNumericSplit::SplitIfBetter( sortedWeights.set_size(sortedLabels.n_elem); // The weights must keep the same order of labels for (size_t i = 0; i < sortedLabels.n_elem; ++i) - sortedWeights[sortedIndices[i]] = weights[i]; + sortedWeights[i] = weights[sortedIndices[i]]; } // Loop through all possible split points, choosing the best one. Also, force From 3521d7d031248694a0192b5a029fc191192173cd Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Thu, 25 May 2017 10:57:01 -0400 Subject: [PATCH 31/84] Fix corner case for information gain. --- src/mlpack/methods/decision_tree/information_gain.hpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/mlpack/methods/decision_tree/information_gain.hpp b/src/mlpack/methods/decision_tree/information_gain.hpp index 9eae6e1158..a1a5a28f71 100644 --- a/src/mlpack/methods/decision_tree/information_gain.hpp +++ b/src/mlpack/methods/decision_tree/information_gain.hpp @@ -60,6 +60,10 @@ class InformationGain accWeights += weights[i]; } + // Corner case: return 0 if no weight. + if (accWeights == 0.0) + return 0.0; + for (size_t i = 0; i < numClasses; ++i) { const double f = ((double) counts[i] / (double) accWeights); From 80b03dbcf66542f479a79fd3b2f161b178bd6bc4 Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Thu, 25 May 2017 10:50:19 -0400 Subject: [PATCH 32/84] Formatting fixes. --- .../core/data/serialization_template_version.hpp | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/src/mlpack/core/data/serialization_template_version.hpp b/src/mlpack/core/data/serialization_template_version.hpp index 78e1c5c1da..e671eb0fe0 100644 --- a/src/mlpack/core/data/serialization_template_version.hpp +++ b/src/mlpack/core/data/serialization_template_version.hpp @@ -29,14 +29,10 @@ struct version> \ typedef mpl::int_ type; \ typedef mpl::integral_c_tag tag; \ BOOST_STATIC_CONSTANT(int, value = version::type::value); \ - BOOST_MPL_ASSERT(( \ - boost::mpl::less< \ - boost::mpl::int_, \ - boost::mpl::int_<256> \ - > \ - )); \ + BOOST_MPL_ASSERT((boost::mpl::less, \ + boost::mpl::int_<256>>)); \ }; \ -} \ -} +} /* namespace serialization */ \ +} /* namespace boost */ #endif From 58946c3fb100ed2437cae12d594390644c69ba4a Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Thu, 25 May 2017 11:03:08 -0400 Subject: [PATCH 33/84] Add tests for information gain and gini impurity weighted learning. --- src/mlpack/tests/decision_tree_test.cpp | 222 +++++++++++++++++++++++- 1 file changed, 220 insertions(+), 2 deletions(-) diff --git a/src/mlpack/tests/decision_tree_test.cpp b/src/mlpack/tests/decision_tree_test.cpp index e5644c167b..7485c45075 100644 --- a/src/mlpack/tests/decision_tree_test.cpp +++ b/src/mlpack/tests/decision_tree_test.cpp @@ -129,7 +129,7 @@ BOOST_AUTO_TEST_SUITE(DecisionTreeTest); */ BOOST_AUTO_TEST_CASE(GiniGainPerfectTest) { - arma::rowvec weights(10); + arma::rowvec weights(10, arma::fill::ones); arma::Row labels; labels.zeros(10); @@ -689,7 +689,7 @@ BOOST_AUTO_TEST_CASE(SimpleGeneralizationTest) if (!data::Load("vc2_labels.txt", labels)) BOOST_FAIL("Cannot load labels for vc2_labels.txt"); - // init a weight martix + // Initialize an all-ones weight matrix. arma::mat weights = arma::ones>(labels.n_rows, labels.n_cols); // Build decision tree. @@ -828,4 +828,222 @@ BOOST_AUTO_TEST_CASE(DecisionStumpTest) BOOST_REQUIRE_EQUAL(stump.Child(1).NumChildren(), 0); } +/** + * Test that we can build a decision tree using weighted data (where the + * low-weighted data is random noise), and that the tree still builds correctly + * enough to get good results. + */ +BOOST_AUTO_TEST_CASE(WeightedDecisionTreeTest) +{ + arma::mat dataset; + arma::Row labels; + data::Load("vc2.csv", dataset); + data::Load("vc2_labels.txt", labels); + + // Add some noise. + arma::mat noise(dataset.n_rows, 1000, arma::fill::randu); + arma::Row noiseLabels(1000); + for (size_t i = 0; i < noiseLabels.n_elem; ++i) + noiseLabels[i] = math::RandInt(3); // Random label. + + // Concatenate data matrices. + arma::mat data = arma::join_rows(dataset, noise); + arma::Row fullLabels = arma::join_rows(labels, noiseLabels); + + // Now set weights. + arma::rowvec weights(dataset.n_cols + 1000); + for (size_t i = 0; i < dataset.n_cols; ++i) + weights[i] = math::Random(0.9, 1.0); + for (size_t i = dataset.n_cols; i < dataset.n_cols + 1000; ++i) + weights[i] = math::Random(0.0, 0.01); // Low weights for false points. + + // Now build the decision tree. I think the syntax is right here. + DecisionTree<> d(data, fullLabels, 3, weights, 10); + + // Now we can check that we get good performance on the VC2 test set. + arma::mat testData; + arma::Row testLabels; + data::Load("vc2_test.csv", testData); + data::Load("vc2_test_labels.txt", testLabels); + + arma::Row predictions; + d.Classify(testData, predictions); + + BOOST_REQUIRE_EQUAL(predictions.n_elem, testData.n_cols); + + // Figure out the accuracy. + double correct = 0.0; + for (size_t i = 0; i < predictions.n_elem; ++i) + if (predictions[i] == testLabels[i]) + ++correct; + correct /= predictions.n_elem; + + BOOST_REQUIRE_GT(correct, 0.75); +} +/** + * Test that we can build a decision tree on a simple categorical dataset using + * weights, with low-weight noise added. + */ +BOOST_AUTO_TEST_CASE(CategoricalWeightedBuildTest) +{ + arma::mat d; + arma::Row l; + data::DatasetInfo di; + MockCategoricalData(d, l, di); + + // Split into a training set and a test set. + arma::mat trainingData = d.cols(0, 4999); + arma::mat testData = d.cols(5000, 9999); + arma::Row trainingLabels = l.subvec(0, 4999); + arma::Row testLabels = l.subvec(5000, 9999); + + // Now create random points. + arma::mat randomNoise(4, 10000); + arma::Row randomLabels(10000); + for (size_t i = 0; i < 10000; ++i) + { + randomNoise(0, i) = math::Random(); + randomNoise(1, i) = math::Random(); + randomNoise(2, i) = math::RandInt(4); + randomNoise(3, i) = math::RandInt(2); + randomLabels[i] = math::RandInt(5); + } + + // Generate weights. + arma::rowvec weights(20000); + for (size_t i = 0; i < 10000; ++i) + weights[i] = math::Random(0.9, 1.0); + for (size_t i = 10000; i < 20000; ++i) + weights[i] = math::Random(0.0, 0.001); + + arma::mat fullData = arma::join_rows(trainingData, randomNoise); + arma::Row fullLabels = arma::join_rows(trainingLabels, randomLabels); + + // Build the tree. + DecisionTree<> tree(fullData, di, fullLabels, 5, weights, 10); + + // Now evaluate the accuracy of the tree. + arma::Row predictions; + tree.Classify(testData, predictions); + + BOOST_REQUIRE_EQUAL(predictions.n_elem, testData.n_cols); + size_t correct = 0; + for (size_t i = 0; i < testData.n_cols; ++i) + if (testLabels[i] == predictions[i]) + ++correct; + + // Make sure we got at least 70% accuracy. + const double correctPct = double(correct) / double(testData.n_cols); + BOOST_REQUIRE_GT(correctPct, 0.70); +} + +/** + * Test that we can build a decision tree using weighted data (where the + * low-weighted data is random noise) with information gain, and that the tree + * still builds correctly enough to get good results. + */ +BOOST_AUTO_TEST_CASE(WeightedDecisionTreeInformationGainTest) +{ + arma::mat dataset; + arma::Row labels; + data::Load("vc2.csv", dataset); + data::Load("vc2_labels.txt", labels); + + // Add some noise. + arma::mat noise(dataset.n_rows, 1000, arma::fill::randu); + arma::Row noiseLabels(1000); + for (size_t i = 0; i < noiseLabels.n_elem; ++i) + noiseLabels[i] = math::RandInt(3); // Random label. + + // Concatenate data matrices. + arma::mat data = arma::join_rows(dataset, noise); + arma::Row fullLabels = arma::join_rows(labels, noiseLabels); + + // Now set weights. + arma::rowvec weights(dataset.n_cols + 1000); + for (size_t i = 0; i < dataset.n_cols; ++i) + weights[i] = math::Random(0.9, 1.0); + for (size_t i = dataset.n_cols; i < dataset.n_cols + 1000; ++i) + weights[i] = math::Random(0.0, 0.01); // Low weights for false points. + + // Now build the decision tree. I think the syntax is right here. + DecisionTree d(data, fullLabels, 3, weights, 10); + + // Now we can check that we get good performance on the VC2 test set. + arma::mat testData; + arma::Row testLabels; + data::Load("vc2_test.csv", testData); + data::Load("vc2_test_labels.txt", testLabels); + + arma::Row predictions; + d.Classify(testData, predictions); + + BOOST_REQUIRE_EQUAL(predictions.n_elem, testData.n_cols); + + // Figure out the accuracy. + double correct = 0.0; + for (size_t i = 0; i < predictions.n_elem; ++i) + if (predictions[i] == testLabels[i]) + ++correct; + correct /= predictions.n_elem; + + BOOST_REQUIRE_GT(correct, 0.75); +} +/** + * Test that we can build a decision tree using information gain on a simple + * categorical dataset using weights, with low-weight noise added. + */ +BOOST_AUTO_TEST_CASE(CategoricalInformationGainWeightedBuildTest) +{ + arma::mat d; + arma::Row l; + data::DatasetInfo di; + MockCategoricalData(d, l, di); + + // Split into a training set and a test set. + arma::mat trainingData = d.cols(0, 4999); + arma::mat testData = d.cols(5000, 9999); + arma::Row trainingLabels = l.subvec(0, 4999); + arma::Row testLabels = l.subvec(5000, 9999); + + // Now create random points. + arma::mat randomNoise(4, 10000); + arma::Row randomLabels(10000); + for (size_t i = 0; i < 10000; ++i) + { + randomNoise(0, i) = math::Random(); + randomNoise(1, i) = math::Random(); + randomNoise(2, i) = math::RandInt(4); + randomNoise(3, i) = math::RandInt(2); + randomLabels[i] = math::RandInt(5); + } + + // Generate weights. + arma::rowvec weights(20000); + for (size_t i = 0; i < 10000; ++i) + weights[i] = math::Random(0.9, 1.0); + for (size_t i = 10000; i < 20000; ++i) + weights[i] = math::Random(0.0, 0.001); + + arma::mat fullData = arma::join_rows(trainingData, randomNoise); + arma::Row fullLabels = arma::join_rows(trainingLabels, randomLabels); + + // Build the tree. + DecisionTree tree(fullData, di, fullLabels, 5, weights, 10); + + // Now evaluate the accuracy of the tree. + arma::Row predictions; + tree.Classify(testData, predictions); + + BOOST_REQUIRE_EQUAL(predictions.n_elem, testData.n_cols); + size_t correct = 0; + for (size_t i = 0; i < testData.n_cols; ++i) + if (testLabels[i] == predictions[i]) + ++correct; + + // Make sure we got at least 70% accuracy. + const double correctPct = double(correct) / double(testData.n_cols); + BOOST_REQUIRE_GT(correctPct, 0.70); +} + BOOST_AUTO_TEST_SUITE_END(); From 35bd82dede36b21da615efb6da7fed10ab963039 Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Thu, 25 May 2017 11:14:28 -0400 Subject: [PATCH 34/84] Make test datasets smaller. --- src/mlpack/tests/decision_tree_test.cpp | 92 ++++++++++++------------- 1 file changed, 43 insertions(+), 49 deletions(-) diff --git a/src/mlpack/tests/decision_tree_test.cpp b/src/mlpack/tests/decision_tree_test.cpp index 7485c45075..56f8c54060 100644 --- a/src/mlpack/tests/decision_tree_test.cpp +++ b/src/mlpack/tests/decision_tree_test.cpp @@ -44,14 +44,14 @@ void MockCategoricalData(arma::mat& d, arma::Row& l, data::DatasetInfo& c2[i] = DiscreteDistribution(probs); } - arma::mat spiralDataset(4, 10000); - arma::Row labels(10000); - for (size_t i = 0; i < 10000; ++i) + arma::mat spiralDataset(4, 4000); + arma::Row labels(4000); + for (size_t i = 0; i < 4000; ++i) { - // One circle every 20000 samples. Plus some noise. - const double magnitude = 2.0 + (double(i) / 2000.0) + + // One circle every 2000 samples. Plus some noise. + const double magnitude = 2.0 + (double(i) / 200.0) + 0.5 * mlpack::math::Random(); - const double angle = (i % 2000) * (2 * M_PI) + mlpack::math::Random(); + const double angle = (i % 200) * (2 * M_PI) + mlpack::math::Random(); const double x = magnitude * cos(angle); const double y = magnitude * sin(angle); @@ -60,25 +60,25 @@ void MockCategoricalData(arma::mat& d, arma::Row& l, data::DatasetInfo& spiralDataset(1, i) = y; // Set categorical features c1 and c2. - if (i < 2000) + if (i < 800) { spiralDataset(2, i) = c1[1].Random()[0]; spiralDataset(3, i) = c2[1].Random()[0]; labels[i] = 1; } - else if (i < 4000) + else if (i < 1600) { spiralDataset(2, i) = c1[3].Random()[0]; spiralDataset(3, i) = c2[3].Random()[0]; labels[i] = 3; } - else if (i < 6000) + else if (i < 2400) { spiralDataset(2, i) = c1[2].Random()[0]; spiralDataset(3, i) = c2[2].Random()[0]; labels[i] = 2; } - else if (i < 8000) + else if (i < 3200) { spiralDataset(2, i) = c1[0].Random()[0]; spiralDataset(3, i) = c2[0].Random()[0]; @@ -105,21 +105,15 @@ void MockCategoricalData(arma::mat& d, arma::Row& l, data::DatasetInfo& datasetInfo.MapString("1", 3); // Now shuffle the dataset. - arma::uvec indices = arma::shuffle(arma::linspace(0, 9999, - 10000)); - d = arma::mat(4, 10000); - l = arma::Row(10000); - for (size_t i = 0; i < 10000; ++i) + arma::uvec indices = arma::shuffle(arma::linspace(0, 3999, + 4000)); + d = arma::mat(4, 4000); + l = arma::Row(4000); + for (size_t i = 0; i < 4000; ++i) { d.col(i) = spiralDataset.col(indices[i]); l[i] = labels[indices[i]]; } - - // Split into a training set and a test set. - arma::mat trainingData = d.cols(0, 4999); - arma::mat testData = d.cols(5000, 9999); - arma::Row trainingLabels = l.subvec(0, 4999); - arma::Row testLabels = l.subvec(5000, 9999); } BOOST_AUTO_TEST_SUITE(DecisionTreeTest); @@ -747,10 +741,10 @@ BOOST_AUTO_TEST_CASE(CategoricalBuildTest) MockCategoricalData(d, l, di); // Split into a training set and a test set. - arma::mat trainingData = d.cols(0, 4999); - arma::mat testData = d.cols(5000, 9999); - arma::Row trainingLabels = l.subvec(0, 4999); - arma::Row testLabels = l.subvec(5000, 9999); + arma::mat trainingData = d.cols(0, 1999); + arma::mat testData = d.cols(2000, 3999); + arma::Row trainingLabels = l.subvec(0, 1999); + arma::Row testLabels = l.subvec(2000, 3999); // Build the tree. DecisionTree<> tree(trainingData, di, trainingLabels, 5, 10); @@ -781,10 +775,10 @@ BOOST_AUTO_TEST_CASE(CategoricalBuildTestWithWeight) MockCategoricalData(d, l, di); // Split into a training set and a test set. - arma::mat trainingData = d.cols(0, 4999); - arma::mat testData = d.cols(5000, 9999); - arma::Row trainingLabels = l.subvec(0, 4999); - arma::Row testLabels = l.subvec(5000, 9999); + arma::mat trainingData = d.cols(0, 1999); + arma::mat testData = d.cols(2000, 3999); + arma::Row trainingLabels = l.subvec(0, 1999); + arma::Row testLabels = l.subvec(2000, 3999); arma::Row weights = arma::ones>(trainingLabels.n_elem); @@ -892,15 +886,15 @@ BOOST_AUTO_TEST_CASE(CategoricalWeightedBuildTest) MockCategoricalData(d, l, di); // Split into a training set and a test set. - arma::mat trainingData = d.cols(0, 4999); - arma::mat testData = d.cols(5000, 9999); - arma::Row trainingLabels = l.subvec(0, 4999); - arma::Row testLabels = l.subvec(5000, 9999); + arma::mat trainingData = d.cols(0, 1999); + arma::mat testData = d.cols(2000, 3999); + arma::Row trainingLabels = l.subvec(0, 1999); + arma::Row testLabels = l.subvec(2000, 3999); // Now create random points. - arma::mat randomNoise(4, 10000); - arma::Row randomLabels(10000); - for (size_t i = 0; i < 10000; ++i) + arma::mat randomNoise(4, 2000); + arma::Row randomLabels(2000); + for (size_t i = 0; i < 2000; ++i) { randomNoise(0, i) = math::Random(); randomNoise(1, i) = math::Random(); @@ -910,10 +904,10 @@ BOOST_AUTO_TEST_CASE(CategoricalWeightedBuildTest) } // Generate weights. - arma::rowvec weights(20000); - for (size_t i = 0; i < 10000; ++i) + arma::rowvec weights(4000); + for (size_t i = 0; i < 2000; ++i) weights[i] = math::Random(0.9, 1.0); - for (size_t i = 10000; i < 20000; ++i) + for (size_t i = 2000; i < 4000; ++i) weights[i] = math::Random(0.0, 0.001); arma::mat fullData = arma::join_rows(trainingData, randomNoise); @@ -1001,15 +995,15 @@ BOOST_AUTO_TEST_CASE(CategoricalInformationGainWeightedBuildTest) MockCategoricalData(d, l, di); // Split into a training set and a test set. - arma::mat trainingData = d.cols(0, 4999); - arma::mat testData = d.cols(5000, 9999); - arma::Row trainingLabels = l.subvec(0, 4999); - arma::Row testLabels = l.subvec(5000, 9999); + arma::mat trainingData = d.cols(0, 1999); + arma::mat testData = d.cols(2000, 3999); + arma::Row trainingLabels = l.subvec(0, 1999); + arma::Row testLabels = l.subvec(2000, 3999); // Now create random points. - arma::mat randomNoise(4, 10000); - arma::Row randomLabels(10000); - for (size_t i = 0; i < 10000; ++i) + arma::mat randomNoise(4, 2000); + arma::Row randomLabels(2000); + for (size_t i = 0; i < 2000; ++i) { randomNoise(0, i) = math::Random(); randomNoise(1, i) = math::Random(); @@ -1019,10 +1013,10 @@ BOOST_AUTO_TEST_CASE(CategoricalInformationGainWeightedBuildTest) } // Generate weights. - arma::rowvec weights(20000); - for (size_t i = 0; i < 10000; ++i) + arma::rowvec weights(4000); + for (size_t i = 0; i < 2000; ++i) weights[i] = math::Random(0.9, 1.0); - for (size_t i = 10000; i < 20000; ++i) + for (size_t i = 2000; i < 4000; ++i) weights[i] = math::Random(0.0, 0.001); arma::mat fullData = arma::join_rows(trainingData, randomNoise); From f19321cad47f7f78f3481b1d3b99ff5c6f29da46 Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Thu, 25 May 2017 11:32:22 -0400 Subject: [PATCH 35/84] Fix style issues. --- .../methods/decision_tree/decision_tree.hpp | 16 +-- .../decision_tree/decision_tree_impl.hpp | 26 +++-- .../methods/decision_tree/gini_gain.hpp | 10 +- .../decision_tree/information_gain.hpp | 13 +-- src/mlpack/tests/decision_tree_test.cpp | 100 +++++++++++------- 5 files changed, 92 insertions(+), 73 deletions(-) diff --git a/src/mlpack/methods/decision_tree/decision_tree.hpp b/src/mlpack/methods/decision_tree/decision_tree.hpp index 535c99953f..71206a30f8 100644 --- a/src/mlpack/methods/decision_tree/decision_tree.hpp +++ b/src/mlpack/methods/decision_tree/decision_tree.hpp @@ -82,10 +82,10 @@ class DecisionTree : const size_t minimumLeafSize = 10); /** - * Construct the decision tree on the given data and labels with weight, where the data - * can be both numeric and categorical. Setting minimumLeafSize too small may - * cause the tree to overfit, but setting it too large may cause it to - * underfit. + * Construct the decision tree on the given data and labels with weights, + * where the data can be both numeric and categorical. Setting + * minimumLeafSize too small may cause the tree to overfit, but setting it too + * large may cause it to underfit. * * @param data Dataset to train on. * @param datasetInfo Type information for each dimension of the dataset. @@ -103,10 +103,10 @@ class DecisionTree : const size_t minimumLeafSize = 10); /** - * Construct the decision tree on the given data and labels with weight, assuming that the - * data is all of the numeric type. Setting minimumLeafSize too small may - * cause the tree to overfit, but setting it too large may cause it to - * underfit. + * Construct the decision tree on the given data and labels with weights, + * assuming that the data is all of the numeric type. Setting minimumLeafSize + * too small may cause the tree to overfit, but setting it too large may cause + * it to underfit. * * @param data Dataset to train on. * @param labels Labels for each training point. diff --git a/src/mlpack/methods/decision_tree/decision_tree_impl.hpp b/src/mlpack/methods/decision_tree/decision_tree_impl.hpp index 608a0a4f3e..e854fd54dc 100644 --- a/src/mlpack/methods/decision_tree/decision_tree_impl.hpp +++ b/src/mlpack/methods/decision_tree/decision_tree_impl.hpp @@ -285,18 +285,20 @@ void DecisionTree(labels, numClasses, weights); + double bestGain = FitnessFunction::template Evaluate(labels, + numClasses, weights); size_t bestDim = datasetInfo.Dimensionality(); // This means "no split". for (size_t i = 0; i < datasetInfo.Dimensionality(); ++i) { double dimGain = -DBL_MAX; if (datasetInfo.Type(i) == data::Datatype::categorical) - dimGain = CategoricalSplit::template SplitIfBetter(bestGain, data.row(i), - datasetInfo.NumMappings(i), labels, numClasses, minimumLeafSize, weights, - classProbabilities, *this); + dimGain = CategoricalSplit::template SplitIfBetter(bestGain, + data.row(i), datasetInfo.NumMappings(i), labels, numClasses, + minimumLeafSize, weights, classProbabilities, *this); else if (datasetInfo.Type(i) == data::Datatype::numeric) - dimGain = NumericSplit::template SplitIfBetter(bestGain, data.row(i), labels, - numClasses, minimumLeafSize, weights, classProbabilities, *this); + dimGain = NumericSplit::template SplitIfBetter(bestGain, + data.row(i), labels, numClasses, minimumLeafSize, weights, + classProbabilities, *this); // Was there an improvement? If so mark that it's the new best dimension. if (dimGain > bestGain) @@ -430,13 +432,14 @@ void DecisionTree(labels, numClasses, weights); + double bestGain = FitnessFunction::template Evaluate(labels, + numClasses, weights); size_t bestDim = data.n_rows; // This means "no split". for (size_t i = 0; i < data.n_rows; ++i) { - double dimGain = NumericSplitType::template SplitIfBetter(bestGain, - data.row(i), labels, numClasses, minimumLeafSize, weights, classProbabilities, - *this); + double dimGain = NumericSplitType::template + SplitIfBetter(bestGain, data.row(i), labels, numClasses, + minimumLeafSize, weights, classProbabilities, *this); if (dimGain > bestGain) { @@ -568,7 +571,8 @@ void DecisionTreeClassify(point, prediction, probabilities); + children[CalculateDirection(point)]->Classify(point, prediction, + probabilities); } //! Return the class for a set of points. diff --git a/src/mlpack/methods/decision_tree/gini_gain.hpp b/src/mlpack/methods/decision_tree/gini_gain.hpp index f3d2e0a479..485b896c87 100644 --- a/src/mlpack/methods/decision_tree/gini_gain.hpp +++ b/src/mlpack/methods/decision_tree/gini_gain.hpp @@ -44,23 +44,19 @@ class GiniGain if (labels.n_elem == 0) return 0.0; - // Count the number of elements in each class. - arma::vec counts(numClasses); - counts.zeros(); + // Count the number of elements in each class. + arma::vec counts(numClasses, arma::fill::zeros); // Calculate the Gini impurity of the un-split node. double impurity = 0.0; if (UseWeights) { - - // sum all the weights up + // Sum all the weights up. double accWeights = 0.0; for (size_t i = 0; i < labels.n_elem; ++i) { - // We just plus one if it's 'no weighted label' and plus 'weight' - // if the label had correspond label. counts[labels[i]] += weights[i]; accWeights += weights[i]; } diff --git a/src/mlpack/methods/decision_tree/information_gain.hpp b/src/mlpack/methods/decision_tree/information_gain.hpp index a1a5a28f71..4152360fc4 100644 --- a/src/mlpack/methods/decision_tree/information_gain.hpp +++ b/src/mlpack/methods/decision_tree/information_gain.hpp @@ -39,23 +39,20 @@ class InformationGain // Edge case: if there are no elements, the gain is zero. if (labels.n_elem == 0) return 0.0; - + // Calculate the information gain. double gain = 0.0; - // Count the number of elements in each class. - arma::Col counts(numClasses); - counts.zeros(); + // Count the number of elements in each class. + arma::Col counts(numClasses, arma::fill::zeros); if (UseWeights) { - // sum all the weights up + // Sum all the weights up. double accWeights = 0.0; - for (size_t i=0; i < labels.n_elem; ++i) + for (size_t i = 0; i < labels.n_elem; ++i) { - // We just plus one if it's 'no weighted label' and plus 'weight' - // if the label had correspond label. counts[labels[i]] += weights[i]; accWeights += weights[i]; } diff --git a/src/mlpack/tests/decision_tree_test.cpp b/src/mlpack/tests/decision_tree_test.cpp index 56f8c54060..51c04c6e49 100644 --- a/src/mlpack/tests/decision_tree_test.cpp +++ b/src/mlpack/tests/decision_tree_test.cpp @@ -22,7 +22,12 @@ using namespace mlpack; using namespace mlpack::tree; using namespace mlpack::distribution; -void MockCategoricalData(arma::mat& d, arma::Row& l, data::DatasetInfo& datasetInfo) +/** + * Create a mock categorical dataset for testing. + */ +void MockCategoricalData(arma::mat& d, + arma::Row& l, + data::DatasetInfo& datasetInfo) { // We'll build a spiral dataset plus two noisy categorical features. We need // to build the distributions for the categorical features (they'll be @@ -186,11 +191,12 @@ BOOST_AUTO_TEST_CASE(GiniGainEvenSplitManyClassTest) labels[i] = i; weights[i] = 1; } - // Calculate Gini gain and make sure it is correct. - BOOST_REQUIRE_CLOSE(GiniGain::Evaluate(labels, c, weights), -(1.0 - 1.0 / c), 1e-5); - BOOST_REQUIRE_CLOSE(GiniGain::Evaluate(labels, c, weights), -(1.0 - 1.0 / c), 1e-5); + BOOST_REQUIRE_CLOSE(GiniGain::Evaluate(labels, c, weights), + -(1.0 - 1.0 / c), 1e-5); + BOOST_REQUIRE_CLOSE(GiniGain::Evaluate(labels, c, weights), + -(1.0 - 1.0 / c), 1e-5); } } @@ -210,8 +216,10 @@ BOOST_AUTO_TEST_CASE(GiniGainManyPoints) for (size_t j = numPoints / 2; j < numPoints; ++j) labels[j] = 1; - BOOST_REQUIRE_CLOSE(GiniGain::Evaluate(labels, 2, weights), -0.5, 1e-5); - BOOST_REQUIRE_CLOSE(GiniGain::Evaluate(labels, 2, weights), -0.5, 1e-5); + BOOST_REQUIRE_CLOSE(GiniGain::Evaluate(labels, 2, weights), -0.5, + 1e-5); + BOOST_REQUIRE_CLOSE(GiniGain::Evaluate(labels, 2, weights), -0.5, + 1e-5); } } @@ -266,9 +274,11 @@ BOOST_AUTO_TEST_CASE(InformationGainEvenSplitTest) // Test that it's -1 regardless of the number of classes. for (size_t c = 2; c < 10; ++c) { - // weighted and unweighted result should make no difference. - BOOST_REQUIRE_CLOSE(InformationGain::Evaluate(labels, c, weights), -1.0, 1e-5); - BOOST_REQUIRE_CLOSE(InformationGain::Evaluate(labels, c, weights), -1.0, 1e-5); + // Weighted and unweighted result should be the same. + BOOST_REQUIRE_CLOSE(InformationGain::Evaluate(labels, c, weights), + -1.0, 1e-5); + BOOST_REQUIRE_CLOSE(InformationGain::Evaluate(labels, c, weights), + -1.0, 1e-5); } } @@ -281,8 +291,10 @@ BOOST_AUTO_TEST_CASE(InformationGainEmptyTest) arma::rowvec weights = arma::ones(10); for (size_t c = 1; c < 10; ++c) { - BOOST_REQUIRE_SMALL(InformationGain::Evaluate(labels, c, weights), 1e-5); - BOOST_REQUIRE_SMALL(InformationGain::Evaluate(labels, c, weights), 1e-5); + BOOST_REQUIRE_SMALL(InformationGain::Evaluate(labels, c, weights), + 1e-5); + BOOST_REQUIRE_SMALL(InformationGain::Evaluate(labels, c, weights), + 1e-5); } } @@ -338,9 +350,12 @@ BOOST_AUTO_TEST_CASE(InformationGainManyPoints) for (size_t j = numPoints / 2; j < numPoints; ++j) labels[j] = 1; - BOOST_REQUIRE_CLOSE(InformationGain::Evaluate(labels, 2, weights), -1.0, 1e-5); - // It should make no difference between weighted and no weight labels. - BOOST_REQUIRE_CLOSE(InformationGain::Evaluate(labels, 2, weights), -1.0, 1e-5); + BOOST_REQUIRE_CLOSE(InformationGain::Evaluate(labels, 2, weights), + -1.0, 1e-5); + // It should make no difference between a weighted and unweighted + // calculation. + BOOST_REQUIRE_CLOSE(InformationGain::Evaluate(labels, 2, weights), + -1.0, 1e-5); } } @@ -360,10 +375,11 @@ BOOST_AUTO_TEST_CASE(BestBinaryNumericSplitSimpleSplitTest) // Call the method to do the splitting. const double bestGain = GiniGain::Evaluate(labels, 2, weights); - const double gain = BestBinaryNumericSplit::SplitIfBetter(bestGain, - values, labels, 2, 3, weights, classProbabilities, aux); - const double weightedGain = BestBinaryNumericSplit::SplitIfBetter(bestGain, - values, labels, 2, 3, weights, classProbabilities, aux); + const double gain = BestBinaryNumericSplit::SplitIfBetter( + bestGain, values, labels, 2, 3, weights, classProbabilities, aux); + const double weightedGain = + BestBinaryNumericSplit::SplitIfBetter(bestGain, values, + labels, 2, 3, weights, classProbabilities, aux); // Make sure that a split was made. BOOST_REQUIRE_GT(gain, bestGain); @@ -396,11 +412,12 @@ BOOST_AUTO_TEST_CASE(BestBinaryNumericSplitMinSamplesTest) // Call the method to do the splitting. const double bestGain = GiniGain::Evaluate(labels, 2, weights); - const double gain = BestBinaryNumericSplit::SplitIfBetter(bestGain, - values, labels, 2, 8, weights, classProbabilities, aux); + const double gain = BestBinaryNumericSplit::SplitIfBetter( + bestGain, values, labels, 2, 8, weights, classProbabilities, aux); // This should make no difference because it won't split at all. - const double weightedGain = BestBinaryNumericSplit::SplitIfBetter(bestGain, - values, labels, 2, 8, weights, classProbabilities, aux); + const double weightedGain = + BestBinaryNumericSplit::SplitIfBetter(bestGain, values, + labels, 2, 8, weights, classProbabilities, aux); // Make sure that no split was made. BOOST_REQUIRE_EQUAL(gain, bestGain); @@ -430,8 +447,8 @@ BOOST_AUTO_TEST_CASE(BestBinaryNumericSplitNoGainTest) // Call the method to do the splitting. const double bestGain = GiniGain::Evaluate(labels, 2, weights); - const double gain = BestBinaryNumericSplit::SplitIfBetter(bestGain, - values, labels, 2, 10, weights, classProbabilities, aux); + const double gain = BestBinaryNumericSplit::SplitIfBetter( + bestGain, values, labels, 2, 10, weights, classProbabilities, aux); // Make sure there was no split. BOOST_REQUIRE_EQUAL(gain, bestGain); @@ -454,10 +471,11 @@ BOOST_AUTO_TEST_CASE(AllCategoricalSplitSimpleSplitTest) // Call the method to do the splitting. const double bestGain = GiniGain::Evaluate(labels, 3, weights); - const double gain = AllCategoricalSplit::SplitIfBetter(bestGain, - values, 4, labels, 3, 3, weights, classProbabilities, aux); - const double weightedGain = AllCategoricalSplit::SplitIfBetter(bestGain, - values, 4, labels, 3, 3, weights, classProbabilities, aux); + const double gain = AllCategoricalSplit::SplitIfBetter( + bestGain, values, 4, labels, 3, 3, weights, classProbabilities, aux); + const double weightedGain = + AllCategoricalSplit::SplitIfBetter(bestGain, values, 4, + labels, 3, 3, weights, classProbabilities, aux); // Make sure that a split was made. BOOST_REQUIRE_GT(gain, bestGain); @@ -488,8 +506,8 @@ BOOST_AUTO_TEST_CASE(AllCategoricalSplitMinSamplesTest) // Call the method to do the splitting. const double bestGain = GiniGain::Evaluate(labels, 3, weights); - const double gain = AllCategoricalSplit::SplitIfBetter(bestGain, - values, 4, labels, 3, 4, weights, classProbabilities, aux); + const double gain = AllCategoricalSplit::SplitIfBetter( + bestGain, values, 4, labels, 3, 4, weights, classProbabilities, aux); // Make sure it's not split. BOOST_REQUIRE_EQUAL(gain, bestGain); @@ -520,10 +538,11 @@ BOOST_AUTO_TEST_CASE(AllCategoricalSplitNoGainTest) // Call the method to do the splitting. const double bestGain = GiniGain::Evaluate(labels, 3, weights); - const double gain = AllCategoricalSplit::SplitIfBetter(bestGain, - values, 10, labels, 3, 10, weights, classProbabilities, aux); - const double weightedGain = AllCategoricalSplit::SplitIfBetter(bestGain, - values, 10, labels, 3, 10, weights, classProbabilities, aux); + const double gain = AllCategoricalSplit::SplitIfBetter( + bestGain, values, 10, labels, 3, 10, weights, classProbabilities, aux); + const double weightedGain = + AllCategoricalSplit::SplitIfBetter(bestGain, values, 10, + labels, 3, 10, weights, classProbabilities, aux); // Make sure that there was no split. BOOST_REQUIRE_EQUAL(gain, bestGain); @@ -684,11 +703,12 @@ BOOST_AUTO_TEST_CASE(SimpleGeneralizationTest) BOOST_FAIL("Cannot load labels for vc2_labels.txt"); // Initialize an all-ones weight matrix. - arma::mat weights = arma::ones>(labels.n_rows, labels.n_cols); + arma::mat weights = arma::ones>(labels.n_rows, + labels.n_cols); // Build decision tree. DecisionTree<> d(inputData, labels, 3, 10); // Leaf size of 10. - DecisionTree<> wd(inputData, labels, 3, weights, 10); // Leaf size of 10 + DecisionTree<> wd(inputData, labels, 3, weights, 10); // Leaf size of 10. // Load testing data. arma::mat testData; @@ -765,7 +785,8 @@ BOOST_AUTO_TEST_CASE(CategoricalBuildTest) } /** - * Test that we can build a decision tree with weighted on a simple categorical dataset. + * Test that we can build a decision tree with weights on a simple categorical + * dataset. */ BOOST_AUTO_TEST_CASE(CategoricalBuildTestWithWeight) { @@ -773,14 +794,15 @@ BOOST_AUTO_TEST_CASE(CategoricalBuildTestWithWeight) arma::Row l; data::DatasetInfo di; MockCategoricalData(d, l, di); - + // Split into a training set and a test set. arma::mat trainingData = d.cols(0, 1999); arma::mat testData = d.cols(2000, 3999); arma::Row trainingLabels = l.subvec(0, 1999); arma::Row testLabels = l.subvec(2000, 3999); - arma::Row weights = arma::ones>(trainingLabels.n_elem); + arma::Row weights = arma::ones>( + trainingLabels.n_elem); // Build the tree. DecisionTree<> tree(trainingData, di, trainingLabels, 5, weights, 10); From a3ca745218a490eec2595a681a5c2c8212d1e064 Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Thu, 25 May 2017 11:32:48 -0400 Subject: [PATCH 36/84] Style fix. --- .../methods/decision_tree/all_categorical_split_impl.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/mlpack/methods/decision_tree/all_categorical_split_impl.hpp b/src/mlpack/methods/decision_tree/all_categorical_split_impl.hpp index 1cd90298b8..c22f04c6ea 100644 --- a/src/mlpack/methods/decision_tree/all_categorical_split_impl.hpp +++ b/src/mlpack/methods/decision_tree/all_categorical_split_impl.hpp @@ -90,8 +90,8 @@ double AllCategoricalSplit::SplitIfBetter( const double childPct = UseWeights ? double(childWeightSums[i]) / sumWeight : double(counts[i]) / double(data.n_elem); - const double childGain = FitnessFunction::template Evaluate(childLabels[i], - numClasses, childWeights[i]); + const double childGain = FitnessFunction::template Evaluate( + childLabels[i], numClasses, childWeights[i]); overallGain += childPct * childGain; } From dbc40018bf771537990efd7fd7bb0c598854f63b Mon Sep 17 00:00:00 2001 From: Abhinav Moudgil Date: Thu, 25 May 2017 10:47:28 +0530 Subject: [PATCH 37/84] cpplint style fixes --- src/mlpack/core/data/imputer.hpp | 1 - src/mlpack/core/data/load_arff_impl.hpp | 3 ++- src/mlpack/core/data/load_csv.hpp | 6 ++--- src/mlpack/core/data/load_impl.hpp | 6 ++--- src/mlpack/core/data/split_data.hpp | 2 +- .../core/dists/discrete_distribution.cpp | 6 ++--- src/mlpack/core/dists/gamma_distribution.cpp | 5 ++--- src/mlpack/core/dists/gamma_distribution.hpp | 8 +++---- .../core/dists/gaussian_distribution.hpp | 5 +++-- .../core/dists/laplace_distribution.cpp | 3 ++- .../core/dists/laplace_distribution.hpp | 1 - .../core/kernels/epanechnikov_kernel.hpp | 1 - src/mlpack/core/kernels/example_kernel.hpp | 2 +- src/mlpack/core/kernels/gaussian_kernel.hpp | 5 +++-- src/mlpack/core/kernels/spherical_kernel.hpp | 2 +- src/mlpack/core/math/lin_alg.cpp | 4 +++- src/mlpack/core/math/random_basis.cpp | 2 +- .../core/metrics/mahalanobis_distance.hpp | 2 +- .../aug_lagrangian_test_functions.hpp | 2 +- .../core/optimizers/lbfgs/lbfgs_impl.hpp | 4 ++-- .../optimizers/rmsprop/rmsprop_update.hpp | 2 +- .../core/optimizers/sdp/lrsdp_function.hpp | 2 -- .../optimizers/sdp/lrsdp_function_impl.hpp | 9 ++++---- .../core/optimizers/sdp/primal_dual.hpp | 2 +- .../core/optimizers/sdp/primal_dual_impl.hpp | 4 +--- src/mlpack/core/optimizers/sdp/sdp.hpp | 1 - src/mlpack/core/optimizers/sdp/sdp_impl.hpp | 4 +--- src/mlpack/core/optimizers/sgd/sgd_impl.hpp | 2 +- .../sgd/update_policies/momentum_update.hpp | 2 +- .../optimizers/smorms3/smorms3_update.hpp | 10 ++++----- src/mlpack/core/tree/address.hpp | 4 ++-- src/mlpack/core/tree/ballbound.hpp | 1 - .../binary_space_tree_impl.hpp | 2 +- .../binary_space_tree/rp_tree_mean_split.hpp | 1 - .../core/tree/cosine_tree/cosine_tree.hpp | 1 - src/mlpack/core/tree/hollow_ball_bound.hpp | 1 - src/mlpack/core/tree/hrectbound.hpp | 2 +- .../discrete_hilbert_value_impl.hpp | 7 +++--- .../rectangle_tree/dual_tree_traverser.hpp | 1 - .../dual_tree_traverser_impl.hpp | 2 +- .../hilbert_r_tree_auxiliary_information.hpp | 2 +- ...bert_r_tree_auxiliary_information_impl.hpp | 8 +++---- .../hilbert_r_tree_split_impl.hpp | 4 ++-- .../no_auxiliary_information.hpp | 10 ++++----- ...r_plus_plus_tree_auxiliary_information.hpp | 2 +- ...s_plus_tree_auxiliary_information_impl.hpp | 18 +++++---------- .../rectangle_tree/r_plus_tree_split_impl.hpp | 2 +- .../tree/rectangle_tree/r_star_tree_split.hpp | 4 ++-- .../rectangle_tree/r_star_tree_split_impl.hpp | 4 ++-- .../core/tree/rectangle_tree/r_tree_split.hpp | 8 +++---- .../tree/rectangle_tree/r_tree_split_impl.hpp | 22 +++++++++---------- .../rectangle_tree/rectangle_tree_impl.hpp | 11 +++++----- .../rectangle_tree/single_tree_traverser.hpp | 1 - .../single_tree_traverser_impl.hpp | 1 - .../core/tree/rectangle_tree/typedef.hpp | 5 +++-- .../x_tree_auxiliary_information.hpp | 1 - .../core/tree/rectangle_tree/x_tree_split.hpp | 5 ++--- .../tree/rectangle_tree/x_tree_split_impl.hpp | 8 +++---- src/mlpack/core/tree/statistic.hpp | 2 +- src/mlpack/core/util/arma_traits.hpp | 16 +++++++------- src/mlpack/core/util/backtrace.cpp | 8 +++---- src/mlpack/core/util/backtrace.hpp | 2 +- src/mlpack/core/util/log.hpp | 2 +- src/mlpack/core/util/prefixedoutstream.cpp | 2 +- src/mlpack/core/util/timers.hpp | 2 +- 65 files changed, 130 insertions(+), 150 deletions(-) diff --git a/src/mlpack/core/data/imputer.hpp b/src/mlpack/core/data/imputer.hpp index a5dddfe539..afd7a9095c 100644 --- a/src/mlpack/core/data/imputer.hpp +++ b/src/mlpack/core/data/imputer.hpp @@ -86,7 +86,6 @@ class Imputer // save columnMajor as a member variable since it is rarely changed. bool columnMajor; - }; // class Imputer } // namespace data diff --git a/src/mlpack/core/data/load_arff_impl.hpp b/src/mlpack/core/data/load_arff_impl.hpp index 81e2df0944..34a6d88410 100644 --- a/src/mlpack/core/data/load_arff_impl.hpp +++ b/src/mlpack/core/data/load_arff_impl.hpp @@ -182,7 +182,8 @@ void LoadARFF(const std::string& filename, // Strip spaces before mapping. std::string token = *it; boost::trim(token); - matrix(col, row) = info.template MapString(token, col); // We load transposed. + // We load transposed. + matrix(col, row) = info.template MapString(token, col); } else if (info.Type(col) == Datatype::numeric) { diff --git a/src/mlpack/core/data/load_csv.hpp b/src/mlpack/core/data/load_csv.hpp index 6cab5c153f..ecaff8c115 100644 --- a/src/mlpack/core/data/load_csv.hpp +++ b/src/mlpack/core/data/load_csv.hpp @@ -204,7 +204,7 @@ class LoadCSV } } -private: + private: using iter_type = boost::iterator_range; /** @@ -256,8 +256,8 @@ private: // Remove whitespace from either side. boost::trim(line); - //parse the numbers from a line(ex : 1,2,3,4), if the parser find the - //number it will execute the setNum function + // parse the numbers from a line(ex : 1,2,3,4), if the parser find the + // number it will execute the setNum function const bool canParse = qi::parse(line.begin(), line.end(), stringRule[setCharClass] % delimiterRule); diff --git a/src/mlpack/core/data/load_impl.hpp b/src/mlpack/core/data/load_impl.hpp index 9f1854af07..31a088f4ae 100644 --- a/src/mlpack/core/data/load_impl.hpp +++ b/src/mlpack/core/data/load_impl.hpp @@ -55,13 +55,13 @@ void TransposeTokens(std::vector> const &input, size_t index) { output.clear(); - for(size_t i = 0; i != input.size(); ++i) + for (size_t i = 0; i != input.size(); ++i) { output.emplace_back(input[i][index]); } } -} //namespace details +} // namespace details template bool inline inplace_transpose(arma::Mat& X) @@ -171,7 +171,7 @@ bool Load(const std::string& filename, // This is taken from load_auto_detect() in diskio_meat.hpp const std::string ARMA_MAT_TXT = "ARMA_MAT_TXT"; - //char* rawHeader = new char[ARMA_MAT_TXT.length() + 1]; + // char* rawHeader = new char[ARMA_MAT_TXT.length() + 1]; std::string rawHeader(ARMA_MAT_TXT.length(), '\0'); std::streampos pos = stream.tellg(); diff --git a/src/mlpack/core/data/split_data.hpp b/src/mlpack/core/data/split_data.hpp index 37d37ba2ce..28b0bcf966 100644 --- a/src/mlpack/core/data/split_data.hpp +++ b/src/mlpack/core/data/split_data.hpp @@ -143,7 +143,7 @@ void Split(const arma::Mat& input, * @return std::tuple containing trainData (arma::Mat), testData * (arma::Mat), trainLabel (arma::Row), and testLabel (arma::Row). */ -template +template std::tuple, arma::Mat, arma::Row, arma::Row> Split(const arma::Mat& input, const arma::Row& inputLabel, diff --git a/src/mlpack/core/dists/discrete_distribution.cpp b/src/mlpack/core/dists/discrete_distribution.cpp index 9866a126bc..dc63aec6f0 100644 --- a/src/mlpack/core/dists/discrete_distribution.cpp +++ b/src/mlpack/core/dists/discrete_distribution.cpp @@ -128,9 +128,9 @@ void DiscreteDistribution::Train(const arma::mat& observations, { for (size_t i = 0; i < dimensions; i++) { - // Add the probability of each observation. The addition of 0.5 to the - // observation is to turn the default flooring operation of the size_t cast - // into a rounding observation. + // Add the probability of each observation. The addition of 0.5 + // to the observation is to turn the default flooring operation + // of the size_t cast into a rounding observation. const size_t obs = size_t(observations(i, r) + 0.5); // Ensure that the observation is within the bounds. diff --git a/src/mlpack/core/dists/gamma_distribution.cpp b/src/mlpack/core/dists/gamma_distribution.cpp index a8da1a28c6..b47b53b74d 100644 --- a/src/mlpack/core/dists/gamma_distribution.cpp +++ b/src/mlpack/core/dists/gamma_distribution.cpp @@ -64,7 +64,7 @@ void GammaDistribution::Train(const arma::mat& rdata, const double tol) Train(logMeanxVec, meanLogxVec, meanxVec, tol); } -//Fits an alpha and beta parameter according to observation probabilities. +// Fits an alpha and beta parameter according to observation probabilities. void GammaDistribution::Train(const arma::mat& rdata, const arma::vec& probabilities, const double tol) @@ -151,7 +151,6 @@ void GammaDistribution::Train(const arma::vec& logMeanxVec, if (aEst <= 0) throw std::logic_error("GammaDistribution::Train(): estimated invalid " "negative value for parameter alpha!"); - } while (!Converged(aEst, aOld, tol)); alpha(row) = aEst; @@ -219,7 +218,7 @@ void GammaDistribution::LogProbability(const arma::mat& observations, double factor = std::exp(-observations(d, i) / beta(d)); double numerator = std::pow(observations(d, i), alpha(d) - 1); - LogProbabilities(i) += std::log( numerator * factor / denominators(d)); + LogProbabilities(i) += std::log(numerator * factor / denominators(d)); } } } diff --git a/src/mlpack/core/dists/gamma_distribution.hpp b/src/mlpack/core/dists/gamma_distribution.hpp index eadf78a55a..9e748ec442 100644 --- a/src/mlpack/core/dists/gamma_distribution.hpp +++ b/src/mlpack/core/dists/gamma_distribution.hpp @@ -50,7 +50,7 @@ namespace distribution { */ class GammaDistribution { - public: + public: /** * Construct the Gamma distribution with the given number of dimensions * (default 0); each parameter will be initialized to 0. @@ -80,7 +80,7 @@ class GammaDistribution /** * Destructor. */ - ~GammaDistribution() {}; + ~GammaDistribution() {} /** * This function trains (fits distribution parameters) to new data or the @@ -192,7 +192,7 @@ class GammaDistribution //! Get the dimensionality of the distribution. size_t Dimensionality() const { return alpha.n_elem; } - private: + private: //! Array of fitted alphas. arma::vec alpha; //! Array of fitted betas. @@ -214,7 +214,7 @@ class GammaDistribution const double tol); }; -} // namespace distributions. +} // namespace distribution. } // namespace mlpack. #endif diff --git a/src/mlpack/core/dists/gaussian_distribution.hpp b/src/mlpack/core/dists/gaussian_distribution.hpp index d817f40ccf..c680ca076a 100644 --- a/src/mlpack/core/dists/gaussian_distribution.hpp +++ b/src/mlpack/core/dists/gaussian_distribution.hpp @@ -174,8 +174,9 @@ class GaussianDistribution * @param x List of observations. * @param probabilities Output log probabilities for each input observation. */ -inline void GaussianDistribution::LogProbability(const arma::mat& x, - arma::vec& logProbabilities) const +inline void GaussianDistribution::LogProbability( + const arma::mat& x, + arma::vec& logProbabilities) const { // Column i of 'diffs' is the difference between x.col(i) and the mean. arma::mat diffs = x - (mean * arma::ones(x.n_cols)); diff --git a/src/mlpack/core/dists/laplace_distribution.cpp b/src/mlpack/core/dists/laplace_distribution.cpp index 2299ed4078..e83a9fd9dc 100644 --- a/src/mlpack/core/dists/laplace_distribution.cpp +++ b/src/mlpack/core/dists/laplace_distribution.cpp @@ -21,7 +21,8 @@ using namespace mlpack::distribution; */ double LaplaceDistribution::LogProbability(const arma::vec& observation) const { - // Evaluate the PDF of the Laplace distribution to determine the log probability. + // Evaluate the PDF of the Laplace distribution to determine + // the log probability. return -log(2. * scale) - arma::norm(observation - mean, 2) / scale; } diff --git a/src/mlpack/core/dists/laplace_distribution.hpp b/src/mlpack/core/dists/laplace_distribution.hpp index e7a59aa426..c49cd77b35 100644 --- a/src/mlpack/core/dists/laplace_distribution.hpp +++ b/src/mlpack/core/dists/laplace_distribution.hpp @@ -155,7 +155,6 @@ class LaplaceDistribution arma::vec mean; //! Scale parameter of the distribution. double scale; - }; } // namespace distribution diff --git a/src/mlpack/core/kernels/epanechnikov_kernel.hpp b/src/mlpack/core/kernels/epanechnikov_kernel.hpp index 07a5eae443..5629bbfe46 100644 --- a/src/mlpack/core/kernels/epanechnikov_kernel.hpp +++ b/src/mlpack/core/kernels/epanechnikov_kernel.hpp @@ -100,7 +100,6 @@ class EpanechnikovKernel double bandwidth; //! Cached value of the inverse bandwidth squared (to speed up computation). double inverseBandwidthSquared; - }; //! Kernel traits for the Epanechnikov kernel. diff --git a/src/mlpack/core/kernels/example_kernel.hpp b/src/mlpack/core/kernels/example_kernel.hpp index 4272535b72..0589b6e901 100644 --- a/src/mlpack/core/kernels/example_kernel.hpp +++ b/src/mlpack/core/kernels/example_kernel.hpp @@ -140,7 +140,7 @@ class ExampleKernel static double Normalizer() { return 0; } // Modified to remove unused variable "dimension" - //static double Normalizer(size_t dimension=1) { return 0; } + // static double Normalizer(size_t dimension=1) { return 0; } }; } // namespace kernel diff --git a/src/mlpack/core/kernels/gaussian_kernel.hpp b/src/mlpack/core/kernels/gaussian_kernel.hpp index 791cea1b66..31095a6ba0 100644 --- a/src/mlpack/core/kernels/gaussian_kernel.hpp +++ b/src/mlpack/core/kernels/gaussian_kernel.hpp @@ -126,8 +126,9 @@ class GaussianKernel template double ConvolutionIntegral(const VecTypeA& a, const VecTypeB& b) { - return Evaluate(sqrt(metric::SquaredEuclideanDistance::Evaluate(a, b) / 2.0)) / - (Normalizer(a.n_rows) * pow(2.0, (double) a.n_rows / 2.0)); + return Evaluate(sqrt(metric:: + SquaredEuclideanDistance::Evaluate(a, b) / 2.0)) / + (Normalizer(a.n_rows) * pow(2.0, (double) a.n_rows / 2.0)); } diff --git a/src/mlpack/core/kernels/spherical_kernel.hpp b/src/mlpack/core/kernels/spherical_kernel.hpp index 962eee6c47..e138fb1648 100644 --- a/src/mlpack/core/kernels/spherical_kernel.hpp +++ b/src/mlpack/core/kernels/spherical_kernel.hpp @@ -68,7 +68,7 @@ class SphericalKernel } double volumeSquared = pow(Normalizer(a.n_rows), 2.0); - switch(a.n_rows) + switch (a.n_rows) { case 1: return 1.0 / volumeSquared * (2.0 * bandwidth - distance); diff --git a/src/mlpack/core/math/lin_alg.cpp b/src/mlpack/core/math/lin_alg.cpp index af7fa9123c..f5336eee05 100644 --- a/src/mlpack/core/math/lin_alg.cpp +++ b/src/mlpack/core/math/lin_alg.cpp @@ -257,7 +257,9 @@ void mlpack::math::Svec(const arma::sp_mat& input, arma::sp_vec& output) void mlpack::math::Smat(const arma::vec& input, arma::mat& output) { - const size_t n = static_cast(ceil((-1. + sqrt(1. + 8. * input.n_elem))/2.)); + const size_t n = static_cast + (ceil((-1. + sqrt(1. + 8. * input.n_elem))/2.)); + output.zeros(n, n); diff --git a/src/mlpack/core/math/random_basis.cpp b/src/mlpack/core/math/random_basis.cpp index bf4e412aa8..7419908ead 100644 --- a/src/mlpack/core/math/random_basis.cpp +++ b/src/mlpack/core/math/random_basis.cpp @@ -18,7 +18,7 @@ namespace math { void RandomBasis(mat& basis, const size_t d) { - while(true) + while (true) { // [Q, R] = qr(randn(d, d)); // Q = Q * diag(sign(diag(R))); diff --git a/src/mlpack/core/metrics/mahalanobis_distance.hpp b/src/mlpack/core/metrics/mahalanobis_distance.hpp index 4ffe453039..343a4b5d2f 100644 --- a/src/mlpack/core/metrics/mahalanobis_distance.hpp +++ b/src/mlpack/core/metrics/mahalanobis_distance.hpp @@ -111,7 +111,7 @@ class MahalanobisDistance arma::mat covariance; }; -} // namespace distance +} // namespace metric } // namespace mlpack #include "mahalanobis_distance_impl.hpp" diff --git a/src/mlpack/core/optimizers/aug_lagrangian/aug_lagrangian_test_functions.hpp b/src/mlpack/core/optimizers/aug_lagrangian/aug_lagrangian_test_functions.hpp index 645feecb5b..9e9e4a028b 100644 --- a/src/mlpack/core/optimizers/aug_lagrangian/aug_lagrangian_test_functions.hpp +++ b/src/mlpack/core/optimizers/aug_lagrangian/aug_lagrangian_test_functions.hpp @@ -67,7 +67,7 @@ class GockenbachFunction double Evaluate(const arma::mat& coordinates); void Gradient(const arma::mat& coordinates, arma::mat& gradient); - size_t NumConstraints() const { return 2; }; + size_t NumConstraints() const { return 2; } double EvaluateConstraint(const size_t index, const arma::mat& coordinates); void GradientConstraint(const size_t index, diff --git a/src/mlpack/core/optimizers/lbfgs/lbfgs_impl.hpp b/src/mlpack/core/optimizers/lbfgs/lbfgs_impl.hpp index 38fa92bd5b..954e4c981e 100644 --- a/src/mlpack/core/optimizers/lbfgs/lbfgs_impl.hpp +++ b/src/mlpack/core/optimizers/lbfgs/lbfgs_impl.hpp @@ -386,7 +386,8 @@ double L_BFGS::Optimize(arma::mat& iterate) function.Evaluate(iterate) << ", gradient norm " << arma::norm(gradient, 2) << ", " << ((prevFunctionValue - functionValue) / - std::max(std::max(fabs(prevFunctionValue), fabs(functionValue)), 1.0)) << "." << std::endl; + std::max(std::max(fabs(prevFunctionValue), fabs(functionValue)), 1.0)) + << "." << std::endl; prevFunctionValue = functionValue; @@ -452,7 +453,6 @@ double L_BFGS::Optimize(arma::mat& iterate) // Overwrite an old basis set. UpdateBasisSet(itNum, iterate, oldIterate, gradient, oldGradient); - } // End of the optimization loop. return function.Evaluate(iterate); diff --git a/src/mlpack/core/optimizers/rmsprop/rmsprop_update.hpp b/src/mlpack/core/optimizers/rmsprop/rmsprop_update.hpp index c86f09f0c3..30f7aacbbe 100644 --- a/src/mlpack/core/optimizers/rmsprop/rmsprop_update.hpp +++ b/src/mlpack/core/optimizers/rmsprop/rmsprop_update.hpp @@ -114,4 +114,4 @@ class RMSPropUpdate } // namespace optimization } // namespace mlpack -#endif \ No newline at end of file +#endif diff --git a/src/mlpack/core/optimizers/sdp/lrsdp_function.hpp b/src/mlpack/core/optimizers/sdp/lrsdp_function.hpp index 6493dfb0fd..00c39f2993 100644 --- a/src/mlpack/core/optimizers/sdp/lrsdp_function.hpp +++ b/src/mlpack/core/optimizers/sdp/lrsdp_function.hpp @@ -27,7 +27,6 @@ template class LRSDPFunction { public: - /** * Construct the LRSDPFunction from the given SDP. * @@ -90,7 +89,6 @@ class LRSDPFunction SDPType& SDP() { return sdp; } private: - //! SDP object representing the problem SDPType sdp; diff --git a/src/mlpack/core/optimizers/sdp/lrsdp_function_impl.hpp b/src/mlpack/core/optimizers/sdp/lrsdp_function_impl.hpp index 7b23b5eead..d3aaf8aefd 100644 --- a/src/mlpack/core/optimizers/sdp/lrsdp_function_impl.hpp +++ b/src/mlpack/core/optimizers/sdp/lrsdp_function_impl.hpp @@ -55,13 +55,14 @@ template void LRSDPFunction::Gradient(const arma::mat& /* coordinates */, arma::mat& /* gradient */) const { - Log::Fatal << "LRSDPFunction::Gradient() not implemented for arbitrary optimizers!" - << std::endl; + Log::Fatal << "LRSDPFunction::Gradient() not implemented for arbitrary + optimizers!" << std::endl; } template -double LRSDPFunction::EvaluateConstraint(const size_t index, - const arma::mat& coordinates) const +double LRSDPFunction::EvaluateConstraint( + const size_t index, + const arma::mat& coordinates) const { const arma::mat rrt = coordinates * trans(coordinates); if (index < SDP().NumSparseConstraints()) diff --git a/src/mlpack/core/optimizers/sdp/primal_dual.hpp b/src/mlpack/core/optimizers/sdp/primal_dual.hpp index 9915115d95..ffa5b5af19 100644 --- a/src/mlpack/core/optimizers/sdp/primal_dual.hpp +++ b/src/mlpack/core/optimizers/sdp/primal_dual.hpp @@ -109,7 +109,7 @@ class PrimalDualSolver arma::vec initialYdense; //! Starting point for Z, the complementary slack variable. Needs to be - //positive definite. + // positive definite. arma::mat initialZ; //! The step size modulating factor. Needs to be a scalar in (0, 1). diff --git a/src/mlpack/core/optimizers/sdp/primal_dual_impl.hpp b/src/mlpack/core/optimizers/sdp/primal_dual_impl.hpp index bc01438e4d..348dc3a603 100644 --- a/src/mlpack/core/optimizers/sdp/primal_dual_impl.hpp +++ b/src/mlpack/core/optimizers/sdp/primal_dual_impl.hpp @@ -46,9 +46,7 @@ PrimalDualSolver::PrimalDualSolver(const SDPType& sdp) primalInfeasTol(1e-7), dualInfeasTol(1e-7), maxIterations(1000) -{ - -} +{ /* Nothing to do. */ } template PrimalDualSolver::PrimalDualSolver(const SDPType& sdp, diff --git a/src/mlpack/core/optimizers/sdp/sdp.hpp b/src/mlpack/core/optimizers/sdp/sdp.hpp index 5a0c89e944..2b7151dd83 100644 --- a/src/mlpack/core/optimizers/sdp/sdp.hpp +++ b/src/mlpack/core/optimizers/sdp/sdp.hpp @@ -39,7 +39,6 @@ template class SDP { public: - typedef ObjectiveMatrixType objective_matrix_type; /** diff --git a/src/mlpack/core/optimizers/sdp/sdp_impl.hpp b/src/mlpack/core/optimizers/sdp/sdp_impl.hpp index 2e0ff76449..88ec79d0ad 100644 --- a/src/mlpack/core/optimizers/sdp/sdp_impl.hpp +++ b/src/mlpack/core/optimizers/sdp/sdp_impl.hpp @@ -23,9 +23,7 @@ SDP::SDP() : sparseB(), denseA(), denseB() -{ - -} +{ /* Nothing to do. */ } template SDP::SDP(const size_t n, diff --git a/src/mlpack/core/optimizers/sgd/sgd_impl.hpp b/src/mlpack/core/optimizers/sgd/sgd_impl.hpp index 5ac19ba960..ac6841e92e 100644 --- a/src/mlpack/core/optimizers/sgd/sgd_impl.hpp +++ b/src/mlpack/core/optimizers/sgd/sgd_impl.hpp @@ -65,7 +65,7 @@ double SGD::Optimize( overallObjective += function.Evaluate(iterate, i); // Initialize the update policy. - updatePolicy.Initialize(iterate.n_rows,iterate.n_cols); + updatePolicy.Initialize(iterate.n_rows, iterate.n_cols); // Now iterate! arma::mat gradient(iterate.n_rows, iterate.n_cols); diff --git a/src/mlpack/core/optimizers/sgd/update_policies/momentum_update.hpp b/src/mlpack/core/optimizers/sgd/update_policies/momentum_update.hpp index 1ea0856034..6d9041a557 100644 --- a/src/mlpack/core/optimizers/sgd/update_policies/momentum_update.hpp +++ b/src/mlpack/core/optimizers/sgd/update_policies/momentum_update.hpp @@ -84,7 +84,7 @@ class MomentumUpdate void Initialize(const size_t rows, const size_t cols) { - //Initialize am empty velocity matrix. + // Initialize am empty velocity matrix. velocity = arma::zeros(rows, cols); } diff --git a/src/mlpack/core/optimizers/smorms3/smorms3_update.hpp b/src/mlpack/core/optimizers/smorms3/smorms3_update.hpp index 3cf35eb7df..67e332f625 100644 --- a/src/mlpack/core/optimizers/smorms3/smorms3_update.hpp +++ b/src/mlpack/core/optimizers/smorms3/smorms3_update.hpp @@ -98,14 +98,14 @@ class SMORMS3Update double& Epsilon() { return epsilon; } private: - //! The value used to initialise the mean squared gradient parameter. - double epsilon; + //! The value used to initialise the mean squared gradient parameter. + double epsilon; - // The parameters mem, g and g2. - arma::mat mem, g, g2; + // The parameters mem, g and g2. + arma::mat mem, g, g2; }; } // namespace optimization } // namespace mlpack -#endif \ No newline at end of file +#endif diff --git a/src/mlpack/core/tree/address.hpp b/src/mlpack/core/tree/address.hpp index b1ab4951fe..03c8cb4f9c 100644 --- a/src/mlpack/core/tree/address.hpp +++ b/src/mlpack/core/tree/address.hpp @@ -82,7 +82,7 @@ void PointToAddress(AddressType& address, const VecType& point) for (size_t i = 0; i < point.n_elem; i++) { int e; - VecElemType normalizedVal = std::frexp(point(i),&e); + VecElemType normalizedVal = std::frexp(point(i), &e); bool sgn = std::signbit(normalizedVal); if (point(i) == 0) @@ -262,6 +262,6 @@ bool Contains(const AddressType1& address, const AddressType2& loBound, } // namespace addr } // namespace bound -} // namespave mlpack +} // namespace mlpack #endif // MLPACK_CORE_TREE_ADDRESS_HPP diff --git a/src/mlpack/core/tree/ballbound.hpp b/src/mlpack/core/tree/ballbound.hpp index 3ae7373c4e..a036a5fbd8 100644 --- a/src/mlpack/core/tree/ballbound.hpp +++ b/src/mlpack/core/tree/ballbound.hpp @@ -54,7 +54,6 @@ class BallBound bool ownsMetric; public: - //! Empty Constructor. BallBound(); diff --git a/src/mlpack/core/tree/binary_space_tree/binary_space_tree_impl.hpp b/src/mlpack/core/tree/binary_space_tree/binary_space_tree_impl.hpp index fd3cbf387c..cea6e71895 100644 --- a/src/mlpack/core/tree/binary_space_tree/binary_space_tree_impl.hpp +++ b/src/mlpack/core/tree/binary_space_tree/binary_space_tree_impl.hpp @@ -413,7 +413,7 @@ BinarySpaceTree(BinarySpaceTree&& other) : other.minimumBoundDistance = 0.0; other.dataset = NULL; - //Set new parent. + // Set new parent. if (left) left->parent = this; if (right) diff --git a/src/mlpack/core/tree/binary_space_tree/rp_tree_mean_split.hpp b/src/mlpack/core/tree/binary_space_tree/rp_tree_mean_split.hpp index 3d8d8a6f8c..6dd08a086b 100644 --- a/src/mlpack/core/tree/binary_space_tree/rp_tree_mean_split.hpp +++ b/src/mlpack/core/tree/binary_space_tree/rp_tree_mean_split.hpp @@ -132,7 +132,6 @@ class RPTreeMeanSplit } private: - /** * Get the average distance between points in the dataset. * diff --git a/src/mlpack/core/tree/cosine_tree/cosine_tree.hpp b/src/mlpack/core/tree/cosine_tree/cosine_tree.hpp index 8e1155dd22..a94562afba 100644 --- a/src/mlpack/core/tree/cosine_tree/cosine_tree.hpp +++ b/src/mlpack/core/tree/cosine_tree/cosine_tree.hpp @@ -246,7 +246,6 @@ class CosineTree class CompareCosineNode { public: - // Comparison function for construction of priority queue. bool operator() (const CosineTree* a, const CosineTree* b) const { diff --git a/src/mlpack/core/tree/hollow_ball_bound.hpp b/src/mlpack/core/tree/hollow_ball_bound.hpp index 76eda7f0f5..a4ba7772ba 100644 --- a/src/mlpack/core/tree/hollow_ball_bound.hpp +++ b/src/mlpack/core/tree/hollow_ball_bound.hpp @@ -55,7 +55,6 @@ class HollowBallBound bool ownsMetric; public: - //! Empty Constructor. HollowBallBound(); diff --git a/src/mlpack/core/tree/hrectbound.hpp b/src/mlpack/core/tree/hrectbound.hpp index da2cd7ef71..d54f6a24c0 100644 --- a/src/mlpack/core/tree/hrectbound.hpp +++ b/src/mlpack/core/tree/hrectbound.hpp @@ -39,7 +39,7 @@ struct IsLMetric> static const bool Value = true; }; -} // namespace util +} // namespace meta /** * Hyper-rectangle bound for an L-metric. This should be used in conjunction diff --git a/src/mlpack/core/tree/rectangle_tree/discrete_hilbert_value_impl.hpp b/src/mlpack/core/tree/rectangle_tree/discrete_hilbert_value_impl.hpp index f757826464..53b23e36c0 100644 --- a/src/mlpack/core/tree/rectangle_tree/discrete_hilbert_value_impl.hpp +++ b/src/mlpack/core/tree/rectangle_tree/discrete_hilbert_value_impl.hpp @@ -166,7 +166,7 @@ CalculateValue(const VecType& pt, for (size_t i = 0; i < pt.n_rows; i++) { int e; - VecElemType normalizedVal = std::frexp(pt(i),&e); + VecElemType normalizedVal = std::frexp(pt(i), &e); bool sgn = std::signbit(normalizedVal); if (pt(i) == 0) @@ -325,7 +325,7 @@ CompareWith(const VecType& pt, if (numValues == 0) return -1; - return CompareValues(localHilbertValues->col(numValues - 1),val); + return CompareValues(localHilbertValues->col(numValues - 1), val); } template @@ -384,7 +384,7 @@ void DiscreteHilbertValue::InsertNode(TreeType* node) { DiscreteHilbertValue &val = node->AuxiliaryInfo().HilbertValue(); - if (CompareWith(node,val) < 0) + if (CompareWith(node, val) < 0) { localHilbertValues = val.LocalHilbertValues(); numValues = val.NumValues(); @@ -396,7 +396,6 @@ template void DiscreteHilbertValue:: DeletePoint(TreeType* /* node */, const size_t localIndex) { - // Delete the Hilbert value from the local dataset for (size_t i = numValues - 1; i > localIndex; i--) localHilbertValues->col(i - 1) = localHilbertValues->col(i); diff --git a/src/mlpack/core/tree/rectangle_tree/dual_tree_traverser.hpp b/src/mlpack/core/tree/rectangle_tree/dual_tree_traverser.hpp index e2a390db7d..8f4607c720 100644 --- a/src/mlpack/core/tree/rectangle_tree/dual_tree_traverser.hpp +++ b/src/mlpack/core/tree/rectangle_tree/dual_tree_traverser.hpp @@ -67,7 +67,6 @@ class RectangleTree::Traverse(RectangleTree& queryNode, if (childScore == DBL_MAX) continue; // We don't require a search in this reference node. - for(size_t ref = 0; ref < referenceNode.Count(); ++ref) + for (size_t ref = 0; ref < referenceNode.Count(); ++ref) rule.BaseCase(queryNode.Point(query), referenceNode.Point(ref)); numBaseCases += referenceNode.Count(); diff --git a/src/mlpack/core/tree/rectangle_tree/hilbert_r_tree_auxiliary_information.hpp b/src/mlpack/core/tree/rectangle_tree/hilbert_r_tree_auxiliary_information.hpp index c80e2d05c3..6ef9164749 100644 --- a/src/mlpack/core/tree/rectangle_tree/hilbert_r_tree_auxiliary_information.hpp +++ b/src/mlpack/core/tree/rectangle_tree/hilbert_r_tree_auxiliary_information.hpp @@ -147,4 +147,4 @@ class HilbertRTreeAuxiliaryInformation #include "hilbert_r_tree_auxiliary_information_impl.hpp" -#endif//MLPACK_CORE_TREE_RECTANGLE_TREE_HILBERT_R_TREE_AUXILIARY_INFORMATION_HPP +#endif // MLPACK_CORE_TREE_RECTANGLE_TREE_HILBERT_R_TREE_AUXILIARY_INFORMATION_HPP diff --git a/src/mlpack/core/tree/rectangle_tree/hilbert_r_tree_auxiliary_information_impl.hpp b/src/mlpack/core/tree/rectangle_tree/hilbert_r_tree_auxiliary_information_impl.hpp index 2fceeede21..dfefd8899d 100644 --- a/src/mlpack/core/tree/rectangle_tree/hilbert_r_tree_auxiliary_information_impl.hpp +++ b/src/mlpack/core/tree/rectangle_tree/hilbert_r_tree_auxiliary_information_impl.hpp @@ -126,7 +126,7 @@ bool HilbertRTreeAuxiliaryInformation:: HandlePointDeletion(TreeType* node, const size_t localIndex) { // Update the largest Hilbert value. - hilbertValue.DeletePoint(node,localIndex); + hilbertValue.DeletePoint(node, localIndex); for (size_t i = localIndex + 1; localIndex < node->NumPoints(); i++) node->Point(i - 1) = node->Point(i); @@ -141,7 +141,7 @@ bool HilbertRTreeAuxiliaryInformation:: HandleNodeRemoval(TreeType* node, const size_t nodeIndex) { // Update the largest Hilbert value. - hilbertValue.RemoveNode(node,nodeIndex); + hilbertValue.RemoveNode(node, nodeIndex); for (size_t i = nodeIndex + 1; nodeIndex < node->NumChildren(); i++) node->children[i - 1] = node->children[i]; @@ -178,7 +178,7 @@ NullifyData() template class HilbertValueType> template -void HilbertRTreeAuxiliaryInformation:: +void HilbertRTreeAuxiliaryInformation:: Serialize(Archive& ar, const unsigned int /* version */) { using data::CreateNVP; @@ -190,4 +190,4 @@ Serialize(Archive& ar, const unsigned int /* version */) } // namespace tree } // namespace mlpack -#endif//MLPACK_CORE_TREE_RECTANGLE_TREE_HILBERT_R_TREE_AUXILIARY_INFORMATION_IMPL_HPP +#endif // MLPACK_CORE_TREE_RECTANGLE_TREE_HILBERT_R_TREE_AUXILIARY_INFORMATION_IMPL_HPP diff --git a/src/mlpack/core/tree/rectangle_tree/hilbert_r_tree_split_impl.hpp b/src/mlpack/core/tree/rectangle_tree/hilbert_r_tree_split_impl.hpp index 32ca6b339a..600665c7bb 100644 --- a/src/mlpack/core/tree/rectangle_tree/hilbert_r_tree_split_impl.hpp +++ b/src/mlpack/core/tree/rectangle_tree/hilbert_r_tree_split_impl.hpp @@ -48,7 +48,7 @@ void HilbertRTreeSplit::SplitLeafNode(TreeType* tree, TreeType* parent = tree->Parent(); size_t iTree = 0; - for (iTree = 0; parent->children[iTree] != tree; iTree++); + for (iTree = 0; parent->children[iTree] != tree; iTree++) { } // Try to find splitOrder cooperating siblings in order to redistribute points // among them and avoid split. @@ -112,7 +112,7 @@ SplitNonLeafNode(TreeType* tree, std::vector& relevels) TreeType* parent = tree->Parent(); size_t iTree = 0; - for (iTree = 0; parent->children[iTree] != tree; iTree++); + for (iTree = 0; parent->children[iTree] != tree; iTree++) { } // Try to find splitOrder cooperating siblings in order to redistribute // children among them and avoid split. diff --git a/src/mlpack/core/tree/rectangle_tree/no_auxiliary_information.hpp b/src/mlpack/core/tree/rectangle_tree/no_auxiliary_information.hpp index 01d18db371..240bf6677f 100644 --- a/src/mlpack/core/tree/rectangle_tree/no_auxiliary_information.hpp +++ b/src/mlpack/core/tree/rectangle_tree/no_auxiliary_information.hpp @@ -21,15 +21,15 @@ class NoAuxiliaryInformation { public: //! Construct the auxiliary information object. - NoAuxiliaryInformation() { }; + NoAuxiliaryInformation() { } //! Construct the auxiliary information object. - NoAuxiliaryInformation(const TreeType* /* node */) { }; + NoAuxiliaryInformation(const TreeType* /* node */) { } //! Construct the auxiliary information object. NoAuxiliaryInformation(const NoAuxiliaryInformation& /* other */, TreeType* /* tree */, - bool /* deepCopy */ = true) { }; + bool /* deepCopy */ = true) { } //! Construct the auxiliary information object. - NoAuxiliaryInformation(NoAuxiliaryInformation&& /* other */) { }; + NoAuxiliaryInformation(NoAuxiliaryInformation&& /* other */) { } //! Copy the auxiliary information object. NoAuxiliaryInformation& operator=(const NoAuxiliaryInformation& /* other */) @@ -141,7 +141,7 @@ class NoAuxiliaryInformation * Serialize the information. */ template - void Serialize(Archive &, const unsigned int /* version */) { }; + void Serialize(Archive &, const unsigned int /* version */) { } }; } // namespace tree diff --git a/src/mlpack/core/tree/rectangle_tree/r_plus_plus_tree_auxiliary_information.hpp b/src/mlpack/core/tree/rectangle_tree/r_plus_plus_tree_auxiliary_information.hpp index 6893352cde..a4e363ebe0 100644 --- a/src/mlpack/core/tree/rectangle_tree/r_plus_plus_tree_auxiliary_information.hpp +++ b/src/mlpack/core/tree/rectangle_tree/r_plus_plus_tree_auxiliary_information.hpp @@ -163,4 +163,4 @@ class RPlusPlusTreeAuxiliaryInformation #include "r_plus_plus_tree_auxiliary_information_impl.hpp" -#endif//MLPACK_CORE_TREE_RECTANGLE_TREE_R_PLUS_PLUS_TREE_AUXILIARY_INFORMATION_HPP +#endif // MLPACK_CORE_TREE_RECTANGLE_TREE_R_PLUS_PLUS_TREE_AUXILIARY_INFORMATION_HPP diff --git a/src/mlpack/core/tree/rectangle_tree/r_plus_plus_tree_auxiliary_information_impl.hpp b/src/mlpack/core/tree/rectangle_tree/r_plus_plus_tree_auxiliary_information_impl.hpp index 06f8becb3b..92bd22f610 100644 --- a/src/mlpack/core/tree/rectangle_tree/r_plus_plus_tree_auxiliary_information_impl.hpp +++ b/src/mlpack/core/tree/rectangle_tree/r_plus_plus_tree_auxiliary_information_impl.hpp @@ -23,9 +23,7 @@ template RPlusPlusTreeAuxiliaryInformation:: RPlusPlusTreeAuxiliaryInformation() : outerBound(0) -{ - -} +{ /* Nothing to do. */ } template RPlusPlusTreeAuxiliaryInformation:: @@ -36,11 +34,13 @@ RPlusPlusTreeAuxiliaryInformation(const TreeType* tree) : { // Initialize the maximum bounding rectangle if the node is the root if (!tree->Parent()) + { for (size_t k = 0; k < outerBound.Dim(); k++) { outerBound[k].Lo() = std::numeric_limits::lowest(); outerBound[k].Hi() = std::numeric_limits::max(); } + } } template @@ -50,17 +50,13 @@ RPlusPlusTreeAuxiliaryInformation( TreeType* /* tree */, bool /* deepCopy */) : outerBound(other.OuterBound()) -{ - -} +{ /* Nothing to do. */ } template RPlusPlusTreeAuxiliaryInformation:: RPlusPlusTreeAuxiliaryInformation(RPlusPlusTreeAuxiliaryInformation&& other) : outerBound(std::move(other.outerBound)) -{ - -} +{ /* Nothing to do. */ } template bool RPlusPlusTreeAuxiliaryInformation::HandlePointInsertion( @@ -122,9 +118,7 @@ void RPlusPlusTreeAuxiliaryInformation::SplitAuxiliaryInfo( template void RPlusPlusTreeAuxiliaryInformation::NullifyData() -{ - -} +{ /* Nothing to do */ } /** * Serialize the information. diff --git a/src/mlpack/core/tree/rectangle_tree/r_plus_tree_split_impl.hpp b/src/mlpack/core/tree/rectangle_tree/r_plus_tree_split_impl.hpp index 66f9e96694..0293f341e0 100644 --- a/src/mlpack/core/tree/rectangle_tree/r_plus_tree_split_impl.hpp +++ b/src/mlpack/core/tree/rectangle_tree/r_plus_tree_split_impl.hpp @@ -137,7 +137,7 @@ SplitNonLeafNode(TreeType* tree, std::vector& relevels) tree->NullifyData(); tree->children[(tree->NumChildren())++] = copy; - RPlusTreeSplit::SplitNonLeafNode(copy,relevels); + RPlusTreeSplit::SplitNonLeafNode(copy, relevels); return true; } size_t cutAxis = tree->Bound().Dim(); diff --git a/src/mlpack/core/tree/rectangle_tree/r_star_tree_split.hpp b/src/mlpack/core/tree/rectangle_tree/r_star_tree_split.hpp index f0c5f5845f..40a8dd0a52 100644 --- a/src/mlpack/core/tree/rectangle_tree/r_star_tree_split.hpp +++ b/src/mlpack/core/tree/rectangle_tree/r_star_tree_split.hpp @@ -32,14 +32,14 @@ class RStarTreeSplit * necessary, this split will propagate upwards through the tree. */ template - static void SplitLeafNode(TreeType *tree,std::vector& relevels); + static void SplitLeafNode(TreeType *tree, std::vector& relevels); /** * Split a non-leaf node using the "default" algorithm. If this is a root * node, the tree increases in depth. */ template - static bool SplitNonLeafNode(TreeType *tree,std::vector& relevels); + static bool SplitNonLeafNode(TreeType *tree, std::vector& relevels); /** * Reinsert any points into the tree, if needed. This returns the number of diff --git a/src/mlpack/core/tree/rectangle_tree/r_star_tree_split_impl.hpp b/src/mlpack/core/tree/rectangle_tree/r_star_tree_split_impl.hpp index 6fe28a6407..d3d3a92514 100644 --- a/src/mlpack/core/tree/rectangle_tree/r_star_tree_split_impl.hpp +++ b/src/mlpack/core/tree/rectangle_tree/r_star_tree_split_impl.hpp @@ -174,7 +174,7 @@ void RStarTreeSplit::PickLeafSplit(TreeType* tree, * new nodes into the tree, spliting the parent if necessary. */ template -void RStarTreeSplit::SplitLeafNode(TreeType *tree,std::vector& relevels) +void RStarTreeSplit::SplitLeafNode(TreeType *tree, std::vector& relevels) { // Convenience typedef. typedef typename TreeType::ElemType ElemType; @@ -268,7 +268,7 @@ void RStarTreeSplit::SplitLeafNode(TreeType *tree,std::vector& relevels) * higher up the tree because they were already updated if necessary. */ template -bool RStarTreeSplit::SplitNonLeafNode(TreeType *tree,std::vector& relevels) +bool RStarTreeSplit::SplitNonLeafNode(TreeType *tree, std::vector& relevels) { // Convenience typedef. typedef typename TreeType::ElemType ElemType; diff --git a/src/mlpack/core/tree/rectangle_tree/r_tree_split.hpp b/src/mlpack/core/tree/rectangle_tree/r_tree_split.hpp index 0375876bba..b140454e5a 100644 --- a/src/mlpack/core/tree/rectangle_tree/r_tree_split.hpp +++ b/src/mlpack/core/tree/rectangle_tree/r_tree_split.hpp @@ -31,27 +31,27 @@ class RTreeSplit * will propagate upwards through the tree. */ template - static void SplitLeafNode(TreeType *tree,std::vector& relevels); + static void SplitLeafNode(TreeType *tree, std::vector& relevels); /** * Split a non-leaf node using the "default" algorithm. If this is a root * node, the tree increases in depth. */ template - static bool SplitNonLeafNode(TreeType *tree,std::vector& relevels); + static bool SplitNonLeafNode(TreeType *tree, std::vector& relevels); private: /** * Get the seeds for splitting a leaf node. */ template - static void GetPointSeeds(const TreeType *tree,int& i, int& j); + static void GetPointSeeds(const TreeType *tree, int& i, int& j); /** * Get the seeds for splitting a non-leaf node. */ template - static void GetBoundSeeds(const TreeType *tree,int& i, int& j); + static void GetBoundSeeds(const TreeType *tree, int& i, int& j); /** * Assign points to the two new nodes. diff --git a/src/mlpack/core/tree/rectangle_tree/r_tree_split_impl.hpp b/src/mlpack/core/tree/rectangle_tree/r_tree_split_impl.hpp index c64c455973..02b1427c85 100644 --- a/src/mlpack/core/tree/rectangle_tree/r_tree_split_impl.hpp +++ b/src/mlpack/core/tree/rectangle_tree/r_tree_split_impl.hpp @@ -26,7 +26,7 @@ namespace tree { * new nodes into the tree, spliting the parent if necessary. */ template -void RTreeSplit::SplitLeafNode(TreeType *tree,std::vector& relevels) +void RTreeSplit::SplitLeafNode(TreeType *tree, std::vector& relevels) { if (tree->Count() <= tree->MaxLeafSize()) return; @@ -42,7 +42,7 @@ void RTreeSplit::SplitLeafNode(TreeType *tree,std::vector& relevels) tree->NullifyData(); // Because this was a leaf node, numChildren must be 0. tree->children[(tree->NumChildren())++] = copy; - RTreeSplit::SplitLeafNode(copy,relevels); + RTreeSplit::SplitLeafNode(copy, relevels); return; } @@ -53,7 +53,7 @@ void RTreeSplit::SplitLeafNode(TreeType *tree,std::vector& relevels) // rectangles, only points. We assume that the tree uses Euclidean Distance. int i = 0; int j = 0; - RTreeSplit::GetPointSeeds(tree,i, j); + RTreeSplit::GetPointSeeds(tree, i, j); TreeType* treeOne = new TreeType(tree->Parent()); TreeType* treeTwo = new TreeType(tree->Parent()); @@ -73,7 +73,7 @@ void RTreeSplit::SplitLeafNode(TreeType *tree,std::vector& relevels) // just in case, we use an assert. assert(par->NumChildren() <= par->MaxNumChildren() + 1); if (par->NumChildren() == par->MaxNumChildren() + 1) - RTreeSplit::SplitNonLeafNode(par,relevels); + RTreeSplit::SplitNonLeafNode(par, relevels); assert(treeOne->Parent()->NumChildren() <= treeOne->MaxNumChildren()); assert(treeOne->Parent()->NumChildren() >= treeOne->MinNumChildren()); @@ -92,7 +92,7 @@ void RTreeSplit::SplitLeafNode(TreeType *tree,std::vector& relevels) * higher up the tree because they were already updated if necessary. */ template -bool RTreeSplit::SplitNonLeafNode(TreeType *tree,std::vector& relevels) +bool RTreeSplit::SplitNonLeafNode(TreeType *tree, std::vector& relevels) { // If we are splitting the root node, we need will do things differently so // that the constructor and other methods don't confuse the end user by giving @@ -105,13 +105,13 @@ bool RTreeSplit::SplitNonLeafNode(TreeType *tree,std::vector& relevels) tree->NumChildren() = 0; tree->NullifyData(); tree->children[(tree->NumChildren())++] = copy; - RTreeSplit::SplitNonLeafNode(copy,relevels); + RTreeSplit::SplitNonLeafNode(copy, relevels); return true; } int i = 0; int j = 0; - RTreeSplit::GetBoundSeeds(tree,i, j); + RTreeSplit::GetBoundSeeds(tree, i, j); assert(i != j); @@ -138,7 +138,7 @@ bool RTreeSplit::SplitNonLeafNode(TreeType *tree,std::vector& relevels) assert(par->NumChildren() <= par->MaxNumChildren() + 1); if (par->NumChildren() == par->MaxNumChildren() + 1) - RTreeSplit::SplitNonLeafNode(par,relevels); + RTreeSplit::SplitNonLeafNode(par, relevels); // We have to update the children of each of these new nodes so that they // record the correct parent. @@ -154,7 +154,7 @@ bool RTreeSplit::SplitNonLeafNode(TreeType *tree,std::vector& relevels) // Because we now have pointers to the information stored under this tree, // we need to delete this node carefully. - tree->SoftDelete(); //currently does nothing but leak memory. + tree->SoftDelete(); // currently does nothing but leak memory. return false; } @@ -164,7 +164,7 @@ bool RTreeSplit::SplitNonLeafNode(TreeType *tree,std::vector& relevels) * The indices of these points will be stored in iRet and jRet. */ template -void RTreeSplit::GetPointSeeds(const TreeType *tree,int& iRet, int& jRet) +void RTreeSplit::GetPointSeeds(const TreeType *tree, int& iRet, int& jRet) { // Here we want to find the pair of points that it is worst to place in the // same node. Because we are just using points, we will simply choose the two @@ -193,7 +193,7 @@ void RTreeSplit::GetPointSeeds(const TreeType *tree,int& iRet, int& jRet) * indices of the bounds will be stored in iRet and jRet. */ template -void RTreeSplit::GetBoundSeeds(const TreeType *tree,int& iRet, int& jRet) +void RTreeSplit::GetBoundSeeds(const TreeType *tree, int& iRet, int& jRet) { // Convenience typedef. typedef typename TreeType::ElemType ElemType; diff --git a/src/mlpack/core/tree/rectangle_tree/rectangle_tree_impl.hpp b/src/mlpack/core/tree/rectangle_tree/rectangle_tree_impl.hpp index 3a31c76778..becd85ad8f 100644 --- a/src/mlpack/core/tree/rectangle_tree/rectangle_tree_impl.hpp +++ b/src/mlpack/core/tree/rectangle_tree/rectangle_tree_impl.hpp @@ -112,7 +112,7 @@ RectangleTree* - parentNode,const size_t numMaxChildren) : + parentNode, const size_t numMaxChildren) : maxNumChildren(numMaxChildren > 0 ? numMaxChildren : parentNode->MaxNumChildren()), minNumChildren(parentNode->MinNumChildren()), @@ -385,7 +385,7 @@ void RectangleTreeInsertPoint(point, relevels); } @@ -879,7 +879,7 @@ void RectangleTreechildren[j] == this) { // Decrement numChildren. - if (!auxiliaryInfo.HandleNodeRemoval(parent,j)) + if (!auxiliaryInfo.HandleNodeRemoval(parent, j)) { parent->children[j] = parent->children[--parent->NumChildren()]; } @@ -1261,7 +1261,6 @@ void RectangleTree::Traverse( const size_t queryIndex, const RectangleTree& referenceNode) { - // If we reach a leaf node, we need to run the base case. if (referenceNode.IsLeaf()) { diff --git a/src/mlpack/core/tree/rectangle_tree/typedef.hpp b/src/mlpack/core/tree/rectangle_tree/typedef.hpp index 20bbfdc7f7..c589732d74 100644 --- a/src/mlpack/core/tree/rectangle_tree/typedef.hpp +++ b/src/mlpack/core/tree/rectangle_tree/typedef.hpp @@ -125,7 +125,7 @@ using XTree = RectangleTree using DiscreteHilbertRTreeAuxiliaryInformation = - HilbertRTreeAuxiliaryInformation; + HilbertRTreeAuxiliaryInformation; template using HilbertRTree = RectangleTree + */ +template using RPlusPlusTree = RectangleTree - static void SplitLeafNode(TreeType *tree,std::vector& relevels); + static void SplitLeafNode(TreeType *tree, std::vector& relevels); /** * Split a non-leaf node using the "default" algorithm. If this is a root * node, the tree increases in depth. */ template - static bool SplitNonLeafNode(TreeType *tree,std::vector& relevels); + static bool SplitNonLeafNode(TreeType *tree, std::vector& relevels); private: /** @@ -68,7 +68,6 @@ class XTreeSplit { return p1.first < p2.first; } - }; } // namespace tree diff --git a/src/mlpack/core/tree/rectangle_tree/x_tree_split_impl.hpp b/src/mlpack/core/tree/rectangle_tree/x_tree_split_impl.hpp index 0457d0a570..bcdeb37c2f 100644 --- a/src/mlpack/core/tree/rectangle_tree/x_tree_split_impl.hpp +++ b/src/mlpack/core/tree/rectangle_tree/x_tree_split_impl.hpp @@ -26,7 +26,7 @@ namespace tree { * new nodes into the tree, spliting the parent if necessary. */ template -void XTreeSplit::SplitLeafNode(TreeType *tree,std::vector& relevels) +void XTreeSplit::SplitLeafNode(TreeType *tree, std::vector& relevels) { // Convenience typedef. typedef typename TreeType::ElemType ElemType; @@ -110,7 +110,7 @@ void XTreeSplit::SplitLeafNode(TreeType *tree,std::vector& relevels) // If we overflowed the parent, split it. if (par && par->NumChildren() == par->MaxNumChildren() + 1) - XTreeSplit::SplitNonLeafNode(par,relevels); + XTreeSplit::SplitNonLeafNode(par, relevels); } /** @@ -121,7 +121,7 @@ void XTreeSplit::SplitLeafNode(TreeType *tree,std::vector& relevels) * higher up the tree because they were already updated if necessary. */ template -bool XTreeSplit::SplitNonLeafNode(TreeType *tree,std::vector& relevels) +bool XTreeSplit::SplitNonLeafNode(TreeType *tree, std::vector& relevels) { // Convenience typedef. typedef typename TreeType::ElemType ElemType; @@ -567,7 +567,7 @@ bool XTreeSplit::SplitNonLeafNode(TreeType *tree,std::vector& relevels) assert(par->NumChildren() <= par->MaxNumChildren() + 1); if (par->NumChildren() == par->MaxNumChildren() + 1) - XTreeSplit::SplitNonLeafNode(par,relevels); + XTreeSplit::SplitNonLeafNode(par, relevels); // We have to update the children of each of these new nodes so that they // record the correct parent. diff --git a/src/mlpack/core/tree/statistic.hpp b/src/mlpack/core/tree/statistic.hpp index 706f5123f3..5ab361be45 100644 --- a/src/mlpack/core/tree/statistic.hpp +++ b/src/mlpack/core/tree/statistic.hpp @@ -23,7 +23,7 @@ namespace tree { */ class EmptyStatistic { - public: + public: EmptyStatistic() { } ~EmptyStatistic() { } diff --git a/src/mlpack/core/util/arma_traits.hpp b/src/mlpack/core/util/arma_traits.hpp index b9661dd3e1..45e5dac125 100644 --- a/src/mlpack/core/util/arma_traits.hpp +++ b/src/mlpack/core/util/arma_traits.hpp @@ -38,43 +38,43 @@ struct IsVector }; // Commenting out the first template per case, because -//Visual Studio doesn't like this instantiaion pattern (error C2910). -//template<> +// Visual Studio doesn't like this instantiaion pattern (error C2910). +// template<> template struct IsVector > { const static bool value = true; }; -//template<> +// template<> template struct IsVector > { const static bool value = true; }; -//template<> +// template<> template struct IsVector > { const static bool value = true; }; -//template<> +// template<> template struct IsVector > { const static bool value = true; }; -//template<> +// template<> template struct IsVector > { const static bool value = true; }; -//template<> +// template<> template struct IsVector > { @@ -84,7 +84,7 @@ struct IsVector > // I'm not so sure about this one. An SpSubview object can be a row or column, // but it can also be a matrix subview. -//template<> +// template<> template struct IsVector > { diff --git a/src/mlpack/core/util/backtrace.cpp b/src/mlpack/core/util/backtrace.cpp index 6e5b2228fa..307dea7dd4 100644 --- a/src/mlpack/core/util/backtrace.cpp +++ b/src/mlpack/core/util/backtrace.cpp @@ -94,10 +94,10 @@ void Backtrace::GetAddress(int maxDepth) { Dl_info addressHandler; - //No backtrace will be printed if no compile flags: -g -rdynamic + // No backtrace will be printed if no compile flags: -g -rdynamic if (TRACE_CONDITION_1) { - return ; + return; } frame.address = addressHandler.dli_saddr; @@ -130,13 +130,13 @@ void Backtrace::DecodeAddress(long addr) return; } - bfd_check_format(abfd,bfd_object); + bfd_check_format(abfd, bfd_object); unsigned storage_needed = bfd_get_symtab_upper_bound(abfd); syms = (asymbol **) malloc(storage_needed); text = bfd_get_section_by_name(abfd, ".text"); - } + } long offset = addr - text->vma; diff --git a/src/mlpack/core/util/backtrace.hpp b/src/mlpack/core/util/backtrace.hpp index 51acdfa106..934a4a1cb8 100644 --- a/src/mlpack/core/util/backtrace.hpp +++ b/src/mlpack/core/util/backtrace.hpp @@ -91,6 +91,6 @@ class Backtrace static std::vector stack; }; -}; //namespace mlpack +}; // namespace mlpack #endif diff --git a/src/mlpack/core/util/log.hpp b/src/mlpack/core/util/log.hpp index 87cdd55372..86e089696d 100644 --- a/src/mlpack/core/util/log.hpp +++ b/src/mlpack/core/util/log.hpp @@ -93,6 +93,6 @@ class Log static std::ostream& cout; }; -}; //namespace mlpack +}; // namespace mlpack #endif diff --git a/src/mlpack/core/util/prefixedoutstream.cpp b/src/mlpack/core/util/prefixedoutstream.cpp index 0db096b77d..8393ca0996 100644 --- a/src/mlpack/core/util/prefixedoutstream.cpp +++ b/src/mlpack/core/util/prefixedoutstream.cpp @@ -36,7 +36,7 @@ PrefixedOutStream& PrefixedOutStream::operator<<(short val) PrefixedOutStream& PrefixedOutStream::operator<<(unsigned short val) { - BaseLogic(val); + BaseLogic(val); return *this; } diff --git a/src/mlpack/core/util/timers.hpp b/src/mlpack/core/util/timers.hpp index 71f5c195d2..021ed56b70 100644 --- a/src/mlpack/core/util/timers.hpp +++ b/src/mlpack/core/util/timers.hpp @@ -18,7 +18,7 @@ #include // chrono library for cross platform timer calculation #if defined(_WIN32) - // uint64_t isn't defined on every windows. + // uint64_t isn't defined on every windows. #if !defined(HAVE_UINT64_T) #if SIZEOF_UNSIGNED_LONG == 8 typedef unsigned long uint64_t; From 1f0ff24740a4a658e2a65b2255905ce3225e2061 Mon Sep 17 00:00:00 2001 From: Abhinav Moudgil Date: Thu, 25 May 2017 19:25:00 +0530 Subject: [PATCH 38/84] Style fix --- src/mlpack/core/optimizers/sdp/lrsdp_function_impl.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/mlpack/core/optimizers/sdp/lrsdp_function_impl.hpp b/src/mlpack/core/optimizers/sdp/lrsdp_function_impl.hpp index d3aaf8aefd..7db085f50e 100644 --- a/src/mlpack/core/optimizers/sdp/lrsdp_function_impl.hpp +++ b/src/mlpack/core/optimizers/sdp/lrsdp_function_impl.hpp @@ -55,8 +55,8 @@ template void LRSDPFunction::Gradient(const arma::mat& /* coordinates */, arma::mat& /* gradient */) const { - Log::Fatal << "LRSDPFunction::Gradient() not implemented for arbitrary - optimizers!" << std::endl; + Log::Fatal << "LRSDPFunction::Gradient() not implemented for arbitrary " + << "optimizers!" << std::endl; } template From 6432c997cf3f4a2a877621e033172313c00d409f Mon Sep 17 00:00:00 2001 From: Abhinav Moudgil Date: Thu, 25 May 2017 21:47:01 +0530 Subject: [PATCH 39/84] cpplint style fixes --- src/mlpack/core/data/load_arff_impl.hpp | 2 +- src/mlpack/core/dists/discrete_distribution.cpp | 4 ++-- src/mlpack/core/dists/laplace_distribution.cpp | 2 +- .../core/optimizers/minibatch_sgd/decay_policies/no_decay.hpp | 4 ++-- .../core/optimizers/minibatch_sgd/minibatch_sgd_impl.hpp | 2 +- src/mlpack/core/optimizers/sgdr/cyclical_decay.hpp | 2 +- src/mlpack/core/optimizers/sgdr/sgdr.hpp | 2 +- src/mlpack/core/optimizers/sgdr/snapshot_ensembles.hpp | 2 +- src/mlpack/core/optimizers/sgdr/snapshot_sgdr.hpp | 2 +- src/mlpack/core/tree/spill_tree/spill_tree_impl.hpp | 2 +- 10 files changed, 12 insertions(+), 12 deletions(-) diff --git a/src/mlpack/core/data/load_arff_impl.hpp b/src/mlpack/core/data/load_arff_impl.hpp index 34a6d88410..8667055082 100644 --- a/src/mlpack/core/data/load_arff_impl.hpp +++ b/src/mlpack/core/data/load_arff_impl.hpp @@ -182,7 +182,7 @@ void LoadARFF(const std::string& filename, // Strip spaces before mapping. std::string token = *it; boost::trim(token); - // We load transposed. + // We load transposed. matrix(col, row) = info.template MapString(token, col); } else if (info.Type(col) == Datatype::numeric) diff --git a/src/mlpack/core/dists/discrete_distribution.cpp b/src/mlpack/core/dists/discrete_distribution.cpp index dc63aec6f0..c2f7e554d1 100644 --- a/src/mlpack/core/dists/discrete_distribution.cpp +++ b/src/mlpack/core/dists/discrete_distribution.cpp @@ -128,8 +128,8 @@ void DiscreteDistribution::Train(const arma::mat& observations, { for (size_t i = 0; i < dimensions; i++) { - // Add the probability of each observation. The addition of 0.5 - // to the observation is to turn the default flooring operation + // Add the probability of each observation. The addition of 0.5 + // to the observation is to turn the default flooring operation // of the size_t cast into a rounding observation. const size_t obs = size_t(observations(i, r) + 0.5); diff --git a/src/mlpack/core/dists/laplace_distribution.cpp b/src/mlpack/core/dists/laplace_distribution.cpp index e83a9fd9dc..75a069bbe7 100644 --- a/src/mlpack/core/dists/laplace_distribution.cpp +++ b/src/mlpack/core/dists/laplace_distribution.cpp @@ -21,7 +21,7 @@ using namespace mlpack::distribution; */ double LaplaceDistribution::LogProbability(const arma::vec& observation) const { - // Evaluate the PDF of the Laplace distribution to determine + // Evaluate the PDF of the Laplace distribution to determine // the log probability. return -log(2. * scale) - arma::norm(observation - mean, 2) / scale; } diff --git a/src/mlpack/core/optimizers/minibatch_sgd/decay_policies/no_decay.hpp b/src/mlpack/core/optimizers/minibatch_sgd/decay_policies/no_decay.hpp index 3c96a37fe8..44f77a8ee5 100644 --- a/src/mlpack/core/optimizers/minibatch_sgd/decay_policies/no_decay.hpp +++ b/src/mlpack/core/optimizers/minibatch_sgd/decay_policies/no_decay.hpp @@ -40,11 +40,11 @@ class NoDecay double& /* stepSize */, const arma::mat& /* gradient */) { - // Nothing to do here. + // Nothing to do here. } }; } // namespace optimization } // namespace mlpack -#endif // MLPACK_CORE_OPTIMIZERS_MINIBATCH_SGD_DECAY_POLICIES_NO_DECAY_HPP \ No newline at end of file +#endif // MLPACK_CORE_OPTIMIZERS_MINIBATCH_SGD_DECAY_POLICIES_NO_DECAY_HPP diff --git a/src/mlpack/core/optimizers/minibatch_sgd/minibatch_sgd_impl.hpp b/src/mlpack/core/optimizers/minibatch_sgd/minibatch_sgd_impl.hpp index 13ea57d1f2..420e3b13af 100644 --- a/src/mlpack/core/optimizers/minibatch_sgd/minibatch_sgd_impl.hpp +++ b/src/mlpack/core/optimizers/minibatch_sgd/minibatch_sgd_impl.hpp @@ -80,7 +80,7 @@ double MiniBatchSGDType< overallObjective += function.Evaluate(iterate, i); // Initialize the update policy. - updatePolicy.Initialize(iterate.n_rows,iterate.n_cols); + updatePolicy.Initialize(iterate.n_rows, iterate.n_cols); // Now iterate! arma::mat gradient(iterate.n_rows, iterate.n_cols); diff --git a/src/mlpack/core/optimizers/sgdr/cyclical_decay.hpp b/src/mlpack/core/optimizers/sgdr/cyclical_decay.hpp index 67e0b14055..cd338fb0fe 100644 --- a/src/mlpack/core/optimizers/sgdr/cyclical_decay.hpp +++ b/src/mlpack/core/optimizers/sgdr/cyclical_decay.hpp @@ -139,4 +139,4 @@ class CyclicalDecay } // namespace optimization } // namespace mlpack -#endif // MLPACK_CORE_OPTIMIZERS_SGDR_CYCLICAL_DECAY_HPP \ No newline at end of file +#endif // MLPACK_CORE_OPTIMIZERS_SGDR_CYCLICAL_DECAY_HPP diff --git a/src/mlpack/core/optimizers/sgdr/sgdr.hpp b/src/mlpack/core/optimizers/sgdr/sgdr.hpp index 50070a56b7..707565ab68 100644 --- a/src/mlpack/core/optimizers/sgdr/sgdr.hpp +++ b/src/mlpack/core/optimizers/sgdr/sgdr.hpp @@ -76,7 +76,7 @@ class SGDR * @param updatePolicy Instantiated update policy used to adjust the given * parameters. */ - SGDR(DecomposableFunctionType& function, + SGDR(DecomposableFunctionType& function, const size_t epochRestart = 50, const double multFactor = 2.0, const size_t batchSize = 1000, diff --git a/src/mlpack/core/optimizers/sgdr/snapshot_ensembles.hpp b/src/mlpack/core/optimizers/sgdr/snapshot_ensembles.hpp index a3c9052a9a..83f3dca542 100644 --- a/src/mlpack/core/optimizers/sgdr/snapshot_ensembles.hpp +++ b/src/mlpack/core/optimizers/sgdr/snapshot_ensembles.hpp @@ -176,4 +176,4 @@ class SnapshotEnsembles } // namespace optimization } // namespace mlpack -#endif // MLPACK_CORE_OPTIMIZERS_SGDR_CYCLICAL_DECAY_HPP \ No newline at end of file +#endif // MLPACK_CORE_OPTIMIZERS_SGDR_CYCLICAL_DECAY_HPP diff --git a/src/mlpack/core/optimizers/sgdr/snapshot_sgdr.hpp b/src/mlpack/core/optimizers/sgdr/snapshot_sgdr.hpp index 7d2030123e..12cd7a55da 100644 --- a/src/mlpack/core/optimizers/sgdr/snapshot_sgdr.hpp +++ b/src/mlpack/core/optimizers/sgdr/snapshot_sgdr.hpp @@ -91,7 +91,7 @@ class SnapshotSGDR * @param updatePolicy Instantiated update policy used to adjust the given * parameters. */ - SnapshotSGDR(DecomposableFunctionType& function, + SnapshotSGDR(DecomposableFunctionType& function, const size_t epochRestart = 50, const double multFactor = 2.0, const size_t batchSize = 1000, diff --git a/src/mlpack/core/tree/spill_tree/spill_tree_impl.hpp b/src/mlpack/core/tree/spill_tree/spill_tree_impl.hpp index bdc9970123..a53f3c3de7 100644 --- a/src/mlpack/core/tree/spill_tree/spill_tree_impl.hpp +++ b/src/mlpack/core/tree/spill_tree/spill_tree_impl.hpp @@ -230,7 +230,7 @@ SpillTree(SpillTree&& other) : other.dataset = NULL; other.localDataset = false; - //Set new parent. + // Set new parent. if (left) left->parent = this; if (right) From b75fb4811d7c20601c69c3508313e8dcccda850b Mon Sep 17 00:00:00 2001 From: Abhinav Moudgil Date: Thu, 25 May 2017 23:05:31 +0530 Subject: [PATCH 40/84] Lines should be <= 80 characters long --- .../core/dists/discrete_distribution.hpp | 12 +++++---- .../optimizers/sdp/lrsdp_function_impl.hpp | 20 +++++++------- .../tree/binary_space_tree/ub_tree_split.hpp | 7 ++--- src/mlpack/core/tree/cellbound_impl.hpp | 22 +++++++++++----- .../cover_tree/dual_tree_traverser_impl.hpp | 3 ++- .../core/tree/hollow_ball_bound_impl.hpp | 3 ++- src/mlpack/core/tree/hrectbound_impl.hpp | 26 +++++++++++++------ .../tree/octree/dual_tree_traverser_impl.hpp | 3 ++- .../minimal_splits_number_sweep_impl.hpp | 5 ++-- .../tree/rectangle_tree/r_star_tree_split.hpp | 5 +++- .../rectangle_tree/r_star_tree_split_impl.hpp | 4 ++- .../tree/rectangle_tree/rectangle_tree.hpp | 3 ++- .../rectangle_tree/rectangle_tree_impl.hpp | 3 ++- .../tree/rectangle_tree/x_tree_split_impl.hpp | 16 +++++++----- 14 files changed, 84 insertions(+), 48 deletions(-) diff --git a/src/mlpack/core/dists/discrete_distribution.hpp b/src/mlpack/core/dists/discrete_distribution.hpp index 523765113e..45d01865bb 100644 --- a/src/mlpack/core/dists/discrete_distribution.hpp +++ b/src/mlpack/core/dists/discrete_distribution.hpp @@ -129,9 +129,10 @@ class DiscreteDistribution // Ensure the observation has the same dimension with the probabilities if (observation.n_elem != probabilities.size()) { - Log::Debug << "the obversation must has the same dimension with the probabilities" - << "the observation's dimension is" << observation.n_elem << "but the dimension of " - << "probabilities is" << probabilities.size() << std::endl; + Log::Debug << "the obversation must has the same dimension with the " + << "probabilities the observation's dimension is " + << observation.n_elem << " but the dimension of probabilities is " + << probabilities.size() << std::endl; return probability; } for (size_t dimension = 0; dimension < observation.n_elem; dimension++) @@ -144,8 +145,9 @@ class DiscreteDistribution if (obs >= probabilities[dimension].n_elem) { Log::Debug << "DiscreteDistribution::Probability(): received observation " - << obs << "; observation must be in [0, " << probabilities[dimension].n_elem - << "] for this distribution." << std::endl; + << obs << "; observation must be in [0, " + << probabilities[dimension].n_elem << "] for this distribution." + << std::endl; } probability *= probabilities[dimension][obs]; } diff --git a/src/mlpack/core/optimizers/sdp/lrsdp_function_impl.hpp b/src/mlpack/core/optimizers/sdp/lrsdp_function_impl.hpp index 7db085f50e..7dbd8cf215 100644 --- a/src/mlpack/core/optimizers/sdp/lrsdp_function_impl.hpp +++ b/src/mlpack/core/optimizers/sdp/lrsdp_function_impl.hpp @@ -72,12 +72,13 @@ double LRSDPFunction::EvaluateConstraint( } template -void LRSDPFunction::GradientConstraint(const size_t /* index */, - const arma::mat& /* coordinates */, - arma::mat& /* gradient */) const +void LRSDPFunction::GradientConstraint( + const size_t /* index */, + const arma::mat& /* coordinates */, + arma::mat& /* gradient */) const { - Log::Fatal << "LRSDPFunction::GradientConstraint() not implemented for arbitrary " - << "optimizers!" << std::endl; + Log::Fatal << "LRSDPFunction::GradientConstraint() not implemented " + << "for arbitrary optimizers!" << std::endl; } //! Utility function for calculating part of the objective when AugLagrangian is @@ -145,10 +146,11 @@ EvaluateImpl(const LRSDPFunction& function, double objective = accu(function.SDP().C() % rrt); // Now each constraint. - UpdateObjective(objective, rrt, function.SDP().SparseA(), function.SDP().SparseB(), - lambda, 0, sigma); - UpdateObjective(objective, rrt, function.SDP().DenseA(), function.SDP().DenseB(), lambda, - function.SDP().NumSparseConstraints(), sigma); + UpdateObjective(objective, rrt, function.SDP().SparseA(), + function.SDP().SparseB(), lambda, 0, sigma); + UpdateObjective(objective, rrt, function.SDP().DenseA(), + function.SDP().DenseB(), lambda, function.SDP().NumSparseConstraints(), + sigma); return objective; } diff --git a/src/mlpack/core/tree/binary_space_tree/ub_tree_split.hpp b/src/mlpack/core/tree/binary_space_tree/ub_tree_split.hpp index 150cbd27a1..5d0f755954 100644 --- a/src/mlpack/core/tree/binary_space_tree/ub_tree_split.hpp +++ b/src/mlpack/core/tree/binary_space_tree/ub_tree_split.hpp @@ -30,9 +30,10 @@ class UBTreeSplit { public: //! The type of an address element. - typedef typename std::conditional::type AddressElemType; + typedef typename std::conditional< + sizeof(typename MatType::elem_type) * CHAR_BIT <= 32, + uint32_t, + uint64_t>::type AddressElemType; //! An information about the partition. struct SplitInfo diff --git a/src/mlpack/core/tree/cellbound_impl.hpp b/src/mlpack/core/tree/cellbound_impl.hpp index a597cf25b7..e117a5e28a 100644 --- a/src/mlpack/core/tree/cellbound_impl.hpp +++ b/src/mlpack/core/tree/cellbound_impl.hpp @@ -82,7 +82,9 @@ inline CellBound::CellBound( * Same as the copy constructor. */ template -inline CellBound& CellBound::operator=( +inline CellBound< + MetricType, + ElemType>& CellBound::operator=( const CellBound& other) { if (dim != other.Dim()) @@ -486,9 +488,10 @@ inline ElemType CellBound::MinDistance( lower = loBound(d, i) - point[d]; higher = point[d] - hiBound(d, i); - // Since only one of 'lower' or 'higher' is negative, if we add each's - // absolute value to itself and then sum those two, our result is the - // nonnegative half of the equation times two; then we raise to power Power. + // Since only one of 'lower' or 'higher' is negative, if we add + // each's absolute value to itself and then sum those two, our + // result is the non negative half of the equation times two; + // then we raise to power Power. if (MetricType::Power == 1) sum += lower + std::fabs(lower) + higher + std::fabs(higher); else if (MetricType::Power == 2) @@ -864,7 +867,9 @@ CellBound::RangeDistance( */ template template -inline CellBound& CellBound::operator|=( +inline CellBound< + MetricType, + ElemType>& CellBound::operator|=( const MatType& data) { Log::Assert(data.n_rows == dim); @@ -893,7 +898,9 @@ inline CellBound& CellBound::operato * Expands this region to encompass another bound. */ template -inline CellBound& CellBound::operator|=( +inline CellBound< + MetricType, + ElemType>& CellBound::operator|=( const CellBound& other) { assert(other.dim == dim); @@ -930,7 +937,8 @@ inline CellBound& CellBound::operato */ template template -inline bool CellBound::Contains(const VecType& point) const +inline bool CellBound::Contains( + const VecType& point) const { for (size_t i = 0; i < point.n_elem; i++) { diff --git a/src/mlpack/core/tree/cover_tree/dual_tree_traverser_impl.hpp b/src/mlpack/core/tree/cover_tree/dual_tree_traverser_impl.hpp index 64dadf3246..692ec9d59e 100644 --- a/src/mlpack/core/tree/cover_tree/dual_tree_traverser_impl.hpp +++ b/src/mlpack/core/tree/cover_tree/dual_tree_traverser_impl.hpp @@ -297,7 +297,8 @@ DualTreeTraverser::ReferenceRecursion( break; // Get a reference to the current largest scale. - std::vector& scaleVector = (*referenceMap.rbegin()).second; + std::vector& scaleVector = + (*referenceMap.rbegin()).second; // Before traversing all the points in this scale, sort by score. std::sort(scaleVector.begin(), scaleVector.end()); diff --git a/src/mlpack/core/tree/hollow_ball_bound_impl.hpp b/src/mlpack/core/tree/hollow_ball_bound_impl.hpp index fa0a8b2b82..182097ff4e 100644 --- a/src/mlpack/core/tree/hollow_ball_bound_impl.hpp +++ b/src/mlpack/core/tree/hollow_ball_bound_impl.hpp @@ -33,7 +33,8 @@ HollowBallBound::HollowBallBound() : * @param dimension Dimensionality of ball bound. */ template -HollowBallBound::HollowBallBound(const size_t dimension) : +HollowBallBound:: +HollowBallBound(const size_t dimension) : radii(std::numeric_limits::lowest(), std::numeric_limits::lowest()), center(dimension), diff --git a/src/mlpack/core/tree/hrectbound_impl.hpp b/src/mlpack/core/tree/hrectbound_impl.hpp index 69e4b97374..0ee0f5a9fe 100644 --- a/src/mlpack/core/tree/hrectbound_impl.hpp +++ b/src/mlpack/core/tree/hrectbound_impl.hpp @@ -60,8 +60,10 @@ inline HRectBound::HRectBound( * Same as the copy constructor. */ template -inline HRectBound& HRectBound::operator=( - const HRectBound& other) +inline HRectBound< + MetricType, + ElemType>& HRectBound::operator=(const HRectBound& other) { if (dim != other.Dim()) { @@ -208,7 +210,8 @@ inline ElemType HRectBound::MinDistance( else { if (MetricType::TakeRoot) - return (ElemType) pow((double) sum, 1.0 / (double) MetricType::Power) / 2.0; + return (ElemType) pow((double) sum, + 1.0 / (double) MetricType::Power) / 2.0; else return sum / pow(2.0, MetricType::Power); } @@ -268,7 +271,8 @@ ElemType HRectBound::MinDistance(const HRectBound& other) else { if (MetricType::TakeRoot) - return (ElemType) pow((double) sum, 1.0 / (double) MetricType::Power) / 2.0; + return (ElemType) pow((double) sum, + 1.0 / (double) MetricType::Power) / 2.0; else return sum / pow(2.0, MetricType::Power); } @@ -503,7 +507,9 @@ HRectBound::RangeDistance( */ template template -inline HRectBound& HRectBound::operator|=( +inline HRectBound< + MetricType, + ElemType>& HRectBound::operator|=( const MatType& data) { Log::Assert(data.n_rows == dim); @@ -527,7 +533,9 @@ inline HRectBound& HRectBound::opera * Expands this region to encompass another bound. */ template -inline HRectBound& HRectBound::operator|=( +inline HRectBound< + MetricType, + ElemType>& HRectBound::operator|=( const HRectBound& other) { assert(other.dim == dim); @@ -549,7 +557,8 @@ inline HRectBound& HRectBound::opera */ template template -inline bool HRectBound::Contains(const VecType& point) const +inline bool HRectBound::Contains( + const VecType& point) const { for (size_t i = 0; i < point.n_elem; i++) { @@ -572,7 +581,8 @@ inline bool HRectBound::Contains( const math::RangeType& r_a = bounds[i]; const math::RangeType& r_b = bound.bounds[i]; - if (r_a.Hi() <= r_b.Lo() || r_a.Lo() >= r_b.Hi()) // If a does not overlap b at all. + // If a does not overlap b at all. + if (r_a.Hi() <= r_b.Lo() || r_a.Lo() >= r_b.Hi()) return false; } diff --git a/src/mlpack/core/tree/octree/dual_tree_traverser_impl.hpp b/src/mlpack/core/tree/octree/dual_tree_traverser_impl.hpp index 7bf14c23b5..81eeec3151 100644 --- a/src/mlpack/core/tree/octree/dual_tree_traverser_impl.hpp +++ b/src/mlpack/core/tree/octree/dual_tree_traverser_impl.hpp @@ -135,7 +135,8 @@ void Octree::DualTreeTraverser:: { if (scores[scoreOrder[i]] == DBL_MAX) { - // We don't need to check any more---all children past here are pruned. + // We don't need to check any more + // All children past here are pruned. numPrunes += scoreOrder.n_elem - i; break; } diff --git a/src/mlpack/core/tree/rectangle_tree/minimal_splits_number_sweep_impl.hpp b/src/mlpack/core/tree/rectangle_tree/minimal_splits_number_sweep_impl.hpp index c0a57ee1e7..9d3e8c28f6 100644 --- a/src/mlpack/core/tree/rectangle_tree/minimal_splits_number_sweep_impl.hpp +++ b/src/mlpack/core/tree/rectangle_tree/minimal_splits_number_sweep_impl.hpp @@ -70,8 +70,9 @@ size_t MinimalSplitsNumberSweep::SweepNonLeafNode( } // Check if the split is possible. - if (numTreeOneChildren <= node->MaxNumChildren() && numTreeOneChildren > 0 && - numTreeTwoChildren <= node->MaxNumChildren() && numTreeTwoChildren > 0) + if (numTreeOneChildren <= node->MaxNumChildren() && + numTreeOneChildren > 0 && numTreeTwoChildren <= node->MaxNumChildren() + && numTreeTwoChildren > 0) { // Evaluate the cost using the number of splits and balancing. size_t balance; diff --git a/src/mlpack/core/tree/rectangle_tree/r_star_tree_split.hpp b/src/mlpack/core/tree/rectangle_tree/r_star_tree_split.hpp index 40a8dd0a52..300d829ef1 100644 --- a/src/mlpack/core/tree/rectangle_tree/r_star_tree_split.hpp +++ b/src/mlpack/core/tree/rectangle_tree/r_star_tree_split.hpp @@ -52,7 +52,10 @@ class RStarTreeSplit * Given a node, return the best dimension and the best index to split on. */ template - static void PickLeafSplit(TreeType* tree, size_t& bestAxis, size_t& bestIndex); + static void PickLeafSplit( + TreeType* tree, + size_t& bestAxis, + size_t& bestIndex); private: /** diff --git a/src/mlpack/core/tree/rectangle_tree/r_star_tree_split_impl.hpp b/src/mlpack/core/tree/rectangle_tree/r_star_tree_split_impl.hpp index d3d3a92514..6452cb92b2 100644 --- a/src/mlpack/core/tree/rectangle_tree/r_star_tree_split_impl.hpp +++ b/src/mlpack/core/tree/rectangle_tree/r_star_tree_split_impl.hpp @@ -268,7 +268,9 @@ void RStarTreeSplit::SplitLeafNode(TreeType *tree, std::vector& relevels) * higher up the tree because they were already updated if necessary. */ template -bool RStarTreeSplit::SplitNonLeafNode(TreeType *tree, std::vector& relevels) +bool RStarTreeSplit::SplitNonLeafNode( + TreeType *tree, + std::vector& relevels) { // Convenience typedef. typedef typename TreeType::ElemType ElemType; diff --git a/src/mlpack/core/tree/rectangle_tree/rectangle_tree.hpp b/src/mlpack/core/tree/rectangle_tree/rectangle_tree.hpp index 1d78c666e6..f965a356bb 100644 --- a/src/mlpack/core/tree/rectangle_tree/rectangle_tree.hpp +++ b/src/mlpack/core/tree/rectangle_tree/rectangle_tree.hpp @@ -49,7 +49,8 @@ template class AuxiliaryInformationType = NoAuxiliaryInformation> + template + class AuxiliaryInformationType = NoAuxiliaryInformation> class RectangleTree { // The metric *must* be the euclidean distance. diff --git a/src/mlpack/core/tree/rectangle_tree/rectangle_tree_impl.hpp b/src/mlpack/core/tree/rectangle_tree/rectangle_tree_impl.hpp index becd85ad8f..1884f77562 100644 --- a/src/mlpack/core/tree/rectangle_tree/rectangle_tree_impl.hpp +++ b/src/mlpack/core/tree/rectangle_tree/rectangle_tree_impl.hpp @@ -1096,7 +1096,8 @@ void RectangleTreeCondenseTree(point, relevels, usePoint); else if (!usePoint && - (ShrinkBoundForBound(bound) || auxiliaryInfo.UpdateAuxiliaryInfo(this)) && + (ShrinkBoundForBound(bound) || + auxiliaryInfo.UpdateAuxiliaryInfo(this)) && parent != NULL) parent->CondenseTree(point, relevels, usePoint); } diff --git a/src/mlpack/core/tree/rectangle_tree/x_tree_split_impl.hpp b/src/mlpack/core/tree/rectangle_tree/x_tree_split_impl.hpp index bcdeb37c2f..3bf23c8cf3 100644 --- a/src/mlpack/core/tree/rectangle_tree/x_tree_split_impl.hpp +++ b/src/mlpack/core/tree/rectangle_tree/x_tree_split_impl.hpp @@ -492,7 +492,8 @@ bool XTreeSplit::SplitNonLeafNode(TreeType *tree, std::vector& relevels) sorted2[i].second = sorted[i].second; } } - std::sort(sorted2.begin(), sorted2.end(), PairComp); + std::sort(sorted2.begin(), sorted2.end(), + PairComp); tree->numDescendants = 0; tree->bound.Clear(); @@ -520,7 +521,7 @@ bool XTreeSplit::SplitNonLeafNode(TreeType *tree, std::vector& relevels) { // We make the root a supernode instead. tree->Parent()->MaxNumChildren() = tree->MaxNumChildren() + - tree->AuxiliaryInfo().NormalNodeMaxNumChildren(); + tree->AuxiliaryInfo().NormalNodeMaxNumChildren(); tree->Parent()->children.resize(tree->Parent()->MaxNumChildren() + 1); tree->Parent()->NumChildren() = tree->NumChildren(); for (size_t i = 0; i < numChildren; ++i) @@ -538,7 +539,7 @@ bool XTreeSplit::SplitNonLeafNode(TreeType *tree, std::vector& relevels) // If we don't have to worry about the root, we just enlarge this node. tree->MaxNumChildren() += - tree->AuxiliaryInfo().NormalNodeMaxNumChildren(); + tree->AuxiliaryInfo().NormalNodeMaxNumChildren(); tree->children.resize(tree->MaxNumChildren() + 1); tree->numChildren = numChildren; for (size_t i = 0; i < numChildren; i++) @@ -627,8 +628,8 @@ bool XTreeSplit::SplitNonLeafNode(TreeType *tree, std::vector& relevels) } // If the split was not good enough, then we try the minimal overlap split. - // If that fails, we create a "super node" (more accurately we resize this one - // to make it a super node). + // If that fails, we create a "super node" (more accurately we resize this + // one to make it a super node). if (useMinOverlapSplit) { // If there is a dimension that might work, try that. @@ -652,7 +653,8 @@ bool XTreeSplit::SplitNonLeafNode(TreeType *tree, std::vector& relevels) sorted2[i].second = sorted[i].second; } } - std::sort(sorted2.begin(), sorted2.end(), PairComp); + std::sort(sorted2.begin(), sorted2.end(), + PairComp); for (size_t i = 0; i < numChildren; i++) { @@ -666,7 +668,7 @@ bool XTreeSplit::SplitNonLeafNode(TreeType *tree, std::vector& relevels) { // Make this node a supernode. tree->MaxNumChildren() += - tree->AuxiliaryInfo().NormalNodeMaxNumChildren(); + tree->AuxiliaryInfo().NormalNodeMaxNumChildren(); tree->children.resize(tree->MaxNumChildren() + 1); tree->numChildren = numChildren; for (size_t i = 0; i < numChildren; i++) From e4625623785f5605e472afe46825cf80e63f3d0c Mon Sep 17 00:00:00 2001 From: Abhinav Moudgil Date: Thu, 25 May 2017 23:14:43 +0530 Subject: [PATCH 41/84] Remove trailing spaces --- src/mlpack/core/dists/discrete_distribution.hpp | 10 +++++----- src/mlpack/core/optimizers/sdp/lrsdp_function_impl.hpp | 2 +- .../core/tree/rectangle_tree/rectangle_tree_impl.hpp | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/mlpack/core/dists/discrete_distribution.hpp b/src/mlpack/core/dists/discrete_distribution.hpp index 45d01865bb..7b770928ce 100644 --- a/src/mlpack/core/dists/discrete_distribution.hpp +++ b/src/mlpack/core/dists/discrete_distribution.hpp @@ -129,9 +129,9 @@ class DiscreteDistribution // Ensure the observation has the same dimension with the probabilities if (observation.n_elem != probabilities.size()) { - Log::Debug << "the obversation must has the same dimension with the " - << "probabilities the observation's dimension is " - << observation.n_elem << " but the dimension of probabilities is " + Log::Debug << "the obversation must has the same dimension with the " + << "probabilities the observation's dimension is " + << observation.n_elem << " but the dimension of probabilities is " << probabilities.size() << std::endl; return probability; } @@ -145,8 +145,8 @@ class DiscreteDistribution if (obs >= probabilities[dimension].n_elem) { Log::Debug << "DiscreteDistribution::Probability(): received observation " - << obs << "; observation must be in [0, " - << probabilities[dimension].n_elem << "] for this distribution." + << obs << "; observation must be in [0, " + << probabilities[dimension].n_elem << "] for this distribution." << std::endl; } probability *= probabilities[dimension][obs]; diff --git a/src/mlpack/core/optimizers/sdp/lrsdp_function_impl.hpp b/src/mlpack/core/optimizers/sdp/lrsdp_function_impl.hpp index 7dbd8cf215..30cb237ffe 100644 --- a/src/mlpack/core/optimizers/sdp/lrsdp_function_impl.hpp +++ b/src/mlpack/core/optimizers/sdp/lrsdp_function_impl.hpp @@ -77,7 +77,7 @@ void LRSDPFunction::GradientConstraint( const arma::mat& /* coordinates */, arma::mat& /* gradient */) const { - Log::Fatal << "LRSDPFunction::GradientConstraint() not implemented " + Log::Fatal << "LRSDPFunction::GradientConstraint() not implemented " << "for arbitrary optimizers!" << std::endl; } diff --git a/src/mlpack/core/tree/rectangle_tree/rectangle_tree_impl.hpp b/src/mlpack/core/tree/rectangle_tree/rectangle_tree_impl.hpp index 1884f77562..1f41931fee 100644 --- a/src/mlpack/core/tree/rectangle_tree/rectangle_tree_impl.hpp +++ b/src/mlpack/core/tree/rectangle_tree/rectangle_tree_impl.hpp @@ -1096,7 +1096,7 @@ void RectangleTreeCondenseTree(point, relevels, usePoint); else if (!usePoint && - (ShrinkBoundForBound(bound) || + (ShrinkBoundForBound(bound) || auxiliaryInfo.UpdateAuxiliaryInfo(this)) && parent != NULL) parent->CondenseTree(point, relevels, usePoint); From bb165eb1073d6b83d665be80f2d2b3564ab24f4e Mon Sep 17 00:00:00 2001 From: Abhinav Moudgil Date: Thu, 25 May 2017 23:28:34 +0530 Subject: [PATCH 42/84] Lines should be <= 80 characters long --- src/mlpack/core/dists/discrete_distribution.hpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/mlpack/core/dists/discrete_distribution.hpp b/src/mlpack/core/dists/discrete_distribution.hpp index 7b770928ce..d8301e246f 100644 --- a/src/mlpack/core/dists/discrete_distribution.hpp +++ b/src/mlpack/core/dists/discrete_distribution.hpp @@ -144,10 +144,10 @@ class DiscreteDistribution // Ensure that the observation is within the bounds. if (obs >= probabilities[dimension].n_elem) { - Log::Debug << "DiscreteDistribution::Probability(): received observation " - << obs << "; observation must be in [0, " - << probabilities[dimension].n_elem << "] for this distribution." - << std::endl; + Log::Debug << "DiscreteDistribution::Probability(): " + << "received observation " << obs << "; observation must be" + << "in [0, " << probabilities[dimension].n_elem << "] for this " + << "distribution." << std::endl; } probability *= probabilities[dimension][obs]; } From 03165a165b677cdd3ebbb9d9f3d72a046b8d8b5b Mon Sep 17 00:00:00 2001 From: Abhinav Moudgil Date: Thu, 25 May 2017 23:45:34 +0530 Subject: [PATCH 43/84] Use brackets while type casting --- src/mlpack/core/tree/spill_tree/spill_tree_impl.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/mlpack/core/tree/spill_tree/spill_tree_impl.hpp b/src/mlpack/core/tree/spill_tree/spill_tree_impl.hpp index a53f3c3de7..8179edd57e 100644 --- a/src/mlpack/core/tree/spill_tree/spill_tree_impl.hpp +++ b/src/mlpack/core/tree/spill_tree/spill_tree_impl.hpp @@ -671,8 +671,8 @@ bool SpillTree:: } } - const double p1 = double (left + rightFrontier) / points.n_elem; - const double p2 = double (right + leftFrontier) / points.n_elem; + const double p1 = (double) (left + rightFrontier) / points.n_elem; + const double p2 = (double) (right + leftFrontier) / points.n_elem; if ((p1 <= rho || rightFrontier == 0) && (p2 <= rho || leftFrontier == 0)) From 163394fe32c8d3c47588f93ce4348671886dd69f Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Thu, 25 May 2017 15:24:18 -0400 Subject: [PATCH 44/84] Reorder arguments to match with DecisionTree better. --- src/mlpack/methods/decision_tree/all_categorical_split.hpp | 2 +- src/mlpack/methods/decision_tree/all_categorical_split_impl.hpp | 2 +- src/mlpack/methods/decision_tree/best_binary_numeric_split.hpp | 2 +- .../methods/decision_tree/best_binary_numeric_split_impl.hpp | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/mlpack/methods/decision_tree/all_categorical_split.hpp b/src/mlpack/methods/decision_tree/all_categorical_split.hpp index 56c66fa625..b92702ebc9 100644 --- a/src/mlpack/methods/decision_tree/all_categorical_split.hpp +++ b/src/mlpack/methods/decision_tree/all_categorical_split.hpp @@ -59,8 +59,8 @@ class AllCategoricalSplit const size_t numCategories, const arma::Row& labels, const size_t numClasses, - const size_t minimumLeafSize, const WeightVecType& weights, + const size_t minimumLeafSize, arma::Col& classProbabilities, AuxiliarySplitInfo& aux); diff --git a/src/mlpack/methods/decision_tree/all_categorical_split_impl.hpp b/src/mlpack/methods/decision_tree/all_categorical_split_impl.hpp index c22f04c6ea..b15c7381f4 100644 --- a/src/mlpack/methods/decision_tree/all_categorical_split_impl.hpp +++ b/src/mlpack/methods/decision_tree/all_categorical_split_impl.hpp @@ -23,8 +23,8 @@ double AllCategoricalSplit::SplitIfBetter( const size_t numCategories, const arma::Row& labels, const size_t numClasses, - const size_t minimumLeafSize, const WeightVecType& weights, + const size_t minimumLeafSize, arma::Col& classProbabilities, AuxiliarySplitInfo& /* aux */) { diff --git a/src/mlpack/methods/decision_tree/best_binary_numeric_split.hpp b/src/mlpack/methods/decision_tree/best_binary_numeric_split.hpp index b81dc0cbcc..4b0f039fad 100644 --- a/src/mlpack/methods/decision_tree/best_binary_numeric_split.hpp +++ b/src/mlpack/methods/decision_tree/best_binary_numeric_split.hpp @@ -56,8 +56,8 @@ class BestBinaryNumericSplit const VecType& data, const arma::Row& labels, const size_t numClasses, - const size_t minimumLeafSize, const WeightVecType& weights, + const size_t minimumLeafSize, arma::Col& classProbabilities, AuxiliarySplitInfo& aux); diff --git a/src/mlpack/methods/decision_tree/best_binary_numeric_split_impl.hpp b/src/mlpack/methods/decision_tree/best_binary_numeric_split_impl.hpp index 88537c035b..f8a0c95841 100644 --- a/src/mlpack/methods/decision_tree/best_binary_numeric_split_impl.hpp +++ b/src/mlpack/methods/decision_tree/best_binary_numeric_split_impl.hpp @@ -22,8 +22,8 @@ double BestBinaryNumericSplit::SplitIfBetter( const VecType& data, const arma::Row& labels, const size_t numClasses, - const size_t minimumLeafSize, const WeightVecType& weights, + const size_t minimumLeafSize, arma::Col& classProbabilities, AuxiliarySplitInfo& /* aux */) { From f3d706c58d9b264ad3f3ce323416c3b891835de3 Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Thu, 25 May 2017 15:24:29 -0400 Subject: [PATCH 45/84] Refactor for universal references. --- .../methods/decision_tree/decision_tree.hpp | 128 ++++----- .../decision_tree/decision_tree_impl.hpp | 254 ++++++++++++------ 2 files changed, 237 insertions(+), 145 deletions(-) diff --git a/src/mlpack/methods/decision_tree/decision_tree.hpp b/src/mlpack/methods/decision_tree/decision_tree.hpp index b3abc9db13..7258b2cfb0 100644 --- a/src/mlpack/methods/decision_tree/decision_tree.hpp +++ b/src/mlpack/methods/decision_tree/decision_tree.hpp @@ -17,6 +17,7 @@ #include "gini_gain.hpp" #include "best_binary_numeric_split.hpp" #include "all_categorical_split.hpp" +#include namespace mlpack { namespace tree { @@ -94,13 +95,16 @@ class DecisionTree : * @param weights The weight list of given label. * @param minimumLeafSize Minimum number of points in each leaf node. */ - template - DecisionTree(const MatType& data, + template + DecisionTree(MatType&& data, const data::DatasetInfo& datasetInfo, - const arma::Row& labels, + LabelsType&& labels, const size_t numClasses, - const arma::rowvec& weights, - const size_t minimumLeafSize = 10); + WeightsType&& weights, + const size_t minimumLeafSize = 10, + const std::enable_if_t::type>::value>* + = 0); /** * Construct the decision tree on the given data and labels with weights, @@ -114,12 +118,15 @@ class DecisionTree : * @param weights The Weight list of given labels. * @param minimumLeafSize Minimum number of points in each leaf node. */ - template - DecisionTree(const MatType& data, - const arma::Row& labels, + template + DecisionTree(MatType&& data, + LabelsType&& labels, const size_t numClasses, - const arma::rowvec& weights, - const size_t minimumLeafSize = 10); + WeightsType&& weights, + const size_t minimumLeafSize = 10, + const std::enable_if_t::type>::value>* + = 0); /** @@ -178,12 +185,11 @@ class DecisionTree : * @param weights Weights of all the labels * @param minimumLeafSize Minimum number of points in each leaf node. */ - template + template void Train(MatType&& data, const data::DatasetInfo& datasetInfo, LabelsType&& labels, const size_t numClasses, - const arma::rowvec& weights, const size_t minimumLeafSize = 10); /** @@ -198,13 +204,57 @@ class DecisionTree : * @param weights Weights of all the labels * @param minimumLeafSize Minimum number of points in each leaf node. */ - template + template void Train(MatType&& data, LabelsType&& labels, const size_t numClasses, - const arma::rowvec& weights, const size_t minimumLeafSize = 10); + /** + * Train the decision tree on the given weighted data. This will overwrite + * the existing model. The data may have numeric and categorical types, + * specified by the datasetInfo parameter. Setting minimumLeafSize too small + * may cause the tree to overfit, but setting it too large may cause it to + * underfit. + * + * @param data Dataset to train on. + * @param datasetInfo Type information for each dimension. + * @param labels Labels for each training point. + * @param numClasses Number of classes in the dataset. + * @param weights Weights of all the labels + * @param minimumLeafSize Minimum number of points in each leaf node. + */ + template + void Train(MatType&& data, + const data::DatasetInfo& datasetInfo, + LabelsType&& labels, + const size_t numClasses, + WeightsType&& weights, + const size_t minimumLeafSize = 10, + const std::enable_if_t::type>::value>* = 0); + + /** + * Train the decision tree on the given weighted data, assuming that all + * dimensions are numeric. This will overwrite the given model. Setting + * minimumLeafSize too small may cause the tree to overfit, but setting it too + * large may cause it to underfit. + * + * @param data Dataset to train on. + * @param labels Labels for each training point. + * @param numClasses Number of classes in the dataset. + * @param weights Weights of all the labels + * @param minimumLeafSize Minimum number of points in each leaf node. + */ + template + void Train(MatType&& data, + LabelsType&& labels, + const size_t numClasses, + WeightsType&& weights, + const size_t minimumLeafSize = 10, + const std::enable_if_t::type>::value>* = 0); + /** * Classify the given point, using the entire tree. The predicted label is * returned. @@ -311,50 +361,6 @@ class DecisionTree : const size_t numClasses, const WeightsRowType& weights); - /** - * Corresponding to the public constructor, this method is designed for - * avoiding unnecessary copies during training. This constructor is called to - * create children. - * - * @param data Dataset to train on. - * @param begin Index of the starting point in the dataset that belongs to - * this node. - * @param count Number of points in this node. - * @param datasetInfo Type information for each dimension of the dataset. - * @param labels Labels for each training point. - * @param numClasses Number of classes in the dataset. - * @param minimumLeafSize Minimum number of points in each leaf node. - */ - template - DecisionTree(MatType& data, - const size_t begin, - const size_t count, - const data::DatasetInfo& datasetInfo, - arma::Row& labels, - const size_t numClasses, - const size_t minimumLeafSize = 10); - - /** - * Corresponding to the public constructor, this method is designed for - * avoiding unnecessary copies during training. This constructor is called to - * create children. - * - * @param data Dataset to train on. - * @param begin Index of the starting point in the dataset that belongs to - * this node. - * @param count Number of points in this node. - * @param labels Labels for each training point. - * @param numClasses Number of classes in the dataset. - * @param minimumLeafSize Minimum number of points in each leaf node. - */ - template - DecisionTree(MatType& data, - const size_t begin, - const size_t count, - arma::Row& labels, - const size_t numClasses, - const size_t minimumLeafSize = 10); - /** * Corresponding to the public Train() method, this method is designed for * avoiding unnecessary copies during training. This function is called to @@ -369,13 +375,14 @@ class DecisionTree : * @param numClasses Number of classes in the dataset. * @param minimumLeafSize Minimum number of points in each leaf node. */ - template + template void Train(MatType& data, const size_t begin, const size_t count, const data::DatasetInfo& datasetInfo, arma::Row& labels, const size_t numClasses, + arma::rowvec& weights, const size_t minimumLeafSize = 10); /** @@ -391,12 +398,13 @@ class DecisionTree : * @param numClasses Number of classes in the dataset. * @param minimumLeafSize Minimum number of points in each leaf node. */ - template + template void Train(MatType& data, const size_t begin, const size_t count, arma::Row& labels, const size_t numClasses, + arma::rowvec& weights, const size_t minimumLeafSize = 10); }; diff --git a/src/mlpack/methods/decision_tree/decision_tree_impl.hpp b/src/mlpack/methods/decision_tree/decision_tree_impl.hpp index a3a9032441..7c9ef7ecc0 100644 --- a/src/mlpack/methods/decision_tree/decision_tree_impl.hpp +++ b/src/mlpack/methods/decision_tree/decision_tree_impl.hpp @@ -32,14 +32,17 @@ DecisionTree::type TrueMatType; typedef typename std::remove_reference::type TrueLabelsType; + TrueMatType tmpData(std::forward(data)); TrueLabelsType tmpLabels(std::forward(labels)); + // Pass off work to the Train() method. - Train(tmpData, 0, tmpData.n_cols, datasetInfo, - tmpLabels, numClasses, minimumLeafSize); + arma::rowvec weights; // Fake weights, not used. + Train(tmpData, 0, tmpData.n_cols, datasetInfo, tmpLabels, numClasses, + weights, minimumLeafSize); } //! Construct and train. @@ -58,106 +61,87 @@ DecisionTree::type TrueMatType; typedef typename std::remove_reference::type TrueLabelsType; TrueMatType tmpData(std::forward(data)); TrueLabelsType tmpLabels(std::forward(labels)); + // Pass off work to the Train() method. - Train(tmpData, 0, tmpData.n_cols, - tmpLabels, numClasses, minimumLeafSize); + arma::rowvec weights; // Fake weights, not used. + Train(tmpData, 0, tmpData.n_cols, tmpLabels, numClasses, weights, + minimumLeafSize); } -//! Construct and train. +//! Construct and train with weights. template class NumericSplitType, template class CategoricalSplitType, typename ElemType, bool NoRecursion> -template +template DecisionTree::DecisionTree(MatType& data, - const size_t begin, - const size_t count, + NoRecursion>::DecisionTree(MatType&& data, const data::DatasetInfo& datasetInfo, - arma::Row& labels, + LabelsType&& labels, const size_t numClasses, - const size_t minimumLeafSize) + WeightsType&& weights, + const size_t minimumLeafSize, + const std::enable_if_t< + arma::is_arma_type< + typename std::remove_reference< + WeightsType>::type>::value>*) { - // Pass to unweighted training function. - arma::rowvec weights; - // Pass off work to the Train() method. - Train(data, begin, count, datasetInfo, labels, numClasses, weights, minimumLeafSize); -} + // Copy or move data. + typedef typename std::remove_reference::type TrueMatType; + typedef typename std::remove_reference::type TrueLabelsType; + typedef typename std::remove_reference::type TrueWeightsType; -//! Construct and train without weight. -template class NumericSplitType, - template class CategoricalSplitType, - typename ElemType, - bool NoRecursion> -template -DecisionTree::DecisionTree(MatType& data, - const size_t begin, - const size_t count, - arma::Row& labels, - const size_t numClasses, - const size_t minimumLeafSize) -{ - // Pass to unweighted training function. - arma::rowvec weights; - // Pass off work to the Train() method. - Train(data, begin, count, labels, numClasses, weights, minimumLeafSize); -} + TrueMatType tmpData(std::forward(data)); + TrueLabelsType tmpLabels(std::forward(labels)); + TrueWeightsType tmpWeights(std::forward(weights)); -//! Construct and train without weight -template class NumericSplitType, - template class CategoricalSplitType, - typename ElemType, - bool NoRecursion> -template -DecisionTree::DecisionTree(const MatType& data, - const data::DatasetInfo& datasetInfo, - const arma::Row& labels, - const size_t numClasses, - const arma::rowvec& weights, - const size_t minimumLeafSize) -{ // Pass off work to the weighted Train() method. - Train(data, datasetInfo, labels, numClasses, weights, minimumLeafSize); + Train(tmpData, 0, tmpData.n_cols, datasetInfo, tmpLabels, numClasses, + tmpWeights, minimumLeafSize); } -//! Construct and train without weight +//! Construct and train with weights. template class NumericSplitType, template class CategoricalSplitType, typename ElemType, bool NoRecursion> -template +template DecisionTree::DecisionTree(const MatType& data, - const arma::Row& labels, + NoRecursion>::DecisionTree(MatType&& data, + LabelsType&& labels, const size_t numClasses, - const arma::rowvec& weights, - const size_t minimumLeafSize) + WeightsType&& weights, + const size_t minimumLeafSize, + const std::enable_if_t< + arma::is_arma_type< + typename std::remove_reference< + WeightsType>::type>::value>*) { + // Copy or move data. + typedef typename std::remove_reference::type TrueMatType; + typedef typename std::remove_reference::type TrueLabelsType; + typedef typename std::remove_reference::type TrueWeightsType; + + TrueMatType tmpData(std::forward(data)); + TrueLabelsType tmpLabels(std::forward(labels)); + TrueWeightsType tmpWeights(std::forward(weights)); + // Pass off work to the weighted Train() method. - Train(data, labels, numClasses, weights, minimumLeafSize); + Train(tmpData, 0, tmpData.n_cols, tmpLabels, numClasses, tmpWeights, + minimumLeafSize); } //! Construct, don't train. @@ -340,13 +324,18 @@ void DecisionTree::type TrueMatType; typedef typename std::remove_reference::type TrueLabelsType; + TrueMatType tmpData(std::forward(data)); TrueLabelsType tmpLabels(std::forward(labels)); + // Pass off work to the Train() method. - Train(tmpData, 0, tmpData.n_cols, datasetInfo, tmpLabels, numClasses, minimumLeafSize); + arma::rowvec weights; // Fake weights, not used. + Train(tmpData, 0, tmpData.n_cols, datasetInfo, tmpLabels, numClasses, + minimumLeafSize); } //! Train on the given data, assuming all dimensions are numeric. @@ -374,14 +363,107 @@ void DecisionTree::type TrueMatType; typedef typename std::remove_reference::type TrueLabelsType; + TrueMatType tmpData(std::forward(data)); TrueLabelsType tmpLabels(std::forward(labels)); + // Pass off work to the Train() method. - Train(tmpData, 0, tmpData.n_cols, - tmpLabels, numClasses, minimumLeafSize); + arma::rowvec weights; // Fake weights, not used. + Train(tmpData, 0, tmpData.n_cols, tmpLabels, numClasses, weights, + minimumLeafSize); +} + +//! Train on the given weighted data. +template class NumericSplitType, + template class CategoricalSplitType, + typename ElemType, + bool NoRecursion> +template +void DecisionTree::Train(MatType&& data, + const data::DatasetInfo& datasetInfo, + LabelsType&& labels, + const size_t numClasses, + WeightsType&& weights, + const size_t minimumLeafSize, + const std::enable_if_t::type>::value>*) +{ + // Sanity check on data. + if (data.n_cols != labels.n_elem) + { + std::ostringstream oss; + oss << "DecisionTree::Train(): number of points (" << data.n_cols << ") " + << "does not match number of labels (" << labels.n_elem << ")!" + << std::endl; + throw std::invalid_argument(oss.str()); + } + + // Copy or move data. + typedef typename std::remove_reference::type TrueMatType; + typedef typename std::remove_reference::type TrueLabelsType; + typedef typename std::remove_reference::type TrueWeightsType; + + TrueMatType tmpData(std::forward(data)); + TrueLabelsType tmpLabels(std::forward(labels)); + TrueWeightsType tmpWeights(std::forward(weights)); + + // Pass off work to the Train() method. + Train(tmpData, 0, tmpData.n_cols, datasetInfo, tmpLabels, numClasses, + tmpWeights, minimumLeafSize); +} + +//! Train on the given weighted data. +template class NumericSplitType, + template class CategoricalSplitType, + typename ElemType, + bool NoRecursion> +template +void DecisionTree::Train(MatType&& data, + LabelsType&& labels, + const size_t numClasses, + WeightsType&& weights, + const size_t minimumLeafSize, + const std::enable_if_t::type>::value>*) +{ + // Sanity check on data. + if (data.n_cols != labels.n_elem) + { + std::ostringstream oss; + oss << "DecisionTree::Train(): number of points (" << data.n_cols << ") " + << "does not match number of labels (" << labels.n_elem << ")!" + << std::endl; + throw std::invalid_argument(oss.str()); + } + + // Copy or move data. + typedef typename std::remove_reference::type TrueMatType; + typedef typename std::remove_reference::type TrueLabelsType; + typedef typename std::remove_reference::type TrueWeightsType; + + TrueMatType tmpData(std::forward(data)); + TrueLabelsType tmpLabels(std::forward(labels)); + TrueWeightsType tmpWeights(std::forward(weights)); + + // Pass off work to the Train() method. + Train(tmpData, 0, tmpData.n_cols, tmpLabels, numClasses, tmpWeights, + minimumLeafSize); } //! Train on the given data. @@ -401,7 +483,7 @@ void DecisionTree& labels, const size_t numClasses, - const arma::rowvec& weights, + arma::rowvec& weights, const size_t minimumLeafSize) { // Clear children if needed. @@ -417,7 +499,7 @@ void DecisionTree( labels.subvec(begin, begin + count - 1), numClasses, - weights.subvec(begin, begin + count - 1)); + UseWeights ? weights.subvec(begin, begin + count - 1) : weights); size_t bestDim = datasetInfo.Dimensionality(); // This means "no split". for (size_t i = 0; i < datasetInfo.Dimensionality(); ++i) { @@ -510,14 +592,15 @@ void DecisionTree(data, currentChildBegin, + children.back()->Train(data, currentChildBegin, currentCol - currentChildBegin, datasetInfo, labels, numClasses, - weights, currentCol - currentChildBegin)); + weights, currentCol - currentChildBegin); else - children.push_back(new DecisionTree(data, currentChildBegin, + children.back()->Train(data, currentChildBegin, currentCol - currentChildBegin, datasetInfo, labels, numClasses, - weights, minimumLeafSize)); + weights, minimumLeafSize); } } else @@ -550,7 +633,7 @@ void DecisionTree& labels, const size_t numClasses, - const arma::rowvec& weights, + arma::rowvec& weights, const size_t minimumLeafSize) { // Clear children if needed. @@ -569,7 +652,7 @@ void DecisionTree( labels.subvec(begin, begin + count - 1), numClasses, - weights.subvec(begin, begin + count - 1)); + UseWeights ? weights.subvec(begin, begin + count - 1) : weights); size_t bestDim = data.n_rows; // This means "no split". for (size_t i = 0; i < data.n_rows; ++i) { @@ -635,14 +718,15 @@ void DecisionTree(data, currentChildBegin, + children.back()->Train(data, currentChildBegin, currentCol - currentChildBegin, - labels, numClasses, weights, currentCol - currentChildBegin)); + labels, numClasses, weights, currentCol - currentChildBegin); else - children.push_back(new DecisionTree(data, currentChildBegin, - currentCol - currentChildBegin, - labels, numClasses, weights, minimumLeafSize)); + children.back()->Train(data, currentChildBegin, + currentCol - currentChildBegin, labels, numClasses, weights, + minimumLeafSize); } } else From b701267e7397e0e72a44c16b35163616ba5f418d Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Thu, 25 May 2017 15:24:38 -0400 Subject: [PATCH 46/84] Fix invalid call to DecisionTree. --- src/mlpack/tests/decision_tree_test.cpp | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/src/mlpack/tests/decision_tree_test.cpp b/src/mlpack/tests/decision_tree_test.cpp index c42589e848..37a3757dee 100644 --- a/src/mlpack/tests/decision_tree_test.cpp +++ b/src/mlpack/tests/decision_tree_test.cpp @@ -376,10 +376,10 @@ BOOST_AUTO_TEST_CASE(BestBinaryNumericSplitSimpleSplitTest) // Call the method to do the splitting. const double bestGain = GiniGain::Evaluate(labels, 2, weights); const double gain = BestBinaryNumericSplit::SplitIfBetter( - bestGain, values, labels, 2, 3, weights, classProbabilities, aux); + bestGain, values, labels, 2, weights, 3, classProbabilities, aux); const double weightedGain = BestBinaryNumericSplit::SplitIfBetter(bestGain, values, - labels, 2, 3, weights, classProbabilities, aux); + labels, 2, weights, 3, classProbabilities, aux); // Make sure that a split was made. BOOST_REQUIRE_GT(gain, bestGain); @@ -413,11 +413,11 @@ BOOST_AUTO_TEST_CASE(BestBinaryNumericSplitMinSamplesTest) // Call the method to do the splitting. const double bestGain = GiniGain::Evaluate(labels, 2, weights); const double gain = BestBinaryNumericSplit::SplitIfBetter( - bestGain, values, labels, 2, 8, weights, classProbabilities, aux); + bestGain, values, labels, 2, weights, 8, classProbabilities, aux); // This should make no difference because it won't split at all. const double weightedGain = BestBinaryNumericSplit::SplitIfBetter(bestGain, values, - labels, 2, 8, weights, classProbabilities, aux); + labels, 2, weights, 8, classProbabilities, aux); // Make sure that no split was made. BOOST_REQUIRE_EQUAL(gain, bestGain); @@ -448,7 +448,7 @@ BOOST_AUTO_TEST_CASE(BestBinaryNumericSplitNoGainTest) // Call the method to do the splitting. const double bestGain = GiniGain::Evaluate(labels, 2, weights); const double gain = BestBinaryNumericSplit::SplitIfBetter( - bestGain, values, labels, 2, 10, weights, classProbabilities, aux); + bestGain, values, labels, 2, weights, 10, classProbabilities, aux); // Make sure there was no split. BOOST_REQUIRE_EQUAL(gain, bestGain); @@ -472,10 +472,10 @@ BOOST_AUTO_TEST_CASE(AllCategoricalSplitSimpleSplitTest) // Call the method to do the splitting. const double bestGain = GiniGain::Evaluate(labels, 3, weights); const double gain = AllCategoricalSplit::SplitIfBetter( - bestGain, values, 4, labels, 3, 3, weights, classProbabilities, aux); + bestGain, values, 4, labels, 3, weights, 3, classProbabilities, aux); const double weightedGain = AllCategoricalSplit::SplitIfBetter(bestGain, values, 4, - labels, 3, 3, weights, classProbabilities, aux); + labels, 3, weights, 3, classProbabilities, aux); // Make sure that a split was made. BOOST_REQUIRE_GT(gain, bestGain); @@ -507,7 +507,7 @@ BOOST_AUTO_TEST_CASE(AllCategoricalSplitMinSamplesTest) // Call the method to do the splitting. const double bestGain = GiniGain::Evaluate(labels, 3, weights); const double gain = AllCategoricalSplit::SplitIfBetter( - bestGain, values, 4, labels, 3, 4, weights, classProbabilities, aux); + bestGain, values, 4, labels, 3, weights, 4, classProbabilities, aux); // Make sure it's not split. BOOST_REQUIRE_EQUAL(gain, bestGain); @@ -539,10 +539,10 @@ BOOST_AUTO_TEST_CASE(AllCategoricalSplitNoGainTest) // Call the method to do the splitting. const double bestGain = GiniGain::Evaluate(labels, 3, weights); const double gain = AllCategoricalSplit::SplitIfBetter( - bestGain, values, 10, labels, 3, 10, weights, classProbabilities, aux); + bestGain, values, 10, labels, 3, weights, 10, classProbabilities, aux); const double weightedGain = AllCategoricalSplit::SplitIfBetter(bestGain, values, 10, - labels, 3, 10, weights, classProbabilities, aux); + labels, 3, weights, 10, classProbabilities, aux); // Make sure that there was no split. BOOST_REQUIRE_EQUAL(gain, bestGain); @@ -703,8 +703,7 @@ BOOST_AUTO_TEST_CASE(SimpleGeneralizationTest) BOOST_FAIL("Cannot load labels for vc2_labels.txt"); // Initialize an all-ones weight matrix. - arma::mat weights = arma::ones>(labels.n_rows, - labels.n_cols); + arma::rowvec weights(labels.n_cols, arma::fill::ones); // Build decision tree. DecisionTree<> d(inputData, labels, 3, 10); // Leaf size of 10. From 4144f66da2d91978e60282746d274c771c6719b2 Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Thu, 25 May 2017 15:51:55 -0400 Subject: [PATCH 47/84] Use correct types in program. --- src/mlpack/methods/decision_tree/decision_tree_main.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/mlpack/methods/decision_tree/decision_tree_main.cpp b/src/mlpack/methods/decision_tree/decision_tree_main.cpp index b2b51a36b1..dc0aab359f 100644 --- a/src/mlpack/methods/decision_tree/decision_tree_main.cpp +++ b/src/mlpack/methods/decision_tree/decision_tree_main.cpp @@ -153,14 +153,14 @@ int main(int argc, char** argv) // Create decision tree with weighted labels. if (CLI::HasParam("weights")) { - const arma::Row weights= std::move(CLI::GetParam>("weights")); - model.tree = DecisionTree<>(dataset, labels.row(0), numClasses, + arma::Row weights = + std::move(CLI::GetParam>("weights")); + model.tree = DecisionTree<>(dataset, labels, numClasses, weights, minLeafSize); } - else { - model.tree = DecisionTree<>(dataset, labels.row(0), numClasses, minLeafSize); + model.tree = DecisionTree<>(dataset, labels, numClasses, minLeafSize); } // Do we need to print training error? From ff6427e3d141891f2da5cc35cf0e62a890f2f0f8 Mon Sep 17 00:00:00 2001 From: Abhinav Moudgil Date: Fri, 26 May 2017 15:55:01 +0530 Subject: [PATCH 48/84] Abbreviate macro names --- .../breadth_first_dual_tree_traverser.hpp | 7 +++---- .../breadth_first_dual_tree_traverser_impl.hpp | 6 +++--- .../hilbert_r_tree_auxiliary_information.hpp | 6 +++--- .../hilbert_r_tree_auxiliary_information_impl.hpp | 6 +++--- .../rectangle_tree/hilbert_r_tree_descent_heuristic.hpp | 6 +++--- .../hilbert_r_tree_descent_heuristic_impl.hpp | 6 +++--- .../r_plus_plus_tree_auxiliary_information.hpp | 6 +++--- .../r_plus_plus_tree_auxiliary_information_impl.hpp | 6 +++--- .../rectangle_tree/r_plus_plus_tree_descent_heuristic.hpp | 6 +++--- .../r_plus_plus_tree_descent_heuristic_impl.hpp | 6 +++--- 10 files changed, 30 insertions(+), 31 deletions(-) diff --git a/src/mlpack/core/tree/binary_space_tree/breadth_first_dual_tree_traverser.hpp b/src/mlpack/core/tree/binary_space_tree/breadth_first_dual_tree_traverser.hpp index 90c361bc97..b313b5a725 100644 --- a/src/mlpack/core/tree/binary_space_tree/breadth_first_dual_tree_traverser.hpp +++ b/src/mlpack/core/tree/binary_space_tree/breadth_first_dual_tree_traverser.hpp @@ -12,8 +12,8 @@ * 3-clause BSD license along with mlpack. If not, see * http://www.opensource.org/licenses/BSD-3-Clause for more information. */ -#ifndef MLPACK_CORE_TREE_BINARY_SPACE_TREE_BREADTH_FIRST_DUAL_TREE_TRAVERSER_HPP -#define MLPACK_CORE_TREE_BINARY_SPACE_TREE_BREADTH_FIRST_DUAL_TREE_TRAVERSER_HPP +#ifndef MLPACK_CORE_TREE_BINARY_SPACE_TREE_BF_DUAL_TREE_TRAVERSER_HPP +#define MLPACK_CORE_TREE_BINARY_SPACE_TREE_BF_DUAL_TREE_TRAVERSER_HPP #include #include @@ -111,5 +111,4 @@ class BinarySpaceTree::Traverse( } // namespace tree } // namespace mlpack -#endif // MLPACK_CORE_TREE_BINARY_SPACE_TREE_BREADTH_FIRST_DUAL_TREE_TRAVERSER_IMPL_HPP +#endif // MLPACK_CORE_TREE_BINARY_SPACE_TREE_BF_DUAL_TREE_TRAVERSER_IMPL_HPP diff --git a/src/mlpack/core/tree/rectangle_tree/hilbert_r_tree_auxiliary_information.hpp b/src/mlpack/core/tree/rectangle_tree/hilbert_r_tree_auxiliary_information.hpp index 6ef9164749..a95b4d8824 100644 --- a/src/mlpack/core/tree/rectangle_tree/hilbert_r_tree_auxiliary_information.hpp +++ b/src/mlpack/core/tree/rectangle_tree/hilbert_r_tree_auxiliary_information.hpp @@ -11,8 +11,8 @@ * 3-clause BSD license along with mlpack. If not, see * http://www.opensource.org/licenses/BSD-3-Clause for more information. */ -#ifndef MLPACK_CORE_TREE_RECTANGLE_TREE_HILBERT_R_TREE_AUXILIARY_INFORMATION_HPP -#define MLPACK_CORE_TREE_RECTANGLE_TREE_HILBERT_R_TREE_AUXILIARY_INFORMATION_HPP +#ifndef MLPACK_CORE_TREE_RECTANGLE_TREE_HR_TREE_AUXILIARY_INFORMATION_HPP +#define MLPACK_CORE_TREE_RECTANGLE_TREE_HR_TREE_AUXILIARY_INFORMATION_HPP namespace mlpack { namespace tree { @@ -147,4 +147,4 @@ class HilbertRTreeAuxiliaryInformation #include "hilbert_r_tree_auxiliary_information_impl.hpp" -#endif // MLPACK_CORE_TREE_RECTANGLE_TREE_HILBERT_R_TREE_AUXILIARY_INFORMATION_HPP +#endif // MLPACK_CORE_TREE_RECTANGLE_TREE_HR_TREE_AUXILIARY_INFORMATION_HPP diff --git a/src/mlpack/core/tree/rectangle_tree/hilbert_r_tree_auxiliary_information_impl.hpp b/src/mlpack/core/tree/rectangle_tree/hilbert_r_tree_auxiliary_information_impl.hpp index dfefd8899d..f30b8016b0 100644 --- a/src/mlpack/core/tree/rectangle_tree/hilbert_r_tree_auxiliary_information_impl.hpp +++ b/src/mlpack/core/tree/rectangle_tree/hilbert_r_tree_auxiliary_information_impl.hpp @@ -10,8 +10,8 @@ * 3-clause BSD license along with mlpack. If not, see * http://www.opensource.org/licenses/BSD-3-Clause for more information. */ -#ifndef MLPACK_CORE_TREE_RECTANGLE_TREE_HILBERT_R_TREE_AUXILIARY_INFORMATION_IMPL_HPP -#define MLPACK_CORE_TREE_RECTANGLE_TREE_HILBERT_R_TREE_AUXILIARY_INFORMATION_IMPL_HPP +#ifndef MLPACK_CORE_TREE_RECTANGLE_TREE_HR_TREE_AUXILIARY_INFORMATION_IMPL_HPP +#define MLPACK_CORE_TREE_RECTANGLE_TREE_HR_TREE_AUXILIARY_INFORMATION_IMPL_HPP #include "hilbert_r_tree_auxiliary_information.hpp" @@ -190,4 +190,4 @@ Serialize(Archive& ar, const unsigned int /* version */) } // namespace tree } // namespace mlpack -#endif // MLPACK_CORE_TREE_RECTANGLE_TREE_HILBERT_R_TREE_AUXILIARY_INFORMATION_IMPL_HPP +#endif // MLPACK_CORE_TREE_RECTANGLE_TREE_HR_TREE_AUXILIARY_INFORMATION_IMPL_HPP diff --git a/src/mlpack/core/tree/rectangle_tree/hilbert_r_tree_descent_heuristic.hpp b/src/mlpack/core/tree/rectangle_tree/hilbert_r_tree_descent_heuristic.hpp index 1a564706dc..74086715f4 100644 --- a/src/mlpack/core/tree/rectangle_tree/hilbert_r_tree_descent_heuristic.hpp +++ b/src/mlpack/core/tree/rectangle_tree/hilbert_r_tree_descent_heuristic.hpp @@ -10,8 +10,8 @@ * 3-clause BSD license along with mlpack. If not, see * http://www.opensource.org/licenses/BSD-3-Clause for more information. */ -#ifndef MLPACK_CORE_TREE_RECTANGLE_TREE_HILBERT_R_TREE_DESCENT_HEURISTIC_HPP -#define MLPACK_CORE_TREE_RECTANGLE_TREE_HILBERT_R_TREE_DESCENT_HEURISTIC_HPP +#ifndef MLPACK_CORE_TREE_RECTANGLE_TREE_HR_TREE_DESCENT_HEURISTIC_HPP +#define MLPACK_CORE_TREE_RECTANGLE_TREE_HR_TREE_DESCENT_HEURISTIC_HPP #include @@ -55,4 +55,4 @@ class HilbertRTreeDescentHeuristic #include "hilbert_r_tree_descent_heuristic_impl.hpp" -#endif // MLPACK_CORE_TREE_RECTANGLE_TREE_HILBERT_R_TREE_DESCENT_HEURISTIC_HPP +#endif // MLPACK_CORE_TREE_RECTANGLE_TREE_HR_TREE_DESCENT_HEURISTIC_HPP diff --git a/src/mlpack/core/tree/rectangle_tree/hilbert_r_tree_descent_heuristic_impl.hpp b/src/mlpack/core/tree/rectangle_tree/hilbert_r_tree_descent_heuristic_impl.hpp index b7f4e08396..64a337552e 100644 --- a/src/mlpack/core/tree/rectangle_tree/hilbert_r_tree_descent_heuristic_impl.hpp +++ b/src/mlpack/core/tree/rectangle_tree/hilbert_r_tree_descent_heuristic_impl.hpp @@ -10,8 +10,8 @@ * 3-clause BSD license along with mlpack. If not, see * http://www.opensource.org/licenses/BSD-3-Clause for more information. */ -#ifndef MLPACK_CORE_TREE_RECTANGLE_TREE_HILBERT_R_TREE_DESCENT_HEURISTIC_IMPL_HPP -#define MLPACK_CORE_TREE_RECTANGLE_TREE_HILBERT_R_TREE_DESCENT_HEURISTIC_IMPL_HPP +#ifndef MLPACK_CORE_TREE_RECTANGLE_TREE_HR_TREE_DESCENT_HEURISTIC_IMPL_HPP +#define MLPACK_CORE_TREE_RECTANGLE_TREE_HR_TREE_DESCENT_HEURISTIC_IMPL_HPP #include "hilbert_r_tree_descent_heuristic.hpp" @@ -51,4 +51,4 @@ size_t HilbertRTreeDescentHeuristic::ChooseDescentNode( } // namespace tree } // namespace mlpack -#endif // MLPACK_CORE_TREE_RECTANGLE_TREE_HILBERT_R_TREE_DESCENT_HEURISTIC_IMPL_HPP +#endif // MLPACK_CORE_TREE_RECTANGLE_TREE_HR_TREE_DESCENT_HEURISTIC_IMPL_HPP diff --git a/src/mlpack/core/tree/rectangle_tree/r_plus_plus_tree_auxiliary_information.hpp b/src/mlpack/core/tree/rectangle_tree/r_plus_plus_tree_auxiliary_information.hpp index a4e363ebe0..74748667b0 100644 --- a/src/mlpack/core/tree/rectangle_tree/r_plus_plus_tree_auxiliary_information.hpp +++ b/src/mlpack/core/tree/rectangle_tree/r_plus_plus_tree_auxiliary_information.hpp @@ -11,8 +11,8 @@ * 3-clause BSD license along with mlpack. If not, see * http://www.opensource.org/licenses/BSD-3-Clause for more information. */ -#ifndef MLPACK_CORE_TREE_RECTANGLE_TREE_R_PLUS_PLUS_TREE_AUXILIARY_INFORMATION_HPP -#define MLPACK_CORE_TREE_RECTANGLE_TREE_R_PLUS_PLUS_TREE_AUXILIARY_INFORMATION_HPP +#ifndef MLPACK_CORE_TREE_RECTANGLE_TREE_RPP_TREE_AUXILIARY_INFORMATION_HPP +#define MLPACK_CORE_TREE_RECTANGLE_TREE_RPP_TREE_AUXILIARY_INFORMATION_HPP #include #include "../hrectbound.hpp" @@ -163,4 +163,4 @@ class RPlusPlusTreeAuxiliaryInformation #include "r_plus_plus_tree_auxiliary_information_impl.hpp" -#endif // MLPACK_CORE_TREE_RECTANGLE_TREE_R_PLUS_PLUS_TREE_AUXILIARY_INFORMATION_HPP +#endif // MLPACK_CORE_TREE_RECTANGLE_TREE_RPP_TREE_AUXILIARY_INFORMATION_HPP diff --git a/src/mlpack/core/tree/rectangle_tree/r_plus_plus_tree_auxiliary_information_impl.hpp b/src/mlpack/core/tree/rectangle_tree/r_plus_plus_tree_auxiliary_information_impl.hpp index 92bd22f610..62988a637c 100644 --- a/src/mlpack/core/tree/rectangle_tree/r_plus_plus_tree_auxiliary_information_impl.hpp +++ b/src/mlpack/core/tree/rectangle_tree/r_plus_plus_tree_auxiliary_information_impl.hpp @@ -11,8 +11,8 @@ * 3-clause BSD license along with mlpack. If not, see * http://www.opensource.org/licenses/BSD-3-Clause for more information. */ -#ifndef MLPACK_CORE_TREE_RECTANGLE_TREE_R_PLUS_PLUS_TREE_AUXILIARY_INFORMATION_IMPL_HPP -#define MLPACK_CORE_TREE_RECTANGLE_TREE_R_PLUS_PLUS_TREE_AUXILIARY_INFORMATION_IMPL_HPP +#ifndef MLPACK_CORE_TREE_RECTANGLE_TREE_RPP_TREE_AUXILIARY_INFORMATION_IMPL_HPP +#define MLPACK_CORE_TREE_RECTANGLE_TREE_RPP_TREE_AUXILIARY_INFORMATION_IMPL_HPP #include "r_plus_plus_tree_auxiliary_information.hpp" @@ -136,4 +136,4 @@ Serialize(Archive& ar, const unsigned int /* version */) } // namespace tree } // namespace mlpack -#endif // MLPACK_CORE_TREE_RECTANGLE_TREE_R_PLUS_PLUS_TREE_AUXILIARY_INFORMATION_IMPL_HPP +#endif // MLPACK_CORE_TREE_RECTANGLE_TREE_RPP_TREE_AUXILIARY_INFORMATION_IMPL_HPP diff --git a/src/mlpack/core/tree/rectangle_tree/r_plus_plus_tree_descent_heuristic.hpp b/src/mlpack/core/tree/rectangle_tree/r_plus_plus_tree_descent_heuristic.hpp index c813470db5..e5efc491b4 100644 --- a/src/mlpack/core/tree/rectangle_tree/r_plus_plus_tree_descent_heuristic.hpp +++ b/src/mlpack/core/tree/rectangle_tree/r_plus_plus_tree_descent_heuristic.hpp @@ -10,8 +10,8 @@ * 3-clause BSD license along with mlpack. If not, see * http://www.opensource.org/licenses/BSD-3-Clause for more information. */ -#ifndef MLPACK_CORE_TREE_RECTANGLE_TREE_R_PLUS_PLUS_TREE_DESCENT_HEURISTIC_HPP -#define MLPACK_CORE_TREE_RECTANGLE_TREE_R_PLUS_PLUS_TREE_DESCENT_HEURISTIC_HPP +#ifndef MLPACK_CORE_TREE_RECTANGLE_TREE_RPP_TREE_DESCENT_HEURISTIC_HPP +#define MLPACK_CORE_TREE_RECTANGLE_TREE_RPP_TREE_DESCENT_HEURISTIC_HPP #include @@ -50,4 +50,4 @@ class RPlusPlusTreeDescentHeuristic #include "r_plus_plus_tree_descent_heuristic_impl.hpp" -#endif // MLPACK_CORE_TREE_RECTANGLE_TREE_R_PLUS_PLUS_TREE_DESCENT_HEURISTIC_HPP +#endif // MLPACK_CORE_TREE_RECTANGLE_TREE_RPP_TREE_DESCENT_HEURISTIC_HPP diff --git a/src/mlpack/core/tree/rectangle_tree/r_plus_plus_tree_descent_heuristic_impl.hpp b/src/mlpack/core/tree/rectangle_tree/r_plus_plus_tree_descent_heuristic_impl.hpp index 141ae0350d..66f47c1365 100644 --- a/src/mlpack/core/tree/rectangle_tree/r_plus_plus_tree_descent_heuristic_impl.hpp +++ b/src/mlpack/core/tree/rectangle_tree/r_plus_plus_tree_descent_heuristic_impl.hpp @@ -10,8 +10,8 @@ * 3-clause BSD license along with mlpack. If not, see * http://www.opensource.org/licenses/BSD-3-Clause for more information. */ -#ifndef MLPACK_CORE_TREE_RECTANGLE_TREE_R_PLUS_PLUS_TREE_DESCENT_HEURISTIC_IMPL_HPP -#define MLPACK_CORE_TREE_RECTANGLE_TREE_R_PLUS_PLUS_TREE_DESCENT_HEURISTIC_IMPL_HPP +#ifndef MLPACK_CORE_TREE_RECTANGLE_TREE_RPP_TREE_DESCENT_HEURISTIC_IMPL_HPP +#define MLPACK_CORE_TREE_RECTANGLE_TREE_RPP_TREE_DESCENT_HEURISTIC_IMPL_HPP #include "r_plus_plus_tree_descent_heuristic.hpp" #include "../hrectbound.hpp" @@ -50,4 +50,4 @@ size_t RPlusPlusTreeDescentHeuristic::ChooseDescentNode( } // namespace tree } // namespace mlpack -#endif // MLPACK_CORE_TREE_RECTANGLE_TREE_R_PLUS_PLUS_TREE_DESCENT_HEURISTIC_IMPL_HPP +#endif // MLPACK_CORE_TREE_RECTANGLE_TREE_RPP_TREE_DESCENT_HEURISTIC_IMPL_HPP From 52b328e4351bd8042f7c45e132eab9c229dbfd15 Mon Sep 17 00:00:00 2001 From: Abhinav Moudgil Date: Fri, 26 May 2017 15:58:01 +0530 Subject: [PATCH 49/84] Fix undefined character escape style error --- src/mlpack/core/data/load_arff_impl.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/mlpack/core/data/load_arff_impl.hpp b/src/mlpack/core/data/load_arff_impl.hpp index 8667055082..578e07b048 100644 --- a/src/mlpack/core/data/load_arff_impl.hpp +++ b/src/mlpack/core/data/load_arff_impl.hpp @@ -49,8 +49,8 @@ void LoadARFF(const std::string& filename, if (line[0] == '@') { typedef boost::tokenizer> Tokenizer; - std::string separators = " \t\%"; // Split on comments too. - boost::escaped_list_separator sep("\\", separators, "\"{"); + std::string separators = " \t%"; // Split on comments too. + boost::escaped_list_separator sep("\\", separators, "{\""); Tokenizer tok(line, sep); Tokenizer::iterator it = tok.begin(); From cb90e0f4685aa051da67f952f98df9490f147df6 Mon Sep 17 00:00:00 2001 From: Abhinav Moudgil Date: Fri, 26 May 2017 16:05:30 +0530 Subject: [PATCH 50/84] Abbreviate macro names --- .../rectangle_tree/hilbert_r_tree_auxiliary_information.hpp | 6 +++--- .../hilbert_r_tree_auxiliary_information_impl.hpp | 6 +++--- .../r_plus_plus_tree_auxiliary_information.hpp | 6 +++--- .../r_plus_plus_tree_auxiliary_information_impl.hpp | 6 +++--- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/src/mlpack/core/tree/rectangle_tree/hilbert_r_tree_auxiliary_information.hpp b/src/mlpack/core/tree/rectangle_tree/hilbert_r_tree_auxiliary_information.hpp index a95b4d8824..42e3fd8fe5 100644 --- a/src/mlpack/core/tree/rectangle_tree/hilbert_r_tree_auxiliary_information.hpp +++ b/src/mlpack/core/tree/rectangle_tree/hilbert_r_tree_auxiliary_information.hpp @@ -11,8 +11,8 @@ * 3-clause BSD license along with mlpack. If not, see * http://www.opensource.org/licenses/BSD-3-Clause for more information. */ -#ifndef MLPACK_CORE_TREE_RECTANGLE_TREE_HR_TREE_AUXILIARY_INFORMATION_HPP -#define MLPACK_CORE_TREE_RECTANGLE_TREE_HR_TREE_AUXILIARY_INFORMATION_HPP +#ifndef MLPACK_CORE_TREE_RECTANGLE_TREE_HR_TREE_AUXILIARY_INFO_HPP +#define MLPACK_CORE_TREE_RECTANGLE_TREE_HR_TREE_AUXILIARY_INFO_HPP namespace mlpack { namespace tree { @@ -147,4 +147,4 @@ class HilbertRTreeAuxiliaryInformation #include "hilbert_r_tree_auxiliary_information_impl.hpp" -#endif // MLPACK_CORE_TREE_RECTANGLE_TREE_HR_TREE_AUXILIARY_INFORMATION_HPP +#endif // MLPACK_CORE_TREE_RECTANGLE_TREE_HR_TREE_AUXILIARY_INFO_HPP diff --git a/src/mlpack/core/tree/rectangle_tree/hilbert_r_tree_auxiliary_information_impl.hpp b/src/mlpack/core/tree/rectangle_tree/hilbert_r_tree_auxiliary_information_impl.hpp index f30b8016b0..e23e929431 100644 --- a/src/mlpack/core/tree/rectangle_tree/hilbert_r_tree_auxiliary_information_impl.hpp +++ b/src/mlpack/core/tree/rectangle_tree/hilbert_r_tree_auxiliary_information_impl.hpp @@ -10,8 +10,8 @@ * 3-clause BSD license along with mlpack. If not, see * http://www.opensource.org/licenses/BSD-3-Clause for more information. */ -#ifndef MLPACK_CORE_TREE_RECTANGLE_TREE_HR_TREE_AUXILIARY_INFORMATION_IMPL_HPP -#define MLPACK_CORE_TREE_RECTANGLE_TREE_HR_TREE_AUXILIARY_INFORMATION_IMPL_HPP +#ifndef MLPACK_CORE_TREE_RECTANGLE_TREE_HR_TREE_AUXILIARY_INFO_IMPL_HPP +#define MLPACK_CORE_TREE_RECTANGLE_TREE_HR_TREE_AUXILIARY_INFO_IMPL_HPP #include "hilbert_r_tree_auxiliary_information.hpp" @@ -190,4 +190,4 @@ Serialize(Archive& ar, const unsigned int /* version */) } // namespace tree } // namespace mlpack -#endif // MLPACK_CORE_TREE_RECTANGLE_TREE_HR_TREE_AUXILIARY_INFORMATION_IMPL_HPP +#endif // MLPACK_CORE_TREE_RECTANGLE_TREE_HR_TREE_AUXILIARY_INFO_IMPL_HPP diff --git a/src/mlpack/core/tree/rectangle_tree/r_plus_plus_tree_auxiliary_information.hpp b/src/mlpack/core/tree/rectangle_tree/r_plus_plus_tree_auxiliary_information.hpp index 74748667b0..55d6cbe9da 100644 --- a/src/mlpack/core/tree/rectangle_tree/r_plus_plus_tree_auxiliary_information.hpp +++ b/src/mlpack/core/tree/rectangle_tree/r_plus_plus_tree_auxiliary_information.hpp @@ -11,8 +11,8 @@ * 3-clause BSD license along with mlpack. If not, see * http://www.opensource.org/licenses/BSD-3-Clause for more information. */ -#ifndef MLPACK_CORE_TREE_RECTANGLE_TREE_RPP_TREE_AUXILIARY_INFORMATION_HPP -#define MLPACK_CORE_TREE_RECTANGLE_TREE_RPP_TREE_AUXILIARY_INFORMATION_HPP +#ifndef MLPACK_CORE_TREE_RECTANGLE_TREE_RPP_TREE_AUXILIARY_INFO_HPP +#define MLPACK_CORE_TREE_RECTANGLE_TREE_RPP_TREE_AUXILIARY_INFO_HPP #include #include "../hrectbound.hpp" @@ -163,4 +163,4 @@ class RPlusPlusTreeAuxiliaryInformation #include "r_plus_plus_tree_auxiliary_information_impl.hpp" -#endif // MLPACK_CORE_TREE_RECTANGLE_TREE_RPP_TREE_AUXILIARY_INFORMATION_HPP +#endif // MLPACK_CORE_TREE_RECTANGLE_TREE_RPP_TREE_AUXILIARY_INFO_HPP diff --git a/src/mlpack/core/tree/rectangle_tree/r_plus_plus_tree_auxiliary_information_impl.hpp b/src/mlpack/core/tree/rectangle_tree/r_plus_plus_tree_auxiliary_information_impl.hpp index 62988a637c..b19149539c 100644 --- a/src/mlpack/core/tree/rectangle_tree/r_plus_plus_tree_auxiliary_information_impl.hpp +++ b/src/mlpack/core/tree/rectangle_tree/r_plus_plus_tree_auxiliary_information_impl.hpp @@ -11,8 +11,8 @@ * 3-clause BSD license along with mlpack. If not, see * http://www.opensource.org/licenses/BSD-3-Clause for more information. */ -#ifndef MLPACK_CORE_TREE_RECTANGLE_TREE_RPP_TREE_AUXILIARY_INFORMATION_IMPL_HPP -#define MLPACK_CORE_TREE_RECTANGLE_TREE_RPP_TREE_AUXILIARY_INFORMATION_IMPL_HPP +#ifndef MLPACK_CORE_TREE_RECTANGLE_TREE_RPP_TREE_AUXILIARY_INFO_IMPL_HPP +#define MLPACK_CORE_TREE_RECTANGLE_TREE_RPP_TREE_AUXILIARY_INFO_IMPL_HPP #include "r_plus_plus_tree_auxiliary_information.hpp" @@ -136,4 +136,4 @@ Serialize(Archive& ar, const unsigned int /* version */) } // namespace tree } // namespace mlpack -#endif // MLPACK_CORE_TREE_RECTANGLE_TREE_RPP_TREE_AUXILIARY_INFORMATION_IMPL_HPP +#endif // MLPACK_CORE_TREE_RECTANGLE_TREE_RPP_TREE_AUXILIARY_INFO_IMPL_HPP From a090df738c120e5feb2ef9fa1afe19cfffa93d12 Mon Sep 17 00:00:00 2001 From: Abhinav Moudgil Date: Fri, 26 May 2017 16:26:42 +0530 Subject: [PATCH 51/84] Lines should be <= 80 characters long --- src/mlpack/core/util/backtrace.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/mlpack/core/util/backtrace.cpp b/src/mlpack/core/util/backtrace.cpp index 307dea7dd4..ce7052f128 100644 --- a/src/mlpack/core/util/backtrace.cpp +++ b/src/mlpack/core/util/backtrace.cpp @@ -48,7 +48,8 @@ // Easier to read Backtrace::DecodeAddress(). #ifdef HAS_BFD_DL #define TRACE_CONDITION_1 (!dladdr(trace[i], &addressHandler)) - #define FIND_LINE (bfd_find_nearest_line(abfd, text, syms, offset, &frame.file, &frame.function, &frame.line) && frame.file) + #define FIND_LINE (bfd_find_nearest_line(abfd, text, syms, offset, \ + &frame.file, &frame.function, &frame.line) && frame.file) #endif using namespace mlpack; From 8b18e42f4801670f3cc4c35074eccc9d6b4b54a8 Mon Sep 17 00:00:00 2001 From: Marcus Edel Date: Fri, 26 May 2017 17:18:31 +0200 Subject: [PATCH 52/84] Fix non-static member function issue; for more information take a look at: #1009. --- .../decision_tree/decision_tree_impl.hpp | 26 +++++++++++++------ 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/src/mlpack/methods/decision_tree/decision_tree_impl.hpp b/src/mlpack/methods/decision_tree/decision_tree_impl.hpp index 7c9ef7ecc0..16288f8434 100644 --- a/src/mlpack/methods/decision_tree/decision_tree_impl.hpp +++ b/src/mlpack/methods/decision_tree/decision_tree_impl.hpp @@ -592,15 +592,20 @@ void DecisionTreeTrain(data, currentChildBegin, + { + child->Train(data, currentChildBegin, currentCol - currentChildBegin, datasetInfo, labels, numClasses, weights, currentCol - currentChildBegin); + } else - children.back()->Train(data, currentChildBegin, + { + child->Train(data, currentChildBegin, currentCol - currentChildBegin, datasetInfo, labels, numClasses, weights, minimumLeafSize); + } + children.push_back(child); } } else @@ -718,15 +723,20 @@ void DecisionTreeTrain(data, currentChildBegin, - currentCol - currentChildBegin, - labels, numClasses, weights, currentCol - currentChildBegin); + { + child->Train(data, currentChildBegin, + currentCol - currentChildBegin, labels, numClasses, weights, + currentCol - currentChildBegin); + } else - children.back()->Train(data, currentChildBegin, + { + child->Train(data, currentChildBegin, currentCol - currentChildBegin, labels, numClasses, weights, minimumLeafSize); + } + children.push_back(child); } } else From f6ea371e6056dff3977735000488d5f9bfcffb07 Mon Sep 17 00:00:00 2001 From: Marcus Edel Date: Fri, 26 May 2017 20:57:32 +0200 Subject: [PATCH 53/84] Update version numbers (mlpack-2.2.3). --- README.md | 2 +- doc/guide/build.hpp | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 3561e74171..ed41237db4 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,7 @@

Download: - current stable version (2.2.2) + current stable version (2.2.3)

diff --git a/doc/guide/build.hpp b/doc/guide/build.hpp index d175747574..ceab1b33fe 100644 --- a/doc/guide/build.hpp +++ b/doc/guide/build.hpp @@ -23,14 +23,14 @@ href="https://keon.io/mlpack/mlpack-on-windows/">Keon's excellent tutorial. @section Download latest mlpack build Download latest mlpack build from here: -mlpack-2.2.2 +mlpack-2.2.3 @section builddir Creating Build Directory Once the mlpack source is unpacked, you should create a build directory. @code -$ cd mlpack-2.2.2 +$ cd mlpack-2.2.3 $ mkdir build @endcode From fa5e37e4de3a02a540e225567537c9d007336702 Mon Sep 17 00:00:00 2001 From: Abhinav Moudgil Date: Sat, 27 May 2017 00:40:00 +0530 Subject: [PATCH 54/84] cpplint style fixes --- .../core/dists/discrete_distribution.hpp | 2 +- src/mlpack/core/kernels/gaussian_kernel.hpp | 5 ++--- src/mlpack/core/math/lin_alg.cpp | 2 +- src/mlpack/core/optimizers/sgdr/sgdr.hpp | 16 +++++++-------- .../core/optimizers/sgdr/snapshot_sgdr.hpp | 20 +++++++++---------- .../tree/rectangle_tree/x_tree_split_impl.hpp | 4 ++-- 6 files changed, 24 insertions(+), 25 deletions(-) diff --git a/src/mlpack/core/dists/discrete_distribution.hpp b/src/mlpack/core/dists/discrete_distribution.hpp index d8301e246f..8f77258bf7 100644 --- a/src/mlpack/core/dists/discrete_distribution.hpp +++ b/src/mlpack/core/dists/discrete_distribution.hpp @@ -145,7 +145,7 @@ class DiscreteDistribution if (obs >= probabilities[dimension].n_elem) { Log::Debug << "DiscreteDistribution::Probability(): " - << "received observation " << obs << "; observation must be" + << " received observation " << obs << "; observation must be " << "in [0, " << probabilities[dimension].n_elem << "] for this " << "distribution." << std::endl; } diff --git a/src/mlpack/core/kernels/gaussian_kernel.hpp b/src/mlpack/core/kernels/gaussian_kernel.hpp index 31095a6ba0..c0f34b4776 100644 --- a/src/mlpack/core/kernels/gaussian_kernel.hpp +++ b/src/mlpack/core/kernels/gaussian_kernel.hpp @@ -126,9 +126,8 @@ class GaussianKernel template double ConvolutionIntegral(const VecTypeA& a, const VecTypeB& b) { - return Evaluate(sqrt(metric:: - SquaredEuclideanDistance::Evaluate(a, b) / 2.0)) / - (Normalizer(a.n_rows) * pow(2.0, (double) a.n_rows / 2.0)); + return Evaluate(sqrt(metric::SquaredEuclideanDistance::Evaluate(a, b) / + 2.0)) / (Normalizer(a.n_rows) * pow(2.0, (double) a.n_rows / 2.0)); } diff --git a/src/mlpack/core/math/lin_alg.cpp b/src/mlpack/core/math/lin_alg.cpp index f5336eee05..915d1a36a9 100644 --- a/src/mlpack/core/math/lin_alg.cpp +++ b/src/mlpack/core/math/lin_alg.cpp @@ -258,7 +258,7 @@ void mlpack::math::Svec(const arma::sp_mat& input, arma::sp_vec& output) void mlpack::math::Smat(const arma::vec& input, arma::mat& output) { const size_t n = static_cast - (ceil((-1. + sqrt(1. + 8. * input.n_elem))/2.)); + (ceil((-1. + sqrt(1. + 8. * input.n_elem))/2.)); output.zeros(n, n); diff --git a/src/mlpack/core/optimizers/sgdr/sgdr.hpp b/src/mlpack/core/optimizers/sgdr/sgdr.hpp index 707565ab68..e404f21a0c 100644 --- a/src/mlpack/core/optimizers/sgdr/sgdr.hpp +++ b/src/mlpack/core/optimizers/sgdr/sgdr.hpp @@ -77,14 +77,14 @@ class SGDR * parameters. */ SGDR(DecomposableFunctionType& function, - const size_t epochRestart = 50, - const double multFactor = 2.0, - const size_t batchSize = 1000, - const double stepSize = 0.01, - const size_t maxIterations = 100000, - const double tolerance = 1e-5, - const bool shuffle = true, - const UpdatePolicyType& updatePolicy = UpdatePolicyType()); + const size_t epochRestart = 50, + const double multFactor = 2.0, + const size_t batchSize = 1000, + const double stepSize = 0.01, + const size_t maxIterations = 100000, + const double tolerance = 1e-5, + const bool shuffle = true, + const UpdatePolicyType& updatePolicy = UpdatePolicyType()); /** * Optimize the given function using SGDR. The given starting point diff --git a/src/mlpack/core/optimizers/sgdr/snapshot_sgdr.hpp b/src/mlpack/core/optimizers/sgdr/snapshot_sgdr.hpp index 12cd7a55da..00f76af853 100644 --- a/src/mlpack/core/optimizers/sgdr/snapshot_sgdr.hpp +++ b/src/mlpack/core/optimizers/sgdr/snapshot_sgdr.hpp @@ -92,16 +92,16 @@ class SnapshotSGDR * parameters. */ SnapshotSGDR(DecomposableFunctionType& function, - const size_t epochRestart = 50, - const double multFactor = 2.0, - const size_t batchSize = 1000, - const double stepSize = 0.01, - const size_t maxIterations = 100000, - const double tolerance = 1e-5, - const bool shuffle = true, - const size_t snapshots = 5, - const bool accumulate = true, - const UpdatePolicyType& updatePolicy = UpdatePolicyType()); + const size_t epochRestart = 50, + const double multFactor = 2.0, + const size_t batchSize = 1000, + const double stepSize = 0.01, + const size_t maxIterations = 100000, + const double tolerance = 1e-5, + const bool shuffle = true, + const size_t snapshots = 5, + const bool accumulate = true, + const UpdatePolicyType& updatePolicy = UpdatePolicyType()); /** * Optimize the given function using SGDR. The given starting point diff --git a/src/mlpack/core/tree/rectangle_tree/x_tree_split_impl.hpp b/src/mlpack/core/tree/rectangle_tree/x_tree_split_impl.hpp index 3bf23c8cf3..a2b1694e7b 100644 --- a/src/mlpack/core/tree/rectangle_tree/x_tree_split_impl.hpp +++ b/src/mlpack/core/tree/rectangle_tree/x_tree_split_impl.hpp @@ -493,7 +493,7 @@ bool XTreeSplit::SplitNonLeafNode(TreeType *tree, std::vector& relevels) } } std::sort(sorted2.begin(), sorted2.end(), - PairComp); + PairComp); tree->numDescendants = 0; tree->bound.Clear(); @@ -654,7 +654,7 @@ bool XTreeSplit::SplitNonLeafNode(TreeType *tree, std::vector& relevels) } } std::sort(sorted2.begin(), sorted2.end(), - PairComp); + PairComp); for (size_t i = 0; i < numChildren; i++) { From 8af0710f4f2aad4b602466d61617652fcd63f279 Mon Sep 17 00:00:00 2001 From: Abhinav Moudgil Date: Sat, 27 May 2017 00:56:29 +0530 Subject: [PATCH 55/84] Remove trailing space --- src/mlpack/core/kernels/gaussian_kernel.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mlpack/core/kernels/gaussian_kernel.hpp b/src/mlpack/core/kernels/gaussian_kernel.hpp index c0f34b4776..4d3446cc7c 100644 --- a/src/mlpack/core/kernels/gaussian_kernel.hpp +++ b/src/mlpack/core/kernels/gaussian_kernel.hpp @@ -126,7 +126,7 @@ class GaussianKernel template double ConvolutionIntegral(const VecTypeA& a, const VecTypeB& b) { - return Evaluate(sqrt(metric::SquaredEuclideanDistance::Evaluate(a, b) / + return Evaluate(sqrt(metric::SquaredEuclideanDistance::Evaluate(a, b) / 2.0)) / (Normalizer(a.n_rows) * pow(2.0, (double) a.n_rows / 2.0)); } From 80afea6fb3485f1e27dbe206315cc9d63ebc3b9d Mon Sep 17 00:00:00 2001 From: Abhinav Moudgil Date: Sat, 27 May 2017 11:42:30 +0530 Subject: [PATCH 56/84] Indent with two spaces --- src/mlpack/core/tree/statistic.hpp | 34 +++++++++++++++--------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/src/mlpack/core/tree/statistic.hpp b/src/mlpack/core/tree/statistic.hpp index 5ab361be45..2a2a0d86bd 100644 --- a/src/mlpack/core/tree/statistic.hpp +++ b/src/mlpack/core/tree/statistic.hpp @@ -24,25 +24,25 @@ namespace tree { class EmptyStatistic { public: - EmptyStatistic() { } - ~EmptyStatistic() { } + EmptyStatistic() { } + ~EmptyStatistic() { } - /** - * This constructor is called when a node is finished being created. The - * node is finished, and its children are finished, but it is not - * necessarily true that the statistics of other nodes are initialized yet. - * - * @param node Node which this corresponds to. - */ - template - EmptyStatistic(TreeType& /* node */) { } + /** + * This constructor is called when a node is finished being created. The + * node is finished, and its children are finished, but it is not + * necessarily true that the statistics of other nodes are initialized yet. + * + * @param node Node which this corresponds to. + */ + template + EmptyStatistic(TreeType& /* node */) { } - /** - * Serialize the statistic (there's nothing to be saved). - */ - template - void Serialize(Archive& /* ar */, const unsigned int /* version */) - { } + /** + * Serialize the statistic (there's nothing to be saved). + */ + template + void Serialize(Archive& /* ar */, const unsigned int /* version */) + { } }; } // namespace tree From c5abec07a6b51dae4eb62294db43fc21348068b4 Mon Sep 17 00:00:00 2001 From: Marcus Edel Date: Sat, 27 May 2017 21:19:25 +0200 Subject: [PATCH 57/84] Accelerate the distracted sequence recall test. --- src/mlpack/tests/recurrent_network_test.cpp | 100 +++++++++++--------- 1 file changed, 53 insertions(+), 47 deletions(-) diff --git a/src/mlpack/tests/recurrent_network_test.cpp b/src/mlpack/tests/recurrent_network_test.cpp index 3941c3d6b8..8395839471 100644 --- a/src/mlpack/tests/recurrent_network_test.cpp +++ b/src/mlpack/tests/recurrent_network_test.cpp @@ -13,6 +13,8 @@ #include #include + +#include #include #include @@ -491,33 +493,45 @@ BOOST_AUTO_TEST_CASE(EmbeddedReberGrammarTest) * * @param input The generated input sequence. * @param input The generated output sequence. + * @param sequences Number of samples. */ -void GenerateDistractedSequence(arma::mat& input, arma::mat& output) +void GenerateDistractedSequence(arma::mat& input, + arma::mat& output, + const size_t sequences) { - input = arma::zeros(10, 10); - output = arma::zeros(3, 10); + input = arma::zeros(100, sequences); + output = arma::zeros(30, sequences); - arma::Col index = arma::shuffle(arma::linspace >( - 0, 7, 8)); - - // Set the target in the input sequence and the corresponding targets in the - // output sequence by following the correct order. - for (size_t i = 0; i < 2; i++) + for (size_t i = 0; i < sequences; ++i) { - size_t idx = rand() % 2; - input(idx, index(i)) = 1; - output(idx, index(i) > index(i == 0) ? 9 : 8) = 1; + arma::mat inputTemp = arma::zeros(10, 10); + arma::mat outputTemp = arma::zeros(3, 10); + + arma::Col index = arma::shuffle( + arma::linspace >(0, 7, 8)); + + // Set the target in the input sequence and the corresponding targets in the + // output sequence by following the correct order. + for (size_t i = 0; i < 2; i++) + { + size_t idx = rand() % 2; + inputTemp(idx, index(i)) = 1; + outputTemp(idx, index(i) > index(i == 0) ? 9 : 8) = 1; + } + + for (size_t i = 2; i < 8; i++) + inputTemp(2 + rand() % 6, index(i)) = 1; + + // Set the prompts which direct the network to give an answer. + inputTemp(8, 8) = 1; + inputTemp(9, 9) = 1; + + inputTemp.reshape(inputTemp.n_elem, 1); + outputTemp.reshape(outputTemp.n_elem, 1); + + input.col(i) = inputTemp; + output.col(i) = outputTemp; } - - for (size_t i = 2; i < 8; i++) - input(2 + rand() % 6, index(i)) = 1; - - // Set the prompts which direct the network to give an answer. - input(8, 8) = 1; - input(9, 9) = 1; - - input.reshape(input.n_elem, 1); - output.reshape(output.n_elem, 1); } /** @@ -529,18 +543,15 @@ void DistractedSequenceRecallTestNetwork() const size_t trainDistractedSequenceCount = 800; const size_t testDistractedSequenceCount = 400; - arma::field trainInput(1, trainDistractedSequenceCount); - arma::field trainLabels(1, trainDistractedSequenceCount); - arma::field testInput(1, testDistractedSequenceCount); - arma::field testLabels(1, testDistractedSequenceCount); + arma::mat trainInput, trainLabels, testInput, testLabels; // Generate the training data. - for (size_t i = 0; i < trainDistractedSequenceCount; i++) - GenerateDistractedSequence(trainInput(0, i), trainLabels(0, i)); + GenerateDistractedSequence(trainInput, trainLabels, + trainDistractedSequenceCount); // Generate the test data. - for (size_t i = 0; i < testDistractedSequenceCount; i++) - GenerateDistractedSequence(testInput(0, i), testLabels(0, i)); + GenerateDistractedSequence(testInput, testLabels, + testDistractedSequenceCount); /* * Construct a network with 10 input units, layerSize hidden units and 3 @@ -560,7 +571,7 @@ void DistractedSequenceRecallTestNetwork() */ const size_t outputSize = 3; const size_t inputSize = 10; - const size_t rho = trainInput.at(0, 0).n_elem / inputSize; + const size_t rho = trainInput.col(0).n_elem / inputSize; // It isn't guaranteed that the recurrent network will converge in the // specified number of iterations using random weights. If this works 1 of 5 @@ -570,26 +581,21 @@ void DistractedSequenceRecallTestNetwork() size_t offset = 0; for (size_t trial = 0; trial < 5; ++trial) { - RNN > model(rho); + RandomInitialization init(-0.5, 0.5); + MeanSquaredError<> output; + RNN, RandomInitialization> model( + rho, false, output, init); model.Add >(); - model.Add >(inputSize, 14); - model.Add >(14, 7, rho); + model.Add >(inputSize, 16); + model.Add >(16, 7, rho); model.Add >(7, outputSize); model.Add >(); - StandardSGD opt(model, 0.1, 2, -50000); + StandardSGD opt(model, 0.1, + trainDistractedSequenceCount * (6 + offset), -1); arma::mat inputTemp, labelsTemp; - for (size_t i = 0; i < (10 + offset); i++) - { - for (size_t j = 0; j < trainDistractedSequenceCount; j++) - { - inputTemp = trainInput.at(0, j); - labelsTemp = trainLabels.at(0, j); - - model.Train(inputTemp, labelsTemp, opt); - } - } + model.Train(trainInput, trainLabels, opt); double error = 0; @@ -598,12 +604,12 @@ void DistractedSequenceRecallTestNetwork() for (size_t i = 0; i < testDistractedSequenceCount; i++) { arma::mat output; - arma::mat input = testInput.at(0, i); + arma::mat input = testInput.col(i); model.Predict(input, output); data::Binarize(output, output, 0.5); - if (arma::accu(arma::abs(testLabels.at(0, i) - output)) != 0) + if (arma::accu(arma::abs(testLabels.col(i) - output)) != 0) error += 1; } From 9251a63d2edabcc051ee27849da34045d93804c3 Mon Sep 17 00:00:00 2001 From: Marcus Edel Date: Sat, 27 May 2017 21:38:57 +0200 Subject: [PATCH 58/84] Update vector layout, for more information take a look at: #1002. --- src/mlpack/tests/serialization_test.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mlpack/tests/serialization_test.cpp b/src/mlpack/tests/serialization_test.cpp index 21ec438908..4e78b258ec 100644 --- a/src/mlpack/tests/serialization_test.cpp +++ b/src/mlpack/tests/serialization_test.cpp @@ -275,7 +275,7 @@ BOOST_AUTO_TEST_CASE(LinearRegressionTest) mat data; data.randn(15, 800); rowvec responses; - responses.randn(800, 1); + responses.randn(800); LinearRegression lr(data, responses, 0.05); // Train the model. LinearRegression xmlLr, textLr, binaryLr; From 42bfdbbbfa279919ce9d5926a649507b1aa044ad Mon Sep 17 00:00:00 2001 From: Shangtong Zhang Date: Sat, 27 May 2017 23:41:44 -0600 Subject: [PATCH 59/84] Add epsilon greedy policy for DQN --- .../policy/CMakeLists.txt | 14 +++ .../policy/greedy_policy.hpp | 103 ++++++++++++++++++ src/mlpack/tests/rl_components_test.cpp | 23 +++- 3 files changed, 137 insertions(+), 3 deletions(-) create mode 100644 src/mlpack/methods/reinforcement_learning/policy/CMakeLists.txt create mode 100644 src/mlpack/methods/reinforcement_learning/policy/greedy_policy.hpp diff --git a/src/mlpack/methods/reinforcement_learning/policy/CMakeLists.txt b/src/mlpack/methods/reinforcement_learning/policy/CMakeLists.txt new file mode 100644 index 0000000000..4ebd441cf3 --- /dev/null +++ b/src/mlpack/methods/reinforcement_learning/policy/CMakeLists.txt @@ -0,0 +1,14 @@ +# Define the files we need to compile +# Anything not in this list will not be compiled into mlpack. +set(SOURCES + greedy_policy.hpp +) + +# Add directory name to sources. +set(DIR_SRCS) +foreach(file ${SOURCES}) + set(DIR_SRCS ${DIR_SRCS} ${CMAKE_CURRENT_SOURCE_DIR}/${file}) +endforeach() +# Append sources (with directory name) to list of all mlpack sources (used at +# the parent scope). +set(MLPACK_SRCS ${MLPACK_SRCS} ${DIR_SRCS} PARENT_SCOPE) diff --git a/src/mlpack/methods/reinforcement_learning/policy/greedy_policy.hpp b/src/mlpack/methods/reinforcement_learning/policy/greedy_policy.hpp new file mode 100644 index 0000000000..d989764865 --- /dev/null +++ b/src/mlpack/methods/reinforcement_learning/policy/greedy_policy.hpp @@ -0,0 +1,103 @@ +/** + * @file greedy_policy.hpp + * @author Shangtong Zhang + * + * This file is an implementation of epsilon greedy policy. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_METHODS_RL_POLICY_GREEDY_POLICY_HPP +#define MLPACK_METHODS_RL_POLICY_GREEDY_POLICY_HPP + +#include + +namespace mlpack { +namespace rl { + +/** + * Implementation for epsilon greedy policy. + * + * In general we will select an action greedily based on the action value, + * however under sometimes we will also randomly select an action to + * encourage exploration. + * + * @tparam EnvironmentType The reinforcement learning task. + */ +template +class GreedyPolicy { + public: + using ActionType = typename EnvironmentType::Action; + + /** + * Constructor for epsilon greedy policy class. + * @param initialEpsilon The initial probability to explore (select a random action). + * @param annealInterval The steps during which the probability to explore will anneal. + * @param minEpsilon Epsilon will never be less than this value. + */ + GreedyPolicy(double initialEpsilon, + size_t annealInterval, + double minEpsilon) : + epsilon(initialEpsilon), + minEpsilon(minEpsilon), + delta((initialEpsilon - minEpsilon) / annealInterval) + { /* Nothing to do here. */ } + + /** + * Sample an action based on given action values. + * @param actionValue Values for each action. + * @return Sampled action + */ + ActionType Sample(const arma::colvec& actionValue) + { + double exploration = math::Random(); + + // Select the action randomly. + if (exploration < epsilon) + return static_cast(math::RandInt(ActionType::size)); + + // Select the action greedily. + size_t bestAction = 0; + double maxActionValue = actionValue[0]; + for (size_t action = 1; action < ActionType::size; ++action) + { + if (maxActionValue < actionValue[action]) + { + maxActionValue = actionValue[action]; + bestAction = action; + } + } + return static_cast(bestAction); + }; + + /** + * Exploration probability will anneal at each step. + */ + void Anneal() + { + epsilon -= delta; + epsilon = std::max(minEpsilon, epsilon); + } + + /** + * @return Current possibility to explore. + */ + const double& Epsilon() const { return epsilon; } + + private: + //! Locally-stored probability to explore. + double epsilon; + + //! Locally-stored lower bound for epsilon. + double minEpsilon; + + //! Locally-stored stride for epsilon to anneal. + double delta; +}; + +} // namespace rl +} // namespace mlpack + +#endif diff --git a/src/mlpack/tests/rl_components_test.cpp b/src/mlpack/tests/rl_components_test.cpp index 31a9ef5991..6ebb44a153 100644 --- a/src/mlpack/tests/rl_components_test.cpp +++ b/src/mlpack/tests/rl_components_test.cpp @@ -15,6 +15,7 @@ #include #include #include +#include #include #include "test_tools.hpp" @@ -87,13 +88,14 @@ BOOST_AUTO_TEST_CASE(RandomReplayTest) BOOST_REQUIRE_EQUAL(1, replay.Size()); //! Overwrite the memory with a nonsense record - for (size_t i = 0; i < 5; ++i) { + for (size_t i = 0; i < 5; ++i) replay.Store(nextState, action, reward, state, true); - } + BOOST_REQUIRE_EQUAL(3, replay.Size()); //! Sample several times, the original record shouldn't appear - for (size_t i = 0; i < 30; ++i) { + for (size_t i = 0; i < 30; ++i) + { replay.Sample(sampledState, sampledAction, sampledReward, sampledNextState, sampledTerminal); CheckMatrices(state.Encode(), sampledNextState); CheckMatrices(nextState.Encode(), sampledState); @@ -101,4 +103,19 @@ BOOST_AUTO_TEST_CASE(RandomReplayTest) } } +/** + * Construct a greedy policy instance and check if it works as + * it should be. + */ +BOOST_AUTO_TEST_CASE(GreedyPolicyTest) +{ + GreedyPolicy policy(1.0, 10, 0.0); + for (int i = 0; i < 15; ++i) + policy.Anneal(); + BOOST_REQUIRE_CLOSE(0.0, policy.Epsilon(), 1e-5); + arma::colvec actionValue = arma::randn(CartPole::Action::size); + CartPole::Action action = policy.Sample(actionValue); + BOOST_REQUIRE_CLOSE(actionValue[action], actionValue.max(), 1e-5); +} + BOOST_AUTO_TEST_SUITE_END() From 7f33916c0d19facfb036fa266e14b310d2908e7c Mon Sep 17 00:00:00 2001 From: Shangtong Zhang Date: Sun, 28 May 2017 23:03:05 -0600 Subject: [PATCH 60/84] Minor style fix --- .../policy/greedy_policy.hpp | 21 ++++++------------- src/mlpack/tests/rl_components_test.cpp | 2 +- 2 files changed, 7 insertions(+), 16 deletions(-) diff --git a/src/mlpack/methods/reinforcement_learning/policy/greedy_policy.hpp b/src/mlpack/methods/reinforcement_learning/policy/greedy_policy.hpp index d989764865..7f38ddeecf 100644 --- a/src/mlpack/methods/reinforcement_learning/policy/greedy_policy.hpp +++ b/src/mlpack/methods/reinforcement_learning/policy/greedy_policy.hpp @@ -21,7 +21,7 @@ namespace rl { * Implementation for epsilon greedy policy. * * In general we will select an action greedily based on the action value, - * however under sometimes we will also randomly select an action to + * however sometimes we will also randomly select an action to * encourage exploration. * * @tparam EnvironmentType The reinforcement learning task. @@ -37,9 +37,9 @@ class GreedyPolicy { * @param annealInterval The steps during which the probability to explore will anneal. * @param minEpsilon Epsilon will never be less than this value. */ - GreedyPolicy(double initialEpsilon, - size_t annealInterval, - double minEpsilon) : + GreedyPolicy(const double initialEpsilon, + const size_t annealInterval, + const double minEpsilon) : epsilon(initialEpsilon), minEpsilon(minEpsilon), delta((initialEpsilon - minEpsilon) / annealInterval) @@ -59,17 +59,8 @@ class GreedyPolicy { return static_cast(math::RandInt(ActionType::size)); // Select the action greedily. - size_t bestAction = 0; - double maxActionValue = actionValue[0]; - for (size_t action = 1; action < ActionType::size; ++action) - { - if (maxActionValue < actionValue[action]) - { - maxActionValue = actionValue[action]; - bestAction = action; - } - } - return static_cast(bestAction); + return static_cast( + arma::as_scalar(arma::find(actionValue == actionValue.max(), 1))); }; /** diff --git a/src/mlpack/tests/rl_components_test.cpp b/src/mlpack/tests/rl_components_test.cpp index 6ebb44a153..708623a286 100644 --- a/src/mlpack/tests/rl_components_test.cpp +++ b/src/mlpack/tests/rl_components_test.cpp @@ -110,7 +110,7 @@ BOOST_AUTO_TEST_CASE(RandomReplayTest) BOOST_AUTO_TEST_CASE(GreedyPolicyTest) { GreedyPolicy policy(1.0, 10, 0.0); - for (int i = 0; i < 15; ++i) + for (size_t i = 0; i < 15; ++i) policy.Anneal(); BOOST_REQUIRE_CLOSE(0.0, policy.Epsilon(), 1e-5); arma::colvec actionValue = arma::randn(CartPole::Action::size); From dfeaf0cb0ce0dc1152bc2728e0ecf237ef29ec1e Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Mon, 29 May 2017 11:39:36 -0400 Subject: [PATCH 61/84] Change debug output to fatal errors. --- src/mlpack/core/dists/discrete_distribution.hpp | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/src/mlpack/core/dists/discrete_distribution.hpp b/src/mlpack/core/dists/discrete_distribution.hpp index 8f77258bf7..60876f257b 100644 --- a/src/mlpack/core/dists/discrete_distribution.hpp +++ b/src/mlpack/core/dists/discrete_distribution.hpp @@ -129,12 +129,11 @@ class DiscreteDistribution // Ensure the observation has the same dimension with the probabilities if (observation.n_elem != probabilities.size()) { - Log::Debug << "the obversation must has the same dimension with the " - << "probabilities the observation's dimension is " - << observation.n_elem << " but the dimension of probabilities is " - << probabilities.size() << std::endl; - return probability; + Log::Fatal << "DiscreteDistribution::Probability(): observation has " + << "incorrect dimension " << observation.n_elem << " but should have " + << "dimension " << probabilities.size() << "!" << std::endl; } + for (size_t dimension = 0; dimension < observation.n_elem; dimension++) { // Adding 0.5 helps ensure that we cast the floating point to a size_t @@ -144,10 +143,10 @@ class DiscreteDistribution // Ensure that the observation is within the bounds. if (obs >= probabilities[dimension].n_elem) { - Log::Debug << "DiscreteDistribution::Probability(): " - << " received observation " << obs << "; observation must be " - << "in [0, " << probabilities[dimension].n_elem << "] for this " - << "distribution." << std::endl; + Log::Fatal << "DiscreteDistribution::Probability(): received " + << "observation " << obs << "; observation must be in [0, " + << probabilities[dimension].n_elem << "] for this distribution." + << std::endl; } probability *= probabilities[dimension][obs]; } From 2e229517515a606f8a93676507e48aa6fd9cbfbb Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Mon, 29 May 2017 11:44:38 -0400 Subject: [PATCH 62/84] Minor style changes. --- src/mlpack/core/data/load_csv.hpp | 4 ++-- src/mlpack/core/dists/gamma_distribution.hpp | 4 ++-- .../core/optimizers/sdp/primal_dual.hpp | 2 +- src/mlpack/core/tree/cellbound_impl.hpp | 12 ++++------- src/mlpack/core/tree/hrectbound_impl.hpp | 20 ++++++++----------- .../tree/rectangle_tree/rectangle_tree.hpp | 4 ++-- 6 files changed, 19 insertions(+), 27 deletions(-) diff --git a/src/mlpack/core/data/load_csv.hpp b/src/mlpack/core/data/load_csv.hpp index ecaff8c115..87c722fdf7 100644 --- a/src/mlpack/core/data/load_csv.hpp +++ b/src/mlpack/core/data/load_csv.hpp @@ -256,8 +256,8 @@ class LoadCSV // Remove whitespace from either side. boost::trim(line); - // parse the numbers from a line(ex : 1,2,3,4), if the parser find the - // number it will execute the setNum function + // Parse the numbers from a line (ex: 1,2,3,4); if the parser finds a + // number it will execute the setNum function. const bool canParse = qi::parse(line.begin(), line.end(), stringRule[setCharClass] % delimiterRule); diff --git a/src/mlpack/core/dists/gamma_distribution.hpp b/src/mlpack/core/dists/gamma_distribution.hpp index 9e748ec442..b4d7c6e639 100644 --- a/src/mlpack/core/dists/gamma_distribution.hpp +++ b/src/mlpack/core/dists/gamma_distribution.hpp @@ -214,7 +214,7 @@ class GammaDistribution const double tol); }; -} // namespace distribution. -} // namespace mlpack. +} // namespace distribution +} // namespace mlpack #endif diff --git a/src/mlpack/core/optimizers/sdp/primal_dual.hpp b/src/mlpack/core/optimizers/sdp/primal_dual.hpp index ffa5b5af19..908852ae87 100644 --- a/src/mlpack/core/optimizers/sdp/primal_dual.hpp +++ b/src/mlpack/core/optimizers/sdp/primal_dual.hpp @@ -109,7 +109,7 @@ class PrimalDualSolver arma::vec initialYdense; //! Starting point for Z, the complementary slack variable. Needs to be - // positive definite. + //! positive definite. arma::mat initialZ; //! The step size modulating factor. Needs to be a scalar in (0, 1). diff --git a/src/mlpack/core/tree/cellbound_impl.hpp b/src/mlpack/core/tree/cellbound_impl.hpp index e117a5e28a..245ec4ce3d 100644 --- a/src/mlpack/core/tree/cellbound_impl.hpp +++ b/src/mlpack/core/tree/cellbound_impl.hpp @@ -867,10 +867,8 @@ CellBound::RangeDistance( */ template template -inline CellBound< - MetricType, - ElemType>& CellBound::operator|=( - const MatType& data) +inline CellBound& +CellBound::operator|=(const MatType& data) { Log::Assert(data.n_rows == dim); @@ -898,10 +896,8 @@ inline CellBound< * Expands this region to encompass another bound. */ template -inline CellBound< - MetricType, - ElemType>& CellBound::operator|=( - const CellBound& other) +inline CellBound& +CellBound::operator|=(const CellBound& other) { assert(other.dim == dim); diff --git a/src/mlpack/core/tree/hrectbound_impl.hpp b/src/mlpack/core/tree/hrectbound_impl.hpp index 0ee0f5a9fe..f262ad4495 100644 --- a/src/mlpack/core/tree/hrectbound_impl.hpp +++ b/src/mlpack/core/tree/hrectbound_impl.hpp @@ -507,10 +507,8 @@ HRectBound::RangeDistance( */ template template -inline HRectBound< - MetricType, - ElemType>& HRectBound::operator|=( - const MatType& data) +inline HRectBound& +HRectBound::operator|=(const MatType& data) { Log::Assert(data.n_rows == dim); @@ -533,10 +531,8 @@ inline HRectBound< * Expands this region to encompass another bound. */ template -inline HRectBound< - MetricType, - ElemType>& HRectBound::operator|=( - const HRectBound& other) +inline HRectBound& +HRectBound::operator|=(const HRectBound& other) { assert(other.dim == dim); @@ -593,8 +589,8 @@ inline bool HRectBound::Contains( * Returns the intersection of this bound and another. */ template -inline HRectBound HRectBound:: -operator&(const HRectBound& bound) const +inline HRectBound +HRectBound::operator&(const HRectBound& bound) const { HRectBound result(dim); @@ -610,8 +606,8 @@ operator&(const HRectBound& bound) const * Intersects this bound with another. */ template -inline HRectBound& HRectBound:: -operator&=(const HRectBound& bound) +inline HRectBound& +HRectBound::operator&=(const HRectBound& bound) { for (size_t k = 0; k < dim; k++) { diff --git a/src/mlpack/core/tree/rectangle_tree/rectangle_tree.hpp b/src/mlpack/core/tree/rectangle_tree/rectangle_tree.hpp index f965a356bb..1609f0be9e 100644 --- a/src/mlpack/core/tree/rectangle_tree/rectangle_tree.hpp +++ b/src/mlpack/core/tree/rectangle_tree/rectangle_tree.hpp @@ -49,8 +49,8 @@ template - class AuxiliaryInformationType = NoAuxiliaryInformation> + template class AuxiliaryInformationType = + NoAuxiliaryInformation> class RectangleTree { // The metric *must* be the euclidean distance. From 05f40a7f7278bb660b3f08972cb7d78b7a41f756 Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Mon, 29 May 2017 13:29:44 -0400 Subject: [PATCH 63/84] Only give IRC notifications on build failures or status changes. --- .travis.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 9dbd5b170d..d486c5c4fa 100644 --- a/.travis.yml +++ b/.travis.yml @@ -20,4 +20,7 @@ notifications: email: - mlpack-git@lists.mlpack.org irc: - - "chat.freenode.net#mlpack" + channels: + - "chat.freenode.net#mlpack" + on_success: change + on_failure: always From 48676e85e9b703fcfc5dd328f4cad060c0844ef3 Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Tue, 30 May 2017 10:09:29 -0400 Subject: [PATCH 64/84] Minor style fixes. --- .../reinforcement_learning/policy/greedy_policy.hpp | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/src/mlpack/methods/reinforcement_learning/policy/greedy_policy.hpp b/src/mlpack/methods/reinforcement_learning/policy/greedy_policy.hpp index 7f38ddeecf..036cd147ed 100644 --- a/src/mlpack/methods/reinforcement_learning/policy/greedy_policy.hpp +++ b/src/mlpack/methods/reinforcement_learning/policy/greedy_policy.hpp @@ -21,18 +21,20 @@ namespace rl { * Implementation for epsilon greedy policy. * * In general we will select an action greedily based on the action value, - * however sometimes we will also randomly select an action to - * encourage exploration. + * however sometimes we will also randomly select an action to encourage + * exploration. * * @tparam EnvironmentType The reinforcement learning task. */ template -class GreedyPolicy { +class GreedyPolicy +{ public: using ActionType = typename EnvironmentType::Action; /** * Constructor for epsilon greedy policy class. + * * @param initialEpsilon The initial probability to explore (select a random action). * @param annealInterval The steps during which the probability to explore will anneal. * @param minEpsilon Epsilon will never be less than this value. @@ -47,8 +49,9 @@ class GreedyPolicy { /** * Sample an action based on given action values. + * * @param actionValue Values for each action. - * @return Sampled action + * @return Sampled action. */ ActionType Sample(const arma::colvec& actionValue) { @@ -61,7 +64,7 @@ class GreedyPolicy { // Select the action greedily. return static_cast( arma::as_scalar(arma::find(actionValue == actionValue.max(), 1))); - }; + } /** * Exploration probability will anneal at each step. From 787f0dea102e07caa08fb3456a8485d3d5fdf12f Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Wed, 31 May 2017 13:21:22 -0400 Subject: [PATCH 65/84] Add policy to select certain dimensions only for splitting. --- .../methods/decision_tree/CMakeLists.txt | 1 + .../decision_tree/all_dimension_select.hpp | 54 +++++++++++++++++++ .../methods/decision_tree/decision_tree.hpp | 6 +++ .../decision_tree/decision_tree_impl.hpp | 52 +++++++++++++++++- src/mlpack/tests/decision_tree_test.cpp | 4 +- 5 files changed, 114 insertions(+), 3 deletions(-) create mode 100644 src/mlpack/methods/decision_tree/all_dimension_select.hpp diff --git a/src/mlpack/methods/decision_tree/CMakeLists.txt b/src/mlpack/methods/decision_tree/CMakeLists.txt index 88654e8b2c..dd615e32b9 100644 --- a/src/mlpack/methods/decision_tree/CMakeLists.txt +++ b/src/mlpack/methods/decision_tree/CMakeLists.txt @@ -3,6 +3,7 @@ cmake_minimum_required(VERSION 2.8) # Define the files we need to compile. # Anything not in this list will not be compiled into mlpack. set(SOURCES + all_dimension_select.hpp decision_tree.hpp decision_tree_impl.hpp all_categorical_split.hpp diff --git a/src/mlpack/methods/decision_tree/all_dimension_select.hpp b/src/mlpack/methods/decision_tree/all_dimension_select.hpp new file mode 100644 index 0000000000..6b9681072b --- /dev/null +++ b/src/mlpack/methods/decision_tree/all_dimension_select.hpp @@ -0,0 +1,54 @@ +/** + * @file all_dimension_select.hpp + * @author Ryan Curtin + * + * Selects all dimensions for a split. + */ +#ifndef MLPACK_METHODS_DECISION_TREE_ALL_DIMENSION_SELECT_HPP +#define MLPACK_METHODS_DECISION_TREE_ALL_DIMENSION_SELECT_HPP + +namespace mlpack { +namespace tree { + +/** + * This dimension selection policy allows any dimension to be selected for + * splitting. + */ +class AllDimensionSelect +{ + public: + /** + * Construct the AllDimensionSelect object for the given number of dimensions. + */ + AllDimensionSelect(const size_t dimensions) : i(0), dimensions(dimensions) { } + + /** + * Get the first dimension to select from. + */ + size_t Begin() + { + i = 0; + return 0; + } + + /** + * Get the last dimension to select from. + */ + size_t End() const { return dimensions; } + + /** + * Get the next dimension. + */ + size_t Next() { return ++i; } + + private: + //! The current dimension we are looking at. + size_t i; + //! The number of dimensions to select from. + const size_t dimensions; +}; + +} // namespace tree +} // namespace mlpack + +#endif diff --git a/src/mlpack/methods/decision_tree/decision_tree.hpp b/src/mlpack/methods/decision_tree/decision_tree.hpp index 7258b2cfb0..9d8e743aed 100644 --- a/src/mlpack/methods/decision_tree/decision_tree.hpp +++ b/src/mlpack/methods/decision_tree/decision_tree.hpp @@ -17,6 +17,7 @@ #include "gini_gain.hpp" #include "best_binary_numeric_split.hpp" #include "all_categorical_split.hpp" +#include "all_dimension_select.hpp" #include namespace mlpack { @@ -32,6 +33,7 @@ namespace tree { template class NumericSplitType = BestBinaryNumericSplit, template class CategoricalSplitType = AllCategoricalSplit, + typename DimensionSelectionType = AllDimensionSelect, typename ElemType = double, bool NoRecursion = false> class DecisionTree : @@ -45,6 +47,8 @@ class DecisionTree : typedef NumericSplitType NumericSplit; //! Allow access to the categorical split type. typedef CategoricalSplitType CategoricalSplit; + //! Allow access to the dimension selection type. + typedef DimensionSelectionType DimensionSelection; /** * Construct the decision tree on the given data and labels, where the data @@ -414,10 +418,12 @@ class DecisionTree : template class NumericSplitType = BestBinaryNumericSplit, template class CategoricalSplitType = AllCategoricalSplit, + typename DimensionSelectType = AllDimensionSelect, typename ElemType = double> using DecisionStump = DecisionTree; diff --git a/src/mlpack/methods/decision_tree/decision_tree_impl.hpp b/src/mlpack/methods/decision_tree/decision_tree_impl.hpp index 16288f8434..7d05042ac9 100644 --- a/src/mlpack/methods/decision_tree/decision_tree_impl.hpp +++ b/src/mlpack/methods/decision_tree/decision_tree_impl.hpp @@ -19,12 +19,14 @@ namespace tree { template class NumericSplitType, template class CategoricalSplitType, + typename DimensionSelectionType, typename ElemType, bool NoRecursion> template DecisionTree::DecisionTree(MatType&& data, const data::DatasetInfo& datasetInfo, @@ -49,12 +51,14 @@ DecisionTree class NumericSplitType, template class CategoricalSplitType, + typename DimensionSelectionType, typename ElemType, bool NoRecursion> template DecisionTree::DecisionTree(MatType&& data, LabelsType&& labels, @@ -77,12 +81,14 @@ DecisionTree class NumericSplitType, template class CategoricalSplitType, + typename DimensionSelectionType, typename ElemType, bool NoRecursion> template DecisionTree::DecisionTree(MatType&& data, const data::DatasetInfo& datasetInfo, @@ -113,12 +119,14 @@ DecisionTree class NumericSplitType, template class CategoricalSplitType, + typename DimensionSelectionType, typename ElemType, bool NoRecursion> template DecisionTree::DecisionTree(MatType&& data, LabelsType&& labels, @@ -148,11 +156,13 @@ DecisionTree class NumericSplitType, template class CategoricalSplitType, + typename DimensionSelectionType, typename ElemType, bool NoRecursion> DecisionTree::DecisionTree(const size_t numClasses) : dimensionTypeOrMajorityClass(0), @@ -166,11 +176,13 @@ DecisionTree class NumericSplitType, template class CategoricalSplitType, + typename DimensionSelectionType, typename ElemType, bool NoRecursion> DecisionTree::DecisionTree(const DecisionTree& other) : NumericAuxiliarySplitInfo(other), @@ -188,11 +200,13 @@ DecisionTree class NumericSplitType, template class CategoricalSplitType, + typename DimensionSelectionType, typename ElemType, bool NoRecursion> DecisionTree::DecisionTree(DecisionTree&& other) : NumericAuxiliarySplitInfo(std::move(other)), @@ -210,16 +224,19 @@ DecisionTree class NumericSplitType, template class CategoricalSplitType, + typename DimensionSelectionType, typename ElemType, bool NoRecursion> DecisionTree& DecisionTree::operator=(const DecisionTree& other) { @@ -248,16 +265,19 @@ DecisionTree class NumericSplitType, template class CategoricalSplitType, + typename DimensionSelectionType, typename ElemType, bool NoRecursion> DecisionTree& DecisionTree::operator=(DecisionTree&& other) { @@ -286,11 +306,13 @@ DecisionTree class NumericSplitType, template class CategoricalSplitType, + typename DimensionSelectionType, typename ElemType, bool NoRecursion> DecisionTree::~DecisionTree() { @@ -302,12 +324,14 @@ DecisionTree class NumericSplitType, template class CategoricalSplitType, + typename DimensionSelectionType, typename ElemType, bool NoRecursion> template void DecisionTree::Train(MatType&& data, const data::DatasetInfo& datasetInfo, @@ -342,12 +366,14 @@ void DecisionTree class NumericSplitType, template class CategoricalSplitType, + typename DimensionSelectionType, typename ElemType, bool NoRecursion> template void DecisionTree::Train(MatType&& data, LabelsType&& labels, @@ -381,12 +407,14 @@ void DecisionTree class NumericSplitType, template class CategoricalSplitType, + typename DimensionSelectionType, typename ElemType, bool NoRecursion> template void DecisionTree::Train(MatType&& data, const data::DatasetInfo& datasetInfo, @@ -426,12 +454,14 @@ void DecisionTree class NumericSplitType, template class CategoricalSplitType, + typename DimensionSelectionType, typename ElemType, bool NoRecursion> template void DecisionTree::Train(MatType&& data, LabelsType&& labels, @@ -470,12 +500,14 @@ void DecisionTree class NumericSplitType, template class CategoricalSplitType, + typename DimensionSelectionType, typename ElemType, bool NoRecursion> template void DecisionTree::Train(MatType& data, const size_t begin, @@ -501,7 +533,9 @@ void DecisionTree class NumericSplitType, template class CategoricalSplitType, + typename DimensionSelectionType, typename ElemType, bool NoRecursion> template void DecisionTree::Train(MatType& data, const size_t begin, @@ -757,12 +793,14 @@ void DecisionTree class NumericSplitType, template class CategoricalSplitType, + typename DimensionSelectionType, typename ElemType, bool NoRecursion> template size_t DecisionTree::Classify(const VecType& point) const { @@ -779,12 +817,14 @@ size_t DecisionTree class NumericSplitType, template class CategoricalSplitType, + typename DimensionSelectionType, typename ElemType, bool NoRecursion> template void DecisionTree::Classify(const VecType& point, size_t& prediction, @@ -805,12 +845,14 @@ void DecisionTree class NumericSplitType, template class CategoricalSplitType, + typename DimensionSelectionType, typename ElemType, bool NoRecursion> template void DecisionTree::Classify(const MatType& data, arma::Row& predictions) const @@ -831,12 +873,14 @@ void DecisionTree class NumericSplitType, template class CategoricalSplitType, + typename DimensionSelectionType, typename ElemType, bool NoRecursion> template void DecisionTree::Classify(const MatType& data, arma::Row& predictions, @@ -868,12 +912,14 @@ void DecisionTree class NumericSplitType, template class CategoricalSplitType, + typename DimensionSelectionType, typename ElemType, bool NoRecursion> template void DecisionTree::Serialize(Archive& ar, const unsigned int /* version */) @@ -914,12 +960,14 @@ void DecisionTree class NumericSplitType, template class CategoricalSplitType, + typename DimensionSelectionType, typename ElemType, bool NoRecursion> template size_t DecisionTree::CalculateDirection(const VecType& point) const { @@ -935,12 +983,14 @@ size_t DecisionTree class NumericSplitType, template class CategoricalSplitType, + typename DimensionSelectionType, typename ElemType, bool NoRecursion> template void DecisionTree::CalculateClassProbabilities( const RowType& labels, diff --git a/src/mlpack/tests/decision_tree_test.cpp b/src/mlpack/tests/decision_tree_test.cpp index 37a3757dee..1704c19271 100644 --- a/src/mlpack/tests/decision_tree_test.cpp +++ b/src/mlpack/tests/decision_tree_test.cpp @@ -833,8 +833,8 @@ BOOST_AUTO_TEST_CASE(DecisionStumpTest) labels[i] = i % 3; // 3 classes. // Build a decision stump. - DecisionTree stump(dataset, labels, 3, 1); + DecisionTree stump(dataset, labels, 3, 1); // Check that it has children. BOOST_REQUIRE_EQUAL(stump.NumChildren(), 2); From fcf9244ebeb5aee47bb97a7e1acb03f10d147f6f Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Wed, 31 May 2017 14:01:52 -0400 Subject: [PATCH 66/84] Add include directories first. --- CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 3a84609915..e7b305f341 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -318,8 +318,8 @@ endif () # some reason. include(CMake/TargetDistclean.cmake OPTIONAL) -include_directories(${CMAKE_SOURCE_DIR}) -include_directories(${MLPACK_INCLUDE_DIRS}) +include_directories(BEFORE ${MLPACK_INCLUDE_DIRS}) +include_directories(BEFORE ${CMAKE_SOURCE_DIR}) # On Windows, things end up under Debug/ or Release/. if (WIN32) From bdf05df7a1730f4837792edf91e2b151eccd939c Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Wed, 31 May 2017 15:02:01 -0400 Subject: [PATCH 67/84] Include mlpack directories first. --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index e7b305f341..01e685af0f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -318,8 +318,8 @@ endif () # some reason. include(CMake/TargetDistclean.cmake OPTIONAL) -include_directories(BEFORE ${MLPACK_INCLUDE_DIRS}) include_directories(BEFORE ${CMAKE_SOURCE_DIR}) +include_directories(BEFORE ${MLPACK_INCLUDE_DIRS}) # On Windows, things end up under Debug/ or Release/. if (WIN32) From fd1cb71ccad275089057269eb426f896d5974655 Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Wed, 31 May 2017 15:24:04 -0400 Subject: [PATCH 68/84] Fix include ordering... hopefully. --- CMakeLists.txt | 2 +- src/mlpack/CMakeLists.txt | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 01e685af0f..416fb96e69 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -318,8 +318,8 @@ endif () # some reason. include(CMake/TargetDistclean.cmake OPTIONAL) -include_directories(BEFORE ${CMAKE_SOURCE_DIR}) include_directories(BEFORE ${MLPACK_INCLUDE_DIRS}) +include_directories(BEFORE ${CMAKE_SOURCE_DIR}/src/) # On Windows, things end up under Debug/ or Release/. if (WIN32) diff --git a/src/mlpack/CMakeLists.txt b/src/mlpack/CMakeLists.txt index 0aad986d5f..af3c21b487 100644 --- a/src/mlpack/CMakeLists.txt +++ b/src/mlpack/CMakeLists.txt @@ -1,4 +1,3 @@ -include_directories(..) # include_directories(${CMAKE_CURRENT_BINARY_DIR}/..) # mlpack/mlpack_export.hpp # Add core.hpp to list of sources. From bdb548d32d45a997bf69338a010a98dc6c86a352 Mon Sep 17 00:00:00 2001 From: Kirill Mishchenko Date: Mon, 29 May 2017 07:43:19 +0500 Subject: [PATCH 69/84] Add the accuracy metric for cross-validation --- src/mlpack/core/CMakeLists.txt | 1 + src/mlpack/core/cv/CMakeLists.txt | 13 +++++ src/mlpack/core/cv/metrics/CMakeLists.txt | 15 ++++++ src/mlpack/core/cv/metrics/accuracy.hpp | 52 ++++++++++++++++++++ src/mlpack/core/cv/metrics/accuracy_impl.hpp | 41 +++++++++++++++ src/mlpack/tests/CMakeLists.txt | 1 + src/mlpack/tests/cv_test.cpp | 39 +++++++++++++++ 7 files changed, 162 insertions(+) create mode 100644 src/mlpack/core/cv/CMakeLists.txt create mode 100644 src/mlpack/core/cv/metrics/CMakeLists.txt create mode 100644 src/mlpack/core/cv/metrics/accuracy.hpp create mode 100644 src/mlpack/core/cv/metrics/accuracy_impl.hpp create mode 100644 src/mlpack/tests/cv_test.cpp diff --git a/src/mlpack/core/CMakeLists.txt b/src/mlpack/core/CMakeLists.txt index d8a49bb99e..6d9194cc2f 100644 --- a/src/mlpack/core/CMakeLists.txt +++ b/src/mlpack/core/CMakeLists.txt @@ -2,6 +2,7 @@ set(DIRS arma_extend boost_backport + cv data dists kernels diff --git a/src/mlpack/core/cv/CMakeLists.txt b/src/mlpack/core/cv/CMakeLists.txt new file mode 100644 index 0000000000..0e6e541a0e --- /dev/null +++ b/src/mlpack/core/cv/CMakeLists.txt @@ -0,0 +1,13 @@ +# Define the files we need to compile +# Anything not in this list will not be compiled into mlpack. +set(SOURCES +) + +# Add directory name to sources. +set(DIR_SRCS) +foreach(file ${SOURCES}) + set(DIR_SRCS ${DIR_SRCS} ${CMAKE_CURRENT_SOURCE_DIR}/${file}) +endforeach() +# Append sources (with directory name) to list of all mlpack sources (used at +# the parent scope). +set(MLPACK_SRCS ${MLPACK_SRCS} ${DIR_SRCS} PARENT_SCOPE) diff --git a/src/mlpack/core/cv/metrics/CMakeLists.txt b/src/mlpack/core/cv/metrics/CMakeLists.txt new file mode 100644 index 0000000000..5560c092e2 --- /dev/null +++ b/src/mlpack/core/cv/metrics/CMakeLists.txt @@ -0,0 +1,15 @@ +# Define the files we need to compile +# Anything not in this list will not be compiled into mlpack. +set(SOURCES + accuracy.hpp + accuracy_impl.hpp +) + +# Add directory name to sources. +set(DIR_SRCS) +foreach(file ${SOURCES}) + set(DIR_SRCS ${DIR_SRCS} ${CMAKE_CURRENT_SOURCE_DIR}/${file}) +endforeach() +# Append sources (with directory name) to list of all mlpack sources (used at +# the parent scope). +set(MLPACK_SRCS ${MLPACK_SRCS} ${DIR_SRCS} PARENT_SCOPE) diff --git a/src/mlpack/core/cv/metrics/accuracy.hpp b/src/mlpack/core/cv/metrics/accuracy.hpp new file mode 100644 index 0000000000..71c6fd8839 --- /dev/null +++ b/src/mlpack/core/cv/metrics/accuracy.hpp @@ -0,0 +1,52 @@ +/** + * @file accuracy.hpp + * @author Kirill Mishchenko + * + * The accuracy metric. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_CORE_CV_METRICS_ACCURACY_HPP +#define MLPACK_CORE_CV_METRICS_ACCURACY_HPP + +#include + +namespace mlpack { +namespace cv { + +/** + * The Accuracy is a metric of performance for classification algorithms that is + * equal to a proportion of correctly labeled test items among all ones for + * given test items. + */ +class Accuracy +{ +public: + /** + * Run classification and calculate accuracy. + * + * @param model A test classification model. + * @data Column-major data containing test items. + * @labels Ground truth (correct) labels for the test items. + */ + template + static double Evaluate(MLAlgorithm& model, const DataType& data, + const arma::Row& labels); + + /** + * Information for hyper-parameter tuning code. It indicates that we want + * to maximize the metric. + */ + static const bool NeedsMinimization = false; +}; + +} // namespace cv +} // namespace mlpack + +// Include implementation. +#include "accuracy_impl.hpp" + +#endif diff --git a/src/mlpack/core/cv/metrics/accuracy_impl.hpp b/src/mlpack/core/cv/metrics/accuracy_impl.hpp new file mode 100644 index 0000000000..e6943d1606 --- /dev/null +++ b/src/mlpack/core/cv/metrics/accuracy_impl.hpp @@ -0,0 +1,41 @@ +/** + * @file accuracy_impl.hpp + * @author Kirill Mishchenko + * + * The implementation of the class Accuracy. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_CORE_CV_METRICS_ACCURACY_IMPL_HPP +#define MLPACK_CORE_CV_METRICS_ACCURACY_IMPL_HPP + +namespace mlpack { +namespace cv { + +template +double Accuracy::Evaluate(MLAlgorithm& model, const DataType& data, + const arma::Row& labels) +{ + if (data.n_cols != labels.n_elem) + { + std::ostringstream oss; + oss << "Accuracy::Evaluate(): number of points (" << data.n_cols << ") " + << "does not match number of labels (" << labels.n_elem << ")!" + << std::endl; + throw std::invalid_argument(oss.str()); + } + + arma::Row predictedLabels; + model.Classify(data, predictedLabels); + size_t amountOfCorrectPredictions = arma::sum(predictedLabels == labels); + + return (double) amountOfCorrectPredictions / labels.n_elem; +} + +} // namespace cv +} // namespace mlpack + +#endif diff --git a/src/mlpack/tests/CMakeLists.txt b/src/mlpack/tests/CMakeLists.txt index 694c23d877..2b34f47c03 100644 --- a/src/mlpack/tests/CMakeLists.txt +++ b/src/mlpack/tests/CMakeLists.txt @@ -18,6 +18,7 @@ add_executable(mlpack_test convolution_test.cpp convolutional_network_test.cpp cosine_tree_test.cpp + cv_test.cpp dbscan_test.cpp decision_stump_test.cpp decision_tree_test.cpp diff --git a/src/mlpack/tests/cv_test.cpp b/src/mlpack/tests/cv_test.cpp new file mode 100644 index 0000000000..ddb9ab1bc0 --- /dev/null +++ b/src/mlpack/tests/cv_test.cpp @@ -0,0 +1,39 @@ +/** + * @file cv_test.cpp + * + * Unit tests for the cross-validation module. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ + +#include +#include + +#include + +using namespace mlpack::cv; +using namespace mlpack::regression; + +BOOST_AUTO_TEST_SUITE(CVTest); + +/* + * Test the accuracy metric. + */ +BOOST_AUTO_TEST_CASE(AccuracyTest) +{ + // Making linearly separable data. + arma::mat data = + arma::mat("1 0; 2 0; 3 0; 4 0; 5 0; 1 1; 2 1; 3 1; 4 1; 5 1").t(); + arma::Row trainingLabels("0 0 0 0 0 1 1 1 1 1"); + + LogisticRegression<> lr(data, trainingLabels); + + arma::Row labels("0 0 1 0 0 1 0 1 0 1"); // 70%-correct labels + + BOOST_REQUIRE_CLOSE(Accuracy::Evaluate(lr, data, labels), 0.7, 1e-5); +} + +BOOST_AUTO_TEST_SUITE_END(); From 3bd91ac1fa51a5e0bb31d7bc811bcf75c4067343 Mon Sep 17 00:00:00 2001 From: Kirill Mishchenko Date: Fri, 2 Jun 2017 12:58:06 +0500 Subject: [PATCH 70/84] Add the mean squared error for cross-validation --- src/mlpack/core/cv/metrics/CMakeLists.txt | 2 + src/mlpack/core/cv/metrics/accuracy.hpp | 15 +++-- src/mlpack/core/cv/metrics/accuracy_impl.hpp | 7 +- src/mlpack/core/cv/metrics/mse.hpp | 67 ++++++++++++++++++++ src/mlpack/core/cv/metrics/mse_impl.hpp | 60 ++++++++++++++++++ src/mlpack/tests/cv_test.cpp | 54 ++++++++++++++++ 6 files changed, 195 insertions(+), 10 deletions(-) create mode 100644 src/mlpack/core/cv/metrics/mse.hpp create mode 100644 src/mlpack/core/cv/metrics/mse_impl.hpp diff --git a/src/mlpack/core/cv/metrics/CMakeLists.txt b/src/mlpack/core/cv/metrics/CMakeLists.txt index 5560c092e2..10445a65e6 100644 --- a/src/mlpack/core/cv/metrics/CMakeLists.txt +++ b/src/mlpack/core/cv/metrics/CMakeLists.txt @@ -3,6 +3,8 @@ set(SOURCES accuracy.hpp accuracy_impl.hpp + mse.hpp + mse_impl.hpp ) # Add directory name to sources. diff --git a/src/mlpack/core/cv/metrics/accuracy.hpp b/src/mlpack/core/cv/metrics/accuracy.hpp index 71c6fd8839..aff7c5acb5 100644 --- a/src/mlpack/core/cv/metrics/accuracy.hpp +++ b/src/mlpack/core/cv/metrics/accuracy.hpp @@ -24,17 +24,18 @@ namespace cv { */ class Accuracy { -public: + public: /** * Run classification and calculate accuracy. * - * @param model A test classification model. - * @data Column-major data containing test items. - * @labels Ground truth (correct) labels for the test items. + * @param model A classification model. + * @param data Column-major data containing test items. + * @param labels Ground truth (correct) labels for the test items. */ - template - static double Evaluate(MLAlgorithm& model, const DataType& data, - const arma::Row& labels); + template + static double Evaluate(MLAlgorithm& model, + const DataType& data, + const arma::Row& labels); /** * Information for hyper-parameter tuning code. It indicates that we want diff --git a/src/mlpack/core/cv/metrics/accuracy_impl.hpp b/src/mlpack/core/cv/metrics/accuracy_impl.hpp index e6943d1606..7b46922a79 100644 --- a/src/mlpack/core/cv/metrics/accuracy_impl.hpp +++ b/src/mlpack/core/cv/metrics/accuracy_impl.hpp @@ -15,9 +15,10 @@ namespace mlpack { namespace cv { -template -double Accuracy::Evaluate(MLAlgorithm& model, const DataType& data, - const arma::Row& labels) +template +double Accuracy::Evaluate(MLAlgorithm& model, + const DataType& data, + const arma::Row& labels) { if (data.n_cols != labels.n_elem) { diff --git a/src/mlpack/core/cv/metrics/mse.hpp b/src/mlpack/core/cv/metrics/mse.hpp new file mode 100644 index 0000000000..1c1509b297 --- /dev/null +++ b/src/mlpack/core/cv/metrics/mse.hpp @@ -0,0 +1,67 @@ +/** + * @file mse.hpp + * @author Kirill Mishchenko + * + * The mean squared error (MSE). + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_CORE_CV_METRICS_MSE_HPP +#define MLPACK_CORE_CV_METRICS_MSE_HPP + +#include + +namespace mlpack { +namespace cv { + +/** + * The MeanSquaredError is a metric of performance for regression algorithms + * that is equal to the mean squared error between predicted values and ground + * truth (correct) values for given test items. + */ +class MSE +{ + public: + /** + * Run prediction and calculate the mean squared error. + * + * @param model A regression model. + * @param data Column-major data containing test items. + * @param responses Ground truth (correct) target values for the test items, + * should be either a row vector or a column-major matrix. + */ + template + static double Evaluate(MLAlgorithm& model, + const DataType& data, + const ResponsesType& responses); + + /** + * Information for hyper-parameter tuning code. It indicates that we want + * to minimize the measurement. + */ + static const bool NeedsMinimization = true; + + private: + /* Predict rowvec responses with the model. */ + template + static void Predict(MLAlgorithm& model, + const DataType& data, + arma::rowvec& responses); + + /* Predict mat responses with the model. */ + template + static void Predict(MLAlgorithm& model, + const DataType& data, + arma::mat& responses); +}; + +} // namespace cv +} // namespace mlpack + +// Include implementation. +#include "mse_impl.hpp" + +#endif diff --git a/src/mlpack/core/cv/metrics/mse_impl.hpp b/src/mlpack/core/cv/metrics/mse_impl.hpp new file mode 100644 index 0000000000..51a2f8a13c --- /dev/null +++ b/src/mlpack/core/cv/metrics/mse_impl.hpp @@ -0,0 +1,60 @@ +/** + * @file mse_impl.hpp + * @author Kirill Mishchenko + * + * The implementation of the class MSE. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_CORE_CV_METRICS_MSE_IMPL_HPP +#define MLPACK_CORE_CV_METRICS_MSE_IMPL_HPP + +namespace mlpack { +namespace cv { + +template +double MSE::Evaluate(MLAlgorithm& model, + const DataType& data, + const ResponsesType& responses) +{ + if (data.n_cols != responses.n_cols) + { + std::ostringstream oss; + oss << "MSE::Evaluate(): number of points (" << data.n_cols << ") " + << "does not match number of responses (" << responses.n_cols << ")!" + << std::endl; + throw std::invalid_argument(oss.str()); + } + + ResponsesType predictedResponses; + Predict(model, data, predictedResponses); + double sum = arma::accu(arma::square(responses - predictedResponses)); + + return sum / responses.n_elem; +} + +template +void MSE::Predict(MLAlgorithm& model, + const DataType& data, + arma::rowvec& responses) +{ + model.Predict(data, responses); +} + +template +void MSE::Predict(MLAlgorithm& model, + const DataType& data, + arma::mat& responses) +{ + // In the case of neural networks data should be passed without const + DataType nonConstData = data; + model.Predict(nonConstData, responses); +} + +} // namespace cv +} // namespace mlpack + +#endif diff --git a/src/mlpack/tests/cv_test.cpp b/src/mlpack/tests/cv_test.cpp index ddb9ab1bc0..620d992d8d 100644 --- a/src/mlpack/tests/cv_test.cpp +++ b/src/mlpack/tests/cv_test.cpp @@ -10,11 +10,19 @@ */ #include +#include +#include +#include +#include +#include +#include #include #include +using namespace mlpack::ann; using namespace mlpack::cv; +using namespace mlpack::optimization; using namespace mlpack::regression; BOOST_AUTO_TEST_SUITE(CVTest); @@ -36,4 +44,50 @@ BOOST_AUTO_TEST_CASE(AccuracyTest) BOOST_REQUIRE_CLOSE(Accuracy::Evaluate(lr, data, labels), 0.7, 1e-5); } +/* + * Test the mean squared error. + */ +BOOST_AUTO_TEST_CASE(MSETest) +{ + // Making two points that define the linear function f(x) = x - 1 + arma::mat trainingData("0 1"); + arma::rowvec trainingResponses("-1 0"); + + LinearRegression lr(trainingData, trainingResponses); + + // Making three responses that differ from the correct ones by 0, 1, and 2 + // respectively + arma::mat data("2 3 4"); + arma::rowvec responses("1 3 5"); + + double expectedMSE = (0 * 0 + 1 * 1 + 2 * 2) / 3.0; + + BOOST_REQUIRE_CLOSE(MSE::Evaluate(lr, data, responses), expectedMSE, 1e-5); +} + +/* + * Test the mean squared error with matrix responses. + */ +BOOST_AUTO_TEST_CASE(MSEMatResponsesTest) +{ + arma::mat data("1 2"); + arma::mat trainingResponses("1 2; 3 4"); + + FFN, ZeroInitialization> ffn; + ffn.Add>(1, 2); + ffn.Add>(); + + RMSProp opt(ffn, 0.2); + opt.Shuffle() = false; + ffn.Train(data, trainingResponses, opt); + + // Making four responses that differ from the correct ones by 0, 1, 2 and 3 + // respectively + arma::mat responses("1 3; 5 7"); + + double expectedMSE = (0 * 0 + 1 * 1 + 2 * 2 + 3 * 3) / 4.0; + + BOOST_REQUIRE_CLOSE(MSE::Evaluate(ffn, data, responses), expectedMSE, 1e-1); +} + BOOST_AUTO_TEST_SUITE_END(); From 400abaa89c89eb20190fc3a3e1d0febc3d415033 Mon Sep 17 00:00:00 2001 From: Marcus Edel Date: Fri, 2 Jun 2017 16:38:41 +0200 Subject: [PATCH 71/84] Add const qualifier to the ann Predict function. --- src/mlpack/methods/ann/ffn.hpp | 2 +- src/mlpack/methods/ann/ffn_impl.hpp | 12 +++++------- src/mlpack/methods/ann/rnn.hpp | 2 +- src/mlpack/methods/ann/rnn_impl.hpp | 10 +++------- 4 files changed, 10 insertions(+), 16 deletions(-) diff --git a/src/mlpack/methods/ann/ffn.hpp b/src/mlpack/methods/ann/ffn.hpp index 05a957df04..2c439d2d0f 100644 --- a/src/mlpack/methods/ann/ffn.hpp +++ b/src/mlpack/methods/ann/ffn.hpp @@ -139,7 +139,7 @@ class FFN * @param predictors Input predictors. * @param results Matrix to put output predictions of responses into. */ - void Predict(arma::mat& predictors, arma::mat& results); + void Predict(const arma::mat& predictors, arma::mat& results); /** * Evaluate the feedforward network with the given parameters. This function diff --git a/src/mlpack/methods/ann/ffn_impl.hpp b/src/mlpack/methods/ann/ffn_impl.hpp index 130089c982..a014852055 100644 --- a/src/mlpack/methods/ann/ffn_impl.hpp +++ b/src/mlpack/methods/ann/ffn_impl.hpp @@ -67,8 +67,8 @@ FFN::~FFN() } template -void FFN::ResetData(const arma::mat &predictors, - const arma::mat &responses) +void FFN::ResetData( + const arma::mat& predictors, const arma::mat& responses) { numFunctions = responses.n_cols; this->predictors = std::move(predictors); @@ -134,7 +134,7 @@ void FFN::Train( template void FFN::Predict( - arma::mat& predictors, arma::mat& results) + const arma::mat& predictors, arma::mat& results) { if (parameter.is_empty()) { @@ -148,8 +148,7 @@ void FFN::Predict( } arma::mat resultsTemp; - Forward(std::move(arma::mat(predictors.colptr(0), - predictors.n_rows, 1, false, true))); + Forward(std::move(predictors.col(0))); resultsTemp = boost::apply_visitor(outputParameterVisitor, network.back()).col(0); @@ -158,8 +157,7 @@ void FFN::Predict( for (size_t i = 1; i < predictors.n_cols; i++) { - Forward(std::move(arma::mat(predictors.colptr(i), - predictors.n_rows, 1, false, true))); + Forward(std::move(predictors.col(i))); resultsTemp = boost::apply_visitor(outputParameterVisitor, network.back()); diff --git a/src/mlpack/methods/ann/rnn.hpp b/src/mlpack/methods/ann/rnn.hpp index fe4aaad661..185faadcee 100644 --- a/src/mlpack/methods/ann/rnn.hpp +++ b/src/mlpack/methods/ann/rnn.hpp @@ -134,7 +134,7 @@ class RNN * @param predictors Input predictors. * @param results Matrix to put output predictions of responses into. */ - void Predict(arma::mat& predictors, arma::mat& results); + void Predict(const arma::mat& predictors, arma::mat& results); /** * Evaluate the recurrent neural network with the given parameters. This diff --git a/src/mlpack/methods/ann/rnn_impl.hpp b/src/mlpack/methods/ann/rnn_impl.hpp index 07809d778f..4130ef0933 100644 --- a/src/mlpack/methods/ann/rnn_impl.hpp +++ b/src/mlpack/methods/ann/rnn_impl.hpp @@ -145,7 +145,7 @@ void RNN::Train( template void RNN::Predict( - arma::mat& predictors, arma::mat& results) + const arma::mat& predictors, arma::mat& results) { if (parameter.is_empty()) { @@ -163,10 +163,7 @@ void RNN::Predict( for (size_t i = 0; i < predictors.n_cols; i++) { - SinglePredict( - arma::mat(predictors.colptr(i), predictors.n_rows, 1, false, true), - resultsTemp); - + SinglePredict(predictors.col(i), resultsTemp); results.col(i) = resultsTemp; } } @@ -202,8 +199,7 @@ double RNN::Evaluate( ResetDeterministic(); } - arma::mat input = arma::mat(predictors.colptr(i), predictors.n_rows, - 1, false, true); + arma::mat input = predictors.col(i); arma::mat target = arma::mat(responses.colptr(i), responses.n_rows, 1, false, true); From f1b04686c3753c8d23743ae7ad566ecb64db9a38 Mon Sep 17 00:00:00 2001 From: Kirill Mishchenko Date: Mon, 5 Jun 2017 08:12:23 +0500 Subject: [PATCH 72/84] Simplify the MSE implementation Simplify the MSE implementation after merging with the master branch. --- src/mlpack/core/cv/metrics/mse.hpp | 13 ------------- src/mlpack/core/cv/metrics/mse_impl.hpp | 20 +------------------- 2 files changed, 1 insertion(+), 32 deletions(-) diff --git a/src/mlpack/core/cv/metrics/mse.hpp b/src/mlpack/core/cv/metrics/mse.hpp index 1c1509b297..b913496f86 100644 --- a/src/mlpack/core/cv/metrics/mse.hpp +++ b/src/mlpack/core/cv/metrics/mse.hpp @@ -43,19 +43,6 @@ class MSE * to minimize the measurement. */ static const bool NeedsMinimization = true; - - private: - /* Predict rowvec responses with the model. */ - template - static void Predict(MLAlgorithm& model, - const DataType& data, - arma::rowvec& responses); - - /* Predict mat responses with the model. */ - template - static void Predict(MLAlgorithm& model, - const DataType& data, - arma::mat& responses); }; } // namespace cv diff --git a/src/mlpack/core/cv/metrics/mse_impl.hpp b/src/mlpack/core/cv/metrics/mse_impl.hpp index 51a2f8a13c..92180e2b83 100644 --- a/src/mlpack/core/cv/metrics/mse_impl.hpp +++ b/src/mlpack/core/cv/metrics/mse_impl.hpp @@ -30,30 +30,12 @@ double MSE::Evaluate(MLAlgorithm& model, } ResponsesType predictedResponses; - Predict(model, data, predictedResponses); + model.Predict(data, predictedResponses); double sum = arma::accu(arma::square(responses - predictedResponses)); return sum / responses.n_elem; } -template -void MSE::Predict(MLAlgorithm& model, - const DataType& data, - arma::rowvec& responses) -{ - model.Predict(data, responses); -} - -template -void MSE::Predict(MLAlgorithm& model, - const DataType& data, - arma::mat& responses) -{ - // In the case of neural networks data should be passed without const - DataType nonConstData = data; - model.Predict(nonConstData, responses); -} - } // namespace cv } // namespace mlpack From 661e78bae89ab1a740ce39458f2598ac16c5c7b3 Mon Sep 17 00:00:00 2001 From: Kirill Mishchenko Date: Mon, 5 Jun 2017 19:05:10 +0500 Subject: [PATCH 73/84] Fix CMakeLists.txt --- src/mlpack/core/cv/CMakeLists.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/mlpack/core/cv/CMakeLists.txt b/src/mlpack/core/cv/CMakeLists.txt index 0e6e541a0e..521e43236d 100644 --- a/src/mlpack/core/cv/CMakeLists.txt +++ b/src/mlpack/core/cv/CMakeLists.txt @@ -1,3 +1,5 @@ +add_subdirectory(metrics) + # Define the files we need to compile # Anything not in this list will not be compiled into mlpack. set(SOURCES From 2f069a8d4a79d40e4fabd65eccf6d76cbc952bf9 Mon Sep 17 00:00:00 2001 From: Marcus Edel Date: Mon, 5 Jun 2017 19:34:31 +0200 Subject: [PATCH 74/84] Add space between code and comment. --- src/mlpack/core.hpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/mlpack/core.hpp b/src/mlpack/core.hpp index 20faeb740c..f9a9ea74a2 100644 --- a/src/mlpack/core.hpp +++ b/src/mlpack/core.hpp @@ -244,7 +244,8 @@ #include #include #include -//mlpack::backtrace only for linux + +// mlpack::backtrace only for linux #ifdef HAS_BFD_DL #include #endif From 5a53431e7def4901bbe6cc313f1158cb7fb4ce65 Mon Sep 17 00:00:00 2001 From: Marcus Edel Date: Mon, 5 Jun 2017 21:03:02 +0200 Subject: [PATCH 75/84] Fix style issues pointed out by cpplint. --- src/mlpack/methods/adaboost/adaboost.hpp | 3 +- .../simple_residue_termination.hpp | 2 +- .../simple_tolerance_termination.hpp | 6 +- .../validation_RMSE_termination.hpp | 10 +- .../methods/amf/update_rules/nmf_mult_div.hpp | 6 +- .../svd_complete_incremental_learning.hpp | 12 +- .../svd_incomplete_incremental_learning.hpp | 28 ++-- .../identity_function.hpp | 2 - .../logistic_function.hpp | 2 +- .../softplus_function.hpp | 3 +- .../softsign_function.hpp | 2 +- .../activation_functions/tanh_function.hpp | 2 +- .../ann/convolution_rules/fft_convolution.hpp | 1 - .../convolution_rules/naive_convolution.hpp | 1 - .../ann/convolution_rules/svd_convolution.hpp | 1 - src/mlpack/methods/ann/ffn.hpp | 3 +- src/mlpack/methods/ann/ffn_impl.hpp | 3 +- src/mlpack/methods/ann/layer/concat_impl.hpp | 3 +- src/mlpack/methods/ann/layer/constant.hpp | 2 +- src/mlpack/methods/ann/layer/convolution.hpp | 5 +- .../methods/ann/layer/convolution_impl.hpp | 6 +- src/mlpack/methods/ann/layer/dropconnect.hpp | 2 +- src/mlpack/methods/ann/layer/elu.hpp | 1 - src/mlpack/methods/ann/layer/glimpse.hpp | 6 +- src/mlpack/methods/ann/layer/layer_traits.hpp | 4 +- src/mlpack/methods/ann/layer/leaky_relu.hpp | 3 +- .../methods/ann/layer/linear_no_bias.hpp | 1 - .../methods/ann/layer/log_softmax_impl.hpp | 4 +- src/mlpack/methods/ann/layer/lookup.hpp | 1 - src/mlpack/methods/ann/layer/lstm.hpp | 1 - src/mlpack/methods/ann/layer/max_pooling.hpp | 3 +- src/mlpack/methods/ann/layer/mean_pooling.hpp | 7 +- .../methods/ann/layer/mean_pooling_impl.hpp | 3 - .../methods/ann/layer/parametric_relu.hpp | 3 +- .../ann/layer/parametric_relu_impl.hpp | 2 +- .../methods/ann/layer/recurrent_impl.hpp | 2 +- src/mlpack/methods/ann/layer/sequential.hpp | 1 - .../methods/ann/visitor/copy_visitor.hpp | 2 +- .../approx_kfn/drusilla_select_impl.hpp | 3 +- src/mlpack/methods/cf/cf.cpp | 2 +- src/mlpack/methods/cf/cf_impl.hpp | 4 +- src/mlpack/methods/cf/svd_wrapper.hpp | 7 +- src/mlpack/methods/cf/svd_wrapper_impl.hpp | 12 +- .../decision_tree/decision_tree_impl.hpp | 15 ++- .../decision_tree/decision_tree_main.cpp | 4 +- src/mlpack/methods/det/dt_utils_impl.hpp | 3 +- src/mlpack/methods/det/dtree.hpp | 2 - src/mlpack/methods/det/dtree_impl.hpp | 31 ++--- src/mlpack/methods/emst/dtb.hpp | 1 - src/mlpack/methods/emst/dtb_impl.hpp | 2 +- src/mlpack/methods/emst/dtb_rules.hpp | 1 - src/mlpack/methods/emst/dtb_stat.hpp | 1 - src/mlpack/methods/emst/edge_pair.hpp | 1 - src/mlpack/methods/gmm/gmm_impl.hpp | 6 +- src/mlpack/methods/hmm/hmm_regression.hpp | 14 +- .../methods/hmm/hmm_regression_impl.hpp | 7 +- .../binary_numeric_split_impl.hpp | 2 +- .../hoeffding_numeric_split_impl.hpp | 1 - .../hoeffding_trees/hoeffding_tree_impl.hpp | 1 - src/mlpack/methods/kernel_pca/kernel_pca.hpp | 1 - .../methods/kernel_pca/kernel_pca_main.cpp | 8 +- .../kernel_pca/kernel_rules/naive_method.hpp | 122 +++++++++--------- .../kernel_rules/nystroem_method.hpp | 88 ++++++------- .../methods/kmeans/elkan_kmeans_impl.hpp | 2 +- src/mlpack/methods/kmeans/kmeans_impl.hpp | 1 - .../kmeans/max_variance_new_cluster_impl.hpp | 7 +- .../methods/kmeans/random_partition.hpp | 4 +- .../linear_regression/linear_regression.cpp | 8 +- .../linear_regression/linear_regression.hpp | 2 +- .../logistic_regression_function_impl.hpp | 2 +- src/mlpack/methods/lsh/lsh_search.hpp | 1 - src/mlpack/methods/lsh/lsh_search_impl.hpp | 29 ++--- .../matrix_completion/matrix_completion.cpp | 22 ++-- .../matrix_completion/matrix_completion.hpp | 5 +- src/mlpack/methods/mvu/mvu.cpp | 1 - src/mlpack/methods/naive_bayes/nbc_main.cpp | 8 +- .../neighbor_search/neighbor_search.hpp | 6 +- .../neighbor_search/neighbor_search_impl.hpp | 2 +- src/mlpack/methods/pca/pca_impl.hpp | 4 +- src/mlpack/methods/pca/pca_main.cpp | 1 - src/mlpack/methods/perceptron/perceptron.hpp | 2 +- .../methods/perceptron/perceptron_main.cpp | 7 +- .../preprocess/preprocess_describe_main.cpp | 2 +- .../preprocess/preprocess_imputer_main.cpp | 3 +- src/mlpack/methods/radical/radical.cpp | 2 +- .../range_search/range_search_stat.hpp | 2 +- src/mlpack/methods/range_search/rs_model.cpp | 4 +- .../methods/range_search/rs_model_impl.hpp | 12 +- src/mlpack/methods/rann/krann_main.cpp | 6 +- .../methods/rann/ra_search_rules_impl.hpp | 2 +- src/mlpack/methods/rann/ra_util.cpp | 4 +- .../regularized_svd/regularized_svd.hpp | 1 - .../regularized_svd_function.cpp | 8 +- .../regularized_svd_function.hpp | 1 - .../environment/cart_pole.hpp | 8 +- .../environment/mountain_car.hpp | 8 +- .../replay/random_replay.hpp | 5 +- .../softmax_regression/softmax_regression.hpp | 8 +- .../softmax_regression_function.cpp | 8 +- .../softmax_regression_impl.hpp | 5 +- .../sparse_autoencoder_function.cpp | 2 +- .../methods/sparse_coding/sparse_coding.cpp | 13 +- src/mlpack/tests/ada_grad_test.cpp | 3 +- src/mlpack/tests/adaboost_test.cpp | 22 ++-- src/mlpack/tests/ann_layer_test.cpp | 6 +- src/mlpack/tests/cli_test.cpp | 35 ++--- src/mlpack/tests/cosine_tree_test.cpp | 6 +- src/mlpack/tests/decision_stump_test.cpp | 3 +- src/mlpack/tests/decision_tree_test.cpp | 60 +++++---- src/mlpack/tests/det_test.cpp | 44 ++++--- src/mlpack/tests/distribution_test.cpp | 1 - src/mlpack/tests/emst_test.cpp | 2 - src/mlpack/tests/fastmks_test.cpp | 15 ++- src/mlpack/tests/feedforward_network_test.cpp | 2 +- src/mlpack/tests/gmm_test.cpp | 7 +- src/mlpack/tests/hmm_test.cpp | 19 +-- src/mlpack/tests/ind2sub_test.cpp | 2 +- src/mlpack/tests/kernel_test.cpp | 27 ++-- src/mlpack/tests/knn_test.cpp | 2 - src/mlpack/tests/ksinit_test.cpp | 4 +- src/mlpack/tests/lars_test.cpp | 5 +- src/mlpack/tests/load_save_test.cpp | 19 ++- .../tests/local_coordinate_coding_test.cpp | 2 +- src/mlpack/tests/lsh_test.cpp | 19 ++- src/mlpack/tests/maximal_inputs_test.cpp | 2 +- src/mlpack/tests/mean_shift_test.cpp | 5 +- src/mlpack/tests/momentum_sgd_test.cpp | 7 +- src/mlpack/tests/nbc_test.cpp | 5 +- src/mlpack/tests/nca_test.cpp | 1 - src/mlpack/tests/nmf_test.cpp | 3 +- src/mlpack/tests/nystroem_method_test.cpp | 6 +- src/mlpack/tests/octree_test.cpp | 8 +- src/mlpack/tests/pca_test.cpp | 5 +- src/mlpack/tests/perceptron_test.cpp | 1 - src/mlpack/tests/prefixedoutstream_test.cpp | 1 - src/mlpack/tests/rectangle_tree_test.cpp | 10 +- src/mlpack/tests/recurrent_network_test.cpp | 104 +++++++-------- src/mlpack/tests/rl_components_test.cpp | 8 +- src/mlpack/tests/sa_test.cpp | 4 +- src/mlpack/tests/sdp_primal_dual_test.cpp | 52 ++++---- src/mlpack/tests/smorms3_test.cpp | 2 +- src/mlpack/tests/softmax_regression_test.cpp | 16 +-- src/mlpack/tests/sparse_autoencoder_test.cpp | 4 +- src/mlpack/tests/sparse_coding_test.cpp | 4 +- src/mlpack/tests/svd_incremental_test.cpp | 4 +- src/mlpack/tests/test_tools.hpp | 10 +- src/mlpack/tests/tree_test.cpp | 5 +- src/mlpack/tests/ub_tree_test.cpp | 1 - src/mlpack/tests/vantage_point_tree_test.cpp | 4 +- 149 files changed, 637 insertions(+), 646 deletions(-) diff --git a/src/mlpack/methods/adaboost/adaboost.hpp b/src/mlpack/methods/adaboost/adaboost.hpp index d2732824df..810cfa09f7 100644 --- a/src/mlpack/methods/adaboost/adaboost.hpp +++ b/src/mlpack/methods/adaboost/adaboost.hpp @@ -161,7 +161,7 @@ class AdaBoost template void Serialize(Archive& ar, const unsigned int /* version */); -private: + private: //! The number of classes in the model. size_t classes; // The tolerance for change in rt and when to stop. @@ -174,7 +174,6 @@ private: //! To check for the bound for the Hamming loss. double ztProduct; - }; // class AdaBoost } // namespace adaboost diff --git a/src/mlpack/methods/amf/termination_policies/simple_residue_termination.hpp b/src/mlpack/methods/amf/termination_policies/simple_residue_termination.hpp index bc659f221a..fb1d2b94a3 100644 --- a/src/mlpack/methods/amf/termination_policies/simple_residue_termination.hpp +++ b/src/mlpack/methods/amf/termination_policies/simple_residue_termination.hpp @@ -99,7 +99,7 @@ class SimpleResidueTermination const double& MinResidue() const { return minResidue; } double& MinResidue() { return minResidue; } -public: + public: //! residue threshold double minResidue; //! iteration threshold diff --git a/src/mlpack/methods/amf/termination_policies/simple_tolerance_termination.hpp b/src/mlpack/methods/amf/termination_policies/simple_tolerance_termination.hpp index 1e29c27bb8..08341cc3e3 100644 --- a/src/mlpack/methods/amf/termination_policies/simple_tolerance_termination.hpp +++ b/src/mlpack/methods/amf/termination_policies/simple_tolerance_termination.hpp @@ -78,12 +78,12 @@ class SimpleToleranceTermination size_t m = V->n_cols; double sum = 0; size_t count = 0; - for(size_t i = 0;i < n;i++) + for (size_t i = 0;i < n; i++) { - for(size_t j = 0;j < m;j++) + for (size_t j = 0;j < m; j++) { double temp = 0; - if ((temp = (*V)(i,j)) != 0) + if ((temp = (*V)(i, j)) != 0) { temp = (temp - WH(i, j)); temp = temp * temp; diff --git a/src/mlpack/methods/amf/termination_policies/validation_RMSE_termination.hpp b/src/mlpack/methods/amf/termination_policies/validation_RMSE_termination.hpp index 00053c3647..9c95c9867c 100644 --- a/src/mlpack/methods/amf/termination_policies/validation_RMSE_termination.hpp +++ b/src/mlpack/methods/amf/termination_policies/validation_RMSE_termination.hpp @@ -63,7 +63,7 @@ class ValidationRMSETermination test_points.zeros(num_test_points, 3); // fill validation set matrix with random chosen entries - for(size_t i = 0; i < num_test_points; i++) + for (size_t i = 0; i < num_test_points; i++) { double t_val; size_t t_row; @@ -72,9 +72,9 @@ class ValidationRMSETermination // pick a random non-zero entry do { - t_row = rand() % n; - t_col = rand() % m; - } while((t_val = V(t_row, t_col)) == 0); + t_row = rand_r() % n; + t_col = rand_r() % m; + } while ((t_val = V(t_row, t_col)) == 0); // add the entry to the validation set test_points(i, 0) = t_row; @@ -122,7 +122,7 @@ class ValidationRMSETermination { rmseOld = rmse; rmse = 0; - for(size_t i = 0; i < num_test_points; i++) + for (size_t i = 0; i < num_test_points; i++) { size_t t_row = test_points(i, 0); size_t t_col = test_points(i, 1); diff --git a/src/mlpack/methods/amf/update_rules/nmf_mult_div.hpp b/src/mlpack/methods/amf/update_rules/nmf_mult_div.hpp index 567b45feb0..38899e0c29 100644 --- a/src/mlpack/methods/amf/update_rules/nmf_mult_div.hpp +++ b/src/mlpack/methods/amf/update_rules/nmf_mult_div.hpp @@ -93,7 +93,7 @@ class NMFMultiplicativeDivergenceUpdate // Writing this as a single expression does not work as of Armadillo // 3.920. This should be fixed in a future release, and then the code // below can be fixed. - //t2 = H.row(j) % V.row(i) / t1.row(i); + // t2 = H.row(j) % V.row(i) / t1.row(i); t2.set_size(H.n_cols); for (size_t k = 0; k < t2.n_elem; ++k) { @@ -137,14 +137,14 @@ class NMFMultiplicativeDivergenceUpdate // Writing this as a single expression does not work as of Armadillo // 3.920. This should be fixed in a future release, and then the code // below can be fixed. - //t2 = W.col(i) % V.col(j) / t1.col(j); + // t2 = W.col(i) % V.col(j) / t1.col(j); t2.set_size(W.n_rows); for (size_t k = 0; k < t2.n_elem; ++k) { t2(k) = W(k, i) * V(k, j) / t1(k, j); } - H(i,j) = H(i,j) * sum(t2) / sum(W.col(i)); + H(i, j) = H(i, j) * sum(t2) / sum(W.col(i)); } } } diff --git a/src/mlpack/methods/amf/update_rules/svd_complete_incremental_learning.hpp b/src/mlpack/methods/amf/update_rules/svd_complete_incremental_learning.hpp index 7e8a0d359f..22174a0f7c 100644 --- a/src/mlpack/methods/amf/update_rules/svd_complete_incremental_learning.hpp +++ b/src/mlpack/methods/amf/update_rules/svd_complete_incremental_learning.hpp @@ -92,7 +92,7 @@ class SVDCompleteIncrementalLearning deltaW.zeros(1, W.n_cols); // Loop until a non-zero entry is found. - while(true) + while (true) { const double val = V(currentItemIndex, currentUserIndex); // Update feature vector if current entry is non-zero and break the loop. @@ -168,7 +168,7 @@ class SVDCompleteIncrementalLearning template<> class SVDCompleteIncrementalLearning { - public: + public: SVDCompleteIncrementalLearning(double u = 0.01, double kw = 0, double kh = 0) @@ -218,8 +218,8 @@ class SVDCompleteIncrementalLearning arma::mat deltaW(1, W.n_cols); deltaW.zeros(); - deltaW += (**it - arma::dot(W.row(currentItemIndex), H.col(currentUserIndex))) - * arma::trans(H.col(currentUserIndex)); + deltaW += (**it - arma::dot(W.row(currentItemIndex), + H.col(currentUserIndex))) * arma::trans(H.col(currentUserIndex)); if (kw != 0) deltaW -= kw * W.row(currentItemIndex); W.row(currentItemIndex) += u*deltaW; @@ -246,8 +246,8 @@ class SVDCompleteIncrementalLearning size_t currentUserIndex = it->col(); size_t currentItemIndex = it->row(); - deltaH += (**it - arma::dot(W.row(currentItemIndex), H.col(currentUserIndex))) - * arma::trans(W.row(currentItemIndex)); + deltaH += (**it - arma::dot(W.row(currentItemIndex), + H.col(currentUserIndex))) * arma::trans(W.row(currentItemIndex)); if (kh != 0) deltaH -= kh * H.col(currentUserIndex); H.col(currentUserIndex) += u * deltaH; diff --git a/src/mlpack/methods/amf/update_rules/svd_incomplete_incremental_learning.hpp b/src/mlpack/methods/amf/update_rules/svd_incomplete_incremental_learning.hpp index 43bce411cc..13da871457 100644 --- a/src/mlpack/methods/amf/update_rules/svd_incomplete_incremental_learning.hpp +++ b/src/mlpack/methods/amf/update_rules/svd_incomplete_incremental_learning.hpp @@ -97,8 +97,10 @@ class SVDIncompleteIncrementalLearning const double val = V(i, currentUserIndex); // Update only if the rating is non-zero. if (val != 0) + { deltaW.row(i) += (val - arma::dot(W.row(i), H.col(currentUserIndex))) * H.col(currentUserIndex).t(); + } // Add regularization. if (kw != 0) deltaW.row(i) -= kw * W.row(i); @@ -130,8 +132,10 @@ class SVDIncompleteIncrementalLearning const double val = V(i, currentUserIndex); // Update only if the rating is non-zero. if (val != 0) + { deltaH += (val - arma::dot(W.row(i), H.col(currentUserIndex))) * W.row(i).t(); + } } // Add regularization. if (kh != 0) @@ -159,20 +163,18 @@ class SVDIncompleteIncrementalLearning //! template specialiazed functions for sparse matrices template<> -inline void SVDIncompleteIncrementalLearning:: - WUpdate(const arma::sp_mat& V, - arma::mat& W, - const arma::mat& H) +inline void SVDIncompleteIncrementalLearning::WUpdate( + const arma::sp_mat& V, arma::mat& W, const arma::mat& H) { arma::mat deltaW(V.n_rows, W.n_cols); deltaW.zeros(); - for(arma::sp_mat::const_iterator it = V.begin_col(currentUserIndex); - it != V.end_col(currentUserIndex);it++) + for (arma::sp_mat::const_iterator it = V.begin_col(currentUserIndex); + it != V.end_col(currentUserIndex); it++) { double val = *it; size_t i = it.row(); deltaW.row(i) += (val - arma::dot(W.row(i), H.col(currentUserIndex))) * - arma::trans(H.col(currentUserIndex)); + arma::trans(H.col(currentUserIndex)); if (kw != 0) deltaW.row(i) -= kw * W.row(i); } @@ -180,22 +182,22 @@ inline void SVDIncompleteIncrementalLearning:: } template<> -inline void SVDIncompleteIncrementalLearning:: - HUpdate(const arma::sp_mat& V, - const arma::mat& W, - arma::mat& H) +inline void SVDIncompleteIncrementalLearning::HUpdate( + const arma::sp_mat& V, const arma::mat& W, arma::mat& H) { arma::mat deltaH(H.n_rows, 1); deltaH.zeros(); for(arma::sp_mat::const_iterator it = V.begin_col(currentUserIndex); - it != V.end_col(currentUserIndex);it++) + it != V.end_col(currentUserIndex); it++) { double val = *it; size_t i = it.row(); if ((val = V(i, currentUserIndex)) != 0) + { deltaH += (val - arma::dot(W.row(i), H.col(currentUserIndex))) * - arma::trans(W.row(i)); + arma::trans(W.row(i)); + } } if (kh != 0) deltaH -= kh * H.col(currentUserIndex); diff --git a/src/mlpack/methods/ann/activation_functions/identity_function.hpp b/src/mlpack/methods/ann/activation_functions/identity_function.hpp index 0de88e57ec..14ab9a8338 100644 --- a/src/mlpack/methods/ann/activation_functions/identity_function.hpp +++ b/src/mlpack/methods/ann/activation_functions/identity_function.hpp @@ -86,8 +86,6 @@ class IdentityFunction { x.ones(y.n_rows, y.n_cols, y.n_slices); } - - }; // class IdentityFunction } // namespace ann diff --git a/src/mlpack/methods/ann/activation_functions/logistic_function.hpp b/src/mlpack/methods/ann/activation_functions/logistic_function.hpp index 75ddd334d4..20e1622270 100644 --- a/src/mlpack/methods/ann/activation_functions/logistic_function.hpp +++ b/src/mlpack/methods/ann/activation_functions/logistic_function.hpp @@ -28,7 +28,7 @@ namespace ann /** Artificial Neural Network. */ { */ class LogisticFunction { - public: + public: /** * Computes the logistic function. * diff --git a/src/mlpack/methods/ann/activation_functions/softplus_function.hpp b/src/mlpack/methods/ann/activation_functions/softplus_function.hpp index 4adc5d746c..dd5875ff83 100644 --- a/src/mlpack/methods/ann/activation_functions/softplus_function.hpp +++ b/src/mlpack/methods/ann/activation_functions/softplus_function.hpp @@ -42,8 +42,7 @@ namespace ann /** Artificial Neural Network. */ { */ class SoftplusFunction { - public: - + public: /** * Computes the softplus function. * diff --git a/src/mlpack/methods/ann/activation_functions/softsign_function.hpp b/src/mlpack/methods/ann/activation_functions/softsign_function.hpp index 2438ee3c7c..eb9db79b69 100644 --- a/src/mlpack/methods/ann/activation_functions/softsign_function.hpp +++ b/src/mlpack/methods/ann/activation_functions/softsign_function.hpp @@ -46,7 +46,7 @@ namespace ann /** Artificial Neural Network. */ { */ class SoftsignFunction { - public: + public: /** * Computes the softsign function. * diff --git a/src/mlpack/methods/ann/activation_functions/tanh_function.hpp b/src/mlpack/methods/ann/activation_functions/tanh_function.hpp index aea406a3d4..63abf0f1ec 100644 --- a/src/mlpack/methods/ann/activation_functions/tanh_function.hpp +++ b/src/mlpack/methods/ann/activation_functions/tanh_function.hpp @@ -28,7 +28,7 @@ namespace ann /** Artificial Neural Network. */ { */ class TanhFunction { - public: + public: /** * Computes the tanh function. * diff --git a/src/mlpack/methods/ann/convolution_rules/fft_convolution.hpp b/src/mlpack/methods/ann/convolution_rules/fft_convolution.hpp index 4eaa038380..f573ce9ca2 100644 --- a/src/mlpack/methods/ann/convolution_rules/fft_convolution.hpp +++ b/src/mlpack/methods/ann/convolution_rules/fft_convolution.hpp @@ -212,7 +212,6 @@ class FFTConvolution output.slice(i) = convOutput; } } - }; // class FFTConvolution } // namespace ann diff --git a/src/mlpack/methods/ann/convolution_rules/naive_convolution.hpp b/src/mlpack/methods/ann/convolution_rules/naive_convolution.hpp index c60d9e5b28..c27225f127 100644 --- a/src/mlpack/methods/ann/convolution_rules/naive_convolution.hpp +++ b/src/mlpack/methods/ann/convolution_rules/naive_convolution.hpp @@ -201,7 +201,6 @@ class NaiveConvolution output.slice(i), dW, dH); } } - }; // class NaiveConvolution } // namespace ann diff --git a/src/mlpack/methods/ann/convolution_rules/svd_convolution.hpp b/src/mlpack/methods/ann/convolution_rules/svd_convolution.hpp index b532ac1a6b..7cd50470ab 100644 --- a/src/mlpack/methods/ann/convolution_rules/svd_convolution.hpp +++ b/src/mlpack/methods/ann/convolution_rules/svd_convolution.hpp @@ -198,7 +198,6 @@ class SVDConvolution output.slice(i) = convOutput; } } - }; // class SVDConvolution } // namespace ann diff --git a/src/mlpack/methods/ann/ffn.hpp b/src/mlpack/methods/ann/ffn.hpp index 2c439d2d0f..f297a56c4e 100644 --- a/src/mlpack/methods/ann/ffn.hpp +++ b/src/mlpack/methods/ann/ffn.hpp @@ -209,7 +209,7 @@ class FFN template void Serialize(Archive& ar, const unsigned int /* version */); -private: + private: // Helper functions. /** * The Forward algorithm (part of the Forward-Backward algorithm). Computes @@ -336,7 +336,6 @@ private: //! Locally-stored copy visitor CopyVisitor copyVisitor; - }; // class FFN } // namespace ann diff --git a/src/mlpack/methods/ann/ffn_impl.hpp b/src/mlpack/methods/ann/ffn_impl.hpp index a014852055..38de848934 100644 --- a/src/mlpack/methods/ann/ffn_impl.hpp +++ b/src/mlpack/methods/ann/ffn_impl.hpp @@ -424,7 +424,8 @@ FFN::FFN( // Build new layers according to source network for (size_t i = 0; i < network.network.size(); ++i) { - this->network.push_back(boost::apply_visitor(copyVisitor, network.network[i])); + this->network.push_back(boost::apply_visitor(copyVisitor, + network.network[i])); } }; diff --git a/src/mlpack/methods/ann/layer/concat_impl.hpp b/src/mlpack/methods/ann/layer/concat_impl.hpp index 1f3ffd47fc..5e3d4cef6b 100644 --- a/src/mlpack/methods/ann/layer/concat_impl.hpp +++ b/src/mlpack/methods/ann/layer/concat_impl.hpp @@ -143,7 +143,8 @@ void Concat::Gradient( { for (size_t i = 0; i < network.size(); ++i) { - boost::apply_visitor(GradientVisitor(std::move(input), std::move(error)), network[i]); + boost::apply_visitor(GradientVisitor(std::move(input), + std::move(error)), network[i]); } } diff --git a/src/mlpack/methods/ann/layer/constant.hpp b/src/mlpack/methods/ann/layer/constant.hpp index 1a2c8e9e52..54c9850a8d 100644 --- a/src/mlpack/methods/ann/layer/constant.hpp +++ b/src/mlpack/methods/ann/layer/constant.hpp @@ -113,4 +113,4 @@ class Constant // Include implementation. #include "constant_impl.hpp" -#endif \ No newline at end of file +#endif diff --git a/src/mlpack/methods/ann/layer/convolution.hpp b/src/mlpack/methods/ann/layer/convolution.hpp index bfef18aa4f..cca4eb21dc 100644 --- a/src/mlpack/methods/ann/layer/convolution.hpp +++ b/src/mlpack/methods/ann/layer/convolution.hpp @@ -45,7 +45,7 @@ template < > class Convolution { -public: + public: //! Create the Convolution object. Convolution(); @@ -168,7 +168,6 @@ public: void Serialize(Archive& ar, const unsigned int /* version */); private: - /* * Return the convolution output size. * @@ -341,4 +340,4 @@ public: // Include implementation. #include "convolution_impl.hpp" -#endif \ No newline at end of file +#endif diff --git a/src/mlpack/methods/ann/layer/convolution_impl.hpp b/src/mlpack/methods/ann/layer/convolution_impl.hpp index 422c994709..d0758b29ca 100644 --- a/src/mlpack/methods/ann/layer/convolution_impl.hpp +++ b/src/mlpack/methods/ann/layer/convolution_impl.hpp @@ -287,12 +287,8 @@ void Convolution< outMap, outMap)); } - // gradient.submat(0, 0, weight.n_elem - 1, 0) = arma::vectorise(gradientTemp); gradient.submat(0, 0, weight.n_elem - 1, 0) = arma::Mat( gradientTemp.memptr(), gradientTemp.n_elem, 1, false, false); - - - // arma::vectorise(gradientTemp); } template< @@ -330,4 +326,4 @@ void Convolution< } // namespace ann } // namespace mlpack -#endif \ No newline at end of file +#endif diff --git a/src/mlpack/methods/ann/layer/dropconnect.hpp b/src/mlpack/methods/ann/layer/dropconnect.hpp index 80e00bfa1c..aa4309b464 100644 --- a/src/mlpack/methods/ann/layer/dropconnect.hpp +++ b/src/mlpack/methods/ann/layer/dropconnect.hpp @@ -162,7 +162,7 @@ class DropConnect template void Serialize(Archive& ar, const unsigned int /* version */); -private: + private: //! The probability of setting a value to zero. double ratio; diff --git a/src/mlpack/methods/ann/layer/elu.hpp b/src/mlpack/methods/ann/layer/elu.hpp index 7a40d453c4..2eb8363ec6 100644 --- a/src/mlpack/methods/ann/layer/elu.hpp +++ b/src/mlpack/methods/ann/layer/elu.hpp @@ -189,7 +189,6 @@ class ELU //! ELU Hyperparameter (0 < alpha) double alpha; - }; // class ELU } // namespace ann diff --git a/src/mlpack/methods/ann/layer/glimpse.hpp b/src/mlpack/methods/ann/layer/glimpse.hpp index e73dd62457..6b4420a8d4 100644 --- a/src/mlpack/methods/ann/layer/glimpse.hpp +++ b/src/mlpack/methods/ann/layer/glimpse.hpp @@ -82,7 +82,6 @@ template < class Glimpse { public: - /** * Create the GlimpseLayer object using the specified ratio and rescale * parameter. @@ -145,7 +144,7 @@ class Glimpse this->location = location; } - //! Get the input width. + //! Get the input width. size_t const& InputWidth() const { return inputWidth; } //! Modify input the width. size_t& InputWidth() { return inputWidth; } @@ -222,7 +221,6 @@ class Glimpse const arma::Mat& input, arma::Mat& output) { - const size_t rStep = kSize; const size_t cStep = kSize; @@ -427,4 +425,4 @@ class Glimpse // Include implementation. #include "glimpse_impl.hpp" -#endif \ No newline at end of file +#endif diff --git a/src/mlpack/methods/ann/layer/layer_traits.hpp b/src/mlpack/methods/ann/layer/layer_traits.hpp index ff4fbf2d38..5b9d4c61cd 100644 --- a/src/mlpack/methods/ann/layer/layer_traits.hpp +++ b/src/mlpack/methods/ann/layer/layer_traits.hpp @@ -64,8 +64,8 @@ HAS_MEM_FUNC(Gradient, HasGradientCheck); // function. HAS_MEM_FUNC(Deterministic, HasDeterministicCheck); -// This gives us a HasParametersCheck type (where U is a function pointer) we -// can use with SFINAE to catch when a type has a Weights() function. +// This gives us a HasParametersCheck type (where U is a function pointer) +// we can use with SFINAE to catch when a type has a Weights() function. HAS_MEM_FUNC(Parameters, HasParametersCheck); // This gives us a HasAddCheck type (where U is a function pointer) we diff --git a/src/mlpack/methods/ann/layer/leaky_relu.hpp b/src/mlpack/methods/ann/layer/leaky_relu.hpp index 8c600f9cff..2e5b423293 100644 --- a/src/mlpack/methods/ann/layer/leaky_relu.hpp +++ b/src/mlpack/methods/ann/layer/leaky_relu.hpp @@ -165,7 +165,6 @@ class LeakyReLU //! Leakyness Parameter in the range 0 ::Forward( // Approximation of the hyperbolic tangent. The acuracy however is // about 0.00001 lower as using tanh. Credits go to Leon Bottou. - output.transform( [](double x) + output.transform([](double x) { //! Fast approximation of exp(-x) for x positive. static constexpr double A0 = 1.0; @@ -55,7 +55,7 @@ void LogSoftMax::Forward( } return 0.0; - } ); + }); output = input - (maxInput + std::log(arma::accu(output))); } diff --git a/src/mlpack/methods/ann/layer/lookup.hpp b/src/mlpack/methods/ann/layer/lookup.hpp index 7e0a5a29fc..e89f3e43f2 100644 --- a/src/mlpack/methods/ann/layer/lookup.hpp +++ b/src/mlpack/methods/ann/layer/lookup.hpp @@ -112,7 +112,6 @@ class Lookup void Serialize(Archive& ar, const unsigned int /* version */); private: - //! Locally-stored number of input units. size_t inSize; diff --git a/src/mlpack/methods/ann/layer/lstm.hpp b/src/mlpack/methods/ann/layer/lstm.hpp index 9c671fea85..773f97bb69 100644 --- a/src/mlpack/methods/ann/layer/lstm.hpp +++ b/src/mlpack/methods/ann/layer/lstm.hpp @@ -139,7 +139,6 @@ class LSTM void Serialize(Archive& ar, const unsigned int /* version */); private: - //! Locally-stored number of input units. size_t inSize; diff --git a/src/mlpack/methods/ann/layer/max_pooling.hpp b/src/mlpack/methods/ann/layer/max_pooling.hpp index faa7f6a9d6..59497a5da7 100644 --- a/src/mlpack/methods/ann/layer/max_pooling.hpp +++ b/src/mlpack/methods/ann/layer/max_pooling.hpp @@ -51,7 +51,7 @@ template < > class MaxPooling { -public: + public: //! Create the MaxPooling object. MaxPooling(); @@ -141,7 +141,6 @@ public: void Serialize(Archive& ar, const unsigned int /* version */); private: - /** * Apply pooling to the input and store the results. * diff --git a/src/mlpack/methods/ann/layer/mean_pooling.hpp b/src/mlpack/methods/ann/layer/mean_pooling.hpp index 8c29df7689..d52f43309e 100644 --- a/src/mlpack/methods/ann/layer/mean_pooling.hpp +++ b/src/mlpack/methods/ann/layer/mean_pooling.hpp @@ -32,7 +32,7 @@ template < > class MeanPooling { -public: + public: //! Create the MeanPooling object. MeanPooling(); @@ -121,7 +121,6 @@ public: void Serialize(Archive& ar, const unsigned int /* version */); private: - /** * Apply pooling to the input and store the results. * @@ -208,13 +207,13 @@ public: //! Locally-stored output height. size_t outputHeight; - //! Locally-stored reset parameter used to initialize the module once. + //! Locally-stored reset parameter used to initialize the module once. bool reset; //! Rounding operation used. bool floor; - //! If true use maximum a posteriori during the forward pass. + //! If true use maximum a posteriori during the forward pass. bool deterministic; //! Locally-stored stored rounding offset. diff --git a/src/mlpack/methods/ann/layer/mean_pooling_impl.hpp b/src/mlpack/methods/ann/layer/mean_pooling_impl.hpp index 5008763752..113845e782 100644 --- a/src/mlpack/methods/ann/layer/mean_pooling_impl.hpp +++ b/src/mlpack/methods/ann/layer/mean_pooling_impl.hpp @@ -75,10 +75,7 @@ void MeanPooling::Forward( slices); for (size_t s = 0; s < inputTemp.n_slices; s++) - { - Pooling(inputTemp.slice(s), outputTemp.slice(s)); - } output = arma::Mat(outputTemp.memptr(), outputTemp.n_elem, 1); diff --git a/src/mlpack/methods/ann/layer/parametric_relu.hpp b/src/mlpack/methods/ann/layer/parametric_relu.hpp index 3f5ba92d86..1362dd6207 100644 --- a/src/mlpack/methods/ann/layer/parametric_relu.hpp +++ b/src/mlpack/methods/ann/layer/parametric_relu.hpp @@ -202,7 +202,6 @@ class PReLU //! Leakyness Parameter given by user in the range 0 < alpha < 1. double user_alpha; - }; // class PReLU } // namespace ann @@ -211,4 +210,4 @@ class PReLU // Include implementation. #include "parametric_relu_impl.hpp" -#endif \ No newline at end of file +#endif diff --git a/src/mlpack/methods/ann/layer/parametric_relu_impl.hpp b/src/mlpack/methods/ann/layer/parametric_relu_impl.hpp index 0c290c2413..94fbd00dcf 100644 --- a/src/mlpack/methods/ann/layer/parametric_relu_impl.hpp +++ b/src/mlpack/methods/ann/layer/parametric_relu_impl.hpp @@ -25,7 +25,7 @@ template PReLU::PReLU( const double user_alpha) : user_alpha(user_alpha) { - alpha.set_size(1,1); + alpha.set_size(1, 1); alpha(0) = user_alpha; } diff --git a/src/mlpack/methods/ann/layer/recurrent_impl.hpp b/src/mlpack/methods/ann/layer/recurrent_impl.hpp index 80184109b6..6d5da1c152 100644 --- a/src/mlpack/methods/ann/layer/recurrent_impl.hpp +++ b/src/mlpack/methods/ann/layer/recurrent_impl.hpp @@ -146,7 +146,7 @@ void Recurrent::Backward( boost::apply_visitor(BackwardVisitor(std::move(boost::apply_visitor( outputParameterVisitor, feedbackModule)), std::move( boost::apply_visitor(deltaVisitor, recurrentModule)), std::move( - boost::apply_visitor(deltaVisitor, feedbackModule))),feedbackModule); + boost::apply_visitor(deltaVisitor, feedbackModule))), feedbackModule); } else { diff --git a/src/mlpack/methods/ann/layer/sequential.hpp b/src/mlpack/methods/ann/layer/sequential.hpp index ad84c7a177..b926f454a9 100644 --- a/src/mlpack/methods/ann/layer/sequential.hpp +++ b/src/mlpack/methods/ann/layer/sequential.hpp @@ -46,7 +46,6 @@ template < class Sequential { public: - /** * Create the Sequential object using the specified parameters. * diff --git a/src/mlpack/methods/ann/visitor/copy_visitor.hpp b/src/mlpack/methods/ann/visitor/copy_visitor.hpp index 3e6812fa1f..75e55e9297 100644 --- a/src/mlpack/methods/ann/visitor/copy_visitor.hpp +++ b/src/mlpack/methods/ann/visitor/copy_visitor.hpp @@ -26,7 +26,7 @@ class CopyVisitor : public boost::static_visitor { public: template - LayerTypes operator () (LayerType* ) const; + LayerTypes operator () (LayerType*) const; }; } // namespace ann diff --git a/src/mlpack/methods/approx_kfn/drusilla_select_impl.hpp b/src/mlpack/methods/approx_kfn/drusilla_select_impl.hpp index a3e8d0ff3a..65c330cddd 100644 --- a/src/mlpack/methods/approx_kfn/drusilla_select_impl.hpp +++ b/src/mlpack/methods/approx_kfn/drusilla_select_impl.hpp @@ -129,7 +129,8 @@ void DrusillaSelect::Train( } }; - std::vector clist(m, std::make_pair(double(-DBL_MAX), size_t(-1))); + std::vector clist( + m, std::make_pair(double(-DBL_MAX), size_t(-1))); std::priority_queue, CandidateCmp> pq(CandidateCmp(), std::move(clist)); diff --git a/src/mlpack/methods/cf/cf.cpp b/src/mlpack/methods/cf/cf.cpp index d063f4bbda..e5586248c2 100644 --- a/src/mlpack/methods/cf/cf.cpp +++ b/src/mlpack/methods/cf/cf.cpp @@ -259,5 +259,5 @@ void CF::CleanData(const arma::mat& data, arma::sp_mat& cleanedData) cleanedData = arma::sp_mat(locations, values, maxItemID, maxUserID); } -} // namespace mlpack } // namespace cf +} // namespace mlpack diff --git a/src/mlpack/methods/cf/cf_impl.hpp b/src/mlpack/methods/cf/cf_impl.hpp index 14e4b7f631..03faf71fbd 100644 --- a/src/mlpack/methods/cf/cf_impl.hpp +++ b/src/mlpack/methods/cf/cf_impl.hpp @@ -91,7 +91,7 @@ CF::CF(const arma::sp_mat& data, { Log::Warn << "CF::CF(): neighbourhood size should be > 0(" << numUsersForSimilarity << " given). Setting value to 5.\n"; - //Setting Default Value of 5 + // Setting Default Value of 5 this->numUsersForSimilarity = 5; } @@ -168,7 +168,7 @@ void CF::Serialize(Archive& ar, const unsigned int /* version */) ar & CreateNVP(cleanedData, "cleanedData"); } -} // namespace mlpack } // namespace cf +} // namespace mlpack #endif diff --git a/src/mlpack/methods/cf/svd_wrapper.hpp b/src/mlpack/methods/cf/svd_wrapper.hpp index 3ae381c52d..8f1a87db97 100644 --- a/src/mlpack/methods/cf/svd_wrapper.hpp +++ b/src/mlpack/methods/cf/svd_wrapper.hpp @@ -41,8 +41,11 @@ class SVDWrapper { public: // empty constructor - SVDWrapper(const Factorizer& factorizer = Factorizer()) - : factorizer(factorizer) {}; + SVDWrapper(const Factorizer& factorizer = Factorizer()) : + factorizer(factorizer) + { + // Nothing to do here. + } /** * Factorizer function which takes SVD of the given matrix and returns the diff --git a/src/mlpack/methods/cf/svd_wrapper_impl.hpp b/src/mlpack/methods/cf/svd_wrapper_impl.hpp index 91935e46ed..2c97eb0ef9 100644 --- a/src/mlpack/methods/cf/svd_wrapper_impl.hpp +++ b/src/mlpack/methods/cf/svd_wrapper_impl.hpp @@ -22,7 +22,7 @@ double mlpack::cf::SVDWrapper::Apply(const arma::mat& V, // construct sigma matrix sigma.zeros(V.n_rows, V.n_cols); - for(size_t i = 0;i < sigma.n_rows && i < sigma.n_cols;i++) + for (size_t i = 0; i < sigma.n_rows && i < sigma.n_cols; i++) sigma(i, i) = E(i, 0); arma::mat V_rec = W * sigma * arma::trans(H); @@ -44,7 +44,7 @@ double mlpack::cf::SVDWrapper::Apply(const arma::mat& V, // construct sigma matrix sigma.zeros(V.n_rows, V.n_cols); - for(size_t i = 0;i < sigma.n_rows && i < sigma.n_cols;i++) + for (size_t i = 0; i < sigma.n_rows && i < sigma.n_cols; i++) sigma(i, i) = E(i, 0); arma::mat V_rec = W * sigma * arma::trans(H); @@ -62,7 +62,9 @@ double mlpack::cf::SVDWrapper::Apply(const arma::mat& V, // check if the given rank is valid if (r > V.n_rows || r > V.n_cols) { - Log::Info << "Rank " << r << ", given for decomposition is invalid." << std::endl; + Log::Info << "Rank " << r << ", given for decomposition is invalid." + << std::endl; + r = (V.n_rows > V.n_cols) ? V.n_cols : V.n_rows; Log::Info << "Setting decomposition rank to " << r << std::endl; } @@ -101,7 +103,9 @@ double mlpack::cf::SVDWrapper::Apply(const arma::mat& V, // check if the given rank is valid if (r > V.n_rows || r > V.n_cols) { - Log::Info << "Rank " << r << ", given for decomposition is invalid." << std::endl; + Log::Info << "Rank " << r << ", given for decomposition is invalid." + << std::endl; + r = (V.n_rows > V.n_cols) ? V.n_cols : V.n_rows; Log::Info << "Setting decomposition rank to " << r << std::endl; } diff --git a/src/mlpack/methods/decision_tree/decision_tree_impl.hpp b/src/mlpack/methods/decision_tree/decision_tree_impl.hpp index 7d05042ac9..7046de3bd3 100644 --- a/src/mlpack/methods/decision_tree/decision_tree_impl.hpp +++ b/src/mlpack/methods/decision_tree/decision_tree_impl.hpp @@ -150,7 +150,7 @@ DecisionTree(tmpData, 0, tmpData.n_cols, tmpLabels, numClasses, tmpWeights, minimumLeafSize); - } +} //! Construct, don't train. template childAssignments(count); for (size_t j = begin; j < begin + count; ++j) - childAssignments[j - begin] = NumericSplit::CalculateDirection(data(bestDim, j), - classProbabilities, *this); + { + childAssignments[j - begin] = NumericSplit::CalculateDirection( + data(bestDim, j), classProbabilities, *this); + } // Calculate counts of children in each node. arma::Row childCounts(numChildren); @@ -786,7 +790,6 @@ void DecisionTree* Trainer(MatType& dataset, std::vector > prunedSequence; while (dtree.SubtreeLeaves() > 1) { - std::pair treeSeq(oldAlpha, dtree.SubtreeLeavesLogNegError()); + std::pair treeSeq(oldAlpha, + dtree.SubtreeLeavesLogNegError()); prunedSequence.push_back(treeSeq); oldAlpha = alpha; alpha = dtree.PruneAndUpdate(oldAlpha, dataset.n_cols, useVolumeReg); diff --git a/src/mlpack/methods/det/dtree.hpp b/src/mlpack/methods/det/dtree.hpp index 4619030992..ccbc79830b 100644 --- a/src/mlpack/methods/det/dtree.hpp +++ b/src/mlpack/methods/det/dtree.hpp @@ -316,7 +316,6 @@ class DTree void Serialize(Archive& ar, const unsigned int /* version */); private: - // Utility methods. /** @@ -336,7 +335,6 @@ class DTree const size_t splitDim, const ElemType splitValue, arma::Col& oldFromNew) const; - }; } // namespace det diff --git a/src/mlpack/methods/det/dtree_impl.hpp b/src/mlpack/methods/det/dtree_impl.hpp index fc7e10350b..1d52f085f9 100644 --- a/src/mlpack/methods/det/dtree_impl.hpp +++ b/src/mlpack/methods/det/dtree_impl.hpp @@ -34,8 +34,7 @@ namespace details { static_assert( std::is_same::value == true, - "The ElemType does not correspond to the matrix's element type." - ); + "The ElemType does not correspond to the matrix's element type."); typedef std::pair SplitItem; const typename MatType::row_type dimVec = @@ -118,9 +117,10 @@ namespace details const ElemType newVal = valsVec[i]; if (lastVal < ElemType(0) && newVal > ElemType(0) && zeroes > 0) { - Log::Assert(padding == 0); // we should arrive here once! + Log::Assert(padding == 0); // We should arrive here once! - // the minLeafSize > 0 also guarantees we're not entering right at the start. + // The minLeafSize > 0 also guarantees we're not entering right at the + // start. if (i >= minLeafSize && i <= n_elem - minLeafSize) splitVec.push_back(SplitItem(lastVal / 2.0, i)); @@ -145,7 +145,7 @@ namespace details lastVal = newVal; } } -}; +}; //namespace details template DTree::DTree() : @@ -412,7 +412,7 @@ double DTree::LogNegativeError(const size_t totalPoints) const 2 * std::log((double) totalPoints); StatType valDiffs = maxVals - minVals; - for (size_t i = 0;i < valDiffs.n_elem; ++i) + for (size_t i = 0; i < valDiffs.n_elem; ++i) { // Ignore very small dimensions to prevent overflow. if (valDiffs[i] > 1e-50) @@ -475,8 +475,9 @@ bool DTree::FindSplit(const MatType& data, // Get the values for splitting. The old implementation: // dimVec = data.row(dim).subvec(start, end - 1); // dimVec = arma::sort(dimVec); - // could be quite inefficient for sparse matrices, due to copy operations (3). - // This one has custom implementation for dense and sparse matrices. + // could be quite inefficient for sparse matrices, due to + // copy operations (3). This one has custom implementation for dense and + // sparse matrices. std::vector splitVec; details::ExtractSplits(splitVec, data, dim, start, end, minLeafSize); @@ -709,7 +710,7 @@ double DTree::Grow(MatType& data, if (useVolReg) { // This is wrong for now! - gT = alphaUpper;// / (subtreeLeavesVTInv - vTInv); + gT = alphaUpper; // / (subtreeLeavesVTInv - vTInv); } else { @@ -740,7 +741,7 @@ double DTree::PruneAndUpdate(const double oldAlpha, // Compute gT value for node t. volatile double gT; if (useVolReg) - gT = alphaUpper;// - std::log(subtreeLeavesVTInv - vTInv); + gT = alphaUpper; // - std::log(subtreeLeavesVTInv - vTInv); else gT = alphaUpper - std::log((double) (subtreeLeaves - 1)); @@ -869,7 +870,8 @@ double DTree::ComputeValue(const VecType& query) const } else { - // Return either of the two children - left or right, depending on the splitValue + // Return either of the two children - left or right, depending on the + // splitValue return (query[splitDim] <= splitValue) ? left->ComputeValue(query) : right->ComputeValue(query); @@ -878,7 +880,6 @@ double DTree::ComputeValue(const VecType& query) const return 0.0; } - // Index the buckets for possible usage later. template TagType DTree::TagTree(const TagType& tag) @@ -895,7 +896,6 @@ TagType DTree::TagTree(const TagType& tag) } } - template TagType DTree::FindBucket(const VecType& query) const { @@ -924,7 +924,7 @@ DTree::ComputeVariableImportance(arma::vec& importances) const std::stack nodes; nodes.push(this); - while(!nodes.empty()) + while (!nodes.empty()) { const DTree& curNode = *nodes.top(); nodes.pop(); @@ -945,7 +945,8 @@ DTree::ComputeVariableImportance(arma::vec& importances) const template template -void DTree::Serialize(Archive& ar, const unsigned int /* version */) +void DTree::Serialize(Archive& ar, + const unsigned int /* version */) { using data::CreateNVP; diff --git a/src/mlpack/methods/emst/dtb.hpp b/src/mlpack/methods/emst/dtb.hpp index c682b4aa0a..4b05bb4ca0 100644 --- a/src/mlpack/methods/emst/dtb.hpp +++ b/src/mlpack/methods/emst/dtb.hpp @@ -202,7 +202,6 @@ class DualTreeBoruvka * The values stored in the tree must be reset on each iteration. */ void Cleanup(); - }; // class DualTreeBoruvka } // namespace emst diff --git a/src/mlpack/methods/emst/dtb_impl.hpp b/src/mlpack/methods/emst/dtb_impl.hpp index 6032075741..a5ba58a314 100644 --- a/src/mlpack/methods/emst/dtb_impl.hpp +++ b/src/mlpack/methods/emst/dtb_impl.hpp @@ -202,7 +202,7 @@ void DualTreeBoruvka::AddAllEdges() size_t outEdge = neighborsOutComponent[component]; if (connections.Find(inEdge) != connections.Find(outEdge)) { - //totalDist = totalDist + dist; + // totalDist = totalDist + dist; // changed to make this agree with the cover tree code totalDist += neighborsDistances[component]; AddEdge(inEdge, outEdge, neighborsDistances[component]); diff --git a/src/mlpack/methods/emst/dtb_rules.hpp b/src/mlpack/methods/emst/dtb_rules.hpp index ee9c319b2d..69d329f395 100644 --- a/src/mlpack/methods/emst/dtb_rules.hpp +++ b/src/mlpack/methods/emst/dtb_rules.hpp @@ -129,7 +129,6 @@ class DTBRules size_t baseCases; //! The number of node combinations that have been scored. size_t scores; - }; // class DTBRules } // emst namespace diff --git a/src/mlpack/methods/emst/dtb_stat.hpp b/src/mlpack/methods/emst/dtb_stat.hpp index ffa5f2b58a..020a953a80 100644 --- a/src/mlpack/methods/emst/dtb_stat.hpp +++ b/src/mlpack/methods/emst/dtb_stat.hpp @@ -87,7 +87,6 @@ class DTBStat int ComponentMembership() const { return componentMembership; } //! Modify the component membership of this node. int& ComponentMembership() { return componentMembership; } - }; // class DTBStat } // namespace emst diff --git a/src/mlpack/methods/emst/edge_pair.hpp b/src/mlpack/methods/emst/edge_pair.hpp index b2a4ebb095..dae44b6a48 100644 --- a/src/mlpack/methods/emst/edge_pair.hpp +++ b/src/mlpack/methods/emst/edge_pair.hpp @@ -63,7 +63,6 @@ class EdgePair double Distance() const { return distance; } //! Modify the distance. double& Distance() { return distance; } - }; // class EdgePair } // namespace emst diff --git a/src/mlpack/methods/gmm/gmm_impl.hpp b/src/mlpack/methods/gmm/gmm_impl.hpp index 021533ad1d..62e586266e 100644 --- a/src/mlpack/methods/gmm/gmm_impl.hpp +++ b/src/mlpack/methods/gmm/gmm_impl.hpp @@ -59,7 +59,8 @@ double GMM::Train(const arma::mat& observations, bestLikelihood = LogLikelihood(observations, dists, weights); - Log::Info << "GMM::Train(): Log-likelihood of trial 0 is " << bestLikelihood << "." << std::endl; + Log::Info << "GMM::Train(): Log-likelihood of trial 0 is " + << bestLikelihood << "." << std::endl; // Now the temporary model. std::vector distsTrial(gaussians, @@ -159,7 +160,8 @@ double GMM::Train(const arma::mat& observations, weightsTrial = weightsOrig; } - fitter.Estimate(observations, probabilities, distsTrial, weightsTrial, useExistingModel); + fitter.Estimate(observations, probabilities, distsTrial, weightsTrial, + useExistingModel); // Check to see if the log-likelihood of this one is better. double newLikelihood = LogLikelihood(observations, distsTrial, diff --git a/src/mlpack/methods/hmm/hmm_regression.hpp b/src/mlpack/methods/hmm/hmm_regression.hpp index bfde35afb5..a6fa7af134 100644 --- a/src/mlpack/methods/hmm/hmm_regression.hpp +++ b/src/mlpack/methods/hmm/hmm_regression.hpp @@ -287,13 +287,13 @@ class HMMRegression : public HMM /** * Utility functions to facilitate the use of the HMM class for HMMR. */ - void StackData(const std::vector& predictors, - const std::vector& responses, - std::vector& dataSeq) const; + void StackData(const std::vector& predictors, + const std::vector& responses, + std::vector& dataSeq) const; - void StackData(const arma::mat& predictors, - const arma::vec& responses, - arma::mat& dataSeq) const; + void StackData(const arma::mat& predictors, + const arma::vec& responses, + arma::mat& dataSeq) const; /** * The Forward algorithm (part of the Forward-Backward algorithm). Computes @@ -327,8 +327,6 @@ class HMMRegression : public HMM const arma::vec& responses, const arma::vec& scales, arma::mat& backwardProb) const; - - }; } // namespace hmm diff --git a/src/mlpack/methods/hmm/hmm_regression_impl.hpp b/src/mlpack/methods/hmm/hmm_regression_impl.hpp index 09cc0b1844..61676305ee 100644 --- a/src/mlpack/methods/hmm/hmm_regression_impl.hpp +++ b/src/mlpack/methods/hmm/hmm_regression_impl.hpp @@ -115,7 +115,7 @@ void HMMRegression::Filter(const arma::mat& predictors, filterSeq.resize(responses.n_elem - ahead); filterSeq.zeros(); arma::vec nextSeq; - for(size_t i = 0; i < emission.size(); i++) + for (size_t i = 0; i < emission.size(); i++) { emission[i].Predict(predictors.cols(ahead, predictors.n_cols-1), nextSeq); filterSeq = filterSeq + nextSeq%(forwardProb.row(i).t()); @@ -137,12 +137,11 @@ void HMMRegression::Smooth(const arma::mat& predictors, smoothSeq.resize(responses.n_elem); smoothSeq.zeros(); arma::vec nextSeq; - for(size_t i = 0; i < emission.size(); i++) + for (size_t i = 0; i < emission.size(); i++) { emission[i].Predict(predictors, nextSeq); smoothSeq = smoothSeq + nextSeq%(stateProb.row(i).t()); } - } /** @@ -174,7 +173,7 @@ void HMMRegression::StackData(const std::vector& predictors, std::vector& dataSeq) const { arma::mat nextSeq; - for(size_t i = 0; i < predictors.size(); i++) + for (size_t i = 0; i < predictors.size(); i++) { nextSeq = predictors[i]; nextSeq.insert_rows(0, responses[i].t()); diff --git a/src/mlpack/methods/hoeffding_trees/binary_numeric_split_impl.hpp b/src/mlpack/methods/hoeffding_trees/binary_numeric_split_impl.hpp index d7a99c121f..1bb5e86974 100644 --- a/src/mlpack/methods/hoeffding_trees/binary_numeric_split_impl.hpp +++ b/src/mlpack/methods/hoeffding_trees/binary_numeric_split_impl.hpp @@ -126,7 +126,7 @@ void BinaryNumericSplit::Split( double min = DBL_MAX; double max = -DBL_MAX; for (typename std::multimap::const_iterator it = - sortedElements.begin();// (*it).first < bestSplit; ++it) + sortedElements.begin(); // (*it).first < bestSplit; ++it) it != sortedElements.end(); ++it) { // Move the point to the correct side of the split. diff --git a/src/mlpack/methods/hoeffding_trees/hoeffding_numeric_split_impl.hpp b/src/mlpack/methods/hoeffding_trees/hoeffding_numeric_split_impl.hpp index 1bbf137856..e7b9fc394c 100644 --- a/src/mlpack/methods/hoeffding_trees/hoeffding_numeric_split_impl.hpp +++ b/src/mlpack/methods/hoeffding_trees/hoeffding_numeric_split_impl.hpp @@ -184,7 +184,6 @@ double HoeffdingNumericSplit:: return double(classCounts.max()) / double(arma::sum(classCounts)); } - } template diff --git a/src/mlpack/methods/hoeffding_trees/hoeffding_tree_impl.hpp b/src/mlpack/methods/hoeffding_trees/hoeffding_tree_impl.hpp index bdf7fe6977..cd8749cc13 100644 --- a/src/mlpack/methods/hoeffding_trees/hoeffding_tree_impl.hpp +++ b/src/mlpack/methods/hoeffding_trees/hoeffding_tree_impl.hpp @@ -648,7 +648,6 @@ void HoeffdingTree< children.push_back(new HoeffdingTree(*datasetInfo, numClasses, successProbability, maxSamples, checkInterval, minSamples, categoricalSplits[0], numericSplits[0], dimensionMappings)); - } children[i]->MajorityClass() = childMajorities[i]; diff --git a/src/mlpack/methods/kernel_pca/kernel_pca.hpp b/src/mlpack/methods/kernel_pca/kernel_pca.hpp index bbe8b5d41c..60ef68e94f 100644 --- a/src/mlpack/methods/kernel_pca/kernel_pca.hpp +++ b/src/mlpack/methods/kernel_pca/kernel_pca.hpp @@ -122,7 +122,6 @@ class KernelPCA //! If true, the data will be scaled (by standard deviation) when Apply() is //! run. bool centerTransformedData; - }; // class KernelPCA } // namespace kpca diff --git a/src/mlpack/methods/kernel_pca/kernel_pca_main.cpp b/src/mlpack/methods/kernel_pca/kernel_pca_main.cpp index 224db7a257..8ea8720df3 100644 --- a/src/mlpack/methods/kernel_pca/kernel_pca_main.cpp +++ b/src/mlpack/methods/kernel_pca/kernel_pca_main.cpp @@ -81,13 +81,13 @@ PROGRAM_INFO("Kernel Principal Components Analysis", "options --bandwidth, --kernel_scale, --offset, or --degree (or a " "combination of those options)." "\n\n" - "Optionally, the nystr\u00F6m method (\"Using the Nystroem method to speed up" - " kernel machines\", 2001) can be used to calculate the kernel matrix by " + "Optionally, the nystr\u00F6m method (\"Using the Nystroem method to speed " + "up kernel machines\", 2001) can be used to calculate the kernel matrix by " "specifying the --nystroem_method (-n) option. This approach works by using" " a subset of the data as basis to reconstruct the kernel matrix; to " "specify the sampling scheme, the --sampling parameter is used, the " - "sampling scheme for the nystr\u00F6m method can be chosen from the following" - " list: kmeans, random, ordered."); + "sampling scheme for the nystr\u00F6m method can be chosen from the " + "following list: kmeans, random, ordered."); PARAM_MATRIX_IN_REQ("input", "Input dataset to perform KPCA on.", "i"); PARAM_MATRIX_OUT("output", "Matrix to save modified dataset to.", "o"); diff --git a/src/mlpack/methods/kernel_pca/kernel_rules/naive_method.hpp b/src/mlpack/methods/kernel_pca/kernel_rules/naive_method.hpp index cb45b583b1..567ab51b83 100644 --- a/src/mlpack/methods/kernel_pca/kernel_rules/naive_method.hpp +++ b/src/mlpack/methods/kernel_pca/kernel_rules/naive_method.hpp @@ -21,71 +21,71 @@ namespace kpca { template class NaiveKernelRule { - public: - /** - * Construct the exact kernel matrix. - * - * @param data Input data points. - * @param transformedData Matrix to output results into. - * @param eigval KPCA eigenvalues will be written to this vector. - * @param eigvec KPCA eigenvectors will be written to this matrix. - * @param rank Rank to be used for matrix approximation. - * @param kernel Kernel to be used for computation. - */ - static void ApplyKernelMatrix(const arma::mat& data, - arma::mat& transformedData, - arma::vec& eigval, - arma::mat& eigvec, - const size_t /* unused */, - KernelType kernel = KernelType()) + public: + /** + * Construct the exact kernel matrix. + * + * @param data Input data points. + * @param transformedData Matrix to output results into. + * @param eigval KPCA eigenvalues will be written to this vector. + * @param eigvec KPCA eigenvectors will be written to this matrix. + * @param rank Rank to be used for matrix approximation. + * @param kernel Kernel to be used for computation. + */ + static void ApplyKernelMatrix(const arma::mat& data, + arma::mat& transformedData, + arma::vec& eigval, + arma::mat& eigvec, + const size_t /* unused */, + KernelType kernel = KernelType()) +{ + // Construct the kernel matrix. + arma::mat kernelMatrix; + // Resize the kernel matrix to the right size. + kernelMatrix.set_size(data.n_cols, data.n_cols); + + // Note that we only need to calculate the upper triangular part of the + // kernel matrix, since it is symmetric. This helps minimize the number of + // kernel evaluations. + for (size_t i = 0; i < data.n_cols; ++i) { - // Construct the kernel matrix. - arma::mat kernelMatrix; - // Resize the kernel matrix to the right size. - kernelMatrix.set_size(data.n_cols, data.n_cols); - - // Note that we only need to calculate the upper triangular part of the - // kernel matrix, since it is symmetric. This helps minimize the number of - // kernel evaluations. - for (size_t i = 0; i < data.n_cols; ++i) + for (size_t j = i; j < data.n_cols; ++j) { - for (size_t j = i; j < data.n_cols; ++j) - { - // Evaluate the kernel on these two points. - kernelMatrix(i, j) = kernel.Evaluate(data.unsafe_col(i), - data.unsafe_col(j)); - } + // Evaluate the kernel on these two points. + kernelMatrix(i, j) = kernel.Evaluate(data.unsafe_col(i), + data.unsafe_col(j)); } - - // Copy to the lower triangular part of the matrix. - for (size_t i = 1; i < data.n_cols; ++i) - for (size_t j = 0; j < i; ++j) - kernelMatrix(i, j) = kernelMatrix(j, i); - - // For PCA the data has to be centered, even if the data is centered. But it - // is not guaranteed that the data, when mapped to the kernel space, is also - // centered. Since we actually never work in the feature space we cannot - // center the data. So, we perform a "psuedo-centering" using the kernel - // matrix. - arma::rowvec rowMean = arma::sum(kernelMatrix, 0) / kernelMatrix.n_cols; - kernelMatrix.each_col() -= arma::sum(kernelMatrix, 1) / kernelMatrix.n_cols; - kernelMatrix.each_row() -= rowMean; - kernelMatrix += arma::sum(rowMean) / kernelMatrix.n_cols; - - // Eigendecompose the centered kernel matrix. - arma::eig_sym(eigval, eigvec, kernelMatrix); - - // Swap the eigenvalues since they are ordered backwards (we need largest to - // smallest). - for (size_t i = 0; i < floor(eigval.n_elem / 2.0); ++i) - eigval.swap_rows(i, (eigval.n_elem - 1) - i); - - // Flip the coefficients to produce the same effect. - eigvec = arma::fliplr(eigvec); - - transformedData = eigvec.t() * kernelMatrix; - transformedData.each_col() /= arma::sqrt(eigval); } + + // Copy to the lower triangular part of the matrix. + for (size_t i = 1; i < data.n_cols; ++i) + for (size_t j = 0; j < i; ++j) + kernelMatrix(i, j) = kernelMatrix(j, i); + + // For PCA the data has to be centered, even if the data is centered. But it + // is not guaranteed that the data, when mapped to the kernel space, is also + // centered. Since we actually never work in the feature space we cannot + // center the data. So, we perform a "psuedo-centering" using the kernel + // matrix. + arma::rowvec rowMean = arma::sum(kernelMatrix, 0) / kernelMatrix.n_cols; + kernelMatrix.each_col() -= arma::sum(kernelMatrix, 1) / kernelMatrix.n_cols; + kernelMatrix.each_row() -= rowMean; + kernelMatrix += arma::sum(rowMean) / kernelMatrix.n_cols; + + // Eigendecompose the centered kernel matrix. + arma::eig_sym(eigval, eigvec, kernelMatrix); + + // Swap the eigenvalues since they are ordered backwards (we need largest to + // smallest). + for (size_t i = 0; i < floor(eigval.n_elem / 2.0); ++i) + eigval.swap_rows(i, (eigval.n_elem - 1) - i); + + // Flip the coefficients to produce the same effect. + eigvec = arma::fliplr(eigvec); + + transformedData = eigvec.t() * kernelMatrix; + transformedData.each_col() /= arma::sqrt(eigval); +} }; } // namespace kpca diff --git a/src/mlpack/methods/kernel_pca/kernel_rules/nystroem_method.hpp b/src/mlpack/methods/kernel_pca/kernel_rules/nystroem_method.hpp index a8f82411e1..3ee0b0aed7 100644 --- a/src/mlpack/methods/kernel_pca/kernel_rules/nystroem_method.hpp +++ b/src/mlpack/methods/kernel_pca/kernel_rules/nystroem_method.hpp @@ -26,56 +26,56 @@ template< > class NystroemKernelRule { - public: - /** - * Construct the kernel matrix approximation using the nystroem method. - * - * @param data Input data points. - * @param transformedData Matrix to output results into. - * @param eigval KPCA eigenvalues will be written to this vector. - * @param eigvec KPCA eigenvectors will be written to this matrix. - * @param rank Rank to be used for matrix approximation. - * @param kernel Kernel to be used for computation. - */ - static void ApplyKernelMatrix(const arma::mat& data, - arma::mat& transformedData, - arma::vec& eigval, - arma::mat& eigvec, - const size_t rank, - KernelType kernel = KernelType()) - { - arma::mat G, v; - kernel::NystroemMethod nm(data, kernel, - rank); - nm.Apply(G); - transformedData = G.t() * G; + public: + /** + * Construct the kernel matrix approximation using the nystroem method. + * + * @param data Input data points. + * @param transformedData Matrix to output results into. + * @param eigval KPCA eigenvalues will be written to this vector. + * @param eigvec KPCA eigenvectors will be written to this matrix. + * @param rank Rank to be used for matrix approximation. + * @param kernel Kernel to be used for computation. + */ + static void ApplyKernelMatrix(const arma::mat& data, + arma::mat& transformedData, + arma::vec& eigval, + arma::mat& eigvec, + const size_t rank, + KernelType kernel = KernelType()) + { + arma::mat G, v; + kernel::NystroemMethod nm(data, kernel, + rank); + nm.Apply(G); + transformedData = G.t() * G; - // Center the reconstructed approximation. - math::Center(transformedData, transformedData); + // Center the reconstructed approximation. + math::Center(transformedData, transformedData); - // For PCA the data has to be centered, even if the data is centered. But - // it is not guaranteed that the data, when mapped to the kernel space, is - // also centered. Since we actually never work in the feature space we - // cannot center the data. So, we perform a "psuedo-centering" using the - // kernel matrix. - arma::colvec colMean = arma::sum(G, 1) / G.n_rows; - G.each_row() -= arma::sum(G, 0) / G.n_rows; - G.each_col() -= colMean; - G += arma::sum(colMean) / G.n_rows; + // For PCA the data has to be centered, even if the data is centered. But + // it is not guaranteed that the data, when mapped to the kernel space, is + // also centered. Since we actually never work in the feature space we + // cannot center the data. So, we perform a "psuedo-centering" using the + // kernel matrix. + arma::colvec colMean = arma::sum(G, 1) / G.n_rows; + G.each_row() -= arma::sum(G, 0) / G.n_rows; + G.each_col() -= colMean; + G += arma::sum(colMean) / G.n_rows; - // Eigendecompose the centered kernel matrix. - arma::eig_sym(eigval, eigvec, transformedData); + // Eigendecompose the centered kernel matrix. + arma::eig_sym(eigval, eigvec, transformedData); - // Swap the eigenvalues since they are ordered backwards (we need largest - // to smallest). - for (size_t i = 0; i < floor(eigval.n_elem / 2.0); ++i) - eigval.swap_rows(i, (eigval.n_elem - 1) - i); + // Swap the eigenvalues since they are ordered backwards (we need largest + // to smallest). + for (size_t i = 0; i < floor(eigval.n_elem / 2.0); ++i) + eigval.swap_rows(i, (eigval.n_elem - 1) - i); - // Flip the coefficients to produce the same effect. - eigvec = arma::fliplr(eigvec); + // Flip the coefficients to produce the same effect. + eigvec = arma::fliplr(eigvec); - transformedData = eigvec.t() * G.t(); - } + transformedData = eigvec.t() * G.t(); + } }; } // namespace kpca diff --git a/src/mlpack/methods/kmeans/elkan_kmeans_impl.hpp b/src/mlpack/methods/kmeans/elkan_kmeans_impl.hpp index 0de4eeb6a0..ffed2d87ca 100644 --- a/src/mlpack/methods/kmeans/elkan_kmeans_impl.hpp +++ b/src/mlpack/methods/kmeans/elkan_kmeans_impl.hpp @@ -25,7 +25,7 @@ ElkanKMeans::ElkanKMeans(const MatType& dataset, metric(metric), distanceCalculations(0) { - + // Nothing to do here. } // Run a single iteration of Elkan's algorithm for Lloyd iterations. diff --git a/src/mlpack/methods/kmeans/kmeans_impl.hpp b/src/mlpack/methods/kmeans/kmeans_impl.hpp index 61467fa105..92755eb7af 100644 --- a/src/mlpack/methods/kmeans/kmeans_impl.hpp +++ b/src/mlpack/methods/kmeans/kmeans_impl.hpp @@ -241,7 +241,6 @@ Cluster(const MatType& data, << cNorm << ".\n"; if (std::isnan(cNorm) || std::isinf(cNorm)) cNorm = 1e-4; // Keep iterating. - } while (cNorm > 1e-5 && iteration != maxIterations); // If we ended on an even iteration, then the centroids are in the diff --git a/src/mlpack/methods/kmeans/max_variance_new_cluster_impl.hpp b/src/mlpack/methods/kmeans/max_variance_new_cluster_impl.hpp index 3ecd74b404..4c23eba88d 100644 --- a/src/mlpack/methods/kmeans/max_variance_new_cluster_impl.hpp +++ b/src/mlpack/methods/kmeans/max_variance_new_cluster_impl.hpp @@ -65,8 +65,8 @@ size_t MaxVarianceNewCluster::EmptyCluster(const MatType& data, // Take that point and add it to the empty cluster. newCentroids.col(maxVarCluster) *= (double(clusterCounts[maxVarCluster]) / double(clusterCounts[maxVarCluster] - 1)); - newCentroids.col(maxVarCluster) -= (1.0 / (clusterCounts[maxVarCluster] - 1.0)) * - arma::vec(data.col(furthestPoint)); + newCentroids.col(maxVarCluster) -= (1.0 / (clusterCounts[maxVarCluster] - + 1.0)) * arma::vec(data.col(furthestPoint)); clusterCounts[maxVarCluster]--; clusterCounts[emptyCluster]++; newCentroids.col(emptyCluster) = arma::vec(data.col(furthestPoint)); @@ -87,7 +87,8 @@ size_t MaxVarianceNewCluster::EmptyCluster(const MatType& data, else { variances[maxVarCluster] = (1.0 / clusterCounts[maxVarCluster]) * - ((clusterCounts[maxVarCluster] + 1) * variances[maxVarCluster] - maxDistance); + ((clusterCounts[maxVarCluster] + 1) * variances[maxVarCluster] - + maxDistance); } // Output some debugging information. diff --git a/src/mlpack/methods/kmeans/random_partition.hpp b/src/mlpack/methods/kmeans/random_partition.hpp index 2d1b99564c..954f3bdfd1 100644 --- a/src/mlpack/methods/kmeans/random_partition.hpp +++ b/src/mlpack/methods/kmeans/random_partition.hpp @@ -55,7 +55,7 @@ class RandomPartition void Serialize(Archive& /* ar */, const unsigned int /* version */) { } }; -} -} +} // namespace kmeans +} // namespace mlpack #endif diff --git a/src/mlpack/methods/linear_regression/linear_regression.cpp b/src/mlpack/methods/linear_regression/linear_regression.cpp index 6a3976a291..cfa6b8a352 100644 --- a/src/mlpack/methods/linear_regression/linear_regression.cpp +++ b/src/mlpack/methods/linear_regression/linear_regression.cpp @@ -88,7 +88,7 @@ void LinearRegression::Train(const arma::mat& predictors, // intercept = false to get a penalized intercept. if (intercept) { - p.insert_rows(0, arma::ones(1,nCols)); + p.insert_rows(0, arma::ones(1, nCols)); } if (weights.n_elem > 0) @@ -122,7 +122,7 @@ void LinearRegression::Train(const arma::mat& predictors, else { // Copy responses into larger vector. - r.insert_cols(nCols,p.n_cols - nCols); + r.insert_cols(nCols, p.n_cols - nCols); arma::solve(parameters, R, arma::trans(r * Q)); } } @@ -152,11 +152,11 @@ void LinearRegression::Predict(const arma::mat& points, } else { - // We want to be sure we have the correct number of dimensions in the dataset. + // We want to be sure we have the correct number of dimensions in + // the dataset. Log::Assert(points.n_rows == parameters.n_rows); predictions = arma::trans(parameters) * points; } - } //! Compute the L2 squared error on the given predictors and responses. diff --git a/src/mlpack/methods/linear_regression/linear_regression.hpp b/src/mlpack/methods/linear_regression/linear_regression.hpp index b8fa6ca78b..99ec44b42b 100644 --- a/src/mlpack/methods/linear_regression/linear_regression.hpp +++ b/src/mlpack/methods/linear_regression/linear_regression.hpp @@ -230,7 +230,7 @@ class LinearRegression bool intercept; }; -} // namespace linear_regression +} // namespace regression } // namespace mlpack #endif // MLPACK_METHODS_LINEAR_REGRESSION_HPP diff --git a/src/mlpack/methods/logistic_regression/logistic_regression_function_impl.hpp b/src/mlpack/methods/logistic_regression/logistic_regression_function_impl.hpp index 1a8e023cca..53b1a953b8 100644 --- a/src/mlpack/methods/logistic_regression/logistic_regression_function_impl.hpp +++ b/src/mlpack/methods/logistic_regression/logistic_regression_function_impl.hpp @@ -48,7 +48,7 @@ LogisticRegressionFunction::LogisticRegressionFunction( responses(responses), lambda(lambda) { - //to check if initialPoint is compatible with predictors + // To check if initialPoint is compatible with predictors. if (initialPoint.n_rows != (predictors.n_rows + 1) || initialPoint.n_cols != 1) this->initialPoint = arma::zeros(predictors.n_rows + 1, 1); diff --git a/src/mlpack/methods/lsh/lsh_search.hpp b/src/mlpack/methods/lsh/lsh_search.hpp index 077ecb43d5..097a94a46f 100644 --- a/src/mlpack/methods/lsh/lsh_search.hpp +++ b/src/mlpack/methods/lsh/lsh_search.hpp @@ -472,7 +472,6 @@ class LSHSearch //! Use a priority queue to represent the list of candidate neighbors. typedef std::priority_queue, CandidateCmp> CandidateList; - }; // class LSHSearch } // namespace neighbor diff --git a/src/mlpack/methods/lsh/lsh_search_impl.hpp b/src/mlpack/methods/lsh/lsh_search_impl.hpp index 9fc56184ca..d0439c5c25 100644 --- a/src/mlpack/methods/lsh/lsh_search_impl.hpp +++ b/src/mlpack/methods/lsh/lsh_search_impl.hpp @@ -298,7 +298,7 @@ void LSHSearch::Train(const arma::mat& referenceSet, // For a single table, let the 'numProj' projections be denoted by 'proj_i' // and the corresponding offset be 'offset_i'. Then the key of a single // point is obtained as: - // key = { floor( ( + offset_i) / 'hashWidth' ) forall i } + // key = { floor(( + offset_i) / 'hashWidth') forall i } arma::mat offsetMat = arma::repmat(offsets.unsafe_col(i), 1, referenceSet.n_cols); arma::mat hashMat = projections.slice(i).t() * (referenceSet); @@ -368,7 +368,6 @@ void LSHSearch::Train(const arma::mat& referenceSet, const size_t index = bucketRowInHashTable[hashInd]; if (bucketContentSize[index] < maxSize) secondHashTable[index](bucketContentSize[index]++) = j; - } // Loop over all points in the reference set. } // Loop over tables. @@ -552,7 +551,6 @@ void LSHSearch::GetAdditionalProbingBins( const size_t T, arma::mat& additionalProbingBins) const { - // No additional bins requested. Our work is done. if (T == 0) return; @@ -626,12 +624,11 @@ void LSHSearch::GetAdditionalProbingBins( // smallest and the second smallest, it's obvious that score(Ae) > // score(As). Therefore the second perturbation vector is ALWAYS the vector // containing only the second-lowest scoring perturbation. - double minscore2 = scores[0]; size_t minloc2 = 0; - for (size_t s = 0; s < (2 * numProj); ++s) // here we can't start from 1 + for (size_t s = 0; s < (2 * numProj); ++s) // Here we can't start from 1. { - if (minscore2 > scores[s] && s != minloc) //second smallest + if (minscore2 > scores[s] && s != minloc) // Second smallest. { minscore2 = scores[s]; minloc2 = s; @@ -644,14 +641,12 @@ void LSHSearch::GetAdditionalProbingBins( } // General case: more than 2 perturbation vectors require use of minheap. - // Sort everything in increasing order. arma::uvec sortidx = arma::sort_index(scores); scores = scores(sortidx); actions = actions(sortidx); positions = positions(sortidx); - // Theory: // A probing sequence is a sequence of T probing bins where a query's // neighbors are most likely to be. Likelihood is dependent only on a bin's @@ -683,7 +678,7 @@ void LSHSearch::GetAdditionalProbingBins( > minHeap; // our minheap // Start by adding the lowest scoring set to the minheap. - minHeap.push( std::make_pair(PerturbationScore(Ao, scores), 0) ); + minHeap.push(std::make_pair(PerturbationScore(Ao, scores), 0)); // Loop invariable: after pvec iterations, additionalProbingBins contains pvec // valid codes of the lowest-scoring bins (bins most likely to contain @@ -710,22 +705,22 @@ void LSHSearch::GetAdditionalProbingBins( // Expand operation on Ai (add max+1 to set). std::vector Ae = Ai; + // Don't add invalid sets. if (PerturbationExpand(Ae) && PerturbationValid(Ae)) - // Don't add invalid sets. { perturbationSets.push_back(Ae); // add expanded set to sets minHeap.push( std::make_pair(PerturbationScore(Ae, scores), perturbationSets.size() - 1)); } - - } while (!PerturbationValid(Ai));//Discard invalid perturbations + } while (!PerturbationValid(Ai)); // Discard invalid perturbations // Found valid perturbation set Ai. Construct perturbation vector from set. for (size_t pos = 0; pos < Ai.size(); ++pos) + { // If Ai[pos] is marked, add action to probing vector. - additionalProbingBins(positions(pos), pvec) - += Ai[pos] ? actions(pos) : 0; + additionalProbingBins(positions(pos), pvec) += Ai[pos] ? actions(pos) : 0; + } } } @@ -756,6 +751,7 @@ void LSHSearch::ReturnIndicesFromTable( arma::mat queryCodesNotFloored(numProj, numTablesToSearch); for (size_t i = 0; i < numTablesToSearch; i++) queryCodesNotFloored.unsafe_col(i) = projections.slice(i).t() * queryPoint; + queryCodesNotFloored += offsets.cols(0, numTablesToSearch - 1); allProjInTables = arma::floor(queryCodesNotFloored / hashWidth); @@ -788,13 +784,12 @@ void LSHSearch::ReturnIndicesFromTable( // the primary hash table). hashMat(arma::span(1, T), i) = // Compute code of rows 1:end of column i arma::conv_to< arma::Col >:: // floor by typecasting to size_t - from( secondHashWeights.t() * additionalProbingBins ); + from(secondHashWeights.t() * additionalProbingBins); for (size_t p = 1; p < T + 1; ++p) hashMat(p, i) = (hashMat(p, i) % secondHashSize); } } - // Count number of points hashed in the same bucket as the query. size_t maxNumPoints = 0; for (size_t i = 0; i < numTablesToSearch; ++i) @@ -1146,7 +1141,7 @@ void LSHSearch::Serialize(Archive& ar, // the value referenceSet->n_cols is seen. size_t len = 0; - for ( ; len < tmpSecondHashTable.n_rows; ++len) + for (; len < tmpSecondHashTable.n_rows; ++len) if (tmpSecondHashTable(len, i) == referenceSet->n_cols) break; diff --git a/src/mlpack/methods/matrix_completion/matrix_completion.cpp b/src/mlpack/methods/matrix_completion/matrix_completion.cpp index fa5c2c4abc..a3058094ab 100644 --- a/src/mlpack/methods/matrix_completion/matrix_completion.cpp +++ b/src/mlpack/methods/matrix_completion/matrix_completion.cpp @@ -54,20 +54,26 @@ MatrixCompletion::MatrixCompletion(const size_t m, void MatrixCompletion::CheckValues() { if (indices.n_rows != 2) - Log::Fatal << "MatrixCompletion::CheckValues(): matrix of constraint indices does " - << "not have 2 rows!" << std::endl; + { + Log::Fatal << "MatrixCompletion::CheckValues(): matrix of constraint " + << "indices does not have 2 rows!" << std::endl; + } if (indices.n_cols != values.n_elem) - Log::Fatal << "MatrixCompletion::CheckValues(): the number of constraint indices " - << "(columns of constraint indices matrix) does not match the number of " - << "constraint values (length of constraint value vector)!" << std::endl; + { + Log::Fatal << "MatrixCompletion::CheckValues(): the number of constraint " + << "indices (columns of constraint indices matrix) does not match the " + << "number of constraint values (length of constraint value vector)!" + << std::endl; + } for (size_t i = 0; i < values.n_elem; i++) { if (indices(0, i) >= m || indices(1, i) >= n) - Log::Fatal << "MatrixCompletion::CheckValues(): indices (" << indices(0, i) << ", " - << indices(1, i) << ") are out of bounds for matrix of size " << m << " x " - << "n!" << std::endl; + Log::Fatal << "MatrixCompletion::CheckValues(): indices (" + << indices(0, i) << ", " << indices(1, i) + << ") are out of bounds for matrix of size " << m << " x n!" + << std::endl; } } diff --git a/src/mlpack/methods/matrix_completion/matrix_completion.hpp b/src/mlpack/methods/matrix_completion/matrix_completion.hpp index a806f105c5..6b0646080e 100644 --- a/src/mlpack/methods/matrix_completion/matrix_completion.hpp +++ b/src/mlpack/methods/matrix_completion/matrix_completion.hpp @@ -112,7 +112,10 @@ class MatrixCompletion void Recover(arma::mat& recovered); //! Return the underlying SDP. - const optimization::LRSDP>& Sdp() const { return sdp; } + const optimization::LRSDP>& Sdp() const + { + return sdp; + } //! Modify the underlying SDP. optimization::LRSDP>& Sdp() { return sdp; } diff --git a/src/mlpack/methods/mvu/mvu.cpp b/src/mlpack/methods/mvu/mvu.cpp index 8c02d0ca6d..b120da16a4 100644 --- a/src/mlpack/methods/mvu/mvu.cpp +++ b/src/mlpack/methods/mvu/mvu.cpp @@ -13,7 +13,6 @@ */ #include "mvu.hpp" -//#include #include #include diff --git a/src/mlpack/methods/naive_bayes/nbc_main.cpp b/src/mlpack/methods/naive_bayes/nbc_main.cpp index bbceaf7825..31a559fe1a 100644 --- a/src/mlpack/methods/naive_bayes/nbc_main.cpp +++ b/src/mlpack/methods/naive_bayes/nbc_main.cpp @@ -28,10 +28,10 @@ PROGRAM_INFO("Parametric Naive Bayes Classifier", "training set, or loads a model from the given model file, and then may use" " that trained model to classify the points in a given test set." "\n\n" - "Labels are expected to be the last row of the training set (--training_file)," - " but labels can also be passed in separately as their own file " - "(--labels_file). If training is not desired, a pre-existing model can be " - "loaded with the --input_model_file (-m) option." + "Labels are expected to be the last row of the training set " + "(--training_file), but labels can also be passed in separately as their " + "own file (--labels_file). If training is not desired, a pre-existing " + "model can be loaded with the --input_model_file (-m) option." "\n\n" "The '--incremental_variance' option can be used to force the training to " "use an incremental algorithm for calculating variance. This is slower, " diff --git a/src/mlpack/methods/neighbor_search/neighbor_search.hpp b/src/mlpack/methods/neighbor_search/neighbor_search.hpp index 3e0a9a01a0..c933e907d0 100644 --- a/src/mlpack/methods/neighbor_search/neighbor_search.hpp +++ b/src/mlpack/methods/neighbor_search/neighbor_search.hpp @@ -26,9 +26,9 @@ #include "neighbor_search_rules.hpp" namespace mlpack { -namespace neighbor /** Neighbor-search routines. These include - * all-nearest-neighbors and all-furthest-neighbors - * searches. */ { +// Neighbor-search routines. These include all-nearest-neighbors and +// all-furthest-neighbors searches. +namespace neighbor { // Forward declaration. template diff --git a/src/mlpack/methods/neighbor_search/neighbor_search_impl.hpp b/src/mlpack/methods/neighbor_search/neighbor_search_impl.hpp index ec814581b5..a238cd940d 100644 --- a/src/mlpack/methods/neighbor_search/neighbor_search_impl.hpp +++ b/src/mlpack/methods/neighbor_search/neighbor_search_impl.hpp @@ -584,7 +584,7 @@ DualTreeTraversalType, SingleTreeTraversalType>::Search( typedef NeighborSearchRules RuleType; - switch(searchMode) + switch (searchMode) { case NAIVE_MODE: { diff --git a/src/mlpack/methods/pca/pca_impl.hpp b/src/mlpack/methods/pca/pca_impl.hpp index 87467f5110..f9c62cb432 100644 --- a/src/mlpack/methods/pca/pca_impl.hpp +++ b/src/mlpack/methods/pca/pca_impl.hpp @@ -26,8 +26,8 @@ namespace mlpack { namespace pca { template -PCAType::PCAType(const bool scaleData, - const DecompositionPolicy& decomposition) : +PCAType::PCAType( + const bool scaleData, const DecompositionPolicy& decomposition) : scaleData(scaleData), decomposition(decomposition) { } diff --git a/src/mlpack/methods/pca/pca_main.cpp b/src/mlpack/methods/pca/pca_main.cpp index 2019f17ec4..c765b3647b 100644 --- a/src/mlpack/methods/pca/pca_main.cpp +++ b/src/mlpack/methods/pca/pca_main.cpp @@ -74,7 +74,6 @@ void RunPCA(arma::mat& dataset, Log::Info << (varRetained * 100) << "% of variance retained (" << dataset.n_rows << " dimensions)." << endl; - } int main(int argc, char** argv) diff --git a/src/mlpack/methods/perceptron/perceptron.hpp b/src/mlpack/methods/perceptron/perceptron.hpp index 9d51fdbbb4..9fb531eb38 100644 --- a/src/mlpack/methods/perceptron/perceptron.hpp +++ b/src/mlpack/methods/perceptron/perceptron.hpp @@ -136,7 +136,7 @@ class Perceptron //! Modify the biases. You had better know what you are doing! arma::vec& Biases() { return biases; } -private: + private: //! The maximum number of iterations during training. size_t maxIterations; diff --git a/src/mlpack/methods/perceptron/perceptron_main.cpp b/src/mlpack/methods/perceptron/perceptron_main.cpp index f22b8b65dc..ec1c56890e 100644 --- a/src/mlpack/methods/perceptron/perceptron_main.cpp +++ b/src/mlpack/methods/perceptron/perceptron_main.cpp @@ -63,8 +63,7 @@ PROGRAM_INFO("Perceptron", "data must match. So you cannot pass a perceptron model trained on 2 " "classes and then re-train with a 4-class dataset. Similarly, attempting " "classification on a 3-dimensional dataset with a perceptron that has been " - "trained on 8 dimensions will cause an error." - ); + "trained on 8 dimensions will cause an error."); // When we save a model, we must also save the class mappings. So we use this // auxiliary structure to store both the perceptron and the mapping, and we'll @@ -94,8 +93,8 @@ class PerceptronModel PARAM_MATRIX_IN("training", "A matrix containing the training set.", "t"); PARAM_UROW_IN("labels", "A matrix containing labels for the training set.", "l"); -PARAM_INT_IN("max_iterations","The maximum number of iterations the perceptron " - "is to be run", "n", 1000); +PARAM_INT_IN("max_iterations", "The maximum number of iterations the " + "perceptron is to be run", "n", 1000); // Model loading/saving. PARAM_MODEL_IN(PerceptronModel, "input_model", "Input perceptron model.", "m"); diff --git a/src/mlpack/methods/preprocess/preprocess_describe_main.cpp b/src/mlpack/methods/preprocess/preprocess_describe_main.cpp index 11e9484af3..2f310b6c8e 100644 --- a/src/mlpack/methods/preprocess/preprocess_describe_main.cpp +++ b/src/mlpack/methods/preprocess/preprocess_describe_main.cpp @@ -144,7 +144,7 @@ double Kurtosis(const arma::rowvec& input, */ double StandardError(const size_t size, const double& fStd) { - return fStd / sqrt(size); + return fStd / sqrt(size); } int main(int argc, char** argv) diff --git a/src/mlpack/methods/preprocess/preprocess_imputer_main.cpp b/src/mlpack/methods/preprocess/preprocess_imputer_main.cpp index 267f3df7d7..f95c7c5cab 100644 --- a/src/mlpack/methods/preprocess/preprocess_imputer_main.cpp +++ b/src/mlpack/methods/preprocess/preprocess_imputer_main.cpp @@ -155,7 +155,8 @@ int main(int argc, char** argv) else if (strategy == "custom") { CustomImputation strat(customValue); - Imputer> imputer(info, strat); + Imputer> imputer( + info, strat); } else { diff --git a/src/mlpack/methods/radical/radical.cpp b/src/mlpack/methods/radical/radical.cpp index 93cf392275..a705e61235 100644 --- a/src/mlpack/methods/radical/radical.cpp +++ b/src/mlpack/methods/radical/radical.cpp @@ -126,7 +126,7 @@ void Radical::DoRadical(const mat& matXT, mat& matY, mat& matW) // In the RADICAL code, they do not copy and perturb initially, although the // paper does. We follow the code as it should match their reported results // and likely does a better job bouncing out of local optima. - //GeneratePerturbedX(X, X); + // GeneratePerturbedX(X, X); // Initialize the unmixing matrix to the whitening matrix. Timer::Start("radical_do_radical"); diff --git a/src/mlpack/methods/range_search/range_search_stat.hpp b/src/mlpack/methods/range_search/range_search_stat.hpp index 8df48d7d12..8ffaf241bc 100644 --- a/src/mlpack/methods/range_search/range_search_stat.hpp +++ b/src/mlpack/methods/range_search/range_search_stat.hpp @@ -56,7 +56,7 @@ class RangeSearchStat double lastDistance; }; -} // namespace neighbor +} // namespace range } // namespace mlpack #endif diff --git a/src/mlpack/methods/range_search/rs_model.cpp b/src/mlpack/methods/range_search/rs_model.cpp index 747c3f600e..529c292f4d 100644 --- a/src/mlpack/methods/range_search/rs_model.cpp +++ b/src/mlpack/methods/range_search/rs_model.cpp @@ -35,7 +35,7 @@ RSModel::RSModel(const RSModel& other) : randomBasis(other.randomBasis), rSearch(other.rSearch) { - + // Nothing to do. } // Move constructor. @@ -128,7 +128,7 @@ void RSModel::BuildModel(arma::mat&& referenceSet, break; case R_TREE: - rSearch = new RSType(naive,singleMode); + rSearch = new RSType(naive, singleMode); break; case R_STAR_TREE: diff --git a/src/mlpack/methods/range_search/rs_model_impl.hpp b/src/mlpack/methods/range_search/rs_model_impl.hpp index f244719f56..243acd6565 100644 --- a/src/mlpack/methods/range_search/rs_model_impl.hpp +++ b/src/mlpack/methods/range_search/rs_model_impl.hpp @@ -206,18 +206,18 @@ void SerializeVisitor::operator()(RSType* rs) const template bool& SingleModeVisitor::operator()(RSType* rs) const { - if (rs) - return rs->SingleMode(); - throw std::runtime_error("no range search model initialized"); + if (rs) + return rs->SingleMode(); + throw std::runtime_error("no range search model initialized"); } //! Exposes Naive() function of given RSType template bool& NaiveVisitor::operator()(RSType* rs) const { - if (rs) - return rs->Naive(); - throw std::runtime_error("no range search model initialized"); + if (rs) + return rs->Naive(); + throw std::runtime_error("no range search model initialized"); } // Serialize the model. diff --git a/src/mlpack/methods/rann/krann_main.cpp b/src/mlpack/methods/rann/krann_main.cpp index aaa98f3f21..760bbd17bc 100644 --- a/src/mlpack/methods/rann/krann_main.cpp +++ b/src/mlpack/methods/rann/krann_main.cpp @@ -31,10 +31,10 @@ PROGRAM_INFO("K-Rank-Approximate-Nearest-Neighbors (kRANN)", "This program will calculate the k rank-approximate-nearest-neighbors of a " "set of points. You may specify a separate set of reference points and " "query points, or just a reference set which will be used as both the " - "reference and query set. You must specify the rank approximation (in \%) " + "reference and query set. You must specify the rank approximation (in \\%) " "(and optionally the success probability)." "\n\n" - "For example, the following will return 5 neighbors from the top 0.1\% of " + "For example, the following will return 5 neighbors from the top 0.1\\% of " "the data (with probability 0.95) for each point in 'input.csv' and store " "the distances in 'distances.csv' and the neighbors in the file " "'neighbors.csv':" @@ -102,7 +102,7 @@ int main(int argc, char *argv[]) math::RandomSeed((size_t) CLI::GetParam("seed")); else math::RandomSeed((size_t) std::time(NULL)); - // A user cannot specify both reference data and a model. + // A user cannot specify both reference data and a model. if (CLI::HasParam("reference") && CLI::HasParam("input_model")) Log::Fatal << "Only one of --reference_file (-r) or --input_model_file (-m)" << " may be specified!" << endl; diff --git a/src/mlpack/methods/rann/ra_search_rules_impl.hpp b/src/mlpack/methods/rann/ra_search_rules_impl.hpp index dc9e9b4555..2a617871be 100644 --- a/src/mlpack/methods/rann/ra_search_rules_impl.hpp +++ b/src/mlpack/methods/rann/ra_search_rules_impl.hpp @@ -84,7 +84,7 @@ RASearchRules(const arma::mat& referenceSet, for (size_t i = 0; i < querySet.n_cols; i++) candidates.push_back(pqueue); - if (naive)// No tree traversal; just do naive sampling here. + if (naive) // No tree traversal; just do naive sampling here. { // Sample enough points. arma::uvec distinctSamples; diff --git a/src/mlpack/methods/rann/ra_util.cpp b/src/mlpack/methods/rann/ra_util.cpp index a5dcd0b1b6..c613a21871 100644 --- a/src/mlpack/methods/rann/ra_util.cpp +++ b/src/mlpack/methods/rann/ra_util.cpp @@ -70,7 +70,6 @@ size_t mlpack::neighbor::RAUtil::MinimumSamplesReqd(const size_t n, } } m = (ub + lb) / 2; - } while (!done); return (std::min(m + 1, n)); @@ -89,7 +88,6 @@ double mlpack::neighbor::RAUtil::SuccessProbability(const size_t n, double eps = (double) t / (double) n; return 1.0 - std::pow(1.0 - eps, (double) m); - } // Faster implementation for topK = 1. else { @@ -153,7 +151,7 @@ double mlpack::neighbor::RAUtil::SuccessProbability(const size_t n, else jTrans = m - j; - for(size_t i = 2; i <= jTrans; i++) + for (size_t i = 2; i <= jTrans; i++) { mCj *= (double) (m - (i - 1)); mCj /= (double) i; diff --git a/src/mlpack/methods/regularized_svd/regularized_svd.hpp b/src/mlpack/methods/regularized_svd/regularized_svd.hpp index 233cac2c9a..ebb07f1f17 100644 --- a/src/mlpack/methods/regularized_svd/regularized_svd.hpp +++ b/src/mlpack/methods/regularized_svd/regularized_svd.hpp @@ -60,7 +60,6 @@ template< class RegularizedSVD { public: - /** * Constructor for Regularized SVD. Obtains the user and item matrices after * training on the passed data. The constructor initiates an object of class diff --git a/src/mlpack/methods/regularized_svd/regularized_svd_function.cpp b/src/mlpack/methods/regularized_svd/regularized_svd_function.cpp index f71daacc92..b360494360 100644 --- a/src/mlpack/methods/regularized_svd/regularized_svd_function.cpp +++ b/src/mlpack/methods/regularized_svd/regularized_svd_function.cpp @@ -42,7 +42,7 @@ double RegularizedSVDFunction::Evaluate(const arma::mat& parameters) const double cost = 0.0; - for(size_t i = 0; i < data.n_cols; i++) + for (size_t i = 0; i < data.n_cols; i++) { // Indices for accessing the the correct parameter columns. const size_t user = data(0, i); @@ -102,7 +102,7 @@ void RegularizedSVDFunction::Gradient(const arma::mat& parameters, gradient.zeros(rank, numUsers + numItems); - for(size_t i = 0; i < data.n_cols; i++) + for (size_t i = 0; i < data.n_cols; i++) { // Indices for accessing the the correct parameter columns. const size_t user = data(0, i); @@ -141,13 +141,13 @@ double StandardSGD::Optimize( double overallObjective = 0; // Calculate the first objective function. - for(size_t i = 0; i < numFunctions; i++) + for (size_t i = 0; i < numFunctions; i++) overallObjective += function.Evaluate(parameters, i); const arma::mat data = function.Dataset(); // Now iterate! - for(size_t i = 1; i != maxIterations; i++, currentFunction++) + for (size_t i = 1; i != maxIterations; i++, currentFunction++) { // Is this iteration the start of a sequence? if ((currentFunction % numFunctions) == 0) diff --git a/src/mlpack/methods/regularized_svd/regularized_svd_function.hpp b/src/mlpack/methods/regularized_svd/regularized_svd_function.hpp index c1ab55a074..11dc0e18db 100644 --- a/src/mlpack/methods/regularized_svd/regularized_svd_function.hpp +++ b/src/mlpack/methods/regularized_svd/regularized_svd_function.hpp @@ -22,7 +22,6 @@ namespace svd { class RegularizedSVDFunction { public: - /** * Constructor for RegularizedSVDFunction class. The constructor calculates * the number of users and items in the passed data. It also randomly diff --git a/src/mlpack/methods/reinforcement_learning/environment/cart_pole.hpp b/src/mlpack/methods/reinforcement_learning/environment/cart_pole.hpp index 20d7ea95c9..0884333c80 100644 --- a/src/mlpack/methods/reinforcement_learning/environment/cart_pole.hpp +++ b/src/mlpack/methods/reinforcement_learning/environment/cart_pole.hpp @@ -26,7 +26,6 @@ namespace rl { class CartPole { public: - /** * Implementation of the state of Cart Pole. Each state is a tuple vector * (position, velocity, angle, angular velocity). @@ -37,7 +36,7 @@ class CartPole /** * Construct a state instance. */ - State() : data(dimension) + State() : data(4) { /* Nothing to do here. */ } /** @@ -74,9 +73,6 @@ class CartPole //! Encode the state to a column vector. const arma::colvec& Encode() const { return data; } - //! Dimension of the encoded state. - static constexpr size_t dimension = 4; - private: //! Locally-stored (position, velocity, angle, angular velocity). arma::colvec data; @@ -229,4 +225,4 @@ class CartPole } // namespace rl } // namespace mlpack -#endif \ No newline at end of file +#endif diff --git a/src/mlpack/methods/reinforcement_learning/environment/mountain_car.hpp b/src/mlpack/methods/reinforcement_learning/environment/mountain_car.hpp index 1c26642cc9..203e90fe6b 100644 --- a/src/mlpack/methods/reinforcement_learning/environment/mountain_car.hpp +++ b/src/mlpack/methods/reinforcement_learning/environment/mountain_car.hpp @@ -27,7 +27,6 @@ namespace rl { class MountainCar { public: - /** * Implementation of state of Mountain Car. Each state is a * (velocity, position) vector. @@ -38,7 +37,7 @@ class MountainCar /** * Construct a state instance. */ - State(): data(dimension, arma::fill::zeros) + State(): data(2, arma::fill::zeros) { /* Nothing to do here. */ } /** @@ -65,9 +64,6 @@ class MountainCar //! Encode the state to a column vector. const arma::colvec& Encode() const { return data; } - //! Dimension of the encoded state. - static constexpr size_t dimension = 2; - private: //! Locally-stored velocity and position vector. arma::colvec data; @@ -193,4 +189,4 @@ class MountainCar } // namespace rl } // namespace mlpack -#endif \ No newline at end of file +#endif diff --git a/src/mlpack/methods/reinforcement_learning/replay/random_replay.hpp b/src/mlpack/methods/reinforcement_learning/replay/random_replay.hpp index 29bc16d1d6..7eb73ce772 100644 --- a/src/mlpack/methods/reinforcement_learning/replay/random_replay.hpp +++ b/src/mlpack/methods/reinforcement_learning/replay/random_replay.hpp @@ -111,8 +111,9 @@ class RandomReplay arma::icolvec& isTerminal) { size_t upperBound = full ? capacity : position; - arma::uvec sampledIndices = - arma::randi(batchSize, arma::distr_param(0, upperBound - 1)); + arma::uvec sampledIndices = arma::randi( + batchSize, arma::distr_param(0, upperBound - 1)); + sampledStates = states.cols(sampledIndices); sampledActions = actions.elem(sampledIndices); sampledRewards = rewards.elem(sampledIndices); diff --git a/src/mlpack/methods/softmax_regression/softmax_regression.hpp b/src/mlpack/methods/softmax_regression/softmax_regression.hpp index 92365438d2..27f245ffaa 100644 --- a/src/mlpack/methods/softmax_regression/softmax_regression.hpp +++ b/src/mlpack/methods/softmax_regression/softmax_regression.hpp @@ -155,8 +155,9 @@ class SoftmaxRegression * @param labels Predicted labels for each point. * @param probabilities Class probabilities for each point. */ - void Classify(const arma::mat& dataset, arma::Row& labels, - arma::mat& probabilites) const; + void Classify(const arma::mat& dataset, + arma::Row& labels, + arma::mat& probabilites) const; /** * Classify the given points, returning class probabilities for each point. @@ -195,7 +196,8 @@ class SoftmaxRegression * @param numClasses Number of classes for classification. * @return Objective value of the final point. */ - double Train(const arma::mat& data, const arma::Row& labels, + double Train(const arma::mat& data, + const arma::Row& labels, const size_t numClasses); //! Sets the number of classes. diff --git a/src/mlpack/methods/softmax_regression/softmax_regression_function.cpp b/src/mlpack/methods/softmax_regression/softmax_regression_function.cpp index 27c18049d4..37d6eff23d 100644 --- a/src/mlpack/methods/softmax_regression/softmax_regression_function.cpp +++ b/src/mlpack/methods/softmax_regression/softmax_regression_function.cpp @@ -9,7 +9,7 @@ * 3-clause BSD license along with mlpack. If not, see * http://www.opensource.org/licenses/BSD-3-Clause for more information. */ - #include "softmax_regression_function.hpp" +#include "softmax_regression_function.hpp" using namespace mlpack; using namespace mlpack::regression; @@ -73,8 +73,8 @@ void SoftmaxRegressionFunction::InitializeWeights( * labels. The output is in the form of a matrix, which leads to simpler * calculations in the Evaluate() and Gradient() methods. */ -void SoftmaxRegressionFunction::GetGroundTruthMatrix(const arma::Row& labels, - arma::sp_mat& groundTruth) +void SoftmaxRegressionFunction::GetGroundTruthMatrix( + const arma::Row& labels, arma::sp_mat& groundTruth) { // Calculate the ground truth matrix according to the labels passed. The // ground truth matrix is a matrix of dimensions 'numClasses * numExamples', @@ -87,7 +87,7 @@ void SoftmaxRegressionFunction::GetGroundTruthMatrix(const arma::Row& la // Row pointers are the labels of the examples, and column pointers are the // number of cumulative entries made uptil that column. - for(size_t i = 0; i < labels.n_elem; i++) + for (size_t i = 0; i < labels.n_elem; i++) { rowPointers(i) = labels(i); colPointers(i+1) = i + 1; diff --git a/src/mlpack/methods/softmax_regression/softmax_regression_impl.hpp b/src/mlpack/methods/softmax_regression/softmax_regression_impl.hpp index 57c4f78b4e..9d5e3ee8fc 100644 --- a/src/mlpack/methods/softmax_regression/softmax_regression_impl.hpp +++ b/src/mlpack/methods/softmax_regression/softmax_regression_impl.hpp @@ -27,9 +27,8 @@ SoftmaxRegression(const size_t inputSize, lambda(0.0001), fitIntercept(fitIntercept) { - SoftmaxRegressionFunction::InitializeWeights(parameters, - inputSize, numClasses, - fitIntercept); + SoftmaxRegressionFunction::InitializeWeights( + parameters, inputSize, numClasses, fitIntercept); } template class OptimizerType> diff --git a/src/mlpack/methods/sparse_autoencoder/sparse_autoencoder_function.cpp b/src/mlpack/methods/sparse_autoencoder/sparse_autoencoder_function.cpp index 050de5efa4..3b719bd725 100644 --- a/src/mlpack/methods/sparse_autoencoder/sparse_autoencoder_function.cpp +++ b/src/mlpack/methods/sparse_autoencoder/sparse_autoencoder_function.cpp @@ -64,7 +64,7 @@ const arma::mat SparseAutoencoderFunction::InitializeWeights() // layers. The formula used is r = sqrt(6) / sqrt(vSize + hSize + 1). const double range = sqrt(6) / sqrt(visibleSize + hiddenSize + 1); - //Shift range of w1 and w2 values from [0, 1] to [-r, r]. + // Shift range of w1 and w2 values from [0, 1] to [-r, r]. parameters.submat(0, 0, 2 * hiddenSize - 1, visibleSize - 1) = 2 * range * (parameters.submat(0, 0, 2 * hiddenSize - 1, visibleSize - 1) - 0.5); diff --git a/src/mlpack/methods/sparse_coding/sparse_coding.cpp b/src/mlpack/methods/sparse_coding/sparse_coding.cpp index 8b8a62be07..6edf7d7eb6 100644 --- a/src/mlpack/methods/sparse_coding/sparse_coding.cpp +++ b/src/mlpack/methods/sparse_coding/sparse_coding.cpp @@ -120,17 +120,17 @@ double SparseCoding::OptimizeDictionary(const arma::mat& data, // numerically stable than just using inv(A) for everything. arma::vec dualVars = arma::zeros(nActiveAtoms); - //vec dualVars = 1e-14 * ones(nActiveAtoms); + // vec dualVars = 1e-14 * ones(nActiveAtoms); // Method used by feature sign code - fails miserably here. Perhaps the // MATLAB optimizer fmincon does something clever? - //vec dualVars = 10.0 * randu(nActiveAtoms, 1); + // vec dualVars = 10.0 * randu(nActiveAtoms, 1); - //vec dualVars = diagvec(solve(dictionary, data * trans(codes)) + // vec dualVars = diagvec(solve(dictionary, data * trans(codes)) // - codes * trans(codes)); - //for (size_t i = 0; i < dualVars.n_elem; i++) - // if (dualVars(i) < 0) - // dualVars(i) = 0; + // for (size_t i = 0; i < dualVars.n_elem; i++) + // if (dualVars(i) < 0) + // dualVars(i) = 0; bool converged = false; @@ -163,7 +163,6 @@ double SparseCoding::OptimizeDictionary(const arma::mat& data, arma::mat hessian = -(-2 * (matAInvZXT * trans(matAInvZXT)) % inv(A)); arma::vec searchDirection = -solve(hessian, gradient); - //printf("%e\n", norm(searchDirection, 2)); // Armijo line search. const double c = 1e-4; diff --git a/src/mlpack/tests/ada_grad_test.cpp b/src/mlpack/tests/ada_grad_test.cpp index 86d5737b84..aed0659a80 100644 --- a/src/mlpack/tests/ada_grad_test.cpp +++ b/src/mlpack/tests/ada_grad_test.cpp @@ -93,7 +93,8 @@ BOOST_AUTO_TEST_CASE(AdaGradLogisticRegressionTest) LogisticRegression<> lr(shuffledData.n_rows, 0.5); LogisticRegressionFunction<> lrf(shuffledData, shuffledResponses, 0.5); - AdaGrad > adagrad(lrf, 0.99, 1e-8, 5000000, 1e-9, true); + AdaGrad > adagrad( + lrf, 0.99, 1e-8, 5000000, 1e-9, true); lr.Train(adagrad); // Ensure that the error is close to zero. diff --git a/src/mlpack/tests/adaboost_test.cpp b/src/mlpack/tests/adaboost_test.cpp index 542eb596bc..ebeda66a26 100644 --- a/src/mlpack/tests/adaboost_test.cpp +++ b/src/mlpack/tests/adaboost_test.cpp @@ -78,7 +78,7 @@ BOOST_AUTO_TEST_CASE(WeakLearnerErrorIris) arma::Mat labels; - if (!data::Load("iris_labels.txt",labels)) + if (!data::Load("iris_labels.txt", labels)) BOOST_FAIL("Cannot load labels for iris iris_labels.txt"); // Define your own weak learner, perceptron in this case. @@ -125,7 +125,7 @@ BOOST_AUTO_TEST_CASE(HammingLossBoundVertebralColumn) BOOST_FAIL("Cannot load test dataset vc2.csv!"); arma::Mat labels; - if (!data::Load("vc2_labels.txt",labels)) + if (!data::Load("vc2_labels.txt", labels)) BOOST_FAIL("Cannot load labels for vc2_labels.txt"); // Define your own weak learner, perceptron in this case. @@ -163,7 +163,7 @@ BOOST_AUTO_TEST_CASE(WeakLearnerErrorVertebralColumn) BOOST_FAIL("Cannot load test dataset vc2.csv!"); arma::Mat labels; - if (!data::Load("vc2_labels.txt",labels)) + if (!data::Load("vc2_labels.txt", labels)) BOOST_FAIL("Cannot load labels for vc2_labels.txt"); // Define your own weak learner, perceptron in this case. @@ -210,7 +210,7 @@ BOOST_AUTO_TEST_CASE(HammingLossBoundNonLinearSepData) BOOST_FAIL("Cannot load test dataset train_nonlinsep.txt!"); arma::Mat labels; - if (!data::Load("train_labels_nonlinsep.txt",labels)) + if (!data::Load("train_labels_nonlinsep.txt", labels)) BOOST_FAIL("Cannot load labels for train_labels_nonlinsep.txt"); // Define your own weak learner, perceptron in this case. @@ -248,7 +248,7 @@ BOOST_AUTO_TEST_CASE(WeakLearnerErrorNonLinearSepData) BOOST_FAIL("Cannot load test dataset train_nonlinsep.txt!"); arma::Mat labels; - if (!data::Load("train_labels_nonlinsep.txt",labels)) + if (!data::Load("train_labels_nonlinsep.txt", labels)) BOOST_FAIL("Cannot load labels for train_labels_nonlinsep.txt"); // Define your own weak learner, perceptron in this case. @@ -295,7 +295,7 @@ BOOST_AUTO_TEST_CASE(HammingLossIris_DS) BOOST_FAIL("Cannot load test dataset iris.csv!"); arma::Mat labels; - if (!data::Load("iris_labels.txt",labels)) + if (!data::Load("iris_labels.txt", labels)) BOOST_FAIL("Cannot load labels for iris_labels.txt"); // Define your own weak learner, decision stumps in this case. @@ -386,7 +386,7 @@ BOOST_AUTO_TEST_CASE(HammingLossBoundVertebralColumn_DS) BOOST_FAIL("Cannot load test dataset vc2.csv!"); arma::Mat labels; - if (!data::Load("vc2_labels.txt",labels)) + if (!data::Load("vc2_labels.txt", labels)) BOOST_FAIL("Cannot load labels for vc2_labels.txt"); // Define your own weak learner, decision stumps in this case. @@ -475,7 +475,7 @@ BOOST_AUTO_TEST_CASE(HammingLossBoundNonLinearSepData_DS) BOOST_FAIL("Cannot load test dataset train_nonlinsep.txt!"); arma::Mat labels; - if (!data::Load("train_labels_nonlinsep.txt",labels)) + if (!data::Load("train_labels_nonlinsep.txt", labels)) BOOST_FAIL("Cannot load labels for train_labels_nonlinsep.txt"); // Define your own weak learner, decision stumps in this case. @@ -516,7 +516,7 @@ BOOST_AUTO_TEST_CASE(WeakLearnerErrorNonLinearSepData_DS) BOOST_FAIL("Cannot load test dataset train_nonlinsep.txt!"); arma::Mat labels; - if (!data::Load("train_labels_nonlinsep.txt",labels)) + if (!data::Load("train_labels_nonlinsep.txt", labels)) BOOST_FAIL("Cannot load labels for train_labels_nonlinsep.txt"); // Define your own weak learner, decision stumps in this case. @@ -565,7 +565,7 @@ BOOST_AUTO_TEST_CASE(ClassifyTest_VERTEBRALCOL) BOOST_FAIL("Cannot load test dataset vc2.csv!"); arma::Mat labels; - if (!data::Load("vc2_labels.txt",labels)) + if (!data::Load("vc2_labels.txt", labels)) BOOST_FAIL("Cannot load labels for vc2_labels.txt"); // Define your own weak learner, perceptron in this case. @@ -579,7 +579,7 @@ BOOST_AUTO_TEST_CASE(ClassifyTest_VERTEBRALCOL) arma::Mat trueTestLabels; - if (!data::Load("vc2_test_labels.txt",trueTestLabels)) + if (!data::Load("vc2_test_labels.txt", trueTestLabels)) BOOST_FAIL("Cannot load labels for vc2_test_labels.txt"); Row perceptronPrediction(labels.n_cols); diff --git a/src/mlpack/tests/ann_layer_test.cpp b/src/mlpack/tests/ann_layer_test.cpp index 3597c04636..08d1cc4537 100644 --- a/src/mlpack/tests/ann_layer_test.cpp +++ b/src/mlpack/tests/ann_layer_test.cpp @@ -32,7 +32,7 @@ void ResetFunction( T& layer, typename std::enable_if::value>::type* = 0) { - layer.Reset(); + layer.Reset(); } template @@ -134,7 +134,7 @@ double JacobianPerformanceTest(ModuleType& module, inputTemp(i) = inputTemp(i) - (2 * eps); double outputB = module.Forward(std::move(input), std::move(target)); - centralDifferenceTemp(i) = (outputA - outputB) / ( 2 * eps); + centralDifferenceTemp(i) = (outputA - outputB) / (2 * eps); inputTemp(i) = inputTemp(i) + eps; } @@ -923,4 +923,4 @@ BOOST_AUTO_TEST_CASE(SimpleLogSoftmaxLayerTest) BOOST_REQUIRE_EQUAL(arma::accu(delta), 0); } -BOOST_AUTO_TEST_SUITE_END(); \ No newline at end of file +BOOST_AUTO_TEST_SUITE_END(); diff --git a/src/mlpack/tests/cli_test.cpp b/src/mlpack/tests/cli_test.cpp index 8a911d8875..c36152c6c5 100644 --- a/src/mlpack/tests/cli_test.cpp +++ b/src/mlpack/tests/cli_test.cpp @@ -144,7 +144,6 @@ BOOST_AUTO_TEST_CASE(TestBooleanOption) BOOST_REQUIRE_EQUAL(CLI::GetParam("flag_test"), true); BOOST_REQUIRE_EQUAL(CLI::HasParam("flag_test"), true); - } /** @@ -215,9 +214,10 @@ BOOST_AUTO_TEST_CASE(InputColVectorParamTest) { AddRequiredCLIOptions(); - CLI::Add(arma::vec(), "vector", "Test vector", 'l', false, true, false); + CLI::Add( + arma::vec(), "vector", "Test vector", 'l', false, true, false); - //fake aruguments + // Fake arguments. const char* argv[3]; argv[0] = "./test"; argv[1] = "-l"; @@ -225,12 +225,12 @@ BOOST_AUTO_TEST_CASE(InputColVectorParamTest) int argc = 3; - // The const-cast is a little hacky but should be fine... + // The const-cast is a little hacky but should be fine... Log::Fatal.ignoreInput = true; CLI::ParseCommandLine(argc, const_cast(argv)); Log::Fatal.ignoreInput = false; - // The --vector parameter should exist. + // The --vector parameter should exist. BOOST_REQUIRE(CLI::HasParam("vector")); // The --vector_file parameter should not exist (it should be transparent from // inside the program). @@ -252,9 +252,10 @@ BOOST_AUTO_TEST_CASE(InputUnsignedColVectorParamTest) { AddRequiredCLIOptions(); - CLI::Add>(arma::Col(), "vector", "Test vector", 'l', false, true, false); + CLI::Add>( + arma::Col(), "vector", "Test vector", 'l', false, true, false); - //fake aruguments + // Fake arguments. const char* argv[3]; argv[0] = "./test"; argv[1] = "-l"; @@ -262,12 +263,12 @@ BOOST_AUTO_TEST_CASE(InputUnsignedColVectorParamTest) int argc = 3; - // The const-cast is a little hacky but should be fine... + // The const-cast is a little hacky but should be fine... Log::Fatal.ignoreInput = true; CLI::ParseCommandLine(argc, const_cast(argv)); Log::Fatal.ignoreInput = false; - // The --vector parameter should exist. + // The --vector parameter should exist. BOOST_REQUIRE(CLI::HasParam("vector")); // The --vector_file parameter should not exist (it should be transparent from // inside the program). @@ -291,7 +292,7 @@ BOOST_AUTO_TEST_CASE(InputRowVectorParamTest) CLI::Add(arma::rowvec(), "row", "Test vector", 'l', false, true, false); - //fake aruguments + // Fake arguments. const char* argv[3]; argv[0] = "./test"; argv[1] = "-l"; @@ -299,12 +300,12 @@ BOOST_AUTO_TEST_CASE(InputRowVectorParamTest) int argc = 3; - // The const-cast is a little hacky but should be fine... + // The const-cast is a little hacky but should be fine... Log::Fatal.ignoreInput = true; CLI::ParseCommandLine(argc, const_cast(argv)); Log::Fatal.ignoreInput = false; - // The --vector parameter should exist. + // The --vector parameter should exist. BOOST_REQUIRE(CLI::HasParam("row")); // The --vector_file parameter should not exist (it should be transparent from // inside the program). @@ -328,7 +329,7 @@ BOOST_AUTO_TEST_CASE(InputUngignedRowVectorParamTest) CLI::Add>(arma::Row(), "row", "Test vector", 'l', false, true, false); - //fake aruguments + // Fake arguments. const char* argv[3]; argv[0] = "./test"; argv[1] = "-l"; @@ -336,12 +337,12 @@ BOOST_AUTO_TEST_CASE(InputUngignedRowVectorParamTest) int argc = 3; - // The const-cast is a little hacky but should be fine... + // The const-cast is a little hacky but should be fine... Log::Fatal.ignoreInput = true; CLI::ParseCommandLine(argc, const_cast(argv)); Log::Fatal.ignoreInput = false; - // The --vector parameter should exist. + // The --vector parameter should exist. BOOST_REQUIRE(CLI::HasParam("row")); // The --vector_file parameter should not exist (it should be transparent from // inside the program). @@ -956,7 +957,7 @@ BOOST_AUTO_TEST_CASE(MatrixAndDatasetInfoTest) f << endl; f << "@attribute three STRING" << endl; f << endl; - f << "\% a comment line " << endl; + f << "\\% a comment line " << endl; f << endl; f << "@data" << endl; f << "hello, 1, moo" << endl; @@ -1053,7 +1054,7 @@ BOOST_AUTO_TEST_CASE(RawDatasetInfoLoadParameter) f << endl; f << "@attribute three STRING" << endl; f << endl; - f << "\% a comment line " << endl; + f << "\\% a comment line " << endl; f << endl; f << "@data" << endl; f << "hello, 1, moo" << endl; diff --git a/src/mlpack/tests/cosine_tree_test.cpp b/src/mlpack/tests/cosine_tree_test.cpp index f09ac273cb..56fe3ba7f1 100644 --- a/src/mlpack/tests/cosine_tree_test.cpp +++ b/src/mlpack/tests/cosine_tree_test.cpp @@ -181,7 +181,7 @@ BOOST_AUTO_TEST_CASE(CosineTreeModifiedGramSchmidt) CosineNodeQueue basisQueue; CosineTree dummyTree(data, epsilon, delta); - for(size_t i = 0; i < numCols; i++) + for (size_t i = 0; i < numCols; i++) { // Make a new CosineNode object. CosineTree* basisNode; @@ -198,7 +198,7 @@ BOOST_AUTO_TEST_CASE(CosineTreeModifiedGramSchmidt) CosineNodeQueue::const_iterator j = basisQueue.begin(); CosineTree* currentNode; - for(; j != basisQueue.end(); j++) + for (; j != basisQueue.end(); j++) { currentNode = *j; BOOST_REQUIRE_SMALL(arma::dot(currentNode->BasisVector(), newBasisVector), @@ -212,7 +212,7 @@ BOOST_AUTO_TEST_CASE(CosineTreeModifiedGramSchmidt) } // Deallocate memory given to the objects. - for(size_t i = 0; i < numCols; i++) + for (size_t i = 0; i < numCols; i++) { CosineTree* currentNode; currentNode = basisQueue.top(); diff --git a/src/mlpack/tests/decision_stump_test.cpp b/src/mlpack/tests/decision_stump_test.cpp index af1e5f7cd6..d888e93f7e 100644 --- a/src/mlpack/tests/decision_stump_test.cpp +++ b/src/mlpack/tests/decision_stump_test.cpp @@ -49,9 +49,8 @@ BOOST_AUTO_TEST_CASE(OneClass) Row predictedLabels; ds.Classify(testingData, predictedLabels); - for (size_t i = 0; i < predictedLabels.size(); i++ ) + for (size_t i = 0; i < predictedLabels.size(); i++) BOOST_CHECK_EQUAL(predictedLabels(i), 1); - } /** diff --git a/src/mlpack/tests/decision_tree_test.cpp b/src/mlpack/tests/decision_tree_test.cpp index 1704c19271..d9b232d05e 100644 --- a/src/mlpack/tests/decision_tree_test.cpp +++ b/src/mlpack/tests/decision_tree_test.cpp @@ -153,11 +153,13 @@ BOOST_AUTO_TEST_CASE(GiniGainEvenSplitTest) // Test that it's -0.5 regardless of the number of classes. for (size_t c = 2; c < 10; ++c) { - BOOST_REQUIRE_CLOSE(GiniGain::Evaluate(labels, c, weights), -0.5, 1e-5); + BOOST_REQUIRE_CLOSE( + GiniGain::Evaluate(labels, c, weights), -0.5, 1e-5); double weightedGain = GiniGain::Evaluate(labels, c, weights); // The weighted gain should stay the same with unweight one - BOOST_REQUIRE_EQUAL(GiniGain::Evaluate(labels, c, weights), weightedGain); + BOOST_REQUIRE_EQUAL( + GiniGain::Evaluate(labels, c, weights), weightedGain); } } @@ -171,7 +173,7 @@ BOOST_AUTO_TEST_CASE(GiniGainEmptyTest) arma::Row labels; for (size_t c = 1; c < 10; ++c) BOOST_REQUIRE_SMALL(GiniGain::Evaluate(labels, c, weights), 1e-5); - + for (size_t c = 1; c < 10; ++c) BOOST_REQUIRE_SMALL(GiniGain::Evaluate(labels, c, weights), 1e-5); } @@ -223,26 +225,28 @@ BOOST_AUTO_TEST_CASE(GiniGainManyPoints) } } -/** + +/** * To make sure the Gini gain can been cacluate proporately with weight. */ - BOOST_AUTO_TEST_CASE(GiniGainWithWeight) - { - arma::Row labels(10); - arma::rowvec weights(10); - for (size_t i = 0; i < 5; ++i) - { - labels[i] = 0; - weights[i] = 0.3; - } - for (size_t i = 5; i < 10; ++i) - { - labels[i] = 1; - weights[i] = 0.7; - } +BOOST_AUTO_TEST_CASE(GiniGainWithWeight) +{ + arma::Row labels(10); + arma::rowvec weights(10); + for (size_t i = 0; i < 5; ++i) + { + labels[i] = 0; + weights[i] = 0.3; + } + for (size_t i = 5; i < 10; ++i) + { + labels[i] = 1; + weights[i] = 0.7; + } - BOOST_REQUIRE_CLOSE(GiniGain::Evaluate(labels, 2, weights), -0.42, 1e-5); - } + BOOST_REQUIRE_CLOSE( + GiniGain::Evaluate(labels, 2, weights), -0.42, 1e-5); +} /** * The information gain should be zero when the labels are perfect. @@ -255,7 +259,10 @@ BOOST_AUTO_TEST_CASE(InformationGainPerfectTest) // Test that it's perfect regardless of number of classes. for (size_t c = 1; c < 10; ++c) - BOOST_REQUIRE_SMALL(InformationGain::Evaluate(labels, c, weights), 1e-5); + { + BOOST_REQUIRE_SMALL( + InformationGain::Evaluate(labels, c, weights), 1e-5); + } } /** @@ -329,9 +336,10 @@ BOOST_AUTO_TEST_CASE(InformationWithWeight) for (size_t i = 5; i < 10; ++i) labels[i] = 1; - // Zero is not a good result as gain, but we just need to prove cacluation works. - BOOST_REQUIRE_CLOSE(InformationGain::Evaluate(labels, 2, weights), 0, 1e-5); - + // Zero is not a good result as gain, but we just need to prove + // cacluation works. + BOOST_REQUIRE_CLOSE( + InformationGain::Evaluate(labels, 2, weights), 0, 1e-5); } @@ -414,7 +422,7 @@ BOOST_AUTO_TEST_CASE(BestBinaryNumericSplitMinSamplesTest) const double bestGain = GiniGain::Evaluate(labels, 2, weights); const double gain = BestBinaryNumericSplit::SplitIfBetter( bestGain, values, labels, 2, weights, 8, classProbabilities, aux); - // This should make no difference because it won't split at all. + // This should make no difference because it won't split at all. const double weightedGain = BestBinaryNumericSplit::SplitIfBetter(bestGain, values, labels, 2, weights, 8, classProbabilities, aux); @@ -758,7 +766,7 @@ BOOST_AUTO_TEST_CASE(CategoricalBuildTest) arma::Row l; data::DatasetInfo di; MockCategoricalData(d, l, di); - + // Split into a training set and a test set. arma::mat trainingData = d.cols(0, 1999); arma::mat testData = d.cols(2000, 3999); diff --git a/src/mlpack/tests/det_test.cpp b/src/mlpack/tests/det_test.cpp index bff2baf583..7c44f2a8dc 100644 --- a/src/mlpack/tests/det_test.cpp +++ b/src/mlpack/tests/det_test.cpp @@ -94,7 +94,7 @@ BOOST_AUTO_TEST_CASE(TestWithinRange) BOOST_AUTO_TEST_CASE(TestFindSplit) { - arma::mat testData(3,5); + arma::mat testData(3, 5); testData << 4 << 5 << 7 << 3 << 5 << arma::endr << 5 << 0 << 1 << 7 << 1 << arma::endr @@ -102,16 +102,17 @@ BOOST_AUTO_TEST_CASE(TestFindSplit) DTree testDTree(testData); - size_t obDim, trueDim; - double trueLeftError, obLeftError, trueRightError, obRightError, obSplit, trueSplit; + size_t obDim; + double obLeftError, obRightError, obSplit; - trueDim = 2; - trueSplit = 5.5; - trueLeftError = 2 * log(2.0 / 5.0) - (log(7.0) + log(4.0) + log(4.5)); - trueRightError = 2 * log(3.0 / 5.0) - (log(7.0) + log(4.0) + log(2.5)); + size_t trueDim = 2; + double trueSplit = 5.5; + double trueLeftError = 2 * log(2.0 / 5.0) - (log(7.0) + log(4.0) + log(4.5)); + double trueRightError = 2 * log(3.0 / 5.0) - (log(7.0) + log(4.0) + log(2.5)); testDTree.logVolume = log(7.0) + log(4.0) + log(7.0); - BOOST_REQUIRE(testDTree.FindSplit(testData, obDim, obSplit, obLeftError, obRightError, 1)); + BOOST_REQUIRE(testDTree.FindSplit( + testData, obDim, obSplit, obLeftError, obRightError, 1)); BOOST_REQUIRE(trueDim == obDim); BOOST_REQUIRE_CLOSE(trueSplit, obSplit, 1e-10); @@ -136,7 +137,8 @@ BOOST_AUTO_TEST_CASE(TestSplitData) size_t splitDim = 2; double trueSplitVal = 5.5; - size_t splitInd = testDTree.SplitData(testData, splitDim, trueSplitVal, oTest); + size_t splitInd = testDTree.SplitData( + testData, splitDim, trueSplitVal, oTest); BOOST_REQUIRE_EQUAL(splitInd, 2); // 2 points on left side. @@ -149,7 +151,7 @@ BOOST_AUTO_TEST_CASE(TestSplitData) BOOST_AUTO_TEST_CASE(TestSparseFindSplit) { - arma::mat realData(4,7); + arma::mat realData(4, 7); realData << .0 << 4 << 5 << 7 << 0 << 5 << 0 << arma::endr << .0 << 5 << 0 << 0 << 1 << 7 << 1 << arma::endr @@ -160,16 +162,19 @@ BOOST_AUTO_TEST_CASE(TestSparseFindSplit) DTree testDTree(testData); - size_t obDim, trueDim; - double trueLeftError, obLeftError, trueRightError, obRightError, obSplit, trueSplit; + size_t obDim; + double obLeftError, obRightError, obSplit; - trueDim = 1; - trueSplit = .5; - trueLeftError = 2 * log(3.0 / 7.0) - (log(7.0) + log(0.5) + log(8.0) + log(6.0)); - trueRightError = 2 * log(4.0 / 7.0) - (log(7.0) + log(6.5) + log(8.0) + log(6.0)); + size_t trueDim = 1; + double trueSplit = .5; + double trueLeftError = 2 * log(3.0 / 7.0) - + (log(7.0) + log(0.5) + log(8.0) + log(6.0)); + double trueRightError = 2 * log(4.0 / 7.0) - + (log(7.0) + log(6.5) + log(8.0) + log(6.0)); testDTree.logVolume = log(7.0) + log(7.0) + log(8.0) + log(6.0); - BOOST_REQUIRE(testDTree.FindSplit(testData, obDim, obSplit, obLeftError, obRightError, 1)); + BOOST_REQUIRE(testDTree.FindSplit( + testData, obDim, obSplit, obLeftError, obRightError, 1)); BOOST_REQUIRE(trueDim == obDim); BOOST_REQUIRE_CLOSE(trueSplit, obSplit, 1e-10); @@ -180,7 +185,7 @@ BOOST_AUTO_TEST_CASE(TestSparseFindSplit) BOOST_AUTO_TEST_CASE(TestSparseSplitData) { - arma::mat realData(4,7); + arma::mat realData(4, 7); realData << .0 << 4 << 5 << 7 << 0 << 5 << 0 << arma::endr << .0 << 5 << 0 << 0 << 1 << 7 << 1 << arma::endr @@ -197,7 +202,8 @@ BOOST_AUTO_TEST_CASE(TestSparseSplitData) size_t splitDim = 1; double trueSplitVal = .5; - size_t splitInd = testDTree.SplitData(testData, splitDim, trueSplitVal, oTest); + size_t splitInd = testDTree.SplitData( + testData, splitDim, trueSplitVal, oTest); BOOST_REQUIRE_EQUAL(splitInd, 3); // 2 points on left side. diff --git a/src/mlpack/tests/distribution_test.cpp b/src/mlpack/tests/distribution_test.cpp index bbf22a6c40..24b02d721b 100644 --- a/src/mlpack/tests/distribution_test.cpp +++ b/src/mlpack/tests/distribution_test.cpp @@ -357,7 +357,6 @@ BOOST_AUTO_TEST_CASE(GaussianMultivariateProbabilityTest) g.Mean() *= -1; BOOST_REQUIRE_CLOSE(g.Probability(-x), 1.4673143531128877e-06, 1e-5); BOOST_REQUIRE_CLOSE(g.Probability(x), 7.7404143494891786e-09, 1e-8); - } /** diff --git a/src/mlpack/tests/emst_test.cpp b/src/mlpack/tests/emst_test.cpp index 435070c9db..d35cfd757b 100644 --- a/src/mlpack/tests/emst_test.cpp +++ b/src/mlpack/tests/emst_test.cpp @@ -247,7 +247,6 @@ BOOST_AUTO_TEST_CASE(CoverTreeTest) BOOST_REQUIRE_EQUAL(bstResults(1, i), coverResults(1, i)); BOOST_REQUIRE_CLOSE(bstResults(2, i), coverResults(2, i), 1e-5); } - } /** @@ -277,7 +276,6 @@ BOOST_AUTO_TEST_CASE(BallTreeTest) BOOST_REQUIRE_EQUAL(bstResults(1, i), ballResults(1, i)); BOOST_REQUIRE_CLOSE(bstResults(2, i), ballResults(2, i), 1e-5); } - } BOOST_AUTO_TEST_SUITE_END(); diff --git a/src/mlpack/tests/fastmks_test.cpp b/src/mlpack/tests/fastmks_test.cpp index 5b1d495930..ed1032b6e5 100644 --- a/src/mlpack/tests/fastmks_test.cpp +++ b/src/mlpack/tests/fastmks_test.cpp @@ -174,12 +174,19 @@ BOOST_AUTO_TEST_CASE(SparsePolynomialFastMKSTest) for (size_t i = 0; i < 100; ++i) for (size_t j = 0; j < 100; ++j) + { if (std::abs(pk.Evaluate(dataset.col(i), dataset.col(j))) < 1e-10) - BOOST_REQUIRE_SMALL(pk.Evaluate(denseset.col(i), denseset.col(j)), 1e-10); + { + BOOST_REQUIRE_SMALL( + pk.Evaluate(denseset.col(i), denseset.col(j)), 1e-10); + } else - BOOST_REQUIRE_CLOSE(pk.Evaluate(dataset.col(i), dataset.col(j)), - pk.Evaluate(denseset.col(i), denseset.col(j)), - 1e-5); + { + BOOST_REQUIRE_CLOSE( + pk.Evaluate(dataset.col(i), dataset.col(j)), + pk.Evaluate(denseset.col(i), denseset.col(j)), 1e-5); + } + } FastMKS sparsepoly(dataset); FastMKS densepoly(denseset); diff --git a/src/mlpack/tests/feedforward_network_test.cpp b/src/mlpack/tests/feedforward_network_test.cpp index d8f82ea023..affa3cc4fa 100644 --- a/src/mlpack/tests/feedforward_network_test.cpp +++ b/src/mlpack/tests/feedforward_network_test.cpp @@ -426,4 +426,4 @@ BOOST_AUTO_TEST_CASE(FFNMiscTest) movedModel = std::move(copiedModel); } -BOOST_AUTO_TEST_SUITE_END(); \ No newline at end of file +BOOST_AUTO_TEST_SUITE_END(); diff --git a/src/mlpack/tests/gmm_test.cpp b/src/mlpack/tests/gmm_test.cpp index 28a1dfcb7b..d3c8cd5c29 100644 --- a/src/mlpack/tests/gmm_test.cpp +++ b/src/mlpack/tests/gmm_test.cpp @@ -494,7 +494,7 @@ BOOST_AUTO_TEST_CASE(GMMLoadSaveTest) } // Remove clutter. - //remove("test-gmm-save.xml"); + // remove("test-gmm-save.xml"); BOOST_REQUIRE_EQUAL(gmm.Gaussians(), gmm2.Gaussians()); BOOST_REQUIRE_EQUAL(gmm.Dimensionality(), gmm2.Dimensionality()); @@ -552,11 +552,10 @@ BOOST_AUTO_TEST_CASE(PositiveDefiniteConstraintTest) arma::mat c; #if (ARMA_VERSION_MAJOR < 4) || \ ((ARMA_VERSION_MAJOR == 4) && (ARMA_VERSION_MINOR < 500)) - BOOST_REQUIRE(arma::chol(c, cov)); + BOOST_REQUIRE(arma::chol(c, cov)); #else - BOOST_REQUIRE(arma::chol(c, cov, "lower")); + BOOST_REQUIRE(arma::chol(c, cov, "lower")); #endif - } } diff --git a/src/mlpack/tests/hmm_test.cpp b/src/mlpack/tests/hmm_test.cpp index 4117571508..4d8656bd02 100644 --- a/src/mlpack/tests/hmm_test.cpp +++ b/src/mlpack/tests/hmm_test.cpp @@ -78,9 +78,12 @@ BOOST_AUTO_TEST_CASE(BorodovskyHMMTestViterbi) "0.5 0.5 0.6"); // Four emission states: A, C, G, T. Start state doesn't emit... std::vector emission(3); - emission[0] = DiscreteDistribution(std::vector{"0.25 0.25 0.25 0.25"}); - emission[1] = DiscreteDistribution(std::vector{"0.20 0.30 0.30 0.20"}); - emission[2] = DiscreteDistribution(std::vector{"0.30 0.20 0.20 0.30"}); + emission[0] = DiscreteDistribution( + std::vector{"0.25 0.25 0.25 0.25"}); + emission[1] = DiscreteDistribution( + std::vector{"0.20 0.30 0.30 0.20"}); + emission[2] = DiscreteDistribution( + std::vector{"0.30 0.20 0.20 0.30"}); HMM hmm(initial, transition, emission); @@ -1002,7 +1005,7 @@ BOOST_AUTO_TEST_CASE(GMMHMMLoadSaveTest) // Create a GMM HMM, save it, and load it. HMM hmm(3, GMM(4, 3)); - for(size_t j = 0; j < hmm.Emission().size(); ++j) + for (size_t j = 0; j < hmm.Emission().size(); ++j) { hmm.Emission()[j].Weights().randu(); for (size_t i = 0; i < hmm.Emission()[j].Gaussians(); ++i) @@ -1055,7 +1058,7 @@ BOOST_AUTO_TEST_CASE(GMMHMMLoadSaveTest) for (size_t k = 0; k < hmm.Emission()[j].Dimensionality(); ++k) { - BOOST_REQUIRE_CLOSE(hmm.Emission()[j].Component(i).Covariance()(l,k), + BOOST_REQUIRE_CLOSE(hmm.Emission()[j].Component(i).Covariance()(l, k), hmm2.Emission()[j].Component(i).Covariance()(l, k), 1e-3); } } @@ -1072,7 +1075,7 @@ BOOST_AUTO_TEST_CASE(GaussianHMMLoadSaveTest) HMM hmm(3, GaussianDistribution(2)); - for(size_t j = 0; j < hmm.Emission().size(); ++j) + for (size_t j = 0; j < hmm.Emission().size(); ++j) { hmm.Emission()[j].Mean().randu(); arma::mat covariance = arma::randu( @@ -1112,7 +1115,7 @@ BOOST_AUTO_TEST_CASE(GaussianHMMLoadSaveTest) hmm2.Emission()[j].Mean()[i], 1e-3); for (size_t k = 0; k < hmm.Emission()[j].Dimensionality(); ++k) { - BOOST_REQUIRE_CLOSE(hmm.Emission()[j].Covariance()(i,k), + BOOST_REQUIRE_CLOSE(hmm.Emission()[j].Covariance()(i, k), hmm2.Emission()[j].Covariance()(i, k), 1e-3); } } @@ -1140,7 +1143,7 @@ BOOST_AUTO_TEST_CASE(DiscreteHMMLoadSaveTest) HMM hmm(3, DiscreteDistribution(3)); - for(size_t j = 0; j < hmm.Emission().size(); ++j) + for (size_t j = 0; j < hmm.Emission().size(); ++j) { hmm.Emission()[j].Probabilities() = arma::randu(3); hmm.Emission()[j].Probabilities() /= accu(emission[j].Probabilities()); diff --git a/src/mlpack/tests/ind2sub_test.cpp b/src/mlpack/tests/ind2sub_test.cpp index 98e425c01d..eb522e9d57 100644 --- a/src/mlpack/tests/ind2sub_test.cpp +++ b/src/mlpack/tests/ind2sub_test.cpp @@ -21,7 +21,7 @@ BOOST_AUTO_TEST_SUITE(ind2subTest); */ BOOST_AUTO_TEST_CASE(ind2sub_test) { - arma::mat A = arma::randu(4,5); + arma::mat A = arma::randu(4, 5); size_t index = 13; arma::uvec u = arma::ind2sub(arma::size(A), index); diff --git a/src/mlpack/tests/kernel_test.cpp b/src/mlpack/tests/kernel_test.cpp index 9e6aa41951..cfe9361624 100644 --- a/src/mlpack/tests/kernel_test.cpp +++ b/src/mlpack/tests/kernel_test.cpp @@ -16,7 +16,6 @@ #include #include #include -#include #include #include #include @@ -254,8 +253,8 @@ BOOST_AUTO_TEST_CASE(LinearKernelTest) arma::vec b = ".56 .21 .623 .82"; LinearKernel lk; - BOOST_REQUIRE_CLOSE(lk.Evaluate(a,b), .5062, 1e-5); - BOOST_REQUIRE_CLOSE(lk.Evaluate(b,a), .5062, 1e-5); + BOOST_REQUIRE_CLOSE(lk.Evaluate(a, b), .5062, 1e-5); + BOOST_REQUIRE_CLOSE(lk.Evaluate(b, a), .5062, 1e-5); } /** @@ -267,8 +266,8 @@ BOOST_AUTO_TEST_CASE(LinearKernelOrthogonalTest) arma::vec b = "0 0 1"; LinearKernel lk; - BOOST_REQUIRE_SMALL(lk.Evaluate(a,b), 1e-5); - BOOST_REQUIRE_SMALL(lk.Evaluate(b,a), 1e-5); + BOOST_REQUIRE_SMALL(lk.Evaluate(a, b), 1e-5); + BOOST_REQUIRE_SMALL(lk.Evaluate(b, a), 1e-5); } BOOST_AUTO_TEST_CASE(GaussianKernelTest) @@ -294,9 +293,9 @@ BOOST_AUTO_TEST_CASE(GaussianKernelTest) BOOST_REQUIRE_CLOSE(gk.Normalizer(3), 1.9687012432153019, 1e-5); BOOST_REQUIRE_CLOSE(gk.Normalizer(4), 2.4674011002723386, 1e-5); /* check the convolution integral */ - BOOST_REQUIRE_CLOSE(gk.ConvolutionIntegral(a,b), 0.024304474038457577, 1e-5); - BOOST_REQUIRE_CLOSE(gk.ConvolutionIntegral(a,c), 0.024304474038457577, 1e-5); - BOOST_REQUIRE_CLOSE(gk.ConvolutionIntegral(b,c), 0.024304474038457577, 1e-5); + BOOST_REQUIRE_CLOSE(gk.ConvolutionIntegral(a, b), 0.024304474038457577, 1e-5); + BOOST_REQUIRE_CLOSE(gk.ConvolutionIntegral(a, c), 0.024304474038457577, 1e-5); + BOOST_REQUIRE_CLOSE(gk.ConvolutionIntegral(b, c), 0.024304474038457577, 1e-5); } BOOST_AUTO_TEST_CASE(GaussianKernelSerializationTest) @@ -334,9 +333,9 @@ BOOST_AUTO_TEST_CASE(SphericalKernelTest) BOOST_REQUIRE_CLOSE(sk.Normalizer(3), 0.52359877559829893, 1e-5); BOOST_REQUIRE_CLOSE(sk.Normalizer(4), 0.30842513753404244, 1e-5); /* check the convolution integral */ - BOOST_REQUIRE_CLOSE(sk.ConvolutionIntegral(a,b), 0.0, 1e-5); - BOOST_REQUIRE_CLOSE(sk.ConvolutionIntegral(a,c), 0.0, 1e-5); - BOOST_REQUIRE_CLOSE(sk.ConvolutionIntegral(b,c), 1.0021155029652784, 1e-5); + BOOST_REQUIRE_CLOSE(sk.ConvolutionIntegral(a, b), 0.0, 1e-5); + BOOST_REQUIRE_CLOSE(sk.ConvolutionIntegral(a, c), 0.0, 1e-5); + BOOST_REQUIRE_CLOSE(sk.ConvolutionIntegral(b, c), 1.0021155029652784, 1e-5); } BOOST_AUTO_TEST_CASE(EpanechnikovKernelTest) @@ -360,9 +359,9 @@ BOOST_AUTO_TEST_CASE(EpanechnikovKernelTest) BOOST_REQUIRE_CLOSE(ek.Normalizer(3), 0.20943951023931956, 1e-5); BOOST_REQUIRE_CLOSE(ek.Normalizer(4), 0.10280837917801415, 1e-5); /* check the convolution integral */ - BOOST_REQUIRE_CLOSE(ek.ConvolutionIntegral(a,b), 0.0, 1e-5); - BOOST_REQUIRE_CLOSE(ek.ConvolutionIntegral(a,c), 0.0, 1e-5); - BOOST_REQUIRE_CLOSE(ek.ConvolutionIntegral(b,c), 1.5263455690698258, 1e-5); + BOOST_REQUIRE_CLOSE(ek.ConvolutionIntegral(a, b), 0.0, 1e-5); + BOOST_REQUIRE_CLOSE(ek.ConvolutionIntegral(a, c), 0.0, 1e-5); + BOOST_REQUIRE_CLOSE(ek.ConvolutionIntegral(b, c), 1.5263455690698258, 1e-5); } BOOST_AUTO_TEST_CASE(PolynomialKernelTest) diff --git a/src/mlpack/tests/knn_test.cpp b/src/mlpack/tests/knn_test.cpp index 6127ffc184..47d2cfe35a 100644 --- a/src/mlpack/tests/knn_test.cpp +++ b/src/mlpack/tests/knn_test.cpp @@ -384,7 +384,6 @@ BOOST_AUTO_TEST_CASE(ExhaustiveSyntheticTest) for (int i = 0; i < 3; i++) { - switch (i) { case 0: // Use the dual-tree method. @@ -649,7 +648,6 @@ BOOST_AUTO_TEST_CASE(ExhaustiveSyntheticTest) BOOST_REQUIRE_CLOSE(distances(8, newFromOld[10]), 3.00, 1e-5); BOOST_REQUIRE_EQUAL(neighbors(9, newFromOld[10]), newFromOld[4]); BOOST_REQUIRE_CLOSE(distances(9, newFromOld[10]), 4.05, 1e-5); - } } diff --git a/src/mlpack/tests/ksinit_test.cpp b/src/mlpack/tests/ksinit_test.cpp index 16ec7457e4..467bee075a 100644 --- a/src/mlpack/tests/ksinit_test.cpp +++ b/src/mlpack/tests/ksinit_test.cpp @@ -242,7 +242,7 @@ BOOST_AUTO_TEST_CASE(IrisDataset) // Normalization used in the paper. dataset /= 10; - //Counter for the number of failures. + // Counter for the number of failures. size_t numFails = 0; // It isn't guaranteed that the network will converge in the specified number @@ -330,4 +330,4 @@ BOOST_AUTO_TEST_CASE(NonLinearFunctionApproximation) BOOST_REQUIRE_LE(numFails, 4); } -BOOST_AUTO_TEST_SUITE_END(); \ No newline at end of file +BOOST_AUTO_TEST_SUITE_END(); diff --git a/src/mlpack/tests/lars_test.cpp b/src/mlpack/tests/lars_test.cpp index b0d2a19ae2..bf78dde33e 100644 --- a/src/mlpack/tests/lars_test.cpp +++ b/src/mlpack/tests/lars_test.cpp @@ -23,7 +23,8 @@ using namespace mlpack::regression; BOOST_AUTO_TEST_SUITE(LARSTest); -void GenerateProblem(arma::mat& X, arma::rowvec& y, size_t nPoints, size_t nDims) +void GenerateProblem( + arma::mat& X, arma::rowvec& y, size_t nPoints, size_t nDims) { X = arma::randn(nDims, nPoints); arma::vec beta = arma::randn(nDims, 1); @@ -34,7 +35,7 @@ void LARSVerifyCorrectness(arma::vec beta, arma::vec errCorr, double lambda) { size_t nDims = beta.n_elem; const double tol = 1e-10; - for(size_t j = 0; j < nDims; j++) + for (size_t j = 0; j < nDims; j++) { if (beta(j) == 0) { diff --git a/src/mlpack/tests/load_save_test.cpp b/src/mlpack/tests/load_save_test.cpp index afd273335e..e2f4e7698d 100644 --- a/src/mlpack/tests/load_save_test.cpp +++ b/src/mlpack/tests/load_save_test.cpp @@ -1682,7 +1682,7 @@ BOOST_AUTO_TEST_CASE(SimpleARFFCategoricalTest) f << endl; f << "@attribute three STRING" << endl; f << endl; - f << "\% a comment line " << endl; + f << "\\% a comment line " << endl; f << endl; f << "@data" << endl; f << "hello, 1, moo" << endl; @@ -1738,15 +1738,15 @@ BOOST_AUTO_TEST_CASE(HarderARFFTest) f << endl; f << "@attribute @@@@flfl numeric" << endl; f << endl; - f << "\% comment" << endl; + f << "\\% comment" << endl; f << "@attribute \"hello world\" string" << endl; f << "@attribute 12345 integer" << endl; f << "@attribute real real" << endl; - f << "@attribute \"blah blah blah \t \" numeric \% comment" << endl; - f << "\% comment" << endl; + f << "@attribute \"blah blah blah \t \" numeric \\% comment" << endl; + f << "\\% comment" << endl; f << "@data" << endl; f << "1, one, 3, 4.5, 6" << endl; - f << "2, two, 4, 5.5, 7 \% comment" << endl; + f << "2, two, 4, 5.5, 7 \\% comment" << endl; f << "3, \"three five, six\", 5, 6.5, 8" << endl; f.close(); @@ -1803,15 +1803,15 @@ BOOST_AUTO_TEST_CASE(BadDatasetInfoARFFTest) f << endl; f << "@attribute @@@@flfl numeric" << endl; f << endl; - f << "\% comment" << endl; + f << "\\% comment" << endl; f << "@attribute \"hello world\" string" << endl; f << "@attribute 12345 integer" << endl; f << "@attribute real real" << endl; - f << "@attribute \"blah blah blah \t \" numeric \% comment" << endl; - f << "\% comment" << endl; + f << "@attribute \"blah blah blah \t \" numeric \\% comment" << endl; + f << "\\% comment" << endl; f << "@data" << endl; f << "1, one, 3, 4.5, 6" << endl; - f << "2, two, 4, 5.5, 7 \% comment" << endl; + f << "2, two, 4, 5.5, 7 \\% comment" << endl; f << "3, \"three five, six\", 5, 6.5, 8" << endl; f.close(); @@ -1992,5 +1992,4 @@ BOOST_AUTO_TEST_CASE(LoadCSVNoTransposeTXTTest) remove("test.txt"); } - BOOST_AUTO_TEST_SUITE_END(); diff --git a/src/mlpack/tests/local_coordinate_coding_test.cpp b/src/mlpack/tests/local_coordinate_coding_test.cpp index 26836d6d43..80477ef9ac 100644 --- a/src/mlpack/tests/local_coordinate_coding_test.cpp +++ b/src/mlpack/tests/local_coordinate_coding_test.cpp @@ -29,7 +29,7 @@ void VerifyCorrectness(vec beta, vec errCorr, double lambda) { const double tol = 1e-12; size_t nDims = beta.n_elem; - for(size_t j = 0; j < nDims; j++) + for (size_t j = 0; j < nDims; j++) { if (beta(j) == 0) { diff --git a/src/mlpack/tests/lsh_test.cpp b/src/mlpack/tests/lsh_test.cpp index 4f96fede9e..2a85baede2 100644 --- a/src/mlpack/tests/lsh_test.cpp +++ b/src/mlpack/tests/lsh_test.cpp @@ -335,7 +335,8 @@ BOOST_AUTO_TEST_CASE(RecallTest) arma::mat lshDistancesExp; lshTestExp.Search(qdata, k, lshNeighborsExp, lshDistancesExp); - const double recallExp = LSHSearch<>::ComputeRecall(lshNeighborsExp, groundTruth); + const double recallExp = LSHSearch<>::ComputeRecall( + lshNeighborsExp, groundTruth); // This run should have recall higher than the threshold. BOOST_REQUIRE_GE(recallExp, recallThreshExp); @@ -408,7 +409,7 @@ BOOST_AUTO_TEST_CASE(DeterministicMerge) // Query 1 is in cluster 3, which under this projection was merged with // cluster 4. Clusters 3 and 4 have points 20:39, so only neighbors among - //those should be found. + // those should be found. q = 0; BOOST_REQUIRE_GE(neighbors(j, q), N / 2); @@ -785,17 +786,19 @@ BOOST_AUTO_TEST_CASE(ParallelBichromatic) arma::mat distances; // Construct an LSH object. By default, it uses the maximum number of threads - LSHSearch<> lshTest(rdata, numProj, numTables); //default parameters + LSHSearch<> lshTest(rdata, numProj, numTables); // Default parameters. lshTest.Search(qdata, k, parallelNeighbors, distances); // Now perform same search but with 1 thread - size_t prevNumThreads = omp_get_max_threads(); // Store number of threads used. + // Store number of threads used. + size_t prevNumThreads = omp_get_max_threads(); omp_set_num_threads(1); lshTest.Search(qdata, k, sequentialNeighbors, distances); omp_set_num_threads(prevNumThreads); // Require both have same results - double recall = LSHSearch<>::ComputeRecall(sequentialNeighbors, parallelNeighbors); + double recall = LSHSearch<>::ComputeRecall( + sequentialNeighbors, parallelNeighbors); BOOST_REQUIRE_EQUAL(recall, 1); } @@ -825,13 +828,15 @@ BOOST_AUTO_TEST_CASE(ParallelMonochromatic) lshTest.Search(k, parallelNeighbors, distances); // Now perform same search but with 1 thread. - size_t prevNumThreads = omp_get_max_threads(); // Store number of threads used. + // Store number of threads used. + size_t prevNumThreads = omp_get_max_threads(); omp_set_num_threads(1); lshTest.Search(k, sequentialNeighbors, distances); omp_set_num_threads(prevNumThreads); // Require both have same results. - double recall = LSHSearch<>::ComputeRecall(sequentialNeighbors, parallelNeighbors); + double recall = LSHSearch<>::ComputeRecall( + sequentialNeighbors, parallelNeighbors); BOOST_REQUIRE_EQUAL(recall, 1); } #endif diff --git a/src/mlpack/tests/maximal_inputs_test.cpp b/src/mlpack/tests/maximal_inputs_test.cpp index c72f340082..cf48e0deaf 100644 --- a/src/mlpack/tests/maximal_inputs_test.cpp +++ b/src/mlpack/tests/maximal_inputs_test.cpp @@ -38,7 +38,7 @@ void TestResults(const arma::mat&actualResult, const arma::mat& expectResult) BOOST_REQUIRE_EQUAL(expectResult.n_rows, actualResult.n_rows); BOOST_REQUIRE_EQUAL(expectResult.n_cols, actualResult.n_cols); - for(size_t i = 0; i != expectResult.n_elem; ++i) + for (size_t i = 0; i != expectResult.n_elem; ++i) { BOOST_REQUIRE_CLOSE(expectResult[i], actualResult[i], 1e-2); } diff --git a/src/mlpack/tests/mean_shift_test.cpp b/src/mlpack/tests/mean_shift_test.cpp index 6d55b36c2d..038c424009 100644 --- a/src/mlpack/tests/mean_shift_test.cpp +++ b/src/mlpack/tests/mean_shift_test.cpp @@ -57,8 +57,8 @@ arma::mat meanShiftData(" 0.0 0.0;" // Class 1. /** * 30-point 3-class test case for Mean Shift. */ -BOOST_AUTO_TEST_CASE(MeanShiftSimpleTest) { - +BOOST_AUTO_TEST_CASE(MeanShiftSimpleTest) +{ MeanShift<> meanShift; arma::Col assignments; @@ -88,7 +88,6 @@ BOOST_AUTO_TEST_CASE(MeanShiftSimpleTest) { for (size_t i = 20; i < 30; i++) BOOST_REQUIRE_EQUAL(assignments(i), thirdClass); - } // Generate samples from four Gaussians, and make sure mean shift nearly diff --git a/src/mlpack/tests/momentum_sgd_test.cpp b/src/mlpack/tests/momentum_sgd_test.cpp index 6a574b4349..2b8184d8bd 100644 --- a/src/mlpack/tests/momentum_sgd_test.cpp +++ b/src/mlpack/tests/momentum_sgd_test.cpp @@ -30,7 +30,8 @@ BOOST_AUTO_TEST_CASE(MomentumSGDSpeedUpTestFunction) { SGDTestFunction f; MomentumUpdate momentumUpdate(0.7); - MomentumSGD s(f, 0.0003, 2500000, 1e-9, true, momentumUpdate); + MomentumSGD s( + f, 0.0003, 2500000, 1e-9, true, momentumUpdate); arma::mat coordinates = f.GetInitialPoint(); double result = s.Optimize(coordinates); @@ -53,7 +54,6 @@ BOOST_AUTO_TEST_CASE(MomentumSGDSpeedUpTestFunction) BOOST_REQUIRE_SMALL(coordinates1[1], 1e-7); BOOST_REQUIRE_SMALL(coordinates1[2], 1e-7); - BOOST_REQUIRE_LE(result,result1); } @@ -65,7 +65,8 @@ BOOST_AUTO_TEST_CASE(GeneralizedRosenbrockTest) // Create the generalized Rosenbrock function. GeneralizedRosenbrockFunction f(i); MomentumUpdate momentumUpdate(0.4); - MomentumSGD s(f, 0.001, 0, 1e-15, true, momentumUpdate); + MomentumSGD s( + f, 0.001, 0, 1e-15, true, momentumUpdate); arma::mat coordinates = f.GetInitialPoint(); double result = s.Optimize(coordinates); diff --git a/src/mlpack/tests/nbc_test.cpp b/src/mlpack/tests/nbc_test.cpp index e7ad6e1d81..a42f8066d7 100644 --- a/src/mlpack/tests/nbc_test.cpp +++ b/src/mlpack/tests/nbc_test.cpp @@ -145,7 +145,10 @@ BOOST_AUTO_TEST_CASE(NaiveBayesClassifierIncrementalTest) for (size_t i = 0; i < testResProba.n_cols; ++i) for (size_t j = 0; j < testResProba.n_rows; ++j) - BOOST_REQUIRE_CLOSE(testResProba(j, i) + .00001, calcProbs(j, i) + .00001, 0.01); + { + BOOST_REQUIRE_CLOSE( + testResProba(j, i) + .00001, calcProbs(j, i) + .00001, 0.01); + } } /** diff --git a/src/mlpack/tests/nca_test.cpp b/src/mlpack/tests/nca_test.cpp index 19db5109b9..d7ff3547fc 100644 --- a/src/mlpack/tests/nca_test.cpp +++ b/src/mlpack/tests/nca_test.cpp @@ -320,7 +320,6 @@ BOOST_AUTO_TEST_CASE(NCALBFGSSimpleDataset) // The solution is not unique, so the best we can do is ensure the gradient // norm is close to 0. BOOST_REQUIRE_LT(arma::norm(finalGradient, 2), 1e-6); - } BOOST_AUTO_TEST_SUITE_END(); diff --git a/src/mlpack/tests/nmf_test.cpp b/src/mlpack/tests/nmf_test.cpp index 69d8a3cdc9..3ab0ef9c32 100644 --- a/src/mlpack/tests/nmf_test.cpp +++ b/src/mlpack/tests/nmf_test.cpp @@ -60,8 +60,7 @@ BOOST_AUTO_TEST_CASE(NMFAcolDistTest) const size_t r = 12; SimpleResidueTermination srt(1e-7, 10000); - AMF > - nmf(srt); + AMF > nmf(srt); nmf.Apply(v, r, w, h); mat wh = w * h; diff --git a/src/mlpack/tests/nystroem_method_test.cpp b/src/mlpack/tests/nystroem_method_test.cpp index 6f3db50426..279cad817c 100644 --- a/src/mlpack/tests/nystroem_method_test.cpp +++ b/src/mlpack/tests/nystroem_method_test.cpp @@ -96,12 +96,12 @@ BOOST_AUTO_TEST_CASE(Rank10Test) size_t successes = 0; for (size_t testTrial = 0; testTrial < 5; ++testTrial) { - // Now use the linear kernel to get a Nystroem approximation; try this several - // times. + // Now use the linear kernel to get a Nystroem approximation; + // try this several times. double normalizedFroAverage = 0.0; for (size_t trial = 0; trial < 20; ++trial) { - while(true) + while (true) { LinearKernel lk; NystroemMethod nm(dataMod, lk, 10); diff --git a/src/mlpack/tests/octree_test.cpp b/src/mlpack/tests/octree_test.cpp index 4f3bca5e74..87460b41d0 100644 --- a/src/mlpack/tests/octree_test.cpp +++ b/src/mlpack/tests/octree_test.cpp @@ -193,15 +193,17 @@ void CheckFurthestDistances(TreeType& node) for (size_t i = 0; i < node.NumPoints(); ++i) { // Handle floating-point inaccuracies. - BOOST_REQUIRE_LE(metric::EuclideanDistance::Evaluate(node.Dataset().col(node.Point(i)), - center), node.FurthestPointDistance() * (1 + 1e-5)); + BOOST_REQUIRE_LE(metric::EuclideanDistance::Evaluate( + node.Dataset().col(node.Point(i)), center), + node.FurthestPointDistance() * (1 + 1e-5)); } // Compare descendants held in the node. for (size_t i = 0; i < node.NumDescendants(); ++i) { // Handle floating-point inaccuracies. - BOOST_REQUIRE_LE(metric::EuclideanDistance::Evaluate(node.Dataset().col(node.Descendant(i)), + BOOST_REQUIRE_LE(metric::EuclideanDistance::Evaluate( + node.Dataset().col(node.Descendant(i)), center), node.FurthestDescendantDistance() * (1 + 1e-5)); } diff --git a/src/mlpack/tests/pca_test.cpp b/src/mlpack/tests/pca_test.cpp index 926fff9311..dae8038542 100644 --- a/src/mlpack/tests/pca_test.cpp +++ b/src/mlpack/tests/pca_test.cpp @@ -238,7 +238,6 @@ BOOST_AUTO_TEST_CASE(QUICPCADimensionalityReductionTest) size_t successes = 0; for (size_t trial = 0; trial < 5; ++trial) { - PCAType exactPCA; const double varRetainedExact = exactPCA.Apply(data, 1); @@ -306,8 +305,8 @@ BOOST_AUTO_TEST_CASE(PCAScalingTest) BOOST_REQUIRE_CLOSE(std::abs(eigvec(2, 1)), 1.0, 0.2); // The third component should have the same absolute value characteristics as - // the first. - BOOST_REQUIRE_CLOSE(std::abs(eigvec(0, 0)), sqrt(2) / 2, 0.2); // 20% tolerance. + // the first (plus 20% tolerance). + BOOST_REQUIRE_CLOSE(std::abs(eigvec(0, 0)), sqrt(2) / 2, 0.2); BOOST_REQUIRE_CLOSE(std::abs(eigvec(1, 0)), sqrt(2) / 2, 0.2); BOOST_REQUIRE_SMALL(eigvec(2, 0), 0.08); // Large tolerance for noise. diff --git a/src/mlpack/tests/perceptron_test.cpp b/src/mlpack/tests/perceptron_test.cpp index fff1ba6e13..ab8adabd60 100644 --- a/src/mlpack/tests/perceptron_test.cpp +++ b/src/mlpack/tests/perceptron_test.cpp @@ -181,7 +181,6 @@ BOOST_AUTO_TEST_CASE(Random3) for (size_t i = 0; i < predictedLabels.n_cols; i++) BOOST_CHECK_EQUAL(predictedLabels(0, i), 0); - } /** diff --git a/src/mlpack/tests/prefixedoutstream_test.cpp b/src/mlpack/tests/prefixedoutstream_test.cpp index eb4a93cae4..d4d4aee41b 100644 --- a/src/mlpack/tests/prefixedoutstream_test.cpp +++ b/src/mlpack/tests/prefixedoutstream_test.cpp @@ -105,7 +105,6 @@ BOOST_AUTO_TEST_CASE(TestArmadilloPrefixedOutStream) BASH_GREEN "[INFO ] " BASH_CLEAR "hello 1.0000 1.5000 2.0000\n" BASH_GREEN "[INFO ] " BASH_CLEAR " 2.5000 3.0000 3.5000\n" BASH_GREEN "[INFO ] " BASH_CLEAR " 4.0000 4.5000 5.0000\n"); - } /** diff --git a/src/mlpack/tests/rectangle_tree_test.cpp b/src/mlpack/tests/rectangle_tree_test.cpp index 8e17d5c2ff..a880595987 100644 --- a/src/mlpack/tests/rectangle_tree_test.cpp +++ b/src/mlpack/tests/rectangle_tree_test.cpp @@ -171,7 +171,7 @@ void CheckExactContainment(const TreeType& tree) { double min = DBL_MAX; double max = -1.0 * DBL_MAX; - for(size_t j = 0; j < tree.Count(); j++) + for (size_t j = 0; j < tree.Count(); j++) { if (tree.Dataset().col(tree.Point(j))[i] < min) min = tree.Dataset().col(tree.Point(j))[i]; @@ -716,8 +716,10 @@ void CheckDiscreteHilbertValueSync(const TreeType& tree) } } else + { for (size_t i = 0; i < tree.NumChildren(); i++) CheckDiscreteHilbertValueSync(tree.Child(i)); + } } BOOST_AUTO_TEST_CASE(DiscreteHilbertValueSyncTest) @@ -726,7 +728,7 @@ BOOST_AUTO_TEST_CASE(DiscreteHilbertValueSyncTest) dataset.randu(8, 1000); // 1000 points in 8 dimensions. typedef HilbertRTree,arma::mat> TreeType; + NeighborSearchStat, arma::mat> TreeType; TreeType hilbertRTree(dataset, 20, 6, 5, 2, 0); CheckDiscreteHilbertValueSync(hilbertRTree); @@ -982,7 +984,7 @@ BOOST_AUTO_TEST_CASE(RPlusTreeOverlapTest) dataset.randu(8, 1000); // 1000 points in 8 dimensions. typedef RPlusTree,arma::mat> TreeType; + NeighborSearchStat, arma::mat> TreeType; TreeType rPlusTree(dataset, 20, 6, 5, 2, 0); CheckOverlap(rPlusTree); @@ -1100,7 +1102,7 @@ BOOST_AUTO_TEST_CASE(RPlusPlusTreeBoundTest) // Check the MinimalCoverageSweep. typedef RPlusPlusTree,arma::mat> TreeType; + NeighborSearchStat, arma::mat> TreeType; TreeType rPlusPlusTree(dataset, 20, 6, 5, 2, 0); CheckRPlusPlusTreeBound(rPlusPlusTree); diff --git a/src/mlpack/tests/recurrent_network_test.cpp b/src/mlpack/tests/recurrent_network_test.cpp index 8395839471..0bbc439790 100644 --- a/src/mlpack/tests/recurrent_network_test.cpp +++ b/src/mlpack/tests/recurrent_network_test.cpp @@ -13,8 +13,6 @@ #include #include - -#include #include #include @@ -177,7 +175,7 @@ void GenerateReber(const arma::Mat& transitions, std::string& reber) do { - const int grammerIdx = rand() % 2; + const int grammerIdx = rand_r() % 2; reber += arma::as_scalar(transitions.submat(idx, grammerIdx, idx, grammerIdx)); @@ -198,7 +196,7 @@ void GenerateEmbeddedReber(const arma::Mat& transitions, std::string& reber) { GenerateReber(transitions, reber); - const char c = (rand() % 2) == 1 ? 'P' : 'T'; + const char c = (rand_r() % 2) == 1 ? 'P' : 'T'; reber = c + reber + c; reber = "B" + reber + "E"; } @@ -493,45 +491,33 @@ BOOST_AUTO_TEST_CASE(EmbeddedReberGrammarTest) * * @param input The generated input sequence. * @param input The generated output sequence. - * @param sequences Number of samples. */ -void GenerateDistractedSequence(arma::mat& input, - arma::mat& output, - const size_t sequences) +void GenerateDistractedSequence(arma::mat& input, arma::mat& output) { - input = arma::zeros(100, sequences); - output = arma::zeros(30, sequences); + input = arma::zeros(10, 10); + output = arma::zeros(3, 10); - for (size_t i = 0; i < sequences; ++i) + arma::Col index = arma::shuffle(arma::linspace >( + 0, 7, 8)); + + // Set the target in the input sequence and the corresponding targets in the + // output sequence by following the correct order. + for (size_t i = 0; i < 2; i++) { - arma::mat inputTemp = arma::zeros(10, 10); - arma::mat outputTemp = arma::zeros(3, 10); - - arma::Col index = arma::shuffle( - arma::linspace >(0, 7, 8)); - - // Set the target in the input sequence and the corresponding targets in the - // output sequence by following the correct order. - for (size_t i = 0; i < 2; i++) - { - size_t idx = rand() % 2; - inputTemp(idx, index(i)) = 1; - outputTemp(idx, index(i) > index(i == 0) ? 9 : 8) = 1; - } - - for (size_t i = 2; i < 8; i++) - inputTemp(2 + rand() % 6, index(i)) = 1; - - // Set the prompts which direct the network to give an answer. - inputTemp(8, 8) = 1; - inputTemp(9, 9) = 1; - - inputTemp.reshape(inputTemp.n_elem, 1); - outputTemp.reshape(outputTemp.n_elem, 1); - - input.col(i) = inputTemp; - output.col(i) = outputTemp; + size_t idx = rand_r() % 2; + input(idx, index(i)) = 1; + output(idx, index(i) > index(i == 0) ? 9 : 8) = 1; } + + for (size_t i = 2; i < 8; i++) + input(2 + rand_r() % 6, index(i)) = 1; + + // Set the prompts which direct the network to give an answer. + input(8, 8) = 1; + input(9, 9) = 1; + + input.reshape(input.n_elem, 1); + output.reshape(output.n_elem, 1); } /** @@ -543,15 +529,18 @@ void DistractedSequenceRecallTestNetwork() const size_t trainDistractedSequenceCount = 800; const size_t testDistractedSequenceCount = 400; - arma::mat trainInput, trainLabels, testInput, testLabels; + arma::field trainInput(1, trainDistractedSequenceCount); + arma::field trainLabels(1, trainDistractedSequenceCount); + arma::field testInput(1, testDistractedSequenceCount); + arma::field testLabels(1, testDistractedSequenceCount); // Generate the training data. - GenerateDistractedSequence(trainInput, trainLabels, - trainDistractedSequenceCount); + for (size_t i = 0; i < trainDistractedSequenceCount; i++) + GenerateDistractedSequence(trainInput(0, i), trainLabels(0, i)); // Generate the test data. - GenerateDistractedSequence(testInput, testLabels, - testDistractedSequenceCount); + for (size_t i = 0; i < testDistractedSequenceCount; i++) + GenerateDistractedSequence(testInput(0, i), testLabels(0, i)); /* * Construct a network with 10 input units, layerSize hidden units and 3 @@ -571,7 +560,7 @@ void DistractedSequenceRecallTestNetwork() */ const size_t outputSize = 3; const size_t inputSize = 10; - const size_t rho = trainInput.col(0).n_elem / inputSize; + const size_t rho = trainInput.at(0, 0).n_elem / inputSize; // It isn't guaranteed that the recurrent network will converge in the // specified number of iterations using random weights. If this works 1 of 5 @@ -581,21 +570,26 @@ void DistractedSequenceRecallTestNetwork() size_t offset = 0; for (size_t trial = 0; trial < 5; ++trial) { - RandomInitialization init(-0.5, 0.5); - MeanSquaredError<> output; - RNN, RandomInitialization> model( - rho, false, output, init); + RNN > model(rho); model.Add >(); - model.Add >(inputSize, 16); - model.Add >(16, 7, rho); + model.Add >(inputSize, 14); + model.Add >(14, 7, rho); model.Add >(7, outputSize); model.Add >(); - StandardSGD opt(model, 0.1, - trainDistractedSequenceCount * (6 + offset), -1); + StandardSGD opt(model, 0.1, 2, -50000); arma::mat inputTemp, labelsTemp; - model.Train(trainInput, trainLabels, opt); + for (size_t i = 0; i < (10 + offset); i++) + { + for (size_t j = 0; j < trainDistractedSequenceCount; j++) + { + inputTemp = trainInput.at(0, j); + labelsTemp = trainLabels.at(0, j); + + model.Train(inputTemp, labelsTemp, opt); + } + } double error = 0; @@ -604,12 +598,12 @@ void DistractedSequenceRecallTestNetwork() for (size_t i = 0; i < testDistractedSequenceCount; i++) { arma::mat output; - arma::mat input = testInput.col(i); + arma::mat input = testInput.at(0, i); model.Predict(input, output); data::Binarize(output, output, 0.5); - if (arma::accu(arma::abs(testLabels.col(i) - output)) != 0) + if (arma::accu(arma::abs(testLabels.at(0, i) - output)) != 0) error += 1; } diff --git a/src/mlpack/tests/rl_components_test.cpp b/src/mlpack/tests/rl_components_test.cpp index 708623a286..5ee77f3350 100644 --- a/src/mlpack/tests/rl_components_test.cpp +++ b/src/mlpack/tests/rl_components_test.cpp @@ -79,7 +79,9 @@ BOOST_AUTO_TEST_CASE(RandomReplayTest) arma::icolvec sampledTerminal; //! So far there should be only one record in the memory - replay.Sample(sampledState, sampledAction, sampledReward, sampledNextState, sampledTerminal); + replay.Sample(sampledState, sampledAction, sampledReward, sampledNextState, + sampledTerminal); + CheckMatrices(state.Encode(), sampledState); BOOST_REQUIRE_EQUAL(action, arma::as_scalar(sampledAction)); BOOST_REQUIRE_CLOSE(reward, arma::as_scalar(sampledReward), 1e-5); @@ -96,7 +98,9 @@ BOOST_AUTO_TEST_CASE(RandomReplayTest) //! Sample several times, the original record shouldn't appear for (size_t i = 0; i < 30; ++i) { - replay.Sample(sampledState, sampledAction, sampledReward, sampledNextState, sampledTerminal); + replay.Sample(sampledState, sampledAction, sampledReward, sampledNextState, + sampledTerminal); + CheckMatrices(state.Encode(), sampledNextState); CheckMatrices(nextState.Encode(), sampledState); BOOST_REQUIRE_EQUAL(true, arma::as_scalar(sampledTerminal)); diff --git a/src/mlpack/tests/sa_test.cpp b/src/mlpack/tests/sa_test.cpp index d3a36851e5..f2314b015e 100644 --- a/src/mlpack/tests/sa_test.cpp +++ b/src/mlpack/tests/sa_test.cpp @@ -61,7 +61,7 @@ BOOST_AUTO_TEST_CASE(RosenbrockTest) { RosenbrockFunction f; ExponentialSchedule schedule(1e-5); - SA //sa(f, schedule); // All default parameters. + SA // sa(f, schedule); // All default parameters. sa(f, schedule, 10000000, 1000., 1000, 100, 1e-11, 3, 20, 0.3, 0.3); arma::mat coordinates = f.GetInitialPoint(); @@ -114,7 +114,7 @@ BOOST_AUTO_TEST_CASE(RastrigrinFunctionTest) { RastrigrinFunction f; ExponentialSchedule schedule(3e-6); - SA //sa(f, schedule); + SA // sa(f, schedule); sa(f, schedule, 20000000, 100, 50, 1000, 1e-12, 2, 0.2, 0.01, 0.1); arma::mat coordinates = f.GetInitialPoint(); diff --git a/src/mlpack/tests/sdp_primal_dual_test.cpp b/src/mlpack/tests/sdp_primal_dual_test.cpp index 9d5103de80..f0f9a06f35 100644 --- a/src/mlpack/tests/sdp_primal_dual_test.cpp +++ b/src/mlpack/tests/sdp_primal_dual_test.cpp @@ -24,7 +24,6 @@ using namespace mlpack::neighbor; class UndirectedGraph { public: - UndirectedGraph() {} size_t NumVertices() const { return numVertices; } @@ -110,7 +109,6 @@ class UndirectedGraph } private: - void ComputeVertices() { numVertices = max(max(edges)) + 1; @@ -269,7 +267,7 @@ BOOST_AUTO_TEST_CASE(SmallMaxCutSdp) // the following was resulting in non-positive Z0 matrices on some // random instances. - //SolveMaxCutFeasibleSDP(sdp); + // SolveMaxCutFeasibleSDP(sdp); SolveMaxCutPositiveSDP(sdp); } @@ -537,7 +535,7 @@ BOOST_AUTO_TEST_CASE(CorrelationCoeffToySdp) BOOST_REQUIRE_CLOSE(obj, 2 * (-0.978), 1e-3); } -///** +// /** // * Maximum variance unfolding (MVU) SDP to learn the unrolled gram matrix. For // * the SDP formulation, see: // * @@ -548,66 +546,66 @@ BOOST_AUTO_TEST_CASE(CorrelationCoeffToySdp) // * @param origData origDim x numPoints // * @param numNeighbors // */ -//static inline SDP ConstructMvuSDP(const arma::mat& origData, +// static inline SDP ConstructMvuSDP(const arma::mat& origData, // size_t numNeighbors) -//{ +// { // const size_t numPoints = origData.n_cols; -// + // assert(numNeighbors <= numPoints); -// + // arma::Mat neighbors; // arma::mat distances; // KNN knn(origData); // knn.Search(numNeighbors, neighbors, distances); -// + // SDP sdp(numPoints, numNeighbors * numPoints, 1); // sdp.C().eye(numPoints, numPoints); // sdp.C() *= -1; // sdp.DenseA()[0].ones(numPoints, numPoints); // sdp.DenseB()[0] = 0; -// + // for (size_t i = 0; i < neighbors.n_cols; ++i) // { // for (size_t j = 0; j < numNeighbors; ++j) // { // // This is the index of the constraint. // const size_t index = (i * numNeighbors) + j; -// + // arma::sp_mat& aRef = sdp.SparseA()[index]; // aRef.zeros(numPoints, numPoints); -// + // // A_ij(i, i) = 1. // aRef(i, i) = 1; -// + // // A_ij(i, j) = -1. // aRef(i, neighbors(j, i)) = -1; -// + // // A_ij(j, i) = -1. // aRef(neighbors(j, i), i) = -1; -// + // // A_ij(j, j) = 1. // aRef(neighbors(j, i), neighbors(j, i)) = 1; -// + // // The constraint b_ij is the distance between these two points. // sdp.SparseB()[index] = distances(j, i); // } // } -// + // return sdp; -//} -// -///** +// } + +// /** // * Maximum variance unfolding // * // * Test doesn't work, because the constraint matrices are not linearly // * independent. // */ -//BOOST_AUTO_TEST_CASE(SmallMvuSdp) -//{ +// BOOST_AUTO_TEST_CASE(SmallMvuSdp) +// { // const size_t n = 20; -// + // arma::mat origData(3, n); -// + // // sample n random points on 3-dim unit sphere // GaussianDistribution gauss(3); // for (size_t i = 0; i < n; i++) @@ -615,14 +613,14 @@ BOOST_AUTO_TEST_CASE(CorrelationCoeffToySdp) // // how european of them // origData.col(i) = arma::normalise(gauss.Random()); // } -// + // auto sdp = ConstructMvuSDP(origData, 5); -// + // PrimalDualSolver> solver(sdp); // arma::mat X, Z; // arma::vec ysparse, ydense; // const auto p = solver.Optimize(X, ysparse, ydense, Z); // BOOST_REQUIRE(p.first); -//} +// } BOOST_AUTO_TEST_SUITE_END(); diff --git a/src/mlpack/tests/smorms3_test.cpp b/src/mlpack/tests/smorms3_test.cpp index 1d62556312..eb4aa48117 100644 --- a/src/mlpack/tests/smorms3_test.cpp +++ b/src/mlpack/tests/smorms3_test.cpp @@ -106,4 +106,4 @@ BOOST_AUTO_TEST_CASE(SMORMS3LogisticRegressionTest) BOOST_REQUIRE_CLOSE(testAcc, 100.0, 0.6); // 0.6% error tolerance. } -BOOST_AUTO_TEST_SUITE_END(); \ No newline at end of file +BOOST_AUTO_TEST_SUITE_END(); diff --git a/src/mlpack/tests/softmax_regression_test.cpp b/src/mlpack/tests/softmax_regression_test.cpp index d5741843b3..63b6f1129f 100644 --- a/src/mlpack/tests/softmax_regression_test.cpp +++ b/src/mlpack/tests/softmax_regression_test.cpp @@ -35,14 +35,14 @@ BOOST_AUTO_TEST_CASE(SoftmaxRegressionFunctionEvaluate) // Create random class labels. arma::Row labels(points); - for(size_t i = 0; i < points; i++) + for (size_t i = 0; i < points; i++) labels(i) = math::RandInt(0, numClasses); // Create a SoftmaxRegressionFunction. Regularization term ignored. SoftmaxRegressionFunction srf(data, labels, numClasses, 0); // Run a number of trials. - for(size_t i = 0; i < trials; i++) + for (size_t i = 0; i < trials; i++) { // Create a random set of parameters. arma::mat parameters; @@ -51,7 +51,7 @@ BOOST_AUTO_TEST_CASE(SoftmaxRegressionFunctionEvaluate) double logLikelihood = 0; // Compute error for each training example. - for(size_t j = 0; j < points; j++) + for (size_t j = 0; j < points; j++) { arma::mat hypothesis, probabilities; @@ -80,7 +80,7 @@ BOOST_AUTO_TEST_CASE(SoftmaxRegressionFunctionRegularizationEvaluate) // Create random class labels. arma::Row labels(points); - for(size_t i = 0; i < points; i++) + for (size_t i = 0; i < points; i++) labels(i) = math::RandInt(0, numClasses); // 3 objects for comparing regularization costs. @@ -121,7 +121,7 @@ BOOST_AUTO_TEST_CASE(SoftmaxRegressionFunctionGradient) // Create random class labels. arma::Row labels(points); - for(size_t i = 0; i < points; i++) + for (size_t i = 0; i < points; i++) labels(i) = math::RandInt(0, numClasses); // 2 objects for 2 terms in the cost function. Each term contributes towards @@ -489,7 +489,7 @@ BOOST_AUTO_TEST_CASE(SoftmaxRegressionClassifySinglePointTest) sr.Classify(data, labels); - for(size_t i = 0; i < data.n_cols; ++i) + for (size_t i = 0; i < data.n_cols; ++i) { BOOST_REQUIRE_EQUAL(sr.Classify(data.col(i)), labels(i)); } @@ -575,7 +575,7 @@ BOOST_AUTO_TEST_CASE(SoftmaxRegressionComputeProbabilitiesTest) BOOST_REQUIRE_EQUAL(probabilities.n_cols, data.n_cols); BOOST_REQUIRE_EQUAL(probabilities.n_rows, sr.NumClasses()); - for(size_t i = 0; i < data.n_cols; ++i) + for (size_t i = 0; i < data.n_cols; ++i) { BOOST_REQUIRE_CLOSE(arma::sum(probabilities.col(i)), 1.0, 1e-5); } @@ -664,7 +664,7 @@ BOOST_AUTO_TEST_CASE(SoftmaxRegressionComputeProbabilitiesAndLabelsTest) BOOST_REQUIRE_EQUAL(probabilities.n_cols, data.n_cols); BOOST_REQUIRE_EQUAL(probabilities.n_rows, sr.NumClasses()); - for(size_t i = 0; i < data.n_cols; ++i) + for (size_t i = 0; i < data.n_cols; ++i) { BOOST_REQUIRE_CLOSE(arma::sum(probabilities.col(i)), 1.0, 1e-5); BOOST_REQUIRE_EQUAL(testLabels(i), labels(i)); diff --git a/src/mlpack/tests/sparse_autoencoder_test.cpp b/src/mlpack/tests/sparse_autoencoder_test.cpp index 463b40d65c..d9e41983cd 100644 --- a/src/mlpack/tests/sparse_autoencoder_test.cpp +++ b/src/mlpack/tests/sparse_autoencoder_test.cpp @@ -91,7 +91,7 @@ BOOST_AUTO_TEST_CASE(SparseAutoencoderFunctionRandomEvaluate) (1 + arma::exp(-(parameters.submat(0, 0, l1 - 1, l2 - 1) * data.col(j) + parameters.submat(0, l2, l1 - 1, l2)))); outputLayer = 1.0 / - (1 + arma::exp(-(parameters.submat(l1, 0, l3 - 1,l2 - 1).t() + (1 + arma::exp(-(parameters.submat(l1, 0, l3 - 1, l2 - 1).t() * hiddenLayer + parameters.submat(l3, 0, l3, l2 - 1).t()))); diff = outputLayer - data.col(j); @@ -167,7 +167,7 @@ BOOST_AUTO_TEST_CASE(SparseAutoencoderFunctionKLDivergenceEvaluate) SparseAutoencoderFunction safBigDiv(data, vSize, hSize, 0, 20, rho); // Run a number of trials. - for(size_t i = 0; i < trials; i++) + for (size_t i = 0; i < trials; i++) { // Create a random set of parameters. arma::mat parameters; diff --git a/src/mlpack/tests/sparse_coding_test.cpp b/src/mlpack/tests/sparse_coding_test.cpp index 93f97024d2..7018a9e4a4 100644 --- a/src/mlpack/tests/sparse_coding_test.cpp +++ b/src/mlpack/tests/sparse_coding_test.cpp @@ -30,7 +30,7 @@ void SCVerifyCorrectness(vec beta, vec errCorr, double lambda) { const double tol = 1e-12; size_t nDims = beta.n_elem; - for(size_t j = 0; j < nDims; j++) + for (size_t j = 0; j < nDims; j++) { if (beta(j) == 0) { @@ -99,7 +99,7 @@ BOOST_AUTO_TEST_CASE(SparseCodingTestCodingStepElasticNet) mat D = sc.Dictionary(); - for(uword i = 0; i < nPoints; ++i) + for (uword i = 0; i < nPoints; ++i) { vec errCorr = (trans(D) * D + lambda2 * eye(nAtoms, nAtoms)) * Z.unsafe_col(i) diff --git a/src/mlpack/tests/svd_incremental_test.cpp b/src/mlpack/tests/svd_incremental_test.cpp index 5b47e4449e..91ac784bdb 100644 --- a/src/mlpack/tests/svd_incremental_test.cpp +++ b/src/mlpack/tests/svd_incremental_test.cpp @@ -38,7 +38,7 @@ BOOST_AUTO_TEST_CASE(SVDIncompleteIncrementalConvergenceTest) RandomInitialization, SVDIncompleteIncrementalLearning> amf(iit, RandomInitialization(), svd); - mat m1,m2; + mat m1, m2; amf.Apply(data, 2, m1, m2); BOOST_REQUIRE_NE(amf.TerminationPolicy().Iteration(), @@ -61,7 +61,7 @@ BOOST_AUTO_TEST_CASE(SVDCompleteIncrementalConvergenceTest) SVDCompleteIncrementalLearning > amf(iit, RandomInitialization(), svd); - mat m1,m2; + mat m1, m2; amf.Apply(data, 2, m1, m2); BOOST_REQUIRE_NE(amf.TerminationPolicy().Iteration(), diff --git a/src/mlpack/tests/test_tools.hpp b/src/mlpack/tests/test_tools.hpp index 53434538ea..b7875b84a9 100644 --- a/src/mlpack/tests/test_tools.hpp +++ b/src/mlpack/tests/test_tools.hpp @@ -17,11 +17,12 @@ // Require the approximation L to be within a relative error of E respect to the // actual value R. -#define REQUIRE_RELATIVE_ERR( L, R, E ) \ - BOOST_REQUIRE_LE( std::abs((R) - (L)), (E) * std::abs(R)) +#define REQUIRE_RELATIVE_ERR(L, R, E) \ + BOOST_REQUIRE_LE(std::abs((R) - (L)), (E) * std::abs(R)) // Check the values of two matrices. -inline void CheckMatrices(const arma::mat& a, const arma::mat& b, +inline void CheckMatrices(const arma::mat& a, + const arma::mat& b, double tolerance = 1e-5) { BOOST_REQUIRE_EQUAL(a.n_rows, b.n_rows); @@ -37,7 +38,8 @@ inline void CheckMatrices(const arma::mat& a, const arma::mat& b, } // Check the values of two unsigned matrices. -inline void CheckMatrices(const arma::Mat& a, const arma::Mat& b) +inline void CheckMatrices(const arma::Mat& a, + const arma::Mat& b) { BOOST_REQUIRE_EQUAL(a.n_rows, b.n_rows); BOOST_REQUIRE_EQUAL(a.n_cols, b.n_cols); diff --git a/src/mlpack/tests/tree_test.cpp b/src/mlpack/tests/tree_test.cpp index 449ebebc05..b794fc94c8 100644 --- a/src/mlpack/tests/tree_test.cpp +++ b/src/mlpack/tests/tree_test.cpp @@ -1573,7 +1573,6 @@ void CheckRPTreeSplit(const TreeType& tree) BOOST_REQUIRE_LE(maxDist, dist * (1.0 + 10.0 * std::numeric_limits::epsilon())); } - } CheckRPTreeSplit(*tree.Left()); @@ -1648,9 +1647,9 @@ BOOST_AUTO_TEST_CASE(BallTreeTest) BOOST_REQUIRE_EQUAL(root.NumDescendants(), size); // Check the forward and backward mappings for correctness. - for(size_t i = 0; i < size; i++) + for (size_t i = 0; i < size; i++) { - for(size_t j = 0; j < dimensions; j++) + for (size_t j = 0; j < dimensions; j++) { BOOST_REQUIRE_EQUAL(treeset(j, i), dataset(j, newToOld[i])); BOOST_REQUIRE_EQUAL(treeset(j, oldToNew[i]), dataset(j, i)); diff --git a/src/mlpack/tests/ub_tree_test.cpp b/src/mlpack/tests/ub_tree_test.cpp index 8b75e2d7e3..cbbf2241d1 100644 --- a/src/mlpack/tests/ub_tree_test.cpp +++ b/src/mlpack/tests/ub_tree_test.cpp @@ -47,7 +47,6 @@ BOOST_AUTO_TEST_CASE(AddressTest) for (size_t k = 0; k < dataset.n_rows; k++) BOOST_REQUIRE_CLOSE(dataset(k, i), point[k], 1e-13); } - } template diff --git a/src/mlpack/tests/vantage_point_tree_test.cpp b/src/mlpack/tests/vantage_point_tree_test.cpp index 2223341f21..5ff0abd197 100644 --- a/src/mlpack/tests/vantage_point_tree_test.cpp +++ b/src/mlpack/tests/vantage_point_tree_test.cpp @@ -220,9 +220,9 @@ BOOST_AUTO_TEST_CASE(VPTreeTest) BOOST_REQUIRE_EQUAL(root.NumDescendants(), size); // Check the forward and backward mappings for correctness. - for(size_t i = 0; i < size; i++) + for (size_t i = 0; i < size; i++) { - for(size_t j = 0; j < dimensions; j++) + for (size_t j = 0; j < dimensions; j++) { BOOST_REQUIRE_EQUAL(treeset(j, i), dataset(j, newToOld[i])); BOOST_REQUIRE_EQUAL(treeset(j, oldToNew[i]), dataset(j, i)); From 664091aa77b2bbdc17c56b20db935b655c10cb0b Mon Sep 17 00:00:00 2001 From: Marcus Edel Date: Mon, 5 Jun 2017 21:40:40 +0200 Subject: [PATCH 76/84] Fix additional style issues pointed out by cpplint. --- .../simple_tolerance_termination.hpp | 4 ++-- .../svd_incomplete_incremental_learning.hpp | 4 ++-- src/mlpack/methods/det/dt_utils_impl.hpp | 2 +- src/mlpack/methods/det/dtree_impl.hpp | 7 ++++--- src/mlpack/methods/emst/dtb_rules.hpp | 4 ++-- .../methods/kernel_pca/kernel_pca_impl.hpp | 2 +- src/mlpack/methods/lsh/lsh_search_impl.hpp | 14 ++++++++++---- src/mlpack/tests/cli_test.cpp | 18 ++++++++++-------- src/mlpack/tests/momentum_sgd_test.cpp | 2 +- 9 files changed, 33 insertions(+), 24 deletions(-) diff --git a/src/mlpack/methods/amf/termination_policies/simple_tolerance_termination.hpp b/src/mlpack/methods/amf/termination_policies/simple_tolerance_termination.hpp index 08341cc3e3..f645494c31 100644 --- a/src/mlpack/methods/amf/termination_policies/simple_tolerance_termination.hpp +++ b/src/mlpack/methods/amf/termination_policies/simple_tolerance_termination.hpp @@ -78,9 +78,9 @@ class SimpleToleranceTermination size_t m = V->n_cols; double sum = 0; size_t count = 0; - for (size_t i = 0;i < n; i++) + for (size_t i = 0; i < n; i++) { - for (size_t j = 0;j < m; j++) + for (size_t j = 0; j < m; j++) { double temp = 0; if ((temp = (*V)(i, j)) != 0) diff --git a/src/mlpack/methods/amf/update_rules/svd_incomplete_incremental_learning.hpp b/src/mlpack/methods/amf/update_rules/svd_incomplete_incremental_learning.hpp index 13da871457..ce5c26f99c 100644 --- a/src/mlpack/methods/amf/update_rules/svd_incomplete_incremental_learning.hpp +++ b/src/mlpack/methods/amf/update_rules/svd_incomplete_incremental_learning.hpp @@ -188,7 +188,7 @@ inline void SVDIncompleteIncrementalLearning::HUpdate( arma::mat deltaH(H.n_rows, 1); deltaH.zeros(); - for(arma::sp_mat::const_iterator it = V.begin_col(currentUserIndex); + for (arma::sp_mat::const_iterator it = V.begin_col(currentUserIndex); it != V.end_col(currentUserIndex); it++) { double val = *it; @@ -205,7 +205,7 @@ inline void SVDIncompleteIncrementalLearning::HUpdate( currentUserIndex = currentUserIndex % V.n_cols; } -} // namepsace amf +} // namespace amf } // namespace mlpack #endif diff --git a/src/mlpack/methods/det/dt_utils_impl.hpp b/src/mlpack/methods/det/dt_utils_impl.hpp index 1ac986c4d9..389cf314fc 100644 --- a/src/mlpack/methods/det/dt_utils_impl.hpp +++ b/src/mlpack/methods/det/dt_utils_impl.hpp @@ -270,7 +270,7 @@ DTree* Trainer(MatType& dataset, cvRegularizationConstants[prunedSequence.size() - 2] += 2.0 * cvVal / (double) cvData.n_cols; - #pragma omp critical (DTreeCVUpdate) + #pragma omp critical(DTreeCVUpdate) regularizationConstants += cvRegularizationConstants; } Timer::Stop("cross_validation"); diff --git a/src/mlpack/methods/det/dtree_impl.hpp b/src/mlpack/methods/det/dtree_impl.hpp index 1d52f085f9..6372b19c0f 100644 --- a/src/mlpack/methods/det/dtree_impl.hpp +++ b/src/mlpack/methods/det/dtree_impl.hpp @@ -145,7 +145,7 @@ namespace details lastVal = newVal; } } -}; //namespace details +}; // namespace details template DTree::DTree() : @@ -480,7 +480,8 @@ bool DTree::FindSplit(const MatType& data, // sparse matrices. std::vector splitVec; - details::ExtractSplits(splitVec, data, dim, start, end, minLeafSize); + details::ExtractSplits(splitVec, data, dim, start, end, + minLeafSize); // Iterate on all the splits for this dimension for (typename std::vector::iterator i = splitVec.begin(); @@ -523,7 +524,7 @@ bool DTree::FindSplit(const MatType& data, - 2 * std::log((double) data.n_cols) - volumeWithoutDim; -#pragma omp critical (DTreeFindUpdate) +#pragma omp critical(DTreeFindUpdate) if ((actualMinDimError > minError) && dimSplitFound) { // Calculate actual error (in logspace) by adding terms back to our diff --git a/src/mlpack/methods/emst/dtb_rules.hpp b/src/mlpack/methods/emst/dtb_rules.hpp index 69d329f395..b89c5b4754 100644 --- a/src/mlpack/methods/emst/dtb_rules.hpp +++ b/src/mlpack/methods/emst/dtb_rules.hpp @@ -131,8 +131,8 @@ class DTBRules size_t scores; }; // class DTBRules -} // emst namespace -} // mlpack namespace +} // namespace emst +} // namespace mlpack #include "dtb_rules_impl.hpp" diff --git a/src/mlpack/methods/kernel_pca/kernel_pca_impl.hpp b/src/mlpack/methods/kernel_pca/kernel_pca_impl.hpp index 5204918fa6..2d106aeb59 100644 --- a/src/mlpack/methods/kernel_pca/kernel_pca_impl.hpp +++ b/src/mlpack/methods/kernel_pca/kernel_pca_impl.hpp @@ -81,7 +81,7 @@ void KernelPCA::Apply(arma::mat& data, data.shed_rows(newDimension, data.n_rows - 1); } -} // namespace mlpack } // namespace kpca +} // namespace mlpack #endif diff --git a/src/mlpack/methods/lsh/lsh_search_impl.hpp b/src/mlpack/methods/lsh/lsh_search_impl.hpp index d0439c5c25..f7994b3688 100644 --- a/src/mlpack/methods/lsh/lsh_search_impl.hpp +++ b/src/mlpack/methods/lsh/lsh_search_impl.hpp @@ -694,8 +694,9 @@ void LSHSearch::GetAdditionalProbingBins( // Shift operation on Ai (replace max with max+1). std::vector As = Ai; + + // Don't add invalid sets. if (PerturbationShift(As) && PerturbationValid(As)) - // Don't add invalid sets. { perturbationSets.push_back(As); // add shifted set to sets minHeap.push( @@ -705,6 +706,7 @@ void LSHSearch::GetAdditionalProbingBins( // Expand operation on Ai (add max+1 to set). std::vector Ae = Ai; + // Don't add invalid sets. if (PerturbationExpand(Ae) && PerturbationValid(Ae)) { @@ -831,9 +833,11 @@ void LSHSearch::ReturnIndicesFromTable( size_t tableRow = bucketRowInHashTable[hashInd]; if (tableRow < secondHashSize && bucketContentSize[tableRow] > 0) + { // Pick the indices in the bucket corresponding to hashInd. for (size_t j = 0; j < bucketContentSize[tableRow]; ++j) refPointsConsidered[ secondHashTable[tableRow](j) ]++; + } } } @@ -860,9 +864,11 @@ void LSHSearch::ReturnIndicesFromTable( const size_t tableRow = bucketRowInHashTable[hashInd]; if (tableRow < secondHashSize) - // Store all secondHashTable points in the candidates set. - for (size_t j = 0; j < bucketContentSize[tableRow]; ++j) - refPointsConsideredSmall(start++) = secondHashTable[tableRow](j); + { + // Store all secondHashTable points in the candidates set. + for (size_t j = 0; j < bucketContentSize[tableRow]; ++j) + refPointsConsideredSmall(start++) = secondHashTable[tableRow](j); + } } } diff --git a/src/mlpack/tests/cli_test.cpp b/src/mlpack/tests/cli_test.cpp index c36152c6c5..1b9d59fba5 100644 --- a/src/mlpack/tests/cli_test.cpp +++ b/src/mlpack/tests/cli_test.cpp @@ -290,7 +290,8 @@ BOOST_AUTO_TEST_CASE(InputRowVectorParamTest) { AddRequiredCLIOptions(); - CLI::Add(arma::rowvec(), "row", "Test vector", 'l', false, true, false); + CLI::Add(arma::rowvec(), "row", "Test vector", 'l', false, + true, false); // Fake arguments. const char* argv[3]; @@ -327,7 +328,8 @@ BOOST_AUTO_TEST_CASE(InputUngignedRowVectorParamTest) { AddRequiredCLIOptions(); - CLI::Add>(arma::Row(), "row", "Test vector", 'l', false, true, false); + CLI::Add>(arma::Row(), "row", "Test vector", 'l', + false, true, false); // Fake arguments. const char* argv[3]; @@ -412,8 +414,8 @@ BOOST_AUTO_TEST_CASE(OutputUnsignedColParamTest) AddRequiredCLIOptions(); // --vector is an output parameter. - CLI::Add>(arma::Col(), "vector", "Test vector", 'l', false, false, - false); + CLI::Add>(arma::Col(), "vector", "Test vector", 'l', + false, false, false); // Set some fake arguments. const char* argv[3]; @@ -459,8 +461,8 @@ BOOST_AUTO_TEST_CASE(OutputRowParamTest) AddRequiredCLIOptions(); // --row is an output parameter. - CLI::Add(arma::rowvec(), "row", "Test vector", 'l', false, false, - false); + CLI::Add(arma::rowvec(), "row", "Test vector", 'l', + false, false, false); // Set some fake arguments. const char* argv[3]; @@ -506,8 +508,8 @@ BOOST_AUTO_TEST_CASE(OutputUnsignedRowParamTest) AddRequiredCLIOptions(); // --row is an output parameter. - CLI::Add>(arma::Row(), "row", "Test vector", 'l', false, false, - false); + CLI::Add>(arma::Row(), "row", "Test vector", 'l', + false, false, false); // Set some fake arguments. const char* argv[3]; diff --git a/src/mlpack/tests/momentum_sgd_test.cpp b/src/mlpack/tests/momentum_sgd_test.cpp index 2b8184d8bd..3d71168c52 100644 --- a/src/mlpack/tests/momentum_sgd_test.cpp +++ b/src/mlpack/tests/momentum_sgd_test.cpp @@ -54,7 +54,7 @@ BOOST_AUTO_TEST_CASE(MomentumSGDSpeedUpTestFunction) BOOST_REQUIRE_SMALL(coordinates1[1], 1e-7); BOOST_REQUIRE_SMALL(coordinates1[2], 1e-7); - BOOST_REQUIRE_LE(result,result1); + BOOST_REQUIRE_LE(result, result1); } BOOST_AUTO_TEST_CASE(GeneralizedRosenbrockTest) From cc8f5e70f0c6f4ffa1409c2c16437fd0f0de8e40 Mon Sep 17 00:00:00 2001 From: Marcus Edel Date: Mon, 5 Jun 2017 21:50:31 +0200 Subject: [PATCH 77/84] Fix minor style issue (Lines should be <= 80 characters long). --- src/mlpack/methods/naive_bayes/nbc_main.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/mlpack/methods/naive_bayes/nbc_main.cpp b/src/mlpack/methods/naive_bayes/nbc_main.cpp index 31a559fe1a..2ef3c4cf56 100644 --- a/src/mlpack/methods/naive_bayes/nbc_main.cpp +++ b/src/mlpack/methods/naive_bayes/nbc_main.cpp @@ -77,8 +77,8 @@ PARAM_FLAG("incremental_variance", "The variance of each class will be " PARAM_MATRIX_IN("test", "A matrix containing the test set.", "T"); PARAM_UROW_OUT("output", "The matrix in which the predicted labels for the" " test set will be written.", "o"); -PARAM_MATRIX_OUT("output_probs", "The matrix in which the predicted probability of labels for the" - " test set will be written.", "p"); +PARAM_MATRIX_OUT("output_probs", "The matrix in which the predicted probability" + " of labels for the test set will be written.", "p"); int main(int argc, char* argv[]) { From 7a00f38e8936b730fed62a592dad586ae4bedb20 Mon Sep 17 00:00:00 2001 From: Marcus Edel Date: Mon, 5 Jun 2017 22:09:18 +0200 Subject: [PATCH 78/84] No need to use rand_r over rand in this case. --- .../termination_policies/validation_RMSE_termination.hpp | 4 ++-- src/mlpack/tests/recurrent_network_test.cpp | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/mlpack/methods/amf/termination_policies/validation_RMSE_termination.hpp b/src/mlpack/methods/amf/termination_policies/validation_RMSE_termination.hpp index 9c95c9867c..9171aa0ae3 100644 --- a/src/mlpack/methods/amf/termination_policies/validation_RMSE_termination.hpp +++ b/src/mlpack/methods/amf/termination_policies/validation_RMSE_termination.hpp @@ -72,8 +72,8 @@ class ValidationRMSETermination // pick a random non-zero entry do { - t_row = rand_r() % n; - t_col = rand_r() % m; + t_row = rand() % n; + t_col = rand() % m; } while ((t_val = V(t_row, t_col)) == 0); // add the entry to the validation set diff --git a/src/mlpack/tests/recurrent_network_test.cpp b/src/mlpack/tests/recurrent_network_test.cpp index 0bbc439790..3941c3d6b8 100644 --- a/src/mlpack/tests/recurrent_network_test.cpp +++ b/src/mlpack/tests/recurrent_network_test.cpp @@ -175,7 +175,7 @@ void GenerateReber(const arma::Mat& transitions, std::string& reber) do { - const int grammerIdx = rand_r() % 2; + const int grammerIdx = rand() % 2; reber += arma::as_scalar(transitions.submat(idx, grammerIdx, idx, grammerIdx)); @@ -196,7 +196,7 @@ void GenerateEmbeddedReber(const arma::Mat& transitions, std::string& reber) { GenerateReber(transitions, reber); - const char c = (rand_r() % 2) == 1 ? 'P' : 'T'; + const char c = (rand() % 2) == 1 ? 'P' : 'T'; reber = c + reber + c; reber = "B" + reber + "E"; } @@ -504,13 +504,13 @@ void GenerateDistractedSequence(arma::mat& input, arma::mat& output) // output sequence by following the correct order. for (size_t i = 0; i < 2; i++) { - size_t idx = rand_r() % 2; + size_t idx = rand() % 2; input(idx, index(i)) = 1; output(idx, index(i) > index(i == 0) ? 9 : 8) = 1; } for (size_t i = 2; i < 8; i++) - input(2 + rand_r() % 6, index(i)) = 1; + input(2 + rand() % 6, index(i)) = 1; // Set the prompts which direct the network to give an answer. input(8, 8) = 1; From 609365c6f62f825e1f01875bb95bbf5a4e1dc50b Mon Sep 17 00:00:00 2001 From: Marcus Edel Date: Mon, 5 Jun 2017 22:47:39 +0200 Subject: [PATCH 79/84] Revert dimension change. --- .../methods/reinforcement_learning/environment/cart_pole.hpp | 5 ++++- .../reinforcement_learning/environment/mountain_car.hpp | 5 ++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/src/mlpack/methods/reinforcement_learning/environment/cart_pole.hpp b/src/mlpack/methods/reinforcement_learning/environment/cart_pole.hpp index 0884333c80..62d168caa4 100644 --- a/src/mlpack/methods/reinforcement_learning/environment/cart_pole.hpp +++ b/src/mlpack/methods/reinforcement_learning/environment/cart_pole.hpp @@ -36,7 +36,7 @@ class CartPole /** * Construct a state instance. */ - State() : data(4) + State() : data(dimension) { /* Nothing to do here. */ } /** @@ -73,6 +73,9 @@ class CartPole //! Encode the state to a column vector. const arma::colvec& Encode() const { return data; } + //! Dimension of the encoded state. + static constexpr size_t dimension = 4; + private: //! Locally-stored (position, velocity, angle, angular velocity). arma::colvec data; diff --git a/src/mlpack/methods/reinforcement_learning/environment/mountain_car.hpp b/src/mlpack/methods/reinforcement_learning/environment/mountain_car.hpp index 203e90fe6b..9ad595fc7e 100644 --- a/src/mlpack/methods/reinforcement_learning/environment/mountain_car.hpp +++ b/src/mlpack/methods/reinforcement_learning/environment/mountain_car.hpp @@ -37,7 +37,7 @@ class MountainCar /** * Construct a state instance. */ - State(): data(2, arma::fill::zeros) + State(): data(dimension, arma::fill::zeros) { /* Nothing to do here. */ } /** @@ -64,6 +64,9 @@ class MountainCar //! Encode the state to a column vector. const arma::colvec& Encode() const { return data; } + //! Dimension of the encoded state. + static constexpr size_t dimension = 2; + private: //! Locally-stored velocity and position vector. arma::colvec data; From 0538de4a26edf71be42a17d7e78f847afe764090 Mon Sep 17 00:00:00 2001 From: Marcus Edel Date: Tue, 6 Jun 2017 00:01:55 +0200 Subject: [PATCH 80/84] Use the correct escape sequence. --- .../decision_tree/decision_tree_main.cpp | 4 ++-- src/mlpack/methods/rann/krann_main.cpp | 4 ++-- src/mlpack/tests/cli_test.cpp | 4 ++-- src/mlpack/tests/load_save_test.cpp | 18 +++++++++--------- 4 files changed, 15 insertions(+), 15 deletions(-) diff --git a/src/mlpack/methods/decision_tree/decision_tree_main.cpp b/src/mlpack/methods/decision_tree/decision_tree_main.cpp index 3fc51f74c2..784eeb4174 100644 --- a/src/mlpack/methods/decision_tree/decision_tree_main.cpp +++ b/src/mlpack/methods/decision_tree/decision_tree_main.cpp @@ -177,7 +177,7 @@ int main(int argc, char** argv) ++correct; // Print number of correct points. - Log::Info << double(correct) / double(dataset.n_cols) * 100 << "\\% " + Log::Info << double(correct) / double(dataset.n_cols) * 100 << "%% " << "correct on training set (" << correct << " / " << dataset.n_cols << ")." << endl; } @@ -209,7 +209,7 @@ int main(int argc, char** argv) ++correct; // Print number of correct points. - Log::Info << double(correct) / double(testPoints.n_cols) * 100 << "\\% " + Log::Info << double(correct) / double(testPoints.n_cols) * 100 << "%% " << "correct on test set (" << correct << " / " << testPoints.n_cols << ")." << endl; } diff --git a/src/mlpack/methods/rann/krann_main.cpp b/src/mlpack/methods/rann/krann_main.cpp index 760bbd17bc..4c4d55d226 100644 --- a/src/mlpack/methods/rann/krann_main.cpp +++ b/src/mlpack/methods/rann/krann_main.cpp @@ -31,10 +31,10 @@ PROGRAM_INFO("K-Rank-Approximate-Nearest-Neighbors (kRANN)", "This program will calculate the k rank-approximate-nearest-neighbors of a " "set of points. You may specify a separate set of reference points and " "query points, or just a reference set which will be used as both the " - "reference and query set. You must specify the rank approximation (in \\%) " + "reference and query set. You must specify the rank approximation (in %%) " "(and optionally the success probability)." "\n\n" - "For example, the following will return 5 neighbors from the top 0.1\\% of " + "For example, the following will return 5 neighbors from the top 0.1%% of " "the data (with probability 0.95) for each point in 'input.csv' and store " "the distances in 'distances.csv' and the neighbors in the file " "'neighbors.csv':" diff --git a/src/mlpack/tests/cli_test.cpp b/src/mlpack/tests/cli_test.cpp index 1b9d59fba5..473edbc4cf 100644 --- a/src/mlpack/tests/cli_test.cpp +++ b/src/mlpack/tests/cli_test.cpp @@ -959,7 +959,7 @@ BOOST_AUTO_TEST_CASE(MatrixAndDatasetInfoTest) f << endl; f << "@attribute three STRING" << endl; f << endl; - f << "\\% a comment line " << endl; + f << "%% a comment line " << endl; f << endl; f << "@data" << endl; f << "hello, 1, moo" << endl; @@ -1056,7 +1056,7 @@ BOOST_AUTO_TEST_CASE(RawDatasetInfoLoadParameter) f << endl; f << "@attribute three STRING" << endl; f << endl; - f << "\\% a comment line " << endl; + f << "%% a comment line " << endl; f << endl; f << "@data" << endl; f << "hello, 1, moo" << endl; diff --git a/src/mlpack/tests/load_save_test.cpp b/src/mlpack/tests/load_save_test.cpp index e2f4e7698d..5c0006b465 100644 --- a/src/mlpack/tests/load_save_test.cpp +++ b/src/mlpack/tests/load_save_test.cpp @@ -1682,7 +1682,7 @@ BOOST_AUTO_TEST_CASE(SimpleARFFCategoricalTest) f << endl; f << "@attribute three STRING" << endl; f << endl; - f << "\\% a comment line " << endl; + f << "%% a comment line " << endl; f << endl; f << "@data" << endl; f << "hello, 1, moo" << endl; @@ -1738,15 +1738,15 @@ BOOST_AUTO_TEST_CASE(HarderARFFTest) f << endl; f << "@attribute @@@@flfl numeric" << endl; f << endl; - f << "\\% comment" << endl; + f << "%% comment" << endl; f << "@attribute \"hello world\" string" << endl; f << "@attribute 12345 integer" << endl; f << "@attribute real real" << endl; - f << "@attribute \"blah blah blah \t \" numeric \\% comment" << endl; - f << "\\% comment" << endl; + f << "@attribute \"blah blah blah \t \" numeric %% comment" << endl; + f << "%% comment" << endl; f << "@data" << endl; f << "1, one, 3, 4.5, 6" << endl; - f << "2, two, 4, 5.5, 7 \\% comment" << endl; + f << "2, two, 4, 5.5, 7 %% comment" << endl; f << "3, \"three five, six\", 5, 6.5, 8" << endl; f.close(); @@ -1803,15 +1803,15 @@ BOOST_AUTO_TEST_CASE(BadDatasetInfoARFFTest) f << endl; f << "@attribute @@@@flfl numeric" << endl; f << endl; - f << "\\% comment" << endl; + f << "%% comment" << endl; f << "@attribute \"hello world\" string" << endl; f << "@attribute 12345 integer" << endl; f << "@attribute real real" << endl; - f << "@attribute \"blah blah blah \t \" numeric \\% comment" << endl; - f << "\\% comment" << endl; + f << "@attribute \"blah blah blah \t \" numeric %% comment" << endl; + f << "%% comment" << endl; f << "@data" << endl; f << "1, one, 3, 4.5, 6" << endl; - f << "2, two, 4, 5.5, 7 \\% comment" << endl; + f << "2, two, 4, 5.5, 7 %% comment" << endl; f << "3, \"three five, six\", 5, 6.5, 8" << endl; f.close(); From 1579f4cb4e9e5c2ef873325bd01721b64e2d637d Mon Sep 17 00:00:00 2001 From: Marcus Edel Date: Tue, 6 Jun 2017 21:27:47 +0200 Subject: [PATCH 81/84] No need to escape % here. --- src/mlpack/methods/rann/krann_main.cpp | 4 ++-- src/mlpack/tests/load_save_test.cpp | 18 +++++++++--------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/src/mlpack/methods/rann/krann_main.cpp b/src/mlpack/methods/rann/krann_main.cpp index 4c4d55d226..73b570fd5a 100644 --- a/src/mlpack/methods/rann/krann_main.cpp +++ b/src/mlpack/methods/rann/krann_main.cpp @@ -31,10 +31,10 @@ PROGRAM_INFO("K-Rank-Approximate-Nearest-Neighbors (kRANN)", "This program will calculate the k rank-approximate-nearest-neighbors of a " "set of points. You may specify a separate set of reference points and " "query points, or just a reference set which will be used as both the " - "reference and query set. You must specify the rank approximation (in %%) " + "reference and query set. You must specify the rank approximation (in %) " "(and optionally the success probability)." "\n\n" - "For example, the following will return 5 neighbors from the top 0.1%% of " + "For example, the following will return 5 neighbors from the top 0.1% of " "the data (with probability 0.95) for each point in 'input.csv' and store " "the distances in 'distances.csv' and the neighbors in the file " "'neighbors.csv':" diff --git a/src/mlpack/tests/load_save_test.cpp b/src/mlpack/tests/load_save_test.cpp index 5c0006b465..9ffce85b8b 100644 --- a/src/mlpack/tests/load_save_test.cpp +++ b/src/mlpack/tests/load_save_test.cpp @@ -1682,7 +1682,7 @@ BOOST_AUTO_TEST_CASE(SimpleARFFCategoricalTest) f << endl; f << "@attribute three STRING" << endl; f << endl; - f << "%% a comment line " << endl; + f << "% a comment line " << endl; f << endl; f << "@data" << endl; f << "hello, 1, moo" << endl; @@ -1738,15 +1738,15 @@ BOOST_AUTO_TEST_CASE(HarderARFFTest) f << endl; f << "@attribute @@@@flfl numeric" << endl; f << endl; - f << "%% comment" << endl; + f << "% comment" << endl; f << "@attribute \"hello world\" string" << endl; f << "@attribute 12345 integer" << endl; f << "@attribute real real" << endl; - f << "@attribute \"blah blah blah \t \" numeric %% comment" << endl; - f << "%% comment" << endl; + f << "@attribute \"blah blah blah \t \" numeric % comment" << endl; + f << "% comment" << endl; f << "@data" << endl; f << "1, one, 3, 4.5, 6" << endl; - f << "2, two, 4, 5.5, 7 %% comment" << endl; + f << "2, two, 4, 5.5, 7 % comment" << endl; f << "3, \"three five, six\", 5, 6.5, 8" << endl; f.close(); @@ -1803,15 +1803,15 @@ BOOST_AUTO_TEST_CASE(BadDatasetInfoARFFTest) f << endl; f << "@attribute @@@@flfl numeric" << endl; f << endl; - f << "%% comment" << endl; + f << "% comment" << endl; f << "@attribute \"hello world\" string" << endl; f << "@attribute 12345 integer" << endl; f << "@attribute real real" << endl; - f << "@attribute \"blah blah blah \t \" numeric %% comment" << endl; - f << "%% comment" << endl; + f << "@attribute \"blah blah blah \t \" numeric % comment" << endl; + f << "% comment" << endl; f << "@data" << endl; f << "1, one, 3, 4.5, 6" << endl; - f << "2, two, 4, 5.5, 7 %% comment" << endl; + f << "2, two, 4, 5.5, 7 % comment" << endl; f << "3, \"three five, six\", 5, 6.5, 8" << endl; f.close(); From c230f71ea6cf41bac50ca66d24c1320fcfc38be7 Mon Sep 17 00:00:00 2001 From: Marcus Edel Date: Tue, 6 Jun 2017 22:14:55 +0200 Subject: [PATCH 82/84] Fix minor operator() style issue (remove spaces). --- src/mlpack/methods/ann/visitor/copy_visitor.hpp | 2 +- src/mlpack/methods/ann/visitor/forward_visitor_impl.hpp | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/mlpack/methods/ann/visitor/copy_visitor.hpp b/src/mlpack/methods/ann/visitor/copy_visitor.hpp index 75e55e9297..b2894ac0ed 100644 --- a/src/mlpack/methods/ann/visitor/copy_visitor.hpp +++ b/src/mlpack/methods/ann/visitor/copy_visitor.hpp @@ -26,7 +26,7 @@ class CopyVisitor : public boost::static_visitor { public: template - LayerTypes operator () (LayerType*) const; + LayerTypes operator()(LayerType*) const; }; } // namespace ann diff --git a/src/mlpack/methods/ann/visitor/forward_visitor_impl.hpp b/src/mlpack/methods/ann/visitor/forward_visitor_impl.hpp index f676a2400c..ce1c36e368 100644 --- a/src/mlpack/methods/ann/visitor/forward_visitor_impl.hpp +++ b/src/mlpack/methods/ann/visitor/forward_visitor_impl.hpp @@ -20,8 +20,8 @@ namespace ann { //! ForwardVisitor visitor class. inline ForwardVisitor::ForwardVisitor(arma::mat&& input, arma::mat&& output) : - input(std::move(input)), - output(std::move(output)) + input(std::move(input)), + output(std::move(output)) { /* Nothing to do here. */ } From 399116c07abaffb559fd72698d2bf4ab39460e78 Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Wed, 7 Jun 2017 17:16:31 -0400 Subject: [PATCH 83/84] Update HISTORY. --- HISTORY.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/HISTORY.md b/HISTORY.md index f03b8ac8bb..d3d4eceebe 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,6 +1,10 @@ ### mlpack ?.?.? ###### ????-??-?? +### mlpack 2.2.3 +###### 2017-05-24 + * Bug fix for --predictions_file in mlpack_decision_tree program. + ### mlpack 2.2.2 ###### 2017-05-04 * Install backwards-compatibility mlpack_allknn and mlpack_allkfn programs; From 61719441c832708c73d6694591695b55a3e59e84 Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Wed, 7 Jun 2017 17:47:37 -0400 Subject: [PATCH 84/84] Add subdirectories that were missing. --- src/mlpack/core/data/CMakeLists.txt | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/mlpack/core/data/CMakeLists.txt b/src/mlpack/core/data/CMakeLists.txt index 510282551a..95afbc1049 100644 --- a/src/mlpack/core/data/CMakeLists.txt +++ b/src/mlpack/core/data/CMakeLists.txt @@ -30,6 +30,10 @@ foreach(file ${SOURCES}) set(DIR_SRCS ${DIR_SRCS} ${CMAKE_CURRENT_SOURCE_DIR}/${file}) endforeach() +# Add subdirectories. +add_subdirectory(imputation_methods) +add_subdirectory(map_policies) + # Append sources (with directory name) to list of all mlpack sources (used at # parent scope). set(MLPACK_SRCS ${MLPACK_SRCS} ${DIR_SRCS} PARENT_SCOPE)