diff --git a/.travis.yml b/.travis.yml index 9167a7c260..1adac031b1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -21,4 +21,7 @@ notifications: email: - mlpack-git@lists.mlpack.org irc: - - "chat.freenode.net#mlpack" + channels: + - "chat.freenode.net#mlpack" + on_success: change + on_failure: always diff --git a/CMakeLists.txt b/CMakeLists.txt index ea4f726e05..d8dbf1a339 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -319,8 +319,8 @@ endif () # some reason. include(CMake/TargetDistclean.cmake OPTIONAL) -include_directories(${CMAKE_SOURCE_DIR}) -include_directories(${MLPACK_INCLUDE_DIRS}) +include_directories(BEFORE ${MLPACK_INCLUDE_DIRS}) +include_directories(BEFORE ${CMAKE_SOURCE_DIR}/src/) # On Windows, things end up under Debug/ or Release/. if (WIN32) diff --git a/HISTORY.md b/HISTORY.md index f03b8ac8bb..d3d4eceebe 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,6 +1,10 @@ ### mlpack ?.?.? ###### ????-??-?? +### mlpack 2.2.3 +###### 2017-05-24 + * Bug fix for --predictions_file in mlpack_decision_tree program. + ### mlpack 2.2.2 ###### 2017-05-04 * Install backwards-compatibility mlpack_allknn and mlpack_allkfn programs; diff --git a/README.md b/README.md index 3561e74171..ed41237db4 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,7 @@

Download: - current stable version (2.2.2) + current stable version (2.2.3)

diff --git a/doc/guide/build.hpp b/doc/guide/build.hpp index d175747574..ceab1b33fe 100644 --- a/doc/guide/build.hpp +++ b/doc/guide/build.hpp @@ -23,14 +23,14 @@ href="https://keon.io/mlpack/mlpack-on-windows/">Keon's excellent tutorial. @section Download latest mlpack build Download latest mlpack build from here: -mlpack-2.2.2 +mlpack-2.2.3 @section builddir Creating Build Directory Once the mlpack source is unpacked, you should create a build directory. @code -$ cd mlpack-2.2.2 +$ cd mlpack-2.2.3 $ mkdir build @endcode diff --git a/src/mlpack/CMakeLists.txt b/src/mlpack/CMakeLists.txt index 1c1d28fed6..d0cbbfad83 100644 --- a/src/mlpack/CMakeLists.txt +++ b/src/mlpack/CMakeLists.txt @@ -1,4 +1,3 @@ -include_directories(..) # include_directories(${CMAKE_CURRENT_BINARY_DIR}/..) # mlpack/mlpack_export.hpp # Add core.hpp to list of sources. diff --git a/src/mlpack/core.hpp b/src/mlpack/core.hpp index 20faeb740c..f9a9ea74a2 100644 --- a/src/mlpack/core.hpp +++ b/src/mlpack/core.hpp @@ -244,7 +244,8 @@ #include #include #include -//mlpack::backtrace only for linux + +// mlpack::backtrace only for linux #ifdef HAS_BFD_DL #include #endif diff --git a/src/mlpack/core/CMakeLists.txt b/src/mlpack/core/CMakeLists.txt index d8a49bb99e..6d9194cc2f 100644 --- a/src/mlpack/core/CMakeLists.txt +++ b/src/mlpack/core/CMakeLists.txt @@ -2,6 +2,7 @@ set(DIRS arma_extend boost_backport + cv data dists kernels diff --git a/src/mlpack/core/cv/CMakeLists.txt b/src/mlpack/core/cv/CMakeLists.txt new file mode 100644 index 0000000000..521e43236d --- /dev/null +++ b/src/mlpack/core/cv/CMakeLists.txt @@ -0,0 +1,15 @@ +add_subdirectory(metrics) + +# Define the files we need to compile +# Anything not in this list will not be compiled into mlpack. +set(SOURCES +) + +# Add directory name to sources. +set(DIR_SRCS) +foreach(file ${SOURCES}) + set(DIR_SRCS ${DIR_SRCS} ${CMAKE_CURRENT_SOURCE_DIR}/${file}) +endforeach() +# Append sources (with directory name) to list of all mlpack sources (used at +# the parent scope). +set(MLPACK_SRCS ${MLPACK_SRCS} ${DIR_SRCS} PARENT_SCOPE) diff --git a/src/mlpack/core/cv/metrics/CMakeLists.txt b/src/mlpack/core/cv/metrics/CMakeLists.txt new file mode 100644 index 0000000000..10445a65e6 --- /dev/null +++ b/src/mlpack/core/cv/metrics/CMakeLists.txt @@ -0,0 +1,17 @@ +# Define the files we need to compile +# Anything not in this list will not be compiled into mlpack. +set(SOURCES + accuracy.hpp + accuracy_impl.hpp + mse.hpp + mse_impl.hpp +) + +# Add directory name to sources. +set(DIR_SRCS) +foreach(file ${SOURCES}) + set(DIR_SRCS ${DIR_SRCS} ${CMAKE_CURRENT_SOURCE_DIR}/${file}) +endforeach() +# Append sources (with directory name) to list of all mlpack sources (used at +# the parent scope). +set(MLPACK_SRCS ${MLPACK_SRCS} ${DIR_SRCS} PARENT_SCOPE) diff --git a/src/mlpack/core/cv/metrics/accuracy.hpp b/src/mlpack/core/cv/metrics/accuracy.hpp new file mode 100644 index 0000000000..aff7c5acb5 --- /dev/null +++ b/src/mlpack/core/cv/metrics/accuracy.hpp @@ -0,0 +1,53 @@ +/** + * @file accuracy.hpp + * @author Kirill Mishchenko + * + * The accuracy metric. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_CORE_CV_METRICS_ACCURACY_HPP +#define MLPACK_CORE_CV_METRICS_ACCURACY_HPP + +#include + +namespace mlpack { +namespace cv { + +/** + * The Accuracy is a metric of performance for classification algorithms that is + * equal to a proportion of correctly labeled test items among all ones for + * given test items. + */ +class Accuracy +{ + public: + /** + * Run classification and calculate accuracy. + * + * @param model A classification model. + * @param data Column-major data containing test items. + * @param labels Ground truth (correct) labels for the test items. + */ + template + static double Evaluate(MLAlgorithm& model, + const DataType& data, + const arma::Row& labels); + + /** + * Information for hyper-parameter tuning code. It indicates that we want + * to maximize the metric. + */ + static const bool NeedsMinimization = false; +}; + +} // namespace cv +} // namespace mlpack + +// Include implementation. +#include "accuracy_impl.hpp" + +#endif diff --git a/src/mlpack/core/cv/metrics/accuracy_impl.hpp b/src/mlpack/core/cv/metrics/accuracy_impl.hpp new file mode 100644 index 0000000000..7b46922a79 --- /dev/null +++ b/src/mlpack/core/cv/metrics/accuracy_impl.hpp @@ -0,0 +1,42 @@ +/** + * @file accuracy_impl.hpp + * @author Kirill Mishchenko + * + * The implementation of the class Accuracy. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_CORE_CV_METRICS_ACCURACY_IMPL_HPP +#define MLPACK_CORE_CV_METRICS_ACCURACY_IMPL_HPP + +namespace mlpack { +namespace cv { + +template +double Accuracy::Evaluate(MLAlgorithm& model, + const DataType& data, + const arma::Row& labels) +{ + if (data.n_cols != labels.n_elem) + { + std::ostringstream oss; + oss << "Accuracy::Evaluate(): number of points (" << data.n_cols << ") " + << "does not match number of labels (" << labels.n_elem << ")!" + << std::endl; + throw std::invalid_argument(oss.str()); + } + + arma::Row predictedLabels; + model.Classify(data, predictedLabels); + size_t amountOfCorrectPredictions = arma::sum(predictedLabels == labels); + + return (double) amountOfCorrectPredictions / labels.n_elem; +} + +} // namespace cv +} // namespace mlpack + +#endif diff --git a/src/mlpack/core/cv/metrics/mse.hpp b/src/mlpack/core/cv/metrics/mse.hpp new file mode 100644 index 0000000000..b913496f86 --- /dev/null +++ b/src/mlpack/core/cv/metrics/mse.hpp @@ -0,0 +1,54 @@ +/** + * @file mse.hpp + * @author Kirill Mishchenko + * + * The mean squared error (MSE). + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_CORE_CV_METRICS_MSE_HPP +#define MLPACK_CORE_CV_METRICS_MSE_HPP + +#include + +namespace mlpack { +namespace cv { + +/** + * The MeanSquaredError is a metric of performance for regression algorithms + * that is equal to the mean squared error between predicted values and ground + * truth (correct) values for given test items. + */ +class MSE +{ + public: + /** + * Run prediction and calculate the mean squared error. + * + * @param model A regression model. + * @param data Column-major data containing test items. + * @param responses Ground truth (correct) target values for the test items, + * should be either a row vector or a column-major matrix. + */ + template + static double Evaluate(MLAlgorithm& model, + const DataType& data, + const ResponsesType& responses); + + /** + * Information for hyper-parameter tuning code. It indicates that we want + * to minimize the measurement. + */ + static const bool NeedsMinimization = true; +}; + +} // namespace cv +} // namespace mlpack + +// Include implementation. +#include "mse_impl.hpp" + +#endif diff --git a/src/mlpack/core/cv/metrics/mse_impl.hpp b/src/mlpack/core/cv/metrics/mse_impl.hpp new file mode 100644 index 0000000000..92180e2b83 --- /dev/null +++ b/src/mlpack/core/cv/metrics/mse_impl.hpp @@ -0,0 +1,42 @@ +/** + * @file mse_impl.hpp + * @author Kirill Mishchenko + * + * The implementation of the class MSE. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_CORE_CV_METRICS_MSE_IMPL_HPP +#define MLPACK_CORE_CV_METRICS_MSE_IMPL_HPP + +namespace mlpack { +namespace cv { + +template +double MSE::Evaluate(MLAlgorithm& model, + const DataType& data, + const ResponsesType& responses) +{ + if (data.n_cols != responses.n_cols) + { + std::ostringstream oss; + oss << "MSE::Evaluate(): number of points (" << data.n_cols << ") " + << "does not match number of responses (" << responses.n_cols << ")!" + << std::endl; + throw std::invalid_argument(oss.str()); + } + + ResponsesType predictedResponses; + model.Predict(data, predictedResponses); + double sum = arma::accu(arma::square(responses - predictedResponses)); + + return sum / responses.n_elem; +} + +} // namespace cv +} // namespace mlpack + +#endif diff --git a/src/mlpack/core/data/CMakeLists.txt b/src/mlpack/core/data/CMakeLists.txt index 510282551a..95afbc1049 100644 --- a/src/mlpack/core/data/CMakeLists.txt +++ b/src/mlpack/core/data/CMakeLists.txt @@ -30,6 +30,10 @@ foreach(file ${SOURCES}) set(DIR_SRCS ${DIR_SRCS} ${CMAKE_CURRENT_SOURCE_DIR}/${file}) endforeach() +# Add subdirectories. +add_subdirectory(imputation_methods) +add_subdirectory(map_policies) + # Append sources (with directory name) to list of all mlpack sources (used at # parent scope). set(MLPACK_SRCS ${MLPACK_SRCS} ${DIR_SRCS} PARENT_SCOPE) diff --git a/src/mlpack/core/data/imputer.hpp b/src/mlpack/core/data/imputer.hpp index a5dddfe539..afd7a9095c 100644 --- a/src/mlpack/core/data/imputer.hpp +++ b/src/mlpack/core/data/imputer.hpp @@ -86,7 +86,6 @@ class Imputer // save columnMajor as a member variable since it is rarely changed. bool columnMajor; - }; // class Imputer } // namespace data diff --git a/src/mlpack/core/data/load_arff_impl.hpp b/src/mlpack/core/data/load_arff_impl.hpp index 81e2df0944..578e07b048 100644 --- a/src/mlpack/core/data/load_arff_impl.hpp +++ b/src/mlpack/core/data/load_arff_impl.hpp @@ -49,8 +49,8 @@ void LoadARFF(const std::string& filename, if (line[0] == '@') { typedef boost::tokenizer> Tokenizer; - std::string separators = " \t\%"; // Split on comments too. - boost::escaped_list_separator sep("\\", separators, "\"{"); + std::string separators = " \t%"; // Split on comments too. + boost::escaped_list_separator sep("\\", separators, "{\""); Tokenizer tok(line, sep); Tokenizer::iterator it = tok.begin(); @@ -182,7 +182,8 @@ void LoadARFF(const std::string& filename, // Strip spaces before mapping. std::string token = *it; boost::trim(token); - matrix(col, row) = info.template MapString(token, col); // We load transposed. + // We load transposed. + matrix(col, row) = info.template MapString(token, col); } else if (info.Type(col) == Datatype::numeric) { diff --git a/src/mlpack/core/data/load_csv.hpp b/src/mlpack/core/data/load_csv.hpp index 6cab5c153f..87c722fdf7 100644 --- a/src/mlpack/core/data/load_csv.hpp +++ b/src/mlpack/core/data/load_csv.hpp @@ -204,7 +204,7 @@ class LoadCSV } } -private: + private: using iter_type = boost::iterator_range; /** @@ -256,8 +256,8 @@ private: // Remove whitespace from either side. boost::trim(line); - //parse the numbers from a line(ex : 1,2,3,4), if the parser find the - //number it will execute the setNum function + // Parse the numbers from a line (ex: 1,2,3,4); if the parser finds a + // number it will execute the setNum function. const bool canParse = qi::parse(line.begin(), line.end(), stringRule[setCharClass] % delimiterRule); diff --git a/src/mlpack/core/data/load_impl.hpp b/src/mlpack/core/data/load_impl.hpp index 9f1854af07..31a088f4ae 100644 --- a/src/mlpack/core/data/load_impl.hpp +++ b/src/mlpack/core/data/load_impl.hpp @@ -55,13 +55,13 @@ void TransposeTokens(std::vector> const &input, size_t index) { output.clear(); - for(size_t i = 0; i != input.size(); ++i) + for (size_t i = 0; i != input.size(); ++i) { output.emplace_back(input[i][index]); } } -} //namespace details +} // namespace details template bool inline inplace_transpose(arma::Mat& X) @@ -171,7 +171,7 @@ bool Load(const std::string& filename, // This is taken from load_auto_detect() in diskio_meat.hpp const std::string ARMA_MAT_TXT = "ARMA_MAT_TXT"; - //char* rawHeader = new char[ARMA_MAT_TXT.length() + 1]; + // char* rawHeader = new char[ARMA_MAT_TXT.length() + 1]; std::string rawHeader(ARMA_MAT_TXT.length(), '\0'); std::streampos pos = stream.tellg(); diff --git a/src/mlpack/core/data/serialization_template_version.hpp b/src/mlpack/core/data/serialization_template_version.hpp index 78e1c5c1da..e671eb0fe0 100644 --- a/src/mlpack/core/data/serialization_template_version.hpp +++ b/src/mlpack/core/data/serialization_template_version.hpp @@ -29,14 +29,10 @@ struct version> \ typedef mpl::int_ type; \ typedef mpl::integral_c_tag tag; \ BOOST_STATIC_CONSTANT(int, value = version::type::value); \ - BOOST_MPL_ASSERT(( \ - boost::mpl::less< \ - boost::mpl::int_, \ - boost::mpl::int_<256> \ - > \ - )); \ + BOOST_MPL_ASSERT((boost::mpl::less, \ + boost::mpl::int_<256>>)); \ }; \ -} \ -} +} /* namespace serialization */ \ +} /* namespace boost */ #endif diff --git a/src/mlpack/core/data/split_data.hpp b/src/mlpack/core/data/split_data.hpp index 37d37ba2ce..28b0bcf966 100644 --- a/src/mlpack/core/data/split_data.hpp +++ b/src/mlpack/core/data/split_data.hpp @@ -143,7 +143,7 @@ void Split(const arma::Mat& input, * @return std::tuple containing trainData (arma::Mat), testData * (arma::Mat), trainLabel (arma::Row), and testLabel (arma::Row). */ -template +template std::tuple, arma::Mat, arma::Row, arma::Row> Split(const arma::Mat& input, const arma::Row& inputLabel, diff --git a/src/mlpack/core/dists/discrete_distribution.cpp b/src/mlpack/core/dists/discrete_distribution.cpp index 9866a126bc..c2f7e554d1 100644 --- a/src/mlpack/core/dists/discrete_distribution.cpp +++ b/src/mlpack/core/dists/discrete_distribution.cpp @@ -128,9 +128,9 @@ void DiscreteDistribution::Train(const arma::mat& observations, { for (size_t i = 0; i < dimensions; i++) { - // Add the probability of each observation. The addition of 0.5 to the - // observation is to turn the default flooring operation of the size_t cast - // into a rounding observation. + // Add the probability of each observation. The addition of 0.5 + // to the observation is to turn the default flooring operation + // of the size_t cast into a rounding observation. const size_t obs = size_t(observations(i, r) + 0.5); // Ensure that the observation is within the bounds. diff --git a/src/mlpack/core/dists/discrete_distribution.hpp b/src/mlpack/core/dists/discrete_distribution.hpp index 523765113e..60876f257b 100644 --- a/src/mlpack/core/dists/discrete_distribution.hpp +++ b/src/mlpack/core/dists/discrete_distribution.hpp @@ -129,11 +129,11 @@ class DiscreteDistribution // Ensure the observation has the same dimension with the probabilities if (observation.n_elem != probabilities.size()) { - Log::Debug << "the obversation must has the same dimension with the probabilities" - << "the observation's dimension is" << observation.n_elem << "but the dimension of " - << "probabilities is" << probabilities.size() << std::endl; - return probability; + Log::Fatal << "DiscreteDistribution::Probability(): observation has " + << "incorrect dimension " << observation.n_elem << " but should have " + << "dimension " << probabilities.size() << "!" << std::endl; } + for (size_t dimension = 0; dimension < observation.n_elem; dimension++) { // Adding 0.5 helps ensure that we cast the floating point to a size_t @@ -143,9 +143,10 @@ class DiscreteDistribution // Ensure that the observation is within the bounds. if (obs >= probabilities[dimension].n_elem) { - Log::Debug << "DiscreteDistribution::Probability(): received observation " - << obs << "; observation must be in [0, " << probabilities[dimension].n_elem - << "] for this distribution." << std::endl; + Log::Fatal << "DiscreteDistribution::Probability(): received " + << "observation " << obs << "; observation must be in [0, " + << probabilities[dimension].n_elem << "] for this distribution." + << std::endl; } probability *= probabilities[dimension][obs]; } diff --git a/src/mlpack/core/dists/gamma_distribution.cpp b/src/mlpack/core/dists/gamma_distribution.cpp index a8da1a28c6..b47b53b74d 100644 --- a/src/mlpack/core/dists/gamma_distribution.cpp +++ b/src/mlpack/core/dists/gamma_distribution.cpp @@ -64,7 +64,7 @@ void GammaDistribution::Train(const arma::mat& rdata, const double tol) Train(logMeanxVec, meanLogxVec, meanxVec, tol); } -//Fits an alpha and beta parameter according to observation probabilities. +// Fits an alpha and beta parameter according to observation probabilities. void GammaDistribution::Train(const arma::mat& rdata, const arma::vec& probabilities, const double tol) @@ -151,7 +151,6 @@ void GammaDistribution::Train(const arma::vec& logMeanxVec, if (aEst <= 0) throw std::logic_error("GammaDistribution::Train(): estimated invalid " "negative value for parameter alpha!"); - } while (!Converged(aEst, aOld, tol)); alpha(row) = aEst; @@ -219,7 +218,7 @@ void GammaDistribution::LogProbability(const arma::mat& observations, double factor = std::exp(-observations(d, i) / beta(d)); double numerator = std::pow(observations(d, i), alpha(d) - 1); - LogProbabilities(i) += std::log( numerator * factor / denominators(d)); + LogProbabilities(i) += std::log(numerator * factor / denominators(d)); } } } diff --git a/src/mlpack/core/dists/gamma_distribution.hpp b/src/mlpack/core/dists/gamma_distribution.hpp index eadf78a55a..b4d7c6e639 100644 --- a/src/mlpack/core/dists/gamma_distribution.hpp +++ b/src/mlpack/core/dists/gamma_distribution.hpp @@ -50,7 +50,7 @@ namespace distribution { */ class GammaDistribution { - public: + public: /** * Construct the Gamma distribution with the given number of dimensions * (default 0); each parameter will be initialized to 0. @@ -80,7 +80,7 @@ class GammaDistribution /** * Destructor. */ - ~GammaDistribution() {}; + ~GammaDistribution() {} /** * This function trains (fits distribution parameters) to new data or the @@ -192,7 +192,7 @@ class GammaDistribution //! Get the dimensionality of the distribution. size_t Dimensionality() const { return alpha.n_elem; } - private: + private: //! Array of fitted alphas. arma::vec alpha; //! Array of fitted betas. @@ -214,7 +214,7 @@ class GammaDistribution const double tol); }; -} // namespace distributions. -} // namespace mlpack. +} // namespace distribution +} // namespace mlpack #endif diff --git a/src/mlpack/core/dists/gaussian_distribution.hpp b/src/mlpack/core/dists/gaussian_distribution.hpp index d817f40ccf..c680ca076a 100644 --- a/src/mlpack/core/dists/gaussian_distribution.hpp +++ b/src/mlpack/core/dists/gaussian_distribution.hpp @@ -174,8 +174,9 @@ class GaussianDistribution * @param x List of observations. * @param probabilities Output log probabilities for each input observation. */ -inline void GaussianDistribution::LogProbability(const arma::mat& x, - arma::vec& logProbabilities) const +inline void GaussianDistribution::LogProbability( + const arma::mat& x, + arma::vec& logProbabilities) const { // Column i of 'diffs' is the difference between x.col(i) and the mean. arma::mat diffs = x - (mean * arma::ones(x.n_cols)); diff --git a/src/mlpack/core/dists/laplace_distribution.cpp b/src/mlpack/core/dists/laplace_distribution.cpp index 2299ed4078..75a069bbe7 100644 --- a/src/mlpack/core/dists/laplace_distribution.cpp +++ b/src/mlpack/core/dists/laplace_distribution.cpp @@ -21,7 +21,8 @@ using namespace mlpack::distribution; */ double LaplaceDistribution::LogProbability(const arma::vec& observation) const { - // Evaluate the PDF of the Laplace distribution to determine the log probability. + // Evaluate the PDF of the Laplace distribution to determine + // the log probability. return -log(2. * scale) - arma::norm(observation - mean, 2) / scale; } diff --git a/src/mlpack/core/dists/laplace_distribution.hpp b/src/mlpack/core/dists/laplace_distribution.hpp index e7a59aa426..c49cd77b35 100644 --- a/src/mlpack/core/dists/laplace_distribution.hpp +++ b/src/mlpack/core/dists/laplace_distribution.hpp @@ -155,7 +155,6 @@ class LaplaceDistribution arma::vec mean; //! Scale parameter of the distribution. double scale; - }; } // namespace distribution diff --git a/src/mlpack/core/dists/regression_distribution.cpp b/src/mlpack/core/dists/regression_distribution.cpp index e306bc8096..50d186b838 100644 --- a/src/mlpack/core/dists/regression_distribution.cpp +++ b/src/mlpack/core/dists/regression_distribution.cpp @@ -23,11 +23,11 @@ using namespace mlpack::distribution; void RegressionDistribution::Train(const arma::mat& observations) { regression::LinearRegression lr(observations.rows(1, observations.n_rows - 1), - (observations.row(0)).t(), 0, true); + arma::rowvec(observations.row(0)), 0, true); rf = lr; - arma::vec fitted; + arma::rowvec fitted; lr.Predict(observations.rows(1, observations.n_rows - 1), fitted); - err.Train(observations.row(0) - fitted.t()); + err.Train(observations.row(0) - fitted); } /** @@ -37,13 +37,19 @@ void RegressionDistribution::Train(const arma::mat& observations) */ void RegressionDistribution::Train(const arma::mat& observations, const arma::vec& weights) +{ + Train(observations, arma::rowvec(weights.t())); +} + +void RegressionDistribution::Train(const arma::mat& observations, + const arma::rowvec& weights) { regression::LinearRegression lr(observations.rows(1, observations.n_rows - 1), - (observations.row(0)).t(), 0, true, weights); + arma::rowvec(observations.row(0)), weights, 0, true); rf = lr; - arma::vec fitted; + arma::rowvec fitted; lr.Predict(observations.rows(1, observations.n_rows - 1), fitted); - err.Train(observations.row(0) - fitted.t(), weights); + err.Train(observations.row(0) - fitted, weights.t()); } /** @@ -53,13 +59,21 @@ void RegressionDistribution::Train(const arma::mat& observations, */ double RegressionDistribution::Probability(const arma::vec& observation) const { - arma::vec fitted; + arma::rowvec fitted; rf.Predict(observation.rows(1, observation.n_rows-1), fitted); - return err.Probability(observation(0)-fitted); + return err.Probability(observation(0)-fitted.t()); } void RegressionDistribution::Predict(const arma::mat& points, arma::vec& predictions) const +{ + arma::rowvec rowPredictions; + Predict(points, rowPredictions); + predictions = rowPredictions.t(); +} + +void RegressionDistribution::Predict(const arma::mat& points, + arma::rowvec& predictions) const { rf.Predict(points, predictions); } diff --git a/src/mlpack/core/dists/regression_distribution.hpp b/src/mlpack/core/dists/regression_distribution.hpp index d38a2a21e2..13b299e0e5 100644 --- a/src/mlpack/core/dists/regression_distribution.hpp +++ b/src/mlpack/core/dists/regression_distribution.hpp @@ -41,6 +41,18 @@ class RegressionDistribution */ RegressionDistribution() { /* nothing to do */ } + /** + * Create a Conditional Gaussian distribution with conditional mean function + * obtained by running RegressionFunction on predictors, responses. + * + * @param predictors Matrix of predictors (X). + * @param responses Vector of responses (y). + */ + mlpack_deprecated RegressionDistribution(const arma::mat& predictors, + const arma::vec& responses) : + RegressionDistribution(predictors, arma::rowvec(responses.t())) + {} + /** * Create a Conditional Gaussian distribution with conditional mean function * obtained by running RegressionFunction on predictors, responses. @@ -49,9 +61,9 @@ class RegressionDistribution * @param responses Vector of responses (y). */ RegressionDistribution(const arma::mat& predictors, - const arma::vec& responses) : - rf(regression::LinearRegression(predictors, responses)) + const arma::rowvec& responses) { + rf.Train(predictors, responses); err = GaussianDistribution(1); arma::mat cov(1, 1); cov(0, 0) = rf.ComputeError(predictors, responses); @@ -90,7 +102,15 @@ class RegressionDistribution * * @param weights probability that given observation is from distribution */ - void Train(const arma::mat& observations, const arma::vec& weights); + mlpack_deprecated void Train(const arma::mat& observations, + const arma::vec& weights); + + /** + * Estimate parameters using provided observation weights + * + * @param weights probability that given observation is from distribution + */ + void Train(const arma::mat& observations, const arma::rowvec& weights); /** * Evaluate probability density function of given observation @@ -114,7 +134,16 @@ class RegressionDistribution * @param points the data points to calculate with. * @param predictions y, will contain calculated values on completion. */ - void Predict(const arma::mat& points, arma::vec& predictions) const; + mlpack_deprecated void Predict(const arma::mat& points, + arma::vec& predictions) const; + + /** + * Calculate y_i for each data point in points. + * + * @param points the data points to calculate with. + * @param predictions y, will contain calculated values on completion. + */ + void Predict(const arma::mat& points, arma::rowvec& predictions) const; //! Return the parameters (the b vector). const arma::vec& Parameters() const { return rf.Parameters(); } diff --git a/src/mlpack/core/kernels/epanechnikov_kernel.hpp b/src/mlpack/core/kernels/epanechnikov_kernel.hpp index 07a5eae443..5629bbfe46 100644 --- a/src/mlpack/core/kernels/epanechnikov_kernel.hpp +++ b/src/mlpack/core/kernels/epanechnikov_kernel.hpp @@ -100,7 +100,6 @@ class EpanechnikovKernel double bandwidth; //! Cached value of the inverse bandwidth squared (to speed up computation). double inverseBandwidthSquared; - }; //! Kernel traits for the Epanechnikov kernel. diff --git a/src/mlpack/core/kernels/example_kernel.hpp b/src/mlpack/core/kernels/example_kernel.hpp index 4272535b72..0589b6e901 100644 --- a/src/mlpack/core/kernels/example_kernel.hpp +++ b/src/mlpack/core/kernels/example_kernel.hpp @@ -140,7 +140,7 @@ class ExampleKernel static double Normalizer() { return 0; } // Modified to remove unused variable "dimension" - //static double Normalizer(size_t dimension=1) { return 0; } + // static double Normalizer(size_t dimension=1) { return 0; } }; } // namespace kernel diff --git a/src/mlpack/core/kernels/gaussian_kernel.hpp b/src/mlpack/core/kernels/gaussian_kernel.hpp index 791cea1b66..4d3446cc7c 100644 --- a/src/mlpack/core/kernels/gaussian_kernel.hpp +++ b/src/mlpack/core/kernels/gaussian_kernel.hpp @@ -126,8 +126,8 @@ class GaussianKernel template double ConvolutionIntegral(const VecTypeA& a, const VecTypeB& b) { - return Evaluate(sqrt(metric::SquaredEuclideanDistance::Evaluate(a, b) / 2.0)) / - (Normalizer(a.n_rows) * pow(2.0, (double) a.n_rows / 2.0)); + return Evaluate(sqrt(metric::SquaredEuclideanDistance::Evaluate(a, b) / + 2.0)) / (Normalizer(a.n_rows) * pow(2.0, (double) a.n_rows / 2.0)); } diff --git a/src/mlpack/core/kernels/spherical_kernel.hpp b/src/mlpack/core/kernels/spherical_kernel.hpp index 962eee6c47..e138fb1648 100644 --- a/src/mlpack/core/kernels/spherical_kernel.hpp +++ b/src/mlpack/core/kernels/spherical_kernel.hpp @@ -68,7 +68,7 @@ class SphericalKernel } double volumeSquared = pow(Normalizer(a.n_rows), 2.0); - switch(a.n_rows) + switch (a.n_rows) { case 1: return 1.0 / volumeSquared * (2.0 * bandwidth - distance); diff --git a/src/mlpack/core/math/lin_alg.cpp b/src/mlpack/core/math/lin_alg.cpp index af7fa9123c..915d1a36a9 100644 --- a/src/mlpack/core/math/lin_alg.cpp +++ b/src/mlpack/core/math/lin_alg.cpp @@ -257,7 +257,9 @@ void mlpack::math::Svec(const arma::sp_mat& input, arma::sp_vec& output) void mlpack::math::Smat(const arma::vec& input, arma::mat& output) { - const size_t n = static_cast(ceil((-1. + sqrt(1. + 8. * input.n_elem))/2.)); + const size_t n = static_cast + (ceil((-1. + sqrt(1. + 8. * input.n_elem))/2.)); + output.zeros(n, n); diff --git a/src/mlpack/core/math/random_basis.cpp b/src/mlpack/core/math/random_basis.cpp index bf4e412aa8..7419908ead 100644 --- a/src/mlpack/core/math/random_basis.cpp +++ b/src/mlpack/core/math/random_basis.cpp @@ -18,7 +18,7 @@ namespace math { void RandomBasis(mat& basis, const size_t d) { - while(true) + while (true) { // [Q, R] = qr(randn(d, d)); // Q = Q * diag(sign(diag(R))); diff --git a/src/mlpack/core/metrics/mahalanobis_distance.hpp b/src/mlpack/core/metrics/mahalanobis_distance.hpp index 4ffe453039..343a4b5d2f 100644 --- a/src/mlpack/core/metrics/mahalanobis_distance.hpp +++ b/src/mlpack/core/metrics/mahalanobis_distance.hpp @@ -111,7 +111,7 @@ class MahalanobisDistance arma::mat covariance; }; -} // namespace distance +} // namespace metric } // namespace mlpack #include "mahalanobis_distance_impl.hpp" diff --git a/src/mlpack/core/optimizers/aug_lagrangian/aug_lagrangian_test_functions.hpp b/src/mlpack/core/optimizers/aug_lagrangian/aug_lagrangian_test_functions.hpp index 645feecb5b..9e9e4a028b 100644 --- a/src/mlpack/core/optimizers/aug_lagrangian/aug_lagrangian_test_functions.hpp +++ b/src/mlpack/core/optimizers/aug_lagrangian/aug_lagrangian_test_functions.hpp @@ -67,7 +67,7 @@ class GockenbachFunction double Evaluate(const arma::mat& coordinates); void Gradient(const arma::mat& coordinates, arma::mat& gradient); - size_t NumConstraints() const { return 2; }; + size_t NumConstraints() const { return 2; } double EvaluateConstraint(const size_t index, const arma::mat& coordinates); void GradientConstraint(const size_t index, diff --git a/src/mlpack/core/optimizers/lbfgs/lbfgs_impl.hpp b/src/mlpack/core/optimizers/lbfgs/lbfgs_impl.hpp index 38fa92bd5b..954e4c981e 100644 --- a/src/mlpack/core/optimizers/lbfgs/lbfgs_impl.hpp +++ b/src/mlpack/core/optimizers/lbfgs/lbfgs_impl.hpp @@ -386,7 +386,8 @@ double L_BFGS::Optimize(arma::mat& iterate) function.Evaluate(iterate) << ", gradient norm " << arma::norm(gradient, 2) << ", " << ((prevFunctionValue - functionValue) / - std::max(std::max(fabs(prevFunctionValue), fabs(functionValue)), 1.0)) << "." << std::endl; + std::max(std::max(fabs(prevFunctionValue), fabs(functionValue)), 1.0)) + << "." << std::endl; prevFunctionValue = functionValue; @@ -452,7 +453,6 @@ double L_BFGS::Optimize(arma::mat& iterate) // Overwrite an old basis set. UpdateBasisSet(itNum, iterate, oldIterate, gradient, oldGradient); - } // End of the optimization loop. return function.Evaluate(iterate); diff --git a/src/mlpack/core/optimizers/minibatch_sgd/decay_policies/CMakeLists.txt b/src/mlpack/core/optimizers/minibatch_sgd/decay_policies/CMakeLists.txt new file mode 100644 index 0000000000..740bdf77ce --- /dev/null +++ b/src/mlpack/core/optimizers/minibatch_sgd/decay_policies/CMakeLists.txt @@ -0,0 +1,10 @@ +set(SOURCES + no_decay.hpp +) + +set(DIR_SRCS) +foreach(file ${SOURCES}) + set(DIR_SRCS ${DIR_SRCS} ${CMAKE_CURRENT_SOURCE_DIR}/${file}) +endforeach() + +set(MLPACK_SRCS ${MLPACK_SRCS} ${DIR_SRCS} PARENT_SCOPE) diff --git a/src/mlpack/core/optimizers/minibatch_sgd/decay_policies/no_decay.hpp b/src/mlpack/core/optimizers/minibatch_sgd/decay_policies/no_decay.hpp new file mode 100644 index 0000000000..44f77a8ee5 --- /dev/null +++ b/src/mlpack/core/optimizers/minibatch_sgd/decay_policies/no_decay.hpp @@ -0,0 +1,50 @@ +/** + * @file no_decay.hpp + * @author Marcus Edel + * + * Definition of the policy type for the decay class. + * + * You should define your own decay update that looks like NoDecay. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ + +#ifndef MLPACK_CORE_OPTIMIZERS_MINIBATCH_SGD_DECAY_POLICIES_NO_DECAY_HPP +#define MLPACK_CORE_OPTIMIZERS_MINIBATCH_SGD_DECAY_POLICIES_NO_DECAY_HPP + +namespace mlpack { +namespace optimization { + +/** + * Definition of the NoDecay class. Use this as a template for your own. + */ +class NoDecay +{ + public: + /** + * This constructor is called before the first iteration. + */ + NoDecay() { } + + /** + * This function is called in each iteration after the policy update. + * + * @param iterate Parameters that minimize the function. + * @param stepSize Step size to be used for the given iteration. + * @param gradient The gradient matrix. + */ + void Update(arma::mat& /* iterate */, + double& /* stepSize */, + const arma::mat& /* gradient */) + { + // Nothing to do here. + } +}; + +} // namespace optimization +} // namespace mlpack + +#endif // MLPACK_CORE_OPTIMIZERS_MINIBATCH_SGD_DECAY_POLICIES_NO_DECAY_HPP diff --git a/src/mlpack/core/optimizers/minibatch_sgd/minibatch_sgd.hpp b/src/mlpack/core/optimizers/minibatch_sgd/minibatch_sgd.hpp index e562b4c6fd..1eba57f522 100644 --- a/src/mlpack/core/optimizers/minibatch_sgd/minibatch_sgd.hpp +++ b/src/mlpack/core/optimizers/minibatch_sgd/minibatch_sgd.hpp @@ -13,6 +13,8 @@ #define MLPACK_CORE_OPTIMIZERS_MINIBATCH_SGD_MINIBATCH_SGD_HPP #include +#include +#include namespace mlpack { namespace optimization { @@ -69,9 +71,22 @@ namespace optimization { * * @tparam DecomposableFunctionType Decomposable objective function type to be * minimized. + * @tparam update Update policy used during the iterative update process. + * By default the vanilla update policy + * (see mlpack::optimization::VanillaUpdate) is used. + * @tparam UpdatePolicyType Update policy used during the iterative update + * process. By default the vanilla update policy + * (see mlpack::optimization::VanillaUpdate) is used. + * @tparam DecayPolicyType Decay policy used during the iterative update + * process to adjust the step size. By default the step size isn't going to + * be adjusted. */ -template -class MiniBatchSGD +template< + typename DecomposableFunctionType, + typename UpdatePolicyType = VanillaUpdate, + typename DecayPolicyType = NoDecay +> +class MiniBatchSGDType { public: /** @@ -89,13 +104,18 @@ class MiniBatchSGD * @param tolerance Maximum absolute tolerance to terminate algorithm. * @param shuffle If true, the mini-batch order is shuffled; otherwise, each * mini-batch is visited in linear order. + * @param updatePolicy Instantiated update policy used to adjust the given + * parameters. + * @param decayPolicy Instantiated decay policy used to adjust the step size. */ - MiniBatchSGD(DecomposableFunctionType& function, - const size_t batchSize = 1000, - const double stepSize = 0.01, - const size_t maxIterations = 100000, - const double tolerance = 1e-5, - const bool shuffle = true); + MiniBatchSGDType(DecomposableFunctionType& function, + const size_t batchSize = 1000, + const double stepSize = 0.01, + const size_t maxIterations = 100000, + const double tolerance = 1e-5, + const bool shuffle = true, + const UpdatePolicyType& updatePolicy = UpdatePolicyType(), + const DecayPolicyType& decayPolicy = DecayPolicyType()); /** * Optimize the given function using mini-batch SGD. The given starting point @@ -137,6 +157,16 @@ class MiniBatchSGD //! Modify whether or not the individual functions are shuffled. bool& Shuffle() { return shuffle; } + //! Get the update policy. + UpdatePolicyType UpdatePolicy() const { return updatePolicy; } + //! Modify the update policy. + UpdatePolicyType& UpdatePolicy() { return updatePolicy; } + + //! Get the decay policy. + DecayPolicyType DecayPolicy() const { return decayPolicy; } + //! Modify the decay policy. + DecayPolicyType& DecayPolicy() { return decayPolicy; } + private: //! The instantiated function. DecomposableFunctionType& function; @@ -156,8 +186,18 @@ class MiniBatchSGD //! Controls whether or not the individual functions are shuffled when //! iterating. bool shuffle; + + //! The update policy used to update the parameters in each iteration. + UpdatePolicyType updatePolicy; + + //! The decay policy used to update the parameters in each iteration. + DecayPolicyType decayPolicy; }; +template +using MiniBatchSGD = MiniBatchSGDType< + DecomposableFunctionType, VanillaUpdate, NoDecay>; + } // namespace optimization } // namespace mlpack diff --git a/src/mlpack/core/optimizers/minibatch_sgd/minibatch_sgd_impl.hpp b/src/mlpack/core/optimizers/minibatch_sgd/minibatch_sgd_impl.hpp index f0dd18c7dc..420e3b13af 100644 --- a/src/mlpack/core/optimizers/minibatch_sgd/minibatch_sgd_impl.hpp +++ b/src/mlpack/core/optimizers/minibatch_sgd/minibatch_sgd_impl.hpp @@ -18,25 +18,44 @@ namespace mlpack { namespace optimization { -template -MiniBatchSGD::MiniBatchSGD( - DecomposableFunctionType& function, - const size_t batchSize, - const double stepSize, - const size_t maxIterations, - const double tolerance, - const bool shuffle) : - function(function), - batchSize(batchSize), - stepSize(stepSize), - maxIterations(maxIterations), - tolerance(tolerance), - shuffle(shuffle) +template< + typename DecomposableFunctionType, + typename UpdatePolicyType, + typename DecayPolicyType +> +MiniBatchSGDType< + DecomposableFunctionType, + UpdatePolicyType, + DecayPolicyType +>::MiniBatchSGDType(DecomposableFunctionType& function, + const size_t batchSize, + const double stepSize, + const size_t maxIterations, + const double tolerance, + const bool shuffle, + const UpdatePolicyType& updatePolicy, + const DecayPolicyType& decayPolicy) : + function(function), + batchSize(batchSize), + stepSize(stepSize), + maxIterations(maxIterations), + tolerance(tolerance), + shuffle(shuffle), + updatePolicy(updatePolicy), + decayPolicy(decayPolicy) { /* Nothing to do. */ } //! Optimize the function (minimize). -template -double MiniBatchSGD::Optimize(arma::mat& iterate) +template< + typename DecomposableFunctionType, + typename UpdatePolicyType, + typename DecayPolicyType +> +double MiniBatchSGDType< + DecomposableFunctionType, + UpdatePolicyType, + DecayPolicyType +>::Optimize(arma::mat& iterate) { // Find the number of functions. const size_t numFunctions = function.NumFunctions(); @@ -60,6 +79,9 @@ double MiniBatchSGD::Optimize(arma::mat& iterate) for (size_t i = 0; i < numFunctions; ++i) overallObjective += function.Evaluate(iterate, i); + // Initialize the update policy. + updatePolicy.Initialize(iterate.n_rows, iterate.n_cols); + // Now iterate! arma::mat gradient(iterate.n_rows, iterate.n_cols); for (size_t i = 1; i != maxIterations; ++i, ++currentBatch) @@ -108,7 +130,7 @@ double MiniBatchSGD::Optimize(arma::mat& iterate) } // Now update the iterate. - iterate -= (stepSize / batchSize) * gradient; + updatePolicy.Update(iterate, stepSize / batchSize, gradient); // Add that to the overall objective function. for (size_t j = 0; j < batchSize; ++j) @@ -130,18 +152,21 @@ double MiniBatchSGD::Optimize(arma::mat& iterate) if (lastBatchSize > 0) { // Now update the iterate. - iterate -= (stepSize / lastBatchSize) * gradient; + updatePolicy.Update(iterate, stepSize / lastBatchSize, gradient); } else { // Now update the iterate. - iterate -= stepSize * gradient; + updatePolicy.Update(iterate, stepSize, gradient); } // Add that to the overall objective function. for (size_t j = 0; j < lastBatchSize; ++j) overallObjective += function.Evaluate(iterate, offset + j); } + + // Now update the learning rate if requested by the user. + decayPolicy.Update(iterate, stepSize, gradient); } Log::Info << "Mini-batch SGD: maximum iterations (" << maxIterations << ") " diff --git a/src/mlpack/core/optimizers/rmsprop/rmsprop_update.hpp b/src/mlpack/core/optimizers/rmsprop/rmsprop_update.hpp index c86f09f0c3..30f7aacbbe 100644 --- a/src/mlpack/core/optimizers/rmsprop/rmsprop_update.hpp +++ b/src/mlpack/core/optimizers/rmsprop/rmsprop_update.hpp @@ -114,4 +114,4 @@ class RMSPropUpdate } // namespace optimization } // namespace mlpack -#endif \ No newline at end of file +#endif diff --git a/src/mlpack/core/optimizers/sdp/lrsdp_function.hpp b/src/mlpack/core/optimizers/sdp/lrsdp_function.hpp index 6493dfb0fd..00c39f2993 100644 --- a/src/mlpack/core/optimizers/sdp/lrsdp_function.hpp +++ b/src/mlpack/core/optimizers/sdp/lrsdp_function.hpp @@ -27,7 +27,6 @@ template class LRSDPFunction { public: - /** * Construct the LRSDPFunction from the given SDP. * @@ -90,7 +89,6 @@ class LRSDPFunction SDPType& SDP() { return sdp; } private: - //! SDP object representing the problem SDPType sdp; diff --git a/src/mlpack/core/optimizers/sdp/lrsdp_function_impl.hpp b/src/mlpack/core/optimizers/sdp/lrsdp_function_impl.hpp index 7b23b5eead..30cb237ffe 100644 --- a/src/mlpack/core/optimizers/sdp/lrsdp_function_impl.hpp +++ b/src/mlpack/core/optimizers/sdp/lrsdp_function_impl.hpp @@ -55,13 +55,14 @@ template void LRSDPFunction::Gradient(const arma::mat& /* coordinates */, arma::mat& /* gradient */) const { - Log::Fatal << "LRSDPFunction::Gradient() not implemented for arbitrary optimizers!" - << std::endl; + Log::Fatal << "LRSDPFunction::Gradient() not implemented for arbitrary " + << "optimizers!" << std::endl; } template -double LRSDPFunction::EvaluateConstraint(const size_t index, - const arma::mat& coordinates) const +double LRSDPFunction::EvaluateConstraint( + const size_t index, + const arma::mat& coordinates) const { const arma::mat rrt = coordinates * trans(coordinates); if (index < SDP().NumSparseConstraints()) @@ -71,12 +72,13 @@ double LRSDPFunction::EvaluateConstraint(const size_t index, } template -void LRSDPFunction::GradientConstraint(const size_t /* index */, - const arma::mat& /* coordinates */, - arma::mat& /* gradient */) const +void LRSDPFunction::GradientConstraint( + const size_t /* index */, + const arma::mat& /* coordinates */, + arma::mat& /* gradient */) const { - Log::Fatal << "LRSDPFunction::GradientConstraint() not implemented for arbitrary " - << "optimizers!" << std::endl; + Log::Fatal << "LRSDPFunction::GradientConstraint() not implemented " + << "for arbitrary optimizers!" << std::endl; } //! Utility function for calculating part of the objective when AugLagrangian is @@ -144,10 +146,11 @@ EvaluateImpl(const LRSDPFunction& function, double objective = accu(function.SDP().C() % rrt); // Now each constraint. - UpdateObjective(objective, rrt, function.SDP().SparseA(), function.SDP().SparseB(), - lambda, 0, sigma); - UpdateObjective(objective, rrt, function.SDP().DenseA(), function.SDP().DenseB(), lambda, - function.SDP().NumSparseConstraints(), sigma); + UpdateObjective(objective, rrt, function.SDP().SparseA(), + function.SDP().SparseB(), lambda, 0, sigma); + UpdateObjective(objective, rrt, function.SDP().DenseA(), + function.SDP().DenseB(), lambda, function.SDP().NumSparseConstraints(), + sigma); return objective; } diff --git a/src/mlpack/core/optimizers/sdp/primal_dual.hpp b/src/mlpack/core/optimizers/sdp/primal_dual.hpp index 9915115d95..908852ae87 100644 --- a/src/mlpack/core/optimizers/sdp/primal_dual.hpp +++ b/src/mlpack/core/optimizers/sdp/primal_dual.hpp @@ -109,7 +109,7 @@ class PrimalDualSolver arma::vec initialYdense; //! Starting point for Z, the complementary slack variable. Needs to be - //positive definite. + //! positive definite. arma::mat initialZ; //! The step size modulating factor. Needs to be a scalar in (0, 1). diff --git a/src/mlpack/core/optimizers/sdp/primal_dual_impl.hpp b/src/mlpack/core/optimizers/sdp/primal_dual_impl.hpp index bc01438e4d..348dc3a603 100644 --- a/src/mlpack/core/optimizers/sdp/primal_dual_impl.hpp +++ b/src/mlpack/core/optimizers/sdp/primal_dual_impl.hpp @@ -46,9 +46,7 @@ PrimalDualSolver::PrimalDualSolver(const SDPType& sdp) primalInfeasTol(1e-7), dualInfeasTol(1e-7), maxIterations(1000) -{ - -} +{ /* Nothing to do. */ } template PrimalDualSolver::PrimalDualSolver(const SDPType& sdp, diff --git a/src/mlpack/core/optimizers/sdp/sdp.hpp b/src/mlpack/core/optimizers/sdp/sdp.hpp index 5a0c89e944..2b7151dd83 100644 --- a/src/mlpack/core/optimizers/sdp/sdp.hpp +++ b/src/mlpack/core/optimizers/sdp/sdp.hpp @@ -39,7 +39,6 @@ template class SDP { public: - typedef ObjectiveMatrixType objective_matrix_type; /** diff --git a/src/mlpack/core/optimizers/sdp/sdp_impl.hpp b/src/mlpack/core/optimizers/sdp/sdp_impl.hpp index 2e0ff76449..88ec79d0ad 100644 --- a/src/mlpack/core/optimizers/sdp/sdp_impl.hpp +++ b/src/mlpack/core/optimizers/sdp/sdp_impl.hpp @@ -23,9 +23,7 @@ SDP::SDP() : sparseB(), denseA(), denseB() -{ - -} +{ /* Nothing to do. */ } template SDP::SDP(const size_t n, diff --git a/src/mlpack/core/optimizers/sgd/sgd_impl.hpp b/src/mlpack/core/optimizers/sgd/sgd_impl.hpp index 5ac19ba960..ac6841e92e 100644 --- a/src/mlpack/core/optimizers/sgd/sgd_impl.hpp +++ b/src/mlpack/core/optimizers/sgd/sgd_impl.hpp @@ -65,7 +65,7 @@ double SGD::Optimize( overallObjective += function.Evaluate(iterate, i); // Initialize the update policy. - updatePolicy.Initialize(iterate.n_rows,iterate.n_cols); + updatePolicy.Initialize(iterate.n_rows, iterate.n_cols); // Now iterate! arma::mat gradient(iterate.n_rows, iterate.n_cols); diff --git a/src/mlpack/core/optimizers/sgd/update_policies/momentum_update.hpp b/src/mlpack/core/optimizers/sgd/update_policies/momentum_update.hpp index 1ea0856034..6d9041a557 100644 --- a/src/mlpack/core/optimizers/sgd/update_policies/momentum_update.hpp +++ b/src/mlpack/core/optimizers/sgd/update_policies/momentum_update.hpp @@ -84,7 +84,7 @@ class MomentumUpdate void Initialize(const size_t rows, const size_t cols) { - //Initialize am empty velocity matrix. + // Initialize am empty velocity matrix. velocity = arma::zeros(rows, cols); } diff --git a/src/mlpack/core/optimizers/sgdr/CMakeLists.txt b/src/mlpack/core/optimizers/sgdr/CMakeLists.txt new file mode 100644 index 0000000000..5c3e185bff --- /dev/null +++ b/src/mlpack/core/optimizers/sgdr/CMakeLists.txt @@ -0,0 +1,15 @@ +set(SOURCES + cyclical_decay.hpp + sgdr.hpp + sgdr_impl.hpp + snapshot_ensembles.hpp + snapshot_sgdr.hpp + snapshot_sgdr_impl.hpp +) + +set(DIR_SRCS) +foreach(file ${SOURCES}) + set(DIR_SRCS ${DIR_SRCS} ${CMAKE_CURRENT_SOURCE_DIR}/${file}) +endforeach() + +set(MLPACK_SRCS ${MLPACK_SRCS} ${DIR_SRCS} PARENT_SCOPE) diff --git a/src/mlpack/core/optimizers/sgdr/cyclical_decay.hpp b/src/mlpack/core/optimizers/sgdr/cyclical_decay.hpp new file mode 100644 index 0000000000..cd338fb0fe --- /dev/null +++ b/src/mlpack/core/optimizers/sgdr/cyclical_decay.hpp @@ -0,0 +1,142 @@ +/** + * @file cyclical_decay.hpp + * @author Marcus Edel + * + * Definition of the warm restart technique (SGDR) described in: + * "SGDR: Stochastic Gradient Descent with Warm Restarts" by + * I. Loshchilov et al. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ + +#ifndef MLPACK_CORE_OPTIMIZERS_SGDR_CYCLICAL_DECAY_HPP +#define MLPACK_CORE_OPTIMIZERS_SGDR_CYCLICAL_DECAY_HPP + +namespace mlpack { +namespace optimization { + +/** + * Simulate a new warm-started run/restart once a number of epochs are + * performed. Importantly, the restarts are not performed from scratch but + * emulated by increasing the step size while the old step size value of as an + * initial parameter. + * + * For more information, please refer to: + * + * @code + * @article{Loshchilov2016, + * title = {Learning representations by back-propagating errors}, + * author = {Ilya Loshchilov and Frank Hutter}, + * title = {{SGDR:} Stochastic Gradient Descent with Restarts}, + * journal = {CoRR}, + * year = {2016}, + * url = {https://arxiv.org/abs/1608.03983} + * } + * @endcode + */ +class CyclicalDecay +{ + public: + /** + * Construct the CyclicalDecay technique a restart method, where the + * step size decays after each batch and peridically resets to its initial + * value. + * + * @param epochRestart Initial epoch where decay is applied. + * @param multFactor Factor to increase the number of epochs before a restart. + * @param stepSize Initial step size for each restart. + * @param batchSize Size of each mini-batch. + * @param numFunctions The number of separable functions (the number of + * predictor points). + */ + CyclicalDecay(const size_t epochRestart, + const double multFactor, + const double stepSize, + const size_t batchSize, + const size_t numFunctions) : + epochRestart(epochRestart), + multFactor(multFactor), + constStepSize(stepSize), + nextRestart(epochRestart), + batchRestart(0), + epochBatches(numFunctions / (double) batchSize), + epoch(0) + { /* Nothing to do here */ } + + /** + * This function is called in each iteration after the policy update. + * + * @param iterate Parameters that minimize the function. + * @param stepSize Step size to be used for the given iteration. + * @param gradient The gradient matrix. + */ + void Update(arma::mat& /* iterate */, + double& stepSize, + const arma::mat& /* gradient */) + { + // Time to adjust the step size. + if (epoch >= epochRestart) + { + // n_t = n_min^i + 0.5(n_max^i - n_min^i)(1 + cos(T_cur/T_i * pi)). + stepSize = 0.5 * constStepSize * (1 + cos((batchRestart / epochBatches) + * M_PI)); + + // Keep track of the number of batches since the last restart. + batchRestart++; + } + + // Time to restart. + if (epoch > nextRestart) + { + batchRestart = 0; + + // Adjust the period of restarts. + epochRestart *= multFactor; + + // Update the time for the next restart. + nextRestart += epochRestart; + } + + epoch++; + } + + //! Get the step size. + double StepSize() const { return constStepSize; } + //! Modify the step size. + double& StepSize() { return constStepSize; } + + //! Get the restart fraction. + double EpochBatches() const { return epochBatches; } + //! Modify the restart fraction. + double& EpochBatches() { return epochBatches; } + + private: + //! Epoch where decay is applied. + size_t epochRestart; + + //! Parameter to increase the number of epochs before a restart. + double multFactor; + + //! The step size for each example. + double constStepSize; + + //! Locally-stored restart time. + size_t nextRestart; + + //! Locally-stored number of batches since the last restart. + size_t batchRestart; + + //! Locally-stored restart fraction. + double epochBatches; + + //! Locally-stored epoch. + size_t epoch; +}; + +} // namespace optimization +} // namespace mlpack + +#endif // MLPACK_CORE_OPTIMIZERS_SGDR_CYCLICAL_DECAY_HPP diff --git a/src/mlpack/core/optimizers/sgdr/sgdr.hpp b/src/mlpack/core/optimizers/sgdr/sgdr.hpp new file mode 100644 index 0000000000..e404f21a0c --- /dev/null +++ b/src/mlpack/core/optimizers/sgdr/sgdr.hpp @@ -0,0 +1,150 @@ +/** + * @file sgdr.hpp + * @author Marcus Edel + * + * Definition of the Stochastic Gradient Descent with Restarts (SGDR) as + * described in: "SGDR: Stochastic Gradient Descent with Warm Restarts" by + * I. Loshchilov et al. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_CORE_OPTIMIZERS_SGDR_SGDR_HPP +#define MLPACK_CORE_OPTIMIZERS_SGDR_SGDR_HPP + +#include + +#include +#include +#include "cyclical_decay.hpp" + +namespace mlpack { +namespace optimization { + +/** + * This class is based on Mini-batch Stochastic Gradient Descent class and + * simulates a new warm-started run/restart once a number of epochs are + * performed. + * + * For more information, please refer to: + * + * @code + * @article{Loshchilov2016, + * title = {{SGDR:} Stochastic Gradient Descent with Restarts}, + * author = {Ilya Loshchilov and Frank Hutter}, + * journal = {CoRR}, + * year = {2016}, + * url = {https://arxiv.org/abs/1608.03983} + * } + * @endcode + * + * @tparam DecomposableFunctionType Decomposable objective function type to be + * minimized. + * @tparam UpdatePolicyType Update policy used during the iterative update + * process. By default the momentum update policy + * (see mlpack::optimization::MomentumUpdate) is used. + */ +template< + typename DecomposableFunctionType, + typename UpdatePolicyType = MomentumUpdate +> +class SGDR +{ + public: + //! Convenience typedef for the internal optimizer construction. + using OptimizerType = MiniBatchSGDType< + DecomposableFunctionType, UpdatePolicyType, CyclicalDecay>; + + /** + * Construct the SGDR optimizer with the given function and + * parameters. The defaults here are not necessarily good for the given + * problem, so it is suggested that the values used be tailored for the task + * at hand. The maximum number of iterations refers to the maximum number of + * mini-batches that are processed. + * + * @param epochRestart Initial epoch where decay is applied. + * @param function Function to be optimized (minimized). + * @param batchSize Size of each mini-batch. + * @param stepSize Step size for each iteration. + * @param maxIterations Maximum number of iterations allowed (0 means no + * limit). + * @param tolerance Maximum absolute tolerance to terminate algorithm. + * @param shuffle If true, the mini-batch order is shuffled; otherwise, each + * mini-batch is visited in linear order. + * @param updatePolicy Instantiated update policy used to adjust the given + * parameters. + */ + SGDR(DecomposableFunctionType& function, + const size_t epochRestart = 50, + const double multFactor = 2.0, + const size_t batchSize = 1000, + const double stepSize = 0.01, + const size_t maxIterations = 100000, + const double tolerance = 1e-5, + const bool shuffle = true, + const UpdatePolicyType& updatePolicy = UpdatePolicyType()); + + /** + * Optimize the given function using SGDR. The given starting point + * will be modified to store the finishing point of the algorithm, and the + * final objective value is returned. + * + * @param iterate Starting point (will be modified). + * @return Objective value of the final point. + */ + double Optimize(arma::mat& iterate); + + //! Get the instantiated function to be optimized. + const DecomposableFunctionType& Function() const + { + return optimizer.Function(); + } + + //! Modify the instantiated function. + DecomposableFunctionType& Function() { return optimizer.Function(); } + + //! Get the batch size. + size_t BatchSize() const { return optimizer.BatchSize(); } + //! Modify the batch size. + size_t& BatchSize() { return optimizer.BatchSize(); } + + //! Get the step size. + double StepSize() const { return optimizer.StepSize(); } + //! Modify the step size. + double& StepSize() { return optimizer.StepSize(); } + + //! Get the maximum number of iterations (0 indicates no limit). + size_t MaxIterations() const { return optimizer.MaxIterations(); } + //! Modify the maximum number of iterations (0 indicates no limit). + size_t& MaxIterations() { return optimizer.MaxIterations(); } + + //! Get the tolerance for termination. + double Tolerance() const { return optimizer.Tolerance(); } + //! Modify the tolerance for termination. + double& Tolerance() { return optimizer.Tolerance(); } + + //! Get whether or not the individual functions are shuffled. + bool Shuffle() const { return optimizer.Shuffle(); } + //! Modify whether or not the individual functions are shuffled. + bool& Shuffle() { return optimizer.Shuffle(); } + + private: + //! The instantiated function. + DecomposableFunctionType& function; + + //! The size of each mini-batch. + size_t batchSize; + + //! Locally-stored optimizer instance. + OptimizerType optimizer; +}; + +} // namespace optimization +} // namespace mlpack + +// Include implementation. +#include "sgdr_impl.hpp" + +#endif diff --git a/src/mlpack/core/optimizers/sgdr/sgdr_impl.hpp b/src/mlpack/core/optimizers/sgdr/sgdr_impl.hpp new file mode 100644 index 0000000000..ce530f7e07 --- /dev/null +++ b/src/mlpack/core/optimizers/sgdr/sgdr_impl.hpp @@ -0,0 +1,77 @@ +/** + * @file sgdr_impl.hpp + * @author Marcus Edel + * + * Implementation of SGDR method. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_CORE_OPTIMIZERS_SGDR_SGDR_IMPL_HPP +#define MLPACK_CORE_OPTIMIZERS_SGDR_SGDR_IMPL_HPP + +// In case it hasn't been included yet. +#include "sgdr.hpp" + +namespace mlpack { +namespace optimization { + +template +SGDR::SGDR( + DecomposableFunctionType& function, + const size_t epochRestart, + const double multFactor, + const size_t batchSize, + const double stepSize, + const size_t maxIterations, + const double tolerance, + const bool shuffle, + const UpdatePolicyType& updatePolicy) : + function(function), + batchSize(batchSize), + optimizer(OptimizerType(function, + batchSize, + stepSize, + maxIterations, + tolerance, + shuffle, + updatePolicy, + CyclicalDecay( + epochRestart, + multFactor, + stepSize, + batchSize, + function.NumFunctions()))) +{ + /* Nothing to do here */ +} + +template +double SGDR::Optimize( + arma::mat& iterate) +{ + // If a user changed the step size he hasn't update the step size of the + // cyclical decay instantiation, so we have to do it here. + if (optimizer.StepSize() != optimizer.DecayPolicy().StepSize()) + { + optimizer.DecayPolicy().StepSize() = optimizer.StepSize(); + } + + // If a user changed the batch size we have to update the restart fraction + // of the cyclical decay instantiation. + if (optimizer.BatchSize() != batchSize) + { + batchSize = optimizer.BatchSize(); + optimizer.DecayPolicy().EpochBatches() = function.NumFunctions() / + double(batchSize); + } + + return optimizer.Optimize(iterate); +} + +} // namespace optimization +} // namespace mlpack + +#endif diff --git a/src/mlpack/core/optimizers/sgdr/snapshot_ensembles.hpp b/src/mlpack/core/optimizers/sgdr/snapshot_ensembles.hpp new file mode 100644 index 0000000000..83f3dca542 --- /dev/null +++ b/src/mlpack/core/optimizers/sgdr/snapshot_ensembles.hpp @@ -0,0 +1,179 @@ +/** + * @file snapshot_ensembles.hpp + * @author Marcus Edel + * + * Definition of the Snapshot ensembles technique described in: + * "Snapshot ensembles: Train 1, get m for free" by G. Huang et al. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ + +#ifndef MLPACK_CORE_OPTIMIZERS_SGDR_SNAPSHOT_ENSEMBLES_HPP +#define MLPACK_CORE_OPTIMIZERS_SGDR_SNAPSHOT_ENSEMBLES_HPP + +namespace mlpack { +namespace optimization { + +/** + * Simulate a new warm-started run/restart once a number of epochs are + * performed. Importantly, the restarts are not performed from scratch but + * emulated by increasing the step size while the old step size value of as an + * initial parameter. + * + * For more information, please refer to: + * + * @code + * @inproceedings{Huang2017, + * title = {Snapshot ensembles: Train 1, get m for free}, + * author = {Gao Huang, Yixuan Li, Geoff Pleiss, Zhuang Liu, + * John E. Hopcroft, and Kilian Q. Weinberger}, + * booktitle = {Proceedings of the International Conference on Learning + * Representations (ICLR)}, + * year = {2017}, + * url = {https://arxiv.org/abs/1704.00109} + * } + * @endcode + */ +class SnapshotEnsembles +{ + public: + /** + * Construct the CyclicalDecay technique a restart method, where the + * step size decays after each batch and peridically resets to its initial + * value. + * + * @param epochRestart Initial epoch where decay is applied. + * @param multFactor Factor to increase the number of epochs before a restart. + * @param stepSize Initial step size for each restart. + * @param batchSize Size of each mini-batch. + * @param numFunctions The number of separable functions (the number of + * predictor points). + * @param maxIterations Maximum number of iterations allowed (0 means no + * limit). + * @param snapshots Maximum number of snapshots. + */ + SnapshotEnsembles(const size_t epochRestart, + const double multFactor, + const double stepSize, + const size_t numFunctions, + const size_t batchSize, + const size_t maxIterations, + const size_t snapshots) : + epochRestart(epochRestart), + multFactor(multFactor), + constStepSize(stepSize), + nextRestart(epochRestart), + batchRestart(0), + epochBatches(numFunctions / (double) batchSize), + epoch(0) + { + snapshotEpochs = 0; + for (size_t i = 0, er = epochRestart, nr = nextRestart; + i < maxIterations; ++i) + { + if (i > nr) + { + er *= multFactor; + nr += er; + snapshotEpochs++; + } + } + + snapshotEpochs = epochRestart * std::pow(multFactor, + snapshotEpochs - snapshots + 1); + } + + /** + * This function is called in each iteration after the policy update. + * + * @param iterate Parameters that minimize the function. + * @param stepSize Step size to be used for the given iteration. + * @param gradient The gradient matrix. + */ + void Update(arma::mat& iterate, + double& stepSize, + const arma::mat& /* gradient */) + { + // Time to adjust the step size. + if (epoch >= epochRestart) + { + // n_t = n_min^i + 0.5(n_max^i - n_min^i)(1 + cos(T_cur/T_i * pi)). + stepSize = 0.5 * constStepSize * (1 + cos((batchRestart / epochBatches) + * M_PI)); + + // Keep track of the number of batches since the last restart. + batchRestart++; + } + + // Time to restart. + if (epoch > nextRestart) + { + batchRestart = 0; + + // Adjust the period of restarts. + epochRestart *= multFactor; + + // Create a new snapshot. + if (epochRestart >= snapshotEpochs) + { + snapshots.push_back(iterate); + } + + // Update the time for the next restart. + nextRestart += epochRestart; + } + + epoch++; + } + + //! Get the step size. + double StepSize() const { return constStepSize; } + //! Modify the step size. + double& StepSize() { return constStepSize; } + + //! Get the restart fraction. + double EpochBatches() const { return epochBatches; } + //! Modify the restart fraction. + double& EpochBatches() { return epochBatches; } + + //! Get the snapshots. + std::vector Snapshots() const { return snapshots; } + //! Modify the snapshots. + std::vector& Snapshots() { return snapshots; } + + private: + //! Epoch where decay is applied. + size_t epochRestart; + + //! Parameter to increase the number of epochs before a restart. + double multFactor; + + //! The step size for each example. + double constStepSize; + + //! Locally-stored restart time. + size_t nextRestart; + + //! Locally-stored number of batches since the last restart. + size_t batchRestart; + + //! Locally-stored restart fraction. + double epochBatches; + + //! Locally-stored epoch. + size_t epoch; + + //! Epochs where a new snapshot is created. + size_t snapshotEpochs; + + //! Locally-stored parameter snapshots. + std::vector snapshots; +}; + +} // namespace optimization +} // namespace mlpack + +#endif // MLPACK_CORE_OPTIMIZERS_SGDR_CYCLICAL_DECAY_HPP diff --git a/src/mlpack/core/optimizers/sgdr/snapshot_sgdr.hpp b/src/mlpack/core/optimizers/sgdr/snapshot_sgdr.hpp new file mode 100644 index 0000000000..00f76af853 --- /dev/null +++ b/src/mlpack/core/optimizers/sgdr/snapshot_sgdr.hpp @@ -0,0 +1,182 @@ +/** + * @file snapshots_sgdr.hpp + * @author Marcus Edel + * + * Definition of the Stochastic Gradient Descent with Restarts (SGDR) as + * described in: "SGDR: Stochastic Gradient Descent with Warm Restarts" by + * I. Loshchilov et al and the Snapshot ensembles technique described in: + * "Snapshot ensembles: Train 1, get m for free" by G. Huang et al. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_CORE_OPTIMIZERS_SGDR_SNAPSHOT_SGDR_HPP +#define MLPACK_CORE_OPTIMIZERS_SGDR_SNAPSHOT_SGDR_HPP + +#include + +#include +#include +#include "snapshot_ensembles.hpp" + +namespace mlpack { +namespace optimization { + +/** + * This class is based on Mini-batch Stochastic Gradient Descent class and + * simulates a new warm-started run/restart once a number of epochs are + * performed using the Snapshot ensembles technique. + * + * For more information, please refer to: + * + * @code + * @article{Loshchilov2016, + * title = {{SGDR:} Stochastic Gradient Descent with Restarts}, + * author = {Ilya Loshchilov and Frank Hutter}, + * journal = {CoRR}, + * year = {2016}, + * url = {https://arxiv.org/abs/1608.03983} + * } + * @endcode + * + * @code + * @inproceedings{Huang2017, + * title = {Snapshot ensembles: Train 1, get m for free}, + * author = {Gao Huang, Yixuan Li, Geoff Pleiss, Zhuang Liu, + * John E. Hopcroft, and Kilian Q. Weinberger}, + * booktitle = {Proceedings of the International Conference on Learning + * Representations (ICLR)}, + * year = {2017}, + * url = {https://arxiv.org/abs/1704.00109} + * } + * @endcode + * + * @tparam DecomposableFunctionType Decomposable objective function type to be + * minimized. + * @tparam UpdatePolicyType Update policy used during the iterative update + * process. By default the momentum update policy + * (see mlpack::optimization::MomentumUpdate) is used. + */ +template< + typename DecomposableFunctionType, + typename UpdatePolicyType = MomentumUpdate +> +class SnapshotSGDR +{ + public: + //! Convenience typedef for the internal optimizer construction. + using OptimizerType = MiniBatchSGDType< + DecomposableFunctionType, UpdatePolicyType, SnapshotEnsembles>; + + /** + * Construct the SnapshotSGDR optimizer with snapshot ensembles with the given + * function and parameters. The defaults here are not necessarily good for + * the given problem, so it is suggested that the values used be tailored for + * the task at hand. The maximum number of iterations refers to the maximum + * number of mini-batches that are processed. + * + * @param epochRestart Initial epoch where decay is applied. + * @param function Function to be optimized (minimized). + * @param batchSize Size of each mini-batch. + * @param stepSize Step size for each iteration. + * @param maxIterations Maximum number of iterations allowed (0 means no + * limit). + * @param tolerance Maximum absolute tolerance to terminate algorithm. + * @param shuffle If true, the mini-batch order is shuffled; otherwise, each + * mini-batch is visited in linear order. + * @param snapshots Maximum number of snapshots. + * @param accumulate Accumulate the snapshot parameter (default true). + * @param updatePolicy Instantiated update policy used to adjust the given + * parameters. + */ + SnapshotSGDR(DecomposableFunctionType& function, + const size_t epochRestart = 50, + const double multFactor = 2.0, + const size_t batchSize = 1000, + const double stepSize = 0.01, + const size_t maxIterations = 100000, + const double tolerance = 1e-5, + const bool shuffle = true, + const size_t snapshots = 5, + const bool accumulate = true, + const UpdatePolicyType& updatePolicy = UpdatePolicyType()); + + /** + * Optimize the given function using SGDR. The given starting point + * will be modified to store the finishing point of the algorithm, and the + * final objective value is returned. + * + * @param iterate Starting point (will be modified). + * @param accumulate Accumulate the snapshot parameter (default true). + * @return Objective value of the final point. + */ + double Optimize(arma::mat& iterate); + + //! Get the instantiated function to be optimized. + const DecomposableFunctionType& Function() const + { + return optimizer.Function(); + } + + //! Modify the instantiated function. + DecomposableFunctionType& Function() { return optimizer.Function(); } + + //! Get the batch size. + size_t BatchSize() const { return optimizer.BatchSize(); } + //! Modify the batch size. + size_t& BatchSize() { return optimizer.BatchSize(); } + + //! Get the step size. + double StepSize() const { return optimizer.StepSize(); } + //! Modify the step size. + double& StepSize() { return optimizer.StepSize(); } + + //! Get the maximum number of iterations (0 indicates no limit). + size_t MaxIterations() const { return optimizer.MaxIterations(); } + //! Modify the maximum number of iterations (0 indicates no limit). + size_t& MaxIterations() { return optimizer.MaxIterations(); } + + //! Get the tolerance for termination. + double Tolerance() const { return optimizer.Tolerance(); } + //! Modify the tolerance for termination. + double& Tolerance() { return optimizer.Tolerance(); } + + //! Get whether or not the individual functions are shuffled. + bool Shuffle() const { return optimizer.Shuffle(); } + //! Modify whether or not the individual functions are shuffled. + bool& Shuffle() { return optimizer.Shuffle(); } + + //! Get the snapshots. + std::vector Snapshots() const + { + return optimizer.DecayPolicy().Snapshots(); + } + //! Modify the snapshots. + std::vector& Snapshots() + { + return optimizer.DecayPolicy().Snapshots(); + } + + private: + //! The instantiated function. + DecomposableFunctionType& function; + + //! The size of each mini-batch. + size_t batchSize; + + //! Whether or not to accumulate the snapshots. + bool accumulate; + + //! Locally-stored optimizer instance. + OptimizerType optimizer; +}; + +} // namespace optimization +} // namespace mlpack + +// Include implementation. +#include "snapshot_sgdr_impl.hpp" + +#endif diff --git a/src/mlpack/core/optimizers/sgdr/snapshot_sgdr_impl.hpp b/src/mlpack/core/optimizers/sgdr/snapshot_sgdr_impl.hpp new file mode 100644 index 0000000000..c58d78627b --- /dev/null +++ b/src/mlpack/core/optimizers/sgdr/snapshot_sgdr_impl.hpp @@ -0,0 +1,99 @@ +/** + * @file snapshots_sgdr_impl.hpp + * @author Marcus Edel + * + * Implementation of SGDR method using snapshots ensembles. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_CORE_OPTIMIZERS_SGDR_SNAPSHOT_SGDR_IMPL_HPP +#define MLPACK_CORE_OPTIMIZERS_SGDR_SNAPSHOT_SGDR_IMPL_HPP + +// In case it hasn't been included yet. +#include "snapshot_sgdr.hpp" + +namespace mlpack { +namespace optimization { + +template +SnapshotSGDR::SnapshotSGDR( + DecomposableFunctionType& function, + const size_t epochRestart, + const double multFactor, + const size_t batchSize, + const double stepSize, + const size_t maxIterations, + const double tolerance, + const bool shuffle, + const size_t snapshots, + const bool accumulate, + const UpdatePolicyType& updatePolicy) : + function(function), + batchSize(batchSize), + accumulate(accumulate), + optimizer(OptimizerType(function, + batchSize, + stepSize, + maxIterations, + tolerance, + shuffle, + updatePolicy, + SnapshotEnsembles( + epochRestart, + multFactor, + stepSize, + batchSize, + function.NumFunctions(), + maxIterations, + snapshots))) +{ + /* Nothing to do here */ +} + +template +double SnapshotSGDR::Optimize( + arma::mat& iterate) +{ + // If a user changed the step size he hasn't update the step size of the + // cyclical decay instantiation, so we have to do here. + if (optimizer.StepSize() != optimizer.DecayPolicy().StepSize()) + { + optimizer.DecayPolicy().StepSize() = optimizer.StepSize(); + } + + // If a user changed the batch size we have to update the restart fraction + // of the cyclical decay instantiation. + if (optimizer.BatchSize() != batchSize) + { + batchSize = optimizer.BatchSize(); + optimizer.DecayPolicy().EpochBatches() = function.NumFunctions() / + double(batchSize); + } + + double overallObjective = optimizer.Optimize(iterate); + + // Accumulate snapshots. + if (accumulate) + { + for (size_t i = 0; i < optimizer.DecayPolicy().Snapshots().size(); ++i) + { + iterate += optimizer.DecayPolicy().Snapshots()[i]; + } + iterate /= (optimizer.DecayPolicy().Snapshots().size() + 1); + + // Calculate final objective. + overallObjective = 0; + for (size_t i = 0; i < function.NumFunctions(); ++i) + overallObjective += function.Evaluate(iterate, i); + } + + return overallObjective; +} + +} // namespace optimization +} // namespace mlpack + +#endif diff --git a/src/mlpack/core/optimizers/smorms3/smorms3_update.hpp b/src/mlpack/core/optimizers/smorms3/smorms3_update.hpp index 3cf35eb7df..67e332f625 100644 --- a/src/mlpack/core/optimizers/smorms3/smorms3_update.hpp +++ b/src/mlpack/core/optimizers/smorms3/smorms3_update.hpp @@ -98,14 +98,14 @@ class SMORMS3Update double& Epsilon() { return epsilon; } private: - //! The value used to initialise the mean squared gradient parameter. - double epsilon; + //! The value used to initialise the mean squared gradient parameter. + double epsilon; - // The parameters mem, g and g2. - arma::mat mem, g, g2; + // The parameters mem, g and g2. + arma::mat mem, g, g2; }; } // namespace optimization } // namespace mlpack -#endif \ No newline at end of file +#endif diff --git a/src/mlpack/core/tree/address.hpp b/src/mlpack/core/tree/address.hpp index b1ab4951fe..03c8cb4f9c 100644 --- a/src/mlpack/core/tree/address.hpp +++ b/src/mlpack/core/tree/address.hpp @@ -82,7 +82,7 @@ void PointToAddress(AddressType& address, const VecType& point) for (size_t i = 0; i < point.n_elem; i++) { int e; - VecElemType normalizedVal = std::frexp(point(i),&e); + VecElemType normalizedVal = std::frexp(point(i), &e); bool sgn = std::signbit(normalizedVal); if (point(i) == 0) @@ -262,6 +262,6 @@ bool Contains(const AddressType1& address, const AddressType2& loBound, } // namespace addr } // namespace bound -} // namespave mlpack +} // namespace mlpack #endif // MLPACK_CORE_TREE_ADDRESS_HPP diff --git a/src/mlpack/core/tree/ballbound.hpp b/src/mlpack/core/tree/ballbound.hpp index 3ae7373c4e..a036a5fbd8 100644 --- a/src/mlpack/core/tree/ballbound.hpp +++ b/src/mlpack/core/tree/ballbound.hpp @@ -54,7 +54,6 @@ class BallBound bool ownsMetric; public: - //! Empty Constructor. BallBound(); diff --git a/src/mlpack/core/tree/binary_space_tree/binary_space_tree_impl.hpp b/src/mlpack/core/tree/binary_space_tree/binary_space_tree_impl.hpp index fd3cbf387c..cea6e71895 100644 --- a/src/mlpack/core/tree/binary_space_tree/binary_space_tree_impl.hpp +++ b/src/mlpack/core/tree/binary_space_tree/binary_space_tree_impl.hpp @@ -413,7 +413,7 @@ BinarySpaceTree(BinarySpaceTree&& other) : other.minimumBoundDistance = 0.0; other.dataset = NULL; - //Set new parent. + // Set new parent. if (left) left->parent = this; if (right) diff --git a/src/mlpack/core/tree/binary_space_tree/breadth_first_dual_tree_traverser.hpp b/src/mlpack/core/tree/binary_space_tree/breadth_first_dual_tree_traverser.hpp index 90c361bc97..b313b5a725 100644 --- a/src/mlpack/core/tree/binary_space_tree/breadth_first_dual_tree_traverser.hpp +++ b/src/mlpack/core/tree/binary_space_tree/breadth_first_dual_tree_traverser.hpp @@ -12,8 +12,8 @@ * 3-clause BSD license along with mlpack. If not, see * http://www.opensource.org/licenses/BSD-3-Clause for more information. */ -#ifndef MLPACK_CORE_TREE_BINARY_SPACE_TREE_BREADTH_FIRST_DUAL_TREE_TRAVERSER_HPP -#define MLPACK_CORE_TREE_BINARY_SPACE_TREE_BREADTH_FIRST_DUAL_TREE_TRAVERSER_HPP +#ifndef MLPACK_CORE_TREE_BINARY_SPACE_TREE_BF_DUAL_TREE_TRAVERSER_HPP +#define MLPACK_CORE_TREE_BINARY_SPACE_TREE_BF_DUAL_TREE_TRAVERSER_HPP #include #include @@ -111,5 +111,4 @@ class BinarySpaceTree::Traverse( } // namespace tree } // namespace mlpack -#endif // MLPACK_CORE_TREE_BINARY_SPACE_TREE_BREADTH_FIRST_DUAL_TREE_TRAVERSER_IMPL_HPP +#endif // MLPACK_CORE_TREE_BINARY_SPACE_TREE_BF_DUAL_TREE_TRAVERSER_IMPL_HPP diff --git a/src/mlpack/core/tree/binary_space_tree/rp_tree_mean_split.hpp b/src/mlpack/core/tree/binary_space_tree/rp_tree_mean_split.hpp index 3d8d8a6f8c..6dd08a086b 100644 --- a/src/mlpack/core/tree/binary_space_tree/rp_tree_mean_split.hpp +++ b/src/mlpack/core/tree/binary_space_tree/rp_tree_mean_split.hpp @@ -132,7 +132,6 @@ class RPTreeMeanSplit } private: - /** * Get the average distance between points in the dataset. * diff --git a/src/mlpack/core/tree/binary_space_tree/ub_tree_split.hpp b/src/mlpack/core/tree/binary_space_tree/ub_tree_split.hpp index 150cbd27a1..5d0f755954 100644 --- a/src/mlpack/core/tree/binary_space_tree/ub_tree_split.hpp +++ b/src/mlpack/core/tree/binary_space_tree/ub_tree_split.hpp @@ -30,9 +30,10 @@ class UBTreeSplit { public: //! The type of an address element. - typedef typename std::conditional::type AddressElemType; + typedef typename std::conditional< + sizeof(typename MatType::elem_type) * CHAR_BIT <= 32, + uint32_t, + uint64_t>::type AddressElemType; //! An information about the partition. struct SplitInfo diff --git a/src/mlpack/core/tree/cellbound_impl.hpp b/src/mlpack/core/tree/cellbound_impl.hpp index a597cf25b7..245ec4ce3d 100644 --- a/src/mlpack/core/tree/cellbound_impl.hpp +++ b/src/mlpack/core/tree/cellbound_impl.hpp @@ -82,7 +82,9 @@ inline CellBound::CellBound( * Same as the copy constructor. */ template -inline CellBound& CellBound::operator=( +inline CellBound< + MetricType, + ElemType>& CellBound::operator=( const CellBound& other) { if (dim != other.Dim()) @@ -486,9 +488,10 @@ inline ElemType CellBound::MinDistance( lower = loBound(d, i) - point[d]; higher = point[d] - hiBound(d, i); - // Since only one of 'lower' or 'higher' is negative, if we add each's - // absolute value to itself and then sum those two, our result is the - // nonnegative half of the equation times two; then we raise to power Power. + // Since only one of 'lower' or 'higher' is negative, if we add + // each's absolute value to itself and then sum those two, our + // result is the non negative half of the equation times two; + // then we raise to power Power. if (MetricType::Power == 1) sum += lower + std::fabs(lower) + higher + std::fabs(higher); else if (MetricType::Power == 2) @@ -864,8 +867,8 @@ CellBound::RangeDistance( */ template template -inline CellBound& CellBound::operator|=( - const MatType& data) +inline CellBound& +CellBound::operator|=(const MatType& data) { Log::Assert(data.n_rows == dim); @@ -893,8 +896,8 @@ inline CellBound& CellBound::operato * Expands this region to encompass another bound. */ template -inline CellBound& CellBound::operator|=( - const CellBound& other) +inline CellBound& +CellBound::operator|=(const CellBound& other) { assert(other.dim == dim); @@ -930,7 +933,8 @@ inline CellBound& CellBound::operato */ template template -inline bool CellBound::Contains(const VecType& point) const +inline bool CellBound::Contains( + const VecType& point) const { for (size_t i = 0; i < point.n_elem; i++) { diff --git a/src/mlpack/core/tree/cosine_tree/cosine_tree.hpp b/src/mlpack/core/tree/cosine_tree/cosine_tree.hpp index 8e1155dd22..a94562afba 100644 --- a/src/mlpack/core/tree/cosine_tree/cosine_tree.hpp +++ b/src/mlpack/core/tree/cosine_tree/cosine_tree.hpp @@ -246,7 +246,6 @@ class CosineTree class CompareCosineNode { public: - // Comparison function for construction of priority queue. bool operator() (const CosineTree* a, const CosineTree* b) const { diff --git a/src/mlpack/core/tree/cover_tree/dual_tree_traverser_impl.hpp b/src/mlpack/core/tree/cover_tree/dual_tree_traverser_impl.hpp index 64dadf3246..692ec9d59e 100644 --- a/src/mlpack/core/tree/cover_tree/dual_tree_traverser_impl.hpp +++ b/src/mlpack/core/tree/cover_tree/dual_tree_traverser_impl.hpp @@ -297,7 +297,8 @@ DualTreeTraverser::ReferenceRecursion( break; // Get a reference to the current largest scale. - std::vector& scaleVector = (*referenceMap.rbegin()).second; + std::vector& scaleVector = + (*referenceMap.rbegin()).second; // Before traversing all the points in this scale, sort by score. std::sort(scaleVector.begin(), scaleVector.end()); diff --git a/src/mlpack/core/tree/hollow_ball_bound.hpp b/src/mlpack/core/tree/hollow_ball_bound.hpp index 76eda7f0f5..a4ba7772ba 100644 --- a/src/mlpack/core/tree/hollow_ball_bound.hpp +++ b/src/mlpack/core/tree/hollow_ball_bound.hpp @@ -55,7 +55,6 @@ class HollowBallBound bool ownsMetric; public: - //! Empty Constructor. HollowBallBound(); diff --git a/src/mlpack/core/tree/hollow_ball_bound_impl.hpp b/src/mlpack/core/tree/hollow_ball_bound_impl.hpp index fa0a8b2b82..182097ff4e 100644 --- a/src/mlpack/core/tree/hollow_ball_bound_impl.hpp +++ b/src/mlpack/core/tree/hollow_ball_bound_impl.hpp @@ -33,7 +33,8 @@ HollowBallBound::HollowBallBound() : * @param dimension Dimensionality of ball bound. */ template -HollowBallBound::HollowBallBound(const size_t dimension) : +HollowBallBound:: +HollowBallBound(const size_t dimension) : radii(std::numeric_limits::lowest(), std::numeric_limits::lowest()), center(dimension), diff --git a/src/mlpack/core/tree/hrectbound.hpp b/src/mlpack/core/tree/hrectbound.hpp index da2cd7ef71..d54f6a24c0 100644 --- a/src/mlpack/core/tree/hrectbound.hpp +++ b/src/mlpack/core/tree/hrectbound.hpp @@ -39,7 +39,7 @@ struct IsLMetric> static const bool Value = true; }; -} // namespace util +} // namespace meta /** * Hyper-rectangle bound for an L-metric. This should be used in conjunction diff --git a/src/mlpack/core/tree/hrectbound_impl.hpp b/src/mlpack/core/tree/hrectbound_impl.hpp index 69e4b97374..f262ad4495 100644 --- a/src/mlpack/core/tree/hrectbound_impl.hpp +++ b/src/mlpack/core/tree/hrectbound_impl.hpp @@ -60,8 +60,10 @@ inline HRectBound::HRectBound( * Same as the copy constructor. */ template -inline HRectBound& HRectBound::operator=( - const HRectBound& other) +inline HRectBound< + MetricType, + ElemType>& HRectBound::operator=(const HRectBound& other) { if (dim != other.Dim()) { @@ -208,7 +210,8 @@ inline ElemType HRectBound::MinDistance( else { if (MetricType::TakeRoot) - return (ElemType) pow((double) sum, 1.0 / (double) MetricType::Power) / 2.0; + return (ElemType) pow((double) sum, + 1.0 / (double) MetricType::Power) / 2.0; else return sum / pow(2.0, MetricType::Power); } @@ -268,7 +271,8 @@ ElemType HRectBound::MinDistance(const HRectBound& other) else { if (MetricType::TakeRoot) - return (ElemType) pow((double) sum, 1.0 / (double) MetricType::Power) / 2.0; + return (ElemType) pow((double) sum, + 1.0 / (double) MetricType::Power) / 2.0; else return sum / pow(2.0, MetricType::Power); } @@ -503,8 +507,8 @@ HRectBound::RangeDistance( */ template template -inline HRectBound& HRectBound::operator|=( - const MatType& data) +inline HRectBound& +HRectBound::operator|=(const MatType& data) { Log::Assert(data.n_rows == dim); @@ -527,8 +531,8 @@ inline HRectBound& HRectBound::opera * Expands this region to encompass another bound. */ template -inline HRectBound& HRectBound::operator|=( - const HRectBound& other) +inline HRectBound& +HRectBound::operator|=(const HRectBound& other) { assert(other.dim == dim); @@ -549,7 +553,8 @@ inline HRectBound& HRectBound::opera */ template template -inline bool HRectBound::Contains(const VecType& point) const +inline bool HRectBound::Contains( + const VecType& point) const { for (size_t i = 0; i < point.n_elem; i++) { @@ -572,7 +577,8 @@ inline bool HRectBound::Contains( const math::RangeType& r_a = bounds[i]; const math::RangeType& r_b = bound.bounds[i]; - if (r_a.Hi() <= r_b.Lo() || r_a.Lo() >= r_b.Hi()) // If a does not overlap b at all. + // If a does not overlap b at all. + if (r_a.Hi() <= r_b.Lo() || r_a.Lo() >= r_b.Hi()) return false; } @@ -583,8 +589,8 @@ inline bool HRectBound::Contains( * Returns the intersection of this bound and another. */ template -inline HRectBound HRectBound:: -operator&(const HRectBound& bound) const +inline HRectBound +HRectBound::operator&(const HRectBound& bound) const { HRectBound result(dim); @@ -600,8 +606,8 @@ operator&(const HRectBound& bound) const * Intersects this bound with another. */ template -inline HRectBound& HRectBound:: -operator&=(const HRectBound& bound) +inline HRectBound& +HRectBound::operator&=(const HRectBound& bound) { for (size_t k = 0; k < dim; k++) { diff --git a/src/mlpack/core/tree/octree/dual_tree_traverser_impl.hpp b/src/mlpack/core/tree/octree/dual_tree_traverser_impl.hpp index 7bf14c23b5..81eeec3151 100644 --- a/src/mlpack/core/tree/octree/dual_tree_traverser_impl.hpp +++ b/src/mlpack/core/tree/octree/dual_tree_traverser_impl.hpp @@ -135,7 +135,8 @@ void Octree::DualTreeTraverser:: { if (scores[scoreOrder[i]] == DBL_MAX) { - // We don't need to check any more---all children past here are pruned. + // We don't need to check any more + // All children past here are pruned. numPrunes += scoreOrder.n_elem - i; break; } diff --git a/src/mlpack/core/tree/rectangle_tree/discrete_hilbert_value_impl.hpp b/src/mlpack/core/tree/rectangle_tree/discrete_hilbert_value_impl.hpp index f757826464..53b23e36c0 100644 --- a/src/mlpack/core/tree/rectangle_tree/discrete_hilbert_value_impl.hpp +++ b/src/mlpack/core/tree/rectangle_tree/discrete_hilbert_value_impl.hpp @@ -166,7 +166,7 @@ CalculateValue(const VecType& pt, for (size_t i = 0; i < pt.n_rows; i++) { int e; - VecElemType normalizedVal = std::frexp(pt(i),&e); + VecElemType normalizedVal = std::frexp(pt(i), &e); bool sgn = std::signbit(normalizedVal); if (pt(i) == 0) @@ -325,7 +325,7 @@ CompareWith(const VecType& pt, if (numValues == 0) return -1; - return CompareValues(localHilbertValues->col(numValues - 1),val); + return CompareValues(localHilbertValues->col(numValues - 1), val); } template @@ -384,7 +384,7 @@ void DiscreteHilbertValue::InsertNode(TreeType* node) { DiscreteHilbertValue &val = node->AuxiliaryInfo().HilbertValue(); - if (CompareWith(node,val) < 0) + if (CompareWith(node, val) < 0) { localHilbertValues = val.LocalHilbertValues(); numValues = val.NumValues(); @@ -396,7 +396,6 @@ template void DiscreteHilbertValue:: DeletePoint(TreeType* /* node */, const size_t localIndex) { - // Delete the Hilbert value from the local dataset for (size_t i = numValues - 1; i > localIndex; i--) localHilbertValues->col(i - 1) = localHilbertValues->col(i); diff --git a/src/mlpack/core/tree/rectangle_tree/dual_tree_traverser.hpp b/src/mlpack/core/tree/rectangle_tree/dual_tree_traverser.hpp index e2a390db7d..8f4607c720 100644 --- a/src/mlpack/core/tree/rectangle_tree/dual_tree_traverser.hpp +++ b/src/mlpack/core/tree/rectangle_tree/dual_tree_traverser.hpp @@ -67,7 +67,6 @@ class RectangleTree::Traverse(RectangleTree& queryNode, if (childScore == DBL_MAX) continue; // We don't require a search in this reference node. - for(size_t ref = 0; ref < referenceNode.Count(); ++ref) + for (size_t ref = 0; ref < referenceNode.Count(); ++ref) rule.BaseCase(queryNode.Point(query), referenceNode.Point(ref)); numBaseCases += referenceNode.Count(); diff --git a/src/mlpack/core/tree/rectangle_tree/hilbert_r_tree_auxiliary_information.hpp b/src/mlpack/core/tree/rectangle_tree/hilbert_r_tree_auxiliary_information.hpp index c80e2d05c3..42e3fd8fe5 100644 --- a/src/mlpack/core/tree/rectangle_tree/hilbert_r_tree_auxiliary_information.hpp +++ b/src/mlpack/core/tree/rectangle_tree/hilbert_r_tree_auxiliary_information.hpp @@ -11,8 +11,8 @@ * 3-clause BSD license along with mlpack. If not, see * http://www.opensource.org/licenses/BSD-3-Clause for more information. */ -#ifndef MLPACK_CORE_TREE_RECTANGLE_TREE_HILBERT_R_TREE_AUXILIARY_INFORMATION_HPP -#define MLPACK_CORE_TREE_RECTANGLE_TREE_HILBERT_R_TREE_AUXILIARY_INFORMATION_HPP +#ifndef MLPACK_CORE_TREE_RECTANGLE_TREE_HR_TREE_AUXILIARY_INFO_HPP +#define MLPACK_CORE_TREE_RECTANGLE_TREE_HR_TREE_AUXILIARY_INFO_HPP namespace mlpack { namespace tree { @@ -147,4 +147,4 @@ class HilbertRTreeAuxiliaryInformation #include "hilbert_r_tree_auxiliary_information_impl.hpp" -#endif//MLPACK_CORE_TREE_RECTANGLE_TREE_HILBERT_R_TREE_AUXILIARY_INFORMATION_HPP +#endif // MLPACK_CORE_TREE_RECTANGLE_TREE_HR_TREE_AUXILIARY_INFO_HPP diff --git a/src/mlpack/core/tree/rectangle_tree/hilbert_r_tree_auxiliary_information_impl.hpp b/src/mlpack/core/tree/rectangle_tree/hilbert_r_tree_auxiliary_information_impl.hpp index 2fceeede21..e23e929431 100644 --- a/src/mlpack/core/tree/rectangle_tree/hilbert_r_tree_auxiliary_information_impl.hpp +++ b/src/mlpack/core/tree/rectangle_tree/hilbert_r_tree_auxiliary_information_impl.hpp @@ -10,8 +10,8 @@ * 3-clause BSD license along with mlpack. If not, see * http://www.opensource.org/licenses/BSD-3-Clause for more information. */ -#ifndef MLPACK_CORE_TREE_RECTANGLE_TREE_HILBERT_R_TREE_AUXILIARY_INFORMATION_IMPL_HPP -#define MLPACK_CORE_TREE_RECTANGLE_TREE_HILBERT_R_TREE_AUXILIARY_INFORMATION_IMPL_HPP +#ifndef MLPACK_CORE_TREE_RECTANGLE_TREE_HR_TREE_AUXILIARY_INFO_IMPL_HPP +#define MLPACK_CORE_TREE_RECTANGLE_TREE_HR_TREE_AUXILIARY_INFO_IMPL_HPP #include "hilbert_r_tree_auxiliary_information.hpp" @@ -126,7 +126,7 @@ bool HilbertRTreeAuxiliaryInformation:: HandlePointDeletion(TreeType* node, const size_t localIndex) { // Update the largest Hilbert value. - hilbertValue.DeletePoint(node,localIndex); + hilbertValue.DeletePoint(node, localIndex); for (size_t i = localIndex + 1; localIndex < node->NumPoints(); i++) node->Point(i - 1) = node->Point(i); @@ -141,7 +141,7 @@ bool HilbertRTreeAuxiliaryInformation:: HandleNodeRemoval(TreeType* node, const size_t nodeIndex) { // Update the largest Hilbert value. - hilbertValue.RemoveNode(node,nodeIndex); + hilbertValue.RemoveNode(node, nodeIndex); for (size_t i = nodeIndex + 1; nodeIndex < node->NumChildren(); i++) node->children[i - 1] = node->children[i]; @@ -178,7 +178,7 @@ NullifyData() template class HilbertValueType> template -void HilbertRTreeAuxiliaryInformation:: +void HilbertRTreeAuxiliaryInformation:: Serialize(Archive& ar, const unsigned int /* version */) { using data::CreateNVP; @@ -190,4 +190,4 @@ Serialize(Archive& ar, const unsigned int /* version */) } // namespace tree } // namespace mlpack -#endif//MLPACK_CORE_TREE_RECTANGLE_TREE_HILBERT_R_TREE_AUXILIARY_INFORMATION_IMPL_HPP +#endif // MLPACK_CORE_TREE_RECTANGLE_TREE_HR_TREE_AUXILIARY_INFO_IMPL_HPP diff --git a/src/mlpack/core/tree/rectangle_tree/hilbert_r_tree_descent_heuristic.hpp b/src/mlpack/core/tree/rectangle_tree/hilbert_r_tree_descent_heuristic.hpp index 1a564706dc..74086715f4 100644 --- a/src/mlpack/core/tree/rectangle_tree/hilbert_r_tree_descent_heuristic.hpp +++ b/src/mlpack/core/tree/rectangle_tree/hilbert_r_tree_descent_heuristic.hpp @@ -10,8 +10,8 @@ * 3-clause BSD license along with mlpack. If not, see * http://www.opensource.org/licenses/BSD-3-Clause for more information. */ -#ifndef MLPACK_CORE_TREE_RECTANGLE_TREE_HILBERT_R_TREE_DESCENT_HEURISTIC_HPP -#define MLPACK_CORE_TREE_RECTANGLE_TREE_HILBERT_R_TREE_DESCENT_HEURISTIC_HPP +#ifndef MLPACK_CORE_TREE_RECTANGLE_TREE_HR_TREE_DESCENT_HEURISTIC_HPP +#define MLPACK_CORE_TREE_RECTANGLE_TREE_HR_TREE_DESCENT_HEURISTIC_HPP #include @@ -55,4 +55,4 @@ class HilbertRTreeDescentHeuristic #include "hilbert_r_tree_descent_heuristic_impl.hpp" -#endif // MLPACK_CORE_TREE_RECTANGLE_TREE_HILBERT_R_TREE_DESCENT_HEURISTIC_HPP +#endif // MLPACK_CORE_TREE_RECTANGLE_TREE_HR_TREE_DESCENT_HEURISTIC_HPP diff --git a/src/mlpack/core/tree/rectangle_tree/hilbert_r_tree_descent_heuristic_impl.hpp b/src/mlpack/core/tree/rectangle_tree/hilbert_r_tree_descent_heuristic_impl.hpp index b7f4e08396..64a337552e 100644 --- a/src/mlpack/core/tree/rectangle_tree/hilbert_r_tree_descent_heuristic_impl.hpp +++ b/src/mlpack/core/tree/rectangle_tree/hilbert_r_tree_descent_heuristic_impl.hpp @@ -10,8 +10,8 @@ * 3-clause BSD license along with mlpack. If not, see * http://www.opensource.org/licenses/BSD-3-Clause for more information. */ -#ifndef MLPACK_CORE_TREE_RECTANGLE_TREE_HILBERT_R_TREE_DESCENT_HEURISTIC_IMPL_HPP -#define MLPACK_CORE_TREE_RECTANGLE_TREE_HILBERT_R_TREE_DESCENT_HEURISTIC_IMPL_HPP +#ifndef MLPACK_CORE_TREE_RECTANGLE_TREE_HR_TREE_DESCENT_HEURISTIC_IMPL_HPP +#define MLPACK_CORE_TREE_RECTANGLE_TREE_HR_TREE_DESCENT_HEURISTIC_IMPL_HPP #include "hilbert_r_tree_descent_heuristic.hpp" @@ -51,4 +51,4 @@ size_t HilbertRTreeDescentHeuristic::ChooseDescentNode( } // namespace tree } // namespace mlpack -#endif // MLPACK_CORE_TREE_RECTANGLE_TREE_HILBERT_R_TREE_DESCENT_HEURISTIC_IMPL_HPP +#endif // MLPACK_CORE_TREE_RECTANGLE_TREE_HR_TREE_DESCENT_HEURISTIC_IMPL_HPP diff --git a/src/mlpack/core/tree/rectangle_tree/hilbert_r_tree_split_impl.hpp b/src/mlpack/core/tree/rectangle_tree/hilbert_r_tree_split_impl.hpp index 32ca6b339a..600665c7bb 100644 --- a/src/mlpack/core/tree/rectangle_tree/hilbert_r_tree_split_impl.hpp +++ b/src/mlpack/core/tree/rectangle_tree/hilbert_r_tree_split_impl.hpp @@ -48,7 +48,7 @@ void HilbertRTreeSplit::SplitLeafNode(TreeType* tree, TreeType* parent = tree->Parent(); size_t iTree = 0; - for (iTree = 0; parent->children[iTree] != tree; iTree++); + for (iTree = 0; parent->children[iTree] != tree; iTree++) { } // Try to find splitOrder cooperating siblings in order to redistribute points // among them and avoid split. @@ -112,7 +112,7 @@ SplitNonLeafNode(TreeType* tree, std::vector& relevels) TreeType* parent = tree->Parent(); size_t iTree = 0; - for (iTree = 0; parent->children[iTree] != tree; iTree++); + for (iTree = 0; parent->children[iTree] != tree; iTree++) { } // Try to find splitOrder cooperating siblings in order to redistribute // children among them and avoid split. diff --git a/src/mlpack/core/tree/rectangle_tree/minimal_splits_number_sweep_impl.hpp b/src/mlpack/core/tree/rectangle_tree/minimal_splits_number_sweep_impl.hpp index c0a57ee1e7..9d3e8c28f6 100644 --- a/src/mlpack/core/tree/rectangle_tree/minimal_splits_number_sweep_impl.hpp +++ b/src/mlpack/core/tree/rectangle_tree/minimal_splits_number_sweep_impl.hpp @@ -70,8 +70,9 @@ size_t MinimalSplitsNumberSweep::SweepNonLeafNode( } // Check if the split is possible. - if (numTreeOneChildren <= node->MaxNumChildren() && numTreeOneChildren > 0 && - numTreeTwoChildren <= node->MaxNumChildren() && numTreeTwoChildren > 0) + if (numTreeOneChildren <= node->MaxNumChildren() && + numTreeOneChildren > 0 && numTreeTwoChildren <= node->MaxNumChildren() + && numTreeTwoChildren > 0) { // Evaluate the cost using the number of splits and balancing. size_t balance; diff --git a/src/mlpack/core/tree/rectangle_tree/no_auxiliary_information.hpp b/src/mlpack/core/tree/rectangle_tree/no_auxiliary_information.hpp index 01d18db371..240bf6677f 100644 --- a/src/mlpack/core/tree/rectangle_tree/no_auxiliary_information.hpp +++ b/src/mlpack/core/tree/rectangle_tree/no_auxiliary_information.hpp @@ -21,15 +21,15 @@ class NoAuxiliaryInformation { public: //! Construct the auxiliary information object. - NoAuxiliaryInformation() { }; + NoAuxiliaryInformation() { } //! Construct the auxiliary information object. - NoAuxiliaryInformation(const TreeType* /* node */) { }; + NoAuxiliaryInformation(const TreeType* /* node */) { } //! Construct the auxiliary information object. NoAuxiliaryInformation(const NoAuxiliaryInformation& /* other */, TreeType* /* tree */, - bool /* deepCopy */ = true) { }; + bool /* deepCopy */ = true) { } //! Construct the auxiliary information object. - NoAuxiliaryInformation(NoAuxiliaryInformation&& /* other */) { }; + NoAuxiliaryInformation(NoAuxiliaryInformation&& /* other */) { } //! Copy the auxiliary information object. NoAuxiliaryInformation& operator=(const NoAuxiliaryInformation& /* other */) @@ -141,7 +141,7 @@ class NoAuxiliaryInformation * Serialize the information. */ template - void Serialize(Archive &, const unsigned int /* version */) { }; + void Serialize(Archive &, const unsigned int /* version */) { } }; } // namespace tree diff --git a/src/mlpack/core/tree/rectangle_tree/r_plus_plus_tree_auxiliary_information.hpp b/src/mlpack/core/tree/rectangle_tree/r_plus_plus_tree_auxiliary_information.hpp index 6893352cde..55d6cbe9da 100644 --- a/src/mlpack/core/tree/rectangle_tree/r_plus_plus_tree_auxiliary_information.hpp +++ b/src/mlpack/core/tree/rectangle_tree/r_plus_plus_tree_auxiliary_information.hpp @@ -11,8 +11,8 @@ * 3-clause BSD license along with mlpack. If not, see * http://www.opensource.org/licenses/BSD-3-Clause for more information. */ -#ifndef MLPACK_CORE_TREE_RECTANGLE_TREE_R_PLUS_PLUS_TREE_AUXILIARY_INFORMATION_HPP -#define MLPACK_CORE_TREE_RECTANGLE_TREE_R_PLUS_PLUS_TREE_AUXILIARY_INFORMATION_HPP +#ifndef MLPACK_CORE_TREE_RECTANGLE_TREE_RPP_TREE_AUXILIARY_INFO_HPP +#define MLPACK_CORE_TREE_RECTANGLE_TREE_RPP_TREE_AUXILIARY_INFO_HPP #include #include "../hrectbound.hpp" @@ -163,4 +163,4 @@ class RPlusPlusTreeAuxiliaryInformation #include "r_plus_plus_tree_auxiliary_information_impl.hpp" -#endif//MLPACK_CORE_TREE_RECTANGLE_TREE_R_PLUS_PLUS_TREE_AUXILIARY_INFORMATION_HPP +#endif // MLPACK_CORE_TREE_RECTANGLE_TREE_RPP_TREE_AUXILIARY_INFO_HPP diff --git a/src/mlpack/core/tree/rectangle_tree/r_plus_plus_tree_auxiliary_information_impl.hpp b/src/mlpack/core/tree/rectangle_tree/r_plus_plus_tree_auxiliary_information_impl.hpp index 06f8becb3b..b19149539c 100644 --- a/src/mlpack/core/tree/rectangle_tree/r_plus_plus_tree_auxiliary_information_impl.hpp +++ b/src/mlpack/core/tree/rectangle_tree/r_plus_plus_tree_auxiliary_information_impl.hpp @@ -11,8 +11,8 @@ * 3-clause BSD license along with mlpack. If not, see * http://www.opensource.org/licenses/BSD-3-Clause for more information. */ -#ifndef MLPACK_CORE_TREE_RECTANGLE_TREE_R_PLUS_PLUS_TREE_AUXILIARY_INFORMATION_IMPL_HPP -#define MLPACK_CORE_TREE_RECTANGLE_TREE_R_PLUS_PLUS_TREE_AUXILIARY_INFORMATION_IMPL_HPP +#ifndef MLPACK_CORE_TREE_RECTANGLE_TREE_RPP_TREE_AUXILIARY_INFO_IMPL_HPP +#define MLPACK_CORE_TREE_RECTANGLE_TREE_RPP_TREE_AUXILIARY_INFO_IMPL_HPP #include "r_plus_plus_tree_auxiliary_information.hpp" @@ -23,9 +23,7 @@ template RPlusPlusTreeAuxiliaryInformation:: RPlusPlusTreeAuxiliaryInformation() : outerBound(0) -{ - -} +{ /* Nothing to do. */ } template RPlusPlusTreeAuxiliaryInformation:: @@ -36,11 +34,13 @@ RPlusPlusTreeAuxiliaryInformation(const TreeType* tree) : { // Initialize the maximum bounding rectangle if the node is the root if (!tree->Parent()) + { for (size_t k = 0; k < outerBound.Dim(); k++) { outerBound[k].Lo() = std::numeric_limits::lowest(); outerBound[k].Hi() = std::numeric_limits::max(); } + } } template @@ -50,17 +50,13 @@ RPlusPlusTreeAuxiliaryInformation( TreeType* /* tree */, bool /* deepCopy */) : outerBound(other.OuterBound()) -{ - -} +{ /* Nothing to do. */ } template RPlusPlusTreeAuxiliaryInformation:: RPlusPlusTreeAuxiliaryInformation(RPlusPlusTreeAuxiliaryInformation&& other) : outerBound(std::move(other.outerBound)) -{ - -} +{ /* Nothing to do. */ } template bool RPlusPlusTreeAuxiliaryInformation::HandlePointInsertion( @@ -122,9 +118,7 @@ void RPlusPlusTreeAuxiliaryInformation::SplitAuxiliaryInfo( template void RPlusPlusTreeAuxiliaryInformation::NullifyData() -{ - -} +{ /* Nothing to do */ } /** * Serialize the information. @@ -142,4 +136,4 @@ Serialize(Archive& ar, const unsigned int /* version */) } // namespace tree } // namespace mlpack -#endif // MLPACK_CORE_TREE_RECTANGLE_TREE_R_PLUS_PLUS_TREE_AUXILIARY_INFORMATION_IMPL_HPP +#endif // MLPACK_CORE_TREE_RECTANGLE_TREE_RPP_TREE_AUXILIARY_INFO_IMPL_HPP diff --git a/src/mlpack/core/tree/rectangle_tree/r_plus_plus_tree_descent_heuristic.hpp b/src/mlpack/core/tree/rectangle_tree/r_plus_plus_tree_descent_heuristic.hpp index c813470db5..e5efc491b4 100644 --- a/src/mlpack/core/tree/rectangle_tree/r_plus_plus_tree_descent_heuristic.hpp +++ b/src/mlpack/core/tree/rectangle_tree/r_plus_plus_tree_descent_heuristic.hpp @@ -10,8 +10,8 @@ * 3-clause BSD license along with mlpack. If not, see * http://www.opensource.org/licenses/BSD-3-Clause for more information. */ -#ifndef MLPACK_CORE_TREE_RECTANGLE_TREE_R_PLUS_PLUS_TREE_DESCENT_HEURISTIC_HPP -#define MLPACK_CORE_TREE_RECTANGLE_TREE_R_PLUS_PLUS_TREE_DESCENT_HEURISTIC_HPP +#ifndef MLPACK_CORE_TREE_RECTANGLE_TREE_RPP_TREE_DESCENT_HEURISTIC_HPP +#define MLPACK_CORE_TREE_RECTANGLE_TREE_RPP_TREE_DESCENT_HEURISTIC_HPP #include @@ -50,4 +50,4 @@ class RPlusPlusTreeDescentHeuristic #include "r_plus_plus_tree_descent_heuristic_impl.hpp" -#endif // MLPACK_CORE_TREE_RECTANGLE_TREE_R_PLUS_PLUS_TREE_DESCENT_HEURISTIC_HPP +#endif // MLPACK_CORE_TREE_RECTANGLE_TREE_RPP_TREE_DESCENT_HEURISTIC_HPP diff --git a/src/mlpack/core/tree/rectangle_tree/r_plus_plus_tree_descent_heuristic_impl.hpp b/src/mlpack/core/tree/rectangle_tree/r_plus_plus_tree_descent_heuristic_impl.hpp index 141ae0350d..66f47c1365 100644 --- a/src/mlpack/core/tree/rectangle_tree/r_plus_plus_tree_descent_heuristic_impl.hpp +++ b/src/mlpack/core/tree/rectangle_tree/r_plus_plus_tree_descent_heuristic_impl.hpp @@ -10,8 +10,8 @@ * 3-clause BSD license along with mlpack. If not, see * http://www.opensource.org/licenses/BSD-3-Clause for more information. */ -#ifndef MLPACK_CORE_TREE_RECTANGLE_TREE_R_PLUS_PLUS_TREE_DESCENT_HEURISTIC_IMPL_HPP -#define MLPACK_CORE_TREE_RECTANGLE_TREE_R_PLUS_PLUS_TREE_DESCENT_HEURISTIC_IMPL_HPP +#ifndef MLPACK_CORE_TREE_RECTANGLE_TREE_RPP_TREE_DESCENT_HEURISTIC_IMPL_HPP +#define MLPACK_CORE_TREE_RECTANGLE_TREE_RPP_TREE_DESCENT_HEURISTIC_IMPL_HPP #include "r_plus_plus_tree_descent_heuristic.hpp" #include "../hrectbound.hpp" @@ -50,4 +50,4 @@ size_t RPlusPlusTreeDescentHeuristic::ChooseDescentNode( } // namespace tree } // namespace mlpack -#endif // MLPACK_CORE_TREE_RECTANGLE_TREE_R_PLUS_PLUS_TREE_DESCENT_HEURISTIC_IMPL_HPP +#endif // MLPACK_CORE_TREE_RECTANGLE_TREE_RPP_TREE_DESCENT_HEURISTIC_IMPL_HPP diff --git a/src/mlpack/core/tree/rectangle_tree/r_plus_tree_split_impl.hpp b/src/mlpack/core/tree/rectangle_tree/r_plus_tree_split_impl.hpp index 66f9e96694..0293f341e0 100644 --- a/src/mlpack/core/tree/rectangle_tree/r_plus_tree_split_impl.hpp +++ b/src/mlpack/core/tree/rectangle_tree/r_plus_tree_split_impl.hpp @@ -137,7 +137,7 @@ SplitNonLeafNode(TreeType* tree, std::vector& relevels) tree->NullifyData(); tree->children[(tree->NumChildren())++] = copy; - RPlusTreeSplit::SplitNonLeafNode(copy,relevels); + RPlusTreeSplit::SplitNonLeafNode(copy, relevels); return true; } size_t cutAxis = tree->Bound().Dim(); diff --git a/src/mlpack/core/tree/rectangle_tree/r_star_tree_split.hpp b/src/mlpack/core/tree/rectangle_tree/r_star_tree_split.hpp index f0c5f5845f..300d829ef1 100644 --- a/src/mlpack/core/tree/rectangle_tree/r_star_tree_split.hpp +++ b/src/mlpack/core/tree/rectangle_tree/r_star_tree_split.hpp @@ -32,14 +32,14 @@ class RStarTreeSplit * necessary, this split will propagate upwards through the tree. */ template - static void SplitLeafNode(TreeType *tree,std::vector& relevels); + static void SplitLeafNode(TreeType *tree, std::vector& relevels); /** * Split a non-leaf node using the "default" algorithm. If this is a root * node, the tree increases in depth. */ template - static bool SplitNonLeafNode(TreeType *tree,std::vector& relevels); + static bool SplitNonLeafNode(TreeType *tree, std::vector& relevels); /** * Reinsert any points into the tree, if needed. This returns the number of @@ -52,7 +52,10 @@ class RStarTreeSplit * Given a node, return the best dimension and the best index to split on. */ template - static void PickLeafSplit(TreeType* tree, size_t& bestAxis, size_t& bestIndex); + static void PickLeafSplit( + TreeType* tree, + size_t& bestAxis, + size_t& bestIndex); private: /** diff --git a/src/mlpack/core/tree/rectangle_tree/r_star_tree_split_impl.hpp b/src/mlpack/core/tree/rectangle_tree/r_star_tree_split_impl.hpp index 6fe28a6407..6452cb92b2 100644 --- a/src/mlpack/core/tree/rectangle_tree/r_star_tree_split_impl.hpp +++ b/src/mlpack/core/tree/rectangle_tree/r_star_tree_split_impl.hpp @@ -174,7 +174,7 @@ void RStarTreeSplit::PickLeafSplit(TreeType* tree, * new nodes into the tree, spliting the parent if necessary. */ template -void RStarTreeSplit::SplitLeafNode(TreeType *tree,std::vector& relevels) +void RStarTreeSplit::SplitLeafNode(TreeType *tree, std::vector& relevels) { // Convenience typedef. typedef typename TreeType::ElemType ElemType; @@ -268,7 +268,9 @@ void RStarTreeSplit::SplitLeafNode(TreeType *tree,std::vector& relevels) * higher up the tree because they were already updated if necessary. */ template -bool RStarTreeSplit::SplitNonLeafNode(TreeType *tree,std::vector& relevels) +bool RStarTreeSplit::SplitNonLeafNode( + TreeType *tree, + std::vector& relevels) { // Convenience typedef. typedef typename TreeType::ElemType ElemType; diff --git a/src/mlpack/core/tree/rectangle_tree/r_tree_split.hpp b/src/mlpack/core/tree/rectangle_tree/r_tree_split.hpp index 0375876bba..b140454e5a 100644 --- a/src/mlpack/core/tree/rectangle_tree/r_tree_split.hpp +++ b/src/mlpack/core/tree/rectangle_tree/r_tree_split.hpp @@ -31,27 +31,27 @@ class RTreeSplit * will propagate upwards through the tree. */ template - static void SplitLeafNode(TreeType *tree,std::vector& relevels); + static void SplitLeafNode(TreeType *tree, std::vector& relevels); /** * Split a non-leaf node using the "default" algorithm. If this is a root * node, the tree increases in depth. */ template - static bool SplitNonLeafNode(TreeType *tree,std::vector& relevels); + static bool SplitNonLeafNode(TreeType *tree, std::vector& relevels); private: /** * Get the seeds for splitting a leaf node. */ template - static void GetPointSeeds(const TreeType *tree,int& i, int& j); + static void GetPointSeeds(const TreeType *tree, int& i, int& j); /** * Get the seeds for splitting a non-leaf node. */ template - static void GetBoundSeeds(const TreeType *tree,int& i, int& j); + static void GetBoundSeeds(const TreeType *tree, int& i, int& j); /** * Assign points to the two new nodes. diff --git a/src/mlpack/core/tree/rectangle_tree/r_tree_split_impl.hpp b/src/mlpack/core/tree/rectangle_tree/r_tree_split_impl.hpp index c64c455973..02b1427c85 100644 --- a/src/mlpack/core/tree/rectangle_tree/r_tree_split_impl.hpp +++ b/src/mlpack/core/tree/rectangle_tree/r_tree_split_impl.hpp @@ -26,7 +26,7 @@ namespace tree { * new nodes into the tree, spliting the parent if necessary. */ template -void RTreeSplit::SplitLeafNode(TreeType *tree,std::vector& relevels) +void RTreeSplit::SplitLeafNode(TreeType *tree, std::vector& relevels) { if (tree->Count() <= tree->MaxLeafSize()) return; @@ -42,7 +42,7 @@ void RTreeSplit::SplitLeafNode(TreeType *tree,std::vector& relevels) tree->NullifyData(); // Because this was a leaf node, numChildren must be 0. tree->children[(tree->NumChildren())++] = copy; - RTreeSplit::SplitLeafNode(copy,relevels); + RTreeSplit::SplitLeafNode(copy, relevels); return; } @@ -53,7 +53,7 @@ void RTreeSplit::SplitLeafNode(TreeType *tree,std::vector& relevels) // rectangles, only points. We assume that the tree uses Euclidean Distance. int i = 0; int j = 0; - RTreeSplit::GetPointSeeds(tree,i, j); + RTreeSplit::GetPointSeeds(tree, i, j); TreeType* treeOne = new TreeType(tree->Parent()); TreeType* treeTwo = new TreeType(tree->Parent()); @@ -73,7 +73,7 @@ void RTreeSplit::SplitLeafNode(TreeType *tree,std::vector& relevels) // just in case, we use an assert. assert(par->NumChildren() <= par->MaxNumChildren() + 1); if (par->NumChildren() == par->MaxNumChildren() + 1) - RTreeSplit::SplitNonLeafNode(par,relevels); + RTreeSplit::SplitNonLeafNode(par, relevels); assert(treeOne->Parent()->NumChildren() <= treeOne->MaxNumChildren()); assert(treeOne->Parent()->NumChildren() >= treeOne->MinNumChildren()); @@ -92,7 +92,7 @@ void RTreeSplit::SplitLeafNode(TreeType *tree,std::vector& relevels) * higher up the tree because they were already updated if necessary. */ template -bool RTreeSplit::SplitNonLeafNode(TreeType *tree,std::vector& relevels) +bool RTreeSplit::SplitNonLeafNode(TreeType *tree, std::vector& relevels) { // If we are splitting the root node, we need will do things differently so // that the constructor and other methods don't confuse the end user by giving @@ -105,13 +105,13 @@ bool RTreeSplit::SplitNonLeafNode(TreeType *tree,std::vector& relevels) tree->NumChildren() = 0; tree->NullifyData(); tree->children[(tree->NumChildren())++] = copy; - RTreeSplit::SplitNonLeafNode(copy,relevels); + RTreeSplit::SplitNonLeafNode(copy, relevels); return true; } int i = 0; int j = 0; - RTreeSplit::GetBoundSeeds(tree,i, j); + RTreeSplit::GetBoundSeeds(tree, i, j); assert(i != j); @@ -138,7 +138,7 @@ bool RTreeSplit::SplitNonLeafNode(TreeType *tree,std::vector& relevels) assert(par->NumChildren() <= par->MaxNumChildren() + 1); if (par->NumChildren() == par->MaxNumChildren() + 1) - RTreeSplit::SplitNonLeafNode(par,relevels); + RTreeSplit::SplitNonLeafNode(par, relevels); // We have to update the children of each of these new nodes so that they // record the correct parent. @@ -154,7 +154,7 @@ bool RTreeSplit::SplitNonLeafNode(TreeType *tree,std::vector& relevels) // Because we now have pointers to the information stored under this tree, // we need to delete this node carefully. - tree->SoftDelete(); //currently does nothing but leak memory. + tree->SoftDelete(); // currently does nothing but leak memory. return false; } @@ -164,7 +164,7 @@ bool RTreeSplit::SplitNonLeafNode(TreeType *tree,std::vector& relevels) * The indices of these points will be stored in iRet and jRet. */ template -void RTreeSplit::GetPointSeeds(const TreeType *tree,int& iRet, int& jRet) +void RTreeSplit::GetPointSeeds(const TreeType *tree, int& iRet, int& jRet) { // Here we want to find the pair of points that it is worst to place in the // same node. Because we are just using points, we will simply choose the two @@ -193,7 +193,7 @@ void RTreeSplit::GetPointSeeds(const TreeType *tree,int& iRet, int& jRet) * indices of the bounds will be stored in iRet and jRet. */ template -void RTreeSplit::GetBoundSeeds(const TreeType *tree,int& iRet, int& jRet) +void RTreeSplit::GetBoundSeeds(const TreeType *tree, int& iRet, int& jRet) { // Convenience typedef. typedef typename TreeType::ElemType ElemType; diff --git a/src/mlpack/core/tree/rectangle_tree/rectangle_tree.hpp b/src/mlpack/core/tree/rectangle_tree/rectangle_tree.hpp index 1d78c666e6..1609f0be9e 100644 --- a/src/mlpack/core/tree/rectangle_tree/rectangle_tree.hpp +++ b/src/mlpack/core/tree/rectangle_tree/rectangle_tree.hpp @@ -49,7 +49,8 @@ template class AuxiliaryInformationType = NoAuxiliaryInformation> + template class AuxiliaryInformationType = + NoAuxiliaryInformation> class RectangleTree { // The metric *must* be the euclidean distance. diff --git a/src/mlpack/core/tree/rectangle_tree/rectangle_tree_impl.hpp b/src/mlpack/core/tree/rectangle_tree/rectangle_tree_impl.hpp index 3a31c76778..1f41931fee 100644 --- a/src/mlpack/core/tree/rectangle_tree/rectangle_tree_impl.hpp +++ b/src/mlpack/core/tree/rectangle_tree/rectangle_tree_impl.hpp @@ -112,7 +112,7 @@ RectangleTree* - parentNode,const size_t numMaxChildren) : + parentNode, const size_t numMaxChildren) : maxNumChildren(numMaxChildren > 0 ? numMaxChildren : parentNode->MaxNumChildren()), minNumChildren(parentNode->MinNumChildren()), @@ -385,7 +385,7 @@ void RectangleTreeInsertPoint(point, relevels); } @@ -879,7 +879,7 @@ void RectangleTreechildren[j] == this) { // Decrement numChildren. - if (!auxiliaryInfo.HandleNodeRemoval(parent,j)) + if (!auxiliaryInfo.HandleNodeRemoval(parent, j)) { parent->children[j] = parent->children[--parent->NumChildren()]; } @@ -1096,7 +1096,8 @@ void RectangleTreeCondenseTree(point, relevels, usePoint); else if (!usePoint && - (ShrinkBoundForBound(bound) || auxiliaryInfo.UpdateAuxiliaryInfo(this)) && + (ShrinkBoundForBound(bound) || + auxiliaryInfo.UpdateAuxiliaryInfo(this)) && parent != NULL) parent->CondenseTree(point, relevels, usePoint); } @@ -1261,7 +1262,6 @@ void RectangleTree::Traverse( const size_t queryIndex, const RectangleTree& referenceNode) { - // If we reach a leaf node, we need to run the base case. if (referenceNode.IsLeaf()) { diff --git a/src/mlpack/core/tree/rectangle_tree/typedef.hpp b/src/mlpack/core/tree/rectangle_tree/typedef.hpp index 20bbfdc7f7..c589732d74 100644 --- a/src/mlpack/core/tree/rectangle_tree/typedef.hpp +++ b/src/mlpack/core/tree/rectangle_tree/typedef.hpp @@ -125,7 +125,7 @@ using XTree = RectangleTree using DiscreteHilbertRTreeAuxiliaryInformation = - HilbertRTreeAuxiliaryInformation; + HilbertRTreeAuxiliaryInformation; template using HilbertRTree = RectangleTree + */ +template using RPlusPlusTree = RectangleTree - static void SplitLeafNode(TreeType *tree,std::vector& relevels); + static void SplitLeafNode(TreeType *tree, std::vector& relevels); /** * Split a non-leaf node using the "default" algorithm. If this is a root * node, the tree increases in depth. */ template - static bool SplitNonLeafNode(TreeType *tree,std::vector& relevels); + static bool SplitNonLeafNode(TreeType *tree, std::vector& relevels); private: /** @@ -68,7 +68,6 @@ class XTreeSplit { return p1.first < p2.first; } - }; } // namespace tree diff --git a/src/mlpack/core/tree/rectangle_tree/x_tree_split_impl.hpp b/src/mlpack/core/tree/rectangle_tree/x_tree_split_impl.hpp index 0457d0a570..a2b1694e7b 100644 --- a/src/mlpack/core/tree/rectangle_tree/x_tree_split_impl.hpp +++ b/src/mlpack/core/tree/rectangle_tree/x_tree_split_impl.hpp @@ -26,7 +26,7 @@ namespace tree { * new nodes into the tree, spliting the parent if necessary. */ template -void XTreeSplit::SplitLeafNode(TreeType *tree,std::vector& relevels) +void XTreeSplit::SplitLeafNode(TreeType *tree, std::vector& relevels) { // Convenience typedef. typedef typename TreeType::ElemType ElemType; @@ -110,7 +110,7 @@ void XTreeSplit::SplitLeafNode(TreeType *tree,std::vector& relevels) // If we overflowed the parent, split it. if (par && par->NumChildren() == par->MaxNumChildren() + 1) - XTreeSplit::SplitNonLeafNode(par,relevels); + XTreeSplit::SplitNonLeafNode(par, relevels); } /** @@ -121,7 +121,7 @@ void XTreeSplit::SplitLeafNode(TreeType *tree,std::vector& relevels) * higher up the tree because they were already updated if necessary. */ template -bool XTreeSplit::SplitNonLeafNode(TreeType *tree,std::vector& relevels) +bool XTreeSplit::SplitNonLeafNode(TreeType *tree, std::vector& relevels) { // Convenience typedef. typedef typename TreeType::ElemType ElemType; @@ -492,7 +492,8 @@ bool XTreeSplit::SplitNonLeafNode(TreeType *tree,std::vector& relevels) sorted2[i].second = sorted[i].second; } } - std::sort(sorted2.begin(), sorted2.end(), PairComp); + std::sort(sorted2.begin(), sorted2.end(), + PairComp); tree->numDescendants = 0; tree->bound.Clear(); @@ -520,7 +521,7 @@ bool XTreeSplit::SplitNonLeafNode(TreeType *tree,std::vector& relevels) { // We make the root a supernode instead. tree->Parent()->MaxNumChildren() = tree->MaxNumChildren() + - tree->AuxiliaryInfo().NormalNodeMaxNumChildren(); + tree->AuxiliaryInfo().NormalNodeMaxNumChildren(); tree->Parent()->children.resize(tree->Parent()->MaxNumChildren() + 1); tree->Parent()->NumChildren() = tree->NumChildren(); for (size_t i = 0; i < numChildren; ++i) @@ -538,7 +539,7 @@ bool XTreeSplit::SplitNonLeafNode(TreeType *tree,std::vector& relevels) // If we don't have to worry about the root, we just enlarge this node. tree->MaxNumChildren() += - tree->AuxiliaryInfo().NormalNodeMaxNumChildren(); + tree->AuxiliaryInfo().NormalNodeMaxNumChildren(); tree->children.resize(tree->MaxNumChildren() + 1); tree->numChildren = numChildren; for (size_t i = 0; i < numChildren; i++) @@ -567,7 +568,7 @@ bool XTreeSplit::SplitNonLeafNode(TreeType *tree,std::vector& relevels) assert(par->NumChildren() <= par->MaxNumChildren() + 1); if (par->NumChildren() == par->MaxNumChildren() + 1) - XTreeSplit::SplitNonLeafNode(par,relevels); + XTreeSplit::SplitNonLeafNode(par, relevels); // We have to update the children of each of these new nodes so that they // record the correct parent. @@ -627,8 +628,8 @@ bool XTreeSplit::SplitNonLeafNode(TreeType *tree,std::vector& relevels) } // If the split was not good enough, then we try the minimal overlap split. - // If that fails, we create a "super node" (more accurately we resize this one - // to make it a super node). + // If that fails, we create a "super node" (more accurately we resize this + // one to make it a super node). if (useMinOverlapSplit) { // If there is a dimension that might work, try that. @@ -652,7 +653,8 @@ bool XTreeSplit::SplitNonLeafNode(TreeType *tree,std::vector& relevels) sorted2[i].second = sorted[i].second; } } - std::sort(sorted2.begin(), sorted2.end(), PairComp); + std::sort(sorted2.begin(), sorted2.end(), + PairComp); for (size_t i = 0; i < numChildren; i++) { @@ -666,7 +668,7 @@ bool XTreeSplit::SplitNonLeafNode(TreeType *tree,std::vector& relevels) { // Make this node a supernode. tree->MaxNumChildren() += - tree->AuxiliaryInfo().NormalNodeMaxNumChildren(); + tree->AuxiliaryInfo().NormalNodeMaxNumChildren(); tree->children.resize(tree->MaxNumChildren() + 1); tree->numChildren = numChildren; for (size_t i = 0; i < numChildren; i++) diff --git a/src/mlpack/core/tree/spill_tree/spill_tree_impl.hpp b/src/mlpack/core/tree/spill_tree/spill_tree_impl.hpp index bdc9970123..8179edd57e 100644 --- a/src/mlpack/core/tree/spill_tree/spill_tree_impl.hpp +++ b/src/mlpack/core/tree/spill_tree/spill_tree_impl.hpp @@ -230,7 +230,7 @@ SpillTree(SpillTree&& other) : other.dataset = NULL; other.localDataset = false; - //Set new parent. + // Set new parent. if (left) left->parent = this; if (right) @@ -671,8 +671,8 @@ bool SpillTree:: } } - const double p1 = double (left + rightFrontier) / points.n_elem; - const double p2 = double (right + leftFrontier) / points.n_elem; + const double p1 = (double) (left + rightFrontier) / points.n_elem; + const double p2 = (double) (right + leftFrontier) / points.n_elem; if ((p1 <= rho || rightFrontier == 0) && (p2 <= rho || leftFrontier == 0)) diff --git a/src/mlpack/core/tree/statistic.hpp b/src/mlpack/core/tree/statistic.hpp index 706f5123f3..2a2a0d86bd 100644 --- a/src/mlpack/core/tree/statistic.hpp +++ b/src/mlpack/core/tree/statistic.hpp @@ -23,26 +23,26 @@ namespace tree { */ class EmptyStatistic { - public: - EmptyStatistic() { } - ~EmptyStatistic() { } + public: + EmptyStatistic() { } + ~EmptyStatistic() { } - /** - * This constructor is called when a node is finished being created. The - * node is finished, and its children are finished, but it is not - * necessarily true that the statistics of other nodes are initialized yet. - * - * @param node Node which this corresponds to. - */ - template - EmptyStatistic(TreeType& /* node */) { } + /** + * This constructor is called when a node is finished being created. The + * node is finished, and its children are finished, but it is not + * necessarily true that the statistics of other nodes are initialized yet. + * + * @param node Node which this corresponds to. + */ + template + EmptyStatistic(TreeType& /* node */) { } - /** - * Serialize the statistic (there's nothing to be saved). - */ - template - void Serialize(Archive& /* ar */, const unsigned int /* version */) - { } + /** + * Serialize the statistic (there's nothing to be saved). + */ + template + void Serialize(Archive& /* ar */, const unsigned int /* version */) + { } }; } // namespace tree diff --git a/src/mlpack/core/util/arma_traits.hpp b/src/mlpack/core/util/arma_traits.hpp index b9661dd3e1..45e5dac125 100644 --- a/src/mlpack/core/util/arma_traits.hpp +++ b/src/mlpack/core/util/arma_traits.hpp @@ -38,43 +38,43 @@ struct IsVector }; // Commenting out the first template per case, because -//Visual Studio doesn't like this instantiaion pattern (error C2910). -//template<> +// Visual Studio doesn't like this instantiaion pattern (error C2910). +// template<> template struct IsVector > { const static bool value = true; }; -//template<> +// template<> template struct IsVector > { const static bool value = true; }; -//template<> +// template<> template struct IsVector > { const static bool value = true; }; -//template<> +// template<> template struct IsVector > { const static bool value = true; }; -//template<> +// template<> template struct IsVector > { const static bool value = true; }; -//template<> +// template<> template struct IsVector > { @@ -84,7 +84,7 @@ struct IsVector > // I'm not so sure about this one. An SpSubview object can be a row or column, // but it can also be a matrix subview. -//template<> +// template<> template struct IsVector > { diff --git a/src/mlpack/core/util/backtrace.cpp b/src/mlpack/core/util/backtrace.cpp index 6e5b2228fa..ce7052f128 100644 --- a/src/mlpack/core/util/backtrace.cpp +++ b/src/mlpack/core/util/backtrace.cpp @@ -48,7 +48,8 @@ // Easier to read Backtrace::DecodeAddress(). #ifdef HAS_BFD_DL #define TRACE_CONDITION_1 (!dladdr(trace[i], &addressHandler)) - #define FIND_LINE (bfd_find_nearest_line(abfd, text, syms, offset, &frame.file, &frame.function, &frame.line) && frame.file) + #define FIND_LINE (bfd_find_nearest_line(abfd, text, syms, offset, \ + &frame.file, &frame.function, &frame.line) && frame.file) #endif using namespace mlpack; @@ -94,10 +95,10 @@ void Backtrace::GetAddress(int maxDepth) { Dl_info addressHandler; - //No backtrace will be printed if no compile flags: -g -rdynamic + // No backtrace will be printed if no compile flags: -g -rdynamic if (TRACE_CONDITION_1) { - return ; + return; } frame.address = addressHandler.dli_saddr; @@ -130,13 +131,13 @@ void Backtrace::DecodeAddress(long addr) return; } - bfd_check_format(abfd,bfd_object); + bfd_check_format(abfd, bfd_object); unsigned storage_needed = bfd_get_symtab_upper_bound(abfd); syms = (asymbol **) malloc(storage_needed); text = bfd_get_section_by_name(abfd, ".text"); - } + } long offset = addr - text->vma; diff --git a/src/mlpack/core/util/backtrace.hpp b/src/mlpack/core/util/backtrace.hpp index 51acdfa106..934a4a1cb8 100644 --- a/src/mlpack/core/util/backtrace.hpp +++ b/src/mlpack/core/util/backtrace.hpp @@ -91,6 +91,6 @@ class Backtrace static std::vector stack; }; -}; //namespace mlpack +}; // namespace mlpack #endif diff --git a/src/mlpack/core/util/log.hpp b/src/mlpack/core/util/log.hpp index 87cdd55372..86e089696d 100644 --- a/src/mlpack/core/util/log.hpp +++ b/src/mlpack/core/util/log.hpp @@ -93,6 +93,6 @@ class Log static std::ostream& cout; }; -}; //namespace mlpack +}; // namespace mlpack #endif diff --git a/src/mlpack/core/util/prefixedoutstream.cpp b/src/mlpack/core/util/prefixedoutstream.cpp index 0db096b77d..8393ca0996 100644 --- a/src/mlpack/core/util/prefixedoutstream.cpp +++ b/src/mlpack/core/util/prefixedoutstream.cpp @@ -36,7 +36,7 @@ PrefixedOutStream& PrefixedOutStream::operator<<(short val) PrefixedOutStream& PrefixedOutStream::operator<<(unsigned short val) { - BaseLogic(val); + BaseLogic(val); return *this; } diff --git a/src/mlpack/core/util/timers.hpp b/src/mlpack/core/util/timers.hpp index 71f5c195d2..021ed56b70 100644 --- a/src/mlpack/core/util/timers.hpp +++ b/src/mlpack/core/util/timers.hpp @@ -18,7 +18,7 @@ #include // chrono library for cross platform timer calculation #if defined(_WIN32) - // uint64_t isn't defined on every windows. + // uint64_t isn't defined on every windows. #if !defined(HAVE_UINT64_T) #if SIZEOF_UNSIGNED_LONG == 8 typedef unsigned long uint64_t; diff --git a/src/mlpack/methods/adaboost/adaboost.hpp b/src/mlpack/methods/adaboost/adaboost.hpp index d2732824df..810cfa09f7 100644 --- a/src/mlpack/methods/adaboost/adaboost.hpp +++ b/src/mlpack/methods/adaboost/adaboost.hpp @@ -161,7 +161,7 @@ class AdaBoost template void Serialize(Archive& ar, const unsigned int /* version */); -private: + private: //! The number of classes in the model. size_t classes; // The tolerance for change in rt and when to stop. @@ -174,7 +174,6 @@ private: //! To check for the bound for the Hamming loss. double ztProduct; - }; // class AdaBoost } // namespace adaboost diff --git a/src/mlpack/methods/amf/termination_policies/simple_residue_termination.hpp b/src/mlpack/methods/amf/termination_policies/simple_residue_termination.hpp index bc659f221a..fb1d2b94a3 100644 --- a/src/mlpack/methods/amf/termination_policies/simple_residue_termination.hpp +++ b/src/mlpack/methods/amf/termination_policies/simple_residue_termination.hpp @@ -99,7 +99,7 @@ class SimpleResidueTermination const double& MinResidue() const { return minResidue; } double& MinResidue() { return minResidue; } -public: + public: //! residue threshold double minResidue; //! iteration threshold diff --git a/src/mlpack/methods/amf/termination_policies/simple_tolerance_termination.hpp b/src/mlpack/methods/amf/termination_policies/simple_tolerance_termination.hpp index 1e29c27bb8..f645494c31 100644 --- a/src/mlpack/methods/amf/termination_policies/simple_tolerance_termination.hpp +++ b/src/mlpack/methods/amf/termination_policies/simple_tolerance_termination.hpp @@ -78,12 +78,12 @@ class SimpleToleranceTermination size_t m = V->n_cols; double sum = 0; size_t count = 0; - for(size_t i = 0;i < n;i++) + for (size_t i = 0; i < n; i++) { - for(size_t j = 0;j < m;j++) + for (size_t j = 0; j < m; j++) { double temp = 0; - if ((temp = (*V)(i,j)) != 0) + if ((temp = (*V)(i, j)) != 0) { temp = (temp - WH(i, j)); temp = temp * temp; diff --git a/src/mlpack/methods/amf/termination_policies/validation_RMSE_termination.hpp b/src/mlpack/methods/amf/termination_policies/validation_RMSE_termination.hpp index 00053c3647..9171aa0ae3 100644 --- a/src/mlpack/methods/amf/termination_policies/validation_RMSE_termination.hpp +++ b/src/mlpack/methods/amf/termination_policies/validation_RMSE_termination.hpp @@ -63,7 +63,7 @@ class ValidationRMSETermination test_points.zeros(num_test_points, 3); // fill validation set matrix with random chosen entries - for(size_t i = 0; i < num_test_points; i++) + for (size_t i = 0; i < num_test_points; i++) { double t_val; size_t t_row; @@ -74,7 +74,7 @@ class ValidationRMSETermination { t_row = rand() % n; t_col = rand() % m; - } while((t_val = V(t_row, t_col)) == 0); + } while ((t_val = V(t_row, t_col)) == 0); // add the entry to the validation set test_points(i, 0) = t_row; @@ -122,7 +122,7 @@ class ValidationRMSETermination { rmseOld = rmse; rmse = 0; - for(size_t i = 0; i < num_test_points; i++) + for (size_t i = 0; i < num_test_points; i++) { size_t t_row = test_points(i, 0); size_t t_col = test_points(i, 1); diff --git a/src/mlpack/methods/amf/update_rules/nmf_mult_div.hpp b/src/mlpack/methods/amf/update_rules/nmf_mult_div.hpp index 567b45feb0..38899e0c29 100644 --- a/src/mlpack/methods/amf/update_rules/nmf_mult_div.hpp +++ b/src/mlpack/methods/amf/update_rules/nmf_mult_div.hpp @@ -93,7 +93,7 @@ class NMFMultiplicativeDivergenceUpdate // Writing this as a single expression does not work as of Armadillo // 3.920. This should be fixed in a future release, and then the code // below can be fixed. - //t2 = H.row(j) % V.row(i) / t1.row(i); + // t2 = H.row(j) % V.row(i) / t1.row(i); t2.set_size(H.n_cols); for (size_t k = 0; k < t2.n_elem; ++k) { @@ -137,14 +137,14 @@ class NMFMultiplicativeDivergenceUpdate // Writing this as a single expression does not work as of Armadillo // 3.920. This should be fixed in a future release, and then the code // below can be fixed. - //t2 = W.col(i) % V.col(j) / t1.col(j); + // t2 = W.col(i) % V.col(j) / t1.col(j); t2.set_size(W.n_rows); for (size_t k = 0; k < t2.n_elem; ++k) { t2(k) = W(k, i) * V(k, j) / t1(k, j); } - H(i,j) = H(i,j) * sum(t2) / sum(W.col(i)); + H(i, j) = H(i, j) * sum(t2) / sum(W.col(i)); } } } diff --git a/src/mlpack/methods/amf/update_rules/svd_complete_incremental_learning.hpp b/src/mlpack/methods/amf/update_rules/svd_complete_incremental_learning.hpp index 7e8a0d359f..22174a0f7c 100644 --- a/src/mlpack/methods/amf/update_rules/svd_complete_incremental_learning.hpp +++ b/src/mlpack/methods/amf/update_rules/svd_complete_incremental_learning.hpp @@ -92,7 +92,7 @@ class SVDCompleteIncrementalLearning deltaW.zeros(1, W.n_cols); // Loop until a non-zero entry is found. - while(true) + while (true) { const double val = V(currentItemIndex, currentUserIndex); // Update feature vector if current entry is non-zero and break the loop. @@ -168,7 +168,7 @@ class SVDCompleteIncrementalLearning template<> class SVDCompleteIncrementalLearning { - public: + public: SVDCompleteIncrementalLearning(double u = 0.01, double kw = 0, double kh = 0) @@ -218,8 +218,8 @@ class SVDCompleteIncrementalLearning arma::mat deltaW(1, W.n_cols); deltaW.zeros(); - deltaW += (**it - arma::dot(W.row(currentItemIndex), H.col(currentUserIndex))) - * arma::trans(H.col(currentUserIndex)); + deltaW += (**it - arma::dot(W.row(currentItemIndex), + H.col(currentUserIndex))) * arma::trans(H.col(currentUserIndex)); if (kw != 0) deltaW -= kw * W.row(currentItemIndex); W.row(currentItemIndex) += u*deltaW; @@ -246,8 +246,8 @@ class SVDCompleteIncrementalLearning size_t currentUserIndex = it->col(); size_t currentItemIndex = it->row(); - deltaH += (**it - arma::dot(W.row(currentItemIndex), H.col(currentUserIndex))) - * arma::trans(W.row(currentItemIndex)); + deltaH += (**it - arma::dot(W.row(currentItemIndex), + H.col(currentUserIndex))) * arma::trans(W.row(currentItemIndex)); if (kh != 0) deltaH -= kh * H.col(currentUserIndex); H.col(currentUserIndex) += u * deltaH; diff --git a/src/mlpack/methods/amf/update_rules/svd_incomplete_incremental_learning.hpp b/src/mlpack/methods/amf/update_rules/svd_incomplete_incremental_learning.hpp index 43bce411cc..ce5c26f99c 100644 --- a/src/mlpack/methods/amf/update_rules/svd_incomplete_incremental_learning.hpp +++ b/src/mlpack/methods/amf/update_rules/svd_incomplete_incremental_learning.hpp @@ -97,8 +97,10 @@ class SVDIncompleteIncrementalLearning const double val = V(i, currentUserIndex); // Update only if the rating is non-zero. if (val != 0) + { deltaW.row(i) += (val - arma::dot(W.row(i), H.col(currentUserIndex))) * H.col(currentUserIndex).t(); + } // Add regularization. if (kw != 0) deltaW.row(i) -= kw * W.row(i); @@ -130,8 +132,10 @@ class SVDIncompleteIncrementalLearning const double val = V(i, currentUserIndex); // Update only if the rating is non-zero. if (val != 0) + { deltaH += (val - arma::dot(W.row(i), H.col(currentUserIndex))) * W.row(i).t(); + } } // Add regularization. if (kh != 0) @@ -159,20 +163,18 @@ class SVDIncompleteIncrementalLearning //! template specialiazed functions for sparse matrices template<> -inline void SVDIncompleteIncrementalLearning:: - WUpdate(const arma::sp_mat& V, - arma::mat& W, - const arma::mat& H) +inline void SVDIncompleteIncrementalLearning::WUpdate( + const arma::sp_mat& V, arma::mat& W, const arma::mat& H) { arma::mat deltaW(V.n_rows, W.n_cols); deltaW.zeros(); - for(arma::sp_mat::const_iterator it = V.begin_col(currentUserIndex); - it != V.end_col(currentUserIndex);it++) + for (arma::sp_mat::const_iterator it = V.begin_col(currentUserIndex); + it != V.end_col(currentUserIndex); it++) { double val = *it; size_t i = it.row(); deltaW.row(i) += (val - arma::dot(W.row(i), H.col(currentUserIndex))) * - arma::trans(H.col(currentUserIndex)); + arma::trans(H.col(currentUserIndex)); if (kw != 0) deltaW.row(i) -= kw * W.row(i); } @@ -180,22 +182,22 @@ inline void SVDIncompleteIncrementalLearning:: } template<> -inline void SVDIncompleteIncrementalLearning:: - HUpdate(const arma::sp_mat& V, - const arma::mat& W, - arma::mat& H) +inline void SVDIncompleteIncrementalLearning::HUpdate( + const arma::sp_mat& V, const arma::mat& W, arma::mat& H) { arma::mat deltaH(H.n_rows, 1); deltaH.zeros(); - for(arma::sp_mat::const_iterator it = V.begin_col(currentUserIndex); - it != V.end_col(currentUserIndex);it++) + for (arma::sp_mat::const_iterator it = V.begin_col(currentUserIndex); + it != V.end_col(currentUserIndex); it++) { double val = *it; size_t i = it.row(); if ((val = V(i, currentUserIndex)) != 0) + { deltaH += (val - arma::dot(W.row(i), H.col(currentUserIndex))) * - arma::trans(W.row(i)); + arma::trans(W.row(i)); + } } if (kh != 0) deltaH -= kh * H.col(currentUserIndex); @@ -203,7 +205,7 @@ inline void SVDIncompleteIncrementalLearning:: currentUserIndex = currentUserIndex % V.n_cols; } -} // namepsace amf +} // namespace amf } // namespace mlpack #endif diff --git a/src/mlpack/methods/ann/activation_functions/identity_function.hpp b/src/mlpack/methods/ann/activation_functions/identity_function.hpp index 0de88e57ec..14ab9a8338 100644 --- a/src/mlpack/methods/ann/activation_functions/identity_function.hpp +++ b/src/mlpack/methods/ann/activation_functions/identity_function.hpp @@ -86,8 +86,6 @@ class IdentityFunction { x.ones(y.n_rows, y.n_cols, y.n_slices); } - - }; // class IdentityFunction } // namespace ann diff --git a/src/mlpack/methods/ann/activation_functions/logistic_function.hpp b/src/mlpack/methods/ann/activation_functions/logistic_function.hpp index 75ddd334d4..20e1622270 100644 --- a/src/mlpack/methods/ann/activation_functions/logistic_function.hpp +++ b/src/mlpack/methods/ann/activation_functions/logistic_function.hpp @@ -28,7 +28,7 @@ namespace ann /** Artificial Neural Network. */ { */ class LogisticFunction { - public: + public: /** * Computes the logistic function. * diff --git a/src/mlpack/methods/ann/activation_functions/softplus_function.hpp b/src/mlpack/methods/ann/activation_functions/softplus_function.hpp index 4adc5d746c..dd5875ff83 100644 --- a/src/mlpack/methods/ann/activation_functions/softplus_function.hpp +++ b/src/mlpack/methods/ann/activation_functions/softplus_function.hpp @@ -42,8 +42,7 @@ namespace ann /** Artificial Neural Network. */ { */ class SoftplusFunction { - public: - + public: /** * Computes the softplus function. * diff --git a/src/mlpack/methods/ann/activation_functions/softsign_function.hpp b/src/mlpack/methods/ann/activation_functions/softsign_function.hpp index 2438ee3c7c..eb9db79b69 100644 --- a/src/mlpack/methods/ann/activation_functions/softsign_function.hpp +++ b/src/mlpack/methods/ann/activation_functions/softsign_function.hpp @@ -46,7 +46,7 @@ namespace ann /** Artificial Neural Network. */ { */ class SoftsignFunction { - public: + public: /** * Computes the softsign function. * diff --git a/src/mlpack/methods/ann/activation_functions/tanh_function.hpp b/src/mlpack/methods/ann/activation_functions/tanh_function.hpp index aea406a3d4..63abf0f1ec 100644 --- a/src/mlpack/methods/ann/activation_functions/tanh_function.hpp +++ b/src/mlpack/methods/ann/activation_functions/tanh_function.hpp @@ -28,7 +28,7 @@ namespace ann /** Artificial Neural Network. */ { */ class TanhFunction { - public: + public: /** * Computes the tanh function. * diff --git a/src/mlpack/methods/ann/convolution_rules/fft_convolution.hpp b/src/mlpack/methods/ann/convolution_rules/fft_convolution.hpp index 4eaa038380..f573ce9ca2 100644 --- a/src/mlpack/methods/ann/convolution_rules/fft_convolution.hpp +++ b/src/mlpack/methods/ann/convolution_rules/fft_convolution.hpp @@ -212,7 +212,6 @@ class FFTConvolution output.slice(i) = convOutput; } } - }; // class FFTConvolution } // namespace ann diff --git a/src/mlpack/methods/ann/convolution_rules/naive_convolution.hpp b/src/mlpack/methods/ann/convolution_rules/naive_convolution.hpp index c60d9e5b28..c27225f127 100644 --- a/src/mlpack/methods/ann/convolution_rules/naive_convolution.hpp +++ b/src/mlpack/methods/ann/convolution_rules/naive_convolution.hpp @@ -201,7 +201,6 @@ class NaiveConvolution output.slice(i), dW, dH); } } - }; // class NaiveConvolution } // namespace ann diff --git a/src/mlpack/methods/ann/convolution_rules/svd_convolution.hpp b/src/mlpack/methods/ann/convolution_rules/svd_convolution.hpp index b532ac1a6b..7cd50470ab 100644 --- a/src/mlpack/methods/ann/convolution_rules/svd_convolution.hpp +++ b/src/mlpack/methods/ann/convolution_rules/svd_convolution.hpp @@ -198,7 +198,6 @@ class SVDConvolution output.slice(i) = convOutput; } } - }; // class SVDConvolution } // namespace ann diff --git a/src/mlpack/methods/ann/ffn.hpp b/src/mlpack/methods/ann/ffn.hpp index 05a957df04..f297a56c4e 100644 --- a/src/mlpack/methods/ann/ffn.hpp +++ b/src/mlpack/methods/ann/ffn.hpp @@ -139,7 +139,7 @@ class FFN * @param predictors Input predictors. * @param results Matrix to put output predictions of responses into. */ - void Predict(arma::mat& predictors, arma::mat& results); + void Predict(const arma::mat& predictors, arma::mat& results); /** * Evaluate the feedforward network with the given parameters. This function @@ -209,7 +209,7 @@ class FFN template void Serialize(Archive& ar, const unsigned int /* version */); -private: + private: // Helper functions. /** * The Forward algorithm (part of the Forward-Backward algorithm). Computes @@ -336,7 +336,6 @@ private: //! Locally-stored copy visitor CopyVisitor copyVisitor; - }; // class FFN } // namespace ann diff --git a/src/mlpack/methods/ann/ffn_impl.hpp b/src/mlpack/methods/ann/ffn_impl.hpp index 130089c982..38de848934 100644 --- a/src/mlpack/methods/ann/ffn_impl.hpp +++ b/src/mlpack/methods/ann/ffn_impl.hpp @@ -67,8 +67,8 @@ FFN::~FFN() } template -void FFN::ResetData(const arma::mat &predictors, - const arma::mat &responses) +void FFN::ResetData( + const arma::mat& predictors, const arma::mat& responses) { numFunctions = responses.n_cols; this->predictors = std::move(predictors); @@ -134,7 +134,7 @@ void FFN::Train( template void FFN::Predict( - arma::mat& predictors, arma::mat& results) + const arma::mat& predictors, arma::mat& results) { if (parameter.is_empty()) { @@ -148,8 +148,7 @@ void FFN::Predict( } arma::mat resultsTemp; - Forward(std::move(arma::mat(predictors.colptr(0), - predictors.n_rows, 1, false, true))); + Forward(std::move(predictors.col(0))); resultsTemp = boost::apply_visitor(outputParameterVisitor, network.back()).col(0); @@ -158,8 +157,7 @@ void FFN::Predict( for (size_t i = 1; i < predictors.n_cols; i++) { - Forward(std::move(arma::mat(predictors.colptr(i), - predictors.n_rows, 1, false, true))); + Forward(std::move(predictors.col(i))); resultsTemp = boost::apply_visitor(outputParameterVisitor, network.back()); @@ -426,7 +424,8 @@ FFN::FFN( // Build new layers according to source network for (size_t i = 0; i < network.network.size(); ++i) { - this->network.push_back(boost::apply_visitor(copyVisitor, network.network[i])); + this->network.push_back(boost::apply_visitor(copyVisitor, + network.network[i])); } }; diff --git a/src/mlpack/methods/ann/layer/concat_impl.hpp b/src/mlpack/methods/ann/layer/concat_impl.hpp index 1f3ffd47fc..5e3d4cef6b 100644 --- a/src/mlpack/methods/ann/layer/concat_impl.hpp +++ b/src/mlpack/methods/ann/layer/concat_impl.hpp @@ -143,7 +143,8 @@ void Concat::Gradient( { for (size_t i = 0; i < network.size(); ++i) { - boost::apply_visitor(GradientVisitor(std::move(input), std::move(error)), network[i]); + boost::apply_visitor(GradientVisitor(std::move(input), + std::move(error)), network[i]); } } diff --git a/src/mlpack/methods/ann/layer/constant.hpp b/src/mlpack/methods/ann/layer/constant.hpp index 1a2c8e9e52..54c9850a8d 100644 --- a/src/mlpack/methods/ann/layer/constant.hpp +++ b/src/mlpack/methods/ann/layer/constant.hpp @@ -113,4 +113,4 @@ class Constant // Include implementation. #include "constant_impl.hpp" -#endif \ No newline at end of file +#endif diff --git a/src/mlpack/methods/ann/layer/convolution.hpp b/src/mlpack/methods/ann/layer/convolution.hpp index bfef18aa4f..cca4eb21dc 100644 --- a/src/mlpack/methods/ann/layer/convolution.hpp +++ b/src/mlpack/methods/ann/layer/convolution.hpp @@ -45,7 +45,7 @@ template < > class Convolution { -public: + public: //! Create the Convolution object. Convolution(); @@ -168,7 +168,6 @@ public: void Serialize(Archive& ar, const unsigned int /* version */); private: - /* * Return the convolution output size. * @@ -341,4 +340,4 @@ public: // Include implementation. #include "convolution_impl.hpp" -#endif \ No newline at end of file +#endif diff --git a/src/mlpack/methods/ann/layer/convolution_impl.hpp b/src/mlpack/methods/ann/layer/convolution_impl.hpp index 422c994709..d0758b29ca 100644 --- a/src/mlpack/methods/ann/layer/convolution_impl.hpp +++ b/src/mlpack/methods/ann/layer/convolution_impl.hpp @@ -287,12 +287,8 @@ void Convolution< outMap, outMap)); } - // gradient.submat(0, 0, weight.n_elem - 1, 0) = arma::vectorise(gradientTemp); gradient.submat(0, 0, weight.n_elem - 1, 0) = arma::Mat( gradientTemp.memptr(), gradientTemp.n_elem, 1, false, false); - - - // arma::vectorise(gradientTemp); } template< @@ -330,4 +326,4 @@ void Convolution< } // namespace ann } // namespace mlpack -#endif \ No newline at end of file +#endif diff --git a/src/mlpack/methods/ann/layer/dropconnect.hpp b/src/mlpack/methods/ann/layer/dropconnect.hpp index 80e00bfa1c..aa4309b464 100644 --- a/src/mlpack/methods/ann/layer/dropconnect.hpp +++ b/src/mlpack/methods/ann/layer/dropconnect.hpp @@ -162,7 +162,7 @@ class DropConnect template void Serialize(Archive& ar, const unsigned int /* version */); -private: + private: //! The probability of setting a value to zero. double ratio; diff --git a/src/mlpack/methods/ann/layer/elu.hpp b/src/mlpack/methods/ann/layer/elu.hpp index 7a40d453c4..2eb8363ec6 100644 --- a/src/mlpack/methods/ann/layer/elu.hpp +++ b/src/mlpack/methods/ann/layer/elu.hpp @@ -189,7 +189,6 @@ class ELU //! ELU Hyperparameter (0 < alpha) double alpha; - }; // class ELU } // namespace ann diff --git a/src/mlpack/methods/ann/layer/glimpse.hpp b/src/mlpack/methods/ann/layer/glimpse.hpp index e73dd62457..6b4420a8d4 100644 --- a/src/mlpack/methods/ann/layer/glimpse.hpp +++ b/src/mlpack/methods/ann/layer/glimpse.hpp @@ -82,7 +82,6 @@ template < class Glimpse { public: - /** * Create the GlimpseLayer object using the specified ratio and rescale * parameter. @@ -145,7 +144,7 @@ class Glimpse this->location = location; } - //! Get the input width. + //! Get the input width. size_t const& InputWidth() const { return inputWidth; } //! Modify input the width. size_t& InputWidth() { return inputWidth; } @@ -222,7 +221,6 @@ class Glimpse const arma::Mat& input, arma::Mat& output) { - const size_t rStep = kSize; const size_t cStep = kSize; @@ -427,4 +425,4 @@ class Glimpse // Include implementation. #include "glimpse_impl.hpp" -#endif \ No newline at end of file +#endif diff --git a/src/mlpack/methods/ann/layer/layer_traits.hpp b/src/mlpack/methods/ann/layer/layer_traits.hpp index ff4fbf2d38..5b9d4c61cd 100644 --- a/src/mlpack/methods/ann/layer/layer_traits.hpp +++ b/src/mlpack/methods/ann/layer/layer_traits.hpp @@ -64,8 +64,8 @@ HAS_MEM_FUNC(Gradient, HasGradientCheck); // function. HAS_MEM_FUNC(Deterministic, HasDeterministicCheck); -// This gives us a HasParametersCheck type (where U is a function pointer) we -// can use with SFINAE to catch when a type has a Weights() function. +// This gives us a HasParametersCheck type (where U is a function pointer) +// we can use with SFINAE to catch when a type has a Weights() function. HAS_MEM_FUNC(Parameters, HasParametersCheck); // This gives us a HasAddCheck type (where U is a function pointer) we diff --git a/src/mlpack/methods/ann/layer/leaky_relu.hpp b/src/mlpack/methods/ann/layer/leaky_relu.hpp index 8c600f9cff..2e5b423293 100644 --- a/src/mlpack/methods/ann/layer/leaky_relu.hpp +++ b/src/mlpack/methods/ann/layer/leaky_relu.hpp @@ -165,7 +165,6 @@ class LeakyReLU //! Leakyness Parameter in the range 0 ::Forward( // Approximation of the hyperbolic tangent. The acuracy however is // about 0.00001 lower as using tanh. Credits go to Leon Bottou. - output.transform( [](double x) + output.transform([](double x) { //! Fast approximation of exp(-x) for x positive. static constexpr double A0 = 1.0; @@ -55,7 +55,7 @@ void LogSoftMax::Forward( } return 0.0; - } ); + }); output = input - (maxInput + std::log(arma::accu(output))); } diff --git a/src/mlpack/methods/ann/layer/lookup.hpp b/src/mlpack/methods/ann/layer/lookup.hpp index 7e0a5a29fc..e89f3e43f2 100644 --- a/src/mlpack/methods/ann/layer/lookup.hpp +++ b/src/mlpack/methods/ann/layer/lookup.hpp @@ -112,7 +112,6 @@ class Lookup void Serialize(Archive& ar, const unsigned int /* version */); private: - //! Locally-stored number of input units. size_t inSize; diff --git a/src/mlpack/methods/ann/layer/lstm.hpp b/src/mlpack/methods/ann/layer/lstm.hpp index 9c671fea85..773f97bb69 100644 --- a/src/mlpack/methods/ann/layer/lstm.hpp +++ b/src/mlpack/methods/ann/layer/lstm.hpp @@ -139,7 +139,6 @@ class LSTM void Serialize(Archive& ar, const unsigned int /* version */); private: - //! Locally-stored number of input units. size_t inSize; diff --git a/src/mlpack/methods/ann/layer/max_pooling.hpp b/src/mlpack/methods/ann/layer/max_pooling.hpp index faa7f6a9d6..59497a5da7 100644 --- a/src/mlpack/methods/ann/layer/max_pooling.hpp +++ b/src/mlpack/methods/ann/layer/max_pooling.hpp @@ -51,7 +51,7 @@ template < > class MaxPooling { -public: + public: //! Create the MaxPooling object. MaxPooling(); @@ -141,7 +141,6 @@ public: void Serialize(Archive& ar, const unsigned int /* version */); private: - /** * Apply pooling to the input and store the results. * diff --git a/src/mlpack/methods/ann/layer/mean_pooling.hpp b/src/mlpack/methods/ann/layer/mean_pooling.hpp index 8c29df7689..d52f43309e 100644 --- a/src/mlpack/methods/ann/layer/mean_pooling.hpp +++ b/src/mlpack/methods/ann/layer/mean_pooling.hpp @@ -32,7 +32,7 @@ template < > class MeanPooling { -public: + public: //! Create the MeanPooling object. MeanPooling(); @@ -121,7 +121,6 @@ public: void Serialize(Archive& ar, const unsigned int /* version */); private: - /** * Apply pooling to the input and store the results. * @@ -208,13 +207,13 @@ public: //! Locally-stored output height. size_t outputHeight; - //! Locally-stored reset parameter used to initialize the module once. + //! Locally-stored reset parameter used to initialize the module once. bool reset; //! Rounding operation used. bool floor; - //! If true use maximum a posteriori during the forward pass. + //! If true use maximum a posteriori during the forward pass. bool deterministic; //! Locally-stored stored rounding offset. diff --git a/src/mlpack/methods/ann/layer/mean_pooling_impl.hpp b/src/mlpack/methods/ann/layer/mean_pooling_impl.hpp index 5008763752..113845e782 100644 --- a/src/mlpack/methods/ann/layer/mean_pooling_impl.hpp +++ b/src/mlpack/methods/ann/layer/mean_pooling_impl.hpp @@ -75,10 +75,7 @@ void MeanPooling::Forward( slices); for (size_t s = 0; s < inputTemp.n_slices; s++) - { - Pooling(inputTemp.slice(s), outputTemp.slice(s)); - } output = arma::Mat(outputTemp.memptr(), outputTemp.n_elem, 1); diff --git a/src/mlpack/methods/ann/layer/parametric_relu.hpp b/src/mlpack/methods/ann/layer/parametric_relu.hpp index 3f5ba92d86..1362dd6207 100644 --- a/src/mlpack/methods/ann/layer/parametric_relu.hpp +++ b/src/mlpack/methods/ann/layer/parametric_relu.hpp @@ -202,7 +202,6 @@ class PReLU //! Leakyness Parameter given by user in the range 0 < alpha < 1. double user_alpha; - }; // class PReLU } // namespace ann @@ -211,4 +210,4 @@ class PReLU // Include implementation. #include "parametric_relu_impl.hpp" -#endif \ No newline at end of file +#endif diff --git a/src/mlpack/methods/ann/layer/parametric_relu_impl.hpp b/src/mlpack/methods/ann/layer/parametric_relu_impl.hpp index 0c290c2413..94fbd00dcf 100644 --- a/src/mlpack/methods/ann/layer/parametric_relu_impl.hpp +++ b/src/mlpack/methods/ann/layer/parametric_relu_impl.hpp @@ -25,7 +25,7 @@ template PReLU::PReLU( const double user_alpha) : user_alpha(user_alpha) { - alpha.set_size(1,1); + alpha.set_size(1, 1); alpha(0) = user_alpha; } diff --git a/src/mlpack/methods/ann/layer/recurrent_impl.hpp b/src/mlpack/methods/ann/layer/recurrent_impl.hpp index 80184109b6..6d5da1c152 100644 --- a/src/mlpack/methods/ann/layer/recurrent_impl.hpp +++ b/src/mlpack/methods/ann/layer/recurrent_impl.hpp @@ -146,7 +146,7 @@ void Recurrent::Backward( boost::apply_visitor(BackwardVisitor(std::move(boost::apply_visitor( outputParameterVisitor, feedbackModule)), std::move( boost::apply_visitor(deltaVisitor, recurrentModule)), std::move( - boost::apply_visitor(deltaVisitor, feedbackModule))),feedbackModule); + boost::apply_visitor(deltaVisitor, feedbackModule))), feedbackModule); } else { diff --git a/src/mlpack/methods/ann/layer/sequential.hpp b/src/mlpack/methods/ann/layer/sequential.hpp index ad84c7a177..b926f454a9 100644 --- a/src/mlpack/methods/ann/layer/sequential.hpp +++ b/src/mlpack/methods/ann/layer/sequential.hpp @@ -46,7 +46,6 @@ template < class Sequential { public: - /** * Create the Sequential object using the specified parameters. * diff --git a/src/mlpack/methods/ann/rnn.hpp b/src/mlpack/methods/ann/rnn.hpp index fe4aaad661..185faadcee 100644 --- a/src/mlpack/methods/ann/rnn.hpp +++ b/src/mlpack/methods/ann/rnn.hpp @@ -134,7 +134,7 @@ class RNN * @param predictors Input predictors. * @param results Matrix to put output predictions of responses into. */ - void Predict(arma::mat& predictors, arma::mat& results); + void Predict(const arma::mat& predictors, arma::mat& results); /** * Evaluate the recurrent neural network with the given parameters. This diff --git a/src/mlpack/methods/ann/rnn_impl.hpp b/src/mlpack/methods/ann/rnn_impl.hpp index 07809d778f..4130ef0933 100644 --- a/src/mlpack/methods/ann/rnn_impl.hpp +++ b/src/mlpack/methods/ann/rnn_impl.hpp @@ -145,7 +145,7 @@ void RNN::Train( template void RNN::Predict( - arma::mat& predictors, arma::mat& results) + const arma::mat& predictors, arma::mat& results) { if (parameter.is_empty()) { @@ -163,10 +163,7 @@ void RNN::Predict( for (size_t i = 0; i < predictors.n_cols; i++) { - SinglePredict( - arma::mat(predictors.colptr(i), predictors.n_rows, 1, false, true), - resultsTemp); - + SinglePredict(predictors.col(i), resultsTemp); results.col(i) = resultsTemp; } } @@ -202,8 +199,7 @@ double RNN::Evaluate( ResetDeterministic(); } - arma::mat input = arma::mat(predictors.colptr(i), predictors.n_rows, - 1, false, true); + arma::mat input = predictors.col(i); arma::mat target = arma::mat(responses.colptr(i), responses.n_rows, 1, false, true); diff --git a/src/mlpack/methods/ann/visitor/copy_visitor.hpp b/src/mlpack/methods/ann/visitor/copy_visitor.hpp index 3e6812fa1f..b2894ac0ed 100644 --- a/src/mlpack/methods/ann/visitor/copy_visitor.hpp +++ b/src/mlpack/methods/ann/visitor/copy_visitor.hpp @@ -26,7 +26,7 @@ class CopyVisitor : public boost::static_visitor { public: template - LayerTypes operator () (LayerType* ) const; + LayerTypes operator()(LayerType*) const; }; } // namespace ann diff --git a/src/mlpack/methods/ann/visitor/forward_visitor_impl.hpp b/src/mlpack/methods/ann/visitor/forward_visitor_impl.hpp index f676a2400c..ce1c36e368 100644 --- a/src/mlpack/methods/ann/visitor/forward_visitor_impl.hpp +++ b/src/mlpack/methods/ann/visitor/forward_visitor_impl.hpp @@ -20,8 +20,8 @@ namespace ann { //! ForwardVisitor visitor class. inline ForwardVisitor::ForwardVisitor(arma::mat&& input, arma::mat&& output) : - input(std::move(input)), - output(std::move(output)) + input(std::move(input)), + output(std::move(output)) { /* Nothing to do here. */ } diff --git a/src/mlpack/methods/approx_kfn/drusilla_select_impl.hpp b/src/mlpack/methods/approx_kfn/drusilla_select_impl.hpp index a3e8d0ff3a..65c330cddd 100644 --- a/src/mlpack/methods/approx_kfn/drusilla_select_impl.hpp +++ b/src/mlpack/methods/approx_kfn/drusilla_select_impl.hpp @@ -129,7 +129,8 @@ void DrusillaSelect::Train( } }; - std::vector clist(m, std::make_pair(double(-DBL_MAX), size_t(-1))); + std::vector clist( + m, std::make_pair(double(-DBL_MAX), size_t(-1))); std::priority_queue, CandidateCmp> pq(CandidateCmp(), std::move(clist)); diff --git a/src/mlpack/methods/cf/cf.cpp b/src/mlpack/methods/cf/cf.cpp index d063f4bbda..e5586248c2 100644 --- a/src/mlpack/methods/cf/cf.cpp +++ b/src/mlpack/methods/cf/cf.cpp @@ -259,5 +259,5 @@ void CF::CleanData(const arma::mat& data, arma::sp_mat& cleanedData) cleanedData = arma::sp_mat(locations, values, maxItemID, maxUserID); } -} // namespace mlpack } // namespace cf +} // namespace mlpack diff --git a/src/mlpack/methods/cf/cf_impl.hpp b/src/mlpack/methods/cf/cf_impl.hpp index 14e4b7f631..03faf71fbd 100644 --- a/src/mlpack/methods/cf/cf_impl.hpp +++ b/src/mlpack/methods/cf/cf_impl.hpp @@ -91,7 +91,7 @@ CF::CF(const arma::sp_mat& data, { Log::Warn << "CF::CF(): neighbourhood size should be > 0(" << numUsersForSimilarity << " given). Setting value to 5.\n"; - //Setting Default Value of 5 + // Setting Default Value of 5 this->numUsersForSimilarity = 5; } @@ -168,7 +168,7 @@ void CF::Serialize(Archive& ar, const unsigned int /* version */) ar & CreateNVP(cleanedData, "cleanedData"); } -} // namespace mlpack } // namespace cf +} // namespace mlpack #endif diff --git a/src/mlpack/methods/cf/svd_wrapper.hpp b/src/mlpack/methods/cf/svd_wrapper.hpp index 3ae381c52d..8f1a87db97 100644 --- a/src/mlpack/methods/cf/svd_wrapper.hpp +++ b/src/mlpack/methods/cf/svd_wrapper.hpp @@ -41,8 +41,11 @@ class SVDWrapper { public: // empty constructor - SVDWrapper(const Factorizer& factorizer = Factorizer()) - : factorizer(factorizer) {}; + SVDWrapper(const Factorizer& factorizer = Factorizer()) : + factorizer(factorizer) + { + // Nothing to do here. + } /** * Factorizer function which takes SVD of the given matrix and returns the diff --git a/src/mlpack/methods/cf/svd_wrapper_impl.hpp b/src/mlpack/methods/cf/svd_wrapper_impl.hpp index 91935e46ed..2c97eb0ef9 100644 --- a/src/mlpack/methods/cf/svd_wrapper_impl.hpp +++ b/src/mlpack/methods/cf/svd_wrapper_impl.hpp @@ -22,7 +22,7 @@ double mlpack::cf::SVDWrapper::Apply(const arma::mat& V, // construct sigma matrix sigma.zeros(V.n_rows, V.n_cols); - for(size_t i = 0;i < sigma.n_rows && i < sigma.n_cols;i++) + for (size_t i = 0; i < sigma.n_rows && i < sigma.n_cols; i++) sigma(i, i) = E(i, 0); arma::mat V_rec = W * sigma * arma::trans(H); @@ -44,7 +44,7 @@ double mlpack::cf::SVDWrapper::Apply(const arma::mat& V, // construct sigma matrix sigma.zeros(V.n_rows, V.n_cols); - for(size_t i = 0;i < sigma.n_rows && i < sigma.n_cols;i++) + for (size_t i = 0; i < sigma.n_rows && i < sigma.n_cols; i++) sigma(i, i) = E(i, 0); arma::mat V_rec = W * sigma * arma::trans(H); @@ -62,7 +62,9 @@ double mlpack::cf::SVDWrapper::Apply(const arma::mat& V, // check if the given rank is valid if (r > V.n_rows || r > V.n_cols) { - Log::Info << "Rank " << r << ", given for decomposition is invalid." << std::endl; + Log::Info << "Rank " << r << ", given for decomposition is invalid." + << std::endl; + r = (V.n_rows > V.n_cols) ? V.n_cols : V.n_rows; Log::Info << "Setting decomposition rank to " << r << std::endl; } @@ -101,7 +103,9 @@ double mlpack::cf::SVDWrapper::Apply(const arma::mat& V, // check if the given rank is valid if (r > V.n_rows || r > V.n_cols) { - Log::Info << "Rank " << r << ", given for decomposition is invalid." << std::endl; + Log::Info << "Rank " << r << ", given for decomposition is invalid." + << std::endl; + r = (V.n_rows > V.n_cols) ? V.n_cols : V.n_rows; Log::Info << "Setting decomposition rank to " << r << std::endl; } diff --git a/src/mlpack/methods/decision_tree/CMakeLists.txt b/src/mlpack/methods/decision_tree/CMakeLists.txt index 178f8ec4dc..f6f99e5183 100644 --- a/src/mlpack/methods/decision_tree/CMakeLists.txt +++ b/src/mlpack/methods/decision_tree/CMakeLists.txt @@ -3,6 +3,7 @@ cmake_minimum_required(VERSION 2.8) # Define the files we need to compile. # Anything not in this list will not be compiled into mlpack. set(SOURCES + all_dimension_select.hpp decision_tree.hpp decision_tree_impl.hpp all_categorical_split.hpp diff --git a/src/mlpack/methods/decision_tree/all_categorical_split.hpp b/src/mlpack/methods/decision_tree/all_categorical_split.hpp index cb09eee30e..b92702ebc9 100644 --- a/src/mlpack/methods/decision_tree/all_categorical_split.hpp +++ b/src/mlpack/methods/decision_tree/all_categorical_split.hpp @@ -52,13 +52,14 @@ class AllCategoricalSplit * @param aux Auxiliary split information, which may be modified on a * successful split. */ - template + template static double SplitIfBetter( const double bestGain, const VecType& data, const size_t numCategories, const arma::Row& labels, const size_t numClasses, + const WeightVecType& weights, const size_t minimumLeafSize, arma::Col& classProbabilities, AuxiliarySplitInfo& aux); diff --git a/src/mlpack/methods/decision_tree/all_categorical_split_impl.hpp b/src/mlpack/methods/decision_tree/all_categorical_split_impl.hpp index 0e364fb7c6..b15c7381f4 100644 --- a/src/mlpack/methods/decision_tree/all_categorical_split_impl.hpp +++ b/src/mlpack/methods/decision_tree/all_categorical_split_impl.hpp @@ -16,24 +16,39 @@ namespace mlpack { namespace tree { template -template +template double AllCategoricalSplit::SplitIfBetter( const double bestGain, const VecType& data, const size_t numCategories, const arma::Row& labels, const size_t numClasses, + const WeightVecType& weights, const size_t minimumLeafSize, arma::Col& classProbabilities, AuxiliarySplitInfo& /* aux */) { // Count the number of elements in each potential child. const double epsilon = 1e-7; // Tolerance for floating-point errors. - arma::Col counts(numCategories); - counts.zeros(); + arma::Col counts(numCategories, arma::fill::zeros); + + // If we are using weighted training, learn the weights for each child too. + arma::vec childWeightSums; + double sumWeight = 0.0; + if (UseWeights) + childWeightSums.zeros(numCategories); + for (size_t i = 0; i < data.n_elem; ++i) + { counts[(size_t) data[i]]++; + if (UseWeights) + { + childWeightSums[(size_t) data[i]] += weights[i]; + sumWeight += weights[i]; + } + } + // If each child will have the minimum number of points in it, we can split. // Otherwise we can't. if (arma::min(counts) < minimumLeafSize) @@ -43,23 +58,40 @@ double AllCategoricalSplit::SplitIfBetter( // that would be assigned to each child. arma::uvec childPositions(numCategories, arma::fill::zeros); std::vector> childLabels(numCategories); + std::vector> childWeights(numCategories); for (size_t i = 0; i < numCategories; ++i) + { + // Labels and weights should have same length. childLabels[i].zeros(counts[i]); + if (UseWeights) + childWeights[i].zeros(counts[i]); + } // Extract labels for each child. for (size_t i = 0; i < data.n_elem; ++i) { const size_t category = (size_t) data[i]; - childLabels[category][childPositions[category]++] = labels[i]; + + if (UseWeights) + { + childLabels[category][childPositions[category]] = labels[i]; + childWeights[category][childPositions[category]++] = weights[i]; + } + else + { + childLabels[category][childPositions[category]++] = labels[i]; + } } double overallGain = 0.0; for (size_t i = 0; i < counts.n_elem; ++i) { // Calculate the gain of this child. - const double childPct = double(counts[i]) / double(data.n_elem); - const double childGain = FitnessFunction::Evaluate(childLabels[i], - numClasses); + const double childPct = UseWeights ? + double(childWeightSums[i]) / sumWeight : + double(counts[i]) / double(data.n_elem); + const double childGain = FitnessFunction::template Evaluate( + childLabels[i], numClasses, childWeights[i]); overallGain += childPct * childGain; } diff --git a/src/mlpack/methods/decision_tree/all_dimension_select.hpp b/src/mlpack/methods/decision_tree/all_dimension_select.hpp new file mode 100644 index 0000000000..6b9681072b --- /dev/null +++ b/src/mlpack/methods/decision_tree/all_dimension_select.hpp @@ -0,0 +1,54 @@ +/** + * @file all_dimension_select.hpp + * @author Ryan Curtin + * + * Selects all dimensions for a split. + */ +#ifndef MLPACK_METHODS_DECISION_TREE_ALL_DIMENSION_SELECT_HPP +#define MLPACK_METHODS_DECISION_TREE_ALL_DIMENSION_SELECT_HPP + +namespace mlpack { +namespace tree { + +/** + * This dimension selection policy allows any dimension to be selected for + * splitting. + */ +class AllDimensionSelect +{ + public: + /** + * Construct the AllDimensionSelect object for the given number of dimensions. + */ + AllDimensionSelect(const size_t dimensions) : i(0), dimensions(dimensions) { } + + /** + * Get the first dimension to select from. + */ + size_t Begin() + { + i = 0; + return 0; + } + + /** + * Get the last dimension to select from. + */ + size_t End() const { return dimensions; } + + /** + * Get the next dimension. + */ + size_t Next() { return ++i; } + + private: + //! The current dimension we are looking at. + size_t i; + //! The number of dimensions to select from. + const size_t dimensions; +}; + +} // namespace tree +} // namespace mlpack + +#endif diff --git a/src/mlpack/methods/decision_tree/best_binary_numeric_split.hpp b/src/mlpack/methods/decision_tree/best_binary_numeric_split.hpp index 791eace4f4..4b0f039fad 100644 --- a/src/mlpack/methods/decision_tree/best_binary_numeric_split.hpp +++ b/src/mlpack/methods/decision_tree/best_binary_numeric_split.hpp @@ -50,12 +50,13 @@ class BestBinaryNumericSplit * @param aux Auxiliary split information, which may be modified on a * successful split. */ - template + template static double SplitIfBetter( const double bestGain, const VecType& data, const arma::Row& labels, const size_t numClasses, + const WeightVecType& weights, const size_t minimumLeafSize, arma::Col& classProbabilities, AuxiliarySplitInfo& aux); diff --git a/src/mlpack/methods/decision_tree/best_binary_numeric_split_impl.hpp b/src/mlpack/methods/decision_tree/best_binary_numeric_split_impl.hpp index 8a57a8c7e9..f8a0c95841 100644 --- a/src/mlpack/methods/decision_tree/best_binary_numeric_split_impl.hpp +++ b/src/mlpack/methods/decision_tree/best_binary_numeric_split_impl.hpp @@ -16,12 +16,13 @@ namespace mlpack { namespace tree { template -template +template double BestBinaryNumericSplit::SplitIfBetter( const double bestGain, const VecType& data, const arma::Row& labels, const size_t numClasses, + const WeightVecType& weights, const size_t minimumLeafSize, arma::Col& classProbabilities, AuxiliarySplitInfo& /* aux */) @@ -33,8 +34,18 @@ double BestBinaryNumericSplit::SplitIfBetter( // Next, sort the data. arma::uvec sortedIndices = arma::sort_index(data); arma::Row sortedLabels(labels.n_elem); + arma::rowvec sortedWeights; for (size_t i = 0; i < sortedLabels.n_elem; ++i) - sortedLabels[sortedIndices[i]] = labels[i]; + sortedLabels[i] = labels[sortedIndices[i]]; + + // Only initialize if we are using weights. + if (UseWeights) + { + sortedWeights.set_size(sortedLabels.n_elem); + // The weights must keep the same order of labels + for (size_t i = 0; i < sortedLabels.n_elem; ++i) + sortedWeights[i] = weights[sortedIndices[i]]; + } // Loop through all possible split points, choosing the best one. Also, force // a minimum leaf size of 1 (empty children don't make sense). @@ -46,18 +57,40 @@ double BestBinaryNumericSplit::SplitIfBetter( if (data[sortedIndices[index]] == data[sortedIndices[index - 1]]) continue; - // Calculate the gain for the left and right child. - const double leftGain = FitnessFunction::Evaluate(sortedLabels.subvec(0, - index - 1), numClasses); - const double rightGain = FitnessFunction::Evaluate(sortedLabels.subvec( - index, sortedLabels.n_elem - 1), numClasses); + // Calculate the gain for the left and right child. Only use weights if + // needed. + const double leftGain = UseWeights ? + FitnessFunction::template Evaluate(sortedLabels.subvec(0, + index - 1), numClasses, sortedWeights.subvec(0, index - 1)) : + FitnessFunction::template Evaluate(sortedLabels.subvec(0, + index - 1), numClasses, sortedWeights /* ignored */); + const double rightGain = UseWeights ? + FitnessFunction::template Evaluate(sortedLabels.subvec(index, + sortedLabels.n_elem - 1), numClasses, sortedWeights.subvec(index, + sortedLabels.n_elem - 1)) : + FitnessFunction::template Evaluate(sortedLabels.subvec(index, + sortedLabels.n_elem - 1), numClasses, sortedWeights /* ignored */); - // Calculate the fraction of points in the left and right children. - const double leftRatio = double(index) / double(sortedLabels.n_elem); - const double rightRatio = 1.0 - leftRatio; + double gain; + if (UseWeights) + { + const double leftWeights = arma::accu(sortedWeights.subvec(0, index - 1)); + const double rightWeights = arma::accu(sortedWeights.subvec(index, + sortedWeights.n_elem - 1)); + const double fullWeight = leftWeights + rightWeights; - // Calculate the gain at this split point. - const double gain = leftRatio * leftGain + rightRatio * rightGain; + gain = (leftWeights / fullWeight) * leftGain + + (rightWeights / fullWeight) * rightGain; + } + else + { + // Calculate the fraction of points in the left and right children. + const double leftRatio = double(index) / double(sortedLabels.n_elem); + const double rightRatio = 1.0 - leftRatio; + + // Calculate the gain at this split point. + gain = leftRatio * leftGain + rightRatio * rightGain; + } // Corner case: is this the best possible split? if (gain == 0.0) diff --git a/src/mlpack/methods/decision_tree/decision_tree.hpp b/src/mlpack/methods/decision_tree/decision_tree.hpp index 7bf5d6d4d9..9d8e743aed 100644 --- a/src/mlpack/methods/decision_tree/decision_tree.hpp +++ b/src/mlpack/methods/decision_tree/decision_tree.hpp @@ -17,6 +17,8 @@ #include "gini_gain.hpp" #include "best_binary_numeric_split.hpp" #include "all_categorical_split.hpp" +#include "all_dimension_select.hpp" +#include namespace mlpack { namespace tree { @@ -31,6 +33,7 @@ namespace tree { template class NumericSplitType = BestBinaryNumericSplit, template class CategoricalSplitType = AllCategoricalSplit, + typename DimensionSelectionType = AllDimensionSelect, typename ElemType = double, bool NoRecursion = false> class DecisionTree : @@ -44,6 +47,8 @@ class DecisionTree : typedef NumericSplitType NumericSplit; //! Allow access to the categorical split type. typedef CategoricalSplitType CategoricalSplit; + //! Allow access to the dimension selection type. + typedef DimensionSelectionType DimensionSelection; /** * Construct the decision tree on the given data and labels, where the data @@ -81,6 +86,53 @@ class DecisionTree : const size_t numClasses, const size_t minimumLeafSize = 10); + /** + * Construct the decision tree on the given data and labels with weights, + * where the data can be both numeric and categorical. Setting + * minimumLeafSize too small may cause the tree to overfit, but setting it too + * large may cause it to underfit. + * + * @param data Dataset to train on. + * @param datasetInfo Type information for each dimension of the dataset. + * @param labels Labels for each training point. + * @param numClasses Number of classes in the dataset. + * @param weights The weight list of given label. + * @param minimumLeafSize Minimum number of points in each leaf node. + */ + template + DecisionTree(MatType&& data, + const data::DatasetInfo& datasetInfo, + LabelsType&& labels, + const size_t numClasses, + WeightsType&& weights, + const size_t minimumLeafSize = 10, + const std::enable_if_t::type>::value>* + = 0); + + /** + * Construct the decision tree on the given data and labels with weights, + * assuming that the data is all of the numeric type. Setting minimumLeafSize + * too small may cause the tree to overfit, but setting it too large may cause + * it to underfit. + * + * @param data Dataset to train on. + * @param labels Labels for each training point. + * @param numClasses Number of classes in the dataset. + * @param weights The Weight list of given labels. + * @param minimumLeafSize Minimum number of points in each leaf node. + */ + template + DecisionTree(MatType&& data, + LabelsType&& labels, + const size_t numClasses, + WeightsType&& weights, + const size_t minimumLeafSize = 10, + const std::enable_if_t::type>::value>* + = 0); + + /** * Construct a decision tree without training it. It will be a leaf node with * equal probabilities for each class. @@ -134,6 +186,7 @@ class DecisionTree : * @param datasetInfo Type information for each dimension. * @param labels Labels for each training point. * @param numClasses Number of classes in the dataset. + * @param weights Weights of all the labels * @param minimumLeafSize Minimum number of points in each leaf node. */ template @@ -152,6 +205,7 @@ class DecisionTree : * @param data Dataset to train on. * @param labels Labels for each training point. * @param numClasses Number of classes in the dataset. + * @param weights Weights of all the labels * @param minimumLeafSize Minimum number of points in each leaf node. */ template @@ -160,6 +214,51 @@ class DecisionTree : const size_t numClasses, const size_t minimumLeafSize = 10); + /** + * Train the decision tree on the given weighted data. This will overwrite + * the existing model. The data may have numeric and categorical types, + * specified by the datasetInfo parameter. Setting minimumLeafSize too small + * may cause the tree to overfit, but setting it too large may cause it to + * underfit. + * + * @param data Dataset to train on. + * @param datasetInfo Type information for each dimension. + * @param labels Labels for each training point. + * @param numClasses Number of classes in the dataset. + * @param weights Weights of all the labels + * @param minimumLeafSize Minimum number of points in each leaf node. + */ + template + void Train(MatType&& data, + const data::DatasetInfo& datasetInfo, + LabelsType&& labels, + const size_t numClasses, + WeightsType&& weights, + const size_t minimumLeafSize = 10, + const std::enable_if_t::type>::value>* = 0); + + /** + * Train the decision tree on the given weighted data, assuming that all + * dimensions are numeric. This will overwrite the given model. Setting + * minimumLeafSize too small may cause the tree to overfit, but setting it too + * large may cause it to underfit. + * + * @param data Dataset to train on. + * @param labels Labels for each training point. + * @param numClasses Number of classes in the dataset. + * @param weights Weights of all the labels + * @param minimumLeafSize Minimum number of points in each leaf node. + */ + template + void Train(MatType&& data, + LabelsType&& labels, + const size_t numClasses, + WeightsType&& weights, + const size_t minimumLeafSize = 10, + const std::enable_if_t::type>::value>* = 0); + /** * Classify the given point, using the entire tree. The predicted label is * returned. @@ -261,53 +360,10 @@ class DecisionTree : /** * Calculate the class probabilities of the given labels. */ - template + template void CalculateClassProbabilities(const RowType& labels, - const size_t numClasses); - - /** - * Corresponding to the public constructor, this method is designed for - * avoiding unnecessary copies during training. This constructor is called to - * create children. - * - * @param data Dataset to train on. - * @param begin Index of the starting point in the dataset that belongs to - * this node. - * @param count Number of points in this node. - * @param datasetInfo Type information for each dimension of the dataset. - * @param labels Labels for each training point. - * @param numClasses Number of classes in the dataset. - * @param minimumLeafSize Minimum number of points in each leaf node. - */ - template - DecisionTree(MatType& data, - const size_t begin, - const size_t count, - const data::DatasetInfo& datasetInfo, - arma::Row& labels, - const size_t numClasses, - const size_t minimumLeafSize = 10); - - /** - * Corresponding to the public constructor, this method is designed for - * avoiding unnecessary copies during training. This constructor is called to - * create children. - * - * @param data Dataset to train on. - * @param begin Index of the starting point in the dataset that belongs to - * this node. - * @param count Number of points in this node. - * @param labels Labels for each training point. - * @param numClasses Number of classes in the dataset. - * @param minimumLeafSize Minimum number of points in each leaf node. - */ - template - DecisionTree(MatType& data, - const size_t begin, - const size_t count, - arma::Row& labels, - const size_t numClasses, - const size_t minimumLeafSize = 10); + const size_t numClasses, + const WeightsRowType& weights); /** * Corresponding to the public Train() method, this method is designed for @@ -323,13 +379,14 @@ class DecisionTree : * @param numClasses Number of classes in the dataset. * @param minimumLeafSize Minimum number of points in each leaf node. */ - template + template void Train(MatType& data, const size_t begin, const size_t count, const data::DatasetInfo& datasetInfo, arma::Row& labels, const size_t numClasses, + arma::rowvec& weights, const size_t minimumLeafSize = 10); /** @@ -345,12 +402,13 @@ class DecisionTree : * @param numClasses Number of classes in the dataset. * @param minimumLeafSize Minimum number of points in each leaf node. */ - template + template void Train(MatType& data, const size_t begin, const size_t count, arma::Row& labels, const size_t numClasses, + arma::rowvec& weights, const size_t minimumLeafSize = 10); }; @@ -360,10 +418,12 @@ class DecisionTree : template class NumericSplitType = BestBinaryNumericSplit, template class CategoricalSplitType = AllCategoricalSplit, + typename DimensionSelectType = AllDimensionSelect, typename ElemType = double> using DecisionStump = DecisionTree; diff --git a/src/mlpack/methods/decision_tree/decision_tree_impl.hpp b/src/mlpack/methods/decision_tree/decision_tree_impl.hpp index 648e5028b7..7046de3bd3 100644 --- a/src/mlpack/methods/decision_tree/decision_tree_impl.hpp +++ b/src/mlpack/methods/decision_tree/decision_tree_impl.hpp @@ -15,16 +15,18 @@ namespace mlpack { namespace tree { -//! Construct and train. +//! Construct and train without weight. template class NumericSplitType, template class CategoricalSplitType, + typename DimensionSelectionType, typename ElemType, bool NoRecursion> template DecisionTree::DecisionTree(MatType&& data, const data::DatasetInfo& datasetInfo, @@ -32,96 +34,135 @@ DecisionTree::type TrueMatType; typedef typename std::remove_reference::type TrueLabelsType; + TrueMatType tmpData(std::forward(data)); TrueLabelsType tmpLabels(std::forward(labels)); + // Pass off work to the Train() method. - Train(tmpData, 0, tmpData.n_cols, datasetInfo, - tmpLabels, numClasses, minimumLeafSize); + arma::rowvec weights; // Fake weights, not used. + Train(tmpData, 0, tmpData.n_cols, datasetInfo, tmpLabels, numClasses, + weights, minimumLeafSize); } //! Construct and train. template class NumericSplitType, template class CategoricalSplitType, + typename DimensionSelectionType, typename ElemType, bool NoRecursion> template DecisionTree::DecisionTree(MatType&& data, LabelsType&& labels, const size_t numClasses, const size_t minimumLeafSize) { - // copy or move data + // Copy or move data. typedef typename std::remove_reference::type TrueMatType; typedef typename std::remove_reference::type TrueLabelsType; TrueMatType tmpData(std::forward(data)); TrueLabelsType tmpLabels(std::forward(labels)); + // Pass off work to the Train() method. - Train(tmpData, 0, tmpData.n_cols, - tmpLabels, numClasses, minimumLeafSize); + arma::rowvec weights; // Fake weights, not used. + Train(tmpData, 0, tmpData.n_cols, tmpLabels, numClasses, weights, + minimumLeafSize); } -//! Construct and train. +//! Construct and train with weights. template class NumericSplitType, template class CategoricalSplitType, + typename DimensionSelectionType, typename ElemType, bool NoRecursion> -template +template DecisionTree::DecisionTree(MatType& data, - const size_t begin, - const size_t count, + NoRecursion>::DecisionTree(MatType&& data, const data::DatasetInfo& datasetInfo, - arma::Row& labels, + LabelsType&& labels, const size_t numClasses, - const size_t minimumLeafSize) + WeightsType&& weights, + const size_t minimumLeafSize, + const std::enable_if_t< + arma::is_arma_type< + typename std::remove_reference< + WeightsType>::type>::value>*) { - // Pass off work to the Train() method. - Train(data, begin, count, datasetInfo, labels, numClasses, minimumLeafSize); + // Copy or move data. + typedef typename std::remove_reference::type TrueMatType; + typedef typename std::remove_reference::type TrueLabelsType; + typedef typename std::remove_reference::type TrueWeightsType; + + TrueMatType tmpData(std::forward(data)); + TrueLabelsType tmpLabels(std::forward(labels)); + TrueWeightsType tmpWeights(std::forward(weights)); + + // Pass off work to the weighted Train() method. + Train(tmpData, 0, tmpData.n_cols, datasetInfo, tmpLabels, numClasses, + tmpWeights, minimumLeafSize); } -//! Construct and train. +//! Construct and train with weights. template class NumericSplitType, template class CategoricalSplitType, + typename DimensionSelectionType, typename ElemType, bool NoRecursion> -template +template DecisionTree::DecisionTree(MatType& data, - const size_t begin, - const size_t count, - arma::Row& labels, + NoRecursion>::DecisionTree(MatType&& data, + LabelsType&& labels, const size_t numClasses, - const size_t minimumLeafSize) + WeightsType&& weights, + const size_t minimumLeafSize, + const std::enable_if_t< + arma::is_arma_type< + typename std::remove_reference< + WeightsType>::type>::value>*) { - // Pass off work to the Train() method. - Train(data, begin, count, labels, numClasses, minimumLeafSize); + // Copy or move data. + typedef typename std::remove_reference::type TrueMatType; + typedef typename std::remove_reference::type TrueLabelsType; + typedef typename std::remove_reference::type TrueWeightsType; + + TrueMatType tmpData(std::forward(data)); + TrueLabelsType tmpLabels(std::forward(labels)); + TrueWeightsType tmpWeights(std::forward(weights)); + + // Pass off work to the weighted Train() method. + Train(tmpData, 0, tmpData.n_cols, tmpLabels, numClasses, tmpWeights, + minimumLeafSize); } //! Construct, don't train. template class NumericSplitType, template class CategoricalSplitType, + typename DimensionSelectionType, typename ElemType, bool NoRecursion> DecisionTree::DecisionTree(const size_t numClasses) : dimensionTypeOrMajorityClass(0), @@ -135,11 +176,13 @@ DecisionTree class NumericSplitType, template class CategoricalSplitType, + typename DimensionSelectionType, typename ElemType, bool NoRecursion> DecisionTree::DecisionTree(const DecisionTree& other) : NumericAuxiliarySplitInfo(other), @@ -157,11 +200,13 @@ DecisionTree class NumericSplitType, template class CategoricalSplitType, + typename DimensionSelectionType, typename ElemType, bool NoRecursion> DecisionTree::DecisionTree(DecisionTree&& other) : NumericAuxiliarySplitInfo(std::move(other)), @@ -179,16 +224,19 @@ DecisionTree class NumericSplitType, template class CategoricalSplitType, + typename DimensionSelectionType, typename ElemType, bool NoRecursion> DecisionTree& DecisionTree::operator=(const DecisionTree& other) { @@ -217,16 +265,19 @@ DecisionTree class NumericSplitType, template class CategoricalSplitType, + typename DimensionSelectionType, typename ElemType, bool NoRecursion> DecisionTree& DecisionTree::operator=(DecisionTree&& other) { @@ -255,11 +306,13 @@ DecisionTree class NumericSplitType, template class CategoricalSplitType, + typename DimensionSelectionType, typename ElemType, bool NoRecursion> DecisionTree::~DecisionTree() { @@ -271,12 +324,14 @@ DecisionTree class NumericSplitType, template class CategoricalSplitType, + typename DimensionSelectionType, typename ElemType, bool NoRecursion> template void DecisionTree::Train(MatType&& data, const data::DatasetInfo& datasetInfo, @@ -293,25 +348,32 @@ void DecisionTree::type TrueMatType; typedef typename std::remove_reference::type TrueLabelsType; + TrueMatType tmpData(std::forward(data)); TrueLabelsType tmpLabels(std::forward(labels)); + // Pass off work to the Train() method. - Train(tmpData, 0, tmpData.n_cols, datasetInfo, tmpLabels, numClasses, minimumLeafSize); + arma::rowvec weights; // Fake weights, not used. + Train(tmpData, 0, tmpData.n_cols, datasetInfo, tmpLabels, numClasses, + minimumLeafSize); } //! Train on the given data, assuming all dimensions are numeric. template class NumericSplitType, template class CategoricalSplitType, + typename DimensionSelectionType, typename ElemType, bool NoRecursion> template void DecisionTree::Train(MatType&& data, LabelsType&& labels, @@ -327,26 +389,125 @@ void DecisionTree::type TrueMatType; typedef typename std::remove_reference::type TrueLabelsType; + TrueMatType tmpData(std::forward(data)); TrueLabelsType tmpLabels(std::forward(labels)); + // Pass off work to the Train() method. - Train(tmpData, 0, tmpData.n_cols, - tmpLabels, numClasses, minimumLeafSize); + arma::rowvec weights; // Fake weights, not used. + Train(tmpData, 0, tmpData.n_cols, tmpLabels, numClasses, weights, + minimumLeafSize); +} + +//! Train on the given weighted data. +template class NumericSplitType, + template class CategoricalSplitType, + typename DimensionSelectionType, + typename ElemType, + bool NoRecursion> +template +void DecisionTree::Train(MatType&& data, + const data::DatasetInfo& datasetInfo, + LabelsType&& labels, + const size_t numClasses, + WeightsType&& weights, + const size_t minimumLeafSize, + const std::enable_if_t::type>::value>*) +{ + // Sanity check on data. + if (data.n_cols != labels.n_elem) + { + std::ostringstream oss; + oss << "DecisionTree::Train(): number of points (" << data.n_cols << ") " + << "does not match number of labels (" << labels.n_elem << ")!" + << std::endl; + throw std::invalid_argument(oss.str()); + } + + // Copy or move data. + typedef typename std::remove_reference::type TrueMatType; + typedef typename std::remove_reference::type TrueLabelsType; + typedef typename std::remove_reference::type TrueWeightsType; + + TrueMatType tmpData(std::forward(data)); + TrueLabelsType tmpLabels(std::forward(labels)); + TrueWeightsType tmpWeights(std::forward(weights)); + + // Pass off work to the Train() method. + Train(tmpData, 0, tmpData.n_cols, datasetInfo, tmpLabels, numClasses, + tmpWeights, minimumLeafSize); +} + +//! Train on the given weighted data. +template class NumericSplitType, + template class CategoricalSplitType, + typename DimensionSelectionType, + typename ElemType, + bool NoRecursion> +template +void DecisionTree::Train(MatType&& data, + LabelsType&& labels, + const size_t numClasses, + WeightsType&& weights, + const size_t minimumLeafSize, + const std::enable_if_t::type>::value>*) +{ + // Sanity check on data. + if (data.n_cols != labels.n_elem) + { + std::ostringstream oss; + oss << "DecisionTree::Train(): number of points (" << data.n_cols << ") " + << "does not match number of labels (" << labels.n_elem << ")!" + << std::endl; + throw std::invalid_argument(oss.str()); + } + + // Copy or move data. + typedef typename std::remove_reference::type TrueMatType; + typedef typename std::remove_reference::type TrueLabelsType; + typedef typename std::remove_reference::type TrueWeightsType; + + TrueMatType tmpData(std::forward(data)); + TrueLabelsType tmpLabels(std::forward(labels)); + TrueWeightsType tmpWeights(std::forward(weights)); + + // Pass off work to the Train() method. + Train(tmpData, 0, tmpData.n_cols, tmpLabels, numClasses, tmpWeights, + minimumLeafSize); } //! Train on the given data. template class NumericSplitType, template class CategoricalSplitType, + typename DimensionSelectionType, typename ElemType, bool NoRecursion> -template +template void DecisionTree::Train(MatType& data, const size_t begin, @@ -354,6 +515,7 @@ void DecisionTree& labels, const size_t numClasses, + arma::rowvec& weights, const size_t minimumLeafSize) { // Clear children if needed. @@ -366,26 +528,39 @@ void DecisionTree( + labels.subvec(begin, begin + count - 1), + numClasses, + UseWeights ? weights.subvec(begin, begin + count - 1) : weights); size_t bestDim = datasetInfo.Dimensionality(); // This means "no split". - for (size_t i = 0; i < datasetInfo.Dimensionality(); ++i) + DimensionSelectionType dimensions(datasetInfo.Dimensionality()); + for (size_t i = dimensions.Begin(); i != dimensions.End(); + i = dimensions.Next()) { double dimGain = -DBL_MAX; if (datasetInfo.Type(i) == data::Datatype::categorical) - dimGain = CategoricalSplit::SplitIfBetter(bestGain, - data.cols(begin, begin + count - 1).row(i), - datasetInfo.NumMappings(i), - labels.subvec(begin, begin + count - 1), - numClasses, minimumLeafSize, - classProbabilities, *this); + { + dimGain = CategoricalSplit::template SplitIfBetter(bestGain, + data.cols(begin, begin + count - 1).row(i), + datasetInfo.NumMappings(i), + labels.subvec(begin, begin + count - 1), + numClasses, + UseWeights ? weights.subvec(begin, begin + count - 1) : weights, + minimumLeafSize, + classProbabilities, + *this); + } else if (datasetInfo.Type(i) == data::Datatype::numeric) - dimGain = NumericSplit::SplitIfBetter(bestGain, - data.cols(begin, begin + count - 1).row(i), - labels.subvec(begin, begin + count - 1), - numClasses, minimumLeafSize, - classProbabilities, *this); + { + dimGain = NumericSplit::template SplitIfBetter(bestGain, + data.cols(begin, begin + count - 1).row(i), + labels.subvec(begin, begin + count - 1), + numClasses, + UseWeights ? weights.subvec(begin, begin + count - 1) : weights, + minimumLeafSize, + classProbabilities, + *this); + } // Was there an improvement? If so mark that it's the new best dimension. if (dimGain > bestGain) @@ -423,8 +598,10 @@ void DecisionTreeTrain(data, currentChildBegin, + currentCol - currentChildBegin, datasetInfo, labels, numClasses, + weights, currentCol - currentChildBegin); + } else - children.push_back(new DecisionTree(data, currentChildBegin, - currentCol - currentChildBegin, datasetInfo, - labels, numClasses, minimumLeafSize)); + { + child->Train(data, currentChildBegin, + currentCol - currentChildBegin, datasetInfo, labels, numClasses, + weights, minimumLeafSize); + } + children.push_back(child); } } else @@ -466,7 +651,10 @@ void DecisionTree( + labels.subvec(begin, begin + count - 1), + numClasses, + UseWeights ? weights.subvec(begin, begin + count - 1) : weights); } } @@ -474,18 +662,21 @@ void DecisionTree class NumericSplitType, template class CategoricalSplitType, + typename DimensionSelectionType, typename ElemType, bool NoRecursion> -template +template void DecisionTree::Train(MatType& data, const size_t begin, const size_t count, arma::Row& labels, const size_t numClasses, + arma::rowvec& weights, const size_t minimumLeafSize) { // Clear children if needed. @@ -501,15 +692,24 @@ void DecisionTree( + labels.subvec(begin, begin + count - 1), + numClasses, + UseWeights ? weights.subvec(begin, begin + count - 1) : weights); size_t bestDim = data.n_rows; // This means "no split". for (size_t i = 0; i < data.n_rows; ++i) { - double dimGain = NumericSplitType::SplitIfBetter(bestGain, - data.cols(begin, begin + count - 1).row(i), - labels.cols(begin, begin + count - 1), - numClasses, minimumLeafSize, classProbabilities, *this); + const double dimGain = NumericSplitType::template + SplitIfBetter(bestGain, + data.cols(begin, begin + count - 1).row(i), + labels.cols(begin, begin + count - 1), + numClasses, + UseWeights ? + weights.cols(begin, begin + count - 1) : + weights, + minimumLeafSize, + classProbabilities, + *this); if (dimGain > bestGain) { @@ -534,8 +734,10 @@ void DecisionTree childAssignments(count); for (size_t j = begin; j < begin + count; ++j) - childAssignments[j - begin] = NumericSplit::CalculateDirection(data(bestDim, j), - classProbabilities, *this); + { + childAssignments[j - begin] = NumericSplit::CalculateDirection( + data(bestDim, j), classProbabilities, *this); + } // Calculate counts of children in each node. arma::Row childCounts(numChildren); @@ -554,19 +756,27 @@ void DecisionTreeTrain(data, currentChildBegin, + currentCol - currentChildBegin, labels, numClasses, weights, + currentCol - currentChildBegin); + } else - children.push_back(new DecisionTree(data, currentChildBegin, - currentCol - currentChildBegin, - labels, numClasses, minimumLeafSize)); + { + child->Train(data, currentChildBegin, + currentCol - currentChildBegin, labels, numClasses, weights, + minimumLeafSize); + } + children.push_back(child); } } else @@ -575,21 +785,25 @@ void DecisionTree( + labels.subvec(begin, begin + count - 1), + numClasses, + UseWeights ? weights.subvec(begin, begin + count - 1) : weights); } - } //! Return the class. template class NumericSplitType, template class CategoricalSplitType, + typename DimensionSelectionType, typename ElemType, bool NoRecursion> template size_t DecisionTree::Classify(const VecType& point) const { @@ -606,12 +820,14 @@ size_t DecisionTree class NumericSplitType, template class CategoricalSplitType, + typename DimensionSelectionType, typename ElemType, bool NoRecursion> template void DecisionTree::Classify(const VecType& point, size_t& prediction, @@ -624,19 +840,22 @@ void DecisionTreeClassify(point, prediction, probabilities); + children[CalculateDirection(point)]->Classify(point, prediction, + probabilities); } //! Return the class for a set of points. template class NumericSplitType, template class CategoricalSplitType, + typename DimensionSelectionType, typename ElemType, bool NoRecursion> template void DecisionTree::Classify(const MatType& data, arma::Row& predictions) const @@ -657,12 +876,14 @@ void DecisionTree class NumericSplitType, template class CategoricalSplitType, + typename DimensionSelectionType, typename ElemType, bool NoRecursion> template void DecisionTree::Classify(const MatType& data, arma::Row& predictions, @@ -694,12 +915,14 @@ void DecisionTree class NumericSplitType, template class CategoricalSplitType, + typename DimensionSelectionType, typename ElemType, bool NoRecursion> template void DecisionTree::Serialize(Archive& ar, const unsigned int /* version */) @@ -740,12 +963,14 @@ void DecisionTree class NumericSplitType, template class CategoricalSplitType, + typename DimensionSelectionType, typename ElemType, bool NoRecursion> template size_t DecisionTree::CalculateDirection(const VecType& point) const { @@ -761,23 +986,37 @@ size_t DecisionTree class NumericSplitType, template class CategoricalSplitType, + typename DimensionSelectionType, typename ElemType, bool NoRecursion> -template +template void DecisionTree::CalculateClassProbabilities( const RowType& labels, - const size_t numClasses) + const size_t numClasses, + const WeightsRowType& weights) { classProbabilities.zeros(numClasses); + double sumWeights = 0.0; for (size_t i = 0; i < labels.n_elem; ++i) - classProbabilities[labels[i]]++; + { + if (UseWeights) + { + classProbabilities[labels[i]] += weights[i]; + sumWeights += weights[i]; + } + else + { + classProbabilities[labels[i]]++; + } + } // Now normalize into probabilities. - classProbabilities /= labels.n_elem; + classProbabilities /= UseWeights ? sumWeights : labels.n_elem; arma::uword maxIndex; classProbabilities.max(maxIndex); dimensionTypeOrMajorityClass = (size_t) maxIndex; diff --git a/src/mlpack/methods/decision_tree/decision_tree_main.cpp b/src/mlpack/methods/decision_tree/decision_tree_main.cpp index 017528b528..d1b7b01412 100644 --- a/src/mlpack/methods/decision_tree/decision_tree_main.cpp +++ b/src/mlpack/methods/decision_tree/decision_tree_main.cpp @@ -72,7 +72,8 @@ PROGRAM_INFO("Decision tree", PARAM_MATRIX_IN("training", "Matrix of training points.", "t"); PARAM_UROW_IN("labels", "Training labels.", "l"); PARAM_MATRIX_IN("test", "Matrix of test points.", "T"); -PARAM_UROW_IN("test_labels", "Test point labels, if accuracy calculation " +PARAM_MATRIX_IN("weights", "The weight of labels", "w"); +PARAM_UMATRIX_IN("test_labels", "Test point labels, if accuracy calculation " "is desired.", "L"); // Training parameters. @@ -171,8 +172,18 @@ void mlpackMain() // Now build the tree. const size_t minLeafSize = (size_t) CLI::GetParam("minimum_leaf_size"); - model.tree = DecisionTree<>(dataset, labels, numClasses, - minLeafSize); + // Create decision tree with weighted labels. + if (CLI::HasParam("weights")) + { + arma::Row weights = + std::move(CLI::GetParam>("weights")); + model.tree = DecisionTree<>(dataset, labels, numClasses, + weights, minLeafSize); + } + else + { + model.tree = DecisionTree<>(dataset, labels, numClasses, minLeafSize); + } // Do we need to print training error? if (CLI::HasParam("print_training_error")) @@ -188,7 +199,7 @@ void mlpackMain() ++correct; // Print number of correct points. - Log::Info << double(correct) / double(dataset.n_cols) * 100 << "\% " + Log::Info << double(correct) / double(dataset.n_cols) * 100 << "%% " << "correct on training set (" << correct << " / " << dataset.n_cols << ")." << endl; } @@ -220,7 +231,7 @@ void mlpackMain() ++correct; // Print number of correct points. - Log::Info << double(correct) / double(testPoints.n_cols) * 100 << "\% " + Log::Info << double(correct) / double(testPoints.n_cols) * 100 << "%% " << "correct on test set (" << correct << " / " << testPoints.n_cols << ")." << endl; } diff --git a/src/mlpack/methods/decision_tree/gini_gain.hpp b/src/mlpack/methods/decision_tree/gini_gain.hpp index c1f08da786..485b896c87 100644 --- a/src/mlpack/methods/decision_tree/gini_gain.hpp +++ b/src/mlpack/methods/decision_tree/gini_gain.hpp @@ -33,26 +33,54 @@ class GiniGain * * @param labels Set of labels to evaluate Gini impurity on. * @param numClasses Number of classes in the dataset. + * @param weights Weight of labels. */ - template + template static double Evaluate(const RowType& labels, - const size_t numClasses) + const size_t numClasses, + const WeightVecType& weights) { // Corner case: if there are no elements, the impurity is zero. if (labels.n_elem == 0) return 0.0; - arma::Col counts(numClasses); - counts.zeros(); - for (size_t i = 0; i < labels.n_elem; ++i) - counts[labels[i]]++; + // Count the number of elements in each class. + arma::vec counts(numClasses, arma::fill::zeros); // Calculate the Gini impurity of the un-split node. double impurity = 0.0; - for (size_t i = 0; i < numClasses; ++i) + + if (UseWeights) { - const double f = ((double) counts[i] / (double) labels.n_elem); - impurity += f * (1.0 - f); + // Sum all the weights up. + double accWeights = 0.0; + + for (size_t i = 0; i < labels.n_elem; ++i) + { + counts[labels[i]] += weights[i]; + accWeights += weights[i]; + } + + // Catch edge case: if there are no weights, the impurity is zero. + if (accWeights == 0.0) + return 0.0; + + for (size_t i = 0; i < numClasses; ++i) + { + const double f = ((double) counts[i] / (double) accWeights); + impurity += f * (1.0 - f); + } + } + else + { + for (size_t i = 0; i < labels.n_elem; ++i) + counts[labels[i]]++; + + for (size_t i = 0; i < numClasses; ++i) + { + const double f = ((double) counts[i] / (double) labels.n_elem); + impurity += f * (1.0 - f); + } } return -impurity; diff --git a/src/mlpack/methods/decision_tree/information_gain.hpp b/src/mlpack/methods/decision_tree/information_gain.hpp index 2dbf814ddd..4152360fc4 100644 --- a/src/mlpack/methods/decision_tree/information_gain.hpp +++ b/src/mlpack/methods/decision_tree/information_gain.hpp @@ -31,26 +31,54 @@ class InformationGain * @param labels Labels of the dataset. * @param numClasses Number of classes in the dataset. */ + template static double Evaluate(const arma::Row& labels, - const size_t numClasses) + const size_t numClasses, + const arma::Row& weights) { - // Edge case: if there are no elements, the gain is zero. - if (labels.n_elem == 0) - return 0.0; - - // Count the number of elements in each class. - arma::Col counts(numClasses); - counts.zeros(); - for (size_t i = 0; i < labels.n_elem; ++i) - counts[labels[i]]++; + // Edge case: if there are no elements, the gain is zero. + if (labels.n_elem == 0) + return 0.0; // Calculate the information gain. double gain = 0.0; - for (size_t i = 0; i < numClasses; ++i) + + // Count the number of elements in each class. + arma::Col counts(numClasses, arma::fill::zeros); + + if (UseWeights) { - const double f = ((double) counts[i] / (double) labels.n_elem); - if (f > 0.0) - gain += f * std::log2(f); + // Sum all the weights up. + double accWeights = 0.0; + + for (size_t i = 0; i < labels.n_elem; ++i) + { + counts[labels[i]] += weights[i]; + accWeights += weights[i]; + } + + // Corner case: return 0 if no weight. + if (accWeights == 0.0) + return 0.0; + + for (size_t i = 0; i < numClasses; ++i) + { + const double f = ((double) counts[i] / (double) accWeights); + if (f > 0.0) + gain += f * std::log2(f); + } + } + else + { + for (size_t i = 0; i < labels.n_elem; ++i) + counts[labels[i]]++; + + for (size_t i = 0; i < numClasses; ++i) + { + const double f = ((double) counts[i] / (double) labels.n_elem); + if (f > 0.0) + gain += f * std::log2(f); + } } return gain; diff --git a/src/mlpack/methods/det/dt_utils_impl.hpp b/src/mlpack/methods/det/dt_utils_impl.hpp index e9e1c61401..389cf314fc 100644 --- a/src/mlpack/methods/det/dt_utils_impl.hpp +++ b/src/mlpack/methods/det/dt_utils_impl.hpp @@ -158,7 +158,8 @@ DTree* Trainer(MatType& dataset, std::vector > prunedSequence; while (dtree.SubtreeLeaves() > 1) { - std::pair treeSeq(oldAlpha, dtree.SubtreeLeavesLogNegError()); + std::pair treeSeq(oldAlpha, + dtree.SubtreeLeavesLogNegError()); prunedSequence.push_back(treeSeq); oldAlpha = alpha; alpha = dtree.PruneAndUpdate(oldAlpha, dataset.n_cols, useVolumeReg); @@ -269,7 +270,7 @@ DTree* Trainer(MatType& dataset, cvRegularizationConstants[prunedSequence.size() - 2] += 2.0 * cvVal / (double) cvData.n_cols; - #pragma omp critical (DTreeCVUpdate) + #pragma omp critical(DTreeCVUpdate) regularizationConstants += cvRegularizationConstants; } Timer::Stop("cross_validation"); diff --git a/src/mlpack/methods/det/dtree.hpp b/src/mlpack/methods/det/dtree.hpp index 46547a79e6..04ce0b0743 100644 --- a/src/mlpack/methods/det/dtree.hpp +++ b/src/mlpack/methods/det/dtree.hpp @@ -316,7 +316,6 @@ class DTree void Serialize(Archive& ar, const unsigned int /* version */); private: - // Utility methods. /** @@ -336,7 +335,6 @@ class DTree const size_t splitDim, const ElemType splitValue, arma::Col& oldFromNew) const; - }; } // namespace det diff --git a/src/mlpack/methods/det/dtree_impl.hpp b/src/mlpack/methods/det/dtree_impl.hpp index fc7e10350b..6372b19c0f 100644 --- a/src/mlpack/methods/det/dtree_impl.hpp +++ b/src/mlpack/methods/det/dtree_impl.hpp @@ -34,8 +34,7 @@ namespace details { static_assert( std::is_same::value == true, - "The ElemType does not correspond to the matrix's element type." - ); + "The ElemType does not correspond to the matrix's element type."); typedef std::pair SplitItem; const typename MatType::row_type dimVec = @@ -118,9 +117,10 @@ namespace details const ElemType newVal = valsVec[i]; if (lastVal < ElemType(0) && newVal > ElemType(0) && zeroes > 0) { - Log::Assert(padding == 0); // we should arrive here once! + Log::Assert(padding == 0); // We should arrive here once! - // the minLeafSize > 0 also guarantees we're not entering right at the start. + // The minLeafSize > 0 also guarantees we're not entering right at the + // start. if (i >= minLeafSize && i <= n_elem - minLeafSize) splitVec.push_back(SplitItem(lastVal / 2.0, i)); @@ -145,7 +145,7 @@ namespace details lastVal = newVal; } } -}; +}; // namespace details template DTree::DTree() : @@ -412,7 +412,7 @@ double DTree::LogNegativeError(const size_t totalPoints) const 2 * std::log((double) totalPoints); StatType valDiffs = maxVals - minVals; - for (size_t i = 0;i < valDiffs.n_elem; ++i) + for (size_t i = 0; i < valDiffs.n_elem; ++i) { // Ignore very small dimensions to prevent overflow. if (valDiffs[i] > 1e-50) @@ -475,11 +475,13 @@ bool DTree::FindSplit(const MatType& data, // Get the values for splitting. The old implementation: // dimVec = data.row(dim).subvec(start, end - 1); // dimVec = arma::sort(dimVec); - // could be quite inefficient for sparse matrices, due to copy operations (3). - // This one has custom implementation for dense and sparse matrices. + // could be quite inefficient for sparse matrices, due to + // copy operations (3). This one has custom implementation for dense and + // sparse matrices. std::vector splitVec; - details::ExtractSplits(splitVec, data, dim, start, end, minLeafSize); + details::ExtractSplits(splitVec, data, dim, start, end, + minLeafSize); // Iterate on all the splits for this dimension for (typename std::vector::iterator i = splitVec.begin(); @@ -522,7 +524,7 @@ bool DTree::FindSplit(const MatType& data, - 2 * std::log((double) data.n_cols) - volumeWithoutDim; -#pragma omp critical (DTreeFindUpdate) +#pragma omp critical(DTreeFindUpdate) if ((actualMinDimError > minError) && dimSplitFound) { // Calculate actual error (in logspace) by adding terms back to our @@ -709,7 +711,7 @@ double DTree::Grow(MatType& data, if (useVolReg) { // This is wrong for now! - gT = alphaUpper;// / (subtreeLeavesVTInv - vTInv); + gT = alphaUpper; // / (subtreeLeavesVTInv - vTInv); } else { @@ -740,7 +742,7 @@ double DTree::PruneAndUpdate(const double oldAlpha, // Compute gT value for node t. volatile double gT; if (useVolReg) - gT = alphaUpper;// - std::log(subtreeLeavesVTInv - vTInv); + gT = alphaUpper; // - std::log(subtreeLeavesVTInv - vTInv); else gT = alphaUpper - std::log((double) (subtreeLeaves - 1)); @@ -869,7 +871,8 @@ double DTree::ComputeValue(const VecType& query) const } else { - // Return either of the two children - left or right, depending on the splitValue + // Return either of the two children - left or right, depending on the + // splitValue return (query[splitDim] <= splitValue) ? left->ComputeValue(query) : right->ComputeValue(query); @@ -878,7 +881,6 @@ double DTree::ComputeValue(const VecType& query) const return 0.0; } - // Index the buckets for possible usage later. template TagType DTree::TagTree(const TagType& tag) @@ -895,7 +897,6 @@ TagType DTree::TagTree(const TagType& tag) } } - template TagType DTree::FindBucket(const VecType& query) const { @@ -924,7 +925,7 @@ DTree::ComputeVariableImportance(arma::vec& importances) const std::stack nodes; nodes.push(this); - while(!nodes.empty()) + while (!nodes.empty()) { const DTree& curNode = *nodes.top(); nodes.pop(); @@ -945,7 +946,8 @@ DTree::ComputeVariableImportance(arma::vec& importances) const template template -void DTree::Serialize(Archive& ar, const unsigned int /* version */) +void DTree::Serialize(Archive& ar, + const unsigned int /* version */) { using data::CreateNVP; diff --git a/src/mlpack/methods/emst/dtb.hpp b/src/mlpack/methods/emst/dtb.hpp index c682b4aa0a..4b05bb4ca0 100644 --- a/src/mlpack/methods/emst/dtb.hpp +++ b/src/mlpack/methods/emst/dtb.hpp @@ -202,7 +202,6 @@ class DualTreeBoruvka * The values stored in the tree must be reset on each iteration. */ void Cleanup(); - }; // class DualTreeBoruvka } // namespace emst diff --git a/src/mlpack/methods/emst/dtb_impl.hpp b/src/mlpack/methods/emst/dtb_impl.hpp index 6032075741..a5ba58a314 100644 --- a/src/mlpack/methods/emst/dtb_impl.hpp +++ b/src/mlpack/methods/emst/dtb_impl.hpp @@ -202,7 +202,7 @@ void DualTreeBoruvka::AddAllEdges() size_t outEdge = neighborsOutComponent[component]; if (connections.Find(inEdge) != connections.Find(outEdge)) { - //totalDist = totalDist + dist; + // totalDist = totalDist + dist; // changed to make this agree with the cover tree code totalDist += neighborsDistances[component]; AddEdge(inEdge, outEdge, neighborsDistances[component]); diff --git a/src/mlpack/methods/emst/dtb_rules.hpp b/src/mlpack/methods/emst/dtb_rules.hpp index ee9c319b2d..b89c5b4754 100644 --- a/src/mlpack/methods/emst/dtb_rules.hpp +++ b/src/mlpack/methods/emst/dtb_rules.hpp @@ -129,11 +129,10 @@ class DTBRules size_t baseCases; //! The number of node combinations that have been scored. size_t scores; - }; // class DTBRules -} // emst namespace -} // mlpack namespace +} // namespace emst +} // namespace mlpack #include "dtb_rules_impl.hpp" diff --git a/src/mlpack/methods/emst/dtb_stat.hpp b/src/mlpack/methods/emst/dtb_stat.hpp index ffa5f2b58a..020a953a80 100644 --- a/src/mlpack/methods/emst/dtb_stat.hpp +++ b/src/mlpack/methods/emst/dtb_stat.hpp @@ -87,7 +87,6 @@ class DTBStat int ComponentMembership() const { return componentMembership; } //! Modify the component membership of this node. int& ComponentMembership() { return componentMembership; } - }; // class DTBStat } // namespace emst diff --git a/src/mlpack/methods/emst/edge_pair.hpp b/src/mlpack/methods/emst/edge_pair.hpp index b2a4ebb095..dae44b6a48 100644 --- a/src/mlpack/methods/emst/edge_pair.hpp +++ b/src/mlpack/methods/emst/edge_pair.hpp @@ -63,7 +63,6 @@ class EdgePair double Distance() const { return distance; } //! Modify the distance. double& Distance() { return distance; } - }; // class EdgePair } // namespace emst diff --git a/src/mlpack/methods/gmm/gmm_impl.hpp b/src/mlpack/methods/gmm/gmm_impl.hpp index 021533ad1d..62e586266e 100644 --- a/src/mlpack/methods/gmm/gmm_impl.hpp +++ b/src/mlpack/methods/gmm/gmm_impl.hpp @@ -59,7 +59,8 @@ double GMM::Train(const arma::mat& observations, bestLikelihood = LogLikelihood(observations, dists, weights); - Log::Info << "GMM::Train(): Log-likelihood of trial 0 is " << bestLikelihood << "." << std::endl; + Log::Info << "GMM::Train(): Log-likelihood of trial 0 is " + << bestLikelihood << "." << std::endl; // Now the temporary model. std::vector distsTrial(gaussians, @@ -159,7 +160,8 @@ double GMM::Train(const arma::mat& observations, weightsTrial = weightsOrig; } - fitter.Estimate(observations, probabilities, distsTrial, weightsTrial, useExistingModel); + fitter.Estimate(observations, probabilities, distsTrial, weightsTrial, + useExistingModel); // Check to see if the log-likelihood of this one is better. double newLikelihood = LogLikelihood(observations, distsTrial, diff --git a/src/mlpack/methods/hmm/hmm_regression.hpp b/src/mlpack/methods/hmm/hmm_regression.hpp index bfde35afb5..a6fa7af134 100644 --- a/src/mlpack/methods/hmm/hmm_regression.hpp +++ b/src/mlpack/methods/hmm/hmm_regression.hpp @@ -287,13 +287,13 @@ class HMMRegression : public HMM /** * Utility functions to facilitate the use of the HMM class for HMMR. */ - void StackData(const std::vector& predictors, - const std::vector& responses, - std::vector& dataSeq) const; + void StackData(const std::vector& predictors, + const std::vector& responses, + std::vector& dataSeq) const; - void StackData(const arma::mat& predictors, - const arma::vec& responses, - arma::mat& dataSeq) const; + void StackData(const arma::mat& predictors, + const arma::vec& responses, + arma::mat& dataSeq) const; /** * The Forward algorithm (part of the Forward-Backward algorithm). Computes @@ -327,8 +327,6 @@ class HMMRegression : public HMM const arma::vec& responses, const arma::vec& scales, arma::mat& backwardProb) const; - - }; } // namespace hmm diff --git a/src/mlpack/methods/hmm/hmm_regression_impl.hpp b/src/mlpack/methods/hmm/hmm_regression_impl.hpp index 09cc0b1844..61676305ee 100644 --- a/src/mlpack/methods/hmm/hmm_regression_impl.hpp +++ b/src/mlpack/methods/hmm/hmm_regression_impl.hpp @@ -115,7 +115,7 @@ void HMMRegression::Filter(const arma::mat& predictors, filterSeq.resize(responses.n_elem - ahead); filterSeq.zeros(); arma::vec nextSeq; - for(size_t i = 0; i < emission.size(); i++) + for (size_t i = 0; i < emission.size(); i++) { emission[i].Predict(predictors.cols(ahead, predictors.n_cols-1), nextSeq); filterSeq = filterSeq + nextSeq%(forwardProb.row(i).t()); @@ -137,12 +137,11 @@ void HMMRegression::Smooth(const arma::mat& predictors, smoothSeq.resize(responses.n_elem); smoothSeq.zeros(); arma::vec nextSeq; - for(size_t i = 0; i < emission.size(); i++) + for (size_t i = 0; i < emission.size(); i++) { emission[i].Predict(predictors, nextSeq); smoothSeq = smoothSeq + nextSeq%(stateProb.row(i).t()); } - } /** @@ -174,7 +173,7 @@ void HMMRegression::StackData(const std::vector& predictors, std::vector& dataSeq) const { arma::mat nextSeq; - for(size_t i = 0; i < predictors.size(); i++) + for (size_t i = 0; i < predictors.size(); i++) { nextSeq = predictors[i]; nextSeq.insert_rows(0, responses[i].t()); diff --git a/src/mlpack/methods/hoeffding_trees/binary_numeric_split_impl.hpp b/src/mlpack/methods/hoeffding_trees/binary_numeric_split_impl.hpp index d7a99c121f..1bb5e86974 100644 --- a/src/mlpack/methods/hoeffding_trees/binary_numeric_split_impl.hpp +++ b/src/mlpack/methods/hoeffding_trees/binary_numeric_split_impl.hpp @@ -126,7 +126,7 @@ void BinaryNumericSplit::Split( double min = DBL_MAX; double max = -DBL_MAX; for (typename std::multimap::const_iterator it = - sortedElements.begin();// (*it).first < bestSplit; ++it) + sortedElements.begin(); // (*it).first < bestSplit; ++it) it != sortedElements.end(); ++it) { // Move the point to the correct side of the split. diff --git a/src/mlpack/methods/hoeffding_trees/hoeffding_numeric_split_impl.hpp b/src/mlpack/methods/hoeffding_trees/hoeffding_numeric_split_impl.hpp index 1bbf137856..e7b9fc394c 100644 --- a/src/mlpack/methods/hoeffding_trees/hoeffding_numeric_split_impl.hpp +++ b/src/mlpack/methods/hoeffding_trees/hoeffding_numeric_split_impl.hpp @@ -184,7 +184,6 @@ double HoeffdingNumericSplit:: return double(classCounts.max()) / double(arma::sum(classCounts)); } - } template diff --git a/src/mlpack/methods/hoeffding_trees/hoeffding_tree_impl.hpp b/src/mlpack/methods/hoeffding_trees/hoeffding_tree_impl.hpp index bdf7fe6977..cd8749cc13 100644 --- a/src/mlpack/methods/hoeffding_trees/hoeffding_tree_impl.hpp +++ b/src/mlpack/methods/hoeffding_trees/hoeffding_tree_impl.hpp @@ -648,7 +648,6 @@ void HoeffdingTree< children.push_back(new HoeffdingTree(*datasetInfo, numClasses, successProbability, maxSamples, checkInterval, minSamples, categoricalSplits[0], numericSplits[0], dimensionMappings)); - } children[i]->MajorityClass() = childMajorities[i]; diff --git a/src/mlpack/methods/kernel_pca/kernel_pca.hpp b/src/mlpack/methods/kernel_pca/kernel_pca.hpp index bbe8b5d41c..60ef68e94f 100644 --- a/src/mlpack/methods/kernel_pca/kernel_pca.hpp +++ b/src/mlpack/methods/kernel_pca/kernel_pca.hpp @@ -122,7 +122,6 @@ class KernelPCA //! If true, the data will be scaled (by standard deviation) when Apply() is //! run. bool centerTransformedData; - }; // class KernelPCA } // namespace kpca diff --git a/src/mlpack/methods/kernel_pca/kernel_pca_impl.hpp b/src/mlpack/methods/kernel_pca/kernel_pca_impl.hpp index 5204918fa6..2d106aeb59 100644 --- a/src/mlpack/methods/kernel_pca/kernel_pca_impl.hpp +++ b/src/mlpack/methods/kernel_pca/kernel_pca_impl.hpp @@ -81,7 +81,7 @@ void KernelPCA::Apply(arma::mat& data, data.shed_rows(newDimension, data.n_rows - 1); } -} // namespace mlpack } // namespace kpca +} // namespace mlpack #endif diff --git a/src/mlpack/methods/kernel_pca/kernel_rules/naive_method.hpp b/src/mlpack/methods/kernel_pca/kernel_rules/naive_method.hpp index cb45b583b1..567ab51b83 100644 --- a/src/mlpack/methods/kernel_pca/kernel_rules/naive_method.hpp +++ b/src/mlpack/methods/kernel_pca/kernel_rules/naive_method.hpp @@ -21,71 +21,71 @@ namespace kpca { template class NaiveKernelRule { - public: - /** - * Construct the exact kernel matrix. - * - * @param data Input data points. - * @param transformedData Matrix to output results into. - * @param eigval KPCA eigenvalues will be written to this vector. - * @param eigvec KPCA eigenvectors will be written to this matrix. - * @param rank Rank to be used for matrix approximation. - * @param kernel Kernel to be used for computation. - */ - static void ApplyKernelMatrix(const arma::mat& data, - arma::mat& transformedData, - arma::vec& eigval, - arma::mat& eigvec, - const size_t /* unused */, - KernelType kernel = KernelType()) + public: + /** + * Construct the exact kernel matrix. + * + * @param data Input data points. + * @param transformedData Matrix to output results into. + * @param eigval KPCA eigenvalues will be written to this vector. + * @param eigvec KPCA eigenvectors will be written to this matrix. + * @param rank Rank to be used for matrix approximation. + * @param kernel Kernel to be used for computation. + */ + static void ApplyKernelMatrix(const arma::mat& data, + arma::mat& transformedData, + arma::vec& eigval, + arma::mat& eigvec, + const size_t /* unused */, + KernelType kernel = KernelType()) +{ + // Construct the kernel matrix. + arma::mat kernelMatrix; + // Resize the kernel matrix to the right size. + kernelMatrix.set_size(data.n_cols, data.n_cols); + + // Note that we only need to calculate the upper triangular part of the + // kernel matrix, since it is symmetric. This helps minimize the number of + // kernel evaluations. + for (size_t i = 0; i < data.n_cols; ++i) { - // Construct the kernel matrix. - arma::mat kernelMatrix; - // Resize the kernel matrix to the right size. - kernelMatrix.set_size(data.n_cols, data.n_cols); - - // Note that we only need to calculate the upper triangular part of the - // kernel matrix, since it is symmetric. This helps minimize the number of - // kernel evaluations. - for (size_t i = 0; i < data.n_cols; ++i) + for (size_t j = i; j < data.n_cols; ++j) { - for (size_t j = i; j < data.n_cols; ++j) - { - // Evaluate the kernel on these two points. - kernelMatrix(i, j) = kernel.Evaluate(data.unsafe_col(i), - data.unsafe_col(j)); - } + // Evaluate the kernel on these two points. + kernelMatrix(i, j) = kernel.Evaluate(data.unsafe_col(i), + data.unsafe_col(j)); } - - // Copy to the lower triangular part of the matrix. - for (size_t i = 1; i < data.n_cols; ++i) - for (size_t j = 0; j < i; ++j) - kernelMatrix(i, j) = kernelMatrix(j, i); - - // For PCA the data has to be centered, even if the data is centered. But it - // is not guaranteed that the data, when mapped to the kernel space, is also - // centered. Since we actually never work in the feature space we cannot - // center the data. So, we perform a "psuedo-centering" using the kernel - // matrix. - arma::rowvec rowMean = arma::sum(kernelMatrix, 0) / kernelMatrix.n_cols; - kernelMatrix.each_col() -= arma::sum(kernelMatrix, 1) / kernelMatrix.n_cols; - kernelMatrix.each_row() -= rowMean; - kernelMatrix += arma::sum(rowMean) / kernelMatrix.n_cols; - - // Eigendecompose the centered kernel matrix. - arma::eig_sym(eigval, eigvec, kernelMatrix); - - // Swap the eigenvalues since they are ordered backwards (we need largest to - // smallest). - for (size_t i = 0; i < floor(eigval.n_elem / 2.0); ++i) - eigval.swap_rows(i, (eigval.n_elem - 1) - i); - - // Flip the coefficients to produce the same effect. - eigvec = arma::fliplr(eigvec); - - transformedData = eigvec.t() * kernelMatrix; - transformedData.each_col() /= arma::sqrt(eigval); } + + // Copy to the lower triangular part of the matrix. + for (size_t i = 1; i < data.n_cols; ++i) + for (size_t j = 0; j < i; ++j) + kernelMatrix(i, j) = kernelMatrix(j, i); + + // For PCA the data has to be centered, even if the data is centered. But it + // is not guaranteed that the data, when mapped to the kernel space, is also + // centered. Since we actually never work in the feature space we cannot + // center the data. So, we perform a "psuedo-centering" using the kernel + // matrix. + arma::rowvec rowMean = arma::sum(kernelMatrix, 0) / kernelMatrix.n_cols; + kernelMatrix.each_col() -= arma::sum(kernelMatrix, 1) / kernelMatrix.n_cols; + kernelMatrix.each_row() -= rowMean; + kernelMatrix += arma::sum(rowMean) / kernelMatrix.n_cols; + + // Eigendecompose the centered kernel matrix. + arma::eig_sym(eigval, eigvec, kernelMatrix); + + // Swap the eigenvalues since they are ordered backwards (we need largest to + // smallest). + for (size_t i = 0; i < floor(eigval.n_elem / 2.0); ++i) + eigval.swap_rows(i, (eigval.n_elem - 1) - i); + + // Flip the coefficients to produce the same effect. + eigvec = arma::fliplr(eigvec); + + transformedData = eigvec.t() * kernelMatrix; + transformedData.each_col() /= arma::sqrt(eigval); +} }; } // namespace kpca diff --git a/src/mlpack/methods/kernel_pca/kernel_rules/nystroem_method.hpp b/src/mlpack/methods/kernel_pca/kernel_rules/nystroem_method.hpp index a8f82411e1..3ee0b0aed7 100644 --- a/src/mlpack/methods/kernel_pca/kernel_rules/nystroem_method.hpp +++ b/src/mlpack/methods/kernel_pca/kernel_rules/nystroem_method.hpp @@ -26,56 +26,56 @@ template< > class NystroemKernelRule { - public: - /** - * Construct the kernel matrix approximation using the nystroem method. - * - * @param data Input data points. - * @param transformedData Matrix to output results into. - * @param eigval KPCA eigenvalues will be written to this vector. - * @param eigvec KPCA eigenvectors will be written to this matrix. - * @param rank Rank to be used for matrix approximation. - * @param kernel Kernel to be used for computation. - */ - static void ApplyKernelMatrix(const arma::mat& data, - arma::mat& transformedData, - arma::vec& eigval, - arma::mat& eigvec, - const size_t rank, - KernelType kernel = KernelType()) - { - arma::mat G, v; - kernel::NystroemMethod nm(data, kernel, - rank); - nm.Apply(G); - transformedData = G.t() * G; + public: + /** + * Construct the kernel matrix approximation using the nystroem method. + * + * @param data Input data points. + * @param transformedData Matrix to output results into. + * @param eigval KPCA eigenvalues will be written to this vector. + * @param eigvec KPCA eigenvectors will be written to this matrix. + * @param rank Rank to be used for matrix approximation. + * @param kernel Kernel to be used for computation. + */ + static void ApplyKernelMatrix(const arma::mat& data, + arma::mat& transformedData, + arma::vec& eigval, + arma::mat& eigvec, + const size_t rank, + KernelType kernel = KernelType()) + { + arma::mat G, v; + kernel::NystroemMethod nm(data, kernel, + rank); + nm.Apply(G); + transformedData = G.t() * G; - // Center the reconstructed approximation. - math::Center(transformedData, transformedData); + // Center the reconstructed approximation. + math::Center(transformedData, transformedData); - // For PCA the data has to be centered, even if the data is centered. But - // it is not guaranteed that the data, when mapped to the kernel space, is - // also centered. Since we actually never work in the feature space we - // cannot center the data. So, we perform a "psuedo-centering" using the - // kernel matrix. - arma::colvec colMean = arma::sum(G, 1) / G.n_rows; - G.each_row() -= arma::sum(G, 0) / G.n_rows; - G.each_col() -= colMean; - G += arma::sum(colMean) / G.n_rows; + // For PCA the data has to be centered, even if the data is centered. But + // it is not guaranteed that the data, when mapped to the kernel space, is + // also centered. Since we actually never work in the feature space we + // cannot center the data. So, we perform a "psuedo-centering" using the + // kernel matrix. + arma::colvec colMean = arma::sum(G, 1) / G.n_rows; + G.each_row() -= arma::sum(G, 0) / G.n_rows; + G.each_col() -= colMean; + G += arma::sum(colMean) / G.n_rows; - // Eigendecompose the centered kernel matrix. - arma::eig_sym(eigval, eigvec, transformedData); + // Eigendecompose the centered kernel matrix. + arma::eig_sym(eigval, eigvec, transformedData); - // Swap the eigenvalues since they are ordered backwards (we need largest - // to smallest). - for (size_t i = 0; i < floor(eigval.n_elem / 2.0); ++i) - eigval.swap_rows(i, (eigval.n_elem - 1) - i); + // Swap the eigenvalues since they are ordered backwards (we need largest + // to smallest). + for (size_t i = 0; i < floor(eigval.n_elem / 2.0); ++i) + eigval.swap_rows(i, (eigval.n_elem - 1) - i); - // Flip the coefficients to produce the same effect. - eigvec = arma::fliplr(eigvec); + // Flip the coefficients to produce the same effect. + eigvec = arma::fliplr(eigvec); - transformedData = eigvec.t() * G.t(); - } + transformedData = eigvec.t() * G.t(); + } }; } // namespace kpca diff --git a/src/mlpack/methods/kmeans/elkan_kmeans_impl.hpp b/src/mlpack/methods/kmeans/elkan_kmeans_impl.hpp index 0de4eeb6a0..ffed2d87ca 100644 --- a/src/mlpack/methods/kmeans/elkan_kmeans_impl.hpp +++ b/src/mlpack/methods/kmeans/elkan_kmeans_impl.hpp @@ -25,7 +25,7 @@ ElkanKMeans::ElkanKMeans(const MatType& dataset, metric(metric), distanceCalculations(0) { - + // Nothing to do here. } // Run a single iteration of Elkan's algorithm for Lloyd iterations. diff --git a/src/mlpack/methods/kmeans/kmeans_impl.hpp b/src/mlpack/methods/kmeans/kmeans_impl.hpp index 61467fa105..92755eb7af 100644 --- a/src/mlpack/methods/kmeans/kmeans_impl.hpp +++ b/src/mlpack/methods/kmeans/kmeans_impl.hpp @@ -241,7 +241,6 @@ Cluster(const MatType& data, << cNorm << ".\n"; if (std::isnan(cNorm) || std::isinf(cNorm)) cNorm = 1e-4; // Keep iterating. - } while (cNorm > 1e-5 && iteration != maxIterations); // If we ended on an even iteration, then the centroids are in the diff --git a/src/mlpack/methods/kmeans/max_variance_new_cluster_impl.hpp b/src/mlpack/methods/kmeans/max_variance_new_cluster_impl.hpp index 3ecd74b404..4c23eba88d 100644 --- a/src/mlpack/methods/kmeans/max_variance_new_cluster_impl.hpp +++ b/src/mlpack/methods/kmeans/max_variance_new_cluster_impl.hpp @@ -65,8 +65,8 @@ size_t MaxVarianceNewCluster::EmptyCluster(const MatType& data, // Take that point and add it to the empty cluster. newCentroids.col(maxVarCluster) *= (double(clusterCounts[maxVarCluster]) / double(clusterCounts[maxVarCluster] - 1)); - newCentroids.col(maxVarCluster) -= (1.0 / (clusterCounts[maxVarCluster] - 1.0)) * - arma::vec(data.col(furthestPoint)); + newCentroids.col(maxVarCluster) -= (1.0 / (clusterCounts[maxVarCluster] - + 1.0)) * arma::vec(data.col(furthestPoint)); clusterCounts[maxVarCluster]--; clusterCounts[emptyCluster]++; newCentroids.col(emptyCluster) = arma::vec(data.col(furthestPoint)); @@ -87,7 +87,8 @@ size_t MaxVarianceNewCluster::EmptyCluster(const MatType& data, else { variances[maxVarCluster] = (1.0 / clusterCounts[maxVarCluster]) * - ((clusterCounts[maxVarCluster] + 1) * variances[maxVarCluster] - maxDistance); + ((clusterCounts[maxVarCluster] + 1) * variances[maxVarCluster] - + maxDistance); } // Output some debugging information. diff --git a/src/mlpack/methods/kmeans/random_partition.hpp b/src/mlpack/methods/kmeans/random_partition.hpp index 2d1b99564c..954f3bdfd1 100644 --- a/src/mlpack/methods/kmeans/random_partition.hpp +++ b/src/mlpack/methods/kmeans/random_partition.hpp @@ -55,7 +55,7 @@ class RandomPartition void Serialize(Archive& /* ar */, const unsigned int /* version */) { } }; -} -} +} // namespace kmeans +} // namespace mlpack #endif diff --git a/src/mlpack/methods/lars/lars.cpp b/src/mlpack/methods/lars/lars.cpp index 1d892bb8a1..721d7563d9 100644 --- a/src/mlpack/methods/lars/lars.cpp +++ b/src/mlpack/methods/lars/lars.cpp @@ -58,7 +58,8 @@ LARS::LARS(const arma::mat& data, lambda2(lambda2), tolerance(tolerance) { - Train(data, responses, transposeData); + arma::rowvec rowResponses = responses.t(); + Train(data, rowResponses, transposeData); } LARS::LARS(const arma::mat& data, @@ -76,6 +77,32 @@ LARS::LARS(const arma::mat& data, elasticNet((lambda1 != 0) && (lambda2 != 0)), lambda2(lambda2), tolerance(tolerance) +{ + arma::rowvec rowResponses = responses.t(); + Train(data, rowResponses, transposeData); +} + +LARS::LARS(const arma::mat& data, + const arma::rowvec& responses, + const bool transposeData, + const bool useCholesky, + const double lambda1, + const double lambda2, + const double tolerance) : + LARS(useCholesky, lambda1, lambda2, tolerance) +{ + Train(data, responses, transposeData); +} + +LARS::LARS(const arma::mat& data, + const arma::rowvec& responses, + const bool transposeData, + const bool useCholesky, + const arma::mat& gramMatrix, + const double lambda1, + const double lambda2, + const double tolerance) : + LARS(useCholesky, gramMatrix, lambda1, lambda2, tolerance) { Train(data, responses, transposeData); } @@ -84,6 +111,24 @@ void LARS::Train(const arma::mat& matX, const arma::vec& y, arma::vec& beta, const bool transposeData) +{ + arma::rowvec rowY = y.t(); + Train(matX, rowY, beta, transposeData); +} + +void LARS::Train(const arma::mat& data, + const arma::vec& responses, + const bool transposeData) +{ + arma::rowvec rowResponses = responses.t(); + arma::vec beta; + Train(data, rowResponses, beta, transposeData); +} + +void LARS::Train(const arma::mat& matX, + const arma::rowvec& y, + arma::vec& beta, + const bool transposeData) { Timer::Start("lars_regression"); @@ -104,7 +149,7 @@ void LARS::Train(const arma::mat& matX, dataTrans = trans(matX); // Compute X' * y. - arma::vec vecXTy = trans(dataRef) * y; + arma::vec vecXTy = trans(y * dataRef); // Set up active set variables. In the beginning, the active set has size 0 // (all dimensions are inactive). @@ -376,7 +421,7 @@ void LARS::Train(const arma::mat& matX, } void LARS::Train(const arma::mat& data, - const arma::vec& responses, + const arma::rowvec& responses, const bool transposeData) { arma::vec beta; @@ -386,12 +431,21 @@ void LARS::Train(const arma::mat& data, void LARS::Predict(const arma::mat& points, arma::vec& predictions, const bool rowMajor) const +{ + arma::rowvec rowPredictions; + Predict(points, rowPredictions, rowMajor); + predictions = rowPredictions.t(); +} + +void LARS::Predict(const arma::mat& points, + arma::rowvec& predictions, + const bool rowMajor) const { // We really only need to store beta internally... if (rowMajor) - predictions = points * betaPath.back(); + predictions = trans(points * betaPath.back()); else - predictions = (betaPath.back().t() * points).t(); + predictions = betaPath.back().t() * points; } // Private functions. diff --git a/src/mlpack/methods/lars/lars.hpp b/src/mlpack/methods/lars/lars.hpp index 1925e02797..baba45f4f6 100644 --- a/src/mlpack/methods/lars/lars.hpp +++ b/src/mlpack/methods/lars/lars.hpp @@ -122,6 +122,54 @@ class LARS const double lambda2 = 0.0, const double tolerance = 1e-16); + /** + * Set the parameters to LARS and run training. Both lambda1 and lambda2 + * are set by default to 0. + * + * @param data Input data. + * @param responses A vector of targets. + * @param transposeData Should be true if the input data is column-major and + * false otherwise. + * @param useCholesky Whether or not to use Cholesky decomposition when + * solving linear system (as opposed to using the full Gram matrix). + * @param lambda1 Regularization parameter for l1-norm penalty. + * @param lambda2 Regularization parameter for l2-norm penalty. + * @param tolerance Run until the maximum correlation of elements in (X^T y) + * is less than this. + */ + mlpack_deprecated LARS(const arma::mat& data, + const arma::vec& responses, + const bool transposeData = true, + const bool useCholesky = false, + const double lambda1 = 0.0, + const double lambda2 = 0.0, + const double tolerance = 1e-16); + + /** + * Set the parameters to LARS, pass in a precalculated Gram matrix, and run + * training. Both lambda1 and lambda2 are set by default to 0. + * + * @param data Input data. + * @param responses A vector of targets. + * @param transposeData Should be true if the input data is column-major and + * false otherwise. + * @param useCholesky Whether or not to use Cholesky decomposition when + * solving linear system (as opposed to using the full Gram matrix). + * @param gramMatrix Gram matrix. + * @param lambda1 Regularization parameter for l1-norm penalty. + * @param lambda2 Regularization parameter for l2-norm penalty. + * @param tolerance Run until the maximum correlation of elements in (X^T y) + * is less than this. + */ + mlpack_deprecated LARS(const arma::mat& data, + const arma::vec& responses, + const bool transposeData, + const bool useCholesky, + const arma::mat& gramMatrix, + const double lambda1 = 0.0, + const double lambda2 = 0.0, + const double tolerance = 1e-16); + /** * Set the parameters to LARS and run training. Both lambda1 and lambda2 * are set by default to 0. @@ -138,7 +186,7 @@ class LARS * is less than this. */ LARS(const arma::mat& data, - const arma::vec& responses, + const arma::rowvec& responses, const bool transposeData = true, const bool useCholesky = false, const double lambda1 = 0.0, @@ -162,7 +210,7 @@ class LARS * is less than this. */ LARS(const arma::mat& data, - const arma::vec& responses, + const arma::rowvec& responses, const bool transposeData, const bool useCholesky, const arma::mat& gramMatrix, @@ -170,6 +218,42 @@ class LARS const double lambda2 = 0.0, const double tolerance = 1e-16); + /** + * Run LARS. The input matrix (like all mlpack matrices) should be + * column-major -- each column is an observation and each row is a dimension. + * However, because LARS is more efficient on a row-major matrix, this method + * will (internally) transpose the matrix. If this transposition is not + * necessary (i.e., you want to pass in a row-major matrix), pass 'false' for + * the transposeData parameter. + * + * @param data Column-major input data (or row-major input data if rowMajor = + * true). + * @param responses A vector of targets. + * @param beta Vector to store the solution (the coefficients) in. + * @param transposeData Set to false if the data is row-major. + */ + mlpack_deprecated void Train(const arma::mat& data, + const arma::vec& responses, + arma::vec& beta, + const bool transposeData = true); + + /** + * Run LARS. The input matrix (like all mlpack matrices) should be + * column-major -- each column is an observation and each row is a dimension. + * However, because LARS is more efficient on a row-major matrix, this method + * will (internally) transpose the matrix. If this transposition is not + * necessary (i.e., you want to pass in a row-major matrix), pass 'false' for + * the transposeData parameter. + * + * @param data Input data. + * @param responses A vector of targets. + * @param transposeData Should be true if the input data is column-major and + * false otherwise. + */ + mlpack_deprecated void Train(const arma::mat& data, + const arma::vec& responses, + const bool transposeData = true); + /** * Run LARS. The input matrix (like all mlpack matrices) should be * column-major -- each column is an observation and each row is a dimension. @@ -185,7 +269,7 @@ class LARS * @param transposeData Set to false if the data is row-major. */ void Train(const arma::mat& data, - const arma::vec& responses, + const arma::rowvec& responses, arma::vec& beta, const bool transposeData = true); @@ -203,7 +287,7 @@ class LARS * false otherwise. */ void Train(const arma::mat& data, - const arma::vec& responses, + const arma::rowvec& responses, const bool transposeData = true); /** @@ -215,8 +299,21 @@ class LARS * @param points The data points to regress on. * @param predictions y, which will contained calculated values on completion. */ + mlpack_deprecated void Predict(const arma::mat& points, + arma::vec& predictions, + const bool rowMajor = false) const; + + /** + * Predict y_i for each data point in the given data matrix using the + * currently-trained LARS model. + * + * @param points The data points to regress on. + * @param predictions y, which will contained calculated values on completion. + * @param rowMajor Should be true if the data points matrix is row-major and + * false otherwise. + */ void Predict(const arma::mat& points, - arma::vec& predictions, + arma::rowvec& predictions, const bool rowMajor = false) const; //! Access the set of active dimensions. diff --git a/src/mlpack/methods/lars/lars_main.cpp b/src/mlpack/methods/lars/lars_main.cpp index e84f7fe808..ffda499dd3 100644 --- a/src/mlpack/methods/lars/lars_main.cpp +++ b/src/mlpack/methods/lars/lars_main.cpp @@ -153,9 +153,9 @@ void mlpackMain() mat matY = std::move(CLI::GetParam("responses")); // Make sure y is oriented the right way. - if (matY.n_rows == 1) + if (matY.n_cols == 1) matY = trans(matY); - if (matY.n_cols > 1) + if (matY.n_rows > 1) Log::Fatal << "Only one column or row allowed in responses file!" << endl; if (matY.n_elem != matX.n_rows) @@ -163,7 +163,8 @@ void mlpackMain() << endl; vec beta; - lars.Train(matX, matY.unsafe_col(0), beta, false /* do not transpose */); + arma::rowvec y = std::move(matY); + lars.Train(matX, y, beta, false /* do not transpose */); } else // We must have --input_model_file. { @@ -184,12 +185,12 @@ void mlpackMain() << "is not equal to the dimensionality of the model (" << lars.BetaPath().back().n_elem << ")!" << endl; - arma::vec predictions; + arma::rowvec predictions; lars.Predict(testPoints.t(), predictions, false); - // Save test predictions. One per line, so, don't transpose on save. + // Save test predictions (one per line). if (CLI::HasParam("output_predictions")) - CLI::GetParam("output_predictions") = std::move(predictions); + CLI::GetParam("output_predictions") = predictions.t(); } if (CLI::HasParam("output_model")) diff --git a/src/mlpack/methods/linear_regression/linear_regression.cpp b/src/mlpack/methods/linear_regression/linear_regression.cpp index cf38e43ca5..cfa6b8a352 100644 --- a/src/mlpack/methods/linear_regression/linear_regression.cpp +++ b/src/mlpack/methods/linear_regression/linear_regression.cpp @@ -21,10 +21,25 @@ LinearRegression::LinearRegression(const arma::mat& predictors, const double lambda, const bool intercept, const arma::vec& weights) : + LinearRegression(predictors, responses.t(), weights.t(), lambda, intercept) +{} + +LinearRegression::LinearRegression(const arma::mat& predictors, + const arma::rowvec& responses, + const double lambda, + const bool intercept) : + LinearRegression(predictors, responses, arma::rowvec(), lambda, intercept) +{} + +LinearRegression::LinearRegression(const arma::mat& predictors, + const arma::rowvec& responses, + const arma::rowvec& weights, + const double lambda, + const bool intercept) : lambda(lambda), intercept(intercept) { - Train(predictors, responses, intercept, weights); + Train(predictors, responses, weights, intercept); } LinearRegression::LinearRegression(const LinearRegression& linearRegression) : @@ -36,6 +51,21 @@ void LinearRegression::Train(const arma::mat& predictors, const arma::vec& responses, const bool intercept, const arma::vec& weights) +{ + Train(predictors, responses.t(), weights.t(), intercept); +} + +void LinearRegression::Train(const arma::mat& predictors, + const arma::rowvec& responses, + const bool intercept) +{ + Train(predictors, responses, arma::rowvec(), intercept); +} + +void LinearRegression::Train(const arma::mat& predictors, + const arma::rowvec& responses, + const arma::rowvec& weights, + const bool intercept) { this->intercept = intercept; @@ -51,14 +81,14 @@ void LinearRegression::Train(const arma::mat& predictors, const size_t nCols = predictors.n_cols; arma::mat p = predictors; - arma::vec r = responses; + arma::rowvec r = responses; // Here we add the row of ones to the predictors. // The intercept is not penalized. Add an "all ones" row to design and set // intercept = false to get a penalized intercept. if (intercept) { - p.insert_rows(0, arma::ones(1,nCols)); + p.insert_rows(0, arma::ones(1, nCols)); } if (weights.n_elem > 0) @@ -88,19 +118,25 @@ void LinearRegression::Train(const arma::mat& predictors, // B = Q^T * responses * R^-1 // If lambda > 0, then we must add a bunch of empty responses. if (lambda == 0.0) - { - arma::solve(parameters, R, arma::trans(Q) * r); - } + arma::solve(parameters, R, arma::trans(r * Q)); else { // Copy responses into larger vector. - r.insert_rows(nCols,p.n_cols - nCols); - arma::solve(parameters, R, arma::trans(Q) * r); + r.insert_cols(nCols, p.n_cols - nCols); + arma::solve(parameters, R, arma::trans(r * Q)); } } void LinearRegression::Predict(const arma::mat& points, arma::vec& predictions) const +{ + arma::rowvec rowPredictions; + Predict(points, rowPredictions); + predictions = arma::trans(rowPredictions); +} + +void LinearRegression::Predict(const arma::mat& points, + arma::rowvec& predictions) const { if (intercept) { @@ -109,23 +145,30 @@ void LinearRegression::Predict(const arma::mat& points, arma::vec& predictions) Log::Assert(points.n_rows == parameters.n_rows - 1); // Get the predictions, but this ignores the intercept value // (parameters[0]). - predictions = arma::trans(arma::trans(parameters.subvec(1, - parameters.n_elem - 1)) * points); + predictions = arma::trans(parameters.subvec(1, parameters.n_elem - 1)) + * points; // Now add the intercept. predictions += parameters(0); } else { - // We want to be sure we have the correct number of dimensions in the dataset. + // We want to be sure we have the correct number of dimensions in + // the dataset. Log::Assert(points.n_rows == parameters.n_rows); - predictions = arma::trans(arma::trans(parameters) * points); + predictions = arma::trans(parameters) * points; } - } //! Compute the L2 squared error on the given predictors and responses. double LinearRegression::ComputeError(const arma::mat& predictors, const arma::vec& responses) const +{ + arma::rowvec rowResponses = responses.t(); + return ComputeError(predictors, rowResponses); +} + +double LinearRegression::ComputeError(const arma::mat& predictors, + const arma::rowvec& responses) const { // Get the number of columns and rows of the dataset. const size_t nCols = predictors.n_cols; @@ -133,7 +176,7 @@ double LinearRegression::ComputeError(const arma::mat& predictors, // Calculate the differences between actual responses and predicted responses. // We must also add the intercept (parameters(0)) to the predictions. - arma::vec temp; + arma::rowvec temp; if (intercept) { // Ensure that we have the correct number of dimensions in the dataset. @@ -142,8 +185,8 @@ double LinearRegression::ComputeError(const arma::mat& predictors, Log::Fatal << "The test data must have the same number of columns as the " "training file." << std::endl; } - temp = responses - arma::trans( (arma::trans(parameters.subvec(1, - parameters.n_elem - 1)) * predictors) + parameters(0)); + temp = responses - (parameters(0) + + arma::trans(parameters.subvec(1, parameters.n_elem - 1)) * predictors); } else { @@ -153,7 +196,7 @@ double LinearRegression::ComputeError(const arma::mat& predictors, Log::Fatal << "The test data must have the same number of columns as the " "training file." << std::endl; } - temp = responses - arma::trans((arma::trans(parameters) * predictors)); + temp = responses - arma::trans(parameters) * predictors; } const double cost = arma::dot(temp, temp) / nCols; diff --git a/src/mlpack/methods/linear_regression/linear_regression.hpp b/src/mlpack/methods/linear_regression/linear_regression.hpp index 7fc9c0b421..99ec44b42b 100644 --- a/src/mlpack/methods/linear_regression/linear_regression.hpp +++ b/src/mlpack/methods/linear_regression/linear_regression.hpp @@ -35,11 +35,39 @@ class LinearRegression * @param intercept Whether or not to include an intercept term. * @param weights Observation weights (for boosting). */ + mlpack_deprecated LinearRegression(const arma::mat& predictors, + const arma::vec& responses, + const double lambda = 0, + const bool intercept = true, + const arma::vec& weights = arma::vec()); + + /** + * Creates the model. + * + * @param predictors X, matrix of data points. + * @param responses y, the measured data for each point in X. + * @param lambda Regularization constant for ridge regression. + * @param intercept Whether or not to include an intercept term. + */ LinearRegression(const arma::mat& predictors, - const arma::vec& responses, + const arma::rowvec& responses, const double lambda = 0, - const bool intercept = true, - const arma::vec& weights = arma::vec()); + const bool intercept = true); + + /** + * Creates the model with weighted learning. + * + * @param predictors X, matrix of data points. + * @param responses y, the measured data for each point in X. + * @param weights Observation weights (for boosting). + * @param lambda Regularization constant for ridge regression. + * @param intercept Whether or not to include an intercept term. + */ + LinearRegression(const arma::mat& predictors, + const arma::rowvec& responses, + const arma::rowvec& weights, + const double lambda = 0, + const bool intercept = true); /** * Copy constructor. @@ -67,10 +95,42 @@ class LinearRegression * @param intercept Whether or not to fit an intercept term. * @param weights Observation weights (for boosting). */ + mlpack_deprecated void Train(const arma::mat& predictors, + const arma::vec& responses, + const bool intercept = true, + const arma::vec& weights = arma::vec()); + + /** + * Train the LinearRegression model on the given data. Careful! This will + * completely ignore and overwrite the existing model. This particular + * implementation does not have an incremental training algorithm. To set the + * regularization parameter lambda, call Lambda() or set a different value in + * the constructor. + * + * @param predictors X, the matrix of data points to train the model on. + * @param responses y, the responses to the data points. + * @param intercept Whether or not to fit an intercept term. + */ void Train(const arma::mat& predictors, - const arma::vec& responses, - const bool intercept = true, - const arma::vec& weights = arma::vec()); + const arma::rowvec& responses, + const bool intercept = true); + + /** + * Train the LinearRegression model on the given data and weights. Careful! + * This will completely ignore and overwrite the existing model. This + * particular implementation does not have an incremental training algorithm. + * To set the regularization parameter lambda, call Lambda() or set a + * different value in the constructor. + * + * @param predictors X, the matrix of data points to train the model on. + * @param responses y, the responses to the data points. + * @param intercept Whether or not to fit an intercept term. + * @param weights Observation weights (for boosting). + */ + void Train(const arma::mat& predictors, + const arma::rowvec& responses, + const arma::rowvec& weights, + const bool intercept = true); /** * Calculate y_i for each data point in points. @@ -78,7 +138,16 @@ class LinearRegression * @param points the data points to calculate with. * @param predictions y, will contain calculated values on completion. */ - void Predict(const arma::mat& points, arma::vec& predictions) const; + mlpack_deprecated void Predict(const arma::mat& points, + arma::vec& predictions) const; + + /** + * Calculate y_i for each data point in points. + * + * @param points the data points to calculate with. + * @param predictions y, will contain calculated values on completion. + */ + void Predict(const arma::mat& points, arma::rowvec& predictions) const; /** * Calculate the L2 squared error on the given predictors and responses using @@ -97,8 +166,28 @@ class LinearRegression * @param points Matrix of predictors (X). * @param responses Vector of responses (y). */ + mlpack_deprecated double ComputeError(const arma::mat& points, + const arma::vec& responses) const; + + /** + * Calculate the L2 squared error on the given predictors and responses using + * this linear regression model. This calculation returns + * + * \f[ + * (1 / n) * \| y - X B \|^2_2 + * \f] + * + * where \f$ y \f$ is the responses vector, \f$ X \f$ is the matrix of + * predictors, and \f$ B \f$ is the parameters of the trained linear + * regression model. + * + * As this number decreases to 0, the linear regression fit is better. + * + * @param points Matrix of predictors (X). + * @param responses Transposed vector of responses (y^T). + */ double ComputeError(const arma::mat& points, - const arma::vec& responses) const; + const arma::rowvec& responses) const; //! Return the parameters (the b vector). const arma::vec& Parameters() const { return parameters; } @@ -141,7 +230,7 @@ class LinearRegression bool intercept; }; -} // namespace linear_regression +} // namespace regression } // namespace mlpack #endif // MLPACK_METHODS_LINEAR_REGRESSION_HPP diff --git a/src/mlpack/methods/linear_regression/linear_regression_main.cpp b/src/mlpack/methods/linear_regression/linear_regression_main.cpp index 054dc49872..c9e5edb45c 100644 --- a/src/mlpack/methods/linear_regression/linear_regression_main.cpp +++ b/src/mlpack/methods/linear_regression/linear_regression_main.cpp @@ -66,7 +66,7 @@ PROGRAM_INFO("Simple Linear Regression and Prediction", PARAM_MATRIX_IN("training", "Matrix containing training set X (regressors).", "t"); -PARAM_COL_IN("training_responses", "Optional vector containing y " +PARAM_ROW_IN("training_responses", "Optional vector containing y " "(responses). If not given, the responses are assumed to be the last row " "of the input file.", "r"); @@ -93,7 +93,7 @@ void mlpackMain() << "(-T) is not specified." << endl; mat regressors; - vec responses; + rowvec responses; LinearRegression lr; lr.Lambda() = lambda; @@ -164,10 +164,10 @@ void mlpackMain() { // The initial predictors for y, Nx1. Timer::Start("load_responses"); - responses = std::move(CLI::GetParam("training_responses")); + responses = CLI::GetParam("training_responses"); Timer::Stop("load_responses"); - if (responses.n_rows != regressors.n_cols) + if (responses.n_cols != regressors.n_cols) Log::Fatal << "The responses must have the same number of rows as the " "training file." << endl; } @@ -207,7 +207,7 @@ void mlpackMain() } // Perform the predictions using our model. - vec predictions; + rowvec predictions; Timer::Start("prediction"); lr.Predict(points, predictions); Timer::Stop("prediction"); diff --git a/src/mlpack/methods/local_coordinate_coding/lcc.cpp b/src/mlpack/methods/local_coordinate_coding/lcc.cpp index 8c420d8413..b066d4c7d5 100644 --- a/src/mlpack/methods/local_coordinate_coding/lcc.cpp +++ b/src/mlpack/methods/local_coordinate_coding/lcc.cpp @@ -57,7 +57,8 @@ void LocalCoordinateCoding::Encode(const arma::mat& data, arma::mat& codes) // Run LARS for this point, by making an alias of the point and passing // that. arma::vec beta = codes.unsafe_col(i); - lars.Train(dictPrime, data.unsafe_col(i), beta, false); + arma::rowvec responses = data.unsafe_col(i).t(); + lars.Train(dictPrime, responses, beta, false); beta %= invW; // Remember, beta is an alias of codes.col(i). } } diff --git a/src/mlpack/methods/logistic_regression/logistic_regression_function_impl.hpp b/src/mlpack/methods/logistic_regression/logistic_regression_function_impl.hpp index 1a8e023cca..53b1a953b8 100644 --- a/src/mlpack/methods/logistic_regression/logistic_regression_function_impl.hpp +++ b/src/mlpack/methods/logistic_regression/logistic_regression_function_impl.hpp @@ -48,7 +48,7 @@ LogisticRegressionFunction::LogisticRegressionFunction( responses(responses), lambda(lambda) { - //to check if initialPoint is compatible with predictors + // To check if initialPoint is compatible with predictors. if (initialPoint.n_rows != (predictors.n_rows + 1) || initialPoint.n_cols != 1) this->initialPoint = arma::zeros(predictors.n_rows + 1, 1); diff --git a/src/mlpack/methods/lsh/lsh_search.hpp b/src/mlpack/methods/lsh/lsh_search.hpp index 077ecb43d5..097a94a46f 100644 --- a/src/mlpack/methods/lsh/lsh_search.hpp +++ b/src/mlpack/methods/lsh/lsh_search.hpp @@ -472,7 +472,6 @@ class LSHSearch //! Use a priority queue to represent the list of candidate neighbors. typedef std::priority_queue, CandidateCmp> CandidateList; - }; // class LSHSearch } // namespace neighbor diff --git a/src/mlpack/methods/lsh/lsh_search_impl.hpp b/src/mlpack/methods/lsh/lsh_search_impl.hpp index 9fc56184ca..f7994b3688 100644 --- a/src/mlpack/methods/lsh/lsh_search_impl.hpp +++ b/src/mlpack/methods/lsh/lsh_search_impl.hpp @@ -298,7 +298,7 @@ void LSHSearch::Train(const arma::mat& referenceSet, // For a single table, let the 'numProj' projections be denoted by 'proj_i' // and the corresponding offset be 'offset_i'. Then the key of a single // point is obtained as: - // key = { floor( ( + offset_i) / 'hashWidth' ) forall i } + // key = { floor(( + offset_i) / 'hashWidth') forall i } arma::mat offsetMat = arma::repmat(offsets.unsafe_col(i), 1, referenceSet.n_cols); arma::mat hashMat = projections.slice(i).t() * (referenceSet); @@ -368,7 +368,6 @@ void LSHSearch::Train(const arma::mat& referenceSet, const size_t index = bucketRowInHashTable[hashInd]; if (bucketContentSize[index] < maxSize) secondHashTable[index](bucketContentSize[index]++) = j; - } // Loop over all points in the reference set. } // Loop over tables. @@ -552,7 +551,6 @@ void LSHSearch::GetAdditionalProbingBins( const size_t T, arma::mat& additionalProbingBins) const { - // No additional bins requested. Our work is done. if (T == 0) return; @@ -626,12 +624,11 @@ void LSHSearch::GetAdditionalProbingBins( // smallest and the second smallest, it's obvious that score(Ae) > // score(As). Therefore the second perturbation vector is ALWAYS the vector // containing only the second-lowest scoring perturbation. - double minscore2 = scores[0]; size_t minloc2 = 0; - for (size_t s = 0; s < (2 * numProj); ++s) // here we can't start from 1 + for (size_t s = 0; s < (2 * numProj); ++s) // Here we can't start from 1. { - if (minscore2 > scores[s] && s != minloc) //second smallest + if (minscore2 > scores[s] && s != minloc) // Second smallest. { minscore2 = scores[s]; minloc2 = s; @@ -644,14 +641,12 @@ void LSHSearch::GetAdditionalProbingBins( } // General case: more than 2 perturbation vectors require use of minheap. - // Sort everything in increasing order. arma::uvec sortidx = arma::sort_index(scores); scores = scores(sortidx); actions = actions(sortidx); positions = positions(sortidx); - // Theory: // A probing sequence is a sequence of T probing bins where a query's // neighbors are most likely to be. Likelihood is dependent only on a bin's @@ -683,7 +678,7 @@ void LSHSearch::GetAdditionalProbingBins( > minHeap; // our minheap // Start by adding the lowest scoring set to the minheap. - minHeap.push( std::make_pair(PerturbationScore(Ao, scores), 0) ); + minHeap.push(std::make_pair(PerturbationScore(Ao, scores), 0)); // Loop invariable: after pvec iterations, additionalProbingBins contains pvec // valid codes of the lowest-scoring bins (bins most likely to contain @@ -699,8 +694,9 @@ void LSHSearch::GetAdditionalProbingBins( // Shift operation on Ai (replace max with max+1). std::vector As = Ai; + + // Don't add invalid sets. if (PerturbationShift(As) && PerturbationValid(As)) - // Don't add invalid sets. { perturbationSets.push_back(As); // add shifted set to sets minHeap.push( @@ -710,22 +706,23 @@ void LSHSearch::GetAdditionalProbingBins( // Expand operation on Ai (add max+1 to set). std::vector Ae = Ai; + + // Don't add invalid sets. if (PerturbationExpand(Ae) && PerturbationValid(Ae)) - // Don't add invalid sets. { perturbationSets.push_back(Ae); // add expanded set to sets minHeap.push( std::make_pair(PerturbationScore(Ae, scores), perturbationSets.size() - 1)); } - - } while (!PerturbationValid(Ai));//Discard invalid perturbations + } while (!PerturbationValid(Ai)); // Discard invalid perturbations // Found valid perturbation set Ai. Construct perturbation vector from set. for (size_t pos = 0; pos < Ai.size(); ++pos) + { // If Ai[pos] is marked, add action to probing vector. - additionalProbingBins(positions(pos), pvec) - += Ai[pos] ? actions(pos) : 0; + additionalProbingBins(positions(pos), pvec) += Ai[pos] ? actions(pos) : 0; + } } } @@ -756,6 +753,7 @@ void LSHSearch::ReturnIndicesFromTable( arma::mat queryCodesNotFloored(numProj, numTablesToSearch); for (size_t i = 0; i < numTablesToSearch; i++) queryCodesNotFloored.unsafe_col(i) = projections.slice(i).t() * queryPoint; + queryCodesNotFloored += offsets.cols(0, numTablesToSearch - 1); allProjInTables = arma::floor(queryCodesNotFloored / hashWidth); @@ -788,13 +786,12 @@ void LSHSearch::ReturnIndicesFromTable( // the primary hash table). hashMat(arma::span(1, T), i) = // Compute code of rows 1:end of column i arma::conv_to< arma::Col >:: // floor by typecasting to size_t - from( secondHashWeights.t() * additionalProbingBins ); + from(secondHashWeights.t() * additionalProbingBins); for (size_t p = 1; p < T + 1; ++p) hashMat(p, i) = (hashMat(p, i) % secondHashSize); } } - // Count number of points hashed in the same bucket as the query. size_t maxNumPoints = 0; for (size_t i = 0; i < numTablesToSearch; ++i) @@ -836,9 +833,11 @@ void LSHSearch::ReturnIndicesFromTable( size_t tableRow = bucketRowInHashTable[hashInd]; if (tableRow < secondHashSize && bucketContentSize[tableRow] > 0) + { // Pick the indices in the bucket corresponding to hashInd. for (size_t j = 0; j < bucketContentSize[tableRow]; ++j) refPointsConsidered[ secondHashTable[tableRow](j) ]++; + } } } @@ -865,9 +864,11 @@ void LSHSearch::ReturnIndicesFromTable( const size_t tableRow = bucketRowInHashTable[hashInd]; if (tableRow < secondHashSize) - // Store all secondHashTable points in the candidates set. - for (size_t j = 0; j < bucketContentSize[tableRow]; ++j) - refPointsConsideredSmall(start++) = secondHashTable[tableRow](j); + { + // Store all secondHashTable points in the candidates set. + for (size_t j = 0; j < bucketContentSize[tableRow]; ++j) + refPointsConsideredSmall(start++) = secondHashTable[tableRow](j); + } } } @@ -1146,7 +1147,7 @@ void LSHSearch::Serialize(Archive& ar, // the value referenceSet->n_cols is seen. size_t len = 0; - for ( ; len < tmpSecondHashTable.n_rows; ++len) + for (; len < tmpSecondHashTable.n_rows; ++len) if (tmpSecondHashTable(len, i) == referenceSet->n_cols) break; diff --git a/src/mlpack/methods/matrix_completion/matrix_completion.cpp b/src/mlpack/methods/matrix_completion/matrix_completion.cpp index fa5c2c4abc..a3058094ab 100644 --- a/src/mlpack/methods/matrix_completion/matrix_completion.cpp +++ b/src/mlpack/methods/matrix_completion/matrix_completion.cpp @@ -54,20 +54,26 @@ MatrixCompletion::MatrixCompletion(const size_t m, void MatrixCompletion::CheckValues() { if (indices.n_rows != 2) - Log::Fatal << "MatrixCompletion::CheckValues(): matrix of constraint indices does " - << "not have 2 rows!" << std::endl; + { + Log::Fatal << "MatrixCompletion::CheckValues(): matrix of constraint " + << "indices does not have 2 rows!" << std::endl; + } if (indices.n_cols != values.n_elem) - Log::Fatal << "MatrixCompletion::CheckValues(): the number of constraint indices " - << "(columns of constraint indices matrix) does not match the number of " - << "constraint values (length of constraint value vector)!" << std::endl; + { + Log::Fatal << "MatrixCompletion::CheckValues(): the number of constraint " + << "indices (columns of constraint indices matrix) does not match the " + << "number of constraint values (length of constraint value vector)!" + << std::endl; + } for (size_t i = 0; i < values.n_elem; i++) { if (indices(0, i) >= m || indices(1, i) >= n) - Log::Fatal << "MatrixCompletion::CheckValues(): indices (" << indices(0, i) << ", " - << indices(1, i) << ") are out of bounds for matrix of size " << m << " x " - << "n!" << std::endl; + Log::Fatal << "MatrixCompletion::CheckValues(): indices (" + << indices(0, i) << ", " << indices(1, i) + << ") are out of bounds for matrix of size " << m << " x n!" + << std::endl; } } diff --git a/src/mlpack/methods/matrix_completion/matrix_completion.hpp b/src/mlpack/methods/matrix_completion/matrix_completion.hpp index a806f105c5..6b0646080e 100644 --- a/src/mlpack/methods/matrix_completion/matrix_completion.hpp +++ b/src/mlpack/methods/matrix_completion/matrix_completion.hpp @@ -112,7 +112,10 @@ class MatrixCompletion void Recover(arma::mat& recovered); //! Return the underlying SDP. - const optimization::LRSDP>& Sdp() const { return sdp; } + const optimization::LRSDP>& Sdp() const + { + return sdp; + } //! Modify the underlying SDP. optimization::LRSDP>& Sdp() { return sdp; } diff --git a/src/mlpack/methods/mvu/mvu.cpp b/src/mlpack/methods/mvu/mvu.cpp index 8c02d0ca6d..b120da16a4 100644 --- a/src/mlpack/methods/mvu/mvu.cpp +++ b/src/mlpack/methods/mvu/mvu.cpp @@ -13,7 +13,6 @@ */ #include "mvu.hpp" -//#include #include #include diff --git a/src/mlpack/methods/naive_bayes/nbc_main.cpp b/src/mlpack/methods/naive_bayes/nbc_main.cpp index 6efb86319f..334fa7625d 100644 --- a/src/mlpack/methods/naive_bayes/nbc_main.cpp +++ b/src/mlpack/methods/naive_bayes/nbc_main.cpp @@ -101,8 +101,8 @@ PARAM_FLAG("incremental_variance", "The variance of each class will be " PARAM_MATRIX_IN("test", "A matrix containing the test set.", "T"); PARAM_UROW_OUT("output", "The matrix in which the predicted labels for the" " test set will be written.", "o"); -PARAM_MATRIX_OUT("output_probs", "The matrix in which the predicted probability of labels for the" - " test set will be written.", "p"); +PARAM_MATRIX_OUT("output_probs", "The matrix in which the predicted probability" + " of labels for the test set will be written.", "p"); void mlpackMain() { diff --git a/src/mlpack/methods/neighbor_search/neighbor_search.hpp b/src/mlpack/methods/neighbor_search/neighbor_search.hpp index 3e0a9a01a0..c933e907d0 100644 --- a/src/mlpack/methods/neighbor_search/neighbor_search.hpp +++ b/src/mlpack/methods/neighbor_search/neighbor_search.hpp @@ -26,9 +26,9 @@ #include "neighbor_search_rules.hpp" namespace mlpack { -namespace neighbor /** Neighbor-search routines. These include - * all-nearest-neighbors and all-furthest-neighbors - * searches. */ { +// Neighbor-search routines. These include all-nearest-neighbors and +// all-furthest-neighbors searches. +namespace neighbor { // Forward declaration. template diff --git a/src/mlpack/methods/neighbor_search/neighbor_search_impl.hpp b/src/mlpack/methods/neighbor_search/neighbor_search_impl.hpp index ec814581b5..a238cd940d 100644 --- a/src/mlpack/methods/neighbor_search/neighbor_search_impl.hpp +++ b/src/mlpack/methods/neighbor_search/neighbor_search_impl.hpp @@ -584,7 +584,7 @@ DualTreeTraversalType, SingleTreeTraversalType>::Search( typedef NeighborSearchRules RuleType; - switch(searchMode) + switch (searchMode) { case NAIVE_MODE: { diff --git a/src/mlpack/methods/pca/pca_impl.hpp b/src/mlpack/methods/pca/pca_impl.hpp index 87467f5110..f9c62cb432 100644 --- a/src/mlpack/methods/pca/pca_impl.hpp +++ b/src/mlpack/methods/pca/pca_impl.hpp @@ -26,8 +26,8 @@ namespace mlpack { namespace pca { template -PCAType::PCAType(const bool scaleData, - const DecompositionPolicy& decomposition) : +PCAType::PCAType( + const bool scaleData, const DecompositionPolicy& decomposition) : scaleData(scaleData), decomposition(decomposition) { } diff --git a/src/mlpack/methods/pca/pca_main.cpp b/src/mlpack/methods/pca/pca_main.cpp index 7626c106c0..87fb48ccd1 100644 --- a/src/mlpack/methods/pca/pca_main.cpp +++ b/src/mlpack/methods/pca/pca_main.cpp @@ -96,7 +96,6 @@ void RunPCA(arma::mat& dataset, Log::Info << (varRetained * 100) << "% of variance retained (" << dataset.n_rows << " dimensions)." << endl; - } void mlpackMain() diff --git a/src/mlpack/methods/perceptron/perceptron.hpp b/src/mlpack/methods/perceptron/perceptron.hpp index 9d51fdbbb4..9fb531eb38 100644 --- a/src/mlpack/methods/perceptron/perceptron.hpp +++ b/src/mlpack/methods/perceptron/perceptron.hpp @@ -136,7 +136,7 @@ class Perceptron //! Modify the biases. You had better know what you are doing! arma::vec& Biases() { return biases; } -private: + private: //! The maximum number of iterations during training. size_t maxIterations; diff --git a/src/mlpack/methods/perceptron/perceptron_main.cpp b/src/mlpack/methods/perceptron/perceptron_main.cpp index 8a896e8468..ad00fda8f7 100644 --- a/src/mlpack/methods/perceptron/perceptron_main.cpp +++ b/src/mlpack/methods/perceptron/perceptron_main.cpp @@ -104,8 +104,8 @@ class PerceptronModel PARAM_MATRIX_IN("training", "A matrix containing the training set.", "t"); PARAM_UROW_IN("labels", "A matrix containing labels for the training set.", "l"); -PARAM_INT_IN("max_iterations","The maximum number of iterations the perceptron " - "is to be run", "n", 1000); +PARAM_INT_IN("max_iterations", "The maximum number of iterations the " + "perceptron is to be run", "n", 1000); // Model loading/saving. PARAM_MODEL_IN(PerceptronModel, "input_model", "Input perceptron model.", "m"); diff --git a/src/mlpack/methods/preprocess/preprocess_describe_main.cpp b/src/mlpack/methods/preprocess/preprocess_describe_main.cpp index 1ed04f75fa..b5e035b26c 100644 --- a/src/mlpack/methods/preprocess/preprocess_describe_main.cpp +++ b/src/mlpack/methods/preprocess/preprocess_describe_main.cpp @@ -149,7 +149,7 @@ double Kurtosis(const arma::rowvec& input, */ double StandardError(const size_t size, const double& fStd) { - return fStd / sqrt(size); + return fStd / sqrt(size); } void mlpackMain() diff --git a/src/mlpack/methods/preprocess/preprocess_imputer_main.cpp b/src/mlpack/methods/preprocess/preprocess_imputer_main.cpp index c603f6712d..d445c80e51 100644 --- a/src/mlpack/methods/preprocess/preprocess_imputer_main.cpp +++ b/src/mlpack/methods/preprocess/preprocess_imputer_main.cpp @@ -154,7 +154,8 @@ void mlpackMain() else if (strategy == "custom") { CustomImputation strat(customValue); - Imputer> imputer(info, strat); + Imputer> imputer( + info, strat); } else { diff --git a/src/mlpack/methods/radical/radical.cpp b/src/mlpack/methods/radical/radical.cpp index 93cf392275..a705e61235 100644 --- a/src/mlpack/methods/radical/radical.cpp +++ b/src/mlpack/methods/radical/radical.cpp @@ -126,7 +126,7 @@ void Radical::DoRadical(const mat& matXT, mat& matY, mat& matW) // In the RADICAL code, they do not copy and perturb initially, although the // paper does. We follow the code as it should match their reported results // and likely does a better job bouncing out of local optima. - //GeneratePerturbedX(X, X); + // GeneratePerturbedX(X, X); // Initialize the unmixing matrix to the whitening matrix. Timer::Start("radical_do_radical"); diff --git a/src/mlpack/methods/range_search/range_search_stat.hpp b/src/mlpack/methods/range_search/range_search_stat.hpp index 8df48d7d12..8ffaf241bc 100644 --- a/src/mlpack/methods/range_search/range_search_stat.hpp +++ b/src/mlpack/methods/range_search/range_search_stat.hpp @@ -56,7 +56,7 @@ class RangeSearchStat double lastDistance; }; -} // namespace neighbor +} // namespace range } // namespace mlpack #endif diff --git a/src/mlpack/methods/range_search/rs_model.cpp b/src/mlpack/methods/range_search/rs_model.cpp index 747c3f600e..529c292f4d 100644 --- a/src/mlpack/methods/range_search/rs_model.cpp +++ b/src/mlpack/methods/range_search/rs_model.cpp @@ -35,7 +35,7 @@ RSModel::RSModel(const RSModel& other) : randomBasis(other.randomBasis), rSearch(other.rSearch) { - + // Nothing to do. } // Move constructor. @@ -128,7 +128,7 @@ void RSModel::BuildModel(arma::mat&& referenceSet, break; case R_TREE: - rSearch = new RSType(naive,singleMode); + rSearch = new RSType(naive, singleMode); break; case R_STAR_TREE: diff --git a/src/mlpack/methods/range_search/rs_model_impl.hpp b/src/mlpack/methods/range_search/rs_model_impl.hpp index f244719f56..243acd6565 100644 --- a/src/mlpack/methods/range_search/rs_model_impl.hpp +++ b/src/mlpack/methods/range_search/rs_model_impl.hpp @@ -206,18 +206,18 @@ void SerializeVisitor::operator()(RSType* rs) const template bool& SingleModeVisitor::operator()(RSType* rs) const { - if (rs) - return rs->SingleMode(); - throw std::runtime_error("no range search model initialized"); + if (rs) + return rs->SingleMode(); + throw std::runtime_error("no range search model initialized"); } //! Exposes Naive() function of given RSType template bool& NaiveVisitor::operator()(RSType* rs) const { - if (rs) - return rs->Naive(); - throw std::runtime_error("no range search model initialized"); + if (rs) + return rs->Naive(); + throw std::runtime_error("no range search model initialized"); } // Serialize the model. diff --git a/src/mlpack/methods/rann/krann_main.cpp b/src/mlpack/methods/rann/krann_main.cpp index c2490538c6..fb8cda6f66 100644 --- a/src/mlpack/methods/rann/krann_main.cpp +++ b/src/mlpack/methods/rann/krann_main.cpp @@ -32,10 +32,10 @@ PROGRAM_INFO("K-Rank-Approximate-Nearest-Neighbors (kRANN)", "This program will calculate the k rank-approximate-nearest-neighbors of a " "set of points. You may specify a separate set of reference points and " "query points, or just a reference set which will be used as both the " - "reference and query set. You must specify the rank approximation (in \%) " + "reference and query set. You must specify the rank approximation (in %) " "(and optionally the success probability)." "\n\n" - "For example, the following will return 5 neighbors from the top 0.1\% of " + "For example, the following will return 5 neighbors from the top 0.1% of " "the data (with probability 0.95) for each point in 'input.csv' and store " "the distances in 'distances.csv' and the neighbors in the file " "'neighbors.csv':" diff --git a/src/mlpack/methods/rann/ra_search_rules_impl.hpp b/src/mlpack/methods/rann/ra_search_rules_impl.hpp index dc9e9b4555..2a617871be 100644 --- a/src/mlpack/methods/rann/ra_search_rules_impl.hpp +++ b/src/mlpack/methods/rann/ra_search_rules_impl.hpp @@ -84,7 +84,7 @@ RASearchRules(const arma::mat& referenceSet, for (size_t i = 0; i < querySet.n_cols; i++) candidates.push_back(pqueue); - if (naive)// No tree traversal; just do naive sampling here. + if (naive) // No tree traversal; just do naive sampling here. { // Sample enough points. arma::uvec distinctSamples; diff --git a/src/mlpack/methods/rann/ra_util.cpp b/src/mlpack/methods/rann/ra_util.cpp index a5dcd0b1b6..c613a21871 100644 --- a/src/mlpack/methods/rann/ra_util.cpp +++ b/src/mlpack/methods/rann/ra_util.cpp @@ -70,7 +70,6 @@ size_t mlpack::neighbor::RAUtil::MinimumSamplesReqd(const size_t n, } } m = (ub + lb) / 2; - } while (!done); return (std::min(m + 1, n)); @@ -89,7 +88,6 @@ double mlpack::neighbor::RAUtil::SuccessProbability(const size_t n, double eps = (double) t / (double) n; return 1.0 - std::pow(1.0 - eps, (double) m); - } // Faster implementation for topK = 1. else { @@ -153,7 +151,7 @@ double mlpack::neighbor::RAUtil::SuccessProbability(const size_t n, else jTrans = m - j; - for(size_t i = 2; i <= jTrans; i++) + for (size_t i = 2; i <= jTrans; i++) { mCj *= (double) (m - (i - 1)); mCj /= (double) i; diff --git a/src/mlpack/methods/regularized_svd/regularized_svd.hpp b/src/mlpack/methods/regularized_svd/regularized_svd.hpp index 233cac2c9a..ebb07f1f17 100644 --- a/src/mlpack/methods/regularized_svd/regularized_svd.hpp +++ b/src/mlpack/methods/regularized_svd/regularized_svd.hpp @@ -60,7 +60,6 @@ template< class RegularizedSVD { public: - /** * Constructor for Regularized SVD. Obtains the user and item matrices after * training on the passed data. The constructor initiates an object of class diff --git a/src/mlpack/methods/regularized_svd/regularized_svd_function.cpp b/src/mlpack/methods/regularized_svd/regularized_svd_function.cpp index f71daacc92..b360494360 100644 --- a/src/mlpack/methods/regularized_svd/regularized_svd_function.cpp +++ b/src/mlpack/methods/regularized_svd/regularized_svd_function.cpp @@ -42,7 +42,7 @@ double RegularizedSVDFunction::Evaluate(const arma::mat& parameters) const double cost = 0.0; - for(size_t i = 0; i < data.n_cols; i++) + for (size_t i = 0; i < data.n_cols; i++) { // Indices for accessing the the correct parameter columns. const size_t user = data(0, i); @@ -102,7 +102,7 @@ void RegularizedSVDFunction::Gradient(const arma::mat& parameters, gradient.zeros(rank, numUsers + numItems); - for(size_t i = 0; i < data.n_cols; i++) + for (size_t i = 0; i < data.n_cols; i++) { // Indices for accessing the the correct parameter columns. const size_t user = data(0, i); @@ -141,13 +141,13 @@ double StandardSGD::Optimize( double overallObjective = 0; // Calculate the first objective function. - for(size_t i = 0; i < numFunctions; i++) + for (size_t i = 0; i < numFunctions; i++) overallObjective += function.Evaluate(parameters, i); const arma::mat data = function.Dataset(); // Now iterate! - for(size_t i = 1; i != maxIterations; i++, currentFunction++) + for (size_t i = 1; i != maxIterations; i++, currentFunction++) { // Is this iteration the start of a sequence? if ((currentFunction % numFunctions) == 0) diff --git a/src/mlpack/methods/regularized_svd/regularized_svd_function.hpp b/src/mlpack/methods/regularized_svd/regularized_svd_function.hpp index c1ab55a074..11dc0e18db 100644 --- a/src/mlpack/methods/regularized_svd/regularized_svd_function.hpp +++ b/src/mlpack/methods/regularized_svd/regularized_svd_function.hpp @@ -22,7 +22,6 @@ namespace svd { class RegularizedSVDFunction { public: - /** * Constructor for RegularizedSVDFunction class. The constructor calculates * the number of users and items in the passed data. It also randomly diff --git a/src/mlpack/methods/reinforcement_learning/environment/cart_pole.hpp b/src/mlpack/methods/reinforcement_learning/environment/cart_pole.hpp index b7040aa4d1..62d168caa4 100644 --- a/src/mlpack/methods/reinforcement_learning/environment/cart_pole.hpp +++ b/src/mlpack/methods/reinforcement_learning/environment/cart_pole.hpp @@ -26,7 +26,6 @@ namespace rl { class CartPole { public: - /** * Implementation of the state of Cart Pole. Each state is a tuple vector * (position, velocity, angle, angular velocity). @@ -37,7 +36,7 @@ class CartPole /** * Construct a state instance. */ - State() : data(4) + State() : data(dimension) { /* Nothing to do here. */ } /** @@ -74,6 +73,9 @@ class CartPole //! Encode the state to a column vector. const arma::colvec& Encode() const { return data; } + //! Dimension of the encoded state. + static constexpr size_t dimension = 4; + private: //! Locally-stored (position, velocity, angle, angular velocity). arma::colvec data; @@ -226,4 +228,4 @@ class CartPole } // namespace rl } // namespace mlpack -#endif \ No newline at end of file +#endif diff --git a/src/mlpack/methods/reinforcement_learning/environment/mountain_car.hpp b/src/mlpack/methods/reinforcement_learning/environment/mountain_car.hpp index c5826009a7..9ad595fc7e 100644 --- a/src/mlpack/methods/reinforcement_learning/environment/mountain_car.hpp +++ b/src/mlpack/methods/reinforcement_learning/environment/mountain_car.hpp @@ -27,7 +27,6 @@ namespace rl { class MountainCar { public: - /** * Implementation of state of Mountain Car. Each state is a * (velocity, position) vector. @@ -38,7 +37,7 @@ class MountainCar /** * Construct a state instance. */ - State(): data(2, arma::fill::zeros) + State(): data(dimension, arma::fill::zeros) { /* Nothing to do here. */ } /** @@ -65,6 +64,9 @@ class MountainCar //! Encode the state to a column vector. const arma::colvec& Encode() const { return data; } + //! Dimension of the encoded state. + static constexpr size_t dimension = 2; + private: //! Locally-stored velocity and position vector. arma::colvec data; @@ -190,4 +192,4 @@ class MountainCar } // namespace rl } // namespace mlpack -#endif \ No newline at end of file +#endif diff --git a/src/mlpack/methods/reinforcement_learning/policy/CMakeLists.txt b/src/mlpack/methods/reinforcement_learning/policy/CMakeLists.txt new file mode 100644 index 0000000000..4ebd441cf3 --- /dev/null +++ b/src/mlpack/methods/reinforcement_learning/policy/CMakeLists.txt @@ -0,0 +1,14 @@ +# Define the files we need to compile +# Anything not in this list will not be compiled into mlpack. +set(SOURCES + greedy_policy.hpp +) + +# Add directory name to sources. +set(DIR_SRCS) +foreach(file ${SOURCES}) + set(DIR_SRCS ${DIR_SRCS} ${CMAKE_CURRENT_SOURCE_DIR}/${file}) +endforeach() +# Append sources (with directory name) to list of all mlpack sources (used at +# the parent scope). +set(MLPACK_SRCS ${MLPACK_SRCS} ${DIR_SRCS} PARENT_SCOPE) diff --git a/src/mlpack/methods/reinforcement_learning/policy/greedy_policy.hpp b/src/mlpack/methods/reinforcement_learning/policy/greedy_policy.hpp new file mode 100644 index 0000000000..036cd147ed --- /dev/null +++ b/src/mlpack/methods/reinforcement_learning/policy/greedy_policy.hpp @@ -0,0 +1,97 @@ +/** + * @file greedy_policy.hpp + * @author Shangtong Zhang + * + * This file is an implementation of epsilon greedy policy. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_METHODS_RL_POLICY_GREEDY_POLICY_HPP +#define MLPACK_METHODS_RL_POLICY_GREEDY_POLICY_HPP + +#include + +namespace mlpack { +namespace rl { + +/** + * Implementation for epsilon greedy policy. + * + * In general we will select an action greedily based on the action value, + * however sometimes we will also randomly select an action to encourage + * exploration. + * + * @tparam EnvironmentType The reinforcement learning task. + */ +template +class GreedyPolicy +{ + public: + using ActionType = typename EnvironmentType::Action; + + /** + * Constructor for epsilon greedy policy class. + * + * @param initialEpsilon The initial probability to explore (select a random action). + * @param annealInterval The steps during which the probability to explore will anneal. + * @param minEpsilon Epsilon will never be less than this value. + */ + GreedyPolicy(const double initialEpsilon, + const size_t annealInterval, + const double minEpsilon) : + epsilon(initialEpsilon), + minEpsilon(minEpsilon), + delta((initialEpsilon - minEpsilon) / annealInterval) + { /* Nothing to do here. */ } + + /** + * Sample an action based on given action values. + * + * @param actionValue Values for each action. + * @return Sampled action. + */ + ActionType Sample(const arma::colvec& actionValue) + { + double exploration = math::Random(); + + // Select the action randomly. + if (exploration < epsilon) + return static_cast(math::RandInt(ActionType::size)); + + // Select the action greedily. + return static_cast( + arma::as_scalar(arma::find(actionValue == actionValue.max(), 1))); + } + + /** + * Exploration probability will anneal at each step. + */ + void Anneal() + { + epsilon -= delta; + epsilon = std::max(minEpsilon, epsilon); + } + + /** + * @return Current possibility to explore. + */ + const double& Epsilon() const { return epsilon; } + + private: + //! Locally-stored probability to explore. + double epsilon; + + //! Locally-stored lower bound for epsilon. + double minEpsilon; + + //! Locally-stored stride for epsilon to anneal. + double delta; +}; + +} // namespace rl +} // namespace mlpack + +#endif diff --git a/src/mlpack/methods/reinforcement_learning/replay/CMakeLists.txt b/src/mlpack/methods/reinforcement_learning/replay/CMakeLists.txt new file mode 100644 index 0000000000..03ff3a5720 --- /dev/null +++ b/src/mlpack/methods/reinforcement_learning/replay/CMakeLists.txt @@ -0,0 +1,14 @@ +# Define the files we need to compile +# Anything not in this list will not be compiled into mlpack. +set(SOURCES + random_replay.hpp +) + +# Add directory name to sources. +set(DIR_SRCS) +foreach(file ${SOURCES}) + set(DIR_SRCS ${DIR_SRCS} ${CMAKE_CURRENT_SOURCE_DIR}/${file}) +endforeach() +# Append sources (with directory name) to list of all mlpack sources (used at +# the parent scope). +set(MLPACK_SRCS ${MLPACK_SRCS} ${DIR_SRCS} PARENT_SCOPE) diff --git a/src/mlpack/methods/reinforcement_learning/replay/random_replay.hpp b/src/mlpack/methods/reinforcement_learning/replay/random_replay.hpp new file mode 100644 index 0000000000..7eb73ce772 --- /dev/null +++ b/src/mlpack/methods/reinforcement_learning/replay/random_replay.hpp @@ -0,0 +1,166 @@ +/** + * @file random_replay.hpp + * @author Shangtong Zhang + * + * This file is an implementation of random experience replay. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_METHODS_RL_REPLAY_RANDOM_REPLAY_HPP +#define MLPACK_METHODS_RL_REPLAY_RANDOM_REPLAY_HPP + +#include + +namespace mlpack { +namespace rl { + +/** + * Implementation of random experience replay. + * + * At each time step, interactions between the agent and the + * environment will be saved to a memory buffer. When necessary, + * we can simply sample previous experiences from the buffer to + * train the agent. Typically this would be a random sample and + * the memory will be a First-In-First-Out buffer. + * + * For more information, see the following. + * + * @code + * @phdthesis{lin1993reinforcement, + * title = {Reinforcement learning for robots using neural networks}, + * author = {Lin, Long-Ji}, + * year = {1993}, + * school = {Fujitsu Laboratories Ltd} + * } + * @endcode + * + * @tparam EnvironmentType Desired task. + */ +template +class RandomReplay +{ + public: + using ActionType = typename EnvironmentType::Action; + using StateType = typename EnvironmentType::State; + + /** + * Construct an instance of random experience replay class. + * + * @param batchSize Number of examples returned at each sample. + * @param capacity Total memory size in terms of number of examples. + * @param dimension The dimension of an encoded state. + */ + RandomReplay(const size_t batchSize, + const size_t capacity, + const size_t dimension = StateType::dimension) : + batchSize(batchSize), + capacity(capacity), + position(0), + states(dimension, capacity), + actions(capacity), + rewards(capacity), + nextStates(dimension, capacity), + isTerminal(capacity), + full(false) + { /* Nothing to do here. */ } + + /** + * Store the given experience. + * + * @param state Given state. + * @param action Given action. + * @param reward Given reward. + * @param nextState Given next state. + * @param isEnd Whether next state is terminal state. + */ + void Store(const StateType& state, + ActionType action, + double reward, + const StateType& nextState, + bool isEnd) + { + states.col(position) = state.Encode(); + actions(position) = action; + rewards(position) = reward; + nextStates.col(position) = nextState.Encode(); + isTerminal(position) = isEnd; + position++; + if (position == capacity) + { + full = true; + position = 0; + } + } + + /** + * Sample some experiences. + * + * @param sampledStates Sampled encoded states. + * @param sampledActions Sampled actions. + * @param sampledRewards Sampled rewards. + * @param sampledNextStates Sampled encoded next states. + * @param isTerminal Indicate whether corresponding next state is terminal state. + */ + void Sample(arma::mat& sampledStates, + arma::icolvec& sampledActions, + arma::colvec& sampledRewards, + arma::mat& sampledNextStates, + arma::icolvec& isTerminal) + { + size_t upperBound = full ? capacity : position; + arma::uvec sampledIndices = arma::randi( + batchSize, arma::distr_param(0, upperBound - 1)); + + sampledStates = states.cols(sampledIndices); + sampledActions = actions.elem(sampledIndices); + sampledRewards = rewards.elem(sampledIndices); + sampledNextStates = nextStates.cols(sampledIndices); + isTerminal = this->isTerminal.elem(sampledIndices); + } + + /** + * Get the number of transitions in the memory. + * + * @return Actual used memory size + */ + const size_t& Size() + { + return full ? capacity : position; + } + + private: + //! Locally-stored number of examples of each sample. + size_t batchSize; + + //! Locally-stored total memory limit. + size_t capacity; + + //! Indicate the position to store new transition. + size_t position; + + //! Locally-stored encoded previous states. + arma::mat states; + + //! Locally-stored previous actions. + arma::icolvec actions; + + //! Locally-stored previous rewards. + arma::colvec rewards; + + //! Locally-stored encoded previous next states. + arma::mat nextStates; + + //! Locally-stored termination information of previous experience. + arma::icolvec isTerminal; + + //! Locally-stored indicator that whether the memory is full or not + bool full; +}; + +} // namespace rl +} // namespace mlpack + +#endif diff --git a/src/mlpack/methods/softmax_regression/softmax_regression.hpp b/src/mlpack/methods/softmax_regression/softmax_regression.hpp index 92365438d2..27f245ffaa 100644 --- a/src/mlpack/methods/softmax_regression/softmax_regression.hpp +++ b/src/mlpack/methods/softmax_regression/softmax_regression.hpp @@ -155,8 +155,9 @@ class SoftmaxRegression * @param labels Predicted labels for each point. * @param probabilities Class probabilities for each point. */ - void Classify(const arma::mat& dataset, arma::Row& labels, - arma::mat& probabilites) const; + void Classify(const arma::mat& dataset, + arma::Row& labels, + arma::mat& probabilites) const; /** * Classify the given points, returning class probabilities for each point. @@ -195,7 +196,8 @@ class SoftmaxRegression * @param numClasses Number of classes for classification. * @return Objective value of the final point. */ - double Train(const arma::mat& data, const arma::Row& labels, + double Train(const arma::mat& data, + const arma::Row& labels, const size_t numClasses); //! Sets the number of classes. diff --git a/src/mlpack/methods/softmax_regression/softmax_regression_function.cpp b/src/mlpack/methods/softmax_regression/softmax_regression_function.cpp index 27c18049d4..37d6eff23d 100644 --- a/src/mlpack/methods/softmax_regression/softmax_regression_function.cpp +++ b/src/mlpack/methods/softmax_regression/softmax_regression_function.cpp @@ -9,7 +9,7 @@ * 3-clause BSD license along with mlpack. If not, see * http://www.opensource.org/licenses/BSD-3-Clause for more information. */ - #include "softmax_regression_function.hpp" +#include "softmax_regression_function.hpp" using namespace mlpack; using namespace mlpack::regression; @@ -73,8 +73,8 @@ void SoftmaxRegressionFunction::InitializeWeights( * labels. The output is in the form of a matrix, which leads to simpler * calculations in the Evaluate() and Gradient() methods. */ -void SoftmaxRegressionFunction::GetGroundTruthMatrix(const arma::Row& labels, - arma::sp_mat& groundTruth) +void SoftmaxRegressionFunction::GetGroundTruthMatrix( + const arma::Row& labels, arma::sp_mat& groundTruth) { // Calculate the ground truth matrix according to the labels passed. The // ground truth matrix is a matrix of dimensions 'numClasses * numExamples', @@ -87,7 +87,7 @@ void SoftmaxRegressionFunction::GetGroundTruthMatrix(const arma::Row& la // Row pointers are the labels of the examples, and column pointers are the // number of cumulative entries made uptil that column. - for(size_t i = 0; i < labels.n_elem; i++) + for (size_t i = 0; i < labels.n_elem; i++) { rowPointers(i) = labels(i); colPointers(i+1) = i + 1; diff --git a/src/mlpack/methods/softmax_regression/softmax_regression_impl.hpp b/src/mlpack/methods/softmax_regression/softmax_regression_impl.hpp index 57c4f78b4e..9d5e3ee8fc 100644 --- a/src/mlpack/methods/softmax_regression/softmax_regression_impl.hpp +++ b/src/mlpack/methods/softmax_regression/softmax_regression_impl.hpp @@ -27,9 +27,8 @@ SoftmaxRegression(const size_t inputSize, lambda(0.0001), fitIntercept(fitIntercept) { - SoftmaxRegressionFunction::InitializeWeights(parameters, - inputSize, numClasses, - fitIntercept); + SoftmaxRegressionFunction::InitializeWeights( + parameters, inputSize, numClasses, fitIntercept); } template class OptimizerType> diff --git a/src/mlpack/methods/sparse_autoencoder/sparse_autoencoder_function.cpp b/src/mlpack/methods/sparse_autoencoder/sparse_autoencoder_function.cpp index 050de5efa4..3b719bd725 100644 --- a/src/mlpack/methods/sparse_autoencoder/sparse_autoencoder_function.cpp +++ b/src/mlpack/methods/sparse_autoencoder/sparse_autoencoder_function.cpp @@ -64,7 +64,7 @@ const arma::mat SparseAutoencoderFunction::InitializeWeights() // layers. The formula used is r = sqrt(6) / sqrt(vSize + hSize + 1). const double range = sqrt(6) / sqrt(visibleSize + hiddenSize + 1); - //Shift range of w1 and w2 values from [0, 1] to [-r, r]. + // Shift range of w1 and w2 values from [0, 1] to [-r, r]. parameters.submat(0, 0, 2 * hiddenSize - 1, visibleSize - 1) = 2 * range * (parameters.submat(0, 0, 2 * hiddenSize - 1, visibleSize - 1) - 0.5); diff --git a/src/mlpack/methods/sparse_coding/sparse_coding.cpp b/src/mlpack/methods/sparse_coding/sparse_coding.cpp index c7f1d67aa2..6edf7d7eb6 100644 --- a/src/mlpack/methods/sparse_coding/sparse_coding.cpp +++ b/src/mlpack/methods/sparse_coding/sparse_coding.cpp @@ -54,7 +54,8 @@ void SparseCoding::Encode(const arma::mat& data, arma::mat& codes) // place the result directly into that; then we will not need to have an // extra copy. arma::vec code = codes.unsafe_col(i); - lars.Train(dictionary, data.unsafe_col(i), code, false); + arma::rowvec responses = data.unsafe_col(i).t(); + lars.Train(dictionary, responses, code, false); } } @@ -119,17 +120,17 @@ double SparseCoding::OptimizeDictionary(const arma::mat& data, // numerically stable than just using inv(A) for everything. arma::vec dualVars = arma::zeros(nActiveAtoms); - //vec dualVars = 1e-14 * ones(nActiveAtoms); + // vec dualVars = 1e-14 * ones(nActiveAtoms); // Method used by feature sign code - fails miserably here. Perhaps the // MATLAB optimizer fmincon does something clever? - //vec dualVars = 10.0 * randu(nActiveAtoms, 1); + // vec dualVars = 10.0 * randu(nActiveAtoms, 1); - //vec dualVars = diagvec(solve(dictionary, data * trans(codes)) + // vec dualVars = diagvec(solve(dictionary, data * trans(codes)) // - codes * trans(codes)); - //for (size_t i = 0; i < dualVars.n_elem; i++) - // if (dualVars(i) < 0) - // dualVars(i) = 0; + // for (size_t i = 0; i < dualVars.n_elem; i++) + // if (dualVars(i) < 0) + // dualVars(i) = 0; bool converged = false; @@ -162,7 +163,6 @@ double SparseCoding::OptimizeDictionary(const arma::mat& data, arma::mat hessian = -(-2 * (matAInvZXT * trans(matAInvZXT)) % inv(A)); arma::vec searchDirection = -solve(hessian, gradient); - //printf("%e\n", norm(searchDirection, 2)); // Armijo line search. const double c = 1e-4; diff --git a/src/mlpack/tests/CMakeLists.txt b/src/mlpack/tests/CMakeLists.txt index 0d39ed84d4..85fec083bf 100644 --- a/src/mlpack/tests/CMakeLists.txt +++ b/src/mlpack/tests/CMakeLists.txt @@ -19,6 +19,7 @@ add_executable(mlpack_test convolution_test.cpp convolutional_network_test.cpp cosine_tree_test.cpp + cv_test.cpp dbscan_test.cpp decision_stump_test.cpp decision_tree_test.cpp @@ -78,11 +79,13 @@ add_executable(mlpack_test recurrent_network_test.cpp rectangle_tree_test.cpp regularized_svd_test.cpp - rl_environment_test.cpp + rl_components_test.cpp rmsprop_test.cpp sa_test.cpp sdp_primal_dual_test.cpp sgd_test.cpp + sgdr_test.cpp + snapshot_ensembles.cpp serialization.hpp serialization.cpp serialization_test.cpp diff --git a/src/mlpack/tests/ada_grad_test.cpp b/src/mlpack/tests/ada_grad_test.cpp index 86d5737b84..aed0659a80 100644 --- a/src/mlpack/tests/ada_grad_test.cpp +++ b/src/mlpack/tests/ada_grad_test.cpp @@ -93,7 +93,8 @@ BOOST_AUTO_TEST_CASE(AdaGradLogisticRegressionTest) LogisticRegression<> lr(shuffledData.n_rows, 0.5); LogisticRegressionFunction<> lrf(shuffledData, shuffledResponses, 0.5); - AdaGrad > adagrad(lrf, 0.99, 1e-8, 5000000, 1e-9, true); + AdaGrad > adagrad( + lrf, 0.99, 1e-8, 5000000, 1e-9, true); lr.Train(adagrad); // Ensure that the error is close to zero. diff --git a/src/mlpack/tests/adaboost_test.cpp b/src/mlpack/tests/adaboost_test.cpp index 542eb596bc..ebeda66a26 100644 --- a/src/mlpack/tests/adaboost_test.cpp +++ b/src/mlpack/tests/adaboost_test.cpp @@ -78,7 +78,7 @@ BOOST_AUTO_TEST_CASE(WeakLearnerErrorIris) arma::Mat labels; - if (!data::Load("iris_labels.txt",labels)) + if (!data::Load("iris_labels.txt", labels)) BOOST_FAIL("Cannot load labels for iris iris_labels.txt"); // Define your own weak learner, perceptron in this case. @@ -125,7 +125,7 @@ BOOST_AUTO_TEST_CASE(HammingLossBoundVertebralColumn) BOOST_FAIL("Cannot load test dataset vc2.csv!"); arma::Mat labels; - if (!data::Load("vc2_labels.txt",labels)) + if (!data::Load("vc2_labels.txt", labels)) BOOST_FAIL("Cannot load labels for vc2_labels.txt"); // Define your own weak learner, perceptron in this case. @@ -163,7 +163,7 @@ BOOST_AUTO_TEST_CASE(WeakLearnerErrorVertebralColumn) BOOST_FAIL("Cannot load test dataset vc2.csv!"); arma::Mat labels; - if (!data::Load("vc2_labels.txt",labels)) + if (!data::Load("vc2_labels.txt", labels)) BOOST_FAIL("Cannot load labels for vc2_labels.txt"); // Define your own weak learner, perceptron in this case. @@ -210,7 +210,7 @@ BOOST_AUTO_TEST_CASE(HammingLossBoundNonLinearSepData) BOOST_FAIL("Cannot load test dataset train_nonlinsep.txt!"); arma::Mat labels; - if (!data::Load("train_labels_nonlinsep.txt",labels)) + if (!data::Load("train_labels_nonlinsep.txt", labels)) BOOST_FAIL("Cannot load labels for train_labels_nonlinsep.txt"); // Define your own weak learner, perceptron in this case. @@ -248,7 +248,7 @@ BOOST_AUTO_TEST_CASE(WeakLearnerErrorNonLinearSepData) BOOST_FAIL("Cannot load test dataset train_nonlinsep.txt!"); arma::Mat labels; - if (!data::Load("train_labels_nonlinsep.txt",labels)) + if (!data::Load("train_labels_nonlinsep.txt", labels)) BOOST_FAIL("Cannot load labels for train_labels_nonlinsep.txt"); // Define your own weak learner, perceptron in this case. @@ -295,7 +295,7 @@ BOOST_AUTO_TEST_CASE(HammingLossIris_DS) BOOST_FAIL("Cannot load test dataset iris.csv!"); arma::Mat labels; - if (!data::Load("iris_labels.txt",labels)) + if (!data::Load("iris_labels.txt", labels)) BOOST_FAIL("Cannot load labels for iris_labels.txt"); // Define your own weak learner, decision stumps in this case. @@ -386,7 +386,7 @@ BOOST_AUTO_TEST_CASE(HammingLossBoundVertebralColumn_DS) BOOST_FAIL("Cannot load test dataset vc2.csv!"); arma::Mat labels; - if (!data::Load("vc2_labels.txt",labels)) + if (!data::Load("vc2_labels.txt", labels)) BOOST_FAIL("Cannot load labels for vc2_labels.txt"); // Define your own weak learner, decision stumps in this case. @@ -475,7 +475,7 @@ BOOST_AUTO_TEST_CASE(HammingLossBoundNonLinearSepData_DS) BOOST_FAIL("Cannot load test dataset train_nonlinsep.txt!"); arma::Mat labels; - if (!data::Load("train_labels_nonlinsep.txt",labels)) + if (!data::Load("train_labels_nonlinsep.txt", labels)) BOOST_FAIL("Cannot load labels for train_labels_nonlinsep.txt"); // Define your own weak learner, decision stumps in this case. @@ -516,7 +516,7 @@ BOOST_AUTO_TEST_CASE(WeakLearnerErrorNonLinearSepData_DS) BOOST_FAIL("Cannot load test dataset train_nonlinsep.txt!"); arma::Mat labels; - if (!data::Load("train_labels_nonlinsep.txt",labels)) + if (!data::Load("train_labels_nonlinsep.txt", labels)) BOOST_FAIL("Cannot load labels for train_labels_nonlinsep.txt"); // Define your own weak learner, decision stumps in this case. @@ -565,7 +565,7 @@ BOOST_AUTO_TEST_CASE(ClassifyTest_VERTEBRALCOL) BOOST_FAIL("Cannot load test dataset vc2.csv!"); arma::Mat labels; - if (!data::Load("vc2_labels.txt",labels)) + if (!data::Load("vc2_labels.txt", labels)) BOOST_FAIL("Cannot load labels for vc2_labels.txt"); // Define your own weak learner, perceptron in this case. @@ -579,7 +579,7 @@ BOOST_AUTO_TEST_CASE(ClassifyTest_VERTEBRALCOL) arma::Mat trueTestLabels; - if (!data::Load("vc2_test_labels.txt",trueTestLabels)) + if (!data::Load("vc2_test_labels.txt", trueTestLabels)) BOOST_FAIL("Cannot load labels for vc2_test_labels.txt"); Row perceptronPrediction(labels.n_cols); diff --git a/src/mlpack/tests/ann_layer_test.cpp b/src/mlpack/tests/ann_layer_test.cpp index 3597c04636..08d1cc4537 100644 --- a/src/mlpack/tests/ann_layer_test.cpp +++ b/src/mlpack/tests/ann_layer_test.cpp @@ -32,7 +32,7 @@ void ResetFunction( T& layer, typename std::enable_if::value>::type* = 0) { - layer.Reset(); + layer.Reset(); } template @@ -134,7 +134,7 @@ double JacobianPerformanceTest(ModuleType& module, inputTemp(i) = inputTemp(i) - (2 * eps); double outputB = module.Forward(std::move(input), std::move(target)); - centralDifferenceTemp(i) = (outputA - outputB) / ( 2 * eps); + centralDifferenceTemp(i) = (outputA - outputB) / (2 * eps); inputTemp(i) = inputTemp(i) + eps; } @@ -923,4 +923,4 @@ BOOST_AUTO_TEST_CASE(SimpleLogSoftmaxLayerTest) BOOST_REQUIRE_EQUAL(arma::accu(delta), 0); } -BOOST_AUTO_TEST_SUITE_END(); \ No newline at end of file +BOOST_AUTO_TEST_SUITE_END(); diff --git a/src/mlpack/tests/cli_test.cpp b/src/mlpack/tests/cli_test.cpp index 415f39fc92..a3e6fc8777 100644 --- a/src/mlpack/tests/cli_test.cpp +++ b/src/mlpack/tests/cli_test.cpp @@ -164,7 +164,6 @@ BOOST_AUTO_TEST_CASE(TestBooleanOption) BOOST_REQUIRE_EQUAL(CLI::GetParam("flag_test"), true); BOOST_REQUIRE_EQUAL(CLI::HasParam("flag_test"), true); - } /** @@ -245,12 +244,12 @@ BOOST_AUTO_TEST_CASE(InputColVectorParamTest) int argc = 3; - // The const-cast is a little hacky but should be fine... + // The const-cast is a little hacky but should be fine... Log::Fatal.ignoreInput = true; ParseCommandLine(argc, const_cast(argv)); Log::Fatal.ignoreInput = false; - // The --vector parameter should exist. + // The --vector parameter should exist. BOOST_REQUIRE(CLI::HasParam("vector")); // The --vector_file parameter should not exist (it should be transparent from // inside the program). @@ -274,7 +273,7 @@ BOOST_AUTO_TEST_CASE(InputUnsignedColVectorParamTest) PARAM_UCOL_IN("vector", "Test vector", "l"); - //fake aruguments + // Fake arguments. const char* argv[3]; argv[0] = "./test"; argv[1] = "-l"; @@ -282,12 +281,12 @@ BOOST_AUTO_TEST_CASE(InputUnsignedColVectorParamTest) int argc = 3; - // The const-cast is a little hacky but should be fine... + // The const-cast is a little hacky but should be fine... Log::Fatal.ignoreInput = true; ParseCommandLine(argc, const_cast(argv)); Log::Fatal.ignoreInput = false; - // The --vector parameter should exist. + // The --vector parameter should exist. BOOST_REQUIRE(CLI::HasParam("vector")); // The --vector_file parameter should not exist (it should be transparent from // inside the program). @@ -311,7 +310,7 @@ BOOST_AUTO_TEST_CASE(InputRowVectorParamTest) PARAM_ROW_IN("row", "Test vector", "l"); - //fake aruguments + // Fake arguments. const char* argv[3]; argv[0] = "./test"; argv[1] = "-l"; @@ -319,12 +318,12 @@ BOOST_AUTO_TEST_CASE(InputRowVectorParamTest) int argc = 3; - // The const-cast is a little hacky but should be fine... + // The const-cast is a little hacky but should be fine... Log::Fatal.ignoreInput = true; ParseCommandLine(argc, const_cast(argv)); Log::Fatal.ignoreInput = false; - // The --vector parameter should exist. + // The --vector parameter should exist. BOOST_REQUIRE(CLI::HasParam("row")); // The --vector_file parameter should not exist (it should be transparent from // inside the program). @@ -348,7 +347,7 @@ BOOST_AUTO_TEST_CASE(InputUnsignedRowVectorParamTest) PARAM_UROW_IN("row", "Test vector", "l"); - //fake aruguments + // Fake arguments. const char* argv[3]; argv[0] = "./test"; argv[1] = "-l"; @@ -356,12 +355,12 @@ BOOST_AUTO_TEST_CASE(InputUnsignedRowVectorParamTest) int argc = 3; - // The const-cast is a little hacky but should be fine... + // The const-cast is a little hacky but should be fine... Log::Fatal.ignoreInput = true; ParseCommandLine(argc, const_cast(argv)); Log::Fatal.ignoreInput = false; - // The --vector parameter should exist. + // The --vector parameter should exist. BOOST_REQUIRE(CLI::HasParam("row")); // The --vector_file parameter should not exist (it should be transparent from // inside the program). @@ -951,7 +950,7 @@ BOOST_AUTO_TEST_CASE(MatrixAndDatasetInfoTest) f << endl; f << "@attribute three STRING" << endl; f << endl; - f << "\% a comment line " << endl; + f << "%% a comment line " << endl; f << endl; f << "@data" << endl; f << "hello, 1, moo" << endl; @@ -1047,7 +1046,7 @@ BOOST_AUTO_TEST_CASE(RawDatasetInfoLoadParameter) f << endl; f << "@attribute three STRING" << endl; f << endl; - f << "\% a comment line " << endl; + f << "%% a comment line " << endl; f << endl; f << "@data" << endl; f << "hello, 1, moo" << endl; diff --git a/src/mlpack/tests/cosine_tree_test.cpp b/src/mlpack/tests/cosine_tree_test.cpp index f09ac273cb..56fe3ba7f1 100644 --- a/src/mlpack/tests/cosine_tree_test.cpp +++ b/src/mlpack/tests/cosine_tree_test.cpp @@ -181,7 +181,7 @@ BOOST_AUTO_TEST_CASE(CosineTreeModifiedGramSchmidt) CosineNodeQueue basisQueue; CosineTree dummyTree(data, epsilon, delta); - for(size_t i = 0; i < numCols; i++) + for (size_t i = 0; i < numCols; i++) { // Make a new CosineNode object. CosineTree* basisNode; @@ -198,7 +198,7 @@ BOOST_AUTO_TEST_CASE(CosineTreeModifiedGramSchmidt) CosineNodeQueue::const_iterator j = basisQueue.begin(); CosineTree* currentNode; - for(; j != basisQueue.end(); j++) + for (; j != basisQueue.end(); j++) { currentNode = *j; BOOST_REQUIRE_SMALL(arma::dot(currentNode->BasisVector(), newBasisVector), @@ -212,7 +212,7 @@ BOOST_AUTO_TEST_CASE(CosineTreeModifiedGramSchmidt) } // Deallocate memory given to the objects. - for(size_t i = 0; i < numCols; i++) + for (size_t i = 0; i < numCols; i++) { CosineTree* currentNode; currentNode = basisQueue.top(); diff --git a/src/mlpack/tests/cv_test.cpp b/src/mlpack/tests/cv_test.cpp new file mode 100644 index 0000000000..620d992d8d --- /dev/null +++ b/src/mlpack/tests/cv_test.cpp @@ -0,0 +1,93 @@ +/** + * @file cv_test.cpp + * + * Unit tests for the cross-validation module. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +using namespace mlpack::ann; +using namespace mlpack::cv; +using namespace mlpack::optimization; +using namespace mlpack::regression; + +BOOST_AUTO_TEST_SUITE(CVTest); + +/* + * Test the accuracy metric. + */ +BOOST_AUTO_TEST_CASE(AccuracyTest) +{ + // Making linearly separable data. + arma::mat data = + arma::mat("1 0; 2 0; 3 0; 4 0; 5 0; 1 1; 2 1; 3 1; 4 1; 5 1").t(); + arma::Row trainingLabels("0 0 0 0 0 1 1 1 1 1"); + + LogisticRegression<> lr(data, trainingLabels); + + arma::Row labels("0 0 1 0 0 1 0 1 0 1"); // 70%-correct labels + + BOOST_REQUIRE_CLOSE(Accuracy::Evaluate(lr, data, labels), 0.7, 1e-5); +} + +/* + * Test the mean squared error. + */ +BOOST_AUTO_TEST_CASE(MSETest) +{ + // Making two points that define the linear function f(x) = x - 1 + arma::mat trainingData("0 1"); + arma::rowvec trainingResponses("-1 0"); + + LinearRegression lr(trainingData, trainingResponses); + + // Making three responses that differ from the correct ones by 0, 1, and 2 + // respectively + arma::mat data("2 3 4"); + arma::rowvec responses("1 3 5"); + + double expectedMSE = (0 * 0 + 1 * 1 + 2 * 2) / 3.0; + + BOOST_REQUIRE_CLOSE(MSE::Evaluate(lr, data, responses), expectedMSE, 1e-5); +} + +/* + * Test the mean squared error with matrix responses. + */ +BOOST_AUTO_TEST_CASE(MSEMatResponsesTest) +{ + arma::mat data("1 2"); + arma::mat trainingResponses("1 2; 3 4"); + + FFN, ZeroInitialization> ffn; + ffn.Add>(1, 2); + ffn.Add>(); + + RMSProp opt(ffn, 0.2); + opt.Shuffle() = false; + ffn.Train(data, trainingResponses, opt); + + // Making four responses that differ from the correct ones by 0, 1, 2 and 3 + // respectively + arma::mat responses("1 3; 5 7"); + + double expectedMSE = (0 * 0 + 1 * 1 + 2 * 2 + 3 * 3) / 4.0; + + BOOST_REQUIRE_CLOSE(MSE::Evaluate(ffn, data, responses), expectedMSE, 1e-1); +} + +BOOST_AUTO_TEST_SUITE_END(); diff --git a/src/mlpack/tests/decision_stump_test.cpp b/src/mlpack/tests/decision_stump_test.cpp index af1e5f7cd6..d888e93f7e 100644 --- a/src/mlpack/tests/decision_stump_test.cpp +++ b/src/mlpack/tests/decision_stump_test.cpp @@ -49,9 +49,8 @@ BOOST_AUTO_TEST_CASE(OneClass) Row predictedLabels; ds.Classify(testingData, predictedLabels); - for (size_t i = 0; i < predictedLabels.size(); i++ ) + for (size_t i = 0; i < predictedLabels.size(); i++) BOOST_CHECK_EQUAL(predictedLabels(i), 1); - } /** diff --git a/src/mlpack/tests/decision_tree_test.cpp b/src/mlpack/tests/decision_tree_test.cpp index 8139eac0d5..d9b232d05e 100644 --- a/src/mlpack/tests/decision_tree_test.cpp +++ b/src/mlpack/tests/decision_tree_test.cpp @@ -22,6 +22,105 @@ using namespace mlpack; using namespace mlpack::tree; using namespace mlpack::distribution; +/** + * Create a mock categorical dataset for testing. + */ +void MockCategoricalData(arma::mat& d, + arma::Row& l, + data::DatasetInfo& datasetInfo) +{ + // We'll build a spiral dataset plus two noisy categorical features. We need + // to build the distributions for the categorical features (they'll be + // discrete distributions). + DiscreteDistribution c1[5]; + // The distribution will be automatically normalized. + for (size_t i = 0; i < 5; ++i) + { + std::vector probs; + probs.push_back(arma::vec(4, arma::fill::randu)); + c1[i] = DiscreteDistribution(probs); + } + + DiscreteDistribution c2[5]; + for (size_t i = 0; i < 5; ++i) + { + std::vector probs; + probs.push_back(arma::vec(2, arma::fill::randu)); + c2[i] = DiscreteDistribution(probs); + } + + arma::mat spiralDataset(4, 4000); + arma::Row labels(4000); + for (size_t i = 0; i < 4000; ++i) + { + // One circle every 2000 samples. Plus some noise. + const double magnitude = 2.0 + (double(i) / 200.0) + + 0.5 * mlpack::math::Random(); + const double angle = (i % 200) * (2 * M_PI) + mlpack::math::Random(); + + const double x = magnitude * cos(angle); + const double y = magnitude * sin(angle); + + spiralDataset(0, i) = x; + spiralDataset(1, i) = y; + + // Set categorical features c1 and c2. + if (i < 800) + { + spiralDataset(2, i) = c1[1].Random()[0]; + spiralDataset(3, i) = c2[1].Random()[0]; + labels[i] = 1; + } + else if (i < 1600) + { + spiralDataset(2, i) = c1[3].Random()[0]; + spiralDataset(3, i) = c2[3].Random()[0]; + labels[i] = 3; + } + else if (i < 2400) + { + spiralDataset(2, i) = c1[2].Random()[0]; + spiralDataset(3, i) = c2[2].Random()[0]; + labels[i] = 2; + } + else if (i < 3200) + { + spiralDataset(2, i) = c1[0].Random()[0]; + spiralDataset(3, i) = c2[0].Random()[0]; + labels[i] = 0; + } + else + { + spiralDataset(2, i) = c1[4].Random()[0]; + spiralDataset(3, i) = c2[4].Random()[0]; + labels[i] = 4; + } + } + + // Now create the dataset info. + datasetInfo = data::DatasetInfo(4); + datasetInfo.Type(2) = data::Datatype::categorical; + datasetInfo.Type(3) = data::Datatype::categorical; + // Set mappings. + datasetInfo.MapString("0", 2); + datasetInfo.MapString("1", 2); + datasetInfo.MapString("2", 2); + datasetInfo.MapString("3", 2); + datasetInfo.MapString("0", 3); + datasetInfo.MapString("1", 3); + + // Now shuffle the dataset. + arma::uvec indices = arma::shuffle(arma::linspace(0, 3999, + 4000)); + d = arma::mat(4, 4000); + l = arma::Row(4000); + for (size_t i = 0; i < 4000; ++i) + { + d.col(i) = spiralDataset.col(indices[i]); + l[i] = labels[indices[i]]; + } +} + BOOST_AUTO_TEST_SUITE(DecisionTreeTest); /** @@ -29,12 +128,13 @@ BOOST_AUTO_TEST_SUITE(DecisionTreeTest); */ BOOST_AUTO_TEST_CASE(GiniGainPerfectTest) { + arma::rowvec weights(10, arma::fill::ones); arma::Row labels; labels.zeros(10); // Test that it's perfect regardless of number of classes. for (size_t c = 1; c < 10; ++c) - BOOST_REQUIRE_SMALL(GiniGain::Evaluate(labels, c), 1e-5); + BOOST_REQUIRE_SMALL(GiniGain::Evaluate(labels, c, weights), 1e-5); } /** @@ -43,6 +143,7 @@ BOOST_AUTO_TEST_CASE(GiniGainPerfectTest) */ BOOST_AUTO_TEST_CASE(GiniGainEvenSplitTest) { + arma::rowvec weights = arma::ones(10); arma::Row labels(10); for (size_t i = 0; i < 5; ++i) labels[i] = 0; @@ -51,7 +152,15 @@ BOOST_AUTO_TEST_CASE(GiniGainEvenSplitTest) // Test that it's -0.5 regardless of the number of classes. for (size_t c = 2; c < 10; ++c) - BOOST_REQUIRE_CLOSE(GiniGain::Evaluate(labels, c), -0.5, 1e-5); + { + BOOST_REQUIRE_CLOSE( + GiniGain::Evaluate(labels, c, weights), -0.5, 1e-5); + double weightedGain = GiniGain::Evaluate(labels, c, weights); + + // The weighted gain should stay the same with unweight one + BOOST_REQUIRE_EQUAL( + GiniGain::Evaluate(labels, c, weights), weightedGain); + } } /** @@ -59,10 +168,14 @@ BOOST_AUTO_TEST_CASE(GiniGainEvenSplitTest) */ BOOST_AUTO_TEST_CASE(GiniGainEmptyTest) { + arma::rowvec weights = arma::ones(10); // Test across some numbers of classes. arma::Row labels; for (size_t c = 1; c < 10; ++c) - BOOST_REQUIRE_SMALL(GiniGain::Evaluate(labels, c), 1e-5); + BOOST_REQUIRE_SMALL(GiniGain::Evaluate(labels, c, weights), 1e-5); + + for (size_t c = 1; c < 10; ++c) + BOOST_REQUIRE_SMALL(GiniGain::Evaluate(labels, c, weights), 1e-5); } /** @@ -74,11 +187,18 @@ BOOST_AUTO_TEST_CASE(GiniGainEvenSplitManyClassTest) for (size_t c = 2; c < 30; ++c) { arma::Row labels(c); + arma::rowvec weights(c); for (size_t i = 0; i < c; ++i) + { labels[i] = i; + weights[i] = 1; + } // Calculate Gini gain and make sure it is correct. - BOOST_REQUIRE_CLOSE(GiniGain::Evaluate(labels, c), -(1.0 - 1.0 / c), 1e-5); + BOOST_REQUIRE_CLOSE(GiniGain::Evaluate(labels, c, weights), + -(1.0 - 1.0 / c), 1e-5); + BOOST_REQUIRE_CLOSE(GiniGain::Evaluate(labels, c, weights), + -(1.0 - 1.0 / c), 1e-5); } } @@ -90,27 +210,59 @@ BOOST_AUTO_TEST_CASE(GiniGainManyPoints) for (size_t i = 1; i < 20; ++i) { const size_t numPoints = 100 * i; + arma::rowvec weights(numPoints); + weights.ones(); arma::Row labels(numPoints); for (size_t j = 0; j < numPoints / 2; ++j) labels[j] = 0; for (size_t j = numPoints / 2; j < numPoints; ++j) labels[j] = 1; - BOOST_REQUIRE_CLOSE(GiniGain::Evaluate(labels, 2), -0.5, 1e-5); + BOOST_REQUIRE_CLOSE(GiniGain::Evaluate(labels, 2, weights), -0.5, + 1e-5); + BOOST_REQUIRE_CLOSE(GiniGain::Evaluate(labels, 2, weights), -0.5, + 1e-5); } } + +/** + * To make sure the Gini gain can been cacluate proporately with weight. + */ +BOOST_AUTO_TEST_CASE(GiniGainWithWeight) +{ + arma::Row labels(10); + arma::rowvec weights(10); + for (size_t i = 0; i < 5; ++i) + { + labels[i] = 0; + weights[i] = 0.3; + } + for (size_t i = 5; i < 10; ++i) + { + labels[i] = 1; + weights[i] = 0.7; + } + + BOOST_REQUIRE_CLOSE( + GiniGain::Evaluate(labels, 2, weights), -0.42, 1e-5); +} + /** * The information gain should be zero when the labels are perfect. */ BOOST_AUTO_TEST_CASE(InformationGainPerfectTest) { + arma::rowvec weights; arma::Row labels; labels.zeros(10); // Test that it's perfect regardless of number of classes. for (size_t c = 1; c < 10; ++c) - BOOST_REQUIRE_SMALL(InformationGain::Evaluate(labels, c), 1e-5); + { + BOOST_REQUIRE_SMALL( + InformationGain::Evaluate(labels, c, weights), 1e-5); + } } /** @@ -119,6 +271,8 @@ BOOST_AUTO_TEST_CASE(InformationGainPerfectTest) BOOST_AUTO_TEST_CASE(InformationGainEvenSplitTest) { arma::Row labels(10); + arma::rowvec weights(10); + weights.ones(); for (size_t i = 0; i < 5; ++i) labels[i] = 0; for (size_t i = 5; i < 10; ++i) @@ -126,7 +280,13 @@ BOOST_AUTO_TEST_CASE(InformationGainEvenSplitTest) // Test that it's -1 regardless of the number of classes. for (size_t c = 2; c < 10; ++c) - BOOST_REQUIRE_CLOSE(InformationGain::Evaluate(labels, c), -1.0, 1e-5); + { + // Weighted and unweighted result should be the same. + BOOST_REQUIRE_CLOSE(InformationGain::Evaluate(labels, c, weights), + -1.0, 1e-5); + BOOST_REQUIRE_CLOSE(InformationGain::Evaluate(labels, c, weights), + -1.0, 1e-5); + } } /** @@ -135,8 +295,14 @@ BOOST_AUTO_TEST_CASE(InformationGainEvenSplitTest) BOOST_AUTO_TEST_CASE(InformationGainEmptyTest) { arma::Row labels; + arma::rowvec weights = arma::ones(10); for (size_t c = 1; c < 10; ++c) - BOOST_REQUIRE_SMALL(InformationGain::Evaluate(labels, c), 1e-5); + { + BOOST_REQUIRE_SMALL(InformationGain::Evaluate(labels, c, weights), + 1e-5); + BOOST_REQUIRE_SMALL(InformationGain::Evaluate(labels, c, weights), + 1e-5); + } } /** @@ -144,6 +310,7 @@ BOOST_AUTO_TEST_CASE(InformationGainEmptyTest) */ BOOST_AUTO_TEST_CASE(InformationGainEvenSplitManyClassTest) { + arma::rowvec weights; // Try with many different numbers of classes. for (size_t c = 2; c < 30; ++c) { @@ -152,11 +319,30 @@ BOOST_AUTO_TEST_CASE(InformationGainEvenSplitManyClassTest) labels[i] = i; // Calculate information gain and make sure it is correct. - BOOST_REQUIRE_CLOSE(InformationGain::Evaluate(labels, c), + BOOST_REQUIRE_CLOSE(InformationGain::Evaluate(labels, c, weights), std::log2(1.0 / c), 1e-5); } } +/** + * Test the information gain with weighted labels + */ +BOOST_AUTO_TEST_CASE(InformationWithWeight) +{ + arma::Row labels(10); + arma::rowvec weights("1 1 1 1 1 0 0 0 0 0"); + for (size_t i = 0; i < 5; ++i) + labels[i] = 0; + for (size_t i = 5; i < 10; ++i) + labels[i] = 1; + + // Zero is not a good result as gain, but we just need to prove + // cacluation works. + BOOST_REQUIRE_CLOSE( + InformationGain::Evaluate(labels, 2, weights), 0, 1e-5); +} + + /** * The information gain should not be sensitive to the number of points. */ @@ -166,12 +352,18 @@ BOOST_AUTO_TEST_CASE(InformationGainManyPoints) { const size_t numPoints = 100 * i; arma::Row labels(numPoints); + arma::rowvec weights = arma::ones(numPoints); for (size_t j = 0; j < numPoints / 2; ++j) labels[j] = 0; for (size_t j = numPoints / 2; j < numPoints; ++j) labels[j] = 1; - BOOST_REQUIRE_CLOSE(InformationGain::Evaluate(labels, 2), -1.0, 1e-5); + BOOST_REQUIRE_CLOSE(InformationGain::Evaluate(labels, 2, weights), + -1.0, 1e-5); + // It should make no difference between a weighted and unweighted + // calculation. + BOOST_REQUIRE_CLOSE(InformationGain::Evaluate(labels, 2, weights), + -1.0, 1e-5); } } @@ -183,18 +375,26 @@ BOOST_AUTO_TEST_CASE(BestBinaryNumericSplitSimpleSplitTest) { arma::vec values("0.0 0.1 0.2 0.3 0.4 0.5 0.6 0.7 0.8 0.9 1.0"); arma::Row labels("0 0 0 0 0 1 1 1 1 1 1"); + arma::rowvec weights(labels.n_elem); + weights.ones(); arma::vec classProbabilities; BestBinaryNumericSplit::template AuxiliarySplitInfo aux; // Call the method to do the splitting. - const double bestGain = GiniGain::Evaluate(labels, 2); - const double gain = BestBinaryNumericSplit::SplitIfBetter(bestGain, - values, labels, 2, 3, classProbabilities, aux); + const double bestGain = GiniGain::Evaluate(labels, 2, weights); + const double gain = BestBinaryNumericSplit::SplitIfBetter( + bestGain, values, labels, 2, weights, 3, classProbabilities, aux); + const double weightedGain = + BestBinaryNumericSplit::SplitIfBetter(bestGain, values, + labels, 2, weights, 3, classProbabilities, aux); // Make sure that a split was made. BOOST_REQUIRE_GT(gain, bestGain); + // Make sure weight works and make no different with no weighted one + BOOST_REQUIRE_EQUAL(gain, weightedGain); + // The split is perfect, so we should be able to accomplish a gain of 0. BOOST_REQUIRE_SMALL(gain, 1e-5); @@ -213,17 +413,23 @@ BOOST_AUTO_TEST_CASE(BestBinaryNumericSplitMinSamplesTest) { arma::vec values("0.0 0.1 0.2 0.3 0.4 0.5 0.6 0.7 0.8 0.9 1.0"); arma::Row labels("0 0 0 0 0 1 1 1 1 1 1"); + arma::rowvec weights(labels.n_elem); arma::vec classProbabilities; BestBinaryNumericSplit::template AuxiliarySplitInfo aux; // Call the method to do the splitting. - const double bestGain = GiniGain::Evaluate(labels, 2); - const double gain = BestBinaryNumericSplit::SplitIfBetter(bestGain, - values, labels, 2, 8, classProbabilities, aux); + const double bestGain = GiniGain::Evaluate(labels, 2, weights); + const double gain = BestBinaryNumericSplit::SplitIfBetter( + bestGain, values, labels, 2, weights, 8, classProbabilities, aux); + // This should make no difference because it won't split at all. + const double weightedGain = + BestBinaryNumericSplit::SplitIfBetter(bestGain, values, + labels, 2, weights, 8, classProbabilities, aux); // Make sure that no split was made. BOOST_REQUIRE_EQUAL(gain, bestGain); + BOOST_REQUIRE_EQUAL(gain, weightedGain); BOOST_REQUIRE_EQUAL(classProbabilities.n_elem, 0); } @@ -235,6 +441,7 @@ BOOST_AUTO_TEST_CASE(BestBinaryNumericSplitNoGainTest) { arma::vec values(100); arma::Row labels(100); + arma::rowvec weights; for (size_t i = 0; i < 100; i += 2) { values[i] = i; @@ -247,9 +454,9 @@ BOOST_AUTO_TEST_CASE(BestBinaryNumericSplitNoGainTest) BestBinaryNumericSplit::template AuxiliarySplitInfo aux; // Call the method to do the splitting. - const double bestGain = GiniGain::Evaluate(labels, 2); - const double gain = BestBinaryNumericSplit::SplitIfBetter(bestGain, - values, labels, 2, 10, classProbabilities, aux); + const double bestGain = GiniGain::Evaluate(labels, 2, weights); + const double gain = BestBinaryNumericSplit::SplitIfBetter( + bestGain, values, labels, 2, weights, 10, classProbabilities, aux); // Make sure there was no split. BOOST_REQUIRE_EQUAL(gain, bestGain); @@ -264,14 +471,19 @@ BOOST_AUTO_TEST_CASE(AllCategoricalSplitSimpleSplitTest) { arma::vec values("0 0 0 1 1 1 2 2 2 3 3 3"); arma::Row labels("0 0 0 2 2 2 1 1 1 2 2 2"); + arma::rowvec weights(labels.n_elem); + weights.ones(); arma::vec classProbabilities; AllCategoricalSplit::template AuxiliarySplitInfo aux; // Call the method to do the splitting. - const double bestGain = GiniGain::Evaluate(labels, 3); - const double gain = AllCategoricalSplit::SplitIfBetter(bestGain, - values, 4, labels, 3, 3, classProbabilities, aux); + const double bestGain = GiniGain::Evaluate(labels, 3, weights); + const double gain = AllCategoricalSplit::SplitIfBetter( + bestGain, values, 4, labels, 3, weights, 3, classProbabilities, aux); + const double weightedGain = + AllCategoricalSplit::SplitIfBetter(bestGain, values, 4, + labels, 3, weights, 3, classProbabilities, aux); // Make sure that a split was made. BOOST_REQUIRE_GT(gain, bestGain); @@ -279,6 +491,8 @@ BOOST_AUTO_TEST_CASE(AllCategoricalSplitSimpleSplitTest) // Since the split is perfect, make sure the new gain is 0. BOOST_REQUIRE_SMALL(gain, 1e-5); + BOOST_REQUIRE_EQUAL(gain, weightedGain); + // Make sure the class probabilities now hold the number of children. BOOST_REQUIRE_EQUAL(classProbabilities.n_elem, 1); BOOST_REQUIRE_EQUAL((size_t) classProbabilities[0], 4); @@ -292,14 +506,16 @@ BOOST_AUTO_TEST_CASE(AllCategoricalSplitMinSamplesTest) { arma::vec values("0 0 0 1 1 1 2 2 2 3 3 3"); arma::Row labels("0 0 0 2 2 2 1 1 1 2 2 2"); + arma::rowvec weights(labels.n_elem); + weights.ones(); arma::vec classProbabilities; AllCategoricalSplit::template AuxiliarySplitInfo aux; // Call the method to do the splitting. - const double bestGain = GiniGain::Evaluate(labels, 3); - const double gain = AllCategoricalSplit::SplitIfBetter(bestGain, - values, 4, labels, 3, 4, classProbabilities, aux); + const double bestGain = GiniGain::Evaluate(labels, 3, weights); + const double gain = AllCategoricalSplit::SplitIfBetter( + bestGain, values, 4, labels, 3, weights, 4, classProbabilities, aux); // Make sure it's not split. BOOST_REQUIRE_EQUAL(gain, bestGain); @@ -313,6 +529,8 @@ BOOST_AUTO_TEST_CASE(AllCategoricalSplitNoGainTest) { arma::vec values(300); arma::Row labels(300); + arma::rowvec weights = arma::ones(300); + for (size_t i = 0; i < 300; i += 3) { values[i] = (i / 3) % 10; @@ -327,12 +545,16 @@ BOOST_AUTO_TEST_CASE(AllCategoricalSplitNoGainTest) AllCategoricalSplit::template AuxiliarySplitInfo aux; // Call the method to do the splitting. - const double bestGain = GiniGain::Evaluate(labels, 3); - const double gain = AllCategoricalSplit::SplitIfBetter(bestGain, - values, 10, labels, 3, 10, classProbabilities, aux); + const double bestGain = GiniGain::Evaluate(labels, 3, weights); + const double gain = AllCategoricalSplit::SplitIfBetter( + bestGain, values, 10, labels, 3, weights, 10, classProbabilities, aux); + const double weightedGain = + AllCategoricalSplit::SplitIfBetter(bestGain, values, 10, + labels, 3, weights, 10, classProbabilities, aux); // Make sure that there was no split. BOOST_REQUIRE_EQUAL(gain, bestGain); + BOOST_REQUIRE_EQUAL(gain, weightedGain); BOOST_REQUIRE_EQUAL(classProbabilities.n_elem, 0); } @@ -344,6 +566,7 @@ BOOST_AUTO_TEST_CASE(BasicConstructionTest) { arma::mat dataset(10, 1000, arma::fill::randu); arma::Row labels(1000); + for (size_t i = 0; i < 1000; ++i) labels[i] = i % 3; // 3 classes. @@ -354,6 +577,28 @@ BOOST_AUTO_TEST_CASE(BasicConstructionTest) BOOST_REQUIRE_GT(d.NumChildren(), 0); } +/** + * Construct a tree with weighted labels. + */ +BOOST_AUTO_TEST_CASE(BasicConstructionTestWithWeight) +{ + arma::mat dataset(10, 1000, arma::fill::randu); + arma::Row labels(1000); + arma::rowvec weights(labels.n_elem); + weights.ones(); + + for (size_t i = 0; i < 1000; ++i) + labels[i] = i % 3; // 3 classes. + + // Use default parameters. + DecisionTree<> wd(dataset, labels, 3, weights, 50); + DecisionTree<> d(dataset, labels, 3, 50); + + // Now require that we have some children. + BOOST_REQUIRE_GT(wd.NumChildren(), 0); + BOOST_REQUIRE_EQUAL(wd.NumChildren(), d.NumChildren()); +} + /** * Construct the decision tree on numeric data only and see that we can fit it * exactly and achieve perfect performance on the training set. @@ -365,6 +610,8 @@ BOOST_AUTO_TEST_CASE(PerfectTrainingSet) arma::Row labels(1000); for (size_t i = 0; i < 1000; ++i) labels[i] = i % 3; // 3 classes. + arma::rowvec weights(labels.n_elem); + weights.ones(); DecisionTree<> d(dataset, labels, 3, 1); // Minimum leaf size of 1. @@ -387,6 +634,41 @@ BOOST_AUTO_TEST_CASE(PerfectTrainingSet) } } +/** + * onstruct the decision tree with weighted labels + */ +BOOST_AUTO_TEST_CASE(PerfectTrainingSetWithWeight) +{ + // Completely random dataset with no structure. + arma::mat dataset(10, 1000, arma::fill::randu); + arma::Row labels(1000); + for (size_t i = 0; i < 1000; ++i) + labels[i] = i % 3; // 3 classes. + arma::rowvec weights(labels.n_elem); + weights.ones(); + + DecisionTree<> d(dataset, labels, 3, weights, 1); // Minimum leaf size of 1. + + // This part of code is dupliacte with no weighted one. + for (size_t i = 0; i < 1000; ++i) + { + size_t prediction; + arma::vec probabilities; + d.Classify(dataset.col(i), prediction, probabilities); + + BOOST_REQUIRE_EQUAL(prediction, labels[i]); + BOOST_REQUIRE_EQUAL(probabilities.n_elem, 3); + for (size_t j = 0; j < 3; ++j) + { + if (labels[i] == j) + BOOST_REQUIRE_CLOSE(probabilities[j], 1.0, 1e-5); + else + BOOST_REQUIRE_SMALL(probabilities[j], 1e-5); + } + } +} + + /** * Make sure class probabilities are computed correctly in the root node. */ @@ -428,8 +710,12 @@ BOOST_AUTO_TEST_CASE(SimpleGeneralizationTest) if (!data::Load("vc2_labels.txt", labels)) BOOST_FAIL("Cannot load labels for vc2_labels.txt"); + // Initialize an all-ones weight matrix. + arma::rowvec weights(labels.n_cols, arma::fill::ones); + // Build decision tree. DecisionTree<> d(inputData, labels, 3, 10); // Leaf size of 10. + DecisionTree<> wd(inputData, labels, 3, weights, 10); // Leaf size of 10. // Load testing data. arma::mat testData; @@ -454,6 +740,21 @@ BOOST_AUTO_TEST_CASE(SimpleGeneralizationTest) correct /= predictions.n_elem; BOOST_REQUIRE_GT(correct, 0.75); + + // reset the prediction + predictions.zeros(); + wd.Classify(testData, predictions); + + BOOST_REQUIRE_EQUAL(predictions.n_elem, testData.n_cols); + + // Figure out the accuracy. + double wdcorrect = 0.0; + for (size_t i = 0; i < predictions.n_elem; ++i) + if (predictions[i] == trueTestLabels[i]) + ++wdcorrect; + wdcorrect /= predictions.n_elem; + + BOOST_REQUIRE_GT(wdcorrect, 0.75); } /** @@ -461,102 +762,16 @@ BOOST_AUTO_TEST_CASE(SimpleGeneralizationTest) */ BOOST_AUTO_TEST_CASE(CategoricalBuildTest) { - // We'll build a spiral dataset plus two noisy categorical features. We need - // to build the distributions for the categorical features (they'll be - // discrete distributions). - DiscreteDistribution c1[5]; - // The distribution will be automatically normalized. - for (size_t i = 0; i < 5; ++i) - { - std::vector probs; - probs.push_back(arma::vec(4, arma::fill::randu)); - c1[i] = DiscreteDistribution(probs); - } - - DiscreteDistribution c2[5]; - for (size_t i = 0; i < 5; ++i) - { - std::vector probs; - probs.push_back(arma::vec(2, arma::fill::randu)); - c2[i] = DiscreteDistribution(probs); - } - - arma::mat spiralDataset(4, 10000); - arma::Row labels(10000); - for (size_t i = 0; i < 10000; ++i) - { - // One circle every 20000 samples. Plus some noise. - const double magnitude = 2.0 + (double(i) / 2000.0) + - 0.5 * mlpack::math::Random(); - const double angle = (i % 2000) * (2 * M_PI) + mlpack::math::Random(); - - const double x = magnitude * cos(angle); - const double y = magnitude * sin(angle); - - spiralDataset(0, i) = x; - spiralDataset(1, i) = y; - - // Set categorical features c1 and c2. - if (i < 2000) - { - spiralDataset(2, i) = c1[1].Random()[0]; - spiralDataset(3, i) = c2[1].Random()[0]; - labels[i] = 1; - } - else if (i < 4000) - { - spiralDataset(2, i) = c1[3].Random()[0]; - spiralDataset(3, i) = c2[3].Random()[0]; - labels[i] = 3; - } - else if (i < 6000) - { - spiralDataset(2, i) = c1[2].Random()[0]; - spiralDataset(3, i) = c2[2].Random()[0]; - labels[i] = 2; - } - else if (i < 8000) - { - spiralDataset(2, i) = c1[0].Random()[0]; - spiralDataset(3, i) = c2[0].Random()[0]; - labels[i] = 0; - } - else - { - spiralDataset(2, i) = c1[4].Random()[0]; - spiralDataset(3, i) = c2[4].Random()[0]; - labels[i] = 4; - } - } - - // Now create the dataset info. - data::DatasetInfo di(4); - di.Type(2) = data::Datatype::categorical; - di.Type(3) = data::Datatype::categorical; - // Set mappings. - di.MapString("0", 2); - di.MapString("1", 2); - di.MapString("2", 2); - di.MapString("3", 2); - di.MapString("0", 3); - di.MapString("1", 3); - - // Now shuffle the dataset. - arma::uvec indices = arma::shuffle(arma::linspace(0, 9999, - 10000)); - arma::mat d(4, 10000); - arma::Row l(10000); - for (size_t i = 0; i < 10000; ++i) - { - d.col(i) = spiralDataset.col(indices[i]); - l[i] = labels[indices[i]]; - } + arma::mat d; + arma::Row l; + data::DatasetInfo di; + MockCategoricalData(d, l, di); // Split into a training set and a test set. - arma::mat trainingData = d.cols(0, 4999); - arma::mat testData = d.cols(5000, 9999); - arma::Row trainingLabels = l.subvec(0, 4999); - arma::Row testLabels = l.subvec(5000, 9999); + arma::mat trainingData = d.cols(0, 1999); + arma::mat testData = d.cols(2000, 3999); + arma::Row trainingLabels = l.subvec(0, 1999); + arma::Row testLabels = l.subvec(2000, 3999); // Build the tree. DecisionTree<> tree(trainingData, di, trainingLabels, 5, 10); @@ -576,6 +791,44 @@ BOOST_AUTO_TEST_CASE(CategoricalBuildTest) BOOST_REQUIRE_GT(correctPct, 0.70); } +/** + * Test that we can build a decision tree with weights on a simple categorical + * dataset. + */ +BOOST_AUTO_TEST_CASE(CategoricalBuildTestWithWeight) +{ + arma::mat d; + arma::Row l; + data::DatasetInfo di; + MockCategoricalData(d, l, di); + + // Split into a training set and a test set. + arma::mat trainingData = d.cols(0, 1999); + arma::mat testData = d.cols(2000, 3999); + arma::Row trainingLabels = l.subvec(0, 1999); + arma::Row testLabels = l.subvec(2000, 3999); + + arma::Row weights = arma::ones>( + trainingLabels.n_elem); + + // Build the tree. + DecisionTree<> tree(trainingData, di, trainingLabels, 5, weights, 10); + + // Now evaluate the accuracy of the tree. + arma::Row predictions; + tree.Classify(testData, predictions); + + BOOST_REQUIRE_EQUAL(predictions.n_elem, testData.n_cols); + size_t correct = 0; + for (size_t i = 0; i < testData.n_cols; ++i) + if (testLabels[i] == predictions[i]) + ++correct; + + // Make sure we got at least 70% accuracy. + const double correctPct = double(correct) / double(testData.n_cols); + BOOST_REQUIRE_GT(correctPct, 0.70); +} + /** * Make sure that when we ask for a decision stump, we get one. */ @@ -588,8 +841,8 @@ BOOST_AUTO_TEST_CASE(DecisionStumpTest) labels[i] = i % 3; // 3 classes. // Build a decision stump. - DecisionTree stump(dataset, labels, 3, 1); + DecisionTree stump(dataset, labels, 3, 1); // Check that it has children. BOOST_REQUIRE_EQUAL(stump.NumChildren(), 2); @@ -598,4 +851,222 @@ BOOST_AUTO_TEST_CASE(DecisionStumpTest) BOOST_REQUIRE_EQUAL(stump.Child(1).NumChildren(), 0); } +/** + * Test that we can build a decision tree using weighted data (where the + * low-weighted data is random noise), and that the tree still builds correctly + * enough to get good results. + */ +BOOST_AUTO_TEST_CASE(WeightedDecisionTreeTest) +{ + arma::mat dataset; + arma::Row labels; + data::Load("vc2.csv", dataset); + data::Load("vc2_labels.txt", labels); + + // Add some noise. + arma::mat noise(dataset.n_rows, 1000, arma::fill::randu); + arma::Row noiseLabels(1000); + for (size_t i = 0; i < noiseLabels.n_elem; ++i) + noiseLabels[i] = math::RandInt(3); // Random label. + + // Concatenate data matrices. + arma::mat data = arma::join_rows(dataset, noise); + arma::Row fullLabels = arma::join_rows(labels, noiseLabels); + + // Now set weights. + arma::rowvec weights(dataset.n_cols + 1000); + for (size_t i = 0; i < dataset.n_cols; ++i) + weights[i] = math::Random(0.9, 1.0); + for (size_t i = dataset.n_cols; i < dataset.n_cols + 1000; ++i) + weights[i] = math::Random(0.0, 0.01); // Low weights for false points. + + // Now build the decision tree. I think the syntax is right here. + DecisionTree<> d(data, fullLabels, 3, weights, 10); + + // Now we can check that we get good performance on the VC2 test set. + arma::mat testData; + arma::Row testLabels; + data::Load("vc2_test.csv", testData); + data::Load("vc2_test_labels.txt", testLabels); + + arma::Row predictions; + d.Classify(testData, predictions); + + BOOST_REQUIRE_EQUAL(predictions.n_elem, testData.n_cols); + + // Figure out the accuracy. + double correct = 0.0; + for (size_t i = 0; i < predictions.n_elem; ++i) + if (predictions[i] == testLabels[i]) + ++correct; + correct /= predictions.n_elem; + + BOOST_REQUIRE_GT(correct, 0.75); +} +/** + * Test that we can build a decision tree on a simple categorical dataset using + * weights, with low-weight noise added. + */ +BOOST_AUTO_TEST_CASE(CategoricalWeightedBuildTest) +{ + arma::mat d; + arma::Row l; + data::DatasetInfo di; + MockCategoricalData(d, l, di); + + // Split into a training set and a test set. + arma::mat trainingData = d.cols(0, 1999); + arma::mat testData = d.cols(2000, 3999); + arma::Row trainingLabels = l.subvec(0, 1999); + arma::Row testLabels = l.subvec(2000, 3999); + + // Now create random points. + arma::mat randomNoise(4, 2000); + arma::Row randomLabels(2000); + for (size_t i = 0; i < 2000; ++i) + { + randomNoise(0, i) = math::Random(); + randomNoise(1, i) = math::Random(); + randomNoise(2, i) = math::RandInt(4); + randomNoise(3, i) = math::RandInt(2); + randomLabels[i] = math::RandInt(5); + } + + // Generate weights. + arma::rowvec weights(4000); + for (size_t i = 0; i < 2000; ++i) + weights[i] = math::Random(0.9, 1.0); + for (size_t i = 2000; i < 4000; ++i) + weights[i] = math::Random(0.0, 0.001); + + arma::mat fullData = arma::join_rows(trainingData, randomNoise); + arma::Row fullLabels = arma::join_rows(trainingLabels, randomLabels); + + // Build the tree. + DecisionTree<> tree(fullData, di, fullLabels, 5, weights, 10); + + // Now evaluate the accuracy of the tree. + arma::Row predictions; + tree.Classify(testData, predictions); + + BOOST_REQUIRE_EQUAL(predictions.n_elem, testData.n_cols); + size_t correct = 0; + for (size_t i = 0; i < testData.n_cols; ++i) + if (testLabels[i] == predictions[i]) + ++correct; + + // Make sure we got at least 70% accuracy. + const double correctPct = double(correct) / double(testData.n_cols); + BOOST_REQUIRE_GT(correctPct, 0.70); +} + +/** + * Test that we can build a decision tree using weighted data (where the + * low-weighted data is random noise) with information gain, and that the tree + * still builds correctly enough to get good results. + */ +BOOST_AUTO_TEST_CASE(WeightedDecisionTreeInformationGainTest) +{ + arma::mat dataset; + arma::Row labels; + data::Load("vc2.csv", dataset); + data::Load("vc2_labels.txt", labels); + + // Add some noise. + arma::mat noise(dataset.n_rows, 1000, arma::fill::randu); + arma::Row noiseLabels(1000); + for (size_t i = 0; i < noiseLabels.n_elem; ++i) + noiseLabels[i] = math::RandInt(3); // Random label. + + // Concatenate data matrices. + arma::mat data = arma::join_rows(dataset, noise); + arma::Row fullLabels = arma::join_rows(labels, noiseLabels); + + // Now set weights. + arma::rowvec weights(dataset.n_cols + 1000); + for (size_t i = 0; i < dataset.n_cols; ++i) + weights[i] = math::Random(0.9, 1.0); + for (size_t i = dataset.n_cols; i < dataset.n_cols + 1000; ++i) + weights[i] = math::Random(0.0, 0.01); // Low weights for false points. + + // Now build the decision tree. I think the syntax is right here. + DecisionTree d(data, fullLabels, 3, weights, 10); + + // Now we can check that we get good performance on the VC2 test set. + arma::mat testData; + arma::Row testLabels; + data::Load("vc2_test.csv", testData); + data::Load("vc2_test_labels.txt", testLabels); + + arma::Row predictions; + d.Classify(testData, predictions); + + BOOST_REQUIRE_EQUAL(predictions.n_elem, testData.n_cols); + + // Figure out the accuracy. + double correct = 0.0; + for (size_t i = 0; i < predictions.n_elem; ++i) + if (predictions[i] == testLabels[i]) + ++correct; + correct /= predictions.n_elem; + + BOOST_REQUIRE_GT(correct, 0.75); +} +/** + * Test that we can build a decision tree using information gain on a simple + * categorical dataset using weights, with low-weight noise added. + */ +BOOST_AUTO_TEST_CASE(CategoricalInformationGainWeightedBuildTest) +{ + arma::mat d; + arma::Row l; + data::DatasetInfo di; + MockCategoricalData(d, l, di); + + // Split into a training set and a test set. + arma::mat trainingData = d.cols(0, 1999); + arma::mat testData = d.cols(2000, 3999); + arma::Row trainingLabels = l.subvec(0, 1999); + arma::Row testLabels = l.subvec(2000, 3999); + + // Now create random points. + arma::mat randomNoise(4, 2000); + arma::Row randomLabels(2000); + for (size_t i = 0; i < 2000; ++i) + { + randomNoise(0, i) = math::Random(); + randomNoise(1, i) = math::Random(); + randomNoise(2, i) = math::RandInt(4); + randomNoise(3, i) = math::RandInt(2); + randomLabels[i] = math::RandInt(5); + } + + // Generate weights. + arma::rowvec weights(4000); + for (size_t i = 0; i < 2000; ++i) + weights[i] = math::Random(0.9, 1.0); + for (size_t i = 2000; i < 4000; ++i) + weights[i] = math::Random(0.0, 0.001); + + arma::mat fullData = arma::join_rows(trainingData, randomNoise); + arma::Row fullLabels = arma::join_rows(trainingLabels, randomLabels); + + // Build the tree. + DecisionTree tree(fullData, di, fullLabels, 5, weights, 10); + + // Now evaluate the accuracy of the tree. + arma::Row predictions; + tree.Classify(testData, predictions); + + BOOST_REQUIRE_EQUAL(predictions.n_elem, testData.n_cols); + size_t correct = 0; + for (size_t i = 0; i < testData.n_cols; ++i) + if (testLabels[i] == predictions[i]) + ++correct; + + // Make sure we got at least 70% accuracy. + const double correctPct = double(correct) / double(testData.n_cols); + BOOST_REQUIRE_GT(correctPct, 0.70); +} + BOOST_AUTO_TEST_SUITE_END(); diff --git a/src/mlpack/tests/det_test.cpp b/src/mlpack/tests/det_test.cpp index bff2baf583..7c44f2a8dc 100644 --- a/src/mlpack/tests/det_test.cpp +++ b/src/mlpack/tests/det_test.cpp @@ -94,7 +94,7 @@ BOOST_AUTO_TEST_CASE(TestWithinRange) BOOST_AUTO_TEST_CASE(TestFindSplit) { - arma::mat testData(3,5); + arma::mat testData(3, 5); testData << 4 << 5 << 7 << 3 << 5 << arma::endr << 5 << 0 << 1 << 7 << 1 << arma::endr @@ -102,16 +102,17 @@ BOOST_AUTO_TEST_CASE(TestFindSplit) DTree testDTree(testData); - size_t obDim, trueDim; - double trueLeftError, obLeftError, trueRightError, obRightError, obSplit, trueSplit; + size_t obDim; + double obLeftError, obRightError, obSplit; - trueDim = 2; - trueSplit = 5.5; - trueLeftError = 2 * log(2.0 / 5.0) - (log(7.0) + log(4.0) + log(4.5)); - trueRightError = 2 * log(3.0 / 5.0) - (log(7.0) + log(4.0) + log(2.5)); + size_t trueDim = 2; + double trueSplit = 5.5; + double trueLeftError = 2 * log(2.0 / 5.0) - (log(7.0) + log(4.0) + log(4.5)); + double trueRightError = 2 * log(3.0 / 5.0) - (log(7.0) + log(4.0) + log(2.5)); testDTree.logVolume = log(7.0) + log(4.0) + log(7.0); - BOOST_REQUIRE(testDTree.FindSplit(testData, obDim, obSplit, obLeftError, obRightError, 1)); + BOOST_REQUIRE(testDTree.FindSplit( + testData, obDim, obSplit, obLeftError, obRightError, 1)); BOOST_REQUIRE(trueDim == obDim); BOOST_REQUIRE_CLOSE(trueSplit, obSplit, 1e-10); @@ -136,7 +137,8 @@ BOOST_AUTO_TEST_CASE(TestSplitData) size_t splitDim = 2; double trueSplitVal = 5.5; - size_t splitInd = testDTree.SplitData(testData, splitDim, trueSplitVal, oTest); + size_t splitInd = testDTree.SplitData( + testData, splitDim, trueSplitVal, oTest); BOOST_REQUIRE_EQUAL(splitInd, 2); // 2 points on left side. @@ -149,7 +151,7 @@ BOOST_AUTO_TEST_CASE(TestSplitData) BOOST_AUTO_TEST_CASE(TestSparseFindSplit) { - arma::mat realData(4,7); + arma::mat realData(4, 7); realData << .0 << 4 << 5 << 7 << 0 << 5 << 0 << arma::endr << .0 << 5 << 0 << 0 << 1 << 7 << 1 << arma::endr @@ -160,16 +162,19 @@ BOOST_AUTO_TEST_CASE(TestSparseFindSplit) DTree testDTree(testData); - size_t obDim, trueDim; - double trueLeftError, obLeftError, trueRightError, obRightError, obSplit, trueSplit; + size_t obDim; + double obLeftError, obRightError, obSplit; - trueDim = 1; - trueSplit = .5; - trueLeftError = 2 * log(3.0 / 7.0) - (log(7.0) + log(0.5) + log(8.0) + log(6.0)); - trueRightError = 2 * log(4.0 / 7.0) - (log(7.0) + log(6.5) + log(8.0) + log(6.0)); + size_t trueDim = 1; + double trueSplit = .5; + double trueLeftError = 2 * log(3.0 / 7.0) - + (log(7.0) + log(0.5) + log(8.0) + log(6.0)); + double trueRightError = 2 * log(4.0 / 7.0) - + (log(7.0) + log(6.5) + log(8.0) + log(6.0)); testDTree.logVolume = log(7.0) + log(7.0) + log(8.0) + log(6.0); - BOOST_REQUIRE(testDTree.FindSplit(testData, obDim, obSplit, obLeftError, obRightError, 1)); + BOOST_REQUIRE(testDTree.FindSplit( + testData, obDim, obSplit, obLeftError, obRightError, 1)); BOOST_REQUIRE(trueDim == obDim); BOOST_REQUIRE_CLOSE(trueSplit, obSplit, 1e-10); @@ -180,7 +185,7 @@ BOOST_AUTO_TEST_CASE(TestSparseFindSplit) BOOST_AUTO_TEST_CASE(TestSparseSplitData) { - arma::mat realData(4,7); + arma::mat realData(4, 7); realData << .0 << 4 << 5 << 7 << 0 << 5 << 0 << arma::endr << .0 << 5 << 0 << 0 << 1 << 7 << 1 << arma::endr @@ -197,7 +202,8 @@ BOOST_AUTO_TEST_CASE(TestSparseSplitData) size_t splitDim = 1; double trueSplitVal = .5; - size_t splitInd = testDTree.SplitData(testData, splitDim, trueSplitVal, oTest); + size_t splitInd = testDTree.SplitData( + testData, splitDim, trueSplitVal, oTest); BOOST_REQUIRE_EQUAL(splitInd, 3); // 2 points on left side. diff --git a/src/mlpack/tests/distribution_test.cpp b/src/mlpack/tests/distribution_test.cpp index bbf22a6c40..24b02d721b 100644 --- a/src/mlpack/tests/distribution_test.cpp +++ b/src/mlpack/tests/distribution_test.cpp @@ -357,7 +357,6 @@ BOOST_AUTO_TEST_CASE(GaussianMultivariateProbabilityTest) g.Mean() *= -1; BOOST_REQUIRE_CLOSE(g.Probability(-x), 1.4673143531128877e-06, 1e-5); BOOST_REQUIRE_CLOSE(g.Probability(x), 7.7404143494891786e-09, 1e-8); - } /** diff --git a/src/mlpack/tests/emst_test.cpp b/src/mlpack/tests/emst_test.cpp index 435070c9db..d35cfd757b 100644 --- a/src/mlpack/tests/emst_test.cpp +++ b/src/mlpack/tests/emst_test.cpp @@ -247,7 +247,6 @@ BOOST_AUTO_TEST_CASE(CoverTreeTest) BOOST_REQUIRE_EQUAL(bstResults(1, i), coverResults(1, i)); BOOST_REQUIRE_CLOSE(bstResults(2, i), coverResults(2, i), 1e-5); } - } /** @@ -277,7 +276,6 @@ BOOST_AUTO_TEST_CASE(BallTreeTest) BOOST_REQUIRE_EQUAL(bstResults(1, i), ballResults(1, i)); BOOST_REQUIRE_CLOSE(bstResults(2, i), ballResults(2, i), 1e-5); } - } BOOST_AUTO_TEST_SUITE_END(); diff --git a/src/mlpack/tests/fastmks_test.cpp b/src/mlpack/tests/fastmks_test.cpp index e9f5cf9a47..ed1032b6e5 100644 --- a/src/mlpack/tests/fastmks_test.cpp +++ b/src/mlpack/tests/fastmks_test.cpp @@ -67,7 +67,7 @@ BOOST_AUTO_TEST_CASE(DualTreeVsNaive) { // First create a random dataset. arma::mat data; - data.randn(10, 5000); + data.randn(10, 2000); LinearKernel lk; // Now run FastMKS naively. @@ -101,7 +101,7 @@ BOOST_AUTO_TEST_CASE(DualTreeVsSingleTree) { // First create a random dataset. arma::mat data; - data.randu(8, 5000); + data.randu(8, 2000); PolynomialKernel pk(5.0, 2.5); FastMKS single(data, pk, true); @@ -174,12 +174,19 @@ BOOST_AUTO_TEST_CASE(SparsePolynomialFastMKSTest) for (size_t i = 0; i < 100; ++i) for (size_t j = 0; j < 100; ++j) + { if (std::abs(pk.Evaluate(dataset.col(i), dataset.col(j))) < 1e-10) - BOOST_REQUIRE_SMALL(pk.Evaluate(denseset.col(i), denseset.col(j)), 1e-10); + { + BOOST_REQUIRE_SMALL( + pk.Evaluate(denseset.col(i), denseset.col(j)), 1e-10); + } else - BOOST_REQUIRE_CLOSE(pk.Evaluate(dataset.col(i), dataset.col(j)), - pk.Evaluate(denseset.col(i), denseset.col(j)), - 1e-5); + { + BOOST_REQUIRE_CLOSE( + pk.Evaluate(dataset.col(i), dataset.col(j)), + pk.Evaluate(denseset.col(i), denseset.col(j)), 1e-5); + } + } FastMKS sparsepoly(dataset); FastMKS densepoly(denseset); diff --git a/src/mlpack/tests/feedforward_network_test.cpp b/src/mlpack/tests/feedforward_network_test.cpp index d8f82ea023..affa3cc4fa 100644 --- a/src/mlpack/tests/feedforward_network_test.cpp +++ b/src/mlpack/tests/feedforward_network_test.cpp @@ -426,4 +426,4 @@ BOOST_AUTO_TEST_CASE(FFNMiscTest) movedModel = std::move(copiedModel); } -BOOST_AUTO_TEST_SUITE_END(); \ No newline at end of file +BOOST_AUTO_TEST_SUITE_END(); diff --git a/src/mlpack/tests/gmm_test.cpp b/src/mlpack/tests/gmm_test.cpp index 28a1dfcb7b..d3c8cd5c29 100644 --- a/src/mlpack/tests/gmm_test.cpp +++ b/src/mlpack/tests/gmm_test.cpp @@ -494,7 +494,7 @@ BOOST_AUTO_TEST_CASE(GMMLoadSaveTest) } // Remove clutter. - //remove("test-gmm-save.xml"); + // remove("test-gmm-save.xml"); BOOST_REQUIRE_EQUAL(gmm.Gaussians(), gmm2.Gaussians()); BOOST_REQUIRE_EQUAL(gmm.Dimensionality(), gmm2.Dimensionality()); @@ -552,11 +552,10 @@ BOOST_AUTO_TEST_CASE(PositiveDefiniteConstraintTest) arma::mat c; #if (ARMA_VERSION_MAJOR < 4) || \ ((ARMA_VERSION_MAJOR == 4) && (ARMA_VERSION_MINOR < 500)) - BOOST_REQUIRE(arma::chol(c, cov)); + BOOST_REQUIRE(arma::chol(c, cov)); #else - BOOST_REQUIRE(arma::chol(c, cov, "lower")); + BOOST_REQUIRE(arma::chol(c, cov, "lower")); #endif - } } diff --git a/src/mlpack/tests/hmm_test.cpp b/src/mlpack/tests/hmm_test.cpp index 4117571508..4d8656bd02 100644 --- a/src/mlpack/tests/hmm_test.cpp +++ b/src/mlpack/tests/hmm_test.cpp @@ -78,9 +78,12 @@ BOOST_AUTO_TEST_CASE(BorodovskyHMMTestViterbi) "0.5 0.5 0.6"); // Four emission states: A, C, G, T. Start state doesn't emit... std::vector emission(3); - emission[0] = DiscreteDistribution(std::vector{"0.25 0.25 0.25 0.25"}); - emission[1] = DiscreteDistribution(std::vector{"0.20 0.30 0.30 0.20"}); - emission[2] = DiscreteDistribution(std::vector{"0.30 0.20 0.20 0.30"}); + emission[0] = DiscreteDistribution( + std::vector{"0.25 0.25 0.25 0.25"}); + emission[1] = DiscreteDistribution( + std::vector{"0.20 0.30 0.30 0.20"}); + emission[2] = DiscreteDistribution( + std::vector{"0.30 0.20 0.20 0.30"}); HMM hmm(initial, transition, emission); @@ -1002,7 +1005,7 @@ BOOST_AUTO_TEST_CASE(GMMHMMLoadSaveTest) // Create a GMM HMM, save it, and load it. HMM hmm(3, GMM(4, 3)); - for(size_t j = 0; j < hmm.Emission().size(); ++j) + for (size_t j = 0; j < hmm.Emission().size(); ++j) { hmm.Emission()[j].Weights().randu(); for (size_t i = 0; i < hmm.Emission()[j].Gaussians(); ++i) @@ -1055,7 +1058,7 @@ BOOST_AUTO_TEST_CASE(GMMHMMLoadSaveTest) for (size_t k = 0; k < hmm.Emission()[j].Dimensionality(); ++k) { - BOOST_REQUIRE_CLOSE(hmm.Emission()[j].Component(i).Covariance()(l,k), + BOOST_REQUIRE_CLOSE(hmm.Emission()[j].Component(i).Covariance()(l, k), hmm2.Emission()[j].Component(i).Covariance()(l, k), 1e-3); } } @@ -1072,7 +1075,7 @@ BOOST_AUTO_TEST_CASE(GaussianHMMLoadSaveTest) HMM hmm(3, GaussianDistribution(2)); - for(size_t j = 0; j < hmm.Emission().size(); ++j) + for (size_t j = 0; j < hmm.Emission().size(); ++j) { hmm.Emission()[j].Mean().randu(); arma::mat covariance = arma::randu( @@ -1112,7 +1115,7 @@ BOOST_AUTO_TEST_CASE(GaussianHMMLoadSaveTest) hmm2.Emission()[j].Mean()[i], 1e-3); for (size_t k = 0; k < hmm.Emission()[j].Dimensionality(); ++k) { - BOOST_REQUIRE_CLOSE(hmm.Emission()[j].Covariance()(i,k), + BOOST_REQUIRE_CLOSE(hmm.Emission()[j].Covariance()(i, k), hmm2.Emission()[j].Covariance()(i, k), 1e-3); } } @@ -1140,7 +1143,7 @@ BOOST_AUTO_TEST_CASE(DiscreteHMMLoadSaveTest) HMM hmm(3, DiscreteDistribution(3)); - for(size_t j = 0; j < hmm.Emission().size(); ++j) + for (size_t j = 0; j < hmm.Emission().size(); ++j) { hmm.Emission()[j].Probabilities() = arma::randu(3); hmm.Emission()[j].Probabilities() /= accu(emission[j].Probabilities()); diff --git a/src/mlpack/tests/ind2sub_test.cpp b/src/mlpack/tests/ind2sub_test.cpp index 98e425c01d..eb522e9d57 100644 --- a/src/mlpack/tests/ind2sub_test.cpp +++ b/src/mlpack/tests/ind2sub_test.cpp @@ -21,7 +21,7 @@ BOOST_AUTO_TEST_SUITE(ind2subTest); */ BOOST_AUTO_TEST_CASE(ind2sub_test) { - arma::mat A = arma::randu(4,5); + arma::mat A = arma::randu(4, 5); size_t index = 13; arma::uvec u = arma::ind2sub(arma::size(A), index); diff --git a/src/mlpack/tests/kernel_test.cpp b/src/mlpack/tests/kernel_test.cpp index 9e6aa41951..cfe9361624 100644 --- a/src/mlpack/tests/kernel_test.cpp +++ b/src/mlpack/tests/kernel_test.cpp @@ -16,7 +16,6 @@ #include #include #include -#include #include #include #include @@ -254,8 +253,8 @@ BOOST_AUTO_TEST_CASE(LinearKernelTest) arma::vec b = ".56 .21 .623 .82"; LinearKernel lk; - BOOST_REQUIRE_CLOSE(lk.Evaluate(a,b), .5062, 1e-5); - BOOST_REQUIRE_CLOSE(lk.Evaluate(b,a), .5062, 1e-5); + BOOST_REQUIRE_CLOSE(lk.Evaluate(a, b), .5062, 1e-5); + BOOST_REQUIRE_CLOSE(lk.Evaluate(b, a), .5062, 1e-5); } /** @@ -267,8 +266,8 @@ BOOST_AUTO_TEST_CASE(LinearKernelOrthogonalTest) arma::vec b = "0 0 1"; LinearKernel lk; - BOOST_REQUIRE_SMALL(lk.Evaluate(a,b), 1e-5); - BOOST_REQUIRE_SMALL(lk.Evaluate(b,a), 1e-5); + BOOST_REQUIRE_SMALL(lk.Evaluate(a, b), 1e-5); + BOOST_REQUIRE_SMALL(lk.Evaluate(b, a), 1e-5); } BOOST_AUTO_TEST_CASE(GaussianKernelTest) @@ -294,9 +293,9 @@ BOOST_AUTO_TEST_CASE(GaussianKernelTest) BOOST_REQUIRE_CLOSE(gk.Normalizer(3), 1.9687012432153019, 1e-5); BOOST_REQUIRE_CLOSE(gk.Normalizer(4), 2.4674011002723386, 1e-5); /* check the convolution integral */ - BOOST_REQUIRE_CLOSE(gk.ConvolutionIntegral(a,b), 0.024304474038457577, 1e-5); - BOOST_REQUIRE_CLOSE(gk.ConvolutionIntegral(a,c), 0.024304474038457577, 1e-5); - BOOST_REQUIRE_CLOSE(gk.ConvolutionIntegral(b,c), 0.024304474038457577, 1e-5); + BOOST_REQUIRE_CLOSE(gk.ConvolutionIntegral(a, b), 0.024304474038457577, 1e-5); + BOOST_REQUIRE_CLOSE(gk.ConvolutionIntegral(a, c), 0.024304474038457577, 1e-5); + BOOST_REQUIRE_CLOSE(gk.ConvolutionIntegral(b, c), 0.024304474038457577, 1e-5); } BOOST_AUTO_TEST_CASE(GaussianKernelSerializationTest) @@ -334,9 +333,9 @@ BOOST_AUTO_TEST_CASE(SphericalKernelTest) BOOST_REQUIRE_CLOSE(sk.Normalizer(3), 0.52359877559829893, 1e-5); BOOST_REQUIRE_CLOSE(sk.Normalizer(4), 0.30842513753404244, 1e-5); /* check the convolution integral */ - BOOST_REQUIRE_CLOSE(sk.ConvolutionIntegral(a,b), 0.0, 1e-5); - BOOST_REQUIRE_CLOSE(sk.ConvolutionIntegral(a,c), 0.0, 1e-5); - BOOST_REQUIRE_CLOSE(sk.ConvolutionIntegral(b,c), 1.0021155029652784, 1e-5); + BOOST_REQUIRE_CLOSE(sk.ConvolutionIntegral(a, b), 0.0, 1e-5); + BOOST_REQUIRE_CLOSE(sk.ConvolutionIntegral(a, c), 0.0, 1e-5); + BOOST_REQUIRE_CLOSE(sk.ConvolutionIntegral(b, c), 1.0021155029652784, 1e-5); } BOOST_AUTO_TEST_CASE(EpanechnikovKernelTest) @@ -360,9 +359,9 @@ BOOST_AUTO_TEST_CASE(EpanechnikovKernelTest) BOOST_REQUIRE_CLOSE(ek.Normalizer(3), 0.20943951023931956, 1e-5); BOOST_REQUIRE_CLOSE(ek.Normalizer(4), 0.10280837917801415, 1e-5); /* check the convolution integral */ - BOOST_REQUIRE_CLOSE(ek.ConvolutionIntegral(a,b), 0.0, 1e-5); - BOOST_REQUIRE_CLOSE(ek.ConvolutionIntegral(a,c), 0.0, 1e-5); - BOOST_REQUIRE_CLOSE(ek.ConvolutionIntegral(b,c), 1.5263455690698258, 1e-5); + BOOST_REQUIRE_CLOSE(ek.ConvolutionIntegral(a, b), 0.0, 1e-5); + BOOST_REQUIRE_CLOSE(ek.ConvolutionIntegral(a, c), 0.0, 1e-5); + BOOST_REQUIRE_CLOSE(ek.ConvolutionIntegral(b, c), 1.5263455690698258, 1e-5); } BOOST_AUTO_TEST_CASE(PolynomialKernelTest) diff --git a/src/mlpack/tests/knn_test.cpp b/src/mlpack/tests/knn_test.cpp index 6127ffc184..47d2cfe35a 100644 --- a/src/mlpack/tests/knn_test.cpp +++ b/src/mlpack/tests/knn_test.cpp @@ -384,7 +384,6 @@ BOOST_AUTO_TEST_CASE(ExhaustiveSyntheticTest) for (int i = 0; i < 3; i++) { - switch (i) { case 0: // Use the dual-tree method. @@ -649,7 +648,6 @@ BOOST_AUTO_TEST_CASE(ExhaustiveSyntheticTest) BOOST_REQUIRE_CLOSE(distances(8, newFromOld[10]), 3.00, 1e-5); BOOST_REQUIRE_EQUAL(neighbors(9, newFromOld[10]), newFromOld[4]); BOOST_REQUIRE_CLOSE(distances(9, newFromOld[10]), 4.05, 1e-5); - } } diff --git a/src/mlpack/tests/ksinit_test.cpp b/src/mlpack/tests/ksinit_test.cpp index 16ec7457e4..467bee075a 100644 --- a/src/mlpack/tests/ksinit_test.cpp +++ b/src/mlpack/tests/ksinit_test.cpp @@ -242,7 +242,7 @@ BOOST_AUTO_TEST_CASE(IrisDataset) // Normalization used in the paper. dataset /= 10; - //Counter for the number of failures. + // Counter for the number of failures. size_t numFails = 0; // It isn't guaranteed that the network will converge in the specified number @@ -330,4 +330,4 @@ BOOST_AUTO_TEST_CASE(NonLinearFunctionApproximation) BOOST_REQUIRE_LE(numFails, 4); } -BOOST_AUTO_TEST_SUITE_END(); \ No newline at end of file +BOOST_AUTO_TEST_SUITE_END(); diff --git a/src/mlpack/tests/lars_test.cpp b/src/mlpack/tests/lars_test.cpp index b33b33e9a5..bf78dde33e 100644 --- a/src/mlpack/tests/lars_test.cpp +++ b/src/mlpack/tests/lars_test.cpp @@ -23,18 +23,19 @@ using namespace mlpack::regression; BOOST_AUTO_TEST_SUITE(LARSTest); -void GenerateProblem(arma::mat& X, arma::vec& y, size_t nPoints, size_t nDims) +void GenerateProblem( + arma::mat& X, arma::rowvec& y, size_t nPoints, size_t nDims) { X = arma::randn(nDims, nPoints); arma::vec beta = arma::randn(nDims, 1); - y = trans(X) * beta; + y = beta.t() * X; } void LARSVerifyCorrectness(arma::vec beta, arma::vec errCorr, double lambda) { size_t nDims = beta.n_elem; const double tol = 1e-10; - for(size_t j = 0; j < nDims; j++) + for (size_t j = 0; j < nDims; j++) { if (beta(j) == 0) { @@ -57,14 +58,14 @@ void LARSVerifyCorrectness(arma::vec beta, arma::vec errCorr, double lambda) void LassoTest(size_t nPoints, size_t nDims, bool elasticNet, bool useCholesky) { arma::mat X; - arma::vec y; + arma::rowvec y; for (size_t i = 0; i < 100; i++) { GenerateProblem(X, y, nPoints, nDims); // Armadillo's median is broken, so... - arma::vec sortedAbsCorr = sort(abs(X * y)); + arma::vec sortedAbsCorr = sort(abs(X * y.t())); double lambda1 = sortedAbsCorr(nDims / 2); double lambda2; if (elasticNet) @@ -78,7 +79,7 @@ void LassoTest(size_t nPoints, size_t nDims, bool elasticNet, bool useCholesky) lars.Train(X, y, betaOpt); arma::vec errCorr = (X * trans(X) + lambda2 * - arma::eye(nDims, nDims)) * betaOpt - X * y; + arma::eye(nDims, nDims)) * betaOpt - X * y.t(); LARSVerifyCorrectness(betaOpt, errCorr, lambda1); } @@ -116,7 +117,7 @@ BOOST_AUTO_TEST_CASE(CholeskySingularityTest) data::Load("lars_dependent_x.csv", X); data::Load("lars_dependent_y.csv", Y); - arma::vec y = Y.row(0).t(); + arma::rowvec y = Y.row(0); // Test for a couple values of lambda1. for (double lambda1 = 0.0; lambda1 < 1.0; lambda1 += 0.1) @@ -125,7 +126,7 @@ BOOST_AUTO_TEST_CASE(CholeskySingularityTest) arma::vec betaOpt; lars.Train(X, y, betaOpt); - arma::vec errCorr = (X * X.t()) * betaOpt - X * y; + arma::vec errCorr = (X * X.t()) * betaOpt - X * y.t(); LARSVerifyCorrectness(betaOpt, errCorr, lambda1); } @@ -140,7 +141,7 @@ BOOST_AUTO_TEST_CASE(NoCholeskySingularityTest) data::Load("lars_dependent_x.csv", X); data::Load("lars_dependent_y.csv", Y); - arma::vec y = Y.row(0).t(); + arma::rowvec y = Y.row(0); // Test for a couple values of lambda1. for (double lambda1 = 0.0; lambda1 < 1.0; lambda1 += 0.1) @@ -149,7 +150,7 @@ BOOST_AUTO_TEST_CASE(NoCholeskySingularityTest) arma::vec betaOpt; lars.Train(X, y, betaOpt); - arma::vec errCorr = (X * X.t()) * betaOpt - X * y; + arma::vec errCorr = (X * X.t()) * betaOpt - X * y.t(); // #373: this test fails on i386 only sometimes. // LARSVerifyCorrectness(betaOpt, errCorr, lambda1); @@ -165,7 +166,7 @@ BOOST_AUTO_TEST_CASE(PredictTest) bool useCholesky = bool(i); arma::mat X; - arma::vec y; + arma::rowvec y; GenerateProblem(X, y, 1000, 100); @@ -180,9 +181,9 @@ BOOST_AUTO_TEST_CASE(PredictTest) // Calculate what the actual error should be with these regression // parameters. arma::vec betaOptPred = (X * X.t()) * betaOpt; - arma::vec predictions; + arma::rowvec predictions; lars.Predict(X, predictions); - arma::vec adjPred = X * predictions; + arma::vec adjPred = X * predictions.t(); BOOST_REQUIRE_EQUAL(predictions.n_elem, 1000); for (size_t i = 0; i < betaOptPred.n_elem; ++i) @@ -200,7 +201,7 @@ BOOST_AUTO_TEST_CASE(PredictTest) BOOST_AUTO_TEST_CASE(PredictRowMajorTest) { arma::mat X; - arma::vec y; + arma::rowvec y; GenerateProblem(X, y, 1000, 100); // Set lambdas to 0. @@ -211,7 +212,7 @@ BOOST_AUTO_TEST_CASE(PredictRowMajorTest) // Get both row-major and column-major predictions. Make sure they are the // same. - arma::vec rowMajorPred, colMajorPred; + arma::rowvec rowMajorPred, colMajorPred; lars.Predict(X, colMajorPred); lars.Predict(X.t(), rowMajorPred, true); @@ -232,11 +233,11 @@ BOOST_AUTO_TEST_CASE(PredictRowMajorTest) BOOST_AUTO_TEST_CASE(RetrainTest) { arma::mat origX; - arma::vec origY; + arma::rowvec origY; GenerateProblem(origX, origY, 1000, 50); arma::mat newX; - arma::vec newY; + arma::rowvec newY; GenerateProblem(newX, newY, 750, 75); LARS lars(false, 0.1, 0.1); @@ -247,7 +248,7 @@ BOOST_AUTO_TEST_CASE(RetrainTest) lars.Train(newX, newY, betaOpt); arma::vec errCorr = (newX * trans(newX) + 0.1 * - arma::eye(75, 75)) * betaOpt - newX * newY; + arma::eye(75, 75)) * betaOpt - newX * newY.t(); LARSVerifyCorrectness(betaOpt, errCorr, 0.1); } @@ -259,11 +260,11 @@ BOOST_AUTO_TEST_CASE(RetrainTest) BOOST_AUTO_TEST_CASE(RetrainCholeskyTest) { arma::mat origX; - arma::vec origY; + arma::rowvec origY; GenerateProblem(origX, origY, 1000, 50); arma::mat newX; - arma::vec newY; + arma::rowvec newY; GenerateProblem(newX, newY, 750, 75); LARS lars(true, 0.1, 0.1); @@ -274,7 +275,7 @@ BOOST_AUTO_TEST_CASE(RetrainCholeskyTest) lars.Train(newX, newY, betaOpt); arma::vec errCorr = (newX * trans(newX) + 0.1 * - arma::eye(75, 75)) * betaOpt - newX * newY; + arma::eye(75, 75)) * betaOpt - newX * newY.t(); LARSVerifyCorrectness(betaOpt, errCorr, 0.1); } @@ -286,7 +287,7 @@ BOOST_AUTO_TEST_CASE(RetrainCholeskyTest) BOOST_AUTO_TEST_CASE(TrainingAndAccessingBetaTest) { arma::mat X; - arma::vec y; + arma::rowvec y; GenerateProblem(X, y, 1000, 100); @@ -309,7 +310,7 @@ BOOST_AUTO_TEST_CASE(TrainingAndAccessingBetaTest) BOOST_AUTO_TEST_CASE(TrainingConstructorWithDefaultsTest) { arma::mat X; - arma::vec y; + arma::rowvec y; GenerateProblem(X, y, 1000, 100); @@ -331,7 +332,7 @@ BOOST_AUTO_TEST_CASE(TrainingConstructorWithDefaultsTest) BOOST_AUTO_TEST_CASE(TrainingConstructorWithNonDefaultsTest) { arma::mat X; - arma::vec y; + arma::rowvec y; GenerateProblem(X, y, 1000, 100); diff --git a/src/mlpack/tests/linear_regression_test.cpp b/src/mlpack/tests/linear_regression_test.cpp index eba31c42b6..74b42a0d6f 100644 --- a/src/mlpack/tests/linear_regression_test.cpp +++ b/src/mlpack/tests/linear_regression_test.cpp @@ -30,10 +30,10 @@ BOOST_AUTO_TEST_CASE(LinearRegressionTestCase) arma::mat points(3, 10); // Responses is the "correct" value for each point in predictors and points. - arma::vec responses(10); + arma::rowvec responses(10); // The values we get back when we predict for points. - arma::vec predictions(10); + arma::rowvec predictions(10); // We'll randomly select some coefficients for the linear response. arma::vec coeffs; @@ -76,7 +76,7 @@ BOOST_AUTO_TEST_CASE(ComputeErrorTest) arma::mat predictors; predictors << 0 << 1 << 2 << 4 << 8 << 16 << arma::endr << 16 << 8 << 4 << 2 << 1 << 0 << arma::endr; - arma::vec responses = "0 2 4 3 8 8"; + arma::rowvec responses = "0 2 4 3 8 8"; // http://www.mlpack.org/trac/ticket/298 // This dataset gives a cost of 1.189500337 (as calculated in Octave). @@ -95,7 +95,7 @@ BOOST_AUTO_TEST_CASE(ComputeErrorPerfectFitTest) arma::mat predictors; predictors << 0 << 1 << 2 << 1 << 6 << 2 << arma::endr << 0 << 1 << 2 << 2 << 2 << 6 << arma::endr; - arma::vec responses = "0 2 4 3 8 8"; + arma::rowvec responses = "0 2 4 3 8 8"; LinearRegression lr(predictors, responses); @@ -111,7 +111,7 @@ BOOST_AUTO_TEST_CASE(RidgeRegressionTest) // Create empty dataset. arma::mat data; data.zeros(10, 5000); // 10-dimensional, 5000 points. - arma::vec responses; + arma::rowvec responses; responses.zeros(5000); // 5000 points. // Any lambda greater than 0 works to make the predictors covariance matrix @@ -121,7 +121,7 @@ BOOST_AUTO_TEST_CASE(RidgeRegressionTest) LinearRegression lr(data, responses, 0.0001); // Now just make sure that it predicts some more zeros. - arma::vec predictedResponses; + arma::rowvec predictedResponses; lr.Predict(data, predictedResponses); for (size_t i = 0; i < 5000; ++i) @@ -140,10 +140,10 @@ BOOST_AUTO_TEST_CASE(RidgeRegressionTestCase) arma::mat points(3, 10); // Responses is the "correct" value for each point in predictors and points. - arma::vec responses(10); + arma::rowvec responses(10); // The values we get back when we predict for points. - arma::vec predictions(10); + arma::rowvec predictions(10); // We'll randomly select some coefficients for the linear response. arma::vec coeffs; @@ -186,7 +186,7 @@ BOOST_AUTO_TEST_CASE(LinearRegressionTrainTest) { // Random dataset. arma::mat dataset = arma::randu(5, 1000); - arma::vec responses = arma::randu(1000); + arma::rowvec responses = arma::randu(1000); LinearRegression lr(dataset, responses, 0.3); LinearRegression lrTrain; diff --git a/src/mlpack/tests/load_save_test.cpp b/src/mlpack/tests/load_save_test.cpp index afd273335e..9ffce85b8b 100644 --- a/src/mlpack/tests/load_save_test.cpp +++ b/src/mlpack/tests/load_save_test.cpp @@ -1682,7 +1682,7 @@ BOOST_AUTO_TEST_CASE(SimpleARFFCategoricalTest) f << endl; f << "@attribute three STRING" << endl; f << endl; - f << "\% a comment line " << endl; + f << "% a comment line " << endl; f << endl; f << "@data" << endl; f << "hello, 1, moo" << endl; @@ -1738,15 +1738,15 @@ BOOST_AUTO_TEST_CASE(HarderARFFTest) f << endl; f << "@attribute @@@@flfl numeric" << endl; f << endl; - f << "\% comment" << endl; + f << "% comment" << endl; f << "@attribute \"hello world\" string" << endl; f << "@attribute 12345 integer" << endl; f << "@attribute real real" << endl; - f << "@attribute \"blah blah blah \t \" numeric \% comment" << endl; - f << "\% comment" << endl; + f << "@attribute \"blah blah blah \t \" numeric % comment" << endl; + f << "% comment" << endl; f << "@data" << endl; f << "1, one, 3, 4.5, 6" << endl; - f << "2, two, 4, 5.5, 7 \% comment" << endl; + f << "2, two, 4, 5.5, 7 % comment" << endl; f << "3, \"three five, six\", 5, 6.5, 8" << endl; f.close(); @@ -1803,15 +1803,15 @@ BOOST_AUTO_TEST_CASE(BadDatasetInfoARFFTest) f << endl; f << "@attribute @@@@flfl numeric" << endl; f << endl; - f << "\% comment" << endl; + f << "% comment" << endl; f << "@attribute \"hello world\" string" << endl; f << "@attribute 12345 integer" << endl; f << "@attribute real real" << endl; - f << "@attribute \"blah blah blah \t \" numeric \% comment" << endl; - f << "\% comment" << endl; + f << "@attribute \"blah blah blah \t \" numeric % comment" << endl; + f << "% comment" << endl; f << "@data" << endl; f << "1, one, 3, 4.5, 6" << endl; - f << "2, two, 4, 5.5, 7 \% comment" << endl; + f << "2, two, 4, 5.5, 7 % comment" << endl; f << "3, \"three five, six\", 5, 6.5, 8" << endl; f.close(); @@ -1992,5 +1992,4 @@ BOOST_AUTO_TEST_CASE(LoadCSVNoTransposeTXTTest) remove("test.txt"); } - BOOST_AUTO_TEST_SUITE_END(); diff --git a/src/mlpack/tests/local_coordinate_coding_test.cpp b/src/mlpack/tests/local_coordinate_coding_test.cpp index 26836d6d43..80477ef9ac 100644 --- a/src/mlpack/tests/local_coordinate_coding_test.cpp +++ b/src/mlpack/tests/local_coordinate_coding_test.cpp @@ -29,7 +29,7 @@ void VerifyCorrectness(vec beta, vec errCorr, double lambda) { const double tol = 1e-12; size_t nDims = beta.n_elem; - for(size_t j = 0; j < nDims; j++) + for (size_t j = 0; j < nDims; j++) { if (beta(j) == 0) { diff --git a/src/mlpack/tests/lsh_test.cpp b/src/mlpack/tests/lsh_test.cpp index 4f96fede9e..2a85baede2 100644 --- a/src/mlpack/tests/lsh_test.cpp +++ b/src/mlpack/tests/lsh_test.cpp @@ -335,7 +335,8 @@ BOOST_AUTO_TEST_CASE(RecallTest) arma::mat lshDistancesExp; lshTestExp.Search(qdata, k, lshNeighborsExp, lshDistancesExp); - const double recallExp = LSHSearch<>::ComputeRecall(lshNeighborsExp, groundTruth); + const double recallExp = LSHSearch<>::ComputeRecall( + lshNeighborsExp, groundTruth); // This run should have recall higher than the threshold. BOOST_REQUIRE_GE(recallExp, recallThreshExp); @@ -408,7 +409,7 @@ BOOST_AUTO_TEST_CASE(DeterministicMerge) // Query 1 is in cluster 3, which under this projection was merged with // cluster 4. Clusters 3 and 4 have points 20:39, so only neighbors among - //those should be found. + // those should be found. q = 0; BOOST_REQUIRE_GE(neighbors(j, q), N / 2); @@ -785,17 +786,19 @@ BOOST_AUTO_TEST_CASE(ParallelBichromatic) arma::mat distances; // Construct an LSH object. By default, it uses the maximum number of threads - LSHSearch<> lshTest(rdata, numProj, numTables); //default parameters + LSHSearch<> lshTest(rdata, numProj, numTables); // Default parameters. lshTest.Search(qdata, k, parallelNeighbors, distances); // Now perform same search but with 1 thread - size_t prevNumThreads = omp_get_max_threads(); // Store number of threads used. + // Store number of threads used. + size_t prevNumThreads = omp_get_max_threads(); omp_set_num_threads(1); lshTest.Search(qdata, k, sequentialNeighbors, distances); omp_set_num_threads(prevNumThreads); // Require both have same results - double recall = LSHSearch<>::ComputeRecall(sequentialNeighbors, parallelNeighbors); + double recall = LSHSearch<>::ComputeRecall( + sequentialNeighbors, parallelNeighbors); BOOST_REQUIRE_EQUAL(recall, 1); } @@ -825,13 +828,15 @@ BOOST_AUTO_TEST_CASE(ParallelMonochromatic) lshTest.Search(k, parallelNeighbors, distances); // Now perform same search but with 1 thread. - size_t prevNumThreads = omp_get_max_threads(); // Store number of threads used. + // Store number of threads used. + size_t prevNumThreads = omp_get_max_threads(); omp_set_num_threads(1); lshTest.Search(k, sequentialNeighbors, distances); omp_set_num_threads(prevNumThreads); // Require both have same results. - double recall = LSHSearch<>::ComputeRecall(sequentialNeighbors, parallelNeighbors); + double recall = LSHSearch<>::ComputeRecall( + sequentialNeighbors, parallelNeighbors); BOOST_REQUIRE_EQUAL(recall, 1); } #endif diff --git a/src/mlpack/tests/maximal_inputs_test.cpp b/src/mlpack/tests/maximal_inputs_test.cpp index c72f340082..cf48e0deaf 100644 --- a/src/mlpack/tests/maximal_inputs_test.cpp +++ b/src/mlpack/tests/maximal_inputs_test.cpp @@ -38,7 +38,7 @@ void TestResults(const arma::mat&actualResult, const arma::mat& expectResult) BOOST_REQUIRE_EQUAL(expectResult.n_rows, actualResult.n_rows); BOOST_REQUIRE_EQUAL(expectResult.n_cols, actualResult.n_cols); - for(size_t i = 0; i != expectResult.n_elem; ++i) + for (size_t i = 0; i != expectResult.n_elem; ++i) { BOOST_REQUIRE_CLOSE(expectResult[i], actualResult[i], 1e-2); } diff --git a/src/mlpack/tests/mean_shift_test.cpp b/src/mlpack/tests/mean_shift_test.cpp index 6d55b36c2d..038c424009 100644 --- a/src/mlpack/tests/mean_shift_test.cpp +++ b/src/mlpack/tests/mean_shift_test.cpp @@ -57,8 +57,8 @@ arma::mat meanShiftData(" 0.0 0.0;" // Class 1. /** * 30-point 3-class test case for Mean Shift. */ -BOOST_AUTO_TEST_CASE(MeanShiftSimpleTest) { - +BOOST_AUTO_TEST_CASE(MeanShiftSimpleTest) +{ MeanShift<> meanShift; arma::Col assignments; @@ -88,7 +88,6 @@ BOOST_AUTO_TEST_CASE(MeanShiftSimpleTest) { for (size_t i = 20; i < 30; i++) BOOST_REQUIRE_EQUAL(assignments(i), thirdClass); - } // Generate samples from four Gaussians, and make sure mean shift nearly diff --git a/src/mlpack/tests/minibatch_sgd_test.cpp b/src/mlpack/tests/minibatch_sgd_test.cpp index 22d65abf57..ee08c73d77 100644 --- a/src/mlpack/tests/minibatch_sgd_test.cpp +++ b/src/mlpack/tests/minibatch_sgd_test.cpp @@ -80,14 +80,14 @@ BOOST_AUTO_TEST_CASE(LogisticRegressionTest) GaussianDistribution g1(arma::vec("1.0 1.0 1.0"), arma::eye(3, 3)); GaussianDistribution g2(arma::vec("9.0 9.0 9.0"), arma::eye(3, 3)); - arma::mat data(3, 1000); - arma::Row responses(1000); - for (size_t i = 0; i < 500; ++i) + arma::mat data(3, 500); + arma::Row responses(500); + for (size_t i = 0; i < 250; ++i) { data.col(i) = g1.Random(); responses[i] = 0; } - for (size_t i = 500; i < 1000; ++i) + for (size_t i = 250; i < 500; ++i) { data.col(i) = g2.Random(); responses[i] = 1; @@ -96,8 +96,8 @@ BOOST_AUTO_TEST_CASE(LogisticRegressionTest) // Shuffle the dataset. arma::uvec indices = arma::shuffle(arma::linspace(0, data.n_cols - 1, data.n_cols)); - arma::mat shuffledData(3, 1000); - arma::Row shuffledResponses(1000); + arma::mat shuffledData(3, 500); + arma::Row shuffledResponses(500); for (size_t i = 0; i < data.n_cols; ++i) { shuffledData.col(i) = data.col(indices[i]); @@ -105,14 +105,14 @@ BOOST_AUTO_TEST_CASE(LogisticRegressionTest) } // Create a test set. - arma::mat testData(3, 1000); - arma::Row testResponses(1000); - for (size_t i = 0; i < 500; ++i) + arma::mat testData(3, 500); + arma::Row testResponses(500); + for (size_t i = 0; i < 250; ++i) { testData.col(i) = g1.Random(); testResponses[i] = 0; } - for (size_t i = 500; i < 1000; ++i) + for (size_t i = 250; i < 500; ++i) { testData.col(i) = g2.Random(); testResponses[i] = 1; diff --git a/src/mlpack/tests/momentum_sgd_test.cpp b/src/mlpack/tests/momentum_sgd_test.cpp index 6a574b4349..3d71168c52 100644 --- a/src/mlpack/tests/momentum_sgd_test.cpp +++ b/src/mlpack/tests/momentum_sgd_test.cpp @@ -30,7 +30,8 @@ BOOST_AUTO_TEST_CASE(MomentumSGDSpeedUpTestFunction) { SGDTestFunction f; MomentumUpdate momentumUpdate(0.7); - MomentumSGD s(f, 0.0003, 2500000, 1e-9, true, momentumUpdate); + MomentumSGD s( + f, 0.0003, 2500000, 1e-9, true, momentumUpdate); arma::mat coordinates = f.GetInitialPoint(); double result = s.Optimize(coordinates); @@ -53,8 +54,7 @@ BOOST_AUTO_TEST_CASE(MomentumSGDSpeedUpTestFunction) BOOST_REQUIRE_SMALL(coordinates1[1], 1e-7); BOOST_REQUIRE_SMALL(coordinates1[2], 1e-7); - - BOOST_REQUIRE_LE(result,result1); + BOOST_REQUIRE_LE(result, result1); } BOOST_AUTO_TEST_CASE(GeneralizedRosenbrockTest) @@ -65,7 +65,8 @@ BOOST_AUTO_TEST_CASE(GeneralizedRosenbrockTest) // Create the generalized Rosenbrock function. GeneralizedRosenbrockFunction f(i); MomentumUpdate momentumUpdate(0.4); - MomentumSGD s(f, 0.001, 0, 1e-15, true, momentumUpdate); + MomentumSGD s( + f, 0.001, 0, 1e-15, true, momentumUpdate); arma::mat coordinates = f.GetInitialPoint(); double result = s.Optimize(coordinates); diff --git a/src/mlpack/tests/nbc_test.cpp b/src/mlpack/tests/nbc_test.cpp index e7ad6e1d81..a42f8066d7 100644 --- a/src/mlpack/tests/nbc_test.cpp +++ b/src/mlpack/tests/nbc_test.cpp @@ -145,7 +145,10 @@ BOOST_AUTO_TEST_CASE(NaiveBayesClassifierIncrementalTest) for (size_t i = 0; i < testResProba.n_cols; ++i) for (size_t j = 0; j < testResProba.n_rows; ++j) - BOOST_REQUIRE_CLOSE(testResProba(j, i) + .00001, calcProbs(j, i) + .00001, 0.01); + { + BOOST_REQUIRE_CLOSE( + testResProba(j, i) + .00001, calcProbs(j, i) + .00001, 0.01); + } } /** diff --git a/src/mlpack/tests/nca_test.cpp b/src/mlpack/tests/nca_test.cpp index 19db5109b9..d7ff3547fc 100644 --- a/src/mlpack/tests/nca_test.cpp +++ b/src/mlpack/tests/nca_test.cpp @@ -320,7 +320,6 @@ BOOST_AUTO_TEST_CASE(NCALBFGSSimpleDataset) // The solution is not unique, so the best we can do is ensure the gradient // norm is close to 0. BOOST_REQUIRE_LT(arma::norm(finalGradient, 2), 1e-6); - } BOOST_AUTO_TEST_SUITE_END(); diff --git a/src/mlpack/tests/nmf_test.cpp b/src/mlpack/tests/nmf_test.cpp index 69d8a3cdc9..3ab0ef9c32 100644 --- a/src/mlpack/tests/nmf_test.cpp +++ b/src/mlpack/tests/nmf_test.cpp @@ -60,8 +60,7 @@ BOOST_AUTO_TEST_CASE(NMFAcolDistTest) const size_t r = 12; SimpleResidueTermination srt(1e-7, 10000); - AMF > - nmf(srt); + AMF > nmf(srt); nmf.Apply(v, r, w, h); mat wh = w * h; diff --git a/src/mlpack/tests/nystroem_method_test.cpp b/src/mlpack/tests/nystroem_method_test.cpp index 2c505768e1..279cad817c 100644 --- a/src/mlpack/tests/nystroem_method_test.cpp +++ b/src/mlpack/tests/nystroem_method_test.cpp @@ -93,28 +93,45 @@ BOOST_AUTO_TEST_CASE(Rank10Test) LinearKernel lk; arma::mat kernel = dataMod.t() * dataMod; - // Now use the linear kernel to get a Nystroem approximation; try this several - // times. - double normalizedFroAverage = 0.0; - for (size_t trial = 0; trial < 20; ++trial) + size_t successes = 0; + for (size_t testTrial = 0; testTrial < 5; ++testTrial) { - LinearKernel lk; - NystroemMethod nm(dataMod, lk, 10); + // Now use the linear kernel to get a Nystroem approximation; + // try this several times. + double normalizedFroAverage = 0.0; + for (size_t trial = 0; trial < 20; ++trial) + { + while (true) + { + LinearKernel lk; + NystroemMethod nm(dataMod, lk, 10); - arma::mat g; - nm.Apply(g); + arma::mat g; + nm.Apply(g); - arma::mat approximation = g * g.t(); + arma::mat approximation = g * g.t(); - // Check the normalized Frobenius norm. - const double normalizedFro = arma::norm(kernel - approximation, "fro") / - arma::norm(kernel, "fro"); + // Check the normalized Frobenius norm. + const double normalizedFro = arma::norm(kernel - approximation, "fro"); - normalizedFroAverage += normalizedFro; + // Sometimes K' is singular. Unlucky. + if (normalizedFro != normalizedFro) + continue; + + normalizedFroAverage += (normalizedFro / arma::norm(kernel, "fro")); + break; + } + } + + normalizedFroAverage /= 20; + if (std::abs(normalizedFroAverage) <= 1e-3) + { + ++successes; + break; + } } - normalizedFroAverage /= 20; - BOOST_REQUIRE_SMALL(normalizedFroAverage, 1e-3); + BOOST_REQUIRE_GE(successes, 1); } /** @@ -153,7 +170,7 @@ BOOST_AUTO_TEST_CASE(GermanTest) { // We will repeat each trial 20 times. double avgError = 0.0; - for (size_t z = 0; z < 20; ++z) + for (size_t z = 1; z < 21; ++z) { NystroemMethod > nm(dataset, gk, size_t((double((trial + 1) * 2) / 100.0) * dataset.n_cols)); diff --git a/src/mlpack/tests/octree_test.cpp b/src/mlpack/tests/octree_test.cpp index 4f3bca5e74..87460b41d0 100644 --- a/src/mlpack/tests/octree_test.cpp +++ b/src/mlpack/tests/octree_test.cpp @@ -193,15 +193,17 @@ void CheckFurthestDistances(TreeType& node) for (size_t i = 0; i < node.NumPoints(); ++i) { // Handle floating-point inaccuracies. - BOOST_REQUIRE_LE(metric::EuclideanDistance::Evaluate(node.Dataset().col(node.Point(i)), - center), node.FurthestPointDistance() * (1 + 1e-5)); + BOOST_REQUIRE_LE(metric::EuclideanDistance::Evaluate( + node.Dataset().col(node.Point(i)), center), + node.FurthestPointDistance() * (1 + 1e-5)); } // Compare descendants held in the node. for (size_t i = 0; i < node.NumDescendants(); ++i) { // Handle floating-point inaccuracies. - BOOST_REQUIRE_LE(metric::EuclideanDistance::Evaluate(node.Dataset().col(node.Descendant(i)), + BOOST_REQUIRE_LE(metric::EuclideanDistance::Evaluate( + node.Dataset().col(node.Descendant(i)), center), node.FurthestDescendantDistance() * (1 + 1e-5)); } diff --git a/src/mlpack/tests/pca_test.cpp b/src/mlpack/tests/pca_test.cpp index 926fff9311..dae8038542 100644 --- a/src/mlpack/tests/pca_test.cpp +++ b/src/mlpack/tests/pca_test.cpp @@ -238,7 +238,6 @@ BOOST_AUTO_TEST_CASE(QUICPCADimensionalityReductionTest) size_t successes = 0; for (size_t trial = 0; trial < 5; ++trial) { - PCAType exactPCA; const double varRetainedExact = exactPCA.Apply(data, 1); @@ -306,8 +305,8 @@ BOOST_AUTO_TEST_CASE(PCAScalingTest) BOOST_REQUIRE_CLOSE(std::abs(eigvec(2, 1)), 1.0, 0.2); // The third component should have the same absolute value characteristics as - // the first. - BOOST_REQUIRE_CLOSE(std::abs(eigvec(0, 0)), sqrt(2) / 2, 0.2); // 20% tolerance. + // the first (plus 20% tolerance). + BOOST_REQUIRE_CLOSE(std::abs(eigvec(0, 0)), sqrt(2) / 2, 0.2); BOOST_REQUIRE_CLOSE(std::abs(eigvec(1, 0)), sqrt(2) / 2, 0.2); BOOST_REQUIRE_SMALL(eigvec(2, 0), 0.08); // Large tolerance for noise. diff --git a/src/mlpack/tests/perceptron_test.cpp b/src/mlpack/tests/perceptron_test.cpp index fff1ba6e13..ab8adabd60 100644 --- a/src/mlpack/tests/perceptron_test.cpp +++ b/src/mlpack/tests/perceptron_test.cpp @@ -181,7 +181,6 @@ BOOST_AUTO_TEST_CASE(Random3) for (size_t i = 0; i < predictedLabels.n_cols; i++) BOOST_CHECK_EQUAL(predictedLabels(0, i), 0); - } /** diff --git a/src/mlpack/tests/prefixedoutstream_test.cpp b/src/mlpack/tests/prefixedoutstream_test.cpp index eb4a93cae4..d4d4aee41b 100644 --- a/src/mlpack/tests/prefixedoutstream_test.cpp +++ b/src/mlpack/tests/prefixedoutstream_test.cpp @@ -105,7 +105,6 @@ BOOST_AUTO_TEST_CASE(TestArmadilloPrefixedOutStream) BASH_GREEN "[INFO ] " BASH_CLEAR "hello 1.0000 1.5000 2.0000\n" BASH_GREEN "[INFO ] " BASH_CLEAR " 2.5000 3.0000 3.5000\n" BASH_GREEN "[INFO ] " BASH_CLEAR " 4.0000 4.5000 5.0000\n"); - } /** diff --git a/src/mlpack/tests/rectangle_tree_test.cpp b/src/mlpack/tests/rectangle_tree_test.cpp index 8e17d5c2ff..a880595987 100644 --- a/src/mlpack/tests/rectangle_tree_test.cpp +++ b/src/mlpack/tests/rectangle_tree_test.cpp @@ -171,7 +171,7 @@ void CheckExactContainment(const TreeType& tree) { double min = DBL_MAX; double max = -1.0 * DBL_MAX; - for(size_t j = 0; j < tree.Count(); j++) + for (size_t j = 0; j < tree.Count(); j++) { if (tree.Dataset().col(tree.Point(j))[i] < min) min = tree.Dataset().col(tree.Point(j))[i]; @@ -716,8 +716,10 @@ void CheckDiscreteHilbertValueSync(const TreeType& tree) } } else + { for (size_t i = 0; i < tree.NumChildren(); i++) CheckDiscreteHilbertValueSync(tree.Child(i)); + } } BOOST_AUTO_TEST_CASE(DiscreteHilbertValueSyncTest) @@ -726,7 +728,7 @@ BOOST_AUTO_TEST_CASE(DiscreteHilbertValueSyncTest) dataset.randu(8, 1000); // 1000 points in 8 dimensions. typedef HilbertRTree,arma::mat> TreeType; + NeighborSearchStat, arma::mat> TreeType; TreeType hilbertRTree(dataset, 20, 6, 5, 2, 0); CheckDiscreteHilbertValueSync(hilbertRTree); @@ -982,7 +984,7 @@ BOOST_AUTO_TEST_CASE(RPlusTreeOverlapTest) dataset.randu(8, 1000); // 1000 points in 8 dimensions. typedef RPlusTree,arma::mat> TreeType; + NeighborSearchStat, arma::mat> TreeType; TreeType rPlusTree(dataset, 20, 6, 5, 2, 0); CheckOverlap(rPlusTree); @@ -1100,7 +1102,7 @@ BOOST_AUTO_TEST_CASE(RPlusPlusTreeBoundTest) // Check the MinimalCoverageSweep. typedef RPlusPlusTree,arma::mat> TreeType; + NeighborSearchStat, arma::mat> TreeType; TreeType rPlusPlusTree(dataset, 20, 6, 5, 2, 0); CheckRPlusPlusTreeBound(rPlusPlusTree); diff --git a/src/mlpack/tests/rl_components_test.cpp b/src/mlpack/tests/rl_components_test.cpp new file mode 100644 index 0000000000..5ee77f3350 --- /dev/null +++ b/src/mlpack/tests/rl_components_test.cpp @@ -0,0 +1,125 @@ +/** + * @file rl_environment_test.hpp + * @author Shangtong Zhang + * + * Basic test for the components of reinforcement learning algorithms. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ + +#include + +#include +#include +#include +#include + +#include +#include "test_tools.hpp" + +using namespace mlpack; +using namespace mlpack::rl; + +BOOST_AUTO_TEST_SUITE(RLComponentsTest) + +/** + * Constructs a MountainCar instance and check if the main rountine works as + * it should be. + */ +BOOST_AUTO_TEST_CASE(SimpleMountainCarTest) +{ + const MountainCar task = MountainCar(); + + MountainCar::State state = task.InitialSample(); + MountainCar::Action action = MountainCar::Action::backward; + double reward = task.Sample(state, action); + + BOOST_REQUIRE_EQUAL(reward, -1.0); + BOOST_REQUIRE(!task.IsTerminal(state)); + BOOST_REQUIRE_EQUAL(3, MountainCar::Action::size); +} + +/** + * Constructs a CartPole instance and check if the main rountine works as + * it should be. + */ +BOOST_AUTO_TEST_CASE(SimpleCartPoleTest) +{ + const CartPole task = CartPole(); + + CartPole::State state = task.InitialSample(); + CartPole::Action action = CartPole::Action::backward; + double reward = task.Sample(state, action); + + BOOST_REQUIRE_EQUAL(reward, 1.0); + BOOST_REQUIRE(!task.IsTerminal(state)); + BOOST_REQUIRE_EQUAL(2, CartPole::Action::size); +} + +/** + * Construct a random replay instance and check if it works as + * it should be. + */ +BOOST_AUTO_TEST_CASE(RandomReplayTest) +{ + RandomReplay replay(1, 3); + MountainCar env; + MountainCar::State state = env.InitialSample(); + MountainCar::Action action = MountainCar::Action::forward; + MountainCar::State nextState; + double reward = env.Sample(state, action, nextState); + replay.Store(state, action, reward, nextState, env.IsTerminal(nextState)); + arma::mat sampledState; + arma::icolvec sampledAction; + arma::colvec sampledReward; + arma::mat sampledNextState; + arma::icolvec sampledTerminal; + + //! So far there should be only one record in the memory + replay.Sample(sampledState, sampledAction, sampledReward, sampledNextState, + sampledTerminal); + + CheckMatrices(state.Encode(), sampledState); + BOOST_REQUIRE_EQUAL(action, arma::as_scalar(sampledAction)); + BOOST_REQUIRE_CLOSE(reward, arma::as_scalar(sampledReward), 1e-5); + CheckMatrices(nextState.Encode(), sampledNextState); + BOOST_REQUIRE_EQUAL(false, arma::as_scalar(sampledTerminal)); + BOOST_REQUIRE_EQUAL(1, replay.Size()); + + //! Overwrite the memory with a nonsense record + for (size_t i = 0; i < 5; ++i) + replay.Store(nextState, action, reward, state, true); + + BOOST_REQUIRE_EQUAL(3, replay.Size()); + + //! Sample several times, the original record shouldn't appear + for (size_t i = 0; i < 30; ++i) + { + replay.Sample(sampledState, sampledAction, sampledReward, sampledNextState, + sampledTerminal); + + CheckMatrices(state.Encode(), sampledNextState); + CheckMatrices(nextState.Encode(), sampledState); + BOOST_REQUIRE_EQUAL(true, arma::as_scalar(sampledTerminal)); + } +} + +/** + * Construct a greedy policy instance and check if it works as + * it should be. + */ +BOOST_AUTO_TEST_CASE(GreedyPolicyTest) +{ + GreedyPolicy policy(1.0, 10, 0.0); + for (size_t i = 0; i < 15; ++i) + policy.Anneal(); + BOOST_REQUIRE_CLOSE(0.0, policy.Epsilon(), 1e-5); + arma::colvec actionValue = arma::randn(CartPole::Action::size); + CartPole::Action action = policy.Sample(actionValue); + BOOST_REQUIRE_CLOSE(actionValue[action], actionValue.max(), 1e-5); +} + +BOOST_AUTO_TEST_SUITE_END() diff --git a/src/mlpack/tests/rl_environment_test.cpp b/src/mlpack/tests/rl_environment_test.cpp deleted file mode 100644 index 57d80e3f34..0000000000 --- a/src/mlpack/tests/rl_environment_test.cpp +++ /dev/null @@ -1,60 +0,0 @@ -/** - * @file rl_environment_test.hpp - * @author Shangtong Zhang - * - * Basic test for the reinforcement learning task environment. - * - * mlpack is free software; you may redistribute it and/or modify it under the - * terms of the 3-clause BSD license. You should have received a copy of the - * 3-clause BSD license along with mlpack. If not, see - * http://www.opensource.org/licenses/BSD-3-Clause for more information. - */ - -#include - -#include -#include - -#include -#include "test_tools.hpp" - -using namespace mlpack; -using namespace mlpack::rl; - -BOOST_AUTO_TEST_SUITE(RLEnvironmentTest) - -/** - * Constructs a MountainCar instance and check if the main rountine works as - * it should be. - */ -BOOST_AUTO_TEST_CASE(SimpleMountainCarTest) -{ - const MountainCar task = MountainCar(); - - MountainCar::State state = task.InitialSample(); - MountainCar::Action action = MountainCar::Action::backward; - double reward = task.Sample(state, action); - - BOOST_REQUIRE_EQUAL(reward, -1.0); - BOOST_REQUIRE(!task.IsTerminal(state)); - BOOST_REQUIRE_EQUAL(3, MountainCar::Action::size); -} - -/** - * Constructs a CartPole instance and check if the main rountine works as - * it should be. - */ -BOOST_AUTO_TEST_CASE(SimpleCartPoleTest) -{ - const CartPole task = CartPole(); - - CartPole::State state = task.InitialSample(); - CartPole::Action action = CartPole::Action::backward; - double reward = task.Sample(state, action); - - BOOST_REQUIRE_EQUAL(reward, 1.0); - BOOST_REQUIRE(!task.IsTerminal(state)); - BOOST_REQUIRE_EQUAL(2, CartPole::Action::size); -} - -BOOST_AUTO_TEST_SUITE_END() diff --git a/src/mlpack/tests/sa_test.cpp b/src/mlpack/tests/sa_test.cpp index d3a36851e5..f2314b015e 100644 --- a/src/mlpack/tests/sa_test.cpp +++ b/src/mlpack/tests/sa_test.cpp @@ -61,7 +61,7 @@ BOOST_AUTO_TEST_CASE(RosenbrockTest) { RosenbrockFunction f; ExponentialSchedule schedule(1e-5); - SA //sa(f, schedule); // All default parameters. + SA // sa(f, schedule); // All default parameters. sa(f, schedule, 10000000, 1000., 1000, 100, 1e-11, 3, 20, 0.3, 0.3); arma::mat coordinates = f.GetInitialPoint(); @@ -114,7 +114,7 @@ BOOST_AUTO_TEST_CASE(RastrigrinFunctionTest) { RastrigrinFunction f; ExponentialSchedule schedule(3e-6); - SA //sa(f, schedule); + SA // sa(f, schedule); sa(f, schedule, 20000000, 100, 50, 1000, 1e-12, 2, 0.2, 0.01, 0.1); arma::mat coordinates = f.GetInitialPoint(); diff --git a/src/mlpack/tests/sdp_primal_dual_test.cpp b/src/mlpack/tests/sdp_primal_dual_test.cpp index 9d5103de80..f0f9a06f35 100644 --- a/src/mlpack/tests/sdp_primal_dual_test.cpp +++ b/src/mlpack/tests/sdp_primal_dual_test.cpp @@ -24,7 +24,6 @@ using namespace mlpack::neighbor; class UndirectedGraph { public: - UndirectedGraph() {} size_t NumVertices() const { return numVertices; } @@ -110,7 +109,6 @@ class UndirectedGraph } private: - void ComputeVertices() { numVertices = max(max(edges)) + 1; @@ -269,7 +267,7 @@ BOOST_AUTO_TEST_CASE(SmallMaxCutSdp) // the following was resulting in non-positive Z0 matrices on some // random instances. - //SolveMaxCutFeasibleSDP(sdp); + // SolveMaxCutFeasibleSDP(sdp); SolveMaxCutPositiveSDP(sdp); } @@ -537,7 +535,7 @@ BOOST_AUTO_TEST_CASE(CorrelationCoeffToySdp) BOOST_REQUIRE_CLOSE(obj, 2 * (-0.978), 1e-3); } -///** +// /** // * Maximum variance unfolding (MVU) SDP to learn the unrolled gram matrix. For // * the SDP formulation, see: // * @@ -548,66 +546,66 @@ BOOST_AUTO_TEST_CASE(CorrelationCoeffToySdp) // * @param origData origDim x numPoints // * @param numNeighbors // */ -//static inline SDP ConstructMvuSDP(const arma::mat& origData, +// static inline SDP ConstructMvuSDP(const arma::mat& origData, // size_t numNeighbors) -//{ +// { // const size_t numPoints = origData.n_cols; -// + // assert(numNeighbors <= numPoints); -// + // arma::Mat neighbors; // arma::mat distances; // KNN knn(origData); // knn.Search(numNeighbors, neighbors, distances); -// + // SDP sdp(numPoints, numNeighbors * numPoints, 1); // sdp.C().eye(numPoints, numPoints); // sdp.C() *= -1; // sdp.DenseA()[0].ones(numPoints, numPoints); // sdp.DenseB()[0] = 0; -// + // for (size_t i = 0; i < neighbors.n_cols; ++i) // { // for (size_t j = 0; j < numNeighbors; ++j) // { // // This is the index of the constraint. // const size_t index = (i * numNeighbors) + j; -// + // arma::sp_mat& aRef = sdp.SparseA()[index]; // aRef.zeros(numPoints, numPoints); -// + // // A_ij(i, i) = 1. // aRef(i, i) = 1; -// + // // A_ij(i, j) = -1. // aRef(i, neighbors(j, i)) = -1; -// + // // A_ij(j, i) = -1. // aRef(neighbors(j, i), i) = -1; -// + // // A_ij(j, j) = 1. // aRef(neighbors(j, i), neighbors(j, i)) = 1; -// + // // The constraint b_ij is the distance between these two points. // sdp.SparseB()[index] = distances(j, i); // } // } -// + // return sdp; -//} -// -///** +// } + +// /** // * Maximum variance unfolding // * // * Test doesn't work, because the constraint matrices are not linearly // * independent. // */ -//BOOST_AUTO_TEST_CASE(SmallMvuSdp) -//{ +// BOOST_AUTO_TEST_CASE(SmallMvuSdp) +// { // const size_t n = 20; -// + // arma::mat origData(3, n); -// + // // sample n random points on 3-dim unit sphere // GaussianDistribution gauss(3); // for (size_t i = 0; i < n; i++) @@ -615,14 +613,14 @@ BOOST_AUTO_TEST_CASE(CorrelationCoeffToySdp) // // how european of them // origData.col(i) = arma::normalise(gauss.Random()); // } -// + // auto sdp = ConstructMvuSDP(origData, 5); -// + // PrimalDualSolver> solver(sdp); // arma::mat X, Z; // arma::vec ysparse, ydense; // const auto p = solver.Optimize(X, ysparse, ydense, Z); // BOOST_REQUIRE(p.first); -//} +// } BOOST_AUTO_TEST_SUITE_END(); diff --git a/src/mlpack/tests/serialization_test.cpp b/src/mlpack/tests/serialization_test.cpp index 2d92f636f1..4e78b258ec 100644 --- a/src/mlpack/tests/serialization_test.cpp +++ b/src/mlpack/tests/serialization_test.cpp @@ -274,8 +274,8 @@ BOOST_AUTO_TEST_CASE(LinearRegressionTest) // Generate some random data. mat data; data.randn(15, 800); - vec responses; - responses.randn(800, 1); + rowvec responses; + responses.randn(800); LinearRegression lr(data, responses, 0.05); // Train the model. LinearRegression xmlLr, textLr, binaryLr; @@ -295,8 +295,8 @@ BOOST_AUTO_TEST_CASE(RegressionDistributionTest) // Generate some random data. mat data; data.randn(15, 800); - vec responses; - responses.randn(800, 1); + rowvec responses; + responses.randn(800); RegressionDistribution rd(data, responses); RegressionDistribution xmlRd, textRd, binaryRd; @@ -1288,7 +1288,7 @@ BOOST_AUTO_TEST_CASE(LARSTest) // Create a dataset. arma::mat X = arma::randn(75, 250); arma::vec beta = arma::randn(75, 1); - arma::vec y = trans(X) * beta; + arma::rowvec y = beta.t() * X; LARS lars(true, 0.1, 0.1); arma::vec betaOpt; @@ -1301,14 +1301,14 @@ BOOST_AUTO_TEST_CASE(LARSTest) // Train textLars. arma::mat textX = arma::randn(25, 150); arma::vec textBeta = arma::randn(25, 1); - arma::vec textY = trans(textX) * textBeta; + arma::rowvec textY = textBeta.t() * textX; arma::vec textBetaOpt; textLars.Train(textX, textY, textBetaOpt); SerializeObjectAll(lars, xmlLars, binaryLars, textLars); // Now, check that predictions are the same. - arma::vec pred, xmlPred, textPred, binaryPred; + arma::rowvec pred, xmlPred, textPred, binaryPred; lars.Predict(X, pred); xmlLars.Predict(X, xmlPred); textLars.Predict(X, textPred); diff --git a/src/mlpack/tests/sgdr_test.cpp b/src/mlpack/tests/sgdr_test.cpp new file mode 100644 index 0000000000..f8190e05dd --- /dev/null +++ b/src/mlpack/tests/sgdr_test.cpp @@ -0,0 +1,131 @@ +/** + * @file sgdr_test.cpp + * @author Marcus Edel + * + * Test file for SGDR. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#include +#include +#include +#include + +#include +#include "test_tools.hpp" + +using namespace std; +using namespace arma; +using namespace mlpack; +using namespace mlpack::optimization; + +using namespace mlpack::distribution; +using namespace mlpack::regression; + +BOOST_AUTO_TEST_SUITE(SGDRTest); + +/* + * Test that the step size resets after a specified number of epochs. + */ +BOOST_AUTO_TEST_CASE(CyclicalResetTest) +{ + const double stepSize = 0.5; + arma::mat iterate; + + // Now run cyclical decay policy with a couple of multiplicators and initial + // restarts. + for (size_t restart = 5; restart < 100; restart += 10) + { + for (size_t mult = 2; mult < 5; ++mult) + { + double epochStepSize = stepSize; + + CyclicalDecay cyclicalDecay(restart, double(mult), stepSize, 10, 1000); + + // Create all restart epochs. + arma::Col nextRestart(1000 / 10 / mult); + nextRestart(0) = restart; + for (size_t j = 1; j < nextRestart.n_elem; ++j) + nextRestart(j) = nextRestart(j - 1) * mult; + + for (size_t i = 0; i < 1000; ++i) + { + cyclicalDecay.Update(iterate, epochStepSize, iterate); + if (i <= restart || arma::accu(arma::find(nextRestart == i)) > 0) + { + BOOST_CHECK_EQUAL(epochStepSize, stepSize); + } + } + } + } +} + +/** + * Run SGDR on logistic regression and make sure the results are acceptable. + */ +BOOST_AUTO_TEST_CASE(LogisticRegressionTest) +{ + // Generate a two-Gaussian dataset. + GaussianDistribution g1(arma::vec("1.0 1.0 1.0"), arma::eye(3, 3)); + GaussianDistribution g2(arma::vec("9.0 9.0 9.0"), arma::eye(3, 3)); + + arma::mat data(3, 1000); + arma::Row responses(1000); + for (size_t i = 0; i < 500; ++i) + { + data.col(i) = g1.Random(); + responses[i] = 0; + } + for (size_t i = 500; i < 1000; ++i) + { + data.col(i) = g2.Random(); + responses[i] = 1; + } + + // Shuffle the dataset. + arma::uvec indices = arma::shuffle(arma::linspace(0, + data.n_cols - 1, data.n_cols)); + arma::mat shuffledData(3, 1000); + arma::Row shuffledResponses(1000); + for (size_t i = 0; i < data.n_cols; ++i) + { + shuffledData.col(i) = data.col(indices[i]); + shuffledResponses[i] = responses[indices[i]]; + } + + // Create a test set. + arma::mat testData(3, 1000); + arma::Row testResponses(1000); + for (size_t i = 0; i < 500; ++i) + { + testData.col(i) = g1.Random(); + testResponses[i] = 0; + } + for (size_t i = 500; i < 1000; ++i) + { + testData.col(i) = g2.Random(); + testResponses[i] = 1; + } + + // Now run SGDR with a couple of batch sizes. + for (size_t batchSize = 5; batchSize < 50; batchSize += 5) + { + LogisticRegression<> lr(shuffledData.n_rows, 0.5); + + LogisticRegressionFunction<> lrf(shuffledData, shuffledResponses, 0.5); + SGDR > sgdr(lrf, 50, 2.0, batchSize); + lr.Train(sgdr); + + // Ensure that the error is close to zero. + const double acc = lr.ComputeAccuracy(data, responses); + BOOST_REQUIRE_CLOSE(acc, 100.0, 0.3); // 0.3% error tolerance. + + const double testAcc = lr.ComputeAccuracy(testData, testResponses); + BOOST_REQUIRE_CLOSE(testAcc, 100.0, 0.6); // 0.6% error tolerance. + } +} + +BOOST_AUTO_TEST_SUITE_END(); diff --git a/src/mlpack/tests/smorms3_test.cpp b/src/mlpack/tests/smorms3_test.cpp index 1d62556312..eb4aa48117 100644 --- a/src/mlpack/tests/smorms3_test.cpp +++ b/src/mlpack/tests/smorms3_test.cpp @@ -106,4 +106,4 @@ BOOST_AUTO_TEST_CASE(SMORMS3LogisticRegressionTest) BOOST_REQUIRE_CLOSE(testAcc, 100.0, 0.6); // 0.6% error tolerance. } -BOOST_AUTO_TEST_SUITE_END(); \ No newline at end of file +BOOST_AUTO_TEST_SUITE_END(); diff --git a/src/mlpack/tests/snapshot_ensembles.cpp b/src/mlpack/tests/snapshot_ensembles.cpp new file mode 100644 index 0000000000..04a3650cd1 --- /dev/null +++ b/src/mlpack/tests/snapshot_ensembles.cpp @@ -0,0 +1,135 @@ +/** + * @file snapshot_ensembles.cpp + * @author Marcus Edel + * + * Test file for SGDR with snapshot ensembles. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#include +#include +#include +#include + +#include +#include "test_tools.hpp" + +using namespace std; +using namespace arma; +using namespace mlpack; +using namespace mlpack::optimization; + +using namespace mlpack::distribution; +using namespace mlpack::regression; + +BOOST_AUTO_TEST_SUITE(SnapshotEnsemblesTest); + +/* + * Test that the step size resets after a specified number of epochs. + */ +BOOST_AUTO_TEST_CASE(SnapshotEnsemblesResetTest) +{ + const double stepSize = 0.5; + arma::mat iterate; + + // Now run cyclical decay policy with a couple of multiplicators and initial + // restarts. + for (size_t restart = 5; restart < 100; restart += 10) + { + for (size_t mult = 2; mult < 5; ++mult) + { + double epochStepSize = stepSize; + + SnapshotEnsembles snapshotEnsembles(restart, double(mult), stepSize, + 10, 1000, 1000, 2); + + // Create all restart epochs. + arma::Col nextRestart(1000 / 10 / mult); + nextRestart(0) = restart; + for (size_t j = 1; j < nextRestart.n_elem; ++j) + nextRestart(j) = nextRestart(j - 1) * mult; + + for (size_t i = 0; i < 1000; ++i) + { + snapshotEnsembles.Update(iterate, epochStepSize, iterate); + if (i <= restart || arma::accu(arma::find(nextRestart == i)) > 0) + { + BOOST_CHECK_EQUAL(epochStepSize, stepSize); + } + } + + BOOST_CHECK_EQUAL(snapshotEnsembles.Snapshots().size(), 2); + } + } +} + +/** + * Run SGDR with snapshot ensembles on logistic regression and make sure the + * results are acceptable. + */ +BOOST_AUTO_TEST_CASE(LogisticRegressionTest) +{ + // Generate a two-Gaussian dataset. + GaussianDistribution g1(arma::vec("1.0 1.0 1.0"), arma::eye(3, 3)); + GaussianDistribution g2(arma::vec("9.0 9.0 9.0"), arma::eye(3, 3)); + + arma::mat data(3, 1000); + arma::Row responses(1000); + for (size_t i = 0; i < 500; ++i) + { + data.col(i) = g1.Random(); + responses[i] = 0; + } + for (size_t i = 500; i < 1000; ++i) + { + data.col(i) = g2.Random(); + responses[i] = 1; + } + + // Shuffle the dataset. + arma::uvec indices = arma::shuffle(arma::linspace(0, + data.n_cols - 1, data.n_cols)); + arma::mat shuffledData(3, 1000); + arma::Row shuffledResponses(1000); + for (size_t i = 0; i < data.n_cols; ++i) + { + shuffledData.col(i) = data.col(indices[i]); + shuffledResponses[i] = responses[indices[i]]; + } + + // Create a test set. + arma::mat testData(3, 1000); + arma::Row testResponses(1000); + for (size_t i = 0; i < 500; ++i) + { + testData.col(i) = g1.Random(); + testResponses[i] = 0; + } + for (size_t i = 500; i < 1000; ++i) + { + testData.col(i) = g2.Random(); + testResponses[i] = 1; + } + + // Now run SGDR with snapshot ensembles on a couple of batch sizes. + for (size_t batchSize = 5; batchSize < 50; batchSize += 5) + { + LogisticRegression<> lr(shuffledData.n_rows, 0.5); + + LogisticRegressionFunction<> lrf(shuffledData, shuffledResponses, 0.5); + SnapshotSGDR > sgdr(lrf, 50, 2.0, batchSize); + lr.Train(sgdr); + + // Ensure that the error is close to zero. + const double acc = lr.ComputeAccuracy(data, responses); + BOOST_REQUIRE_CLOSE(acc, 100.0, 0.3); // 0.3% error tolerance. + + const double testAcc = lr.ComputeAccuracy(testData, testResponses); + BOOST_REQUIRE_CLOSE(testAcc, 100.0, 0.6); // 0.6% error tolerance. + } +} + +BOOST_AUTO_TEST_SUITE_END(); diff --git a/src/mlpack/tests/softmax_regression_test.cpp b/src/mlpack/tests/softmax_regression_test.cpp index d5741843b3..63b6f1129f 100644 --- a/src/mlpack/tests/softmax_regression_test.cpp +++ b/src/mlpack/tests/softmax_regression_test.cpp @@ -35,14 +35,14 @@ BOOST_AUTO_TEST_CASE(SoftmaxRegressionFunctionEvaluate) // Create random class labels. arma::Row labels(points); - for(size_t i = 0; i < points; i++) + for (size_t i = 0; i < points; i++) labels(i) = math::RandInt(0, numClasses); // Create a SoftmaxRegressionFunction. Regularization term ignored. SoftmaxRegressionFunction srf(data, labels, numClasses, 0); // Run a number of trials. - for(size_t i = 0; i < trials; i++) + for (size_t i = 0; i < trials; i++) { // Create a random set of parameters. arma::mat parameters; @@ -51,7 +51,7 @@ BOOST_AUTO_TEST_CASE(SoftmaxRegressionFunctionEvaluate) double logLikelihood = 0; // Compute error for each training example. - for(size_t j = 0; j < points; j++) + for (size_t j = 0; j < points; j++) { arma::mat hypothesis, probabilities; @@ -80,7 +80,7 @@ BOOST_AUTO_TEST_CASE(SoftmaxRegressionFunctionRegularizationEvaluate) // Create random class labels. arma::Row labels(points); - for(size_t i = 0; i < points; i++) + for (size_t i = 0; i < points; i++) labels(i) = math::RandInt(0, numClasses); // 3 objects for comparing regularization costs. @@ -121,7 +121,7 @@ BOOST_AUTO_TEST_CASE(SoftmaxRegressionFunctionGradient) // Create random class labels. arma::Row labels(points); - for(size_t i = 0; i < points; i++) + for (size_t i = 0; i < points; i++) labels(i) = math::RandInt(0, numClasses); // 2 objects for 2 terms in the cost function. Each term contributes towards @@ -489,7 +489,7 @@ BOOST_AUTO_TEST_CASE(SoftmaxRegressionClassifySinglePointTest) sr.Classify(data, labels); - for(size_t i = 0; i < data.n_cols; ++i) + for (size_t i = 0; i < data.n_cols; ++i) { BOOST_REQUIRE_EQUAL(sr.Classify(data.col(i)), labels(i)); } @@ -575,7 +575,7 @@ BOOST_AUTO_TEST_CASE(SoftmaxRegressionComputeProbabilitiesTest) BOOST_REQUIRE_EQUAL(probabilities.n_cols, data.n_cols); BOOST_REQUIRE_EQUAL(probabilities.n_rows, sr.NumClasses()); - for(size_t i = 0; i < data.n_cols; ++i) + for (size_t i = 0; i < data.n_cols; ++i) { BOOST_REQUIRE_CLOSE(arma::sum(probabilities.col(i)), 1.0, 1e-5); } @@ -664,7 +664,7 @@ BOOST_AUTO_TEST_CASE(SoftmaxRegressionComputeProbabilitiesAndLabelsTest) BOOST_REQUIRE_EQUAL(probabilities.n_cols, data.n_cols); BOOST_REQUIRE_EQUAL(probabilities.n_rows, sr.NumClasses()); - for(size_t i = 0; i < data.n_cols; ++i) + for (size_t i = 0; i < data.n_cols; ++i) { BOOST_REQUIRE_CLOSE(arma::sum(probabilities.col(i)), 1.0, 1e-5); BOOST_REQUIRE_EQUAL(testLabels(i), labels(i)); diff --git a/src/mlpack/tests/sparse_autoencoder_test.cpp b/src/mlpack/tests/sparse_autoencoder_test.cpp index 463b40d65c..d9e41983cd 100644 --- a/src/mlpack/tests/sparse_autoencoder_test.cpp +++ b/src/mlpack/tests/sparse_autoencoder_test.cpp @@ -91,7 +91,7 @@ BOOST_AUTO_TEST_CASE(SparseAutoencoderFunctionRandomEvaluate) (1 + arma::exp(-(parameters.submat(0, 0, l1 - 1, l2 - 1) * data.col(j) + parameters.submat(0, l2, l1 - 1, l2)))); outputLayer = 1.0 / - (1 + arma::exp(-(parameters.submat(l1, 0, l3 - 1,l2 - 1).t() + (1 + arma::exp(-(parameters.submat(l1, 0, l3 - 1, l2 - 1).t() * hiddenLayer + parameters.submat(l3, 0, l3, l2 - 1).t()))); diff = outputLayer - data.col(j); @@ -167,7 +167,7 @@ BOOST_AUTO_TEST_CASE(SparseAutoencoderFunctionKLDivergenceEvaluate) SparseAutoencoderFunction safBigDiv(data, vSize, hSize, 0, 20, rho); // Run a number of trials. - for(size_t i = 0; i < trials; i++) + for (size_t i = 0; i < trials; i++) { // Create a random set of parameters. arma::mat parameters; diff --git a/src/mlpack/tests/sparse_coding_test.cpp b/src/mlpack/tests/sparse_coding_test.cpp index 93f97024d2..7018a9e4a4 100644 --- a/src/mlpack/tests/sparse_coding_test.cpp +++ b/src/mlpack/tests/sparse_coding_test.cpp @@ -30,7 +30,7 @@ void SCVerifyCorrectness(vec beta, vec errCorr, double lambda) { const double tol = 1e-12; size_t nDims = beta.n_elem; - for(size_t j = 0; j < nDims; j++) + for (size_t j = 0; j < nDims; j++) { if (beta(j) == 0) { @@ -99,7 +99,7 @@ BOOST_AUTO_TEST_CASE(SparseCodingTestCodingStepElasticNet) mat D = sc.Dictionary(); - for(uword i = 0; i < nPoints; ++i) + for (uword i = 0; i < nPoints; ++i) { vec errCorr = (trans(D) * D + lambda2 * eye(nAtoms, nAtoms)) * Z.unsafe_col(i) diff --git a/src/mlpack/tests/svd_batch_test.cpp b/src/mlpack/tests/svd_batch_test.cpp index fee96ec7d5..dd9e66dc26 100644 --- a/src/mlpack/tests/svd_batch_test.cpp +++ b/src/mlpack/tests/svd_batch_test.cpp @@ -91,7 +91,7 @@ BOOST_AUTO_TEST_CASE(SVDBatchMomentumTest) // Create the initial matrices. SpecificRandomInitialization sri(cleanedData.n_rows, 2, cleanedData.n_cols); - ValidationRMSETermination vrt(cleanedData, 2000); + ValidationRMSETermination vrt(cleanedData, 100); AMF, SpecificRandomInitialization, SVDBatchLearning> amf1(vrt, sri, SVDBatchLearning(0.0009, 0, 0, 0)); diff --git a/src/mlpack/tests/svd_incremental_test.cpp b/src/mlpack/tests/svd_incremental_test.cpp index 5b47e4449e..91ac784bdb 100644 --- a/src/mlpack/tests/svd_incremental_test.cpp +++ b/src/mlpack/tests/svd_incremental_test.cpp @@ -38,7 +38,7 @@ BOOST_AUTO_TEST_CASE(SVDIncompleteIncrementalConvergenceTest) RandomInitialization, SVDIncompleteIncrementalLearning> amf(iit, RandomInitialization(), svd); - mat m1,m2; + mat m1, m2; amf.Apply(data, 2, m1, m2); BOOST_REQUIRE_NE(amf.TerminationPolicy().Iteration(), @@ -61,7 +61,7 @@ BOOST_AUTO_TEST_CASE(SVDCompleteIncrementalConvergenceTest) SVDCompleteIncrementalLearning > amf(iit, RandomInitialization(), svd); - mat m1,m2; + mat m1, m2; amf.Apply(data, 2, m1, m2); BOOST_REQUIRE_NE(amf.TerminationPolicy().Iteration(), diff --git a/src/mlpack/tests/test_tools.hpp b/src/mlpack/tests/test_tools.hpp index 53434538ea..b7875b84a9 100644 --- a/src/mlpack/tests/test_tools.hpp +++ b/src/mlpack/tests/test_tools.hpp @@ -17,11 +17,12 @@ // Require the approximation L to be within a relative error of E respect to the // actual value R. -#define REQUIRE_RELATIVE_ERR( L, R, E ) \ - BOOST_REQUIRE_LE( std::abs((R) - (L)), (E) * std::abs(R)) +#define REQUIRE_RELATIVE_ERR(L, R, E) \ + BOOST_REQUIRE_LE(std::abs((R) - (L)), (E) * std::abs(R)) // Check the values of two matrices. -inline void CheckMatrices(const arma::mat& a, const arma::mat& b, +inline void CheckMatrices(const arma::mat& a, + const arma::mat& b, double tolerance = 1e-5) { BOOST_REQUIRE_EQUAL(a.n_rows, b.n_rows); @@ -37,7 +38,8 @@ inline void CheckMatrices(const arma::mat& a, const arma::mat& b, } // Check the values of two unsigned matrices. -inline void CheckMatrices(const arma::Mat& a, const arma::Mat& b) +inline void CheckMatrices(const arma::Mat& a, + const arma::Mat& b) { BOOST_REQUIRE_EQUAL(a.n_rows, b.n_rows); BOOST_REQUIRE_EQUAL(a.n_cols, b.n_cols); diff --git a/src/mlpack/tests/tree_test.cpp b/src/mlpack/tests/tree_test.cpp index 449ebebc05..b794fc94c8 100644 --- a/src/mlpack/tests/tree_test.cpp +++ b/src/mlpack/tests/tree_test.cpp @@ -1573,7 +1573,6 @@ void CheckRPTreeSplit(const TreeType& tree) BOOST_REQUIRE_LE(maxDist, dist * (1.0 + 10.0 * std::numeric_limits::epsilon())); } - } CheckRPTreeSplit(*tree.Left()); @@ -1648,9 +1647,9 @@ BOOST_AUTO_TEST_CASE(BallTreeTest) BOOST_REQUIRE_EQUAL(root.NumDescendants(), size); // Check the forward and backward mappings for correctness. - for(size_t i = 0; i < size; i++) + for (size_t i = 0; i < size; i++) { - for(size_t j = 0; j < dimensions; j++) + for (size_t j = 0; j < dimensions; j++) { BOOST_REQUIRE_EQUAL(treeset(j, i), dataset(j, newToOld[i])); BOOST_REQUIRE_EQUAL(treeset(j, oldToNew[i]), dataset(j, i)); diff --git a/src/mlpack/tests/ub_tree_test.cpp b/src/mlpack/tests/ub_tree_test.cpp index 8b75e2d7e3..cbbf2241d1 100644 --- a/src/mlpack/tests/ub_tree_test.cpp +++ b/src/mlpack/tests/ub_tree_test.cpp @@ -47,7 +47,6 @@ BOOST_AUTO_TEST_CASE(AddressTest) for (size_t k = 0; k < dataset.n_rows; k++) BOOST_REQUIRE_CLOSE(dataset(k, i), point[k], 1e-13); } - } template diff --git a/src/mlpack/tests/vantage_point_tree_test.cpp b/src/mlpack/tests/vantage_point_tree_test.cpp index 2223341f21..5ff0abd197 100644 --- a/src/mlpack/tests/vantage_point_tree_test.cpp +++ b/src/mlpack/tests/vantage_point_tree_test.cpp @@ -220,9 +220,9 @@ BOOST_AUTO_TEST_CASE(VPTreeTest) BOOST_REQUIRE_EQUAL(root.NumDescendants(), size); // Check the forward and backward mappings for correctness. - for(size_t i = 0; i < size; i++) + for (size_t i = 0; i < size; i++) { - for(size_t j = 0; j < dimensions; j++) + for (size_t j = 0; j < dimensions; j++) { BOOST_REQUIRE_EQUAL(treeset(j, i), dataset(j, newToOld[i])); BOOST_REQUIRE_EQUAL(treeset(j, oldToNew[i]), dataset(j, i));