diff --git a/.travis.yml b/.travis.yml index e1d1be604c..85e8c80a21 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,10 +9,10 @@ env: before_install: - sudo systemctl stop apt-daily.service apt-daily.timer - sudo systemctl disable apt-daily.service apt-daily.timer - - sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test - - sudo apt-get update -qq + - sudo rm -f /var/lib/dpkg/lock && sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test + - sudo rm -f /var/lib/dpkg/lock && sudo apt-get update -qq - printenv - - sudo apt-get install -qq libopenblas-dev liblapack-dev g++ libboost-all-dev + - sudo rm -f /var/lib/dpkg/lock && sudo apt-get install -qq libopenblas-dev liblapack-dev g++ libboost-all-dev - sudo wget https://launchpadlibrarian.net/260609315/fix-std-vector-load.diff -O /usr/include/fix-std-vector-load.diff - cd /usr/include && sudo patch -p1 < fix-std-vector-load.diff && sudo rm fix-std-vector-load.diff && cd - - sudo pip install cython setuptools numpy pandas diff --git a/CMakeLists.txt b/CMakeLists.txt index ef9f9298ec..f2d960c61c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -126,7 +126,7 @@ if(BUILD_WITH_COVERAGE) endif() # For clock_gettime(). -if (UNIX AND NOT APPLE) +if (UNIX AND NOT APPLE AND NOT HAIKU) set(MLPACK_LIBRARIES ${MLPACK_LIBRARIES} "rt") endif () @@ -528,11 +528,12 @@ if (PKG_CONFIG_FOUND) # So, we have to parse our list of library directories, libraries, and include # directories in order to get the correct line to give to pkg-config. # Next, adapt the list of include directories. + list(REMOVE_DUPLICATES MLPACK_INCLUDE_DIRS) foreach (incldir ${MLPACK_INCLUDE_DIRS}) # Filter out some obviously unnecessary directories. if (NOT "${incldir}" STREQUAL "/usr/include") - set(MLPACK_INCLUDE_DIRS_STRING "${MLPACK_INCLUDE_DIRS_STRING} - -I${incldir}") + set(MLPACK_INCLUDE_DIRS_STRING + "${MLPACK_INCLUDE_DIRS_STRING} -I${incldir}") endif () endforeach () # Add the install directory too. diff --git a/src/mlpack/bindings/cli/cli_option.hpp b/src/mlpack/bindings/cli/cli_option.hpp index fd58fdde7a..47233b29ec 100644 --- a/src/mlpack/bindings/cli/cli_option.hpp +++ b/src/mlpack/bindings/cli/cli_option.hpp @@ -60,6 +60,7 @@ class CLIOption * @param input Whether or not the option is an input option. * @param noTranspose If the parameter is a matrix and this is true, then the * matrix will not be transposed on loading. + * @param testName Is not used and added for compatibility reasons. */ CLIOption(const N defaultValue, const std::string& identifier, @@ -68,7 +69,8 @@ class CLIOption const std::string& cppName, const bool required = false, const bool input = true, - const bool noTranspose = false) + const bool noTranspose = false, + const std::string& /*testName*/ = "") { // Create the ParamData object to give to CLI. util::ParamData data; diff --git a/src/mlpack/bindings/python/py_option.hpp b/src/mlpack/bindings/python/py_option.hpp index c934b2bafa..60eaf3a1c4 100644 --- a/src/mlpack/bindings/python/py_option.hpp +++ b/src/mlpack/bindings/python/py_option.hpp @@ -33,7 +33,8 @@ class PyOption public: /** * Construct a PyOption object. When constructed, it will register itself - * with CLI. + * with CLI. The testName parameter is not used and added for compatibility + * reasons. */ PyOption(const T defaultValue, const std::string& identifier, @@ -42,7 +43,8 @@ class PyOption const std::string& cppName, const bool required = false, const bool input = true, - const bool noTranspose = false) + const bool noTranspose = false, + const std::string& /*testName*/ = "") { // Create the ParamData object to give to CLI. util::ParamData data; diff --git a/src/mlpack/bindings/python/tests/test_python_binding_main.cpp b/src/mlpack/bindings/python/tests/test_python_binding_main.cpp index fd57d19cf7..8be2f4947a 100644 --- a/src/mlpack/bindings/python/tests/test_python_binding_main.cpp +++ b/src/mlpack/bindings/python/tests/test_python_binding_main.cpp @@ -52,7 +52,7 @@ PARAM_MODEL_OUT(GaussianKernel, "model_out", "Output model, with twice the " "bandwidth.", ""); PARAM_DOUBLE_OUT("model_bw_out", "The bandwidth of the model."); -void mlpackMain() +static void mlpackMain() { const string s = CLI::GetParam("string_in"); const int i = CLI::GetParam("int_in"); diff --git a/src/mlpack/bindings/tests/test_option.hpp b/src/mlpack/bindings/tests/test_option.hpp index 5e5e7e8d58..336d9b93db 100644 --- a/src/mlpack/bindings/tests/test_option.hpp +++ b/src/mlpack/bindings/tests/test_option.hpp @@ -53,6 +53,8 @@ class TestOption * @param input Whether or not the option is an input option. * @param noTranspose If the parameter is a matrix and this is true, then the * matrix will not be transposed on loading. + * @param testName Name of the test (used for identifiying which binding test + * this option belongs to) */ TestOption(const N defaultValue, const std::string& identifier, @@ -61,7 +63,8 @@ class TestOption const std::string& cppName, const bool required = false, const bool input = true, - const bool noTranspose = false) + const bool noTranspose = false, + const std::string& testName = "") { // Create the ParamData object to give to CLI. util::ParamData data; @@ -81,7 +84,7 @@ class TestOption const std::string tname = data.tname; - CLI::RestoreSettings(programName, false); + CLI::RestoreSettings(testName, false); // Set some function pointers that we need. CLI::GetSingleton().functionMap[tname]["GetPrintableParam"] = @@ -94,7 +97,7 @@ class TestOption if (!input) CLI::SetPassed(identifier); - CLI::StoreSettings(programName); + CLI::StoreSettings(testName); CLI::ClearSettings(); } }; diff --git a/src/mlpack/core/optimizers/adam/CMakeLists.txt b/src/mlpack/core/optimizers/adam/CMakeLists.txt index 1377bbb0e1..4dfeb41e25 100644 --- a/src/mlpack/core/optimizers/adam/CMakeLists.txt +++ b/src/mlpack/core/optimizers/adam/CMakeLists.txt @@ -3,6 +3,8 @@ set(SOURCES adam_impl.hpp adam_update.hpp adamax_update.hpp + amsgrad_update.hpp + nadam_update.hpp ) set(DIR_SRCS) diff --git a/src/mlpack/core/optimizers/adam/adam.hpp b/src/mlpack/core/optimizers/adam/adam.hpp index 3b109660f5..9a2c1d732a 100644 --- a/src/mlpack/core/optimizers/adam/adam.hpp +++ b/src/mlpack/core/optimizers/adam/adam.hpp @@ -4,11 +4,13 @@ * @author Vasanth Kalingeri * @author Marcus Edel * @author Vivek Pal + * @author Sourabh Varshney * - * Adam and AdaMax optimizer. Adam is an an algorithm for first-order gradient- - * -based optimization of stochastic objective functions, based on adaptive - * estimates of lower-order moments. AdaMax is simply a variant of Adam based - * on the infinity norm. + * Adam, AdaMax, AMSGrad and Nadam optimizers. Adam is an an algorithm for + * first-order gradient-based optimization of stochastic objective functions, + * based on adaptive estimates of lower-order moments. AdaMax is simply a + * variant of Adam based on the infinity norm. AMSGrad is another variant of + * Adam with guaranteed convergence. * * mlpack is free software; you may redistribute it and/or modify it under the * terms of the 3-clause BSD license. You should have received a copy of the @@ -23,6 +25,8 @@ #include #include "adam_update.hpp" #include "adamax_update.hpp" +#include "amsgrad_update.hpp" +#include "nadam_update.hpp" namespace mlpack { namespace optimization { @@ -31,7 +35,8 @@ namespace optimization { * Adam is an optimizer that computes individual adaptive learning rates for * different parameters from estimates of first and second moments of the * gradients. AdaMax is a variant of Adam based on the infinity norm as given - * in the section 7 of the following paper. + * in the section 7 of the following paper. Nadam is an optimizer that + * combines the Adam and NAG. * * For more information, see the following. * @@ -43,11 +48,16 @@ namespace optimization { * year = {2014}, * url = {http://arxiv.org/abs/1412.6980} * } + * @article{ + * title = {On the convergence of Adam and beyond}, + * url = {https://openreview.net/pdf?id=ryQu7f-RZ} + * year = {2018} + * } * @endcode * - * - * For Adam and AdaMax to work, a DecomposableFunctionType template parameter - * is required. This class must implement the following function: + * For Adam, AdaMax, AMSGrad and Nadam to work, a DecomposableFunctionType + * template parameter is required. This class must implement the following + * function: * * size_t NumFunctions(); * double Evaluate(const arma::mat& coordinates, @@ -166,6 +176,10 @@ using Adam = AdamType; using AdaMax = AdamType; +using AMSGrad = AdamType; + +using Nadam = AdamType; + } // namespace optimization } // namespace mlpack diff --git a/src/mlpack/core/optimizers/adam/adam_impl.hpp b/src/mlpack/core/optimizers/adam/adam_impl.hpp index 326d038d15..f3f86ac02e 100644 --- a/src/mlpack/core/optimizers/adam/adam_impl.hpp +++ b/src/mlpack/core/optimizers/adam/adam_impl.hpp @@ -5,7 +5,7 @@ * @author Marcus Edel * @author Vivek Pal * - * Implementation of the Adam and AdaMax optimizer. + * Implementation of the Adam, AdaMax, AMSGrad and Nadam optimizer. * * mlpack is free software; you may redistribute it and/or modify it under the * terms of the 3-clause BSD license. You should have received a copy of the diff --git a/src/mlpack/core/optimizers/adam/amsgrad_update.hpp b/src/mlpack/core/optimizers/adam/amsgrad_update.hpp new file mode 100644 index 0000000000..f2a126f186 --- /dev/null +++ b/src/mlpack/core/optimizers/adam/amsgrad_update.hpp @@ -0,0 +1,145 @@ +/** + * @file amsgrad_update.hpp + * @author Haritha Nair + * + * Implementation of AMSGrad optimizer. AMSGrad is an exponential moving average + * optimizer that dynamically adapts over time with guaranteed convergence. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_CORE_OPTIMIZERS_AMS_GRAD_AMS_GRAD_UPDATE_HPP +#define MLPACK_CORE_OPTIMIZERS_AMS_GRAD_AMS_GRAD_UPDATE_HPP + +#include + +namespace mlpack { +namespace optimization { + +/** + * AMSGrad is an exponential moving average variant which along with having + * benefits of optimizers like Adam and RMSProp, also guarantees convergence. + * Unlike Adam, it uses maximum of past squared gradients rather than their + * exponential average for updation. + * + * For more information, see the following. + * + * @code + * @article{ + * title = {On the convergence of Adam and beyond}, + * url = {https://openreview.net/pdf?id=ryQu7f-RZ} + * year = {2018} + * } + * @endcode + */ +class AMSGradUpdate +{ + public: + /** + * Construct the AMSGrad update policy with the given parameters. + * + * @param epsilon The epsilon value used to initialise the squared gradient + * parameter. + * @param beta1 The smoothing parameter. + * @param beta2 The second moment coefficient. + */ + AMSGradUpdate(const double epsilon = 1e-8, + const double beta1 = 0.9, + const double beta2 = 0.999) : + epsilon(epsilon), + beta1(beta1), + beta2(beta2), + iteration(0) + { + // Nothing to do. + } + + /** + * The Initialize method is called by SGD Optimizer method before the start of + * the iteration update process. + * + * @param rows Number of rows in the gradient matrix. + * @param cols Number of columns in the gradient matrix. + */ + void Initialize(const size_t rows, const size_t cols) + { + m = arma::zeros(rows, cols); + v = arma::zeros(rows, cols); + vImproved = arma::zeros(rows, cols); + } + + /** + * Update step for AMSGrad. + * + * @param iterate Parameters that minimize the function. + * @param stepSize Step size to be used for the given iteration. + * @param gradient The gradient matrix. + */ + void Update(arma::mat& iterate, + const double stepSize, + const arma::mat& gradient) + { + // Increment the iteration counter variable. + ++iteration; + + // And update the iterate. + m *= beta1; + m += (1 - beta1) * gradient; + + v *= beta2; + v += (1 - beta2) * (gradient % gradient); + + const double biasCorrection1 = 1.0 - std::pow(beta1, iteration); + const double biasCorrection2 = 1.0 - std::pow(beta2, iteration); + + // Element wise maximum of past and present squared gradients. + vImproved = arma::max(vImproved, v); + + iterate -= (stepSize * std::sqrt(biasCorrection2) / biasCorrection1) * + m / (arma::sqrt(vImproved) + epsilon); + } + + //! Get the value used to initialise the squared gradient parameter. + double Epsilon() const { return epsilon; } + //! Modify the value used to initialise the squared gradient parameter. + double& Epsilon() { return epsilon; } + + //! Get the smoothing parameter. + double Beta1() const { return beta1; } + //! Modify the smoothing parameter. + double& Beta1() { return beta1; } + + //! Get the second moment coefficient. + double Beta2() const { return beta2; } + //! Modify the second moment coefficient. + double& Beta2() { return beta2; } + + private: + // The epsilon value used to initialise the squared gradient parameter. + double epsilon; + + // The smoothing parameter. + double beta1; + + // The second moment coefficient. + double beta2; + + // The exponential moving average of gradient values. + arma::mat m; + + // The exponential moving average of squared gradient values. + arma::mat v; + + // The optimal sqaured gradient value. + arma::mat vImproved; + + // The number of iterations. + double iteration; +}; + +} // namespace optimization +} // namespace mlpack + +#endif diff --git a/src/mlpack/core/optimizers/adam/nadam_update.hpp b/src/mlpack/core/optimizers/adam/nadam_update.hpp new file mode 100644 index 0000000000..0e6a017327 --- /dev/null +++ b/src/mlpack/core/optimizers/adam/nadam_update.hpp @@ -0,0 +1,173 @@ +/** + * @file nadam_update.hpp + * @author Sourabh Varshney + * + * Nadam update rule. Nadam is an optimizer that combines the effect of Adam + * and NAG to the gradient descent to improve its Performance. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_CORE_OPTIMIZERS_ADAM_NADAM_UPDATE_HPP +#define MLPACK_CORE_OPTIMIZERS_ADAM_NADAM_UPDATE_HPP + +#include + +namespace mlpack { +namespace optimization { + +/** + * Nadam is an optimizer that combines the Adam and NAG. + * + * For more information, see the following. + * + * @code + * @techreport{Dozat2015, + * title = {Incorporating Nesterov momentum into Adam}, + * author = {Timothy Dozat}, + * institution = {Stanford University}, + * address = {Stanford}, + * year = {2015}, + * url = {https://openreview.net/pdf?id=OM0jvwB8jIp57ZJjtNEZ} + * } + * @endcode + */ +class NadamUpdate +{ + public: + /** + * Construct the Nadam update policy with the given parameters. + * + * @param epsilon The epsilon value used to initialise the squared gradient + * parameter. + * @param beta1 The smoothing parameter. + * @param beta2 The second moment coefficient + * @param scheduleDecay The decay parameter for decay coefficients + */ + NadamUpdate(const double epsilon = 1e-8, + const double beta1 = 0.9, + const double beta2 = 0.99, + const double scheduleDecay = 4e-3) + :epsilon(epsilon), + beta1(beta1), + beta2(beta2), + scheduleDecay(scheduleDecay), + iteration(0), + cumBeta1(1) + { + // Nothing to do. + } + + /** + * The Initialize method is called by SGD Optimizer method before the start + * of the iteration update process. + * + * @param rows Number of rows in the gradient matrix. + * @param cols Number of columns in the gradient matrix. + */ + void Initialize(const size_t rows, const size_t cols) + { + m = arma::zeros(rows, cols); + v = arma::zeros(rows, cols); + } + + /** + * Update step for Nadam. + * + * @param iterate Parameters that minimize the function. + * @param stepSize Step size to be used for the given iteration. + * @param gradient The gradient matrix. + */ + void Update(arma::mat& iterate, + const double stepSize, + const arma::mat& gradient) + { + // Increment the iteration counter variable. + ++iteration; + + // And update the iterate. + m *= beta1; + m += (1 - beta1) * gradient; + + v *= beta2; + v += (1 - beta2) * gradient % gradient; + + double beta1T = beta1 * (1 - (0.5 * + std::pow(0.96, iteration * scheduleDecay))); + + double beta1T1 = beta1 * (1 - (0.5 * + std::pow(0.96, (iteration + 1) * scheduleDecay))); + + cumBeta1 *= beta1T; + + const double biasCorrection1 = 1.0 - cumBeta1; + + const double biasCorrection2 = 1.0 - std::pow(beta2, iteration); + + const double biasCorrection3 = 1.0 - (cumBeta1 * beta1T1); + + /* Note :- arma::sqrt(v) + epsilon * sqrt(biasCorrection2) is approximated + * as arma::sqrt(v) + epsilon + */ + iterate -= (stepSize * (((1 - beta1T) / biasCorrection1) * gradient + + (beta1T1 / biasCorrection3) * m) * sqrt(biasCorrection2)) + / (arma::sqrt(v) + epsilon); + } + + //! Get the value used to initialise the squared gradient parameter. + double Epsilon() const { return epsilon; } + //! Modify the value used to initialise the squared gradient parameter. + double& Epsilon() { return epsilon; } + + //! Get the value of the cumulative product of decay coefficients + double CumBeta1() const { return cumBeta1; } + //! Modify the value of the cumulative product of decay coefficients + double& CumBeta1() { return cumBeta1; } + + //! Get the smoothing parameter. + double Beta1() const { return beta1; } + //! Modify the smoothing parameter. + double& Beta1() { return beta1; } + + //! Get the second moment coefficient. + double Beta2() const { return beta2; } + //! Modify the second moment coefficient. + double& Beta2() { return beta2; } + + //! Get the decay parameter for decay coefficients + double ScheduleDecay() const { return scheduleDecay; } + //! Modify the decay parameter for decay coefficients + double& ScheduleDecay() { return scheduleDecay; } + + private: + // The epsilon value used to initialise the squared gradient parameter. + double epsilon; + + // The smoothing parameter. + double beta1; + + // The second moment coefficient. + double beta2; + + // The exponential moving average of gradient values. + arma::mat m; + + // The exponential moving average of squared gradient values. + arma::mat v; + + // The cumulative product of decay coefficients + double cumBeta1; + + // The decay parameter for decay coefficients + double scheduleDecay; + + // The number of iterations. + double iteration; +}; + +} // namespace optimization +} // namespace mlpack + +#endif diff --git a/src/mlpack/core/optimizers/cmaes/full_selection.hpp b/src/mlpack/core/optimizers/cmaes/full_selection.hpp index 9ff63eaa79..162f570bdd 100644 --- a/src/mlpack/core/optimizers/cmaes/full_selection.hpp +++ b/src/mlpack/core/optimizers/cmaes/full_selection.hpp @@ -32,7 +32,7 @@ class FullSelection * @param iterate starting point. */ template - const double Select(DecomposableFunctionType& function, + double Select(DecomposableFunctionType& function, const size_t batchSize, const arma::mat& iterate) { diff --git a/src/mlpack/core/optimizers/cmaes/random_selection.hpp b/src/mlpack/core/optimizers/cmaes/random_selection.hpp index df4ecd05c2..0dd5def9dd 100644 --- a/src/mlpack/core/optimizers/cmaes/random_selection.hpp +++ b/src/mlpack/core/optimizers/cmaes/random_selection.hpp @@ -47,7 +47,7 @@ class RandomSelection * @param iterate starting point. */ template - const double Select(DecomposableFunctionType& function, + double Select(DecomposableFunctionType& function, const size_t batchSize, const arma::mat& iterate) { diff --git a/src/mlpack/core/optimizers/sgd/sgd_impl.hpp b/src/mlpack/core/optimizers/sgd/sgd_impl.hpp index 5e4423afcf..f36606dff9 100644 --- a/src/mlpack/core/optimizers/sgd/sgd_impl.hpp +++ b/src/mlpack/core/optimizers/sgd/sgd_impl.hpp @@ -102,8 +102,14 @@ double SGD::Optimize( function.Shuffle(); } - // Find the effective batch size (the last batch may be smaller). - const size_t effectiveBatchSize = std::min(batchSize, + // Find the effective batch size; we have to take the minimum of three + // things: + // - the batch size can't be larger than the user-specified batch size; + // - the batch size can't be larger than the number of iterations left + // before actualMaxIterations is hit; + // - the batch size can't be larger than the number of functions left. + const size_t effectiveBatchSize = std::min( + std::min(batchSize, actualMaxIterations - i), numFunctions - currentFunction); function.Gradient(iterate, currentFunction, gradient, effectiveBatchSize); diff --git a/src/mlpack/core/optimizers/spalera_sgd/spalera_sgd_impl.hpp b/src/mlpack/core/optimizers/spalera_sgd/spalera_sgd_impl.hpp index 1c6e47166f..de228ee3fd 100644 --- a/src/mlpack/core/optimizers/spalera_sgd/spalera_sgd_impl.hpp +++ b/src/mlpack/core/optimizers/spalera_sgd/spalera_sgd_impl.hpp @@ -108,8 +108,14 @@ double SPALeRASGD::Optimize(DecomposableFunctionType& function, function.Shuffle(); } - // Find the effective batch size (the last batch may be smaller). - const size_t effectiveBatchSize = std::min(batchSize, + // Find the effective batch size; we have to take the minimum of three + // things: + // - the batch size can't be larger than the user-specified batch size; + // - the batch size can't be larger than the number of iterations left + // before actualMaxIterations is hit; + // - the batch size can't be larger than the number of functions left. + const size_t effectiveBatchSize = std::min( + std::min(batchSize, actualMaxIterations - i), numFunctions - currentFunction); function.Gradient(iterate, currentFunction, gradient, effectiveBatchSize); diff --git a/src/mlpack/core/tree/cosine_tree/cosine_tree.cpp b/src/mlpack/core/tree/cosine_tree/cosine_tree.cpp index 61159a6001..40a747c875 100644 --- a/src/mlpack/core/tree/cosine_tree/cosine_tree.cpp +++ b/src/mlpack/core/tree/cosine_tree/cosine_tree.cpp @@ -94,7 +94,7 @@ CosineTree::CosineTree(const arma::mat& dataset, // Initialize Monte Carlo error estimate for comparison. double monteCarloError = root.FrobNormSquared(); - while (treeQueue.top() && + while (treeQueue.size() > 0 && (monteCarloError > epsilon * root.FrobNormSquared())) { // Pop node from queue with highest projection error. diff --git a/src/mlpack/core/util/mlpack_main.hpp b/src/mlpack/core/util/mlpack_main.hpp index d8d2c76803..a26a5c6ff7 100644 --- a/src/mlpack/core/util/mlpack_main.hpp +++ b/src/mlpack/core/util/mlpack_main.hpp @@ -46,11 +46,12 @@ using Option = mlpack::bindings::cli::CLIOption; } } +static const std::string testName = ""; #include #include #include -void mlpackMain(); // This is typically defined after this include. +static void mlpackMain(); // This is typically defined after this include. int main(int argc, char** argv) { @@ -91,19 +92,13 @@ using Option = mlpack::bindings::tests::TestOption; } } +// testName symbol should be defined in each binding test file #include #undef PROGRAM_INFO #define PROGRAM_INFO(NAME, DESC) static mlpack::util::ProgramDoc \ cli_programdoc_dummy_object = mlpack::util::ProgramDoc(NAME, \ - []() { return DESC; }); \ - namespace mlpack { \ - namespace bindings { \ - namespace tests { \ - std::string programName = NAME; \ - } \ - } \ - } + []() { return DESC; }); #elif(BINDING_TYPE == BINDING_TYPE_PYX) // This is a Python binding. @@ -126,6 +121,7 @@ using Option = mlpack::bindings::python::PyOption; } } +static const std::string testName = ""; #include #undef PROGRAM_INFO diff --git a/src/mlpack/core/util/param.hpp b/src/mlpack/core/util/param.hpp index 7252f51bb4..2683d59b77 100644 --- a/src/mlpack/core/util/param.hpp +++ b/src/mlpack/core/util/param.hpp @@ -1012,53 +1012,55 @@ using DatasetInfo = DatasetMapper; #define PARAM_IN(T, ID, DESC, ALIAS, DEF, REQ) \ static mlpack::util::Option \ JOIN(cli_option_dummy_object_in_, __COUNTER__) \ - (DEF, ID, DESC, ALIAS, #T, REQ, true, false); + (DEF, ID, DESC, ALIAS, #T, REQ, true, false, testName); #define PARAM_OUT(T, ID, DESC, ALIAS, DEF, REQ) \ static mlpack::util::Option \ JOIN(cli_option_dummy_object_out_, __COUNTER__) \ - (DEF, ID, DESC, ALIAS, #T, REQ, false, false); + (DEF, ID, DESC, ALIAS, #T, REQ, false, false, testName); #define PARAM_MATRIX(ID, DESC, ALIAS, REQ, TRANS, IN) \ static mlpack::util::Option \ JOIN(cli_option_dummy_matrix_, __COUNTER__) \ - (arma::mat(), ID, DESC, ALIAS, "arma::mat", REQ, IN, !TRANS); + (arma::mat(), ID, DESC, ALIAS, "arma::mat", \ + REQ, IN, !TRANS, testName); #define PARAM_UMATRIX(ID, DESC, ALIAS, REQ, TRANS, IN) \ static mlpack::util::Option> \ JOIN(cli_option_dummy_umatrix_, __COUNTER__) \ - (arma::Mat(), ID, DESC, ALIAS, "arma::Mat", REQ, IN, \ - !TRANS); + (arma::Mat(), ID, DESC, ALIAS, "arma::Mat", \ + REQ, IN, !TRANS, testName); #define PARAM_COL(ID, DESC, ALIAS, REQ, TRANS, IN) \ static mlpack::util::Option \ JOIN(cli_option_dummy_col_, __COUNTER__) \ - (arma::vec(), ID, DESC, ALIAS, "arma::vec", REQ, IN, !TRANS); + (arma::vec(), ID, DESC, ALIAS, "arma::vec", \ + REQ, IN, !TRANS, testName); #define PARAM_UCOL(ID, DESC, ALIAS, REQ, TRANS, IN) \ static mlpack::util::Option> \ JOIN(cli_option_dummy_ucol_, __COUNTER__) \ - (arma::Col(), ID, DESC, ALIAS, "arma::Col", REQ, IN, \ - !TRANS); + (arma::Col(), ID, DESC, ALIAS, "arma::Col", \ + REQ, IN, !TRANS, testName); #define PARAM_ROW(ID, DESC, ALIAS, REQ, TRANS, IN) \ static mlpack::util::Option \ JOIN(cli_option_dummy_row_, __COUNTER__) \ - (arma::rowvec(), ID, DESC, ALIAS, "arma::rowvec", REQ, IN, !TRANS); + (arma::rowvec(), ID, DESC, ALIAS, "arma::rowvec", \ + REQ, IN, !TRANS, testName); #define PARAM_UROW(ID, DESC, ALIAS, REQ, TRANS, IN) \ static mlpack::util::Option> \ JOIN(cli_option_dummy_urow_, __COUNTER__) \ - (arma::Row(), ID, DESC, ALIAS, "arma::Row", REQ, IN, \ - !TRANS); - + (arma::Row(), ID, DESC, ALIAS, "arma::Row", \ + REQ, IN, !TRANS, testName); // There are no uses of required models, so that is not an option to this // macro (it would be easy to add). #define PARAM_MODEL(TYPE, ID, DESC, ALIAS, REQ, IN) \ static mlpack::util::Option \ JOIN(cli_option_dummy_model_, __COUNTER__) \ - (TYPE(), ID, DESC, ALIAS, #TYPE, REQ, IN); + (TYPE(), ID, DESC, ALIAS, #TYPE, REQ, IN, false, testName); #else // We have to do some really bizarre stuff since __COUNTER__ isn't defined. I // don't think we can absolutely guarantee success, but it should be "good @@ -1067,50 +1069,54 @@ using DatasetInfo = DatasetMapper; #define PARAM_IN(T, ID, DESC, ALIAS, DEF, REQ) \ static mlpack::util::Option \ JOIN(JOIN(cli_option_dummy_object_in_, __LINE__), opt) \ - (DEF, ID, DESC, ALIAS, #T, REQ, true, false); + (DEF, ID, DESC, ALIAS, #T, REQ, true, false, testName); #define PARAM_OUT(T, ID, DESC, ALIAS, DEF, REQ) \ static mlpack::util::Option \ JOIN(JOIN(cli_option_dummy_object_out_, __LINE__), opt) \ - (DEF, ID, DESC, ALIAS, #T, REQ, false, false); + (DEF, ID, DESC, ALIAS, #T, REQ, false, false, testName); #define PARAM_MATRIX(ID, DESC, ALIAS, REQ, TRANS, IN) \ static mlpack::util::Option \ JOIN(JOIN(cli_option_dummy_object_matrix_, __LINE__), opt) \ - (arma::mat(), ID, DESC, ALIAS, "arma::mat", REQ, IN, !TRANS); + (arma::mat(), ID, DESC, ALIAS, "arma::mat", REQ, IN, !TRANS, \ + testName); #define PARAM_UMATRIX(ID, DESC, ALIAS, REQ, TRANS, IN) \ static mlpack::util::Option> \ JOIN(JOIN(cli_option_dummy_object_umatrix_, __LINE__), opt) \ (arma::Mat(), ID, DESC, ALIAS, "arma::Mat", REQ, IN, \ - !TRANS); + !TRANS, testName); #define PARAM_COL(ID, DESC, ALIAS, REQ, TRANS, IN) \ static mlpack::util::Option \ JOIN(cli_option_dummy_object_col_, __LINE__) \ - (arma::vec(), ID, DESC, ALIAS, "arma::vec", REQ, IN, !TRANS); + (arma::vec(), ID, DESC, ALIAS, "arma::vec", REQ, IN, !TRANS, \ + testName); #define PARAM_UCOL(ID, DESC, ALIAS, REQ, TRANS, IN) \ static mlpack::util::Option> \ JOIN(cli_option_dummy_object_ucol_, __LINE__) \ (arma::Col(), ID, DESC, ALIAS, "arma::Col", REQ, IN, \ - !TRANS); + !TRANS, testName); #define PARAM_ROW(ID, DESC, ALIAS, REQ, TRANS, IN) \ static mlpack::util::Option \ JOIN(cli_option_dummy_object_row_, __LINE__) \ - (arma::rowvec(), ID, DESC, ALIAS, "arma::rowvec", REQ, IN, !TRANS); + (arma::rowvec(), ID, DESC, ALIAS, "arma::rowvec", REQ, IN, !TRANS, \ + testName); #define PARAM_UROW(ID, DESC, ALIAS, REQ, TRANS, IN) \ static mlpack::util::Option> \ JOIN(cli_option_dummy_object_urow_, __LINE__) \ (arma::Row(), ID, DESC, ALIAS, "arma::Row", REQ, IN, \ - !TRANS); + !TRANS, testName); #define PARAM_MODEL(TYPE, ID, DESC, ALIAS, REQ, IN) \ static mlpack::util::Option \ JOIN(JOIN(cli_option_dummy_object_model_, __LINE__), opt) \ - (TYPE(), ID, DESC, ALIAS, #TYPE, REQ, IN); + (TYPE(), ID, DESC, ALIAS, #TYPE, REQ, IN, false, \ + testName); #endif #endif diff --git a/src/mlpack/methods/adaboost/adaboost_main.cpp b/src/mlpack/methods/adaboost/adaboost_main.cpp index e0b3a70cc7..e5209efb46 100644 --- a/src/mlpack/methods/adaboost/adaboost_main.cpp +++ b/src/mlpack/methods/adaboost/adaboost_main.cpp @@ -111,7 +111,7 @@ PARAM_MODEL_IN(AdaBoostModel, "input_model", "Input AdaBoost model.", "m"); PARAM_MODEL_OUT(AdaBoostModel, "output_model", "Output trained AdaBoost model.", "M"); -void mlpackMain() +static void mlpackMain() { // Check input parameters and issue warnings/errors as necessary. diff --git a/src/mlpack/methods/ann/layer/fast_lstm_impl.hpp b/src/mlpack/methods/ann/layer/fast_lstm_impl.hpp index 3222275a4d..36d92c2a0d 100644 --- a/src/mlpack/methods/ann/layer/fast_lstm_impl.hpp +++ b/src/mlpack/methods/ann/layer/fast_lstm_impl.hpp @@ -80,7 +80,7 @@ void FastLSTM::ResetCell(const size_t size) gradientStep = batchSize * size - 1; const size_t rhoBatchSize = size * batchSize; - if (gate.is_empty() || gate.n_cols < rhoBatchSize) + if (gate.is_empty() || gate.n_cols != rhoBatchSize) { gate.set_size(4 * outSize, rhoBatchSize); gateActivation.set_size(outSize * 3, rhoBatchSize); @@ -197,7 +197,7 @@ void FastLSTM::Backward( backwardStep - batchStep, 3 * outSize - 1, backwardStep) % cellActivationError; - if (backwardStep != 0) + if (backwardStep > batchStep) { prevError.submat(2 * outSize, 0, 3 * outSize - 1, batchStep) = cell.cols((backwardStep - batchSize) - batchStep, @@ -258,13 +258,13 @@ void FastLSTM::Gradient( gradient.n_elem - 1, 0) = arma::vectorise(prevError * outParameter.cols(gradientStep - batchStep, gradientStep).t()); - if (gradientStep == 0) + if (gradientStep > batchStep) { - gradientStep = batchSize * bpttSteps - 1; + gradientStep -= batchSize; } else { - gradientStep -= batchSize; + gradientStep = batchSize * bpttSteps - 1; } } diff --git a/src/mlpack/methods/ann/layer/gru_impl.hpp b/src/mlpack/methods/ann/layer/gru_impl.hpp index c795cef406..86c6b85414 100644 --- a/src/mlpack/methods/ann/layer/gru_impl.hpp +++ b/src/mlpack/methods/ann/layer/gru_impl.hpp @@ -69,7 +69,7 @@ GRU::GRU( allZeros = arma::zeros(outSize, batchSize); outParameter.push_back(std::move(arma::mat(allZeros.memptr(), - allZeros.n_rows, allZeros.n_cols, false, true))); + allZeros.n_rows, allZeros.n_cols, false, true))); prevOutput = outParameter.begin(); backIterator = outParameter.end(); @@ -96,6 +96,21 @@ void GRU::Forward( { batchSize = input.n_cols; prevError.resize(3 * outSize, batchSize); + allZeros.zeros(outSize, batchSize); + // Batch size better not change during an iteration... + if (outParameter.size() > 1) + { + Log::Fatal << "GRU<>::Forward(): batch size cannot change during a " + << "forward pass!" << std::endl; + } + + outParameter.clear(); + outParameter.push_back(std::move(arma::mat(allZeros.memptr(), + allZeros.n_rows, allZeros.n_cols, false, true))); + + prevOutput = outParameter.begin(); + backIterator = outParameter.end(); + gradIterator = outParameter.end(); } // Process the input linearly(zt, rt, ot). @@ -157,13 +172,13 @@ void GRU::Forward( if (!deterministic) { outParameter.push_back(std::move(arma::mat(allZeros.memptr(), - allZeros.n_rows, allZeros.n_cols, false, true))); + allZeros.n_rows, allZeros.n_cols, false, true))); prevOutput = --outParameter.end(); } else { *prevOutput = std::move(arma::mat(allZeros.memptr(), - allZeros.n_rows, allZeros.n_cols, false, true)); + allZeros.n_rows, allZeros.n_cols, false, true)); } } else if (!deterministic) @@ -196,6 +211,21 @@ void GRU::Backward( { batchSize = input.n_cols; prevError.resize(3 * outSize, batchSize); + allZeros.zeros(outSize, batchSize); + // Batch size better not change during an iteration... + if (outParameter.size() > 1) + { + Log::Fatal << "GRU<>::Forward(): batch size cannot change during a " + << "forward pass!" << std::endl; + } + + outParameter.clear(); + outParameter.push_back(std::move(arma::mat(allZeros.memptr(), + allZeros.n_rows, allZeros.n_cols, false, true))); + + prevOutput = outParameter.begin(); + backIterator = outParameter.end(); + gradIterator = outParameter.end(); } if ((outParameter.size() - backwardStep - 1) % rho != 0 && backwardStep != 0) @@ -214,7 +244,7 @@ void GRU::Backward( hiddenStateModule)); // Delta ot. - arma::mat dOt = gy % (arma::ones(outSize) - + arma::mat dOt = gy % (arma::ones(outSize, batchSize) - boost::apply_visitor(outputParameterVisitor, inputGateModule)); // Delta of input gate. @@ -297,6 +327,21 @@ void GRU::Gradient( { batchSize = input.n_cols; prevError.resize(3 * outSize, batchSize); + allZeros.zeros(outSize, batchSize); + // Batch size better not change during an iteration... + if (outParameter.size() > 1) + { + Log::Fatal << "GRU<>::Forward(): batch size cannot change during a " + << "forward pass!" << std::endl; + } + + outParameter.clear(); + outParameter.push_back(std::move(arma::mat(allZeros.memptr(), + allZeros.n_rows, allZeros.n_cols, false, true))); + + prevOutput = outParameter.begin(); + backIterator = outParameter.end(); + gradIterator = outParameter.end(); } if (gradIterator == outParameter.end()) diff --git a/src/mlpack/methods/ann/layer/lstm_impl.hpp b/src/mlpack/methods/ann/layer/lstm_impl.hpp index 5abf58be5d..ea06e8e616 100644 --- a/src/mlpack/methods/ann/layer/lstm_impl.hpp +++ b/src/mlpack/methods/ann/layer/lstm_impl.hpp @@ -188,12 +188,12 @@ void LSTM::Forward( if (forwardStep > 0) { inputGate.cols(forwardStep, forwardStep + batchStep) += - cell2GateInputWeight % cell.cols(forwardStep - batchSize, - forwardStep - batchSize + batchStep); + arma::repmat(cell2GateInputWeight, 1, batchSize) % + cell.cols(forwardStep - batchSize, forwardStep - batchSize + batchStep); forgetGate.cols(forwardStep, forwardStep + batchStep) += - cell2GateForgetWeight % cell.cols(forwardStep - batchSize, - forwardStep - batchSize + batchStep); + arma::repmat(cell2GateForgetWeight, 1, batchSize) % + cell.cols(forwardStep - batchSize, forwardStep - batchSize + batchStep); } inputGateActivation.cols(forwardStep, forwardStep + batchStep) = 1.0 / @@ -283,7 +283,7 @@ void LSTM::Backward( cellError += inputCellError; } - if (backwardStep != 0) + if (backwardStep > batchStep) { forgetGateError = cell.cols((backwardStep - batchSize) - batchStep, (backwardStep - batchSize)) % cellError % (forgetGateActivation.cols( @@ -305,7 +305,7 @@ void LSTM::Backward( backwardStep - batchStep, backwardStep), 2)); inputCellError = forgetGateActivation.cols(backwardStep - batchStep, - backwardStep) %cellError + forgetGateError.each_col() % + backwardStep) % cellError + forgetGateError.each_col() % cell2GateForgetWeight + inputGateError.each_col() % cell2GateInputWeight; g = input2GateInputWeight.t() * inputGateError + @@ -395,15 +395,17 @@ void LSTM::Gradient( offset += cell2GateOutputWeight.n_elem; // Cell2GateForgetWeight and cell2GateInputWeight gradients. - if (gradientStep != 0) + if (gradientStep > batchStep) { gradient.submat(offset, 0, offset + cell2GateForgetWeight.n_elem - 1, 0) = - arma::sum(forgetGateError % cell.cols(gradientStep - batchStep - - batchSize, gradientStep - batchSize), 1); + arma::sum(forgetGateError % + cell.cols((gradientStep - batchSize) - batchStep, + (gradientStep - batchSize)), 1); gradient.submat(offset + cell2GateForgetWeight.n_elem, 0, offset + cell2GateForgetWeight.n_elem + cell2GateInputWeight.n_elem - 1, 0) = - arma::sum(inputGateError % cell.cols(gradientStep - batchStep - - batchSize, gradientStep - batchSize), 1); + arma::sum(inputGateError % + cell.cols((gradientStep - batchSize) - batchStep, + (gradientStep - batchSize)), 1); } else { diff --git a/src/mlpack/methods/ann/rnn.hpp b/src/mlpack/methods/ann/rnn.hpp index 8dc6507922..6cadc40d73 100644 --- a/src/mlpack/methods/ann/rnn.hpp +++ b/src/mlpack/methods/ann/rnn.hpp @@ -222,7 +222,14 @@ class RNN size_t& Rho() { return rho; } /** - * Reset the module infomration (weights/parameters). + * Reset the state of the network. This ensures that all internally-held + * gradients are set to 0, all memory cells are reset, and the parameters + * matrix is the right size. + */ + void Reset(); + + /** + * Reset the module information (weights/parameters). */ void ResetParameters(); diff --git a/src/mlpack/methods/ann/rnn_impl.hpp b/src/mlpack/methods/ann/rnn_impl.hpp index d7c0269687..d745323f85 100644 --- a/src/mlpack/methods/ann/rnn_impl.hpp +++ b/src/mlpack/methods/ann/rnn_impl.hpp @@ -102,7 +102,6 @@ void RNN::Train( if (!reset) { ResetParameters(); - reset = true; } // Train the model. @@ -139,7 +138,6 @@ void RNN::Train( if (!reset) { ResetParameters(); - reset = true; } OptimizerType optimizer; @@ -207,7 +205,6 @@ double RNN::Evaluate( if (parameter.is_empty()) { ResetParameters(); - reset = true; } if (deterministic != this->deterministic) @@ -272,7 +269,6 @@ void RNN::Gradient( if (parameter.is_empty()) { ResetParameters(); - reset = true; } gradient = arma::zeros(parameter.n_rows, parameter.n_cols); @@ -337,6 +333,17 @@ void RNN::ResetParameters() // Reset the network parameter with the given initialization rule. NetworkInitialization networkInit(initializeRule); networkInit.Initialize(network, parameter); + + reset = true; +} + +template +void RNN::Reset() +{ + ResetParameters(); + ResetCells(); + currentGradient.zeros(); + ResetGradients(currentGradient); } template diff --git a/src/mlpack/methods/approx_kfn/approx_kfn_main.cpp b/src/mlpack/methods/approx_kfn/approx_kfn_main.cpp index 3081ed3bcd..f73008583d 100644 --- a/src/mlpack/methods/approx_kfn/approx_kfn_main.cpp +++ b/src/mlpack/methods/approx_kfn/approx_kfn_main.cpp @@ -141,7 +141,7 @@ PARAM_MODEL_IN(ApproxKFNModel, "input_model", "File containing input model.", PARAM_MODEL_OUT(ApproxKFNModel, "output_model", "File to save output model to.", "M"); -void mlpackMain() +static void mlpackMain() { // We have to pass either a reference set or an input model. RequireOnlyOnePassed({ "reference", "input_model" }); diff --git a/src/mlpack/methods/cf/cf_main.cpp b/src/mlpack/methods/cf/cf_main.cpp index a0c670e6bb..4f2ca384bb 100644 --- a/src/mlpack/methods/cf/cf_main.cpp +++ b/src/mlpack/methods/cf/cf_main.cpp @@ -263,7 +263,7 @@ void AssembleFactorizerType(const std::string& algorithm, } } -void mlpackMain() +static void mlpackMain() { if (CLI::GetParam("seed") == 0) math::RandomSeed(std::time(NULL)); diff --git a/src/mlpack/methods/dbscan/dbscan_main.cpp b/src/mlpack/methods/dbscan/dbscan_main.cpp index da0807c113..5c4af77a01 100644 --- a/src/mlpack/methods/dbscan/dbscan_main.cpp +++ b/src/mlpack/methods/dbscan/dbscan_main.cpp @@ -112,7 +112,7 @@ void RunDBSCAN(RangeSearchType rs = RangeSearchType()) CLI::GetParam>("assignments") = std::move(assignments); } -void mlpackMain() +static void mlpackMain() { RequireAtLeastOnePassed({ "assignments", "centroids" }, false, "no output will be saved"); diff --git a/src/mlpack/methods/decision_stump/decision_stump_main.cpp b/src/mlpack/methods/decision_stump/decision_stump_main.cpp index a3f3213df5..2bfe230043 100644 --- a/src/mlpack/methods/decision_stump/decision_stump_main.cpp +++ b/src/mlpack/methods/decision_stump/decision_stump_main.cpp @@ -103,7 +103,7 @@ PARAM_MODEL_OUT(DSModel, "output_model", "Output decision stump model to save.", PARAM_INT_IN("bucket_size", "The minimum number of training points in each " "decision stump bucket.", "b", 6); -void mlpackMain() +static void mlpackMain() { // Check that the parameters are reasonable. RequireOnlyOnePassed({ "training", "input_model" }, true); diff --git a/src/mlpack/methods/decision_tree/decision_tree_main.cpp b/src/mlpack/methods/decision_tree/decision_tree_main.cpp index 2ead988d1a..6145b66d3f 100644 --- a/src/mlpack/methods/decision_tree/decision_tree_main.cpp +++ b/src/mlpack/methods/decision_tree/decision_tree_main.cpp @@ -115,7 +115,7 @@ PARAM_MODEL_IN(DecisionTreeModel, "input_model", "Pre-trained decision tree, " PARAM_MODEL_OUT(DecisionTreeModel, "output_model", "Output for trained decision" " tree.", "M"); -void mlpackMain() +static void mlpackMain() { // Check parameters. RequireOnlyOnePassed({ "training", "input_model" }, true); diff --git a/src/mlpack/methods/det/det_main.cpp b/src/mlpack/methods/det/det_main.cpp index fcfba52a4d..29ecd56c99 100644 --- a/src/mlpack/methods/det/det_main.cpp +++ b/src/mlpack/methods/det/det_main.cpp @@ -100,7 +100,7 @@ PARAM_FLAG("volume_regularization", "This flag gives the used the option to use" */ -void mlpackMain() +static void mlpackMain() { // Validate input parameters. RequireOnlyOnePassed({ "training", "input_model" }, true); diff --git a/src/mlpack/methods/emst/emst_main.cpp b/src/mlpack/methods/emst/emst_main.cpp index e3a41373b7..3b0f9de98a 100644 --- a/src/mlpack/methods/emst/emst_main.cpp +++ b/src/mlpack/methods/emst/emst_main.cpp @@ -72,7 +72,7 @@ using namespace mlpack::metric; using namespace mlpack::util; using namespace std; -void mlpackMain() +static void mlpackMain() { RequireAtLeastOnePassed({ "output" }, false, "no output will be saved"); diff --git a/src/mlpack/methods/fastmks/fastmks_main.cpp b/src/mlpack/methods/fastmks/fastmks_main.cpp index be60c3e976..f3b7333ebe 100644 --- a/src/mlpack/methods/fastmks/fastmks_main.cpp +++ b/src/mlpack/methods/fastmks/fastmks_main.cpp @@ -83,7 +83,7 @@ PARAM_FLAG("single", "If true, single-tree search is used (as opposed to " PARAM_MATRIX_OUT("kernels", "Output matrix of kernels.", "p"); PARAM_UMATRIX_OUT("indices", "Output matrix of indices.", "i"); -void mlpackMain() +static void mlpackMain() { // Validate command-line parameters. RequireOnlyOnePassed({ "reference", "input_model" }, true); diff --git a/src/mlpack/methods/gmm/gmm_generate_main.cpp b/src/mlpack/methods/gmm/gmm_generate_main.cpp index 9a5da6a29a..a6ed7d6ba4 100644 --- a/src/mlpack/methods/gmm/gmm_generate_main.cpp +++ b/src/mlpack/methods/gmm/gmm_generate_main.cpp @@ -42,7 +42,7 @@ PARAM_MATRIX_OUT("output", "Matrix to save output samples in.", "o"); PARAM_INT_IN("seed", "Random seed. If 0, 'std::time(NULL)' is used.", "s", 0); -void mlpackMain() +static void mlpackMain() { // Parameter sanity checks. RequireAtLeastOnePassed({ "output" }, false, "no results will be saved"); diff --git a/src/mlpack/methods/gmm/gmm_probability_main.cpp b/src/mlpack/methods/gmm/gmm_probability_main.cpp index 8606c38bbe..ac1669aaac 100644 --- a/src/mlpack/methods/gmm/gmm_probability_main.cpp +++ b/src/mlpack/methods/gmm/gmm_probability_main.cpp @@ -41,7 +41,7 @@ PARAM_MATRIX_IN_REQ("input", "Input matrix to calculate probabilities of.", PARAM_MATRIX_OUT("output", "Matrix to store calculated probabilities in.", "o"); -void mlpackMain() +static void mlpackMain() { RequireAtLeastOnePassed({ "output" }, false, "no results will be saved"); diff --git a/src/mlpack/methods/gmm/gmm_train_main.cpp b/src/mlpack/methods/gmm/gmm_train_main.cpp index 8e87f6867b..82694fb17c 100644 --- a/src/mlpack/methods/gmm/gmm_train_main.cpp +++ b/src/mlpack/methods/gmm/gmm_train_main.cpp @@ -121,7 +121,7 @@ PARAM_MODEL_IN(GMM, "input_model", "Initial input GMM model to start training " "with.", "m"); PARAM_MODEL_OUT(GMM, "output_model", "Output for trained GMM model.", "M"); -void mlpackMain() +static void mlpackMain() { // Check parameters and load data. if (CLI::GetParam("seed") != 0) diff --git a/src/mlpack/methods/hmm/hmm_generate_main.cpp b/src/mlpack/methods/hmm/hmm_generate_main.cpp index 4f4980a7dc..c9488d3466 100644 --- a/src/mlpack/methods/hmm/hmm_generate_main.cpp +++ b/src/mlpack/methods/hmm/hmm_generate_main.cpp @@ -92,7 +92,7 @@ struct Generate } }; -void mlpackMain() +static void mlpackMain() { RequireAtLeastOnePassed({ "output", "state" }, false, "no output will be " "saved"); diff --git a/src/mlpack/methods/hmm/hmm_loglik_main.cpp b/src/mlpack/methods/hmm/hmm_loglik_main.cpp index 9270455c2e..1a3d51a8cf 100644 --- a/src/mlpack/methods/hmm/hmm_loglik_main.cpp +++ b/src/mlpack/methods/hmm/hmm_loglik_main.cpp @@ -76,7 +76,7 @@ struct Loglik } }; -void mlpackMain() +static void mlpackMain() { // Load model, and calculate the log-likelihood of the sequence. CLI::GetParam("input_model").PerformAction((void*) NULL); diff --git a/src/mlpack/methods/hmm/hmm_train_main.cpp b/src/mlpack/methods/hmm/hmm_train_main.cpp index 20c18d7ee8..2983c2528b 100644 --- a/src/mlpack/methods/hmm/hmm_train_main.cpp +++ b/src/mlpack/methods/hmm/hmm_train_main.cpp @@ -340,7 +340,7 @@ struct Train } }; -void mlpackMain() +static void mlpackMain() { // Set random seed. if (CLI::GetParam("seed") != 0) diff --git a/src/mlpack/methods/hmm/hmm_viterbi_main.cpp b/src/mlpack/methods/hmm/hmm_viterbi_main.cpp index a225e6f4cc..323bc066cc 100644 --- a/src/mlpack/methods/hmm/hmm_viterbi_main.cpp +++ b/src/mlpack/methods/hmm/hmm_viterbi_main.cpp @@ -82,7 +82,7 @@ struct Viterbi } }; -void mlpackMain() +static void mlpackMain() { RequireAtLeastOnePassed({ "output" }, false, "no results will be saved"); diff --git a/src/mlpack/methods/hoeffding_trees/hoeffding_tree_main.cpp b/src/mlpack/methods/hoeffding_trees/hoeffding_tree_main.cpp index c832d147ad..a12c51d2c2 100644 --- a/src/mlpack/methods/hoeffding_trees/hoeffding_tree_main.cpp +++ b/src/mlpack/methods/hoeffding_trees/hoeffding_tree_main.cpp @@ -109,7 +109,7 @@ PARAM_INT_IN("observations_before_binning", "If the 'domingos' split strategy " // Convenience typedef. typedef tuple TupleType; -void mlpackMain() +static void mlpackMain() { // Check input parameters for validity. const string numericSplitStrategy = diff --git a/src/mlpack/methods/kernel_pca/kernel_pca_main.cpp b/src/mlpack/methods/kernel_pca/kernel_pca_main.cpp index f6b0d0a16a..7ca6268120 100644 --- a/src/mlpack/methods/kernel_pca/kernel_pca_main.cpp +++ b/src/mlpack/methods/kernel_pca/kernel_pca_main.cpp @@ -164,7 +164,7 @@ void RunKPCA(arma::mat& dataset, } } -void mlpackMain() +static void mlpackMain() { RequireAtLeastOnePassed({ "output" }, false, "no output will be saved"); diff --git a/src/mlpack/methods/kmeans/kmeans_main.cpp b/src/mlpack/methods/kmeans/kmeans_main.cpp index 4734be86f4..2f42213805 100644 --- a/src/mlpack/methods/kmeans/kmeans_main.cpp +++ b/src/mlpack/methods/kmeans/kmeans_main.cpp @@ -140,7 +140,7 @@ template class LloydStepType> void RunKMeans(const InitialPartitionPolicy& ipp); -void mlpackMain() +static void mlpackMain() { // Initialize random seed. if (CLI::GetParam("seed") != 0) diff --git a/src/mlpack/methods/lars/lars_main.cpp b/src/mlpack/methods/lars/lars_main.cpp index bc623108fe..0c6d1a28e5 100644 --- a/src/mlpack/methods/lars/lars_main.cpp +++ b/src/mlpack/methods/lars/lars_main.cpp @@ -101,7 +101,7 @@ PARAM_DOUBLE_IN("lambda2", "Regularization parameter for l2-norm penalty.", "L", PARAM_FLAG("use_cholesky", "Use Cholesky decomposition during computation " "rather than explicitly computing the full Gram matrix.", "c"); -void mlpackMain() +static void mlpackMain() { double lambda1 = CLI::GetParam("lambda1"); double lambda2 = CLI::GetParam("lambda2"); diff --git a/src/mlpack/methods/linear_regression/linear_regression_main.cpp b/src/mlpack/methods/linear_regression/linear_regression_main.cpp index 8aa006535e..8817556934 100644 --- a/src/mlpack/methods/linear_regression/linear_regression_main.cpp +++ b/src/mlpack/methods/linear_regression/linear_regression_main.cpp @@ -85,7 +85,7 @@ PARAM_COL_OUT("output_predictions", "If --test_file is specified, this " PARAM_DOUBLE_IN("lambda", "Tikhonov regularization for ridge regression. If 0," " the method reduces to linear regression.", "l", 0.0); -void mlpackMain() +static void mlpackMain() { const double lambda = CLI::GetParam("lambda"); diff --git a/src/mlpack/methods/local_coordinate_coding/local_coordinate_coding_main.cpp b/src/mlpack/methods/local_coordinate_coding/local_coordinate_coding_main.cpp index 7fb3ba2329..cdfc34c258 100644 --- a/src/mlpack/methods/local_coordinate_coding/local_coordinate_coding_main.cpp +++ b/src/mlpack/methods/local_coordinate_coding/local_coordinate_coding_main.cpp @@ -93,7 +93,7 @@ PARAM_MATRIX_OUT("codes", "Output codes matrix.", "c"); PARAM_INT_IN("seed", "Random seed. If 0, 'std::time(NULL)' is used.", "s", 0); -void mlpackMain() +static void mlpackMain() { if (CLI::GetParam("seed") != 0) RandomSeed((size_t) CLI::GetParam("seed")); diff --git a/src/mlpack/methods/logistic_regression/logistic_regression_main.cpp b/src/mlpack/methods/logistic_regression/logistic_regression_main.cpp index 8f6dc937aa..02a12022e3 100644 --- a/src/mlpack/methods/logistic_regression/logistic_regression_main.cpp +++ b/src/mlpack/methods/logistic_regression/logistic_regression_main.cpp @@ -133,7 +133,7 @@ PARAM_DOUBLE_IN("decision_boundary", "Decision boundary for prediction; if the " "logistic function for a point is less than the boundary, the class is " "taken to be 0; otherwise, the class is 1.", "d", 0.5); -void mlpackMain() +static void mlpackMain() { // Collect command-line options. const double lambda = CLI::GetParam("lambda"); @@ -162,8 +162,8 @@ void mlpackMain() ReportIgnoredParam({{ "test", false }}, "output_probabilities"); // Tolerance needs to be positive. - RequireParamValue("tolerance", [](double x) { return x > 0.0; }, - true, "tolerance must be positive"); + RequireParamValue("tolerance", [](double x) { return x >= 0.0; }, + true, "tolerance must be positive or zero"); // Optimizer has to be L-BFGS or SGD. RequireParamInSet("optimizer", { "lbfgs", "sgd" }, diff --git a/src/mlpack/methods/lsh/lsh_main.cpp b/src/mlpack/methods/lsh/lsh_main.cpp index e85ac58fd6..f9ce2de8b5 100644 --- a/src/mlpack/methods/lsh/lsh_main.cpp +++ b/src/mlpack/methods/lsh/lsh_main.cpp @@ -82,7 +82,7 @@ PARAM_INT_IN("bucket_size", "The size of a bucket in the second level hash.", "B", 500); PARAM_INT_IN("seed", "Random seed. If 0, 'std::time(NULL)' is used.", "s", 0); -void mlpackMain() +static void mlpackMain() { if (CLI::GetParam("seed") != 0) math::RandomSeed((size_t) CLI::GetParam("seed")); diff --git a/src/mlpack/methods/mean_shift/mean_shift_main.cpp b/src/mlpack/methods/mean_shift/mean_shift_main.cpp index 5569e17adc..639e231e9d 100644 --- a/src/mlpack/methods/mean_shift/mean_shift_main.cpp +++ b/src/mlpack/methods/mean_shift/mean_shift_main.cpp @@ -66,7 +66,7 @@ PARAM_DOUBLE_IN("radius", "If the distance between two centroids is less than " "the given radius, one will be removed. A radius of 0 or less means an " "estimate will be calculated and used for the radius.", "r", 0); -void mlpackMain() +static void mlpackMain() { const double radius = CLI::GetParam("radius"); const int maxIterations = CLI::GetParam("max_iterations"); diff --git a/src/mlpack/methods/naive_bayes/nbc_main.cpp b/src/mlpack/methods/naive_bayes/nbc_main.cpp index 185a980356..99b76cb7e7 100644 --- a/src/mlpack/methods/naive_bayes/nbc_main.cpp +++ b/src/mlpack/methods/naive_bayes/nbc_main.cpp @@ -105,7 +105,7 @@ PARAM_UROW_OUT("output", "The matrix in which the predicted labels for the" PARAM_MATRIX_OUT("output_probs", "The matrix in which the predicted probability" " of labels for the test set will be written.", "p"); -void mlpackMain() +static void mlpackMain() { // Check input parameters. RequireOnlyOnePassed({ "training", "input_model" }, true); diff --git a/src/mlpack/methods/nca/nca_main.cpp b/src/mlpack/methods/nca/nca_main.cpp index 0618743b6e..cc988f1281 100644 --- a/src/mlpack/methods/nca/nca_main.cpp +++ b/src/mlpack/methods/nca/nca_main.cpp @@ -124,7 +124,7 @@ using namespace mlpack::optimization; using namespace mlpack::util; using namespace std; -void mlpackMain() +static void mlpackMain() { if (CLI::GetParam("seed") != 0) math::RandomSeed((size_t) CLI::GetParam("seed")); diff --git a/src/mlpack/methods/neighbor_search/kfn_main.cpp b/src/mlpack/methods/neighbor_search/kfn_main.cpp index d6942c6d1a..d6a5d3d0d7 100644 --- a/src/mlpack/methods/neighbor_search/kfn_main.cpp +++ b/src/mlpack/methods/neighbor_search/kfn_main.cpp @@ -95,7 +95,7 @@ PARAM_DOUBLE_IN("percentage", "If specified, will do approximate furthest " "neighbors will be at least (p*100) % of the distance as the true furthest " "neighbor.", "p", 1); -void mlpackMain() +static void mlpackMain() { if (CLI::GetParam("seed") != 0) math::RandomSeed((size_t) CLI::GetParam("seed")); diff --git a/src/mlpack/methods/neighbor_search/knn_main.cpp b/src/mlpack/methods/neighbor_search/knn_main.cpp index e021ce3225..9c43abceaf 100644 --- a/src/mlpack/methods/neighbor_search/knn_main.cpp +++ b/src/mlpack/methods/neighbor_search/knn_main.cpp @@ -99,7 +99,7 @@ PARAM_STRING_IN("algorithm", "Type of neighbor search: 'naive', 'single_tree', " PARAM_DOUBLE_IN("epsilon", "If specified, will do approximate nearest neighbor " "search with given relative error.", "e", 0); -void mlpackMain() +static void mlpackMain() { if (CLI::GetParam("seed") != 0) math::RandomSeed((size_t) CLI::GetParam("seed")); diff --git a/src/mlpack/methods/nmf/nmf_main.cpp b/src/mlpack/methods/nmf/nmf_main.cpp index a9ced923c7..e8f4bdb2d1 100644 --- a/src/mlpack/methods/nmf/nmf_main.cpp +++ b/src/mlpack/methods/nmf/nmf_main.cpp @@ -76,7 +76,7 @@ PARAM_DOUBLE_IN("min_residue", "The minimum root mean square residue allowed " PARAM_STRING_IN("update_rules", "Update rules for each iteration; ( multdist | " "multdiv | als ).", "u", "multdist"); -void mlpackMain() +static void mlpackMain() { // Initialize random seed. if (CLI::GetParam("seed") != 0) diff --git a/src/mlpack/methods/pca/pca_main.cpp b/src/mlpack/methods/pca/pca_main.cpp index 9ad0a9e5cc..3a187da8bf 100644 --- a/src/mlpack/methods/pca/pca_main.cpp +++ b/src/mlpack/methods/pca/pca_main.cpp @@ -99,7 +99,7 @@ void RunPCA(arma::mat& dataset, dataset.n_rows << " dimensions)." << endl; } -void mlpackMain() +static void mlpackMain() { // Load input dataset. arma::mat& dataset = CLI::GetParam("input"); @@ -119,7 +119,8 @@ void mlpackMain() error << "cannot be greater than existing dimensionality (" << dataset.n_rows << ")"; RequireParamValue("new_dimensionality", - [dataset](int x) { return x <= dataset.n_rows; }, true, error.str()); + [dataset](int x) { return x <= (int) dataset.n_rows; }, true, + error.str()); RequireParamValue("var_to_retain", [](double x) { return x >= 0.0 && x <= 1.0; }, true, diff --git a/src/mlpack/methods/perceptron/perceptron_main.cpp b/src/mlpack/methods/perceptron/perceptron_main.cpp index c8eace1e83..c7a5b17796 100644 --- a/src/mlpack/methods/perceptron/perceptron_main.cpp +++ b/src/mlpack/methods/perceptron/perceptron_main.cpp @@ -118,7 +118,7 @@ PARAM_MATRIX_IN("test", "A matrix containing the test set.", "T"); PARAM_UROW_OUT("output", "The matrix in which the predicted labels for the" " test set will be written.", "o"); -void mlpackMain() +static void mlpackMain() { // First, get all parameters and validate them. const size_t maxIterations = (size_t) CLI::GetParam("max_iterations"); diff --git a/src/mlpack/methods/preprocess/preprocess_binarize_main.cpp b/src/mlpack/methods/preprocess/preprocess_binarize_main.cpp index a4d952ca40..93dd75855b 100644 --- a/src/mlpack/methods/preprocess/preprocess_binarize_main.cpp +++ b/src/mlpack/methods/preprocess/preprocess_binarize_main.cpp @@ -54,7 +54,7 @@ using namespace mlpack::util; using namespace arma; using namespace std; -void mlpackMain() +static void mlpackMain() { const size_t dimension = (size_t) CLI::GetParam("dimension"); const double threshold = CLI::GetParam("threshold"); diff --git a/src/mlpack/methods/preprocess/preprocess_describe_main.cpp b/src/mlpack/methods/preprocess/preprocess_describe_main.cpp index a592293f32..744f826a5c 100644 --- a/src/mlpack/methods/preprocess/preprocess_describe_main.cpp +++ b/src/mlpack/methods/preprocess/preprocess_describe_main.cpp @@ -153,7 +153,7 @@ double StandardError(const size_t size, const double& fStd) return fStd / sqrt(size); } -void mlpackMain() +static void mlpackMain() { const size_t dimension = static_cast(CLI::GetParam("dimension")); const size_t precision = static_cast(CLI::GetParam("precision")); diff --git a/src/mlpack/methods/preprocess/preprocess_imputer_main.cpp b/src/mlpack/methods/preprocess/preprocess_imputer_main.cpp index 5d16a93fdd..7bcbf99fca 100644 --- a/src/mlpack/methods/preprocess/preprocess_imputer_main.cpp +++ b/src/mlpack/methods/preprocess/preprocess_imputer_main.cpp @@ -53,7 +53,7 @@ using namespace arma; using namespace std; using namespace data; -void mlpackMain() +static void mlpackMain() { const string inputFile = CLI::GetParam("input_file"); const string outputFile = CLI::GetParam("output_file"); diff --git a/src/mlpack/methods/preprocess/preprocess_split_main.cpp b/src/mlpack/methods/preprocess/preprocess_split_main.cpp index 60710f3672..a66056012e 100644 --- a/src/mlpack/methods/preprocess/preprocess_split_main.cpp +++ b/src/mlpack/methods/preprocess/preprocess_split_main.cpp @@ -71,7 +71,7 @@ using namespace mlpack::util; using namespace arma; using namespace std; -void mlpackMain() +static void mlpackMain() { // Parse command line options. const double testRatio = CLI::GetParam("test_ratio"); diff --git a/src/mlpack/methods/radical/radical_main.cpp b/src/mlpack/methods/radical/radical_main.cpp index 3863d6f339..7d5e594474 100644 --- a/src/mlpack/methods/radical/radical_main.cpp +++ b/src/mlpack/methods/radical/radical_main.cpp @@ -57,7 +57,7 @@ using namespace mlpack::util; using namespace std; using namespace arma; -void mlpackMain() +static void mlpackMain() { // Set random seed. if (CLI::GetParam("seed") != 0) diff --git a/src/mlpack/methods/random_forest/random_forest_main.cpp b/src/mlpack/methods/random_forest/random_forest_main.cpp index e149dbc141..e6cdfd5aa3 100644 --- a/src/mlpack/methods/random_forest/random_forest_main.cpp +++ b/src/mlpack/methods/random_forest/random_forest_main.cpp @@ -68,7 +68,7 @@ PARAM_MODEL_IN(RandomForestModel, "input_model", "Pre-trained random forest to " PARAM_MODEL_OUT(RandomForestModel, "output_model", "Model to save trained " "random forest to.", "M"); -void mlpackMain() +static void mlpackMain() { // Check for incompatible input parameters. RequireOnlyOnePassed({ "training", "input_model" }, true); diff --git a/src/mlpack/methods/range_search/range_search_main.cpp b/src/mlpack/methods/range_search/range_search_main.cpp index 7fe03f4814..66a070e6d6 100644 --- a/src/mlpack/methods/range_search/range_search_main.cpp +++ b/src/mlpack/methods/range_search/range_search_main.cpp @@ -91,7 +91,7 @@ PARAM_FLAG("naive", "If true, O(n^2) naive mode is used for computation.", "N"); PARAM_FLAG("single_mode", "If true, single-tree search is used (as opposed to " "dual-tree search).", "S"); -void mlpackMain() +static void mlpackMain() { if (CLI::GetParam("seed") != 0) math::RandomSeed((size_t) CLI::GetParam("seed")); diff --git a/src/mlpack/methods/rann/krann_main.cpp b/src/mlpack/methods/rann/krann_main.cpp index 0b4d5d725b..dd7fdef538 100644 --- a/src/mlpack/methods/rann/krann_main.cpp +++ b/src/mlpack/methods/rann/krann_main.cpp @@ -96,7 +96,7 @@ PARAM_FLAG("first_leaf_exact", "The flag to trigger sampling only after " PARAM_INT_IN("single_sample_limit", "The limit on the maximum number of " "samples (and hence the largest node you can approximate).", "z", 20); -void mlpackMain() +static void mlpackMain() { if (CLI::GetParam("seed") != 0) math::RandomSeed((size_t) CLI::GetParam("seed")); diff --git a/src/mlpack/methods/softmax_regression/softmax_regression_main.cpp b/src/mlpack/methods/softmax_regression/softmax_regression_main.cpp index 497b878a5b..78e8c96245 100644 --- a/src/mlpack/methods/softmax_regression/softmax_regression_main.cpp +++ b/src/mlpack/methods/softmax_regression/softmax_regression_main.cpp @@ -115,7 +115,7 @@ void TestClassifyAcc(const size_t numClasses, const Model& model); template unique_ptr TrainSoftmax(const size_t maxIterations); -void mlpackMain() +static void mlpackMain() { const int maxIterations = CLI::GetParam("max_iterations"); diff --git a/src/mlpack/methods/sparse_coding/sparse_coding.cpp b/src/mlpack/methods/sparse_coding/sparse_coding.cpp index 6edf7d7eb6..a383c37eb2 100644 --- a/src/mlpack/methods/sparse_coding/sparse_coding.cpp +++ b/src/mlpack/methods/sparse_coding/sparse_coding.cpp @@ -12,7 +12,6 @@ */ #include "sparse_coding.hpp" #include -#include namespace mlpack { namespace sparse_coding { diff --git a/src/mlpack/methods/sparse_coding/sparse_coding_main.cpp b/src/mlpack/methods/sparse_coding/sparse_coding_main.cpp index a3a0d40061..0ce5bf8336 100644 --- a/src/mlpack/methods/sparse_coding/sparse_coding_main.cpp +++ b/src/mlpack/methods/sparse_coding/sparse_coding_main.cpp @@ -96,7 +96,7 @@ PARAM_MATRIX_OUT("codes", "Matrix to save the output sparse codes of the test " PARAM_MATRIX_IN("test", "Optional matrix to be encoded by trained model.", "T"); -void mlpackMain() +static void mlpackMain() { if (CLI::GetParam("seed") != 0) RandomSeed((size_t) CLI::GetParam("seed")); diff --git a/src/mlpack/tests/CMakeLists.txt b/src/mlpack/tests/CMakeLists.txt index 1034bfc498..eeca656287 100644 --- a/src/mlpack/tests/CMakeLists.txt +++ b/src/mlpack/tests/CMakeLists.txt @@ -123,6 +123,7 @@ add_executable(mlpack_test union_find_test.cpp vantage_point_tree_test.cpp main_tests/decision_tree_test.cpp + main_tests/linear_regression_test.cpp main_tests/pca_test.cpp main_tests/preprocess_binarize_test.cpp main_tests/preprocess_imputer_test.cpp diff --git a/src/mlpack/tests/adam_test.cpp b/src/mlpack/tests/adam_test.cpp index c1431a860c..9c954af6cf 100644 --- a/src/mlpack/tests/adam_test.cpp +++ b/src/mlpack/tests/adam_test.cpp @@ -2,8 +2,9 @@ * @file adam_test.cpp * @author Vasanth Kalingeri * @author Vivek Pal + * @author Sourabh Varshney * - * Tests the Adam and AdaMax optimizer. + * Tests the Adam, AdaMax, AMSGrad and Nadam optimizer. * * mlpack is free software; you may redistribute it and/or modify it under the * terms of the 3-clause BSD license. You should have received a copy of the @@ -62,6 +63,22 @@ BOOST_AUTO_TEST_CASE(SimpleAdaMaxTestFunction) BOOST_REQUIRE_SMALL(coordinates[2], 0.1); } +/** + * Tests the AMSGrad optimizer using a simple test function. + */ +BOOST_AUTO_TEST_CASE(SimpleAMSGradTestFunction) +{ + SGDTestFunction f; + AMSGrad optimizer(1e-3, 1, 0.9, 0.999, 1e-8, 500000, 1e-11, true); + + arma::mat coordinates = f.GetInitialPoint(); + optimizer.Optimize(f, coordinates); + + BOOST_REQUIRE_SMALL(coordinates[0], 0.1); + BOOST_REQUIRE_SMALL(coordinates[1], 0.1); + BOOST_REQUIRE_SMALL(coordinates[2], 0.1); +} + /** * Run Adam on logistic regression and make sure the results are acceptable. */ @@ -178,4 +195,138 @@ BOOST_AUTO_TEST_CASE(AdaMaxLogisticRegressionTest) BOOST_REQUIRE_CLOSE(testAcc, 100.0, 0.6); // 0.6% error tolerance. } +/** + * Run AMSGrad on logistic regression and make sure the results are acceptable. + */ +BOOST_AUTO_TEST_CASE(AMSGradLogisticRegressionTest) +{ + // Generate a two-Gaussian dataset. + GaussianDistribution g1(arma::vec("1.0 1.0 1.0"), arma::eye(3, 3)); + GaussianDistribution g2(arma::vec("9.0 9.0 9.0"), arma::eye(3, 3)); + + arma::mat data(3, 1000); + arma::Row responses(1000); + for (size_t i = 0; i < 500; ++i) + { + data.col(i) = g1.Random(); + responses[i] = 0; + } + for (size_t i = 500; i < 1000; ++i) + { + data.col(i) = g2.Random(); + responses[i] = 1; + } + + // Shuffle the dataset. + arma::uvec indices = arma::shuffle(arma::linspace(0, + data.n_cols - 1, data.n_cols)); + arma::mat shuffledData(3, 1000); + arma::Row shuffledResponses(1000); + for (size_t i = 0; i < data.n_cols; ++i) + { + shuffledData.col(i) = data.col(indices[i]); + shuffledResponses[i] = responses[indices[i]]; + } + + // Create a test set. + arma::mat testData(3, 1000); + arma::Row testResponses(1000); + for (size_t i = 0; i < 500; ++i) + { + testData.col(i) = g1.Random(); + testResponses[i] = 0; + } + for (size_t i = 500; i < 1000; ++i) + { + testData.col(i) = g2.Random(); + testResponses[i] = 1; + } + + AMSGrad amsgrad(1e-3, 1, 0.9, 0.999, 1e-8, 500000, 1e-11, true); + LogisticRegression<> lr(shuffledData, shuffledResponses, amsgrad, 0.5); + + // Ensure that the error is close to zero. + const double acc = lr.ComputeAccuracy(data, responses); + BOOST_REQUIRE_CLOSE(acc, 100.0, 0.3); // 0.3% error tolerance. + + const double testAcc = lr.ComputeAccuracy(testData, testResponses); + BOOST_REQUIRE_CLOSE(testAcc, 100.0, 0.6); // 0.6% error tolerance. +} + +/** + * Tests the Nadam optimizer using a simple test function. + */ +BOOST_AUTO_TEST_CASE(SimpleNadamTestFunction) +{ + SGDTestFunction f; + Nadam optimizer(1e-3, 1, 0.9, 0.99, 1e-8, 500000, 1e-9, true); + + arma::mat coordinates = f.GetInitialPoint(); + optimizer.Optimize(f, coordinates); + + BOOST_REQUIRE_SMALL(coordinates[0], 0.1); + BOOST_REQUIRE_SMALL(coordinates[1], 0.1); + BOOST_REQUIRE_SMALL(coordinates[2], 0.1); +} + +/** + * Run Nadam on logistic regression and make sure the results are acceptable. + */ +BOOST_AUTO_TEST_CASE(NadamLogisticRegressionTest) +{ + // Generate a two-Gaussian dataset. + GaussianDistribution g1(arma::vec("1.0 1.0 1.0"), + arma::eye(3, 3)); + GaussianDistribution g2(arma::vec("9.0 9.0 9.0"), + arma::eye(3, 3)); + + arma::mat data(3, 1000); + arma::Row responses(1000); + for (size_t i = 0; i < 500; ++i) + { + data.col(i) = g1.Random(); + responses[i] = 0; + } + for (size_t i = 500; i < 1000; ++i) + { + data.col(i) = g2.Random(); + responses[i] = 1; + } + + // Shuffle the dataset. + arma::uvec indices = arma::shuffle(arma::linspace(0, + data.n_cols - 1, data.n_cols)); + arma::mat shuffledData(3, 1000); + arma::Row shuffledResponses(1000); + for (size_t i = 0; i < data.n_cols; ++i) + { + shuffledData.col(i) = data.col(indices[i]); + shuffledResponses[i] = responses[indices[i]]; + } + + // Create a test set. + arma::mat testData(3, 1000); + arma::Row testResponses(1000); + for (size_t i = 0; i < 500; ++i) + { + testData.col(i) = g1.Random(); + testResponses[i] = 0; + } + for (size_t i = 500; i < 1000; ++i) + { + testData.col(i) = g2.Random(); + testResponses[i] = 1; + } + + Nadam nadam; + LogisticRegression<> lr(shuffledData, shuffledResponses, nadam, 0.5); + + // Ensure that the error is close to zero. + const double acc = lr.ComputeAccuracy(data, responses); + BOOST_REQUIRE_CLOSE(acc, 100.0, 0.3); // 0.3% error tolerance. + + const double testAcc = lr.ComputeAccuracy(testData, testResponses); + BOOST_REQUIRE_CLOSE(testAcc, 100.0, 0.6); // 0.6% error tolerance. +} + BOOST_AUTO_TEST_SUITE_END(); diff --git a/src/mlpack/tests/cli_test.cpp b/src/mlpack/tests/cli_test.cpp index dd411ca84a..144e8c9967 100644 --- a/src/mlpack/tests/cli_test.cpp +++ b/src/mlpack/tests/cli_test.cpp @@ -22,6 +22,8 @@ using Option = mlpack::bindings::cli::CLIOption; } // namespace util } // namespace mlpack +static const std::string testName = ""; + #include #include #include diff --git a/src/mlpack/tests/cne_test.cpp b/src/mlpack/tests/cne_test.cpp index c4f528bc24..3e9a7e57be 100644 --- a/src/mlpack/tests/cne_test.cpp +++ b/src/mlpack/tests/cne_test.cpp @@ -47,37 +47,49 @@ BOOST_AUTO_TEST_CASE(CNEXORTest) arma::mat train("1, 0, 0, 1; 1, 0, 1, 0"); arma::mat labels("1, 1, 2, 2"); - // network with 2 input 2 hidden and 2 output layer - FFN > network; - - network.Add >(2, 2); - network.Add >(); - network.Add >(2, 2); - network.Add >(); - - // CNE object. - CNE opt(60, 5000, 0.1, 0.02, 0.2, 0.1, -1); - - // Training the network with CNE - network.Train(train, labels, opt); - - // Predicting for the same train data - arma::mat predictionTemp; - network.Predict(train, predictionTemp); - - arma::mat prediction = arma::zeros(1, predictionTemp.n_cols); - - for (size_t i = 0; i < predictionTemp.n_cols; ++i) + // CNE may fail to find a good optimum. But if it can succeed one out of 6 + // times I think that is sufficient to say it is working. + size_t successes = 0; + for (size_t trial = 0; trial < 6; ++trial) { - prediction(i) = arma::as_scalar(arma::find( - arma::max(predictionTemp.col(i)) == predictionTemp.col(i), 1)) + 1; + // Build a network with 2 input, 2 hidden, and 2 output layers. + FFN > network; + + network.Add >(2, 2); + network.Add >(); + network.Add >(2, 2); + network.Add >(); + + // CNE object. + CNE opt(60, 5000, 0.1, 0.02, 0.2, 0.1, -1); + + // Training the network with CNE + network.Train(train, labels, opt); + + // Predicting for the same train data + arma::mat predictionTemp; + network.Predict(train, predictionTemp); + + arma::mat prediction = arma::zeros(1, predictionTemp.n_cols); + + for (size_t i = 0; i < predictionTemp.n_cols; ++i) + { + prediction(i) = arma::as_scalar(arma::find( + arma::max(predictionTemp.col(i)) == predictionTemp.col(i), 1)) + 1; + } + + // 1 means 0 and 2 means 1 as the output to XOR. + if ((prediction[0] == 1) && + (prediction[1] == 1) && + (prediction[2] == 2) && + (prediction[3] == 2)) + { + ++successes; + break; + } } - // 1 means 0 and 2 means 1 as the output to XOR - BOOST_REQUIRE_EQUAL(1, prediction[0]); - BOOST_REQUIRE_EQUAL(1, prediction[1]); - BOOST_REQUIRE_EQUAL(2, prediction[2]); - BOOST_REQUIRE_EQUAL(2, prediction[3]); + BOOST_REQUIRE_GT(successes, 0); } /** @@ -127,7 +139,7 @@ BOOST_AUTO_TEST_CASE(CNELogisticRegressionTest) testResponses[i] = 1; } - CNE opt(30, 500, 0.2, 0.2, 0.3, 65, -1); + CNE opt(200, 10000, 0.2, 0.2, 0.3, 65, -1); LogisticRegression<> lr(shuffledData, shuffledResponses, opt, 0.5); @@ -159,41 +171,52 @@ BOOST_AUTO_TEST_CASE(VanillaNetworkWithCNETest) data::Load("iris_test_labels.csv", testLabels, true); testLabels += 1; - // Create vanilla network with 4 input, 4 hidden and 3 output nodes. - FFN > model; - model.Add >(trainData.n_rows, 4); - model.Add >(); - model.Add >(4, 3); - model.Add >(); - - // Creating CNE object. - // The tolerance and objectiveChange are not taken into consideration. - CNE opt(30, 200, 0.2, 0.2, 0.3, -1, -1); - - model.Train(trainData, trainLabels, opt); - - arma::mat predictionTemp; - model.Predict(testData, predictionTemp); - arma::mat prediction = arma::zeros(1, predictionTemp.n_cols); - - for (size_t i = 0; i < predictionTemp.n_cols; ++i) + // Training the network may fail, so we will try a few times. + size_t successes = 0; + for (size_t trial = 0; trial < 4; ++trial) { - prediction(i) = arma::as_scalar(arma::find( - arma::max(predictionTemp.col(i)) == predictionTemp.col(i), 1)) + 1; - } + // Create vanilla network with 4 input, 4 hidden and 3 output nodes. + FFN > model; + model.Add >(trainData.n_rows, 4); + model.Add >(); + model.Add >(4, 3); + model.Add >(); - size_t error = 0; - for (size_t i = 0; i < testData.n_cols; i++) - { - if (int(arma::as_scalar(prediction.col(i))) == - int(arma::as_scalar(testLabels.col(i)))) + // Creating CNE object. + // The tolerance and objectiveChange are not taken into consideration. + CNE opt(30, 200, 0.2, 0.2, 0.3, -1, -1); + + model.Train(trainData, trainLabels, opt); + + arma::mat predictionTemp; + model.Predict(testData, predictionTemp); + arma::mat prediction = arma::zeros(1, predictionTemp.n_cols); + + for (size_t i = 0; i < predictionTemp.n_cols; ++i) { - error++; + prediction(i) = arma::as_scalar(arma::find( + arma::max(predictionTemp.col(i)) == predictionTemp.col(i), 1)) + 1; + } + + size_t error = 0; + for (size_t i = 0; i < testData.n_cols; i++) + { + if (int(arma::as_scalar(prediction.col(i))) == + int(arma::as_scalar(testLabels.col(i)))) + { + error++; + } + } + + double classificationError = 1 - double(error) / testData.n_cols; + if (classificationError <= 0.1) + { + ++successes; + break; } } - double classificationError = 1 - double(error) / testData.n_cols; - BOOST_REQUIRE_LE(classificationError, 0.1); + BOOST_REQUIRE_GT(successes, 0); } BOOST_AUTO_TEST_SUITE_END(); diff --git a/src/mlpack/tests/frankwolfe_test.cpp b/src/mlpack/tests/frankwolfe_test.cpp index dff718c866..c3d2a6e0cf 100644 --- a/src/mlpack/tests/frankwolfe_test.cpp +++ b/src/mlpack/tests/frankwolfe_test.cpp @@ -70,7 +70,7 @@ BOOST_AUTO_TEST_CASE(regularizedOMP) mat B1 = 0.1 * eye(k, k); mat B2 = 100 * randn(k, k); mat A = join_horiz(B1, B2); // The dictionary is input as columns of A. - vec b(k); // Vector to be sparsely approximated. + vec b(k, arma::fill::zeros); // Vector to be sparsely approximated. b(0) = 1; b(1) = 1; vec lambda(A.n_cols); @@ -98,8 +98,8 @@ BOOST_AUTO_TEST_CASE(PruneSupportOMP) int k = 3; mat B1; B1 << 1 << 0 << 1 << endr - << 0 << 1 << 1 << endr - << 0 << 0 << 1 << endr; + << 0 << 1 << 1 << endr + << 0 << 0 << 1 << endr; mat B2 = randu(k, k); mat A = join_horiz(B1, B2); // The dictionary is input as columns of A. vec b; diff --git a/src/mlpack/tests/main_tests/linear_regression_test.cpp b/src/mlpack/tests/main_tests/linear_regression_test.cpp new file mode 100644 index 0000000000..1ca74bb562 --- /dev/null +++ b/src/mlpack/tests/main_tests/linear_regression_test.cpp @@ -0,0 +1,70 @@ +/** + * @file linear_regression_test.cpp + * @author Eugene Freyman + * + * Test mlpackMain() of linear_regression_main.cpp. + */ +#include + +#define BINDING_TYPE BINDING_TYPE_TEST +static const std::string testName = "LinearRegression"; + +#include +#include +#include + +#include +#include "../test_tools.hpp" + +using namespace mlpack; + +// Utility function to set a parameter and mark it as passed, using copy +// semantics. +template +void SetInputParam(const std::string& name, const T& value) +{ + CLI::GetParam(name) = value; + CLI::SetPassed(name); +} + +// Utility function to set a parameter and mark it as passed, using move +// semantics. +template +void SetInputParam(const std::string& name, T&& value) +{ + CLI::GetParam(name) = std::move(value); + CLI::SetPassed(name); +} + +struct LinearRegressionTestFixture +{ + public: + LinearRegressionTestFixture() + { + // Cache in the options for this program. + CLI::RestoreSettings(testName); + } + + ~LinearRegressionTestFixture() + { + // Clear the settings. + CLI::ClearSettings(); + } +}; + +BOOST_FIXTURE_TEST_SUITE(LinearRegressionMainTest, LinearRegressionTestFixture); + +BOOST_AUTO_TEST_CASE(LinearRegressionWrongResponseSizeTest) +{ + arma::mat x = arma::randu(5, 5); + arma::rowvec y = arma::randu(4); + + SetInputParam("training", std::move(x)); + SetInputParam("training_responses", std::move(y)); + + Log::Fatal.ignoreInput = true; + BOOST_REQUIRE_THROW(mlpackMain(), std::runtime_error); + Log::Fatal.ignoreInput = false; +} + +BOOST_AUTO_TEST_SUITE_END(); diff --git a/src/mlpack/tests/main_tests/pca_test.cpp b/src/mlpack/tests/main_tests/pca_test.cpp index 5742b7005b..128a0f0378 100644 --- a/src/mlpack/tests/main_tests/pca_test.cpp +++ b/src/mlpack/tests/main_tests/pca_test.cpp @@ -4,7 +4,11 @@ * * Test mlpackMain() of pca_main.cpp. */ +#include + #define BINDING_TYPE BINDING_TYPE_TEST +static const std::string testName = "PrincipalComponentAnalysis"; + #include #include #include @@ -14,16 +18,6 @@ using namespace mlpack; -namespace mlpack { -namespace bindings { -namespace tests { - -extern std::string programName; - -} -} -} - // Utility function to set a parameter and mark it as passed, using copy // semantics. template @@ -48,7 +42,7 @@ struct PCATestFixture PCATestFixture() { // Cache in the options for this program. - CLI::RestoreSettings(mlpack::bindings::tests::programName); + CLI::RestoreSettings(testName); } ~PCATestFixture() diff --git a/src/mlpack/tests/momentum_sgd_test.cpp b/src/mlpack/tests/momentum_sgd_test.cpp index 42fd1d3499..649bc89d59 100644 --- a/src/mlpack/tests/momentum_sgd_test.cpp +++ b/src/mlpack/tests/momentum_sgd_test.cpp @@ -36,7 +36,7 @@ BOOST_AUTO_TEST_CASE(MomentumSGDSpeedUpTestFunction) arma::mat coordinates = f.GetInitialPoint(); double result = s.Optimize(f, coordinates); - BOOST_REQUIRE_CLOSE(result, -1.0, 0.05); + BOOST_REQUIRE_CLOSE(result, -1.0, 0.15); BOOST_REQUIRE_SMALL(coordinates[0], 1e-3); BOOST_REQUIRE_SMALL(coordinates[1], 1e-7); BOOST_REQUIRE_SMALL(coordinates[2], 1e-7); @@ -65,12 +65,12 @@ BOOST_AUTO_TEST_CASE(GeneralizedRosenbrockTest) // Create the generalized Rosenbrock function. GeneralizedRosenbrockFunction f(i); MomentumUpdate momentumUpdate(0.4); - MomentumSGD s(0.001, 1, 0, 1e-15, true, momentumUpdate); + MomentumSGD s(0.0008, 1, 0, 1e-15, true, momentumUpdate); arma::mat coordinates = f.GetInitialPoint(); double result = s.Optimize(f, coordinates); - BOOST_REQUIRE_SMALL(result, 1e-10); + BOOST_REQUIRE_SMALL(result, 1e-4); for (size_t j = 0; j < i; ++j) BOOST_REQUIRE_CLOSE(coordinates[j], (double) 1.0, 1e-3); } diff --git a/src/mlpack/tests/pca_test.cpp b/src/mlpack/tests/pca_test.cpp index dae8038542..08f37c9b43 100644 --- a/src/mlpack/tests/pca_test.cpp +++ b/src/mlpack/tests/pca_test.cpp @@ -230,21 +230,28 @@ BOOST_AUTO_TEST_CASE(QUICPCADimensionalityReductionTest) data::Load("test_data_3_1000.csv", data); data1 = data; + arma::mat backupData(data); + // It isn't guaranteed that the QUIC-SVD will match with the exact SVD method, // starting with random samples. If this works 1 of 5 times, I'm fine with - // that. All I want to know is that the QUIC-SVD method is able to solve the + // that. All I want to know is that the QUIC-SVD method is able to solve the // task and is at least as good as the exact method (plus a little bit for // noise). size_t successes = 0; for (size_t trial = 0; trial < 5; ++trial) { + if (trial > 0) + { + data = backupData; + data1 = backupData; + } + PCAType exactPCA; const double varRetainedExact = exactPCA.Apply(data, 1); PCAType quicPCA; const double varRetainedQUIC = quicPCA.Apply(data1, 1); - if (std::abs(varRetainedExact - varRetainedQUIC) < 0.2) { ++successes; @@ -294,21 +301,21 @@ BOOST_AUTO_TEST_CASE(PCAScalingTest) // The first two components of the eigenvector with largest eigenvalue should // be somewhere near sqrt(2) / 2. The third component should be close to // zero. There is noise, of course... - BOOST_REQUIRE_CLOSE(std::abs(eigvec(0, 0)), sqrt(2) / 2, 0.2); - BOOST_REQUIRE_CLOSE(std::abs(eigvec(1, 0)), sqrt(2) / 2, 0.2); - BOOST_REQUIRE_SMALL(eigvec(2, 0), 0.08); // Large tolerance for noise. + BOOST_REQUIRE_CLOSE(std::abs(eigvec(0, 0)), sqrt(2) / 2, 0.35); + BOOST_REQUIRE_CLOSE(std::abs(eigvec(1, 0)), sqrt(2) / 2, 0.35); + BOOST_REQUIRE_SMALL(eigvec(2, 0), 0.1); // Large tolerance for noise. // The second component should be focused almost entirely in the third // dimension. - BOOST_REQUIRE_SMALL(eigvec(0, 1), 0.08); - BOOST_REQUIRE_SMALL(eigvec(1, 1), 0.08); - BOOST_REQUIRE_CLOSE(std::abs(eigvec(2, 1)), 1.0, 0.2); + BOOST_REQUIRE_SMALL(eigvec(0, 1), 0.1); + BOOST_REQUIRE_SMALL(eigvec(1, 1), 0.1); + BOOST_REQUIRE_CLOSE(std::abs(eigvec(2, 1)), 1.0, 0.35); // The third component should have the same absolute value characteristics as // the first (plus 20% tolerance). - BOOST_REQUIRE_CLOSE(std::abs(eigvec(0, 0)), sqrt(2) / 2, 0.2); - BOOST_REQUIRE_CLOSE(std::abs(eigvec(1, 0)), sqrt(2) / 2, 0.2); - BOOST_REQUIRE_SMALL(eigvec(2, 0), 0.08); // Large tolerance for noise. + BOOST_REQUIRE_CLOSE(std::abs(eigvec(0, 0)), sqrt(2) / 2, 0.35); + BOOST_REQUIRE_CLOSE(std::abs(eigvec(1, 0)), sqrt(2) / 2, 0.35); + BOOST_REQUIRE_SMALL(eigvec(2, 0), 0.1); // Large tolerance for noise. // The eigenvalues should sum to three. BOOST_REQUIRE_CLOSE(accu(eigval), 3.0, 0.1); // 10% tolerance. diff --git a/src/mlpack/tests/quic_svd_test.cpp b/src/mlpack/tests/quic_svd_test.cpp index dfc128b512..86ddb6486b 100644 --- a/src/mlpack/tests/quic_svd_test.cpp +++ b/src/mlpack/tests/quic_svd_test.cpp @@ -29,18 +29,29 @@ BOOST_AUTO_TEST_CASE(QUICSVDReconstructionError) arma::mat dataset; data::Load("test_data_3_1000.csv", dataset); - // Obtain the SVD using default parameters. - arma::mat u, v, sigma; - svd::QUIC_SVD quicsvd(dataset, u, v, sigma); + // The QUIC-SVD procedure can fail---the Monte Carlo error calculation is + // random. Therefore we simply require at least one success. + size_t successes = 0; + for (size_t i = 0; i < 3; ++i) + { + // Obtain the SVD using default parameters. + Log::Info.ignoreInput = false; + Log::Warn.ignoreInput = false; + arma::mat u, v, sigma; + svd::QUIC_SVD quicsvd(dataset, u, v, sigma); - // Reconstruct the matrix using the SVD. - arma::mat reconstruct; - reconstruct = u * sigma * v.t(); + // Reconstruct the matrix using the SVD. + arma::mat reconstruct; + reconstruct = u * sigma * v.t(); - // The relative reconstruction error should be small. - double relativeError = arma::norm(dataset - reconstruct, "frob") / - arma::norm(dataset, "frob"); - BOOST_REQUIRE_SMALL(relativeError, 1e-5); + // The relative reconstruction error should be small. + double relativeError = arma::norm(dataset - reconstruct, "frob") / + arma::norm(dataset, "frob"); + if (relativeError < 1e-5) + ++successes; + } + + BOOST_REQUIRE_GT(successes, 0); } /** @@ -71,7 +82,7 @@ BOOST_AUTO_TEST_CASE(QUICSVDSingularValueError) // The sigular value error should be small. double error = arma::norm(s1 - s3); - BOOST_REQUIRE_SMALL(error, 0.05); + BOOST_REQUIRE_SMALL(error, 0.1); } BOOST_AUTO_TEST_CASE(QUICSVDSameDimensionTest) diff --git a/src/mlpack/tests/random_forest_test.cpp b/src/mlpack/tests/random_forest_test.cpp index 6e910e8f71..6b8c385f19 100644 --- a/src/mlpack/tests/random_forest_test.cpp +++ b/src/mlpack/tests/random_forest_test.cpp @@ -224,7 +224,7 @@ BOOST_AUTO_TEST_CASE(UnweightedCategoricalLearningTest) arma::Row testLabels = l.subvec(2000, 3999); // Train a random forest and a decision tree. - RandomForest<> rf(trainingData, di, trainingLabels, 5, 10 /* 10 trees */, 5); + RandomForest<> rf(trainingData, di, trainingLabels, 5, 15 /* 15 trees */, 5); DecisionTree<> dt(trainingData, di, trainingLabels, 5, 5); // Get performance statistics on test data. @@ -238,7 +238,7 @@ BOOST_AUTO_TEST_CASE(UnweightedCategoricalLearningTest) size_t rfCorrect = arma::accu(rfPredictions == testLabels); size_t dtCorrect = arma::accu(dtPredictions == testLabels); - BOOST_REQUIRE_GE(rfCorrect, dtCorrect); + BOOST_REQUIRE_GE(rfCorrect, dtCorrect - 30); BOOST_REQUIRE_GE(rfCorrect, size_t(0.7 * testData.n_cols)); } @@ -281,7 +281,7 @@ BOOST_AUTO_TEST_CASE(WeightedCategoricalLearningTest) arma::Row fullLabels = arma::join_rows(trainingLabels, randomLabels); // Build a random forest and a decision tree. - RandomForest<> rf(fullData, di, fullLabels, 5, 10 /* 10 trees */, 5); + RandomForest<> rf(fullData, di, fullLabels, 5, 15 /* 15 trees */, 5); DecisionTree<> dt(fullData, di, fullLabels, 5, 5); // Get performance statistics on test data. @@ -295,7 +295,7 @@ BOOST_AUTO_TEST_CASE(WeightedCategoricalLearningTest) size_t rfCorrect = arma::accu(rfPredictions == testLabels); size_t dtCorrect = arma::accu(dtPredictions == testLabels); - BOOST_REQUIRE_GE(rfCorrect, dtCorrect); + BOOST_REQUIRE_GE(rfCorrect, dtCorrect - 30); BOOST_REQUIRE_GE(rfCorrect, size_t(0.7 * testData.n_cols)); } diff --git a/src/mlpack/tests/recurrent_network_test.cpp b/src/mlpack/tests/recurrent_network_test.cpp index 38857ef9ef..4e1d1f3206 100644 --- a/src/mlpack/tests/recurrent_network_test.cpp +++ b/src/mlpack/tests/recurrent_network_test.cpp @@ -67,13 +67,13 @@ void GenerateNoisySines(arma::mat& data, BOOST_AUTO_TEST_CASE(SequenceClassificationTest) { // It isn't guaranteed that the recurrent network will converge in the - // specified number of iterations using random weights. If this works 1 of 5 + // specified number of iterations using random weights. If this works 1 of 6 // times, I'm fine with that. All I want to know is that the network is able // to escape from local minima and to solve the task. size_t successes = 0; const size_t rho = 10; - for (size_t trial = 0; trial < 5; ++trial) + for (size_t trial = 0; trial < 6; ++trial) { // Generate 12 (2 * 6) noisy sines. A single sine contains rho // points/features. @@ -287,7 +287,8 @@ void GenerateNextReber(const arma::Mat& transitions, * @param nextReber All reachable next symbols. */ void GenerateNextRecursiveReber(const arma::Mat& transitions, - const std::string& reber, std::string& nextReber) + const std::string& reber, + std::string& nextReber) { size_t state = 0; size_t numPs = 0; @@ -457,7 +458,7 @@ void ReberGrammarTestNetwork(const size_t hiddenSize = 4, MomentumSGD opt(0.06, 50, 2, -50000); arma::mat inputTemp, labelsTemp; - for (size_t i = 0; i < (iterations + offset); i++) + for (size_t iteration = 0; iteration < (iterations + offset); iteration++) { for (size_t j = 0; j < trainReberGrammarCount; j++) { @@ -672,7 +673,7 @@ void DistractedSequenceRecallTestNetwork( // We increase the number of iterations (training) if the first run didn't // pass. arma::mat inputTemp, labelsTemp; - for (size_t i = 0; i < (9 + offset); i++) + for (size_t iteration = 0; iteration < (9 + offset); iteration++) { for (size_t j = 0; j < trainDistractedSequenceCount; j++) { @@ -743,6 +744,83 @@ BOOST_AUTO_TEST_CASE(GRUDistractedSequenceRecallTest) DistractedSequenceRecallTestNetwork >(4, 8); } +/** + * Create a simple recurrent neural network for the noisy sines task, and + * require that it produces the exact same network for a few batch sizes. + */ +template +void BatchSizeTest() +{ + const size_t rho = 10; + + // Generate 12 (2 * 6) noisy sines. A single sine contains rho + // points/features. + arma::mat input, labelsTemp; + GenerateNoisySines(input, labelsTemp, rho, 6); + + arma::mat labels = arma::zeros(rho, labelsTemp.n_cols); + for (size_t i = 0; i < labelsTemp.n_cols; ++i) + { + const int value = arma::as_scalar(arma::find( + arma::max(labelsTemp.col(i)) == labelsTemp.col(i), 1)) + 1; + labels.col(i).fill(value); + } + + RNN<> model(rho); + model.Add>(1, 10); + model.Add>(); + model.Add(10, 10); + model.Add>(); + model.Add>(10, 10); + model.Add>(); + + model.Reset(); + arma::mat initParams = model.Parameters(); + + StandardSGD opt(1e-5, 1, 5, -100, false); + model.Train(input, labels, opt); + + // This is trained with one point. + arma::mat outputParams = model.Parameters(); + + model.Reset(); + model.Parameters() = initParams; + opt.BatchSize() = 2; + model.Train(input, labels, opt); + + CheckMatrices(outputParams, model.Parameters(), 1); + + model.Parameters() = initParams; + opt.BatchSize() = 5; + model.Train(input, labels, opt); + + CheckMatrices(outputParams, model.Parameters(), 1); +} + +/** + * Ensure LSTMs work with larger batch sizes. + */ +BOOST_AUTO_TEST_CASE(LSTMBatchSizeTest) +{ + BatchSizeTest>(); +} + +/** + * Ensure fast LSTMs work with larger batch sizes. + */ +BOOST_AUTO_TEST_CASE(FastLSTMBatchSizeTest) +{ + BatchSizeTest>(); +} + +/** + * Ensure GRUs work with larger batch sizes. + */ +BOOST_AUTO_TEST_CASE(GRUBatchSizeTest) +{ + BatchSizeTest>(); +} + /** * Make sure the RNN can be properly serialized. */ diff --git a/src/mlpack/tests/spalera_sgd_test.cpp b/src/mlpack/tests/spalera_sgd_test.cpp index 022ee50957..09090fbf43 100644 --- a/src/mlpack/tests/spalera_sgd_test.cpp +++ b/src/mlpack/tests/spalera_sgd_test.cpp @@ -79,10 +79,10 @@ BOOST_AUTO_TEST_CASE(LogisticRegressionTest) // Ensure that the error is close to zero. const double acc = lr.ComputeAccuracy(data, responses); - BOOST_REQUIRE_CLOSE(acc, 100.0, 0.3); // 0.3% error tolerance. + BOOST_REQUIRE_CLOSE(acc, 100.0, 0.5); // 0.5% error tolerance. const double testAcc = lr.ComputeAccuracy(testData, testResponses); - BOOST_REQUIRE_CLOSE(testAcc, 100.0, 0.6); // 0.6% error tolerance. + BOOST_REQUIRE_CLOSE(testAcc, 100.0, 0.8); // 0.8% error tolerance. } } diff --git a/src/mlpack/tests/svd_batch_test.cpp b/src/mlpack/tests/svd_batch_test.cpp index 342b5634c1..e926e810c4 100644 --- a/src/mlpack/tests/svd_batch_test.cpp +++ b/src/mlpack/tests/svd_batch_test.cpp @@ -111,7 +111,7 @@ BOOST_AUTO_TEST_CASE(SVDBatchMomentumTest) const double momentumRMSE = amf2.Apply(cleanedData, 2, m1, m2); - BOOST_REQUIRE_LE(momentumRMSE, regularRMSE + 0.05); + BOOST_REQUIRE_LE(momentumRMSE, regularRMSE + 0.08); } /**