diff --git a/src/mlpack/core/optimizers/adam/CMakeLists.txt b/src/mlpack/core/optimizers/adam/CMakeLists.txt index 3cbcfd84e7..1377bbb0e1 100644 --- a/src/mlpack/core/optimizers/adam/CMakeLists.txt +++ b/src/mlpack/core/optimizers/adam/CMakeLists.txt @@ -1,6 +1,8 @@ set(SOURCES adam.hpp adam_impl.hpp + adam_update.hpp + adamax_update.hpp ) set(DIR_SRCS) diff --git a/src/mlpack/core/optimizers/adam/adam.hpp b/src/mlpack/core/optimizers/adam/adam.hpp index 62af4c6ac8..d4808c6cd9 100644 --- a/src/mlpack/core/optimizers/adam/adam.hpp +++ b/src/mlpack/core/optimizers/adam/adam.hpp @@ -20,6 +20,10 @@ #include +#include +#include "adam_update.hpp" +#include "adamax_update.hpp" + namespace mlpack { namespace optimization { @@ -36,7 +40,8 @@ namespace optimization { * author = {Diederik P. Kingma and Jimmy Ba}, * title = {Adam: {A} Method for Stochastic Optimization}, * journal = {CoRR}, - * year = {2014} + * year = {2014}, + * url = {http://arxiv.org/abs/1412.6980} * } * @endcode * @@ -60,9 +65,13 @@ namespace optimization { * * @tparam DecomposableFunctionType Decomposable objective function type to be * minimized. + * @tparam UpdateRule Adam optimizer update rule to be used. */ -template -class Adam +template< + typename DecomposableFunctionType, + typename UpdateRule = AdamUpdate +> +class AdamType { public: /** @@ -84,18 +93,15 @@ class Adam * @param tolerance Maximum absolute tolerance to terminate algorithm. * @param shuffle If true, the function order is shuffled; otherwise, each * function is visited in linear order. - * @param adaMax If true, then the AdaMax optimizer is used; otherwise, by - * default the Adam optimizer is used. */ - Adam(DecomposableFunctionType& function, - const double stepSize = 0.001, - const double beta1 = 0.9, - const double beta2 = 0.999, - const double eps = 1e-8, - const size_t maxIterations = 100000, - const double tolerance = 1e-5, - const bool shuffle = true, - const bool adaMax = false); + AdamType(DecomposableFunctionType& function, + const double stepSize = 0.001, + const double beta1 = 0.9, + const double beta2 = 0.999, + const double eps = 1e-8, + const size_t maxIterations = 100000, + const double tolerance = 1e-5, + const bool shuffle = true); /** * Optimize the given function using Adam. The given starting point will be @@ -105,83 +111,62 @@ class Adam * @param iterate Starting point (will be modified). * @return Objective value of the final point. */ - double Optimize(arma::mat& iterate); + double Optimize(arma::mat& iterate){ return optimizer.Optimize(iterate); } //! Get the instantiated function to be optimized. - const DecomposableFunctionType& Function() const { return function; } + const DecomposableFunctionType& Function() const + { + return optimizer.Function(); + } //! Modify the instantiated function. - DecomposableFunctionType& Function() { return function; } + DecomposableFunctionType& Function() { return optimizer.Function(); } //! Get the step size. - double StepSize() const { return stepSize; } + double StepSize() const { return optimizer.StepSize(); } //! Modify the step size. - double& StepSize() { return stepSize; } + double& StepSize() { return optimizer.StepSize(); } //! Get the smoothing parameter. - double Beta1() const { return beta1; } + double Beta1() const { return optimizer.UpdatePolicy().Beta1(); } //! Modify the smoothing parameter. - double& Beta1() { return beta1; } + double& Beta1() { return optimizer.UpdatePolicy().Beta1(); } //! Get the second moment coefficient. - double Beta2() const { return beta2; } + double Beta2() const { return optimizer.UpdatePolicy().Beta2(); } //! Modify the second moment coefficient. - double& Beta2() { return beta2; } + double& Beta2() { return optimizer.UpdatePolicy().Beta2(); } //! Get the value used to initialise the mean squared gradient parameter. - double Epsilon() const { return eps; } + double Epsilon() const { return optimizer.UpdatePolicy().Epsilon(); } //! Modify the value used to initialise the mean squared gradient parameter. - double& Epsilon() { return eps; } + double& Epsilon() { return optimizer.UpdatePolicy().Epsilon(); } //! Get the maximum number of iterations (0 indicates no limit). - size_t MaxIterations() const { return maxIterations; } + size_t MaxIterations() const { return optimizer.MaxIterations(); } //! Modify the maximum number of iterations (0 indicates no limit). - size_t& MaxIterations() { return maxIterations; } + size_t& MaxIterations() { return optimizer.MaxIterations(); } //! Get the tolerance for termination. - double Tolerance() const { return tolerance; } + double Tolerance() const { return optimizer.Tolerance(); } //! Modify the tolerance for termination. - double& Tolerance() { return tolerance; } + double& Tolerance() { return optimizer.Tolerance(); } //! Get whether or not the individual functions are shuffled. - bool Shuffle() const { return shuffle; } + bool Shuffle() const { return optimizer.Shuffle(); } //! Modify whether or not the individual functions are shuffled. - bool& Shuffle() { return shuffle; } - - //! Get whether or not the AdaMax optimizer is specified. - bool AdaMax() const { return adaMax; } - //! Modify wehther or not the AdaMax optimizer is to be used. - bool& AdaMax() { return adaMax; } + bool& Shuffle() { return optimizer.Shuffle(); } private: - //! The instantiated function. - DecomposableFunctionType& function; - - //! The step size for each example. - double stepSize; - - //! Exponential decay rate for the first moment estimates. - double beta1; - - //! Exponential decay rate for the weighted infinity norm estimates. - double beta2; - - //! The value used to initialise the mean squared gradient parameter. - double eps; - - //! The maximum number of allowed iterations. - size_t maxIterations; - - //! The tolerance for termination. - double tolerance; - - //! Controls whether or not the individual functions are shuffled when - //! iterating. - bool shuffle; - - //! Specifies whether or not the AdaMax optimizer is to be used. - bool adaMax; + //! The Stochastic Gradient Descent object with Adam policy. + SGD optimizer; }; +template +using Adam = AdamType; + +template +using AdaMax = AdamType; + } // namespace optimization } // namespace mlpack diff --git a/src/mlpack/core/optimizers/adam/adam_impl.hpp b/src/mlpack/core/optimizers/adam/adam_impl.hpp index 8a6be66c07..82408de71e 100644 --- a/src/mlpack/core/optimizers/adam/adam_impl.hpp +++ b/src/mlpack/core/optimizers/adam/adam_impl.hpp @@ -21,161 +21,26 @@ namespace mlpack { namespace optimization { -template -Adam::Adam(DecomposableFunctionType& function, - const double stepSize, - const double beta1, - const double beta2, - const double eps, - const size_t maxIterations, - const double tolerance, - const bool shuffle, - const bool adaMax) : - function(function), - stepSize(stepSize), - beta1(beta1), - beta2(beta2), - eps(eps), - maxIterations(maxIterations), - tolerance(tolerance), - shuffle(shuffle), - adaMax(adaMax) +template +AdamType::AdamType( + DecomposableFunctionType& function, + const double stepSize, + const double beta1, + const double beta2, + const double epsilon, + const size_t maxIterations, + const double tolerance, + const bool shuffle) : + optimizer(function, + stepSize, + maxIterations, + tolerance, + shuffle, + UpdateRule(epsilon, + beta1, + beta2)) { /* Nothing to do. */ } -//! Optimize the function (minimize). -template -double Adam::Optimize(arma::mat& iterate) -{ - // Find the number of functions to use. - const size_t numFunctions = function.NumFunctions(); - - // This is used only if shuffle is true. - arma::Col visitationOrder; - if (shuffle) - visitationOrder = arma::shuffle(arma::linspace>(0, - (numFunctions - 1), numFunctions)); - - // To keep track of where we are and how things are going. - size_t currentFunction = 0; - double overallObjective = 0; - double lastObjective = DBL_MAX; - - // Calculate the first objective function. - for (size_t i = 0; i < numFunctions; ++i) - overallObjective += function.Evaluate(iterate, i); - - // Now iterate! - arma::mat gradient(iterate.n_rows, iterate.n_cols); - - // Exponential moving average of gradient values. - arma::mat m = arma::zeros(iterate.n_rows, iterate.n_cols); - - /** - * Initialize either the exponentially weighted infinity norm for AdaMax - * optimizer (u) or exponential moving average of squared gradient values - * for Adam optimizer (v). - */ - arma::mat u, v; - if (adaMax) - { - u = arma::zeros(iterate.n_rows, iterate.n_cols); - } - else - { - v = arma::zeros(iterate.n_rows, iterate.n_cols); - } - - for (size_t i = 1; i != maxIterations; ++i, ++currentFunction) - { - // Is this iteration the start of a sequence? - if ((currentFunction % numFunctions) == 0) - { - // Output current objective function. - Log::Info << "Adam: iteration " << i << ", objective " << overallObjective - << "." << std::endl; - - if (std::isnan(overallObjective) || std::isinf(overallObjective)) - { - Log::Warn << "Adam: converged to " << overallObjective - << "; terminating with failure. Try a smaller step size?" - << std::endl; - return overallObjective; - } - - if (std::abs(lastObjective - overallObjective) < tolerance) - { - Log::Info << "Adam: minimized within tolerance " << tolerance << "; " - << "terminating optimization." << std::endl; - return overallObjective; - } - - // Reset the counter variables. - lastObjective = overallObjective; - overallObjective = 0; - currentFunction = 0; - - if (shuffle) // Determine order of visitation. - visitationOrder = arma::shuffle(visitationOrder); - } - - // Evaluate the gradient for this iteration. - if (shuffle) - function.Gradient(iterate, visitationOrder[currentFunction], gradient); - else - function.Gradient(iterate, currentFunction, gradient); - - // And update the iterate. - m *= beta1; - m += (1 - beta1) * gradient; - - if (adaMax) - { - // Update the exponentially weighted infinity norm. - u *= beta2; - u = arma::max(u, arma::abs(gradient)); - } - else - { - v *= beta2; - v += (1 - beta2) * (gradient % gradient); - } - - const double biasCorrection1 = 1.0 - std::pow(beta1, (double) i); - const double biasCorrection2 = 1.0 - std::pow(beta2, (double) i); - - if (adaMax) - { - if (biasCorrection1 != 0.0) - iterate -= (stepSize / biasCorrection1 * m / (u + eps)); - } - else - { - /** - * It should be noted that the term, m / (arma::sqrt(v) + eps), in the - * following expression is an approximation of the following actual term; - * m / (arma::sqrt(v) + (arma::sqrt(biasCorrection2) * eps). - */ - iterate -= (stepSize * std::sqrt(biasCorrection2) / biasCorrection1) * - m / (arma::sqrt(v) + eps); - } - - // Now add that to the overall objective function. - if (shuffle) - overallObjective += function.Evaluate(iterate, - visitationOrder[currentFunction]); - else - overallObjective += function.Evaluate(iterate, currentFunction); - } - - Log::Info << "Adam: maximum iterations (" << maxIterations << ") reached; " - << "terminating optimization." << std::endl; - // Calculate final objective. - overallObjective = 0; - for (size_t i = 0; i < numFunctions; ++i) - overallObjective += function.Evaluate(iterate, i); - return overallObjective; -} - } // namespace optimization } // namespace mlpack diff --git a/src/mlpack/core/optimizers/adam/adam_update.hpp b/src/mlpack/core/optimizers/adam/adam_update.hpp new file mode 100644 index 0000000000..540ae1ce01 --- /dev/null +++ b/src/mlpack/core/optimizers/adam/adam_update.hpp @@ -0,0 +1,149 @@ +/** + * @file adam_update.hpp + * @author Ryan Curtin + * @author Vasanth Kalingeri + * @author Marcus Edel + * @author Vivek Pal + * + * Adam optimizer. Adam is an an algorithm for first-order gradient-based + * optimization of stochastic objective functions, based on adaptive estimates + * of lower-order moments. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_CORE_OPTIMIZERS_ADAM_ADAM_UPDATE_HPP +#define MLPACK_CORE_OPTIMIZERS_ADAM_ADAM_UPDATE_HPP + +#include + +namespace mlpack { +namespace optimization { + +/** + * Adam is an optimizer that computes individual adaptive learning rates for + * different parameters from estimates of first and second moments of the + * gradients as given in the section 7 of the following paper. + * + * For more information, see the following. + * + * @code + * @article{Kingma2014, + * author = {Diederik P. Kingma and Jimmy Ba}, + * title = {Adam: {A} Method for Stochastic Optimization}, + * journal = {CoRR}, + * year = {2014}, + * url = {http://arxiv.org/abs/1412.6980} + * } + * @endcode + */ +class AdamUpdate +{ + public: + /** + * Construct the Adam update policy with the given parameters. + * + * @param epsilon The epsilon value used to initialise the squared gradient + * parameter. + * @param beta1 The smoothing parameter. + * @param beta2 The second moment coefficient. + */ + AdamUpdate(const double epsilon = 1e-8, + const double beta1 = 0.9, + const double beta2 = 0.999) : + epsilon(epsilon), + beta1(beta1), + beta2(beta2), + iteration(0) + { + // Nothing to do. + } + + /** + * The Initialize method is called by SGD Optimizer method before the start of + * the iteration update process. + * + * @param rows number of rows in the gradient matrix. + * @param cols number of columns in the gradient matrix. + */ + void Initialize(const size_t rows, + const size_t cols) + { + m = arma::zeros(rows, cols); + v = arma::zeros(rows, cols); + } + + /** + * Update step for Adam. + * + * @param iterate Parameters that minimize the function. + * @param stepSize Step size to be used for the given iteration. + * @param gradient The gradient matrix. + */ + void Update(arma::mat& iterate, + const double stepSize, + const arma::mat& gradient) + { + // Increment the iteration counter variable. + ++iteration; + + // And update the iterate. + m *= beta1; + m += (1 - beta1) * gradient; + + v *= beta2; + v += (1 - beta2) * (gradient % gradient); + + const double biasCorrection1 = 1.0 - std::pow(beta1, (double) iteration); + const double biasCorrection2 = 1.0 - std::pow(beta2, (double) iteration); + + /** + * It should be noted that the term, m / (arma::sqrt(v) + eps), in the + * following expression is an approximation of the following actual term; + * m / (arma::sqrt(v) + (arma::sqrt(biasCorrection2) * eps). + */ + iterate -= (stepSize * std::sqrt(biasCorrection2) / biasCorrection1) * + m / (arma::sqrt(v) + epsilon); + } + + //! Get the value used to initialise the squared gradient parameter. + double Epsilon() const { return epsilon; } + //! Modify the value used to initialise the squared gradient parameter. + double& Epsilon() { return epsilon; } + + //! Get the smoothing parameter. + double Beta1() const { return beta1; } + //! Modify the smoothing parameter. + double& Beta1() { return beta1; } + + //! Get the second moment coefficient. + double Beta2() const { return beta2; } + //! Modify the second moment coefficient. + double& Beta2() { return beta2; } + + private: + // The epsilon value used to initialise the squared gradient parameter. + double epsilon; + + // The smoothing parameter. + double beta1; + + // The second moment coefficient. + double beta2; + + // The exponential moving average of gradient values. + arma::mat m; + + // The exponential moving average of squared gradient values. + arma::mat v; + + // The number of iterations. + double iteration; +}; + +} // namespace optimization +} // namespace mlpack + +#endif diff --git a/src/mlpack/core/optimizers/adam/adamax_update.hpp b/src/mlpack/core/optimizers/adam/adamax_update.hpp new file mode 100644 index 0000000000..6337efaae3 --- /dev/null +++ b/src/mlpack/core/optimizers/adam/adamax_update.hpp @@ -0,0 +1,146 @@ +/** + * @file adamax_update.hpp + * @author Ryan Curtin + * @author Vasanth Kalingeri + * @author Marcus Edel + * @author Vivek Pal + * + * AdaMax update rule. Adam is an an algorithm for first-order gradient- + * -based optimization of stochastic objective functions, based on adaptive + * estimates of lower-order moments. AdaMax is simply a variant of Adam based + * on the infinity norm. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_CORE_OPTIMIZERS_ADAM_ADAMAX_UPDATE_HPP +#define MLPACK_CORE_OPTIMIZERS_ADAM_ADAMAX_UPDATE_HPP + +#include + +namespace mlpack { +namespace optimization { + +/** + * AdaMax is a variant of Adam, an optimizer that computes individual adaptive + * learning rates for different parameters from estimates of first and second + * moments of the gradients.based on the infinity norm as given in the section + * 7 of the following paper. + * + * For more information, see the following. + * + * @code + * @article{Kingma2014, + * author = {Diederik P. Kingma and Jimmy Ba}, + * title = {Adam: {A} Method for Stochastic Optimization}, + * journal = {CoRR}, + * year = {2014}, + * url = {http://arxiv.org/abs/1412.6980} + * } + * @endcode + */ +class AdaMaxUpdate +{ + public: + /** + * Construct the AdaMax update policy with the given parameters. + * + * @param epsilon The epsilon value used to initialise the squared gradient + * parameter. + * @param beta1 The smoothing parameter. + * @param beta2 The second moment coefficient. + */ + AdaMaxUpdate(const double epsilon = 1e-8, + const double beta1 = 0.9, + const double beta2 = 0.999) : + epsilon(epsilon), + beta1(beta1), + beta2(beta2), + iteration(0) + { + // Nothing to do. + } + + /** + * The Initialize method is called by SGD Optimizer method before the start of + * the iteration update process. + * + * @param rows number of rows in the gradient matrix. + * @param cols number of columns in the gradient matrix. + */ + void Initialize(const size_t rows, + const size_t cols) + { + m = arma::zeros(rows, cols); + u = arma::zeros(rows, cols); + } + + /** + * Update step for Adam. + * + * @param iterate Parameters that minimize the function. + * @param stepSize Step size to be used for the given iteration. + * @param gradient The gradient matrix. + */ + void Update(arma::mat& iterate, + const double stepSize, + const arma::mat& gradient) + { + // Increment the iteration counter variable. + ++iteration; + + // And update the iterate. + m *= beta1; + m += (1 - beta1) * gradient; + + // Update the exponentially weighted infinity norm. + u *= beta2; + u = arma::max(u, arma::abs(gradient)); + + const double biasCorrection1 = 1.0 - std::pow(beta1, iteration); + + if (biasCorrection1 != 0) + iterate -= (stepSize / biasCorrection1 * m / (u + epsilon)); + } + + //! Get the value used to initialise the squared gradient parameter. + double Epsilon() const { return epsilon; } + //! Modify the value used to initialise the squared gradient parameter. + double& Epsilon() { return epsilon; } + + //! Get the smoothing parameter. + double Beta1() const { return beta1; } + //! Modify the smoothing parameter. + double& Beta1() { return beta1; } + + //! Get the second moment coefficient. + double Beta2() const { return beta2; } + //! Modify the second moment coefficient. + double& Beta2() { return beta2; } + + private: + // The epsilon value used to initialise the squared gradient parameter. + double epsilon; + + // The smoothing parameter. + double beta1; + + // The second moment coefficient. + double beta2; + + // The exponential moving average of gradient values. + arma::mat m; + + // The exponentially weighted infinity norm. + arma::mat u; + + // The number of iterations. + double iteration; +}; + +} // namespace optimization +} // namespace mlpack + +#endif diff --git a/src/mlpack/methods/logistic_regression/logistic_regression.hpp b/src/mlpack/methods/logistic_regression/logistic_regression.hpp index 7a03a0a04e..a19f31b876 100644 --- a/src/mlpack/methods/logistic_regression/logistic_regression.hpp +++ b/src/mlpack/methods/logistic_regression/logistic_regression.hpp @@ -119,7 +119,7 @@ class LogisticRegression * @param responses Outputs results from input training variables. */ template< - template class OptimizerType = mlpack::optimization::L_BFGS + template class OptimizerType = mlpack::optimization::L_BFGS > void Train(const MatType& predictors, const arma::Row& responses); diff --git a/src/mlpack/methods/logistic_regression/logistic_regression_impl.hpp b/src/mlpack/methods/logistic_regression/logistic_regression_impl.hpp index 4974c975a5..9494525847 100644 --- a/src/mlpack/methods/logistic_regression/logistic_regression_impl.hpp +++ b/src/mlpack/methods/logistic_regression/logistic_regression_impl.hpp @@ -66,7 +66,7 @@ LogisticRegression::LogisticRegression( } template -template class OptimizerType> +template class OptimizerType> void LogisticRegression::Train(const MatType& predictors, const arma::Row& responses) { diff --git a/src/mlpack/tests/adam_test.cpp b/src/mlpack/tests/adam_test.cpp index 88d839fd5c..a88e3618f8 100644 --- a/src/mlpack/tests/adam_test.cpp +++ b/src/mlpack/tests/adam_test.cpp @@ -36,7 +36,8 @@ BOOST_AUTO_TEST_SUITE(AdamTest); BOOST_AUTO_TEST_CASE(SimpleAdamTestFunction) { SGDTestFunction f; - Adam optimizer(f, 1e-3, 0.9, 0.999, 1e-8, 5000000, 1e-9, true); + Adam optimizer(f, 1e-3, 0.9, 0.999, 1e-8, 5000000, 1e-9, + true); arma::mat coordinates = f.GetInitialPoint(); optimizer.Optimize(coordinates); @@ -52,8 +53,8 @@ BOOST_AUTO_TEST_CASE(SimpleAdamTestFunction) BOOST_AUTO_TEST_CASE(SimpleAdaMaxTestFunction) { SGDTestFunction f; - Adam optimizer(f, 2e-3, 0.9, 0.999, 1e-8, 5000000, 1e-9, true - ,true); + AdaMax optimizer(f, 2e-3, 0.9, 0.999, 1e-8, 5000000, 1e-9, + true); arma::mat coordinates = f.GetInitialPoint(); optimizer.Optimize(coordinates); @@ -174,8 +175,9 @@ BOOST_AUTO_TEST_CASE(AdaMaxLogisticRegressionTest) LogisticRegression<> lr(shuffledData.n_rows, 0.5); LogisticRegressionFunction<> lrf(shuffledData, shuffledResponses, 0.5); - Adam > adamax(lrf, 1e-3, 0.9, 0.999, 1e-8, 5000000, - 1e-9, true, true); + AdaMax > adamax(lrf, 1e-3, 0.9, 0.999, 1e-8, + 5000000, 1e-9, true); + lr.Train(adamax); // Ensure that the error is close to zero.