From 7890547e2aebf1e2ed06736bb3535f00d30c94ce Mon Sep 17 00:00:00 2001 From: Abhinav Moudgil Date: Sat, 25 Mar 2017 15:46:35 +0530 Subject: [PATCH] Add AdaDelta policy class --- src/mlpack/core/optimizers/CMakeLists.txt | 2 +- .../{adadelta => ada_delta}/CMakeLists.txt | 1 + .../{adadelta => ada_delta}/ada_delta.hpp | 63 ++++---- .../optimizers/ada_delta/ada_delta_impl.hpp | 41 +++++ .../optimizers/ada_delta/ada_delta_update.hpp | 121 +++++++++++++++ .../optimizers/adadelta/ada_delta_impl.hpp | 141 ------------------ src/mlpack/tests/ada_delta_test.cpp | 5 +- 7 files changed, 194 insertions(+), 180 deletions(-) rename src/mlpack/core/optimizers/{adadelta => ada_delta}/CMakeLists.txt (90%) rename src/mlpack/core/optimizers/{adadelta => ada_delta}/ada_delta.hpp (74%) create mode 100644 src/mlpack/core/optimizers/ada_delta/ada_delta_impl.hpp create mode 100644 src/mlpack/core/optimizers/ada_delta/ada_delta_update.hpp delete mode 100644 src/mlpack/core/optimizers/adadelta/ada_delta_impl.hpp diff --git a/src/mlpack/core/optimizers/CMakeLists.txt b/src/mlpack/core/optimizers/CMakeLists.txt index e722a3a763..c0b5f563f5 100644 --- a/src/mlpack/core/optimizers/CMakeLists.txt +++ b/src/mlpack/core/optimizers/CMakeLists.txt @@ -1,5 +1,5 @@ set(DIRS - adadelta + ada_delta ada_grad adam aug_lagrangian diff --git a/src/mlpack/core/optimizers/adadelta/CMakeLists.txt b/src/mlpack/core/optimizers/ada_delta/CMakeLists.txt similarity index 90% rename from src/mlpack/core/optimizers/adadelta/CMakeLists.txt rename to src/mlpack/core/optimizers/ada_delta/CMakeLists.txt index 3cd516b87a..5fc89c659e 100644 --- a/src/mlpack/core/optimizers/adadelta/CMakeLists.txt +++ b/src/mlpack/core/optimizers/ada_delta/CMakeLists.txt @@ -1,6 +1,7 @@ set(SOURCES ada_delta.hpp ada_delta_impl.hpp + ada_delta_update.hpp ) set(DIR_SRCS) diff --git a/src/mlpack/core/optimizers/adadelta/ada_delta.hpp b/src/mlpack/core/optimizers/ada_delta/ada_delta.hpp similarity index 74% rename from src/mlpack/core/optimizers/adadelta/ada_delta.hpp rename to src/mlpack/core/optimizers/ada_delta/ada_delta.hpp index dec379f9d4..59f78b1865 100644 --- a/src/mlpack/core/optimizers/adadelta/ada_delta.hpp +++ b/src/mlpack/core/optimizers/ada_delta/ada_delta.hpp @@ -2,10 +2,11 @@ * @file ada_delta.hpp * @author Ryan Curtin * @author Vasanth Kalingeri + * @author Abhinav Moudgil * - * Implementation of the Adadelta optimizer. Adadelta is an optimizer that + * Implementation of the AdaDelta optimizer. AdaDelta is an optimizer that * dynamically adapts over time using only first order information. - * Additionally, Adadelta requires no manual tuning of a learning rate. + * Additionally, AdaDelta requires no manual tuning of a learning rate. * * mlpack is free software; you may redistribute it and/or modify it under the * terms of the 3-clause BSD license. You should have received a copy of the @@ -16,12 +17,14 @@ #define __MLPACK_CORE_OPTIMIZERS_ADADELTA_ADA_DELTA_HPP #include +#include +#include "ada_delta_update.hpp" namespace mlpack { namespace optimization { /** - * Adadelta is an optimizer that uses two ideas to improve upon the two main + * AdaDelta is an optimizer that uses two ideas to improve upon the two main * drawbacks of the Adagrad method: * * - Accumulate Over Window @@ -38,7 +41,6 @@ namespace optimization { * } * @endcode * - * For AdaDelta to work, a DecomposableFunctionType template parameter is * required. This class must implement the following function: * @@ -81,6 +83,7 @@ class AdaDelta * function is visited in linear order. */ AdaDelta(DecomposableFunctionType& function, + const double stepSize = 1.0, const double rho = 0.95, const double eps = 1e-6, const size_t maxIterations = 100000, @@ -95,57 +98,46 @@ class AdaDelta * @param iterate Starting point (will be modified). * @return Objective value of the final point. */ - double Optimize(arma::mat& iterate); + double Optimize(arma::mat& iterate) { return optimizer.Optimize(iterate); } //! Get the instantiated function to be optimized. - const DecomposableFunctionType& Function() const { return function; } + const DecomposableFunctionType& Function() const { return optimizer.Function(); } //! Modify the instantiated function. - DecomposableFunctionType& Function() { return function; } + DecomposableFunctionType& Function() { return optimizer.Function(); } + + //! Get the step size. + double StepSize() const { return optimizer.StepSize(); } + //! Modify the step size. + double& StepSize() { return optimizer.StepSize(); } //! Get the smoothing parameter. - double Rho() const { return rho; } + double Rho() const { return optimizer.UpdatePolicy().Rho(); } //! Modify the smoothing parameter. - double& Rho() { return rho; } + double& Rho() { return optimizer.UpdatePolicy().Rho(); } //! Get the value used to initialise the mean squared gradient parameter. - double Epsilon() const { return eps; } + double Epsilon() const { return optimizer.UpdatePolicy().Epsilon(); } //! Modify the value used to initialise the mean squared gradient parameter. - double& Epsilon() { return eps; } + double& Epsilon() { return optimizer.UpdatePolicy().Epsilon(); } //! Get the maximum number of iterations (0 indicates no limit). - size_t MaxIterations() const { return maxIterations; } + size_t MaxIterations() const { return optimizer.MaxIterations(); } //! Modify the maximum number of iterations (0 indicates no limit). - size_t& MaxIterations() { return maxIterations; } + size_t& MaxIterations() { return optimizer.MaxIterations(); } //! Get the tolerance for termination. - double Tolerance() const { return tolerance; } + double Tolerance() const { return optimizer.Tolerance(); } //! Modify the tolerance for termination. - double& Tolerance() { return tolerance; } + double& Tolerance() { return optimizer.Tolerance(); } //! Get whether or not the individual functions are shuffled. - bool Shuffle() const { return shuffle; } + bool Shuffle() const { return optimizer.Shuffle(); } //! Modify whether or not the individual functions are shuffled. - bool& Shuffle() { return shuffle; } + bool& Shuffle() { return optimizer.Shuffle(); } private: - //! The instantiated function. - DecomposableFunctionType& function; - - //! The smoothing parameter. - double rho; - - //! The value used to initialise the mean squared gradient parameter. - double eps; - - //! The maximum number of allowed iterations. - size_t maxIterations; - - //! The tolerance for termination. - double tolerance; - - //! Controls whether or not the individual functions are shuffled when - //! iterating. - bool shuffle; + //! The Stochastic Gradient Descent object with AdaDelta policy. + SGD optimizer; }; } // namespace optimization @@ -155,4 +147,3 @@ class AdaDelta #include "ada_delta_impl.hpp" #endif - diff --git a/src/mlpack/core/optimizers/ada_delta/ada_delta_impl.hpp b/src/mlpack/core/optimizers/ada_delta/ada_delta_impl.hpp new file mode 100644 index 0000000000..ef026e6e58 --- /dev/null +++ b/src/mlpack/core/optimizers/ada_delta/ada_delta_impl.hpp @@ -0,0 +1,41 @@ +/** + * @file ada_delta_impl.hpp + * @author Ryan Curtin + * @author Vasanth Kalingeri + * @author Abhinav Moudgil + * + * Implementation of the AdaDelta optimizer. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_CORE_OPTIMIZERS_ADADELTA_ADA_DELTA_IMPL_HPP +#define MLPACK_CORE_OPTIMIZERS_ADADELTA_ADA_DELTA_IMPL_HPP + +#include "ada_delta.hpp" + +namespace mlpack { +namespace optimization { + +template +AdaDelta::AdaDelta(DecomposableFunctionType& function, + const double stepSize, + const double rho, + const double epsilon, + const size_t maxIterations, + const double tolerance, + const bool shuffle) : + optimizer(function, + stepSize, + maxIterations, + tolerance, + shuffle, + AdaDeltaUpdate(rho, epsilon)) +{ /* Nothing to do. */ } + +} // namespace optimization +} // namespace mlpack + +#endif diff --git a/src/mlpack/core/optimizers/ada_delta/ada_delta_update.hpp b/src/mlpack/core/optimizers/ada_delta/ada_delta_update.hpp new file mode 100644 index 0000000000..122efd84f1 --- /dev/null +++ b/src/mlpack/core/optimizers/ada_delta/ada_delta_update.hpp @@ -0,0 +1,121 @@ +/** + * @file ada_delta_update.hpp + * @author Abhinav Moudgil + * + * AdaDelta update for Stochastic Gradient Descent. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_CORE_OPTIMIZERS_SGD_ADA_DELTA_UPDATE_HPP +#define MLPACK_CORE_OPTIMIZERS_SGD_ADA_DELTA_UPDATE_HPP + +#include + +namespace mlpack { +namespace optimization { + +/** + * Implementation of the AdaDelta update policy. AdaDelta is an optimizer that + * uses two ideas to improve upon the two main drawbacks of the AdaGrad method: + * + * - Accumulate Over Window + * - Correct Units with Hessian Approximation + * + * For more information, see the following. + * + * @code + * @article{Zeiler2012, + * author = {Matthew D. Zeiler}, + * title = {{ADADELTA:} An Adaptive Learning Rate Method}, + * journal = {CoRR}, + * year = {2012} + * } + * @endcode + * + */ +class AdaDeltaUpdate +{ + public: + /** + * Construct the AdaDelta update policy with given rho and epsilon parameters. + * + * @param rho The smoothing parameter. + * @param epsilon The epsilon value used to initialise the squared gradient parameter. + */ + AdaDeltaUpdate(const double rho = 0.95, const double epsilon = 1e-8) : + rho(rho), + epsilon(epsilon) + { /* Do nothing. */ }; + + /** + * The Initialize method is called by SGD Optimizer method before the start of the + * iteration update process. In AdaDelta update policy, the mean squared and the delta + * mean squared gradient matrices are initialized to the zeros matrix with the same + * size as gradient matrix (see mlpack::optimization::SGD::Optimizer ) + * + * @param rows number of rows in the gradient matrix. + * @param cols number of columns in the gradient matrix. + */ + void Initialize(const size_t rows, + const size_t cols) + { + // Initialize empty matrices for mean sum of squares of parameter gradient. + meanSquaredGradient = arma::zeros(rows, cols); + meanSquaredGradientDx = arma::zeros(rows, cols); + } + + /** + * Update step for SGD. The AdaDelta update dynamically adapts over time using only + * first order information. Additionally, AdaDelta requires no manual tuning + * of a learning rate. + * + * @param iterate Parameters that minimize the function. + * @param stepSize Step size to be used for the given iteration. + * @param gradient The gradient matrix. + */ + void Update(arma::mat& iterate, + const double stepSize, + const arma::mat& gradient) + { + // Accumulate gradient. + meanSquaredGradient *= rho; + meanSquaredGradient += (1 - rho) * (gradient % gradient); + arma::mat dx = arma::sqrt((meanSquaredGradientDx + eps) / + (meanSquaredGradient + eps)) % gradient; + + // Accumulate updates. + meanSquaredGradientDx *= rho; + meanSquaredGradientDx += (1 - rho) * (dx % dx); + + // Apply update. + iterate -= (stepSize * dx); + } + + //! Get the smoothing parameter. + double Rho() const { return rho; } + //! Modify the smoothing parameter. + double& Rho() { return rho; } + + //! Get the value used to initialise the mean squared gradient parameter. + double Epsilon() const { return epsilon; } + //! Modify the value used to initialise the mean squared gradient parameter. + double& Epsilon() { return epsilon; } + + private: + // The smoothing parameter. + double rho; + // The epsilon value used to initialise the mean squared gradient parameter. + double epsilon; + // The mean squared gradient matrix. + arma::mat meanSquaredGradient; + // The delta mean squared gradient matrix. + arma::mat meanSquaredGradientDx; +}; + +} // namespace optimization +} // namespace mlpack + +#endif diff --git a/src/mlpack/core/optimizers/adadelta/ada_delta_impl.hpp b/src/mlpack/core/optimizers/adadelta/ada_delta_impl.hpp deleted file mode 100644 index ee34a7a110..0000000000 --- a/src/mlpack/core/optimizers/adadelta/ada_delta_impl.hpp +++ /dev/null @@ -1,141 +0,0 @@ -/** - * @file ada_delta_impl.hpp - * @author Ryan Curtin - * @author Vasanth Kalingeri - * - * Implementation of the Adadelta optimizer. - * - * mlpack is free software; you may redistribute it and/or modify it under the - * terms of the 3-clause BSD license. You should have received a copy of the - * 3-clause BSD license along with mlpack. If not, see - * http://www.opensource.org/licenses/BSD-3-Clause for more information. - */ -#ifndef __MLPACK_CORE_OPTIMIZERS_ADADELTA_ADA_DELTA_IMPL_HPP -#define __MLPACK_CORE_OPTIMIZERS_ADADELTA_ADA_DELTA_IMPL_HPP - -#include "ada_delta.hpp" - -namespace mlpack { -namespace optimization { - -template -AdaDelta::AdaDelta(DecomposableFunctionType& function, - const double rho, - const double eps, - const size_t maxIterations, - const double tolerance, - const bool shuffle) : - function(function), - rho(rho), - eps(eps), - maxIterations(maxIterations), - tolerance(tolerance), - shuffle(shuffle) -{ /* Nothing to do. */ } - -//! Optimize the function (minimize). -template -double AdaDelta::Optimize(arma::mat& iterate) -{ - // Find the number of functions to use. - const size_t numFunctions = function.NumFunctions(); - - // This is used only if shuffle is true. - arma::Col visitationOrder; - if (shuffle) - visitationOrder = arma::shuffle(arma::linspace>(0, - (numFunctions - 1), numFunctions)); - - // To keep track of where we are and how things are going. - size_t currentFunction = 0; - double overallObjective = 0; - double lastObjective = DBL_MAX; - - // Calculate the first objective function. - for (size_t i = 0; i < numFunctions; ++i) - overallObjective += function.Evaluate(iterate, i); - - // Now iterate! - arma::mat gradient(iterate.n_rows, iterate.n_cols); - - // Leaky sum of squares of parameter gradient. - arma::mat meanSquaredGradient = arma::zeros(iterate.n_rows, - iterate.n_cols); - - // Leaky sum of squares of parameter gradient. - arma::mat meanSquaredGradientDx = arma::zeros(iterate.n_rows, - iterate.n_cols); - - for (size_t i = 1; i != maxIterations; ++i, ++currentFunction) - { - // Is this iteration the start of a sequence? - if ((currentFunction % numFunctions) == 0) - { - // Output current objective function. - Log::Info << "AdaDelta: iteration " << i << ", objective " - << overallObjective << "." << std::endl; - - if (std::isnan(overallObjective) || std::isinf(overallObjective)) - { - Log::Warn << "AdaDelta: converged to " << overallObjective - << "; terminating with failure. Try a smaller step size?" - << std::endl; - return overallObjective; - } - - if (std::abs(lastObjective - overallObjective) < tolerance) - { - Log::Info << "AdaDelta: minimized within tolerance " << tolerance - << "; terminating optimization." << std::endl; - return overallObjective; - } - - // Reset the counter variables. - lastObjective = overallObjective; - overallObjective = 0; - currentFunction = 0; - - if (shuffle) // Determine order of visitation. - visitationOrder = arma::shuffle(visitationOrder); - } - - // Evaluate the gradient for this iteration. - if (shuffle) - function.Gradient(iterate, visitationOrder[currentFunction], gradient); - else - function.Gradient(iterate, currentFunction, gradient); - - // Accumulate gradient. - meanSquaredGradient *= rho; - meanSquaredGradient += (1 - rho) * (gradient % gradient); - arma::mat dx = arma::sqrt((meanSquaredGradientDx + eps) / - (meanSquaredGradient + eps)) % gradient; - - // Accumulate updates. - meanSquaredGradientDx *= rho; - meanSquaredGradientDx += (1 - rho) * (dx % dx); - - // Apply update. - iterate -= dx; - - // Now add that to the overall objective function. - if (shuffle) - overallObjective += function.Evaluate(iterate, - visitationOrder[currentFunction]); - else - overallObjective += function.Evaluate(iterate, currentFunction); - } - - Log::Info << "AdaDelta: maximum iterations (" << maxIterations - << ") reached; terminating optimization." << std::endl; - // Calculate final objective. - overallObjective = 0; - for (size_t i = 0; i < numFunctions; ++i) - overallObjective += function.Evaluate(iterate, i); - return overallObjective; -} - -} // namespace optimization -} // namespace mlpack - -#endif diff --git a/src/mlpack/tests/ada_delta_test.cpp b/src/mlpack/tests/ada_delta_test.cpp index 481e117c52..295b8ce037 100644 --- a/src/mlpack/tests/ada_delta_test.cpp +++ b/src/mlpack/tests/ada_delta_test.cpp @@ -2,6 +2,7 @@ * @file ada_delta_test.cpp * @author Marcus Edel * @author Vasanth Kalingeri + * @author Abhinav Moudgil * * Tests the AdaDelta optimizer * @@ -12,7 +13,7 @@ */ #include -#include +#include #include #include @@ -36,7 +37,7 @@ BOOST_AUTO_TEST_SUITE(AdaDeltaTest); BOOST_AUTO_TEST_CASE(SimpleAdaDeltaTestFunction) { SGDTestFunction f; - AdaDelta optimizer(f, 0.99, 1e-8, 5000000, 1e-9, true); + AdaDelta optimizer(f, 1.0, 0.99, 1e-8, 5000000, 1e-9, true); arma::mat coordinates = f.GetInitialPoint(); optimizer.Optimize(coordinates);