From 4d0c2bb2548b79eadc969e6ed87ae70465ee8d74 Mon Sep 17 00:00:00 2001 From: Marcus Edel Date: Sat, 10 Feb 2018 19:27:09 +0100 Subject: [PATCH] Address comments and minor style fixes. --- .../core/optimizers/katyusha/katyusha.hpp | 7 +++--- .../optimizers/katyusha/katyusha_impl.hpp | 22 +++++++++++++------ .../core/optimizers/sarah/CMakeLists.txt | 2 +- src/mlpack/core/optimizers/sarah/sarah.hpp | 3 ++- .../core/optimizers/sarah/sarah_impl.hpp | 4 ++-- src/mlpack/core/optimizers/svrg/svrg.hpp | 3 ++- src/mlpack/core/optimizers/svrg/svrg_impl.hpp | 4 ++-- 7 files changed, 28 insertions(+), 17 deletions(-) diff --git a/src/mlpack/core/optimizers/katyusha/katyusha.hpp b/src/mlpack/core/optimizers/katyusha/katyusha.hpp index 5768e5c53c..2d2d68cc76 100644 --- a/src/mlpack/core/optimizers/katyusha/katyusha.hpp +++ b/src/mlpack/core/optimizers/katyusha/katyusha.hpp @@ -19,7 +19,7 @@ namespace optimization { /** * Katyusha is a direct, primal-only stochastic gradient method which uses a - * negative momentum” on top of Nesterov’s momentum. + * "negative momentum" on top of Nesterov’s momentum. * * For more information, see the following. * @@ -59,7 +59,7 @@ namespace optimization { * * @tparam proximal Whether the proximal update should be used or not. */ -template +template class KatyushaType { public: @@ -77,7 +77,8 @@ class KatyushaType * @param maxIterations Maximum number of iterations allowed (0 means no * limit). * @param innerIterations The number of inner iterations allowed (0 means - * n / batchSize). + * n / batchSize). Note that the full gradient is only calculated in + * the outer iteration. * @param tolerance Maximum absolute tolerance to terminate algorithm. * @param shuffle If true, the function order is shuffled; otherwise, each * function is visited in linear order. diff --git a/src/mlpack/core/optimizers/katyusha/katyusha_impl.hpp b/src/mlpack/core/optimizers/katyusha/katyusha_impl.hpp index 9a91b61ec6..0affb3e7a0 100644 --- a/src/mlpack/core/optimizers/katyusha/katyusha_impl.hpp +++ b/src/mlpack/core/optimizers/katyusha/katyusha_impl.hpp @@ -18,8 +18,8 @@ namespace mlpack { namespace optimization { -template -KatyushaType::KatyushaType( +template +KatyushaType::KatyushaType( const double convexity, const double lipschitz, const size_t batchSize, @@ -37,9 +37,9 @@ KatyushaType::KatyushaType( { /* Nothing to do. */ } //! Optimize the function (minimize). -template +template template -double KatyushaType::Optimize( +double KatyushaType::Optimize( DecomposableFunctionType& function, arma::mat& iterate) { @@ -66,7 +66,7 @@ double KatyushaType::Optimize( double normalizer = 1; for (size_t i = 0; i < numBatches; i++) { - normalizer = r * (normalizer + 1.0); + normalizer = r * (normalizer + 1.0); } normalizer = 1.0 / normalizer; @@ -156,15 +156,23 @@ double KatyushaType::Optimize( function.Gradient(iterate0, currentFunction, gradient0, effectiveBatchSize); + // By the minimality definition of z_{k + 1}, we have that: + // z_{k+1} − z_k + \alpha * \sigma_{k+1} + \alpha g = 0. arma::mat zNew = z - alpha * (fullGradient + (gradient - gradient0) / (double) batchSize); // Proximal update, choose between Option I and Option II. Shift relative // to the Lipschitz constant or take a constant step using the given step // size. - if (proximal) + if (Proximal) { - y = iterate + 1.0 / (3.0 * lipschitz) * (zNew - z); + // yk = x0 − 1 / (3L) * \delta1, k = 1 + // yk = x0 − 1 / (3L) * \delta2 - ((1 - tau) / (3L)) + tau * alpha) + // * \delta1, k = 2 + // yk = x0 − 1 / (3L) * \delta3 - ((1 - tau) / (3L)) + tau * alpha) + // * \delta2 - ((1-tau)^2 / (3L) + (1 - (1 - tau)^2) * alpha) * \delta1, + // k = 3. + y = iterate + 1.0 / (3.0 * lipschitz) * w; } else { diff --git a/src/mlpack/core/optimizers/sarah/CMakeLists.txt b/src/mlpack/core/optimizers/sarah/CMakeLists.txt index a39a81807b..bbd4a9ea98 100644 --- a/src/mlpack/core/optimizers/sarah/CMakeLists.txt +++ b/src/mlpack/core/optimizers/sarah/CMakeLists.txt @@ -1,5 +1,5 @@ set(SOURCES - sarah.hpp + sarah.hpp sarah_impl.hpp sarah_plus_update.hpp sarah_update.hpp diff --git a/src/mlpack/core/optimizers/sarah/sarah.hpp b/src/mlpack/core/optimizers/sarah/sarah.hpp index 0dd5b0755b..3ead9e137f 100644 --- a/src/mlpack/core/optimizers/sarah/sarah.hpp +++ b/src/mlpack/core/optimizers/sarah/sarah.hpp @@ -79,7 +79,8 @@ class SARAHType * @param maxIterations Maximum number of iterations allowed (0 means no * limit). * @param innerIterations The number of inner iterations allowed (0 means - * n / batchSize). + * n / batchSize). Note that the full gradient is only calculated in + * the outer iteration. * @param tolerance Maximum absolute tolerance to terminate algorithm. * @param shuffle If true, the function order is shuffled; otherwise, each * function is visited in linear order. diff --git a/src/mlpack/core/optimizers/sarah/sarah_impl.hpp b/src/mlpack/core/optimizers/sarah/sarah_impl.hpp index 010c76e3da..2a25ce59dc 100644 --- a/src/mlpack/core/optimizers/sarah/sarah_impl.hpp +++ b/src/mlpack/core/optimizers/sarah/sarah_impl.hpp @@ -78,7 +78,7 @@ double SARAHType::Optimize( if (std::isnan(overallObjective) || std::isinf(overallObjective)) { - Log::Warn << "Katyusha: converged to " << overallObjective + Log::Warn << "SARAH: converged to " << overallObjective << "; terminating with failure. Try a smaller step size?" << std::endl; return overallObjective; @@ -86,7 +86,7 @@ double SARAHType::Optimize( if (std::abs(lastObjective - overallObjective) < tolerance) { - Log::Info << "Katyusha: minimized within tolerance " << tolerance + Log::Info << "SARAH: minimized within tolerance " << tolerance << "; terminating optimization." << std::endl; return overallObjective; } diff --git a/src/mlpack/core/optimizers/svrg/svrg.hpp b/src/mlpack/core/optimizers/svrg/svrg.hpp index 45a3011de6..fdc2282a91 100644 --- a/src/mlpack/core/optimizers/svrg/svrg.hpp +++ b/src/mlpack/core/optimizers/svrg/svrg.hpp @@ -114,7 +114,8 @@ class SVRGType * @param maxIterations Maximum number of iterations allowed (0 means no * limit). * @param innerIterations The number of inner iterations allowed (0 means - * n / batchSize). + * n / batchSize). Note that the full gradient is only calculated in + * the outer iteration. * @param tolerance Maximum absolute tolerance to terminate algorithm. * @param shuffle If true, the function order is shuffled; otherwise, each * function is visited in linear order. diff --git a/src/mlpack/core/optimizers/svrg/svrg_impl.hpp b/src/mlpack/core/optimizers/svrg/svrg_impl.hpp index 75c016d37c..00bc1edfdb 100644 --- a/src/mlpack/core/optimizers/svrg/svrg_impl.hpp +++ b/src/mlpack/core/optimizers/svrg/svrg_impl.hpp @@ -86,7 +86,7 @@ double SVRGType::Optimize( if (std::isnan(overallObjective) || std::isinf(overallObjective)) { - Log::Warn << "Katyusha: converged to " << overallObjective + Log::Warn << "SVRG: converged to " << overallObjective << "; terminating with failure. Try a smaller step size?" << std::endl; return overallObjective; @@ -94,7 +94,7 @@ double SVRGType::Optimize( if (std::abs(lastObjective - overallObjective) < tolerance) { - Log::Info << "Katyusha: minimized within tolerance " << tolerance + Log::Info << "SVRG: minimized within tolerance " << tolerance << "; terminating optimization." << std::endl; return overallObjective; }