From b624518249b4f19e5e8adf447ce0883f41e43d48 Mon Sep 17 00:00:00 2001 From: Sourabh Varshney Date: Sun, 25 Feb 2018 22:06:12 +0530 Subject: [PATCH 01/24] Added base declaration --- .../nesterov_momentum_update.hpp | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 src/mlpack/core/optimizers/sgd/update_policies/nesterov_momentum_update.hpp diff --git a/src/mlpack/core/optimizers/sgd/update_policies/nesterov_momentum_update.hpp b/src/mlpack/core/optimizers/sgd/update_policies/nesterov_momentum_update.hpp new file mode 100644 index 0000000000..eacad0f80f --- /dev/null +++ b/src/mlpack/core/optimizers/sgd/update_policies/nesterov_momentum_update.hpp @@ -0,0 +1,27 @@ +/** + * @file nesterov_momentum_update.hpp + * @author Sourabh Varshney + * + * Nesterov Momentum Update for Stochastic Gradient Descent. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_CORE_OPTIMIZERS_SGD_NESTEROV_MOMENTUM_UPDATE_HPP +#define MLPACK_CORE_OPTIMIZERS_SGD_NESTEROV_MOMENTUM_UPDATE_HPP + +#include + +namespace mlpack { +namespace optimization { + +/* + */ + + +} // namespace optimization +} // namespace mlpack + +#endif From e51263eb8ef808b5f4c1b69a3ee4a4f9d576d7bb Mon Sep 17 00:00:00 2001 From: Sourabh Varshney Date: Sun, 25 Feb 2018 22:12:03 +0530 Subject: [PATCH 02/24] Made changes for adding nesterov momentum in sgd --- src/mlpack/core/optimizers/sgd/sgd.hpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/mlpack/core/optimizers/sgd/sgd.hpp b/src/mlpack/core/optimizers/sgd/sgd.hpp index 023ee63950..f73e1f923a 100644 --- a/src/mlpack/core/optimizers/sgd/sgd.hpp +++ b/src/mlpack/core/optimizers/sgd/sgd.hpp @@ -3,6 +3,7 @@ * @author Ryan Curtin * @author Arun Reddy * @author Abhinav Moudgil + * @author Sourabh Varshney * * Stochastic Gradient Descent (SGD). * @@ -17,6 +18,7 @@ #include #include "update_policies/vanilla_update.hpp" #include "update_policies/momentum_update.hpp" +#include "update_policies/nesterov_momentum_update.hpp" #include "decay_policies/no_decay.hpp" namespace mlpack { @@ -202,6 +204,8 @@ using StandardSGD = SGD; using MomentumSGD = SGD; +using NesterovSGD = SGD; + } // namespace optimization } // namespace mlpack From e00a3fbf208cfb58a3df3d3d922d0e0c70a85081 Mon Sep 17 00:00:00 2001 From: Sourabh Varshney Date: Mon, 26 Feb 2018 19:39:11 +0530 Subject: [PATCH 03/24] Added class declarations --- .../nesterov_momentum_update.hpp | 46 +++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/src/mlpack/core/optimizers/sgd/update_policies/nesterov_momentum_update.hpp b/src/mlpack/core/optimizers/sgd/update_policies/nesterov_momentum_update.hpp index eacad0f80f..8eeae6df27 100644 --- a/src/mlpack/core/optimizers/sgd/update_policies/nesterov_momentum_update.hpp +++ b/src/mlpack/core/optimizers/sgd/update_policies/nesterov_momentum_update.hpp @@ -20,6 +20,52 @@ namespace optimization { /* */ +class NesterovMomentumUpdate +{ + public: + /* + */ + NesterovMomentumUpdate(const double beta1 = 0.99 , + const double scheduleDecay = 4e-3) : + beta1(beta1), + scheduleDecay(scheduleDecay), + iteration(0) + { + // Nothing to do. + } + + /** + * The Initialize method is called by SGD Optimizer method before the start of + * the iteration update process. In the momentum update policy the velocity + * matrix is initialized to the zeros matrix with the same size as the + * gradient matrix (see mlpack::optimization::SGD::Optimizer ) + * + * @param rows Number of rows in the gradient matrix. + * @param cols Number of columns in the gradient matrix. + */ + void Initialize(const size_t rows, const size_t cols) + { + // Initialize am empty velocity matrix. + velocity = arma::zeros(rows, cols); + } + + /** + * Update step for SGD. The momentum term makes the convergence faster on the + * way as momentum term increases for dimensions pointing in the same and + * reduces updates for dimensions whose gradients change directions. + * + * @param iterate Parameters that minimize the function. + * @param stepSize Step size to be used for the given iteration. + * @param gradient The gradient matrix. + */ + void Update(arma::mat& iterate, + const double stepSize, + const arma::mat& gradient) + { + + } + +}; } // namespace optimization } // namespace mlpack From 409a2331940c1c6712724ba729072511dcfad055 Mon Sep 17 00:00:00 2001 From: Sourabh Varshney Date: Mon, 26 Feb 2018 20:06:48 +0530 Subject: [PATCH 04/24] Add update formula --- .../nesterov_momentum_update.hpp | 29 +++++++++++++++++-- 1 file changed, 27 insertions(+), 2 deletions(-) diff --git a/src/mlpack/core/optimizers/sgd/update_policies/nesterov_momentum_update.hpp b/src/mlpack/core/optimizers/sgd/update_policies/nesterov_momentum_update.hpp index 8eeae6df27..14964f720f 100644 --- a/src/mlpack/core/optimizers/sgd/update_policies/nesterov_momentum_update.hpp +++ b/src/mlpack/core/optimizers/sgd/update_policies/nesterov_momentum_update.hpp @@ -26,7 +26,7 @@ class NesterovMomentumUpdate /* */ NesterovMomentumUpdate(const double beta1 = 0.99 , - const double scheduleDecay = 4e-3) : + const double scheduleDecay = 4e-3) : beta1(beta1), scheduleDecay(scheduleDecay), iteration(0) @@ -62,9 +62,34 @@ class NesterovMomentumUpdate const double stepSize, const arma::mat& gradient) { - + double beta1T = beta1 * (1 - (0.5 * + std::pow(0.96, (iteration - 1) * scheduleDecay))); + + double beta1T1 = beta1 * (1 - (0.5 * + std::pow(0.96, iteration * scheduleDecay))); + + iterate = iterate + (beta1T * beta1T1 * velocity) - ((1 + beta1T1) * stepSize * gradient); + + velocity = beta1T * velocity - stepSize * gradient; } + //! Get the smoothing parameter. + double Beta1() const { return beta1; } + //! Modify the smoothing parameter. + double& Beta1() { return beta1; } + + //! Get the decay parameter for decay coefficients + double ScheduleDecay() const { return scheduleDecay; } + //! Modify the decay parameter for decay coefficients + double& ScheduleDecay() { return scheduleDecay; } + + private: + // The smoothing parameter. + double beta1; + + // The velocity matrix. + arma::mat velocity; + }; } // namespace optimization From 0e0cdbc768a069574568de2e9c771261778a9a51 Mon Sep 17 00:00:00 2001 From: Sourabh Varshney Date: Mon, 26 Feb 2018 22:08:23 +0530 Subject: [PATCH 05/24] Add comments and reference paper --- .../nesterov_momentum_update.hpp | 37 +++++++++++++++---- 1 file changed, 30 insertions(+), 7 deletions(-) diff --git a/src/mlpack/core/optimizers/sgd/update_policies/nesterov_momentum_update.hpp b/src/mlpack/core/optimizers/sgd/update_policies/nesterov_momentum_update.hpp index 14964f720f..918bb2c893 100644 --- a/src/mlpack/core/optimizers/sgd/update_policies/nesterov_momentum_update.hpp +++ b/src/mlpack/core/optimizers/sgd/update_policies/nesterov_momentum_update.hpp @@ -17,14 +17,37 @@ namespace mlpack { namespace optimization { -/* +/** + * Nesterov Momentum update policy for Stochastic Gradient Descent (SGD). + * + * Learning with SGD can be slow. Applying Standard momentum can accelerate + * the rate of convergence. Nesterov Momentum application can accelerate the + * rate of convergence to O(1/k^2). + * + * @code + * @techreport{Nesterov1983, + * title = {A Method Of Solving A Convex Programming Problem With + * Convergence Rate O(1/K^2)}, + * author = {Yuri Nesterov}, + * institution = {Soviet Math. Dokl.}, + * volume = {27}, + * year = {1983}, + * url = {http://www.cis.pku.edu.cn/faculty/vision/zlin/1983-A%20 + Method%20of%20Solving%20a%20Convex%20Programming%20Problem + %20with%20Convergence%20Rate%20O(k%5E(-2))_Nesterov.pdf} + * } + * @endcode */ class NesterovMomentumUpdate { public: - /* - */ + /** + * Construct the Nesterov Momentum update policy with the given parameters. + * + * @param beta1 The second moment coefficient. + * @param scheduleDecay The decay parameter for decay coefficients + */ NesterovMomentumUpdate(const double beta1 = 0.99 , const double scheduleDecay = 4e-3) : beta1(beta1), @@ -70,12 +93,12 @@ class NesterovMomentumUpdate iterate = iterate + (beta1T * beta1T1 * velocity) - ((1 + beta1T1) * stepSize * gradient); - velocity = beta1T * velocity - stepSize * gradient; + velocity = beta1T * velocity - stepSize * gradient;pd } - //! Get the smoothing parameter. + //! Get the second moment coefficient. double Beta1() const { return beta1; } - //! Modify the smoothing parameter. + //! Modify the second moment coefficient. double& Beta1() { return beta1; } //! Get the decay parameter for decay coefficients @@ -84,7 +107,7 @@ class NesterovMomentumUpdate double& ScheduleDecay() { return scheduleDecay; } private: - // The smoothing parameter. + // The second moment coefficient. double beta1; // The velocity matrix. From ebb37286dd0d8f23ac60860fa6c9493a152c58d2 Mon Sep 17 00:00:00 2001 From: Sourabh Varshney Date: Mon, 26 Feb 2018 22:11:48 +0530 Subject: [PATCH 06/24] Add method in Cmakelist --- src/mlpack/core/optimizers/sgd/CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/src/mlpack/core/optimizers/sgd/CMakeLists.txt b/src/mlpack/core/optimizers/sgd/CMakeLists.txt index 343d024d90..b4c266316c 100644 --- a/src/mlpack/core/optimizers/sgd/CMakeLists.txt +++ b/src/mlpack/core/optimizers/sgd/CMakeLists.txt @@ -2,6 +2,7 @@ set(SOURCES decay_policies/no_decay.hpp update_policies/gradient_clipping.hpp update_policies/momentum_update.hpp + update_policies/nesterov_momentum_update.hpp update_policies/vanilla_update.hpp sgd.hpp sgd_impl.hpp From e42f9a84ccc6e12dda748d70b57781f4316ef0be Mon Sep 17 00:00:00 2001 From: Sourabh Varshney Date: Tue, 27 Feb 2018 11:27:56 +0530 Subject: [PATCH 07/24] Removed typo mistakes --- src/mlpack/core/optimizers/sgd/sgd.hpp | 2 +- .../sgd/update_policies/nesterov_momentum_update.hpp | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/mlpack/core/optimizers/sgd/sgd.hpp b/src/mlpack/core/optimizers/sgd/sgd.hpp index f73e1f923a..523539ec73 100644 --- a/src/mlpack/core/optimizers/sgd/sgd.hpp +++ b/src/mlpack/core/optimizers/sgd/sgd.hpp @@ -204,7 +204,7 @@ using StandardSGD = SGD; using MomentumSGD = SGD; -using NesterovSGD = SGD; +using NesterovMomentumSGD = SGD; } // namespace optimization } // namespace mlpack diff --git a/src/mlpack/core/optimizers/sgd/update_policies/nesterov_momentum_update.hpp b/src/mlpack/core/optimizers/sgd/update_policies/nesterov_momentum_update.hpp index 918bb2c893..3e0bc6899d 100644 --- a/src/mlpack/core/optimizers/sgd/update_policies/nesterov_momentum_update.hpp +++ b/src/mlpack/core/optimizers/sgd/update_policies/nesterov_momentum_update.hpp @@ -49,7 +49,7 @@ class NesterovMomentumUpdate * @param scheduleDecay The decay parameter for decay coefficients */ NesterovMomentumUpdate(const double beta1 = 0.99 , - const double scheduleDecay = 4e-3) : + const double scheduleDecay = 4e-3) : beta1(beta1), scheduleDecay(scheduleDecay), iteration(0) @@ -93,7 +93,7 @@ class NesterovMomentumUpdate iterate = iterate + (beta1T * beta1T1 * velocity) - ((1 + beta1T1) * stepSize * gradient); - velocity = beta1T * velocity - stepSize * gradient;pd + velocity = beta1T * velocity - stepSize * gradient; } //! Get the second moment coefficient. From e50eb991589109d3ae68561fdac01034007bbf5e Mon Sep 17 00:00:00 2001 From: Sourabh Varshney Date: Tue, 27 Feb 2018 11:29:19 +0530 Subject: [PATCH 08/24] Add tests for nesterov momentum sgd --- .../tests/nesterov_momentum_sgd_test.cpp | 81 +++++++++++++++++++ 1 file changed, 81 insertions(+) create mode 100644 src/mlpack/tests/nesterov_momentum_sgd_test.cpp diff --git a/src/mlpack/tests/nesterov_momentum_sgd_test.cpp b/src/mlpack/tests/nesterov_momentum_sgd_test.cpp new file mode 100644 index 0000000000..ee63b260be --- /dev/null +++ b/src/mlpack/tests/nesterov_momentum_sgd_test.cpp @@ -0,0 +1,81 @@ +/** + * @file nesterov_momentum_sgd_test.cpp + * @author Sourabh Varshney + * + * Test file for NesterovMomentumSGD (Stochastic gradient descent with + * nesterov momentum updates). + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#include +#include +#include +#include +#include +#include + +#include +#include "test_tools.hpp" + +using namespace std; +using namespace arma; +using namespace mlpack; +using namespace mlpack::optimization; +using namespace mlpack::optimization::test; + +BOOST_AUTO_TEST_SUITE(NesterovMomentumSGDTest); + +BOOST_AUTO_TEST_CASE(NesterovMomentumSGDSpeedUpTestFunction) +{ + SGDTestFunction f; + NesterovMomentumUpdate nesterovMomentumUpdate(0.99, 4e-3); + NesterovMomentumSGD s(0.0003, 1, 2500000, 1e-9, true, + nesterovMomentumUpdate); + + arma::mat coordinates = f.GetInitialPoint(); + double result = s.Optimize(f, coordinates); + + BOOST_REQUIRE_CLOSE(result, -1.0, 0.15); + BOOST_REQUIRE_SMALL(coordinates[0], 1e-3); + BOOST_REQUIRE_SMALL(coordinates[1], 1e-7); + BOOST_REQUIRE_SMALL(coordinates[2], 1e-7); + + // Compare with SGD with vanilla update. + SGDTestFunction f1; + StandardSGD s1(0.0003, 1, 2500000, 1e-9, true); + + arma::mat coordinates1 = f.GetInitialPoint(); + double result1 = s1.Optimize(f1, coordinates1); + + // Result doesn't converge in 2500000 iterations. + BOOST_REQUIRE_GT(result1 + 1.0, 0.05); + BOOST_REQUIRE_GE(coordinates1[0], 1e-3); + BOOST_REQUIRE_SMALL(coordinates1[1], 1e-7); + BOOST_REQUIRE_SMALL(coordinates1[2], 1e-7); + + BOOST_REQUIRE_LE(result, result1); +} + +BOOST_AUTO_TEST_CASE(GeneralizedRosenbrockTest) +{ + // Loop over several variants. + for (size_t i = 10; i < 50; i += 5) + { + // Create the generalized Rosenbrock function. + GeneralizedRosenbrockFunction f(i); + NesterovMomentumUpdate nesterovMomentumUpdate(0.88, 4e-3); + NesterovMomentumSGD s(0.0008, 1, 0, 1e-15, true, nesterovMomentumUpdate); + + arma::mat coordinates = f.GetInitialPoint(); + double result = s.Optimize(f, coordinates); + + BOOST_REQUIRE_SMALL(result, 1e-4); + for (size_t j = 0; j < i; ++j) + BOOST_REQUIRE_CLOSE(coordinates[j], (double) 1.0, 1e-3); + } +} + +BOOST_AUTO_TEST_SUITE_END(); From e7dea3a553448518427cae36f6580610b327465d Mon Sep 17 00:00:00 2001 From: Sourabh Varshney Date: Tue, 27 Feb 2018 12:27:40 +0530 Subject: [PATCH 09/24] Fix style checks and declared remaining variables --- .../update_policies/nesterov_momentum_update.hpp | 16 +++++++++++----- src/mlpack/tests/nesterov_momentum_sgd_test.cpp | 2 +- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/src/mlpack/core/optimizers/sgd/update_policies/nesterov_momentum_update.hpp b/src/mlpack/core/optimizers/sgd/update_policies/nesterov_momentum_update.hpp index 3e0bc6899d..45ef0026e2 100644 --- a/src/mlpack/core/optimizers/sgd/update_policies/nesterov_momentum_update.hpp +++ b/src/mlpack/core/optimizers/sgd/update_policies/nesterov_momentum_update.hpp @@ -49,7 +49,7 @@ class NesterovMomentumUpdate * @param scheduleDecay The decay parameter for decay coefficients */ NesterovMomentumUpdate(const double beta1 = 0.99 , - const double scheduleDecay = 4e-3) : + const double scheduleDecay = 4e-3) : beta1(beta1), scheduleDecay(scheduleDecay), iteration(0) @@ -85,15 +85,16 @@ class NesterovMomentumUpdate const double stepSize, const arma::mat& gradient) { - double beta1T = beta1 * (1 - (0.5 * + double beta1T = beta1 * (1 - (0.5 * std::pow(0.96, (iteration - 1) * scheduleDecay))); double beta1T1 = beta1 * (1 - (0.5 * std::pow(0.96, iteration * scheduleDecay))); - iterate = iterate + (beta1T * beta1T1 * velocity) - ((1 + beta1T1) * stepSize * gradient); - - velocity = beta1T * velocity - stepSize * gradient; + iterate = iterate + (beta1T * beta1T1 * velocity) - ((1 + beta1T1) + * stepSize * gradient); + + velocity = beta1T * velocity - stepSize * gradient; } //! Get the second moment coefficient. @@ -113,6 +114,11 @@ class NesterovMomentumUpdate // The velocity matrix. arma::mat velocity; + // The decay parameter for decay coefficients. + double scheduleDecay; + + // The number of iterations. + double iteration; }; } // namespace optimization diff --git a/src/mlpack/tests/nesterov_momentum_sgd_test.cpp b/src/mlpack/tests/nesterov_momentum_sgd_test.cpp index ee63b260be..37e3ed3b74 100644 --- a/src/mlpack/tests/nesterov_momentum_sgd_test.cpp +++ b/src/mlpack/tests/nesterov_momentum_sgd_test.cpp @@ -33,7 +33,7 @@ BOOST_AUTO_TEST_CASE(NesterovMomentumSGDSpeedUpTestFunction) SGDTestFunction f; NesterovMomentumUpdate nesterovMomentumUpdate(0.99, 4e-3); NesterovMomentumSGD s(0.0003, 1, 2500000, 1e-9, true, - nesterovMomentumUpdate); + nesterovMomentumUpdate); arma::mat coordinates = f.GetInitialPoint(); double result = s.Optimize(f, coordinates); From 7cf5718b04d00706e781c0e6d26b675b0bf28a85 Mon Sep 17 00:00:00 2001 From: Sourabh Varshney Date: Tue, 27 Feb 2018 12:40:02 +0530 Subject: [PATCH 10/24] Mention test for nesterov sgd in CMakelists --- src/mlpack/tests/CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/src/mlpack/tests/CMakeLists.txt b/src/mlpack/tests/CMakeLists.txt index 69910edfa8..11466a9098 100644 --- a/src/mlpack/tests/CMakeLists.txt +++ b/src/mlpack/tests/CMakeLists.txt @@ -74,6 +74,7 @@ add_executable(mlpack_test momentum_sgd_test.cpp nbc_test.cpp nca_test.cpp + nesterov_momentum_sgd_test.cpp nmf_test.cpp nystroem_method_test.cpp octree_test.cpp From d27c95f8da2521ffb5d742598ab21b0e33c7205f Mon Sep 17 00:00:00 2001 From: Sourabh Varshney Date: Wed, 28 Feb 2018 10:18:08 +0530 Subject: [PATCH 11/24] Solved pedantic style issues --- .../sgd/update_policies/nesterov_momentum_update.hpp | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/mlpack/core/optimizers/sgd/update_policies/nesterov_momentum_update.hpp b/src/mlpack/core/optimizers/sgd/update_policies/nesterov_momentum_update.hpp index 45ef0026e2..f571a42b64 100644 --- a/src/mlpack/core/optimizers/sgd/update_policies/nesterov_momentum_update.hpp +++ b/src/mlpack/core/optimizers/sgd/update_policies/nesterov_momentum_update.hpp @@ -27,14 +27,11 @@ namespace optimization { * @code * @techreport{Nesterov1983, * title = {A Method Of Solving A Convex Programming Problem With - * Convergence Rate O(1/K^2)}, + * Convergence Rate O(1/K^2)}, * author = {Yuri Nesterov}, * institution = {Soviet Math. Dokl.}, * volume = {27}, * year = {1983}, - * url = {http://www.cis.pku.edu.cn/faculty/vision/zlin/1983-A%20 - Method%20of%20Solving%20a%20Convex%20Programming%20Problem - %20with%20Convergence%20Rate%20O(k%5E(-2))_Nesterov.pdf} * } * @endcode */ From c0c0f2befb1701ec157d071fd09ca635f378329f Mon Sep 17 00:00:00 2001 From: Sourabh Varshney Date: Thu, 1 Mar 2018 23:04:55 +0530 Subject: [PATCH 12/24] Updated nesterov_momentum formula --- .../nesterov_momentum_update.hpp | 35 +++---------------- 1 file changed, 5 insertions(+), 30 deletions(-) diff --git a/src/mlpack/core/optimizers/sgd/update_policies/nesterov_momentum_update.hpp b/src/mlpack/core/optimizers/sgd/update_policies/nesterov_momentum_update.hpp index f571a42b64..96f01b5824 100644 --- a/src/mlpack/core/optimizers/sgd/update_policies/nesterov_momentum_update.hpp +++ b/src/mlpack/core/optimizers/sgd/update_policies/nesterov_momentum_update.hpp @@ -42,14 +42,8 @@ class NesterovMomentumUpdate /** * Construct the Nesterov Momentum update policy with the given parameters. * - * @param beta1 The second moment coefficient. - * @param scheduleDecay The decay parameter for decay coefficients */ - NesterovMomentumUpdate(const double beta1 = 0.99 , - const double scheduleDecay = 4e-3) : - beta1(beta1), - scheduleDecay(scheduleDecay), - iteration(0) + NesterovMomentumUpdate() : iteration(0) { // Nothing to do. } @@ -82,38 +76,19 @@ class NesterovMomentumUpdate const double stepSize, const arma::mat& gradient) { - double beta1T = beta1 * (1 - (0.5 * - std::pow(0.96, (iteration - 1) * scheduleDecay))); + iteration++; - double beta1T1 = beta1 * (1 - (0.5 * - std::pow(0.96, iteration * scheduleDecay))); + double momentum = 1 - (3 / (iteration + 5)); - iterate = iterate + (beta1T * beta1T1 * velocity) - ((1 + beta1T1) - * stepSize * gradient); + velocity = momentum * velocity - stepSize * gradient; - velocity = beta1T * velocity - stepSize * gradient; + iterate += velocity; } - //! Get the second moment coefficient. - double Beta1() const { return beta1; } - //! Modify the second moment coefficient. - double& Beta1() { return beta1; } - - //! Get the decay parameter for decay coefficients - double ScheduleDecay() const { return scheduleDecay; } - //! Modify the decay parameter for decay coefficients - double& ScheduleDecay() { return scheduleDecay; } - private: - // The second moment coefficient. - double beta1; - // The velocity matrix. arma::mat velocity; - // The decay parameter for decay coefficients. - double scheduleDecay; - // The number of iterations. double iteration; }; From e3641f3f0fb19b859bc227c700b4d553e7231205 Mon Sep 17 00:00:00 2001 From: Sourabh Varshney Date: Thu, 1 Mar 2018 23:05:50 +0530 Subject: [PATCH 13/24] Updated tests for nesterov momentum --- src/mlpack/tests/nesterov_momentum_sgd_test.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/mlpack/tests/nesterov_momentum_sgd_test.cpp b/src/mlpack/tests/nesterov_momentum_sgd_test.cpp index 37e3ed3b74..39c1e02119 100644 --- a/src/mlpack/tests/nesterov_momentum_sgd_test.cpp +++ b/src/mlpack/tests/nesterov_momentum_sgd_test.cpp @@ -31,7 +31,7 @@ BOOST_AUTO_TEST_SUITE(NesterovMomentumSGDTest); BOOST_AUTO_TEST_CASE(NesterovMomentumSGDSpeedUpTestFunction) { SGDTestFunction f; - NesterovMomentumUpdate nesterovMomentumUpdate(0.99, 4e-3); + NesterovMomentumUpdate nesterovMomentumUpdate(); NesterovMomentumSGD s(0.0003, 1, 2500000, 1e-9, true, nesterovMomentumUpdate); @@ -66,7 +66,7 @@ BOOST_AUTO_TEST_CASE(GeneralizedRosenbrockTest) { // Create the generalized Rosenbrock function. GeneralizedRosenbrockFunction f(i); - NesterovMomentumUpdate nesterovMomentumUpdate(0.88, 4e-3); + NesterovMomentumUpdate nesterovMomentumUpdate(); NesterovMomentumSGD s(0.0008, 1, 0, 1e-15, true, nesterovMomentumUpdate); arma::mat coordinates = f.GetInitialPoint(); From bb24b37ea3ef6ab7abd23a59f3bcccd6527300d8 Mon Sep 17 00:00:00 2001 From: Sourabh Varshney Date: Thu, 1 Mar 2018 23:34:50 +0530 Subject: [PATCH 14/24] Corrected declaration --- src/mlpack/tests/nesterov_momentum_sgd_test.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/mlpack/tests/nesterov_momentum_sgd_test.cpp b/src/mlpack/tests/nesterov_momentum_sgd_test.cpp index 39c1e02119..18ab96ae0b 100644 --- a/src/mlpack/tests/nesterov_momentum_sgd_test.cpp +++ b/src/mlpack/tests/nesterov_momentum_sgd_test.cpp @@ -31,7 +31,7 @@ BOOST_AUTO_TEST_SUITE(NesterovMomentumSGDTest); BOOST_AUTO_TEST_CASE(NesterovMomentumSGDSpeedUpTestFunction) { SGDTestFunction f; - NesterovMomentumUpdate nesterovMomentumUpdate(); + NesterovMomentumUpdate nesterovMomentumUpdate; NesterovMomentumSGD s(0.0003, 1, 2500000, 1e-9, true, nesterovMomentumUpdate); @@ -66,7 +66,7 @@ BOOST_AUTO_TEST_CASE(GeneralizedRosenbrockTest) { // Create the generalized Rosenbrock function. GeneralizedRosenbrockFunction f(i); - NesterovMomentumUpdate nesterovMomentumUpdate(); + NesterovMomentumUpdate nesterovMomentumUpdate; NesterovMomentumSGD s(0.0008, 1, 0, 1e-15, true, nesterovMomentumUpdate); arma::mat coordinates = f.GetInitialPoint(); From 833f5f577baa3f926e156a3722db0017ed0936f1 Mon Sep 17 00:00:00 2001 From: Sourabh Varshney Date: Fri, 9 Mar 2018 21:54:56 +0530 Subject: [PATCH 15/24] Modified method rule --- src/mlpack/core/optimizers/sgd/sgd.hpp | 2 +- .../nesterov_momentum_update.hpp | 17 ++++++++++++++--- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/src/mlpack/core/optimizers/sgd/sgd.hpp b/src/mlpack/core/optimizers/sgd/sgd.hpp index 523539ec73..047e871048 100644 --- a/src/mlpack/core/optimizers/sgd/sgd.hpp +++ b/src/mlpack/core/optimizers/sgd/sgd.hpp @@ -107,7 +107,7 @@ class SGD * @param resetPolicy Flag that determines whether update policy parameters * are reset before every Optimize call. */ - SGD(const double stepSize = 0.01, + SGD(const double stepSize = 0.001, const size_t batchSize = 32, const size_t maxIterations = 100000, const double tolerance = 1e-5, diff --git a/src/mlpack/core/optimizers/sgd/update_policies/nesterov_momentum_update.hpp b/src/mlpack/core/optimizers/sgd/update_policies/nesterov_momentum_update.hpp index 96f01b5824..c79f58e739 100644 --- a/src/mlpack/core/optimizers/sgd/update_policies/nesterov_momentum_update.hpp +++ b/src/mlpack/core/optimizers/sgd/update_policies/nesterov_momentum_update.hpp @@ -43,7 +43,9 @@ class NesterovMomentumUpdate * Construct the Nesterov Momentum update policy with the given parameters. * */ - NesterovMomentumUpdate() : iteration(0) + NesterovMomentumUpdate(const double maxMomentum = 0.999) : + iteration(0), + maxMomentum(maxMomentum) { // Nothing to do. } @@ -78,19 +80,28 @@ class NesterovMomentumUpdate { iteration++; - double momentum = 1 - (3 / (iteration + 5)); + double momentumT = std::min((1 - std::pow(2,(- 1 - ((log(floor(iteration + / 250)) +1) / log(2))))) , maxMomentum); - velocity = momentum * velocity - stepSize * gradient; + velocity = momentumT * velocity - stepSize * gradient; iterate += velocity; } + //! Get the value used to initialise the maximum momentum coefficient. + double MaxMomentum() const { return maxMomentum; } + //! Modify the value used to initialise the maximum momentum coefficient. + double& MaxMomentum() { return maxMomentum; } + private: // The velocity matrix. arma::mat velocity; // The number of iterations. double iteration; + + // Maximum momentum coefficient + double maxMomentum; }; } // namespace optimization From 1340c79f637fd0d0438f6fad9f6d68eef5217859 Mon Sep 17 00:00:00 2001 From: Sourabh Varshney Date: Fri, 9 Mar 2018 21:58:31 +0530 Subject: [PATCH 16/24] Modified test --- src/mlpack/tests/nesterov_momentum_sgd_test.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/mlpack/tests/nesterov_momentum_sgd_test.cpp b/src/mlpack/tests/nesterov_momentum_sgd_test.cpp index 18ab96ae0b..442fa80ed7 100644 --- a/src/mlpack/tests/nesterov_momentum_sgd_test.cpp +++ b/src/mlpack/tests/nesterov_momentum_sgd_test.cpp @@ -31,7 +31,7 @@ BOOST_AUTO_TEST_SUITE(NesterovMomentumSGDTest); BOOST_AUTO_TEST_CASE(NesterovMomentumSGDSpeedUpTestFunction) { SGDTestFunction f; - NesterovMomentumUpdate nesterovMomentumUpdate; + NesterovMomentumUpdate nesterovMomentumUpdate(0.999); NesterovMomentumSGD s(0.0003, 1, 2500000, 1e-9, true, nesterovMomentumUpdate); @@ -66,7 +66,7 @@ BOOST_AUTO_TEST_CASE(GeneralizedRosenbrockTest) { // Create the generalized Rosenbrock function. GeneralizedRosenbrockFunction f(i); - NesterovMomentumUpdate nesterovMomentumUpdate; + NesterovMomentumUpdate nesterovMomentumUpdate(0.999); NesterovMomentumSGD s(0.0008, 1, 0, 1e-15, true, nesterovMomentumUpdate); arma::mat coordinates = f.GetInitialPoint(); From 5f2baa73d44353c0225a5c97b507c2489ac30b14 Mon Sep 17 00:00:00 2001 From: Sourabh Varshney Date: Fri, 9 Mar 2018 22:01:59 +0530 Subject: [PATCH 17/24] Style fix --- .../optimizers/sgd/update_policies/nesterov_momentum_update.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mlpack/core/optimizers/sgd/update_policies/nesterov_momentum_update.hpp b/src/mlpack/core/optimizers/sgd/update_policies/nesterov_momentum_update.hpp index c79f58e739..75eea974ca 100644 --- a/src/mlpack/core/optimizers/sgd/update_policies/nesterov_momentum_update.hpp +++ b/src/mlpack/core/optimizers/sgd/update_policies/nesterov_momentum_update.hpp @@ -80,7 +80,7 @@ class NesterovMomentumUpdate { iteration++; - double momentumT = std::min((1 - std::pow(2,(- 1 - ((log(floor(iteration + double momentumT = std::min((1 - std::pow(2 , (- 1 - ((log(floor(iteration / 250)) +1) / log(2))))) , maxMomentum); velocity = momentumT * velocity - stepSize * gradient; From 5c72999fc283a2ce91d3e24b3ba4bc238a52e069 Mon Sep 17 00:00:00 2001 From: Sourabh Varshney Date: Sat, 10 Mar 2018 15:10:25 +0530 Subject: [PATCH 18/24] Tried tests for constant momentum --- .../sgd/update_policies/nesterov_momentum_update.hpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/mlpack/core/optimizers/sgd/update_policies/nesterov_momentum_update.hpp b/src/mlpack/core/optimizers/sgd/update_policies/nesterov_momentum_update.hpp index 75eea974ca..54a55224ef 100644 --- a/src/mlpack/core/optimizers/sgd/update_policies/nesterov_momentum_update.hpp +++ b/src/mlpack/core/optimizers/sgd/update_policies/nesterov_momentum_update.hpp @@ -80,8 +80,7 @@ class NesterovMomentumUpdate { iteration++; - double momentumT = std::min((1 - std::pow(2 , (- 1 - ((log(floor(iteration - / 250)) +1) / log(2))))) , maxMomentum); + double momentumT = 0.9; velocity = momentumT * velocity - stepSize * gradient; From 9d6ac4d6e77143c4a929cc91969c1a1105335d0e Mon Sep 17 00:00:00 2001 From: Sourabh Varshney Date: Sat, 10 Mar 2018 22:37:44 +0530 Subject: [PATCH 19/24] Changed momentum to check whether tests are running correctly --- .../optimizers/sgd/update_policies/nesterov_momentum_update.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mlpack/core/optimizers/sgd/update_policies/nesterov_momentum_update.hpp b/src/mlpack/core/optimizers/sgd/update_policies/nesterov_momentum_update.hpp index 54a55224ef..8a2dfda640 100644 --- a/src/mlpack/core/optimizers/sgd/update_policies/nesterov_momentum_update.hpp +++ b/src/mlpack/core/optimizers/sgd/update_policies/nesterov_momentum_update.hpp @@ -80,7 +80,7 @@ class NesterovMomentumUpdate { iteration++; - double momentumT = 0.9; + double momentumT = 0.5; velocity = momentumT * velocity - stepSize * gradient; From c11f5f8a7b664ba22cfc0c60e6f68bd7e2526336 Mon Sep 17 00:00:00 2001 From: Sourabh Varshney Date: Sun, 11 Mar 2018 20:23:16 +0530 Subject: [PATCH 20/24] Formula Correction --- .../nesterov_momentum_update.hpp | 27 +++++++------------ .../tests/nesterov_momentum_sgd_test.cpp | 4 +-- 2 files changed, 12 insertions(+), 19 deletions(-) diff --git a/src/mlpack/core/optimizers/sgd/update_policies/nesterov_momentum_update.hpp b/src/mlpack/core/optimizers/sgd/update_policies/nesterov_momentum_update.hpp index 8a2dfda640..5616fd68cd 100644 --- a/src/mlpack/core/optimizers/sgd/update_policies/nesterov_momentum_update.hpp +++ b/src/mlpack/core/optimizers/sgd/update_policies/nesterov_momentum_update.hpp @@ -43,9 +43,9 @@ class NesterovMomentumUpdate * Construct the Nesterov Momentum update policy with the given parameters. * */ - NesterovMomentumUpdate(const double maxMomentum = 0.999) : + NesterovMomentumUpdate(const double momentum = 0.5) : iteration(0), - maxMomentum(maxMomentum) + momentum(momentum) { // Nothing to do. } @@ -78,29 +78,22 @@ class NesterovMomentumUpdate const double stepSize, const arma::mat& gradient) { - iteration++; + velocity = momentum * velocity - stepSize * gradient; - double momentumT = 0.5; - - velocity = momentumT * velocity - stepSize * gradient; - - iterate += velocity; + iterate += momentum * velocity - stepSize * gradient; } - //! Get the value used to initialise the maximum momentum coefficient. - double MaxMomentum() const { return maxMomentum; } - //! Modify the value used to initialise the maximum momentum coefficient. - double& MaxMomentum() { return maxMomentum; } + //! Get the value used to initialise the momentum coefficient. + double Momentum() const { return momentum; } + //! Modify the value used to initialise the momentum coefficient. + double& Momentum() { return momentum; } private: // The velocity matrix. arma::mat velocity; - // The number of iterations. - double iteration; - - // Maximum momentum coefficient - double maxMomentum; + // Momentum coefficient + double momentum; }; } // namespace optimization diff --git a/src/mlpack/tests/nesterov_momentum_sgd_test.cpp b/src/mlpack/tests/nesterov_momentum_sgd_test.cpp index 442fa80ed7..e24e97148c 100644 --- a/src/mlpack/tests/nesterov_momentum_sgd_test.cpp +++ b/src/mlpack/tests/nesterov_momentum_sgd_test.cpp @@ -31,7 +31,7 @@ BOOST_AUTO_TEST_SUITE(NesterovMomentumSGDTest); BOOST_AUTO_TEST_CASE(NesterovMomentumSGDSpeedUpTestFunction) { SGDTestFunction f; - NesterovMomentumUpdate nesterovMomentumUpdate(0.999); + NesterovMomentumUpdate nesterovMomentumUpdate(0.7); NesterovMomentumSGD s(0.0003, 1, 2500000, 1e-9, true, nesterovMomentumUpdate); @@ -66,7 +66,7 @@ BOOST_AUTO_TEST_CASE(GeneralizedRosenbrockTest) { // Create the generalized Rosenbrock function. GeneralizedRosenbrockFunction f(i); - NesterovMomentumUpdate nesterovMomentumUpdate(0.999); + NesterovMomentumUpdate nesterovMomentumUpdate(0.4); NesterovMomentumSGD s(0.0008, 1, 0, 1e-15, true, nesterovMomentumUpdate); arma::mat coordinates = f.GetInitialPoint(); From 765c1be1572bca2052934198cc4568020af4b4e1 Mon Sep 17 00:00:00 2001 From: Sourabh Varshney Date: Sun, 11 Mar 2018 20:25:42 +0530 Subject: [PATCH 21/24] Updated tests --- src/mlpack/tests/nesterov_momentum_sgd_test.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/mlpack/tests/nesterov_momentum_sgd_test.cpp b/src/mlpack/tests/nesterov_momentum_sgd_test.cpp index e24e97148c..50feec5b64 100644 --- a/src/mlpack/tests/nesterov_momentum_sgd_test.cpp +++ b/src/mlpack/tests/nesterov_momentum_sgd_test.cpp @@ -31,7 +31,7 @@ BOOST_AUTO_TEST_SUITE(NesterovMomentumSGDTest); BOOST_AUTO_TEST_CASE(NesterovMomentumSGDSpeedUpTestFunction) { SGDTestFunction f; - NesterovMomentumUpdate nesterovMomentumUpdate(0.7); + NesterovMomentumUpdate nesterovMomentumUpdate(0.9); NesterovMomentumSGD s(0.0003, 1, 2500000, 1e-9, true, nesterovMomentumUpdate); @@ -66,8 +66,8 @@ BOOST_AUTO_TEST_CASE(GeneralizedRosenbrockTest) { // Create the generalized Rosenbrock function. GeneralizedRosenbrockFunction f(i); - NesterovMomentumUpdate nesterovMomentumUpdate(0.4); - NesterovMomentumSGD s(0.0008, 1, 0, 1e-15, true, nesterovMomentumUpdate); + NesterovMomentumUpdate nesterovMomentumUpdate(0.9); + NesterovMomentumSGD s(0.0001, 1, 0, 1e-15, true, nesterovMomentumUpdate); arma::mat coordinates = f.GetInitialPoint(); double result = s.Optimize(f, coordinates); From e81fb5c8a5bea80ab1c659b4900131cbcacd1d29 Mon Sep 17 00:00:00 2001 From: Sourabh Varshney Date: Sun, 11 Mar 2018 20:33:18 +0530 Subject: [PATCH 22/24] Remove unused initialization --- .../optimizers/sgd/update_policies/nesterov_momentum_update.hpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/mlpack/core/optimizers/sgd/update_policies/nesterov_momentum_update.hpp b/src/mlpack/core/optimizers/sgd/update_policies/nesterov_momentum_update.hpp index 5616fd68cd..ab4c66320d 100644 --- a/src/mlpack/core/optimizers/sgd/update_policies/nesterov_momentum_update.hpp +++ b/src/mlpack/core/optimizers/sgd/update_policies/nesterov_momentum_update.hpp @@ -44,7 +44,6 @@ class NesterovMomentumUpdate * */ NesterovMomentumUpdate(const double momentum = 0.5) : - iteration(0), momentum(momentum) { // Nothing to do. From d0b6f69d9e71b3b3f812c249429fc7e69e8982c5 Mon Sep 17 00:00:00 2001 From: Sourabh Varshney Date: Wed, 14 Mar 2018 17:27:58 +0530 Subject: [PATCH 23/24] Minor style fixes --- src/mlpack/core/optimizers/sgd/sgd.hpp | 2 +- .../nesterov_momentum_update.hpp | 3 +-- .../tests/nesterov_momentum_sgd_test.cpp | 21 ++++++------------- 3 files changed, 8 insertions(+), 18 deletions(-) diff --git a/src/mlpack/core/optimizers/sgd/sgd.hpp b/src/mlpack/core/optimizers/sgd/sgd.hpp index 047e871048..523539ec73 100644 --- a/src/mlpack/core/optimizers/sgd/sgd.hpp +++ b/src/mlpack/core/optimizers/sgd/sgd.hpp @@ -107,7 +107,7 @@ class SGD * @param resetPolicy Flag that determines whether update policy parameters * are reset before every Optimize call. */ - SGD(const double stepSize = 0.001, + SGD(const double stepSize = 0.01, const size_t batchSize = 32, const size_t maxIterations = 100000, const double tolerance = 1e-5, diff --git a/src/mlpack/core/optimizers/sgd/update_policies/nesterov_momentum_update.hpp b/src/mlpack/core/optimizers/sgd/update_policies/nesterov_momentum_update.hpp index ab4c66320d..8bbede71c7 100644 --- a/src/mlpack/core/optimizers/sgd/update_policies/nesterov_momentum_update.hpp +++ b/src/mlpack/core/optimizers/sgd/update_policies/nesterov_momentum_update.hpp @@ -35,7 +35,6 @@ namespace optimization { * } * @endcode */ - class NesterovMomentumUpdate { public: @@ -91,7 +90,7 @@ class NesterovMomentumUpdate // The velocity matrix. arma::mat velocity; - // Momentum coefficient + // The Momentum coefficient. double momentum; }; diff --git a/src/mlpack/tests/nesterov_momentum_sgd_test.cpp b/src/mlpack/tests/nesterov_momentum_sgd_test.cpp index 50feec5b64..265a112965 100644 --- a/src/mlpack/tests/nesterov_momentum_sgd_test.cpp +++ b/src/mlpack/tests/nesterov_momentum_sgd_test.cpp @@ -28,6 +28,9 @@ using namespace mlpack::optimization::test; BOOST_AUTO_TEST_SUITE(NesterovMomentumSGDTest); +/* +* Tests the Nesterov Momentum SGD update policy. +*/ BOOST_AUTO_TEST_CASE(NesterovMomentumSGDSpeedUpTestFunction) { SGDTestFunction f; @@ -42,23 +45,11 @@ BOOST_AUTO_TEST_CASE(NesterovMomentumSGDSpeedUpTestFunction) BOOST_REQUIRE_SMALL(coordinates[0], 1e-3); BOOST_REQUIRE_SMALL(coordinates[1], 1e-7); BOOST_REQUIRE_SMALL(coordinates[2], 1e-7); - - // Compare with SGD with vanilla update. - SGDTestFunction f1; - StandardSGD s1(0.0003, 1, 2500000, 1e-9, true); - - arma::mat coordinates1 = f.GetInitialPoint(); - double result1 = s1.Optimize(f1, coordinates1); - - // Result doesn't converge in 2500000 iterations. - BOOST_REQUIRE_GT(result1 + 1.0, 0.05); - BOOST_REQUIRE_GE(coordinates1[0], 1e-3); - BOOST_REQUIRE_SMALL(coordinates1[1], 1e-7); - BOOST_REQUIRE_SMALL(coordinates1[2], 1e-7); - - BOOST_REQUIRE_LE(result, result1); } +/* +* Tests the Nesterov Momentum SGD with Generalized Rosenbrock Test. +*/ BOOST_AUTO_TEST_CASE(GeneralizedRosenbrockTest) { // Loop over several variants. From 642d3516d81f22cd32183b2fb35e02a8bf753579 Mon Sep 17 00:00:00 2001 From: Sourabh Varshney Date: Thu, 15 Mar 2018 21:49:54 +0530 Subject: [PATCH 24/24] Minor grammar fixes --- .../sgd/update_policies/nesterov_momentum_update.hpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/mlpack/core/optimizers/sgd/update_policies/nesterov_momentum_update.hpp b/src/mlpack/core/optimizers/sgd/update_policies/nesterov_momentum_update.hpp index 8bbede71c7..befb974913 100644 --- a/src/mlpack/core/optimizers/sgd/update_policies/nesterov_momentum_update.hpp +++ b/src/mlpack/core/optimizers/sgd/update_policies/nesterov_momentum_update.hpp @@ -59,14 +59,14 @@ class NesterovMomentumUpdate */ void Initialize(const size_t rows, const size_t cols) { - // Initialize am empty velocity matrix. + // Initialize an empty velocity matrix. velocity = arma::zeros(rows, cols); } /** * Update step for SGD. The momentum term makes the convergence faster on the - * way as momentum term increases for dimensions pointing in the same and - * reduces updates for dimensions whose gradients change directions. + * way as momentum term increases for dimensions pointing in the same direction + * and reduces updates for dimensions whose gradients change directions. * * @param iterate Parameters that minimize the function. * @param stepSize Step size to be used for the given iteration. @@ -81,9 +81,9 @@ class NesterovMomentumUpdate iterate += momentum * velocity - stepSize * gradient; } - //! Get the value used to initialise the momentum coefficient. + //! Get the value used to initialize the momentum coefficient. double Momentum() const { return momentum; } - //! Modify the value used to initialise the momentum coefficient. + //! Modify the value used to initialize the momentum coefficient. double& Momentum() { return momentum; } private: