Merge pull request #1279 from sourabhvarshney111/master

Nesterov Momentum Method.

src/mlpack/core/optimizers/sgd/CMakeLists.txt

@@ -2,6 +2,7 @@ set(SOURCES
   decay_policies/no_decay.hpp
   update_policies/gradient_clipping.hpp
   update_policies/momentum_update.hpp
+  update_policies/nesterov_momentum_update.hpp
   update_policies/vanilla_update.hpp
   sgd.hpp
   sgd_impl.hpp

src/mlpack/core/optimizers/sgd/sgd.hpp

@@ -3,6 +3,7 @@
  * @author Ryan Curtin
  * @author Arun Reddy
  * @author Abhinav Moudgil
+ * @author Sourabh Varshney
  *
  * Stochastic Gradient Descent (SGD).
  *
@@ -17,6 +18,7 @@
 #include <mlpack/prereqs.hpp>
 #include "update_policies/vanilla_update.hpp"
 #include "update_policies/momentum_update.hpp"
+#include "update_policies/nesterov_momentum_update.hpp"
 #include "decay_policies/no_decay.hpp"
 
 namespace mlpack {
@@ -202,6 +204,8 @@ using StandardSGD = SGD<VanillaUpdate>;
 
 using MomentumSGD = SGD<MomentumUpdate>;
 
+using NesterovMomentumSGD = SGD<NesterovMomentumUpdate>;
+
 } // namespace optimization
 } // namespace mlpack
 

src/mlpack/core/optimizers/sgd/update_policies/nesterov_momentum_update.hpp

@@ -0,0 +1,100 @@
+/**
+ * @file nesterov_momentum_update.hpp
+ * @author Sourabh Varshney
+ *
+ * Nesterov Momentum Update for Stochastic Gradient Descent.
+ *
+ * mlpack is free software; you may redistribute it and/or modify it under the
+ * terms of the 3-clause BSD license.  You should have received a copy of the
+ * 3-clause BSD license along with mlpack.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef MLPACK_CORE_OPTIMIZERS_SGD_NESTEROV_MOMENTUM_UPDATE_HPP
+#define MLPACK_CORE_OPTIMIZERS_SGD_NESTEROV_MOMENTUM_UPDATE_HPP
+
+#include <mlpack/prereqs.hpp>
+
+namespace mlpack {
+namespace optimization {
+
+/**
+ * Nesterov Momentum update policy for Stochastic Gradient Descent (SGD).
+ *
+ * Learning with SGD can be slow. Applying Standard momentum can accelerate
+ * the rate of convergence. Nesterov Momentum application can accelerate the
+ * rate of convergence to O(1/k^2).
+ * 
+ * @code
+ * @techreport{Nesterov1983,
+ *   title       = {A Method Of Solving A Convex Programming Problem With 
+ *                  Convergence Rate O(1/K^2)},
+ *   author      = {Yuri Nesterov},
+ *   institution = {Soviet Math. Dokl.},
+ *   volume      = {27},
+ *   year        = {1983},
+ * }
+ * @endcode
+ */
+class NesterovMomentumUpdate
+{
+ public:
+  /**
+   * Construct the Nesterov Momentum update policy with the given parameters.
+   *
+   */
+  NesterovMomentumUpdate(const double momentum = 0.5) :
+      momentum(momentum)
+  {
+    // Nothing to do.
+  }
+
+  /**
+   * The Initialize method is called by SGD Optimizer method before the start of
+   * the iteration update process.  In the momentum update policy the velocity
+   * matrix is initialized to the zeros matrix with the same size as the
+   * gradient matrix (see mlpack::optimization::SGD::Optimizer )
+   *
+   * @param rows Number of rows in the gradient matrix.
+   * @param cols Number of columns in the gradient matrix.
+   */
+  void Initialize(const size_t rows, const size_t cols)
+  {
+    // Initialize an empty velocity matrix.
+    velocity = arma::zeros<arma::mat>(rows, cols);
+  }
+
+  /**
+   * Update step for SGD.  The momentum term makes the convergence faster on the
+   * way as momentum term increases for dimensions pointing in the same direction
+   * and reduces updates for dimensions whose gradients change directions.
+   *
+   * @param iterate Parameters that minimize the function.
+   * @param stepSize Step size to be used for the given iteration.
+   * @param gradient The gradient matrix.
+   */
+  void Update(arma::mat& iterate,
+              const double stepSize,
+              const arma::mat& gradient)
+  {
+    velocity = momentum * velocity - stepSize * gradient;
+
+    iterate += momentum * velocity - stepSize * gradient;
+  }
+
+  //! Get the value used to initialize the momentum coefficient.
+  double Momentum() const { return momentum; }
+  //! Modify the value used to initialize the momentum coefficient.
+  double& Momentum() { return momentum; }
+
+ private:
+  // The velocity matrix.
+  arma::mat velocity;
+
+  // The Momentum coefficient.
+  double momentum;
+};
+
+} // namespace optimization
+} // namespace mlpack
+
+#endif

src/mlpack/tests/CMakeLists.txt

@@ -75,6 +75,7 @@ add_executable(mlpack_test
   momentum_sgd_test.cpp
   nbc_test.cpp
   nca_test.cpp
+  nesterov_momentum_sgd_test.cpp
   nmf_test.cpp
   nystroem_method_test.cpp
   octree_test.cpp

src/mlpack/tests/nesterov_momentum_sgd_test.cpp

@@ -0,0 +1,72 @@
+/**
+ * @file nesterov_momentum_sgd_test.cpp
+ * @author Sourabh Varshney
+ *
+ * Test file for NesterovMomentumSGD (Stochastic gradient descent with
+ * nesterov momentum updates).
+ *
+ * mlpack is free software; you may redistribute it and/or modify it under the
+ * terms of the 3-clause BSD license.  You should have received a copy of the
+ * 3-clause BSD license along with mlpack.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#include <mlpack/core.hpp>
+#include <mlpack/core/optimizers/sgd/sgd.hpp>
+#include <mlpack/core/optimizers/sgd/update_policies/gradient_clipping.hpp>
+#include <mlpack/core/optimizers/sgd/update_policies/nesterov_momentum_update.hpp>
+#include <mlpack/core/optimizers/problems/generalized_rosenbrock_function.hpp>
+#include <mlpack/core/optimizers/problems/sgd_test_function.hpp>
+
+#include <boost/test/unit_test.hpp>
+#include "test_tools.hpp"
+
+using namespace std;
+using namespace arma;
+using namespace mlpack;
+using namespace mlpack::optimization;
+using namespace mlpack::optimization::test;
+
+BOOST_AUTO_TEST_SUITE(NesterovMomentumSGDTest);
+
+/*
+* Tests the Nesterov Momentum SGD update policy.
+*/
+BOOST_AUTO_TEST_CASE(NesterovMomentumSGDSpeedUpTestFunction)
+{
+  SGDTestFunction f;
+  NesterovMomentumUpdate nesterovMomentumUpdate(0.9);
+  NesterovMomentumSGD s(0.0003, 1, 2500000, 1e-9, true,
+                        nesterovMomentumUpdate);
+
+  arma::mat coordinates = f.GetInitialPoint();
+  double result = s.Optimize(f, coordinates);
+
+  BOOST_REQUIRE_CLOSE(result, -1.0, 0.15);
+  BOOST_REQUIRE_SMALL(coordinates[0], 1e-3);
+  BOOST_REQUIRE_SMALL(coordinates[1], 1e-7);
+  BOOST_REQUIRE_SMALL(coordinates[2], 1e-7);
+}
+
+/*
+* Tests the Nesterov Momentum SGD with Generalized Rosenbrock Test.
+*/
+BOOST_AUTO_TEST_CASE(GeneralizedRosenbrockTest)
+{
+  // Loop over several variants.
+  for (size_t i = 10; i < 50; i += 5)
+  {
+    // Create the generalized Rosenbrock function.
+    GeneralizedRosenbrockFunction f(i);
+    NesterovMomentumUpdate nesterovMomentumUpdate(0.9);
+    NesterovMomentumSGD s(0.0001, 1, 0, 1e-15, true, nesterovMomentumUpdate);
+
+    arma::mat coordinates = f.GetInitialPoint();
+    double result = s.Optimize(f, coordinates);
+
+    BOOST_REQUIRE_SMALL(result, 1e-4);
+    for (size_t j = 0; j < i; ++j)
+      BOOST_REQUIRE_CLOSE(coordinates[j], (double) 1.0, 1e-3);
+  }
+}
+
+BOOST_AUTO_TEST_SUITE_END();