From 28e471299bae009574127c0703af6eeb8d3d7a6f Mon Sep 17 00:00:00 2001
From: Haritha <haritha1313@gmail.com>
Date: Wed, 13 Dec 2017 13:08:24 +0530
Subject: [PATCH 1/7] ams_grad optimizer implementation

---
 src/mlpack/core/optimizers/CMakeLists.txt     |   1 +
 .../core/optimizers/ams_grad/CMakeLists.txt   |  12 ++
 .../core/optimizers/ams_grad/ams_grad.cpp     |  37 +++++
 .../core/optimizers/ams_grad/ams_grad.hpp     | 157 ++++++++++++++++++
 .../optimizers/ams_grad/amsgrad_update.hpp    | 140 ++++++++++++++++
 src/mlpack/tests/CMakeLists.txt               |   1 +
 src/mlpack/tests/ams_grad_test.cpp            | 107 ++++++++++++
 7 files changed, 455 insertions(+)
 create mode 100644 src/mlpack/core/optimizers/ams_grad/CMakeLists.txt
 create mode 100644 src/mlpack/core/optimizers/ams_grad/ams_grad.cpp
 create mode 100644 src/mlpack/core/optimizers/ams_grad/ams_grad.hpp
 create mode 100644 src/mlpack/core/optimizers/ams_grad/amsgrad_update.hpp
 create mode 100644 src/mlpack/tests/ams_grad_test.cpp

diff --git a/src/mlpack/core/optimizers/CMakeLists.txt b/src/mlpack/core/optimizers/CMakeLists.txt
index 1dd8bb4e51..76493dc5a0 100644
--- a/src/mlpack/core/optimizers/CMakeLists.txt
+++ b/src/mlpack/core/optimizers/CMakeLists.txt
@@ -2,6 +2,7 @@ set(DIRS
   ada_delta
   ada_grad
   adam
+  ams_grad
   aug_lagrangian
   cmaes
   cne
diff --git a/src/mlpack/core/optimizers/ams_grad/CMakeLists.txt b/src/mlpack/core/optimizers/ams_grad/CMakeLists.txt
new file mode 100644
index 0000000000..f138890d5b
--- /dev/null
+++ b/src/mlpack/core/optimizers/ams_grad/CMakeLists.txt
@@ -0,0 +1,12 @@
+set(SOURCES
+  ams_grad.cpp
+  ams_grad.hpp
+  amsgrad_update.hpp
+)
+
+set(DIR_SRCS)
+foreach(file ${SOURCES})
+  set(DIR_SRCS ${DIR_SRCS} ${CMAKE_CURRENT_SOURCE_DIR}/${file})
+endforeach()
+
+set(MLPACK_SRCS ${MLPACK_SRCS} ${DIR_SRCS} PARENT_SCOPE)
diff --git a/src/mlpack/core/optimizers/ams_grad/ams_grad.cpp b/src/mlpack/core/optimizers/ams_grad/ams_grad.cpp
new file mode 100644
index 0000000000..dca009419e
--- /dev/null
+++ b/src/mlpack/core/optimizers/ams_grad/ams_grad.cpp
@@ -0,0 +1,37 @@
+/**
+ * @file ams_grad_impl.hpp
+ * @author Haritha Nair
+ *
+ * Implementation of the AMSGrad Optimizer.
+ *
+ * mlpack is free software; you may redistribute it and/or modify it under the
+ * terms of the 3-clause BSD license.  You should have received a copy of the
+ * 3-clause BSD license along with mlpack.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+// In case it hasn't been included yet.
+#include "ams_grad.hpp"
+
+namespace mlpack {
+namespace optimization {
+
+AmsGrad::AmsGrad(
+    const double stepSize,
+    const size_t batchSize,
+    const double beta1,
+    const double beta2,
+    const double epsilon,
+    const size_t maxIterations,
+    const double tolerance,
+    const bool shuffle) :
+    optimizer(stepSize,
+              batchSize,
+              maxIterations,
+              tolerance,
+              shuffle,
+              AmsGradUpdate(epsilon, beta1, beta2))
+{ /* Nothing to do. */ }
+
+} // namespace optimization
+} // namespace mlpack
+
diff --git a/src/mlpack/core/optimizers/ams_grad/ams_grad.hpp b/src/mlpack/core/optimizers/ams_grad/ams_grad.hpp
new file mode 100644
index 0000000000..76f096cc4d
--- /dev/null
+++ b/src/mlpack/core/optimizers/ams_grad/ams_grad.hpp
@@ -0,0 +1,157 @@
+/**
+ * @file ams_grad.hpp
+ * @author Haritha Nair
+ *
+ * Implementation of AMSGrad optimizer. AMSGrad is an exponential moving average 
+ * optimizer that dynamically adapts over time with guaranteed convergence.
+ *
+ * mlpack is free software; you may redistribute it and/or modify it under the
+ * terms of the 3-clause BSD license.  You should have received a copy of the
+ * 3-clause BSD license along with mlpack.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef MLPACK_CORE_OPTIMIZERS_AMS_GRAD_AMS_GRAD_HPP
+#define MLPACK_CORE_OPTIMIZERS_AMS_GRAD_AMS_GRAD_HPP
+
+#include <mlpack/prereqs.hpp>
+
+#include <mlpack/core/optimizers/sgd/sgd.hpp>
+#include "amsgrad_update.hpp"
+
+namespace mlpack {
+namespace optimization {
+
+/**
+ * AMSGrad is an exponential moving average variant which along with having 
+ * benefits of optimizers like Adam and RmsProp, also guarantees convergence.
+ * Unlike adam, it uses maximum of past squared gradients rather than their 
+ * exponential average for updation.
+ *
+ * For more information, see the following.
+ *
+ * @code
+ * @article{
+ *   title   = {On the convergence of Adam and beyond},
+ *   url     = {https://openreview.net/pdf?id=ryQu7f-RZ}
+ * }
+ * @endcode
+ *
+ *
+ * For AMSGrad to work, a DecomposableFunctionType template parameter
+ * is required. This class must implement the following function:
+ *
+ *   size_t NumFunctions();
+ *   double Evaluate(const arma::mat& coordinates,
+ *                   const size_t i,
+ *                   const size_t batchSize);
+ *   void Gradient(const arma::mat& coordinates,
+ *                 const size_t i,
+ *                 arma::mat& gradient,
+ *                 const size_t batchSize);
+ *
+ * NumFunctions() should return the number of functions (\f$n\f$), and in the
+ * other two functions, the parameter i refers to which individual function (or
+ * gradient) is being evaluated.  So, for the case of a data-dependent function,
+ * such as NCA (see mlpack::nca::NCA), NumFunctions() should return the number
+ * of points in the dataset, and Evaluate(coordinates, 0) will evaluate the
+ * objective function on the first point in the dataset (presumably, the dataset
+ * is held internally in the DecomposableFunctionType).
+ */
+
+class AmsGrad
+{
+ public:
+  /**
+   * Construct the AMSGrad optimizer with the given function and parameters. The
+   * defaults here are not necessarily good for the given problem, so it is
+   * suggested that the values used be tailored to the task at hand.  The
+   * maximum number of iterations refers to the maximum number of points that
+   * are processed (i.e., one iteration equals one point; one iteration does not
+   * equal one pass over the dataset).
+   *
+   * @param stepSize Step size for each iteration.
+   * @param batchSize Number of points to process in a single step.
+   * @param beta1 Exponential decay rate for the first moment estimates.
+   * @param beta2 Exponential decay rate for the weighted infinity norm
+            estimates.
+   * @param eps Value used to initialise the mean squared gradient parameter.
+   * @param maxIterations Maximum number of iterations allowed (0 means no
+   *        limit).
+   * @param tolerance Maximum absolute tolerance to terminate algorithm.
+   * @param shuffle If true, the function order is shuffled; otherwise, each
+   *        function is visited in linear order.
+   */
+  AmsGrad(const double stepSize = 0.001,
+           const size_t batchSize = 32,
+           const double beta1 = 0.9,
+           const double beta2 = 0.999,
+           const double eps = 1e-8,
+           const size_t maxIterations = 100000,
+           const double tolerance = 1e-5,
+           const bool shuffle = true);
+
+  /**
+   * Optimize the given function using AMSGrad. The given starting point will be
+   * modified to store the finishing point of the algorithm, and the final
+   * objective value is returned.
+   *
+   * @tparam DecomposableFunctionType Type of the function to optimize.
+   * @param function Function to optimize.
+   * @param iterate Starting point (will be modified).
+   * @return Objective value of the final point.
+   */
+  template<typename DecomposableFunctionType>
+  double Optimize(DecomposableFunctionType& function, arma::mat& iterate)
+  {
+    return optimizer.Optimize(function, iterate);
+  }
+
+  //! Get the step size.
+  double StepSize() const { return optimizer.StepSize(); }
+  //! Modify the step size.
+  double& StepSize() { return optimizer.StepSize(); }
+
+  //! Get the batch size.
+  size_t BatchSize() const { return optimizer.BatchSize(); }
+  //! Modify the batch size.
+  size_t& BatchSize() { return optimizer.BatchSize(); }
+
+  //! Get the smoothing parameter.
+  double Beta1() const { return optimizer.UpdatePolicy().Beta1(); }
+  //! Modify the smoothing parameter.
+  double& Beta1() { return optimizer.UpdatePolicy().Beta1(); }
+
+  //! Get the second moment coefficient.
+  double Beta2() const { return optimizer.UpdatePolicy().Beta2(); }
+  //! Modify the second moment coefficient.
+  double& Beta2() { return optimizer.UpdatePolicy().Beta2(); }
+
+  //! Get the value used to initialise the mean squared gradient parameter.
+  double Epsilon() const { return optimizer.UpdatePolicy().Epsilon(); }
+  //! Modify the value used to initialise the mean squared gradient parameter.
+  double& Epsilon() { return optimizer.UpdatePolicy().Epsilon(); }
+
+  //! Get the maximum number of iterations (0 indicates no limit).
+  size_t MaxIterations() const { return optimizer.MaxIterations(); }
+  //! Modify the maximum number of iterations (0 indicates no limit).
+  size_t& MaxIterations() { return optimizer.MaxIterations(); }
+
+  //! Get the tolerance for termination.
+  double Tolerance() const { return optimizer.Tolerance(); }
+  //! Modify the tolerance for termination.
+  double& Tolerance() { return optimizer.Tolerance(); }
+
+  //! Get whether or not the individual functions are shuffled.
+  bool Shuffle() const { return optimizer.Shuffle(); }
+  //! Modify whether or not the individual functions are shuffled.
+  bool& Shuffle() { return optimizer.Shuffle(); }
+
+ private:
+  //! The Stochastic Gradient Descent object with AMSGrad policy.
+  SGD<AmsGradUpdate> optimizer;
+};
+
+} // namespace optimization
+} // namespace mlpack
+
+#endif
diff --git a/src/mlpack/core/optimizers/ams_grad/amsgrad_update.hpp b/src/mlpack/core/optimizers/ams_grad/amsgrad_update.hpp
new file mode 100644
index 0000000000..82d94e7143
--- /dev/null
+++ b/src/mlpack/core/optimizers/ams_grad/amsgrad_update.hpp
@@ -0,0 +1,140 @@
+/**
+ * @file amsgrad_update.hpp
+ * @author Haritha Nair
+ *
+ * Implementation of AMSGrad optimizer. AMSGrad is an exponential moving average 
+ * optimizer that dynamically adapts over time with guaranteed convergence.
+ *
+ * mlpack is free software; you may redistribute it and/or modify it under the
+ * terms of the 3-clause BSD license.  You should have received a copy of the
+ * 3-clause BSD license along with mlpack.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef MLPACK_CORE_OPTIMIZERS_AMS_GRAD_AMS_GRAD_UPDATE_HPP
+#define MLPACK_CORE_OPTIMIZERS_AMS_GRAD_AMS_GRAD_UPDATE_HPP
+
+#include <mlpack/prereqs.hpp>
+
+namespace mlpack {
+namespace optimization {
+
+/**
+ * AMSGrad is an exponential moving average variant which along with having 
+ * benefits of optimizers like Adam and RmsProp, also guarantees convergence.
+ * Unlike adam, it uses maximum of past squared gradients rather than their 
+ * exponential average for updation.
+ *
+ * For more information, see the following.
+ *
+ * @code
+ * @article{
+ *   title   = {On the convergence of Adam and beyond},
+ *   url     = {https://openreview.net/pdf?id=ryQu7f-RZ}
+ * }
+ * @endcode
+ */
+class AmsGradUpdate
+{
+ public:
+  /**
+   * Construct the AMSGrad update policy with the given parameters.
+   *
+   * @param epsilon The epsilon value used to initialise the squared gradient
+   *        parameter.
+   * @param beta1 The smoothing parameter.
+   * @param beta2 The second moment coefficient.
+   */
+  AmsGradUpdate(const double epsilon = 1e-8,
+             const double beta1 = 0.9,
+             const double beta2 = 0.999) :
+    epsilon(epsilon),
+    beta1(beta1),
+    beta2(beta2),
+    iteration(0)
+  {
+    // Nothing to do.
+  }
+
+  /**
+   * The Initialize method is called by SGD Optimizer method before the start of
+   * the iteration update process.
+   *
+   * @param rows Number of rows in the gradient matrix.
+   * @param cols Number of columns in the gradient matrix.
+   */
+  void Initialize(const size_t rows, const size_t cols)
+  {
+    m = arma::zeros<arma::mat>(rows, cols);
+    v = arma::zeros<arma::mat>(rows, cols);
+    vImproved = arma::zeros<arma::mat>(rows, cols);
+  }
+
+  /**
+   * Update step for AMSGrad.
+   *
+   * @param iterate Parameters that minimize the function.
+   * @param stepSize Step size to be used for the given iteration.
+   * @param gradient The gradient matrix.
+   */
+  void Update(arma::mat& iterate,
+              const double stepSize,
+              const arma::mat& gradient)
+  {
+    // Increment the iteration counter variable.
+    ++iteration;
+
+    // And update the iterate.
+    m *= beta1;
+    m += (1 - beta1) * gradient;
+
+    v *= beta2;
+    v += (1 - beta2) * (gradient % gradient);
+
+    vImproved = arma::max(vImproved, v);
+    // Element wise maximum of past and present squared gradients.
+
+    iterate -= (stepSize * m) / (arma::sqrt(vImproved) + epsilon);
+  }
+
+  //! Get the value used to initialise the squared gradient parameter.
+  double Epsilon() const { return epsilon; }
+  //! Modify the value used to initialise the squared gradient parameter.
+  double& Epsilon() { return epsilon; }
+
+  //! Get the smoothing parameter.
+  double Beta1() const { return beta1; }
+  //! Modify the smoothing parameter.
+  double& Beta1() { return beta1; }
+
+  //! Get the second moment coefficient.
+  double Beta2() const { return beta2; }
+  //! Modify the second moment coefficient.
+  double& Beta2() { return beta2; }
+
+ private:
+  // The epsilon value used to initialise the squared gradient parameter.
+  double epsilon;
+
+  // The smoothing parameter.
+  double beta1;
+
+  // The second moment coefficient.
+  double beta2;
+
+  // The exponential moving average of gradient values.
+  arma::mat m;
+
+  // The exponential moving average of squared gradient values.
+  arma::mat v;
+
+  //The optimal sqaured gradient value.
+  arma::mat vImproved;
+
+  // The number of iterations.
+  double iteration;
+};
+
+} // namespace optimization
+} // namespace mlpack
+
+#endif
diff --git a/src/mlpack/tests/CMakeLists.txt b/src/mlpack/tests/CMakeLists.txt
index fc6ee57e48..5dbe1b096c 100644
--- a/src/mlpack/tests/CMakeLists.txt
+++ b/src/mlpack/tests/CMakeLists.txt
@@ -7,6 +7,7 @@ add_executable(mlpack_test
   ada_grad_test.cpp
   akfn_test.cpp
   aknn_test.cpp
+  ams_grad_test.cpp
   ann_layer_test.cpp
   armadillo_svd_test.cpp
   arma_extend_test.cpp
diff --git a/src/mlpack/tests/ams_grad_test.cpp b/src/mlpack/tests/ams_grad_test.cpp
new file mode 100644
index 0000000000..1a834b0e92
--- /dev/null
+++ b/src/mlpack/tests/ams_grad_test.cpp
@@ -0,0 +1,107 @@
+/**
+ * @file ams_grad_test.cpp
+ * @author Haritha Nair
+ *
+ * Tests the AMSGrad optimizer.
+ *
+ * mlpack is free software; you may redistribute it and/or modify it under the
+ * terms of the 3-clause BSD license.  You should have received a copy of the
+ * 3-clause BSD license along with mlpack.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#include <mlpack/core.hpp>
+
+#include <mlpack/core/optimizers/ams_grad/ams_grad.hpp>
+#include <mlpack/core/optimizers/sgd/test_function.hpp>
+#include <mlpack/methods/logistic_regression/logistic_regression.hpp>
+
+#include <boost/test/unit_test.hpp>
+#include "test_tools.hpp"
+
+using namespace arma;
+using namespace mlpack::optimization;
+using namespace mlpack::optimization::test;
+
+using namespace mlpack::distribution;
+using namespace mlpack::regression;
+
+using namespace mlpack;
+
+BOOST_AUTO_TEST_SUITE(AmsGradTest);
+
+/**
+ * Tests the AMSGrad optimizer using a simple test function.
+ */
+BOOST_AUTO_TEST_CASE(SimpleAmsGradTestFunction)
+{
+  SGDTestFunction f;
+  AmsGrad optimizer(1e-3, 1, 0.9, 0.999, 1e-8, 500000, 1e-9, true);
+
+  arma::mat coordinates = f.GetInitialPoint();
+  optimizer.Optimize(f, coordinates);
+
+  BOOST_REQUIRE_SMALL(coordinates[0], 0.1);
+  BOOST_REQUIRE_SMALL(coordinates[1], 0.1);
+  BOOST_REQUIRE_SMALL(coordinates[2], 0.1);
+}
+
+
+/**
+ * Run AMSGrad on logistic regression and make sure the results are acceptable.
+ */
+BOOST_AUTO_TEST_CASE(AmsGradLogisticRegressionTest)
+{
+  // Generate a two-Gaussian dataset.
+  GaussianDistribution g1(arma::vec("1.0 1.0 1.0"), arma::eye<arma::mat>(3, 3));
+  GaussianDistribution g2(arma::vec("9.0 9.0 9.0"), arma::eye<arma::mat>(3, 3));
+
+  arma::mat data(3, 1000);
+  arma::Row<size_t> responses(1000);
+  for (size_t i = 0; i < 500; ++i)
+  {
+    data.col(i) = g1.Random();
+    responses[i] = 0;
+  }
+  for (size_t i = 500; i < 1000; ++i)
+  {
+    data.col(i) = g2.Random();
+    responses[i] = 1;
+  }
+
+  // Shuffle the dataset.
+  arma::uvec indices = arma::shuffle(arma::linspace<arma::uvec>(0,
+      data.n_cols - 1, data.n_cols));
+  arma::mat shuffledData(3, 1000);
+  arma::Row<size_t> shuffledResponses(1000);
+  for (size_t i = 0; i < data.n_cols; ++i)
+  {
+    shuffledData.col(i) = data.col(indices[i]);
+    shuffledResponses[i] = responses[indices[i]];
+  }
+
+  // Create a test set.
+  arma::mat testData(3, 1000);
+  arma::Row<size_t> testResponses(1000);
+  for (size_t i = 0; i < 500; ++i)
+  {
+    testData.col(i) = g1.Random();
+    testResponses[i] = 0;
+  }
+  for (size_t i = 500; i < 1000; ++i)
+  {
+    testData.col(i) = g2.Random();
+    testResponses[i] = 1;
+  }
+
+  AmsGrad amsgrad;
+  LogisticRegression<> lr(shuffledData, shuffledResponses, amsgrad, 0.5);
+
+  // Ensure that the error is close to zero.
+  const double acc = lr.ComputeAccuracy(data, responses);
+  BOOST_REQUIRE_CLOSE(acc, 100.0, 0.3); // 0.3% error tolerance.
+
+  const double testAcc = lr.ComputeAccuracy(testData, testResponses);
+  BOOST_REQUIRE_CLOSE(testAcc, 100.0, 0.6); // 0.6% error tolerance.
+}
+
+BOOST_AUTO_TEST_SUITE_END();

From b86f7d1fcf4d272768e0b3c52ac146ecf96318e2 Mon Sep 17 00:00:00 2001
From: Haritha <haritha1313@gmail.com>
Date: Wed, 13 Dec 2017 21:50:07 +0530
Subject: [PATCH 2/7] AMSGrad optimizer

---
 src/mlpack/core/optimizers/ams_grad/ams_grad.hpp       | 2 +-
 src/mlpack/core/optimizers/ams_grad/amsgrad_update.hpp | 6 +++---
 src/mlpack/tests/ams_grad_test.cpp                     | 2 +-
 3 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/src/mlpack/core/optimizers/ams_grad/ams_grad.hpp b/src/mlpack/core/optimizers/ams_grad/ams_grad.hpp
index 76f096cc4d..a8f543ca52 100644
--- a/src/mlpack/core/optimizers/ams_grad/ams_grad.hpp
+++ b/src/mlpack/core/optimizers/ams_grad/ams_grad.hpp
@@ -87,7 +87,7 @@ class AmsGrad
            const double beta2 = 0.999,
            const double eps = 1e-8,
            const size_t maxIterations = 100000,
-           const double tolerance = 1e-5,
+           const double tolerance = 1e-11,
            const bool shuffle = true);
 
   /**
diff --git a/src/mlpack/core/optimizers/ams_grad/amsgrad_update.hpp b/src/mlpack/core/optimizers/ams_grad/amsgrad_update.hpp
index 82d94e7143..b12525daab 100644
--- a/src/mlpack/core/optimizers/ams_grad/amsgrad_update.hpp
+++ b/src/mlpack/core/optimizers/ams_grad/amsgrad_update.hpp
@@ -90,9 +90,9 @@ class AmsGradUpdate
     v *= beta2;
     v += (1 - beta2) * (gradient % gradient);
 
-    vImproved = arma::max(vImproved, v);
     // Element wise maximum of past and present squared gradients.
-
+    vImproved = arma::max(vImproved, v);
+    
     iterate -= (stepSize * m) / (arma::sqrt(vImproved) + epsilon);
   }
 
@@ -127,7 +127,7 @@ class AmsGradUpdate
   // The exponential moving average of squared gradient values.
   arma::mat v;
 
-  //The optimal sqaured gradient value.
+  // The optimal sqaured gradient value.
   arma::mat vImproved;
 
   // The number of iterations.
diff --git a/src/mlpack/tests/ams_grad_test.cpp b/src/mlpack/tests/ams_grad_test.cpp
index 1a834b0e92..ecfe9a4bbd 100644
--- a/src/mlpack/tests/ams_grad_test.cpp
+++ b/src/mlpack/tests/ams_grad_test.cpp
@@ -35,7 +35,7 @@ BOOST_AUTO_TEST_SUITE(AmsGradTest);
 BOOST_AUTO_TEST_CASE(SimpleAmsGradTestFunction)
 {
   SGDTestFunction f;
-  AmsGrad optimizer(1e-3, 1, 0.9, 0.999, 1e-8, 500000, 1e-9, true);
+  AmsGrad optimizer(1e-3, 1, 0.9, 0.999, 1e-8, 500000, 1e-11, true);
 
   arma::mat coordinates = f.GetInitialPoint();
   optimizer.Optimize(f, coordinates);

From ebd0cb3f301822efaf23217399ce2fc654c18bf8 Mon Sep 17 00:00:00 2001
From: Haritha <haritha1313@gmail.com>
Date: Wed, 13 Dec 2017 21:53:20 +0530
Subject: [PATCH 3/7] AMSGrad optimizer

---
 src/mlpack/core/optimizers/ams_grad/amsgrad_update.hpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/mlpack/core/optimizers/ams_grad/amsgrad_update.hpp b/src/mlpack/core/optimizers/ams_grad/amsgrad_update.hpp
index b12525daab..321e85665e 100644
--- a/src/mlpack/core/optimizers/ams_grad/amsgrad_update.hpp
+++ b/src/mlpack/core/optimizers/ams_grad/amsgrad_update.hpp
@@ -92,7 +92,7 @@ class AmsGradUpdate
 
     // Element wise maximum of past and present squared gradients.
     vImproved = arma::max(vImproved, v);
-    
+
     iterate -= (stepSize * m) / (arma::sqrt(vImproved) + epsilon);
   }
 

From d267dd45f9eb03790f6589df8816b8bb718c7d0f Mon Sep 17 00:00:00 2001
From: Haritha <haritha1313@gmail.com>
Date: Thu, 14 Dec 2017 13:49:09 +0530
Subject: [PATCH 4/7] ams_grad optimizer

---
 src/mlpack/core/optimizers/CMakeLists.txt     |   1 -
 .../core/optimizers/adam/CMakeLists.txt       |   1 +
 src/mlpack/core/optimizers/adam/adam.hpp      |  13 +-
 src/mlpack/core/optimizers/adam/adam_impl.hpp |   2 +-
 .../{ams_grad => adam}/amsgrad_update.hpp     |  10 +-
 .../core/optimizers/ams_grad/CMakeLists.txt   |  12 --
 .../core/optimizers/ams_grad/ams_grad.cpp     |  37 -----
 .../core/optimizers/ams_grad/ams_grad.hpp     | 157 ------------------
 src/mlpack/tests/CMakeLists.txt               |   1 -
 src/mlpack/tests/adam_test.cpp                |  76 ++++++++-
 src/mlpack/tests/ams_grad_test.cpp            | 107 ------------
 11 files changed, 94 insertions(+), 323 deletions(-)
 rename src/mlpack/core/optimizers/{ams_grad => adam}/amsgrad_update.hpp (91%)
 delete mode 100644 src/mlpack/core/optimizers/ams_grad/CMakeLists.txt
 delete mode 100644 src/mlpack/core/optimizers/ams_grad/ams_grad.cpp
 delete mode 100644 src/mlpack/core/optimizers/ams_grad/ams_grad.hpp
 delete mode 100644 src/mlpack/tests/ams_grad_test.cpp

diff --git a/src/mlpack/core/optimizers/CMakeLists.txt b/src/mlpack/core/optimizers/CMakeLists.txt
index 76493dc5a0..1dd8bb4e51 100644
--- a/src/mlpack/core/optimizers/CMakeLists.txt
+++ b/src/mlpack/core/optimizers/CMakeLists.txt
@@ -2,7 +2,6 @@ set(DIRS
   ada_delta
   ada_grad
   adam
-  ams_grad
   aug_lagrangian
   cmaes
   cne
diff --git a/src/mlpack/core/optimizers/adam/CMakeLists.txt b/src/mlpack/core/optimizers/adam/CMakeLists.txt
index 1377bbb0e1..644797090f 100644
--- a/src/mlpack/core/optimizers/adam/CMakeLists.txt
+++ b/src/mlpack/core/optimizers/adam/CMakeLists.txt
@@ -3,6 +3,7 @@ set(SOURCES
   adam_impl.hpp
   adam_update.hpp
   adamax_update.hpp
+  amsgrad_update.hpp
 )
 
 set(DIR_SRCS)
diff --git a/src/mlpack/core/optimizers/adam/adam.hpp b/src/mlpack/core/optimizers/adam/adam.hpp
index 3b109660f5..a1ec694543 100644
--- a/src/mlpack/core/optimizers/adam/adam.hpp
+++ b/src/mlpack/core/optimizers/adam/adam.hpp
@@ -5,10 +5,10 @@
  * @author Marcus Edel
  * @author Vivek Pal
  *
- * Adam and AdaMax optimizer. Adam is an an algorithm for first-order gradient-
+ * Adam, AdaMax and AMSGrad optimizer. Adam is an an algorithm for first-order gradient-
  * -based optimization of stochastic objective functions, based on adaptive
  * estimates of lower-order moments. AdaMax is simply a variant of Adam based
- * on the infinity norm.
+ * on the infinity norm. AMSGrad is another variant of Adam with guaranteed convergence.
  *
  * mlpack is free software; you may redistribute it and/or modify it under the
  * terms of the 3-clause BSD license.  You should have received a copy of the
@@ -23,6 +23,7 @@
 #include <mlpack/core/optimizers/sgd/sgd.hpp>
 #include "adam_update.hpp"
 #include "adamax_update.hpp"
+#include "amsgrad_update.hpp"
 
 namespace mlpack {
 namespace optimization {
@@ -43,10 +44,14 @@ namespace optimization {
  *   year    = {2014},
  *   url     = {http://arxiv.org/abs/1412.6980}
  * }
+ * @article{
+ *   title   = {On the convergence of Adam and beyond},
+ *   url     = {https://openreview.net/pdf?id=ryQu7f-RZ}
+ * }
  * @endcode
  *
  *
- * For Adam and AdaMax to work, a DecomposableFunctionType template parameter
+ * For Adam, AdaMax and AMSGrad to work, a DecomposableFunctionType template parameter
  * is required. This class must implement the following function:
  *
  *   size_t NumFunctions();
@@ -166,6 +171,8 @@ using Adam = AdamType<AdamUpdate>;
 
 using AdaMax = AdamType<AdaMaxUpdate>;
 
+using AmsGrad = AdamType<AmsGradUpdate>;
+
 } // namespace optimization
 } // namespace mlpack
 
diff --git a/src/mlpack/core/optimizers/adam/adam_impl.hpp b/src/mlpack/core/optimizers/adam/adam_impl.hpp
index 326d038d15..49497f5af5 100644
--- a/src/mlpack/core/optimizers/adam/adam_impl.hpp
+++ b/src/mlpack/core/optimizers/adam/adam_impl.hpp
@@ -5,7 +5,7 @@
  * @author Marcus Edel
  * @author Vivek Pal
  *
- * Implementation of the Adam and AdaMax optimizer.
+ * Implementation of the Adam, AdaMax and AMSGrad optimizer.
  *
  * mlpack is free software; you may redistribute it and/or modify it under the
  * terms of the 3-clause BSD license.  You should have received a copy of the
diff --git a/src/mlpack/core/optimizers/ams_grad/amsgrad_update.hpp b/src/mlpack/core/optimizers/adam/amsgrad_update.hpp
similarity index 91%
rename from src/mlpack/core/optimizers/ams_grad/amsgrad_update.hpp
rename to src/mlpack/core/optimizers/adam/amsgrad_update.hpp
index 321e85665e..49a8a4ca6e 100644
--- a/src/mlpack/core/optimizers/ams_grad/amsgrad_update.hpp
+++ b/src/mlpack/core/optimizers/adam/amsgrad_update.hpp
@@ -45,8 +45,8 @@ class AmsGradUpdate
    * @param beta2 The second moment coefficient.
    */
   AmsGradUpdate(const double epsilon = 1e-8,
-             const double beta1 = 0.9,
-             const double beta2 = 0.999) :
+                const double beta1 = 0.9,
+                const double beta2 = 0.999) :
     epsilon(epsilon),
     beta1(beta1),
     beta2(beta2),
@@ -90,10 +90,14 @@ class AmsGradUpdate
     v *= beta2;
     v += (1 - beta2) * (gradient % gradient);
 
+    const double biasCorrection1 = 1.0 - std::pow(beta1, iteration);
+    const double biasCorrection2 = 1.0 - std::pow(beta2, iteration);
+
     // Element wise maximum of past and present squared gradients.
     vImproved = arma::max(vImproved, v);
 
-    iterate -= (stepSize * m) / (arma::sqrt(vImproved) + epsilon);
+    iterate -= (stepSize * std::sqrt(biasCorrection2) / biasCorrection1) *
+                m / (arma::sqrt(vImproved) + epsilon);
   }
 
   //! Get the value used to initialise the squared gradient parameter.
diff --git a/src/mlpack/core/optimizers/ams_grad/CMakeLists.txt b/src/mlpack/core/optimizers/ams_grad/CMakeLists.txt
deleted file mode 100644
index f138890d5b..0000000000
--- a/src/mlpack/core/optimizers/ams_grad/CMakeLists.txt
+++ /dev/null
@@ -1,12 +0,0 @@
-set(SOURCES
-  ams_grad.cpp
-  ams_grad.hpp
-  amsgrad_update.hpp
-)
-
-set(DIR_SRCS)
-foreach(file ${SOURCES})
-  set(DIR_SRCS ${DIR_SRCS} ${CMAKE_CURRENT_SOURCE_DIR}/${file})
-endforeach()
-
-set(MLPACK_SRCS ${MLPACK_SRCS} ${DIR_SRCS} PARENT_SCOPE)
diff --git a/src/mlpack/core/optimizers/ams_grad/ams_grad.cpp b/src/mlpack/core/optimizers/ams_grad/ams_grad.cpp
deleted file mode 100644
index dca009419e..0000000000
--- a/src/mlpack/core/optimizers/ams_grad/ams_grad.cpp
+++ /dev/null
@@ -1,37 +0,0 @@
-/**
- * @file ams_grad_impl.hpp
- * @author Haritha Nair
- *
- * Implementation of the AMSGrad Optimizer.
- *
- * mlpack is free software; you may redistribute it and/or modify it under the
- * terms of the 3-clause BSD license.  You should have received a copy of the
- * 3-clause BSD license along with mlpack.  If not, see
- * http://www.opensource.org/licenses/BSD-3-Clause for more information.
- */
-// In case it hasn't been included yet.
-#include "ams_grad.hpp"
-
-namespace mlpack {
-namespace optimization {
-
-AmsGrad::AmsGrad(
-    const double stepSize,
-    const size_t batchSize,
-    const double beta1,
-    const double beta2,
-    const double epsilon,
-    const size_t maxIterations,
-    const double tolerance,
-    const bool shuffle) :
-    optimizer(stepSize,
-              batchSize,
-              maxIterations,
-              tolerance,
-              shuffle,
-              AmsGradUpdate(epsilon, beta1, beta2))
-{ /* Nothing to do. */ }
-
-} // namespace optimization
-} // namespace mlpack
-
diff --git a/src/mlpack/core/optimizers/ams_grad/ams_grad.hpp b/src/mlpack/core/optimizers/ams_grad/ams_grad.hpp
deleted file mode 100644
index a8f543ca52..0000000000
--- a/src/mlpack/core/optimizers/ams_grad/ams_grad.hpp
+++ /dev/null
@@ -1,157 +0,0 @@
-/**
- * @file ams_grad.hpp
- * @author Haritha Nair
- *
- * Implementation of AMSGrad optimizer. AMSGrad is an exponential moving average 
- * optimizer that dynamically adapts over time with guaranteed convergence.
- *
- * mlpack is free software; you may redistribute it and/or modify it under the
- * terms of the 3-clause BSD license.  You should have received a copy of the
- * 3-clause BSD license along with mlpack.  If not, see
- * http://www.opensource.org/licenses/BSD-3-Clause for more information.
- */
-#ifndef MLPACK_CORE_OPTIMIZERS_AMS_GRAD_AMS_GRAD_HPP
-#define MLPACK_CORE_OPTIMIZERS_AMS_GRAD_AMS_GRAD_HPP
-
-#include <mlpack/prereqs.hpp>
-
-#include <mlpack/core/optimizers/sgd/sgd.hpp>
-#include "amsgrad_update.hpp"
-
-namespace mlpack {
-namespace optimization {
-
-/**
- * AMSGrad is an exponential moving average variant which along with having 
- * benefits of optimizers like Adam and RmsProp, also guarantees convergence.
- * Unlike adam, it uses maximum of past squared gradients rather than their 
- * exponential average for updation.
- *
- * For more information, see the following.
- *
- * @code
- * @article{
- *   title   = {On the convergence of Adam and beyond},
- *   url     = {https://openreview.net/pdf?id=ryQu7f-RZ}
- * }
- * @endcode
- *
- *
- * For AMSGrad to work, a DecomposableFunctionType template parameter
- * is required. This class must implement the following function:
- *
- *   size_t NumFunctions();
- *   double Evaluate(const arma::mat& coordinates,
- *                   const size_t i,
- *                   const size_t batchSize);
- *   void Gradient(const arma::mat& coordinates,
- *                 const size_t i,
- *                 arma::mat& gradient,
- *                 const size_t batchSize);
- *
- * NumFunctions() should return the number of functions (\f$n\f$), and in the
- * other two functions, the parameter i refers to which individual function (or
- * gradient) is being evaluated.  So, for the case of a data-dependent function,
- * such as NCA (see mlpack::nca::NCA), NumFunctions() should return the number
- * of points in the dataset, and Evaluate(coordinates, 0) will evaluate the
- * objective function on the first point in the dataset (presumably, the dataset
- * is held internally in the DecomposableFunctionType).
- */
-
-class AmsGrad
-{
- public:
-  /**
-   * Construct the AMSGrad optimizer with the given function and parameters. The
-   * defaults here are not necessarily good for the given problem, so it is
-   * suggested that the values used be tailored to the task at hand.  The
-   * maximum number of iterations refers to the maximum number of points that
-   * are processed (i.e., one iteration equals one point; one iteration does not
-   * equal one pass over the dataset).
-   *
-   * @param stepSize Step size for each iteration.
-   * @param batchSize Number of points to process in a single step.
-   * @param beta1 Exponential decay rate for the first moment estimates.
-   * @param beta2 Exponential decay rate for the weighted infinity norm
-            estimates.
-   * @param eps Value used to initialise the mean squared gradient parameter.
-   * @param maxIterations Maximum number of iterations allowed (0 means no
-   *        limit).
-   * @param tolerance Maximum absolute tolerance to terminate algorithm.
-   * @param shuffle If true, the function order is shuffled; otherwise, each
-   *        function is visited in linear order.
-   */
-  AmsGrad(const double stepSize = 0.001,
-           const size_t batchSize = 32,
-           const double beta1 = 0.9,
-           const double beta2 = 0.999,
-           const double eps = 1e-8,
-           const size_t maxIterations = 100000,
-           const double tolerance = 1e-11,
-           const bool shuffle = true);
-
-  /**
-   * Optimize the given function using AMSGrad. The given starting point will be
-   * modified to store the finishing point of the algorithm, and the final
-   * objective value is returned.
-   *
-   * @tparam DecomposableFunctionType Type of the function to optimize.
-   * @param function Function to optimize.
-   * @param iterate Starting point (will be modified).
-   * @return Objective value of the final point.
-   */
-  template<typename DecomposableFunctionType>
-  double Optimize(DecomposableFunctionType& function, arma::mat& iterate)
-  {
-    return optimizer.Optimize(function, iterate);
-  }
-
-  //! Get the step size.
-  double StepSize() const { return optimizer.StepSize(); }
-  //! Modify the step size.
-  double& StepSize() { return optimizer.StepSize(); }
-
-  //! Get the batch size.
-  size_t BatchSize() const { return optimizer.BatchSize(); }
-  //! Modify the batch size.
-  size_t& BatchSize() { return optimizer.BatchSize(); }
-
-  //! Get the smoothing parameter.
-  double Beta1() const { return optimizer.UpdatePolicy().Beta1(); }
-  //! Modify the smoothing parameter.
-  double& Beta1() { return optimizer.UpdatePolicy().Beta1(); }
-
-  //! Get the second moment coefficient.
-  double Beta2() const { return optimizer.UpdatePolicy().Beta2(); }
-  //! Modify the second moment coefficient.
-  double& Beta2() { return optimizer.UpdatePolicy().Beta2(); }
-
-  //! Get the value used to initialise the mean squared gradient parameter.
-  double Epsilon() const { return optimizer.UpdatePolicy().Epsilon(); }
-  //! Modify the value used to initialise the mean squared gradient parameter.
-  double& Epsilon() { return optimizer.UpdatePolicy().Epsilon(); }
-
-  //! Get the maximum number of iterations (0 indicates no limit).
-  size_t MaxIterations() const { return optimizer.MaxIterations(); }
-  //! Modify the maximum number of iterations (0 indicates no limit).
-  size_t& MaxIterations() { return optimizer.MaxIterations(); }
-
-  //! Get the tolerance for termination.
-  double Tolerance() const { return optimizer.Tolerance(); }
-  //! Modify the tolerance for termination.
-  double& Tolerance() { return optimizer.Tolerance(); }
-
-  //! Get whether or not the individual functions are shuffled.
-  bool Shuffle() const { return optimizer.Shuffle(); }
-  //! Modify whether or not the individual functions are shuffled.
-  bool& Shuffle() { return optimizer.Shuffle(); }
-
- private:
-  //! The Stochastic Gradient Descent object with AMSGrad policy.
-  SGD<AmsGradUpdate> optimizer;
-};
-
-} // namespace optimization
-} // namespace mlpack
-
-#endif
diff --git a/src/mlpack/tests/CMakeLists.txt b/src/mlpack/tests/CMakeLists.txt
index 5dbe1b096c..fc6ee57e48 100644
--- a/src/mlpack/tests/CMakeLists.txt
+++ b/src/mlpack/tests/CMakeLists.txt
@@ -7,7 +7,6 @@ add_executable(mlpack_test
   ada_grad_test.cpp
   akfn_test.cpp
   aknn_test.cpp
-  ams_grad_test.cpp
   ann_layer_test.cpp
   armadillo_svd_test.cpp
   arma_extend_test.cpp
diff --git a/src/mlpack/tests/adam_test.cpp b/src/mlpack/tests/adam_test.cpp
index c1431a860c..b0279aa1ea 100644
--- a/src/mlpack/tests/adam_test.cpp
+++ b/src/mlpack/tests/adam_test.cpp
@@ -3,7 +3,7 @@
  * @author Vasanth Kalingeri
  * @author Vivek Pal
  *
- * Tests the Adam and AdaMax optimizer.
+ * Tests the Adam, AdaMax and AMSGrad optimizer.
  *
  * mlpack is free software; you may redistribute it and/or modify it under the
  * terms of the 3-clause BSD license.  You should have received a copy of the
@@ -62,6 +62,22 @@ BOOST_AUTO_TEST_CASE(SimpleAdaMaxTestFunction)
   BOOST_REQUIRE_SMALL(coordinates[2], 0.1);
 }
 
+/**
+ * Tests the AMSGrad optimizer using a simple test function.
+ */
+BOOST_AUTO_TEST_CASE(SimpleAmsGradTestFunction)
+{
+  SGDTestFunction f;
+  AmsGrad optimizer(1e-3, 1, 0.9, 0.999, 1e-8, 500000, 1e-11, true);
+
+  arma::mat coordinates = f.GetInitialPoint();
+  optimizer.Optimize(f, coordinates);
+
+  BOOST_REQUIRE_SMALL(coordinates[0], 0.1);
+  BOOST_REQUIRE_SMALL(coordinates[1], 0.1);
+  BOOST_REQUIRE_SMALL(coordinates[2], 0.1);
+}
+
 /**
  * Run Adam on logistic regression and make sure the results are acceptable.
  */
@@ -178,4 +194,62 @@ BOOST_AUTO_TEST_CASE(AdaMaxLogisticRegressionTest)
   BOOST_REQUIRE_CLOSE(testAcc, 100.0, 0.6); // 0.6% error tolerance.
 }
 
+/**
+ * Run AMSGrad on logistic regression and make sure the results are acceptable.
+ */
+BOOST_AUTO_TEST_CASE(AmsGradLogisticRegressionTest)
+{
+  // Generate a two-Gaussian dataset.
+  GaussianDistribution g1(arma::vec("1.0 1.0 1.0"), arma::eye<arma::mat>(3, 3));
+  GaussianDistribution g2(arma::vec("9.0 9.0 9.0"), arma::eye<arma::mat>(3, 3));
+
+  arma::mat data(3, 1000);
+  arma::Row<size_t> responses(1000);
+  for (size_t i = 0; i < 500; ++i)
+  {
+    data.col(i) = g1.Random();
+    responses[i] = 0;
+  }
+  for (size_t i = 500; i < 1000; ++i)
+  {
+    data.col(i) = g2.Random();
+    responses[i] = 1;
+  }
+
+  // Shuffle the dataset.
+  arma::uvec indices = arma::shuffle(arma::linspace<arma::uvec>(0,
+      data.n_cols - 1, data.n_cols));
+  arma::mat shuffledData(3, 1000);
+  arma::Row<size_t> shuffledResponses(1000);
+  for (size_t i = 0; i < data.n_cols; ++i)
+  {
+    shuffledData.col(i) = data.col(indices[i]);
+    shuffledResponses[i] = responses[indices[i]];
+  }
+
+  // Create a test set.
+  arma::mat testData(3, 1000);
+  arma::Row<size_t> testResponses(1000);
+  for (size_t i = 0; i < 500; ++i)
+  {
+    testData.col(i) = g1.Random();
+    testResponses[i] = 0;
+  }
+  for (size_t i = 500; i < 1000; ++i)
+  {
+    testData.col(i) = g2.Random();
+    testResponses[i] = 1;
+  }
+
+  AmsGrad amsgrad(1e-3, 1, 0.9, 0.999, 1e-8, 500000, 1e-11, true);
+  LogisticRegression<> lr(shuffledData, shuffledResponses, amsgrad, 0.5);
+
+  // Ensure that the error is close to zero.
+  const double acc = lr.ComputeAccuracy(data, responses);
+  BOOST_REQUIRE_CLOSE(acc, 100.0, 0.3); // 0.3% error tolerance.
+
+  const double testAcc = lr.ComputeAccuracy(testData, testResponses);
+  BOOST_REQUIRE_CLOSE(testAcc, 100.0, 0.6); // 0.6% error tolerance.
+}
+
 BOOST_AUTO_TEST_SUITE_END();
diff --git a/src/mlpack/tests/ams_grad_test.cpp b/src/mlpack/tests/ams_grad_test.cpp
deleted file mode 100644
index ecfe9a4bbd..0000000000
--- a/src/mlpack/tests/ams_grad_test.cpp
+++ /dev/null
@@ -1,107 +0,0 @@
-/**
- * @file ams_grad_test.cpp
- * @author Haritha Nair
- *
- * Tests the AMSGrad optimizer.
- *
- * mlpack is free software; you may redistribute it and/or modify it under the
- * terms of the 3-clause BSD license.  You should have received a copy of the
- * 3-clause BSD license along with mlpack.  If not, see
- * http://www.opensource.org/licenses/BSD-3-Clause for more information.
- */
-#include <mlpack/core.hpp>
-
-#include <mlpack/core/optimizers/ams_grad/ams_grad.hpp>
-#include <mlpack/core/optimizers/sgd/test_function.hpp>
-#include <mlpack/methods/logistic_regression/logistic_regression.hpp>
-
-#include <boost/test/unit_test.hpp>
-#include "test_tools.hpp"
-
-using namespace arma;
-using namespace mlpack::optimization;
-using namespace mlpack::optimization::test;
-
-using namespace mlpack::distribution;
-using namespace mlpack::regression;
-
-using namespace mlpack;
-
-BOOST_AUTO_TEST_SUITE(AmsGradTest);
-
-/**
- * Tests the AMSGrad optimizer using a simple test function.
- */
-BOOST_AUTO_TEST_CASE(SimpleAmsGradTestFunction)
-{
-  SGDTestFunction f;
-  AmsGrad optimizer(1e-3, 1, 0.9, 0.999, 1e-8, 500000, 1e-11, true);
-
-  arma::mat coordinates = f.GetInitialPoint();
-  optimizer.Optimize(f, coordinates);
-
-  BOOST_REQUIRE_SMALL(coordinates[0], 0.1);
-  BOOST_REQUIRE_SMALL(coordinates[1], 0.1);
-  BOOST_REQUIRE_SMALL(coordinates[2], 0.1);
-}
-
-
-/**
- * Run AMSGrad on logistic regression and make sure the results are acceptable.
- */
-BOOST_AUTO_TEST_CASE(AmsGradLogisticRegressionTest)
-{
-  // Generate a two-Gaussian dataset.
-  GaussianDistribution g1(arma::vec("1.0 1.0 1.0"), arma::eye<arma::mat>(3, 3));
-  GaussianDistribution g2(arma::vec("9.0 9.0 9.0"), arma::eye<arma::mat>(3, 3));
-
-  arma::mat data(3, 1000);
-  arma::Row<size_t> responses(1000);
-  for (size_t i = 0; i < 500; ++i)
-  {
-    data.col(i) = g1.Random();
-    responses[i] = 0;
-  }
-  for (size_t i = 500; i < 1000; ++i)
-  {
-    data.col(i) = g2.Random();
-    responses[i] = 1;
-  }
-
-  // Shuffle the dataset.
-  arma::uvec indices = arma::shuffle(arma::linspace<arma::uvec>(0,
-      data.n_cols - 1, data.n_cols));
-  arma::mat shuffledData(3, 1000);
-  arma::Row<size_t> shuffledResponses(1000);
-  for (size_t i = 0; i < data.n_cols; ++i)
-  {
-    shuffledData.col(i) = data.col(indices[i]);
-    shuffledResponses[i] = responses[indices[i]];
-  }
-
-  // Create a test set.
-  arma::mat testData(3, 1000);
-  arma::Row<size_t> testResponses(1000);
-  for (size_t i = 0; i < 500; ++i)
-  {
-    testData.col(i) = g1.Random();
-    testResponses[i] = 0;
-  }
-  for (size_t i = 500; i < 1000; ++i)
-  {
-    testData.col(i) = g2.Random();
-    testResponses[i] = 1;
-  }
-
-  AmsGrad amsgrad;
-  LogisticRegression<> lr(shuffledData, shuffledResponses, amsgrad, 0.5);
-
-  // Ensure that the error is close to zero.
-  const double acc = lr.ComputeAccuracy(data, responses);
-  BOOST_REQUIRE_CLOSE(acc, 100.0, 0.3); // 0.3% error tolerance.
-
-  const double testAcc = lr.ComputeAccuracy(testData, testResponses);
-  BOOST_REQUIRE_CLOSE(testAcc, 100.0, 0.6); // 0.6% error tolerance.
-}
-
-BOOST_AUTO_TEST_SUITE_END();

From 403f767751113ff07831a412b156ef16fb0fc2e2 Mon Sep 17 00:00:00 2001
From: Haritha Sreedharan Nair <haritha1313@gmail.com>
Date: Thu, 14 Dec 2017 20:26:08 +0530
Subject: [PATCH 5/7] Update adam.hpp

---
 src/mlpack/core/optimizers/adam/adam.hpp | 13 +++++++------
 1 file changed, 7 insertions(+), 6 deletions(-)

diff --git a/src/mlpack/core/optimizers/adam/adam.hpp b/src/mlpack/core/optimizers/adam/adam.hpp
index a1ec694543..8b0420d9c1 100644
--- a/src/mlpack/core/optimizers/adam/adam.hpp
+++ b/src/mlpack/core/optimizers/adam/adam.hpp
@@ -5,10 +5,11 @@
  * @author Marcus Edel
  * @author Vivek Pal
  *
- * Adam, AdaMax and AMSGrad optimizer. Adam is an an algorithm for first-order gradient-
- * -based optimization of stochastic objective functions, based on adaptive
- * estimates of lower-order moments. AdaMax is simply a variant of Adam based
- * on the infinity norm. AMSGrad is another variant of Adam with guaranteed convergence.
+ * Adam, AdaMax and AMSGrad optimizers. Adam is an an algorithm for 
+ * first-order gradient-based optimization of stochastic objective 
+ * functions, based on adaptive estimates of lower-order moments. 
+ * AdaMax is simply a variant of Adam based on the infinity norm. 
+ * AMSGrad is another variant of Adam with guaranteed convergence.
  *
  * mlpack is free software; you may redistribute it and/or modify it under the
  * terms of the 3-clause BSD license.  You should have received a copy of the
@@ -51,8 +52,8 @@ namespace optimization {
  * @endcode
  *
  *
- * For Adam, AdaMax and AMSGrad to work, a DecomposableFunctionType template parameter
- * is required. This class must implement the following function:
+ * For Adam, AdaMax and AMSGrad to work, a DecomposableFunctionType template
+ * parameter is required. This class must implement the following function:
  *
  *   size_t NumFunctions();
  *   double Evaluate(const arma::mat& coordinates,

From 51e7a2a01ff513006103867b642eaa6517f80f9f Mon Sep 17 00:00:00 2001
From: Haritha <haritha1313@gmail.com>
Date: Wed, 13 Dec 2017 13:08:24 +0530
Subject: [PATCH 6/7] ams_grad optimizer implementation

---
 .../core/optimizers/adam/CMakeLists.txt       |   1 +
 src/mlpack/core/optimizers/adam/adam.hpp      |  21 ++-
 src/mlpack/core/optimizers/adam/adam_impl.hpp |   2 +-
 .../core/optimizers/adam/amsgrad_update.hpp   | 145 ++++++++++++++++++
 src/mlpack/tests/adam_test.cpp                |  76 ++++++++-
 5 files changed, 237 insertions(+), 8 deletions(-)
 create mode 100644 src/mlpack/core/optimizers/adam/amsgrad_update.hpp

diff --git a/src/mlpack/core/optimizers/adam/CMakeLists.txt b/src/mlpack/core/optimizers/adam/CMakeLists.txt
index 1377bbb0e1..644797090f 100644
--- a/src/mlpack/core/optimizers/adam/CMakeLists.txt
+++ b/src/mlpack/core/optimizers/adam/CMakeLists.txt
@@ -3,6 +3,7 @@ set(SOURCES
   adam_impl.hpp
   adam_update.hpp
   adamax_update.hpp
+  amsgrad_update.hpp
 )
 
 set(DIR_SRCS)
diff --git a/src/mlpack/core/optimizers/adam/adam.hpp b/src/mlpack/core/optimizers/adam/adam.hpp
index 3b109660f5..8548d6781a 100644
--- a/src/mlpack/core/optimizers/adam/adam.hpp
+++ b/src/mlpack/core/optimizers/adam/adam.hpp
@@ -5,10 +5,11 @@
  * @author Marcus Edel
  * @author Vivek Pal
  *
- * Adam and AdaMax optimizer. Adam is an an algorithm for first-order gradient-
- * -based optimization of stochastic objective functions, based on adaptive
- * estimates of lower-order moments. AdaMax is simply a variant of Adam based
- * on the infinity norm.
+ * Adam, AdaMax and AMSGrad optimizers. Adam is an an algorithm for 
+ * first-order gradient-based optimization of stochastic objective 
+ * functions, based on adaptive estimates of lower-order moments. 
+ * AdaMax is simply a variant of Adam based on the infinity norm. 
+ * AMSGrad is another variant of Adam with guaranteed convergence.
  *
  * mlpack is free software; you may redistribute it and/or modify it under the
  * terms of the 3-clause BSD license.  You should have received a copy of the
@@ -23,6 +24,7 @@
 #include <mlpack/core/optimizers/sgd/sgd.hpp>
 #include "adam_update.hpp"
 #include "adamax_update.hpp"
+#include "amsgrad_update.hpp"
 
 namespace mlpack {
 namespace optimization {
@@ -43,11 +45,16 @@ namespace optimization {
  *   year    = {2014},
  *   url     = {http://arxiv.org/abs/1412.6980}
  * }
+ * @article{
+ *   title   = {On the convergence of Adam and beyond},
+ *   url     = {https://openreview.net/pdf?id=ryQu7f-RZ}
+ *   year    = {2018}
+ * }
  * @endcode
  *
  *
- * For Adam and AdaMax to work, a DecomposableFunctionType template parameter
- * is required. This class must implement the following function:
+ * For Adam, AdaMax and AMSGrad to work, a DecomposableFunctionType template
+ * parameter is required. This class must implement the following function:
  *
  *   size_t NumFunctions();
  *   double Evaluate(const arma::mat& coordinates,
@@ -166,6 +173,8 @@ using Adam = AdamType<AdamUpdate>;
 
 using AdaMax = AdamType<AdaMaxUpdate>;
 
+using AMSGrad = AdamType<AMSGradUpdate>;
+
 } // namespace optimization
 } // namespace mlpack
 
diff --git a/src/mlpack/core/optimizers/adam/adam_impl.hpp b/src/mlpack/core/optimizers/adam/adam_impl.hpp
index 326d038d15..49497f5af5 100644
--- a/src/mlpack/core/optimizers/adam/adam_impl.hpp
+++ b/src/mlpack/core/optimizers/adam/adam_impl.hpp
@@ -5,7 +5,7 @@
  * @author Marcus Edel
  * @author Vivek Pal
  *
- * Implementation of the Adam and AdaMax optimizer.
+ * Implementation of the Adam, AdaMax and AMSGrad optimizer.
  *
  * mlpack is free software; you may redistribute it and/or modify it under the
  * terms of the 3-clause BSD license.  You should have received a copy of the
diff --git a/src/mlpack/core/optimizers/adam/amsgrad_update.hpp b/src/mlpack/core/optimizers/adam/amsgrad_update.hpp
new file mode 100644
index 0000000000..ae5ac681a6
--- /dev/null
+++ b/src/mlpack/core/optimizers/adam/amsgrad_update.hpp
@@ -0,0 +1,145 @@
+/**
+ * @file amsgrad_update.hpp
+ * @author Haritha Nair
+ *
+ * Implementation of AMSGrad optimizer. AMSGrad is an exponential moving average 
+ * optimizer that dynamically adapts over time with guaranteed convergence.
+ *
+ * mlpack is free software; you may redistribute it and/or modify it under the
+ * terms of the 3-clause BSD license.  You should have received a copy of the
+ * 3-clause BSD license along with mlpack.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef MLPACK_CORE_OPTIMIZERS_AMS_GRAD_AMS_GRAD_UPDATE_HPP
+#define MLPACK_CORE_OPTIMIZERS_AMS_GRAD_AMS_GRAD_UPDATE_HPP
+
+#include <mlpack/prereqs.hpp>
+
+namespace mlpack {
+namespace optimization {
+
+/**
+ * AMSGrad is an exponential moving average variant which along with having 
+ * benefits of optimizers like Adam and RMSProp, also guarantees convergence.
+ * Unlike Adam, it uses maximum of past squared gradients rather than their 
+ * exponential average for updation.
+ *
+ * For more information, see the following.
+ *
+ * @code
+ * @article{
+ *   title   = {On the convergence of Adam and beyond},
+ *   url     = {https://openreview.net/pdf?id=ryQu7f-RZ}
+ *   year    = {2018}
+ * }
+ * @endcode
+ */
+class AMSGradUpdate
+{
+ public:
+  /**
+   * Construct the AMSGrad update policy with the given parameters.
+   *
+   * @param epsilon The epsilon value used to initialise the squared gradient
+   *        parameter.
+   * @param beta1 The smoothing parameter.
+   * @param beta2 The second moment coefficient.
+   */
+  AMSGradUpdate(const double epsilon = 1e-8,
+                const double beta1 = 0.9,
+                const double beta2 = 0.999) :
+    epsilon(epsilon),
+    beta1(beta1),
+    beta2(beta2),
+    iteration(0)
+  {
+    // Nothing to do.
+  }
+
+  /**
+   * The Initialize method is called by SGD Optimizer method before the start of
+   * the iteration update process.
+   *
+   * @param rows Number of rows in the gradient matrix.
+   * @param cols Number of columns in the gradient matrix.
+   */
+  void Initialize(const size_t rows, const size_t cols)
+  {
+    m = arma::zeros<arma::mat>(rows, cols);
+    v = arma::zeros<arma::mat>(rows, cols);
+    vImproved = arma::zeros<arma::mat>(rows, cols);
+  }
+
+  /**
+   * Update step for AMSGrad.
+   *
+   * @param iterate Parameters that minimize the function.
+   * @param stepSize Step size to be used for the given iteration.
+   * @param gradient The gradient matrix.
+   */
+  void Update(arma::mat& iterate,
+              const double stepSize,
+              const arma::mat& gradient)
+  {
+    // Increment the iteration counter variable.
+    ++iteration;
+
+    // And update the iterate.
+    m *= beta1;
+    m += (1 - beta1) * gradient;
+
+    v *= beta2;
+    v += (1 - beta2) * (gradient % gradient);
+
+    const double biasCorrection1 = 1.0 - std::pow(beta1, iteration);
+    const double biasCorrection2 = 1.0 - std::pow(beta2, iteration);
+
+    // Element wise maximum of past and present squared gradients.
+    vImproved = arma::max(vImproved, v);
+
+    iterate -= (stepSize * std::sqrt(biasCorrection2) / biasCorrection1) *
+                m / (arma::sqrt(vImproved) + epsilon);
+  }
+
+  //! Get the value used to initialise the squared gradient parameter.
+  double Epsilon() const { return epsilon; }
+  //! Modify the value used to initialise the squared gradient parameter.
+  double& Epsilon() { return epsilon; }
+
+  //! Get the smoothing parameter.
+  double Beta1() const { return beta1; }
+  //! Modify the smoothing parameter.
+  double& Beta1() { return beta1; }
+
+  //! Get the second moment coefficient.
+  double Beta2() const { return beta2; }
+  //! Modify the second moment coefficient.
+  double& Beta2() { return beta2; }
+
+ private:
+  // The epsilon value used to initialise the squared gradient parameter.
+  double epsilon;
+
+  // The smoothing parameter.
+  double beta1;
+
+  // The second moment coefficient.
+  double beta2;
+
+  // The exponential moving average of gradient values.
+  arma::mat m;
+
+  // The exponential moving average of squared gradient values.
+  arma::mat v;
+
+  // The optimal sqaured gradient value.
+  arma::mat vImproved;
+
+  // The number of iterations.
+  double iteration;
+};
+
+} // namespace optimization
+} // namespace mlpack
+
+#endif
diff --git a/src/mlpack/tests/adam_test.cpp b/src/mlpack/tests/adam_test.cpp
index c1431a860c..dee211cb84 100644
--- a/src/mlpack/tests/adam_test.cpp
+++ b/src/mlpack/tests/adam_test.cpp
@@ -3,7 +3,7 @@
  * @author Vasanth Kalingeri
  * @author Vivek Pal
  *
- * Tests the Adam and AdaMax optimizer.
+ * Tests the Adam, AdaMax and AMSGrad optimizer.
  *
  * mlpack is free software; you may redistribute it and/or modify it under the
  * terms of the 3-clause BSD license.  You should have received a copy of the
@@ -62,6 +62,22 @@ BOOST_AUTO_TEST_CASE(SimpleAdaMaxTestFunction)
   BOOST_REQUIRE_SMALL(coordinates[2], 0.1);
 }
 
+/**
+ * Tests the AMSGrad optimizer using a simple test function.
+ */
+BOOST_AUTO_TEST_CASE(SimpleAMSGradTestFunction)
+{
+  SGDTestFunction f;
+  AMSGrad optimizer(1e-3, 1, 0.9, 0.999, 1e-8, 500000, 1e-11, true);
+
+  arma::mat coordinates = f.GetInitialPoint();
+  optimizer.Optimize(f, coordinates);
+
+  BOOST_REQUIRE_SMALL(coordinates[0], 0.1);
+  BOOST_REQUIRE_SMALL(coordinates[1], 0.1);
+  BOOST_REQUIRE_SMALL(coordinates[2], 0.1);
+}
+
 /**
  * Run Adam on logistic regression and make sure the results are acceptable.
  */
@@ -178,4 +194,62 @@ BOOST_AUTO_TEST_CASE(AdaMaxLogisticRegressionTest)
   BOOST_REQUIRE_CLOSE(testAcc, 100.0, 0.6); // 0.6% error tolerance.
 }
 
+/**
+ * Run AMSGrad on logistic regression and make sure the results are acceptable.
+ */
+BOOST_AUTO_TEST_CASE(AMSGradLogisticRegressionTest)
+{
+  // Generate a two-Gaussian dataset.
+  GaussianDistribution g1(arma::vec("1.0 1.0 1.0"), arma::eye<arma::mat>(3, 3));
+  GaussianDistribution g2(arma::vec("9.0 9.0 9.0"), arma::eye<arma::mat>(3, 3));
+
+  arma::mat data(3, 1000);
+  arma::Row<size_t> responses(1000);
+  for (size_t i = 0; i < 500; ++i)
+  {
+    data.col(i) = g1.Random();
+    responses[i] = 0;
+  }
+  for (size_t i = 500; i < 1000; ++i)
+  {
+    data.col(i) = g2.Random();
+    responses[i] = 1;
+  }
+
+  // Shuffle the dataset.
+  arma::uvec indices = arma::shuffle(arma::linspace<arma::uvec>(0,
+      data.n_cols - 1, data.n_cols));
+  arma::mat shuffledData(3, 1000);
+  arma::Row<size_t> shuffledResponses(1000);
+  for (size_t i = 0; i < data.n_cols; ++i)
+  {
+    shuffledData.col(i) = data.col(indices[i]);
+    shuffledResponses[i] = responses[indices[i]];
+  }
+
+  // Create a test set.
+  arma::mat testData(3, 1000);
+  arma::Row<size_t> testResponses(1000);
+  for (size_t i = 0; i < 500; ++i)
+  {
+    testData.col(i) = g1.Random();
+    testResponses[i] = 0;
+  }
+  for (size_t i = 500; i < 1000; ++i)
+  {
+    testData.col(i) = g2.Random();
+    testResponses[i] = 1;
+  }
+
+  AMSGrad amsgrad(1e-3, 1, 0.9, 0.999, 1e-8, 500000, 1e-11, true);
+  LogisticRegression<> lr(shuffledData, shuffledResponses, amsgrad, 0.5);
+
+  // Ensure that the error is close to zero.
+  const double acc = lr.ComputeAccuracy(data, responses);
+  BOOST_REQUIRE_CLOSE(acc, 100.0, 0.3); // 0.3% error tolerance.
+
+  const double testAcc = lr.ComputeAccuracy(testData, testResponses);
+  BOOST_REQUIRE_CLOSE(testAcc, 100.0, 0.6); // 0.6% error tolerance.
+}
+
 BOOST_AUTO_TEST_SUITE_END();

From 0555e733378ce0233dc1b1eab74e4109d6f6eff4 Mon Sep 17 00:00:00 2001
From: Haritha <haritha1313@gmail.com>
Date: Wed, 20 Dec 2017 11:58:26 +0530
Subject: [PATCH 7/7] ams_grad style edits

---
 src/mlpack/core/optimizers/adam/amsgrad_update.hpp | 9 ++-------
 1 file changed, 2 insertions(+), 7 deletions(-)

diff --git a/src/mlpack/core/optimizers/adam/amsgrad_update.hpp b/src/mlpack/core/optimizers/adam/amsgrad_update.hpp
index 21130e214a..f2a126f186 100644
--- a/src/mlpack/core/optimizers/adam/amsgrad_update.hpp
+++ b/src/mlpack/core/optimizers/adam/amsgrad_update.hpp
@@ -19,14 +19,9 @@ namespace mlpack {
 namespace optimization {
 
 /**
- * AMSGrad is an exponential moving average variant which along with having 
-<<<<<<< HEAD
+ * AMSGrad is an exponential moving average variant which along with having
  * benefits of optimizers like Adam and RMSProp, also guarantees convergence.
- * Unlike Adam, it uses maximum of past squared gradients rather than their 
-=======
- * benefits of optimizers like Adam and RmsProp, also guarantees convergence.
- * Unlike adam, it uses maximum of past squared gradients rather than their 
->>>>>>> 403f767751113ff07831a412b156ef16fb0fc2e2
+ * Unlike Adam, it uses maximum of past squared gradients rather than their
  * exponential average for updation.
  *
  * For more information, see the following.