From ab7a5adc528beeda16419cdaad881544733bf0a9 Mon Sep 17 00:00:00 2001
From: Vivek Pal <vivekpal.dtu@gmail.com>
Date: Thu, 30 Mar 2017 17:07:37 +0530
Subject: [PATCH 01/12] Add implementation of the Adam update policy

---
 .../core/optimizers/adam/CMakeLists.txt       |   1 +
 src/mlpack/core/optimizers/adam/adam.hpp      |  73 +++----
 src/mlpack/core/optimizers/adam/adam_impl.hpp | 154 +--------------
 .../core/optimizers/adam/adam_update.hpp      | 180 ++++++++++++++++++
 4 files changed, 218 insertions(+), 190 deletions(-)
 create mode 100644 src/mlpack/core/optimizers/adam/adam_update.hpp

diff --git a/src/mlpack/core/optimizers/adam/CMakeLists.txt b/src/mlpack/core/optimizers/adam/CMakeLists.txt
index 3cbcfd84e7..eabdbbaafa 100644
--- a/src/mlpack/core/optimizers/adam/CMakeLists.txt
+++ b/src/mlpack/core/optimizers/adam/CMakeLists.txt
@@ -1,6 +1,7 @@
 set(SOURCES
   adam.hpp
   adam_impl.hpp
+  adam_update.hpp
 )
 
 set(DIR_SRCS)
diff --git a/src/mlpack/core/optimizers/adam/adam.hpp b/src/mlpack/core/optimizers/adam/adam.hpp
index 62af4c6ac8..9c729b4d88 100644
--- a/src/mlpack/core/optimizers/adam/adam.hpp
+++ b/src/mlpack/core/optimizers/adam/adam.hpp
@@ -20,6 +20,9 @@
 
 #include <mlpack/prereqs.hpp>
 
+#include <mlpack/core/optimizers/sgd/sgd.hpp>
+#include "adam_update.hpp"
+
 namespace mlpack {
 namespace optimization {
 
@@ -105,81 +108,59 @@ class Adam
    * @param iterate Starting point (will be modified).
    * @return Objective value of the final point.
    */
-  double Optimize(arma::mat& iterate);
+  double Optimize(arma::mat& iterate){ return optimizer.Optimize(iterate); }
 
   //! Get the instantiated function to be optimized.
-  const DecomposableFunctionType& Function() const { return function; }
+  const DecomposableFunctionType& Function() const
+  {
+    return optimizer.Function();
+  }
   //! Modify the instantiated function.
-  DecomposableFunctionType& Function() { return function; }
+  DecomposableFunctionType& Function() { return optimizer.Function(); }
 
   //! Get the step size.
-  double StepSize() const { return stepSize; }
+  double StepSize() const { return optimizer.StepSize(); }
   //! Modify the step size.
-  double& StepSize() { return stepSize; }
+  double& StepSize() { return optimizer.StepSize(); }
 
   //! Get the smoothing parameter.
-  double Beta1() const { return beta1; }
+  double Beta1() const { return optimizer.UpdatePolicy().Beta1(); }
   //! Modify the smoothing parameter.
-  double& Beta1() { return beta1; }
+  double& Beta1() { return optimizer.UpdatePolicy().Beta1(); }
 
   //! Get the second moment coefficient.
-  double Beta2() const { return beta2; }
+  double Beta2() const { return optimizer.UpdatePolicy().Beta2(); }
   //! Modify the second moment coefficient.
-  double& Beta2() { return beta2; }
+  double& Beta2() { return optimizer.UpdatePolicy().Beta2(); }
 
   //! Get the value used to initialise the mean squared gradient parameter.
-  double Epsilon() const { return eps; }
+  double Epsilon() const { return optimizer.UpdatePolicy().Epsilon(); }
   //! Modify the value used to initialise the mean squared gradient parameter.
-  double& Epsilon() { return eps; }
+  double& Epsilon() { return optimizer.UpdatePolicy().Epsilon(); }
 
   //! Get the maximum number of iterations (0 indicates no limit).
-  size_t MaxIterations() const { return maxIterations; }
+  size_t MaxIterations() const { return optimizer.MaxIterations(); }
   //! Modify the maximum number of iterations (0 indicates no limit).
-  size_t& MaxIterations() { return maxIterations; }
+  size_t& MaxIterations() { return optimizer.MaxIterations(); }
 
   //! Get the tolerance for termination.
-  double Tolerance() const { return tolerance; }
+  double Tolerance() const { return optimizer.Tolerance(); }
   //! Modify the tolerance for termination.
-  double& Tolerance() { return tolerance; }
+  double& Tolerance() { return optimizer.Tolerance(); }
 
   //! Get whether or not the individual functions are shuffled.
-  bool Shuffle() const { return shuffle; }
+  bool Shuffle() const { return optimizer.Shuffle(); }
   //! Modify whether or not the individual functions are shuffled.
-  bool& Shuffle() { return shuffle; }
+  bool& Shuffle() { return optimizer.Shuffle(); }
 
   //! Get whether or not the AdaMax optimizer is specified.
-  bool AdaMax() const { return adaMax; }
+  bool AdaMax() const { return optimizer.UpdatePolicy().AdaMax(); }
   //! Modify wehther or not the AdaMax optimizer is to be used.
-  bool& AdaMax() { return adaMax; }
+  bool& AdaMax() { return optimizer.UpdatePolicy().AdaMax(); }
 
  private:
-  //! The instantiated function.
-  DecomposableFunctionType& function;
-
-  //! The step size for each example.
-  double stepSize;
-
-  //! Exponential decay rate for the first moment estimates.
-  double beta1;
-
-  //! Exponential decay rate for the weighted infinity norm estimates.
-  double beta2;
-
-  //! The value used to initialise the mean squared gradient parameter.
-  double eps;
-
-  //! The maximum number of allowed iterations.
-  size_t maxIterations;
-
-  //! The tolerance for termination.
-  double tolerance;
-
-  //! Controls whether or not the individual functions are shuffled when
-  //! iterating.
-  bool shuffle;
-
-  //! Specifies whether or not the AdaMax optimizer is to be used.
-  bool adaMax;
+  //! The Stochastic Gradient Descent object with Adam policy.
+  SGD<DecomposableFunctionType, AdamUpdate> optimizer;
 };
 
 } // namespace optimization
diff --git a/src/mlpack/core/optimizers/adam/adam_impl.hpp b/src/mlpack/core/optimizers/adam/adam_impl.hpp
index 8a6be66c07..b9faf1226d 100644
--- a/src/mlpack/core/optimizers/adam/adam_impl.hpp
+++ b/src/mlpack/core/optimizers/adam/adam_impl.hpp
@@ -26,156 +26,22 @@ Adam<DecomposableFunctionType>::Adam(DecomposableFunctionType& function,
                                      const double stepSize,
                                      const double beta1,
                                      const double beta2,
-                                     const double eps,
+                                     const double epsilon,
                                      const size_t maxIterations,
                                      const double tolerance,
                                      const bool shuffle,
                                      const bool adaMax) :
-    function(function),
-    stepSize(stepSize),
-    beta1(beta1),
-    beta2(beta2),
-    eps(eps),
-    maxIterations(maxIterations),
-    tolerance(tolerance),
-    shuffle(shuffle),
-    adaMax(adaMax)
+    optimizer(function,
+              stepSize,
+              maxIterations,
+              tolerance,
+              shuffle,
+              AdamUpdate(epsilon,
+                         beta1,
+                         beta2,
+                         adaMax))
 { /* Nothing to do. */ }
 
-//! Optimize the function (minimize).
-template<typename DecomposableFunctionType>
-double Adam<DecomposableFunctionType>::Optimize(arma::mat& iterate)
-{
-  // Find the number of functions to use.
-  const size_t numFunctions = function.NumFunctions();
-
-  // This is used only if shuffle is true.
-  arma::Col<size_t> visitationOrder;
-  if (shuffle)
-    visitationOrder = arma::shuffle(arma::linspace<arma::Col<size_t>>(0,
-        (numFunctions - 1), numFunctions));
-
-  // To keep track of where we are and how things are going.
-  size_t currentFunction = 0;
-  double overallObjective = 0;
-  double lastObjective = DBL_MAX;
-
-  // Calculate the first objective function.
-  for (size_t i = 0; i < numFunctions; ++i)
-    overallObjective += function.Evaluate(iterate, i);
-
-  // Now iterate!
-  arma::mat gradient(iterate.n_rows, iterate.n_cols);
-
-  // Exponential moving average of gradient values.
-  arma::mat m = arma::zeros<arma::mat>(iterate.n_rows, iterate.n_cols);
-
-  /**
-   * Initialize  either the exponentially weighted infinity norm for AdaMax
-   * optimizer (u) or exponential moving average of squared gradient values
-   * for Adam optimizer (v).
-   */
-  arma::mat u, v;
-  if (adaMax)
-  {
-    u = arma::zeros<arma::mat>(iterate.n_rows, iterate.n_cols);
-  }
-  else
-  {
-    v = arma::zeros<arma::mat>(iterate.n_rows, iterate.n_cols);
-  }
-
-  for (size_t i = 1; i != maxIterations; ++i, ++currentFunction)
-  {
-    // Is this iteration the start of a sequence?
-    if ((currentFunction % numFunctions) == 0)
-    {
-      // Output current objective function.
-      Log::Info << "Adam: iteration " << i << ", objective " << overallObjective
-          << "." << std::endl;
-
-      if (std::isnan(overallObjective) || std::isinf(overallObjective))
-      {
-        Log::Warn << "Adam: converged to " << overallObjective
-            << "; terminating with failure. Try a smaller step size?"
-            << std::endl;
-        return overallObjective;
-      }
-
-      if (std::abs(lastObjective - overallObjective) < tolerance)
-      {
-        Log::Info << "Adam: minimized within tolerance " << tolerance << "; "
-            << "terminating optimization." << std::endl;
-        return overallObjective;
-      }
-
-      // Reset the counter variables.
-      lastObjective = overallObjective;
-      overallObjective = 0;
-      currentFunction = 0;
-
-      if (shuffle) // Determine order of visitation.
-        visitationOrder = arma::shuffle(visitationOrder);
-    }
-
-    // Evaluate the gradient for this iteration.
-    if (shuffle)
-      function.Gradient(iterate, visitationOrder[currentFunction], gradient);
-    else
-      function.Gradient(iterate, currentFunction, gradient);
-
-    // And update the iterate.
-    m *= beta1;
-    m += (1 - beta1) * gradient;
-
-    if (adaMax)
-    {
-      // Update the exponentially weighted infinity norm.
-      u *= beta2;
-      u = arma::max(u, arma::abs(gradient));
-    }
-    else
-    {
-      v *= beta2;
-      v += (1 - beta2) * (gradient % gradient);
-    }
-
-    const double biasCorrection1 = 1.0 - std::pow(beta1, (double) i);
-    const double biasCorrection2 = 1.0 - std::pow(beta2, (double) i);
-
-    if (adaMax)
-    {
-      if (biasCorrection1 != 0.0)
-        iterate -= (stepSize / biasCorrection1 * m / (u + eps));
-    }
-    else
-    {
-      /**
-       * It should be noted that the term, m / (arma::sqrt(v) + eps), in the
-       * following expression is an approximation of the following actual term;
-       * m / (arma::sqrt(v) + (arma::sqrt(biasCorrection2) * eps).
-       */
-      iterate -= (stepSize * std::sqrt(biasCorrection2) / biasCorrection1) *
-                  m / (arma::sqrt(v) + eps);
-    }
-
-    // Now add that to the overall objective function.
-    if (shuffle)
-      overallObjective += function.Evaluate(iterate,
-          visitationOrder[currentFunction]);
-    else
-      overallObjective += function.Evaluate(iterate, currentFunction);
-  }
-
-  Log::Info << "Adam: maximum iterations (" << maxIterations << ") reached; "
-      << "terminating optimization." << std::endl;
-  // Calculate final objective.
-  overallObjective = 0;
-  for (size_t i = 0; i < numFunctions; ++i)
-    overallObjective += function.Evaluate(iterate, i);
-  return overallObjective;
-}
-
 } // namespace optimization
 } // namespace mlpack
 
diff --git a/src/mlpack/core/optimizers/adam/adam_update.hpp b/src/mlpack/core/optimizers/adam/adam_update.hpp
new file mode 100644
index 0000000000..fba575f4ce
--- /dev/null
+++ b/src/mlpack/core/optimizers/adam/adam_update.hpp
@@ -0,0 +1,180 @@
+/**
+ * @file adam.hpp
+ * @author Ryan Curtin
+ * @author Vasanth Kalingeri
+ * @author Marcus Edel
+ * @author Vivek Pal
+ *
+ * Adam and AdaMax optimizer. Adam is an an algorithm for first-order gradient-
+ * -based optimization of stochastic objective functions, based on adaptive
+ * estimates of lower-order moments. AdaMax is simply a variant of Adam based
+ * on the infinity norm.
+ *
+ * mlpack is free software; you may redistribute it and/or modify it under the
+ * terms of the 3-clause BSD license.  You should have received a copy of the
+ * 3-clause BSD license along with mlpack.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef MLPACK_CORE_OPTIMIZERS_ADAM_ADAM_UPDATE_HPP
+#define MLPACK_CORE_OPTIMIZERS_ADAM_ADAM_UPDATE_HPP
+
+#include <mlpack/prereqs.hpp>
+
+namespace mlpack {
+namespace optimization {
+
+/**
+ * Adam is an optimizer that computes individual adaptive learning rates for
+ * different parameters from estimates of first and second moments of the
+ * gradients. AdaMax is a variant of Adam based on the infinity norm as given
+ * in the section 7 of the following paper.
+ *
+ * For more information, see the following.
+ *
+ * @code
+ * @article{Kingma2014,
+ *   author    = {Diederik P. Kingma and Jimmy Ba},
+ *   title     = {Adam: {A} Method for Stochastic Optimization},
+ *   journal   = {CoRR},
+ *   year      = {2014}
+ * }
+ * @endcode
+ *
+ */
+class AdamUpdate
+{
+ public:
+  /**
+   * Construct the Adam update policy with the given epsilon parameter.
+   *
+   * @param epsilon The epsilon value used to initialise the squared gradient
+   *        parameter.
+   */
+  AdamUpdate(const double epsilon = 1e-8,
+             const double beta1 = 0.9,
+             const double beta2 = 0.999,
+             const bool adaMax = false) :
+    epsilon(epsilon),
+    beta1(beta1),
+    beta2(beta2),
+    adaMax(adaMax)
+  {
+    // Nothing to do.
+  }
+
+  /**
+   * The Initialize method is called by SGD Optimizer method before the start of
+   * the iteration update process.
+   *
+   * @param rows number of rows in the gradient matrix.
+   * @param cols number of columns in the gradient matrix.
+   */
+  void Initialize(const size_t rows,
+                  const size_t cols)
+  {
+    m = arma::zeros<arma::mat>(rows, cols);
+    if (adaMax)
+    {
+      u = arma::zeros<arma::mat>(rows, cols);
+    }
+    else
+    {
+      v = arma::zeros<arma::mat>(rows, cols);
+    }
+  }
+
+  /**
+   * Update step for Adam.
+   *
+   * @param iterate Parameters that minimize the function.
+   * @param stepSize Step size to be used for the given iteration.
+   * @param gradient The gradient matrix.
+   */
+  void Update(arma::mat& iterate,
+              const double stepSize,
+              const arma::mat& gradient,
+              const size_t i)
+  {
+    // And update the iterate.
+    m *= beta1;
+    m += (1 - beta1) * gradient;
+
+    if (adaMax)
+    {
+      // Update the exponentially weighted infinity norm.
+      u *= beta2;
+      u = arma::max(u, arma::abs(gradient));
+    }
+    else
+    {
+      v *= beta2;
+      v += (1 - beta2) * (gradient % gradient);
+    }
+
+    const double biasCorrection1 = 1.0 - std::pow(beta1, (double) i);
+    const double biasCorrection2 = 1.0 - std::pow(beta2, (double) i);
+
+    if (adaMax)
+    {
+      if (biasCorrection1 != 0.0)
+        iterate -= (stepSize / biasCorrection1 * m / (u + epsilon));
+    }
+    else
+    {
+      /**
+       * It should be noted that the term, m / (arma::sqrt(v) + eps), in the
+       * following expression is an approximation of the following actual term;
+       * m / (arma::sqrt(v) + (arma::sqrt(biasCorrection2) * eps).
+       */
+      iterate -= (stepSize * std::sqrt(biasCorrection2) / biasCorrection1) *
+                  m / (arma::sqrt(v) + epsilon);
+    }
+  }
+
+  //! Get the value used to initialise the squared gradient parameter.
+  double Epsilon() const { return epsilon; }
+  //! Modify the value used to initialise the squared gradient parameter.
+  double& Epsilon() { return epsilon; }
+
+  //! Get the smoothing parameter.
+  double Beta1() const { return beta1; }
+  //! Modify the smoothing parameter.
+  double& Beta1() { return beta1; }
+
+  //! Get the second moment coefficient.
+  double Beta2() const { return beta2; }
+  //! Modify the second moment coefficient.
+  double& Beta2() { return beta2; }
+
+  //! Get whether or not the AdaMax optimizer is specified.
+  bool AdaMax() const { return adaMax; }
+  //! Modify wehther or not the AdaMax optimizer is to be used.
+  bool& AdaMax() { return adaMax; }
+
+ private:
+  // The epsilon value used to initialise the squared gradient parameter.
+  double epsilon;
+
+  // The smoothing parameter.
+  double beta1;
+
+  // The second moment coefficient.
+  double beta2;
+
+  //! Specifies whether or not the AdaMax optimizer is to be used.
+  bool adaMax;
+
+  // The exponential moving average of gradient values.
+  arma::mat m;
+
+  // The exponentially weighted infinity norm.
+  arma::mat u;
+
+  // The exponential moving average of squared gradient values.
+  arma::mat v;
+};
+
+} // namespace optimization
+} // namespace mlpack
+
+#endif

From bf7e38d7657ae51ce9dfbe9a20619932501a943b Mon Sep 17 00:00:00 2001
From: Vivek Pal <vivekpal.dtu@gmail.com>
Date: Thu, 30 Mar 2017 17:13:21 +0530
Subject: [PATCH 02/12] Add iteration parameter "i" to the Update function

Used in the Update step of the Adam optimizer.
---
 src/mlpack/core/optimizers/ada_delta/ada_delta_update.hpp      | 3 ++-
 src/mlpack/core/optimizers/ada_grad/ada_grad_update.hpp        | 3 ++-
 src/mlpack/core/optimizers/sgd/sgd_impl.hpp                    | 2 +-
 .../core/optimizers/sgd/update_policies/momentum_update.hpp    | 3 ++-
 .../core/optimizers/sgd/update_policies/vanilla_update.hpp     | 3 ++-
 5 files changed, 9 insertions(+), 5 deletions(-)

diff --git a/src/mlpack/core/optimizers/ada_delta/ada_delta_update.hpp b/src/mlpack/core/optimizers/ada_delta/ada_delta_update.hpp
index 391e2a88bd..537d2d067c 100644
--- a/src/mlpack/core/optimizers/ada_delta/ada_delta_update.hpp
+++ b/src/mlpack/core/optimizers/ada_delta/ada_delta_update.hpp
@@ -83,7 +83,8 @@ class AdaDeltaUpdate
    */
   void Update(arma::mat& iterate,
               const double stepSize,
-              const arma::mat& gradient)
+              const arma::mat& gradient,
+              const size_t i)
   {
     // Accumulate gradient.
     meanSquaredGradient *= rho;
diff --git a/src/mlpack/core/optimizers/ada_grad/ada_grad_update.hpp b/src/mlpack/core/optimizers/ada_grad/ada_grad_update.hpp
index 2fb52573b1..12634a4eff 100644
--- a/src/mlpack/core/optimizers/ada_grad/ada_grad_update.hpp
+++ b/src/mlpack/core/optimizers/ada_grad/ada_grad_update.hpp
@@ -79,7 +79,8 @@ class AdaGradUpdate
    */
   void Update(arma::mat& iterate,
               const double stepSize,
-              const arma::mat& gradient)
+              const arma::mat& gradient,
+              const size_t i)
   {
     squaredGradient += (gradient % gradient);
     iterate -= (stepSize * gradient) / (arma::sqrt(squaredGradient) + epsilon);
diff --git a/src/mlpack/core/optimizers/sgd/sgd_impl.hpp b/src/mlpack/core/optimizers/sgd/sgd_impl.hpp
index 5ac19ba960..ca756cb20e 100644
--- a/src/mlpack/core/optimizers/sgd/sgd_impl.hpp
+++ b/src/mlpack/core/optimizers/sgd/sgd_impl.hpp
@@ -108,7 +108,7 @@ double SGD<DecomposableFunctionType, UpdatePolicyType>::Optimize(
       function.Gradient(iterate, currentFunction, gradient);
 
     // Use the update policy to take a step.
-    updatePolicy.Update(iterate, stepSize, gradient);
+    updatePolicy.Update(iterate, stepSize, gradient, i);
 
     // Now add that to the overall objective function.
     if (shuffle)
diff --git a/src/mlpack/core/optimizers/sgd/update_policies/momentum_update.hpp b/src/mlpack/core/optimizers/sgd/update_policies/momentum_update.hpp
index 2947c355f9..889d725504 100644
--- a/src/mlpack/core/optimizers/sgd/update_policies/momentum_update.hpp
+++ b/src/mlpack/core/optimizers/sgd/update_policies/momentum_update.hpp
@@ -99,7 +99,8 @@ class MomentumUpdate
    */
   void Update(arma::mat& iterate,
               const double stepSize,
-              const arma::mat& gradient)
+              const arma::mat& gradient,
+              const size_t i)
   {
     velocity = momentum * velocity - stepSize * gradient;
     iterate += velocity;
diff --git a/src/mlpack/core/optimizers/sgd/update_policies/vanilla_update.hpp b/src/mlpack/core/optimizers/sgd/update_policies/vanilla_update.hpp
index 1bd85bf1c3..2c75a4b4fd 100644
--- a/src/mlpack/core/optimizers/sgd/update_policies/vanilla_update.hpp
+++ b/src/mlpack/core/optimizers/sgd/update_policies/vanilla_update.hpp
@@ -52,7 +52,8 @@ class VanillaUpdate
   */
   void Update(arma::mat& iterate,
               const double stepSize,
-              const arma::mat& gradient)
+              const arma::mat& gradient,
+              const size_t i)
   {
     // Perform the vanilla SGD update.
     iterate -= stepSize * gradient;

From c0eb047e8cebe43795fc30ce7e8bfb10638ec8b3 Mon Sep 17 00:00:00 2001
From: Vivek Pal <vivekpal.dtu@gmail.com>
Date: Thu, 30 Mar 2017 17:15:44 +0530
Subject: [PATCH 03/12] Suppress compiler warnings on unused parameter i

---
 src/mlpack/core/optimizers/ada_delta/ada_delta_update.hpp       | 2 +-
 src/mlpack/core/optimizers/ada_grad/ada_grad_update.hpp         | 2 +-
 .../core/optimizers/sgd/update_policies/momentum_update.hpp     | 2 +-
 .../core/optimizers/sgd/update_policies/vanilla_update.hpp      | 2 +-
 4 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/mlpack/core/optimizers/ada_delta/ada_delta_update.hpp b/src/mlpack/core/optimizers/ada_delta/ada_delta_update.hpp
index 537d2d067c..cb582239a8 100644
--- a/src/mlpack/core/optimizers/ada_delta/ada_delta_update.hpp
+++ b/src/mlpack/core/optimizers/ada_delta/ada_delta_update.hpp
@@ -84,7 +84,7 @@ class AdaDeltaUpdate
   void Update(arma::mat& iterate,
               const double stepSize,
               const arma::mat& gradient,
-              const size_t i)
+              const size_t /*i*/)
   {
     // Accumulate gradient.
     meanSquaredGradient *= rho;
diff --git a/src/mlpack/core/optimizers/ada_grad/ada_grad_update.hpp b/src/mlpack/core/optimizers/ada_grad/ada_grad_update.hpp
index 12634a4eff..bc0f1296ae 100644
--- a/src/mlpack/core/optimizers/ada_grad/ada_grad_update.hpp
+++ b/src/mlpack/core/optimizers/ada_grad/ada_grad_update.hpp
@@ -80,7 +80,7 @@ class AdaGradUpdate
   void Update(arma::mat& iterate,
               const double stepSize,
               const arma::mat& gradient,
-              const size_t i)
+              const size_t /*i*/)
   {
     squaredGradient += (gradient % gradient);
     iterate -= (stepSize * gradient) / (arma::sqrt(squaredGradient) + epsilon);
diff --git a/src/mlpack/core/optimizers/sgd/update_policies/momentum_update.hpp b/src/mlpack/core/optimizers/sgd/update_policies/momentum_update.hpp
index 889d725504..7e4a1deeed 100644
--- a/src/mlpack/core/optimizers/sgd/update_policies/momentum_update.hpp
+++ b/src/mlpack/core/optimizers/sgd/update_policies/momentum_update.hpp
@@ -100,7 +100,7 @@ class MomentumUpdate
   void Update(arma::mat& iterate,
               const double stepSize,
               const arma::mat& gradient,
-              const size_t i)
+              const size_t /*i*/)
   {
     velocity = momentum * velocity - stepSize * gradient;
     iterate += velocity;
diff --git a/src/mlpack/core/optimizers/sgd/update_policies/vanilla_update.hpp b/src/mlpack/core/optimizers/sgd/update_policies/vanilla_update.hpp
index 2c75a4b4fd..159d7f3398 100644
--- a/src/mlpack/core/optimizers/sgd/update_policies/vanilla_update.hpp
+++ b/src/mlpack/core/optimizers/sgd/update_policies/vanilla_update.hpp
@@ -53,7 +53,7 @@ class VanillaUpdate
   void Update(arma::mat& iterate,
               const double stepSize,
               const arma::mat& gradient,
-              const size_t i)
+              const size_t /*i*/)
   {
     // Perform the vanilla SGD update.
     iterate -= stepSize * gradient;

From cda0e912402461730a5091c897319296f91154ca Mon Sep 17 00:00:00 2001
From: Vivek Pal <vivekpal.dtu@gmail.com>
Date: Sat, 1 Apr 2017 06:51:45 +0530
Subject: [PATCH 04/12] Address review comments

* Use a local variable to keep count of iterations.
* Change adaMax parameter to a template parameter.
---
 .../optimizers/ada_delta/ada_delta_update.hpp |  3 +-
 .../optimizers/ada_grad/ada_grad_update.hpp   |  3 +-
 src/mlpack/core/optimizers/adam/adam.hpp      | 14 ++-------
 src/mlpack/core/optimizers/adam/adam_impl.hpp | 26 +++++++---------
 .../core/optimizers/adam/adam_update.hpp      | 31 +++++++++----------
 src/mlpack/core/optimizers/sgd/sgd_impl.hpp   |  2 +-
 .../sgd/update_policies/momentum_update.hpp   |  3 +-
 .../sgd/update_policies/vanilla_update.hpp    |  3 +-
 .../logistic_regression.hpp                   |  2 +-
 .../logistic_regression_impl.hpp              |  2 +-
 src/mlpack/tests/adam_test.cpp                | 12 ++++---
 11 files changed, 44 insertions(+), 57 deletions(-)

diff --git a/src/mlpack/core/optimizers/ada_delta/ada_delta_update.hpp b/src/mlpack/core/optimizers/ada_delta/ada_delta_update.hpp
index cb582239a8..391e2a88bd 100644
--- a/src/mlpack/core/optimizers/ada_delta/ada_delta_update.hpp
+++ b/src/mlpack/core/optimizers/ada_delta/ada_delta_update.hpp
@@ -83,8 +83,7 @@ class AdaDeltaUpdate
    */
   void Update(arma::mat& iterate,
               const double stepSize,
-              const arma::mat& gradient,
-              const size_t /*i*/)
+              const arma::mat& gradient)
   {
     // Accumulate gradient.
     meanSquaredGradient *= rho;
diff --git a/src/mlpack/core/optimizers/ada_grad/ada_grad_update.hpp b/src/mlpack/core/optimizers/ada_grad/ada_grad_update.hpp
index bc0f1296ae..2fb52573b1 100644
--- a/src/mlpack/core/optimizers/ada_grad/ada_grad_update.hpp
+++ b/src/mlpack/core/optimizers/ada_grad/ada_grad_update.hpp
@@ -79,8 +79,7 @@ class AdaGradUpdate
    */
   void Update(arma::mat& iterate,
               const double stepSize,
-              const arma::mat& gradient,
-              const size_t /*i*/)
+              const arma::mat& gradient)
   {
     squaredGradient += (gradient % gradient);
     iterate -= (stepSize * gradient) / (arma::sqrt(squaredGradient) + epsilon);
diff --git a/src/mlpack/core/optimizers/adam/adam.hpp b/src/mlpack/core/optimizers/adam/adam.hpp
index 9c729b4d88..e590c0b281 100644
--- a/src/mlpack/core/optimizers/adam/adam.hpp
+++ b/src/mlpack/core/optimizers/adam/adam.hpp
@@ -64,7 +64,7 @@ namespace optimization {
  * @tparam DecomposableFunctionType Decomposable objective function type to be
  *     minimized.
  */
-template<typename DecomposableFunctionType>
+template<typename DecomposableFunctionType, bool adaMax = false>
 class Adam
 {
  public:
@@ -87,8 +87,6 @@ class Adam
    * @param tolerance Maximum absolute tolerance to terminate algorithm.
    * @param shuffle If true, the function order is shuffled; otherwise, each
    *        function is visited in linear order.
-   * @param adaMax If true, then the AdaMax optimizer is used; otherwise, by
-   *        default the Adam optimizer is used.
    */
   Adam(DecomposableFunctionType& function,
       const double stepSize = 0.001,
@@ -97,8 +95,7 @@ class Adam
       const double eps = 1e-8,
       const size_t maxIterations = 100000,
       const double tolerance = 1e-5,
-      const bool shuffle = true,
-      const bool adaMax = false);
+      const bool shuffle = true);
 
   /**
    * Optimize the given function using Adam. The given starting point will be
@@ -153,14 +150,9 @@ class Adam
   //! Modify whether or not the individual functions are shuffled.
   bool& Shuffle() { return optimizer.Shuffle(); }
 
-  //! Get whether or not the AdaMax optimizer is specified.
-  bool AdaMax() const { return optimizer.UpdatePolicy().AdaMax(); }
-  //! Modify wehther or not the AdaMax optimizer is to be used.
-  bool& AdaMax() { return optimizer.UpdatePolicy().AdaMax(); }
-
  private:
   //! The Stochastic Gradient Descent object with Adam policy.
-  SGD<DecomposableFunctionType, AdamUpdate> optimizer;
+  SGD<DecomposableFunctionType, AdamUpdate<adaMax> > optimizer;
 };
 
 } // namespace optimization
diff --git a/src/mlpack/core/optimizers/adam/adam_impl.hpp b/src/mlpack/core/optimizers/adam/adam_impl.hpp
index b9faf1226d..4905a0cec2 100644
--- a/src/mlpack/core/optimizers/adam/adam_impl.hpp
+++ b/src/mlpack/core/optimizers/adam/adam_impl.hpp
@@ -21,25 +21,23 @@
 namespace mlpack {
 namespace optimization {
 
-template<typename DecomposableFunctionType>
-Adam<DecomposableFunctionType>::Adam(DecomposableFunctionType& function,
-                                     const double stepSize,
-                                     const double beta1,
-                                     const double beta2,
-                                     const double epsilon,
-                                     const size_t maxIterations,
-                                     const double tolerance,
-                                     const bool shuffle,
-                                     const bool adaMax) :
+template<typename DecomposableFunctionType, bool adaMax>
+Adam<DecomposableFunctionType, adaMax>::Adam(DecomposableFunctionType& function,
+                                             const double stepSize,
+                                             const double beta1,
+                                             const double beta2,
+                                             const double epsilon,
+                                             const size_t maxIterations,
+                                             const double tolerance,
+                                             const bool shuffle) :
     optimizer(function,
               stepSize,
               maxIterations,
               tolerance,
               shuffle,
-              AdamUpdate(epsilon,
-                         beta1,
-                         beta2,
-                         adaMax))
+              AdamUpdate<adaMax>(epsilon,
+                                 beta1,
+                                 beta2))
 { /* Nothing to do. */ }
 
 } // namespace optimization
diff --git a/src/mlpack/core/optimizers/adam/adam_update.hpp b/src/mlpack/core/optimizers/adam/adam_update.hpp
index fba575f4ce..d5a74f17b4 100644
--- a/src/mlpack/core/optimizers/adam/adam_update.hpp
+++ b/src/mlpack/core/optimizers/adam/adam_update.hpp
@@ -39,8 +39,11 @@ namespace optimization {
  *   year      = {2014}
  * }
  * @endcode
- *
+ * 
+ * @param adaMax If true, then the AdaMax optimizer is used; otherwise, by
+ *        default the Adam optimizer is used.
  */
+template<bool adaMax>
 class AdamUpdate
 {
  public:
@@ -52,12 +55,11 @@ class AdamUpdate
    */
   AdamUpdate(const double epsilon = 1e-8,
              const double beta1 = 0.9,
-             const double beta2 = 0.999,
-             const bool adaMax = false) :
+             const double beta2 = 0.999) :
     epsilon(epsilon),
     beta1(beta1),
     beta2(beta2),
-    adaMax(adaMax)
+    iteration(0)
   {
     // Nothing to do.
   }
@@ -92,9 +94,11 @@ class AdamUpdate
    */
   void Update(arma::mat& iterate,
               const double stepSize,
-              const arma::mat& gradient,
-              const size_t i)
+              const arma::mat& gradient)
   {
+    // Increment the iteration counter variable.
+    ++iteration;
+
     // And update the iterate.
     m *= beta1;
     m += (1 - beta1) * gradient;
@@ -111,8 +115,8 @@ class AdamUpdate
       v += (1 - beta2) * (gradient % gradient);
     }
 
-    const double biasCorrection1 = 1.0 - std::pow(beta1, (double) i);
-    const double biasCorrection2 = 1.0 - std::pow(beta2, (double) i);
+    const double biasCorrection1 = 1.0 - std::pow(beta1, (double) iteration);
+    const double biasCorrection2 = 1.0 - std::pow(beta2, (double) iteration);
 
     if (adaMax)
     {
@@ -146,11 +150,6 @@ class AdamUpdate
   //! Modify the second moment coefficient.
   double& Beta2() { return beta2; }
 
-  //! Get whether or not the AdaMax optimizer is specified.
-  bool AdaMax() const { return adaMax; }
-  //! Modify wehther or not the AdaMax optimizer is to be used.
-  bool& AdaMax() { return adaMax; }
-
  private:
   // The epsilon value used to initialise the squared gradient parameter.
   double epsilon;
@@ -161,9 +160,6 @@ class AdamUpdate
   // The second moment coefficient.
   double beta2;
 
-  //! Specifies whether or not the AdaMax optimizer is to be used.
-  bool adaMax;
-
   // The exponential moving average of gradient values.
   arma::mat m;
 
@@ -172,6 +168,9 @@ class AdamUpdate
 
   // The exponential moving average of squared gradient values.
   arma::mat v;
+
+  // The number of iterations.
+  double iteration;
 };
 
 } // namespace optimization
diff --git a/src/mlpack/core/optimizers/sgd/sgd_impl.hpp b/src/mlpack/core/optimizers/sgd/sgd_impl.hpp
index ca756cb20e..5ac19ba960 100644
--- a/src/mlpack/core/optimizers/sgd/sgd_impl.hpp
+++ b/src/mlpack/core/optimizers/sgd/sgd_impl.hpp
@@ -108,7 +108,7 @@ double SGD<DecomposableFunctionType, UpdatePolicyType>::Optimize(
       function.Gradient(iterate, currentFunction, gradient);
 
     // Use the update policy to take a step.
-    updatePolicy.Update(iterate, stepSize, gradient, i);
+    updatePolicy.Update(iterate, stepSize, gradient);
 
     // Now add that to the overall objective function.
     if (shuffle)
diff --git a/src/mlpack/core/optimizers/sgd/update_policies/momentum_update.hpp b/src/mlpack/core/optimizers/sgd/update_policies/momentum_update.hpp
index 7e4a1deeed..2947c355f9 100644
--- a/src/mlpack/core/optimizers/sgd/update_policies/momentum_update.hpp
+++ b/src/mlpack/core/optimizers/sgd/update_policies/momentum_update.hpp
@@ -99,8 +99,7 @@ class MomentumUpdate
    */
   void Update(arma::mat& iterate,
               const double stepSize,
-              const arma::mat& gradient,
-              const size_t /*i*/)
+              const arma::mat& gradient)
   {
     velocity = momentum * velocity - stepSize * gradient;
     iterate += velocity;
diff --git a/src/mlpack/core/optimizers/sgd/update_policies/vanilla_update.hpp b/src/mlpack/core/optimizers/sgd/update_policies/vanilla_update.hpp
index 159d7f3398..1bd85bf1c3 100644
--- a/src/mlpack/core/optimizers/sgd/update_policies/vanilla_update.hpp
+++ b/src/mlpack/core/optimizers/sgd/update_policies/vanilla_update.hpp
@@ -52,8 +52,7 @@ class VanillaUpdate
   */
   void Update(arma::mat& iterate,
               const double stepSize,
-              const arma::mat& gradient,
-              const size_t /*i*/)
+              const arma::mat& gradient)
   {
     // Perform the vanilla SGD update.
     iterate -= stepSize * gradient;
diff --git a/src/mlpack/methods/logistic_regression/logistic_regression.hpp b/src/mlpack/methods/logistic_regression/logistic_regression.hpp
index 7a03a0a04e..a19f31b876 100644
--- a/src/mlpack/methods/logistic_regression/logistic_regression.hpp
+++ b/src/mlpack/methods/logistic_regression/logistic_regression.hpp
@@ -119,7 +119,7 @@ class LogisticRegression
    * @param responses Outputs results from input training variables.
    */
   template<
-      template<typename> class OptimizerType = mlpack::optimization::L_BFGS
+      template<typename...> class OptimizerType = mlpack::optimization::L_BFGS
   >
   void Train(const MatType& predictors,
              const arma::Row<size_t>& responses);
diff --git a/src/mlpack/methods/logistic_regression/logistic_regression_impl.hpp b/src/mlpack/methods/logistic_regression/logistic_regression_impl.hpp
index 4974c975a5..9494525847 100644
--- a/src/mlpack/methods/logistic_regression/logistic_regression_impl.hpp
+++ b/src/mlpack/methods/logistic_regression/logistic_regression_impl.hpp
@@ -66,7 +66,7 @@ LogisticRegression<MatType>::LogisticRegression(
 }
 
 template<typename MatType>
-template<template<typename> class OptimizerType>
+template<template<typename...> class OptimizerType>
 void LogisticRegression<MatType>::Train(const MatType& predictors,
                                         const arma::Row<size_t>& responses)
 {
diff --git a/src/mlpack/tests/adam_test.cpp b/src/mlpack/tests/adam_test.cpp
index 88d839fd5c..6dae1e4320 100644
--- a/src/mlpack/tests/adam_test.cpp
+++ b/src/mlpack/tests/adam_test.cpp
@@ -36,7 +36,8 @@ BOOST_AUTO_TEST_SUITE(AdamTest);
 BOOST_AUTO_TEST_CASE(SimpleAdamTestFunction)
 {
   SGDTestFunction f;
-  Adam<SGDTestFunction> optimizer(f, 1e-3, 0.9, 0.999, 1e-8, 5000000, 1e-9, true);
+  Adam<SGDTestFunction> optimizer(f, 1e-3, 0.9, 0.999, 1e-8, 5000000, 1e-9,
+                                  true);
 
   arma::mat coordinates = f.GetInitialPoint();
   optimizer.Optimize(coordinates);
@@ -52,8 +53,8 @@ BOOST_AUTO_TEST_CASE(SimpleAdamTestFunction)
 BOOST_AUTO_TEST_CASE(SimpleAdaMaxTestFunction)
 {
   SGDTestFunction f;
-  Adam<SGDTestFunction> optimizer(f, 2e-3, 0.9, 0.999, 1e-8, 5000000, 1e-9, true
-                                  ,true);
+  Adam<SGDTestFunction, true> optimizer(f, 2e-3, 0.9, 0.999, 1e-8, 5000000, 1e-9,
+                                        true);
 
   arma::mat coordinates = f.GetInitialPoint();
   optimizer.Optimize(coordinates);
@@ -174,8 +175,9 @@ BOOST_AUTO_TEST_CASE(AdaMaxLogisticRegressionTest)
   LogisticRegression<> lr(shuffledData.n_rows, 0.5);
 
   LogisticRegressionFunction<> lrf(shuffledData, shuffledResponses, 0.5);
-  Adam<LogisticRegressionFunction<> > adamax(lrf, 1e-3, 0.9, 0.999, 1e-8, 5000000,
-                                             1e-9, true, true);
+  Adam<LogisticRegressionFunction<>, true> adamax(lrf, 1e-3, 0.9, 0.999, 1e-8,
+                                                  5000000, 1e-9, true);
+
   lr.Train(adamax);
 
   // Ensure that the error is close to zero.

From e01bb43b2474169f03668bc90467e65efe3443a1 Mon Sep 17 00:00:00 2001
From: Vivek Pal <vivekpal.dtu@gmail.com>
Date: Sun, 2 Apr 2017 01:20:06 +0530
Subject: [PATCH 05/12] Create separate UpdateRule classes for Adam and AdaMax

---
 .../core/optimizers/adam/CMakeLists.txt       |   1 +
 src/mlpack/core/optimizers/adam/adam.hpp      |  19 ++-
 src/mlpack/core/optimizers/adam/adam_impl.hpp |  25 +--
 .../core/optimizers/adam/adam_update.hpp      |  51 ++-----
 .../core/optimizers/adam/adamax_update.hpp    | 143 ++++++++++++++++++
 src/mlpack/tests/adam_test.cpp                |   8 +-
 6 files changed, 186 insertions(+), 61 deletions(-)
 create mode 100644 src/mlpack/core/optimizers/adam/adamax_update.hpp

diff --git a/src/mlpack/core/optimizers/adam/CMakeLists.txt b/src/mlpack/core/optimizers/adam/CMakeLists.txt
index eabdbbaafa..1377bbb0e1 100644
--- a/src/mlpack/core/optimizers/adam/CMakeLists.txt
+++ b/src/mlpack/core/optimizers/adam/CMakeLists.txt
@@ -2,6 +2,7 @@ set(SOURCES
   adam.hpp
   adam_impl.hpp
   adam_update.hpp
+  adamax_update.hpp
 )
 
 set(DIR_SRCS)
diff --git a/src/mlpack/core/optimizers/adam/adam.hpp b/src/mlpack/core/optimizers/adam/adam.hpp
index e590c0b281..f81b3efb0e 100644
--- a/src/mlpack/core/optimizers/adam/adam.hpp
+++ b/src/mlpack/core/optimizers/adam/adam.hpp
@@ -22,6 +22,7 @@
 
 #include <mlpack/core/optimizers/sgd/sgd.hpp>
 #include "adam_update.hpp"
+#include "adamax_update.hpp"
 
 namespace mlpack {
 namespace optimization {
@@ -63,9 +64,13 @@ namespace optimization {
  *
  * @tparam DecomposableFunctionType Decomposable objective function type to be
  *     minimized.
+ * @tparam UpdateRule Adam optimizer update rule to be used.
  */
-template<typename DecomposableFunctionType, bool adaMax = false>
-class Adam
+template<
+    typename DecomposableFunctionType,
+    typename UpdateRule = AdamUpdate
+>
+class AdamType
 {
  public:
   /**
@@ -88,7 +93,7 @@ class Adam
    * @param shuffle If true, the function order is shuffled; otherwise, each
    *        function is visited in linear order.
    */
-  Adam(DecomposableFunctionType& function,
+  AdamType(DecomposableFunctionType& function,
       const double stepSize = 0.001,
       const double beta1 = 0.9,
       const double beta2 = 0.999,
@@ -152,9 +157,15 @@ class Adam
 
  private:
   //! The Stochastic Gradient Descent object with Adam policy.
-  SGD<DecomposableFunctionType, AdamUpdate<adaMax> > optimizer;
+  SGD<DecomposableFunctionType, UpdateRule> optimizer;
 };
 
+template<typename DecomposableFunctionType>
+using Adam = AdamType<DecomposableFunctionType, AdamUpdate>;
+
+template<typename DecomposableFunctionType>
+using AdaMax = AdamType<DecomposableFunctionType, AdaMaxUpdate>;
+
 } // namespace optimization
 } // namespace mlpack
 
diff --git a/src/mlpack/core/optimizers/adam/adam_impl.hpp b/src/mlpack/core/optimizers/adam/adam_impl.hpp
index 4905a0cec2..82408de71e 100644
--- a/src/mlpack/core/optimizers/adam/adam_impl.hpp
+++ b/src/mlpack/core/optimizers/adam/adam_impl.hpp
@@ -21,23 +21,24 @@
 namespace mlpack {
 namespace optimization {
 
-template<typename DecomposableFunctionType, bool adaMax>
-Adam<DecomposableFunctionType, adaMax>::Adam(DecomposableFunctionType& function,
-                                             const double stepSize,
-                                             const double beta1,
-                                             const double beta2,
-                                             const double epsilon,
-                                             const size_t maxIterations,
-                                             const double tolerance,
-                                             const bool shuffle) :
+template<typename DecomposableFunctionType, typename UpdateRule>
+AdamType<DecomposableFunctionType, UpdateRule>::AdamType(
+    DecomposableFunctionType& function,
+    const double stepSize,
+    const double beta1,
+    const double beta2,
+    const double epsilon,
+    const size_t maxIterations,
+    const double tolerance,
+    const bool shuffle) :
     optimizer(function,
               stepSize,
               maxIterations,
               tolerance,
               shuffle,
-              AdamUpdate<adaMax>(epsilon,
-                                 beta1,
-                                 beta2))
+              UpdateRule(epsilon,
+                         beta1,
+                         beta2))
 { /* Nothing to do. */ }
 
 } // namespace optimization
diff --git a/src/mlpack/core/optimizers/adam/adam_update.hpp b/src/mlpack/core/optimizers/adam/adam_update.hpp
index d5a74f17b4..aa20e2f752 100644
--- a/src/mlpack/core/optimizers/adam/adam_update.hpp
+++ b/src/mlpack/core/optimizers/adam/adam_update.hpp
@@ -39,11 +39,7 @@ namespace optimization {
  *   year      = {2014}
  * }
  * @endcode
- * 
- * @param adaMax If true, then the AdaMax optimizer is used; otherwise, by
- *        default the Adam optimizer is used.
  */
-template<bool adaMax>
 class AdamUpdate
 {
  public:
@@ -75,14 +71,7 @@ class AdamUpdate
                   const size_t cols)
   {
     m = arma::zeros<arma::mat>(rows, cols);
-    if (adaMax)
-    {
-      u = arma::zeros<arma::mat>(rows, cols);
-    }
-    else
-    {
-      v = arma::zeros<arma::mat>(rows, cols);
-    }
+    v = arma::zeros<arma::mat>(rows, cols);
   }
 
   /**
@@ -103,36 +92,19 @@ class AdamUpdate
     m *= beta1;
     m += (1 - beta1) * gradient;
 
-    if (adaMax)
-    {
-      // Update the exponentially weighted infinity norm.
-      u *= beta2;
-      u = arma::max(u, arma::abs(gradient));
-    }
-    else
-    {
-      v *= beta2;
-      v += (1 - beta2) * (gradient % gradient);
-    }
+    v *= beta2;
+    v += (1 - beta2) * (gradient % gradient);
 
     const double biasCorrection1 = 1.0 - std::pow(beta1, (double) iteration);
     const double biasCorrection2 = 1.0 - std::pow(beta2, (double) iteration);
 
-    if (adaMax)
-    {
-      if (biasCorrection1 != 0.0)
-        iterate -= (stepSize / biasCorrection1 * m / (u + epsilon));
-    }
-    else
-    {
-      /**
-       * It should be noted that the term, m / (arma::sqrt(v) + eps), in the
-       * following expression is an approximation of the following actual term;
-       * m / (arma::sqrt(v) + (arma::sqrt(biasCorrection2) * eps).
-       */
-      iterate -= (stepSize * std::sqrt(biasCorrection2) / biasCorrection1) *
-                  m / (arma::sqrt(v) + epsilon);
-    }
+    /**
+     * It should be noted that the term, m / (arma::sqrt(v) + eps), in the
+     * following expression is an approximation of the following actual term;
+     * m / (arma::sqrt(v) + (arma::sqrt(biasCorrection2) * eps).
+     */
+    iterate -= (stepSize * std::sqrt(biasCorrection2) / biasCorrection1) *
+                m / (arma::sqrt(v) + epsilon);
   }
 
   //! Get the value used to initialise the squared gradient parameter.
@@ -163,9 +135,6 @@ class AdamUpdate
   // The exponential moving average of gradient values.
   arma::mat m;
 
-  // The exponentially weighted infinity norm.
-  arma::mat u;
-
   // The exponential moving average of squared gradient values.
   arma::mat v;
 
diff --git a/src/mlpack/core/optimizers/adam/adamax_update.hpp b/src/mlpack/core/optimizers/adam/adamax_update.hpp
new file mode 100644
index 0000000000..7aa5b6b990
--- /dev/null
+++ b/src/mlpack/core/optimizers/adam/adamax_update.hpp
@@ -0,0 +1,143 @@
+/**
+ * @file adam.hpp
+ * @author Ryan Curtin
+ * @author Vasanth Kalingeri
+ * @author Marcus Edel
+ * @author Vivek Pal
+ *
+ * AdaMax update rule. Adam is an an algorithm for first-order gradient-
+ * -based optimization of stochastic objective functions, based on adaptive
+ * estimates of lower-order moments. AdaMax is simply a variant of Adam based
+ * on the infinity norm.
+ *
+ * mlpack is free software; you may redistribute it and/or modify it under the
+ * terms of the 3-clause BSD license.  You should have received a copy of the
+ * 3-clause BSD license along with mlpack.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef MLPACK_CORE_OPTIMIZERS_ADAM_ADAMAX_UPDATE_HPP
+#define MLPACK_CORE_OPTIMIZERS_ADAM_ADAMAX_UPDATE_HPP
+
+#include <mlpack/prereqs.hpp>
+
+namespace mlpack {
+namespace optimization {
+
+/**
+ * AdaMax is a variant of Adam, an optimizer that computes individual adaptive
+ * learning rates for different parameters from estimates of first and second
+ * moments of the gradients.based on the infinity norm as given in the section
+ * 7 of the following paper.
+ *
+ * For more information, see the following.
+ *
+ * @code
+ * @article{Kingma2014,
+ *   author    = {Diederik P. Kingma and Jimmy Ba},
+ *   title     = {Adam: {A} Method for Stochastic Optimization},
+ *   journal   = {CoRR},
+ *   year      = {2014}
+ * }
+ * @endcode
+ */
+class AdaMaxUpdate
+{
+ public:
+  /**
+   * Construct the AdaMax update policy with the given epsilon parameter.
+   *
+   * @param epsilon The epsilon value used to initialise the squared gradient
+   *        parameter.
+   */
+  AdaMaxUpdate(const double epsilon = 1e-8,
+               const double beta1 = 0.9,
+               const double beta2 = 0.999) :
+    epsilon(epsilon),
+    beta1(beta1),
+    beta2(beta2),
+    iteration(0)
+  {
+    // Nothing to do.
+  }
+
+  /**
+   * The Initialize method is called by SGD Optimizer method before the start of
+   * the iteration update process.
+   *
+   * @param rows number of rows in the gradient matrix.
+   * @param cols number of columns in the gradient matrix.
+   */
+  void Initialize(const size_t rows,
+                  const size_t cols)
+  {
+    m = arma::zeros<arma::mat>(rows, cols);
+    u = arma::zeros<arma::mat>(rows, cols);
+  }
+
+  /**
+   * Update step for Adam.
+   *
+   * @param iterate Parameters that minimize the function.
+   * @param stepSize Step size to be used for the given iteration.
+   * @param gradient The gradient matrix.
+   */
+  void Update(arma::mat& iterate,
+              const double stepSize,
+              const arma::mat& gradient)
+  {
+    // Increment the iteration counter variable.
+    ++iteration;
+
+    // And update the iterate.
+    m *= beta1;
+    m += (1 - beta1) * gradient;
+
+    // Update the exponentially weighted infinity norm.
+    u *= beta2;
+    u = arma::max(u, arma::abs(gradient));
+
+    const double biasCorrection1 = 1.0 - std::pow(beta1, (double) iteration);
+
+    if (biasCorrection1 != 0)
+      iterate -= (stepSize / biasCorrection1 * m / (u + epsilon));
+  }
+
+  //! Get the value used to initialise the squared gradient parameter.
+  double Epsilon() const { return epsilon; }
+  //! Modify the value used to initialise the squared gradient parameter.
+  double& Epsilon() { return epsilon; }
+
+  //! Get the smoothing parameter.
+  double Beta1() const { return beta1; }
+  //! Modify the smoothing parameter.
+  double& Beta1() { return beta1; }
+
+  //! Get the second moment coefficient.
+  double Beta2() const { return beta2; }
+  //! Modify the second moment coefficient.
+  double& Beta2() { return beta2; }
+
+ private:
+  // The epsilon value used to initialise the squared gradient parameter.
+  double epsilon;
+
+  // The smoothing parameter.
+  double beta1;
+
+  // The second moment coefficient.
+  double beta2;
+
+  // The exponential moving average of gradient values.
+  arma::mat m;
+
+  // The exponentially weighted infinity norm.
+  arma::mat u;
+
+  // The number of iterations.
+  double iteration;
+};
+
+} // namespace optimization
+} // namespace mlpack
+
+#endif
diff --git a/src/mlpack/tests/adam_test.cpp b/src/mlpack/tests/adam_test.cpp
index 6dae1e4320..a88e3618f8 100644
--- a/src/mlpack/tests/adam_test.cpp
+++ b/src/mlpack/tests/adam_test.cpp
@@ -53,8 +53,8 @@ BOOST_AUTO_TEST_CASE(SimpleAdamTestFunction)
 BOOST_AUTO_TEST_CASE(SimpleAdaMaxTestFunction)
 {
   SGDTestFunction f;
-  Adam<SGDTestFunction, true> optimizer(f, 2e-3, 0.9, 0.999, 1e-8, 5000000, 1e-9,
-                                        true);
+  AdaMax<SGDTestFunction> optimizer(f, 2e-3, 0.9, 0.999, 1e-8, 5000000, 1e-9,
+                                    true);
 
   arma::mat coordinates = f.GetInitialPoint();
   optimizer.Optimize(coordinates);
@@ -175,8 +175,8 @@ BOOST_AUTO_TEST_CASE(AdaMaxLogisticRegressionTest)
   LogisticRegression<> lr(shuffledData.n_rows, 0.5);
 
   LogisticRegressionFunction<> lrf(shuffledData, shuffledResponses, 0.5);
-  Adam<LogisticRegressionFunction<>, true> adamax(lrf, 1e-3, 0.9, 0.999, 1e-8,
-                                                  5000000, 1e-9, true);
+  AdaMax<LogisticRegressionFunction<> > adamax(lrf, 1e-3, 0.9, 0.999, 1e-8,
+                                               5000000, 1e-9, true);
 
   lr.Train(adamax);
 

From d77444f5b1a9aa760cefb873dbb60ba873731c4a Mon Sep 17 00:00:00 2001
From: Vivek Pal <vivekpal.dtu@gmail.com>
Date: Sun, 2 Apr 2017 19:44:47 +0530
Subject: [PATCH 06/12] Add parameters description

---
 src/mlpack/core/optimizers/adam/adam_update.hpp   | 2 ++
 src/mlpack/core/optimizers/adam/adamax_update.hpp | 2 ++
 2 files changed, 4 insertions(+)

diff --git a/src/mlpack/core/optimizers/adam/adam_update.hpp b/src/mlpack/core/optimizers/adam/adam_update.hpp
index aa20e2f752..71a283632a 100644
--- a/src/mlpack/core/optimizers/adam/adam_update.hpp
+++ b/src/mlpack/core/optimizers/adam/adam_update.hpp
@@ -48,6 +48,8 @@ class AdamUpdate
    *
    * @param epsilon The epsilon value used to initialise the squared gradient
    *        parameter.
+   * @param beta1 The smoothing parameter.
+   * @param beta2 The second moment coefficient.
    */
   AdamUpdate(const double epsilon = 1e-8,
              const double beta1 = 0.9,
diff --git a/src/mlpack/core/optimizers/adam/adamax_update.hpp b/src/mlpack/core/optimizers/adam/adamax_update.hpp
index 7aa5b6b990..a23996f30a 100644
--- a/src/mlpack/core/optimizers/adam/adamax_update.hpp
+++ b/src/mlpack/core/optimizers/adam/adamax_update.hpp
@@ -48,6 +48,8 @@ class AdaMaxUpdate
    *
    * @param epsilon The epsilon value used to initialise the squared gradient
    *        parameter.
+   * @param beta1 The smoothing parameter.
+   * @param beta2 The second moment coefficient.
    */
   AdaMaxUpdate(const double epsilon = 1e-8,
                const double beta1 = 0.9,

From 565242ef2f0b85616e419e1246d4ce617330771a Mon Sep 17 00:00:00 2001
From: Vivek Pal <vivekpal.dtu@gmail.com>
Date: Sun, 2 Apr 2017 19:45:37 +0530
Subject: [PATCH 07/12] Fix alignment issue

---
 src/mlpack/core/optimizers/adam/adam.hpp | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/src/mlpack/core/optimizers/adam/adam.hpp b/src/mlpack/core/optimizers/adam/adam.hpp
index f81b3efb0e..37411d9bb0 100644
--- a/src/mlpack/core/optimizers/adam/adam.hpp
+++ b/src/mlpack/core/optimizers/adam/adam.hpp
@@ -94,13 +94,13 @@ class AdamType
    *        function is visited in linear order.
    */
   AdamType(DecomposableFunctionType& function,
-      const double stepSize = 0.001,
-      const double beta1 = 0.9,
-      const double beta2 = 0.999,
-      const double eps = 1e-8,
-      const size_t maxIterations = 100000,
-      const double tolerance = 1e-5,
-      const bool shuffle = true);
+          const double stepSize = 0.001,
+          const double beta1 = 0.9,
+          const double beta2 = 0.999,
+          const double eps = 1e-8,
+          const size_t maxIterations = 100000,
+          const double tolerance = 1e-5,
+          const bool shuffle = true);
 
   /**
    * Optimize the given function using Adam. The given starting point will be

From ce245d1dbf96d2b81334e9ff0f807015029a3433 Mon Sep 17 00:00:00 2001
From: Vivek Pal <vivekpal.dtu@gmail.com>
Date: Sun, 2 Apr 2017 22:37:08 +0530
Subject: [PATCH 08/12] Correct the file name

---
 src/mlpack/core/optimizers/adam/adam_update.hpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/mlpack/core/optimizers/adam/adam_update.hpp b/src/mlpack/core/optimizers/adam/adam_update.hpp
index 71a283632a..c6e61b1ff4 100644
--- a/src/mlpack/core/optimizers/adam/adam_update.hpp
+++ b/src/mlpack/core/optimizers/adam/adam_update.hpp
@@ -1,5 +1,5 @@
 /**
- * @file adam.hpp
+ * @file adam_update.hpp
  * @author Ryan Curtin
  * @author Vasanth Kalingeri
  * @author Marcus Edel

From de7ff08cc59d7f790e9d68d8247f3aec1dcb1a17 Mon Sep 17 00:00:00 2001
From: Vivek Pal <vivekpal.dtu@gmail.com>
Date: Tue, 4 Apr 2017 18:35:15 +0530
Subject: [PATCH 09/12] Remove comments about AdaMax in adam_update.hpp

---
 src/mlpack/core/optimizers/adam/adam_update.hpp | 10 ++++------
 1 file changed, 4 insertions(+), 6 deletions(-)

diff --git a/src/mlpack/core/optimizers/adam/adam_update.hpp b/src/mlpack/core/optimizers/adam/adam_update.hpp
index c6e61b1ff4..7e42132dfe 100644
--- a/src/mlpack/core/optimizers/adam/adam_update.hpp
+++ b/src/mlpack/core/optimizers/adam/adam_update.hpp
@@ -5,10 +5,9 @@
  * @author Marcus Edel
  * @author Vivek Pal
  *
- * Adam and AdaMax optimizer. Adam is an an algorithm for first-order gradient-
- * -based optimization of stochastic objective functions, based on adaptive
- * estimates of lower-order moments. AdaMax is simply a variant of Adam based
- * on the infinity norm.
+ * Adam optimizer. Adam is an an algorithm for first-order gradient-based
+ * optimization of stochastic objective functions, based on adaptive estimates
+ * of lower-order moments.
  *
  * mlpack is free software; you may redistribute it and/or modify it under the
  * terms of the 3-clause BSD license.  You should have received a copy of the
@@ -26,8 +25,7 @@ namespace optimization {
 /**
  * Adam is an optimizer that computes individual adaptive learning rates for
  * different parameters from estimates of first and second moments of the
- * gradients. AdaMax is a variant of Adam based on the infinity norm as given
- * in the section 7 of the following paper.
+ * gradients as given in the section 7 of the following paper.
  *
  * For more information, see the following.
  *

From bec9a20185e7c45973db222fbef1662c3d1b5524 Mon Sep 17 00:00:00 2001
From: Vivek Pal <vivekpal.dtu@gmail.com>
Date: Tue, 4 Apr 2017 18:36:35 +0530
Subject: [PATCH 10/12] Remove a redundant type cast

---
 src/mlpack/core/optimizers/adam/adamax_update.hpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/mlpack/core/optimizers/adam/adamax_update.hpp b/src/mlpack/core/optimizers/adam/adamax_update.hpp
index a23996f30a..17a33d2d26 100644
--- a/src/mlpack/core/optimizers/adam/adamax_update.hpp
+++ b/src/mlpack/core/optimizers/adam/adamax_update.hpp
@@ -98,7 +98,7 @@ class AdaMaxUpdate
     u *= beta2;
     u = arma::max(u, arma::abs(gradient));
 
-    const double biasCorrection1 = 1.0 - std::pow(beta1, (double) iteration);
+    const double biasCorrection1 = 1.0 - std::pow(beta1, iteration);
 
     if (biasCorrection1 != 0)
       iterate -= (stepSize / biasCorrection1 * m / (u + epsilon));

From c049330b118e7006545fb2ddbeb3bad1ec65ea79 Mon Sep 17 00:00:00 2001
From: Vivek Pal <vivekpal.dtu@gmail.com>
Date: Tue, 4 Apr 2017 18:40:12 +0530
Subject: [PATCH 11/12] Update a few comments

---
 src/mlpack/core/optimizers/adam/adam_update.hpp   | 2 +-
 src/mlpack/core/optimizers/adam/adamax_update.hpp | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/mlpack/core/optimizers/adam/adam_update.hpp b/src/mlpack/core/optimizers/adam/adam_update.hpp
index 7e42132dfe..95920c14ee 100644
--- a/src/mlpack/core/optimizers/adam/adam_update.hpp
+++ b/src/mlpack/core/optimizers/adam/adam_update.hpp
@@ -42,7 +42,7 @@ class AdamUpdate
 {
  public:
   /**
-   * Construct the Adam update policy with the given epsilon parameter.
+   * Construct the Adam update policy with the given parameters.
    *
    * @param epsilon The epsilon value used to initialise the squared gradient
    *        parameter.
diff --git a/src/mlpack/core/optimizers/adam/adamax_update.hpp b/src/mlpack/core/optimizers/adam/adamax_update.hpp
index 17a33d2d26..97e26fe683 100644
--- a/src/mlpack/core/optimizers/adam/adamax_update.hpp
+++ b/src/mlpack/core/optimizers/adam/adamax_update.hpp
@@ -1,5 +1,5 @@
 /**
- * @file adam.hpp
+ * @file adamax_update.hpp
  * @author Ryan Curtin
  * @author Vasanth Kalingeri
  * @author Marcus Edel
@@ -44,7 +44,7 @@ class AdaMaxUpdate
 {
  public:
   /**
-   * Construct the AdaMax update policy with the given epsilon parameter.
+   * Construct the AdaMax update policy with the given parameters.
    *
    * @param epsilon The epsilon value used to initialise the squared gradient
    *        parameter.

From e7bfbb19dfbc8cecb5ae094fc3cc45ca0ef87e6a Mon Sep 17 00:00:00 2001
From: Vivek Pal <vivekpal.dtu@gmail.com>
Date: Fri, 14 Apr 2017 23:05:35 +0530
Subject: [PATCH 12/12] Add paper url to the citation

---
 src/mlpack/core/optimizers/adam/adam.hpp          | 3 ++-
 src/mlpack/core/optimizers/adam/adam_update.hpp   | 3 ++-
 src/mlpack/core/optimizers/adam/adamax_update.hpp | 3 ++-
 3 files changed, 6 insertions(+), 3 deletions(-)

diff --git a/src/mlpack/core/optimizers/adam/adam.hpp b/src/mlpack/core/optimizers/adam/adam.hpp
index 37411d9bb0..d4808c6cd9 100644
--- a/src/mlpack/core/optimizers/adam/adam.hpp
+++ b/src/mlpack/core/optimizers/adam/adam.hpp
@@ -40,7 +40,8 @@ namespace optimization {
  *   author    = {Diederik P. Kingma and Jimmy Ba},
  *   title     = {Adam: {A} Method for Stochastic Optimization},
  *   journal   = {CoRR},
- *   year      = {2014}
+ *   year      = {2014},
+ *   url       = {http://arxiv.org/abs/1412.6980}
  * }
  * @endcode
  *
diff --git a/src/mlpack/core/optimizers/adam/adam_update.hpp b/src/mlpack/core/optimizers/adam/adam_update.hpp
index 95920c14ee..540ae1ce01 100644
--- a/src/mlpack/core/optimizers/adam/adam_update.hpp
+++ b/src/mlpack/core/optimizers/adam/adam_update.hpp
@@ -34,7 +34,8 @@ namespace optimization {
  *   author    = {Diederik P. Kingma and Jimmy Ba},
  *   title     = {Adam: {A} Method for Stochastic Optimization},
  *   journal   = {CoRR},
- *   year      = {2014}
+ *   year      = {2014},
+ *   url       = {http://arxiv.org/abs/1412.6980}
  * }
  * @endcode
  */
diff --git a/src/mlpack/core/optimizers/adam/adamax_update.hpp b/src/mlpack/core/optimizers/adam/adamax_update.hpp
index 97e26fe683..6337efaae3 100644
--- a/src/mlpack/core/optimizers/adam/adamax_update.hpp
+++ b/src/mlpack/core/optimizers/adam/adamax_update.hpp
@@ -36,7 +36,8 @@ namespace optimization {
  *   author    = {Diederik P. Kingma and Jimmy Ba},
  *   title     = {Adam: {A} Method for Stochastic Optimization},
  *   journal   = {CoRR},
- *   year      = {2014}
+ *   year      = {2014},
+ *   url       = {http://arxiv.org/abs/1412.6980}
  * }
  * @endcode
  */