From 2c106049a61e0128c978b2ee64f100920ef1af32 Mon Sep 17 00:00:00 2001
From: Sourabh Varshney <sourabhvarshney111@gmail.com>
Date: Mon, 11 Dec 2017 17:42:26 +0530
Subject: [PATCH 01/67] Added Optimizer Nadam

---
 .../core/optimizers/Nadam/CMakeLists.txt      |  12 ++
 src/mlpack/core/optimizers/Nadam/nadam.hpp    | 158 ++++++++++++++++++
 .../core/optimizers/Nadam/nadam_impl.hpp      |  41 +++++
 .../core/optimizers/Nadam/nadam_update.hpp    | 117 +++++++++++++
 4 files changed, 328 insertions(+)
 create mode 100644 src/mlpack/core/optimizers/Nadam/CMakeLists.txt
 create mode 100644 src/mlpack/core/optimizers/Nadam/nadam.hpp
 create mode 100644 src/mlpack/core/optimizers/Nadam/nadam_impl.hpp
 create mode 100644 src/mlpack/core/optimizers/Nadam/nadam_update.hpp

diff --git a/src/mlpack/core/optimizers/Nadam/CMakeLists.txt b/src/mlpack/core/optimizers/Nadam/CMakeLists.txt
new file mode 100644
index 0000000000..09bb246794
--- /dev/null
+++ b/src/mlpack/core/optimizers/Nadam/CMakeLists.txt
@@ -0,0 +1,12 @@
+set(SOURCES
+  nadam.hpp
+  nadam_impl.hpp
+  nadam_update.hpp
+)
+
+set(DIR_SRCS)
+foreach(file ${SOURCES})
+  set(DIR_SRCS ${DIR_SRCS} ${CMAKE_CURRENT_SOURCE_DIR}/${file})
+endforeach()
+
+set(MLPACK_SRCS ${MLPACK_SRCS} ${DIR_SRCS} PARENT_SCOPE)
diff --git a/src/mlpack/core/optimizers/Nadam/nadam.hpp b/src/mlpack/core/optimizers/Nadam/nadam.hpp
new file mode 100644
index 0000000000..fe300f9956
--- /dev/null
+++ b/src/mlpack/core/optimizers/Nadam/nadam.hpp
@@ -0,0 +1,158 @@
+/**
+ * @file nadam.hpp
+ * @author Sourabh Varshney
+ *
+ * Nadam optimizer. Nadam is an optimizer that combines the effect of Adam and 
+ * NAG to the gradient descent to improve its Performance.
+ *
+ * mlpack is free software; you may redistribute it and/or modify it under the
+ * terms of the 3-clause BSD license.  You should have received a copy of the
+ * 3-clause BSD license along with mlpack.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef MLPACK_CORE_OPTIMIZERS_NADAM_NADAM_HPP
+#define MLPACK_CORE_OPTIMIZERS_NADAM_NADAM_HPP
+
+#include <mlpack/prereqs.hpp>
+
+#include <mlpack/core/optimizers/sgd/sgd.hpp>
+#include "nadam_update.hpp"
+
+namespace mlpack {
+namespace optimization {
+
+/**
+ * Nadam is an optimizer that combines the Adam and NAG.
+ *
+ * For more information, see the following.
+ *
+ * @code
+ * @article{
+ *   author  = {Sebastian Ruder},
+ *   title   = {An overview of gradient descent optimization algorithms},
+ *   journal = {CoRR},
+ *   year    = {2016},
+ *   url     = {https://arxiv.org/abs/1609.04747v2}
+ * }
+ * @endcode
+ *
+ *
+ * For Nadam to work, a DecomposableFunctionType template parameter
+ * is required. This class must implement the following function:
+ *
+ *   size_t NumFunctions();
+ *   double Evaluate(const arma::mat& coordinates,
+ *                   const size_t i,
+ *                   const size_t batchSize);
+ *   void Gradient(const arma::mat& coordinates,
+ *                 const size_t i,
+ *                 arma::mat& gradient,
+ *                 const size_t batchSize);
+ *
+ * NumFunctions() should return the number of functions (\f$n\f$), and in the
+ * other two functions, the parameter i refers to which individual function (or
+ * gradient) is being evaluated.  So, for the case of a data-dependent function,
+ * such as NCA (see mlpack::nca::NCA), NumFunctions() should return the number
+ * of points in the dataset, and Evaluate(coordinates, 0) will evaluate the
+ * objective function on the first point in the dataset (presumably, the dataset
+ * is held internally in the DecomposableFunctionType).
+ *
+ * @tparam UpdateRule Nadam optimizer update rule to be used.
+ */
+template<typename UpdateRule = NadamUpdate>
+class NadamType
+{
+ public:
+  /**
+   * Construct the Nadam optimizer with the given function and parameters. The
+   * defaults here are not necessarily good for the given problem, so it is
+   * suggested that the values used be tailored to the task at hand.  The
+   * maximum number of iterations refers to the maximum number of points that
+   * are processed (i.e., one iteration equals one point; one iteration does not
+   * equal one pass over the dataset).
+   *
+   * @param stepSize Step size for each iteration.
+   * @param batchSize Number of points to process in a single step.
+   * @param beta1 Exponential decay rate for the first moment estimates.
+   * @param beta2 Exponential decay rate for the weighted infinity norm
+            estimates.
+   * @param eps Value used to initialise the mean squared gradient parameter.
+   * @param maxIterations Maximum number of iterations allowed (0 means no
+   *        limit).
+   * @param tolerance Maximum absolute tolerance to terminate algorithm.
+   * @param shuffle If true, the function order is shuffled; otherwise, each
+   *        function is visited in linear order.
+   */
+  NadamType(const double stepSize = 0.001,
+           const size_t batchSize = 32,
+           const double beta1 = 0.9,
+           const double eps = 1e-8,
+           const size_t maxIterations = 100000,
+           const double tolerance = 1e-5,
+           const bool shuffle = true);
+
+  /**
+   * Optimize the given function using Nadam. The given starting point will be
+   * modified to store the finishing point of the algorithm, and the final
+   * objective value is returned.
+   *
+   * @tparam DecomposableFunctionType Type of the function to optimize.
+   * @param function Function to optimize.
+   * @param iterate Starting point (will be modified).
+   * @return Objective value of the final point.
+   */
+  template<typename DecomposableFunctionType>
+  double Optimize(DecomposableFunctionType& function, arma::mat& iterate)
+  {
+    return optimizer.Optimize(function, iterate);
+  }
+
+  //! Get the step size.
+  double StepSize() const { return optimizer.StepSize(); }
+  //! Modify the step size.
+  double& StepSize() { return optimizer.StepSize(); }
+
+  //! Get the batch size.
+  size_t BatchSize() const { return optimizer.BatchSize(); }
+  //! Modify the batch size.
+  size_t& BatchSize() { return optimizer.BatchSize(); }
+
+  //! Get the smoothing parameter.
+  double Beta1() const { return optimizer.UpdatePolicy().Beta1(); }
+  //! Modify the smoothing parameter.
+  double& Beta1() { return optimizer.UpdatePolicy().Beta1(); }
+
+  //! Get the value used to initialise the mean squared gradient parameter.
+  double Epsilon() const { return optimizer.UpdatePolicy().Epsilon(); }
+  //! Modify the value used to initialise the mean squared gradient parameter.
+  double& Epsilon() { return optimizer.UpdatePolicy().Epsilon(); }
+
+  //! Get the maximum number of iterations (0 indicates no limit).
+  size_t MaxIterations() const { return optimizer.MaxIterations(); }
+  //! Modify the maximum number of iterations (0 indicates no limit).
+  size_t& MaxIterations() { return optimizer.MaxIterations(); }
+
+  //! Get the tolerance for termination.
+  double Tolerance() const { return optimizer.Tolerance(); }
+  //! Modify the tolerance for termination.
+  double& Tolerance() { return optimizer.Tolerance(); }
+
+  //! Get whether or not the individual functions are shuffled.
+  bool Shuffle() const { return optimizer.Shuffle(); }
+  //! Modify whether or not the individual functions are shuffled.
+  bool& Shuffle() { return optimizer.Shuffle(); }
+
+ private:
+  //! The Stochastic Gradient Descent object with Nadam policy.
+  SGD<UpdateRule> optimizer;
+};
+
+using Nadam = NadamType<NadamUpdate>;
+
+} // namespace optimization
+} // namespace mlpack
+
+// Include implementation.
+#include "nadam_impl.hpp"
+
+#endif
diff --git a/src/mlpack/core/optimizers/Nadam/nadam_impl.hpp b/src/mlpack/core/optimizers/Nadam/nadam_impl.hpp
new file mode 100644
index 0000000000..238e951acb
--- /dev/null
+++ b/src/mlpack/core/optimizers/Nadam/nadam_impl.hpp
@@ -0,0 +1,41 @@
+/**
+ * @file nadam_impl.hpp
+ * @author Sourabh Varshney
+ *
+ * Implementation of the Nadam optimizer.
+ *
+ * mlpack is free software; you may redistribute it and/or modify it under the
+ * terms of the 3-clause BSD license.  You should have received a copy of the
+ * 3-clause BSD license along with mlpack.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef MLPACK_CORE_OPTIMIZERS_NADAM_NADAM_IMPL_HPP
+#define MLPACK_CORE_OPTIMIZERS_NADAM_NADAM_IMPL_HPP
+
+// In case it hasn't been included yet.
+#include "nadam.hpp"
+
+namespace mlpack {
+namespace optimization {
+
+template<typename UpdateRule>
+NadamType<UpdateRule>::NadamType(
+    const double stepSize,
+    const size_t batchSize,
+    const double beta1,
+    const double epsilon,
+    const size_t maxIterations,
+    const double tolerance,
+    const bool shuffle) :
+    optimizer(stepSize,
+              batchSize,
+              maxIterations,
+              tolerance,
+              shuffle,
+              UpdateRule(epsilon, beta1))
+{ /* Nothing to do. */ }
+
+} // namespace optimization
+} // namespace mlpack
+
+#endif
diff --git a/src/mlpack/core/optimizers/Nadam/nadam_update.hpp b/src/mlpack/core/optimizers/Nadam/nadam_update.hpp
new file mode 100644
index 0000000000..6c3b3139ce
--- /dev/null
+++ b/src/mlpack/core/optimizers/Nadam/nadam_update.hpp
@@ -0,0 +1,117 @@
+/**
+ * @file nadam_update.hpp
+ * @author Sourabh Varshney
+ *
+ * Nadam optimizer. Nadam is an optimizer that combines the effect of Adam and 
+ * NAG to the gradient descent to improve its Performance.
+ *
+ * mlpack is free software; you may redistribute it and/or modify it under the
+ * terms of the 3-clause BSD license.  You should have received a copy of the
+ * 3-clause BSD license along with mlpack.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef MLPACK_CORE_OPTIMIZERS_NADAM_NADAM_UPDATE_HPP
+#define MLPACK_CORE_OPTIMIZERS_NADAM_NADAM_UPDATE_HPP
+
+#include <mlpack/prereqs.hpp>
+
+namespace mlpack {
+namespace optimization {
+
+/**
+ * Nadam is an optimizer that combines the Adam and NAG.
+ *
+ * For more information, see the following.
+ *
+ * @code
+ * @article{
+ *   author  = {Sebastian Ruder},
+ *   title   = {An overview of gradient descent optimization algorithms},
+ *   journal = {CoRR},
+ *   year    = {2016},
+ *   url     = {https://arxiv.org/abs/1609.04747v2}
+ * }
+ * @endcode
+ */
+class NadamUpdate
+{
+ public:
+  /**
+   * Construct the Nadam update policy with the given parameters.
+   *
+   * @param epsilon The epsilon value used to initialise the squared gradient
+   *        parameter.
+   * @param beta1 The smoothing parameter.
+   */
+  NadamUpdate(const double epsilon = 1e-8,const double beta1 = 0.9) :epsilon(epsilon),beta1(beta1),iteration(0)
+  {
+    // Nothing to do.
+  }
+
+  /**
+   * The Initialize method is called by SGD Optimizer method before the start of
+   * the iteration update process.
+   *
+   * @param rows Number of rows in the gradient matrix.
+   * @param cols Number of columns in the gradient matrix.
+   */
+  void Initialize(const size_t rows, const size_t cols)
+  {
+    m = arma::zeros<arma::mat>(rows, cols);
+    v = arma::zeros<arma::mat>(rows, cols);
+  }
+
+  /**
+   * Update step for Nadam.
+   *
+   * @param iterate Parameters that minimize the function.
+   * @param stepSize Step size to be used for the given iteration.
+   * @param gradient The gradient matrix.
+   */
+  void Update(arma::mat& iterate,const double stepSize,const arma::mat& gradient)
+  {
+    // Increment the iteration counter variable.
+    ++iteration;
+
+    // And update the iterate.
+    m *= beta1;
+    m += (1 - beta1) * gradient;
+    //biasCorrection=1-beta1^iteration
+    const double biasCorrection = 1.0 - std::pow(beta1, iteration);
+    /*
+	iterate=iterate-((stepsize/(sqrt(v)+epsilon))*(m/biasCorrection))
+    */
+    iterate -= ((stepSize * m)/(biasCorrection1 * (arma::sqrt(v) + epsilon)));
+  }
+
+  //! Get the value used to initialise the squared gradient parameter.
+  double Epsilon() const { return epsilon; }
+  //! Modify the value used to initialise the squared gradient parameter.
+  double& Epsilon() { return epsilon; }
+
+  //! Get the smoothing parameter.
+  double Beta1() const { return beta1; }
+  //! Modify the smoothing parameter.
+  double& Beta1() { return beta1; }
+
+  private:
+  // The epsilon value used to initialise the squared gradient parameter.
+  double epsilon;
+
+  // The smoothing parameter.
+  double beta1;
+
+  // The exponential moving average of gradient values.
+  arma::mat m;
+
+  // The exponential moving average of squared gradient values.
+  arma::mat v;
+
+  // The number of iterations.
+  double iteration;
+};
+
+} // namespace optimization
+} // namespace mlpack
+
+#endif

From 3b41080fc65f044c590aaa5e301c27184afcfd26 Mon Sep 17 00:00:00 2001
From: Sourabh Varshney <sourabhvarshney111@gmail.com>
Date: Mon, 11 Dec 2017 19:45:18 +0530
Subject: [PATCH 02/67] Added tests for Nadam

---
 src/mlpack/tests/nadam_test.cpp | 108 ++++++++++++++++++++++++++++++++
 1 file changed, 108 insertions(+)
 create mode 100644 src/mlpack/tests/nadam_test.cpp

diff --git a/src/mlpack/tests/nadam_test.cpp b/src/mlpack/tests/nadam_test.cpp
new file mode 100644
index 0000000000..f21f3e7a3e
--- /dev/null
+++ b/src/mlpack/tests/nadam_test.cpp
@@ -0,0 +1,108 @@
+/**
+ * @file nadam_test.cpp
+ * @author Sourabh Varshney
+ *
+ * Tests the Nadam optimizer.
+ *
+ * mlpack is free software; you may redistribute it and/or modify it under the
+ * terms of the 3-clause BSD license.  You should have received a copy of the
+ * 3-clause BSD license along with mlpack.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#include <mlpack/core.hpp>
+
+#include <mlpack/core/optimizers/nadam/nadam.hpp>
+#include <mlpack/core/optimizers/sgd/test_function.hpp>
+#include <mlpack/methods/logistic_regression/logistic_regression.hpp>
+
+#include <boost/test/unit_test.hpp>
+#include "test_tools.hpp"
+
+using namespace arma;
+using namespace mlpack::optimization;
+using namespace mlpack::optimization::test;
+
+using namespace mlpack::distribution;
+using namespace mlpack::regression;
+
+using namespace mlpack;
+
+BOOST_AUTO_TEST_SUITE(NadamTest);
+
+/**
+ * Tests the Adam optimizer using a simple test function.
+ */
+BOOST_AUTO_TEST_CASE(SimpleNadamTestFunction)
+{
+  SGDTestFunction f;
+  Nadam optimizer(1e-3, 1, 0.9, 1e-8, 500000, 1e-9, true);
+
+  arma::mat coordinates = f.GetInitialPoint();
+  optimizer.Optimize(f, coordinates);
+
+  BOOST_REQUIRE_SMALL(coordinates[0], 0.1);
+  BOOST_REQUIRE_SMALL(coordinates[1], 0.1);
+  BOOST_REQUIRE_SMALL(coordinates[2], 0.1);
+}
+
+
+/**
+ * Run Nadam on logistic regression and make sure the results are acceptable.
+ */
+BOOST_AUTO_TEST_CASE(NadamLogisticRegressionTest)
+{
+  // Generate a two-Gaussian dataset.
+  GaussianDistribution g1(arma::vec("1.0 1.0 1.0"), arma::eye<arma::mat>(3, 3));
+  GaussianDistribution g2(arma::vec("9.0 9.0 9.0"), arma::eye<arma::mat>(3, 3));
+
+  arma::mat data(3, 1000);
+  arma::Row<size_t> responses(1000);
+  for (size_t i = 0; i < 500; ++i)
+  {
+    data.col(i) = g1.Random();
+    responses[i] = 0;
+  }
+  for (size_t i = 500; i < 1000; ++i)
+  {
+    data.col(i) = g2.Random();
+    responses[i] = 1;
+  }
+
+  // Shuffle the dataset.
+  arma::uvec indices = arma::shuffle(arma::linspace<arma::uvec>(0,data.n_cols - 1, data.n_cols));
+  arma::mat shuffledData(3, 1000);
+  arma::Row<size_t> shuffledResponses(1000);
+  for (size_t i = 0; i < data.n_cols; ++i)
+  {
+    shuffledData.col(i) = data.col(indices[i]);
+    shuffledResponses[i] = responses[indices[i]];
+  }
+
+  // Create a test set.
+  arma::mat testData(3, 1000);
+  arma::Row<size_t> testResponses(1000);
+  for (size_t i = 0; i < 500; ++i)
+  {
+    testData.col(i) = g1.Random();
+    testResponses[i] = 0;
+  }
+  for (size_t i = 500; i < 1000; ++i)
+  {
+    testData.col(i) = g2.Random();
+    testResponses[i] = 1;
+  }
+
+  Nadam nadam;
+  LogisticRegression<> lr(shuffledData, shuffledResponses, nadam, 0.5);
+
+  // Ensure that the error is close to zero.
+  const double acc = lr.ComputeAccuracy(data, responses);
+  BOOST_REQUIRE_CLOSE(acc, 100.0, 0.3); // 0.3% error tolerance.
+
+  const double testAcc = lr.ComputeAccuracy(testData, testResponses);
+  BOOST_REQUIRE_CLOSE(testAcc, 100.0, 0.6); // 0.6% error tolerance.
+}
+
+
+
+BOOST_AUTO_TEST_SUITE_END();

From 6eeced182c722148b130fd8fd8afb68fdeab78af Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Zolt=C3=A1n=20Mizsei?= <zmizsei@extrowerk.com>
Date: Tue, 12 Dec 2017 10:28:32 +0100
Subject: [PATCH 03/67] Haiku got no -lrt

---
 CMakeLists.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index ef9f9298ec..5d18bc4733 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -126,7 +126,7 @@ if(BUILD_WITH_COVERAGE)
 endif()
 
 # For clock_gettime().
-if (UNIX AND NOT APPLE)
+if (UNIX AND NOT APPLE AND NOT HAIKU)
   set(MLPACK_LIBRARIES ${MLPACK_LIBRARIES} "rt")
 endif ()
 

From 28e471299bae009574127c0703af6eeb8d3d7a6f Mon Sep 17 00:00:00 2001
From: Haritha <haritha1313@gmail.com>
Date: Wed, 13 Dec 2017 13:08:24 +0530
Subject: [PATCH 04/67] ams_grad optimizer implementation

---
 src/mlpack/core/optimizers/CMakeLists.txt     |   1 +
 .../core/optimizers/ams_grad/CMakeLists.txt   |  12 ++
 .../core/optimizers/ams_grad/ams_grad.cpp     |  37 +++++
 .../core/optimizers/ams_grad/ams_grad.hpp     | 157 ++++++++++++++++++
 .../optimizers/ams_grad/amsgrad_update.hpp    | 140 ++++++++++++++++
 src/mlpack/tests/CMakeLists.txt               |   1 +
 src/mlpack/tests/ams_grad_test.cpp            | 107 ++++++++++++
 7 files changed, 455 insertions(+)
 create mode 100644 src/mlpack/core/optimizers/ams_grad/CMakeLists.txt
 create mode 100644 src/mlpack/core/optimizers/ams_grad/ams_grad.cpp
 create mode 100644 src/mlpack/core/optimizers/ams_grad/ams_grad.hpp
 create mode 100644 src/mlpack/core/optimizers/ams_grad/amsgrad_update.hpp
 create mode 100644 src/mlpack/tests/ams_grad_test.cpp

diff --git a/src/mlpack/core/optimizers/CMakeLists.txt b/src/mlpack/core/optimizers/CMakeLists.txt
index 1dd8bb4e51..76493dc5a0 100644
--- a/src/mlpack/core/optimizers/CMakeLists.txt
+++ b/src/mlpack/core/optimizers/CMakeLists.txt
@@ -2,6 +2,7 @@ set(DIRS
   ada_delta
   ada_grad
   adam
+  ams_grad
   aug_lagrangian
   cmaes
   cne
diff --git a/src/mlpack/core/optimizers/ams_grad/CMakeLists.txt b/src/mlpack/core/optimizers/ams_grad/CMakeLists.txt
new file mode 100644
index 0000000000..f138890d5b
--- /dev/null
+++ b/src/mlpack/core/optimizers/ams_grad/CMakeLists.txt
@@ -0,0 +1,12 @@
+set(SOURCES
+  ams_grad.cpp
+  ams_grad.hpp
+  amsgrad_update.hpp
+)
+
+set(DIR_SRCS)
+foreach(file ${SOURCES})
+  set(DIR_SRCS ${DIR_SRCS} ${CMAKE_CURRENT_SOURCE_DIR}/${file})
+endforeach()
+
+set(MLPACK_SRCS ${MLPACK_SRCS} ${DIR_SRCS} PARENT_SCOPE)
diff --git a/src/mlpack/core/optimizers/ams_grad/ams_grad.cpp b/src/mlpack/core/optimizers/ams_grad/ams_grad.cpp
new file mode 100644
index 0000000000..dca009419e
--- /dev/null
+++ b/src/mlpack/core/optimizers/ams_grad/ams_grad.cpp
@@ -0,0 +1,37 @@
+/**
+ * @file ams_grad_impl.hpp
+ * @author Haritha Nair
+ *
+ * Implementation of the AMSGrad Optimizer.
+ *
+ * mlpack is free software; you may redistribute it and/or modify it under the
+ * terms of the 3-clause BSD license.  You should have received a copy of the
+ * 3-clause BSD license along with mlpack.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+// In case it hasn't been included yet.
+#include "ams_grad.hpp"
+
+namespace mlpack {
+namespace optimization {
+
+AmsGrad::AmsGrad(
+    const double stepSize,
+    const size_t batchSize,
+    const double beta1,
+    const double beta2,
+    const double epsilon,
+    const size_t maxIterations,
+    const double tolerance,
+    const bool shuffle) :
+    optimizer(stepSize,
+              batchSize,
+              maxIterations,
+              tolerance,
+              shuffle,
+              AmsGradUpdate(epsilon, beta1, beta2))
+{ /* Nothing to do. */ }
+
+} // namespace optimization
+} // namespace mlpack
+
diff --git a/src/mlpack/core/optimizers/ams_grad/ams_grad.hpp b/src/mlpack/core/optimizers/ams_grad/ams_grad.hpp
new file mode 100644
index 0000000000..76f096cc4d
--- /dev/null
+++ b/src/mlpack/core/optimizers/ams_grad/ams_grad.hpp
@@ -0,0 +1,157 @@
+/**
+ * @file ams_grad.hpp
+ * @author Haritha Nair
+ *
+ * Implementation of AMSGrad optimizer. AMSGrad is an exponential moving average 
+ * optimizer that dynamically adapts over time with guaranteed convergence.
+ *
+ * mlpack is free software; you may redistribute it and/or modify it under the
+ * terms of the 3-clause BSD license.  You should have received a copy of the
+ * 3-clause BSD license along with mlpack.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef MLPACK_CORE_OPTIMIZERS_AMS_GRAD_AMS_GRAD_HPP
+#define MLPACK_CORE_OPTIMIZERS_AMS_GRAD_AMS_GRAD_HPP
+
+#include <mlpack/prereqs.hpp>
+
+#include <mlpack/core/optimizers/sgd/sgd.hpp>
+#include "amsgrad_update.hpp"
+
+namespace mlpack {
+namespace optimization {
+
+/**
+ * AMSGrad is an exponential moving average variant which along with having 
+ * benefits of optimizers like Adam and RmsProp, also guarantees convergence.
+ * Unlike adam, it uses maximum of past squared gradients rather than their 
+ * exponential average for updation.
+ *
+ * For more information, see the following.
+ *
+ * @code
+ * @article{
+ *   title   = {On the convergence of Adam and beyond},
+ *   url     = {https://openreview.net/pdf?id=ryQu7f-RZ}
+ * }
+ * @endcode
+ *
+ *
+ * For AMSGrad to work, a DecomposableFunctionType template parameter
+ * is required. This class must implement the following function:
+ *
+ *   size_t NumFunctions();
+ *   double Evaluate(const arma::mat& coordinates,
+ *                   const size_t i,
+ *                   const size_t batchSize);
+ *   void Gradient(const arma::mat& coordinates,
+ *                 const size_t i,
+ *                 arma::mat& gradient,
+ *                 const size_t batchSize);
+ *
+ * NumFunctions() should return the number of functions (\f$n\f$), and in the
+ * other two functions, the parameter i refers to which individual function (or
+ * gradient) is being evaluated.  So, for the case of a data-dependent function,
+ * such as NCA (see mlpack::nca::NCA), NumFunctions() should return the number
+ * of points in the dataset, and Evaluate(coordinates, 0) will evaluate the
+ * objective function on the first point in the dataset (presumably, the dataset
+ * is held internally in the DecomposableFunctionType).
+ */
+
+class AmsGrad
+{
+ public:
+  /**
+   * Construct the AMSGrad optimizer with the given function and parameters. The
+   * defaults here are not necessarily good for the given problem, so it is
+   * suggested that the values used be tailored to the task at hand.  The
+   * maximum number of iterations refers to the maximum number of points that
+   * are processed (i.e., one iteration equals one point; one iteration does not
+   * equal one pass over the dataset).
+   *
+   * @param stepSize Step size for each iteration.
+   * @param batchSize Number of points to process in a single step.
+   * @param beta1 Exponential decay rate for the first moment estimates.
+   * @param beta2 Exponential decay rate for the weighted infinity norm
+            estimates.
+   * @param eps Value used to initialise the mean squared gradient parameter.
+   * @param maxIterations Maximum number of iterations allowed (0 means no
+   *        limit).
+   * @param tolerance Maximum absolute tolerance to terminate algorithm.
+   * @param shuffle If true, the function order is shuffled; otherwise, each
+   *        function is visited in linear order.
+   */
+  AmsGrad(const double stepSize = 0.001,
+           const size_t batchSize = 32,
+           const double beta1 = 0.9,
+           const double beta2 = 0.999,
+           const double eps = 1e-8,
+           const size_t maxIterations = 100000,
+           const double tolerance = 1e-5,
+           const bool shuffle = true);
+
+  /**
+   * Optimize the given function using AMSGrad. The given starting point will be
+   * modified to store the finishing point of the algorithm, and the final
+   * objective value is returned.
+   *
+   * @tparam DecomposableFunctionType Type of the function to optimize.
+   * @param function Function to optimize.
+   * @param iterate Starting point (will be modified).
+   * @return Objective value of the final point.
+   */
+  template<typename DecomposableFunctionType>
+  double Optimize(DecomposableFunctionType& function, arma::mat& iterate)
+  {
+    return optimizer.Optimize(function, iterate);
+  }
+
+  //! Get the step size.
+  double StepSize() const { return optimizer.StepSize(); }
+  //! Modify the step size.
+  double& StepSize() { return optimizer.StepSize(); }
+
+  //! Get the batch size.
+  size_t BatchSize() const { return optimizer.BatchSize(); }
+  //! Modify the batch size.
+  size_t& BatchSize() { return optimizer.BatchSize(); }
+
+  //! Get the smoothing parameter.
+  double Beta1() const { return optimizer.UpdatePolicy().Beta1(); }
+  //! Modify the smoothing parameter.
+  double& Beta1() { return optimizer.UpdatePolicy().Beta1(); }
+
+  //! Get the second moment coefficient.
+  double Beta2() const { return optimizer.UpdatePolicy().Beta2(); }
+  //! Modify the second moment coefficient.
+  double& Beta2() { return optimizer.UpdatePolicy().Beta2(); }
+
+  //! Get the value used to initialise the mean squared gradient parameter.
+  double Epsilon() const { return optimizer.UpdatePolicy().Epsilon(); }
+  //! Modify the value used to initialise the mean squared gradient parameter.
+  double& Epsilon() { return optimizer.UpdatePolicy().Epsilon(); }
+
+  //! Get the maximum number of iterations (0 indicates no limit).
+  size_t MaxIterations() const { return optimizer.MaxIterations(); }
+  //! Modify the maximum number of iterations (0 indicates no limit).
+  size_t& MaxIterations() { return optimizer.MaxIterations(); }
+
+  //! Get the tolerance for termination.
+  double Tolerance() const { return optimizer.Tolerance(); }
+  //! Modify the tolerance for termination.
+  double& Tolerance() { return optimizer.Tolerance(); }
+
+  //! Get whether or not the individual functions are shuffled.
+  bool Shuffle() const { return optimizer.Shuffle(); }
+  //! Modify whether or not the individual functions are shuffled.
+  bool& Shuffle() { return optimizer.Shuffle(); }
+
+ private:
+  //! The Stochastic Gradient Descent object with AMSGrad policy.
+  SGD<AmsGradUpdate> optimizer;
+};
+
+} // namespace optimization
+} // namespace mlpack
+
+#endif
diff --git a/src/mlpack/core/optimizers/ams_grad/amsgrad_update.hpp b/src/mlpack/core/optimizers/ams_grad/amsgrad_update.hpp
new file mode 100644
index 0000000000..82d94e7143
--- /dev/null
+++ b/src/mlpack/core/optimizers/ams_grad/amsgrad_update.hpp
@@ -0,0 +1,140 @@
+/**
+ * @file amsgrad_update.hpp
+ * @author Haritha Nair
+ *
+ * Implementation of AMSGrad optimizer. AMSGrad is an exponential moving average 
+ * optimizer that dynamically adapts over time with guaranteed convergence.
+ *
+ * mlpack is free software; you may redistribute it and/or modify it under the
+ * terms of the 3-clause BSD license.  You should have received a copy of the
+ * 3-clause BSD license along with mlpack.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef MLPACK_CORE_OPTIMIZERS_AMS_GRAD_AMS_GRAD_UPDATE_HPP
+#define MLPACK_CORE_OPTIMIZERS_AMS_GRAD_AMS_GRAD_UPDATE_HPP
+
+#include <mlpack/prereqs.hpp>
+
+namespace mlpack {
+namespace optimization {
+
+/**
+ * AMSGrad is an exponential moving average variant which along with having 
+ * benefits of optimizers like Adam and RmsProp, also guarantees convergence.
+ * Unlike adam, it uses maximum of past squared gradients rather than their 
+ * exponential average for updation.
+ *
+ * For more information, see the following.
+ *
+ * @code
+ * @article{
+ *   title   = {On the convergence of Adam and beyond},
+ *   url     = {https://openreview.net/pdf?id=ryQu7f-RZ}
+ * }
+ * @endcode
+ */
+class AmsGradUpdate
+{
+ public:
+  /**
+   * Construct the AMSGrad update policy with the given parameters.
+   *
+   * @param epsilon The epsilon value used to initialise the squared gradient
+   *        parameter.
+   * @param beta1 The smoothing parameter.
+   * @param beta2 The second moment coefficient.
+   */
+  AmsGradUpdate(const double epsilon = 1e-8,
+             const double beta1 = 0.9,
+             const double beta2 = 0.999) :
+    epsilon(epsilon),
+    beta1(beta1),
+    beta2(beta2),
+    iteration(0)
+  {
+    // Nothing to do.
+  }
+
+  /**
+   * The Initialize method is called by SGD Optimizer method before the start of
+   * the iteration update process.
+   *
+   * @param rows Number of rows in the gradient matrix.
+   * @param cols Number of columns in the gradient matrix.
+   */
+  void Initialize(const size_t rows, const size_t cols)
+  {
+    m = arma::zeros<arma::mat>(rows, cols);
+    v = arma::zeros<arma::mat>(rows, cols);
+    vImproved = arma::zeros<arma::mat>(rows, cols);
+  }
+
+  /**
+   * Update step for AMSGrad.
+   *
+   * @param iterate Parameters that minimize the function.
+   * @param stepSize Step size to be used for the given iteration.
+   * @param gradient The gradient matrix.
+   */
+  void Update(arma::mat& iterate,
+              const double stepSize,
+              const arma::mat& gradient)
+  {
+    // Increment the iteration counter variable.
+    ++iteration;
+
+    // And update the iterate.
+    m *= beta1;
+    m += (1 - beta1) * gradient;
+
+    v *= beta2;
+    v += (1 - beta2) * (gradient % gradient);
+
+    vImproved = arma::max(vImproved, v);
+    // Element wise maximum of past and present squared gradients.
+
+    iterate -= (stepSize * m) / (arma::sqrt(vImproved) + epsilon);
+  }
+
+  //! Get the value used to initialise the squared gradient parameter.
+  double Epsilon() const { return epsilon; }
+  //! Modify the value used to initialise the squared gradient parameter.
+  double& Epsilon() { return epsilon; }
+
+  //! Get the smoothing parameter.
+  double Beta1() const { return beta1; }
+  //! Modify the smoothing parameter.
+  double& Beta1() { return beta1; }
+
+  //! Get the second moment coefficient.
+  double Beta2() const { return beta2; }
+  //! Modify the second moment coefficient.
+  double& Beta2() { return beta2; }
+
+ private:
+  // The epsilon value used to initialise the squared gradient parameter.
+  double epsilon;
+
+  // The smoothing parameter.
+  double beta1;
+
+  // The second moment coefficient.
+  double beta2;
+
+  // The exponential moving average of gradient values.
+  arma::mat m;
+
+  // The exponential moving average of squared gradient values.
+  arma::mat v;
+
+  //The optimal sqaured gradient value.
+  arma::mat vImproved;
+
+  // The number of iterations.
+  double iteration;
+};
+
+} // namespace optimization
+} // namespace mlpack
+
+#endif
diff --git a/src/mlpack/tests/CMakeLists.txt b/src/mlpack/tests/CMakeLists.txt
index fc6ee57e48..5dbe1b096c 100644
--- a/src/mlpack/tests/CMakeLists.txt
+++ b/src/mlpack/tests/CMakeLists.txt
@@ -7,6 +7,7 @@ add_executable(mlpack_test
   ada_grad_test.cpp
   akfn_test.cpp
   aknn_test.cpp
+  ams_grad_test.cpp
   ann_layer_test.cpp
   armadillo_svd_test.cpp
   arma_extend_test.cpp
diff --git a/src/mlpack/tests/ams_grad_test.cpp b/src/mlpack/tests/ams_grad_test.cpp
new file mode 100644
index 0000000000..1a834b0e92
--- /dev/null
+++ b/src/mlpack/tests/ams_grad_test.cpp
@@ -0,0 +1,107 @@
+/**
+ * @file ams_grad_test.cpp
+ * @author Haritha Nair
+ *
+ * Tests the AMSGrad optimizer.
+ *
+ * mlpack is free software; you may redistribute it and/or modify it under the
+ * terms of the 3-clause BSD license.  You should have received a copy of the
+ * 3-clause BSD license along with mlpack.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#include <mlpack/core.hpp>
+
+#include <mlpack/core/optimizers/ams_grad/ams_grad.hpp>
+#include <mlpack/core/optimizers/sgd/test_function.hpp>
+#include <mlpack/methods/logistic_regression/logistic_regression.hpp>
+
+#include <boost/test/unit_test.hpp>
+#include "test_tools.hpp"
+
+using namespace arma;
+using namespace mlpack::optimization;
+using namespace mlpack::optimization::test;
+
+using namespace mlpack::distribution;
+using namespace mlpack::regression;
+
+using namespace mlpack;
+
+BOOST_AUTO_TEST_SUITE(AmsGradTest);
+
+/**
+ * Tests the AMSGrad optimizer using a simple test function.
+ */
+BOOST_AUTO_TEST_CASE(SimpleAmsGradTestFunction)
+{
+  SGDTestFunction f;
+  AmsGrad optimizer(1e-3, 1, 0.9, 0.999, 1e-8, 500000, 1e-9, true);
+
+  arma::mat coordinates = f.GetInitialPoint();
+  optimizer.Optimize(f, coordinates);
+
+  BOOST_REQUIRE_SMALL(coordinates[0], 0.1);
+  BOOST_REQUIRE_SMALL(coordinates[1], 0.1);
+  BOOST_REQUIRE_SMALL(coordinates[2], 0.1);
+}
+
+
+/**
+ * Run AMSGrad on logistic regression and make sure the results are acceptable.
+ */
+BOOST_AUTO_TEST_CASE(AmsGradLogisticRegressionTest)
+{
+  // Generate a two-Gaussian dataset.
+  GaussianDistribution g1(arma::vec("1.0 1.0 1.0"), arma::eye<arma::mat>(3, 3));
+  GaussianDistribution g2(arma::vec("9.0 9.0 9.0"), arma::eye<arma::mat>(3, 3));
+
+  arma::mat data(3, 1000);
+  arma::Row<size_t> responses(1000);
+  for (size_t i = 0; i < 500; ++i)
+  {
+    data.col(i) = g1.Random();
+    responses[i] = 0;
+  }
+  for (size_t i = 500; i < 1000; ++i)
+  {
+    data.col(i) = g2.Random();
+    responses[i] = 1;
+  }
+
+  // Shuffle the dataset.
+  arma::uvec indices = arma::shuffle(arma::linspace<arma::uvec>(0,
+      data.n_cols - 1, data.n_cols));
+  arma::mat shuffledData(3, 1000);
+  arma::Row<size_t> shuffledResponses(1000);
+  for (size_t i = 0; i < data.n_cols; ++i)
+  {
+    shuffledData.col(i) = data.col(indices[i]);
+    shuffledResponses[i] = responses[indices[i]];
+  }
+
+  // Create a test set.
+  arma::mat testData(3, 1000);
+  arma::Row<size_t> testResponses(1000);
+  for (size_t i = 0; i < 500; ++i)
+  {
+    testData.col(i) = g1.Random();
+    testResponses[i] = 0;
+  }
+  for (size_t i = 500; i < 1000; ++i)
+  {
+    testData.col(i) = g2.Random();
+    testResponses[i] = 1;
+  }
+
+  AmsGrad amsgrad;
+  LogisticRegression<> lr(shuffledData, shuffledResponses, amsgrad, 0.5);
+
+  // Ensure that the error is close to zero.
+  const double acc = lr.ComputeAccuracy(data, responses);
+  BOOST_REQUIRE_CLOSE(acc, 100.0, 0.3); // 0.3% error tolerance.
+
+  const double testAcc = lr.ComputeAccuracy(testData, testResponses);
+  BOOST_REQUIRE_CLOSE(testAcc, 100.0, 0.6); // 0.6% error tolerance.
+}
+
+BOOST_AUTO_TEST_SUITE_END();

From 3633cd3cacba366c49538a683e32c4cfeb9b86b0 Mon Sep 17 00:00:00 2001
From: Sourabh Varshney <sourabhvarshney111@gmail.com>
Date: Wed, 13 Dec 2017 16:06:38 +0530
Subject: [PATCH 05/67] Applied style improvements

---
 src/mlpack/core/optimizers/Nadam/nadam_update.hpp | 3 ++-
 src/mlpack/tests/nadam_test.cpp                   | 3 ++-
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/src/mlpack/core/optimizers/Nadam/nadam_update.hpp b/src/mlpack/core/optimizers/Nadam/nadam_update.hpp
index 6c3b3139ce..646bb161aa 100644
--- a/src/mlpack/core/optimizers/Nadam/nadam_update.hpp
+++ b/src/mlpack/core/optimizers/Nadam/nadam_update.hpp
@@ -43,7 +43,8 @@ class NadamUpdate
    *        parameter.
    * @param beta1 The smoothing parameter.
    */
-  NadamUpdate(const double epsilon = 1e-8,const double beta1 = 0.9) :epsilon(epsilon),beta1(beta1),iteration(0)
+  NadamUpdate(const double epsilon = 1e-8,const double beta1 = 0.9) 
+  :epsilon(epsilon),beta1(beta1),iteration(0)
   {
     // Nothing to do.
   }
diff --git a/src/mlpack/tests/nadam_test.cpp b/src/mlpack/tests/nadam_test.cpp
index f21f3e7a3e..47c5ec5ce4 100644
--- a/src/mlpack/tests/nadam_test.cpp
+++ b/src/mlpack/tests/nadam_test.cpp
@@ -69,7 +69,8 @@ BOOST_AUTO_TEST_CASE(NadamLogisticRegressionTest)
   }
 
   // Shuffle the dataset.
-  arma::uvec indices = arma::shuffle(arma::linspace<arma::uvec>(0,data.n_cols - 1, data.n_cols));
+  arma::uvec indices = arma::shuffle(arma::linspace<arma::uvec>(0,
+  data.n_cols - 1, data.n_cols));
   arma::mat shuffledData(3, 1000);
   arma::Row<size_t> shuffledResponses(1000);
   for (size_t i = 0; i < data.n_cols; ++i)

From fc45099367e685246b346eae7987f238fc0da3c2 Mon Sep 17 00:00:00 2001
From: Sourabh Varshney <sourabhvarshney111@gmail.com>
Date: Wed, 13 Dec 2017 16:19:25 +0530
Subject: [PATCH 06/67] Applied more style improvements

---
 src/mlpack/core/optimizers/Nadam/nadam.hpp      | 17 +++++++++--------
 .../core/optimizers/Nadam/nadam_update.hpp      |  9 +++++----
 src/mlpack/tests/nadam_test.cpp                 |  6 ++++--
 3 files changed, 18 insertions(+), 14 deletions(-)

diff --git a/src/mlpack/core/optimizers/Nadam/nadam.hpp b/src/mlpack/core/optimizers/Nadam/nadam.hpp
index fe300f9956..4e01c39305 100644
--- a/src/mlpack/core/optimizers/Nadam/nadam.hpp
+++ b/src/mlpack/core/optimizers/Nadam/nadam.hpp
@@ -2,7 +2,7 @@
  * @file nadam.hpp
  * @author Sourabh Varshney
  *
- * Nadam optimizer. Nadam is an optimizer that combines the effect of Adam and 
+ * Nadam optimizer. Nadam is an optimizer that combines the effect of Adam and
  * NAG to the gradient descent to improve its Performance.
  *
  * mlpack is free software; you may redistribute it and/or modify it under the
@@ -51,11 +51,12 @@ namespace optimization {
  *
  * NumFunctions() should return the number of functions (\f$n\f$), and in the
  * other two functions, the parameter i refers to which individual function (or
- * gradient) is being evaluated.  So, for the case of a data-dependent function,
- * such as NCA (see mlpack::nca::NCA), NumFunctions() should return the number
- * of points in the dataset, and Evaluate(coordinates, 0) will evaluate the
- * objective function on the first point in the dataset (presumably, the dataset
- * is held internally in the DecomposableFunctionType).
+ * gradient) is being evaluated.  So, for the case of a data-dependent 
+ * function, such as NCA (see mlpack::nca::NCA), NumFunctions() should return 
+ * the number of points in the dataset, and Evaluate(coordinates, 0) will 
+ * evaluate the objective function on the first point in the dataset 
+ * (presumably, the dataset is held internally in the 
+ * DecomposableFunctionType).
  *
  * @tparam UpdateRule Nadam optimizer update rule to be used.
  */
@@ -68,8 +69,8 @@ class NadamType
    * defaults here are not necessarily good for the given problem, so it is
    * suggested that the values used be tailored to the task at hand.  The
    * maximum number of iterations refers to the maximum number of points that
-   * are processed (i.e., one iteration equals one point; one iteration does not
-   * equal one pass over the dataset).
+   * are processed (i.e., one iteration equals one point; one iteration does
+   * not equal one pass over the dataset).
    *
    * @param stepSize Step size for each iteration.
    * @param batchSize Number of points to process in a single step.
diff --git a/src/mlpack/core/optimizers/Nadam/nadam_update.hpp b/src/mlpack/core/optimizers/Nadam/nadam_update.hpp
index 646bb161aa..bbd85aabb3 100644
--- a/src/mlpack/core/optimizers/Nadam/nadam_update.hpp
+++ b/src/mlpack/core/optimizers/Nadam/nadam_update.hpp
@@ -43,15 +43,15 @@ class NadamUpdate
    *        parameter.
    * @param beta1 The smoothing parameter.
    */
-  NadamUpdate(const double epsilon = 1e-8,const double beta1 = 0.9) 
+  NadamUpdate(const double epsilon = 1e-8,const double beta1 = 0.9)
   :epsilon(epsilon),beta1(beta1),iteration(0)
   {
     // Nothing to do.
   }
 
   /**
-   * The Initialize method is called by SGD Optimizer method before the start of
-   * the iteration update process.
+   * The Initialize method is called by SGD Optimizer method before the start 
+   * of the iteration update process.
    *
    * @param rows Number of rows in the gradient matrix.
    * @param cols Number of columns in the gradient matrix.
@@ -69,7 +69,8 @@ class NadamUpdate
    * @param stepSize Step size to be used for the given iteration.
    * @param gradient The gradient matrix.
    */
-  void Update(arma::mat& iterate,const double stepSize,const arma::mat& gradient)
+  void Update(arma::mat& iterate, const double stepSize,
+  const arma::mat& gradient)
   {
     // Increment the iteration counter variable.
     ++iteration;
diff --git a/src/mlpack/tests/nadam_test.cpp b/src/mlpack/tests/nadam_test.cpp
index 47c5ec5ce4..97f53c1f12 100644
--- a/src/mlpack/tests/nadam_test.cpp
+++ b/src/mlpack/tests/nadam_test.cpp
@@ -52,8 +52,10 @@ BOOST_AUTO_TEST_CASE(SimpleNadamTestFunction)
 BOOST_AUTO_TEST_CASE(NadamLogisticRegressionTest)
 {
   // Generate a two-Gaussian dataset.
-  GaussianDistribution g1(arma::vec("1.0 1.0 1.0"), arma::eye<arma::mat>(3, 3));
-  GaussianDistribution g2(arma::vec("9.0 9.0 9.0"), arma::eye<arma::mat>(3, 3));
+  GaussianDistribution g1(arma::vec("1.0 1.0 1.0"),
+  arma::eye<arma::mat>(3, 3));
+  GaussianDistribution g2(arma::vec("9.0 9.0 9.0"),
+  arma::eye<arma::mat>(3, 3));
 
   arma::mat data(3, 1000);
   arma::Row<size_t> responses(1000);

From 948516ef8e3b140f252398c8073dafa0be236006 Mon Sep 17 00:00:00 2001
From: Sourabh Varshney <sourabhvarshney111@gmail.com>
Date: Wed, 13 Dec 2017 16:40:00 +0530
Subject: [PATCH 07/67] Applied more style changes

---
 src/mlpack/core/optimizers/Nadam/nadam_update.hpp | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/mlpack/core/optimizers/Nadam/nadam_update.hpp b/src/mlpack/core/optimizers/Nadam/nadam_update.hpp
index bbd85aabb3..c5ad289b3a 100644
--- a/src/mlpack/core/optimizers/Nadam/nadam_update.hpp
+++ b/src/mlpack/core/optimizers/Nadam/nadam_update.hpp
@@ -43,8 +43,8 @@ class NadamUpdate
    *        parameter.
    * @param beta1 The smoothing parameter.
    */
-  NadamUpdate(const double epsilon = 1e-8,const double beta1 = 0.9)
-  :epsilon(epsilon),beta1(beta1),iteration(0)
+  NadamUpdate(const double epsilon = 1e-8, const double beta1 = 0.9)
+  :epsilon(epsilon), beta1(beta1), iteration(0)
   {
     // Nothing to do.
   }
@@ -78,7 +78,7 @@ class NadamUpdate
     // And update the iterate.
     m *= beta1;
     m += (1 - beta1) * gradient;
-    //biasCorrection=1-beta1^iteration
+    // biasCorrection=1-beta1^iteration
     const double biasCorrection = 1.0 - std::pow(beta1, iteration);
     /*
 	iterate=iterate-((stepsize/(sqrt(v)+epsilon))*(m/biasCorrection))
@@ -96,7 +96,7 @@ class NadamUpdate
   //! Modify the smoothing parameter.
   double& Beta1() { return beta1; }
 
-  private:
+ private:
   // The epsilon value used to initialise the squared gradient parameter.
   double epsilon;
 

From c8a97fb0186419a14c20cdad593cc0acc293ae9f Mon Sep 17 00:00:00 2001
From: Ryan Curtin <ryan@ratml.org>
Date: Wed, 13 Dec 2017 10:33:34 -0500
Subject: [PATCH 08/67] Another attempt to fix when apt refuses to work on
 Travis.

---
 .travis.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index e1d1be604c..35dc821267 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -9,10 +9,10 @@ env:
 before_install:
   - sudo systemctl stop apt-daily.service apt-daily.timer
   - sudo systemctl disable apt-daily.service apt-daily.timer
-  - sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test
-  - sudo apt-get update -qq
+  - sudo rm -f /var/lib/dpkg/lock && add-apt-repository -y ppa:ubuntu-toolchain-r/test
+  - sudo rm -f /var/lib/dpkg/lock && apt-get update -qq
   - printenv
-  - sudo apt-get install -qq libopenblas-dev liblapack-dev g++ libboost-all-dev
+  - sudo rm -f /var/lib/dpkg/lock && apt-get install -qq libopenblas-dev liblapack-dev g++ libboost-all-dev
   - sudo wget https://launchpadlibrarian.net/260609315/fix-std-vector-load.diff -O /usr/include/fix-std-vector-load.diff
   - cd /usr/include && sudo patch -p1 < fix-std-vector-load.diff && sudo rm fix-std-vector-load.diff && cd -
   - sudo pip install cython setuptools numpy pandas

From b86f7d1fcf4d272768e0b3c52ac146ecf96318e2 Mon Sep 17 00:00:00 2001
From: Haritha <haritha1313@gmail.com>
Date: Wed, 13 Dec 2017 21:50:07 +0530
Subject: [PATCH 09/67] AMSGrad optimizer

---
 src/mlpack/core/optimizers/ams_grad/ams_grad.hpp       | 2 +-
 src/mlpack/core/optimizers/ams_grad/amsgrad_update.hpp | 6 +++---
 src/mlpack/tests/ams_grad_test.cpp                     | 2 +-
 3 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/src/mlpack/core/optimizers/ams_grad/ams_grad.hpp b/src/mlpack/core/optimizers/ams_grad/ams_grad.hpp
index 76f096cc4d..a8f543ca52 100644
--- a/src/mlpack/core/optimizers/ams_grad/ams_grad.hpp
+++ b/src/mlpack/core/optimizers/ams_grad/ams_grad.hpp
@@ -87,7 +87,7 @@ class AmsGrad
            const double beta2 = 0.999,
            const double eps = 1e-8,
            const size_t maxIterations = 100000,
-           const double tolerance = 1e-5,
+           const double tolerance = 1e-11,
            const bool shuffle = true);
 
   /**
diff --git a/src/mlpack/core/optimizers/ams_grad/amsgrad_update.hpp b/src/mlpack/core/optimizers/ams_grad/amsgrad_update.hpp
index 82d94e7143..b12525daab 100644
--- a/src/mlpack/core/optimizers/ams_grad/amsgrad_update.hpp
+++ b/src/mlpack/core/optimizers/ams_grad/amsgrad_update.hpp
@@ -90,9 +90,9 @@ class AmsGradUpdate
     v *= beta2;
     v += (1 - beta2) * (gradient % gradient);
 
-    vImproved = arma::max(vImproved, v);
     // Element wise maximum of past and present squared gradients.
-
+    vImproved = arma::max(vImproved, v);
+    
     iterate -= (stepSize * m) / (arma::sqrt(vImproved) + epsilon);
   }
 
@@ -127,7 +127,7 @@ class AmsGradUpdate
   // The exponential moving average of squared gradient values.
   arma::mat v;
 
-  //The optimal sqaured gradient value.
+  // The optimal sqaured gradient value.
   arma::mat vImproved;
 
   // The number of iterations.
diff --git a/src/mlpack/tests/ams_grad_test.cpp b/src/mlpack/tests/ams_grad_test.cpp
index 1a834b0e92..ecfe9a4bbd 100644
--- a/src/mlpack/tests/ams_grad_test.cpp
+++ b/src/mlpack/tests/ams_grad_test.cpp
@@ -35,7 +35,7 @@ BOOST_AUTO_TEST_SUITE(AmsGradTest);
 BOOST_AUTO_TEST_CASE(SimpleAmsGradTestFunction)
 {
   SGDTestFunction f;
-  AmsGrad optimizer(1e-3, 1, 0.9, 0.999, 1e-8, 500000, 1e-9, true);
+  AmsGrad optimizer(1e-3, 1, 0.9, 0.999, 1e-8, 500000, 1e-11, true);
 
   arma::mat coordinates = f.GetInitialPoint();
   optimizer.Optimize(f, coordinates);

From ebd0cb3f301822efaf23217399ce2fc654c18bf8 Mon Sep 17 00:00:00 2001
From: Haritha <haritha1313@gmail.com>
Date: Wed, 13 Dec 2017 21:53:20 +0530
Subject: [PATCH 10/67] AMSGrad optimizer

---
 src/mlpack/core/optimizers/ams_grad/amsgrad_update.hpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/mlpack/core/optimizers/ams_grad/amsgrad_update.hpp b/src/mlpack/core/optimizers/ams_grad/amsgrad_update.hpp
index b12525daab..321e85665e 100644
--- a/src/mlpack/core/optimizers/ams_grad/amsgrad_update.hpp
+++ b/src/mlpack/core/optimizers/ams_grad/amsgrad_update.hpp
@@ -92,7 +92,7 @@ class AmsGradUpdate
 
     // Element wise maximum of past and present squared gradients.
     vImproved = arma::max(vImproved, v);
-    
+
     iterate -= (stepSize * m) / (arma::sqrt(vImproved) + epsilon);
   }
 

From 8ca93e49d00cf9b1b19991cb421eda7d931d888a Mon Sep 17 00:00:00 2001
From: Ryan Curtin <ryan@ratml.org>
Date: Wed, 13 Dec 2017 11:23:37 -0500
Subject: [PATCH 11/67] && needs another sudo. :)

---
 .travis.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 35dc821267..85e8c80a21 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -9,10 +9,10 @@ env:
 before_install:
   - sudo systemctl stop apt-daily.service apt-daily.timer
   - sudo systemctl disable apt-daily.service apt-daily.timer
-  - sudo rm -f /var/lib/dpkg/lock && add-apt-repository -y ppa:ubuntu-toolchain-r/test
-  - sudo rm -f /var/lib/dpkg/lock && apt-get update -qq
+  - sudo rm -f /var/lib/dpkg/lock && sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test
+  - sudo rm -f /var/lib/dpkg/lock && sudo apt-get update -qq
   - printenv
-  - sudo rm -f /var/lib/dpkg/lock && apt-get install -qq libopenblas-dev liblapack-dev g++ libboost-all-dev
+  - sudo rm -f /var/lib/dpkg/lock && sudo apt-get install -qq libopenblas-dev liblapack-dev g++ libboost-all-dev
   - sudo wget https://launchpadlibrarian.net/260609315/fix-std-vector-load.diff -O /usr/include/fix-std-vector-load.diff
   - cd /usr/include && sudo patch -p1 < fix-std-vector-load.diff && sudo rm fix-std-vector-load.diff && cd -
   - sudo pip install cython setuptools numpy pandas

From 2290fe2be7a330fc4ba22144f0fa037504fa6747 Mon Sep 17 00:00:00 2001
From: Ryan Curtin <ryan@ratml.org>
Date: Wed, 13 Dec 2017 16:26:54 -0500
Subject: [PATCH 12/67] Fix recurrent layers' compatibility with batch sizes >
 2.

Also add tests.
---
 .../methods/ann/layer/fast_lstm_impl.hpp      |  8 +-
 src/mlpack/methods/ann/layer/gru_impl.hpp     | 59 +++++++++++-
 src/mlpack/methods/ann/layer/lstm_impl.hpp    | 18 ++--
 src/mlpack/tests/recurrent_network_test.cpp   | 91 ++++++++++++++++++-
 4 files changed, 158 insertions(+), 18 deletions(-)

diff --git a/src/mlpack/methods/ann/layer/fast_lstm_impl.hpp b/src/mlpack/methods/ann/layer/fast_lstm_impl.hpp
index 3222275a4d..349ece44e5 100644
--- a/src/mlpack/methods/ann/layer/fast_lstm_impl.hpp
+++ b/src/mlpack/methods/ann/layer/fast_lstm_impl.hpp
@@ -197,7 +197,7 @@ void FastLSTM<InputDataType, OutputDataType>::Backward(
       backwardStep - batchStep, 3 * outSize - 1, backwardStep) %
       cellActivationError;
 
-  if (backwardStep != 0)
+  if (backwardStep > batchStep)
   {
     prevError.submat(2 * outSize, 0, 3 * outSize - 1, batchStep) =
         cell.cols((backwardStep - batchSize) - batchStep,
@@ -258,13 +258,13 @@ void FastLSTM<InputDataType, OutputDataType>::Gradient(
       gradient.n_elem - 1, 0) = arma::vectorise(prevError *
       outParameter.cols(gradientStep - batchStep, gradientStep).t());
 
-  if (gradientStep == 0)
+  if (gradientStep > batchStep)
   {
-    gradientStep = batchSize * bpttSteps - 1;
+    gradientStep -= batchSize;
   }
   else
   {
-    gradientStep -= batchSize;
+    gradientStep = batchSize * bpttSteps - 1;
   }
 }
 
diff --git a/src/mlpack/methods/ann/layer/gru_impl.hpp b/src/mlpack/methods/ann/layer/gru_impl.hpp
index c795cef406..07a09b6584 100644
--- a/src/mlpack/methods/ann/layer/gru_impl.hpp
+++ b/src/mlpack/methods/ann/layer/gru_impl.hpp
@@ -69,7 +69,7 @@ GRU<InputDataType, OutputDataType>::GRU(
   allZeros = arma::zeros<arma::mat>(outSize, batchSize);
 
   outParameter.push_back(std::move(arma::mat(allZeros.memptr(),
-    allZeros.n_rows, allZeros.n_cols, false, true)));
+      allZeros.n_rows, allZeros.n_cols, false, true)));
 
   prevOutput = outParameter.begin();
   backIterator = outParameter.end();
@@ -92,10 +92,27 @@ template<typename eT>
 void GRU<InputDataType, OutputDataType>::Forward(
     arma::Mat<eT>&& input, arma::Mat<eT>&& output)
 {
+  std::cout << "GRU::Forward(): input " << input.n_rows << "x" << input.n_cols
+<< "; batchSize " << batchSize << "; forwardStep " << forwardStep << "\n";
   if (input.n_cols != batchSize)
   {
     batchSize = input.n_cols;
     prevError.resize(3 * outSize, batchSize);
+    allZeros.zeros(outSize, batchSize);
+    // Batch size better not change during an iteration...
+    if (outParameter.size() > 1)
+    {
+      Log::Fatal << "GRU<>::Forward(): batch size cannot change during a "
+          << "forward pass!" << std::endl;
+    }
+
+    outParameter.clear();
+    outParameter.push_back(std::move(arma::mat(allZeros.memptr(),
+        allZeros.n_rows, allZeros.n_cols, false, true)));
+
+    prevOutput = outParameter.begin();
+    backIterator = outParameter.end();
+    gradIterator = outParameter.end();
   }
 
   // Process the input linearly(zt, rt, ot).
@@ -157,13 +174,13 @@ void GRU<InputDataType, OutputDataType>::Forward(
     if (!deterministic)
     {
       outParameter.push_back(std::move(arma::mat(allZeros.memptr(),
-        allZeros.n_rows, allZeros.n_cols, false, true)));
+          allZeros.n_rows, allZeros.n_cols, false, true)));
       prevOutput = --outParameter.end();
     }
     else
     {
       *prevOutput = std::move(arma::mat(allZeros.memptr(),
-        allZeros.n_rows, allZeros.n_cols, false, true));
+          allZeros.n_rows, allZeros.n_cols, false, true));
     }
   }
   else if (!deterministic)
@@ -192,10 +209,27 @@ template<typename eT>
 void GRU<InputDataType, OutputDataType>::Backward(
   const arma::Mat<eT>&& input, arma::Mat<eT>&& gy, arma::Mat<eT>&& g)
 {
+  std::cout << "GRU::Backward(): input " << input.n_rows << "x" << input.n_cols
+<< "; batchSize " << batchSize << "; backwardStep " << backwardStep << "\n";
   if (input.n_cols != batchSize)
   {
     batchSize = input.n_cols;
     prevError.resize(3 * outSize, batchSize);
+    allZeros.zeros(outSize, batchSize);
+    // Batch size better not change during an iteration...
+    if (outParameter.size() > 1)
+    {
+      Log::Fatal << "GRU<>::Forward(): batch size cannot change during a "
+          << "forward pass!" << std::endl;
+    }
+
+    outParameter.clear();
+    outParameter.push_back(std::move(arma::mat(allZeros.memptr(),
+        allZeros.n_rows, allZeros.n_cols, false, true)));
+
+    prevOutput = outParameter.begin();
+    backIterator = outParameter.end();
+    gradIterator = outParameter.end();
   }
 
   if ((outParameter.size() - backwardStep  - 1) % rho != 0 && backwardStep != 0)
@@ -214,7 +248,7 @@ void GRU<InputDataType, OutputDataType>::Backward(
       hiddenStateModule));
 
   // Delta ot.
-  arma::mat dOt = gy % (arma::ones<arma::vec>(outSize) -
+  arma::mat dOt = gy % (arma::ones<arma::mat>(outSize, batchSize) -
       boost::apply_visitor(outputParameterVisitor, inputGateModule));
 
   // Delta of input gate.
@@ -293,10 +327,27 @@ void GRU<InputDataType, OutputDataType>::Gradient(
     arma::Mat<eT>&& /* error */,
     arma::Mat<eT>&& /* gradient */)
 {
+  std::cout << "GRU::Gradient(): input " << input.n_rows << "x" << input.n_cols
+<< "; batchSize " << batchSize << "; gradientStep " << gradientStep << "\n";
   if (input.n_cols != batchSize)
   {
     batchSize = input.n_cols;
     prevError.resize(3 * outSize, batchSize);
+    allZeros.zeros(outSize, batchSize);
+    // Batch size better not change during an iteration...
+    if (outParameter.size() > 1)
+    {
+      Log::Fatal << "GRU<>::Forward(): batch size cannot change during a "
+          << "forward pass!" << std::endl;
+    }
+
+    outParameter.clear();
+    outParameter.push_back(std::move(arma::mat(allZeros.memptr(),
+        allZeros.n_rows, allZeros.n_cols, false, true)));
+
+    prevOutput = outParameter.begin();
+    backIterator = outParameter.end();
+    gradIterator = outParameter.end();
   }
 
   if (gradIterator == outParameter.end())
diff --git a/src/mlpack/methods/ann/layer/lstm_impl.hpp b/src/mlpack/methods/ann/layer/lstm_impl.hpp
index 5abf58be5d..d14082597f 100644
--- a/src/mlpack/methods/ann/layer/lstm_impl.hpp
+++ b/src/mlpack/methods/ann/layer/lstm_impl.hpp
@@ -188,11 +188,11 @@ void LSTM<InputDataType, OutputDataType>::Forward(
   if (forwardStep > 0)
   {
     inputGate.cols(forwardStep, forwardStep + batchStep) +=
-        cell2GateInputWeight % cell.cols(forwardStep - batchSize,
+        arma::repmat(cell2GateInputWeight, 1, batchSize) % cell.cols(forwardStep - batchSize,
         forwardStep - batchSize + batchStep);
 
     forgetGate.cols(forwardStep, forwardStep + batchStep) +=
-        cell2GateForgetWeight % cell.cols(forwardStep - batchSize,
+        arma::repmat(cell2GateForgetWeight, 1, batchSize) % cell.cols(forwardStep - batchSize,
         forwardStep - batchSize + batchStep);
   }
 
@@ -283,7 +283,7 @@ void LSTM<InputDataType, OutputDataType>::Backward(
     cellError += inputCellError;
   }
 
-  if (backwardStep != 0)
+  if (backwardStep > batchStep)
   {
     forgetGateError = cell.cols((backwardStep - batchSize) - batchStep,
       (backwardStep - batchSize)) % cellError % (forgetGateActivation.cols(
@@ -305,7 +305,7 @@ void LSTM<InputDataType, OutputDataType>::Backward(
       backwardStep - batchStep, backwardStep), 2));
 
   inputCellError = forgetGateActivation.cols(backwardStep - batchStep,
-      backwardStep) %cellError + forgetGateError.each_col() %
+      backwardStep) % cellError + forgetGateError.each_col() %
       cell2GateForgetWeight + inputGateError.each_col() % cell2GateInputWeight;
 
   g = input2GateInputWeight.t() * inputGateError +
@@ -395,15 +395,15 @@ void LSTM<InputDataType, OutputDataType>::Gradient(
   offset += cell2GateOutputWeight.n_elem;
 
   // Cell2GateForgetWeight and cell2GateInputWeight gradients.
-  if (gradientStep != 0)
+  if (gradientStep > batchStep)
   {
     gradient.submat(offset, 0, offset + cell2GateForgetWeight.n_elem - 1, 0) =
-        arma::sum(forgetGateError % cell.cols(gradientStep - batchStep -
-        batchSize, gradientStep - batchSize), 1);
+        arma::sum(forgetGateError % cell.cols((gradientStep - batchSize) - batchStep,
+        (gradientStep - batchSize)), 1);
     gradient.submat(offset + cell2GateForgetWeight.n_elem, 0, offset +
         cell2GateForgetWeight.n_elem + cell2GateInputWeight.n_elem - 1, 0) =
-        arma::sum(inputGateError % cell.cols(gradientStep - batchStep -
-        batchSize, gradientStep - batchSize), 1);
+        arma::sum(inputGateError % cell.cols((gradientStep - batchSize) - batchStep,
+        (gradientStep - batchSize)), 1);
   }
   else
   {
diff --git a/src/mlpack/tests/recurrent_network_test.cpp b/src/mlpack/tests/recurrent_network_test.cpp
index 38857ef9ef..600f2544d2 100644
--- a/src/mlpack/tests/recurrent_network_test.cpp
+++ b/src/mlpack/tests/recurrent_network_test.cpp
@@ -287,7 +287,8 @@ void GenerateNextReber(const arma::Mat<char>& transitions,
  * @param nextReber All reachable next symbols.
  */
 void GenerateNextRecursiveReber(const arma::Mat<char>& transitions,
-                               const std::string& reber, std::string& nextReber)
+                                const std::string& reber,
+                                std::string& nextReber)
 {
   size_t state = 0;
   size_t numPs = 0;
@@ -743,6 +744,94 @@ BOOST_AUTO_TEST_CASE(GRUDistractedSequenceRecallTest)
   DistractedSequenceRecallTestNetwork<GRU<> >(4, 8);
 }
 
+/**
+ * Create a simple recurrent neural network for the noisy sines task, and ensure
+ * that it achieves adequate performance when training with the specified batch
+ * size.
+ */
+template<typename RecurrentLayerType>
+void BatchSizeTest(const size_t batchSize)
+{
+  const size_t rho = 10;
+
+  // Generate 12 (2 * 6) noisy sines. A single sine contains rho
+  // points/features.
+  arma::mat input, labelsTemp;
+  GenerateNoisySines(input, labelsTemp, rho, 6);
+
+  arma::mat labels = arma::zeros<arma::mat>(rho, labelsTemp.n_cols);
+  for (size_t i = 0; i < labelsTemp.n_cols; ++i)
+  {
+    const int value = arma::as_scalar(arma::find(
+        arma::max(labelsTemp.col(i)) == labelsTemp.col(i), 1)) + 1;
+    labels.col(i).fill(value);
+  }
+
+  RNN<> model(rho);
+  model.Add<Linear<>>(1, 10);
+  model.Add<SigmoidLayer<>>();
+  model.Add<RecurrentLayerType>(10, 10);
+  model.Add<SigmoidLayer<>>();
+  model.Add<Linear<>>(10, 10);
+  model.Add<SigmoidLayer<>>();
+
+  StandardSGD opt(0.1, batchSize, 500 * input.n_cols, -100);
+  model.Train(input, labels, opt);
+
+  arma::mat prediction;
+  model.Predict(input, prediction);
+
+  size_t error = 0;
+  for (size_t i = 0; i < prediction.n_cols; ++i)
+  {
+    arma::mat singlePrediction = prediction.submat((rho - 1) * rho, i,
+        rho * rho - 1, i);
+
+    const int predictionValue = arma::as_scalar(arma::find(
+        arma::max(singlePrediction.col(0)) ==
+        singlePrediction.col(0), 1) + 1);
+
+    const int targetValue = arma::as_scalar(arma::find(
+        arma::max(labelsTemp.col(i)) == labelsTemp.col(i), 1)) + 1;
+
+    if (predictionValue == targetValue)
+    {
+      error++;
+    }
+  }
+
+  double classificationError = 1 - double(error) / prediction.n_cols;
+
+  BOOST_REQUIRE_LE(classificationError, 0.2);
+}
+
+/**
+ * Ensure LSTMs work with larger batch sizes.
+ */
+BOOST_AUTO_TEST_CASE(LSTMBatchSizeTest)
+{
+  BatchSizeTest<LSTM<>>(2);
+  BatchSizeTest<LSTM<>>(50);
+}
+
+/**
+ * Ensure fast LSTMs work with larger batch sizes.
+ */
+BOOST_AUTO_TEST_CASE(FastLSTMBatchSizeTest)
+{
+  BatchSizeTest<FastLSTM<>>(2);
+  BatchSizeTest<FastLSTM<>>(50);
+}
+
+/**
+ * Ensure GRUs work with larger batch sizes.
+ */
+BOOST_AUTO_TEST_CASE(GRUBatchSizeTest)
+{
+  BatchSizeTest<GRU<>>(2);
+  BatchSizeTest<GRU<>>(50);
+}
+
 /**
  * Make sure the RNN can be properly serialized.
  */

From d267dd45f9eb03790f6589df8816b8bb718c7d0f Mon Sep 17 00:00:00 2001
From: Haritha <haritha1313@gmail.com>
Date: Thu, 14 Dec 2017 13:49:09 +0530
Subject: [PATCH 13/67] ams_grad optimizer

---
 src/mlpack/core/optimizers/CMakeLists.txt     |   1 -
 .../core/optimizers/adam/CMakeLists.txt       |   1 +
 src/mlpack/core/optimizers/adam/adam.hpp      |  13 +-
 src/mlpack/core/optimizers/adam/adam_impl.hpp |   2 +-
 .../{ams_grad => adam}/amsgrad_update.hpp     |  10 +-
 .../core/optimizers/ams_grad/CMakeLists.txt   |  12 --
 .../core/optimizers/ams_grad/ams_grad.cpp     |  37 -----
 .../core/optimizers/ams_grad/ams_grad.hpp     | 157 ------------------
 src/mlpack/tests/CMakeLists.txt               |   1 -
 src/mlpack/tests/adam_test.cpp                |  76 ++++++++-
 src/mlpack/tests/ams_grad_test.cpp            | 107 ------------
 11 files changed, 94 insertions(+), 323 deletions(-)
 rename src/mlpack/core/optimizers/{ams_grad => adam}/amsgrad_update.hpp (91%)
 delete mode 100644 src/mlpack/core/optimizers/ams_grad/CMakeLists.txt
 delete mode 100644 src/mlpack/core/optimizers/ams_grad/ams_grad.cpp
 delete mode 100644 src/mlpack/core/optimizers/ams_grad/ams_grad.hpp
 delete mode 100644 src/mlpack/tests/ams_grad_test.cpp

diff --git a/src/mlpack/core/optimizers/CMakeLists.txt b/src/mlpack/core/optimizers/CMakeLists.txt
index 76493dc5a0..1dd8bb4e51 100644
--- a/src/mlpack/core/optimizers/CMakeLists.txt
+++ b/src/mlpack/core/optimizers/CMakeLists.txt
@@ -2,7 +2,6 @@ set(DIRS
   ada_delta
   ada_grad
   adam
-  ams_grad
   aug_lagrangian
   cmaes
   cne
diff --git a/src/mlpack/core/optimizers/adam/CMakeLists.txt b/src/mlpack/core/optimizers/adam/CMakeLists.txt
index 1377bbb0e1..644797090f 100644
--- a/src/mlpack/core/optimizers/adam/CMakeLists.txt
+++ b/src/mlpack/core/optimizers/adam/CMakeLists.txt
@@ -3,6 +3,7 @@ set(SOURCES
   adam_impl.hpp
   adam_update.hpp
   adamax_update.hpp
+  amsgrad_update.hpp
 )
 
 set(DIR_SRCS)
diff --git a/src/mlpack/core/optimizers/adam/adam.hpp b/src/mlpack/core/optimizers/adam/adam.hpp
index 3b109660f5..a1ec694543 100644
--- a/src/mlpack/core/optimizers/adam/adam.hpp
+++ b/src/mlpack/core/optimizers/adam/adam.hpp
@@ -5,10 +5,10 @@
  * @author Marcus Edel
  * @author Vivek Pal
  *
- * Adam and AdaMax optimizer. Adam is an an algorithm for first-order gradient-
+ * Adam, AdaMax and AMSGrad optimizer. Adam is an an algorithm for first-order gradient-
  * -based optimization of stochastic objective functions, based on adaptive
  * estimates of lower-order moments. AdaMax is simply a variant of Adam based
- * on the infinity norm.
+ * on the infinity norm. AMSGrad is another variant of Adam with guaranteed convergence.
  *
  * mlpack is free software; you may redistribute it and/or modify it under the
  * terms of the 3-clause BSD license.  You should have received a copy of the
@@ -23,6 +23,7 @@
 #include <mlpack/core/optimizers/sgd/sgd.hpp>
 #include "adam_update.hpp"
 #include "adamax_update.hpp"
+#include "amsgrad_update.hpp"
 
 namespace mlpack {
 namespace optimization {
@@ -43,10 +44,14 @@ namespace optimization {
  *   year    = {2014},
  *   url     = {http://arxiv.org/abs/1412.6980}
  * }
+ * @article{
+ *   title   = {On the convergence of Adam and beyond},
+ *   url     = {https://openreview.net/pdf?id=ryQu7f-RZ}
+ * }
  * @endcode
  *
  *
- * For Adam and AdaMax to work, a DecomposableFunctionType template parameter
+ * For Adam, AdaMax and AMSGrad to work, a DecomposableFunctionType template parameter
  * is required. This class must implement the following function:
  *
  *   size_t NumFunctions();
@@ -166,6 +171,8 @@ using Adam = AdamType<AdamUpdate>;
 
 using AdaMax = AdamType<AdaMaxUpdate>;
 
+using AmsGrad = AdamType<AmsGradUpdate>;
+
 } // namespace optimization
 } // namespace mlpack
 
diff --git a/src/mlpack/core/optimizers/adam/adam_impl.hpp b/src/mlpack/core/optimizers/adam/adam_impl.hpp
index 326d038d15..49497f5af5 100644
--- a/src/mlpack/core/optimizers/adam/adam_impl.hpp
+++ b/src/mlpack/core/optimizers/adam/adam_impl.hpp
@@ -5,7 +5,7 @@
  * @author Marcus Edel
  * @author Vivek Pal
  *
- * Implementation of the Adam and AdaMax optimizer.
+ * Implementation of the Adam, AdaMax and AMSGrad optimizer.
  *
  * mlpack is free software; you may redistribute it and/or modify it under the
  * terms of the 3-clause BSD license.  You should have received a copy of the
diff --git a/src/mlpack/core/optimizers/ams_grad/amsgrad_update.hpp b/src/mlpack/core/optimizers/adam/amsgrad_update.hpp
similarity index 91%
rename from src/mlpack/core/optimizers/ams_grad/amsgrad_update.hpp
rename to src/mlpack/core/optimizers/adam/amsgrad_update.hpp
index 321e85665e..49a8a4ca6e 100644
--- a/src/mlpack/core/optimizers/ams_grad/amsgrad_update.hpp
+++ b/src/mlpack/core/optimizers/adam/amsgrad_update.hpp
@@ -45,8 +45,8 @@ class AmsGradUpdate
    * @param beta2 The second moment coefficient.
    */
   AmsGradUpdate(const double epsilon = 1e-8,
-             const double beta1 = 0.9,
-             const double beta2 = 0.999) :
+                const double beta1 = 0.9,
+                const double beta2 = 0.999) :
     epsilon(epsilon),
     beta1(beta1),
     beta2(beta2),
@@ -90,10 +90,14 @@ class AmsGradUpdate
     v *= beta2;
     v += (1 - beta2) * (gradient % gradient);
 
+    const double biasCorrection1 = 1.0 - std::pow(beta1, iteration);
+    const double biasCorrection2 = 1.0 - std::pow(beta2, iteration);
+
     // Element wise maximum of past and present squared gradients.
     vImproved = arma::max(vImproved, v);
 
-    iterate -= (stepSize * m) / (arma::sqrt(vImproved) + epsilon);
+    iterate -= (stepSize * std::sqrt(biasCorrection2) / biasCorrection1) *
+                m / (arma::sqrt(vImproved) + epsilon);
   }
 
   //! Get the value used to initialise the squared gradient parameter.
diff --git a/src/mlpack/core/optimizers/ams_grad/CMakeLists.txt b/src/mlpack/core/optimizers/ams_grad/CMakeLists.txt
deleted file mode 100644
index f138890d5b..0000000000
--- a/src/mlpack/core/optimizers/ams_grad/CMakeLists.txt
+++ /dev/null
@@ -1,12 +0,0 @@
-set(SOURCES
-  ams_grad.cpp
-  ams_grad.hpp
-  amsgrad_update.hpp
-)
-
-set(DIR_SRCS)
-foreach(file ${SOURCES})
-  set(DIR_SRCS ${DIR_SRCS} ${CMAKE_CURRENT_SOURCE_DIR}/${file})
-endforeach()
-
-set(MLPACK_SRCS ${MLPACK_SRCS} ${DIR_SRCS} PARENT_SCOPE)
diff --git a/src/mlpack/core/optimizers/ams_grad/ams_grad.cpp b/src/mlpack/core/optimizers/ams_grad/ams_grad.cpp
deleted file mode 100644
index dca009419e..0000000000
--- a/src/mlpack/core/optimizers/ams_grad/ams_grad.cpp
+++ /dev/null
@@ -1,37 +0,0 @@
-/**
- * @file ams_grad_impl.hpp
- * @author Haritha Nair
- *
- * Implementation of the AMSGrad Optimizer.
- *
- * mlpack is free software; you may redistribute it and/or modify it under the
- * terms of the 3-clause BSD license.  You should have received a copy of the
- * 3-clause BSD license along with mlpack.  If not, see
- * http://www.opensource.org/licenses/BSD-3-Clause for more information.
- */
-// In case it hasn't been included yet.
-#include "ams_grad.hpp"
-
-namespace mlpack {
-namespace optimization {
-
-AmsGrad::AmsGrad(
-    const double stepSize,
-    const size_t batchSize,
-    const double beta1,
-    const double beta2,
-    const double epsilon,
-    const size_t maxIterations,
-    const double tolerance,
-    const bool shuffle) :
-    optimizer(stepSize,
-              batchSize,
-              maxIterations,
-              tolerance,
-              shuffle,
-              AmsGradUpdate(epsilon, beta1, beta2))
-{ /* Nothing to do. */ }
-
-} // namespace optimization
-} // namespace mlpack
-
diff --git a/src/mlpack/core/optimizers/ams_grad/ams_grad.hpp b/src/mlpack/core/optimizers/ams_grad/ams_grad.hpp
deleted file mode 100644
index a8f543ca52..0000000000
--- a/src/mlpack/core/optimizers/ams_grad/ams_grad.hpp
+++ /dev/null
@@ -1,157 +0,0 @@
-/**
- * @file ams_grad.hpp
- * @author Haritha Nair
- *
- * Implementation of AMSGrad optimizer. AMSGrad is an exponential moving average 
- * optimizer that dynamically adapts over time with guaranteed convergence.
- *
- * mlpack is free software; you may redistribute it and/or modify it under the
- * terms of the 3-clause BSD license.  You should have received a copy of the
- * 3-clause BSD license along with mlpack.  If not, see
- * http://www.opensource.org/licenses/BSD-3-Clause for more information.
- */
-#ifndef MLPACK_CORE_OPTIMIZERS_AMS_GRAD_AMS_GRAD_HPP
-#define MLPACK_CORE_OPTIMIZERS_AMS_GRAD_AMS_GRAD_HPP
-
-#include <mlpack/prereqs.hpp>
-
-#include <mlpack/core/optimizers/sgd/sgd.hpp>
-#include "amsgrad_update.hpp"
-
-namespace mlpack {
-namespace optimization {
-
-/**
- * AMSGrad is an exponential moving average variant which along with having 
- * benefits of optimizers like Adam and RmsProp, also guarantees convergence.
- * Unlike adam, it uses maximum of past squared gradients rather than their 
- * exponential average for updation.
- *
- * For more information, see the following.
- *
- * @code
- * @article{
- *   title   = {On the convergence of Adam and beyond},
- *   url     = {https://openreview.net/pdf?id=ryQu7f-RZ}
- * }
- * @endcode
- *
- *
- * For AMSGrad to work, a DecomposableFunctionType template parameter
- * is required. This class must implement the following function:
- *
- *   size_t NumFunctions();
- *   double Evaluate(const arma::mat& coordinates,
- *                   const size_t i,
- *                   const size_t batchSize);
- *   void Gradient(const arma::mat& coordinates,
- *                 const size_t i,
- *                 arma::mat& gradient,
- *                 const size_t batchSize);
- *
- * NumFunctions() should return the number of functions (\f$n\f$), and in the
- * other two functions, the parameter i refers to which individual function (or
- * gradient) is being evaluated.  So, for the case of a data-dependent function,
- * such as NCA (see mlpack::nca::NCA), NumFunctions() should return the number
- * of points in the dataset, and Evaluate(coordinates, 0) will evaluate the
- * objective function on the first point in the dataset (presumably, the dataset
- * is held internally in the DecomposableFunctionType).
- */
-
-class AmsGrad
-{
- public:
-  /**
-   * Construct the AMSGrad optimizer with the given function and parameters. The
-   * defaults here are not necessarily good for the given problem, so it is
-   * suggested that the values used be tailored to the task at hand.  The
-   * maximum number of iterations refers to the maximum number of points that
-   * are processed (i.e., one iteration equals one point; one iteration does not
-   * equal one pass over the dataset).
-   *
-   * @param stepSize Step size for each iteration.
-   * @param batchSize Number of points to process in a single step.
-   * @param beta1 Exponential decay rate for the first moment estimates.
-   * @param beta2 Exponential decay rate for the weighted infinity norm
-            estimates.
-   * @param eps Value used to initialise the mean squared gradient parameter.
-   * @param maxIterations Maximum number of iterations allowed (0 means no
-   *        limit).
-   * @param tolerance Maximum absolute tolerance to terminate algorithm.
-   * @param shuffle If true, the function order is shuffled; otherwise, each
-   *        function is visited in linear order.
-   */
-  AmsGrad(const double stepSize = 0.001,
-           const size_t batchSize = 32,
-           const double beta1 = 0.9,
-           const double beta2 = 0.999,
-           const double eps = 1e-8,
-           const size_t maxIterations = 100000,
-           const double tolerance = 1e-11,
-           const bool shuffle = true);
-
-  /**
-   * Optimize the given function using AMSGrad. The given starting point will be
-   * modified to store the finishing point of the algorithm, and the final
-   * objective value is returned.
-   *
-   * @tparam DecomposableFunctionType Type of the function to optimize.
-   * @param function Function to optimize.
-   * @param iterate Starting point (will be modified).
-   * @return Objective value of the final point.
-   */
-  template<typename DecomposableFunctionType>
-  double Optimize(DecomposableFunctionType& function, arma::mat& iterate)
-  {
-    return optimizer.Optimize(function, iterate);
-  }
-
-  //! Get the step size.
-  double StepSize() const { return optimizer.StepSize(); }
-  //! Modify the step size.
-  double& StepSize() { return optimizer.StepSize(); }
-
-  //! Get the batch size.
-  size_t BatchSize() const { return optimizer.BatchSize(); }
-  //! Modify the batch size.
-  size_t& BatchSize() { return optimizer.BatchSize(); }
-
-  //! Get the smoothing parameter.
-  double Beta1() const { return optimizer.UpdatePolicy().Beta1(); }
-  //! Modify the smoothing parameter.
-  double& Beta1() { return optimizer.UpdatePolicy().Beta1(); }
-
-  //! Get the second moment coefficient.
-  double Beta2() const { return optimizer.UpdatePolicy().Beta2(); }
-  //! Modify the second moment coefficient.
-  double& Beta2() { return optimizer.UpdatePolicy().Beta2(); }
-
-  //! Get the value used to initialise the mean squared gradient parameter.
-  double Epsilon() const { return optimizer.UpdatePolicy().Epsilon(); }
-  //! Modify the value used to initialise the mean squared gradient parameter.
-  double& Epsilon() { return optimizer.UpdatePolicy().Epsilon(); }
-
-  //! Get the maximum number of iterations (0 indicates no limit).
-  size_t MaxIterations() const { return optimizer.MaxIterations(); }
-  //! Modify the maximum number of iterations (0 indicates no limit).
-  size_t& MaxIterations() { return optimizer.MaxIterations(); }
-
-  //! Get the tolerance for termination.
-  double Tolerance() const { return optimizer.Tolerance(); }
-  //! Modify the tolerance for termination.
-  double& Tolerance() { return optimizer.Tolerance(); }
-
-  //! Get whether or not the individual functions are shuffled.
-  bool Shuffle() const { return optimizer.Shuffle(); }
-  //! Modify whether or not the individual functions are shuffled.
-  bool& Shuffle() { return optimizer.Shuffle(); }
-
- private:
-  //! The Stochastic Gradient Descent object with AMSGrad policy.
-  SGD<AmsGradUpdate> optimizer;
-};
-
-} // namespace optimization
-} // namespace mlpack
-
-#endif
diff --git a/src/mlpack/tests/CMakeLists.txt b/src/mlpack/tests/CMakeLists.txt
index 5dbe1b096c..fc6ee57e48 100644
--- a/src/mlpack/tests/CMakeLists.txt
+++ b/src/mlpack/tests/CMakeLists.txt
@@ -7,7 +7,6 @@ add_executable(mlpack_test
   ada_grad_test.cpp
   akfn_test.cpp
   aknn_test.cpp
-  ams_grad_test.cpp
   ann_layer_test.cpp
   armadillo_svd_test.cpp
   arma_extend_test.cpp
diff --git a/src/mlpack/tests/adam_test.cpp b/src/mlpack/tests/adam_test.cpp
index c1431a860c..b0279aa1ea 100644
--- a/src/mlpack/tests/adam_test.cpp
+++ b/src/mlpack/tests/adam_test.cpp
@@ -3,7 +3,7 @@
  * @author Vasanth Kalingeri
  * @author Vivek Pal
  *
- * Tests the Adam and AdaMax optimizer.
+ * Tests the Adam, AdaMax and AMSGrad optimizer.
  *
  * mlpack is free software; you may redistribute it and/or modify it under the
  * terms of the 3-clause BSD license.  You should have received a copy of the
@@ -62,6 +62,22 @@ BOOST_AUTO_TEST_CASE(SimpleAdaMaxTestFunction)
   BOOST_REQUIRE_SMALL(coordinates[2], 0.1);
 }
 
+/**
+ * Tests the AMSGrad optimizer using a simple test function.
+ */
+BOOST_AUTO_TEST_CASE(SimpleAmsGradTestFunction)
+{
+  SGDTestFunction f;
+  AmsGrad optimizer(1e-3, 1, 0.9, 0.999, 1e-8, 500000, 1e-11, true);
+
+  arma::mat coordinates = f.GetInitialPoint();
+  optimizer.Optimize(f, coordinates);
+
+  BOOST_REQUIRE_SMALL(coordinates[0], 0.1);
+  BOOST_REQUIRE_SMALL(coordinates[1], 0.1);
+  BOOST_REQUIRE_SMALL(coordinates[2], 0.1);
+}
+
 /**
  * Run Adam on logistic regression and make sure the results are acceptable.
  */
@@ -178,4 +194,62 @@ BOOST_AUTO_TEST_CASE(AdaMaxLogisticRegressionTest)
   BOOST_REQUIRE_CLOSE(testAcc, 100.0, 0.6); // 0.6% error tolerance.
 }
 
+/**
+ * Run AMSGrad on logistic regression and make sure the results are acceptable.
+ */
+BOOST_AUTO_TEST_CASE(AmsGradLogisticRegressionTest)
+{
+  // Generate a two-Gaussian dataset.
+  GaussianDistribution g1(arma::vec("1.0 1.0 1.0"), arma::eye<arma::mat>(3, 3));
+  GaussianDistribution g2(arma::vec("9.0 9.0 9.0"), arma::eye<arma::mat>(3, 3));
+
+  arma::mat data(3, 1000);
+  arma::Row<size_t> responses(1000);
+  for (size_t i = 0; i < 500; ++i)
+  {
+    data.col(i) = g1.Random();
+    responses[i] = 0;
+  }
+  for (size_t i = 500; i < 1000; ++i)
+  {
+    data.col(i) = g2.Random();
+    responses[i] = 1;
+  }
+
+  // Shuffle the dataset.
+  arma::uvec indices = arma::shuffle(arma::linspace<arma::uvec>(0,
+      data.n_cols - 1, data.n_cols));
+  arma::mat shuffledData(3, 1000);
+  arma::Row<size_t> shuffledResponses(1000);
+  for (size_t i = 0; i < data.n_cols; ++i)
+  {
+    shuffledData.col(i) = data.col(indices[i]);
+    shuffledResponses[i] = responses[indices[i]];
+  }
+
+  // Create a test set.
+  arma::mat testData(3, 1000);
+  arma::Row<size_t> testResponses(1000);
+  for (size_t i = 0; i < 500; ++i)
+  {
+    testData.col(i) = g1.Random();
+    testResponses[i] = 0;
+  }
+  for (size_t i = 500; i < 1000; ++i)
+  {
+    testData.col(i) = g2.Random();
+    testResponses[i] = 1;
+  }
+
+  AmsGrad amsgrad(1e-3, 1, 0.9, 0.999, 1e-8, 500000, 1e-11, true);
+  LogisticRegression<> lr(shuffledData, shuffledResponses, amsgrad, 0.5);
+
+  // Ensure that the error is close to zero.
+  const double acc = lr.ComputeAccuracy(data, responses);
+  BOOST_REQUIRE_CLOSE(acc, 100.0, 0.3); // 0.3% error tolerance.
+
+  const double testAcc = lr.ComputeAccuracy(testData, testResponses);
+  BOOST_REQUIRE_CLOSE(testAcc, 100.0, 0.6); // 0.6% error tolerance.
+}
+
 BOOST_AUTO_TEST_SUITE_END();
diff --git a/src/mlpack/tests/ams_grad_test.cpp b/src/mlpack/tests/ams_grad_test.cpp
deleted file mode 100644
index ecfe9a4bbd..0000000000
--- a/src/mlpack/tests/ams_grad_test.cpp
+++ /dev/null
@@ -1,107 +0,0 @@
-/**
- * @file ams_grad_test.cpp
- * @author Haritha Nair
- *
- * Tests the AMSGrad optimizer.
- *
- * mlpack is free software; you may redistribute it and/or modify it under the
- * terms of the 3-clause BSD license.  You should have received a copy of the
- * 3-clause BSD license along with mlpack.  If not, see
- * http://www.opensource.org/licenses/BSD-3-Clause for more information.
- */
-#include <mlpack/core.hpp>
-
-#include <mlpack/core/optimizers/ams_grad/ams_grad.hpp>
-#include <mlpack/core/optimizers/sgd/test_function.hpp>
-#include <mlpack/methods/logistic_regression/logistic_regression.hpp>
-
-#include <boost/test/unit_test.hpp>
-#include "test_tools.hpp"
-
-using namespace arma;
-using namespace mlpack::optimization;
-using namespace mlpack::optimization::test;
-
-using namespace mlpack::distribution;
-using namespace mlpack::regression;
-
-using namespace mlpack;
-
-BOOST_AUTO_TEST_SUITE(AmsGradTest);
-
-/**
- * Tests the AMSGrad optimizer using a simple test function.
- */
-BOOST_AUTO_TEST_CASE(SimpleAmsGradTestFunction)
-{
-  SGDTestFunction f;
-  AmsGrad optimizer(1e-3, 1, 0.9, 0.999, 1e-8, 500000, 1e-11, true);
-
-  arma::mat coordinates = f.GetInitialPoint();
-  optimizer.Optimize(f, coordinates);
-
-  BOOST_REQUIRE_SMALL(coordinates[0], 0.1);
-  BOOST_REQUIRE_SMALL(coordinates[1], 0.1);
-  BOOST_REQUIRE_SMALL(coordinates[2], 0.1);
-}
-
-
-/**
- * Run AMSGrad on logistic regression and make sure the results are acceptable.
- */
-BOOST_AUTO_TEST_CASE(AmsGradLogisticRegressionTest)
-{
-  // Generate a two-Gaussian dataset.
-  GaussianDistribution g1(arma::vec("1.0 1.0 1.0"), arma::eye<arma::mat>(3, 3));
-  GaussianDistribution g2(arma::vec("9.0 9.0 9.0"), arma::eye<arma::mat>(3, 3));
-
-  arma::mat data(3, 1000);
-  arma::Row<size_t> responses(1000);
-  for (size_t i = 0; i < 500; ++i)
-  {
-    data.col(i) = g1.Random();
-    responses[i] = 0;
-  }
-  for (size_t i = 500; i < 1000; ++i)
-  {
-    data.col(i) = g2.Random();
-    responses[i] = 1;
-  }
-
-  // Shuffle the dataset.
-  arma::uvec indices = arma::shuffle(arma::linspace<arma::uvec>(0,
-      data.n_cols - 1, data.n_cols));
-  arma::mat shuffledData(3, 1000);
-  arma::Row<size_t> shuffledResponses(1000);
-  for (size_t i = 0; i < data.n_cols; ++i)
-  {
-    shuffledData.col(i) = data.col(indices[i]);
-    shuffledResponses[i] = responses[indices[i]];
-  }
-
-  // Create a test set.
-  arma::mat testData(3, 1000);
-  arma::Row<size_t> testResponses(1000);
-  for (size_t i = 0; i < 500; ++i)
-  {
-    testData.col(i) = g1.Random();
-    testResponses[i] = 0;
-  }
-  for (size_t i = 500; i < 1000; ++i)
-  {
-    testData.col(i) = g2.Random();
-    testResponses[i] = 1;
-  }
-
-  AmsGrad amsgrad;
-  LogisticRegression<> lr(shuffledData, shuffledResponses, amsgrad, 0.5);
-
-  // Ensure that the error is close to zero.
-  const double acc = lr.ComputeAccuracy(data, responses);
-  BOOST_REQUIRE_CLOSE(acc, 100.0, 0.3); // 0.3% error tolerance.
-
-  const double testAcc = lr.ComputeAccuracy(testData, testResponses);
-  BOOST_REQUIRE_CLOSE(testAcc, 100.0, 0.6); // 0.6% error tolerance.
-}
-
-BOOST_AUTO_TEST_SUITE_END();

From e463bb03d50c91503a80e2e4e383ac2196964d6a Mon Sep 17 00:00:00 2001
From: Sourabh Varshney <sourabhvarshney111@gmail.com>
Date: Thu, 14 Dec 2017 15:12:23 +0530
Subject: [PATCH 14/67] Changed Nadam as optimizing update on Adam

---
 .../core/optimizers/Nadam/CMakeLists.txt      |  12 --
 src/mlpack/core/optimizers/Nadam/nadam.hpp    | 159 ------------------
 .../core/optimizers/Nadam/nadam_impl.hpp      |  41 -----
 .../core/optimizers/adam/CMakeLists.txt       |   1 +
 src/mlpack/core/optimizers/adam/adam.hpp      |  12 +-
 src/mlpack/core/optimizers/adam/adam_impl.hpp |   2 +-
 .../{Nadam => adam}/nadam_update.hpp          |  78 ++++++---
 src/mlpack/tests/adam_test.cpp                |  79 ++++++++-
 src/mlpack/tests/nadam_test.cpp               | 111 ------------
 9 files changed, 147 insertions(+), 348 deletions(-)
 delete mode 100644 src/mlpack/core/optimizers/Nadam/CMakeLists.txt
 delete mode 100644 src/mlpack/core/optimizers/Nadam/nadam.hpp
 delete mode 100644 src/mlpack/core/optimizers/Nadam/nadam_impl.hpp
 rename src/mlpack/core/optimizers/{Nadam => adam}/nadam_update.hpp (54%)
 delete mode 100644 src/mlpack/tests/nadam_test.cpp

diff --git a/src/mlpack/core/optimizers/Nadam/CMakeLists.txt b/src/mlpack/core/optimizers/Nadam/CMakeLists.txt
deleted file mode 100644
index 09bb246794..0000000000
--- a/src/mlpack/core/optimizers/Nadam/CMakeLists.txt
+++ /dev/null
@@ -1,12 +0,0 @@
-set(SOURCES
-  nadam.hpp
-  nadam_impl.hpp
-  nadam_update.hpp
-)
-
-set(DIR_SRCS)
-foreach(file ${SOURCES})
-  set(DIR_SRCS ${DIR_SRCS} ${CMAKE_CURRENT_SOURCE_DIR}/${file})
-endforeach()
-
-set(MLPACK_SRCS ${MLPACK_SRCS} ${DIR_SRCS} PARENT_SCOPE)
diff --git a/src/mlpack/core/optimizers/Nadam/nadam.hpp b/src/mlpack/core/optimizers/Nadam/nadam.hpp
deleted file mode 100644
index 4e01c39305..0000000000
--- a/src/mlpack/core/optimizers/Nadam/nadam.hpp
+++ /dev/null
@@ -1,159 +0,0 @@
-/**
- * @file nadam.hpp
- * @author Sourabh Varshney
- *
- * Nadam optimizer. Nadam is an optimizer that combines the effect of Adam and
- * NAG to the gradient descent to improve its Performance.
- *
- * mlpack is free software; you may redistribute it and/or modify it under the
- * terms of the 3-clause BSD license.  You should have received a copy of the
- * 3-clause BSD license along with mlpack.  If not, see
- * http://www.opensource.org/licenses/BSD-3-Clause for more information.
- */
-#ifndef MLPACK_CORE_OPTIMIZERS_NADAM_NADAM_HPP
-#define MLPACK_CORE_OPTIMIZERS_NADAM_NADAM_HPP
-
-#include <mlpack/prereqs.hpp>
-
-#include <mlpack/core/optimizers/sgd/sgd.hpp>
-#include "nadam_update.hpp"
-
-namespace mlpack {
-namespace optimization {
-
-/**
- * Nadam is an optimizer that combines the Adam and NAG.
- *
- * For more information, see the following.
- *
- * @code
- * @article{
- *   author  = {Sebastian Ruder},
- *   title   = {An overview of gradient descent optimization algorithms},
- *   journal = {CoRR},
- *   year    = {2016},
- *   url     = {https://arxiv.org/abs/1609.04747v2}
- * }
- * @endcode
- *
- *
- * For Nadam to work, a DecomposableFunctionType template parameter
- * is required. This class must implement the following function:
- *
- *   size_t NumFunctions();
- *   double Evaluate(const arma::mat& coordinates,
- *                   const size_t i,
- *                   const size_t batchSize);
- *   void Gradient(const arma::mat& coordinates,
- *                 const size_t i,
- *                 arma::mat& gradient,
- *                 const size_t batchSize);
- *
- * NumFunctions() should return the number of functions (\f$n\f$), and in the
- * other two functions, the parameter i refers to which individual function (or
- * gradient) is being evaluated.  So, for the case of a data-dependent 
- * function, such as NCA (see mlpack::nca::NCA), NumFunctions() should return 
- * the number of points in the dataset, and Evaluate(coordinates, 0) will 
- * evaluate the objective function on the first point in the dataset 
- * (presumably, the dataset is held internally in the 
- * DecomposableFunctionType).
- *
- * @tparam UpdateRule Nadam optimizer update rule to be used.
- */
-template<typename UpdateRule = NadamUpdate>
-class NadamType
-{
- public:
-  /**
-   * Construct the Nadam optimizer with the given function and parameters. The
-   * defaults here are not necessarily good for the given problem, so it is
-   * suggested that the values used be tailored to the task at hand.  The
-   * maximum number of iterations refers to the maximum number of points that
-   * are processed (i.e., one iteration equals one point; one iteration does
-   * not equal one pass over the dataset).
-   *
-   * @param stepSize Step size for each iteration.
-   * @param batchSize Number of points to process in a single step.
-   * @param beta1 Exponential decay rate for the first moment estimates.
-   * @param beta2 Exponential decay rate for the weighted infinity norm
-            estimates.
-   * @param eps Value used to initialise the mean squared gradient parameter.
-   * @param maxIterations Maximum number of iterations allowed (0 means no
-   *        limit).
-   * @param tolerance Maximum absolute tolerance to terminate algorithm.
-   * @param shuffle If true, the function order is shuffled; otherwise, each
-   *        function is visited in linear order.
-   */
-  NadamType(const double stepSize = 0.001,
-           const size_t batchSize = 32,
-           const double beta1 = 0.9,
-           const double eps = 1e-8,
-           const size_t maxIterations = 100000,
-           const double tolerance = 1e-5,
-           const bool shuffle = true);
-
-  /**
-   * Optimize the given function using Nadam. The given starting point will be
-   * modified to store the finishing point of the algorithm, and the final
-   * objective value is returned.
-   *
-   * @tparam DecomposableFunctionType Type of the function to optimize.
-   * @param function Function to optimize.
-   * @param iterate Starting point (will be modified).
-   * @return Objective value of the final point.
-   */
-  template<typename DecomposableFunctionType>
-  double Optimize(DecomposableFunctionType& function, arma::mat& iterate)
-  {
-    return optimizer.Optimize(function, iterate);
-  }
-
-  //! Get the step size.
-  double StepSize() const { return optimizer.StepSize(); }
-  //! Modify the step size.
-  double& StepSize() { return optimizer.StepSize(); }
-
-  //! Get the batch size.
-  size_t BatchSize() const { return optimizer.BatchSize(); }
-  //! Modify the batch size.
-  size_t& BatchSize() { return optimizer.BatchSize(); }
-
-  //! Get the smoothing parameter.
-  double Beta1() const { return optimizer.UpdatePolicy().Beta1(); }
-  //! Modify the smoothing parameter.
-  double& Beta1() { return optimizer.UpdatePolicy().Beta1(); }
-
-  //! Get the value used to initialise the mean squared gradient parameter.
-  double Epsilon() const { return optimizer.UpdatePolicy().Epsilon(); }
-  //! Modify the value used to initialise the mean squared gradient parameter.
-  double& Epsilon() { return optimizer.UpdatePolicy().Epsilon(); }
-
-  //! Get the maximum number of iterations (0 indicates no limit).
-  size_t MaxIterations() const { return optimizer.MaxIterations(); }
-  //! Modify the maximum number of iterations (0 indicates no limit).
-  size_t& MaxIterations() { return optimizer.MaxIterations(); }
-
-  //! Get the tolerance for termination.
-  double Tolerance() const { return optimizer.Tolerance(); }
-  //! Modify the tolerance for termination.
-  double& Tolerance() { return optimizer.Tolerance(); }
-
-  //! Get whether or not the individual functions are shuffled.
-  bool Shuffle() const { return optimizer.Shuffle(); }
-  //! Modify whether or not the individual functions are shuffled.
-  bool& Shuffle() { return optimizer.Shuffle(); }
-
- private:
-  //! The Stochastic Gradient Descent object with Nadam policy.
-  SGD<UpdateRule> optimizer;
-};
-
-using Nadam = NadamType<NadamUpdate>;
-
-} // namespace optimization
-} // namespace mlpack
-
-// Include implementation.
-#include "nadam_impl.hpp"
-
-#endif
diff --git a/src/mlpack/core/optimizers/Nadam/nadam_impl.hpp b/src/mlpack/core/optimizers/Nadam/nadam_impl.hpp
deleted file mode 100644
index 238e951acb..0000000000
--- a/src/mlpack/core/optimizers/Nadam/nadam_impl.hpp
+++ /dev/null
@@ -1,41 +0,0 @@
-/**
- * @file nadam_impl.hpp
- * @author Sourabh Varshney
- *
- * Implementation of the Nadam optimizer.
- *
- * mlpack is free software; you may redistribute it and/or modify it under the
- * terms of the 3-clause BSD license.  You should have received a copy of the
- * 3-clause BSD license along with mlpack.  If not, see
- * http://www.opensource.org/licenses/BSD-3-Clause for more information.
- */
-#ifndef MLPACK_CORE_OPTIMIZERS_NADAM_NADAM_IMPL_HPP
-#define MLPACK_CORE_OPTIMIZERS_NADAM_NADAM_IMPL_HPP
-
-// In case it hasn't been included yet.
-#include "nadam.hpp"
-
-namespace mlpack {
-namespace optimization {
-
-template<typename UpdateRule>
-NadamType<UpdateRule>::NadamType(
-    const double stepSize,
-    const size_t batchSize,
-    const double beta1,
-    const double epsilon,
-    const size_t maxIterations,
-    const double tolerance,
-    const bool shuffle) :
-    optimizer(stepSize,
-              batchSize,
-              maxIterations,
-              tolerance,
-              shuffle,
-              UpdateRule(epsilon, beta1))
-{ /* Nothing to do. */ }
-
-} // namespace optimization
-} // namespace mlpack
-
-#endif
diff --git a/src/mlpack/core/optimizers/adam/CMakeLists.txt b/src/mlpack/core/optimizers/adam/CMakeLists.txt
index 1377bbb0e1..d5a62574ee 100644
--- a/src/mlpack/core/optimizers/adam/CMakeLists.txt
+++ b/src/mlpack/core/optimizers/adam/CMakeLists.txt
@@ -3,6 +3,7 @@ set(SOURCES
   adam_impl.hpp
   adam_update.hpp
   adamax_update.hpp
+  nadam_update.hpp
 )
 
 set(DIR_SRCS)
diff --git a/src/mlpack/core/optimizers/adam/adam.hpp b/src/mlpack/core/optimizers/adam/adam.hpp
index 3b109660f5..c886e99f35 100644
--- a/src/mlpack/core/optimizers/adam/adam.hpp
+++ b/src/mlpack/core/optimizers/adam/adam.hpp
@@ -5,10 +5,11 @@
  * @author Marcus Edel
  * @author Vivek Pal
  *
- * Adam and AdaMax optimizer. Adam is an an algorithm for first-order gradient-
+ * Adam, AdaMax and Nadam optimizer. Adam is an an algorithm for first-order gradient-
  * -based optimization of stochastic objective functions, based on adaptive
  * estimates of lower-order moments. AdaMax is simply a variant of Adam based
- * on the infinity norm.
+ * on the infinity norm. Nadam is an optimizer that combines the effect of Adam and
+ * NAG to the gradient descent to improve its Performance.
  *
  * mlpack is free software; you may redistribute it and/or modify it under the
  * terms of the 3-clause BSD license.  You should have received a copy of the
@@ -31,7 +32,8 @@ namespace optimization {
  * Adam is an optimizer that computes individual adaptive learning rates for
  * different parameters from estimates of first and second moments of the
  * gradients. AdaMax is a variant of Adam based on the infinity norm as given
- * in the section 7 of the following paper.
+ * in the section 7 of the following paper. Nadam is an optimizer that 
+ * combines the Adam and NAG.
  *
  * For more information, see the following.
  *
@@ -46,7 +48,7 @@ namespace optimization {
  * @endcode
  *
  *
- * For Adam and AdaMax to work, a DecomposableFunctionType template parameter
+ * For Adam, AdaMax and Nadam to work, a DecomposableFunctionType template parameter
  * is required. This class must implement the following function:
  *
  *   size_t NumFunctions();
@@ -166,6 +168,8 @@ using Adam = AdamType<AdamUpdate>;
 
 using AdaMax = AdamType<AdaMaxUpdate>;
 
+using Nadam = NadamType<NadamUpdate>;
+
 } // namespace optimization
 } // namespace mlpack
 
diff --git a/src/mlpack/core/optimizers/adam/adam_impl.hpp b/src/mlpack/core/optimizers/adam/adam_impl.hpp
index 326d038d15..7953f6e2f7 100644
--- a/src/mlpack/core/optimizers/adam/adam_impl.hpp
+++ b/src/mlpack/core/optimizers/adam/adam_impl.hpp
@@ -5,7 +5,7 @@
  * @author Marcus Edel
  * @author Vivek Pal
  *
- * Implementation of the Adam and AdaMax optimizer.
+ * Implementation of the Adam, AdaMax and Nadam optimizer.
  *
  * mlpack is free software; you may redistribute it and/or modify it under the
  * terms of the 3-clause BSD license.  You should have received a copy of the
diff --git a/src/mlpack/core/optimizers/Nadam/nadam_update.hpp b/src/mlpack/core/optimizers/adam/nadam_update.hpp
similarity index 54%
rename from src/mlpack/core/optimizers/Nadam/nadam_update.hpp
rename to src/mlpack/core/optimizers/adam/nadam_update.hpp
index c5ad289b3a..62c360f109 100644
--- a/src/mlpack/core/optimizers/Nadam/nadam_update.hpp
+++ b/src/mlpack/core/optimizers/adam/nadam_update.hpp
@@ -2,7 +2,7 @@
  * @file nadam_update.hpp
  * @author Sourabh Varshney
  *
- * Nadam optimizer. Nadam is an optimizer that combines the effect of Adam and 
+ * Nadam update rule. Nadam is an optimizer that combines the effect of Adam and 
  * NAG to the gradient descent to improve its Performance.
  *
  * mlpack is free software; you may redistribute it and/or modify it under the
@@ -10,8 +10,8 @@
  * 3-clause BSD license along with mlpack.  If not, see
  * http://www.opensource.org/licenses/BSD-3-Clause for more information.
  */
-#ifndef MLPACK_CORE_OPTIMIZERS_NADAM_NADAM_UPDATE_HPP
-#define MLPACK_CORE_OPTIMIZERS_NADAM_NADAM_UPDATE_HPP
+#ifndef MLPACK_CORE_OPTIMIZERS_ADAM_NADAM_UPDATE_HPP
+#define MLPACK_CORE_OPTIMIZERS_ADAM_NADAM_UPDATE_HPP
 
 #include <mlpack/prereqs.hpp>
 
@@ -24,12 +24,12 @@ namespace optimization {
  * For more information, see the following.
  *
  * @code
- * @article{
- *   author  = {Sebastian Ruder},
- *   title   = {An overview of gradient descent optimization algorithms},
- *   journal = {CoRR},
- *   year    = {2016},
- *   url     = {https://arxiv.org/abs/1609.04747v2}
+ * @misc{
+ *   author  = {},
+ *   title   = {},
+ *   journal = {},
+ *   year    = {},
+ *   url     = {}
  * }
  * @endcode
  */
@@ -42,9 +42,12 @@ class NadamUpdate
    * @param epsilon The epsilon value used to initialise the squared gradient
    *        parameter.
    * @param beta1 The smoothing parameter.
+   * @param beta2 The second moment coefficient
    */
-  NadamUpdate(const double epsilon = 1e-8, const double beta1 = 0.9)
-  :epsilon(epsilon), beta1(beta1), iteration(0)
+  NadamUpdate(const double epsilon = 1e-8,
+	      const double beta1 = 0.9,
+	      const double beta2=0.99)
+  	      :epsilon(epsilon), beta1(beta1), beta2(beta2), iteration(0)
   {
     // Nothing to do.
   }
@@ -60,6 +63,7 @@ class NadamUpdate
   {
     m = arma::zeros<arma::mat>(rows, cols);
     v = arma::zeros<arma::mat>(rows, cols);
+    cum_beta1 = 1;
   }
 
   /**
@@ -69,8 +73,9 @@ class NadamUpdate
    * @param stepSize Step size to be used for the given iteration.
    * @param gradient The gradient matrix.
    */
-  void Update(arma::mat& iterate, const double stepSize,
-  const arma::mat& gradient)
+  void Update(arma::mat& iterate,
+              const double stepSize,
+              const arma::mat& gradient)
   {
     // Increment the iteration counter variable.
     ++iteration;
@@ -78,12 +83,31 @@ class NadamUpdate
     // And update the iterate.
     m *= beta1;
     m += (1 - beta1) * gradient;
-    // biasCorrection=1-beta1^iteration
-    const double biasCorrection = 1.0 - std::pow(beta1, iteration);
-    /*
-	iterate=iterate-((stepsize/(sqrt(v)+epsilon))*(m/biasCorrection))
-    */
-    iterate -= ((stepSize * m)/(biasCorrection1 * (arma::sqrt(v) + epsilon)));
+    
+    v *= beta2;
+    v += (1 - beta2) * gradient % gradient;
+
+    // beta1_t = beta1 * (1 - (0.5 * (0.96 ^ (iteration / 250))))
+    double beta1_t = beta1 * (1 - (0.5 * std::pow(0.96, (iteration / 250))));
+
+    // beta1_t1 = beta1 * (1 - (0.5 * (0.96 ^ ((iteration + 1)/ 250))))
+    double beta1_t1 = beta1 * (1 - (0.5 * std::pow(0.96, ((iteration + 1) / 250))));
+
+    // cum_beta1 *= beta1_t
+    cum_beta1 *= beta1_t;
+    
+    // biasCorrection = 1 - cum_beta1
+    const double biasCorrection1 = 1.0 - cum_beta1;
+
+    // biasCorrection2 = 1 - beta2 ^ iteration
+    const double biasCorrection2 = 1.0 - std::pow(beta2, iteration);
+
+    /* Note :- arma::sqrt(v) + epsilon * sqrt(biasCorrection2) is approximated as
+     *  arma::sqrt(v) + epsilon
+     */
+    iterate -= (stepSize * ((1 - beta1_t) * gradient +beta1_t1 * m)
+               * sqrt(biasCorrection2)) / ((arma::sqrt(v) + epsilon)
+	       * biasCorrection1)
   }
 
   //! Get the value used to initialise the squared gradient parameter.
@@ -91,11 +115,21 @@ class NadamUpdate
   //! Modify the value used to initialise the squared gradient parameter.
   double& Epsilon() { return epsilon; }
 
+  //! Get the value of the cumulative product of decay constants
+  double Cum_beta1() const { return cum_beta1; }
+  //! Modify the value of the cumulative product of decay constants
+  double& Cum_beta1() { return cum_beta1; }
+
   //! Get the smoothing parameter.
   double Beta1() const { return beta1; }
   //! Modify the smoothing parameter.
   double& Beta1() { return beta1; }
 
+  //! Get the second moment coefficient.
+  double Beta2() const { return beta2; }
+  //! Modify the second moment coefficient.
+  double& Beta2() { return beta2; }
+
  private:
   // The epsilon value used to initialise the squared gradient parameter.
   double epsilon;
@@ -103,12 +137,18 @@ class NadamUpdate
   // The smoothing parameter.
   double beta1;
 
+  // The second moment coefficient.
+  double beta2; 
+
   // The exponential moving average of gradient values.
   arma::mat m;
 
   // The exponential moving average of squared gradient values.
   arma::mat v;
 
+  // The cumulative product of decay constants
+  double cum_beta1;
+
   // The number of iterations.
   double iteration;
 };
diff --git a/src/mlpack/tests/adam_test.cpp b/src/mlpack/tests/adam_test.cpp
index c1431a860c..10c7f0bd5a 100644
--- a/src/mlpack/tests/adam_test.cpp
+++ b/src/mlpack/tests/adam_test.cpp
@@ -2,8 +2,9 @@
  * @file adam_test.cpp
  * @author Vasanth Kalingeri
  * @author Vivek Pal
+ * @author Sourabh Varshney
  *
- * Tests the Adam and AdaMax optimizer.
+ * Tests the Adam, AdaMax and Nadam optimizer.
  *
  * mlpack is free software; you may redistribute it and/or modify it under the
  * terms of the 3-clause BSD license.  You should have received a copy of the
@@ -62,6 +63,22 @@ BOOST_AUTO_TEST_CASE(SimpleAdaMaxTestFunction)
   BOOST_REQUIRE_SMALL(coordinates[2], 0.1);
 }
 
+/**
+ * Tests the Nadam optimizer using a simple test function.
+ */
+BOOST_AUTO_TEST_CASE(SimpleNadamTestFunction)
+{
+  SGDTestFunction f;
+  Nadam optimizer(1e-3, 1, 0.9, 0.99, 1e-8, 500000, 1e-9, true);
+
+  arma::mat coordinates = f.GetInitialPoint();
+  optimizer.Optimize(f, coordinates);
+
+  BOOST_REQUIRE_SMALL(coordinates[0], 0.1);
+  BOOST_REQUIRE_SMALL(coordinates[1], 0.1);
+  BOOST_REQUIRE_SMALL(coordinates[2], 0.1);
+}
+
 /**
  * Run Adam on logistic regression and make sure the results are acceptable.
  */
@@ -178,4 +195,64 @@ BOOST_AUTO_TEST_CASE(AdaMaxLogisticRegressionTest)
   BOOST_REQUIRE_CLOSE(testAcc, 100.0, 0.6); // 0.6% error tolerance.
 }
 
+/**
+ * Run Nadam on logistic regression and make sure the results are acceptable.
+ */
+BOOST_AUTO_TEST_CASE(NadamLogisticRegressionTest)
+{
+  // Generate a two-Gaussian dataset.
+  GaussianDistribution g1(arma::vec("1.0 1.0 1.0"),
+  arma::eye<arma::mat>(3, 3));
+  GaussianDistribution g2(arma::vec("9.0 9.0 9.0"),
+  arma::eye<arma::mat>(3, 3));
+
+  arma::mat data(3, 1000);
+  arma::Row<size_t> responses(1000);
+  for (size_t i = 0; i < 500; ++i)
+  {
+    data.col(i) = g1.Random();
+    responses[i] = 0;
+  }
+  for (size_t i = 500; i < 1000; ++i)
+  {
+    data.col(i) = g2.Random();
+    responses[i] = 1;
+  }
+
+  // Shuffle the dataset.
+  arma::uvec indices = arma::shuffle(arma::linspace<arma::uvec>(0,
+  data.n_cols - 1, data.n_cols));
+  arma::mat shuffledData(3, 1000);
+  arma::Row<size_t> shuffledResponses(1000);
+  for (size_t i = 0; i < data.n_cols; ++i)
+  {
+    shuffledData.col(i) = data.col(indices[i]);
+    shuffledResponses[i] = responses[indices[i]];
+  }
+
+  // Create a test set.
+  arma::mat testData(3, 1000);
+  arma::Row<size_t> testResponses(1000);
+  for (size_t i = 0; i < 500; ++i)
+  {
+    testData.col(i) = g1.Random();
+    testResponses[i] = 0;
+  }
+  for (size_t i = 500; i < 1000; ++i)
+  {
+    testData.col(i) = g2.Random();
+    testResponses[i] = 1;
+  }
+
+  Nadam nadam;
+  LogisticRegression<> lr(shuffledData, shuffledResponses, nadam, 0.5);
+
+  // Ensure that the error is close to zero.
+  const double acc = lr.ComputeAccuracy(data, responses);
+  BOOST_REQUIRE_CLOSE(acc, 100.0, 0.3); // 0.3% error tolerance.
+
+  const double testAcc = lr.ComputeAccuracy(testData, testResponses);
+  BOOST_REQUIRE_CLOSE(testAcc, 100.0, 0.6); // 0.6% error tolerance.
+}
+
 BOOST_AUTO_TEST_SUITE_END();
diff --git a/src/mlpack/tests/nadam_test.cpp b/src/mlpack/tests/nadam_test.cpp
deleted file mode 100644
index 97f53c1f12..0000000000
--- a/src/mlpack/tests/nadam_test.cpp
+++ /dev/null
@@ -1,111 +0,0 @@
-/**
- * @file nadam_test.cpp
- * @author Sourabh Varshney
- *
- * Tests the Nadam optimizer.
- *
- * mlpack is free software; you may redistribute it and/or modify it under the
- * terms of the 3-clause BSD license.  You should have received a copy of the
- * 3-clause BSD license along with mlpack.  If not, see
- * http://www.opensource.org/licenses/BSD-3-Clause for more information.
- */
-#include <mlpack/core.hpp>
-
-#include <mlpack/core/optimizers/nadam/nadam.hpp>
-#include <mlpack/core/optimizers/sgd/test_function.hpp>
-#include <mlpack/methods/logistic_regression/logistic_regression.hpp>
-
-#include <boost/test/unit_test.hpp>
-#include "test_tools.hpp"
-
-using namespace arma;
-using namespace mlpack::optimization;
-using namespace mlpack::optimization::test;
-
-using namespace mlpack::distribution;
-using namespace mlpack::regression;
-
-using namespace mlpack;
-
-BOOST_AUTO_TEST_SUITE(NadamTest);
-
-/**
- * Tests the Adam optimizer using a simple test function.
- */
-BOOST_AUTO_TEST_CASE(SimpleNadamTestFunction)
-{
-  SGDTestFunction f;
-  Nadam optimizer(1e-3, 1, 0.9, 1e-8, 500000, 1e-9, true);
-
-  arma::mat coordinates = f.GetInitialPoint();
-  optimizer.Optimize(f, coordinates);
-
-  BOOST_REQUIRE_SMALL(coordinates[0], 0.1);
-  BOOST_REQUIRE_SMALL(coordinates[1], 0.1);
-  BOOST_REQUIRE_SMALL(coordinates[2], 0.1);
-}
-
-
-/**
- * Run Nadam on logistic regression and make sure the results are acceptable.
- */
-BOOST_AUTO_TEST_CASE(NadamLogisticRegressionTest)
-{
-  // Generate a two-Gaussian dataset.
-  GaussianDistribution g1(arma::vec("1.0 1.0 1.0"),
-  arma::eye<arma::mat>(3, 3));
-  GaussianDistribution g2(arma::vec("9.0 9.0 9.0"),
-  arma::eye<arma::mat>(3, 3));
-
-  arma::mat data(3, 1000);
-  arma::Row<size_t> responses(1000);
-  for (size_t i = 0; i < 500; ++i)
-  {
-    data.col(i) = g1.Random();
-    responses[i] = 0;
-  }
-  for (size_t i = 500; i < 1000; ++i)
-  {
-    data.col(i) = g2.Random();
-    responses[i] = 1;
-  }
-
-  // Shuffle the dataset.
-  arma::uvec indices = arma::shuffle(arma::linspace<arma::uvec>(0,
-  data.n_cols - 1, data.n_cols));
-  arma::mat shuffledData(3, 1000);
-  arma::Row<size_t> shuffledResponses(1000);
-  for (size_t i = 0; i < data.n_cols; ++i)
-  {
-    shuffledData.col(i) = data.col(indices[i]);
-    shuffledResponses[i] = responses[indices[i]];
-  }
-
-  // Create a test set.
-  arma::mat testData(3, 1000);
-  arma::Row<size_t> testResponses(1000);
-  for (size_t i = 0; i < 500; ++i)
-  {
-    testData.col(i) = g1.Random();
-    testResponses[i] = 0;
-  }
-  for (size_t i = 500; i < 1000; ++i)
-  {
-    testData.col(i) = g2.Random();
-    testResponses[i] = 1;
-  }
-
-  Nadam nadam;
-  LogisticRegression<> lr(shuffledData, shuffledResponses, nadam, 0.5);
-
-  // Ensure that the error is close to zero.
-  const double acc = lr.ComputeAccuracy(data, responses);
-  BOOST_REQUIRE_CLOSE(acc, 100.0, 0.3); // 0.3% error tolerance.
-
-  const double testAcc = lr.ComputeAccuracy(testData, testResponses);
-  BOOST_REQUIRE_CLOSE(testAcc, 100.0, 0.6); // 0.6% error tolerance.
-}
-
-
-
-BOOST_AUTO_TEST_SUITE_END();

From d2baab465bbcd7330309aa1cb65f419a8379bcb3 Mon Sep 17 00:00:00 2001
From: Sourabh Varshney <sourabhvarshney111@gmail.com>
Date: Thu, 14 Dec 2017 15:36:51 +0530
Subject: [PATCH 15/67] Applied Style Improvements

---
 .../core/optimizers/adam/nadam_update.hpp     | 19 ++++++++++---------
 1 file changed, 10 insertions(+), 9 deletions(-)

diff --git a/src/mlpack/core/optimizers/adam/nadam_update.hpp b/src/mlpack/core/optimizers/adam/nadam_update.hpp
index 62c360f109..cefc001e53 100644
--- a/src/mlpack/core/optimizers/adam/nadam_update.hpp
+++ b/src/mlpack/core/optimizers/adam/nadam_update.hpp
@@ -45,9 +45,9 @@ class NadamUpdate
    * @param beta2 The second moment coefficient
    */
   NadamUpdate(const double epsilon = 1e-8,
-	      const double beta1 = 0.9,
-	      const double beta2=0.99)
-  	      :epsilon(epsilon), beta1(beta1), beta2(beta2), iteration(0)
+              const double beta1 = 0.9,
+              const double beta2 = 0.99)
+              :epsilon(epsilon), beta1(beta1), beta2(beta2), iteration(0)
   {
     // Nothing to do.
   }
@@ -83,7 +83,7 @@ class NadamUpdate
     // And update the iterate.
     m *= beta1;
     m += (1 - beta1) * gradient;
-    
+
     v *= beta2;
     v += (1 - beta2) * gradient % gradient;
 
@@ -91,11 +91,12 @@ class NadamUpdate
     double beta1_t = beta1 * (1 - (0.5 * std::pow(0.96, (iteration / 250))));
 
     // beta1_t1 = beta1 * (1 - (0.5 * (0.96 ^ ((iteration + 1)/ 250))))
-    double beta1_t1 = beta1 * (1 - (0.5 * std::pow(0.96, ((iteration + 1) / 250))));
+    double beta1_t1 = beta1 * (1 - (0.5 *
+                      std::pow(0.96, ((iteration + 1) / 250))));
 
     // cum_beta1 *= beta1_t
     cum_beta1 *= beta1_t;
-    
+
     // biasCorrection = 1 - cum_beta1
     const double biasCorrection1 = 1.0 - cum_beta1;
 
@@ -103,11 +104,11 @@ class NadamUpdate
     const double biasCorrection2 = 1.0 - std::pow(beta2, iteration);
 
     /* Note :- arma::sqrt(v) + epsilon * sqrt(biasCorrection2) is approximated as
-     *  arma::sqrt(v) + epsilon
+     * arma::sqrt(v) + epsilon
      */
     iterate -= (stepSize * ((1 - beta1_t) * gradient +beta1_t1 * m)
                * sqrt(biasCorrection2)) / ((arma::sqrt(v) + epsilon)
-	       * biasCorrection1)
+               * biasCorrection1)
   }
 
   //! Get the value used to initialise the squared gradient parameter.
@@ -138,7 +139,7 @@ class NadamUpdate
   double beta1;
 
   // The second moment coefficient.
-  double beta2; 
+  double beta2;
 
   // The exponential moving average of gradient values.
   arma::mat m;

From 5777c6ad40c04096e903ee3044de29265e3c0fa0 Mon Sep 17 00:00:00 2001
From: Sourabh Varshney <sourabhvarshney111@gmail.com>
Date: Thu, 14 Dec 2017 16:23:54 +0530
Subject: [PATCH 16/67] Applied Style Improvements

---
 src/mlpack/core/optimizers/adam/adam.hpp | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/mlpack/core/optimizers/adam/adam.hpp b/src/mlpack/core/optimizers/adam/adam.hpp
index c886e99f35..807408c692 100644
--- a/src/mlpack/core/optimizers/adam/adam.hpp
+++ b/src/mlpack/core/optimizers/adam/adam.hpp
@@ -4,6 +4,7 @@
  * @author Vasanth Kalingeri
  * @author Marcus Edel
  * @author Vivek Pal
+ * @author Sourabh Varshney
  *
  * Adam, AdaMax and Nadam optimizer. Adam is an an algorithm for first-order gradient-
  * -based optimization of stochastic objective functions, based on adaptive
@@ -24,6 +25,7 @@
 #include <mlpack/core/optimizers/sgd/sgd.hpp>
 #include "adam_update.hpp"
 #include "adamax_update.hpp"
+#include "nadam_update.hpp"
 
 namespace mlpack {
 namespace optimization {
@@ -168,7 +170,7 @@ using Adam = AdamType<AdamUpdate>;
 
 using AdaMax = AdamType<AdaMaxUpdate>;
 
-using Nadam = NadamType<NadamUpdate>;
+using Nadam = AdamType<NadamUpdate>;
 
 } // namespace optimization
 } // namespace mlpack

From aac0270c460e4e5c03cdebac7aab56c04deea22a Mon Sep 17 00:00:00 2001
From: Sourabh Varshney <sourabhvarshney111@gmail.com>
Date: Thu, 14 Dec 2017 16:34:33 +0530
Subject: [PATCH 17/67] Corrected Typo

---
 src/mlpack/core/optimizers/adam/nadam_update.hpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/mlpack/core/optimizers/adam/nadam_update.hpp b/src/mlpack/core/optimizers/adam/nadam_update.hpp
index cefc001e53..3c37a4a4cb 100644
--- a/src/mlpack/core/optimizers/adam/nadam_update.hpp
+++ b/src/mlpack/core/optimizers/adam/nadam_update.hpp
@@ -108,7 +108,7 @@ class NadamUpdate
      */
     iterate -= (stepSize * ((1 - beta1_t) * gradient +beta1_t1 * m)
                * sqrt(biasCorrection2)) / ((arma::sqrt(v) + epsilon)
-               * biasCorrection1)
+               * biasCorrection1);
   }
 
   //! Get the value used to initialise the squared gradient parameter.

From 966a1503f4a83300ebab1a0f9c2053a33a7c795e Mon Sep 17 00:00:00 2001
From: Sourabh Varshney <sourabhvarshney111@gmail.com>
Date: Thu, 14 Dec 2017 16:59:07 +0530
Subject: [PATCH 18/67] Constructor Initialization for a parameter

---
 src/mlpack/core/optimizers/adam/nadam_update.hpp | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/src/mlpack/core/optimizers/adam/nadam_update.hpp b/src/mlpack/core/optimizers/adam/nadam_update.hpp
index 3c37a4a4cb..a70d697226 100644
--- a/src/mlpack/core/optimizers/adam/nadam_update.hpp
+++ b/src/mlpack/core/optimizers/adam/nadam_update.hpp
@@ -47,7 +47,7 @@ class NadamUpdate
   NadamUpdate(const double epsilon = 1e-8,
               const double beta1 = 0.9,
               const double beta2 = 0.99)
-              :epsilon(epsilon), beta1(beta1), beta2(beta2), iteration(0)
+              :epsilon(epsilon), beta1(beta1), beta2(beta2), iteration(0), cum_beta1(1)
   {
     // Nothing to do.
   }
@@ -63,7 +63,6 @@ class NadamUpdate
   {
     m = arma::zeros<arma::mat>(rows, cols);
     v = arma::zeros<arma::mat>(rows, cols);
-    cum_beta1 = 1;
   }
 
   /**

From 9883434c17e196dc63512408d471752e086700b4 Mon Sep 17 00:00:00 2001
From: Sourabh Varshney <sourabhvarshney111@gmail.com>
Date: Thu, 14 Dec 2017 17:02:39 +0530
Subject: [PATCH 19/67] Corrected Line confliction

---
 src/mlpack/core/optimizers/adam/nadam_update.hpp | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/mlpack/core/optimizers/adam/nadam_update.hpp b/src/mlpack/core/optimizers/adam/nadam_update.hpp
index a70d697226..af5ee555ac 100644
--- a/src/mlpack/core/optimizers/adam/nadam_update.hpp
+++ b/src/mlpack/core/optimizers/adam/nadam_update.hpp
@@ -47,7 +47,8 @@ class NadamUpdate
   NadamUpdate(const double epsilon = 1e-8,
               const double beta1 = 0.9,
               const double beta2 = 0.99)
-              :epsilon(epsilon), beta1(beta1), beta2(beta2), iteration(0), cum_beta1(1)
+              :epsilon(epsilon), beta1(beta1), beta2(beta2), iteration(0),
+              cum_beta1(1)
   {
     // Nothing to do.
   }

From 403f767751113ff07831a412b156ef16fb0fc2e2 Mon Sep 17 00:00:00 2001
From: Haritha Sreedharan Nair <haritha1313@gmail.com>
Date: Thu, 14 Dec 2017 20:26:08 +0530
Subject: [PATCH 20/67] Update adam.hpp

---
 src/mlpack/core/optimizers/adam/adam.hpp | 13 +++++++------
 1 file changed, 7 insertions(+), 6 deletions(-)

diff --git a/src/mlpack/core/optimizers/adam/adam.hpp b/src/mlpack/core/optimizers/adam/adam.hpp
index a1ec694543..8b0420d9c1 100644
--- a/src/mlpack/core/optimizers/adam/adam.hpp
+++ b/src/mlpack/core/optimizers/adam/adam.hpp
@@ -5,10 +5,11 @@
  * @author Marcus Edel
  * @author Vivek Pal
  *
- * Adam, AdaMax and AMSGrad optimizer. Adam is an an algorithm for first-order gradient-
- * -based optimization of stochastic objective functions, based on adaptive
- * estimates of lower-order moments. AdaMax is simply a variant of Adam based
- * on the infinity norm. AMSGrad is another variant of Adam with guaranteed convergence.
+ * Adam, AdaMax and AMSGrad optimizers. Adam is an an algorithm for 
+ * first-order gradient-based optimization of stochastic objective 
+ * functions, based on adaptive estimates of lower-order moments. 
+ * AdaMax is simply a variant of Adam based on the infinity norm. 
+ * AMSGrad is another variant of Adam with guaranteed convergence.
  *
  * mlpack is free software; you may redistribute it and/or modify it under the
  * terms of the 3-clause BSD license.  You should have received a copy of the
@@ -51,8 +52,8 @@ namespace optimization {
  * @endcode
  *
  *
- * For Adam, AdaMax and AMSGrad to work, a DecomposableFunctionType template parameter
- * is required. This class must implement the following function:
+ * For Adam, AdaMax and AMSGrad to work, a DecomposableFunctionType template
+ * parameter is required. This class must implement the following function:
  *
  *   size_t NumFunctions();
  *   double Evaluate(const arma::mat& coordinates,

From 22dd024aa6b4e85a8c3e2330925e72db6c087d18 Mon Sep 17 00:00:00 2001
From: Eugene Freyman <evg.freyman@gmail.com>
Date: Fri, 15 Dec 2017 22:56:15 +0200
Subject: [PATCH 21/67] fixing issue with redefinition at binding tests

---
 .../main_tests/linear_regression_test.cpp     | 62 +++++++++++++++++++
 1 file changed, 62 insertions(+)
 create mode 100644 src/mlpack/tests/main_tests/linear_regression_test.cpp

diff --git a/src/mlpack/tests/main_tests/linear_regression_test.cpp b/src/mlpack/tests/main_tests/linear_regression_test.cpp
new file mode 100644
index 0000000000..061c713367
--- /dev/null
+++ b/src/mlpack/tests/main_tests/linear_regression_test.cpp
@@ -0,0 +1,62 @@
+/**
+ * @file linear_regression_test.cpp
+ * @author Eugene Freyman
+ *
+ * Test mlpackMain() of linear_regression_main.cpp.
+ */
+#define BINDING_TYPE BINDING_TYPE_TEST
+#define PROGRAM_NAME linearRegressionProgramName
+
+#include <mlpack/core.hpp>
+#include <mlpack/core/util/mlpack_main.hpp>
+#include <mlpack/methods/linear_regression/linear_regression_main.cpp>
+
+#include <boost/test/unit_test.hpp>
+#include "../test_tools.hpp"
+
+using namespace mlpack;
+
+// Utility function to set a parameter and mark it as passed, using copy
+// semantics.
+template<typename T>
+void SetInputParam(const std::string& name, const T& value)
+{
+  CLI::GetParam<T>(name) = value;
+  CLI::SetPassed(name);
+}
+
+// Utility function to set a parameter and mark it as passed, using move
+// semantics.
+template<typename T>
+void SetInputParam(const std::string& name, T&& value)
+{
+  CLI::GetParam<T>(name) = std::move(value);
+  CLI::SetPassed(name);
+}
+
+struct LinearRegressionTestFixture
+{
+ public:
+  LinearRegressionTestFixture()
+  {
+    // Cache in the options for this program.
+    CLI::RestoreSettings(mlpack::bindings::tests::programName);
+  }
+
+  ~LinearRegressionTestFixture()
+  {
+    // Clear the settings.
+    CLI::ClearSettings();
+  }
+};
+
+BOOST_FIXTURE_TEST_SUITE(LinearRegressionMainTest, LinearRegressionTestFixture);
+
+BOOST_AUTO_TEST_CASE(LinearRegressionWringResponseSizeTest)
+{
+  std::cout << "1\n";
+  SetInputParam("lambda", 1.0);
+  std::cout << "2\n";
+}
+
+BOOST_AUTO_TEST_SUITE_END();

From b720c6b603c596d325d40a50a56222681fb694dc Mon Sep 17 00:00:00 2001
From: Eugene Freyman <evg.freyman@gmail.com>
Date: Fri, 15 Dec 2017 23:04:24 +0200
Subject: [PATCH 22/67] fixed issue with redefinition at binding tests
 (forgotten files)

---
 src/mlpack/bindings/cli/cli_option.hpp        |  3 +-
 src/mlpack/bindings/python/py_option.hpp      |  3 +-
 src/mlpack/bindings/tests/test_option.hpp     |  3 +-
 src/mlpack/core/util/mlpack_main.hpp          | 11 +--
 src/mlpack/core/util/param.hpp                | 68 +++++++++++++------
 .../linear_regression_main.cpp                |  2 +-
 src/mlpack/methods/pca/pca_main.cpp           |  2 +-
 src/mlpack/tests/CMakeLists.txt               |  1 +
 .../main_tests/linear_regression_test.cpp     | 19 ++++--
 src/mlpack/tests/main_tests/pca_test.cpp      | 14 ++--
 10 files changed, 81 insertions(+), 45 deletions(-)

diff --git a/src/mlpack/bindings/cli/cli_option.hpp b/src/mlpack/bindings/cli/cli_option.hpp
index fd58fdde7a..4229230bf1 100644
--- a/src/mlpack/bindings/cli/cli_option.hpp
+++ b/src/mlpack/bindings/cli/cli_option.hpp
@@ -68,7 +68,8 @@ class CLIOption
             const std::string& cppName,
             const bool required = false,
             const bool input = true,
-            const bool noTranspose = false)
+            const bool noTranspose = false,
+	    const std::string& /*programName*/ = "")
   {
     // Create the ParamData object to give to CLI.
     util::ParamData data;
diff --git a/src/mlpack/bindings/python/py_option.hpp b/src/mlpack/bindings/python/py_option.hpp
index c934b2bafa..f4c38d9eb4 100644
--- a/src/mlpack/bindings/python/py_option.hpp
+++ b/src/mlpack/bindings/python/py_option.hpp
@@ -42,7 +42,8 @@ class PyOption
            const std::string& cppName,
            const bool required = false,
            const bool input = true,
-           const bool noTranspose = false)
+           const bool noTranspose = false,
+	   const std::string& /*programName*/ = "")
   {
     // Create the ParamData object to give to CLI.
     util::ParamData data;
diff --git a/src/mlpack/bindings/tests/test_option.hpp b/src/mlpack/bindings/tests/test_option.hpp
index 5e5e7e8d58..1f88b69f58 100644
--- a/src/mlpack/bindings/tests/test_option.hpp
+++ b/src/mlpack/bindings/tests/test_option.hpp
@@ -61,7 +61,8 @@ class TestOption
              const std::string& cppName,
              const bool required = false,
              const bool input = true,
-             const bool noTranspose = false)
+             const bool noTranspose = false,
+	     const std::string& programName = "")
   {
     // Create the ParamData object to give to CLI.
     util::ParamData data;
diff --git a/src/mlpack/core/util/mlpack_main.hpp b/src/mlpack/core/util/mlpack_main.hpp
index d8d2c76803..6762329eec 100644
--- a/src/mlpack/core/util/mlpack_main.hpp
+++ b/src/mlpack/core/util/mlpack_main.hpp
@@ -96,14 +96,7 @@ using Option = mlpack::bindings::tests::TestOption<T>;
 #undef PROGRAM_INFO
 #define PROGRAM_INFO(NAME, DESC) static mlpack::util::ProgramDoc \
     cli_programdoc_dummy_object = mlpack::util::ProgramDoc(NAME, \
-        []() { return DESC; }); \
-    namespace mlpack { \
-    namespace bindings { \
-    namespace tests { \
-    std::string programName = NAME; \
-    } \
-    } \
-    }
+        []() { return DESC; });
 
 #elif(BINDING_TYPE == BINDING_TYPE_PYX) // This is a Python binding.
 
@@ -155,3 +148,5 @@ PARAM_FLAG("verbose", "Display informational messages and the full list of "
 #include "param_checks.hpp"
 
 #endif
+
+
diff --git a/src/mlpack/core/util/param.hpp b/src/mlpack/core/util/param.hpp
index 7252f51bb4..43212ad3ae 100644
--- a/src/mlpack/core/util/param.hpp
+++ b/src/mlpack/core/util/param.hpp
@@ -15,6 +15,24 @@
 #ifndef MLPACK_CORE_UTIL_PARAM_HPP
 #define MLPACK_CORE_UTIL_PARAM_HPP
 
+#include <string>
+
+//PROGRAM_NAME is used for mlpackMain modification (mlpackMain##PROGRAM_NAME)
+//PROGRAM_NAME_SUBSTUTIDE is used for passing unique program name to TestOption
+#ifdef PROGRAM_NAME
+#define PROGRAM_NAME_SUBSTITUDE PROGRAM_NAME
+#else
+#define PROGRAM_NAME
+#define PROGRAM_NAME_SUBSTITUDE programNameSubstitude
+static const std::string programNameSubstitude = "programNameSubstitude";
+#endif
+//The MAIN macro is mlpackMain##PROGRAM_NAME. The goal is to have different
+//names of main procedure for main tests when all method's  main functions
+//are linked into one executable (mlpack_test)
+#define TOKENPASTE1(x, y) x ## y
+#define TOKENPASTE2(x, y) TOKENPASTE1(x, y)
+#define MAIN TOKENPASTE2(mlpackMain, PROGRAM_NAME)
+
 // Required forward declarations.
 namespace mlpack {
 namespace data {
@@ -1012,53 +1030,55 @@ using DatasetInfo = DatasetMapper<IncrementPolicy, std::string>;
   #define PARAM_IN(T, ID, DESC, ALIAS, DEF, REQ) \
       static mlpack::util::Option<T> \
       JOIN(cli_option_dummy_object_in_, __COUNTER__) \
-      (DEF, ID, DESC, ALIAS, #T, REQ, true, false);
+      (DEF, ID, DESC, ALIAS, #T, REQ, true, false, PROGRAM_NAME_SUBSTITUDE);
 
   #define PARAM_OUT(T, ID, DESC, ALIAS, DEF, REQ) \
       static mlpack::util::Option<T> \
       JOIN(cli_option_dummy_object_out_, __COUNTER__) \
-      (DEF, ID, DESC, ALIAS, #T, REQ, false, false);
+      (DEF, ID, DESC, ALIAS, #T, REQ, false, false, PROGRAM_NAME_SUBSTITUDE);
 
   #define PARAM_MATRIX(ID, DESC, ALIAS, REQ, TRANS, IN) \
       static mlpack::util::Option<arma::mat> \
       JOIN(cli_option_dummy_matrix_, __COUNTER__) \
-      (arma::mat(), ID, DESC, ALIAS, "arma::mat", REQ, IN, !TRANS);
+      (arma::mat(), ID, DESC, ALIAS, "arma::mat", \
+      REQ, IN, !TRANS, PROGRAM_NAME_SUBSTITUDE);
 
   #define PARAM_UMATRIX(ID, DESC, ALIAS, REQ, TRANS, IN) \
       static mlpack::util::Option<arma::Mat<size_t>> \
       JOIN(cli_option_dummy_umatrix_, __COUNTER__) \
-      (arma::Mat<size_t>(), ID, DESC, ALIAS, "arma::Mat<size_t>", REQ, IN, \
-      !TRANS);
+      (arma::Mat<size_t>(), ID, DESC, ALIAS, "arma::Mat<size_t>", \
+      REQ, IN, !TRANS, PROGRAM_NAME_SUBSTITUDE);
 
   #define PARAM_COL(ID, DESC, ALIAS, REQ, TRANS, IN) \
       static mlpack::util::Option<arma::vec> \
       JOIN(cli_option_dummy_col_, __COUNTER__) \
-      (arma::vec(), ID, DESC, ALIAS, "arma::vec", REQ, IN, !TRANS);
+      (arma::vec(), ID, DESC, ALIAS, "arma::vec", \
+      REQ, IN, !TRANS, PROGRAM_NAME_SUBSTITUDE);
 
   #define PARAM_UCOL(ID, DESC, ALIAS, REQ, TRANS, IN) \
       static mlpack::util::Option<arma::Col<size_t>> \
       JOIN(cli_option_dummy_ucol_, __COUNTER__) \
-      (arma::Col<size_t>(), ID, DESC, ALIAS, "arma::Col<size_t>", REQ, IN, \
-      !TRANS);
+      (arma::Col<size_t>(), ID, DESC, ALIAS, "arma::Col<size_t>", \
+      REQ, IN, !TRANS, PROGRAM_NAME_SUBSTITUDE);
 
   #define PARAM_ROW(ID, DESC, ALIAS, REQ, TRANS, IN) \
       static mlpack::util::Option<arma::rowvec> \
       JOIN(cli_option_dummy_row_, __COUNTER__) \
-      (arma::rowvec(), ID, DESC, ALIAS, "arma::rowvec", REQ, IN, !TRANS);
+      (arma::rowvec(), ID, DESC, ALIAS, "arma::rowvec", \
+      REQ, IN, !TRANS, PROGRAM_NAME_SUBSTITUDE);
 
   #define PARAM_UROW(ID, DESC, ALIAS, REQ, TRANS, IN) \
       static mlpack::util::Option<arma::Row<size_t>> \
       JOIN(cli_option_dummy_urow_, __COUNTER__) \
-      (arma::Row<size_t>(), ID, DESC, ALIAS, "arma::Row<size_t>", REQ, IN, \
-      !TRANS);
-
+      (arma::Row<size_t>(), ID, DESC, ALIAS, "arma::Row<size_t>", \
+      REQ, IN, !TRANS, PROGRAM_NAME_SUBSTITUDE);
 
   // There are no uses of required models, so that is not an option to this
   // macro (it would be easy to add).
   #define PARAM_MODEL(TYPE, ID, DESC, ALIAS, REQ, IN) \
       static mlpack::util::Option<TYPE> \
       JOIN(cli_option_dummy_model_, __COUNTER__) \
-      (TYPE(), ID, DESC, ALIAS, #TYPE, REQ, IN);
+      (TYPE(), ID, DESC, ALIAS, #TYPE, REQ, IN, false, PROGRAM_NAME_SUBSTITUDE);
 #else
   // We have to do some really bizarre stuff since __COUNTER__ isn't defined. I
   // don't think we can absolutely guarantee success, but it should be "good
@@ -1067,50 +1087,54 @@ using DatasetInfo = DatasetMapper<IncrementPolicy, std::string>;
   #define PARAM_IN(T, ID, DESC, ALIAS, DEF, REQ) \
       static mlpack::util::Option<T> \
       JOIN(JOIN(cli_option_dummy_object_in_, __LINE__), opt) \
-      (DEF, ID, DESC, ALIAS, #T, REQ, true, false);
+      (DEF, ID, DESC, ALIAS, #T, REQ, true, false, PROGRAM_NAME_SUBSTITUDE);
 
   #define PARAM_OUT(T, ID, DESC, ALIAS, DEF, REQ) \
       static mlpack::util::Option<T> \
       JOIN(JOIN(cli_option_dummy_object_out_, __LINE__), opt) \
-      (DEF, ID, DESC, ALIAS, #T, REQ, false, false);
+      (DEF, ID, DESC, ALIAS, #T, REQ, false, false, PROGRAM_NAME_SUBSTITUDE);
 
   #define PARAM_MATRIX(ID, DESC, ALIAS, REQ, TRANS, IN) \
       static mlpack::util::Option<arma::mat> \
       JOIN(JOIN(cli_option_dummy_object_matrix_, __LINE__), opt) \
-      (arma::mat(), ID, DESC, ALIAS, "arma::mat", REQ, IN, !TRANS);
+      (arma::mat(), ID, DESC, ALIAS, "arma::mat", REQ, IN, !TRANS, \
+      PROGRAM_NAME_SUBSTITUDE);
 
   #define PARAM_UMATRIX(ID, DESC, ALIAS, REQ, TRANS, IN) \
       static mlpack::util::Option<arma::Mat<size_t>> \
       JOIN(JOIN(cli_option_dummy_object_umatrix_, __LINE__), opt) \
       (arma::Mat<size_t>(), ID, DESC, ALIAS, "arma::Mat<size_t>", REQ, IN, \
-      !TRANS);
+      !TRANS, PROGRAM_NAME_SUBSTITUDE);
 
   #define PARAM_COL(ID, DESC, ALIAS, REQ, TRANS, IN) \
       static mlpack::util::Option<arma::vec> \
       JOIN(cli_option_dummy_object_col_, __LINE__) \
-      (arma::vec(), ID, DESC, ALIAS, "arma::vec", REQ, IN, !TRANS);
+      (arma::vec(), ID, DESC, ALIAS, "arma::vec", REQ, IN, !TRANS, \
+      PROGRAM_NAME_SUBSTITUDE);
 
   #define PARAM_UCOL(ID, DESC, ALIAS, REQ, TRANS, IN) \
       static mlpack::util::Option<arma::Col<size_t>> \
       JOIN(cli_option_dummy_object_ucol_, __LINE__) \
       (arma::Col<size_t>(), ID, DESC, ALIAS, "arma::Col<size_t>", REQ, IN, \
-      !TRANS);
+      !TRANS, PROGRAM_NAME_SUBSTITUDE);
 
   #define PARAM_ROW(ID, DESC, ALIAS, REQ, TRANS, IN) \
       static mlpack::util::Option<arma::rowvec> \
       JOIN(cli_option_dummy_object_row_, __LINE__) \
-      (arma::rowvec(), ID, DESC, ALIAS, "arma::rowvec", REQ, IN, !TRANS);
+      (arma::rowvec(), ID, DESC, ALIAS, "arma::rowvec", REQ, IN, !TRANS, \
+      PROGRAM_NAME_SUBSTITUDE);
 
   #define PARAM_UROW(ID, DESC, ALIAS, REQ, TRANS, IN) \
       static mlpack::util::Option<arma::Row<size_t>> \
       JOIN(cli_option_dummy_object_urow_, __LINE__) \
       (arma::Row<size_t>(), ID, DESC, ALIAS, "arma::Row<size_t>", REQ, IN, \
-      !TRANS);
+      !TRANS, PROGRAM_NAME_SUBSTITUDE);
 
   #define PARAM_MODEL(TYPE, ID, DESC, ALIAS, REQ, IN) \
       static mlpack::util::Option<TYPE> \
       JOIN(JOIN(cli_option_dummy_object_model_, __LINE__), opt) \
-      (TYPE(), ID, DESC, ALIAS, #TYPE, REQ, IN);
+      (TYPE(), ID, DESC, ALIAS, #TYPE, REQ, IN, false, \
+      PROGRAM_NAME_SUBSTITUDE);
 #endif
 
 #endif
diff --git a/src/mlpack/methods/linear_regression/linear_regression_main.cpp b/src/mlpack/methods/linear_regression/linear_regression_main.cpp
index 8aa006535e..d5df454af0 100644
--- a/src/mlpack/methods/linear_regression/linear_regression_main.cpp
+++ b/src/mlpack/methods/linear_regression/linear_regression_main.cpp
@@ -85,7 +85,7 @@ PARAM_COL_OUT("output_predictions", "If --test_file is specified, this "
 PARAM_DOUBLE_IN("lambda", "Tikhonov regularization for ridge regression.  If 0,"
     " the method reduces to linear regression.", "l", 0.0);
 
-void mlpackMain()
+void MAIN()
 {
   const double lambda = CLI::GetParam<double>("lambda");
 
diff --git a/src/mlpack/methods/pca/pca_main.cpp b/src/mlpack/methods/pca/pca_main.cpp
index 9ad0a9e5cc..e0751d1b9c 100644
--- a/src/mlpack/methods/pca/pca_main.cpp
+++ b/src/mlpack/methods/pca/pca_main.cpp
@@ -99,7 +99,7 @@ void RunPCA(arma::mat& dataset,
       dataset.n_rows << " dimensions)." << endl;
 }
 
-void mlpackMain()
+void MAIN()
 {
   // Load input dataset.
   arma::mat& dataset = CLI::GetParam<arma::mat>("input");
diff --git a/src/mlpack/tests/CMakeLists.txt b/src/mlpack/tests/CMakeLists.txt
index fc6ee57e48..8201906cc1 100644
--- a/src/mlpack/tests/CMakeLists.txt
+++ b/src/mlpack/tests/CMakeLists.txt
@@ -123,6 +123,7 @@ add_executable(mlpack_test
   union_find_test.cpp
   vantage_point_tree_test.cpp
   main_tests/pca_test.cpp
+  main_tests/linear_regression_test.cpp
 )
 
 # Link dependencies of test executable.
diff --git a/src/mlpack/tests/main_tests/linear_regression_test.cpp b/src/mlpack/tests/main_tests/linear_regression_test.cpp
index 061c713367..9eb8b0319c 100644
--- a/src/mlpack/tests/main_tests/linear_regression_test.cpp
+++ b/src/mlpack/tests/main_tests/linear_regression_test.cpp
@@ -4,8 +4,11 @@
  *
  * Test mlpackMain() of linear_regression_main.cpp.
  */
+#include <string>
+
 #define BINDING_TYPE BINDING_TYPE_TEST
 #define PROGRAM_NAME linearRegressionProgramName
+static const std::string linearRegressionProgramName = "LinearRegression";
 
 #include <mlpack/core.hpp>
 #include <mlpack/core/util/mlpack_main.hpp>
@@ -40,7 +43,7 @@ struct LinearRegressionTestFixture
   LinearRegressionTestFixture()
   {
     // Cache in the options for this program.
-    CLI::RestoreSettings(mlpack::bindings::tests::programName);
+    CLI::RestoreSettings(linearRegressionProgramName);
   }
 
   ~LinearRegressionTestFixture()
@@ -52,11 +55,17 @@ struct LinearRegressionTestFixture
 
 BOOST_FIXTURE_TEST_SUITE(LinearRegressionMainTest, LinearRegressionTestFixture);
 
-BOOST_AUTO_TEST_CASE(LinearRegressionWringResponseSizeTest)
+BOOST_AUTO_TEST_CASE(LinearRegressionWrongResponseSizeTest)
 {
-  std::cout << "1\n";
-  SetInputParam("lambda", 1.0);
-  std::cout << "2\n";
+  arma::mat x = arma::randu<arma::mat>(5, 5);
+  arma::rowvec y = arma::randu<arma::rowvec>(4);
+
+  SetInputParam("training", std::move(x));
+  SetInputParam("training_responses", std::move(y));
+
+  Log::Fatal.ignoreInput = true;
+  BOOST_REQUIRE_THROW(MAIN(), std::runtime_error);
+  Log::Fatal.ignoreInput = false;
 }
 
 BOOST_AUTO_TEST_SUITE_END();
diff --git a/src/mlpack/tests/main_tests/pca_test.cpp b/src/mlpack/tests/main_tests/pca_test.cpp
index 5742b7005b..26aaf5983a 100644
--- a/src/mlpack/tests/main_tests/pca_test.cpp
+++ b/src/mlpack/tests/main_tests/pca_test.cpp
@@ -4,7 +4,11 @@
  *
  * Test mlpackMain() of pca_main.cpp.
  */
+#include <string>
+
 #define BINDING_TYPE BINDING_TYPE_TEST
+#define PROGRAM_NAME pcaProgramName
+static const std::string pcaProgramName = "PrincipalComponentAnalysis";
 #include <mlpack/core.hpp>
 #include <mlpack/core/util/mlpack_main.hpp>
 #include <mlpack/methods/pca/pca_main.cpp>
@@ -48,7 +52,7 @@ struct PCATestFixture
   PCATestFixture()
   {
     // Cache in the options for this program.
-    CLI::RestoreSettings(mlpack::bindings::tests::programName);
+    CLI::RestoreSettings(pcaProgramName);
   }
 
   ~PCATestFixture()
@@ -71,7 +75,7 @@ BOOST_AUTO_TEST_CASE(PCADimensionTest)
   SetInputParam("input", std::move(x));
   SetInputParam("new_dimensionality", (int) 3);
 
-  mlpackMain();
+  MAIN();
 
   // Now check that the output has 3 dimensions.
   BOOST_REQUIRE_EQUAL(CLI::GetParam<arma::mat>("output").n_rows, 3);
@@ -91,7 +95,7 @@ BOOST_AUTO_TEST_CASE(PCAVarRetainTest)
   SetInputParam("scale", true);
   SetInputParam("new_dimensionality", (int) 3); // Should be ignored.
 
-  mlpackMain();
+  MAIN();
 
   // Check that the output has 5 dimensions.
   BOOST_REQUIRE_EQUAL(CLI::GetParam<arma::mat>("output").n_rows, 4);
@@ -110,7 +114,7 @@ BOOST_AUTO_TEST_CASE(PCANoVarRetainTest)
   SetInputParam("scale", true);
   SetInputParam("new_dimensionality", (int) 3); // Should be ignored.
 
-  mlpackMain();
+  MAIN();
 
   // Check that the output has 1 dimensions.
   BOOST_REQUIRE_EQUAL(CLI::GetParam<arma::mat>("output").n_rows, 1);
@@ -128,7 +132,7 @@ BOOST_AUTO_TEST_CASE(PCATooHighNewDimensionalityTest)
   SetInputParam("new_dimensionality", (int) 7); // Invalid.
 
   Log::Fatal.ignoreInput = true;
-  BOOST_REQUIRE_THROW(mlpackMain(), std::runtime_error);
+  BOOST_REQUIRE_THROW(MAIN(), std::runtime_error);
   Log::Fatal.ignoreInput = false;
 }
 

From 84ad1fabb5c43d960b395faf7a0dd4efb401b0eb Mon Sep 17 00:00:00 2001
From: Eugene Freyman <evg.freyman@gmail.com>
Date: Sat, 16 Dec 2017 00:38:14 +0200
Subject: [PATCH 23/67] fixed checkstyle issues

---
 src/mlpack/bindings/cli/cli_option.hpp    |  2 +-
 src/mlpack/bindings/python/py_option.hpp  |  2 +-
 src/mlpack/bindings/tests/test_option.hpp |  2 +-
 src/mlpack/core/util/mlpack_main.hpp      |  2 --
 src/mlpack/core/util/param.hpp            | 10 +++++-----
 5 files changed, 8 insertions(+), 10 deletions(-)

diff --git a/src/mlpack/bindings/cli/cli_option.hpp b/src/mlpack/bindings/cli/cli_option.hpp
index 4229230bf1..93003eccc1 100644
--- a/src/mlpack/bindings/cli/cli_option.hpp
+++ b/src/mlpack/bindings/cli/cli_option.hpp
@@ -69,7 +69,7 @@ class CLIOption
             const bool required = false,
             const bool input = true,
             const bool noTranspose = false,
-	    const std::string& /*programName*/ = "")
+            const std::string& /*programName*/ = "")
   {
     // Create the ParamData object to give to CLI.
     util::ParamData data;
diff --git a/src/mlpack/bindings/python/py_option.hpp b/src/mlpack/bindings/python/py_option.hpp
index f4c38d9eb4..e5c4b059d4 100644
--- a/src/mlpack/bindings/python/py_option.hpp
+++ b/src/mlpack/bindings/python/py_option.hpp
@@ -43,7 +43,7 @@ class PyOption
            const bool required = false,
            const bool input = true,
            const bool noTranspose = false,
-	   const std::string& /*programName*/ = "")
+           const std::string& /*programName*/ = "")
   {
     // Create the ParamData object to give to CLI.
     util::ParamData data;
diff --git a/src/mlpack/bindings/tests/test_option.hpp b/src/mlpack/bindings/tests/test_option.hpp
index 1f88b69f58..f80596d580 100644
--- a/src/mlpack/bindings/tests/test_option.hpp
+++ b/src/mlpack/bindings/tests/test_option.hpp
@@ -62,7 +62,7 @@ class TestOption
              const bool required = false,
              const bool input = true,
              const bool noTranspose = false,
-	     const std::string& programName = "")
+             const std::string& programName = "")
   {
     // Create the ParamData object to give to CLI.
     util::ParamData data;
diff --git a/src/mlpack/core/util/mlpack_main.hpp b/src/mlpack/core/util/mlpack_main.hpp
index 6762329eec..bc9846b27c 100644
--- a/src/mlpack/core/util/mlpack_main.hpp
+++ b/src/mlpack/core/util/mlpack_main.hpp
@@ -148,5 +148,3 @@ PARAM_FLAG("verbose", "Display informational messages and the full list of "
 #include "param_checks.hpp"
 
 #endif
-
-
diff --git a/src/mlpack/core/util/param.hpp b/src/mlpack/core/util/param.hpp
index 43212ad3ae..ad23e42ac0 100644
--- a/src/mlpack/core/util/param.hpp
+++ b/src/mlpack/core/util/param.hpp
@@ -17,8 +17,8 @@
 
 #include <string>
 
-//PROGRAM_NAME is used for mlpackMain modification (mlpackMain##PROGRAM_NAME)
-//PROGRAM_NAME_SUBSTUTIDE is used for passing unique program name to TestOption
+// PROGRAM_NAME is used for mlpackMain modification (mlpackMain##PROGRAM_NAME)
+// PROGRAM_NAME_SUBSTUTIDE is used for passing unique program name to TestOption
 #ifdef PROGRAM_NAME
 #define PROGRAM_NAME_SUBSTITUDE PROGRAM_NAME
 #else
@@ -26,9 +26,9 @@
 #define PROGRAM_NAME_SUBSTITUDE programNameSubstitude
 static const std::string programNameSubstitude = "programNameSubstitude";
 #endif
-//The MAIN macro is mlpackMain##PROGRAM_NAME. The goal is to have different
-//names of main procedure for main tests when all method's  main functions
-//are linked into one executable (mlpack_test)
+// The MAIN macro is mlpackMain##PROGRAM_NAME. The goal is to have different
+// names of main procedure for main tests when all method's  main functions
+// are linked into one executable (mlpack_test)
 #define TOKENPASTE1(x, y) x ## y
 #define TOKENPASTE2(x, y) TOKENPASTE1(x, y)
 #define MAIN TOKENPASTE2(mlpackMain, PROGRAM_NAME)

From 19ff6e8fc0d6b59f9ca4fa9686526cc8cc8a1fed Mon Sep 17 00:00:00 2001
From: Sourabh Varshney <sourabhvarshney111@gmail.com>
Date: Sat, 16 Dec 2017 11:00:38 +0530
Subject: [PATCH 24/67] Made the quoted changes

---
 .../core/optimizers/adam/nadam_update.hpp     | 63 +++++++++++--------
 1 file changed, 36 insertions(+), 27 deletions(-)

diff --git a/src/mlpack/core/optimizers/adam/nadam_update.hpp b/src/mlpack/core/optimizers/adam/nadam_update.hpp
index af5ee555ac..e6e1e46793 100644
--- a/src/mlpack/core/optimizers/adam/nadam_update.hpp
+++ b/src/mlpack/core/optimizers/adam/nadam_update.hpp
@@ -24,12 +24,12 @@ namespace optimization {
  * For more information, see the following.
  *
  * @code
- * @misc{
- *   author  = {},
- *   title   = {},
- *   journal = {},
- *   year    = {},
- *   url     = {}
+ * @techreport{Dozat2015,
+ *   title       = {Incorporating Nesterov momentum into Adam},
+ *   author      = {Timothy Dozat},
+ *   institution = {Stanford University},
+ *   address     = {Stanford},
+ *   year        = {2015}
  * }
  * @endcode
  */
@@ -46,8 +46,13 @@ class NadamUpdate
    */
   NadamUpdate(const double epsilon = 1e-8,
               const double beta1 = 0.9,
-              const double beta2 = 0.99)
-              :epsilon(epsilon), beta1(beta1), beta2(beta2), iteration(0),
+              const double beta2 = 0.99,
+              const double scheduleDecay = 4e-3)
+              :epsilon(epsilon),
+              beta1(beta1),
+              beta2(beta2),
+              scheduleDecay(scheduleDecay),
+              iteration(0),
               cum_beta1(1)
   {
     // Nothing to do.
@@ -87,28 +92,24 @@ class NadamUpdate
     v *= beta2;
     v += (1 - beta2) * gradient % gradient;
 
-    // beta1_t = beta1 * (1 - (0.5 * (0.96 ^ (iteration / 250))))
-    double beta1_t = beta1 * (1 - (0.5 * std::pow(0.96, (iteration / 250))));
+    double beta1T = beta1 * (1 - (0.5 * 
+        std::pow(0.96, iteration * scheduleDecay)));
 
-    // beta1_t1 = beta1 * (1 - (0.5 * (0.96 ^ ((iteration + 1)/ 250))))
-    double beta1_t1 = beta1 * (1 - (0.5 *
-                      std::pow(0.96, ((iteration + 1) / 250))));
+    double beta1T1 = beta1 * (1 - (0.5 *
+        std::pow(0.96, (iteration + 1) * scheduleDecay)));
 
-    // cum_beta1 *= beta1_t
-    cum_beta1 *= beta1_t;
+    cumBeta1 *= beta1T;
 
-    // biasCorrection = 1 - cum_beta1
-    const double biasCorrection1 = 1.0 - cum_beta1;
+    const double biasCorrection1 = 1.0 - cumBeta1;
 
-    // biasCorrection2 = 1 - beta2 ^ iteration
     const double biasCorrection2 = 1.0 - std::pow(beta2, iteration);
 
     /* Note :- arma::sqrt(v) + epsilon * sqrt(biasCorrection2) is approximated as
      * arma::sqrt(v) + epsilon
      */
-    iterate -= (stepSize * ((1 - beta1_t) * gradient +beta1_t1 * m)
-               * sqrt(biasCorrection2)) / ((arma::sqrt(v) + epsilon)
-               * biasCorrection1);
+    iterate -= (stepSize * ((1 - beta1T) * gradient +beta1T1 * m)
+        * sqrt(biasCorrection2)) / ((arma::sqrt(v) + epsilon)
+        * biasCorrection1);
   }
 
   //! Get the value used to initialise the squared gradient parameter.
@@ -116,10 +117,10 @@ class NadamUpdate
   //! Modify the value used to initialise the squared gradient parameter.
   double& Epsilon() { return epsilon; }
 
-  //! Get the value of the cumulative product of decay constants
-  double Cum_beta1() const { return cum_beta1; }
-  //! Modify the value of the cumulative product of decay constants
-  double& Cum_beta1() { return cum_beta1; }
+  //! Get the value of the cumulative product of decay coefficients
+  double CumBeta1() const { return cumBeta1; }
+  //! Modify the value of the cumulative product of decay coefficients
+  double& CumBeta1() { return cumBeta1; }
 
   //! Get the smoothing parameter.
   double Beta1() const { return beta1; }
@@ -131,6 +132,11 @@ class NadamUpdate
   //! Modify the second moment coefficient.
   double& Beta2() { return beta2; }
 
+  //! Get the dacay parameter for decay coefficients
+  double ScheduleDecay() const { return scheduleDecay; }
+  //! Modify the dacay parameter for decay coefficients
+  double& ScheduleDecay() { return scheduleDecay; }
+
  private:
   // The epsilon value used to initialise the squared gradient parameter.
   double epsilon;
@@ -147,8 +153,11 @@ class NadamUpdate
   // The exponential moving average of squared gradient values.
   arma::mat v;
 
-  // The cumulative product of decay constants
-  double cum_beta1;
+  // The cumulative product of decay coefficients
+  double cumBeta1;
+
+  // The decay parameter for decay coefficients
+  double scheduleDecay;
 
   // The number of iterations.
   double iteration;

From ba66ef08c250c1486f6956b48f3ab6eda57770e4 Mon Sep 17 00:00:00 2001
From: Sourabh Varshney <sourabhvarshney111@gmail.com>
Date: Sat, 16 Dec 2017 11:04:15 +0530
Subject: [PATCH 25/67] Removed extra whitespace

---
 src/mlpack/core/optimizers/adam/nadam_update.hpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/mlpack/core/optimizers/adam/nadam_update.hpp b/src/mlpack/core/optimizers/adam/nadam_update.hpp
index e6e1e46793..b54fc6ca1c 100644
--- a/src/mlpack/core/optimizers/adam/nadam_update.hpp
+++ b/src/mlpack/core/optimizers/adam/nadam_update.hpp
@@ -92,7 +92,7 @@ class NadamUpdate
     v *= beta2;
     v += (1 - beta2) * gradient % gradient;
 
-    double beta1T = beta1 * (1 - (0.5 * 
+    double beta1T = beta1 * (1 - (0.5 *
         std::pow(0.96, iteration * scheduleDecay)));
 
     double beta1T1 = beta1 * (1 - (0.5 *

From 1705902e918d0d6e1e5f31460796e3b863bdca22 Mon Sep 17 00:00:00 2001
From: Sourabh Varshney <sourabhvarshney111@gmail.com>
Date: Sat, 16 Dec 2017 11:12:26 +0530
Subject: [PATCH 26/67] Removed Typo Error

---
 src/mlpack/core/optimizers/adam/nadam_update.hpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/mlpack/core/optimizers/adam/nadam_update.hpp b/src/mlpack/core/optimizers/adam/nadam_update.hpp
index b54fc6ca1c..eeb3757d42 100644
--- a/src/mlpack/core/optimizers/adam/nadam_update.hpp
+++ b/src/mlpack/core/optimizers/adam/nadam_update.hpp
@@ -53,7 +53,7 @@ class NadamUpdate
               beta2(beta2),
               scheduleDecay(scheduleDecay),
               iteration(0),
-              cum_beta1(1)
+              cumBeta1(1)
   {
     // Nothing to do.
   }

From 70efb6886fdac7a2d79f6155df2569df4c2cfb99 Mon Sep 17 00:00:00 2001
From: Sourabh Varshney <sourabhvarshney111@gmail.com>
Date: Sat, 16 Dec 2017 18:51:12 +0530
Subject: [PATCH 27/67] Some more style changes

---
 .../core/optimizers/adam/nadam_update.hpp       | 17 +++++++++--------
 1 file changed, 9 insertions(+), 8 deletions(-)

diff --git a/src/mlpack/core/optimizers/adam/nadam_update.hpp b/src/mlpack/core/optimizers/adam/nadam_update.hpp
index eeb3757d42..752ba87937 100644
--- a/src/mlpack/core/optimizers/adam/nadam_update.hpp
+++ b/src/mlpack/core/optimizers/adam/nadam_update.hpp
@@ -43,17 +43,18 @@ class NadamUpdate
    *        parameter.
    * @param beta1 The smoothing parameter.
    * @param beta2 The second moment coefficient
+   * @param scheduleDecay The decay parameter for decay coefficients
    */
   NadamUpdate(const double epsilon = 1e-8,
               const double beta1 = 0.9,
               const double beta2 = 0.99,
               const double scheduleDecay = 4e-3)
-              :epsilon(epsilon),
-              beta1(beta1),
-              beta2(beta2),
-              scheduleDecay(scheduleDecay),
-              iteration(0),
-              cumBeta1(1)
+      :epsilon(epsilon),
+      beta1(beta1),
+      beta2(beta2),
+      scheduleDecay(scheduleDecay),
+      iteration(0),
+      cumBeta1(1)
   {
     // Nothing to do.
   }
@@ -132,9 +133,9 @@ class NadamUpdate
   //! Modify the second moment coefficient.
   double& Beta2() { return beta2; }
 
-  //! Get the dacay parameter for decay coefficients
+  //! Get the decay parameter for decay coefficients
   double ScheduleDecay() const { return scheduleDecay; }
-  //! Modify the dacay parameter for decay coefficients
+  //! Modify the decay parameter for decay coefficients
   double& ScheduleDecay() { return scheduleDecay; }
 
  private:

From 1aaf6d27e3ed6c309e0731f8e49f3715b7f26f87 Mon Sep 17 00:00:00 2001
From: Sourabh Varshney <sourabhvarshney111@gmail.com>
Date: Sun, 17 Dec 2017 11:31:06 +0530
Subject: [PATCH 28/67] Added some correction in implementation

---
 src/mlpack/core/optimizers/adam/nadam_update.hpp | 16 +++++++++-------
 1 file changed, 9 insertions(+), 7 deletions(-)

diff --git a/src/mlpack/core/optimizers/adam/nadam_update.hpp b/src/mlpack/core/optimizers/adam/nadam_update.hpp
index 752ba87937..e22a559ec6 100644
--- a/src/mlpack/core/optimizers/adam/nadam_update.hpp
+++ b/src/mlpack/core/optimizers/adam/nadam_update.hpp
@@ -2,8 +2,8 @@
  * @file nadam_update.hpp
  * @author Sourabh Varshney
  *
- * Nadam update rule. Nadam is an optimizer that combines the effect of Adam and 
- * NAG to the gradient descent to improve its Performance.
+ * Nadam update rule. Nadam is an optimizer that combines the effect of Adam
+ * and NAG to the gradient descent to improve its Performance.
  *
  * mlpack is free software; you may redistribute it and/or modify it under the
  * terms of the 3-clause BSD license.  You should have received a copy of the
@@ -105,12 +105,14 @@ class NadamUpdate
 
     const double biasCorrection2 = 1.0 - std::pow(beta2, iteration);
 
-    /* Note :- arma::sqrt(v) + epsilon * sqrt(biasCorrection2) is approximated as
-     * arma::sqrt(v) + epsilon
+    const double biasCorrection3 = 1.0 - (cumBeta1 * beta1T1);
+
+    /* Note :- arma::sqrt(v) + epsilon * sqrt(biasCorrection2) is approximated
+     * as arma::sqrt(v) + epsilon
      */
-    iterate -= (stepSize * ((1 - beta1T) * gradient +beta1T1 * m)
-        * sqrt(biasCorrection2)) / ((arma::sqrt(v) + epsilon)
-        * biasCorrection1);
+    iterate -= (stepSize * (((1 - beta1T) / biasCorrection1) * gradient
+        + (beta1T1 / biasCorrection3) * m) * sqrt(biasCorrection2))
+        / (arma::sqrt(v) + epsilon);
   }
 
   //! Get the value used to initialise the squared gradient parameter.

From 211346f4280f0053c3f342cb35af1ba03fc20fe3 Mon Sep 17 00:00:00 2001
From: Ryan Curtin <ryan@ratml.org>
Date: Mon, 18 Dec 2017 14:47:26 -0500
Subject: [PATCH 29/67] Refactor batch size tests.  This meant I needed to add
 RNN::Reset().

---
 src/mlpack/methods/ann/rnn.hpp              |  9 +++-
 src/mlpack/methods/ann/rnn_impl.hpp         | 15 ++++--
 src/mlpack/tests/recurrent_network_test.cpp | 53 ++++++++-------------
 3 files changed, 40 insertions(+), 37 deletions(-)

diff --git a/src/mlpack/methods/ann/rnn.hpp b/src/mlpack/methods/ann/rnn.hpp
index 8dc6507922..6cadc40d73 100644
--- a/src/mlpack/methods/ann/rnn.hpp
+++ b/src/mlpack/methods/ann/rnn.hpp
@@ -222,7 +222,14 @@ class RNN
   size_t& Rho() { return rho; }
 
   /**
-   * Reset the module infomration (weights/parameters).
+   * Reset the state of the network.  This ensures that all internally-held
+   * gradients are set to 0, all memory cells are reset, and the parameters
+   * matrix is the right size.
+   */
+  void Reset();
+
+  /**
+   * Reset the module information (weights/parameters).
    */
   void ResetParameters();
 
diff --git a/src/mlpack/methods/ann/rnn_impl.hpp b/src/mlpack/methods/ann/rnn_impl.hpp
index d7c0269687..d745323f85 100644
--- a/src/mlpack/methods/ann/rnn_impl.hpp
+++ b/src/mlpack/methods/ann/rnn_impl.hpp
@@ -102,7 +102,6 @@ void RNN<OutputLayerType, InitializationRuleType>::Train(
   if (!reset)
   {
     ResetParameters();
-    reset = true;
   }
 
   // Train the model.
@@ -139,7 +138,6 @@ void RNN<OutputLayerType, InitializationRuleType>::Train(
   if (!reset)
   {
     ResetParameters();
-    reset = true;
   }
 
   OptimizerType optimizer;
@@ -207,7 +205,6 @@ double RNN<OutputLayerType, InitializationRuleType>::Evaluate(
   if (parameter.is_empty())
   {
     ResetParameters();
-    reset = true;
   }
 
   if (deterministic != this->deterministic)
@@ -272,7 +269,6 @@ void RNN<OutputLayerType, InitializationRuleType>::Gradient(
     if (parameter.is_empty())
     {
       ResetParameters();
-      reset = true;
     }
 
     gradient = arma::zeros<arma::mat>(parameter.n_rows, parameter.n_cols);
@@ -337,6 +333,17 @@ void RNN<OutputLayerType, InitializationRuleType>::ResetParameters()
   // Reset the network parameter with the given initialization rule.
   NetworkInitialization<InitializationRuleType> networkInit(initializeRule);
   networkInit.Initialize(network, parameter);
+
+  reset = true;
+}
+
+template<typename OutputLayerType, typename InitializationRuleType>
+void RNN<OutputLayerType, InitializationRuleType>::Reset()
+{
+  ResetParameters();
+  ResetCells();
+  currentGradient.zeros();
+  ResetGradients(currentGradient);
 }
 
 template<typename OutputLayerType, typename InitializationRuleType>
diff --git a/src/mlpack/tests/recurrent_network_test.cpp b/src/mlpack/tests/recurrent_network_test.cpp
index 600f2544d2..805d25d759 100644
--- a/src/mlpack/tests/recurrent_network_test.cpp
+++ b/src/mlpack/tests/recurrent_network_test.cpp
@@ -745,12 +745,11 @@ BOOST_AUTO_TEST_CASE(GRUDistractedSequenceRecallTest)
 }
 
 /**
- * Create a simple recurrent neural network for the noisy sines task, and ensure
- * that it achieves adequate performance when training with the specified batch
- * size.
+ * Create a simple recurrent neural network for the noisy sines task, and
+ * require that it produces the exact same network for a few batch sizes.
  */
 template<typename RecurrentLayerType>
-void BatchSizeTest(const size_t batchSize)
+void BatchSizeTest()
 {
   const size_t rho = 10;
 
@@ -775,34 +774,27 @@ void BatchSizeTest(const size_t batchSize)
   model.Add<Linear<>>(10, 10);
   model.Add<SigmoidLayer<>>();
 
-  StandardSGD opt(0.1, batchSize, 500 * input.n_cols, -100);
+  model.Reset();
+  arma::mat initParams = model.Parameters();
+
+  StandardSGD opt(1e-5, 1, 5, -100, false);
   model.Train(input, labels, opt);
 
-  arma::mat prediction;
-  model.Predict(input, prediction);
+  // This is trained with one point.
+  arma::mat outputParams = model.Parameters();
 
-  size_t error = 0;
-  for (size_t i = 0; i < prediction.n_cols; ++i)
-  {
-    arma::mat singlePrediction = prediction.submat((rho - 1) * rho, i,
-        rho * rho - 1, i);
+  model.Reset();
+  model.Parameters() = initParams;
+  opt.BatchSize() = 2;
+  model.Train(input, labels, opt);
 
-    const int predictionValue = arma::as_scalar(arma::find(
-        arma::max(singlePrediction.col(0)) ==
-        singlePrediction.col(0), 1) + 1);
+  CheckMatrices(outputParams, model.Parameters(), 1);
 
-    const int targetValue = arma::as_scalar(arma::find(
-        arma::max(labelsTemp.col(i)) == labelsTemp.col(i), 1)) + 1;
+  model.Parameters() = initParams;
+  opt.BatchSize() = 5;
+  model.Train(input, labels, opt);
 
-    if (predictionValue == targetValue)
-    {
-      error++;
-    }
-  }
-
-  double classificationError = 1 - double(error) / prediction.n_cols;
-
-  BOOST_REQUIRE_LE(classificationError, 0.2);
+  CheckMatrices(outputParams, model.Parameters(), 1);
 }
 
 /**
@@ -810,8 +802,7 @@ void BatchSizeTest(const size_t batchSize)
  */
 BOOST_AUTO_TEST_CASE(LSTMBatchSizeTest)
 {
-  BatchSizeTest<LSTM<>>(2);
-  BatchSizeTest<LSTM<>>(50);
+  BatchSizeTest<LSTM<>>();
 }
 
 /**
@@ -819,8 +810,7 @@ BOOST_AUTO_TEST_CASE(LSTMBatchSizeTest)
  */
 BOOST_AUTO_TEST_CASE(FastLSTMBatchSizeTest)
 {
-  BatchSizeTest<FastLSTM<>>(2);
-  BatchSizeTest<FastLSTM<>>(50);
+  BatchSizeTest<FastLSTM<>>();
 }
 
 /**
@@ -828,8 +818,7 @@ BOOST_AUTO_TEST_CASE(FastLSTMBatchSizeTest)
  */
 BOOST_AUTO_TEST_CASE(GRUBatchSizeTest)
 {
-  BatchSizeTest<GRU<>>(2);
-  BatchSizeTest<GRU<>>(50);
+  BatchSizeTest<GRU<>>();
 }
 
 /**

From 45d580ddc15075e2d735f3178054b32b61840cb4 Mon Sep 17 00:00:00 2001
From: Ryan Curtin <ryan@ratml.org>
Date: Mon, 18 Dec 2017 14:47:56 -0500
Subject: [PATCH 30/67] Handle case where batchSize > maxIterations. Don't do
 extra iterations!

---
 src/mlpack/core/optimizers/sgd/sgd_impl.hpp | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/mlpack/core/optimizers/sgd/sgd_impl.hpp b/src/mlpack/core/optimizers/sgd/sgd_impl.hpp
index 5e4423afcf..ff004f1c1f 100644
--- a/src/mlpack/core/optimizers/sgd/sgd_impl.hpp
+++ b/src/mlpack/core/optimizers/sgd/sgd_impl.hpp
@@ -103,7 +103,8 @@ double SGD<UpdatePolicyType, DecayPolicyType>::Optimize(
     }
 
     // Find the effective batch size (the last batch may be smaller).
-    const size_t effectiveBatchSize = std::min(batchSize,
+    const size_t effectiveBatchSize = std::min(
+        std::min(batchSize, actualMaxIterations - i),
         numFunctions - currentFunction);
 
     function.Gradient(iterate, currentFunction, gradient, effectiveBatchSize);

From 735d2b97a1df880b51b6942f1af872436fcda531 Mon Sep 17 00:00:00 2001
From: Ryan Curtin <ryan@ratml.org>
Date: Mon, 18 Dec 2017 14:48:17 -0500
Subject: [PATCH 31/67] Reset size of internal parameters whenever batch size
 changes.

We can call set_size() even when the batch size decreased---Armadillo will reuse
memory if we shrink the matrix size (if it can).
---
 src/mlpack/methods/ann/layer/fast_lstm_impl.hpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/mlpack/methods/ann/layer/fast_lstm_impl.hpp b/src/mlpack/methods/ann/layer/fast_lstm_impl.hpp
index 349ece44e5..36d92c2a0d 100644
--- a/src/mlpack/methods/ann/layer/fast_lstm_impl.hpp
+++ b/src/mlpack/methods/ann/layer/fast_lstm_impl.hpp
@@ -80,7 +80,7 @@ void FastLSTM<InputDataType, OutputDataType>::ResetCell(const size_t size)
   gradientStep = batchSize * size - 1;
 
   const size_t rhoBatchSize = size * batchSize;
-  if (gate.is_empty() || gate.n_cols < rhoBatchSize)
+  if (gate.is_empty() || gate.n_cols != rhoBatchSize)
   {
     gate.set_size(4 * outSize, rhoBatchSize);
     gateActivation.set_size(outSize * 3, rhoBatchSize);

From 0e0201a5f55ecfc80d4c5bb9415dfbc46a0f225b Mon Sep 17 00:00:00 2001
From: Ryan Curtin <ryan@ratml.org>
Date: Mon, 18 Dec 2017 14:49:28 -0500
Subject: [PATCH 32/67] Remove debugging output from GRU.

---
 src/mlpack/methods/ann/layer/gru_impl.hpp | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/src/mlpack/methods/ann/layer/gru_impl.hpp b/src/mlpack/methods/ann/layer/gru_impl.hpp
index 07a09b6584..86c6b85414 100644
--- a/src/mlpack/methods/ann/layer/gru_impl.hpp
+++ b/src/mlpack/methods/ann/layer/gru_impl.hpp
@@ -92,8 +92,6 @@ template<typename eT>
 void GRU<InputDataType, OutputDataType>::Forward(
     arma::Mat<eT>&& input, arma::Mat<eT>&& output)
 {
-  std::cout << "GRU::Forward(): input " << input.n_rows << "x" << input.n_cols
-<< "; batchSize " << batchSize << "; forwardStep " << forwardStep << "\n";
   if (input.n_cols != batchSize)
   {
     batchSize = input.n_cols;
@@ -209,8 +207,6 @@ template<typename eT>
 void GRU<InputDataType, OutputDataType>::Backward(
   const arma::Mat<eT>&& input, arma::Mat<eT>&& gy, arma::Mat<eT>&& g)
 {
-  std::cout << "GRU::Backward(): input " << input.n_rows << "x" << input.n_cols
-<< "; batchSize " << batchSize << "; backwardStep " << backwardStep << "\n";
   if (input.n_cols != batchSize)
   {
     batchSize = input.n_cols;
@@ -327,8 +323,6 @@ void GRU<InputDataType, OutputDataType>::Gradient(
     arma::Mat<eT>&& /* error */,
     arma::Mat<eT>&& /* gradient */)
 {
-  std::cout << "GRU::Gradient(): input " << input.n_rows << "x" << input.n_cols
-<< "; batchSize " << batchSize << "; gradientStep " << gradientStep << "\n";
   if (input.n_cols != batchSize)
   {
     batchSize = input.n_cols;

From aa83402f7e8529f6577d301fddca09f33b97198c Mon Sep 17 00:00:00 2001
From: Ryan Curtin <ryan@ratml.org>
Date: Mon, 18 Dec 2017 16:20:50 -0500
Subject: [PATCH 33/67] Fix too-long lines.

---
 src/mlpack/methods/ann/layer/lstm_impl.hpp | 18 ++++++++++--------
 1 file changed, 10 insertions(+), 8 deletions(-)

diff --git a/src/mlpack/methods/ann/layer/lstm_impl.hpp b/src/mlpack/methods/ann/layer/lstm_impl.hpp
index d14082597f..ea06e8e616 100644
--- a/src/mlpack/methods/ann/layer/lstm_impl.hpp
+++ b/src/mlpack/methods/ann/layer/lstm_impl.hpp
@@ -188,12 +188,12 @@ void LSTM<InputDataType, OutputDataType>::Forward(
   if (forwardStep > 0)
   {
     inputGate.cols(forwardStep, forwardStep + batchStep) +=
-        arma::repmat(cell2GateInputWeight, 1, batchSize) % cell.cols(forwardStep - batchSize,
-        forwardStep - batchSize + batchStep);
+        arma::repmat(cell2GateInputWeight, 1, batchSize) %
+        cell.cols(forwardStep - batchSize, forwardStep - batchSize + batchStep);
 
     forgetGate.cols(forwardStep, forwardStep + batchStep) +=
-        arma::repmat(cell2GateForgetWeight, 1, batchSize) % cell.cols(forwardStep - batchSize,
-        forwardStep - batchSize + batchStep);
+        arma::repmat(cell2GateForgetWeight, 1, batchSize) %
+        cell.cols(forwardStep - batchSize, forwardStep - batchSize + batchStep);
   }
 
   inputGateActivation.cols(forwardStep, forwardStep + batchStep) = 1.0 /
@@ -398,12 +398,14 @@ void LSTM<InputDataType, OutputDataType>::Gradient(
   if (gradientStep > batchStep)
   {
     gradient.submat(offset, 0, offset + cell2GateForgetWeight.n_elem - 1, 0) =
-        arma::sum(forgetGateError % cell.cols((gradientStep - batchSize) - batchStep,
-        (gradientStep - batchSize)), 1);
+        arma::sum(forgetGateError %
+                  cell.cols((gradientStep - batchSize) - batchStep,
+                            (gradientStep - batchSize)), 1);
     gradient.submat(offset + cell2GateForgetWeight.n_elem, 0, offset +
         cell2GateForgetWeight.n_elem + cell2GateInputWeight.n_elem - 1, 0) =
-        arma::sum(inputGateError % cell.cols((gradientStep - batchSize) - batchStep,
-        (gradientStep - batchSize)), 1);
+        arma::sum(inputGateError %
+                  cell.cols((gradientStep - batchSize) - batchStep,
+                            (gradientStep - batchSize)), 1);
   }
   else
   {

From 534bc65ee799aa0ddd2af553167a17d51ff53def Mon Sep 17 00:00:00 2001
From: Sourabh Varshney <sourabhvarshney111@gmail.com>
Date: Tue, 19 Dec 2017 21:33:42 +0530
Subject: [PATCH 34/67] Added citation url

---
 src/mlpack/core/optimizers/adam/nadam_update.hpp | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/mlpack/core/optimizers/adam/nadam_update.hpp b/src/mlpack/core/optimizers/adam/nadam_update.hpp
index e22a559ec6..0e6a017327 100644
--- a/src/mlpack/core/optimizers/adam/nadam_update.hpp
+++ b/src/mlpack/core/optimizers/adam/nadam_update.hpp
@@ -29,7 +29,8 @@ namespace optimization {
  *   author      = {Timothy Dozat},
  *   institution = {Stanford University},
  *   address     = {Stanford},
- *   year        = {2015}
+ *   year        = {2015},
+ *   url         = {https://openreview.net/pdf?id=OM0jvwB8jIp57ZJjtNEZ}
  * }
  * @endcode
  */

From 51e7a2a01ff513006103867b642eaa6517f80f9f Mon Sep 17 00:00:00 2001
From: Haritha <haritha1313@gmail.com>
Date: Wed, 13 Dec 2017 13:08:24 +0530
Subject: [PATCH 35/67] ams_grad optimizer implementation

---
 .../core/optimizers/adam/CMakeLists.txt       |   1 +
 src/mlpack/core/optimizers/adam/adam.hpp      |  21 ++-
 src/mlpack/core/optimizers/adam/adam_impl.hpp |   2 +-
 .../core/optimizers/adam/amsgrad_update.hpp   | 145 ++++++++++++++++++
 src/mlpack/tests/adam_test.cpp                |  76 ++++++++-
 5 files changed, 237 insertions(+), 8 deletions(-)
 create mode 100644 src/mlpack/core/optimizers/adam/amsgrad_update.hpp

diff --git a/src/mlpack/core/optimizers/adam/CMakeLists.txt b/src/mlpack/core/optimizers/adam/CMakeLists.txt
index 1377bbb0e1..644797090f 100644
--- a/src/mlpack/core/optimizers/adam/CMakeLists.txt
+++ b/src/mlpack/core/optimizers/adam/CMakeLists.txt
@@ -3,6 +3,7 @@ set(SOURCES
   adam_impl.hpp
   adam_update.hpp
   adamax_update.hpp
+  amsgrad_update.hpp
 )
 
 set(DIR_SRCS)
diff --git a/src/mlpack/core/optimizers/adam/adam.hpp b/src/mlpack/core/optimizers/adam/adam.hpp
index 3b109660f5..8548d6781a 100644
--- a/src/mlpack/core/optimizers/adam/adam.hpp
+++ b/src/mlpack/core/optimizers/adam/adam.hpp
@@ -5,10 +5,11 @@
  * @author Marcus Edel
  * @author Vivek Pal
  *
- * Adam and AdaMax optimizer. Adam is an an algorithm for first-order gradient-
- * -based optimization of stochastic objective functions, based on adaptive
- * estimates of lower-order moments. AdaMax is simply a variant of Adam based
- * on the infinity norm.
+ * Adam, AdaMax and AMSGrad optimizers. Adam is an an algorithm for 
+ * first-order gradient-based optimization of stochastic objective 
+ * functions, based on adaptive estimates of lower-order moments. 
+ * AdaMax is simply a variant of Adam based on the infinity norm. 
+ * AMSGrad is another variant of Adam with guaranteed convergence.
  *
  * mlpack is free software; you may redistribute it and/or modify it under the
  * terms of the 3-clause BSD license.  You should have received a copy of the
@@ -23,6 +24,7 @@
 #include <mlpack/core/optimizers/sgd/sgd.hpp>
 #include "adam_update.hpp"
 #include "adamax_update.hpp"
+#include "amsgrad_update.hpp"
 
 namespace mlpack {
 namespace optimization {
@@ -43,11 +45,16 @@ namespace optimization {
  *   year    = {2014},
  *   url     = {http://arxiv.org/abs/1412.6980}
  * }
+ * @article{
+ *   title   = {On the convergence of Adam and beyond},
+ *   url     = {https://openreview.net/pdf?id=ryQu7f-RZ}
+ *   year    = {2018}
+ * }
  * @endcode
  *
  *
- * For Adam and AdaMax to work, a DecomposableFunctionType template parameter
- * is required. This class must implement the following function:
+ * For Adam, AdaMax and AMSGrad to work, a DecomposableFunctionType template
+ * parameter is required. This class must implement the following function:
  *
  *   size_t NumFunctions();
  *   double Evaluate(const arma::mat& coordinates,
@@ -166,6 +173,8 @@ using Adam = AdamType<AdamUpdate>;
 
 using AdaMax = AdamType<AdaMaxUpdate>;
 
+using AMSGrad = AdamType<AMSGradUpdate>;
+
 } // namespace optimization
 } // namespace mlpack
 
diff --git a/src/mlpack/core/optimizers/adam/adam_impl.hpp b/src/mlpack/core/optimizers/adam/adam_impl.hpp
index 326d038d15..49497f5af5 100644
--- a/src/mlpack/core/optimizers/adam/adam_impl.hpp
+++ b/src/mlpack/core/optimizers/adam/adam_impl.hpp
@@ -5,7 +5,7 @@
  * @author Marcus Edel
  * @author Vivek Pal
  *
- * Implementation of the Adam and AdaMax optimizer.
+ * Implementation of the Adam, AdaMax and AMSGrad optimizer.
  *
  * mlpack is free software; you may redistribute it and/or modify it under the
  * terms of the 3-clause BSD license.  You should have received a copy of the
diff --git a/src/mlpack/core/optimizers/adam/amsgrad_update.hpp b/src/mlpack/core/optimizers/adam/amsgrad_update.hpp
new file mode 100644
index 0000000000..ae5ac681a6
--- /dev/null
+++ b/src/mlpack/core/optimizers/adam/amsgrad_update.hpp
@@ -0,0 +1,145 @@
+/**
+ * @file amsgrad_update.hpp
+ * @author Haritha Nair
+ *
+ * Implementation of AMSGrad optimizer. AMSGrad is an exponential moving average 
+ * optimizer that dynamically adapts over time with guaranteed convergence.
+ *
+ * mlpack is free software; you may redistribute it and/or modify it under the
+ * terms of the 3-clause BSD license.  You should have received a copy of the
+ * 3-clause BSD license along with mlpack.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef MLPACK_CORE_OPTIMIZERS_AMS_GRAD_AMS_GRAD_UPDATE_HPP
+#define MLPACK_CORE_OPTIMIZERS_AMS_GRAD_AMS_GRAD_UPDATE_HPP
+
+#include <mlpack/prereqs.hpp>
+
+namespace mlpack {
+namespace optimization {
+
+/**
+ * AMSGrad is an exponential moving average variant which along with having 
+ * benefits of optimizers like Adam and RMSProp, also guarantees convergence.
+ * Unlike Adam, it uses maximum of past squared gradients rather than their 
+ * exponential average for updation.
+ *
+ * For more information, see the following.
+ *
+ * @code
+ * @article{
+ *   title   = {On the convergence of Adam and beyond},
+ *   url     = {https://openreview.net/pdf?id=ryQu7f-RZ}
+ *   year    = {2018}
+ * }
+ * @endcode
+ */
+class AMSGradUpdate
+{
+ public:
+  /**
+   * Construct the AMSGrad update policy with the given parameters.
+   *
+   * @param epsilon The epsilon value used to initialise the squared gradient
+   *        parameter.
+   * @param beta1 The smoothing parameter.
+   * @param beta2 The second moment coefficient.
+   */
+  AMSGradUpdate(const double epsilon = 1e-8,
+                const double beta1 = 0.9,
+                const double beta2 = 0.999) :
+    epsilon(epsilon),
+    beta1(beta1),
+    beta2(beta2),
+    iteration(0)
+  {
+    // Nothing to do.
+  }
+
+  /**
+   * The Initialize method is called by SGD Optimizer method before the start of
+   * the iteration update process.
+   *
+   * @param rows Number of rows in the gradient matrix.
+   * @param cols Number of columns in the gradient matrix.
+   */
+  void Initialize(const size_t rows, const size_t cols)
+  {
+    m = arma::zeros<arma::mat>(rows, cols);
+    v = arma::zeros<arma::mat>(rows, cols);
+    vImproved = arma::zeros<arma::mat>(rows, cols);
+  }
+
+  /**
+   * Update step for AMSGrad.
+   *
+   * @param iterate Parameters that minimize the function.
+   * @param stepSize Step size to be used for the given iteration.
+   * @param gradient The gradient matrix.
+   */
+  void Update(arma::mat& iterate,
+              const double stepSize,
+              const arma::mat& gradient)
+  {
+    // Increment the iteration counter variable.
+    ++iteration;
+
+    // And update the iterate.
+    m *= beta1;
+    m += (1 - beta1) * gradient;
+
+    v *= beta2;
+    v += (1 - beta2) * (gradient % gradient);
+
+    const double biasCorrection1 = 1.0 - std::pow(beta1, iteration);
+    const double biasCorrection2 = 1.0 - std::pow(beta2, iteration);
+
+    // Element wise maximum of past and present squared gradients.
+    vImproved = arma::max(vImproved, v);
+
+    iterate -= (stepSize * std::sqrt(biasCorrection2) / biasCorrection1) *
+                m / (arma::sqrt(vImproved) + epsilon);
+  }
+
+  //! Get the value used to initialise the squared gradient parameter.
+  double Epsilon() const { return epsilon; }
+  //! Modify the value used to initialise the squared gradient parameter.
+  double& Epsilon() { return epsilon; }
+
+  //! Get the smoothing parameter.
+  double Beta1() const { return beta1; }
+  //! Modify the smoothing parameter.
+  double& Beta1() { return beta1; }
+
+  //! Get the second moment coefficient.
+  double Beta2() const { return beta2; }
+  //! Modify the second moment coefficient.
+  double& Beta2() { return beta2; }
+
+ private:
+  // The epsilon value used to initialise the squared gradient parameter.
+  double epsilon;
+
+  // The smoothing parameter.
+  double beta1;
+
+  // The second moment coefficient.
+  double beta2;
+
+  // The exponential moving average of gradient values.
+  arma::mat m;
+
+  // The exponential moving average of squared gradient values.
+  arma::mat v;
+
+  // The optimal sqaured gradient value.
+  arma::mat vImproved;
+
+  // The number of iterations.
+  double iteration;
+};
+
+} // namespace optimization
+} // namespace mlpack
+
+#endif
diff --git a/src/mlpack/tests/adam_test.cpp b/src/mlpack/tests/adam_test.cpp
index c1431a860c..dee211cb84 100644
--- a/src/mlpack/tests/adam_test.cpp
+++ b/src/mlpack/tests/adam_test.cpp
@@ -3,7 +3,7 @@
  * @author Vasanth Kalingeri
  * @author Vivek Pal
  *
- * Tests the Adam and AdaMax optimizer.
+ * Tests the Adam, AdaMax and AMSGrad optimizer.
  *
  * mlpack is free software; you may redistribute it and/or modify it under the
  * terms of the 3-clause BSD license.  You should have received a copy of the
@@ -62,6 +62,22 @@ BOOST_AUTO_TEST_CASE(SimpleAdaMaxTestFunction)
   BOOST_REQUIRE_SMALL(coordinates[2], 0.1);
 }
 
+/**
+ * Tests the AMSGrad optimizer using a simple test function.
+ */
+BOOST_AUTO_TEST_CASE(SimpleAMSGradTestFunction)
+{
+  SGDTestFunction f;
+  AMSGrad optimizer(1e-3, 1, 0.9, 0.999, 1e-8, 500000, 1e-11, true);
+
+  arma::mat coordinates = f.GetInitialPoint();
+  optimizer.Optimize(f, coordinates);
+
+  BOOST_REQUIRE_SMALL(coordinates[0], 0.1);
+  BOOST_REQUIRE_SMALL(coordinates[1], 0.1);
+  BOOST_REQUIRE_SMALL(coordinates[2], 0.1);
+}
+
 /**
  * Run Adam on logistic regression and make sure the results are acceptable.
  */
@@ -178,4 +194,62 @@ BOOST_AUTO_TEST_CASE(AdaMaxLogisticRegressionTest)
   BOOST_REQUIRE_CLOSE(testAcc, 100.0, 0.6); // 0.6% error tolerance.
 }
 
+/**
+ * Run AMSGrad on logistic regression and make sure the results are acceptable.
+ */
+BOOST_AUTO_TEST_CASE(AMSGradLogisticRegressionTest)
+{
+  // Generate a two-Gaussian dataset.
+  GaussianDistribution g1(arma::vec("1.0 1.0 1.0"), arma::eye<arma::mat>(3, 3));
+  GaussianDistribution g2(arma::vec("9.0 9.0 9.0"), arma::eye<arma::mat>(3, 3));
+
+  arma::mat data(3, 1000);
+  arma::Row<size_t> responses(1000);
+  for (size_t i = 0; i < 500; ++i)
+  {
+    data.col(i) = g1.Random();
+    responses[i] = 0;
+  }
+  for (size_t i = 500; i < 1000; ++i)
+  {
+    data.col(i) = g2.Random();
+    responses[i] = 1;
+  }
+
+  // Shuffle the dataset.
+  arma::uvec indices = arma::shuffle(arma::linspace<arma::uvec>(0,
+      data.n_cols - 1, data.n_cols));
+  arma::mat shuffledData(3, 1000);
+  arma::Row<size_t> shuffledResponses(1000);
+  for (size_t i = 0; i < data.n_cols; ++i)
+  {
+    shuffledData.col(i) = data.col(indices[i]);
+    shuffledResponses[i] = responses[indices[i]];
+  }
+
+  // Create a test set.
+  arma::mat testData(3, 1000);
+  arma::Row<size_t> testResponses(1000);
+  for (size_t i = 0; i < 500; ++i)
+  {
+    testData.col(i) = g1.Random();
+    testResponses[i] = 0;
+  }
+  for (size_t i = 500; i < 1000; ++i)
+  {
+    testData.col(i) = g2.Random();
+    testResponses[i] = 1;
+  }
+
+  AMSGrad amsgrad(1e-3, 1, 0.9, 0.999, 1e-8, 500000, 1e-11, true);
+  LogisticRegression<> lr(shuffledData, shuffledResponses, amsgrad, 0.5);
+
+  // Ensure that the error is close to zero.
+  const double acc = lr.ComputeAccuracy(data, responses);
+  BOOST_REQUIRE_CLOSE(acc, 100.0, 0.3); // 0.3% error tolerance.
+
+  const double testAcc = lr.ComputeAccuracy(testData, testResponses);
+  BOOST_REQUIRE_CLOSE(testAcc, 100.0, 0.6); // 0.6% error tolerance.
+}
+
 BOOST_AUTO_TEST_SUITE_END();

From 0555e733378ce0233dc1b1eab74e4109d6f6eff4 Mon Sep 17 00:00:00 2001
From: Haritha <haritha1313@gmail.com>
Date: Wed, 20 Dec 2017 11:58:26 +0530
Subject: [PATCH 36/67] ams_grad style edits

---
 src/mlpack/core/optimizers/adam/amsgrad_update.hpp | 9 ++-------
 1 file changed, 2 insertions(+), 7 deletions(-)

diff --git a/src/mlpack/core/optimizers/adam/amsgrad_update.hpp b/src/mlpack/core/optimizers/adam/amsgrad_update.hpp
index 21130e214a..f2a126f186 100644
--- a/src/mlpack/core/optimizers/adam/amsgrad_update.hpp
+++ b/src/mlpack/core/optimizers/adam/amsgrad_update.hpp
@@ -19,14 +19,9 @@ namespace mlpack {
 namespace optimization {
 
 /**
- * AMSGrad is an exponential moving average variant which along with having 
-<<<<<<< HEAD
+ * AMSGrad is an exponential moving average variant which along with having
  * benefits of optimizers like Adam and RMSProp, also guarantees convergence.
- * Unlike Adam, it uses maximum of past squared gradients rather than their 
-=======
- * benefits of optimizers like Adam and RmsProp, also guarantees convergence.
- * Unlike adam, it uses maximum of past squared gradients rather than their 
->>>>>>> 403f767751113ff07831a412b156ef16fb0fc2e2
+ * Unlike Adam, it uses maximum of past squared gradients rather than their
  * exponential average for updation.
  *
  * For more information, see the following.

From 09050cd9fdcdd7b111820bda3410dddef63b2771 Mon Sep 17 00:00:00 2001
From: Eugene Freyman <evg.freyman@gmail.com>
Date: Wed, 20 Dec 2017 08:53:40 +0200
Subject: [PATCH 37/67] solving redefinition problem at binding tests with
 static functions without macros

---
 .../python/tests/test_python_binding_main.cpp |  2 +-
 src/mlpack/core/util/mlpack_main.hpp          |  5 +-
 src/mlpack/core/util/param.hpp                | 54 +++++++------------
 src/mlpack/methods/adaboost/adaboost_main.cpp |  2 +-
 .../methods/approx_kfn/approx_kfn_main.cpp    |  2 +-
 src/mlpack/methods/cf/cf_main.cpp             |  2 +-
 src/mlpack/methods/dbscan/dbscan_main.cpp     |  2 +-
 .../decision_stump/decision_stump_main.cpp    |  2 +-
 .../decision_tree/decision_tree_main.cpp      |  2 +-
 src/mlpack/methods/det/det_main.cpp           |  2 +-
 src/mlpack/methods/emst/emst_main.cpp         |  2 +-
 src/mlpack/methods/fastmks/fastmks_main.cpp   |  2 +-
 src/mlpack/methods/gmm/gmm_generate_main.cpp  |  2 +-
 .../methods/gmm/gmm_probability_main.cpp      |  2 +-
 src/mlpack/methods/gmm/gmm_train_main.cpp     |  2 +-
 src/mlpack/methods/hmm/hmm_generate_main.cpp  |  2 +-
 src/mlpack/methods/hmm/hmm_loglik_main.cpp    |  2 +-
 src/mlpack/methods/hmm/hmm_train_main.cpp     |  2 +-
 src/mlpack/methods/hmm/hmm_viterbi_main.cpp   |  2 +-
 .../hoeffding_trees/hoeffding_tree_main.cpp   |  2 +-
 .../methods/kernel_pca/kernel_pca_main.cpp    |  2 +-
 src/mlpack/methods/kmeans/kmeans_main.cpp     |  2 +-
 src/mlpack/methods/lars/lars_main.cpp         |  2 +-
 .../linear_regression_main.cpp                |  2 +-
 .../local_coordinate_coding_main.cpp          |  2 +-
 .../logistic_regression_main.cpp              |  2 +-
 src/mlpack/methods/lsh/lsh_main.cpp           |  2 +-
 .../methods/mean_shift/mean_shift_main.cpp    |  2 +-
 src/mlpack/methods/naive_bayes/nbc_main.cpp   |  2 +-
 src/mlpack/methods/nca/nca_main.cpp           |  2 +-
 .../methods/neighbor_search/kfn_main.cpp      |  2 +-
 .../methods/neighbor_search/knn_main.cpp      |  2 +-
 src/mlpack/methods/nmf/nmf_main.cpp           |  2 +-
 src/mlpack/methods/pca/pca_main.cpp           |  2 +-
 .../methods/perceptron/perceptron_main.cpp    |  2 +-
 .../preprocess/preprocess_binarize_main.cpp   |  2 +-
 .../preprocess/preprocess_describe_main.cpp   |  2 +-
 .../preprocess/preprocess_imputer_main.cpp    |  2 +-
 .../preprocess/preprocess_split_main.cpp      |  2 +-
 src/mlpack/methods/radical/radical_main.cpp   |  2 +-
 .../random_forest/random_forest_main.cpp      |  2 +-
 .../range_search/range_search_main.cpp        |  2 +-
 src/mlpack/methods/rann/krann_main.cpp        |  2 +-
 .../softmax_regression_main.cpp               |  2 +-
 .../methods/sparse_coding/sparse_coding.cpp   |  1 -
 .../sparse_coding/sparse_coding_main.cpp      |  2 +-
 src/mlpack/tests/cli_test.cpp                 |  2 +
 .../main_tests/linear_regression_test.cpp     |  7 ++-
 src/mlpack/tests/main_tests/pca_test.cpp      | 24 +++------
 49 files changed, 77 insertions(+), 102 deletions(-)

diff --git a/src/mlpack/bindings/python/tests/test_python_binding_main.cpp b/src/mlpack/bindings/python/tests/test_python_binding_main.cpp
index fd57d19cf7..8be2f4947a 100644
--- a/src/mlpack/bindings/python/tests/test_python_binding_main.cpp
+++ b/src/mlpack/bindings/python/tests/test_python_binding_main.cpp
@@ -52,7 +52,7 @@ PARAM_MODEL_OUT(GaussianKernel, "model_out", "Output model, with twice the "
     "bandwidth.", "");
 PARAM_DOUBLE_OUT("model_bw_out", "The bandwidth of the model.");
 
-void mlpackMain()
+static void mlpackMain()
 {
   const string s = CLI::GetParam<string>("string_in");
   const int i = CLI::GetParam<int>("int_in");
diff --git a/src/mlpack/core/util/mlpack_main.hpp b/src/mlpack/core/util/mlpack_main.hpp
index bc9846b27c..11c21ee770 100644
--- a/src/mlpack/core/util/mlpack_main.hpp
+++ b/src/mlpack/core/util/mlpack_main.hpp
@@ -46,11 +46,12 @@ using Option = mlpack::bindings::cli::CLIOption<T>;
 }
 }
 
+static const std::string testName = "";
 #include <mlpack/core/util/param.hpp>
 #include <mlpack/bindings/cli/parse_command_line.hpp>
 #include <mlpack/bindings/cli/end_program.hpp>
 
-void mlpackMain(); // This is typically defined after this include.
+static void mlpackMain(); // This is typically defined after this include.
 
 int main(int argc, char** argv)
 {
@@ -91,6 +92,7 @@ using Option = mlpack::bindings::tests::TestOption<T>;
 }
 }
 
+//testName symbol should be defined in each binding test file
 #include <mlpack/core/util/param.hpp>
 
 #undef PROGRAM_INFO
@@ -119,6 +121,7 @@ using Option = mlpack::bindings::python::PyOption<T>;
 }
 }
 
+static const std::string testName = "";
 #include <mlpack/core/util/param.hpp>
 
 #undef PROGRAM_INFO
diff --git a/src/mlpack/core/util/param.hpp b/src/mlpack/core/util/param.hpp
index ad23e42ac0..2683d59b77 100644
--- a/src/mlpack/core/util/param.hpp
+++ b/src/mlpack/core/util/param.hpp
@@ -15,24 +15,6 @@
 #ifndef MLPACK_CORE_UTIL_PARAM_HPP
 #define MLPACK_CORE_UTIL_PARAM_HPP
 
-#include <string>
-
-// PROGRAM_NAME is used for mlpackMain modification (mlpackMain##PROGRAM_NAME)
-// PROGRAM_NAME_SUBSTUTIDE is used for passing unique program name to TestOption
-#ifdef PROGRAM_NAME
-#define PROGRAM_NAME_SUBSTITUDE PROGRAM_NAME
-#else
-#define PROGRAM_NAME
-#define PROGRAM_NAME_SUBSTITUDE programNameSubstitude
-static const std::string programNameSubstitude = "programNameSubstitude";
-#endif
-// The MAIN macro is mlpackMain##PROGRAM_NAME. The goal is to have different
-// names of main procedure for main tests when all method's  main functions
-// are linked into one executable (mlpack_test)
-#define TOKENPASTE1(x, y) x ## y
-#define TOKENPASTE2(x, y) TOKENPASTE1(x, y)
-#define MAIN TOKENPASTE2(mlpackMain, PROGRAM_NAME)
-
 // Required forward declarations.
 namespace mlpack {
 namespace data {
@@ -1030,55 +1012,55 @@ using DatasetInfo = DatasetMapper<IncrementPolicy, std::string>;
   #define PARAM_IN(T, ID, DESC, ALIAS, DEF, REQ) \
       static mlpack::util::Option<T> \
       JOIN(cli_option_dummy_object_in_, __COUNTER__) \
-      (DEF, ID, DESC, ALIAS, #T, REQ, true, false, PROGRAM_NAME_SUBSTITUDE);
+      (DEF, ID, DESC, ALIAS, #T, REQ, true, false, testName);
 
   #define PARAM_OUT(T, ID, DESC, ALIAS, DEF, REQ) \
       static mlpack::util::Option<T> \
       JOIN(cli_option_dummy_object_out_, __COUNTER__) \
-      (DEF, ID, DESC, ALIAS, #T, REQ, false, false, PROGRAM_NAME_SUBSTITUDE);
+      (DEF, ID, DESC, ALIAS, #T, REQ, false, false, testName);
 
   #define PARAM_MATRIX(ID, DESC, ALIAS, REQ, TRANS, IN) \
       static mlpack::util::Option<arma::mat> \
       JOIN(cli_option_dummy_matrix_, __COUNTER__) \
       (arma::mat(), ID, DESC, ALIAS, "arma::mat", \
-      REQ, IN, !TRANS, PROGRAM_NAME_SUBSTITUDE);
+      REQ, IN, !TRANS, testName);
 
   #define PARAM_UMATRIX(ID, DESC, ALIAS, REQ, TRANS, IN) \
       static mlpack::util::Option<arma::Mat<size_t>> \
       JOIN(cli_option_dummy_umatrix_, __COUNTER__) \
       (arma::Mat<size_t>(), ID, DESC, ALIAS, "arma::Mat<size_t>", \
-      REQ, IN, !TRANS, PROGRAM_NAME_SUBSTITUDE);
+      REQ, IN, !TRANS, testName);
 
   #define PARAM_COL(ID, DESC, ALIAS, REQ, TRANS, IN) \
       static mlpack::util::Option<arma::vec> \
       JOIN(cli_option_dummy_col_, __COUNTER__) \
       (arma::vec(), ID, DESC, ALIAS, "arma::vec", \
-      REQ, IN, !TRANS, PROGRAM_NAME_SUBSTITUDE);
+      REQ, IN, !TRANS, testName);
 
   #define PARAM_UCOL(ID, DESC, ALIAS, REQ, TRANS, IN) \
       static mlpack::util::Option<arma::Col<size_t>> \
       JOIN(cli_option_dummy_ucol_, __COUNTER__) \
       (arma::Col<size_t>(), ID, DESC, ALIAS, "arma::Col<size_t>", \
-      REQ, IN, !TRANS, PROGRAM_NAME_SUBSTITUDE);
+      REQ, IN, !TRANS, testName);
 
   #define PARAM_ROW(ID, DESC, ALIAS, REQ, TRANS, IN) \
       static mlpack::util::Option<arma::rowvec> \
       JOIN(cli_option_dummy_row_, __COUNTER__) \
       (arma::rowvec(), ID, DESC, ALIAS, "arma::rowvec", \
-      REQ, IN, !TRANS, PROGRAM_NAME_SUBSTITUDE);
+      REQ, IN, !TRANS, testName);
 
   #define PARAM_UROW(ID, DESC, ALIAS, REQ, TRANS, IN) \
       static mlpack::util::Option<arma::Row<size_t>> \
       JOIN(cli_option_dummy_urow_, __COUNTER__) \
       (arma::Row<size_t>(), ID, DESC, ALIAS, "arma::Row<size_t>", \
-      REQ, IN, !TRANS, PROGRAM_NAME_SUBSTITUDE);
+      REQ, IN, !TRANS, testName);
 
   // There are no uses of required models, so that is not an option to this
   // macro (it would be easy to add).
   #define PARAM_MODEL(TYPE, ID, DESC, ALIAS, REQ, IN) \
       static mlpack::util::Option<TYPE> \
       JOIN(cli_option_dummy_model_, __COUNTER__) \
-      (TYPE(), ID, DESC, ALIAS, #TYPE, REQ, IN, false, PROGRAM_NAME_SUBSTITUDE);
+      (TYPE(), ID, DESC, ALIAS, #TYPE, REQ, IN, false, testName);
 #else
   // We have to do some really bizarre stuff since __COUNTER__ isn't defined. I
   // don't think we can absolutely guarantee success, but it should be "good
@@ -1087,54 +1069,54 @@ using DatasetInfo = DatasetMapper<IncrementPolicy, std::string>;
   #define PARAM_IN(T, ID, DESC, ALIAS, DEF, REQ) \
       static mlpack::util::Option<T> \
       JOIN(JOIN(cli_option_dummy_object_in_, __LINE__), opt) \
-      (DEF, ID, DESC, ALIAS, #T, REQ, true, false, PROGRAM_NAME_SUBSTITUDE);
+      (DEF, ID, DESC, ALIAS, #T, REQ, true, false, testName);
 
   #define PARAM_OUT(T, ID, DESC, ALIAS, DEF, REQ) \
       static mlpack::util::Option<T> \
       JOIN(JOIN(cli_option_dummy_object_out_, __LINE__), opt) \
-      (DEF, ID, DESC, ALIAS, #T, REQ, false, false, PROGRAM_NAME_SUBSTITUDE);
+      (DEF, ID, DESC, ALIAS, #T, REQ, false, false, testName);
 
   #define PARAM_MATRIX(ID, DESC, ALIAS, REQ, TRANS, IN) \
       static mlpack::util::Option<arma::mat> \
       JOIN(JOIN(cli_option_dummy_object_matrix_, __LINE__), opt) \
       (arma::mat(), ID, DESC, ALIAS, "arma::mat", REQ, IN, !TRANS, \
-      PROGRAM_NAME_SUBSTITUDE);
+      testName);
 
   #define PARAM_UMATRIX(ID, DESC, ALIAS, REQ, TRANS, IN) \
       static mlpack::util::Option<arma::Mat<size_t>> \
       JOIN(JOIN(cli_option_dummy_object_umatrix_, __LINE__), opt) \
       (arma::Mat<size_t>(), ID, DESC, ALIAS, "arma::Mat<size_t>", REQ, IN, \
-      !TRANS, PROGRAM_NAME_SUBSTITUDE);
+      !TRANS, testName);
 
   #define PARAM_COL(ID, DESC, ALIAS, REQ, TRANS, IN) \
       static mlpack::util::Option<arma::vec> \
       JOIN(cli_option_dummy_object_col_, __LINE__) \
       (arma::vec(), ID, DESC, ALIAS, "arma::vec", REQ, IN, !TRANS, \
-      PROGRAM_NAME_SUBSTITUDE);
+      testName);
 
   #define PARAM_UCOL(ID, DESC, ALIAS, REQ, TRANS, IN) \
       static mlpack::util::Option<arma::Col<size_t>> \
       JOIN(cli_option_dummy_object_ucol_, __LINE__) \
       (arma::Col<size_t>(), ID, DESC, ALIAS, "arma::Col<size_t>", REQ, IN, \
-      !TRANS, PROGRAM_NAME_SUBSTITUDE);
+      !TRANS, testName);
 
   #define PARAM_ROW(ID, DESC, ALIAS, REQ, TRANS, IN) \
       static mlpack::util::Option<arma::rowvec> \
       JOIN(cli_option_dummy_object_row_, __LINE__) \
       (arma::rowvec(), ID, DESC, ALIAS, "arma::rowvec", REQ, IN, !TRANS, \
-      PROGRAM_NAME_SUBSTITUDE);
+      testName);
 
   #define PARAM_UROW(ID, DESC, ALIAS, REQ, TRANS, IN) \
       static mlpack::util::Option<arma::Row<size_t>> \
       JOIN(cli_option_dummy_object_urow_, __LINE__) \
       (arma::Row<size_t>(), ID, DESC, ALIAS, "arma::Row<size_t>", REQ, IN, \
-      !TRANS, PROGRAM_NAME_SUBSTITUDE);
+      !TRANS, testName);
 
   #define PARAM_MODEL(TYPE, ID, DESC, ALIAS, REQ, IN) \
       static mlpack::util::Option<TYPE> \
       JOIN(JOIN(cli_option_dummy_object_model_, __LINE__), opt) \
       (TYPE(), ID, DESC, ALIAS, #TYPE, REQ, IN, false, \
-      PROGRAM_NAME_SUBSTITUDE);
+      testName);
 #endif
 
 #endif
diff --git a/src/mlpack/methods/adaboost/adaboost_main.cpp b/src/mlpack/methods/adaboost/adaboost_main.cpp
index e0b3a70cc7..e5209efb46 100644
--- a/src/mlpack/methods/adaboost/adaboost_main.cpp
+++ b/src/mlpack/methods/adaboost/adaboost_main.cpp
@@ -111,7 +111,7 @@ PARAM_MODEL_IN(AdaBoostModel, "input_model", "Input AdaBoost model.", "m");
 PARAM_MODEL_OUT(AdaBoostModel, "output_model", "Output trained AdaBoost model.",
     "M");
 
-void mlpackMain()
+static void mlpackMain()
 {
   // Check input parameters and issue warnings/errors as necessary.
 
diff --git a/src/mlpack/methods/approx_kfn/approx_kfn_main.cpp b/src/mlpack/methods/approx_kfn/approx_kfn_main.cpp
index 3081ed3bcd..f73008583d 100644
--- a/src/mlpack/methods/approx_kfn/approx_kfn_main.cpp
+++ b/src/mlpack/methods/approx_kfn/approx_kfn_main.cpp
@@ -141,7 +141,7 @@ PARAM_MODEL_IN(ApproxKFNModel, "input_model", "File containing input model.",
 PARAM_MODEL_OUT(ApproxKFNModel, "output_model", "File to save output model to.",
     "M");
 
-void mlpackMain()
+static void mlpackMain()
 {
   // We have to pass either a reference set or an input model.
   RequireOnlyOnePassed({ "reference", "input_model" });
diff --git a/src/mlpack/methods/cf/cf_main.cpp b/src/mlpack/methods/cf/cf_main.cpp
index a0c670e6bb..4f2ca384bb 100644
--- a/src/mlpack/methods/cf/cf_main.cpp
+++ b/src/mlpack/methods/cf/cf_main.cpp
@@ -263,7 +263,7 @@ void AssembleFactorizerType(const std::string& algorithm,
   }
 }
 
-void mlpackMain()
+static void mlpackMain()
 {
   if (CLI::GetParam<int>("seed") == 0)
     math::RandomSeed(std::time(NULL));
diff --git a/src/mlpack/methods/dbscan/dbscan_main.cpp b/src/mlpack/methods/dbscan/dbscan_main.cpp
index da0807c113..5c4af77a01 100644
--- a/src/mlpack/methods/dbscan/dbscan_main.cpp
+++ b/src/mlpack/methods/dbscan/dbscan_main.cpp
@@ -112,7 +112,7 @@ void RunDBSCAN(RangeSearchType rs = RangeSearchType())
     CLI::GetParam<arma::Row<size_t>>("assignments") = std::move(assignments);
 }
 
-void mlpackMain()
+static void mlpackMain()
 {
   RequireAtLeastOnePassed({ "assignments", "centroids" }, false,
       "no output will be saved");
diff --git a/src/mlpack/methods/decision_stump/decision_stump_main.cpp b/src/mlpack/methods/decision_stump/decision_stump_main.cpp
index a3f3213df5..2bfe230043 100644
--- a/src/mlpack/methods/decision_stump/decision_stump_main.cpp
+++ b/src/mlpack/methods/decision_stump/decision_stump_main.cpp
@@ -103,7 +103,7 @@ PARAM_MODEL_OUT(DSModel, "output_model", "Output decision stump model to save.",
 PARAM_INT_IN("bucket_size", "The minimum number of training points in each "
     "decision stump bucket.", "b", 6);
 
-void mlpackMain()
+static void mlpackMain()
 {
   // Check that the parameters are reasonable.
   RequireOnlyOnePassed({ "training", "input_model" }, true);
diff --git a/src/mlpack/methods/decision_tree/decision_tree_main.cpp b/src/mlpack/methods/decision_tree/decision_tree_main.cpp
index 2ead988d1a..6145b66d3f 100644
--- a/src/mlpack/methods/decision_tree/decision_tree_main.cpp
+++ b/src/mlpack/methods/decision_tree/decision_tree_main.cpp
@@ -115,7 +115,7 @@ PARAM_MODEL_IN(DecisionTreeModel, "input_model", "Pre-trained decision tree, "
 PARAM_MODEL_OUT(DecisionTreeModel, "output_model", "Output for trained decision"
     " tree.", "M");
 
-void mlpackMain()
+static void mlpackMain()
 {
   // Check parameters.
   RequireOnlyOnePassed({ "training", "input_model" }, true);
diff --git a/src/mlpack/methods/det/det_main.cpp b/src/mlpack/methods/det/det_main.cpp
index fcfba52a4d..29ecd56c99 100644
--- a/src/mlpack/methods/det/det_main.cpp
+++ b/src/mlpack/methods/det/det_main.cpp
@@ -100,7 +100,7 @@ PARAM_FLAG("volume_regularization", "This flag gives the used the option to use"
 */
 
 
-void mlpackMain()
+static void mlpackMain()
 {
   // Validate input parameters.
   RequireOnlyOnePassed({ "training", "input_model" }, true);
diff --git a/src/mlpack/methods/emst/emst_main.cpp b/src/mlpack/methods/emst/emst_main.cpp
index e3a41373b7..3b0f9de98a 100644
--- a/src/mlpack/methods/emst/emst_main.cpp
+++ b/src/mlpack/methods/emst/emst_main.cpp
@@ -72,7 +72,7 @@ using namespace mlpack::metric;
 using namespace mlpack::util;
 using namespace std;
 
-void mlpackMain()
+static void mlpackMain()
 {
   RequireAtLeastOnePassed({ "output" }, false, "no output will be saved");
 
diff --git a/src/mlpack/methods/fastmks/fastmks_main.cpp b/src/mlpack/methods/fastmks/fastmks_main.cpp
index be60c3e976..f3b7333ebe 100644
--- a/src/mlpack/methods/fastmks/fastmks_main.cpp
+++ b/src/mlpack/methods/fastmks/fastmks_main.cpp
@@ -83,7 +83,7 @@ PARAM_FLAG("single", "If true, single-tree search is used (as opposed to "
 PARAM_MATRIX_OUT("kernels", "Output matrix of kernels.", "p");
 PARAM_UMATRIX_OUT("indices", "Output matrix of indices.", "i");
 
-void mlpackMain()
+static void mlpackMain()
 {
   // Validate command-line parameters.
   RequireOnlyOnePassed({ "reference", "input_model" }, true);
diff --git a/src/mlpack/methods/gmm/gmm_generate_main.cpp b/src/mlpack/methods/gmm/gmm_generate_main.cpp
index 9a5da6a29a..a6ed7d6ba4 100644
--- a/src/mlpack/methods/gmm/gmm_generate_main.cpp
+++ b/src/mlpack/methods/gmm/gmm_generate_main.cpp
@@ -42,7 +42,7 @@ PARAM_MATRIX_OUT("output", "Matrix to save output samples in.", "o");
 
 PARAM_INT_IN("seed", "Random seed.  If 0, 'std::time(NULL)' is used.", "s", 0);
 
-void mlpackMain()
+static void mlpackMain()
 {
   // Parameter sanity checks.
   RequireAtLeastOnePassed({ "output" }, false, "no results will be saved");
diff --git a/src/mlpack/methods/gmm/gmm_probability_main.cpp b/src/mlpack/methods/gmm/gmm_probability_main.cpp
index 8606c38bbe..ac1669aaac 100644
--- a/src/mlpack/methods/gmm/gmm_probability_main.cpp
+++ b/src/mlpack/methods/gmm/gmm_probability_main.cpp
@@ -41,7 +41,7 @@ PARAM_MATRIX_IN_REQ("input", "Input matrix to calculate probabilities of.",
 
 PARAM_MATRIX_OUT("output", "Matrix to store calculated probabilities in.", "o");
 
-void mlpackMain()
+static void mlpackMain()
 {
   RequireAtLeastOnePassed({ "output" }, false, "no results will be saved");
 
diff --git a/src/mlpack/methods/gmm/gmm_train_main.cpp b/src/mlpack/methods/gmm/gmm_train_main.cpp
index 8e87f6867b..82694fb17c 100644
--- a/src/mlpack/methods/gmm/gmm_train_main.cpp
+++ b/src/mlpack/methods/gmm/gmm_train_main.cpp
@@ -121,7 +121,7 @@ PARAM_MODEL_IN(GMM, "input_model", "Initial input GMM model to start training "
     "with.", "m");
 PARAM_MODEL_OUT(GMM, "output_model", "Output for trained GMM model.", "M");
 
-void mlpackMain()
+static void mlpackMain()
 {
   // Check parameters and load data.
   if (CLI::GetParam<int>("seed") != 0)
diff --git a/src/mlpack/methods/hmm/hmm_generate_main.cpp b/src/mlpack/methods/hmm/hmm_generate_main.cpp
index 4f4980a7dc..c9488d3466 100644
--- a/src/mlpack/methods/hmm/hmm_generate_main.cpp
+++ b/src/mlpack/methods/hmm/hmm_generate_main.cpp
@@ -92,7 +92,7 @@ struct Generate
   }
 };
 
-void mlpackMain()
+static void mlpackMain()
 {
   RequireAtLeastOnePassed({ "output", "state" }, false, "no output will be "
       "saved");
diff --git a/src/mlpack/methods/hmm/hmm_loglik_main.cpp b/src/mlpack/methods/hmm/hmm_loglik_main.cpp
index 9270455c2e..1a3d51a8cf 100644
--- a/src/mlpack/methods/hmm/hmm_loglik_main.cpp
+++ b/src/mlpack/methods/hmm/hmm_loglik_main.cpp
@@ -76,7 +76,7 @@ struct Loglik
   }
 };
 
-void mlpackMain()
+static void mlpackMain()
 {
   // Load model, and calculate the log-likelihood of the sequence.
   CLI::GetParam<HMMModel>("input_model").PerformAction<Loglik>((void*) NULL);
diff --git a/src/mlpack/methods/hmm/hmm_train_main.cpp b/src/mlpack/methods/hmm/hmm_train_main.cpp
index 20c18d7ee8..2983c2528b 100644
--- a/src/mlpack/methods/hmm/hmm_train_main.cpp
+++ b/src/mlpack/methods/hmm/hmm_train_main.cpp
@@ -340,7 +340,7 @@ struct Train
   }
 };
 
-void mlpackMain()
+static void mlpackMain()
 {
   // Set random seed.
   if (CLI::GetParam<int>("seed") != 0)
diff --git a/src/mlpack/methods/hmm/hmm_viterbi_main.cpp b/src/mlpack/methods/hmm/hmm_viterbi_main.cpp
index a225e6f4cc..323bc066cc 100644
--- a/src/mlpack/methods/hmm/hmm_viterbi_main.cpp
+++ b/src/mlpack/methods/hmm/hmm_viterbi_main.cpp
@@ -82,7 +82,7 @@ struct Viterbi
   }
 };
 
-void mlpackMain()
+static void mlpackMain()
 {
   RequireAtLeastOnePassed({ "output" }, false, "no results will be saved");
 
diff --git a/src/mlpack/methods/hoeffding_trees/hoeffding_tree_main.cpp b/src/mlpack/methods/hoeffding_trees/hoeffding_tree_main.cpp
index c832d147ad..a12c51d2c2 100644
--- a/src/mlpack/methods/hoeffding_trees/hoeffding_tree_main.cpp
+++ b/src/mlpack/methods/hoeffding_trees/hoeffding_tree_main.cpp
@@ -109,7 +109,7 @@ PARAM_INT_IN("observations_before_binning", "If the 'domingos' split strategy "
 // Convenience typedef.
 typedef tuple<DatasetInfo, arma::mat> TupleType;
 
-void mlpackMain()
+static void mlpackMain()
 {
   // Check input parameters for validity.
   const string numericSplitStrategy =
diff --git a/src/mlpack/methods/kernel_pca/kernel_pca_main.cpp b/src/mlpack/methods/kernel_pca/kernel_pca_main.cpp
index f6b0d0a16a..7ca6268120 100644
--- a/src/mlpack/methods/kernel_pca/kernel_pca_main.cpp
+++ b/src/mlpack/methods/kernel_pca/kernel_pca_main.cpp
@@ -164,7 +164,7 @@ void RunKPCA(arma::mat& dataset,
   }
 }
 
-void mlpackMain()
+static void mlpackMain()
 {
   RequireAtLeastOnePassed({ "output" }, false, "no output will be saved");
 
diff --git a/src/mlpack/methods/kmeans/kmeans_main.cpp b/src/mlpack/methods/kmeans/kmeans_main.cpp
index 4734be86f4..2f42213805 100644
--- a/src/mlpack/methods/kmeans/kmeans_main.cpp
+++ b/src/mlpack/methods/kmeans/kmeans_main.cpp
@@ -140,7 +140,7 @@ template<typename InitialPartitionPolicy,
          template<class, class> class LloydStepType>
 void RunKMeans(const InitialPartitionPolicy& ipp);
 
-void mlpackMain()
+static void mlpackMain()
 {
   // Initialize random seed.
   if (CLI::GetParam<int>("seed") != 0)
diff --git a/src/mlpack/methods/lars/lars_main.cpp b/src/mlpack/methods/lars/lars_main.cpp
index bc623108fe..0c6d1a28e5 100644
--- a/src/mlpack/methods/lars/lars_main.cpp
+++ b/src/mlpack/methods/lars/lars_main.cpp
@@ -101,7 +101,7 @@ PARAM_DOUBLE_IN("lambda2", "Regularization parameter for l2-norm penalty.", "L",
 PARAM_FLAG("use_cholesky", "Use Cholesky decomposition during computation "
     "rather than explicitly computing the full Gram matrix.", "c");
 
-void mlpackMain()
+static void mlpackMain()
 {
   double lambda1 = CLI::GetParam<double>("lambda1");
   double lambda2 = CLI::GetParam<double>("lambda2");
diff --git a/src/mlpack/methods/linear_regression/linear_regression_main.cpp b/src/mlpack/methods/linear_regression/linear_regression_main.cpp
index d5df454af0..8817556934 100644
--- a/src/mlpack/methods/linear_regression/linear_regression_main.cpp
+++ b/src/mlpack/methods/linear_regression/linear_regression_main.cpp
@@ -85,7 +85,7 @@ PARAM_COL_OUT("output_predictions", "If --test_file is specified, this "
 PARAM_DOUBLE_IN("lambda", "Tikhonov regularization for ridge regression.  If 0,"
     " the method reduces to linear regression.", "l", 0.0);
 
-void MAIN()
+static void mlpackMain()
 {
   const double lambda = CLI::GetParam<double>("lambda");
 
diff --git a/src/mlpack/methods/local_coordinate_coding/local_coordinate_coding_main.cpp b/src/mlpack/methods/local_coordinate_coding/local_coordinate_coding_main.cpp
index 7fb3ba2329..cdfc34c258 100644
--- a/src/mlpack/methods/local_coordinate_coding/local_coordinate_coding_main.cpp
+++ b/src/mlpack/methods/local_coordinate_coding/local_coordinate_coding_main.cpp
@@ -93,7 +93,7 @@ PARAM_MATRIX_OUT("codes", "Output codes matrix.", "c");
 
 PARAM_INT_IN("seed", "Random seed.  If 0, 'std::time(NULL)' is used.", "s", 0);
 
-void mlpackMain()
+static void mlpackMain()
 {
   if (CLI::GetParam<int>("seed") != 0)
     RandomSeed((size_t) CLI::GetParam<int>("seed"));
diff --git a/src/mlpack/methods/logistic_regression/logistic_regression_main.cpp b/src/mlpack/methods/logistic_regression/logistic_regression_main.cpp
index 8f6dc937aa..3ecf5ff8b1 100644
--- a/src/mlpack/methods/logistic_regression/logistic_regression_main.cpp
+++ b/src/mlpack/methods/logistic_regression/logistic_regression_main.cpp
@@ -133,7 +133,7 @@ PARAM_DOUBLE_IN("decision_boundary", "Decision boundary for prediction; if the "
     "logistic function for a point is less than the boundary, the class is "
     "taken to be 0; otherwise, the class is 1.", "d", 0.5);
 
-void mlpackMain()
+static void mlpackMain()
 {
   // Collect command-line options.
   const double lambda = CLI::GetParam<double>("lambda");
diff --git a/src/mlpack/methods/lsh/lsh_main.cpp b/src/mlpack/methods/lsh/lsh_main.cpp
index e85ac58fd6..f9ce2de8b5 100644
--- a/src/mlpack/methods/lsh/lsh_main.cpp
+++ b/src/mlpack/methods/lsh/lsh_main.cpp
@@ -82,7 +82,7 @@ PARAM_INT_IN("bucket_size", "The size of a bucket in the second level hash.",
     "B", 500);
 PARAM_INT_IN("seed", "Random seed.  If 0, 'std::time(NULL)' is used.", "s", 0);
 
-void mlpackMain()
+static void mlpackMain()
 {
   if (CLI::GetParam<int>("seed") != 0)
     math::RandomSeed((size_t) CLI::GetParam<int>("seed"));
diff --git a/src/mlpack/methods/mean_shift/mean_shift_main.cpp b/src/mlpack/methods/mean_shift/mean_shift_main.cpp
index 5569e17adc..639e231e9d 100644
--- a/src/mlpack/methods/mean_shift/mean_shift_main.cpp
+++ b/src/mlpack/methods/mean_shift/mean_shift_main.cpp
@@ -66,7 +66,7 @@ PARAM_DOUBLE_IN("radius", "If the distance between two centroids is less than "
     "the given radius, one will be removed.  A radius of 0 or less means an "
     "estimate will be calculated and used for the radius.", "r", 0);
 
-void mlpackMain()
+static void mlpackMain()
 {
   const double radius = CLI::GetParam<double>("radius");
   const int maxIterations = CLI::GetParam<int>("max_iterations");
diff --git a/src/mlpack/methods/naive_bayes/nbc_main.cpp b/src/mlpack/methods/naive_bayes/nbc_main.cpp
index 185a980356..99b76cb7e7 100644
--- a/src/mlpack/methods/naive_bayes/nbc_main.cpp
+++ b/src/mlpack/methods/naive_bayes/nbc_main.cpp
@@ -105,7 +105,7 @@ PARAM_UROW_OUT("output", "The matrix in which the predicted labels for the"
 PARAM_MATRIX_OUT("output_probs", "The matrix in which the predicted probability"
     " of labels for the test set will be written.", "p");
 
-void mlpackMain()
+static void mlpackMain()
 {
   // Check input parameters.
   RequireOnlyOnePassed({ "training", "input_model" }, true);
diff --git a/src/mlpack/methods/nca/nca_main.cpp b/src/mlpack/methods/nca/nca_main.cpp
index 0618743b6e..cc988f1281 100644
--- a/src/mlpack/methods/nca/nca_main.cpp
+++ b/src/mlpack/methods/nca/nca_main.cpp
@@ -124,7 +124,7 @@ using namespace mlpack::optimization;
 using namespace mlpack::util;
 using namespace std;
 
-void mlpackMain()
+static void mlpackMain()
 {
   if (CLI::GetParam<int>("seed") != 0)
     math::RandomSeed((size_t) CLI::GetParam<int>("seed"));
diff --git a/src/mlpack/methods/neighbor_search/kfn_main.cpp b/src/mlpack/methods/neighbor_search/kfn_main.cpp
index d6942c6d1a..d6a5d3d0d7 100644
--- a/src/mlpack/methods/neighbor_search/kfn_main.cpp
+++ b/src/mlpack/methods/neighbor_search/kfn_main.cpp
@@ -95,7 +95,7 @@ PARAM_DOUBLE_IN("percentage", "If specified, will do approximate furthest "
     "neighbors will be at least (p*100) % of the distance as the true furthest "
     "neighbor.", "p", 1);
 
-void mlpackMain()
+static void mlpackMain()
 {
   if (CLI::GetParam<int>("seed") != 0)
     math::RandomSeed((size_t) CLI::GetParam<int>("seed"));
diff --git a/src/mlpack/methods/neighbor_search/knn_main.cpp b/src/mlpack/methods/neighbor_search/knn_main.cpp
index e021ce3225..9c43abceaf 100644
--- a/src/mlpack/methods/neighbor_search/knn_main.cpp
+++ b/src/mlpack/methods/neighbor_search/knn_main.cpp
@@ -99,7 +99,7 @@ PARAM_STRING_IN("algorithm", "Type of neighbor search: 'naive', 'single_tree', "
 PARAM_DOUBLE_IN("epsilon", "If specified, will do approximate nearest neighbor "
     "search with given relative error.", "e", 0);
 
-void mlpackMain()
+static void mlpackMain()
 {
   if (CLI::GetParam<int>("seed") != 0)
     math::RandomSeed((size_t) CLI::GetParam<int>("seed"));
diff --git a/src/mlpack/methods/nmf/nmf_main.cpp b/src/mlpack/methods/nmf/nmf_main.cpp
index a9ced923c7..e8f4bdb2d1 100644
--- a/src/mlpack/methods/nmf/nmf_main.cpp
+++ b/src/mlpack/methods/nmf/nmf_main.cpp
@@ -76,7 +76,7 @@ PARAM_DOUBLE_IN("min_residue", "The minimum root mean square residue allowed "
 PARAM_STRING_IN("update_rules", "Update rules for each iteration; ( multdist | "
     "multdiv | als ).", "u", "multdist");
 
-void mlpackMain()
+static void mlpackMain()
 {
   // Initialize random seed.
   if (CLI::GetParam<int>("seed") != 0)
diff --git a/src/mlpack/methods/pca/pca_main.cpp b/src/mlpack/methods/pca/pca_main.cpp
index e0751d1b9c..742ad08b4c 100644
--- a/src/mlpack/methods/pca/pca_main.cpp
+++ b/src/mlpack/methods/pca/pca_main.cpp
@@ -99,7 +99,7 @@ void RunPCA(arma::mat& dataset,
       dataset.n_rows << " dimensions)." << endl;
 }
 
-void MAIN()
+static void mlpackMain()
 {
   // Load input dataset.
   arma::mat& dataset = CLI::GetParam<arma::mat>("input");
diff --git a/src/mlpack/methods/perceptron/perceptron_main.cpp b/src/mlpack/methods/perceptron/perceptron_main.cpp
index c8eace1e83..c7a5b17796 100644
--- a/src/mlpack/methods/perceptron/perceptron_main.cpp
+++ b/src/mlpack/methods/perceptron/perceptron_main.cpp
@@ -118,7 +118,7 @@ PARAM_MATRIX_IN("test", "A matrix containing the test set.", "T");
 PARAM_UROW_OUT("output", "The matrix in which the predicted labels for the"
     " test set will be written.", "o");
 
-void mlpackMain()
+static void mlpackMain()
 {
   // First, get all parameters and validate them.
   const size_t maxIterations = (size_t) CLI::GetParam<int>("max_iterations");
diff --git a/src/mlpack/methods/preprocess/preprocess_binarize_main.cpp b/src/mlpack/methods/preprocess/preprocess_binarize_main.cpp
index a4d952ca40..93dd75855b 100644
--- a/src/mlpack/methods/preprocess/preprocess_binarize_main.cpp
+++ b/src/mlpack/methods/preprocess/preprocess_binarize_main.cpp
@@ -54,7 +54,7 @@ using namespace mlpack::util;
 using namespace arma;
 using namespace std;
 
-void mlpackMain()
+static void mlpackMain()
 {
   const size_t dimension = (size_t) CLI::GetParam<int>("dimension");
   const double threshold = CLI::GetParam<double>("threshold");
diff --git a/src/mlpack/methods/preprocess/preprocess_describe_main.cpp b/src/mlpack/methods/preprocess/preprocess_describe_main.cpp
index a592293f32..744f826a5c 100644
--- a/src/mlpack/methods/preprocess/preprocess_describe_main.cpp
+++ b/src/mlpack/methods/preprocess/preprocess_describe_main.cpp
@@ -153,7 +153,7 @@ double StandardError(const size_t size, const double& fStd)
   return fStd / sqrt(size);
 }
 
-void mlpackMain()
+static void mlpackMain()
 {
   const size_t dimension = static_cast<size_t>(CLI::GetParam<int>("dimension"));
   const size_t precision = static_cast<size_t>(CLI::GetParam<int>("precision"));
diff --git a/src/mlpack/methods/preprocess/preprocess_imputer_main.cpp b/src/mlpack/methods/preprocess/preprocess_imputer_main.cpp
index 5d16a93fdd..7bcbf99fca 100644
--- a/src/mlpack/methods/preprocess/preprocess_imputer_main.cpp
+++ b/src/mlpack/methods/preprocess/preprocess_imputer_main.cpp
@@ -53,7 +53,7 @@ using namespace arma;
 using namespace std;
 using namespace data;
 
-void mlpackMain()
+static void mlpackMain()
 {
   const string inputFile = CLI::GetParam<string>("input_file");
   const string outputFile = CLI::GetParam<string>("output_file");
diff --git a/src/mlpack/methods/preprocess/preprocess_split_main.cpp b/src/mlpack/methods/preprocess/preprocess_split_main.cpp
index 60710f3672..a66056012e 100644
--- a/src/mlpack/methods/preprocess/preprocess_split_main.cpp
+++ b/src/mlpack/methods/preprocess/preprocess_split_main.cpp
@@ -71,7 +71,7 @@ using namespace mlpack::util;
 using namespace arma;
 using namespace std;
 
-void mlpackMain()
+static void mlpackMain()
 {
   // Parse command line options.
   const double testRatio = CLI::GetParam<double>("test_ratio");
diff --git a/src/mlpack/methods/radical/radical_main.cpp b/src/mlpack/methods/radical/radical_main.cpp
index 3863d6f339..7d5e594474 100644
--- a/src/mlpack/methods/radical/radical_main.cpp
+++ b/src/mlpack/methods/radical/radical_main.cpp
@@ -57,7 +57,7 @@ using namespace mlpack::util;
 using namespace std;
 using namespace arma;
 
-void mlpackMain()
+static void mlpackMain()
 {
   // Set random seed.
   if (CLI::GetParam<int>("seed") != 0)
diff --git a/src/mlpack/methods/random_forest/random_forest_main.cpp b/src/mlpack/methods/random_forest/random_forest_main.cpp
index e149dbc141..e6cdfd5aa3 100644
--- a/src/mlpack/methods/random_forest/random_forest_main.cpp
+++ b/src/mlpack/methods/random_forest/random_forest_main.cpp
@@ -68,7 +68,7 @@ PARAM_MODEL_IN(RandomForestModel, "input_model", "Pre-trained random forest to "
 PARAM_MODEL_OUT(RandomForestModel, "output_model", "Model to save trained "
     "random forest to.", "M");
 
-void mlpackMain()
+static void mlpackMain()
 {
   // Check for incompatible input parameters.
   RequireOnlyOnePassed({ "training", "input_model" }, true);
diff --git a/src/mlpack/methods/range_search/range_search_main.cpp b/src/mlpack/methods/range_search/range_search_main.cpp
index 7fe03f4814..66a070e6d6 100644
--- a/src/mlpack/methods/range_search/range_search_main.cpp
+++ b/src/mlpack/methods/range_search/range_search_main.cpp
@@ -91,7 +91,7 @@ PARAM_FLAG("naive", "If true, O(n^2) naive mode is used for computation.", "N");
 PARAM_FLAG("single_mode", "If true, single-tree search is used (as opposed to "
     "dual-tree search).", "S");
 
-void mlpackMain()
+static void mlpackMain()
 {
   if (CLI::GetParam<int>("seed") != 0)
     math::RandomSeed((size_t) CLI::GetParam<int>("seed"));
diff --git a/src/mlpack/methods/rann/krann_main.cpp b/src/mlpack/methods/rann/krann_main.cpp
index 0b4d5d725b..dd7fdef538 100644
--- a/src/mlpack/methods/rann/krann_main.cpp
+++ b/src/mlpack/methods/rann/krann_main.cpp
@@ -96,7 +96,7 @@ PARAM_FLAG("first_leaf_exact", "The flag to trigger sampling only after "
 PARAM_INT_IN("single_sample_limit", "The limit on the maximum number of "
     "samples (and hence the largest node you can approximate).", "z", 20);
 
-void mlpackMain()
+static void mlpackMain()
 {
   if (CLI::GetParam<int>("seed") != 0)
     math::RandomSeed((size_t) CLI::GetParam<int>("seed"));
diff --git a/src/mlpack/methods/softmax_regression/softmax_regression_main.cpp b/src/mlpack/methods/softmax_regression/softmax_regression_main.cpp
index 497b878a5b..78e8c96245 100644
--- a/src/mlpack/methods/softmax_regression/softmax_regression_main.cpp
+++ b/src/mlpack/methods/softmax_regression/softmax_regression_main.cpp
@@ -115,7 +115,7 @@ void TestClassifyAcc(const size_t numClasses, const Model& model);
 template<typename Model>
 unique_ptr<Model> TrainSoftmax(const size_t maxIterations);
 
-void mlpackMain()
+static void mlpackMain()
 {
   const int maxIterations = CLI::GetParam<int>("max_iterations");
 
diff --git a/src/mlpack/methods/sparse_coding/sparse_coding.cpp b/src/mlpack/methods/sparse_coding/sparse_coding.cpp
index 6edf7d7eb6..a383c37eb2 100644
--- a/src/mlpack/methods/sparse_coding/sparse_coding.cpp
+++ b/src/mlpack/methods/sparse_coding/sparse_coding.cpp
@@ -12,7 +12,6 @@
  */
 #include "sparse_coding.hpp"
 #include <mlpack/core/math/lin_alg.hpp>
-#include <mlpack/core/util/param.hpp>
 
 namespace mlpack {
 namespace sparse_coding {
diff --git a/src/mlpack/methods/sparse_coding/sparse_coding_main.cpp b/src/mlpack/methods/sparse_coding/sparse_coding_main.cpp
index a3a0d40061..0ce5bf8336 100644
--- a/src/mlpack/methods/sparse_coding/sparse_coding_main.cpp
+++ b/src/mlpack/methods/sparse_coding/sparse_coding_main.cpp
@@ -96,7 +96,7 @@ PARAM_MATRIX_OUT("codes", "Matrix to save the output sparse codes of the test "
 
 PARAM_MATRIX_IN("test", "Optional matrix to be encoded by trained model.", "T");
 
-void mlpackMain()
+static void mlpackMain()
 {
   if (CLI::GetParam<int>("seed") != 0)
     RandomSeed((size_t) CLI::GetParam<int>("seed"));
diff --git a/src/mlpack/tests/cli_test.cpp b/src/mlpack/tests/cli_test.cpp
index dd411ca84a..144e8c9967 100644
--- a/src/mlpack/tests/cli_test.cpp
+++ b/src/mlpack/tests/cli_test.cpp
@@ -22,6 +22,8 @@ using Option = mlpack::bindings::cli::CLIOption<T>;
 } // namespace util
 } // namespace mlpack
 
+static const std::string testName = "";
+
 #include <mlpack/core/util/param.hpp>
 #include <mlpack/bindings/cli/parse_command_line.hpp>
 #include <mlpack/bindings/cli/end_program.hpp>
diff --git a/src/mlpack/tests/main_tests/linear_regression_test.cpp b/src/mlpack/tests/main_tests/linear_regression_test.cpp
index 9eb8b0319c..1ca74bb562 100644
--- a/src/mlpack/tests/main_tests/linear_regression_test.cpp
+++ b/src/mlpack/tests/main_tests/linear_regression_test.cpp
@@ -7,8 +7,7 @@
 #include <string>
 
 #define BINDING_TYPE BINDING_TYPE_TEST
-#define PROGRAM_NAME linearRegressionProgramName
-static const std::string linearRegressionProgramName = "LinearRegression";
+static const std::string testName = "LinearRegression";
 
 #include <mlpack/core.hpp>
 #include <mlpack/core/util/mlpack_main.hpp>
@@ -43,7 +42,7 @@ struct LinearRegressionTestFixture
   LinearRegressionTestFixture()
   {
     // Cache in the options for this program.
-    CLI::RestoreSettings(linearRegressionProgramName);
+    CLI::RestoreSettings(testName);
   }
 
   ~LinearRegressionTestFixture()
@@ -64,7 +63,7 @@ BOOST_AUTO_TEST_CASE(LinearRegressionWrongResponseSizeTest)
   SetInputParam("training_responses", std::move(y));
 
   Log::Fatal.ignoreInput = true;
-  BOOST_REQUIRE_THROW(MAIN(), std::runtime_error);
+  BOOST_REQUIRE_THROW(mlpackMain(), std::runtime_error);
   Log::Fatal.ignoreInput = false;
 }
 
diff --git a/src/mlpack/tests/main_tests/pca_test.cpp b/src/mlpack/tests/main_tests/pca_test.cpp
index 26aaf5983a..128a0f0378 100644
--- a/src/mlpack/tests/main_tests/pca_test.cpp
+++ b/src/mlpack/tests/main_tests/pca_test.cpp
@@ -7,8 +7,8 @@
 #include <string>
 
 #define BINDING_TYPE BINDING_TYPE_TEST
-#define PROGRAM_NAME pcaProgramName
-static const std::string pcaProgramName = "PrincipalComponentAnalysis";
+static const std::string testName = "PrincipalComponentAnalysis";
+
 #include <mlpack/core.hpp>
 #include <mlpack/core/util/mlpack_main.hpp>
 #include <mlpack/methods/pca/pca_main.cpp>
@@ -18,16 +18,6 @@ static const std::string pcaProgramName = "PrincipalComponentAnalysis";
 
 using namespace mlpack;
 
-namespace mlpack {
-namespace bindings {
-namespace tests {
-
-extern std::string programName;
-
-}
-}
-}
-
 // Utility function to set a parameter and mark it as passed, using copy
 // semantics.
 template<typename T>
@@ -52,7 +42,7 @@ struct PCATestFixture
   PCATestFixture()
   {
     // Cache in the options for this program.
-    CLI::RestoreSettings(pcaProgramName);
+    CLI::RestoreSettings(testName);
   }
 
   ~PCATestFixture()
@@ -75,7 +65,7 @@ BOOST_AUTO_TEST_CASE(PCADimensionTest)
   SetInputParam("input", std::move(x));
   SetInputParam("new_dimensionality", (int) 3);
 
-  MAIN();
+  mlpackMain();
 
   // Now check that the output has 3 dimensions.
   BOOST_REQUIRE_EQUAL(CLI::GetParam<arma::mat>("output").n_rows, 3);
@@ -95,7 +85,7 @@ BOOST_AUTO_TEST_CASE(PCAVarRetainTest)
   SetInputParam("scale", true);
   SetInputParam("new_dimensionality", (int) 3); // Should be ignored.
 
-  MAIN();
+  mlpackMain();
 
   // Check that the output has 5 dimensions.
   BOOST_REQUIRE_EQUAL(CLI::GetParam<arma::mat>("output").n_rows, 4);
@@ -114,7 +104,7 @@ BOOST_AUTO_TEST_CASE(PCANoVarRetainTest)
   SetInputParam("scale", true);
   SetInputParam("new_dimensionality", (int) 3); // Should be ignored.
 
-  MAIN();
+  mlpackMain();
 
   // Check that the output has 1 dimensions.
   BOOST_REQUIRE_EQUAL(CLI::GetParam<arma::mat>("output").n_rows, 1);
@@ -132,7 +122,7 @@ BOOST_AUTO_TEST_CASE(PCATooHighNewDimensionalityTest)
   SetInputParam("new_dimensionality", (int) 7); // Invalid.
 
   Log::Fatal.ignoreInput = true;
-  BOOST_REQUIRE_THROW(MAIN(), std::runtime_error);
+  BOOST_REQUIRE_THROW(mlpackMain(), std::runtime_error);
   Log::Fatal.ignoreInput = false;
 }
 

From cb58baa2195c1024a921cd88388bffc39571152f Mon Sep 17 00:00:00 2001
From: Eugene Freyman <evg.freyman@gmail.com>
Date: Wed, 20 Dec 2017 09:19:30 +0200
Subject: [PATCH 38/67] fixed checkstyle issue

---
 src/mlpack/core/util/mlpack_main.hpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/mlpack/core/util/mlpack_main.hpp b/src/mlpack/core/util/mlpack_main.hpp
index 11c21ee770..a26a5c6ff7 100644
--- a/src/mlpack/core/util/mlpack_main.hpp
+++ b/src/mlpack/core/util/mlpack_main.hpp
@@ -92,7 +92,7 @@ using Option = mlpack::bindings::tests::TestOption<T>;
 }
 }
 
-//testName symbol should be defined in each binding test file
+// testName symbol should be defined in each binding test file
 #include <mlpack/core/util/param.hpp>
 
 #undef PROGRAM_INFO

From 70d98e3ed84a872bbd325e71de6b143a8636a089 Mon Sep 17 00:00:00 2001
From: Marcus Edel <marcus.edel@fu-berlin.de>
Date: Wed, 20 Dec 2017 14:27:55 +0100
Subject: [PATCH 39/67] Remove 'const' type qualifier since it has no effect on
 the on return type.

---
 src/mlpack/core/optimizers/cmaes/full_selection.hpp   | 2 +-
 src/mlpack/core/optimizers/cmaes/random_selection.hpp | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/mlpack/core/optimizers/cmaes/full_selection.hpp b/src/mlpack/core/optimizers/cmaes/full_selection.hpp
index 9ff63eaa79..162f570bdd 100644
--- a/src/mlpack/core/optimizers/cmaes/full_selection.hpp
+++ b/src/mlpack/core/optimizers/cmaes/full_selection.hpp
@@ -32,7 +32,7 @@ class FullSelection
    * @param iterate starting point.
    */
   template<typename DecomposableFunctionType>
-  const double Select(DecomposableFunctionType& function,
+  double Select(DecomposableFunctionType& function,
                       const size_t batchSize,
                       const arma::mat& iterate)
   {
diff --git a/src/mlpack/core/optimizers/cmaes/random_selection.hpp b/src/mlpack/core/optimizers/cmaes/random_selection.hpp
index df4ecd05c2..0dd5def9dd 100644
--- a/src/mlpack/core/optimizers/cmaes/random_selection.hpp
+++ b/src/mlpack/core/optimizers/cmaes/random_selection.hpp
@@ -47,7 +47,7 @@ class RandomSelection
    * @param iterate starting point.
    */
   template<typename DecomposableFunctionType>
-  const double Select(DecomposableFunctionType& function,
+  double Select(DecomposableFunctionType& function,
                       const size_t batchSize,
                       const arma::mat& iterate)
   {

From d494dd16f95a979dd9fcae0da0b2f8c893835d0f Mon Sep 17 00:00:00 2001
From: Ryan Curtin <ryan@ratml.org>
Date: Wed, 20 Dec 2017 09:56:16 -0500
Subject: [PATCH 40/67] Update comments and apply fix to spalera_sgd.

---
 src/mlpack/core/optimizers/sgd/sgd_impl.hpp          |  7 ++++++-
 .../core/optimizers/spalera_sgd/spalera_sgd_impl.hpp | 12 +++++++++---
 2 files changed, 15 insertions(+), 4 deletions(-)

diff --git a/src/mlpack/core/optimizers/sgd/sgd_impl.hpp b/src/mlpack/core/optimizers/sgd/sgd_impl.hpp
index ff004f1c1f..f36606dff9 100644
--- a/src/mlpack/core/optimizers/sgd/sgd_impl.hpp
+++ b/src/mlpack/core/optimizers/sgd/sgd_impl.hpp
@@ -102,7 +102,12 @@ double SGD<UpdatePolicyType, DecayPolicyType>::Optimize(
         function.Shuffle();
     }
 
-    // Find the effective batch size (the last batch may be smaller).
+    // Find the effective batch size; we have to take the minimum of three
+    // things:
+    // - the batch size can't be larger than the user-specified batch size;
+    // - the batch size can't be larger than the number of iterations left
+    //       before actualMaxIterations is hit;
+    // - the batch size can't be larger than the number of functions left.
     const size_t effectiveBatchSize = std::min(
         std::min(batchSize, actualMaxIterations - i),
         numFunctions - currentFunction);
diff --git a/src/mlpack/core/optimizers/spalera_sgd/spalera_sgd_impl.hpp b/src/mlpack/core/optimizers/spalera_sgd/spalera_sgd_impl.hpp
index 1c6e47166f..afd0ca1ccc 100644
--- a/src/mlpack/core/optimizers/spalera_sgd/spalera_sgd_impl.hpp
+++ b/src/mlpack/core/optimizers/spalera_sgd/spalera_sgd_impl.hpp
@@ -108,9 +108,15 @@ double SPALeRASGD<DecayPolicyType>::Optimize(DecomposableFunctionType& function,
         function.Shuffle();
     }
 
-    // Find the effective batch size (the last batch may be smaller).
-    const size_t effectiveBatchSize = std::min(batchSize,
-        numFunctions - currentFunction);
+    // Find the effective batch size; we have to take the minimum of three
+    // things:
+    // - the batch size can't be larger than the user-specified batch size;
+    // - the batch size can't be larger than the number of iterations left
+    //       before actualMaxIterations is hit;
+    // - the batch size can't be larger than the number of functions left.
+    const size_t effectiveBatchSize = std::min(
+        std::min(batchSize, actualMaxIterations - i),
+        numFunctions - currentFunction);ons - currentFunction);
 
     function.Gradient(iterate, currentFunction, gradient, effectiveBatchSize);
 

From 83128ddfccbfd5380606eedcbaf2ec9a508ea59e Mon Sep 17 00:00:00 2001
From: Ryan Curtin <ryan@ratml.org>
Date: Wed, 20 Dec 2017 10:41:51 -0500
Subject: [PATCH 41/67] The tolerance can also be 0.

---
 .../methods/logistic_regression/logistic_regression_main.cpp  | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/mlpack/methods/logistic_regression/logistic_regression_main.cpp b/src/mlpack/methods/logistic_regression/logistic_regression_main.cpp
index 8f6dc937aa..425e89504e 100644
--- a/src/mlpack/methods/logistic_regression/logistic_regression_main.cpp
+++ b/src/mlpack/methods/logistic_regression/logistic_regression_main.cpp
@@ -162,8 +162,8 @@ void mlpackMain()
   ReportIgnoredParam({{ "test", false }}, "output_probabilities");
 
   // Tolerance needs to be positive.
-  RequireParamValue<double>("tolerance", [](double x) { return x > 0.0; },
-      true, "tolerance must be positive");
+  RequireParamValue<double>("tolerance", [](double x) { return x >= 0.0; },
+      true, "tolerance must be positive or zero");
 
   // Optimizer has to be L-BFGS or SGD.
   RequireParamInSet<string>("optimizer", { "lbfgs", "sgd" },

From f241d9ac13272e4bf2406505cc101d172392ad05 Mon Sep 17 00:00:00 2001
From: Ryan Curtin <ryan@ratml.org>
Date: Wed, 20 Dec 2017 13:27:03 -0500
Subject: [PATCH 42/67] Not sure how this mistake got in...

---
 src/mlpack/core/optimizers/spalera_sgd/spalera_sgd_impl.hpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/mlpack/core/optimizers/spalera_sgd/spalera_sgd_impl.hpp b/src/mlpack/core/optimizers/spalera_sgd/spalera_sgd_impl.hpp
index afd0ca1ccc..de228ee3fd 100644
--- a/src/mlpack/core/optimizers/spalera_sgd/spalera_sgd_impl.hpp
+++ b/src/mlpack/core/optimizers/spalera_sgd/spalera_sgd_impl.hpp
@@ -116,7 +116,7 @@ double SPALeRASGD<DecayPolicyType>::Optimize(DecomposableFunctionType& function,
     // - the batch size can't be larger than the number of functions left.
     const size_t effectiveBatchSize = std::min(
         std::min(batchSize, actualMaxIterations - i),
-        numFunctions - currentFunction);ons - currentFunction);
+        numFunctions - currentFunction);
 
     function.Gradient(iterate, currentFunction, gradient, effectiveBatchSize);
 

From 7d45f511921dfb0c227cfc7aa002381163190736 Mon Sep 17 00:00:00 2001
From: Ryan Curtin <ryan@ratml.org>
Date: Wed, 20 Dec 2017 20:37:09 -0500
Subject: [PATCH 43/67] Fix compilation warning.

---
 src/mlpack/methods/pca/pca_main.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/mlpack/methods/pca/pca_main.cpp b/src/mlpack/methods/pca/pca_main.cpp
index 9ad0a9e5cc..ff7f9307ac 100644
--- a/src/mlpack/methods/pca/pca_main.cpp
+++ b/src/mlpack/methods/pca/pca_main.cpp
@@ -119,7 +119,7 @@ void mlpackMain()
   error << "cannot be greater than existing dimensionality (" << dataset.n_rows
       << ")";
   RequireParamValue<int>("new_dimensionality",
-      [dataset](int x) { return x <= dataset.n_rows; }, true, error.str());
+      [dataset](int x) { return x <= (int) dataset.n_rows; }, true, error.str());
 
   RequireParamValue<double>("var_to_retain",
       [](double x) { return x >= 0.0 && x <= 1.0; }, true,

From 27288163b957f94449057a98e35c362f8a2e0bc1 Mon Sep 17 00:00:00 2001
From: Ryan Curtin <ryan@ratml.org>
Date: Wed, 20 Dec 2017 20:40:00 -0500
Subject: [PATCH 44/67] Fix uninitialized memory.

This tricky one only showed up on one particular compiler/library combination.
---
 src/mlpack/tests/frankwolfe_test.cpp | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/src/mlpack/tests/frankwolfe_test.cpp b/src/mlpack/tests/frankwolfe_test.cpp
index dff718c866..9cbe59e75d 100644
--- a/src/mlpack/tests/frankwolfe_test.cpp
+++ b/src/mlpack/tests/frankwolfe_test.cpp
@@ -70,7 +70,7 @@ BOOST_AUTO_TEST_CASE(regularizedOMP)
   mat B1 = 0.1 * eye(k, k);
   mat B2 = 100 * randn(k, k);
   mat A = join_horiz(B1, B2); // The dictionary is input as columns of A.
-  vec b(k); // Vector to be sparsely approximated.
+  vec b(k, arma::fill::zeros); // Vector to be sparsely approximated.
   b(0) = 1;
   b(1) = 1;
   vec lambda(A.n_cols);
@@ -81,6 +81,7 @@ BOOST_AUTO_TEST_CASE(regularizedOMP)
   ConstrLpBallSolver linearConstrSolver(1, lambda);
   UpdateSpan updateRule;
 
+  Log::Info.ignoreInput = false;
   OMP s(linearConstrSolver, updateRule);
 
   vec coordinates = zeros<vec>(2 * k);
@@ -98,8 +99,8 @@ BOOST_AUTO_TEST_CASE(PruneSupportOMP)
   int k = 3;
   mat B1;
   B1 << 1 << 0 << 1 << endr
-    << 0 << 1 << 1 << endr
-    << 0 << 0 << 1 << endr;
+     << 0 << 1 << 1 << endr
+     << 0 << 0 << 1 << endr;
   mat B2 = randu(k, k);
   mat A = join_horiz(B1, B2); // The dictionary is input as columns of A.
   vec b;

From e6b84867e043fc78028450ad3ff34decf423ce37 Mon Sep 17 00:00:00 2001
From: Ryan Curtin <ryan@ratml.org>
Date: Wed, 20 Dec 2017 20:40:37 -0500
Subject: [PATCH 45/67] Disable debugging output.

---
 src/mlpack/tests/frankwolfe_test.cpp | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/mlpack/tests/frankwolfe_test.cpp b/src/mlpack/tests/frankwolfe_test.cpp
index 9cbe59e75d..c3d2a6e0cf 100644
--- a/src/mlpack/tests/frankwolfe_test.cpp
+++ b/src/mlpack/tests/frankwolfe_test.cpp
@@ -81,7 +81,6 @@ BOOST_AUTO_TEST_CASE(regularizedOMP)
   ConstrLpBallSolver linearConstrSolver(1, lambda);
   UpdateSpan updateRule;
 
-  Log::Info.ignoreInput = false;
   OMP s(linearConstrSolver, updateRule);
 
   vec coordinates = zeros<vec>(2 * k);

From 84dd7bfa45d52b9d409e78a8f48869f0123cb092 Mon Sep 17 00:00:00 2001
From: Eugene Freyman <evg.freyman@gmail.com>
Date: Thu, 21 Dec 2017 11:24:32 +0200
Subject: [PATCH 46/67] renaming programName to testName

---
 src/mlpack/bindings/tests/test_option.hpp | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/src/mlpack/bindings/tests/test_option.hpp b/src/mlpack/bindings/tests/test_option.hpp
index f80596d580..8f915d3022 100644
--- a/src/mlpack/bindings/tests/test_option.hpp
+++ b/src/mlpack/bindings/tests/test_option.hpp
@@ -53,6 +53,8 @@ class TestOption
    * @param input Whether or not the option is an input option.
    * @param noTranspose If the parameter is a matrix and this is true, then the
    *      matrix will not be transposed on loading.
+   * @param testName Name of the test (used for identifiying which binding test 
+   * this option belongs to)
    */
   TestOption(const N defaultValue,
              const std::string& identifier,
@@ -62,7 +64,7 @@ class TestOption
              const bool required = false,
              const bool input = true,
              const bool noTranspose = false,
-             const std::string& programName = "")
+             const std::string& testName = "")
   {
     // Create the ParamData object to give to CLI.
     util::ParamData data;
@@ -82,7 +84,7 @@ class TestOption
 
     const std::string tname = data.tname;
 
-    CLI::RestoreSettings(programName, false);
+    CLI::RestoreSettings(testName, false);
 
     // Set some function pointers that we need.
     CLI::GetSingleton().functionMap[tname]["GetPrintableParam"] =
@@ -95,7 +97,7 @@ class TestOption
     if (!input)
       CLI::SetPassed(identifier);
 
-    CLI::StoreSettings(programName);
+    CLI::StoreSettings(testName);
     CLI::ClearSettings();
   }
 };

From c34b1af04bc884a75d339bfd2059be3ae809ed50 Mon Sep 17 00:00:00 2001
From: Eugene Freyman <evg.freyman@gmail.com>
Date: Thu, 21 Dec 2017 11:28:18 +0200
Subject: [PATCH 47/67] Added comments about testName

---
 src/mlpack/bindings/python/py_option.hpp | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/mlpack/bindings/python/py_option.hpp b/src/mlpack/bindings/python/py_option.hpp
index e5c4b059d4..60eaf3a1c4 100644
--- a/src/mlpack/bindings/python/py_option.hpp
+++ b/src/mlpack/bindings/python/py_option.hpp
@@ -33,7 +33,8 @@ class PyOption
  public:
   /**
    * Construct a PyOption object.  When constructed, it will register itself
-   * with CLI.
+   * with CLI. The testName parameter is not used and added for compatibility 
+   * reasons.
    */
   PyOption(const T defaultValue,
            const std::string& identifier,
@@ -43,7 +44,7 @@ class PyOption
            const bool required = false,
            const bool input = true,
            const bool noTranspose = false,
-           const std::string& /*programName*/ = "")
+           const std::string& /*testName*/ = "")
   {
     // Create the ParamData object to give to CLI.
     util::ParamData data;

From 1b3477c6adf829e83c343c618666cade3a30da0c Mon Sep 17 00:00:00 2001
From: Eugene Freyman <evg.freyman@gmail.com>
Date: Thu, 21 Dec 2017 11:31:47 +0200
Subject: [PATCH 48/67] comment on testName

---
 src/mlpack/bindings/cli/cli_option.hpp | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/mlpack/bindings/cli/cli_option.hpp b/src/mlpack/bindings/cli/cli_option.hpp
index 93003eccc1..47233b29ec 100644
--- a/src/mlpack/bindings/cli/cli_option.hpp
+++ b/src/mlpack/bindings/cli/cli_option.hpp
@@ -60,6 +60,7 @@ class CLIOption
    * @param input Whether or not the option is an input option.
    * @param noTranspose If the parameter is a matrix and this is true, then the
    *      matrix will not be transposed on loading.
+   * @param testName Is not used and added for compatibility reasons.
    */
   CLIOption(const N defaultValue,
             const std::string& identifier,
@@ -69,7 +70,7 @@ class CLIOption
             const bool required = false,
             const bool input = true,
             const bool noTranspose = false,
-            const std::string& /*programName*/ = "")
+            const std::string& /*testName*/ = "")
   {
     // Create the ParamData object to give to CLI.
     util::ParamData data;

From 19cd926b907ccb70caa0e650baf5cc7a763c9f78 Mon Sep 17 00:00:00 2001
From: Eugene Freyman <evg.freyman@gmail.com>
Date: Thu, 21 Dec 2017 11:33:14 +0200
Subject: [PATCH 49/67] minor style fixes

---
 src/mlpack/bindings/tests/test_option.hpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/mlpack/bindings/tests/test_option.hpp b/src/mlpack/bindings/tests/test_option.hpp
index 8f915d3022..336d9b93db 100644
--- a/src/mlpack/bindings/tests/test_option.hpp
+++ b/src/mlpack/bindings/tests/test_option.hpp
@@ -54,7 +54,7 @@ class TestOption
    * @param noTranspose If the parameter is a matrix and this is true, then the
    *      matrix will not be transposed on loading.
    * @param testName Name of the test (used for identifiying which binding test 
-   * this option belongs to)
+   *      this option belongs to)
    */
   TestOption(const N defaultValue,
              const std::string& identifier,

From a81956777b957fb0b9ce3dd60638d31f36057750 Mon Sep 17 00:00:00 2001
From: Marcus Edel <marcus.edel@fu-berlin.de>
Date: Thu, 21 Dec 2017 15:09:55 +0100
Subject: [PATCH 50/67] Resolve merge and minor style issues.

---
 src/mlpack/core/optimizers/adam/adam.hpp | 19 ++++++++++---------
 src/mlpack/tests/adam_test.cpp           |  2 +-
 2 files changed, 11 insertions(+), 10 deletions(-)

diff --git a/src/mlpack/core/optimizers/adam/adam.hpp b/src/mlpack/core/optimizers/adam/adam.hpp
index 07184dcb6e..9a2c1d732a 100644
--- a/src/mlpack/core/optimizers/adam/adam.hpp
+++ b/src/mlpack/core/optimizers/adam/adam.hpp
@@ -6,11 +6,11 @@
  * @author Vivek Pal
  * @author Sourabh Varshney
  *
- * Adam, AdaMax, AMSGrad and Nadam optimizers. Adam is an an algorithm for 
- * first-order gradient-based optimization of stochastic objective 
- * functions, based on adaptive estimates of lower-order moments. 
- * AdaMax is simply a variant of Adam based on the infinity norm. 
- * AMSGrad is another variant of Adam with guaranteed convergence.
+ * Adam, AdaMax, AMSGrad and Nadam optimizers. Adam is an an algorithm for
+ * first-order gradient-based optimization of stochastic objective functions,
+ * based on adaptive estimates of lower-order moments. AdaMax is simply a
+ * variant of Adam based on the infinity norm. AMSGrad is another variant of
+ * Adam with guaranteed convergence.
  *
  * mlpack is free software; you may redistribute it and/or modify it under the
  * terms of the 3-clause BSD license.  You should have received a copy of the
@@ -25,7 +25,7 @@
 #include <mlpack/core/optimizers/sgd/sgd.hpp>
 #include "adam_update.hpp"
 #include "adamax_update.hpp"
-#include "amsgrad_update.hpp
+#include "amsgrad_update.hpp"
 #include "nadam_update.hpp"
 
 namespace mlpack {
@@ -35,7 +35,7 @@ namespace optimization {
  * Adam is an optimizer that computes individual adaptive learning rates for
  * different parameters from estimates of first and second moments of the
  * gradients. AdaMax is a variant of Adam based on the infinity norm as given
- * in the section 7 of the following paper. Nadam is an optimizer that 
+ * in the section 7 of the following paper. Nadam is an optimizer that
  * combines the Adam and NAG.
  *
  * For more information, see the following.
@@ -55,8 +55,9 @@ namespace optimization {
  * }
  * @endcode
  *
- * For Adam, AdaMax, AMSGrad and Nadam to work, a DecomposableFunctionType template
- * parameter is required. This class must implement the following function:
+ * For Adam, AdaMax, AMSGrad and Nadam to work, a DecomposableFunctionType
+ * template parameter is required. This class must implement the following
+ * function:
  *
  *   size_t NumFunctions();
  *   double Evaluate(const arma::mat& coordinates,
diff --git a/src/mlpack/tests/adam_test.cpp b/src/mlpack/tests/adam_test.cpp
index 8c9c2062f3..9c954af6cf 100644
--- a/src/mlpack/tests/adam_test.cpp
+++ b/src/mlpack/tests/adam_test.cpp
@@ -329,4 +329,4 @@ BOOST_AUTO_TEST_CASE(NadamLogisticRegressionTest)
   BOOST_REQUIRE_CLOSE(testAcc, 100.0, 0.6); // 0.6% error tolerance.
 }
 
-BOOST_AUTO_TEST_SUITE_END();
\ No newline at end of file
+BOOST_AUTO_TEST_SUITE_END();

From 32d1cff3c201536704cad0d84290f2ecbaec59e4 Mon Sep 17 00:00:00 2001
From: Ryan Curtin <ryan@ratml.org>
Date: Thu, 21 Dec 2017 14:39:24 -0500
Subject: [PATCH 51/67] Sometimes QUIC-SVD can fail, so perform multiple runs.

Also increase the tolerance for another test to prevent failures.
---
 src/mlpack/tests/pca_test.cpp           |  3 ++-
 src/mlpack/tests/quic_svd_test.cpp      | 33 ++++++++++++++++---------
 src/mlpack/tests/random_forest_test.cpp |  8 +++---
 3 files changed, 28 insertions(+), 16 deletions(-)

diff --git a/src/mlpack/tests/pca_test.cpp b/src/mlpack/tests/pca_test.cpp
index dae8038542..904a6eeb4f 100644
--- a/src/mlpack/tests/pca_test.cpp
+++ b/src/mlpack/tests/pca_test.cpp
@@ -226,13 +226,14 @@ BOOST_AUTO_TEST_CASE(RandomizedPCADimensionalityReductionTest)
  */
 BOOST_AUTO_TEST_CASE(QUICPCADimensionalityReductionTest)
 {
+  math::RandomSeed(std::time(NULL));
   arma::mat data, data1;
   data::Load("test_data_3_1000.csv", data);
   data1 = data;
 
   // It isn't guaranteed that the QUIC-SVD will match with the exact SVD method,
   // starting with random samples. If this works 1 of 5 times, I'm fine with
-  // that. All I want to know is that the QUIC-SVD method is  able to solve the
+  // that. All I want to know is that the QUIC-SVD method is able to solve the
   // task and is at least as good as the exact method (plus a little bit for
   // noise).
   size_t successes = 0;
diff --git a/src/mlpack/tests/quic_svd_test.cpp b/src/mlpack/tests/quic_svd_test.cpp
index dfc128b512..86ddb6486b 100644
--- a/src/mlpack/tests/quic_svd_test.cpp
+++ b/src/mlpack/tests/quic_svd_test.cpp
@@ -29,18 +29,29 @@ BOOST_AUTO_TEST_CASE(QUICSVDReconstructionError)
   arma::mat dataset;
   data::Load("test_data_3_1000.csv", dataset);
 
-  // Obtain the SVD using default parameters.
-  arma::mat u, v, sigma;
-  svd::QUIC_SVD quicsvd(dataset, u, v, sigma);
+  // The QUIC-SVD procedure can fail---the Monte Carlo error calculation is
+  // random.  Therefore we simply require at least one success.
+  size_t successes = 0;
+  for (size_t i = 0; i < 3; ++i)
+  {
+    // Obtain the SVD using default parameters.
+    Log::Info.ignoreInput = false;
+    Log::Warn.ignoreInput = false;
+    arma::mat u, v, sigma;
+    svd::QUIC_SVD quicsvd(dataset, u, v, sigma);
 
-  // Reconstruct the matrix using the SVD.
-  arma::mat reconstruct;
-  reconstruct = u * sigma * v.t();
+    // Reconstruct the matrix using the SVD.
+    arma::mat reconstruct;
+    reconstruct = u * sigma * v.t();
 
-  // The relative reconstruction error should be small.
-  double relativeError = arma::norm(dataset - reconstruct, "frob") /
-                         arma::norm(dataset, "frob");
-  BOOST_REQUIRE_SMALL(relativeError, 1e-5);
+    // The relative reconstruction error should be small.
+    double relativeError = arma::norm(dataset - reconstruct, "frob") /
+                           arma::norm(dataset, "frob");
+    if (relativeError < 1e-5)
+      ++successes;
+  }
+
+  BOOST_REQUIRE_GT(successes, 0);
 }
 
 /**
@@ -71,7 +82,7 @@ BOOST_AUTO_TEST_CASE(QUICSVDSingularValueError)
 
   // The sigular value error should be small.
   double error = arma::norm(s1 - s3);
-  BOOST_REQUIRE_SMALL(error, 0.05);
+  BOOST_REQUIRE_SMALL(error, 0.1);
 }
 
 BOOST_AUTO_TEST_CASE(QUICSVDSameDimensionTest)
diff --git a/src/mlpack/tests/random_forest_test.cpp b/src/mlpack/tests/random_forest_test.cpp
index 6e910e8f71..6b8c385f19 100644
--- a/src/mlpack/tests/random_forest_test.cpp
+++ b/src/mlpack/tests/random_forest_test.cpp
@@ -224,7 +224,7 @@ BOOST_AUTO_TEST_CASE(UnweightedCategoricalLearningTest)
   arma::Row<size_t> testLabels = l.subvec(2000, 3999);
 
   // Train a random forest and a decision tree.
-  RandomForest<> rf(trainingData, di, trainingLabels, 5, 10 /* 10 trees */, 5);
+  RandomForest<> rf(trainingData, di, trainingLabels, 5, 15 /* 15 trees */, 5);
   DecisionTree<> dt(trainingData, di, trainingLabels, 5, 5);
 
   // Get performance statistics on test data.
@@ -238,7 +238,7 @@ BOOST_AUTO_TEST_CASE(UnweightedCategoricalLearningTest)
   size_t rfCorrect = arma::accu(rfPredictions == testLabels);
   size_t dtCorrect = arma::accu(dtPredictions == testLabels);
 
-  BOOST_REQUIRE_GE(rfCorrect, dtCorrect);
+  BOOST_REQUIRE_GE(rfCorrect, dtCorrect - 30);
   BOOST_REQUIRE_GE(rfCorrect, size_t(0.7 * testData.n_cols));
 }
 
@@ -281,7 +281,7 @@ BOOST_AUTO_TEST_CASE(WeightedCategoricalLearningTest)
   arma::Row<size_t> fullLabels = arma::join_rows(trainingLabels, randomLabels);
 
   // Build a random forest and a decision tree.
-  RandomForest<> rf(fullData, di, fullLabels, 5, 10 /* 10 trees */, 5);
+  RandomForest<> rf(fullData, di, fullLabels, 5, 15 /* 15 trees */, 5);
   DecisionTree<> dt(fullData, di, fullLabels, 5, 5);
 
   // Get performance statistics on test data.
@@ -295,7 +295,7 @@ BOOST_AUTO_TEST_CASE(WeightedCategoricalLearningTest)
   size_t rfCorrect = arma::accu(rfPredictions == testLabels);
   size_t dtCorrect = arma::accu(dtPredictions == testLabels);
 
-  BOOST_REQUIRE_GE(rfCorrect, dtCorrect);
+  BOOST_REQUIRE_GE(rfCorrect, dtCorrect - 30);
   BOOST_REQUIRE_GE(rfCorrect, size_t(0.7 * testData.n_cols));
 }
 

From b49d7af4bf4fbf0cfe8385a785f85d4ca4796597 Mon Sep 17 00:00:00 2001
From: Ryan Curtin <ryan@ratml.org>
Date: Thu, 21 Dec 2017 16:53:25 -0500
Subject: [PATCH 52/67] Make sure to restore the data before a second trial.

---
 src/mlpack/tests/pca_test.cpp | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/src/mlpack/tests/pca_test.cpp b/src/mlpack/tests/pca_test.cpp
index 904a6eeb4f..5b4ec46830 100644
--- a/src/mlpack/tests/pca_test.cpp
+++ b/src/mlpack/tests/pca_test.cpp
@@ -226,11 +226,12 @@ BOOST_AUTO_TEST_CASE(RandomizedPCADimensionalityReductionTest)
  */
 BOOST_AUTO_TEST_CASE(QUICPCADimensionalityReductionTest)
 {
-  math::RandomSeed(std::time(NULL));
   arma::mat data, data1;
   data::Load("test_data_3_1000.csv", data);
   data1 = data;
 
+  arma::mat backupData(data);
+
   // It isn't guaranteed that the QUIC-SVD will match with the exact SVD method,
   // starting with random samples. If this works 1 of 5 times, I'm fine with
   // that. All I want to know is that the QUIC-SVD method is able to solve the
@@ -239,13 +240,18 @@ BOOST_AUTO_TEST_CASE(QUICPCADimensionalityReductionTest)
   size_t successes = 0;
   for (size_t trial = 0; trial < 5; ++trial)
   {
+    if (trial > 0)
+    {
+      data = backupData;
+      data1 = backupData;
+    }
+
     PCAType<ExactSVDPolicy> exactPCA;
     const double varRetainedExact = exactPCA.Apply(data, 1);
 
     PCAType<QUICSVDPolicy> quicPCA;
     const double varRetainedQUIC = quicPCA.Apply(data1, 1);
 
-
     if (std::abs(varRetainedExact - varRetainedQUIC) < 0.2)
     {
       ++successes;

From f1363cd8f1bbceaf57a1fe642fa9f724b2045b6d Mon Sep 17 00:00:00 2001
From: Ryan Curtin <ryan@ratml.org>
Date: Thu, 21 Dec 2017 16:54:10 -0500
Subject: [PATCH 53/67] Use safer check to see if the queue is empty.

---
 src/mlpack/core/tree/cosine_tree/cosine_tree.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/mlpack/core/tree/cosine_tree/cosine_tree.cpp b/src/mlpack/core/tree/cosine_tree/cosine_tree.cpp
index 61159a6001..40a747c875 100644
--- a/src/mlpack/core/tree/cosine_tree/cosine_tree.cpp
+++ b/src/mlpack/core/tree/cosine_tree/cosine_tree.cpp
@@ -94,7 +94,7 @@ CosineTree::CosineTree(const arma::mat& dataset,
   // Initialize Monte Carlo error estimate for comparison.
   double monteCarloError = root.FrobNormSquared();
 
-  while (treeQueue.top() &&
+  while (treeQueue.size() > 0 &&
          (monteCarloError > epsilon * root.FrobNormSquared()))
   {
     // Pop node from queue with highest projection error.

From 59c234c443d837cc1013d331b644089779e84bf4 Mon Sep 17 00:00:00 2001
From: Ryan Curtin <ryan@ratml.org>
Date: Thu, 21 Dec 2017 18:30:13 -0500
Subject: [PATCH 54/67] Adjust tolerances for PCAScalingTest.

---
 src/mlpack/tests/pca_test.cpp | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/src/mlpack/tests/pca_test.cpp b/src/mlpack/tests/pca_test.cpp
index 5b4ec46830..08f37c9b43 100644
--- a/src/mlpack/tests/pca_test.cpp
+++ b/src/mlpack/tests/pca_test.cpp
@@ -301,21 +301,21 @@ BOOST_AUTO_TEST_CASE(PCAScalingTest)
   // The first two components of the eigenvector with largest eigenvalue should
   // be somewhere near sqrt(2) / 2.  The third component should be close to
   // zero.  There is noise, of course...
-  BOOST_REQUIRE_CLOSE(std::abs(eigvec(0, 0)), sqrt(2) / 2, 0.2);
-  BOOST_REQUIRE_CLOSE(std::abs(eigvec(1, 0)), sqrt(2) / 2, 0.2);
-  BOOST_REQUIRE_SMALL(eigvec(2, 0), 0.08); // Large tolerance for noise.
+  BOOST_REQUIRE_CLOSE(std::abs(eigvec(0, 0)), sqrt(2) / 2, 0.35);
+  BOOST_REQUIRE_CLOSE(std::abs(eigvec(1, 0)), sqrt(2) / 2, 0.35);
+  BOOST_REQUIRE_SMALL(eigvec(2, 0), 0.1); // Large tolerance for noise.
 
   // The second component should be focused almost entirely in the third
   // dimension.
-  BOOST_REQUIRE_SMALL(eigvec(0, 1), 0.08);
-  BOOST_REQUIRE_SMALL(eigvec(1, 1), 0.08);
-  BOOST_REQUIRE_CLOSE(std::abs(eigvec(2, 1)), 1.0, 0.2);
+  BOOST_REQUIRE_SMALL(eigvec(0, 1), 0.1);
+  BOOST_REQUIRE_SMALL(eigvec(1, 1), 0.1);
+  BOOST_REQUIRE_CLOSE(std::abs(eigvec(2, 1)), 1.0, 0.35);
 
   // The third component should have the same absolute value characteristics as
   // the first (plus 20% tolerance).
-  BOOST_REQUIRE_CLOSE(std::abs(eigvec(0, 0)), sqrt(2) / 2, 0.2);
-  BOOST_REQUIRE_CLOSE(std::abs(eigvec(1, 0)), sqrt(2) / 2, 0.2);
-  BOOST_REQUIRE_SMALL(eigvec(2, 0), 0.08); // Large tolerance for noise.
+  BOOST_REQUIRE_CLOSE(std::abs(eigvec(0, 0)), sqrt(2) / 2, 0.35);
+  BOOST_REQUIRE_CLOSE(std::abs(eigvec(1, 0)), sqrt(2) / 2, 0.35);
+  BOOST_REQUIRE_SMALL(eigvec(2, 0), 0.1); // Large tolerance for noise.
 
   // The eigenvalues should sum to three.
   BOOST_REQUIRE_CLOSE(accu(eigval), 3.0, 0.1); // 10% tolerance.

From 7e4eec1c3b5eba64ca8c7c699eb18d7dd3b37331 Mon Sep 17 00:00:00 2001
From: Ryan Curtin <ryan@ratml.org>
Date: Thu, 21 Dec 2017 18:30:29 -0500
Subject: [PATCH 55/67] Adjust tolerances for MomentumSGD tests.

---
 src/mlpack/tests/momentum_sgd_test.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/mlpack/tests/momentum_sgd_test.cpp b/src/mlpack/tests/momentum_sgd_test.cpp
index 42fd1d3499..f17aa2aa7f 100644
--- a/src/mlpack/tests/momentum_sgd_test.cpp
+++ b/src/mlpack/tests/momentum_sgd_test.cpp
@@ -36,7 +36,7 @@ BOOST_AUTO_TEST_CASE(MomentumSGDSpeedUpTestFunction)
   arma::mat coordinates = f.GetInitialPoint();
   double result = s.Optimize(f, coordinates);
 
-  BOOST_REQUIRE_CLOSE(result, -1.0, 0.05);
+  BOOST_REQUIRE_CLOSE(result, -1.0, 0.15);
   BOOST_REQUIRE_SMALL(coordinates[0], 1e-3);
   BOOST_REQUIRE_SMALL(coordinates[1], 1e-7);
   BOOST_REQUIRE_SMALL(coordinates[2], 1e-7);

From 23a49d96fed3109db9ac0f0af78e78f337d4508f Mon Sep 17 00:00:00 2001
From: Ryan Curtin <ryan@ratml.org>
Date: Thu, 21 Dec 2017 18:43:55 -0500
Subject: [PATCH 56/67] Adjust optimizer settings to prevent failures.

---
 src/mlpack/tests/cne_test.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/mlpack/tests/cne_test.cpp b/src/mlpack/tests/cne_test.cpp
index c4f528bc24..a1597b0de0 100644
--- a/src/mlpack/tests/cne_test.cpp
+++ b/src/mlpack/tests/cne_test.cpp
@@ -127,7 +127,7 @@ BOOST_AUTO_TEST_CASE(CNELogisticRegressionTest)
     testResponses[i] = 1;
   }
 
-  CNE opt(30, 500, 0.2, 0.2, 0.3, 65, -1);
+  CNE opt(200, 10000, 0.2, 0.2, 0.3, 65, -1);
 
   LogisticRegression<> lr(shuffledData, shuffledResponses, opt, 0.5);
 

From e3afe032d263940dae294e48eff87006fb35e5c2 Mon Sep 17 00:00:00 2001
From: Ryan Curtin <ryan@ratml.org>
Date: Fri, 22 Dec 2017 10:40:50 -0500
Subject: [PATCH 57/67] Run the XOR test multiple times if needed.

---
 src/mlpack/tests/cne_test.cpp | 68 ++++++++++++++++++++---------------
 1 file changed, 40 insertions(+), 28 deletions(-)

diff --git a/src/mlpack/tests/cne_test.cpp b/src/mlpack/tests/cne_test.cpp
index a1597b0de0..f0c13505ad 100644
--- a/src/mlpack/tests/cne_test.cpp
+++ b/src/mlpack/tests/cne_test.cpp
@@ -47,37 +47,49 @@ BOOST_AUTO_TEST_CASE(CNEXORTest)
   arma::mat train("1, 0, 0, 1; 1, 0, 1, 0");
   arma::mat labels("1, 1, 2, 2");
 
-  // network with 2 input 2 hidden and 2 output layer
-  FFN<NegativeLogLikelihood<> > network;
-
-  network.Add<Linear<> >(2, 2);
-  network.Add<SigmoidLayer<> >();
-  network.Add<Linear<> >(2, 2);
-  network.Add<LogSoftMax<> >();
-
-  // CNE object.
-  CNE opt(60, 5000, 0.1, 0.02, 0.2, 0.1, -1);
-
-  // Training the network with CNE
-  network.Train(train, labels, opt);
-
-  // Predicting for the same train data
-  arma::mat predictionTemp;
-  network.Predict(train, predictionTemp);
-
-  arma::mat prediction = arma::zeros<arma::mat>(1, predictionTemp.n_cols);
-
-  for (size_t i = 0; i < predictionTemp.n_cols; ++i)
+  // CNE may fail to find a good optimum.  But if it can succeed one out of 6
+  // times I think that is sufficient to say it is working.
+  size_t successes = 0;
+  for (size_t trial = 0; trial < 6; ++trial)
   {
-    prediction(i) = arma::as_scalar(arma::find(
-        arma::max(predictionTemp.col(i)) == predictionTemp.col(i), 1)) + 1;
+    // Build a network with 2 input, 2 hidden, and 2 output layers.
+    FFN<NegativeLogLikelihood<> > network;
+
+    network.Add<Linear<> >(2, 2);
+    network.Add<SigmoidLayer<> >();
+    network.Add<Linear<> >(2, 2);
+    network.Add<LogSoftMax<> >();
+
+    // CNE object.
+    CNE opt(60, 5000, 0.1, 0.02, 0.2, 0.1, -1);
+
+    // Training the network with CNE
+    network.Train(train, labels, opt);
+
+    // Predicting for the same train data
+    arma::mat predictionTemp;
+    network.Predict(train, predictionTemp);
+
+    arma::mat prediction = arma::zeros<arma::mat>(1, predictionTemp.n_cols);
+
+    for (size_t i = 0; i < predictionTemp.n_cols; ++i)
+    {
+      prediction(i) = arma::as_scalar(arma::find(
+          arma::max(predictionTemp.col(i)) == predictionTemp.col(i), 1)) + 1;
+    }
+
+    // 1 means 0 and 2 means 1 as the output to XOR.
+    if ((prediction[0] == 1) &&
+        (prediction[1] == 1) &&
+        (prediction[2] == 2) &&
+        (prediction[3] == 2))
+    {
+      ++successes;
+      break;
+    }
   }
 
-  // 1 means 0 and 2 means 1 as the output to XOR
-  BOOST_REQUIRE_EQUAL(1, prediction[0]);
-  BOOST_REQUIRE_EQUAL(1, prediction[1]);
-  BOOST_REQUIRE_EQUAL(2, prediction[2]);
-  BOOST_REQUIRE_EQUAL(2, prediction[3]);
+  BOOST_REQUIRE_GT(successes, 0);
 }
 
 /**

From 73f9d96757e7dc38e950e8a06101a9c1b67c5cea Mon Sep 17 00:00:00 2001
From: Ryan Curtin <ryan@ratml.org>
Date: Fri, 22 Dec 2017 10:41:10 -0500
Subject: [PATCH 58/67] Adjust tolerances for SPALeRA test to avoid random
 failures.

---
 src/mlpack/tests/spalera_sgd_test.cpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/mlpack/tests/spalera_sgd_test.cpp b/src/mlpack/tests/spalera_sgd_test.cpp
index 022ee50957..09090fbf43 100644
--- a/src/mlpack/tests/spalera_sgd_test.cpp
+++ b/src/mlpack/tests/spalera_sgd_test.cpp
@@ -79,10 +79,10 @@ BOOST_AUTO_TEST_CASE(LogisticRegressionTest)
 
     // Ensure that the error is close to zero.
     const double acc = lr.ComputeAccuracy(data, responses);
-    BOOST_REQUIRE_CLOSE(acc, 100.0, 0.3); // 0.3% error tolerance.
+    BOOST_REQUIRE_CLOSE(acc, 100.0, 0.5); // 0.5% error tolerance.
 
     const double testAcc = lr.ComputeAccuracy(testData, testResponses);
-    BOOST_REQUIRE_CLOSE(testAcc, 100.0, 0.6); // 0.6% error tolerance.
+    BOOST_REQUIRE_CLOSE(testAcc, 100.0, 0.8); // 0.8% error tolerance.
   }
 }
 

From 2d04abc9fc5d1a132e031c4e182ec2d6a5d51e8a Mon Sep 17 00:00:00 2001
From: Ryan Curtin <ryan@ratml.org>
Date: Fri, 22 Dec 2017 11:38:15 -0500
Subject: [PATCH 59/67] Adjust tolerance to reduce failures.

---
 src/mlpack/tests/svd_batch_test.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/mlpack/tests/svd_batch_test.cpp b/src/mlpack/tests/svd_batch_test.cpp
index 342b5634c1..e926e810c4 100644
--- a/src/mlpack/tests/svd_batch_test.cpp
+++ b/src/mlpack/tests/svd_batch_test.cpp
@@ -111,7 +111,7 @@ BOOST_AUTO_TEST_CASE(SVDBatchMomentumTest)
 
   const double momentumRMSE = amf2.Apply(cleanedData, 2, m1, m2);
 
-  BOOST_REQUIRE_LE(momentumRMSE, regularRMSE + 0.05);
+  BOOST_REQUIRE_LE(momentumRMSE, regularRMSE + 0.08);
 }
 
 /**

From 60ac702b0f7c199dc62771ca0274b2ded9e005fd Mon Sep 17 00:00:00 2001
From: Ryan Curtin <ryan@ratml.org>
Date: Fri, 22 Dec 2017 15:32:34 -0500
Subject: [PATCH 60/67] Adjust tolerances to prevent random failures.

---
 src/mlpack/tests/momentum_sgd_test.cpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/mlpack/tests/momentum_sgd_test.cpp b/src/mlpack/tests/momentum_sgd_test.cpp
index f17aa2aa7f..649bc89d59 100644
--- a/src/mlpack/tests/momentum_sgd_test.cpp
+++ b/src/mlpack/tests/momentum_sgd_test.cpp
@@ -65,12 +65,12 @@ BOOST_AUTO_TEST_CASE(GeneralizedRosenbrockTest)
     // Create the generalized Rosenbrock function.
     GeneralizedRosenbrockFunction f(i);
     MomentumUpdate momentumUpdate(0.4);
-    MomentumSGD s(0.001, 1, 0, 1e-15, true, momentumUpdate);
+    MomentumSGD s(0.0008, 1, 0, 1e-15, true, momentumUpdate);
 
     arma::mat coordinates = f.GetInitialPoint();
     double result = s.Optimize(f, coordinates);
 
-    BOOST_REQUIRE_SMALL(result, 1e-10);
+    BOOST_REQUIRE_SMALL(result, 1e-4);
     for (size_t j = 0; j < i; ++j)
       BOOST_REQUIRE_CLOSE(coordinates[j], (double) 1.0, 1e-3);
   }

From 240065954d74d9c1537a79f9e0cdc71c0018e256 Mon Sep 17 00:00:00 2001
From: Ryan Curtin <ryan@ratml.org>
Date: Fri, 22 Dec 2017 16:03:33 -0500
Subject: [PATCH 61/67] Perform multiple trials of VanillaNetworkWithCNETest.

---
 src/mlpack/tests/cne_test.cpp | 69 ++++++++++++++++++++---------------
 1 file changed, 40 insertions(+), 29 deletions(-)

diff --git a/src/mlpack/tests/cne_test.cpp b/src/mlpack/tests/cne_test.cpp
index f0c13505ad..3e9a7e57be 100644
--- a/src/mlpack/tests/cne_test.cpp
+++ b/src/mlpack/tests/cne_test.cpp
@@ -171,41 +171,52 @@ BOOST_AUTO_TEST_CASE(VanillaNetworkWithCNETest)
   data::Load("iris_test_labels.csv", testLabels, true);
   testLabels += 1;
 
-  // Create vanilla network with 4 input, 4 hidden and 3 output nodes.
-  FFN<NegativeLogLikelihood<> > model;
-  model.Add<Linear<> >(trainData.n_rows, 4);
-  model.Add<SigmoidLayer<> >();
-  model.Add<Linear<> >(4, 3);
-  model.Add<LogSoftMax<> >();
-
-  // Creating CNE object.
-  // The tolerance and objectiveChange are not taken into consideration.
-  CNE opt(30, 200, 0.2, 0.2, 0.3, -1, -1);
-
-  model.Train(trainData, trainLabels, opt);
-
-  arma::mat predictionTemp;
-  model.Predict(testData, predictionTemp);
-  arma::mat prediction = arma::zeros<arma::mat>(1, predictionTemp.n_cols);
-
-  for (size_t i = 0; i < predictionTemp.n_cols; ++i)
+  // Training the network may fail, so we will try a few times.
+  size_t successes = 0;
+  for (size_t trial = 0; trial < 4; ++trial)
   {
-    prediction(i) = arma::as_scalar(arma::find(
-        arma::max(predictionTemp.col(i)) == predictionTemp.col(i), 1)) + 1;
-  }
+    // Create vanilla network with 4 input, 4 hidden and 3 output nodes.
+    FFN<NegativeLogLikelihood<> > model;
+    model.Add<Linear<> >(trainData.n_rows, 4);
+    model.Add<SigmoidLayer<> >();
+    model.Add<Linear<> >(4, 3);
+    model.Add<LogSoftMax<> >();
 
-  size_t error = 0;
-  for (size_t i = 0; i < testData.n_cols; i++)
-  {
-    if (int(arma::as_scalar(prediction.col(i))) ==
-        int(arma::as_scalar(testLabels.col(i))))
+    // Creating CNE object.
+    // The tolerance and objectiveChange are not taken into consideration.
+    CNE opt(30, 200, 0.2, 0.2, 0.3, -1, -1);
+
+    model.Train(trainData, trainLabels, opt);
+
+    arma::mat predictionTemp;
+    model.Predict(testData, predictionTemp);
+    arma::mat prediction = arma::zeros<arma::mat>(1, predictionTemp.n_cols);
+
+    for (size_t i = 0; i < predictionTemp.n_cols; ++i)
     {
-      error++;
+      prediction(i) = arma::as_scalar(arma::find(
+          arma::max(predictionTemp.col(i)) == predictionTemp.col(i), 1)) + 1;
+    }
+
+    size_t error = 0;
+    for (size_t i = 0; i < testData.n_cols; i++)
+    {
+      if (int(arma::as_scalar(prediction.col(i))) ==
+          int(arma::as_scalar(testLabels.col(i))))
+      {
+        error++;
+      }
+    }
+
+    double classificationError = 1 - double(error) / testData.n_cols;
+    if (classificationError <= 0.1)
+    {
+      ++successes;
+      break;
     }
   }
 
-  double classificationError = 1 - double(error) / testData.n_cols;
-  BOOST_REQUIRE_LE(classificationError, 0.1);
+  BOOST_REQUIRE_GT(successes, 0);
 }
 
 BOOST_AUTO_TEST_SUITE_END();

From bd831e05fa80547b8cd3111d232047567e55b45c Mon Sep 17 00:00:00 2001
From: Ryan Curtin <ryan@ratml.org>
Date: Fri, 22 Dec 2017 17:42:34 -0500
Subject: [PATCH 62/67] Relax tolerance slightly to prevent random failures.

---
 src/mlpack/tests/recurrent_network_test.cpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/mlpack/tests/recurrent_network_test.cpp b/src/mlpack/tests/recurrent_network_test.cpp
index 38857ef9ef..beb30a5ff6 100644
--- a/src/mlpack/tests/recurrent_network_test.cpp
+++ b/src/mlpack/tests/recurrent_network_test.cpp
@@ -67,13 +67,13 @@ void GenerateNoisySines(arma::mat& data,
 BOOST_AUTO_TEST_CASE(SequenceClassificationTest)
 {
   // It isn't guaranteed that the recurrent network will converge in the
-  // specified number of iterations using random weights. If this works 1 of 5
+  // specified number of iterations using random weights. If this works 1 of 6
   // times, I'm fine with that. All I want to know is that the network is able
   // to escape from local minima and to solve the task.
   size_t successes = 0;
   const size_t rho = 10;
 
-  for (size_t trial = 0; trial < 5; ++trial)
+  for (size_t trial = 0; trial < 6; ++trial)
   {
     // Generate 12 (2 * 6) noisy sines. A single sine contains rho
     // points/features.

From 443c24d93d5d528da623253280349830a2089362 Mon Sep 17 00:00:00 2001
From: Ryan Curtin <ryan@ratml.org>
Date: Sat, 23 Dec 2017 16:28:52 -0500
Subject: [PATCH 63/67] Avoid line wrap which made invalid .pc files.

---
 CMakeLists.txt | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 5d18bc4733..651e03fd67 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -531,8 +531,8 @@ if (PKG_CONFIG_FOUND)
   foreach (incldir ${MLPACK_INCLUDE_DIRS})
     # Filter out some obviously unnecessary directories.
     if (NOT "${incldir}" STREQUAL "/usr/include")
-      set(MLPACK_INCLUDE_DIRS_STRING "${MLPACK_INCLUDE_DIRS_STRING}
-          -I${incldir}")
+      set(MLPACK_INCLUDE_DIRS_STRING
+          "${MLPACK_INCLUDE_DIRS_STRING} -I${incldir}")
     endif ()
   endforeach ()
   # Add the install directory too.

From 1174d5c2979e15f33b0ff2c6b18aa5d4668c127b Mon Sep 17 00:00:00 2001
From: Ryan Curtin <ryan@ratml.org>
Date: Sat, 23 Dec 2017 16:49:01 -0500
Subject: [PATCH 64/67] Remove any duplicate include directories.

---
 CMakeLists.txt | 1 +
 1 file changed, 1 insertion(+)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 651e03fd67..f2d960c61c 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -528,6 +528,7 @@ if (PKG_CONFIG_FOUND)
   # So, we have to parse our list of library directories, libraries, and include
   # directories in order to get the correct line to give to pkg-config.
   # Next, adapt the list of include directories.
+  list(REMOVE_DUPLICATES MLPACK_INCLUDE_DIRS)
   foreach (incldir ${MLPACK_INCLUDE_DIRS})
     # Filter out some obviously unnecessary directories.
     if (NOT "${incldir}" STREQUAL "/usr/include")

From 9b508e91bdb22c832ef90919ef70274b961d8cb2 Mon Sep 17 00:00:00 2001
From: Ryan Curtin <ryan@ratml.org>
Date: Tue, 26 Dec 2017 09:57:31 -0500
Subject: [PATCH 65/67] Fix line length.

---
 src/mlpack/methods/pca/pca_main.cpp | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/mlpack/methods/pca/pca_main.cpp b/src/mlpack/methods/pca/pca_main.cpp
index ff7f9307ac..8715c11eed 100644
--- a/src/mlpack/methods/pca/pca_main.cpp
+++ b/src/mlpack/methods/pca/pca_main.cpp
@@ -119,7 +119,8 @@ void mlpackMain()
   error << "cannot be greater than existing dimensionality (" << dataset.n_rows
       << ")";
   RequireParamValue<int>("new_dimensionality",
-      [dataset](int x) { return x <= (int) dataset.n_rows; }, true, error.str());
+      [dataset](int x) { return x <= (int) dataset.n_rows; }, true,
+      error.str());
 
   RequireParamValue<double>("var_to_retain",
       [](double x) { return x >= 0.0 && x <= 1.0; }, true,

From 64360a13ccb585c67ef4e273b34b17cc170fb3da Mon Sep 17 00:00:00 2001
From: Ryan Curtin <ryan@ratml.org>
Date: Tue, 26 Dec 2017 10:00:24 -0500
Subject: [PATCH 66/67] Try to fix static warnings by changing loop counter
 variable name.

---
 src/mlpack/tests/recurrent_network_test.cpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/mlpack/tests/recurrent_network_test.cpp b/src/mlpack/tests/recurrent_network_test.cpp
index beb30a5ff6..e37db108e9 100644
--- a/src/mlpack/tests/recurrent_network_test.cpp
+++ b/src/mlpack/tests/recurrent_network_test.cpp
@@ -457,7 +457,7 @@ void ReberGrammarTestNetwork(const size_t hiddenSize = 4,
     MomentumSGD opt(0.06, 50, 2, -50000);
 
     arma::mat inputTemp, labelsTemp;
-    for (size_t i = 0; i < (iterations + offset); i++)
+    for (size_t iteration = 0; i < (iterations + offset); iteration++)
     {
       for (size_t j = 0; j < trainReberGrammarCount; j++)
       {
@@ -672,7 +672,7 @@ void DistractedSequenceRecallTestNetwork(
     // We increase the number of iterations (training) if the first run didn't
     // pass.
     arma::mat inputTemp, labelsTemp;
-    for (size_t i = 0; i < (9 + offset); i++)
+    for (size_t iteration = 0; iteration < (9 + offset); iteration++)
     {
       for (size_t j = 0; j < trainDistractedSequenceCount; j++)
       {

From e2af169e730742a67fd2cf7c67db2468382b4c30 Mon Sep 17 00:00:00 2001
From: Ryan Curtin <ryan@ratml.org>
Date: Tue, 26 Dec 2017 12:20:26 -0500
Subject: [PATCH 67/67] Fix stupid oversight in changing code.

---
 src/mlpack/tests/recurrent_network_test.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/mlpack/tests/recurrent_network_test.cpp b/src/mlpack/tests/recurrent_network_test.cpp
index e37db108e9..9e71431302 100644
--- a/src/mlpack/tests/recurrent_network_test.cpp
+++ b/src/mlpack/tests/recurrent_network_test.cpp
@@ -457,7 +457,7 @@ void ReberGrammarTestNetwork(const size_t hiddenSize = 4,
     MomentumSGD opt(0.06, 50, 2, -50000);
 
     arma::mat inputTemp, labelsTemp;
-    for (size_t iteration = 0; i < (iterations + offset); iteration++)
+    for (size_t iteration = 0; iteration < (iterations + offset); iteration++)
     {
       for (size_t j = 0; j < trainReberGrammarCount; j++)
       {