Merge branch 'master' into master2

.travis.yml

@@ -9,10 +9,10 @@ env:
 before_install:
   - sudo systemctl stop apt-daily.service apt-daily.timer
   - sudo systemctl disable apt-daily.service apt-daily.timer
-  - sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test
-  - sudo apt-get update -qq
+  - sudo rm -f /var/lib/dpkg/lock && sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test
+  - sudo rm -f /var/lib/dpkg/lock && sudo apt-get update -qq
   - printenv
-  - sudo apt-get install -qq libopenblas-dev liblapack-dev g++ libboost-all-dev
+  - sudo rm -f /var/lib/dpkg/lock && sudo apt-get install -qq libopenblas-dev liblapack-dev g++ libboost-all-dev
   - sudo wget https://launchpadlibrarian.net/260609315/fix-std-vector-load.diff -O /usr/include/fix-std-vector-load.diff
   - cd /usr/include && sudo patch -p1 < fix-std-vector-load.diff && sudo rm fix-std-vector-load.diff && cd -
   - sudo pip install cython setuptools numpy pandas

CMakeLists.txt

@@ -126,7 +126,7 @@ if(BUILD_WITH_COVERAGE)
 endif()
 
 # For clock_gettime().
-if (UNIX AND NOT APPLE)
+if (UNIX AND NOT APPLE AND NOT HAIKU)
   set(MLPACK_LIBRARIES ${MLPACK_LIBRARIES} "rt")
 endif ()
 
@@ -528,11 +528,12 @@ if (PKG_CONFIG_FOUND)
   # So, we have to parse our list of library directories, libraries, and include
   # directories in order to get the correct line to give to pkg-config.
   # Next, adapt the list of include directories.
+  list(REMOVE_DUPLICATES MLPACK_INCLUDE_DIRS)
   foreach (incldir ${MLPACK_INCLUDE_DIRS})
     # Filter out some obviously unnecessary directories.
     if (NOT "${incldir}" STREQUAL "/usr/include")
-      set(MLPACK_INCLUDE_DIRS_STRING "${MLPACK_INCLUDE_DIRS_STRING}
-          -I${incldir}")
+      set(MLPACK_INCLUDE_DIRS_STRING
+          "${MLPACK_INCLUDE_DIRS_STRING} -I${incldir}")
     endif ()
   endforeach ()
   # Add the install directory too.

src/mlpack/bindings/cli/cli_option.hpp

@@ -60,6 +60,7 @@ class CLIOption
    * @param input Whether or not the option is an input option.
    * @param noTranspose If the parameter is a matrix and this is true, then the
    *      matrix will not be transposed on loading.
+   * @param testName Is not used and added for compatibility reasons.
    */
   CLIOption(const N defaultValue,
             const std::string& identifier,
@@ -68,7 +69,8 @@ class CLIOption
             const std::string& cppName,
             const bool required = false,
             const bool input = true,
-            const bool noTranspose = false)
+            const bool noTranspose = false,
+            const std::string& /*testName*/ = "")
   {
     // Create the ParamData object to give to CLI.
     util::ParamData data;

src/mlpack/bindings/python/py_option.hpp

@@ -33,7 +33,8 @@ class PyOption
  public:
   /**
    * Construct a PyOption object.  When constructed, it will register itself
-   * with CLI.
+   * with CLI. The testName parameter is not used and added for compatibility 
+   * reasons.
    */
   PyOption(const T defaultValue,
            const std::string& identifier,
@@ -42,7 +43,8 @@ class PyOption
            const std::string& cppName,
            const bool required = false,
            const bool input = true,
-           const bool noTranspose = false)
+           const bool noTranspose = false,
+           const std::string& /*testName*/ = "")
   {
     // Create the ParamData object to give to CLI.
     util::ParamData data;

src/mlpack/bindings/python/tests/test_python_binding_main.cpp

@@ -52,7 +52,7 @@ PARAM_MODEL_OUT(GaussianKernel, "model_out", "Output model, with twice the "
     "bandwidth.", "");
 PARAM_DOUBLE_OUT("model_bw_out", "The bandwidth of the model.");
 
-void mlpackMain()
+static void mlpackMain()
 {
   const string s = CLI::GetParam<string>("string_in");
   const int i = CLI::GetParam<int>("int_in");

src/mlpack/bindings/tests/test_option.hpp

@@ -53,6 +53,8 @@ class TestOption
    * @param input Whether or not the option is an input option.
    * @param noTranspose If the parameter is a matrix and this is true, then the
    *      matrix will not be transposed on loading.
+   * @param testName Name of the test (used for identifiying which binding test 
+   *      this option belongs to)
    */
   TestOption(const N defaultValue,
              const std::string& identifier,
@@ -61,7 +63,8 @@ class TestOption
              const std::string& cppName,
              const bool required = false,
              const bool input = true,
-             const bool noTranspose = false)
+             const bool noTranspose = false,
+             const std::string& testName = "")
   {
     // Create the ParamData object to give to CLI.
     util::ParamData data;
@@ -81,7 +84,7 @@ class TestOption
 
     const std::string tname = data.tname;
 
-    CLI::RestoreSettings(programName, false);
+    CLI::RestoreSettings(testName, false);
 
     // Set some function pointers that we need.
     CLI::GetSingleton().functionMap[tname]["GetPrintableParam"] =
@@ -94,7 +97,7 @@ class TestOption
     if (!input)
       CLI::SetPassed(identifier);
 
-    CLI::StoreSettings(programName);
+    CLI::StoreSettings(testName);
     CLI::ClearSettings();
   }
 };

src/mlpack/core/optimizers/adam/CMakeLists.txt

@@ -3,6 +3,8 @@ set(SOURCES
   adam_impl.hpp
   adam_update.hpp
   adamax_update.hpp
+  amsgrad_update.hpp
+  nadam_update.hpp
 )
 
 set(DIR_SRCS)

src/mlpack/core/optimizers/adam/adam.hpp

@@ -4,11 +4,13 @@
  * @author Vasanth Kalingeri
  * @author Marcus Edel
  * @author Vivek Pal
+ * @author Sourabh Varshney
  *
- * Adam and AdaMax optimizer. Adam is an an algorithm for first-order gradient-
- * -based optimization of stochastic objective functions, based on adaptive
- * estimates of lower-order moments. AdaMax is simply a variant of Adam based
- * on the infinity norm.
+ * Adam, AdaMax, AMSGrad and Nadam optimizers. Adam is an an algorithm for
+ * first-order gradient-based optimization of stochastic objective functions,
+ * based on adaptive estimates of lower-order moments. AdaMax is simply a
+ * variant of Adam based on the infinity norm. AMSGrad is another variant of
+ * Adam with guaranteed convergence.
  *
  * mlpack is free software; you may redistribute it and/or modify it under the
  * terms of the 3-clause BSD license.  You should have received a copy of the
@@ -23,6 +25,8 @@
 #include <mlpack/core/optimizers/sgd/sgd.hpp>
 #include "adam_update.hpp"
 #include "adamax_update.hpp"
+#include "amsgrad_update.hpp"
+#include "nadam_update.hpp"
 
 namespace mlpack {
 namespace optimization {
@@ -31,7 +35,8 @@ namespace optimization {
  * Adam is an optimizer that computes individual adaptive learning rates for
  * different parameters from estimates of first and second moments of the
  * gradients. AdaMax is a variant of Adam based on the infinity norm as given
- * in the section 7 of the following paper.
+ * in the section 7 of the following paper. Nadam is an optimizer that
+ * combines the Adam and NAG.
  *
  * For more information, see the following.
  *
@@ -43,11 +48,16 @@ namespace optimization {
  *   year    = {2014},
  *   url     = {http://arxiv.org/abs/1412.6980}
  * }
+ * @article{
+ *   title   = {On the convergence of Adam and beyond},
+ *   url     = {https://openreview.net/pdf?id=ryQu7f-RZ}
+ *   year    = {2018}
+ * }
  * @endcode
  *
- *
- * For Adam and AdaMax to work, a DecomposableFunctionType template parameter
- * is required. This class must implement the following function:
+ * For Adam, AdaMax, AMSGrad and Nadam to work, a DecomposableFunctionType
+ * template parameter is required. This class must implement the following
+ * function:
  *
  *   size_t NumFunctions();
  *   double Evaluate(const arma::mat& coordinates,
@@ -166,6 +176,10 @@ using Adam = AdamType<AdamUpdate>;
 
 using AdaMax = AdamType<AdaMaxUpdate>;
 
+using AMSGrad = AdamType<AMSGradUpdate>;
+
+using Nadam = AdamType<NadamUpdate>;
+
 } // namespace optimization
 } // namespace mlpack
 

src/mlpack/core/optimizers/adam/adam_impl.hpp

@@ -5,7 +5,7 @@
  * @author Marcus Edel
  * @author Vivek Pal
  *
- * Implementation of the Adam and AdaMax optimizer.
+ * Implementation of the Adam, AdaMax, AMSGrad and Nadam optimizer.
  *
  * mlpack is free software; you may redistribute it and/or modify it under the
  * terms of the 3-clause BSD license.  You should have received a copy of the

src/mlpack/core/optimizers/adam/amsgrad_update.hpp

@@ -0,0 +1,145 @@
+/**
+ * @file amsgrad_update.hpp
+ * @author Haritha Nair
+ *
+ * Implementation of AMSGrad optimizer. AMSGrad is an exponential moving average 
+ * optimizer that dynamically adapts over time with guaranteed convergence.
+ *
+ * mlpack is free software; you may redistribute it and/or modify it under the
+ * terms of the 3-clause BSD license.  You should have received a copy of the
+ * 3-clause BSD license along with mlpack.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef MLPACK_CORE_OPTIMIZERS_AMS_GRAD_AMS_GRAD_UPDATE_HPP
+#define MLPACK_CORE_OPTIMIZERS_AMS_GRAD_AMS_GRAD_UPDATE_HPP
+
+#include <mlpack/prereqs.hpp>
+
+namespace mlpack {
+namespace optimization {
+
+/**
+ * AMSGrad is an exponential moving average variant which along with having
+ * benefits of optimizers like Adam and RMSProp, also guarantees convergence.
+ * Unlike Adam, it uses maximum of past squared gradients rather than their
+ * exponential average for updation.
+ *
+ * For more information, see the following.
+ *
+ * @code
+ * @article{
+ *   title   = {On the convergence of Adam and beyond},
+ *   url     = {https://openreview.net/pdf?id=ryQu7f-RZ}
+ *   year    = {2018}
+ * }
+ * @endcode
+ */
+class AMSGradUpdate
+{
+ public:
+  /**
+   * Construct the AMSGrad update policy with the given parameters.
+   *
+   * @param epsilon The epsilon value used to initialise the squared gradient
+   *        parameter.
+   * @param beta1 The smoothing parameter.
+   * @param beta2 The second moment coefficient.
+   */
+  AMSGradUpdate(const double epsilon = 1e-8,
+                const double beta1 = 0.9,
+                const double beta2 = 0.999) :
+    epsilon(epsilon),
+    beta1(beta1),
+    beta2(beta2),
+    iteration(0)
+  {
+    // Nothing to do.
+  }
+
+  /**
+   * The Initialize method is called by SGD Optimizer method before the start of
+   * the iteration update process.
+   *
+   * @param rows Number of rows in the gradient matrix.
+   * @param cols Number of columns in the gradient matrix.
+   */
+  void Initialize(const size_t rows, const size_t cols)
+  {
+    m = arma::zeros<arma::mat>(rows, cols);
+    v = arma::zeros<arma::mat>(rows, cols);
+    vImproved = arma::zeros<arma::mat>(rows, cols);
+  }
+
+  /**
+   * Update step for AMSGrad.
+   *
+   * @param iterate Parameters that minimize the function.
+   * @param stepSize Step size to be used for the given iteration.
+   * @param gradient The gradient matrix.
+   */
+  void Update(arma::mat& iterate,
+              const double stepSize,
+              const arma::mat& gradient)
+  {
+    // Increment the iteration counter variable.
+    ++iteration;
+
+    // And update the iterate.
+    m *= beta1;
+    m += (1 - beta1) * gradient;
+
+    v *= beta2;
+    v += (1 - beta2) * (gradient % gradient);
+
+    const double biasCorrection1 = 1.0 - std::pow(beta1, iteration);
+    const double biasCorrection2 = 1.0 - std::pow(beta2, iteration);
+
+    // Element wise maximum of past and present squared gradients.
+    vImproved = arma::max(vImproved, v);
+
+    iterate -= (stepSize * std::sqrt(biasCorrection2) / biasCorrection1) *
+                m / (arma::sqrt(vImproved) + epsilon);
+  }
+
+  //! Get the value used to initialise the squared gradient parameter.
+  double Epsilon() const { return epsilon; }
+  //! Modify the value used to initialise the squared gradient parameter.
+  double& Epsilon() { return epsilon; }
+
+  //! Get the smoothing parameter.
+  double Beta1() const { return beta1; }
+  //! Modify the smoothing parameter.
+  double& Beta1() { return beta1; }
+
+  //! Get the second moment coefficient.
+  double Beta2() const { return beta2; }
+  //! Modify the second moment coefficient.
+  double& Beta2() { return beta2; }
+
+ private:
+  // The epsilon value used to initialise the squared gradient parameter.
+  double epsilon;
+
+  // The smoothing parameter.
+  double beta1;
+
+  // The second moment coefficient.
+  double beta2;
+
+  // The exponential moving average of gradient values.
+  arma::mat m;
+
+  // The exponential moving average of squared gradient values.
+  arma::mat v;
+
+  // The optimal sqaured gradient value.
+  arma::mat vImproved;
+
+  // The number of iterations.
+  double iteration;
+};
+
+} // namespace optimization
+} // namespace mlpack
+
+#endif

src/mlpack/core/optimizers/adam/nadam_update.hpp

@@ -0,0 +1,173 @@
+/**
+ * @file nadam_update.hpp
+ * @author Sourabh Varshney
+ *
+ * Nadam update rule. Nadam is an optimizer that combines the effect of Adam
+ * and NAG to the gradient descent to improve its Performance.
+ *
+ * mlpack is free software; you may redistribute it and/or modify it under the
+ * terms of the 3-clause BSD license.  You should have received a copy of the
+ * 3-clause BSD license along with mlpack.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef MLPACK_CORE_OPTIMIZERS_ADAM_NADAM_UPDATE_HPP
+#define MLPACK_CORE_OPTIMIZERS_ADAM_NADAM_UPDATE_HPP
+
+#include <mlpack/prereqs.hpp>
+
+namespace mlpack {
+namespace optimization {
+
+/**
+ * Nadam is an optimizer that combines the Adam and NAG.
+ *
+ * For more information, see the following.
+ *
+ * @code
+ * @techreport{Dozat2015,
+ *   title       = {Incorporating Nesterov momentum into Adam},
+ *   author      = {Timothy Dozat},
+ *   institution = {Stanford University},
+ *   address     = {Stanford},
+ *   year        = {2015},
+ *   url         = {https://openreview.net/pdf?id=OM0jvwB8jIp57ZJjtNEZ}
+ * }
+ * @endcode
+ */
+class NadamUpdate
+{
+ public:
+  /**
+   * Construct the Nadam update policy with the given parameters.
+   *
+   * @param epsilon The epsilon value used to initialise the squared gradient
+   *        parameter.
+   * @param beta1 The smoothing parameter.
+   * @param beta2 The second moment coefficient
+   * @param scheduleDecay The decay parameter for decay coefficients
+   */
+  NadamUpdate(const double epsilon = 1e-8,
+              const double beta1 = 0.9,
+              const double beta2 = 0.99,
+              const double scheduleDecay = 4e-3)
+      :epsilon(epsilon),
+      beta1(beta1),
+      beta2(beta2),
+      scheduleDecay(scheduleDecay),
+      iteration(0),
+      cumBeta1(1)
+  {
+    // Nothing to do.
+  }
+
+  /**
+   * The Initialize method is called by SGD Optimizer method before the start 
+   * of the iteration update process.
+   *
+   * @param rows Number of rows in the gradient matrix.
+   * @param cols Number of columns in the gradient matrix.
+   */
+  void Initialize(const size_t rows, const size_t cols)
+  {
+    m = arma::zeros<arma::mat>(rows, cols);
+    v = arma::zeros<arma::mat>(rows, cols);
+  }
+
+  /**
+   * Update step for Nadam.
+   *
+   * @param iterate Parameters that minimize the function.
+   * @param stepSize Step size to be used for the given iteration.
+   * @param gradient The gradient matrix.
+   */
+  void Update(arma::mat& iterate,
+              const double stepSize,
+              const arma::mat& gradient)
+  {
+    // Increment the iteration counter variable.
+    ++iteration;
+
+    // And update the iterate.
+    m *= beta1;
+    m += (1 - beta1) * gradient;
+
+    v *= beta2;
+    v += (1 - beta2) * gradient % gradient;
+
+    double beta1T = beta1 * (1 - (0.5 *
+        std::pow(0.96, iteration * scheduleDecay)));
+
+    double beta1T1 = beta1 * (1 - (0.5 *
+        std::pow(0.96, (iteration + 1) * scheduleDecay)));
+
+    cumBeta1 *= beta1T;
+
+    const double biasCorrection1 = 1.0 - cumBeta1;
+
+    const double biasCorrection2 = 1.0 - std::pow(beta2, iteration);
+
+    const double biasCorrection3 = 1.0 - (cumBeta1 * beta1T1);
+
+    /* Note :- arma::sqrt(v) + epsilon * sqrt(biasCorrection2) is approximated
+     * as arma::sqrt(v) + epsilon
+     */
+    iterate -= (stepSize * (((1 - beta1T) / biasCorrection1) * gradient
+        + (beta1T1 / biasCorrection3) * m) * sqrt(biasCorrection2))
+        / (arma::sqrt(v) + epsilon);
+  }
+
+  //! Get the value used to initialise the squared gradient parameter.
+  double Epsilon() const { return epsilon; }
+  //! Modify the value used to initialise the squared gradient parameter.
+  double& Epsilon() { return epsilon; }
+
+  //! Get the value of the cumulative product of decay coefficients
+  double CumBeta1() const { return cumBeta1; }
+  //! Modify the value of the cumulative product of decay coefficients
+  double& CumBeta1() { return cumBeta1; }
+
+  //! Get the smoothing parameter.
+  double Beta1() const { return beta1; }
+  //! Modify the smoothing parameter.
+  double& Beta1() { return beta1; }
+
+  //! Get the second moment coefficient.
+  double Beta2() const { return beta2; }
+  //! Modify the second moment coefficient.
+  double& Beta2() { return beta2; }
+
+  //! Get the decay parameter for decay coefficients
+  double ScheduleDecay() const { return scheduleDecay; }
+  //! Modify the decay parameter for decay coefficients
+  double& ScheduleDecay() { return scheduleDecay; }
+
+ private:
+  // The epsilon value used to initialise the squared gradient parameter.
+  double epsilon;
+
+  // The smoothing parameter.
+  double beta1;
+
+  // The second moment coefficient.
+  double beta2;
+
+  // The exponential moving average of gradient values.
+  arma::mat m;
+
+  // The exponential moving average of squared gradient values.
+  arma::mat v;
+
+  // The cumulative product of decay coefficients
+  double cumBeta1;
+
+  // The decay parameter for decay coefficients
+  double scheduleDecay;
+
+  // The number of iterations.
+  double iteration;
+};
+
+} // namespace optimization
+} // namespace mlpack
+
+#endif

src/mlpack/core/optimizers/cmaes/full_selection.hpp

@@ -32,7 +32,7 @@ class FullSelection
    * @param iterate starting point.
    */
   template<typename DecomposableFunctionType>
-  const double Select(DecomposableFunctionType& function,
+  double Select(DecomposableFunctionType& function,
                       const size_t batchSize,
                       const arma::mat& iterate)
   {

src/mlpack/core/optimizers/cmaes/random_selection.hpp

@@ -47,7 +47,7 @@ class RandomSelection
    * @param iterate starting point.
    */
   template<typename DecomposableFunctionType>
-  const double Select(DecomposableFunctionType& function,
+  double Select(DecomposableFunctionType& function,
                       const size_t batchSize,
                       const arma::mat& iterate)
   {

src/mlpack/core/optimizers/sgd/sgd_impl.hpp

@@ -102,8 +102,14 @@ double SGD<UpdatePolicyType, DecayPolicyType>::Optimize(
         function.Shuffle();
     }
 
-    // Find the effective batch size (the last batch may be smaller).
-    const size_t effectiveBatchSize = std::min(batchSize,
+    // Find the effective batch size; we have to take the minimum of three
+    // things:
+    // - the batch size can't be larger than the user-specified batch size;
+    // - the batch size can't be larger than the number of iterations left
+    //       before actualMaxIterations is hit;
+    // - the batch size can't be larger than the number of functions left.
+    const size_t effectiveBatchSize = std::min(
+        std::min(batchSize, actualMaxIterations - i),
         numFunctions - currentFunction);
 
     function.Gradient(iterate, currentFunction, gradient, effectiveBatchSize);

src/mlpack/core/optimizers/spalera_sgd/spalera_sgd_impl.hpp

@@ -108,8 +108,14 @@ double SPALeRASGD<DecayPolicyType>::Optimize(DecomposableFunctionType& function,
         function.Shuffle();
     }
 
-    // Find the effective batch size (the last batch may be smaller).
-    const size_t effectiveBatchSize = std::min(batchSize,
+    // Find the effective batch size; we have to take the minimum of three
+    // things:
+    // - the batch size can't be larger than the user-specified batch size;
+    // - the batch size can't be larger than the number of iterations left
+    //       before actualMaxIterations is hit;
+    // - the batch size can't be larger than the number of functions left.
+    const size_t effectiveBatchSize = std::min(
+        std::min(batchSize, actualMaxIterations - i),
         numFunctions - currentFunction);
 
     function.Gradient(iterate, currentFunction, gradient, effectiveBatchSize);

src/mlpack/core/tree/cosine_tree/cosine_tree.cpp

@@ -94,7 +94,7 @@ CosineTree::CosineTree(const arma::mat& dataset,
   // Initialize Monte Carlo error estimate for comparison.
   double monteCarloError = root.FrobNormSquared();
 
-  while (treeQueue.top() &&
+  while (treeQueue.size() > 0 &&
          (monteCarloError > epsilon * root.FrobNormSquared()))
   {
     // Pop node from queue with highest projection error.

src/mlpack/core/util/mlpack_main.hpp

@@ -46,11 +46,12 @@ using Option = mlpack::bindings::cli::CLIOption<T>;
 }
 }
 
+static const std::string testName = "";
 #include <mlpack/core/util/param.hpp>
 #include <mlpack/bindings/cli/parse_command_line.hpp>
 #include <mlpack/bindings/cli/end_program.hpp>
 
-void mlpackMain(); // This is typically defined after this include.
+static void mlpackMain(); // This is typically defined after this include.
 
 int main(int argc, char** argv)
 {
@@ -91,19 +92,13 @@ using Option = mlpack::bindings::tests::TestOption<T>;
 }
 }
 
+// testName symbol should be defined in each binding test file
 #include <mlpack/core/util/param.hpp>
 
 #undef PROGRAM_INFO
 #define PROGRAM_INFO(NAME, DESC) static mlpack::util::ProgramDoc \
     cli_programdoc_dummy_object = mlpack::util::ProgramDoc(NAME, \
-        []() { return DESC; }); \
-    namespace mlpack { \
-    namespace bindings { \
-    namespace tests { \
-    std::string programName = NAME; \
-    } \
-    } \
-    }
+        []() { return DESC; });
 
 #elif(BINDING_TYPE == BINDING_TYPE_PYX) // This is a Python binding.
 
@@ -126,6 +121,7 @@ using Option = mlpack::bindings::python::PyOption<T>;
 }
 }
 
+static const std::string testName = "";
 #include <mlpack/core/util/param.hpp>
 
 #undef PROGRAM_INFO

src/mlpack/core/util/param.hpp

@@ -1012,53 +1012,55 @@ using DatasetInfo = DatasetMapper<IncrementPolicy, std::string>;
   #define PARAM_IN(T, ID, DESC, ALIAS, DEF, REQ) \
       static mlpack::util::Option<T> \
       JOIN(cli_option_dummy_object_in_, __COUNTER__) \
-      (DEF, ID, DESC, ALIAS, #T, REQ, true, false);
+      (DEF, ID, DESC, ALIAS, #T, REQ, true, false, testName);
 
   #define PARAM_OUT(T, ID, DESC, ALIAS, DEF, REQ) \
       static mlpack::util::Option<T> \
       JOIN(cli_option_dummy_object_out_, __COUNTER__) \
-      (DEF, ID, DESC, ALIAS, #T, REQ, false, false);
+      (DEF, ID, DESC, ALIAS, #T, REQ, false, false, testName);
 
   #define PARAM_MATRIX(ID, DESC, ALIAS, REQ, TRANS, IN) \
       static mlpack::util::Option<arma::mat> \
       JOIN(cli_option_dummy_matrix_, __COUNTER__) \
-      (arma::mat(), ID, DESC, ALIAS, "arma::mat", REQ, IN, !TRANS);
+      (arma::mat(), ID, DESC, ALIAS, "arma::mat", \
+      REQ, IN, !TRANS, testName);
 
   #define PARAM_UMATRIX(ID, DESC, ALIAS, REQ, TRANS, IN) \
       static mlpack::util::Option<arma::Mat<size_t>> \
       JOIN(cli_option_dummy_umatrix_, __COUNTER__) \
-      (arma::Mat<size_t>(), ID, DESC, ALIAS, "arma::Mat<size_t>", REQ, IN, \
-      !TRANS);
+      (arma::Mat<size_t>(), ID, DESC, ALIAS, "arma::Mat<size_t>", \
+      REQ, IN, !TRANS, testName);
 
   #define PARAM_COL(ID, DESC, ALIAS, REQ, TRANS, IN) \
       static mlpack::util::Option<arma::vec> \
       JOIN(cli_option_dummy_col_, __COUNTER__) \
-      (arma::vec(), ID, DESC, ALIAS, "arma::vec", REQ, IN, !TRANS);
+      (arma::vec(), ID, DESC, ALIAS, "arma::vec", \
+      REQ, IN, !TRANS, testName);
 
   #define PARAM_UCOL(ID, DESC, ALIAS, REQ, TRANS, IN) \
       static mlpack::util::Option<arma::Col<size_t>> \
       JOIN(cli_option_dummy_ucol_, __COUNTER__) \
-      (arma::Col<size_t>(), ID, DESC, ALIAS, "arma::Col<size_t>", REQ, IN, \
-      !TRANS);
+      (arma::Col<size_t>(), ID, DESC, ALIAS, "arma::Col<size_t>", \
+      REQ, IN, !TRANS, testName);
 
   #define PARAM_ROW(ID, DESC, ALIAS, REQ, TRANS, IN) \
       static mlpack::util::Option<arma::rowvec> \
       JOIN(cli_option_dummy_row_, __COUNTER__) \
-      (arma::rowvec(), ID, DESC, ALIAS, "arma::rowvec", REQ, IN, !TRANS);
+      (arma::rowvec(), ID, DESC, ALIAS, "arma::rowvec", \
+      REQ, IN, !TRANS, testName);
 
   #define PARAM_UROW(ID, DESC, ALIAS, REQ, TRANS, IN) \
       static mlpack::util::Option<arma::Row<size_t>> \
       JOIN(cli_option_dummy_urow_, __COUNTER__) \
-      (arma::Row<size_t>(), ID, DESC, ALIAS, "arma::Row<size_t>", REQ, IN, \
-      !TRANS);
-
+      (arma::Row<size_t>(), ID, DESC, ALIAS, "arma::Row<size_t>", \
+      REQ, IN, !TRANS, testName);
 
   // There are no uses of required models, so that is not an option to this
   // macro (it would be easy to add).
   #define PARAM_MODEL(TYPE, ID, DESC, ALIAS, REQ, IN) \
       static mlpack::util::Option<TYPE> \
       JOIN(cli_option_dummy_model_, __COUNTER__) \
-      (TYPE(), ID, DESC, ALIAS, #TYPE, REQ, IN);
+      (TYPE(), ID, DESC, ALIAS, #TYPE, REQ, IN, false, testName);
 #else
   // We have to do some really bizarre stuff since __COUNTER__ isn't defined. I
   // don't think we can absolutely guarantee success, but it should be "good
@@ -1067,50 +1069,54 @@ using DatasetInfo = DatasetMapper<IncrementPolicy, std::string>;
   #define PARAM_IN(T, ID, DESC, ALIAS, DEF, REQ) \
       static mlpack::util::Option<T> \
       JOIN(JOIN(cli_option_dummy_object_in_, __LINE__), opt) \
-      (DEF, ID, DESC, ALIAS, #T, REQ, true, false);
+      (DEF, ID, DESC, ALIAS, #T, REQ, true, false, testName);
 
   #define PARAM_OUT(T, ID, DESC, ALIAS, DEF, REQ) \
       static mlpack::util::Option<T> \
       JOIN(JOIN(cli_option_dummy_object_out_, __LINE__), opt) \
-      (DEF, ID, DESC, ALIAS, #T, REQ, false, false);
+      (DEF, ID, DESC, ALIAS, #T, REQ, false, false, testName);
 
   #define PARAM_MATRIX(ID, DESC, ALIAS, REQ, TRANS, IN) \
       static mlpack::util::Option<arma::mat> \
       JOIN(JOIN(cli_option_dummy_object_matrix_, __LINE__), opt) \
-      (arma::mat(), ID, DESC, ALIAS, "arma::mat", REQ, IN, !TRANS);
+      (arma::mat(), ID, DESC, ALIAS, "arma::mat", REQ, IN, !TRANS, \
+      testName);
 
   #define PARAM_UMATRIX(ID, DESC, ALIAS, REQ, TRANS, IN) \
       static mlpack::util::Option<arma::Mat<size_t>> \
       JOIN(JOIN(cli_option_dummy_object_umatrix_, __LINE__), opt) \
       (arma::Mat<size_t>(), ID, DESC, ALIAS, "arma::Mat<size_t>", REQ, IN, \
-      !TRANS);
+      !TRANS, testName);
 
   #define PARAM_COL(ID, DESC, ALIAS, REQ, TRANS, IN) \
       static mlpack::util::Option<arma::vec> \
       JOIN(cli_option_dummy_object_col_, __LINE__) \
-      (arma::vec(), ID, DESC, ALIAS, "arma::vec", REQ, IN, !TRANS);
+      (arma::vec(), ID, DESC, ALIAS, "arma::vec", REQ, IN, !TRANS, \
+      testName);
 
   #define PARAM_UCOL(ID, DESC, ALIAS, REQ, TRANS, IN) \
       static mlpack::util::Option<arma::Col<size_t>> \
       JOIN(cli_option_dummy_object_ucol_, __LINE__) \
       (arma::Col<size_t>(), ID, DESC, ALIAS, "arma::Col<size_t>", REQ, IN, \
-      !TRANS);
+      !TRANS, testName);
 
   #define PARAM_ROW(ID, DESC, ALIAS, REQ, TRANS, IN) \
       static mlpack::util::Option<arma::rowvec> \
       JOIN(cli_option_dummy_object_row_, __LINE__) \
-      (arma::rowvec(), ID, DESC, ALIAS, "arma::rowvec", REQ, IN, !TRANS);
+      (arma::rowvec(), ID, DESC, ALIAS, "arma::rowvec", REQ, IN, !TRANS, \
+      testName);
 
   #define PARAM_UROW(ID, DESC, ALIAS, REQ, TRANS, IN) \
       static mlpack::util::Option<arma::Row<size_t>> \
       JOIN(cli_option_dummy_object_urow_, __LINE__) \
       (arma::Row<size_t>(), ID, DESC, ALIAS, "arma::Row<size_t>", REQ, IN, \
-      !TRANS);
+      !TRANS, testName);
 
   #define PARAM_MODEL(TYPE, ID, DESC, ALIAS, REQ, IN) \
       static mlpack::util::Option<TYPE> \
       JOIN(JOIN(cli_option_dummy_object_model_, __LINE__), opt) \
-      (TYPE(), ID, DESC, ALIAS, #TYPE, REQ, IN);
+      (TYPE(), ID, DESC, ALIAS, #TYPE, REQ, IN, false, \
+      testName);
 #endif
 
 #endif

src/mlpack/methods/adaboost/adaboost_main.cpp

@@ -111,7 +111,7 @@ PARAM_MODEL_IN(AdaBoostModel, "input_model", "Input AdaBoost model.", "m");
 PARAM_MODEL_OUT(AdaBoostModel, "output_model", "Output trained AdaBoost model.",
     "M");
 
-void mlpackMain()
+static void mlpackMain()
 {
   // Check input parameters and issue warnings/errors as necessary.
 

src/mlpack/methods/ann/layer/fast_lstm_impl.hpp

@@ -80,7 +80,7 @@ void FastLSTM<InputDataType, OutputDataType>::ResetCell(const size_t size)
   gradientStep = batchSize * size - 1;
 
   const size_t rhoBatchSize = size * batchSize;
-  if (gate.is_empty() || gate.n_cols < rhoBatchSize)
+  if (gate.is_empty() || gate.n_cols != rhoBatchSize)
   {
     gate.set_size(4 * outSize, rhoBatchSize);
     gateActivation.set_size(outSize * 3, rhoBatchSize);
@@ -197,7 +197,7 @@ void FastLSTM<InputDataType, OutputDataType>::Backward(
       backwardStep - batchStep, 3 * outSize - 1, backwardStep) %
       cellActivationError;
 
-  if (backwardStep != 0)
+  if (backwardStep > batchStep)
   {
     prevError.submat(2 * outSize, 0, 3 * outSize - 1, batchStep) =
         cell.cols((backwardStep - batchSize) - batchStep,
@@ -258,13 +258,13 @@ void FastLSTM<InputDataType, OutputDataType>::Gradient(
       gradient.n_elem - 1, 0) = arma::vectorise(prevError *
       outParameter.cols(gradientStep - batchStep, gradientStep).t());
 
-  if (gradientStep == 0)
+  if (gradientStep > batchStep)
   {
-    gradientStep = batchSize * bpttSteps - 1;
+    gradientStep -= batchSize;
   }
   else
   {
-    gradientStep -= batchSize;
+    gradientStep = batchSize * bpttSteps - 1;
   }
 }
 

src/mlpack/methods/ann/layer/gru_impl.hpp

@@ -69,7 +69,7 @@ GRU<InputDataType, OutputDataType>::GRU(
   allZeros = arma::zeros<arma::mat>(outSize, batchSize);
 
   outParameter.push_back(std::move(arma::mat(allZeros.memptr(),
-    allZeros.n_rows, allZeros.n_cols, false, true)));
+      allZeros.n_rows, allZeros.n_cols, false, true)));
 
   prevOutput = outParameter.begin();
   backIterator = outParameter.end();
@@ -96,6 +96,21 @@ void GRU<InputDataType, OutputDataType>::Forward(
   {
     batchSize = input.n_cols;
     prevError.resize(3 * outSize, batchSize);
+    allZeros.zeros(outSize, batchSize);
+    // Batch size better not change during an iteration...
+    if (outParameter.size() > 1)
+    {
+      Log::Fatal << "GRU<>::Forward(): batch size cannot change during a "
+          << "forward pass!" << std::endl;
+    }
+
+    outParameter.clear();
+    outParameter.push_back(std::move(arma::mat(allZeros.memptr(),
+        allZeros.n_rows, allZeros.n_cols, false, true)));
+
+    prevOutput = outParameter.begin();
+    backIterator = outParameter.end();
+    gradIterator = outParameter.end();
   }
 
   // Process the input linearly(zt, rt, ot).
@@ -157,13 +172,13 @@ void GRU<InputDataType, OutputDataType>::Forward(
     if (!deterministic)
     {
       outParameter.push_back(std::move(arma::mat(allZeros.memptr(),
-        allZeros.n_rows, allZeros.n_cols, false, true)));
+          allZeros.n_rows, allZeros.n_cols, false, true)));
       prevOutput = --outParameter.end();
     }
     else
     {
       *prevOutput = std::move(arma::mat(allZeros.memptr(),
-        allZeros.n_rows, allZeros.n_cols, false, true));
+          allZeros.n_rows, allZeros.n_cols, false, true));
     }
   }
   else if (!deterministic)
@@ -196,6 +211,21 @@ void GRU<InputDataType, OutputDataType>::Backward(
   {
     batchSize = input.n_cols;
     prevError.resize(3 * outSize, batchSize);
+    allZeros.zeros(outSize, batchSize);
+    // Batch size better not change during an iteration...
+    if (outParameter.size() > 1)
+    {
+      Log::Fatal << "GRU<>::Forward(): batch size cannot change during a "
+          << "forward pass!" << std::endl;
+    }
+
+    outParameter.clear();
+    outParameter.push_back(std::move(arma::mat(allZeros.memptr(),
+        allZeros.n_rows, allZeros.n_cols, false, true)));
+
+    prevOutput = outParameter.begin();
+    backIterator = outParameter.end();
+    gradIterator = outParameter.end();
   }
 
   if ((outParameter.size() - backwardStep  - 1) % rho != 0 && backwardStep != 0)
@@ -214,7 +244,7 @@ void GRU<InputDataType, OutputDataType>::Backward(
       hiddenStateModule));
 
   // Delta ot.
-  arma::mat dOt = gy % (arma::ones<arma::vec>(outSize) -
+  arma::mat dOt = gy % (arma::ones<arma::mat>(outSize, batchSize) -
       boost::apply_visitor(outputParameterVisitor, inputGateModule));
 
   // Delta of input gate.
@@ -297,6 +327,21 @@ void GRU<InputDataType, OutputDataType>::Gradient(
   {
     batchSize = input.n_cols;
     prevError.resize(3 * outSize, batchSize);
+    allZeros.zeros(outSize, batchSize);
+    // Batch size better not change during an iteration...
+    if (outParameter.size() > 1)
+    {
+      Log::Fatal << "GRU<>::Forward(): batch size cannot change during a "
+          << "forward pass!" << std::endl;
+    }
+
+    outParameter.clear();
+    outParameter.push_back(std::move(arma::mat(allZeros.memptr(),
+        allZeros.n_rows, allZeros.n_cols, false, true)));
+
+    prevOutput = outParameter.begin();
+    backIterator = outParameter.end();
+    gradIterator = outParameter.end();
   }
 
   if (gradIterator == outParameter.end())

src/mlpack/methods/ann/layer/lstm_impl.hpp

@@ -188,12 +188,12 @@ void LSTM<InputDataType, OutputDataType>::Forward(
   if (forwardStep > 0)
   {
     inputGate.cols(forwardStep, forwardStep + batchStep) +=
-        cell2GateInputWeight % cell.cols(forwardStep - batchSize,
-        forwardStep - batchSize + batchStep);
+        arma::repmat(cell2GateInputWeight, 1, batchSize) %
+        cell.cols(forwardStep - batchSize, forwardStep - batchSize + batchStep);
 
     forgetGate.cols(forwardStep, forwardStep + batchStep) +=
-        cell2GateForgetWeight % cell.cols(forwardStep - batchSize,
-        forwardStep - batchSize + batchStep);
+        arma::repmat(cell2GateForgetWeight, 1, batchSize) %
+        cell.cols(forwardStep - batchSize, forwardStep - batchSize + batchStep);
   }
 
   inputGateActivation.cols(forwardStep, forwardStep + batchStep) = 1.0 /
@@ -283,7 +283,7 @@ void LSTM<InputDataType, OutputDataType>::Backward(
     cellError += inputCellError;
   }
 
-  if (backwardStep != 0)
+  if (backwardStep > batchStep)
   {
     forgetGateError = cell.cols((backwardStep - batchSize) - batchStep,
       (backwardStep - batchSize)) % cellError % (forgetGateActivation.cols(
@@ -305,7 +305,7 @@ void LSTM<InputDataType, OutputDataType>::Backward(
       backwardStep - batchStep, backwardStep), 2));
 
   inputCellError = forgetGateActivation.cols(backwardStep - batchStep,
-      backwardStep) %cellError + forgetGateError.each_col() %
+      backwardStep) % cellError + forgetGateError.each_col() %
       cell2GateForgetWeight + inputGateError.each_col() % cell2GateInputWeight;
 
   g = input2GateInputWeight.t() * inputGateError +
@@ -395,15 +395,17 @@ void LSTM<InputDataType, OutputDataType>::Gradient(
   offset += cell2GateOutputWeight.n_elem;
 
   // Cell2GateForgetWeight and cell2GateInputWeight gradients.
-  if (gradientStep != 0)
+  if (gradientStep > batchStep)
   {
     gradient.submat(offset, 0, offset + cell2GateForgetWeight.n_elem - 1, 0) =
-        arma::sum(forgetGateError % cell.cols(gradientStep - batchStep -
-        batchSize, gradientStep - batchSize), 1);
+        arma::sum(forgetGateError %
+                  cell.cols((gradientStep - batchSize) - batchStep,
+                            (gradientStep - batchSize)), 1);
     gradient.submat(offset + cell2GateForgetWeight.n_elem, 0, offset +
         cell2GateForgetWeight.n_elem + cell2GateInputWeight.n_elem - 1, 0) =
-        arma::sum(inputGateError % cell.cols(gradientStep - batchStep -
-        batchSize, gradientStep - batchSize), 1);
+        arma::sum(inputGateError %
+                  cell.cols((gradientStep - batchSize) - batchStep,
+                            (gradientStep - batchSize)), 1);
   }
   else
   {

src/mlpack/methods/ann/rnn.hpp

@@ -222,7 +222,14 @@ class RNN
   size_t& Rho() { return rho; }
 
   /**
-   * Reset the module infomration (weights/parameters).
+   * Reset the state of the network.  This ensures that all internally-held
+   * gradients are set to 0, all memory cells are reset, and the parameters
+   * matrix is the right size.
+   */
+  void Reset();
+
+  /**
+   * Reset the module information (weights/parameters).
    */
   void ResetParameters();
 

src/mlpack/methods/ann/rnn_impl.hpp

@@ -102,7 +102,6 @@ void RNN<OutputLayerType, InitializationRuleType>::Train(
   if (!reset)
   {
     ResetParameters();
-    reset = true;
   }
 
   // Train the model.
@@ -139,7 +138,6 @@ void RNN<OutputLayerType, InitializationRuleType>::Train(
   if (!reset)
   {
     ResetParameters();
-    reset = true;
   }
 
   OptimizerType optimizer;
@@ -207,7 +205,6 @@ double RNN<OutputLayerType, InitializationRuleType>::Evaluate(
   if (parameter.is_empty())
   {
     ResetParameters();
-    reset = true;
   }
 
   if (deterministic != this->deterministic)
@@ -272,7 +269,6 @@ void RNN<OutputLayerType, InitializationRuleType>::Gradient(
     if (parameter.is_empty())
     {
       ResetParameters();
-      reset = true;
     }
 
     gradient = arma::zeros<arma::mat>(parameter.n_rows, parameter.n_cols);
@@ -337,6 +333,17 @@ void RNN<OutputLayerType, InitializationRuleType>::ResetParameters()
   // Reset the network parameter with the given initialization rule.
   NetworkInitialization<InitializationRuleType> networkInit(initializeRule);
   networkInit.Initialize(network, parameter);
+
+  reset = true;
+}
+
+template<typename OutputLayerType, typename InitializationRuleType>
+void RNN<OutputLayerType, InitializationRuleType>::Reset()
+{
+  ResetParameters();
+  ResetCells();
+  currentGradient.zeros();
+  ResetGradients(currentGradient);
 }
 
 template<typename OutputLayerType, typename InitializationRuleType>

src/mlpack/methods/approx_kfn/approx_kfn_main.cpp

@@ -141,7 +141,7 @@ PARAM_MODEL_IN(ApproxKFNModel, "input_model", "File containing input model.",
 PARAM_MODEL_OUT(ApproxKFNModel, "output_model", "File to save output model to.",
     "M");
 
-void mlpackMain()
+static void mlpackMain()
 {
   // We have to pass either a reference set or an input model.
   RequireOnlyOnePassed({ "reference", "input_model" });

src/mlpack/methods/cf/cf_main.cpp

@@ -263,7 +263,7 @@ void AssembleFactorizerType(const std::string& algorithm,
   }
 }
 
-void mlpackMain()
+static void mlpackMain()
 {
   if (CLI::GetParam<int>("seed") == 0)
     math::RandomSeed(std::time(NULL));

src/mlpack/methods/dbscan/dbscan_main.cpp

@@ -112,7 +112,7 @@ void RunDBSCAN(RangeSearchType rs = RangeSearchType())
     CLI::GetParam<arma::Row<size_t>>("assignments") = std::move(assignments);
 }
 
-void mlpackMain()
+static void mlpackMain()
 {
   RequireAtLeastOnePassed({ "assignments", "centroids" }, false,
       "no output will be saved");

src/mlpack/methods/decision_stump/decision_stump_main.cpp

@@ -103,7 +103,7 @@ PARAM_MODEL_OUT(DSModel, "output_model", "Output decision stump model to save.",
 PARAM_INT_IN("bucket_size", "The minimum number of training points in each "
     "decision stump bucket.", "b", 6);
 
-void mlpackMain()
+static void mlpackMain()
 {
   // Check that the parameters are reasonable.
   RequireOnlyOnePassed({ "training", "input_model" }, true);

src/mlpack/methods/decision_tree/decision_tree_main.cpp

@@ -115,7 +115,7 @@ PARAM_MODEL_IN(DecisionTreeModel, "input_model", "Pre-trained decision tree, "
 PARAM_MODEL_OUT(DecisionTreeModel, "output_model", "Output for trained decision"
     " tree.", "M");
 
-void mlpackMain()
+static void mlpackMain()
 {
   // Check parameters.
   RequireOnlyOnePassed({ "training", "input_model" }, true);

src/mlpack/methods/det/det_main.cpp

@@ -100,7 +100,7 @@ PARAM_FLAG("volume_regularization", "This flag gives the used the option to use"
 */
 
 
-void mlpackMain()
+static void mlpackMain()
 {
   // Validate input parameters.
   RequireOnlyOnePassed({ "training", "input_model" }, true);

src/mlpack/methods/emst/emst_main.cpp

@@ -72,7 +72,7 @@ using namespace mlpack::metric;
 using namespace mlpack::util;
 using namespace std;
 
-void mlpackMain()
+static void mlpackMain()
 {
   RequireAtLeastOnePassed({ "output" }, false, "no output will be saved");
 

src/mlpack/methods/fastmks/fastmks_main.cpp

@@ -83,7 +83,7 @@ PARAM_FLAG("single", "If true, single-tree search is used (as opposed to "
 PARAM_MATRIX_OUT("kernels", "Output matrix of kernels.", "p");
 PARAM_UMATRIX_OUT("indices", "Output matrix of indices.", "i");
 
-void mlpackMain()
+static void mlpackMain()
 {
   // Validate command-line parameters.
   RequireOnlyOnePassed({ "reference", "input_model" }, true);

src/mlpack/methods/gmm/gmm_generate_main.cpp

@@ -42,7 +42,7 @@ PARAM_MATRIX_OUT("output", "Matrix to save output samples in.", "o");
 
 PARAM_INT_IN("seed", "Random seed.  If 0, 'std::time(NULL)' is used.", "s", 0);
 
-void mlpackMain()
+static void mlpackMain()
 {
   // Parameter sanity checks.
   RequireAtLeastOnePassed({ "output" }, false, "no results will be saved");

src/mlpack/methods/gmm/gmm_probability_main.cpp

@@ -41,7 +41,7 @@ PARAM_MATRIX_IN_REQ("input", "Input matrix to calculate probabilities of.",
 
 PARAM_MATRIX_OUT("output", "Matrix to store calculated probabilities in.", "o");
 
-void mlpackMain()
+static void mlpackMain()
 {
   RequireAtLeastOnePassed({ "output" }, false, "no results will be saved");
 

src/mlpack/methods/gmm/gmm_train_main.cpp

@@ -121,7 +121,7 @@ PARAM_MODEL_IN(GMM, "input_model", "Initial input GMM model to start training "
     "with.", "m");
 PARAM_MODEL_OUT(GMM, "output_model", "Output for trained GMM model.", "M");
 
-void mlpackMain()
+static void mlpackMain()
 {
   // Check parameters and load data.
   if (CLI::GetParam<int>("seed") != 0)

src/mlpack/methods/hmm/hmm_generate_main.cpp

@@ -92,7 +92,7 @@ struct Generate
   }
 };
 
-void mlpackMain()
+static void mlpackMain()
 {
   RequireAtLeastOnePassed({ "output", "state" }, false, "no output will be "
       "saved");

src/mlpack/methods/hmm/hmm_loglik_main.cpp

@@ -76,7 +76,7 @@ struct Loglik
   }
 };
 
-void mlpackMain()
+static void mlpackMain()
 {
   // Load model, and calculate the log-likelihood of the sequence.
   CLI::GetParam<HMMModel>("input_model").PerformAction<Loglik>((void*) NULL);

src/mlpack/methods/hmm/hmm_train_main.cpp

@@ -340,7 +340,7 @@ struct Train
   }
 };
 
-void mlpackMain()
+static void mlpackMain()
 {
   // Set random seed.
   if (CLI::GetParam<int>("seed") != 0)

src/mlpack/methods/hmm/hmm_viterbi_main.cpp

@@ -82,7 +82,7 @@ struct Viterbi
   }
 };
 
-void mlpackMain()
+static void mlpackMain()
 {
   RequireAtLeastOnePassed({ "output" }, false, "no results will be saved");
 

src/mlpack/methods/hoeffding_trees/hoeffding_tree_main.cpp

@@ -109,7 +109,7 @@ PARAM_INT_IN("observations_before_binning", "If the 'domingos' split strategy "
 // Convenience typedef.
 typedef tuple<DatasetInfo, arma::mat> TupleType;
 
-void mlpackMain()
+static void mlpackMain()
 {
   // Check input parameters for validity.
   const string numericSplitStrategy =

src/mlpack/methods/kernel_pca/kernel_pca_main.cpp

@@ -164,7 +164,7 @@ void RunKPCA(arma::mat& dataset,
   }
 }
 
-void mlpackMain()
+static void mlpackMain()
 {
   RequireAtLeastOnePassed({ "output" }, false, "no output will be saved");
 

src/mlpack/methods/kmeans/kmeans_main.cpp

@@ -140,7 +140,7 @@ template<typename InitialPartitionPolicy,
          template<class, class> class LloydStepType>
 void RunKMeans(const InitialPartitionPolicy& ipp);
 
-void mlpackMain()
+static void mlpackMain()
 {
   // Initialize random seed.
   if (CLI::GetParam<int>("seed") != 0)

src/mlpack/methods/lars/lars_main.cpp

@@ -101,7 +101,7 @@ PARAM_DOUBLE_IN("lambda2", "Regularization parameter for l2-norm penalty.", "L",
 PARAM_FLAG("use_cholesky", "Use Cholesky decomposition during computation "
     "rather than explicitly computing the full Gram matrix.", "c");
 
-void mlpackMain()
+static void mlpackMain()
 {
   double lambda1 = CLI::GetParam<double>("lambda1");
   double lambda2 = CLI::GetParam<double>("lambda2");

src/mlpack/methods/linear_regression/linear_regression_main.cpp

@@ -85,7 +85,7 @@ PARAM_COL_OUT("output_predictions", "If --test_file is specified, this "
 PARAM_DOUBLE_IN("lambda", "Tikhonov regularization for ridge regression.  If 0,"
     " the method reduces to linear regression.", "l", 0.0);
 
-void mlpackMain()
+static void mlpackMain()
 {
   const double lambda = CLI::GetParam<double>("lambda");
 

src/mlpack/methods/local_coordinate_coding/local_coordinate_coding_main.cpp

@@ -93,7 +93,7 @@ PARAM_MATRIX_OUT("codes", "Output codes matrix.", "c");
 
 PARAM_INT_IN("seed", "Random seed.  If 0, 'std::time(NULL)' is used.", "s", 0);
 
-void mlpackMain()
+static void mlpackMain()
 {
   if (CLI::GetParam<int>("seed") != 0)
     RandomSeed((size_t) CLI::GetParam<int>("seed"));

src/mlpack/methods/logistic_regression/logistic_regression_main.cpp

@@ -133,7 +133,7 @@ PARAM_DOUBLE_IN("decision_boundary", "Decision boundary for prediction; if the "
     "logistic function for a point is less than the boundary, the class is "
     "taken to be 0; otherwise, the class is 1.", "d", 0.5);
 
-void mlpackMain()
+static void mlpackMain()
 {
   // Collect command-line options.
   const double lambda = CLI::GetParam<double>("lambda");
@@ -162,8 +162,8 @@ void mlpackMain()
   ReportIgnoredParam({{ "test", false }}, "output_probabilities");
 
   // Tolerance needs to be positive.
-  RequireParamValue<double>("tolerance", [](double x) { return x > 0.0; },
-      true, "tolerance must be positive");
+  RequireParamValue<double>("tolerance", [](double x) { return x >= 0.0; },
+      true, "tolerance must be positive or zero");
 
   // Optimizer has to be L-BFGS or SGD.
   RequireParamInSet<string>("optimizer", { "lbfgs", "sgd" },

src/mlpack/methods/lsh/lsh_main.cpp

@@ -82,7 +82,7 @@ PARAM_INT_IN("bucket_size", "The size of a bucket in the second level hash.",
     "B", 500);
 PARAM_INT_IN("seed", "Random seed.  If 0, 'std::time(NULL)' is used.", "s", 0);
 
-void mlpackMain()
+static void mlpackMain()
 {
   if (CLI::GetParam<int>("seed") != 0)
     math::RandomSeed((size_t) CLI::GetParam<int>("seed"));

src/mlpack/methods/mean_shift/mean_shift_main.cpp

@@ -66,7 +66,7 @@ PARAM_DOUBLE_IN("radius", "If the distance between two centroids is less than "
     "the given radius, one will be removed.  A radius of 0 or less means an "
     "estimate will be calculated and used for the radius.", "r", 0);
 
-void mlpackMain()
+static void mlpackMain()
 {
   const double radius = CLI::GetParam<double>("radius");
   const int maxIterations = CLI::GetParam<int>("max_iterations");

src/mlpack/methods/naive_bayes/nbc_main.cpp

@@ -105,7 +105,7 @@ PARAM_UROW_OUT("output", "The matrix in which the predicted labels for the"
 PARAM_MATRIX_OUT("output_probs", "The matrix in which the predicted probability"
     " of labels for the test set will be written.", "p");
 
-void mlpackMain()
+static void mlpackMain()
 {
   // Check input parameters.
   RequireOnlyOnePassed({ "training", "input_model" }, true);

src/mlpack/methods/nca/nca_main.cpp

@@ -124,7 +124,7 @@ using namespace mlpack::optimization;
 using namespace mlpack::util;
 using namespace std;
 
-void mlpackMain()
+static void mlpackMain()
 {
   if (CLI::GetParam<int>("seed") != 0)
     math::RandomSeed((size_t) CLI::GetParam<int>("seed"));

src/mlpack/methods/neighbor_search/kfn_main.cpp

@@ -95,7 +95,7 @@ PARAM_DOUBLE_IN("percentage", "If specified, will do approximate furthest "
     "neighbors will be at least (p*100) % of the distance as the true furthest "
     "neighbor.", "p", 1);
 
-void mlpackMain()
+static void mlpackMain()
 {
   if (CLI::GetParam<int>("seed") != 0)
     math::RandomSeed((size_t) CLI::GetParam<int>("seed"));

src/mlpack/methods/neighbor_search/knn_main.cpp

@@ -99,7 +99,7 @@ PARAM_STRING_IN("algorithm", "Type of neighbor search: 'naive', 'single_tree', "
 PARAM_DOUBLE_IN("epsilon", "If specified, will do approximate nearest neighbor "
     "search with given relative error.", "e", 0);
 
-void mlpackMain()
+static void mlpackMain()
 {
   if (CLI::GetParam<int>("seed") != 0)
     math::RandomSeed((size_t) CLI::GetParam<int>("seed"));

src/mlpack/methods/nmf/nmf_main.cpp

@@ -76,7 +76,7 @@ PARAM_DOUBLE_IN("min_residue", "The minimum root mean square residue allowed "
 PARAM_STRING_IN("update_rules", "Update rules for each iteration; ( multdist | "
     "multdiv | als ).", "u", "multdist");
 
-void mlpackMain()
+static void mlpackMain()
 {
   // Initialize random seed.
   if (CLI::GetParam<int>("seed") != 0)

src/mlpack/methods/pca/pca_main.cpp

@@ -99,7 +99,7 @@ void RunPCA(arma::mat& dataset,
       dataset.n_rows << " dimensions)." << endl;
 }
 
-void mlpackMain()
+static void mlpackMain()
 {
   // Load input dataset.
   arma::mat& dataset = CLI::GetParam<arma::mat>("input");
@@ -119,7 +119,8 @@ void mlpackMain()
   error << "cannot be greater than existing dimensionality (" << dataset.n_rows
       << ")";
   RequireParamValue<int>("new_dimensionality",
-      [dataset](int x) { return x <= dataset.n_rows; }, true, error.str());
+      [dataset](int x) { return x <= (int) dataset.n_rows; }, true,
+      error.str());
 
   RequireParamValue<double>("var_to_retain",
       [](double x) { return x >= 0.0 && x <= 1.0; }, true,

src/mlpack/methods/perceptron/perceptron_main.cpp

@@ -118,7 +118,7 @@ PARAM_MATRIX_IN("test", "A matrix containing the test set.", "T");
 PARAM_UROW_OUT("output", "The matrix in which the predicted labels for the"
     " test set will be written.", "o");
 
-void mlpackMain()
+static void mlpackMain()
 {
   // First, get all parameters and validate them.
   const size_t maxIterations = (size_t) CLI::GetParam<int>("max_iterations");

src/mlpack/methods/preprocess/preprocess_binarize_main.cpp

@@ -54,7 +54,7 @@ using namespace mlpack::util;
 using namespace arma;
 using namespace std;
 
-void mlpackMain()
+static void mlpackMain()
 {
   const size_t dimension = (size_t) CLI::GetParam<int>("dimension");
   const double threshold = CLI::GetParam<double>("threshold");

src/mlpack/methods/preprocess/preprocess_describe_main.cpp

@@ -153,7 +153,7 @@ double StandardError(const size_t size, const double& fStd)
   return fStd / sqrt(size);
 }
 
-void mlpackMain()
+static void mlpackMain()
 {
   const size_t dimension = static_cast<size_t>(CLI::GetParam<int>("dimension"));
   const size_t precision = static_cast<size_t>(CLI::GetParam<int>("precision"));

src/mlpack/methods/preprocess/preprocess_imputer_main.cpp

@@ -53,7 +53,7 @@ using namespace arma;
 using namespace std;
 using namespace data;
 
-void mlpackMain()
+static void mlpackMain()
 {
   const string inputFile = CLI::GetParam<string>("input_file");
   const string outputFile = CLI::GetParam<string>("output_file");

src/mlpack/methods/preprocess/preprocess_split_main.cpp

@@ -71,7 +71,7 @@ using namespace mlpack::util;
 using namespace arma;
 using namespace std;
 
-void mlpackMain()
+static void mlpackMain()
 {
   // Parse command line options.
   const double testRatio = CLI::GetParam<double>("test_ratio");

src/mlpack/methods/radical/radical_main.cpp

@@ -57,7 +57,7 @@ using namespace mlpack::util;
 using namespace std;
 using namespace arma;
 
-void mlpackMain()
+static void mlpackMain()
 {
   // Set random seed.
   if (CLI::GetParam<int>("seed") != 0)

src/mlpack/methods/random_forest/random_forest_main.cpp

@@ -68,7 +68,7 @@ PARAM_MODEL_IN(RandomForestModel, "input_model", "Pre-trained random forest to "
 PARAM_MODEL_OUT(RandomForestModel, "output_model", "Model to save trained "
     "random forest to.", "M");
 
-void mlpackMain()
+static void mlpackMain()
 {
   // Check for incompatible input parameters.
   RequireOnlyOnePassed({ "training", "input_model" }, true);

src/mlpack/methods/range_search/range_search_main.cpp

@@ -91,7 +91,7 @@ PARAM_FLAG("naive", "If true, O(n^2) naive mode is used for computation.", "N");
 PARAM_FLAG("single_mode", "If true, single-tree search is used (as opposed to "
     "dual-tree search).", "S");
 
-void mlpackMain()
+static void mlpackMain()
 {
   if (CLI::GetParam<int>("seed") != 0)
     math::RandomSeed((size_t) CLI::GetParam<int>("seed"));

src/mlpack/methods/rann/krann_main.cpp

@@ -96,7 +96,7 @@ PARAM_FLAG("first_leaf_exact", "The flag to trigger sampling only after "
 PARAM_INT_IN("single_sample_limit", "The limit on the maximum number of "
     "samples (and hence the largest node you can approximate).", "z", 20);
 
-void mlpackMain()
+static void mlpackMain()
 {
   if (CLI::GetParam<int>("seed") != 0)
     math::RandomSeed((size_t) CLI::GetParam<int>("seed"));

src/mlpack/methods/softmax_regression/softmax_regression_main.cpp

@@ -115,7 +115,7 @@ void TestClassifyAcc(const size_t numClasses, const Model& model);
 template<typename Model>
 unique_ptr<Model> TrainSoftmax(const size_t maxIterations);
 
-void mlpackMain()
+static void mlpackMain()
 {
   const int maxIterations = CLI::GetParam<int>("max_iterations");
 

src/mlpack/methods/sparse_coding/sparse_coding.cpp

@@ -12,7 +12,6 @@
  */
 #include "sparse_coding.hpp"
 #include <mlpack/core/math/lin_alg.hpp>
-#include <mlpack/core/util/param.hpp>
 
 namespace mlpack {
 namespace sparse_coding {

src/mlpack/methods/sparse_coding/sparse_coding_main.cpp

@@ -96,7 +96,7 @@ PARAM_MATRIX_OUT("codes", "Matrix to save the output sparse codes of the test "
 
 PARAM_MATRIX_IN("test", "Optional matrix to be encoded by trained model.", "T");
 
-void mlpackMain()
+static void mlpackMain()
 {
   if (CLI::GetParam<int>("seed") != 0)
     RandomSeed((size_t) CLI::GetParam<int>("seed"));

src/mlpack/tests/CMakeLists.txt

@@ -123,6 +123,7 @@ add_executable(mlpack_test
   union_find_test.cpp
   vantage_point_tree_test.cpp
   main_tests/decision_tree_test.cpp
+  main_tests/linear_regression_test.cpp
   main_tests/pca_test.cpp
   main_tests/preprocess_binarize_test.cpp
   main_tests/preprocess_imputer_test.cpp

src/mlpack/tests/adam_test.cpp

@@ -2,8 +2,9 @@
  * @file adam_test.cpp
  * @author Vasanth Kalingeri
  * @author Vivek Pal
+ * @author Sourabh Varshney
  *
- * Tests the Adam and AdaMax optimizer.
+ * Tests the Adam, AdaMax, AMSGrad and Nadam optimizer.
  *
  * mlpack is free software; you may redistribute it and/or modify it under the
  * terms of the 3-clause BSD license.  You should have received a copy of the
@@ -62,6 +63,22 @@ BOOST_AUTO_TEST_CASE(SimpleAdaMaxTestFunction)
   BOOST_REQUIRE_SMALL(coordinates[2], 0.1);
 }
 
+/**
+ * Tests the AMSGrad optimizer using a simple test function.
+ */
+BOOST_AUTO_TEST_CASE(SimpleAMSGradTestFunction)
+{
+  SGDTestFunction f;
+  AMSGrad optimizer(1e-3, 1, 0.9, 0.999, 1e-8, 500000, 1e-11, true);
+
+  arma::mat coordinates = f.GetInitialPoint();
+  optimizer.Optimize(f, coordinates);
+
+  BOOST_REQUIRE_SMALL(coordinates[0], 0.1);
+  BOOST_REQUIRE_SMALL(coordinates[1], 0.1);
+  BOOST_REQUIRE_SMALL(coordinates[2], 0.1);
+}
+
 /**
  * Run Adam on logistic regression and make sure the results are acceptable.
  */
@@ -178,4 +195,138 @@ BOOST_AUTO_TEST_CASE(AdaMaxLogisticRegressionTest)
   BOOST_REQUIRE_CLOSE(testAcc, 100.0, 0.6); // 0.6% error tolerance.
 }
 
+/**
+ * Run AMSGrad on logistic regression and make sure the results are acceptable.
+ */
+BOOST_AUTO_TEST_CASE(AMSGradLogisticRegressionTest)
+{
+  // Generate a two-Gaussian dataset.
+  GaussianDistribution g1(arma::vec("1.0 1.0 1.0"), arma::eye<arma::mat>(3, 3));
+  GaussianDistribution g2(arma::vec("9.0 9.0 9.0"), arma::eye<arma::mat>(3, 3));
+
+  arma::mat data(3, 1000);
+  arma::Row<size_t> responses(1000);
+  for (size_t i = 0; i < 500; ++i)
+  {
+    data.col(i) = g1.Random();
+    responses[i] = 0;
+  }
+  for (size_t i = 500; i < 1000; ++i)
+  {
+    data.col(i) = g2.Random();
+    responses[i] = 1;
+  }
+
+  // Shuffle the dataset.
+  arma::uvec indices = arma::shuffle(arma::linspace<arma::uvec>(0,
+      data.n_cols - 1, data.n_cols));
+  arma::mat shuffledData(3, 1000);
+  arma::Row<size_t> shuffledResponses(1000);
+  for (size_t i = 0; i < data.n_cols; ++i)
+  {
+    shuffledData.col(i) = data.col(indices[i]);
+    shuffledResponses[i] = responses[indices[i]];
+  }
+
+  // Create a test set.
+  arma::mat testData(3, 1000);
+  arma::Row<size_t> testResponses(1000);
+  for (size_t i = 0; i < 500; ++i)
+  {
+    testData.col(i) = g1.Random();
+    testResponses[i] = 0;
+  }
+  for (size_t i = 500; i < 1000; ++i)
+  {
+    testData.col(i) = g2.Random();
+    testResponses[i] = 1;
+  }
+
+  AMSGrad amsgrad(1e-3, 1, 0.9, 0.999, 1e-8, 500000, 1e-11, true);
+  LogisticRegression<> lr(shuffledData, shuffledResponses, amsgrad, 0.5);
+
+  // Ensure that the error is close to zero.
+  const double acc = lr.ComputeAccuracy(data, responses);
+  BOOST_REQUIRE_CLOSE(acc, 100.0, 0.3); // 0.3% error tolerance.
+
+  const double testAcc = lr.ComputeAccuracy(testData, testResponses);
+  BOOST_REQUIRE_CLOSE(testAcc, 100.0, 0.6); // 0.6% error tolerance.
+}
+
+/**
+ * Tests the Nadam optimizer using a simple test function.
+ */
+BOOST_AUTO_TEST_CASE(SimpleNadamTestFunction)
+{
+  SGDTestFunction f;
+  Nadam optimizer(1e-3, 1, 0.9, 0.99, 1e-8, 500000, 1e-9, true);
+
+  arma::mat coordinates = f.GetInitialPoint();
+  optimizer.Optimize(f, coordinates);
+
+  BOOST_REQUIRE_SMALL(coordinates[0], 0.1);
+  BOOST_REQUIRE_SMALL(coordinates[1], 0.1);
+  BOOST_REQUIRE_SMALL(coordinates[2], 0.1);
+}
+
+/**
+ * Run Nadam on logistic regression and make sure the results are acceptable.
+ */
+BOOST_AUTO_TEST_CASE(NadamLogisticRegressionTest)
+{
+  // Generate a two-Gaussian dataset.
+  GaussianDistribution g1(arma::vec("1.0 1.0 1.0"),
+  arma::eye<arma::mat>(3, 3));
+  GaussianDistribution g2(arma::vec("9.0 9.0 9.0"),
+  arma::eye<arma::mat>(3, 3));
+
+  arma::mat data(3, 1000);
+  arma::Row<size_t> responses(1000);
+  for (size_t i = 0; i < 500; ++i)
+  {
+    data.col(i) = g1.Random();
+    responses[i] = 0;
+  }
+  for (size_t i = 500; i < 1000; ++i)
+  {
+    data.col(i) = g2.Random();
+    responses[i] = 1;
+  }
+
+  // Shuffle the dataset.
+  arma::uvec indices = arma::shuffle(arma::linspace<arma::uvec>(0,
+  data.n_cols - 1, data.n_cols));
+  arma::mat shuffledData(3, 1000);
+  arma::Row<size_t> shuffledResponses(1000);
+  for (size_t i = 0; i < data.n_cols; ++i)
+  {
+    shuffledData.col(i) = data.col(indices[i]);
+    shuffledResponses[i] = responses[indices[i]];
+  }
+
+  // Create a test set.
+  arma::mat testData(3, 1000);
+  arma::Row<size_t> testResponses(1000);
+  for (size_t i = 0; i < 500; ++i)
+  {
+    testData.col(i) = g1.Random();
+    testResponses[i] = 0;
+  }
+  for (size_t i = 500; i < 1000; ++i)
+  {
+    testData.col(i) = g2.Random();
+    testResponses[i] = 1;
+  }
+
+  Nadam nadam;
+  LogisticRegression<> lr(shuffledData, shuffledResponses, nadam, 0.5);
+
+  // Ensure that the error is close to zero.
+  const double acc = lr.ComputeAccuracy(data, responses);
+  BOOST_REQUIRE_CLOSE(acc, 100.0, 0.3); // 0.3% error tolerance.
+
+  const double testAcc = lr.ComputeAccuracy(testData, testResponses);
+  BOOST_REQUIRE_CLOSE(testAcc, 100.0, 0.6); // 0.6% error tolerance.
+}
+
 BOOST_AUTO_TEST_SUITE_END();

src/mlpack/tests/cli_test.cpp

@@ -22,6 +22,8 @@ using Option = mlpack::bindings::cli::CLIOption<T>;
 } // namespace util
 } // namespace mlpack
 
+static const std::string testName = "";
+
 #include <mlpack/core/util/param.hpp>
 #include <mlpack/bindings/cli/parse_command_line.hpp>
 #include <mlpack/bindings/cli/end_program.hpp>

src/mlpack/tests/cne_test.cpp

@@ -47,37 +47,49 @@ BOOST_AUTO_TEST_CASE(CNEXORTest)
   arma::mat train("1, 0, 0, 1; 1, 0, 1, 0");
   arma::mat labels("1, 1, 2, 2");
 
-  // network with 2 input 2 hidden and 2 output layer
-  FFN<NegativeLogLikelihood<> > network;
-
-  network.Add<Linear<> >(2, 2);
-  network.Add<SigmoidLayer<> >();
-  network.Add<Linear<> >(2, 2);
-  network.Add<LogSoftMax<> >();
-
-  // CNE object.
-  CNE opt(60, 5000, 0.1, 0.02, 0.2, 0.1, -1);
-
-  // Training the network with CNE
-  network.Train(train, labels, opt);
-
-  // Predicting for the same train data
-  arma::mat predictionTemp;
-  network.Predict(train, predictionTemp);
-
-  arma::mat prediction = arma::zeros<arma::mat>(1, predictionTemp.n_cols);
-
-  for (size_t i = 0; i < predictionTemp.n_cols; ++i)
+  // CNE may fail to find a good optimum.  But if it can succeed one out of 6
+  // times I think that is sufficient to say it is working.
+  size_t successes = 0;
+  for (size_t trial = 0; trial < 6; ++trial)
   {
-    prediction(i) = arma::as_scalar(arma::find(
-        arma::max(predictionTemp.col(i)) == predictionTemp.col(i), 1)) + 1;
+    // Build a network with 2 input, 2 hidden, and 2 output layers.
+    FFN<NegativeLogLikelihood<> > network;
+
+    network.Add<Linear<> >(2, 2);
+    network.Add<SigmoidLayer<> >();
+    network.Add<Linear<> >(2, 2);
+    network.Add<LogSoftMax<> >();
+
+    // CNE object.
+    CNE opt(60, 5000, 0.1, 0.02, 0.2, 0.1, -1);
+
+    // Training the network with CNE
+    network.Train(train, labels, opt);
+
+    // Predicting for the same train data
+    arma::mat predictionTemp;
+    network.Predict(train, predictionTemp);
+
+    arma::mat prediction = arma::zeros<arma::mat>(1, predictionTemp.n_cols);
+
+    for (size_t i = 0; i < predictionTemp.n_cols; ++i)
+    {
+      prediction(i) = arma::as_scalar(arma::find(
+          arma::max(predictionTemp.col(i)) == predictionTemp.col(i), 1)) + 1;
+    }
+
+    // 1 means 0 and 2 means 1 as the output to XOR.
+    if ((prediction[0] == 1) &&
+        (prediction[1] == 1) &&
+        (prediction[2] == 2) &&
+        (prediction[3] == 2))
+    {
+      ++successes;
+      break;
+    }
   }
 
-  // 1 means 0 and 2 means 1 as the output to XOR
-  BOOST_REQUIRE_EQUAL(1, prediction[0]);
-  BOOST_REQUIRE_EQUAL(1, prediction[1]);
-  BOOST_REQUIRE_EQUAL(2, prediction[2]);
-  BOOST_REQUIRE_EQUAL(2, prediction[3]);
+  BOOST_REQUIRE_GT(successes, 0);
 }
 
 /**
@@ -127,7 +139,7 @@ BOOST_AUTO_TEST_CASE(CNELogisticRegressionTest)
     testResponses[i] = 1;
   }
 
-  CNE opt(30, 500, 0.2, 0.2, 0.3, 65, -1);
+  CNE opt(200, 10000, 0.2, 0.2, 0.3, 65, -1);
 
   LogisticRegression<> lr(shuffledData, shuffledResponses, opt, 0.5);
 
@@ -159,41 +171,52 @@ BOOST_AUTO_TEST_CASE(VanillaNetworkWithCNETest)
   data::Load("iris_test_labels.csv", testLabels, true);
   testLabels += 1;
 
-  // Create vanilla network with 4 input, 4 hidden and 3 output nodes.
-  FFN<NegativeLogLikelihood<> > model;
-  model.Add<Linear<> >(trainData.n_rows, 4);
-  model.Add<SigmoidLayer<> >();
-  model.Add<Linear<> >(4, 3);
-  model.Add<LogSoftMax<> >();
-
-  // Creating CNE object.
-  // The tolerance and objectiveChange are not taken into consideration.
-  CNE opt(30, 200, 0.2, 0.2, 0.3, -1, -1);
-
-  model.Train(trainData, trainLabels, opt);
-
-  arma::mat predictionTemp;
-  model.Predict(testData, predictionTemp);
-  arma::mat prediction = arma::zeros<arma::mat>(1, predictionTemp.n_cols);
-
-  for (size_t i = 0; i < predictionTemp.n_cols; ++i)
+  // Training the network may fail, so we will try a few times.
+  size_t successes = 0;
+  for (size_t trial = 0; trial < 4; ++trial)
   {
-    prediction(i) = arma::as_scalar(arma::find(
-        arma::max(predictionTemp.col(i)) == predictionTemp.col(i), 1)) + 1;
-  }
+    // Create vanilla network with 4 input, 4 hidden and 3 output nodes.
+    FFN<NegativeLogLikelihood<> > model;
+    model.Add<Linear<> >(trainData.n_rows, 4);
+    model.Add<SigmoidLayer<> >();
+    model.Add<Linear<> >(4, 3);
+    model.Add<LogSoftMax<> >();
 
-  size_t error = 0;
-  for (size_t i = 0; i < testData.n_cols; i++)
-  {
-    if (int(arma::as_scalar(prediction.col(i))) ==
-        int(arma::as_scalar(testLabels.col(i))))
+    // Creating CNE object.
+    // The tolerance and objectiveChange are not taken into consideration.
+    CNE opt(30, 200, 0.2, 0.2, 0.3, -1, -1);
+
+    model.Train(trainData, trainLabels, opt);
+
+    arma::mat predictionTemp;
+    model.Predict(testData, predictionTemp);
+    arma::mat prediction = arma::zeros<arma::mat>(1, predictionTemp.n_cols);
+
+    for (size_t i = 0; i < predictionTemp.n_cols; ++i)
     {
-      error++;
+      prediction(i) = arma::as_scalar(arma::find(
+          arma::max(predictionTemp.col(i)) == predictionTemp.col(i), 1)) + 1;
+    }
+
+    size_t error = 0;
+    for (size_t i = 0; i < testData.n_cols; i++)
+    {
+      if (int(arma::as_scalar(prediction.col(i))) ==
+          int(arma::as_scalar(testLabels.col(i))))
+      {
+        error++;
+      }
+    }
+
+    double classificationError = 1 - double(error) / testData.n_cols;
+    if (classificationError <= 0.1)
+    {
+      ++successes;
+      break;
     }
   }
 
-  double classificationError = 1 - double(error) / testData.n_cols;
-  BOOST_REQUIRE_LE(classificationError, 0.1);
+  BOOST_REQUIRE_GT(successes, 0);
 }
 
 BOOST_AUTO_TEST_SUITE_END();

src/mlpack/tests/frankwolfe_test.cpp

@@ -70,7 +70,7 @@ BOOST_AUTO_TEST_CASE(regularizedOMP)
   mat B1 = 0.1 * eye(k, k);
   mat B2 = 100 * randn(k, k);
   mat A = join_horiz(B1, B2); // The dictionary is input as columns of A.
-  vec b(k); // Vector to be sparsely approximated.
+  vec b(k, arma::fill::zeros); // Vector to be sparsely approximated.
   b(0) = 1;
   b(1) = 1;
   vec lambda(A.n_cols);
@@ -98,8 +98,8 @@ BOOST_AUTO_TEST_CASE(PruneSupportOMP)
   int k = 3;
   mat B1;
   B1 << 1 << 0 << 1 << endr
-    << 0 << 1 << 1 << endr
-    << 0 << 0 << 1 << endr;
+     << 0 << 1 << 1 << endr
+     << 0 << 0 << 1 << endr;
   mat B2 = randu(k, k);
   mat A = join_horiz(B1, B2); // The dictionary is input as columns of A.
   vec b;

src/mlpack/tests/main_tests/linear_regression_test.cpp

@@ -0,0 +1,70 @@
+/**
+ * @file linear_regression_test.cpp
+ * @author Eugene Freyman
+ *
+ * Test mlpackMain() of linear_regression_main.cpp.
+ */
+#include <string>
+
+#define BINDING_TYPE BINDING_TYPE_TEST
+static const std::string testName = "LinearRegression";
+
+#include <mlpack/core.hpp>
+#include <mlpack/core/util/mlpack_main.hpp>
+#include <mlpack/methods/linear_regression/linear_regression_main.cpp>
+
+#include <boost/test/unit_test.hpp>
+#include "../test_tools.hpp"
+
+using namespace mlpack;
+
+// Utility function to set a parameter and mark it as passed, using copy
+// semantics.
+template<typename T>
+void SetInputParam(const std::string& name, const T& value)
+{
+  CLI::GetParam<T>(name) = value;
+  CLI::SetPassed(name);
+}
+
+// Utility function to set a parameter and mark it as passed, using move
+// semantics.
+template<typename T>
+void SetInputParam(const std::string& name, T&& value)
+{
+  CLI::GetParam<T>(name) = std::move(value);
+  CLI::SetPassed(name);
+}
+
+struct LinearRegressionTestFixture
+{
+ public:
+  LinearRegressionTestFixture()
+  {
+    // Cache in the options for this program.
+    CLI::RestoreSettings(testName);
+  }
+
+  ~LinearRegressionTestFixture()
+  {
+    // Clear the settings.
+    CLI::ClearSettings();
+  }
+};
+
+BOOST_FIXTURE_TEST_SUITE(LinearRegressionMainTest, LinearRegressionTestFixture);
+
+BOOST_AUTO_TEST_CASE(LinearRegressionWrongResponseSizeTest)
+{
+  arma::mat x = arma::randu<arma::mat>(5, 5);
+  arma::rowvec y = arma::randu<arma::rowvec>(4);
+
+  SetInputParam("training", std::move(x));
+  SetInputParam("training_responses", std::move(y));
+
+  Log::Fatal.ignoreInput = true;
+  BOOST_REQUIRE_THROW(mlpackMain(), std::runtime_error);
+  Log::Fatal.ignoreInput = false;
+}
+
+BOOST_AUTO_TEST_SUITE_END();

src/mlpack/tests/main_tests/pca_test.cpp

@@ -4,7 +4,11 @@
  *
  * Test mlpackMain() of pca_main.cpp.
  */
+#include <string>
+
 #define BINDING_TYPE BINDING_TYPE_TEST
+static const std::string testName = "PrincipalComponentAnalysis";
+
 #include <mlpack/core.hpp>
 #include <mlpack/core/util/mlpack_main.hpp>
 #include <mlpack/methods/pca/pca_main.cpp>
@@ -14,16 +18,6 @@
 
 using namespace mlpack;
 
-namespace mlpack {
-namespace bindings {
-namespace tests {
-
-extern std::string programName;
-
-}
-}
-}
-
 // Utility function to set a parameter and mark it as passed, using copy
 // semantics.
 template<typename T>
@@ -48,7 +42,7 @@ struct PCATestFixture
   PCATestFixture()
   {
     // Cache in the options for this program.
-    CLI::RestoreSettings(mlpack::bindings::tests::programName);
+    CLI::RestoreSettings(testName);
   }
 
   ~PCATestFixture()

src/mlpack/tests/momentum_sgd_test.cpp

@@ -36,7 +36,7 @@ BOOST_AUTO_TEST_CASE(MomentumSGDSpeedUpTestFunction)
   arma::mat coordinates = f.GetInitialPoint();
   double result = s.Optimize(f, coordinates);
 
-  BOOST_REQUIRE_CLOSE(result, -1.0, 0.05);
+  BOOST_REQUIRE_CLOSE(result, -1.0, 0.15);
   BOOST_REQUIRE_SMALL(coordinates[0], 1e-3);
   BOOST_REQUIRE_SMALL(coordinates[1], 1e-7);
   BOOST_REQUIRE_SMALL(coordinates[2], 1e-7);
@@ -65,12 +65,12 @@ BOOST_AUTO_TEST_CASE(GeneralizedRosenbrockTest)
     // Create the generalized Rosenbrock function.
     GeneralizedRosenbrockFunction f(i);
     MomentumUpdate momentumUpdate(0.4);
-    MomentumSGD s(0.001, 1, 0, 1e-15, true, momentumUpdate);
+    MomentumSGD s(0.0008, 1, 0, 1e-15, true, momentumUpdate);
 
     arma::mat coordinates = f.GetInitialPoint();
     double result = s.Optimize(f, coordinates);
 
-    BOOST_REQUIRE_SMALL(result, 1e-10);
+    BOOST_REQUIRE_SMALL(result, 1e-4);
     for (size_t j = 0; j < i; ++j)
       BOOST_REQUIRE_CLOSE(coordinates[j], (double) 1.0, 1e-3);
   }

src/mlpack/tests/pca_test.cpp

@@ -230,21 +230,28 @@ BOOST_AUTO_TEST_CASE(QUICPCADimensionalityReductionTest)
   data::Load("test_data_3_1000.csv", data);
   data1 = data;
 
+  arma::mat backupData(data);
+
   // It isn't guaranteed that the QUIC-SVD will match with the exact SVD method,
   // starting with random samples. If this works 1 of 5 times, I'm fine with
-  // that. All I want to know is that the QUIC-SVD method is  able to solve the
+  // that. All I want to know is that the QUIC-SVD method is able to solve the
   // task and is at least as good as the exact method (plus a little bit for
   // noise).
   size_t successes = 0;
   for (size_t trial = 0; trial < 5; ++trial)
   {
+    if (trial > 0)
+    {
+      data = backupData;
+      data1 = backupData;
+    }
+
     PCAType<ExactSVDPolicy> exactPCA;
     const double varRetainedExact = exactPCA.Apply(data, 1);
 
     PCAType<QUICSVDPolicy> quicPCA;
     const double varRetainedQUIC = quicPCA.Apply(data1, 1);
 
-
     if (std::abs(varRetainedExact - varRetainedQUIC) < 0.2)
     {
       ++successes;
@@ -294,21 +301,21 @@ BOOST_AUTO_TEST_CASE(PCAScalingTest)
   // The first two components of the eigenvector with largest eigenvalue should
   // be somewhere near sqrt(2) / 2.  The third component should be close to
   // zero.  There is noise, of course...
-  BOOST_REQUIRE_CLOSE(std::abs(eigvec(0, 0)), sqrt(2) / 2, 0.2);
-  BOOST_REQUIRE_CLOSE(std::abs(eigvec(1, 0)), sqrt(2) / 2, 0.2);
-  BOOST_REQUIRE_SMALL(eigvec(2, 0), 0.08); // Large tolerance for noise.
+  BOOST_REQUIRE_CLOSE(std::abs(eigvec(0, 0)), sqrt(2) / 2, 0.35);
+  BOOST_REQUIRE_CLOSE(std::abs(eigvec(1, 0)), sqrt(2) / 2, 0.35);
+  BOOST_REQUIRE_SMALL(eigvec(2, 0), 0.1); // Large tolerance for noise.
 
   // The second component should be focused almost entirely in the third
   // dimension.
-  BOOST_REQUIRE_SMALL(eigvec(0, 1), 0.08);
-  BOOST_REQUIRE_SMALL(eigvec(1, 1), 0.08);
-  BOOST_REQUIRE_CLOSE(std::abs(eigvec(2, 1)), 1.0, 0.2);
+  BOOST_REQUIRE_SMALL(eigvec(0, 1), 0.1);
+  BOOST_REQUIRE_SMALL(eigvec(1, 1), 0.1);
+  BOOST_REQUIRE_CLOSE(std::abs(eigvec(2, 1)), 1.0, 0.35);
 
   // The third component should have the same absolute value characteristics as
   // the first (plus 20% tolerance).
-  BOOST_REQUIRE_CLOSE(std::abs(eigvec(0, 0)), sqrt(2) / 2, 0.2);
-  BOOST_REQUIRE_CLOSE(std::abs(eigvec(1, 0)), sqrt(2) / 2, 0.2);
-  BOOST_REQUIRE_SMALL(eigvec(2, 0), 0.08); // Large tolerance for noise.
+  BOOST_REQUIRE_CLOSE(std::abs(eigvec(0, 0)), sqrt(2) / 2, 0.35);
+  BOOST_REQUIRE_CLOSE(std::abs(eigvec(1, 0)), sqrt(2) / 2, 0.35);
+  BOOST_REQUIRE_SMALL(eigvec(2, 0), 0.1); // Large tolerance for noise.
 
   // The eigenvalues should sum to three.
   BOOST_REQUIRE_CLOSE(accu(eigval), 3.0, 0.1); // 10% tolerance.

src/mlpack/tests/quic_svd_test.cpp

@@ -29,18 +29,29 @@ BOOST_AUTO_TEST_CASE(QUICSVDReconstructionError)
   arma::mat dataset;
   data::Load("test_data_3_1000.csv", dataset);
 
-  // Obtain the SVD using default parameters.
-  arma::mat u, v, sigma;
-  svd::QUIC_SVD quicsvd(dataset, u, v, sigma);
+  // The QUIC-SVD procedure can fail---the Monte Carlo error calculation is
+  // random.  Therefore we simply require at least one success.
+  size_t successes = 0;
+  for (size_t i = 0; i < 3; ++i)
+  {
+    // Obtain the SVD using default parameters.
+    Log::Info.ignoreInput = false;
+    Log::Warn.ignoreInput = false;
+    arma::mat u, v, sigma;
+    svd::QUIC_SVD quicsvd(dataset, u, v, sigma);
 
-  // Reconstruct the matrix using the SVD.
-  arma::mat reconstruct;
-  reconstruct = u * sigma * v.t();
+    // Reconstruct the matrix using the SVD.
+    arma::mat reconstruct;
+    reconstruct = u * sigma * v.t();
 
-  // The relative reconstruction error should be small.
-  double relativeError = arma::norm(dataset - reconstruct, "frob") /
-                         arma::norm(dataset, "frob");
-  BOOST_REQUIRE_SMALL(relativeError, 1e-5);
+    // The relative reconstruction error should be small.
+    double relativeError = arma::norm(dataset - reconstruct, "frob") /
+                           arma::norm(dataset, "frob");
+    if (relativeError < 1e-5)
+      ++successes;
+  }
+
+  BOOST_REQUIRE_GT(successes, 0);
 }
 
 /**
@@ -71,7 +82,7 @@ BOOST_AUTO_TEST_CASE(QUICSVDSingularValueError)
 
   // The sigular value error should be small.
   double error = arma::norm(s1 - s3);
-  BOOST_REQUIRE_SMALL(error, 0.05);
+  BOOST_REQUIRE_SMALL(error, 0.1);
 }
 
 BOOST_AUTO_TEST_CASE(QUICSVDSameDimensionTest)

src/mlpack/tests/random_forest_test.cpp

@@ -224,7 +224,7 @@ BOOST_AUTO_TEST_CASE(UnweightedCategoricalLearningTest)
   arma::Row<size_t> testLabels = l.subvec(2000, 3999);
 
   // Train a random forest and a decision tree.
-  RandomForest<> rf(trainingData, di, trainingLabels, 5, 10 /* 10 trees */, 5);
+  RandomForest<> rf(trainingData, di, trainingLabels, 5, 15 /* 15 trees */, 5);
   DecisionTree<> dt(trainingData, di, trainingLabels, 5, 5);
 
   // Get performance statistics on test data.
@@ -238,7 +238,7 @@ BOOST_AUTO_TEST_CASE(UnweightedCategoricalLearningTest)
   size_t rfCorrect = arma::accu(rfPredictions == testLabels);
   size_t dtCorrect = arma::accu(dtPredictions == testLabels);
 
-  BOOST_REQUIRE_GE(rfCorrect, dtCorrect);
+  BOOST_REQUIRE_GE(rfCorrect, dtCorrect - 30);
   BOOST_REQUIRE_GE(rfCorrect, size_t(0.7 * testData.n_cols));
 }
 
@@ -281,7 +281,7 @@ BOOST_AUTO_TEST_CASE(WeightedCategoricalLearningTest)
   arma::Row<size_t> fullLabels = arma::join_rows(trainingLabels, randomLabels);
 
   // Build a random forest and a decision tree.
-  RandomForest<> rf(fullData, di, fullLabels, 5, 10 /* 10 trees */, 5);
+  RandomForest<> rf(fullData, di, fullLabels, 5, 15 /* 15 trees */, 5);
   DecisionTree<> dt(fullData, di, fullLabels, 5, 5);
 
   // Get performance statistics on test data.
@@ -295,7 +295,7 @@ BOOST_AUTO_TEST_CASE(WeightedCategoricalLearningTest)
   size_t rfCorrect = arma::accu(rfPredictions == testLabels);
   size_t dtCorrect = arma::accu(dtPredictions == testLabels);
 
-  BOOST_REQUIRE_GE(rfCorrect, dtCorrect);
+  BOOST_REQUIRE_GE(rfCorrect, dtCorrect - 30);
   BOOST_REQUIRE_GE(rfCorrect, size_t(0.7 * testData.n_cols));
 }
 

src/mlpack/tests/recurrent_network_test.cpp

@@ -67,13 +67,13 @@ void GenerateNoisySines(arma::mat& data,
 BOOST_AUTO_TEST_CASE(SequenceClassificationTest)
 {
   // It isn't guaranteed that the recurrent network will converge in the
-  // specified number of iterations using random weights. If this works 1 of 5
+  // specified number of iterations using random weights. If this works 1 of 6
   // times, I'm fine with that. All I want to know is that the network is able
   // to escape from local minima and to solve the task.
   size_t successes = 0;
   const size_t rho = 10;
 
-  for (size_t trial = 0; trial < 5; ++trial)
+  for (size_t trial = 0; trial < 6; ++trial)
   {
     // Generate 12 (2 * 6) noisy sines. A single sine contains rho
     // points/features.
@@ -287,7 +287,8 @@ void GenerateNextReber(const arma::Mat<char>& transitions,
  * @param nextReber All reachable next symbols.
  */
 void GenerateNextRecursiveReber(const arma::Mat<char>& transitions,
-                               const std::string& reber, std::string& nextReber)
+                                const std::string& reber,
+                                std::string& nextReber)
 {
   size_t state = 0;
   size_t numPs = 0;
@@ -457,7 +458,7 @@ void ReberGrammarTestNetwork(const size_t hiddenSize = 4,
     MomentumSGD opt(0.06, 50, 2, -50000);
 
     arma::mat inputTemp, labelsTemp;
-    for (size_t i = 0; i < (iterations + offset); i++)
+    for (size_t iteration = 0; iteration < (iterations + offset); iteration++)
     {
       for (size_t j = 0; j < trainReberGrammarCount; j++)
       {
@@ -672,7 +673,7 @@ void DistractedSequenceRecallTestNetwork(
     // We increase the number of iterations (training) if the first run didn't
     // pass.
     arma::mat inputTemp, labelsTemp;
-    for (size_t i = 0; i < (9 + offset); i++)
+    for (size_t iteration = 0; iteration < (9 + offset); iteration++)
     {
       for (size_t j = 0; j < trainDistractedSequenceCount; j++)
       {
@@ -743,6 +744,83 @@ BOOST_AUTO_TEST_CASE(GRUDistractedSequenceRecallTest)
   DistractedSequenceRecallTestNetwork<GRU<> >(4, 8);
 }
 
+/**
+ * Create a simple recurrent neural network for the noisy sines task, and
+ * require that it produces the exact same network for a few batch sizes.
+ */
+template<typename RecurrentLayerType>
+void BatchSizeTest()
+{
+  const size_t rho = 10;
+
+  // Generate 12 (2 * 6) noisy sines. A single sine contains rho
+  // points/features.
+  arma::mat input, labelsTemp;
+  GenerateNoisySines(input, labelsTemp, rho, 6);
+
+  arma::mat labels = arma::zeros<arma::mat>(rho, labelsTemp.n_cols);
+  for (size_t i = 0; i < labelsTemp.n_cols; ++i)
+  {
+    const int value = arma::as_scalar(arma::find(
+        arma::max(labelsTemp.col(i)) == labelsTemp.col(i), 1)) + 1;
+    labels.col(i).fill(value);
+  }
+
+  RNN<> model(rho);
+  model.Add<Linear<>>(1, 10);
+  model.Add<SigmoidLayer<>>();
+  model.Add<RecurrentLayerType>(10, 10);
+  model.Add<SigmoidLayer<>>();
+  model.Add<Linear<>>(10, 10);
+  model.Add<SigmoidLayer<>>();
+
+  model.Reset();
+  arma::mat initParams = model.Parameters();
+
+  StandardSGD opt(1e-5, 1, 5, -100, false);
+  model.Train(input, labels, opt);
+
+  // This is trained with one point.
+  arma::mat outputParams = model.Parameters();
+
+  model.Reset();
+  model.Parameters() = initParams;
+  opt.BatchSize() = 2;
+  model.Train(input, labels, opt);
+
+  CheckMatrices(outputParams, model.Parameters(), 1);
+
+  model.Parameters() = initParams;
+  opt.BatchSize() = 5;
+  model.Train(input, labels, opt);
+
+  CheckMatrices(outputParams, model.Parameters(), 1);
+}
+
+/**
+ * Ensure LSTMs work with larger batch sizes.
+ */
+BOOST_AUTO_TEST_CASE(LSTMBatchSizeTest)
+{
+  BatchSizeTest<LSTM<>>();
+}
+
+/**
+ * Ensure fast LSTMs work with larger batch sizes.
+ */
+BOOST_AUTO_TEST_CASE(FastLSTMBatchSizeTest)
+{
+  BatchSizeTest<FastLSTM<>>();
+}
+
+/**
+ * Ensure GRUs work with larger batch sizes.
+ */
+BOOST_AUTO_TEST_CASE(GRUBatchSizeTest)
+{
+  BatchSizeTest<GRU<>>();
+}
+
 /**
  * Make sure the RNN can be properly serialized.
  */

src/mlpack/tests/spalera_sgd_test.cpp

@@ -79,10 +79,10 @@ BOOST_AUTO_TEST_CASE(LogisticRegressionTest)
 
     // Ensure that the error is close to zero.
     const double acc = lr.ComputeAccuracy(data, responses);
-    BOOST_REQUIRE_CLOSE(acc, 100.0, 0.3); // 0.3% error tolerance.
+    BOOST_REQUIRE_CLOSE(acc, 100.0, 0.5); // 0.5% error tolerance.
 
     const double testAcc = lr.ComputeAccuracy(testData, testResponses);
-    BOOST_REQUIRE_CLOSE(testAcc, 100.0, 0.6); // 0.6% error tolerance.
+    BOOST_REQUIRE_CLOSE(testAcc, 100.0, 0.8); // 0.8% error tolerance.
   }
 }
 

src/mlpack/tests/svd_batch_test.cpp

@@ -111,7 +111,7 @@ BOOST_AUTO_TEST_CASE(SVDBatchMomentumTest)
 
   const double momentumRMSE = amf2.Apply(cleanedData, 2, m1, m2);
 
-  BOOST_REQUIRE_LE(momentumRMSE, regularRMSE + 0.05);
+  BOOST_REQUIRE_LE(momentumRMSE, regularRMSE + 0.08);
 }
 
 /**