Merge branch 'master' into bindings

.travis.yml

@@ -21,4 +21,7 @@ notifications:
   email:
     - mlpack-git@lists.mlpack.org
   irc:
-    - "chat.freenode.net#mlpack"
+    channels:
+      - "chat.freenode.net#mlpack"
+    on_success: change
+    on_failure: always

CMakeLists.txt

@@ -319,8 +319,8 @@ endif ()
 # some reason.
 include(CMake/TargetDistclean.cmake OPTIONAL)
 
-include_directories(${CMAKE_SOURCE_DIR})
-include_directories(${MLPACK_INCLUDE_DIRS})
+include_directories(BEFORE ${MLPACK_INCLUDE_DIRS})
+include_directories(BEFORE ${CMAKE_SOURCE_DIR}/src/)
 
 # On Windows, things end up under Debug/ or Release/.
 if (WIN32)

HISTORY.md

@@ -1,6 +1,10 @@
 ### mlpack ?.?.?
 ###### ????-??-??
 
+### mlpack 2.2.3
+###### 2017-05-24
+  * Bug fix for --predictions_file in mlpack_decision_tree program.
+
 ### mlpack 2.2.2
 ###### 2017-05-04
   * Install backwards-compatibility mlpack_allknn and mlpack_allkfn programs;

README.md

@@ -20,7 +20,7 @@
 <p align="center">
   <em>
     Download:
-    <a href="http://www.mlpack.org/files/mlpack-2.2.2.tar.gz">current stable version (2.2.2)</a>
+    <a href="http://www.mlpack.org/files/mlpack-2.2.3.tar.gz">current stable version (2.2.3)</a>
   </em>
 </p>
 

doc/guide/build.hpp

@@ -23,14 +23,14 @@ href="https://keon.io/mlpack/mlpack-on-windows/">Keon's excellent tutorial</a>.
 
 @section Download latest mlpack build
 Download latest mlpack build from here:
-<a href="http://www.mlpack.org/files/mlpack-2.2.2.tar.gz">mlpack-2.2.2</a>
+<a href="http://www.mlpack.org/files/mlpack-2.2.3.tar.gz">mlpack-2.2.3</a>
 
 @section builddir Creating Build Directory
 
 Once the mlpack source is unpacked, you should create a build directory.
 
 @code
-$ cd mlpack-2.2.2
+$ cd mlpack-2.2.3
 $ mkdir build
 @endcode
 

src/mlpack/CMakeLists.txt

@@ -1,4 +1,3 @@
-include_directories(..) # <mlpack/[whatever]>
 include_directories(${CMAKE_CURRENT_BINARY_DIR}/..) # mlpack/mlpack_export.hpp
 
 # Add core.hpp to list of sources.

src/mlpack/core.hpp

@@ -244,7 +244,8 @@
 #include <mlpack/core/dists/gaussian_distribution.hpp>
 #include <mlpack/core/dists/laplace_distribution.hpp>
 #include <mlpack/core/dists/gamma_distribution.hpp>
-//mlpack::backtrace only for linux
+
+// mlpack::backtrace only for linux
 #ifdef HAS_BFD_DL
   #include <mlpack/core/util/backtrace.hpp>
 #endif

src/mlpack/core/CMakeLists.txt

@@ -2,6 +2,7 @@
 set(DIRS
   arma_extend
   boost_backport
+  cv
   data
   dists
   kernels

src/mlpack/core/cv/CMakeLists.txt

@@ -0,0 +1,15 @@
+add_subdirectory(metrics)
+
+# Define the files we need to compile
+# Anything not in this list will not be compiled into mlpack.
+set(SOURCES
+)
+
+# Add directory name to sources.
+set(DIR_SRCS)
+foreach(file ${SOURCES})
+  set(DIR_SRCS ${DIR_SRCS} ${CMAKE_CURRENT_SOURCE_DIR}/${file})
+endforeach()
+# Append sources (with directory name) to list of all mlpack sources (used at
+# the parent scope).
+set(MLPACK_SRCS ${MLPACK_SRCS} ${DIR_SRCS} PARENT_SCOPE)

src/mlpack/core/cv/metrics/CMakeLists.txt

@@ -0,0 +1,17 @@
+# Define the files we need to compile
+# Anything not in this list will not be compiled into mlpack.
+set(SOURCES
+  accuracy.hpp
+  accuracy_impl.hpp
+  mse.hpp
+  mse_impl.hpp
+)
+
+# Add directory name to sources.
+set(DIR_SRCS)
+foreach(file ${SOURCES})
+  set(DIR_SRCS ${DIR_SRCS} ${CMAKE_CURRENT_SOURCE_DIR}/${file})
+endforeach()
+# Append sources (with directory name) to list of all mlpack sources (used at
+# the parent scope).
+set(MLPACK_SRCS ${MLPACK_SRCS} ${DIR_SRCS} PARENT_SCOPE)

src/mlpack/core/cv/metrics/accuracy.hpp

@@ -0,0 +1,53 @@
+/**
+ * @file accuracy.hpp
+ * @author Kirill Mishchenko
+ *
+ * The accuracy metric.
+ *
+ * mlpack is free software; you may redistribute it and/or modify it under the
+ * terms of the 3-clause BSD license.  You should have received a copy of the
+ * 3-clause BSD license along with mlpack.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef MLPACK_CORE_CV_METRICS_ACCURACY_HPP
+#define MLPACK_CORE_CV_METRICS_ACCURACY_HPP
+
+#include <mlpack/core.hpp>
+
+namespace mlpack {
+namespace cv {
+
+/**
+ * The Accuracy is a metric of performance for classification algorithms that is
+ * equal to a proportion of correctly labeled test items among all ones for
+ * given test items.
+ */
+class Accuracy
+{
+ public:
+  /**
+   * Run classification and calculate accuracy.
+   *
+   * @param model A classification model.
+   * @param data Column-major data containing test items.
+   * @param labels Ground truth (correct) labels for the test items.
+   */
+  template<typename MLAlgorithm, typename DataType>
+  static double Evaluate(MLAlgorithm& model,
+                         const DataType& data,
+                         const arma::Row<size_t>& labels);
+
+  /**
+   * Information for hyper-parameter tuning code. It indicates that we want
+   * to maximize the metric.
+   */
+  static const bool NeedsMinimization = false;
+};
+
+} // namespace cv
+} // namespace mlpack
+
+// Include implementation.
+#include "accuracy_impl.hpp"
+
+#endif

src/mlpack/core/cv/metrics/accuracy_impl.hpp

@@ -0,0 +1,42 @@
+/**
+ * @file accuracy_impl.hpp
+ * @author Kirill Mishchenko
+ *
+ * The implementation of the class Accuracy.
+ *
+ * mlpack is free software; you may redistribute it and/or modify it under the
+ * terms of the 3-clause BSD license.  You should have received a copy of the
+ * 3-clause BSD license along with mlpack.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef MLPACK_CORE_CV_METRICS_ACCURACY_IMPL_HPP
+#define MLPACK_CORE_CV_METRICS_ACCURACY_IMPL_HPP
+
+namespace mlpack {
+namespace cv {
+
+template<typename MLAlgorithm, typename DataType>
+double Accuracy::Evaluate(MLAlgorithm& model,
+                          const DataType& data,
+                          const arma::Row<size_t>& labels)
+{
+  if (data.n_cols != labels.n_elem)
+  {
+    std::ostringstream oss;
+    oss << "Accuracy::Evaluate(): number of points (" << data.n_cols << ") "
+        << "does not match number of labels (" << labels.n_elem << ")!"
+        << std::endl;
+    throw std::invalid_argument(oss.str());
+  }
+
+  arma::Row<size_t> predictedLabels;
+  model.Classify(data, predictedLabels);
+  size_t amountOfCorrectPredictions = arma::sum(predictedLabels == labels);
+
+  return (double) amountOfCorrectPredictions / labels.n_elem;
+}
+
+} // namespace cv
+} // namespace mlpack
+
+#endif

src/mlpack/core/cv/metrics/mse.hpp

@@ -0,0 +1,54 @@
+/**
+ * @file mse.hpp
+ * @author Kirill Mishchenko
+ *
+ * The mean squared error (MSE).
+ *
+ * mlpack is free software; you may redistribute it and/or modify it under the
+ * terms of the 3-clause BSD license.  You should have received a copy of the
+ * 3-clause BSD license along with mlpack.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef MLPACK_CORE_CV_METRICS_MSE_HPP
+#define MLPACK_CORE_CV_METRICS_MSE_HPP
+
+#include <mlpack/core.hpp>
+
+namespace mlpack {
+namespace cv {
+
+/**
+ * The MeanSquaredError is a metric of performance for regression algorithms
+ * that is equal to the mean squared error between predicted values and ground
+ * truth (correct) values for given test items.
+ */
+class MSE
+{
+ public:
+  /**
+   * Run prediction and calculate the mean squared error.
+   *
+   * @param model A regression model.
+   * @param data Column-major data containing test items.
+   * @param responses Ground truth (correct) target values for the test items,
+   *     should be either a row vector or a column-major matrix.
+   */
+  template<typename MLAlgorithm, typename DataType, typename ResponsesType>
+  static double Evaluate(MLAlgorithm& model,
+                         const DataType& data,
+                         const ResponsesType& responses);
+
+  /**
+   * Information for hyper-parameter tuning code. It indicates that we want
+   * to minimize the measurement.
+   */
+  static const bool NeedsMinimization = true;
+};
+
+} // namespace cv
+} // namespace mlpack
+
+// Include implementation.
+#include "mse_impl.hpp"
+
+#endif

src/mlpack/core/cv/metrics/mse_impl.hpp

@@ -0,0 +1,42 @@
+/**
+ * @file mse_impl.hpp
+ * @author Kirill Mishchenko
+ *
+ * The implementation of the class MSE.
+ *
+ * mlpack is free software; you may redistribute it and/or modify it under the
+ * terms of the 3-clause BSD license.  You should have received a copy of the
+ * 3-clause BSD license along with mlpack.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef MLPACK_CORE_CV_METRICS_MSE_IMPL_HPP
+#define MLPACK_CORE_CV_METRICS_MSE_IMPL_HPP
+
+namespace mlpack {
+namespace cv {
+
+template<typename MLAlgorithm, typename DataType, typename ResponsesType>
+double MSE::Evaluate(MLAlgorithm& model,
+                     const DataType& data,
+                     const ResponsesType& responses)
+{
+  if (data.n_cols != responses.n_cols)
+  {
+    std::ostringstream oss;
+    oss << "MSE::Evaluate(): number of points (" << data.n_cols << ") "
+        << "does not match number of responses (" << responses.n_cols << ")!"
+        << std::endl;
+    throw std::invalid_argument(oss.str());
+  }
+
+  ResponsesType predictedResponses;
+  model.Predict(data, predictedResponses);
+  double sum = arma::accu(arma::square(responses - predictedResponses));
+
+  return sum / responses.n_elem;
+}
+
+} // namespace cv
+} // namespace mlpack
+
+#endif

src/mlpack/core/data/CMakeLists.txt

@@ -30,6 +30,10 @@ foreach(file ${SOURCES})
     set(DIR_SRCS ${DIR_SRCS} ${CMAKE_CURRENT_SOURCE_DIR}/${file})
 endforeach()
 
+# Add subdirectories.
+add_subdirectory(imputation_methods)
+add_subdirectory(map_policies)
+
 # Append sources (with directory name) to list of all mlpack sources (used at
 # parent scope).
 set(MLPACK_SRCS ${MLPACK_SRCS} ${DIR_SRCS} PARENT_SCOPE)

src/mlpack/core/data/imputer.hpp

@@ -86,7 +86,6 @@ class Imputer
 
   // save columnMajor as a member variable since it is rarely changed.
   bool columnMajor;
-
 }; // class Imputer
 
 } // namespace data

src/mlpack/core/data/load_arff_impl.hpp

@@ -49,8 +49,8 @@ void LoadARFF(const std::string& filename,
     if (line[0] == '@')
     {
       typedef boost::tokenizer<boost::escaped_list_separator<char>> Tokenizer;
-      std::string separators = " \t\%"; // Split on comments too.
-      boost::escaped_list_separator<char> sep("\\", separators, "\"{");
+      std::string separators = " \t%"; // Split on comments too.
+      boost::escaped_list_separator<char> sep("\\", separators, "{\"");
       Tokenizer tok(line, sep);
       Tokenizer::iterator it = tok.begin();
 
@@ -182,7 +182,8 @@ void LoadARFF(const std::string& filename,
         // Strip spaces before mapping.
         std::string token = *it;
         boost::trim(token);
-        matrix(col, row) = info.template MapString<eT>(token, col); // We load transposed.
+        // We load transposed.
+        matrix(col, row) = info.template MapString<eT>(token, col);
       }
       else if (info.Type(col) == Datatype::numeric)
       {

src/mlpack/core/data/load_csv.hpp

@@ -204,7 +204,7 @@ class LoadCSV
     }
   }
 
-private:
+ private:
   using iter_type = boost::iterator_range<std::string::iterator>;
 
   /**
@@ -256,8 +256,8 @@ private:
       // Remove whitespace from either side.
       boost::trim(line);
 
-      //parse the numbers from a line(ex : 1,2,3,4), if the parser find the
-      //number it will execute the setNum function
+      // Parse the numbers from a line (ex: 1,2,3,4); if the parser finds a
+      // number it will execute the setNum function.
       const bool canParse = qi::parse(line.begin(), line.end(),
           stringRule[setCharClass] % delimiterRule);
 

src/mlpack/core/data/load_impl.hpp

@@ -55,13 +55,13 @@ void TransposeTokens(std::vector<std::vector<std::string>> const &input,
                      size_t index)
 {
   output.clear();
-  for(size_t i = 0; i != input.size(); ++i)
+  for (size_t i = 0; i != input.size(); ++i)
   {
     output.emplace_back(input[i][index]);
   }
 }
 
-} //namespace details
+} // namespace details
 
 template<typename eT>
 bool inline inplace_transpose(arma::Mat<eT>& X)
@@ -171,7 +171,7 @@ bool Load(const std::string& filename,
 
     // This is taken from load_auto_detect() in diskio_meat.hpp
     const std::string ARMA_MAT_TXT = "ARMA_MAT_TXT";
-    //char* rawHeader = new char[ARMA_MAT_TXT.length() + 1];
+    // char* rawHeader = new char[ARMA_MAT_TXT.length() + 1];
     std::string rawHeader(ARMA_MAT_TXT.length(), '\0');
     std::streampos pos = stream.tellg();
 

src/mlpack/core/data/serialization_template_version.hpp

@@ -29,14 +29,10 @@ struct version<mlpack::data::SecondShim<T>> \
   typedef mpl::int_<N> type; \
   typedef mpl::integral_c_tag tag; \
   BOOST_STATIC_CONSTANT(int, value = version::type::value); \
-  BOOST_MPL_ASSERT(( \
-      boost::mpl::less< \
-          boost::mpl::int_<N>, \
-          boost::mpl::int_<256> \
-      > \
-  )); \
+  BOOST_MPL_ASSERT((boost::mpl::less<boost::mpl::int_<N>, \
+                                     boost::mpl::int_<256>>)); \
 }; \
-} \
-}
+} /* namespace serialization */ \
+} /* namespace boost */
 
 #endif

src/mlpack/core/data/split_data.hpp

@@ -143,7 +143,7 @@ void Split(const arma::Mat<T>& input,
  * @return std::tuple containing trainData (arma::Mat<T>), testData
  *      (arma::Mat<T>), trainLabel (arma::Row<U>), and testLabel (arma::Row<U>).
  */
-template<typename T,typename U>
+template<typename T, typename U>
 std::tuple<arma::Mat<T>, arma::Mat<T>, arma::Row<U>, arma::Row<U>>
 Split(const arma::Mat<T>& input,
       const arma::Row<U>& inputLabel,

src/mlpack/core/dists/discrete_distribution.cpp

@@ -128,9 +128,9 @@ void DiscreteDistribution::Train(const arma::mat& observations,
   {
     for (size_t i = 0; i < dimensions; i++)
     {
-      // Add the probability of each observation.  The addition of 0.5 to the
-      // observation is to turn the default flooring operation of the size_t cast
-      // into a rounding observation.
+      // Add the probability of each observation. The addition of 0.5
+      // to the observation is to turn the default flooring operation
+      // of the size_t cast into a rounding observation.
       const size_t obs = size_t(observations(i, r) + 0.5);
 
       // Ensure that the observation is within the bounds.

src/mlpack/core/dists/discrete_distribution.hpp

@@ -129,11 +129,11 @@ class DiscreteDistribution
     // Ensure the observation has the same dimension with the probabilities
     if (observation.n_elem != probabilities.size())
     {
-      Log::Debug << "the obversation must has the same dimension with the probabilities"
-          << "the observation's dimension is" << observation.n_elem << "but the dimension of "
-          << "probabilities is" << probabilities.size() << std::endl;
-      return probability;
+      Log::Fatal << "DiscreteDistribution::Probability(): observation has "
+          << "incorrect dimension " << observation.n_elem << " but should have "
+          << "dimension " << probabilities.size() << "!" << std::endl;
     }
+
     for (size_t dimension = 0; dimension < observation.n_elem; dimension++)
     {
       // Adding 0.5 helps ensure that we cast the floating point to a size_t
@@ -143,9 +143,10 @@ class DiscreteDistribution
       // Ensure that the observation is within the bounds.
       if (obs >= probabilities[dimension].n_elem)
       {
-        Log::Debug << "DiscreteDistribution::Probability(): received observation "
-             << obs << "; observation must be in [0, " << probabilities[dimension].n_elem
-             << "] for this distribution." << std::endl;
+        Log::Fatal << "DiscreteDistribution::Probability(): received "
+            << "observation " << obs << "; observation must be in [0, "
+            << probabilities[dimension].n_elem << "] for this distribution."
+            << std::endl;
       }
       probability *= probabilities[dimension][obs];
     }

src/mlpack/core/dists/gamma_distribution.cpp

@@ -64,7 +64,7 @@ void GammaDistribution::Train(const arma::mat& rdata, const double tol)
   Train(logMeanxVec, meanLogxVec, meanxVec, tol);
 }
 
-//Fits an alpha and beta parameter according to observation probabilities.
+// Fits an alpha and beta parameter according to observation probabilities.
 void GammaDistribution::Train(const arma::mat& rdata,
                               const arma::vec& probabilities,
                               const double tol)
@@ -151,7 +151,6 @@ void GammaDistribution::Train(const arma::vec& logMeanxVec,
       if (aEst <= 0)
         throw std::logic_error("GammaDistribution::Train(): estimated invalid "
             "negative value for parameter alpha!");
-
     } while (!Converged(aEst, aOld, tol));
 
     alpha(row) = aEst;
@@ -219,7 +218,7 @@ void GammaDistribution::LogProbability(const arma::mat& observations,
       double factor = std::exp(-observations(d, i) / beta(d));
       double numerator = std::pow(observations(d, i), alpha(d) - 1);
 
-      LogProbabilities(i) += std::log( numerator * factor / denominators(d));
+      LogProbabilities(i) += std::log(numerator * factor / denominators(d));
     }
   }
 }

src/mlpack/core/dists/gamma_distribution.hpp

@@ -50,7 +50,7 @@ namespace distribution {
  */
 class GammaDistribution
 {
-  public:
+ public:
     /**
      * Construct the Gamma distribution with the given number of dimensions
      * (default 0); each parameter will be initialized to 0.
@@ -80,7 +80,7 @@ class GammaDistribution
     /**
      * Destructor.
      */
-    ~GammaDistribution() {};
+    ~GammaDistribution() {}
 
     /**
      * This function trains (fits distribution parameters) to new data or the
@@ -192,7 +192,7 @@ class GammaDistribution
     //! Get the dimensionality of the distribution.
     size_t Dimensionality() const { return alpha.n_elem; }
 
-  private:
+ private:
     //! Array of fitted alphas.
     arma::vec alpha;
     //! Array of fitted betas.
@@ -214,7 +214,7 @@ class GammaDistribution
                           const double tol);
 };
 
-} // namespace distributions.
-} // namespace mlpack.
+} // namespace distribution
+} // namespace mlpack
 
 #endif

src/mlpack/core/dists/gaussian_distribution.hpp

@@ -174,8 +174,9 @@ class GaussianDistribution
 * @param x List of observations.
 * @param probabilities Output log probabilities for each input observation.
 */
-inline void GaussianDistribution::LogProbability(const arma::mat& x,
-                                                 arma::vec& logProbabilities) const
+inline void GaussianDistribution::LogProbability(
+    const arma::mat& x,
+    arma::vec& logProbabilities) const
 {
   // Column i of 'diffs' is the difference between x.col(i) and the mean.
   arma::mat diffs = x - (mean * arma::ones<arma::rowvec>(x.n_cols));

src/mlpack/core/dists/laplace_distribution.cpp

@@ -21,7 +21,8 @@ using namespace mlpack::distribution;
  */
 double LaplaceDistribution::LogProbability(const arma::vec& observation) const
 {
-  // Evaluate the PDF of the Laplace distribution to determine the log probability.
+  // Evaluate the PDF of the Laplace distribution to determine
+  // the log probability.
   return -log(2. * scale) - arma::norm(observation - mean, 2) / scale;
 }
 

src/mlpack/core/dists/laplace_distribution.hpp

@@ -155,7 +155,6 @@ class LaplaceDistribution
   arma::vec mean;
   //! Scale parameter of the distribution.
   double scale;
-
 };
 
 } // namespace distribution

src/mlpack/core/dists/regression_distribution.cpp

@@ -23,11 +23,11 @@ using namespace mlpack::distribution;
 void RegressionDistribution::Train(const arma::mat& observations)
 {
   regression::LinearRegression lr(observations.rows(1, observations.n_rows - 1),
-      (observations.row(0)).t(), 0, true);
+      arma::rowvec(observations.row(0)), 0, true);
   rf = lr;
-  arma::vec fitted;
+  arma::rowvec fitted;
   lr.Predict(observations.rows(1, observations.n_rows - 1), fitted);
-  err.Train(observations.row(0) - fitted.t());
+  err.Train(observations.row(0) - fitted);
 }
 
 /**
@@ -37,13 +37,19 @@ void RegressionDistribution::Train(const arma::mat& observations)
  */
 void RegressionDistribution::Train(const arma::mat& observations,
                                    const arma::vec& weights)
+{
+  Train(observations, arma::rowvec(weights.t()));
+}
+
+void RegressionDistribution::Train(const arma::mat& observations,
+                                   const arma::rowvec& weights)
 {
   regression::LinearRegression lr(observations.rows(1, observations.n_rows - 1),
-      (observations.row(0)).t(), 0, true, weights);
+      arma::rowvec(observations.row(0)), weights, 0, true);
   rf = lr;
-  arma::vec fitted;
+  arma::rowvec fitted;
   lr.Predict(observations.rows(1, observations.n_rows - 1), fitted);
-  err.Train(observations.row(0) - fitted.t(), weights);
+  err.Train(observations.row(0) - fitted, weights.t());
 }
 
 /**
@@ -53,13 +59,21 @@ void RegressionDistribution::Train(const arma::mat& observations,
  */
 double RegressionDistribution::Probability(const arma::vec& observation) const
 {
-  arma::vec fitted;
+  arma::rowvec fitted;
   rf.Predict(observation.rows(1, observation.n_rows-1), fitted);
-  return err.Probability(observation(0)-fitted);
+  return err.Probability(observation(0)-fitted.t());
 }
 
 void RegressionDistribution::Predict(const arma::mat& points,
                                      arma::vec& predictions) const
+{
+  arma::rowvec rowPredictions;
+  Predict(points, rowPredictions);
+  predictions = rowPredictions.t();
+}
+
+void RegressionDistribution::Predict(const arma::mat& points,
+                                     arma::rowvec& predictions) const
 {
   rf.Predict(points, predictions);
 }

src/mlpack/core/dists/regression_distribution.hpp

@@ -41,6 +41,18 @@ class RegressionDistribution
    */
   RegressionDistribution() { /* nothing to do */ }
 
+  /**
+   * Create a Conditional Gaussian distribution with conditional mean function
+   * obtained by running RegressionFunction on predictors, responses.
+   *
+   * @param predictors Matrix of predictors (X).
+   * @param responses Vector of responses (y).
+   */
+  mlpack_deprecated RegressionDistribution(const arma::mat& predictors,
+                                           const arma::vec& responses) :
+    RegressionDistribution(predictors, arma::rowvec(responses.t()))
+  {}
+
   /**
    * Create a Conditional Gaussian distribution with conditional mean function
    * obtained by running RegressionFunction on predictors, responses.
@@ -49,9 +61,9 @@ class RegressionDistribution
    * @param responses Vector of responses (y).
    */
   RegressionDistribution(const arma::mat& predictors,
-                         const arma::vec& responses) :
-      rf(regression::LinearRegression(predictors, responses))
+                         const arma::rowvec& responses)
   {
+    rf.Train(predictors, responses);
     err = GaussianDistribution(1);
     arma::mat cov(1, 1);
     cov(0, 0) = rf.ComputeError(predictors, responses);
@@ -90,7 +102,15 @@ class RegressionDistribution
    *
    * @param weights probability that given observation is from distribution
    */
-  void Train(const arma::mat& observations, const arma::vec& weights);
+  mlpack_deprecated void Train(const arma::mat& observations,
+                               const arma::vec& weights);
+
+  /**
+   * Estimate parameters using provided observation weights
+   *
+   * @param weights probability that given observation is from distribution
+   */
+  void Train(const arma::mat& observations, const arma::rowvec& weights);
 
   /**
   * Evaluate probability density function of given observation
@@ -114,7 +134,16 @@ class RegressionDistribution
    * @param points the data points to calculate with.
    * @param predictions y, will contain calculated values on completion.
    */
-  void Predict(const arma::mat& points, arma::vec& predictions) const;
+  mlpack_deprecated void Predict(const arma::mat& points,
+                                 arma::vec& predictions) const;
+
+  /**
+   * Calculate y_i for each data point in points.
+   *
+   * @param points the data points to calculate with.
+   * @param predictions y, will contain calculated values on completion.
+   */
+  void Predict(const arma::mat& points, arma::rowvec& predictions) const;
 
   //! Return the parameters (the b vector).
   const arma::vec& Parameters() const { return rf.Parameters(); }

src/mlpack/core/kernels/epanechnikov_kernel.hpp

@@ -100,7 +100,6 @@ class EpanechnikovKernel
   double bandwidth;
   //! Cached value of the inverse bandwidth squared (to speed up computation).
   double inverseBandwidthSquared;
-
 };
 
 //! Kernel traits for the Epanechnikov kernel.

src/mlpack/core/kernels/example_kernel.hpp

@@ -140,7 +140,7 @@ class ExampleKernel
   static double Normalizer() { return 0; }
 
   // Modified to remove unused variable "dimension"
-  //static double Normalizer(size_t dimension=1) { return 0; }
+  // static double Normalizer(size_t dimension=1) { return 0; }
 };
 
 } // namespace kernel

src/mlpack/core/kernels/gaussian_kernel.hpp

@@ -126,8 +126,8 @@ class GaussianKernel
   template<typename VecTypeA, typename VecTypeB>
   double ConvolutionIntegral(const VecTypeA& a, const VecTypeB& b)
   {
-    return Evaluate(sqrt(metric::SquaredEuclideanDistance::Evaluate(a, b) / 2.0)) /
-        (Normalizer(a.n_rows) * pow(2.0, (double) a.n_rows / 2.0));
+    return Evaluate(sqrt(metric::SquaredEuclideanDistance::Evaluate(a, b) /
+        2.0)) / (Normalizer(a.n_rows) * pow(2.0, (double) a.n_rows / 2.0));
   }
 
 

src/mlpack/core/kernels/spherical_kernel.hpp

@@ -68,7 +68,7 @@ class SphericalKernel
     }
     double volumeSquared = pow(Normalizer(a.n_rows), 2.0);
 
-    switch(a.n_rows)
+    switch (a.n_rows)
     {
       case 1:
         return 1.0 / volumeSquared * (2.0 * bandwidth - distance);

src/mlpack/core/math/lin_alg.cpp

@@ -257,7 +257,9 @@ void mlpack::math::Svec(const arma::sp_mat& input, arma::sp_vec& output)
 
 void mlpack::math::Smat(const arma::vec& input, arma::mat& output)
 {
-  const size_t n = static_cast<size_t>(ceil((-1. + sqrt(1. + 8. * input.n_elem))/2.));
+  const size_t n = static_cast<size_t>
+      (ceil((-1. + sqrt(1. + 8. * input.n_elem))/2.));
+
 
   output.zeros(n, n);
 

src/mlpack/core/math/random_basis.cpp

@@ -18,7 +18,7 @@ namespace math {
 
 void RandomBasis(mat& basis, const size_t d)
 {
-  while(true)
+  while (true)
   {
     // [Q, R] = qr(randn(d, d));
     // Q = Q * diag(sign(diag(R)));

src/mlpack/core/metrics/mahalanobis_distance.hpp

@@ -111,7 +111,7 @@ class MahalanobisDistance
   arma::mat covariance;
 };
 
-} // namespace distance
+} // namespace metric
 } // namespace mlpack
 
 #include "mahalanobis_distance_impl.hpp"

src/mlpack/core/optimizers/aug_lagrangian/aug_lagrangian_test_functions.hpp

@@ -67,7 +67,7 @@ class GockenbachFunction
   double Evaluate(const arma::mat& coordinates);
   void Gradient(const arma::mat& coordinates, arma::mat& gradient);
 
-  size_t NumConstraints() const { return 2; };
+  size_t NumConstraints() const { return 2; }
 
   double EvaluateConstraint(const size_t index, const arma::mat& coordinates);
   void GradientConstraint(const size_t index,

src/mlpack/core/optimizers/lbfgs/lbfgs_impl.hpp

@@ -386,7 +386,8 @@ double L_BFGS<FunctionType>::Optimize(arma::mat& iterate)
         function.Evaluate(iterate) << ", gradient norm " <<
         arma::norm(gradient, 2) << ", " <<
         ((prevFunctionValue - functionValue) /
-         std::max(std::max(fabs(prevFunctionValue), fabs(functionValue)), 1.0)) << "." << std::endl;
+         std::max(std::max(fabs(prevFunctionValue), fabs(functionValue)), 1.0))
+        << "." << std::endl;
 
     prevFunctionValue = functionValue;
 
@@ -452,7 +453,6 @@ double L_BFGS<FunctionType>::Optimize(arma::mat& iterate)
 
     // Overwrite an old basis set.
     UpdateBasisSet(itNum, iterate, oldIterate, gradient, oldGradient);
-
   } // End of the optimization loop.
 
   return function.Evaluate(iterate);

src/mlpack/core/optimizers/minibatch_sgd/decay_policies/CMakeLists.txt

@@ -0,0 +1,10 @@
+set(SOURCES
+  no_decay.hpp
+)
+
+set(DIR_SRCS)
+foreach(file ${SOURCES})
+  set(DIR_SRCS ${DIR_SRCS} ${CMAKE_CURRENT_SOURCE_DIR}/${file})
+endforeach()
+
+set(MLPACK_SRCS ${MLPACK_SRCS} ${DIR_SRCS} PARENT_SCOPE)

src/mlpack/core/optimizers/minibatch_sgd/decay_policies/no_decay.hpp

@@ -0,0 +1,50 @@
+/**
+ * @file no_decay.hpp
+ * @author Marcus Edel
+ *
+ * Definition of the policy type for the decay class.
+ *
+ * You should define your own decay update that looks like NoDecay.
+ *
+ * mlpack is free software; you may redistribute it and/or modify it under the
+ * terms of the 3-clause BSD license.  You should have received a copy of the
+ * 3-clause BSD license along with mlpack.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+
+#ifndef MLPACK_CORE_OPTIMIZERS_MINIBATCH_SGD_DECAY_POLICIES_NO_DECAY_HPP
+#define MLPACK_CORE_OPTIMIZERS_MINIBATCH_SGD_DECAY_POLICIES_NO_DECAY_HPP
+
+namespace mlpack {
+namespace optimization {
+
+/**
+ * Definition of the NoDecay class. Use this as a template for your own.
+ */
+class NoDecay
+{
+ public:
+  /**
+   * This constructor is called before the first iteration.
+   */
+  NoDecay() { }
+
+  /**
+   * This function is called in each iteration after the policy update.
+   *
+  * @param iterate Parameters that minimize the function.
+  * @param stepSize Step size to be used for the given iteration.
+  * @param gradient The gradient matrix.
+  */
+  void Update(arma::mat& /* iterate */,
+              double& /* stepSize */,
+              const arma::mat& /* gradient */)
+  {
+    // Nothing to do here.
+  }
+};
+
+} // namespace optimization
+} // namespace mlpack
+
+#endif // MLPACK_CORE_OPTIMIZERS_MINIBATCH_SGD_DECAY_POLICIES_NO_DECAY_HPP

src/mlpack/core/optimizers/minibatch_sgd/minibatch_sgd.hpp

@@ -13,6 +13,8 @@
 #define MLPACK_CORE_OPTIMIZERS_MINIBATCH_SGD_MINIBATCH_SGD_HPP
 
 #include <mlpack/prereqs.hpp>
+#include <mlpack/core/optimizers/sgd/update_policies/vanilla_update.hpp>
+#include <mlpack/core/optimizers/minibatch_sgd/decay_policies/no_decay.hpp>
 
 namespace mlpack {
 namespace optimization {
@@ -69,9 +71,22 @@ namespace optimization {
  *
  * @tparam DecomposableFunctionType Decomposable objective function type to be
  *     minimized.
+ * @tparam update Update policy used during the iterative update process.
+ *     By default the vanilla update policy
+ *     (see mlpack::optimization::VanillaUpdate) is used.
+ * @tparam UpdatePolicyType Update policy used during the iterative update
+ *     process. By default the vanilla update policy
+ *     (see mlpack::optimization::VanillaUpdate) is used.
+ * @tparam DecayPolicyType Decay policy used during the iterative update
+ *     process to adjust the step size. By default the step size isn't going to
+ *     be adjusted.
  */
-template<typename DecomposableFunctionType>
-class MiniBatchSGD
+template<
+    typename DecomposableFunctionType,
+    typename UpdatePolicyType = VanillaUpdate,
+    typename DecayPolicyType = NoDecay
+>
+class MiniBatchSGDType
 {
  public:
   /**
@@ -89,13 +104,18 @@ class MiniBatchSGD
    * @param tolerance Maximum absolute tolerance to terminate algorithm.
    * @param shuffle If true, the mini-batch order is shuffled; otherwise, each
    *     mini-batch is visited in linear order.
+   * @param updatePolicy Instantiated update policy used to adjust the given
+   *     parameters.
+   * @param decayPolicy Instantiated decay policy used to adjust the step size.
    */
-  MiniBatchSGD(DecomposableFunctionType& function,
-               const size_t batchSize = 1000,
-               const double stepSize = 0.01,
-               const size_t maxIterations = 100000,
-               const double tolerance = 1e-5,
-               const bool shuffle = true);
+  MiniBatchSGDType(DecomposableFunctionType& function,
+                   const size_t batchSize = 1000,
+                   const double stepSize = 0.01,
+                   const size_t maxIterations = 100000,
+                   const double tolerance = 1e-5,
+                   const bool shuffle = true,
+                   const UpdatePolicyType& updatePolicy = UpdatePolicyType(),
+                   const DecayPolicyType& decayPolicy = DecayPolicyType());
 
   /**
    * Optimize the given function using mini-batch SGD.  The given starting point
@@ -137,6 +157,16 @@ class MiniBatchSGD
   //! Modify whether or not the individual functions are shuffled.
   bool& Shuffle() { return shuffle; }
 
+  //! Get the update policy.
+  UpdatePolicyType UpdatePolicy() const { return updatePolicy; }
+  //! Modify the update policy.
+  UpdatePolicyType& UpdatePolicy() { return updatePolicy; }
+
+  //! Get the decay policy.
+  DecayPolicyType DecayPolicy() const { return decayPolicy; }
+  //! Modify the decay policy.
+  DecayPolicyType& DecayPolicy() { return decayPolicy; }
+
  private:
   //! The instantiated function.
   DecomposableFunctionType& function;
@@ -156,8 +186,18 @@ class MiniBatchSGD
   //! Controls whether or not the individual functions are shuffled when
   //! iterating.
   bool shuffle;
+
+  //! The update policy used to update the parameters in each iteration.
+  UpdatePolicyType updatePolicy;
+
+  //! The decay policy used to update the parameters in each iteration.
+  DecayPolicyType decayPolicy;
 };
 
+template<typename DecomposableFunctionType>
+using MiniBatchSGD = MiniBatchSGDType<
+    DecomposableFunctionType, VanillaUpdate, NoDecay>;
+
 } // namespace optimization
 } // namespace mlpack
 

src/mlpack/core/optimizers/minibatch_sgd/minibatch_sgd_impl.hpp

@@ -18,25 +18,44 @@
 namespace mlpack {
 namespace optimization {
 
-template<typename DecomposableFunctionType>
-MiniBatchSGD<DecomposableFunctionType>::MiniBatchSGD(
-    DecomposableFunctionType& function,
-    const size_t batchSize,
-    const double stepSize,
-    const size_t maxIterations,
-    const double tolerance,
-    const bool shuffle) :
-    function(function),
-    batchSize(batchSize),
-    stepSize(stepSize),
-    maxIterations(maxIterations),
-    tolerance(tolerance),
-    shuffle(shuffle)
+template<
+    typename DecomposableFunctionType,
+    typename UpdatePolicyType,
+    typename DecayPolicyType
+>
+MiniBatchSGDType<
+    DecomposableFunctionType,
+    UpdatePolicyType,
+    DecayPolicyType
+>::MiniBatchSGDType(DecomposableFunctionType& function,
+                    const size_t batchSize,
+                    const double stepSize,
+                    const size_t maxIterations,
+                    const double tolerance,
+                    const bool shuffle,
+                    const UpdatePolicyType& updatePolicy,
+                    const DecayPolicyType& decayPolicy) :
+                    function(function),
+                    batchSize(batchSize),
+                    stepSize(stepSize),
+                    maxIterations(maxIterations),
+                    tolerance(tolerance),
+                    shuffle(shuffle),
+                    updatePolicy(updatePolicy),
+                    decayPolicy(decayPolicy)
 { /* Nothing to do. */ }
 
 //! Optimize the function (minimize).
-template<typename DecomposableFunctionType>
-double MiniBatchSGD<DecomposableFunctionType>::Optimize(arma::mat& iterate)
+template<
+    typename DecomposableFunctionType,
+    typename UpdatePolicyType,
+    typename DecayPolicyType
+>
+double MiniBatchSGDType<
+    DecomposableFunctionType,
+    UpdatePolicyType,
+    DecayPolicyType
+>::Optimize(arma::mat& iterate)
 {
   // Find the number of functions.
   const size_t numFunctions = function.NumFunctions();
@@ -60,6 +79,9 @@ double MiniBatchSGD<DecomposableFunctionType>::Optimize(arma::mat& iterate)
   for (size_t i = 0; i < numFunctions; ++i)
     overallObjective += function.Evaluate(iterate, i);
 
+  // Initialize the update policy.
+  updatePolicy.Initialize(iterate.n_rows, iterate.n_cols);
+
   // Now iterate!
   arma::mat gradient(iterate.n_rows, iterate.n_cols);
   for (size_t i = 1; i != maxIterations; ++i, ++currentBatch)
@@ -108,7 +130,7 @@ double MiniBatchSGD<DecomposableFunctionType>::Optimize(arma::mat& iterate)
       }
 
       // Now update the iterate.
-      iterate -= (stepSize / batchSize) * gradient;
+      updatePolicy.Update(iterate, stepSize / batchSize, gradient);
 
       // Add that to the overall objective function.
       for (size_t j = 0; j < batchSize; ++j)
@@ -130,18 +152,21 @@ double MiniBatchSGD<DecomposableFunctionType>::Optimize(arma::mat& iterate)
       if (lastBatchSize > 0)
       {
         // Now update the iterate.
-        iterate -= (stepSize / lastBatchSize) * gradient;
+        updatePolicy.Update(iterate, stepSize / lastBatchSize, gradient);
       }
       else
       {
         // Now update the iterate.
-        iterate -= stepSize * gradient;
+        updatePolicy.Update(iterate, stepSize, gradient);
       }
 
       // Add that to the overall objective function.
       for (size_t j = 0; j < lastBatchSize; ++j)
         overallObjective += function.Evaluate(iterate, offset + j);
     }
+
+    // Now update the learning rate if requested by the user.
+    decayPolicy.Update(iterate, stepSize, gradient);
   }
 
   Log::Info << "Mini-batch SGD: maximum iterations (" << maxIterations << ") "

src/mlpack/core/optimizers/rmsprop/rmsprop_update.hpp

@@ -114,4 +114,4 @@ class RMSPropUpdate
 } // namespace optimization
 } // namespace mlpack
 
-#endif
+#endif

src/mlpack/core/optimizers/sdp/lrsdp_function.hpp

@@ -27,7 +27,6 @@ template <typename SDPType>
 class LRSDPFunction
 {
  public:
-
   /**
    * Construct the LRSDPFunction from the given SDP.
    *
@@ -90,7 +89,6 @@ class LRSDPFunction
   SDPType& SDP() { return sdp; }
 
  private:
-
   //! SDP object representing the problem
   SDPType sdp;
 

src/mlpack/core/optimizers/sdp/lrsdp_function_impl.hpp

@@ -55,13 +55,14 @@ template <typename SDPType>
 void LRSDPFunction<SDPType>::Gradient(const arma::mat& /* coordinates */,
                                       arma::mat& /* gradient */) const
 {
-  Log::Fatal << "LRSDPFunction::Gradient() not implemented for arbitrary optimizers!"
-      << std::endl;
+  Log::Fatal << "LRSDPFunction::Gradient() not implemented for arbitrary "
+      << "optimizers!" << std::endl;
 }
 
 template <typename SDPType>
-double LRSDPFunction<SDPType>::EvaluateConstraint(const size_t index,
-                                                  const arma::mat& coordinates) const
+double LRSDPFunction<SDPType>::EvaluateConstraint(
+    const size_t index,
+    const arma::mat& coordinates) const
 {
   const arma::mat rrt = coordinates * trans(coordinates);
   if (index < SDP().NumSparseConstraints())
@@ -71,12 +72,13 @@ double LRSDPFunction<SDPType>::EvaluateConstraint(const size_t index,
 }
 
 template <typename SDPType>
-void LRSDPFunction<SDPType>::GradientConstraint(const size_t /* index */,
-                                                const arma::mat& /* coordinates */,
-                                                arma::mat& /* gradient */) const
+void LRSDPFunction<SDPType>::GradientConstraint(
+    const size_t /* index */,
+    const arma::mat& /* coordinates */,
+    arma::mat& /* gradient */) const
 {
-  Log::Fatal << "LRSDPFunction::GradientConstraint() not implemented for arbitrary "
-      << "optimizers!" << std::endl;
+  Log::Fatal << "LRSDPFunction::GradientConstraint() not implemented "
+      << "for arbitrary optimizers!" << std::endl;
 }
 
 //! Utility function for calculating part of the objective when AugLagrangian is
@@ -144,10 +146,11 @@ EvaluateImpl(const LRSDPFunction<SDPType>& function,
   double objective = accu(function.SDP().C() % rrt);
 
   // Now each constraint.
-  UpdateObjective(objective, rrt, function.SDP().SparseA(), function.SDP().SparseB(),
-      lambda, 0, sigma);
-  UpdateObjective(objective, rrt, function.SDP().DenseA(), function.SDP().DenseB(), lambda,
-      function.SDP().NumSparseConstraints(), sigma);
+  UpdateObjective(objective, rrt, function.SDP().SparseA(),
+      function.SDP().SparseB(), lambda, 0, sigma);
+  UpdateObjective(objective, rrt, function.SDP().DenseA(),
+      function.SDP().DenseB(), lambda, function.SDP().NumSparseConstraints(),
+      sigma);
 
   return objective;
 }

src/mlpack/core/optimizers/sdp/primal_dual.hpp

@@ -109,7 +109,7 @@ class PrimalDualSolver
   arma::vec initialYdense;
 
   //! Starting point for Z, the complementary slack variable. Needs to be
-  //positive definite.
+  //! positive definite.
   arma::mat initialZ;
 
   //! The step size modulating factor. Needs to be a scalar in (0, 1).

src/mlpack/core/optimizers/sdp/primal_dual_impl.hpp

@@ -46,9 +46,7 @@ PrimalDualSolver<SDPType>::PrimalDualSolver(const SDPType& sdp)
     primalInfeasTol(1e-7),
     dualInfeasTol(1e-7),
     maxIterations(1000)
-{
-
-}
+{ /* Nothing to do. */ }
 
 template <typename SDPType>
 PrimalDualSolver<SDPType>::PrimalDualSolver(const SDPType& sdp,

src/mlpack/core/optimizers/sdp/sdp.hpp

@@ -39,7 +39,6 @@ template <typename ObjectiveMatrixType>
 class SDP
 {
  public:
-
   typedef ObjectiveMatrixType objective_matrix_type;
 
   /**

src/mlpack/core/optimizers/sdp/sdp_impl.hpp

@@ -23,9 +23,7 @@ SDP<ObjectiveMatrixType>::SDP() :
     sparseB(),
     denseA(),
     denseB()
-{
-
-}
+{ /* Nothing to do. */ }
 
 template <typename ObjectiveMatrixType>
 SDP<ObjectiveMatrixType>::SDP(const size_t n,

src/mlpack/core/optimizers/sgd/sgd_impl.hpp

@@ -65,7 +65,7 @@ double SGD<DecomposableFunctionType, UpdatePolicyType>::Optimize(
     overallObjective += function.Evaluate(iterate, i);
 
   // Initialize the update policy.
-  updatePolicy.Initialize(iterate.n_rows,iterate.n_cols);
+  updatePolicy.Initialize(iterate.n_rows, iterate.n_cols);
 
   // Now iterate!
   arma::mat gradient(iterate.n_rows, iterate.n_cols);

src/mlpack/core/optimizers/sgd/update_policies/momentum_update.hpp

@@ -84,7 +84,7 @@ class MomentumUpdate
   void Initialize(const size_t rows,
                   const size_t cols)
   {
-    //Initialize am empty velocity matrix.
+    // Initialize am empty velocity matrix.
     velocity = arma::zeros<arma::mat>(rows, cols);
   }
 

src/mlpack/core/optimizers/sgdr/CMakeLists.txt

@@ -0,0 +1,15 @@
+set(SOURCES
+  cyclical_decay.hpp
+  sgdr.hpp
+  sgdr_impl.hpp
+  snapshot_ensembles.hpp
+  snapshot_sgdr.hpp
+  snapshot_sgdr_impl.hpp
+)
+
+set(DIR_SRCS)
+foreach(file ${SOURCES})
+  set(DIR_SRCS ${DIR_SRCS} ${CMAKE_CURRENT_SOURCE_DIR}/${file})
+endforeach()
+
+set(MLPACK_SRCS ${MLPACK_SRCS} ${DIR_SRCS} PARENT_SCOPE)

src/mlpack/core/optimizers/sgdr/cyclical_decay.hpp

@@ -0,0 +1,142 @@
+/**
+ * @file cyclical_decay.hpp
+ * @author Marcus Edel
+ *
+ * Definition of the warm restart technique (SGDR) described in:
+ * "SGDR: Stochastic Gradient Descent with Warm Restarts" by
+ * I. Loshchilov et al.
+ *
+ * mlpack is free software; you may redistribute it and/or modify it under the
+ * terms of the 3-clause BSD license.  You should have received a copy of the
+ * 3-clause BSD license along with mlpack.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+
+#ifndef MLPACK_CORE_OPTIMIZERS_SGDR_CYCLICAL_DECAY_HPP
+#define MLPACK_CORE_OPTIMIZERS_SGDR_CYCLICAL_DECAY_HPP
+
+namespace mlpack {
+namespace optimization {
+
+/**
+ * Simulate a new warm-started run/restart once a number of epochs are
+ * performed. Importantly, the restarts are not performed from scratch but
+ * emulated by increasing the step size while the old step size value of as an
+ * initial parameter.
+ *
+ * For more information, please refer to:
+ *
+ * @code
+ * @article{Loshchilov2016,
+ *   title   = {Learning representations by back-propagating errors},
+ *   author  = {Ilya Loshchilov and Frank Hutter},
+ *   title   = {{SGDR:} Stochastic Gradient Descent with Restarts},
+ *   journal = {CoRR},
+ *   year    = {2016},
+ *   url     = {https://arxiv.org/abs/1608.03983}
+ * }
+ * @endcode
+ */
+class CyclicalDecay
+{
+ public:
+  /**
+   * Construct the CyclicalDecay technique a restart method, where the
+   * step size decays after each batch and peridically resets to its initial
+   * value.
+   *
+   * @param epochRestart Initial epoch where decay is applied.
+   * @param multFactor Factor to increase the number of epochs before a restart.
+   * @param stepSize Initial step size for each restart.
+   * @param batchSize Size of each mini-batch.
+   * @param numFunctions The number of separable functions (the number of
+   *        predictor points).
+   */
+  CyclicalDecay(const size_t epochRestart,
+                const double multFactor,
+                const double stepSize,
+                const size_t batchSize,
+                const size_t numFunctions) :
+      epochRestart(epochRestart),
+      multFactor(multFactor),
+      constStepSize(stepSize),
+      nextRestart(epochRestart),
+      batchRestart(0),
+      epochBatches(numFunctions / (double) batchSize),
+      epoch(0)
+  { /* Nothing to do here */ }
+
+  /**
+   * This function is called in each iteration after the policy update.
+   *
+   * @param iterate Parameters that minimize the function.
+   * @param stepSize Step size to be used for the given iteration.
+   * @param gradient The gradient matrix.
+   */
+  void Update(arma::mat& /* iterate */,
+              double& stepSize,
+              const arma::mat& /* gradient */)
+  {
+    // Time to adjust the step size.
+    if (epoch >= epochRestart)
+    {
+      // n_t = n_min^i + 0.5(n_max^i - n_min^i)(1 + cos(T_cur/T_i * pi)).
+      stepSize = 0.5 * constStepSize * (1 + cos((batchRestart / epochBatches)
+          * M_PI));
+
+      // Keep track of the number of batches since the last restart.
+      batchRestart++;
+    }
+
+    // Time to restart.
+    if (epoch > nextRestart)
+    {
+      batchRestart = 0;
+
+      // Adjust the period of restarts.
+      epochRestart *= multFactor;
+
+      // Update the time for the next restart.
+      nextRestart += epochRestart;
+    }
+
+    epoch++;
+  }
+
+  //! Get the step size.
+  double StepSize() const { return constStepSize; }
+  //! Modify the step size.
+  double& StepSize() { return constStepSize; }
+
+  //! Get the restart fraction.
+  double EpochBatches() const { return epochBatches; }
+  //! Modify the restart fraction.
+  double& EpochBatches() { return epochBatches; }
+
+ private:
+  //! Epoch where decay is applied.
+  size_t epochRestart;
+
+  //! Parameter to increase the number of epochs before a restart.
+  double multFactor;
+
+  //! The step size for each example.
+  double constStepSize;
+
+  //! Locally-stored restart time.
+  size_t nextRestart;
+
+  //! Locally-stored number of batches since the last restart.
+  size_t batchRestart;
+
+  //! Locally-stored restart fraction.
+  double epochBatches;
+
+  //! Locally-stored epoch.
+  size_t epoch;
+};
+
+} // namespace optimization
+} // namespace mlpack
+
+#endif // MLPACK_CORE_OPTIMIZERS_SGDR_CYCLICAL_DECAY_HPP

src/mlpack/core/optimizers/sgdr/sgdr.hpp

@@ -0,0 +1,150 @@
+/**
+ * @file sgdr.hpp
+ * @author Marcus Edel
+ *
+ * Definition of the Stochastic Gradient Descent with Restarts (SGDR) as
+ * described in: "SGDR: Stochastic Gradient Descent with Warm Restarts" by
+ * I. Loshchilov et al.
+ *
+ * mlpack is free software; you may redistribute it and/or modify it under the
+ * terms of the 3-clause BSD license.  You should have received a copy of the
+ * 3-clause BSD license along with mlpack.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef MLPACK_CORE_OPTIMIZERS_SGDR_SGDR_HPP
+#define MLPACK_CORE_OPTIMIZERS_SGDR_SGDR_HPP
+
+#include <mlpack/prereqs.hpp>
+
+#include <mlpack/core/optimizers/minibatch_sgd/minibatch_sgd.hpp>
+#include <mlpack/core/optimizers/sgd/update_policies/momentum_update.hpp>
+#include "cyclical_decay.hpp"
+
+namespace mlpack {
+namespace optimization {
+
+/**
+ * This class is based on Mini-batch Stochastic Gradient Descent class and
+ * simulates a new warm-started run/restart once a number of epochs are
+ * performed.
+ *
+ * For more information, please refer to:
+ *
+ * @code
+ * @article{Loshchilov2016,
+ *   title   = {{SGDR:} Stochastic Gradient Descent with Restarts},
+ *   author  = {Ilya Loshchilov and Frank Hutter},
+ *   journal = {CoRR},
+ *   year    = {2016},
+ *   url     = {https://arxiv.org/abs/1608.03983}
+ * }
+ * @endcode
+ *
+ * @tparam DecomposableFunctionType Decomposable objective function type to be
+ *         minimized.
+ * @tparam UpdatePolicyType Update policy used during the iterative update
+ *         process. By default the momentum update policy
+ *         (see mlpack::optimization::MomentumUpdate) is used.
+ */
+template<
+    typename DecomposableFunctionType,
+    typename UpdatePolicyType = MomentumUpdate
+>
+class SGDR
+{
+ public:
+  //! Convenience typedef for the internal optimizer construction.
+  using OptimizerType = MiniBatchSGDType<
+      DecomposableFunctionType, UpdatePolicyType, CyclicalDecay>;
+
+  /**
+   * Construct the SGDR optimizer with the given function and
+   * parameters.  The defaults here are not necessarily good for the given
+   * problem, so it is suggested that the values used be tailored for the task
+   * at hand.  The maximum number of iterations refers to the maximum number of
+   * mini-batches that are processed.
+   *
+   * @param epochRestart Initial epoch where decay is applied.
+   * @param function Function to be optimized (minimized).
+   * @param batchSize Size of each mini-batch.
+   * @param stepSize Step size for each iteration.
+   * @param maxIterations Maximum number of iterations allowed (0 means no
+   *        limit).
+   * @param tolerance Maximum absolute tolerance to terminate algorithm.
+   * @param shuffle If true, the mini-batch order is shuffled; otherwise, each
+   *        mini-batch is visited in linear order.
+   * @param updatePolicy Instantiated update policy used to adjust the given
+   *        parameters.
+   */
+  SGDR(DecomposableFunctionType& function,
+       const size_t epochRestart = 50,
+       const double multFactor = 2.0,
+       const size_t batchSize = 1000,
+       const double stepSize = 0.01,
+       const size_t maxIterations = 100000,
+       const double tolerance = 1e-5,
+       const bool shuffle = true,
+       const UpdatePolicyType& updatePolicy = UpdatePolicyType());
+
+  /**
+   * Optimize the given function using SGDR.  The given starting point
+   * will be modified to store the finishing point of the algorithm, and the
+   * final objective value is returned.
+   *
+   * @param iterate Starting point (will be modified).
+   * @return Objective value of the final point.
+   */
+  double Optimize(arma::mat& iterate);
+
+  //! Get the instantiated function to be optimized.
+  const DecomposableFunctionType& Function() const
+  {
+    return optimizer.Function();
+  }
+
+  //! Modify the instantiated function.
+  DecomposableFunctionType& Function() { return optimizer.Function(); }
+
+  //! Get the batch size.
+  size_t BatchSize() const { return optimizer.BatchSize(); }
+  //! Modify the batch size.
+  size_t& BatchSize() { return optimizer.BatchSize(); }
+
+  //! Get the step size.
+  double StepSize() const { return optimizer.StepSize(); }
+  //! Modify the step size.
+  double& StepSize() { return optimizer.StepSize(); }
+
+  //! Get the maximum number of iterations (0 indicates no limit).
+  size_t MaxIterations() const { return optimizer.MaxIterations(); }
+  //! Modify the maximum number of iterations (0 indicates no limit).
+  size_t& MaxIterations() { return optimizer.MaxIterations(); }
+
+  //! Get the tolerance for termination.
+  double Tolerance() const { return optimizer.Tolerance(); }
+  //! Modify the tolerance for termination.
+  double& Tolerance() { return optimizer.Tolerance(); }
+
+  //! Get whether or not the individual functions are shuffled.
+  bool Shuffle() const { return optimizer.Shuffle(); }
+  //! Modify whether or not the individual functions are shuffled.
+  bool& Shuffle() { return optimizer.Shuffle(); }
+
+ private:
+  //! The instantiated function.
+  DecomposableFunctionType& function;
+
+  //! The size of each mini-batch.
+  size_t batchSize;
+
+  //! Locally-stored optimizer instance.
+  OptimizerType optimizer;
+};
+
+} // namespace optimization
+} // namespace mlpack
+
+// Include implementation.
+#include "sgdr_impl.hpp"
+
+#endif

src/mlpack/core/optimizers/sgdr/sgdr_impl.hpp

@@ -0,0 +1,77 @@
+/**
+ * @file sgdr_impl.hpp
+ * @author Marcus Edel
+ *
+ * Implementation of SGDR method.
+ *
+ * mlpack is free software; you may redistribute it and/or modify it under the
+ * terms of the 3-clause BSD license.  You should have received a copy of the
+ * 3-clause BSD license along with mlpack.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef MLPACK_CORE_OPTIMIZERS_SGDR_SGDR_IMPL_HPP
+#define MLPACK_CORE_OPTIMIZERS_SGDR_SGDR_IMPL_HPP
+
+// In case it hasn't been included yet.
+#include "sgdr.hpp"
+
+namespace mlpack {
+namespace optimization {
+
+template<typename DecomposableFunctionType, typename UpdatePolicyType>
+SGDR<DecomposableFunctionType, UpdatePolicyType>::SGDR(
+    DecomposableFunctionType& function,
+    const size_t epochRestart,
+    const double multFactor,
+    const size_t batchSize,
+    const double stepSize,
+    const size_t maxIterations,
+    const double tolerance,
+    const bool shuffle,
+    const UpdatePolicyType& updatePolicy) :
+    function(function),
+    batchSize(batchSize),
+    optimizer(OptimizerType(function,
+                            batchSize,
+                            stepSize,
+                            maxIterations,
+                            tolerance,
+                            shuffle,
+                            updatePolicy,
+                            CyclicalDecay(
+                                epochRestart,
+                                multFactor,
+                                stepSize,
+                                batchSize,
+                                function.NumFunctions())))
+{
+  /* Nothing to do here */
+}
+
+template<typename DecomposableFunctionType, typename UpdatePolicyType>
+double SGDR<DecomposableFunctionType, UpdatePolicyType>::Optimize(
+    arma::mat& iterate)
+{
+  // If a user changed the step size he hasn't update the step size of the
+  // cyclical decay instantiation, so we have to do it here.
+  if (optimizer.StepSize() != optimizer.DecayPolicy().StepSize())
+  {
+    optimizer.DecayPolicy().StepSize() = optimizer.StepSize();
+  }
+
+  // If a user changed the batch size we have to update the restart fraction
+  // of the cyclical decay instantiation.
+  if (optimizer.BatchSize() != batchSize)
+  {
+    batchSize = optimizer.BatchSize();
+    optimizer.DecayPolicy().EpochBatches() = function.NumFunctions() /
+        double(batchSize);
+  }
+
+  return optimizer.Optimize(iterate);
+}
+
+} // namespace optimization
+} // namespace mlpack
+
+#endif

src/mlpack/core/optimizers/sgdr/snapshot_ensembles.hpp

@@ -0,0 +1,179 @@
+/**
+ * @file snapshot_ensembles.hpp
+ * @author Marcus Edel
+ *
+ * Definition of the Snapshot ensembles technique described in:
+ * "Snapshot ensembles: Train 1, get m for free" by G. Huang et al.
+ *
+ * mlpack is free software; you may redistribute it and/or modify it under the
+ * terms of the 3-clause BSD license.  You should have received a copy of the
+ * 3-clause BSD license along with mlpack.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+
+#ifndef MLPACK_CORE_OPTIMIZERS_SGDR_SNAPSHOT_ENSEMBLES_HPP
+#define MLPACK_CORE_OPTIMIZERS_SGDR_SNAPSHOT_ENSEMBLES_HPP
+
+namespace mlpack {
+namespace optimization {
+
+/**
+ * Simulate a new warm-started run/restart once a number of epochs are
+ * performed. Importantly, the restarts are not performed from scratch but
+ * emulated by increasing the step size while the old step size value of as an
+ * initial parameter.
+ *
+ * For more information, please refer to:
+ *
+ * @code
+ * @inproceedings{Huang2017,
+ *   title     = {Snapshot ensembles: Train 1, get m for free},
+ *   author    = {Gao Huang, Yixuan Li, Geoff Pleiss, Zhuang Liu,
+ *                John E. Hopcroft, and Kilian Q. Weinberger},
+ *   booktitle = {Proceedings of the International Conference on Learning
+ *                Representations (ICLR)},
+ *   year      = {2017},
+ *   url       = {https://arxiv.org/abs/1704.00109}
+ * }
+ * @endcode
+ */
+class SnapshotEnsembles
+{
+ public:
+  /**
+   * Construct the CyclicalDecay technique a restart method, where the
+   * step size decays after each batch and peridically resets to its initial
+   * value.
+   *
+   * @param epochRestart Initial epoch where decay is applied.
+   * @param multFactor Factor to increase the number of epochs before a restart.
+   * @param stepSize Initial step size for each restart.
+   * @param batchSize Size of each mini-batch.
+   * @param numFunctions The number of separable functions (the number of
+   *        predictor points).
+   * @param maxIterations Maximum number of iterations allowed (0 means no
+   *        limit).
+   * @param snapshots Maximum number of snapshots.
+   */
+  SnapshotEnsembles(const size_t epochRestart,
+                    const double multFactor,
+                    const double stepSize,
+                    const size_t numFunctions,
+                    const size_t batchSize,
+                    const size_t maxIterations,
+                    const size_t snapshots) :
+    epochRestart(epochRestart),
+    multFactor(multFactor),
+    constStepSize(stepSize),
+    nextRestart(epochRestart),
+    batchRestart(0),
+    epochBatches(numFunctions / (double) batchSize),
+    epoch(0)
+  {
+    snapshotEpochs = 0;
+    for (size_t i = 0, er = epochRestart, nr = nextRestart;
+        i < maxIterations; ++i)
+    {
+      if (i > nr)
+      {
+        er *= multFactor;
+        nr += er;
+        snapshotEpochs++;
+      }
+    }
+
+    snapshotEpochs = epochRestart * std::pow(multFactor,
+        snapshotEpochs - snapshots + 1);
+  }
+
+  /**
+   * This function is called in each iteration after the policy update.
+   *
+   * @param iterate Parameters that minimize the function.
+   * @param stepSize Step size to be used for the given iteration.
+   * @param gradient The gradient matrix.
+   */
+  void Update(arma::mat& iterate,
+              double& stepSize,
+              const arma::mat& /* gradient */)
+  {
+    // Time to adjust the step size.
+    if (epoch >= epochRestart)
+    {
+      // n_t = n_min^i + 0.5(n_max^i - n_min^i)(1 + cos(T_cur/T_i * pi)).
+      stepSize = 0.5 * constStepSize * (1 + cos((batchRestart / epochBatches)
+          * M_PI));
+
+      // Keep track of the number of batches since the last restart.
+      batchRestart++;
+    }
+
+    // Time to restart.
+    if (epoch > nextRestart)
+    {
+      batchRestart = 0;
+
+      // Adjust the period of restarts.
+      epochRestart *= multFactor;
+
+      // Create a new snapshot.
+      if (epochRestart >= snapshotEpochs)
+      {
+        snapshots.push_back(iterate);
+      }
+
+      // Update the time for the next restart.
+      nextRestart += epochRestart;
+    }
+
+    epoch++;
+  }
+
+  //! Get the step size.
+  double StepSize() const { return constStepSize; }
+  //! Modify the step size.
+  double& StepSize() { return constStepSize; }
+
+  //! Get the restart fraction.
+  double EpochBatches() const { return epochBatches; }
+  //! Modify the restart fraction.
+  double& EpochBatches() { return epochBatches; }
+
+  //! Get the snapshots.
+  std::vector<arma::mat> Snapshots() const { return snapshots; }
+  //! Modify the snapshots.
+  std::vector<arma::mat>& Snapshots() { return snapshots; }
+
+ private:
+  //! Epoch where decay is applied.
+  size_t epochRestart;
+
+  //! Parameter to increase the number of epochs before a restart.
+  double multFactor;
+
+  //! The step size for each example.
+  double constStepSize;
+
+  //! Locally-stored restart time.
+  size_t nextRestart;
+
+  //! Locally-stored number of batches since the last restart.
+  size_t batchRestart;
+
+  //! Locally-stored restart fraction.
+  double epochBatches;
+
+  //! Locally-stored epoch.
+  size_t epoch;
+
+  //! Epochs where a new snapshot is created.
+  size_t snapshotEpochs;
+
+  //! Locally-stored parameter snapshots.
+  std::vector<arma::mat> snapshots;
+};
+
+} // namespace optimization
+} // namespace mlpack
+
+#endif // MLPACK_CORE_OPTIMIZERS_SGDR_CYCLICAL_DECAY_HPP

src/mlpack/core/optimizers/sgdr/snapshot_sgdr.hpp

@@ -0,0 +1,182 @@
+/**
+ * @file snapshots_sgdr.hpp
+ * @author Marcus Edel
+ *
+ * Definition of the Stochastic Gradient Descent with Restarts (SGDR) as
+ * described in: "SGDR: Stochastic Gradient Descent with Warm Restarts" by
+ * I. Loshchilov et al and the Snapshot ensembles technique described in:
+ * "Snapshot ensembles: Train 1, get m for free" by G. Huang et al.
+ *
+ * mlpack is free software; you may redistribute it and/or modify it under the
+ * terms of the 3-clause BSD license.  You should have received a copy of the
+ * 3-clause BSD license along with mlpack.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef MLPACK_CORE_OPTIMIZERS_SGDR_SNAPSHOT_SGDR_HPP
+#define MLPACK_CORE_OPTIMIZERS_SGDR_SNAPSHOT_SGDR_HPP
+
+#include <mlpack/prereqs.hpp>
+
+#include <mlpack/core/optimizers/minibatch_sgd/minibatch_sgd.hpp>
+#include <mlpack/core/optimizers/sgd/update_policies/momentum_update.hpp>
+#include "snapshot_ensembles.hpp"
+
+namespace mlpack {
+namespace optimization {
+
+/**
+ * This class is based on Mini-batch Stochastic Gradient Descent class and
+ * simulates a new warm-started run/restart once a number of epochs are
+ * performed using the Snapshot ensembles technique.
+ *
+ * For more information, please refer to:
+ *
+ * @code
+ * @article{Loshchilov2016,
+ *   title   = {{SGDR:} Stochastic Gradient Descent with Restarts},
+ *   author  = {Ilya Loshchilov and Frank Hutter},
+ *   journal = {CoRR},
+ *   year    = {2016},
+ *   url     = {https://arxiv.org/abs/1608.03983}
+ * }
+ * @endcode
+ *
+ * @code
+ * @inproceedings{Huang2017,
+ *   title     = {Snapshot ensembles: Train 1, get m for free},
+ *   author    = {Gao Huang, Yixuan Li, Geoff Pleiss, Zhuang Liu,
+ *                John E. Hopcroft, and Kilian Q. Weinberger},
+ *   booktitle = {Proceedings of the International Conference on Learning
+ *                Representations (ICLR)},
+ *   year      = {2017},
+ *   url       = {https://arxiv.org/abs/1704.00109}
+ * }
+ * @endcode
+ *
+ * @tparam DecomposableFunctionType Decomposable objective function type to be
+ *         minimized.
+ * @tparam UpdatePolicyType Update policy used during the iterative update
+ *         process. By default the momentum update policy
+ *         (see mlpack::optimization::MomentumUpdate) is used.
+ */
+template<
+    typename DecomposableFunctionType,
+    typename UpdatePolicyType = MomentumUpdate
+>
+class SnapshotSGDR
+{
+ public:
+  //! Convenience typedef for the internal optimizer construction.
+  using OptimizerType = MiniBatchSGDType<
+      DecomposableFunctionType, UpdatePolicyType, SnapshotEnsembles>;
+
+  /**
+   * Construct the SnapshotSGDR optimizer with snapshot ensembles with the given
+   * function and parameters. The defaults here are not necessarily good for
+   * the given problem, so it is suggested that the values used be tailored for
+   * the task at hand.  The maximum number of iterations refers to the maximum
+   * number of mini-batches that are processed.
+   *
+   * @param epochRestart Initial epoch where decay is applied.
+   * @param function Function to be optimized (minimized).
+   * @param batchSize Size of each mini-batch.
+   * @param stepSize Step size for each iteration.
+   * @param maxIterations Maximum number of iterations allowed (0 means no
+   *        limit).
+   * @param tolerance Maximum absolute tolerance to terminate algorithm.
+   * @param shuffle If true, the mini-batch order is shuffled; otherwise, each
+   *        mini-batch is visited in linear order.
+   * @param snapshots Maximum number of snapshots.
+   * @param accumulate Accumulate the snapshot parameter (default true).
+   * @param updatePolicy Instantiated update policy used to adjust the given
+   *        parameters.
+   */
+  SnapshotSGDR(DecomposableFunctionType& function,
+               const size_t epochRestart = 50,
+               const double multFactor = 2.0,
+               const size_t batchSize = 1000,
+               const double stepSize = 0.01,
+               const size_t maxIterations = 100000,
+               const double tolerance = 1e-5,
+               const bool shuffle = true,
+               const size_t snapshots = 5,
+               const bool accumulate = true,
+               const UpdatePolicyType& updatePolicy = UpdatePolicyType());
+
+  /**
+   * Optimize the given function using SGDR.  The given starting point
+   * will be modified to store the finishing point of the algorithm, and the
+   * final objective value is returned.
+   *
+   * @param iterate Starting point (will be modified).
+   * @param accumulate Accumulate the snapshot parameter (default true).
+   * @return Objective value of the final point.
+   */
+  double Optimize(arma::mat& iterate);
+
+  //! Get the instantiated function to be optimized.
+  const DecomposableFunctionType& Function() const
+  {
+    return optimizer.Function();
+  }
+
+  //! Modify the instantiated function.
+  DecomposableFunctionType& Function() { return optimizer.Function(); }
+
+  //! Get the batch size.
+  size_t BatchSize() const { return optimizer.BatchSize(); }
+  //! Modify the batch size.
+  size_t& BatchSize() { return optimizer.BatchSize(); }
+
+  //! Get the step size.
+  double StepSize() const { return optimizer.StepSize(); }
+  //! Modify the step size.
+  double& StepSize() { return optimizer.StepSize(); }
+
+  //! Get the maximum number of iterations (0 indicates no limit).
+  size_t MaxIterations() const { return optimizer.MaxIterations(); }
+  //! Modify the maximum number of iterations (0 indicates no limit).
+  size_t& MaxIterations() { return optimizer.MaxIterations(); }
+
+  //! Get the tolerance for termination.
+  double Tolerance() const { return optimizer.Tolerance(); }
+  //! Modify the tolerance for termination.
+  double& Tolerance() { return optimizer.Tolerance(); }
+
+  //! Get whether or not the individual functions are shuffled.
+  bool Shuffle() const { return optimizer.Shuffle(); }
+  //! Modify whether or not the individual functions are shuffled.
+  bool& Shuffle() { return optimizer.Shuffle(); }
+
+  //! Get the snapshots.
+  std::vector<arma::mat> Snapshots() const
+  {
+    return optimizer.DecayPolicy().Snapshots();
+  }
+  //! Modify the snapshots.
+  std::vector<arma::mat>& Snapshots()
+  {
+    return optimizer.DecayPolicy().Snapshots();
+  }
+
+ private:
+  //! The instantiated function.
+  DecomposableFunctionType& function;
+
+  //! The size of each mini-batch.
+  size_t batchSize;
+
+  //! Whether or not to accumulate the snapshots.
+  bool accumulate;
+
+  //! Locally-stored optimizer instance.
+  OptimizerType optimizer;
+};
+
+} // namespace optimization
+} // namespace mlpack
+
+// Include implementation.
+#include "snapshot_sgdr_impl.hpp"
+
+#endif

src/mlpack/core/optimizers/sgdr/snapshot_sgdr_impl.hpp

@@ -0,0 +1,99 @@
+/**
+ * @file snapshots_sgdr_impl.hpp
+ * @author Marcus Edel
+ *
+ * Implementation of SGDR method using snapshots ensembles.
+ *
+ * mlpack is free software; you may redistribute it and/or modify it under the
+ * terms of the 3-clause BSD license.  You should have received a copy of the
+ * 3-clause BSD license along with mlpack.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef MLPACK_CORE_OPTIMIZERS_SGDR_SNAPSHOT_SGDR_IMPL_HPP
+#define MLPACK_CORE_OPTIMIZERS_SGDR_SNAPSHOT_SGDR_IMPL_HPP
+
+// In case it hasn't been included yet.
+#include "snapshot_sgdr.hpp"
+
+namespace mlpack {
+namespace optimization {
+
+template<typename DecomposableFunctionType, typename UpdatePolicyType>
+SnapshotSGDR<DecomposableFunctionType, UpdatePolicyType>::SnapshotSGDR(
+    DecomposableFunctionType& function,
+    const size_t epochRestart,
+    const double multFactor,
+    const size_t batchSize,
+    const double stepSize,
+    const size_t maxIterations,
+    const double tolerance,
+    const bool shuffle,
+    const size_t snapshots,
+    const bool accumulate,
+    const UpdatePolicyType& updatePolicy) :
+    function(function),
+    batchSize(batchSize),
+    accumulate(accumulate),
+    optimizer(OptimizerType(function,
+                            batchSize,
+                            stepSize,
+                            maxIterations,
+                            tolerance,
+                            shuffle,
+                            updatePolicy,
+                            SnapshotEnsembles(
+                                epochRestart,
+                                multFactor,
+                                stepSize,
+                                batchSize,
+                                function.NumFunctions(),
+                                maxIterations,
+                                snapshots)))
+{
+  /* Nothing to do here */
+}
+
+template<typename DecomposableFunctionType, typename UpdatePolicyType>
+double SnapshotSGDR<DecomposableFunctionType, UpdatePolicyType>::Optimize(
+    arma::mat& iterate)
+{
+  // If a user changed the step size he hasn't update the step size of the
+  // cyclical decay instantiation, so we have to do here.
+  if (optimizer.StepSize() != optimizer.DecayPolicy().StepSize())
+  {
+    optimizer.DecayPolicy().StepSize() = optimizer.StepSize();
+  }
+
+  // If a user changed the batch size we have to update the restart fraction
+  // of the cyclical decay instantiation.
+  if (optimizer.BatchSize() != batchSize)
+  {
+    batchSize = optimizer.BatchSize();
+    optimizer.DecayPolicy().EpochBatches() = function.NumFunctions() /
+        double(batchSize);
+  }
+
+  double overallObjective = optimizer.Optimize(iterate);
+
+  // Accumulate snapshots.
+  if (accumulate)
+  {
+    for (size_t i = 0; i < optimizer.DecayPolicy().Snapshots().size(); ++i)
+    {
+      iterate += optimizer.DecayPolicy().Snapshots()[i];
+    }
+    iterate /= (optimizer.DecayPolicy().Snapshots().size() + 1);
+
+    // Calculate final objective.
+    overallObjective = 0;
+    for (size_t i = 0; i < function.NumFunctions(); ++i)
+      overallObjective += function.Evaluate(iterate, i);
+  }
+
+  return overallObjective;
+}
+
+} // namespace optimization
+} // namespace mlpack
+
+#endif

src/mlpack/core/optimizers/smorms3/smorms3_update.hpp

@@ -98,14 +98,14 @@ class SMORMS3Update
   double& Epsilon() { return epsilon; }
 
  private:
-   //! The value used to initialise the mean squared gradient parameter.
-   double epsilon;
+  //! The value used to initialise the mean squared gradient parameter.
+  double epsilon;
 
-   // The parameters mem, g and g2.
-   arma::mat mem, g, g2;
+  // The parameters mem, g and g2.
+  arma::mat mem, g, g2;
 };
 
 } // namespace optimization
 } // namespace mlpack
 
-#endif
+#endif

src/mlpack/core/tree/address.hpp

@@ -82,7 +82,7 @@ void PointToAddress(AddressType& address, const VecType& point)
   for (size_t i = 0; i < point.n_elem; i++)
   {
     int e;
-    VecElemType normalizedVal = std::frexp(point(i),&e);
+    VecElemType normalizedVal = std::frexp(point(i), &e);
     bool sgn = std::signbit(normalizedVal);
 
     if (point(i) == 0)
@@ -262,6 +262,6 @@ bool Contains(const AddressType1& address, const AddressType2& loBound,
 
 } // namespace addr
 } // namespace bound
-} // namespave mlpack
+} // namespace mlpack
 
 #endif // MLPACK_CORE_TREE_ADDRESS_HPP

src/mlpack/core/tree/ballbound.hpp

@@ -54,7 +54,6 @@ class BallBound
   bool ownsMetric;
 
  public:
-
   //! Empty Constructor.
   BallBound();
 

src/mlpack/core/tree/binary_space_tree/binary_space_tree_impl.hpp

@@ -413,7 +413,7 @@ BinarySpaceTree(BinarySpaceTree&& other) :
   other.minimumBoundDistance = 0.0;
   other.dataset = NULL;
 
-  //Set new parent.
+  // Set new parent.
   if (left)
     left->parent = this;
   if (right)

src/mlpack/core/tree/binary_space_tree/breadth_first_dual_tree_traverser.hpp

@@ -12,8 +12,8 @@
  * 3-clause BSD license along with mlpack.  If not, see
  * http://www.opensource.org/licenses/BSD-3-Clause for more information.
  */
-#ifndef MLPACK_CORE_TREE_BINARY_SPACE_TREE_BREADTH_FIRST_DUAL_TREE_TRAVERSER_HPP
-#define MLPACK_CORE_TREE_BINARY_SPACE_TREE_BREADTH_FIRST_DUAL_TREE_TRAVERSER_HPP
+#ifndef MLPACK_CORE_TREE_BINARY_SPACE_TREE_BF_DUAL_TREE_TRAVERSER_HPP
+#define MLPACK_CORE_TREE_BINARY_SPACE_TREE_BF_DUAL_TREE_TRAVERSER_HPP
 
 #include <mlpack/prereqs.hpp>
 #include <queue>
@@ -111,5 +111,4 @@ class BinarySpaceTree<MetricType, StatisticType, MatType, BoundType,
 // Include implementation.
 #include "breadth_first_dual_tree_traverser_impl.hpp"
 
-#endif // MLPACK_CORE_TREE_BINARY_SPACE_TREE_BREADTH_FIRST_DUAL_TREE_TRAVERSER_HPP
-
+#endif // MLPACK_CORE_TREE_BINARY_SPACE_TREE_BF_DUAL_TREE_TRAVERSER_HPP

src/mlpack/core/tree/binary_space_tree/breadth_first_dual_tree_traverser_impl.hpp

@@ -11,8 +11,8 @@
  * 3-clause BSD license along with mlpack.  If not, see
  * http://www.opensource.org/licenses/BSD-3-Clause for more information.
  */
-#ifndef MLPACK_CORE_TREE_BINARY_SPACE_TREE_BREADTH_FIRST_DUAL_TREE_TRAVERSER_IMPL_HPP
-#define MLPACK_CORE_TREE_BINARY_SPACE_TREE_BREADTH_FIRST_DUAL_TREE_TRAVERSER_IMPL_HPP
+#ifndef MLPACK_CORE_TREE_BINARY_SPACE_TREE_BF_DUAL_TREE_TRAVERSER_IMPL_HPP
+#define MLPACK_CORE_TREE_BINARY_SPACE_TREE_BF_DUAL_TREE_TRAVERSER_IMPL_HPP
 
 // In case it hasn't been included yet.
 #include "breadth_first_dual_tree_traverser.hpp"
@@ -207,4 +207,4 @@ BreadthFirstDualTreeTraverser<RuleType>::Traverse(
 } // namespace tree
 } // namespace mlpack
 
-#endif // MLPACK_CORE_TREE_BINARY_SPACE_TREE_BREADTH_FIRST_DUAL_TREE_TRAVERSER_IMPL_HPP
+#endif // MLPACK_CORE_TREE_BINARY_SPACE_TREE_BF_DUAL_TREE_TRAVERSER_IMPL_HPP

src/mlpack/core/tree/binary_space_tree/rp_tree_mean_split.hpp

@@ -132,7 +132,6 @@ class RPTreeMeanSplit
   }
 
  private:
-
   /**
    * Get the average distance between points in the dataset.
    *

src/mlpack/core/tree/binary_space_tree/ub_tree_split.hpp

@@ -30,9 +30,10 @@ class UBTreeSplit
 {
  public:
   //! The type of an address element.
-  typedef typename std::conditional<sizeof(typename MatType::elem_type) * CHAR_BIT <= 32,
-                                    uint32_t,
-                                    uint64_t>::type AddressElemType;
+  typedef typename std::conditional<
+      sizeof(typename MatType::elem_type) * CHAR_BIT <= 32,
+      uint32_t,
+      uint64_t>::type AddressElemType;
 
   //! An information about the partition.
   struct SplitInfo

src/mlpack/core/tree/cellbound_impl.hpp

@@ -82,7 +82,9 @@ inline CellBound<MetricType, ElemType>::CellBound(
  * Same as the copy constructor.
  */
 template<typename MetricType, typename ElemType>
-inline CellBound<MetricType, ElemType>& CellBound<MetricType, ElemType>::operator=(
+inline CellBound<
+    MetricType,
+    ElemType>& CellBound<MetricType, ElemType>::operator=(
     const CellBound<MetricType, ElemType>& other)
 {
   if (dim != other.Dim())
@@ -486,9 +488,10 @@ inline ElemType CellBound<MetricType, ElemType>::MinDistance(
       lower = loBound(d, i) - point[d];
       higher = point[d] - hiBound(d, i);
 
-      // Since only one of 'lower' or 'higher' is negative, if we add each's
-      // absolute value to itself and then sum those two, our result is the
-      // nonnegative half of the equation times two; then we raise to power Power.
+      // Since only one of 'lower' or 'higher' is negative, if we add
+      // each's absolute value to itself and then sum those two, our
+      // result is the non negative half of the equation times two;
+      // then we raise to power Power.
       if (MetricType::Power == 1)
         sum += lower + std::fabs(lower) + higher + std::fabs(higher);
       else if (MetricType::Power == 2)
@@ -864,8 +867,8 @@ CellBound<MetricType, ElemType>::RangeDistance(
  */
 template<typename MetricType, typename ElemType>
 template<typename MatType>
-inline CellBound<MetricType, ElemType>& CellBound<MetricType, ElemType>::operator|=(
-    const MatType& data)
+inline CellBound<MetricType, ElemType>&
+CellBound<MetricType, ElemType>::operator|=(const MatType& data)
 {
   Log::Assert(data.n_rows == dim);
 
@@ -893,8 +896,8 @@ inline CellBound<MetricType, ElemType>& CellBound<MetricType, ElemType>::operato
  * Expands this region to encompass another bound.
  */
 template<typename MetricType, typename ElemType>
-inline CellBound<MetricType, ElemType>& CellBound<MetricType, ElemType>::operator|=(
-    const CellBound& other)
+inline CellBound<MetricType, ElemType>&
+CellBound<MetricType, ElemType>::operator|=(const CellBound& other)
 {
   assert(other.dim == dim);
 
@@ -930,7 +933,8 @@ inline CellBound<MetricType, ElemType>& CellBound<MetricType, ElemType>::operato
  */
 template<typename MetricType, typename ElemType>
 template<typename VecType>
-inline bool CellBound<MetricType, ElemType>::Contains(const VecType& point) const
+inline bool CellBound<MetricType, ElemType>::Contains(
+    const VecType& point) const
 {
   for (size_t i = 0; i < point.n_elem; i++)
   {

src/mlpack/core/tree/cosine_tree/cosine_tree.hpp

@@ -246,7 +246,6 @@ class CosineTree
 class CompareCosineNode
 {
  public:
-
   // Comparison function for construction of priority queue.
   bool operator() (const CosineTree* a, const CosineTree* b) const
   {

src/mlpack/core/tree/cover_tree/dual_tree_traverser_impl.hpp

@@ -297,7 +297,8 @@ DualTreeTraverser<RuleType>::ReferenceRecursion(
       break;
 
     // Get a reference to the current largest scale.
-    std::vector<DualCoverTreeMapEntry>& scaleVector = (*referenceMap.rbegin()).second;
+    std::vector<DualCoverTreeMapEntry>& scaleVector =
+        (*referenceMap.rbegin()).second;
 
     // Before traversing all the points in this scale, sort by score.
     std::sort(scaleVector.begin(), scaleVector.end());

src/mlpack/core/tree/hollow_ball_bound.hpp

@@ -55,7 +55,6 @@ class HollowBallBound
   bool ownsMetric;
 
  public:
-
   //! Empty Constructor.
   HollowBallBound();
 

src/mlpack/core/tree/hollow_ball_bound_impl.hpp

@@ -33,7 +33,8 @@ HollowBallBound<TMetricType, ElemType>::HollowBallBound() :
  * @param dimension Dimensionality of ball bound.
  */
 template<typename TMetricType, typename ElemType>
-HollowBallBound<TMetricType, ElemType>::HollowBallBound(const size_t dimension) :
+HollowBallBound<TMetricType, ElemType>::
+HollowBallBound(const size_t dimension) :
     radii(std::numeric_limits<ElemType>::lowest(),
           std::numeric_limits<ElemType>::lowest()),
     center(dimension),

src/mlpack/core/tree/hrectbound.hpp

@@ -39,7 +39,7 @@ struct IsLMetric<metric::LMetric<Power, TakeRoot>>
   static const bool Value = true;
 };
 
-} // namespace util
+} // namespace meta
 
 /**
  * Hyper-rectangle bound for an L-metric.  This should be used in conjunction

src/mlpack/core/tree/hrectbound_impl.hpp

@@ -60,8 +60,10 @@ inline HRectBound<MetricType, ElemType>::HRectBound(
  * Same as the copy constructor.
  */
 template<typename MetricType, typename ElemType>
-inline HRectBound<MetricType, ElemType>& HRectBound<MetricType, ElemType>::operator=(
-    const HRectBound<MetricType, ElemType>& other)
+inline HRectBound<
+    MetricType,
+    ElemType>& HRectBound<MetricType,
+    ElemType>::operator=(const HRectBound<MetricType, ElemType>& other)
 {
   if (dim != other.Dim())
   {
@@ -208,7 +210,8 @@ inline ElemType HRectBound<MetricType, ElemType>::MinDistance(
   else
   {
     if (MetricType::TakeRoot)
-      return (ElemType) pow((double) sum, 1.0 / (double) MetricType::Power) / 2.0;
+      return (ElemType) pow((double) sum,
+          1.0 / (double) MetricType::Power) / 2.0;
     else
       return sum / pow(2.0, MetricType::Power);
   }
@@ -268,7 +271,8 @@ ElemType HRectBound<MetricType, ElemType>::MinDistance(const HRectBound& other)
   else
   {
     if (MetricType::TakeRoot)
-      return (ElemType) pow((double) sum, 1.0 / (double) MetricType::Power) / 2.0;
+      return (ElemType) pow((double) sum,
+          1.0 / (double) MetricType::Power) / 2.0;
     else
       return sum / pow(2.0, MetricType::Power);
   }
@@ -503,8 +507,8 @@ HRectBound<MetricType, ElemType>::RangeDistance(
  */
 template<typename MetricType, typename ElemType>
 template<typename MatType>
-inline HRectBound<MetricType, ElemType>& HRectBound<MetricType, ElemType>::operator|=(
-    const MatType& data)
+inline HRectBound<MetricType, ElemType>&
+HRectBound<MetricType, ElemType>::operator|=(const MatType& data)
 {
   Log::Assert(data.n_rows == dim);
 
@@ -527,8 +531,8 @@ inline HRectBound<MetricType, ElemType>& HRectBound<MetricType, ElemType>::opera
  * Expands this region to encompass another bound.
  */
 template<typename MetricType, typename ElemType>
-inline HRectBound<MetricType, ElemType>& HRectBound<MetricType, ElemType>::operator|=(
-    const HRectBound& other)
+inline HRectBound<MetricType, ElemType>&
+HRectBound<MetricType, ElemType>::operator|=(const HRectBound& other)
 {
   assert(other.dim == dim);
 
@@ -549,7 +553,8 @@ inline HRectBound<MetricType, ElemType>& HRectBound<MetricType, ElemType>::opera
  */
 template<typename MetricType, typename ElemType>
 template<typename VecType>
-inline bool HRectBound<MetricType, ElemType>::Contains(const VecType& point) const
+inline bool HRectBound<MetricType, ElemType>::Contains(
+    const VecType& point) const
 {
   for (size_t i = 0; i < point.n_elem; i++)
   {
@@ -572,7 +577,8 @@ inline bool HRectBound<MetricType, ElemType>::Contains(
     const math::RangeType<ElemType>& r_a = bounds[i];
     const math::RangeType<ElemType>& r_b = bound.bounds[i];
 
-    if (r_a.Hi() <= r_b.Lo() || r_a.Lo() >= r_b.Hi()) // If a does not overlap b at all.
+    // If a does not overlap b at all.
+    if (r_a.Hi() <= r_b.Lo() || r_a.Lo() >= r_b.Hi())
       return false;
   }
 
@@ -583,8 +589,8 @@ inline bool HRectBound<MetricType, ElemType>::Contains(
  * Returns the intersection of this bound and another.
  */
 template<typename MetricType, typename ElemType>
-inline HRectBound<MetricType, ElemType> HRectBound<MetricType, ElemType>::
-operator&(const HRectBound& bound) const
+inline HRectBound<MetricType, ElemType>
+HRectBound<MetricType, ElemType>::operator&(const HRectBound& bound) const
 {
   HRectBound<MetricType, ElemType> result(dim);
 
@@ -600,8 +606,8 @@ operator&(const HRectBound& bound) const
  * Intersects this bound with another.
  */
 template<typename MetricType, typename ElemType>
-inline HRectBound<MetricType, ElemType>& HRectBound<MetricType, ElemType>::
-operator&=(const HRectBound& bound)
+inline HRectBound<MetricType, ElemType>&
+HRectBound<MetricType, ElemType>::operator&=(const HRectBound& bound)
 {
   for (size_t k = 0; k < dim; k++)
   {

src/mlpack/core/tree/octree/dual_tree_traverser_impl.hpp

@@ -135,7 +135,8 @@ void Octree<MetricType, StatisticType, MatType>::DualTreeTraverser<RuleType>::
       {
         if (scores[scoreOrder[i]] == DBL_MAX)
         {
-          // We don't need to check any more---all children past here are pruned.
+          // We don't need to check any more
+          // All children past here are pruned.
           numPrunes += scoreOrder.n_elem - i;
           break;
         }

src/mlpack/core/tree/rectangle_tree/discrete_hilbert_value_impl.hpp

@@ -166,7 +166,7 @@ CalculateValue(const VecType& pt,
   for (size_t i = 0; i < pt.n_rows; i++)
   {
     int e;
-    VecElemType normalizedVal = std::frexp(pt(i),&e);
+    VecElemType normalizedVal = std::frexp(pt(i), &e);
     bool sgn = std::signbit(normalizedVal);
 
     if (pt(i) == 0)
@@ -325,7 +325,7 @@ CompareWith(const VecType& pt,
   if (numValues == 0)
     return -1;
 
-  return CompareValues(localHilbertValues->col(numValues - 1),val);
+  return CompareValues(localHilbertValues->col(numValues - 1), val);
 }
 
 template<typename TreeElemType>
@@ -384,7 +384,7 @@ void DiscreteHilbertValue<TreeElemType>::InsertNode(TreeType* node)
 {
   DiscreteHilbertValue &val = node->AuxiliaryInfo().HilbertValue();
 
-  if (CompareWith(node,val) < 0)
+  if (CompareWith(node, val) < 0)
   {
     localHilbertValues = val.LocalHilbertValues();
     numValues = val.NumValues();
@@ -396,7 +396,6 @@ template<typename TreeType>
 void DiscreteHilbertValue<TreeElemType>::
 DeletePoint(TreeType* /* node */, const size_t localIndex)
 {
-
   // Delete the Hilbert value from the local dataset
   for (size_t i = numValues - 1; i > localIndex; i--)
     localHilbertValues->col(i - 1) = localHilbertValues->col(i);

src/mlpack/core/tree/rectangle_tree/dual_tree_traverser.hpp

@@ -67,7 +67,6 @@ class RectangleTree<MetricType, StatisticType, MatType, SplitType,
   size_t& NumBaseCases() { return numBaseCases; }
 
  private:
-
   // We use this struct and this function to make the sorting and scoring easy
   // and efficient:
   struct NodeAndScore

src/mlpack/core/tree/rectangle_tree/dual_tree_traverser_impl.hpp

@@ -78,7 +78,7 @@ DualTreeTraverser<RuleType>::Traverse(RectangleTree& queryNode,
       if (childScore == DBL_MAX)
         continue;  // We don't require a search in this reference node.
 
-      for(size_t ref = 0; ref < referenceNode.Count(); ++ref)
+      for (size_t ref = 0; ref < referenceNode.Count(); ++ref)
         rule.BaseCase(queryNode.Point(query), referenceNode.Point(ref));
 
       numBaseCases += referenceNode.Count();

src/mlpack/core/tree/rectangle_tree/hilbert_r_tree_auxiliary_information.hpp

@@ -11,8 +11,8 @@
  * 3-clause BSD license along with mlpack.  If not, see
  * http://www.opensource.org/licenses/BSD-3-Clause for more information.
  */
-#ifndef MLPACK_CORE_TREE_RECTANGLE_TREE_HILBERT_R_TREE_AUXILIARY_INFORMATION_HPP
-#define MLPACK_CORE_TREE_RECTANGLE_TREE_HILBERT_R_TREE_AUXILIARY_INFORMATION_HPP
+#ifndef MLPACK_CORE_TREE_RECTANGLE_TREE_HR_TREE_AUXILIARY_INFO_HPP
+#define MLPACK_CORE_TREE_RECTANGLE_TREE_HR_TREE_AUXILIARY_INFO_HPP
 
 namespace mlpack {
 namespace tree {
@@ -147,4 +147,4 @@ class HilbertRTreeAuxiliaryInformation
 
 #include "hilbert_r_tree_auxiliary_information_impl.hpp"
 
-#endif//MLPACK_CORE_TREE_RECTANGLE_TREE_HILBERT_R_TREE_AUXILIARY_INFORMATION_HPP
+#endif // MLPACK_CORE_TREE_RECTANGLE_TREE_HR_TREE_AUXILIARY_INFO_HPP

src/mlpack/core/tree/rectangle_tree/hilbert_r_tree_auxiliary_information_impl.hpp

@@ -10,8 +10,8 @@
  * 3-clause BSD license along with mlpack.  If not, see
  * http://www.opensource.org/licenses/BSD-3-Clause for more information.
  */
-#ifndef MLPACK_CORE_TREE_RECTANGLE_TREE_HILBERT_R_TREE_AUXILIARY_INFORMATION_IMPL_HPP
-#define MLPACK_CORE_TREE_RECTANGLE_TREE_HILBERT_R_TREE_AUXILIARY_INFORMATION_IMPL_HPP
+#ifndef MLPACK_CORE_TREE_RECTANGLE_TREE_HR_TREE_AUXILIARY_INFO_IMPL_HPP
+#define MLPACK_CORE_TREE_RECTANGLE_TREE_HR_TREE_AUXILIARY_INFO_IMPL_HPP
 
 #include "hilbert_r_tree_auxiliary_information.hpp"
 
@@ -126,7 +126,7 @@ bool HilbertRTreeAuxiliaryInformation<TreeType, HilbertValueType>::
 HandlePointDeletion(TreeType* node, const size_t localIndex)
 {
   // Update the largest Hilbert value.
-  hilbertValue.DeletePoint(node,localIndex);
+  hilbertValue.DeletePoint(node, localIndex);
 
   for (size_t i = localIndex + 1; localIndex < node->NumPoints(); i++)
     node->Point(i - 1) = node->Point(i);
@@ -141,7 +141,7 @@ bool HilbertRTreeAuxiliaryInformation<TreeType, HilbertValueType>::
 HandleNodeRemoval(TreeType* node, const size_t nodeIndex)
 {
   // Update the largest Hilbert value.
-  hilbertValue.RemoveNode(node,nodeIndex);
+  hilbertValue.RemoveNode(node, nodeIndex);
 
   for (size_t i = nodeIndex + 1; nodeIndex < node->NumChildren(); i++)
     node->children[i - 1] = node->children[i];
@@ -178,7 +178,7 @@ NullifyData()
 template<typename TreeType,
          template<typename> class HilbertValueType>
 template<typename Archive>
-void HilbertRTreeAuxiliaryInformation<TreeType ,HilbertValueType>::
+void HilbertRTreeAuxiliaryInformation<TreeType, HilbertValueType>::
 Serialize(Archive& ar, const unsigned int /* version */)
 {
   using data::CreateNVP;
@@ -190,4 +190,4 @@ Serialize(Archive& ar, const unsigned int /* version */)
 } // namespace tree
 } // namespace mlpack
 
-#endif//MLPACK_CORE_TREE_RECTANGLE_TREE_HILBERT_R_TREE_AUXILIARY_INFORMATION_IMPL_HPP
+#endif // MLPACK_CORE_TREE_RECTANGLE_TREE_HR_TREE_AUXILIARY_INFO_IMPL_HPP

src/mlpack/core/tree/rectangle_tree/hilbert_r_tree_descent_heuristic.hpp

@@ -10,8 +10,8 @@
  * 3-clause BSD license along with mlpack.  If not, see
  * http://www.opensource.org/licenses/BSD-3-Clause for more information.
  */
-#ifndef MLPACK_CORE_TREE_RECTANGLE_TREE_HILBERT_R_TREE_DESCENT_HEURISTIC_HPP
-#define MLPACK_CORE_TREE_RECTANGLE_TREE_HILBERT_R_TREE_DESCENT_HEURISTIC_HPP
+#ifndef MLPACK_CORE_TREE_RECTANGLE_TREE_HR_TREE_DESCENT_HEURISTIC_HPP
+#define MLPACK_CORE_TREE_RECTANGLE_TREE_HR_TREE_DESCENT_HEURISTIC_HPP
 
 #include <mlpack/prereqs.hpp>
 
@@ -55,4 +55,4 @@ class HilbertRTreeDescentHeuristic
 
 #include "hilbert_r_tree_descent_heuristic_impl.hpp"
 
-#endif // MLPACK_CORE_TREE_RECTANGLE_TREE_HILBERT_R_TREE_DESCENT_HEURISTIC_HPP
+#endif // MLPACK_CORE_TREE_RECTANGLE_TREE_HR_TREE_DESCENT_HEURISTIC_HPP

src/mlpack/core/tree/rectangle_tree/hilbert_r_tree_descent_heuristic_impl.hpp

@@ -10,8 +10,8 @@
  * 3-clause BSD license along with mlpack.  If not, see
  * http://www.opensource.org/licenses/BSD-3-Clause for more information.
  */
-#ifndef MLPACK_CORE_TREE_RECTANGLE_TREE_HILBERT_R_TREE_DESCENT_HEURISTIC_IMPL_HPP
-#define MLPACK_CORE_TREE_RECTANGLE_TREE_HILBERT_R_TREE_DESCENT_HEURISTIC_IMPL_HPP
+#ifndef MLPACK_CORE_TREE_RECTANGLE_TREE_HR_TREE_DESCENT_HEURISTIC_IMPL_HPP
+#define MLPACK_CORE_TREE_RECTANGLE_TREE_HR_TREE_DESCENT_HEURISTIC_IMPL_HPP
 
 #include "hilbert_r_tree_descent_heuristic.hpp"
 
@@ -51,4 +51,4 @@ size_t HilbertRTreeDescentHeuristic::ChooseDescentNode(
 } //  namespace tree
 } //  namespace mlpack
 
-#endif // MLPACK_CORE_TREE_RECTANGLE_TREE_HILBERT_R_TREE_DESCENT_HEURISTIC_IMPL_HPP
+#endif // MLPACK_CORE_TREE_RECTANGLE_TREE_HR_TREE_DESCENT_HEURISTIC_IMPL_HPP

src/mlpack/core/tree/rectangle_tree/hilbert_r_tree_split_impl.hpp

@@ -48,7 +48,7 @@ void HilbertRTreeSplit<splitOrder>::SplitLeafNode(TreeType* tree,
 
   TreeType* parent = tree->Parent();
   size_t iTree = 0;
-  for (iTree = 0; parent->children[iTree] != tree; iTree++);
+  for (iTree = 0; parent->children[iTree] != tree; iTree++) { }
 
   // Try to find splitOrder cooperating siblings in order to redistribute points
   // among them and avoid split.
@@ -112,7 +112,7 @@ SplitNonLeafNode(TreeType* tree, std::vector<bool>& relevels)
   TreeType* parent = tree->Parent();
 
   size_t iTree = 0;
-  for (iTree = 0; parent->children[iTree] != tree; iTree++);
+  for (iTree = 0; parent->children[iTree] != tree; iTree++) { }
 
   // Try to find splitOrder cooperating siblings in order to redistribute
   // children among them and avoid split.

src/mlpack/core/tree/rectangle_tree/minimal_splits_number_sweep_impl.hpp

@@ -70,8 +70,9 @@ size_t MinimalSplitsNumberSweep<SplitPolicy>::SweepNonLeafNode(
     }
 
     // Check if the split is possible.
-    if (numTreeOneChildren <= node->MaxNumChildren() && numTreeOneChildren > 0 &&
-        numTreeTwoChildren <= node->MaxNumChildren() && numTreeTwoChildren > 0)
+    if (numTreeOneChildren <= node->MaxNumChildren() &&
+        numTreeOneChildren > 0 && numTreeTwoChildren <= node->MaxNumChildren()
+        && numTreeTwoChildren > 0)
     {
       // Evaluate the cost using the number of splits and balancing.
       size_t balance;

src/mlpack/core/tree/rectangle_tree/no_auxiliary_information.hpp

@@ -21,15 +21,15 @@ class NoAuxiliaryInformation
 {
  public:
   //! Construct the auxiliary information object.
-  NoAuxiliaryInformation() { };
+  NoAuxiliaryInformation() { }
   //! Construct the auxiliary information object.
-  NoAuxiliaryInformation(const TreeType* /* node */) { };
+  NoAuxiliaryInformation(const TreeType* /* node */) { }
   //! Construct the auxiliary information object.
   NoAuxiliaryInformation(const NoAuxiliaryInformation& /* other */,
                          TreeType* /* tree */,
-                         bool /* deepCopy */ = true) { };
+                         bool /* deepCopy */ = true) { }
   //! Construct the auxiliary information object.
-  NoAuxiliaryInformation(NoAuxiliaryInformation&& /* other */) { };
+  NoAuxiliaryInformation(NoAuxiliaryInformation&& /* other */) { }
 
   //! Copy the auxiliary information object.
   NoAuxiliaryInformation& operator=(const NoAuxiliaryInformation& /* other */)
@@ -141,7 +141,7 @@ class NoAuxiliaryInformation
    * Serialize the information.
    */
   template<typename Archive>
-  void Serialize(Archive &, const unsigned int /* version */) { };
+  void Serialize(Archive &, const unsigned int /* version */) { }
 };
 
 } // namespace tree

src/mlpack/core/tree/rectangle_tree/r_plus_plus_tree_auxiliary_information.hpp

@@ -11,8 +11,8 @@
  * 3-clause BSD license along with mlpack.  If not, see
  * http://www.opensource.org/licenses/BSD-3-Clause for more information.
  */
-#ifndef MLPACK_CORE_TREE_RECTANGLE_TREE_R_PLUS_PLUS_TREE_AUXILIARY_INFORMATION_HPP
-#define MLPACK_CORE_TREE_RECTANGLE_TREE_R_PLUS_PLUS_TREE_AUXILIARY_INFORMATION_HPP
+#ifndef MLPACK_CORE_TREE_RECTANGLE_TREE_RPP_TREE_AUXILIARY_INFO_HPP
+#define MLPACK_CORE_TREE_RECTANGLE_TREE_RPP_TREE_AUXILIARY_INFO_HPP
 
 #include <mlpack/prereqs.hpp>
 #include "../hrectbound.hpp"
@@ -163,4 +163,4 @@ class RPlusPlusTreeAuxiliaryInformation
 
 #include "r_plus_plus_tree_auxiliary_information_impl.hpp"
 
-#endif//MLPACK_CORE_TREE_RECTANGLE_TREE_R_PLUS_PLUS_TREE_AUXILIARY_INFORMATION_HPP
+#endif // MLPACK_CORE_TREE_RECTANGLE_TREE_RPP_TREE_AUXILIARY_INFO_HPP

src/mlpack/core/tree/rectangle_tree/r_plus_plus_tree_auxiliary_information_impl.hpp

@@ -11,8 +11,8 @@
  * 3-clause BSD license along with mlpack.  If not, see
  * http://www.opensource.org/licenses/BSD-3-Clause for more information.
  */
-#ifndef MLPACK_CORE_TREE_RECTANGLE_TREE_R_PLUS_PLUS_TREE_AUXILIARY_INFORMATION_IMPL_HPP
-#define MLPACK_CORE_TREE_RECTANGLE_TREE_R_PLUS_PLUS_TREE_AUXILIARY_INFORMATION_IMPL_HPP
+#ifndef MLPACK_CORE_TREE_RECTANGLE_TREE_RPP_TREE_AUXILIARY_INFO_IMPL_HPP
+#define MLPACK_CORE_TREE_RECTANGLE_TREE_RPP_TREE_AUXILIARY_INFO_IMPL_HPP
 
 #include "r_plus_plus_tree_auxiliary_information.hpp"
 
@@ -23,9 +23,7 @@ template<typename TreeType>
 RPlusPlusTreeAuxiliaryInformation<TreeType>::
 RPlusPlusTreeAuxiliaryInformation() :
     outerBound(0)
-{
-
-}
+{ /* Nothing to do. */ }
 
 template<typename TreeType>
 RPlusPlusTreeAuxiliaryInformation<TreeType>::
@@ -36,11 +34,13 @@ RPlusPlusTreeAuxiliaryInformation(const TreeType* tree) :
 {
   // Initialize the maximum bounding rectangle if the node is the root
   if (!tree->Parent())
+  {
     for (size_t k = 0; k < outerBound.Dim(); k++)
     {
       outerBound[k].Lo() = std::numeric_limits<ElemType>::lowest();
       outerBound[k].Hi() = std::numeric_limits<ElemType>::max();
     }
+  }
 }
 
 template<typename TreeType>
@@ -50,17 +50,13 @@ RPlusPlusTreeAuxiliaryInformation(
     TreeType* /* tree */,
     bool /* deepCopy */) :
     outerBound(other.OuterBound())
-{
-
-}
+{ /* Nothing to do. */ }
 
 template<typename TreeType>
 RPlusPlusTreeAuxiliaryInformation<TreeType>::
 RPlusPlusTreeAuxiliaryInformation(RPlusPlusTreeAuxiliaryInformation&& other) :
     outerBound(std::move(other.outerBound))
-{
-
-}
+{ /* Nothing to do. */ }
 
 template<typename TreeType>
 bool RPlusPlusTreeAuxiliaryInformation<TreeType>::HandlePointInsertion(
@@ -122,9 +118,7 @@ void RPlusPlusTreeAuxiliaryInformation<TreeType>::SplitAuxiliaryInfo(
 
 template<typename TreeType>
 void RPlusPlusTreeAuxiliaryInformation<TreeType>::NullifyData()
-{
-
-}
+{ /* Nothing to do */ }
 
 /**
  * Serialize the information.
@@ -142,4 +136,4 @@ Serialize(Archive& ar, const unsigned int /* version */)
 } // namespace tree
 } // namespace mlpack
 
-#endif // MLPACK_CORE_TREE_RECTANGLE_TREE_R_PLUS_PLUS_TREE_AUXILIARY_INFORMATION_IMPL_HPP
+#endif // MLPACK_CORE_TREE_RECTANGLE_TREE_RPP_TREE_AUXILIARY_INFO_IMPL_HPP

src/mlpack/core/tree/rectangle_tree/r_plus_plus_tree_descent_heuristic.hpp

@@ -10,8 +10,8 @@
  * 3-clause BSD license along with mlpack.  If not, see
  * http://www.opensource.org/licenses/BSD-3-Clause for more information.
  */
-#ifndef MLPACK_CORE_TREE_RECTANGLE_TREE_R_PLUS_PLUS_TREE_DESCENT_HEURISTIC_HPP
-#define MLPACK_CORE_TREE_RECTANGLE_TREE_R_PLUS_PLUS_TREE_DESCENT_HEURISTIC_HPP
+#ifndef MLPACK_CORE_TREE_RECTANGLE_TREE_RPP_TREE_DESCENT_HEURISTIC_HPP
+#define MLPACK_CORE_TREE_RECTANGLE_TREE_RPP_TREE_DESCENT_HEURISTIC_HPP
 
 #include <mlpack/prereqs.hpp>
 
@@ -50,4 +50,4 @@ class RPlusPlusTreeDescentHeuristic
 
 #include "r_plus_plus_tree_descent_heuristic_impl.hpp"
 
-#endif // MLPACK_CORE_TREE_RECTANGLE_TREE_R_PLUS_PLUS_TREE_DESCENT_HEURISTIC_HPP
+#endif // MLPACK_CORE_TREE_RECTANGLE_TREE_RPP_TREE_DESCENT_HEURISTIC_HPP

src/mlpack/core/tree/rectangle_tree/r_plus_plus_tree_descent_heuristic_impl.hpp

@@ -10,8 +10,8 @@
  * 3-clause BSD license along with mlpack.  If not, see
  * http://www.opensource.org/licenses/BSD-3-Clause for more information.
  */
-#ifndef MLPACK_CORE_TREE_RECTANGLE_TREE_R_PLUS_PLUS_TREE_DESCENT_HEURISTIC_IMPL_HPP
-#define MLPACK_CORE_TREE_RECTANGLE_TREE_R_PLUS_PLUS_TREE_DESCENT_HEURISTIC_IMPL_HPP
+#ifndef MLPACK_CORE_TREE_RECTANGLE_TREE_RPP_TREE_DESCENT_HEURISTIC_IMPL_HPP
+#define MLPACK_CORE_TREE_RECTANGLE_TREE_RPP_TREE_DESCENT_HEURISTIC_IMPL_HPP
 
 #include "r_plus_plus_tree_descent_heuristic.hpp"
 #include "../hrectbound.hpp"
@@ -50,4 +50,4 @@ size_t RPlusPlusTreeDescentHeuristic::ChooseDescentNode(
 } // namespace tree
 } // namespace mlpack
 
-#endif // MLPACK_CORE_TREE_RECTANGLE_TREE_R_PLUS_PLUS_TREE_DESCENT_HEURISTIC_IMPL_HPP
+#endif // MLPACK_CORE_TREE_RECTANGLE_TREE_RPP_TREE_DESCENT_HEURISTIC_IMPL_HPP

src/mlpack/core/tree/rectangle_tree/r_plus_tree_split_impl.hpp

@@ -137,7 +137,7 @@ SplitNonLeafNode(TreeType* tree, std::vector<bool>& relevels)
     tree->NullifyData();
     tree->children[(tree->NumChildren())++] = copy;
 
-    RPlusTreeSplit::SplitNonLeafNode(copy,relevels);
+    RPlusTreeSplit::SplitNonLeafNode(copy, relevels);
     return true;
   }
   size_t cutAxis = tree->Bound().Dim();

src/mlpack/core/tree/rectangle_tree/r_star_tree_split.hpp

@@ -32,14 +32,14 @@ class RStarTreeSplit
    * necessary, this split will propagate upwards through the tree.
    */
   template <typename TreeType>
-  static void SplitLeafNode(TreeType *tree,std::vector<bool>& relevels);
+  static void SplitLeafNode(TreeType *tree, std::vector<bool>& relevels);
 
   /**
    * Split a non-leaf node using the "default" algorithm.  If this is a root
    * node, the tree increases in depth.
    */
   template <typename TreeType>
-  static bool SplitNonLeafNode(TreeType *tree,std::vector<bool>& relevels);
+  static bool SplitNonLeafNode(TreeType *tree, std::vector<bool>& relevels);
 
   /**
    * Reinsert any points into the tree, if needed.  This returns the number of
@@ -52,7 +52,10 @@ class RStarTreeSplit
    * Given a node, return the best dimension and the best index to split on.
    */
   template<typename TreeType>
-  static void PickLeafSplit(TreeType* tree, size_t& bestAxis, size_t& bestIndex);
+  static void PickLeafSplit(
+      TreeType* tree,
+      size_t& bestAxis,
+      size_t& bestIndex);
 
  private:
   /**

src/mlpack/core/tree/rectangle_tree/r_star_tree_split_impl.hpp

@@ -174,7 +174,7 @@ void RStarTreeSplit::PickLeafSplit(TreeType* tree,
  * new nodes into the tree, spliting the parent if necessary.
  */
 template<typename TreeType>
-void RStarTreeSplit::SplitLeafNode(TreeType *tree,std::vector<bool>& relevels)
+void RStarTreeSplit::SplitLeafNode(TreeType *tree, std::vector<bool>& relevels)
 {
   // Convenience typedef.
   typedef typename TreeType::ElemType ElemType;
@@ -268,7 +268,9 @@ void RStarTreeSplit::SplitLeafNode(TreeType *tree,std::vector<bool>& relevels)
  * higher up the tree because they were already updated if necessary.
  */
 template<typename TreeType>
-bool RStarTreeSplit::SplitNonLeafNode(TreeType *tree,std::vector<bool>& relevels)
+bool RStarTreeSplit::SplitNonLeafNode(
+    TreeType *tree,
+    std::vector<bool>& relevels)
 {
   // Convenience typedef.
   typedef typename TreeType::ElemType ElemType;

src/mlpack/core/tree/rectangle_tree/r_tree_split.hpp

@@ -31,27 +31,27 @@ class RTreeSplit
    * will propagate upwards through the tree.
    */
   template<typename TreeType>
-  static void SplitLeafNode(TreeType *tree,std::vector<bool>& relevels);
+  static void SplitLeafNode(TreeType *tree, std::vector<bool>& relevels);
 
   /**
    * Split a non-leaf node using the "default" algorithm.  If this is a root
    * node, the tree increases in depth.
    */
   template<typename TreeType>
-  static bool SplitNonLeafNode(TreeType *tree,std::vector<bool>& relevels);
+  static bool SplitNonLeafNode(TreeType *tree, std::vector<bool>& relevels);
 
  private:
   /**
    * Get the seeds for splitting a leaf node.
    */
   template<typename TreeType>
-  static void GetPointSeeds(const TreeType *tree,int& i, int& j);
+  static void GetPointSeeds(const TreeType *tree, int& i, int& j);
 
   /**
    * Get the seeds for splitting a non-leaf node.
    */
   template<typename TreeType>
-  static void GetBoundSeeds(const TreeType *tree,int& i, int& j);
+  static void GetBoundSeeds(const TreeType *tree, int& i, int& j);
 
   /**
    * Assign points to the two new nodes.

src/mlpack/core/tree/rectangle_tree/r_tree_split_impl.hpp

@@ -26,7 +26,7 @@ namespace tree {
  * new nodes into the tree, spliting the parent if necessary.
  */
 template<typename TreeType>
-void RTreeSplit::SplitLeafNode(TreeType *tree,std::vector<bool>& relevels)
+void RTreeSplit::SplitLeafNode(TreeType *tree, std::vector<bool>& relevels)
 {
   if (tree->Count() <= tree->MaxLeafSize())
     return;
@@ -42,7 +42,7 @@ void RTreeSplit::SplitLeafNode(TreeType *tree,std::vector<bool>& relevels)
     tree->NullifyData();
     // Because this was a leaf node, numChildren must be 0.
     tree->children[(tree->NumChildren())++] = copy;
-    RTreeSplit::SplitLeafNode(copy,relevels);
+    RTreeSplit::SplitLeafNode(copy, relevels);
     return;
   }
 
@@ -53,7 +53,7 @@ void RTreeSplit::SplitLeafNode(TreeType *tree,std::vector<bool>& relevels)
   // rectangles, only points.  We assume that the tree uses Euclidean Distance.
   int i = 0;
   int j = 0;
-  RTreeSplit::GetPointSeeds(tree,i, j);
+  RTreeSplit::GetPointSeeds(tree, i, j);
 
   TreeType* treeOne = new TreeType(tree->Parent());
   TreeType* treeTwo = new TreeType(tree->Parent());
@@ -73,7 +73,7 @@ void RTreeSplit::SplitLeafNode(TreeType *tree,std::vector<bool>& relevels)
   // just in case, we use an assert.
   assert(par->NumChildren() <= par->MaxNumChildren() + 1);
   if (par->NumChildren() == par->MaxNumChildren() + 1)
-    RTreeSplit::SplitNonLeafNode(par,relevels);
+    RTreeSplit::SplitNonLeafNode(par, relevels);
 
   assert(treeOne->Parent()->NumChildren() <= treeOne->MaxNumChildren());
   assert(treeOne->Parent()->NumChildren() >= treeOne->MinNumChildren());
@@ -92,7 +92,7 @@ void RTreeSplit::SplitLeafNode(TreeType *tree,std::vector<bool>& relevels)
  * higher up the tree because they were already updated if necessary.
  */
 template<typename TreeType>
-bool RTreeSplit::SplitNonLeafNode(TreeType *tree,std::vector<bool>& relevels)
+bool RTreeSplit::SplitNonLeafNode(TreeType *tree, std::vector<bool>& relevels)
 {
   // If we are splitting the root node, we need will do things differently so
   // that the constructor and other methods don't confuse the end user by giving
@@ -105,13 +105,13 @@ bool RTreeSplit::SplitNonLeafNode(TreeType *tree,std::vector<bool>& relevels)
     tree->NumChildren() = 0;
     tree->NullifyData();
     tree->children[(tree->NumChildren())++] = copy;
-    RTreeSplit::SplitNonLeafNode(copy,relevels);
+    RTreeSplit::SplitNonLeafNode(copy, relevels);
     return true;
   }
 
   int i = 0;
   int j = 0;
-  RTreeSplit::GetBoundSeeds(tree,i, j);
+  RTreeSplit::GetBoundSeeds(tree, i, j);
 
   assert(i != j);
 
@@ -138,7 +138,7 @@ bool RTreeSplit::SplitNonLeafNode(TreeType *tree,std::vector<bool>& relevels)
   assert(par->NumChildren() <= par->MaxNumChildren() + 1);
 
   if (par->NumChildren() == par->MaxNumChildren() + 1)
-    RTreeSplit::SplitNonLeafNode(par,relevels);
+    RTreeSplit::SplitNonLeafNode(par, relevels);
 
   // We have to update the children of each of these new nodes so that they
   // record the correct parent.
@@ -154,7 +154,7 @@ bool RTreeSplit::SplitNonLeafNode(TreeType *tree,std::vector<bool>& relevels)
 
   // Because we now have pointers to the information stored under this tree,
   // we need to delete this node carefully.
-  tree->SoftDelete(); //currently does nothing but leak memory.
+  tree->SoftDelete(); // currently does nothing but leak memory.
 
   return false;
 }
@@ -164,7 +164,7 @@ bool RTreeSplit::SplitNonLeafNode(TreeType *tree,std::vector<bool>& relevels)
  * The indices of these points will be stored in iRet and jRet.
  */
 template<typename TreeType>
-void RTreeSplit::GetPointSeeds(const TreeType *tree,int& iRet, int& jRet)
+void RTreeSplit::GetPointSeeds(const TreeType *tree, int& iRet, int& jRet)
 {
   // Here we want to find the pair of points that it is worst to place in the
   // same node.  Because we are just using points, we will simply choose the two
@@ -193,7 +193,7 @@ void RTreeSplit::GetPointSeeds(const TreeType *tree,int& iRet, int& jRet)
  * indices of the bounds will be stored in iRet and jRet.
  */
 template<typename TreeType>
-void RTreeSplit::GetBoundSeeds(const TreeType *tree,int& iRet, int& jRet)
+void RTreeSplit::GetBoundSeeds(const TreeType *tree, int& iRet, int& jRet)
 {
   // Convenience typedef.
   typedef typename TreeType::ElemType ElemType;

src/mlpack/core/tree/rectangle_tree/rectangle_tree.hpp

@@ -49,7 +49,8 @@ template<typename MetricType = metric::EuclideanDistance,
          typename MatType = arma::mat,
          typename SplitType = RTreeSplit,
          typename DescentType = RTreeDescentHeuristic,
-         template<typename> class AuxiliaryInformationType = NoAuxiliaryInformation>
+         template<typename> class AuxiliaryInformationType =
+             NoAuxiliaryInformation>
 class RectangleTree
 {
   // The metric *must* be the euclidean distance.

src/mlpack/core/tree/rectangle_tree/rectangle_tree_impl.hpp

@@ -112,7 +112,7 @@ RectangleTree<MetricType, StatisticType, MatType, SplitType, DescentType,
 RectangleTree(
     RectangleTree<MetricType, StatisticType, MatType, SplitType, DescentType,
                   AuxiliaryInformationType>*
-        parentNode,const size_t numMaxChildren) :
+        parentNode, const size_t numMaxChildren) :
     maxNumChildren(numMaxChildren > 0 ? numMaxChildren :
                                         parentNode->MaxNumChildren()),
     minNumChildren(parentNode->MinNumChildren()),
@@ -385,7 +385,7 @@ void RectangleTree<MetricType, StatisticType, MatType, SplitType, DescentType,
   // If it is not a leaf node, we use the DescentHeuristic to choose a child
   // to which we recurse.
   auxiliaryInfo.HandlePointInsertion(this, point);
-  const size_t descentNode = DescentType::ChooseDescentNode(this,point);
+  const size_t descentNode = DescentType::ChooseDescentNode(this, point);
   children[descentNode]->InsertPoint(point, relevels);
 }
 
@@ -879,7 +879,7 @@ void RectangleTree<MetricType, StatisticType, MatType, SplitType, DescentType,
     // node contains only one point.
 
     // The SplitType takes care of this and of moving up the tree if necessary.
-    SplitType::SplitLeafNode(this,relevels);
+    SplitType::SplitLeafNode(this, relevels);
   }
   else
   {
@@ -889,7 +889,7 @@ void RectangleTree<MetricType, StatisticType, MatType, SplitType, DescentType,
 
     // If we are full, then we need to split (or at least try).  The SplitType
     // takes care of this and of moving up the tree if necessary.
-    SplitType::SplitNonLeafNode(this,relevels);
+    SplitType::SplitNonLeafNode(this, relevels);
   }
 }
 
@@ -1004,7 +1004,7 @@ void RectangleTree<MetricType, StatisticType, MatType, SplitType, DescentType,
         if (parent->children[j] == this)
         {
           // Decrement numChildren.
-          if (!auxiliaryInfo.HandleNodeRemoval(parent,j))
+          if (!auxiliaryInfo.HandleNodeRemoval(parent, j))
           {
             parent->children[j] = parent->children[--parent->NumChildren()];
           }
@@ -1096,7 +1096,8 @@ void RectangleTree<MetricType, StatisticType, MatType, SplitType, DescentType,
       parent != NULL)
     parent->CondenseTree(point, relevels, usePoint);
   else if (!usePoint &&
-           (ShrinkBoundForBound(bound) || auxiliaryInfo.UpdateAuxiliaryInfo(this)) &&
+           (ShrinkBoundForBound(bound) ||
+           auxiliaryInfo.UpdateAuxiliaryInfo(this)) &&
            parent != NULL)
     parent->CondenseTree(point, relevels, usePoint);
 }
@@ -1261,7 +1262,6 @@ void RectangleTree<MetricType, StatisticType, MatType, SplitType, DescentType,
 
     if (ownsDataset && dataset)
       delete dataset;
-
   }
 
   ar & CreateNVP(maxNumChildren, "maxNumChildren");

src/mlpack/core/tree/rectangle_tree/single_tree_traverser.hpp

@@ -52,7 +52,6 @@ class RectangleTree<MetricType, StatisticType, MatType, SplitType,
   size_t& NumPrunes() { return numPrunes; }
 
  private:
-
   // We use this class and this function to make the sorting and scoring easy
   // and efficient:
   struct NodeAndScore

src/mlpack/core/tree/rectangle_tree/single_tree_traverser_impl.hpp

@@ -49,7 +49,6 @@ SingleTreeTraverser<RuleType>::Traverse(
     const size_t queryIndex,
     const RectangleTree& referenceNode)
 {
-
   // If we reach a leaf node, we need to run the base case.
   if (referenceNode.IsLeaf())
   {

src/mlpack/core/tree/rectangle_tree/typedef.hpp

@@ -125,7 +125,7 @@ using XTree = RectangleTree<MetricType,
  */
 template<typename TreeType>
 using DiscreteHilbertRTreeAuxiliaryInformation =
-      HilbertRTreeAuxiliaryInformation<TreeType,DiscreteHilbertValue>;
+      HilbertRTreeAuxiliaryInformation<TreeType, DiscreteHilbertValue>;
 
 template<typename MetricType, typename StatisticType, typename MatType>
 using HilbertRTree = RectangleTree<MetricType,
@@ -186,7 +186,8 @@ using RPlusTree = RectangleTree<MetricType,
  * @endcode
  *
  * @see @ref trees, RTree, RTree, RPlusTree, RPlusPlusTree
- */template<typename MetricType, typename StatisticType, typename MatType>
+ */
+template<typename MetricType, typename StatisticType, typename MatType>
 using RPlusPlusTree = RectangleTree<MetricType,
                             StatisticType,
                             MatType,

src/mlpack/core/tree/rectangle_tree/x_tree_auxiliary_information.hpp

@@ -231,7 +231,6 @@ class XTreeAuxiliaryInformation
     ar & CreateNVP(normalNodeMaxNumChildren, "normalNodeMaxNumChildren");
     ar & CreateNVP(splitHistory, "splitHistory");
   }
-
 };
 
 } // namespace tree