diff --git a/src/mlpack/core/optimizers/ada_delta/ada_delta.hpp b/src/mlpack/core/optimizers/ada_delta/ada_delta.hpp index 8589374460..9a209c63e7 100644 --- a/src/mlpack/core/optimizers/ada_delta/ada_delta.hpp +++ b/src/mlpack/core/optimizers/ada_delta/ada_delta.hpp @@ -34,10 +34,10 @@ namespace optimization { * * @code * @article{Zeiler2012, - * author = {Matthew D. Zeiler}, - * title = {{ADADELTA:} An Adaptive Learning Rate Method}, - * journal = {CoRR}, - * year = {2012} + * author = {Matthew D. Zeiler}, + * title = {{ADADELTA:} An Adaptive Learning Rate Method}, + * journal = {CoRR}, + * year = {2012} * } * @endcode * diff --git a/src/mlpack/core/optimizers/ada_delta/ada_delta_update.hpp b/src/mlpack/core/optimizers/ada_delta/ada_delta_update.hpp index 391e2a88bd..fc3d0bcde0 100644 --- a/src/mlpack/core/optimizers/ada_delta/ada_delta_update.hpp +++ b/src/mlpack/core/optimizers/ada_delta/ada_delta_update.hpp @@ -29,10 +29,10 @@ namespace optimization { * * @code * @article{Zeiler2012, - * author = {Matthew D. Zeiler}, - * title = {{ADADELTA:} An Adaptive Learning Rate Method}, - * journal = {CoRR}, - * year = {2012} + * author = {Matthew D. Zeiler}, + * title = {{ADADELTA:} An Adaptive Learning Rate Method}, + * journal = {CoRR}, + * year = {2012} * } * @endcode * diff --git a/src/mlpack/core/optimizers/ada_grad/ada_grad.hpp b/src/mlpack/core/optimizers/ada_grad/ada_grad.hpp index e96b619e85..6c622c8e3e 100644 --- a/src/mlpack/core/optimizers/ada_grad/ada_grad.hpp +++ b/src/mlpack/core/optimizers/ada_grad/ada_grad.hpp @@ -30,13 +30,13 @@ namespace optimization { * * @code * @article{duchi2011adaptive, - * author = {Duchi, John and Hazan, Elad and Singer, Yoram}, - * title = {Adaptive subgradient methods for online learning and stochastic optimization}, - * journal = {Journal of Machine Learning Research}, - * volume = {12}, - * number = {Jul}, - * pages = {2121--2159}, - * year = {2011} + * author = {Duchi, John and Hazan, Elad and Singer, Yoram}, + * title = {Adaptive subgradient methods for online learning and stochastic optimization}, + * journal = {Journal of Machine Learning Research}, + * volume = {12}, + * number = {Jul}, + * pages = {2121--2159}, + * year = {2011} * } * @endcode * diff --git a/src/mlpack/core/optimizers/ada_grad/ada_grad_update.hpp b/src/mlpack/core/optimizers/ada_grad/ada_grad_update.hpp index 2fb52573b1..b086926388 100644 --- a/src/mlpack/core/optimizers/ada_grad/ada_grad_update.hpp +++ b/src/mlpack/core/optimizers/ada_grad/ada_grad_update.hpp @@ -58,8 +58,8 @@ class AdaGradUpdate * gradient matrix is initialized to the zeros matrix with the same size as * gradient matrix (see mlpack::optimization::SGD::Optimizer). * - * @param rows number of rows in the gradient matrix. - * @param cols number of columns in the gradient matrix. + * @param rows Number of rows in the gradient matrix. + * @param cols Number of columns in the gradient matrix. */ void Initialize(const size_t rows, const size_t cols) diff --git a/src/mlpack/core/optimizers/adam/adam.hpp b/src/mlpack/core/optimizers/adam/adam.hpp index d4808c6cd9..23615fdd9e 100644 --- a/src/mlpack/core/optimizers/adam/adam.hpp +++ b/src/mlpack/core/optimizers/adam/adam.hpp @@ -37,11 +37,11 @@ namespace optimization { * * @code * @article{Kingma2014, - * author = {Diederik P. Kingma and Jimmy Ba}, - * title = {Adam: {A} Method for Stochastic Optimization}, - * journal = {CoRR}, - * year = {2014}, - * url = {http://arxiv.org/abs/1412.6980} + * author = {Diederik P. Kingma and Jimmy Ba}, + * title = {Adam: {A} Method for Stochastic Optimization}, + * journal = {CoRR}, + * year = {2014}, + * url = {http://arxiv.org/abs/1412.6980} * } * @endcode * @@ -95,13 +95,13 @@ class AdamType * function is visited in linear order. */ AdamType(DecomposableFunctionType& function, - const double stepSize = 0.001, - const double beta1 = 0.9, - const double beta2 = 0.999, - const double eps = 1e-8, - const size_t maxIterations = 100000, - const double tolerance = 1e-5, - const bool shuffle = true); + const double stepSize = 0.001, + const double beta1 = 0.9, + const double beta2 = 0.999, + const double eps = 1e-8, + const size_t maxIterations = 100000, + const double tolerance = 1e-5, + const bool shuffle = true); /** * Optimize the given function using Adam. The given starting point will be diff --git a/src/mlpack/core/optimizers/adam/adam_update.hpp b/src/mlpack/core/optimizers/adam/adam_update.hpp index b420b57311..0f64c82327 100644 --- a/src/mlpack/core/optimizers/adam/adam_update.hpp +++ b/src/mlpack/core/optimizers/adam/adam_update.hpp @@ -31,11 +31,11 @@ namespace optimization { * * @code * @article{Kingma2014, - * author = {Diederik P. Kingma and Jimmy Ba}, - * title = {Adam: {A} Method for Stochastic Optimization}, - * journal = {CoRR}, - * year = {2014}, - * url = {http://arxiv.org/abs/1412.6980} + * author = {Diederik P. Kingma and Jimmy Ba}, + * title = {Adam: {A} Method for Stochastic Optimization}, + * journal = {CoRR}, + * year = {2014}, + * url = {http://arxiv.org/abs/1412.6980} * } * @endcode */ @@ -65,8 +65,8 @@ class AdamUpdate * The Initialize method is called by SGD Optimizer method before the start of * the iteration update process. * - * @param rows number of rows in the gradient matrix. - * @param cols number of columns in the gradient matrix. + * @param rows Number of rows in the gradient matrix. + * @param cols Number of columns in the gradient matrix. */ void Initialize(const size_t rows, const size_t cols) @@ -105,7 +105,7 @@ class AdamUpdate * m / (arma::sqrt(v) + (arma::sqrt(biasCorrection2) * eps). */ iterate -= (stepSize * std::sqrt(biasCorrection2) / biasCorrection1) * - m / (arma::sqrt(v) + epsilon); + m / (arma::sqrt(v) + epsilon); } //! Get the value used to initialise the squared gradient parameter. diff --git a/src/mlpack/core/optimizers/adam/adamax_update.hpp b/src/mlpack/core/optimizers/adam/adamax_update.hpp index 6337efaae3..9ac281cca8 100644 --- a/src/mlpack/core/optimizers/adam/adamax_update.hpp +++ b/src/mlpack/core/optimizers/adam/adamax_update.hpp @@ -67,8 +67,8 @@ class AdaMaxUpdate * The Initialize method is called by SGD Optimizer method before the start of * the iteration update process. * - * @param rows number of rows in the gradient matrix. - * @param cols number of columns in the gradient matrix. + * @param rows Number of rows in the gradient matrix. + * @param cols Number of columns in the gradient matrix. */ void Initialize(const size_t rows, const size_t cols) diff --git a/src/mlpack/core/optimizers/gradient_descent/gradient_descent.hpp b/src/mlpack/core/optimizers/gradient_descent/gradient_descent.hpp index 78ee103813..79fb8dded7 100644 --- a/src/mlpack/core/optimizers/gradient_descent/gradient_descent.hpp +++ b/src/mlpack/core/optimizers/gradient_descent/gradient_descent.hpp @@ -66,9 +66,9 @@ class GradientDescent * @param tolerance Maximum absolute tolerance to terminate algorithm. */ GradientDescent(FunctionType& function, - const double stepSize = 0.01, - const size_t maxIterations = 100000, - const double tolerance = 1e-5); + const double stepSize = 0.01, + const size_t maxIterations = 100000, + const double tolerance = 1e-5); /** * Optimize the given function using gradient descent. The given starting diff --git a/src/mlpack/core/optimizers/rmsprop/rmsprop.hpp b/src/mlpack/core/optimizers/rmsprop/rmsprop.hpp index 4d67dd906b..c3da802d70 100644 --- a/src/mlpack/core/optimizers/rmsprop/rmsprop.hpp +++ b/src/mlpack/core/optimizers/rmsprop/rmsprop.hpp @@ -38,9 +38,9 @@ namespace optimization { * * @code * @misc{tieleman2012, - * title={Lecture 6.5 - rmsprop, COURSERA: Neural Networks for Machine - * Learning}, - * year={2012} + * title = {Lecture 6.5 - rmsprop, COURSERA: Neural Networks for Machine + * Learning}, + * year = {2012} * } * @endcode * diff --git a/src/mlpack/core/optimizers/rmsprop/rmsprop_update.hpp b/src/mlpack/core/optimizers/rmsprop/rmsprop_update.hpp index b9a821a8c4..c86f09f0c3 100644 --- a/src/mlpack/core/optimizers/rmsprop/rmsprop_update.hpp +++ b/src/mlpack/core/optimizers/rmsprop/rmsprop_update.hpp @@ -35,9 +35,9 @@ namespace optimization { * * @code * @misc{tieleman2012, - * title={Lecture 6.5 - rmsprop, COURSERA: Neural Networks for Machine - * Learning}, - * year={2012} + * title = {Lecture 6.5 - rmsprop, COURSERA: Neural Networks for Machine + * Learning}, + * year = {2012} * } * @endcode */ @@ -63,8 +63,8 @@ class RMSPropUpdate * The Initialize method is called by SGD Optimizer method before the start of * the iteration update process. * - * @param rows number of rows in the gradient matrix. - * @param cols number of columns in the gradient matrix. + * @param rows Number of rows in the gradient matrix. + * @param cols Number of columns in the gradient matrix. */ void Initialize(const size_t rows, const size_t cols) @@ -86,7 +86,8 @@ class RMSPropUpdate { meanSquaredGradient *= alpha; meanSquaredGradient += (1 - alpha) * (gradient % gradient); - iterate -= stepSize * gradient / (arma::sqrt(meanSquaredGradient) + epsilon); + iterate -= stepSize * gradient / (arma::sqrt(meanSquaredGradient) + + epsilon); } //! Get the value used to initialise the squared gradient parameter. diff --git a/src/mlpack/core/optimizers/sgd/update_policies/momentum_update.hpp b/src/mlpack/core/optimizers/sgd/update_policies/momentum_update.hpp index 2947c355f9..1ea0856034 100644 --- a/src/mlpack/core/optimizers/sgd/update_policies/momentum_update.hpp +++ b/src/mlpack/core/optimizers/sgd/update_policies/momentum_update.hpp @@ -42,23 +42,23 @@ namespace optimization { * * @code * @article{rumelhart1988learning, - * title={Learning representations by back-propagating errors}, - * author={Rumelhart, David E. and Hinton, Geoffrey E. and - * Williams, Ronald J.}, - * journal={Cognitive Modeling}, - * volume={5}, - * number={3}, - * pages={1}, - * year={1988} + * title = {Learning representations by back-propagating errors}, + * author = {Rumelhart, David E. and Hinton, Geoffrey E. and + * Williams, Ronald J.}, + * journal = {Cognitive Modeling}, + * volume = {5}, + * number = {3}, + * pages = {1}, + * year = {1988} * } * * @code * @book{Goodfellow-et-al-2016, - * title={Deep Learning}, - * author={Ian Goodfellow and Yoshua Bengio and Aaron Courville}, - * publisher={MIT Press}, - * note={\url{http://www.deeplearningbook.org}}, - * year={2016} + * title = {Deep Learning}, + * author = {Ian Goodfellow and Yoshua Bengio and Aaron Courville}, + * publisher = {MIT Press}, + * note = {\url{http://www.deeplearningbook.org}}, + * year = {2016} * } */ class MomentumUpdate @@ -78,14 +78,14 @@ class MomentumUpdate * matrix is initialized to the zeros matrix with the same size as the * gradient matrix (see mlpack::optimization::SGD::Optimizer ) * - * @param n_rows number of rows in the gradient matrix. - * @param n_cols number of columns in the gradient matrix. + * @param rows Number of rows in the gradient matrix. + * @param cols Number of columns in the gradient matrix. */ - void Initialize(const size_t n_rows, - const size_t n_cols) + void Initialize(const size_t rows, + const size_t cols) { //Initialize am empty velocity matrix. - velocity = arma::zeros(n_rows, n_cols); + velocity = arma::zeros(rows, cols); } /** diff --git a/src/mlpack/core/optimizers/sgd/update_policies/vanilla_update.hpp b/src/mlpack/core/optimizers/sgd/update_policies/vanilla_update.hpp index 1bd85bf1c3..d76264864b 100644 --- a/src/mlpack/core/optimizers/sgd/update_policies/vanilla_update.hpp +++ b/src/mlpack/core/optimizers/sgd/update_policies/vanilla_update.hpp @@ -36,10 +36,10 @@ class VanillaUpdate * the iteration update process. The vanilla update doesn't initialize * anything. * - * @param n_rows number of rows in the gradient matrix. - * @param n_cols number of columns in the gradient matrix. + * @param rows Number of rows in the gradient matrix. + * @param cols Number of columns in the gradient matrix. */ - void Initialize(const size_t /* n_rows */, const size_t /* n_cols */) + void Initialize(const size_t /* rows */, const size_t /* cols */) { /* Do nothing. */ } /** diff --git a/src/mlpack/core/optimizers/smorms3/smorms3.hpp b/src/mlpack/core/optimizers/smorms3/smorms3.hpp index 41643c8023..d8b6749707 100644 --- a/src/mlpack/core/optimizers/smorms3/smorms3.hpp +++ b/src/mlpack/core/optimizers/smorms3/smorms3.hpp @@ -31,16 +31,16 @@ namespace optimization { * * @code * @misc{Funk2015, - * author = {Simon Funk}, - * title = {RMSprop loses to SMORMS3 - Beware the Epsilon!}, - * year = {2015} - * url = {http://sifter.org/~simon/journal/20150420.html} + * author = {Simon Funk}, + * title = {RMSprop loses to SMORMS3 - Beware the Epsilon!}, + * year = {2015} + * url = {http://sifter.org/~simon/journal/20150420.html} * } * @endcode * * - * For SMORMS3 to work, a DecomposableFunctionType template parameter is required. - * This class must implement the following function: + * For SMORMS3 to work, a DecomposableFunctionType template parameter is + * required. This class must implement the following function: * * size_t NumFunctions(); * double Evaluate(const arma::mat& coordinates, const size_t i); @@ -57,7 +57,7 @@ namespace optimization { * is held internally in the DecomposableFunctionType). * * @tparam DecomposableFunctionType Decomposable objective function type to be - * minimized. + * minimized. */ template class SMORMS3 @@ -73,7 +73,8 @@ class SMORMS3 * * @param function Function to be optimized (minimized). * @param stepSize Step size for each iteration. - * @param epsilon Value used to initialise the mean squared gradient parameter. + * @param epsilon Value used to initialise the mean squared gradient + * parameter. * @param maxIterations Maximum number of iterations allowed (0 means no * limit). * @param tolerance Maximum absolute tolerance to terminate algorithm. @@ -81,14 +82,14 @@ class SMORMS3 * function is visited in linear order. */ SMORMS3(DecomposableFunctionType& function, - const double stepSize = 0.001, - const double epsilon = 1e-16, - const size_t maxIterations = 100000, - const double tolerance = 1e-5, - const bool shuffle = true); + const double stepSize = 0.001, + const double epsilon = 1e-16, + const size_t maxIterations = 100000, + const double tolerance = 1e-5, + const bool shuffle = true); /** - * Optimize the given function using SMORMS3. The given starting point will + * Optimize the given function using SMORMS3. The given starting point will * be modified to store the finishing point of the algorithm, and the final * objective value is returned. * @@ -98,7 +99,10 @@ class SMORMS3 double Optimize(arma::mat& iterate) { return optimizer.Optimize(iterate); } //! Get the instantiated function to be optimized. - const DecomposableFunctionType& Function() const { return optimizer.Function(); } + const DecomposableFunctionType& Function() const + { + return optimizer.Function(); + } //! Modify the instantiated function. DecomposableFunctionType& Function() { return optimizer.Function(); } diff --git a/src/mlpack/core/optimizers/smorms3/smorms3_update.hpp b/src/mlpack/core/optimizers/smorms3/smorms3_update.hpp index fbc534926b..3cf35eb7df 100644 --- a/src/mlpack/core/optimizers/smorms3/smorms3_update.hpp +++ b/src/mlpack/core/optimizers/smorms3/smorms3_update.hpp @@ -26,10 +26,10 @@ namespace optimization { * * @code * @misc{Funk2015, - * author = {Simon Funk}, - * title = {RMSprop loses to SMORMS3 - Beware the Epsilon!}, - * year = {2015} - * url = {http://sifter.org/~simon/journal/20150420.html} + * author = {Simon Funk}, + * title = {RMSprop loses to SMORMS3 - Beware the Epsilon!}, + * year = {2015} + * url = {http://sifter.org/~simon/journal/20150420.html} * } * @endcode */ @@ -40,23 +40,18 @@ class SMORMS3Update /** * Construct the SMORMS3 update policy with given epsilon parameter. * - * @param epsilon Value used to initialise the mean squared gradient parameter. + * @param epsilon Value used to initialise the mean squared gradient + * parameter. */ - SMORMS3Update(const double epsilon = 1e-16) : - epsilon(epsilon) + SMORMS3Update(const double epsilon = 1e-16) : epsilon(epsilon) { /* Do nothing. */ } - //! Get the value used to initialise the mean squared gradient parameter. - double Epsilon() const { return epsilon; } - //! Modify the value used to initialise the mean squared gradient parameter. - double& Epsilon() { return epsilon; } - /** * The Initialize method is called by SGD::Optimize method with UpdatePolicy * SMORMS3Update before the start of the iteration update process. * - * @param rows number of rows in the gradient matrix. - * @param cols number of columns in the gradient matrix. + * @param rows Number of rows in the gradient matrix. + * @param cols Number of columns in the gradient matrix. */ void Initialize(const size_t rows, const size_t cols) @@ -96,10 +91,16 @@ class SMORMS3Update mem %= (1 - x); mem += 1; } + + //! Get the value used to initialise the mean squared gradient parameter. + double Epsilon() const { return epsilon; } + //! Modify the value used to initialise the mean squared gradient parameter. + double& Epsilon() { return epsilon; } + private: //! The value used to initialise the mean squared gradient parameter. double epsilon; - + // The parameters mem, g and g2. arma::mat mem, g, g2; };