apaam/mlpack
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #972 from ivmarkp/adam

Browse Source 
Adam and AdaMax update policy implementation
This commit is contained in:
Ryan Curtin
2017-04-14 14:46:47 -04:00
committed by GitHub
parent dec8a4ffcb e7bfbb19df
commit dcfd118ebe
8 changed files with 372 additions and 223 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/mlpack/core/optimizers/adam/CMakeLists.txt
+2
View File
@@ -1,6 +1,8 @@
set(SOURCES
adam.hpp
adam_impl.hpp
adam_update.hpp
adamax_update.hpp
)
set(DIR_SRCS)
src/mlpack/core/optimizers/adam/adam.hpp
+48 -63
View File
@@ -20,6 +20,10 @@
#include <mlpack/prereqs.hpp>
#include <mlpack/core/optimizers/sgd/sgd.hpp>
#include "adam_update.hpp"
#include "adamax_update.hpp"
namespace mlpack {
namespace optimization {
@@ -36,7 +40,8 @@ namespace optimization {
* author = {Diederik P. Kingma and Jimmy Ba},
* title = {Adam: {A} Method for Stochastic Optimization},
* journal = {CoRR},
* year = {2014}
* year = {2014},
* url = {http://arxiv.org/abs/1412.6980}
* }
* @endcode
*
@@ -60,9 +65,13 @@ namespace optimization {
*
* @tparam DecomposableFunctionType Decomposable objective function type to be
* minimized.
* @tparam UpdateRule Adam optimizer update rule to be used.
*/
template<typename DecomposableFunctionType>
class Adam
template<
typename DecomposableFunctionType,
typename UpdateRule = AdamUpdate
>
class AdamType
{
public:
/**
@@ -84,18 +93,15 @@ class Adam
* @param tolerance Maximum absolute tolerance to terminate algorithm.
* @param shuffle If true, the function order is shuffled; otherwise, each
* function is visited in linear order.
* @param adaMax If true, then the AdaMax optimizer is used; otherwise, by
* default the Adam optimizer is used.
*/
Adam(DecomposableFunctionType& function,
const double stepSize = 0.001,
const double beta1 = 0.9,
const double beta2 = 0.999,
const double eps = 1e-8,
const size_t maxIterations = 100000,
const double tolerance = 1e-5,
const bool shuffle = true,
const bool adaMax = false);
AdamType(DecomposableFunctionType& function,
const double stepSize = 0.001,
const double beta1 = 0.9,
const double beta2 = 0.999,
const double eps = 1e-8,
const size_t maxIterations = 100000,
const double tolerance = 1e-5,
const bool shuffle = true);
/**
* Optimize the given function using Adam. The given starting point will be
@@ -105,83 +111,62 @@ class Adam
* @param iterate Starting point (will be modified).
* @return Objective value of the final point.
*/
double Optimize(arma::mat& iterate);
double Optimize(arma::mat& iterate){ return optimizer.Optimize(iterate); }
//! Get the instantiated function to be optimized.
const DecomposableFunctionType& Function() const { return function; }
const DecomposableFunctionType& Function() const
{
return optimizer.Function();
}
//! Modify the instantiated function.
DecomposableFunctionType& Function() { return function; }
DecomposableFunctionType& Function() { return optimizer.Function(); }
//! Get the step size.
double StepSize() const { return stepSize; }
double StepSize() const { return optimizer.StepSize(); }
//! Modify the step size.
double& StepSize() { return stepSize; }
double& StepSize() { return optimizer.StepSize(); }
//! Get the smoothing parameter.
double Beta1() const { return beta1; }
double Beta1() const { return optimizer.UpdatePolicy().Beta1(); }
//! Modify the smoothing parameter.
double& Beta1() { return beta1; }
double& Beta1() { return optimizer.UpdatePolicy().Beta1(); }
//! Get the second moment coefficient.
double Beta2() const { return beta2; }
double Beta2() const { return optimizer.UpdatePolicy().Beta2(); }
//! Modify the second moment coefficient.
double& Beta2() { return beta2; }
double& Beta2() { return optimizer.UpdatePolicy().Beta2(); }
//! Get the value used to initialise the mean squared gradient parameter.
double Epsilon() const { return eps; }
double Epsilon() const { return optimizer.UpdatePolicy().Epsilon(); }
//! Modify the value used to initialise the mean squared gradient parameter.
double& Epsilon() { return eps; }
double& Epsilon() { return optimizer.UpdatePolicy().Epsilon(); }
//! Get the maximum number of iterations (0 indicates no limit).
size_t MaxIterations() const { return maxIterations; }
size_t MaxIterations() const { return optimizer.MaxIterations(); }
//! Modify the maximum number of iterations (0 indicates no limit).
size_t& MaxIterations() { return maxIterations; }
size_t& MaxIterations() { return optimizer.MaxIterations(); }
//! Get the tolerance for termination.
double Tolerance() const { return tolerance; }
double Tolerance() const { return optimizer.Tolerance(); }
//! Modify the tolerance for termination.
double& Tolerance() { return tolerance; }
double& Tolerance() { return optimizer.Tolerance(); }
//! Get whether or not the individual functions are shuffled.
bool Shuffle() const { return shuffle; }
bool Shuffle() const { return optimizer.Shuffle(); }
//! Modify whether or not the individual functions are shuffled.
bool& Shuffle() { return shuffle; }
//! Get whether or not the AdaMax optimizer is specified.
bool AdaMax() const { return adaMax; }
//! Modify wehther or not the AdaMax optimizer is to be used.
bool& AdaMax() { return adaMax; }
bool& Shuffle() { return optimizer.Shuffle(); }
private:
//! The instantiated function.
DecomposableFunctionType& function;
//! The step size for each example.
double stepSize;
//! Exponential decay rate for the first moment estimates.
double beta1;
//! Exponential decay rate for the weighted infinity norm estimates.
double beta2;
//! The value used to initialise the mean squared gradient parameter.
double eps;
//! The maximum number of allowed iterations.
size_t maxIterations;
//! The tolerance for termination.
double tolerance;
//! Controls whether or not the individual functions are shuffled when
//! iterating.
bool shuffle;
//! Specifies whether or not the AdaMax optimizer is to be used.
bool adaMax;
//! The Stochastic Gradient Descent object with Adam policy.
SGD<DecomposableFunctionType, UpdateRule> optimizer;
};
template<typename DecomposableFunctionType>
using Adam = AdamType<DecomposableFunctionType, AdamUpdate>;
template<typename DecomposableFunctionType>
using AdaMax = AdamType<DecomposableFunctionType, AdaMaxUpdate>;
} // namespace optimization
} // namespace mlpack
src/mlpack/core/optimizers/adam/adam_impl.hpp
+18 -153
View File
@@ -21,161 +21,26 @@
namespace mlpack {
namespace optimization {
template<typename DecomposableFunctionType>
Adam<DecomposableFunctionType>::Adam(DecomposableFunctionType& function,
const double stepSize,
const double beta1,
const double beta2,
const double eps,
const size_t maxIterations,
const double tolerance,
const bool shuffle,
const bool adaMax) :
function(function),
stepSize(stepSize),
beta1(beta1),
beta2(beta2),
eps(eps),
maxIterations(maxIterations),
tolerance(tolerance),
shuffle(shuffle),
adaMax(adaMax)
template<typename DecomposableFunctionType, typename UpdateRule>
AdamType<DecomposableFunctionType, UpdateRule>::AdamType(
DecomposableFunctionType& function,
const double stepSize,
const double beta1,
const double beta2,
const double epsilon,
const size_t maxIterations,
const double tolerance,
const bool shuffle) :
optimizer(function,
stepSize,
maxIterations,
tolerance,
shuffle,
UpdateRule(epsilon,
beta1,
beta2))
{ /* Nothing to do. */ }
//! Optimize the function (minimize).
template<typename DecomposableFunctionType>
double Adam<DecomposableFunctionType>::Optimize(arma::mat& iterate)
{
// Find the number of functions to use.
const size_t numFunctions = function.NumFunctions();
// This is used only if shuffle is true.
arma::Col<size_t> visitationOrder;
if (shuffle)
visitationOrder = arma::shuffle(arma::linspace<arma::Col<size_t>>(0,
(numFunctions - 1), numFunctions));
// To keep track of where we are and how things are going.
size_t currentFunction = 0;
double overallObjective = 0;
double lastObjective = DBL_MAX;
// Calculate the first objective function.
for (size_t i = 0; i < numFunctions; ++i)
overallObjective += function.Evaluate(iterate, i);
// Now iterate!
arma::mat gradient(iterate.n_rows, iterate.n_cols);
// Exponential moving average of gradient values.
arma::mat m = arma::zeros<arma::mat>(iterate.n_rows, iterate.n_cols);
/**
* Initialize either the exponentially weighted infinity norm for AdaMax
* optimizer (u) or exponential moving average of squared gradient values
* for Adam optimizer (v).
*/
arma::mat u, v;
if (adaMax)
{
u = arma::zeros<arma::mat>(iterate.n_rows, iterate.n_cols);
}
else
{
v = arma::zeros<arma::mat>(iterate.n_rows, iterate.n_cols);
}
for (size_t i = 1; i != maxIterations; ++i, ++currentFunction)
{
// Is this iteration the start of a sequence?
if ((currentFunction % numFunctions) == 0)
{
// Output current objective function.
Log::Info << "Adam: iteration " << i << ", objective " << overallObjective
<< "." << std::endl;
if (std::isnan(overallObjective) || std::isinf(overallObjective))
{
Log::Warn << "Adam: converged to " << overallObjective
<< "; terminating with failure. Try a smaller step size?"
<< std::endl;
return overallObjective;
}
if (std::abs(lastObjective - overallObjective) < tolerance)
{
Log::Info << "Adam: minimized within tolerance " << tolerance << "; "
<< "terminating optimization." << std::endl;
return overallObjective;
}
// Reset the counter variables.
lastObjective = overallObjective;
overallObjective = 0;
currentFunction = 0;
if (shuffle) // Determine order of visitation.
visitationOrder = arma::shuffle(visitationOrder);
}
// Evaluate the gradient for this iteration.
if (shuffle)
function.Gradient(iterate, visitationOrder[currentFunction], gradient);
else
function.Gradient(iterate, currentFunction, gradient);
// And update the iterate.
m *= beta1;
m += (1 - beta1) * gradient;
if (adaMax)
{
// Update the exponentially weighted infinity norm.
u *= beta2;
u = arma::max(u, arma::abs(gradient));
}
else
{
v *= beta2;
v += (1 - beta2) * (gradient % gradient);
}
const double biasCorrection1 = 1.0 - std::pow(beta1, (double) i);
const double biasCorrection2 = 1.0 - std::pow(beta2, (double) i);
if (adaMax)
{
if (biasCorrection1 != 0.0)
iterate -= (stepSize / biasCorrection1 * m / (u + eps));
}
else
{
/**
* It should be noted that the term, m / (arma::sqrt(v) + eps), in the
* following expression is an approximation of the following actual term;
* m / (arma::sqrt(v) + (arma::sqrt(biasCorrection2) * eps).
*/
iterate -= (stepSize * std::sqrt(biasCorrection2) / biasCorrection1) *
m / (arma::sqrt(v) + eps);
}
// Now add that to the overall objective function.
if (shuffle)
overallObjective += function.Evaluate(iterate,
visitationOrder[currentFunction]);
else
overallObjective += function.Evaluate(iterate, currentFunction);
}
Log::Info << "Adam: maximum iterations (" << maxIterations << ") reached; "
<< "terminating optimization." << std::endl;
// Calculate final objective.
overallObjective = 0;
for (size_t i = 0; i < numFunctions; ++i)
overallObjective += function.Evaluate(iterate, i);
return overallObjective;
}
} // namespace optimization
} // namespace mlpack
src/mlpack/core/optimizers/adam/adam_update.hpp
+149
View File
@@ -0,0 +1,149 @@
/**
* @file adam_update.hpp
* @author Ryan Curtin
* @author Vasanth Kalingeri
* @author Marcus Edel
* @author Vivek Pal
*
* Adam optimizer. Adam is an an algorithm for first-order gradient-based
* optimization of stochastic objective functions, based on adaptive estimates
* of lower-order moments.
*
* mlpack is free software; you may redistribute it and/or modify it under the
* terms of the 3-clause BSD license. You should have received a copy of the
* 3-clause BSD license along with mlpack. If not, see
* http://www.opensource.org/licenses/BSD-3-Clause for more information.
*/
#ifndef MLPACK_CORE_OPTIMIZERS_ADAM_ADAM_UPDATE_HPP
#define MLPACK_CORE_OPTIMIZERS_ADAM_ADAM_UPDATE_HPP
#include <mlpack/prereqs.hpp>
namespace mlpack {
namespace optimization {
/**
* Adam is an optimizer that computes individual adaptive learning rates for
* different parameters from estimates of first and second moments of the
* gradients as given in the section 7 of the following paper.
*
* For more information, see the following.
*
* @code
* @article{Kingma2014,
* author = {Diederik P. Kingma and Jimmy Ba},
* title = {Adam: {A} Method for Stochastic Optimization},
* journal = {CoRR},
* year = {2014},
* url = {http://arxiv.org/abs/1412.6980}
* }
* @endcode
*/
class AdamUpdate
{
public:
/**
* Construct the Adam update policy with the given parameters.
*
* @param epsilon The epsilon value used to initialise the squared gradient
* parameter.
* @param beta1 The smoothing parameter.
* @param beta2 The second moment coefficient.
*/
AdamUpdate(const double epsilon = 1e-8,
const double beta1 = 0.9,
const double beta2 = 0.999) :
epsilon(epsilon),
beta1(beta1),
beta2(beta2),
iteration(0)
{
// Nothing to do.
}
/**
* The Initialize method is called by SGD Optimizer method before the start of
* the iteration update process.
*
* @param rows number of rows in the gradient matrix.
* @param cols number of columns in the gradient matrix.
*/
void Initialize(const size_t rows,
const size_t cols)
{
m = arma::zeros<arma::mat>(rows, cols);
v = arma::zeros<arma::mat>(rows, cols);
}
/**
* Update step for Adam.
*
* @param iterate Parameters that minimize the function.
* @param stepSize Step size to be used for the given iteration.
* @param gradient The gradient matrix.
*/
void Update(arma::mat& iterate,
const double stepSize,
const arma::mat& gradient)
{
// Increment the iteration counter variable.
++iteration;
// And update the iterate.
m *= beta1;
m += (1 - beta1) * gradient;
v *= beta2;
v += (1 - beta2) * (gradient % gradient);
const double biasCorrection1 = 1.0 - std::pow(beta1, (double) iteration);
const double biasCorrection2 = 1.0 - std::pow(beta2, (double) iteration);
/**
* It should be noted that the term, m / (arma::sqrt(v) + eps), in the
* following expression is an approximation of the following actual term;
* m / (arma::sqrt(v) + (arma::sqrt(biasCorrection2) * eps).
*/
iterate -= (stepSize * std::sqrt(biasCorrection2) / biasCorrection1) *
m / (arma::sqrt(v) + epsilon);
}
//! Get the value used to initialise the squared gradient parameter.
double Epsilon() const { return epsilon; }
//! Modify the value used to initialise the squared gradient parameter.
double& Epsilon() { return epsilon; }
//! Get the smoothing parameter.
double Beta1() const { return beta1; }
//! Modify the smoothing parameter.
double& Beta1() { return beta1; }
//! Get the second moment coefficient.
double Beta2() const { return beta2; }
//! Modify the second moment coefficient.
double& Beta2() { return beta2; }
private:
// The epsilon value used to initialise the squared gradient parameter.
double epsilon;
// The smoothing parameter.
double beta1;
// The second moment coefficient.
double beta2;
// The exponential moving average of gradient values.
arma::mat m;
// The exponential moving average of squared gradient values.
arma::mat v;
// The number of iterations.
double iteration;
};
} // namespace optimization
} // namespace mlpack
#endif
src/mlpack/core/optimizers/adam/adamax_update.hpp
+146
View File
@@ -0,0 +1,146 @@
/**
* @file adamax_update.hpp
* @author Ryan Curtin
* @author Vasanth Kalingeri
* @author Marcus Edel
* @author Vivek Pal
*
* AdaMax update rule. Adam is an an algorithm for first-order gradient-
* -based optimization of stochastic objective functions, based on adaptive
* estimates of lower-order moments. AdaMax is simply a variant of Adam based
* on the infinity norm.
*
* mlpack is free software; you may redistribute it and/or modify it under the
* terms of the 3-clause BSD license. You should have received a copy of the
* 3-clause BSD license along with mlpack. If not, see
* http://www.opensource.org/licenses/BSD-3-Clause for more information.
*/
#ifndef MLPACK_CORE_OPTIMIZERS_ADAM_ADAMAX_UPDATE_HPP
#define MLPACK_CORE_OPTIMIZERS_ADAM_ADAMAX_UPDATE_HPP
#include <mlpack/prereqs.hpp>
namespace mlpack {
namespace optimization {
/**
* AdaMax is a variant of Adam, an optimizer that computes individual adaptive
* learning rates for different parameters from estimates of first and second
* moments of the gradients.based on the infinity norm as given in the section
* 7 of the following paper.
*
* For more information, see the following.
*
* @code
* @article{Kingma2014,
* author = {Diederik P. Kingma and Jimmy Ba},
* title = {Adam: {A} Method for Stochastic Optimization},
* journal = {CoRR},
* year = {2014},
* url = {http://arxiv.org/abs/1412.6980}
* }
* @endcode
*/
class AdaMaxUpdate
{
public:
/**
* Construct the AdaMax update policy with the given parameters.
*
* @param epsilon The epsilon value used to initialise the squared gradient
* parameter.
* @param beta1 The smoothing parameter.
* @param beta2 The second moment coefficient.
*/
AdaMaxUpdate(const double epsilon = 1e-8,
const double beta1 = 0.9,
const double beta2 = 0.999) :
epsilon(epsilon),
beta1(beta1),
beta2(beta2),
iteration(0)
{
// Nothing to do.
}
/**
* The Initialize method is called by SGD Optimizer method before the start of
* the iteration update process.
*
* @param rows number of rows in the gradient matrix.
* @param cols number of columns in the gradient matrix.
*/
void Initialize(const size_t rows,
const size_t cols)
{
m = arma::zeros<arma::mat>(rows, cols);
u = arma::zeros<arma::mat>(rows, cols);
}
/**
* Update step for Adam.
*
* @param iterate Parameters that minimize the function.
* @param stepSize Step size to be used for the given iteration.
* @param gradient The gradient matrix.
*/
void Update(arma::mat& iterate,
const double stepSize,
const arma::mat& gradient)
{
// Increment the iteration counter variable.
++iteration;
// And update the iterate.
m *= beta1;
m += (1 - beta1) * gradient;
// Update the exponentially weighted infinity norm.
u *= beta2;
u = arma::max(u, arma::abs(gradient));
const double biasCorrection1 = 1.0 - std::pow(beta1, iteration);
if (biasCorrection1 != 0)
iterate -= (stepSize / biasCorrection1 * m / (u + epsilon));
}
//! Get the value used to initialise the squared gradient parameter.
double Epsilon() const { return epsilon; }
//! Modify the value used to initialise the squared gradient parameter.
double& Epsilon() { return epsilon; }
//! Get the smoothing parameter.
double Beta1() const { return beta1; }
//! Modify the smoothing parameter.
double& Beta1() { return beta1; }
//! Get the second moment coefficient.
double Beta2() const { return beta2; }
//! Modify the second moment coefficient.
double& Beta2() { return beta2; }
private:
// The epsilon value used to initialise the squared gradient parameter.
double epsilon;
// The smoothing parameter.
double beta1;
// The second moment coefficient.
double beta2;
// The exponential moving average of gradient values.
arma::mat m;
// The exponentially weighted infinity norm.
arma::mat u;
// The number of iterations.
double iteration;
};
} // namespace optimization
} // namespace mlpack
#endif
src/mlpack/methods/logistic_regression/logistic_regression.hpp
+1 -1
View File
@@ -119,7 +119,7 @@ class LogisticRegression
* @param responses Outputs results from input training variables.
*/
template<
template<typename> class OptimizerType = mlpack::optimization::L_BFGS
template<typename...> class OptimizerType = mlpack::optimization::L_BFGS
>
void Train(const MatType& predictors,
const arma::Row<size_t>& responses);
src/mlpack/methods/logistic_regression/logistic_regression_impl.hpp
+1 -1
View File
@@ -66,7 +66,7 @@ LogisticRegression<MatType>::LogisticRegression(
}
template<typename MatType>
template<template<typename> class OptimizerType>
template<template<typename...> class OptimizerType>
void LogisticRegression<MatType>::Train(const MatType& predictors,
const arma::Row<size_t>& responses)
{
src/mlpack/tests/adam_test.cpp
+7 -5
View File
@@ -36,7 +36,8 @@ BOOST_AUTO_TEST_SUITE(AdamTest);
BOOST_AUTO_TEST_CASE(SimpleAdamTestFunction)
{
SGDTestFunction f;
Adam<SGDTestFunction> optimizer(f, 1e-3, 0.9, 0.999, 1e-8, 5000000, 1e-9, true);
Adam<SGDTestFunction> optimizer(f, 1e-3, 0.9, 0.999, 1e-8, 5000000, 1e-9,
true);
arma::mat coordinates = f.GetInitialPoint();
optimizer.Optimize(coordinates);
@@ -52,8 +53,8 @@ BOOST_AUTO_TEST_CASE(SimpleAdamTestFunction)
BOOST_AUTO_TEST_CASE(SimpleAdaMaxTestFunction)
{
SGDTestFunction f;
Adam<SGDTestFunction> optimizer(f, 2e-3, 0.9, 0.999, 1e-8, 5000000, 1e-9, true
,true);
AdaMax<SGDTestFunction> optimizer(f, 2e-3, 0.9, 0.999, 1e-8, 5000000, 1e-9,
true);
arma::mat coordinates = f.GetInitialPoint();
optimizer.Optimize(coordinates);
@@ -174,8 +175,9 @@ BOOST_AUTO_TEST_CASE(AdaMaxLogisticRegressionTest)
LogisticRegression<> lr(shuffledData.n_rows, 0.5);
LogisticRegressionFunction<> lrf(shuffledData, shuffledResponses, 0.5);
Adam<LogisticRegressionFunction<> > adamax(lrf, 1e-3, 0.9, 0.999, 1e-8, 5000000,
1e-9, true, true);
AdaMax<LogisticRegressionFunction<> > adamax(lrf, 1e-3, 0.9, 0.999, 1e-8,
5000000, 1e-9, true);
lr.Train(adamax);
// Ensure that the error is close to zero.