apaam/mlpack
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add AdaDelta policy class

Browse Source
This commit is contained in:
Abhinav Moudgil
2017-03-25 15:46:35 +05:30
parent c134852855
commit 7890547e2a
7 changed files with 194 additions and 180 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/mlpack/core/optimizers/CMakeLists.txt
+1 -1
View File
@@ -1,5 +1,5 @@
set(DIRS
adadelta
ada_delta
ada_grad
adam
aug_lagrangian
src/mlpack/core/optimizers/adadelta/CMakeLists.txt → src/mlpack/core/optimizers/ada_delta/CMakeLists.txt
+1
View File
@@ -1,6 +1,7 @@
set(SOURCES
ada_delta.hpp
ada_delta_impl.hpp
ada_delta_update.hpp
)
set(DIR_SRCS)
src/mlpack/core/optimizers/adadelta/ada_delta.hpp → src/mlpack/core/optimizers/ada_delta/ada_delta.hpp
+27 -36
View File
@@ -2,10 +2,11 @@
* @file ada_delta.hpp
* @author Ryan Curtin
* @author Vasanth Kalingeri
* @author Abhinav Moudgil
*
* Implementation of the Adadelta optimizer. Adadelta is an optimizer that
* Implementation of the AdaDelta optimizer. AdaDelta is an optimizer that
* dynamically adapts over time using only first order information.
* Additionally, Adadelta requires no manual tuning of a learning rate.
* Additionally, AdaDelta requires no manual tuning of a learning rate.
*
* mlpack is free software; you may redistribute it and/or modify it under the
* terms of the 3-clause BSD license. You should have received a copy of the
@@ -16,12 +17,14 @@
#define __MLPACK_CORE_OPTIMIZERS_ADADELTA_ADA_DELTA_HPP
#include <mlpack/prereqs.hpp>
#include <mlpack/core/optimizers/sgd/sgd.hpp>
#include "ada_delta_update.hpp"
namespace mlpack {
namespace optimization {
/**
* Adadelta is an optimizer that uses two ideas to improve upon the two main
* AdaDelta is an optimizer that uses two ideas to improve upon the two main
* drawbacks of the Adagrad method:
*
* - Accumulate Over Window
@@ -38,7 +41,6 @@ namespace optimization {
* }
* @endcode
*
* For AdaDelta to work, a DecomposableFunctionType template parameter is
* required. This class must implement the following function:
*
@@ -81,6 +83,7 @@ class AdaDelta
* function is visited in linear order.
*/
AdaDelta(DecomposableFunctionType& function,
const double stepSize = 1.0,
const double rho = 0.95,
const double eps = 1e-6,
const size_t maxIterations = 100000,
@@ -95,57 +98,46 @@ class AdaDelta
* @param iterate Starting point (will be modified).
* @return Objective value of the final point.
*/
double Optimize(arma::mat& iterate);
double Optimize(arma::mat& iterate) { return optimizer.Optimize(iterate); }
//! Get the instantiated function to be optimized.
const DecomposableFunctionType& Function() const { return function; }
const DecomposableFunctionType& Function() const { return optimizer.Function(); }
//! Modify the instantiated function.
DecomposableFunctionType& Function() { return function; }
DecomposableFunctionType& Function() { return optimizer.Function(); }
//! Get the step size.
double StepSize() const { return optimizer.StepSize(); }
//! Modify the step size.
double& StepSize() { return optimizer.StepSize(); }
//! Get the smoothing parameter.
double Rho() const { return rho; }
double Rho() const { return optimizer.UpdatePolicy().Rho(); }
//! Modify the smoothing parameter.
double& Rho() { return rho; }
double& Rho() { return optimizer.UpdatePolicy().Rho(); }
//! Get the value used to initialise the mean squared gradient parameter.
double Epsilon() const { return eps; }
double Epsilon() const { return optimizer.UpdatePolicy().Epsilon(); }
//! Modify the value used to initialise the mean squared gradient parameter.
double& Epsilon() { return eps; }
double& Epsilon() { return optimizer.UpdatePolicy().Epsilon(); }
//! Get the maximum number of iterations (0 indicates no limit).
size_t MaxIterations() const { return maxIterations; }
size_t MaxIterations() const { return optimizer.MaxIterations(); }
//! Modify the maximum number of iterations (0 indicates no limit).
size_t& MaxIterations() { return maxIterations; }
size_t& MaxIterations() { return optimizer.MaxIterations(); }
//! Get the tolerance for termination.
double Tolerance() const { return tolerance; }
double Tolerance() const { return optimizer.Tolerance(); }
//! Modify the tolerance for termination.
double& Tolerance() { return tolerance; }
double& Tolerance() { return optimizer.Tolerance(); }
//! Get whether or not the individual functions are shuffled.
bool Shuffle() const { return shuffle; }
bool Shuffle() const { return optimizer.Shuffle(); }
//! Modify whether or not the individual functions are shuffled.
bool& Shuffle() { return shuffle; }
bool& Shuffle() { return optimizer.Shuffle(); }
private:
//! The instantiated function.
DecomposableFunctionType& function;
//! The smoothing parameter.
double rho;
//! The value used to initialise the mean squared gradient parameter.
double eps;
//! The maximum number of allowed iterations.
size_t maxIterations;
//! The tolerance for termination.
double tolerance;
//! Controls whether or not the individual functions are shuffled when
//! iterating.
bool shuffle;
//! The Stochastic Gradient Descent object with AdaDelta policy.
SGD<DecomposableFunctionType, AdaDeltaUpdate> optimizer;
};
} // namespace optimization
@@ -155,4 +147,3 @@ class AdaDelta
#include "ada_delta_impl.hpp"
#endif
src/mlpack/core/optimizers/ada_delta/ada_delta_impl.hpp
+41
View File
@@ -0,0 +1,41 @@
/**
* @file ada_delta_impl.hpp
* @author Ryan Curtin
* @author Vasanth Kalingeri
* @author Abhinav Moudgil
*
* Implementation of the AdaDelta optimizer.
*
* mlpack is free software; you may redistribute it and/or modify it under the
* terms of the 3-clause BSD license. You should have received a copy of the
* 3-clause BSD license along with mlpack. If not, see
* http://www.opensource.org/licenses/BSD-3-Clause for more information.
*/
#ifndef MLPACK_CORE_OPTIMIZERS_ADADELTA_ADA_DELTA_IMPL_HPP
#define MLPACK_CORE_OPTIMIZERS_ADADELTA_ADA_DELTA_IMPL_HPP
#include "ada_delta.hpp"
namespace mlpack {
namespace optimization {
template<typename DecomposableFunctionType>
AdaDelta<DecomposableFunctionType>::AdaDelta(DecomposableFunctionType& function,
const double stepSize,
const double rho,
const double epsilon,
const size_t maxIterations,
const double tolerance,
const bool shuffle) :
optimizer(function,
stepSize,
maxIterations,
tolerance,
shuffle,
AdaDeltaUpdate(rho, epsilon))
{ /* Nothing to do. */ }
} // namespace optimization
} // namespace mlpack
#endif
src/mlpack/core/optimizers/ada_delta/ada_delta_update.hpp
+121
View File
@@ -0,0 +1,121 @@
/**
* @file ada_delta_update.hpp
* @author Abhinav Moudgil
*
* AdaDelta update for Stochastic Gradient Descent.
*
* mlpack is free software; you may redistribute it and/or modify it under the
* terms of the 3-clause BSD license. You should have received a copy of the
* 3-clause BSD license along with mlpack. If not, see
* http://www.opensource.org/licenses/BSD-3-Clause for more information.
*/
#ifndef MLPACK_CORE_OPTIMIZERS_SGD_ADA_DELTA_UPDATE_HPP
#define MLPACK_CORE_OPTIMIZERS_SGD_ADA_DELTA_UPDATE_HPP
#include <mlpack/prereqs.hpp>
namespace mlpack {
namespace optimization {
/**
* Implementation of the AdaDelta update policy. AdaDelta is an optimizer that
* uses two ideas to improve upon the two main drawbacks of the AdaGrad method:
*
* - Accumulate Over Window
* - Correct Units with Hessian Approximation
*
* For more information, see the following.
*
* @code
* @article{Zeiler2012,
* author = {Matthew D. Zeiler},
* title = {{ADADELTA:} An Adaptive Learning Rate Method},
* journal = {CoRR},
* year = {2012}
* }
* @endcode
*
*/
class AdaDeltaUpdate
{
public:
/**
* Construct the AdaDelta update policy with given rho and epsilon parameters.
*
* @param rho The smoothing parameter.
* @param epsilon The epsilon value used to initialise the squared gradient parameter.
*/
AdaDeltaUpdate(const double rho = 0.95, const double epsilon = 1e-8) :
rho(rho),
epsilon(epsilon)
{ /* Do nothing. */ };
/**
* The Initialize method is called by SGD Optimizer method before the start of the
* iteration update process. In AdaDelta update policy, the mean squared and the delta
* mean squared gradient matrices are initialized to the zeros matrix with the same
* size as gradient matrix (see mlpack::optimization::SGD::Optimizer )
*
* @param rows number of rows in the gradient matrix.
* @param cols number of columns in the gradient matrix.
*/
void Initialize(const size_t rows,
const size_t cols)
{
// Initialize empty matrices for mean sum of squares of parameter gradient.
meanSquaredGradient = arma::zeros<arma::mat>(rows, cols);
meanSquaredGradientDx = arma::zeros<arma::mat>(rows, cols);
}
/**
* Update step for SGD. The AdaDelta update dynamically adapts over time using only
* first order information. Additionally, AdaDelta requires no manual tuning
* of a learning rate.
*
* @param iterate Parameters that minimize the function.
* @param stepSize Step size to be used for the given iteration.
* @param gradient The gradient matrix.
*/
void Update(arma::mat& iterate,
const double stepSize,
const arma::mat& gradient)
{
// Accumulate gradient.
meanSquaredGradient *= rho;
meanSquaredGradient += (1 - rho) * (gradient % gradient);
arma::mat dx = arma::sqrt((meanSquaredGradientDx + eps) /
(meanSquaredGradient + eps)) % gradient;
// Accumulate updates.
meanSquaredGradientDx *= rho;
meanSquaredGradientDx += (1 - rho) * (dx % dx);
// Apply update.
iterate -= (stepSize * dx);
}
//! Get the smoothing parameter.
double Rho() const { return rho; }
//! Modify the smoothing parameter.
double& Rho() { return rho; }
//! Get the value used to initialise the mean squared gradient parameter.
double Epsilon() const { return epsilon; }
//! Modify the value used to initialise the mean squared gradient parameter.
double& Epsilon() { return epsilon; }
private:
// The smoothing parameter.
double rho;
// The epsilon value used to initialise the mean squared gradient parameter.
double epsilon;
// The mean squared gradient matrix.
arma::mat meanSquaredGradient;
// The delta mean squared gradient matrix.
arma::mat meanSquaredGradientDx;
};
} // namespace optimization
} // namespace mlpack
#endif
src/mlpack/core/optimizers/adadelta/ada_delta_impl.hpp
-141
View File
@@ -1,141 +0,0 @@
/**
* @file ada_delta_impl.hpp
* @author Ryan Curtin
* @author Vasanth Kalingeri
*
* Implementation of the Adadelta optimizer.
*
* mlpack is free software; you may redistribute it and/or modify it under the
* terms of the 3-clause BSD license. You should have received a copy of the
* 3-clause BSD license along with mlpack. If not, see
* http://www.opensource.org/licenses/BSD-3-Clause for more information.
*/
#ifndef __MLPACK_CORE_OPTIMIZERS_ADADELTA_ADA_DELTA_IMPL_HPP
#define __MLPACK_CORE_OPTIMIZERS_ADADELTA_ADA_DELTA_IMPL_HPP
#include "ada_delta.hpp"
namespace mlpack {
namespace optimization {
template<typename DecomposableFunctionType>
AdaDelta<DecomposableFunctionType>::AdaDelta(DecomposableFunctionType& function,
const double rho,
const double eps,
const size_t maxIterations,
const double tolerance,
const bool shuffle) :
function(function),
rho(rho),
eps(eps),
maxIterations(maxIterations),
tolerance(tolerance),
shuffle(shuffle)
{ /* Nothing to do. */ }
//! Optimize the function (minimize).
template<typename DecomposableFunctionType>
double AdaDelta<DecomposableFunctionType>::Optimize(arma::mat& iterate)
{
// Find the number of functions to use.
const size_t numFunctions = function.NumFunctions();
// This is used only if shuffle is true.
arma::Col<size_t> visitationOrder;
if (shuffle)
visitationOrder = arma::shuffle(arma::linspace<arma::Col<size_t>>(0,
(numFunctions - 1), numFunctions));
// To keep track of where we are and how things are going.
size_t currentFunction = 0;
double overallObjective = 0;
double lastObjective = DBL_MAX;
// Calculate the first objective function.
for (size_t i = 0; i < numFunctions; ++i)
overallObjective += function.Evaluate(iterate, i);
// Now iterate!
arma::mat gradient(iterate.n_rows, iterate.n_cols);
// Leaky sum of squares of parameter gradient.
arma::mat meanSquaredGradient = arma::zeros<arma::mat>(iterate.n_rows,
iterate.n_cols);
// Leaky sum of squares of parameter gradient.
arma::mat meanSquaredGradientDx = arma::zeros<arma::mat>(iterate.n_rows,
iterate.n_cols);
for (size_t i = 1; i != maxIterations; ++i, ++currentFunction)
{
// Is this iteration the start of a sequence?
if ((currentFunction % numFunctions) == 0)
{
// Output current objective function.
Log::Info << "AdaDelta: iteration " << i << ", objective "
<< overallObjective << "." << std::endl;
if (std::isnan(overallObjective) || std::isinf(overallObjective))
{
Log::Warn << "AdaDelta: converged to " << overallObjective
<< "; terminating with failure. Try a smaller step size?"
<< std::endl;
return overallObjective;
}
if (std::abs(lastObjective - overallObjective) < tolerance)
{
Log::Info << "AdaDelta: minimized within tolerance " << tolerance
<< "; terminating optimization." << std::endl;
return overallObjective;
}
// Reset the counter variables.
lastObjective = overallObjective;
overallObjective = 0;
currentFunction = 0;
if (shuffle) // Determine order of visitation.
visitationOrder = arma::shuffle(visitationOrder);
}
// Evaluate the gradient for this iteration.
if (shuffle)
function.Gradient(iterate, visitationOrder[currentFunction], gradient);
else
function.Gradient(iterate, currentFunction, gradient);
// Accumulate gradient.
meanSquaredGradient *= rho;
meanSquaredGradient += (1 - rho) * (gradient % gradient);
arma::mat dx = arma::sqrt((meanSquaredGradientDx + eps) /
(meanSquaredGradient + eps)) % gradient;
// Accumulate updates.
meanSquaredGradientDx *= rho;
meanSquaredGradientDx += (1 - rho) * (dx % dx);
// Apply update.
iterate -= dx;
// Now add that to the overall objective function.
if (shuffle)
overallObjective += function.Evaluate(iterate,
visitationOrder[currentFunction]);
else
overallObjective += function.Evaluate(iterate, currentFunction);
}
Log::Info << "AdaDelta: maximum iterations (" << maxIterations
<< ") reached; terminating optimization." << std::endl;
// Calculate final objective.
overallObjective = 0;
for (size_t i = 0; i < numFunctions; ++i)
overallObjective += function.Evaluate(iterate, i);
return overallObjective;
}
} // namespace optimization
} // namespace mlpack
#endif
src/mlpack/tests/ada_delta_test.cpp
+3 -2
View File
@@ -2,6 +2,7 @@
* @file ada_delta_test.cpp
* @author Marcus Edel
* @author Vasanth Kalingeri
* @author Abhinav Moudgil
*
* Tests the AdaDelta optimizer
*
@@ -12,7 +13,7 @@
*/
#include <mlpack/core.hpp>
#include <mlpack/core/optimizers/adadelta/ada_delta.hpp>
#include <mlpack/core/optimizers/ada_delta/ada_delta.hpp>
#include <mlpack/core/optimizers/sgd/test_function.hpp>
#include <mlpack/methods/logistic_regression/logistic_regression.hpp>
@@ -36,7 +37,7 @@ BOOST_AUTO_TEST_SUITE(AdaDeltaTest);
BOOST_AUTO_TEST_CASE(SimpleAdaDeltaTestFunction)
{
SGDTestFunction f;
AdaDelta<SGDTestFunction> optimizer(f, 0.99, 1e-8, 5000000, 1e-9, true);
AdaDelta<SGDTestFunction> optimizer(f, 1.0, 0.99, 1e-8, 5000000, 1e-9, true);
arma::mat coordinates = f.GetInitialPoint();
optimizer.Optimize(coordinates);