apaam/mlpack
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Changed Nadam as optimizing update on Adam

Browse Source
This commit is contained in:
Sourabh Varshney
2017-12-14 15:12:23 +05:30
parent 948516ef8e
commit e463bb03d5
9 changed files with 147 additions and 348 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/mlpack/core/optimizers/Nadam/CMakeLists.txt
-12
View File
@@ -1,12 +0,0 @@
set(SOURCES
nadam.hpp
nadam_impl.hpp
nadam_update.hpp
)
set(DIR_SRCS)
foreach(file ${SOURCES})
set(DIR_SRCS ${DIR_SRCS} ${CMAKE_CURRENT_SOURCE_DIR}/${file})
endforeach()
set(MLPACK_SRCS ${MLPACK_SRCS} ${DIR_SRCS} PARENT_SCOPE)
src/mlpack/core/optimizers/Nadam/nadam.hpp
-159
View File
@@ -1,159 +0,0 @@
/**
* @file nadam.hpp
* @author Sourabh Varshney
*
* Nadam optimizer. Nadam is an optimizer that combines the effect of Adam and
* NAG to the gradient descent to improve its Performance.
*
* mlpack is free software; you may redistribute it and/or modify it under the
* terms of the 3-clause BSD license. You should have received a copy of the
* 3-clause BSD license along with mlpack. If not, see
* http://www.opensource.org/licenses/BSD-3-Clause for more information.
*/
#ifndef MLPACK_CORE_OPTIMIZERS_NADAM_NADAM_HPP
#define MLPACK_CORE_OPTIMIZERS_NADAM_NADAM_HPP
#include <mlpack/prereqs.hpp>
#include <mlpack/core/optimizers/sgd/sgd.hpp>
#include "nadam_update.hpp"
namespace mlpack {
namespace optimization {
/**
* Nadam is an optimizer that combines the Adam and NAG.
*
* For more information, see the following.
*
* @code
* @article{
* author = {Sebastian Ruder},
* title = {An overview of gradient descent optimization algorithms},
* journal = {CoRR},
* year = {2016},
* url = {https://arxiv.org/abs/1609.04747v2}
* }
* @endcode
*
*
* For Nadam to work, a DecomposableFunctionType template parameter
* is required. This class must implement the following function:
*
* size_t NumFunctions();
* double Evaluate(const arma::mat& coordinates,
* const size_t i,
* const size_t batchSize);
* void Gradient(const arma::mat& coordinates,
* const size_t i,
* arma::mat& gradient,
* const size_t batchSize);
*
* NumFunctions() should return the number of functions (\f$n\f$), and in the
* other two functions, the parameter i refers to which individual function (or
* gradient) is being evaluated. So, for the case of a data-dependent
* function, such as NCA (see mlpack::nca::NCA), NumFunctions() should return
* the number of points in the dataset, and Evaluate(coordinates, 0) will
* evaluate the objective function on the first point in the dataset
* (presumably, the dataset is held internally in the
* DecomposableFunctionType).
*
* @tparam UpdateRule Nadam optimizer update rule to be used.
*/
template<typename UpdateRule = NadamUpdate>
class NadamType
{
public:
/**
* Construct the Nadam optimizer with the given function and parameters. The
* defaults here are not necessarily good for the given problem, so it is
* suggested that the values used be tailored to the task at hand. The
* maximum number of iterations refers to the maximum number of points that
* are processed (i.e., one iteration equals one point; one iteration does
* not equal one pass over the dataset).
*
* @param stepSize Step size for each iteration.
* @param batchSize Number of points to process in a single step.
* @param beta1 Exponential decay rate for the first moment estimates.
* @param beta2 Exponential decay rate for the weighted infinity norm
estimates.
* @param eps Value used to initialise the mean squared gradient parameter.
* @param maxIterations Maximum number of iterations allowed (0 means no
* limit).
* @param tolerance Maximum absolute tolerance to terminate algorithm.
* @param shuffle If true, the function order is shuffled; otherwise, each
* function is visited in linear order.
*/
NadamType(const double stepSize = 0.001,
const size_t batchSize = 32,
const double beta1 = 0.9,
const double eps = 1e-8,
const size_t maxIterations = 100000,
const double tolerance = 1e-5,
const bool shuffle = true);
/**
* Optimize the given function using Nadam. The given starting point will be
* modified to store the finishing point of the algorithm, and the final
* objective value is returned.
*
* @tparam DecomposableFunctionType Type of the function to optimize.
* @param function Function to optimize.
* @param iterate Starting point (will be modified).
* @return Objective value of the final point.
*/
template<typename DecomposableFunctionType>
double Optimize(DecomposableFunctionType& function, arma::mat& iterate)
{
return optimizer.Optimize(function, iterate);
}
//! Get the step size.
double StepSize() const { return optimizer.StepSize(); }
//! Modify the step size.
double& StepSize() { return optimizer.StepSize(); }
//! Get the batch size.
size_t BatchSize() const { return optimizer.BatchSize(); }
//! Modify the batch size.
size_t& BatchSize() { return optimizer.BatchSize(); }
//! Get the smoothing parameter.
double Beta1() const { return optimizer.UpdatePolicy().Beta1(); }
//! Modify the smoothing parameter.
double& Beta1() { return optimizer.UpdatePolicy().Beta1(); }
//! Get the value used to initialise the mean squared gradient parameter.
double Epsilon() const { return optimizer.UpdatePolicy().Epsilon(); }
//! Modify the value used to initialise the mean squared gradient parameter.
double& Epsilon() { return optimizer.UpdatePolicy().Epsilon(); }
//! Get the maximum number of iterations (0 indicates no limit).
size_t MaxIterations() const { return optimizer.MaxIterations(); }
//! Modify the maximum number of iterations (0 indicates no limit).
size_t& MaxIterations() { return optimizer.MaxIterations(); }
//! Get the tolerance for termination.
double Tolerance() const { return optimizer.Tolerance(); }
//! Modify the tolerance for termination.
double& Tolerance() { return optimizer.Tolerance(); }
//! Get whether or not the individual functions are shuffled.
bool Shuffle() const { return optimizer.Shuffle(); }
//! Modify whether or not the individual functions are shuffled.
bool& Shuffle() { return optimizer.Shuffle(); }
private:
//! The Stochastic Gradient Descent object with Nadam policy.
SGD<UpdateRule> optimizer;
};
using Nadam = NadamType<NadamUpdate>;
} // namespace optimization
} // namespace mlpack
// Include implementation.
#include "nadam_impl.hpp"
#endif
src/mlpack/core/optimizers/Nadam/nadam_impl.hpp
-41
View File
@@ -1,41 +0,0 @@
/**
* @file nadam_impl.hpp
* @author Sourabh Varshney
*
* Implementation of the Nadam optimizer.
*
* mlpack is free software; you may redistribute it and/or modify it under the
* terms of the 3-clause BSD license. You should have received a copy of the
* 3-clause BSD license along with mlpack. If not, see
* http://www.opensource.org/licenses/BSD-3-Clause for more information.
*/
#ifndef MLPACK_CORE_OPTIMIZERS_NADAM_NADAM_IMPL_HPP
#define MLPACK_CORE_OPTIMIZERS_NADAM_NADAM_IMPL_HPP
// In case it hasn't been included yet.
#include "nadam.hpp"
namespace mlpack {
namespace optimization {
template<typename UpdateRule>
NadamType<UpdateRule>::NadamType(
const double stepSize,
const size_t batchSize,
const double beta1,
const double epsilon,
const size_t maxIterations,
const double tolerance,
const bool shuffle) :
optimizer(stepSize,
batchSize,
maxIterations,
tolerance,
shuffle,
UpdateRule(epsilon, beta1))
{ /* Nothing to do. */ }
} // namespace optimization
} // namespace mlpack
#endif
src/mlpack/core/optimizers/adam/CMakeLists.txt
+1
View File
@@ -3,6 +3,7 @@ set(SOURCES
adam_impl.hpp
adam_update.hpp
adamax_update.hpp
nadam_update.hpp
)
set(DIR_SRCS)
src/mlpack/core/optimizers/adam/adam.hpp
+8 -4
View File
@@ -5,10 +5,11 @@
* @author Marcus Edel
* @author Vivek Pal
*
* Adam and AdaMax optimizer. Adam is an an algorithm for first-order gradient-
* Adam, AdaMax and Nadam optimizer. Adam is an an algorithm for first-order gradient-
* -based optimization of stochastic objective functions, based on adaptive
* estimates of lower-order moments. AdaMax is simply a variant of Adam based
* on the infinity norm.
* on the infinity norm. Nadam is an optimizer that combines the effect of Adam and
* NAG to the gradient descent to improve its Performance.
*
* mlpack is free software; you may redistribute it and/or modify it under the
* terms of the 3-clause BSD license. You should have received a copy of the
@@ -31,7 +32,8 @@ namespace optimization {
* Adam is an optimizer that computes individual adaptive learning rates for
* different parameters from estimates of first and second moments of the
* gradients. AdaMax is a variant of Adam based on the infinity norm as given
* in the section 7 of the following paper.
* in the section 7 of the following paper. Nadam is an optimizer that
* combines the Adam and NAG.
*
* For more information, see the following.
*
@@ -46,7 +48,7 @@ namespace optimization {
* @endcode
*
*
* For Adam and AdaMax to work, a DecomposableFunctionType template parameter
* For Adam, AdaMax and Nadam to work, a DecomposableFunctionType template parameter
* is required. This class must implement the following function:
*
* size_t NumFunctions();
@@ -166,6 +168,8 @@ using Adam = AdamType<AdamUpdate>;
using AdaMax = AdamType<AdaMaxUpdate>;
using Nadam = NadamType<NadamUpdate>;
} // namespace optimization
} // namespace mlpack
src/mlpack/core/optimizers/adam/adam_impl.hpp
+1 -1
View File
@@ -5,7 +5,7 @@
* @author Marcus Edel
* @author Vivek Pal
*
* Implementation of the Adam and AdaMax optimizer.
* Implementation of the Adam, AdaMax and Nadam optimizer.
*
* mlpack is free software; you may redistribute it and/or modify it under the
* terms of the 3-clause BSD license. You should have received a copy of the
src/mlpack/core/optimizers/Nadam/nadam_update.hpp → src/mlpack/core/optimizers/adam/nadam_update.hpp
+59 -19
View File
@@ -2,7 +2,7 @@
* @file nadam_update.hpp
* @author Sourabh Varshney
*
* Nadam optimizer. Nadam is an optimizer that combines the effect of Adam and
* Nadam update rule. Nadam is an optimizer that combines the effect of Adam and
* NAG to the gradient descent to improve its Performance.
*
* mlpack is free software; you may redistribute it and/or modify it under the
@@ -10,8 +10,8 @@
* 3-clause BSD license along with mlpack. If not, see
* http://www.opensource.org/licenses/BSD-3-Clause for more information.
*/
#ifndef MLPACK_CORE_OPTIMIZERS_NADAM_NADAM_UPDATE_HPP
#define MLPACK_CORE_OPTIMIZERS_NADAM_NADAM_UPDATE_HPP
#ifndef MLPACK_CORE_OPTIMIZERS_ADAM_NADAM_UPDATE_HPP
#define MLPACK_CORE_OPTIMIZERS_ADAM_NADAM_UPDATE_HPP
#include <mlpack/prereqs.hpp>
@@ -24,12 +24,12 @@ namespace optimization {
* For more information, see the following.
*
* @code
* @article{
* author = {Sebastian Ruder},
* title = {An overview of gradient descent optimization algorithms},
* journal = {CoRR},
* year = {2016},
* url = {https://arxiv.org/abs/1609.04747v2}
* @misc{
* author = {},
* title = {},
* journal = {},
* year = {},
* url = {}
* }
* @endcode
*/
@@ -42,9 +42,12 @@ class NadamUpdate
* @param epsilon The epsilon value used to initialise the squared gradient
* parameter.
* @param beta1 The smoothing parameter.
* @param beta2 The second moment coefficient
*/
NadamUpdate(const double epsilon = 1e-8, const double beta1 = 0.9)
:epsilon(epsilon), beta1(beta1), iteration(0)
NadamUpdate(const double epsilon = 1e-8,
const double beta1 = 0.9,
const double beta2=0.99)
:epsilon(epsilon), beta1(beta1), beta2(beta2), iteration(0)
{
// Nothing to do.
}
@@ -60,6 +63,7 @@ class NadamUpdate
{
m = arma::zeros<arma::mat>(rows, cols);
v = arma::zeros<arma::mat>(rows, cols);
cum_beta1 = 1;
}
/**
@@ -69,8 +73,9 @@ class NadamUpdate
* @param stepSize Step size to be used for the given iteration.
* @param gradient The gradient matrix.
*/
void Update(arma::mat& iterate, const double stepSize,
const arma::mat& gradient)
void Update(arma::mat& iterate,
const double stepSize,
const arma::mat& gradient)
{
// Increment the iteration counter variable.
++iteration;
@@ -78,12 +83,31 @@ class NadamUpdate
// And update the iterate.
m *= beta1;
m += (1 - beta1) * gradient;
// biasCorrection=1-beta1^iteration
const double biasCorrection = 1.0 - std::pow(beta1, iteration);
/*
iterate=iterate-((stepsize/(sqrt(v)+epsilon))*(m/biasCorrection))
*/
iterate -= ((stepSize * m)/(biasCorrection1 * (arma::sqrt(v) + epsilon)));
v *= beta2;
v += (1 - beta2) * gradient % gradient;
// beta1_t = beta1 * (1 - (0.5 * (0.96 ^ (iteration / 250))))
double beta1_t = beta1 * (1 - (0.5 * std::pow(0.96, (iteration / 250))));
// beta1_t1 = beta1 * (1 - (0.5 * (0.96 ^ ((iteration + 1)/ 250))))
double beta1_t1 = beta1 * (1 - (0.5 * std::pow(0.96, ((iteration + 1) / 250))));
// cum_beta1 *= beta1_t
cum_beta1 *= beta1_t;
// biasCorrection = 1 - cum_beta1
const double biasCorrection1 = 1.0 - cum_beta1;
// biasCorrection2 = 1 - beta2 ^ iteration
const double biasCorrection2 = 1.0 - std::pow(beta2, iteration);
/* Note :- arma::sqrt(v) + epsilon * sqrt(biasCorrection2) is approximated as
* arma::sqrt(v) + epsilon
*/
iterate -= (stepSize * ((1 - beta1_t) * gradient +beta1_t1 * m)
* sqrt(biasCorrection2)) / ((arma::sqrt(v) + epsilon)
* biasCorrection1)
}
//! Get the value used to initialise the squared gradient parameter.
@@ -91,11 +115,21 @@ class NadamUpdate
//! Modify the value used to initialise the squared gradient parameter.
double& Epsilon() { return epsilon; }
//! Get the value of the cumulative product of decay constants
double Cum_beta1() const { return cum_beta1; }
//! Modify the value of the cumulative product of decay constants
double& Cum_beta1() { return cum_beta1; }
//! Get the smoothing parameter.
double Beta1() const { return beta1; }
//! Modify the smoothing parameter.
double& Beta1() { return beta1; }
//! Get the second moment coefficient.
double Beta2() const { return beta2; }
//! Modify the second moment coefficient.
double& Beta2() { return beta2; }
private:
// The epsilon value used to initialise the squared gradient parameter.
double epsilon;
@@ -103,12 +137,18 @@ class NadamUpdate
// The smoothing parameter.
double beta1;
// The second moment coefficient.
double beta2;
// The exponential moving average of gradient values.
arma::mat m;
// The exponential moving average of squared gradient values.
arma::mat v;
// The cumulative product of decay constants
double cum_beta1;
// The number of iterations.
double iteration;
};
src/mlpack/tests/adam_test.cpp
+78 -1
View File
@@ -2,8 +2,9 @@
* @file adam_test.cpp
* @author Vasanth Kalingeri
* @author Vivek Pal
* @author Sourabh Varshney
*
* Tests the Adam and AdaMax optimizer.
* Tests the Adam, AdaMax and Nadam optimizer.
*
* mlpack is free software; you may redistribute it and/or modify it under the
* terms of the 3-clause BSD license. You should have received a copy of the
@@ -62,6 +63,22 @@ BOOST_AUTO_TEST_CASE(SimpleAdaMaxTestFunction)
BOOST_REQUIRE_SMALL(coordinates[2], 0.1);
}
/**
* Tests the Nadam optimizer using a simple test function.
*/
BOOST_AUTO_TEST_CASE(SimpleNadamTestFunction)
{
SGDTestFunction f;
Nadam optimizer(1e-3, 1, 0.9, 0.99, 1e-8, 500000, 1e-9, true);
arma::mat coordinates = f.GetInitialPoint();
optimizer.Optimize(f, coordinates);
BOOST_REQUIRE_SMALL(coordinates[0], 0.1);
BOOST_REQUIRE_SMALL(coordinates[1], 0.1);
BOOST_REQUIRE_SMALL(coordinates[2], 0.1);
}
/**
* Run Adam on logistic regression and make sure the results are acceptable.
*/
@@ -178,4 +195,64 @@ BOOST_AUTO_TEST_CASE(AdaMaxLogisticRegressionTest)
BOOST_REQUIRE_CLOSE(testAcc, 100.0, 0.6); // 0.6% error tolerance.
}
/**
* Run Nadam on logistic regression and make sure the results are acceptable.
*/
BOOST_AUTO_TEST_CASE(NadamLogisticRegressionTest)
{
// Generate a two-Gaussian dataset.
GaussianDistribution g1(arma::vec("1.0 1.0 1.0"),
arma::eye<arma::mat>(3, 3));
GaussianDistribution g2(arma::vec("9.0 9.0 9.0"),
arma::eye<arma::mat>(3, 3));
arma::mat data(3, 1000);
arma::Row<size_t> responses(1000);
for (size_t i = 0; i < 500; ++i)
{
data.col(i) = g1.Random();
responses[i] = 0;
}
for (size_t i = 500; i < 1000; ++i)
{
data.col(i) = g2.Random();
responses[i] = 1;
}
// Shuffle the dataset.
arma::uvec indices = arma::shuffle(arma::linspace<arma::uvec>(0,
data.n_cols - 1, data.n_cols));
arma::mat shuffledData(3, 1000);
arma::Row<size_t> shuffledResponses(1000);
for (size_t i = 0; i < data.n_cols; ++i)
{
shuffledData.col(i) = data.col(indices[i]);
shuffledResponses[i] = responses[indices[i]];
}
// Create a test set.
arma::mat testData(3, 1000);
arma::Row<size_t> testResponses(1000);
for (size_t i = 0; i < 500; ++i)
{
testData.col(i) = g1.Random();
testResponses[i] = 0;
}
for (size_t i = 500; i < 1000; ++i)
{
testData.col(i) = g2.Random();
testResponses[i] = 1;
}
Nadam nadam;
LogisticRegression<> lr(shuffledData, shuffledResponses, nadam, 0.5);
// Ensure that the error is close to zero.
const double acc = lr.ComputeAccuracy(data, responses);
BOOST_REQUIRE_CLOSE(acc, 100.0, 0.3); // 0.3% error tolerance.
const double testAcc = lr.ComputeAccuracy(testData, testResponses);
BOOST_REQUIRE_CLOSE(testAcc, 100.0, 0.6); // 0.6% error tolerance.
}
BOOST_AUTO_TEST_SUITE_END();
src/mlpack/tests/nadam_test.cpp
-111
View File
@@ -1,111 +0,0 @@
/**
* @file nadam_test.cpp
* @author Sourabh Varshney
*
* Tests the Nadam optimizer.
*
* mlpack is free software; you may redistribute it and/or modify it under the
* terms of the 3-clause BSD license. You should have received a copy of the
* 3-clause BSD license along with mlpack. If not, see
* http://www.opensource.org/licenses/BSD-3-Clause for more information.
*/
#include <mlpack/core.hpp>
#include <mlpack/core/optimizers/nadam/nadam.hpp>
#include <mlpack/core/optimizers/sgd/test_function.hpp>
#include <mlpack/methods/logistic_regression/logistic_regression.hpp>
#include <boost/test/unit_test.hpp>
#include "test_tools.hpp"
using namespace arma;
using namespace mlpack::optimization;
using namespace mlpack::optimization::test;
using namespace mlpack::distribution;
using namespace mlpack::regression;
using namespace mlpack;
BOOST_AUTO_TEST_SUITE(NadamTest);
/**
* Tests the Adam optimizer using a simple test function.
*/
BOOST_AUTO_TEST_CASE(SimpleNadamTestFunction)
{
SGDTestFunction f;
Nadam optimizer(1e-3, 1, 0.9, 1e-8, 500000, 1e-9, true);
arma::mat coordinates = f.GetInitialPoint();
optimizer.Optimize(f, coordinates);
BOOST_REQUIRE_SMALL(coordinates[0], 0.1);
BOOST_REQUIRE_SMALL(coordinates[1], 0.1);
BOOST_REQUIRE_SMALL(coordinates[2], 0.1);
}
/**
* Run Nadam on logistic regression and make sure the results are acceptable.
*/
BOOST_AUTO_TEST_CASE(NadamLogisticRegressionTest)
{
// Generate a two-Gaussian dataset.
GaussianDistribution g1(arma::vec("1.0 1.0 1.0"),
arma::eye<arma::mat>(3, 3));
GaussianDistribution g2(arma::vec("9.0 9.0 9.0"),
arma::eye<arma::mat>(3, 3));
arma::mat data(3, 1000);
arma::Row<size_t> responses(1000);
for (size_t i = 0; i < 500; ++i)
{
data.col(i) = g1.Random();
responses[i] = 0;
}
for (size_t i = 500; i < 1000; ++i)
{
data.col(i) = g2.Random();
responses[i] = 1;
}
// Shuffle the dataset.
arma::uvec indices = arma::shuffle(arma::linspace<arma::uvec>(0,
data.n_cols - 1, data.n_cols));
arma::mat shuffledData(3, 1000);
arma::Row<size_t> shuffledResponses(1000);
for (size_t i = 0; i < data.n_cols; ++i)
{
shuffledData.col(i) = data.col(indices[i]);
shuffledResponses[i] = responses[indices[i]];
}
// Create a test set.
arma::mat testData(3, 1000);
arma::Row<size_t> testResponses(1000);
for (size_t i = 0; i < 500; ++i)
{
testData.col(i) = g1.Random();
testResponses[i] = 0;
}
for (size_t i = 500; i < 1000; ++i)
{
testData.col(i) = g2.Random();
testResponses[i] = 1;
}
Nadam nadam;
LogisticRegression<> lr(shuffledData, shuffledResponses, nadam, 0.5);
// Ensure that the error is close to zero.
const double acc = lr.ComputeAccuracy(data, responses);
BOOST_REQUIRE_CLOSE(acc, 100.0, 0.3); // 0.3% error tolerance.
const double testAcc = lr.ComputeAccuracy(testData, testResponses);
BOOST_REQUIRE_CLOSE(testAcc, 100.0, 0.6); // 0.6% error tolerance.
}
BOOST_AUTO_TEST_SUITE_END();