Merge branch 'master' into ada_sqrt
This commit is contained in:
Marcus Edel
@@ -1,5 +1,11 @@
|
||||
### ensmallen ?.??.?: "???"
|
||||
###### ????-??-??
|
||||
* Add Yogi optimizer
|
||||
([#232](https://github.com/mlpack/ensmallen/pull/232)).
|
||||
|
||||
* Add AdaBelief optimizer
|
||||
([#233](https://github.com/mlpack/ensmallen/pull/233)).
|
||||
|
||||
* Add AdaSqrt optimizer
|
||||
([#234](https://github.com/mlpack/ensmallen/pull/234)).
|
||||
|
||||
|
||||
@@ -554,6 +554,7 @@ Each of the implemented methods is allowed to have additional cv-modifiers
|
||||
|
||||
The following optimizers can be used with differentiable separable functions:
|
||||
|
||||
- [AdaBelief](#adabelief)
|
||||
- [AdaBound](#adabound)
|
||||
- [AdaDelta](#adadelta)
|
||||
- [AdaGrad](#adagrad)
|
||||
|
||||
+112
-1
@@ -1,3 +1,61 @@
|
||||
## AdaBelief
|
||||
|
||||
*An optimizer for [differentiable separable functions](#differentiable-separable-functions).*
|
||||
|
||||
AdaBelief uses a different denominator from Adam, and is orthogonal to other
|
||||
techniques such as recification, decoupled weight decay. The intuition for
|
||||
AdaBelief is to adapt the stepsize according to the "belief" in the current
|
||||
gradient direction.
|
||||
|
||||
#### Constructors
|
||||
|
||||
* `AdaBelief()`
|
||||
* `AdaBelief(`_`stepSize, batchSize`_`)`
|
||||
* `AdaBelief(`_`stepSize, batchSize, beta1, beta2, epsilon, maxIterations, tolerance, shuffle`_`)`
|
||||
* `AdaBelief(`_`stepSize, batchSize, beta1, beta2, epsilon, maxIterations, tolerance, shuffle, resetPolicy, exactObjective`_`)`
|
||||
|
||||
#### Attributes
|
||||
|
||||
| **type** | **name** | **description** | **default** |
|
||||
|----------|----------|-----------------|-------------|
|
||||
| `double` | **`stepSize`** | Step size for each iteration. | `0.001` |
|
||||
| `size_t` | **`batchSize`** | Number of points to process in a single step. | `32` |
|
||||
| `double` | **`beta1`** | The exponential decay rate for the 1st moment estimates. | `0.9` |
|
||||
| `double` | **`beta2`** | The exponential decay rate for the 2nd moment estimates. | `0.999` |
|
||||
| `double` | **`epsilon`** | A small constant for numerical stability. | `1e-8` |
|
||||
| `size_t` | **`max_iterations`** | Maximum number of iterations allowed (0 means no limit). | `100000` |
|
||||
| `double` | **`tolerance`** | Maximum absolute tolerance to terminate algorithm. | `1e-5` |
|
||||
| `bool` | **`shuffle`** | If true, the function order is shuffled; otherwise, each function is visited in linear order. | `true` |
|
||||
| `bool` | **`resetPolicy`** | If true, parameters are reset before every Optimize call; otherwise, their values are retained. | `true` |
|
||||
| `bool` | **`exactObjective`** | Calculate the exact objective (Default: estimate the final objective obtained on the last pass over the data). | `false` |
|
||||
|
||||
The attributes of the optimizer may also be modified via the member methods
|
||||
`StepSize()`, `BatchSize()`, `Beta1()`, `Beta2()`, `Epsilon()`, `MaxIterations()`,
|
||||
`Tolerance()`, `Shuffle()`, `ResetPolicy()`, and `ExactObjective()`.
|
||||
|
||||
#### Examples
|
||||
|
||||
<details open>
|
||||
<summary>Click to collapse/expand example code.
|
||||
</summary>
|
||||
|
||||
```c++
|
||||
RosenbrockFunction f;
|
||||
arma::mat coordinates = f.GetInitialPoint();
|
||||
|
||||
AdaBelief optimizer(0.001, 32, 0.9, 0.999, 1e-12, 100000, 1e-5, true);
|
||||
optimizer.Optimize(f, coordinates);
|
||||
```
|
||||
|
||||
</details>
|
||||
|
||||
#### See also:
|
||||
|
||||
* [SGD in Wikipedia](https://en.wikipedia.org/wiki/Stochastic_gradient_descent)
|
||||
* [SGD](#standard-sgd)
|
||||
* [AdaBelief Optimizer: Adapting Stepsizes by the Belief in Observed Gradients](https://arxiv.org/abs/2010.07468)
|
||||
* [Differentiable separable functions](#differentiable-separable-functions)
|
||||
|
||||
## AdaBound
|
||||
|
||||
*An optimizer for [differentiable separable functions](#differentiable-separable-functions).*
|
||||
@@ -267,7 +325,7 @@ with _`UpdateRule`_` = AdamUpdate`.
|
||||
| `bool` | **`exactObjective`** | Calculate the exact objective (Default: estimate the final objective obtained on the last pass over the data). | `false` |
|
||||
|
||||
The attributes of the optimizer may also be modified via the member methods
|
||||
`StepSize()`, `BatchSize()`, `Beta1()`, `Beta2()`, `Eps()`, `MaxIterations()`,
|
||||
`StepSize()`, `BatchSize()`, `Beta1()`, `Beta2()`, `Epsilon()`, `MaxIterations()`,
|
||||
`Tolerance()`, `Shuffle()`, `ResetPolicy()`, and `ExactObjective()`.
|
||||
|
||||
#### Examples
|
||||
@@ -3034,3 +3092,56 @@ optimizer.Optimize(f, coordinates);
|
||||
* [SGD](#standard-sgd)
|
||||
* [SGD in Wikipedia](https://en.wikipedia.org/wiki/Stochastic_gradient_descent)
|
||||
* [Differentiable separable functions](#differentiable-separable-functions)
|
||||
|
||||
## Yogi
|
||||
|
||||
*An optimizer for [differentiable separable functions](#differentiable-separable-functions).*
|
||||
|
||||
Yogi is an optimization algorithm based on Adam with more fine-grained effective
|
||||
learning rate control, which uses additive updates instead of multiplicative
|
||||
updates for the moving average of the squared gradient. In addition, Yogi has
|
||||
similar theoretical guarantees on convergence as Adam.
|
||||
|
||||
#### Constructors
|
||||
|
||||
* `Yogi()`
|
||||
* `Yogi(`_`stepSize, batchSize`_`)`
|
||||
* `Yogi(`_`stepSize, batchSize, beta1, beta2, eps, maxIterations`_`)`
|
||||
* `Yogi(`_`stepSize, batchSize, beta1, beta2, eps, maxIterations, tolerance, shuffle, resetPolicy, exactObjective`_`)`
|
||||
|
||||
#### Attributes
|
||||
|
||||
| **type** | **name** | **description** | **default** |
|
||||
|----------|----------|-----------------|-------------|
|
||||
| `double` | **`stepSize`** | Step size for each iteration. | `0.001` |
|
||||
| `size_t` | **`batchSize`** | Number of points to process in a single step. | `32` |
|
||||
| `double` | **`beta1`** | Exponential decay rate for the first moment estimates. | `0.9` |
|
||||
| `double` | **`beta2`** | Exponential decay rate for the weighted infinity norm estimates. | `0.999` |
|
||||
| `double` | **`eps`** | Value used to initialize the mean squared gradient parameter. | `1e-8` |
|
||||
| `size_t` | **`max_iterations`** | Maximum number of iterations allowed (0 means no limit). | `100000` |
|
||||
| `double` | **`tolerance`** | Maximum absolute tolerance to terminate algorithm. | `1e-5` |
|
||||
| `bool` | **`shuffle`** | If true, the function order is shuffled; otherwise, each function is visited in linear order. | `true` |
|
||||
| `bool` | **`resetPolicy`** | If true, parameters are reset before every Optimize call; otherwise, their values are retained. | `true` |
|
||||
| `bool` | **`exactObjective`** | Calculate the exact objective (Default: estimate the final objective obtained on the last pass over the data). | `false` |
|
||||
|
||||
The attributes of the optimizer may also be modified via the member methods
|
||||
`StepSize()`, `BatchSize()`, `Beta1()`, `Beta2()`, `Eps()`, `MaxIterations()`,
|
||||
`Tolerance()`, `Shuffle()`, `ResetPolicy()`, and `ExactObjective()`.
|
||||
|
||||
#### Examples
|
||||
|
||||
```c++
|
||||
RosenbrockFunction f;
|
||||
arma::mat coordinates = f.GetInitialPoint();
|
||||
|
||||
Yogi optimizer(0.001, 32, 0.9, 0.999, 1e-8, 100000, 1e-5, true);
|
||||
optimizer.Optimize(f, coordinates);
|
||||
```
|
||||
|
||||
#### See also:
|
||||
|
||||
* [Adaptive Methods for Nonconvex Optimization](https://papers.nips.cc/paper/8186-adaptive-methods-for-nonconvex-optimization)
|
||||
* [SGD in Wikipedia](https://en.wikipedia.org/wiki/Stochastic_gradient_descent)
|
||||
* [SGD](#standard-sgd)
|
||||
* [Adam](#adam)
|
||||
* [Differentiable separable functions](#differentiable-separable-functions)
|
||||
|
||||
@@ -85,6 +85,7 @@
|
||||
|
||||
#include "ensmallen_bits/problems/problems.hpp" // TODO: should move to another place
|
||||
|
||||
#include "ensmallen_bits/ada_belief/ada_belief.hpp"
|
||||
#include "ensmallen_bits/ada_bound/ada_bound.hpp"
|
||||
#include "ensmallen_bits/ada_delta/ada_delta.hpp"
|
||||
#include "ensmallen_bits/ada_grad/ada_grad.hpp"
|
||||
@@ -132,5 +133,6 @@
|
||||
#include "ensmallen_bits/svrg/svrg.hpp"
|
||||
#include "ensmallen_bits/swats/swats.hpp"
|
||||
#include "ensmallen_bits/wn_grad/wn_grad.hpp"
|
||||
#include "ensmallen_bits/yogi/yogi.hpp"
|
||||
|
||||
#endif
|
||||
|
||||
@@ -0,0 +1,186 @@
|
||||
/**
|
||||
* @file ada_belief.hpp
|
||||
* @author Marcus Edel
|
||||
*
|
||||
* Class wrapper for the AdaBelief update Policy. The intuition for AdaBelief is
|
||||
* to adapt the stepsize according to the "belief" in the current gradient
|
||||
* direction.
|
||||
*
|
||||
* ensmallen is free software; you may redistribute it and/or modify it under
|
||||
* the terms of the 3-clause BSD license. You should have received a copy of
|
||||
* the 3-clause BSD license along with ensmallen. If not, see
|
||||
* http://www.opensource.org/licenses/BSD-3-Clause for more information.
|
||||
*/
|
||||
#ifndef ENSMALLEN_ADA_BELIEF_HPP
|
||||
#define ENSMALLEN_ADA_BELIEF_HPP
|
||||
|
||||
#include <ensmallen_bits/sgd/sgd.hpp>
|
||||
#include "ada_belief_update.hpp"
|
||||
|
||||
namespace ens {
|
||||
|
||||
/**
|
||||
* The intuition for AdaBelief is to adapt the stepsize according to the
|
||||
* "belief" in the current gradient direction. For more information, see the
|
||||
* following.
|
||||
*
|
||||
* @code
|
||||
* @misc{zhuang2020adabelief,
|
||||
* title = {AdaBelief Optimizer: Adapting Stepsizes by the Belief in
|
||||
* Observed Gradients},
|
||||
* author = {Juntang Zhuang and Tommy Tang and Sekhar Tatikonda and
|
||||
* Nicha Dvornek and Yifan Ding and Xenophon Papademetris
|
||||
* and James S. Duncan},
|
||||
* year = {2020},
|
||||
* eprint = {2010.07468},
|
||||
* archivePrefix = {arXiv},
|
||||
* }
|
||||
* @endcode
|
||||
*
|
||||
* AdaBelief can optimize differentiable separable functions. For more details,
|
||||
* see the documentation on function types included with this distribution or
|
||||
* on the ensmallen website.
|
||||
*/
|
||||
class AdaBelief
|
||||
{
|
||||
public:
|
||||
/**
|
||||
* Construct the AdaBelief optimizer with the given function and parameters.
|
||||
* AdaBelief is sensitive to its parameters and hence a good hyperparameter
|
||||
* selection is necessary as its default may not fit every case.
|
||||
*
|
||||
* The maximum number of iterations refers to the maximum number of
|
||||
* points that are processed (i.e., one iteration equals one point; one
|
||||
* iteration does not equal one pass over the dataset).
|
||||
*
|
||||
* @param stepSize Step size for each iteration.
|
||||
* @param batchSize Number of points to process in a single step.
|
||||
* @param beta1 The exponential decay rate for the 1st moment estimates.
|
||||
* @param beta2 The exponential decay rate for the 2nd moment estimates.
|
||||
* @param epsilon A small constant for numerical stability.
|
||||
* @param maxIterations Maximum number of iterations allowed (0 means no
|
||||
* limit).
|
||||
* @param tolerance Maximum absolute tolerance to terminate algorithm.
|
||||
* @param shuffle If true, the function order is shuffled; otherwise, each
|
||||
* function is visited in linear order.
|
||||
* @param resetPolicy If true, parameters are reset before every Optimize
|
||||
* call; otherwise, their values are retained.
|
||||
* @param exactObjective Calculate the exact objective (Default: estimate the
|
||||
* final objective obtained on the last pass over the data).
|
||||
*/
|
||||
AdaBelief(const double stepSize = 0.001,
|
||||
const size_t batchSize = 32,
|
||||
const double beta1 = 0.9,
|
||||
const double beta2 = 0.999,
|
||||
const double epsilon = 1e-12,
|
||||
const size_t maxIterations = 100000,
|
||||
const double tolerance = 1e-5,
|
||||
const bool shuffle = true,
|
||||
const bool resetPolicy = true,
|
||||
const bool exactObjective = false);
|
||||
|
||||
/**
|
||||
* Optimize the given function using AdaBelief. The given starting point will
|
||||
* be modified to store the finishing point of the algorithm, and the final
|
||||
* objective value is returned.
|
||||
*
|
||||
* @tparam SeparableFunctionType Type of the function to optimize.
|
||||
* @tparam MatType Type of matrix to optimize with.
|
||||
* @tparam GradType Type of matrix to use to represent function gradients.
|
||||
* @tparam CallbackTypes Types of callback functions.
|
||||
* @param function Function to optimize.
|
||||
* @param iterate Starting point (will be modified).
|
||||
* @param callbacks Callback functions.
|
||||
* @return Objective value of the final point.
|
||||
*/
|
||||
template<typename SeparableFunctionType,
|
||||
typename MatType,
|
||||
typename GradType,
|
||||
typename... CallbackTypes>
|
||||
typename std::enable_if<IsArmaType<GradType>::value,
|
||||
typename MatType::elem_type>::type
|
||||
Optimize(SeparableFunctionType& function,
|
||||
MatType& iterate,
|
||||
CallbackTypes&&... callbacks)
|
||||
{
|
||||
return optimizer.Optimize<SeparableFunctionType, MatType, GradType,
|
||||
CallbackTypes...>(function, iterate,
|
||||
std::forward<CallbackTypes>(callbacks)...);
|
||||
}
|
||||
|
||||
//! Forward the MatType as GradType.
|
||||
template<typename SeparableFunctionType,
|
||||
typename MatType,
|
||||
typename... CallbackTypes>
|
||||
typename MatType::elem_type Optimize(SeparableFunctionType& function,
|
||||
MatType& iterate,
|
||||
CallbackTypes&&... callbacks)
|
||||
{
|
||||
return Optimize<SeparableFunctionType, MatType, MatType,
|
||||
CallbackTypes...>(function, iterate,
|
||||
std::forward<CallbackTypes>(callbacks)...);
|
||||
}
|
||||
|
||||
//! Get the step size.
|
||||
double StepSize() const { return optimizer.StepSize(); }
|
||||
//! Modify the step size.
|
||||
double& StepSize() { return optimizer.StepSize(); }
|
||||
|
||||
//! Get the batch size.
|
||||
size_t BatchSize() const { return optimizer.BatchSize(); }
|
||||
//! Modify the batch size.
|
||||
size_t& BatchSize() { return optimizer.BatchSize(); }
|
||||
|
||||
//! Get the exponential decay rate for the 1st moment estimates.
|
||||
double Beta1() const { return optimizer.UpdatePolicy().Beta1(); }
|
||||
//! Modify the exponential decay rate for the 1st moment estimates.
|
||||
double& Beta1() { return optimizer.UpdatePolicy().Beta1(); }
|
||||
|
||||
//! Get the exponential decay rate for the 2nd moment estimates.
|
||||
double Beta2() const { return optimizer.UpdatePolicy().Beta2(); }
|
||||
//! Get the second moment coefficient.
|
||||
double& Beta2() { return optimizer.UpdatePolicy().Beta2(); }
|
||||
|
||||
//! Get the value for numerical stability.
|
||||
double Epsilon() const { return optimizer.UpdatePolicy().Epsilon(); }
|
||||
//! Modify the value used for numerical stability.
|
||||
double& Epsilon() { return optimizer.UpdatePolicy().Epsilon(); }
|
||||
|
||||
//! Get the maximum number of iterations (0 indicates no limit).
|
||||
size_t MaxIterations() const { return optimizer.MaxIterations(); }
|
||||
//! Modify the maximum number of iterations (0 indicates no limit).
|
||||
size_t& MaxIterations() { return optimizer.MaxIterations(); }
|
||||
|
||||
//! Get the tolerance for termination.
|
||||
double Tolerance() const { return optimizer.Tolerance(); }
|
||||
//! Modify the tolerance for termination.
|
||||
double& Tolerance() { return optimizer.Tolerance(); }
|
||||
|
||||
//! Get whether or not the individual functions are shuffled.
|
||||
bool Shuffle() const { return optimizer.Shuffle(); }
|
||||
//! Modify whether or not the individual functions are shuffled.
|
||||
bool& Shuffle() { return optimizer.Shuffle(); }
|
||||
|
||||
//! Get whether or not the actual objective is calculated.
|
||||
bool ExactObjective() const { return optimizer.ExactObjective(); }
|
||||
//! Modify whether or not the actual objective is calculated.
|
||||
bool& ExactObjective() { return optimizer.ExactObjective(); }
|
||||
|
||||
//! Get whether or not the update policy parameters are reset before
|
||||
//! Optimize call.
|
||||
bool ResetPolicy() const { return optimizer.ResetPolicy(); }
|
||||
//! Modify whether or not the update policy parameters
|
||||
//! are reset before Optimize call.
|
||||
bool& ResetPolicy() { return optimizer.ResetPolicy(); }
|
||||
|
||||
private:
|
||||
//! The Stochastic Gradient Descent object with AdaBelief policy.
|
||||
SGD<AdaBeliefUpdate> optimizer;
|
||||
};
|
||||
|
||||
} // namespace ens
|
||||
|
||||
// Include implementation.
|
||||
#include "ada_belief_impl.hpp"
|
||||
|
||||
#endif
|
||||
@@ -0,0 +1,44 @@
|
||||
/**
|
||||
* @file ada_belief_impl.hpp
|
||||
* @author Marcus Edel
|
||||
*
|
||||
* Implementation of AdaBelief class wrapper.
|
||||
*
|
||||
* ensmallen is free software; you may redistribute it and/or modify it under
|
||||
* the terms of the 3-clause BSD license. You should have received a copy of
|
||||
* the 3-clause BSD license along with ensmallen. If not, see
|
||||
* http://www.opensource.org/licenses/BSD-3-Clause for more information.
|
||||
*/
|
||||
#ifndef ENSMALLEN_ADA_BELIEF_ADA_BELIEF_IMPL_HPP
|
||||
#define ENSMALLEN_ADA_BELIEF_ADA_BELIEF_IMPL_HPP
|
||||
|
||||
// In case it hasn't been included yet.
|
||||
#include "ada_belief.hpp"
|
||||
|
||||
namespace ens {
|
||||
|
||||
inline AdaBelief::AdaBelief(
|
||||
const double stepSize,
|
||||
const size_t batchSize,
|
||||
const double beta1,
|
||||
const double beta2,
|
||||
const double epsilon,
|
||||
const size_t maxIterations,
|
||||
const double tolerance,
|
||||
const bool shuffle,
|
||||
const bool resetPolicy,
|
||||
const bool exactObjective) :
|
||||
optimizer(stepSize,
|
||||
batchSize,
|
||||
maxIterations,
|
||||
tolerance,
|
||||
shuffle,
|
||||
AdaBeliefUpdate(epsilon, beta1, beta2),
|
||||
NoDecay(),
|
||||
resetPolicy,
|
||||
exactObjective)
|
||||
{ /* Nothing to do. */ }
|
||||
|
||||
} // namespace ens
|
||||
|
||||
#endif
|
||||
@@ -0,0 +1,160 @@
|
||||
/**
|
||||
* @file ada_belief_update.hpp
|
||||
* @author Marcus Edel
|
||||
*
|
||||
* AdaBelief optimizer update policy. The intuition for AdaBelief is to adapt
|
||||
* the stepsize according to the "belief" in the current gradient direction.
|
||||
*
|
||||
* ensmallen is free software; you may redistribute it and/or modify it under
|
||||
* the terms of the 3-clause BSD license. You should have received a copy of
|
||||
* the 3-clause BSD license along with ensmallen. If not, see
|
||||
* http://www.opensource.org/licenses/BSD-3-Clause for more information.
|
||||
*/
|
||||
#ifndef ENSMALLEN_ADA_BELIEF_ADA_BELIEF_UPDATE_HPP
|
||||
#define ENSMALLEN_ADA_BELIEF_ADA_BELIEF_UPDATE_HPP
|
||||
|
||||
namespace ens {
|
||||
|
||||
/**
|
||||
* The intuition for AdaBelief is to adapt the stepsize according to the
|
||||
* "belief" in the current gradient direction.
|
||||
*
|
||||
* For more information, see the following.
|
||||
*
|
||||
* @code
|
||||
* @misc{zhuang2020adabelief,
|
||||
* title = {AdaBelief Optimizer: Adapting Stepsizes by the Belief in
|
||||
* Observed Gradients},
|
||||
* author = {Juntang Zhuang and Tommy Tang and Sekhar Tatikonda and
|
||||
* Nicha Dvornek and Yifan Ding and Xenophon Papademetris
|
||||
* and James S. Duncan},
|
||||
* year = {2020},
|
||||
* eprint = {2010.07468},
|
||||
* archivePrefix = {arXiv},
|
||||
* }
|
||||
* @endcode
|
||||
*/
|
||||
class AdaBeliefUpdate
|
||||
{
|
||||
public:
|
||||
/**
|
||||
* Construct the AdaBelief update policy with the given parameters.
|
||||
*
|
||||
* @param epsilon A small constant for numerical stability.
|
||||
* @param beta1 The exponential decay rate for the 1st moment estimates.
|
||||
* @param beta2 The exponential decay rate for the 2nd moment estimates.
|
||||
*/
|
||||
AdaBeliefUpdate(const double epsilon = 1e-8,
|
||||
const double beta1 = 0.9,
|
||||
const double beta2 = 0.999) :
|
||||
epsilon(epsilon),
|
||||
beta1(beta1),
|
||||
beta2(beta2),
|
||||
iteration(0)
|
||||
{
|
||||
// Nothing to do.
|
||||
}
|
||||
|
||||
//! Get the value for numerical stability.
|
||||
double Epsilon() const { return epsilon; }
|
||||
//! Modify the value used for numerical stability.
|
||||
double& Epsilon() { return epsilon; }
|
||||
|
||||
//! Get the exponential decay rate for the 1st moment estimates.
|
||||
double Beta1() const { return beta1; }
|
||||
//! Modify the exponential decay rate for the 1st moment estimates.
|
||||
double& Beta1() { return beta1; }
|
||||
|
||||
//! Get the exponential decay rate for the 2nd moment estimates.
|
||||
double Beta2() const { return beta2; }
|
||||
//! Modify the exponential decay rate for the 2nd moment estimates.
|
||||
double& Beta2() { return beta2; }
|
||||
|
||||
//! Get the current iteration number.
|
||||
size_t Iteration() const { return iteration; }
|
||||
//! Modify the current iteration number.
|
||||
size_t& Iteration() { return iteration; }
|
||||
|
||||
/**
|
||||
* The UpdatePolicyType policy classes must contain an internal 'Policy'
|
||||
* template class with two template arguments: MatType and GradType. This is
|
||||
* instantiated at the start of the optimization, and holds parameters
|
||||
* specific to an individual optimization.
|
||||
*/
|
||||
template<typename MatType, typename GradType>
|
||||
class Policy
|
||||
{
|
||||
public:
|
||||
/**
|
||||
* This constructor is called by the SGD Optimize() method before the start
|
||||
* of the iteration update process.
|
||||
*
|
||||
* @param parent AdaBeliefUpdate object.
|
||||
* @param rows Number of rows in the gradient matrix.
|
||||
* @param cols Number of columns in the gradient matrix.
|
||||
*/
|
||||
Policy(AdaBeliefUpdate& parent, const size_t rows, const size_t cols) :
|
||||
parent(parent)
|
||||
{
|
||||
m.zeros(rows, cols);
|
||||
s.zeros(rows, cols);
|
||||
}
|
||||
|
||||
/**
|
||||
* Update step for AdaBelief.
|
||||
*
|
||||
* @param iterate Parameters that minimize the function.
|
||||
* @param stepSize Step size to be used for the given iteration.
|
||||
* @param gradient The gradient matrix.
|
||||
*/
|
||||
void Update(MatType& iterate,
|
||||
const double stepSize,
|
||||
const GradType& gradient)
|
||||
{
|
||||
// Increment the iteration counter variable.
|
||||
++parent.iteration;
|
||||
|
||||
m *= parent.beta1;
|
||||
m += (1 - parent.beta1) * gradient;
|
||||
|
||||
s *= parent.beta2;
|
||||
s += (1 - parent.beta2) * arma::pow(gradient - m, 2.0) + parent.epsilon;
|
||||
|
||||
const double biasCorrection1 = 1.0 - std::pow(parent.beta1,
|
||||
parent.iteration);
|
||||
const double biasCorrection2 = 1.0 - std::pow(parent.beta2,
|
||||
parent.iteration);
|
||||
|
||||
// And update the iterate.
|
||||
iterate -= ((m / biasCorrection1) * stepSize) / (arma::sqrt(s /
|
||||
biasCorrection2) + parent.epsilon);
|
||||
}
|
||||
|
||||
private:
|
||||
//! Instantiated parent object.
|
||||
AdaBeliefUpdate& parent;
|
||||
|
||||
//! The exponential moving average of gradient values.
|
||||
GradType m;
|
||||
|
||||
// The exponential moving average of squared gradient values.
|
||||
GradType s;
|
||||
};
|
||||
|
||||
private:
|
||||
// The epsilon value used to initialise the squared gradient parameter.
|
||||
double epsilon;
|
||||
|
||||
// The xponential decay rate for the 1st moment estimates.
|
||||
double beta1;
|
||||
|
||||
// The exponential decay rate for the 2nd moment estimates.
|
||||
double beta2;
|
||||
|
||||
// The number of iterations.
|
||||
size_t iteration;
|
||||
};
|
||||
|
||||
} // namespace ens
|
||||
|
||||
#endif
|
||||
@@ -0,0 +1,189 @@
|
||||
/**
|
||||
* @file yogi.hpp
|
||||
* @author Marcus Edel
|
||||
*
|
||||
* Class wrapper for the Yogi update Policy. Yogi is based on Adam with more
|
||||
* fine grained effective learning rate control.
|
||||
*
|
||||
* ensmallen is free software; you may redistribute it and/or modify it under
|
||||
* the terms of the 3-clause BSD license. You should have received a copy of
|
||||
* the 3-clause BSD license along with ensmallen. If not, see
|
||||
* http://www.opensource.org/licenses/BSD-3-Clause for more information.
|
||||
*/
|
||||
#ifndef ENSMALLEN_YOGI_YOGI_HPP
|
||||
#define ENSMALLEN_YOGI_YOGI_HPP
|
||||
|
||||
#include <ensmallen_bits/sgd/sgd.hpp>
|
||||
#include "yogi_update.hpp"
|
||||
|
||||
namespace ens {
|
||||
|
||||
/**
|
||||
* Yogi is an variation of Adam with more fine grained effective learning rate
|
||||
* control.
|
||||
*
|
||||
* For more information, see the following.
|
||||
*
|
||||
* @code
|
||||
* @inproceedings{Zaheer2018,
|
||||
* author = {Zaheer, Manzil and Reddi, Sashank J. and Sachan, Devendra
|
||||
* and Kale, Satyen and Kumar, Sanjiv},
|
||||
* title = {Adaptive Methods for Nonconvex Optimization},
|
||||
* year = {2018},
|
||||
* publisher = {Curran Associates Inc.},
|
||||
* booktitle = {Proceedings of the 32nd International Conference on Neural
|
||||
* Information Processing Systems},
|
||||
* pages = {9815–9825},
|
||||
* series = {NIPS'18}
|
||||
* }
|
||||
* @endcode
|
||||
*
|
||||
* Yogi can optimize differentiable separable functions. For more details,
|
||||
* see the documentation on function types included with this distribution or
|
||||
* on the ensmallen website.
|
||||
*/
|
||||
class Yogi
|
||||
{
|
||||
public:
|
||||
/**
|
||||
* Construct the Yogi optimizer with the given function and parameters.
|
||||
* Yogi is sensitive to its paramters and hence a good hyper paramater
|
||||
* selection is necessary as its default may not fit every case.
|
||||
*
|
||||
* The maximum number of iterations refers to the maximum number of
|
||||
* points that are processed (i.e., one iteration equals one point; one
|
||||
* iteration does not equal one pass over the dataset).
|
||||
*
|
||||
* @param stepSize Step size for each iteration.
|
||||
* @param batchSize Number of points to process in a single step.
|
||||
* @param beta1 Exponential decay rate for the first moment estimates.
|
||||
* @param beta2 Exponential decay rate for the weighted infinity norm
|
||||
* estimates.
|
||||
* @param epsilon Value used to initialise the mean squared gradient
|
||||
* parameter.
|
||||
* @param maxIterations Maximum number of iterations allowed (0 means no
|
||||
* limit).
|
||||
* @param tolerance Maximum absolute tolerance to terminate algorithm.
|
||||
* @param shuffle If true, the function order is shuffled; otherwise, each
|
||||
* function is visited in linear order.
|
||||
* @param resetPolicy If true, parameters are reset before every Optimize
|
||||
* call; otherwise, their values are retained.
|
||||
* @param exactObjective Calculate the exact objective (Default: estimate the
|
||||
* final objective obtained on the last pass over the data).
|
||||
*/
|
||||
Yogi(const double stepSize = 0.001,
|
||||
const size_t batchSize = 32,
|
||||
const double beta1 = 0.9,
|
||||
const double beta2 = 0.999,
|
||||
const double epsilon = 1e-8,
|
||||
const size_t maxIterations = 100000,
|
||||
const double tolerance = 1e-5,
|
||||
const bool shuffle = true,
|
||||
const bool resetPolicy = true,
|
||||
const bool exactObjective = false);
|
||||
|
||||
/**
|
||||
* Optimize the given function using Yogi. The given starting point will be
|
||||
* modified to store the finishing point of the algorithm, and the final
|
||||
* objective value is returned.
|
||||
*
|
||||
* @tparam SeparableFunctionType Type of the function to optimize.
|
||||
* @tparam MatType Type of matrix to optimize with.
|
||||
* @tparam GradType Type of matrix to use to represent function gradients.
|
||||
* @tparam CallbackTypes Types of callback functions.
|
||||
* @param function Function to optimize.
|
||||
* @param iterate Starting point (will be modified).
|
||||
* @param callbacks Callback functions.
|
||||
* @return Objective value of the final point.
|
||||
*/
|
||||
template<typename SeparableFunctionType,
|
||||
typename MatType,
|
||||
typename GradType,
|
||||
typename... CallbackTypes>
|
||||
typename std::enable_if<IsArmaType<GradType>::value,
|
||||
typename MatType::elem_type>::type
|
||||
Optimize(SeparableFunctionType& function,
|
||||
MatType& iterate,
|
||||
CallbackTypes&&... callbacks)
|
||||
{
|
||||
return optimizer.Optimize<SeparableFunctionType, MatType, GradType,
|
||||
CallbackTypes...>(function, iterate,
|
||||
std::forward<CallbackTypes>(callbacks)...);
|
||||
}
|
||||
|
||||
//! Forward the MatType as GradType.
|
||||
template<typename SeparableFunctionType,
|
||||
typename MatType,
|
||||
typename... CallbackTypes>
|
||||
typename MatType::elem_type Optimize(SeparableFunctionType& function,
|
||||
MatType& iterate,
|
||||
CallbackTypes&&... callbacks)
|
||||
{
|
||||
return Optimize<SeparableFunctionType, MatType, MatType,
|
||||
CallbackTypes...>(function, iterate,
|
||||
std::forward<CallbackTypes>(callbacks)...);
|
||||
}
|
||||
|
||||
//! Get the step size.
|
||||
double StepSize() const { return optimizer.StepSize(); }
|
||||
//! Modify the step size.
|
||||
double& StepSize() { return optimizer.StepSize(); }
|
||||
|
||||
//! Get the batch size.
|
||||
size_t BatchSize() const { return optimizer.BatchSize(); }
|
||||
//! Modify the batch size.
|
||||
size_t& BatchSize() { return optimizer.BatchSize(); }
|
||||
|
||||
//! Get the smoothing parameter.
|
||||
double Beta1() const { return optimizer.UpdatePolicy().Beta1(); }
|
||||
//! Modify the smoothing parameter.
|
||||
double& Beta1() { return optimizer.UpdatePolicy().Beta1(); }
|
||||
|
||||
//! Get the second moment coefficient.
|
||||
double Beta2() const { return optimizer.UpdatePolicy().Beta2(); }
|
||||
//! Modify the second moment coefficient.
|
||||
double& Beta2() { return optimizer.UpdatePolicy().Beta2(); }
|
||||
|
||||
//! Get the value used to initialise the mean squared gradient parameter.
|
||||
double Epsilon() const { return optimizer.UpdatePolicy().Epsilon(); }
|
||||
//! Modify the value used to initialise the mean squared gradient parameter.
|
||||
double& Epsilon() { return optimizer.UpdatePolicy().Epsilon(); }
|
||||
|
||||
//! Get the maximum number of iterations (0 indicates no limit).
|
||||
size_t MaxIterations() const { return optimizer.MaxIterations(); }
|
||||
//! Modify the maximum number of iterations (0 indicates no limit).
|
||||
size_t& MaxIterations() { return optimizer.MaxIterations(); }
|
||||
|
||||
//! Get the tolerance for termination.
|
||||
double Tolerance() const { return optimizer.Tolerance(); }
|
||||
//! Modify the tolerance for termination.
|
||||
double& Tolerance() { return optimizer.Tolerance(); }
|
||||
|
||||
//! Get whether or not the individual functions are shuffled.
|
||||
bool Shuffle() const { return optimizer.Shuffle(); }
|
||||
//! Modify whether or not the individual functions are shuffled.
|
||||
bool& Shuffle() { return optimizer.Shuffle(); }
|
||||
|
||||
//! Get whether or not the actual objective is calculated.
|
||||
bool ExactObjective() const { return optimizer.ExactObjective(); }
|
||||
//! Modify whether or not the actual objective is calculated.
|
||||
bool& ExactObjective() { return optimizer.ExactObjective(); }
|
||||
|
||||
//! Get whether or not the update policy parameters are reset before
|
||||
//! Optimize call.
|
||||
bool ResetPolicy() const { return optimizer.ResetPolicy(); }
|
||||
//! Modify whether or not the update policy parameters
|
||||
//! are reset before Optimize call.
|
||||
bool& ResetPolicy() { return optimizer.ResetPolicy(); }
|
||||
|
||||
private:
|
||||
//! The Stochastic Gradient Descent object with Yogi policy.
|
||||
SGD<YogiUpdate> optimizer;
|
||||
};
|
||||
|
||||
} // namespace ens
|
||||
|
||||
// Include implementation.
|
||||
#include "yogi_impl.hpp"
|
||||
|
||||
#endif
|
||||
@@ -0,0 +1,44 @@
|
||||
/**
|
||||
* @file yogi_impl.hpp
|
||||
* @author Marcus Edel
|
||||
*
|
||||
* Implementation of Yogi class wrapper.
|
||||
*
|
||||
* ensmallen is free software; you may redistribute it and/or modify it under
|
||||
* the terms of the 3-clause BSD license. You should have received a copy of
|
||||
* the 3-clause BSD license along with ensmallen. If not, see
|
||||
* http://www.opensource.org/licenses/BSD-3-Clause for more information.
|
||||
*/
|
||||
#ifndef ENSMALLEN_YOGI_YOGI_IMPL_HPP
|
||||
#define ENSMALLEN_YOGI_YOGI_IMPL_HPP
|
||||
|
||||
// In case it hasn't been included yet.
|
||||
#include "yogi.hpp"
|
||||
|
||||
namespace ens {
|
||||
|
||||
inline Yogi::Yogi(
|
||||
const double stepSize,
|
||||
const size_t batchSize,
|
||||
const double beta1,
|
||||
const double beta2,
|
||||
const double epsilon,
|
||||
const size_t maxIterations,
|
||||
const double tolerance,
|
||||
const bool shuffle,
|
||||
const bool resetPolicy,
|
||||
const bool exactObjective) :
|
||||
optimizer(stepSize,
|
||||
batchSize,
|
||||
maxIterations,
|
||||
tolerance,
|
||||
shuffle,
|
||||
YogiUpdate(epsilon, beta1, beta2),
|
||||
NoDecay(),
|
||||
resetPolicy,
|
||||
exactObjective)
|
||||
{ /* Nothing to do. */ }
|
||||
|
||||
} // namespace ens
|
||||
|
||||
#endif
|
||||
@@ -0,0 +1,158 @@
|
||||
/**
|
||||
* @file yogi_update.hpp
|
||||
* @author Marcus Edel
|
||||
*
|
||||
* Implements the Yogi Optimizer. Yogi is a variant of Adam with more fine
|
||||
* grained effective learning rate control.
|
||||
*
|
||||
* ensmallen is free software; you may redistribute it and/or modify it under
|
||||
* the terms of the 3-clause BSD license. You should have received a copy of
|
||||
* the 3-clause BSD license along with ensmallen. If not, see
|
||||
* http://www.opensource.org/licenses/BSD-3-Clause for more information.
|
||||
*/
|
||||
#ifndef ENSMALLEN_YOGI_YOGI_UPDATE_HPP
|
||||
#define ENSMALLEN_YOGI_YOGI_UPDATE_HPP
|
||||
|
||||
namespace ens {
|
||||
|
||||
/**
|
||||
* Yogi builds upon the Adam update strategy but provides more fine grained
|
||||
* effective learning rate control.
|
||||
*
|
||||
* For more information, see the following.
|
||||
*
|
||||
* @code
|
||||
* @inproceedings{Zaheer2018,
|
||||
* author = {Zaheer, Manzil and Reddi, Sashank J. and Sachan, Devendra
|
||||
* and Kale, Satyen and Kumar, Sanjiv},
|
||||
* title = {Adaptive Methods for Nonconvex Optimization},
|
||||
* year = {2018},
|
||||
* publisher = {Curran Associates Inc.},
|
||||
* booktitle = {Proceedings of the 32nd International Conference on Neural
|
||||
* Information Processing Systems},
|
||||
* pages = {9815–9825},
|
||||
* series = {NIPS'18}
|
||||
* }
|
||||
* @endcode
|
||||
*/
|
||||
class YogiUpdate
|
||||
{
|
||||
public:
|
||||
/**
|
||||
* Construct the Yogi update policy with the given parameters.
|
||||
*
|
||||
* @param epsilon The epsilon value used to initialise the squared gradient
|
||||
* parameter.
|
||||
* @param beta1 The smoothing parameter.
|
||||
* @param beta2 The second moment coefficient.
|
||||
* @param v1 The first quasi-hyperbolic term.
|
||||
* @param v1 The second quasi-hyperbolic term.
|
||||
*/
|
||||
YogiUpdate(const double epsilon = 1e-8,
|
||||
const double beta1 = 0.9,
|
||||
const double beta2 = 0.999) :
|
||||
epsilon(epsilon),
|
||||
beta1(beta1),
|
||||
beta2(beta2),
|
||||
iteration(0)
|
||||
{
|
||||
// Nothing to do.
|
||||
}
|
||||
|
||||
//! Get the value used to initialise the squared gradient parameter.
|
||||
double Epsilon() const { return epsilon; }
|
||||
//! Modify the value used to initialise the squared gradient parameter.
|
||||
double& Epsilon() { return epsilon; }
|
||||
|
||||
//! Get the smoothing parameter.
|
||||
double Beta1() const { return beta1; }
|
||||
//! Modify the smoothing parameter.
|
||||
double& Beta1() { return beta1; }
|
||||
|
||||
//! Get the second moment coefficient.
|
||||
double Beta2() const { return beta2; }
|
||||
//! Modify the second moment coefficient.
|
||||
double& Beta2() { return beta2; }
|
||||
|
||||
//! Get the current iteration number.
|
||||
size_t Iteration() const { return iteration; }
|
||||
//! Modify the current iteration number.
|
||||
size_t& Iteration() { return iteration; }
|
||||
|
||||
/**
|
||||
* The UpdatePolicyType policy classes must contain an internal 'Policy'
|
||||
* template class with two template arguments: MatType and GradType. This is
|
||||
* instantiated at the start of the optimization, and holds parameters
|
||||
* specific to an individual optimization.
|
||||
*/
|
||||
template<typename MatType, typename GradType>
|
||||
class Policy
|
||||
{
|
||||
public:
|
||||
/**
|
||||
* This constructor is called by the SGD Optimize() method before the start
|
||||
* of the iteration update process.
|
||||
*
|
||||
* @param parent YogiUpdate object.
|
||||
* @param rows Number of rows in the gradient matrix.
|
||||
* @param cols Number of columns in the gradient matrix.
|
||||
*/
|
||||
Policy(YogiUpdate& parent, const size_t rows, const size_t cols) :
|
||||
parent(parent)
|
||||
{
|
||||
m.zeros(rows, cols);
|
||||
v.zeros(rows, cols);
|
||||
}
|
||||
|
||||
/**
|
||||
* Update step for Yogi.
|
||||
*
|
||||
* @param iterate Parameters that minimize the function.
|
||||
* @param stepSize Step size to be used for the given iteration.
|
||||
* @param gradient The gradient matrix.
|
||||
*/
|
||||
void Update(MatType& iterate,
|
||||
const double stepSize,
|
||||
const GradType& gradient)
|
||||
{
|
||||
// Increment the iteration counter variable.
|
||||
++parent.iteration;
|
||||
|
||||
m *= parent.beta1;
|
||||
m += (1 - parent.beta1) * gradient;
|
||||
|
||||
const MatType gSquared = arma::square(gradient);
|
||||
v -= (1 - parent.beta2) * arma::sign(v - gSquared) % gSquared;
|
||||
|
||||
// And update the iterate.
|
||||
iterate -= stepSize * m / (arma::sqrt(v) + parent.epsilon);
|
||||
}
|
||||
|
||||
private:
|
||||
//! Instantiated parent object.
|
||||
YogiUpdate& parent;
|
||||
|
||||
//! The exponential moving average of gradient values.
|
||||
GradType m;
|
||||
|
||||
// The exponential moving average of squared gradient values.
|
||||
GradType v;
|
||||
};
|
||||
|
||||
private:
|
||||
// The epsilon value used to initialise the squared gradient parameter.
|
||||
double epsilon;
|
||||
|
||||
// The smoothing parameter.
|
||||
double beta1;
|
||||
|
||||
// The second moment coefficient.
|
||||
double beta2;
|
||||
|
||||
// The number of iterations.
|
||||
size_t iteration;
|
||||
};
|
||||
|
||||
} // namespace ens
|
||||
|
||||
#endif
|
||||
@@ -1,6 +1,7 @@
|
||||
# The tests that need to be compiled.
|
||||
set(ENSMALLEN_TESTS_SOURCES
|
||||
main.cpp
|
||||
ada_belief_test.cpp
|
||||
ada_bound_test.cpp
|
||||
ada_delta_test.cpp
|
||||
ada_grad_test.cpp
|
||||
@@ -46,6 +47,7 @@ set(ENSMALLEN_TESTS_SOURCES
|
||||
svrg_test.cpp
|
||||
swats_test.cpp
|
||||
wn_grad_test.cpp
|
||||
yogi_test.cpp
|
||||
)
|
||||
|
||||
set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR})
|
||||
|
||||
@@ -0,0 +1,71 @@
|
||||
/**
|
||||
* @file ada_belief_test.cpp
|
||||
* @author Marcus Edel
|
||||
*
|
||||
* ensmallen is free software; you may redistribute it and/or modify it under
|
||||
* the terms of the 3-clause BSD license. You should have received a copy of
|
||||
* the 3-clause BSD license along with ensmallen. If not, see
|
||||
* http://www.opensource.org/licenses/BSD-3-Clause for more information.
|
||||
*/
|
||||
|
||||
#include <ensmallen.hpp>
|
||||
#include "catch.hpp"
|
||||
#include "test_function_tools.hpp"
|
||||
|
||||
using namespace ens;
|
||||
using namespace ens::test;
|
||||
|
||||
/**
|
||||
* Test the AdaBelief optimizer on the Sphere function.
|
||||
*/
|
||||
TEST_CASE("AdaBeliefSphereFunctionTest", "[AdaBeliefTest]")
|
||||
{
|
||||
AdaBelief optimizer(0.5, 2, 0.7, 0.999, 1e-8, 500000, 1e-3, false);
|
||||
FunctionTest<SphereFunction>(optimizer, 0.5, 0.1);
|
||||
}
|
||||
|
||||
/**
|
||||
* Test the AdaBelief optimizer on the Sphere function with arma::fmat.
|
||||
*/
|
||||
TEST_CASE("AdaBeliefSphereFunctionTestFMat", "[AdaBeliefTest]")
|
||||
{
|
||||
AdaBelief optimizer(0.5, 2, 0.7, 0.999, 1e-8, 500000, 1e-3, false);
|
||||
FunctionTest<SphereFunction, arma::fmat>(optimizer, 0.5, 0.1);
|
||||
}
|
||||
|
||||
/**
|
||||
* Test the AdaBelief optimizer on the McCormick function.
|
||||
*/
|
||||
TEST_CASE("AdaBeliefMcCormickFunctionTest", "[AdaBeliefTest]")
|
||||
{
|
||||
AdaBelief optimizer(0.5, 1, 0.7, 0.999, 1e-8, 500000, 1e-5, false);
|
||||
FunctionTest<McCormickFunction>(optimizer, 0.5, 0.1);
|
||||
}
|
||||
|
||||
/**
|
||||
* Run AdaBelief on logistic regression and make sure the results are
|
||||
* acceptable.
|
||||
*/
|
||||
TEST_CASE("AdaBeliefLogisticRegressionTest", "[AdaBeliefTest]")
|
||||
{
|
||||
AdaBelief optimizer;
|
||||
LogisticRegressionFunctionTest(optimizer, 0.003, 0.006);
|
||||
}
|
||||
|
||||
/**
|
||||
* Run AdaBelief on logistic regression and make sure the results are
|
||||
* acceptable, using arma::fmat.
|
||||
*/
|
||||
TEST_CASE("AdaBeliefLogisticRegressionFMatTest", "[AdaBeliefTest]")
|
||||
{
|
||||
arma::fmat data, testData, shuffledData;
|
||||
arma::Row<size_t> responses, testResponses, shuffledResponses;
|
||||
|
||||
LogisticRegressionTestData(data, testData, shuffledData,
|
||||
responses, testResponses, shuffledResponses);
|
||||
LogisticRegression<arma::fmat> lr(shuffledData, shuffledResponses, 0.5);
|
||||
|
||||
AdaBelief optimizer;
|
||||
LogisticRegressionFunctionTest<arma::fmat>(optimizer, 0.003, 0.006);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,74 @@
|
||||
/**
|
||||
* @file yogi_test.cpp
|
||||
* @author Marcus Edel
|
||||
*
|
||||
* ensmallen is free software; you may redistribute it and/or modify it under
|
||||
* the terms of the 3-clause BSD license. You should have received a copy of
|
||||
* the 3-clause BSD license along with ensmallen. If not, see
|
||||
* http://www.opensource.org/licenses/BSD-3-Clause for more information.
|
||||
*/
|
||||
|
||||
#include <ensmallen.hpp>
|
||||
#include "catch.hpp"
|
||||
#include "test_function_tools.hpp"
|
||||
|
||||
using namespace ens;
|
||||
using namespace ens::test;
|
||||
|
||||
/**
|
||||
* Test the Yogi optimizer on the Sphere function.
|
||||
*/
|
||||
TEST_CASE("YogiSphereFunctionTest", "[YogiTest]")
|
||||
{
|
||||
SphereFunction f(2);
|
||||
Yogi optimizer(0.5, 2, 0.7, 0.999, 1e-8, 500000, 1e-3, false);
|
||||
|
||||
arma::mat coordinates = f.GetInitialPoint();
|
||||
optimizer.Optimize(f, coordinates);
|
||||
|
||||
REQUIRE(coordinates(0) == Approx(0.0).margin(0.1));
|
||||
REQUIRE(coordinates(1) == Approx(0.0).margin(0.1));
|
||||
}
|
||||
|
||||
/**
|
||||
* Test the Yogi optimizer on the Sphere function with arma::fmat.
|
||||
*/
|
||||
TEST_CASE("YogiSphereFunctionTestFMat", "[YogiTest]")
|
||||
{
|
||||
SphereFunction f(2);
|
||||
Yogi optimizer(0.5, 2, 0.7, 0.999, 1e-8, 500000, 1e-3, false);
|
||||
|
||||
arma::fmat coordinates = f.GetInitialPoint<arma::fmat>();
|
||||
optimizer.Optimize(f, coordinates);
|
||||
|
||||
REQUIRE(coordinates(0) == Approx(0.0).margin(0.1));
|
||||
REQUIRE(coordinates(1) == Approx(0.0).margin(0.1));
|
||||
}
|
||||
|
||||
/**
|
||||
* Test the Yogi optimizer on the McCormick function.
|
||||
*/
|
||||
TEST_CASE("YogiMcCormickFunctionTest", "[YogiTest]")
|
||||
{
|
||||
Yogi optimizer(0.5, 1, 0.7, 0.999, 1e-8, 500000, 1e-5, false);
|
||||
FunctionTest<McCormickFunction>(optimizer, 0.5, 0.1);
|
||||
}
|
||||
|
||||
/**
|
||||
* Run Yogi on logistic regression and make sure the results are acceptable.
|
||||
*/
|
||||
TEST_CASE("YogiLogisticRegressionTest", "[YogiTest]")
|
||||
{
|
||||
Yogi optimizer;
|
||||
LogisticRegressionFunctionTest(optimizer, 0.003, 0.006);
|
||||
}
|
||||
|
||||
/**
|
||||
* Run Yogi on logistic regression and make sure the results are acceptable,
|
||||
* using arma::fmat.
|
||||
*/
|
||||
TEST_CASE("YogiLogisticRegressionFMatTest", "[YogiTest]")
|
||||
{
|
||||
Yogi optimizer;
|
||||
LogisticRegressionFunctionTest<arma::fmat>(optimizer, 0.003, 0.006);
|
||||
}
|
||||
Reference in New Issue
Block a user