apaam/mlpack
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Address comments and minor style fixes.

Browse Source
This commit is contained in:
Marcus Edel
2018-02-10 19:27:09 +01:00
parent 4eb3018077
commit 4d0c2bb254
7 changed files with 28 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/mlpack/core/optimizers/katyusha/katyusha.hpp
+4 -3
View File
@@ -19,7 +19,7 @@ namespace optimization {
/**
* Katyusha is a direct, primal-only stochastic gradient method which uses a
* negative momentum” on top of Nesterov’s momentum.
* "negative momentum" on top of Nesterov’s momentum.
*
* For more information, see the following.
*
@@ -59,7 +59,7 @@ namespace optimization {
*
* @tparam proximal Whether the proximal update should be used or not.
*/
template<bool proximal = false>
template<bool Proximal = false>
class KatyushaType
{
public:
@@ -77,7 +77,8 @@ class KatyushaType
* @param maxIterations Maximum number of iterations allowed (0 means no
* limit).
* @param innerIterations The number of inner iterations allowed (0 means
* n / batchSize).
* n / batchSize). Note that the full gradient is only calculated in
* the outer iteration.
* @param tolerance Maximum absolute tolerance to terminate algorithm.
* @param shuffle If true, the function order is shuffled; otherwise, each
* function is visited in linear order.
src/mlpack/core/optimizers/katyusha/katyusha_impl.hpp
+15 -7
View File
@@ -18,8 +18,8 @@
namespace mlpack {
namespace optimization {
template<bool proximal>
KatyushaType<proximal>::KatyushaType(
template<bool Proximal>
KatyushaType<Proximal>::KatyushaType(
const double convexity,
const double lipschitz,
const size_t batchSize,
@@ -37,9 +37,9 @@ KatyushaType<proximal>::KatyushaType(
{ /* Nothing to do. */ }
//! Optimize the function (minimize).
template<bool proximal>
template<bool Proximal>
template<typename DecomposableFunctionType>
double KatyushaType<proximal>::Optimize(
double KatyushaType<Proximal>::Optimize(
DecomposableFunctionType& function,
arma::mat& iterate)
{
@@ -66,7 +66,7 @@ double KatyushaType<proximal>::Optimize(
double normalizer = 1;
for (size_t i = 0; i < numBatches; i++)
{
normalizer = r * (normalizer + 1.0);
normalizer = r * (normalizer + 1.0);
}
normalizer = 1.0 / normalizer;
@@ -156,15 +156,23 @@ double KatyushaType<proximal>::Optimize(
function.Gradient(iterate0, currentFunction, gradient0,
effectiveBatchSize);
// By the minimality definition of z_{k + 1}, we have that:
// z_{k+1} − z_k + \alpha * \sigma_{k+1} + \alpha g = 0.
arma::mat zNew = z - alpha * (fullGradient + (gradient - gradient0) /
(double) batchSize);
// Proximal update, choose between Option I and Option II. Shift relative
// to the Lipschitz constant or take a constant step using the given step
// size.
if (proximal)
if (Proximal)
{
y = iterate + 1.0 / (3.0 * lipschitz) * (zNew - z);
// yk = x0 − 1 / (3L) * \delta1, k = 1
// yk = x0 − 1 / (3L) * \delta2 - ((1 - tau) / (3L)) + tau * alpha)
// * \delta1, k = 2
// yk = x0 − 1 / (3L) * \delta3 - ((1 - tau) / (3L)) + tau * alpha)
// * \delta2 - ((1-tau)^2 / (3L) + (1 - (1 - tau)^2) * alpha) * \delta1,
// k = 3.
y = iterate + 1.0 / (3.0 * lipschitz) * w;
}
else
{
src/mlpack/core/optimizers/sarah/CMakeLists.txt
+1 -1
View File
@@ -1,5 +1,5 @@
set(SOURCES
sarah.hpp
sarah.hpp
sarah_impl.hpp
sarah_plus_update.hpp
sarah_update.hpp
src/mlpack/core/optimizers/sarah/sarah.hpp
+2 -1
View File
@@ -79,7 +79,8 @@ class SARAHType
* @param maxIterations Maximum number of iterations allowed (0 means no
* limit).
* @param innerIterations The number of inner iterations allowed (0 means
* n / batchSize).
* n / batchSize). Note that the full gradient is only calculated in
* the outer iteration.
* @param tolerance Maximum absolute tolerance to terminate algorithm.
* @param shuffle If true, the function order is shuffled; otherwise, each
* function is visited in linear order.
src/mlpack/core/optimizers/sarah/sarah_impl.hpp
+2 -2
View File
@@ -78,7 +78,7 @@ double SARAHType<UpdatePolicyType>::Optimize(
if (std::isnan(overallObjective) || std::isinf(overallObjective))
{
Log::Warn << "Katyusha: converged to " << overallObjective
Log::Warn << "SARAH: converged to " << overallObjective
<< "; terminating with failure. Try a smaller step size?"
<< std::endl;
return overallObjective;
@@ -86,7 +86,7 @@ double SARAHType<UpdatePolicyType>::Optimize(
if (std::abs(lastObjective - overallObjective) < tolerance)
{
Log::Info << "Katyusha: minimized within tolerance " << tolerance
Log::Info << "SARAH: minimized within tolerance " << tolerance
<< "; terminating optimization." << std::endl;
return overallObjective;
}
src/mlpack/core/optimizers/svrg/svrg.hpp
+2 -1
View File
@@ -114,7 +114,8 @@ class SVRGType
* @param maxIterations Maximum number of iterations allowed (0 means no
* limit).
* @param innerIterations The number of inner iterations allowed (0 means
* n / batchSize).
* n / batchSize). Note that the full gradient is only calculated in
* the outer iteration.
* @param tolerance Maximum absolute tolerance to terminate algorithm.
* @param shuffle If true, the function order is shuffled; otherwise, each
* function is visited in linear order.
src/mlpack/core/optimizers/svrg/svrg_impl.hpp
+2 -2
View File
@@ -86,7 +86,7 @@ double SVRGType<UpdatePolicyType, DecayPolicyType>::Optimize(
if (std::isnan(overallObjective) || std::isinf(overallObjective))
{
Log::Warn << "Katyusha: converged to " << overallObjective
Log::Warn << "SVRG: converged to " << overallObjective
<< "; terminating with failure. Try a smaller step size?"
<< std::endl;
return overallObjective;
@@ -94,7 +94,7 @@ double SVRGType<UpdatePolicyType, DecayPolicyType>::Optimize(
if (std::abs(lastObjective - overallObjective) < tolerance)
{
Log::Info << "Katyusha: minimized within tolerance " << tolerance
Log::Info << "SVRG: minimized within tolerance " << tolerance
<< "; terminating optimization." << std::endl;
return overallObjective;
}