apaam/mlpack
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Added proper unit tests for gradient clipping + some other minor fixes

Browse Source
This commit is contained in:
Konstantin Sidorov
2017-07-24 15:51:47 +03:00
parent 336855ccde
commit ed7a486576
8 changed files with 114 additions and 50 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/mlpack/core/optimizers/sgd/update_policies/gradient_clipping.hpp
+21 -8
View File
@@ -25,7 +25,7 @@ namespace optimization {
*
* @tparam UpdatePolicy A type of UpdatePolicy that sould be wrapped around.
*/
template<typename UpdatePolicy>
template<typename UpdatePolicyType>
class GradientClipping
{
public:
@@ -34,12 +34,12 @@ class GradientClipping
*
* @param minGradient Minimum possible value of gradient element.
* @param maxGradient Maximum possible value of gradient element.
* @param updatePolicy An instance of the UpdatePolicy
* @param updatePolicy An instance of the UpdatePolicyType
* used for actual optimization.
*/
GradientClipping(const double minGradient,
const double maxGradient,
UpdatePolicy updatePolicy) :
UpdatePolicyType& updatePolicy) :
minGradient(minGradient),
maxGradient(maxGradient),
updatePolicy(updatePolicy)
@@ -73,12 +73,25 @@ class GradientClipping
const arma::mat& gradient)
{
// First, clip the gradient.
gradient.transform(
[&](double val)
{ return std::min(std::max(val, minGradient), maxGradient); });
arma::mat clippedGradient = arma::clamp(gradient, minGradient, maxGradient);
// And only then do the update.
updatePolicy.Update(iterate, stepSize, gradient);
updatePolicy.Update(iterate, stepSize, clippedGradient);
}
//! Get the update policy.
UpdatePolicyType& UpdatePolicy() const { return updatePolicy; }
//! Modify the update policy.
UpdatePolicyType& UpdatePolicy() { return updatePolicy; }
//! Get the minimum gradient value.
double MinGradient() const { return minGradient; }
//! Modify the minimum gradient value.
double& MinGradient() { return minGradient; }
//! Get the maximum gradient value.
double MaxGradient() const { return maxGradient; }
//! Modify the maximum gradient value.
double& MaxGradient() { return maxGradient; }
private:
//! Minimum possible value of gradient element.
double minGradient;
@@ -87,7 +100,7 @@ class GradientClipping
double maxGradient;
//! An instance of the UpdatePolicy used for actual optimization.
UpdatePolicy updatePolicy;
UpdatePolicyType updatePolicy;
};
} // namespace optimization
src/mlpack/methods/ann/layer/cross_entropy_error.hpp
+12 -1
View File
@@ -36,8 +36,11 @@ class CrossEntropyError
public:
/**
* Create the CrossEntropyError object.
*
* @param eps The minimum value used for computing logarithms
* and denominators in a numerically stable way.
*/
CrossEntropyError();
CrossEntropyError(double eps = 1e-10);
/*
* Computes the cross-entropy function.
@@ -74,6 +77,11 @@ class CrossEntropyError
//! Modify the delta.
OutputDataType& Delta() { return delta; }
//! Get the epsilon.
double Eps() const { return eps; }
//! Modify the epsilon.
double& Eps() { return eps; }
/**
* Serialize the layer
*/
@@ -89,6 +97,9 @@ class CrossEntropyError
//! Locally-stored output parameter object.
OutputDataType outputParameter;
//! The minimum value used for computing logarithms and denominators
double eps;
}; // class CrossEntropyError
} // namespace ann
src/mlpack/methods/ann/layer/cross_entropy_error_impl.hpp
+5 -4
View File
@@ -19,7 +19,8 @@ namespace mlpack {
namespace ann /** Artificial Neural Network. */ {
template<typename InputDataType, typename OutputDataType>
CrossEntropyError<InputDataType, OutputDataType>::CrossEntropyError()
CrossEntropyError<InputDataType, OutputDataType>::CrossEntropyError(double eps)
: eps(eps)
{
// Nothing to do here.
}
@@ -29,8 +30,8 @@ template<typename eT>
double CrossEntropyError<InputDataType, OutputDataType>::Forward(
const arma::Mat<eT>&& input, const arma::Mat<eT>&& target)
{
return -arma::accu(target % arma::trunc_log(input) +
(1. - target) % arma::trunc_log(1. - input));
return -arma::accu(target % arma::log(input + eps) +
(1. - target) % arma::log(1. - input + eps));
}
template<typename InputDataType, typename OutputDataType>
@@ -40,7 +41,7 @@ void CrossEntropyError<InputDataType, OutputDataType>::Backward(
const arma::Mat<eT>&& target,
arma::Mat<eT>&& output)
{
output = (1. - target) / (1. - input + 1e-2) - target / (input + 1e-2);
output = (1. - target) / (1. - input + eps) - target / (input + eps);
}
template<typename InputDataType, typename OutputDataType>
src/mlpack/tests/CMakeLists.txt
+1
View File
@@ -29,6 +29,7 @@ add_executable(mlpack_test
fastmks_test.cpp
feedforward_network_test.cpp
gmm_test.cpp
gradient_clipping_test.cpp
gradient_descent_test.cpp
hmm_test.cpp
hoeffding_tree_test.cpp
src/mlpack/tests/ann_layer_test.cpp
+2 -2
View File
@@ -26,7 +26,7 @@ using namespace mlpack::ann;
BOOST_AUTO_TEST_SUITE(ANNLayerTest);
// Helper function whcih calls the Reset function of the given module.
// Helper function which calls the Reset function of the given module.
template<class T>
void ResetFunction(
T& layer,
@@ -531,7 +531,7 @@ BOOST_AUTO_TEST_CASE(JacobianLinearNoBiasLayerTest)
/**
* LinearNoBias layer numerically gradient test.
*/
BOOST_AUTO_TEST_CASE(GradientLinearNoBiadLayerTest)
BOOST_AUTO_TEST_CASE(GradientLinearNoBiasLayerTest)
{
// LinearNoBias function gradient instantiation.
struct GradientFunction
src/mlpack/tests/gradient_clipping_test.cpp
+73
View File
@@ -0,0 +1,73 @@
/**
* @file gradient_clipping_test.cpp
* @author Konstantin Sidorov
*
* Test file for gradient clipping.
*
* mlpack is free software; you may redistribute it and/or modify it under the
* terms of the 3-clause BSD license. You should have received a copy of the
* 3-clause BSD license along with mlpack. If not, see
* http://www.opensource.org/licenses/BSD-3-Clause for more information.
*/
#include <mlpack/core.hpp>
#include <mlpack/core/optimizers/sgd/update_policies/gradient_clipping.hpp>
#include <mlpack/core/optimizers/sgd/update_policies/vanilla_update.hpp>
#include <mlpack/core/optimizers/sgd/update_policies/momentum_update.hpp>
#include <mlpack/core/optimizers/lbfgs/test_functions.hpp>
#include <mlpack/core/optimizers/sgd/test_function.hpp>
#include <boost/test/unit_test.hpp>
#include "test_tools.hpp"
using namespace std;
using namespace arma;
using namespace mlpack;
using namespace mlpack::optimization;
using namespace mlpack::optimization::test;
BOOST_AUTO_TEST_SUITE(GradientClippingTest);
// Test checking that gradient clipping works with vanilla update.
BOOST_AUTO_TEST_CASE(ClippedVanillaUpdateTest)
{
VanillaUpdate vanillaUpdate;
GradientClipping<VanillaUpdate> update(-3.0, +3.0, vanillaUpdate);
update.Initialize(3, 3);
arma::mat coordinates = arma::zeros(3, 3);
// Setting step = 1 to make math easy.
double stepSize = 1.0;
arma::mat dummyGradient = { {-6, +6, 0}, {1, 2, 3}, {-3, 0, +4} };
update.Update(coordinates, stepSize, dummyGradient);
// After clipping, we should get the following coordinates:
arma::mat targetCoordinates = { { 3, -3, 0}, {-1, -2, -3}, {3, 0, -3} };
BOOST_REQUIRE_SMALL(arma::abs(coordinates - targetCoordinates).max(), 1e-7);
}
// Test checking that gradient clipping works with momentum update.
BOOST_AUTO_TEST_CASE(ClippedMomentumUpdateTest)
{
// Once again, setting momentum = 1 for easy math
// (now momentum = -stepSize * [sum of gradients])
MomentumUpdate momentumUpdate(1);
GradientClipping<MomentumUpdate> update(-3.0, +3.0, momentumUpdate);
update.Initialize(3, 3);
arma::mat coordinates = arma::zeros(3, 3);
double stepSize = 1.0;
arma::mat dummyGradient = { {-6, +6, 0}, {1, 2, 3}, {-3, 0, +4} };
update.Update(coordinates, stepSize, dummyGradient);
arma::mat targetCoordinates = { { 3, -3, 0}, {-1, -2, -3}, {3, 0, -3} };
// On the first Update() call the parameters
// should just be equal to (-gradient).
BOOST_REQUIRE_SMALL(arma::abs(coordinates - targetCoordinates).max(), 1e-7);
update.Update(coordinates, stepSize, dummyGradient);
// On the second Update() call the Momentum update will subtract
// the gradient from the momentum, which gives 2 * gradient value
// for the momentum on that step. Adding that to the gradient which
// was subtracted earlier yiels the 3 * gradient in the following check.
BOOST_REQUIRE_SMALL(arma::abs(coordinates - 3 * targetCoordinates).max(),
1e-7);
}
BOOST_AUTO_TEST_SUITE_END();
src/mlpack/tests/momentum_sgd_test.cpp
-18
View File
@@ -57,24 +57,6 @@ BOOST_AUTO_TEST_CASE(MomentumSGDSpeedUpTestFunction)
BOOST_REQUIRE_LE(result, result1);
}
// Test checking that gradient clipping works with momentum SGD.
BOOST_AUTO_TEST_CASE(ClippedSGDTestFunction)
{
SGDTestFunction f;
MomentumUpdate momentumUpdate;
GradientClipping<MomentumUpdate> update(-3., +3., momentumUpdate);
StandardSGD s(0.0003, 5000000, 1e-9, true);
arma::mat coordinates = f.GetInitialPoint();
double result = s.Optimize(f, coordinates);
BOOST_REQUIRE_CLOSE(result, -1.0, 0.05);
BOOST_REQUIRE_SMALL(coordinates[0], 1e-3);
BOOST_REQUIRE_SMALL(coordinates[1], 1e-7);
BOOST_REQUIRE_SMALL(coordinates[2], 1e-7);
}
BOOST_AUTO_TEST_CASE(GeneralizedRosenbrockTest)
{
// Loop over several variants.
src/mlpack/tests/sgd_test.cpp
-17
View File
@@ -41,23 +41,6 @@ BOOST_AUTO_TEST_CASE(SimpleSGDTestFunction)
BOOST_REQUIRE_SMALL(coordinates[2], 1e-7);
}
// Test checking that gradient clipping works with vanilla SGD.
BOOST_AUTO_TEST_CASE(ClippedSGDTestFunction)
{
SGDTestFunction f;
VanillaUpdate vanillaUpdate;
GradientClipping<VanillaUpdate> update(-3., +3., vanillaUpdate);
StandardSGD s(0.0003, 5000000, 1e-9, true);
arma::mat coordinates = f.GetInitialPoint();
double result = s.Optimize(f, coordinates);
BOOST_REQUIRE_CLOSE(result, -1.0, 0.05);
BOOST_REQUIRE_SMALL(coordinates[0], 1e-3);
BOOST_REQUIRE_SMALL(coordinates[1], 1e-7);
BOOST_REQUIRE_SMALL(coordinates[2], 1e-7);
}
BOOST_AUTO_TEST_CASE(GeneralizedRosenbrockTest)
{
// Loop over several variants.