Marcus Edel
4ae22bc9a1
The AdamSchafferFunctionN2Test fails on some specific aarch64 machines, since we are unable to reproduce the issue on other devices, we decided to remove the test from the Adam test suite. The Adam optimizer is still tested against many different functions, which all pass, so we don't think the implementation is wrong. It might be possible to optimize the initial function and optimizer settings to help Adam to converge in certain situations, but since we can't reproduce the issue, it's unclear what those settings are. For more details, see #253.
325 lines
8.9 KiB
C++
325 lines
8.9 KiB
C++
/**
|
|
* @file adam_test.cpp
|
|
* @author Vasanth Kalingeri
|
|
* @author Vivek Pal
|
|
* @author Sourabh Varshney
|
|
* @author Haritha Nair
|
|
* @author Marcus Edel
|
|
* @author Conrad Sanderson
|
|
*
|
|
* ensmallen is free software; you may redistribute it and/or modify it under
|
|
* the terms of the 3-clause BSD license. You should have received a copy of
|
|
* the 3-clause BSD license along with ensmallen. If not, see
|
|
* http://www.opensource.org/licenses/BSD-3-Clause for more information.
|
|
*/
|
|
|
|
#include <ensmallen.hpp>
|
|
#include "catch.hpp"
|
|
#include "test_function_tools.hpp"
|
|
|
|
using namespace ens;
|
|
using namespace ens::test;
|
|
|
|
/**
|
|
* Test the Adam optimizer on the Sphere function.
|
|
*/
|
|
TEST_CASE("AdamSphereFunctionTest", "[AdamTest]")
|
|
{
|
|
Adam optimizer(0.5, 2, 0.7, 0.999, 1e-8, 500000, 1e-3, false);
|
|
FunctionTest<SphereFunction>(optimizer, 0.5, 0.2);
|
|
}
|
|
|
|
/**
|
|
* Test the Adam optimizer on the Sphere function with arma::fmat.
|
|
*/
|
|
TEST_CASE("AdamSphereFunctionTestFMat", "[AdamTest]")
|
|
{
|
|
Adam optimizer(0.5, 2, 0.7, 0.999, 1e-8, 500000, 1e-3, false);
|
|
FunctionTest<SphereFunction, arma::fmat>(optimizer, 0.5, 0.2);
|
|
}
|
|
|
|
#if ARMA_VERSION_MAJOR > 9 ||\
|
|
(ARMA_VERSION_MAJOR == 9 && ARMA_VERSION_MINOR >= 400)
|
|
|
|
/**
|
|
* Test the AMSGrad optimizer on the Sphere function with arma::sp_mat.
|
|
*/
|
|
TEST_CASE("AdamSphereFunctionTestSpMat", "[AdamTest]")
|
|
{
|
|
Adam optimizer(0.5, 2, 0.7, 0.999, 1e-8, 500000, 1e-3, false);
|
|
FunctionTest<SphereFunction, arma::sp_mat>(optimizer, 0.5, 0.2);
|
|
}
|
|
|
|
/**
|
|
* Test the AMSGrad optimizer on the Sphere function with arma::sp_mat but a
|
|
* dense (arma::mat) gradient.
|
|
*/
|
|
TEST_CASE("AdamSphereFunctionTestSpMatDenseGradient", "[AdamTest]")
|
|
{
|
|
SphereFunction f(2);
|
|
Adam optimizer(0.5, 2, 0.7, 0.999, 1e-8, 500000, 1e-3, false);
|
|
|
|
arma::sp_mat coordinates = f.GetInitialPoint<arma::sp_mat>();
|
|
optimizer.Optimize<decltype(f), arma::sp_mat, arma::mat>(f, coordinates);
|
|
|
|
REQUIRE(coordinates(0) == Approx(0.0).margin(0.1));
|
|
REQUIRE(coordinates(1) == Approx(0.0).margin(0.1));
|
|
}
|
|
|
|
#endif
|
|
|
|
/**
|
|
* Test the Adam optimizer on the Wood function.
|
|
*/
|
|
TEST_CASE("AdamStyblinskiTangFunctionTest", "[AdamTest]")
|
|
{
|
|
Adam optimizer(0.5, 2, 0.7, 0.999, 1e-8, 500000, 1e-3, false);
|
|
FunctionTest<StyblinskiTangFunction>(optimizer, 0.5, 0.1);
|
|
}
|
|
|
|
/**
|
|
* Test the Adam optimizer on the McCormick function.
|
|
*/
|
|
TEST_CASE("AdamMcCormickFunctionTest", "[AdamTest]")
|
|
{
|
|
Adam optimizer(0.5, 1, 0.7, 0.999, 1e-8, 500000, 1e-5, false);
|
|
FunctionTest<McCormickFunction>(optimizer, 0.5, 0.1);
|
|
}
|
|
|
|
/**
|
|
* Test the Adam optimizer on the Matyas function.
|
|
*/
|
|
TEST_CASE("AdamMatyasFunctionTest", "[AdamTest]")
|
|
{
|
|
Adam optimizer(0.5, 1, 0.7, 0.999, 1e-8, 500000, 1e-5, false);
|
|
FunctionTest<MatyasFunction>(optimizer, 0.1, 0.01);
|
|
}
|
|
|
|
/**
|
|
* Test the Adam optimizer on the Easom function.
|
|
*/
|
|
TEST_CASE("AdamEasomFunctionTest", "[AdamTest]")
|
|
{
|
|
Adam optimizer(0.2, 1, 0.7, 0.999, 1e-8, 500000, 1e-5, false);
|
|
FunctionTest<EasomFunction>(optimizer, 1.5, 0.01);
|
|
}
|
|
|
|
/**
|
|
* Test the Adam optimizer on the Booth function.
|
|
*/
|
|
TEST_CASE("AdamBoothFunctionTest", "[AdamTest]")
|
|
{
|
|
Adam optimizer(1e-1, 1, 0.7, 0.999, 1e-8, 500000, 1e-9, true);
|
|
FunctionTest<BoothFunction>(optimizer);
|
|
}
|
|
|
|
|
|
/**
|
|
* Test the AMSGrad optimizer on the Sphere function with arma::fmat.
|
|
*/
|
|
TEST_CASE("AMSGradSphereFunctionTestFMat", "[AdamTest]")
|
|
{
|
|
AMSGrad optimizer(1e-3, 1, 0.9, 0.999, 1e-8, 500000, 1e-11, true);
|
|
FunctionTest<SphereFunction, arma::fmat>(optimizer, 0.5, 0.1);
|
|
}
|
|
|
|
#if ARMA_VERSION_MAJOR > 9 || \
|
|
(ARMA_VERSION_MAJOR == 9 && ARMA_VERSION_MINOR >= 400)
|
|
|
|
/**
|
|
* Test the AMSGrad optimizer on the Sphere function with arma::sp_mat.
|
|
*/
|
|
TEST_CASE("AMSGradSphereFunctionTestSpMat", "[AdamTest]")
|
|
{
|
|
AMSGrad optimizer(1e-3, 1, 0.9, 0.999, 1e-8, 500000, 1e-11, true);
|
|
FunctionTest<SphereFunction, arma::sp_mat>(optimizer, 0.5, 0.1);
|
|
}
|
|
|
|
/**
|
|
* Test the AMSGrad optimizer on the Sphere function with arma::sp_mat but a
|
|
* dense (arma::mat) gradient.
|
|
*/
|
|
TEST_CASE("AMSGradSphereFunctionTestSpMatDenseGradient", "[AdamTest]")
|
|
{
|
|
SphereFunction f(2);
|
|
AMSGrad optimizer(1e-3, 1, 0.9, 0.999, 1e-8, 500000, 1e-11, true);
|
|
|
|
arma::sp_mat coordinates = f.GetInitialPoint<arma::sp_mat>();
|
|
optimizer.Optimize<decltype(f), arma::sp_mat, arma::mat>(f, coordinates);
|
|
|
|
REQUIRE(coordinates(0) == Approx(0.0).margin(0.1));
|
|
REQUIRE(coordinates(1) == Approx(0.0).margin(0.1));
|
|
}
|
|
|
|
#endif
|
|
|
|
/**
|
|
* Run Adam on logistic regression and make sure the results are acceptable.
|
|
*/
|
|
TEST_CASE("AdamLogisticRegressionTest", "[AdamTest]")
|
|
{
|
|
Adam adam;
|
|
LogisticRegressionFunctionTest(adam, 0.003, 0.006);
|
|
}
|
|
|
|
/**
|
|
* Run AdaMax on logistic regression and make sure the results are acceptable.
|
|
*/
|
|
TEST_CASE("AdaMaxLogisticRegressionTest", "[AdamTest]")
|
|
{
|
|
AdaMax adamax(1e-3, 1, 0.9, 0.999, 1e-8, 5000000, 1e-9, true);
|
|
LogisticRegressionFunctionTest(adamax, 0.003, 0.006);
|
|
}
|
|
|
|
/**
|
|
* Run AMSGrad on logistic regression and make sure the results are acceptable.
|
|
*/
|
|
TEST_CASE("AMSGradLogisticRegressionTest", "[AdamTest]")
|
|
{
|
|
AMSGrad amsgrad(1e-3, 1, 0.9, 0.999, 1e-8, 500000, 1e-11, true);
|
|
LogisticRegressionFunctionTest(amsgrad, 0.003, 0.006);
|
|
}
|
|
|
|
/**
|
|
* Run Nadam on logistic regression and make sure the results are acceptable.
|
|
*/
|
|
TEST_CASE("NadamLogisticRegressionTest", "[AdamTest]")
|
|
{
|
|
Nadam nadam;
|
|
LogisticRegressionFunctionTest(nadam, 0.003, 0.006);
|
|
}
|
|
|
|
/**
|
|
* Run NadaMax on logistic regression and make sure the results are acceptable.
|
|
*/
|
|
TEST_CASE("NadaMaxLogisticRegressionTest", "[AdamTest]")
|
|
{
|
|
NadaMax nadamax(1e-3, 1, 0.9, 0.999, 1e-8, 5000000, 1e-9, true);
|
|
LogisticRegressionFunctionTest(nadamax, 0.003, 0.006);
|
|
}
|
|
|
|
/**
|
|
* Run OptimisticAdam on logistic regression and make sure the results are
|
|
* acceptable.
|
|
*/
|
|
TEST_CASE("OptimisticAdamLogisticRegressionTest", "[AdamTest]")
|
|
{
|
|
OptimisticAdam optimisticAdam;
|
|
LogisticRegressionFunctionTest(optimisticAdam, 0.003, 0.006);
|
|
}
|
|
|
|
/**
|
|
* Run Padam on logistic regression and make sure the results are acceptable.
|
|
*/
|
|
TEST_CASE("PadamLogisticRegressionTest", "[AdamTest]")
|
|
{
|
|
Padam optimizer;
|
|
LogisticRegressionFunctionTest(optimizer, 0.003, 0.006);
|
|
}
|
|
|
|
/**
|
|
* Run QHAdam on logistic regression and make sure the results are acceptable.
|
|
*/
|
|
TEST_CASE("QHAdamLogisticRegressionTest", "[AdamTest]")
|
|
{
|
|
QHAdam optimizer;
|
|
LogisticRegressionFunctionTest(optimizer, 0.003, 0.006);
|
|
}
|
|
|
|
/**
|
|
* Run QHAdam on logistic regression and make sure the results are acceptable,
|
|
* using arma::fmat.
|
|
*/
|
|
TEST_CASE("QHAdamLogisticRegressionFMatTest", "[AdamTest]")
|
|
{
|
|
QHAdam optimizer;
|
|
LogisticRegressionFunctionTest<arma::fmat>(optimizer, 0.03, 0.06);
|
|
}
|
|
|
|
#if ARMA_VERSION_MAJOR > 9 ||\
|
|
(ARMA_VERSION_MAJOR == 9 && ARMA_VERSION_MINOR >= 400)
|
|
|
|
/**
|
|
* Run QHAdam on logistic regression and make sure the results are acceptable,
|
|
* using arma::sp_mat.
|
|
*/
|
|
TEST_CASE("QHAdamLogisticRegressionSpMatTest", "[AdamTest]")
|
|
{
|
|
QHAdam optimizer;
|
|
LogisticRegressionFunctionTest<arma::sp_mat>(optimizer, 0.003, 0.006);
|
|
}
|
|
|
|
#endif
|
|
|
|
/**
|
|
* Test the Adam optimizer on the Ackley function.
|
|
* This is to test the Ackley function and not Adam.
|
|
* This test will be removed later.
|
|
*/
|
|
TEST_CASE("AdamAckleyFunctionTest", "[AdamTest]")
|
|
{
|
|
Adam optimizer(0.001, 2, 0.7, 0.999, 1e-8, 500000, 1e-7, false);
|
|
FunctionTest<AckleyFunction>(optimizer);
|
|
}
|
|
|
|
/**
|
|
* Test the Adam optimizer on the Beale function.
|
|
* This is to test the Beale function and not Adam.
|
|
* This test will be removed later.
|
|
*/
|
|
TEST_CASE("AdamBealeFunctionTest", "[AdamTest]")
|
|
{
|
|
Adam optimizer(0.001, 2, 0.7, 0.999, 1e-8, 500000, 1e-7, false);
|
|
FunctionTest<BealeFunction>(optimizer, 0.1, 0.01);
|
|
}
|
|
|
|
/**
|
|
* Test the Adam optimizer on the Goldstein-Price function.
|
|
* This is to test the Goldstein-Price function and not Adam.
|
|
* This test will be removed later.
|
|
*/
|
|
TEST_CASE("AdamGoldsteinPriceFunctionTest", "[AdamTest]")
|
|
{
|
|
Adam optimizer(0.0001, 2, 0.7, 0.999, 1e-8, 500000, 1e-9, false);
|
|
FunctionTest<GoldsteinPriceFunction>(optimizer, 0.1, 0.01);
|
|
}
|
|
|
|
/**
|
|
* Test the Adam optimizer on the Levi function.
|
|
* This is to test the Levi function and not Adam.
|
|
* This test will be removed later.
|
|
*/
|
|
TEST_CASE("AdamLevyFunctionTest", "[AdamTest]")
|
|
{
|
|
Adam optimizer(0.001, 2, 0.7, 0.999, 1e-8, 500000, 1e-9, false);
|
|
FunctionTest<LevyFunctionN13>(optimizer, 0.1, 0.01);
|
|
}
|
|
|
|
/**
|
|
* Test the Adam optimizer on the Himmelblau function.
|
|
* This is to test the Himmelblau function and not Adam.
|
|
* This test will be removed later.
|
|
*/
|
|
TEST_CASE("AdamHimmelblauFunctionTest", "[AdamTest]")
|
|
{
|
|
HimmelblauFunction f;
|
|
Adam optimizer(0.001, 2, 0.7, 0.999, 1e-8, 500000, 1e-9, false);
|
|
|
|
arma::mat coordinates = arma::mat("2.9; 1.9");
|
|
optimizer.Optimize(f, coordinates);
|
|
|
|
REQUIRE(coordinates(0) == Approx(3.0).margin(0.05));
|
|
REQUIRE(coordinates(1) == Approx(2.0).margin(0.05));
|
|
}
|
|
|
|
/**
|
|
* Test the Adam optimizer on the Three-hump camel function.
|
|
* This is to test the Three-hump camel function and not Adam.
|
|
* This test will be removed later.
|
|
*/
|
|
TEST_CASE("AdamThreeHumpCamelFunctionTest", "[AdamTest]")
|
|
{
|
|
Adam optimizer(0.001, 2, 0.7, 0.999, 1e-8, 500000, 1e-9, false);
|
|
FunctionTest<ThreeHumpCamelFunction>(optimizer, 0.1, 0.01);
|
|
}
|