From 6f3650ce4a3dd23209e346d1f1fdb791bdfd7887 Mon Sep 17 00:00:00 2001 From: Marcus Edel Date: Thu, 15 Nov 2018 21:07:55 +0100 Subject: [PATCH] Remove optimizer test cases. --- src/mlpack/tests/CMakeLists.txt | 33 +- src/mlpack/tests/ada_delta_test.cpp | 108 --- src/mlpack/tests/ada_grad_test.cpp | 103 --- src/mlpack/tests/adam_test.cpp | 587 ---------------- src/mlpack/tests/aug_lagrangian_test.cpp | 69 -- src/mlpack/tests/bigbatch_sgd_test.cpp | 135 ---- src/mlpack/tests/cmaes_test.cpp | 165 ----- src/mlpack/tests/cne_test.cpp | 222 ------ src/mlpack/tests/frankwolfe_test.cpp | 193 ------ src/mlpack/tests/gradient_clipping_test.cpp | 72 -- src/mlpack/tests/gradient_descent_test.cpp | 58 -- src/mlpack/tests/iqn_test.cpp | 88 --- src/mlpack/tests/katyusha_test.cpp | 77 --- src/mlpack/tests/lbfgs_test.cpp | 140 ---- src/mlpack/tests/line_search_test.cpp | 49 -- src/mlpack/tests/lrsdp_test.cpp | 317 --------- src/mlpack/tests/momentum_sgd_test.cpp | 79 --- .../tests/nesterov_momentum_sgd_test.cpp | 72 -- src/mlpack/tests/parallel_sgd_test.cpp | 127 ---- src/mlpack/tests/proximal_test.cpp | 91 --- src/mlpack/tests/rmsprop_test.cpp | 106 --- src/mlpack/tests/sa_test.cpp | 113 --- src/mlpack/tests/sarah_test.cpp | 78 --- src/mlpack/tests/scd_test.cpp | 218 ------ src/mlpack/tests/sdp_primal_dual_test.cpp | 648 ------------------ src/mlpack/tests/sgd_test.cpp | 61 -- src/mlpack/tests/sgdr_test.cpp | 129 ---- src/mlpack/tests/smorms3_test.cpp | 106 --- src/mlpack/tests/snapshot_ensembles.cpp | 133 ---- src/mlpack/tests/spalera_sgd_test.cpp | 89 --- src/mlpack/tests/svrg_test.cpp | 77 --- 31 files changed, 1 insertion(+), 4542 deletions(-) delete mode 100644 src/mlpack/tests/ada_delta_test.cpp delete mode 100644 src/mlpack/tests/ada_grad_test.cpp delete mode 100644 src/mlpack/tests/adam_test.cpp delete mode 100644 src/mlpack/tests/aug_lagrangian_test.cpp delete mode 100644 src/mlpack/tests/bigbatch_sgd_test.cpp delete mode 100644 src/mlpack/tests/cmaes_test.cpp delete mode 100644 src/mlpack/tests/cne_test.cpp delete mode 100644 src/mlpack/tests/frankwolfe_test.cpp delete mode 100644 src/mlpack/tests/gradient_clipping_test.cpp delete mode 100644 src/mlpack/tests/gradient_descent_test.cpp delete mode 100644 src/mlpack/tests/iqn_test.cpp delete mode 100644 src/mlpack/tests/katyusha_test.cpp delete mode 100644 src/mlpack/tests/lbfgs_test.cpp delete mode 100644 src/mlpack/tests/line_search_test.cpp delete mode 100644 src/mlpack/tests/lrsdp_test.cpp delete mode 100644 src/mlpack/tests/momentum_sgd_test.cpp delete mode 100644 src/mlpack/tests/nesterov_momentum_sgd_test.cpp delete mode 100644 src/mlpack/tests/parallel_sgd_test.cpp delete mode 100644 src/mlpack/tests/proximal_test.cpp delete mode 100644 src/mlpack/tests/rmsprop_test.cpp delete mode 100644 src/mlpack/tests/sa_test.cpp delete mode 100644 src/mlpack/tests/sarah_test.cpp delete mode 100644 src/mlpack/tests/scd_test.cpp delete mode 100644 src/mlpack/tests/sdp_primal_dual_test.cpp delete mode 100644 src/mlpack/tests/sgd_test.cpp delete mode 100644 src/mlpack/tests/sgdr_test.cpp delete mode 100644 src/mlpack/tests/smorms3_test.cpp delete mode 100644 src/mlpack/tests/snapshot_ensembles.cpp delete mode 100644 src/mlpack/tests/spalera_sgd_test.cpp delete mode 100644 src/mlpack/tests/svrg_test.cpp diff --git a/src/mlpack/tests/CMakeLists.txt b/src/mlpack/tests/CMakeLists.txt index 255404e5f7..9e65198b0f 100644 --- a/src/mlpack/tests/CMakeLists.txt +++ b/src/mlpack/tests/CMakeLists.txt @@ -1,10 +1,7 @@ # mlpack test executable. add_executable(mlpack_test activation_functions_test.cpp - ada_delta_test.cpp - ada_grad_test.cpp adaboost_test.cpp - adam_test.cpp akfn_test.cpp aknn_test.cpp ann_dist_test.cpp @@ -13,17 +10,13 @@ add_executable(mlpack_test arma_extend_test.cpp armadillo_svd_test.cpp async_learning_test.cpp - aug_lagrangian_test.cpp augmented_rnns_tasks_test.cpp bias_svd_test.cpp - bigbatch_sgd_test.cpp binarize_test.cpp block_krylov_svd_test.cpp cf_test.cpp cli_binding_test.cpp cli_test.cpp - cmaes_test.cpp - cne_test.cpp convolution_test.cpp convolutional_network_test.cpp cosine_tree_test.cpp @@ -38,20 +31,14 @@ add_executable(mlpack_test emst_test.cpp fastmks_test.cpp feedforward_network_test.cpp - frankwolfe_test.cpp - function_test.cpp gan_test.cpp gmm_test.cpp - gradient_clipping_test.cpp - gradient_descent_test.cpp hmm_test.cpp hoeffding_tree_test.cpp hpt_test.cpp hyperplane_test.cpp imputation_test.cpp init_rules_test.cpp - iqn_test.cpp - katyusha_test.cpp kernel_pca_test.cpp kernel_test.cpp kernel_traits_test.cpp @@ -61,9 +48,7 @@ add_executable(mlpack_test krann_search_test.cpp ksinit_test.cpp lars_test.cpp - lbfgs_test.cpp lin_alg_test.cpp - line_search_test.cpp linear_regression_test.cpp lmnn_test.cpp load_save_test.cpp @@ -71,7 +56,6 @@ add_executable(mlpack_test log_test.cpp logistic_regression_test.cpp loss_functions_test.cpp - lrsdp_test.cpp lsh_test.cpp math_test.cpp matrix_completion_test.cpp @@ -80,18 +64,14 @@ add_executable(mlpack_test metric_test.cpp mlpack_test.cpp mock_categorical_data.hpp - momentum_sgd_test.cpp nbc_test.cpp nca_test.cpp - nesterov_momentum_sgd_test.cpp nmf_test.cpp nystroem_method_test.cpp octree_test.cpp - parallel_sgd_test.cpp pca_test.cpp perceptron_test.cpp prefixedoutstream_test.cpp - proximal_test.cpp python_binding_test.cpp q_learning_test.cpp qdafn_test.cpp @@ -107,22 +87,12 @@ add_executable(mlpack_test regularized_svd_test.cpp reward_clipping_test.cpp rl_components_test.cpp - rmsprop_test.cpp - sa_test.cpp - sarah_test.cpp - scd_test.cpp - sdp_primal_dual_test.cpp serialization.cpp serialization.hpp serialization_test.cpp sfinae_test.cpp - sgd_test.cpp - sgdr_test.cpp - smorms3_test.cpp - snapshot_ensembles.cpp softmax_regression_test.cpp sort_policy_test.cpp - spalera_sgd_test.cpp sparse_autoencoder_test.cpp sparse_coding_test.cpp spill_tree_test.cpp @@ -130,7 +100,6 @@ add_executable(mlpack_test svd_batch_test.cpp svd_incremental_test.cpp svdplusplus_test.cpp - svrg_test.cpp termination_policy_test.cpp test_function_tools.hpp test_tools.hpp @@ -205,7 +174,7 @@ add_custom_command(TARGET mlpack_test # The list of long running parallel tests set(parallel_tests "AsyncLearningTest" - "SdpPrimalDualTest;SVDIncrementalTest;SVDBatchTest;" + "SVDIncrementalTest;SVDBatchTest;" "LocalCoordinateCodingTest;FeedForwardNetworkTest;SparseAutoencoderTest;" "GMMTest;CFTest;ConvolutionalNetworkTest;HMMTest;LARSTest;" "LogisticRegressionTest") diff --git a/src/mlpack/tests/ada_delta_test.cpp b/src/mlpack/tests/ada_delta_test.cpp deleted file mode 100644 index 873f162278..0000000000 --- a/src/mlpack/tests/ada_delta_test.cpp +++ /dev/null @@ -1,108 +0,0 @@ -/** - * @file ada_delta_test.cpp - * @author Marcus Edel - * @author Vasanth Kalingeri - * @author Abhinav Moudgil - * - * Tests the AdaDelta optimizer - * - * mlpack is free software; you may redistribute it and/or modify it under the - * terms of the 3-clause BSD license. You should have received a copy of the - * 3-clause BSD license along with mlpack. If not, see - * http://www.opensource.org/licenses/BSD-3-Clause for more information. - */ -#include - -#include -#include -#include - -#include -#include "test_tools.hpp" - -using namespace arma; -using namespace mlpack::optimization; -using namespace mlpack::optimization::test; - -using namespace mlpack::distribution; -using namespace mlpack::regression; - -using namespace mlpack; - -BOOST_AUTO_TEST_SUITE(AdaDeltaTest); - -/** - * Tests the Adadelta optimizer using a simple test function. - */ -BOOST_AUTO_TEST_CASE(SimpleAdaDeltaTestFunction) -{ - SGDTestFunction f; - AdaDelta optimizer(1.0, 1, 0.99, 1e-8, 5000000, 1e-9, true); - - arma::mat coordinates = f.GetInitialPoint(); - optimizer.Optimize(f, coordinates); - - BOOST_REQUIRE_SMALL(coordinates[0], 0.003); - BOOST_REQUIRE_SMALL(coordinates[1], 0.003); - BOOST_REQUIRE_SMALL(coordinates[2], 0.003); -} - -/** - * Run AdaDelta on logistic regression and make sure the results are acceptable. - */ -BOOST_AUTO_TEST_CASE(LogisticRegressionTest) -{ - // Generate a two-Gaussian dataset. - GaussianDistribution g1(arma::vec("1.0 1.0 1.0"), arma::eye(3, 3)); - GaussianDistribution g2(arma::vec("9.0 9.0 9.0"), arma::eye(3, 3)); - - arma::mat data(3, 1000); - arma::Row responses(1000); - for (size_t i = 0; i < 500; ++i) - { - data.col(i) = g1.Random(); - responses[i] = 0; - } - for (size_t i = 500; i < 1000; ++i) - { - data.col(i) = g2.Random(); - responses[i] = 1; - } - - // Shuffle the dataset. - arma::uvec indices = arma::shuffle(arma::linspace(0, - data.n_cols - 1, data.n_cols)); - arma::mat shuffledData(3, 1000); - arma::Row shuffledResponses(1000); - for (size_t i = 0; i < data.n_cols; ++i) - { - shuffledData.col(i) = data.col(indices[i]); - shuffledResponses[i] = responses[indices[i]]; - } - - // Create a test set. - arma::mat testData(3, 1000); - arma::Row testResponses(1000); - for (size_t i = 0; i < 500; ++i) - { - testData.col(i) = g1.Random(); - testResponses[i] = 0; - } - for (size_t i = 500; i < 1000; ++i) - { - testData.col(i) = g2.Random(); - testResponses[i] = 1; - } - - AdaDelta adaDelta; - LogisticRegression<> lr(shuffledData, shuffledResponses, adaDelta, 0.5); - - // Ensure that the error is close to zero. - const double acc = lr.ComputeAccuracy(data, responses); - BOOST_REQUIRE_CLOSE(acc, 100.0, 0.3); // 0.3% error tolerance. - - const double testAcc = lr.ComputeAccuracy(testData, testResponses); - BOOST_REQUIRE_CLOSE(testAcc, 100.0, 0.6); // 0.6% error tolerance. -} - -BOOST_AUTO_TEST_SUITE_END(); diff --git a/src/mlpack/tests/ada_grad_test.cpp b/src/mlpack/tests/ada_grad_test.cpp deleted file mode 100644 index 484b6a7306..0000000000 --- a/src/mlpack/tests/ada_grad_test.cpp +++ /dev/null @@ -1,103 +0,0 @@ -/** - * @file ada_grad_test.cpp - * @author Abhinav Moudgil - * - * Test file for AdaGrad (stochastic gradient descent with AdaGrad updates). - * - * mlpack is free software; you may redistribute it and/or modify it under the - * terms of the 3-clause BSD license. You should have received a copy of the - * 3-clause BSD license along with mlpack. If not, see - * http://www.opensource.org/licenses/BSD-3-Clause for more information. - */ -#include -#include -#include -#include - -#include -#include "test_tools.hpp" - -using namespace std; -using namespace arma; -using namespace mlpack; -using namespace mlpack::optimization; -using namespace mlpack::optimization::test; -using namespace mlpack::distribution; -using namespace mlpack::regression; - -BOOST_AUTO_TEST_SUITE(AdaGradTest); - -/** - * Tests the Adagrad optimizer using a simple test function. - */ -BOOST_AUTO_TEST_CASE(SimpleAdaGradTestFunction) -{ - SGDTestFunction f; - AdaGrad optimizer(0.99, 1, 1e-8, 5000000, 1e-9, true); - - arma::mat coordinates = f.GetInitialPoint(); - optimizer.Optimize(f, coordinates); - - BOOST_REQUIRE_SMALL(coordinates[0], 0.003); - BOOST_REQUIRE_SMALL(coordinates[1], 0.003); - BOOST_REQUIRE_SMALL(coordinates[2], 0.003); -} - -/** - * Run AdaGrad on logistic regression and make sure the results are acceptable. - */ -BOOST_AUTO_TEST_CASE(AdaGradLogisticRegressionTest) -{ - // Generate a two-Gaussian dataset. - GaussianDistribution g1(arma::vec("1.0 1.0 1.0"), arma::eye(3, 3)); - GaussianDistribution g2(arma::vec("9.0 9.0 9.0"), arma::eye(3, 3)); - - arma::mat data(3, 1000); - arma::Row responses(1000); - for (size_t i = 0; i < 500; ++i) - { - data.col(i) = g1.Random(); - responses[i] = 0; - } - for (size_t i = 500; i < 1000; ++i) - { - data.col(i) = g2.Random(); - responses[i] = 1; - } - - // Shuffle the dataset. - arma::uvec indices = arma::shuffle(arma::linspace(0, - data.n_cols - 1, data.n_cols)); - arma::mat shuffledData(3, 1000); - arma::Row shuffledResponses(1000); - for (size_t i = 0; i < data.n_cols; ++i) - { - shuffledData.col(i) = data.col(indices[i]); - shuffledResponses[i] = responses[indices[i]]; - } - - // Create a test set. - arma::mat testData(3, 1000); - arma::Row testResponses(1000); - for (size_t i = 0; i < 500; ++i) - { - testData.col(i) = g1.Random(); - testResponses[i] = 0; - } - for (size_t i = 500; i < 1000; ++i) - { - testData.col(i) = g2.Random(); - testResponses[i] = 1; - } - - AdaGrad adagrad(0.99, 32, 1e-8, 5000000, 1e-9, true); - LogisticRegression<> lr(shuffledData, shuffledResponses, adagrad, 0.5); - - // Ensure that the error is close to zero. - const double acc = lr.ComputeAccuracy(data, responses); - BOOST_REQUIRE_CLOSE(acc, 100.0, 0.3); // 0.3% error tolerance. - - const double testAcc = lr.ComputeAccuracy(testData, testResponses); - BOOST_REQUIRE_CLOSE(testAcc, 100.0, 0.6); // 0.6% error tolerance. -} -BOOST_AUTO_TEST_SUITE_END(); diff --git a/src/mlpack/tests/adam_test.cpp b/src/mlpack/tests/adam_test.cpp deleted file mode 100644 index dff4844db3..0000000000 --- a/src/mlpack/tests/adam_test.cpp +++ /dev/null @@ -1,587 +0,0 @@ -/** - * @file adam_test.cpp - * @author Vasanth Kalingeri - * @author Vivek Pal - * @author Sourabh Varshney - * @author Haritha Nair - * - * Tests the Adam, AdaMax, AMSGrad, Nadam and NadaMax optimizer. - * - * mlpack is free software; you may redistribute it and/or modify it under the - * terms of the 3-clause BSD license. You should have received a copy of the - * 3-clause BSD license along with mlpack. If not, see - * http://www.opensource.org/licenses/BSD-3-Clause for more information. - */ - -#include - -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include "test_tools.hpp" - -using namespace arma; -using namespace mlpack::optimization; -using namespace mlpack::optimization::test; - -using namespace mlpack::distribution; -using namespace mlpack::regression; - -using namespace mlpack; - -BOOST_AUTO_TEST_SUITE(AdamTest); - - -/** - * Test the Adam optimizer on the Sphere function. - */ -BOOST_AUTO_TEST_CASE(AdamSphereFunctionTest) -{ - SphereFunction f(2); - Adam optimizer(0.5, 2, 0.7, 0.999, 1e-8, 500000, 1e-3, false); - - arma::mat coordinates = f.GetInitialPoint(); - optimizer.Optimize(f, coordinates); - - BOOST_REQUIRE_SMALL(coordinates[0], 0.1); - BOOST_REQUIRE_SMALL(coordinates[1], 0.1); -} - -/** - * Test the Adam optimizer on the Wood function. - */ -BOOST_AUTO_TEST_CASE(AdamStyblinskiTangFunctionTest) -{ - StyblinskiTangFunction f(2); - Adam optimizer(0.5, 2, 0.7, 0.999, 1e-8, 500000, 1e-3, false); - - arma::mat coordinates = f.GetInitialPoint(); - optimizer.Optimize(f, coordinates); - - BOOST_REQUIRE_CLOSE(coordinates[0], -2.9, 1.0); // 1% error tolerance. - BOOST_REQUIRE_CLOSE(coordinates[1], -2.9, 1.0); // 1% error tolerance. -} - -/** - * Test the Adam optimizer on the McCormick function. - */ -BOOST_AUTO_TEST_CASE(AdamMcCormickFunctionTest) -{ - McCormickFunction f; - Adam optimizer(0.5, 1, 0.7, 0.999, 1e-8, 500000, 1e-5, false); - - arma::mat coordinates = f.GetInitialPoint(); - optimizer.Optimize(f, coordinates); - - BOOST_REQUIRE_CLOSE(coordinates[0], -0.547, 3.0); // 3% error tolerance. - BOOST_REQUIRE_CLOSE(coordinates[1], -1.547, 3.0); // 3% error tolerance. -} - -/** - * Test the Adam optimizer on the Matyas function. - */ -BOOST_AUTO_TEST_CASE(AdamMatyasFunctionTest) -{ - MatyasFunction f; - Adam optimizer(0.5, 1, 0.7, 0.999, 1e-8, 500000, 1e-5, false); - - arma::mat coordinates = f.GetInitialPoint(); - optimizer.Optimize(f, coordinates); - - // 3% error tolerance. - BOOST_REQUIRE_CLOSE(std::trunc(100.0 * coordinates[0]) / 100.0, 0.0, 3.0); - BOOST_REQUIRE_CLOSE(std::trunc(100.0 * coordinates[1]) / 100.0, 0.0, 3.0); -} - -/** - * Test the Adam optimizer on the Easom function. - */ -BOOST_AUTO_TEST_CASE(AdamEasomFunctionTest) -{ - EasomFunction f; - Adam optimizer(0.2, 1, 0.7, 0.999, 1e-8, 500000, 1e-5, false); - - arma::mat coordinates = arma::mat("2.9; 2.9"); - optimizer.Optimize(f, coordinates); - - // 5% error tolerance. - BOOST_REQUIRE_CLOSE(std::trunc(100.0 * coordinates[0]) / 100.0, 3.14, 3.0); - BOOST_REQUIRE_CLOSE(std::trunc(100.0 * coordinates[1]) / 100.0, 3.14, 3.0); -} - -/** - * Test the Adam optimizer on the Booth function. - */ -BOOST_AUTO_TEST_CASE(AdamBoothFunctionTest) -{ - BoothFunction f; - Adam optimizer(1e-1, 1, 0.7, 0.999, 1e-8, 500000, 1e-9, true); - - arma::mat coordinates = f.GetInitialPoint(); - optimizer.Optimize(f, coordinates); - - BOOST_REQUIRE_CLOSE(coordinates[0], 1.0, 0.2); - BOOST_REQUIRE_CLOSE(coordinates[1], 3.0, 0.2); -} - -/** - * Tests the Adam optimizer using a simple test function. - */ -BOOST_AUTO_TEST_CASE(SimpleAdamTestFunction) -{ - SGDTestFunction f; - Adam optimizer(1e-3, 1, 0.9, 0.999, 1e-8, 500000, 1e-9, true); - - arma::mat coordinates = f.GetInitialPoint(); - optimizer.Optimize(f, coordinates); - - BOOST_REQUIRE_SMALL(coordinates[0], 0.1); - BOOST_REQUIRE_SMALL(coordinates[1], 0.1); - BOOST_REQUIRE_SMALL(coordinates[2], 0.1); -} - -/** - * Tests the AdaMax optimizer using a simple test function. - */ -BOOST_AUTO_TEST_CASE(SimpleAdaMaxTestFunction) -{ - SGDTestFunction f; - AdaMax optimizer(2e-3, 1, 0.9, 0.999, 1e-8, 500000, 1e-9, true); - - arma::mat coordinates = f.GetInitialPoint(); - optimizer.Optimize(f, coordinates); - - BOOST_REQUIRE_SMALL(coordinates[0], 0.1); - BOOST_REQUIRE_SMALL(coordinates[1], 0.1); - BOOST_REQUIRE_SMALL(coordinates[2], 0.1); -} - -/** - * Tests the AMSGrad optimizer using a simple test function. - */ -BOOST_AUTO_TEST_CASE(SimpleAMSGradTestFunction) -{ - SGDTestFunction f; - AMSGrad optimizer(1e-3, 1, 0.9, 0.999, 1e-8, 500000, 1e-11, true); - - arma::mat coordinates = f.GetInitialPoint(); - optimizer.Optimize(f, coordinates); - - BOOST_REQUIRE_SMALL(coordinates[0], 0.1); - BOOST_REQUIRE_SMALL(coordinates[1], 0.1); - BOOST_REQUIRE_SMALL(coordinates[2], 0.1); -} - -/** - * Run Adam on logistic regression and make sure the results are acceptable. - */ -BOOST_AUTO_TEST_CASE(AdamLogisticRegressionTest) -{ - // Generate a two-Gaussian dataset. - GaussianDistribution g1(arma::vec("1.0 1.0 1.0"), arma::eye(3, 3)); - GaussianDistribution g2(arma::vec("9.0 9.0 9.0"), arma::eye(3, 3)); - - arma::mat data(3, 1000); - arma::Row responses(1000); - for (size_t i = 0; i < 500; ++i) - { - data.col(i) = g1.Random(); - responses[i] = 0; - } - for (size_t i = 500; i < 1000; ++i) - { - data.col(i) = g2.Random(); - responses[i] = 1; - } - - // Shuffle the dataset. - arma::uvec indices = arma::shuffle(arma::linspace(0, - data.n_cols - 1, data.n_cols)); - arma::mat shuffledData(3, 1000); - arma::Row shuffledResponses(1000); - for (size_t i = 0; i < data.n_cols; ++i) - { - shuffledData.col(i) = data.col(indices[i]); - shuffledResponses[i] = responses[indices[i]]; - } - - // Create a test set. - arma::mat testData(3, 1000); - arma::Row testResponses(1000); - for (size_t i = 0; i < 500; ++i) - { - testData.col(i) = g1.Random(); - testResponses[i] = 0; - } - for (size_t i = 500; i < 1000; ++i) - { - testData.col(i) = g2.Random(); - testResponses[i] = 1; - } - - Adam adam; - LogisticRegression<> lr(shuffledData, shuffledResponses, adam, 0.5); - - // Ensure that the error is close to zero. - const double acc = lr.ComputeAccuracy(data, responses); - BOOST_REQUIRE_CLOSE(acc, 100.0, 0.3); // 0.3% error tolerance. - - const double testAcc = lr.ComputeAccuracy(testData, testResponses); - BOOST_REQUIRE_CLOSE(testAcc, 100.0, 0.6); // 0.6% error tolerance. -} - -/** - * Run AdaMax on logistic regression and make sure the results are acceptable. - */ -BOOST_AUTO_TEST_CASE(AdaMaxLogisticRegressionTest) -{ - // Generate a two-Gaussian dataset. - GaussianDistribution g1(arma::vec("1.0 1.0 1.0"), arma::eye(3, 3)); - GaussianDistribution g2(arma::vec("9.0 9.0 9.0"), arma::eye(3, 3)); - - arma::mat data(3, 1000); - arma::Row responses(1000); - for (size_t i = 0; i < 500; ++i) - { - data.col(i) = g1.Random(); - responses[i] = 0; - } - for (size_t i = 500; i < 1000; ++i) - { - data.col(i) = g2.Random(); - responses[i] = 1; - } - - // Shuffle the dataset. - arma::uvec indices = arma::shuffle(arma::linspace(0, - data.n_cols - 1, data.n_cols)); - arma::mat shuffledData(3, 1000); - arma::Row shuffledResponses(1000); - for (size_t i = 0; i < data.n_cols; ++i) - { - shuffledData.col(i) = data.col(indices[i]); - shuffledResponses[i] = responses[indices[i]]; - } - - // Create a test set. - arma::mat testData(3, 1000); - arma::Row testResponses(1000); - for (size_t i = 0; i < 500; ++i) - { - testData.col(i) = g1.Random(); - testResponses[i] = 0; - } - for (size_t i = 500; i < 1000; ++i) - { - testData.col(i) = g2.Random(); - testResponses[i] = 1; - } - - AdaMax adamax(1e-3, 1, 0.9, 0.999, 1e-8, 5000000, 1e-9, true); - LogisticRegression<> lr(shuffledData, shuffledResponses, adamax, 0.5); - - // Ensure that the error is close to zero. - const double acc = lr.ComputeAccuracy(data, responses); - BOOST_REQUIRE_CLOSE(acc, 100.0, 0.3); // 0.3% error tolerance. - - const double testAcc = lr.ComputeAccuracy(testData, testResponses); - BOOST_REQUIRE_CLOSE(testAcc, 100.0, 0.6); // 0.6% error tolerance. -} - -/** - * Run AMSGrad on logistic regression and make sure the results are acceptable. - */ -BOOST_AUTO_TEST_CASE(AMSGradLogisticRegressionTest) -{ - // Generate a two-Gaussian dataset. - GaussianDistribution g1(arma::vec("1.0 1.0 1.0"), arma::eye(3, 3)); - GaussianDistribution g2(arma::vec("9.0 9.0 9.0"), arma::eye(3, 3)); - - arma::mat data(3, 1000); - arma::Row responses(1000); - for (size_t i = 0; i < 500; ++i) - { - data.col(i) = g1.Random(); - responses[i] = 0; - } - for (size_t i = 500; i < 1000; ++i) - { - data.col(i) = g2.Random(); - responses[i] = 1; - } - - // Shuffle the dataset. - arma::uvec indices = arma::shuffle(arma::linspace(0, - data.n_cols - 1, data.n_cols)); - arma::mat shuffledData(3, 1000); - arma::Row shuffledResponses(1000); - for (size_t i = 0; i < data.n_cols; ++i) - { - shuffledData.col(i) = data.col(indices[i]); - shuffledResponses[i] = responses[indices[i]]; - } - - // Create a test set. - arma::mat testData(3, 1000); - arma::Row testResponses(1000); - for (size_t i = 0; i < 500; ++i) - { - testData.col(i) = g1.Random(); - testResponses[i] = 0; - } - for (size_t i = 500; i < 1000; ++i) - { - testData.col(i) = g2.Random(); - testResponses[i] = 1; - } - - AMSGrad amsgrad(1e-3, 1, 0.9, 0.999, 1e-8, 500000, 1e-11, true); - LogisticRegression<> lr(shuffledData, shuffledResponses, amsgrad, 0.5); - - // Ensure that the error is close to zero. - const double acc = lr.ComputeAccuracy(data, responses); - BOOST_REQUIRE_CLOSE(acc, 100.0, 0.3); // 0.3% error tolerance. - - const double testAcc = lr.ComputeAccuracy(testData, testResponses); - BOOST_REQUIRE_CLOSE(testAcc, 100.0, 0.6); // 0.6% error tolerance. -} - -/** - * Tests the Nadam optimizer using a simple test function. - */ -BOOST_AUTO_TEST_CASE(SimpleNadamTestFunction) -{ - SGDTestFunction f; - Nadam optimizer(1e-3, 1, 0.9, 0.99, 1e-8, 500000, 1e-9, true); - - arma::mat coordinates = f.GetInitialPoint(); - optimizer.Optimize(f, coordinates); - - BOOST_REQUIRE_SMALL(coordinates[0], 0.1); - BOOST_REQUIRE_SMALL(coordinates[1], 0.1); - BOOST_REQUIRE_SMALL(coordinates[2], 0.1); -} - -/** - * Run Nadam on logistic regression and make sure the results are acceptable. - */ -BOOST_AUTO_TEST_CASE(NadamLogisticRegressionTest) -{ - // Generate a two-Gaussian dataset. - GaussianDistribution g1(arma::vec("1.0 1.0 1.0"), - arma::eye(3, 3)); - GaussianDistribution g2(arma::vec("9.0 9.0 9.0"), - arma::eye(3, 3)); - - arma::mat data(3, 1000); - arma::Row responses(1000); - for (size_t i = 0; i < 500; ++i) - { - data.col(i) = g1.Random(); - responses[i] = 0; - } - for (size_t i = 500; i < 1000; ++i) - { - data.col(i) = g2.Random(); - responses[i] = 1; - } - - // Shuffle the dataset. - arma::uvec indices = arma::shuffle(arma::linspace(0, - data.n_cols - 1, data.n_cols)); - arma::mat shuffledData(3, 1000); - arma::Row shuffledResponses(1000); - for (size_t i = 0; i < data.n_cols; ++i) - { - shuffledData.col(i) = data.col(indices[i]); - shuffledResponses[i] = responses[indices[i]]; - } - - // Create a test set. - arma::mat testData(3, 1000); - arma::Row testResponses(1000); - for (size_t i = 0; i < 500; ++i) - { - testData.col(i) = g1.Random(); - testResponses[i] = 0; - } - for (size_t i = 500; i < 1000; ++i) - { - testData.col(i) = g2.Random(); - testResponses[i] = 1; - } - - Nadam nadam; - LogisticRegression<> lr(shuffledData, shuffledResponses, nadam, 0.5); - - // Ensure that the error is close to zero. - const double acc = lr.ComputeAccuracy(data, responses); - BOOST_REQUIRE_CLOSE(acc, 100.0, 0.3); // 0.3% error tolerance. - - const double testAcc = lr.ComputeAccuracy(testData, testResponses); - BOOST_REQUIRE_CLOSE(testAcc, 100.0, 0.6); // 0.6% error tolerance. -} - -/** - * Tests the NadaMax optimizer using a simple test function. - */ -BOOST_AUTO_TEST_CASE(SimpleNadaMaxTestFunction) -{ - SGDTestFunction f; - NadaMax optimizer(1e-3, 1, 0.9, 0.99, 1e-8, 500000, 1e-9, true); - - arma::mat coordinates = f.GetInitialPoint(); - optimizer.Optimize(f, coordinates); - - BOOST_REQUIRE_SMALL(coordinates[0], 0.1); - BOOST_REQUIRE_SMALL(coordinates[1], 0.1); - BOOST_REQUIRE_SMALL(coordinates[2], 0.1); -} - -/** - * Run NadaMax on logistic regression and make sure the results are acceptable. - */ -BOOST_AUTO_TEST_CASE(NadaMaxLogisticRegressionTest) -{ - // Generate a two-Gaussian dataset. - GaussianDistribution g1(arma::vec("1.0 1.0 1.0"), - arma::eye(3, 3)); - GaussianDistribution g2(arma::vec("9.0 9.0 9.0"), - arma::eye(3, 3)); - - arma::mat data(3, 1000); - arma::Row responses(1000); - for (size_t i = 0; i < 500; ++i) - { - data.col(i) = g1.Random(); - responses[i] = 0; - } - for (size_t i = 500; i < 1000; ++i) - { - data.col(i) = g2.Random(); - responses[i] = 1; - } - - // Shuffle the dataset. - arma::uvec indices = arma::shuffle(arma::linspace(0, - data.n_cols - 1, data.n_cols)); - arma::mat shuffledData(3, 1000); - arma::Row shuffledResponses(1000); - for (size_t i = 0; i < data.n_cols; ++i) - { - shuffledData.col(i) = data.col(indices[i]); - shuffledResponses[i] = responses[indices[i]]; - } - - // Create a test set. - arma::mat testData(3, 1000); - arma::Row testResponses(1000); - for (size_t i = 0; i < 500; ++i) - { - testData.col(i) = g1.Random(); - testResponses[i] = 0; - } - for (size_t i = 500; i < 1000; ++i) - { - testData.col(i) = g2.Random(); - testResponses[i] = 1; - } - - NadaMax nadamax; - LogisticRegression<> lr(shuffledData, shuffledResponses, nadamax, 0.5); - - // Ensure that the error is close to zero. - const double acc = lr.ComputeAccuracy(data, responses); - BOOST_REQUIRE_CLOSE(acc, 100.0, 0.3); // 0.3% error tolerance. - - const double testAcc = lr.ComputeAccuracy(testData, testResponses); - BOOST_REQUIRE_CLOSE(testAcc, 100.0, 0.6); // 0.6% error tolerance. -} - -/** - * Tests the OptimisticAdam optimizer using a simple test function. - */ -BOOST_AUTO_TEST_CASE(SimpleOptimisticAdamTestFunction) -{ - SGDTestFunction f; - OptimisticAdam optimizer(1e-2, 1, 0.9, 0.99, 1e-8); - - arma::mat coordinates = f.GetInitialPoint(); - optimizer.Optimize(f, coordinates); - - BOOST_REQUIRE_SMALL(coordinates[0], 0.1); - BOOST_REQUIRE_SMALL(coordinates[1], 0.1); - BOOST_REQUIRE_SMALL(coordinates[2], 0.1); -} - -/** - * Run OptimisticAdam on logistic regression and make sure the results are acceptable. - */ -BOOST_AUTO_TEST_CASE(OptimisticAdamLogisticRegressionTest) -{ - // Generate a two-Gaussian dataset. - GaussianDistribution g1(arma::vec("1.0 1.0 1.0"), - arma::eye(3, 3)); - GaussianDistribution g2(arma::vec("9.0 9.0 9.0"), - arma::eye(3, 3)); - - arma::mat data(3, 1000); - arma::Row responses(1000); - for (size_t i = 0; i < 500; ++i) - { - data.col(i) = g1.Random(); - responses[i] = 0; - } - for (size_t i = 500; i < 1000; ++i) - { - data.col(i) = g2.Random(); - responses[i] = 1; - } - - // Shuffle the dataset. - arma::uvec indices = arma::shuffle(arma::linspace(0, - data.n_cols - 1, data.n_cols)); - arma::mat shuffledData(3, 1000); - arma::Row shuffledResponses(1000); - for (size_t i = 0; i < data.n_cols; ++i) - { - shuffledData.col(i) = data.col(indices[i]); - shuffledResponses[i] = responses[indices[i]]; - } - - // Create a test set. - arma::mat testData(3, 1000); - arma::Row testResponses(1000); - for (size_t i = 0; i < 500; ++i) - { - testData.col(i) = g1.Random(); - testResponses[i] = 0; - } - for (size_t i = 500; i < 1000; ++i) - { - testData.col(i) = g2.Random(); - testResponses[i] = 1; - } - - OptimisticAdam optimisticAdam; - LogisticRegression<> lr(shuffledData, shuffledResponses, optimisticAdam, 0.5); - - // Ensure that the error is close to zero. - const double acc = lr.ComputeAccuracy(data, responses); - BOOST_REQUIRE_CLOSE(acc, 100.0, 0.3); // 0.3% error tolerance. - - const double testAcc = lr.ComputeAccuracy(testData, testResponses); - BOOST_REQUIRE_CLOSE(testAcc, 100.0, 0.6); // 0.6% error tolerance. -} - -BOOST_AUTO_TEST_SUITE_END(); diff --git a/src/mlpack/tests/aug_lagrangian_test.cpp b/src/mlpack/tests/aug_lagrangian_test.cpp deleted file mode 100644 index b5f98658ca..0000000000 --- a/src/mlpack/tests/aug_lagrangian_test.cpp +++ /dev/null @@ -1,69 +0,0 @@ -/** - * @file aug_lagrangian_test.cpp - * @author Ryan Curtin - * - * Test of the AugmentedLagrangian class using the test functions defined in - * aug_lagrangian_test_functions.hpp. - * - * mlpack is free software; you may redistribute it and/or modify it under the - * terms of the 3-clause BSD license. You should have received a copy of the - * 3-clause BSD license along with mlpack. If not, see - * http://www.opensource.org/licenses/BSD-3-Clause for more information. - */ -#include -#include -#include -#include -#include "test_tools.hpp" - -using namespace mlpack; -using namespace mlpack::optimization; - -BOOST_AUTO_TEST_SUITE(AugLagrangianTest); - -/** - * Tests the Augmented Lagrangian optimizer using the - * AugmentedLagrangianTestFunction class. - */ -BOOST_AUTO_TEST_CASE(AugLagrangianTestFunctionTest) -{ - // The choice of 10 memory slots is arbitrary. - AugLagrangianTestFunction f; - AugLagrangian aug; - - arma::vec coords = f.GetInitialPoint(); - - if (!aug.Optimize(f, coords, 0)) - BOOST_FAIL("Optimization reported failure."); - - double finalValue = f.Evaluate(coords); - - BOOST_REQUIRE_CLOSE(finalValue, 70.0, 1e-5); - BOOST_REQUIRE_CLOSE(coords[0], 1.0, 1e-5); - BOOST_REQUIRE_CLOSE(coords[1], 4.0, 1e-5); -} - -/** - * Tests the Augmented Lagrangian optimizer using the Gockenbach function. - */ -BOOST_AUTO_TEST_CASE(GockenbachFunctionTest) -{ - GockenbachFunction f; - AugLagrangian aug; - - arma::vec coords = f.GetInitialPoint(); - - if (!aug.Optimize(f, coords, 0)) - BOOST_FAIL("Optimization reported failure."); - - double finalValue = f.Evaluate(coords); - - // Higher tolerance for smaller values. - BOOST_REQUIRE_CLOSE(finalValue, 29.633926, 1e-5); - BOOST_REQUIRE_CLOSE(coords[0], 0.12288178, 1e-3); - BOOST_REQUIRE_CLOSE(coords[1], -1.10778185, 1e-5); - BOOST_REQUIRE_CLOSE(coords[2], 0.015099932, 1e-3); -} - -BOOST_AUTO_TEST_SUITE_END(); - diff --git a/src/mlpack/tests/bigbatch_sgd_test.cpp b/src/mlpack/tests/bigbatch_sgd_test.cpp deleted file mode 100644 index a710563439..0000000000 --- a/src/mlpack/tests/bigbatch_sgd_test.cpp +++ /dev/null @@ -1,135 +0,0 @@ -/** - * @file bigbatch_sgd_test.cpp - * @author Marcus Edel - * - * Test file for big-batch SGD. - * - * mlpack is free software; you may redistribute it and/or modify it under the - * terms of the 3-clause BSD license. You should have received a copy of the - * 3-clause BSD license along with mlpack. If not, see - * http://www.opensource.org/licenses/BSD-3-Clause for more information. - */ -#include -#include -#include - -#include -#include "test_tools.hpp" - -using namespace arma; -using namespace mlpack; -using namespace mlpack::optimization; - -using namespace mlpack::distribution; -using namespace mlpack::regression; - -BOOST_AUTO_TEST_SUITE(BigBatchSGDTest); - -/** - * Create the data for the logistic regression test case. - */ -void CreateLogisticRegressionTestData(arma::mat& data, - arma::mat& testData, - arma::mat& shuffledData, - arma::Row& responses, - arma::Row& testResponses, - arma::Row& shuffledResponses) -{ - // Generate a two-Gaussian dataset. - GaussianDistribution g1(arma::vec("1.0 1.0 1.0"), arma::eye(3, 3)); - GaussianDistribution g2(arma::vec("9.0 9.0 9.0"), arma::eye(3, 3)); - - data = arma::mat(3, 1000); - responses = arma::Row(1000); - for (size_t i = 0; i < 500; ++i) - { - data.col(i) = g1.Random(); - responses[i] = 0; - } - for (size_t i = 500; i < 1000; ++i) - { - data.col(i) = g2.Random(); - responses[i] = 1; - } - - // Shuffle the dataset. - arma::uvec indices = arma::shuffle(arma::linspace(0, - data.n_cols - 1, data.n_cols)); - shuffledData = arma::mat(3, 1000); - shuffledResponses = arma::Row(1000); - for (size_t i = 0; i < data.n_cols; ++i) - { - shuffledData.col(i) = data.col(indices[i]); - shuffledResponses[i] = responses[indices[i]]; - } - - // Create a test set. - testData = arma::mat(3, 1000); - testResponses = arma::Row(1000); - for (size_t i = 0; i < 500; ++i) - { - testData.col(i) = g1.Random(); - testResponses[i] = 0; - } - for (size_t i = 500; i < 1000; ++i) - { - testData.col(i) = g2.Random(); - testResponses[i] = 1; - } -} - -/** - * Run big-batch SGD using BBS_BB on logistic regression and make sure the - * results are acceptable. - */ -BOOST_AUTO_TEST_CASE(BBSBBLogisticRegressionTest) -{ - arma::mat data, testData, shuffledData; - arma::Row responses, testResponses, shuffledResponses; - - CreateLogisticRegressionTestData(data, testData, shuffledData, - responses, testResponses, shuffledResponses); - - // Now run big-batch SGD with a couple of batch sizes. - for (size_t batchSize = 30; batchSize < 40; batchSize += 5) - { - BBS_BB bbsgd(batchSize, 0.01, 0.1, 6000, 1e-3); - LogisticRegression<> lr(shuffledData, shuffledResponses, bbsgd, 0.5); - - // Ensure that the error is close to zero. - const double acc = lr.ComputeAccuracy(data, responses); - BOOST_REQUIRE_CLOSE(acc, 100.0, 0.3); // 0.3% error tolerance. - - const double testAcc = lr.ComputeAccuracy(testData, testResponses); - BOOST_REQUIRE_CLOSE(testAcc, 100.0, 0.6); // 0.6% error tolerance. - } -} - -/** - * Run big-batch SGD using BBS_Armijo on logistic regression and make sure the - * results are acceptable. - */ -BOOST_AUTO_TEST_CASE(BBSArmijoLogisticRegressionTest) -{ - arma::mat data, testData, shuffledData; - arma::Row responses, testResponses, shuffledResponses; - - CreateLogisticRegressionTestData(data, testData, shuffledData, - responses, testResponses, shuffledResponses); - - // Now run big-batch SGD with a couple of batch sizes. - for (size_t batchSize = 30; batchSize < 60; batchSize += 1) - { - BBS_Armijo bbsgd(batchSize, 0.01, 0.1, 6000, 1e-3); - LogisticRegression<> lr(shuffledData, shuffledResponses, bbsgd, 0.5); - - // Ensure that the error is close to zero. - const double acc = lr.ComputeAccuracy(data, responses); - BOOST_REQUIRE_CLOSE(acc, 100.0, 0.3); // 0.3% error tolerance. - - const double testAcc = lr.ComputeAccuracy(testData, testResponses); - BOOST_REQUIRE_CLOSE(testAcc, 100.0, 0.6); // 0.6% error tolerance. - } -} - -BOOST_AUTO_TEST_SUITE_END(); diff --git a/src/mlpack/tests/cmaes_test.cpp b/src/mlpack/tests/cmaes_test.cpp deleted file mode 100644 index ef6b9778e1..0000000000 --- a/src/mlpack/tests/cmaes_test.cpp +++ /dev/null @@ -1,165 +0,0 @@ -/** - * @file cmaes_test.cpp - * @author Marcus Edel - * @author Kartik Nighania - * - * Test file for CMA-ES. - * - * mlpack is free software; you may redistribute it and/or modify it under the - * terms of the 3-clause BSD license. You should have received a copy of the - * 3-clause BSD license along with mlpack. If not, see - * http://www.opensource.org/licenses/BSD-3-Clause for more information. - */ -#include -#include -#include -#include - -#include -#include "test_tools.hpp" - -using namespace arma; -using namespace mlpack::optimization; -using namespace mlpack::optimization::test; - -using namespace mlpack::distribution; -using namespace mlpack::regression; - -using namespace mlpack; - -BOOST_AUTO_TEST_SUITE(CMAESTest); - -/** - * Tests the CMA-ES optimizer using a simple test function. - */ -BOOST_AUTO_TEST_CASE(SimpleTestFunction) -{ - SGDTestFunction f; - CMAES<> optimizer(0, -1, 1, 32, 200, -1); - - arma::mat coordinates = f.GetInitialPoint(); - optimizer.Optimize(f, coordinates); - - BOOST_REQUIRE_SMALL(coordinates[0], 0.003); - BOOST_REQUIRE_SMALL(coordinates[1], 0.003); - BOOST_REQUIRE_SMALL(coordinates[2], 0.003); -} - -/** - * Create the data for the logistic regression test case. - */ -void CreateLogisticRegressionTestData(arma::mat& data, - arma::mat& testData, - arma::mat& shuffledData, - arma::Row& responses, - arma::Row& testResponses, - arma::Row& shuffledResponses) -{ - // Generate a two-Gaussian dataset. - GaussianDistribution g1(arma::vec("1.0 1.0 1.0"), arma::eye(3, 3)); - GaussianDistribution g2(arma::vec("9.0 9.0 9.0"), arma::eye(3, 3)); - - data = arma::mat(3, 1000); - responses = arma::Row(1000); - for (size_t i = 0; i < 500; ++i) - { - data.col(i) = g1.Random(); - responses[i] = 0; - } - for (size_t i = 500; i < 1000; ++i) - { - data.col(i) = g2.Random(); - responses[i] = 1; - } - - // Shuffle the dataset. - arma::uvec indices = arma::shuffle(arma::linspace(0, - data.n_cols - 1, data.n_cols)); - shuffledData = arma::mat(3, 1000); - shuffledResponses = arma::Row(1000); - for (size_t i = 0; i < data.n_cols; ++i) - { - shuffledData.col(i) = data.col(indices[i]); - shuffledResponses[i] = responses[indices[i]]; - } - - // Create a test set. - testData = arma::mat(3, 1000); - testResponses = arma::Row(1000); - for (size_t i = 0; i < 500; ++i) - { - testData.col(i) = g1.Random(); - testResponses[i] = 0; - } - for (size_t i = 500; i < 1000; ++i) - { - testData.col(i) = g2.Random(); - testResponses[i] = 1; - } -} - -/** - * Run CMA-ES with the full selection policy on logistic regression and - * make sure the results are acceptable. - */ -BOOST_AUTO_TEST_CASE(CMAESLogisticRegressionTest) -{ - const size_t trials = 3; - bool success = false; - for (size_t trial = 0; trial < trials; ++trial) - { - arma::mat data, testData, shuffledData; - arma::Row responses, testResponses, shuffledResponses; - - CreateLogisticRegressionTestData(data, testData, shuffledData, - responses, testResponses, shuffledResponses); - - CMAES<> cmaes(0, -1, 1, 32, 200, 1e-3); - LogisticRegression<> lr(shuffledData, shuffledResponses, cmaes, 0.5); - - // Ensure that the error is close to zero. - const double acc = lr.ComputeAccuracy(data, responses); - const double testAcc = lr.ComputeAccuracy(testData, testResponses); - if (acc >= 99.7 && testAcc >= 99.4) - { - success = true; - break; - } - } - - BOOST_REQUIRE_EQUAL(success, true); -} - -/** - * Run CMA-ES with the random selection policy on logistic regression and - * make sure the results are acceptable. - */ -BOOST_AUTO_TEST_CASE(ApproxCMAESLogisticRegressionTest) -{ - const size_t trials = 3; - bool success = false; - for (size_t trial = 0; trial < trials; ++trial) - { - arma::mat data, testData, shuffledData; - arma::Row responses, testResponses, shuffledResponses; - - CreateLogisticRegressionTestData(data, testData, shuffledData, - responses, testResponses, shuffledResponses); - - ApproxCMAES<> cmaes(0, -1, 1, 32, 200, 1e-3); - LogisticRegression<> lr(shuffledData, shuffledResponses, cmaes, 0.5); - - // Ensure that the error is close to zero. - const double acc = lr.ComputeAccuracy(data, responses); - const double testAcc = lr.ComputeAccuracy(testData, testResponses); - if (acc >= 99.7 && testAcc >= 99.4) - { - success = true; - break; - } - } - - BOOST_REQUIRE_EQUAL(success, true); -} - -BOOST_AUTO_TEST_SUITE_END(); diff --git a/src/mlpack/tests/cne_test.cpp b/src/mlpack/tests/cne_test.cpp deleted file mode 100644 index 3e9a7e57be..0000000000 --- a/src/mlpack/tests/cne_test.cpp +++ /dev/null @@ -1,222 +0,0 @@ -/** - * @file cne_test.cpp - * @author Marcus Edel - * @author Kartik Nighania - * - * Test file for CNE (Conventional Neural Evolution). - * - * mlpack is free software; you may redistribute it and/or modify it under the - * terms of the 3-clause BSD license. You should have received a copy of the - * 3-clause BSD license along with mlpack. If not, see - * http://www.opensource.org/licenses/BSD-3-Clause for more information. - */ -#include - -#include -#include -#include - -#include - -#include -#include "test_tools.hpp" - -using namespace mlpack; -using namespace mlpack::ann; -using namespace mlpack::optimization; - -using namespace mlpack::distribution; -using namespace mlpack::regression; - -BOOST_AUTO_TEST_SUITE(CNETest); - -/** - * Training a vanilla network for 2 input XOR function - */ -BOOST_AUTO_TEST_CASE(CNEXORTest) -{ - /* - * Create the four cases for XOR with two variable - * - * Input Output - * 0 XOR 0 = 0 - * 1 XOR 1 = 0 - * 0 XOR 1 = 1 - * 1 XOR 0 = 1 - */ - arma::mat train("1, 0, 0, 1; 1, 0, 1, 0"); - arma::mat labels("1, 1, 2, 2"); - - // CNE may fail to find a good optimum. But if it can succeed one out of 6 - // times I think that is sufficient to say it is working. - size_t successes = 0; - for (size_t trial = 0; trial < 6; ++trial) - { - // Build a network with 2 input, 2 hidden, and 2 output layers. - FFN > network; - - network.Add >(2, 2); - network.Add >(); - network.Add >(2, 2); - network.Add >(); - - // CNE object. - CNE opt(60, 5000, 0.1, 0.02, 0.2, 0.1, -1); - - // Training the network with CNE - network.Train(train, labels, opt); - - // Predicting for the same train data - arma::mat predictionTemp; - network.Predict(train, predictionTemp); - - arma::mat prediction = arma::zeros(1, predictionTemp.n_cols); - - for (size_t i = 0; i < predictionTemp.n_cols; ++i) - { - prediction(i) = arma::as_scalar(arma::find( - arma::max(predictionTemp.col(i)) == predictionTemp.col(i), 1)) + 1; - } - - // 1 means 0 and 2 means 1 as the output to XOR. - if ((prediction[0] == 1) && - (prediction[1] == 1) && - (prediction[2] == 2) && - (prediction[3] == 2)) - { - ++successes; - break; - } - } - - BOOST_REQUIRE_GT(successes, 0); -} - -/** - * Train and test a logistic regression function using CNE optimizer - */ -BOOST_AUTO_TEST_CASE(CNELogisticRegressionTest) -{ - // Generate a two-Gaussian dataset. - GaussianDistribution g1(arma::vec("1.0 1.0 1.0"), arma::eye(3, 3)); - GaussianDistribution g2(arma::vec("9.0 9.0 9.0"), arma::eye(3, 3)); - - arma::mat data(3, 1000); - arma::Row responses(1000); - for (size_t i = 0; i < 500; ++i) - { - data.col(i) = g1.Random(); - responses[i] = 0; - } - for (size_t i = 500; i < 1000; ++i) - { - data.col(i) = g2.Random(); - responses[i] = 1; - } - - // Shuffle the dataset. - arma::uvec indices = arma::shuffle(arma::linspace(0, - data.n_cols - 1, data.n_cols)); - arma::mat shuffledData(3, 1000); - arma::Row shuffledResponses(1000); - for (size_t i = 0; i < data.n_cols; ++i) - { - shuffledData.col(i) = data.col(indices[i]); - shuffledResponses[i] = responses[indices[i]]; - } - - // Create a test set. - arma::mat testData(3, 1000); - arma::Row testResponses(1000); - for (size_t i = 0; i < 500; ++i) - { - testData.col(i) = g1.Random(); - testResponses[i] = 0; - } - for (size_t i = 500; i < 1000; ++i) - { - testData.col(i) = g2.Random(); - testResponses[i] = 1; - } - - CNE opt(200, 10000, 0.2, 0.2, 0.3, 65, -1); - - LogisticRegression<> lr(shuffledData, shuffledResponses, opt, 0.5); - - // Ensure that the error is close to zero. - const double acc = lr.ComputeAccuracy(data, responses); - BOOST_REQUIRE_CLOSE(acc, 100.0, 0.3); // 0.3% error tolerance. - - const double testAcc = lr.ComputeAccuracy(testData, testResponses); - BOOST_REQUIRE_CLOSE(testAcc, 100.0, 0.6); // 0.6% error tolerance. -} - -/** - * Training a vanilla network on a larger dataset using CNE optimizer. - */ -BOOST_AUTO_TEST_CASE(VanillaNetworkWithCNETest) -{ - // Load the datasets. - arma::mat trainData; - data::Load("iris_train.csv", trainData, true); - - arma::mat testData; - data::Load("iris_test.csv", testData, true); - - arma::mat trainLabels; - data::Load("iris_train_labels.csv", trainLabels, true); - trainLabels += 1; - - arma::mat testLabels; - data::Load("iris_test_labels.csv", testLabels, true); - testLabels += 1; - - // Training the network may fail, so we will try a few times. - size_t successes = 0; - for (size_t trial = 0; trial < 4; ++trial) - { - // Create vanilla network with 4 input, 4 hidden and 3 output nodes. - FFN > model; - model.Add >(trainData.n_rows, 4); - model.Add >(); - model.Add >(4, 3); - model.Add >(); - - // Creating CNE object. - // The tolerance and objectiveChange are not taken into consideration. - CNE opt(30, 200, 0.2, 0.2, 0.3, -1, -1); - - model.Train(trainData, trainLabels, opt); - - arma::mat predictionTemp; - model.Predict(testData, predictionTemp); - arma::mat prediction = arma::zeros(1, predictionTemp.n_cols); - - for (size_t i = 0; i < predictionTemp.n_cols; ++i) - { - prediction(i) = arma::as_scalar(arma::find( - arma::max(predictionTemp.col(i)) == predictionTemp.col(i), 1)) + 1; - } - - size_t error = 0; - for (size_t i = 0; i < testData.n_cols; i++) - { - if (int(arma::as_scalar(prediction.col(i))) == - int(arma::as_scalar(testLabels.col(i)))) - { - error++; - } - } - - double classificationError = 1 - double(error) / testData.n_cols; - if (classificationError <= 0.1) - { - ++successes; - break; - } - } - - BOOST_REQUIRE_GT(successes, 0); -} - -BOOST_AUTO_TEST_SUITE_END(); diff --git a/src/mlpack/tests/frankwolfe_test.cpp b/src/mlpack/tests/frankwolfe_test.cpp deleted file mode 100644 index c3d2a6e0cf..0000000000 --- a/src/mlpack/tests/frankwolfe_test.cpp +++ /dev/null @@ -1,193 +0,0 @@ -/** - * @file frankwolfe_test.cpp - * @author Chenzhe Diao - * - * Test file for Frank-Wolfe type optimizer. - * - * mlpack is free software; you may redistribute it and/or modify it under the - * terms of the 3-clause BSD license. You should have received a copy of the - * 3-clause BSD license along with mlpack. If not, see - * http://www.opensource.org/licenses/BSD-3-Clause for more information. - */ -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include "test_tools.hpp" - -using namespace std; -using namespace arma; -using namespace mlpack; -using namespace mlpack::optimization; - -BOOST_AUTO_TEST_SUITE(FrankWolfeTest); - - -/** - * Simple test of Orthogonal Matching Pursuit algorithm. - */ -BOOST_AUTO_TEST_CASE(OMPTest) -{ - int k = 5; - mat B1 = eye(3, 3); - mat B2 = 0.1 * randn(3, k); - mat A = join_horiz(B1, B2); // The dictionary is input as columns of A. - vec b; - b << 1 << 1 << 0; // Vector to be sparsely approximated. - - FuncSq f(A, b); - ConstrLpBallSolver linearConstrSolver(1); - UpdateSpan updateRule; - - OMP s(linearConstrSolver, updateRule); - - vec coordinates = zeros(k + 3); - double result = s.Optimize(f, coordinates); - - BOOST_REQUIRE_SMALL(result, 1e-10); - BOOST_REQUIRE_SMALL(coordinates[0] - 1, 1e-10); - BOOST_REQUIRE_SMALL(coordinates[1] - 1, 1e-10); - BOOST_REQUIRE_SMALL(coordinates[2], 1e-10); - for (int ii = 0; ii < k; ++ii) - { - BOOST_REQUIRE_SMALL(coordinates[ii + 3], 1e-10); - } -} - -/** - * Simple test of Orthogonal Matching Pursuit with regularization. - */ -BOOST_AUTO_TEST_CASE(regularizedOMP) -{ - int k = 10; - mat B1 = 0.1 * eye(k, k); - mat B2 = 100 * randn(k, k); - mat A = join_horiz(B1, B2); // The dictionary is input as columns of A. - vec b(k, arma::fill::zeros); // Vector to be sparsely approximated. - b(0) = 1; - b(1) = 1; - vec lambda(A.n_cols); - for (size_t ii = 0; ii < A.n_cols; ii++) - lambda(ii) = norm(A.col(ii), 2); - - FuncSq f(A, b); - ConstrLpBallSolver linearConstrSolver(1, lambda); - UpdateSpan updateRule; - - OMP s(linearConstrSolver, updateRule); - - vec coordinates = zeros(2 * k); - double result = s.Optimize(f, coordinates); - - BOOST_REQUIRE_SMALL(result, 1e-10); -} - -/** - * Simple test of Orthogonal Matching Pursuit with support prune. - */ -BOOST_AUTO_TEST_CASE(PruneSupportOMP) -{ - // The dictionary is input as columns of A. - int k = 3; - mat B1; - B1 << 1 << 0 << 1 << endr - << 0 << 1 << 1 << endr - << 0 << 0 << 1 << endr; - mat B2 = randu(k, k); - mat A = join_horiz(B1, B2); // The dictionary is input as columns of A. - vec b; - b << 1 << 1 << 0; // Vector to be sparsely approximated. - - FuncSq f(A, b); - ConstrLpBallSolver linearConstrSolver(1); - UpdateSpan updateRule(true); - - OMP s(linearConstrSolver, updateRule); - - vec coordinates = zeros(k + 3); - double result = s.Optimize(f, coordinates); - - BOOST_REQUIRE_SMALL(result, 1e-10); -} - -/** - * Simple test of sparse soluton in atom domain with atom norm constraint. - */ -BOOST_AUTO_TEST_CASE(AtomNormConstraint) -{ - int k = 5; - mat B1 = eye(3, 3); - mat B2 = 0.1 * randn(3, k); - mat A = join_horiz(B1, B2); // The dictionary is input as columns of A. - vec b; - b << 1 << 1 << 0; // Vector to be sparsely approximated. - - FuncSq f(A, b); - ConstrLpBallSolver linearConstrSolver(1); - UpdateFullCorrection updateRule(2, 0.2); - - FrankWolfe - s(linearConstrSolver, updateRule); - - vec coordinates = zeros(k + 3); - double result = s.Optimize(f, coordinates); - - BOOST_REQUIRE_SMALL(result, 1e-10); -} - - -/** - * A very simple test of classic Frank-Wolfe algorithm. - * The constrained domain used is unit lp ball. - */ -BOOST_AUTO_TEST_CASE(ClassicFW) -{ - TestFuncFW f; - double p = 2; // Constraint set is unit lp ball. - ConstrLpBallSolver linearConstrSolver(p); - UpdateClassic updateRule; - - FrankWolfe - s(linearConstrSolver, updateRule); - - vec coordinates = randu(3); - double result = s.Optimize(f, coordinates); - - BOOST_REQUIRE_SMALL(result, 1e-4); - BOOST_REQUIRE_SMALL(coordinates[0] - 0.1, 1e-4); - BOOST_REQUIRE_SMALL(coordinates[1] - 0.2, 1e-4); - BOOST_REQUIRE_SMALL(coordinates[2] - 0.3, 1e-4); -} - -/** - * Exactly the same problem with ClassicFW. - * The update step performs a line search now. - * It converges much faster. - */ -BOOST_AUTO_TEST_CASE(FWLineSearch) -{ - TestFuncFW f; - double p = 2; // Constraint set is unit lp ball. - ConstrLpBallSolver linearConstrSolver(p); - UpdateLineSearch updateRule; - - FrankWolfe - s(linearConstrSolver, updateRule); - - vec coordinates = randu(3); - double result = s.Optimize(f, coordinates); - - BOOST_REQUIRE_SMALL(result, 1e-4); - BOOST_REQUIRE_SMALL(coordinates[0] - 0.1, 1e-4); - BOOST_REQUIRE_SMALL(coordinates[1] - 0.2, 1e-4); - BOOST_REQUIRE_SMALL(coordinates[2] - 0.3, 1e-4); -} - -BOOST_AUTO_TEST_SUITE_END(); diff --git a/src/mlpack/tests/gradient_clipping_test.cpp b/src/mlpack/tests/gradient_clipping_test.cpp deleted file mode 100644 index 87afd669c5..0000000000 --- a/src/mlpack/tests/gradient_clipping_test.cpp +++ /dev/null @@ -1,72 +0,0 @@ -/** - * @file gradient_clipping_test.cpp - * @author Konstantin Sidorov - * - * Test file for gradient clipping. - * - * mlpack is free software; you may redistribute it and/or modify it under the - * terms of the 3-clause BSD license. You should have received a copy of the - * 3-clause BSD license along with mlpack. If not, see - * http://www.opensource.org/licenses/BSD-3-Clause for more information. - */ -#include -#include -#include -#include -#include - -#include -#include "test_tools.hpp" - -using namespace std; -using namespace arma; -using namespace mlpack; -using namespace mlpack::optimization; -using namespace mlpack::optimization::test; - -BOOST_AUTO_TEST_SUITE(GradientClippingTest); - -// Test checking that gradient clipping works with vanilla update. -BOOST_AUTO_TEST_CASE(ClippedVanillaUpdateTest) -{ - VanillaUpdate vanillaUpdate; - GradientClipping update(-3.0, +3.0, vanillaUpdate); - update.Initialize(3, 3); - - arma::mat coordinates = arma::zeros(3, 3); - // Setting step = 1 to make math easy. - double stepSize = 1.0; - arma::mat dummyGradient("-6 6 0; 1 2 3; -3 0 4;"); - update.Update(coordinates, stepSize, dummyGradient); - // After clipping, we should get the following coordinates: - arma::mat targetCoordinates("3 -3 0; -1 -2 -3; 3 0 -3;"); - BOOST_REQUIRE_SMALL(arma::abs(coordinates - targetCoordinates).max(), 1e-7); -} - -// Test checking that gradient clipping works with momentum update. -BOOST_AUTO_TEST_CASE(ClippedMomentumUpdateTest) -{ - // Once again, setting momentum = 1 for easy math - // (now momentum = -stepSize * [sum of gradients]) - MomentumUpdate momentumUpdate(1); - GradientClipping update(-3.0, +3.0, momentumUpdate); - update.Initialize(3, 3); - - arma::mat coordinates = arma::zeros(3, 3); - double stepSize = 1.0; - arma::mat dummyGradient("-6 6 0; 1 2 3; -3 0 4;"); - update.Update(coordinates, stepSize, dummyGradient); - arma::mat targetCoordinates("3 -3 0; -1 -2 -3; 3 0 -3;"); - // On the first Update() call the parameters - // should just be equal to (-gradient). - BOOST_REQUIRE_SMALL(arma::abs(coordinates - targetCoordinates).max(), 1e-7); - update.Update(coordinates, stepSize, dummyGradient); - // On the second Update() call the Momentum update will subtract - // the gradient from the momentum, which gives 2 * gradient value - // for the momentum on that step. Adding that to the gradient which - // was subtracted earlier yiels the 3 * gradient in the following check. - BOOST_REQUIRE_SMALL( - arma::abs(coordinates - 3 * targetCoordinates).max(), 1e-7); -} - -BOOST_AUTO_TEST_SUITE_END(); diff --git a/src/mlpack/tests/gradient_descent_test.cpp b/src/mlpack/tests/gradient_descent_test.cpp deleted file mode 100644 index 12048d9d8a..0000000000 --- a/src/mlpack/tests/gradient_descent_test.cpp +++ /dev/null @@ -1,58 +0,0 @@ -/** - * @file gradient_descent_test.cpp - * @author Sumedh Ghaisas - * - * Test file for Gradient Descent optimizer. - * - * mlpack is free software; you may redistribute it and/or modify it under the - * terms of the 3-clause BSD license. You should have received a copy of the - * 3-clause BSD license along with mlpack. If not, see - * http://www.opensource.org/licenses/BSD-3-Clause for more information. - */ -#include -#include - -#include -#include - -#include -#include "test_tools.hpp" - -using namespace std; -using namespace arma; -using namespace mlpack; -using namespace mlpack::optimization; -using namespace mlpack::optimization::test; - -BOOST_AUTO_TEST_SUITE(GradientDescentTest); - -BOOST_AUTO_TEST_CASE(SimpleGDTestFunction) -{ - GDTestFunction f; - GradientDescent s(0.01, 5000000, 1e-9); - - arma::vec coordinates = f.GetInitialPoint(); - double result = s.Optimize(f, coordinates); - - BOOST_REQUIRE_SMALL(result, 1e-4); - BOOST_REQUIRE_SMALL(coordinates[0], 1e-2); - BOOST_REQUIRE_SMALL(coordinates[1], 1e-2); - BOOST_REQUIRE_SMALL(coordinates[2], 1e-2); -} - -BOOST_AUTO_TEST_CASE(RosenbrockTest) -{ - // Create the Rosenbrock function. - RosenbrockFunction f; - - GradientDescent s(0.001, 0, 1e-15); - - arma::mat coordinates = f.GetInitialPoint(); - double result = s.Optimize(f, coordinates); - - BOOST_REQUIRE_SMALL(result, 1e-10); - for (size_t j = 0; j < 2; ++j) - BOOST_REQUIRE_CLOSE(coordinates[j], (double) 1.0, 1e-3); -} - -BOOST_AUTO_TEST_SUITE_END(); diff --git a/src/mlpack/tests/iqn_test.cpp b/src/mlpack/tests/iqn_test.cpp deleted file mode 100644 index b8758af114..0000000000 --- a/src/mlpack/tests/iqn_test.cpp +++ /dev/null @@ -1,88 +0,0 @@ -/** - * @file iqn_test.cpp - * @author Marcus Edel - * - * Test file for IQN (incremental Quasi-Newton). - * - * mlpack is free software; you may redistribute it and/or modify it under the - * terms of the 3-clause BSD license. You should have received a copy of the - * 3-clause BSD license along with mlpack. If not, see - * http://www.opensource.org/licenses/BSD-3-Clause for more information. - */ -#include -#include -#include - -#include -#include "test_tools.hpp" - -using namespace mlpack::optimization; -using namespace mlpack::distribution; -using namespace mlpack::regression; -using namespace mlpack; - -BOOST_AUTO_TEST_SUITE(IQNTest); - -/** - * Run IQN on logistic regression and make sure the results are acceptable. - */ -BOOST_AUTO_TEST_CASE(LogisticRegressionTest) -{ - // Generate a two-Gaussian dataset. - GaussianDistribution g1(arma::vec("1.0 1.0 1.0"), arma::eye(3, 3)); - GaussianDistribution g2(arma::vec("9.0 9.0 9.0"), arma::eye(3, 3)); - - arma::mat data(3, 1000); - arma::Row responses(1000); - for (size_t i = 0; i < 500; ++i) - { - data.col(i) = g1.Random(); - responses[i] = 0; - } - for (size_t i = 500; i < 1000; ++i) - { - data.col(i) = g2.Random(); - responses[i] = 1; - } - - // Shuffle the dataset. - arma::uvec indices = arma::shuffle(arma::linspace(0, - data.n_cols - 1, data.n_cols)); - arma::mat shuffledData(3, 1000); - arma::Row shuffledResponses(1000); - for (size_t i = 0; i < data.n_cols; ++i) - { - shuffledData.col(i) = data.col(indices[i]); - shuffledResponses[i] = responses[indices[i]]; - } - - // Create a test set. - arma::mat testData(3, 1000); - arma::Row testResponses(1000); - for (size_t i = 0; i < 500; ++i) - { - testData.col(i) = g1.Random(); - testResponses[i] = 0; - } - for (size_t i = 500; i < 1000; ++i) - { - testData.col(i) = g2.Random(); - testResponses[i] = 1; - } - - // Now run SGDR with snapshot ensembles on a couple of batch sizes. - for (size_t batchSize = 1; batchSize < 9; batchSize += 4) - { - IQN iqn(0.01, batchSize, 5000, 1e-3); - LogisticRegression<> lr(shuffledData, shuffledResponses, iqn, 0.5); - - // Ensure that the error is close to zero. - const double acc = lr.ComputeAccuracy(data, responses); - BOOST_REQUIRE_CLOSE(acc, 100.0, 1.3); // 1.3% error tolerance. - - const double testAcc = lr.ComputeAccuracy(testData, testResponses); - BOOST_REQUIRE_CLOSE(testAcc, 100.0, 1.6); // 1.6% error tolerance. - } -} - -BOOST_AUTO_TEST_SUITE_END(); diff --git a/src/mlpack/tests/katyusha_test.cpp b/src/mlpack/tests/katyusha_test.cpp deleted file mode 100644 index e4666c4144..0000000000 --- a/src/mlpack/tests/katyusha_test.cpp +++ /dev/null @@ -1,77 +0,0 @@ -/** - * @file katyusha_test.cpp - * @author Marcus Edel - * - * Test file for Katyusha. - * - * mlpack is free software; you may redistribute it and/or modify it under the - * terms of the 3-clause BSD license. You should have received a copy of the - * 3-clause BSD license along with mlpack. If not, see - * http://www.opensource.org/licenses/BSD-3-Clause for more information. - */ -#include -#include - -#include -#include "test_tools.hpp" -#include "test_function_tools.hpp" - -using namespace mlpack; -using namespace mlpack::optimization; - -BOOST_AUTO_TEST_SUITE(KatyushaTest); - -/** - * Run Katyusha on logistic regression and make sure the results are acceptable. - */ -BOOST_AUTO_TEST_CASE(KatyushaLogisticRegressionTest) -{ - arma::mat data, testData, shuffledData; - arma::Row responses, testResponses, shuffledResponses; - - LogisticRegressionTestData(data, testData, shuffledData, - responses, testResponses, shuffledResponses); - - // Now run big-batch SGD with a couple of batch sizes. - for (size_t batchSize = 30; batchSize < 45; batchSize += 5) - { - Katyusha optimizer(1.0, 10.0, batchSize, 100, 0, 1e-10, true); - LogisticRegression<> lr(shuffledData, shuffledResponses, optimizer, 0.5); - - // Ensure that the error is close to zero. - const double acc = lr.ComputeAccuracy(data, responses); - BOOST_REQUIRE_CLOSE(acc, 100.0, 1.5); // 1.5% error tolerance. - - const double testAcc = lr.ComputeAccuracy(testData, testResponses); - BOOST_REQUIRE_CLOSE(testAcc, 100.0, 1.5); // 1.5% error tolerance. - } -} - -/** - * Run Proximal Katyusha on logistic regression and make sure the results are - * acceptable. - */ -BOOST_AUTO_TEST_CASE(KatyushaProximalLogisticRegressionTest) -{ - arma::mat data, testData, shuffledData; - arma::Row responses, testResponses, shuffledResponses; - - LogisticRegressionTestData(data, testData, shuffledData, - responses, testResponses, shuffledResponses); - - // Now run big-batch SGD with a couple of batch sizes. - for (size_t batchSize = 30; batchSize < 45; batchSize += 5) - { - KatyushaProximal optimizer(1.0, 10.0, batchSize, 100, 0, 1e-10, true); - LogisticRegression<> lr(shuffledData, shuffledResponses, optimizer, 0.5); - - // Ensure that the error is close to zero. - const double acc = lr.ComputeAccuracy(data, responses); - BOOST_REQUIRE_CLOSE(acc, 100.0, 1.5); // 1.5% error tolerance. - - const double testAcc = lr.ComputeAccuracy(testData, testResponses); - BOOST_REQUIRE_CLOSE(testAcc, 100.0, 1.5); // 1.5% error tolerance. - } -} - -BOOST_AUTO_TEST_SUITE_END(); diff --git a/src/mlpack/tests/lbfgs_test.cpp b/src/mlpack/tests/lbfgs_test.cpp deleted file mode 100644 index dc95ae0f24..0000000000 --- a/src/mlpack/tests/lbfgs_test.cpp +++ /dev/null @@ -1,140 +0,0 @@ -/** - * @file lbfgs_test.cpp - * - * Tests the L-BFGS optimizer on a couple test functions. - * - * @author Ryan Curtin (gth671b@mail.gatech.edu) - * - * mlpack is free software; you may redistribute it and/or modify it under the - * terms of the 3-clause BSD license. You should have received a copy of the - * 3-clause BSD license along with mlpack. If not, see - * http://www.opensource.org/licenses/BSD-3-Clause for more information. - */ -#include -#include - -#include -#include -#include - -#include -#include "test_tools.hpp" - -using namespace mlpack::optimization; -using namespace mlpack::optimization::test; - -BOOST_AUTO_TEST_SUITE(LBFGSTest); - -/** - * Tests the L-BFGS optimizer using the Rosenbrock Function. - */ -BOOST_AUTO_TEST_CASE(RosenbrockFunctionTest) -{ - RosenbrockFunction f; - L_BFGS lbfgs; - lbfgs.MaxIterations() = 10000; - - arma::vec coords = f.GetInitialPoint(); - if (!lbfgs.Optimize(f, coords)) - BOOST_FAIL("L-BFGS optimization reported failure."); - - double finalValue = f.Evaluate(coords); - - BOOST_REQUIRE_SMALL(finalValue, 1e-5); - BOOST_REQUIRE_CLOSE(coords[0], 1.0, 1e-5); - BOOST_REQUIRE_CLOSE(coords[1], 1.0, 1e-5); -} - -/** - * Tests the L-BFGS optimizer using the Colville Function. - */ -BOOST_AUTO_TEST_CASE(ColvilleFunctionTest) -{ - ColvilleFunction f; - L_BFGS lbfgs; - lbfgs.MaxIterations() = 10000; - - arma::vec coords = f.GetInitialPoint(); - if (!lbfgs.Optimize(f, coords)) - BOOST_FAIL("L-BFGS optimization reported failure."); - - BOOST_REQUIRE_CLOSE(coords[0], 1.0, 1e-5); - BOOST_REQUIRE_CLOSE(coords[1], 1.0, 1e-5); -} - -/** - * Tests the L-BFGS optimizer using the Wood Function. - */ -BOOST_AUTO_TEST_CASE(WoodFunctionTest) -{ - WoodFunction f; - L_BFGS lbfgs; - lbfgs.MaxIterations() = 10000; - - arma::vec coords = f.GetInitialPoint(); - if (!lbfgs.Optimize(f, coords)) - BOOST_FAIL("L-BFGS optimization reported failure."); - - double finalValue = f.Evaluate(coords); - - BOOST_REQUIRE_SMALL(finalValue, 1e-5); - BOOST_REQUIRE_CLOSE(coords[0], 1.0, 1e-5); - BOOST_REQUIRE_CLOSE(coords[1], 1.0, 1e-5); - BOOST_REQUIRE_CLOSE(coords[2], 1.0, 1e-5); - BOOST_REQUIRE_CLOSE(coords[3], 1.0, 1e-5); -} - -/** - * Tests the L-BFGS optimizer using the generalized Rosenbrock function. This - * is actually multiple tests, increasing the dimension by powers of 2, from 4 - * dimensions to 1024 dimensions. - */ -BOOST_AUTO_TEST_CASE(GeneralizedRosenbrockFunctionTest) -{ - for (int i = 2; i < 10; i++) - { - // Dimension: powers of 2 - int dim = std::pow(2.0, i); - - GeneralizedRosenbrockFunction f(dim); - L_BFGS lbfgs(20); - lbfgs.MaxIterations() = 10000; - - arma::vec coords = f.GetInitialPoint(); - if (!lbfgs.Optimize(f, coords)) - BOOST_FAIL("L-BFGS optimization reported failure."); - - double finalValue = f.Evaluate(coords); - - // Test the output to make sure it is correct. - BOOST_REQUIRE_SMALL(finalValue, 1e-5); - for (int j = 0; j < dim; j++) - BOOST_REQUIRE_CLOSE(coords[j], 1.0, 1e-5); - } -} - -/** - * Tests the L-BFGS optimizer using the Rosenbrock-Wood combined function. This - * is a test on optimizing a matrix of coordinates. - */ -BOOST_AUTO_TEST_CASE(RosenbrockWoodFunctionTest) -{ - RosenbrockWoodFunction f; - L_BFGS lbfgs; - lbfgs.MaxIterations() = 10000; - - arma::mat coords = f.GetInitialPoint(); - if (!lbfgs.Optimize(f, coords)) - BOOST_FAIL("L-BFGS optimization reported failure."); - - double finalValue = f.Evaluate(coords); - - BOOST_REQUIRE_SMALL(finalValue, 1e-5); - for (int row = 0; row < 4; row++) - { - BOOST_REQUIRE_CLOSE((coords(row, 0)), 1.0, 1e-5); - BOOST_REQUIRE_CLOSE((coords(row, 1)), 1.0, 1e-5); - } -} - -BOOST_AUTO_TEST_SUITE_END(); diff --git a/src/mlpack/tests/line_search_test.cpp b/src/mlpack/tests/line_search_test.cpp deleted file mode 100644 index 688362e71b..0000000000 --- a/src/mlpack/tests/line_search_test.cpp +++ /dev/null @@ -1,49 +0,0 @@ -/** - * @file line_search_test.cpp - * @author Chenzhe Diao - * - * Test file for line search optimizer. - * - * mlpack is free software; you may redistribute it and/or modify it under the - * terms of the 3-clause BSD license. You should have received a copy of the - * 3-clause BSD license along with mlpack. If not, see - * http://www.opensource.org/licenses/BSD-3-Clause for more information. - */ - - -#include -#include -#include - -#include -#include "test_tools.hpp" - -using namespace std; -using namespace arma; -using namespace mlpack; -using namespace mlpack::optimization; - -BOOST_AUTO_TEST_SUITE(LineSearchTest); - -/** - * Simple test of Line Search with TestFuncFW function. - */ -BOOST_AUTO_TEST_CASE(FuncFWTest) -{ - vec x1 = zeros(3); - vec x2; - x2 << 0.2 << 0.4 << 0.6; - - TestFuncFW f; - LineSearch s; - - double result = s.Optimize(f, x1, x2); - - BOOST_REQUIRE_SMALL(result, 1e-10); - BOOST_REQUIRE_SMALL(x2[0] - 0.1, 1e-10); - BOOST_REQUIRE_SMALL(x2[1] - 0.2, 1e-10); - BOOST_REQUIRE_SMALL(x2[2] - 0.3, 1e-10); -} - - -BOOST_AUTO_TEST_SUITE_END(); diff --git a/src/mlpack/tests/lrsdp_test.cpp b/src/mlpack/tests/lrsdp_test.cpp deleted file mode 100644 index eb1b9d3b66..0000000000 --- a/src/mlpack/tests/lrsdp_test.cpp +++ /dev/null @@ -1,317 +0,0 @@ -/** - * @file lrsdp_test.cpp - * @author Ryan Curtin - * - * Tests for LR-SDP (core/optimizers/sdp/). - * - * mlpack is free software; you may redistribute it and/or modify it under the - * terms of the 3-clause BSD license. You should have received a copy of the - * 3-clause BSD license along with mlpack. If not, see - * http://www.opensource.org/licenses/BSD-3-Clause for more information. - */ -#include -#include - -#include -#include "test_tools.hpp" - -using namespace mlpack; -using namespace mlpack::optimization; - -BOOST_AUTO_TEST_SUITE(LRSDPTest); - -/** - * Create a Lovasz-Theta initial point. - */ -void CreateLovaszThetaInitialPoint(const arma::mat& edges, - arma::mat& coordinates) -{ - // Get the number of vertices in the problem. - const size_t vertices = max(max(edges)) + 1; - - const size_t m = edges.n_cols + 1; - float r = 0.5 + sqrt(0.25 + 2 * m); - if (ceil(r) > vertices) - r = vertices; // An upper bound on the dimension. - - coordinates.set_size(vertices, ceil(r)); - - // Now we set the entries of the initial matrix according to the formula given - // in Section 4 of Monteiro and Burer. - for (size_t i = 0; i < vertices; ++i) - { - for (size_t j = 0; j < ceil(r); ++j) - { - if (i == j) - coordinates(i, j) = sqrt(1.0 / r) + sqrt(1.0 / (vertices * m)); - else - coordinates(i, j) = sqrt(1.0 / (vertices * m)); - } - } -} - -/** - * Prepare an LRSDP object to solve the Lovasz-Theta SDP in the manner detailed - * in Monteiro + Burer 2004. The list of edges in the graph must be given; that - * is all that is necessary to set up the problem. A matrix which will contain - * initial point coordinates should be given also. - */ -void SetupLovaszTheta(const arma::mat& edges, - LRSDP>& lovasz) -{ - // Get the number of vertices in the problem. - const size_t vertices = max(max(edges)) + 1; - - // C = -(e e^T) = -ones(). - lovasz.SDP().C().ones(vertices, vertices); - lovasz.SDP().C() *= -1; - - // b_0 = 1; else = 0. - lovasz.SDP().SparseB().zeros(edges.n_cols + 1); - lovasz.SDP().SparseB()[0] = 1; - - // A_0 = I_n. - lovasz.SDP().SparseA()[0].eye(vertices, vertices); - - // A_ij only has ones at (i, j) and (j, i) and 0 elsewhere. - for (size_t i = 0; i < edges.n_cols; ++i) - { - lovasz.SDP().SparseA()[i + 1].zeros(vertices, vertices); - lovasz.SDP().SparseA()[i + 1](edges(0, i), edges(1, i)) = 1.; - lovasz.SDP().SparseA()[i + 1](edges(1, i), edges(0, i)) = 1.; - } - - // Set the Lagrange multipliers right. - lovasz.AugLag().Lambda().ones(edges.n_cols + 1); - lovasz.AugLag().Lambda() *= -1; - lovasz.AugLag().Lambda()[0] = -double(vertices); -} - -/** - * johnson8-4-4.co test case for Lovasz-Theta LRSDP. - * See Monteiro and Burer 2004. - */ -BOOST_AUTO_TEST_CASE(Johnson844LovaszThetaSDP) -{ - // Load the edges. - arma::mat edges; - data::Load("johnson8-4-4.csv", edges, true); - - // The LRSDP itself and the initial point. - arma::mat coordinates; - - CreateLovaszThetaInitialPoint(edges, coordinates); - - LRSDP> lovasz(edges.n_cols + 1, 0, coordinates); - - SetupLovaszTheta(edges, lovasz); - - double finalValue = lovasz.Optimize(coordinates); - - // Final value taken from Monteiro + Burer 2004. - BOOST_REQUIRE_CLOSE(finalValue, -14.0, 1e-5); - - // Now ensure that all the constraints are satisfied. - arma::mat rrt = coordinates * trans(coordinates); - BOOST_REQUIRE_CLOSE(trace(rrt), 1.0, 1e-5); - - // All those edge constraints... - for (size_t i = 0; i < edges.n_cols; ++i) - { - BOOST_REQUIRE_SMALL(rrt(edges(0, i), edges(1, i)), 1e-5); - BOOST_REQUIRE_SMALL(rrt(edges(1, i), edges(0, i)), 1e-5); - } -} - -/** - * Create an unweighted graph laplacian from the edges. - */ -void CreateSparseGraphLaplacian(const arma::mat& edges, - arma::sp_mat& laplacian) -{ - // Get the number of vertices in the problem. - const size_t vertices = max(max(edges)) + 1; - - laplacian.zeros(vertices, vertices); - - for (size_t i = 0; i < edges.n_cols; ++i) - { - laplacian(edges(0, i), edges(1, i)) = -1.0; - laplacian(edges(1, i), edges(0, i)) = -1.0; - } - - for (size_t i = 0; i < vertices; ++i) - { - laplacian(i, i) = -arma::accu(laplacian.row(i)); - } -} - -BOOST_AUTO_TEST_CASE(ErdosRenyiRandomGraphMaxCutSDP) -{ - // Load the edges. - arma::mat edges; - data::Load("erdosrenyi-n100.csv", edges, true); - - arma::sp_mat laplacian; - CreateSparseGraphLaplacian(edges, laplacian); - - float r = 0.5 + sqrt(0.25 + 2 * edges.n_cols); - if (ceil(r) > laplacian.n_rows) - r = laplacian.n_rows; - - // initialize coordinates to a feasible point - arma::mat coordinates(laplacian.n_rows, ceil(r)); - coordinates.zeros(); - for (size_t i = 0; i < coordinates.n_rows; ++i) - { - coordinates(i, i % coordinates.n_cols) = 1.; - } - - LRSDP> maxcut(laplacian.n_rows, 0, coordinates); - maxcut.SDP().C() = laplacian; - maxcut.SDP().C() *= -1.; // need to minimize the negative - maxcut.SDP().SparseB().ones(laplacian.n_rows); - for (size_t i = 0; i < laplacian.n_rows; ++i) - { - maxcut.SDP().SparseA()[i].zeros(laplacian.n_rows, laplacian.n_rows); - maxcut.SDP().SparseA()[i](i, i) = 1.; - } - - const double finalValue = maxcut.Optimize(coordinates); - const arma::mat rrt = coordinates * trans(coordinates); - - for (size_t i = 0; i < laplacian.n_rows; ++i) - { - BOOST_REQUIRE_CLOSE(rrt(i, i), 1., 1e-5); - } - - // Final value taken by solving with Mosek - BOOST_REQUIRE_CLOSE(finalValue, -3672.7, 1e-1); -} - -/* - * Test a nuclear norm minimization SDP. - * - * Specifically, fix an unknown m x n matrix X. Our goal is to recover X from p - * measurements of X, where the i-th measurement is of the form - * - * b_i = dot(A_i, X) - * - * where the A_i's have iid entries from Normal(0, 1/p). We do this by solving - * the the following semi-definite program - * - * min ||X||_* subj to dot(A_i, X) = b_i, i=1,...,p - * - * where ||X||_* denotes the nuclear norm (sum of singular values) of X. The - * equivalent SDP is - * - * min tr(W1) + tr(W2) : [ W1, X ; X', W2 ] is PSD, - * dot(A_i, X) = b_i, i = 1, ..., p - * - * For more details on matrix sensing and nuclear norm minimization, see - * - * Guaranteed Minimum-Rank Solutions of Linear Matrix Equations via Nuclear - * Norm Minimization. - * Benjamin Recht, Maryam Fazel, Pablo Parrilo. - * SIAM Review 2010. - * - */ -BOOST_AUTO_TEST_CASE(GaussianMatrixSensingSDP) -{ - arma::mat Xorig, A; - - // read the unknown matrix X and the measurement matrices A_i in - data::Load("sensing_X.csv", Xorig, true, false); - data::Load("sensing_A.csv", A, true, false); - - const size_t m = Xorig.n_rows; - const size_t n = Xorig.n_cols; - const size_t p = A.n_rows; - assert(A.n_cols == m * m); - - arma::vec b(p); - for (size_t i = 0; i < p; ++i) - { - const arma::mat Ai = arma::reshape(A.row(i), n, m); - b(i) = arma::dot(trans(Ai), Xorig); - } - - float r = 0.5 + sqrt(0.25 + 2 * p); - if (ceil(r) > m + n) - r = m + n; - - arma::mat coordinates; - coordinates.eye(m + n, ceil(r)); - - LRSDP> sensing(0, p, coordinates, 15); - sensing.SDP().C().eye(m + n, m + n); - sensing.SDP().DenseB() = 2. * b; - - const auto blockRows = arma::span(0, m - 1); - const auto blockCols = arma::span(m, m + n - 1); - - for (size_t i = 0; i < p; ++i) - { - const arma::mat Ai = arma::reshape(A.row(i), n, m); - sensing.SDP().DenseA()[i].zeros(m + n, m + n); - sensing.SDP().DenseA()[i](blockRows, blockCols) = trans(Ai); - sensing.SDP().DenseA()[i](blockCols, blockRows) = Ai; - } - - double finalValue = sensing.Optimize(coordinates); - BOOST_REQUIRE_CLOSE(finalValue, 44.7550132629, 1e-1); - - const arma::mat rrt = coordinates * trans(coordinates); - for (size_t i = 0; i < p; ++i) - { - const arma::mat Ai = arma::reshape(A.row(i), n, m); - const double measurement = - arma::dot(trans(Ai), rrt(blockRows, blockCols)); - BOOST_REQUIRE_CLOSE(measurement, b(i), 0.05); - } - - // check matrix recovery - const double err = arma::norm(Xorig - rrt(blockRows, blockCols), "fro") / - arma::norm(Xorig, "fro"); - BOOST_REQUIRE_SMALL(err, 0.05); -} - -/** - * keller4.co test case for Lovasz-Theta LRSDP. - * This is commented out because it takes a long time to run. - * See Monteiro and Burer 2004. - * -BOOST_AUTO_TEST_CASE(Keller4LovaszThetaSDP) -{ - // Load the edges. - arma::mat edges; - data::Load("keller4.csv", edges, true); - - // The LRSDP itself and the initial point. - arma::mat coordinates; - - CreateLovaszThetaInitialPoint(edges, coordinates); - - LRSDP> lovasz(edges.n_cols, coordinates); - - SetupLovaszTheta(edges, lovasz); - - double finalValue = lovasz.Optimize(coordinates); - - // Final value taken from Monteiro + Burer 2004. - BOOST_REQUIRE_CLOSE(finalValue, -14.013, 1e-2); // Not as much precision... - // The SB method came to -14.013, but M&B's method only came to -14.005. - - // Now ensure that all the constraints are satisfied. - arma::mat rrt = coordinates * trans(coordinates); - BOOST_REQUIRE_CLOSE(trace(rrt), 1.0, 1e-5); - - // All those edge constraints... - for (size_t i = 0; i < edges.n_cols; ++i) - { - BOOST_REQUIRE_SMALL(rrt(edges(0, i), edges(1, i)), 1e-3); - BOOST_REQUIRE_SMALL(rrt(edges(1, i), edges(0, i)), 1e-3); - } -}*/ - -BOOST_AUTO_TEST_SUITE_END(); diff --git a/src/mlpack/tests/momentum_sgd_test.cpp b/src/mlpack/tests/momentum_sgd_test.cpp deleted file mode 100644 index f80146a482..0000000000 --- a/src/mlpack/tests/momentum_sgd_test.cpp +++ /dev/null @@ -1,79 +0,0 @@ -/** - * @file momentum_sgd_test.cpp - * @author Ryan Curtin - * - * Test file for MomentumSGD (stochastic gradient descent with momentum updates). - * - * mlpack is free software; you may redistribute it and/or modify it under the - * terms of the 3-clause BSD license. You should have received a copy of the - * 3-clause BSD license along with mlpack. If not, see - * http://www.opensource.org/licenses/BSD-3-Clause for more information. - */ -#include -#include -#include -#include -#include -#include - -#include -#include "test_tools.hpp" - -using namespace std; -using namespace arma; -using namespace mlpack; -using namespace mlpack::optimization; -using namespace mlpack::optimization::test; - -BOOST_AUTO_TEST_SUITE(MomentumSGDTest); - -BOOST_AUTO_TEST_CASE(MomentumSGDSpeedUpTestFunction) -{ - SGDTestFunction f; - MomentumUpdate momentumUpdate(0.7); - MomentumSGD s(0.0003, 1, 2500000, 1e-9, true, momentumUpdate); - - arma::mat coordinates = f.GetInitialPoint(); - double result = s.Optimize(f, coordinates); - - BOOST_REQUIRE_CLOSE(result, -1.0, 0.15); - BOOST_REQUIRE_SMALL(coordinates[0], 0.015); - BOOST_REQUIRE_SMALL(coordinates[1], 1e-6); - BOOST_REQUIRE_SMALL(coordinates[2], 1e-6); - - // Compare with SGD with vanilla update. - SGDTestFunction f1; - StandardSGD s1(0.0003, 1, 2500000, 1e-9, true); - - arma::mat coordinates1 = f.GetInitialPoint(); - double result1 = s1.Optimize(f1, coordinates1); - - // Result doesn't converge in 2500000 iterations. - BOOST_REQUIRE_GT(result1 + 1.0, 0.05); - BOOST_REQUIRE_GE(coordinates1[0], 0.015); - BOOST_REQUIRE_SMALL(coordinates1[1], 1e-6); - BOOST_REQUIRE_SMALL(coordinates1[2], 1e-6); - - BOOST_REQUIRE_LE(result, result1); -} - -BOOST_AUTO_TEST_CASE(GeneralizedRosenbrockTest) -{ - // Loop over several variants. - for (size_t i = 10; i < 50; i += 5) - { - // Create the generalized Rosenbrock function. - GeneralizedRosenbrockFunction f(i); - MomentumUpdate momentumUpdate(0.4); - MomentumSGD s(0.0008, 1, 0, 1e-15, true, momentumUpdate); - - arma::mat coordinates = f.GetInitialPoint(); - double result = s.Optimize(f, coordinates); - - BOOST_REQUIRE_SMALL(result, 1e-4); - for (size_t j = 0; j < i; ++j) - BOOST_REQUIRE_CLOSE(coordinates[j], (double) 1.0, 1e-3); - } -} - -BOOST_AUTO_TEST_SUITE_END(); diff --git a/src/mlpack/tests/nesterov_momentum_sgd_test.cpp b/src/mlpack/tests/nesterov_momentum_sgd_test.cpp deleted file mode 100644 index 4cb7e7bb90..0000000000 --- a/src/mlpack/tests/nesterov_momentum_sgd_test.cpp +++ /dev/null @@ -1,72 +0,0 @@ -/** - * @file nesterov_momentum_sgd_test.cpp - * @author Sourabh Varshney - * - * Test file for NesterovMomentumSGD (Stochastic gradient descent with - * nesterov momentum updates). - * - * mlpack is free software; you may redistribute it and/or modify it under the - * terms of the 3-clause BSD license. You should have received a copy of the - * 3-clause BSD license along with mlpack. If not, see - * http://www.opensource.org/licenses/BSD-3-Clause for more information. - */ -#include -#include -#include -#include -#include -#include - -#include -#include "test_tools.hpp" - -using namespace std; -using namespace arma; -using namespace mlpack; -using namespace mlpack::optimization; -using namespace mlpack::optimization::test; - -BOOST_AUTO_TEST_SUITE(NesterovMomentumSGDTest); - -/* -* Tests the Nesterov Momentum SGD update policy. -*/ -BOOST_AUTO_TEST_CASE(NesterovMomentumSGDSpeedUpTestFunction) -{ - SGDTestFunction f; - NesterovMomentumUpdate nesterovMomentumUpdate(0.9); - NesterovMomentumSGD s(0.0003, 1, 2500000, 1e-9, true, - nesterovMomentumUpdate); - - arma::mat coordinates = f.GetInitialPoint(); - double result = s.Optimize(f, coordinates); - - BOOST_REQUIRE_CLOSE(result, -1.0, 0.25); - BOOST_REQUIRE_SMALL(coordinates[0], 3e-3); - BOOST_REQUIRE_SMALL(coordinates[1], 1e-6); - BOOST_REQUIRE_SMALL(coordinates[2], 1e-6); -} - -/* -* Tests the Nesterov Momentum SGD with Generalized Rosenbrock Test. -*/ -BOOST_AUTO_TEST_CASE(GeneralizedRosenbrockTest) -{ - // Loop over several variants. - for (size_t i = 10; i < 50; i += 5) - { - // Create the generalized Rosenbrock function. - GeneralizedRosenbrockFunction f(i); - NesterovMomentumUpdate nesterovMomentumUpdate(0.9); - NesterovMomentumSGD s(0.0001, 1, 0, 1e-15, true, nesterovMomentumUpdate); - - arma::mat coordinates = f.GetInitialPoint(); - double result = s.Optimize(f, coordinates); - - BOOST_REQUIRE_SMALL(result, 1e-4); - for (size_t j = 0; j < i; ++j) - BOOST_REQUIRE_CLOSE(coordinates[j], (double) 1.0, 1e-3); - } -} - -BOOST_AUTO_TEST_SUITE_END(); diff --git a/src/mlpack/tests/parallel_sgd_test.cpp b/src/mlpack/tests/parallel_sgd_test.cpp deleted file mode 100644 index c19d0d15ca..0000000000 --- a/src/mlpack/tests/parallel_sgd_test.cpp +++ /dev/null @@ -1,127 +0,0 @@ -/** - * @file parallel_sgd_test.cpp - * @author Shikhar Bhardwaj - * - * Test file for Parallel SGD. - * - * mlpack is free software; you may redistribute it and/or modify it under the - * terms of the 3-clause BSD license. You should have received a copy of the - * 3-clause BSD license along with mlpack. If not, see - * http://www.opensource.org/licenses/BSD-3-Clause for more information. - */ -#include -#include -#include -#include -#include - -// We need some thorough testing. -#define private public -#include -#undef private - -#include -#include "test_tools.hpp" - -using namespace std; -using namespace arma; -using namespace mlpack; -using namespace mlpack::optimization; -using namespace mlpack::optimization::test; - -BOOST_AUTO_TEST_SUITE(ParallelSGDTest); - - -// These tests are only compiled if the user has specified OpenMP to be -// used. -#ifdef HAS_OPENMP - -/** - * Test the correctness of the Parallel SGD implementation using a specified - * sparse test function, with guaranteed disjoint updates between different - * threads. - */ -BOOST_AUTO_TEST_CASE(SimpleParallelSGDTest) -{ - SparseTestFunction f; - - ConstantStep decayPolicy(0.4); - - // The batch size for this test should be chosen according to the threads - // available on the system. If the update does not touch each datapoint, the - // test will fail. - - size_t threadsAvailable = omp_get_max_threads(); - - for (size_t i = threadsAvailable; i > 0; --i) - { - omp_set_num_threads(i); - - size_t batchSize = std::ceil((float) f.NumFunctions() / i); - - ParallelSGD s(10000, batchSize, 1e-5, true, decayPolicy); - - arma::mat coordinates = f.GetInitialPoint(); - double result = s.Optimize(f, coordinates); - - // The final value of the objective function should be close to the optimal - // value, that is the sum of values at the vertices of the parabolas. - BOOST_REQUIRE_CLOSE(result, 123.75, 0.01); - - // The co-ordinates should be the vertices of the parabolas. - BOOST_REQUIRE_CLOSE(coordinates[0], 2, 0.02); - BOOST_REQUIRE_CLOSE(coordinates[1], 1, 0.02); - BOOST_REQUIRE_CLOSE(coordinates[2], 1.5, 0.02); - BOOST_REQUIRE_CLOSE(coordinates[3], 4, 0.02); - } -} - -/** - * When run with a single thread, parallel SGD should be identical to normal - * SGD. - */ -BOOST_AUTO_TEST_CASE(GeneralizedRosenbrockTest) -{ - // Loop over several variants. - for (size_t i = 10; i < 50; i += 5) - { - // Create the generalized Rosenbrock function. - GeneralizedRosenbrockFunction f(i); - - ConstantStep decayPolicy(0.001); - - ParallelSGD s(0, f.NumFunctions(), 1e-12, true, decayPolicy); - - arma::mat coordinates = f.GetInitialPoint(); - - omp_set_num_threads(1); - double result = s.Optimize(f, coordinates); - - BOOST_REQUIRE_SMALL(result, 1e-8); - for (size_t j = 0; j < i; ++j) - BOOST_REQUIRE_CLOSE(coordinates[j], (double) 1.0, 0.01); - } -} - -#endif - -/** - * Test the correctness of the Exponential backoff stepsize decay policy. - */ -BOOST_AUTO_TEST_CASE(ExponentialBackoffDecayTest) -{ - ExponentialBackoff decayPolicy(100, 100, 0.9); - - // At the first iteration, stepsize should be unchanged - BOOST_REQUIRE_EQUAL(decayPolicy.StepSize(1), 100); - // At the 99th iteration, stepsize should be unchanged - BOOST_REQUIRE_EQUAL(decayPolicy.StepSize(99), 100); - // At the 100th iteration, stepsize should be changed - BOOST_REQUIRE_EQUAL(decayPolicy.StepSize(100), 90); - // At the 210th iteration, stepsize should be unchanged - BOOST_REQUIRE_EQUAL(decayPolicy.StepSize(210), 90); - // At the 211th iteration, stepsize should be changed - BOOST_REQUIRE_EQUAL(decayPolicy.StepSize(211), 81); -} - -BOOST_AUTO_TEST_SUITE_END(); diff --git a/src/mlpack/tests/proximal_test.cpp b/src/mlpack/tests/proximal_test.cpp deleted file mode 100644 index 42ee27fa74..0000000000 --- a/src/mlpack/tests/proximal_test.cpp +++ /dev/null @@ -1,91 +0,0 @@ -/** - * @file proximal_test.cpp - * @author Chenzhe Diao - * - * Test file for proximal optimizer. - * - * mlpack is free software; you may redistribute it and/or modify it under the - * terms of the 3-clause BSD license. You should have received a copy of the - * 3-clause BSD license along with mlpack. If not, see - * http://www.opensource.org/licenses/BSD-3-Clause for more information. - */ -#include -#include - -#include -#include "test_tools.hpp" - -using namespace std; -using namespace arma; -using namespace mlpack; -using namespace mlpack::optimization; - -BOOST_AUTO_TEST_SUITE(ProximalTest); - -/** - * Approximate vector using a vector with l1 norm small than or equal to tau. - */ -BOOST_AUTO_TEST_CASE(ProjectToL1) -{ - int D = 100; // Dimension of the problem. - - // Norm of L1 ball. - double tau1 = 1.5; - double tau2 = 0.5; - - // Vector to be projected, with unit l1 norm. - vec v = randu(D); - v = normalise(v, 1); - - // v is inside the l1 ball, so the projection will not change v. - vec v1 = v; - Proximal::ProjectToL1Ball(v1, tau1); - BOOST_REQUIRE_SMALL(norm(v - v1, 2), 1e-10); - - // v is outside the l1 ball, so the projection should find the closest. - vec v2 = v; - Proximal::ProjectToL1Ball(v2, tau2); - double distance = norm(v2 - v, 2); - for (size_t i = 1; i < 1000; i++) - { - // Randomly generate a vector on the surface of the l1 ball with norm tau2. - vec vSurface = randu(D); - vSurface = tau2 * normalise(vSurface, 1); - - double distanceNew = norm(vSurface - v, 2); - - BOOST_REQUIRE_GE(distanceNew, distance); - } -} - -/** - * Approximate a vector with a tau-sparse vector. - */ -BOOST_AUTO_TEST_CASE(ProjectToL0) -{ - int D = 100; // Dimension of the problem. - int tau = 25; // Sparsity requirement. - - // Vector to be projected. - vec v = randn(D); - - vec v0 = v; - Proximal::ProjectToL0Ball(v0, tau); - double distance = norm(v0 - v, 2); - - for (size_t i = 1; i < 1000; i++) - { - // Randomly find a subset of the support of v, generate a tau-sparse - // vector by restricting v to this support. - uvec indices = linspace(0, D - 1, D); - indices = shuffle(indices); - indices = indices.head(tau); - vec vNew = zeros(D); - vNew.elem(indices) = v.elem(indices); - - double distanceNew = norm(v - vNew, 2); - BOOST_REQUIRE_GE(distanceNew, distance); - } -} - -BOOST_AUTO_TEST_SUITE_END(); diff --git a/src/mlpack/tests/rmsprop_test.cpp b/src/mlpack/tests/rmsprop_test.cpp deleted file mode 100644 index 7135d06c28..0000000000 --- a/src/mlpack/tests/rmsprop_test.cpp +++ /dev/null @@ -1,106 +0,0 @@ -/** - * @file rmsprop_test.cpp - * @author Marcus Edel - * - * Tests the RMSProp optimizer. - * - * mlpack is free software; you may redistribute it and/or modify it under the - * terms of the 3-clause BSD license. You should have received a copy of the - * 3-clause BSD license along with mlpack. If not, see - * http://www.opensource.org/licenses/BSD-3-Clause for more information. - */ -#include - -#include -#include - -#include - -#include -#include "test_tools.hpp" - -using namespace arma; -using namespace mlpack; -using namespace mlpack::optimization; -using namespace mlpack::optimization::test; - -using namespace mlpack::distribution; -using namespace mlpack::regression; - -BOOST_AUTO_TEST_SUITE(RMSPropTest); - -/** - * Tests the RMSProp optimizer using a simple test function. - */ -BOOST_AUTO_TEST_CASE(SimpleRMSPropTestFunction) -{ - SGDTestFunction f; - RMSProp optimizer(1e-3, 1, 0.99, 1e-8, 5000000, 1e-9, true); - - arma::mat coordinates = f.GetInitialPoint(); - optimizer.Optimize(f, coordinates); - - BOOST_REQUIRE_SMALL(coordinates[0], 0.1); - BOOST_REQUIRE_SMALL(coordinates[1], 0.1); - BOOST_REQUIRE_SMALL(coordinates[2], 0.1); -} - -/** - * Run RMSProp on logistic regression and make sure the results are acceptable. - */ -BOOST_AUTO_TEST_CASE(LogisticRegressionTest) -{ - // Generate a two-Gaussian dataset. - GaussianDistribution g1(arma::vec("1.0 1.0 1.0"), arma::eye(3, 3)); - GaussianDistribution g2(arma::vec("9.0 9.0 9.0"), arma::eye(3, 3)); - - arma::mat data(3, 1000); - arma::Row responses(1000); - for (size_t i = 0; i < 500; ++i) - { - data.col(i) = g1.Random(); - responses[i] = 0; - } - for (size_t i = 500; i < 1000; ++i) - { - data.col(i) = g2.Random(); - responses[i] = 1; - } - - // Shuffle the dataset. - arma::uvec indices = arma::shuffle(arma::linspace(0, - data.n_cols - 1, data.n_cols)); - arma::mat shuffledData(3, 1000); - arma::Row shuffledResponses(1000); - for (size_t i = 0; i < data.n_cols; ++i) - { - shuffledData.col(i) = data.col(indices[i]); - shuffledResponses[i] = responses[indices[i]]; - } - - // Create a test set. - arma::mat testData(3, 1000); - arma::Row testResponses(1000); - for (size_t i = 0; i < 500; ++i) - { - testData.col(i) = g1.Random(); - testResponses[i] = 0; - } - for (size_t i = 500; i < 1000; ++i) - { - testData.col(i) = g2.Random(); - testResponses[i] = 1; - } - - RMSProp rmsprop; - LogisticRegression<> lr(shuffledData, shuffledResponses, rmsprop, 0.5); - - // Ensure that the error is close to zero. - const double acc = lr.ComputeAccuracy(data, responses); - BOOST_REQUIRE_CLOSE(acc, 100.0, 0.3); // 0.3% error tolerance. - - const double testAcc = lr.ComputeAccuracy(testData, testResponses); - BOOST_REQUIRE_CLOSE(testAcc, 100.0, 0.6); // 0.6% error tolerance. -} - -BOOST_AUTO_TEST_SUITE_END(); diff --git a/src/mlpack/tests/sa_test.cpp b/src/mlpack/tests/sa_test.cpp deleted file mode 100644 index 9f964166c0..0000000000 --- a/src/mlpack/tests/sa_test.cpp +++ /dev/null @@ -1,113 +0,0 @@ -/* - * @file sa_test.cpp - * @auther Zhihao Lou - * - * Test file for SA (simulated annealing). - * - * mlpack is free software; you may redistribute it and/or modify it under the - * terms of the 3-clause BSD license. You should have received a copy of the - * 3-clause BSD license along with mlpack. If not, see - * http://www.opensource.org/licenses/BSD-3-Clause for more information. - */ -#include -#include -#include -#include -#include -#include - -#include -#include -#include - -#include -#include "test_tools.hpp" - -using namespace std; -using namespace arma; -using namespace mlpack; -using namespace mlpack::optimization; -using namespace mlpack::optimization::test; -using namespace mlpack::metric; - -BOOST_AUTO_TEST_SUITE(SATest); - -// The Generalized-Rosenbrock function is a simple function to optimize. -BOOST_AUTO_TEST_CASE(GeneralizedRosenbrockTest) -{ - size_t dim = 10; - GeneralizedRosenbrockFunction f(dim); - - double iteration = 0; - double result = DBL_MAX; - arma::mat coordinates; - while (result > 1e-6) - { - ExponentialSchedule schedule; - // The convergence is very sensitive to the choices of maxMove and initMove. - SA sa(schedule, 1000000, 1000., 1000, 100, 1e-10, 3, - 1.5, 0.5, 0.3); - coordinates = f.GetInitialPoint(); - result = sa.Optimize(f, coordinates); - ++iteration; - - BOOST_REQUIRE_LT(iteration, 4); // No more than three tries. - } - - // 0.1% tolerance for each coordinate. - BOOST_REQUIRE_SMALL(result, 1e-6); - for (size_t j = 0; j < dim; ++j) - BOOST_REQUIRE_CLOSE(coordinates[j], (double) 1.0, 0.1); -} - -// The Rosenbrock function is a simple function to optimize. -BOOST_AUTO_TEST_CASE(RosenbrockTest) -{ - RosenbrockFunction f; - ExponentialSchedule schedule; - // The convergence is very sensitive to the choices of maxMove and initMove. - SA<> sa(schedule, 1000000, 1000., 1000, 100, 1e-11, 3, 1.5, 0.3, 0.3); - arma::mat coordinates = f.GetInitialPoint(); - - const double result = sa.Optimize(f, coordinates); - - BOOST_REQUIRE_SMALL(result, 1e-5); - BOOST_REQUIRE_CLOSE(coordinates[0], 1.0, 1e-2); - BOOST_REQUIRE_CLOSE(coordinates[1], 1.0, 1e-2); -} - -/** - * The Rastrigrin function, a (not very) simple nonconvex function. It has very - * many local minima, so finding the true global minimum is difficult. - */ -BOOST_AUTO_TEST_CASE(RastrigrinFunctionTest) -{ - // Simulated annealing isn't guaranteed to converge (except in very specific - // situations). If this works 1 of 4 times, I'm fine with that. All I want - // to know is that this implementation will escape from local minima. - size_t successes = 0; - - for (size_t trial = 0; trial < 4; ++trial) - { - RastriginFunction f(2); - ExponentialSchedule schedule; - // The convergence is very sensitive to the choices of maxMove and initMove. - // SA<> sa(schedule, 2000000, 100, 50, 1000, 1e-12, 2, 2.0, 0.5, 0.1); - SA<> sa(schedule, 2000000, 100, 50, 1000, 1e-12, 2, 2.0, 0.5, 0.1); - arma::mat coordinates = f.GetInitialPoint(); - - const double result = sa.Optimize(f, coordinates); - - if ((std::abs(result) < 1e-3) && - (std::abs(coordinates[0]) < 1e-3) && - (std::abs(coordinates[1]) < 1e-3)) - { - ++successes; - break; // No need to continue. - } - } - - BOOST_REQUIRE_GE(successes, 1); -} - -BOOST_AUTO_TEST_SUITE_END(); diff --git a/src/mlpack/tests/sarah_test.cpp b/src/mlpack/tests/sarah_test.cpp deleted file mode 100644 index d164cecdcb..0000000000 --- a/src/mlpack/tests/sarah_test.cpp +++ /dev/null @@ -1,78 +0,0 @@ -/** - * @file sarah_test.cpp - * @author Marcus Edel - * - * Test file for SARAH and SARAH+. - * - * mlpack is free software; you may redistribute it and/or modify it under the - * terms of the 3-clause BSD license. You should have received a copy of the - * 3-clause BSD license along with mlpack. If not, see - * http://www.opensource.org/licenses/BSD-3-Clause for more information. - */ -#include -#include - -#include -#include "test_tools.hpp" -#include "test_function_tools.hpp" - -using namespace mlpack; -using namespace mlpack::optimization; - -BOOST_AUTO_TEST_SUITE(SARAHTest); - -/** - * Run SARAH on logistic regression and make sure the results are - * acceptable. - */ -BOOST_AUTO_TEST_CASE(SAHRALogisticRegressionTest) -{ - arma::mat data, testData, shuffledData; - arma::Row responses, testResponses, shuffledResponses; - - LogisticRegressionTestData(data, testData, shuffledData, - responses, testResponses, shuffledResponses); - - // Now run big-batch SGD with a couple of batch sizes. - for (size_t batchSize = 35; batchSize < 45; batchSize += 5) - { - SARAH optimizer(0.01, batchSize, 250, 0, 1e-5, true); - LogisticRegression<> lr(shuffledData, shuffledResponses, optimizer, 0.5); - - // Ensure that the error is close to zero. - const double acc = lr.ComputeAccuracy(data, responses); - BOOST_REQUIRE_CLOSE(acc, 100.0, 1.5); // 1.5% error tolerance. - - const double testAcc = lr.ComputeAccuracy(testData, testResponses); - BOOST_REQUIRE_CLOSE(testAcc, 100.0, 1.5); // 1.5% error tolerance. - } -} - -/** - * Run SARAH_Plus on logistic regression and make sure the results are - * acceptable. - */ -BOOST_AUTO_TEST_CASE(SAHRAPlusLogisticRegressionTest) -{ - arma::mat data, testData, shuffledData; - arma::Row responses, testResponses, shuffledResponses; - - LogisticRegressionTestData(data, testData, shuffledData, - responses, testResponses, shuffledResponses); - - // Now run big-batch SGD with a couple of batch sizes. - for (size_t batchSize = 35; batchSize < 45; batchSize += 5) - { - SARAH_Plus optimizer(0.01, batchSize, 250, 0, 1e-5, true); - LogisticRegression<> lr(shuffledData, shuffledResponses, optimizer, 0.5); - - // Ensure that the error is close to zero. - const double acc = lr.ComputeAccuracy(data, responses); - BOOST_REQUIRE_CLOSE(acc, 100.0, 1.5); // 1.5% error tolerance. - - const double testAcc = lr.ComputeAccuracy(testData, testResponses); - BOOST_REQUIRE_CLOSE(testAcc, 100.0, 1.5); // 1.5% error tolerance. - } -} - -BOOST_AUTO_TEST_SUITE_END(); diff --git a/src/mlpack/tests/scd_test.cpp b/src/mlpack/tests/scd_test.cpp deleted file mode 100644 index 02c1ca82e7..0000000000 --- a/src/mlpack/tests/scd_test.cpp +++ /dev/null @@ -1,218 +0,0 @@ -/** - * @file scd_test.cpp - * @author Shikhar Bhardwaj - * - * Test file for SCD (stochastic coordinate descent). - * - * mlpack is free software; you may redistribute it and/or modify it under the - * terms of the 3-clause BSD license. You should have received a copy of the - * 3-clause BSD license along with mlpack. If not, see - * http://www.opensource.org/licenses/BSD-3-Clause for more information. - */ -#include -#include -#include -#include -#include -#include -#include - -#include -#include "test_tools.hpp" - -using namespace std; -using namespace mlpack; -using namespace mlpack::math; -using namespace mlpack::optimization; -using namespace mlpack::optimization::test; -using namespace mlpack::regression; - -BOOST_AUTO_TEST_SUITE(SCDTest); - -/** - * Test the correctness of the SCD implementation by using a dataset with a - * precalculated minima. - */ -BOOST_AUTO_TEST_CASE(PreCalcSCDTest) -{ - arma::mat predictors("0 0 0.4; 0 0 0.6; 0 0.3 0; 0.2 0 0; 0.2 -0.5 0;"); - arma::Row responses("1 1 0;"); - - LogisticRegressionFunction f(predictors, responses, 0.0001); - - SCD<> s(0.02, 60000, 1e-5); - arma::mat iterate = f.InitialPoint(); - - double objective = s.Optimize(f, iterate); - - BOOST_REQUIRE_LE(objective, 0.055); -} - -/** - * Test the correctness of the SCD implemenation by using the sparse test - * function, with dijoint features which optimize to a precalculated minima. - */ -BOOST_AUTO_TEST_CASE(DisjointFeatureTest) -{ - // The test function for parallel SGD should work with SCD, as the gradients - // of the individual functions are projections into the ith dimension. - SparseTestFunction f; - SCD<> s(0.4); - - arma::mat iterate = f.GetInitialPoint(); - - double result = s.Optimize(f, iterate); - - // The final value of the objective function should be close to the optimal - // value, that is the sum of values at the vertices of the parabolas. - BOOST_REQUIRE_CLOSE(result, 123.75, 0.01); - - // The co-ordinates should be the vertices of the parabolas. - BOOST_REQUIRE_CLOSE(iterate[0], 2, 0.02); - BOOST_REQUIRE_CLOSE(iterate[1], 1, 0.02); - BOOST_REQUIRE_CLOSE(iterate[2], 1.5, 0.02); - BOOST_REQUIRE_CLOSE(iterate[3], 4, 0.02); -} - -/** - * Test the greedy descent policy. - */ -BOOST_AUTO_TEST_CASE(GreedyDescentTest) -{ - // In the sparse test function, the given point has the maximum gradient at - // the feature with index 2. - arma::mat point("1; 2; 3; 4;"); - - SparseTestFunction f; - - GreedyDescent descentPolicy; - - BOOST_REQUIRE_EQUAL(descentPolicy.DescentFeature(0, point, f), 2); - - // Changing the point under consideration, so that the maximum gradient is at - // index 1. - point[1] = 10; - - BOOST_REQUIRE_EQUAL(descentPolicy.DescentFeature(0, point, f), 1); -} - -/** - * Test the cyclic descent policy. - */ -BOOST_AUTO_TEST_CASE(CyclicDescentTest) -{ - const size_t features = 10; - struct DummyFunction - { - static size_t NumFeatures() - { - return features; - } - }; - - DummyFunction dummy; - - CyclicDescent descentPolicy; - - for (size_t i = 0; i < 15; ++i) - { - BOOST_REQUIRE_EQUAL(descentPolicy.DescentFeature(i, arma::mat(), dummy), i % - features); - } -} - -/** - * Test the random descent policy. - */ -BOOST_AUTO_TEST_CASE(RandomDescentTest) -{ - const size_t features = 10; - struct DummyFunction - { - static size_t NumFeatures() - { - return features; - } - }; - - DummyFunction dummy; - - CyclicDescent descentPolicy; - - for (size_t i = 0; i < 100; ++i) - { - size_t j = descentPolicy.DescentFeature(i, arma::mat(), dummy); - BOOST_REQUIRE_LT(j, features); - BOOST_REQUIRE_GE(j, 0); - } -} - -/** - * Test that LogisticRegressionFunction::PartialGradient() works as expected. - */ -BOOST_AUTO_TEST_CASE(LogisticRegressionFunctionPartialGradientTest) -{ - // Evaluate the gradient and feature gradient and equate. - arma::mat predictors("0 0 0.4; 0 0 0.6; 0 0.3 0; 0.2 0 0; 0.2 -0.5 0;"); - arma::Row responses("1 1 0;"); - - LogisticRegressionFunction f(predictors, responses, 0.0001); - - arma::mat testPoint(1, f.NumFeatures(), arma::fill::randu); - - arma::mat testGradient; - - f.Gradient(testPoint, testGradient); - - for (size_t i = 0; i < f.NumFeatures(); ++i) - { - arma::sp_mat fGrad; - f.PartialGradient(testPoint, i, fGrad); - - CheckMatrices(testGradient.col(i), arma::mat(fGrad.col(i))); - } -} - -/** - * Test that SoftmaxRegressionFunction::PartialGradient() works as expected. - */ -BOOST_AUTO_TEST_CASE(SoftmaxRegressionFunctionPartialGradientTest) -{ - const size_t points = 1000; - const size_t inputSize = 10; - const size_t numClasses = 5; - - // Initialize a random dataset. - arma::mat data; - data.randu(inputSize, points); - - // Create random class labels. - arma::Row labels(points); - for (size_t i = 0; i < points; i++) - labels(i) = RandInt(0, numClasses); - - // 2 objects for 2 terms in the cost function. Each term contributes towards - // the gradient and thus need to be checked independently. - SoftmaxRegressionFunction srf(data, labels, numClasses, 0); - - // Create a random set of parameters. - arma::mat parameters; - parameters.randu(numClasses, inputSize); - - // Get gradients for the current parameters. - arma::mat gradient; - srf.Gradient(parameters, gradient); - - // For each parameter. - for (size_t j = 0; j < inputSize; j++) - { - // Get the gradient for this feature. - arma::sp_mat fGrad; - - srf.PartialGradient(parameters, j, fGrad); - - CheckMatrices(gradient.col(j), arma::mat(fGrad.col(j))); - } -} - -BOOST_AUTO_TEST_SUITE_END(); diff --git a/src/mlpack/tests/sdp_primal_dual_test.cpp b/src/mlpack/tests/sdp_primal_dual_test.cpp deleted file mode 100644 index 3d5a0cc9f9..0000000000 --- a/src/mlpack/tests/sdp_primal_dual_test.cpp +++ /dev/null @@ -1,648 +0,0 @@ -/** - * @file sdp_primal_dual_test.cpp - * @author Stephen Tu - * - * - * mlpack is free software; you may redistribute it and/or modify it under the - * terms of the 3-clause BSD license. You should have received a copy of the - * 3-clause BSD license along with mlpack. If not, see - * http://www.opensource.org/licenses/BSD-3-Clause for more information. - */ -#include -#include -#include -#include - -#include -#include "test_tools.hpp" - -using namespace mlpack; -using namespace mlpack::optimization; -using namespace mlpack::distribution; -using namespace mlpack::neighbor; - -class UndirectedGraph -{ - public: - UndirectedGraph() : numVertices(0) { } - - size_t NumVertices() const { return numVertices; } - size_t NumEdges() const { return edges.n_cols; } - - const arma::umat& Edges() const { return edges; } - const arma::vec& Weights() const { return weights; } - - void Laplacian(arma::sp_mat& laplacian) const - { - laplacian.zeros(numVertices, numVertices); - - for (size_t i = 0; i < edges.n_cols; ++i) - { - laplacian(edges(0, i), edges(1, i)) = -weights(i); - laplacian(edges(1, i), edges(0, i)) = -weights(i); - } - - for (size_t i = 0; i < numVertices; ++i) - { - laplacian(i, i) = -arma::accu(laplacian.row(i)); - } - } - - static void LoadFromEdges(UndirectedGraph& g, - const std::string& edgesFilename, - bool transposeEdges) - { - data::Load(edgesFilename, g.edges, true, transposeEdges); - if (g.edges.n_rows != 2) - Log::Fatal << "Invalid datafile" << std::endl; - g.weights.ones(g.edges.n_cols); - g.ComputeVertices(); - } - - static void LoadFromEdgesAndWeights(UndirectedGraph& g, - const std::string& edgesFilename, - bool transposeEdges, - const std::string& weightsFilename, - bool transposeWeights) - { - data::Load(edgesFilename, g.edges, true, transposeEdges); - if (g.edges.n_rows != 2) - Log::Fatal << "Invalid datafile" << std::endl; - data::Load(weightsFilename, g.weights, true, transposeWeights); - if (g.weights.n_elem != g.edges.n_cols) - Log::Fatal << "Size mismatch" << std::endl; - g.ComputeVertices(); - } - - static void ErdosRenyiRandomGraph(UndirectedGraph& g, - size_t numVertices, - double edgeProbability, - bool weighted, - bool selfLoops = false) - { - if (edgeProbability < 0. || edgeProbability > 1.) - Log::Fatal << "edgeProbability not in [0, 1]" << std::endl; - - std::vector> edges; - std::vector weights; - - for (size_t i = 0; i < numVertices; i ++) - { - for (size_t j = (selfLoops ? i : i + 1); j < numVertices; j++) - { - if (math::Random() > edgeProbability) - continue; - edges.emplace_back(i, j); - weights.push_back(weighted ? math::Random() : 1.); - } - } - - g.edges.set_size(2, edges.size()); - for (size_t i = 0; i < edges.size(); i++) - { - g.edges(0, i) = edges[i].first; - g.edges(1, i) = edges[i].second; - } - g.weights = arma::vec(weights); - - g.numVertices = numVertices; - } - - private: - void ComputeVertices() - { - numVertices = max(max(edges)) + 1; - } - - arma::umat edges; - arma::vec weights; - size_t numVertices; -}; - -static inline SDP -ConstructMaxCutSDPFromGraph(const UndirectedGraph& g) -{ - SDP sdp(g.NumVertices(), g.NumVertices(), 0); - g.Laplacian(sdp.C()); - sdp.C() *= -1; - for (size_t i = 0; i < g.NumVertices(); i++) - { - sdp.SparseA()[i].zeros(g.NumVertices(), g.NumVertices()); - sdp.SparseA()[i](i, i) = 1.; - } - sdp.SparseB().ones(); - return sdp; -} - -static inline SDP -ConstructLovaszThetaSDPFromGraph(const UndirectedGraph& g) -{ - SDP sdp(g.NumVertices(), g.NumEdges() + 1, 0); - sdp.C().ones(); - sdp.C() *= -1.; - sdp.SparseA()[0].eye(g.NumVertices(), g.NumVertices()); - for (size_t i = 0; i < g.NumEdges(); i++) - { - sdp.SparseA()[i + 1].zeros(g.NumVertices(), g.NumVertices()); - sdp.SparseA()[i + 1](g.Edges()(0, i), g.Edges()(1, i)) = 1.; - sdp.SparseA()[i + 1](g.Edges()(1, i), g.Edges()(0, i)) = 1.; - } - sdp.SparseB().zeros(); - sdp.SparseB()[0] = 1.; - return sdp; -} - -static inline SDP -ConstructMaxCutSDPFromLaplacian(const std::string& laplacianFilename) -{ - arma::mat laplacian; - data::Load(laplacianFilename, laplacian, true, false); - if (laplacian.n_rows != laplacian.n_cols) - Log::Fatal << "laplacian not square" << std::endl; - SDP sdp(laplacian.n_rows, laplacian.n_rows, 0); - sdp.C() = -arma::sp_mat(laplacian); - for (size_t i = 0; i < laplacian.n_rows; i++) - { - sdp.SparseA()[i].zeros(laplacian.n_rows, laplacian.n_rows); - sdp.SparseA()[i](i, i) = 1.; - } - sdp.SparseB().ones(); - return sdp; -} - -static bool CheckPositiveSemiDefinite(const arma::mat& X) -{ - const auto evals = arma::eig_sym(X); - return (evals(0) > 1e-20); -} - -template -static bool CheckKKT(const SDPType& sdp, - const arma::mat& X, - const arma::vec& ysparse, - const arma::vec& ydense, - const arma::mat& Z) -{ - // Require that the KKT optimality conditions for sdp are satisfied - // by the primal-dual pair (X, y, Z). - - if (!CheckPositiveSemiDefinite(X)) - return false; - if (!CheckPositiveSemiDefinite(Z)) - return false; - - bool success = true; - const double normXz = arma::norm(X * Z, "fro"); - success &= (std::abs(normXz) < 1e-5); - - for (size_t i = 0; i < sdp.NumSparseConstraints(); i++) - { - success &= (std::abs( - arma::dot(sdp.SparseA()[i], X) - sdp.SparseB()[i]) < 1e-5); - } - - for (size_t i = 0; i < sdp.NumDenseConstraints(); i++) - { - success &= (std::abs( - arma::dot(sdp.DenseA()[i], X) - sdp.DenseB()[i]) < 1e-5); - } - - arma::mat dualCheck = Z - sdp.C(); - for (size_t i = 0; i < sdp.NumSparseConstraints(); i++) - dualCheck += ysparse(i) * sdp.SparseA()[i]; - for (size_t i = 0; i < sdp.NumDenseConstraints(); i++) - dualCheck += ydense(i) * sdp.DenseA()[i]; - const double dualInfeas = arma::norm(dualCheck, "fro"); - success &= (dualInfeas < 1e-5); - - return success; -} - -BOOST_AUTO_TEST_SUITE(SdpPrimalDualTest); - -static void SolveMaxCutFeasibleSDP(const SDP& sdp) -{ - arma::mat X0, Z0; - arma::vec ysparse0, ydense0; - ydense0.set_size(0); - - // strictly feasible starting point - X0.eye(sdp.N(), sdp.N()); - ysparse0 = -1.1 * arma::vec(arma::sum(arma::abs(sdp.C()), 0).t()); - Z0 = -arma::diagmat(ysparse0) + sdp.C(); - - PrimalDualSolver> solver(sdp, X0, ysparse0, ydense0, Z0); - - arma::mat X, Z; - arma::vec ysparse, ydense; - solver.Optimize(X, ysparse, ydense, Z); - CheckKKT(sdp, X, ysparse, ydense, Z); -} - -static void SolveMaxCutPositiveSDP(const SDP& sdp) -{ - arma::mat X0, Z0; - arma::vec ysparse0, ydense0; - ydense0.set_size(0); - - // infeasible, but positive starting point - X0 = arma::eye(sdp.N(), sdp.N()); - ysparse0 = arma::randu(sdp.NumSparseConstraints()); - Z0.eye(sdp.N(), sdp.N()); - - PrimalDualSolver> solver(sdp, X0, ysparse0, ydense0, Z0); - - arma::mat X, Z; - arma::vec ysparse, ydense; - solver.Optimize(X, ysparse, ydense, Z); - CheckKKT(sdp, X, ysparse, ydense, Z); -} - -BOOST_AUTO_TEST_CASE(SmallMaxCutSdp) -{ - auto sdp = ConstructMaxCutSDPFromLaplacian("r10.txt"); - SolveMaxCutFeasibleSDP(sdp); - SolveMaxCutPositiveSDP(sdp); - - UndirectedGraph g; - UndirectedGraph::ErdosRenyiRandomGraph(g, 10, 0.3, true); - sdp = ConstructMaxCutSDPFromGraph(g); - - // the following was resulting in non-positive Z0 matrices on some - // random instances. - // SolveMaxCutFeasibleSDP(sdp); - - SolveMaxCutPositiveSDP(sdp); -} - -BOOST_AUTO_TEST_CASE(SmallLovaszThetaSdp) -{ - UndirectedGraph g; - UndirectedGraph::LoadFromEdges(g, "johnson8-4-4.csv", true); - auto sdp = ConstructLovaszThetaSDPFromGraph(g); - - PrimalDualSolver> solver(sdp); - - arma::mat X, Z; - arma::vec ysparse, ydense; - solver.Optimize(X, ysparse, ydense, Z); - CheckKKT(sdp, X, ysparse, ydense, Z); -} - -static inline arma::sp_mat -RepeatBlockDiag(const arma::sp_mat& block, size_t repeat) -{ - assert(block.n_rows == block.n_cols); - arma::sp_mat ret(block.n_rows * repeat, block.n_rows * repeat); - ret.zeros(); - for (size_t i = 0; i < repeat; i++) - ret(arma::span(i * block.n_rows, (i + 1) * block.n_rows - 1), - arma::span(i * block.n_rows, (i + 1) * block.n_rows - 1)) = block; - return ret; -} - -static inline arma::sp_mat -BlockDiag(const std::vector& blocks) -{ - // assumes all blocks are the same size - const size_t n = blocks.front().n_rows; - assert(blocks.front().n_cols == n); - arma::sp_mat ret(n * blocks.size(), n * blocks.size()); - ret.zeros(); - for (size_t i = 0; i < blocks.size(); i++) - ret(arma::span(i * n, (i + 1) * n - 1), - arma::span(i * n, (i + 1) * n - 1)) = blocks[i]; - return ret; -} - -static inline SDP -ConstructLogChebychevApproxSdp(const arma::mat& A, const arma::vec& b) -{ - if (A.n_rows != b.n_elem) - Log::Fatal << "A.n_rows != len(b)" << std::endl; - const size_t p = A.n_rows; - const size_t k = A.n_cols; - - // [0, 0, 0] - // [0, 0, 1] - // [0, 1, 0] - arma::sp_mat cblock(3, 3); - cblock(1, 2) = cblock(2, 1) = 1.; - const arma::sp_mat C = RepeatBlockDiag(cblock, p); - - SDP sdp(C.n_rows, k + 1, 0); - sdp.C() = C; - sdp.SparseB().zeros(); - sdp.SparseB()[0] = -1; - - // [1, 0, 0] - // [0, 0, 0] - // [0, 0, 1] - arma::sp_mat a0block(3, 3); - a0block(0, 0) = a0block(2, 2) = 1.; - sdp.SparseA()[0] = RepeatBlockDiag(a0block, p); - sdp.SparseA()[0] *= -1.; - - for (size_t i = 0; i < k; i++) - { - std::vector blocks; - for (size_t j = 0; j < p; j++) - { - arma::sp_mat block(3, 3); - const double f = A(j, i) / b(j); - // [ -a_j(i)/b_j 0 0 ] - // [ 0 a_j(i)/b_j 0 ] - // [ 0 0 0 ] - block(0, 0) = -f; - block(1, 1) = f; - blocks.emplace_back(block); - } - sdp.SparseA()[i + 1] = BlockDiag(blocks); - sdp.SparseA()[i + 1] *= -1; - } - - return sdp; -} - -static inline arma::mat -RandomOrthogonalMatrix(size_t rows, size_t cols) -{ - arma::mat Q, R; - if (!arma::qr(Q, R, arma::randu(rows, cols))) - Log::Fatal << "could not compute QR decomposition" << std::endl; - return Q; -} - -static inline arma::mat -RandomFullRowRankMatrix(size_t rows, size_t cols) -{ - const arma::mat U = RandomOrthogonalMatrix(rows, rows); - const arma::mat V = RandomOrthogonalMatrix(cols, cols); - arma::mat S; - S.zeros(rows, cols); - for (size_t i = 0; i < std::min(rows, cols); i++) - { - S(i, i) = math::Random() + 1e-3; - } - return U * S * V; -} - -/** - * See the examples section, Eq. 9, of - * - * Semidefinite Programming. - * Lieven Vandenberghe and Stephen Boyd. - * SIAM Review. 1996. - * - * The logarithmic Chebychev approximation to Ax = b, A is p x k and b is - * length p is given by the SDP: - * - * min t - * s.t. - * [ t - dot(a_i, x) 0 0 ] - * [ 0 dot(a_i, x) / b_i 1 ] >= 0, i=1,...,p - * [ 0 1 t ] - * - */ -BOOST_AUTO_TEST_CASE(LogChebychevApproxSdp) -{ - // Sometimes, the optimization can fail randomly, so we will run the test - // three times and make sure it succeeds at least once. - bool success = false; - for (size_t i = 0; i < 3; ++i) - { - const size_t p0 = 5; - const size_t k0 = 10; - const arma::mat A0 = RandomFullRowRankMatrix(p0, k0); - const arma::vec b0 = arma::randu(p0); - const auto sdp0 = ConstructLogChebychevApproxSdp(A0, b0); - PrimalDualSolver> solver0(sdp0); - arma::mat X0, Z0; - arma::vec ysparse0, ydense0; - solver0.Optimize(X0, ysparse0, ydense0, Z0); - success = CheckKKT(sdp0, X0, ysparse0, ydense0, Z0); - if (success) - break; - } - - BOOST_REQUIRE_EQUAL(success, true); - - success = false; - for (size_t i = 0; i < 3; ++i) - { - const size_t p1 = 10; - const size_t k1 = 5; - const arma::mat A1 = RandomFullRowRankMatrix(p1, k1); - const arma::vec b1 = arma::randu(p1); - const auto sdp1 = ConstructLogChebychevApproxSdp(A1, b1); - PrimalDualSolver> solver1(sdp1); - arma::mat X1, Z1; - arma::vec ysparse1, ydense1; - solver1.Optimize(X1, ysparse1, ydense1, Z1); - success = CheckKKT(sdp1, X1, ysparse1, ydense1, Z1); - if (success) - break; - } - - BOOST_REQUIRE_EQUAL(success, true); -} - -/** - * Example 1 on the SDP wiki - * - * min x_13 - * s.t. - * -0.2 <= x_12 <= -0.1 - * 0.4 <= x_23 <= 0.5 - * x_11 = x_22 = x_33 = 1 - * X >= 0 - * - */ -BOOST_AUTO_TEST_CASE(CorrelationCoeffToySdp) -{ - // The semi-definite constraint looks like: - // - // [ 1 x_12 x_13 0 0 0 0 ] - // [ 1 x_23 0 0 0 0 ] - // [ 1 0 0 0 0 ] - // [ s1 0 0 0 ] >= 0 - // [ s2 0 0 ] - // [ s3 0 ] - // [ s4 ] - - - // x_11 == 0 - arma::sp_mat A0(7, 7); A0.zeros(); - A0(0, 0) = 1.; - - // x_22 == 0 - arma::sp_mat A1(7, 7); A1.zeros(); - A1(1, 1) = 1.; - - // x_33 == 0 - arma::sp_mat A2(7, 7); A2.zeros(); - A2(2, 2) = 1.; - - // x_12 <= -0.1 <==> x_12 + s1 == -0.1, s1 >= 0 - arma::sp_mat A3(7, 7); A3.zeros(); - A3(1, 0) = A3(0, 1) = 1.; A3(3, 3) = 2.; - - // -0.2 <= x_12 <==> x_12 - s2 == -0.2, s2 >= 0 - arma::sp_mat A4(7, 7); A4.zeros(); - A4(1, 0) = A4(0, 1) = 1.; A4(4, 4) = -2.; - - // x_23 <= 0.5 <==> x_23 + s3 == 0.5, s3 >= 0 - arma::sp_mat A5(7, 7); A5.zeros(); - A5(2, 1) = A5(1, 2) = 1.; A5(5, 5) = 2.; - - // 0.4 <= x_23 <==> x_23 - s4 == 0.4, s4 >= 0 - arma::sp_mat A6(7, 7); A6.zeros(); - A6(2, 1) = A6(1, 2) = 1.; A6(6, 6) = -2.; - - std::vector ais({A0, A1, A2, A3, A4, A5, A6}); - - SDP sdp(7, 7 + 4 + 4 + 4 + 3 + 2 + 1, 0); - - for (size_t j = 0; j < 3; j++) - { - // x_j4 == x_j5 == x_j6 == x_j7 == 0 - for (size_t i = 0; i < 4; i++) - { - arma::sp_mat A(7, 7); A.zeros(); - A(i + 3, j) = A(j, i + 3) = 1; - ais.emplace_back(A); - } - } - - // x_45 == x_46 == x_47 == 0 - for (size_t i = 0; i < 3; i++) - { - arma::sp_mat A(7, 7); A.zeros(); - A(i + 4, 3) = A(3, i + 4) = 1; - ais.emplace_back(A); - } - - // x_56 == x_57 == 0 - for (size_t i = 0; i < 2; i++) - { - arma::sp_mat A(7, 7); A.zeros(); - A(i + 5, 4) = A(4, i + 5) = 1; - ais.emplace_back(A); - } - - // x_67 == 0 - arma::sp_mat A(7, 7); A.zeros(); - A(6, 5) = A(5, 6) = 1; - ais.emplace_back(A); - - std::swap(sdp.SparseA(), ais); - - sdp.SparseB().zeros(); - - sdp.SparseB()[0] = sdp.SparseB()[1] = sdp.SparseB()[2] = 1.; - - sdp.SparseB()[3] = -0.2; sdp.SparseB()[4] = -0.4; - - sdp.SparseB()[5] = 1.; sdp.SparseB()[6] = 0.8; - - sdp.C().zeros(); - sdp.C()(0, 2) = sdp.C()(2, 0) = 1.; - - PrimalDualSolver> solver(sdp); - arma::mat X, Z; - arma::vec ysparse, ydense; - const double obj = solver.Optimize(X, ysparse, ydense, Z); - bool success = CheckKKT(sdp, X, ysparse, ydense, Z); - BOOST_REQUIRE_EQUAL(success, true); - BOOST_REQUIRE_CLOSE(obj, 2 * (-0.978), 1e-3); -} - -// /** -// * Maximum variance unfolding (MVU) SDP to learn the unrolled gram matrix. For -// * the SDP formulation, see: -// * -// * Unsupervised learning of image manifolds by semidefinite programming. -// * Kilian Weinberger and Lawrence Saul. CVPR 04. -// * http://repository.upenn.edu/cgi/viewcontent.cgi?article=1000&context=cis_papers -// * -// * @param origData origDim x numPoints -// * @param numNeighbors -// */ -// static inline SDP ConstructMvuSDP(const arma::mat& origData, -// size_t numNeighbors) -// { -// const size_t numPoints = origData.n_cols; - -// assert(numNeighbors <= numPoints); - -// arma::Mat neighbors; -// arma::mat distances; -// KNN knn(origData); -// knn.Search(numNeighbors, neighbors, distances); - -// SDP sdp(numPoints, numNeighbors * numPoints, 1); -// sdp.C().eye(numPoints, numPoints); -// sdp.C() *= -1; -// sdp.DenseA()[0].ones(numPoints, numPoints); -// sdp.DenseB()[0] = 0; - -// for (size_t i = 0; i < neighbors.n_cols; ++i) -// { -// for (size_t j = 0; j < numNeighbors; ++j) -// { -// // This is the index of the constraint. -// const size_t index = (i * numNeighbors) + j; - -// arma::sp_mat& aRef = sdp.SparseA()[index]; -// aRef.zeros(numPoints, numPoints); - -// // A_ij(i, i) = 1. -// aRef(i, i) = 1; - -// // A_ij(i, j) = -1. -// aRef(i, neighbors(j, i)) = -1; - -// // A_ij(j, i) = -1. -// aRef(neighbors(j, i), i) = -1; - -// // A_ij(j, j) = 1. -// aRef(neighbors(j, i), neighbors(j, i)) = 1; - -// // The constraint b_ij is the distance between these two points. -// sdp.SparseB()[index] = distances(j, i); -// } -// } - -// return sdp; -// } - -// /** -// * Maximum variance unfolding -// * -// * Test doesn't work, because the constraint matrices are not linearly -// * independent. -// */ -// BOOST_AUTO_TEST_CASE(SmallMvuSdp) -// { -// const size_t n = 20; - -// arma::mat origData(3, n); - -// // sample n random points on 3-dim unit sphere -// GaussianDistribution gauss(3); -// for (size_t i = 0; i < n; i++) -// { -// // how european of them -// origData.col(i) = arma::normalise(gauss.Random()); -// } - -// auto sdp = ConstructMvuSDP(origData, 5); - -// PrimalDualSolver> solver(sdp); -// arma::mat X, Z; -// arma::vec ysparse, ydense; -// const auto p = solver.Optimize(X, ysparse, ydense, Z); -// BOOST_REQUIRE(p.first); -// } - -BOOST_AUTO_TEST_SUITE_END(); diff --git a/src/mlpack/tests/sgd_test.cpp b/src/mlpack/tests/sgd_test.cpp deleted file mode 100644 index b4015daef0..0000000000 --- a/src/mlpack/tests/sgd_test.cpp +++ /dev/null @@ -1,61 +0,0 @@ -/** - * @file sgd_test.cpp - * @author Ryan Curtin - * - * Test file for SGD (stochastic gradient descent). - * - * mlpack is free software; you may redistribute it and/or modify it under the - * terms of the 3-clause BSD license. You should have received a copy of the - * 3-clause BSD license along with mlpack. If not, see - * http://www.opensource.org/licenses/BSD-3-Clause for more information. - */ -#include -#include -#include -#include - -#include -#include "test_tools.hpp" - -using namespace std; -using namespace arma; -using namespace mlpack; -using namespace mlpack::optimization; -using namespace mlpack::optimization::test; - -BOOST_AUTO_TEST_SUITE(SGDTest); - -BOOST_AUTO_TEST_CASE(SimpleSGDTestFunction) -{ - SGDTestFunction f; - StandardSGD s(0.0003, 1, 5000000, 1e-9, true); - - arma::mat coordinates = f.GetInitialPoint(); - double result = s.Optimize(f, coordinates); - - BOOST_REQUIRE_CLOSE(result, -1.0, 0.05); - BOOST_REQUIRE_SMALL(coordinates[0], 1e-3); - BOOST_REQUIRE_SMALL(coordinates[1], 1e-7); - BOOST_REQUIRE_SMALL(coordinates[2], 1e-7); -} - -BOOST_AUTO_TEST_CASE(GeneralizedRosenbrockTest) -{ - // Loop over several variants. - for (size_t i = 10; i < 50; i += 5) - { - // Create the generalized Rosenbrock function. - GeneralizedRosenbrockFunction f(i); - - StandardSGD s(0.001, 1, 0, 1e-15, true); - - arma::mat coordinates = f.GetInitialPoint(); - double result = s.Optimize(f, coordinates); - - BOOST_REQUIRE_SMALL(result, 1e-10); - for (size_t j = 0; j < i; ++j) - BOOST_REQUIRE_CLOSE(coordinates[j], (double) 1.0, 1e-3); - } -} - -BOOST_AUTO_TEST_SUITE_END(); diff --git a/src/mlpack/tests/sgdr_test.cpp b/src/mlpack/tests/sgdr_test.cpp deleted file mode 100644 index 835b3d3be9..0000000000 --- a/src/mlpack/tests/sgdr_test.cpp +++ /dev/null @@ -1,129 +0,0 @@ -/** - * @file sgdr_test.cpp - * @author Marcus Edel - * - * Test file for SGDR. - * - * mlpack is free software; you may redistribute it and/or modify it under the - * terms of the 3-clause BSD license. You should have received a copy of the - * 3-clause BSD license along with mlpack. If not, see - * http://www.opensource.org/licenses/BSD-3-Clause for more information. - */ -#include -#include -#include -#include - -#include -#include "test_tools.hpp" - -using namespace std; -using namespace arma; -using namespace mlpack; -using namespace mlpack::optimization; - -using namespace mlpack::distribution; -using namespace mlpack::regression; - -BOOST_AUTO_TEST_SUITE(SGDRTest); - -/* - * Test that the step size resets after a specified number of epochs. - */ -BOOST_AUTO_TEST_CASE(CyclicalResetTest) -{ - const double stepSize = 0.5; - arma::mat iterate; - - // Now run cyclical decay policy with a couple of multiplicators and initial - // restarts. - for (size_t restart = 5; restart < 100; restart += 10) - { - for (size_t mult = 2; mult < 5; ++mult) - { - double epochStepSize = stepSize; - - CyclicalDecay cyclicalDecay(restart, double(mult), stepSize); - cyclicalDecay.EpochBatches() = (double) 1000 / 10; - - // Create all restart epochs. - arma::Col nextRestart(1000 / 10 / mult); - nextRestart(0) = restart; - for (size_t j = 1; j < nextRestart.n_elem; ++j) - nextRestart(j) = nextRestart(j - 1) * mult; - - for (size_t i = 0; i < 1000; ++i) - { - cyclicalDecay.Update(iterate, epochStepSize, iterate); - if (i <= restart || arma::accu(arma::find(nextRestart == i)) > 0) - { - BOOST_CHECK_EQUAL(epochStepSize, stepSize); - } - } - } - } -} - -/** - * Run SGDR on logistic regression and make sure the results are acceptable. - */ -BOOST_AUTO_TEST_CASE(LogisticRegressionTest) -{ - // Generate a two-Gaussian dataset. - GaussianDistribution g1(arma::vec("1.0 1.0 1.0"), arma::eye(3, 3)); - GaussianDistribution g2(arma::vec("9.0 9.0 9.0"), arma::eye(3, 3)); - - arma::mat data(3, 1000); - arma::Row responses(1000); - for (size_t i = 0; i < 500; ++i) - { - data.col(i) = g1.Random(); - responses[i] = 0; - } - for (size_t i = 500; i < 1000; ++i) - { - data.col(i) = g2.Random(); - responses[i] = 1; - } - - // Shuffle the dataset. - arma::uvec indices = arma::shuffle(arma::linspace(0, - data.n_cols - 1, data.n_cols)); - arma::mat shuffledData(3, 1000); - arma::Row shuffledResponses(1000); - for (size_t i = 0; i < data.n_cols; ++i) - { - shuffledData.col(i) = data.col(indices[i]); - shuffledResponses[i] = responses[indices[i]]; - } - - // Create a test set. - arma::mat testData(3, 1000); - arma::Row testResponses(1000); - for (size_t i = 0; i < 500; ++i) - { - testData.col(i) = g1.Random(); - testResponses[i] = 0; - } - for (size_t i = 500; i < 1000; ++i) - { - testData.col(i) = g2.Random(); - testResponses[i] = 1; - } - - // Now run SGDR with a couple of batch sizes. - for (size_t batchSize = 5; batchSize < 50; batchSize += 5) - { - SGDR<> sgdr(50, 2.0, batchSize, 0.01, 10000, 1e-3); - LogisticRegression<> lr(shuffledData, shuffledResponses, sgdr, 0.5); - - // Ensure that the error is close to zero. - const double acc = lr.ComputeAccuracy(data, responses); - BOOST_REQUIRE_CLOSE(acc, 100.0, 0.3); // 0.3% error tolerance. - - const double testAcc = lr.ComputeAccuracy(testData, testResponses); - BOOST_REQUIRE_CLOSE(testAcc, 100.0, 0.6); // 0.6% error tolerance. - } -} - -BOOST_AUTO_TEST_SUITE_END(); diff --git a/src/mlpack/tests/smorms3_test.cpp b/src/mlpack/tests/smorms3_test.cpp deleted file mode 100644 index de5c1f569c..0000000000 --- a/src/mlpack/tests/smorms3_test.cpp +++ /dev/null @@ -1,106 +0,0 @@ -/** - * @file smorms3_test.cpp - * @author Vivek Pal - * - * Tests the SMORMS3 optimizer. - * - * mlpack is free software; you may redistribute it and/or modify it under the - * terms of the 3-clause BSD license. You should have received a copy of the - * 3-clause BSD license along with mlpack. If not, see - * http://www.opensource.org/licenses/BSD-3-Clause for more information. - */ -#include - -#include -#include -#include - -#include -#include "test_tools.hpp" - -using namespace arma; -using namespace mlpack::optimization; -using namespace mlpack::optimization::test; - -using namespace mlpack::distribution; -using namespace mlpack::regression; - -using namespace mlpack; - -BOOST_AUTO_TEST_SUITE(SMORMS3Test); - -/** - * Tests the SMORMS3 optimizer using a simple test function. - */ -BOOST_AUTO_TEST_CASE(SimpleSMORMS3TestFunction) -{ - SGDTestFunction f; - SMORMS3 s(0.001, 1, 1e-16, 5000000, 1e-9, true); - - arma::mat coordinates = f.GetInitialPoint(); - s.Optimize(f, coordinates); - - BOOST_REQUIRE_SMALL(coordinates[0], 0.1); - BOOST_REQUIRE_SMALL(coordinates[1], 0.1); - BOOST_REQUIRE_SMALL(coordinates[2], 0.1); -} - -/** - * Run SMORMS3 on logistic regression and make sure the results are acceptable. - */ -BOOST_AUTO_TEST_CASE(SMORMS3LogisticRegressionTest) -{ - // Generate a two-Gaussian dataset. - GaussianDistribution g1(arma::vec("1.0 1.0 1.0"), arma::eye(3, 3)); - GaussianDistribution g2(arma::vec("9.0 9.0 9.0"), arma::eye(3, 3)); - - arma::mat data(3, 1000); - arma::Row responses(1000); - for (size_t i = 0; i < 500; ++i) - { - data.col(i) = g1.Random(); - responses[i] = 0; - } - for (size_t i = 500; i < 1000; ++i) - { - data.col(i) = g2.Random(); - responses[i] = 1; - } - - // Shuffle the dataset. - arma::uvec indices = arma::shuffle(arma::linspace(0, - data.n_cols - 1, data.n_cols)); - arma::mat shuffledData(3, 1000); - arma::Row shuffledResponses(1000); - for (size_t i = 0; i < data.n_cols; ++i) - { - shuffledData.col(i) = data.col(indices[i]); - shuffledResponses[i] = responses[indices[i]]; - } - - // Create a test set. - arma::mat testData(3, 1000); - arma::Row testResponses(1000); - for (size_t i = 0; i < 500; ++i) - { - testData.col(i) = g1.Random(); - testResponses[i] = 0; - } - for (size_t i = 500; i < 1000; ++i) - { - testData.col(i) = g2.Random(); - testResponses[i] = 1; - } - - SMORMS3 smorms3; - LogisticRegression<> lr(shuffledData, shuffledResponses, smorms3, 0.5); - - // Ensure that the error is close to zero. - const double acc = lr.ComputeAccuracy(data, responses); - BOOST_REQUIRE_CLOSE(acc, 100.0, 0.3); // 0.3% error tolerance. - - const double testAcc = lr.ComputeAccuracy(testData, testResponses); - BOOST_REQUIRE_CLOSE(testAcc, 100.0, 0.6); // 0.6% error tolerance. -} - -BOOST_AUTO_TEST_SUITE_END(); diff --git a/src/mlpack/tests/snapshot_ensembles.cpp b/src/mlpack/tests/snapshot_ensembles.cpp deleted file mode 100644 index 790e646b2b..0000000000 --- a/src/mlpack/tests/snapshot_ensembles.cpp +++ /dev/null @@ -1,133 +0,0 @@ -/** - * @file snapshot_ensembles.cpp - * @author Marcus Edel - * - * Test file for SGDR with snapshot ensembles. - * - * mlpack is free software; you may redistribute it and/or modify it under the - * terms of the 3-clause BSD license. You should have received a copy of the - * 3-clause BSD license along with mlpack. If not, see - * http://www.opensource.org/licenses/BSD-3-Clause for more information. - */ -#include -#include -#include -#include - -#include -#include "test_tools.hpp" - -using namespace std; -using namespace arma; -using namespace mlpack; -using namespace mlpack::optimization; - -using namespace mlpack::distribution; -using namespace mlpack::regression; - -BOOST_AUTO_TEST_SUITE(SnapshotEnsemblesTest); - -/* - * Test that the step size resets after a specified number of epochs. - */ -BOOST_AUTO_TEST_CASE(SnapshotEnsemblesResetTest) -{ - const double stepSize = 0.5; - arma::mat iterate; - - // Now run cyclical decay policy with a couple of multiplicators and initial - // restarts. - for (size_t restart = 5; restart < 100; restart += 10) - { - for (size_t mult = 2; mult < 5; ++mult) - { - double epochStepSize = stepSize; - - SnapshotEnsembles snapshotEnsembles(restart, - double(mult), stepSize, 1000, 2); - - snapshotEnsembles.EpochBatches() = 10 / (double)1000; - // Create all restart epochs. - arma::Col nextRestart(1000 / 10 / mult); - nextRestart(0) = restart; - for (size_t j = 1; j < nextRestart.n_elem; ++j) - nextRestart(j) = nextRestart(j - 1) * mult; - - for (size_t i = 0; i < 1000; ++i) - { - snapshotEnsembles.Update(iterate, epochStepSize, iterate); - if (i <= restart || arma::accu(arma::find(nextRestart == i)) > 0) - { - BOOST_CHECK_EQUAL(epochStepSize, stepSize); - } - } - - BOOST_CHECK_EQUAL(snapshotEnsembles.Snapshots().size(), 2); - } - } -} - -/** - * Run SGDR with snapshot ensembles on logistic regression and make sure the - * results are acceptable. - */ -BOOST_AUTO_TEST_CASE(LogisticRegressionTest) -{ - // Generate a two-Gaussian dataset. - GaussianDistribution g1(arma::vec("1.0 1.0 1.0"), arma::eye(3, 3)); - GaussianDistribution g2(arma::vec("9.0 9.0 9.0"), arma::eye(3, 3)); - - arma::mat data(3, 1000); - arma::Row responses(1000); - for (size_t i = 0; i < 500; ++i) - { - data.col(i) = g1.Random(); - responses[i] = 0; - } - for (size_t i = 500; i < 1000; ++i) - { - data.col(i) = g2.Random(); - responses[i] = 1; - } - - // Shuffle the dataset. - arma::uvec indices = arma::shuffle(arma::linspace(0, - data.n_cols - 1, data.n_cols)); - arma::mat shuffledData(3, 1000); - arma::Row shuffledResponses(1000); - for (size_t i = 0; i < data.n_cols; ++i) - { - shuffledData.col(i) = data.col(indices[i]); - shuffledResponses[i] = responses[indices[i]]; - } - - // Create a test set. - arma::mat testData(3, 1000); - arma::Row testResponses(1000); - for (size_t i = 0; i < 500; ++i) - { - testData.col(i) = g1.Random(); - testResponses[i] = 0; - } - for (size_t i = 500; i < 1000; ++i) - { - testData.col(i) = g2.Random(); - testResponses[i] = 1; - } - - // Now run SGDR with snapshot ensembles on a couple of batch sizes. - for (size_t batchSize = 5; batchSize < 50; batchSize += 5) - { - SnapshotSGDR<> sgdr(50, 2.0, batchSize, 0.01, 10000, 1e-3); - LogisticRegression<> lr(shuffledData, shuffledResponses, sgdr, 0.5); - - // Ensure that the error is close to zero. - const double acc = lr.ComputeAccuracy(data, responses); - BOOST_REQUIRE_CLOSE(acc, 100.0, 0.3); // 0.3% error tolerance. - - const double testAcc = lr.ComputeAccuracy(testData, testResponses); - BOOST_REQUIRE_CLOSE(testAcc, 100.0, 0.6); // 0.6% error tolerance. - } -} - -BOOST_AUTO_TEST_SUITE_END(); diff --git a/src/mlpack/tests/spalera_sgd_test.cpp b/src/mlpack/tests/spalera_sgd_test.cpp deleted file mode 100644 index bf2698c3bb..0000000000 --- a/src/mlpack/tests/spalera_sgd_test.cpp +++ /dev/null @@ -1,89 +0,0 @@ -/** - * @file spalera_sgd_test.cpp - * @author Marcus Edel - * - * Test file for SGD (stochastic gradient descent). - * - * mlpack is free software; you may redistribute it and/or modify it under the - * terms of the 3-clause BSD license. You should have received a copy of the - * 3-clause BSD license along with mlpack. If not, see - * http://www.opensource.org/licenses/BSD-3-Clause for more information. - */ -#include -#include -#include - -#include -#include "test_tools.hpp" - -using namespace mlpack; -using namespace mlpack::optimization; -using namespace mlpack::distribution; -using namespace mlpack::regression; - -BOOST_AUTO_TEST_SUITE(SPALeRASGDTest); - -/** - * Run SPALeRA SGD on logistic regression and make sure the results are - * acceptable. - */ -BOOST_AUTO_TEST_CASE(LogisticRegressionTest) -{ - // Generate a two-Gaussian dataset. - GaussianDistribution g1(arma::vec("1.0 1.0 1.0"), arma::eye(3, 3)); - GaussianDistribution g2(arma::vec("9.0 9.0 9.0"), arma::eye(3, 3)); - - arma::mat data(3, 500); - arma::Row responses(500); - for (size_t i = 0; i < 250; ++i) - { - data.col(i) = g1.Random(); - responses[i] = 0; - } - for (size_t i = 250; i < 500; ++i) - { - data.col(i) = g2.Random(); - responses[i] = 1; - } - - // Shuffle the dataset. - arma::uvec indices = arma::shuffle(arma::linspace(0, - data.n_cols - 1, data.n_cols)); - arma::mat shuffledData(3, 500); - arma::Row shuffledResponses(500); - for (size_t i = 0; i < data.n_cols; ++i) - { - shuffledData.col(i) = data.col(indices[i]); - shuffledResponses[i] = responses[indices[i]]; - } - - // Create a test set. - arma::mat testData(3, 500); - arma::Row testResponses(500); - for (size_t i = 0; i < 250; ++i) - { - testData.col(i) = g1.Random(); - testResponses[i] = 0; - } - for (size_t i = 250; i < 500; ++i) - { - testData.col(i) = g2.Random(); - testResponses[i] = 1; - } - - // Now run mini-batch SGD with a couple of batch sizes. - for (size_t batchSize = 30; batchSize < 50; batchSize += 5) - { - SPALeRASGD<> mbsgd(0.05 / batchSize, batchSize, 10000, 1e-4); - LogisticRegression<> lr(shuffledData, shuffledResponses, mbsgd, 0.5); - - // Ensure that the error is close to zero. - const double acc = lr.ComputeAccuracy(data, responses); - BOOST_REQUIRE_CLOSE(acc, 100.0, 1.5); // 1.5% error tolerance. - - const double testAcc = lr.ComputeAccuracy(testData, testResponses); - BOOST_REQUIRE_CLOSE(testAcc, 100.0, 2.4); // 2.4% error tolerance. - } -} - -BOOST_AUTO_TEST_SUITE_END(); diff --git a/src/mlpack/tests/svrg_test.cpp b/src/mlpack/tests/svrg_test.cpp deleted file mode 100644 index 5f3e31847e..0000000000 --- a/src/mlpack/tests/svrg_test.cpp +++ /dev/null @@ -1,77 +0,0 @@ -/** - * @file svrg_test.cpp - * @author Marcus Edel - * - * Test file for SVRG. - * - * mlpack is free software; you may redistribute it and/or modify it under the - * terms of the 3-clause BSD license. You should have received a copy of the - * 3-clause BSD license along with mlpack. If not, see - * http://www.opensource.org/licenses/BSD-3-Clause for more information. - */ -#include -#include - -#include -#include "test_tools.hpp" -#include "test_function_tools.hpp" - -using namespace mlpack; -using namespace mlpack::optimization; - -BOOST_AUTO_TEST_SUITE(SVRGTest); - -/** - * Run SVRG on logistic regression and make sure the results are acceptable. - */ -BOOST_AUTO_TEST_CASE(SVRGLogisticRegressionTest) -{ - arma::mat data, testData, shuffledData; - arma::Row responses, testResponses, shuffledResponses; - - LogisticRegressionTestData(data, testData, shuffledData, - responses, testResponses, shuffledResponses); - - // Now run big-batch SGD with a couple of batch sizes. - for (size_t batchSize = 35; batchSize < 50; batchSize += 5) - { - SVRG optimizer(0.001, batchSize, 250, 0, 1e-3, true); - LogisticRegression<> lr(shuffledData, shuffledResponses, optimizer, 0.5); - - // Ensure that the error is close to zero. - const double acc = lr.ComputeAccuracy(data, responses); - BOOST_REQUIRE_CLOSE(acc, 100.0, 1.5); // 1.5% error tolerance. - - const double testAcc = lr.ComputeAccuracy(testData, testResponses); - BOOST_REQUIRE_CLOSE(testAcc, 100.0, 1.5); // 1.5% error tolerance. - } -} - -/** - * Run SVRG_BB on logistic regression and make sure the results are acceptable. - */ -BOOST_AUTO_TEST_CASE(SVRGBBLogisticRegressionTest) -{ - arma::mat data, testData, shuffledData; - arma::Row responses, testResponses, shuffledResponses; - - LogisticRegressionTestData(data, testData, shuffledData, - responses, testResponses, shuffledResponses); - - // Now run big-batch SGD with a couple of batch sizes. - for (size_t batchSize = 35; batchSize < 50; batchSize += 5) - { - SVRG_BB optimizer(0.001, batchSize, 250, 0, 1e-5, true, - SVRGUpdate(), BarzilaiBorweinDecay(0.1)); - LogisticRegression<> lr(shuffledData, shuffledResponses, optimizer, 0.5); - - // Ensure that the error is close to zero. - const double acc = lr.ComputeAccuracy(data, responses); - BOOST_REQUIRE_CLOSE(acc, 100.0, 1.5); // 1.5% error tolerance. - - const double testAcc = lr.ComputeAccuracy(testData, testResponses); - BOOST_REQUIRE_CLOSE(testAcc, 100.0, 1.5); // 1.5% error tolerance. - } -} - -BOOST_AUTO_TEST_SUITE_END();