From ac40c7231220b0da893752db090e7d561bbb31f0 Mon Sep 17 00:00:00 2001 From: Ayush Date: Fri, 22 Feb 2019 01:25:31 +0530 Subject: [PATCH] added tests for LinearSVM with some Optimizers --- src/mlpack/tests/CMakeLists.txt | 3 +- src/mlpack/tests/linear_svm_test.cpp | 177 ++++++++++++++++++++++----- 2 files changed, 147 insertions(+), 33 deletions(-) diff --git a/src/mlpack/tests/CMakeLists.txt b/src/mlpack/tests/CMakeLists.txt index 44e8d321f9..32b3b08be9 100644 --- a/src/mlpack/tests/CMakeLists.txt +++ b/src/mlpack/tests/CMakeLists.txt @@ -184,7 +184,8 @@ set(parallel_tests "SVDIncrementalTest;SVDBatchTest;" "LocalCoordinateCodingTest;FeedForwardNetworkTest;SparseAutoencoderTest;" "GMMTest;CFTest;ConvolutionalNetworkTest;HMMTest;LARSTest;" - "LogisticRegressionTest") + "LogisticRegressionTest" + "LinearSVMTest") # Add tests to the testing framework # Get the list of sources from the test target diff --git a/src/mlpack/tests/linear_svm_test.cpp b/src/mlpack/tests/linear_svm_test.cpp index 37f384c36d..d6e3e0b1a8 100644 --- a/src/mlpack/tests/linear_svm_test.cpp +++ b/src/mlpack/tests/linear_svm_test.cpp @@ -439,31 +439,6 @@ BOOST_AUTO_TEST_CASE(LinearSVMFunctionSeparableGradient) } } -/** - * Test training of linear svm on a simple dataset using - * Parallel SGD optimizer. - */ -BOOST_AUTO_TEST_CASE(LinearSVMPSGDSimpleTest) -{ - // A very simple fake dataset - arma::mat dataset = "2 0 0;" - "0 0 0;" - "0 2 1;" - "1 0 2;" - "0 1 0"; - - // Corresponding labels - arma::Row labels = "1 0 1"; - - // Create a linear svm object using a custom Parallel - // SGD object. - ens::ParallelSGD<> psgd(1000, 3, 1e-5); - LinearSVM lsvm(dataset, labels, 2, 0.0001, psgd); - - // Compare training accuracy to 100. - const double acc = lsvm.ComputeAccuracy(dataset, labels); - BOOST_REQUIRE_CLOSE(acc, 100.0, 0.5); -} /** * Test training of linear svm on a simple dataset using @@ -471,6 +446,9 @@ BOOST_AUTO_TEST_CASE(LinearSVMPSGDSimpleTest) */ BOOST_AUTO_TEST_CASE(LinearSVMLGFGSSimpleTest) { + const size_t numClasses = 2; + const double lambda = 0.0001; + // A very simple fake dataset arma::mat dataset = "2 0 0;" "0 0 0;" @@ -481,14 +459,46 @@ BOOST_AUTO_TEST_CASE(LinearSVMLGFGSSimpleTest) // Corresponding labels arma::Row labels = "1 0 1"; - // Create a linear svm object using a custom L-BFGS object. - LinearSVM lsvm(dataset, labels, 2, 0.0001, ens::L_BFGS()); + // Create a linear svm object using L-BFGS optimizer. + LinearSVM lsvm(dataset, labels, numClasses, lambda, + ens::L_BFGS()); // Compare training accuracy to 100. const double acc = lsvm.ComputeAccuracy(dataset, labels); BOOST_REQUIRE_CLOSE(acc, 100.0, 0.5); } +/** + * Test training of linear svm on a simple dataset using + * Gradient Descent optimizer + */ +BOOST_AUTO_TEST_CASE(LinearSVMGradientDescentSimpleTest) +{ + const size_t numClasses = 2; + const size_t maxIterations = 10000; + const double stepSize = 0.01; + const double tolerance = 1e-5; + const double lambda = 0.0001; + + // A very simple fake dataset + arma::mat dataset = "2 0 0;" + "0 0 0;" + "0 2 1;" + "1 0 2;" + "0 1 0"; + + // Corresponding labels + arma::Row labels = "1 0 1"; + + // Create a linear svm object using custom gradient descent optimizer. + ens::GradientDescent optimizer(stepSize, maxIterations, tolerance); + LinearSVM lsvm(dataset, labels, numClasses, lambda, optimizer); + + // Compare training accuracy to 100. + const double acc = lsvm.ComputeAccuracy(dataset, labels); + BOOST_REQUIRE_CLOSE(acc, 100.0, 0.5); +} + /** * Test training of linear svm for two classes on a complex gaussian dataset * using L-BFGS optimizer. @@ -518,8 +528,9 @@ BOOST_AUTO_TEST_CASE(LinearSVMLBFGSTwoClasses) labels(i) = 1; } - // Train linear svm object using L-BFGS optimizer. - LinearSVM lsvm(data, labels, numClasses, lambda, ens::L_BFGS()); + // Create a linear svm object using L-BFGS optimizer. + LinearSVM lsvm(data, labels, numClasses, lambda, + ens::L_BFGS()); // Compare training accuracy to 100. const double acc = lsvm.ComputeAccuracy(data, labels); @@ -542,6 +553,108 @@ BOOST_AUTO_TEST_CASE(LinearSVMLBFGSTwoClasses) BOOST_REQUIRE_CLOSE(testAcc, 100.0, 0.6); } +/** + * The test is only compiled if the user has specified OpenMP to be + * used. + */ +#ifdef HAS_OPENMP + +/** + * Test training of linear svm on a simple dataset using + * Parallel SGD optimizer. + */ +BOOST_AUTO_TEST_CASE(LinearSVMPSGDSimpleTest) +{ + const size_t numClasses = 2; + const double lambda = 0.5; + const double alpha = 0.01; + + // A very simple fake dataset + arma::mat dataset = "2 0 0;" + "0 0 0;" + "0 2 1;" + "1 0 2;" + "0 1 0"; + + // Corresponding labels + arma::Row labels = "1 0 1"; + + ens::ConstantStep decayPolicy(alpha); + + // Train linear svm object using Parallel SGD optimizer. + // The threadShareSize is chosen such that each function gets optimized. + ens::ParallelSGD optimizer(0, + std::ceil((float) dataset.n_cols / omp_get_max_threads()), + 1e-5, true, decayPolicy); + LinearSVM lsvm(dataset, labels, numClasses, lambda, optimizer); + + // Compare training accuracy to 100. + const double acc = lsvm.ComputeAccuracy(dataset, labels); + BOOST_REQUIRE_CLOSE(acc, 100.0, 0.5); +} + +/** + * Test training of linear svm for two classes on a complex gaussian dataset + * using Parallel SGD optimizer. + */ +BOOST_AUTO_TEST_CASE(LinearSVMParallelSGDTwoClasses) +{ + const size_t points = 500; + const size_t inputSize = 3; + const size_t numClasses = 2; + const double lambda = 0.5; + const double alpha = 0.01; + + // Generate two-Gaussian dataset. + GaussianDistribution g1(arma::vec("1.0 9.0 1.0"), arma::eye(3, 3)); + GaussianDistribution g2(arma::vec("4.0 3.0 4.0"), arma::eye(3, 3)); + + arma::mat data(inputSize, points); + arma::Row labels(points); + + for (size_t i = 0; i < points / 2; i++) + { + data.col(i) = g1.Random(); + labels(i) = 0; + } + for (size_t i = points / 2; i < points; i++) + { + data.col(i) = g2.Random(); + labels(i) = 1; + } + + ens::ConstantStep decayPolicy(alpha); + + // Train linear svm object using Parallel SGD optimizer. + // The threadShareSize is chosen such that each function gets optimized. + ens::ParallelSGD optimizer(0, + std::ceil((float) data.n_cols / omp_get_max_threads()), + 1e-5, true, decayPolicy); + LinearSVM lsvm(data, labels, numClasses, lambda, optimizer); + + // Compare training accuracy to 100. + const double acc = lsvm.ComputeAccuracy(data, labels); + BOOST_REQUIRE_CLOSE(acc, 100.0, 0.5); + + // Create test dataset. + for (size_t i = 0; i < points / 2; i++) + { + data.col(i) = g1.Random(); + labels(i) = 0; + } + for (size_t i = points / 2; i < points; i++) + { + data.col(i) = g2.Random(); + labels(i) = 1; + } + + // Compare test accuracy to 100. + const double testAcc = lsvm.ComputeAccuracy(data, labels); + BOOST_REQUIRE_CLOSE(testAcc, 100.0, 0.6); +} + +#endif + /** * Test sparse and dense linear svm and make sure they both work the * same using the L-BFGS optimizer. @@ -667,9 +780,9 @@ BOOST_AUTO_TEST_CASE(LinearSVMTrainTest) LinearSVM lsvm(dataset.n_rows, 2); LinearSVM lsvm2(dataset.n_rows, 2); lsvm.Parameters() = lsvm2.Parameters(); - ens::L_BFGS lbfgs; - lsvm.Train(dataset, labels, 2, std::move(lbfgs)); - lsvm2.Train(dataset, labels, 2, std::move(lbfgs)); + ens::L_BFGS optimizer; + lsvm.Train(dataset, labels, 2, std::move(optimizer)); + lsvm2.Train(dataset, labels, 2, std::move(optimizer)); // Ensure that the parameters are the same. BOOST_REQUIRE_EQUAL(lsvm.Parameters().n_rows, lsvm2.Parameters().n_rows);