From d93b0b499cddf5da1a72da991864f083f520c8bc Mon Sep 17 00:00:00 2001 From: Ayush Date: Mon, 4 Feb 2019 18:26:57 +0530 Subject: [PATCH] Added new test for SparseSVMFunction. --- src/mlpack/methods/sparse_svm/sparse_svm.hpp | 51 +++- .../sparse_svm/sparse_svm_function.hpp | 8 - .../sparse_svm/sparse_svm_function_impl.hpp | 103 ++++--- .../methods/sparse_svm/sparse_svm_impl.hpp | 86 +++++- src/mlpack/tests/CMakeLists.txt | 1 + src/mlpack/tests/sparse_svm_test.cpp | 258 ++++++++++++++++-- 6 files changed, 409 insertions(+), 98 deletions(-) diff --git a/src/mlpack/methods/sparse_svm/sparse_svm.hpp b/src/mlpack/methods/sparse_svm/sparse_svm.hpp index 0088b58f83..444f50b9ba 100644 --- a/src/mlpack/methods/sparse_svm/sparse_svm.hpp +++ b/src/mlpack/methods/sparse_svm/sparse_svm.hpp @@ -55,15 +55,38 @@ class SparseSVM OptimizerType optimizer = OptimizerType()); /** - * Classify the given points, returning predicted class label for - * each data point. + * Classify the given points, returning the predicted labels for each point. * - * @param dataset Matrix of data points to be classified. + * @param data Set of points to classify. * @param labels Predicted labels for each point. */ - void Classify(const MatType& dataset, + void Classify(const MatType& data, arma::Row& labels) const; + /** + * Classify the given points, returning class scores and predicted + * class label for each point. + * The function calculates the scores for every class, given a data + * point. It then chooses the class which has the highest probability among + * all. + * + * @param data Matrix of data points to be classified. + * @param labels Predicted labels for each point. + * @param scores Class probabilities for each point. + */ + void Classify(const MatType& data, + arma::Row& labels, + arma::mat& scores) const; + + /** + * Classify the given points, returning class scores for each point. + * + * @param data Matrix of data points to be classified. + * @param scores Class scores for each point. + */ + void Classify(const MatType& data, + arma::mat& scores) const; + /** * Computes accuracy of the learned model given the feature data and the * labels associated with each data point. Predictions are made using the @@ -94,11 +117,25 @@ class SparseSVM const double lambda = 0.0001, OptimizerType optimizer = OptimizerType()); + + //! Sets the number of classes. + size_t& NumClasses() { return numClasses; } + //! Gets the number of classes. + size_t NumClasses() const { return numClasses; } + + //! Sets the regularization parameter. + double& Lambda() { return lambda; } + //! Gets the regularization parameter. + double Lambda() const { return lambda; } + //! Set the model parameters. arma::mat& Parameters() { return parameters; } //! Get the model parameters. const arma::mat& Parameters() const { return parameters; } + //! Gets the features size of the training data + size_t FeatureSize() const { return parameters.n_cols; } + /** * Serialize the SparseSVM model. */ @@ -106,11 +143,17 @@ class SparseSVM void serialize(Archive& ar, const unsigned int /* version */) { ar & BOOST_SERIALIZATION_NVP(parameters); + ar & BOOST_SERIALIZATION_NVP(numClasses); + ar & BOOST_SERIALIZATION_NVP(lambda); } private: //! Parameters after optimization. arma::mat parameters; + //! Number of classes. + size_t numClasses; + //! L2-Regularization constant. + double lambda; }; } // namespace svm diff --git a/src/mlpack/methods/sparse_svm/sparse_svm_function.hpp b/src/mlpack/methods/sparse_svm/sparse_svm_function.hpp index 26bb930906..6a05273df0 100644 --- a/src/mlpack/methods/sparse_svm/sparse_svm_function.hpp +++ b/src/mlpack/methods/sparse_svm/sparse_svm_function.hpp @@ -174,11 +174,6 @@ class SparseSVMFunction //! Modify the dataset. arma::sp_mat& Dataset() { return dataset; } - //! Get the labels. - const arma::vec& Labels() const { return labels; } - //! Modify the labels. - arma::vec& Labels() { return labels; } - //! Sets the regularization parameter. double& Lambda() { return lambda; } //! Gets the regularization parameter. @@ -201,9 +196,6 @@ class SparseSVMFunction //! Number of Classes. size_t numClasses; - //! The labels, y_i. - arma::Row labels; - //! The regularization parameter for L2-regularization. double lambda; }; diff --git a/src/mlpack/methods/sparse_svm/sparse_svm_function_impl.hpp b/src/mlpack/methods/sparse_svm/sparse_svm_function_impl.hpp index b54834a5f8..117f445b9e 100644 --- a/src/mlpack/methods/sparse_svm/sparse_svm_function_impl.hpp +++ b/src/mlpack/methods/sparse_svm/sparse_svm_function_impl.hpp @@ -30,8 +30,6 @@ SparseSVMFunction::SparseSVMFunction( const size_t numClasses, const double lambda) : dataset(math::MakeAlias(const_cast(dataset), false)), - labels(math::MakeAlias(const_cast&>(labels), - false)), numClasses(numClasses), lambda(lambda) { @@ -158,12 +156,12 @@ double SparseSVMFunction::Evaluate( // Calculate the loss and regularization terms. double loss, regularization, cost; - arma::mat scores = dataset.t() * parameters; - arma::mat correct = scores - % arma::conv_to::from(groundTruth).t(); - arma::mat margin = scores - arma::repmat(correct - * arma::ones(numClasses), 1, numClasses) + 1 - - groundTruth.t(); + arma::mat scores = parameters * dataset; + arma::mat correct = arma::conv_to::from(scores + % groundTruth); + arma::mat margin = scores + - arma::repmat(arma::ones(numClasses).t() * correct, numClasses, 1) + + 1 - groundTruth; // The Hinge Loss Function loss = arma::accu(arma::clamp(margin, 0.0, margin.max())); @@ -188,13 +186,12 @@ double SparseSVMFunction::Evaluate( // Calculate the loss and regularization terms. double loss, regularization, cost; - arma::mat scores = dataset.cols(firstId, lastId).t() - * parameters; - arma::mat correct = scores - % arma::conv_to::from(groundTruth).cols(firstId, lastId).t(); - arma::mat margin = scores - arma::repmat(correct - * arma::ones(numClasses), 1, numClasses) + 1 - - arma::conv_to::from(groundTruth).cols(firstId, lastId).t(); + arma::mat scores = parameters * dataset.cols(firstId, lastId); + arma::mat correct = arma::conv_to::from(scores + % groundTruth.cols(firstId, lastId)); + arma::mat margin = scores + - arma::repmat(arma::ones(numClasses).t() * correct, numClasses, 1) + + 1 - groundTruth.cols(firstId, lastId); // The Hinge Loss Function loss = arma::accu(arma::clamp(margin, 0.0, margin.max())); @@ -214,23 +211,23 @@ void SparseSVMFunction::Gradient( const arma::mat& parameters, GradType& gradient) { - arma::mat scores = dataset.t() * parameters; - arma::mat correct = scores - % arma::conv_to::from(groundTruth).t(); - arma::mat margin = scores - arma::repmat(correct - * arma::ones(numClasses), 1, numClasses) + 1 - - groundTruth.t(); + arma::mat scores = parameters * dataset; + arma::mat correct = arma::conv_to::from(scores + % groundTruth); + arma::mat margin = scores + - arma::repmat(arma::ones(numClasses).t() * correct, numClasses, 1) + + 1 - groundTruth; // For each sample, find the total number of classes where // ( margin > 0 ) arma::mat mask = margin.for_each([](arma::mat::elem_type& val) { val = (val > 0) ? 1: 0; }); - arma::mat incorrectLabels = arma::conv_to::from(groundTruth).t() - % (-arma::repmat(arma::sum(mask, 1), 1, numClasses)); + arma::sp_mat incorrectLabels = groundTruth + % (-arma::repmat(arma::sum(mask), numClasses, 1)); arma::mat difference = incorrectLabels + mask; - gradient = dataset * difference; + gradient = difference * dataset.t(); gradient /= dataset.n_cols; // Adding the regularization contribution to the gradient. @@ -246,25 +243,23 @@ void SparseSVMFunction::Gradient( const size_t batchSize) { const size_t lastId = firstId + batchSize - 1; - arma::mat scores = dataset.cols(firstId, lastId).t() - * parameters; - arma::mat correct = scores - % arma::conv_to::from(groundTruth).cols(firstId, lastId).t(); - arma::mat margin = scores - arma::repmat(correct - * arma::ones(numClasses), 1, numClasses) + 1 - - arma::conv_to::from(groundTruth).cols(firstId, lastId).t(); + arma::mat scores = parameters * dataset.cols(firstId, lastId); + arma::mat correct = arma::conv_to::from(scores + % groundTruth.cols(firstId, lastId)); + arma::mat margin = scores + - arma::repmat(arma::ones(numClasses).t() * correct, numClasses, 1) + + 1 - groundTruth.cols(firstId, lastId); // For each sample, find the total number of classes where // ( margin > 0 ) arma::mat mask = margin.for_each([](arma::mat::elem_type& val) { val = (val > 0) ? 1: 0; }); - arma::mat incorrectLabels = - arma::conv_to::from(groundTruth).cols(firstId, lastId).t() % - (-arma::repmat(arma::sum(mask.rows(firstId, lastId), 1), 1, numClasses)); + arma::sp_mat incorrectLabels = groundTruth.cols(firstId, lastId) + % (-arma::repmat(arma::sum(mask.cols(firstId, lastId)), numClasses, 1)); arma::mat difference = incorrectLabels + mask; - gradient = dataset.cols(firstId, lastId) * difference; + gradient = difference * dataset.cols(firstId, lastId).t(); gradient /= batchSize; // Adding the regularization contribution to the gradient. @@ -279,23 +274,23 @@ double SparseSVMFunction::EvaluateWithGradient( { double loss, regularization, cost; - arma::mat scores = dataset.t() * parameters; - arma::mat correct = scores - % arma::conv_to::from(groundTruth).t(); - arma::mat margin = scores - arma::repmat(correct - * arma::ones(numClasses), 1, numClasses) + 1 - - groundTruth.t(); + arma::mat scores = parameters * dataset; + arma::mat correct = arma::conv_to::from(scores + % groundTruth); + arma::mat margin = scores + - arma::repmat(arma::ones(numClasses).t() * correct, numClasses, 1) + + 1 - groundTruth; // For each sample, find the total number of classes where // ( margin > 0 ) arma::mat mask = margin.for_each([](arma::mat::elem_type& val) { val = (val > 0) ? 1: 0; }); - arma::mat incorrectLabels = arma::conv_to::from(groundTruth).t() - % (-arma::repmat(arma::sum(mask, 1), 1, numClasses)); + arma::sp_mat incorrectLabels = groundTruth + % (-arma::repmat(arma::sum(mask), numClasses, 1)); arma::mat difference = incorrectLabels + mask; - gradient = dataset * difference; + gradient = difference * dataset.t(); gradient /= dataset.n_cols; // Adding the regularization contribution to the gradient. @@ -326,25 +321,23 @@ double SparseSVMFunction::EvaluateWithGradient( // Calculate the loss and regularization terms. double loss, regularization, cost; - arma::mat scores = dataset.cols(firstId, lastId).t() - * parameters; - arma::mat correct = scores - % arma::conv_to::from(groundTruth).cols(firstId, lastId).t(); - arma::mat margin = scores - arma::repmat(correct - * arma::ones(numClasses), 1, numClasses) + 1 - - arma::conv_to::from(groundTruth).cols(firstId, lastId).t(); + arma::mat scores = parameters * dataset.cols(firstId, lastId); + arma::mat correct = arma::conv_to::from(scores + % groundTruth.cols(firstId, lastId)); + arma::mat margin = scores + - arma::repmat(arma::ones(numClasses).t() * correct, numClasses, 1) + + 1 - groundTruth.cols(firstId, lastId); // For each sample, find the total number of classes where // ( margin > 0 ) arma::mat mask = margin.for_each([](arma::mat::elem_type& val) { val = (val > 0) ? 1: 0; }); - arma::mat incorrectLabels = - arma::conv_to::from(groundTruth).cols(firstId, lastId).t() % - (-arma::repmat(arma::sum(mask.rows(firstId, lastId), 1), 1, numClasses)); + arma::sp_mat incorrectLabels = groundTruth.cols(firstId, lastId) + % (-arma::repmat(arma::sum(mask.cols(firstId, lastId)), numClasses, 1)); arma::mat difference = incorrectLabels + mask; - gradient = dataset.cols(firstId, lastId) * difference; + gradient = difference * dataset.cols(firstId, lastId).t(); gradient /= batchSize; // Adding the regularization contribution to the gradient. diff --git a/src/mlpack/methods/sparse_svm/sparse_svm_impl.hpp b/src/mlpack/methods/sparse_svm/sparse_svm_impl.hpp index 5692bb7044..a6dbb831f8 100644 --- a/src/mlpack/methods/sparse_svm/sparse_svm_impl.hpp +++ b/src/mlpack/methods/sparse_svm/sparse_svm_impl.hpp @@ -25,7 +25,9 @@ SparseSVM::SparseSVM( const arma::Row& labels, const size_t numClasses, const double lambda, - OptimizerType optimizer) + OptimizerType optimizer) : + numClasses(numClasses), + lambda(lambda) { Train(data, labels, numClasses, lambda, optimizer); } @@ -39,8 +41,8 @@ double SparseSVM::Train( const double lambda, OptimizerType optimizer) { - SparseSVMFunction svm(arma::conv_to::from(data), - labels, numClasses, lambda); + SparseSVMFunction svm(data, labels, + numClasses, lambda); if (parameters.is_empty()) parameters = svm.InitialPoint(); @@ -57,13 +59,81 @@ double SparseSVM::Train( template void SparseSVM::Classify( - const MatType& dataset, - arma::Row& labels) + const MatType& data, + arma::Row &labels) const { - // Classify each point of dataset into their suitable class. - labels = arma::conv_to>::from( - arma::index_max(parameters.t() * dataset)); + arma::mat scores; + Classify(data, scores); + + // Prepare necessary data + labels.zeros(data.n_cols); + double maxScore = 0; + + // For each test input. + for (size_t i = 0; i < data.n_cols; ++i) + { + // For each class. + for (size_t j = 0; j < numClasses; ++j) { + // If a higher class probability is encountered, change score. + if (scores(j, i) > maxScore) + { + maxScore = scores(j, i); + labels(i) = j; + } + } + + // Set maximum probability to zero for next input. + maxScore = 0; + } +} + +template +void SparseSVM::Classify( + const MatType& data, + arma::Row &labels, + arma::mat& scores) +const +{ + Classify(data, scores); + + // Prepare necessary data + labels.zeros(data.n_cols); + double maxScore = 0; + + // For each test input. + for (size_t i = 0; i < data.n_cols; ++i) + { + // For each class. + for (size_t j = 0; j < numClasses; ++j) { + // If a higher class probability is encountered, change score. + if (scores(j, i) > maxScore) + { + maxScore = scores(j, i); + labels(i) = j; + } + } + + // Set maximum probability to zero for next input. + maxScore = 0; + } +} + +template +void SparseSVM::Classify( + const MatType& data, + arma::mat& scores) +const +{ + if (data.n_rows != FeatureSize()) + { + std::ostringstream oss; + oss << "SparseSVM::Classify(): dataset has " << data.n_rows + << " dimensions, but model has " << FeatureSize() << " dimensions!"; + throw std::invalid_argument(oss.str()); + } + + scores = parameters * data; } template diff --git a/src/mlpack/tests/CMakeLists.txt b/src/mlpack/tests/CMakeLists.txt index 8354d3423c..73b81fe370 100644 --- a/src/mlpack/tests/CMakeLists.txt +++ b/src/mlpack/tests/CMakeLists.txt @@ -96,6 +96,7 @@ add_executable(mlpack_test sort_policy_test.cpp sparse_autoencoder_test.cpp sparse_coding_test.cpp + sparse_svm_test.cpp spill_tree_test.cpp split_data_test.cpp svd_batch_test.cpp diff --git a/src/mlpack/tests/sparse_svm_test.cpp b/src/mlpack/tests/sparse_svm_test.cpp index 10d578e59e..14b6705a82 100644 --- a/src/mlpack/tests/sparse_svm_test.cpp +++ b/src/mlpack/tests/sparse_svm_test.cpp @@ -23,44 +23,256 @@ using namespace mlpack::distribution; BOOST_AUTO_TEST_SUITE(SparseSVMTest); /** - * A more complicated test for the SparseSVMFunction. + * A simple test for SparseSVMFunction */ BOOST_AUTO_TEST_CASE(SparseSVMFunctionEvaluate) { - const size_t dimension = 10; + // A very simple fake dataset + arma::mat dataset = "2 0 0;" + "0 0 0;" + "0 2 1;" + "1 0 2;" + "0 1 0"; + + // Corresponding labels + arma::Row labels = "1 0 1"; + + SparseSVMFunction svmf(dataset, labels, 2, + 0.0 /* no regularization */); + + // These were hand-calculated using Python. + arma::mat parameters = "1 1 1 1 1;" + " 1 1 1 1 1"; + BOOST_REQUIRE_CLOSE(svmf.Evaluate(parameters), 1.0, 1e-5); + + parameters = "2 0 1 2 2;" + " 1 2 2 2 2"; + BOOST_REQUIRE_CLOSE(svmf.Evaluate(parameters), 2.0, 1e-5); + + parameters = "-0.1425 8.3228 0.1724 -0.3374 0.1548;" + "0.1435 0.0009 -0.1736 0.3356 -0.1544"; + BOOST_REQUIRE_CLOSE(svmf.Evaluate(parameters), 0.0, 1e-5); + + parameters = "100 3 4 5 23;" + "43 54 67 32 64"; + BOOST_REQUIRE_CLOSE(svmf.Evaluate(parameters), 85.33333333, 1e-5); + + parameters = "3 71 22 12 6;" + "100 39 30 57 22"; + BOOST_REQUIRE_CLOSE(svmf.Evaluate(parameters), 11.0, 1e-5); +} + +/** + * A complicated test for the SparseSVMFunction for binary-class + * classification. + */ +BOOST_AUTO_TEST_CASE(SparseSVMFunctionRandomBinaryEvaluate) +{ const size_t points = 1000; - const size_t trials = 50; + const size_t trials = 25; + const size_t inputSize = 10; + const size_t numClasses = 2; // Initialize a random dataset. - arma::sp_mat data; - data = arma::sprandu(dimension, points, 0.2); + arma::mat data; + data.randu(inputSize, points); - // Create random response. - arma::Row responses(points); - for (size_t i = 0; i < points; ++i) - responses(i) = math::RandInt(0, 2); + // Create random class labels. + arma::Row labels(points); + for (size_t i = 0; i < points; i++) + labels(i) = math::RandInt(0, numClasses); - // Create a SparseSVMFunction. - SparseSVMFunction<> svm(data, responses, 0.0); + // Create a SparseSVMFunction, Regularization term ignored. + SparseSVMFunction svmf(data, labels, numClasses, + 0.0 /* no regularization */); - // Run a bunch of trials. + // Run a number of trials. + for (size_t i = 0; i < trials; ++i) { + // Create a random set of parameters. + arma::mat parameters; + parameters.randu(numClasses, inputSize); + + // Hand-calculate the loss function + double hingeLoss = 0; + + // Compute error for each training example. + for (size_t j = 0; j < points; ++j) { + arma::mat score = parameters * data.col(j); + double correct = score[labels(j)]; + for (size_t k = 0; k < numClasses; ++k) { + if (k == labels[j]) + continue; + double margin = score[k] - correct + 1; + if (margin > 0) + hingeLoss += margin; + } + } + hingeLoss /= points; + + // Compare with the value returned by the function. + BOOST_REQUIRE_CLOSE(svmf.Evaluate(parameters), hingeLoss, 1e-5); + } +} + +/** + * A complicated test for the SparseSVMFunction for multi-class + * classification. + */ +BOOST_AUTO_TEST_CASE(SparseSVMFunctionRandomEvaluate) +{ + const size_t points = 1000; + const size_t trials = 25; + const size_t inputSize = 10; + const size_t numClasses = 5; + + // Initialize a random dataset. + arma::mat data; + data.randu(inputSize, points); + + // Create random class labels. + arma::Row labels(points); + for (size_t i = 0; i < points; i++) + labels(i) = math::RandInt(0, numClasses); + + // Create a SparseSVMFunction, Regularization term ignored. + SparseSVMFunction svmf(data, labels, numClasses, + 0.0 /* no regularization */); + + // Run a number of trials. + for (size_t i = 0; i < trials; ++i) { + // Create a random set of parameters. + arma::mat parameters; + parameters.randu(numClasses, inputSize); + + // Hand-calculate the loss function + double hingeLoss = 0; + + // Compute error for each training example. + for (size_t j = 0; j < points; ++j) { + arma::mat score = parameters * data.col(j); + double correct = score[labels(j)]; + for (size_t k = 0; k < numClasses; ++k) { + if (k == labels[j]) + continue; + double margin = score[k] - correct + 1; + if (margin > 0) + hingeLoss += margin; + } + } + hingeLoss /= points; + + // Compare with the value returned by the function. + BOOST_REQUIRE_CLOSE(svmf.Evaluate(parameters), hingeLoss, 1e-5); + } +} + +/** + * Test regularization for the SparseSVMFunction Evaluate() + * function. + */ +BOOST_AUTO_TEST_CASE(SparseSVMFunctionRegularizationEvaluate) +{ + const size_t points = 1000; + const size_t trials = 50; + const size_t inputSize = 10; + const size_t numClasses = 5; + + // Initialize a random dataset. + arma::mat data; + data.randu(inputSize, points); + + // Create random class labels. + arma::Row labels(points); + for (size_t i = 0; i < points; i++) + labels(i) = math::RandInt(0, numClasses); + + // 3 objects for comparing regularization costs. + SparseSVMFunction svmfNoReg(data, labels, numClasses, 0); + SparseSVMFunction svmfSmallReg(data, labels, numClasses, 1); + SparseSVMFunction svmfBigReg(data, labels, numClasses, 20); + + // Run a number of trials. for (size_t i = 0; i < trials; i++) { - // Generate a random set of parameters. + // Create a random set of parameters. arma::mat parameters; - parameters = arma::randu(1, dimension + 1); + parameters.randu(numClasses, inputSize); - // Hand-calculate the Hinge Loss Function. - double hingeloss = 0.0; - for (size_t j = 0; j < points; j++) + double wL2SquaredNorm; + wL2SquaredNorm = arma::accu(parameters % parameters); + + // Calculate regularization terms. + const double smallRegTerm = 0.5 * wL2SquaredNorm; + const double bigRegTerm = 10 * wL2SquaredNorm; + + BOOST_REQUIRE_CLOSE(svmfNoReg.Evaluate(parameters) + smallRegTerm, + svmfSmallReg.Evaluate(parameters), 1e-5); + BOOST_REQUIRE_CLOSE(svmfNoReg.Evaluate(parameters) + bigRegTerm, + svmfBigReg.Evaluate(parameters), 1e-5); + } +} + +BOOST_AUTO_TEST_CASE(SparseSVMFunctionGradient) +{ + const size_t points = 1000; + const size_t inputSize = 10; + const size_t numClasses = 3; + + // Initialize a random dataset. + arma::mat data; + data.randu(inputSize, points); + + // Create random class labels. + arma::Row labels(points); + for (size_t i = 0; i < points; i++) + labels(i) = math::RandInt(0, numClasses); + + // 2 objects for 2 terms in the cost function. Each term contributes towards + // the gradient and thus need to be checked independently. + SparseSVMFunction svmf1(data, labels, numClasses, 0); + SparseSVMFunction svmf2(data, labels, numClasses, 10); + + // Create a random set of parameters. + arma::mat parameters; + parameters.randu(numClasses, inputSize); + + // Get gradients for the current parameters. + arma::mat gradient1, gradient2; + svmf1.Gradient(parameters, gradient1); + svmf2.Gradient(parameters, gradient2); + + // Perturbation constant. + const double epsilon = 0.001; + double costPlus1, costMinus1, numGradient1; + double costPlus2, costMinus2, numGradient2; + + + // For each parameter. + for (size_t i = 0; i < numClasses; i++) + { + for (size_t j = 0; j < inputSize; j++) { - hingeloss += std::max(0.0, 1 - (2 * (double)responses(j) - 1) * - (arma::dot(data.col(j), parameters.head_cols(parameters.n_cols - 1)) - + parameters(parameters.n_cols - 1))); - } - hingeloss /= points; + // Perturb parameter with a positive constant and get costs. + parameters(i, j) += epsilon; + costPlus1 = svmf1.Evaluate(parameters); + costPlus2 = svmf2.Evaluate(parameters); - BOOST_REQUIRE_CLOSE(svm.Evaluate(parameters), hingeloss, 1e-5); + // Perturb parameter with a negative constant and get costs. + parameters(i, j) -= 2 * epsilon; + costMinus1 = svmf1.Evaluate(parameters); + costMinus2 = svmf2.Evaluate(parameters); + + // Compute numerical gradients using the costs calculated above. + numGradient1 = (costPlus1 - costMinus1) / (2 * epsilon); + numGradient2 = (costPlus2 - costMinus2) / (2 * epsilon); + + // Restore the parameter value. + parameters(i, j) += epsilon; + + // Compare numerical and backpropagation gradient values. + BOOST_REQUIRE_CLOSE(numGradient1, gradient1(i, j), 1e-5); + BOOST_REQUIRE_CLOSE(numGradient2, gradient2(i, j), 1e-5); + } } }