Added new test for SparseSVMFunction.

src/mlpack/methods/sparse_svm/sparse_svm.hpp

@@ -55,15 +55,38 @@ class SparseSVM
             OptimizerType optimizer = OptimizerType());
 
   /**
-   * Classify the given points, returning  predicted class label  for 
-   * each data point. 
+   * Classify the given points, returning the predicted labels for each point.
    *
-   * @param dataset Matrix of data points to be classified.
+   * @param data Set of points to classify.
    * @param labels Predicted labels for each point.
    */
-  void Classify(const MatType& dataset,
+  void Classify(const MatType& data,
                 arma::Row<size_t>& labels) const;
 
+  /**
+   * Classify the given points, returning class scores and predicted
+   * class label for each point.
+   * The function calculates the scores for every class, given a data
+   * point. It then chooses the class which has the highest probability among
+   * all.
+   *
+   * @param data Matrix of data points to be classified.
+   * @param labels Predicted labels for each point.
+   * @param scores Class probabilities for each point.
+   */
+  void Classify(const MatType& data,
+                arma::Row<size_t>& labels,
+                arma::mat& scores) const;
+
+    /**
+   * Classify the given points, returning class scores for each point.
+   *
+   * @param data Matrix of data points to be classified.
+   * @param scores Class scores for each point.
+   */
+    void Classify(const MatType& data,
+                  arma::mat& scores) const;
+
   /**
    * Computes accuracy of the learned model given the feature data and the
    * labels associated with each data point. Predictions are made using the
@@ -94,11 +117,25 @@ class SparseSVM
                const double lambda = 0.0001,
                OptimizerType optimizer = OptimizerType());
 
+
+  //! Sets the number of classes.
+  size_t& NumClasses() { return numClasses; }
+  //! Gets the number of classes.
+  size_t NumClasses() const { return numClasses; }
+
+  //! Sets the regularization parameter.
+  double& Lambda() { return lambda; }
+  //! Gets the regularization parameter.
+  double Lambda() const { return lambda; }
+
   //! Set the model parameters.
   arma::mat& Parameters() { return parameters; }
   //! Get the model parameters.
   const arma::mat& Parameters() const { return parameters; }
 
+  //! Gets the features size of the training data
+  size_t FeatureSize() const { return parameters.n_cols; }
+
   /**
    * Serialize the SparseSVM model.
    */
@@ -106,11 +143,17 @@ class SparseSVM
   void serialize(Archive& ar, const unsigned int /* version */)
   {
     ar & BOOST_SERIALIZATION_NVP(parameters);
+    ar & BOOST_SERIALIZATION_NVP(numClasses);
+    ar & BOOST_SERIALIZATION_NVP(lambda);
   }
 
  private:
   //! Parameters after optimization.
   arma::mat parameters;
+  //! Number of classes.
+  size_t numClasses;
+  //! L2-Regularization constant.
+  double lambda;
 };
 
 } // namespace svm

src/mlpack/methods/sparse_svm/sparse_svm_function.hpp

@@ -174,11 +174,6 @@ class SparseSVMFunction
   //! Modify the dataset.
   arma::sp_mat& Dataset() { return dataset; }
 
-  //! Get the labels.
-  const arma::vec& Labels() const { return labels; }
-  //! Modify the labels.
-  arma::vec& Labels() { return labels; }
-
   //! Sets the regularization parameter.
   double& Lambda() { return lambda; }
   //! Gets the regularization parameter.
@@ -201,9 +196,6 @@ class SparseSVMFunction
   //! Number of Classes.
   size_t numClasses;
 
-  //! The labels, y_i.
-  arma::Row<size_t> labels;
-
   //! The regularization parameter for L2-regularization.
   double lambda;
 };

src/mlpack/methods/sparse_svm/sparse_svm_function_impl.hpp

@@ -30,8 +30,6 @@ SparseSVMFunction<MatType>::SparseSVMFunction(
   const size_t numClasses,
   const double lambda) :
   dataset(math::MakeAlias(const_cast<MatType&>(dataset), false)),
-  labels(math::MakeAlias(const_cast<arma::Row<size_t>&>(labels),
-      false)),
   numClasses(numClasses),
   lambda(lambda)
 {
@@ -158,12 +156,12 @@ double SparseSVMFunction<MatType>::Evaluate(
   // Calculate the loss and regularization terms.
   double loss, regularization, cost;
 
-  arma::mat scores = dataset.t() * parameters;
-  arma::mat correct = scores
-      % arma::conv_to<arma::mat>::from(groundTruth).t();
-  arma::mat margin = scores - arma::repmat(correct
-      * arma::ones(numClasses), 1, numClasses) + 1
-      - groundTruth.t();
+  arma::mat scores = parameters * dataset;
+  arma::mat correct = arma::conv_to<arma::mat>::from(scores
+      % groundTruth);
+  arma::mat margin = scores
+      - arma::repmat(arma::ones(numClasses).t() * correct, numClasses, 1)
+      + 1 - groundTruth;
 
   // The Hinge Loss Function
   loss = arma::accu(arma::clamp(margin, 0.0, margin.max()));
@@ -188,13 +186,12 @@ double SparseSVMFunction<MatType>::Evaluate(
   // Calculate the loss and regularization terms.
   double loss, regularization, cost;
 
-  arma::mat scores = dataset.cols(firstId, lastId).t()
-      * parameters;
-  arma::mat correct = scores
-      % arma::conv_to<arma::mat>::from(groundTruth).cols(firstId, lastId).t();
-  arma::mat margin = scores - arma::repmat(correct
-      * arma::ones(numClasses), 1, numClasses) + 1
-      - arma::conv_to<arma::mat>::from(groundTruth).cols(firstId, lastId).t();
+  arma::mat scores = parameters * dataset.cols(firstId, lastId);
+  arma::mat correct = arma::conv_to<arma::mat>::from(scores
+      % groundTruth.cols(firstId, lastId));
+  arma::mat margin = scores
+      - arma::repmat(arma::ones(numClasses).t() * correct, numClasses, 1)
+      + 1 - groundTruth.cols(firstId, lastId);
 
   // The Hinge Loss Function
   loss = arma::accu(arma::clamp(margin, 0.0, margin.max()));
@@ -214,23 +211,23 @@ void SparseSVMFunction<MatType>::Gradient(
     const arma::mat& parameters,
     GradType& gradient)
 {
-  arma::mat scores = dataset.t() * parameters;
-  arma::mat correct = scores
-      % arma::conv_to<arma::mat>::from(groundTruth).t();
-  arma::mat margin = scores - arma::repmat(correct
-      * arma::ones(numClasses), 1, numClasses) + 1
-      - groundTruth.t();
+  arma::mat scores = parameters * dataset;
+  arma::mat correct = arma::conv_to<arma::mat>::from(scores
+      % groundTruth);
+  arma::mat margin = scores
+      - arma::repmat(arma::ones(numClasses).t() * correct, numClasses, 1)
+      + 1 - groundTruth;
 
   // For each sample, find the total number of classes where
   // ( margin > 0 )
   arma::mat mask = margin.for_each([](arma::mat::elem_type& val)
       { val = (val > 0) ? 1: 0; });
 
-  arma::mat incorrectLabels = arma::conv_to<arma::mat>::from(groundTruth).t()
-      % (-arma::repmat(arma::sum(mask, 1), 1, numClasses));
+  arma::sp_mat incorrectLabels = groundTruth
+      % (-arma::repmat(arma::sum(mask), numClasses, 1));
   arma::mat difference = incorrectLabels + mask;
 
-  gradient = dataset * difference;
+  gradient = difference * dataset.t();
   gradient /= dataset.n_cols;
 
   // Adding the regularization contribution to the gradient.
@@ -246,25 +243,23 @@ void SparseSVMFunction<MatType>::Gradient(
     const size_t batchSize)
 {
   const size_t lastId = firstId + batchSize - 1;
-  arma::mat scores = dataset.cols(firstId, lastId).t()
-      * parameters;
-  arma::mat correct = scores
-      % arma::conv_to<arma::mat>::from(groundTruth).cols(firstId, lastId).t();
-  arma::mat margin = scores - arma::repmat(correct
-      * arma::ones(numClasses), 1, numClasses) + 1
-      - arma::conv_to<arma::mat>::from(groundTruth).cols(firstId, lastId).t();
+  arma::mat scores = parameters * dataset.cols(firstId, lastId);
+  arma::mat correct = arma::conv_to<arma::mat>::from(scores
+      % groundTruth.cols(firstId, lastId));
+  arma::mat margin = scores
+      - arma::repmat(arma::ones(numClasses).t() * correct, numClasses, 1)
+      + 1 - groundTruth.cols(firstId, lastId);
 
   // For each sample, find the total number of classes where
   // ( margin > 0 )
   arma::mat mask = margin.for_each([](arma::mat::elem_type& val)
       { val = (val > 0) ? 1: 0; });
 
-  arma::mat incorrectLabels =
-      arma::conv_to<arma::mat>::from(groundTruth).cols(firstId, lastId).t() %
-      (-arma::repmat(arma::sum(mask.rows(firstId, lastId), 1), 1, numClasses));
+  arma::sp_mat incorrectLabels = groundTruth.cols(firstId, lastId)
+      % (-arma::repmat(arma::sum(mask.cols(firstId, lastId)), numClasses, 1));
   arma::mat difference = incorrectLabels + mask;
 
-  gradient = dataset.cols(firstId, lastId) * difference;
+  gradient = difference * dataset.cols(firstId, lastId).t();
   gradient /= batchSize;
 
   // Adding the regularization contribution to the gradient.
@@ -279,23 +274,23 @@ double SparseSVMFunction<MatType>::EvaluateWithGradient(
 {
   double loss, regularization, cost;
 
-  arma::mat scores = dataset.t() * parameters;
-  arma::mat correct = scores
-      % arma::conv_to<arma::mat>::from(groundTruth).t();
-  arma::mat margin = scores - arma::repmat(correct
-       * arma::ones(numClasses), 1, numClasses) + 1
-       - groundTruth.t();
+  arma::mat scores = parameters * dataset;
+  arma::mat correct = arma::conv_to<arma::mat>::from(scores
+      % groundTruth);
+  arma::mat margin = scores
+      - arma::repmat(arma::ones(numClasses).t() * correct, numClasses, 1)
+      + 1 - groundTruth;
 
   // For each sample, find the total number of classes where
   // ( margin > 0 )
   arma::mat mask = margin.for_each([](arma::mat::elem_type& val)
       { val = (val > 0) ? 1: 0; });
 
-  arma::mat incorrectLabels = arma::conv_to<arma::mat>::from(groundTruth).t()
-      % (-arma::repmat(arma::sum(mask, 1), 1, numClasses));
+  arma::sp_mat incorrectLabels = groundTruth
+       % (-arma::repmat(arma::sum(mask), numClasses, 1));
   arma::mat difference = incorrectLabels + mask;
 
-  gradient = dataset * difference;
+  gradient = difference * dataset.t();
   gradient /= dataset.n_cols;
 
   // Adding the regularization contribution to the gradient.
@@ -326,25 +321,23 @@ double SparseSVMFunction<MatType>::EvaluateWithGradient(
   // Calculate the loss and regularization terms.
   double loss, regularization, cost;
 
-  arma::mat scores = dataset.cols(firstId, lastId).t()
-                     * parameters;
-  arma::mat correct = scores
-      % arma::conv_to<arma::mat>::from(groundTruth).cols(firstId, lastId).t();
-  arma::mat margin = scores - arma::repmat(correct
-      * arma::ones(numClasses), 1, numClasses) + 1
-      - arma::conv_to<arma::mat>::from(groundTruth).cols(firstId, lastId).t();
+  arma::mat scores = parameters * dataset.cols(firstId, lastId);
+  arma::mat correct = arma::conv_to<arma::mat>::from(scores
+      % groundTruth.cols(firstId, lastId));
+  arma::mat margin = scores
+      - arma::repmat(arma::ones(numClasses).t() * correct, numClasses, 1)
+      + 1 - groundTruth.cols(firstId, lastId);
 
   // For each sample, find the total number of classes where
   // ( margin > 0 )
   arma::mat mask = margin.for_each([](arma::mat::elem_type& val)
       { val = (val > 0) ? 1: 0; });
 
-  arma::mat incorrectLabels =
-      arma::conv_to<arma::mat>::from(groundTruth).cols(firstId, lastId).t() %
-      (-arma::repmat(arma::sum(mask.rows(firstId, lastId), 1), 1, numClasses));
+  arma::sp_mat incorrectLabels = groundTruth.cols(firstId, lastId)
+      % (-arma::repmat(arma::sum(mask.cols(firstId, lastId)), numClasses, 1));
   arma::mat difference = incorrectLabels + mask;
 
-  gradient = dataset.cols(firstId, lastId) * difference;
+  gradient = difference * dataset.cols(firstId, lastId).t();
   gradient /= batchSize;
 
   // Adding the regularization contribution to the gradient.

src/mlpack/methods/sparse_svm/sparse_svm_impl.hpp

@@ -25,7 +25,9 @@ SparseSVM<MatType>::SparseSVM(
     const arma::Row<size_t>& labels,
     const size_t numClasses,
     const double lambda,
-    OptimizerType optimizer)
+    OptimizerType optimizer) :
+    numClasses(numClasses),
+    lambda(lambda)
 {
   Train(data, labels, numClasses, lambda, optimizer);
 }
@@ -39,8 +41,8 @@ double SparseSVM<MatType>::Train(
     const double lambda,
     OptimizerType optimizer)
 {
-  SparseSVMFunction<MatType> svm(arma::conv_to<arma::sp_mat>::from(data),
-                                 labels, numClasses, lambda);
+  SparseSVMFunction<MatType> svm(data, labels,
+      numClasses, lambda);
   if (parameters.is_empty())
     parameters = svm.InitialPoint();
 
@@ -57,13 +59,81 @@ double SparseSVM<MatType>::Train(
 
 template <typename MatType>
 void SparseSVM<MatType>::Classify(
-    const MatType& dataset,
-    arma::Row<size_t>& labels)
+    const MatType& data,
+    arma::Row<size_t> &labels)
 const
 {
-  // Classify each point of dataset into their suitable class.
-  labels = arma::conv_to<arma::Row<size_t>>::from(
-      arma::index_max(parameters.t() * dataset));
+  arma::mat scores;
+  Classify(data, scores);
+
+  // Prepare necessary data
+  labels.zeros(data.n_cols);
+  double maxScore = 0;
+
+  // For each test input.
+  for (size_t i = 0; i < data.n_cols; ++i)
+  {
+    // For each class.
+    for (size_t j = 0; j < numClasses; ++j) {
+      // If a higher class probability is encountered, change score.
+      if (scores(j, i) > maxScore)
+      {
+        maxScore = scores(j, i);
+        labels(i) = j;
+      }
+    }
+
+    // Set maximum probability to zero for next input.
+    maxScore = 0;
+  }
+}
+
+template <typename MatType>
+void SparseSVM<MatType>::Classify(
+        const MatType& data,
+        arma::Row<size_t> &labels,
+        arma::mat& scores)
+const
+{
+  Classify(data, scores);
+
+  // Prepare necessary data
+  labels.zeros(data.n_cols);
+  double maxScore = 0;
+
+  // For each test input.
+  for (size_t i = 0; i < data.n_cols; ++i)
+  {
+    // For each class.
+    for (size_t j = 0; j < numClasses; ++j) {
+      // If a higher class probability is encountered, change score.
+      if (scores(j, i) > maxScore)
+      {
+        maxScore = scores(j, i);
+        labels(i) = j;
+      }
+    }
+
+    // Set maximum probability to zero for next input.
+    maxScore = 0;
+  }
+}
+
+template <typename MatType>
+void SparseSVM<MatType>::Classify(
+        const MatType& data,
+        arma::mat& scores)
+const
+{
+  if (data.n_rows != FeatureSize())
+  {
+    std::ostringstream oss;
+    oss << "SparseSVM::Classify(): dataset has " << data.n_rows
+        << " dimensions, but model has " << FeatureSize() << " dimensions!";
+    throw std::invalid_argument(oss.str());
+  }
+
+  scores = parameters * data;
 }
 
 template <typename MatType>

src/mlpack/tests/CMakeLists.txt

@@ -96,6 +96,7 @@ add_executable(mlpack_test
   sort_policy_test.cpp
   sparse_autoencoder_test.cpp
   sparse_coding_test.cpp
+  sparse_svm_test.cpp
   spill_tree_test.cpp
   split_data_test.cpp
   svd_batch_test.cpp

src/mlpack/tests/sparse_svm_test.cpp

@@ -23,44 +23,256 @@ using namespace mlpack::distribution;
 BOOST_AUTO_TEST_SUITE(SparseSVMTest);
 
 /**
- * A more complicated test for the SparseSVMFunction.
+ * A simple test for SparseSVMFunction
  */
 BOOST_AUTO_TEST_CASE(SparseSVMFunctionEvaluate)
 {
-  const size_t dimension = 10;
+  // A very simple fake dataset
+  arma::mat dataset = "2 0 0;"
+                      "0 0 0;"
+                      "0 2 1;"
+                      "1 0 2;"
+                      "0 1 0";
+
+  //  Corresponding labels
+  arma::Row<size_t> labels = "1 0 1";
+
+  SparseSVMFunction<arma::mat> svmf(dataset, labels, 2,
+      0.0 /* no regularization */);
+
+  // These were hand-calculated using Python.
+  arma::mat parameters = "1 1 1 1 1;"
+                         " 1 1 1 1 1";
+  BOOST_REQUIRE_CLOSE(svmf.Evaluate(parameters), 1.0, 1e-5);
+
+  parameters = "2 0 1 2 2;"
+               " 1 2 2 2 2";
+  BOOST_REQUIRE_CLOSE(svmf.Evaluate(parameters), 2.0, 1e-5);
+
+  parameters = "-0.1425 8.3228 0.1724 -0.3374 0.1548;"
+               "0.1435 0.0009 -0.1736 0.3356 -0.1544";
+  BOOST_REQUIRE_CLOSE(svmf.Evaluate(parameters), 0.0, 1e-5);
+
+  parameters = "100 3 4 5 23;"
+               "43 54 67 32 64";
+  BOOST_REQUIRE_CLOSE(svmf.Evaluate(parameters), 85.33333333, 1e-5);
+
+  parameters = "3 71 22 12 6;"
+               "100 39 30 57 22";
+  BOOST_REQUIRE_CLOSE(svmf.Evaluate(parameters), 11.0, 1e-5);
+}
+
+/**
+ * A complicated test for the SparseSVMFunction for binary-class
+ * classification.
+ */
+BOOST_AUTO_TEST_CASE(SparseSVMFunctionRandomBinaryEvaluate)
+{
   const size_t points = 1000;
-  const size_t trials = 50;
+  const size_t trials = 25;
+  const size_t inputSize = 10;
+  const size_t numClasses = 2;
 
   // Initialize a random dataset.
-  arma::sp_mat data;
-  data = arma::sprandu(dimension, points, 0.2);
+  arma::mat data;
+  data.randu(inputSize, points);
 
-  // Create random response.
-  arma::Row<size_t> responses(points);
-  for (size_t i = 0; i < points; ++i)
-    responses(i) = math::RandInt(0, 2);
+  // Create random class labels.
+  arma::Row<size_t> labels(points);
+  for (size_t i = 0; i < points; i++)
+    labels(i) = math::RandInt(0, numClasses);
 
-  // Create a SparseSVMFunction.
-  SparseSVMFunction<> svm(data, responses, 0.0);
+  // Create a SparseSVMFunction, Regularization term ignored.
+  SparseSVMFunction<arma::mat> svmf(data, labels, numClasses,
+      0.0 /* no regularization */);
 
-  // Run a bunch of trials.
+  // Run a number of trials.
+  for (size_t i = 0; i < trials; ++i) {
+    // Create a random set of parameters.
+    arma::mat parameters;
+    parameters.randu(numClasses, inputSize);
+
+    // Hand-calculate the loss function
+    double hingeLoss = 0;
+
+    // Compute error for each training example.
+    for (size_t j = 0; j < points; ++j) {
+      arma::mat score = parameters * data.col(j);
+      double correct = score[labels(j)];
+      for (size_t k = 0; k < numClasses; ++k) {
+        if (k == labels[j])
+          continue;
+        double margin = score[k] - correct + 1;
+        if (margin > 0)
+          hingeLoss += margin;
+      }
+    }
+    hingeLoss /= points;
+
+    // Compare with the value returned by the function.
+    BOOST_REQUIRE_CLOSE(svmf.Evaluate(parameters), hingeLoss, 1e-5);
+  }
+}
+
+/**
+ * A complicated test for the SparseSVMFunction for multi-class
+ * classification.
+ */
+BOOST_AUTO_TEST_CASE(SparseSVMFunctionRandomEvaluate)
+{
+  const size_t points = 1000;
+  const size_t trials = 25;
+  const size_t inputSize = 10;
+  const size_t numClasses = 5;
+
+  // Initialize a random dataset.
+  arma::mat data;
+  data.randu(inputSize, points);
+
+  // Create random class labels.
+  arma::Row<size_t> labels(points);
+  for (size_t i = 0; i < points; i++)
+    labels(i) = math::RandInt(0, numClasses);
+
+  // Create a SparseSVMFunction, Regularization term ignored.
+  SparseSVMFunction<arma::mat> svmf(data, labels, numClasses,
+      0.0 /* no regularization */);
+
+  // Run a number of trials.
+  for (size_t i = 0; i < trials; ++i) {
+    // Create a random set of parameters.
+    arma::mat parameters;
+    parameters.randu(numClasses, inputSize);
+
+    // Hand-calculate the loss function
+    double hingeLoss = 0;
+
+    // Compute error for each training example.
+    for (size_t j = 0; j < points; ++j) {
+      arma::mat score = parameters * data.col(j);
+      double correct = score[labels(j)];
+      for (size_t k = 0; k < numClasses; ++k) {
+        if (k == labels[j])
+          continue;
+        double margin = score[k] - correct + 1;
+        if (margin > 0)
+          hingeLoss += margin;
+      }
+    }
+    hingeLoss /= points;
+
+    // Compare with the value returned by the function.
+    BOOST_REQUIRE_CLOSE(svmf.Evaluate(parameters), hingeLoss, 1e-5);
+  }
+}
+
+/**
+ * Test regularization for the SparseSVMFunction Evaluate()
+ * function.
+ */
+BOOST_AUTO_TEST_CASE(SparseSVMFunctionRegularizationEvaluate)
+{
+  const size_t points = 1000;
+  const size_t trials = 50;
+  const size_t inputSize = 10;
+  const size_t numClasses = 5;
+
+  // Initialize a random dataset.
+  arma::mat data;
+  data.randu(inputSize, points);
+
+  // Create random class labels.
+  arma::Row<size_t> labels(points);
+  for (size_t i = 0; i < points; i++)
+    labels(i) = math::RandInt(0, numClasses);
+
+  // 3 objects for comparing regularization costs.
+  SparseSVMFunction<arma::mat> svmfNoReg(data, labels, numClasses, 0);
+  SparseSVMFunction<arma::mat> svmfSmallReg(data, labels, numClasses, 1);
+  SparseSVMFunction<arma::mat> svmfBigReg(data, labels, numClasses, 20);
+
+  // Run a number of trials.
   for (size_t i = 0; i < trials; i++)
   {
-    // Generate a random set of parameters.
+    // Create a random set of parameters.
     arma::mat parameters;
-    parameters = arma::randu<arma::mat>(1, dimension + 1);
+    parameters.randu(numClasses, inputSize);
 
-    // Hand-calculate the Hinge Loss Function.
-    double hingeloss = 0.0;
-    for (size_t j = 0; j < points; j++)
+    double wL2SquaredNorm;
+    wL2SquaredNorm = arma::accu(parameters % parameters);
+
+    // Calculate regularization terms.
+    const double smallRegTerm = 0.5 * wL2SquaredNorm;
+    const double bigRegTerm = 10 * wL2SquaredNorm;
+
+    BOOST_REQUIRE_CLOSE(svmfNoReg.Evaluate(parameters) + smallRegTerm,
+                        svmfSmallReg.Evaluate(parameters), 1e-5);
+    BOOST_REQUIRE_CLOSE(svmfNoReg.Evaluate(parameters) + bigRegTerm,
+                        svmfBigReg.Evaluate(parameters), 1e-5);
+  }
+}
+
+BOOST_AUTO_TEST_CASE(SparseSVMFunctionGradient)
+{
+  const size_t points = 1000;
+  const size_t inputSize = 10;
+  const size_t numClasses = 3;
+
+  // Initialize a random dataset.
+  arma::mat data;
+  data.randu(inputSize, points);
+
+  // Create random class labels.
+  arma::Row<size_t> labels(points);
+  for (size_t i = 0; i < points; i++)
+    labels(i) = math::RandInt(0, numClasses);
+
+  // 2 objects for 2 terms in the cost function. Each term contributes towards
+  // the gradient and thus need to be checked independently.
+  SparseSVMFunction<arma::mat> svmf1(data, labels, numClasses, 0);
+  SparseSVMFunction<arma::mat> svmf2(data, labels, numClasses, 10);
+
+  // Create a random set of parameters.
+  arma::mat parameters;
+  parameters.randu(numClasses, inputSize);
+
+  // Get gradients for the current parameters.
+  arma::mat gradient1, gradient2;
+  svmf1.Gradient(parameters, gradient1);
+  svmf2.Gradient(parameters, gradient2);
+
+  // Perturbation constant.
+  const double epsilon = 0.001;
+  double costPlus1, costMinus1, numGradient1;
+  double costPlus2, costMinus2, numGradient2;
+
+
+  // For each parameter.
+  for (size_t i = 0; i < numClasses; i++)
+  {
+    for (size_t j = 0; j < inputSize; j++)
     {
-      hingeloss += std::max(0.0, 1 - (2 * (double)responses(j) - 1) *
-          (arma::dot(data.col(j), parameters.head_cols(parameters.n_cols - 1))
-          + parameters(parameters.n_cols - 1)));
-    }
-    hingeloss /= points;
+      // Perturb parameter with a positive constant and get costs.
+      parameters(i, j) += epsilon;
+      costPlus1 = svmf1.Evaluate(parameters);
+      costPlus2 = svmf2.Evaluate(parameters);
 
-    BOOST_REQUIRE_CLOSE(svm.Evaluate(parameters), hingeloss, 1e-5);
+      // Perturb parameter with a negative constant and get costs.
+      parameters(i, j) -= 2 * epsilon;
+      costMinus1 = svmf1.Evaluate(parameters);
+      costMinus2 = svmf2.Evaluate(parameters);
+
+      // Compute numerical gradients using the costs calculated above.
+      numGradient1 = (costPlus1 - costMinus1) / (2 * epsilon);
+      numGradient2 = (costPlus2 - costMinus2) / (2 * epsilon);
+
+      // Restore the parameter value.
+      parameters(i, j) += epsilon;
+
+      // Compare numerical and backpropagation gradient values.
+      BOOST_REQUIRE_CLOSE(numGradient1, gradient1(i, j), 1e-5);
+      BOOST_REQUIRE_CLOSE(numGradient2, gradient2(i, j), 1e-5);
+    }
   }
 }