add ROC AUC for binary classification

src/mlpack/core/cv/metrics/CMakeLists.txt

@@ -13,6 +13,8 @@ set(SOURCES
   precision_impl.hpp
   recall.hpp
   recall_impl.hpp
+  roc_auc_score.hpp
+  roc_auc_score_impl.hpp
   r2_score.hpp
   r2_score_impl.hpp
 )

src/mlpack/core/cv/metrics/roc_auc_score.hpp

@@ -0,0 +1,63 @@
+/**
+ * @file core/cv/metrics/roc_auc_score.hpp
+ * @author Sri Madhan M
+ *
+ * The area under Receiver Operating Characteristic curve (ROC-AUC) score.
+ *
+ * mlpack is free software; you may redistribute it and/or modify it under the
+ * terms of the 3-clause BSD license.  You should have received a copy of the
+ * 3-clause BSD license along with mlpack.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef MLPACK_CORE_CV_METRICS_ROCAUCSCORE_HPP
+#define MLPACK_CORE_CV_METRICS_ROCAUCSCORE_HPP
+
+#include <mlpack/core.hpp>
+
+namespace mlpack {
+namespace cv {
+
+/**
+ * ROC-AUC is a metric of performance for classification algorithms that for
+ * binary classification is equal to area under the curve formed by
+ * @f$ (fpr, tpr) @f$, where @f$ fpr @f$ and @f$ tpr @f$ are the true positive
+ * rate and false positive rate, which is calculated for many different
+ * thresholds. For each thresholds, @f$ tpr @f$ and @f$ fpr @f$ are calculated
+ * as, @f$ tpr = tp / (tp + fn) @f$ and @f$ fpr = fp / (fp + tn) @f$,
+ * where @f$ tp @f$, @f$ tn @f$, @f$ fp @f$ and @f$ fn @f$ are the numbers of
+ * true positives, true negatives, false positives and false negatives
+ * respectively.
+ *
+ * @tparam PositiveClass Positives are assumed to have labels equal to this
+ *     value.
+ */
+template<size_t PositiveClass = 1>
+class ROC_AUC
+{
+ public:
+  /**
+   * Run classification and calculate area under the ROC curve.
+   *
+   * @param model A classification model.
+   * @param data Column-major data containing test items.
+   * @param labels Ground truth (correct) labels for the test items.
+   */
+  template<typename MLAlgorithm, typename DataType>
+  static double Evaluate(MLAlgorithm& model,
+                         const DataType& data,
+                         const arma::Row<size_t>& labels);
+
+  /**
+   * Information for hyper-parameter tuning code. It indicates that we want
+   * to maximize the metric.
+   */
+  static const bool NeedsMinimization = false;
+};
+
+} // namespace cv
+} // namespace mlpack
+
+// Include implementation.
+#include "roc_auc_score_impl.hpp"
+
+#endif

src/mlpack/core/cv/metrics/roc_auc_score_impl.hpp

@@ -0,0 +1,78 @@
+/**
+ * @file core/cv/metrics/roc_auc_score_impl.hpp
+ * @author Sri Madhan M
+ *
+ * Implementation of the area under Receiver Operating Characteristic curve
+ * (ROC-AUC) score.
+ *
+ * mlpack is free software; you may redistribute it and/or modify it under the
+ * terms of the 3-clause BSD license.  You should have received a copy of the
+ * 3-clause BSD license along with mlpack.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef MLPACK_CORE_CV_METRICS_ROCAUCSCORE_IMPL_HPP
+#define MLPACK_CORE_CV_METRICS_ROCAUCSCORE_IMPL_HPP
+
+#include <mlpack/core/cv/metrics/accuracy.hpp>
+#include <iostream>
+
+namespace mlpack {
+namespace cv {
+
+template<size_t PC /* PositiveClass */>
+template<typename MLAlgorithm, typename DataType>
+double ROC_AUC<PC>::Evaluate(MLAlgorithm& model,
+                              const DataType& data,
+                              const arma::Row<size_t>& labels)
+{
+  util::CheckSameSizes(data, labels, "ROC_AUC::Evaluate()");
+
+  arma::mat probabilities;
+  model.Classify(data, probabilities);
+
+  // compute labels with "1" for positive class and "0" for the other
+  arma::Col<size_t> binaryLabels = arma::conv_to<arma::Col<size_t>>::from(
+                                        ((arma::umat) (labels == PC)).row(0));
+
+  // probability scores of positive class
+  arma::Col<double> scoresOfPC   = arma::conv_to<arma::Col<double>>::from(
+                                        probabilities.row(PC));
+
+  size_t numberOfTrueLabels  = arma::sum(binaryLabels);
+  size_t numberOfFalseLabels = binaryLabels.n_rows - numberOfTrueLabels;
+
+  // sort labels and probabilities, using probability scores
+  arma::ucolvec sortedScoreIndices = arma::stable_sort_index(
+                                        scoresOfPC, "descend");
+  arma::Col<size_t> sortedLabels = binaryLabels(sortedScoreIndices);
+  arma::Col<double> sortedScores = scoresOfPC(sortedScoreIndices);
+
+  // compute indices of unique probability scores
+  arma::ucolvec uniqueScoreIndices = arma::find(arma::diff(sortedScores));
+  uniqueScoreIndices.insert_rows(uniqueScoreIndices.n_rows, 1);
+  uniqueScoreIndices(uniqueScoreIndices.n_rows - 1) = scoresOfPC.n_rows - 1;
+
+  // compute true positive rate, and false positive rate
+  arma::Col<size_t> cummulativeSum = arma::cumsum(sortedLabels);
+  cummulativeSum = cummulativeSum(uniqueScoreIndices);
+
+  arma::Col<double> tpr, fpr;
+  tpr = arma::conv_to<arma::Col<double>>::from(cummulativeSum);
+  fpr = 1 + uniqueScoreIndices - tpr;
+  tpr /= numberOfTrueLabels;
+  fpr /= numberOfFalseLabels;
+
+  // to ensure that the (fpr, tpr) starts at (0, 0)
+  tpr.insert_rows(0, 1);
+  fpr.insert_rows(0, 1);
+  tpr(0) = fpr(0) = 0;
+
+  // compute area under the curve using trapezoidal rule
+  arma::mat auc = arma::trapz(fpr, tpr);
+  return auc(0, 0);
+}
+
+} // namespace cv
+} // namespace mlpack
+
+#endif

src/mlpack/tests/cv_test.cpp

@@ -17,6 +17,7 @@
 #include <mlpack/core/cv/metrics/mse.hpp>
 #include <mlpack/core/cv/metrics/precision.hpp>
 #include <mlpack/core/cv/metrics/recall.hpp>
+#include <mlpack/core/cv/metrics/roc_auc_score.hpp>
 #include <mlpack/core/cv/metrics/r2_score.hpp>
 #include <mlpack/core/cv/metrics/silhouette_score.hpp>
 #include <mlpack/core/cv/simple_cv.hpp>
@@ -74,6 +75,9 @@ TEST_CASE("BinaryClassificationMetricsTest", "[CVTest]")
 
   double f1 = 2 * 0.6 * 0.75 / (0.6 + 0.75);
   REQUIRE(F1<Binary>::Evaluate(lr, data, labels) == Approx(f1).epsilon(1e-7));
+
+  REQUIRE(ROC_AUC<>::Evaluate(lr, data, labels)
+          == Approx((double) 2 / 3).epsilon(1e-7));
 }
 
 /**