From d3b66f4b367483c3e305dc9ebf2efb072b59004b Mon Sep 17 00:00:00 2001 From: Sri Madhan M <22276366+srimadhan11@users.noreply.github.com> Date: Mon, 8 Nov 2021 02:49:09 +0530 Subject: [PATCH] add ROC AUC for binary classification --- src/mlpack/core/cv/metrics/CMakeLists.txt | 2 + src/mlpack/core/cv/metrics/roc_auc_score.hpp | 63 +++++++++++++++ .../core/cv/metrics/roc_auc_score_impl.hpp | 78 +++++++++++++++++++ src/mlpack/tests/cv_test.cpp | 4 + 4 files changed, 147 insertions(+) create mode 100644 src/mlpack/core/cv/metrics/roc_auc_score.hpp create mode 100644 src/mlpack/core/cv/metrics/roc_auc_score_impl.hpp diff --git a/src/mlpack/core/cv/metrics/CMakeLists.txt b/src/mlpack/core/cv/metrics/CMakeLists.txt index b9edacaf9a..2f908b759e 100644 --- a/src/mlpack/core/cv/metrics/CMakeLists.txt +++ b/src/mlpack/core/cv/metrics/CMakeLists.txt @@ -13,6 +13,8 @@ set(SOURCES precision_impl.hpp recall.hpp recall_impl.hpp + roc_auc_score.hpp + roc_auc_score_impl.hpp r2_score.hpp r2_score_impl.hpp ) diff --git a/src/mlpack/core/cv/metrics/roc_auc_score.hpp b/src/mlpack/core/cv/metrics/roc_auc_score.hpp new file mode 100644 index 0000000000..cacfac7d0b --- /dev/null +++ b/src/mlpack/core/cv/metrics/roc_auc_score.hpp @@ -0,0 +1,63 @@ +/** + * @file core/cv/metrics/roc_auc_score.hpp + * @author Sri Madhan M + * + * The area under Receiver Operating Characteristic curve (ROC-AUC) score. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_CORE_CV_METRICS_ROCAUCSCORE_HPP +#define MLPACK_CORE_CV_METRICS_ROCAUCSCORE_HPP + +#include + +namespace mlpack { +namespace cv { + +/** + * ROC-AUC is a metric of performance for classification algorithms that for + * binary classification is equal to area under the curve formed by + * @f$ (fpr, tpr) @f$, where @f$ fpr @f$ and @f$ tpr @f$ are the true positive + * rate and false positive rate, which is calculated for many different + * thresholds. For each thresholds, @f$ tpr @f$ and @f$ fpr @f$ are calculated + * as, @f$ tpr = tp / (tp + fn) @f$ and @f$ fpr = fp / (fp + tn) @f$, + * where @f$ tp @f$, @f$ tn @f$, @f$ fp @f$ and @f$ fn @f$ are the numbers of + * true positives, true negatives, false positives and false negatives + * respectively. + * + * @tparam PositiveClass Positives are assumed to have labels equal to this + * value. + */ +template +class ROC_AUC +{ + public: + /** + * Run classification and calculate area under the ROC curve. + * + * @param model A classification model. + * @param data Column-major data containing test items. + * @param labels Ground truth (correct) labels for the test items. + */ + template + static double Evaluate(MLAlgorithm& model, + const DataType& data, + const arma::Row& labels); + + /** + * Information for hyper-parameter tuning code. It indicates that we want + * to maximize the metric. + */ + static const bool NeedsMinimization = false; +}; + +} // namespace cv +} // namespace mlpack + +// Include implementation. +#include "roc_auc_score_impl.hpp" + +#endif diff --git a/src/mlpack/core/cv/metrics/roc_auc_score_impl.hpp b/src/mlpack/core/cv/metrics/roc_auc_score_impl.hpp new file mode 100644 index 0000000000..3f2a6a9fff --- /dev/null +++ b/src/mlpack/core/cv/metrics/roc_auc_score_impl.hpp @@ -0,0 +1,78 @@ +/** + * @file core/cv/metrics/roc_auc_score_impl.hpp + * @author Sri Madhan M + * + * Implementation of the area under Receiver Operating Characteristic curve + * (ROC-AUC) score. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_CORE_CV_METRICS_ROCAUCSCORE_IMPL_HPP +#define MLPACK_CORE_CV_METRICS_ROCAUCSCORE_IMPL_HPP + +#include +#include + +namespace mlpack { +namespace cv { + +template +template +double ROC_AUC::Evaluate(MLAlgorithm& model, + const DataType& data, + const arma::Row& labels) +{ + util::CheckSameSizes(data, labels, "ROC_AUC::Evaluate()"); + + arma::mat probabilities; + model.Classify(data, probabilities); + + // compute labels with "1" for positive class and "0" for the other + arma::Col binaryLabels = arma::conv_to>::from( + ((arma::umat) (labels == PC)).row(0)); + + // probability scores of positive class + arma::Col scoresOfPC = arma::conv_to>::from( + probabilities.row(PC)); + + size_t numberOfTrueLabels = arma::sum(binaryLabels); + size_t numberOfFalseLabels = binaryLabels.n_rows - numberOfTrueLabels; + + // sort labels and probabilities, using probability scores + arma::ucolvec sortedScoreIndices = arma::stable_sort_index( + scoresOfPC, "descend"); + arma::Col sortedLabels = binaryLabels(sortedScoreIndices); + arma::Col sortedScores = scoresOfPC(sortedScoreIndices); + + // compute indices of unique probability scores + arma::ucolvec uniqueScoreIndices = arma::find(arma::diff(sortedScores)); + uniqueScoreIndices.insert_rows(uniqueScoreIndices.n_rows, 1); + uniqueScoreIndices(uniqueScoreIndices.n_rows - 1) = scoresOfPC.n_rows - 1; + + // compute true positive rate, and false positive rate + arma::Col cummulativeSum = arma::cumsum(sortedLabels); + cummulativeSum = cummulativeSum(uniqueScoreIndices); + + arma::Col tpr, fpr; + tpr = arma::conv_to>::from(cummulativeSum); + fpr = 1 + uniqueScoreIndices - tpr; + tpr /= numberOfTrueLabels; + fpr /= numberOfFalseLabels; + + // to ensure that the (fpr, tpr) starts at (0, 0) + tpr.insert_rows(0, 1); + fpr.insert_rows(0, 1); + tpr(0) = fpr(0) = 0; + + // compute area under the curve using trapezoidal rule + arma::mat auc = arma::trapz(fpr, tpr); + return auc(0, 0); +} + +} // namespace cv +} // namespace mlpack + +#endif diff --git a/src/mlpack/tests/cv_test.cpp b/src/mlpack/tests/cv_test.cpp index 75f1e87998..aa18af9ccb 100644 --- a/src/mlpack/tests/cv_test.cpp +++ b/src/mlpack/tests/cv_test.cpp @@ -17,6 +17,7 @@ #include #include #include +#include #include #include #include @@ -74,6 +75,9 @@ TEST_CASE("BinaryClassificationMetricsTest", "[CVTest]") double f1 = 2 * 0.6 * 0.75 / (0.6 + 0.75); REQUIRE(F1::Evaluate(lr, data, labels) == Approx(f1).epsilon(1e-7)); + + REQUIRE(ROC_AUC<>::Evaluate(lr, data, labels) + == Approx((double) 2 / 3).epsilon(1e-7)); } /**