From d3b66f4b367483c3e305dc9ebf2efb072b59004b Mon Sep 17 00:00:00 2001 From: Sri Madhan M <22276366+srimadhan11@users.noreply.github.com> Date: Mon, 8 Nov 2021 02:49:09 +0530 Subject: [PATCH 01/10] add ROC AUC for binary classification --- src/mlpack/core/cv/metrics/CMakeLists.txt | 2 + src/mlpack/core/cv/metrics/roc_auc_score.hpp | 63 +++++++++++++++ .../core/cv/metrics/roc_auc_score_impl.hpp | 78 +++++++++++++++++++ src/mlpack/tests/cv_test.cpp | 4 + 4 files changed, 147 insertions(+) create mode 100644 src/mlpack/core/cv/metrics/roc_auc_score.hpp create mode 100644 src/mlpack/core/cv/metrics/roc_auc_score_impl.hpp diff --git a/src/mlpack/core/cv/metrics/CMakeLists.txt b/src/mlpack/core/cv/metrics/CMakeLists.txt index b9edacaf9a..2f908b759e 100644 --- a/src/mlpack/core/cv/metrics/CMakeLists.txt +++ b/src/mlpack/core/cv/metrics/CMakeLists.txt @@ -13,6 +13,8 @@ set(SOURCES precision_impl.hpp recall.hpp recall_impl.hpp + roc_auc_score.hpp + roc_auc_score_impl.hpp r2_score.hpp r2_score_impl.hpp ) diff --git a/src/mlpack/core/cv/metrics/roc_auc_score.hpp b/src/mlpack/core/cv/metrics/roc_auc_score.hpp new file mode 100644 index 0000000000..cacfac7d0b --- /dev/null +++ b/src/mlpack/core/cv/metrics/roc_auc_score.hpp @@ -0,0 +1,63 @@ +/** + * @file core/cv/metrics/roc_auc_score.hpp + * @author Sri Madhan M + * + * The area under Receiver Operating Characteristic curve (ROC-AUC) score. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_CORE_CV_METRICS_ROCAUCSCORE_HPP +#define MLPACK_CORE_CV_METRICS_ROCAUCSCORE_HPP + +#include + +namespace mlpack { +namespace cv { + +/** + * ROC-AUC is a metric of performance for classification algorithms that for + * binary classification is equal to area under the curve formed by + * @f$ (fpr, tpr) @f$, where @f$ fpr @f$ and @f$ tpr @f$ are the true positive + * rate and false positive rate, which is calculated for many different + * thresholds. For each thresholds, @f$ tpr @f$ and @f$ fpr @f$ are calculated + * as, @f$ tpr = tp / (tp + fn) @f$ and @f$ fpr = fp / (fp + tn) @f$, + * where @f$ tp @f$, @f$ tn @f$, @f$ fp @f$ and @f$ fn @f$ are the numbers of + * true positives, true negatives, false positives and false negatives + * respectively. + * + * @tparam PositiveClass Positives are assumed to have labels equal to this + * value. + */ +template +class ROC_AUC +{ + public: + /** + * Run classification and calculate area under the ROC curve. + * + * @param model A classification model. + * @param data Column-major data containing test items. + * @param labels Ground truth (correct) labels for the test items. + */ + template + static double Evaluate(MLAlgorithm& model, + const DataType& data, + const arma::Row& labels); + + /** + * Information for hyper-parameter tuning code. It indicates that we want + * to maximize the metric. + */ + static const bool NeedsMinimization = false; +}; + +} // namespace cv +} // namespace mlpack + +// Include implementation. +#include "roc_auc_score_impl.hpp" + +#endif diff --git a/src/mlpack/core/cv/metrics/roc_auc_score_impl.hpp b/src/mlpack/core/cv/metrics/roc_auc_score_impl.hpp new file mode 100644 index 0000000000..3f2a6a9fff --- /dev/null +++ b/src/mlpack/core/cv/metrics/roc_auc_score_impl.hpp @@ -0,0 +1,78 @@ +/** + * @file core/cv/metrics/roc_auc_score_impl.hpp + * @author Sri Madhan M + * + * Implementation of the area under Receiver Operating Characteristic curve + * (ROC-AUC) score. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_CORE_CV_METRICS_ROCAUCSCORE_IMPL_HPP +#define MLPACK_CORE_CV_METRICS_ROCAUCSCORE_IMPL_HPP + +#include +#include + +namespace mlpack { +namespace cv { + +template +template +double ROC_AUC::Evaluate(MLAlgorithm& model, + const DataType& data, + const arma::Row& labels) +{ + util::CheckSameSizes(data, labels, "ROC_AUC::Evaluate()"); + + arma::mat probabilities; + model.Classify(data, probabilities); + + // compute labels with "1" for positive class and "0" for the other + arma::Col binaryLabels = arma::conv_to>::from( + ((arma::umat) (labels == PC)).row(0)); + + // probability scores of positive class + arma::Col scoresOfPC = arma::conv_to>::from( + probabilities.row(PC)); + + size_t numberOfTrueLabels = arma::sum(binaryLabels); + size_t numberOfFalseLabels = binaryLabels.n_rows - numberOfTrueLabels; + + // sort labels and probabilities, using probability scores + arma::ucolvec sortedScoreIndices = arma::stable_sort_index( + scoresOfPC, "descend"); + arma::Col sortedLabels = binaryLabels(sortedScoreIndices); + arma::Col sortedScores = scoresOfPC(sortedScoreIndices); + + // compute indices of unique probability scores + arma::ucolvec uniqueScoreIndices = arma::find(arma::diff(sortedScores)); + uniqueScoreIndices.insert_rows(uniqueScoreIndices.n_rows, 1); + uniqueScoreIndices(uniqueScoreIndices.n_rows - 1) = scoresOfPC.n_rows - 1; + + // compute true positive rate, and false positive rate + arma::Col cummulativeSum = arma::cumsum(sortedLabels); + cummulativeSum = cummulativeSum(uniqueScoreIndices); + + arma::Col tpr, fpr; + tpr = arma::conv_to>::from(cummulativeSum); + fpr = 1 + uniqueScoreIndices - tpr; + tpr /= numberOfTrueLabels; + fpr /= numberOfFalseLabels; + + // to ensure that the (fpr, tpr) starts at (0, 0) + tpr.insert_rows(0, 1); + fpr.insert_rows(0, 1); + tpr(0) = fpr(0) = 0; + + // compute area under the curve using trapezoidal rule + arma::mat auc = arma::trapz(fpr, tpr); + return auc(0, 0); +} + +} // namespace cv +} // namespace mlpack + +#endif diff --git a/src/mlpack/tests/cv_test.cpp b/src/mlpack/tests/cv_test.cpp index 75f1e87998..aa18af9ccb 100644 --- a/src/mlpack/tests/cv_test.cpp +++ b/src/mlpack/tests/cv_test.cpp @@ -17,6 +17,7 @@ #include #include #include +#include #include #include #include @@ -74,6 +75,9 @@ TEST_CASE("BinaryClassificationMetricsTest", "[CVTest]") double f1 = 2 * 0.6 * 0.75 / (0.6 + 0.75); REQUIRE(F1::Evaluate(lr, data, labels) == Approx(f1).epsilon(1e-7)); + + REQUIRE(ROC_AUC<>::Evaluate(lr, data, labels) + == Approx((double) 2 / 3).epsilon(1e-7)); } /** From 729997f182af7323f2df92ff58969165de104320 Mon Sep 17 00:00:00 2001 From: Sri Madhan M <22276366+srimadhan11@users.noreply.github.com> Date: Sun, 5 Dec 2021 17:38:47 +0530 Subject: [PATCH 02/10] follow design guidelines --- src/mlpack/core/cv/metrics/roc_auc_score.hpp | 4 +- .../core/cv/metrics/roc_auc_score_impl.hpp | 39 +++++++++---------- src/mlpack/tests/cv_test.cpp | 2 +- 3 files changed, 22 insertions(+), 23 deletions(-) diff --git a/src/mlpack/core/cv/metrics/roc_auc_score.hpp b/src/mlpack/core/cv/metrics/roc_auc_score.hpp index cacfac7d0b..8bc4af4011 100644 --- a/src/mlpack/core/cv/metrics/roc_auc_score.hpp +++ b/src/mlpack/core/cv/metrics/roc_auc_score.hpp @@ -29,10 +29,10 @@ namespace cv { * respectively. * * @tparam PositiveClass Positives are assumed to have labels equal to this - * value. + * value. Defaults to 1. */ template -class ROC_AUC +class ROCAUCScore { public: /** diff --git a/src/mlpack/core/cv/metrics/roc_auc_score_impl.hpp b/src/mlpack/core/cv/metrics/roc_auc_score_impl.hpp index 3f2a6a9fff..46d377d638 100644 --- a/src/mlpack/core/cv/metrics/roc_auc_score_impl.hpp +++ b/src/mlpack/core/cv/metrics/roc_auc_score_impl.hpp @@ -14,60 +14,59 @@ #define MLPACK_CORE_CV_METRICS_ROCAUCSCORE_IMPL_HPP #include -#include namespace mlpack { namespace cv { -template +template template -double ROC_AUC::Evaluate(MLAlgorithm& model, - const DataType& data, - const arma::Row& labels) +double ROCAUCScore::Evaluate(MLAlgorithm& model, + const DataType& data, + const arma::Row& labels) { - util::CheckSameSizes(data, labels, "ROC_AUC::Evaluate()"); + util::CheckSameSizes(data, labels, "ROCAUCScore::Evaluate()"); arma::mat probabilities; model.Classify(data, probabilities); - // compute labels with "1" for positive class and "0" for the other + // Compute labels with "1" for positive class and "0" for the other. arma::Col binaryLabels = arma::conv_to>::from( - ((arma::umat) (labels == PC)).row(0)); + (labels == PositiveClass)); - // probability scores of positive class - arma::Col scoresOfPC = arma::conv_to>::from( - probabilities.row(PC)); + // Probability scores of positive class. + arma::Col scoresOfPC = arma::conv_to>::from( + probabilities.row(PositiveClass)); size_t numberOfTrueLabels = arma::sum(binaryLabels); size_t numberOfFalseLabels = binaryLabels.n_rows - numberOfTrueLabels; - // sort labels and probabilities, using probability scores + // Sort labels and probabilities, using probability scores. arma::ucolvec sortedScoreIndices = arma::stable_sort_index( - scoresOfPC, "descend"); + scoresOfPC, "descend"); arma::Col sortedLabels = binaryLabels(sortedScoreIndices); arma::Col sortedScores = scoresOfPC(sortedScoreIndices); - // compute indices of unique probability scores + // Compute indices of unique probability scores. arma::ucolvec uniqueScoreIndices = arma::find(arma::diff(sortedScores)); uniqueScoreIndices.insert_rows(uniqueScoreIndices.n_rows, 1); uniqueScoreIndices(uniqueScoreIndices.n_rows - 1) = scoresOfPC.n_rows - 1; - // compute true positive rate, and false positive rate - arma::Col cummulativeSum = arma::cumsum(sortedLabels); - cummulativeSum = cummulativeSum(uniqueScoreIndices); + // Compute true positive rate, and false positive rate. + arma::Col cumulativeSum = arma::cumsum(sortedLabels); + cumulativeSum = cumulativeSum(uniqueScoreIndices); arma::Col tpr, fpr; - tpr = arma::conv_to>::from(cummulativeSum); + tpr = arma::conv_to>::from(cumulativeSum); fpr = 1 + uniqueScoreIndices - tpr; tpr /= numberOfTrueLabels; fpr /= numberOfFalseLabels; - // to ensure that the (fpr, tpr) starts at (0, 0) + // To ensure that the (fpr, tpr) starts at (0, 0). tpr.insert_rows(0, 1); fpr.insert_rows(0, 1); tpr(0) = fpr(0) = 0; - // compute area under the curve using trapezoidal rule + // Compute area under the curve using trapezoidal rule. arma::mat auc = arma::trapz(fpr, tpr); return auc(0, 0); } diff --git a/src/mlpack/tests/cv_test.cpp b/src/mlpack/tests/cv_test.cpp index aa18af9ccb..634b6d9fba 100644 --- a/src/mlpack/tests/cv_test.cpp +++ b/src/mlpack/tests/cv_test.cpp @@ -76,7 +76,7 @@ TEST_CASE("BinaryClassificationMetricsTest", "[CVTest]") double f1 = 2 * 0.6 * 0.75 / (0.6 + 0.75); REQUIRE(F1::Evaluate(lr, data, labels) == Approx(f1).epsilon(1e-7)); - REQUIRE(ROC_AUC<>::Evaluate(lr, data, labels) + REQUIRE(ROCAUCScore<>::Evaluate(lr, data, labels) == Approx((double) 2 / 3).epsilon(1e-7)); } From 979da1b9627933aee778e3d68f947c2639608176 Mon Sep 17 00:00:00 2001 From: Sri Madhan M <22276366+srimadhan11@users.noreply.github.com> Date: Sun, 5 Dec 2021 17:44:45 +0530 Subject: [PATCH 03/10] instead of using arma::find(arma::diff()), implement 'for-loop' for finding 'changing points' of probabiliites --- src/mlpack/core/cv/metrics/roc_auc_score_impl.hpp | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/mlpack/core/cv/metrics/roc_auc_score_impl.hpp b/src/mlpack/core/cv/metrics/roc_auc_score_impl.hpp index 46d377d638..432c2d5878 100644 --- a/src/mlpack/core/cv/metrics/roc_auc_score_impl.hpp +++ b/src/mlpack/core/cv/metrics/roc_auc_score_impl.hpp @@ -47,9 +47,15 @@ double ROCAUCScore::Evaluate(MLAlgorithm& model, arma::Col sortedScores = scoresOfPC(sortedScoreIndices); // Compute indices of unique probability scores. - arma::ucolvec uniqueScoreIndices = arma::find(arma::diff(sortedScores)); - uniqueScoreIndices.insert_rows(uniqueScoreIndices.n_rows, 1); - uniqueScoreIndices(uniqueScoreIndices.n_rows - 1) = scoresOfPC.n_rows - 1; + arma::uword uniqueScoreIndicesLength = 0; + arma::ucolvec uniqueScoreIndices(sortedScores.n_rows); + for (arma::uword idx = 0; idx < sortedScores.n_rows - 1; idx++) { + if (sortedScores(idx) != sortedScores(idx + 1)) { + uniqueScoreIndices(uniqueScoreIndicesLength++) = idx; + } + } + uniqueScoreIndices(uniqueScoreIndicesLength++) = sortedScores.n_rows - 1; + uniqueScoreIndices.resize(uniqueScoreIndicesLength); // Compute true positive rate, and false positive rate. arma::Col cumulativeSum = arma::cumsum(sortedLabels); From c95b4247bf883090711057f3cb1f63b0d685e9eb Mon Sep 17 00:00:00 2001 From: Sri Madhan M <22276366+srimadhan11@users.noreply.github.com> Date: Sun, 5 Dec 2021 18:00:57 +0530 Subject: [PATCH 04/10] use rectangular integration, instead of trapezoidal integration --- src/mlpack/core/cv/metrics/roc_auc_score_impl.hpp | 9 ++++++--- src/mlpack/tests/cv_test.cpp | 4 +++- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/src/mlpack/core/cv/metrics/roc_auc_score_impl.hpp b/src/mlpack/core/cv/metrics/roc_auc_score_impl.hpp index 432c2d5878..8bd448b970 100644 --- a/src/mlpack/core/cv/metrics/roc_auc_score_impl.hpp +++ b/src/mlpack/core/cv/metrics/roc_auc_score_impl.hpp @@ -72,9 +72,12 @@ double ROCAUCScore::Evaluate(MLAlgorithm& model, fpr.insert_rows(0, 1); tpr(0) = fpr(0) = 0; - // Compute area under the curve using trapezoidal rule. - arma::mat auc = arma::trapz(fpr, tpr); - return auc(0, 0); + // Compute area under the curve using rectangular integration. + double auc = 0; + for (arma::uword idx = 0; idx < tpr.n_rows - 1; idx++) { + auc += (fpr(idx + 1) - fpr(idx)) * tpr(idx); + } + return auc; } } // namespace cv diff --git a/src/mlpack/tests/cv_test.cpp b/src/mlpack/tests/cv_test.cpp index 634b6d9fba..d15b02b381 100644 --- a/src/mlpack/tests/cv_test.cpp +++ b/src/mlpack/tests/cv_test.cpp @@ -76,8 +76,10 @@ TEST_CASE("BinaryClassificationMetricsTest", "[CVTest]") double f1 = 2 * 0.6 * 0.75 / (0.6 + 0.75); REQUIRE(F1::Evaluate(lr, data, labels) == Approx(f1).epsilon(1e-7)); + double rocAuc = + ((1.0/3 - 0) * 0) + ((2.0/3 - 1.0/3) * 0.75) + ((1 - 2.0/3) * 1); REQUIRE(ROCAUCScore<>::Evaluate(lr, data, labels) - == Approx((double) 2 / 3).epsilon(1e-7)); + == Approx(rocAuc).epsilon(1e-7)); } /** From 74160bcfbfed5d617bdb52917830fbd0a45e10a9 Mon Sep 17 00:00:00 2001 From: Sri Madhan M <22276366+srimadhan11@users.noreply.github.com> Date: Sun, 5 Dec 2021 18:06:32 +0530 Subject: [PATCH 05/10] add more test cases --- .../core/cv/metrics/roc_auc_score_impl.hpp | 14 ++++- src/mlpack/tests/cv_test.cpp | 63 ++++++++++++++++++- 2 files changed, 72 insertions(+), 5 deletions(-) diff --git a/src/mlpack/core/cv/metrics/roc_auc_score_impl.hpp b/src/mlpack/core/cv/metrics/roc_auc_score_impl.hpp index 8bd448b970..42371f0e6e 100644 --- a/src/mlpack/core/cv/metrics/roc_auc_score_impl.hpp +++ b/src/mlpack/core/cv/metrics/roc_auc_score_impl.hpp @@ -26,6 +26,12 @@ double ROCAUCScore::Evaluate(MLAlgorithm& model, { util::CheckSameSizes(data, labels, "ROCAUCScore::Evaluate()"); + if (data.n_cols == 0) { + throw std::invalid_argument( + "ROCAUCScore::Evaluate(): " + "number of points in input data cannot be zero"); + } + arma::mat probabilities; model.Classify(data, probabilities); @@ -64,8 +70,12 @@ double ROCAUCScore::Evaluate(MLAlgorithm& model, arma::Col tpr, fpr; tpr = arma::conv_to>::from(cumulativeSum); fpr = 1 + uniqueScoreIndices - tpr; - tpr /= numberOfTrueLabels; - fpr /= numberOfFalseLabels; + + // Check for "0" to avoid "nan". + if (numberOfTrueLabels != 0) + tpr /= numberOfTrueLabels; + if (numberOfFalseLabels != 0) + fpr /= numberOfFalseLabels; // To ensure that the (fpr, tpr) starts at (0, 0). tpr.insert_rows(0, 1); diff --git a/src/mlpack/tests/cv_test.cpp b/src/mlpack/tests/cv_test.cpp index d15b02b381..519997cc4c 100644 --- a/src/mlpack/tests/cv_test.cpp +++ b/src/mlpack/tests/cv_test.cpp @@ -76,10 +76,67 @@ TEST_CASE("BinaryClassificationMetricsTest", "[CVTest]") double f1 = 2 * 0.6 * 0.75 / (0.6 + 0.75); REQUIRE(F1::Evaluate(lr, data, labels) == Approx(f1).epsilon(1e-7)); - double rocAuc = - ((1.0/3 - 0) * 0) + ((2.0/3 - 1.0/3) * 0.75) + ((1 - 2.0/3) * 1); - REQUIRE(ROCAUCScore<>::Evaluate(lr, data, labels) + + // Testing binary ROC-AUC Score. + double rocAuc; + + // With "0" as PositiveClass. + rocAuc = ((0.25 - 0) * 0) + ((0.5 - 0.25) * 2.0/3) + ((1 - 0.5) * 2.0/3); + REQUIRE(ROCAUCScore<0>::Evaluate(lr, data, labels) == Approx(rocAuc).epsilon(1e-7)); + + // With "1" as PositiveClass. + rocAuc = ((1.0/3 - 0) * 0) + ((2.0/3 - 1.0/3) * 0.75) + ((1 - 2.0/3) * 1); + REQUIRE(ROCAUCScore<1>::Evaluate(lr, data, labels) + == Approx(rocAuc).epsilon(1e-7)); + + // Using perfect classifier. + arma::mat data2 = arma::linspace(1.0, 10.0, 20); + arma::Row correctLabels( + "0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1"); + arma::Row incorrectLabels( + "1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0"); + LogisticRegression<> lr2(data2, correctLabels); + + // Testing perfect and perfectly incorrect classification. + REQUIRE(ROCAUCScore<1>::Evaluate(lr2, data2, correctLabels) + == Approx(1).epsilon(1e-7)); + REQUIRE(ROCAUCScore<1>::Evaluate(lr2, data2, incorrectLabels) + == Approx(0).epsilon(1e-7)); + + // Testing mismatched input data and label size. + arma::Row smallLabels("0 0 0 0 0 1 1 1 1 1"); + Log::Fatal.ignoreInput = true; + REQUIRE_THROWS_AS(ROCAUCScore<0>::Evaluate(lr2, data2, smallLabels), + std::invalid_argument); + REQUIRE_THROWS_AS(ROCAUCScore<1>::Evaluate(lr2, data2, smallLabels), + std::invalid_argument); + Log::Fatal.ignoreInput = false; + + // Testing with input data and labels, of size zero. + arma::mat zeroData; + arma::Row zeroLabels; + Log::Fatal.ignoreInput = true; + REQUIRE_THROWS_AS( + ROCAUCScore<1>::Evaluate(lr2, zeroData, zeroLabels), + std::invalid_argument); + Log::Fatal.ignoreInput = false; + + // Testing with input data and labels, of size one. + arma::mat oneData("1"); + arma::Row oneLabels("1"); + REQUIRE(ROCAUCScore<1>::Evaluate(lr2, oneData, oneLabels) + == Approx(0).epsilon(1e-7)); + + // Testing with same labels. + arma::Row sameLabels("1 1 1 1 1 1 1 1 1 1"); + LogisticRegression<> lr3(data, sameLabels); + // All the labels as positive labels. + REQUIRE(ROCAUCScore<1>::Evaluate(lr3, data, sameLabels) + == Approx(0).epsilon(1e-7)); + // All the labels as non-positive labels. + REQUIRE(ROCAUCScore<0>::Evaluate(lr3, data, sameLabels) + == Approx(0).epsilon(1e-7)); } /** From c35adc6c326710d6416c26252e569779e0cc328f Mon Sep 17 00:00:00 2001 From: Sri Madhan M <22276366+srimadhan11@users.noreply.github.com> Date: Tue, 1 Feb 2022 19:12:13 +0530 Subject: [PATCH 06/10] using trapezoidal integration instead of rectangular integration, and directly get probability scores of positive class as function argument in 'ROCAUCScore::Evaluate()' --- src/mlpack/core/cv/metrics/roc_auc_score.hpp | 13 +-- .../core/cv/metrics/roc_auc_score_impl.hpp | 48 ++++---- src/mlpack/tests/cv_test.cpp | 103 ++++++++++-------- 3 files changed, 83 insertions(+), 81 deletions(-) diff --git a/src/mlpack/core/cv/metrics/roc_auc_score.hpp b/src/mlpack/core/cv/metrics/roc_auc_score.hpp index 8bc4af4011..75522daf4f 100644 --- a/src/mlpack/core/cv/metrics/roc_auc_score.hpp +++ b/src/mlpack/core/cv/metrics/roc_auc_score.hpp @@ -36,16 +36,13 @@ class ROCAUCScore { public: /** - * Run classification and calculate area under the ROC curve. + * Calculate area under the ROC curve. * - * @param model A classification model. - * @param data Column-major data containing test items. - * @param labels Ground truth (correct) labels for the test items. + * @param labels Ground truth (correct) labels. + * @param scores Probability scores of positive class. */ - template - static double Evaluate(MLAlgorithm& model, - const DataType& data, - const arma::Row& labels); + static double Evaluate(const arma::Row& labels, + const arma::Row& scores); /** * Information for hyper-parameter tuning code. It indicates that we want diff --git a/src/mlpack/core/cv/metrics/roc_auc_score_impl.hpp b/src/mlpack/core/cv/metrics/roc_auc_score_impl.hpp index 42371f0e6e..a218f064f4 100644 --- a/src/mlpack/core/cv/metrics/roc_auc_score_impl.hpp +++ b/src/mlpack/core/cv/metrics/roc_auc_score_impl.hpp @@ -13,44 +13,43 @@ #ifndef MLPACK_CORE_CV_METRICS_ROCAUCSCORE_IMPL_HPP #define MLPACK_CORE_CV_METRICS_ROCAUCSCORE_IMPL_HPP -#include - namespace mlpack { namespace cv { template -template -double ROCAUCScore::Evaluate(MLAlgorithm& model, - const DataType& data, - const arma::Row& labels) +double ROCAUCScore::Evaluate(const arma::Row& labels, + const arma::Row& scores) { - util::CheckSameSizes(data, labels, "ROCAUCScore::Evaluate()"); + util::CheckSameSizes(labels, scores, "ROCAUCScore::Evaluate()"); - if (data.n_cols == 0) { + if (labels.n_cols == 0) { throw std::invalid_argument( "ROCAUCScore::Evaluate(): " "number of points in input data cannot be zero"); } - arma::mat probabilities; - model.Classify(data, probabilities); - // Compute labels with "1" for positive class and "0" for the other. arma::Col binaryLabels = arma::conv_to>::from( (labels == PositiveClass)); - // Probability scores of positive class. - arma::Col scoresOfPC = arma::conv_to>::from( - probabilities.row(PositiveClass)); + // Converting probability scores of PositiveClass, from row to column vector. + arma::Col colScores = arma::conv_to>::from(scores); size_t numberOfTrueLabels = arma::sum(binaryLabels); size_t numberOfFalseLabels = binaryLabels.n_rows - numberOfTrueLabels; + // Check if only one class is given in labels. + if (numberOfTrueLabels == 0 || numberOfFalseLabels == 0) { + throw std::invalid_argument( + "ROCAUCScore::Evaluate(): " + "only one class is given in labels, ROCAUCScore is undefined"); + } + // Sort labels and probabilities, using probability scores. arma::ucolvec sortedScoreIndices = arma::stable_sort_index( - scoresOfPC, "descend"); + colScores, "descend"); arma::Col sortedLabels = binaryLabels(sortedScoreIndices); - arma::Col sortedScores = scoresOfPC(sortedScoreIndices); + arma::Col sortedScores = colScores(sortedScoreIndices); // Compute indices of unique probability scores. arma::uword uniqueScoreIndicesLength = 0; @@ -70,24 +69,17 @@ double ROCAUCScore::Evaluate(MLAlgorithm& model, arma::Col tpr, fpr; tpr = arma::conv_to>::from(cumulativeSum); fpr = 1 + uniqueScoreIndices - tpr; - - // Check for "0" to avoid "nan". - if (numberOfTrueLabels != 0) - tpr /= numberOfTrueLabels; - if (numberOfFalseLabels != 0) - fpr /= numberOfFalseLabels; + tpr /= numberOfTrueLabels; + fpr /= numberOfFalseLabels; // To ensure that the (fpr, tpr) starts at (0, 0). tpr.insert_rows(0, 1); fpr.insert_rows(0, 1); tpr(0) = fpr(0) = 0; - // Compute area under the curve using rectangular integration. - double auc = 0; - for (arma::uword idx = 0; idx < tpr.n_rows - 1; idx++) { - auc += (fpr(idx + 1) - fpr(idx)) * tpr(idx); - } - return auc; + // Compute area under the curve using trapezoidal rule. + arma::mat auc = arma::trapz(fpr, tpr); + return auc(0, 0); } } // namespace cv diff --git a/src/mlpack/tests/cv_test.cpp b/src/mlpack/tests/cv_test.cpp index 519997cc4c..be42641c61 100644 --- a/src/mlpack/tests/cv_test.cpp +++ b/src/mlpack/tests/cv_test.cpp @@ -78,65 +78,78 @@ TEST_CASE("BinaryClassificationMetricsTest", "[CVTest]") // Testing binary ROC-AUC Score. - double rocAuc; + arma::Row rocTrueLabels; + arma::Row rocScoresOfPC; + double rocAucScore; - // With "0" as PositiveClass. - rocAuc = ((0.25 - 0) * 0) + ((0.5 - 0.25) * 2.0/3) + ((1 - 0.5) * 2.0/3); - REQUIRE(ROCAUCScore<0>::Evaluate(lr, data, labels) - == Approx(rocAuc).epsilon(1e-7)); + // Test - 1 + rocTrueLabels = arma::Row("0 0 0 0 0 1 1 1 1 1"); + rocScoresOfPC = arma::Row("0.1 0.2 0.3 0.4 0.5 0.6 0.7 0.8 0.9 1"); - // With "1" as PositiveClass. - rocAuc = ((1.0/3 - 0) * 0) + ((2.0/3 - 1.0/3) * 0.75) + ((1 - 2.0/3) * 1); - REQUIRE(ROCAUCScore<1>::Evaluate(lr, data, labels) - == Approx(rocAuc).epsilon(1e-7)); + rocAucScore = 0; + REQUIRE(ROCAUCScore<0>::Evaluate(rocTrueLabels, rocScoresOfPC) + == Approx(rocAucScore).epsilon(1e-7)); - // Using perfect classifier. - arma::mat data2 = arma::linspace(1.0, 10.0, 20); - arma::Row correctLabels( - "0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1"); - arma::Row incorrectLabels( - "1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0"); - LogisticRegression<> lr2(data2, correctLabels); + rocAucScore = 1; + REQUIRE(ROCAUCScore<1>::Evaluate(rocTrueLabels, rocScoresOfPC) + == Approx(rocAucScore).epsilon(1e-7)); - // Testing perfect and perfectly incorrect classification. - REQUIRE(ROCAUCScore<1>::Evaluate(lr2, data2, correctLabels) - == Approx(1).epsilon(1e-7)); - REQUIRE(ROCAUCScore<1>::Evaluate(lr2, data2, incorrectLabels) - == Approx(0).epsilon(1e-7)); + // Test - 2 + rocTrueLabels = arma::Row("1 0 1 0 1 0 1 0 1 0"); + rocScoresOfPC = arma::Row("0.1 0.2 0.3 0.4 0.5 0.6 0.7 0.8 0.9 1"); + + rocAucScore = 0.6; + REQUIRE(ROCAUCScore<0>::Evaluate(rocTrueLabels, rocScoresOfPC) + == Approx(rocAucScore).epsilon(1e-7)); + + rocAucScore = 0.4; + REQUIRE(ROCAUCScore<1>::Evaluate(rocTrueLabels, rocScoresOfPC) + == Approx(rocAucScore).epsilon(1e-7)); + + // Test - 3 + rocTrueLabels = arma::Row("1 0 1 0 1 0 1 0 1 0"); + rocScoresOfPC = arma::Row("0.8 0.3 0.5 0.4 0.9 0.2 0.7 0.6 0 0.1"); + + rocAucScore = 0.24; + REQUIRE(ROCAUCScore<0>::Evaluate(rocTrueLabels, rocScoresOfPC) + == Approx(rocAucScore).epsilon(1e-7)); + + rocAucScore = 0.76; + REQUIRE(ROCAUCScore<1>::Evaluate(rocTrueLabels, rocScoresOfPC) + == Approx(rocAucScore).epsilon(1e-7)); + + // Test - 4 (labels and scores with zero size) + rocTrueLabels = arma::Row(); + rocScoresOfPC = arma::Row(); - // Testing mismatched input data and label size. - arma::Row smallLabels("0 0 0 0 0 1 1 1 1 1"); Log::Fatal.ignoreInput = true; - REQUIRE_THROWS_AS(ROCAUCScore<0>::Evaluate(lr2, data2, smallLabels), + REQUIRE_THROWS_AS(ROCAUCScore<0>::Evaluate(rocTrueLabels, rocScoresOfPC), std::invalid_argument); - REQUIRE_THROWS_AS(ROCAUCScore<1>::Evaluate(lr2, data2, smallLabels), + REQUIRE_THROWS_AS(ROCAUCScore<1>::Evaluate(rocTrueLabels, rocScoresOfPC), std::invalid_argument); Log::Fatal.ignoreInput = false; - // Testing with input data and labels, of size zero. - arma::mat zeroData; - arma::Row zeroLabels; + // Test - 5 (labels and scores with one size) + rocTrueLabels = arma::Row("1"); + rocScoresOfPC = arma::Row("0.8"); + Log::Fatal.ignoreInput = true; - REQUIRE_THROWS_AS( - ROCAUCScore<1>::Evaluate(lr2, zeroData, zeroLabels), - std::invalid_argument); + REQUIRE_THROWS_AS(ROCAUCScore<0>::Evaluate(rocTrueLabels, rocScoresOfPC), + std::invalid_argument); + REQUIRE_THROWS_AS(ROCAUCScore<1>::Evaluate(rocTrueLabels, rocScoresOfPC), + std::invalid_argument); Log::Fatal.ignoreInput = false; - // Testing with input data and labels, of size one. - arma::mat oneData("1"); - arma::Row oneLabels("1"); - REQUIRE(ROCAUCScore<1>::Evaluate(lr2, oneData, oneLabels) - == Approx(0).epsilon(1e-7)); + // Test - 6 (mismatch labels and scores size) + rocTrueLabels = arma::Row("1 0 1 0 1 0 1 0 1 0"); + rocScoresOfPC = arma::Row("0.1 0.2 0.3 0.4 0.5"); - // Testing with same labels. - arma::Row sameLabels("1 1 1 1 1 1 1 1 1 1"); - LogisticRegression<> lr3(data, sameLabels); - // All the labels as positive labels. - REQUIRE(ROCAUCScore<1>::Evaluate(lr3, data, sameLabels) - == Approx(0).epsilon(1e-7)); - // All the labels as non-positive labels. - REQUIRE(ROCAUCScore<0>::Evaluate(lr3, data, sameLabels) - == Approx(0).epsilon(1e-7)); + Log::Fatal.ignoreInput = true; + REQUIRE_THROWS_AS(ROCAUCScore<0>::Evaluate(rocTrueLabels, rocScoresOfPC), + std::invalid_argument); + REQUIRE_THROWS_AS(ROCAUCScore<1>::Evaluate(rocTrueLabels, rocScoresOfPC), + std::invalid_argument); + Log::Fatal.ignoreInput = false; } /** From eb59051c1c6a883e7768d4257e6f1077fb365af6 Mon Sep 17 00:00:00 2001 From: Sri Madhan M <22276366+srimadhan11@users.noreply.github.com> Date: Sun, 6 Feb 2022 11:54:01 +0530 Subject: [PATCH 07/10] fix style issues --- .../core/cv/metrics/roc_auc_score_impl.hpp | 25 +++++++++++-------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/src/mlpack/core/cv/metrics/roc_auc_score_impl.hpp b/src/mlpack/core/cv/metrics/roc_auc_score_impl.hpp index a218f064f4..7f48c32cef 100644 --- a/src/mlpack/core/cv/metrics/roc_auc_score_impl.hpp +++ b/src/mlpack/core/cv/metrics/roc_auc_score_impl.hpp @@ -18,11 +18,12 @@ namespace cv { template double ROCAUCScore::Evaluate(const arma::Row& labels, - const arma::Row& scores) + const arma::rowvec& scores) { util::CheckSameSizes(labels, scores, "ROCAUCScore::Evaluate()"); - if (labels.n_cols == 0) { + if (labels.n_cols == 0) + { throw std::invalid_argument( "ROCAUCScore::Evaluate(): " "number of points in input data cannot be zero"); @@ -33,29 +34,31 @@ double ROCAUCScore::Evaluate(const arma::Row& labels, (labels == PositiveClass)); // Converting probability scores of PositiveClass, from row to column vector. - arma::Col colScores = arma::conv_to>::from(scores); + arma::vec colScores = arma::conv_to::from(scores); size_t numberOfTrueLabels = arma::sum(binaryLabels); size_t numberOfFalseLabels = binaryLabels.n_rows - numberOfTrueLabels; // Check if only one class is given in labels. - if (numberOfTrueLabels == 0 || numberOfFalseLabels == 0) { + if (numberOfTrueLabels == 0 || numberOfFalseLabels == 0) + { throw std::invalid_argument( "ROCAUCScore::Evaluate(): " "only one class is given in labels, ROCAUCScore is undefined"); } // Sort labels and probabilities, using probability scores. - arma::ucolvec sortedScoreIndices = arma::stable_sort_index( - colScores, "descend"); + arma::uvec sortedScoreIndices = arma::stable_sort_index(colScores, "descend"); arma::Col sortedLabels = binaryLabels(sortedScoreIndices); - arma::Col sortedScores = colScores(sortedScoreIndices); + arma::vec sortedScores = colScores(sortedScoreIndices); // Compute indices of unique probability scores. arma::uword uniqueScoreIndicesLength = 0; arma::ucolvec uniqueScoreIndices(sortedScores.n_rows); - for (arma::uword idx = 0; idx < sortedScores.n_rows - 1; idx++) { - if (sortedScores(idx) != sortedScores(idx + 1)) { + for (arma::uword idx = 0; idx < sortedScores.n_rows - 1; idx++) + { + if (sortedScores(idx) != sortedScores(idx + 1)) + { uniqueScoreIndices(uniqueScoreIndicesLength++) = idx; } } @@ -66,8 +69,8 @@ double ROCAUCScore::Evaluate(const arma::Row& labels, arma::Col cumulativeSum = arma::cumsum(sortedLabels); cumulativeSum = cumulativeSum(uniqueScoreIndices); - arma::Col tpr, fpr; - tpr = arma::conv_to>::from(cumulativeSum); + arma::vec tpr, fpr; + tpr = arma::conv_to::from(cumulativeSum); fpr = 1 + uniqueScoreIndices - tpr; tpr /= numberOfTrueLabels; fpr /= numberOfFalseLabels; From cfe3e11bc2626272d7a7e70153d95ba2453ba8cc Mon Sep 17 00:00:00 2001 From: Sri Madhan M <22276366+srimadhan11@users.noreply.github.com> Date: Sat, 19 Feb 2022 18:03:46 +0530 Subject: [PATCH 08/10] remove excess newline --- src/mlpack/tests/cv_test.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/mlpack/tests/cv_test.cpp b/src/mlpack/tests/cv_test.cpp index be42641c61..135fb5d092 100644 --- a/src/mlpack/tests/cv_test.cpp +++ b/src/mlpack/tests/cv_test.cpp @@ -76,7 +76,6 @@ TEST_CASE("BinaryClassificationMetricsTest", "[CVTest]") double f1 = 2 * 0.6 * 0.75 / (0.6 + 0.75); REQUIRE(F1::Evaluate(lr, data, labels) == Approx(f1).epsilon(1e-7)); - // Testing binary ROC-AUC Score. arma::Row rocTrueLabels; arma::Row rocScoresOfPC; From c236183961670499cc8d7bb96bed8a6fe4b92f26 Mon Sep 17 00:00:00 2001 From: Sri Madhan M <22276366+srimadhan11@users.noreply.github.com> Date: Sat, 19 Feb 2022 18:05:17 +0530 Subject: [PATCH 09/10] add testcases comparison link --- src/mlpack/tests/cv_test.cpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/mlpack/tests/cv_test.cpp b/src/mlpack/tests/cv_test.cpp index 135fb5d092..aafee5db11 100644 --- a/src/mlpack/tests/cv_test.cpp +++ b/src/mlpack/tests/cv_test.cpp @@ -77,6 +77,12 @@ TEST_CASE("BinaryClassificationMetricsTest", "[CVTest]") REQUIRE(F1::Evaluate(lr, data, labels) == Approx(f1).epsilon(1e-7)); // Testing binary ROC-AUC Score. + // + // NOTE: + // For comparing these ROCAUCScore testcases with "scikit-learn" + // library's "roc_auc_score", refer the pull request thread comment + // https://github.com/mlpack/mlpack/pull/3086#issuecomment-1046003548 + // arma::Row rocTrueLabels; arma::Row rocScoresOfPC; double rocAucScore; From 56202ee7fcfcf778fd71a75313536aebff3e76ea Mon Sep 17 00:00:00 2001 From: Sri Madhan M <22276366+srimadhan11@users.noreply.github.com> Date: Sat, 19 Feb 2022 18:08:20 +0530 Subject: [PATCH 10/10] update contributors list --- COPYRIGHT.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/COPYRIGHT.txt b/COPYRIGHT.txt index 4ab00dfb7a..c849978f77 100644 --- a/COPYRIGHT.txt +++ b/COPYRIGHT.txt @@ -146,6 +146,7 @@ Copyright: Copyright 2021, Muhammad Fawwaz Mayda Copyright 2021, Roshan Nrusing Swain Copyright 2021, Suvarsha Chennareddy + Copyright 2022, Sri Madhan M License: BSD-3-clause All rights reserved.