diff --git a/COPYRIGHT.txt b/COPYRIGHT.txt index 94b2383e6f..09007f2719 100644 --- a/COPYRIGHT.txt +++ b/COPYRIGHT.txt @@ -147,6 +147,7 @@ Copyright: Copyright 2021, Roshan Nrusing Swain Copyright 2021, Suvarsha Chennareddy Copyright 2021, Shubham Agrawal + Copyright 2022, Sri Madhan M License: BSD-3-clause All rights reserved. diff --git a/src/mlpack/core/cv/metrics/CMakeLists.txt b/src/mlpack/core/cv/metrics/CMakeLists.txt index b9edacaf9a..2f908b759e 100644 --- a/src/mlpack/core/cv/metrics/CMakeLists.txt +++ b/src/mlpack/core/cv/metrics/CMakeLists.txt @@ -13,6 +13,8 @@ set(SOURCES precision_impl.hpp recall.hpp recall_impl.hpp + roc_auc_score.hpp + roc_auc_score_impl.hpp r2_score.hpp r2_score_impl.hpp ) diff --git a/src/mlpack/core/cv/metrics/roc_auc_score.hpp b/src/mlpack/core/cv/metrics/roc_auc_score.hpp new file mode 100644 index 0000000000..75522daf4f --- /dev/null +++ b/src/mlpack/core/cv/metrics/roc_auc_score.hpp @@ -0,0 +1,60 @@ +/** + * @file core/cv/metrics/roc_auc_score.hpp + * @author Sri Madhan M + * + * The area under Receiver Operating Characteristic curve (ROC-AUC) score. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_CORE_CV_METRICS_ROCAUCSCORE_HPP +#define MLPACK_CORE_CV_METRICS_ROCAUCSCORE_HPP + +#include + +namespace mlpack { +namespace cv { + +/** + * ROC-AUC is a metric of performance for classification algorithms that for + * binary classification is equal to area under the curve formed by + * @f$ (fpr, tpr) @f$, where @f$ fpr @f$ and @f$ tpr @f$ are the true positive + * rate and false positive rate, which is calculated for many different + * thresholds. For each thresholds, @f$ tpr @f$ and @f$ fpr @f$ are calculated + * as, @f$ tpr = tp / (tp + fn) @f$ and @f$ fpr = fp / (fp + tn) @f$, + * where @f$ tp @f$, @f$ tn @f$, @f$ fp @f$ and @f$ fn @f$ are the numbers of + * true positives, true negatives, false positives and false negatives + * respectively. + * + * @tparam PositiveClass Positives are assumed to have labels equal to this + * value. Defaults to 1. + */ +template +class ROCAUCScore +{ + public: + /** + * Calculate area under the ROC curve. + * + * @param labels Ground truth (correct) labels. + * @param scores Probability scores of positive class. + */ + static double Evaluate(const arma::Row& labels, + const arma::Row& scores); + + /** + * Information for hyper-parameter tuning code. It indicates that we want + * to maximize the metric. + */ + static const bool NeedsMinimization = false; +}; + +} // namespace cv +} // namespace mlpack + +// Include implementation. +#include "roc_auc_score_impl.hpp" + +#endif diff --git a/src/mlpack/core/cv/metrics/roc_auc_score_impl.hpp b/src/mlpack/core/cv/metrics/roc_auc_score_impl.hpp new file mode 100644 index 0000000000..7f48c32cef --- /dev/null +++ b/src/mlpack/core/cv/metrics/roc_auc_score_impl.hpp @@ -0,0 +1,91 @@ +/** + * @file core/cv/metrics/roc_auc_score_impl.hpp + * @author Sri Madhan M + * + * Implementation of the area under Receiver Operating Characteristic curve + * (ROC-AUC) score. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_CORE_CV_METRICS_ROCAUCSCORE_IMPL_HPP +#define MLPACK_CORE_CV_METRICS_ROCAUCSCORE_IMPL_HPP + +namespace mlpack { +namespace cv { + +template +double ROCAUCScore::Evaluate(const arma::Row& labels, + const arma::rowvec& scores) +{ + util::CheckSameSizes(labels, scores, "ROCAUCScore::Evaluate()"); + + if (labels.n_cols == 0) + { + throw std::invalid_argument( + "ROCAUCScore::Evaluate(): " + "number of points in input data cannot be zero"); + } + + // Compute labels with "1" for positive class and "0" for the other. + arma::Col binaryLabels = arma::conv_to>::from( + (labels == PositiveClass)); + + // Converting probability scores of PositiveClass, from row to column vector. + arma::vec colScores = arma::conv_to::from(scores); + + size_t numberOfTrueLabels = arma::sum(binaryLabels); + size_t numberOfFalseLabels = binaryLabels.n_rows - numberOfTrueLabels; + + // Check if only one class is given in labels. + if (numberOfTrueLabels == 0 || numberOfFalseLabels == 0) + { + throw std::invalid_argument( + "ROCAUCScore::Evaluate(): " + "only one class is given in labels, ROCAUCScore is undefined"); + } + + // Sort labels and probabilities, using probability scores. + arma::uvec sortedScoreIndices = arma::stable_sort_index(colScores, "descend"); + arma::Col sortedLabels = binaryLabels(sortedScoreIndices); + arma::vec sortedScores = colScores(sortedScoreIndices); + + // Compute indices of unique probability scores. + arma::uword uniqueScoreIndicesLength = 0; + arma::ucolvec uniqueScoreIndices(sortedScores.n_rows); + for (arma::uword idx = 0; idx < sortedScores.n_rows - 1; idx++) + { + if (sortedScores(idx) != sortedScores(idx + 1)) + { + uniqueScoreIndices(uniqueScoreIndicesLength++) = idx; + } + } + uniqueScoreIndices(uniqueScoreIndicesLength++) = sortedScores.n_rows - 1; + uniqueScoreIndices.resize(uniqueScoreIndicesLength); + + // Compute true positive rate, and false positive rate. + arma::Col cumulativeSum = arma::cumsum(sortedLabels); + cumulativeSum = cumulativeSum(uniqueScoreIndices); + + arma::vec tpr, fpr; + tpr = arma::conv_to::from(cumulativeSum); + fpr = 1 + uniqueScoreIndices - tpr; + tpr /= numberOfTrueLabels; + fpr /= numberOfFalseLabels; + + // To ensure that the (fpr, tpr) starts at (0, 0). + tpr.insert_rows(0, 1); + fpr.insert_rows(0, 1); + tpr(0) = fpr(0) = 0; + + // Compute area under the curve using trapezoidal rule. + arma::mat auc = arma::trapz(fpr, tpr); + return auc(0, 0); +} + +} // namespace cv +} // namespace mlpack + +#endif diff --git a/src/mlpack/tests/cv_test.cpp b/src/mlpack/tests/cv_test.cpp index 75f1e87998..aafee5db11 100644 --- a/src/mlpack/tests/cv_test.cpp +++ b/src/mlpack/tests/cv_test.cpp @@ -17,6 +17,7 @@ #include #include #include +#include #include #include #include @@ -74,6 +75,86 @@ TEST_CASE("BinaryClassificationMetricsTest", "[CVTest]") double f1 = 2 * 0.6 * 0.75 / (0.6 + 0.75); REQUIRE(F1::Evaluate(lr, data, labels) == Approx(f1).epsilon(1e-7)); + + // Testing binary ROC-AUC Score. + // + // NOTE: + // For comparing these ROCAUCScore testcases with "scikit-learn" + // library's "roc_auc_score", refer the pull request thread comment + // https://github.com/mlpack/mlpack/pull/3086#issuecomment-1046003548 + // + arma::Row rocTrueLabels; + arma::Row rocScoresOfPC; + double rocAucScore; + + // Test - 1 + rocTrueLabels = arma::Row("0 0 0 0 0 1 1 1 1 1"); + rocScoresOfPC = arma::Row("0.1 0.2 0.3 0.4 0.5 0.6 0.7 0.8 0.9 1"); + + rocAucScore = 0; + REQUIRE(ROCAUCScore<0>::Evaluate(rocTrueLabels, rocScoresOfPC) + == Approx(rocAucScore).epsilon(1e-7)); + + rocAucScore = 1; + REQUIRE(ROCAUCScore<1>::Evaluate(rocTrueLabels, rocScoresOfPC) + == Approx(rocAucScore).epsilon(1e-7)); + + // Test - 2 + rocTrueLabels = arma::Row("1 0 1 0 1 0 1 0 1 0"); + rocScoresOfPC = arma::Row("0.1 0.2 0.3 0.4 0.5 0.6 0.7 0.8 0.9 1"); + + rocAucScore = 0.6; + REQUIRE(ROCAUCScore<0>::Evaluate(rocTrueLabels, rocScoresOfPC) + == Approx(rocAucScore).epsilon(1e-7)); + + rocAucScore = 0.4; + REQUIRE(ROCAUCScore<1>::Evaluate(rocTrueLabels, rocScoresOfPC) + == Approx(rocAucScore).epsilon(1e-7)); + + // Test - 3 + rocTrueLabels = arma::Row("1 0 1 0 1 0 1 0 1 0"); + rocScoresOfPC = arma::Row("0.8 0.3 0.5 0.4 0.9 0.2 0.7 0.6 0 0.1"); + + rocAucScore = 0.24; + REQUIRE(ROCAUCScore<0>::Evaluate(rocTrueLabels, rocScoresOfPC) + == Approx(rocAucScore).epsilon(1e-7)); + + rocAucScore = 0.76; + REQUIRE(ROCAUCScore<1>::Evaluate(rocTrueLabels, rocScoresOfPC) + == Approx(rocAucScore).epsilon(1e-7)); + + // Test - 4 (labels and scores with zero size) + rocTrueLabels = arma::Row(); + rocScoresOfPC = arma::Row(); + + Log::Fatal.ignoreInput = true; + REQUIRE_THROWS_AS(ROCAUCScore<0>::Evaluate(rocTrueLabels, rocScoresOfPC), + std::invalid_argument); + REQUIRE_THROWS_AS(ROCAUCScore<1>::Evaluate(rocTrueLabels, rocScoresOfPC), + std::invalid_argument); + Log::Fatal.ignoreInput = false; + + // Test - 5 (labels and scores with one size) + rocTrueLabels = arma::Row("1"); + rocScoresOfPC = arma::Row("0.8"); + + Log::Fatal.ignoreInput = true; + REQUIRE_THROWS_AS(ROCAUCScore<0>::Evaluate(rocTrueLabels, rocScoresOfPC), + std::invalid_argument); + REQUIRE_THROWS_AS(ROCAUCScore<1>::Evaluate(rocTrueLabels, rocScoresOfPC), + std::invalid_argument); + Log::Fatal.ignoreInput = false; + + // Test - 6 (mismatch labels and scores size) + rocTrueLabels = arma::Row("1 0 1 0 1 0 1 0 1 0"); + rocScoresOfPC = arma::Row("0.1 0.2 0.3 0.4 0.5"); + + Log::Fatal.ignoreInput = true; + REQUIRE_THROWS_AS(ROCAUCScore<0>::Evaluate(rocTrueLabels, rocScoresOfPC), + std::invalid_argument); + REQUIRE_THROWS_AS(ROCAUCScore<1>::Evaluate(rocTrueLabels, rocScoresOfPC), + std::invalid_argument); + Log::Fatal.ignoreInput = false; } /**