Merge pull request #3086 from srimadhan11/roc-auc
add ROC AUC for binary classification
This commit is contained in:
Ryan Curtin
@@ -147,6 +147,7 @@ Copyright:
|
||||
Copyright 2021, Roshan Nrusing Swain <swainroshan001@gmail.com>
|
||||
Copyright 2021, Suvarsha Chennareddy <suvarshachennareddy@gmail.com>
|
||||
Copyright 2021, Shubham Agrawal <shubham.agra1206@gmail.com>
|
||||
Copyright 2022, Sri Madhan M <srimadhan11@gmail.com>
|
||||
|
||||
License: BSD-3-clause
|
||||
All rights reserved.
|
||||
|
||||
@@ -13,6 +13,8 @@ set(SOURCES
|
||||
precision_impl.hpp
|
||||
recall.hpp
|
||||
recall_impl.hpp
|
||||
roc_auc_score.hpp
|
||||
roc_auc_score_impl.hpp
|
||||
r2_score.hpp
|
||||
r2_score_impl.hpp
|
||||
)
|
||||
|
||||
@@ -0,0 +1,60 @@
|
||||
/**
|
||||
* @file core/cv/metrics/roc_auc_score.hpp
|
||||
* @author Sri Madhan M
|
||||
*
|
||||
* The area under Receiver Operating Characteristic curve (ROC-AUC) score.
|
||||
*
|
||||
* mlpack is free software; you may redistribute it and/or modify it under the
|
||||
* terms of the 3-clause BSD license. You should have received a copy of the
|
||||
* 3-clause BSD license along with mlpack. If not, see
|
||||
* http://www.opensource.org/licenses/BSD-3-Clause for more information.
|
||||
*/
|
||||
#ifndef MLPACK_CORE_CV_METRICS_ROCAUCSCORE_HPP
|
||||
#define MLPACK_CORE_CV_METRICS_ROCAUCSCORE_HPP
|
||||
|
||||
#include <mlpack/core.hpp>
|
||||
|
||||
namespace mlpack {
|
||||
namespace cv {
|
||||
|
||||
/**
|
||||
* ROC-AUC is a metric of performance for classification algorithms that for
|
||||
* binary classification is equal to area under the curve formed by
|
||||
* @f$ (fpr, tpr) @f$, where @f$ fpr @f$ and @f$ tpr @f$ are the true positive
|
||||
* rate and false positive rate, which is calculated for many different
|
||||
* thresholds. For each thresholds, @f$ tpr @f$ and @f$ fpr @f$ are calculated
|
||||
* as, @f$ tpr = tp / (tp + fn) @f$ and @f$ fpr = fp / (fp + tn) @f$,
|
||||
* where @f$ tp @f$, @f$ tn @f$, @f$ fp @f$ and @f$ fn @f$ are the numbers of
|
||||
* true positives, true negatives, false positives and false negatives
|
||||
* respectively.
|
||||
*
|
||||
* @tparam PositiveClass Positives are assumed to have labels equal to this
|
||||
* value. Defaults to 1.
|
||||
*/
|
||||
template<size_t PositiveClass = 1>
|
||||
class ROCAUCScore
|
||||
{
|
||||
public:
|
||||
/**
|
||||
* Calculate area under the ROC curve.
|
||||
*
|
||||
* @param labels Ground truth (correct) labels.
|
||||
* @param scores Probability scores of positive class.
|
||||
*/
|
||||
static double Evaluate(const arma::Row<size_t>& labels,
|
||||
const arma::Row<double>& scores);
|
||||
|
||||
/**
|
||||
* Information for hyper-parameter tuning code. It indicates that we want
|
||||
* to maximize the metric.
|
||||
*/
|
||||
static const bool NeedsMinimization = false;
|
||||
};
|
||||
|
||||
} // namespace cv
|
||||
} // namespace mlpack
|
||||
|
||||
// Include implementation.
|
||||
#include "roc_auc_score_impl.hpp"
|
||||
|
||||
#endif
|
||||
@@ -0,0 +1,91 @@
|
||||
/**
|
||||
* @file core/cv/metrics/roc_auc_score_impl.hpp
|
||||
* @author Sri Madhan M
|
||||
*
|
||||
* Implementation of the area under Receiver Operating Characteristic curve
|
||||
* (ROC-AUC) score.
|
||||
*
|
||||
* mlpack is free software; you may redistribute it and/or modify it under the
|
||||
* terms of the 3-clause BSD license. You should have received a copy of the
|
||||
* 3-clause BSD license along with mlpack. If not, see
|
||||
* http://www.opensource.org/licenses/BSD-3-Clause for more information.
|
||||
*/
|
||||
#ifndef MLPACK_CORE_CV_METRICS_ROCAUCSCORE_IMPL_HPP
|
||||
#define MLPACK_CORE_CV_METRICS_ROCAUCSCORE_IMPL_HPP
|
||||
|
||||
namespace mlpack {
|
||||
namespace cv {
|
||||
|
||||
template<size_t PositiveClass>
|
||||
double ROCAUCScore<PositiveClass>::Evaluate(const arma::Row<size_t>& labels,
|
||||
const arma::rowvec& scores)
|
||||
{
|
||||
util::CheckSameSizes(labels, scores, "ROCAUCScore::Evaluate()");
|
||||
|
||||
if (labels.n_cols == 0)
|
||||
{
|
||||
throw std::invalid_argument(
|
||||
"ROCAUCScore::Evaluate(): "
|
||||
"number of points in input data cannot be zero");
|
||||
}
|
||||
|
||||
// Compute labels with "1" for positive class and "0" for the other.
|
||||
arma::Col<size_t> binaryLabels = arma::conv_to<arma::Col<size_t>>::from(
|
||||
(labels == PositiveClass));
|
||||
|
||||
// Converting probability scores of PositiveClass, from row to column vector.
|
||||
arma::vec colScores = arma::conv_to<arma::vec>::from(scores);
|
||||
|
||||
size_t numberOfTrueLabels = arma::sum(binaryLabels);
|
||||
size_t numberOfFalseLabels = binaryLabels.n_rows - numberOfTrueLabels;
|
||||
|
||||
// Check if only one class is given in labels.
|
||||
if (numberOfTrueLabels == 0 || numberOfFalseLabels == 0)
|
||||
{
|
||||
throw std::invalid_argument(
|
||||
"ROCAUCScore::Evaluate(): "
|
||||
"only one class is given in labels, ROCAUCScore is undefined");
|
||||
}
|
||||
|
||||
// Sort labels and probabilities, using probability scores.
|
||||
arma::uvec sortedScoreIndices = arma::stable_sort_index(colScores, "descend");
|
||||
arma::Col<size_t> sortedLabels = binaryLabels(sortedScoreIndices);
|
||||
arma::vec sortedScores = colScores(sortedScoreIndices);
|
||||
|
||||
// Compute indices of unique probability scores.
|
||||
arma::uword uniqueScoreIndicesLength = 0;
|
||||
arma::ucolvec uniqueScoreIndices(sortedScores.n_rows);
|
||||
for (arma::uword idx = 0; idx < sortedScores.n_rows - 1; idx++)
|
||||
{
|
||||
if (sortedScores(idx) != sortedScores(idx + 1))
|
||||
{
|
||||
uniqueScoreIndices(uniqueScoreIndicesLength++) = idx;
|
||||
}
|
||||
}
|
||||
uniqueScoreIndices(uniqueScoreIndicesLength++) = sortedScores.n_rows - 1;
|
||||
uniqueScoreIndices.resize(uniqueScoreIndicesLength);
|
||||
|
||||
// Compute true positive rate, and false positive rate.
|
||||
arma::Col<size_t> cumulativeSum = arma::cumsum(sortedLabels);
|
||||
cumulativeSum = cumulativeSum(uniqueScoreIndices);
|
||||
|
||||
arma::vec tpr, fpr;
|
||||
tpr = arma::conv_to<arma::vec>::from(cumulativeSum);
|
||||
fpr = 1 + uniqueScoreIndices - tpr;
|
||||
tpr /= numberOfTrueLabels;
|
||||
fpr /= numberOfFalseLabels;
|
||||
|
||||
// To ensure that the (fpr, tpr) starts at (0, 0).
|
||||
tpr.insert_rows(0, 1);
|
||||
fpr.insert_rows(0, 1);
|
||||
tpr(0) = fpr(0) = 0;
|
||||
|
||||
// Compute area under the curve using trapezoidal rule.
|
||||
arma::mat auc = arma::trapz(fpr, tpr);
|
||||
return auc(0, 0);
|
||||
}
|
||||
|
||||
} // namespace cv
|
||||
} // namespace mlpack
|
||||
|
||||
#endif
|
||||
@@ -17,6 +17,7 @@
|
||||
#include <mlpack/core/cv/metrics/mse.hpp>
|
||||
#include <mlpack/core/cv/metrics/precision.hpp>
|
||||
#include <mlpack/core/cv/metrics/recall.hpp>
|
||||
#include <mlpack/core/cv/metrics/roc_auc_score.hpp>
|
||||
#include <mlpack/core/cv/metrics/r2_score.hpp>
|
||||
#include <mlpack/core/cv/metrics/silhouette_score.hpp>
|
||||
#include <mlpack/core/cv/simple_cv.hpp>
|
||||
@@ -74,6 +75,86 @@ TEST_CASE("BinaryClassificationMetricsTest", "[CVTest]")
|
||||
|
||||
double f1 = 2 * 0.6 * 0.75 / (0.6 + 0.75);
|
||||
REQUIRE(F1<Binary>::Evaluate(lr, data, labels) == Approx(f1).epsilon(1e-7));
|
||||
|
||||
// Testing binary ROC-AUC Score.
|
||||
//
|
||||
// NOTE:
|
||||
// For comparing these ROCAUCScore testcases with "scikit-learn"
|
||||
// library's "roc_auc_score", refer the pull request thread comment
|
||||
// https://github.com/mlpack/mlpack/pull/3086#issuecomment-1046003548
|
||||
//
|
||||
arma::Row<size_t> rocTrueLabels;
|
||||
arma::Row<double> rocScoresOfPC;
|
||||
double rocAucScore;
|
||||
|
||||
// Test - 1
|
||||
rocTrueLabels = arma::Row<size_t>("0 0 0 0 0 1 1 1 1 1");
|
||||
rocScoresOfPC = arma::Row<double>("0.1 0.2 0.3 0.4 0.5 0.6 0.7 0.8 0.9 1");
|
||||
|
||||
rocAucScore = 0;
|
||||
REQUIRE(ROCAUCScore<0>::Evaluate(rocTrueLabels, rocScoresOfPC)
|
||||
== Approx(rocAucScore).epsilon(1e-7));
|
||||
|
||||
rocAucScore = 1;
|
||||
REQUIRE(ROCAUCScore<1>::Evaluate(rocTrueLabels, rocScoresOfPC)
|
||||
== Approx(rocAucScore).epsilon(1e-7));
|
||||
|
||||
// Test - 2
|
||||
rocTrueLabels = arma::Row<size_t>("1 0 1 0 1 0 1 0 1 0");
|
||||
rocScoresOfPC = arma::Row<double>("0.1 0.2 0.3 0.4 0.5 0.6 0.7 0.8 0.9 1");
|
||||
|
||||
rocAucScore = 0.6;
|
||||
REQUIRE(ROCAUCScore<0>::Evaluate(rocTrueLabels, rocScoresOfPC)
|
||||
== Approx(rocAucScore).epsilon(1e-7));
|
||||
|
||||
rocAucScore = 0.4;
|
||||
REQUIRE(ROCAUCScore<1>::Evaluate(rocTrueLabels, rocScoresOfPC)
|
||||
== Approx(rocAucScore).epsilon(1e-7));
|
||||
|
||||
// Test - 3
|
||||
rocTrueLabels = arma::Row<size_t>("1 0 1 0 1 0 1 0 1 0");
|
||||
rocScoresOfPC = arma::Row<double>("0.8 0.3 0.5 0.4 0.9 0.2 0.7 0.6 0 0.1");
|
||||
|
||||
rocAucScore = 0.24;
|
||||
REQUIRE(ROCAUCScore<0>::Evaluate(rocTrueLabels, rocScoresOfPC)
|
||||
== Approx(rocAucScore).epsilon(1e-7));
|
||||
|
||||
rocAucScore = 0.76;
|
||||
REQUIRE(ROCAUCScore<1>::Evaluate(rocTrueLabels, rocScoresOfPC)
|
||||
== Approx(rocAucScore).epsilon(1e-7));
|
||||
|
||||
// Test - 4 (labels and scores with zero size)
|
||||
rocTrueLabels = arma::Row<size_t>();
|
||||
rocScoresOfPC = arma::Row<double>();
|
||||
|
||||
Log::Fatal.ignoreInput = true;
|
||||
REQUIRE_THROWS_AS(ROCAUCScore<0>::Evaluate(rocTrueLabels, rocScoresOfPC),
|
||||
std::invalid_argument);
|
||||
REQUIRE_THROWS_AS(ROCAUCScore<1>::Evaluate(rocTrueLabels, rocScoresOfPC),
|
||||
std::invalid_argument);
|
||||
Log::Fatal.ignoreInput = false;
|
||||
|
||||
// Test - 5 (labels and scores with one size)
|
||||
rocTrueLabels = arma::Row<size_t>("1");
|
||||
rocScoresOfPC = arma::Row<double>("0.8");
|
||||
|
||||
Log::Fatal.ignoreInput = true;
|
||||
REQUIRE_THROWS_AS(ROCAUCScore<0>::Evaluate(rocTrueLabels, rocScoresOfPC),
|
||||
std::invalid_argument);
|
||||
REQUIRE_THROWS_AS(ROCAUCScore<1>::Evaluate(rocTrueLabels, rocScoresOfPC),
|
||||
std::invalid_argument);
|
||||
Log::Fatal.ignoreInput = false;
|
||||
|
||||
// Test - 6 (mismatch labels and scores size)
|
||||
rocTrueLabels = arma::Row<size_t>("1 0 1 0 1 0 1 0 1 0");
|
||||
rocScoresOfPC = arma::Row<double>("0.1 0.2 0.3 0.4 0.5");
|
||||
|
||||
Log::Fatal.ignoreInput = true;
|
||||
REQUIRE_THROWS_AS(ROCAUCScore<0>::Evaluate(rocTrueLabels, rocScoresOfPC),
|
||||
std::invalid_argument);
|
||||
REQUIRE_THROWS_AS(ROCAUCScore<1>::Evaluate(rocTrueLabels, rocScoresOfPC),
|
||||
std::invalid_argument);
|
||||
Log::Fatal.ignoreInput = false;
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
Reference in New Issue
Block a user