Merge pull request #2022 from KimSangYeon-DGU/nbc

Fix some potential infinity errors in Naive Bayes Classifier.
2019-09-29 20:45:40 +09:00
parent c996ba583d 259a2afd8d
commit 42d154fd0b
8 changed files with 578 additions and 20 deletions
@@ -6,6 +6,8 @@

 ### mlpack 3.2.0
 ###### 2019-09-25
+  * Fix some potential infinity errors in Naive Bayes Classifier (#2022).
+
  * Fix occasionally-failing RADICAL test (#1924).

  * Fix gcc 9 OpenMP compilation issue (#1970).
@@ -78,12 +78,14 @@ class NaiveBayesClassifier
   * @param incrementalVariance If true, an incremental algorithm is used to
   *     calculate the variance; this can prevent loss of precision in some
   *     cases, but will be somewhat slower to calculate.
+   * @param epsilon Small value to prevent log of zero.
   */
  template<typename MatType>
  NaiveBayesClassifier(const MatType& data,
                       const arma::Row<size_t>& labels,
                       const size_t numClasses,
-                       const bool incrementalVariance = false);
+                       const bool incrementalVariance = false,
+                       const double epsilon = 1e-10);

  /**
   * Initialize the Naive Bayes classifier without performing training.  All of
@@ -92,7 +94,8 @@ class NaiveBayesClassifier
   * meaningless.
   */
  NaiveBayesClassifier(const size_t dimensionality = 0,
-                       const size_t numClasses = 0);
+                       const size_t numClasses = 0,
+                       const double epsilon = 1e-10);

  /**
   * Train the Naive Bayes classifier on the given dataset.  If the incremental
@@ -224,6 +227,8 @@ class NaiveBayesClassifier
  ModelMatType probabilities;
  //! Number of training points seen so far.
  size_t trainingPoints;
+  //! Small value to prevent log of zero.
+  double epsilon;

  /**
   * Compute the unnormalized posterior log probability of given points (log
@@ -31,8 +31,10 @@ NaiveBayesClassifier<ModelMatType>::NaiveBayesClassifier(
    const MatType& data,
    const arma::Row<size_t>& labels,
    const size_t numClasses,
-    const bool incremental) :
-    trainingPoints(0) // Set when we call Train().
+    const bool incremental,
+    const double epsilon) :
+    trainingPoints(0), // Set when we call Train().
+    epsilon(epsilon)
 {
  static_assert(std::is_same<ElemType, typename MatType::elem_type>::value,
      "NaiveBayesClassifier: element type of given data must match the element "
@@ -60,8 +62,10 @@ NaiveBayesClassifier<ModelMatType>::NaiveBayesClassifier(
 template<typename ModelMatType>
 NaiveBayesClassifier<ModelMatType>::NaiveBayesClassifier(
    const size_t dimensionality,
-    const size_t numClasses) :
-    trainingPoints(0)
+    const size_t numClasses,
+    const double epsilon) :
+    trainingPoints(0),
+    epsilon(epsilon)
 {
  // Initialize model to 0.
  probabilities.zeros(numClasses);
@@ -164,10 +168,8 @@ void NaiveBayesClassifier<ModelMatType>::Train(
        variances.col(i) /= (probabilities[i] - 1);
  }

-  // Ensure that the variances are invertible.
-  for (size_t i = 0; i < variances.n_elem; ++i)
-    if (variances[i] == 0.0)
-      variances[i] = 1e-50;
+  // Add epsilon to prevent log of zero.
+  variances += epsilon;

  probabilities /= data.n_cols;
  trainingPoints += data.n_cols;
@@ -224,7 +226,7 @@ void NaiveBayesClassifier<ModelMatType>::LogLikelihood(
    arma::Mat<ElemType> exponents = arma::sum(diffs % rhs, 0);

    logLikelihoods.row(i) += (data.n_rows / -2.0 * log(2 * M_PI) - 0.5 *
-        std::log(arma::det(arma::diagmat(variances.col(i)))) + exponents);
+        arma::accu(arma::log(variances.col(i))) + exponents);
  }
 }

@@ -266,13 +268,17 @@ void NaiveBayesClassifier<ModelMatType>::Classify(
  // term.
  ModelMatType logLikelihoods;
  LogLikelihood(point, logLikelihoods);
-  const double logProbX = log(arma::accu(exp(logLikelihoods))); // Log(Prob(X)).
-  logLikelihoods -= logProbX;
+
+  // To prevent underflow in log of sum of exp of x operation (where x is a
+  // small negative value), we use logsumexp(x - max(x)) + max(x).
+  const double maxValue = arma::max(logLikelihoods);
+  const double logProbX = log(arma::accu(exp(logLikelihoods - maxValue))) +
+      maxValue;
+  probabilities = exp(logLikelihoods - logProbX); // log(exp(value)) == value.

  arma::uword maxIndex = 0;
  logLikelihoods.max(maxIndex);
  prediction = (size_t) maxIndex;
-  probabilities = exp(logLikelihoods); // log(exp(value)) == value.
 }

 template<typename ModelMatType>
@@ -318,16 +324,20 @@ void NaiveBayesClassifier<ModelMatType>::Classify(
  ModelMatType logLikelihoods;
  LogLikelihood(data, logLikelihoods);

-  // This will hold log(Prob(X)) for each point.
-  arma::Col<ElemType> logProbX(data.n_cols);
+  predictionProbs.set_size(arma::size(logLikelihoods));
+  double maxValue, logProbX;
  for (size_t j = 0; j < data.n_cols; ++j)
  {
-    logProbX(j) = log(arma::accu(exp(logLikelihoods.col(j))));
-    logLikelihoods.col(j) -= logProbX(j);
+    // The LogLikelihood() gives us the unnormalized log likelihood which is
+    // Log(Prob(X|Y)) + Log(Prob(Y)), so we subtract the normalization term.
+    // Besides, to prevent underflow in log of sum of exp of x operation (where
+    // x is a small negative value), we use logsumexp(x - max(x)) + max(x).
+    maxValue = arma::max(logLikelihoods.col(j));
+    logProbX = log(arma::accu(exp(logLikelihoods.col(j) -
+        maxValue))) + maxValue;
+    predictionProbs.col(j) = arma::exp(logLikelihoods.col(j) - logProbX);
  }

-  predictionProbs = arma::exp(logLikelihoods);
-
  // Now calculate maximum probabilities for each point.
  for (size_t i = 0; i < data.n_cols; ++i)
  {
@@ -0,0 +1,50 @@
+3
+2
+0
+0
+0
+1
+2
+3
+3
+2
+4
+2
+1
+2
+3
+1
+2
+4
+4
+1
+3
+0
+2
+0
+0
+2
+0
+1
+3
+3
+2
+2
+2
+3
+3
+3
+3
+3
+0
+0
+4
+3
+3
+0
+3
+2
+3
+2
+1
+1
@@ -0,0 +1,200 @@
+1
+4
+2
+2
+1
+0
+1
+0
+0
+4
+0
+4
+3
+4
+3
+2
+4
+2
+2
+2
+4
+1
+2
+1
+3
+0
+4
+1
+4
+4
+4
+0
+3
+4
+3
+1
+3
+2
+3
+0
+4
+1
+4
+1
+4
+2
+1
+4
+2
+1
+2
+0
+2
+2
+4
+2
+0
+2
+0
+3
+3
+3
+0
+2
+1
+4
+3
+1
+2
+2
+4
+0
+1
+3
+4
+4
+4
+2
+4
+2
+3
+4
+4
+3
+2
+3
+3
+4
+3
+4
+2
+4
+0
+3
+3
+1
+3
+4
+2
+1
+2
+3
+1
+3
+3
+0
+4
+0
+0
+3
+2
+1
+0
+3
+2
+1
+0
+0
+1
+0
+2
+2
+4
+2
+3
+1
+4
+4
+2
+3
+4
+0
+2
+2
+0
+4
+0
+3
+1
+4
+4
+2
+0
+0
+0
+0
+3
+4
+3
+2
+0
+4
+3
+3
+4
+0
+3
+1
+3
+4
+3
+2
+2
+4
+0
+0
+0
+0
+1
+4
+0
+3
+4
+3
+1
+4
+0
+1
+4
+3
+2
+1
+3
+2
+4
+3
+2
+0
+1
+4
+2
+0
+2
+3
+0
+0
+2
+1
+3
+1
@@ -314,4 +314,45 @@ BOOST_AUTO_TEST_CASE(SeparateTrainIndividualIncrementalTest)
  }
 }

+/**
+ * Check if NaiveBayesClassifier::Classify() works properly for a high
+ * dimension datasets.
+ */
+BOOST_AUTO_TEST_CASE(NaiveBayesClassifierHighDimensionsTest)
+{
+  // Set file names of dataset of training and test.
+  // The training dataset has 5 classes and each class has 1,000 dimensions.
+  const char* trainFilename = "nbc_high_dim_train.csv";
+  const char* testFilename = "nbc_high_dim_test.csv";
+  const char* trainLabelsFileName = "nbc_high_dim_train_labels.csv";
+  const char* testLabelsFilename = "nbc_high_dim_test_labels.csv";
+
+  size_t classes = 5;
+
+  // Create variables for training and assign data to them.
+  arma::mat trainData;
+  arma::Row<size_t> trainLabels;
+  data::Load(trainFilename, trainData, true);
+  data::Load(trainLabelsFileName, trainLabels, true);
+
+  // Initialize and train a NBC model.
+  NaiveBayesClassifier<> nbcTest(trainData, trainLabels, classes);
+
+  // Create variables for test and assign data to them.
+  arma::mat testData, calcProbs;
+  arma::Row<size_t> testLabels;
+  arma::Row<size_t> calcVec;
+  data::Load(testFilename, testData, true);
+  data::Load(testLabelsFilename, testLabels, true);
+
+  // Classify observations in the test dataset. To use Classify() method with
+  // a parameter for probabilities of predictions, we pass 'calcProbs' to the
+  // method.
+  nbcTest.Classify(testData, calcVec, calcProbs);
+
+  // Check the results.
+  for (size_t i = 0; i < calcVec.n_cols; i++)
+    BOOST_REQUIRE_EQUAL(calcVec(i), testLabels(i));
+}
+
 BOOST_AUTO_TEST_SUITE_END();