Merge pull request #3190 from mlpack/fix-perceptron-cv

Add weighted data constructor to `Perceptron`

COPYRIGHT.txt

@@ -7,7 +7,7 @@ Source:
 
 Files: *
 Copyright:
-  Copyright 2008-2021, Ryan Curtin <ryan@ratml.org>
+  Copyright 2008-2022, Ryan Curtin <ryan@ratml.org>
   Copyright 2008-2013, Bill March <march@gatech.edu>
   Copyright 2008-2012, Dongryeol Lee <dongryel@cc.gatech.edu>
   Copyright 2008-2013, Nishant Mehta <niche@cc.gatech.edu>

HISTORY.md

@@ -1,5 +1,7 @@
 ### mlpack ?.?.?
 ###### ????-??-??
+  * Fix `Perceptron` to work with cross-validation framework (#3190).
+
   * Migrate from boost tests to Catch2 framework (#2523), (#2584).
 
   * Bump minimum armadillo version from 8.400 to 9.800 (#3043), (#3048).

src/mlpack/methods/perceptron/perceptron.hpp

@@ -68,6 +68,29 @@ class Perceptron
              const size_t numClasses,
              const size_t maxIterations = 1000);
 
+  /**
+   * Constructor: construct the perceptron by building the weights matrix, which
+   * is later used in classification.  The number of classes should be specified
+   * separately, and the labels vector should contain values in the range [0,
+   * numClasses - 1].  The data::NormalizeLabels() function can be used if the
+   * labels vector does not contain values in the required range.
+   *
+   * This constructor supports weights for each data point.
+   *
+   * @param data Input, training data.
+   * @param labels Labels of dataset.
+   * @param numClasses Number of classes in the dataset.
+   * @param instanceWeights Weight vector to use for each training point while
+   *     training.
+   * @param maxIterations Maximum number of iterations for the perceptron
+   *     learning algorithm.
+   */
+  Perceptron(const MatType& data,
+             const arma::Row<size_t>& labels,
+             const size_t numClasses,
+             const arma::rowvec& instanceWeights,
+             const size_t maxIterations = 1000);
+
   /**
    * Alternate constructor which copies parameters from an already initiated
    * perceptron.

src/mlpack/methods/perceptron/perceptron_impl.hpp

@@ -62,6 +62,32 @@ Perceptron<LearnPolicy, WeightInitializationPolicy, MatType>::Perceptron(
   Train(data, labels, numClasses);
 }
 
+/**
+ * Constructor: construct the perceptron by building the weights matrix, which
+ * is later used in classification.  The number of classes should be specified
+ * separately, and the labels vector should contain values in the range [0,
+ * numClasses - 1].  The data::NormalizeLabels() function can be used if the
+ * labels vector does not contain values in the required range.
+ *
+ * This constructor supports weights for each data point.
+ */
+template<
+    typename LearnPolicy,
+    typename WeightInitializationPolicy,
+    typename MatType
+>
+Perceptron<LearnPolicy, WeightInitializationPolicy, MatType>::Perceptron(
+    const MatType& data,
+    const arma::Row<size_t>& labels,
+    const size_t numClasses,
+    const arma::rowvec& instanceWeights,
+    const size_t maxIterations) :
+    maxIterations(maxIterations)
+{
+  // Start training.
+  Train(data, labels, numClasses, instanceWeights);
+}
+
 /**
  * Alternate constructor which copies parameters from an already initiated
  * perceptron.
@@ -108,13 +134,14 @@ void Perceptron<LearnPolicy, WeightInitializationPolicy, MatType>::Classify(
 {
   arma::vec tempLabelMat;
   arma::uword maxIndex = 0;
+  predictedLabels.set_size(test.n_cols);
 
   // Could probably be faster if done in batch.
   for (size_t i = 0; i < test.n_cols; ++i)
   {
     tempLabelMat = weights.t() * test.col(i) + biases;
     tempLabelMat.max(maxIndex);
-    predictedLabels(0, i) = maxIndex;
+    predictedLabels(i) = maxIndex;
   }
 }
 

src/mlpack/tests/cv_test.cpp

@@ -33,6 +33,7 @@
 #include <mlpack/methods/linear_regression/linear_regression.hpp>
 #include <mlpack/methods/logistic_regression/logistic_regression.hpp>
 #include <mlpack/methods/naive_bayes/naive_bayes_classifier.hpp>
+#include <mlpack/methods/perceptron/perceptron.hpp>
 #include <mlpack/methods/softmax_regression/softmax_regression.hpp>
 #include <mlpack/core/data/confusion_matrix.hpp>
 #include <ensmallen.hpp>
@@ -44,6 +45,7 @@ using namespace mlpack;
 using namespace mlpack::ann;
 using namespace mlpack::cv;
 using namespace mlpack::naive_bayes;
+using namespace mlpack::perceptron;
 using namespace mlpack::regression;
 using namespace mlpack::tree;
 using namespace mlpack::data;
@@ -612,6 +614,33 @@ TEST_CASE("KFoldCVAccuracyTest", "[CVTest]")
   REQUIRE_NOTHROW(cv.Model());
 }
 
+/**
+ * Test k-fold cross-validation with the perceptron.
+ */
+TEST_CASE("KFoldCVPerceptronTest", "[CVTest]")
+{
+  // The same as the test above (for Naive Bayes), but with the perceptron.
+
+  // Making a 10-points dataset. The last point should be classified wrong when
+  // it is tested separately.
+  arma::mat data("0 1 2 3 100 101 102 103 104 5");
+  arma::Row<size_t> labels("0 0 0 0 1 1 1 1 1 1");
+  size_t numClasses = 2;
+
+  // 10-fold cross-validation, no shuffling.
+  KFoldCV<Perceptron<>, Accuracy> cv(10, data, labels, numClasses);
+
+  // We should succeed in classifying separately the first nine samples, and
+  // fail with the remaining one.
+  double expectedAccuracy = (9 * 1.0 + 0.0) / 10;
+
+  REQUIRE(cv.Evaluate() == Approx(expectedAccuracy).epsilon(1e-7));
+
+  // Assert we can access a trained model without the exception of
+  // uninitialization.
+  REQUIRE_NOTHROW(cv.Model());
+}
+
 /**
  * Test k-fold cross-validation with weighted linear regression.
  */

src/mlpack/tests/perceptron_test.cpp

@@ -120,7 +120,7 @@ TEST_CASE("And", "[PerceptronTest]")
   mat testData;
   testData = { { 0, 1, 1, 0 },
                { 1, 0, 1, 0 } };
-  Row<size_t> predictedLabels(testData.n_cols);
+  Row<size_t> predictedLabels;
   p.Classify(testData, predictedLabels);
 
   CHECK(predictedLabels(0, 0) == 0);
@@ -146,7 +146,7 @@ TEST_CASE("Or", "[PerceptronTest]")
   mat testData;
   testData = { { 0, 1, 1, 0 },
                { 1, 0, 1, 0 } };
-  Row<size_t> predictedLabels(testData.n_cols);
+  Row<size_t> predictedLabels;
   p.Classify(testData, predictedLabels);
 
   CHECK(predictedLabels(0, 0) == 1);
@@ -173,7 +173,7 @@ TEST_CASE("Random3", "[PerceptronTest]")
   mat testData;
   testData = { { 0, 1, 1 },
                { 1, 0, 1 } };
-  Row<size_t> predictedLabels(testData.n_cols);
+  Row<size_t> predictedLabels;
   p.Classify(testData, predictedLabels);
 
   for (size_t i = 0; i < predictedLabels.n_cols; ++i)
@@ -198,7 +198,7 @@ TEST_CASE("TwoPoints", "[PerceptronTest]")
   mat testData;
   testData = { { 0, 1 },
                { 1, 0 } };
-  Row<size_t> predictedLabels(testData.n_cols);
+  Row<size_t> predictedLabels;
   p.Classify(testData, predictedLabels);
 
   CHECK(predictedLabels(0, 0) == 0);
@@ -223,7 +223,7 @@ TEST_CASE("NonLinearlySeparableDataset", "[PerceptronTest]")
   mat testData;
   testData = { { 3,   4,   5,   6 },
                { 3, 2.3, 1.7, 1.5 } };
-  Row<size_t> predictedLabels(testData.n_cols);
+  Row<size_t> predictedLabels;
   p.Classify(testData, predictedLabels);
 
   CHECK(predictedLabels(0, 0) == 0);
@@ -244,4 +244,28 @@ TEST_CASE("SecondaryConstructor", "[PerceptronTest]")
   Perceptron<> p1(trainData, labels.row(0), 2, 1000);
 
   Perceptron<> p2(p1);
+
+  REQUIRE(p1.Weights().n_elem > 0);
+  REQUIRE(p2.Weights().n_elem > 0);
+}
+
+/**
+ * This tests that we can build the Perceptron when specifying instance weights.
+ */
+TEST_CASE("InstanceWeightsConstructor", "[PerceptronTest]")
+{
+  mat trainData;
+  trainData = { { 1, 2, 3, 4, 5, 6, 7, 8, 1, 2, 3, 4, 5, 6, 7, 8 },
+                { 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2 } };
+
+  Mat<size_t> labels;
+  labels = { 0, 0, 0, 1, 0, 1, 1, 1, 0, 0, 0, 1, 0, 1, 1, 1 };
+
+  rowvec instanceWeights;
+  instanceWeights = { 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0, 0.9,
+      0.8, 0.7, 0.6, 0.5, 0.4 };
+
+  Perceptron<> p(trainData, labels.row(0), 2, instanceWeights, 1000);
+
+  REQUIRE(p.Weights().n_elem > 0);
 }