Add ShuffleData() with weights.

2018-05-31 16:36:19 -04:00
parent c37df2f8d4
commit cd37c054e2
2 changed files with 285 additions and 4 deletions
@@ -148,6 +148,97 @@ void ShuffleData(const MatType& inputPoints,
  }
 }

+/**
+ * Shuffle a dataset and associated labels (or responses) and weights.  It is
+ * expected that inputPoints and inputLabels and inputWeights have the same
+ * number of columns (so, be sure that inputLabels, if it is a vector, is a row
+ * vector).
+ *
+ * Shuffled data will be output into outputPoints and outputLabels and
+ * outputWeights.
+ */
+template<typename MatType, typename LabelsType, typename WeightsType>
+void ShuffleData(const MatType& inputPoints,
+                 const LabelsType& inputLabels,
+                 const WeightsType& inputWeights,
+                 MatType& outputPoints,
+                 LabelsType& outputLabels,
+                 WeightsType& outputWeights,
+                 const std::enable_if_t<!arma::is_SpMat<MatType>::value>* = 0,
+                 const std::enable_if_t<!arma::is_Cube<MatType>::value>* = 0)
+{
+  // Generate ordering.
+  arma::uvec ordering = arma::shuffle(arma::linspace<arma::uvec>(0,
+      inputPoints.n_cols - 1, inputPoints.n_cols));
+
+  outputPoints = inputPoints.cols(ordering);
+  outputLabels = inputLabels.cols(ordering);
+  outputWeights = inputWeights.cols(ordering);
+}
+
+/**
+ * Shuffle a sparse dataset and associated labels (or responses) and weights.
+ * It is expected that inputPoints and inputLabels and inputWeights have the
+ * same number of columns (so, be sure that inputLabels, if it is a vector, is a
+ * row vector).
+ *
+ * Shuffled data will be output into outputPoints and outputLabels and
+ * outputWeights.
+ */
+template<typename MatType, typename LabelsType, typename WeightsType>
+void ShuffleData(const MatType& inputPoints,
+                 const LabelsType& inputLabels,
+                 const WeightsType& inputWeights,
+                 MatType& outputPoints,
+                 LabelsType& outputLabels,
+                 WeightsType& outputWeights,
+                 const std::enable_if_t<arma::is_SpMat<MatType>::value>* = 0,
+                 const std::enable_if_t<!arma::is_Cube<MatType>::value>* = 0)
+{
+  // Generate ordering.
+  arma::uvec ordering = arma::shuffle(arma::linspace<arma::uvec>(0,
+      inputPoints.n_cols - 1, inputPoints.n_cols));
+
+  // Extract coordinate list representation.
+  arma::umat locations(2, inputPoints.n_nonzero);
+  arma::Col<typename MatType::elem_type> values(
+      const_cast<typename MatType::elem_type*>(inputPoints.values),
+      inputPoints.n_nonzero, false, true);
+  typename MatType::const_iterator it = inputPoints.begin();
+  size_t index = 0;
+  while (it != inputPoints.end())
+  {
+    locations(0, index) = it.row();
+    locations(1, index) = ordering[it.col()];
+    ++it;
+    ++index;
+  }
+
+  if (&inputPoints == &outputPoints || &inputLabels == &outputLabels ||
+      &inputWeights == &outputWeights)
+  {
+    MatType newOutputPoints(locations, values, inputPoints.n_rows,
+        inputPoints.n_cols, true);
+    LabelsType newOutputLabels(inputLabels.n_elem);
+    WeightsType newOutputWeights(inputWeights.n_elem);
+    newOutputLabels.cols(ordering) = inputLabels;
+    newOutputWeights.cols(ordering) = inputWeights;
+
+    outputPoints = std::move(newOutputPoints);
+    outputLabels = std::move(newOutputLabels);
+    outputWeights = std::move(newOutputWeights);
+  }
+  else
+  {
+    outputPoints = MatType(locations, values, inputPoints.n_rows,
+        inputPoints.n_cols, true);
+    outputLabels.set_size(inputLabels.n_elem);
+    outputLabels.cols(ordering) = inputLabels;
+    outputWeights.set_size(inputWeights.n_elem);
+    outputWeights.cols(ordering) = inputWeights;
+  }
+}
+
 } // namespace math
 } // namespace mlpack

@@ -612,7 +612,7 @@ BOOST_AUTO_TEST_CASE(ShuffleTest)
    BOOST_REQUIRE_EQUAL((size_t) outputData(0, i), outputLabels[i]);
    BOOST_REQUIRE_SMALL(outputData(1, i), 1e-5);
    BOOST_REQUIRE_SMALL(outputData(2, i), 1e-5);
-    counts[labels[i]]++;
+    counts[outputLabels[i]]++;
  }

  for (size_t i = 0; i < 10; ++i)
@@ -650,7 +650,7 @@ BOOST_AUTO_TEST_CASE(SparseShuffleTest)
    BOOST_REQUIRE_EQUAL((size_t) outputData(0, i), outputLabels[i]);
    BOOST_REQUIRE_SMALL((double) outputData(1, i), 1e-5);
    BOOST_REQUIRE_SMALL((double) outputData(2, i), 1e-5);
-    counts[labels[i]]++;
+    counts[outputLabels[i]]++;
  }

  for (size_t i = 0; i < 10; ++i)
@@ -701,6 +701,100 @@ BOOST_AUTO_TEST_CASE(CubeShuffleTest)
    BOOST_REQUIRE_EQUAL(counts[i], data.n_slices);
 }

+/**
+ * Make sure shuffling data with weights works.
+ */
+BOOST_AUTO_TEST_CASE(ShuffleWeightsTest)
+{
+  arma::mat data(3, 10, arma::fill::zeros);
+  arma::Row<size_t> labels(10);
+  arma::rowvec weights(10);
+  for (size_t i = 0; i < 10; ++i)
+  {
+    data(0, i) = i;
+    labels[i] = i;
+    weights[i] = i;
+  }
+
+  arma::mat outputData;
+  arma::Row<size_t> outputLabels;
+  arma::rowvec outputWeights;
+
+  ShuffleData(data, labels, weights, outputData, outputLabels, outputWeights);
+
+  BOOST_REQUIRE_EQUAL(outputData.n_rows, data.n_rows);
+  BOOST_REQUIRE_EQUAL(outputData.n_cols, data.n_cols);
+  BOOST_REQUIRE_EQUAL(outputLabels.n_elem, labels.n_elem);
+  BOOST_REQUIRE_EQUAL(outputWeights.n_elem, weights.n_elem);
+
+  // Make sure we only have each point once.
+  arma::Row<size_t> counts(10, arma::fill::zeros);
+  arma::Row<size_t> weightCounts(10, arma::fill::zeros);
+  for (size_t i = 0; i < 10; ++i)
+  {
+    BOOST_REQUIRE_EQUAL((size_t) outputData(0, i), outputLabels[i]);
+    BOOST_REQUIRE_EQUAL((size_t) outputData(0, i), (size_t) outputWeights[i]);
+    BOOST_REQUIRE_SMALL(outputData(1, i), 1e-5);
+    BOOST_REQUIRE_SMALL(outputData(2, i), 1e-5);
+    counts[outputLabels[i]]++;
+    weightCounts[(size_t) outputWeights[i]]++;
+  }
+
+  for (size_t i = 0; i < 10; ++i)
+  {
+    BOOST_REQUIRE_EQUAL(counts[i], 1);
+    BOOST_REQUIRE_EQUAL(weightCounts[i], 1);
+  }
+}
+
+/**
+ * Make sure shuffling sparse data with weights works.
+ */
+BOOST_AUTO_TEST_CASE(SparseShuffleWeightsTest)
+{
+  arma::sp_mat data(3, 10);
+  arma::Row<size_t> labels(10);
+  arma::rowvec weights(10);
+  for (size_t i = 0; i < 10; ++i)
+  {
+    data(0, i) = i;
+    labels[i] = i;
+    weights[i] = i;
+  }
+  // This appears to be a necessary workaround for an Armadillo 8 bug.
+  data *= 1.0;
+
+  arma::sp_mat outputData;
+  arma::Row<size_t> outputLabels;
+  arma::rowvec outputWeights;
+
+  ShuffleData(data, labels, weights, outputData, outputLabels, outputWeights);
+
+  BOOST_REQUIRE_EQUAL(outputData.n_rows, data.n_rows);
+  BOOST_REQUIRE_EQUAL(outputData.n_cols, data.n_cols);
+  BOOST_REQUIRE_EQUAL(outputLabels.n_elem, labels.n_elem);
+  BOOST_REQUIRE_EQUAL(outputWeights.n_elem, weights.n_elem);
+
+  // Make sure we only have each point once.
+  arma::Row<size_t> counts(10, arma::fill::zeros);
+  arma::Row<size_t> weightCounts(10, arma::fill::zeros);
+  for (size_t i = 0; i < 10; ++i)
+  {
+    BOOST_REQUIRE_EQUAL((size_t) outputData(0, i), outputLabels[i]);
+    BOOST_REQUIRE_EQUAL((size_t) outputData(0, i), (size_t) outputWeights[i]);
+    BOOST_REQUIRE_SMALL((double) outputData(1, i), 1e-5);
+    BOOST_REQUIRE_SMALL((double) outputData(2, i), 1e-5);
+    counts[outputLabels[i]]++;
+    weightCounts[(size_t) outputWeights[i]]++;
+  }
+
+  for (size_t i = 0; i < 10; ++i)
+  {
+    BOOST_REQUIRE_EQUAL(counts[i], 1);
+    BOOST_REQUIRE_EQUAL(weightCounts[i], 1);
+  }
+}
+
 /**
 * Make sure shuffling data works when the same matrices are given as input and
 * output.
@@ -731,7 +825,7 @@ BOOST_AUTO_TEST_CASE(InplaceShuffleTest)
    BOOST_REQUIRE_EQUAL((size_t) outputData(0, i), outputLabels[i]);
    BOOST_REQUIRE_SMALL(outputData(1, i), 1e-5);
    BOOST_REQUIRE_SMALL(outputData(2, i), 1e-5);
-    counts[labels[i]]++;
+    counts[outputLabels[i]]++;
  }

  for (size_t i = 0; i < 10; ++i)
@@ -768,7 +862,7 @@ BOOST_AUTO_TEST_CASE(InplaceSparseShuffleTest)
    BOOST_REQUIRE_EQUAL((size_t) outputData(0, i), outputLabels[i]);
    BOOST_REQUIRE_SMALL((double) outputData(1, i), 1e-5);
    BOOST_REQUIRE_SMALL((double) outputData(2, i), 1e-5);
-    counts[labels[i]]++;
+    counts[outputLabels[i]]++;
  }

  for (size_t i = 0; i < 10; ++i)
@@ -819,4 +913,100 @@ BOOST_AUTO_TEST_CASE(InplaceCubeShuffleTest)
    BOOST_REQUIRE_EQUAL(counts[i], data.n_slices);
 }

+/**
+ * Make sure shuffling data with weights works when the same matrices are given
+ * as input and output.
+ */
+BOOST_AUTO_TEST_CASE(InplaceShuffleWeightsTest)
+{
+  arma::mat data(3, 10, arma::fill::zeros);
+  arma::Row<size_t> labels(10);
+  arma::rowvec weights(10);
+  for (size_t i = 0; i < 10; ++i)
+  {
+    data(0, i) = i;
+    labels[i] = i;
+    weights[i] = i;
+  }
+
+  arma::mat outputData(data);
+  arma::Row<size_t> outputLabels(labels);
+  arma::rowvec outputWeights(weights);
+
+  ShuffleData(outputData, outputLabels, outputWeights, outputData, outputLabels,
+      outputWeights);
+
+  BOOST_REQUIRE_EQUAL(outputData.n_rows, data.n_rows);
+  BOOST_REQUIRE_EQUAL(outputData.n_cols, data.n_cols);
+  BOOST_REQUIRE_EQUAL(outputLabels.n_elem, labels.n_elem);
+  BOOST_REQUIRE_EQUAL(outputWeights.n_elem, weights.n_elem);
+
+  // Make sure we only have each point once.
+  arma::Row<size_t> counts(10, arma::fill::zeros);
+  arma::Row<size_t> weightCounts(10, arma::fill::zeros);
+  for (size_t i = 0; i < 10; ++i)
+  {
+    BOOST_REQUIRE_EQUAL((size_t) outputData(0, i), outputLabels[i]);
+    BOOST_REQUIRE_EQUAL((size_t) outputData(0, i), (size_t) outputWeights[i]);
+    BOOST_REQUIRE_SMALL(outputData(1, i), 1e-5);
+    BOOST_REQUIRE_SMALL(outputData(2, i), 1e-5);
+    counts[outputLabels[i]]++;
+    weightCounts[(size_t) outputWeights[i]]++;
+  }
+
+  for (size_t i = 0; i < 10; ++i)
+  {
+    BOOST_REQUIRE_EQUAL(counts[i], 1);
+    BOOST_REQUIRE_EQUAL(weightCounts[i], 1);
+  }
+}
+
+/**
+ * Make sure shuffling sparse data with weights works when the input and output
+ * matrices are the same.
+ */
+BOOST_AUTO_TEST_CASE(InplaceSparseShuffleWeightsTest)
+{
+  arma::sp_mat data(3, 10);
+  arma::Row<size_t> labels(10);
+  arma::rowvec weights(10);
+  for (size_t i = 0; i < 10; ++i)
+  {
+    data(0, i) = i;
+    labels[i] = i;
+    weights[i] = i;
+  }
+
+  arma::sp_mat outputData(data);
+  arma::Row<size_t> outputLabels(labels);
+  arma::rowvec outputWeights(weights);
+
+  ShuffleData(outputData, outputLabels, outputWeights, outputData, outputLabels,
+      outputWeights);
+
+  BOOST_REQUIRE_EQUAL(outputData.n_rows, data.n_rows);
+  BOOST_REQUIRE_EQUAL(outputData.n_cols, data.n_cols);
+  BOOST_REQUIRE_EQUAL(outputLabels.n_elem, labels.n_elem);
+  BOOST_REQUIRE_EQUAL(outputWeights.n_elem, weights.n_elem);
+
+  // Make sure we only have each point once.
+  arma::Row<size_t> counts(10, arma::fill::zeros);
+  arma::Row<size_t> weightCounts(10, arma::fill::zeros);
+  for (size_t i = 0; i < 10; ++i)
+  {
+    BOOST_REQUIRE_EQUAL((size_t) outputData(0, i), outputLabels[i]);
+    BOOST_REQUIRE_EQUAL((size_t) outputData(0, i), (size_t) outputWeights[i]);
+    BOOST_REQUIRE_SMALL((double) outputData(1, i), 1e-5);
+    BOOST_REQUIRE_SMALL((double) outputData(2, i), 1e-5);
+    counts[outputLabels[i]]++;
+    weightCounts[(size_t) outputWeights[i]]++;
+  }
+
+  for (size_t i = 0; i < 10; ++i)
+  {
+    BOOST_REQUIRE_EQUAL(counts[i], 1);
+    BOOST_REQUIRE_EQUAL(weightCounts[i], 1);
+  }
+}
+
 BOOST_AUTO_TEST_SUITE_END();