From cd37c054e20ecaeecc5c576a5f8133a1bae2ae4a Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Thu, 31 May 2018 16:36:19 -0400 Subject: [PATCH] Add ShuffleData() with weights. --- src/mlpack/core/math/shuffle_data.hpp | 91 ++++++++++++ src/mlpack/tests/math_test.cpp | 198 +++++++++++++++++++++++++- 2 files changed, 285 insertions(+), 4 deletions(-) diff --git a/src/mlpack/core/math/shuffle_data.hpp b/src/mlpack/core/math/shuffle_data.hpp index 497c7ff89d..e90882d6f6 100644 --- a/src/mlpack/core/math/shuffle_data.hpp +++ b/src/mlpack/core/math/shuffle_data.hpp @@ -148,6 +148,97 @@ void ShuffleData(const MatType& inputPoints, } } +/** + * Shuffle a dataset and associated labels (or responses) and weights. It is + * expected that inputPoints and inputLabels and inputWeights have the same + * number of columns (so, be sure that inputLabels, if it is a vector, is a row + * vector). + * + * Shuffled data will be output into outputPoints and outputLabels and + * outputWeights. + */ +template +void ShuffleData(const MatType& inputPoints, + const LabelsType& inputLabels, + const WeightsType& inputWeights, + MatType& outputPoints, + LabelsType& outputLabels, + WeightsType& outputWeights, + const std::enable_if_t::value>* = 0, + const std::enable_if_t::value>* = 0) +{ + // Generate ordering. + arma::uvec ordering = arma::shuffle(arma::linspace(0, + inputPoints.n_cols - 1, inputPoints.n_cols)); + + outputPoints = inputPoints.cols(ordering); + outputLabels = inputLabels.cols(ordering); + outputWeights = inputWeights.cols(ordering); +} + +/** + * Shuffle a sparse dataset and associated labels (or responses) and weights. + * It is expected that inputPoints and inputLabels and inputWeights have the + * same number of columns (so, be sure that inputLabels, if it is a vector, is a + * row vector). + * + * Shuffled data will be output into outputPoints and outputLabels and + * outputWeights. + */ +template +void ShuffleData(const MatType& inputPoints, + const LabelsType& inputLabels, + const WeightsType& inputWeights, + MatType& outputPoints, + LabelsType& outputLabels, + WeightsType& outputWeights, + const std::enable_if_t::value>* = 0, + const std::enable_if_t::value>* = 0) +{ + // Generate ordering. + arma::uvec ordering = arma::shuffle(arma::linspace(0, + inputPoints.n_cols - 1, inputPoints.n_cols)); + + // Extract coordinate list representation. + arma::umat locations(2, inputPoints.n_nonzero); + arma::Col values( + const_cast(inputPoints.values), + inputPoints.n_nonzero, false, true); + typename MatType::const_iterator it = inputPoints.begin(); + size_t index = 0; + while (it != inputPoints.end()) + { + locations(0, index) = it.row(); + locations(1, index) = ordering[it.col()]; + ++it; + ++index; + } + + if (&inputPoints == &outputPoints || &inputLabels == &outputLabels || + &inputWeights == &outputWeights) + { + MatType newOutputPoints(locations, values, inputPoints.n_rows, + inputPoints.n_cols, true); + LabelsType newOutputLabels(inputLabels.n_elem); + WeightsType newOutputWeights(inputWeights.n_elem); + newOutputLabels.cols(ordering) = inputLabels; + newOutputWeights.cols(ordering) = inputWeights; + + outputPoints = std::move(newOutputPoints); + outputLabels = std::move(newOutputLabels); + outputWeights = std::move(newOutputWeights); + } + else + { + outputPoints = MatType(locations, values, inputPoints.n_rows, + inputPoints.n_cols, true); + outputLabels.set_size(inputLabels.n_elem); + outputLabels.cols(ordering) = inputLabels; + outputWeights.set_size(inputWeights.n_elem); + outputWeights.cols(ordering) = inputWeights; + } +} + } // namespace math } // namespace mlpack diff --git a/src/mlpack/tests/math_test.cpp b/src/mlpack/tests/math_test.cpp index 724991500b..2c7460b194 100644 --- a/src/mlpack/tests/math_test.cpp +++ b/src/mlpack/tests/math_test.cpp @@ -612,7 +612,7 @@ BOOST_AUTO_TEST_CASE(ShuffleTest) BOOST_REQUIRE_EQUAL((size_t) outputData(0, i), outputLabels[i]); BOOST_REQUIRE_SMALL(outputData(1, i), 1e-5); BOOST_REQUIRE_SMALL(outputData(2, i), 1e-5); - counts[labels[i]]++; + counts[outputLabels[i]]++; } for (size_t i = 0; i < 10; ++i) @@ -650,7 +650,7 @@ BOOST_AUTO_TEST_CASE(SparseShuffleTest) BOOST_REQUIRE_EQUAL((size_t) outputData(0, i), outputLabels[i]); BOOST_REQUIRE_SMALL((double) outputData(1, i), 1e-5); BOOST_REQUIRE_SMALL((double) outputData(2, i), 1e-5); - counts[labels[i]]++; + counts[outputLabels[i]]++; } for (size_t i = 0; i < 10; ++i) @@ -701,6 +701,100 @@ BOOST_AUTO_TEST_CASE(CubeShuffleTest) BOOST_REQUIRE_EQUAL(counts[i], data.n_slices); } +/** + * Make sure shuffling data with weights works. + */ +BOOST_AUTO_TEST_CASE(ShuffleWeightsTest) +{ + arma::mat data(3, 10, arma::fill::zeros); + arma::Row labels(10); + arma::rowvec weights(10); + for (size_t i = 0; i < 10; ++i) + { + data(0, i) = i; + labels[i] = i; + weights[i] = i; + } + + arma::mat outputData; + arma::Row outputLabels; + arma::rowvec outputWeights; + + ShuffleData(data, labels, weights, outputData, outputLabels, outputWeights); + + BOOST_REQUIRE_EQUAL(outputData.n_rows, data.n_rows); + BOOST_REQUIRE_EQUAL(outputData.n_cols, data.n_cols); + BOOST_REQUIRE_EQUAL(outputLabels.n_elem, labels.n_elem); + BOOST_REQUIRE_EQUAL(outputWeights.n_elem, weights.n_elem); + + // Make sure we only have each point once. + arma::Row counts(10, arma::fill::zeros); + arma::Row weightCounts(10, arma::fill::zeros); + for (size_t i = 0; i < 10; ++i) + { + BOOST_REQUIRE_EQUAL((size_t) outputData(0, i), outputLabels[i]); + BOOST_REQUIRE_EQUAL((size_t) outputData(0, i), (size_t) outputWeights[i]); + BOOST_REQUIRE_SMALL(outputData(1, i), 1e-5); + BOOST_REQUIRE_SMALL(outputData(2, i), 1e-5); + counts[outputLabels[i]]++; + weightCounts[(size_t) outputWeights[i]]++; + } + + for (size_t i = 0; i < 10; ++i) + { + BOOST_REQUIRE_EQUAL(counts[i], 1); + BOOST_REQUIRE_EQUAL(weightCounts[i], 1); + } +} + +/** + * Make sure shuffling sparse data with weights works. + */ +BOOST_AUTO_TEST_CASE(SparseShuffleWeightsTest) +{ + arma::sp_mat data(3, 10); + arma::Row labels(10); + arma::rowvec weights(10); + for (size_t i = 0; i < 10; ++i) + { + data(0, i) = i; + labels[i] = i; + weights[i] = i; + } + // This appears to be a necessary workaround for an Armadillo 8 bug. + data *= 1.0; + + arma::sp_mat outputData; + arma::Row outputLabels; + arma::rowvec outputWeights; + + ShuffleData(data, labels, weights, outputData, outputLabels, outputWeights); + + BOOST_REQUIRE_EQUAL(outputData.n_rows, data.n_rows); + BOOST_REQUIRE_EQUAL(outputData.n_cols, data.n_cols); + BOOST_REQUIRE_EQUAL(outputLabels.n_elem, labels.n_elem); + BOOST_REQUIRE_EQUAL(outputWeights.n_elem, weights.n_elem); + + // Make sure we only have each point once. + arma::Row counts(10, arma::fill::zeros); + arma::Row weightCounts(10, arma::fill::zeros); + for (size_t i = 0; i < 10; ++i) + { + BOOST_REQUIRE_EQUAL((size_t) outputData(0, i), outputLabels[i]); + BOOST_REQUIRE_EQUAL((size_t) outputData(0, i), (size_t) outputWeights[i]); + BOOST_REQUIRE_SMALL((double) outputData(1, i), 1e-5); + BOOST_REQUIRE_SMALL((double) outputData(2, i), 1e-5); + counts[outputLabels[i]]++; + weightCounts[(size_t) outputWeights[i]]++; + } + + for (size_t i = 0; i < 10; ++i) + { + BOOST_REQUIRE_EQUAL(counts[i], 1); + BOOST_REQUIRE_EQUAL(weightCounts[i], 1); + } +} + /** * Make sure shuffling data works when the same matrices are given as input and * output. @@ -731,7 +825,7 @@ BOOST_AUTO_TEST_CASE(InplaceShuffleTest) BOOST_REQUIRE_EQUAL((size_t) outputData(0, i), outputLabels[i]); BOOST_REQUIRE_SMALL(outputData(1, i), 1e-5); BOOST_REQUIRE_SMALL(outputData(2, i), 1e-5); - counts[labels[i]]++; + counts[outputLabels[i]]++; } for (size_t i = 0; i < 10; ++i) @@ -768,7 +862,7 @@ BOOST_AUTO_TEST_CASE(InplaceSparseShuffleTest) BOOST_REQUIRE_EQUAL((size_t) outputData(0, i), outputLabels[i]); BOOST_REQUIRE_SMALL((double) outputData(1, i), 1e-5); BOOST_REQUIRE_SMALL((double) outputData(2, i), 1e-5); - counts[labels[i]]++; + counts[outputLabels[i]]++; } for (size_t i = 0; i < 10; ++i) @@ -819,4 +913,100 @@ BOOST_AUTO_TEST_CASE(InplaceCubeShuffleTest) BOOST_REQUIRE_EQUAL(counts[i], data.n_slices); } +/** + * Make sure shuffling data with weights works when the same matrices are given + * as input and output. + */ +BOOST_AUTO_TEST_CASE(InplaceShuffleWeightsTest) +{ + arma::mat data(3, 10, arma::fill::zeros); + arma::Row labels(10); + arma::rowvec weights(10); + for (size_t i = 0; i < 10; ++i) + { + data(0, i) = i; + labels[i] = i; + weights[i] = i; + } + + arma::mat outputData(data); + arma::Row outputLabels(labels); + arma::rowvec outputWeights(weights); + + ShuffleData(outputData, outputLabels, outputWeights, outputData, outputLabels, + outputWeights); + + BOOST_REQUIRE_EQUAL(outputData.n_rows, data.n_rows); + BOOST_REQUIRE_EQUAL(outputData.n_cols, data.n_cols); + BOOST_REQUIRE_EQUAL(outputLabels.n_elem, labels.n_elem); + BOOST_REQUIRE_EQUAL(outputWeights.n_elem, weights.n_elem); + + // Make sure we only have each point once. + arma::Row counts(10, arma::fill::zeros); + arma::Row weightCounts(10, arma::fill::zeros); + for (size_t i = 0; i < 10; ++i) + { + BOOST_REQUIRE_EQUAL((size_t) outputData(0, i), outputLabels[i]); + BOOST_REQUIRE_EQUAL((size_t) outputData(0, i), (size_t) outputWeights[i]); + BOOST_REQUIRE_SMALL(outputData(1, i), 1e-5); + BOOST_REQUIRE_SMALL(outputData(2, i), 1e-5); + counts[outputLabels[i]]++; + weightCounts[(size_t) outputWeights[i]]++; + } + + for (size_t i = 0; i < 10; ++i) + { + BOOST_REQUIRE_EQUAL(counts[i], 1); + BOOST_REQUIRE_EQUAL(weightCounts[i], 1); + } +} + +/** + * Make sure shuffling sparse data with weights works when the input and output + * matrices are the same. + */ +BOOST_AUTO_TEST_CASE(InplaceSparseShuffleWeightsTest) +{ + arma::sp_mat data(3, 10); + arma::Row labels(10); + arma::rowvec weights(10); + for (size_t i = 0; i < 10; ++i) + { + data(0, i) = i; + labels[i] = i; + weights[i] = i; + } + + arma::sp_mat outputData(data); + arma::Row outputLabels(labels); + arma::rowvec outputWeights(weights); + + ShuffleData(outputData, outputLabels, outputWeights, outputData, outputLabels, + outputWeights); + + BOOST_REQUIRE_EQUAL(outputData.n_rows, data.n_rows); + BOOST_REQUIRE_EQUAL(outputData.n_cols, data.n_cols); + BOOST_REQUIRE_EQUAL(outputLabels.n_elem, labels.n_elem); + BOOST_REQUIRE_EQUAL(outputWeights.n_elem, weights.n_elem); + + // Make sure we only have each point once. + arma::Row counts(10, arma::fill::zeros); + arma::Row weightCounts(10, arma::fill::zeros); + for (size_t i = 0; i < 10; ++i) + { + BOOST_REQUIRE_EQUAL((size_t) outputData(0, i), outputLabels[i]); + BOOST_REQUIRE_EQUAL((size_t) outputData(0, i), (size_t) outputWeights[i]); + BOOST_REQUIRE_SMALL((double) outputData(1, i), 1e-5); + BOOST_REQUIRE_SMALL((double) outputData(2, i), 1e-5); + counts[outputLabels[i]]++; + weightCounts[(size_t) outputWeights[i]]++; + } + + for (size_t i = 0; i < 10; ++i) + { + BOOST_REQUIRE_EQUAL(counts[i], 1); + BOOST_REQUIRE_EQUAL(weightCounts[i], 1); + } +} + BOOST_AUTO_TEST_SUITE_END();