From d8e177b60282135e44d2d13d079c1ae12a516436 Mon Sep 17 00:00:00 2001 From: Shikhar S Date: Tue, 31 Dec 2019 19:43:32 +0530 Subject: [PATCH 01/55] Refactored format guessing code, added parameter to specify data file type --- HISTORY.md | 2 + src/mlpack/core/data/load.cpp | 18 ++- src/mlpack/core/data/load.hpp | 21 ++-- src/mlpack/core/data/load_impl.hpp | 109 ++++++++++------- src/mlpack/core/data/save.hpp | 3 +- src/mlpack/core/data/save_impl.hpp | 186 ++++++++++++++++++----------- 6 files changed, 211 insertions(+), 128 deletions(-) diff --git a/HISTORY.md b/HISTORY.md index 51c6793dd8..f9cd3b9953 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,5 +1,7 @@ ### mlpack ?.?.? ###### ????-??-?? + * Added option to specify type of data file while loading/saving data. + * Added `probabilities_file` parameter to get the probabilities matrix of AdaBoost classifier (#2050). diff --git a/src/mlpack/core/data/load.cpp b/src/mlpack/core/data/load.cpp index 16cae0bc0f..c7e775f54f 100644 --- a/src/mlpack/core/data/load.cpp +++ b/src/mlpack/core/data/load.cpp @@ -18,32 +18,38 @@ namespace data /** Functions to load and save matrices and models. */ { template bool Load(const std::string&, arma::Mat&, const bool, - const bool); + const bool, + arma::file_type); template bool Load(const std::string&, arma::Mat&, const bool, - const bool); + const bool, + arma::file_type); template bool Load(const std::string&, arma::Mat&, const bool, - const bool); + const bool, + arma::file_type); template bool Load(const std::string&, arma::Mat&, const bool, - const bool); + const bool, + arma::file_type); template bool Load(const std::string&, arma::Mat&, const bool, - const bool); + const bool, + arma::file_type); template bool Load(const std::string&, arma::Mat&, const bool, - const bool); + const bool, + arma::file_type); template bool Load(const std::string&, arma::Mat&, diff --git a/src/mlpack/core/data/load.hpp b/src/mlpack/core/data/load.hpp index 7974227409..0fbb6a2df6 100644 --- a/src/mlpack/core/data/load.hpp +++ b/src/mlpack/core/data/load.hpp @@ -63,7 +63,8 @@ template bool Load(const std::string& filename, arma::Mat& matrix, const bool fatal = false, - const bool transpose = true); + const bool transpose = true, + arma::file_type inputLoadType = arma::auto_detect); /** * Don't document these with doxygen; these declarations aren't helpful to @@ -75,33 +76,39 @@ bool Load(const std::string& filename, extern template bool Load(const std::string&, arma::Mat&, const bool, - const bool); + const bool, + arma::file_type); // size_t and uword should be one of these three typedefs. extern template bool Load(const std::string&, arma::Mat&, const bool, - const bool); + const bool, + arma::file_type); extern template bool Load(const std::string&, arma::Mat&, const bool, - const bool); + const bool, + arma::file_type); extern template bool Load(const std::string&, arma::Mat&, const bool, - const bool); + const bool, + arma::file_type); extern template bool Load(const std::string&, arma::Mat&, const bool, - const bool); + const bool, + arma::file_type); extern template bool Load(const std::string&, arma::Mat&, const bool, - const bool); + const bool, + arma::file_type); /** * @endcond diff --git a/src/mlpack/core/data/load_impl.hpp b/src/mlpack/core/data/load_impl.hpp index 1d65743455..4b0e3562e3 100644 --- a/src/mlpack/core/data/load_impl.hpp +++ b/src/mlpack/core/data/load_impl.hpp @@ -77,51 +77,25 @@ bool inline inplace_transpose(arma::Mat& X) } } -template -bool Load(const std::string& filename, - arma::Mat& matrix, - const bool fatal, - const bool transpose) +inline +std::string AutoDetect(std::fstream& stream,const std::string filename,arma::file_type& detectedLoadType,const bool fatal) { - Timer::Start("loading_data"); - // Get the extension. std::string extension = Extension(filename); - - // Catch nonexistent files by opening the stream ourselves. - std::fstream stream; -#ifdef _WIN32 // Always open in binary mode on Windows. - stream.open(filename.c_str(), std::fstream::in | std::fstream::binary); -#else - stream.open(filename.c_str(), std::fstream::in); -#endif - if (!stream.is_open()) - { - Timer::Stop("loading_data"); - if (fatal) - Log::Fatal << "Cannot open file '" << filename << "'. " << std::endl; - else - Log::Warn << "Cannot open file '" << filename << "'; load failed." - << std::endl; - - return false; - } - - bool unknownType = false; - arma::file_type loadType; - std::string stringType; + std::string stringType=""; + bool unknownType=false; if (extension == "csv" || extension == "tsv") { - loadType = arma::diskio::guess_file_type(stream); - if (loadType == arma::csv_ascii) + detectedLoadType = arma::diskio::guess_file_type(stream); + if (detectedLoadType == arma::csv_ascii) { if (extension == "tsv") Log::Warn << "'" << filename << "' is comma-separated, not " "tab-separated!" << std::endl; stringType = "CSV data"; } - else if (loadType == arma::raw_ascii) // .csv file can be tsv. + else if (detectedLoadType == arma::raw_ascii) // .csv file can be tsv. { if (extension == "csv") { @@ -150,7 +124,7 @@ bool Load(const std::string& filename, else { unknownType = true; - loadType = arma::raw_binary; // Won't be used; prevent a warning. + detectedLoadType = arma::raw_binary; // Won't be used; prevent a warning. stringType = ""; } } @@ -172,16 +146,16 @@ bool Load(const std::string& filename, if (rawHeader == ARMA_MAT_TXT) { - loadType = arma::arma_ascii; + detectedLoadType = arma::arma_ascii; stringType = "Armadillo ASCII formatted data"; } else // It's not arma_ascii. Now we let Armadillo guess. { - loadType = arma::diskio::guess_file_type(stream); + detectedLoadType = arma::diskio::guess_file_type(stream); - if (loadType == arma::raw_ascii) // Raw ASCII (space-separated). + if (detectedLoadType == arma::raw_ascii) // Raw ASCII (space-separated). stringType = "raw ASCII formatted data"; - else if (loadType == arma::csv_ascii) // CSV can be .txt too. + else if (detectedLoadType == arma::csv_ascii) // CSV can be .txt too. stringType = "CSV data"; else // Unknown .txt... we will throw an error. unknownType = true; @@ -203,24 +177,24 @@ bool Load(const std::string& filename, if (rawHeader == ARMA_MAT_BIN) { stringType = "Armadillo binary formatted data"; - loadType = arma::arma_binary; + detectedLoadType = arma::arma_binary; } else // We can only assume it's raw binary. { stringType = "raw binary formatted data"; - loadType = arma::raw_binary; + detectedLoadType = arma::raw_binary; } } else if (extension == "pgm") { - loadType = arma::pgm_binary; + detectedLoadType = arma::pgm_binary; stringType = "PGM data"; } else if (extension == "h5" || extension == "hdf5" || extension == "hdf" || extension == "he5") { #ifdef ARMA_USE_HDF5 - loadType = arma::hdf5_binary; + detectedLoadType = arma::hdf5_binary; stringType = "HDF5 data"; #else Timer::Stop("loading_data"); @@ -233,19 +207,20 @@ bool Load(const std::string& filename, << "Armadillo was compiled without HDF5 support. Load failed." << std::endl; - return false; + return ""; #endif } else // Unknown extension... { unknownType = true; - loadType = arma::raw_binary; // Won't be used; prevent a warning. + detectedLoadType = arma::raw_binary; // Won't be used; prevent a warning. stringType = ""; } // Provide error if we don't know the type. if (unknownType) { + stringType=""; Timer::Stop("loading_data"); if (fatal) Log::Fatal << "Unable to detect type of '" << filename << "'; " @@ -253,7 +228,53 @@ bool Load(const std::string& filename, else Log::Warn << "Unable to detect type of '" << filename << "'; load failed." << " Incorrect extension?" << std::endl; + } + return stringType; //Empty string denotes undetected file type. +} +template +bool Load(const std::string& filename, + arma::Mat& matrix, + const bool fatal, + const bool transpose, + arma::file_type inputLoadType) +{ + Timer::Start("loading_data"); + + // Catch nonexistent files by opening the stream ourselves. + std::fstream stream; +#ifdef _WIN32 // Always open in binary mode on Windows. + stream.open(filename.c_str(), std::fstream::in | std::fstream::binary); +#else + stream.open(filename.c_str(), std::fstream::in); +#endif + if (!stream.is_open()) + { + Timer::Stop("loading_data"); + if (fatal) + Log::Fatal << "Cannot open file '" << filename << "'. " << std::endl; + else + Log::Warn << "Cannot open file '" << filename << "'; load failed." + << std::endl; + + return false; + } + + arma::file_type loadType; + std::string stringType; + + if(inputLoadType==arma::file_type::auto_detect) + { + stringType = AutoDetect(stream,filename,loadType,fatal); + } + else + { + loadType=inputLoadType; + stringType=GetStringType(loadType); + } + //If file type is not detected, return failure for load. + if(stringType=="") + { return false; } diff --git a/src/mlpack/core/data/save.hpp b/src/mlpack/core/data/save.hpp index 121265ac81..16015afe06 100644 --- a/src/mlpack/core/data/save.hpp +++ b/src/mlpack/core/data/save.hpp @@ -57,7 +57,8 @@ template bool Save(const std::string& filename, const arma::Mat& matrix, const bool fatal = false, - bool transpose = true); + bool transpose = true, + arma::file_type inputSaveType = arma::auto_detect); /** * Saves a model to file, guessing the filetype from the extension, or, diff --git a/src/mlpack/core/data/save_impl.hpp b/src/mlpack/core/data/save_impl.hpp index 18a61530a5..5f96ae43a0 100644 --- a/src/mlpack/core/data/save_impl.hpp +++ b/src/mlpack/core/data/save_impl.hpp @@ -41,14 +41,32 @@ bool Save(const std::string& filename, return Save(filename, rowvec, fatal, true); } -template -bool Save(const std::string& filename, - const arma::Mat& matrix, - const bool fatal, - bool transpose) +inline +std::string GetStringType(const arma::file_type& loadType) { - Timer::Start("saving_data"); + switch(loadType) + { + case arma::csv_ascii : return "CSV data"; + case arma::raw_ascii : return "raw ASCII formatted data"; + case arma::raw_binary : return "raw binary formatted data"; + case arma::arma_ascii : return "Armadillo ASCII formatted data"; + case arma::arma_binary : return "Armadillo binary formatted data"; + case arma::pgm_binary : return "PGM data"; + case arma::hdf5_binary : + { + #ifdef ARMA_USE_HDF5 + return "HDF5 data"; + #else + return ""; + #endif + } + default : return ""; + } +} +inline +std::string AutoDetect(const std::string& filename,arma::file_type& detectedSaveType,const bool fatal) +{ // First we will try to discriminate by file extension. std::string extension = Extension(filename); if (extension == "") @@ -61,6 +79,98 @@ bool Save(const std::string& filename, Log::Warn << "No extension given with filename '" << filename << "'; " << "type unknown. Save failed." << std::endl; + return ""; + } + + std::string stringType; + bool unknownType=false; + + if (extension == "csv") + { + detectedSaveType = arma::csv_ascii; + stringType = "CSV data"; + } + else if (extension == "txt") + { + detectedSaveType = arma::raw_ascii; + stringType = "raw ASCII formatted data"; + } + else if (extension == "bin") + { + detectedSaveType = arma::arma_binary; + stringType = "Armadillo binary formatted data"; + } + else if (extension == "pgm") + { + detectedSaveType = arma::pgm_binary; + stringType = "PGM data"; + } + else if (extension == "h5" || extension == "hdf5" || extension == "hdf" || + extension == "he5") + { +#ifdef ARMA_USE_HDF5 + detectedSaveType = arma::hdf5_binary; + stringType = "HDF5 data"; +#else + Timer::Stop("saving_data"); + if (fatal) + Log::Fatal << "Attempted to save HDF5 data to '" << filename << "', but " + << "Armadillo was compiled without HDF5 support. Save failed." + << std::endl; + else + Log::Warn << "Attempted to save HDF5 data to '" << filename << "', but " + << "Armadillo was compiled without HDF5 support. Save failed." + << std::endl; + + return ""; +#endif + } + else + { + unknownType=true; + detectedSaveType = arma::raw_binary; // Won't be used; prevent a warning. + stringType = ""; + } + + // Provide error if we don't know the type. + if (unknownType) + { + Timer::Stop("saving_data"); + if (fatal) + Log::Fatal << "Unable to determine format to save to from filename '" + << filename << "'. Save failed." << std::endl; + else + Log::Warn << "Unable to determine format to save to from filename '" + << filename << "'. Save failed." << std::endl; + + } + return stringType; +} + +template +bool Save(const std::string& filename, + const arma::Mat& matrix, + const bool fatal, + bool transpose, + arma::file_type inputSaveType) +{ + Timer::Start("saving_data"); + + arma::file_type saveType; + std::string stringType = ""; + + if(inputSaveType == arma::auto_detect) + { + stringType = AutoDetect(filename,saveType,fatal); + } + else + { + stringType = GetStringType(saveType); + } + //If File Type is not automatically detected from extension or no extension + //is specified then return failure. + if(stringType == "") + { return false; } @@ -84,70 +194,6 @@ bool Save(const std::string& filename, return false; } - bool unknownType = false; - arma::file_type saveType; - std::string stringType; - - if (extension == "csv") - { - saveType = arma::csv_ascii; - stringType = "CSV data"; - } - else if (extension == "txt") - { - saveType = arma::raw_ascii; - stringType = "raw ASCII formatted data"; - } - else if (extension == "bin") - { - saveType = arma::arma_binary; - stringType = "Armadillo binary formatted data"; - } - else if (extension == "pgm") - { - saveType = arma::pgm_binary; - stringType = "PGM data"; - } - else if (extension == "h5" || extension == "hdf5" || extension == "hdf" || - extension == "he5") - { -#ifdef ARMA_USE_HDF5 - saveType = arma::hdf5_binary; - stringType = "HDF5 data"; -#else - Timer::Stop("saving_data"); - if (fatal) - Log::Fatal << "Attempted to save HDF5 data to '" << filename << "', but " - << "Armadillo was compiled without HDF5 support. Save failed." - << std::endl; - else - Log::Warn << "Attempted to save HDF5 data to '" << filename << "', but " - << "Armadillo was compiled without HDF5 support. Save failed." - << std::endl; - - return false; -#endif - } - else - { - unknownType = true; - saveType = arma::raw_binary; // Won't be used; prevent a warning. - stringType = ""; - } - - // Provide error if we don't know the type. - if (unknownType) - { - Timer::Stop("saving_data"); - if (fatal) - Log::Fatal << "Unable to determine format to save to from filename '" - << filename << "'. Save failed." << std::endl; - else - Log::Warn << "Unable to determine format to save to from filename '" - << filename << "'. Save failed." << std::endl; - - return false; - } // Try to save the file. Log::Info << "Saving " << stringType << " to '" << filename << "'." From 9731a2f18a3beebe8627f93c23fdfa005f969616 Mon Sep 17 00:00:00 2001 From: Shikhar S Date: Tue, 31 Dec 2019 20:24:20 +0530 Subject: [PATCH 02/55] fix style errors --- src/mlpack/core/data/load_impl.hpp | 27 +++++++++++++++------------ src/mlpack/core/data/save_impl.hpp | 29 +++++++++++++++-------------- 2 files changed, 30 insertions(+), 26 deletions(-) diff --git a/src/mlpack/core/data/load_impl.hpp b/src/mlpack/core/data/load_impl.hpp index 4b0e3562e3..93b3d37861 100644 --- a/src/mlpack/core/data/load_impl.hpp +++ b/src/mlpack/core/data/load_impl.hpp @@ -78,12 +78,15 @@ bool inline inplace_transpose(arma::Mat& X) } inline -std::string AutoDetect(std::fstream& stream,const std::string filename,arma::file_type& detectedLoadType,const bool fatal) +std::string AutoDetect(std::fstream& stream, + const std::string filename, + arma::file_type& detectedLoadType, + const bool fatal) { // Get the extension. std::string extension = Extension(filename); - std::string stringType=""; - bool unknownType=false; + std::string stringType = ""; + bool unknownType = false; if (extension == "csv" || extension == "tsv") { @@ -220,7 +223,7 @@ std::string AutoDetect(std::fstream& stream,const std::string filename,arma::fil // Provide error if we don't know the type. if (unknownType) { - stringType=""; + stringType = ""; Timer::Stop("loading_data"); if (fatal) Log::Fatal << "Unable to detect type of '" << filename << "'; " @@ -229,7 +232,7 @@ std::string AutoDetect(std::fstream& stream,const std::string filename,arma::fil Log::Warn << "Unable to detect type of '" << filename << "'; load failed." << " Incorrect extension?" << std::endl; } - return stringType; //Empty string denotes undetected file type. + return stringType; // Empty string denotes undetected file type. } template @@ -262,18 +265,18 @@ bool Load(const std::string& filename, arma::file_type loadType; std::string stringType; - - if(inputLoadType==arma::file_type::auto_detect) + + if (inputLoadType == arma::file_type::auto_detect) { - stringType = AutoDetect(stream,filename,loadType,fatal); + stringType = AutoDetect(stream, filename, loadType, fatal); } else { - loadType=inputLoadType; - stringType=GetStringType(loadType); + loadType = inputLoadType; + stringType = GetStringType(loadType); } - //If file type is not detected, return failure for load. - if(stringType=="") + // If file type is not detected, return failure for load. + if (stringType == "") { return false; } diff --git a/src/mlpack/core/data/save_impl.hpp b/src/mlpack/core/data/save_impl.hpp index 5f96ae43a0..fed16d0adb 100644 --- a/src/mlpack/core/data/save_impl.hpp +++ b/src/mlpack/core/data/save_impl.hpp @@ -44,7 +44,7 @@ bool Save(const std::string& filename, inline std::string GetStringType(const arma::file_type& loadType) { - switch(loadType) + switch (loadType) { case arma::csv_ascii : return "CSV data"; case arma::raw_ascii : return "raw ASCII formatted data"; @@ -52,7 +52,7 @@ std::string GetStringType(const arma::file_type& loadType) case arma::arma_ascii : return "Armadillo ASCII formatted data"; case arma::arma_binary : return "Armadillo binary formatted data"; case arma::pgm_binary : return "PGM data"; - case arma::hdf5_binary : + case arma::hdf5_binary : { #ifdef ARMA_USE_HDF5 return "HDF5 data"; @@ -65,7 +65,9 @@ std::string GetStringType(const arma::file_type& loadType) } inline -std::string AutoDetect(const std::string& filename,arma::file_type& detectedSaveType,const bool fatal) +std::string AutoDetect(const std::string& filename, + arma::file_type& detectedSaveType, + const bool fatal) { // First we will try to discriminate by file extension. std::string extension = Extension(filename); @@ -83,7 +85,7 @@ std::string AutoDetect(const std::string& filename,arma::file_type& detectedSave } std::string stringType; - bool unknownType=false; + bool unknownType = false; if (extension == "csv") { @@ -127,11 +129,11 @@ std::string AutoDetect(const std::string& filename,arma::file_type& detectedSave } else { - unknownType=true; + unknownType = true; detectedSaveType = arma::raw_binary; // Won't be used; prevent a warning. stringType = ""; } - + // Provide error if we don't know the type. if (unknownType) { @@ -142,7 +144,6 @@ std::string AutoDetect(const std::string& filename,arma::file_type& detectedSave else Log::Warn << "Unable to determine format to save to from filename '" << filename << "'. Save failed." << std::endl; - } return stringType; } @@ -155,21 +156,21 @@ bool Save(const std::string& filename, arma::file_type inputSaveType) { Timer::Start("saving_data"); - + arma::file_type saveType; std::string stringType = ""; - if(inputSaveType == arma::auto_detect) + if (inputSaveType == arma::auto_detect) { - stringType = AutoDetect(filename,saveType,fatal); + stringType = AutoDetect(filename, saveType, fatal); } else { - stringType = GetStringType(saveType); + stringType = GetStringType(saveType); } - //If File Type is not automatically detected from extension or no extension - //is specified then return failure. - if(stringType == "") + // If File Type is not automatically detected from extension or no extension + // is specified then return failure. + if (stringType == "") { return false; } From 1b0769d8dd1f5e3312472430bfd9d5ec16d56657 Mon Sep 17 00:00:00 2001 From: Shikhar S Date: Wed, 1 Jan 2020 01:06:57 +0530 Subject: [PATCH 03/55] Issue description in HISTORY.md --- HISTORY.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/HISTORY.md b/HISTORY.md index f9cd3b9953..0af589dd1e 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,6 +1,6 @@ ### mlpack ?.?.? ###### ????-??-?? - * Added option to specify type of data file while loading/saving data. + * Add manual type specification support to `data::Load()` and `data::Save()` (#2084). * Added `probabilities_file` parameter to get the probabilities matrix of AdaBoost classifier (#2050). From cd8904867133769dbb2e9e7ea245bb371ed09627 Mon Sep 17 00:00:00 2001 From: Shikhar S Date: Thu, 2 Jan 2020 20:28:10 +0530 Subject: [PATCH 04/55] Added Wrong Extension Failure Check --- src/mlpack/tests/load_save_test.cpp | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/src/mlpack/tests/load_save_test.cpp b/src/mlpack/tests/load_save_test.cpp index 783e0065b0..91f490881f 100644 --- a/src/mlpack/tests/load_save_test.cpp +++ b/src/mlpack/tests/load_save_test.cpp @@ -50,6 +50,28 @@ BOOST_AUTO_TEST_CASE(NotExistLoad) BOOST_REQUIRE(data::Load("nonexistentfile_______________.csv", out) == false); } +/** + * Make sure load fails if the file extension is wrong in automatic detection mode. + */ +BOOST_AUTO_TEST_CASE(WrongExtensionLoad) +{ + //Try to load arma::arma_binary file with ".csv" extension + arma::mat test = "1 5;" + "2 6;" + "3 7;" + "4 8;"; + + arma::mat testTrans = trans(test); + BOOST_REQUIRE(testTrans.quiet_save("test_file.csv", arma::arma_binary) + == true); + + // Now reload through our interface. + BOOST_REQUIRE(data::Load("test_file.csv", test) == false); + + // Remove the file. + remove("test_file.csv"); +} + /** * Make sure a CSV is loaded correctly. */ From 0d2a99b8aed8abb452622cfedbaebed5bc35e6dd Mon Sep 17 00:00:00 2001 From: Shikhar S Date: Thu, 2 Jan 2020 20:43:57 +0530 Subject: [PATCH 05/55] Fix style errors --- src/mlpack/tests/load_save_test.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mlpack/tests/load_save_test.cpp b/src/mlpack/tests/load_save_test.cpp index 91f490881f..5294901fde 100644 --- a/src/mlpack/tests/load_save_test.cpp +++ b/src/mlpack/tests/load_save_test.cpp @@ -55,7 +55,7 @@ BOOST_AUTO_TEST_CASE(NotExistLoad) */ BOOST_AUTO_TEST_CASE(WrongExtensionLoad) { - //Try to load arma::arma_binary file with ".csv" extension + // Try to load arma::arma_binary file with ".csv" extension arma::mat test = "1 5;" "2 6;" "3 7;" From 916f606aa61b4e9cb46ee336c3f6ea1f75abbaf8 Mon Sep 17 00:00:00 2001 From: Shikhar S Date: Tue, 7 Jan 2020 14:34:08 +0530 Subject: [PATCH 06/55] Added Tests, Added documentation for additional parameter, Changed auto_detect constant --- src/mlpack/core/data/load.hpp | 1 + src/mlpack/core/data/load_impl.hpp | 3 +-- src/mlpack/tests/load_save_test.cpp | 30 ++++++++++++++++++++++++++++- 3 files changed, 31 insertions(+), 3 deletions(-) diff --git a/src/mlpack/core/data/load.hpp b/src/mlpack/core/data/load.hpp index 0fbb6a2df6..11b263e3f7 100644 --- a/src/mlpack/core/data/load.hpp +++ b/src/mlpack/core/data/load.hpp @@ -57,6 +57,7 @@ namespace data /** Functions to load and save matrices and models. */ { * @param matrix Matrix to load contents of file into. * @param fatal If an error should be reported as fatal (default false). * @param transpose If true, transpose the matrix after loading. + * @param inputLoadType Used to determine the type of file to load (default arma::auto_detect). * @return Boolean value indicating success or failure of load. */ template diff --git a/src/mlpack/core/data/load_impl.hpp b/src/mlpack/core/data/load_impl.hpp index 93b3d37861..db90bc4b2f 100644 --- a/src/mlpack/core/data/load_impl.hpp +++ b/src/mlpack/core/data/load_impl.hpp @@ -265,8 +265,7 @@ bool Load(const std::string& filename, arma::file_type loadType; std::string stringType; - - if (inputLoadType == arma::file_type::auto_detect) + if (inputLoadType == arma::auto_detect) { stringType = AutoDetect(stream, filename, loadType, fatal); } diff --git a/src/mlpack/tests/load_save_test.cpp b/src/mlpack/tests/load_save_test.cpp index 5294901fde..f7a47ea56f 100644 --- a/src/mlpack/tests/load_save_test.cpp +++ b/src/mlpack/tests/load_save_test.cpp @@ -53,7 +53,7 @@ BOOST_AUTO_TEST_CASE(NotExistLoad) /** * Make sure load fails if the file extension is wrong in automatic detection mode. */ -BOOST_AUTO_TEST_CASE(WrongExtensionLoad) +BOOST_AUTO_TEST_CASE(WrongExtensionWrongLoad) { // Try to load arma::arma_binary file with ".csv" extension arma::mat test = "1 5;" @@ -72,6 +72,34 @@ BOOST_AUTO_TEST_CASE(WrongExtensionLoad) remove("test_file.csv"); } +/** + * Make sure load is successful even if the file extension is wrong when file type is specified. + */ +BOOST_AUTO_TEST_CASE(WrongExtensionCorrectLoad) +{ + // Try to load arma::arma_binary file with ".csv" extension + arma::mat test = "1 5;" + "2 6;" + "3 7;" + "4 8;"; + + arma::mat testTrans = trans(test); + BOOST_REQUIRE(testTrans.quiet_save("test_file.csv", arma::arma_binary) + == true); + + // Now reload through our interface. + BOOST_REQUIRE(data::Load("test_file.csv", test,false,true,arma::arma_binary) == true); + + BOOST_REQUIRE_EQUAL(test.n_rows, 4); + BOOST_REQUIRE_EQUAL(test.n_cols, 2); + + for (size_t i = 0; i < 8; i++) + BOOST_REQUIRE_CLOSE(test[i], (double) (i + 1), 1e-5); + + // Remove the file. + remove("test_file.csv"); +} + /** * Make sure a CSV is loaded correctly. */ From 7dc5861ab5415ec30e6ac8ed582087a9ee5b9b4e Mon Sep 17 00:00:00 2001 From: Shikhar S Date: Wed, 8 Jan 2020 09:40:32 +0530 Subject: [PATCH 07/55] Fix style errors, Copied guess_file_type from armadillo --- src/mlpack/core/data/load_impl.hpp | 75 ++++++++++++++++++++++++++++- src/mlpack/tests/load_save_test.cpp | 4 +- 2 files changed, 76 insertions(+), 3 deletions(-) diff --git a/src/mlpack/core/data/load_impl.hpp b/src/mlpack/core/data/load_impl.hpp index db90bc4b2f..7d89c98bcc 100644 --- a/src/mlpack/core/data/load_impl.hpp +++ b/src/mlpack/core/data/load_impl.hpp @@ -77,6 +77,77 @@ bool inline inplace_transpose(arma::Mat& X) } } +inline arma::file_type guess_file_type(std::istream& f) +{ + //Taken from armadillo's function guess_file_type_internal + f.clear(); + const std::fstream::pos_type pos1 = f.tellg(); + + f.clear(); + f.seekg(0, arma::ios::end); + + f.clear(); + const std::fstream::pos_type pos2 = f.tellg(); + + const arma::uword N_max = ((pos1 >= 0) && (pos2 >= 0) && (pos2 > pos1)) ? + arma::uword(pos2 - pos1) : arma::uword(0); + + f.clear(); + f.seekg(pos1); + + if (N_max == 0) + return arma::file_type_unknown; + + const arma::uword N_use = (std::min)(N_max, arma::uword(4096)); + + arma::podarray data(N_use); + data.zeros(); + + unsigned char* data_mem = data.memptr(); + + f.clear(); + f.read( reinterpret_cast(data_mem), std::streamsize(N_use)); + + const bool load_okay = f.good(); + + f.clear(); + f.seekg(pos1); + + if (load_okay == false) + return arma::file_type_unknown; + + bool has_binary = false; + bool has_bracket = false; + bool has_comma = false; + + for(arma::uword i=0; i= 123)) + { + has_binary = true; + break; + } // the range checking can be made more elaborate + + if ((val == '(') || (val == ')')) + { + has_bracket = true; + } + if (val == ',') + { + has_comma = true; + } + } + + if (has_binary) + return arma::raw_binary; + + if (has_comma && (has_bracket == false)) + return arma::csv_ascii; + + return arma::raw_ascii; +} + inline std::string AutoDetect(std::fstream& stream, const std::string filename, @@ -90,7 +161,7 @@ std::string AutoDetect(std::fstream& stream, if (extension == "csv" || extension == "tsv") { - detectedLoadType = arma::diskio::guess_file_type(stream); + detectedLoadType = guess_file_type(stream); if (detectedLoadType == arma::csv_ascii) { if (extension == "tsv") @@ -154,7 +225,7 @@ std::string AutoDetect(std::fstream& stream, } else // It's not arma_ascii. Now we let Armadillo guess. { - detectedLoadType = arma::diskio::guess_file_type(stream); + detectedLoadType = guess_file_type(stream); if (detectedLoadType == arma::raw_ascii) // Raw ASCII (space-separated). stringType = "raw ASCII formatted data"; diff --git a/src/mlpack/tests/load_save_test.cpp b/src/mlpack/tests/load_save_test.cpp index f7a47ea56f..453ebba93e 100644 --- a/src/mlpack/tests/load_save_test.cpp +++ b/src/mlpack/tests/load_save_test.cpp @@ -88,7 +88,9 @@ BOOST_AUTO_TEST_CASE(WrongExtensionCorrectLoad) == true); // Now reload through our interface. - BOOST_REQUIRE(data::Load("test_file.csv", test,false,true,arma::arma_binary) == true); + BOOST_REQUIRE( + data::Load("test_file.csv", test, false, true, arma::arma_binary) + == true); BOOST_REQUIRE_EQUAL(test.n_rows, 4); BOOST_REQUIRE_EQUAL(test.n_cols, 2); From f61ef1b6a4da72bce1ad4cc96ec9eb221a299ec4 Mon Sep 17 00:00:00 2001 From: Shikhar S Date: Wed, 8 Jan 2020 09:50:42 +0530 Subject: [PATCH 08/55] Fix style errors --- src/mlpack/core/data/load_impl.hpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/mlpack/core/data/load_impl.hpp b/src/mlpack/core/data/load_impl.hpp index 7d89c98bcc..b6b1139bf5 100644 --- a/src/mlpack/core/data/load_impl.hpp +++ b/src/mlpack/core/data/load_impl.hpp @@ -79,7 +79,7 @@ bool inline inplace_transpose(arma::Mat& X) inline arma::file_type guess_file_type(std::istream& f) { - //Taken from armadillo's function guess_file_type_internal + // Taken from armadillo's function guess_file_type_internal f.clear(); const std::fstream::pos_type pos1 = f.tellg(); @@ -106,7 +106,7 @@ inline arma::file_type guess_file_type(std::istream& f) unsigned char* data_mem = data.memptr(); f.clear(); - f.read( reinterpret_cast(data_mem), std::streamsize(N_use)); + f.read(reinterpret_cast(data_mem), std::streamsize(N_use)); const bool load_okay = f.good(); @@ -120,7 +120,7 @@ inline arma::file_type guess_file_type(std::istream& f) bool has_bracket = false; bool has_comma = false; - for(arma::uword i=0; i= 123)) From 405225aecaeef1c450058d1d7bfc959c6c92f267 Mon Sep 17 00:00:00 2001 From: Shikhar S Date: Wed, 8 Jan 2020 10:14:08 +0530 Subject: [PATCH 09/55] Function name style fix --- src/mlpack/core/data/load_impl.hpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/mlpack/core/data/load_impl.hpp b/src/mlpack/core/data/load_impl.hpp index b6b1139bf5..fd1d5fdd0f 100644 --- a/src/mlpack/core/data/load_impl.hpp +++ b/src/mlpack/core/data/load_impl.hpp @@ -77,7 +77,7 @@ bool inline inplace_transpose(arma::Mat& X) } } -inline arma::file_type guess_file_type(std::istream& f) +inline arma::file_type GuessFileType(std::istream& f) { // Taken from armadillo's function guess_file_type_internal f.clear(); @@ -161,7 +161,7 @@ std::string AutoDetect(std::fstream& stream, if (extension == "csv" || extension == "tsv") { - detectedLoadType = guess_file_type(stream); + detectedLoadType = GuessFileType(stream); if (detectedLoadType == arma::csv_ascii) { if (extension == "tsv") @@ -225,7 +225,7 @@ std::string AutoDetect(std::fstream& stream, } else // It's not arma_ascii. Now we let Armadillo guess. { - detectedLoadType = guess_file_type(stream); + detectedLoadType = GuessFileType(stream); if (detectedLoadType == arma::raw_ascii) // Raw ASCII (space-separated). stringType = "raw ASCII formatted data"; From 9ea45ce30723df97c6afb5e74c3350055cdf0ad3 Mon Sep 17 00:00:00 2001 From: Shikhar S Date: Wed, 8 Jan 2020 13:24:39 +0530 Subject: [PATCH 10/55] Removed dependency on arma internal PODvector, Added comments --- src/mlpack/core/data/load_impl.hpp | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/src/mlpack/core/data/load_impl.hpp b/src/mlpack/core/data/load_impl.hpp index fd1d5fdd0f..d8ba3dceca 100644 --- a/src/mlpack/core/data/load_impl.hpp +++ b/src/mlpack/core/data/load_impl.hpp @@ -84,26 +84,27 @@ inline arma::file_type GuessFileType(std::istream& f) const std::fstream::pos_type pos1 = f.tellg(); f.clear(); - f.seekg(0, arma::ios::end); + f.seekg(0, std::ios::end); f.clear(); - const std::fstream::pos_type pos2 = f.tellg(); + const std::fstream::pos_type pos2 = f.tellg(); // pos2 holds length of stream + // Compute length of stream in N_max const arma::uword N_max = ((pos1 >= 0) && (pos2 >= 0) && (pos2 > pos1)) ? arma::uword(pos2 - pos1) : arma::uword(0); f.clear(); f.seekg(pos1); + // Handle empty files. if (N_max == 0) return arma::file_type_unknown; const arma::uword N_use = (std::min)(N_max, arma::uword(4096)); - arma::podarray data(N_use); - data.zeros(); - - unsigned char* data_mem = data.memptr(); + unsigned char* data_mem = (unsigned char*) + malloc(sizeof(unsigned char) * N_use); + memset(data_mem,0,N_use); f.clear(); f.read(reinterpret_cast(data_mem), std::streamsize(N_use)); @@ -114,7 +115,10 @@ inline arma::file_type GuessFileType(std::istream& f) f.seekg(pos1); if (load_okay == false) + { + delete data_mem; return arma::file_type_unknown; + } bool has_binary = false; bool has_bracket = false; @@ -139,6 +143,8 @@ inline arma::file_type GuessFileType(std::istream& f) } } + delete data_mem; + if (has_binary) return arma::raw_binary; From d8ff8983fde576e3ce88bd10a2b55d5cf956b14b Mon Sep 17 00:00:00 2001 From: Shikhar S Date: Wed, 8 Jan 2020 13:30:58 +0530 Subject: [PATCH 11/55] Fix style errors! --- src/mlpack/core/data/load_impl.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/mlpack/core/data/load_impl.hpp b/src/mlpack/core/data/load_impl.hpp index d8ba3dceca..4fdfd22efa 100644 --- a/src/mlpack/core/data/load_impl.hpp +++ b/src/mlpack/core/data/load_impl.hpp @@ -102,9 +102,9 @@ inline arma::file_type GuessFileType(std::istream& f) const arma::uword N_use = (std::min)(N_max, arma::uword(4096)); - unsigned char* data_mem = (unsigned char*) + unsigned char* data_mem = (unsigned char*) malloc(sizeof(unsigned char) * N_use); - memset(data_mem,0,N_use); + memset(data_mem, 0, N_use); f.clear(); f.read(reinterpret_cast(data_mem), std::streamsize(N_use)); From e2fe5bb8c47dc96dc76ea5508d9416f80052c40c Mon Sep 17 00:00:00 2001 From: jeffin143 Date: Sat, 29 Aug 2020 20:16:09 +0530 Subject: [PATCH 12/55] migrate _network_* test --- src/mlpack/tests/CMakeLists.txt | 8 +-- src/mlpack/tests/feedforward_network_test.cpp | 41 +++++------ src/mlpack/tests/rbm_network_test.cpp | 23 +++--- src/mlpack/tests/recurrent_network_test.cpp | 71 +++++++++---------- 4 files changed, 63 insertions(+), 80 deletions(-) diff --git a/src/mlpack/tests/CMakeLists.txt b/src/mlpack/tests/CMakeLists.txt index cc9fad9bff..ffbfe0d1bc 100644 --- a/src/mlpack/tests/CMakeLists.txt +++ b/src/mlpack/tests/CMakeLists.txt @@ -16,7 +16,6 @@ add_executable(mlpack_test emst_test.cpp fastmks_test.cpp facilities_test.cpp - feedforward_network_test.cpp gan_test.cpp gmm_test.cpp hmm_test.cpp @@ -60,9 +59,7 @@ add_executable(mlpack_test random_forest_test.cpp random_test.cpp range_search_test.cpp - rbm_network_test.cpp rectangle_tree_test.cpp - recurrent_network_test.cpp reward_clipping_test.cpp rl_components_test.cpp serialization.cpp @@ -137,6 +134,7 @@ add_executable(mlpack_catch_test cv_test.cpp decision_stump_test.cpp decision_tree_test.cpp + feedforward_network_test.cpp image_load_test.cpp imputation_test.cpp kernel_pca_test.cpp @@ -150,6 +148,8 @@ add_executable(mlpack_catch_test one_hot_encoding_test.cpp quic_svd_test.cpp randomized_svd_test.cpp + rbm_network_test.cpp + recurrent_network_test.cpp regularized_svd_test.cpp scaling_test.cpp serialization_catch.cpp @@ -229,8 +229,6 @@ add_custom_command(TARGET mlpack_test set(parallel_tests "AsyncLearningTest;" "LocalCoordinateCodingTest;" - "FeedForwardNetworkTest;" - "RecurrentNetworkTest;" "GMMTest;" "CFTest;" "HMMTest;" diff --git a/src/mlpack/tests/feedforward_network_test.cpp b/src/mlpack/tests/feedforward_network_test.cpp index 47842204e6..f9a338e0b3 100644 --- a/src/mlpack/tests/feedforward_network_test.cpp +++ b/src/mlpack/tests/feedforward_network_test.cpp @@ -19,17 +19,14 @@ #include -#include -#include "test_tools.hpp" -#include "serialization.hpp" +#include "catch.hpp" +#include "serialization_catch.hpp" #include "custom_layer.hpp" using namespace mlpack; using namespace mlpack::ann; using namespace mlpack::kmeans; -BOOST_AUTO_TEST_SUITE(FeedForwardNetworkTest); - /** * Train and evaluate a model with the specified structure. */ @@ -57,13 +54,13 @@ void TestNetwork(ModelType& model, size_t correct = arma::accu(prediction == testLabels); double classificationError = 1 - double(correct) / testData.n_cols; - BOOST_REQUIRE_LE(classificationError, classificationErrorThreshold); + REQUIRE(classificationError <= classificationErrorThreshold); } /** * Train the vanilla network on a larger dataset. */ -BOOST_AUTO_TEST_CASE(VanillaNetworkTest) +TEST_CASE("FFVanillaNetworkTest", "[FeedForwardNetworkTest]") { // Load the dataset. arma::mat trainData; @@ -131,7 +128,7 @@ BOOST_AUTO_TEST_CASE(VanillaNetworkTest) TestNetwork<>(model1, dataset, labels, dataset, labels, 10, 0.2); } -BOOST_AUTO_TEST_CASE(ForwardBackwardTest) +TEST_CASE("ForwardBackwardTest", "[FeedForwardNetworkTest]") { arma::mat dataset; dataset.load("mnist_first250_training_4s_and_9s.arm"); @@ -204,13 +201,13 @@ BOOST_AUTO_TEST_CASE(ForwardBackwardTest) } } - BOOST_REQUIRE(converged); + REQUIRE(converged); } /** * Train the dropout network on a larger dataset. */ -BOOST_AUTO_TEST_CASE(DropoutNetworkTest) +TEST_CASE("DropoutNetworkTest", "[FeedForwardNetworkTest]") { // Load the dataset. arma::mat trainData; @@ -284,7 +281,7 @@ BOOST_AUTO_TEST_CASE(DropoutNetworkTest) /** * Train the highway network on a larger dataset. */ -BOOST_AUTO_TEST_CASE(HighwayNetworkTest) +TEST_CASE("HighwayNetworkTest", "[FeedForwardNetworkTest]") { arma::mat dataset; dataset.load("mnist_first250_training_4s_and_9s.arm"); @@ -311,7 +308,7 @@ BOOST_AUTO_TEST_CASE(HighwayNetworkTest) /** * Train the DropConnect network on a larger dataset. */ -BOOST_AUTO_TEST_CASE(DropConnectNetworkTest) +TEST_CASE("DropConnectNetworkTest", "[FeedForwardNetworkTest]") { // Load the dataset. arma::mat trainData; @@ -385,7 +382,7 @@ BOOST_AUTO_TEST_CASE(DropConnectNetworkTest) * Test miscellaneous things of FFN, * e.g. copy/move constructor, assignment operator. */ -BOOST_AUTO_TEST_CASE(FFNMiscTest) +TEST_CASE("FFNMiscTest", "[FeedForwardNetworkTest]") { FFN> model; model.Add>(2, 3); @@ -400,7 +397,7 @@ BOOST_AUTO_TEST_CASE(FFNMiscTest) /** * Test that serialization works ok. */ -BOOST_AUTO_TEST_CASE(SerializationTest) +TEST_CASE("FFSerializationTest", "[FeedForwardNetworkTest]") { // Load the dataset. arma::mat trainData; @@ -449,7 +446,7 @@ BOOST_AUTO_TEST_CASE(SerializationTest) * Test if the custom layers work. The target is to see if the code compiles * when the Train and Prediction are called. */ -BOOST_AUTO_TEST_CASE(CustomLayerTest) +TEST_CASE("CustomLayerTest", "[FeedForwardNetworkTest]") { // Load the dataset. arma::mat trainData; @@ -481,7 +478,7 @@ BOOST_AUTO_TEST_CASE(CustomLayerTest) /** * Test the overload of Forward function which allows partial forward pass. */ -BOOST_AUTO_TEST_CASE(PartialForwardTest) +TEST_CASE("PartialForwardTest", "[FeedForwardNetworkTest]") { FFN, RandomInitialization> model; model.Add >(5, 10); @@ -528,7 +525,7 @@ BOOST_AUTO_TEST_CASE(PartialForwardTest) /** * Test that FFN::Train() returns finite objective value. */ -BOOST_AUTO_TEST_CASE(FFNTrainReturnObjective) +TEST_CASE("FFNTrainReturnObjective", "[FeedForwardNetworkTest]") { // Load the dataset. arma::mat trainData; @@ -557,13 +554,13 @@ BOOST_AUTO_TEST_CASE(FFNTrainReturnObjective) double objVal = model.Train(trainData, trainLabels, opt); - BOOST_REQUIRE_EQUAL(std::isfinite(objVal), true); + REQUIRE(std::isfinite(objVal) == true); } /** * Test that FFN::Model() allows us to access the instantiated network. */ -BOOST_AUTO_TEST_CASE(FFNReturnModel) +TEST_CASE("FFNReturnModel", "[FeedForwardNetworkTest]") { // Create dummy network. FFN > model; @@ -598,7 +595,7 @@ BOOST_AUTO_TEST_CASE(FFNReturnModel) * Test to see if the FFN code compiles when the Optimizer * doesn't have the MaxIterations() method. */ -BOOST_AUTO_TEST_CASE(OptimizerTest) +TEST_CASE("OptimizerTest", "[FeedForwardNetworkTest]") { // Load the dataset. arma::mat trainData; @@ -626,7 +623,7 @@ BOOST_AUTO_TEST_CASE(OptimizerTest) /** * Train the RBF network on a larger dataset. */ -BOOST_AUTO_TEST_CASE(RBFNetworkTest) +TEST_CASE("RBFNetworkTest", "[FeedForwardNetworkTest]") { // Load the dataset. arma::mat trainData; @@ -703,5 +700,3 @@ BOOST_AUTO_TEST_CASE(RBFNetworkTest) // RBFN neural net with MeanSquaredError. TestNetwork<>(model1, dataset, labels1, dataset, labels, 10, 0.1); } - -BOOST_AUTO_TEST_SUITE_END(); diff --git a/src/mlpack/tests/rbm_network_test.cpp b/src/mlpack/tests/rbm_network_test.cpp index 820f9616f5..f14e2726bd 100644 --- a/src/mlpack/tests/rbm_network_test.cpp +++ b/src/mlpack/tests/rbm_network_test.cpp @@ -26,20 +26,17 @@ #include #include -#include -#include "test_tools.hpp" +#include "catch.hpp" using namespace mlpack; using namespace mlpack::ann; using namespace ens; using namespace mlpack::regression; -BOOST_AUTO_TEST_SUITE(RBMNetworkTest); - /* * Tests the BinaryRBM implementation on the Digits dataset. */ -BOOST_AUTO_TEST_CASE(BinaryRBMClassificationTest) +TEST_CASE("BinaryRBMClassificationTest", "[RBMNetworkTest]") { // Normalised dataset. int hiddenLayerSize = 100; @@ -84,7 +81,7 @@ BOOST_AUTO_TEST_CASE(BinaryRBMClassificationTest) double objVal = model.Train(msgd); // Test that objective value returned by RBM::Train() is finite. - BOOST_REQUIRE_EQUAL(std::isfinite(objVal), true); + REQUIRE(std::isfinite(objVal) == true); for (size_t i = 0; i < trainData.n_cols; ++i) { @@ -117,13 +114,13 @@ BOOST_AUTO_TEST_CASE(BinaryRBMClassificationTest) // We allow a 6% tolerance because the RBM may not reconstruct samples as // well. (Typically it does, but we have no guarantee.) - BOOST_REQUIRE_GE(rbmClassificationAccuracy, classificationAccuracy - 6.0); + REQUIRE(rbmClassificationAccuracy >= classificationAccuracy - 6.0); } /* * Tests the SpikeSlabRBM implementation on the Digits dataset. */ -BOOST_AUTO_TEST_CASE(ssRBMClassificationTest) +TEST_CASE("ssRBMClassificationTest", "[RBMNetworkTest]") { size_t batchSize = 10; size_t numEpoches = 3; @@ -184,7 +181,7 @@ BOOST_AUTO_TEST_CASE(ssRBMClassificationTest) double objVal = modelssRBM.Train(msgd); // Test that objective value returned by RBM::Train() is finite. - BOOST_REQUIRE_EQUAL(std::isfinite(objVal), true); + REQUIRE(std::isfinite(objVal) == true); for (size_t i = 0; i < trainData.n_cols; ++i) { @@ -211,7 +208,7 @@ BOOST_AUTO_TEST_CASE(ssRBMClassificationTest) // omitted here for speed. We add a margin of 3% since ssRBM isn't guaranteed // to give us better results (we just generally expect it to be about as good // or better). - BOOST_REQUIRE_GE(ssRbmClassificationAccuracy, 76.18 - 3.0); + REQUIRE(ssRbmClassificationAccuracy >= 76.18 - 3.0); } template @@ -239,13 +236,13 @@ void BuildVanillaNetwork(MatType& trainData, } for (size_t i = 0; i < freeEnergy.n_elem; ++i) - BOOST_REQUIRE_CLOSE(calculatedFreeEnergy(i), freeEnergy(i), 1e-3); + REQUIRE(calculatedFreeEnergy(i) == Approx(freeEnergy(i)).epsilon(1e-5)); } /* * Train and evaluate a Vanilla network with the specified structure. */ -BOOST_AUTO_TEST_CASE(MiscTest) +TEST_CASE("MiscTest", "[RBMNetworkTest]") { arma::Mat X = arma::Mat("0.0, 0.0, 0.0;" "0.0, 1.0, 1.0;" @@ -254,5 +251,3 @@ BOOST_AUTO_TEST_CASE(MiscTest) X = X.t(); BuildVanillaNetwork>(X, 2); } - -BOOST_AUTO_TEST_SUITE_END(); diff --git a/src/mlpack/tests/recurrent_network_test.cpp b/src/mlpack/tests/recurrent_network_test.cpp index 10635e80fb..5bedcbd3cc 100644 --- a/src/mlpack/tests/recurrent_network_test.cpp +++ b/src/mlpack/tests/recurrent_network_test.cpp @@ -19,9 +19,8 @@ #include #include -#include -#include "test_tools.hpp" -#include "serialization.hpp" +#include "catch.hpp" +#include "serialization_catch.hpp" #include "custom_layer.hpp" using namespace mlpack; @@ -29,8 +28,6 @@ using namespace mlpack::ann; using namespace ens; using namespace mlpack::math; -BOOST_AUTO_TEST_SUITE(RecurrentNetworkTest); - /** * Construct a 2-class dataset out of noisy sines. * @@ -75,7 +72,7 @@ void GenerateNoisySines(arma::cube& data, /** * Train the BRNN on a larger dataset. */ -BOOST_AUTO_TEST_CASE(SequenceClassificationBRNNTest) +TEST_CASE("SequenceClassificationBRNNTest", "[RecurrentNetworkTest]") { // Using same test for RNN below. size_t successes = 0; @@ -111,10 +108,10 @@ BOOST_AUTO_TEST_CASE(SequenceClassificationBRNNTest) StandardSGD opt(0.1, 1, 500 * input.n_cols, -100); model.Train(input, labels, opt); - BOOST_TEST_CHECKPOINT("Training over"); + INFO("Training over"); arma::cube prediction; model.Predict(input, prediction); - BOOST_TEST_CHECKPOINT("Prediction over"); + INFO("Prediction over"); size_t error = 0; for (size_t i = 0; i < prediction.n_cols; ++i) @@ -133,7 +130,7 @@ BOOST_AUTO_TEST_CASE(SequenceClassificationBRNNTest) } double classificationError = 1 - double(error) / prediction.n_cols; - BOOST_TEST_CHECKPOINT(classificationError); + INFO(classificationError); if (classificationError <= 0.2) { ++successes; @@ -141,13 +138,13 @@ BOOST_AUTO_TEST_CASE(SequenceClassificationBRNNTest) } } - BOOST_REQUIRE_GE(successes, 1); + REQUIRE(successes >= 1); } /** * Train the vanilla network on a larger dataset. */ -BOOST_AUTO_TEST_CASE(SequenceClassificationTest) +TEST_CASE("SequenceClassificationTest", "[RecurrentNetworkTest]") { // It isn't guaranteed that the recurrent network will converge in the // specified number of iterations using random weights. If this works 1 of 6 @@ -231,7 +228,7 @@ BOOST_AUTO_TEST_CASE(SequenceClassificationTest) } } - BOOST_REQUIRE_GE(successes, 1); + REQUIRE(successes >= 1); } /** @@ -645,13 +642,13 @@ void ReberGrammarTestNetwork(ModelType& model, offset += 3; } - BOOST_REQUIRE_GE(successes, 1); + REQUIRE(successes >= 1); } /** * Train the specified networks on an embedded Reber grammar dataset. */ -BOOST_AUTO_TEST_CASE(LSTMReberGrammarTest) +TEST_CASE("LSTMReberGrammarTest", "[RecurrentNetworkTest]") { RNN > model(5); model.Add >(7, 10); @@ -664,7 +661,7 @@ BOOST_AUTO_TEST_CASE(LSTMReberGrammarTest) /** * Train the specified networks on an embedded Reber grammar dataset. */ -BOOST_AUTO_TEST_CASE(FastLSTMReberGrammarTest) +TEST_CASE("FastLSTMReberGrammarTest", "[RecurrentNetworkTest]") { RNN > model(5); model.Add >(7, 8); @@ -677,7 +674,7 @@ BOOST_AUTO_TEST_CASE(FastLSTMReberGrammarTest) /** * Train the specified networks on an embedded Reber grammar dataset. */ -BOOST_AUTO_TEST_CASE(GRURecursiveReberGrammarTest) +TEST_CASE("GRURecursiveReberGrammarTest", "[RecurrentNetworkTest]") { RNN > model(5); model.Add >(7, 16); @@ -690,7 +687,7 @@ BOOST_AUTO_TEST_CASE(GRURecursiveReberGrammarTest) /** * Train BLSTM on an embedded Reber grammar dataset. */ -BOOST_AUTO_TEST_CASE(BRNNReberGrammarTest) +TEST_CASE("BRNNReberGrammarTest", "[RecurrentNetworkTest]") { BRNN, AddMerge<>, SigmoidLayer<> > model(5); model.Add >(7, 10); @@ -869,14 +866,14 @@ void DistractedSequenceRecallTestNetwork( offset += 2; } - BOOST_REQUIRE_GE(successes, 1); + REQUIRE(successes >= 1); } /** * Train the specified networks on the Derek D. Monner's distracted sequence * recall task. */ -BOOST_AUTO_TEST_CASE(LSTMDistractedSequenceRecallTest) +TEST_CASE("LSTMDistractedSequenceRecallTest", "[RecurrentNetworkTest]") { DistractedSequenceRecallTestNetwork >(4, 8); } @@ -885,7 +882,7 @@ BOOST_AUTO_TEST_CASE(LSTMDistractedSequenceRecallTest) * Train the specified networks on the Derek D. Monner's distracted sequence * recall task. */ -BOOST_AUTO_TEST_CASE(FastLSTMDistractedSequenceRecallTest) +TEST_CASE("FastLSTMDistractedSequenceRecallTest", "[RecurrentNetworkTest]") { DistractedSequenceRecallTestNetwork >(4, 8); } @@ -894,7 +891,7 @@ BOOST_AUTO_TEST_CASE(FastLSTMDistractedSequenceRecallTest) * Train the specified networks on the Derek D. Monner's distracted sequence * recall task. */ -BOOST_AUTO_TEST_CASE(GRUDistractedSequenceRecallTest) +TEST_CASE("GRUDistractedSequenceRecallTest", "[RecurrentNetworkTest]") { DistractedSequenceRecallTestNetwork >(4, 8); } @@ -956,7 +953,7 @@ void BatchSizeTest() /** * Ensure LSTMs work with larger batch sizes. */ -BOOST_AUTO_TEST_CASE(LSTMBatchSizeTest) +TEST_CASE("LSTMBatchSizeTest", "[RecurrentNetworkTest]") { BatchSizeTest>(); } @@ -964,7 +961,7 @@ BOOST_AUTO_TEST_CASE(LSTMBatchSizeTest) /** * Ensure fast LSTMs work with larger batch sizes. */ -BOOST_AUTO_TEST_CASE(FastLSTMBatchSizeTest) +TEST_CASE("FastLSTMBatchSizeTest", "[RecurrentNetworkTest]") { BatchSizeTest>(); } @@ -972,7 +969,7 @@ BOOST_AUTO_TEST_CASE(FastLSTMBatchSizeTest) /** * Ensure GRUs work with larger batch sizes. */ -BOOST_AUTO_TEST_CASE(GRUBatchSizeTest) +TEST_CASE("GRUBatchSizeTest", "[RecurrentNetworkTest]") { BatchSizeTest>(); } @@ -980,7 +977,7 @@ BOOST_AUTO_TEST_CASE(GRUBatchSizeTest) /** * Make sure the RNN can be properly serialized. */ -BOOST_AUTO_TEST_CASE(SerializationTest) +TEST_CASE("RNNSerializationTest", "[RecurrentNetworkTest]") { const size_t rho = 10; @@ -1172,13 +1169,13 @@ void ReberGrammarTestCustomNetwork(const size_t hiddenSize = 4, offset += 3; } - BOOST_REQUIRE_GE(successes, 1); + REQUIRE(successes >= 1); } /** * Train the specified networks on an embedded Reber grammar dataset. */ -BOOST_AUTO_TEST_CASE(CustomRecursiveReberGrammarTest) +TEST_CASE("CustomRecursiveReberGrammarTest", "[RecurrentNetworkTest]") { ReberGrammarTestCustomNetwork(16, true); } @@ -1312,16 +1309,16 @@ double RNNSineTest(size_t hiddenUnits, size_t rho, size_t numEpochs = 100) /** * Test RNN using multiple timestep input and single output. */ -BOOST_AUTO_TEST_CASE(MultiTimestepTest) +TEST_CASE("MultiTimestepTest", "[RecurrentNetworkTest]") { double err = RNNSineTest(4, 10, 20); - BOOST_REQUIRE_LE(err, 0.025); + REQUIRE(err <= 0.025); } /** * Test that RNN::Train() returns finite objective value. */ -BOOST_AUTO_TEST_CASE(RNNTrainReturnObjective) +TEST_CASE("RNNTrainReturnObjective", "[RecurrentNetworkTest]") { const size_t rho = 10; @@ -1371,13 +1368,13 @@ BOOST_AUTO_TEST_CASE(RNNTrainReturnObjective) StandardSGD opt(0.1, 1, input.n_cols /* 1 epoch */, -100); double objVal = model.Train(input, labels, opt); - BOOST_REQUIRE_EQUAL(std::isfinite(objVal), true); + REQUIRE(std::isfinite(objVal) == true); } /** * Test that BRNN::Train() returns finite objective value. */ -BOOST_AUTO_TEST_CASE(BRNNTrainReturnObjective) +TEST_CASE("BRNNTrainReturnObjective", "[RecurrentNetworkTest]") { const size_t rho = 10; @@ -1407,16 +1404,16 @@ BOOST_AUTO_TEST_CASE(BRNNTrainReturnObjective) StandardSGD opt(0.1, 1, 500 * input.n_cols, -100); double objVal = model.Train(input, labels, opt); - BOOST_TEST_CHECKPOINT("Training over"); + INFO("Training over"); // Test that BRNN::Train() returns finite objective value. - BOOST_REQUIRE_EQUAL(std::isfinite(objVal), true); + REQUIRE(std::isfinite(objVal) == true); } /** * Test that RNN::Train() does not give an error for large rho. */ -BOOST_AUTO_TEST_CASE(LargeRhoValueRnnTest) +TEST_CASE("LargeRhoValueRnnTest", "[RecurrentNetworkTest]") { // Setting rho value greater than sequence length which is 17. const size_t rho = 100; @@ -1473,7 +1470,5 @@ BOOST_AUTO_TEST_CASE(LargeRhoValueRnnTest) } ens::SGD<> opt(0.01, 1, 100); model.Train(inputs[0], targets[0], opt); - BOOST_TEST_CHECKPOINT("Training over"); + INFO("Training over"); } - -BOOST_AUTO_TEST_SUITE_END(); From 584cd7ad1f491c7ce5a70fadd23367d4bc8d7875 Mon Sep 17 00:00:00 2001 From: jeffin143 Date: Sun, 30 Aug 2020 01:17:33 +0530 Subject: [PATCH 13/55] migrate Kmeans test --- src/mlpack/tests/CMakeLists.txt | 4 +- src/mlpack/tests/kmeans_test.cpp | 165 ++++++++++---------- src/mlpack/tests/main_tests/kmeans_test.cpp | 90 ++++++----- 3 files changed, 130 insertions(+), 129 deletions(-) diff --git a/src/mlpack/tests/CMakeLists.txt b/src/mlpack/tests/CMakeLists.txt index 7b88490c04..14f716245e 100644 --- a/src/mlpack/tests/CMakeLists.txt +++ b/src/mlpack/tests/CMakeLists.txt @@ -25,7 +25,6 @@ add_executable(mlpack_test hyperplane_test.cpp init_rules_test.cpp kde_test.cpp - kmeans_test.cpp krann_search_test.cpp ksinit_test.cpp lars_test.cpp @@ -97,7 +96,6 @@ add_executable(mlpack_test main_tests/hmm_viterbi_test.cpp main_tests/hoeffding_tree_test.cpp main_tests/kde_test.cpp - main_tests/kmeans_test.cpp main_tests/krann_test.cpp main_tests/linear_svm_test.cpp main_tests/lmnn_test.cpp @@ -141,6 +139,7 @@ add_executable(mlpack_catch_test kernel_test.cpp kernel_traits_test.cpp kfn_test.cpp + kmeans_test.cpp knn_test.cpp linear_regression_test.cpp load_save_test.cpp @@ -169,6 +168,7 @@ add_executable(mlpack_catch_test main_tests/image_converter_test.cpp main_tests/kernel_pca_test.cpp main_tests/kfn_test.cpp + main_tests/kmeans_test.cpp main_tests/knn_test.cpp main_tests/linear_regression_test.cpp main_tests/nca_test.cpp diff --git a/src/mlpack/tests/kmeans_test.cpp b/src/mlpack/tests/kmeans_test.cpp index df879a048f..5d4bae2abf 100644 --- a/src/mlpack/tests/kmeans_test.cpp +++ b/src/mlpack/tests/kmeans_test.cpp @@ -22,9 +22,8 @@ #include #include -#include +#include "catch.hpp" #include -#include "test_tools.hpp" using namespace mlpack; using namespace mlpack::kmeans; @@ -32,8 +31,6 @@ using namespace mlpack::metric; using namespace mlpack::tree; using namespace mlpack::neighbor; -BOOST_AUTO_TEST_SUITE(KMeansTest); - // Generate dataset; written transposed because it's easier to read. arma::mat kMeansData(" 0.0 0.0;" // Class 1. " 0.3 0.4;" @@ -69,7 +66,7 @@ arma::mat kMeansData(" 0.0 0.0;" // Class 1. /** * 30-point 3-class test case for K-Means. */ -BOOST_AUTO_TEST_CASE(KMeansSimpleTest) +TEST_CASE("KMeansSimpleTest", "[KMeansTest]") { // This test was originally written to use RandomPartition, and is left that // way because RandomPartition gives better initializations here. @@ -83,30 +80,30 @@ BOOST_AUTO_TEST_CASE(KMeansSimpleTest) size_t firstClass = assignments(0); for (size_t i = 1; i < 13; ++i) - BOOST_REQUIRE_EQUAL(assignments(i), firstClass); + REQUIRE(assignments(i) == firstClass); size_t secondClass = assignments(13); // To ensure that class 1 != class 2. - BOOST_REQUIRE_NE(firstClass, secondClass); + REQUIRE(firstClass != secondClass); for (size_t i = 13; i < 20; ++i) - BOOST_REQUIRE_EQUAL(assignments(i), secondClass); + REQUIRE(assignments(i) == secondClass); size_t thirdClass = assignments(20); // To ensure that this is the third class which we haven't seen yet. - BOOST_REQUIRE_NE(firstClass, thirdClass); - BOOST_REQUIRE_NE(secondClass, thirdClass); + REQUIRE(firstClass != thirdClass); + REQUIRE(secondClass != thirdClass); for (size_t i = 20; i < 30; ++i) - BOOST_REQUIRE_EQUAL(assignments(i), thirdClass); + REQUIRE(assignments(i) == thirdClass); } /** * Make sure the empty cluster policy class does nothing. */ -BOOST_AUTO_TEST_CASE(AllowEmptyClusterTest) +TEST_CASE("AllowEmptyClusterTest", "[KMeansTest]") { arma::Row assignments; assignments.randu(30); @@ -129,17 +126,17 @@ BOOST_AUTO_TEST_CASE(AllowEmptyClusterTest) // Make sure no assignments were changed. for (size_t i = 0; i < assignments.n_elem; ++i) - BOOST_REQUIRE_EQUAL(assignments[i], assignmentsOld[i]); + REQUIRE(assignments[i] == assignmentsOld[i]); // Make sure no counts were changed. for (size_t i = 0; i < 3; ++i) - BOOST_REQUIRE_EQUAL(counts[i], countsOld[i]); + REQUIRE(counts[i] == countsOld[i]); } /** * Make sure kill empty cluster policy removes the empty cluster. */ -BOOST_AUTO_TEST_CASE(KillEmptyClusterTest) +TEST_CASE("KillEmptyClusterTest", "[KMeansTest]") { arma::Row assignments; assignments.randu(30); @@ -162,20 +159,20 @@ BOOST_AUTO_TEST_CASE(KillEmptyClusterTest) // Make sure no assignments were changed. for (size_t i = 0; i < assignments.n_elem; ++i) - BOOST_REQUIRE_EQUAL(assignments[i], assignmentsOld[i]); + REQUIRE(assignments[i] == assignmentsOld[i]); // Make sure no counts were changed for clusters that are not empty. for (size_t i = 0; i < 2; ++i) - BOOST_REQUIRE_EQUAL(counts[i], countsOld[i]); + REQUIRE(counts[i] == countsOld[i]); // Make sure that counts contain one less element than old counts. - BOOST_REQUIRE_GT(countsOld.n_elem, counts.n_elem); + REQUIRE(countsOld.n_elem > counts.n_elem); } /** * Make sure the max variance method finds the correct point. */ -BOOST_AUTO_TEST_CASE(MaxVarianceNewClusterTest) +TEST_CASE("MaxVarianceNewClusterTest", "[KMeansTest]") { // Five points. arma::mat data("0.4 1.0 5.0 -2.0 -2.5;" @@ -220,22 +217,22 @@ BOOST_AUTO_TEST_CASE(MaxVarianceNewClusterTest) assignments[i] = closestCluster; } - BOOST_REQUIRE_EQUAL(assignments[0], 0); - BOOST_REQUIRE_EQUAL(assignments[1], 0); - BOOST_REQUIRE_EQUAL(assignments[2], 2); - BOOST_REQUIRE_EQUAL(assignments[3], 1); - BOOST_REQUIRE_EQUAL(assignments[4], 1); + REQUIRE(assignments[0] == 0); + REQUIRE(assignments[1] == 0); + REQUIRE(assignments[2] == 2); + REQUIRE(assignments[3] == 1); + REQUIRE(assignments[4] == 1); // Ensure that the counts are right. - BOOST_REQUIRE_EQUAL(counts[0], 2); - BOOST_REQUIRE_EQUAL(counts[1], 2); - BOOST_REQUIRE_EQUAL(counts[2], 1); + REQUIRE(counts[0] == 2); + REQUIRE(counts[1] == 2); + REQUIRE(counts[2] == 1); } /** * Make sure the random partitioner seems to return valid results. */ -BOOST_AUTO_TEST_CASE(RandomPartitionTest) +TEST_CASE("RandomPartitionTest", "[KMeansTest]") { arma::mat data; data.randu(2, 1000); // One thousand points. @@ -246,17 +243,17 @@ BOOST_AUTO_TEST_CASE(RandomPartitionTest) RandomPartition::Cluster(data, 18, assignments); // Ensure that the right number of assignments were given. - BOOST_REQUIRE_EQUAL(assignments.n_elem, 1000); + REQUIRE(assignments.n_elem == 1000); // Ensure that no value is greater than 17 (the maximum valid cluster). for (size_t i = 0; i < 1000; ++i) - BOOST_REQUIRE_LT(assignments[i], 18); + REQUIRE(assignments[i] < 18); } /** * Make sure that random initialization fails for a corner case dataset. */ -BOOST_AUTO_TEST_CASE(RandomInitialAssignmentFailureTest) +TEST_CASE("RandomInitialAssignmentFailureTest", "[KMeansTest]") { // This is a very synthetic dataset. It is one Gaussian with a huge number of // points combined with one faraway Gaussian with very few points. Normally, @@ -292,14 +289,14 @@ BOOST_AUTO_TEST_CASE(RandomInitialAssignmentFailureTest) // Only one success allowed. The probability of two successes should be // infinitesimal. - BOOST_REQUIRE_LT(successes, 2); + REQUIRE(successes < 2); } /** * Make sure that specifying initial assignments is successful for a corner case * dataset which doesn't usually converge otherwise. */ -BOOST_AUTO_TEST_CASE(InitialAssignmentTest) +TEST_CASE("InitialAssignmentTest", "[KMeansTest]") { // For a better description of this dataset, see // RandomInitialAssignmentFailureTest. @@ -321,9 +318,9 @@ BOOST_AUTO_TEST_CASE(InitialAssignmentTest) // Check results. for (size_t i = 0; i < 10000; ++i) - BOOST_REQUIRE_EQUAL(assignments[i], 0); + REQUIRE(assignments[i] == 0); for (size_t i = 10000; i < 10002; ++i) - BOOST_REQUIRE_EQUAL(assignments[i], 1); + REQUIRE(assignments[i] == 1); // Now, slightly harder. Give it one incorrect assignment in each cluster. // The wrong assignment should be quickly fixed. @@ -334,16 +331,16 @@ BOOST_AUTO_TEST_CASE(InitialAssignmentTest) // Check results. for (size_t i = 0; i < 10000; ++i) - BOOST_REQUIRE_EQUAL(assignments[i], 0); + REQUIRE(assignments[i] == 0); for (size_t i = 10000; i < 10002; ++i) - BOOST_REQUIRE_EQUAL(assignments[i], 1); + REQUIRE(assignments[i] == 1); } /** * Make sure specifying initial centroids is successful for a corner case which * doesn't usually converge otherwise. */ -BOOST_AUTO_TEST_CASE(InitialCentroidTest) +TEST_CASE("InitialCentroidTest", "[KMeansTest]") { // For a better description of this dataset, see // RandomInitialAssignmentFailureTest. @@ -365,9 +362,9 @@ BOOST_AUTO_TEST_CASE(InitialCentroidTest) // Check results. for (size_t i = 0; i < 10000; ++i) - BOOST_REQUIRE_EQUAL(assignments[i], 0); + REQUIRE(assignments[i] == 0); for (size_t i = 10000; i < 10002; ++i) - BOOST_REQUIRE_EQUAL(assignments[i], 1); + REQUIRE(assignments[i] == 1); // Now add a little noise to the initial centroids. centroids.col(0) = arma::vec("3 4"); @@ -377,15 +374,15 @@ BOOST_AUTO_TEST_CASE(InitialCentroidTest) // Check results. for (size_t i = 0; i < 10000; ++i) - BOOST_REQUIRE_EQUAL(assignments[i], 0); + REQUIRE(assignments[i] == 0); for (size_t i = 10000; i < 10002; ++i) - BOOST_REQUIRE_EQUAL(assignments[i], 1); + REQUIRE(assignments[i] == 1); } /** * Ensure that initial assignments override initial centroids. */ -BOOST_AUTO_TEST_CASE(InitialAssignmentOverrideTest) +TEST_CASE("InitialAssignmentOverrideTest", "[KMeansTest]") { // For a better description of this dataset, see // RandomInitialAssignmentFailureTest. @@ -412,22 +409,22 @@ BOOST_AUTO_TEST_CASE(InitialAssignmentOverrideTest) // Because the initial assignments guess should take priority, we should get // those same results back. for (size_t i = 0; i < 10000; ++i) - BOOST_REQUIRE_EQUAL(assignments[i], 0); + REQUIRE(assignments[i] == 0); for (size_t i = 10000; i < 10002; ++i) - BOOST_REQUIRE_EQUAL(assignments[i], 1); + REQUIRE(assignments[i] == 1); // Make sure the centroids are about right too. - BOOST_REQUIRE_LT(centroids(0, 0), 10.0); - BOOST_REQUIRE_LT(centroids(1, 0), 10.0); - BOOST_REQUIRE_GT(centroids(0, 1), 40.0); - BOOST_REQUIRE_GT(centroids(1, 1), 40.0); + REQUIRE(centroids(0, 0) < 10.0); + REQUIRE(centroids(1, 0) < 10.0); + REQUIRE(centroids(0, 1) > 40.0); + REQUIRE(centroids(1, 1) > 40.0); } /** * Test that the refined starting policy returns decent initial cluster * estimates. */ -BOOST_AUTO_TEST_CASE(RefinedStartTest) +TEST_CASE("RefinedStartTest", "[KMeansTest]") { // Our dataset will be five Gaussians of largely varying numbers of points and // we expect that the refined starting policy should return good guesses at @@ -486,14 +483,14 @@ BOOST_AUTO_TEST_CASE(RefinedStartTest) // figure is a corner case which actually does not give good clusters), and // random initial starts give distortion around 22000. So we'll require that // our distortion is less than 14000. - BOOST_REQUIRE_LT(distortion, 14000.0); + REQUIRE(distortion < 14000.0); } #ifdef ARMA_HAS_SPMAT /** * Make sure sparse k-means works okay. */ -BOOST_AUTO_TEST_CASE(SparseKMeansTest) +TEST_CASE("SparseKMeansTest", "[KMeansTest]") { // Huge dimensionality, few points. arma::SpMat data(5000, 12); @@ -520,23 +517,23 @@ BOOST_AUTO_TEST_CASE(SparseKMeansTest) size_t clusterOne = assignments[0]; size_t clusterTwo = assignments[6]; - BOOST_REQUIRE_EQUAL(assignments[0], clusterOne); - BOOST_REQUIRE_EQUAL(assignments[1], clusterOne); - BOOST_REQUIRE_EQUAL(assignments[2], clusterOne); - BOOST_REQUIRE_EQUAL(assignments[3], clusterOne); - BOOST_REQUIRE_EQUAL(assignments[4], clusterOne); - BOOST_REQUIRE_EQUAL(assignments[5], clusterOne); - BOOST_REQUIRE_EQUAL(assignments[6], clusterTwo); - BOOST_REQUIRE_EQUAL(assignments[7], clusterTwo); - BOOST_REQUIRE_EQUAL(assignments[8], clusterTwo); - BOOST_REQUIRE_EQUAL(assignments[9], clusterTwo); - BOOST_REQUIRE_EQUAL(assignments[10], clusterTwo); - BOOST_REQUIRE_EQUAL(assignments[11], clusterTwo); + REQUIRE(assignments[0] == clusterOne); + REQUIRE(assignments[1] == clusterOne); + REQUIRE(assignments[2] == clusterOne); + REQUIRE(assignments[3] == clusterOne); + REQUIRE(assignments[4] == clusterOne); + REQUIRE(assignments[5] == clusterOne); + REQUIRE(assignments[6] == clusterTwo); + REQUIRE(assignments[7] == clusterTwo); + REQUIRE(assignments[8] == clusterTwo); + REQUIRE(assignments[9] == clusterTwo); + REQUIRE(assignments[10] == clusterTwo); + REQUIRE(assignments[11] == clusterTwo); } #endif // ARMA_HAS_SPMAT -BOOST_AUTO_TEST_CASE(ElkanTest) +TEST_CASE("ElkanTest", "[KMeansTest]") { const size_t trials = 5; @@ -563,14 +560,14 @@ BOOST_AUTO_TEST_CASE(ElkanTest) elkan.Cluster(dataset, k, elkanAssignments, elkanCentroids, false, true); for (size_t i = 0; i < dataset.n_cols; ++i) - BOOST_REQUIRE_EQUAL(assignments[i], elkanAssignments[i]); + REQUIRE(assignments[i] == elkanAssignments[i]); for (size_t i = 0; i < centroids.n_elem; ++i) - BOOST_REQUIRE_CLOSE(naiveCentroids[i], elkanCentroids[i], 1e-5); + REQUIRE(naiveCentroids[i] == Approx(elkanCentroids[i]).epsilon(1e-7)); } } -BOOST_AUTO_TEST_CASE(HamerlyTest) +TEST_CASE("HamerlyTest", "[KMeansTest]") { const size_t trials = 5; @@ -598,14 +595,14 @@ BOOST_AUTO_TEST_CASE(HamerlyTest) true); for (size_t i = 0; i < dataset.n_cols; ++i) - BOOST_REQUIRE_EQUAL(assignments[i], hamerlyAssignments[i]); + REQUIRE(assignments[i] == hamerlyAssignments[i]); for (size_t i = 0; i < centroids.n_elem; ++i) - BOOST_REQUIRE_CLOSE(naiveCentroids[i], hamerlyCentroids[i], 1e-5); + REQUIRE(naiveCentroids[i] == Approx(hamerlyCentroids[i]).epsilon(1e-7)); } } -BOOST_AUTO_TEST_CASE(PellegMooreTest) +TEST_CASE("PellegMooreTest", "[KMeansTest]") { const size_t trials = 5; @@ -632,14 +629,14 @@ BOOST_AUTO_TEST_CASE(PellegMooreTest) pellegMoore.Cluster(dataset, k, pmAssignments, pmCentroids, false, true); for (size_t i = 0; i < dataset.n_cols; ++i) - BOOST_REQUIRE_EQUAL(assignments[i], pmAssignments[i]); + REQUIRE(assignments[i] == pmAssignments[i]); for (size_t i = 0; i < centroids.n_elem; ++i) - BOOST_REQUIRE_CLOSE(naiveCentroids[i], pmCentroids[i], 1e-5); + REQUIRE(naiveCentroids[i] == Approx(pmCentroids[i]).epsilon(1e-7)); } } -BOOST_AUTO_TEST_CASE(DTNNTest) +TEST_CASE("DTNNTest", "[KMeansTest]") { const size_t trials = 5; @@ -664,14 +661,14 @@ BOOST_AUTO_TEST_CASE(DTNNTest) dtnn.Cluster(dataset, k, dtnnAssignments, dtnnCentroids, false, true); for (size_t i = 0; i < dataset.n_cols; ++i) - BOOST_REQUIRE_EQUAL(assignments[i], dtnnAssignments[i]); + REQUIRE(assignments[i] == dtnnAssignments[i]); for (size_t i = 0; i < centroids.n_elem; ++i) - BOOST_REQUIRE_CLOSE(naiveCentroids[i], dtnnCentroids[i], 1e-5); + REQUIRE(naiveCentroids[i] == Approx(dtnnCentroids[i]).epsilon(1e-7)); } } -BOOST_AUTO_TEST_CASE(DTNNCoverTreeTest) +TEST_CASE("DTNNCoverTreeTest", "[KMeansTest]") { const size_t trials = 5; @@ -696,10 +693,10 @@ BOOST_AUTO_TEST_CASE(DTNNCoverTreeTest) dtnn.Cluster(dataset, k, dtnnAssignments, dtnnCentroids, false, true); for (size_t i = 0; i < dataset.n_cols; ++i) - BOOST_REQUIRE_EQUAL(assignments[i], dtnnAssignments[i]); + REQUIRE(assignments[i] == dtnnAssignments[i]); for (size_t i = 0; i < centroids.n_elem; ++i) - BOOST_REQUIRE_CLOSE(naiveCentroids[i], dtnnCentroids[i], 1e-5); + REQUIRE(naiveCentroids[i] == Approx(dtnnCentroids[i]).epsilon(1e-7)); } } @@ -707,7 +704,7 @@ BOOST_AUTO_TEST_CASE(DTNNCoverTreeTest) * Make sure that the sample initialization strategy successfully samples points * from the dataset. */ -BOOST_AUTO_TEST_CASE(SampleInitializationTest) +TEST_CASE("SampleInitializationTest", "[KMeansTest]") { arma::mat dataset = arma::randu(5, 100); const size_t clusters = 10; @@ -716,8 +713,8 @@ BOOST_AUTO_TEST_CASE(SampleInitializationTest) SampleInitialization::Cluster(dataset, clusters, centroids); // Check that the size of the matrix is correct. - BOOST_REQUIRE_EQUAL(centroids.n_cols, 10); - BOOST_REQUIRE_EQUAL(centroids.n_rows, 5); + REQUIRE(centroids.n_cols == 10); + REQUIRE(centroids.n_rows == 5); // Check that each entry in the matrix is some sample from the dataset. for (size_t i = 0; i < clusters; ++i) @@ -733,8 +730,6 @@ BOOST_AUTO_TEST_CASE(SampleInitializationTest) break; } - BOOST_REQUIRE_LT(j, dataset.n_cols); + REQUIRE(j < dataset.n_cols); } } - -BOOST_AUTO_TEST_SUITE_END(); diff --git a/src/mlpack/tests/main_tests/kmeans_test.cpp b/src/mlpack/tests/main_tests/kmeans_test.cpp index 9de3bc1e09..6db26d1fcb 100644 --- a/src/mlpack/tests/main_tests/kmeans_test.cpp +++ b/src/mlpack/tests/main_tests/kmeans_test.cpp @@ -19,8 +19,8 @@ static const std::string testName = "Kmeans"; #include "test_helper.hpp" #include -#include -#include "../test_tools.hpp" +#include "../catch.hpp" +#include "../test_catch_tools.hpp" using namespace mlpack; @@ -46,22 +46,21 @@ void ResetKmSettings() IO::RestoreSettings(testName); } -BOOST_FIXTURE_TEST_SUITE(KmeansMainTest, KmTestFixture); - /** * Checking that number of Clusters are non negative */ -BOOST_AUTO_TEST_CASE(NonNegativeClustersTest) +TEST_CASE_METHOD(KmTestFixture, "NonNegativeClustersTest", + "[KmeansMainTest][BindingTests]") { arma::mat inputData; if (!data::Load("vc2.csv", inputData)) - BOOST_FAIL("Unable to load train dataset vc2.csv!"); + FAIL("Unable to load train dataset vc2.csv!"); SetInputParam("input", std::move(inputData)); SetInputParam("clusters", (int) -1); // Invalid Log::Fatal.ignoreInput = true; - BOOST_REQUIRE_THROW(mlpackMain(), std::runtime_error); + REQUIRE_THROWS_AS(mlpackMain(), std::runtime_error); Log::Fatal.ignoreInput = false; } @@ -69,7 +68,8 @@ BOOST_AUTO_TEST_CASE(NonNegativeClustersTest) /** * Checking that initial centroids are provided if clusters are to be auto detected */ -BOOST_AUTO_TEST_CASE(AutoDetectClusterTest) +TEST_CASE_METHOD(KmTestFixture, "AutoDetectClusterTest", + "[KmeansMainTest][BindingTests]") { constexpr int N = 10; constexpr int D = 4; @@ -80,7 +80,7 @@ BOOST_AUTO_TEST_CASE(AutoDetectClusterTest) SetInputParam("clusters", (int) 0); // Invalid Log::Fatal.ignoreInput = true; - BOOST_REQUIRE_THROW(mlpackMain(), std::runtime_error); + REQUIRE_THROWS_AS(mlpackMain(), std::runtime_error); Log::Fatal.ignoreInput = false; } @@ -88,13 +88,14 @@ BOOST_AUTO_TEST_CASE(AutoDetectClusterTest) /** * Checking that percentage is between 0 and 1 when --refined_start is specified */ -BOOST_AUTO_TEST_CASE(RefinedStartPercentageTest) +TEST_CASE_METHOD(KmTestFixture, "RefinedStartPercentageTest", + "[KmeansMainTest][BindingTests]") { int c = 2; double P = 2.0; arma::mat inputData; if (!data::Load("vc2.csv", inputData)) - BOOST_FAIL("Unable to load train dataset vc2.csv!"); + FAIL("Unable to load train dataset vc2.csv!"); SetInputParam("input", std::move(inputData)); SetInputParam("refined_start", true); @@ -102,7 +103,7 @@ BOOST_AUTO_TEST_CASE(RefinedStartPercentageTest) SetInputParam("percentage", std::move(P)); // Invalid Log::Fatal.ignoreInput = true; - BOOST_REQUIRE_THROW(mlpackMain(), std::runtime_error); + REQUIRE_THROWS_AS(mlpackMain(), std::runtime_error); Log::Fatal.ignoreInput = false; } @@ -110,13 +111,14 @@ BOOST_AUTO_TEST_CASE(RefinedStartPercentageTest) /** * Checking percentage is non-negative when --refined_start is specified */ -BOOST_AUTO_TEST_CASE(NonNegativePercentageTest) +TEST_CASE_METHOD(KmTestFixture, "NonNegativePercentageTest", + "[KmeansMainTest][BindingTests]") { int c = 2; double P = -1.0; arma::mat inputData; if (!data::Load("vc2.csv", inputData)) - BOOST_FAIL("Unable to load train dataset vc2.csv!"); + FAIL("Unable to load train dataset vc2.csv!"); SetInputParam("input", std::move(inputData)); SetInputParam("refined_start", true); @@ -124,7 +126,7 @@ BOOST_AUTO_TEST_CASE(NonNegativePercentageTest) SetInputParam("percentage", P); // Invalid Log::Fatal.ignoreInput = true; - BOOST_REQUIRE_THROW(mlpackMain(), std::runtime_error); + REQUIRE_THROWS_AS(mlpackMain(), std::runtime_error); Log::Fatal.ignoreInput = false; } @@ -132,12 +134,13 @@ BOOST_AUTO_TEST_CASE(NonNegativePercentageTest) /** * Checking that size and dimensionality of prediction is correct. */ -BOOST_AUTO_TEST_CASE(KmClusteringSizeCheck) +TEST_CASE_METHOD(KmTestFixture, "KmClusteringSizeCheck", + "[KmeansMainTest][BindingTests]") { int c = 2; arma::mat inputData; if (!data::Load("vc2.csv", inputData)) - BOOST_FAIL("Unable to load train dataset vc2.csv!"); + FAIL("Unable to load train dataset vc2.csv!"); size_t col = inputData.n_cols; size_t row = inputData.n_rows; @@ -147,22 +150,23 @@ BOOST_AUTO_TEST_CASE(KmClusteringSizeCheck) mlpackMain(); - BOOST_REQUIRE_EQUAL(IO::GetParam("output").n_rows, row+1); - BOOST_REQUIRE_EQUAL(IO::GetParam("output").n_cols, col); - BOOST_REQUIRE_EQUAL(IO::GetParam("centroid").n_rows, row); - BOOST_REQUIRE_EQUAL(IO::GetParam("centroid").n_cols, c); + REQUIRE(IO::GetParam("output").n_rows == row+1); + REQUIRE(IO::GetParam("output").n_cols == col); + REQUIRE(IO::GetParam("centroid").n_rows == row); + REQUIRE(IO::GetParam("centroid").n_cols == c); } /** * Checking that size and dimensionality of prediction is correct when --labels_only is specified */ -BOOST_AUTO_TEST_CASE(KmClusteringSizeCheckLabelOnly) +TEST_CASE_METHOD(KmTestFixture, "KmClusteringSizeCheckLabelOnly", + "[KmeansMainTest][BindingTests]") { int c = 2; arma::mat inputData; if (!data::Load("vc2.csv", inputData)) - BOOST_FAIL("Unable to load train dataset vc2.csv!"); + FAIL("Unable to load train dataset vc2.csv!"); size_t col = inputData.n_cols; size_t row = inputData.n_rows; @@ -172,24 +176,25 @@ BOOST_AUTO_TEST_CASE(KmClusteringSizeCheckLabelOnly) mlpackMain(); - BOOST_REQUIRE_EQUAL(IO::GetParam("output").n_rows, 1); - BOOST_REQUIRE_EQUAL(IO::GetParam("output").n_cols, col); - BOOST_REQUIRE_EQUAL(IO::GetParam("centroid").n_rows, row); - BOOST_REQUIRE_EQUAL(IO::GetParam("centroid").n_cols, c); + REQUIRE(IO::GetParam("output").n_rows == 1); + REQUIRE(IO::GetParam("output").n_cols == col); + REQUIRE(IO::GetParam("centroid").n_rows == row); + REQUIRE(IO::GetParam("centroid").n_cols == c); } /** * Checking that predictions are not same when --allow_empty_clusters or kill_empty_clusters are specified */ -BOOST_AUTO_TEST_CASE(KmClusteringEmptyClustersCheck) +TEST_CASE_METHOD(KmTestFixture, "KmClusteringEmptyClustersCheck", + "[KmeansMainTest][BindingTests]") { int c = 400; int iterations = 100; arma::mat inputData; if (!data::Load("test_data_3_1000.csv", inputData)) - BOOST_FAIL("Unable to load train dataset test_data_3_1000.csv!"); + FAIL("Unable to load train dataset test_data_3_1000.csv!"); arma::mat initCentroid = arma::randu(inputData.n_rows, c); SetInputParam("input", inputData); @@ -235,23 +240,24 @@ BOOST_AUTO_TEST_CASE(KmClusteringEmptyClustersCheck) if (killEmptyOutput.n_elem == allowEmptyOutput.n_elem) { - BOOST_REQUIRE_GT(arma::accu(killEmptyOutput != allowEmptyOutput), 1); - BOOST_REQUIRE_GT(arma::accu(killEmptyOutput != normalOutput), 1); + REQUIRE(arma::accu(killEmptyOutput != allowEmptyOutput) > 1); + REQUIRE(arma::accu(killEmptyOutput != normalOutput) > 1); } - BOOST_REQUIRE_GT(arma::accu(normalOutput != allowEmptyOutput), 1); + REQUIRE(arma::accu(normalOutput != allowEmptyOutput) > 1); } /** * Checking that that size and dimensionality of Final Input File is correct * when flag --in_place is specified */ -BOOST_AUTO_TEST_CASE(KmClusteringResultSizeCheck) +TEST_CASE_METHOD(KmTestFixture, "KmClusteringResultSizeCheck", + "[KmeansMainTest][BindingTests]") { int c = 2; arma::mat inputData; if (!data::Load("vc2.csv", inputData)) - BOOST_FAIL("Unable to load train dataset vc2.csv!"); + FAIL("Unable to load train dataset vc2.csv!"); size_t row = inputData.n_rows; size_t col = inputData.n_cols; @@ -265,30 +271,32 @@ BOOST_AUTO_TEST_CASE(KmClusteringResultSizeCheck) // here input is actually accessed through output // due to a little trick in kmeans_main - BOOST_REQUIRE_EQUAL(processedInput.n_cols, col); - BOOST_REQUIRE_EQUAL(processedInput.n_rows, row+1); + REQUIRE(processedInput.n_cols == col); + REQUIRE(processedInput.n_rows == row+1); } /** * Ensuring that absence of Number of Clusters is checked. */ -BOOST_AUTO_TEST_CASE(KmClustersNotDefined) +TEST_CASE_METHOD(KmTestFixture, "KmClustersNotDefined", + "[KmeansMainTest][BindingTests]") { arma::mat inputData; if (!data::Load("vc2.csv", inputData)) - BOOST_FAIL("Unable to load train dataset vc2.csv!"); + FAIL("Unable to load train dataset vc2.csv!"); SetInputParam("input", std::move(inputData)); Log::Fatal.ignoreInput = true; - BOOST_REQUIRE_THROW(mlpackMain(), std::runtime_error); + REQUIRE_THROWS_AS(mlpackMain(), std::runtime_error); Log::Fatal.ignoreInput = false; } /** * Checking that all the algorithms yield same results */ -BOOST_AUTO_TEST_CASE(AlgorithmsSimilarTest) +TEST_CASE_METHOD(KmTestFixture, "AlgorithmsSimilarTest", + "[KmeansMainTest][BindingTests]") { int c = 5; arma::mat inputData(10, 1000); @@ -393,5 +401,3 @@ BOOST_AUTO_TEST_CASE(AlgorithmsSimilarTest) CheckMatrices(naiveCentroid, dualTreeCentroid); CheckMatrices(naiveCentroid, dualCoverTreeCentroid); } - -BOOST_AUTO_TEST_SUITE_END(); From 71fce9a6e3e13361222555dd299fa2f3508cddbc Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Sat, 29 Aug 2020 20:24:41 -0400 Subject: [PATCH 14/55] Add test for NaN values in metrics. --- src/mlpack/tests/cv_test.cpp | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/src/mlpack/tests/cv_test.cpp b/src/mlpack/tests/cv_test.cpp index 635afb761f..48bb9a6481 100644 --- a/src/mlpack/tests/cv_test.cpp +++ b/src/mlpack/tests/cv_test.cpp @@ -380,6 +380,26 @@ TEST_CASE("SimpleCVMSETest", "[CVTest]") REQUIRE(std::abs(weightedCV2.Evaluate() - expectedMSE) > 1e-5); } +/** + * Test that scores of -nan are filtered out. + */ +TEST_CASE("FilterNANCVTest", "[CVTest]") +{ + // Create a dataset with only one positive label, so it will not be in every + // fold. + arma::mat data(3, 10, arma::fill::randu); + arma::Row labels(10, arma::fill::zeros); + labels[0] = 1; + + const size_t numClasses = 2; + KFoldCV, F1> kfoldcv(2, data, labels, + numClasses); + + const double result = kfoldcv.Evaluate(); + REQUIRE(!std::isnan(result)); + REQUIRE(!std::isinf(result)); +} + template arma::Row PredictLabelsWithDT(const arma::mat& data, const DTArgs&... args) From 96f737b95c5b30490d480c15e18d83ad6d9000b7 Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Sat, 29 Aug 2020 20:24:57 -0400 Subject: [PATCH 15/55] Better handling of NaN metric values. --- src/mlpack/core/cv/k_fold_cv_impl.hpp | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/src/mlpack/core/cv/k_fold_cv_impl.hpp b/src/mlpack/core/cv/k_fold_cv_impl.hpp index 46b0a9bf89..8dd9672053 100644 --- a/src/mlpack/core/cv/k_fold_cv_impl.hpp +++ b/src/mlpack/core/cv/k_fold_cv_impl.hpp @@ -246,17 +246,32 @@ double KFoldCV Date: Sat, 29 Aug 2020 20:43:38 -0400 Subject: [PATCH 16/55] Update HISTORY. --- HISTORY.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/HISTORY.md b/HISTORY.md index 596b2f9165..e75eed284d 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,5 +1,7 @@ ### mlpack ?.?.? ###### ????-??-?? + * Issue warnings when metrics produce NaNs in KFoldCV (#2595). + * Added Spatial Dropout layer (#2564). * Force CMake to show error when it didn't find Python/modules (#2568). From c70ece7bafab0b2495461327c1883040508b9ca0 Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Mon, 31 Aug 2020 09:50:24 -0400 Subject: [PATCH 17/55] Update HISTORY.md Co-authored-by: Marcus Edel --- HISTORY.md | 1 - 1 file changed, 1 deletion(-) diff --git a/HISTORY.md b/HISTORY.md index d9ddade3ab..b25d71a58a 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -13,7 +13,6 @@ methods/hoeffding_trees/information_gain.hpp (#2556). * Added macro for changing stream of printing and warnings/errors (#2556). ->>>>>>> origin/master * Added Spatial Dropout layer (#2564). From 1f2335edb18a2b9146a2950233c1e87ac85da715 Mon Sep 17 00:00:00 2001 From: Marcus Edel Date: Tue, 1 Sep 2020 01:00:16 +0200 Subject: [PATCH 18/55] Let's see if there is another python version installed. --- .ci/linux-steps.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.ci/linux-steps.yaml b/.ci/linux-steps.yaml index 8d0a76bb69..bbac934b73 100644 --- a/.ci/linux-steps.yaml +++ b/.ci/linux-steps.yaml @@ -23,6 +23,8 @@ steps: sudo apt-get install -y --allow-unauthenticated libopenblas-dev liblapack-dev g++ libboost1.70-dev libarmadillo-dev xz-utils + ls -l /usr/bin/ + if [ "$(binding)" == "python" ]; then /usr/bin/python3 -m pip install --upgrade pip /usr/bin/python3 -m pip install --upgrade --ignore-installed setuptools cython pandas From ecfef0ca05469e7daeca1ab7af1ee9caf36ff9c1 Mon Sep 17 00:00:00 2001 From: Marcus Edel Date: Tue, 1 Sep 2020 11:13:04 +0200 Subject: [PATCH 19/55] Let's see if that picks up the correct python bin. --- .ci/linux-steps.yaml | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/.ci/linux-steps.yaml b/.ci/linux-steps.yaml index bbac934b73..69acf73201 100644 --- a/.ci/linux-steps.yaml +++ b/.ci/linux-steps.yaml @@ -24,10 +24,12 @@ steps: sudo apt-get install -y --allow-unauthenticated libopenblas-dev liblapack-dev g++ libboost1.70-dev libarmadillo-dev xz-utils ls -l /usr/bin/ + which python if [ "$(binding)" == "python" ]; then - /usr/bin/python3 -m pip install --upgrade pip - /usr/bin/python3 -m pip install --upgrade --ignore-installed setuptools cython pandas + export PYBIN=$(which python) + $PYBIN -m pip install --upgrade pip + $PYBIN -m pip install --upgrade --ignore-installed setuptools cython pandas fi if [ "a$(julia.version)" != "a" ]; then @@ -57,7 +59,11 @@ steps: export GOPATH=$PWD/src/mlpack/bindings/go go get -u -t gonum.org/v1/gonum/... fi - cmake $(CMakeArgs) .. + if [ "$(binding)" == "python" ]; then + cmake $(CMakeArgs) -DPYTHON_EXECUTABLE=$PYBIN .. + else + cmake $(CMakeArgs) .. + fi displayName: 'CMake' # Build mlpack From 7045fcd7f1df4d119444d52696ba7c937f08847b Mon Sep 17 00:00:00 2001 From: Marcus Edel Date: Tue, 1 Sep 2020 13:07:43 +0200 Subject: [PATCH 20/55] Export Python binary path. --- .ci/linux-steps.yaml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.ci/linux-steps.yaml b/.ci/linux-steps.yaml index 69acf73201..fd74a4553a 100644 --- a/.ci/linux-steps.yaml +++ b/.ci/linux-steps.yaml @@ -23,9 +23,6 @@ steps: sudo apt-get install -y --allow-unauthenticated libopenblas-dev liblapack-dev g++ libboost1.70-dev libarmadillo-dev xz-utils - ls -l /usr/bin/ - which python - if [ "$(binding)" == "python" ]; then export PYBIN=$(which python) $PYBIN -m pip install --upgrade pip @@ -60,6 +57,7 @@ steps: go get -u -t gonum.org/v1/gonum/... fi if [ "$(binding)" == "python" ]; then + export PYBIN=$(which python) cmake $(CMakeArgs) -DPYTHON_EXECUTABLE=$PYBIN .. else cmake $(CMakeArgs) .. From 5a22416184bf86d8ae6565e7118e50e5b6ce6b17 Mon Sep 17 00:00:00 2001 From: Marcus Edel Date: Tue, 1 Sep 2020 16:21:51 +0200 Subject: [PATCH 21/55] Always set PYTHON_EXECUTABLE simplify the config. Co-authored-by: Ryan Curtin --- .ci/linux-steps.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/linux-steps.yaml b/.ci/linux-steps.yaml index fd74a4553a..4c38dfcc89 100644 --- a/.ci/linux-steps.yaml +++ b/.ci/linux-steps.yaml @@ -61,7 +61,7 @@ steps: cmake $(CMakeArgs) -DPYTHON_EXECUTABLE=$PYBIN .. else cmake $(CMakeArgs) .. - fi +cmake $(CMakeArgs) -DPYTHON_EXECUTABLE=`which python` .. displayName: 'CMake' # Build mlpack From e601fc59765bbdc63c9cda127ae60a2a47b2cf94 Mon Sep 17 00:00:00 2001 From: Marcus Edel Date: Tue, 1 Sep 2020 16:33:04 +0200 Subject: [PATCH 22/55] Simplify config. --- .ci/linux-steps.yaml | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/.ci/linux-steps.yaml b/.ci/linux-steps.yaml index 4c38dfcc89..84538f59b4 100644 --- a/.ci/linux-steps.yaml +++ b/.ci/linux-steps.yaml @@ -56,12 +56,7 @@ steps: export GOPATH=$PWD/src/mlpack/bindings/go go get -u -t gonum.org/v1/gonum/... fi - if [ "$(binding)" == "python" ]; then - export PYBIN=$(which python) - cmake $(CMakeArgs) -DPYTHON_EXECUTABLE=$PYBIN .. - else - cmake $(CMakeArgs) .. -cmake $(CMakeArgs) -DPYTHON_EXECUTABLE=`which python` .. + cmake $(CMakeArgs) -DPYTHON_EXECUTABLE=`which python` .. displayName: 'CMake' # Build mlpack From f2bec6cd9501326cc9845290f102587718fa233b Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Tue, 1 Sep 2020 17:02:50 -0400 Subject: [PATCH 23/55] Add missing license. --- .../bayesian_linear_regression.hpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/mlpack/methods/bayesian_linear_regression/bayesian_linear_regression.hpp b/src/mlpack/methods/bayesian_linear_regression/bayesian_linear_regression.hpp index 415f63e595..09a9a09fb3 100644 --- a/src/mlpack/methods/bayesian_linear_regression/bayesian_linear_regression.hpp +++ b/src/mlpack/methods/bayesian_linear_regression/bayesian_linear_regression.hpp @@ -5,6 +5,11 @@ * Definition of the BayesianRidge class, which performs the * bayesian linear regression. According to the armadillo standards, * all the functions consider data in column-major format. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. **/ #ifndef MLPACK_METHODS_BAYESIAN_LINEAR_REGRESSION_HPP From 3efae284a5981e063bd0870fef72fd618a3bedc9 Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Tue, 1 Sep 2020 17:11:15 -0400 Subject: [PATCH 24/55] Update version to 3.4.0. --- CMakeLists.txt | 8 ++++---- Doxyfile | 2 +- HISTORY.md | 4 ++-- README.md | 4 ++-- .../sample-ml-app/sample-ml-app.vcxproj | 8 ++++---- doc/guide/build.hpp | 12 ++++++------ doc/guide/python_quickstart.hpp | 6 +++--- doc/guide/sample_ml_app.hpp | 8 ++++---- src/mlpack/CMakeLists.txt | 2 +- src/mlpack/core/util/version.hpp | 4 ++-- 10 files changed, 29 insertions(+), 29 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index a1f1a45937..ca8031408b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -360,14 +360,14 @@ endif () find_package(Ensmallen "${ENSMALLEN_VERSION}") if (NOT ENSMALLEN_FOUND) if (DOWNLOAD_ENSMALLEN) - file(DOWNLOAD http://www.ensmallen.org/files/ensmallen-latest.tar.gz - "${CMAKE_BINARY_DIR}/deps/ensmallen-latest.tar.gz" + file(DOWNLOAD http://www.ensmallen.org/files/ensmallen-2.14.1.tar.gz + "${CMAKE_BINARY_DIR}/deps/ensmallen-2.14.1.tar.gz" STATUS ENS_DOWNLOAD_STATUS_LIST LOG ENS_DOWNLOAD_LOG SHOW_PROGRESS) list(GET ENS_DOWNLOAD_STATUS_LIST 0 ENS_DOWNLOAD_STATUS) if (ENS_DOWNLOAD_STATUS EQUAL 0) execute_process(COMMAND ${CMAKE_COMMAND} -E - tar xzf "${CMAKE_BINARY_DIR}/deps/ensmallen-latest.tar.gz" + tar xzf "${CMAKE_BINARY_DIR}/deps/ensmallen-2.14.1.tar.gz" WORKING_DIRECTORY "${CMAKE_BINARY_DIR}/deps/") # Get the name of the directory. @@ -375,7 +375,7 @@ if (NOT ENSMALLEN_FOUND) "${CMAKE_BINARY_DIR}/deps/ensmallen-[0-9]*.[0-9]*.[0-9]*") # list(FILTER) is not available on 3.5 or older, but try to keep # configuring without filtering the list anyway (it might work if only - # the file ensmallen-latest.tar.gz is present. + # the file ensmallen-2.14.1.tar.gz is present. if (${CMAKE_VERSION} VERSION_GREATER_EQUAL "3.6.0") list(FILTER ENS_DIRECTORIES EXCLUDE REGEX "ensmallen-.*\.tar\.gz") endif () diff --git a/Doxyfile b/Doxyfile index 23597e8a5f..5f28220461 100644 --- a/Doxyfile +++ b/Doxyfile @@ -4,7 +4,7 @@ # Project related configuration options #--------------------------------------------------------------------------- PROJECT_NAME = mlpack -PROJECT_NUMBER = 3.3.2 +PROJECT_NUMBER = 3.4.0 OUTPUT_DIRECTORY = ./doc CREATE_SUBDIRS = NO OUTPUT_LANGUAGE = English diff --git a/HISTORY.md b/HISTORY.md index b25d71a58a..cb19b56368 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,5 +1,5 @@ -### mlpack ?.?.? -###### ????-??-?? +### mlpack 3.4.0 +###### 2020-09-01 * Issue warnings when metrics produce NaNs in KFoldCV (#2595). diff --git a/README.md b/README.md index d8657d15ac..4e5cc4aaa5 100644 --- a/README.md +++ b/README.md @@ -23,7 +23,7 @@ src="https://cdn.rawgit.com/mlpack/mlpack.org/e7d36ed8/mlpack-black.svg" style="

Download: - current stable version (3.2.2) + current stable version (3.4.0)

@@ -152,7 +152,7 @@ on Ubuntu, you can install mlpack with the following command: Note: Older Ubuntu versions may not have the most recent version of mlpack available---for instance, at the time of this writing, Ubuntu 16.04 only has -mlpack 3.2.2 available. Options include upgrading your Ubuntu version, finding +mlpack 3.4.0 available. Options include upgrading your Ubuntu version, finding a PPA or other non-official sources, or installing with a manual build. There are some useful pages to consult in addition to this section: diff --git a/doc/examples/sample-ml-app/sample-ml-app/sample-ml-app.vcxproj b/doc/examples/sample-ml-app/sample-ml-app/sample-ml-app.vcxproj index 2c427cfc3a..29b8d98818 100644 --- a/doc/examples/sample-ml-app/sample-ml-app/sample-ml-app.vcxproj +++ b/doc/examples/sample-ml-app/sample-ml-app/sample-ml-app.vcxproj @@ -104,16 +104,16 @@ true _DEBUG;_CONSOLE;%(PreprocessorDefinitions) false - C:\boost\boost_1_66_0;C:\mlpack\armadillo-8.500.1\include;C:\mlpack\mlpack-3.2.2\build\include;%(AdditionalIncludeDirectories) + C:\boost\boost_1_66_0;C:\mlpack\armadillo-8.500.1\include;C:\mlpack\mlpack-3.4.0\build\include;%(AdditionalIncludeDirectories) Console true - C:\mlpack\mlpack-3.2.2\build\Debug\mlpack.lib;C:\boost\boost_1_66_0\lib64-msvc-14.1\libboost_serialization-vc141-mt-gd-x64-1_66.lib;%(AdditionalDependencies) + C:\mlpack\mlpack-3.4.0\build\Debug\mlpack.lib;C:\boost\boost_1_66_0\lib64-msvc-14.1\libboost_serialization-vc141-mt-gd-x64-1_66.lib;%(AdditionalDependencies) - xcopy /y "C:\mlpack\mlpack-3.2.2\build\Debug\mlpack.dll" $(OutDir) -xcopy /y "C:\mlpack\mlpack-3.2.2\packages\OpenBLAS.0.2.14.1\lib\native\bin\x64\*.dll" $(OutDir) + xcopy /y "C:\mlpack\mlpack-3.4.0\build\Debug\mlpack.dll" $(OutDir) +xcopy /y "C:\mlpack\mlpack-3.4.0\packages\OpenBLAS.0.2.14.1\lib\native\bin\x64\*.dll" $(OutDir) xcopy /y "$(ProjectDir)..\..\..\..\src\mlpack\tests\data\german.csv" "$(ProjectDir)data\german.csv*" diff --git a/doc/guide/build.hpp b/doc/guide/build.hpp index 1e8f4be536..f60cee3aa5 100644 --- a/doc/guide/build.hpp +++ b/doc/guide/build.hpp @@ -30,7 +30,7 @@ to build mlpack on Windows, see \ref build_windows (alternatively, you can read is based on older versions). You can download the latest mlpack release from here: -mlpack-3.2.2 +mlpack-3.4.0 @section build_simple Simple Linux build instructions @@ -38,9 +38,9 @@ Assuming all dependencies are installed in the system, you can run the commands below directly to build and install mlpack. @code -$ wget https://www.mlpack.org/files/mlpack-3.2.2.tar.gz -$ tar -xvzpf mlpack-3.2.2.tar.gz -$ mkdir mlpack-3.2.2/build && cd mlpack-3.2.2/build +$ wget https://www.mlpack.org/files/mlpack-3.4.0.tar.gz +$ tar -xvzpf mlpack-3.4.0.tar.gz +$ mkdir mlpack-3.4.0/build && cd mlpack-3.4.0/build $ cmake ../ $ make -j4 # The -j is the number of cores you want to use for a build. $ sudo make install @@ -65,8 +65,8 @@ configure mlpack. First we should unpack the mlpack source and create a build directory. @code -$ tar -xvzpf mlpack-3.2.2.tar.gz -$ cd mlpack-3.2.2 +$ tar -xvzpf mlpack-3.4.0.tar.gz +$ cd mlpack-3.4.0 $ mkdir build @endcode diff --git a/doc/guide/python_quickstart.hpp b/doc/guide/python_quickstart.hpp index 981c1449f3..47aeefed15 100644 --- a/doc/guide/python_quickstart.hpp +++ b/doc/guide/python_quickstart.hpp @@ -32,9 +32,9 @@ build and install mlpack. You can copy-paste the commands into your shell. @code{.sh} sudo apt-get install libboost-all-dev g++ cmake libarmadillo-dev python-pip wget sudo pip install cython setuptools distutils numpy pandas -wget https://www.mlpack.org/files/mlpack-3.2.2.tar.gz -tar -xvzpf mlpack-3.2.2.tar.gz -mkdir -p mlpack-3.2.2/build/ && cd mlpack-3.2.2/build/ +wget https://www.mlpack.org/files/mlpack-3.4.0.tar.gz +tar -xvzpf mlpack-3.4.0.tar.gz +mkdir -p mlpack-3.4.0/build/ && cd mlpack-3.4.0/build/ cmake ../ && make -j4 && sudo make install @endcode diff --git a/doc/guide/sample_ml_app.hpp b/doc/guide/sample_ml_app.hpp index 6a1b1aa30a..fa71905602 100644 --- a/doc/guide/sample_ml_app.hpp +++ b/doc/guide/sample_ml_app.hpp @@ -29,17 +29,17 @@ mlpack and dependencies in Release Mode). @code - C:\boost\boost_1_71_0\lib\native\include - C:\mlpack\armadillo-9.800.3\include - - C:\mlpack\mlpack-3.2.2\build\include + - C:\mlpack\mlpack-3.4.0\build\include @endcode - Under Linker > Input > Additional Dependencies add: @code - - C:\mlpack\mlpack-3.2.2\build\Debug\mlpack.lib + - C:\mlpack\mlpack-3.4.0\build\Debug\mlpack.lib - C:\boost\boost_1_71_0\lib64-msvc-14.2\libboost_serialization-vc142-mt-gd-x64-1_71.lib @endcode - Under Build Events > Post-Build Event > Command Line add: @code - - xcopy /y "C:\mlpack\mlpack-3.2.2\build\Debug\mlpack.dll" $(OutDir) - - xcopy /y "C:\mlpack\mlpack-3.2.2\packages\OpenBLAS.0.2.14.1\lib\native\bin\x64\*.dll" $(OutDir) + - xcopy /y "C:\mlpack\mlpack-3.4.0\build\Debug\mlpack.dll" $(OutDir) + - xcopy /y "C:\mlpack\mlpack-3.4.0\packages\OpenBLAS.0.2.14.1\lib\native\bin\x64\*.dll" $(OutDir) @endcode @note Recent versions of Visual Studio set "Conformance Mode" enabled by default. This causes some issues with diff --git a/src/mlpack/CMakeLists.txt b/src/mlpack/CMakeLists.txt index b284d60dfc..12ff343a34 100644 --- a/src/mlpack/CMakeLists.txt +++ b/src/mlpack/CMakeLists.txt @@ -54,7 +54,7 @@ target_link_libraries(mlpack ${MLPACK_LIBRARIES}) set_target_properties(mlpack PROPERTIES - VERSION 3.3 + VERSION 3.4 SOVERSION 3 ) diff --git a/src/mlpack/core/util/version.hpp b/src/mlpack/core/util/version.hpp index 09e1042691..83b6336a56 100644 --- a/src/mlpack/core/util/version.hpp +++ b/src/mlpack/core/util/version.hpp @@ -17,8 +17,8 @@ // The version of mlpack. If this is a git repository, this will be a version // with higher number than the most recent release. #define MLPACK_VERSION_MAJOR 3 -#define MLPACK_VERSION_MINOR 2 -#define MLPACK_VERSION_PATCH 3 +#define MLPACK_VERSION_MINOR 4 +#define MLPACK_VERSION_PATCH 0 // The name of the version (for use by --version). namespace mlpack { From b3039c694216088786042f5faf8d5fa03ff0ba83 Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Tue, 1 Sep 2020 17:11:15 -0400 Subject: [PATCH 25/55] Update version to next release version. --- CMakeLists.txt | 8 ++++---- src/mlpack/core/util/version.hpp | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index ca8031408b..a1f1a45937 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -360,14 +360,14 @@ endif () find_package(Ensmallen "${ENSMALLEN_VERSION}") if (NOT ENSMALLEN_FOUND) if (DOWNLOAD_ENSMALLEN) - file(DOWNLOAD http://www.ensmallen.org/files/ensmallen-2.14.1.tar.gz - "${CMAKE_BINARY_DIR}/deps/ensmallen-2.14.1.tar.gz" + file(DOWNLOAD http://www.ensmallen.org/files/ensmallen-latest.tar.gz + "${CMAKE_BINARY_DIR}/deps/ensmallen-latest.tar.gz" STATUS ENS_DOWNLOAD_STATUS_LIST LOG ENS_DOWNLOAD_LOG SHOW_PROGRESS) list(GET ENS_DOWNLOAD_STATUS_LIST 0 ENS_DOWNLOAD_STATUS) if (ENS_DOWNLOAD_STATUS EQUAL 0) execute_process(COMMAND ${CMAKE_COMMAND} -E - tar xzf "${CMAKE_BINARY_DIR}/deps/ensmallen-2.14.1.tar.gz" + tar xzf "${CMAKE_BINARY_DIR}/deps/ensmallen-latest.tar.gz" WORKING_DIRECTORY "${CMAKE_BINARY_DIR}/deps/") # Get the name of the directory. @@ -375,7 +375,7 @@ if (NOT ENSMALLEN_FOUND) "${CMAKE_BINARY_DIR}/deps/ensmallen-[0-9]*.[0-9]*.[0-9]*") # list(FILTER) is not available on 3.5 or older, but try to keep # configuring without filtering the list anyway (it might work if only - # the file ensmallen-2.14.1.tar.gz is present. + # the file ensmallen-latest.tar.gz is present. if (${CMAKE_VERSION} VERSION_GREATER_EQUAL "3.6.0") list(FILTER ENS_DIRECTORIES EXCLUDE REGEX "ensmallen-.*\.tar\.gz") endif () diff --git a/src/mlpack/core/util/version.hpp b/src/mlpack/core/util/version.hpp index 83b6336a56..3a8437d332 100644 --- a/src/mlpack/core/util/version.hpp +++ b/src/mlpack/core/util/version.hpp @@ -18,7 +18,7 @@ // with higher number than the most recent release. #define MLPACK_VERSION_MAJOR 3 #define MLPACK_VERSION_MINOR 4 -#define MLPACK_VERSION_PATCH 0 +#define MLPACK_VERSION_PATCH 1 // The name of the version (for use by --version). namespace mlpack { From bb72def957fbd124d252d3edc659c1e5bea60d71 Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Tue, 1 Sep 2020 17:11:15 -0400 Subject: [PATCH 26/55] Add new block to HISTORY.md for next version. --- HISTORY.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/HISTORY.md b/HISTORY.md index cb19b56368..0a8ab2a16a 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,3 +1,6 @@ +### mlpack ?.?.? +###### ????-??-?? + ### mlpack 3.4.0 ###### 2020-09-01 From 38047948f98cca80cc6067cee8c6923a35be8aa1 Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Tue, 1 Sep 2020 19:41:10 -0400 Subject: [PATCH 27/55] Use const arma::file_type (pedantry). --- src/mlpack/core/data/load.cpp | 12 ++++++------ src/mlpack/core/data/load.hpp | 14 +++++++------- src/mlpack/core/data/load_impl.hpp | 2 +- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/src/mlpack/core/data/load.cpp b/src/mlpack/core/data/load.cpp index 78283b9803..71ad35cb64 100644 --- a/src/mlpack/core/data/load.cpp +++ b/src/mlpack/core/data/load.cpp @@ -19,37 +19,37 @@ template bool Load(const std::string&, arma::Mat&, const bool, const bool, - arma::file_type); + const arma::file_type); template bool Load(const std::string&, arma::Mat&, const bool, const bool, - arma::file_type); + const arma::file_type); template bool Load(const std::string&, arma::Mat&, const bool, const bool, - arma::file_type); + const arma::file_type); template bool Load(const std::string&, arma::Mat&, const bool, const bool, - arma::file_type); + const arma::file_type); template bool Load(const std::string&, arma::Mat&, const bool, const bool, - arma::file_type); + const arma::file_type); template bool Load(const std::string&, arma::Mat&, const bool, const bool, - arma::file_type); + const arma::file_type); template bool Load(const std::string&, arma::SpMat&, diff --git a/src/mlpack/core/data/load.hpp b/src/mlpack/core/data/load.hpp index da45eb9836..ae01f01f25 100644 --- a/src/mlpack/core/data/load.hpp +++ b/src/mlpack/core/data/load.hpp @@ -65,7 +65,7 @@ bool Load(const std::string& filename, arma::Mat& matrix, const bool fatal = false, const bool transpose = true, - arma::file_type inputLoadType = arma::auto_detect); + const arma::file_type inputLoadType = arma::auto_detect); /** * Loads a sparse matrix from file, using arma::coord_ascii format. This @@ -113,38 +113,38 @@ extern template bool Load(const std::string&, arma::Mat&, const bool, const bool, - arma::file_type); + const arma::file_type); // size_t and uword should be one of these three typedefs. extern template bool Load(const std::string&, arma::Mat&, const bool, const bool, - arma::file_type); + const arma::file_type); extern template bool Load(const std::string&, arma::Mat&, const bool, const bool, - arma::file_type); + const arma::file_type); extern template bool Load(const std::string&, arma::Mat&, const bool, const bool, - arma::file_type); + const arma::file_type); extern template bool Load(const std::string&, arma::Mat&, const bool, const bool, - arma::file_type); + const arma::file_type); extern template bool Load(const std::string&, arma::Mat&, const bool, const bool, - arma::file_type); + const arma::file_type); extern template bool Load(const std::string&, arma::Mat&, diff --git a/src/mlpack/core/data/load_impl.hpp b/src/mlpack/core/data/load_impl.hpp index 9409cb6bb7..9ce790de8f 100644 --- a/src/mlpack/core/data/load_impl.hpp +++ b/src/mlpack/core/data/load_impl.hpp @@ -333,7 +333,7 @@ bool Load(const std::string& filename, arma::Mat& matrix, const bool fatal, const bool transpose, - arma::file_type inputLoadType) + const arma::file_type inputLoadType) { Timer::Start("loading_data"); From 4011aa401a127bbcad9803db9d6377ed08e984b0 Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Wed, 2 Sep 2020 12:52:26 -0400 Subject: [PATCH 28/55] Prepend "--" to options when checking that they are there. --- .../bindings/cli/parse_command_line.hpp | 10 +- src/mlpack/tests/io_test.cpp | 99 +++++++++++++++++++ 2 files changed, 105 insertions(+), 4 deletions(-) diff --git a/src/mlpack/bindings/cli/parse_command_line.hpp b/src/mlpack/bindings/cli/parse_command_line.hpp index a611e178fb..381d749bc1 100644 --- a/src/mlpack/bindings/cli/parse_command_line.hpp +++ b/src/mlpack/bindings/cli/parse_command_line.hpp @@ -47,8 +47,8 @@ void ParseCommandLine(int argc, char** argv) { // Add the parameter to desc. util::ParamData& d = it->second; - IO::GetSingleton().functionMap[d.tname]["AddToCLI11"] - (d, NULL, (void*) &app); + IO::GetSingleton().functionMap[d.tname]["AddToCLI11"](d, NULL, (void*) + &app); } // Mark that we did parsing. @@ -136,13 +136,15 @@ void ParseCommandLine(int argc, char** argv) util::ParamData d = iter->second; if (d.required) { - const std::string cliName; + // CLI11 expects the parameter name to have "--" prepended. + std::string cliName; IO::GetSingleton().functionMap[d.tname]["MapParameterName"](d, NULL, (void*) &cliName); + cliName = "--" + cliName; if (!app.count(cliName)) { - Log::Fatal << "Required option --" << cliName << " is undefined." + Log::Fatal << "Required option " << cliName << " is undefined." << std::endl; } } diff --git a/src/mlpack/tests/io_test.cpp b/src/mlpack/tests/io_test.cpp index 766e36facc..80aef17fbe 100644 --- a/src/mlpack/tests/io_test.cpp +++ b/src/mlpack/tests/io_test.cpp @@ -598,6 +598,105 @@ BOOST_AUTO_TEST_CASE(InputMatrixParamTest) BOOST_REQUIRE_CLOSE(dataset[i], dataset2[i], 1e-10); } +// Make sure we can correctly load required matrix parameters. +BOOST_AUTO_TEST_CASE(RequiredInputMatrixParamTest) +{ + AddRequiredCLIOptions(); + + // --matrix is an input parameter; it won't be transposed. + PARAM_MATRIX_IN_REQ("matrix", "Test matrix", "m"); + + // Set some fake arguments. + const char* argv[3]; + argv[0] = "./test"; + argv[1] = "--matrix_file"; + argv[2] = "test_data_3_1000.csv"; + + int argc = 3; + + // The const-cast is a little hacky but should be fine... + ParseCommandLine(argc, const_cast(argv)); + + // The --matrix parameter should exist. + BOOST_REQUIRE(IO::HasParam("matrix")); + // The --matrix_file parameter should not exist (it should be transparent from + // inside the program). + Log::Fatal.ignoreInput = true; + BOOST_REQUIRE_THROW(IO::HasParam("matrix_file"), runtime_error); + Log::Fatal.ignoreInput = false; + + arma::mat dataset = IO::GetParam("matrix"); + arma::mat dataset2 = IO::GetParam("matrix"); + + BOOST_REQUIRE_EQUAL(dataset.n_rows, 3); + BOOST_REQUIRE_EQUAL(dataset.n_cols, 1000); + BOOST_REQUIRE_EQUAL(dataset2.n_rows, 3); + BOOST_REQUIRE_EQUAL(dataset2.n_cols, 1000); + + for (size_t i = 0; i < dataset.n_elem; ++i) + BOOST_REQUIRE_CLOSE(dataset[i], dataset2[i], 1e-10); +} + +// Make sure loading required matrix options by alias succeeds. +BOOST_AUTO_TEST_CASE(RequiredInputMatrixParamAliasTest) +{ + AddRequiredCLIOptions(); + + // --matrix is an input parameter; it won't be transposed. + PARAM_MATRIX_IN_REQ("matrix", "Test matrix", "m"); + + // Set some fake arguments. + const char* argv[3]; + argv[0] = "./test"; + argv[1] = "-m"; + argv[2] = "test_data_3_1000.csv"; + + int argc = 3; + + // The const-cast is a little hacky but should be fine... + ParseCommandLine(argc, const_cast(argv)); + + // The --matrix parameter should exist. + BOOST_REQUIRE(IO::HasParam("matrix")); + // The --matrix_file parameter should not exist (it should be transparent from + // inside the program). + Log::Fatal.ignoreInput = true; + BOOST_REQUIRE_THROW(IO::HasParam("matrix_file"), runtime_error); + Log::Fatal.ignoreInput = false; + + arma::mat dataset = IO::GetParam("matrix"); + arma::mat dataset2 = IO::GetParam("matrix"); + + BOOST_REQUIRE_EQUAL(dataset.n_rows, 3); + BOOST_REQUIRE_EQUAL(dataset.n_cols, 1000); + BOOST_REQUIRE_EQUAL(dataset2.n_rows, 3); + BOOST_REQUIRE_EQUAL(dataset2.n_cols, 1000); + + for (size_t i = 0; i < dataset.n_elem; ++i) + BOOST_REQUIRE_CLOSE(dataset[i], dataset2[i], 1e-10); +} + +// Make sure that when we don't pass a required matrix, parsing fails. +BOOST_AUTO_TEST_CASE(RequiredUnspecifiedInputMatrixParamTest) +{ + AddRequiredCLIOptions(); + + // --matrix is an input parameter; it won't be transposed. + PARAM_MATRIX_IN_REQ("matrix", "Test matrix", "m"); + + // Set some fake arguments. + const char* argv[1]; + argv[0] = "./test"; + + int argc = 1; + + // The const-cast is a little hacky but should be fine... + Log::Fatal.ignoreInput = true; + BOOST_REQUIRE_THROW(ParseCommandLine(argc, const_cast(argv)), + std::exception); + Log::Fatal.ignoreInput = false; +} + BOOST_AUTO_TEST_CASE(InputMatrixNoTransposeParamTest) { AddRequiredCLIOptions(); From 008607c754fd651d45e63c09e32bf96cbd655f68 Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Wed, 2 Sep 2020 13:07:11 -0400 Subject: [PATCH 29/55] Update HISTORY.md. --- HISTORY.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/HISTORY.md b/HISTORY.md index 0a8ab2a16a..d03be5e5e5 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,5 +1,7 @@ ### mlpack ?.?.? ###### ????-??-?? + * Fix incorrect parsing of required matrix/model parameters for command-line + bindings (#2600). ### mlpack 3.4.0 ###### 2020-09-01 From 75b6ffb0e6a5f2d97c97c01d516efaec0a720ca6 Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Thu, 3 Sep 2020 11:55:11 -0400 Subject: [PATCH 30/55] Fix missing parameter. --- src/mlpack/core/data/load.hpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/mlpack/core/data/load.hpp b/src/mlpack/core/data/load.hpp index ae01f01f25..50184e594d 100644 --- a/src/mlpack/core/data/load.hpp +++ b/src/mlpack/core/data/load.hpp @@ -149,7 +149,8 @@ extern template bool Load(const std::string&, extern template bool Load(const std::string&, arma::Mat&, const bool, - const bool); + const bool, + const arma::file_type); extern template bool Load(const std::string&, arma::SpMat&, From 637eafc7e373e92da409f34546a08239fe0e9f30 Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Thu, 3 Sep 2020 11:56:46 -0400 Subject: [PATCH 31/55] Refactor auto-detection into a separate file. --- src/mlpack/core/data/CMakeLists.txt | 2 + src/mlpack/core/data/detect_file_type.cpp | 291 ++++++++++++++++++++++ src/mlpack/core/data/detect_file_type.hpp | 63 +++++ src/mlpack/core/data/load_impl.hpp | 287 +++------------------ src/mlpack/core/data/save_impl.hpp | 147 ++--------- 5 files changed, 413 insertions(+), 377 deletions(-) create mode 100644 src/mlpack/core/data/detect_file_type.cpp create mode 100644 src/mlpack/core/data/detect_file_type.hpp diff --git a/src/mlpack/core/data/CMakeLists.txt b/src/mlpack/core/data/CMakeLists.txt index cc99c7b175..6f838754f6 100644 --- a/src/mlpack/core/data/CMakeLists.txt +++ b/src/mlpack/core/data/CMakeLists.txt @@ -3,6 +3,8 @@ set(SOURCES dataset_mapper.hpp dataset_mapper_impl.hpp + detect_file_type.hpp + detect_file_type.cpp extension.hpp format.hpp has_serialize.hpp diff --git a/src/mlpack/core/data/detect_file_type.cpp b/src/mlpack/core/data/detect_file_type.cpp new file mode 100644 index 0000000000..520f9bec1d --- /dev/null +++ b/src/mlpack/core/data/detect_file_type.cpp @@ -0,0 +1,291 @@ +/** + * @file detect_file_type.cpp + * @author Conrad Sanderson + * @author Ryan Curtin + * + * Functionality to guess the type of a file by inspecting it. Parts of the + * implementation are adapted from the Armadillo sources and relicensed to be a + * part of mlpack with permission from Conrad. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#include "extension.hpp" +#include "detect_file_type.hpp" + +#include +#include +#include + +namespace mlpack { +namespace data { + +/** + * Given a file type, return a logical name corresponding to that file type. + * + * @param type Type to get the logical name of. + */ +std::string GetStringType(const arma::file_type& type) +{ + switch (type) + { + case arma::csv_ascii: return "CSV data"; + case arma::raw_ascii: return "raw ASCII formatted data"; + case arma::raw_binary: return "raw binary formatted data"; + case arma::arma_ascii: return "Armadillo ASCII formatted data"; + case arma::arma_binary: return "Armadillo binary formatted data"; + case arma::pgm_binary: return "PGM data"; + case arma::hdf5_binary: return "HDF5 data"; + default: return ""; + } +} + +/** + * Given an istream, attempt to guess the file type. This is taken originally + * from Armadillo's function guess_file_type_internal(), but we avoid using + * internal Armadillo functionality. + * + * @param f Opened istream to look into to guess the file type. + */ +arma::file_type GuessFileType(std::istream& f) +{ + f.clear(); + const std::fstream::pos_type pos1 = f.tellg(); + + f.clear(); + f.seekg(0, std::ios::end); + + f.clear(); + // Get the length of the stream. + const std::fstream::pos_type pos2 = f.tellg(); + + // Compute length of the stream. + const arma::uword nMax = ((pos1 >= 0) && (pos2 >= 0) && (pos2 > pos1)) ? + arma::uword(pos2 - pos1) : arma::uword(0); + + f.clear(); + f.seekg(pos1); + + // Handle empty files. + if (nMax == 0) + return arma::file_type_unknown; + + const arma::uword nUse = std::min(nMax, arma::uword(4096)); + + unsigned char* dataMem = + (unsigned char*) malloc(sizeof(unsigned char) *nUse); + memset(dataMem, 0, nUse); + + f.clear(); + f.read(reinterpret_cast(dataMem), std::streamsize(nUse)); + + const bool loadOkay = f.good(); + + f.clear(); + f.seekg(pos1); + + if (!loadOkay) + { + delete dataMem; + return arma::file_type_unknown; + } + + bool hasBinary = false; + bool hasBracket = false; + bool hasComma = false; + + for (arma::uword i = 0; i < nUse; ++i) + { + const unsigned char val = dataMem[i]; + if ((val <= 8) || (val >= 123)) + { + hasBinary = true; + break; + } // The range checking can be made more elaborate. + + if ((val == '(') || (val == ')')) + { + hasBracket = true; + } + if (val == ',') + { + hasComma = true; + } + } + + delete dataMem; + + if (hasBinary) + return arma::raw_binary; + + if (hasComma && (hasBracket == false)) + return arma::csv_ascii; + + return arma::raw_ascii; +} + +/** + * Attempt to auto-detect the type of a file given its extension, and by + * inspecting the parts of the file to disambiguate between types when + * necessary. (For instance, a .csv file could be delimited by spaces, commas, + * or tabs.) This is meant to be used during loading. + * + * @param stream Opened file stream to look into for autodetection. + * @param filename Name of the file. + * @return The detected file type. + */ +arma::file_type AutoDetect(std::fstream& stream, + const std::string filename) +{ + // Get the extension. + std::string extension = Extension(filename); + arma::file_type detectedLoadType = arma::file_type_unknown; + + if (extension == "csv" || extension == "tsv") + { + detectedLoadType = GuessFileType(stream); + if (detectedLoadType == arma::csv_ascii) + { + if (extension == "tsv") + Log::Warn << "'" << filename << "' is comma-separated, not " + "tab-separated!" << std::endl; + } + else if (detectedLoadType == arma::raw_ascii) // .csv file can be tsv. + { + if (extension == "csv") + { + // We should issue a warning, but we don't want to issue the warning if + // there is only one column in the CSV (since there will be no commas + // anyway, and it will be detected as arma::raw_ascii). + const std::streampos pos = stream.tellg(); + std::string line; + std::getline(stream, line, '\n'); + boost::trim(line); + + // Reset stream position. + stream.seekg(pos); + + // If there are no spaces or whitespace in the line, then we shouldn't + // print the warning. + if ((line.find(' ') != std::string::npos) || + (line.find('\t') != std::string::npos)) + { + Log::Warn << "'" << filename << "' is not a standard csv file." + << std::endl; + } + } + } + else + { + detectedLoadType = arma::file_type_unknown; + } + } + else if (extension == "txt") + { + // This could be raw ASCII or Armadillo ASCII (ASCII with size header). + // We'll let Armadillo do its guessing (although we have to check if it is + // arma_ascii ourselves) and see what we come up with. + + // This is taken from load_auto_detect() in diskio_meat.hpp + const std::string ARMA_MAT_TXT = "ARMA_MAT_TXT"; + // char* rawHeader = new char[ARMA_MAT_TXT.length() + 1]; + std::string rawHeader(ARMA_MAT_TXT.length(), '\0'); + std::streampos pos = stream.tellg(); + + stream.read(&rawHeader[0], std::streamsize(ARMA_MAT_TXT.length())); + stream.clear(); + stream.seekg(pos); // Reset stream position after peeking. + + if (rawHeader == ARMA_MAT_TXT) + { + detectedLoadType = arma::arma_ascii; + } + else // It's not arma_ascii. Now we let Armadillo guess. + { + detectedLoadType = GuessFileType(stream); + + if (detectedLoadType != arma::raw_ascii && + detectedLoadType != arma::csv_ascii) + detectedLoadType = arma::file_type_unknown; + } + } + else if (extension == "bin") + { + // This could be raw binary or Armadillo binary (binary with header). We + // will check to see if it is Armadillo binary. + const std::string ARMA_MAT_BIN = "ARMA_MAT_BIN"; + std::string rawHeader(ARMA_MAT_BIN.length(), '\0'); + + std::streampos pos = stream.tellg(); + + stream.read(&rawHeader[0], std::streamsize(ARMA_MAT_BIN.length())); + stream.clear(); + stream.seekg(pos); // Reset stream position after peeking. + + if (rawHeader == ARMA_MAT_BIN) + { + detectedLoadType = arma::arma_binary; + } + else // We can only assume it's raw binary. + { + detectedLoadType = arma::raw_binary; + } + } + else if (extension == "pgm") + { + detectedLoadType = arma::pgm_binary; + } + else if (extension == "h5" || extension == "hdf5" || extension == "hdf" || + extension == "he5") + { + detectedLoadType = arma::hdf5_binary; + } + else // Unknown extension... + { + detectedLoadType = arma::file_type_unknown; + } + + return detectedLoadType; +} + +/** + * Return the type based only on the extension. + * + * @param filename Name of the file whose type we should detect. + * @return Detected type of file. + */ +arma::file_type DetectFromExtension(const std::string& filename) +{ + const std::string extension = Extension(filename); + + if (extension == "csv") + { + return arma::csv_ascii; + } + else if (extension == "txt") + { + return arma::raw_ascii; + } + else if (extension == "bin") + { + return arma::arma_binary; + } + else if (extension == "pgm") + { + return arma::pgm_binary; + } + else if (extension == "h5" || extension == "hdf5" || extension == "hdf" || + extension == "he5") + { + return arma::hdf5_binary; + } + else + { + return arma::file_type_unknown; + } +} + +} // namespace data +} // namespace mlpack diff --git a/src/mlpack/core/data/detect_file_type.hpp b/src/mlpack/core/data/detect_file_type.hpp new file mode 100644 index 0000000000..8c99c8c1ab --- /dev/null +++ b/src/mlpack/core/data/detect_file_type.hpp @@ -0,0 +1,63 @@ +/** + * @file detect_file_type.hpp + * @author Conrad Sanderson + * @author Ryan Curtin + * + * Functionality to guess the type of a file by inspecting it. Parts of the + * implementation are adapted from the Armadillo sources and relicensed to be a + * part of mlpack with permission from Conrad. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_CORE_DATA_DETECT_FILE_TYPE_HPP +#define MLPACK_CORE_DATA_DETECT_FILE_TYPE_HPP + +namespace mlpack { +namespace data { + +/** + * Given a file type, return a logical name corresponding to that file type. + * + * @param type Type to get the logical name of. + */ +std::string GetStringType(const arma::file_type& type); + +/** + * Given an istream, attempt to guess the file type. This is taken originally + * from Armadillo's function guess_file_type_internal(), but we avoid using + * internal Armadillo functionality. + * + * @param f Opened istream to look into to guess the file type. + */ +arma::file_type GuessFileType(std::istream& f); + +/** + * Attempt to auto-detect the type of a file given its extension, and by + * inspecting the parts of the file to disambiguate between types when + * necessary. (For instance, a .csv file could be delimited by spaces, commas, + * or tabs.) This is meant to be used during loading. + * + * @param stream Opened file stream to look into for autodetection. + * @param filename Name of the file. + * @return The detected file type. arma::file_type_unknown if unknown. + */ +arma::file_type AutoDetect(std::fstream& stream, + const std::string filename); + +/** + * Return the type based only on the extension. + * + * @param filename Name of the file whose type we should detect. + * @param fatal If true, then if the format is not known, an exception will be + * thrown. + * @return Detected type of file. arma::file_type_unknown if unknown. + */ +arma::file_type DetectFromExtension(const std::string& filename); + +} // namespace data +} // namespace mlpack + +#endif diff --git a/src/mlpack/core/data/load_impl.hpp b/src/mlpack/core/data/load_impl.hpp index 9ce790de8f..344a8b6c13 100644 --- a/src/mlpack/core/data/load_impl.hpp +++ b/src/mlpack/core/data/load_impl.hpp @@ -21,6 +21,7 @@ #include "load_csv.hpp" #include "load.hpp" #include "extension.hpp" +#include "detect_file_type.hpp" #include #include @@ -84,250 +85,6 @@ bool inline inplace_transpose(MatType& X, bool fatal) } } -inline arma::file_type GuessFileType(std::istream& f) -{ - // Taken from armadillo's function guess_file_type_internal - f.clear(); - const std::fstream::pos_type pos1 = f.tellg(); - - f.clear(); - f.seekg(0, std::ios::end); - - f.clear(); - const std::fstream::pos_type pos2 = f.tellg(); // pos2 holds length of stream - - // Compute length of stream in N_max - const arma::uword N_max = ((pos1 >= 0) && (pos2 >= 0) && (pos2 > pos1)) ? - arma::uword(pos2 - pos1) : arma::uword(0); - - f.clear(); - f.seekg(pos1); - - // Handle empty files. - if (N_max == 0) - return arma::file_type_unknown; - - const arma::uword N_use = (std::min)(N_max, arma::uword(4096)); - - unsigned char* data_mem = (unsigned char*) - malloc(sizeof(unsigned char) * N_use); - memset(data_mem, 0, N_use); - - f.clear(); - f.read(reinterpret_cast(data_mem), std::streamsize(N_use)); - - const bool load_okay = f.good(); - - f.clear(); - f.seekg(pos1); - - if (load_okay == false) - { - delete data_mem; - return arma::file_type_unknown; - } - - bool has_binary = false; - bool has_bracket = false; - bool has_comma = false; - - for (arma::uword i = 0; i < N_use; ++i) - { - const unsigned char val = data_mem[i]; - if ((val <= 8) || (val >= 123)) - { - has_binary = true; - break; - } // the range checking can be made more elaborate - - if ((val == '(') || (val == ')')) - { - has_bracket = true; - } - if (val == ',') - { - has_comma = true; - } - } - - delete data_mem; - - if (has_binary) - return arma::raw_binary; - - if (has_comma && (has_bracket == false)) - return arma::csv_ascii; - - return arma::raw_ascii; -} - -inline -std::string AutoDetect(std::fstream& stream, - const std::string filename, - arma::file_type& detectedLoadType, - const bool fatal) -{ - // Get the extension. - std::string extension = Extension(filename); - std::string stringType = ""; - bool unknownType = false; - - if (extension == "csv" || extension == "tsv") - { - detectedLoadType = GuessFileType(stream); - if (detectedLoadType == arma::csv_ascii) - { - if (extension == "tsv") - Log::Warn << "'" << filename << "' is comma-separated, not " - "tab-separated!" << std::endl; - stringType = "CSV data"; - } - else if (detectedLoadType == arma::raw_ascii) // .csv file can be tsv. - { - if (extension == "csv") - { - // We should issue a warning, but we don't want to issue the warning if - // there is only one column in the CSV (since there will be no commas - // anyway, and it will be detected as arma::raw_ascii). - const std::streampos pos = stream.tellg(); - std::string line; - std::getline(stream, line, '\n'); - boost::trim(line); - - // Reset stream position. - stream.seekg(pos); - - // If there are no spaces or whitespace in the line, then we shouldn't - // print the warning. - if ((line.find(' ') != std::string::npos) || - (line.find('\t') != std::string::npos)) - { - Log::Warn << "'" << filename << "' is not a standard csv file." - << std::endl; - } - } - stringType = "raw ASCII formatted data"; - } - else - { - unknownType = true; - detectedLoadType = arma::raw_binary; // Won't be used; prevent a warning. - stringType = ""; - } - } - else if (extension == "txt") - { - // This could be raw ASCII or Armadillo ASCII (ASCII with size header). - // We'll let Armadillo do its guessing (although we have to check if it is - // arma_ascii ourselves) and see what we come up with. - - // This is taken from load_auto_detect() in diskio_meat.hpp - const std::string ARMA_MAT_TXT = "ARMA_MAT_TXT"; - // char* rawHeader = new char[ARMA_MAT_TXT.length() + 1]; - std::string rawHeader(ARMA_MAT_TXT.length(), '\0'); - std::streampos pos = stream.tellg(); - - stream.read(&rawHeader[0], std::streamsize(ARMA_MAT_TXT.length())); - stream.clear(); - stream.seekg(pos); // Reset stream position after peeking. - - if (rawHeader == ARMA_MAT_TXT) - { - detectedLoadType = arma::arma_ascii; - stringType = "Armadillo ASCII formatted data"; - } - else // It's not arma_ascii. Now we let Armadillo guess. - { -<<<<<<< HEAD - detectedLoadType = GuessFileType(stream); - - if (detectedLoadType == arma::raw_ascii) // Raw ASCII (space-separated). -======= -#if (ARMA_VERSION_MAJOR == 9 && ARMA_VERSION_MINOR >= 800) - loadType = arma::diskio::guess_file_type_internal(stream); -#else - loadType = arma::diskio::guess_file_type(stream); -#endif - if (loadType == arma::raw_ascii) // Raw ASCII (space-separated). ->>>>>>> origin/master - stringType = "raw ASCII formatted data"; - else if (detectedLoadType == arma::csv_ascii) // CSV can be .txt too. - stringType = "CSV data"; - else // Unknown .txt... we will throw an error. - unknownType = true; - } - } - else if (extension == "bin") - { - // This could be raw binary or Armadillo binary (binary with header). We - // will check to see if it is Armadillo binary. - const std::string ARMA_MAT_BIN = "ARMA_MAT_BIN"; - std::string rawHeader(ARMA_MAT_BIN.length(), '\0'); - - std::streampos pos = stream.tellg(); - - stream.read(&rawHeader[0], std::streamsize(ARMA_MAT_BIN.length())); - stream.clear(); - stream.seekg(pos); // Reset stream position after peeking. - - if (rawHeader == ARMA_MAT_BIN) - { - stringType = "Armadillo binary formatted data"; - detectedLoadType = arma::arma_binary; - } - else // We can only assume it's raw binary. - { - stringType = "raw binary formatted data"; - detectedLoadType = arma::raw_binary; - } - } - else if (extension == "pgm") - { - detectedLoadType = arma::pgm_binary; - stringType = "PGM data"; - } - else if (extension == "h5" || extension == "hdf5" || extension == "hdf" || - extension == "he5") - { -#ifdef ARMA_USE_HDF5 - detectedLoadType = arma::hdf5_binary; - stringType = "HDF5 data"; -#else - Timer::Stop("loading_data"); - if (fatal) - Log::Fatal << "Attempted to load '" << filename << "' as HDF5 data, but " - << "Armadillo was compiled without HDF5 support. Load failed." - << std::endl; - else - Log::Warn << "Attempted to load '" << filename << "' as HDF5 data, but " - << "Armadillo was compiled without HDF5 support. Load failed." - << std::endl; - - return ""; -#endif - } - else // Unknown extension... - { - unknownType = true; - detectedLoadType = arma::raw_binary; // Won't be used; prevent a warning. - stringType = ""; - } - - // Provide error if we don't know the type. - if (unknownType) - { - stringType = ""; - Timer::Stop("loading_data"); - if (fatal) - Log::Fatal << "Unable to detect type of '" << filename << "'; " - << "incorrect extension?" << std::endl; - else - Log::Warn << "Unable to detect type of '" << filename << "'; load failed." - << " Incorrect extension?" << std::endl; - } - return stringType; // Empty string denotes undetected file type. -} - template bool Load(const std::string& filename, arma::Mat& matrix, @@ -356,22 +113,46 @@ bool Load(const std::string& filename, return false; } - arma::file_type loadType; + arma::file_type loadType = inputLoadType; std::string stringType; if (inputLoadType == arma::auto_detect) { - stringType = AutoDetect(stream, filename, loadType, fatal); + // Attempt to auto-detect the type from the given file. + loadType = AutoDetect(stream, filename); + // Provide error if we don't know the type. + if (loadType == arma::file_type_unknown) + { + Timer::Stop("loading_data"); + if (fatal) + Log::Fatal << "Unable to detect type of '" << filename << "'; " + << "incorrect extension?" << std::endl; + else + Log::Warn << "Unable to detect type of '" << filename << "'; load " + << " failed. Incorrect extension?" << std::endl; + + return false; + } } - else - { - loadType = inputLoadType; - stringType = GetStringType(loadType); - } - // If file type is not detected, return failure for load. - if (stringType == "") + + stringType = GetStringType(loadType); + +#ifndef ARMA_USE_HDF5 + if (inputLoadType == arma::hdf5_binary) { + // Ensure that HDF5 is supported. + Timer::Stop("loading_data"); + if (fatal) + Log::Fatal << "Attempted to load '" << filename << "' as HDF5 data, but " + << "Armadillo was compiled without HDF5 support. Load failed." + << std::endl; + else + Log::Warn << "Attempted to load '" << filename << "' as HDF5 data, but " + << "Armadillo was compiled without HDF5 support. Load failed." + << std::endl; + return false; } +#endif // Try to load the file; but if it's raw_binary, it could be a problem. if (loadType == arma::raw_binary) diff --git a/src/mlpack/core/data/save_impl.hpp b/src/mlpack/core/data/save_impl.hpp index c55e10fc49..f0d37fc14f 100644 --- a/src/mlpack/core/data/save_impl.hpp +++ b/src/mlpack/core/data/save_impl.hpp @@ -15,6 +15,7 @@ // In case it hasn't already been included. #include "save.hpp" #include "extension.hpp" +#include "detect_file_type.hpp" #include #include @@ -27,125 +28,20 @@ namespace data { template bool Save(const std::string& filename, const arma::Col& vec, - const bool fatal) + const bool fatal, + arma::file_type inputSaveType) { // Don't transpose: one observation per line (for CSVs at least). - return Save(filename, vec, fatal, false); + return Save(filename, vec, fatal, false, inputSaveType); } template bool Save(const std::string& filename, const arma::Row& rowvec, - const bool fatal) + const bool fatal, + arma::file_type inputSaveType) { - return Save(filename, rowvec, fatal, true); -} - -inline -std::string GetStringType(const arma::file_type& loadType) -{ - switch (loadType) - { - case arma::csv_ascii : return "CSV data"; - case arma::raw_ascii : return "raw ASCII formatted data"; - case arma::raw_binary : return "raw binary formatted data"; - case arma::arma_ascii : return "Armadillo ASCII formatted data"; - case arma::arma_binary : return "Armadillo binary formatted data"; - case arma::pgm_binary : return "PGM data"; - case arma::hdf5_binary : - { - #ifdef ARMA_USE_HDF5 - return "HDF5 data"; - #else - return ""; - #endif - } - default : return ""; - } -} - -inline -std::string AutoDetect(const std::string& filename, - arma::file_type& detectedSaveType, - const bool fatal) -{ - // First we will try to discriminate by file extension. - std::string extension = Extension(filename); - if (extension == "") - { - Timer::Stop("saving_data"); - if (fatal) - Log::Fatal << "No extension given with filename '" << filename << "'; " - << "type unknown. Save failed." << std::endl; - else - Log::Warn << "No extension given with filename '" << filename << "'; " - << "type unknown. Save failed." << std::endl; - - return ""; - } - - std::string stringType; - bool unknownType = false; - - if (extension == "csv") - { - detectedSaveType = arma::csv_ascii; - stringType = "CSV data"; - } - else if (extension == "txt") - { - detectedSaveType = arma::raw_ascii; - stringType = "raw ASCII formatted data"; - } - else if (extension == "bin") - { - detectedSaveType = arma::arma_binary; - stringType = "Armadillo binary formatted data"; - } - else if (extension == "pgm") - { - detectedSaveType = arma::pgm_binary; - stringType = "PGM data"; - } - else if (extension == "h5" || extension == "hdf5" || extension == "hdf" || - extension == "he5") - { -#ifdef ARMA_USE_HDF5 - detectedSaveType = arma::hdf5_binary; - stringType = "HDF5 data"; -#else - Timer::Stop("saving_data"); - if (fatal) - Log::Fatal << "Attempted to save HDF5 data to '" << filename << "', but " - << "Armadillo was compiled without HDF5 support. Save failed." - << std::endl; - else - Log::Warn << "Attempted to save HDF5 data to '" << filename << "', but " - << "Armadillo was compiled without HDF5 support. Save failed." - << std::endl; - - return ""; -#endif - } - else - { - unknownType = true; - detectedSaveType = arma::raw_binary; // Won't be used; prevent a warning. - stringType = ""; - } - - // Provide error if we don't know the type. - if (unknownType) - { - Timer::Stop("saving_data"); - if (fatal) - Log::Fatal << "Unable to determine format to save to from filename '" - << filename << "'. Save failed." << std::endl; - else - Log::Warn << "Unable to determine format to save to from filename '" - << filename << "'. Save failed." << std::endl; - } - return stringType; + return Save(filename, rowvec, fatal, true, inputSaveType); } template @@ -157,24 +53,28 @@ bool Save(const std::string& filename, { Timer::Start("saving_data"); - arma::file_type saveType; + arma::file_type saveType = inputSaveType; std::string stringType = ""; if (inputSaveType == arma::auto_detect) { - stringType = AutoDetect(filename, saveType, fatal); - } - else - { - stringType = GetStringType(saveType); - } - // If File Type is not automatically detected from extension or no extension - // is specified then return failure. - if (stringType == "") - { - return false; + // Detect the file type using only the extension. + saveType = DetectFromExtension(filename); + if (saveType == arma::file_type_unknown) + { + if (fatal) + Log::Fatal << "Could not detect type of file '" << filename << "' for " + << "writing. Save failed." << std::endl; + else + Log::Warn << "Could not detect type of file '" << filename << "' for " + << "writing. Save failed." << std::endl; + + return false; + } } + stringType = GetStringType(saveType); + // Catch errors opening the file. std::fstream stream; #ifdef _WIN32 // Always open in binary mode on Windows. @@ -195,7 +95,6 @@ bool Save(const std::string& filename, return false; } - // Try to save the file. Log::Info << "Saving " << stringType << " to '" << filename << "'." << std::endl; From f95d0f078a92af0d9fbd5f8673c0b04b449de364 Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Thu, 3 Sep 2020 11:56:55 -0400 Subject: [PATCH 32/55] Add tests for manually specified file format. --- src/mlpack/tests/load_save_test.cpp | 72 ++++++++++++++++++++++++----- 1 file changed, 61 insertions(+), 11 deletions(-) diff --git a/src/mlpack/tests/load_save_test.cpp b/src/mlpack/tests/load_save_test.cpp index e32042aaf2..66bba5ccea 100644 --- a/src/mlpack/tests/load_save_test.cpp +++ b/src/mlpack/tests/load_save_test.cpp @@ -51,7 +51,7 @@ TEST_CASE("NotExistLoad", "[LoadSaveTest]") /** * Make sure load fails if the file extension is wrong in automatic detection mode. */ -BOOST_AUTO_TEST_CASE(WrongExtensionWrongLoad) +TEST_CASE("WrongExtensionWrongLoad", "[LoadSaveTest]") { // Try to load arma::arma_binary file with ".csv" extension arma::mat test = "1 5;" @@ -60,11 +60,10 @@ BOOST_AUTO_TEST_CASE(WrongExtensionWrongLoad) "4 8;"; arma::mat testTrans = trans(test); - BOOST_REQUIRE(testTrans.quiet_save("test_file.csv", arma::arma_binary) - == true); + REQUIRE(testTrans.quiet_save("test_file.csv", arma::arma_binary) == true); // Now reload through our interface. - BOOST_REQUIRE(data::Load("test_file.csv", test) == false); + REQUIRE(data::Load("test_file.csv", test) == false); // Remove the file. remove("test_file.csv"); @@ -73,7 +72,7 @@ BOOST_AUTO_TEST_CASE(WrongExtensionWrongLoad) /** * Make sure load is successful even if the file extension is wrong when file type is specified. */ -BOOST_AUTO_TEST_CASE(WrongExtensionCorrectLoad) +TEST_CASE("WrongExtensionCorrectLoad", "[LoadSaveTest]") { // Try to load arma::arma_binary file with ".csv" extension arma::mat test = "1 5;" @@ -82,19 +81,18 @@ BOOST_AUTO_TEST_CASE(WrongExtensionCorrectLoad) "4 8;"; arma::mat testTrans = trans(test); - BOOST_REQUIRE(testTrans.quiet_save("test_file.csv", arma::arma_binary) - == true); + REQUIRE(testTrans.quiet_save("test_file.csv", arma::arma_binary) == true); // Now reload through our interface. - BOOST_REQUIRE( + REQUIRE( data::Load("test_file.csv", test, false, true, arma::arma_binary) == true); - BOOST_REQUIRE_EQUAL(test.n_rows, 4); - BOOST_REQUIRE_EQUAL(test.n_cols, 2); + REQUIRE(test.n_rows == 4); + REQUIRE(test.n_cols == 2); for (size_t i = 0; i < 8; i++) - BOOST_REQUIRE_CLOSE(test[i], (double) (i + 1), 1e-5); + REQUIRE(test[i] == Approx((double) (i + 1)).epsilon(1e-3)); // Remove the file. remove("test_file.csv"); @@ -257,6 +255,32 @@ TEST_CASE("LoadTSVExtensionTest", "[LoadSaveTest]") remove("test_file.tsv"); } +/** + * Test that we can manually specify the format for loading. + */ +TEST_CASE("LoadAnyExtensionFileTest", "[LoadSaveTest]") +{ + fstream f; + f.open("test_file.blah", fstream::out); + + f << "1\t2\t3\t4" << endl; + f << "5\t6\t7\t8" << endl; + + f.close(); + + arma::mat test; + REQUIRE(data::Load("test_file.blah", test, false, true, arma::raw_ascii)); + + REQUIRE(test.n_rows == 4); + REQUIRE(test.n_cols == 2); + + for (size_t i = 0; i < 8; ++i) + REQUIRE(test[i] == Approx((double) (i + 1)).epsilon(1e-7)); + + // Remove the file. + remove("test_file.blah"); +} + /** * Make sure a CSV is saved correctly. */ @@ -944,6 +968,32 @@ TEST_CASE("SaveArmaBinaryTest", "[LoadSaveTest]") remove("test_file.bin"); } +/** + * Make sure that we can manually specify the format. + */ +TEST_CASE("SaveArmaBinaryArbitraryExtensionTest", "[LoadSaveTest]") +{ + arma::mat test = "1 5;" + "2 6;" + "3 7;" + "4 8;"; + + REQUIRE(data::Save("test_file.blerp.blah", test, false, true, + arma::arma_binary) == true); + + REQUIRE(data::Load("test_file.blerp.blah", test, false, true, + arma::arma_binary) == true); + + REQUIRE(test.n_rows == 4); + REQUIRE(test.n_cols == 2); + + for (size_t i = 0; i < 8; ++i) + REQUIRE(test[i] == Approx((double) (i + 1)).epsilon(1e-7)); + + // Remove the file. + remove("test_file.blerp.blah"); +} + /** * Make sure raw_binary is loaded correctly. */ From 3ca76032c8b4b57c4e27e1067622745bb06ccd7a Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Thu, 3 Sep 2020 12:07:14 -0400 Subject: [PATCH 33/55] Update documentation. --- src/mlpack/core/data/load.hpp | 28 +++++++++++++++------------- src/mlpack/core/data/save.hpp | 18 ++++++++++++------ 2 files changed, 27 insertions(+), 19 deletions(-) diff --git a/src/mlpack/core/data/load.hpp b/src/mlpack/core/data/load.hpp index 50184e594d..d5da43a1ff 100644 --- a/src/mlpack/core/data/load.hpp +++ b/src/mlpack/core/data/load.hpp @@ -28,23 +28,25 @@ namespace data /** Functions to load and save matrices and models. */ { /** * Loads a matrix from file, guessing the filetype from the extension. This * will transpose the matrix at load time (unless the transpose parameter is set - * to false). If the filetype cannot be determined, an error will be given. + * to false). * * The supported types of files are the same as found in Armadillo: * - * - CSV (csv_ascii), denoted by .csv, or optionally .txt - * - TSV (raw_ascii), denoted by .tsv, .csv, or .txt - * - ASCII (raw_ascii), denoted by .txt - * - Armadillo ASCII (arma_ascii), also denoted by .txt - * - PGM (pgm_binary), denoted by .pgm - * - PPM (ppm_binary), denoted by .ppm - * - Raw binary (raw_binary), denoted by .bin - * - Armadillo binary (arma_binary), denoted by .bin - * - HDF5, denoted by .hdf, .hdf5, .h5, or .he5 + * - CSV (arma::csv_ascii), denoted by .csv, or optionally .txt + * - TSV (arma::raw_ascii), denoted by .tsv, .csv, or .txt + * - ASCII (arma::raw_ascii), denoted by .txt + * - Armadillo ASCII (arma::arma_ascii), also denoted by .txt + * - PGM (arma::pgm_binary), denoted by .pgm + * - PPM (arma::ppm_binary), denoted by .ppm + * - Raw binary (arma::raw_binary), denoted by .bin + * - Armadillo binary (arma::arma_binary), denoted by .bin + * - HDF5 (arma::hdf5_binary), denoted by .hdf, .hdf5, .h5, or .he5 * - * If the file extension is not one of those types, an error will be given. - * This is preferable to Armadillo's default behavior of loading an unknown - * filetype as raw_binary, which can have very confusing effects. + * By default, this function will try to automatically determine the type of + * file to load based on its extension and by inspecting the file. If you know + * the file type and want to specify it manually, override the default + * `inputLoadType` parameter with the correct type above (e.g. + * `arma::csv_ascii`.) * * If the parameter 'fatal' is set to true, a std::runtime_error exception will * be thrown if the matrix does not load successfully. The parameter diff --git a/src/mlpack/core/data/save.hpp b/src/mlpack/core/data/save.hpp index f09473879a..49ddd51e1d 100644 --- a/src/mlpack/core/data/save.hpp +++ b/src/mlpack/core/data/save.hpp @@ -40,17 +40,23 @@ namespace data /** Functions to load and save matrices. */ { * - Armadillo binary (arma_binary), denoted by .bin * - HDF5 (hdf5_binary), denoted by .hdf5, .hdf, .h5, or .he5 * - * If the file extension is not one of those types, an error will be given. If - * the 'fatal' parameter is set to true, a std::runtime_error exception will be - * thrown upon failure. If the 'transpose' parameter is set to true, the matrix - * will be transposed before saving. Generally, because mlpack stores matrices - * in a column-major format and most datasets are stored on disk as row-major, - * this parameter should be left at its default value of 'true'. + * By default, this function will try to automatically determine the format to + * save with based only on the filename's extension. If you would prefer to + * specify a file type manually, override the default + * `inputSaveType` parameter with the correct type above (e.g. + * `arma::csv_ascii`.) + * + * If the 'fatal' parameter is set to true, a std::runtime_error exception will + * be thrown upon failure. If the 'transpose' parameter is set to true, the + * matrix will be transposed before saving. Generally, because mlpack stores + * matrices in a column-major format and most datasets are stored on disk as + * row-major, this parameter should be left at its default value of 'true'. * * @param filename Name of file to save to. * @param matrix Matrix to save into file. * @param fatal If an error should be reported as fatal (default false). * @param transpose If true, transpose the matrix before saving (default true). + * @param inputSaveType File type to save to (defaults to arma::auto_detect). * @return Boolean value indicating success or failure of save. */ template From 4b8349889f39a2658292a8277298b21d8497fd39 Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Thu, 3 Sep 2020 12:07:39 -0400 Subject: [PATCH 34/55] Explicitly specify type namespace. --- src/mlpack/core/data/save.hpp | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/mlpack/core/data/save.hpp b/src/mlpack/core/data/save.hpp index 49ddd51e1d..cf8d78f3e1 100644 --- a/src/mlpack/core/data/save.hpp +++ b/src/mlpack/core/data/save.hpp @@ -31,14 +31,14 @@ namespace data /** Functions to load and save matrices. */ { * * The supported types of files are the same as found in Armadillo: * - * - CSV (csv_ascii), denoted by .csv, or optionally .txt - * - ASCII (raw_ascii), denoted by .txt - * - Armadillo ASCII (arma_ascii), also denoted by .txt - * - PGM (pgm_binary), denoted by .pgm - * - PPM (ppm_binary), denoted by .ppm - * - Raw binary (raw_binary), denoted by .bin - * - Armadillo binary (arma_binary), denoted by .bin - * - HDF5 (hdf5_binary), denoted by .hdf5, .hdf, .h5, or .he5 + * - CSV (arma::csv_ascii), denoted by .csv, or optionally .txt + * - ASCII (arma::raw_ascii), denoted by .txt + * - Armadillo ASCII (arma::arma_ascii), also denoted by .txt + * - PGM (arma::pgm_binary), denoted by .pgm + * - PPM (arma::ppm_binary), denoted by .ppm + * - Raw binary (arma::raw_binary), denoted by .bin + * - Armadillo binary (arma::arma_binary), denoted by .bin + * - HDF5 (arma::hdf5_binary), denoted by .hdf5, .hdf, .h5, or .he5 * * By default, this function will try to automatically determine the format to * save with based only on the filename's extension. If you would prefer to From ad0f0e30e1ad8c1ed5dc05c686ecd69f43d83218 Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Thu, 3 Sep 2020 12:10:53 -0400 Subject: [PATCH 35/55] Remove accidentally documented parameter. --- src/mlpack/core/data/detect_file_type.hpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/mlpack/core/data/detect_file_type.hpp b/src/mlpack/core/data/detect_file_type.hpp index 8c99c8c1ab..c262699490 100644 --- a/src/mlpack/core/data/detect_file_type.hpp +++ b/src/mlpack/core/data/detect_file_type.hpp @@ -51,8 +51,6 @@ arma::file_type AutoDetect(std::fstream& stream, * Return the type based only on the extension. * * @param filename Name of the file whose type we should detect. - * @param fatal If true, then if the format is not known, an exception will be - * thrown. * @return Detected type of file. arma::file_type_unknown if unknown. */ arma::file_type DetectFromExtension(const std::string& filename); From ff72427833d45baba8012de822631c008574d786 Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Thu, 3 Sep 2020 12:11:34 -0400 Subject: [PATCH 36/55] Update HISTORY. --- HISTORY.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/HISTORY.md b/HISTORY.md index 8f6812bf3a..0583eec9ac 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,6 +1,9 @@ ### mlpack ?.?.? ###### ????-??-?? - * Add manual type specification support to `data::Load()` and `data::Save()` (#2084). + * Add manual type specification support to `data::Load()` and `data::Save()` + (#2084, #2135, #2602). + + * Remove use of internal Armadillo functionality (#2596, #2601, #2602). ### mlpack 3.4.0 ###### 2020-09-01 From 0cab9b0dd005a71df52da5516383a53204fe4e62 Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Thu, 3 Sep 2020 14:12:11 -0400 Subject: [PATCH 37/55] Fix static code analysis issues. --- src/mlpack/core/data/detect_file_type.cpp | 5 ++--- src/mlpack/core/data/detect_file_type.hpp | 2 +- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/src/mlpack/core/data/detect_file_type.cpp b/src/mlpack/core/data/detect_file_type.cpp index 520f9bec1d..1ade1340aa 100644 --- a/src/mlpack/core/data/detect_file_type.cpp +++ b/src/mlpack/core/data/detect_file_type.cpp @@ -74,8 +74,7 @@ arma::file_type GuessFileType(std::istream& f) const arma::uword nUse = std::min(nMax, arma::uword(4096)); - unsigned char* dataMem = - (unsigned char*) malloc(sizeof(unsigned char) *nUse); + unsigned char* dataMem = new unsigned char[nUse]; memset(dataMem, 0, nUse); f.clear(); @@ -137,7 +136,7 @@ arma::file_type GuessFileType(std::istream& f) * @return The detected file type. */ arma::file_type AutoDetect(std::fstream& stream, - const std::string filename) + const std::string& filename) { // Get the extension. std::string extension = Extension(filename); diff --git a/src/mlpack/core/data/detect_file_type.hpp b/src/mlpack/core/data/detect_file_type.hpp index c262699490..5c3989539d 100644 --- a/src/mlpack/core/data/detect_file_type.hpp +++ b/src/mlpack/core/data/detect_file_type.hpp @@ -45,7 +45,7 @@ arma::file_type GuessFileType(std::istream& f); * @return The detected file type. arma::file_type_unknown if unknown. */ arma::file_type AutoDetect(std::fstream& stream, - const std::string filename); + const std::string& filename); /** * Return the type based only on the extension. From bb4df4d7c97db0d5403b799a7ec64fbbe93cc400 Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Thu, 3 Sep 2020 17:10:14 -0400 Subject: [PATCH 38/55] Oops, use delete[]. --- src/mlpack/core/data/detect_file_type.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/mlpack/core/data/detect_file_type.cpp b/src/mlpack/core/data/detect_file_type.cpp index 1ade1340aa..4344bb5299 100644 --- a/src/mlpack/core/data/detect_file_type.cpp +++ b/src/mlpack/core/data/detect_file_type.cpp @@ -87,7 +87,7 @@ arma::file_type GuessFileType(std::istream& f) if (!loadOkay) { - delete dataMem; + delete[] dataMem; return arma::file_type_unknown; } @@ -114,7 +114,7 @@ arma::file_type GuessFileType(std::istream& f) } } - delete dataMem; + delete[] dataMem; if (hasBinary) return arma::raw_binary; From cf5cb4ced43bac8d8e578beef3eccc00d5e94504 Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Thu, 3 Sep 2020 23:00:57 -0400 Subject: [PATCH 39/55] Update HISTORY.md Co-authored-by: James J Balamuta --- HISTORY.md | 1 - 1 file changed, 1 deletion(-) diff --git a/HISTORY.md b/HISTORY.md index 0583eec9ac..7943e77272 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -137,7 +137,6 @@ * The DecisionStump class has been marked deprecated; use the `DecisionTree` class with `NoRecursion=true` or use `ID3DecisionStump` instead (#2099). ->>>>>>> origin/master * Added `probabilities_file` parameter to get the probabilities matrix of AdaBoost classifier (#2050). From d5e2cd3975a93477bf7b62cf0837db8c8e592b2d Mon Sep 17 00:00:00 2001 From: 1sarthakbhadwaj <7sarthakbhardwaj@gmail.com> Date: Fri, 4 Sep 2020 23:54:06 +0530 Subject: [PATCH 40/55] upgrading the configuration variables --- CMakeLists.txt | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index a1f1a45937..62e89f7be5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -430,7 +430,9 @@ set(Boost_ADDITIONAL_VERSIONS "1.61.1" "1.61.0" "1.61" "1.60.1" "1.60.0" "1.60" "1.59.1" "1.59.0" "1.59" - "1.58.1" "1.58.0" "1.58") + "1.58.1" "1.58.0" "1.58" + "1.74.0" "1.73" + "17.4.0" "17.4" ) # Disable forced config-mode CMake search for Boost, which only imports targets # and does not set the variables that we need. # From 7b77d5fc61499161cafc6d57b2422e58ebe1a591 Mon Sep 17 00:00:00 2001 From: 1sarthakbhadwaj <7sarthakbhardwaj@gmail.com> Date: Sat, 5 Sep 2020 16:11:20 +0530 Subject: [PATCH 41/55] upgrading the configuration variable --- CMakeLists.txt | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 62e89f7be5..fc7a2fd764 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -416,6 +416,9 @@ endif () # Unfortunately this configuration variable is necessary and will need to be # updated as time goes on and new versions are released. set(Boost_ADDITIONAL_VERSIONS + + "1.74.0" "1.73" + "17.4.0" "17.4" "1.72.0" "1.72" "1.71.0" "1.71" "1.70.0" "1.70" @@ -430,9 +433,7 @@ set(Boost_ADDITIONAL_VERSIONS "1.61.1" "1.61.0" "1.61" "1.60.1" "1.60.0" "1.60" "1.59.1" "1.59.0" "1.59" - "1.58.1" "1.58.0" "1.58" - "1.74.0" "1.73" - "17.4.0" "17.4" ) + "1.58.1" "1.58.0" "1.58") # Disable forced config-mode CMake search for Boost, which only imports targets # and does not set the variables that we need. # From f6a5c871e5853bac74201b033914c3232946a193 Mon Sep 17 00:00:00 2001 From: Sarthak Bhardwaj <7sarthakbhardwaj@gmail.com> Date: Sat, 5 Sep 2020 16:34:48 +0530 Subject: [PATCH 42/55] Update CMakeLists.txt Co-authored-by: Marcus Edel --- CMakeLists.txt | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index fc7a2fd764..f041e321e6 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -416,9 +416,8 @@ endif () # Unfortunately this configuration variable is necessary and will need to be # updated as time goes on and new versions are released. set(Boost_ADDITIONAL_VERSIONS - - "1.74.0" "1.73" - "17.4.0" "17.4" + "1.74.0" "1.74" + "17.3.0" "17.3" "1.72.0" "1.72" "1.71.0" "1.71" "1.70.0" "1.70" From b7b36787b4ea517f55220f2208315bf866bb2ef1 Mon Sep 17 00:00:00 2001 From: jeffin143 Date: Sat, 29 Aug 2020 20:16:09 +0530 Subject: [PATCH 43/55] migrate _network_* test --- src/mlpack/tests/CMakeLists.txt | 8 +-- src/mlpack/tests/feedforward_network_test.cpp | 41 +++++------ src/mlpack/tests/rbm_network_test.cpp | 23 +++--- src/mlpack/tests/recurrent_network_test.cpp | 71 +++++++++---------- 4 files changed, 63 insertions(+), 80 deletions(-) diff --git a/src/mlpack/tests/CMakeLists.txt b/src/mlpack/tests/CMakeLists.txt index 14f716245e..224a14c215 100644 --- a/src/mlpack/tests/CMakeLists.txt +++ b/src/mlpack/tests/CMakeLists.txt @@ -16,7 +16,6 @@ add_executable(mlpack_test emst_test.cpp fastmks_test.cpp facilities_test.cpp - feedforward_network_test.cpp gan_test.cpp gmm_test.cpp hmm_test.cpp @@ -58,9 +57,7 @@ add_executable(mlpack_test random_forest_test.cpp random_test.cpp range_search_test.cpp - rbm_network_test.cpp rectangle_tree_test.cpp - recurrent_network_test.cpp reward_clipping_test.cpp rl_components_test.cpp serialization.cpp @@ -133,6 +130,7 @@ add_executable(mlpack_catch_test cv_test.cpp decision_stump_test.cpp decision_tree_test.cpp + feedforward_network_test.cpp image_load_test.cpp imputation_test.cpp kernel_pca_test.cpp @@ -148,6 +146,8 @@ add_executable(mlpack_catch_test one_hot_encoding_test.cpp quic_svd_test.cpp randomized_svd_test.cpp + rbm_network_test.cpp + recurrent_network_test.cpp regularized_svd_test.cpp scaling_test.cpp serialization_catch.cpp @@ -229,8 +229,6 @@ add_custom_command(TARGET mlpack_test set(parallel_tests "AsyncLearningTest;" "LocalCoordinateCodingTest;" - "FeedForwardNetworkTest;" - "RecurrentNetworkTest;" "GMMTest;" "CFTest;" "HMMTest;" diff --git a/src/mlpack/tests/feedforward_network_test.cpp b/src/mlpack/tests/feedforward_network_test.cpp index 47842204e6..f9a338e0b3 100644 --- a/src/mlpack/tests/feedforward_network_test.cpp +++ b/src/mlpack/tests/feedforward_network_test.cpp @@ -19,17 +19,14 @@ #include -#include -#include "test_tools.hpp" -#include "serialization.hpp" +#include "catch.hpp" +#include "serialization_catch.hpp" #include "custom_layer.hpp" using namespace mlpack; using namespace mlpack::ann; using namespace mlpack::kmeans; -BOOST_AUTO_TEST_SUITE(FeedForwardNetworkTest); - /** * Train and evaluate a model with the specified structure. */ @@ -57,13 +54,13 @@ void TestNetwork(ModelType& model, size_t correct = arma::accu(prediction == testLabels); double classificationError = 1 - double(correct) / testData.n_cols; - BOOST_REQUIRE_LE(classificationError, classificationErrorThreshold); + REQUIRE(classificationError <= classificationErrorThreshold); } /** * Train the vanilla network on a larger dataset. */ -BOOST_AUTO_TEST_CASE(VanillaNetworkTest) +TEST_CASE("FFVanillaNetworkTest", "[FeedForwardNetworkTest]") { // Load the dataset. arma::mat trainData; @@ -131,7 +128,7 @@ BOOST_AUTO_TEST_CASE(VanillaNetworkTest) TestNetwork<>(model1, dataset, labels, dataset, labels, 10, 0.2); } -BOOST_AUTO_TEST_CASE(ForwardBackwardTest) +TEST_CASE("ForwardBackwardTest", "[FeedForwardNetworkTest]") { arma::mat dataset; dataset.load("mnist_first250_training_4s_and_9s.arm"); @@ -204,13 +201,13 @@ BOOST_AUTO_TEST_CASE(ForwardBackwardTest) } } - BOOST_REQUIRE(converged); + REQUIRE(converged); } /** * Train the dropout network on a larger dataset. */ -BOOST_AUTO_TEST_CASE(DropoutNetworkTest) +TEST_CASE("DropoutNetworkTest", "[FeedForwardNetworkTest]") { // Load the dataset. arma::mat trainData; @@ -284,7 +281,7 @@ BOOST_AUTO_TEST_CASE(DropoutNetworkTest) /** * Train the highway network on a larger dataset. */ -BOOST_AUTO_TEST_CASE(HighwayNetworkTest) +TEST_CASE("HighwayNetworkTest", "[FeedForwardNetworkTest]") { arma::mat dataset; dataset.load("mnist_first250_training_4s_and_9s.arm"); @@ -311,7 +308,7 @@ BOOST_AUTO_TEST_CASE(HighwayNetworkTest) /** * Train the DropConnect network on a larger dataset. */ -BOOST_AUTO_TEST_CASE(DropConnectNetworkTest) +TEST_CASE("DropConnectNetworkTest", "[FeedForwardNetworkTest]") { // Load the dataset. arma::mat trainData; @@ -385,7 +382,7 @@ BOOST_AUTO_TEST_CASE(DropConnectNetworkTest) * Test miscellaneous things of FFN, * e.g. copy/move constructor, assignment operator. */ -BOOST_AUTO_TEST_CASE(FFNMiscTest) +TEST_CASE("FFNMiscTest", "[FeedForwardNetworkTest]") { FFN> model; model.Add>(2, 3); @@ -400,7 +397,7 @@ BOOST_AUTO_TEST_CASE(FFNMiscTest) /** * Test that serialization works ok. */ -BOOST_AUTO_TEST_CASE(SerializationTest) +TEST_CASE("FFSerializationTest", "[FeedForwardNetworkTest]") { // Load the dataset. arma::mat trainData; @@ -449,7 +446,7 @@ BOOST_AUTO_TEST_CASE(SerializationTest) * Test if the custom layers work. The target is to see if the code compiles * when the Train and Prediction are called. */ -BOOST_AUTO_TEST_CASE(CustomLayerTest) +TEST_CASE("CustomLayerTest", "[FeedForwardNetworkTest]") { // Load the dataset. arma::mat trainData; @@ -481,7 +478,7 @@ BOOST_AUTO_TEST_CASE(CustomLayerTest) /** * Test the overload of Forward function which allows partial forward pass. */ -BOOST_AUTO_TEST_CASE(PartialForwardTest) +TEST_CASE("PartialForwardTest", "[FeedForwardNetworkTest]") { FFN, RandomInitialization> model; model.Add >(5, 10); @@ -528,7 +525,7 @@ BOOST_AUTO_TEST_CASE(PartialForwardTest) /** * Test that FFN::Train() returns finite objective value. */ -BOOST_AUTO_TEST_CASE(FFNTrainReturnObjective) +TEST_CASE("FFNTrainReturnObjective", "[FeedForwardNetworkTest]") { // Load the dataset. arma::mat trainData; @@ -557,13 +554,13 @@ BOOST_AUTO_TEST_CASE(FFNTrainReturnObjective) double objVal = model.Train(trainData, trainLabels, opt); - BOOST_REQUIRE_EQUAL(std::isfinite(objVal), true); + REQUIRE(std::isfinite(objVal) == true); } /** * Test that FFN::Model() allows us to access the instantiated network. */ -BOOST_AUTO_TEST_CASE(FFNReturnModel) +TEST_CASE("FFNReturnModel", "[FeedForwardNetworkTest]") { // Create dummy network. FFN > model; @@ -598,7 +595,7 @@ BOOST_AUTO_TEST_CASE(FFNReturnModel) * Test to see if the FFN code compiles when the Optimizer * doesn't have the MaxIterations() method. */ -BOOST_AUTO_TEST_CASE(OptimizerTest) +TEST_CASE("OptimizerTest", "[FeedForwardNetworkTest]") { // Load the dataset. arma::mat trainData; @@ -626,7 +623,7 @@ BOOST_AUTO_TEST_CASE(OptimizerTest) /** * Train the RBF network on a larger dataset. */ -BOOST_AUTO_TEST_CASE(RBFNetworkTest) +TEST_CASE("RBFNetworkTest", "[FeedForwardNetworkTest]") { // Load the dataset. arma::mat trainData; @@ -703,5 +700,3 @@ BOOST_AUTO_TEST_CASE(RBFNetworkTest) // RBFN neural net with MeanSquaredError. TestNetwork<>(model1, dataset, labels1, dataset, labels, 10, 0.1); } - -BOOST_AUTO_TEST_SUITE_END(); diff --git a/src/mlpack/tests/rbm_network_test.cpp b/src/mlpack/tests/rbm_network_test.cpp index 820f9616f5..f14e2726bd 100644 --- a/src/mlpack/tests/rbm_network_test.cpp +++ b/src/mlpack/tests/rbm_network_test.cpp @@ -26,20 +26,17 @@ #include #include -#include -#include "test_tools.hpp" +#include "catch.hpp" using namespace mlpack; using namespace mlpack::ann; using namespace ens; using namespace mlpack::regression; -BOOST_AUTO_TEST_SUITE(RBMNetworkTest); - /* * Tests the BinaryRBM implementation on the Digits dataset. */ -BOOST_AUTO_TEST_CASE(BinaryRBMClassificationTest) +TEST_CASE("BinaryRBMClassificationTest", "[RBMNetworkTest]") { // Normalised dataset. int hiddenLayerSize = 100; @@ -84,7 +81,7 @@ BOOST_AUTO_TEST_CASE(BinaryRBMClassificationTest) double objVal = model.Train(msgd); // Test that objective value returned by RBM::Train() is finite. - BOOST_REQUIRE_EQUAL(std::isfinite(objVal), true); + REQUIRE(std::isfinite(objVal) == true); for (size_t i = 0; i < trainData.n_cols; ++i) { @@ -117,13 +114,13 @@ BOOST_AUTO_TEST_CASE(BinaryRBMClassificationTest) // We allow a 6% tolerance because the RBM may not reconstruct samples as // well. (Typically it does, but we have no guarantee.) - BOOST_REQUIRE_GE(rbmClassificationAccuracy, classificationAccuracy - 6.0); + REQUIRE(rbmClassificationAccuracy >= classificationAccuracy - 6.0); } /* * Tests the SpikeSlabRBM implementation on the Digits dataset. */ -BOOST_AUTO_TEST_CASE(ssRBMClassificationTest) +TEST_CASE("ssRBMClassificationTest", "[RBMNetworkTest]") { size_t batchSize = 10; size_t numEpoches = 3; @@ -184,7 +181,7 @@ BOOST_AUTO_TEST_CASE(ssRBMClassificationTest) double objVal = modelssRBM.Train(msgd); // Test that objective value returned by RBM::Train() is finite. - BOOST_REQUIRE_EQUAL(std::isfinite(objVal), true); + REQUIRE(std::isfinite(objVal) == true); for (size_t i = 0; i < trainData.n_cols; ++i) { @@ -211,7 +208,7 @@ BOOST_AUTO_TEST_CASE(ssRBMClassificationTest) // omitted here for speed. We add a margin of 3% since ssRBM isn't guaranteed // to give us better results (we just generally expect it to be about as good // or better). - BOOST_REQUIRE_GE(ssRbmClassificationAccuracy, 76.18 - 3.0); + REQUIRE(ssRbmClassificationAccuracy >= 76.18 - 3.0); } template @@ -239,13 +236,13 @@ void BuildVanillaNetwork(MatType& trainData, } for (size_t i = 0; i < freeEnergy.n_elem; ++i) - BOOST_REQUIRE_CLOSE(calculatedFreeEnergy(i), freeEnergy(i), 1e-3); + REQUIRE(calculatedFreeEnergy(i) == Approx(freeEnergy(i)).epsilon(1e-5)); } /* * Train and evaluate a Vanilla network with the specified structure. */ -BOOST_AUTO_TEST_CASE(MiscTest) +TEST_CASE("MiscTest", "[RBMNetworkTest]") { arma::Mat X = arma::Mat("0.0, 0.0, 0.0;" "0.0, 1.0, 1.0;" @@ -254,5 +251,3 @@ BOOST_AUTO_TEST_CASE(MiscTest) X = X.t(); BuildVanillaNetwork>(X, 2); } - -BOOST_AUTO_TEST_SUITE_END(); diff --git a/src/mlpack/tests/recurrent_network_test.cpp b/src/mlpack/tests/recurrent_network_test.cpp index 10635e80fb..5bedcbd3cc 100644 --- a/src/mlpack/tests/recurrent_network_test.cpp +++ b/src/mlpack/tests/recurrent_network_test.cpp @@ -19,9 +19,8 @@ #include #include -#include -#include "test_tools.hpp" -#include "serialization.hpp" +#include "catch.hpp" +#include "serialization_catch.hpp" #include "custom_layer.hpp" using namespace mlpack; @@ -29,8 +28,6 @@ using namespace mlpack::ann; using namespace ens; using namespace mlpack::math; -BOOST_AUTO_TEST_SUITE(RecurrentNetworkTest); - /** * Construct a 2-class dataset out of noisy sines. * @@ -75,7 +72,7 @@ void GenerateNoisySines(arma::cube& data, /** * Train the BRNN on a larger dataset. */ -BOOST_AUTO_TEST_CASE(SequenceClassificationBRNNTest) +TEST_CASE("SequenceClassificationBRNNTest", "[RecurrentNetworkTest]") { // Using same test for RNN below. size_t successes = 0; @@ -111,10 +108,10 @@ BOOST_AUTO_TEST_CASE(SequenceClassificationBRNNTest) StandardSGD opt(0.1, 1, 500 * input.n_cols, -100); model.Train(input, labels, opt); - BOOST_TEST_CHECKPOINT("Training over"); + INFO("Training over"); arma::cube prediction; model.Predict(input, prediction); - BOOST_TEST_CHECKPOINT("Prediction over"); + INFO("Prediction over"); size_t error = 0; for (size_t i = 0; i < prediction.n_cols; ++i) @@ -133,7 +130,7 @@ BOOST_AUTO_TEST_CASE(SequenceClassificationBRNNTest) } double classificationError = 1 - double(error) / prediction.n_cols; - BOOST_TEST_CHECKPOINT(classificationError); + INFO(classificationError); if (classificationError <= 0.2) { ++successes; @@ -141,13 +138,13 @@ BOOST_AUTO_TEST_CASE(SequenceClassificationBRNNTest) } } - BOOST_REQUIRE_GE(successes, 1); + REQUIRE(successes >= 1); } /** * Train the vanilla network on a larger dataset. */ -BOOST_AUTO_TEST_CASE(SequenceClassificationTest) +TEST_CASE("SequenceClassificationTest", "[RecurrentNetworkTest]") { // It isn't guaranteed that the recurrent network will converge in the // specified number of iterations using random weights. If this works 1 of 6 @@ -231,7 +228,7 @@ BOOST_AUTO_TEST_CASE(SequenceClassificationTest) } } - BOOST_REQUIRE_GE(successes, 1); + REQUIRE(successes >= 1); } /** @@ -645,13 +642,13 @@ void ReberGrammarTestNetwork(ModelType& model, offset += 3; } - BOOST_REQUIRE_GE(successes, 1); + REQUIRE(successes >= 1); } /** * Train the specified networks on an embedded Reber grammar dataset. */ -BOOST_AUTO_TEST_CASE(LSTMReberGrammarTest) +TEST_CASE("LSTMReberGrammarTest", "[RecurrentNetworkTest]") { RNN > model(5); model.Add >(7, 10); @@ -664,7 +661,7 @@ BOOST_AUTO_TEST_CASE(LSTMReberGrammarTest) /** * Train the specified networks on an embedded Reber grammar dataset. */ -BOOST_AUTO_TEST_CASE(FastLSTMReberGrammarTest) +TEST_CASE("FastLSTMReberGrammarTest", "[RecurrentNetworkTest]") { RNN > model(5); model.Add >(7, 8); @@ -677,7 +674,7 @@ BOOST_AUTO_TEST_CASE(FastLSTMReberGrammarTest) /** * Train the specified networks on an embedded Reber grammar dataset. */ -BOOST_AUTO_TEST_CASE(GRURecursiveReberGrammarTest) +TEST_CASE("GRURecursiveReberGrammarTest", "[RecurrentNetworkTest]") { RNN > model(5); model.Add >(7, 16); @@ -690,7 +687,7 @@ BOOST_AUTO_TEST_CASE(GRURecursiveReberGrammarTest) /** * Train BLSTM on an embedded Reber grammar dataset. */ -BOOST_AUTO_TEST_CASE(BRNNReberGrammarTest) +TEST_CASE("BRNNReberGrammarTest", "[RecurrentNetworkTest]") { BRNN, AddMerge<>, SigmoidLayer<> > model(5); model.Add >(7, 10); @@ -869,14 +866,14 @@ void DistractedSequenceRecallTestNetwork( offset += 2; } - BOOST_REQUIRE_GE(successes, 1); + REQUIRE(successes >= 1); } /** * Train the specified networks on the Derek D. Monner's distracted sequence * recall task. */ -BOOST_AUTO_TEST_CASE(LSTMDistractedSequenceRecallTest) +TEST_CASE("LSTMDistractedSequenceRecallTest", "[RecurrentNetworkTest]") { DistractedSequenceRecallTestNetwork >(4, 8); } @@ -885,7 +882,7 @@ BOOST_AUTO_TEST_CASE(LSTMDistractedSequenceRecallTest) * Train the specified networks on the Derek D. Monner's distracted sequence * recall task. */ -BOOST_AUTO_TEST_CASE(FastLSTMDistractedSequenceRecallTest) +TEST_CASE("FastLSTMDistractedSequenceRecallTest", "[RecurrentNetworkTest]") { DistractedSequenceRecallTestNetwork >(4, 8); } @@ -894,7 +891,7 @@ BOOST_AUTO_TEST_CASE(FastLSTMDistractedSequenceRecallTest) * Train the specified networks on the Derek D. Monner's distracted sequence * recall task. */ -BOOST_AUTO_TEST_CASE(GRUDistractedSequenceRecallTest) +TEST_CASE("GRUDistractedSequenceRecallTest", "[RecurrentNetworkTest]") { DistractedSequenceRecallTestNetwork >(4, 8); } @@ -956,7 +953,7 @@ void BatchSizeTest() /** * Ensure LSTMs work with larger batch sizes. */ -BOOST_AUTO_TEST_CASE(LSTMBatchSizeTest) +TEST_CASE("LSTMBatchSizeTest", "[RecurrentNetworkTest]") { BatchSizeTest>(); } @@ -964,7 +961,7 @@ BOOST_AUTO_TEST_CASE(LSTMBatchSizeTest) /** * Ensure fast LSTMs work with larger batch sizes. */ -BOOST_AUTO_TEST_CASE(FastLSTMBatchSizeTest) +TEST_CASE("FastLSTMBatchSizeTest", "[RecurrentNetworkTest]") { BatchSizeTest>(); } @@ -972,7 +969,7 @@ BOOST_AUTO_TEST_CASE(FastLSTMBatchSizeTest) /** * Ensure GRUs work with larger batch sizes. */ -BOOST_AUTO_TEST_CASE(GRUBatchSizeTest) +TEST_CASE("GRUBatchSizeTest", "[RecurrentNetworkTest]") { BatchSizeTest>(); } @@ -980,7 +977,7 @@ BOOST_AUTO_TEST_CASE(GRUBatchSizeTest) /** * Make sure the RNN can be properly serialized. */ -BOOST_AUTO_TEST_CASE(SerializationTest) +TEST_CASE("RNNSerializationTest", "[RecurrentNetworkTest]") { const size_t rho = 10; @@ -1172,13 +1169,13 @@ void ReberGrammarTestCustomNetwork(const size_t hiddenSize = 4, offset += 3; } - BOOST_REQUIRE_GE(successes, 1); + REQUIRE(successes >= 1); } /** * Train the specified networks on an embedded Reber grammar dataset. */ -BOOST_AUTO_TEST_CASE(CustomRecursiveReberGrammarTest) +TEST_CASE("CustomRecursiveReberGrammarTest", "[RecurrentNetworkTest]") { ReberGrammarTestCustomNetwork(16, true); } @@ -1312,16 +1309,16 @@ double RNNSineTest(size_t hiddenUnits, size_t rho, size_t numEpochs = 100) /** * Test RNN using multiple timestep input and single output. */ -BOOST_AUTO_TEST_CASE(MultiTimestepTest) +TEST_CASE("MultiTimestepTest", "[RecurrentNetworkTest]") { double err = RNNSineTest(4, 10, 20); - BOOST_REQUIRE_LE(err, 0.025); + REQUIRE(err <= 0.025); } /** * Test that RNN::Train() returns finite objective value. */ -BOOST_AUTO_TEST_CASE(RNNTrainReturnObjective) +TEST_CASE("RNNTrainReturnObjective", "[RecurrentNetworkTest]") { const size_t rho = 10; @@ -1371,13 +1368,13 @@ BOOST_AUTO_TEST_CASE(RNNTrainReturnObjective) StandardSGD opt(0.1, 1, input.n_cols /* 1 epoch */, -100); double objVal = model.Train(input, labels, opt); - BOOST_REQUIRE_EQUAL(std::isfinite(objVal), true); + REQUIRE(std::isfinite(objVal) == true); } /** * Test that BRNN::Train() returns finite objective value. */ -BOOST_AUTO_TEST_CASE(BRNNTrainReturnObjective) +TEST_CASE("BRNNTrainReturnObjective", "[RecurrentNetworkTest]") { const size_t rho = 10; @@ -1407,16 +1404,16 @@ BOOST_AUTO_TEST_CASE(BRNNTrainReturnObjective) StandardSGD opt(0.1, 1, 500 * input.n_cols, -100); double objVal = model.Train(input, labels, opt); - BOOST_TEST_CHECKPOINT("Training over"); + INFO("Training over"); // Test that BRNN::Train() returns finite objective value. - BOOST_REQUIRE_EQUAL(std::isfinite(objVal), true); + REQUIRE(std::isfinite(objVal) == true); } /** * Test that RNN::Train() does not give an error for large rho. */ -BOOST_AUTO_TEST_CASE(LargeRhoValueRnnTest) +TEST_CASE("LargeRhoValueRnnTest", "[RecurrentNetworkTest]") { // Setting rho value greater than sequence length which is 17. const size_t rho = 100; @@ -1473,7 +1470,5 @@ BOOST_AUTO_TEST_CASE(LargeRhoValueRnnTest) } ens::SGD<> opt(0.01, 1, 100); model.Train(inputs[0], targets[0], opt); - BOOST_TEST_CHECKPOINT("Training over"); + INFO("Training over"); } - -BOOST_AUTO_TEST_SUITE_END(); From 966b091bc05b673495d2509d82f6791aac724b59 Mon Sep 17 00:00:00 2001 From: Marcus Edel Date: Tue, 1 Sep 2020 01:00:16 +0200 Subject: [PATCH 44/55] Let's see if there is another python version installed. --- .ci/linux-steps.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.ci/linux-steps.yaml b/.ci/linux-steps.yaml index 8d0a76bb69..bbac934b73 100644 --- a/.ci/linux-steps.yaml +++ b/.ci/linux-steps.yaml @@ -23,6 +23,8 @@ steps: sudo apt-get install -y --allow-unauthenticated libopenblas-dev liblapack-dev g++ libboost1.70-dev libarmadillo-dev xz-utils + ls -l /usr/bin/ + if [ "$(binding)" == "python" ]; then /usr/bin/python3 -m pip install --upgrade pip /usr/bin/python3 -m pip install --upgrade --ignore-installed setuptools cython pandas From 40983f225c78fe5b03ec5aa16585964bf2caf35c Mon Sep 17 00:00:00 2001 From: Marcus Edel Date: Tue, 1 Sep 2020 11:13:04 +0200 Subject: [PATCH 45/55] Let's see if that picks up the correct python bin. --- .ci/linux-steps.yaml | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/.ci/linux-steps.yaml b/.ci/linux-steps.yaml index bbac934b73..69acf73201 100644 --- a/.ci/linux-steps.yaml +++ b/.ci/linux-steps.yaml @@ -24,10 +24,12 @@ steps: sudo apt-get install -y --allow-unauthenticated libopenblas-dev liblapack-dev g++ libboost1.70-dev libarmadillo-dev xz-utils ls -l /usr/bin/ + which python if [ "$(binding)" == "python" ]; then - /usr/bin/python3 -m pip install --upgrade pip - /usr/bin/python3 -m pip install --upgrade --ignore-installed setuptools cython pandas + export PYBIN=$(which python) + $PYBIN -m pip install --upgrade pip + $PYBIN -m pip install --upgrade --ignore-installed setuptools cython pandas fi if [ "a$(julia.version)" != "a" ]; then @@ -57,7 +59,11 @@ steps: export GOPATH=$PWD/src/mlpack/bindings/go go get -u -t gonum.org/v1/gonum/... fi - cmake $(CMakeArgs) .. + if [ "$(binding)" == "python" ]; then + cmake $(CMakeArgs) -DPYTHON_EXECUTABLE=$PYBIN .. + else + cmake $(CMakeArgs) .. + fi displayName: 'CMake' # Build mlpack From 8fbea698f03d171dc5a86067401edb3f125029f1 Mon Sep 17 00:00:00 2001 From: Marcus Edel Date: Tue, 1 Sep 2020 13:07:43 +0200 Subject: [PATCH 46/55] Export Python binary path. --- .ci/linux-steps.yaml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.ci/linux-steps.yaml b/.ci/linux-steps.yaml index 69acf73201..fd74a4553a 100644 --- a/.ci/linux-steps.yaml +++ b/.ci/linux-steps.yaml @@ -23,9 +23,6 @@ steps: sudo apt-get install -y --allow-unauthenticated libopenblas-dev liblapack-dev g++ libboost1.70-dev libarmadillo-dev xz-utils - ls -l /usr/bin/ - which python - if [ "$(binding)" == "python" ]; then export PYBIN=$(which python) $PYBIN -m pip install --upgrade pip @@ -60,6 +57,7 @@ steps: go get -u -t gonum.org/v1/gonum/... fi if [ "$(binding)" == "python" ]; then + export PYBIN=$(which python) cmake $(CMakeArgs) -DPYTHON_EXECUTABLE=$PYBIN .. else cmake $(CMakeArgs) .. From 7f411aecfc70fa124b4a45f7defe60fa67d0c648 Mon Sep 17 00:00:00 2001 From: Marcus Edel Date: Tue, 1 Sep 2020 16:21:51 +0200 Subject: [PATCH 47/55] Always set PYTHON_EXECUTABLE simplify the config. Co-authored-by: Ryan Curtin --- .ci/linux-steps.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/linux-steps.yaml b/.ci/linux-steps.yaml index fd74a4553a..4c38dfcc89 100644 --- a/.ci/linux-steps.yaml +++ b/.ci/linux-steps.yaml @@ -61,7 +61,7 @@ steps: cmake $(CMakeArgs) -DPYTHON_EXECUTABLE=$PYBIN .. else cmake $(CMakeArgs) .. - fi +cmake $(CMakeArgs) -DPYTHON_EXECUTABLE=`which python` .. displayName: 'CMake' # Build mlpack From f67ed0b5f2daabf399f94c5a3f059205a4870c5b Mon Sep 17 00:00:00 2001 From: Marcus Edel Date: Tue, 1 Sep 2020 16:33:04 +0200 Subject: [PATCH 48/55] Simplify config. --- .ci/linux-steps.yaml | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/.ci/linux-steps.yaml b/.ci/linux-steps.yaml index 4c38dfcc89..84538f59b4 100644 --- a/.ci/linux-steps.yaml +++ b/.ci/linux-steps.yaml @@ -56,12 +56,7 @@ steps: export GOPATH=$PWD/src/mlpack/bindings/go go get -u -t gonum.org/v1/gonum/... fi - if [ "$(binding)" == "python" ]; then - export PYBIN=$(which python) - cmake $(CMakeArgs) -DPYTHON_EXECUTABLE=$PYBIN .. - else - cmake $(CMakeArgs) .. -cmake $(CMakeArgs) -DPYTHON_EXECUTABLE=`which python` .. + cmake $(CMakeArgs) -DPYTHON_EXECUTABLE=`which python` .. displayName: 'CMake' # Build mlpack From 61dde92c681b1128306fad034b047e32162f88f9 Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Wed, 2 Sep 2020 12:52:26 -0400 Subject: [PATCH 49/55] Prepend "--" to options when checking that they are there. --- .../bindings/cli/parse_command_line.hpp | 10 +- src/mlpack/tests/io_test.cpp | 99 +++++++++++++++++++ 2 files changed, 105 insertions(+), 4 deletions(-) diff --git a/src/mlpack/bindings/cli/parse_command_line.hpp b/src/mlpack/bindings/cli/parse_command_line.hpp index a611e178fb..381d749bc1 100644 --- a/src/mlpack/bindings/cli/parse_command_line.hpp +++ b/src/mlpack/bindings/cli/parse_command_line.hpp @@ -47,8 +47,8 @@ void ParseCommandLine(int argc, char** argv) { // Add the parameter to desc. util::ParamData& d = it->second; - IO::GetSingleton().functionMap[d.tname]["AddToCLI11"] - (d, NULL, (void*) &app); + IO::GetSingleton().functionMap[d.tname]["AddToCLI11"](d, NULL, (void*) + &app); } // Mark that we did parsing. @@ -136,13 +136,15 @@ void ParseCommandLine(int argc, char** argv) util::ParamData d = iter->second; if (d.required) { - const std::string cliName; + // CLI11 expects the parameter name to have "--" prepended. + std::string cliName; IO::GetSingleton().functionMap[d.tname]["MapParameterName"](d, NULL, (void*) &cliName); + cliName = "--" + cliName; if (!app.count(cliName)) { - Log::Fatal << "Required option --" << cliName << " is undefined." + Log::Fatal << "Required option " << cliName << " is undefined." << std::endl; } } diff --git a/src/mlpack/tests/io_test.cpp b/src/mlpack/tests/io_test.cpp index 766e36facc..80aef17fbe 100644 --- a/src/mlpack/tests/io_test.cpp +++ b/src/mlpack/tests/io_test.cpp @@ -598,6 +598,105 @@ BOOST_AUTO_TEST_CASE(InputMatrixParamTest) BOOST_REQUIRE_CLOSE(dataset[i], dataset2[i], 1e-10); } +// Make sure we can correctly load required matrix parameters. +BOOST_AUTO_TEST_CASE(RequiredInputMatrixParamTest) +{ + AddRequiredCLIOptions(); + + // --matrix is an input parameter; it won't be transposed. + PARAM_MATRIX_IN_REQ("matrix", "Test matrix", "m"); + + // Set some fake arguments. + const char* argv[3]; + argv[0] = "./test"; + argv[1] = "--matrix_file"; + argv[2] = "test_data_3_1000.csv"; + + int argc = 3; + + // The const-cast is a little hacky but should be fine... + ParseCommandLine(argc, const_cast(argv)); + + // The --matrix parameter should exist. + BOOST_REQUIRE(IO::HasParam("matrix")); + // The --matrix_file parameter should not exist (it should be transparent from + // inside the program). + Log::Fatal.ignoreInput = true; + BOOST_REQUIRE_THROW(IO::HasParam("matrix_file"), runtime_error); + Log::Fatal.ignoreInput = false; + + arma::mat dataset = IO::GetParam("matrix"); + arma::mat dataset2 = IO::GetParam("matrix"); + + BOOST_REQUIRE_EQUAL(dataset.n_rows, 3); + BOOST_REQUIRE_EQUAL(dataset.n_cols, 1000); + BOOST_REQUIRE_EQUAL(dataset2.n_rows, 3); + BOOST_REQUIRE_EQUAL(dataset2.n_cols, 1000); + + for (size_t i = 0; i < dataset.n_elem; ++i) + BOOST_REQUIRE_CLOSE(dataset[i], dataset2[i], 1e-10); +} + +// Make sure loading required matrix options by alias succeeds. +BOOST_AUTO_TEST_CASE(RequiredInputMatrixParamAliasTest) +{ + AddRequiredCLIOptions(); + + // --matrix is an input parameter; it won't be transposed. + PARAM_MATRIX_IN_REQ("matrix", "Test matrix", "m"); + + // Set some fake arguments. + const char* argv[3]; + argv[0] = "./test"; + argv[1] = "-m"; + argv[2] = "test_data_3_1000.csv"; + + int argc = 3; + + // The const-cast is a little hacky but should be fine... + ParseCommandLine(argc, const_cast(argv)); + + // The --matrix parameter should exist. + BOOST_REQUIRE(IO::HasParam("matrix")); + // The --matrix_file parameter should not exist (it should be transparent from + // inside the program). + Log::Fatal.ignoreInput = true; + BOOST_REQUIRE_THROW(IO::HasParam("matrix_file"), runtime_error); + Log::Fatal.ignoreInput = false; + + arma::mat dataset = IO::GetParam("matrix"); + arma::mat dataset2 = IO::GetParam("matrix"); + + BOOST_REQUIRE_EQUAL(dataset.n_rows, 3); + BOOST_REQUIRE_EQUAL(dataset.n_cols, 1000); + BOOST_REQUIRE_EQUAL(dataset2.n_rows, 3); + BOOST_REQUIRE_EQUAL(dataset2.n_cols, 1000); + + for (size_t i = 0; i < dataset.n_elem; ++i) + BOOST_REQUIRE_CLOSE(dataset[i], dataset2[i], 1e-10); +} + +// Make sure that when we don't pass a required matrix, parsing fails. +BOOST_AUTO_TEST_CASE(RequiredUnspecifiedInputMatrixParamTest) +{ + AddRequiredCLIOptions(); + + // --matrix is an input parameter; it won't be transposed. + PARAM_MATRIX_IN_REQ("matrix", "Test matrix", "m"); + + // Set some fake arguments. + const char* argv[1]; + argv[0] = "./test"; + + int argc = 1; + + // The const-cast is a little hacky but should be fine... + Log::Fatal.ignoreInput = true; + BOOST_REQUIRE_THROW(ParseCommandLine(argc, const_cast(argv)), + std::exception); + Log::Fatal.ignoreInput = false; +} + BOOST_AUTO_TEST_CASE(InputMatrixNoTransposeParamTest) { AddRequiredCLIOptions(); From 2bc4e9fea627c7b5957f05750b0b104f716323e3 Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Wed, 2 Sep 2020 13:07:11 -0400 Subject: [PATCH 50/55] Update HISTORY.md. --- HISTORY.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/HISTORY.md b/HISTORY.md index 7943e77272..c2acee062f 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,5 +1,8 @@ ### mlpack ?.?.? ###### ????-??-?? + * Fix incorrect parsing of required matrix/model parameters for command-line + bindings (#2600). + * Add manual type specification support to `data::Load()` and `data::Save()` (#2084, #2135, #2602). From 065ae8b963b10db72b236bf86924bb286c17e245 Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Sun, 6 Sep 2020 07:36:05 -0400 Subject: [PATCH 51/55] Remove unnecessary line. --- src/mlpack/core/data/detect_file_type.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/mlpack/core/data/detect_file_type.cpp b/src/mlpack/core/data/detect_file_type.cpp index 4344bb5299..7a600d97c7 100644 --- a/src/mlpack/core/data/detect_file_type.cpp +++ b/src/mlpack/core/data/detect_file_type.cpp @@ -187,9 +187,8 @@ arma::file_type AutoDetect(std::fstream& stream, // We'll let Armadillo do its guessing (although we have to check if it is // arma_ascii ourselves) and see what we come up with. - // This is taken from load_auto_detect() in diskio_meat.hpp + // This is adapted from load_auto_detect() in diskio_meat.hpp. const std::string ARMA_MAT_TXT = "ARMA_MAT_TXT"; - // char* rawHeader = new char[ARMA_MAT_TXT.length() + 1]; std::string rawHeader(ARMA_MAT_TXT.length(), '\0'); std::streampos pos = stream.tellg(); From c289d0c8f0f60fce2fa8dd50741dee6da835b4d7 Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Sun, 6 Sep 2020 07:36:53 -0400 Subject: [PATCH 52/55] Apply suggestions from code review Thanks @zoq! Co-authored-by: Marcus Edel --- src/mlpack/core/data/detect_file_type.cpp | 2 +- src/mlpack/core/data/detect_file_type.hpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/mlpack/core/data/detect_file_type.cpp b/src/mlpack/core/data/detect_file_type.cpp index 7a600d97c7..740ffabd93 100644 --- a/src/mlpack/core/data/detect_file_type.cpp +++ b/src/mlpack/core/data/detect_file_type.cpp @@ -1,5 +1,5 @@ /** - * @file detect_file_type.cpp + * @file core/data/detect_file_type.cpp * @author Conrad Sanderson * @author Ryan Curtin * diff --git a/src/mlpack/core/data/detect_file_type.hpp b/src/mlpack/core/data/detect_file_type.hpp index 5c3989539d..ab387ad0ba 100644 --- a/src/mlpack/core/data/detect_file_type.hpp +++ b/src/mlpack/core/data/detect_file_type.hpp @@ -1,5 +1,5 @@ /** - * @file detect_file_type.hpp + * @file core/data/detect_file_type.hpp * @author Conrad Sanderson * @author Ryan Curtin * From 7ae9ddda86c1751b6509ceb48b27d182feaae439 Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Mon, 7 Sep 2020 11:52:36 -0400 Subject: [PATCH 53/55] Update version to 3.4.1. --- CMakeLists.txt | 8 ++++---- Doxyfile | 2 +- HISTORY.md | 4 ++-- README.md | 4 ++-- .../sample-ml-app/sample-ml-app.vcxproj | 8 ++++---- doc/guide/build.hpp | 12 ++++++------ doc/guide/python_quickstart.hpp | 6 +++--- doc/guide/sample_ml_app.hpp | 8 ++++---- 8 files changed, 26 insertions(+), 26 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index f041e321e6..6fe6d8105a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -360,14 +360,14 @@ endif () find_package(Ensmallen "${ENSMALLEN_VERSION}") if (NOT ENSMALLEN_FOUND) if (DOWNLOAD_ENSMALLEN) - file(DOWNLOAD http://www.ensmallen.org/files/ensmallen-latest.tar.gz - "${CMAKE_BINARY_DIR}/deps/ensmallen-latest.tar.gz" + file(DOWNLOAD http://www.ensmallen.org/files/ensmallen-2.14.2.tar.gz + "${CMAKE_BINARY_DIR}/deps/ensmallen-2.14.2.tar.gz" STATUS ENS_DOWNLOAD_STATUS_LIST LOG ENS_DOWNLOAD_LOG SHOW_PROGRESS) list(GET ENS_DOWNLOAD_STATUS_LIST 0 ENS_DOWNLOAD_STATUS) if (ENS_DOWNLOAD_STATUS EQUAL 0) execute_process(COMMAND ${CMAKE_COMMAND} -E - tar xzf "${CMAKE_BINARY_DIR}/deps/ensmallen-latest.tar.gz" + tar xzf "${CMAKE_BINARY_DIR}/deps/ensmallen-2.14.2.tar.gz" WORKING_DIRECTORY "${CMAKE_BINARY_DIR}/deps/") # Get the name of the directory. @@ -375,7 +375,7 @@ if (NOT ENSMALLEN_FOUND) "${CMAKE_BINARY_DIR}/deps/ensmallen-[0-9]*.[0-9]*.[0-9]*") # list(FILTER) is not available on 3.5 or older, but try to keep # configuring without filtering the list anyway (it might work if only - # the file ensmallen-latest.tar.gz is present. + # the file ensmallen-2.14.2.tar.gz is present. if (${CMAKE_VERSION} VERSION_GREATER_EQUAL "3.6.0") list(FILTER ENS_DIRECTORIES EXCLUDE REGEX "ensmallen-.*\.tar\.gz") endif () diff --git a/Doxyfile b/Doxyfile index 5f28220461..8c5b7ec5b4 100644 --- a/Doxyfile +++ b/Doxyfile @@ -4,7 +4,7 @@ # Project related configuration options #--------------------------------------------------------------------------- PROJECT_NAME = mlpack -PROJECT_NUMBER = 3.4.0 +PROJECT_NUMBER = 3.4.1 OUTPUT_DIRECTORY = ./doc CREATE_SUBDIRS = NO OUTPUT_LANGUAGE = English diff --git a/HISTORY.md b/HISTORY.md index c2acee062f..d71294702c 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,5 +1,5 @@ -### mlpack ?.?.? -###### ????-??-?? +### mlpack 3.4.1 +###### 2020-09-07 * Fix incorrect parsing of required matrix/model parameters for command-line bindings (#2600). diff --git a/README.md b/README.md index 4e5cc4aaa5..a0f9ea1822 100644 --- a/README.md +++ b/README.md @@ -23,7 +23,7 @@ src="https://cdn.rawgit.com/mlpack/mlpack.org/e7d36ed8/mlpack-black.svg" style="

Download: - current stable version (3.4.0) + current stable version (3.4.1)

@@ -152,7 +152,7 @@ on Ubuntu, you can install mlpack with the following command: Note: Older Ubuntu versions may not have the most recent version of mlpack available---for instance, at the time of this writing, Ubuntu 16.04 only has -mlpack 3.4.0 available. Options include upgrading your Ubuntu version, finding +mlpack 3.4.1 available. Options include upgrading your Ubuntu version, finding a PPA or other non-official sources, or installing with a manual build. There are some useful pages to consult in addition to this section: diff --git a/doc/examples/sample-ml-app/sample-ml-app/sample-ml-app.vcxproj b/doc/examples/sample-ml-app/sample-ml-app/sample-ml-app.vcxproj index 29b8d98818..92c3d1ae0a 100644 --- a/doc/examples/sample-ml-app/sample-ml-app/sample-ml-app.vcxproj +++ b/doc/examples/sample-ml-app/sample-ml-app/sample-ml-app.vcxproj @@ -104,16 +104,16 @@ true _DEBUG;_CONSOLE;%(PreprocessorDefinitions) false - C:\boost\boost_1_66_0;C:\mlpack\armadillo-8.500.1\include;C:\mlpack\mlpack-3.4.0\build\include;%(AdditionalIncludeDirectories) + C:\boost\boost_1_66_0;C:\mlpack\armadillo-8.500.1\include;C:\mlpack\mlpack-3.4.1\build\include;%(AdditionalIncludeDirectories) Console true - C:\mlpack\mlpack-3.4.0\build\Debug\mlpack.lib;C:\boost\boost_1_66_0\lib64-msvc-14.1\libboost_serialization-vc141-mt-gd-x64-1_66.lib;%(AdditionalDependencies) + C:\mlpack\mlpack-3.4.1\build\Debug\mlpack.lib;C:\boost\boost_1_66_0\lib64-msvc-14.1\libboost_serialization-vc141-mt-gd-x64-1_66.lib;%(AdditionalDependencies) - xcopy /y "C:\mlpack\mlpack-3.4.0\build\Debug\mlpack.dll" $(OutDir) -xcopy /y "C:\mlpack\mlpack-3.4.0\packages\OpenBLAS.0.2.14.1\lib\native\bin\x64\*.dll" $(OutDir) + xcopy /y "C:\mlpack\mlpack-3.4.1\build\Debug\mlpack.dll" $(OutDir) +xcopy /y "C:\mlpack\mlpack-3.4.1\packages\OpenBLAS.0.2.14.1\lib\native\bin\x64\*.dll" $(OutDir) xcopy /y "$(ProjectDir)..\..\..\..\src\mlpack\tests\data\german.csv" "$(ProjectDir)data\german.csv*" diff --git a/doc/guide/build.hpp b/doc/guide/build.hpp index f60cee3aa5..33f564475b 100644 --- a/doc/guide/build.hpp +++ b/doc/guide/build.hpp @@ -30,7 +30,7 @@ to build mlpack on Windows, see \ref build_windows (alternatively, you can read is based on older versions). You can download the latest mlpack release from here: -mlpack-3.4.0 +mlpack-3.4.1 @section build_simple Simple Linux build instructions @@ -38,9 +38,9 @@ Assuming all dependencies are installed in the system, you can run the commands below directly to build and install mlpack. @code -$ wget https://www.mlpack.org/files/mlpack-3.4.0.tar.gz -$ tar -xvzpf mlpack-3.4.0.tar.gz -$ mkdir mlpack-3.4.0/build && cd mlpack-3.4.0/build +$ wget https://www.mlpack.org/files/mlpack-3.4.1.tar.gz +$ tar -xvzpf mlpack-3.4.1.tar.gz +$ mkdir mlpack-3.4.1/build && cd mlpack-3.4.1/build $ cmake ../ $ make -j4 # The -j is the number of cores you want to use for a build. $ sudo make install @@ -65,8 +65,8 @@ configure mlpack. First we should unpack the mlpack source and create a build directory. @code -$ tar -xvzpf mlpack-3.4.0.tar.gz -$ cd mlpack-3.4.0 +$ tar -xvzpf mlpack-3.4.1.tar.gz +$ cd mlpack-3.4.1 $ mkdir build @endcode diff --git a/doc/guide/python_quickstart.hpp b/doc/guide/python_quickstart.hpp index 47aeefed15..1f442fc39e 100644 --- a/doc/guide/python_quickstart.hpp +++ b/doc/guide/python_quickstart.hpp @@ -32,9 +32,9 @@ build and install mlpack. You can copy-paste the commands into your shell. @code{.sh} sudo apt-get install libboost-all-dev g++ cmake libarmadillo-dev python-pip wget sudo pip install cython setuptools distutils numpy pandas -wget https://www.mlpack.org/files/mlpack-3.4.0.tar.gz -tar -xvzpf mlpack-3.4.0.tar.gz -mkdir -p mlpack-3.4.0/build/ && cd mlpack-3.4.0/build/ +wget https://www.mlpack.org/files/mlpack-3.4.1.tar.gz +tar -xvzpf mlpack-3.4.1.tar.gz +mkdir -p mlpack-3.4.1/build/ && cd mlpack-3.4.1/build/ cmake ../ && make -j4 && sudo make install @endcode diff --git a/doc/guide/sample_ml_app.hpp b/doc/guide/sample_ml_app.hpp index fa71905602..d2a350c85d 100644 --- a/doc/guide/sample_ml_app.hpp +++ b/doc/guide/sample_ml_app.hpp @@ -29,17 +29,17 @@ mlpack and dependencies in Release Mode). @code - C:\boost\boost_1_71_0\lib\native\include - C:\mlpack\armadillo-9.800.3\include - - C:\mlpack\mlpack-3.4.0\build\include + - C:\mlpack\mlpack-3.4.1\build\include @endcode - Under Linker > Input > Additional Dependencies add: @code - - C:\mlpack\mlpack-3.4.0\build\Debug\mlpack.lib + - C:\mlpack\mlpack-3.4.1\build\Debug\mlpack.lib - C:\boost\boost_1_71_0\lib64-msvc-14.2\libboost_serialization-vc142-mt-gd-x64-1_71.lib @endcode - Under Build Events > Post-Build Event > Command Line add: @code - - xcopy /y "C:\mlpack\mlpack-3.4.0\build\Debug\mlpack.dll" $(OutDir) - - xcopy /y "C:\mlpack\mlpack-3.4.0\packages\OpenBLAS.0.2.14.1\lib\native\bin\x64\*.dll" $(OutDir) + - xcopy /y "C:\mlpack\mlpack-3.4.1\build\Debug\mlpack.dll" $(OutDir) + - xcopy /y "C:\mlpack\mlpack-3.4.1\packages\OpenBLAS.0.2.14.1\lib\native\bin\x64\*.dll" $(OutDir) @endcode @note Recent versions of Visual Studio set "Conformance Mode" enabled by default. This causes some issues with From 089eb1fd489e17278e6259a38d94c38e3974535f Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Mon, 7 Sep 2020 11:52:37 -0400 Subject: [PATCH 54/55] Update version to next release version. --- CMakeLists.txt | 8 ++++---- src/mlpack/core/util/version.hpp | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 6fe6d8105a..f041e321e6 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -360,14 +360,14 @@ endif () find_package(Ensmallen "${ENSMALLEN_VERSION}") if (NOT ENSMALLEN_FOUND) if (DOWNLOAD_ENSMALLEN) - file(DOWNLOAD http://www.ensmallen.org/files/ensmallen-2.14.2.tar.gz - "${CMAKE_BINARY_DIR}/deps/ensmallen-2.14.2.tar.gz" + file(DOWNLOAD http://www.ensmallen.org/files/ensmallen-latest.tar.gz + "${CMAKE_BINARY_DIR}/deps/ensmallen-latest.tar.gz" STATUS ENS_DOWNLOAD_STATUS_LIST LOG ENS_DOWNLOAD_LOG SHOW_PROGRESS) list(GET ENS_DOWNLOAD_STATUS_LIST 0 ENS_DOWNLOAD_STATUS) if (ENS_DOWNLOAD_STATUS EQUAL 0) execute_process(COMMAND ${CMAKE_COMMAND} -E - tar xzf "${CMAKE_BINARY_DIR}/deps/ensmallen-2.14.2.tar.gz" + tar xzf "${CMAKE_BINARY_DIR}/deps/ensmallen-latest.tar.gz" WORKING_DIRECTORY "${CMAKE_BINARY_DIR}/deps/") # Get the name of the directory. @@ -375,7 +375,7 @@ if (NOT ENSMALLEN_FOUND) "${CMAKE_BINARY_DIR}/deps/ensmallen-[0-9]*.[0-9]*.[0-9]*") # list(FILTER) is not available on 3.5 or older, but try to keep # configuring without filtering the list anyway (it might work if only - # the file ensmallen-2.14.2.tar.gz is present. + # the file ensmallen-latest.tar.gz is present. if (${CMAKE_VERSION} VERSION_GREATER_EQUAL "3.6.0") list(FILTER ENS_DIRECTORIES EXCLUDE REGEX "ensmallen-.*\.tar\.gz") endif () diff --git a/src/mlpack/core/util/version.hpp b/src/mlpack/core/util/version.hpp index 3a8437d332..2ba6d122eb 100644 --- a/src/mlpack/core/util/version.hpp +++ b/src/mlpack/core/util/version.hpp @@ -18,7 +18,7 @@ // with higher number than the most recent release. #define MLPACK_VERSION_MAJOR 3 #define MLPACK_VERSION_MINOR 4 -#define MLPACK_VERSION_PATCH 1 +#define MLPACK_VERSION_PATCH 2 // The name of the version (for use by --version). namespace mlpack { From e11c3e320c4008a3c6ed16abeef69d53c0584366 Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Mon, 7 Sep 2020 11:52:37 -0400 Subject: [PATCH 55/55] Add new block to HISTORY.md for next version. --- HISTORY.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/HISTORY.md b/HISTORY.md index d71294702c..4d3807e380 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,3 +1,6 @@ +### mlpack ?.?.? +###### ????-??-?? + ### mlpack 3.4.1 ###### 2020-09-07 * Fix incorrect parsing of required matrix/model parameters for command-line