From dc402b0d106443d91f22ab5b24fff7375578d50c Mon Sep 17 00:00:00 2001 From: jeffinsam Date: Mon, 5 Aug 2019 18:21:57 +0530 Subject: [PATCH 001/111] Added bow encoding policy and tfidf policy (buggy) --- src/mlpack/core/data/string_encoding_impl.hpp | 7 +- .../string_encoding_policies/CMakeLists.txt | 2 + .../bow_encoding_policy.hpp | 142 +++++++++++++++ .../dictionary_encoding_policy.hpp | 10 ++ .../tf_idf_encoding_policy.hpp | 167 ++++++++++++++++++ src/mlpack/tests/string_encoding_test.cpp | 150 ++++++++++++++++ 6 files changed, 474 insertions(+), 4 deletions(-) create mode 100644 src/mlpack/core/data/string_encoding_policies/bow_encoding_policy.hpp create mode 100644 src/mlpack/core/data/string_encoding_policies/tf_idf_encoding_policy.hpp diff --git a/src/mlpack/core/data/string_encoding_impl.hpp b/src/mlpack/core/data/string_encoding_impl.hpp index b84781c26f..959d5b7b91 100644 --- a/src/mlpack/core/data/string_encoding_impl.hpp +++ b/src/mlpack/core/data/string_encoding_impl.hpp @@ -110,9 +110,9 @@ EncodeHelper(const std::vector& input, { size_t numColumns = 0; - for (const std::string& line : input) + for (size_t i = 0; i < input.size(); i++) { - boost::string_view strView(line); + boost::string_view strView(input[i]); auto token = tokenizer(strView); static_assert( @@ -128,7 +128,7 @@ EncodeHelper(const std::vector& input, { if (!dictionary.HasToken(token)) dictionary.AddToken(token); - + policy.PreprocessToken(i, numTokens, dictionary.Value(token)); token = tokenizer(strView); numTokens++; } @@ -136,7 +136,6 @@ EncodeHelper(const std::vector& input, } policy.InitMatrix(output, input.size(), numColumns, dictionary.Size()); - for (size_t i = 0; i < input.size(); i++) { boost::string_view strView(input[i]); diff --git a/src/mlpack/core/data/string_encoding_policies/CMakeLists.txt b/src/mlpack/core/data/string_encoding_policies/CMakeLists.txt index 8a8c7ab65c..ad88e9c0fc 100644 --- a/src/mlpack/core/data/string_encoding_policies/CMakeLists.txt +++ b/src/mlpack/core/data/string_encoding_policies/CMakeLists.txt @@ -2,6 +2,8 @@ # Anything not in this list will not be compiled into mlpack. set(SOURCES dictionary_encoding_policy.hpp + bow_encoding_policy.hpp + tf_idf_encoding_policy.hpp policy_traits.hpp ) diff --git a/src/mlpack/core/data/string_encoding_policies/bow_encoding_policy.hpp b/src/mlpack/core/data/string_encoding_policies/bow_encoding_policy.hpp new file mode 100644 index 0000000000..14c8566c94 --- /dev/null +++ b/src/mlpack/core/data/string_encoding_policies/bow_encoding_policy.hpp @@ -0,0 +1,142 @@ +/** + * @file bow_encoding_policy.hpp + * @author Jeffin Sam + * + * Definition of the BagOfWordsEncodingPolicy class. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_CORE_DATA_BAG_OF_WORDS_ENCODING_POLICY_HPP +#define MLPACK_CORE_DATA_BAG_OF_WORDS_ENCODING_POLICY_HPP + +#include +#include +#include + +namespace mlpack { +namespace data { +/** + * Definition of the BagOfWordsEncodingPolicy class. + * + * DicitonaryEnocding is used as a helper class for StringEncoding. + * The encoder assigns a positive integer number to each unique token and treat + * the dataset as categorical. The numbers are assigned sequentially starting + * from one. The tokens are labeled in the order of their occurrence + * in the input dataset. + */ +class BagOfWordsEncodingPolicy +{ + public: + /** + * The function initializes the output matrix. + * + * @param output Output matrix to store the encoded results (sp_mat or mat). + * @param datasetSize The number of strings in the input dataset. + * @param maxNumTokens The maximum number of tokens in the strings of the + input dataset (not used). + * @param dictionarySize The size of the dictionary. + */ + template + static void InitMatrix(MatType& output, + size_t datasetSize, + size_t /*maxNumTokens*/, + size_t dictionarySize) + { + output.zeros(datasetSize, dictionarySize); + } + + /** + * The function initializes the output matrix. + * Overloaded function to store result in vector> + * + * @param output Output matrix to store the encoded results. + * @param datasetSize The number of strings in the input dataset. + * @param maxNumTokens The maximum number of tokens in the strings of the + input dataset. + * @param dictionarySize The size of the dictionary (not used). + */ + static void InitMatrix(std::vector >& output, + size_t datasetSize, + size_t /*maxNumTokens*/, + size_t dictionarySize) + { + output.resize(datasetSize, std::vector (dictionarySize,0)); + } + + /** + * The function performs the bagofwords encoding algorithm i.e. it writes + * the encoded token to the ouput. + * + * @param output Output matrix to store the encoded results (sp_mat or mat). + * @param value The encoded token. + * @param row The row number at which the encoding is performed. + * @param col The row token number at which the encoding is performed. + */ + template + static void Encode(MatType& output, size_t value, size_t row, size_t /*col*/) + { + // Important since Mapping starts from 1 whereas allowed column value is 0. + output(row, value-1) = 1; + } + + /** + * The function performs the bagofwords encoding algorithm i.e. it writes + * the encoded token to the ouput. + * Overload function to accepted vector> as output type. + * + * @param output Output matrix to store the encoded results. + * @param value The encoded token. + * @param row The row number at which the encoding is performed. + * @param col The row token number at which the encoding is performed. + */ + static void Encode(std::vector >& output, size_t value, + size_t row, size_t /*col*/) + { + // Important since Mapping starts from 1 whereas allowed column value is 0. + output[row][value-1] = 1; + } + + /** + * Serialize the class to the given archive. + */ + template + void serialize(Archive& /* ar */, const unsigned int /* version */) + { + // Nothing to serialize. + } + + /** + * Empty function, Important for tf-idf encoding policy + * + * @param row The row number at which the encoding is performed. + * @param numToken The count of token parsed till now. + * @param value The encoded token. + */ + static void PreprocessToken(size_t /*row*/, size_t /*numTokens*/, + size_t /*value*/) { } +}; + +/** + * The specialization provides some information about the dictionary encoding + * policy. + */ +template<> +struct StringEncodingPolicyTraits +{ + /** + * Indicates if the policy is able to encode the token at once without + * any information about other tokens as well as the total tokens count. + */ + static const bool onePassEncoding = false; +}; + +template +using BowEncoding = StringEncoding>; +} // namespace data +} // namespace mlpack + +#endif \ No newline at end of file diff --git a/src/mlpack/core/data/string_encoding_policies/dictionary_encoding_policy.hpp b/src/mlpack/core/data/string_encoding_policies/dictionary_encoding_policy.hpp index 10239d7fbb..18b96b24a3 100644 --- a/src/mlpack/core/data/string_encoding_policies/dictionary_encoding_policy.hpp +++ b/src/mlpack/core/data/string_encoding_policies/dictionary_encoding_policy.hpp @@ -76,6 +76,16 @@ class DictionaryEncodingPolicy output.push_back(value); } + /** + * Empty function, Important for tf-idf encoding policy + * + * @param row The row number at which the encoding is performed. + * @param numToken The count of token parsed till now. + * @param value The encoded token. + */ + static void PreprocessToken(size_t /*row*/, size_t /*numTokens*/, + size_t /*value*/) { } + /** * Serialize the class to the given archive. */ diff --git a/src/mlpack/core/data/string_encoding_policies/tf_idf_encoding_policy.hpp b/src/mlpack/core/data/string_encoding_policies/tf_idf_encoding_policy.hpp new file mode 100644 index 0000000000..1c9565b8cb --- /dev/null +++ b/src/mlpack/core/data/string_encoding_policies/tf_idf_encoding_policy.hpp @@ -0,0 +1,167 @@ +/** + * @file td_idf_encoding_policy.hpp + * @author Jeffin Sam + * + * Definition of the TfIdfEncodingPolicy class. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_CORE_DATA_TF_IDF_ENCODING_POLICY_HPP +#define MLPACK_CORE_DATA_TF_IDF_ENCODING_POLICY_HPP + +#include +#include +#include +namespace mlpack { +namespace data { +/** + * Definition of the TfIdfEncodingPolicy class. + * + * DicitonaryEnocding is used as a helper class for StringEncoding. + * The encoder assigns a positive integer number to each unique token and treat + * the dataset as categorical. The numbers are assigned sequentially starting + * from one. The tokens are labeled in the order of their occurrence + * in the input dataset. + */ +class TfIdfEncodingPolicy +{ + public: + + /** + * The function initializes the output matrix. + * + * @param output Output matrix to store the encoded results (sp_mat or mat). + * @param datasetSize The number of strings in the input dataset. + * @param maxNumTokens The maximum number of tokens in the strings of the + input dataset. + * @param dictionarySize The size of the dictionary (not used). + */ + template + static void InitMatrix(MatType& output, + size_t datasetSize, + size_t /*maxNumTokens*/, + size_t dictionarySize) + { + output.zeros(datasetSize, dictionarySize); + std::cout<<"Init matrix "<> + * + * @param output Output matrix to store the encoded results. + * @param datasetSize The number of strings in the input dataset. + * @param maxNumTokens The maximum number of tokens in the strings of the + input dataset. + * @param dictionarySize The size of the dictionary (not used). + */ + static void InitMatrix(std::vector >& output, + size_t datasetSize, + size_t /*maxNumTokens*/, + size_t dictionarySize) + { + output.resize(datasetSize, std::vector (dictionarySize,0)); + } + + /** + * The function performs the TfIdf encoding algorithm i.e. it writes + * the encoded token to the ouput. + * + * @param output Output matrix to store the encoded results (sp_mat or mat). + * @param value The encoded token. + * @param row The row number at which the encoding is performed. + * @param col The row token number at which the encoding is performed. + */ + template + static void Encode(MatType& output, size_t value, size_t row, size_t /*col*/) + { + // Important since Mapping starts from 1 whereas allowed column value is 0. + std::cout<<"tokencoutn for "<> as output type. + * + * @param output Output matrix to store the encoded results (sp_mat or mat). + * @param value The encoded token. + * @param row The row number at which the encoding is performed. + * @param col The row token number at which the encoding is performed. + */ + static void Encode(std::vector >& output, size_t value, + size_t row, size_t /*col*/) + { + // Important since Mapping starts from 1 whereas allowed column value is 0. + output[row][value-1] = (tokenCount[row][value - 1] / (row_size[row]-1)) * + std::log10(output.size() / idfdict[value-1]); + } + + /** + * Serialize the class to the given archive. + */ + template + void serialize(Archive& /* ar */, const unsigned int /* version */) + { + // Nothing to serialize. + } + + /* + * The function is used to create the datastrcutre will be important to find + * out idfvalue of words, and then wrtiting the output based on their count. + * + * @param row The row number at which the encoding is performed. + * @param numToken The count of token parsed till now. + * @param value The encoded token. + */ + static void PreprocessToken(size_t row, size_t numTokens, + size_t value) + { + if(row>=tokenCount.size()) + { + row_size.push_back(0); + tokenCount.push_back(std::unordered_map()); + } + tokenCount.back()[value-1]++; + if(tokenCount.back()[value-1]==1) + idfdict[value-1]++; + row_size.back()++; + } + private: + static std::vector> tokenCount; + static std::unordered_map idfdict; + static std::vector row_size; +}; + +std::vector TfIdfEncodingPolicy::row_size = {}; +std::unordered_map TfIdfEncodingPolicy::idfdict = {}; +std::vector> TfIdfEncodingPolicy::tokenCount = {}; + +/** + * The specialization provides some information about the dictionary encoding + * policy. + */ +template<> +struct StringEncodingPolicyTraits +{ + /** + * Indicates if the policy is able to encode the token at once without + * any information about other tokens as well as the total tokens count. + */ + static const bool onePassEncoding = false; +}; + +template +using TfIdfEncoding = StringEncoding>; +} // namespace data +} // namespace mlpack + +#endif \ No newline at end of file diff --git a/src/mlpack/tests/string_encoding_test.cpp b/src/mlpack/tests/string_encoding_test.cpp index a819ef4328..f7f614e099 100644 --- a/src/mlpack/tests/string_encoding_test.cpp +++ b/src/mlpack/tests/string_encoding_test.cpp @@ -15,6 +15,8 @@ #include #include #include +#include +#include #include #include #include "test_tools.hpp" @@ -37,6 +39,11 @@ static vector stringEncodingInput = { "command-line programs and Python bindings." }; +static vector stringEncodingInputSmall = { + "hello how are you", + "i am good", + "Good how are you", +}; /** * Test the dictionary encoding algorithm. @@ -401,5 +408,148 @@ BOOST_AUTO_TEST_CASE(CharExtractDictionaryEncodingSerialization) CheckMatrices(output, xmlOutput, textOutput, binaryOutput); } +BOOST_AUTO_TEST_CASE(BowEncodingTest) +{ + using DictionaryType = StringEncodingDictionary; + + arma::mat output; + BowEncoding encoder; + SplitByAnyOf tokenizer(" "); + + encoder.Encode(stringEncodingInputSmall, output, tokenizer); + + const DictionaryType& dictionary = encoder.Dictionary(); + + // Checking that everything is mapped to different numbers + std::unordered_map keysCount; + for (auto& keyValue : dictionary.Mapping()) + { + keysCount[keyValue.second]++; + // Every token should be mapped only once + BOOST_REQUIRE_EQUAL(keysCount[keyValue.second], 1); + } + arma::mat expected = { + { 1, 1, 1, 1, 0, 0, 0, 0 }, + { 0, 0, 0, 0, 1, 1, 1, 0 }, + { 0, 1, 1, 1, 0, 0, 0, 1 } + }; + CheckMatrices(output, expected); +} + +/** + * Test the one pass modification of the Bag of Words encoding algorithm. + */ +BOOST_AUTO_TEST_CASE(OnePassBowEncodingTest) +{ + using DictionaryType = StringEncodingDictionary; + + vector> output; + BowEncoding encoder( + (BagOfWordsEncodingPolicy())); + SplitByAnyOf tokenizer(" "); + + encoder.Encode(stringEncodingInputSmall, output, tokenizer); + + const DictionaryType& dictionary = encoder.Dictionary(); + + // Checking that everything is mapped to different numbers + std::unordered_map keysCount; + for (auto& keyValue : dictionary.Mapping()) + { + keysCount[keyValue.second]++; + // Every token should be mapped only once + BOOST_REQUIRE_EQUAL(keysCount[keyValue.second], 1); + } + + vector> expected = { + { 1, 1, 1, 1, 0, 0, 0, 0 }, + { 0, 0, 0, 0, 1, 1, 1, 0 }, + { 0, 1, 1, 1, 0, 0, 0, 1 } + }; + + BOOST_REQUIRE(output == expected); +} + +/** +* Test Bag of Words encoding for characters using lamda function. +*/ +BOOST_AUTO_TEST_CASE(BowEncodingIndividualCharactersTest) +{ + vector input = { + "GACCA", + "ABCABCD", + "GAB" + }; + + arma::mat output; + BowEncoding encoder; + + // Passing a empty string to encode characters + encoder.Encode(input, output, CharExtract()); + arma::mat target = { + { 1, 1, 1, 0, 0 }, + { 0, 1, 1, 1, 1 }, + { 1, 1, 0, 1, 0 } + }; + + CheckMatrices(output, target); +} + +/** + * Test the one pass modification of the Bag of Words encoding algorithm + * in case of individual character encoding. + */ +BOOST_AUTO_TEST_CASE(OnePassBowEncodingIndividualCharactersTest) +{ + std::vector input = { + "GACCA", + "ABCABCD", + "GAB" + }; + + vector> output; + BowEncoding encoder; + + // Passing a empty string to encode characters + encoder.Encode(input, output, CharExtract()); + + vector> expected = { + { 1, 1, 1, 0, 0 }, + { 0, 1, 1, 1, 1 }, + { 1, 1, 0, 1, 0 } + }; + + BOOST_REQUIRE(output == expected); +} + +BOOST_AUTO_TEST_CASE(TfIdfEncodingTest) +{ + using DictionaryType = StringEncodingDictionary; + + arma::mat output; + TfIdfEncoding encoder; + SplitByAnyOf tokenizer(" "); + + encoder.Encode(stringEncodingInputSmall, output, tokenizer); + + const DictionaryType& dictionary = encoder.Dictionary(); + + // Checking that everything is mapped to different numbers + std::unordered_map keysCount; + for (auto& keyValue : dictionary.Mapping()) + { + keysCount[keyValue.second]++; + // Every token should be mapped only once + BOOST_REQUIRE_EQUAL(keysCount[keyValue.second], 1); + } + std::cout<<"inside \n"< Date: Mon, 5 Aug 2019 21:42:19 +0530 Subject: [PATCH 002/111] solved the bug in Tf Idf Encoding Algorithim --- .../tf_idf_encoding_policy.hpp | 7 ++---- src/mlpack/tests/string_encoding_test.cpp | 25 ++++++++++++++++++- 2 files changed, 26 insertions(+), 6 deletions(-) diff --git a/src/mlpack/core/data/string_encoding_policies/tf_idf_encoding_policy.hpp b/src/mlpack/core/data/string_encoding_policies/tf_idf_encoding_policy.hpp index 1c9565b8cb..4fcd659b37 100644 --- a/src/mlpack/core/data/string_encoding_policies/tf_idf_encoding_policy.hpp +++ b/src/mlpack/core/data/string_encoding_policies/tf_idf_encoding_policy.hpp @@ -46,7 +46,6 @@ class TfIdfEncodingPolicy size_t dictionarySize) { output.zeros(datasetSize, dictionarySize); - std::cout<<"Init matrix "< input = { + "GACCA", + "ABCABCD", + "GAB" + }; + + arma::mat output; + TfIdfEncoding encoder; + + // Passing a empty string to encode characters + encoder.Encode(input, output, CharExtract()); + arma::mat target = { + { 0.0440, 0.0440, 0.0440, 0, 0 }, + { 0, 0, 0, 0, 0.0587 }, + { 0.0185, 0.0556, 0, 0.0371, 0 } + }; + CheckMatrices(output, target, 1e-01); +} + BOOST_AUTO_TEST_SUITE_END(); From 1935a9efdcb8430bef7aba8ca9bf6c75dc210586 Mon Sep 17 00:00:00 2001 From: jeffinsam Date: Mon, 5 Aug 2019 22:26:44 +0530 Subject: [PATCH 003/111] add support for vector> for tfidf --- src/mlpack/core/data/string_encoding.hpp | 5 +- src/mlpack/core/data/string_encoding_impl.hpp | 5 +- .../bow_encoding_policy.hpp | 8 +-- .../dictionary_encoding_policy.hpp | 3 +- .../tf_idf_encoding_policy.hpp | 18 +++++-- src/mlpack/tests/string_encoding_test.cpp | 50 +++++++++++++++++++ 6 files changed, 77 insertions(+), 12 deletions(-) diff --git a/src/mlpack/core/data/string_encoding.hpp b/src/mlpack/core/data/string_encoding.hpp index 54356bd06b..3377f29abb 100644 --- a/src/mlpack/core/data/string_encoding.hpp +++ b/src/mlpack/core/data/string_encoding.hpp @@ -172,9 +172,10 @@ class StringEncoding * and the IsTokenEmpty() method that accepts the given token and returns true * if the given token is empty. */ - template + template void EncodeHelper(const std::vector& input, - std::vector>& output, + std::vector>& output, const TokenizerType& tokenizer, PolicyType& policy, typename std::enable_if& input, } template -template +template void StringEncoding:: EncodeHelper(const std::vector& input, - std::vector>& output, + std::vector>& output, const TokenizerType& tokenizer, PolicyType& policy, typename std::enable_if >& output, + template + static void InitMatrix(std::vector >& output, size_t datasetSize, size_t /*maxNumTokens*/, size_t dictionarySize) { - output.resize(datasetSize, std::vector (dictionarySize,0)); + output.resize(datasetSize, std::vector (dictionarySize,0)); } /** @@ -92,7 +93,8 @@ class BagOfWordsEncodingPolicy * @param row The row number at which the encoding is performed. * @param col The row token number at which the encoding is performed. */ - static void Encode(std::vector >& output, size_t value, + template + static void Encode(std::vector >& output, size_t value, size_t row, size_t /*col*/) { // Important since Mapping starts from 1 whereas allowed column value is 0. diff --git a/src/mlpack/core/data/string_encoding_policies/dictionary_encoding_policy.hpp b/src/mlpack/core/data/string_encoding_policies/dictionary_encoding_policy.hpp index 18b96b24a3..f11e7bb566 100644 --- a/src/mlpack/core/data/string_encoding_policies/dictionary_encoding_policy.hpp +++ b/src/mlpack/core/data/string_encoding_policies/dictionary_encoding_policy.hpp @@ -71,7 +71,8 @@ class DictionaryEncodingPolicy * @param output Output vector to store the encoded results. * @param value The encoded token. */ - static void Encode(std::vector& output, size_t value) + template + static void Encode(std::vector& output, size_t value) { output.push_back(value); } diff --git a/src/mlpack/core/data/string_encoding_policies/tf_idf_encoding_policy.hpp b/src/mlpack/core/data/string_encoding_policies/tf_idf_encoding_policy.hpp index 4fcd659b37..6d8e49f4e3 100644 --- a/src/mlpack/core/data/string_encoding_policies/tf_idf_encoding_policy.hpp +++ b/src/mlpack/core/data/string_encoding_policies/tf_idf_encoding_policy.hpp @@ -58,12 +58,13 @@ class TfIdfEncodingPolicy input dataset. * @param dictionarySize The size of the dictionary (not used). */ - static void InitMatrix(std::vector >& output, + template + static void InitMatrix(std::vector >& output, size_t datasetSize, size_t /*maxNumTokens*/, size_t dictionarySize) { - output.resize(datasetSize, std::vector (dictionarySize,0)); + output.resize(datasetSize, std::vector (dictionarySize,0)); } /** @@ -79,8 +80,12 @@ class TfIdfEncodingPolicy static void Encode(MatType& output, size_t value, size_t row, size_t /*col*/) { // Important since Mapping starts from 1 whereas allowed column value is 0. + // std::cout<<"divide "<<(tokenCount[row][value - 1] / row_size[row])<<"\n"; + // std::cout<<"output size "< >& output, size_t value, + template + static void Encode(std::vector >& output, size_t value, size_t row, size_t /*col*/) { // Important since Mapping starts from 1 whereas allowed column value is 0. + // std::cout<<"divide "<<(tokenCount[row][value - 1] / row_size[row])<<"\n"; + // std::cout<<"output size "<=tokenCount.size()) diff --git a/src/mlpack/tests/string_encoding_test.cpp b/src/mlpack/tests/string_encoding_test.cpp index 9eeadd4ae2..38f306a4ca 100644 --- a/src/mlpack/tests/string_encoding_test.cpp +++ b/src/mlpack/tests/string_encoding_test.cpp @@ -550,6 +550,56 @@ BOOST_AUTO_TEST_CASE(TfIdfEncodingTest) CheckMatrices(output, expected, 1e-01); } +/** + * Test the one pass modification of the TfIdf encoding algorithm. + */ +BOOST_AUTO_TEST_CASE(OnePassTfIdfEncodingTest) +{ + using DictionaryType = StringEncodingDictionary; + + vector> output; + TfIdfEncoding encoder( + (TfIdfEncodingPolicy())); + SplitByAnyOf tokenizer(" "); + + encoder.Encode(stringEncodingInputSmall, output, tokenizer); + + const DictionaryType& dictionary = encoder.Dictionary(); + + // Checking that everything is mapped to different numbers + std::unordered_map keysCount; + for (auto& keyValue : dictionary.Mapping()) + { + keysCount[keyValue.second]++; + // Every token should be mapped only once + BOOST_REQUIRE_EQUAL(keysCount[keyValue.second], 1); + } + std::cout<<"print output "<> expected = { + { 0.1193, 0.0440, 0.0440, 0.0440, 0, 0, 0, 0 }, + { 0, 0, 0, 0, 0.1590, 0.1590, 0.1590, 0 }, + { 0, 0.0440, 0.0440, 0.0440, 0, 0, 0, 0.1193 } + }; + std::cout<<"normal expected"< Date: Mon, 5 Aug 2019 22:43:28 +0530 Subject: [PATCH 004/111] Y are tfidf values different? --- .../tf_idf_encoding_policy.hpp | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/mlpack/core/data/string_encoding_policies/tf_idf_encoding_policy.hpp b/src/mlpack/core/data/string_encoding_policies/tf_idf_encoding_policy.hpp index 6d8e49f4e3..be34671af3 100644 --- a/src/mlpack/core/data/string_encoding_policies/tf_idf_encoding_policy.hpp +++ b/src/mlpack/core/data/string_encoding_policies/tf_idf_encoding_policy.hpp @@ -46,6 +46,13 @@ class TfIdfEncodingPolicy size_t dictionarySize) { output.zeros(datasetSize, dictionarySize); + std::cout<<"arma::mat;\n"; + for(auto it=idfdict.begin();it!=idfdict.end();it++) + { + std::cout<first<<" "<second<<"\n "; + } + std::cout<<"\n"; + } /** @@ -65,6 +72,12 @@ class TfIdfEncodingPolicy size_t dictionarySize) { output.resize(datasetSize, std::vector (dictionarySize,0)); + std::cout<<"std::vector v;\n"; + for(auto it=idfdict.begin();it!=idfdict.end();it++) + { + std::cout<first<<" "<second<<"\n"; + } + std::cout<<"\n"; } /** From 22362cb03ae7bd1a643ea8c5541cea2fd1e25ede Mon Sep 17 00:00:00 2001 From: jeffinsam Date: Mon, 5 Aug 2019 23:18:04 +0530 Subject: [PATCH 005/111] completed with TF IDF Policy --- .../tf_idf_encoding_policy.hpp | 37 ++--------- src/mlpack/tests/string_encoding_test.cpp | 62 +++++++++++-------- 2 files changed, 43 insertions(+), 56 deletions(-) diff --git a/src/mlpack/core/data/string_encoding_policies/tf_idf_encoding_policy.hpp b/src/mlpack/core/data/string_encoding_policies/tf_idf_encoding_policy.hpp index be34671af3..a3a76b2128 100644 --- a/src/mlpack/core/data/string_encoding_policies/tf_idf_encoding_policy.hpp +++ b/src/mlpack/core/data/string_encoding_policies/tf_idf_encoding_policy.hpp @@ -46,13 +46,6 @@ class TfIdfEncodingPolicy size_t dictionarySize) { output.zeros(datasetSize, dictionarySize); - std::cout<<"arma::mat;\n"; - for(auto it=idfdict.begin();it!=idfdict.end();it++) - { - std::cout<first<<" "<second<<"\n "; - } - std::cout<<"\n"; - } /** @@ -72,12 +65,6 @@ class TfIdfEncodingPolicy size_t dictionarySize) { output.resize(datasetSize, std::vector (dictionarySize,0)); - std::cout<<"std::vector v;\n"; - for(auto it=idfdict.begin();it!=idfdict.end();it++) - { - std::cout<first<<" "<second<<"\n"; - } - std::cout<<"\n"; } /** @@ -90,15 +77,11 @@ class TfIdfEncodingPolicy * @param col The row token number at which the encoding is performed. */ template - static void Encode(MatType& output, size_t value, size_t row, size_t /*col*/) + void Encode(MatType& output, size_t value, size_t row, size_t /*col*/) { // Important since Mapping starts from 1 whereas allowed column value is 0. - // std::cout<<"divide "<<(tokenCount[row][value - 1] / row_size[row])<<"\n"; - // std::cout<<"output size "< - static void Encode(std::vector >& output, size_t value, + void Encode(std::vector >& output, size_t value, size_t row, size_t /*col*/) { // Important since Mapping starts from 1 whereas allowed column value is 0. - // std::cout<<"divide "<<(tokenCount[row][value - 1] / row_size[row])<<"\n"; - // std::cout<<"output size "<=tokenCount.size()) @@ -155,15 +134,11 @@ class TfIdfEncodingPolicy row_size.back()++; } private: - static std::vector> tokenCount; - static std::unordered_map idfdict; - static std::vector row_size; + std::vector> tokenCount; + std::unordered_map idfdict; + std::vector row_size; }; -std::vector TfIdfEncodingPolicy::row_size = {}; -std::unordered_map TfIdfEncodingPolicy::idfdict = {}; -std::vector> TfIdfEncodingPolicy::tokenCount = {}; - /** * The specialization provides some information about the dictionary encoding * policy. diff --git a/src/mlpack/tests/string_encoding_test.cpp b/src/mlpack/tests/string_encoding_test.cpp index 38f306a4ca..4ec1ab4b7c 100644 --- a/src/mlpack/tests/string_encoding_test.cpp +++ b/src/mlpack/tests/string_encoding_test.cpp @@ -574,30 +574,15 @@ BOOST_AUTO_TEST_CASE(OnePassTfIdfEncodingTest) // Every token should be mapped only once BOOST_REQUIRE_EQUAL(keysCount[keyValue.second], 1); } - std::cout<<"print output "<> expected = { - { 0.1193, 0.0440, 0.0440, 0.0440, 0, 0, 0, 0 }, - { 0, 0, 0, 0, 0.1590, 0.1590, 0.1590, 0 }, - { 0, 0.0440, 0.0440, 0.0440, 0, 0, 0, 0.1193 } + { 0.11928, 0.0440228, 0.0440228, 0.0440228, 0, 0, 0, 0 }, + { 0, 0, 0, 0, 0.15904, 0.15904, 0.15904, 0 }, + { 0, 0.044028, 0.0440228, 0.0440228, 0, 0, 0, 0.11928 } }; - std::cout<<"normal expected"< input = { + "GACCA", + "ABCABCD", + "GAB" + }; + + vector> output; + TfIdfEncoding encoder; + + // Passing a empty string to encode characters + encoder.Encode(input, output, CharExtract()); + vector> expected = { + { 0.0352, 0, 0.0704, 0, 0 }, + { 0, 0, 0.0503, 0.0503, 0.0682 }, + { 0.0587, 0, 0, 0.0587, 0 } + }; + for(size_t i=0;i Date: Tue, 6 Aug 2019 16:16:57 +0530 Subject: [PATCH 006/111] resolve comments about header and eps --- .../bow_encoding_policy.hpp | 12 ++--- .../tf_idf_encoding_policy.hpp | 6 +-- src/mlpack/tests/string_encoding_test.cpp | 48 +++++++++---------- 3 files changed, 33 insertions(+), 33 deletions(-) diff --git a/src/mlpack/core/data/string_encoding_policies/bow_encoding_policy.hpp b/src/mlpack/core/data/string_encoding_policies/bow_encoding_policy.hpp index 377687dcdb..9660c939e9 100644 --- a/src/mlpack/core/data/string_encoding_policies/bow_encoding_policy.hpp +++ b/src/mlpack/core/data/string_encoding_policies/bow_encoding_policy.hpp @@ -9,8 +9,8 @@ * 3-clause BSD license along with mlpack. If not, see * http://www.opensource.org/licenses/BSD-3-Clause for more information. */ -#ifndef MLPACK_CORE_DATA_BAG_OF_WORDS_ENCODING_POLICY_HPP -#define MLPACK_CORE_DATA_BAG_OF_WORDS_ENCODING_POLICY_HPP +#ifndef MLPACK_CORE_DATA_ENCODING_POLICIES_BAG_OF_WORDS_ENCODING_POLICY_HPP +#define MLPACK_CORE_DATA_ENCODING_POLICIES_BAG_OF_WORDS_ENCODING_POLICY_HPP #include #include @@ -64,11 +64,11 @@ class BagOfWordsEncodingPolicy size_t /*maxNumTokens*/, size_t dictionarySize) { - output.resize(datasetSize, std::vector (dictionarySize,0)); + output.resize(datasetSize, std::vector (dictionarySize, 0)); } /** - * The function performs the bagofwords encoding algorithm i.e. it writes + * The function performs the bag of words encoding algorithm i.e. it writes * the encoded token to the ouput. * * @param output Output matrix to store the encoded results (sp_mat or mat). @@ -84,7 +84,7 @@ class BagOfWordsEncodingPolicy } /** - * The function performs the bagofwords encoding algorithm i.e. it writes + * The function performs the bag of words encoding algorithm i.e. it writess * the encoded token to the ouput. * Overload function to accepted vector> as output type. * @@ -136,7 +136,7 @@ struct StringEncodingPolicyTraits }; template -using BowEncoding = StringEncoding>; } // namespace data } // namespace mlpack diff --git a/src/mlpack/core/data/string_encoding_policies/tf_idf_encoding_policy.hpp b/src/mlpack/core/data/string_encoding_policies/tf_idf_encoding_policy.hpp index a3a76b2128..16b4cbbc29 100644 --- a/src/mlpack/core/data/string_encoding_policies/tf_idf_encoding_policy.hpp +++ b/src/mlpack/core/data/string_encoding_policies/tf_idf_encoding_policy.hpp @@ -9,8 +9,8 @@ * 3-clause BSD license along with mlpack. If not, see * http://www.opensource.org/licenses/BSD-3-Clause for more information. */ -#ifndef MLPACK_CORE_DATA_TF_IDF_ENCODING_POLICY_HPP -#define MLPACK_CORE_DATA_TF_IDF_ENCODING_POLICY_HPP +#ifndef MLPACK_CORE_DATA_ENCODING_POLICIES_TF_IDF_ENCODING_POLICY_HPP +#define MLPACK_CORE_DATA_ENCODING_POLICIES_TF_IDF_ENCODING_POLICY_HPP #include #include @@ -64,7 +64,7 @@ class TfIdfEncodingPolicy size_t /*maxNumTokens*/, size_t dictionarySize) { - output.resize(datasetSize, std::vector (dictionarySize,0)); + output.resize(datasetSize, std::vector (dictionarySize, 0)); } /** diff --git a/src/mlpack/tests/string_encoding_test.cpp b/src/mlpack/tests/string_encoding_test.cpp index 4ec1ab4b7c..74cc5b99ba 100644 --- a/src/mlpack/tests/string_encoding_test.cpp +++ b/src/mlpack/tests/string_encoding_test.cpp @@ -408,12 +408,12 @@ BOOST_AUTO_TEST_CASE(CharExtractDictionaryEncodingSerialization) CheckMatrices(output, xmlOutput, textOutput, binaryOutput); } -BOOST_AUTO_TEST_CASE(BowEncodingTest) +BOOST_AUTO_TEST_CASE(BagOfWordsEncodingTest) { using DictionaryType = StringEncodingDictionary; arma::mat output; - BowEncoding encoder; + BagOfWordsEncoding encoder; SplitByAnyOf tokenizer(" "); encoder.Encode(stringEncodingInputSmall, output, tokenizer); @@ -439,12 +439,12 @@ BOOST_AUTO_TEST_CASE(BowEncodingTest) /** * Test the one pass modification of the Bag of Words encoding algorithm. */ -BOOST_AUTO_TEST_CASE(OnePassBowEncodingTest) +BOOST_AUTO_TEST_CASE(OnePassBagOfWordsEncodingTest) { using DictionaryType = StringEncodingDictionary; vector> output; - BowEncoding encoder( + BagOfWordsEncoding encoder( (BagOfWordsEncodingPolicy())); SplitByAnyOf tokenizer(" "); @@ -473,7 +473,7 @@ BOOST_AUTO_TEST_CASE(OnePassBowEncodingTest) /** * Test Bag of Words encoding for characters using lamda function. */ -BOOST_AUTO_TEST_CASE(BowEncodingIndividualCharactersTest) +BOOST_AUTO_TEST_CASE(BagOfWordsEncodingIndividualCharactersTest) { vector input = { "GACCA", @@ -482,7 +482,7 @@ BOOST_AUTO_TEST_CASE(BowEncodingIndividualCharactersTest) }; arma::mat output; - BowEncoding encoder; + BagOfWordsEncoding encoder; // Passing a empty string to encode characters encoder.Encode(input, output, CharExtract()); @@ -499,7 +499,7 @@ BOOST_AUTO_TEST_CASE(BowEncodingIndividualCharactersTest) * Test the one pass modification of the Bag of Words encoding algorithm * in case of individual character encoding. */ -BOOST_AUTO_TEST_CASE(OnePassBowEncodingIndividualCharactersTest) +BOOST_AUTO_TEST_CASE(OnePassBagOfWordsEncodingIndividualCharactersTest) { std::vector input = { "GACCA", @@ -508,7 +508,7 @@ BOOST_AUTO_TEST_CASE(OnePassBowEncodingIndividualCharactersTest) }; vector> output; - BowEncoding encoder; + BagOfWordsEncoding encoder; // Passing a empty string to encode characters encoder.Encode(input, output, CharExtract()); @@ -543,11 +543,11 @@ BOOST_AUTO_TEST_CASE(TfIdfEncodingTest) BOOST_REQUIRE_EQUAL(keysCount[keyValue.second], 1); } arma::mat expected = { - { 0.1193, 0.0440, 0.0440, 0.0440, 0, 0, 0, 0 }, - { 0, 0, 0, 0, 0.1590, 0.1590, 0.1590, 0 }, - { 0, 0.0440, 0.0440, 0.0440, 0, 0, 0, 0.1193 } + { 0.1192803136799, 0.0440228147639, 0.0440228147639, 0.0440228147639, 0, 0, 0, 0 }, + { 0, 0, 0, 0, 0.1590404182399, 0.1590404182399, 0.1590404182399, 0 }, + { 0, 0.0440228147639 , 0.0440228147639 , 0.0440228147639 , 0, 0, 0, 0.1192803136799 } }; - CheckMatrices(output, expected, 1e-01); + CheckMatrices(output, expected, 1e-10); } /** @@ -576,13 +576,13 @@ BOOST_AUTO_TEST_CASE(OnePassTfIdfEncodingTest) } vector> expected = { - { 0.11928, 0.0440228, 0.0440228, 0.0440228, 0, 0, 0, 0 }, - { 0, 0, 0, 0, 0.15904, 0.15904, 0.15904, 0 }, - { 0, 0.044028, 0.0440228, 0.0440228, 0, 0, 0, 0.11928 } + { 0.1192803136799, 0.0440228147639, 0.0440228147639, 0.0440228147639, 0, 0, 0, 0 }, + { 0, 0, 0, 0, 0.1590404182399, 0.1590404182399, 0.1590404182399, 0 }, + { 0, 0.0440228147639 , 0.0440228147639 , 0.0440228147639 , 0, 0, 0, 0.1192803136799 } }; for(size_t i=0;i> expected = { - { 0.0352, 0, 0.0704, 0, 0 }, - { 0, 0, 0.0503, 0.0503, 0.0682 }, - { 0.0587, 0, 0, 0.0587, 0 } + { 0.0352182518111, 0, 0.0704365036223, 0, 0 }, + { 0, 0, 0.0503117883016, 0.0503117883016, 0.0681601792457 }, + { 0.0586970863519, 0, 0, 0.0586970863519, 0 } }; for(size_t i=0;i Date: Tue, 6 Aug 2019 18:44:00 +0530 Subject: [PATCH 007/111] rewriting test for long inputs as accuracy as eps = 1e-12 --- src/mlpack/tests/string_encoding_test.cpp | 103 +++++++++++++++------- 1 file changed, 70 insertions(+), 33 deletions(-) diff --git a/src/mlpack/tests/string_encoding_test.cpp b/src/mlpack/tests/string_encoding_test.cpp index 74cc5b99ba..185e8820ef 100644 --- a/src/mlpack/tests/string_encoding_test.cpp +++ b/src/mlpack/tests/string_encoding_test.cpp @@ -39,12 +39,6 @@ static vector stringEncodingInput = { "command-line programs and Python bindings." }; -static vector stringEncodingInputSmall = { - "hello how are you", - "i am good", - "Good how are you", -}; - /** * Test the dictionary encoding algorithm. */ @@ -416,8 +410,7 @@ BOOST_AUTO_TEST_CASE(BagOfWordsEncodingTest) BagOfWordsEncoding encoder; SplitByAnyOf tokenizer(" "); - encoder.Encode(stringEncodingInputSmall, output, tokenizer); - + encoder.Encode(stringEncodingInput, output, tokenizer); const DictionaryType& dictionary = encoder.Dictionary(); // Checking that everything is mapped to different numbers @@ -429,9 +422,12 @@ BOOST_AUTO_TEST_CASE(BagOfWordsEncodingTest) BOOST_REQUIRE_EQUAL(keysCount[keyValue.second], 1); } arma::mat expected = { - { 1, 1, 1, 1, 0, 0, 0, 0 }, - { 0, 0, 0, 0, 1, 1, 1, 0 }, - { 0, 1, 1, 1, 0, 0, 0, 1 } + { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, + { 0, 1, 0, 0, 0, 1, 0, 0, 1, 1, 0, 0, 0, 1, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, + { 1, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 } }; CheckMatrices(output, expected); } @@ -448,7 +444,7 @@ BOOST_AUTO_TEST_CASE(OnePassBagOfWordsEncodingTest) (BagOfWordsEncodingPolicy())); SplitByAnyOf tokenizer(" "); - encoder.Encode(stringEncodingInputSmall, output, tokenizer); + encoder.Encode(stringEncodingInput, output, tokenizer); const DictionaryType& dictionary = encoder.Dictionary(); @@ -462,9 +458,12 @@ BOOST_AUTO_TEST_CASE(OnePassBagOfWordsEncodingTest) } vector> expected = { - { 1, 1, 1, 1, 0, 0, 0, 0 }, - { 0, 0, 0, 0, 1, 1, 1, 0 }, - { 0, 1, 1, 1, 0, 0, 0, 1 } + { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, + { 0, 1, 0, 0, 0, 1, 0, 0, 1, 1, 0, 0, 0, 1, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, + { 1, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 } }; BOOST_REQUIRE(output == expected); @@ -530,7 +529,7 @@ BOOST_AUTO_TEST_CASE(TfIdfEncodingTest) TfIdfEncoding encoder; SplitByAnyOf tokenizer(" "); - encoder.Encode(stringEncodingInputSmall, output, tokenizer); + encoder.Encode(stringEncodingInput, output, tokenizer); const DictionaryType& dictionary = encoder.Dictionary(); @@ -543,11 +542,29 @@ BOOST_AUTO_TEST_CASE(TfIdfEncodingTest) BOOST_REQUIRE_EQUAL(keysCount[keyValue.second], 1); } arma::mat expected = { - { 0.1192803136799, 0.0440228147639, 0.0440228147639, 0.0440228147639, 0, 0, 0, 0 }, - { 0, 0, 0, 0, 0.1590404182399, 0.1590404182399, 0.1590404182399, 0 }, - { 0, 0.0440228147639 , 0.0440228147639 , 0.0440228147639 , 0, 0, 0, 0.1192803136799 } + { 0.01100570369098, 0.01100570369098, 0.029820078419979, + 0.029820078419979, 0.029820078419979, 0, 0.029820078419979, + 0.01100570369098, 0.01100570369098, 0.01100570369098, 0.029820078419979, + 0.029820078419979, 0.029820078419979, 0, 0.029820078419979, + 0.029820078419979, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, + { 0, 0.0053360987592630683, 0, 0, 0, 0, 0, 0, 0.016008296277789203, + 0.016008296277789203, 0, 0, 0, 0, 0, 0, 0.014458219839989772, + 0.014458219839989772, 0.014458219839989772, 0.043374659519969, + 0.014458219839989772, 0.014458219839989772, 0.014458219839989772, + 0.014458219839989772, 0.014458219839989772, 0.014458219839989772, + 0.014458219839989772, 0.014458219839989772, 0.014458219839989772, + 0.014458219839989772, 0.014458219839989772, 0.014458219839989772, + 0.014458219839989772, 0.014458219839989772, 0.014458219839989772, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, + { 0.011739417270378749, 0, 0, 0, 0, 0, 0, 0.011739417270378749, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0.031808083647977492, 0.031808083647977492, 0.031808083647977492, + 0.031808083647977492, 0.031808083647977492, 0.031808083647977492, + 0.031808083647977492, 0.031808083647977492, 0.031808083647977492, + 0.031808083647977492, 0.031808083647977492 } }; - CheckMatrices(output, expected, 1e-10); + CheckMatrices(output, expected, 1e-12); } /** @@ -562,7 +579,7 @@ BOOST_AUTO_TEST_CASE(OnePassTfIdfEncodingTest) (TfIdfEncodingPolicy())); SplitByAnyOf tokenizer(" "); - encoder.Encode(stringEncodingInputSmall, output, tokenizer); + encoder.Encode(stringEncodingInput, output, tokenizer); const DictionaryType& dictionary = encoder.Dictionary(); @@ -576,13 +593,31 @@ BOOST_AUTO_TEST_CASE(OnePassTfIdfEncodingTest) } vector> expected = { - { 0.1192803136799, 0.0440228147639, 0.0440228147639, 0.0440228147639, 0, 0, 0, 0 }, - { 0, 0, 0, 0, 0.1590404182399, 0.1590404182399, 0.1590404182399, 0 }, - { 0, 0.0440228147639 , 0.0440228147639 , 0.0440228147639 , 0, 0, 0, 0.1192803136799 } + { 0.01100570369098, 0.01100570369098, 0.029820078419979, + 0.029820078419979, 0.029820078419979, 0, 0.029820078419979, + 0.01100570369098, 0.01100570369098, 0.01100570369098, 0.029820078419979, + 0.029820078419979, 0.029820078419979, 0, 0.029820078419979, + 0.029820078419979, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, + { 0, 0.0053360987592630683, 0, 0, 0, 0, 0, 0, 0.016008296277789203, + 0.016008296277789203, 0, 0, 0, 0, 0, 0, 0.014458219839989772, + 0.014458219839989772, 0.014458219839989772, 0.043374659519969, + 0.014458219839989772, 0.014458219839989772, 0.014458219839989772, + 0.014458219839989772, 0.014458219839989772, 0.014458219839989772, + 0.014458219839989772, 0.014458219839989772, 0.014458219839989772, + 0.014458219839989772, 0.014458219839989772, 0.014458219839989772, + 0.014458219839989772, 0.014458219839989772, 0.014458219839989772, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, + { 0.011739417270378749, 0, 0, 0, 0, 0, 0, 0.011739417270378749, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0.031808083647977492, 0.031808083647977492, 0.031808083647977492, + 0.031808083647977492, 0.031808083647977492, 0.031808083647977492, + 0.031808083647977492, 0.031808083647977492, 0.031808083647977492, + 0.031808083647977492, 0.031808083647977492 } }; for(size_t i=0;i> expected = { - { 0.0352182518111, 0, 0.0704365036223, 0, 0 }, - { 0, 0, 0.0503117883016, 0.0503117883016, 0.0681601792457 }, - { 0.0586970863519, 0, 0, 0.0586970863519, 0 } + { 0.0352182518111362474755310, 0, 0.0704365036222724949510621, 0, 0 }, + { 0, 0, 0.0503117883016232086967889, 0.0503117883016232086967889, + 0.0681601792456660582342209 }, + { 0.0586970863518937457925517, 0, 0, 0.0586970863518937457925517, 0 } }; for(size_t i=0;i Date: Wed, 7 Aug 2019 01:01:35 +0530 Subject: [PATCH 008/111] Added some different implimentation of tf-idf along with test --- .../tf_idf_encoding_policy.hpp | 42 ++- src/mlpack/tests/string_encoding_test.cpp | 298 ++++++++++++++---- 2 files changed, 283 insertions(+), 57 deletions(-) diff --git a/src/mlpack/core/data/string_encoding_policies/tf_idf_encoding_policy.hpp b/src/mlpack/core/data/string_encoding_policies/tf_idf_encoding_policy.hpp index 16b4cbbc29..aded2bf404 100644 --- a/src/mlpack/core/data/string_encoding_policies/tf_idf_encoding_policy.hpp +++ b/src/mlpack/core/data/string_encoding_policies/tf_idf_encoding_policy.hpp @@ -30,6 +30,11 @@ class TfIdfEncodingPolicy { public: + TfIdfEncodingPolicy(bool binary = false, bool smooth_idf = true, + bool sublinear_tf = false, bool raw_count = true, + bool term_frequency = false) : binary (binary), + smooth_idf (smooth_idf), sublinear_tf (sublinear_tf), + raw_count (raw_count), term_frequency (term_frequency) {} /** * The function initializes the output matrix. * @@ -80,8 +85,20 @@ class TfIdfEncodingPolicy void Encode(MatType& output, size_t value, size_t row, size_t /*col*/) { // Important since Mapping starts from 1 whereas allowed column value is 0. - output(row, value-1) = (tokenCount[row][value - 1] / row_size[row]) * - std::log10(output.n_rows / idfdict[value-1]); + double idf, tf; + if(smooth_idf) + idf = std::log((output.n_rows + 1) / (1 + idfdict[value-1])) + 1; + else + idf = std::log(output.n_rows / idfdict[value - 1]) + 1; + if (sublinear_tf) + tf = tokenCount[row][value - 1] / row_size[row]; + else if (term_frequency) + tf = std::log(tokenCount[row][value - 1]) + 1; + else if (binary) + tf = tokenCount[row][value - 1] > 0 ? 1 : 0; + else + tf = tokenCount[row][value - 1]; + output(row, value-1) = tf * idf; } /** @@ -99,8 +116,20 @@ class TfIdfEncodingPolicy size_t row, size_t /*col*/) { // Important since Mapping starts from 1 whereas allowed column value is 0. - output[row][value-1] = (tokenCount[row][value - 1] / row_size[row]) * - std::log10(output.size() / idfdict[value-1]); + double idf, tf; + if(smooth_idf) + idf = std::log((output.size() + 1) / (1 + idfdict[value-1])) + 1; + else + idf = std::log(output.size() / idfdict[value - 1]) + 1; + if (sublinear_tf) + tf = tokenCount[row][value - 1] / row_size[row]; + else if (term_frequency) + tf = std::log(tokenCount[row][value - 1]) + 1; + else if (binary) + tf = tokenCount[row][value - 1] > 0 ? 1 : 0; + else + tf = tokenCount[row][value - 1]; + output[row][value-1] = tf * idf; } /** @@ -137,6 +166,11 @@ class TfIdfEncodingPolicy std::vector> tokenCount; std::unordered_map idfdict; std::vector row_size; + bool binary; + bool smooth_idf; + bool sublinear_tf; + bool raw_count; + bool term_frequency; }; /** diff --git a/src/mlpack/tests/string_encoding_test.cpp b/src/mlpack/tests/string_encoding_test.cpp index 185e8820ef..1075eace4a 100644 --- a/src/mlpack/tests/string_encoding_test.cpp +++ b/src/mlpack/tests/string_encoding_test.cpp @@ -521,7 +521,7 @@ BOOST_AUTO_TEST_CASE(OnePassBagOfWordsEncodingIndividualCharactersTest) BOOST_REQUIRE(output == expected); } -BOOST_AUTO_TEST_CASE(TfIdfEncodingTest) +BOOST_AUTO_TEST_CASE(TfIdfrawcountsmoothidftrueEncodingTest) { using DictionaryType = StringEncodingDictionary; @@ -542,27 +542,24 @@ BOOST_AUTO_TEST_CASE(TfIdfEncodingTest) BOOST_REQUIRE_EQUAL(keysCount[keyValue.second], 1); } arma::mat expected = { - { 0.01100570369098, 0.01100570369098, 0.029820078419979, - 0.029820078419979, 0.029820078419979, 0, 0.029820078419979, - 0.01100570369098, 0.01100570369098, 0.01100570369098, 0.029820078419979, - 0.029820078419979, 0.029820078419979, 0, 0.029820078419979, - 0.029820078419979, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, - { 0, 0.0053360987592630683, 0, 0, 0, 0, 0, 0, 0.016008296277789203, - 0.016008296277789203, 0, 0, 0, 0, 0, 0, 0.014458219839989772, - 0.014458219839989772, 0.014458219839989772, 0.043374659519969, - 0.014458219839989772, 0.014458219839989772, 0.014458219839989772, - 0.014458219839989772, 0.014458219839989772, 0.014458219839989772, - 0.014458219839989772, 0.014458219839989772, 0.014458219839989772, - 0.014458219839989772, 0.014458219839989772, 0.014458219839989772, - 0.014458219839989772, 0.014458219839989772, 0.014458219839989772, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, - { 0.011739417270378749, 0, 0, 0, 0, 0, 0, 0.011739417270378749, 0, 0, 0, + { 1.28768207245178, 1.28768207245178, 1.69314718055995, 1.69314718055995, + 1.69314718055995, 1, 1.69314718055995, 1.28768207245178, + 1.28768207245178, 1.28768207245178, 1.69314718055995, 1.69314718055995, + 1.69314718055995, 1, 1.69314718055995, 1.69314718055995, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0.031808083647977492, 0.031808083647977492, 0.031808083647977492, - 0.031808083647977492, 0.031808083647977492, 0.031808083647977492, - 0.031808083647977492, 0.031808083647977492, 0.031808083647977492, - 0.031808083647977492, 0.031808083647977492 } + 0 }, + { 0, 1.28768207245178, 0, 0, 0, 2, 0, 0, 3.86304621735534, + 3.86304621735534, 0, 0, 0, 3, 0, 0, 1.69314718055995, 1.69314718055995, + 1.69314718055995, 5.07944154167984, 1.69314718055995, 1.69314718055995, + 1.69314718055995, 1.69314718055995, 1.69314718055995, 1.69314718055995, + 1.69314718055995, 1.69314718055995, 1.69314718055995, 1.69314718055995, + 1.69314718055995, 1.69314718055995, 1.69314718055995, 1.69314718055995, + 1.69314718055995, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, + { 1.28768207245178, 0, 0, 0, 0, 1, 0, 1.28768207245178, 0, 0, 0, 0, 0, 1, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 1.69314718055995, 1.69314718055995, 1.69314718055995, 1.69314718055995, + 1.69314718055995, 1.69314718055995, 1.69314718055995, 1.69314718055995, + 1.69314718055995, 1.69314718055995, 1.69314718055995 } }; CheckMatrices(output, expected, 1e-12); } @@ -570,7 +567,7 @@ BOOST_AUTO_TEST_CASE(TfIdfEncodingTest) /** * Test the one pass modification of the TfIdf encoding algorithm. */ -BOOST_AUTO_TEST_CASE(OnePassTfIdfEncodingTest) +BOOST_AUTO_TEST_CASE(OnePassTfIdfrawcountsmoothidftrueEncodingTest) { using DictionaryType = StringEncodingDictionary; @@ -593,27 +590,24 @@ BOOST_AUTO_TEST_CASE(OnePassTfIdfEncodingTest) } vector> expected = { - { 0.01100570369098, 0.01100570369098, 0.029820078419979, - 0.029820078419979, 0.029820078419979, 0, 0.029820078419979, - 0.01100570369098, 0.01100570369098, 0.01100570369098, 0.029820078419979, - 0.029820078419979, 0.029820078419979, 0, 0.029820078419979, - 0.029820078419979, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, - { 0, 0.0053360987592630683, 0, 0, 0, 0, 0, 0, 0.016008296277789203, - 0.016008296277789203, 0, 0, 0, 0, 0, 0, 0.014458219839989772, - 0.014458219839989772, 0.014458219839989772, 0.043374659519969, - 0.014458219839989772, 0.014458219839989772, 0.014458219839989772, - 0.014458219839989772, 0.014458219839989772, 0.014458219839989772, - 0.014458219839989772, 0.014458219839989772, 0.014458219839989772, - 0.014458219839989772, 0.014458219839989772, 0.014458219839989772, - 0.014458219839989772, 0.014458219839989772, 0.014458219839989772, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, - { 0.011739417270378749, 0, 0, 0, 0, 0, 0, 0.011739417270378749, 0, 0, 0, + { 1.28768207245178, 1.28768207245178, 1.69314718055995, 1.69314718055995, + 1.69314718055995, 1, 1.69314718055995, 1.28768207245178, + 1.28768207245178, 1.28768207245178, 1.69314718055995, 1.69314718055995, + 1.69314718055995, 1, 1.69314718055995, 1.69314718055995, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0.031808083647977492, 0.031808083647977492, 0.031808083647977492, - 0.031808083647977492, 0.031808083647977492, 0.031808083647977492, - 0.031808083647977492, 0.031808083647977492, 0.031808083647977492, - 0.031808083647977492, 0.031808083647977492 } + 0 }, + { 0, 1.28768207245178, 0, 0, 0, 2, 0, 0, 3.86304621735534, + 3.86304621735534, 0, 0, 0, 3, 0, 0, 1.69314718055995, 1.69314718055995, + 1.69314718055995, 5.07944154167984, 1.69314718055995, 1.69314718055995, + 1.69314718055995, 1.69314718055995, 1.69314718055995, 1.69314718055995, + 1.69314718055995, 1.69314718055995, 1.69314718055995, 1.69314718055995, + 1.69314718055995, 1.69314718055995, 1.69314718055995, 1.69314718055995, + 1.69314718055995, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, + { 1.28768207245178, 0, 0, 0, 0, 1, 0, 1.28768207245178, 0, 0, 0, 0, 0, 1, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 1.69314718055995, 1.69314718055995, 1.69314718055995, 1.69314718055995, + 1.69314718055995, 1.69314718055995, 1.69314718055995, 1.69314718055995, + 1.69314718055995, 1.69314718055995, 1.69314718055995 } }; for(size_t i=0;i input = { "GACCA", @@ -634,13 +628,12 @@ BOOST_AUTO_TEST_CASE(TfIdfEncodingIndividualCharactersTest) arma::mat output; TfIdfEncoding encoder; - // Passing a empty string to encode characters + // Passing a empty string to encode charactersrawcountsmoothidftrue encoder.Encode(input, output, CharExtract()); arma::mat target = { - { 0.0352182518111362474755310, 0, 0.0704365036222724949510621, 0, 0 }, - { 0, 0, 0.0503117883016232086967889, 0.0503117883016232086967889, - 0.0681601792456660582342209 }, - { 0.0586970863518937457925517, 0, 0, 0.0586970863518937457925517, 0 } + { 1.2876820724517808, 2, 2.5753641449035616, 0, 0 }, + { 0, 2, 2.5753641449035616, 2.5753641449035616, 1.6931471805599454 }, + { 1.2876820724517808, 1, 0, 1.2876820724517808, 0 } }; CheckMatrices(output, target, 1e-12); } @@ -649,7 +642,7 @@ BOOST_AUTO_TEST_CASE(TfIdfEncodingIndividualCharactersTest) * Test the one pass modification of the Bag of Words encoding algorithm * in case of individual character encoding. */ -BOOST_AUTO_TEST_CASE(OnePassTfIdfEncodingIndividualCharactersTest) +BOOST_AUTO_TEST_CASE(OnePassTfIdfrawcountsmoothidftrueEncodingIndividualCharactersTest) { std::vector input = { "GACCA", @@ -663,15 +656,214 @@ BOOST_AUTO_TEST_CASE(OnePassTfIdfEncodingIndividualCharactersTest) // Passing a empty string to encode characters encoder.Encode(input, output, CharExtract()); vector> expected = { - { 0.0352182518111362474755310, 0, 0.0704365036222724949510621, 0, 0 }, - { 0, 0, 0.0503117883016232086967889, 0.0503117883016232086967889, - 0.0681601792456660582342209 }, - { 0.0586970863518937457925517, 0, 0, 0.0586970863518937457925517, 0 } + { 1.2876820724517808, 2, 2.5753641449035616, 0, 0 }, + { 0, 2, 2.5753641449035616, 2.5753641449035616, 1.6931471805599454 }, + { 1.2876820724517808, 1, 0, 1.2876820724517808, 0 } }; for(size_t i=0;i; + + arma::mat output; + TfIdfEncoding encoder( + (TfIdfEncodingPolicy(false, false))); + SplitByAnyOf tokenizer(" "); + + encoder.Encode(stringEncodingInput, output, tokenizer); + + const DictionaryType& dictionary = encoder.Dictionary(); + + // Checking that everything is mapped to different numbers + std::unordered_map keysCount; + for (auto& keyValue : dictionary.Mapping()) + { + keysCount[keyValue.second]++; + // Every token should be mapped only once + BOOST_REQUIRE_EQUAL(keysCount[keyValue.second], 1); + } + + arma::mat expected = { + { 1.40546510810816, 1.40546510810816, 2.09861228866811, 2.09861228866811, + 2.09861228866811, 1, 2.09861228866811, 1.40546510810816, + 1.40546510810816, 1.40546510810816, 2.09861228866811, 2.09861228866811, + 2.09861228866811, 1, 2.09861228866811, 2.09861228866811, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0 }, + { 0, 1.40546510810816, 0, 0, 0, 2, 0, 0, 4.21639532432449, + 4.21639532432449, 0, 0, 0, 3, 0, 0, 2.09861228866811, 2.09861228866811, + 2.09861228866811, 6.29583686600433, 2.09861228866811, 2.09861228866811, + 2.09861228866811, 2.09861228866811, 2.09861228866811, 2.09861228866811, + 2.09861228866811, 2.09861228866811, 2.09861228866811, 2.09861228866811, + 2.09861228866811, 2.09861228866811, 2.09861228866811, 2.09861228866811, + 2.09861228866811, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, + { 1.40546510810816, 0, 0, 0, 0, 1, 0, 1.40546510810816, 0, 0, 0, 0, 0, 1, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 2.09861228866811, 2.09861228866811, 2.09861228866811, 2.09861228866811, + 2.09861228866811, 2.09861228866811, 2.09861228866811, 2.09861228866811, + 2.09861228866811, 2.09861228866811, 2.09861228866811 } + }; + CheckMatrices(output, expected, 1e-12); +} + +/** + * Test the one pass modification of the TfIdf encoding algorithm. + */ +BOOST_AUTO_TEST_CASE(OnePassTfIdfrawcountsmoothidffalseEncodingTest) +{ + using DictionaryType = StringEncodingDictionary; + + vector> output; + TfIdfEncoding encoder( + (TfIdfEncodingPolicy(false, false))); + SplitByAnyOf tokenizer(" "); + + encoder.Encode(stringEncodingInput, output, tokenizer); + + const DictionaryType& dictionary = encoder.Dictionary(); + + // Checking that everything is mapped to different numbers + std::unordered_map keysCount; + for (auto& keyValue : dictionary.Mapping()) + { + keysCount[keyValue.second]++; + // Every token should be mapped only once + BOOST_REQUIRE_EQUAL(keysCount[keyValue.second], 1); + } + + vector> expected = { + { 1.40546510810816, 1.40546510810816, 2.09861228866811, 2.09861228866811, + 2.09861228866811, 1, 2.09861228866811, 1.40546510810816, + 1.40546510810816, 1.40546510810816, 2.09861228866811, 2.09861228866811, + 2.09861228866811, 1, 2.09861228866811, 2.09861228866811, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0 }, + { 0, 1.40546510810816, 0, 0, 0, 2, 0, 0, 4.21639532432449, + 4.21639532432449, 0, 0, 0, 3, 0, 0, 2.09861228866811, 2.09861228866811, + 2.09861228866811, 6.29583686600433, 2.09861228866811, 2.09861228866811, + 2.09861228866811, 2.09861228866811, 2.09861228866811, 2.09861228866811, + 2.09861228866811, 2.09861228866811, 2.09861228866811, 2.09861228866811, + 2.09861228866811, 2.09861228866811, 2.09861228866811, 2.09861228866811, + 2.09861228866811, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, + { 1.40546510810816, 0, 0, 0, 0, 1, 0, 1.40546510810816, 0, 0, 0, 0, 0, 1, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 2.09861228866811, 2.09861228866811, 2.09861228866811, 2.09861228866811, + 2.09861228866811, 2.09861228866811, 2.09861228866811, 2.09861228866811, + 2.09861228866811, 2.09861228866811, 2.09861228866811 } + }; + for(size_t i=0;i input = { + "GACCA", + "ABCABCD", + "GAB" + }; + + arma::mat output; + TfIdfEncoding encoder( + (TfIdfEncodingPolicy(false, false))); + + // Passing a empty string to encode charactersrawcountsmoothidftrue + encoder.Encode(input, output, CharExtract()); + arma::mat target = { + { 1.4054651081081644, 2, 2.8109302162163288, 0, 0 }, + { 0, 2, 2.8109302162163288, 2.8109302162163288, 2.0986122886681100 }, + { 1.4054651081081644, 1, 0, 1.4054651081081644, 0 } + }; + CheckMatrices(output, target, 1e-12); +} + +/** + * Test the one pass modification of the Bag of Words encoding algorithm + * in case of individual character encoding. + */ +BOOST_AUTO_TEST_CASE(OnePassTfIdfrawcountsmoothidfalseEncodingIndividualCharactersTest) +{ + std::vector input = { + "GACCA", + "ABCABCD", + "GAB" + }; + + vector> output; + TfIdfEncoding encoder( + (TfIdfEncodingPolicy(false, false))); + + // Passing a empty string to encode characters + encoder.Encode(input, output, CharExtract()); + vector> expected = { + { 1.4054651081081644, 2, 2.8109302162163288, 0, 0 }, + { 0, 2, 2.8109302162163288, 2.8109302162163288, 2.0986122886681100 }, + { 1.4054651081081644, 1, 0, 1.4054651081081644, 0 } + }; + for(size_t i=0;i input = { + "GACCA", + "ABCABCD", + "GAB" + }; + + arma::mat output; + TfIdfEncoding encoder( + (TfIdfEncodingPolicy(true))); + + // Passing a empty string to encode charactersrawcountsmoothidftrue + encoder.Encode(input, output, CharExtract()); + arma::mat target = { + { 1.2876820724517808, 1, 1.2876820724517808, 0, 0 }, + { 0, 1, 1.2876820724517808, 1.2876820724517808, 1.6931471805599454 }, + { 1.2876820724517808, 1, 0, 1.2876820724517808, 0 } + }; + CheckMatrices(output, target, 1e-12); +} + +/** + * Test the one pass modification of the Bag of Words encoding algorithm + * in case of individual character encoding. + */ +BOOST_AUTO_TEST_CASE(OnePassTfIdfbnarysmoothidtrueEncodingIndividualCharactersTest) +{ + std::vector input = { + "GACCA", + "ABCABCD", + "GAB" + }; + + vector> output; + TfIdfEncoding encoder( + (TfIdfEncodingPolicy(true))); + + // Passing a empty string to encode characters + encoder.Encode(input, output, CharExtract()); + vector> expected = { + { 1.2876820724517808, 1, 1.2876820724517808, 0, 0 }, + { 0, 1, 1.2876820724517808, 1.2876820724517808, 1.6931471805599454 }, + { 1.2876820724517808, 1, 0, 1.2876820724517808, 0 } + }; + for(size_t i=0;i Date: Wed, 7 Aug 2019 01:14:30 +0530 Subject: [PATCH 009/111] renamed file bow_encoding_policy.hpp to bag_of_words_encoding_policy.hpp --- src/mlpack/core/data/string_encoding_policies/CMakeLists.txt | 2 +- ...bow_encoding_policy.hpp => bag_of_words_encoding_policy.hpp} | 2 +- src/mlpack/tests/string_encoding_test.cpp | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) rename src/mlpack/core/data/string_encoding_policies/{bow_encoding_policy.hpp => bag_of_words_encoding_policy.hpp} (99%) diff --git a/src/mlpack/core/data/string_encoding_policies/CMakeLists.txt b/src/mlpack/core/data/string_encoding_policies/CMakeLists.txt index ad88e9c0fc..b78b409453 100644 --- a/src/mlpack/core/data/string_encoding_policies/CMakeLists.txt +++ b/src/mlpack/core/data/string_encoding_policies/CMakeLists.txt @@ -2,7 +2,7 @@ # Anything not in this list will not be compiled into mlpack. set(SOURCES dictionary_encoding_policy.hpp - bow_encoding_policy.hpp + bag_of_words_encoding_policy.hpp tf_idf_encoding_policy.hpp policy_traits.hpp ) diff --git a/src/mlpack/core/data/string_encoding_policies/bow_encoding_policy.hpp b/src/mlpack/core/data/string_encoding_policies/bag_of_words_encoding_policy.hpp similarity index 99% rename from src/mlpack/core/data/string_encoding_policies/bow_encoding_policy.hpp rename to src/mlpack/core/data/string_encoding_policies/bag_of_words_encoding_policy.hpp index 9660c939e9..2ba40ede27 100644 --- a/src/mlpack/core/data/string_encoding_policies/bow_encoding_policy.hpp +++ b/src/mlpack/core/data/string_encoding_policies/bag_of_words_encoding_policy.hpp @@ -1,5 +1,5 @@ /** - * @file bow_encoding_policy.hpp + * @file bag_of_words_encoding_policy.hpp * @author Jeffin Sam * * Definition of the BagOfWordsEncodingPolicy class. diff --git a/src/mlpack/tests/string_encoding_test.cpp b/src/mlpack/tests/string_encoding_test.cpp index 1075eace4a..a3c0f5e392 100644 --- a/src/mlpack/tests/string_encoding_test.cpp +++ b/src/mlpack/tests/string_encoding_test.cpp @@ -15,7 +15,7 @@ #include #include #include -#include +#include #include #include #include From 195461fe172ef454b46579821d8fd276a0eaf00a Mon Sep 17 00:00:00 2001 From: jeffinsam Date: Wed, 7 Aug 2019 07:48:11 +0530 Subject: [PATCH 010/111] use enum for tfTypes instead of bool values --- .../tf_idf_encoding_policy.hpp | 32 ++-- src/mlpack/tests/string_encoding_test.cpp | 160 ++++++++++++++++-- 2 files changed, 162 insertions(+), 30 deletions(-) diff --git a/src/mlpack/core/data/string_encoding_policies/tf_idf_encoding_policy.hpp b/src/mlpack/core/data/string_encoding_policies/tf_idf_encoding_policy.hpp index aded2bf404..90e5e2ecc9 100644 --- a/src/mlpack/core/data/string_encoding_policies/tf_idf_encoding_policy.hpp +++ b/src/mlpack/core/data/string_encoding_policies/tf_idf_encoding_policy.hpp @@ -29,12 +29,17 @@ namespace data { class TfIdfEncodingPolicy { public: + + enum tfTypes + { + rawCount, + binary, + sublinearTf, + termFrequency, + }; - TfIdfEncodingPolicy(bool binary = false, bool smooth_idf = true, - bool sublinear_tf = false, bool raw_count = true, - bool term_frequency = false) : binary (binary), - smooth_idf (smooth_idf), sublinear_tf (sublinear_tf), - raw_count (raw_count), term_frequency (term_frequency) {} + TfIdfEncodingPolicy(size_t tfType = 0, bool smooth_idf = true) : + tfType (tfType), smooth_idf (smooth_idf) {} /** * The function initializes the output matrix. * @@ -90,11 +95,11 @@ class TfIdfEncodingPolicy idf = std::log((output.n_rows + 1) / (1 + idfdict[value-1])) + 1; else idf = std::log(output.n_rows / idfdict[value - 1]) + 1; - if (sublinear_tf) + if (tfType == tfTypes::termFrequency) tf = tokenCount[row][value - 1] / row_size[row]; - else if (term_frequency) + else if (tfType == tfTypes::sublinearTf) tf = std::log(tokenCount[row][value - 1]) + 1; - else if (binary) + else if (tfType == tfTypes::binary) tf = tokenCount[row][value - 1] > 0 ? 1 : 0; else tf = tokenCount[row][value - 1]; @@ -121,11 +126,11 @@ class TfIdfEncodingPolicy idf = std::log((output.size() + 1) / (1 + idfdict[value-1])) + 1; else idf = std::log(output.size() / idfdict[value - 1]) + 1; - if (sublinear_tf) + if (tfType == tfTypes::termFrequency) tf = tokenCount[row][value - 1] / row_size[row]; - else if (term_frequency) + else if (tfType == tfTypes::sublinearTf) tf = std::log(tokenCount[row][value - 1]) + 1; - else if (binary) + else if (tfType == tfTypes::binary) tf = tokenCount[row][value - 1] > 0 ? 1 : 0; else tf = tokenCount[row][value - 1]; @@ -166,11 +171,8 @@ class TfIdfEncodingPolicy std::vector> tokenCount; std::unordered_map idfdict; std::vector row_size; - bool binary; bool smooth_idf; - bool sublinear_tf; - bool raw_count; - bool term_frequency; + size_t tfType; }; /** diff --git a/src/mlpack/tests/string_encoding_test.cpp b/src/mlpack/tests/string_encoding_test.cpp index a3c0f5e392..97cd82ba7c 100644 --- a/src/mlpack/tests/string_encoding_test.cpp +++ b/src/mlpack/tests/string_encoding_test.cpp @@ -521,7 +521,7 @@ BOOST_AUTO_TEST_CASE(OnePassBagOfWordsEncodingIndividualCharactersTest) BOOST_REQUIRE(output == expected); } -BOOST_AUTO_TEST_CASE(TfIdfrawcountsmoothidftrueEncodingTest) +BOOST_AUTO_TEST_CASE(RawCountSmoothIdfEncodingTest) { using DictionaryType = StringEncodingDictionary; @@ -567,7 +567,7 @@ BOOST_AUTO_TEST_CASE(TfIdfrawcountsmoothidftrueEncodingTest) /** * Test the one pass modification of the TfIdf encoding algorithm. */ -BOOST_AUTO_TEST_CASE(OnePassTfIdfrawcountsmoothidftrueEncodingTest) +BOOST_AUTO_TEST_CASE(OnePassRawCountSmoothIdfEncodingTest) { using DictionaryType = StringEncodingDictionary; @@ -617,7 +617,7 @@ BOOST_AUTO_TEST_CASE(OnePassTfIdfrawcountsmoothidftrueEncodingTest) /** * Test TFIDF encoding for characters using lamda function. */ -BOOST_AUTO_TEST_CASE(TfIdfrawcountsmoothidftrueEncodingIndividualCharactersTest) +BOOST_AUTO_TEST_CASE(RawCountSmoothIdfEncodingIndividualCharactersTest) { vector input = { "GACCA", @@ -642,7 +642,7 @@ BOOST_AUTO_TEST_CASE(TfIdfrawcountsmoothidftrueEncodingIndividualCharactersTest) * Test the one pass modification of the Bag of Words encoding algorithm * in case of individual character encoding. */ -BOOST_AUTO_TEST_CASE(OnePassTfIdfrawcountsmoothidftrueEncodingIndividualCharactersTest) +BOOST_AUTO_TEST_CASE(OnePassRawCountSmoothIdfEncodingIndividualCharactersTest) { std::vector input = { "GACCA", @@ -665,13 +665,13 @@ BOOST_AUTO_TEST_CASE(OnePassTfIdfrawcountsmoothidftrueEncodingIndividualCharacte BOOST_REQUIRE_CLOSE(expected[i][j], output[i][j], 1e-12); } -BOOST_AUTO_TEST_CASE(TfIdfrawcountsmoothidffalseEncodingTest) +BOOST_AUTO_TEST_CASE(TfIdfRawCountEncodingTest) { using DictionaryType = StringEncodingDictionary; arma::mat output; TfIdfEncoding encoder( - (TfIdfEncodingPolicy(false, false))); + (TfIdfEncodingPolicy(0, false))); SplitByAnyOf tokenizer(" "); encoder.Encode(stringEncodingInput, output, tokenizer); @@ -713,13 +713,13 @@ BOOST_AUTO_TEST_CASE(TfIdfrawcountsmoothidffalseEncodingTest) /** * Test the one pass modification of the TfIdf encoding algorithm. */ -BOOST_AUTO_TEST_CASE(OnePassTfIdfrawcountsmoothidffalseEncodingTest) +BOOST_AUTO_TEST_CASE(OnePassTfIdfRawCountEncodingTest) { using DictionaryType = StringEncodingDictionary; vector> output; TfIdfEncoding encoder( - (TfIdfEncodingPolicy(false, false))); + (TfIdfEncodingPolicy(0, false))); SplitByAnyOf tokenizer(" "); encoder.Encode(stringEncodingInput, output, tokenizer); @@ -763,7 +763,7 @@ BOOST_AUTO_TEST_CASE(OnePassTfIdfrawcountsmoothidffalseEncodingTest) /** * Test TFIDF encoding for characters using lamda function. */ -BOOST_AUTO_TEST_CASE(TfIdfrawcountsmoothidffalseEncodingIndividualCharactersTest) +BOOST_AUTO_TEST_CASE(RawcountTfIdfEncodingIndividualCharactersTest) { vector input = { "GACCA", @@ -773,7 +773,7 @@ BOOST_AUTO_TEST_CASE(TfIdfrawcountsmoothidffalseEncodingIndividualCharactersTest arma::mat output; TfIdfEncoding encoder( - (TfIdfEncodingPolicy(false, false))); + (TfIdfEncodingPolicy(0, false))); // Passing a empty string to encode charactersrawcountsmoothidftrue encoder.Encode(input, output, CharExtract()); @@ -789,7 +789,7 @@ BOOST_AUTO_TEST_CASE(TfIdfrawcountsmoothidffalseEncodingIndividualCharactersTest * Test the one pass modification of the Bag of Words encoding algorithm * in case of individual character encoding. */ -BOOST_AUTO_TEST_CASE(OnePassTfIdfrawcountsmoothidfalseEncodingIndividualCharactersTest) +BOOST_AUTO_TEST_CASE(OnePassRawcountEncodingIndividualCharactersTest) { std::vector input = { "GACCA", @@ -816,7 +816,7 @@ BOOST_AUTO_TEST_CASE(OnePassTfIdfrawcountsmoothidfalseEncodingIndividualCharacte /** * Test TFIDF encoding for characters using lamda function. */ -BOOST_AUTO_TEST_CASE(TfIdfbinarysmoothidftrueEncodingIndividualCharactersTest) +BOOST_AUTO_TEST_CASE(BinarySmoothIdfEncodingIndividualCharactersTest) { vector input = { "GACCA", @@ -826,7 +826,7 @@ BOOST_AUTO_TEST_CASE(TfIdfbinarysmoothidftrueEncodingIndividualCharactersTest) arma::mat output; TfIdfEncoding encoder( - (TfIdfEncodingPolicy(true))); + (TfIdfEncodingPolicy(1, true))); // Passing a empty string to encode charactersrawcountsmoothidftrue encoder.Encode(input, output, CharExtract()); @@ -842,7 +842,7 @@ BOOST_AUTO_TEST_CASE(TfIdfbinarysmoothidftrueEncodingIndividualCharactersTest) * Test the one pass modification of the Bag of Words encoding algorithm * in case of individual character encoding. */ -BOOST_AUTO_TEST_CASE(OnePassTfIdfbnarysmoothidtrueEncodingIndividualCharactersTest) +BOOST_AUTO_TEST_CASE(OnePassBnarySmoothIdfEncodingIndividualCharactersTest) { std::vector input = { "GACCA", @@ -852,7 +852,7 @@ BOOST_AUTO_TEST_CASE(OnePassTfIdfbnarysmoothidtrueEncodingIndividualCharactersTe vector> output; TfIdfEncoding encoder( - (TfIdfEncodingPolicy(true))); + (TfIdfEncodingPolicy(1, true))); // Passing a empty string to encode characters encoder.Encode(input, output, CharExtract()); @@ -865,5 +865,135 @@ BOOST_AUTO_TEST_CASE(OnePassTfIdfbnarysmoothidtrueEncodingIndividualCharactersTe for(size_t j=0;j input = { + "GACCA", + "ABCABCD", + "GAB" + }; + + arma::mat output; + TfIdfEncoding encoder( + (TfIdfEncodingPolicy(1, false))); + + // Passing a empty string to encode charactersrawcountsmoothidftrue + encoder.Encode(input, output, CharExtract()); + arma::mat target = { + { 1.4054651081081644, 1, 1.4054651081081644, 0, 0 }, + { 0, 1, 1.4054651081081644, 1.4054651081081644, 2.0986122886681100 }, + { 1.4054651081081644, 1, 0, 1.4054651081081644, 0 } + }; + CheckMatrices(output, target, 1e-12); +} + +/** +* Test TFIDF encoding for characters using lamda function. +*/ +BOOST_AUTO_TEST_CASE(SublinearSmoothIdfEncodingIndividualCharactersTest) +{ + vector input = { + "GACCA", + "ABCABCD", + "GAB" + }; + + arma::mat output; + TfIdfEncoding encoder( + (TfIdfEncodingPolicy(2, true))); + + // Passing a empty string to encode charactersrawcountsmoothidftrue + encoder.Encode(input, output, CharExtract()); + arma::mat target = { + { 1.2876820724517808, 1.6931471805599454, 2.1802352704293200, 0, 0 }, + { 0, 1.6931471805599454, 2.1802352704293200, 2.1802352704293200, + 1.6931471805599454 }, + { 1.2876820724517808, 1, 0, 1.2876820724517808, 0 } + }; + CheckMatrices(output, target, 1e-12); +} + +/** +* Test TFIDF encoding for characters using lamda function. +*/ +BOOST_AUTO_TEST_CASE(SublinearTfIdfEncodingIndividualCharactersTest) +{ + vector input = { + "GACCA", + "ABCABCD", + "GAB" + }; + + arma::mat output; + TfIdfEncoding encoder( + (TfIdfEncodingPolicy(2, false))); + + // Passing a empty string to encode charactersrawcountsmoothidftrue + encoder.Encode(input, output, CharExtract()); + arma::mat target = { + { 1.4054651081081644, 1.6931471805599454, 2.3796592851687173, 0, 0 }, + { 0, 1.6931471805599454, 2.3796592851687173, 2.3796592851687173, + 2.0986122886681100 }, + { 1.4054651081081644, 1, 0, 1.4054651081081644, 0 } + }; + CheckMatrices(output, target, 1e-12); +} + +/** +* Test TFIDF encoding for characters using lamda function. +*/ +BOOST_AUTO_TEST_CASE(TermFrequencySmoothIdfEncodingIndividualCharactersTest) +{ + vector input = { + "GACCA", + "ABCABCD", + "GAB" + }; + + arma::mat output; + TfIdfEncoding encoder( + (TfIdfEncodingPolicy(3, true))); + + // Passing a empty string to encode charactersrawcountsmoothidftrue + encoder.Encode(input, output, CharExtract()); + arma::mat target = { + { 0.2575364144903562, 0.4, 0.5150728289807124, 0, 0 }, + { 0, 0.2857142857142857, 0.3679091635576516, 0.3679091635576516, + 0.2418781686514208 }, + { 0.4292273574839269, 0.3333333333333333, 0, 0.4292273574839269, 0 } + }; + CheckMatrices(output, target, 1e-12); +} + +/** +* Test TFIDF encoding for characters using lamda function. +*/ +BOOST_AUTO_TEST_CASE(TermFrequencyTfIdfEncodingIndividualCharactersTest) +{ + vector input = { + "GACCA", + "ABCABCD", + "GAB" + }; + + arma::mat output; + TfIdfEncoding encoder( + (TfIdfEncodingPolicy(3, false))); + + // Passing a empty string to encode charactersrawcountsmoothidftrue + encoder.Encode(input, output, CharExtract()); + arma::mat target = { + { 0.2810930216216329, 0.4, 0.5621860432432658, 0, 0 }, + { 0, 0.2857142857142857, 0.4015614594594755, 0.4015614594594755, + 0.2998017555240157 }, + { 0.4684883693693881, 0.3333333333333333, 0, 0.4684883693693881, 0 } + }; + CheckMatrices(output, target, 1e-12); +} + BOOST_AUTO_TEST_SUITE_END(); From 2e615024d3e2af066c48d040005327ddb1dc1c78 Mon Sep 17 00:00:00 2001 From: jeffinsam Date: Wed, 7 Aug 2019 11:23:56 +0530 Subject: [PATCH 011/111] Done with the documetnation and adapted test accordingly --- .../bag_of_words_encoding_policy.hpp | 25 ++++--- .../dictionary_encoding_policy.hpp | 10 ++- .../tf_idf_encoding_policy.hpp | 56 ++++++++++++---- src/mlpack/tests/string_encoding_test.cpp | 66 ++++++++++++------- 4 files changed, 106 insertions(+), 51 deletions(-) diff --git a/src/mlpack/core/data/string_encoding_policies/bag_of_words_encoding_policy.hpp b/src/mlpack/core/data/string_encoding_policies/bag_of_words_encoding_policy.hpp index 2ba40ede27..e692cfce1b 100644 --- a/src/mlpack/core/data/string_encoding_policies/bag_of_words_encoding_policy.hpp +++ b/src/mlpack/core/data/string_encoding_policies/bag_of_words_encoding_policy.hpp @@ -21,11 +21,10 @@ namespace data { /** * Definition of the BagOfWordsEncodingPolicy class. * - * DicitonaryEnocding is used as a helper class for StringEncoding. - * The encoder assigns a positive integer number to each unique token and treat - * the dataset as categorical. The numbers are assigned sequentially starting - * from one. The tokens are labeled in the order of their occurrence - * in the input dataset. + * BagOfWords is used as a helper class for StringEncoding. + * The encoder create a vector of all the unique token and then assigns + * 1 if the token is present in the document, 0 if not present. The tokens + * are labeled in the order of their occurrence in the input dataset. */ class BagOfWordsEncodingPolicy { @@ -77,7 +76,10 @@ class BagOfWordsEncodingPolicy * @param col The row token number at which the encoding is performed. */ template - static void Encode(MatType& output, size_t value, size_t row, size_t /*col*/) + static void Encode(MatType& output, + size_t value, + size_t row, + size_t /*col*/) { // Important since Mapping starts from 1 whereas allowed column value is 0. output(row, value-1) = 1; @@ -94,8 +96,10 @@ class BagOfWordsEncodingPolicy * @param col The row token number at which the encoding is performed. */ template - static void Encode(std::vector >& output, size_t value, - size_t row, size_t /*col*/) + static void Encode(std::vector >& output, + size_t value, + size_t row, + size_t /*col*/) { // Important since Mapping starts from 1 whereas allowed column value is 0. output[row][value-1] = 1; @@ -117,8 +121,9 @@ class BagOfWordsEncodingPolicy * @param numToken The count of token parsed till now. * @param value The encoded token. */ - static void PreprocessToken(size_t /*row*/, size_t /*numTokens*/, - size_t /*value*/) { } + static void PreprocessToken(size_t /*row*/, + size_t /*numTokens*/, + size_t /*value*/) { } }; /** diff --git a/src/mlpack/core/data/string_encoding_policies/dictionary_encoding_policy.hpp b/src/mlpack/core/data/string_encoding_policies/dictionary_encoding_policy.hpp index f11e7bb566..9c972fd856 100644 --- a/src/mlpack/core/data/string_encoding_policies/dictionary_encoding_policy.hpp +++ b/src/mlpack/core/data/string_encoding_policies/dictionary_encoding_policy.hpp @@ -58,7 +58,10 @@ class DictionaryEncodingPolicy * @param col The row token number at which the encoding is performed. */ template - static void Encode(MatType& output, size_t value, size_t row, size_t col) + static void Encode(MatType& output, + size_t value, + size_t row, + size_t col) { output(row, col) = value; } @@ -84,8 +87,9 @@ class DictionaryEncodingPolicy * @param numToken The count of token parsed till now. * @param value The encoded token. */ - static void PreprocessToken(size_t /*row*/, size_t /*numTokens*/, - size_t /*value*/) { } + static void PreprocessToken(size_t /*row*/, + size_t /*numTokens*/, + size_t /*value*/) {} /** * Serialize the class to the given archive. diff --git a/src/mlpack/core/data/string_encoding_policies/tf_idf_encoding_policy.hpp b/src/mlpack/core/data/string_encoding_policies/tf_idf_encoding_policy.hpp index 90e5e2ecc9..4cdf8095e7 100644 --- a/src/mlpack/core/data/string_encoding_policies/tf_idf_encoding_policy.hpp +++ b/src/mlpack/core/data/string_encoding_policies/tf_idf_encoding_policy.hpp @@ -20,16 +20,33 @@ namespace data { /** * Definition of the TfIdfEncodingPolicy class. * - * DicitonaryEnocding is used as a helper class for StringEncoding. - * The encoder assigns a positive integer number to each unique token and treat - * the dataset as categorical. The numbers are assigned sequentially starting - * from one. The tokens are labeled in the order of their occurrence - * in the input dataset. + * Tf means term-frequency while tf-idf means term-frequency times inverse + * document-frequency. This is a common term weighting scheme in information + * retrieval, that has also found good use in document classification. + * The goal of using tf-idf instead of the raw frequencies of occurrence of a + * token in a given document is to scale down the impact of tokens that occur + * very frequently in a given corpus and that are hence empirically less + * informative than features that occur in a small fraction of the training + * corpus. + * TfIdfEncodingPolicy is used as a helper class for StringEncoding. + * The encoder assigns a tf-idf number to each unique token and treat + * the dataset as categorical. The tokens are labeled in the order of their + * occurrence in the input dataset. */ class TfIdfEncodingPolicy { public: + /* + * Enum Class used to identify the type of tf encoding + * + * Follwing are the defination of the types + * binary : binary weighting scheme (0,1) + * rawCount : raw count weighting scheme (count of token for every row) + * termFrequency : term frequency weighting scheme (count / length(row)) + * subinerTf : logarthimic weighting scheme (log(tf) + 1) + * + */ enum tfTypes { rawCount, @@ -87,12 +104,15 @@ class TfIdfEncodingPolicy * @param col The row token number at which the encoding is performed. */ template - void Encode(MatType& output, size_t value, size_t row, size_t /*col*/) + void Encode(MatType& output, + size_t value, + size_t row, + size_t /*col*/) { // Important since Mapping starts from 1 whereas allowed column value is 0. double idf, tf; if(smooth_idf) - idf = std::log((output.n_rows + 1) / (1 + idfdict[value-1])) + 1; + idf = std::log((output.n_rows + 1) / (1 + idfdict[value - 1])) + 1; else idf = std::log(output.n_rows / idfdict[value - 1]) + 1; if (tfType == tfTypes::termFrequency) @@ -117,13 +137,15 @@ class TfIdfEncodingPolicy * @param col The row token number at which the encoding is performed. */ template - void Encode(std::vector >& output, size_t value, - size_t row, size_t /*col*/) + void Encode(std::vector >& output, + size_t value, + size_t row, + size_t /*col*/) { // Important since Mapping starts from 1 whereas allowed column value is 0. double idf, tf; if(smooth_idf) - idf = std::log((output.size() + 1) / (1 + idfdict[value-1])) + 1; + idf = std::log((output.size() + 1) / (1 + idfdict[value - 1])) + 1; else idf = std::log(output.size() / idfdict[value - 1]) + 1; if (tfType == tfTypes::termFrequency) @@ -154,8 +176,9 @@ class TfIdfEncodingPolicy * @param numToken The count of token parsed till now. * @param value The encoded token. */ - void PreprocessToken(size_t row, size_t /*numTokens*/, - size_t value) + void PreprocessToken(size_t row, + size_t /*numTokens*/, + size_t value) { if(row>=tokenCount.size()) { @@ -163,15 +186,20 @@ class TfIdfEncodingPolicy tokenCount.push_back(std::unordered_map()); } tokenCount.back()[value-1]++; - if(tokenCount.back()[value-1]==1) - idfdict[value-1]++; + if(tokenCount.back()[value - 1]==1) + idfdict[value - 1]++; row_size.back()++; } private: + // Used to store the count of token for each row. std::vector> tokenCount; + // Used to store the idf values. std::unordered_map idfdict; + // Used to store the number of tokens in each row. std::vector row_size; + // smooth_idf variable to indicate smoothining. bool smooth_idf; + // Type of Term Frequency to use. size_t tfType; }; diff --git a/src/mlpack/tests/string_encoding_test.cpp b/src/mlpack/tests/string_encoding_test.cpp index 97cd82ba7c..544e8fe160 100644 --- a/src/mlpack/tests/string_encoding_test.cpp +++ b/src/mlpack/tests/string_encoding_test.cpp @@ -521,6 +521,10 @@ BOOST_AUTO_TEST_CASE(OnePassBagOfWordsEncodingIndividualCharactersTest) BOOST_REQUIRE(output == expected); } +/** + * Test the Tf-Idf Encoding using rawcount type and smoothidf as true, + * which is the deafult values used for algorithim. + */ BOOST_AUTO_TEST_CASE(RawCountSmoothIdfEncodingTest) { using DictionaryType = StringEncodingDictionary; @@ -565,7 +569,8 @@ BOOST_AUTO_TEST_CASE(RawCountSmoothIdfEncodingTest) } /** - * Test the one pass modification of the TfIdf encoding algorithm. + * Test the one pass modification of the TfIdf encoding algorithm, using rawcount + * as type of tf and smoothIdf as true. */ BOOST_AUTO_TEST_CASE(OnePassRawCountSmoothIdfEncodingTest) { @@ -615,8 +620,9 @@ BOOST_AUTO_TEST_CASE(OnePassRawCountSmoothIdfEncodingTest) } /** -* Test TFIDF encoding for characters using lamda function. -*/ + * Test TFIDF encoding for characters using lamda function, using rawcount as tf + * type and smoothidf as true. + */ BOOST_AUTO_TEST_CASE(RawCountSmoothIdfEncodingIndividualCharactersTest) { vector input = { @@ -639,8 +645,8 @@ BOOST_AUTO_TEST_CASE(RawCountSmoothIdfEncodingIndividualCharactersTest) } /** - * Test the one pass modification of the Bag of Words encoding algorithm - * in case of individual character encoding. + * Test the one pass modification of the Tf-Idf encoding algorithm + * in case of individual character encoding using default values. */ BOOST_AUTO_TEST_CASE(OnePassRawCountSmoothIdfEncodingIndividualCharactersTest) { @@ -665,6 +671,9 @@ BOOST_AUTO_TEST_CASE(OnePassRawCountSmoothIdfEncodingIndividualCharactersTest) BOOST_REQUIRE_CLOSE(expected[i][j], output[i][j], 1e-12); } +/** + * Test the Tf-Idf Encoding using rawcount type and smoothidf as false. + */ BOOST_AUTO_TEST_CASE(TfIdfRawCountEncodingTest) { using DictionaryType = StringEncodingDictionary; @@ -711,7 +720,8 @@ BOOST_AUTO_TEST_CASE(TfIdfRawCountEncodingTest) } /** - * Test the one pass modification of the TfIdf encoding algorithm. + * Test the one pass modification of the TfIdf encoding algorithm, with rawcount + * as type, but with smoothidf as false. */ BOOST_AUTO_TEST_CASE(OnePassTfIdfRawCountEncodingTest) { @@ -761,8 +771,9 @@ BOOST_AUTO_TEST_CASE(OnePassTfIdfRawCountEncodingTest) } /** -* Test TFIDF encoding for characters using lamda function. -*/ + * Test TFIDF encoding for characters using lamda function, using rawcount as + * tf type and smoothidf as false. + */ BOOST_AUTO_TEST_CASE(RawcountTfIdfEncodingIndividualCharactersTest) { vector input = { @@ -786,8 +797,9 @@ BOOST_AUTO_TEST_CASE(RawcountTfIdfEncodingIndividualCharactersTest) } /** - * Test the one pass modification of the Bag of Words encoding algorithm - * in case of individual character encoding. + * Test the one pass modification of the Tf Idf encoding algorithm + * in case of individual character encoding, using raw count as type, + * and smoothidf as false. */ BOOST_AUTO_TEST_CASE(OnePassRawcountEncodingIndividualCharactersTest) { @@ -814,8 +826,9 @@ BOOST_AUTO_TEST_CASE(OnePassRawcountEncodingIndividualCharactersTest) } /** -* Test TFIDF encoding for characters using lamda function. -*/ + * Test TFIDF encoding for characters using lamda function, using binary + * weighting scheme for tf and smoothidf as true. + */ BOOST_AUTO_TEST_CASE(BinarySmoothIdfEncodingIndividualCharactersTest) { vector input = { @@ -839,8 +852,8 @@ BOOST_AUTO_TEST_CASE(BinarySmoothIdfEncodingIndividualCharactersTest) } /** - * Test the one pass modification of the Bag of Words encoding algorithm - * in case of individual character encoding. + * Test TFIDF encoding for characters using lamda function, using binary + * weighting scheme for tf and smoothidf as true. */ BOOST_AUTO_TEST_CASE(OnePassBnarySmoothIdfEncodingIndividualCharactersTest) { @@ -867,8 +880,9 @@ BOOST_AUTO_TEST_CASE(OnePassBnarySmoothIdfEncodingIndividualCharactersTest) } /** -* Test TFIDF encoding for characters using lamda function. -*/ + * Test TFIDF encoding for characters using lamda function, using binary + * as weighting scheme and smoothidf as false. + */ BOOST_AUTO_TEST_CASE(BinaryTfIdfEncodingIndividualCharactersTest) { vector input = { @@ -892,8 +906,9 @@ BOOST_AUTO_TEST_CASE(BinaryTfIdfEncodingIndividualCharactersTest) } /** -* Test TFIDF encoding for characters using lamda function. -*/ + * Test TFIDF encoding for characters using lamda function, using sublinear + * as weighting scheme and smoothidf as true. + */ BOOST_AUTO_TEST_CASE(SublinearSmoothIdfEncodingIndividualCharactersTest) { vector input = { @@ -918,8 +933,9 @@ BOOST_AUTO_TEST_CASE(SublinearSmoothIdfEncodingIndividualCharactersTest) } /** -* Test TFIDF encoding for characters using lamda function. -*/ + * Test TFIDF encoding for characters using lamda function, using sublinear + * as weighting scheme and smoothidf as false. + */ BOOST_AUTO_TEST_CASE(SublinearTfIdfEncodingIndividualCharactersTest) { vector input = { @@ -944,8 +960,9 @@ BOOST_AUTO_TEST_CASE(SublinearTfIdfEncodingIndividualCharactersTest) } /** -* Test TFIDF encoding for characters using lamda function. -*/ + * Test TFIDF encoding for characters using lamda function, using term + * Frequency as weighting scheme and smoothidf as true. + */ BOOST_AUTO_TEST_CASE(TermFrequencySmoothIdfEncodingIndividualCharactersTest) { vector input = { @@ -970,8 +987,9 @@ BOOST_AUTO_TEST_CASE(TermFrequencySmoothIdfEncodingIndividualCharactersTest) } /** -* Test TFIDF encoding for characters using lamda function. -*/ + * Test TFIDF encoding for characters using lamda function, using Term + * Frequency as weighting scheme and smoothidf as false. + */ BOOST_AUTO_TEST_CASE(TermFrequencyTfIdfEncodingIndividualCharactersTest) { vector input = { From efdd833b0aca76394eaa4ac70d8f9f848644e48b Mon Sep 17 00:00:00 2001 From: jeffinsam Date: Wed, 7 Aug 2019 11:38:54 +0530 Subject: [PATCH 012/111] Style fixes --- .../bag_of_words_encoding_policy.hpp | 2 +- .../tf_idf_encoding_policy.hpp | 16 ++++++++------ src/mlpack/tests/string_encoding_test.cpp | 22 +++++++++---------- 3 files changed, 21 insertions(+), 19 deletions(-) diff --git a/src/mlpack/core/data/string_encoding_policies/bag_of_words_encoding_policy.hpp b/src/mlpack/core/data/string_encoding_policies/bag_of_words_encoding_policy.hpp index e692cfce1b..fdfc26bab9 100644 --- a/src/mlpack/core/data/string_encoding_policies/bag_of_words_encoding_policy.hpp +++ b/src/mlpack/core/data/string_encoding_policies/bag_of_words_encoding_policy.hpp @@ -146,4 +146,4 @@ using BagOfWordsEncoding = StringEncoding=tokenCount.size()) + if (row >= tokenCount.size()) { row_size.push_back(0); tokenCount.push_back(std::unordered_map()); } tokenCount.back()[value-1]++; - if(tokenCount.back()[value - 1]==1) + if (tokenCount.back()[value - 1] == 1) idfdict[value - 1]++; row_size.back()++; } @@ -223,4 +225,4 @@ using TfIdfEncoding = StringEncoding Date: Wed, 7 Aug 2019 11:47:16 +0530 Subject: [PATCH 013/111] style fixes part2 --- .../bag_of_words_encoding_policy.hpp | 2 +- .../tf_idf_encoding_policy.hpp | 11 +++++------ src/mlpack/tests/string_encoding_test.cpp | 2 +- 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/src/mlpack/core/data/string_encoding_policies/bag_of_words_encoding_policy.hpp b/src/mlpack/core/data/string_encoding_policies/bag_of_words_encoding_policy.hpp index fdfc26bab9..87656a1c4c 100644 --- a/src/mlpack/core/data/string_encoding_policies/bag_of_words_encoding_policy.hpp +++ b/src/mlpack/core/data/string_encoding_policies/bag_of_words_encoding_policy.hpp @@ -146,4 +146,4 @@ using BagOfWordsEncoding = StringEncoding= tokenCount.size()) { @@ -225,4 +224,4 @@ using TfIdfEncoding = StringEncoding Date: Thu, 8 Aug 2019 19:05:23 +0530 Subject: [PATCH 014/111] resolve comments regarding test, have to add test for serialisation --- .../tf_idf_encoding_policy.hpp | 44 +++++++++++-------- src/mlpack/tests/string_encoding_test.cpp | 30 +++++-------- 2 files changed, 36 insertions(+), 38 deletions(-) diff --git a/src/mlpack/core/data/string_encoding_policies/tf_idf_encoding_policy.hpp b/src/mlpack/core/data/string_encoding_policies/tf_idf_encoding_policy.hpp index dc10f29e5d..83e4062bda 100644 --- a/src/mlpack/core/data/string_encoding_policies/tf_idf_encoding_policy.hpp +++ b/src/mlpack/core/data/string_encoding_policies/tf_idf_encoding_policy.hpp @@ -48,14 +48,14 @@ class TfIdfEncodingPolicy */ enum tfTypes { - rawCount, - binary, - sublinearTf, - termFrequency, + RAW_COUNT, + BINARY, + SUBLINEAR_TF, + TERM_FREQUENCY, }; - TfIdfEncodingPolicy(size_t tfType = 0, bool smooth_idf = true) : - tfType(tfType), smooth_idf(smooth_idf) + TfIdfEncodingPolicy(size_t tfType = 0, bool smoothIdf = true) : + tfType(tfType), smoothIdf(smoothIdf) { } /** @@ -112,18 +112,20 @@ class TfIdfEncodingPolicy { // Important since Mapping starts from 1 whereas allowed column value is 0. double idf, tf; - if (smooth_idf) + if (smoothIdf) idf = std::log((output.n_rows + 1) / (1 + idfdict[value - 1])) + 1; else idf = std::log(output.n_rows / idfdict[value - 1]) + 1; - if (tfType == tfTypes::termFrequency) + + if (tfType == tfTypes::TERM_FREQUENCY) tf = tokenCount[row][value - 1] / row_size[row]; - else if (tfType == tfTypes::sublinearTf) + else if (tfType == tfTypes::SUBLINEAR_TF) tf = std::log(tokenCount[row][value - 1]) + 1; - else if (tfType == tfTypes::binary) + else if (tfType == tfTypes::BINARY) tf = tokenCount[row][value - 1] > 0 ? 1 : 0; else tf = tokenCount[row][value - 1]; + output(row, value-1) = tf * idf; } @@ -145,18 +147,20 @@ class TfIdfEncodingPolicy { // Important since Mapping starts from 1 whereas allowed column value is 0. double idf, tf; - if (smooth_idf) + if (smoothIdf) idf = std::log((output.size() + 1) / (1 + idfdict[value - 1])) + 1; else idf = std::log(output.size() / idfdict[value - 1]) + 1; - if (tfType == tfTypes::termFrequency) + + if (tfType == tfTypes::TERM_FREQUENCY) tf = tokenCount[row][value - 1] / row_size[row]; - else if (tfType == tfTypes::sublinearTf) + else if (tfType == tfTypes::SUBLINEAR_TF) tf = std::log(tokenCount[row][value - 1]) + 1; - else if (tfType == tfTypes::binary) + else if (tfType == tfTypes::BINARY) tf = tokenCount[row][value - 1] > 0 ? 1 : 0; else tf = tokenCount[row][value - 1]; + output[row][value-1] = tf * idf; } @@ -164,9 +168,13 @@ class TfIdfEncodingPolicy * Serialize the class to the given archive. */ template - void serialize(Archive& /* ar */, const unsigned int /* version */) + void serialize(Archive& ar , const unsigned int /* version */) { - // Nothing to serialize. + ar & BOOST_SERIALIZATION_NVP(tfType); + ar & BOOST_SERIALIZATION_NVP(tokenCount); + ar & BOOST_SERIALIZATION_NVP(idfdict); + ar & BOOST_SERIALIZATION_NVP(smoothIdf); + ar & BOOST_SERIALIZATION_NVP(row_size); } /* @@ -198,8 +206,8 @@ class TfIdfEncodingPolicy std::unordered_map idfdict; // Used to store the number of tokens in each row. std::vector row_size; - // smooth_idf variable to indicate smoothining. - bool smooth_idf; + // smoothIdf variable to indicate smoothining. + bool smoothIdf; // Type of Term Frequency to use. size_t tfType; }; diff --git a/src/mlpack/tests/string_encoding_test.cpp b/src/mlpack/tests/string_encoding_test.cpp index 2cb1201097..87c358ed81 100644 --- a/src/mlpack/tests/string_encoding_test.cpp +++ b/src/mlpack/tests/string_encoding_test.cpp @@ -728,8 +728,7 @@ BOOST_AUTO_TEST_CASE(OnePassTfIdfRawCountEncodingTest) using DictionaryType = StringEncodingDictionary; vector> output; - TfIdfEncoding encoder( - (TfIdfEncodingPolicy(0, false))); + TfIdfEncoding encoder(0, false); SplitByAnyOf tokenizer(" "); encoder.Encode(stringEncodingInput, output, tokenizer); @@ -783,8 +782,7 @@ BOOST_AUTO_TEST_CASE(RawcountTfIdfEncodingIndividualCharactersTest) }; arma::mat output; - TfIdfEncoding encoder( - (TfIdfEncodingPolicy(0, false))); + TfIdfEncoding encoder(0, false); // Passing a empty string to encode charactersrawcountsmoothidftrue encoder.Encode(input, output, CharExtract()); @@ -810,8 +808,7 @@ BOOST_AUTO_TEST_CASE(OnePassRawcountEncodingIndividualCharactersTest) }; vector> output; - TfIdfEncoding encoder( - (TfIdfEncodingPolicy(false, false))); + TfIdfEncoding encoder(0, false); // Passing a empty string to encode characters encoder.Encode(input, output, CharExtract()); @@ -838,8 +835,7 @@ BOOST_AUTO_TEST_CASE(BinarySmoothIdfEncodingIndividualCharactersTest) }; arma::mat output; - TfIdfEncoding encoder( - (TfIdfEncodingPolicy(1, true))); + TfIdfEncoding encoder(1, true); // Passing a empty string to encode charactersrawcountsmoothidftrue encoder.Encode(input, output, CharExtract()); @@ -864,8 +860,7 @@ BOOST_AUTO_TEST_CASE(OnePassBnarySmoothIdfEncodingIndividualCharactersTest) }; vector> output; - TfIdfEncoding encoder( - (TfIdfEncodingPolicy(1, true))); + TfIdfEncoding encoder(1, true); // Passing a empty string to encode characters encoder.Encode(input, output, CharExtract()); @@ -892,8 +887,7 @@ BOOST_AUTO_TEST_CASE(BinaryTfIdfEncodingIndividualCharactersTest) }; arma::mat output; - TfIdfEncoding encoder( - (TfIdfEncodingPolicy(1, false))); + TfIdfEncoding encoder(1, false); // Passing a empty string to encode charactersrawcountsmoothidftrue encoder.Encode(input, output, CharExtract()); @@ -918,8 +912,7 @@ BOOST_AUTO_TEST_CASE(SublinearSmoothIdfEncodingIndividualCharactersTest) }; arma::mat output; - TfIdfEncoding encoder( - (TfIdfEncodingPolicy(2, true))); + TfIdfEncoding encoder(2, true); // Passing a empty string to encode charactersrawcountsmoothidftrue encoder.Encode(input, output, CharExtract()); @@ -945,8 +938,7 @@ BOOST_AUTO_TEST_CASE(SublinearTfIdfEncodingIndividualCharactersTest) }; arma::mat output; - TfIdfEncoding encoder( - (TfIdfEncodingPolicy(2, false))); + TfIdfEncoding encoder(2, false); // Passing a empty string to encode charactersrawcountsmoothidftrue encoder.Encode(input, output, CharExtract()); @@ -972,8 +964,7 @@ BOOST_AUTO_TEST_CASE(TermFrequencySmoothIdfEncodingIndividualCharactersTest) }; arma::mat output; - TfIdfEncoding encoder( - (TfIdfEncodingPolicy(3, true))); + TfIdfEncoding encoder(3, true); // Passing a empty string to encode charactersrawcountsmoothidftrue encoder.Encode(input, output, CharExtract()); @@ -999,8 +990,7 @@ BOOST_AUTO_TEST_CASE(TermFrequencyTfIdfEncodingIndividualCharactersTest) }; arma::mat output; - TfIdfEncoding encoder( - (TfIdfEncodingPolicy(3, false))); + TfIdfEncoding encoder(3, false); // Passing a empty string to encode charactersrawcountsmoothidftrue encoder.Encode(input, output, CharExtract()); From 818c4269739c6fd77a44821c9f721d0cd092d208 Mon Sep 17 00:00:00 2001 From: jeffinsam Date: Sat, 10 Aug 2019 11:48:58 +0530 Subject: [PATCH 015/111] temp commit to seek help --- src/mlpack/core/data/string_encoding_impl.hpp | 2 +- .../bag_of_words_encoding_policy.hpp | 7 +- .../tf_idf_encoding_policy.hpp | 25 +- src/mlpack/tests/string_encoding_test.cpp | 1618 +++++++++-------- 4 files changed, 850 insertions(+), 802 deletions(-) diff --git a/src/mlpack/core/data/string_encoding_impl.hpp b/src/mlpack/core/data/string_encoding_impl.hpp index a621ba0982..058539dc79 100644 --- a/src/mlpack/core/data/string_encoding_impl.hpp +++ b/src/mlpack/core/data/string_encoding_impl.hpp @@ -134,7 +134,7 @@ EncodeHelper(const std::vector& input, } numColumns = std::max(numColumns, numTokens); } - + std::cout<<"dictionary size is "< struct StringEncodingPolicyTraits diff --git a/src/mlpack/core/data/string_encoding_policies/tf_idf_encoding_policy.hpp b/src/mlpack/core/data/string_encoding_policies/tf_idf_encoding_policy.hpp index 83e4062bda..42b16aa3b3 100644 --- a/src/mlpack/core/data/string_encoding_policies/tf_idf_encoding_policy.hpp +++ b/src/mlpack/core/data/string_encoding_policies/tf_idf_encoding_policy.hpp @@ -46,7 +46,7 @@ class TfIdfEncodingPolicy * subinerTf : logarthimic weighting scheme (log(tf) + 1) * */ - enum tfTypes + enum class TfTypes { RAW_COUNT, BINARY, @@ -55,7 +55,8 @@ class TfIdfEncodingPolicy }; TfIdfEncodingPolicy(size_t tfType = 0, bool smoothIdf = true) : - tfType(tfType), smoothIdf(smoothIdf) + tfType(tfType), + smoothIdf(smoothIdf) { } /** @@ -73,6 +74,7 @@ class TfIdfEncodingPolicy size_t /*maxNumTokens*/, size_t dictionarySize) { + std::cout<<"dataset "<(TfTypes::TERM_FREQUENCY)) tf = tokenCount[row][value - 1] / row_size[row]; - else if (tfType == tfTypes::SUBLINEAR_TF) + else if (tfType == static_cast(TfTypes::SUBLINEAR_TF)) tf = std::log(tokenCount[row][value - 1]) + 1; - else if (tfType == tfTypes::BINARY) + else if (tfType == static_cast(TfTypes::BINARY)) tf = tokenCount[row][value - 1] > 0 ? 1 : 0; else tf = tokenCount[row][value - 1]; @@ -152,11 +154,11 @@ class TfIdfEncodingPolicy else idf = std::log(output.size() / idfdict[value - 1]) + 1; - if (tfType == tfTypes::TERM_FREQUENCY) + if (tfType == static_cast(TfTypes::TERM_FREQUENCY)) tf = tokenCount[row][value - 1] / row_size[row]; - else if (tfType == tfTypes::SUBLINEAR_TF) + else if (tfType == static_cast(TfTypes::SUBLINEAR_TF)) tf = std::log(tokenCount[row][value - 1]) + 1; - else if (tfType == tfTypes::BINARY) + else if (tfType == static_cast(TfTypes::BINARY)) tf = tokenCount[row][value - 1] > 0 ? 1 : 0; else tf = tokenCount[row][value - 1]; @@ -174,7 +176,8 @@ class TfIdfEncodingPolicy ar & BOOST_SERIALIZATION_NVP(tokenCount); ar & BOOST_SERIALIZATION_NVP(idfdict); ar & BOOST_SERIALIZATION_NVP(smoothIdf); - ar & BOOST_SERIALIZATION_NVP(row_size); + ar & BOOST_SERIALIZATION_NVP(row_size); + std::cout<<"hello world"<= tokenCount.size()) { row_size.push_back(0); tokenCount.push_back(std::unordered_map()); } + std::cout<<"error"< stringEncodingInput = { "command-line programs and Python bindings." }; -/** - * Test the dictionary encoding algorithm. - */ -BOOST_AUTO_TEST_CASE(DictionaryEncodingTest) -{ - using DictionaryType = StringEncodingDictionary; +// /** +// * Test the dictionary encoding algorithm. +// */ +// BOOST_AUTO_TEST_CASE(DictionaryEncodingTest) +// { +// using DictionaryType = StringEncodingDictionary; - arma::mat output; - DictionaryEncoding encoder; - SplitByAnyOf tokenizer(" .,\""); +// arma::mat output; +// DictionaryEncoding encoder; +// SplitByAnyOf tokenizer(" .,\""); - encoder.Encode(stringEncodingInput, output, tokenizer); +// encoder.Encode(stringEncodingInput, output, tokenizer); - const DictionaryType& dictionary = encoder.Dictionary(); +// const DictionaryType& dictionary = encoder.Dictionary(); - // Checking that everything is mapped to different numbers - std::unordered_map keysCount; - for (auto& keyValue : dictionary.Mapping()) - { - keysCount[keyValue.second]++; - // Every token should be mapped only once - BOOST_REQUIRE_EQUAL(keysCount[keyValue.second], 1); - } +// // Checking that everything is mapped to different numbers +// std::unordered_map keysCount; +// for (auto& keyValue : dictionary.Mapping()) +// { +// keysCount[keyValue.second]++; +// // Every token should be mapped only once +// BOOST_REQUIRE_EQUAL(keysCount[keyValue.second], 1); +// } - arma::mat expected = { - { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, - { 17, 2, 18, 14, 19, 20, 9, 10, 21, 14, 22, 6, 23, 14, 24, 20, 25, - 26, 27, 9, 10, 28, 6, 29, 30, 20, 31, 32, 33, 34, 9, 10, 35 }, - { 36, 37, 14, 38, 39, 8, 40, 1, 41, 42, 43, 44, 6, 45, 13, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 } - }; +// arma::mat expected = { +// { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 0, +// 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, +// { 17, 2, 18, 14, 19, 20, 9, 10, 21, 14, 22, 6, 23, 14, 24, 20, 25, +// 26, 27, 9, 10, 28, 6, 29, 30, 20, 31, 32, 33, 34, 9, 10, 35 }, +// { 36, 37, 14, 38, 39, 8, 40, 1, 41, 42, 43, 44, 6, 45, 13, 0, 0, +// 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 } +// }; - CheckMatrices(output, expected); -} +// CheckMatrices(output, expected); +// } -/** - * Test the one pass modification of the dictionary encoding algorithm. - */ -BOOST_AUTO_TEST_CASE(OnePassDictionaryEncodingTest) -{ - using DictionaryType = StringEncodingDictionary; +// /** +// * Test the one pass modification of the dictionary encoding algorithm. +// */ +// BOOST_AUTO_TEST_CASE(OnePassDictionaryEncodingTest) +// { +// using DictionaryType = StringEncodingDictionary; - vector> output; - DictionaryEncoding encoder( - (DictionaryEncodingPolicy())); - SplitByAnyOf tokenizer(" .,\""); +// vector> output; +// DictionaryEncoding encoder( +// (DictionaryEncodingPolicy())); +// SplitByAnyOf tokenizer(" .,\""); - encoder.Encode(stringEncodingInput, output, tokenizer); +// encoder.Encode(stringEncodingInput, output, tokenizer); - const DictionaryType& dictionary = encoder.Dictionary(); +// const DictionaryType& dictionary = encoder.Dictionary(); - // Checking that everything is mapped to different numbers - std::unordered_map keysCount; - for (auto& keyValue : dictionary.Mapping()) - { - keysCount[keyValue.second]++; - // Every token should be mapped only once - BOOST_REQUIRE_EQUAL(keysCount[keyValue.second], 1); - } +// // Checking that everything is mapped to different numbers +// std::unordered_map keysCount; +// for (auto& keyValue : dictionary.Mapping()) +// { +// keysCount[keyValue.second]++; +// // Every token should be mapped only once +// BOOST_REQUIRE_EQUAL(keysCount[keyValue.second], 1); +// } - vector> expected = { - { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16 }, - { 17, 2, 18, 14, 19, 20, 9, 10, 21, 14, 22, 6, 23, 14, 24, 20, 25, - 26, 27, 9, 10, 28, 6, 29, 30, 20, 31, 32, 33, 34, 9, 10, 35 }, - { 36, 37, 14, 38, 39, 8, 40, 1, 41, 42, 43, 44, 6, 45, 13 } - }; +// vector> expected = { +// { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16 }, +// { 17, 2, 18, 14, 19, 20, 9, 10, 21, 14, 22, 6, 23, 14, 24, 20, 25, +// 26, 27, 9, 10, 28, 6, 29, 30, 20, 31, 32, 33, 34, 9, 10, 35 }, +// { 36, 37, 14, 38, 39, 8, 40, 1, 41, 42, 43, 44, 6, 45, 13 } +// }; - BOOST_REQUIRE(output == expected); -} +// BOOST_REQUIRE(output == expected); +// } -/** - * Test for the SplitByAnyOf tokenizer. - */ -BOOST_AUTO_TEST_CASE(SplitByAnyOfTokenizerTest) -{ - std::vector tokens; - boost::string_view line(stringEncodingInput[0]); - SplitByAnyOf tokenizer(" ,."); - boost::string_view token = tokenizer(line); +// /** +// * Test for the SplitByAnyOf tokenizer. +// */ +// BOOST_AUTO_TEST_CASE(SplitByAnyOfTokenizerTest) +// { +// std::vector tokens; +// boost::string_view line(stringEncodingInput[0]); +// SplitByAnyOf tokenizer(" ,."); +// boost::string_view token = tokenizer(line); - while (!token.empty()) - { - tokens.push_back(token); - token = tokenizer(line); - } +// while (!token.empty()) +// { +// tokens.push_back(token); +// token = tokenizer(line); +// } - vector expected = { "mlpack", "is", "an", "intuitive", "fast", - "and", "flexible", "C++", "machine", "learning", "library", "with", - "bindings", "to", "other", "languages" - }; +// vector expected = { "mlpack", "is", "an", "intuitive", "fast", +// "and", "flexible", "C++", "machine", "learning", "library", "with", +// "bindings", "to", "other", "languages" +// }; - BOOST_REQUIRE_EQUAL(tokens.size(), expected.size()); +// BOOST_REQUIRE_EQUAL(tokens.size(), expected.size()); - for (size_t i = 0; i < tokens.size(); i++) - BOOST_REQUIRE_EQUAL(tokens[i], expected[i]); -} +// for (size_t i = 0; i < tokens.size(); i++) +// BOOST_REQUIRE_EQUAL(tokens[i], expected[i]); +// } -/** -* Test Dictionary encoding for characters using lamda function. -*/ -BOOST_AUTO_TEST_CASE(DictionaryEncodingIndividualCharactersTest) -{ - vector input = { - "GACCA", - "ABCABCD", - "GAB" - }; +// /** +// * Test Dictionary encoding for characters using lamda function. +// */ +// BOOST_AUTO_TEST_CASE(DictionaryEncodingIndividualCharactersTest) +// { +// vector input = { +// "GACCA", +// "ABCABCD", +// "GAB" +// }; - arma::mat output; - DictionaryEncoding encoder; +// arma::mat output; +// DictionaryEncoding encoder; - // Passing a empty string to encode characters - encoder.Encode(input, output, CharExtract()); +// // Passing a empty string to encode characters +// encoder.Encode(input, output, CharExtract()); - arma::mat target = { - { 1, 2, 3, 3, 2, 0, 0 }, - { 2, 4, 3, 2, 4, 3, 5 }, - { 1, 2, 4, 0, 0, 0, 0 } - }; - CheckMatrices(output, target); -} +// arma::mat target = { +// { 1, 2, 3, 3, 2, 0, 0 }, +// { 2, 4, 3, 2, 4, 3, 5 }, +// { 1, 2, 4, 0, 0, 0, 0 } +// }; +// CheckMatrices(output, target); +// } -/** - * Test the one pass modification of the dictionary encoding algorithm - * in case of individual character encoding. - */ -BOOST_AUTO_TEST_CASE(OnePassDictionaryEncodingIndividualCharactersTest) -{ - std::vector input = { - "GACCA", - "ABCABCD", - "GAB" - }; +// /** +// * Test the one pass modification of the dictionary encoding algorithm +// * in case of individual character encoding. +// */ +// BOOST_AUTO_TEST_CASE(OnePassDictionaryEncodingIndividualCharactersTest) +// { +// std::vector input = { +// "GACCA", +// "ABCABCD", +// "GAB" +// }; - vector> output; - DictionaryEncoding encoder; +// vector> output; +// DictionaryEncoding encoder; - // Passing a empty string to encode characters - encoder.Encode(input, output, CharExtract()); +// // Passing a empty string to encode characters +// encoder.Encode(input, output, CharExtract()); - vector> expected = { - { 1, 2, 3, 3, 2 }, - { 2, 4, 3, 2, 4, 3, 5 }, - { 1, 2, 4 } - }; +// vector> expected = { +// { 1, 2, 3, 3, 2 }, +// { 2, 4, 3, 2, 4, 3, 5 }, +// { 1, 2, 4 } +// }; - BOOST_REQUIRE(output == expected); -} +// BOOST_REQUIRE(output == expected); +// } -/** - * Test the functionality of copy constructor. - */ -BOOST_AUTO_TEST_CASE(StringEncodingCopyTest) -{ - using DictionaryType = StringEncodingDictionary; - arma::sp_mat output; - DictionaryEncoding encoderCopy; - SplitByAnyOf tokenizer(" ,."); +// /** +// * Test the functionality of copy constructor. +// */ +// BOOST_AUTO_TEST_CASE(StringEncodingCopyTest) +// { +// using DictionaryType = StringEncodingDictionary; +// arma::sp_mat output; +// DictionaryEncoding encoderCopy; +// SplitByAnyOf tokenizer(" ,."); - vector> naiveDictionary; +// vector> naiveDictionary; - { - DictionaryEncoding encoder; - encoder.Encode(stringEncodingInput, output, tokenizer); +// { +// DictionaryEncoding encoder; +// encoder.Encode(stringEncodingInput, output, tokenizer); - for (const string& token : encoder.Dictionary().Tokens()) - { - naiveDictionary.emplace_back(token, encoder.Dictionary().Value(token)); - } +// for (const string& token : encoder.Dictionary().Tokens()) +// { +// naiveDictionary.emplace_back(token, encoder.Dictionary().Value(token)); +// } - encoderCopy = DictionaryEncoding(encoder); - } +// encoderCopy = DictionaryEncoding(encoder); +// } - const DictionaryType& copiedDictionary = encoderCopy.Dictionary(); +// const DictionaryType& copiedDictionary = encoderCopy.Dictionary(); - BOOST_REQUIRE_EQUAL(naiveDictionary.size(), copiedDictionary.Size()); +// BOOST_REQUIRE_EQUAL(naiveDictionary.size(), copiedDictionary.Size()); - for (const pair& keyValue : naiveDictionary) - { - BOOST_REQUIRE(copiedDictionary.HasToken(keyValue.first)); - BOOST_REQUIRE_EQUAL(copiedDictionary.Value(keyValue.first), - keyValue.second); - } -} +// for (const pair& keyValue : naiveDictionary) +// { +// BOOST_REQUIRE(copiedDictionary.HasToken(keyValue.first)); +// BOOST_REQUIRE_EQUAL(copiedDictionary.Value(keyValue.first), +// keyValue.second); +// } +// } -/** - * Test the move assignment operator. - */ -BOOST_AUTO_TEST_CASE(StringEncodingMoveTest) -{ - using DictionaryType = StringEncodingDictionary; - arma::sp_mat output; - DictionaryEncoding encoderCopy; - SplitByAnyOf tokenizer(" ,."); +// /** +// * Test the move assignment operator. +// */ +// BOOST_AUTO_TEST_CASE(StringEncodingMoveTest) +// { +// using DictionaryType = StringEncodingDictionary; +// arma::sp_mat output; +// DictionaryEncoding encoderCopy; +// SplitByAnyOf tokenizer(" ,."); - vector> naiveDictionary; +// vector> naiveDictionary; - { - DictionaryEncoding encoder; - encoder.Encode(stringEncodingInput, output, tokenizer); +// { +// DictionaryEncoding encoder; +// encoder.Encode(stringEncodingInput, output, tokenizer); - for (const string& token : encoder.Dictionary().Tokens()) - { - naiveDictionary.emplace_back(token, encoder.Dictionary().Value(token)); - } +// for (const string& token : encoder.Dictionary().Tokens()) +// { +// naiveDictionary.emplace_back(token, encoder.Dictionary().Value(token)); +// } - encoderCopy = std::move(encoder); - } +// encoderCopy = std::move(encoder); +// } - const DictionaryType& copiedDictionary = encoderCopy.Dictionary(); +// const DictionaryType& copiedDictionary = encoderCopy.Dictionary(); - BOOST_REQUIRE_EQUAL(naiveDictionary.size(), copiedDictionary.Size()); +// BOOST_REQUIRE_EQUAL(naiveDictionary.size(), copiedDictionary.Size()); - for (const pair& keyValue : naiveDictionary) - { - BOOST_REQUIRE(copiedDictionary.HasToken(keyValue.first)); - BOOST_REQUIRE_EQUAL(copiedDictionary.Value(keyValue.first), - keyValue.second); - } -} +// for (const pair& keyValue : naiveDictionary) +// { +// BOOST_REQUIRE(copiedDictionary.HasToken(keyValue.first)); +// BOOST_REQUIRE_EQUAL(copiedDictionary.Value(keyValue.first), +// keyValue.second); +// } +// } /** * The function checks that the given dictionaries contain the same data. @@ -342,35 +342,35 @@ void CheckDictionaries(const StringEncodingDictionary& expected, } } -/** - * Serialization test for the dictionary encoding algorithm with - * the SplitByAnyOf tokenizer. - */ -BOOST_AUTO_TEST_CASE(SplitByAnyOfDictionaryEncodingSerialization) -{ - using EncoderType = DictionaryEncoding; +// /** +// * Serialization test for the dictionary encoding algorithm with +// * the SplitByAnyOf tokenizer. +// */ +// BOOST_AUTO_TEST_CASE(SplitByAnyOfDictionaryEncodingSerialization) +// { +// using EncoderType = DictionaryEncoding; - EncoderType encoder; - SplitByAnyOf tokenizer(" ,."); - arma::mat output; +// EncoderType encoder; +// SplitByAnyOf tokenizer(" ,."); +// arma::mat output; - encoder.Encode(stringEncodingInput, output, tokenizer); +// encoder.Encode(stringEncodingInput, output, tokenizer); - EncoderType xmlEncoder, textEncoder, binaryEncoder; - arma::mat xmlOutput, textOutput, binaryOutput; +// EncoderType xmlEncoder, textEncoder, binaryEncoder; +// arma::mat xmlOutput, textOutput, binaryOutput; - SerializeObjectAll(encoder, xmlEncoder, textEncoder, binaryEncoder); +// SerializeObjectAll(encoder, xmlEncoder, textEncoder, binaryEncoder); - CheckDictionaries(encoder.Dictionary(), xmlEncoder.Dictionary()); - CheckDictionaries(encoder.Dictionary(), textEncoder.Dictionary()); - CheckDictionaries(encoder.Dictionary(), binaryEncoder.Dictionary()); +// CheckDictionaries(encoder.Dictionary(), xmlEncoder.Dictionary()); +// CheckDictionaries(encoder.Dictionary(), textEncoder.Dictionary()); +// CheckDictionaries(encoder.Dictionary(), binaryEncoder.Dictionary()); - xmlEncoder.Encode(stringEncodingInput, xmlOutput, tokenizer); - textEncoder.Encode(stringEncodingInput, textOutput, tokenizer); - binaryEncoder.Encode(stringEncodingInput, binaryOutput, tokenizer); +// xmlEncoder.Encode(stringEncodingInput, xmlOutput, tokenizer); +// textEncoder.Encode(stringEncodingInput, textOutput, tokenizer); +// binaryEncoder.Encode(stringEncodingInput, binaryOutput, tokenizer); - CheckMatrices(output, xmlOutput, textOutput, binaryOutput); -} +// CheckMatrices(output, xmlOutput, textOutput, binaryOutput); +// } /** * Serialization test for the dictionary encoding algorithm with @@ -378,13 +378,17 @@ BOOST_AUTO_TEST_CASE(SplitByAnyOfDictionaryEncodingSerialization) */ BOOST_AUTO_TEST_CASE(CharExtractDictionaryEncodingSerialization) { - using EncoderType = DictionaryEncoding; + using EncoderType = BagOfWordsEncoding; EncoderType encoder; CharExtract tokenizer; arma::mat output; - - encoder.Encode(stringEncodingInput, output, tokenizer); + std::vector input = { + "GACCA", + "ABCABCD", + "GAB" + }; + encoder.Encode(input, output, tokenizer); EncoderType xmlEncoder, textEncoder, binaryEncoder; arma::mat xmlOutput, textOutput, binaryOutput; @@ -394,613 +398,649 @@ BOOST_AUTO_TEST_CASE(CharExtractDictionaryEncodingSerialization) CheckDictionaries(encoder.Dictionary(), xmlEncoder.Dictionary()); CheckDictionaries(encoder.Dictionary(), textEncoder.Dictionary()); CheckDictionaries(encoder.Dictionary(), binaryEncoder.Dictionary()); - - xmlEncoder.Encode(stringEncodingInput, xmlOutput, tokenizer); - textEncoder.Encode(stringEncodingInput, textOutput, tokenizer); - binaryEncoder.Encode(stringEncodingInput, binaryOutput, tokenizer); + std::cout<<"Calling the actual object"<; +// BOOST_AUTO_TEST_CASE(BagOfWordsEncodingTest) +// { +// using DictionaryType = StringEncodingDictionary; - arma::mat output; - BagOfWordsEncoding encoder; - SplitByAnyOf tokenizer(" "); +// arma::mat output; +// BagOfWordsEncoding encoder; +// SplitByAnyOf tokenizer(" "); - encoder.Encode(stringEncodingInput, output, tokenizer); - const DictionaryType& dictionary = encoder.Dictionary(); +// encoder.Encode(stringEncodingInput, output, tokenizer); +// const DictionaryType& dictionary = encoder.Dictionary(); - // Checking that everything is mapped to different numbers - std::unordered_map keysCount; - for (auto& keyValue : dictionary.Mapping()) - { - keysCount[keyValue.second]++; - // Every token should be mapped only once - BOOST_REQUIRE_EQUAL(keysCount[keyValue.second], 1); - } - arma::mat expected = { - { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, - { 0, 1, 0, 0, 0, 1, 0, 0, 1, 1, 0, 0, 0, 1, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, - { 1, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 } - }; - CheckMatrices(output, expected); -} +// // Checking that everything is mapped to different numbers +// std::unordered_map keysCount; +// for (auto& keyValue : dictionary.Mapping()) +// { +// keysCount[keyValue.second]++; +// // Every token should be mapped only once +// BOOST_REQUIRE_EQUAL(keysCount[keyValue.second], 1); +// } +// arma::mat expected = { +// { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, +// 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, +// { 0, 1, 0, 0, 0, 1, 0, 0, 1, 1, 0, 0, 0, 1, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, +// 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, +// { 1, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +// 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 } +// }; +// CheckMatrices(output, expected); +// } + +// * +// * Test the one pass modification of the Bag of Words encoding algorithm. + +// BOOST_AUTO_TEST_CASE(OnePassBagOfWordsEncodingTest) +// { +// using DictionaryType = StringEncodingDictionary; + +// vector> output; +// BagOfWordsEncoding encoder( +// (BagOfWordsEncodingPolicy())); +// SplitByAnyOf tokenizer(" "); + +// encoder.Encode(stringEncodingInput, output, tokenizer); + +// const DictionaryType& dictionary = encoder.Dictionary(); + +// // Checking that everything is mapped to different numbers +// std::unordered_map keysCount; +// for (auto& keyValue : dictionary.Mapping()) +// { +// keysCount[keyValue.second]++; +// // Every token should be mapped only once +// BOOST_REQUIRE_EQUAL(keysCount[keyValue.second], 1); +// } + +// vector> expected = { +// { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, +// 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, +// { 0, 1, 0, 0, 0, 1, 0, 0, 1, 1, 0, 0, 0, 1, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, +// 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, +// { 1, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +// 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 } +// }; + +// BOOST_REQUIRE(output == expected); +// } + +// /** +// * Test Bag of Words encoding for characters using lamda function. +// */ +// BOOST_AUTO_TEST_CASE(BagOfWordsEncodingIndividualCharactersTest) +// { +// vector input = { +// "GACCA", +// "ABCABCD", +// "GAB" +// }; + +// arma::mat output; +// BagOfWordsEncoding encoder; + +// // Passing a empty string to encode characters +// encoder.Encode(input, output, CharExtract()); +// arma::mat target = { +// { 1, 1, 1, 0, 0 }, +// { 0, 1, 1, 1, 1 }, +// { 1, 1, 0, 1, 0 } +// }; + +// CheckMatrices(output, target); +// } + +// /** +// * Test the one pass modification of the Bag of Words encoding algorithm +// * in case of individual character encoding. +// */ +// BOOST_AUTO_TEST_CASE(OnePassBagOfWordsEncodingIndividualCharactersTest) +// { +// std::vector input = { +// "GACCA", +// "ABCABCD", +// "GAB" +// }; + +// vector> output; +// BagOfWordsEncoding encoder; + +// // Passing a empty string to encode characters +// encoder.Encode(input, output, CharExtract()); + +// vector> expected = { +// { 1, 1, 1, 0, 0 }, +// { 0, 1, 1, 1, 1 }, +// { 1, 1, 0, 1, 0 } +// }; + +// BOOST_REQUIRE(output == expected); +// } + +// /** +// * Test the Tf-Idf Encoding using rawcount type and smoothidf as true, +// * which is the deafult values used for algorithim. +// */ +// BOOST_AUTO_TEST_CASE(RawCountSmoothIdfEncodingTest) +// { +// using DictionaryType = StringEncodingDictionary; + +// arma::mat output; +// TfIdfEncoding encoder; +// SplitByAnyOf tokenizer(" "); + +// encoder.Encode(stringEncodingInput, output, tokenizer); + +// const DictionaryType& dictionary = encoder.Dictionary(); + +// // Checking that everything is mapped to different numbers +// std::unordered_map keysCount; +// for (auto& keyValue : dictionary.Mapping()) +// { +// keysCount[keyValue.second]++; +// // Every token should be mapped only once +// BOOST_REQUIRE_EQUAL(keysCount[keyValue.second], 1); +// } +// arma::mat expected = { +// { 1.28768207245178, 1.28768207245178, 1.69314718055995, 1.69314718055995, +// 1.69314718055995, 1, 1.69314718055995, 1.28768207245178, +// 1.28768207245178, 1.28768207245178, 1.69314718055995, 1.69314718055995, +// 1.69314718055995, 1, 1.69314718055995, 1.69314718055995, 0, 0, 0, 0, 0, +// 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +// 0 }, +// { 0, 1.28768207245178, 0, 0, 0, 2, 0, 0, 3.86304621735534, +// 3.86304621735534, 0, 0, 0, 3, 0, 0, 1.69314718055995, 1.69314718055995, +// 1.69314718055995, 5.07944154167984, 1.69314718055995, 1.69314718055995, +// 1.69314718055995, 1.69314718055995, 1.69314718055995, 1.69314718055995, +// 1.69314718055995, 1.69314718055995, 1.69314718055995, 1.69314718055995, +// 1.69314718055995, 1.69314718055995, 1.69314718055995, 1.69314718055995, +// 1.69314718055995, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, +// { 1.28768207245178, 0, 0, 0, 0, 1, 0, 1.28768207245178, 0, 0, 0, 0, 0, 1, +// 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +// 1.69314718055995, 1.69314718055995, 1.69314718055995, 1.69314718055995, +// 1.69314718055995, 1.69314718055995, 1.69314718055995, 1.69314718055995, +// 1.69314718055995, 1.69314718055995, 1.69314718055995 } +// }; +// CheckMatrices(output, expected, 1e-12); +// } + +// /** +// * Test the one pass modification of the TfIdf encoding algorithm, using rawcount +// * as type of tf and smoothIdf as true. +// */ +// BOOST_AUTO_TEST_CASE(OnePassRawCountSmoothIdfEncodingTest) +// { +// using DictionaryType = StringEncodingDictionary; + +// vector> output; +// TfIdfEncoding encoder( +// (TfIdfEncodingPolicy())); +// SplitByAnyOf tokenizer(" "); + +// encoder.Encode(stringEncodingInput, output, tokenizer); + +// const DictionaryType& dictionary = encoder.Dictionary(); + +// // Checking that everything is mapped to different numbers +// std::unordered_map keysCount; +// for (auto& keyValue : dictionary.Mapping()) +// { +// keysCount[keyValue.second]++; +// // Every token should be mapped only once +// BOOST_REQUIRE_EQUAL(keysCount[keyValue.second], 1); +// } + +// vector> expected = { +// { 1.28768207245178, 1.28768207245178, 1.69314718055995, 1.69314718055995, +// 1.69314718055995, 1, 1.69314718055995, 1.28768207245178, +// 1.28768207245178, 1.28768207245178, 1.69314718055995, 1.69314718055995, +// 1.69314718055995, 1, 1.69314718055995, 1.69314718055995, 0, 0, 0, 0, 0, +// 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +// 0 }, +// { 0, 1.28768207245178, 0, 0, 0, 2, 0, 0, 3.86304621735534, +// 3.86304621735534, 0, 0, 0, 3, 0, 0, 1.69314718055995, 1.69314718055995, +// 1.69314718055995, 5.07944154167984, 1.69314718055995, 1.69314718055995, +// 1.69314718055995, 1.69314718055995, 1.69314718055995, 1.69314718055995, +// 1.69314718055995, 1.69314718055995, 1.69314718055995, 1.69314718055995, +// 1.69314718055995, 1.69314718055995, 1.69314718055995, 1.69314718055995, +// 1.69314718055995, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, +// { 1.28768207245178, 0, 0, 0, 0, 1, 0, 1.28768207245178, 0, 0, 0, 0, 0, 1, +// 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +// 1.69314718055995, 1.69314718055995, 1.69314718055995, 1.69314718055995, +// 1.69314718055995, 1.69314718055995, 1.69314718055995, 1.69314718055995, +// 1.69314718055995, 1.69314718055995, 1.69314718055995 } +// }; +// for (size_t i = 0; i < expected.size(); i++) +// for (size_t j = 0; j < expected[i].size(); j++) +// BOOST_REQUIRE_CLOSE(expected[i][j], output[i][j], 1e-12); +// } + +// /** +// * Test TFIDF encoding for characters using lamda function, using rawcount as tf +// * type and smoothidf as true. +// */ +// BOOST_AUTO_TEST_CASE(RawCountSmoothIdfEncodingIndividualCharactersTest) +// { +// vector input = { +// "GACCA", +// "ABCABCD", +// "GAB" +// }; + +// arma::mat output; +// TfIdfEncoding encoder; + +// // Passing a empty string to encode charactersrawcountsmoothidftrue +// encoder.Encode(input, output, CharExtract()); +// arma::mat target = { +// { 1.2876820724517808, 2, 2.5753641449035616, 0, 0 }, +// { 0, 2, 2.5753641449035616, 2.5753641449035616, 1.6931471805599454 }, +// { 1.2876820724517808, 1, 0, 1.2876820724517808, 0 } +// }; +// CheckMatrices(output, target, 1e-12); +// } + +// /** +// * Test the one pass modification of the Tf-Idf encoding algorithm +// * in case of individual character encoding using default values. +// */ +// BOOST_AUTO_TEST_CASE(OnePassRawCountSmoothIdfEncodingIndividualCharactersTest) +// { +// std::vector input = { +// "GACCA", +// "ABCABCD", +// "GAB" +// }; + +// vector> output; +// TfIdfEncoding encoder; + +// // Passing a empty string to encode characters +// encoder.Encode(input, output, CharExtract()); +// vector> expected = { +// { 1.2876820724517808, 2, 2.5753641449035616, 0, 0 }, +// { 0, 2, 2.5753641449035616, 2.5753641449035616, 1.6931471805599454 }, +// { 1.2876820724517808, 1, 0, 1.2876820724517808, 0 } +// }; +// for (size_t i = 0; i < expected.size(); i++) +// for (size_t j = 0; j < expected[i].size(); j++) +// BOOST_REQUIRE_CLOSE(expected[i][j], output[i][j], 1e-12); +// } + +// /** +// * Test the Tf-Idf Encoding using rawcount type and smoothidf as false. +// */ +// BOOST_AUTO_TEST_CASE(TfIdfRawCountEncodingTest) +// { +// using DictionaryType = StringEncodingDictionary; + +// arma::mat output; +// TfIdfEncoding encoder( +// (TfIdfEncodingPolicy(0, false))); +// SplitByAnyOf tokenizer(" "); + +// encoder.Encode(stringEncodingInput, output, tokenizer); + +// const DictionaryType& dictionary = encoder.Dictionary(); + +// // Checking that everything is mapped to different numbers +// std::unordered_map keysCount; +// for (auto& keyValue : dictionary.Mapping()) +// { +// keysCount[keyValue.second]++; +// // Every token should be mapped only once +// BOOST_REQUIRE_EQUAL(keysCount[keyValue.second], 1); +// } + +// arma::mat expected = { +// { 1.40546510810816, 1.40546510810816, 2.09861228866811, 2.09861228866811, +// 2.09861228866811, 1, 2.09861228866811, 1.40546510810816, +// 1.40546510810816, 1.40546510810816, 2.09861228866811, 2.09861228866811, +// 2.09861228866811, 1, 2.09861228866811, 2.09861228866811, 0, 0, 0, 0, 0, +// 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +// 0 }, +// { 0, 1.40546510810816, 0, 0, 0, 2, 0, 0, 4.21639532432449, +// 4.21639532432449, 0, 0, 0, 3, 0, 0, 2.09861228866811, 2.09861228866811, +// 2.09861228866811, 6.29583686600433, 2.09861228866811, 2.09861228866811, +// 2.09861228866811, 2.09861228866811, 2.09861228866811, 2.09861228866811, +// 2.09861228866811, 2.09861228866811, 2.09861228866811, 2.09861228866811, +// 2.09861228866811, 2.09861228866811, 2.09861228866811, 2.09861228866811, +// 2.09861228866811, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, +// { 1.40546510810816, 0, 0, 0, 0, 1, 0, 1.40546510810816, 0, 0, 0, 0, 0, 1, +// 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +// 2.09861228866811, 2.09861228866811, 2.09861228866811, 2.09861228866811, +// 2.09861228866811, 2.09861228866811, 2.09861228866811, 2.09861228866811, +// 2.09861228866811, 2.09861228866811, 2.09861228866811 } +// }; +// CheckMatrices(output, expected, 1e-12); +// } + +// * +// * Test the one pass modification of the TfIdf encoding algorithm, with rawcount +// * as type, but with smoothidf as false. + +// BOOST_AUTO_TEST_CASE(OnePassTfIdfRawCountEncodingTest) +// { +// using DictionaryType = StringEncodingDictionary; + +// vector> output; +// TfIdfEncoding encoder(0, false); +// SplitByAnyOf tokenizer(" "); + +// encoder.Encode(stringEncodingInput, output, tokenizer); + +// const DictionaryType& dictionary = encoder.Dictionary(); + +// // Checking that everything is mapped to different numbers +// std::unordered_map keysCount; +// for (auto& keyValue : dictionary.Mapping()) +// { +// keysCount[keyValue.second]++; +// // Every token should be mapped only once +// BOOST_REQUIRE_EQUAL(keysCount[keyValue.second], 1); +// } + +// vector> expected = { +// { 1.40546510810816, 1.40546510810816, 2.09861228866811, 2.09861228866811, +// 2.09861228866811, 1, 2.09861228866811, 1.40546510810816, +// 1.40546510810816, 1.40546510810816, 2.09861228866811, 2.09861228866811, +// 2.09861228866811, 1, 2.09861228866811, 2.09861228866811, 0, 0, 0, 0, 0, +// 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +// 0 }, +// { 0, 1.40546510810816, 0, 0, 0, 2, 0, 0, 4.21639532432449, +// 4.21639532432449, 0, 0, 0, 3, 0, 0, 2.09861228866811, 2.09861228866811, +// 2.09861228866811, 6.29583686600433, 2.09861228866811, 2.09861228866811, +// 2.09861228866811, 2.09861228866811, 2.09861228866811, 2.09861228866811, +// 2.09861228866811, 2.09861228866811, 2.09861228866811, 2.09861228866811, +// 2.09861228866811, 2.09861228866811, 2.09861228866811, 2.09861228866811, +// 2.09861228866811, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, +// { 1.40546510810816, 0, 0, 0, 0, 1, 0, 1.40546510810816, 0, 0, 0, 0, 0, 1, +// 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +// 2.09861228866811, 2.09861228866811, 2.09861228866811, 2.09861228866811, +// 2.09861228866811, 2.09861228866811, 2.09861228866811, 2.09861228866811, +// 2.09861228866811, 2.09861228866811, 2.09861228866811 } +// }; +// for (size_t i = 0; i < expected.size(); i++) +// for (size_t j = 0; j < expected[i].size(); j++) +// BOOST_REQUIRE_CLOSE(expected[i][j], output[i][j], 1e-12); +// } + +// /** +// * Test TFIDF encoding for characters using lamda function, using rawcount as +// * tf type and smoothidf as false. +// */ +// BOOST_AUTO_TEST_CASE(RawcountTfIdfEncodingIndividualCharactersTest) +// { +// vector input = { +// "GACCA", +// "ABCABCD", +// "GAB" +// }; + +// arma::mat output; +// TfIdfEncoding encoder(0, false); + +// // Passing a empty string to encode charactersrawcountsmoothidftrue +// encoder.Encode(input, output, CharExtract()); +// arma::mat target = { +// { 1.4054651081081644, 2, 2.8109302162163288, 0, 0 }, +// { 0, 2, 2.8109302162163288, 2.8109302162163288, 2.0986122886681100 }, +// { 1.4054651081081644, 1, 0, 1.4054651081081644, 0 } +// }; +// CheckMatrices(output, target, 1e-12); +// } + +// /** +// * Test the one pass modification of the Tf Idf encoding algorithm +// * in case of individual character encoding, using raw count as type, +// * and smoothidf as false. +// */ +// BOOST_AUTO_TEST_CASE(OnePassRawcountEncodingIndividualCharactersTest) +// { +// std::vector input = { +// "GACCA", +// "ABCABCD", +// "GAB" +// }; + +// vector> output; +// TfIdfEncoding encoder(0, false); + +// // Passing a empty string to encode characters +// encoder.Encode(input, output, CharExtract()); +// vector> expected = { +// { 1.4054651081081644, 2, 2.8109302162163288, 0, 0 }, +// { 0, 2, 2.8109302162163288, 2.8109302162163288, 2.0986122886681100 }, +// { 1.4054651081081644, 1, 0, 1.4054651081081644, 0 } +// }; +// for (size_t i = 0; i < expected.size(); i++) +// for (size_t j = 0; j < expected[i].size(); j++) +// BOOST_REQUIRE_CLOSE(expected[i][j], output[i][j], 1e-12); +// } + +// /** +// * Test TFIDF encoding for characters using lamda function, using binary +// * weighting scheme for tf and smoothidf as true. +// */ +// BOOST_AUTO_TEST_CASE(BinarySmoothIdfEncodingIndividualCharactersTest) +// { +// vector input = { +// "GACCA", +// "ABCABCD", +// "GAB" +// }; + +// arma::mat output; +// TfIdfEncoding encoder(1, true); + +// // Passing a empty string to encode charactersrawcountsmoothidftrue +// encoder.Encode(input, output, CharExtract()); +// arma::mat target = { +// { 1.2876820724517808, 1, 1.2876820724517808, 0, 0 }, +// { 0, 1, 1.2876820724517808, 1.2876820724517808, 1.6931471805599454 }, +// { 1.2876820724517808, 1, 0, 1.2876820724517808, 0 } +// }; +// CheckMatrices(output, target, 1e-12); +// } + +// /** +// * Test TFIDF encoding for characters using lamda function, using binary +// * weighting scheme for tf and smoothidf as true. +// */ +// BOOST_AUTO_TEST_CASE(OnePassBnarySmoothIdfEncodingIndividualCharactersTest) +// { +// std::vector input = { +// "GACCA", +// "ABCABCD", +// "GAB" +// }; + +// vector> output; +// TfIdfEncoding encoder(1, true); + +// // Passing a empty string to encode characters +// encoder.Encode(input, output, CharExtract()); +// vector> expected = { +// { 1.2876820724517808, 1, 1.2876820724517808, 0, 0 }, +// { 0, 1, 1.2876820724517808, 1.2876820724517808, 1.6931471805599454 }, +// { 1.2876820724517808, 1, 0, 1.2876820724517808, 0 } +// }; +// for (size_t i = 0; i < expected.size(); i++) +// for (size_t j = 0; j < expected[i].size(); j++) +// BOOST_REQUIRE_CLOSE(expected[i][j], output[i][j], 1e-12); +// } + +// /** +// * Test TFIDF encoding for characters using lamda function, using binary +// * as weighting scheme and smoothidf as false. +// */ +// BOOST_AUTO_TEST_CASE(BinaryTfIdfEncodingIndividualCharactersTest) +// { +// vector input = { +// "GACCA", +// "ABCABCD", +// "GAB" +// }; + +// arma::mat output; +// TfIdfEncoding encoder(1, false); + +// // Passing a empty string to encode charactersrawcountsmoothidftrue +// encoder.Encode(input, output, CharExtract()); +// arma::mat target = { +// { 1.4054651081081644, 1, 1.4054651081081644, 0, 0 }, +// { 0, 1, 1.4054651081081644, 1.4054651081081644, 2.0986122886681100 }, +// { 1.4054651081081644, 1, 0, 1.4054651081081644, 0 } +// }; +// CheckMatrices(output, target, 1e-12); +// } + +// /** +// * Test TFIDF encoding for characters using lamda function, using sublinear +// * as weighting scheme and smoothidf as true. +// */ +// BOOST_AUTO_TEST_CASE(SublinearSmoothIdfEncodingIndividualCharactersTest) +// { +// vector input = { +// "GACCA", +// "ABCABCD", +// "GAB" +// }; + +// arma::mat output; +// TfIdfEncoding encoder(2, true); + +// // Passing a empty string to encode charactersrawcountsmoothidftrue +// encoder.Encode(input, output, CharExtract()); +// arma::mat target = { +// { 1.2876820724517808, 1.6931471805599454, 2.1802352704293200, 0, 0 }, +// { 0, 1.6931471805599454, 2.1802352704293200, 2.1802352704293200, +// 1.6931471805599454 }, +// { 1.2876820724517808, 1, 0, 1.2876820724517808, 0 } +// }; +// CheckMatrices(output, target, 1e-12); +// } + +// /** +// * Test TFIDF encoding for characters using lamda function, using sublinear +// * as weighting scheme and smoothidf as false. +// */ +// BOOST_AUTO_TEST_CASE(SublinearTfIdfEncodingIndividualCharactersTest) +// { +// vector input = { +// "GACCA", +// "ABCABCD", +// "GAB" +// }; + +// arma::mat output; +// TfIdfEncoding encoder(2, false); + +// // Passing a empty string to encode charactersrawcountsmoothidftrue +// encoder.Encode(input, output, CharExtract()); +// arma::mat target = { +// { 1.4054651081081644, 1.6931471805599454, 2.3796592851687173, 0, 0 }, +// { 0, 1.6931471805599454, 2.3796592851687173, 2.3796592851687173, +// 2.0986122886681100 }, +// { 1.4054651081081644, 1, 0, 1.4054651081081644, 0 } +// }; +// CheckMatrices(output, target, 1e-12); +// } + +// /** +// * Test TFIDF encoding for characters using lamda function, using term +// * Frequency as weighting scheme and smoothidf as true. +// */ +// BOOST_AUTO_TEST_CASE(TermFrequencySmoothIdfEncodingIndividualCharactersTest) +// { +// vector input = { +// "GACCA", +// "ABCABCD", +// "GAB" +// }; + +// arma::mat output; +// TfIdfEncoding encoder(3, true); + +// // Passing a empty string to encode charactersrawcountsmoothidftrue +// encoder.Encode(input, output, CharExtract()); +// arma::mat target = { +// { 0.2575364144903562, 0.4, 0.5150728289807124, 0, 0 }, +// { 0, 0.2857142857142857, 0.3679091635576516, 0.3679091635576516, +// 0.2418781686514208 }, +// { 0.4292273574839269, 0.3333333333333333, 0, 0.4292273574839269, 0 } +// }; +// CheckMatrices(output, target, 1e-12); +// } + +// /** +// * Test TFIDF encoding for characters using lamda function, using Term +// * Frequency as weighting scheme and smoothidf as false. +// */ +// BOOST_AUTO_TEST_CASE(TermFrequencyTfIdfEncodingIndividualCharactersTest) +// { +// vector input = { +// "GACCA", +// "ABCABCD", +// "GAB" +// }; + +// arma::mat output; +// TfIdfEncoding encoder(3, false); + +// // Passing a empty string to encode charactersrawcountsmoothidftrue +// encoder.Encode(input, output, CharExtract()); +// arma::mat target = { +// { 0.2810930216216329, 0.4, 0.5621860432432658, 0, 0 }, +// { 0, 0.2857142857142857, 0.4015614594594755, 0.4015614594594755, +// 0.2998017555240157 }, +// { 0.4684883693693881, 0.3333333333333333, 0, 0.4684883693693881, 0 } +// }; +// CheckMatrices(output, target, 1e-12); +// } /** - * Test the one pass modification of the Bag of Words encoding algorithm. + * Serialization test for the dictionary encoding algorithm with + * the CharExtract tokenizer. */ -BOOST_AUTO_TEST_CASE(OnePassBagOfWordsEncodingTest) +BOOST_AUTO_TEST_CASE(CharExtractBagOfWordsEncodingSerialization) { - using DictionaryType = StringEncodingDictionary; + using EncoderType = TfIdfEncoding; - vector> output; - BagOfWordsEncoding encoder( - (BagOfWordsEncodingPolicy())); - SplitByAnyOf tokenizer(" "); - - encoder.Encode(stringEncodingInput, output, tokenizer); - - const DictionaryType& dictionary = encoder.Dictionary(); - - // Checking that everything is mapped to different numbers - std::unordered_map keysCount; - for (auto& keyValue : dictionary.Mapping()) - { - keysCount[keyValue.second]++; - // Every token should be mapped only once - BOOST_REQUIRE_EQUAL(keysCount[keyValue.second], 1); - } - - vector> expected = { - { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, - { 0, 1, 0, 0, 0, 1, 0, 0, 1, 1, 0, 0, 0, 1, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, - { 1, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 } - }; - - BOOST_REQUIRE(output == expected); -} - -/** -* Test Bag of Words encoding for characters using lamda function. -*/ -BOOST_AUTO_TEST_CASE(BagOfWordsEncodingIndividualCharactersTest) -{ + EncoderType encoder; + CharExtract tokenizer; + arma::mat output; vector input = { "GACCA", "ABCABCD", "GAB" }; + encoder.Encode(input, output, tokenizer); - arma::mat output; - BagOfWordsEncoding encoder; + EncoderType xmlEncoder, textEncoder, binaryEncoder; + arma::mat xmlOutput, textOutput, binaryOutput; - // Passing a empty string to encode characters - encoder.Encode(input, output, CharExtract()); - arma::mat target = { - { 1, 1, 1, 0, 0 }, - { 0, 1, 1, 1, 1 }, - { 1, 1, 0, 1, 0 } - }; + SerializeObjectAll(encoder, xmlEncoder, textEncoder, binaryEncoder); + std::cout<<"HHHHHHHHHHHHHH\n"; + CheckDictionaries(encoder.Dictionary(), xmlEncoder.Dictionary()); + CheckDictionaries(encoder.Dictionary(), textEncoder.Dictionary()); + CheckDictionaries(encoder.Dictionary(), binaryEncoder.Dictionary()); + std::cout<<"FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFSDDDDDDDDDDDDDDDD\n"; + xmlEncoder.Encode(input, xmlOutput, tokenizer); + std::cout<<"@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@\n"; - CheckMatrices(output, target); -} + textEncoder.Encode(input, textOutput, tokenizer); + binaryEncoder.Encode(input, binaryOutput, tokenizer); -/** - * Test the one pass modification of the Bag of Words encoding algorithm - * in case of individual character encoding. - */ -BOOST_AUTO_TEST_CASE(OnePassBagOfWordsEncodingIndividualCharactersTest) -{ - std::vector input = { - "GACCA", - "ABCABCD", - "GAB" - }; - - vector> output; - BagOfWordsEncoding encoder; - - // Passing a empty string to encode characters - encoder.Encode(input, output, CharExtract()); - - vector> expected = { - { 1, 1, 1, 0, 0 }, - { 0, 1, 1, 1, 1 }, - { 1, 1, 0, 1, 0 } - }; - - BOOST_REQUIRE(output == expected); -} - -/** - * Test the Tf-Idf Encoding using rawcount type and smoothidf as true, - * which is the deafult values used for algorithim. - */ -BOOST_AUTO_TEST_CASE(RawCountSmoothIdfEncodingTest) -{ - using DictionaryType = StringEncodingDictionary; - - arma::mat output; - TfIdfEncoding encoder; - SplitByAnyOf tokenizer(" "); - - encoder.Encode(stringEncodingInput, output, tokenizer); - - const DictionaryType& dictionary = encoder.Dictionary(); - - // Checking that everything is mapped to different numbers - std::unordered_map keysCount; - for (auto& keyValue : dictionary.Mapping()) - { - keysCount[keyValue.second]++; - // Every token should be mapped only once - BOOST_REQUIRE_EQUAL(keysCount[keyValue.second], 1); - } - arma::mat expected = { - { 1.28768207245178, 1.28768207245178, 1.69314718055995, 1.69314718055995, - 1.69314718055995, 1, 1.69314718055995, 1.28768207245178, - 1.28768207245178, 1.28768207245178, 1.69314718055995, 1.69314718055995, - 1.69314718055995, 1, 1.69314718055995, 1.69314718055995, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0 }, - { 0, 1.28768207245178, 0, 0, 0, 2, 0, 0, 3.86304621735534, - 3.86304621735534, 0, 0, 0, 3, 0, 0, 1.69314718055995, 1.69314718055995, - 1.69314718055995, 5.07944154167984, 1.69314718055995, 1.69314718055995, - 1.69314718055995, 1.69314718055995, 1.69314718055995, 1.69314718055995, - 1.69314718055995, 1.69314718055995, 1.69314718055995, 1.69314718055995, - 1.69314718055995, 1.69314718055995, 1.69314718055995, 1.69314718055995, - 1.69314718055995, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, - { 1.28768207245178, 0, 0, 0, 0, 1, 0, 1.28768207245178, 0, 0, 0, 0, 0, 1, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 1.69314718055995, 1.69314718055995, 1.69314718055995, 1.69314718055995, - 1.69314718055995, 1.69314718055995, 1.69314718055995, 1.69314718055995, - 1.69314718055995, 1.69314718055995, 1.69314718055995 } - }; - CheckMatrices(output, expected, 1e-12); -} - -/** - * Test the one pass modification of the TfIdf encoding algorithm, using rawcount - * as type of tf and smoothIdf as true. - */ -BOOST_AUTO_TEST_CASE(OnePassRawCountSmoothIdfEncodingTest) -{ - using DictionaryType = StringEncodingDictionary; - - vector> output; - TfIdfEncoding encoder( - (TfIdfEncodingPolicy())); - SplitByAnyOf tokenizer(" "); - - encoder.Encode(stringEncodingInput, output, tokenizer); - - const DictionaryType& dictionary = encoder.Dictionary(); - - // Checking that everything is mapped to different numbers - std::unordered_map keysCount; - for (auto& keyValue : dictionary.Mapping()) - { - keysCount[keyValue.second]++; - // Every token should be mapped only once - BOOST_REQUIRE_EQUAL(keysCount[keyValue.second], 1); - } - - vector> expected = { - { 1.28768207245178, 1.28768207245178, 1.69314718055995, 1.69314718055995, - 1.69314718055995, 1, 1.69314718055995, 1.28768207245178, - 1.28768207245178, 1.28768207245178, 1.69314718055995, 1.69314718055995, - 1.69314718055995, 1, 1.69314718055995, 1.69314718055995, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0 }, - { 0, 1.28768207245178, 0, 0, 0, 2, 0, 0, 3.86304621735534, - 3.86304621735534, 0, 0, 0, 3, 0, 0, 1.69314718055995, 1.69314718055995, - 1.69314718055995, 5.07944154167984, 1.69314718055995, 1.69314718055995, - 1.69314718055995, 1.69314718055995, 1.69314718055995, 1.69314718055995, - 1.69314718055995, 1.69314718055995, 1.69314718055995, 1.69314718055995, - 1.69314718055995, 1.69314718055995, 1.69314718055995, 1.69314718055995, - 1.69314718055995, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, - { 1.28768207245178, 0, 0, 0, 0, 1, 0, 1.28768207245178, 0, 0, 0, 0, 0, 1, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 1.69314718055995, 1.69314718055995, 1.69314718055995, 1.69314718055995, - 1.69314718055995, 1.69314718055995, 1.69314718055995, 1.69314718055995, - 1.69314718055995, 1.69314718055995, 1.69314718055995 } - }; - for (size_t i = 0; i < expected.size(); i++) - for (size_t j = 0; j < expected[i].size(); j++) - BOOST_REQUIRE_CLOSE(expected[i][j], output[i][j], 1e-12); -} - -/** - * Test TFIDF encoding for characters using lamda function, using rawcount as tf - * type and smoothidf as true. - */ -BOOST_AUTO_TEST_CASE(RawCountSmoothIdfEncodingIndividualCharactersTest) -{ - vector input = { - "GACCA", - "ABCABCD", - "GAB" - }; - - arma::mat output; - TfIdfEncoding encoder; - - // Passing a empty string to encode charactersrawcountsmoothidftrue - encoder.Encode(input, output, CharExtract()); - arma::mat target = { - { 1.2876820724517808, 2, 2.5753641449035616, 0, 0 }, - { 0, 2, 2.5753641449035616, 2.5753641449035616, 1.6931471805599454 }, - { 1.2876820724517808, 1, 0, 1.2876820724517808, 0 } - }; - CheckMatrices(output, target, 1e-12); -} - -/** - * Test the one pass modification of the Tf-Idf encoding algorithm - * in case of individual character encoding using default values. - */ -BOOST_AUTO_TEST_CASE(OnePassRawCountSmoothIdfEncodingIndividualCharactersTest) -{ - std::vector input = { - "GACCA", - "ABCABCD", - "GAB" - }; - - vector> output; - TfIdfEncoding encoder; - - // Passing a empty string to encode characters - encoder.Encode(input, output, CharExtract()); - vector> expected = { - { 1.2876820724517808, 2, 2.5753641449035616, 0, 0 }, - { 0, 2, 2.5753641449035616, 2.5753641449035616, 1.6931471805599454 }, - { 1.2876820724517808, 1, 0, 1.2876820724517808, 0 } - }; - for (size_t i = 0; i < expected.size(); i++) - for (size_t j = 0; j < expected[i].size(); j++) - BOOST_REQUIRE_CLOSE(expected[i][j], output[i][j], 1e-12); -} - -/** - * Test the Tf-Idf Encoding using rawcount type and smoothidf as false. - */ -BOOST_AUTO_TEST_CASE(TfIdfRawCountEncodingTest) -{ - using DictionaryType = StringEncodingDictionary; - - arma::mat output; - TfIdfEncoding encoder( - (TfIdfEncodingPolicy(0, false))); - SplitByAnyOf tokenizer(" "); - - encoder.Encode(stringEncodingInput, output, tokenizer); - - const DictionaryType& dictionary = encoder.Dictionary(); - - // Checking that everything is mapped to different numbers - std::unordered_map keysCount; - for (auto& keyValue : dictionary.Mapping()) - { - keysCount[keyValue.second]++; - // Every token should be mapped only once - BOOST_REQUIRE_EQUAL(keysCount[keyValue.second], 1); - } - - arma::mat expected = { - { 1.40546510810816, 1.40546510810816, 2.09861228866811, 2.09861228866811, - 2.09861228866811, 1, 2.09861228866811, 1.40546510810816, - 1.40546510810816, 1.40546510810816, 2.09861228866811, 2.09861228866811, - 2.09861228866811, 1, 2.09861228866811, 2.09861228866811, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0 }, - { 0, 1.40546510810816, 0, 0, 0, 2, 0, 0, 4.21639532432449, - 4.21639532432449, 0, 0, 0, 3, 0, 0, 2.09861228866811, 2.09861228866811, - 2.09861228866811, 6.29583686600433, 2.09861228866811, 2.09861228866811, - 2.09861228866811, 2.09861228866811, 2.09861228866811, 2.09861228866811, - 2.09861228866811, 2.09861228866811, 2.09861228866811, 2.09861228866811, - 2.09861228866811, 2.09861228866811, 2.09861228866811, 2.09861228866811, - 2.09861228866811, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, - { 1.40546510810816, 0, 0, 0, 0, 1, 0, 1.40546510810816, 0, 0, 0, 0, 0, 1, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 2.09861228866811, 2.09861228866811, 2.09861228866811, 2.09861228866811, - 2.09861228866811, 2.09861228866811, 2.09861228866811, 2.09861228866811, - 2.09861228866811, 2.09861228866811, 2.09861228866811 } - }; - CheckMatrices(output, expected, 1e-12); -} - -/** - * Test the one pass modification of the TfIdf encoding algorithm, with rawcount - * as type, but with smoothidf as false. - */ -BOOST_AUTO_TEST_CASE(OnePassTfIdfRawCountEncodingTest) -{ - using DictionaryType = StringEncodingDictionary; - - vector> output; - TfIdfEncoding encoder(0, false); - SplitByAnyOf tokenizer(" "); - - encoder.Encode(stringEncodingInput, output, tokenizer); - - const DictionaryType& dictionary = encoder.Dictionary(); - - // Checking that everything is mapped to different numbers - std::unordered_map keysCount; - for (auto& keyValue : dictionary.Mapping()) - { - keysCount[keyValue.second]++; - // Every token should be mapped only once - BOOST_REQUIRE_EQUAL(keysCount[keyValue.second], 1); - } - - vector> expected = { - { 1.40546510810816, 1.40546510810816, 2.09861228866811, 2.09861228866811, - 2.09861228866811, 1, 2.09861228866811, 1.40546510810816, - 1.40546510810816, 1.40546510810816, 2.09861228866811, 2.09861228866811, - 2.09861228866811, 1, 2.09861228866811, 2.09861228866811, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0 }, - { 0, 1.40546510810816, 0, 0, 0, 2, 0, 0, 4.21639532432449, - 4.21639532432449, 0, 0, 0, 3, 0, 0, 2.09861228866811, 2.09861228866811, - 2.09861228866811, 6.29583686600433, 2.09861228866811, 2.09861228866811, - 2.09861228866811, 2.09861228866811, 2.09861228866811, 2.09861228866811, - 2.09861228866811, 2.09861228866811, 2.09861228866811, 2.09861228866811, - 2.09861228866811, 2.09861228866811, 2.09861228866811, 2.09861228866811, - 2.09861228866811, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, - { 1.40546510810816, 0, 0, 0, 0, 1, 0, 1.40546510810816, 0, 0, 0, 0, 0, 1, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 2.09861228866811, 2.09861228866811, 2.09861228866811, 2.09861228866811, - 2.09861228866811, 2.09861228866811, 2.09861228866811, 2.09861228866811, - 2.09861228866811, 2.09861228866811, 2.09861228866811 } - }; - for (size_t i = 0; i < expected.size(); i++) - for (size_t j = 0; j < expected[i].size(); j++) - BOOST_REQUIRE_CLOSE(expected[i][j], output[i][j], 1e-12); -} - -/** - * Test TFIDF encoding for characters using lamda function, using rawcount as - * tf type and smoothidf as false. - */ -BOOST_AUTO_TEST_CASE(RawcountTfIdfEncodingIndividualCharactersTest) -{ - vector input = { - "GACCA", - "ABCABCD", - "GAB" - }; - - arma::mat output; - TfIdfEncoding encoder(0, false); - - // Passing a empty string to encode charactersrawcountsmoothidftrue - encoder.Encode(input, output, CharExtract()); - arma::mat target = { - { 1.4054651081081644, 2, 2.8109302162163288, 0, 0 }, - { 0, 2, 2.8109302162163288, 2.8109302162163288, 2.0986122886681100 }, - { 1.4054651081081644, 1, 0, 1.4054651081081644, 0 } - }; - CheckMatrices(output, target, 1e-12); -} - -/** - * Test the one pass modification of the Tf Idf encoding algorithm - * in case of individual character encoding, using raw count as type, - * and smoothidf as false. - */ -BOOST_AUTO_TEST_CASE(OnePassRawcountEncodingIndividualCharactersTest) -{ - std::vector input = { - "GACCA", - "ABCABCD", - "GAB" - }; - - vector> output; - TfIdfEncoding encoder(0, false); - - // Passing a empty string to encode characters - encoder.Encode(input, output, CharExtract()); - vector> expected = { - { 1.4054651081081644, 2, 2.8109302162163288, 0, 0 }, - { 0, 2, 2.8109302162163288, 2.8109302162163288, 2.0986122886681100 }, - { 1.4054651081081644, 1, 0, 1.4054651081081644, 0 } - }; - for (size_t i = 0; i < expected.size(); i++) - for (size_t j = 0; j < expected[i].size(); j++) - BOOST_REQUIRE_CLOSE(expected[i][j], output[i][j], 1e-12); -} - -/** - * Test TFIDF encoding for characters using lamda function, using binary - * weighting scheme for tf and smoothidf as true. - */ -BOOST_AUTO_TEST_CASE(BinarySmoothIdfEncodingIndividualCharactersTest) -{ - vector input = { - "GACCA", - "ABCABCD", - "GAB" - }; - - arma::mat output; - TfIdfEncoding encoder(1, true); - - // Passing a empty string to encode charactersrawcountsmoothidftrue - encoder.Encode(input, output, CharExtract()); - arma::mat target = { - { 1.2876820724517808, 1, 1.2876820724517808, 0, 0 }, - { 0, 1, 1.2876820724517808, 1.2876820724517808, 1.6931471805599454 }, - { 1.2876820724517808, 1, 0, 1.2876820724517808, 0 } - }; - CheckMatrices(output, target, 1e-12); -} - -/** - * Test TFIDF encoding for characters using lamda function, using binary - * weighting scheme for tf and smoothidf as true. - */ -BOOST_AUTO_TEST_CASE(OnePassBnarySmoothIdfEncodingIndividualCharactersTest) -{ - std::vector input = { - "GACCA", - "ABCABCD", - "GAB" - }; - - vector> output; - TfIdfEncoding encoder(1, true); - - // Passing a empty string to encode characters - encoder.Encode(input, output, CharExtract()); - vector> expected = { - { 1.2876820724517808, 1, 1.2876820724517808, 0, 0 }, - { 0, 1, 1.2876820724517808, 1.2876820724517808, 1.6931471805599454 }, - { 1.2876820724517808, 1, 0, 1.2876820724517808, 0 } - }; - for (size_t i = 0; i < expected.size(); i++) - for (size_t j = 0; j < expected[i].size(); j++) - BOOST_REQUIRE_CLOSE(expected[i][j], output[i][j], 1e-12); -} - -/** - * Test TFIDF encoding for characters using lamda function, using binary - * as weighting scheme and smoothidf as false. - */ -BOOST_AUTO_TEST_CASE(BinaryTfIdfEncodingIndividualCharactersTest) -{ - vector input = { - "GACCA", - "ABCABCD", - "GAB" - }; - - arma::mat output; - TfIdfEncoding encoder(1, false); - - // Passing a empty string to encode charactersrawcountsmoothidftrue - encoder.Encode(input, output, CharExtract()); - arma::mat target = { - { 1.4054651081081644, 1, 1.4054651081081644, 0, 0 }, - { 0, 1, 1.4054651081081644, 1.4054651081081644, 2.0986122886681100 }, - { 1.4054651081081644, 1, 0, 1.4054651081081644, 0 } - }; - CheckMatrices(output, target, 1e-12); -} - -/** - * Test TFIDF encoding for characters using lamda function, using sublinear - * as weighting scheme and smoothidf as true. - */ -BOOST_AUTO_TEST_CASE(SublinearSmoothIdfEncodingIndividualCharactersTest) -{ - vector input = { - "GACCA", - "ABCABCD", - "GAB" - }; - - arma::mat output; - TfIdfEncoding encoder(2, true); - - // Passing a empty string to encode charactersrawcountsmoothidftrue - encoder.Encode(input, output, CharExtract()); - arma::mat target = { - { 1.2876820724517808, 1.6931471805599454, 2.1802352704293200, 0, 0 }, - { 0, 1.6931471805599454, 2.1802352704293200, 2.1802352704293200, - 1.6931471805599454 }, - { 1.2876820724517808, 1, 0, 1.2876820724517808, 0 } - }; - CheckMatrices(output, target, 1e-12); -} - -/** - * Test TFIDF encoding for characters using lamda function, using sublinear - * as weighting scheme and smoothidf as false. - */ -BOOST_AUTO_TEST_CASE(SublinearTfIdfEncodingIndividualCharactersTest) -{ - vector input = { - "GACCA", - "ABCABCD", - "GAB" - }; - - arma::mat output; - TfIdfEncoding encoder(2, false); - - // Passing a empty string to encode charactersrawcountsmoothidftrue - encoder.Encode(input, output, CharExtract()); - arma::mat target = { - { 1.4054651081081644, 1.6931471805599454, 2.3796592851687173, 0, 0 }, - { 0, 1.6931471805599454, 2.3796592851687173, 2.3796592851687173, - 2.0986122886681100 }, - { 1.4054651081081644, 1, 0, 1.4054651081081644, 0 } - }; - CheckMatrices(output, target, 1e-12); -} - -/** - * Test TFIDF encoding for characters using lamda function, using term - * Frequency as weighting scheme and smoothidf as true. - */ -BOOST_AUTO_TEST_CASE(TermFrequencySmoothIdfEncodingIndividualCharactersTest) -{ - vector input = { - "GACCA", - "ABCABCD", - "GAB" - }; - - arma::mat output; - TfIdfEncoding encoder(3, true); - - // Passing a empty string to encode charactersrawcountsmoothidftrue - encoder.Encode(input, output, CharExtract()); - arma::mat target = { - { 0.2575364144903562, 0.4, 0.5150728289807124, 0, 0 }, - { 0, 0.2857142857142857, 0.3679091635576516, 0.3679091635576516, - 0.2418781686514208 }, - { 0.4292273574839269, 0.3333333333333333, 0, 0.4292273574839269, 0 } - }; - CheckMatrices(output, target, 1e-12); -} - -/** - * Test TFIDF encoding for characters using lamda function, using Term - * Frequency as weighting scheme and smoothidf as false. - */ -BOOST_AUTO_TEST_CASE(TermFrequencyTfIdfEncodingIndividualCharactersTest) -{ - vector input = { - "GACCA", - "ABCABCD", - "GAB" - }; - - arma::mat output; - TfIdfEncoding encoder(3, false); - - // Passing a empty string to encode charactersrawcountsmoothidftrue - encoder.Encode(input, output, CharExtract()); - arma::mat target = { - { 0.2810930216216329, 0.4, 0.5621860432432658, 0, 0 }, - { 0, 0.2857142857142857, 0.4015614594594755, 0.4015614594594755, - 0.2998017555240157 }, - { 0.4684883693693881, 0.3333333333333333, 0, 0.4684883693693881, 0 } - }; - CheckMatrices(output, target, 1e-12); + CheckMatrices(output, xmlOutput, textOutput, binaryOutput); } BOOST_AUTO_TEST_SUITE_END(); From b663dd83d7232508698cb85b9352a0b747c71605 Mon Sep 17 00:00:00 2001 From: jeffinsam Date: Sat, 10 Aug 2019 12:18:04 +0530 Subject: [PATCH 016/111] resolved serialization issue with dictionary class --- .../core/data/string_encoding_dictionary.hpp | 1 + src/mlpack/core/data/string_encoding_impl.hpp | 1 - .../bag_of_words_encoding_policy.hpp | 1 - src/mlpack/tests/string_encoding_test.cpp | 639 +++++++++--------- 4 files changed, 318 insertions(+), 324 deletions(-) diff --git a/src/mlpack/core/data/string_encoding_dictionary.hpp b/src/mlpack/core/data/string_encoding_dictionary.hpp index db5943f61a..2ccec8fb89 100644 --- a/src/mlpack/core/data/string_encoding_dictionary.hpp +++ b/src/mlpack/core/data/string_encoding_dictionary.hpp @@ -318,6 +318,7 @@ class StringEncodingDictionary void serialize(Archive& ar, const unsigned int /* version */) { ar & BOOST_SERIALIZATION_NVP(mapping); + ar & BOOST_SERIALIZATION_NVP(size); } private: diff --git a/src/mlpack/core/data/string_encoding_impl.hpp b/src/mlpack/core/data/string_encoding_impl.hpp index 058539dc79..11fd3644d1 100644 --- a/src/mlpack/core/data/string_encoding_impl.hpp +++ b/src/mlpack/core/data/string_encoding_impl.hpp @@ -134,7 +134,6 @@ EncodeHelper(const std::vector& input, } numColumns = std::max(numColumns, numTokens); } - std::cout<<"dictionary size is "< stringEncodingInput = { "command-line programs and Python bindings." }; -// /** -// * Test the dictionary encoding algorithm. -// */ -// BOOST_AUTO_TEST_CASE(DictionaryEncodingTest) -// { -// using DictionaryType = StringEncodingDictionary; +/** + * Test the dictionary encoding algorithm. + */ +BOOST_AUTO_TEST_CASE(DictionaryEncodingTest) +{ + using DictionaryType = StringEncodingDictionary; -// arma::mat output; -// DictionaryEncoding encoder; -// SplitByAnyOf tokenizer(" .,\""); + arma::mat output; + DictionaryEncoding encoder; + SplitByAnyOf tokenizer(" .,\""); -// encoder.Encode(stringEncodingInput, output, tokenizer); + encoder.Encode(stringEncodingInput, output, tokenizer); -// const DictionaryType& dictionary = encoder.Dictionary(); + const DictionaryType& dictionary = encoder.Dictionary(); -// // Checking that everything is mapped to different numbers -// std::unordered_map keysCount; -// for (auto& keyValue : dictionary.Mapping()) -// { -// keysCount[keyValue.second]++; -// // Every token should be mapped only once -// BOOST_REQUIRE_EQUAL(keysCount[keyValue.second], 1); -// } + // Checking that everything is mapped to different numbers + std::unordered_map keysCount; + for (auto& keyValue : dictionary.Mapping()) + { + keysCount[keyValue.second]++; + // Every token should be mapped only once + BOOST_REQUIRE_EQUAL(keysCount[keyValue.second], 1); + } -// arma::mat expected = { -// { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 0, -// 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, -// { 17, 2, 18, 14, 19, 20, 9, 10, 21, 14, 22, 6, 23, 14, 24, 20, 25, -// 26, 27, 9, 10, 28, 6, 29, 30, 20, 31, 32, 33, 34, 9, 10, 35 }, -// { 36, 37, 14, 38, 39, 8, 40, 1, 41, 42, 43, 44, 6, 45, 13, 0, 0, -// 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 } -// }; + arma::mat expected = { + { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, + { 17, 2, 18, 14, 19, 20, 9, 10, 21, 14, 22, 6, 23, 14, 24, 20, 25, + 26, 27, 9, 10, 28, 6, 29, 30, 20, 31, 32, 33, 34, 9, 10, 35 }, + { 36, 37, 14, 38, 39, 8, 40, 1, 41, 42, 43, 44, 6, 45, 13, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 } + }; -// CheckMatrices(output, expected); -// } + CheckMatrices(output, expected); +} -// /** -// * Test the one pass modification of the dictionary encoding algorithm. -// */ -// BOOST_AUTO_TEST_CASE(OnePassDictionaryEncodingTest) -// { -// using DictionaryType = StringEncodingDictionary; +/** + * Test the one pass modification of the dictionary encoding algorithm. + */ +BOOST_AUTO_TEST_CASE(OnePassDictionaryEncodingTest) +{ + using DictionaryType = StringEncodingDictionary; -// vector> output; -// DictionaryEncoding encoder( -// (DictionaryEncodingPolicy())); -// SplitByAnyOf tokenizer(" .,\""); + vector> output; + DictionaryEncoding encoder( + (DictionaryEncodingPolicy())); + SplitByAnyOf tokenizer(" .,\""); -// encoder.Encode(stringEncodingInput, output, tokenizer); + encoder.Encode(stringEncodingInput, output, tokenizer); -// const DictionaryType& dictionary = encoder.Dictionary(); + const DictionaryType& dictionary = encoder.Dictionary(); -// // Checking that everything is mapped to different numbers -// std::unordered_map keysCount; -// for (auto& keyValue : dictionary.Mapping()) -// { -// keysCount[keyValue.second]++; -// // Every token should be mapped only once -// BOOST_REQUIRE_EQUAL(keysCount[keyValue.second], 1); -// } + // Checking that everything is mapped to different numbers + std::unordered_map keysCount; + for (auto& keyValue : dictionary.Mapping()) + { + keysCount[keyValue.second]++; + // Every token should be mapped only once + BOOST_REQUIRE_EQUAL(keysCount[keyValue.second], 1); + } -// vector> expected = { -// { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16 }, -// { 17, 2, 18, 14, 19, 20, 9, 10, 21, 14, 22, 6, 23, 14, 24, 20, 25, -// 26, 27, 9, 10, 28, 6, 29, 30, 20, 31, 32, 33, 34, 9, 10, 35 }, -// { 36, 37, 14, 38, 39, 8, 40, 1, 41, 42, 43, 44, 6, 45, 13 } -// }; + vector> expected = { + { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16 }, + { 17, 2, 18, 14, 19, 20, 9, 10, 21, 14, 22, 6, 23, 14, 24, 20, 25, + 26, 27, 9, 10, 28, 6, 29, 30, 20, 31, 32, 33, 34, 9, 10, 35 }, + { 36, 37, 14, 38, 39, 8, 40, 1, 41, 42, 43, 44, 6, 45, 13 } + }; -// BOOST_REQUIRE(output == expected); -// } + BOOST_REQUIRE(output == expected); +} -// /** -// * Test for the SplitByAnyOf tokenizer. -// */ -// BOOST_AUTO_TEST_CASE(SplitByAnyOfTokenizerTest) -// { -// std::vector tokens; -// boost::string_view line(stringEncodingInput[0]); -// SplitByAnyOf tokenizer(" ,."); -// boost::string_view token = tokenizer(line); +/** + * Test for the SplitByAnyOf tokenizer. + */ +BOOST_AUTO_TEST_CASE(SplitByAnyOfTokenizerTest) +{ + std::vector tokens; + boost::string_view line(stringEncodingInput[0]); + SplitByAnyOf tokenizer(" ,."); + boost::string_view token = tokenizer(line); -// while (!token.empty()) -// { -// tokens.push_back(token); -// token = tokenizer(line); -// } + while (!token.empty()) + { + tokens.push_back(token); + token = tokenizer(line); + } -// vector expected = { "mlpack", "is", "an", "intuitive", "fast", -// "and", "flexible", "C++", "machine", "learning", "library", "with", -// "bindings", "to", "other", "languages" -// }; + vector expected = { "mlpack", "is", "an", "intuitive", "fast", + "and", "flexible", "C++", "machine", "learning", "library", "with", + "bindings", "to", "other", "languages" + }; -// BOOST_REQUIRE_EQUAL(tokens.size(), expected.size()); + BOOST_REQUIRE_EQUAL(tokens.size(), expected.size()); -// for (size_t i = 0; i < tokens.size(); i++) -// BOOST_REQUIRE_EQUAL(tokens[i], expected[i]); -// } + for (size_t i = 0; i < tokens.size(); i++) + BOOST_REQUIRE_EQUAL(tokens[i], expected[i]); +} -// /** -// * Test Dictionary encoding for characters using lamda function. -// */ -// BOOST_AUTO_TEST_CASE(DictionaryEncodingIndividualCharactersTest) -// { -// vector input = { -// "GACCA", -// "ABCABCD", -// "GAB" -// }; +/** +* Test Dictionary encoding for characters using lamda function. +*/ +BOOST_AUTO_TEST_CASE(DictionaryEncodingIndividualCharactersTest) +{ + vector input = { + "GACCA", + "ABCABCD", + "GAB" + }; -// arma::mat output; -// DictionaryEncoding encoder; + arma::mat output; + DictionaryEncoding encoder; -// // Passing a empty string to encode characters -// encoder.Encode(input, output, CharExtract()); + // Passing a empty string to encode characters + encoder.Encode(input, output, CharExtract()); -// arma::mat target = { -// { 1, 2, 3, 3, 2, 0, 0 }, -// { 2, 4, 3, 2, 4, 3, 5 }, -// { 1, 2, 4, 0, 0, 0, 0 } -// }; -// CheckMatrices(output, target); -// } + arma::mat target = { + { 1, 2, 3, 3, 2, 0, 0 }, + { 2, 4, 3, 2, 4, 3, 5 }, + { 1, 2, 4, 0, 0, 0, 0 } + }; + CheckMatrices(output, target); +} -// /** -// * Test the one pass modification of the dictionary encoding algorithm -// * in case of individual character encoding. -// */ -// BOOST_AUTO_TEST_CASE(OnePassDictionaryEncodingIndividualCharactersTest) -// { -// std::vector input = { -// "GACCA", -// "ABCABCD", -// "GAB" -// }; +/** + * Test the one pass modification of the dictionary encoding algorithm + * in case of individual character encoding. + */ +BOOST_AUTO_TEST_CASE(OnePassDictionaryEncodingIndividualCharactersTest) +{ + std::vector input = { + "GACCA", + "ABCABCD", + "GAB" + }; -// vector> output; -// DictionaryEncoding encoder; + vector> output; + DictionaryEncoding encoder; -// // Passing a empty string to encode characters -// encoder.Encode(input, output, CharExtract()); + // Passing a empty string to encode characters + encoder.Encode(input, output, CharExtract()); -// vector> expected = { -// { 1, 2, 3, 3, 2 }, -// { 2, 4, 3, 2, 4, 3, 5 }, -// { 1, 2, 4 } -// }; + vector> expected = { + { 1, 2, 3, 3, 2 }, + { 2, 4, 3, 2, 4, 3, 5 }, + { 1, 2, 4 } + }; -// BOOST_REQUIRE(output == expected); -// } + BOOST_REQUIRE(output == expected); +} -// /** -// * Test the functionality of copy constructor. -// */ -// BOOST_AUTO_TEST_CASE(StringEncodingCopyTest) -// { -// using DictionaryType = StringEncodingDictionary; -// arma::sp_mat output; -// DictionaryEncoding encoderCopy; -// SplitByAnyOf tokenizer(" ,."); +/** + * Test the functionality of copy constructor. + */ +BOOST_AUTO_TEST_CASE(StringEncodingCopyTest) +{ + using DictionaryType = StringEncodingDictionary; + arma::sp_mat output; + DictionaryEncoding encoderCopy; + SplitByAnyOf tokenizer(" ,."); -// vector> naiveDictionary; + vector> naiveDictionary; -// { -// DictionaryEncoding encoder; -// encoder.Encode(stringEncodingInput, output, tokenizer); + { + DictionaryEncoding encoder; + encoder.Encode(stringEncodingInput, output, tokenizer); -// for (const string& token : encoder.Dictionary().Tokens()) -// { -// naiveDictionary.emplace_back(token, encoder.Dictionary().Value(token)); -// } + for (const string& token : encoder.Dictionary().Tokens()) + { + naiveDictionary.emplace_back(token, encoder.Dictionary().Value(token)); + } -// encoderCopy = DictionaryEncoding(encoder); -// } + encoderCopy = DictionaryEncoding(encoder); + } -// const DictionaryType& copiedDictionary = encoderCopy.Dictionary(); + const DictionaryType& copiedDictionary = encoderCopy.Dictionary(); -// BOOST_REQUIRE_EQUAL(naiveDictionary.size(), copiedDictionary.Size()); + BOOST_REQUIRE_EQUAL(naiveDictionary.size(), copiedDictionary.Size()); -// for (const pair& keyValue : naiveDictionary) -// { -// BOOST_REQUIRE(copiedDictionary.HasToken(keyValue.first)); -// BOOST_REQUIRE_EQUAL(copiedDictionary.Value(keyValue.first), -// keyValue.second); -// } -// } + for (const pair& keyValue : naiveDictionary) + { + BOOST_REQUIRE(copiedDictionary.HasToken(keyValue.first)); + BOOST_REQUIRE_EQUAL(copiedDictionary.Value(keyValue.first), + keyValue.second); + } +} -// /** -// * Test the move assignment operator. -// */ -// BOOST_AUTO_TEST_CASE(StringEncodingMoveTest) -// { -// using DictionaryType = StringEncodingDictionary; -// arma::sp_mat output; -// DictionaryEncoding encoderCopy; -// SplitByAnyOf tokenizer(" ,."); +/** + * Test the move assignment operator. + */ +BOOST_AUTO_TEST_CASE(StringEncodingMoveTest) +{ + using DictionaryType = StringEncodingDictionary; + arma::sp_mat output; + DictionaryEncoding encoderCopy; + SplitByAnyOf tokenizer(" ,."); -// vector> naiveDictionary; + vector> naiveDictionary; -// { -// DictionaryEncoding encoder; -// encoder.Encode(stringEncodingInput, output, tokenizer); + { + DictionaryEncoding encoder; + encoder.Encode(stringEncodingInput, output, tokenizer); -// for (const string& token : encoder.Dictionary().Tokens()) -// { -// naiveDictionary.emplace_back(token, encoder.Dictionary().Value(token)); -// } + for (const string& token : encoder.Dictionary().Tokens()) + { + naiveDictionary.emplace_back(token, encoder.Dictionary().Value(token)); + } -// encoderCopy = std::move(encoder); -// } + encoderCopy = std::move(encoder); + } -// const DictionaryType& copiedDictionary = encoderCopy.Dictionary(); + const DictionaryType& copiedDictionary = encoderCopy.Dictionary(); -// BOOST_REQUIRE_EQUAL(naiveDictionary.size(), copiedDictionary.Size()); + BOOST_REQUIRE_EQUAL(naiveDictionary.size(), copiedDictionary.Size()); -// for (const pair& keyValue : naiveDictionary) -// { -// BOOST_REQUIRE(copiedDictionary.HasToken(keyValue.first)); -// BOOST_REQUIRE_EQUAL(copiedDictionary.Value(keyValue.first), -// keyValue.second); -// } -// } + for (const pair& keyValue : naiveDictionary) + { + BOOST_REQUIRE(copiedDictionary.HasToken(keyValue.first)); + BOOST_REQUIRE_EQUAL(copiedDictionary.Value(keyValue.first), + keyValue.second); + } +} /** * The function checks that the given dictionaries contain the same data. @@ -342,53 +342,19 @@ void CheckDictionaries(const StringEncodingDictionary& expected, } } -// /** -// * Serialization test for the dictionary encoding algorithm with -// * the SplitByAnyOf tokenizer. -// */ -// BOOST_AUTO_TEST_CASE(SplitByAnyOfDictionaryEncodingSerialization) -// { -// using EncoderType = DictionaryEncoding; - -// EncoderType encoder; -// SplitByAnyOf tokenizer(" ,."); -// arma::mat output; - -// encoder.Encode(stringEncodingInput, output, tokenizer); - -// EncoderType xmlEncoder, textEncoder, binaryEncoder; -// arma::mat xmlOutput, textOutput, binaryOutput; - -// SerializeObjectAll(encoder, xmlEncoder, textEncoder, binaryEncoder); - -// CheckDictionaries(encoder.Dictionary(), xmlEncoder.Dictionary()); -// CheckDictionaries(encoder.Dictionary(), textEncoder.Dictionary()); -// CheckDictionaries(encoder.Dictionary(), binaryEncoder.Dictionary()); - -// xmlEncoder.Encode(stringEncodingInput, xmlOutput, tokenizer); -// textEncoder.Encode(stringEncodingInput, textOutput, tokenizer); -// binaryEncoder.Encode(stringEncodingInput, binaryOutput, tokenizer); - -// CheckMatrices(output, xmlOutput, textOutput, binaryOutput); -// } - /** * Serialization test for the dictionary encoding algorithm with - * the CharExtract tokenizer. + * the SplitByAnyOf tokenizer. */ -BOOST_AUTO_TEST_CASE(CharExtractDictionaryEncodingSerialization) +BOOST_AUTO_TEST_CASE(SplitByAnyOfDictionaryEncodingSerialization) { - using EncoderType = BagOfWordsEncoding; + using EncoderType = DictionaryEncoding; EncoderType encoder; - CharExtract tokenizer; + SplitByAnyOf tokenizer(" ,."); arma::mat output; - std::vector input = { - "GACCA", - "ABCABCD", - "GAB" - }; - encoder.Encode(input, output, tokenizer); + + encoder.Encode(stringEncodingInput, output, tokenizer); EncoderType xmlEncoder, textEncoder, binaryEncoder; arma::mat xmlOutput, textOutput, binaryOutput; @@ -398,132 +364,161 @@ BOOST_AUTO_TEST_CASE(CharExtractDictionaryEncodingSerialization) CheckDictionaries(encoder.Dictionary(), xmlEncoder.Dictionary()); CheckDictionaries(encoder.Dictionary(), textEncoder.Dictionary()); CheckDictionaries(encoder.Dictionary(), binaryEncoder.Dictionary()); - std::cout<<"Calling the actual object"<; +/** + * Serialization test for the Bag Of Words encoding algorithm with + * the CharExtract tokenizer. + */ +BOOST_AUTO_TEST_CASE(CharExtractBagOfWordsEncodingSerialization) +{ + using EncoderType = BagOfWordsEncoding; -// arma::mat output; -// BagOfWordsEncoding encoder; -// SplitByAnyOf tokenizer(" "); + EncoderType encoder; + CharExtract tokenizer; + arma::mat output; + encoder.Encode(stringEncodingInput, output, tokenizer); -// encoder.Encode(stringEncodingInput, output, tokenizer); -// const DictionaryType& dictionary = encoder.Dictionary(); + EncoderType xmlEncoder, textEncoder, binaryEncoder; + arma::mat xmlOutput, textOutput, binaryOutput; -// // Checking that everything is mapped to different numbers -// std::unordered_map keysCount; -// for (auto& keyValue : dictionary.Mapping()) -// { -// keysCount[keyValue.second]++; -// // Every token should be mapped only once -// BOOST_REQUIRE_EQUAL(keysCount[keyValue.second], 1); -// } -// arma::mat expected = { -// { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, -// 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, -// { 0, 1, 0, 0, 0, 1, 0, 0, 1, 1, 0, 0, 0, 1, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, -// 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, -// { 1, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -// 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 } -// }; -// CheckMatrices(output, expected); -// } + SerializeObjectAll(encoder, xmlEncoder, textEncoder, binaryEncoder); -// * -// * Test the one pass modification of the Bag of Words encoding algorithm. + CheckDictionaries(encoder.Dictionary(), xmlEncoder.Dictionary()); + CheckDictionaries(encoder.Dictionary(), textEncoder.Dictionary()); + CheckDictionaries(encoder.Dictionary(), binaryEncoder.Dictionary()); + + xmlEncoder.Encode(stringEncodingInput, xmlOutput, tokenizer); + textEncoder.Encode(stringEncodingInput, textOutput, tokenizer); + binaryEncoder.Encode(stringEncodingInput, binaryOutput, tokenizer); + + CheckMatrices(output, xmlOutput, textOutput, binaryOutput); +} + +BOOST_AUTO_TEST_CASE(BagOfWordsEncodingTest) +{ + using DictionaryType = StringEncodingDictionary; + + arma::mat output; + BagOfWordsEncoding encoder; + SplitByAnyOf tokenizer(" "); + + encoder.Encode(stringEncodingInput, output, tokenizer); + const DictionaryType& dictionary = encoder.Dictionary(); + + // Checking that everything is mapped to different numbers + std::unordered_map keysCount; + for (auto& keyValue : dictionary.Mapping()) + { + keysCount[keyValue.second]++; + // Every token should be mapped only once + BOOST_REQUIRE_EQUAL(keysCount[keyValue.second], 1); + } + arma::mat expected = { + { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, + { 0, 1, 0, 0, 0, 1, 0, 0, 1, 1, 0, 0, 0, 1, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, + { 1, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 } + }; + CheckMatrices(output, expected); +} + +* + * Test the one pass modification of the Bag of Words encoding algorithm. -// BOOST_AUTO_TEST_CASE(OnePassBagOfWordsEncodingTest) -// { -// using DictionaryType = StringEncodingDictionary; +BOOST_AUTO_TEST_CASE(OnePassBagOfWordsEncodingTest) +{ + using DictionaryType = StringEncodingDictionary; -// vector> output; -// BagOfWordsEncoding encoder( -// (BagOfWordsEncodingPolicy())); -// SplitByAnyOf tokenizer(" "); + vector> output; + BagOfWordsEncoding encoder( + (BagOfWordsEncodingPolicy())); + SplitByAnyOf tokenizer(" "); -// encoder.Encode(stringEncodingInput, output, tokenizer); + encoder.Encode(stringEncodingInput, output, tokenizer); -// const DictionaryType& dictionary = encoder.Dictionary(); + const DictionaryType& dictionary = encoder.Dictionary(); -// // Checking that everything is mapped to different numbers -// std::unordered_map keysCount; -// for (auto& keyValue : dictionary.Mapping()) -// { -// keysCount[keyValue.second]++; -// // Every token should be mapped only once -// BOOST_REQUIRE_EQUAL(keysCount[keyValue.second], 1); -// } + // Checking that everything is mapped to different numbers + std::unordered_map keysCount; + for (auto& keyValue : dictionary.Mapping()) + { + keysCount[keyValue.second]++; + // Every token should be mapped only once + BOOST_REQUIRE_EQUAL(keysCount[keyValue.second], 1); + } -// vector> expected = { -// { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, -// 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, -// { 0, 1, 0, 0, 0, 1, 0, 0, 1, 1, 0, 0, 0, 1, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, -// 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, -// { 1, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -// 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 } -// }; + vector> expected = { + { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, + { 0, 1, 0, 0, 0, 1, 0, 0, 1, 1, 0, 0, 0, 1, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, + { 1, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 } + }; -// BOOST_REQUIRE(output == expected); -// } + BOOST_REQUIRE(output == expected); +} -// /** -// * Test Bag of Words encoding for characters using lamda function. -// */ -// BOOST_AUTO_TEST_CASE(BagOfWordsEncodingIndividualCharactersTest) -// { -// vector input = { -// "GACCA", -// "ABCABCD", -// "GAB" -// }; +/** +* Test Bag of Words encoding for characters using lamda function. +*/ +BOOST_AUTO_TEST_CASE(BagOfWordsEncodingIndividualCharactersTest) +{ + vector input = { + "GACCA", + "ABCABCD", + "GAB" + }; -// arma::mat output; -// BagOfWordsEncoding encoder; + arma::mat output; + BagOfWordsEncoding encoder; -// // Passing a empty string to encode characters -// encoder.Encode(input, output, CharExtract()); -// arma::mat target = { -// { 1, 1, 1, 0, 0 }, -// { 0, 1, 1, 1, 1 }, -// { 1, 1, 0, 1, 0 } -// }; + // Passing a empty string to encode characters + encoder.Encode(input, output, CharExtract()); + arma::mat target = { + { 1, 1, 1, 0, 0 }, + { 0, 1, 1, 1, 1 }, + { 1, 1, 0, 1, 0 } + }; -// CheckMatrices(output, target); -// } + CheckMatrices(output, target); +} -// /** -// * Test the one pass modification of the Bag of Words encoding algorithm -// * in case of individual character encoding. -// */ -// BOOST_AUTO_TEST_CASE(OnePassBagOfWordsEncodingIndividualCharactersTest) -// { -// std::vector input = { -// "GACCA", -// "ABCABCD", -// "GAB" -// }; +/** + * Test the one pass modification of the Bag of Words encoding algorithm + * in case of individual character encoding. + */ +BOOST_AUTO_TEST_CASE(OnePassBagOfWordsEncodingIndividualCharactersTest) +{ + std::vector input = { + "GACCA", + "ABCABCD", + "GAB" + }; -// vector> output; -// BagOfWordsEncoding encoder; + vector> output; + BagOfWordsEncoding encoder; -// // Passing a empty string to encode characters -// encoder.Encode(input, output, CharExtract()); + // Passing a empty string to encode characters + encoder.Encode(input, output, CharExtract()); -// vector> expected = { -// { 1, 1, 1, 0, 0 }, -// { 0, 1, 1, 1, 1 }, -// { 1, 1, 0, 1, 0 } -// }; + vector> expected = { + { 1, 1, 1, 0, 0 }, + { 0, 1, 1, 1, 1 }, + { 1, 1, 0, 1, 0 } + }; -// BOOST_REQUIRE(output == expected); -// } + BOOST_REQUIRE(output == expected); +} // /** // * Test the Tf-Idf Encoding using rawcount type and smoothidf as true, @@ -1008,10 +1003,10 @@ BOOST_AUTO_TEST_CASE(CharExtractDictionaryEncodingSerialization) // } /** - * Serialization test for the dictionary encoding algorithm with + * Serialization test for the TF-IDF encoding algorithm with * the CharExtract tokenizer. */ -BOOST_AUTO_TEST_CASE(CharExtractBagOfWordsEncodingSerialization) +BOOST_AUTO_TEST_CASE(CharExtractTfIdfEncodingSerialization) { using EncoderType = TfIdfEncoding; From f91a81e10731bd19c059fe4a93838490d7e4e374 Mon Sep 17 00:00:00 2001 From: jeffinsam Date: Sat, 10 Aug 2019 12:23:07 +0530 Subject: [PATCH 017/111] uncommenting out test which is needed, next step to debug tfidf --- src/mlpack/tests/string_encoding_test.cpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/mlpack/tests/string_encoding_test.cpp b/src/mlpack/tests/string_encoding_test.cpp index 3bf3cbf68a..1e61d1f3f4 100644 --- a/src/mlpack/tests/string_encoding_test.cpp +++ b/src/mlpack/tests/string_encoding_test.cpp @@ -373,12 +373,12 @@ BOOST_AUTO_TEST_CASE(SplitByAnyOfDictionaryEncodingSerialization) } /** - * Serialization test for the Bag Of Words encoding algorithm with + * Serialization test for the dictionary encoding algorithm with * the CharExtract tokenizer. */ -BOOST_AUTO_TEST_CASE(CharExtractBagOfWordsEncodingSerialization) +BOOST_AUTO_TEST_CASE(CharExtractDictionaryEncodingSerialization) { - using EncoderType = BagOfWordsEncoding; + using EncoderType = DictionaryEncoding; EncoderType encoder; CharExtract tokenizer; @@ -431,9 +431,9 @@ BOOST_AUTO_TEST_CASE(BagOfWordsEncodingTest) CheckMatrices(output, expected); } -* - * Test the one pass modification of the Bag of Words encoding algorithm. - +/** +* Test the one pass modification of the Bag of Words encoding algorithm. +*/ BOOST_AUTO_TEST_CASE(OnePassBagOfWordsEncodingTest) { using DictionaryType = StringEncodingDictionary; From bf72cc958125519ed4b503d3f089689cc5c29eaa Mon Sep 17 00:00:00 2001 From: jeffinsam Date: Sat, 10 Aug 2019 12:44:38 +0530 Subject: [PATCH 018/111] Wrote serialization test, next to resolve comments --- .../tf_idf_encoding_policy.hpp | 11 +- src/mlpack/tests/string_encoding_test.cpp | 900 +++++++++--------- 2 files changed, 463 insertions(+), 448 deletions(-) diff --git a/src/mlpack/core/data/string_encoding_policies/tf_idf_encoding_policy.hpp b/src/mlpack/core/data/string_encoding_policies/tf_idf_encoding_policy.hpp index 42b16aa3b3..88bbdbff01 100644 --- a/src/mlpack/core/data/string_encoding_policies/tf_idf_encoding_policy.hpp +++ b/src/mlpack/core/data/string_encoding_policies/tf_idf_encoding_policy.hpp @@ -69,12 +69,11 @@ class TfIdfEncodingPolicy * @param dictionarySize The size of the dictionary (not used). */ template - static void InitMatrix(MatType& output, + void InitMatrix(MatType& output, size_t datasetSize, size_t /*maxNumTokens*/, size_t dictionarySize) { - std::cout<<"dataset "<= tokenCount.size()) { row_size.push_back(0); tokenCount.push_back(std::unordered_map()); } - std::cout<<"error"<; +/** + * Test the Tf-Idf Encoding using rawcount type and smoothidf as true, + * which is the deafult values used for algorithim. + */ +BOOST_AUTO_TEST_CASE(RawCountSmoothIdfEncodingTest) +{ + using DictionaryType = StringEncodingDictionary; -// arma::mat output; -// TfIdfEncoding encoder; -// SplitByAnyOf tokenizer(" "); + arma::mat output; + TfIdfEncoding encoder; + SplitByAnyOf tokenizer(" "); -// encoder.Encode(stringEncodingInput, output, tokenizer); + encoder.Encode(stringEncodingInput, output, tokenizer); -// const DictionaryType& dictionary = encoder.Dictionary(); + const DictionaryType& dictionary = encoder.Dictionary(); -// // Checking that everything is mapped to different numbers -// std::unordered_map keysCount; -// for (auto& keyValue : dictionary.Mapping()) -// { -// keysCount[keyValue.second]++; -// // Every token should be mapped only once -// BOOST_REQUIRE_EQUAL(keysCount[keyValue.second], 1); -// } -// arma::mat expected = { -// { 1.28768207245178, 1.28768207245178, 1.69314718055995, 1.69314718055995, -// 1.69314718055995, 1, 1.69314718055995, 1.28768207245178, -// 1.28768207245178, 1.28768207245178, 1.69314718055995, 1.69314718055995, -// 1.69314718055995, 1, 1.69314718055995, 1.69314718055995, 0, 0, 0, 0, 0, -// 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -// 0 }, -// { 0, 1.28768207245178, 0, 0, 0, 2, 0, 0, 3.86304621735534, -// 3.86304621735534, 0, 0, 0, 3, 0, 0, 1.69314718055995, 1.69314718055995, -// 1.69314718055995, 5.07944154167984, 1.69314718055995, 1.69314718055995, -// 1.69314718055995, 1.69314718055995, 1.69314718055995, 1.69314718055995, -// 1.69314718055995, 1.69314718055995, 1.69314718055995, 1.69314718055995, -// 1.69314718055995, 1.69314718055995, 1.69314718055995, 1.69314718055995, -// 1.69314718055995, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, -// { 1.28768207245178, 0, 0, 0, 0, 1, 0, 1.28768207245178, 0, 0, 0, 0, 0, 1, -// 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -// 1.69314718055995, 1.69314718055995, 1.69314718055995, 1.69314718055995, -// 1.69314718055995, 1.69314718055995, 1.69314718055995, 1.69314718055995, -// 1.69314718055995, 1.69314718055995, 1.69314718055995 } -// }; -// CheckMatrices(output, expected, 1e-12); -// } + // Checking that everything is mapped to different numbers + std::unordered_map keysCount; + for (auto& keyValue : dictionary.Mapping()) + { + keysCount[keyValue.second]++; + // Every token should be mapped only once + BOOST_REQUIRE_EQUAL(keysCount[keyValue.second], 1); + } + arma::mat expected = { + { 1.28768207245178, 1.28768207245178, 1.69314718055995, 1.69314718055995, + 1.69314718055995, 1, 1.69314718055995, 1.28768207245178, + 1.28768207245178, 1.28768207245178, 1.69314718055995, 1.69314718055995, + 1.69314718055995, 1, 1.69314718055995, 1.69314718055995, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0 }, + { 0, 1.28768207245178, 0, 0, 0, 2, 0, 0, 3.86304621735534, + 3.86304621735534, 0, 0, 0, 3, 0, 0, 1.69314718055995, 1.69314718055995, + 1.69314718055995, 5.07944154167984, 1.69314718055995, 1.69314718055995, + 1.69314718055995, 1.69314718055995, 1.69314718055995, 1.69314718055995, + 1.69314718055995, 1.69314718055995, 1.69314718055995, 1.69314718055995, + 1.69314718055995, 1.69314718055995, 1.69314718055995, 1.69314718055995, + 1.69314718055995, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, + { 1.28768207245178, 0, 0, 0, 0, 1, 0, 1.28768207245178, 0, 0, 0, 0, 0, 1, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 1.69314718055995, 1.69314718055995, 1.69314718055995, 1.69314718055995, + 1.69314718055995, 1.69314718055995, 1.69314718055995, 1.69314718055995, + 1.69314718055995, 1.69314718055995, 1.69314718055995 } + }; + CheckMatrices(output, expected, 1e-12); +} -// /** -// * Test the one pass modification of the TfIdf encoding algorithm, using rawcount -// * as type of tf and smoothIdf as true. -// */ -// BOOST_AUTO_TEST_CASE(OnePassRawCountSmoothIdfEncodingTest) -// { -// using DictionaryType = StringEncodingDictionary; +/** + * Test the one pass modification of the TfIdf encoding algorithm, using rawcount + * as type of tf and smoothIdf as true. + */ +BOOST_AUTO_TEST_CASE(OnePassRawCountSmoothIdfEncodingTest) +{ + using DictionaryType = StringEncodingDictionary; -// vector> output; -// TfIdfEncoding encoder( -// (TfIdfEncodingPolicy())); -// SplitByAnyOf tokenizer(" "); + vector> output; + TfIdfEncoding encoder( + (TfIdfEncodingPolicy())); + SplitByAnyOf tokenizer(" "); -// encoder.Encode(stringEncodingInput, output, tokenizer); + encoder.Encode(stringEncodingInput, output, tokenizer); -// const DictionaryType& dictionary = encoder.Dictionary(); + const DictionaryType& dictionary = encoder.Dictionary(); -// // Checking that everything is mapped to different numbers -// std::unordered_map keysCount; -// for (auto& keyValue : dictionary.Mapping()) -// { -// keysCount[keyValue.second]++; -// // Every token should be mapped only once -// BOOST_REQUIRE_EQUAL(keysCount[keyValue.second], 1); -// } + // Checking that everything is mapped to different numbers + std::unordered_map keysCount; + for (auto& keyValue : dictionary.Mapping()) + { + keysCount[keyValue.second]++; + // Every token should be mapped only once + BOOST_REQUIRE_EQUAL(keysCount[keyValue.second], 1); + } -// vector> expected = { -// { 1.28768207245178, 1.28768207245178, 1.69314718055995, 1.69314718055995, -// 1.69314718055995, 1, 1.69314718055995, 1.28768207245178, -// 1.28768207245178, 1.28768207245178, 1.69314718055995, 1.69314718055995, -// 1.69314718055995, 1, 1.69314718055995, 1.69314718055995, 0, 0, 0, 0, 0, -// 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -// 0 }, -// { 0, 1.28768207245178, 0, 0, 0, 2, 0, 0, 3.86304621735534, -// 3.86304621735534, 0, 0, 0, 3, 0, 0, 1.69314718055995, 1.69314718055995, -// 1.69314718055995, 5.07944154167984, 1.69314718055995, 1.69314718055995, -// 1.69314718055995, 1.69314718055995, 1.69314718055995, 1.69314718055995, -// 1.69314718055995, 1.69314718055995, 1.69314718055995, 1.69314718055995, -// 1.69314718055995, 1.69314718055995, 1.69314718055995, 1.69314718055995, -// 1.69314718055995, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, -// { 1.28768207245178, 0, 0, 0, 0, 1, 0, 1.28768207245178, 0, 0, 0, 0, 0, 1, -// 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -// 1.69314718055995, 1.69314718055995, 1.69314718055995, 1.69314718055995, -// 1.69314718055995, 1.69314718055995, 1.69314718055995, 1.69314718055995, -// 1.69314718055995, 1.69314718055995, 1.69314718055995 } -// }; -// for (size_t i = 0; i < expected.size(); i++) -// for (size_t j = 0; j < expected[i].size(); j++) -// BOOST_REQUIRE_CLOSE(expected[i][j], output[i][j], 1e-12); -// } + vector> expected = { + { 1.28768207245178, 1.28768207245178, 1.69314718055995, 1.69314718055995, + 1.69314718055995, 1, 1.69314718055995, 1.28768207245178, + 1.28768207245178, 1.28768207245178, 1.69314718055995, 1.69314718055995, + 1.69314718055995, 1, 1.69314718055995, 1.69314718055995, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0 }, + { 0, 1.28768207245178, 0, 0, 0, 2, 0, 0, 3.86304621735534, + 3.86304621735534, 0, 0, 0, 3, 0, 0, 1.69314718055995, 1.69314718055995, + 1.69314718055995, 5.07944154167984, 1.69314718055995, 1.69314718055995, + 1.69314718055995, 1.69314718055995, 1.69314718055995, 1.69314718055995, + 1.69314718055995, 1.69314718055995, 1.69314718055995, 1.69314718055995, + 1.69314718055995, 1.69314718055995, 1.69314718055995, 1.69314718055995, + 1.69314718055995, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, + { 1.28768207245178, 0, 0, 0, 0, 1, 0, 1.28768207245178, 0, 0, 0, 0, 0, 1, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 1.69314718055995, 1.69314718055995, 1.69314718055995, 1.69314718055995, + 1.69314718055995, 1.69314718055995, 1.69314718055995, 1.69314718055995, + 1.69314718055995, 1.69314718055995, 1.69314718055995 } + }; + for (size_t i = 0; i < expected.size(); i++) + for (size_t j = 0; j < expected[i].size(); j++) + BOOST_REQUIRE_CLOSE(expected[i][j], output[i][j], 1e-12); +} -// /** -// * Test TFIDF encoding for characters using lamda function, using rawcount as tf -// * type and smoothidf as true. -// */ -// BOOST_AUTO_TEST_CASE(RawCountSmoothIdfEncodingIndividualCharactersTest) -// { -// vector input = { -// "GACCA", -// "ABCABCD", -// "GAB" -// }; +/** + * Test TFIDF encoding for characters using lamda function, using rawcount as tf + * type and smoothidf as true. + */ +BOOST_AUTO_TEST_CASE(RawCountSmoothIdfEncodingIndividualCharactersTest) +{ + vector input = { + "GACCA", + "ABCABCD", + "GAB" + }; -// arma::mat output; -// TfIdfEncoding encoder; + arma::mat output; + TfIdfEncoding encoder; -// // Passing a empty string to encode charactersrawcountsmoothidftrue -// encoder.Encode(input, output, CharExtract()); -// arma::mat target = { -// { 1.2876820724517808, 2, 2.5753641449035616, 0, 0 }, -// { 0, 2, 2.5753641449035616, 2.5753641449035616, 1.6931471805599454 }, -// { 1.2876820724517808, 1, 0, 1.2876820724517808, 0 } -// }; -// CheckMatrices(output, target, 1e-12); -// } + // Passing a empty string to encode charactersrawcountsmoothidftrue + encoder.Encode(input, output, CharExtract()); + arma::mat target = { + { 1.2876820724517808, 2, 2.5753641449035616, 0, 0 }, + { 0, 2, 2.5753641449035616, 2.5753641449035616, 1.6931471805599454 }, + { 1.2876820724517808, 1, 0, 1.2876820724517808, 0 } + }; + CheckMatrices(output, target, 1e-12); +} -// /** -// * Test the one pass modification of the Tf-Idf encoding algorithm -// * in case of individual character encoding using default values. -// */ -// BOOST_AUTO_TEST_CASE(OnePassRawCountSmoothIdfEncodingIndividualCharactersTest) -// { -// std::vector input = { -// "GACCA", -// "ABCABCD", -// "GAB" -// }; +/** + * Test the one pass modification of the Tf-Idf encoding algorithm + * in case of individual character encoding using default values. + */ +BOOST_AUTO_TEST_CASE(OnePassRawCountSmoothIdfEncodingIndividualCharactersTest) +{ + std::vector input = { + "GACCA", + "ABCABCD", + "GAB" + }; -// vector> output; -// TfIdfEncoding encoder; + vector> output; + TfIdfEncoding encoder; -// // Passing a empty string to encode characters -// encoder.Encode(input, output, CharExtract()); -// vector> expected = { -// { 1.2876820724517808, 2, 2.5753641449035616, 0, 0 }, -// { 0, 2, 2.5753641449035616, 2.5753641449035616, 1.6931471805599454 }, -// { 1.2876820724517808, 1, 0, 1.2876820724517808, 0 } -// }; -// for (size_t i = 0; i < expected.size(); i++) -// for (size_t j = 0; j < expected[i].size(); j++) -// BOOST_REQUIRE_CLOSE(expected[i][j], output[i][j], 1e-12); -// } + // Passing a empty string to encode characters + encoder.Encode(input, output, CharExtract()); + vector> expected = { + { 1.2876820724517808, 2, 2.5753641449035616, 0, 0 }, + { 0, 2, 2.5753641449035616, 2.5753641449035616, 1.6931471805599454 }, + { 1.2876820724517808, 1, 0, 1.2876820724517808, 0 } + }; + for (size_t i = 0; i < expected.size(); i++) + for (size_t j = 0; j < expected[i].size(); j++) + BOOST_REQUIRE_CLOSE(expected[i][j], output[i][j], 1e-12); +} -// /** -// * Test the Tf-Idf Encoding using rawcount type and smoothidf as false. -// */ -// BOOST_AUTO_TEST_CASE(TfIdfRawCountEncodingTest) -// { -// using DictionaryType = StringEncodingDictionary; +/** + * Test the Tf-Idf Encoding using rawcount type and smoothidf as false. + */ +BOOST_AUTO_TEST_CASE(TfIdfRawCountEncodingTest) +{ + using DictionaryType = StringEncodingDictionary; -// arma::mat output; -// TfIdfEncoding encoder( -// (TfIdfEncodingPolicy(0, false))); -// SplitByAnyOf tokenizer(" "); + arma::mat output; + TfIdfEncoding encoder( + (TfIdfEncodingPolicy(0, false))); + SplitByAnyOf tokenizer(" "); -// encoder.Encode(stringEncodingInput, output, tokenizer); + encoder.Encode(stringEncodingInput, output, tokenizer); -// const DictionaryType& dictionary = encoder.Dictionary(); + const DictionaryType& dictionary = encoder.Dictionary(); -// // Checking that everything is mapped to different numbers -// std::unordered_map keysCount; -// for (auto& keyValue : dictionary.Mapping()) -// { -// keysCount[keyValue.second]++; -// // Every token should be mapped only once -// BOOST_REQUIRE_EQUAL(keysCount[keyValue.second], 1); -// } + // Checking that everything is mapped to different numbers + std::unordered_map keysCount; + for (auto& keyValue : dictionary.Mapping()) + { + keysCount[keyValue.second]++; + // Every token should be mapped only once + BOOST_REQUIRE_EQUAL(keysCount[keyValue.second], 1); + } -// arma::mat expected = { -// { 1.40546510810816, 1.40546510810816, 2.09861228866811, 2.09861228866811, -// 2.09861228866811, 1, 2.09861228866811, 1.40546510810816, -// 1.40546510810816, 1.40546510810816, 2.09861228866811, 2.09861228866811, -// 2.09861228866811, 1, 2.09861228866811, 2.09861228866811, 0, 0, 0, 0, 0, -// 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -// 0 }, -// { 0, 1.40546510810816, 0, 0, 0, 2, 0, 0, 4.21639532432449, -// 4.21639532432449, 0, 0, 0, 3, 0, 0, 2.09861228866811, 2.09861228866811, -// 2.09861228866811, 6.29583686600433, 2.09861228866811, 2.09861228866811, -// 2.09861228866811, 2.09861228866811, 2.09861228866811, 2.09861228866811, -// 2.09861228866811, 2.09861228866811, 2.09861228866811, 2.09861228866811, -// 2.09861228866811, 2.09861228866811, 2.09861228866811, 2.09861228866811, -// 2.09861228866811, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, -// { 1.40546510810816, 0, 0, 0, 0, 1, 0, 1.40546510810816, 0, 0, 0, 0, 0, 1, -// 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -// 2.09861228866811, 2.09861228866811, 2.09861228866811, 2.09861228866811, -// 2.09861228866811, 2.09861228866811, 2.09861228866811, 2.09861228866811, -// 2.09861228866811, 2.09861228866811, 2.09861228866811 } -// }; -// CheckMatrices(output, expected, 1e-12); -// } + arma::mat expected = { + { 1.40546510810816, 1.40546510810816, 2.09861228866811, 2.09861228866811, + 2.09861228866811, 1, 2.09861228866811, 1.40546510810816, + 1.40546510810816, 1.40546510810816, 2.09861228866811, 2.09861228866811, + 2.09861228866811, 1, 2.09861228866811, 2.09861228866811, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0 }, + { 0, 1.40546510810816, 0, 0, 0, 2, 0, 0, 4.21639532432449, + 4.21639532432449, 0, 0, 0, 3, 0, 0, 2.09861228866811, 2.09861228866811, + 2.09861228866811, 6.29583686600433, 2.09861228866811, 2.09861228866811, + 2.09861228866811, 2.09861228866811, 2.09861228866811, 2.09861228866811, + 2.09861228866811, 2.09861228866811, 2.09861228866811, 2.09861228866811, + 2.09861228866811, 2.09861228866811, 2.09861228866811, 2.09861228866811, + 2.09861228866811, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, + { 1.40546510810816, 0, 0, 0, 0, 1, 0, 1.40546510810816, 0, 0, 0, 0, 0, 1, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 2.09861228866811, 2.09861228866811, 2.09861228866811, 2.09861228866811, + 2.09861228866811, 2.09861228866811, 2.09861228866811, 2.09861228866811, + 2.09861228866811, 2.09861228866811, 2.09861228866811 } + }; + CheckMatrices(output, expected, 1e-12); +} -// * -// * Test the one pass modification of the TfIdf encoding algorithm, with rawcount -// * as type, but with smoothidf as false. - -// BOOST_AUTO_TEST_CASE(OnePassTfIdfRawCountEncodingTest) -// { -// using DictionaryType = StringEncodingDictionary; +/** + * Test the one pass modification of the TfIdf encoding algorithm, with rawcount + * as type, but with smoothidf as false. + */ +BOOST_AUTO_TEST_CASE(OnePassTfIdfRawCountEncodingTest) +{ + using DictionaryType = StringEncodingDictionary; -// vector> output; -// TfIdfEncoding encoder(0, false); -// SplitByAnyOf tokenizer(" "); + vector> output; + TfIdfEncoding encoder(0, false); + SplitByAnyOf tokenizer(" "); -// encoder.Encode(stringEncodingInput, output, tokenizer); + encoder.Encode(stringEncodingInput, output, tokenizer); -// const DictionaryType& dictionary = encoder.Dictionary(); + const DictionaryType& dictionary = encoder.Dictionary(); -// // Checking that everything is mapped to different numbers -// std::unordered_map keysCount; -// for (auto& keyValue : dictionary.Mapping()) -// { -// keysCount[keyValue.second]++; -// // Every token should be mapped only once -// BOOST_REQUIRE_EQUAL(keysCount[keyValue.second], 1); -// } + // Checking that everything is mapped to different numbers + std::unordered_map keysCount; + for (auto& keyValue : dictionary.Mapping()) + { + keysCount[keyValue.second]++; + // Every token should be mapped only once + BOOST_REQUIRE_EQUAL(keysCount[keyValue.second], 1); + } -// vector> expected = { -// { 1.40546510810816, 1.40546510810816, 2.09861228866811, 2.09861228866811, -// 2.09861228866811, 1, 2.09861228866811, 1.40546510810816, -// 1.40546510810816, 1.40546510810816, 2.09861228866811, 2.09861228866811, -// 2.09861228866811, 1, 2.09861228866811, 2.09861228866811, 0, 0, 0, 0, 0, -// 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -// 0 }, -// { 0, 1.40546510810816, 0, 0, 0, 2, 0, 0, 4.21639532432449, -// 4.21639532432449, 0, 0, 0, 3, 0, 0, 2.09861228866811, 2.09861228866811, -// 2.09861228866811, 6.29583686600433, 2.09861228866811, 2.09861228866811, -// 2.09861228866811, 2.09861228866811, 2.09861228866811, 2.09861228866811, -// 2.09861228866811, 2.09861228866811, 2.09861228866811, 2.09861228866811, -// 2.09861228866811, 2.09861228866811, 2.09861228866811, 2.09861228866811, -// 2.09861228866811, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, -// { 1.40546510810816, 0, 0, 0, 0, 1, 0, 1.40546510810816, 0, 0, 0, 0, 0, 1, -// 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -// 2.09861228866811, 2.09861228866811, 2.09861228866811, 2.09861228866811, -// 2.09861228866811, 2.09861228866811, 2.09861228866811, 2.09861228866811, -// 2.09861228866811, 2.09861228866811, 2.09861228866811 } -// }; -// for (size_t i = 0; i < expected.size(); i++) -// for (size_t j = 0; j < expected[i].size(); j++) -// BOOST_REQUIRE_CLOSE(expected[i][j], output[i][j], 1e-12); -// } + vector> expected = { + { 1.40546510810816, 1.40546510810816, 2.09861228866811, 2.09861228866811, + 2.09861228866811, 1, 2.09861228866811, 1.40546510810816, + 1.40546510810816, 1.40546510810816, 2.09861228866811, 2.09861228866811, + 2.09861228866811, 1, 2.09861228866811, 2.09861228866811, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0 }, + { 0, 1.40546510810816, 0, 0, 0, 2, 0, 0, 4.21639532432449, + 4.21639532432449, 0, 0, 0, 3, 0, 0, 2.09861228866811, 2.09861228866811, + 2.09861228866811, 6.29583686600433, 2.09861228866811, 2.09861228866811, + 2.09861228866811, 2.09861228866811, 2.09861228866811, 2.09861228866811, + 2.09861228866811, 2.09861228866811, 2.09861228866811, 2.09861228866811, + 2.09861228866811, 2.09861228866811, 2.09861228866811, 2.09861228866811, + 2.09861228866811, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, + { 1.40546510810816, 0, 0, 0, 0, 1, 0, 1.40546510810816, 0, 0, 0, 0, 0, 1, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 2.09861228866811, 2.09861228866811, 2.09861228866811, 2.09861228866811, + 2.09861228866811, 2.09861228866811, 2.09861228866811, 2.09861228866811, + 2.09861228866811, 2.09861228866811, 2.09861228866811 } + }; + for (size_t i = 0; i < expected.size(); i++) + for (size_t j = 0; j < expected[i].size(); j++) + BOOST_REQUIRE_CLOSE(expected[i][j], output[i][j], 1e-12); +} -// /** -// * Test TFIDF encoding for characters using lamda function, using rawcount as -// * tf type and smoothidf as false. -// */ -// BOOST_AUTO_TEST_CASE(RawcountTfIdfEncodingIndividualCharactersTest) -// { -// vector input = { -// "GACCA", -// "ABCABCD", -// "GAB" -// }; +/** + * Test TFIDF encoding for characters using lamda function, using rawcount as + * tf type and smoothidf as false. + */ +BOOST_AUTO_TEST_CASE(RawcountTfIdfEncodingIndividualCharactersTest) +{ + vector input = { + "GACCA", + "ABCABCD", + "GAB" + }; -// arma::mat output; -// TfIdfEncoding encoder(0, false); + arma::mat output; + TfIdfEncoding encoder(0, false); -// // Passing a empty string to encode charactersrawcountsmoothidftrue -// encoder.Encode(input, output, CharExtract()); -// arma::mat target = { -// { 1.4054651081081644, 2, 2.8109302162163288, 0, 0 }, -// { 0, 2, 2.8109302162163288, 2.8109302162163288, 2.0986122886681100 }, -// { 1.4054651081081644, 1, 0, 1.4054651081081644, 0 } -// }; -// CheckMatrices(output, target, 1e-12); -// } + // Passing a empty string to encode charactersrawcountsmoothidftrue + encoder.Encode(input, output, CharExtract()); + arma::mat target = { + { 1.4054651081081644, 2, 2.8109302162163288, 0, 0 }, + { 0, 2, 2.8109302162163288, 2.8109302162163288, 2.0986122886681100 }, + { 1.4054651081081644, 1, 0, 1.4054651081081644, 0 } + }; + CheckMatrices(output, target, 1e-12); +} -// /** -// * Test the one pass modification of the Tf Idf encoding algorithm -// * in case of individual character encoding, using raw count as type, -// * and smoothidf as false. -// */ -// BOOST_AUTO_TEST_CASE(OnePassRawcountEncodingIndividualCharactersTest) -// { -// std::vector input = { -// "GACCA", -// "ABCABCD", -// "GAB" -// }; +/** + * Test the one pass modification of the Tf Idf encoding algorithm + * in case of individual character encoding, using raw count as type, + * and smoothidf as false. + */ +BOOST_AUTO_TEST_CASE(OnePassRawcountEncodingIndividualCharactersTest) +{ + std::vector input = { + "GACCA", + "ABCABCD", + "GAB" + }; -// vector> output; -// TfIdfEncoding encoder(0, false); + vector> output; + TfIdfEncoding encoder(0, false); -// // Passing a empty string to encode characters -// encoder.Encode(input, output, CharExtract()); -// vector> expected = { -// { 1.4054651081081644, 2, 2.8109302162163288, 0, 0 }, -// { 0, 2, 2.8109302162163288, 2.8109302162163288, 2.0986122886681100 }, -// { 1.4054651081081644, 1, 0, 1.4054651081081644, 0 } -// }; -// for (size_t i = 0; i < expected.size(); i++) -// for (size_t j = 0; j < expected[i].size(); j++) -// BOOST_REQUIRE_CLOSE(expected[i][j], output[i][j], 1e-12); -// } + // Passing a empty string to encode characters + encoder.Encode(input, output, CharExtract()); + vector> expected = { + { 1.4054651081081644, 2, 2.8109302162163288, 0, 0 }, + { 0, 2, 2.8109302162163288, 2.8109302162163288, 2.0986122886681100 }, + { 1.4054651081081644, 1, 0, 1.4054651081081644, 0 } + }; + for (size_t i = 0; i < expected.size(); i++) + for (size_t j = 0; j < expected[i].size(); j++) + BOOST_REQUIRE_CLOSE(expected[i][j], output[i][j], 1e-12); +} -// /** -// * Test TFIDF encoding for characters using lamda function, using binary -// * weighting scheme for tf and smoothidf as true. -// */ -// BOOST_AUTO_TEST_CASE(BinarySmoothIdfEncodingIndividualCharactersTest) -// { -// vector input = { -// "GACCA", -// "ABCABCD", -// "GAB" -// }; +/** + * Test TFIDF encoding for characters using lamda function, using binary + * weighting scheme for tf and smoothidf as true. + */ +BOOST_AUTO_TEST_CASE(BinarySmoothIdfEncodingIndividualCharactersTest) +{ + vector input = { + "GACCA", + "ABCABCD", + "GAB" + }; -// arma::mat output; -// TfIdfEncoding encoder(1, true); + arma::mat output; + TfIdfEncoding encoder(1, true); -// // Passing a empty string to encode charactersrawcountsmoothidftrue -// encoder.Encode(input, output, CharExtract()); -// arma::mat target = { -// { 1.2876820724517808, 1, 1.2876820724517808, 0, 0 }, -// { 0, 1, 1.2876820724517808, 1.2876820724517808, 1.6931471805599454 }, -// { 1.2876820724517808, 1, 0, 1.2876820724517808, 0 } -// }; -// CheckMatrices(output, target, 1e-12); -// } + // Passing a empty string to encode charactersrawcountsmoothidftrue + encoder.Encode(input, output, CharExtract()); + arma::mat target = { + { 1.2876820724517808, 1, 1.2876820724517808, 0, 0 }, + { 0, 1, 1.2876820724517808, 1.2876820724517808, 1.6931471805599454 }, + { 1.2876820724517808, 1, 0, 1.2876820724517808, 0 } + }; + CheckMatrices(output, target, 1e-12); +} -// /** -// * Test TFIDF encoding for characters using lamda function, using binary -// * weighting scheme for tf and smoothidf as true. -// */ -// BOOST_AUTO_TEST_CASE(OnePassBnarySmoothIdfEncodingIndividualCharactersTest) -// { -// std::vector input = { -// "GACCA", -// "ABCABCD", -// "GAB" -// }; +/** + * Test TFIDF encoding for characters using lamda function, using binary + * weighting scheme for tf and smoothidf as true. + */ +BOOST_AUTO_TEST_CASE(OnePassBnarySmoothIdfEncodingIndividualCharactersTest) +{ + std::vector input = { + "GACCA", + "ABCABCD", + "GAB" + }; -// vector> output; -// TfIdfEncoding encoder(1, true); + vector> output; + TfIdfEncoding encoder(1, true); -// // Passing a empty string to encode characters -// encoder.Encode(input, output, CharExtract()); -// vector> expected = { -// { 1.2876820724517808, 1, 1.2876820724517808, 0, 0 }, -// { 0, 1, 1.2876820724517808, 1.2876820724517808, 1.6931471805599454 }, -// { 1.2876820724517808, 1, 0, 1.2876820724517808, 0 } -// }; -// for (size_t i = 0; i < expected.size(); i++) -// for (size_t j = 0; j < expected[i].size(); j++) -// BOOST_REQUIRE_CLOSE(expected[i][j], output[i][j], 1e-12); -// } + // Passing a empty string to encode characters + encoder.Encode(input, output, CharExtract()); + vector> expected = { + { 1.2876820724517808, 1, 1.2876820724517808, 0, 0 }, + { 0, 1, 1.2876820724517808, 1.2876820724517808, 1.6931471805599454 }, + { 1.2876820724517808, 1, 0, 1.2876820724517808, 0 } + }; + for (size_t i = 0; i < expected.size(); i++) + for (size_t j = 0; j < expected[i].size(); j++) + BOOST_REQUIRE_CLOSE(expected[i][j], output[i][j], 1e-12); +} -// /** -// * Test TFIDF encoding for characters using lamda function, using binary -// * as weighting scheme and smoothidf as false. -// */ -// BOOST_AUTO_TEST_CASE(BinaryTfIdfEncodingIndividualCharactersTest) -// { -// vector input = { -// "GACCA", -// "ABCABCD", -// "GAB" -// }; +/** + * Test TFIDF encoding for characters using lamda function, using binary + * as weighting scheme and smoothidf as false. + */ +BOOST_AUTO_TEST_CASE(BinaryTfIdfEncodingIndividualCharactersTest) +{ + vector input = { + "GACCA", + "ABCABCD", + "GAB" + }; -// arma::mat output; -// TfIdfEncoding encoder(1, false); + arma::mat output; + TfIdfEncoding encoder(1, false); -// // Passing a empty string to encode charactersrawcountsmoothidftrue -// encoder.Encode(input, output, CharExtract()); -// arma::mat target = { -// { 1.4054651081081644, 1, 1.4054651081081644, 0, 0 }, -// { 0, 1, 1.4054651081081644, 1.4054651081081644, 2.0986122886681100 }, -// { 1.4054651081081644, 1, 0, 1.4054651081081644, 0 } -// }; -// CheckMatrices(output, target, 1e-12); -// } + // Passing a empty string to encode charactersrawcountsmoothidftrue + encoder.Encode(input, output, CharExtract()); + arma::mat target = { + { 1.4054651081081644, 1, 1.4054651081081644, 0, 0 }, + { 0, 1, 1.4054651081081644, 1.4054651081081644, 2.0986122886681100 }, + { 1.4054651081081644, 1, 0, 1.4054651081081644, 0 } + }; + CheckMatrices(output, target, 1e-12); +} -// /** -// * Test TFIDF encoding for characters using lamda function, using sublinear -// * as weighting scheme and smoothidf as true. -// */ -// BOOST_AUTO_TEST_CASE(SublinearSmoothIdfEncodingIndividualCharactersTest) -// { -// vector input = { -// "GACCA", -// "ABCABCD", -// "GAB" -// }; +/** + * Test TFIDF encoding for characters using lamda function, using sublinear + * as weighting scheme and smoothidf as true. + */ +BOOST_AUTO_TEST_CASE(SublinearSmoothIdfEncodingIndividualCharactersTest) +{ + vector input = { + "GACCA", + "ABCABCD", + "GAB" + }; -// arma::mat output; -// TfIdfEncoding encoder(2, true); + arma::mat output; + TfIdfEncoding encoder(2, true); -// // Passing a empty string to encode charactersrawcountsmoothidftrue -// encoder.Encode(input, output, CharExtract()); -// arma::mat target = { -// { 1.2876820724517808, 1.6931471805599454, 2.1802352704293200, 0, 0 }, -// { 0, 1.6931471805599454, 2.1802352704293200, 2.1802352704293200, -// 1.6931471805599454 }, -// { 1.2876820724517808, 1, 0, 1.2876820724517808, 0 } -// }; -// CheckMatrices(output, target, 1e-12); -// } + // Passing a empty string to encode charactersrawcountsmoothidftrue + encoder.Encode(input, output, CharExtract()); + arma::mat target = { + { 1.2876820724517808, 1.6931471805599454, 2.1802352704293200, 0, 0 }, + { 0, 1.6931471805599454, 2.1802352704293200, 2.1802352704293200, + 1.6931471805599454 }, + { 1.2876820724517808, 1, 0, 1.2876820724517808, 0 } + }; + CheckMatrices(output, target, 1e-12); +} -// /** -// * Test TFIDF encoding for characters using lamda function, using sublinear -// * as weighting scheme and smoothidf as false. -// */ -// BOOST_AUTO_TEST_CASE(SublinearTfIdfEncodingIndividualCharactersTest) -// { -// vector input = { -// "GACCA", -// "ABCABCD", -// "GAB" -// }; +/** + * Test TFIDF encoding for characters using lamda function, using sublinear + * as weighting scheme and smoothidf as false. + */ +BOOST_AUTO_TEST_CASE(SublinearTfIdfEncodingIndividualCharactersTest) +{ + vector input = { + "GACCA", + "ABCABCD", + "GAB" + }; -// arma::mat output; -// TfIdfEncoding encoder(2, false); + arma::mat output; + TfIdfEncoding encoder(2, false); -// // Passing a empty string to encode charactersrawcountsmoothidftrue -// encoder.Encode(input, output, CharExtract()); -// arma::mat target = { -// { 1.4054651081081644, 1.6931471805599454, 2.3796592851687173, 0, 0 }, -// { 0, 1.6931471805599454, 2.3796592851687173, 2.3796592851687173, -// 2.0986122886681100 }, -// { 1.4054651081081644, 1, 0, 1.4054651081081644, 0 } -// }; -// CheckMatrices(output, target, 1e-12); -// } + // Passing a empty string to encode charactersrawcountsmoothidftrue + encoder.Encode(input, output, CharExtract()); + arma::mat target = { + { 1.4054651081081644, 1.6931471805599454, 2.3796592851687173, 0, 0 }, + { 0, 1.6931471805599454, 2.3796592851687173, 2.3796592851687173, + 2.0986122886681100 }, + { 1.4054651081081644, 1, 0, 1.4054651081081644, 0 } + }; + CheckMatrices(output, target, 1e-12); +} -// /** -// * Test TFIDF encoding for characters using lamda function, using term -// * Frequency as weighting scheme and smoothidf as true. -// */ -// BOOST_AUTO_TEST_CASE(TermFrequencySmoothIdfEncodingIndividualCharactersTest) -// { -// vector input = { -// "GACCA", -// "ABCABCD", -// "GAB" -// }; +/** + * Test TFIDF encoding for characters using lamda function, using term + * Frequency as weighting scheme and smoothidf as true. + */ +BOOST_AUTO_TEST_CASE(TermFrequencySmoothIdfEncodingIndividualCharactersTest) +{ + vector input = { + "GACCA", + "ABCABCD", + "GAB" + }; -// arma::mat output; -// TfIdfEncoding encoder(3, true); + arma::mat output; + TfIdfEncoding encoder(3, true); -// // Passing a empty string to encode charactersrawcountsmoothidftrue -// encoder.Encode(input, output, CharExtract()); -// arma::mat target = { -// { 0.2575364144903562, 0.4, 0.5150728289807124, 0, 0 }, -// { 0, 0.2857142857142857, 0.3679091635576516, 0.3679091635576516, -// 0.2418781686514208 }, -// { 0.4292273574839269, 0.3333333333333333, 0, 0.4292273574839269, 0 } -// }; -// CheckMatrices(output, target, 1e-12); -// } + // Passing a empty string to encode charactersrawcountsmoothidftrue + encoder.Encode(input, output, CharExtract()); + arma::mat target = { + { 0.2575364144903562, 0.4, 0.5150728289807124, 0, 0 }, + { 0, 0.2857142857142857, 0.3679091635576516, 0.3679091635576516, + 0.2418781686514208 }, + { 0.4292273574839269, 0.3333333333333333, 0, 0.4292273574839269, 0 } + }; + CheckMatrices(output, target, 1e-12); +} -// /** -// * Test TFIDF encoding for characters using lamda function, using Term -// * Frequency as weighting scheme and smoothidf as false. -// */ -// BOOST_AUTO_TEST_CASE(TermFrequencyTfIdfEncodingIndividualCharactersTest) -// { -// vector input = { -// "GACCA", -// "ABCABCD", -// "GAB" -// }; +/** + * Test TFIDF encoding for characters using lamda function, using Term + * Frequency as weighting scheme and smoothidf as false. + */ +BOOST_AUTO_TEST_CASE(TermFrequencyTfIdfEncodingIndividualCharactersTest) +{ + vector input = { + "GACCA", + "ABCABCD", + "GAB" + }; -// arma::mat output; -// TfIdfEncoding encoder(3, false); + arma::mat output; + TfIdfEncoding encoder(3, false); -// // Passing a empty string to encode charactersrawcountsmoothidftrue -// encoder.Encode(input, output, CharExtract()); -// arma::mat target = { -// { 0.2810930216216329, 0.4, 0.5621860432432658, 0, 0 }, -// { 0, 0.2857142857142857, 0.4015614594594755, 0.4015614594594755, -// 0.2998017555240157 }, -// { 0.4684883693693881, 0.3333333333333333, 0, 0.4684883693693881, 0 } -// }; -// CheckMatrices(output, target, 1e-12); -// } + // Passing a empty string to encode charactersrawcountsmoothidftrue + encoder.Encode(input, output, CharExtract()); + arma::mat target = { + { 0.2810930216216329, 0.4, 0.5621860432432658, 0, 0 }, + { 0, 0.2857142857142857, 0.4015614594594755, 0.4015614594594755, + 0.2998017555240157 }, + { 0.4684883693693881, 0.3333333333333333, 0, 0.4684883693693881, 0 } + }; + CheckMatrices(output, target, 1e-12); +} /** * Serialization test for the TF-IDF encoding algorithm with @@ -1013,27 +1013,51 @@ BOOST_AUTO_TEST_CASE(CharExtractTfIdfEncodingSerialization) EncoderType encoder; CharExtract tokenizer; arma::mat output; - vector input = { - "GACCA", - "ABCABCD", - "GAB" - }; - encoder.Encode(input, output, tokenizer); + + encoder.Encode(stringEncodingInput, output, tokenizer); EncoderType xmlEncoder, textEncoder, binaryEncoder; arma::mat xmlOutput, textOutput, binaryOutput; SerializeObjectAll(encoder, xmlEncoder, textEncoder, binaryEncoder); - std::cout<<"HHHHHHHHHHHHHH\n"; + CheckDictionaries(encoder.Dictionary(), xmlEncoder.Dictionary()); CheckDictionaries(encoder.Dictionary(), textEncoder.Dictionary()); CheckDictionaries(encoder.Dictionary(), binaryEncoder.Dictionary()); - std::cout<<"FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFSDDDDDDDDDDDDDDDD\n"; - xmlEncoder.Encode(input, xmlOutput, tokenizer); - std::cout<<"@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@\n"; - textEncoder.Encode(input, textOutput, tokenizer); - binaryEncoder.Encode(input, binaryOutput, tokenizer); + xmlEncoder.Encode(stringEncodingInput, xmlOutput, tokenizer); + textEncoder.Encode(stringEncodingInput, textOutput, tokenizer); + binaryEncoder.Encode(stringEncodingInput, binaryOutput, tokenizer); + + CheckMatrices(output, xmlOutput, textOutput, binaryOutput); +} + +/** + * Serialization test for the TF-IDF encoding algorithm with + * the SplitByAnyOf tokenizer. + */ +BOOST_AUTO_TEST_CASE(SplitByAnyOfTfIdfEncodingSerialization) +{ + using EncoderType = TfIdfEncoding; + + EncoderType encoder; + SplitByAnyOf tokenizer(" "); + arma::mat output; + + encoder.Encode(stringEncodingInput, output, tokenizer); + + EncoderType xmlEncoder, textEncoder, binaryEncoder; + arma::mat xmlOutput, textOutput, binaryOutput; + + SerializeObjectAll(encoder, xmlEncoder, textEncoder, binaryEncoder); + + CheckDictionaries(encoder.Dictionary(), xmlEncoder.Dictionary()); + CheckDictionaries(encoder.Dictionary(), textEncoder.Dictionary()); + CheckDictionaries(encoder.Dictionary(), binaryEncoder.Dictionary()); + + xmlEncoder.Encode(stringEncodingInput, xmlOutput, tokenizer); + textEncoder.Encode(stringEncodingInput, textOutput, tokenizer); + binaryEncoder.Encode(stringEncodingInput, binaryOutput, tokenizer); CheckMatrices(output, xmlOutput, textOutput, binaryOutput); } From 494d7164f21831838dbdf7d5140dda9e835fe958 Mon Sep 17 00:00:00 2001 From: jeffinsam Date: Sat, 10 Aug 2019 23:49:10 +0530 Subject: [PATCH 019/111] correction in test, update documentation, next target update minor documentation --- .../bag_of_words_encoding_policy.hpp | 34 ++-- .../dictionary_encoding_policy.hpp | 3 + .../tf_idf_encoding_policy.hpp | 67 ++++--- src/mlpack/tests/string_encoding_test.cpp | 177 +++++++++--------- 4 files changed, 141 insertions(+), 140 deletions(-) diff --git a/src/mlpack/core/data/string_encoding_policies/bag_of_words_encoding_policy.hpp b/src/mlpack/core/data/string_encoding_policies/bag_of_words_encoding_policy.hpp index e5b253cbf4..01e113b64d 100644 --- a/src/mlpack/core/data/string_encoding_policies/bag_of_words_encoding_policy.hpp +++ b/src/mlpack/core/data/string_encoding_policies/bag_of_words_encoding_policy.hpp @@ -37,6 +37,7 @@ class BagOfWordsEncodingPolicy * @param maxNumTokens The maximum number of tokens in the strings of the input dataset (not used). * @param dictionarySize The size of the dictionary. + * @tparam MatType The type of output matrix. */ template static void InitMatrix(MatType& output, @@ -49,13 +50,14 @@ class BagOfWordsEncodingPolicy /** * The function initializes the output matrix. - * Overloaded function to store result in vector> + * Overloaded function to store result in vector> * * @param output Output matrix to store the encoded results. * @param datasetSize The number of strings in the input dataset. * @param maxNumTokens The maximum number of tokens in the strings of the input dataset. * @param dictionarySize The size of the dictionary (not used). + * @tparam OutputType The type of output vector. */ template static void InitMatrix(std::vector >& output, @@ -74,6 +76,7 @@ class BagOfWordsEncodingPolicy * @param value The encoded token. * @param row The row number at which the encoding is performed. * @param col The row token number at which the encoding is performed. + * @tparam MatType The type of output matrix. */ template static void Encode(MatType& output, @@ -81,19 +84,21 @@ class BagOfWordsEncodingPolicy size_t row, size_t /*col*/) { - // Important since Mapping starts from 1 whereas allowed column value is 0. - output(row, value-1) = 1; + // Important since Mapping of words,Dcitionary Encoding starts from 1, + // whereas allowed column value is 0. + output(row, value - 1) = 1; } /** - * The function performs the bag of words encoding algorithm i.e. it writess + * The function performs the bag of words encoding algorithm i.e. it writes * the encoded token to the ouput. - * Overload function to accepted vector> as output type. + * Overload function to accepted vector> as output type. * * @param output Output matrix to store the encoded results. * @param value The encoded token. * @param row The row number at which the encoding is performed. * @param col The row token number at which the encoding is performed. + * @tparam OutputType The type of output vector. */ template static void Encode(std::vector >& output, @@ -101,8 +106,9 @@ class BagOfWordsEncodingPolicy size_t row, size_t /*col*/) { - // Important since Mapping starts from 1 whereas allowed column value is 0. - output[row][value-1] = 1; + // Important since Mapping of words,Dcitionary Encoding starts from 1, + // whereas allowed column value is 0. + output[row][value - 1] = 1; } /** @@ -126,20 +132,6 @@ class BagOfWordsEncodingPolicy size_t /*value*/) { } }; -/** - * The specialization provides some information about the about the bag of - * words encoding policy. - */ -template<> -struct StringEncodingPolicyTraits -{ - /** - * Indicates if the policy is able to encode the token at once without - * any information about other tokens as well as the total tokens count. - */ - static const bool onePassEncoding = false; -}; - template using BagOfWordsEncoding = StringEncoding>; diff --git a/src/mlpack/core/data/string_encoding_policies/dictionary_encoding_policy.hpp b/src/mlpack/core/data/string_encoding_policies/dictionary_encoding_policy.hpp index 9c972fd856..d3fd97efc7 100644 --- a/src/mlpack/core/data/string_encoding_policies/dictionary_encoding_policy.hpp +++ b/src/mlpack/core/data/string_encoding_policies/dictionary_encoding_policy.hpp @@ -38,6 +38,7 @@ class DictionaryEncodingPolicy * @param maxNumTokens The maximum number of tokens in the strings of the input dataset. * @param dictionarySize The size of the dictionary (not used). + * @tparam MatType The type of output matrix. */ template static void InitMatrix(MatType& output, @@ -56,6 +57,7 @@ class DictionaryEncodingPolicy * @param value The encoded token. * @param row The row number at which the encoding is performed. * @param col The row token number at which the encoding is performed. + * @tparam MatType The type of output matrix. */ template static void Encode(MatType& output, @@ -73,6 +75,7 @@ class DictionaryEncodingPolicy * * @param output Output vector to store the encoded results. * @param value The encoded token. + * @tparam OutputType The type of output vector. */ template static void Encode(std::vector& output, size_t value) diff --git a/src/mlpack/core/data/string_encoding_policies/tf_idf_encoding_policy.hpp b/src/mlpack/core/data/string_encoding_policies/tf_idf_encoding_policy.hpp index 88bbdbff01..86719066e4 100644 --- a/src/mlpack/core/data/string_encoding_policies/tf_idf_encoding_policy.hpp +++ b/src/mlpack/core/data/string_encoding_policies/tf_idf_encoding_policy.hpp @@ -37,13 +37,13 @@ class TfIdfEncodingPolicy { public: /* - * Enum Class used to identify the type of tf encoding + * Enum class used to identify the type of tf encoding * * Follwing are the defination of the types - * binary : binary weighting scheme (0,1) - * rawCount : raw count weighting scheme (count of token for every row) - * termFrequency : term frequency weighting scheme (count / length(row)) - * subinerTf : logarthimic weighting scheme (log(tf) + 1) + * BINARY : binary weighting scheme (0,1) + * RAW_COUNT : raw count weighting scheme (count of token for every row) + * TERM_FREQUENCY : term frequency weighting scheme (count / length(row)) + * SUBLINEAR_TF : logarthimic weighting scheme (log(tf) + 1) * */ enum class TfTypes @@ -54,7 +54,8 @@ class TfIdfEncodingPolicy TERM_FREQUENCY, }; - TfIdfEncodingPolicy(size_t tfType = 0, bool smoothIdf = true) : + TfIdfEncodingPolicy(TfTypes tfType = TfTypes::RAW_COUNT, + bool smoothIdf = true) : tfType(tfType), smoothIdf(smoothIdf) { @@ -67,9 +68,10 @@ class TfIdfEncodingPolicy * @param maxNumTokens The maximum number of tokens in the strings of the input dataset. * @param dictionarySize The size of the dictionary (not used). + * @tparam MatType The type of output matrix. */ template - void InitMatrix(MatType& output, + static void InitMatrix(MatType& output, size_t datasetSize, size_t /*maxNumTokens*/, size_t dictionarySize) @@ -79,13 +81,14 @@ class TfIdfEncodingPolicy /** * The function initializes the output matrix. - * Overloaded function to store result in vector> + * Overloaded function to store result in vector> * * @param output Output matrix to store the encoded results. * @param datasetSize The number of strings in the input dataset. * @param maxNumTokens The maximum number of tokens in the strings of the input dataset. * @param dictionarySize The size of the dictionary (not used). + * @tparam OutputType The type of output vector. */ template static void InitMatrix(std::vector >& output, @@ -104,6 +107,7 @@ class TfIdfEncodingPolicy * @param value The encoded token. * @param row The row number at which the encoding is performed. * @param col The row token number at which the encoding is performed. + * @tparam MatType The type of output matrix. */ template void Encode(MatType& output, @@ -111,18 +115,19 @@ class TfIdfEncodingPolicy size_t row, size_t /*col*/) { - // Important since Mapping starts from 1 whereas allowed column value is 0. + // Important since Mapping of words,Dcitionary Encoding starts from 1, + // whereas allowed column value is 0. double idf, tf; if (smoothIdf) idf = std::log((output.n_rows + 1) / (1 + idfdict[value - 1])) + 1; else idf = std::log(output.n_rows / idfdict[value - 1]) + 1; - if (tfType == static_cast(TfTypes::TERM_FREQUENCY)) + if (tfType == TfTypes::TERM_FREQUENCY) tf = tokenCount[row][value - 1] / row_size[row]; - else if (tfType == static_cast(TfTypes::SUBLINEAR_TF)) + else if (tfType == TfTypes::SUBLINEAR_TF) tf = std::log(tokenCount[row][value - 1]) + 1; - else if (tfType == static_cast(TfTypes::BINARY)) + else if (tfType == TfTypes::BINARY) tf = tokenCount[row][value - 1] > 0 ? 1 : 0; else tf = tokenCount[row][value - 1]; @@ -133,12 +138,14 @@ class TfIdfEncodingPolicy /** * The function performs the TfIdf encoding algorithm i.e. it writes * the encoded token to the ouput. - * Overload function to accepted vector> as output type. + * Overload function to accepted vector> as output type. * - * @param output Output matrix to store the encoded results (sp_mat or mat). + * @param output Output matrix to store the encoded results. * @param value The encoded token. * @param row The row number at which the encoding is performed. * @param col The row token number at which the encoding is performed. + * @tparam OutputType The type of output vector. + * @tparam OutputType The type of output vector. */ template void Encode(std::vector >& output, @@ -146,18 +153,19 @@ class TfIdfEncodingPolicy size_t row, size_t /*col*/) { - // Important since Mapping starts from 1 whereas allowed column value is 0. + // Important since Mapping of words,Dcitionary Encoding starts from 1, + // whereas allowed column value is 0. double idf, tf; if (smoothIdf) idf = std::log((output.size() + 1) / (1 + idfdict[value - 1])) + 1; else idf = std::log(output.size() / idfdict[value - 1]) + 1; - if (tfType == static_cast(TfTypes::TERM_FREQUENCY)) + if (tfType == TfTypes::TERM_FREQUENCY) tf = tokenCount[row][value - 1] / row_size[row]; - else if (tfType == static_cast(TfTypes::SUBLINEAR_TF)) + else if (tfType == TfTypes::SUBLINEAR_TF) tf = std::log(tokenCount[row][value - 1]) + 1; - else if (tfType == static_cast(TfTypes::BINARY)) + else if (tfType == TfTypes::BINARY) tf = tokenCount[row][value - 1] > 0 ? 1 : 0; else tf = tokenCount[row][value - 1]; @@ -190,11 +198,14 @@ class TfIdfEncodingPolicy if (row >= tokenCount.size()) { row_size.push_back(0); - tokenCount.push_back(std::unordered_map()); + tokenCount.emplace_back(); + ///tokenCount.push_back(std::unordered_map()); } tokenCount.back()[value-1]++; + if (tokenCount.back()[value - 1] == 1) idfdict[value - 1]++; + row_size.back()++; } private: @@ -207,26 +218,12 @@ class TfIdfEncodingPolicy // smoothIdf variable to indicate smoothining. bool smoothIdf; // Type of Term Frequency to use. - size_t tfType; -}; - -/** - * The specialization provides some information about the dictionary encoding - * policy. - */ -template<> -struct StringEncodingPolicyTraits -{ - /** - * Indicates if the policy is able to encode the token at once without - * any information about other tokens as well as the total tokens count. - */ - static const bool onePassEncoding = false; + TfTypes tfType; }; template using TfIdfEncoding = StringEncoding>; + StringEncodingDictionary>; } // namespace data } // namespace mlpack diff --git a/src/mlpack/tests/string_encoding_test.cpp b/src/mlpack/tests/string_encoding_test.cpp index 220eed1cc9..184abc67fb 100644 --- a/src/mlpack/tests/string_encoding_test.cpp +++ b/src/mlpack/tests/string_encoding_test.cpp @@ -407,7 +407,7 @@ BOOST_AUTO_TEST_CASE(BagOfWordsEncodingTest) arma::mat output; BagOfWordsEncoding encoder; - SplitByAnyOf tokenizer(" "); + SplitByAnyOf tokenizer(" ,."); encoder.Encode(stringEncodingInput, output, tokenizer); const DictionaryType& dictionary = encoder.Dictionary(); @@ -422,26 +422,26 @@ BOOST_AUTO_TEST_CASE(BagOfWordsEncodingTest) } arma::mat expected = { { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, { 0, 1, 0, 0, 0, 1, 0, 0, 1, 1, 0, 0, 0, 1, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, - { 1, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 } + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, + { 1, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 } }; CheckMatrices(output, expected); } /** -* Test the one pass modification of the Bag of Words encoding algorithm. +* Bag of Words encoding algorithm output saved in a vector. */ -BOOST_AUTO_TEST_CASE(OnePassBagOfWordsEncodingTest) +BOOST_AUTO_TEST_CASE(VectorBagOfWordsEncodingTest) { using DictionaryType = StringEncodingDictionary; vector> output; BagOfWordsEncoding encoder( (BagOfWordsEncodingPolicy())); - SplitByAnyOf tokenizer(" "); + SplitByAnyOf tokenizer(" ,."); encoder.Encode(stringEncodingInput, output, tokenizer); @@ -458,18 +458,18 @@ BOOST_AUTO_TEST_CASE(OnePassBagOfWordsEncodingTest) vector> expected = { { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, { 0, 1, 0, 0, 0, 1, 0, 0, 1, 1, 0, 0, 0, 1, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, - { 1, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 } + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, + { 1, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 } }; BOOST_REQUIRE(output == expected); } /** -* Test Bag of Words encoding for characters using lamda function. +* Test Bag of Words encoding for characters. */ BOOST_AUTO_TEST_CASE(BagOfWordsEncodingIndividualCharactersTest) { @@ -494,8 +494,8 @@ BOOST_AUTO_TEST_CASE(BagOfWordsEncodingIndividualCharactersTest) } /** - * Test the one pass modification of the Bag of Words encoding algorithm - * in case of individual character encoding. + * Test the Bag of Words encoding algorithm in case of individual + * character encoding, storing resulting in vector>. */ BOOST_AUTO_TEST_CASE(OnePassBagOfWordsEncodingIndividualCharactersTest) { @@ -530,10 +530,9 @@ BOOST_AUTO_TEST_CASE(RawCountSmoothIdfEncodingTest) arma::mat output; TfIdfEncoding encoder; - SplitByAnyOf tokenizer(" "); + SplitByAnyOf tokenizer(" ,."); encoder.Encode(stringEncodingInput, output, tokenizer); - const DictionaryType& dictionary = encoder.Dictionary(); // Checking that everything is mapped to different numbers @@ -548,37 +547,37 @@ BOOST_AUTO_TEST_CASE(RawCountSmoothIdfEncodingTest) { 1.28768207245178, 1.28768207245178, 1.69314718055995, 1.69314718055995, 1.69314718055995, 1, 1.69314718055995, 1.28768207245178, 1.28768207245178, 1.28768207245178, 1.69314718055995, 1.69314718055995, - 1.69314718055995, 1, 1.69314718055995, 1.69314718055995, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0 }, + 1.28768207245178, 1, 1.69314718055995, 1.69314718055995, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0 }, { 0, 1.28768207245178, 0, 0, 0, 2, 0, 0, 3.86304621735534, 3.86304621735534, 0, 0, 0, 3, 0, 0, 1.69314718055995, 1.69314718055995, 1.69314718055995, 5.07944154167984, 1.69314718055995, 1.69314718055995, 1.69314718055995, 1.69314718055995, 1.69314718055995, 1.69314718055995, 1.69314718055995, 1.69314718055995, 1.69314718055995, 1.69314718055995, 1.69314718055995, 1.69314718055995, 1.69314718055995, 1.69314718055995, - 1.69314718055995, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, - { 1.28768207245178, 0, 0, 0, 0, 1, 0, 1.28768207245178, 0, 0, 0, 0, 0, 1, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 1.69314718055995, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, + { 1.28768207245178, 0, 0, 0, 0, 1, 0, 1.28768207245178, 0, 0, 0, 0, + 1.28768207245178, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 1.69314718055995, 1.69314718055995, 1.69314718055995, 1.69314718055995, 1.69314718055995, 1.69314718055995, 1.69314718055995, - 1.69314718055995, 1.69314718055995, 1.69314718055995, 1.69314718055995, - 1.69314718055995, 1.69314718055995, 1.69314718055995 } + 1.69314718055995, 1.69314718055995, 1.69314718055995 } }; CheckMatrices(output, expected, 1e-12); } /** - * Test the one pass modification of the TfIdf encoding algorithm, using rawcount - * as type of tf and smoothIdf as true. + * Test the TfIdf encoding algorithm, using rawcount as type of tf and smoothIdf + * as true, storing the result in a vector>. */ -BOOST_AUTO_TEST_CASE(OnePassRawCountSmoothIdfEncodingTest) +BOOST_AUTO_TEST_CASE(VectorRawCountSmoothIdfEncodingTest) { using DictionaryType = StringEncodingDictionary; vector> output; TfIdfEncoding encoder( (TfIdfEncodingPolicy())); - SplitByAnyOf tokenizer(" "); + SplitByAnyOf tokenizer(" ,."); encoder.Encode(stringEncodingInput, output, tokenizer); @@ -597,21 +596,21 @@ BOOST_AUTO_TEST_CASE(OnePassRawCountSmoothIdfEncodingTest) { 1.28768207245178, 1.28768207245178, 1.69314718055995, 1.69314718055995, 1.69314718055995, 1, 1.69314718055995, 1.28768207245178, 1.28768207245178, 1.28768207245178, 1.69314718055995, 1.69314718055995, - 1.69314718055995, 1, 1.69314718055995, 1.69314718055995, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0 }, + 1.28768207245178, 1, 1.69314718055995, 1.69314718055995, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0 }, { 0, 1.28768207245178, 0, 0, 0, 2, 0, 0, 3.86304621735534, 3.86304621735534, 0, 0, 0, 3, 0, 0, 1.69314718055995, 1.69314718055995, 1.69314718055995, 5.07944154167984, 1.69314718055995, 1.69314718055995, 1.69314718055995, 1.69314718055995, 1.69314718055995, 1.69314718055995, 1.69314718055995, 1.69314718055995, 1.69314718055995, 1.69314718055995, 1.69314718055995, 1.69314718055995, 1.69314718055995, 1.69314718055995, - 1.69314718055995, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, - { 1.28768207245178, 0, 0, 0, 0, 1, 0, 1.28768207245178, 0, 0, 0, 0, 0, 1, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 1.69314718055995, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, + { 1.28768207245178, 0, 0, 0, 0, 1, 0, 1.28768207245178, 0, 0, 0, 0, + 1.28768207245178, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 1.69314718055995, 1.69314718055995, 1.69314718055995, 1.69314718055995, 1.69314718055995, 1.69314718055995, 1.69314718055995, - 1.69314718055995, 1.69314718055995, 1.69314718055995, 1.69314718055995, - 1.69314718055995, 1.69314718055995, 1.69314718055995 } + 1.69314718055995, 1.69314718055995, 1.69314718055995 } }; for (size_t i = 0; i < expected.size(); i++) for (size_t j = 0; j < expected[i].size(); j++) @@ -619,8 +618,8 @@ BOOST_AUTO_TEST_CASE(OnePassRawCountSmoothIdfEncodingTest) } /** - * Test TFIDF encoding for characters using lamda function, using rawcount as tf - * type and smoothidf as true. + * Test TFIDF encoding for characters, using rawcount as tf type and + * smoothidf as true. */ BOOST_AUTO_TEST_CASE(RawCountSmoothIdfEncodingIndividualCharactersTest) { @@ -644,10 +643,10 @@ BOOST_AUTO_TEST_CASE(RawCountSmoothIdfEncodingIndividualCharactersTest) } /** - * Test the one pass modification of the Tf-Idf encoding algorithm + * Test the Tf-Idf encoding algorithm to store result in vector * in case of individual character encoding using default values. */ -BOOST_AUTO_TEST_CASE(OnePassRawCountSmoothIdfEncodingIndividualCharactersTest) +BOOST_AUTO_TEST_CASE(VectorRawCountSmoothIdfEncodingIndividualCharactersTest) { std::vector input = { "GACCA", @@ -679,8 +678,8 @@ BOOST_AUTO_TEST_CASE(TfIdfRawCountEncodingTest) arma::mat output; TfIdfEncoding encoder( - (TfIdfEncodingPolicy(0, false))); - SplitByAnyOf tokenizer(" "); + (TfIdfEncodingPolicy(TfIdfEncodingPolicy::TfTypes::RAW_COUNT, false))); + SplitByAnyOf tokenizer(" ,."); encoder.Encode(stringEncodingInput, output, tokenizer); @@ -699,36 +698,37 @@ BOOST_AUTO_TEST_CASE(TfIdfRawCountEncodingTest) { 1.40546510810816, 1.40546510810816, 2.09861228866811, 2.09861228866811, 2.09861228866811, 1, 2.09861228866811, 1.40546510810816, 1.40546510810816, 1.40546510810816, 2.09861228866811, 2.09861228866811, - 2.09861228866811, 1, 2.09861228866811, 2.09861228866811, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0 }, + 1.40546510810816, 1, 2.09861228866811, 2.09861228866811, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0 }, { 0, 1.40546510810816, 0, 0, 0, 2, 0, 0, 4.21639532432449, 4.21639532432449, 0, 0, 0, 3, 0, 0, 2.09861228866811, 2.09861228866811, 2.09861228866811, 6.29583686600433, 2.09861228866811, 2.09861228866811, 2.09861228866811, 2.09861228866811, 2.09861228866811, 2.09861228866811, 2.09861228866811, 2.09861228866811, 2.09861228866811, 2.09861228866811, 2.09861228866811, 2.09861228866811, 2.09861228866811, 2.09861228866811, - 2.09861228866811, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, - { 1.40546510810816, 0, 0, 0, 0, 1, 0, 1.40546510810816, 0, 0, 0, 0, 0, 1, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 2.09861228866811, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, + { 1.40546510810816, 0, 0, 0, 0, 1, 0, 1.40546510810816, 0, 0, 0, 0, + 1.40546510810816, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 2.09861228866811, 2.09861228866811, 2.09861228866811, 2.09861228866811, 2.09861228866811, 2.09861228866811, 2.09861228866811, - 2.09861228866811, 2.09861228866811, 2.09861228866811, 2.09861228866811, - 2.09861228866811, 2.09861228866811, 2.09861228866811 } + 2.09861228866811, 2.09861228866811, 2.09861228866811 } }; CheckMatrices(output, expected, 1e-12); } /** - * Test the one pass modification of the TfIdf encoding algorithm, with rawcount + * Test the TfIdf encoding algorithm for output type as vector, with rawcount * as type, but with smoothidf as false. */ -BOOST_AUTO_TEST_CASE(OnePassTfIdfRawCountEncodingTest) +BOOST_AUTO_TEST_CASE(VectorTfIdfRawCountEncodingTest) { using DictionaryType = StringEncodingDictionary; vector> output; - TfIdfEncoding encoder(0, false); - SplitByAnyOf tokenizer(" "); + TfIdfEncoding + encoder(TfIdfEncodingPolicy::TfTypes::RAW_COUNT, false); + SplitByAnyOf tokenizer(" ,."); encoder.Encode(stringEncodingInput, output, tokenizer); @@ -747,21 +747,21 @@ BOOST_AUTO_TEST_CASE(OnePassTfIdfRawCountEncodingTest) { 1.40546510810816, 1.40546510810816, 2.09861228866811, 2.09861228866811, 2.09861228866811, 1, 2.09861228866811, 1.40546510810816, 1.40546510810816, 1.40546510810816, 2.09861228866811, 2.09861228866811, - 2.09861228866811, 1, 2.09861228866811, 2.09861228866811, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0 }, + 1.40546510810816, 1, 2.09861228866811, 2.09861228866811, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0 }, { 0, 1.40546510810816, 0, 0, 0, 2, 0, 0, 4.21639532432449, 4.21639532432449, 0, 0, 0, 3, 0, 0, 2.09861228866811, 2.09861228866811, 2.09861228866811, 6.29583686600433, 2.09861228866811, 2.09861228866811, 2.09861228866811, 2.09861228866811, 2.09861228866811, 2.09861228866811, 2.09861228866811, 2.09861228866811, 2.09861228866811, 2.09861228866811, 2.09861228866811, 2.09861228866811, 2.09861228866811, 2.09861228866811, - 2.09861228866811, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, - { 1.40546510810816, 0, 0, 0, 0, 1, 0, 1.40546510810816, 0, 0, 0, 0, 0, 1, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 2.09861228866811, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, + { 1.40546510810816, 0, 0, 0, 0, 1, 0, 1.40546510810816, 0, 0, 0, 0, + 1.40546510810816, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 2.09861228866811, 2.09861228866811, 2.09861228866811, 2.09861228866811, 2.09861228866811, 2.09861228866811, 2.09861228866811, - 2.09861228866811, 2.09861228866811, 2.09861228866811, 2.09861228866811, - 2.09861228866811, 2.09861228866811, 2.09861228866811 } + 2.09861228866811, 2.09861228866811, 2.09861228866811 } }; for (size_t i = 0; i < expected.size(); i++) for (size_t j = 0; j < expected[i].size(); j++) @@ -769,7 +769,7 @@ BOOST_AUTO_TEST_CASE(OnePassTfIdfRawCountEncodingTest) } /** - * Test TFIDF encoding for characters using lamda function, using rawcount as + * Test TFIDF encoding for characters, using rawcount as * tf type and smoothidf as false. */ BOOST_AUTO_TEST_CASE(RawcountTfIdfEncodingIndividualCharactersTest) @@ -781,7 +781,8 @@ BOOST_AUTO_TEST_CASE(RawcountTfIdfEncodingIndividualCharactersTest) }; arma::mat output; - TfIdfEncoding encoder(0, false); + TfIdfEncoding + encoder(TfIdfEncodingPolicy::TfTypes::RAW_COUNT, false); // Passing a empty string to encode charactersrawcountsmoothidftrue encoder.Encode(input, output, CharExtract()); @@ -794,11 +795,11 @@ BOOST_AUTO_TEST_CASE(RawcountTfIdfEncodingIndividualCharactersTest) } /** - * Test the one pass modification of the Tf Idf encoding algorithm + * Test the Tf Idf encoding algorithm to store result in vector * in case of individual character encoding, using raw count as type, * and smoothidf as false. */ -BOOST_AUTO_TEST_CASE(OnePassRawcountEncodingIndividualCharactersTest) +BOOST_AUTO_TEST_CASE(VectorRawcountEncodingIndividualCharactersTest) { std::vector input = { "GACCA", @@ -807,7 +808,8 @@ BOOST_AUTO_TEST_CASE(OnePassRawcountEncodingIndividualCharactersTest) }; vector> output; - TfIdfEncoding encoder(0, false); + TfIdfEncoding + encoder(TfIdfEncodingPolicy::TfTypes::RAW_COUNT, false); // Passing a empty string to encode characters encoder.Encode(input, output, CharExtract()); @@ -822,8 +824,8 @@ BOOST_AUTO_TEST_CASE(OnePassRawcountEncodingIndividualCharactersTest) } /** - * Test TFIDF encoding for characters using lamda function, using binary - * weighting scheme for tf and smoothidf as true. + * Test TFIDF encoding for characters, using binary weighting scheme + * for tf and smoothidf as true. */ BOOST_AUTO_TEST_CASE(BinarySmoothIdfEncodingIndividualCharactersTest) { @@ -834,7 +836,8 @@ BOOST_AUTO_TEST_CASE(BinarySmoothIdfEncodingIndividualCharactersTest) }; arma::mat output; - TfIdfEncoding encoder(1, true); + TfIdfEncoding + encoder(TfIdfEncodingPolicy::TfTypes::BINARY, true); // Passing a empty string to encode charactersrawcountsmoothidftrue encoder.Encode(input, output, CharExtract()); @@ -847,10 +850,10 @@ BOOST_AUTO_TEST_CASE(BinarySmoothIdfEncodingIndividualCharactersTest) } /** - * Test TFIDF encoding for characters using lamda function, using binary + * Test TFIDF encoding for characters to store results in vector, using binary * weighting scheme for tf and smoothidf as true. */ -BOOST_AUTO_TEST_CASE(OnePassBnarySmoothIdfEncodingIndividualCharactersTest) +BOOST_AUTO_TEST_CASE(VectorBnarySmoothIdfEncodingIndividualCharactersTest) { std::vector input = { "GACCA", @@ -859,7 +862,8 @@ BOOST_AUTO_TEST_CASE(OnePassBnarySmoothIdfEncodingIndividualCharactersTest) }; vector> output; - TfIdfEncoding encoder(1, true); + TfIdfEncoding + encoder(TfIdfEncodingPolicy::TfTypes::BINARY, true); // Passing a empty string to encode characters encoder.Encode(input, output, CharExtract()); @@ -874,7 +878,7 @@ BOOST_AUTO_TEST_CASE(OnePassBnarySmoothIdfEncodingIndividualCharactersTest) } /** - * Test TFIDF encoding for characters using lamda function, using binary + * Test TFIDF encoding for characters, using binary * as weighting scheme and smoothidf as false. */ BOOST_AUTO_TEST_CASE(BinaryTfIdfEncodingIndividualCharactersTest) @@ -886,7 +890,8 @@ BOOST_AUTO_TEST_CASE(BinaryTfIdfEncodingIndividualCharactersTest) }; arma::mat output; - TfIdfEncoding encoder(1, false); + TfIdfEncoding + encoder(TfIdfEncodingPolicy::TfTypes::BINARY, false); // Passing a empty string to encode charactersrawcountsmoothidftrue encoder.Encode(input, output, CharExtract()); @@ -899,7 +904,7 @@ BOOST_AUTO_TEST_CASE(BinaryTfIdfEncodingIndividualCharactersTest) } /** - * Test TFIDF encoding for characters using lamda function, using sublinear + * Test TFIDF encoding for characters, using sublinear * as weighting scheme and smoothidf as true. */ BOOST_AUTO_TEST_CASE(SublinearSmoothIdfEncodingIndividualCharactersTest) @@ -911,7 +916,8 @@ BOOST_AUTO_TEST_CASE(SublinearSmoothIdfEncodingIndividualCharactersTest) }; arma::mat output; - TfIdfEncoding encoder(2, true); + TfIdfEncoding + encoder(TfIdfEncodingPolicy::TfTypes::SUBLINEAR_TF, true); // Passing a empty string to encode charactersrawcountsmoothidftrue encoder.Encode(input, output, CharExtract()); @@ -925,7 +931,7 @@ BOOST_AUTO_TEST_CASE(SublinearSmoothIdfEncodingIndividualCharactersTest) } /** - * Test TFIDF encoding for characters using lamda function, using sublinear + * Test TFIDF encoding for characters, using sublinear * as weighting scheme and smoothidf as false. */ BOOST_AUTO_TEST_CASE(SublinearTfIdfEncodingIndividualCharactersTest) @@ -937,7 +943,8 @@ BOOST_AUTO_TEST_CASE(SublinearTfIdfEncodingIndividualCharactersTest) }; arma::mat output; - TfIdfEncoding encoder(2, false); + TfIdfEncoding + encoder(TfIdfEncodingPolicy::TfTypes::SUBLINEAR_TF, false); // Passing a empty string to encode charactersrawcountsmoothidftrue encoder.Encode(input, output, CharExtract()); @@ -951,7 +958,7 @@ BOOST_AUTO_TEST_CASE(SublinearTfIdfEncodingIndividualCharactersTest) } /** - * Test TFIDF encoding for characters using lamda function, using term + * Test TFIDF encoding for characters, using term * Frequency as weighting scheme and smoothidf as true. */ BOOST_AUTO_TEST_CASE(TermFrequencySmoothIdfEncodingIndividualCharactersTest) @@ -963,7 +970,8 @@ BOOST_AUTO_TEST_CASE(TermFrequencySmoothIdfEncodingIndividualCharactersTest) }; arma::mat output; - TfIdfEncoding encoder(3, true); + TfIdfEncoding + encoder(TfIdfEncodingPolicy::TfTypes::TERM_FREQUENCY, true); // Passing a empty string to encode charactersrawcountsmoothidftrue encoder.Encode(input, output, CharExtract()); @@ -977,7 +985,7 @@ BOOST_AUTO_TEST_CASE(TermFrequencySmoothIdfEncodingIndividualCharactersTest) } /** - * Test TFIDF encoding for characters using lamda function, using Term + * Test TFIDF encoding for characters, using Term * Frequency as weighting scheme and smoothidf as false. */ BOOST_AUTO_TEST_CASE(TermFrequencyTfIdfEncodingIndividualCharactersTest) @@ -989,7 +997,8 @@ BOOST_AUTO_TEST_CASE(TermFrequencyTfIdfEncodingIndividualCharactersTest) }; arma::mat output; - TfIdfEncoding encoder(3, false); + TfIdfEncoding + encoder(TfIdfEncodingPolicy::TfTypes::TERM_FREQUENCY, false); // Passing a empty string to encode charactersrawcountsmoothidftrue encoder.Encode(input, output, CharExtract()); From cde8ad96e1c52f0ac3bccc907bc6e3e0703c08ee Mon Sep 17 00:00:00 2001 From: jeffinsam Date: Sun, 11 Aug 2019 02:00:39 +0530 Subject: [PATCH 020/111] Resolve style comments and other minor comments --- .../tf_idf_encoding_policy.hpp | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/src/mlpack/core/data/string_encoding_policies/tf_idf_encoding_policy.hpp b/src/mlpack/core/data/string_encoding_policies/tf_idf_encoding_policy.hpp index 86719066e4..9e8a4a290a 100644 --- a/src/mlpack/core/data/string_encoding_policies/tf_idf_encoding_policy.hpp +++ b/src/mlpack/core/data/string_encoding_policies/tf_idf_encoding_policy.hpp @@ -23,11 +23,10 @@ namespace data { * Tf means term-frequency while tf-idf means term-frequency times inverse * document-frequency. This is a common term weighting scheme in information * retrieval, that has also found good use in document classification. - * The goal of using tf-idf instead of the raw frequencies of occurrence of a - * token in a given document is to scale down the impact of tokens that occur - * very frequently in a given corpus and that are hence empirically less - * informative than features that occur in a small fraction of the training - * corpus. + * The goal of using tf-idf is to scale down the impact of tokens that occur + * very frequently in a given corpus while using the raw frequencies of a token + * and that are hence empirically less informative than features that occur in + * a small fraction of the training corpus. * TfIdfEncodingPolicy is used as a helper class for StringEncoding. * The encoder assigns a tf-idf number to each unique token and treat * the dataset as categorical. The tokens are labeled in the order of their @@ -199,9 +198,8 @@ class TfIdfEncodingPolicy { row_size.push_back(0); tokenCount.emplace_back(); - ///tokenCount.push_back(std::unordered_map()); } - tokenCount.back()[value-1]++; + tokenCount.back()[value - 1]++; if (tokenCount.back()[value - 1] == 1) idfdict[value - 1]++; From 96cc5e26068041054f6fe6af37b94fedf6bc75f3 Mon Sep 17 00:00:00 2001 From: jeffinsam Date: Sun, 11 Aug 2019 02:42:16 +0530 Subject: [PATCH 021/111] merge conflict removed --- src/mlpack/core/data/string_encoding_impl.hpp | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/mlpack/core/data/string_encoding_impl.hpp b/src/mlpack/core/data/string_encoding_impl.hpp index 96c81641ca..86a14a107c 100644 --- a/src/mlpack/core/data/string_encoding_impl.hpp +++ b/src/mlpack/core/data/string_encoding_impl.hpp @@ -112,12 +112,8 @@ EncodeHelper(const std::vector& input, { size_t numColumns = 0; -<<<<<<< HEAD - for (size_t i = 0; i < input.size(); i++) -======= // The first pass adds the extracted tokens to the dictionary. - for (const std::string& line : input) ->>>>>>> lozhnikov/string-encoding-fixes + for (size_t i = 0; i < input.size(); i++) { boost::string_view strView(input[i]); auto token = tokenizer(strView); From 0c2323485fb6c1eb9d625232b8ce62a031176a1f Mon Sep 17 00:00:00 2001 From: jeffinsam Date: Mon, 12 Aug 2019 13:38:36 +0530 Subject: [PATCH 022/111] update documentation --- .../string_encoding_policies/CMakeLists.txt | 4 +-- .../tf_idf_encoding_policy.hpp | 25 +++++++++++++------ src/mlpack/tests/string_encoding_test.cpp | 4 +-- 3 files changed, 22 insertions(+), 11 deletions(-) diff --git a/src/mlpack/core/data/string_encoding_policies/CMakeLists.txt b/src/mlpack/core/data/string_encoding_policies/CMakeLists.txt index b78b409453..9f570e08d7 100644 --- a/src/mlpack/core/data/string_encoding_policies/CMakeLists.txt +++ b/src/mlpack/core/data/string_encoding_policies/CMakeLists.txt @@ -1,10 +1,10 @@ # Define the files that we need to compile. # Anything not in this list will not be compiled into mlpack. set(SOURCES - dictionary_encoding_policy.hpp bag_of_words_encoding_policy.hpp - tf_idf_encoding_policy.hpp + dictionary_encoding_policy.hpp policy_traits.hpp + tf_idf_encoding_policy.hpp ) # add directory name to sources diff --git a/src/mlpack/core/data/string_encoding_policies/tf_idf_encoding_policy.hpp b/src/mlpack/core/data/string_encoding_policies/tf_idf_encoding_policy.hpp index 9e8a4a290a..d1724ccd3d 100644 --- a/src/mlpack/core/data/string_encoding_policies/tf_idf_encoding_policy.hpp +++ b/src/mlpack/core/data/string_encoding_policies/tf_idf_encoding_policy.hpp @@ -20,13 +20,12 @@ namespace data { /** * Definition of the TfIdfEncodingPolicy class. * - * Tf means term-frequency while tf-idf means term-frequency times inverse - * document-frequency. This is a common term weighting scheme in information - * retrieval, that has also found good use in document classification. + * Tf-idf is weighing scheme that stands for term-frequency multiplied by + * inverse document-frequency. * The goal of using tf-idf is to scale down the impact of tokens that occur - * very frequently in a given corpus while using the raw frequencies of a token - * and that are hence empirically less informative than features that occur in - * a small fraction of the training corpus. + * very frequently in a given corpus while using the type of Term-Frequency + * of a token and that are hence empirically less informative than features + * that occur in a small fraction of the training corpus. * TfIdfEncodingPolicy is used as a helper class for StringEncoding. * The encoder assigns a tf-idf number to each unique token and treat * the dataset as categorical. The tokens are labeled in the order of their @@ -35,7 +34,7 @@ namespace data { class TfIdfEncodingPolicy { public: - /* + /** * Enum class used to identify the type of tf encoding * * Follwing are the defination of the types @@ -53,6 +52,18 @@ class TfIdfEncodingPolicy TERM_FREQUENCY, }; + /** + * A constructor for the class which is use to set the type of term frequency + * and also the value for somoothIdf. + * + * @param tfType The type of term frequency, The avialbale option are + * RAW_COUNT : The count of a specific token + * BINARY : 1 if token occurs in document and 0 otherwise; + * TERM_FREQUENCY : Raw_count ÷ (number of words in document) + * SUBLINEAR_TF : log(Raw_Count) + 1 + * + * @param smoothIdf Used to indicate whether to use smooth idf or not. + */ TfIdfEncodingPolicy(TfTypes tfType = TfTypes::RAW_COUNT, bool smoothIdf = true) : tfType(tfType), diff --git a/src/mlpack/tests/string_encoding_test.cpp b/src/mlpack/tests/string_encoding_test.cpp index 5d8bf27803..d2fea48ef0 100644 --- a/src/mlpack/tests/string_encoding_test.cpp +++ b/src/mlpack/tests/string_encoding_test.cpp @@ -533,7 +533,7 @@ BOOST_AUTO_TEST_CASE(BagOfWordsEncodingIndividualCharactersTest) * Test the Bag of Words encoding algorithm in case of individual * character encoding, storing resulting in vector>. */ -BOOST_AUTO_TEST_CASE(OnePassBagOfWordsEncodingIndividualCharactersTest) +BOOST_AUTO_TEST_CASE(VectorBagOfWordsEncodingIndividualCharactersTest) { std::vector input = { "GACCA", @@ -1086,7 +1086,7 @@ BOOST_AUTO_TEST_CASE(SplitByAnyOfTfIdfEncodingSerialization) using EncoderType = TfIdfEncoding; EncoderType encoder; - SplitByAnyOf tokenizer(" "); + SplitByAnyOf tokenizer(" ,.\""); arma::mat output; encoder.Encode(stringEncodingInput, output, tokenizer); From fe950e933318d26f588ff18d61bf0c44e800658b Mon Sep 17 00:00:00 2001 From: jeffinsam Date: Mon, 12 Aug 2019 20:45:42 +0530 Subject: [PATCH 023/111] remove a test --- src/mlpack/tests/string_encoding_test.cpp | 32 ++--------------------- 1 file changed, 2 insertions(+), 30 deletions(-) diff --git a/src/mlpack/tests/string_encoding_test.cpp b/src/mlpack/tests/string_encoding_test.cpp index d2fea48ef0..926e8867fc 100644 --- a/src/mlpack/tests/string_encoding_test.cpp +++ b/src/mlpack/tests/string_encoding_test.cpp @@ -337,6 +337,8 @@ void CheckDictionaries(const StringEncodingDictionary& expected, const MapType& expectedMapping = expected.Mapping(); const MapType& mapping = obtained.Mapping(); + BOOST_REQUIRE_EQUAL(expected.Size(), obtained.Size()); + for (size_t i = 0; i < mapping.size(); i++) { BOOST_REQUIRE_EQUAL(mapping[i], expectedMapping[i]); @@ -1047,36 +1049,6 @@ BOOST_AUTO_TEST_CASE(TermFrequencyTfIdfEncodingIndividualCharactersTest) CheckMatrices(output, target, 1e-12); } -/** - * Serialization test for the TF-IDF encoding algorithm with - * the CharExtract tokenizer. - */ -BOOST_AUTO_TEST_CASE(CharExtractTfIdfEncodingSerialization) -{ - using EncoderType = TfIdfEncoding; - - EncoderType encoder; - CharExtract tokenizer; - arma::mat output; - - encoder.Encode(stringEncodingInput, output, tokenizer); - - EncoderType xmlEncoder, textEncoder, binaryEncoder; - arma::mat xmlOutput, textOutput, binaryOutput; - - SerializeObjectAll(encoder, xmlEncoder, textEncoder, binaryEncoder); - - CheckDictionaries(encoder.Dictionary(), xmlEncoder.Dictionary()); - CheckDictionaries(encoder.Dictionary(), textEncoder.Dictionary()); - CheckDictionaries(encoder.Dictionary(), binaryEncoder.Dictionary()); - - xmlEncoder.Encode(stringEncodingInput, xmlOutput, tokenizer); - textEncoder.Encode(stringEncodingInput, textOutput, tokenizer); - binaryEncoder.Encode(stringEncodingInput, binaryOutput, tokenizer); - - CheckMatrices(output, xmlOutput, textOutput, binaryOutput); -} - /** * Serialization test for the TF-IDF encoding algorithm with * the SplitByAnyOf tokenizer. From 497c3832f23dbf4ffaf5c505afec5978cad6fcbe Mon Sep 17 00:00:00 2001 From: jeffinsam Date: Sun, 15 Sep 2019 23:02:40 +0530 Subject: [PATCH 024/111] Binding for Loading and Saving images --- src/mlpack/core/data/load_image_impl.hpp | 8 +- src/mlpack/methods/preprocess/CMakeLists.txt | 4 + .../preprocess/load_save_image_main.cpp | 94 +++++++++++++++++++ 3 files changed, 102 insertions(+), 4 deletions(-) create mode 100644 src/mlpack/methods/preprocess/load_save_image_main.cpp diff --git a/src/mlpack/core/data/load_image_impl.hpp b/src/mlpack/core/data/load_image_impl.hpp index e4fa8230af..cc4a37595f 100644 --- a/src/mlpack/core/data/load_image_impl.hpp +++ b/src/mlpack/core/data/load_image_impl.hpp @@ -77,8 +77,8 @@ bool Load(const std::string& filename, info.Channels() = tempChannels; // Copy image into armadillo Mat. - matrix = arma::Mat(image, info.Width() * info.Height() * - info.Channels(), 1, true, true); + matrix = arma::conv_to >::from(arma::Mat(image, info.Width() * info.Height() * + info.Channels(), 1, true, true)); // Free the image pointer. free(image); @@ -108,11 +108,11 @@ bool Load(const std::vector& files, // Decide matrix dimension using the image height and width. matrix.set_size(info.Width() * info.Height() * info.Channels(), files.size()); - matrix.col(0) = img; + matrix.col(0) = arma::conv_to>::from(img); for (size_t i = 1; i < files.size() ; i++) { - arma::Mat colImg(matrix.colptr(i), matrix.n_rows, 1, + arma::Mat colImg(matrix.colptr(i), matrix.n_rows, 1, false, true); status &= Load(files[i], colImg, info, fatal, transpose); } diff --git a/src/mlpack/methods/preprocess/CMakeLists.txt b/src/mlpack/methods/preprocess/CMakeLists.txt index e1128afebc..36eb498bab 100644 --- a/src/mlpack/methods/preprocess/CMakeLists.txt +++ b/src/mlpack/methods/preprocess/CMakeLists.txt @@ -35,3 +35,7 @@ add_markdown_docs(preprocess_imputer "cli" "preprocessing") add_cli_executable(preprocess_scale) add_python_binding(preprocess_scale) add_markdown_docs(preprocess_scale "cli;python" "preprocessing") + +add_cli_executable(load_save_image) +add_python_binding(load_save_image) +add_markdown_docs(load_save_image "cli;python" "preprocessing") diff --git a/src/mlpack/methods/preprocess/load_save_image_main.cpp b/src/mlpack/methods/preprocess/load_save_image_main.cpp new file mode 100644 index 0000000000..d39b8a11a3 --- /dev/null +++ b/src/mlpack/methods/preprocess/load_save_image_main.cpp @@ -0,0 +1,94 @@ +/** + * @file load_save_image_main.cpp + * @author Jeffin Sam + * + * A CLI executable to load and image dataset. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifdef HAS_STB // Compile this only if stb is present. + +#include +#include +#include +#include +#include +#include + +using namespace mlpack; +using namespace mlpack::util; +using namespace arma; +using namespace std; + +PROGRAM_INFO("Load Image", + // Short description. + "A utility to load and save image dataset. This utility will allow you to " + "load and save a single image or an array of images.", + // Long description. + "This utility takes a images or an array of images and loads them to arma matrix." + "You can specify the height " + PRINT_PARAM_STRING("height") + "width " + + PRINT_PARAM_STRING("width") + " and channel " + PRINT_PARAM_STRING("channel") + + "of the images that needs to be loaded. \n There are other options too, that" + "can be specified such as quality " + PRINT_PARAM_STRING("quality") + " and " + + PRINT_PARAM_STRING("transpose") + + "\n\n" + + "You can also provide a dataset and save them as image using " + + PRINT_PARAM_STRING("dataset") + "and " + PRINT_PARAM_STRING("save") + + "as an parameter. An example to load an " + "image" + "\n\n" + + PRINT_CALL("load_save_image", "input", "X", "height", 256, "width", 256, + "channel", 3, "output", "Y") + "\n\n" + + " An example to save an image is :" + "\n\n" + + PRINT_CALL("load_save_image", "input", "X", "height", 256, "width", 256, + "channel", 3, "dataset", "Y"), + SEE_ALSO("@preprocess_binarize", "#preprocess_binarize"), + SEE_ALSO("@preprocess_describe", "#preprocess_describe"), + SEE_ALSO("@preprocess_imputer", "#preprocess_imputer")); + +//DEFINE PATAM +PARAM_VECTOR_IN_REQ(string, "input", "Image filenames which has to " + "be loaded/saved", "i"); + +PARAM_INT_IN_REQ("width", "Width of the image", "W"); +PARAM_INT_IN_REQ("channel", "Number of channel", "C"); + + +PARAM_MATRIX_OUT("output", "Matrix to save images data to.", "o"); + +PARAM_INT_IN("quality", "Compression of the image if saved as jpg (0-100).", "q",90); + +PARAM_FLAG("transpose", "Loaded dataset to be transposed", "t"); + +PARAM_INT_IN_REQ("height", "Height of the images", "H"); +PARAM_FLAG("save", "Save a dataset as images", "s"); +PARAM_MATRIX_IN("dataset", "Input matrix to save as images.", "I"); + +static void mlpackMain() +{ + // Parse command line options. + const int height = CLI::GetParam("height"); + const int width = CLI::GetParam("width"); + const int channel = CLI::GetParam("channel"); + const int quality = CLI::GetParam("quality"); + data::ImageInfo info(height, width, channel, quality); + const vector fileNames = + CLI::GetParam >("input"); + arma::mat out; + if(!CLI::HasParam("save")) + data::Load(fileNames, out, info, false, !CLI::HasParam("transpose")); + if (CLI::HasParam("output")) + CLI::GetParam("output") = std::move(out); + if(CLI::HasParam("save")) + { + if(!CLI::HasParam("dataset")) + { + throw std::runtime_error("Please provide a input matrix to save images from"); + } + data::Save(fileNames, CLI::GetParam("dataset"), info, false, !CLI::HasParam("transpose")); + } +} + +#endif // HAS_STB. From 3671ce47bb6b5dd4f1ce3c6d988d986d29e17a9e Mon Sep 17 00:00:00 2001 From: jeffinsam Date: Sun, 15 Sep 2019 23:03:43 +0530 Subject: [PATCH 025/111] correct saving of images implementation --- src/mlpack/core/data/save_impl.hpp | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/src/mlpack/core/data/save_impl.hpp b/src/mlpack/core/data/save_impl.hpp index 18a61530a5..e65d93f8ca 100644 --- a/src/mlpack/core/data/save_impl.hpp +++ b/src/mlpack/core/data/save_impl.hpp @@ -325,7 +325,8 @@ bool Save(const std::string& filename, bool status = false; try { - unsigned char* image = matrix.memptr(); + arma::Mat temp = arma::conv_to >::from(matrix); + unsigned char* image = temp.memptr(); if ("png" == Extension(filename)) { @@ -384,17 +385,10 @@ bool Save(const std::vector& files, // We transpose by default. So, un-transpose if necessary. if (!transpose) matrix = arma::trans(matrix); - - arma::Mat img; - bool status = Save(files[0], img, info, fatal, transpose); - - // Decide matrix dimension using the image height and width. - matrix.set_size(info.Width() * info.Height() * info.Channels(), files.size()); - matrix.col(0) = img; - - for (size_t i = 1; i < files.size() ; i++) + bool status = true; + for (size_t i = 0; i < files.size() ; i++) { - arma::Mat colImg(matrix.colptr(i), matrix.n_rows, 1, + arma::Mat colImg(matrix.colptr(i), matrix.n_rows, 1, false, true); status &= Save(files[i], colImg, info, fatal, transpose); } From e44eb663ca2e8dad7f72ebf84bb57496afb7d38b Mon Sep 17 00:00:00 2001 From: jeffinsam Date: Mon, 16 Sep 2019 23:26:35 +0530 Subject: [PATCH 026/111] style fixing and rectify transpose option --- src/mlpack/core/data/save_impl.hpp | 7 +-- .../preprocess/load_save_image_main.cpp | 34 +++++------ src/mlpack/tests/image_load_test.cpp | 57 +++++++++++++++++++ 3 files changed, 76 insertions(+), 22 deletions(-) diff --git a/src/mlpack/core/data/save_impl.hpp b/src/mlpack/core/data/save_impl.hpp index e65d93f8ca..08ae1d3807 100644 --- a/src/mlpack/core/data/save_impl.hpp +++ b/src/mlpack/core/data/save_impl.hpp @@ -293,9 +293,6 @@ bool Save(const std::string& filename, const bool transpose) { Timer::Start("saving_image"); - // We transpose by default. So, un-transpose if necessary. - if (!transpose) - matrix = arma::trans(matrix); int tempWidth, tempHeight, tempChannels, tempQuality; @@ -382,9 +379,7 @@ bool Save(const std::vector& files, throw std::runtime_error(oss.str()); return false; } - // We transpose by default. So, un-transpose if necessary. - if (!transpose) - matrix = arma::trans(matrix); + bool status = true; for (size_t i = 0; i < files.size() ; i++) { diff --git a/src/mlpack/methods/preprocess/load_save_image_main.cpp b/src/mlpack/methods/preprocess/load_save_image_main.cpp index d39b8a11a3..3780674328 100644 --- a/src/mlpack/methods/preprocess/load_save_image_main.cpp +++ b/src/mlpack/methods/preprocess/load_save_image_main.cpp @@ -23,27 +23,26 @@ using namespace mlpack::util; using namespace arma; using namespace std; -PROGRAM_INFO("Load Image", +PROGRAM_INFO("Load Save Image", // Short description. "A utility to load and save image dataset. This utility will allow you to " "load and save a single image or an array of images.", // Long description. - "This utility takes a images or an array of images and loads them to arma matrix." - "You can specify the height " + PRINT_PARAM_STRING("height") + "width " + - PRINT_PARAM_STRING("width") + " and channel " + PRINT_PARAM_STRING("channel") + - "of the images that needs to be loaded. \n There are other options too, that" - "can be specified such as quality " + PRINT_PARAM_STRING("quality") + " and " + - PRINT_PARAM_STRING("transpose") + - "\n\n" + - "You can also provide a dataset and save them as image using " + - PRINT_PARAM_STRING("dataset") + "and " + PRINT_PARAM_STRING("save") + - "as an parameter. An example to load an " - "image" + "\n\n" + + "This utility takes a image or an array of images and loads them to a" + "matrix. You can specify the height " + PRINT_PARAM_STRING("height") + + " width " + PRINT_PARAM_STRING("width") + " and channel " + + PRINT_PARAM_STRING("channel") + "of the images that needs to be loaded. " + "\nThere are other options too, that can be specified such as " + + PRINT_PARAM_STRING("quality") + " and " + PRINT_PARAM_STRING("transpose") + + ".\n\n" + + "You can also provide a dataset and save them as images using " + + PRINT_PARAM_STRING("dataset") + " and " + PRINT_PARAM_STRING("save") + + "as an parameter. An example to load an image : " + "\n\n" + PRINT_CALL("load_save_image", "input", "X", "height", 256, "width", 256, "channel", 3, "output", "Y") + "\n\n" + " An example to save an image is :" + "\n\n" + PRINT_CALL("load_save_image", "input", "X", "height", 256, "width", 256, - "channel", 3, "dataset", "Y"), + "channel", 3, "dataset", "Y", "save", true), SEE_ALSO("@preprocess_binarize", "#preprocess_binarize"), SEE_ALSO("@preprocess_describe", "#preprocess_describe"), SEE_ALSO("@preprocess_imputer", "#preprocess_imputer")); @@ -58,7 +57,8 @@ PARAM_INT_IN_REQ("channel", "Number of channel", "C"); PARAM_MATRIX_OUT("output", "Matrix to save images data to.", "o"); -PARAM_INT_IN("quality", "Compression of the image if saved as jpg (0-100).", "q",90); +PARAM_INT_IN("quality", "Compression of the image if saved as jpg (0-100).", + "q", 90); PARAM_FLAG("transpose", "Loaded dataset to be transposed", "t"); @@ -85,9 +85,11 @@ static void mlpackMain() { if(!CLI::HasParam("dataset")) { - throw std::runtime_error("Please provide a input matrix to save images from"); + throw std::runtime_error("Please provide a input matrix to save " + "images from"); } - data::Save(fileNames, CLI::GetParam("dataset"), info, false, !CLI::HasParam("transpose")); + data::Save(fileNames, CLI::GetParam("dataset"), info, false, + !CLI::HasParam("transpose")); } } diff --git a/src/mlpack/tests/image_load_test.cpp b/src/mlpack/tests/image_load_test.cpp index 0e2bbb0459..65527af0e0 100644 --- a/src/mlpack/tests/image_load_test.cpp +++ b/src/mlpack/tests/image_load_test.cpp @@ -67,6 +67,63 @@ BOOST_AUTO_TEST_CASE(SaveImageAPITest) BOOST_REQUIRE_EQUAL(im1[i], im2[i]); } +/** + * Test if the image is saved correctly using API for transpose. + */ +BOOST_AUTO_TEST_CASE(SaveImageTransposeAPITest) +{ + data::ImageInfo info(5, 5, 3, 90); + + arma::Mat im1; + size_t dimension = info.Width() * info.Height() * info.Channels(); + im1 = arma::randi>(dimension, 1); + BOOST_REQUIRE(data::Save("APITest.bmp", im1, info, false, false) == true); + + arma::Mat im2; + BOOST_REQUIRE(data::Load("APITest.bmp", im2, info, false, false) == true); + + BOOST_REQUIRE_EQUAL(im1.n_cols, im2.n_cols); + BOOST_REQUIRE_EQUAL(im1.n_rows, im2.n_rows); + for (size_t i = 10; i < im1.n_elem; ++i) + BOOST_REQUIRE_EQUAL(im1[i], im2[i]); +} + +/** + * Test that the image is loaded correctly into the matrix using the API + * for vectors. + */ +BOOST_AUTO_TEST_CASE(LoadVectorImageAPITest) +{ + arma::Mat matrix; + data::ImageInfo info; + std::vector file = {"test_image.png", "test_image.png"}; + BOOST_REQUIRE(data::Load(file, matrix, info, false, + true) == true); + BOOST_REQUIRE_EQUAL(matrix.n_rows, 50 * 50 * 3); // width * height * channels. + BOOST_REQUIRE_EQUAL(matrix.n_cols, 2); +} + +/** + * Test if the image is saved correctly using vector saving API for transpose. + */ +BOOST_AUTO_TEST_CASE(SaveImageVectorAPITest) +{ + data::ImageInfo info(5, 5, 3, 90); + + arma::Mat im1; + size_t dimension = info.Width() * info.Height() * info.Channels(); + im1 = arma::randi>(dimension, 2); + std::vector file = {"APITest1.bmp", "APITest2.bmp"}; + BOOST_REQUIRE(data::Save(file, im1, info, false, false) == true); + + arma::Mat im2; + BOOST_REQUIRE(data::Load(file, im2, info, false, false) == true); + + BOOST_REQUIRE_EQUAL(im1.n_cols, im2.n_cols); + BOOST_REQUIRE_EQUAL(im1.n_rows, im2.n_rows); + for (size_t i = 10; i < im1.n_elem; ++i) + BOOST_REQUIRE_EQUAL(im1[i], im2[i]); +} BOOST_AUTO_TEST_SUITE_END(); #endif // HAS_STB. From a34307e81d78a277ad76896fa1bbdb0f4ca9b42b Mon Sep 17 00:00:00 2001 From: jeffinsam Date: Mon, 16 Sep 2019 23:53:21 +0530 Subject: [PATCH 027/111] Add serialize to Image Info class, add save and load image info for the cli --- src/mlpack/core/data/image_info.hpp | 3 ++ src/mlpack/core/data/image_info_impl.hpp | 8 ++++ .../preprocess/load_save_image_main.cpp | 42 ++++++++++++++----- 3 files changed, 43 insertions(+), 10 deletions(-) diff --git a/src/mlpack/core/data/image_info.hpp b/src/mlpack/core/data/image_info.hpp index cd8c86ee8e..64f2e8f2bf 100644 --- a/src/mlpack/core/data/image_info.hpp +++ b/src/mlpack/core/data/image_info.hpp @@ -81,6 +81,9 @@ class ImageInfo //! Modify the image quality. size_t& Quality() { return quality; } + template + void serialize(Archive& ar, const unsigned int /* version */); + private: // To store the image width. size_t width; diff --git a/src/mlpack/core/data/image_info_impl.hpp b/src/mlpack/core/data/image_info_impl.hpp index dac80d163e..64ffefe8c8 100644 --- a/src/mlpack/core/data/image_info_impl.hpp +++ b/src/mlpack/core/data/image_info_impl.hpp @@ -63,6 +63,14 @@ inline ImageInfo::ImageInfo(const size_t width, // Do nothing. } + template + void ImageInfo::serialize(Archive& ar, const unsigned int /* version */) + { + ar & BOOST_SERIALIZATION_NVP(width); + ar & BOOST_SERIALIZATION_NVP(channels); + ar & BOOST_SERIALIZATION_NVP(height); + ar & BOOST_SERIALIZATION_NVP(quality); + } } // namespace data } // namespace mlpack diff --git a/src/mlpack/methods/preprocess/load_save_image_main.cpp b/src/mlpack/methods/preprocess/load_save_image_main.cpp index 3780674328..ce2919032d 100644 --- a/src/mlpack/methods/preprocess/load_save_image_main.cpp +++ b/src/mlpack/methods/preprocess/load_save_image_main.cpp @@ -22,6 +22,7 @@ using namespace mlpack; using namespace mlpack::util; using namespace arma; using namespace std; +using namespace mlpack::data; PROGRAM_INFO("Load Save Image", // Short description. @@ -51,8 +52,8 @@ PROGRAM_INFO("Load Save Image", PARAM_VECTOR_IN_REQ(string, "input", "Image filenames which has to " "be loaded/saved", "i"); -PARAM_INT_IN_REQ("width", "Width of the image", "W"); -PARAM_INT_IN_REQ("channel", "Number of channel", "C"); +PARAM_INT_IN("width", "Width of the image", "W", 256); +PARAM_INT_IN("channel", "Number of channel", "C", 3); PARAM_MATRIX_OUT("output", "Matrix to save images data to.", "o"); @@ -62,23 +63,41 @@ PARAM_INT_IN("quality", "Compression of the image if saved as jpg (0-100).", PARAM_FLAG("transpose", "Loaded dataset to be transposed", "t"); -PARAM_INT_IN_REQ("height", "Height of the images", "H"); +PARAM_INT_IN("height", "Height of the images", "H", 256); PARAM_FLAG("save", "Save a dataset as images", "s"); PARAM_MATRIX_IN("dataset", "Input matrix to save as images.", "I"); +// Loading/saving of a Image Info model. +PARAM_MODEL_IN(ImageInfo, "input_model", "Input Image Info model.", "m"); +PARAM_MODEL_OUT(ImageInfo, "output_model", "Output Image Info model.", "M"); static void mlpackMain() { // Parse command line options. - const int height = CLI::GetParam("height"); - const int width = CLI::GetParam("width"); - const int channel = CLI::GetParam("channel"); - const int quality = CLI::GetParam("quality"); - data::ImageInfo info(height, width, channel, quality); + ImageInfo* info; + Timer::Start("Loading/Saving Image"); + if (CLI::HasParam("input_model")) + { + info = CLI::GetParam("input_model"); + } + else + { + if(!CLI::HasParam("width") || !CLI::HasParam("height") || + !CLI::HasParam("channel")) + { + throw std::runtime_error("Please provide height, width and " + "number of channels of the images."); + } + const int height = CLI::GetParam("height"); + const int width = CLI::GetParam("width"); + const int channel = CLI::GetParam("channel"); + const int quality = CLI::GetParam("quality"); + info = new ImageInfo(height, width, channel, quality); + } const vector fileNames = CLI::GetParam >("input"); arma::mat out; if(!CLI::HasParam("save")) - data::Load(fileNames, out, info, false, !CLI::HasParam("transpose")); + Load(fileNames, out, *info, false, !CLI::HasParam("transpose")); if (CLI::HasParam("output")) CLI::GetParam("output") = std::move(out); if(CLI::HasParam("save")) @@ -88,9 +107,12 @@ static void mlpackMain() throw std::runtime_error("Please provide a input matrix to save " "images from"); } - data::Save(fileNames, CLI::GetParam("dataset"), info, false, + Save(fileNames, CLI::GetParam("dataset"), *info, false, !CLI::HasParam("transpose")); } + if (CLI::HasParam("output_model")) + CLI::GetParam("output_model") = info; + } #endif // HAS_STB. From f2e1bc547e5a7e5750983b5381984b04b836fbb5 Mon Sep 17 00:00:00 2001 From: jeffinsam Date: Tue, 17 Sep 2019 00:02:28 +0530 Subject: [PATCH 028/111] style fixes --- src/mlpack/core/data/load_image_impl.hpp | 4 +-- .../preprocess/load_save_image_main.cpp | 25 +++++++++---------- 2 files changed, 14 insertions(+), 15 deletions(-) diff --git a/src/mlpack/core/data/load_image_impl.hpp b/src/mlpack/core/data/load_image_impl.hpp index cc4a37595f..9b69dcbd92 100644 --- a/src/mlpack/core/data/load_image_impl.hpp +++ b/src/mlpack/core/data/load_image_impl.hpp @@ -77,8 +77,8 @@ bool Load(const std::string& filename, info.Channels() = tempChannels; // Copy image into armadillo Mat. - matrix = arma::conv_to >::from(arma::Mat(image, info.Width() * info.Height() * - info.Channels(), 1, true, true)); + matrix = arma::conv_to >::from(arma::Mat(image, + info.Width() * info.Height() * info.Channels(), 1, true, true)); // Free the image pointer. free(image); diff --git a/src/mlpack/methods/preprocess/load_save_image_main.cpp b/src/mlpack/methods/preprocess/load_save_image_main.cpp index ce2919032d..673ff08744 100644 --- a/src/mlpack/methods/preprocess/load_save_image_main.cpp +++ b/src/mlpack/methods/preprocess/load_save_image_main.cpp @@ -29,19 +29,19 @@ PROGRAM_INFO("Load Save Image", "A utility to load and save image dataset. This utility will allow you to " "load and save a single image or an array of images.", // Long description. - "This utility takes a image or an array of images and loads them to a" + "This utility takes a image or an array of images and loads them to a" "matrix. You can specify the height " + PRINT_PARAM_STRING("height") + " width " + PRINT_PARAM_STRING("width") + " and channel " + PRINT_PARAM_STRING("channel") + "of the images that needs to be loaded. " - "\nThere are other options too, that can be specified such as " + + "\nThere are other options too, that can be specified such as " + PRINT_PARAM_STRING("quality") + " and " + PRINT_PARAM_STRING("transpose") + ".\n\n" + - "You can also provide a dataset and save them as images using " + + "You can also provide a dataset and save them as images using " + PRINT_PARAM_STRING("dataset") + " and " + PRINT_PARAM_STRING("save") + - "as an parameter. An example to load an image : " + "\n\n" + + "as an parameter. An example to load an image : " + "\n\n" + PRINT_CALL("load_save_image", "input", "X", "height", 256, "width", 256, "channel", 3, "output", "Y") + "\n\n" + - " An example to save an image is :" + "\n\n" + + " An example to save an image is :" + "\n\n" + PRINT_CALL("load_save_image", "input", "X", "height", 256, "width", 256, "channel", 3, "dataset", "Y", "save", true), SEE_ALSO("@preprocess_binarize", "#preprocess_binarize"), @@ -50,12 +50,11 @@ PROGRAM_INFO("Load Save Image", //DEFINE PATAM PARAM_VECTOR_IN_REQ(string, "input", "Image filenames which has to " - "be loaded/saved", "i"); + "be loaded/saved", "i"); PARAM_INT_IN("width", "Width of the image", "W", 256); PARAM_INT_IN("channel", "Number of channel", "C", 3); - PARAM_MATRIX_OUT("output", "Matrix to save images data to.", "o"); PARAM_INT_IN("quality", "Compression of the image if saved as jpg (0-100).", @@ -81,7 +80,7 @@ static void mlpackMain() } else { - if(!CLI::HasParam("width") || !CLI::HasParam("height") || + if (!CLI::HasParam("width") || !CLI::HasParam("height") || !CLI::HasParam("channel")) { throw std::runtime_error("Please provide height, width and " @@ -96,22 +95,22 @@ static void mlpackMain() const vector fileNames = CLI::GetParam >("input"); arma::mat out; - if(!CLI::HasParam("save")) + if (!CLI::HasParam("save")) Load(fileNames, out, *info, false, !CLI::HasParam("transpose")); if (CLI::HasParam("output")) CLI::GetParam("output") = std::move(out); - if(CLI::HasParam("save")) + if (CLI::HasParam("save")) { - if(!CLI::HasParam("dataset")) + if (!CLI::HasParam("dataset")) { throw std::runtime_error("Please provide a input matrix to save " "images from"); } - Save(fileNames, CLI::GetParam("dataset"), *info, false, + Save (fileNames, CLI::GetParam ("dataset"), *info, false, !CLI::HasParam("transpose")); } if (CLI::HasParam("output_model")) - CLI::GetParam("output_model") = info; + CLI::GetParam ("output_model") = info; } From 080e619fb2661c50448a9d0be89836ed1adb3aab Mon Sep 17 00:00:00 2001 From: jeffinsam Date: Sun, 22 Sep 2019 05:05:41 +0530 Subject: [PATCH 029/111] Rectify build failures and add serialization for Image info class --- src/mlpack/core/data/image_info.hpp | 26 ++++++-- src/mlpack/core/data/image_info_impl.hpp | 16 ++--- .../preprocess/load_save_image_main.cpp | 28 +++++---- src/mlpack/tests/image_load_test.cpp | 60 +++++++++++++++++-- 4 files changed, 96 insertions(+), 34 deletions(-) diff --git a/src/mlpack/core/data/image_info.hpp b/src/mlpack/core/data/image_info.hpp index 64f2e8f2bf..199528e20d 100644 --- a/src/mlpack/core/data/image_info.hpp +++ b/src/mlpack/core/data/image_info.hpp @@ -56,10 +56,10 @@ class ImageInfo * @param channels Number of channels in the image. * @param quality Compression of the image if saved as jpg (0 - 100). */ - ImageInfo(const size_t width = 0, - const size_t height = 0, - const size_t channels = 3, - const size_t quality = 90); + ImageInfo(const size_t& width = 0, + const size_t& height = 0, + const size_t& channels = 3, + const size_t& quality = 90); //! Get the image width. const size_t& Width() const { return width; } @@ -82,7 +82,13 @@ class ImageInfo size_t& Quality() { return quality; } template - void serialize(Archive& ar, const unsigned int /* version */); + void serialize(Archive& ar, const unsigned int /* version */) + { + ar & BOOST_SERIALIZATION_NVP(width); + ar & BOOST_SERIALIZATION_NVP(channels); + ar & BOOST_SERIALIZATION_NVP(height); + ar & BOOST_SERIALIZATION_NVP(quality); + } private: // To store the image width. @@ -98,7 +104,15 @@ class ImageInfo size_t quality; }; #else -class ImageInfo { }; +class ImageInfo +{ + public: + template + void serialize(Archive& ar, const unsigned int /* version */) + { + // Nothing to do + } +}; #endif // HAS_STB. diff --git a/src/mlpack/core/data/image_info_impl.hpp b/src/mlpack/core/data/image_info_impl.hpp index 64ffefe8c8..b4e44a23bf 100644 --- a/src/mlpack/core/data/image_info_impl.hpp +++ b/src/mlpack/core/data/image_info_impl.hpp @@ -51,10 +51,10 @@ inline bool ImageFormatSupported(const std::string& fileName, const bool save) return false; } -inline ImageInfo::ImageInfo(const size_t width, - const size_t height, - const size_t channels, - const size_t quality) : +inline ImageInfo::ImageInfo(const size_t& width, + const size_t& height, + const size_t& channels, + const size_t& quality) : width(width), height(height), channels(channels), @@ -63,14 +63,6 @@ inline ImageInfo::ImageInfo(const size_t width, // Do nothing. } - template - void ImageInfo::serialize(Archive& ar, const unsigned int /* version */) - { - ar & BOOST_SERIALIZATION_NVP(width); - ar & BOOST_SERIALIZATION_NVP(channels); - ar & BOOST_SERIALIZATION_NVP(height); - ar & BOOST_SERIALIZATION_NVP(quality); - } } // namespace data } // namespace mlpack diff --git a/src/mlpack/methods/preprocess/load_save_image_main.cpp b/src/mlpack/methods/preprocess/load_save_image_main.cpp index 673ff08744..9eaf538377 100644 --- a/src/mlpack/methods/preprocess/load_save_image_main.cpp +++ b/src/mlpack/methods/preprocess/load_save_image_main.cpp @@ -9,14 +9,10 @@ * 3-clause BSD license along with mlpack. If not, see * http://www.opensource.org/licenses/BSD-3-Clause for more information. */ -#ifdef HAS_STB // Compile this only if stb is present. - #include #include -#include #include -#include -#include +#include using namespace mlpack; using namespace mlpack::util; @@ -24,6 +20,8 @@ using namespace arma; using namespace std; using namespace mlpack::data; +#ifdef HAS_STB // Compile this only if stb is present. + PROGRAM_INFO("Load Save Image", // Short description. "A utility to load and save image dataset. This utility will allow you to " @@ -65,6 +63,7 @@ PARAM_FLAG("transpose", "Loaded dataset to be transposed", "t"); PARAM_INT_IN("height", "Height of the images", "H", 256); PARAM_FLAG("save", "Save a dataset as images", "s"); PARAM_MATRIX_IN("dataset", "Input matrix to save as images.", "I"); + // Loading/saving of a Image Info model. PARAM_MODEL_IN(ImageInfo, "input_model", "Input Image Info model.", "m"); PARAM_MODEL_OUT(ImageInfo, "output_model", "Output Image Info model.", "M"); @@ -72,7 +71,9 @@ PARAM_MODEL_OUT(ImageInfo, "output_model", "Output Image Info model.", "M"); static void mlpackMain() { // Parse command line options. - ImageInfo* info; + + data::ImageInfo* info; + Timer::Start("Loading/Saving Image"); if (CLI::HasParam("input_model")) { @@ -86,11 +87,11 @@ static void mlpackMain() throw std::runtime_error("Please provide height, width and " "number of channels of the images."); } - const int height = CLI::GetParam("height"); - const int width = CLI::GetParam("width"); - const int channel = CLI::GetParam("channel"); - const int quality = CLI::GetParam("quality"); - info = new ImageInfo(height, width, channel, quality); + const size_t& height = CLI::GetParam("height"); + const size_t& width = CLI::GetParam("width"); + const size_t& channel = CLI::GetParam("channel"); + const size_t& quality = CLI::GetParam("quality"); + info = new data::ImageInfo(width, height, channel, quality); } const vector fileNames = CLI::GetParam >("input"); @@ -113,5 +114,8 @@ static void mlpackMain() CLI::GetParam ("output_model") = info; } +#else -#endif // HAS_STB. +static void mlpackMain() {} + +#endif // HAS_STB. \ No newline at end of file diff --git a/src/mlpack/tests/image_load_test.cpp b/src/mlpack/tests/image_load_test.cpp index 65527af0e0..9a1f689436 100644 --- a/src/mlpack/tests/image_load_test.cpp +++ b/src/mlpack/tests/image_load_test.cpp @@ -12,6 +12,9 @@ #include #include +#include +#include "test_tools.hpp" +#include "serialization.hpp" using namespace mlpack; using namespace mlpack::data; @@ -72,15 +75,15 @@ BOOST_AUTO_TEST_CASE(SaveImageAPITest) */ BOOST_AUTO_TEST_CASE(SaveImageTransposeAPITest) { - data::ImageInfo info(5, 5, 3, 90); + data::ImageInfo* info = new ImageInfo(5, 5, 3, 90); arma::Mat im1; - size_t dimension = info.Width() * info.Height() * info.Channels(); + size_t dimension = info->Width() * info->Height() * info->Channels(); im1 = arma::randi>(dimension, 1); - BOOST_REQUIRE(data::Save("APITest.bmp", im1, info, false, false) == true); + BOOST_REQUIRE(data::Save("APITest.bmp", im1, *info, false, false) == true); arma::Mat im2; - BOOST_REQUIRE(data::Load("APITest.bmp", im2, info, false, false) == true); + BOOST_REQUIRE(data::Load("APITest.bmp", im2, *info, false, false) == true); BOOST_REQUIRE_EQUAL(im1.n_cols, im2.n_cols); BOOST_REQUIRE_EQUAL(im1.n_rows, im2.n_rows); @@ -124,6 +127,55 @@ BOOST_AUTO_TEST_CASE(SaveImageVectorAPITest) for (size_t i = 10; i < im1.n_elem; ++i) BOOST_REQUIRE_EQUAL(im1[i], im2[i]); } + +/** + * Test if the image is saved correctly using API for arm mat. + */ +BOOST_AUTO_TEST_CASE(SaveImageMatAPITest) +{ + data::ImageInfo* info = new ImageInfo(5, 5, 3, 90); + + arma::Mat im1; + size_t dimension = info->Width() * info->Height() * info->Channels(); + im1 = arma::randi>(dimension, 1); + arma::mat input = arma::conv_to::from(im1); + BOOST_REQUIRE(Save("APITest.bmp", input, *info, false, false) == true); + + arma::mat output; + BOOST_REQUIRE(Load("APITest.bmp", output, *info, false, false) == true); + + BOOST_REQUIRE_EQUAL(input.n_cols, output.n_cols); + BOOST_REQUIRE_EQUAL(input.n_rows, output.n_rows); + for (size_t i = 10; i < input.n_elem; ++i) + BOOST_REQUIRE_CLOSE(input[i], output[i], 1e-5); +} + +/** + * Serialization test for the ImageInfo class. + */ +BOOST_AUTO_TEST_CASE(ImageInfoSerialization) +{ + + data::ImageInfo info(5, 5, 3, 90); + data::ImageInfo xmlInfo, textInfo, binaryInfo; + + SerializeObjectAll(info, xmlInfo, textInfo, binaryInfo); + + BOOST_REQUIRE_EQUAL(info.Width(), xmlInfo.Width()); + BOOST_REQUIRE_EQUAL(info.Height(), xmlInfo.Height()); + BOOST_REQUIRE_EQUAL(info.Channels(), xmlInfo.Channels()); + BOOST_REQUIRE_EQUAL(info.Quality(), xmlInfo.Quality()); + BOOST_REQUIRE_EQUAL(info.Width(), textInfo.Width()); + BOOST_REQUIRE_EQUAL(info.Height(), textInfo.Height()); + BOOST_REQUIRE_EQUAL(info.Channels(), textInfo.Channels()); + BOOST_REQUIRE_EQUAL(info.Quality(), textInfo.Quality()); + BOOST_REQUIRE_EQUAL(info.Width(), binaryInfo.Width()); + BOOST_REQUIRE_EQUAL(info.Height(), binaryInfo.Height()); + BOOST_REQUIRE_EQUAL(info.Channels(), binaryInfo.Channels()); + BOOST_REQUIRE_EQUAL(info.Quality(), binaryInfo.Quality()); + +} + BOOST_AUTO_TEST_SUITE_END(); #endif // HAS_STB. From 178d243b0abe718600c0a4daac45947af1a22607 Mon Sep 17 00:00:00 2001 From: jeffinsam Date: Sun, 22 Sep 2019 05:17:10 +0530 Subject: [PATCH 030/111] Remove extra lines and whitespace --- src/mlpack/core/data/image_info.hpp | 2 +- src/mlpack/core/data/save_impl.hpp | 3 ++- src/mlpack/methods/preprocess/load_save_image_main.cpp | 7 +++---- src/mlpack/tests/image_load_test.cpp | 2 -- 4 files changed, 6 insertions(+), 8 deletions(-) diff --git a/src/mlpack/core/data/image_info.hpp b/src/mlpack/core/data/image_info.hpp index 199528e20d..ae0c54dd26 100644 --- a/src/mlpack/core/data/image_info.hpp +++ b/src/mlpack/core/data/image_info.hpp @@ -87,7 +87,7 @@ class ImageInfo ar & BOOST_SERIALIZATION_NVP(width); ar & BOOST_SERIALIZATION_NVP(channels); ar & BOOST_SERIALIZATION_NVP(height); - ar & BOOST_SERIALIZATION_NVP(quality); + ar & BOOST_SERIALIZATION_NVP(quality); } private: diff --git a/src/mlpack/core/data/save_impl.hpp b/src/mlpack/core/data/save_impl.hpp index 08ae1d3807..22b9449962 100644 --- a/src/mlpack/core/data/save_impl.hpp +++ b/src/mlpack/core/data/save_impl.hpp @@ -322,7 +322,8 @@ bool Save(const std::string& filename, bool status = false; try { - arma::Mat temp = arma::conv_to >::from(matrix); + arma::Mat temp = arma::conv_to > + ::from(matrix); unsigned char* image = temp.memptr(); if ("png" == Extension(filename)) diff --git a/src/mlpack/methods/preprocess/load_save_image_main.cpp b/src/mlpack/methods/preprocess/load_save_image_main.cpp index 9eaf538377..45a9c79ff7 100644 --- a/src/mlpack/methods/preprocess/load_save_image_main.cpp +++ b/src/mlpack/methods/preprocess/load_save_image_main.cpp @@ -46,7 +46,7 @@ PROGRAM_INFO("Load Save Image", SEE_ALSO("@preprocess_describe", "#preprocess_describe"), SEE_ALSO("@preprocess_imputer", "#preprocess_imputer")); -//DEFINE PATAM +// DEFINE PARAM PARAM_VECTOR_IN_REQ(string, "input", "Image filenames which has to " "be loaded/saved", "i"); @@ -107,15 +107,14 @@ static void mlpackMain() throw std::runtime_error("Please provide a input matrix to save " "images from"); } - Save (fileNames, CLI::GetParam ("dataset"), *info, false, + Save(fileNames, CLI::GetParam ("dataset"), *info, false, !CLI::HasParam("transpose")); } if (CLI::HasParam("output_model")) CLI::GetParam ("output_model") = info; - } #else static void mlpackMain() {} -#endif // HAS_STB. \ No newline at end of file +#endif // HAS_STB. diff --git a/src/mlpack/tests/image_load_test.cpp b/src/mlpack/tests/image_load_test.cpp index 9a1f689436..f585b25e68 100644 --- a/src/mlpack/tests/image_load_test.cpp +++ b/src/mlpack/tests/image_load_test.cpp @@ -155,7 +155,6 @@ BOOST_AUTO_TEST_CASE(SaveImageMatAPITest) */ BOOST_AUTO_TEST_CASE(ImageInfoSerialization) { - data::ImageInfo info(5, 5, 3, 90); data::ImageInfo xmlInfo, textInfo, binaryInfo; @@ -173,7 +172,6 @@ BOOST_AUTO_TEST_CASE(ImageInfoSerialization) BOOST_REQUIRE_EQUAL(info.Height(), binaryInfo.Height()); BOOST_REQUIRE_EQUAL(info.Channels(), binaryInfo.Channels()); BOOST_REQUIRE_EQUAL(info.Quality(), binaryInfo.Quality()); - } BOOST_AUTO_TEST_SUITE_END(); From d9de571a4797b165d08f9982a010d319d0a7fed0 Mon Sep 17 00:00:00 2001 From: jeffinsam Date: Sun, 22 Sep 2019 14:32:34 +0530 Subject: [PATCH 031/111] add some CLI binding test --- .../preprocess/load_save_image_main.cpp | 7 +- src/mlpack/tests/CMakeLists.txt | 11 ++- .../tests/main_tests/load_save_image_test.cpp | 97 +++++++++++++++++++ 3 files changed, 107 insertions(+), 8 deletions(-) create mode 100644 src/mlpack/tests/main_tests/load_save_image_test.cpp diff --git a/src/mlpack/methods/preprocess/load_save_image_main.cpp b/src/mlpack/methods/preprocess/load_save_image_main.cpp index 45a9c79ff7..8342a98086 100644 --- a/src/mlpack/methods/preprocess/load_save_image_main.cpp +++ b/src/mlpack/methods/preprocess/load_save_image_main.cpp @@ -71,7 +71,6 @@ PARAM_MODEL_OUT(ImageInfo, "output_model", "Output Image Info model.", "M"); static void mlpackMain() { // Parse command line options. - data::ImageInfo* info; Timer::Start("Loading/Saving Image"); @@ -97,9 +96,11 @@ static void mlpackMain() CLI::GetParam >("input"); arma::mat out; if (!CLI::HasParam("save")) + { Load(fileNames, out, *info, false, !CLI::HasParam("transpose")); - if (CLI::HasParam("output")) - CLI::GetParam("output") = std::move(out); + if (CLI::HasParam("output")) + CLI::GetParam("output") = std::move(out); + } if (CLI::HasParam("save")) { if (!CLI::HasParam("dataset")) diff --git a/src/mlpack/tests/CMakeLists.txt b/src/mlpack/tests/CMakeLists.txt index d443e3ec24..e74297a441 100644 --- a/src/mlpack/tests/CMakeLists.txt +++ b/src/mlpack/tests/CMakeLists.txt @@ -134,6 +134,7 @@ add_executable(mlpack_test main_tests/kfn_test.cpp main_tests/knn_test.cpp main_tests/linear_regression_test.cpp + main_tests/load_save_image_test.cpp main_tests/logistic_regression_test.cpp main_tests/local_coordinate_coding_test.cpp main_tests/lmnn_test.cpp @@ -182,11 +183,11 @@ add_custom_command(TARGET mlpack_test ) add_custom_command(TARGET mlpack_test POST_BUILD - COMMAND ${CMAKE_COMMAND} -E tar xjpf mnist_first250_training_4s_and_9s.tar.bz2 - COMMAND ${CMAKE_COMMAND} -E tar xjpf digits_train.tar.bz2 - COMMAND ${CMAKE_COMMAND} -E tar xjpf digits_test.tar.bz2 - COMMAND ${CMAKE_COMMAND} -E tar xjpf digits_train_label.tar.bz2 - COMMAND ${CMAKE_COMMAND} -E tar xjpf digits_test_label.tar.bz2 + COMMAND ${CMAKE_COMMAND} -E tar xjf mnist_first250_training_4s_and_9s.tar.bz2 + COMMAND ${CMAKE_COMMAND} -E tar xjf digits_train.tar.bz2 + COMMAND ${CMAKE_COMMAND} -E tar xjf digits_test.tar.bz2 + COMMAND ${CMAKE_COMMAND} -E tar xjf digits_train_label.tar.bz2 + COMMAND ${CMAKE_COMMAND} -E tar xjf digits_test_label.tar.bz2 WORKING_DIRECTORY ${PROJECT_BINARY_DIR} ) diff --git a/src/mlpack/tests/main_tests/load_save_image_test.cpp b/src/mlpack/tests/main_tests/load_save_image_test.cpp new file mode 100644 index 0000000000..84c56b1992 --- /dev/null +++ b/src/mlpack/tests/main_tests/load_save_image_test.cpp @@ -0,0 +1,97 @@ +/** + * @file load_save_image_test.cpp + * @author Jeffin Sam + * + * Test mlpackMain() of load_save_image_main.cpp. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#define BINDING_TYPE BINDING_TYPE_TEST + +#include +static const std::string testName = "LoadSaveImage"; + +#include +#include + +#include "test_helper.hpp" +#include +#include "../test_tools.hpp" + +using namespace mlpack; + +struct LoadSaveImageTestFixture +{ + public: + LoadSaveImageTestFixture() + { + // Cache in the options for this program. + CLI::RestoreSettings(testName); + } + + ~LoadSaveImageTestFixture() + { + // Clear the settings. + bindings::tests::CleanMemory(); + CLI::ClearSettings(); + } +}; + +BOOST_FIXTURE_TEST_SUITE(LoadSaveImageMainTest, + LoadSaveImageTestFixture); + + +/** + * Check that two different scalers give two different output. + */ + +// arma::mat testimage = arma::conv_to::from( +// arma::randi>((50 * 50 * 3), 2)); + +// Global variable to avoid creating of multiple files +arma::mat input; +arma::mat output; + +BOOST_AUTO_TEST_CASE(LoadImageTest) +{ + SetInputParam>("input", {"test_image.png", "test_image.png"}); + SetInputParam("height", 50); + SetInputParam("width", 50); + SetInputParam("channel", 3); + + mlpackMain(); + output = CLI::GetParam("output"); + BOOST_REQUIRE_EQUAL(output.n_rows, 50 * 50 * 3); // width * height * channels. + BOOST_REQUIRE_EQUAL(output.n_cols, 2); +} + +BOOST_AUTO_TEST_CASE(SaveImageTest) +{ + // SetInputParam>("input", {"test_image777.png", "test_image999.png"}); + // SetInputParam("height", 50); + // SetInputParam("width", 50); + // SetInputParam("channel", 3); + // SetInputParam("save", true); + // SetInputParam("dataset", output); + // mlpackMain(); + // std::cout<<"output given is "<>("input", {"test_image777.png", "test_image999.png"}); + // SetInputParam("height", 50); + // SetInputParam("width", 50); + // SetInputParam("channel", 3); + // SetInputParam("save", false); + + // mlpackMain(); + // input = CLI::GetParam("output"); + // std::cout<<"outputnew is "< Date: Mon, 23 Sep 2019 13:52:11 +0530 Subject: [PATCH 032/111] Tried rectifyin test --- .../tests/main_tests/load_save_image_test.cpp | 66 +++++++++++++------ 1 file changed, 45 insertions(+), 21 deletions(-) diff --git a/src/mlpack/tests/main_tests/load_save_image_test.cpp b/src/mlpack/tests/main_tests/load_save_image_test.cpp index 84c56b1992..3a9c62e129 100644 --- a/src/mlpack/tests/main_tests/load_save_image_test.cpp +++ b/src/mlpack/tests/main_tests/load_save_image_test.cpp @@ -66,32 +66,56 @@ BOOST_AUTO_TEST_CASE(LoadImageTest) output = CLI::GetParam("output"); BOOST_REQUIRE_EQUAL(output.n_rows, 50 * 50 * 3); // width * height * channels. BOOST_REQUIRE_EQUAL(output.n_cols, 2); + + // SetInputParam>("input", {"test_image777.png", "test_image999.png"}); + // SetInputParam("height", 50); + // SetInputParam("width", 50); + // SetInputParam("channel", 3); + // SetInputParam("save", true); + // SetInputParam("dataset", output); + // mlpackMain(); + // std::cout<<"output given is "<>("input", {"test_image777.png", "test_image999.png"}); + // SetInputParam("height", 50); + // SetInputParam("width", 50); + // SetInputParam("channel", 3); + + // mlpackMain(); + // input = CLI::GetParam("output"); + // std::cout<<"outputnew is "<>("input", {"test_image777.png", "test_image999.png"}); - // SetInputParam("height", 50); - // SetInputParam("width", 50); - // SetInputParam("channel", 3); - // SetInputParam("save", true); - // SetInputParam("dataset", output); - // mlpackMain(); - // std::cout<<"output given is "<>("input", {"test_image777.png", "test_image999.png"}); - // SetInputParam("height", 50); - // SetInputParam("width", 50); - // SetInputParam("channel", 3); - // SetInputParam("save", false); + // std::cout<<"output entry is "<>("input", {"test_image777.png", "test_image999.png"}); + // SetInputParam("height", 50); + // SetInputParam("width", 50); + // SetInputParam("channel", 3); + // SetInputParam("save", true); + // SetInputParam("dataset", output); + // mlpackMain(); + // std::cout<<"output given is "<>("input", {"test_image777.png", "test_image999.png"}); + // SetInputParam("height", 50); + // SetInputParam("width", 50); + // SetInputParam("channel", 3); + + // mlpackMain(); + // input = CLI::GetParam("output"); + // std::cout<<"outputnew is "<("output"); - // std::cout<<"outputnew is "< Date: Mon, 23 Sep 2019 14:19:53 +0530 Subject: [PATCH 033/111] correctly build .so file --- src/mlpack/methods/preprocess/load_save_image_main.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/mlpack/methods/preprocess/load_save_image_main.cpp b/src/mlpack/methods/preprocess/load_save_image_main.cpp index 8342a98086..2c696e91d6 100644 --- a/src/mlpack/methods/preprocess/load_save_image_main.cpp +++ b/src/mlpack/methods/preprocess/load_save_image_main.cpp @@ -20,7 +20,6 @@ using namespace arma; using namespace std; using namespace mlpack::data; -#ifdef HAS_STB // Compile this only if stb is present. PROGRAM_INFO("Load Save Image", // Short description. @@ -68,6 +67,8 @@ PARAM_MATRIX_IN("dataset", "Input matrix to save as images.", "I"); PARAM_MODEL_IN(ImageInfo, "input_model", "Input Image Info model.", "m"); PARAM_MODEL_OUT(ImageInfo, "output_model", "Output Image Info model.", "M"); +#ifdef HAS_STB // Compile this only if stb is present. + static void mlpackMain() { // Parse command line options. From e15d479348d76d6961d883e6b5cab9b93dcaae9b Mon Sep 17 00:00:00 2001 From: jeffinsam Date: Mon, 23 Sep 2019 14:36:36 +0530 Subject: [PATCH 034/111] add some more cli binding test --- .../tests/main_tests/load_save_image_test.cpp | 45 +++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/src/mlpack/tests/main_tests/load_save_image_test.cpp b/src/mlpack/tests/main_tests/load_save_image_test.cpp index 3a9c62e129..d6d73614a9 100644 --- a/src/mlpack/tests/main_tests/load_save_image_test.cpp +++ b/src/mlpack/tests/main_tests/load_save_image_test.cpp @@ -118,4 +118,49 @@ BOOST_AUTO_TEST_CASE(SaveImageTest) } +/** + * Check Saved model is working. + */ +BOOST_AUTO_TEST_CASE(SavedModelTest) +{ + SetInputParam>("input", {"test_image.png", "test_image.png"}); + SetInputParam("height", 50); + SetInputParam("width", 50); + SetInputParam("channel", 3); + + mlpackMain(); + arma::mat randomOutput = CLI::GetParam("output"); + + SetInputParam>("input", {"test_image.png", "test_image.png"}); + SetInputParam("input_model", + CLI::GetParam("output_model")); + + mlpackMain(); + arma::mat savedOutput = CLI::GetParam("output"); + CheckMatrices(randomOutput, savedOutput); +} + +/** + * Check transpose option give two different output. + */ +BOOST_AUTO_TEST_CASE(TransposeTest) +{ + SetInputParam>("input", {"test_image.png", "test_image.png"}); + SetInputParam("height", 50); + SetInputParam("width", 50); + SetInputParam("channel", 3); + + mlpackMain(); + arma::mat normalOutput = CLI::GetParam("output"); + + SetInputParam>("input", {"test_image.png", "test_image.png"}); + SetInputParam("input_model", + CLI::GetParam("output_model")); + SetInputParam("transpose", true); + mlpackMain(); + arma::mat transposeOutput = CLI::GetParam("output"); + + CheckMatricesNotEqual(normalOutput, transposeOutput); +} + BOOST_AUTO_TEST_SUITE_END(); From 9b8b703f17235f7d0da117a50e0e810b27d4aebd Mon Sep 17 00:00:00 2001 From: jeffinsam Date: Mon, 23 Sep 2019 19:04:25 +0530 Subject: [PATCH 035/111] Just for @rcurtin or @zoq to debug :) --- src/mlpack/methods/preprocess/load_save_image_main.cpp | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/mlpack/methods/preprocess/load_save_image_main.cpp b/src/mlpack/methods/preprocess/load_save_image_main.cpp index 2c696e91d6..36240ed17f 100644 --- a/src/mlpack/methods/preprocess/load_save_image_main.cpp +++ b/src/mlpack/methods/preprocess/load_save_image_main.cpp @@ -117,6 +117,13 @@ static void mlpackMain() } #else -static void mlpackMain() {} +static void mlpackMain() +{ + if (!CLI::HasParam("input") || !CLI::HasParam("width") || + !CLI::HasParam("height") || !CLI::HasParam("channel")) + { + throw std::runtime_error("Are you sure We are on correct path"); + } +} #endif // HAS_STB. From a7d923d75549b3a28a3822f06a5a27785589550b Mon Sep 17 00:00:00 2001 From: jeffinsam Date: Mon, 23 Sep 2019 23:39:47 +0530 Subject: [PATCH 036/111] Resolved python binding , add Variable in setup.py.in --- CMakeLists.txt | 2 ++ src/mlpack/bindings/python/setup.py.in | 7 ++++++- src/mlpack/methods/preprocess/load_save_image_main.cpp | 9 +-------- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index c60590cf8e..bfa79c5877 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -344,6 +344,7 @@ if (NOT STB_IMAGE_FOUND) install(FILES ${CMAKE_BINARY_DIR}/deps/${STB_DIR}/stb_image.h DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}) install(FILES ${CMAKE_BINARY_DIR}/deps/${STB_DIR}/stb_image_write.h DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}) add_definitions(-DHAS_STB) + set(STB_AVAILABLE "1") else () list(GET STB_IMAGE_DOWNLOAD_STATUS_LIST 1 STB_DOWNLOAD_ERROR) message(WARNING @@ -357,6 +358,7 @@ else () # Already has STB installed. add_definitions(-DHAS_STB) set(MLPACK_INCLUDE_DIRS ${MLPACK_INCLUDE_DIRS} ${STB_IMAGE_INCLUDE_DIR}) + set(STB_AVAILABLE "1") endif () diff --git a/src/mlpack/bindings/python/setup.py.in b/src/mlpack/bindings/python/setup.py.in index 475afd85f1..793c3a4338 100644 --- a/src/mlpack/bindings/python/setup.py.in +++ b/src/mlpack/bindings/python/setup.py.in @@ -29,13 +29,18 @@ if not '${OpenMP_CXX_FLAGS}': else: extra_link_args=['${OpenMP_CXX_FLAGS}'] +if '${STB_AVAILABLE}': + extra_args = ['-DHAS_STB'] +else : + extra_args = [] + # Only build the extensions if we are asked to. if os.getenv('NO_BUILD') == '1': modules = [] else: cxx_flags = '${CMAKE_CXX_FLAGS}'.strip() cxx_flags = re.sub(' +', ' ', cxx_flags) - extra_args = ['-DBINDING_TYPE=BINDING_TYPE_PYX', + extra_args = extra_args + ['-DBINDING_TYPE=BINDING_TYPE_PYX', '-std=c++11'] + cxx_flags.split(' ') # This is used for parallel builds; CMake will set PYX_TO_BUILD accordingly. if module is not None: diff --git a/src/mlpack/methods/preprocess/load_save_image_main.cpp b/src/mlpack/methods/preprocess/load_save_image_main.cpp index 36240ed17f..2c696e91d6 100644 --- a/src/mlpack/methods/preprocess/load_save_image_main.cpp +++ b/src/mlpack/methods/preprocess/load_save_image_main.cpp @@ -117,13 +117,6 @@ static void mlpackMain() } #else -static void mlpackMain() -{ - if (!CLI::HasParam("input") || !CLI::HasParam("width") || - !CLI::HasParam("height") || !CLI::HasParam("channel")) - { - throw std::runtime_error("Are you sure We are on correct path"); - } -} +static void mlpackMain() {} #endif // HAS_STB. From f54b330b3d6b4e01a9b95e875f87e3e6d3e14ebb Mon Sep 17 00:00:00 2001 From: jeffinsam Date: Tue, 24 Sep 2019 09:57:48 +0530 Subject: [PATCH 037/111] rectified test for load_save_image --- .../preprocess/load_save_image_main.cpp | 13 ++ .../tests/main_tests/load_save_image_test.cpp | 113 +++++++++--------- 2 files changed, 69 insertions(+), 57 deletions(-) diff --git a/src/mlpack/methods/preprocess/load_save_image_main.cpp b/src/mlpack/methods/preprocess/load_save_image_main.cpp index 2c696e91d6..b9f5867520 100644 --- a/src/mlpack/methods/preprocess/load_save_image_main.cpp +++ b/src/mlpack/methods/preprocess/load_save_image_main.cpp @@ -87,6 +87,19 @@ static void mlpackMain() throw std::runtime_error("Please provide height, width and " "number of channels of the images."); } + // Positive value for width. + RequireParamValue("width", [](int x) { return x >= 0;}, true, + "width must be Positive"); + // Positive value for height. + RequireParamValue("height", [](int x) { return x >= 0;}, true, + "height must be Positive"); + // Positive value for channel. + RequireParamValue("channel", [](int x) { return x >= 0;}, true, + "channel must be Positive"); + // Positive value for quality. + RequireParamValue("quality", [](int x) { return x >= 0;}, true, + "quality must be Positive"); + const size_t& height = CLI::GetParam("height"); const size_t& width = CLI::GetParam("width"); const size_t& channel = CLI::GetParam("channel"); diff --git a/src/mlpack/tests/main_tests/load_save_image_test.cpp b/src/mlpack/tests/main_tests/load_save_image_test.cpp index d6d73614a9..f25d6a3f66 100644 --- a/src/mlpack/tests/main_tests/load_save_image_test.cpp +++ b/src/mlpack/tests/main_tests/load_save_image_test.cpp @@ -43,18 +43,6 @@ struct LoadSaveImageTestFixture BOOST_FIXTURE_TEST_SUITE(LoadSaveImageMainTest, LoadSaveImageTestFixture); - -/** - * Check that two different scalers give two different output. - */ - -// arma::mat testimage = arma::conv_to::from( -// arma::randi>((50 * 50 * 3), 2)); - -// Global variable to avoid creating of multiple files -arma::mat input; -arma::mat output; - BOOST_AUTO_TEST_CASE(LoadImageTest) { SetInputParam>("input", {"test_image.png", "test_image.png"}); @@ -63,59 +51,40 @@ BOOST_AUTO_TEST_CASE(LoadImageTest) SetInputParam("channel", 3); mlpackMain(); - output = CLI::GetParam("output"); - BOOST_REQUIRE_EQUAL(output.n_rows, 50 * 50 * 3); // width * height * channels. + arma::mat output = CLI::GetParam("output"); + // width * height * channels. + BOOST_REQUIRE_EQUAL(output.n_rows, 50 * 50 * 3); BOOST_REQUIRE_EQUAL(output.n_cols, 2); - - // SetInputParam>("input", {"test_image777.png", "test_image999.png"}); - // SetInputParam("height", 50); - // SetInputParam("width", 50); - // SetInputParam("channel", 3); - // SetInputParam("save", true); - // SetInputParam("dataset", output); - // mlpackMain(); - // std::cout<<"output given is "<>("input", {"test_image777.png", "test_image999.png"}); - // SetInputParam("height", 50); - // SetInputParam("width", 50); - // SetInputParam("channel", 3); - - // mlpackMain(); - // input = CLI::GetParam("output"); - // std::cout<<"outputnew is "<::from( + arma::randi>((5 * 5 * 3), 2)); + SetInputParam>("input", {"test_image777.png", + "test_image999.png"}); + SetInputParam("height", 5); + SetInputParam("width", 5); + SetInputParam("channel", 3); + SetInputParam("save", true); + SetInputParam("dataset", testimage); + mlpackMain(); - // SetInputParam>("input", {"test_image777.png", "test_image999.png"}); - // SetInputParam("height", 50); - // SetInputParam("width", 50); - // SetInputParam("channel", 3); - // SetInputParam("save", true); - // SetInputParam("dataset", output); - // mlpackMain(); - // std::cout<<"output given is "<>("input", {"test_image777.png", "test_image999.png"}); - // SetInputParam("height", 50); - // SetInputParam("width", 50); - // SetInputParam("channel", 3); - - // mlpackMain(); - // input = CLI::GetParam("output"); - // std::cout<<"outputnew is "<>("input", {"test_image777.png", + "test_image999.png"}); + SetInputParam("height", 5); + SetInputParam("width", 5); + SetInputParam("channel", 3); + mlpackMain(); + arma::mat output = CLI::GetParam("output"); + BOOST_REQUIRE_EQUAL(output.n_rows, 5 * 5 * 3); + BOOST_REQUIRE_EQUAL(output.n_cols, 2); + for (size_t i = 0; i < output.n_elem; ++i) + BOOST_REQUIRE_CLOSE(testimage[i], output[i], 1e-5); } /** @@ -163,4 +132,34 @@ BOOST_AUTO_TEST_CASE(TransposeTest) CheckMatricesNotEqual(normalOutput, transposeOutput); } +/** + * Check whether binding throws error if height, width or channel are not + * specified. + */ +BOOST_AUTO_TEST_CASE(IncompleteTest) +{ + SetInputParam>("input", {"test_image.png", "test_image.png"}); + SetInputParam("height", 50); + SetInputParam("width", 50); + + Log::Fatal.ignoreInput = true; + BOOST_REQUIRE_THROW(mlpackMain(), std::runtime_error); + Log::Fatal.ignoreInput = false; +} + +/** + * Check for invalid height values. + */ +BOOST_AUTO_TEST_CASE(InvalidInputTest) +{ + SetInputParam>("input", {"test_image.png", "test_image.png"}); + SetInputParam("height", -50); + SetInputParam("width", 50); + SetInputParam("channel", 3); + + Log::Fatal.ignoreInput = true; + BOOST_REQUIRE_THROW(mlpackMain(), std::runtime_error); + Log::Fatal.ignoreInput = false; +} + BOOST_AUTO_TEST_SUITE_END(); From 27365b28e13b592c75ad5b9167fa9b1769947527 Mon Sep 17 00:00:00 2001 From: jeffinsam Date: Wed, 25 Sep 2019 17:52:21 +0530 Subject: [PATCH 038/111] update documentation --- doc/tutorials/image/image.txt | 13 ++++++++----- src/mlpack/core/data/image_info.hpp | 8 ++++---- src/mlpack/core/data/image_info_impl.hpp | 8 ++++---- .../methods/preprocess/load_save_image_main.cpp | 8 ++++++-- src/mlpack/tests/image_load_test.cpp | 14 +++++++------- 5 files changed, 29 insertions(+), 22 deletions(-) diff --git a/doc/tutorials/image/image.txt b/doc/tutorials/image/image.txt index 8a635bf260..1397d4f8dc 100644 --- a/doc/tutorials/image/image.txt +++ b/doc/tutorials/image/image.txt @@ -46,7 +46,6 @@ ImageInfo class contains the metadata of the images. const size_t channels); @endcode Other public memebers include: - - flipVertical Flip the image vertical upon loading. - quality Compression of the image if saved as jpg (0-100). @section load_api_imagetut Load @@ -61,7 +60,8 @@ Standalone loading of images. * @param matrix Matrix to load the image into. * @param info An object of ImageInfo class. * @param fatal If an error should be reported as fatal (default false). - * @param transpose If true, transpose the matrix after loading. + * @param transpose If true, Filps the image, same as transposing the + * matrix after loading. * @return Boolean value indicating success or failure of load. */ template @@ -97,7 +97,8 @@ Loading multiple images: * @param matrix Matrix to save the image from. * @param info An object of ImageInfo class. * @param fatal If an error should be reported as fatal (default false). - * @param transpose If true, transpose the matrix after loading. + * @param transpose If true, Filps the image, same as transposing the + * matrix after loading. * @return Boolean value indicating success or failure of load. */ template @@ -129,7 +130,8 @@ Saving one image: * @param matrix Matrix to save the image from. * @param info An object of ImageInfo class. * @param fatal If an error should be reported as fatal (default false). - * @param transpose If true, transpose the matrix after loading. + * @param transpose If true, Filps the image, same as transposing the + * matrix after loading. * @return Boolean value indicating success or failure of load. */ template @@ -160,7 +162,8 @@ Saving multiple images: * @param matrix Matrix to save the image from. * @param info An object of ImageInfo class. * @param fatal If an error should be reported as fatal (default false). - * @param transpose If true, transpose the matrix after loading. + * @param transpose If true, Filps the image, same as transposing the + * matrix after loading. * @return Boolean value indicating success or failure of load. */ template diff --git a/src/mlpack/core/data/image_info.hpp b/src/mlpack/core/data/image_info.hpp index ae0c54dd26..315d7c3afd 100644 --- a/src/mlpack/core/data/image_info.hpp +++ b/src/mlpack/core/data/image_info.hpp @@ -56,10 +56,10 @@ class ImageInfo * @param channels Number of channels in the image. * @param quality Compression of the image if saved as jpg (0 - 100). */ - ImageInfo(const size_t& width = 0, - const size_t& height = 0, - const size_t& channels = 3, - const size_t& quality = 90); + ImageInfo(const size_t width = 0, + const size_t height = 0, + const size_t channels = 3, + const size_t quality = 90); //! Get the image width. const size_t& Width() const { return width; } diff --git a/src/mlpack/core/data/image_info_impl.hpp b/src/mlpack/core/data/image_info_impl.hpp index b4e44a23bf..dac80d163e 100644 --- a/src/mlpack/core/data/image_info_impl.hpp +++ b/src/mlpack/core/data/image_info_impl.hpp @@ -51,10 +51,10 @@ inline bool ImageFormatSupported(const std::string& fileName, const bool save) return false; } -inline ImageInfo::ImageInfo(const size_t& width, - const size_t& height, - const size_t& channels, - const size_t& quality) : +inline ImageInfo::ImageInfo(const size_t width, + const size_t height, + const size_t channels, + const size_t quality) : width(width), height(height), channels(channels), diff --git a/src/mlpack/methods/preprocess/load_save_image_main.cpp b/src/mlpack/methods/preprocess/load_save_image_main.cpp index b9f5867520..c6b122312f 100644 --- a/src/mlpack/methods/preprocess/load_save_image_main.cpp +++ b/src/mlpack/methods/preprocess/load_save_image_main.cpp @@ -2,7 +2,7 @@ * @file load_save_image_main.cpp * @author Jeffin Sam * - * A CLI executable to load and image dataset. + * A CLI executable to load and save a image dataset. * * mlpack is free software; you may redistribute it and/or modify it under the * terms of the 3-clause BSD license. You should have received a copy of the @@ -40,7 +40,11 @@ PROGRAM_INFO("Load Save Image", "channel", 3, "output", "Y") + "\n\n" + " An example to save an image is :" + "\n\n" + PRINT_CALL("load_save_image", "input", "X", "height", 256, "width", 256, - "channel", 3, "dataset", "Y", "save", true), + "channel", 3, "dataset", "Y", "save", true) + "\n\n" + + " An example to load an image and also flipping it while loading is :" + + "\n\n" + + PRINT_CALL("load_save_image", "input", "X", "height", 256, "width", 256, + "channel", 3, "output", "Y", "transpose", true), SEE_ALSO("@preprocess_binarize", "#preprocess_binarize"), SEE_ALSO("@preprocess_describe", "#preprocess_describe"), SEE_ALSO("@preprocess_imputer", "#preprocess_imputer")); diff --git a/src/mlpack/tests/image_load_test.cpp b/src/mlpack/tests/image_load_test.cpp index f585b25e68..0acbee14bb 100644 --- a/src/mlpack/tests/image_load_test.cpp +++ b/src/mlpack/tests/image_load_test.cpp @@ -66,7 +66,7 @@ BOOST_AUTO_TEST_CASE(SaveImageAPITest) BOOST_REQUIRE_EQUAL(im1.n_cols, im2.n_cols); BOOST_REQUIRE_EQUAL(im1.n_rows, im2.n_rows); - for (size_t i = 10; i < im1.n_elem; ++i) + for (size_t i = 0; i < im1.n_elem; ++i) BOOST_REQUIRE_EQUAL(im1[i], im2[i]); } @@ -87,7 +87,7 @@ BOOST_AUTO_TEST_CASE(SaveImageTransposeAPITest) BOOST_REQUIRE_EQUAL(im1.n_cols, im2.n_cols); BOOST_REQUIRE_EQUAL(im1.n_rows, im2.n_rows); - for (size_t i = 10; i < im1.n_elem; ++i) + for (size_t i = 0; i < im1.n_elem; ++i) BOOST_REQUIRE_EQUAL(im1[i], im2[i]); } @@ -111,7 +111,7 @@ BOOST_AUTO_TEST_CASE(LoadVectorImageAPITest) */ BOOST_AUTO_TEST_CASE(SaveImageVectorAPITest) { - data::ImageInfo info(5, 5, 3, 90); + data::ImageInfo info(5, 5, 3); arma::Mat im1; size_t dimension = info.Width() * info.Height() * info.Channels(); @@ -124,16 +124,16 @@ BOOST_AUTO_TEST_CASE(SaveImageVectorAPITest) BOOST_REQUIRE_EQUAL(im1.n_cols, im2.n_cols); BOOST_REQUIRE_EQUAL(im1.n_rows, im2.n_rows); - for (size_t i = 10; i < im1.n_elem; ++i) + for (size_t i = 0; i < im1.n_elem; ++i) BOOST_REQUIRE_EQUAL(im1[i], im2[i]); } /** - * Test if the image is saved correctly using API for arm mat. + * Test if the image is saved correctly using API for arma mat. */ BOOST_AUTO_TEST_CASE(SaveImageMatAPITest) { - data::ImageInfo* info = new ImageInfo(5, 5, 3, 90); + data::ImageInfo* info = new ImageInfo(5, 5, 3); arma::Mat im1; size_t dimension = info->Width() * info->Height() * info->Channels(); @@ -146,7 +146,7 @@ BOOST_AUTO_TEST_CASE(SaveImageMatAPITest) BOOST_REQUIRE_EQUAL(input.n_cols, output.n_cols); BOOST_REQUIRE_EQUAL(input.n_rows, output.n_rows); - for (size_t i = 10; i < input.n_elem; ++i) + for (size_t i = 0; i < input.n_elem; ++i) BOOST_REQUIRE_CLOSE(input[i], output[i], 1e-5); } From 357e57cdff3e33c2b21453f01e635039986acb82 Mon Sep 17 00:00:00 2001 From: jeffinsam Date: Wed, 25 Sep 2019 18:03:39 +0530 Subject: [PATCH 039/111] remove a white space --- src/mlpack/methods/preprocess/load_save_image_main.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mlpack/methods/preprocess/load_save_image_main.cpp b/src/mlpack/methods/preprocess/load_save_image_main.cpp index c6b122312f..19f5d3b2c7 100644 --- a/src/mlpack/methods/preprocess/load_save_image_main.cpp +++ b/src/mlpack/methods/preprocess/load_save_image_main.cpp @@ -40,7 +40,7 @@ PROGRAM_INFO("Load Save Image", "channel", 3, "output", "Y") + "\n\n" + " An example to save an image is :" + "\n\n" + PRINT_CALL("load_save_image", "input", "X", "height", 256, "width", 256, - "channel", 3, "dataset", "Y", "save", true) + "\n\n" + + "channel", 3, "dataset", "Y", "save", true) + "\n\n" + " An example to load an image and also flipping it while loading is :" + "\n\n" + PRINT_CALL("load_save_image", "input", "X", "height", 256, "width", 256, From 8c412673aaa648ba509d0235e7e1245e6e91417b Mon Sep 17 00:00:00 2001 From: jeffinsam Date: Thu, 3 Oct 2019 12:37:17 +0530 Subject: [PATCH 040/111] add serialization function which was deleted when resolving merge conflicts --- src/mlpack/core/data/image_info.hpp | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/mlpack/core/data/image_info.hpp b/src/mlpack/core/data/image_info.hpp index e7f444447c..16f69922ac 100644 --- a/src/mlpack/core/data/image_info.hpp +++ b/src/mlpack/core/data/image_info.hpp @@ -87,6 +87,15 @@ class ImageInfo //! Modify the image quality. size_t& Quality() { return quality; } + template + void serialize(Archive& ar, const unsigned int /* version */) + { + ar & BOOST_SERIALIZATION_NVP(width); + ar & BOOST_SERIALIZATION_NVP(channels); + ar & BOOST_SERIALIZATION_NVP(height); + ar & BOOST_SERIALIZATION_NVP(quality); + } + private: // To store the image width. size_t width; From 99cf51253c31843ae6615e685f2e5cb079c150ca Mon Sep 17 00:00:00 2001 From: jeffinsam Date: Mon, 4 Nov 2019 20:36:21 +0530 Subject: [PATCH 041/111] correctly appending the list --- src/mlpack/bindings/python/setup.py.in | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/mlpack/bindings/python/setup.py.in b/src/mlpack/bindings/python/setup.py.in index 06243f7ada..1e341844e1 100644 --- a/src/mlpack/bindings/python/setup.py.in +++ b/src/mlpack/bindings/python/setup.py.in @@ -56,14 +56,12 @@ if os.getenv('NO_BUILD') == '1': else: cxx_flags = '${CMAKE_CXX_FLAGS}'.strip() cxx_flags = re.sub(' +', ' ', cxx_flags) - extra_args = extra_args + ['-DBINDING_TYPE=BINDING_TYPE_PYX', - '-std=c++11'] + cxx_flags.split(' ') if cxx_flags: - extra_args = ['-DBINDING_TYPE=BINDING_TYPE_PYX', + extra_args = extra_args + ['-DBINDING_TYPE=BINDING_TYPE_PYX', '-std=c++11', '${OpenMP_CXX_FLAGS}'] + cxx_flags.split(' ') else: - extra_args = ['-DBINDING_TYPE=BINDING_TYPE_PYX', + extra_args = extra_args + ['-DBINDING_TYPE=BINDING_TYPE_PYX', '-std=c++11', '${OpenMP_CXX_FLAGS}'] From e0ef23c796580ae4604d4b1f5e9418e7144d564d Mon Sep 17 00:00:00 2001 From: jeffin143 Date: Thu, 30 Jan 2020 15:52:26 +0530 Subject: [PATCH 042/111] Improvements and adding test --- src/mlpack/methods/preprocess/CMakeLists.txt | 8 ++-- .../preprocess/load_save_image_main.cpp | 32 ++++++++----- src/mlpack/tests/image_load_test.cpp | 26 +++++----- .../tests/main_tests/load_save_image_test.cpp | 48 +++++++++++++++++++ 4 files changed, 85 insertions(+), 29 deletions(-) diff --git a/src/mlpack/methods/preprocess/CMakeLists.txt b/src/mlpack/methods/preprocess/CMakeLists.txt index 56cdb1846c..39810a1c87 100644 --- a/src/mlpack/methods/preprocess/CMakeLists.txt +++ b/src/mlpack/methods/preprocess/CMakeLists.txt @@ -42,6 +42,8 @@ add_cli_executable(preprocess_scale) add_python_binding(preprocess_scale) add_markdown_docs(preprocess_scale "cli;python" "preprocessing") -add_cli_executable(load_save_image) -add_python_binding(load_save_image) -add_markdown_docs(load_save_image "cli;python" "preprocessing") +if (NOT HAS_STB) + add_cli_executable(load_save_image) + add_python_binding(load_save_image) + add_markdown_docs(load_save_image "cli;python" "preprocessing") +endif () \ No newline at end of file diff --git a/src/mlpack/methods/preprocess/load_save_image_main.cpp b/src/mlpack/methods/preprocess/load_save_image_main.cpp index 31454cefcb..8673861e42 100644 --- a/src/mlpack/methods/preprocess/load_save_image_main.cpp +++ b/src/mlpack/methods/preprocess/load_save_image_main.cpp @@ -23,24 +23,30 @@ using namespace mlpack::data; PROGRAM_INFO("Load Save Image", // Short description. - "A utility to load and save image dataset. This utility will allow you to " - "load and save a single image or an list of images.", + "A utility to load an image or set of images into a single dataset that" + "can then be used by other mlpack methods and utilities. This can also" + "unpack an image dataset into individual files.", // Long description. "This utility takes a image or an array of images and loads them to a" " matrix. You can specify the height " + PRINT_PARAM_STRING("height") + " width " + PRINT_PARAM_STRING("width") + " and channel " + PRINT_PARAM_STRING("channel") + " of the images that needs to be loaded. " - "\nThere are other options too, that can be specified such as " + + "\n" + "There are other options too, that can be specified such as " + PRINT_PARAM_STRING("quality") + " and " + PRINT_PARAM_STRING("transpose") + ".\n\n" + "You can also provide a dataset and save them as images using " + PRINT_PARAM_STRING("dataset") + " and " + PRINT_PARAM_STRING("save") + - " as an parameter. An example to load an image : " + "\n\n" + + " as an parameter. An example to load an image : " + + "\n\n" + PRINT_CALL("load_save_image", "input", "X", "height", 256, "width", 256, - "channel", 3, "output", "Y") + "\n\n" + - " An example to save an image is :" + "\n\n" + + "channel", 3, "output", "Y") + + "\n\n" + + " An example to save an image is :" + + "\n\n" + PRINT_CALL("load_save_image", "input", "X", "height", 256, "width", 256, - "channel", 3, "dataset", "Y", "save", true) + "\n\n" + + "channel", 3, "dataset", "Y", "save", true) + + "\n\n" + " An example to load an image and also flipping it while loading is :" + "\n\n" + PRINT_CALL("load_save_image", "input", "X", "height", 256, "width", 256, @@ -50,8 +56,8 @@ PROGRAM_INFO("Load Save Image", SEE_ALSO("@preprocess_imputer", "#preprocess_imputer")); // DEFINE PARAM -PARAM_VECTOR_IN_REQ(string, "input", "Image filenames which has to " - "be loaded/saved", "i"); +PARAM_VECTOR_IN_REQ(string, "input", "Image filenames which have to " + "be loaded/saved.", "i"); PARAM_INT_IN("width", "Width of the image", "W", 256); PARAM_INT_IN("channel", "Number of channel", "C", 3); @@ -93,16 +99,16 @@ static void mlpackMain() } // Positive value for width. RequireParamValue("width", [](int x) { return x >= 0;}, true, - "width must be Positive"); + "width must be positive"); // Positive value for height. RequireParamValue("height", [](int x) { return x >= 0;}, true, - "height must be Positive"); + "height must be positive"); // Positive value for channel. RequireParamValue("channel", [](int x) { return x >= 0;}, true, - "channel must be Positive"); + "channel must be positive"); // Positive value for quality. RequireParamValue("quality", [](int x) { return x >= 0;}, true, - "quality must be Positive"); + "quality must be positive"); const size_t& height = CLI::GetParam("height"); const size_t& width = CLI::GetParam("width"); diff --git a/src/mlpack/tests/image_load_test.cpp b/src/mlpack/tests/image_load_test.cpp index 0acbee14bb..83a46b48ad 100644 --- a/src/mlpack/tests/image_load_test.cpp +++ b/src/mlpack/tests/image_load_test.cpp @@ -75,15 +75,15 @@ BOOST_AUTO_TEST_CASE(SaveImageAPITest) */ BOOST_AUTO_TEST_CASE(SaveImageTransposeAPITest) { - data::ImageInfo* info = new ImageInfo(5, 5, 3, 90); + data::ImageInfo info(5, 5, 3, 90); arma::Mat im1; - size_t dimension = info->Width() * info->Height() * info->Channels(); + size_t dimension = info.Width() * info.Height() * info.Channels(); im1 = arma::randi>(dimension, 1); - BOOST_REQUIRE(data::Save("APITest.bmp", im1, *info, false, false) == true); + BOOST_REQUIRE(data::Save("APITest.bmp", im1, info, false, false) == true); arma::Mat im2; - BOOST_REQUIRE(data::Load("APITest.bmp", im2, *info, false, false) == true); + BOOST_REQUIRE(data::Load("APITest.bmp", im2, info, false, false) == true); BOOST_REQUIRE_EQUAL(im1.n_cols, im2.n_cols); BOOST_REQUIRE_EQUAL(im1.n_rows, im2.n_rows); @@ -99,8 +99,8 @@ BOOST_AUTO_TEST_CASE(LoadVectorImageAPITest) { arma::Mat matrix; data::ImageInfo info; - std::vector file = {"test_image.png", "test_image.png"}; - BOOST_REQUIRE(data::Load(file, matrix, info, false, + std::vector files = {"test_image.png", "test_image.png"}; + BOOST_REQUIRE(data::Load(files, matrix, info, false, true) == true); BOOST_REQUIRE_EQUAL(matrix.n_rows, 50 * 50 * 3); // width * height * channels. BOOST_REQUIRE_EQUAL(matrix.n_cols, 2); @@ -116,11 +116,11 @@ BOOST_AUTO_TEST_CASE(SaveImageVectorAPITest) arma::Mat im1; size_t dimension = info.Width() * info.Height() * info.Channels(); im1 = arma::randi>(dimension, 2); - std::vector file = {"APITest1.bmp", "APITest2.bmp"}; - BOOST_REQUIRE(data::Save(file, im1, info, false, false) == true); + std::vector files = {"APITest1.bmp", "APITest2.bmp"}; + BOOST_REQUIRE(data::Save(files, im1, info, false, false) == true); arma::Mat im2; - BOOST_REQUIRE(data::Load(file, im2, info, false, false) == true); + BOOST_REQUIRE(data::Load(files, im2, info, false, false) == true); BOOST_REQUIRE_EQUAL(im1.n_cols, im2.n_cols); BOOST_REQUIRE_EQUAL(im1.n_rows, im2.n_rows); @@ -133,16 +133,16 @@ BOOST_AUTO_TEST_CASE(SaveImageVectorAPITest) */ BOOST_AUTO_TEST_CASE(SaveImageMatAPITest) { - data::ImageInfo* info = new ImageInfo(5, 5, 3); + data::ImageInfo info(5, 5, 3); arma::Mat im1; - size_t dimension = info->Width() * info->Height() * info->Channels(); + size_t dimension = info.Width() * info.Height() * info.Channels(); im1 = arma::randi>(dimension, 1); arma::mat input = arma::conv_to::from(im1); - BOOST_REQUIRE(Save("APITest.bmp", input, *info, false, false) == true); + BOOST_REQUIRE(Save("APITest.bmp", input, info, false, false) == true); arma::mat output; - BOOST_REQUIRE(Load("APITest.bmp", output, *info, false, false) == true); + BOOST_REQUIRE(Load("APITest.bmp", output, info, false, false) == true); BOOST_REQUIRE_EQUAL(input.n_cols, output.n_cols); BOOST_REQUIRE_EQUAL(input.n_rows, output.n_rows); diff --git a/src/mlpack/tests/main_tests/load_save_image_test.cpp b/src/mlpack/tests/main_tests/load_save_image_test.cpp index f25d6a3f66..016b3424e3 100644 --- a/src/mlpack/tests/main_tests/load_save_image_test.cpp +++ b/src/mlpack/tests/main_tests/load_save_image_test.cpp @@ -130,6 +130,8 @@ BOOST_AUTO_TEST_CASE(TransposeTest) arma::mat transposeOutput = CLI::GetParam("output"); CheckMatricesNotEqual(normalOutput, transposeOutput); + BOOST_REQUIRE_EQUAL(normalOutput.n_rows, transposeOutput.n_rows); + BOOST_REQUIRE_EQUAL(normalOutput.n_cols, transposeOutput.n_cols); } /** @@ -162,4 +164,50 @@ BOOST_AUTO_TEST_CASE(InvalidInputTest) Log::Fatal.ignoreInput = false; } +/** + * Check for invalid width values. + */ +BOOST_AUTO_TEST_CASE(InvalidWidthTest) +{ + SetInputParam>("input", {"test_image.png", "test_image.png"}); + SetInputParam("height", 50); + SetInputParam("width", -50); + SetInputParam("channel", 3); + + Log::Fatal.ignoreInput = true; + BOOST_REQUIRE_THROW(mlpackMain(), std::runtime_error); + Log::Fatal.ignoreInput = false; +} + +/** + * Check for invalid channel values. + */ +BOOST_AUTO_TEST_CASE(InvalidChannelTest) +{ + SetInputParam>("input", {"test_image.png", "test_image.png"}); + SetInputParam("height", 50); + SetInputParam("width", 50); + SetInputParam("channel", -1); + + Log::Fatal.ignoreInput = true; + BOOST_REQUIRE_THROW(mlpackMain(), std::runtime_error); + Log::Fatal.ignoreInput = false; +} + +/** + * Check for invalid input values. + */ +BOOST_AUTO_TEST_CASE(EmptyinputTest) +{ + SetInputParam>("input", {}); + SetInputParam("height", 50); + SetInputParam("width", 50); + SetInputParam("channel", 50); + + Log::Fatal.ignoreInput = true; + BOOST_REQUIRE_THROW(mlpackMain(), std::runtime_error); + Log::Fatal.ignoreInput = false; +} + + BOOST_AUTO_TEST_SUITE_END(); From 491e0308421525a0f313fcce5bde7e160ff0fabd Mon Sep 17 00:00:00 2001 From: AndreiMihalea Date: Sat, 7 Mar 2020 12:37:13 +0200 Subject: [PATCH 043/111] Implemented margin ranking loss --- .../methods/ann/loss_functions/CMakeLists.txt | 2 + .../loss_functions/margin_ranking_loss.hpp | 96 +++++++++++++++++++ .../margin_ranking_loss_impl.hpp | 74 ++++++++++++++ src/mlpack/tests/loss_functions_test.cpp | 54 +++++++++++ 4 files changed, 226 insertions(+) create mode 100644 src/mlpack/methods/ann/loss_functions/margin_ranking_loss.hpp create mode 100644 src/mlpack/methods/ann/loss_functions/margin_ranking_loss_impl.hpp diff --git a/src/mlpack/methods/ann/loss_functions/CMakeLists.txt b/src/mlpack/methods/ann/loss_functions/CMakeLists.txt index 5a74ca4a2e..d7e311bf55 100644 --- a/src/mlpack/methods/ann/loss_functions/CMakeLists.txt +++ b/src/mlpack/methods/ann/loss_functions/CMakeLists.txt @@ -9,6 +9,8 @@ set(SOURCES earth_mover_distance_impl.hpp kl_divergence.hpp kl_divergence_impl.hpp + margin_ranking_loss.hpp + margin_ranking_loss_impl.hpp mean_bias_error.hpp mean_bias_error_impl.hpp mean_squared_error.hpp diff --git a/src/mlpack/methods/ann/loss_functions/margin_ranking_loss.hpp b/src/mlpack/methods/ann/loss_functions/margin_ranking_loss.hpp new file mode 100644 index 0000000000..764c884bea --- /dev/null +++ b/src/mlpack/methods/ann/loss_functions/margin_ranking_loss.hpp @@ -0,0 +1,96 @@ +/** + * @file margin_ranking_loss.hpp + * @author Andrei Mihalea + * + * Definition of the Margin Ranking Loss function. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_ANN_LOSS_FUNCTION_MARGIN_RANKING_LOSS_HPP +#define MLPACK_ANN_LOSS_FUNCTION_MARGIN_RANKING_LOSS_HPP + +#include + +namespace mlpack { +namespace ann /** Artificial Neural Network. */ { + +template < + typename InputDataType = arma::mat, + typename OutputDataType = arma::mat +> +class MarginRankingLoss +{ + public: + /** + * Create the MarginRankingLoss object with Hyperparameter margin. + */ + MarginRankingLoss(const double margin = 1.0); + + /** + * Computes the Margin Ranking Loss function. + * Measures the loss between two intputs and a label with -1 and 1 values. + * a value of 1 in the label means the first input should be ranked higher + * and a value of -1 means the second input should be ranked higher. + * + * @param x1 First input data used for evaluating the specified function. + * @param x2 Second input data used for evaluating the specified function. + * @param y The label vector which contains -1 or 1 values. + */ + template + double Forward(const FirstInputType&& x1, + const SecondInputType&& x2, + const ThirdInputType&& y); + + /** + * Ordinary feed backward pass of a neural network. + * + * @param x1 The propagated first input activation. + * @param x2 The propagated second input activation. + * @param y The label vector which contains -1 or 1 values. + * @param output The calculated error. + */ + template < + typename FirstInputType, + typename SecondInputType, + typename ThirdInputType, + typename OutputType + > + void Backward(const FirstInputType&& x1, + const SecondInputType&& x2, + const ThirdInputType&& y, + OutputType&& output); + + //! Get the output parameter. + OutputDataType& OutputParameter() const { return outputParameter; } + //! Modify the output parameter. + OutputDataType& OutputParameter() { return outputParameter; } + + //! Get the output parameter. + double Margin() const { return margin; } + //! Modify the output parameter. + double& Margin() { return margin; } + + /** + * Serialize the layer. + */ + template + void serialize(Archive& ar, const unsigned int /* version */); + + private: + //! Locally-stored output parameter object. + OutputDataType outputParameter; + + //! The margin value used in calculating Margin Ranking Loss. + double margin; +}; // class MarginRankingLoss + +} // namespace ann +} // namespace mlpack + +// include implementation. +#include "margin_ranking_loss_impl.hpp" + +#endif diff --git a/src/mlpack/methods/ann/loss_functions/margin_ranking_loss_impl.hpp b/src/mlpack/methods/ann/loss_functions/margin_ranking_loss_impl.hpp new file mode 100644 index 0000000000..b36bb58212 --- /dev/null +++ b/src/mlpack/methods/ann/loss_functions/margin_ranking_loss_impl.hpp @@ -0,0 +1,74 @@ +/** + * @file margin_ranking_loss_impl.hpp + * @author Andrei Mihalea + * + * Implementation of the Margin Ranking Loss function. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_METHODS_ANN_LOSS_FUNCTION_MARGIN_IMPL_LOSS_HPP +#define MLPACK_METHODS_ANN_LOSS_FUNCTION_MARGIN_IMPL_LOSS_HPP + +// In case it hasn't been included. +#include "margin_ranking_loss.hpp" + +namespace mlpack { +namespace ann /** Artifical Neural Network. */ { + +template +MarginRankingLoss::MarginRankingLoss( + const double margin) : margin(margin) +{ + // Nothing to do here. +} + +template +template < + typename FirstInputType, + typename SecondInputType, + typename ThirdInputType +> +double MarginRankingLoss::Forward( + const FirstInputType&& x1, + const SecondInputType&& x2, + const ThirdInputType&& y) +{ + return arma::accu(arma::max(arma::zeros(size(y)), + -y % (x1 - x2) + margin)) / y.n_cols; +} + +template +template < + typename FirstInputType, + typename SecondInputType, + typename ThirdInputType, + typename OutputType +> +void MarginRankingLoss::Backward( + const FirstInputType&& x1, + const SecondInputType&& x2, + const ThirdInputType&& y, + OutputType&& output) +{ + output = -y % (x1 - x2) + margin; + output.elem(arma::find(output >= 0)).ones(); + output.elem(arma::find(output < 0)).zeros(); + output = (x2 - x1) % output / y.n_cols; +} + +template +template +void MarginRankingLoss::serialize( + Archive& ar, + const unsigned int /* version */) +{ + ar & BOOST_SERIALIZATION_NVP(margin); +} + +} // namespace ann +} // namespace mlpack + +#endif diff --git a/src/mlpack/tests/loss_functions_test.cpp b/src/mlpack/tests/loss_functions_test.cpp index 7988a961d0..ed08a844de 100644 --- a/src/mlpack/tests/loss_functions_test.cpp +++ b/src/mlpack/tests/loss_functions_test.cpp @@ -23,6 +23,7 @@ #include #include #include +#include #include #include #include @@ -471,4 +472,57 @@ BOOST_AUTO_TEST_CASE(SimpleMeanBiasErrorTest) BOOST_REQUIRE_EQUAL(output.n_elem, 1); } +/* + * Simple test for the Margin Ranking Loss function. + */ +BOOST_AUTO_TEST_CASE(MarginRankingLossTest) +{ + arma::mat x1, x2, y, output; + MarginRankingLoss<> module; + + // Test the Forward function on a user generator input and compare it against + // the manually calculated result. + x1 = arma::mat("1 2 5 7 -1 -3"); + x2 = arma::mat("-1 3 -4 11 3 -3"); + y = arma::mat("1 -1 -1 1 -1 1"); + double error = module.Forward(std::move(x1), std::move(x2), std::move(y)); + // Computed using PyTorch + // >>> import torch + // >>> import torch.nn.functional as F + // >>> x1 = torch.tensor([1., 2., 5., 7., -1., -3.]) + // >>> x2 = torch.tensor([-1., 3., -4., 11., 3., -3.]) + // >>> y = torch.tensor([1., -1., -1., 1., -1., 1.], requires_grad=True) + // >>> loss = F.margin_ranking_loss(x1, x2, y, margin=1.) + // >>> loss.item() + // 2.6666667461395264 + // >>> y.grad + // tensor([-0.0000, 0.1667, -1.5000, 0.6667, 0.0000, -0.0000]) + BOOST_REQUIRE_CLOSE(error, 2.66667, 1e-3); + + // Test the Backward function. + module.Backward(std::move(x1), std::move(x2), std::move(y), + std::move(output)); + + CheckMatrices(output, arma::mat("-0.000000 0.166667 -1.500000 0.666667 " + "0.000000 -0.000000"), 1e-3); + BOOST_REQUIRE_EQUAL(output.n_rows, y.n_rows); + BOOST_REQUIRE_EQUAL(output.n_cols, y.n_cols); + + // Test the error function on another input. + x1 = arma::mat("0.4287 -1.6208 -1.5006 -0.4473 1.5208 -4.5184 9.3574 " + "-4.8090 4.3455 5.2070"); + x2 = arma::mat("-4.5288 -9.2766 -0.5882 -5.6643 -6.0175 8.8506 3.4759 " + "-9.4886 2.2755 8.4951"); + y = arma::mat("1 1 -1 1 -1 1 1 1 -1 1"); + error = module.Forward(std::move(x1), std::move(x2), std::move(y)); + BOOST_REQUIRE_CLOSE(error, 3.03530, 1e-3); + + // Test the Backward function on the second input. + module.Backward(std::move(x1), std::move(x2), std::move(y), + std::move(output)); + + CheckMatrices(output, arma::mat("0.000000 0.000000 0.091240 0.000000 " + "-0.753830 1.336900 0.000000 0.000000 -0.207000 0.328810"), 1e-6); +} + BOOST_AUTO_TEST_SUITE_END(); From a820fd919d86efe3fddcc38cc46b5e8920977967 Mon Sep 17 00:00:00 2001 From: AndreiMihalea Date: Sat, 7 Mar 2020 12:50:31 +0200 Subject: [PATCH 044/111] Pulled the latest update to loss_functions_test --- src/mlpack/tests/loss_functions_test.cpp | 41 +++++++++++++++++++++++- 1 file changed, 40 insertions(+), 1 deletion(-) diff --git a/src/mlpack/tests/loss_functions_test.cpp b/src/mlpack/tests/loss_functions_test.cpp index ed08a844de..02f618bf75 100644 --- a/src/mlpack/tests/loss_functions_test.cpp +++ b/src/mlpack/tests/loss_functions_test.cpp @@ -23,8 +23,8 @@ #include #include #include -#include #include +#include #include #include @@ -472,6 +472,45 @@ BOOST_AUTO_TEST_CASE(SimpleMeanBiasErrorTest) BOOST_REQUIRE_EQUAL(output.n_elem, 1); } +/** + * Simple test for the Log-Hyperbolic-Cosine loss function. + */ +BOOST_AUTO_TEST_CASE(LogCoshLossTest) +{ + arma::mat input, target, output; + double loss; + LogCoshLoss<> module(2); + + // Test the Forward function. Loss should be 0 if input = target. + input = arma::ones(10, 1); + target = arma::ones(10, 1); + loss = module.Forward(std::move(input), std::move(target)); + BOOST_REQUIRE_EQUAL(loss, 0); + + // Test the Backward function for input = target. + module.Backward(std::move(input), std::move(target), std::move(output)); + for (double el : output) + { + // For input = target we should get 0.0 everywhere. + BOOST_REQUIRE_CLOSE(el, 0.0, 1e-5); + } + + BOOST_REQUIRE_EQUAL(output.n_rows, input.n_rows); + BOOST_REQUIRE_EQUAL(output.n_cols, input.n_cols); + + // Test the Forward function. Loss should be 0.546621. + input = arma::mat("1 2 3 4 5"); + target = arma::mat("1 2.4 3.4 4.2 5.5"); + loss = module.Forward(std::move(input), std::move(target)); + BOOST_REQUIRE_CLOSE(loss, 0.546621, 1e-3); + + // Test the Backward function. + module.Backward(std::move(input), std::move(target), std::move(output)); + BOOST_REQUIRE_CLOSE(arma::accu(output), 2.46962, 1e-3); + BOOST_REQUIRE_EQUAL(output.n_rows, input.n_rows); + BOOST_REQUIRE_EQUAL(output.n_cols, input.n_cols); +} + /* * Simple test for the Margin Ranking Loss function. */ From fa5f2e9d88458ecd1e0ae5d7a7c7450151a6f5ee Mon Sep 17 00:00:00 2001 From: AndreiMihalea Date: Sat, 7 Mar 2020 12:57:35 +0200 Subject: [PATCH 045/111] Added the margin ranking loss import --- src/mlpack/tests/loss_functions_test.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/mlpack/tests/loss_functions_test.cpp b/src/mlpack/tests/loss_functions_test.cpp index 02f618bf75..8db9a75917 100644 --- a/src/mlpack/tests/loss_functions_test.cpp +++ b/src/mlpack/tests/loss_functions_test.cpp @@ -21,6 +21,7 @@ #include #include #include +#include #include #include #include From 6f72b588a7a515e7b1b4848ca7ea791727ca5811 Mon Sep 17 00:00:00 2001 From: AndreiMihalea Date: Sat, 7 Mar 2020 13:08:23 +0200 Subject: [PATCH 046/111] Solved some style checks --- .../methods/ann/loss_functions/margin_ranking_loss.hpp | 2 +- .../methods/ann/loss_functions/margin_ranking_loss_impl.hpp | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/mlpack/methods/ann/loss_functions/margin_ranking_loss.hpp b/src/mlpack/methods/ann/loss_functions/margin_ranking_loss.hpp index 764c884bea..72198396df 100644 --- a/src/mlpack/methods/ann/loss_functions/margin_ranking_loss.hpp +++ b/src/mlpack/methods/ann/loss_functions/margin_ranking_loss.hpp @@ -93,4 +93,4 @@ class MarginRankingLoss // include implementation. #include "margin_ranking_loss_impl.hpp" -#endif +#endif diff --git a/src/mlpack/methods/ann/loss_functions/margin_ranking_loss_impl.hpp b/src/mlpack/methods/ann/loss_functions/margin_ranking_loss_impl.hpp index b36bb58212..4950eb6b94 100644 --- a/src/mlpack/methods/ann/loss_functions/margin_ranking_loss_impl.hpp +++ b/src/mlpack/methods/ann/loss_functions/margin_ranking_loss_impl.hpp @@ -28,7 +28,7 @@ MarginRankingLoss::MarginRankingLoss( template template < typename FirstInputType, - typename SecondInputType, + typename SecondInputType, typename ThirdInputType > double MarginRankingLoss::Forward( @@ -36,7 +36,7 @@ double MarginRankingLoss::Forward( const SecondInputType&& x2, const ThirdInputType&& y) { - return arma::accu(arma::max(arma::zeros(size(y)), + return arma::accu(arma::max(arma::zeros(size(y)), -y % (x1 - x2) + margin)) / y.n_cols; } @@ -71,4 +71,4 @@ void MarginRankingLoss::serialize( } // namespace ann } // namespace mlpack -#endif +#endif From e0b76b7a595ee5cc92e381e575f0c951700f6044 Mon Sep 17 00:00:00 2001 From: AndreiMihalea Date: Sat, 7 Mar 2020 13:21:32 +0200 Subject: [PATCH 047/111] Solved the rest of style checks --- .../methods/ann/loss_functions/margin_ranking_loss.hpp | 7 ++++++- src/mlpack/tests/loss_functions_test.cpp | 4 ++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/src/mlpack/methods/ann/loss_functions/margin_ranking_loss.hpp b/src/mlpack/methods/ann/loss_functions/margin_ranking_loss.hpp index 72198396df..7f02c15ceb 100644 --- a/src/mlpack/methods/ann/loss_functions/margin_ranking_loss.hpp +++ b/src/mlpack/methods/ann/loss_functions/margin_ranking_loss.hpp @@ -39,7 +39,12 @@ class MarginRankingLoss * @param x2 Second input data used for evaluating the specified function. * @param y The label vector which contains -1 or 1 values. */ - template + template < + typename FirstInputType, + typename SecondInputType, + typename ThirdInputType + > + double Forward(const FirstInputType&& x1, const SecondInputType&& x2, const ThirdInputType&& y); diff --git a/src/mlpack/tests/loss_functions_test.cpp b/src/mlpack/tests/loss_functions_test.cpp index 8db9a75917..3136a1a0ca 100644 --- a/src/mlpack/tests/loss_functions_test.cpp +++ b/src/mlpack/tests/loss_functions_test.cpp @@ -540,7 +540,7 @@ BOOST_AUTO_TEST_CASE(MarginRankingLossTest) BOOST_REQUIRE_CLOSE(error, 2.66667, 1e-3); // Test the Backward function. - module.Backward(std::move(x1), std::move(x2), std::move(y), + module.Backward(std::move(x1), std::move(x2), std::move(y), std::move(output)); CheckMatrices(output, arma::mat("-0.000000 0.166667 -1.500000 0.666667 " @@ -560,7 +560,7 @@ BOOST_AUTO_TEST_CASE(MarginRankingLossTest) // Test the Backward function on the second input. module.Backward(std::move(x1), std::move(x2), std::move(y), std::move(output)); - + CheckMatrices(output, arma::mat("0.000000 0.000000 0.091240 0.000000 " "-0.753830 1.336900 0.000000 0.000000 -0.207000 0.328810"), 1e-6); } From 9949edc365ad1f55e586a33172ea53c071220436 Mon Sep 17 00:00:00 2001 From: AndreiMihalea Date: Sat, 7 Mar 2020 14:51:01 +0200 Subject: [PATCH 048/111] Added description for hyperparameter margin and a template description for input / output --- .../ann/loss_functions/margin_ranking_loss.hpp | 10 +++++++++- src/mlpack/tests/loss_functions_test.cpp | 12 +----------- 2 files changed, 10 insertions(+), 12 deletions(-) diff --git a/src/mlpack/methods/ann/loss_functions/margin_ranking_loss.hpp b/src/mlpack/methods/ann/loss_functions/margin_ranking_loss.hpp index 7f02c15ceb..cb0f7ca1d3 100644 --- a/src/mlpack/methods/ann/loss_functions/margin_ranking_loss.hpp +++ b/src/mlpack/methods/ann/loss_functions/margin_ranking_loss.hpp @@ -17,6 +17,13 @@ namespace mlpack { namespace ann /** Artificial Neural Network. */ { +/** + * @tparam InputDataType Type of the input data (arma::colvec, arma::mat, + * arma::sp_mat or arma::cube). + * @tparam OutputDataType Type of the output data (arma::colvec, arma::mat, + * arma::sp_mat or arma::cube). + */ + template < typename InputDataType = arma::mat, typename OutputDataType = arma::mat @@ -26,6 +33,8 @@ class MarginRankingLoss public: /** * Create the MarginRankingLoss object with Hyperparameter margin. + * Hyperparameter margin defines a minimum distance between correctly ranked + * samples. */ MarginRankingLoss(const double margin = 1.0); @@ -44,7 +53,6 @@ class MarginRankingLoss typename SecondInputType, typename ThirdInputType > - double Forward(const FirstInputType&& x1, const SecondInputType&& x2, const ThirdInputType&& y); diff --git a/src/mlpack/tests/loss_functions_test.cpp b/src/mlpack/tests/loss_functions_test.cpp index 3136a1a0ca..1640206357 100644 --- a/src/mlpack/tests/loss_functions_test.cpp +++ b/src/mlpack/tests/loss_functions_test.cpp @@ -526,17 +526,7 @@ BOOST_AUTO_TEST_CASE(MarginRankingLossTest) x2 = arma::mat("-1 3 -4 11 3 -3"); y = arma::mat("1 -1 -1 1 -1 1"); double error = module.Forward(std::move(x1), std::move(x2), std::move(y)); - // Computed using PyTorch - // >>> import torch - // >>> import torch.nn.functional as F - // >>> x1 = torch.tensor([1., 2., 5., 7., -1., -3.]) - // >>> x2 = torch.tensor([-1., 3., -4., 11., 3., -3.]) - // >>> y = torch.tensor([1., -1., -1., 1., -1., 1.], requires_grad=True) - // >>> loss = F.margin_ranking_loss(x1, x2, y, margin=1.) - // >>> loss.item() - // 2.6666667461395264 - // >>> y.grad - // tensor([-0.0000, 0.1667, -1.5000, 0.6667, 0.0000, -0.0000]) + // Computed using torch.nn.functional.margin_ranking_loss() BOOST_REQUIRE_CLOSE(error, 2.66667, 1e-3); // Test the Backward function. From c600714c413bb30cbd1d3591d22e7fefd3a14fc3 Mon Sep 17 00:00:00 2001 From: AndreiMihalea Date: Mon, 9 Mar 2020 08:55:51 +0200 Subject: [PATCH 049/111] retrigger checks; see if python27 passes now From f0e73c73d672b227f43f466cd931fefc885a77fe Mon Sep 17 00:00:00 2001 From: jeffin143 Date: Sun, 15 Mar 2020 00:22:34 +0530 Subject: [PATCH 050/111] Rename LoadSave to ImageConverter --- src/mlpack/methods/preprocess/CMakeLists.txt | 6 +++--- ...image_main.cpp => image_converter_main.cpp} | 16 +++++----------- src/mlpack/tests/CMakeLists.txt | 2 +- ...image_test.cpp => image_converter_test.cpp} | 18 ++++++++++-------- 4 files changed, 19 insertions(+), 23 deletions(-) rename src/mlpack/methods/preprocess/{load_save_image_main.cpp => image_converter_main.cpp} (93%) rename src/mlpack/tests/main_tests/{load_save_image_test.cpp => image_converter_test.cpp} (93%) diff --git a/src/mlpack/methods/preprocess/CMakeLists.txt b/src/mlpack/methods/preprocess/CMakeLists.txt index 39810a1c87..48d0378614 100644 --- a/src/mlpack/methods/preprocess/CMakeLists.txt +++ b/src/mlpack/methods/preprocess/CMakeLists.txt @@ -43,7 +43,7 @@ add_python_binding(preprocess_scale) add_markdown_docs(preprocess_scale "cli;python" "preprocessing") if (NOT HAS_STB) - add_cli_executable(load_save_image) - add_python_binding(load_save_image) - add_markdown_docs(load_save_image "cli;python" "preprocessing") + add_cli_executable(image_converter) + add_python_binding(limage_converter) + add_markdown_docs(image_converter "cli;python" "preprocessing") endif () \ No newline at end of file diff --git a/src/mlpack/methods/preprocess/load_save_image_main.cpp b/src/mlpack/methods/preprocess/image_converter_main.cpp similarity index 93% rename from src/mlpack/methods/preprocess/load_save_image_main.cpp rename to src/mlpack/methods/preprocess/image_converter_main.cpp index 8673861e42..d5ee142c4d 100644 --- a/src/mlpack/methods/preprocess/load_save_image_main.cpp +++ b/src/mlpack/methods/preprocess/image_converter_main.cpp @@ -1,5 +1,5 @@ /** - * @file load_save_image_main.cpp + * @file image_converter_main.cpp * @author Jeffin Sam * * A CLI executable to load and save a image dataset. @@ -21,7 +21,7 @@ using namespace std; using namespace mlpack::data; -PROGRAM_INFO("Load Save Image", +PROGRAM_INFO("Image Converter", // Short description. "A utility to load an image or set of images into a single dataset that" "can then be used by other mlpack methods and utilities. This can also" @@ -39,17 +39,17 @@ PROGRAM_INFO("Load Save Image", PRINT_PARAM_STRING("dataset") + " and " + PRINT_PARAM_STRING("save") + " as an parameter. An example to load an image : " + "\n\n" + - PRINT_CALL("load_save_image", "input", "X", "height", 256, "width", 256, + PRINT_CALL("image_converter", "input", "X", "height", 256, "width", 256, "channel", 3, "output", "Y") + "\n\n" + " An example to save an image is :" + "\n\n" + - PRINT_CALL("load_save_image", "input", "X", "height", 256, "width", 256, + PRINT_CALL("image_converter", "input", "X", "height", 256, "width", 256, "channel", 3, "dataset", "Y", "save", true) + "\n\n" + " An example to load an image and also flipping it while loading is :" + "\n\n" + - PRINT_CALL("load_save_image", "input", "X", "height", 256, "width", 256, + PRINT_CALL("image_converter", "input", "X", "height", 256, "width", 256, "channel", 3, "output", "Y", "transpose", true), SEE_ALSO("@preprocess_binarize", "#preprocess_binarize"), SEE_ALSO("@preprocess_describe", "#preprocess_describe"), @@ -77,8 +77,6 @@ PARAM_MATRIX_IN("dataset", "Input matrix to save as images.", "I"); PARAM_MODEL_IN(ImageInfo, "input_model", "Input Image Info model.", "m"); PARAM_MODEL_OUT(ImageInfo, "output_model", "Output Image Info model.", "M"); -#ifdef HAS_STB // Compile this only if stb is present. - static void mlpackMain() { // Parse command line options. @@ -138,8 +136,4 @@ static void mlpackMain() if (CLI::HasParam("output_model")) CLI::GetParam ("output_model") = info; } -#else -static void mlpackMain() {} - -#endif // HAS_STB. diff --git a/src/mlpack/tests/CMakeLists.txt b/src/mlpack/tests/CMakeLists.txt index c23405426c..77e4fcc2e4 100644 --- a/src/mlpack/tests/CMakeLists.txt +++ b/src/mlpack/tests/CMakeLists.txt @@ -137,7 +137,7 @@ add_executable(mlpack_test main_tests/kfn_test.cpp main_tests/knn_test.cpp main_tests/linear_regression_test.cpp - main_tests/load_save_image_test.cpp + main_tests/image_converter_test.cpp main_tests/linear_svm_test.cpp main_tests/logistic_regression_test.cpp main_tests/local_coordinate_coding_test.cpp diff --git a/src/mlpack/tests/main_tests/load_save_image_test.cpp b/src/mlpack/tests/main_tests/image_converter_test.cpp similarity index 93% rename from src/mlpack/tests/main_tests/load_save_image_test.cpp rename to src/mlpack/tests/main_tests/image_converter_test.cpp index 016b3424e3..bd34831c72 100644 --- a/src/mlpack/tests/main_tests/load_save_image_test.cpp +++ b/src/mlpack/tests/main_tests/image_converter_test.cpp @@ -12,10 +12,10 @@ #define BINDING_TYPE BINDING_TYPE_TEST #include -static const std::string testName = "LoadSaveImage"; +static const std::string testName = "ImageConverter"; #include -#include +#include #include "test_helper.hpp" #include @@ -23,25 +23,27 @@ static const std::string testName = "LoadSaveImage"; using namespace mlpack; -struct LoadSaveImageTestFixture +struct ImageConverterTestFixture { public: - LoadSaveImageTestFixture() + ImageConverterTestFixture() { // Cache in the options for this program. CLI::RestoreSettings(testName); } - ~LoadSaveImageTestFixture() + ~ImageConverterTestFixture() { // Clear the settings. + remove("test_image777.png"); + remove("test_image999.png"); bindings::tests::CleanMemory(); CLI::ClearSettings(); } }; -BOOST_FIXTURE_TEST_SUITE(LoadSaveImageMainTest, - LoadSaveImageTestFixture); +BOOST_FIXTURE_TEST_SUITE(ImageConverterMainTest, + ImageConverterTestFixture); BOOST_AUTO_TEST_CASE(LoadImageTest) { @@ -197,7 +199,7 @@ BOOST_AUTO_TEST_CASE(InvalidChannelTest) /** * Check for invalid input values. */ -BOOST_AUTO_TEST_CASE(EmptyinputTest) +BOOST_AUTO_TEST_CASE(EmptyInputTest) { SetInputParam>("input", {}); SetInputParam("height", 50); From b268531dafb7992ba6dd66ae499d22074985e3b0 Mon Sep 17 00:00:00 2001 From: jeffin143 Date: Sun, 15 Mar 2020 02:40:29 +0530 Subject: [PATCH 051/111] Style-fixes for New policy --- src/mlpack/core/data/string_encoding.hpp | 7 +- src/mlpack/core/data/string_encoding_impl.hpp | 3 +- .../bag_of_words_encoding_policy.hpp | 95 +++++------ .../dictionary_encoding_policy.hpp | 58 +++---- .../tf_idf_encoding_policy.hpp | 149 +++++++++--------- src/mlpack/tests/string_encoding_test.cpp | 67 ++++---- 6 files changed, 190 insertions(+), 189 deletions(-) diff --git a/src/mlpack/core/data/string_encoding.hpp b/src/mlpack/core/data/string_encoding.hpp index 1457fa92c7..ac9d67c366 100644 --- a/src/mlpack/core/data/string_encoding.hpp +++ b/src/mlpack/core/data/string_encoding.hpp @@ -153,9 +153,7 @@ class StringEncoding * 2. IsTokenEmpty() that accepts a token and returns true if the given * token is empty. */ - template + template void EncodeHelper(const std::vector& input, OutputType& output, const TokenizerType& tokenizer, @@ -182,8 +180,7 @@ class StringEncoding * 2. IsTokenEmpty() that accepts a token and returns true if the given * token is empty. */ - template + template void EncodeHelper(const std::vector& input, std::vector>& output, const TokenizerType& tokenizer, diff --git a/src/mlpack/core/data/string_encoding_impl.hpp b/src/mlpack/core/data/string_encoding_impl.hpp index b233fba6e5..16a533245b 100644 --- a/src/mlpack/core/data/string_encoding_impl.hpp +++ b/src/mlpack/core/data/string_encoding_impl.hpp @@ -152,8 +152,7 @@ EncodeHelper(const std::vector& input, } template -template +template void StringEncoding:: EncodeHelper(const std::vector& input, std::vector>& output, diff --git a/src/mlpack/core/data/string_encoding_policies/bag_of_words_encoding_policy.hpp b/src/mlpack/core/data/string_encoding_policies/bag_of_words_encoding_policy.hpp index 01e113b64d..a2225aafd4 100644 --- a/src/mlpack/core/data/string_encoding_policies/bag_of_words_encoding_policy.hpp +++ b/src/mlpack/core/data/string_encoding_policies/bag_of_words_encoding_policy.hpp @@ -18,6 +18,7 @@ namespace mlpack { namespace data { + /** * Definition of the BagOfWordsEncodingPolicy class. * @@ -30,15 +31,15 @@ class BagOfWordsEncodingPolicy { public: /** - * The function initializes the output matrix. - * - * @param output Output matrix to store the encoded results (sp_mat or mat). - * @param datasetSize The number of strings in the input dataset. - * @param maxNumTokens The maximum number of tokens in the strings of the - input dataset (not used). - * @param dictionarySize The size of the dictionary. - * @tparam MatType The type of output matrix. - */ + * The function initializes the output matrix. + * + * @param output Output matrix to store the encoded results (sp_mat or mat). + * @param datasetSize The number of strings in the input dataset. + * @param maxNumTokens The maximum number of tokens in the strings of the + input dataset (not used). + * @param dictionarySize The size of the dictionary. + * @tparam MatType The type of output matrix. + */ template static void InitMatrix(MatType& output, size_t datasetSize, @@ -49,64 +50,64 @@ class BagOfWordsEncodingPolicy } /** - * The function initializes the output matrix. - * Overloaded function to store result in vector> - * - * @param output Output matrix to store the encoded results. - * @param datasetSize The number of strings in the input dataset. - * @param maxNumTokens The maximum number of tokens in the strings of the - input dataset. - * @param dictionarySize The size of the dictionary (not used). - * @tparam OutputType The type of output vector. - */ + * The function initializes the output matrix. + * Overloaded function to store result in vector> + * + * @param output Output matrix to store the encoded results. + * @param datasetSize The number of strings in the input dataset. + * @param maxNumTokens The maximum number of tokens in the strings of the + input dataset. + * @param dictionarySize The size of the dictionary (not used). + * @tparam OutputType The type of output vector. + */ template static void InitMatrix(std::vector >& output, size_t datasetSize, size_t /*maxNumTokens*/, size_t dictionarySize) { - output.resize(datasetSize, std::vector (dictionarySize, 0)); + output.resize(datasetSize, std::vector(dictionarySize, 0)); } /** - * The function performs the bag of words encoding algorithm i.e. it writes - * the encoded token to the ouput. - * - * @param output Output matrix to store the encoded results (sp_mat or mat). - * @param value The encoded token. - * @param row The row number at which the encoding is performed. - * @param col The row token number at which the encoding is performed. - * @tparam MatType The type of output matrix. - */ + * The function performs the bag of words encoding algorithm i.e. it writes + * the encoded token to the output. + * + * @param output Output matrix to store the encoded results (sp_mat or mat). + * @param value The encoded token. + * @param row The row number at which the encoding is performed. + * @param col The row token number at which the encoding is performed. + * @tparam MatType The type of output matrix. + */ template static void Encode(MatType& output, size_t value, size_t row, size_t /*col*/) { - // Important since Mapping of words,Dcitionary Encoding starts from 1, + // Important since Mapping of words, Dictionary Encoding starts from 1, // whereas allowed column value is 0. output(row, value - 1) = 1; } /** - * The function performs the bag of words encoding algorithm i.e. it writes - * the encoded token to the ouput. - * Overload function to accepted vector> as output type. - * - * @param output Output matrix to store the encoded results. - * @param value The encoded token. - * @param row The row number at which the encoding is performed. - * @param col The row token number at which the encoding is performed. - * @tparam OutputType The type of output vector. - */ + * The function performs the bag of words encoding algorithm i.e. it writes + * the encoded token to the output. + * Overload function to accepted vector> as output type. + * + * @param output Output matrix to store the encoded results. + * @param value The encoded token. + * @param row The row number at which the encoding is performed. + * @param col The row token number at which the encoding is performed. + * @tparam OutputType The type of output vector. + */ template static void Encode(std::vector >& output, size_t value, size_t row, size_t /*col*/) { - // Important since Mapping of words,Dcitionary Encoding starts from 1, + // Important since Mapping of words in Dictionary Encoding starts from 1, // whereas allowed column value is 0. output[row][value - 1] = 1; } @@ -121,12 +122,12 @@ class BagOfWordsEncodingPolicy } /** - * Empty function, Important for tf-idf encoding policy - * - * @param row The row number at which the encoding is performed. - * @param numToken The count of token parsed till now. - * @param value The encoded token. - */ + * Empty function, Important for tf-idf encoding policy. + * + * @param row The row number at which the encoding is performed. + * @param numToken The count of token parsed till now. + * @param value The encoded token. + */ static void PreprocessToken(size_t /*row*/, size_t /*numTokens*/, size_t /*value*/) { } diff --git a/src/mlpack/core/data/string_encoding_policies/dictionary_encoding_policy.hpp b/src/mlpack/core/data/string_encoding_policies/dictionary_encoding_policy.hpp index f1eb26313b..3af0bf8277 100644 --- a/src/mlpack/core/data/string_encoding_policies/dictionary_encoding_policy.hpp +++ b/src/mlpack/core/data/string_encoding_policies/dictionary_encoding_policy.hpp @@ -31,17 +31,17 @@ class DictionaryEncodingPolicy { public: /** - * The function initializes the output matrix. - * - * @tparam MatType The output matrix type. - * - * @param output Output matrix to store the encoded results (sp_mat or mat). - * @param datasetSize The number of strings in the input dataset. - * @param maxNumTokens The maximum number of tokens in the strings of the - input dataset. - * @param dictionarySize The size of the dictionary (not used). - * @tparam MatType The type of output matrix. - */ + * The function initializes the output matrix. + * + * @tparam MatType The output matrix type. + * + * @param output Output matrix to store the encoded results (sp_mat or mat). + * @param datasetSize The number of strings in the input dataset. + * @param maxNumTokens The maximum number of tokens in the strings of the + input dataset. + * @param dictionarySize The size of the dictionary (not used). + * @tparam MatType The type of output matrix. + */ template static void InitMatrix(MatType& output, const size_t datasetSize, @@ -52,17 +52,17 @@ class DictionaryEncodingPolicy } /** - * The function performs the dictionary encoding algorithm i.e. it writes - * the encoded token to the ouput. - * - * @tparam MatType The output matrix type. - * - * @param output Output matrix to store the encoded results (sp_mat or mat). - * @param value The encoded token. - * @param row The row number at which the encoding is performed. - * @param col The token index in the row. - * @tparam MatType The type of output matrix. - */ + * The function performs the dictionary encoding algorithm i.e. it writes + * the encoded token to the ouput. + * + * @tparam MatType The output matrix type. + * + * @param output Output matrix to store the encoded results (sp_mat or mat). + * @param value The encoded token. + * @param row The row number at which the encoding is performed. + * @param col The token index in the row. + * @tparam MatType The type of output matrix. + */ template static void Encode(MatType& output, const size_t value, @@ -88,12 +88,14 @@ class DictionaryEncodingPolicy } /** - * Empty function, Important for tf-idf encoding policy - * - * @param row The row number at which the encoding is performed. - * @param numToken The count of token parsed till now. - * @param value The encoded token. - */ + * Empty function, Important for tf-idf encoding policy. + * This function has been only used in tf-idf encoding policy and has no + * relevance here. + * + * @param row The row number at which the encoding is performed. + * @param numToken The count of token parsed till now. + * @param value The encoded token. + */ static void PreprocessToken(size_t /*row*/, size_t /*numTokens*/, size_t /*value*/) {} diff --git a/src/mlpack/core/data/string_encoding_policies/tf_idf_encoding_policy.hpp b/src/mlpack/core/data/string_encoding_policies/tf_idf_encoding_policy.hpp index d1724ccd3d..8f80ab73d8 100644 --- a/src/mlpack/core/data/string_encoding_policies/tf_idf_encoding_policy.hpp +++ b/src/mlpack/core/data/string_encoding_policies/tf_idf_encoding_policy.hpp @@ -1,5 +1,5 @@ /** - * @file td_idf_encoding_policy.hpp + * @file tf_idf_encoding_policy.hpp * @author Jeffin Sam * * Definition of the TfIdfEncodingPolicy class. @@ -15,12 +15,14 @@ #include #include #include + namespace mlpack { namespace data { + /** * Definition of the TfIdfEncodingPolicy class. * - * Tf-idf is weighing scheme that stands for term-frequency multiplied by + * Tf-idf is a weighting scheme that stands for term-frequency multiplied by * inverse document-frequency. * The goal of using tf-idf is to scale down the impact of tokens that occur * very frequently in a given corpus while using the type of Term-Frequency @@ -35,15 +37,15 @@ class TfIdfEncodingPolicy { public: /** - * Enum class used to identify the type of tf encoding - * - * Follwing are the defination of the types - * BINARY : binary weighting scheme (0,1) - * RAW_COUNT : raw count weighting scheme (count of token for every row) - * TERM_FREQUENCY : term frequency weighting scheme (count / length(row)) - * SUBLINEAR_TF : logarthimic weighting scheme (log(tf) + 1) - * - */ + * Enum class used to identify the type of tf encoding. + * + * Following are the type definitions: + * BINARY : binary weighting scheme (0,1) + * RAW_COUNT : raw count weighting scheme (count of token for every row) + * TERM_FREQUENCY : term frequency weighting scheme (count / length(row)) + * SUBLINEAR_TF : logarithmic weighting scheme (log(tf) + 1) + * + */ enum class TfTypes { RAW_COUNT, @@ -53,33 +55,34 @@ class TfIdfEncodingPolicy }; /** - * A constructor for the class which is use to set the type of term frequency - * and also the value for somoothIdf. - * - * @param tfType The type of term frequency, The avialbale option are - * RAW_COUNT : The count of a specific token - * BINARY : 1 if token occurs in document and 0 otherwise; - * TERM_FREQUENCY : Raw_count ÷ (number of words in document) - * SUBLINEAR_TF : log(Raw_Count) + 1 - * - * @param smoothIdf Used to indicate whether to use smooth idf or not. - */ + * A constructor for the class which is use to set the type of term frequency + * and also the value for smoothIdf. + * + * @param tfType The type of term frequency, The avialbale option are + * RAW_COUNT : The count of a specific token + * BINARY : 1 if token occurs in document and 0 otherwise; + * TERM_FREQUENCY : Raw_count ÷ (number of words in document) + * SUBLINEAR_TF : log(Raw_Count) + 1 + * + * @param smoothIdf Used to indicate whether to use smooth idf or not. + */ TfIdfEncodingPolicy(TfTypes tfType = TfTypes::RAW_COUNT, bool smoothIdf = true) : tfType(tfType), smoothIdf(smoothIdf) { } + /** - * The function initializes the output matrix. - * - * @param output Output matrix to store the encoded results (sp_mat or mat). - * @param datasetSize The number of strings in the input dataset. - * @param maxNumTokens The maximum number of tokens in the strings of the + * The function initializes the output matrix. + * + * @param output Output matrix to store the encoded results (sp_mat or mat). + * @param datasetSize The number of strings in the input dataset. + * @param maxNumTokens The maximum number of tokens in the strings of the input dataset. - * @param dictionarySize The size of the dictionary (not used). - * @tparam MatType The type of output matrix. - */ + * @param dictionarySize The size of the dictionary (not used). + * @tparam MatType The type of output matrix. + */ template static void InitMatrix(MatType& output, size_t datasetSize, @@ -90,43 +93,41 @@ class TfIdfEncodingPolicy } /** - * The function initializes the output matrix. - * Overloaded function to store result in vector> - * - * @param output Output matrix to store the encoded results. - * @param datasetSize The number of strings in the input dataset. - * @param maxNumTokens The maximum number of tokens in the strings of the - input dataset. - * @param dictionarySize The size of the dictionary (not used). - * @tparam OutputType The type of output vector. - */ + * The function initializes the output matrix. + * Overloaded function to store result in vector> + * + * @param output Output matrix to store the encoded results. + * @param datasetSize The number of strings in the input dataset. + * @param maxNumTokens The maximum number of tokens in the strings of the + input dataset. + * @param dictionarySize The size of the dictionary (not used). + * @tparam OutputType The type of output vector. + */ template static void InitMatrix(std::vector >& output, size_t datasetSize, size_t /*maxNumTokens*/, size_t dictionarySize) { - output.resize(datasetSize, std::vector (dictionarySize, 0)); + output.resize(datasetSize, std::vector(dictionarySize, 0)); } /** - * The function performs the TfIdf encoding algorithm i.e. it writes - * the encoded token to the ouput. - * - * @param output Output matrix to store the encoded results (sp_mat or mat). - * @param value The encoded token. - * @param row The row number at which the encoding is performed. - * @param col The row token number at which the encoding is performed. - * @tparam MatType The type of output matrix. - */ + * The function performs the TfIdf encoding algorithm i.e. it writes + * the encoded token to the output. + * + * @param output Output matrix to store the encoded results (sp_mat or mat). + * @param value The encoded token. + * @param row The row number at which the encoding is performed. + * @param col The row token number at which the encoding is performed. + * @tparam MatType The type of output matrix. + */ template void Encode(MatType& output, size_t value, size_t row, size_t /*col*/) { - // Important since Mapping of words,Dcitionary Encoding starts from 1, - // whereas allowed column value is 0. double idf, tf; if (smoothIdf) idf = std::log((output.n_rows + 1) / (1 + idfdict[value - 1])) + 1; @@ -142,29 +143,27 @@ class TfIdfEncodingPolicy else tf = tokenCount[row][value - 1]; - output(row, value-1) = tf * idf; + output(row, value - 1) = tf * idf; } /** - * The function performs the TfIdf encoding algorithm i.e. it writes - * the encoded token to the ouput. - * Overload function to accepted vector> as output type. - * - * @param output Output matrix to store the encoded results. - * @param value The encoded token. - * @param row The row number at which the encoding is performed. - * @param col The row token number at which the encoding is performed. - * @tparam OutputType The type of output vector. - * @tparam OutputType The type of output vector. - */ + * The function performs the TfIdf encoding algorithm i.e. it writes + * the encoded token to the output. + * Overload function to accepted vector> as output type. + * + * @param output Output matrix to store the encoded results. + * @param value The encoded token. + * @param row The row number at which the encoding is performed. + * @param col The row token number at which the encoding is performed. + * @tparam OutputType The type of output vector. + * @tparam OutputType The type of output vector. + */ template void Encode(std::vector >& output, size_t value, size_t row, size_t /*col*/) { - // Important since Mapping of words,Dcitionary Encoding starts from 1, - // whereas allowed column value is 0. double idf, tf; if (smoothIdf) idf = std::log((output.size() + 1) / (1 + idfdict[value - 1])) + 1; @@ -180,27 +179,27 @@ class TfIdfEncodingPolicy else tf = tokenCount[row][value - 1]; - output[row][value-1] = tf * idf; + output[row][value - 1] = tf * idf; } /** * Serialize the class to the given archive. */ template - void serialize(Archive& ar , const unsigned int /* version */) + void serialize(Archive& ar, const unsigned int /* version */) { ar & BOOST_SERIALIZATION_NVP(tfType); ar & BOOST_SERIALIZATION_NVP(smoothIdf); } /* - * The function is used to create the datastrcutre will be important to find - * out idfvalue of words, and then wrtiting the output based on their count. - * - * @param row The row number at which the encoding is performed. - * @param numToken The count of token parsed till now. - * @param value The encoded token. - */ + * The function is used to create the datastrcutre will be important to find + * out idfvalue of words, and then wrtiting the output based on their count. + * + * @param row The row number at which the encoding is performed. + * @param numToken The count of token parsed till now. + * @param value The encoded token. + */ void PreprocessToken(size_t row, size_t /*numTokens*/, size_t value) diff --git a/src/mlpack/tests/string_encoding_test.cpp b/src/mlpack/tests/string_encoding_test.cpp index 866acb54ba..8e4b8107b9 100644 --- a/src/mlpack/tests/string_encoding_test.cpp +++ b/src/mlpack/tests/string_encoding_test.cpp @@ -224,8 +224,8 @@ BOOST_AUTO_TEST_CASE(SplitByAnyOfTokenizerUnicodeTest) } /** -* Test the CharExtract tokenizer. -*/ + * Test the CharExtract tokenizer. + */ BOOST_AUTO_TEST_CASE(DictionaryEncodingIndividualCharactersTest) { vector input = { @@ -521,6 +521,9 @@ BOOST_AUTO_TEST_CASE(CharExtractDictionaryEncodingSerialization) CheckMatrices(output, xmlOutput, textOutput, binaryOutput); } +/** + * Test for Bag of Words encoding algorithm. + */ BOOST_AUTO_TEST_CASE(BagOfWordsEncodingTest) { using DictionaryType = StringEncodingDictionary; @@ -532,12 +535,12 @@ BOOST_AUTO_TEST_CASE(BagOfWordsEncodingTest) encoder.Encode(stringEncodingInput, output, tokenizer); const DictionaryType& dictionary = encoder.Dictionary(); - // Checking that everything is mapped to different numbers + // Checking that everything is mapped to different numbers. std::unordered_map keysCount; for (auto& keyValue : dictionary.Mapping()) { keysCount[keyValue.second]++; - // Every token should be mapped only once + // Every token should be mapped only once. BOOST_REQUIRE_EQUAL(keysCount[keyValue.second], 1); } arma::mat expected = { @@ -552,8 +555,8 @@ BOOST_AUTO_TEST_CASE(BagOfWordsEncodingTest) } /** -* Bag of Words encoding algorithm output saved in a vector. -*/ + * Bag of Words encoding algorithm output saved in a vector. + */ BOOST_AUTO_TEST_CASE(VectorBagOfWordsEncodingTest) { using DictionaryType = StringEncodingDictionary; @@ -567,12 +570,12 @@ BOOST_AUTO_TEST_CASE(VectorBagOfWordsEncodingTest) const DictionaryType& dictionary = encoder.Dictionary(); - // Checking that everything is mapped to different numbers + // Checking that everything is mapped to different numbers. std::unordered_map keysCount; for (auto& keyValue : dictionary.Mapping()) { keysCount[keyValue.second]++; - // Every token should be mapped only once + // Every token should be mapped only once. BOOST_REQUIRE_EQUAL(keysCount[keyValue.second], 1); } @@ -589,8 +592,8 @@ BOOST_AUTO_TEST_CASE(VectorBagOfWordsEncodingTest) } /** -* Test Bag of Words encoding for characters. -*/ + * Test Bag of Words encoding for characters. + */ BOOST_AUTO_TEST_CASE(BagOfWordsEncodingIndividualCharactersTest) { vector input = { @@ -602,7 +605,7 @@ BOOST_AUTO_TEST_CASE(BagOfWordsEncodingIndividualCharactersTest) arma::mat output; BagOfWordsEncoding encoder; - // Passing a empty string to encode characters + // Passing a empty string to encode characters. encoder.Encode(input, output, CharExtract()); arma::mat target = { { 1, 1, 1, 0, 0 }, @@ -628,7 +631,7 @@ BOOST_AUTO_TEST_CASE(VectorBagOfWordsEncodingIndividualCharactersTest) vector> output; BagOfWordsEncoding encoder; - // Passing a empty string to encode characters + // Passing a empty string to encode characters. encoder.Encode(input, output, CharExtract()); vector> expected = { @@ -642,7 +645,7 @@ BOOST_AUTO_TEST_CASE(VectorBagOfWordsEncodingIndividualCharactersTest) /** * Test the Tf-Idf Encoding using rawcount type and smoothidf as true, - * which is the deafult values used for algorithim. + * which is the default value used for algorithm. */ BOOST_AUTO_TEST_CASE(RawCountSmoothIdfEncodingTest) { @@ -655,12 +658,12 @@ BOOST_AUTO_TEST_CASE(RawCountSmoothIdfEncodingTest) encoder.Encode(stringEncodingInput, output, tokenizer); const DictionaryType& dictionary = encoder.Dictionary(); - // Checking that everything is mapped to different numbers + // Checking that everything is mapped to different numbers. std::unordered_map keysCount; for (auto& keyValue : dictionary.Mapping()) { keysCount[keyValue.second]++; - // Every token should be mapped only once + // Every token should be mapped only once. BOOST_REQUIRE_EQUAL(keysCount[keyValue.second], 1); } arma::mat expected = { @@ -703,12 +706,12 @@ BOOST_AUTO_TEST_CASE(VectorRawCountSmoothIdfEncodingTest) const DictionaryType& dictionary = encoder.Dictionary(); - // Checking that everything is mapped to different numbers + // Checking that everything is mapped to different numbers. std::unordered_map keysCount; for (auto& keyValue : dictionary.Mapping()) { keysCount[keyValue.second]++; - // Every token should be mapped only once + // Every token should be mapped only once. BOOST_REQUIRE_EQUAL(keysCount[keyValue.second], 1); } @@ -752,7 +755,7 @@ BOOST_AUTO_TEST_CASE(RawCountSmoothIdfEncodingIndividualCharactersTest) arma::mat output; TfIdfEncoding encoder; - // Passing a empty string to encode charactersrawcountsmoothidftrue + // Passing a empty string to encode characters. encoder.Encode(input, output, CharExtract()); arma::mat target = { { 1.2876820724517808, 2, 2.5753641449035616, 0, 0 }, @@ -777,7 +780,7 @@ BOOST_AUTO_TEST_CASE(VectorRawCountSmoothIdfEncodingIndividualCharactersTest) vector> output; TfIdfEncoding encoder; - // Passing a empty string to encode characters + // Passing a empty string to encode characters. encoder.Encode(input, output, CharExtract()); vector> expected = { { 1.2876820724517808, 2, 2.5753641449035616, 0, 0 }, @@ -805,12 +808,12 @@ BOOST_AUTO_TEST_CASE(TfIdfRawCountEncodingTest) const DictionaryType& dictionary = encoder.Dictionary(); - // Checking that everything is mapped to different numbers + // Checking that everything is mapped to different numbers. std::unordered_map keysCount; for (auto& keyValue : dictionary.Mapping()) { keysCount[keyValue.second]++; - // Every token should be mapped only once + // Every token should be mapped only once. BOOST_REQUIRE_EQUAL(keysCount[keyValue.second], 1); } @@ -854,12 +857,12 @@ BOOST_AUTO_TEST_CASE(VectorTfIdfRawCountEncodingTest) const DictionaryType& dictionary = encoder.Dictionary(); - // Checking that everything is mapped to different numbers + // Checking that everything is mapped to different numbers. std::unordered_map keysCount; for (auto& keyValue : dictionary.Mapping()) { keysCount[keyValue.second]++; - // Every token should be mapped only once + // Every token should be mapped only once. BOOST_REQUIRE_EQUAL(keysCount[keyValue.second], 1); } @@ -904,7 +907,7 @@ BOOST_AUTO_TEST_CASE(RawcountTfIdfEncodingIndividualCharactersTest) TfIdfEncoding encoder(TfIdfEncodingPolicy::TfTypes::RAW_COUNT, false); - // Passing a empty string to encode charactersrawcountsmoothidftrue + // Passing a empty string to encode charactersrawcountsmoothidftrue. encoder.Encode(input, output, CharExtract()); arma::mat target = { { 1.4054651081081644, 2, 2.8109302162163288, 0, 0 }, @@ -931,7 +934,7 @@ BOOST_AUTO_TEST_CASE(VectorRawcountEncodingIndividualCharactersTest) TfIdfEncoding encoder(TfIdfEncodingPolicy::TfTypes::RAW_COUNT, false); - // Passing a empty string to encode characters + // Passing a empty string to encode characters. encoder.Encode(input, output, CharExtract()); vector> expected = { { 1.4054651081081644, 2, 2.8109302162163288, 0, 0 }, @@ -959,7 +962,7 @@ BOOST_AUTO_TEST_CASE(BinarySmoothIdfEncodingIndividualCharactersTest) TfIdfEncoding encoder(TfIdfEncodingPolicy::TfTypes::BINARY, true); - // Passing a empty string to encode charactersrawcountsmoothidftrue + // Passing a empty string to encode charactersrawcountsmoothidftrue. encoder.Encode(input, output, CharExtract()); arma::mat target = { { 1.2876820724517808, 1, 1.2876820724517808, 0, 0 }, @@ -985,7 +988,7 @@ BOOST_AUTO_TEST_CASE(VectorBnarySmoothIdfEncodingIndividualCharactersTest) TfIdfEncoding encoder(TfIdfEncodingPolicy::TfTypes::BINARY, true); - // Passing a empty string to encode characters + // Passing a empty string to encode characters. encoder.Encode(input, output, CharExtract()); vector> expected = { { 1.2876820724517808, 1, 1.2876820724517808, 0, 0 }, @@ -1013,7 +1016,7 @@ BOOST_AUTO_TEST_CASE(BinaryTfIdfEncodingIndividualCharactersTest) TfIdfEncoding encoder(TfIdfEncodingPolicy::TfTypes::BINARY, false); - // Passing a empty string to encode charactersrawcountsmoothidftrue + // Passing a empty string to encode charactersrawcountsmoothidftrue. encoder.Encode(input, output, CharExtract()); arma::mat target = { { 1.4054651081081644, 1, 1.4054651081081644, 0, 0 }, @@ -1039,7 +1042,7 @@ BOOST_AUTO_TEST_CASE(SublinearSmoothIdfEncodingIndividualCharactersTest) TfIdfEncoding encoder(TfIdfEncodingPolicy::TfTypes::SUBLINEAR_TF, true); - // Passing a empty string to encode charactersrawcountsmoothidftrue + // Passing a empty string to encode charactersrawcountsmoothidftrue. encoder.Encode(input, output, CharExtract()); arma::mat target = { { 1.2876820724517808, 1.6931471805599454, 2.1802352704293200, 0, 0 }, @@ -1066,7 +1069,7 @@ BOOST_AUTO_TEST_CASE(SublinearTfIdfEncodingIndividualCharactersTest) TfIdfEncoding encoder(TfIdfEncodingPolicy::TfTypes::SUBLINEAR_TF, false); - // Passing a empty string to encode charactersrawcountsmoothidftrue + // Passing a empty string to encode charactersrawcountsmoothidftrue. encoder.Encode(input, output, CharExtract()); arma::mat target = { { 1.4054651081081644, 1.6931471805599454, 2.3796592851687173, 0, 0 }, @@ -1093,7 +1096,7 @@ BOOST_AUTO_TEST_CASE(TermFrequencySmoothIdfEncodingIndividualCharactersTest) TfIdfEncoding encoder(TfIdfEncodingPolicy::TfTypes::TERM_FREQUENCY, true); - // Passing a empty string to encode charactersrawcountsmoothidftrue + // Passing a empty string to encode charactersrawcountsmoothidftrue. encoder.Encode(input, output, CharExtract()); arma::mat target = { { 0.2575364144903562, 0.4, 0.5150728289807124, 0, 0 }, @@ -1120,7 +1123,7 @@ BOOST_AUTO_TEST_CASE(TermFrequencyTfIdfEncodingIndividualCharactersTest) TfIdfEncoding encoder(TfIdfEncodingPolicy::TfTypes::TERM_FREQUENCY, false); - // Passing a empty string to encode charactersrawcountsmoothidftrue + // Passing a empty string to encode charactersrawcountsmoothidftrue. encoder.Encode(input, output, CharExtract()); arma::mat target = { { 0.2810930216216329, 0.4, 0.5621860432432658, 0, 0 }, From 5a746abd70781c0e587b7cfa255de9fde3b85bd5 Mon Sep 17 00:00:00 2001 From: Mikhail Lozhnikov Date: Mon, 16 Mar 2020 18:43:10 +0300 Subject: [PATCH 052/111] Style fixes for the following files: - src/mlpack/core/data/string_encoding_impl.hpp - src/mlpack/core/data/string_encoding_policies/bag_of_words_encoding_policy.hpp - src/mlpack/core/data/string_encoding_policies/dictionary_encoding_policy.hpp - src/mlpack/core/data/string_encoding_policies/tf_idf_encoding_policy.hpp --- src/mlpack/core/data/string_encoding_impl.hpp | 1 + .../bag_of_words_encoding_policy.hpp | 92 ++--- .../dictionary_encoding_policy.hpp | 20 +- .../tf_idf_encoding_policy.hpp | 331 +++++++++++------- 4 files changed, 272 insertions(+), 172 deletions(-) diff --git a/src/mlpack/core/data/string_encoding_impl.hpp b/src/mlpack/core/data/string_encoding_impl.hpp index 16a533245b..2194486f23 100644 --- a/src/mlpack/core/data/string_encoding_impl.hpp +++ b/src/mlpack/core/data/string_encoding_impl.hpp @@ -126,6 +126,7 @@ EncodeHelper(const std::vector& input, { if (!dictionary.HasToken(token)) dictionary.AddToken(std::move(token)); + policy.PreprocessToken(i, numTokens, dictionary.Value(token)); token = tokenizer(strView); diff --git a/src/mlpack/core/data/string_encoding_policies/bag_of_words_encoding_policy.hpp b/src/mlpack/core/data/string_encoding_policies/bag_of_words_encoding_policy.hpp index a2225aafd4..4ca76b4a08 100644 --- a/src/mlpack/core/data/string_encoding_policies/bag_of_words_encoding_policy.hpp +++ b/src/mlpack/core/data/string_encoding_policies/bag_of_words_encoding_policy.hpp @@ -9,8 +9,8 @@ * 3-clause BSD license along with mlpack. If not, see * http://www.opensource.org/licenses/BSD-3-Clause for more information. */ -#ifndef MLPACK_CORE_DATA_ENCODING_POLICIES_BAG_OF_WORDS_ENCODING_POLICY_HPP -#define MLPACK_CORE_DATA_ENCODING_POLICIES_BAG_OF_WORDS_ENCODING_POLICY_HPP +#ifndef MLPACK_CORE_DATA_STRING_ENCODING_POLICIES_BAG_OF_WORDS_ENCODING_POLICY_HPP +#define MLPACK_CORE_DATA_STRING_ENCODING_POLICIES_BAG_OF_WORDS_ENCODING_POLICY_HPP #include #include @@ -22,10 +22,12 @@ namespace data { /** * Definition of the BagOfWordsEncodingPolicy class. * - * BagOfWords is used as a helper class for StringEncoding. - * The encoder creates a vector of all the unique token and then assigns - * 1 if the token is present in the document, 0 if not present. The tokens - * are labeled in the order of their occurrence in the input dataset. + * BagOfWords is used as a helper class for StringEncoding. The encoder maps + * each dataset item to a vector of size N, where N is equal to the total number + * of tokens. If an item of the dataset has the i-th token, then the i-th + * coordinate of the corresponding vector is equal to 1, otherwise it's equal to + * zero. The order in which the tokens are labeled is defined by the dictionary + * used by the StringEncoding class. */ class BagOfWordsEncodingPolicy { @@ -33,18 +35,19 @@ class BagOfWordsEncodingPolicy /** * The function initializes the output matrix. * + * @tparam MatType The output matrix type. + * * @param output Output matrix to store the encoded results (sp_mat or mat). * @param datasetSize The number of strings in the input dataset. * @param maxNumTokens The maximum number of tokens in the strings of the input dataset (not used). * @param dictionarySize The size of the dictionary. - * @tparam MatType The type of output matrix. */ template static void InitMatrix(MatType& output, - size_t datasetSize, - size_t /*maxNumTokens*/, - size_t dictionarySize) + const size_t datasetSize, + const size_t /* maxNumTokens */, + const size_t dictionarySize) { output.zeros(datasetSize, dictionarySize); } @@ -53,18 +56,19 @@ class BagOfWordsEncodingPolicy * The function initializes the output matrix. * Overloaded function to store result in vector> * + * @tparam OutputType Type of the output vector. + * * @param output Output matrix to store the encoded results. * @param datasetSize The number of strings in the input dataset. * @param maxNumTokens The maximum number of tokens in the strings of the - input dataset. - * @param dictionarySize The size of the dictionary (not used). - * @tparam OutputType The type of output vector. + input dataset (not used). + * @param dictionarySize The size of the dictionary. */ template - static void InitMatrix(std::vector >& output, - size_t datasetSize, - size_t /*maxNumTokens*/, - size_t dictionarySize) + static void InitMatrix(std::vector>& output, + const size_t datasetSize, + const size_t /* maxNumTokens */, + const size_t dictionarySize) { output.resize(datasetSize, std::vector(dictionarySize, 0)); } @@ -73,20 +77,20 @@ class BagOfWordsEncodingPolicy * The function performs the bag of words encoding algorithm i.e. it writes * the encoded token to the output. * + * @tparam MatType The output matrix type. + * * @param output Output matrix to store the encoded results (sp_mat or mat). * @param value The encoded token. * @param row The row number at which the encoding is performed. - * @param col The row token number at which the encoding is performed. - * @tparam MatType The type of output matrix. + * @param col The token index in the row. */ template static void Encode(MatType& output, - size_t value, - size_t row, - size_t /*col*/) + const size_t value, + const size_t row, + const size_t /* col */) { - // Important since Mapping of words, Dictionary Encoding starts from 1, - // whereas allowed column value is 0. + // The labels are assigned sequentially starting from one. output(row, value - 1) = 1; } @@ -102,16 +106,27 @@ class BagOfWordsEncodingPolicy * @tparam OutputType The type of output vector. */ template - static void Encode(std::vector >& output, - size_t value, - size_t row, - size_t /*col*/) + static void Encode(std::vector>& output, + const size_t value, + const size_t row, + const size_t /* col */) { - // Important since Mapping of words in Dictionary Encoding starts from 1, - // whereas allowed column value is 0. + // The labels are assigned sequentially starting from one. output[row][value - 1] = 1; } + /** + * The function is not used by the bag of words encoding policy. + * + * @param row The row number at which the encoding is performed. + * @param col The token sequence number in the row. + * @param value The encoded token. + */ + static void PreprocessToken(size_t /* row */, + size_t /* col */, + size_t /* value */) + { } + /** * Serialize the class to the given archive. */ @@ -120,19 +135,14 @@ class BagOfWordsEncodingPolicy { // Nothing to serialize. } - - /** - * Empty function, Important for tf-idf encoding policy. - * - * @param row The row number at which the encoding is performed. - * @param numToken The count of token parsed till now. - * @param value The encoded token. - */ - static void PreprocessToken(size_t /*row*/, - size_t /*numTokens*/, - size_t /*value*/) { } }; +/** + * A convenient alias for the StringEncoding class with BagOfWordsEncodingPolicy + * and the default dictionary for the given token type. + * + * @tparam TokenType Type of the tokens. + */ template using BagOfWordsEncoding = StringEncoding>; diff --git a/src/mlpack/core/data/string_encoding_policies/dictionary_encoding_policy.hpp b/src/mlpack/core/data/string_encoding_policies/dictionary_encoding_policy.hpp index 3af0bf8277..e3de62e526 100644 --- a/src/mlpack/core/data/string_encoding_policies/dictionary_encoding_policy.hpp +++ b/src/mlpack/core/data/string_encoding_policies/dictionary_encoding_policy.hpp @@ -40,13 +40,12 @@ class DictionaryEncodingPolicy * @param maxNumTokens The maximum number of tokens in the strings of the input dataset. * @param dictionarySize The size of the dictionary (not used). - * @tparam MatType The type of output matrix. */ template static void InitMatrix(MatType& output, const size_t datasetSize, const size_t maxNumTokens, - const size_t /*dictionarySize*/) + const size_t /* dictionarySize */) { output.zeros(datasetSize, maxNumTokens); } @@ -61,7 +60,6 @@ class DictionaryEncodingPolicy * @param value The encoded token. * @param row The row number at which the encoding is performed. * @param col The token index in the row. - * @tparam MatType The type of output matrix. */ template static void Encode(MatType& output, @@ -77,9 +75,10 @@ class DictionaryEncodingPolicy * the encoded token to the ouput. This is an overload function which saves * the result into the given vector to avoid padding. * + * @tparam OutputType Type of the output vector. + * * @param output Output vector to store the encoded results. * @param value The encoded token. - * @tparam OutputType The type of output vector. */ template static void Encode(std::vector& output, size_t value) @@ -88,17 +87,16 @@ class DictionaryEncodingPolicy } /** - * Empty function, Important for tf-idf encoding policy. - * This function has been only used in tf-idf encoding policy and has no - * relevance here. + * The function is not used by the dictionary encoding policy. * * @param row The row number at which the encoding is performed. - * @param numToken The count of token parsed till now. + * @param col The token sequence number in the row. * @param value The encoded token. */ - static void PreprocessToken(size_t /*row*/, - size_t /*numTokens*/, - size_t /*value*/) {} + static void PreprocessToken(const size_t /* row */, + const size_t /* col */, + const size_t /* value */) + { } /** * Serialize the class to the given archive. diff --git a/src/mlpack/core/data/string_encoding_policies/tf_idf_encoding_policy.hpp b/src/mlpack/core/data/string_encoding_policies/tf_idf_encoding_policy.hpp index 8f80ab73d8..ecba9f1a8d 100644 --- a/src/mlpack/core/data/string_encoding_policies/tf_idf_encoding_policy.hpp +++ b/src/mlpack/core/data/string_encoding_policies/tf_idf_encoding_policy.hpp @@ -9,8 +9,8 @@ * 3-clause BSD license along with mlpack. If not, see * http://www.opensource.org/licenses/BSD-3-Clause for more information. */ -#ifndef MLPACK_CORE_DATA_ENCODING_POLICIES_TF_IDF_ENCODING_POLICY_HPP -#define MLPACK_CORE_DATA_ENCODING_POLICIES_TF_IDF_ENCODING_POLICY_HPP +#ifndef MLPACK_CORE_DATA_STRING_ENCODING_POLICIES_TF_IDF_ENCODING_POLICY_HPP +#define MLPACK_CORE_DATA_STRING_ENCODING_POLICIES_TF_IDF_ENCODING_POLICY_HPP #include #include @@ -20,94 +20,98 @@ namespace mlpack { namespace data { /** - * Definition of the TfIdfEncodingPolicy class. + * Definition of the TfIdfEncodingPolicy class. TfIdfEncodingPolicy is used + * as a helper class for StringEncoding. * - * Tf-idf is a weighting scheme that stands for term-frequency multiplied by - * inverse document-frequency. - * The goal of using tf-idf is to scale down the impact of tokens that occur - * very frequently in a given corpus while using the type of Term-Frequency - * of a token and that are hence empirically less informative than features - * that occur in a small fraction of the training corpus. - * TfIdfEncodingPolicy is used as a helper class for StringEncoding. - * The encoder assigns a tf-idf number to each unique token and treat - * the dataset as categorical. The tokens are labeled in the order of their - * occurrence in the input dataset. + * Tf-idf is a weighting scheme that takes into account the importance of + * encoded tokens. The tf-idf statistics is equal to term frequency (tf) + * multiplied by inverse document frequency (idf). + * The encoder assigns the corresponding tf-idf value to each token. The order + * in which the tokens are labeled is defined by the dictionary used by the + * StringEncoding class. */ class TfIdfEncodingPolicy { public: /** - * Enum class used to identify the type of tf encoding. + * Enum class used to identify the type of term frequency encoding. * - * Following are the type definitions: - * BINARY : binary weighting scheme (0,1) - * RAW_COUNT : raw count weighting scheme (count of token for every row) - * TERM_FREQUENCY : term frequency weighting scheme (count / length(row)) - * SUBLINEAR_TF : logarithmic weighting scheme (log(tf) + 1) - * + * The present implementation supports the following types: + * BINARY Term frequency equals 1 if the row contains the encoded + * token and 0 otherwise. + * RAW_COUNT Term frequency equals the number of times when the encoded + * token occurs in the row. + * TERM_FREQUENCY Term frequency equals the number of times when the encoded + * token occurs in the row divided by the row size. + * SUBLINEAR_TF Term frequency equals \f$ 1 + log(rawCount), \f$ where + rawCount is equal to the number of times when the encoded + * token occurs in the row. */ enum class TfTypes { - RAW_COUNT, BINARY, - SUBLINEAR_TF, + RAW_COUNT, TERM_FREQUENCY, + SUBLINEAR_TF, }; /** * A constructor for the class which is use to set the type of term frequency * and also the value for smoothIdf. * - * @param tfType The type of term frequency, The avialbale option are - * RAW_COUNT : The count of a specific token - * BINARY : 1 if token occurs in document and 0 otherwise; - * TERM_FREQUENCY : Raw_count ÷ (number of words in document) - * SUBLINEAR_TF : log(Raw_Count) + 1 - * + * @param tfType Type of the term frequency statistics. * @param smoothIdf Used to indicate whether to use smooth idf or not. + * If idf is smooth it's calculated by the following formula: + * \f$ idf(T) = \log \frac{1 + N}{1 + df(T)} + 1, \f$ where + * \f$ N \f$ is the total number of strings in the document, + * \f$ T \f$ is the current encoded token, \f$ df(T) \f$ + * equals the number of strings which contain the token. + * If idf isn't smooth then the following rule applies: + * \f$ idf(T) = \log \frac{N}{df(T)} + 1. \f$ */ - TfIdfEncodingPolicy(TfTypes tfType = TfTypes::RAW_COUNT, - bool smoothIdf = true) : - tfType(tfType), - smoothIdf(smoothIdf) - { - } + TfIdfEncodingPolicy(const TfTypes tfType = TfTypes::RAW_COUNT, + const bool smoothIdf = true) : + tfType(tfType), + smoothIdf(smoothIdf) + { } /** * The function initializes the output matrix. * + * @tparam MatType The output matrix type. + * * @param output Output matrix to store the encoded results (sp_mat or mat). * @param datasetSize The number of strings in the input dataset. * @param maxNumTokens The maximum number of tokens in the strings of the - input dataset. - * @param dictionarySize The size of the dictionary (not used). - * @tparam MatType The type of output matrix. + input dataset (not used). + * @param dictionarySize The size of the dictionary. */ template static void InitMatrix(MatType& output, - size_t datasetSize, - size_t /*maxNumTokens*/, - size_t dictionarySize) + const size_t datasetSize, + const size_t /* maxNumTokens */, + const size_t dictionarySize) { output.zeros(datasetSize, dictionarySize); } /** * The function initializes the output matrix. - * Overloaded function to store result in vector> + * Overloaded function to store result in vector>. * + * @tparam OutputType Type of the output vector. + * * @param output Output matrix to store the encoded results. * @param datasetSize The number of strings in the input dataset. * @param maxNumTokens The maximum number of tokens in the strings of the - input dataset. - * @param dictionarySize The size of the dictionary (not used). - * @tparam OutputType The type of output vector. + input dataset (not used). + * @param dictionarySize The size of the dictionary. */ template - static void InitMatrix(std::vector >& output, - size_t datasetSize, - size_t /*maxNumTokens*/, - size_t dictionarySize) + static void InitMatrix(std::vector>& output, + const size_t datasetSize, + const size_t /* maxNumTokens */, + const size_t dictionarySize) { output.resize(datasetSize, std::vector(dictionarySize, 0)); } @@ -116,32 +120,26 @@ class TfIdfEncodingPolicy * The function performs the TfIdf encoding algorithm i.e. it writes * the encoded token to the output. * + * @tparam MatType The output matrix type. + * * @param output Output matrix to store the encoded results (sp_mat or mat). * @param value The encoded token. * @param row The row number at which the encoding is performed. - * @param col The row token number at which the encoding is performed. - * @tparam MatType The type of output matrix. + * @param col The token index in the row. */ template void Encode(MatType& output, - size_t value, - size_t row, - size_t /*col*/) + const size_t value, + const size_t row, + const size_t /* col */) { - double idf, tf; - if (smoothIdf) - idf = std::log((output.n_rows + 1) / (1 + idfdict[value - 1])) + 1; - else - idf = std::log(output.n_rows / idfdict[value - 1]) + 1; + const typename MatType::elem_type tf = + TermFrequency( + tokensFrequences[row][value], rowsSizes[row]); - if (tfType == TfTypes::TERM_FREQUENCY) - tf = tokenCount[row][value - 1] / row_size[row]; - else if (tfType == TfTypes::SUBLINEAR_TF) - tf = std::log(tokenCount[row][value - 1]) + 1; - else if (tfType == TfTypes::BINARY) - tf = tokenCount[row][value - 1] > 0 ? 1 : 0; - else - tf = tokenCount[row][value - 1]; + const typename MatType::elem_type idf = + InverseDocumentFrequency( + output.n_rows, numContainingStrings[value]); output(row, value - 1) = tf * idf; } @@ -149,39 +147,93 @@ class TfIdfEncodingPolicy /** * The function performs the TfIdf encoding algorithm i.e. it writes * the encoded token to the output. - * Overload function to accepted vector> as output type. + * Overloaded function to accept vector> as the output + * type. + * + * @tparam OutputType Type of the output vector. * * @param output Output matrix to store the encoded results. * @param value The encoded token. * @param row The row number at which the encoding is performed. - * @param col The row token number at which the encoding is performed. - * @tparam OutputType The type of output vector. - * @tparam OutputType The type of output vector. + * @param col The token index in the row. */ template void Encode(std::vector >& output, - size_t value, - size_t row, - size_t /*col*/) + const size_t value, + const size_t row, + const size_t /* col */) { - double idf, tf; - if (smoothIdf) - idf = std::log((output.size() + 1) / (1 + idfdict[value - 1])) + 1; - else - idf = std::log(output.size() / idfdict[value - 1]) + 1; + const OutputType tf = TermFrequency( + tokensFrequences[row][value], rowsSizes[row]); - if (tfType == TfTypes::TERM_FREQUENCY) - tf = tokenCount[row][value - 1] / row_size[row]; - else if (tfType == TfTypes::SUBLINEAR_TF) - tf = std::log(tokenCount[row][value - 1]) + 1; - else if (tfType == TfTypes::BINARY) - tf = tokenCount[row][value - 1] > 0 ? 1 : 0; - else - tf = tokenCount[row][value - 1]; + const OutputType idf = InverseDocumentFrequency( + output.size(), numContainingStrings[value]); output[row][value - 1] = tf * idf; } + /* + * The function calculates the necessary statistics for the purpose + * of the tf-idf algorithm during the first pass through the dataset. + * + * @param row The row number at which the encoding is performed. + * @param col The token sequence number in the row. + * @param value The encoded token. + */ + void PreprocessToken(const size_t row, + const size_t /* col */, + const size_t value) + { + if (row >= tokensFrequences.size()) + { + rowsSizes.resize(row + 1); + tokensFrequences.resize(row + 1); + } + + tokensFrequences[row][value]++; + + if (tokensFrequences[row][value] == 1) + numContainingStrings[value]++; + + rowsSizes[row]++; + } + + //! Return token frequencies. + const std::vector>& + TokensFrequences() const { return tokensFrequences; } + //! Modify token frequencies. + std::vector>& TokensFrequences() + { + return tokensFrequences; + } + + //! Get the number of containing strings depending on the given token. + const std::unordered_map& NumContainingStrings() const + { + return numContainingStrings; + } + + //! Modify the number of containing strings depending on the given token. + std::unordered_map& NumContainingStrings() + { + return numContainingStrings; + } + + //! Return the rows sizes. + const std::vector& RowsSizes() const { return rowsSizes; } + //! Modify the rows sizes. + std::vector& RowsSizes() { return rowsSizes; } + + //! Return the term frequency type. + TfTypes TfType() const { return tfType; } + //! Modify the term frequency type. + TfTypes& TfType() { return tfType; } + + //! Determine the idf algorithm type (whether it's smooth or not). + bool SmoothIdf() const { return smoothIdf; } + //! Modify the idf algorithm type (whether it's smooth or not). + bool& SmoothIdf() { return smoothIdf; } + /** * Serialize the class to the given archive. */ @@ -192,46 +244,85 @@ class TfIdfEncodingPolicy ar & BOOST_SERIALIZATION_NVP(smoothIdf); } - /* - * The function is used to create the datastrcutre will be important to find - * out idfvalue of words, and then wrtiting the output based on their count. - * - * @param row The row number at which the encoding is performed. - * @param numToken The count of token parsed till now. - * @param value The encoded token. - */ - void PreprocessToken(size_t row, - size_t /*numTokens*/, - size_t value) - { - if (row >= tokenCount.size()) - { - row_size.push_back(0); - tokenCount.emplace_back(); - } - tokenCount.back()[value - 1]++; - - if (tokenCount.back()[value - 1] == 1) - idfdict[value - 1]++; - - row_size.back()++; - } private: - // Used to store the count of token for each row. - std::vector> tokenCount; - // Used to store the idf values. - std::unordered_map idfdict; - // Used to store the number of tokens in each row. - std::vector row_size; - // smoothIdf variable to indicate smoothining. - bool smoothIdf; - // Type of Term Frequency to use. + /** + * The function calculates the term frequency statistics. + * + * @tparam ValueType Type of the returned value. + * + * @param numOccurrences The number of the given token occurrences in the row. + * @param numTokens The total number of tokens in the row. + */ + template + ValueType TermFrequency(const size_t numOccurrences, + const size_t numTokens) + { + switch (tfType) + { + case TfTypes::BINARY: + return numOccurrences > 0; + case TfTypes::RAW_COUNT: + return numOccurrences; + case TfTypes::TERM_FREQUENCY: + return static_cast(numOccurrences) / numTokens; + case TfTypes::SUBLINEAR_TF: + return std::log(static_cast(numOccurrences)) + 1; + default: + Log::Fatal << "Incorrect term frequency type!"; + return 0; + } + } + + /** + * The function calculates the inverse document frequency statistics. + * + * @tparam ValueType Type of the returned value. + * + * @param totalNumRows The total number of strings in the input dataset. + * @param numOccurrences The number of strings in the input dataset + * which contain the current token. + */ + template + ValueType InverseDocumentFrequency(const size_t totalNumRows, + const size_t numOccurrences) + { + if (smoothIdf) + { + return std::log(static_cast(totalNumRows + 1) / + (1 + numOccurrences)) + 1.0; + } + else + { + return std::log(static_cast(totalNumRows) / + numOccurrences) + 1.0; + } + } + + private: + //! Used to store the total number of tokens for each row. + std::vector> tokensFrequences; + /** + * Used to store the number of strings which contain a token depending + * on the given token. + */ + std::unordered_map numContainingStrings; + //! Used to store the number of tokens in each row. + std::vector rowsSizes; + //! Type of the term frequency scheme. TfTypes tfType; + //! Indicates whether the idf scheme is smooth or not. + bool smoothIdf; }; +/** + * A convenient alias for the StringEncoding class with TfIdfEncodingPolicy + * and the default dictionary for the given token type. + * + * @tparam TokenType Type of the tokens. + */ template using TfIdfEncoding = StringEncoding>; + StringEncodingDictionary>; } // namespace data } // namespace mlpack From d2abb44332ac8282b813fb521a726b011baa8cd2 Mon Sep 17 00:00:00 2001 From: Mikhail Lozhnikov Date: Mon, 16 Mar 2020 18:48:51 +0300 Subject: [PATCH 053/111] Fixed the automatic style checks. --- .../string_encoding_policies/bag_of_words_encoding_policy.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/mlpack/core/data/string_encoding_policies/bag_of_words_encoding_policy.hpp b/src/mlpack/core/data/string_encoding_policies/bag_of_words_encoding_policy.hpp index 4ca76b4a08..b6f164db36 100644 --- a/src/mlpack/core/data/string_encoding_policies/bag_of_words_encoding_policy.hpp +++ b/src/mlpack/core/data/string_encoding_policies/bag_of_words_encoding_policy.hpp @@ -9,8 +9,8 @@ * 3-clause BSD license along with mlpack. If not, see * http://www.opensource.org/licenses/BSD-3-Clause for more information. */ -#ifndef MLPACK_CORE_DATA_STRING_ENCODING_POLICIES_BAG_OF_WORDS_ENCODING_POLICY_HPP -#define MLPACK_CORE_DATA_STRING_ENCODING_POLICIES_BAG_OF_WORDS_ENCODING_POLICY_HPP +#ifndef MLPACK_CORE_DATA_STR_ENCODING_POLICIES_BAG_OF_WORDS_ENCODING_POLICY_HPP +#define MLPACK_CORE_DATA_STR_ENCODING_POLICIES_BAG_OF_WORDS_ENCODING_POLICY_HPP #include #include From 7233ca19d325b9c0e75e39a5694656aebb6e5ba0 Mon Sep 17 00:00:00 2001 From: AndreiMihalea Date: Tue, 17 Mar 2020 16:19:01 +0200 Subject: [PATCH 054/111] Removed rvalues references --- .vscode/settings.json | 68 ++++++++++++++ .../loss_functions/margin_ranking_loss.hpp | 14 +-- .../margin_ranking_loss_impl.hpp | 14 +-- src/mlpack/tests/loss_functions_test.cpp | 88 +++++++++---------- 4 files changed, 125 insertions(+), 59 deletions(-) create mode 100644 .vscode/settings.json diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000000..45fd170b7b --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,68 @@ +{ + "files.associations": { + "cctype": "cpp", + "clocale": "cpp", + "cmath": "cpp", + "cstdarg": "cpp", + "cstddef": "cpp", + "cstdio": "cpp", + "cstdlib": "cpp", + "cstring": "cpp", + "ctime": "cpp", + "cwchar": "cpp", + "cwctype": "cpp", + "array": "cpp", + "atomic": "cpp", + "strstream": "cpp", + "*.tcc": "cpp", + "bitset": "cpp", + "chrono": "cpp", + "codecvt": "cpp", + "complex": "cpp", + "condition_variable": "cpp", + "cstdint": "cpp", + "deque": "cpp", + "list": "cpp", + "unordered_map": "cpp", + "vector": "cpp", + "exception": "cpp", + "algorithm": "cpp", + "functional": "cpp", + "iterator": "cpp", + "map": "cpp", + "memory": "cpp", + "memory_resource": "cpp", + "numeric": "cpp", + "optional": "cpp", + "random": "cpp", + "ratio": "cpp", + "regex": "cpp", + "set": "cpp", + "string": "cpp", + "string_view": "cpp", + "system_error": "cpp", + "tuple": "cpp", + "type_traits": "cpp", + "utility": "cpp", + "fstream": "cpp", + "initializer_list": "cpp", + "iomanip": "cpp", + "iosfwd": "cpp", + "iostream": "cpp", + "istream": "cpp", + "limits": "cpp", + "mutex": "cpp", + "new": "cpp", + "ostream": "cpp", + "sstream": "cpp", + "stdexcept": "cpp", + "streambuf": "cpp", + "thread": "cpp", + "cfenv": "cpp", + "cinttypes": "cpp", + "typeindex": "cpp", + "typeinfo": "cpp", + "valarray": "cpp", + "variant": "cpp" + } +} \ No newline at end of file diff --git a/src/mlpack/methods/ann/loss_functions/margin_ranking_loss.hpp b/src/mlpack/methods/ann/loss_functions/margin_ranking_loss.hpp index cb0f7ca1d3..cd9e71ba9f 100644 --- a/src/mlpack/methods/ann/loss_functions/margin_ranking_loss.hpp +++ b/src/mlpack/methods/ann/loss_functions/margin_ranking_loss.hpp @@ -53,9 +53,9 @@ class MarginRankingLoss typename SecondInputType, typename ThirdInputType > - double Forward(const FirstInputType&& x1, - const SecondInputType&& x2, - const ThirdInputType&& y); + double Forward(const FirstInputType& x1, + const SecondInputType& x2, + const ThirdInputType& y); /** * Ordinary feed backward pass of a neural network. @@ -71,10 +71,10 @@ class MarginRankingLoss typename ThirdInputType, typename OutputType > - void Backward(const FirstInputType&& x1, - const SecondInputType&& x2, - const ThirdInputType&& y, - OutputType&& output); + void Backward(const FirstInputType& x1, + const SecondInputType& x2, + const ThirdInputType& y, + OutputType& output); //! Get the output parameter. OutputDataType& OutputParameter() const { return outputParameter; } diff --git a/src/mlpack/methods/ann/loss_functions/margin_ranking_loss_impl.hpp b/src/mlpack/methods/ann/loss_functions/margin_ranking_loss_impl.hpp index 4950eb6b94..f99af349f6 100644 --- a/src/mlpack/methods/ann/loss_functions/margin_ranking_loss_impl.hpp +++ b/src/mlpack/methods/ann/loss_functions/margin_ranking_loss_impl.hpp @@ -32,9 +32,9 @@ template < typename ThirdInputType > double MarginRankingLoss::Forward( - const FirstInputType&& x1, - const SecondInputType&& x2, - const ThirdInputType&& y) + const FirstInputType& x1, + const SecondInputType& x2, + const ThirdInputType& y) { return arma::accu(arma::max(arma::zeros(size(y)), -y % (x1 - x2) + margin)) / y.n_cols; @@ -48,10 +48,10 @@ template < typename OutputType > void MarginRankingLoss::Backward( - const FirstInputType&& x1, - const SecondInputType&& x2, - const ThirdInputType&& y, - OutputType&& output) + const FirstInputType& x1, + const SecondInputType& x2, + const ThirdInputType& y, + OutputType& output) { output = -y % (x1 - x2) + margin; output.elem(arma::find(output >= 0)).ones(); diff --git a/src/mlpack/tests/loss_functions_test.cpp b/src/mlpack/tests/loss_functions_test.cpp index 1640206357..bbeca9addf 100644 --- a/src/mlpack/tests/loss_functions_test.cpp +++ b/src/mlpack/tests/loss_functions_test.cpp @@ -50,7 +50,7 @@ BOOST_AUTO_TEST_CASE(SimpleKLDivergenceTest) // Test the Forward function. Loss should be 0 if input = target. input = arma::ones(10, 1); target = arma::ones(10, 1); - loss = module.Forward(std::move(input), std::move(target)); + loss = module.Forward(input), target)); BOOST_REQUIRE_SMALL(loss, 0.00001); } @@ -66,11 +66,11 @@ BOOST_AUTO_TEST_CASE(SimpleMeanSquaredLogarithmicErrorTest) // the manually calculated result. input = arma::zeros(1, 8); target = arma::zeros(1, 8); - double error = module.Forward(std::move(input), std::move(target)); + double error = module.Forward(input), target)); BOOST_REQUIRE_SMALL(error, 0.00001); // Test the Backward function. - module.Backward(std::move(input), std::move(target), std::move(output)); + module.Backward(input), target), output)); // The output should be equal to 0. CheckMatrices(input, output); BOOST_REQUIRE_EQUAL(output.n_rows, input.n_rows); @@ -79,11 +79,11 @@ BOOST_AUTO_TEST_CASE(SimpleMeanSquaredLogarithmicErrorTest) // Test the error function on a single input. input = arma::mat("2"); target = arma::mat("3"); - error = module.Forward(std::move(input), std::move(target)); + error = module.Forward(input), target)); BOOST_REQUIRE_CLOSE(error, 0.082760974810151655, 0.001); // Test the Backward function on a single input. - module.Backward(std::move(input), std::move(target), std::move(output)); + module.Backward(input), target), output)); BOOST_REQUIRE_CLOSE(arma::accu(output), -0.1917880483011872, 0.001); BOOST_REQUIRE_EQUAL(output.n_elem, 1); } @@ -101,11 +101,11 @@ BOOST_AUTO_TEST_CASE(KLDivergenceMeanTest) input = arma::mat("1 1 1 1 1 1 1 1 1 1"); target = arma::exp(arma::mat("2 1 1 1 1 1 1 1 1 1")); - loss = module.Forward(std::move(input), std::move(target)); + loss = module.Forward(input), target)); BOOST_REQUIRE_CLOSE_FRACTION(loss, -1.1 , 0.00001); // Test the Backward function. - module.Backward(std::move(input), std::move(target), std::move(output)); + module.Backward(input), target), output)); BOOST_REQUIRE_CLOSE_FRACTION(arma::as_scalar(output), -0.1, 0.00001); } @@ -122,11 +122,11 @@ BOOST_AUTO_TEST_CASE(KLDivergenceNoMeanTest) input = arma::mat("1 1 1 1 1 1 1 1 1 1"); target = arma::exp(arma::mat("2 1 1 1 1 1 1 1 1 1")); - loss = module.Forward(std::move(input), std::move(target)); + loss = module.Forward(input), target)); BOOST_REQUIRE_CLOSE_FRACTION(loss, -11, 0.00001); // Test the Backward function. - module.Backward(std::move(input), std::move(target), std::move(output)); + module.Backward(input), target), output)); BOOST_REQUIRE_CLOSE_FRACTION(arma::as_scalar(output), -1, 0.00001); } @@ -142,11 +142,11 @@ BOOST_AUTO_TEST_CASE(SimpleMeanSquaredErrorTest) // the manually calculated result. input = arma::mat("1.0 0.0 1.0 0.0 -1.0 0.0 -1.0 0.0"); target = arma::zeros(1, 8); - double error = module.Forward(std::move(input), std::move(target)); + double error = module.Forward(input), target)); BOOST_REQUIRE_EQUAL(error, 0.5); // Test the Backward function. - module.Backward(std::move(input), std::move(target), std::move(output)); + module.Backward(input), target), output)); // We subtract a zero vector, so according to the used backward formula: // output = 2 * (input - target) / target.n_cols, // output * nofColumns / 2 should be equal to input. @@ -157,11 +157,11 @@ BOOST_AUTO_TEST_CASE(SimpleMeanSquaredErrorTest) // Test the error function on a single input. input = arma::mat("2"); target = arma::mat("3"); - error = module.Forward(std::move(input), std::move(target)); + error = module.Forward(input), target)); BOOST_REQUIRE_EQUAL(error, 1.0); // Test the Backward function on a single input. - module.Backward(std::move(input), std::move(target), std::move(output)); + module.Backward(input), target), output)); // Test whether the output is negative. BOOST_REQUIRE_EQUAL(arma::accu(output), -2); BOOST_REQUIRE_EQUAL(output.n_elem, 1); @@ -179,16 +179,16 @@ BOOST_AUTO_TEST_CASE(SimpleCrossEntropyErrorTest) // the manually calculated result. input1 = arma::mat("0.5 0.5 0.5 0.5 0.5 0.5 0.5 0.5"); target1 = arma::zeros(1, 8); - double error1 = module.Forward(std::move(input1), std::move(target1)); + double error1 = module.Forward(input1), target1)); BOOST_REQUIRE_SMALL(error1 - 8 * std::log(2), 2e-5); input2 = arma::mat("0 1 1 0 1 0 0 1"); target2 = arma::mat("0 1 1 0 1 0 0 1"); - double error2 = module.Forward(std::move(input2), std::move(target2)); + double error2 = module.Forward(input2), target2)); BOOST_REQUIRE_SMALL(error2, 1e-5); // Test the Backward function. - module.Backward(std::move(input1), std::move(target1), std::move(output)); + module.Backward(input1), target1), output)); for (double el : output) { // For the 0.5 constant vector we should get 1 / (1 - 0.5) = 2 everywhere. @@ -197,7 +197,7 @@ BOOST_AUTO_TEST_CASE(SimpleCrossEntropyErrorTest) BOOST_REQUIRE_EQUAL(output.n_rows, input1.n_rows); BOOST_REQUIRE_EQUAL(output.n_cols, input1.n_cols); - module.Backward(std::move(input2), std::move(target2), std::move(output)); + module.Backward(input2), target2), output)); for (size_t i = 0; i < 8; ++i) { double el = output.at(0, i); @@ -223,25 +223,25 @@ BOOST_AUTO_TEST_CASE(SimpleSigmoidCrossEntropyErrorTest) // the calculated result. input1 = arma::mat("0.5 0.5 0.5 0.5 0.5 0.5 0.5 0.5"); target1 = arma::zeros(1, 8); - double error1 = module.Forward(std::move(input1), std::move(target1)); + double error1 = module.Forward(input1), target1)); double expected = 0.97407699; // Value computed using tensorflow. BOOST_REQUIRE_SMALL(error1 / input1.n_elem - expected, 1e-7); input2 = arma::mat("1 2 3 4 5"); target2 = arma::mat("0 0 1 0 1"); - double error2 = module.Forward(std::move(input2), std::move(target2)); + double error2 = module.Forward(input2), target2)); expected = 1.5027283; BOOST_REQUIRE_SMALL(error2 / input2.n_elem - expected, 1e-6); input3 = arma::mat("0 -1 -1 0 -1 0 0 -1"); target3 = arma::mat("0 -1 -1 0 -1 0 0 -1"); - double error3 = module.Forward(std::move(input3), std::move(target3)); + double error3 = module.Forward(input3), target3)); expected = 0.00320443; BOOST_REQUIRE_SMALL(error3 / input3.n_elem - expected, 1e-6); // Test the Backward function. - module.Backward(std::move(input1), std::move(target1), std::move(output)); + module.Backward(input1), target1), output)); expected = 0.62245929; for (size_t i = 0; i < output.n_elem; i++) BOOST_REQUIRE_SMALL(output(i) - expected, 1e-5); @@ -250,13 +250,13 @@ BOOST_AUTO_TEST_CASE(SimpleSigmoidCrossEntropyErrorTest) expectedOutput = arma::mat( "0.7310586 0.88079709 -0.04742587 0.98201376 -0.00669285"); - module.Backward(std::move(input2), std::move(target2), std::move(output)); + module.Backward(input2), target2), output)); for (size_t i = 0; i < output.n_elem; i++) BOOST_REQUIRE_SMALL(output(i) - expectedOutput(i), 1e-5); BOOST_REQUIRE_EQUAL(output.n_rows, input2.n_rows); BOOST_REQUIRE_EQUAL(output.n_cols, input2.n_cols); - module.Backward(std::move(input3), std::move(target3), std::move(output)); + module.Backward(input3), target3), output)); expectedOutput = arma::mat("0.5 1.2689414"); for (size_t i = 0; i < 8; ++i) { @@ -282,18 +282,18 @@ BOOST_AUTO_TEST_CASE(SimpleEarthMoverDistanceLayerTest) // the manually calculated result. input1 = arma::mat("0.5 0.5 0.5 0.5 0.5 0.5 0.5 0.5"); target1 = arma::zeros(1, 8); - double error1 = module.Forward(std::move(input1), std::move(target1)); + double error1 = module.Forward(input1), target1)); double expected = 0.0; BOOST_REQUIRE_SMALL(error1 / input1.n_elem - expected, 1e-7); input2 = arma::mat("1 2 3 4 5"); target2 = arma::mat("1 0 1 0 1"); - double error2 = module.Forward(std::move(input2), std::move(target2)); + double error2 = module.Forward(input2), target2)); expected = -1.8; BOOST_REQUIRE_SMALL(error2 / input2.n_elem - expected, 1e-6); // Test the Backward function. - module.Backward(std::move(input1), std::move(target1), std::move(output)); + module.Backward(input1), target1), output)); expected = 0.0; for (size_t i = 0; i < output.n_elem; i++) BOOST_REQUIRE_SMALL(output(i) - expected, 1e-5); @@ -301,7 +301,7 @@ BOOST_AUTO_TEST_CASE(SimpleEarthMoverDistanceLayerTest) BOOST_REQUIRE_EQUAL(output.n_cols, input1.n_cols); expectedOutput = arma::mat("-1 0 -1 0 -1"); - module.Backward(std::move(input2), std::move(target2), std::move(output)); + module.Backward(input2), target2), output)); for (size_t i = 0; i < output.n_elem; i++) BOOST_REQUIRE_SMALL(output(i) - expectedOutput(i), 1e-5); BOOST_REQUIRE_EQUAL(output.n_rows, input2.n_rows); @@ -406,16 +406,16 @@ BOOST_AUTO_TEST_CASE(DiceLossTest) // Test the Forward function. Loss should be 0 if input = target. input1 = arma::ones(10, 1); target = arma::ones(10, 1); - loss = module.Forward(std::move(input1), std::move(target)); + loss = module.Forward(input1), target)); BOOST_REQUIRE_SMALL(loss, 0.00001); // Test the Forward function. Loss should be 0.185185185. input2 = arma::ones(10, 1) * 0.5; - loss = module.Forward(std::move(input2), std::move(target)); + loss = module.Forward(input2), target)); BOOST_REQUIRE_CLOSE(loss, 0.185185185, 0.00001); // Test the Backward function for input = target. - module.Backward(std::move(input1), std::move(target), std::move(output)); + module.Backward(input1), target), output)); for (double el : output) { // For input = target we should get 0.0 everywhere. @@ -425,7 +425,7 @@ BOOST_AUTO_TEST_CASE(DiceLossTest) BOOST_REQUIRE_EQUAL(output.n_cols, input1.n_cols); // Test the Backward function. - module.Backward(std::move(input2), std::move(target), std::move(output)); + module.Backward(input2), target), output)); for (double el : output) { // For the 0.5 constant vector we should get -0.0877914951989026 everywhere. @@ -447,11 +447,11 @@ BOOST_AUTO_TEST_CASE(SimpleMeanBiasErrorTest) // the manually calculated result. input = arma::mat("1.0 0.0 1.0 -1.0 -1.0 0.0 -1.0 0.0"); target = arma::zeros(1, 8); - double error = module.Forward(std::move(input), std::move(target)); + double error = module.Forward(input), target)); BOOST_REQUIRE_EQUAL(error, 0.125); // Test the Backward function. - module.Backward(std::move(input), std::move(target), std::move(output)); + module.Backward(input), target), output)); // We should get a vector with -1 everywhere. for (double el : output) { @@ -463,11 +463,11 @@ BOOST_AUTO_TEST_CASE(SimpleMeanBiasErrorTest) // Test the error function on a single input. input = arma::mat("2"); target = arma::mat("3"); - error = module.Forward(std::move(input), std::move(target)); + error = module.Forward(input), target)); BOOST_REQUIRE_EQUAL(error, 1.0); // Test the Backward function on a single input. - module.Backward(std::move(input), std::move(target), std::move(output)); + module.Backward(input), target), output)); // Test whether the output is negative. BOOST_REQUIRE_EQUAL(arma::accu(output), -1); BOOST_REQUIRE_EQUAL(output.n_elem, 1); @@ -485,11 +485,11 @@ BOOST_AUTO_TEST_CASE(LogCoshLossTest) // Test the Forward function. Loss should be 0 if input = target. input = arma::ones(10, 1); target = arma::ones(10, 1); - loss = module.Forward(std::move(input), std::move(target)); + loss = module.Forward(input), target)); BOOST_REQUIRE_EQUAL(loss, 0); // Test the Backward function for input = target. - module.Backward(std::move(input), std::move(target), std::move(output)); + module.Backward(input), target), output)); for (double el : output) { // For input = target we should get 0.0 everywhere. @@ -502,11 +502,11 @@ BOOST_AUTO_TEST_CASE(LogCoshLossTest) // Test the Forward function. Loss should be 0.546621. input = arma::mat("1 2 3 4 5"); target = arma::mat("1 2.4 3.4 4.2 5.5"); - loss = module.Forward(std::move(input), std::move(target)); + loss = module.Forward(input), target)); BOOST_REQUIRE_CLOSE(loss, 0.546621, 1e-3); // Test the Backward function. - module.Backward(std::move(input), std::move(target), std::move(output)); + module.Backward(input), target), output)); BOOST_REQUIRE_CLOSE(arma::accu(output), 2.46962, 1e-3); BOOST_REQUIRE_EQUAL(output.n_rows, input.n_rows); BOOST_REQUIRE_EQUAL(output.n_cols, input.n_cols); @@ -525,13 +525,12 @@ BOOST_AUTO_TEST_CASE(MarginRankingLossTest) x1 = arma::mat("1 2 5 7 -1 -3"); x2 = arma::mat("-1 3 -4 11 3 -3"); y = arma::mat("1 -1 -1 1 -1 1"); - double error = module.Forward(std::move(x1), std::move(x2), std::move(y)); + double error = module.Forward(x1, x2, y); // Computed using torch.nn.functional.margin_ranking_loss() BOOST_REQUIRE_CLOSE(error, 2.66667, 1e-3); // Test the Backward function. - module.Backward(std::move(x1), std::move(x2), std::move(y), - std::move(output)); + module.Backward(x1, x2, y, output); CheckMatrices(output, arma::mat("-0.000000 0.166667 -1.500000 0.666667 " "0.000000 -0.000000"), 1e-3); @@ -544,12 +543,11 @@ BOOST_AUTO_TEST_CASE(MarginRankingLossTest) x2 = arma::mat("-4.5288 -9.2766 -0.5882 -5.6643 -6.0175 8.8506 3.4759 " "-9.4886 2.2755 8.4951"); y = arma::mat("1 1 -1 1 -1 1 1 1 -1 1"); - error = module.Forward(std::move(x1), std::move(x2), std::move(y)); + error = module.Forward(x1, x2, y); BOOST_REQUIRE_CLOSE(error, 3.03530, 1e-3); // Test the Backward function on the second input. - module.Backward(std::move(x1), std::move(x2), std::move(y), - std::move(output)); + module.Backward(x1, x2, y, output); CheckMatrices(output, arma::mat("0.000000 0.000000 0.091240 0.000000 " "-0.753830 1.336900 0.000000 0.000000 -0.207000 0.328810"), 1e-6); From 5559e4f4eaca59a2b452cd27fb0a40c2993f593f Mon Sep 17 00:00:00 2001 From: AndreiMihalea Date: Tue, 17 Mar 2020 16:27:42 +0200 Subject: [PATCH 055/111] Removed rvalues references --- src/mlpack/tests/loss_functions_test.cpp | 78 ++++++++++++------------ 1 file changed, 39 insertions(+), 39 deletions(-) diff --git a/src/mlpack/tests/loss_functions_test.cpp b/src/mlpack/tests/loss_functions_test.cpp index bbeca9addf..58ad1f3826 100644 --- a/src/mlpack/tests/loss_functions_test.cpp +++ b/src/mlpack/tests/loss_functions_test.cpp @@ -50,7 +50,7 @@ BOOST_AUTO_TEST_CASE(SimpleKLDivergenceTest) // Test the Forward function. Loss should be 0 if input = target. input = arma::ones(10, 1); target = arma::ones(10, 1); - loss = module.Forward(input), target)); + loss = module.Forward(input, target); BOOST_REQUIRE_SMALL(loss, 0.00001); } @@ -66,11 +66,11 @@ BOOST_AUTO_TEST_CASE(SimpleMeanSquaredLogarithmicErrorTest) // the manually calculated result. input = arma::zeros(1, 8); target = arma::zeros(1, 8); - double error = module.Forward(input), target)); + double error = module.Forward(input, target); BOOST_REQUIRE_SMALL(error, 0.00001); // Test the Backward function. - module.Backward(input), target), output)); + module.Backward(input, target, output); // The output should be equal to 0. CheckMatrices(input, output); BOOST_REQUIRE_EQUAL(output.n_rows, input.n_rows); @@ -79,11 +79,11 @@ BOOST_AUTO_TEST_CASE(SimpleMeanSquaredLogarithmicErrorTest) // Test the error function on a single input. input = arma::mat("2"); target = arma::mat("3"); - error = module.Forward(input), target)); + error = module.Forward(input, target); BOOST_REQUIRE_CLOSE(error, 0.082760974810151655, 0.001); // Test the Backward function on a single input. - module.Backward(input), target), output)); + module.Backward(input, target, output); BOOST_REQUIRE_CLOSE(arma::accu(output), -0.1917880483011872, 0.001); BOOST_REQUIRE_EQUAL(output.n_elem, 1); } @@ -101,11 +101,11 @@ BOOST_AUTO_TEST_CASE(KLDivergenceMeanTest) input = arma::mat("1 1 1 1 1 1 1 1 1 1"); target = arma::exp(arma::mat("2 1 1 1 1 1 1 1 1 1")); - loss = module.Forward(input), target)); + loss = module.Forward(input, target); BOOST_REQUIRE_CLOSE_FRACTION(loss, -1.1 , 0.00001); // Test the Backward function. - module.Backward(input), target), output)); + module.Backward(input, target, output); BOOST_REQUIRE_CLOSE_FRACTION(arma::as_scalar(output), -0.1, 0.00001); } @@ -122,11 +122,11 @@ BOOST_AUTO_TEST_CASE(KLDivergenceNoMeanTest) input = arma::mat("1 1 1 1 1 1 1 1 1 1"); target = arma::exp(arma::mat("2 1 1 1 1 1 1 1 1 1")); - loss = module.Forward(input), target)); + loss = module.Forward(input, target); BOOST_REQUIRE_CLOSE_FRACTION(loss, -11, 0.00001); // Test the Backward function. - module.Backward(input), target), output)); + module.Backward(input, target, output); BOOST_REQUIRE_CLOSE_FRACTION(arma::as_scalar(output), -1, 0.00001); } @@ -142,11 +142,11 @@ BOOST_AUTO_TEST_CASE(SimpleMeanSquaredErrorTest) // the manually calculated result. input = arma::mat("1.0 0.0 1.0 0.0 -1.0 0.0 -1.0 0.0"); target = arma::zeros(1, 8); - double error = module.Forward(input), target)); + double error = module.Forward(input, target); BOOST_REQUIRE_EQUAL(error, 0.5); // Test the Backward function. - module.Backward(input), target), output)); + module.Backward(input, target, output); // We subtract a zero vector, so according to the used backward formula: // output = 2 * (input - target) / target.n_cols, // output * nofColumns / 2 should be equal to input. @@ -157,11 +157,11 @@ BOOST_AUTO_TEST_CASE(SimpleMeanSquaredErrorTest) // Test the error function on a single input. input = arma::mat("2"); target = arma::mat("3"); - error = module.Forward(input), target)); + error = module.Forward(input, target); BOOST_REQUIRE_EQUAL(error, 1.0); // Test the Backward function on a single input. - module.Backward(input), target), output)); + module.Backward(input, target, output); // Test whether the output is negative. BOOST_REQUIRE_EQUAL(arma::accu(output), -2); BOOST_REQUIRE_EQUAL(output.n_elem, 1); @@ -179,16 +179,16 @@ BOOST_AUTO_TEST_CASE(SimpleCrossEntropyErrorTest) // the manually calculated result. input1 = arma::mat("0.5 0.5 0.5 0.5 0.5 0.5 0.5 0.5"); target1 = arma::zeros(1, 8); - double error1 = module.Forward(input1), target1)); + double error1 = module.Forward(input1, target1); BOOST_REQUIRE_SMALL(error1 - 8 * std::log(2), 2e-5); input2 = arma::mat("0 1 1 0 1 0 0 1"); target2 = arma::mat("0 1 1 0 1 0 0 1"); - double error2 = module.Forward(input2), target2)); + double error2 = module.Forward(input2, target2); BOOST_REQUIRE_SMALL(error2, 1e-5); // Test the Backward function. - module.Backward(input1), target1), output)); + module.Backward(input1, target1, output); for (double el : output) { // For the 0.5 constant vector we should get 1 / (1 - 0.5) = 2 everywhere. @@ -197,7 +197,7 @@ BOOST_AUTO_TEST_CASE(SimpleCrossEntropyErrorTest) BOOST_REQUIRE_EQUAL(output.n_rows, input1.n_rows); BOOST_REQUIRE_EQUAL(output.n_cols, input1.n_cols); - module.Backward(input2), target2), output)); + module.Backward(input2, target2, output); for (size_t i = 0; i < 8; ++i) { double el = output.at(0, i); @@ -223,25 +223,25 @@ BOOST_AUTO_TEST_CASE(SimpleSigmoidCrossEntropyErrorTest) // the calculated result. input1 = arma::mat("0.5 0.5 0.5 0.5 0.5 0.5 0.5 0.5"); target1 = arma::zeros(1, 8); - double error1 = module.Forward(input1), target1)); + double error1 = module.Forward(input1, target1); double expected = 0.97407699; // Value computed using tensorflow. BOOST_REQUIRE_SMALL(error1 / input1.n_elem - expected, 1e-7); input2 = arma::mat("1 2 3 4 5"); target2 = arma::mat("0 0 1 0 1"); - double error2 = module.Forward(input2), target2)); + double error2 = module.Forward(input2, target2); expected = 1.5027283; BOOST_REQUIRE_SMALL(error2 / input2.n_elem - expected, 1e-6); input3 = arma::mat("0 -1 -1 0 -1 0 0 -1"); target3 = arma::mat("0 -1 -1 0 -1 0 0 -1"); - double error3 = module.Forward(input3), target3)); + double error3 = module.Forward(input3, target3); expected = 0.00320443; BOOST_REQUIRE_SMALL(error3 / input3.n_elem - expected, 1e-6); // Test the Backward function. - module.Backward(input1), target1), output)); + module.Backward(input1, target1, output); expected = 0.62245929; for (size_t i = 0; i < output.n_elem; i++) BOOST_REQUIRE_SMALL(output(i) - expected, 1e-5); @@ -250,13 +250,13 @@ BOOST_AUTO_TEST_CASE(SimpleSigmoidCrossEntropyErrorTest) expectedOutput = arma::mat( "0.7310586 0.88079709 -0.04742587 0.98201376 -0.00669285"); - module.Backward(input2), target2), output)); + module.Backward(input2, target2, output); for (size_t i = 0; i < output.n_elem; i++) BOOST_REQUIRE_SMALL(output(i) - expectedOutput(i), 1e-5); BOOST_REQUIRE_EQUAL(output.n_rows, input2.n_rows); BOOST_REQUIRE_EQUAL(output.n_cols, input2.n_cols); - module.Backward(input3), target3), output)); + module.Backward(input3, target3, output); expectedOutput = arma::mat("0.5 1.2689414"); for (size_t i = 0; i < 8; ++i) { @@ -282,18 +282,18 @@ BOOST_AUTO_TEST_CASE(SimpleEarthMoverDistanceLayerTest) // the manually calculated result. input1 = arma::mat("0.5 0.5 0.5 0.5 0.5 0.5 0.5 0.5"); target1 = arma::zeros(1, 8); - double error1 = module.Forward(input1), target1)); + double error1 = module.Forward(input1, target1); double expected = 0.0; BOOST_REQUIRE_SMALL(error1 / input1.n_elem - expected, 1e-7); input2 = arma::mat("1 2 3 4 5"); target2 = arma::mat("1 0 1 0 1"); - double error2 = module.Forward(input2), target2)); + double error2 = module.Forward(input2, target2); expected = -1.8; BOOST_REQUIRE_SMALL(error2 / input2.n_elem - expected, 1e-6); // Test the Backward function. - module.Backward(input1), target1), output)); + module.Backward(input1, target1, output); expected = 0.0; for (size_t i = 0; i < output.n_elem; i++) BOOST_REQUIRE_SMALL(output(i) - expected, 1e-5); @@ -301,7 +301,7 @@ BOOST_AUTO_TEST_CASE(SimpleEarthMoverDistanceLayerTest) BOOST_REQUIRE_EQUAL(output.n_cols, input1.n_cols); expectedOutput = arma::mat("-1 0 -1 0 -1"); - module.Backward(input2), target2), output)); + module.Backward(input2, target2, output); for (size_t i = 0; i < output.n_elem; i++) BOOST_REQUIRE_SMALL(output(i) - expectedOutput(i), 1e-5); BOOST_REQUIRE_EQUAL(output.n_rows, input2.n_rows); @@ -406,16 +406,16 @@ BOOST_AUTO_TEST_CASE(DiceLossTest) // Test the Forward function. Loss should be 0 if input = target. input1 = arma::ones(10, 1); target = arma::ones(10, 1); - loss = module.Forward(input1), target)); + loss = module.Forward(input1, target); BOOST_REQUIRE_SMALL(loss, 0.00001); // Test the Forward function. Loss should be 0.185185185. input2 = arma::ones(10, 1) * 0.5; - loss = module.Forward(input2), target)); + loss = module.Forward(input2, target); BOOST_REQUIRE_CLOSE(loss, 0.185185185, 0.00001); // Test the Backward function for input = target. - module.Backward(input1), target), output)); + module.Backward(input1, target, output); for (double el : output) { // For input = target we should get 0.0 everywhere. @@ -425,7 +425,7 @@ BOOST_AUTO_TEST_CASE(DiceLossTest) BOOST_REQUIRE_EQUAL(output.n_cols, input1.n_cols); // Test the Backward function. - module.Backward(input2), target), output)); + module.Backward(input2, target, output); for (double el : output) { // For the 0.5 constant vector we should get -0.0877914951989026 everywhere. @@ -447,11 +447,11 @@ BOOST_AUTO_TEST_CASE(SimpleMeanBiasErrorTest) // the manually calculated result. input = arma::mat("1.0 0.0 1.0 -1.0 -1.0 0.0 -1.0 0.0"); target = arma::zeros(1, 8); - double error = module.Forward(input), target)); + double error = module.Forward(input, target); BOOST_REQUIRE_EQUAL(error, 0.125); // Test the Backward function. - module.Backward(input), target), output)); + module.Backward(input, target, output); // We should get a vector with -1 everywhere. for (double el : output) { @@ -463,11 +463,11 @@ BOOST_AUTO_TEST_CASE(SimpleMeanBiasErrorTest) // Test the error function on a single input. input = arma::mat("2"); target = arma::mat("3"); - error = module.Forward(input), target)); + error = module.Forward(input, target); BOOST_REQUIRE_EQUAL(error, 1.0); // Test the Backward function on a single input. - module.Backward(input), target), output)); + module.Backward(input, target, output); // Test whether the output is negative. BOOST_REQUIRE_EQUAL(arma::accu(output), -1); BOOST_REQUIRE_EQUAL(output.n_elem, 1); @@ -485,11 +485,11 @@ BOOST_AUTO_TEST_CASE(LogCoshLossTest) // Test the Forward function. Loss should be 0 if input = target. input = arma::ones(10, 1); target = arma::ones(10, 1); - loss = module.Forward(input), target)); + loss = module.Forward(input, target); BOOST_REQUIRE_EQUAL(loss, 0); // Test the Backward function for input = target. - module.Backward(input), target), output)); + module.Backward(input, target, output); for (double el : output) { // For input = target we should get 0.0 everywhere. @@ -502,11 +502,11 @@ BOOST_AUTO_TEST_CASE(LogCoshLossTest) // Test the Forward function. Loss should be 0.546621. input = arma::mat("1 2 3 4 5"); target = arma::mat("1 2.4 3.4 4.2 5.5"); - loss = module.Forward(input), target)); + loss = module.Forward(input, target); BOOST_REQUIRE_CLOSE(loss, 0.546621, 1e-3); // Test the Backward function. - module.Backward(input), target), output)); + module.Backward(input, target, output); BOOST_REQUIRE_CLOSE(arma::accu(output), 2.46962, 1e-3); BOOST_REQUIRE_EQUAL(output.n_rows, input.n_rows); BOOST_REQUIRE_EQUAL(output.n_cols, input.n_cols); From 8e34c3a1e219f437d72485fd6f812a939d8e2fc0 Mon Sep 17 00:00:00 2001 From: Andrei Mihalea Date: Tue, 17 Mar 2020 16:29:50 +0200 Subject: [PATCH 056/111] Delete settings.json Deleted .vscode --- .vscode/settings.json | 68 ------------------------------------------- 1 file changed, 68 deletions(-) delete mode 100644 .vscode/settings.json diff --git a/.vscode/settings.json b/.vscode/settings.json deleted file mode 100644 index 45fd170b7b..0000000000 --- a/.vscode/settings.json +++ /dev/null @@ -1,68 +0,0 @@ -{ - "files.associations": { - "cctype": "cpp", - "clocale": "cpp", - "cmath": "cpp", - "cstdarg": "cpp", - "cstddef": "cpp", - "cstdio": "cpp", - "cstdlib": "cpp", - "cstring": "cpp", - "ctime": "cpp", - "cwchar": "cpp", - "cwctype": "cpp", - "array": "cpp", - "atomic": "cpp", - "strstream": "cpp", - "*.tcc": "cpp", - "bitset": "cpp", - "chrono": "cpp", - "codecvt": "cpp", - "complex": "cpp", - "condition_variable": "cpp", - "cstdint": "cpp", - "deque": "cpp", - "list": "cpp", - "unordered_map": "cpp", - "vector": "cpp", - "exception": "cpp", - "algorithm": "cpp", - "functional": "cpp", - "iterator": "cpp", - "map": "cpp", - "memory": "cpp", - "memory_resource": "cpp", - "numeric": "cpp", - "optional": "cpp", - "random": "cpp", - "ratio": "cpp", - "regex": "cpp", - "set": "cpp", - "string": "cpp", - "string_view": "cpp", - "system_error": "cpp", - "tuple": "cpp", - "type_traits": "cpp", - "utility": "cpp", - "fstream": "cpp", - "initializer_list": "cpp", - "iomanip": "cpp", - "iosfwd": "cpp", - "iostream": "cpp", - "istream": "cpp", - "limits": "cpp", - "mutex": "cpp", - "new": "cpp", - "ostream": "cpp", - "sstream": "cpp", - "stdexcept": "cpp", - "streambuf": "cpp", - "thread": "cpp", - "cfenv": "cpp", - "cinttypes": "cpp", - "typeindex": "cpp", - "typeinfo": "cpp", - "valarray": "cpp", - "variant": "cpp" - } -} \ No newline at end of file From bd892fe07b06a1dcbc11dd47578ab99f3185a46d Mon Sep 17 00:00:00 2001 From: AndreiMihalea Date: Tue, 17 Mar 2020 18:25:09 +0200 Subject: [PATCH 057/111] Modified the parameter names and the comments before the margin getter/setter --- .../loss_functions/margin_ranking_loss.hpp | 31 +++++++++---------- .../margin_ranking_loss_impl.hpp | 24 +++++++------- 2 files changed, 27 insertions(+), 28 deletions(-) diff --git a/src/mlpack/methods/ann/loss_functions/margin_ranking_loss.hpp b/src/mlpack/methods/ann/loss_functions/margin_ranking_loss.hpp index cd9e71ba9f..54d8928b19 100644 --- a/src/mlpack/methods/ann/loss_functions/margin_ranking_loss.hpp +++ b/src/mlpack/methods/ann/loss_functions/margin_ranking_loss.hpp @@ -23,7 +23,6 @@ namespace ann /** Artificial Neural Network. */ { * @tparam OutputDataType Type of the output data (arma::colvec, arma::mat, * arma::sp_mat or arma::cube). */ - template < typename InputDataType = arma::mat, typename OutputDataType = arma::mat @@ -44,36 +43,36 @@ class MarginRankingLoss * a value of 1 in the label means the first input should be ranked higher * and a value of -1 means the second input should be ranked higher. * - * @param x1 First input data used for evaluating the specified function. - * @param x2 Second input data used for evaluating the specified function. + * @param input1 First input data used for evaluating the specified function. + * @param input2 Second input data used for evaluating the specified function. * @param y The label vector which contains -1 or 1 values. */ template < typename FirstInputType, typename SecondInputType, - typename ThirdInputType + typename TargetType > - double Forward(const FirstInputType& x1, - const SecondInputType& x2, - const ThirdInputType& y); + double Forward(const FirstInputType& input1, + const SecondInputType& input2, + const TargetType& target); /** * Ordinary feed backward pass of a neural network. * - * @param x1 The propagated first input activation. - * @param x2 The propagated second input activation. - * @param y The label vector which contains -1 or 1 values. + * @param input1 The propagated first input activation. + * @param input2 The propagated second input activation. + * @param target The label vector which contains -1 or 1 values. * @param output The calculated error. */ template < typename FirstInputType, typename SecondInputType, - typename ThirdInputType, + typename TargetType, typename OutputType > - void Backward(const FirstInputType& x1, - const SecondInputType& x2, - const ThirdInputType& y, + void Backward(const FirstInputType& input1, + const SecondInputType& input2, + const TargetType& target, OutputType& output); //! Get the output parameter. @@ -81,9 +80,9 @@ class MarginRankingLoss //! Modify the output parameter. OutputDataType& OutputParameter() { return outputParameter; } - //! Get the output parameter. + //! Get the margin parameter. double Margin() const { return margin; } - //! Modify the output parameter. + //! Modify the margin parameter. double& Margin() { return margin; } /** diff --git a/src/mlpack/methods/ann/loss_functions/margin_ranking_loss_impl.hpp b/src/mlpack/methods/ann/loss_functions/margin_ranking_loss_impl.hpp index f99af349f6..5e908dfede 100644 --- a/src/mlpack/methods/ann/loss_functions/margin_ranking_loss_impl.hpp +++ b/src/mlpack/methods/ann/loss_functions/margin_ranking_loss_impl.hpp @@ -29,34 +29,34 @@ template template < typename FirstInputType, typename SecondInputType, - typename ThirdInputType + typename TargetType > double MarginRankingLoss::Forward( - const FirstInputType& x1, - const SecondInputType& x2, - const ThirdInputType& y) + const FirstInputType& input1, + const SecondInputType& input2, + const TargetType& target) { - return arma::accu(arma::max(arma::zeros(size(y)), - -y % (x1 - x2) + margin)) / y.n_cols; + return arma::accu(arma::max(arma::zeros(size(target)), + -target % (input1 - input2) + margin)) / target.n_cols; } template template < typename FirstInputType, typename SecondInputType, - typename ThirdInputType, + typename TargetType, typename OutputType > void MarginRankingLoss::Backward( - const FirstInputType& x1, - const SecondInputType& x2, - const ThirdInputType& y, + const FirstInputType& input1, + const SecondInputType& input2, + const TargetType& target, OutputType& output) { - output = -y % (x1 - x2) + margin; + output = -target % (input1 - input2) + margin; output.elem(arma::find(output >= 0)).ones(); output.elem(arma::find(output < 0)).zeros(); - output = (x2 - x1) % output / y.n_cols; + output = (input2 - input1) % output / target.n_cols; } template From 32d6d89dccd9e11f62383542e872d8dec38fcb6d Mon Sep 17 00:00:00 2001 From: AndreiMihalea Date: Tue, 17 Mar 2020 18:37:03 +0200 Subject: [PATCH 058/111] Modified y to target --- src/mlpack/methods/ann/loss_functions/margin_ranking_loss.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mlpack/methods/ann/loss_functions/margin_ranking_loss.hpp b/src/mlpack/methods/ann/loss_functions/margin_ranking_loss.hpp index 54d8928b19..ec9c78b11f 100644 --- a/src/mlpack/methods/ann/loss_functions/margin_ranking_loss.hpp +++ b/src/mlpack/methods/ann/loss_functions/margin_ranking_loss.hpp @@ -45,7 +45,7 @@ class MarginRankingLoss * * @param input1 First input data used for evaluating the specified function. * @param input2 Second input data used for evaluating the specified function. - * @param y The label vector which contains -1 or 1 values. + * @param target The label vector which contains -1 or 1 values. */ template < typename FirstInputType, From a555f47d41756142cdd3b5ca20d8b49d8f8ff01a Mon Sep 17 00:00:00 2001 From: AndreiMihalea Date: Wed, 18 Mar 2020 13:25:59 +0200 Subject: [PATCH 059/111] Changed variable names int the test file --- src/mlpack/tests/loss_functions_test.cpp | 26 ++++++++++++------------ 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/src/mlpack/tests/loss_functions_test.cpp b/src/mlpack/tests/loss_functions_test.cpp index f4e007507c..8402212e2b 100644 --- a/src/mlpack/tests/loss_functions_test.cpp +++ b/src/mlpack/tests/loss_functions_test.cpp @@ -585,37 +585,37 @@ BOOST_AUTO_TEST_CASE(HingeEmbeddingLossTest) */ BOOST_AUTO_TEST_CASE(MarginRankingLossTest) { - arma::mat x1, x2, y, output; + arma::mat input1, input2, target, output; MarginRankingLoss<> module; // Test the Forward function on a user generator input and compare it against // the manually calculated result. - x1 = arma::mat("1 2 5 7 -1 -3"); - x2 = arma::mat("-1 3 -4 11 3 -3"); - y = arma::mat("1 -1 -1 1 -1 1"); - double error = module.Forward(x1, x2, y); + input1 = arma::mat("1 2 5 7 -1 -3"); + input2 = arma::mat("-1 3 -4 11 3 -3"); + target = arma::mat("1 -1 -1 1 -1 1"); + double error = module.Forward(input1, input2, target); // Computed using torch.nn.functional.margin_ranking_loss() BOOST_REQUIRE_CLOSE(error, 2.66667, 1e-3); // Test the Backward function. - module.Backward(x1, x2, y, output); + module.Backward(input1, input2, target, output); CheckMatrices(output, arma::mat("-0.000000 0.166667 -1.500000 0.666667 " "0.000000 -0.000000"), 1e-3); - BOOST_REQUIRE_EQUAL(output.n_rows, y.n_rows); - BOOST_REQUIRE_EQUAL(output.n_cols, y.n_cols); + BOOST_REQUIRE_EQUAL(output.n_rows, target.n_rows); + BOOST_REQUIRE_EQUAL(output.n_cols, target.n_cols); // Test the error function on another input. - x1 = arma::mat("0.4287 -1.6208 -1.5006 -0.4473 1.5208 -4.5184 9.3574 " + input1 = arma::mat("0.4287 -1.6208 -1.5006 -0.4473 1.5208 -4.5184 9.3574 " "-4.8090 4.3455 5.2070"); - x2 = arma::mat("-4.5288 -9.2766 -0.5882 -5.6643 -6.0175 8.8506 3.4759 " + input2 = arma::mat("-4.5288 -9.2766 -0.5882 -5.6643 -6.0175 8.8506 3.4759 " "-9.4886 2.2755 8.4951"); - y = arma::mat("1 1 -1 1 -1 1 1 1 -1 1"); - error = module.Forward(x1, x2, y); + target = arma::mat("1 1 -1 1 -1 1 1 1 -1 1"); + error = module.Forward(input1, input2, target); BOOST_REQUIRE_CLOSE(error, 3.03530, 1e-3); // Test the Backward function on the second input. - module.Backward(x1, x2, y, output); + module.Backward(input1, input2, target, output); CheckMatrices(output, arma::mat("0.000000 0.000000 0.091240 0.000000 " "-0.753830 1.336900 0.000000 0.000000 -0.207000 0.328810"), 1e-6); From cdbdbe70c63915f0299d2e73c854127e82ec9978 Mon Sep 17 00:00:00 2001 From: AndreiMihalea Date: Wed, 18 Mar 2020 13:42:39 +0200 Subject: [PATCH 060/111] Rebuild From 888631202ae66e9b94424e6b0bd3ff26a2cfa008 Mon Sep 17 00:00:00 2001 From: jeffin143 Date: Wed, 18 Mar 2020 23:52:56 +0530 Subject: [PATCH 061/111] Set HAS_STB --- CMakeLists.txt | 1 + src/mlpack/methods/preprocess/CMakeLists.txt | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 4dc6d0beb8..817a947b6e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -328,6 +328,7 @@ if (NOT STB_IMAGE_FOUND) install(FILES "${CMAKE_BINARY_DIR}/deps/${STB_DIR}/stb_image.h" DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}") install(FILES "${CMAKE_BINARY_DIR}/deps/${STB_DIR}/stb_image_write.h" DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}") add_definitions(-DHAS_STB) + set(HAS_STB 1) set(STB_AVAILABLE "1") else () message(WARNING diff --git a/src/mlpack/methods/preprocess/CMakeLists.txt b/src/mlpack/methods/preprocess/CMakeLists.txt index 48d0378614..4faccbf6d6 100644 --- a/src/mlpack/methods/preprocess/CMakeLists.txt +++ b/src/mlpack/methods/preprocess/CMakeLists.txt @@ -42,7 +42,7 @@ add_cli_executable(preprocess_scale) add_python_binding(preprocess_scale) add_markdown_docs(preprocess_scale "cli;python" "preprocessing") -if (NOT HAS_STB) +if (HAS_STB) add_cli_executable(image_converter) add_python_binding(limage_converter) add_markdown_docs(image_converter "cli;python" "preprocessing") From 19f5766f12c4f24dbec1573a7902b172badb58a5 Mon Sep 17 00:00:00 2001 From: jeffin143 Date: Thu, 19 Mar 2020 00:05:24 +0530 Subject: [PATCH 062/111] HAS_STB DOESN'T WORK --- src/mlpack/methods/preprocess/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mlpack/methods/preprocess/CMakeLists.txt b/src/mlpack/methods/preprocess/CMakeLists.txt index 4faccbf6d6..48d0378614 100644 --- a/src/mlpack/methods/preprocess/CMakeLists.txt +++ b/src/mlpack/methods/preprocess/CMakeLists.txt @@ -42,7 +42,7 @@ add_cli_executable(preprocess_scale) add_python_binding(preprocess_scale) add_markdown_docs(preprocess_scale "cli;python" "preprocessing") -if (HAS_STB) +if (NOT HAS_STB) add_cli_executable(image_converter) add_python_binding(limage_converter) add_markdown_docs(image_converter "cli;python" "preprocessing") From 0524c11b40b0cb458035aa85fd6aa28c9f21d6ad Mon Sep 17 00:00:00 2001 From: jeffin143 Date: Fri, 20 Mar 2020 22:06:37 +0530 Subject: [PATCH 063/111] Removed tranpose option --- CMakeLists.txt | 1 - src/mlpack/core/data/load.hpp | 9 ++-- src/mlpack/core/data/load_image.cpp | 8 +-- src/mlpack/core/data/load_image_impl.hpp | 12 ++--- src/mlpack/core/data/save.hpp | 9 ++-- src/mlpack/core/data/save_image.cpp | 8 +-- src/mlpack/core/data/save_impl.hpp | 10 ++-- src/mlpack/methods/preprocess/CMakeLists.txt | 2 +- .../preprocess/image_converter_main.cpp | 15 ++---- src/mlpack/tests/image_load_test.cpp | 52 ++----------------- .../tests/main_tests/image_converter_test.cpp | 25 --------- 11 files changed, 29 insertions(+), 122 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 817a947b6e..4dc6d0beb8 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -328,7 +328,6 @@ if (NOT STB_IMAGE_FOUND) install(FILES "${CMAKE_BINARY_DIR}/deps/${STB_DIR}/stb_image.h" DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}") install(FILES "${CMAKE_BINARY_DIR}/deps/${STB_DIR}/stb_image_write.h" DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}") add_definitions(-DHAS_STB) - set(HAS_STB 1) set(STB_AVAILABLE "1") else () message(WARNING diff --git a/src/mlpack/core/data/load.hpp b/src/mlpack/core/data/load.hpp index 586ddfcf16..256163f72b 100644 --- a/src/mlpack/core/data/load.hpp +++ b/src/mlpack/core/data/load.hpp @@ -305,8 +305,7 @@ template bool Load(const std::string& filename, arma::Mat& matrix, ImageInfo& info, - const bool fatal = false, - const bool transpose = true); + const bool fatal = false); /** * Load the image file into the given matrix. @@ -322,15 +321,13 @@ template bool Load(const std::vector& files, arma::Mat& matrix, ImageInfo& info, - const bool fatal = false, - const bool transpose = true); + const bool fatal = false); // Implementation found in load_image.cpp. bool LoadImage(const std::string& filename, arma::Mat& matrix, ImageInfo& info, - const bool fatal = false, - const bool transpose = true); + const bool fatal = false); } // namespace data } // namespace mlpack diff --git a/src/mlpack/core/data/load_image.cpp b/src/mlpack/core/data/load_image.cpp index 53922628d5..2796e9a1f4 100644 --- a/src/mlpack/core/data/load_image.cpp +++ b/src/mlpack/core/data/load_image.cpp @@ -23,8 +23,7 @@ namespace data { bool LoadImage(const std::string& filename, arma::Mat& matrix, ImageInfo& info, - const bool fatal, - const bool transpose) + const bool fatal) { unsigned char* image; @@ -49,8 +48,6 @@ bool LoadImage(const std::string& filename, return false; } - stbi_set_flip_vertically_on_load(transpose); - // Temporary variables needed as stb_image.h supports int parameters. int tempWidth, tempHeight, tempChannels; @@ -106,8 +103,7 @@ namespace data { bool LoadImage(const std::string& /* filename */, arma::Mat& /* matrix */, ImageInfo& /* info */, - const bool fatal, - const bool transpose) + const bool fatal) { if (fatal) { diff --git a/src/mlpack/core/data/load_image_impl.hpp b/src/mlpack/core/data/load_image_impl.hpp index 43540504e4..b5be00861b 100644 --- a/src/mlpack/core/data/load_image_impl.hpp +++ b/src/mlpack/core/data/load_image_impl.hpp @@ -24,15 +24,14 @@ template bool Load(const std::string& filename, arma::Mat& matrix, ImageInfo& info, - const bool fatal, - const bool transpose) + const bool fatal) { Timer::Start("loading_image"); // STB loads into unsigned char matrices, so we may have to convert once // loaded. arma::Mat tempMatrix; - const bool result = LoadImage(filename, tempMatrix, info, fatal, transpose); + const bool result = LoadImage(filename, tempMatrix, info, fatal); // If fatal is true, then the program will have already thrown an exception. if (!result) @@ -51,8 +50,7 @@ template bool Load(const std::vector& files, arma::Mat& matrix, ImageInfo& info, - const bool fatal, - const bool transpose) + const bool fatal) { if (files.size() == 0) { @@ -68,7 +66,7 @@ bool Load(const std::vector& files, } arma::Mat img; - bool status = LoadImage(files[0], img, info, fatal, transpose); + bool status = LoadImage(files[0], img, info, fatal); if (!status) return false; @@ -82,7 +80,7 @@ bool Load(const std::vector& files, { arma::Mat colImg(tmpMatrix.colptr(i), tmpMatrix.n_rows, 1, false, true); - status = LoadImage(files[i], colImg, info, fatal, transpose); + status = LoadImage(files[i], colImg, info, fatal); if (!status) return false; diff --git a/src/mlpack/core/data/save.hpp b/src/mlpack/core/data/save.hpp index de1860d0ac..0fb3fe8438 100644 --- a/src/mlpack/core/data/save.hpp +++ b/src/mlpack/core/data/save.hpp @@ -105,8 +105,7 @@ template bool Save(const std::string& filename, arma::Mat& matrix, ImageInfo& info, - const bool fatal = false, - const bool transpose = true); + const bool fatal = false); /** * Save the image file from the given matrix. @@ -122,8 +121,7 @@ template bool Save(const std::vector& files, arma::Mat& matrix, ImageInfo& info, - const bool fatal = false, - const bool transpose = true); + const bool fatal = false); /** * Helper function to save files. Implementation in save_image.cpp. @@ -131,8 +129,7 @@ bool Save(const std::vector& files, bool SaveImage(const std::string& filename, arma::Mat& image, ImageInfo& info, - const bool fatal = false, - const bool transpose = true); + const bool fatal = false); } // namespace data } // namespace mlpack diff --git a/src/mlpack/core/data/save_image.cpp b/src/mlpack/core/data/save_image.cpp index b97cda57e8..c1802073e4 100644 --- a/src/mlpack/core/data/save_image.cpp +++ b/src/mlpack/core/data/save_image.cpp @@ -22,8 +22,7 @@ namespace data { bool SaveImage(const std::string& filename, arma::Mat& image, ImageInfo& info, - const bool fatal, - const bool transpose) + const bool fatal) { // Check to see if the file type is supported. if (!ImageFormatSupported(filename, true)) @@ -55,8 +54,6 @@ bool SaveImage(const std::string& filename, Log::Warn << "Only the first image will be saved!" << std::endl; } - stbi_flip_vertically_on_write(transpose); - bool status = false; unsigned char* imageMem = image.memptr(); @@ -116,8 +113,7 @@ namespace data { bool SaveImage(const std::string& /* filename */, arma::Mat& /* image */, ImageInfo& /* info */, - const bool fatal, - const bool transpose) + const bool fatal) { if (fatal) { diff --git a/src/mlpack/core/data/save_impl.hpp b/src/mlpack/core/data/save_impl.hpp index 8750ac063d..516e0aaf93 100644 --- a/src/mlpack/core/data/save_impl.hpp +++ b/src/mlpack/core/data/save_impl.hpp @@ -295,14 +295,13 @@ template bool Save(const std::string& filename, arma::Mat& matrix, ImageInfo& info, - const bool fatal, - const bool transpose) + const bool fatal) { arma::Mat tmpMatrix = arma::conv_to>::from(matrix); // Call out to .cpp implementation. - return SaveImage(filename, tmpMatrix, info, fatal, transpose); + return SaveImage(filename, tmpMatrix, info, fatal); } // Image saving API for multiple files. @@ -310,8 +309,7 @@ template bool Save(const std::vector& files, arma::Mat& matrix, ImageInfo& info, - const bool fatal, - const bool transpose) + const bool fatal) { if (files.size() == 0) { @@ -336,7 +334,7 @@ bool Save(const std::vector& files, { arma::Mat colImg(matrix.colptr(i), matrix.n_rows, 1, false, true); - status &= Save(files[i], colImg, info, fatal, transpose); + status &= Save(files[i], colImg, info, fatal); } return status; diff --git a/src/mlpack/methods/preprocess/CMakeLists.txt b/src/mlpack/methods/preprocess/CMakeLists.txt index 48d0378614..2f9f4b0725 100644 --- a/src/mlpack/methods/preprocess/CMakeLists.txt +++ b/src/mlpack/methods/preprocess/CMakeLists.txt @@ -42,7 +42,7 @@ add_cli_executable(preprocess_scale) add_python_binding(preprocess_scale) add_markdown_docs(preprocess_scale "cli;python" "preprocessing") -if (NOT HAS_STB) +if (STB_AVAILABLE) add_cli_executable(image_converter) add_python_binding(limage_converter) add_markdown_docs(image_converter "cli;python" "preprocessing") diff --git a/src/mlpack/methods/preprocess/image_converter_main.cpp b/src/mlpack/methods/preprocess/image_converter_main.cpp index f8e26755b1..ac90eace66 100644 --- a/src/mlpack/methods/preprocess/image_converter_main.cpp +++ b/src/mlpack/methods/preprocess/image_converter_main.cpp @@ -33,7 +33,7 @@ PROGRAM_INFO("Image Converter", PRINT_PARAM_STRING("channel") + " of the images that needs to be loaded. " "\n" "There are other options too, that can be specified such as " + - PRINT_PARAM_STRING("quality") + " and " + PRINT_PARAM_STRING("transpose") + PRINT_PARAM_STRING("quality") + ".\n\n" + "You can also provide a dataset and save them as images using " + PRINT_PARAM_STRING("dataset") + " and " + PRINT_PARAM_STRING("save") + @@ -45,12 +45,7 @@ PROGRAM_INFO("Image Converter", " An example to save an image is :" + "\n\n" + PRINT_CALL("image_converter", "input", "X", "height", 256, "width", 256, - "channel", 3, "dataset", "Y", "save", true) + - "\n\n" + - " An example to load an image and also flipping it while loading is :" - + "\n\n" + - PRINT_CALL("image_converter", "input", "X", "height", 256, "width", 256, - "channel", 3, "output", "Y", "transpose", true), + "channel", 3, "dataset", "Y", "save", true), SEE_ALSO("@preprocess_binarize", "#preprocess_binarize"), SEE_ALSO("@preprocess_describe", "#preprocess_describe"), SEE_ALSO("@preprocess_imputer", "#preprocess_imputer")); @@ -67,7 +62,6 @@ PARAM_MATRIX_OUT("output", "Matrix to save images data to.", "o"); PARAM_INT_IN("quality", "Compression of the image if saved as jpg (0-100).", "q", 90); -PARAM_FLAG("transpose", "Loaded dataset to be transposed", "t"); PARAM_INT_IN("height", "Height of the images", "H", 256); PARAM_FLAG("save", "Save a dataset as images", "s"); @@ -119,7 +113,7 @@ static void mlpackMain() arma::mat out; if (!CLI::HasParam("save")) { - Load(fileNames, out, *info, true, !CLI::HasParam("transpose")); + Load(fileNames, out, *info, true); if (CLI::HasParam("output")) CLI::GetParam("output") = std::move(out); } @@ -130,8 +124,7 @@ static void mlpackMain() throw std::runtime_error("Please provide a input matrix to save " "images from"); } - Save(fileNames, CLI::GetParam ("dataset"), *info, true, - !CLI::HasParam("transpose")); + Save(fileNames, CLI::GetParam ("dataset"), *info, true); } if (CLI::HasParam("output_model")) CLI::GetParam ("output_model") = info; diff --git a/src/mlpack/tests/image_load_test.cpp b/src/mlpack/tests/image_load_test.cpp index 213c01e46e..396cb536da 100644 --- a/src/mlpack/tests/image_load_test.cpp +++ b/src/mlpack/tests/image_load_test.cpp @@ -69,27 +69,7 @@ BOOST_AUTO_TEST_CASE(SaveImageAPITest) BOOST_REQUIRE_EQUAL(im1.n_rows, im2.n_rows); for (size_t i = 0; i < im1.n_elem; ++i) BOOST_REQUIRE_EQUAL(im1[i], im2[i]); -} - -/** - * Test if the image is saved correctly using API for transpose. - */ -BOOST_AUTO_TEST_CASE(SaveImageTransposeAPITest) -{ - data::ImageInfo info(5, 5, 3, 90); - - arma::Mat im1; - size_t dimension = info.Width() * info.Height() * info.Channels(); - im1 = arma::randi>(dimension, 1); - BOOST_REQUIRE(data::Save("APITest.bmp", im1, info, false, false) == true); - - arma::Mat im2; - BOOST_REQUIRE(data::Load("APITest.bmp", im2, info, false, false) == true); - - BOOST_REQUIRE_EQUAL(im1.n_cols, im2.n_cols); - BOOST_REQUIRE_EQUAL(im1.n_rows, im2.n_rows); - for (size_t i = 0; i < im1.n_elem; ++i) - BOOST_REQUIRE_EQUAL(im1[i], im2[i]); + remove("APITest.bmp"); } /** @@ -101,34 +81,11 @@ BOOST_AUTO_TEST_CASE(LoadVectorImageAPITest) arma::Mat matrix; data::ImageInfo info; std::vector files = {"test_image.png", "test_image.png"}; - BOOST_REQUIRE(data::Load(files, matrix, info, false, - true) == true); + BOOST_REQUIRE(data::Load(files, matrix, info, false) == true); BOOST_REQUIRE_EQUAL(matrix.n_rows, 50 * 50 * 3); // width * height * channels. BOOST_REQUIRE_EQUAL(matrix.n_cols, 2); } -/** - * Test if the image is saved correctly using vector saving API for transpose. - */ -BOOST_AUTO_TEST_CASE(SaveImageVectorAPITest) -{ - data::ImageInfo info(5, 5, 3); - - arma::Mat im1; - size_t dimension = info.Width() * info.Height() * info.Channels(); - im1 = arma::randi>(dimension, 2); - std::vector files = {"APITest1.bmp", "APITest2.bmp"}; - BOOST_REQUIRE(data::Save(files, im1, info, false, false) == true); - - arma::Mat im2; - BOOST_REQUIRE(data::Load(files, im2, info, false, false) == true); - - BOOST_REQUIRE_EQUAL(im1.n_cols, im2.n_cols); - BOOST_REQUIRE_EQUAL(im1.n_rows, im2.n_rows); - for (size_t i = 0; i < im1.n_elem; ++i) - BOOST_REQUIRE_EQUAL(im1[i], im2[i]); -} - /** * Test if the image is saved correctly using API for arma mat. */ @@ -140,15 +97,16 @@ BOOST_AUTO_TEST_CASE(SaveImageMatAPITest) size_t dimension = info.Width() * info.Height() * info.Channels(); im1 = arma::randi>(dimension, 1); arma::mat input = arma::conv_to::from(im1); - BOOST_REQUIRE(Save("APITest.bmp", input, info, false, false) == true); + BOOST_REQUIRE(Save("APITest.bmp", input, info, false) == true); arma::mat output; - BOOST_REQUIRE(Load("APITest.bmp", output, info, false, false) == true); + BOOST_REQUIRE(Load("APITest.bmp", output, info, false) == true); BOOST_REQUIRE_EQUAL(input.n_cols, output.n_cols); BOOST_REQUIRE_EQUAL(input.n_rows, output.n_rows); for (size_t i = 0; i < input.n_elem; ++i) BOOST_REQUIRE_CLOSE(input[i], output[i], 1e-5); + remove("APITest.bmp"); } /** diff --git a/src/mlpack/tests/main_tests/image_converter_test.cpp b/src/mlpack/tests/main_tests/image_converter_test.cpp index bd34831c72..dac1924ad6 100644 --- a/src/mlpack/tests/main_tests/image_converter_test.cpp +++ b/src/mlpack/tests/main_tests/image_converter_test.cpp @@ -111,31 +111,6 @@ BOOST_AUTO_TEST_CASE(SavedModelTest) CheckMatrices(randomOutput, savedOutput); } -/** - * Check transpose option give two different output. - */ -BOOST_AUTO_TEST_CASE(TransposeTest) -{ - SetInputParam>("input", {"test_image.png", "test_image.png"}); - SetInputParam("height", 50); - SetInputParam("width", 50); - SetInputParam("channel", 3); - - mlpackMain(); - arma::mat normalOutput = CLI::GetParam("output"); - - SetInputParam>("input", {"test_image.png", "test_image.png"}); - SetInputParam("input_model", - CLI::GetParam("output_model")); - SetInputParam("transpose", true); - mlpackMain(); - arma::mat transposeOutput = CLI::GetParam("output"); - - CheckMatricesNotEqual(normalOutput, transposeOutput); - BOOST_REQUIRE_EQUAL(normalOutput.n_rows, transposeOutput.n_rows); - BOOST_REQUIRE_EQUAL(normalOutput.n_cols, transposeOutput.n_cols); -} - /** * Check whether binding throws error if height, width or channel are not * specified. From 5bbed7ea78a5f050751ba5df59bcb3990fb9df51 Mon Sep 17 00:00:00 2001 From: jeffin143 Date: Tue, 17 Mar 2020 23:25:01 +0530 Subject: [PATCH 064/111] Row major to Column Major --- .../bag_of_words_encoding_policy.hpp | 6 +- .../dictionary_encoding_policy.hpp | 6 +- .../tf_idf_encoding_policy.hpp | 8 ++- src/mlpack/tests/string_encoding_test.cpp | 61 ++++++++++--------- 4 files changed, 44 insertions(+), 37 deletions(-) diff --git a/src/mlpack/core/data/string_encoding_policies/bag_of_words_encoding_policy.hpp b/src/mlpack/core/data/string_encoding_policies/bag_of_words_encoding_policy.hpp index b6f164db36..8c7bc83036 100644 --- a/src/mlpack/core/data/string_encoding_policies/bag_of_words_encoding_policy.hpp +++ b/src/mlpack/core/data/string_encoding_policies/bag_of_words_encoding_policy.hpp @@ -49,7 +49,7 @@ class BagOfWordsEncodingPolicy const size_t /* maxNumTokens */, const size_t dictionarySize) { - output.zeros(datasetSize, dictionarySize); + output.zeros(dictionarySize, datasetSize); } /** @@ -76,6 +76,7 @@ class BagOfWordsEncodingPolicy /** * The function performs the bag of words encoding algorithm i.e. it writes * the encoded token to the output. + * Returns the encodings in column-major format. * * @tparam MatType The output matrix type. * @@ -91,13 +92,14 @@ class BagOfWordsEncodingPolicy const size_t /* col */) { // The labels are assigned sequentially starting from one. - output(row, value - 1) = 1; + output(value - 1, row) = 1; } /** * The function performs the bag of words encoding algorithm i.e. it writes * the encoded token to the output. * Overload function to accepted vector> as output type. + * Returns the encodings in row-major format. * * @param output Output matrix to store the encoded results. * @param value The encoded token. diff --git a/src/mlpack/core/data/string_encoding_policies/dictionary_encoding_policy.hpp b/src/mlpack/core/data/string_encoding_policies/dictionary_encoding_policy.hpp index e3de62e526..e907df5c1d 100644 --- a/src/mlpack/core/data/string_encoding_policies/dictionary_encoding_policy.hpp +++ b/src/mlpack/core/data/string_encoding_policies/dictionary_encoding_policy.hpp @@ -47,12 +47,13 @@ class DictionaryEncodingPolicy const size_t maxNumTokens, const size_t /* dictionarySize */) { - output.zeros(datasetSize, maxNumTokens); + output.zeros(maxNumTokens, datasetSize); } /** * The function performs the dictionary encoding algorithm i.e. it writes * the encoded token to the ouput. + * Returns the encodings in column-major format. * * @tparam MatType The output matrix type. * @@ -67,13 +68,14 @@ class DictionaryEncodingPolicy const size_t row, const size_t col) { - output(row, col) = value; + output(col, row) = value; } /** * The function performs the dictionary encoding algorithm i.e. it writes * the encoded token to the ouput. This is an overload function which saves * the result into the given vector to avoid padding. + * Returns the encodings in row-major format. * * @tparam OutputType Type of the output vector. * diff --git a/src/mlpack/core/data/string_encoding_policies/tf_idf_encoding_policy.hpp b/src/mlpack/core/data/string_encoding_policies/tf_idf_encoding_policy.hpp index ecba9f1a8d..eba42ee0ca 100644 --- a/src/mlpack/core/data/string_encoding_policies/tf_idf_encoding_policy.hpp +++ b/src/mlpack/core/data/string_encoding_policies/tf_idf_encoding_policy.hpp @@ -92,7 +92,7 @@ class TfIdfEncodingPolicy const size_t /* maxNumTokens */, const size_t dictionarySize) { - output.zeros(datasetSize, dictionarySize); + output.zeros(dictionarySize, datasetSize); } /** @@ -119,6 +119,7 @@ class TfIdfEncodingPolicy /** * The function performs the TfIdf encoding algorithm i.e. it writes * the encoded token to the output. + * Returns the encodings in column-major format. * * @tparam MatType The output matrix type. * @@ -139,9 +140,9 @@ class TfIdfEncodingPolicy const typename MatType::elem_type idf = InverseDocumentFrequency( - output.n_rows, numContainingStrings[value]); + output.n_cols, numContainingStrings[value]); - output(row, value - 1) = tf * idf; + output(value - 1, row) = tf * idf; } /** @@ -149,6 +150,7 @@ class TfIdfEncodingPolicy * the encoded token to the output. * Overloaded function to accept vector> as the output * type. + * Returns the encodings in row-major format. * * @tparam OutputType Type of the output vector. * diff --git a/src/mlpack/tests/string_encoding_test.cpp b/src/mlpack/tests/string_encoding_test.cpp index 8e4b8107b9..c8f3821c4e 100644 --- a/src/mlpack/tests/string_encoding_test.cpp +++ b/src/mlpack/tests/string_encoding_test.cpp @@ -56,6 +56,17 @@ static vector stringEncodingUtf8Input = { "\xE2\x93\x82\xE2\x93\x81\xE2\x93\x85\xE2\x92\xB6\xE2\x92\xB8\xE2\x93\x80" }; +/** + * Function used to compare two vectors. + */ +template +void checkVector(vector> A, vector> B) +{ + for (size_t i = 0; i < A.size(); i++) + for (size_t j = 0; j < A[i].size(); j++) + BOOST_REQUIRE_CLOSE(A[i][j], B[i][j], 1e-12); +} + /** * Test the dictionary encoding algorithm. */ @@ -90,7 +101,7 @@ BOOST_AUTO_TEST_CASE(DictionaryEncodingTest) 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 } }; - CheckMatrices(output, expected); + CheckMatrices(output, expected.t()); } /** @@ -124,7 +135,7 @@ BOOST_AUTO_TEST_CASE(UnicodeDictionaryEncodingTest) { 5, 2, 3, 5, 4 } }; - CheckMatrices(output, expected); + CheckMatrices(output, expected.t()); } /** @@ -244,7 +255,7 @@ BOOST_AUTO_TEST_CASE(DictionaryEncodingIndividualCharactersTest) { 2, 4, 3, 2, 4, 3, 5 }, { 1, 2, 4, 0, 0, 0, 0 } }; - CheckMatrices(output, target); + CheckMatrices(output, target.t()); } /** @@ -551,7 +562,7 @@ BOOST_AUTO_TEST_CASE(BagOfWordsEncodingTest) { 1, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 } }; - CheckMatrices(output, expected); + CheckMatrices(output, expected.t()); } /** @@ -613,7 +624,7 @@ BOOST_AUTO_TEST_CASE(BagOfWordsEncodingIndividualCharactersTest) { 1, 1, 0, 1, 0 } }; - CheckMatrices(output, target); + CheckMatrices(output, target.t()); } /** @@ -686,7 +697,7 @@ BOOST_AUTO_TEST_CASE(RawCountSmoothIdfEncodingTest) 1.69314718055995, 1.69314718055995, 1.69314718055995, 1.69314718055995, 1.69314718055995, 1.69314718055995, 1.69314718055995 } }; - CheckMatrices(output, expected, 1e-12); + CheckMatrices(output, expected.t(), 1e-12); } /** @@ -735,9 +746,7 @@ BOOST_AUTO_TEST_CASE(VectorRawCountSmoothIdfEncodingTest) 1.69314718055995, 1.69314718055995, 1.69314718055995, 1.69314718055995, 1.69314718055995, 1.69314718055995, 1.69314718055995 } }; - for (size_t i = 0; i < expected.size(); i++) - for (size_t j = 0; j < expected[i].size(); j++) - BOOST_REQUIRE_CLOSE(expected[i][j], output[i][j], 1e-12); + checkVector(output, expected); } /** @@ -762,7 +771,7 @@ BOOST_AUTO_TEST_CASE(RawCountSmoothIdfEncodingIndividualCharactersTest) { 0, 2, 2.5753641449035616, 2.5753641449035616, 1.6931471805599454 }, { 1.2876820724517808, 1, 0, 1.2876820724517808, 0 } }; - CheckMatrices(output, target, 1e-12); + CheckMatrices(output, target.t(), 1e-12); } /** @@ -787,9 +796,7 @@ BOOST_AUTO_TEST_CASE(VectorRawCountSmoothIdfEncodingIndividualCharactersTest) { 0, 2, 2.5753641449035616, 2.5753641449035616, 1.6931471805599454 }, { 1.2876820724517808, 1, 0, 1.2876820724517808, 0 } }; - for (size_t i = 0; i < expected.size(); i++) - for (size_t j = 0; j < expected[i].size(); j++) - BOOST_REQUIRE_CLOSE(expected[i][j], output[i][j], 1e-12); + checkVector(output, expected); } /** @@ -837,7 +844,7 @@ BOOST_AUTO_TEST_CASE(TfIdfRawCountEncodingTest) 2.09861228866811, 2.09861228866811, 2.09861228866811, 2.09861228866811, 2.09861228866811, 2.09861228866811, 2.09861228866811 } }; - CheckMatrices(output, expected, 1e-12); + CheckMatrices(output, expected.t(), 1e-12); } /** @@ -886,9 +893,7 @@ BOOST_AUTO_TEST_CASE(VectorTfIdfRawCountEncodingTest) 2.09861228866811, 2.09861228866811, 2.09861228866811, 2.09861228866811, 2.09861228866811, 2.09861228866811, 2.09861228866811 } }; - for (size_t i = 0; i < expected.size(); i++) - for (size_t j = 0; j < expected[i].size(); j++) - BOOST_REQUIRE_CLOSE(expected[i][j], output[i][j], 1e-12); + checkVector(output, expected); } /** @@ -914,7 +919,7 @@ BOOST_AUTO_TEST_CASE(RawcountTfIdfEncodingIndividualCharactersTest) { 0, 2, 2.8109302162163288, 2.8109302162163288, 2.0986122886681100 }, { 1.4054651081081644, 1, 0, 1.4054651081081644, 0 } }; - CheckMatrices(output, target, 1e-12); + CheckMatrices(output, target.t(), 1e-12); } /** @@ -941,9 +946,7 @@ BOOST_AUTO_TEST_CASE(VectorRawcountEncodingIndividualCharactersTest) { 0, 2, 2.8109302162163288, 2.8109302162163288, 2.0986122886681100 }, { 1.4054651081081644, 1, 0, 1.4054651081081644, 0 } }; - for (size_t i = 0; i < expected.size(); i++) - for (size_t j = 0; j < expected[i].size(); j++) - BOOST_REQUIRE_CLOSE(expected[i][j], output[i][j], 1e-12); + checkVector(output, expected); } /** @@ -969,7 +972,7 @@ BOOST_AUTO_TEST_CASE(BinarySmoothIdfEncodingIndividualCharactersTest) { 0, 1, 1.2876820724517808, 1.2876820724517808, 1.6931471805599454 }, { 1.2876820724517808, 1, 0, 1.2876820724517808, 0 } }; - CheckMatrices(output, target, 1e-12); + CheckMatrices(output, target.t(), 1e-12); } /** @@ -995,9 +998,7 @@ BOOST_AUTO_TEST_CASE(VectorBnarySmoothIdfEncodingIndividualCharactersTest) { 0, 1, 1.2876820724517808, 1.2876820724517808, 1.6931471805599454 }, { 1.2876820724517808, 1, 0, 1.2876820724517808, 0 } }; - for (size_t i = 0; i < expected.size(); i++) - for (size_t j = 0; j < expected[i].size(); j++) - BOOST_REQUIRE_CLOSE(expected[i][j], output[i][j], 1e-12); + checkVector(output, expected); } /** @@ -1023,7 +1024,7 @@ BOOST_AUTO_TEST_CASE(BinaryTfIdfEncodingIndividualCharactersTest) { 0, 1, 1.4054651081081644, 1.4054651081081644, 2.0986122886681100 }, { 1.4054651081081644, 1, 0, 1.4054651081081644, 0 } }; - CheckMatrices(output, target, 1e-12); + CheckMatrices(output, target.t(), 1e-12); } /** @@ -1050,7 +1051,7 @@ BOOST_AUTO_TEST_CASE(SublinearSmoothIdfEncodingIndividualCharactersTest) 1.6931471805599454 }, { 1.2876820724517808, 1, 0, 1.2876820724517808, 0 } }; - CheckMatrices(output, target, 1e-12); + CheckMatrices(output, target.t(), 1e-12); } /** @@ -1077,7 +1078,7 @@ BOOST_AUTO_TEST_CASE(SublinearTfIdfEncodingIndividualCharactersTest) 2.0986122886681100 }, { 1.4054651081081644, 1, 0, 1.4054651081081644, 0 } }; - CheckMatrices(output, target, 1e-12); + CheckMatrices(output, target.t(), 1e-12); } /** @@ -1104,7 +1105,7 @@ BOOST_AUTO_TEST_CASE(TermFrequencySmoothIdfEncodingIndividualCharactersTest) 0.2418781686514208 }, { 0.4292273574839269, 0.3333333333333333, 0, 0.4292273574839269, 0 } }; - CheckMatrices(output, target, 1e-12); + CheckMatrices(output, target.t(), 1e-12); } /** @@ -1131,7 +1132,7 @@ BOOST_AUTO_TEST_CASE(TermFrequencyTfIdfEncodingIndividualCharactersTest) 0.2998017555240157 }, { 0.4684883693693881, 0.3333333333333333, 0, 0.4684883693693881, 0 } }; - CheckMatrices(output, target, 1e-12); + CheckMatrices(output, target.t(), 1e-12); } /** From 10b26d26277982a719b58512c7233552d4dd5c4f Mon Sep 17 00:00:00 2001 From: Mikhail Lozhnikov Date: Wed, 25 Mar 2020 02:26:20 +0300 Subject: [PATCH 065/111] Added detailed description for the string encoding output format. Described the column-major case and the row-major case. --- src/mlpack/core/data/string_encoding.hpp | 21 ++++- .../bag_of_words_encoding_policy.hpp | 54 ++++++----- .../dictionary_encoding_policy.hpp | 37 ++++---- .../tf_idf_encoding_policy.hpp | 90 ++++++++++--------- src/mlpack/tests/string_encoding_test.cpp | 30 ++++--- 5 files changed, 134 insertions(+), 98 deletions(-) diff --git a/src/mlpack/core/data/string_encoding.hpp b/src/mlpack/core/data/string_encoding.hpp index ac9d67c366..db5896576c 100644 --- a/src/mlpack/core/data/string_encoding.hpp +++ b/src/mlpack/core/data/string_encoding.hpp @@ -24,7 +24,8 @@ namespace data { /** * The class translates a set of strings into numbers using various encoding - * algorithms. + * algorithms. The encoder writes data either in the column-major order or + * in the row-major order depending on the output data type. * * @tparam EncodingPolicyType Type of the encoding algorithm itself. * @tparam DictionaryType Type of the dictionary. @@ -90,7 +91,13 @@ class StringEncoding void Clear(); /** - * Encode the given text and write the result to the given output. + * Encode the given text and write the result to the given output. The encoder + * writes data in the column-major order or in the row-major order depending + * on the output data type. + * + * If the output type is either arma::mat or arma::sp_mat then the function + * writes it in the column-major order. If the output type is 2D std::vector + * then the function writes it in the row major order. * * @tparam OutputType Type of the output container. The function supports * the following types: arma::mat, arma::sp_mat, @@ -132,7 +139,12 @@ class StringEncoding private: /** * A helper function to encode the given text and write the result to - * the given output. + * the given output. The encoder writes data in the column-major order or + * in the row-major order depending on the output data type. + * + * If the output type is either arma::mat or arma::sp_mat then the function + * writes it in the column-major order. If the output type is 2D std::vector + * then the function writes it in the row major order. * * @tparam OutputType Type of the output container. The function supports * the following types: arma::mat, arma::sp_mat, @@ -162,7 +174,8 @@ class StringEncoding /** * A helper function to encode the given text and write the result to * the given output. This is an optimized overload for policies that support - * the one pass encoding algorithm. + * the one pass encoding algorithm. The encoder writes data in the row-major + * order. * * @tparam TokenizerType Type of the tokenizer. * @tparam PolicyType The type of the encoding policy. It has to be diff --git a/src/mlpack/core/data/string_encoding_policies/bag_of_words_encoding_policy.hpp b/src/mlpack/core/data/string_encoding_policies/bag_of_words_encoding_policy.hpp index 8c7bc83036..e3be0f5faa 100644 --- a/src/mlpack/core/data/string_encoding_policies/bag_of_words_encoding_policy.hpp +++ b/src/mlpack/core/data/string_encoding_policies/bag_of_words_encoding_policy.hpp @@ -27,13 +27,16 @@ namespace data { * of tokens. If an item of the dataset has the i-th token, then the i-th * coordinate of the corresponding vector is equal to 1, otherwise it's equal to * zero. The order in which the tokens are labeled is defined by the dictionary - * used by the StringEncoding class. + * used by the StringEncoding class. The encoder writes data either in the + * column-major order or in the row-major order depending on the output data + * type. */ class BagOfWordsEncodingPolicy { public: /** - * The function initializes the output matrix. + * The function initializes the output matrix. The encoder writes data + * in the column-major order. * * @tparam MatType The output matrix type. * @@ -53,8 +56,10 @@ class BagOfWordsEncodingPolicy } /** - * The function initializes the output matrix. - * Overloaded function to store result in vector> + * The function initializes the output matrix. The encoder writes data + * in the row-major order. + * + * Overloaded function to store result in vector>. * * @tparam OutputType Type of the output vector. * @@ -73,59 +78,60 @@ class BagOfWordsEncodingPolicy output.resize(datasetSize, std::vector(dictionarySize, 0)); } - /** + /** * The function performs the bag of words encoding algorithm i.e. it writes - * the encoded token to the output. - * Returns the encodings in column-major format. + * the encoded token to the output. The encoder writes data in the + * column-major order. * * @tparam MatType The output matrix type. * * @param output Output matrix to store the encoded results (sp_mat or mat). * @param value The encoded token. - * @param row The row number at which the encoding is performed. - * @param col The token index in the row. + * @param line The line number at which the encoding is performed. + * @param index The token index in the line. */ template static void Encode(MatType& output, const size_t value, - const size_t row, - const size_t /* col */) + const size_t line, + const size_t /* index */) { // The labels are assigned sequentially starting from one. - output(value - 1, row) = 1; + output(value - 1, line) = 1; } - /** + /** * The function performs the bag of words encoding algorithm i.e. it writes - * the encoded token to the output. + * the encoded token to the output. The encoder writes data in the + * row-major order. + * * Overload function to accepted vector> as output type. - * Returns the encodings in row-major format. * * @param output Output matrix to store the encoded results. * @param value The encoded token. - * @param row The row number at which the encoding is performed. - * @param col The row token number at which the encoding is performed. + * @param line The line number at which the encoding is performed. + * @param index The line token number at which the encoding is performed. * @tparam OutputType The type of output vector. */ template static void Encode(std::vector>& output, const size_t value, - const size_t row, - const size_t /* col */) + const size_t line, + const size_t /* index */) { // The labels are assigned sequentially starting from one. - output[row][value - 1] = 1; + output[line][value - 1] = 1; } /** * The function is not used by the bag of words encoding policy. * - * @param row The row number at which the encoding is performed. - * @param col The token sequence number in the row. + * @param line The line number at which the encoding is performed. + * @param index The token sequence number in the line. * @param value The encoded token. */ - static void PreprocessToken(size_t /* row */, - size_t /* col */, + static void PreprocessToken(size_t /* line */, + size_t /* index */, size_t /* value */) { } diff --git a/src/mlpack/core/data/string_encoding_policies/dictionary_encoding_policy.hpp b/src/mlpack/core/data/string_encoding_policies/dictionary_encoding_policy.hpp index e907df5c1d..4b26860abd 100644 --- a/src/mlpack/core/data/string_encoding_policies/dictionary_encoding_policy.hpp +++ b/src/mlpack/core/data/string_encoding_policies/dictionary_encoding_policy.hpp @@ -25,13 +25,16 @@ namespace data { * The encoder assigns a positive integer number to each unique token and treats * the dataset as categorical. The numbers are assigned sequentially starting * from one. The order in which the tokens are labeled is defined by - * the dictionary used by the StringEncoding class. + * the dictionary used by the StringEncoding class. The encoder writes data + * either in the column-major order or in the row-major order depending on + * the output data type. */ class DictionaryEncodingPolicy { public: /** - * The function initializes the output matrix. + * The function initializes the output matrix. The encoder writes data + * in the column-major order. * * @tparam MatType The output matrix type. * @@ -50,32 +53,32 @@ class DictionaryEncodingPolicy output.zeros(maxNumTokens, datasetSize); } - /** + /** * The function performs the dictionary encoding algorithm i.e. it writes - * the encoded token to the ouput. - * Returns the encodings in column-major format. + * the encoded token to the ouput. The encoder writes data in the + * column-major order. * * @tparam MatType The output matrix type. * * @param output Output matrix to store the encoded results (sp_mat or mat). * @param value The encoded token. - * @param row The row number at which the encoding is performed. - * @param col The token index in the row. + * @param line The line number at which the encoding is performed. + * @param index The token index in the line. */ template static void Encode(MatType& output, const size_t value, - const size_t row, - const size_t col) + const size_t line, + const size_t index) { - output(col, row) = value; + output(index, line) = value; } - /** + /** * The function performs the dictionary encoding algorithm i.e. it writes * the encoded token to the ouput. This is an overload function which saves - * the result into the given vector to avoid padding. - * Returns the encodings in row-major format. + * the result into the given vector to avoid padding. The encoder writes data + * in the row-major order. * * @tparam OutputType Type of the output vector. * @@ -91,12 +94,12 @@ class DictionaryEncodingPolicy /** * The function is not used by the dictionary encoding policy. * - * @param row The row number at which the encoding is performed. - * @param col The token sequence number in the row. + * @param line The line number at which the encoding is performed. + * @param index The token sequence number in the line. * @param value The encoded token. */ - static void PreprocessToken(const size_t /* row */, - const size_t /* col */, + static void PreprocessToken(const size_t /* line */, + const size_t /* index */, const size_t /* value */) { } diff --git a/src/mlpack/core/data/string_encoding_policies/tf_idf_encoding_policy.hpp b/src/mlpack/core/data/string_encoding_policies/tf_idf_encoding_policy.hpp index eba42ee0ca..2ad8b15e9f 100644 --- a/src/mlpack/core/data/string_encoding_policies/tf_idf_encoding_policy.hpp +++ b/src/mlpack/core/data/string_encoding_policies/tf_idf_encoding_policy.hpp @@ -28,7 +28,8 @@ namespace data { * multiplied by inverse document frequency (idf). * The encoder assigns the corresponding tf-idf value to each token. The order * in which the tokens are labeled is defined by the dictionary used by the - * StringEncoding class. + * StringEncoding class. The encoder writes data either in the column-major + * order or in the row-major order depending on the output data type. */ class TfIdfEncodingPolicy { @@ -76,7 +77,8 @@ class TfIdfEncodingPolicy { } /** - * The function initializes the output matrix. + * The function initializes the output matrix. The encoder writes data + * in the row-major order. * * @tparam MatType The output matrix type. * @@ -96,7 +98,9 @@ class TfIdfEncodingPolicy } /** - * The function initializes the output matrix. + * The function initializes the output matrix. The encoder writes data + * in the row-major order. + * * Overloaded function to store result in vector>. * * @tparam OutputType Type of the output vector. @@ -118,86 +122,87 @@ class TfIdfEncodingPolicy /** * The function performs the TfIdf encoding algorithm i.e. it writes - * the encoded token to the output. - * Returns the encodings in column-major format. + * the encoded token to the output. The encoder writes data in the + * column-major order. * * @tparam MatType The output matrix type. * * @param output Output matrix to store the encoded results (sp_mat or mat). * @param value The encoded token. - * @param row The row number at which the encoding is performed. - * @param col The token index in the row. + * @param line The line number at which the encoding is performed. + * @param index The token index in the line. */ template void Encode(MatType& output, const size_t value, - const size_t row, - const size_t /* col */) + const size_t line, + const size_t /* index */) { const typename MatType::elem_type tf = TermFrequency( - tokensFrequences[row][value], rowsSizes[row]); + tokensFrequences[line][value], linesSizes[line]); const typename MatType::elem_type idf = InverseDocumentFrequency( output.n_cols, numContainingStrings[value]); - output(value - 1, row) = tf * idf; + output(value - 1, line) = tf * idf; } /** * The function performs the TfIdf encoding algorithm i.e. it writes - * the encoded token to the output. + * the encoded token to the output. The encoder writes data in the + * row-major order. + * * Overloaded function to accept vector> as the output * type. - * Returns the encodings in row-major format. * * @tparam OutputType Type of the output vector. * * @param output Output matrix to store the encoded results. * @param value The encoded token. - * @param row The row number at which the encoding is performed. - * @param col The token index in the row. + * @param line The line number at which the encoding is performed. + * @param index The token index in the line. */ template void Encode(std::vector >& output, const size_t value, - const size_t row, - const size_t /* col */) + const size_t line, + const size_t /* index */) { const OutputType tf = TermFrequency( - tokensFrequences[row][value], rowsSizes[row]); + tokensFrequences[line][value], linesSizes[line]); const OutputType idf = InverseDocumentFrequency( output.size(), numContainingStrings[value]); - output[row][value - 1] = tf * idf; + output[line][value - 1] = tf * idf; } /* * The function calculates the necessary statistics for the purpose * of the tf-idf algorithm during the first pass through the dataset. * - * @param row The row number at which the encoding is performed. - * @param col The token sequence number in the row. + * @param line The line number at which the encoding is performed. + * @param index The token sequence number in the line. * @param value The encoded token. */ - void PreprocessToken(const size_t row, - const size_t /* col */, + void PreprocessToken(const size_t line, + const size_t /* index */, const size_t value) { - if (row >= tokensFrequences.size()) + if (line >= tokensFrequences.size()) { - rowsSizes.resize(row + 1); - tokensFrequences.resize(row + 1); + linesSizes.resize(line + 1); + tokensFrequences.resize(line + 1); } - tokensFrequences[row][value]++; + tokensFrequences[line][value]++; - if (tokensFrequences[row][value] == 1) + if (tokensFrequences[line][value] == 1) numContainingStrings[value]++; - rowsSizes[row]++; + linesSizes[line]++; } //! Return token frequencies. @@ -221,10 +226,10 @@ class TfIdfEncodingPolicy return numContainingStrings; } - //! Return the rows sizes. - const std::vector& RowsSizes() const { return rowsSizes; } - //! Modify the rows sizes. - std::vector& RowsSizes() { return rowsSizes; } + //! Return the lines sizes. + const std::vector& LinesSizes() const { return linesSizes; } + //! Modify the lines sizes. + std::vector& LinesSizes() { return linesSizes; } //! Return the term frequency type. TfTypes TfType() const { return tfType; } @@ -252,8 +257,9 @@ class TfIdfEncodingPolicy * * @tparam ValueType Type of the returned value. * - * @param numOccurrences The number of the given token occurrences in the row. - * @param numTokens The total number of tokens in the row. + * @param numOccurrences The number of the given token occurrences in + * the line. + * @param numTokens The total number of tokens in the line. */ template ValueType TermFrequency(const size_t numOccurrences, @@ -280,36 +286,36 @@ class TfIdfEncodingPolicy * * @tparam ValueType Type of the returned value. * - * @param totalNumRows The total number of strings in the input dataset. + * @param totalNumLines The total number of strings in the input dataset. * @param numOccurrences The number of strings in the input dataset * which contain the current token. */ template - ValueType InverseDocumentFrequency(const size_t totalNumRows, + ValueType InverseDocumentFrequency(const size_t totalNumLines, const size_t numOccurrences) { if (smoothIdf) { - return std::log(static_cast(totalNumRows + 1) / + return std::log(static_cast(totalNumLines + 1) / (1 + numOccurrences)) + 1.0; } else { - return std::log(static_cast(totalNumRows) / + return std::log(static_cast(totalNumLines) / numOccurrences) + 1.0; } } private: - //! Used to store the total number of tokens for each row. + //! Used to store the total number of tokens for each line. std::vector> tokensFrequences; /** * Used to store the number of strings which contain a token depending * on the given token. */ std::unordered_map numContainingStrings; - //! Used to store the number of tokens in each row. - std::vector rowsSizes; + //! Used to store the number of tokens in each line. + std::vector linesSizes; //! Type of the term frequency scheme. TfTypes tfType; //! Indicates whether the idf scheme is smooth or not. diff --git a/src/mlpack/tests/string_encoding_test.cpp b/src/mlpack/tests/string_encoding_test.cpp index c8f3821c4e..baec566bcb 100644 --- a/src/mlpack/tests/string_encoding_test.cpp +++ b/src/mlpack/tests/string_encoding_test.cpp @@ -57,14 +57,22 @@ static vector stringEncodingUtf8Input = { }; /** - * Function used to compare two vectors. + * Check the values of two 2D vectors. */ -template -void checkVector(vector> A, vector> B) +template +void CheckVectors(const vector>& a, + const vector>& b, + const ValueType tolerance = 1e-5) { - for (size_t i = 0; i < A.size(); i++) - for (size_t j = 0; j < A[i].size(); j++) - BOOST_REQUIRE_CLOSE(A[i][j], B[i][j], 1e-12); + BOOST_REQUIRE_EQUAL(a.size(), b.size()); + + for (size_t i = 0; i < a.size(); i++) + { + BOOST_REQUIRE_EQUAL(a[i].size(), b[i].size()); + + for (size_t j = 0; j < a[i].size(); j++) + BOOST_REQUIRE_CLOSE(a[i][j], b[i][j], tolerance); + } } /** @@ -746,7 +754,7 @@ BOOST_AUTO_TEST_CASE(VectorRawCountSmoothIdfEncodingTest) 1.69314718055995, 1.69314718055995, 1.69314718055995, 1.69314718055995, 1.69314718055995, 1.69314718055995, 1.69314718055995 } }; - checkVector(output, expected); + CheckVectors(output, expected, 1e-12); } /** @@ -796,7 +804,7 @@ BOOST_AUTO_TEST_CASE(VectorRawCountSmoothIdfEncodingIndividualCharactersTest) { 0, 2, 2.5753641449035616, 2.5753641449035616, 1.6931471805599454 }, { 1.2876820724517808, 1, 0, 1.2876820724517808, 0 } }; - checkVector(output, expected); + CheckVectors(output, expected, 1e-12); } /** @@ -893,7 +901,7 @@ BOOST_AUTO_TEST_CASE(VectorTfIdfRawCountEncodingTest) 2.09861228866811, 2.09861228866811, 2.09861228866811, 2.09861228866811, 2.09861228866811, 2.09861228866811, 2.09861228866811 } }; - checkVector(output, expected); + CheckVectors(output, expected, 1e-12); } /** @@ -946,7 +954,7 @@ BOOST_AUTO_TEST_CASE(VectorRawcountEncodingIndividualCharactersTest) { 0, 2, 2.8109302162163288, 2.8109302162163288, 2.0986122886681100 }, { 1.4054651081081644, 1, 0, 1.4054651081081644, 0 } }; - checkVector(output, expected); + CheckVectors(output, expected, 1e-12); } /** @@ -998,7 +1006,7 @@ BOOST_AUTO_TEST_CASE(VectorBnarySmoothIdfEncodingIndividualCharactersTest) { 0, 1, 1.2876820724517808, 1.2876820724517808, 1.6931471805599454 }, { 1.2876820724517808, 1, 0, 1.2876820724517808, 0 } }; - checkVector(output, expected); + CheckVectors(output, expected, 1e-12); } /** From 90898dbb960ea42aa37f7344b4d599dd45abec53 Mon Sep 17 00:00:00 2001 From: AndreiMihalea Date: Sat, 28 Mar 2020 14:52:02 +0200 Subject: [PATCH 066/111] Started working on changing the input in margin ranking loss --- .../loss_functions/margin_ranking_loss.hpp | 19 +++++++------------ .../margin_ranking_loss_impl.hpp | 6 ++---- 2 files changed, 9 insertions(+), 16 deletions(-) diff --git a/src/mlpack/methods/ann/loss_functions/margin_ranking_loss.hpp b/src/mlpack/methods/ann/loss_functions/margin_ranking_loss.hpp index ec9c78b11f..ae21d38d50 100644 --- a/src/mlpack/methods/ann/loss_functions/margin_ranking_loss.hpp +++ b/src/mlpack/methods/ann/loss_functions/margin_ranking_loss.hpp @@ -43,35 +43,30 @@ class MarginRankingLoss * a value of 1 in the label means the first input should be ranked higher * and a value of -1 means the second input should be ranked higher. * - * @param input1 First input data used for evaluating the specified function. - * @param input2 Second input data used for evaluating the specified function. + * @param input Concatenation of the two inputs for evaluating the specified + * function. * @param target The label vector which contains -1 or 1 values. */ template < - typename FirstInputType, - typename SecondInputType, + typename InputType, typename TargetType > - double Forward(const FirstInputType& input1, - const SecondInputType& input2, + double Forward(const InputType& input, const TargetType& target); /** * Ordinary feed backward pass of a neural network. * - * @param input1 The propagated first input activation. - * @param input2 The propagated second input activation. + * @param input The propagated concatenated input activation. * @param target The label vector which contains -1 or 1 values. * @param output The calculated error. */ template < - typename FirstInputType, - typename SecondInputType, + typename InputType, typename TargetType, typename OutputType > - void Backward(const FirstInputType& input1, - const SecondInputType& input2, + void Backward(const InputType& input1, const TargetType& target, OutputType& output); diff --git a/src/mlpack/methods/ann/loss_functions/margin_ranking_loss_impl.hpp b/src/mlpack/methods/ann/loss_functions/margin_ranking_loss_impl.hpp index 5e908dfede..fc06881c36 100644 --- a/src/mlpack/methods/ann/loss_functions/margin_ranking_loss_impl.hpp +++ b/src/mlpack/methods/ann/loss_functions/margin_ranking_loss_impl.hpp @@ -27,13 +27,11 @@ MarginRankingLoss::MarginRankingLoss( template template < - typename FirstInputType, - typename SecondInputType, + typename InputType, typename TargetType > double MarginRankingLoss::Forward( - const FirstInputType& input1, - const SecondInputType& input2, + const InputType& input, const TargetType& target) { return arma::accu(arma::max(arma::zeros(size(target)), From c6c2f9e85e6831aec0c9b9fe1f4334a8957f7622 Mon Sep 17 00:00:00 2001 From: AndreiMihalea Date: Sat, 28 Mar 2020 18:37:47 +0200 Subject: [PATCH 067/111] Merged the two inputs into a single one for the margin ranking loss --- .../ann/loss_functions/margin_ranking_loss_impl.hpp | 12 ++++++++---- src/mlpack/tests/loss_functions_test.cpp | 12 +++++++----- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/src/mlpack/methods/ann/loss_functions/margin_ranking_loss_impl.hpp b/src/mlpack/methods/ann/loss_functions/margin_ranking_loss_impl.hpp index fc06881c36..9920b07085 100644 --- a/src/mlpack/methods/ann/loss_functions/margin_ranking_loss_impl.hpp +++ b/src/mlpack/methods/ann/loss_functions/margin_ranking_loss_impl.hpp @@ -34,23 +34,27 @@ double MarginRankingLoss::Forward( const InputType& input, const TargetType& target) { + int input_rows = input.n_rows; + const InputType& input1 = input.rows(0, input_rows / 2 - 1); + const InputType& input2 = input.rows(input_rows / 2, input_rows - 1); return arma::accu(arma::max(arma::zeros(size(target)), -target % (input1 - input2) + margin)) / target.n_cols; } template template < - typename FirstInputType, - typename SecondInputType, + typename InputType, typename TargetType, typename OutputType > void MarginRankingLoss::Backward( - const FirstInputType& input1, - const SecondInputType& input2, + const InputType& input, const TargetType& target, OutputType& output) { + int input_rows = input.n_rows; + const InputType& input1 = input.rows(0, input_rows / 2 - 1); + const InputType& input2 = input.rows(input_rows / 2, input_rows - 1); output = -target % (input1 - input2) + margin; output.elem(arma::find(output >= 0)).ones(); output.elem(arma::find(output < 0)).zeros(); diff --git a/src/mlpack/tests/loss_functions_test.cpp b/src/mlpack/tests/loss_functions_test.cpp index 8402212e2b..7deeb6b6e7 100644 --- a/src/mlpack/tests/loss_functions_test.cpp +++ b/src/mlpack/tests/loss_functions_test.cpp @@ -585,20 +585,21 @@ BOOST_AUTO_TEST_CASE(HingeEmbeddingLossTest) */ BOOST_AUTO_TEST_CASE(MarginRankingLossTest) { - arma::mat input1, input2, target, output; + arma::mat input, input1, input2, target, output; MarginRankingLoss<> module; // Test the Forward function on a user generator input and compare it against // the manually calculated result. input1 = arma::mat("1 2 5 7 -1 -3"); input2 = arma::mat("-1 3 -4 11 3 -3"); + input = arma::join_cols(input1, input2); target = arma::mat("1 -1 -1 1 -1 1"); - double error = module.Forward(input1, input2, target); + double error = module.Forward(input, target); // Computed using torch.nn.functional.margin_ranking_loss() BOOST_REQUIRE_CLOSE(error, 2.66667, 1e-3); // Test the Backward function. - module.Backward(input1, input2, target, output); + module.Backward(input, target, output); CheckMatrices(output, arma::mat("-0.000000 0.166667 -1.500000 0.666667 " "0.000000 -0.000000"), 1e-3); @@ -610,12 +611,13 @@ BOOST_AUTO_TEST_CASE(MarginRankingLossTest) "-4.8090 4.3455 5.2070"); input2 = arma::mat("-4.5288 -9.2766 -0.5882 -5.6643 -6.0175 8.8506 3.4759 " "-9.4886 2.2755 8.4951"); + input = arma::join_cols(input1, input2); target = arma::mat("1 1 -1 1 -1 1 1 1 -1 1"); - error = module.Forward(input1, input2, target); + error = module.Forward(input, target); BOOST_REQUIRE_CLOSE(error, 3.03530, 1e-3); // Test the Backward function on the second input. - module.Backward(input1, input2, target, output); + module.Backward(input, target, output); CheckMatrices(output, arma::mat("0.000000 0.000000 0.091240 0.000000 " "-0.753830 1.336900 0.000000 0.000000 -0.207000 0.328810"), 1e-6); From 5c842717d33c38ae0f2c5a4a4ace749bfbfdc5d7 Mon Sep 17 00:00:00 2001 From: AndreiMihalea Date: Sat, 28 Mar 2020 22:17:20 +0200 Subject: [PATCH 068/111] Solved type in input variable name --- src/mlpack/methods/ann/loss_functions/margin_ranking_loss.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mlpack/methods/ann/loss_functions/margin_ranking_loss.hpp b/src/mlpack/methods/ann/loss_functions/margin_ranking_loss.hpp index ae21d38d50..2360b6b0fb 100644 --- a/src/mlpack/methods/ann/loss_functions/margin_ranking_loss.hpp +++ b/src/mlpack/methods/ann/loss_functions/margin_ranking_loss.hpp @@ -66,7 +66,7 @@ class MarginRankingLoss typename TargetType, typename OutputType > - void Backward(const InputType& input1, + void Backward(const InputType& input, const TargetType& target, OutputType& output); From dcc9fcb6c843b5876d51958ba7a3208d0ea3126a Mon Sep 17 00:00:00 2001 From: AndreiMihalea Date: Sun, 29 Mar 2020 18:38:59 +0300 Subject: [PATCH 069/111] Rebuild From a41b0de91e29f86447d7ab4cc0fe5fbcfd834fde Mon Sep 17 00:00:00 2001 From: jeffin143 Date: Mon, 30 Mar 2020 19:23:59 +0530 Subject: [PATCH 070/111] Remove saving of parameteres --- src/mlpack/bindings/python/setup.py.in | 9 +-- .../preprocess/image_converter_main.cpp | 64 ++++++++----------- src/mlpack/tests/image_load_test.cpp | 12 +++- .../tests/main_tests/image_converter_test.cpp | 55 ++++++++-------- 4 files changed, 63 insertions(+), 77 deletions(-) diff --git a/src/mlpack/bindings/python/setup.py.in b/src/mlpack/bindings/python/setup.py.in index cad63e3420..cbb3839127 100644 --- a/src/mlpack/bindings/python/setup.py.in +++ b/src/mlpack/bindings/python/setup.py.in @@ -30,11 +30,6 @@ if not '${OpenMP_CXX_FLAGS}': else: extra_link_args=['${OpenMP_CXX_FLAGS}'] -if '${STB_AVAILABLE}': - extra_args = ['-DHAS_STB'] -else : - extra_args = [] - # Get list of library dirs. # Get list of library dirs. Note that for the list of directories, any # directories with a (valid) space in the name will be given to us as '\ '; so, @@ -61,11 +56,11 @@ else: cxx_flags = '${CMAKE_CXX_FLAGS}'.strip() cxx_flags = re.sub(' +', ' ', cxx_flags) if cxx_flags: - extra_args = extra_args + ['-DBINDING_TYPE=BINDING_TYPE_PYX', + extra_args = ['-DBINDING_TYPE=BINDING_TYPE_PYX', '-std=c++11', '${OpenMP_CXX_FLAGS}'] + cxx_flags.split(' ') else: - extra_args = extra_args + ['-DBINDING_TYPE=BINDING_TYPE_PYX', + extra_args = ['-DBINDING_TYPE=BINDING_TYPE_PYX', '-std=c++11', '${OpenMP_CXX_FLAGS}'] diff --git a/src/mlpack/methods/preprocess/image_converter_main.cpp b/src/mlpack/methods/preprocess/image_converter_main.cpp index ac90eace66..0565c6d433 100644 --- a/src/mlpack/methods/preprocess/image_converter_main.cpp +++ b/src/mlpack/methods/preprocess/image_converter_main.cpp @@ -28,9 +28,11 @@ PROGRAM_INFO("Image Converter", "unpack an image dataset into individual files.", // Long description. "This utility takes a image or an array of images and loads them to a" - " matrix. You can specify the height " + PRINT_PARAM_STRING("height") + - " width " + PRINT_PARAM_STRING("width") + " and channel " + - PRINT_PARAM_STRING("channel") + " of the images that needs to be loaded. " + " matrix. You can optionally specify the height " + + PRINT_PARAM_STRING("height") + " width " + PRINT_PARAM_STRING("width") + + " and channel " + PRINT_PARAM_STRING("channel") + " of the images that" + "needs to be loaded; otherwise, these parameters will be automatically" + "detected from the image." "\n" "There are other options too, that can be specified such as " + PRINT_PARAM_STRING("quality") @@ -54,32 +56,32 @@ PROGRAM_INFO("Image Converter", PARAM_VECTOR_IN_REQ(string, "input", "Image filenames which have to " "be loaded/saved.", "i"); -PARAM_INT_IN("width", "Width of the image", "W", 256); -PARAM_INT_IN("channel", "Number of channel", "C", 3); +PARAM_INT_IN("width", "Width of the image.", "w", 256); +PARAM_INT_IN("channel", "Number of channel.", "c", 3); -PARAM_MATRIX_OUT("output", "Matrix to save images data to.", "o"); +PARAM_MATRIX_OUT("output", "Matrix to save images data to, Only" + "needed if you are specifing save option.", "o"); PARAM_INT_IN("quality", "Compression of the image if saved as jpg (0-100).", "q", 90); - -PARAM_INT_IN("height", "Height of the images", "H", 256); -PARAM_FLAG("save", "Save a dataset as images", "s"); +PARAM_INT_IN("height", "Height of the images.", "H", 256); +PARAM_FLAG("save", "Save a dataset as images.", "s"); PARAM_MATRIX_IN("dataset", "Input matrix to save as images.", "I"); -// Loading/saving of a Image Info model. -PARAM_MODEL_IN(ImageInfo, "input_model", "Input Image Info model.", "m"); -PARAM_MODEL_OUT(ImageInfo, "output_model", "Output Image Info model.", "M"); - static void mlpackMain() { - // Parse command line options. - data::ImageInfo* info; - Timer::Start("Loading/Saving Image"); - if (CLI::HasParam("input_model")) + // Parse command line options. + const vector fileNames = CLI::GetParam >("input"); + arma::mat out; + + if (!CLI::HasParam("save")) { - info = CLI::GetParam("input_model"); + data::ImageInfo info; + Load(fileNames, out, info, true); + if (CLI::HasParam("output")) + CLI::GetParam("output") = std::move(out); } else { @@ -102,31 +104,17 @@ static void mlpackMain() RequireParamValue("quality", [](int x) { return x >= 0;}, true, "quality must be positive"); - const size_t& height = CLI::GetParam("height"); - const size_t& width = CLI::GetParam("width"); - const size_t& channel = CLI::GetParam("channel"); - const size_t& quality = CLI::GetParam("quality"); - info = new data::ImageInfo(width, height, channel, quality); - } - const vector fileNames = - CLI::GetParam >("input"); - arma::mat out; - if (!CLI::HasParam("save")) - { - Load(fileNames, out, *info, true); - if (CLI::HasParam("output")) - CLI::GetParam("output") = std::move(out); - } - if (CLI::HasParam("save")) - { + const size_t height = CLI::GetParam("height"); + const size_t width = CLI::GetParam("width"); + const size_t channel = CLI::GetParam("channel"); + const size_t quality = CLI::GetParam("quality"); + data::ImageInfo info(width, height, channel, quality); if (!CLI::HasParam("dataset")) { throw std::runtime_error("Please provide a input matrix to save " "images from"); } - Save(fileNames, CLI::GetParam ("dataset"), *info, true); + Save(fileNames, CLI::GetParam("dataset"), info, true); } - if (CLI::HasParam("output_model")) - CLI::GetParam ("output_model") = info; } diff --git a/src/mlpack/tests/image_load_test.cpp b/src/mlpack/tests/image_load_test.cpp index 396cb536da..7d2108264c 100644 --- a/src/mlpack/tests/image_load_test.cpp +++ b/src/mlpack/tests/image_load_test.cpp @@ -46,7 +46,11 @@ BOOST_AUTO_TEST_CASE(LoadImageAPITest) arma::Mat matrix; data::ImageInfo info; BOOST_REQUIRE(data::Load("test_image.png", matrix, info, false) == true); - BOOST_REQUIRE_EQUAL(matrix.n_rows, 50 * 50 * 3); // width * height * channels. + // width * height * channels. + BOOST_REQUIRE_EQUAL(matrix.n_rows, 50 * 50 * 3); + BOOST_REQUIRE_EQUAL(info.Height(), 50); + BOOST_REQUIRE_EQUAL(info.Width(), 50); + BOOST_REQUIRE_EQUAL(info.Channels(), 3); BOOST_REQUIRE_EQUAL(matrix.n_cols, 1); } @@ -82,7 +86,11 @@ BOOST_AUTO_TEST_CASE(LoadVectorImageAPITest) data::ImageInfo info; std::vector files = {"test_image.png", "test_image.png"}; BOOST_REQUIRE(data::Load(files, matrix, info, false) == true); - BOOST_REQUIRE_EQUAL(matrix.n_rows, 50 * 50 * 3); // width * height * channels. + // width * height * channels. + BOOST_REQUIRE_EQUAL(matrix.n_rows, 50 * 50 * 3); + BOOST_REQUIRE_EQUAL(info.Height(), 50); + BOOST_REQUIRE_EQUAL(info.Width(), 50); + BOOST_REQUIRE_EQUAL(info.Channels(), 3); BOOST_REQUIRE_EQUAL(matrix.n_cols, 2); } diff --git a/src/mlpack/tests/main_tests/image_converter_test.cpp b/src/mlpack/tests/main_tests/image_converter_test.cpp index dac1924ad6..86397a888e 100644 --- a/src/mlpack/tests/main_tests/image_converter_test.cpp +++ b/src/mlpack/tests/main_tests/image_converter_test.cpp @@ -48,9 +48,6 @@ BOOST_FIXTURE_TEST_SUITE(ImageConverterMainTest, BOOST_AUTO_TEST_CASE(LoadImageTest) { SetInputParam>("input", {"test_image.png", "test_image.png"}); - SetInputParam("height", 50); - SetInputParam("width", 50); - SetInputParam("channel", 3); mlpackMain(); arma::mat output = CLI::GetParam("output"); @@ -89,37 +86,20 @@ BOOST_AUTO_TEST_CASE(SaveImageTest) BOOST_REQUIRE_CLOSE(testimage[i], output[i], 1e-5); } -/** - * Check Saved model is working. - */ -BOOST_AUTO_TEST_CASE(SavedModelTest) -{ - SetInputParam>("input", {"test_image.png", "test_image.png"}); - SetInputParam("height", 50); - SetInputParam("width", 50); - SetInputParam("channel", 3); - - mlpackMain(); - arma::mat randomOutput = CLI::GetParam("output"); - - SetInputParam>("input", {"test_image.png", "test_image.png"}); - SetInputParam("input_model", - CLI::GetParam("output_model")); - - mlpackMain(); - arma::mat savedOutput = CLI::GetParam("output"); - CheckMatrices(randomOutput, savedOutput); -} - /** * Check whether binding throws error if height, width or channel are not * specified. */ BOOST_AUTO_TEST_CASE(IncompleteTest) { - SetInputParam>("input", {"test_image.png", "test_image.png"}); + arma::mat testimage = arma::conv_to::from( + arma::randi>((5 * 5 * 3), 2)); + SetInputParam>("input", {"test_image777.png", + "test_image999.png"}); + SetInputParam("save", true); SetInputParam("height", 50); SetInputParam("width", 50); + SetInputParam("dataset", testimage); Log::Fatal.ignoreInput = true; BOOST_REQUIRE_THROW(mlpackMain(), std::runtime_error); @@ -131,7 +111,13 @@ BOOST_AUTO_TEST_CASE(IncompleteTest) */ BOOST_AUTO_TEST_CASE(InvalidInputTest) { - SetInputParam>("input", {"test_image.png", "test_image.png"}); + arma::mat testimage = arma::conv_to::from( + arma::randi>((5 * 5 * 3), 2)); + SetInputParam>("input", {"test_image777.png", + "test_image999.png"}); + SetInputParam("save", true); + SetInputParam("dataset", testimage); + SetInputParam("height", -50); SetInputParam("width", 50); SetInputParam("channel", 3); @@ -146,7 +132,12 @@ BOOST_AUTO_TEST_CASE(InvalidInputTest) */ BOOST_AUTO_TEST_CASE(InvalidWidthTest) { - SetInputParam>("input", {"test_image.png", "test_image.png"}); + arma::mat testimage = arma::conv_to::from( + arma::randi>((5 * 5 * 3), 2)); + SetInputParam>("input", {"test_image777.png", + "test_image999.png"}); + SetInputParam("save", true); + SetInputParam("dataset", testimage); SetInputParam("height", 50); SetInputParam("width", -50); SetInputParam("channel", 3); @@ -161,7 +152,12 @@ BOOST_AUTO_TEST_CASE(InvalidWidthTest) */ BOOST_AUTO_TEST_CASE(InvalidChannelTest) { - SetInputParam>("input", {"test_image.png", "test_image.png"}); + arma::mat testimage = arma::conv_to::from( + arma::randi>((5 * 5 * 3), 2)); + SetInputParam>("input", {"test_image777.png", + "test_image999.png"}); + SetInputParam("save", true); + SetInputParam("dataset", testimage); SetInputParam("height", 50); SetInputParam("width", 50); SetInputParam("channel", -1); @@ -186,5 +182,4 @@ BOOST_AUTO_TEST_CASE(EmptyInputTest) Log::Fatal.ignoreInput = false; } - BOOST_AUTO_TEST_SUITE_END(); From b3df3884dfe226a1c27062cecc34471d632b6720 Mon Sep 17 00:00:00 2001 From: jeffin143 Date: Tue, 31 Mar 2020 13:50:50 +0530 Subject: [PATCH 071/111] Rewrote checks in binding and also added to History --- HISTORY.md | 3 ++ src/mlpack/bindings/python/setup.py.in | 1 - .../preprocess/image_converter_main.cpp | 37 +++++++++---------- .../tests/main_tests/image_converter_test.cpp | 12 +++--- 4 files changed, 27 insertions(+), 26 deletions(-) diff --git a/HISTORY.md b/HISTORY.md index 639f180a9f..06067e1b4e 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -31,6 +31,9 @@ * CMake fix for finding STB include directory (#2145). + * Add bindings for Loading and Saving image (#2019); `mlpack_image_converter` + from the command-line, `mlpack_image_converter()` from Python. + * Add normalization support for CF binding (#2136). * Add Mish activation function (#2158). diff --git a/src/mlpack/bindings/python/setup.py.in b/src/mlpack/bindings/python/setup.py.in index cbb3839127..913063191c 100644 --- a/src/mlpack/bindings/python/setup.py.in +++ b/src/mlpack/bindings/python/setup.py.in @@ -30,7 +30,6 @@ if not '${OpenMP_CXX_FLAGS}': else: extra_link_args=['${OpenMP_CXX_FLAGS}'] -# Get list of library dirs. # Get list of library dirs. Note that for the list of directories, any # directories with a (valid) space in the name will be given to us as '\ '; so, # in order to split these right, we first convert all spaces to ';', then diff --git a/src/mlpack/methods/preprocess/image_converter_main.cpp b/src/mlpack/methods/preprocess/image_converter_main.cpp index 0565c6d433..af05853f7a 100644 --- a/src/mlpack/methods/preprocess/image_converter_main.cpp +++ b/src/mlpack/methods/preprocess/image_converter_main.cpp @@ -20,17 +20,17 @@ using namespace arma; using namespace std; using namespace mlpack::data; - PROGRAM_INFO("Image Converter", // Short description. "A utility to load an image or set of images into a single dataset that" "can then be used by other mlpack methods and utilities. This can also" - "unpack an image dataset into individual files.", + "unpack an image dataset into individual files, for instance after mlpack" + "methods have been used.", // Long description. "This utility takes a image or an array of images and loads them to a" " matrix. You can optionally specify the height " + PRINT_PARAM_STRING("height") + " width " + PRINT_PARAM_STRING("width") - + " and channel " + PRINT_PARAM_STRING("channel") + " of the images that" + + " and channel " + PRINT_PARAM_STRING("channels") + " of the images that" "needs to be loaded; otherwise, these parameters will be automatically" "detected from the image." "\n" @@ -42,12 +42,12 @@ PROGRAM_INFO("Image Converter", " as an parameter. An example to load an image : " + "\n\n" + PRINT_CALL("image_converter", "input", "X", "height", 256, "width", 256, - "channel", 3, "output", "Y") + + "channels", 3, "output", "Y") + "\n\n" + " An example to save an image is :" + "\n\n" + PRINT_CALL("image_converter", "input", "X", "height", 256, "width", 256, - "channel", 3, "dataset", "Y", "save", true), + "channels", 3, "dataset", "Y", "save", true), SEE_ALSO("@preprocess_binarize", "#preprocess_binarize"), SEE_ALSO("@preprocess_describe", "#preprocess_describe"), SEE_ALSO("@preprocess_imputer", "#preprocess_imputer")); @@ -56,16 +56,16 @@ PROGRAM_INFO("Image Converter", PARAM_VECTOR_IN_REQ(string, "input", "Image filenames which have to " "be loaded/saved.", "i"); -PARAM_INT_IN("width", "Width of the image.", "w", 256); -PARAM_INT_IN("channel", "Number of channel.", "c", 3); +PARAM_INT_IN("width", "Width of the image.", "w", 0); +PARAM_INT_IN("channels", "Number of channels in the image.", "c", 0); PARAM_MATRIX_OUT("output", "Matrix to save images data to, Only" - "needed if you are specifing save option.", "o"); + "needed if you are specifying 'save' option.", "o"); PARAM_INT_IN("quality", "Compression of the image if saved as jpg (0-100).", "q", 90); -PARAM_INT_IN("height", "Height of the images.", "H", 256); +PARAM_INT_IN("height", "Height of the images.", "H", 0); PARAM_FLAG("save", "Save a dataset as images.", "s"); PARAM_MATRIX_IN("dataset", "Input matrix to save as images.", "I"); @@ -78,6 +78,9 @@ static void mlpackMain() if (!CLI::HasParam("save")) { + ReportIgnoredParam("width", "Width of image is determined from file."); + ReportIgnoredParam("height", "Height of image is determined from file."); + ReportIgnoredParam("channels", "Number of channels determined from file."); data::ImageInfo info; Load(fileNames, out, info, true); if (CLI::HasParam("output")) @@ -85,12 +88,8 @@ static void mlpackMain() } else { - if (!CLI::HasParam("width") || !CLI::HasParam("height") || - !CLI::HasParam("channel")) - { - throw std::runtime_error("Please provide height, width and " - "number of channels of the images."); - } + RequireNoneOrAllPassed({ "save", "width", "height", "channels" }, true, + "image size information is needed when 'save' is specified!"); // Positive value for width. RequireParamValue("width", [](int x) { return x >= 0;}, true, "width must be positive"); @@ -98,17 +97,17 @@ static void mlpackMain() RequireParamValue("height", [](int x) { return x >= 0;}, true, "height must be positive"); // Positive value for channel. - RequireParamValue("channel", [](int x) { return x >= 0;}, true, - "channel must be positive"); + RequireParamValue("channels", [](int x) { return x >= 0;}, true, + "channels must be positive"); // Positive value for quality. RequireParamValue("quality", [](int x) { return x >= 0;}, true, "quality must be positive"); const size_t height = CLI::GetParam("height"); const size_t width = CLI::GetParam("width"); - const size_t channel = CLI::GetParam("channel"); + const size_t channels = CLI::GetParam("channels"); const size_t quality = CLI::GetParam("quality"); - data::ImageInfo info(width, height, channel, quality); + data::ImageInfo info(width, height, channels, quality); if (!CLI::HasParam("dataset")) { throw std::runtime_error("Please provide a input matrix to save " diff --git a/src/mlpack/tests/main_tests/image_converter_test.cpp b/src/mlpack/tests/main_tests/image_converter_test.cpp index 86397a888e..35e140c45e 100644 --- a/src/mlpack/tests/main_tests/image_converter_test.cpp +++ b/src/mlpack/tests/main_tests/image_converter_test.cpp @@ -64,7 +64,7 @@ BOOST_AUTO_TEST_CASE(SaveImageTest) "test_image999.png"}); SetInputParam("height", 5); SetInputParam("width", 5); - SetInputParam("channel", 3); + SetInputParam("channels", 3); SetInputParam("save", true); SetInputParam("dataset", testimage); mlpackMain(); @@ -76,7 +76,7 @@ BOOST_AUTO_TEST_CASE(SaveImageTest) "test_image999.png"}); SetInputParam("height", 5); SetInputParam("width", 5); - SetInputParam("channel", 3); + SetInputParam("channels", 3); mlpackMain(); arma::mat output = CLI::GetParam("output"); @@ -120,7 +120,7 @@ BOOST_AUTO_TEST_CASE(InvalidInputTest) SetInputParam("height", -50); SetInputParam("width", 50); - SetInputParam("channel", 3); + SetInputParam("channels", 3); Log::Fatal.ignoreInput = true; BOOST_REQUIRE_THROW(mlpackMain(), std::runtime_error); @@ -140,7 +140,7 @@ BOOST_AUTO_TEST_CASE(InvalidWidthTest) SetInputParam("dataset", testimage); SetInputParam("height", 50); SetInputParam("width", -50); - SetInputParam("channel", 3); + SetInputParam("channels", 3); Log::Fatal.ignoreInput = true; BOOST_REQUIRE_THROW(mlpackMain(), std::runtime_error); @@ -160,7 +160,7 @@ BOOST_AUTO_TEST_CASE(InvalidChannelTest) SetInputParam("dataset", testimage); SetInputParam("height", 50); SetInputParam("width", 50); - SetInputParam("channel", -1); + SetInputParam("channels", -1); Log::Fatal.ignoreInput = true; BOOST_REQUIRE_THROW(mlpackMain(), std::runtime_error); @@ -175,7 +175,7 @@ BOOST_AUTO_TEST_CASE(EmptyInputTest) SetInputParam>("input", {}); SetInputParam("height", 50); SetInputParam("width", 50); - SetInputParam("channel", 50); + SetInputParam("channels", 50); Log::Fatal.ignoreInput = true; BOOST_REQUIRE_THROW(mlpackMain(), std::runtime_error); From d3068f0fdc83e5ca0aa3b960672b36d7790532cf Mon Sep 17 00:00:00 2001 From: Andrei Mihalea Date: Tue, 31 Mar 2020 18:16:17 +0300 Subject: [PATCH 072/111] Update src/mlpack/methods/ann/loss_functions/margin_ranking_loss.hpp Co-Authored-By: favre49 <40389657+favre49@users.noreply.github.com> --- src/mlpack/methods/ann/loss_functions/margin_ranking_loss.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mlpack/methods/ann/loss_functions/margin_ranking_loss.hpp b/src/mlpack/methods/ann/loss_functions/margin_ranking_loss.hpp index 2360b6b0fb..cdd8d8e5f1 100644 --- a/src/mlpack/methods/ann/loss_functions/margin_ranking_loss.hpp +++ b/src/mlpack/methods/ann/loss_functions/margin_ranking_loss.hpp @@ -39,7 +39,7 @@ class MarginRankingLoss /** * Computes the Margin Ranking Loss function. - * Measures the loss between two intputs and a label with -1 and 1 values. + * Measures the loss between two inputs and a label with -1 and 1 values. * a value of 1 in the label means the first input should be ranked higher * and a value of -1 means the second input should be ranked higher. * From 3208350409002c18846a7483c013df8ca6c63e22 Mon Sep 17 00:00:00 2001 From: Andrei Mihalea Date: Tue, 31 Mar 2020 18:16:31 +0300 Subject: [PATCH 073/111] Update src/mlpack/methods/ann/loss_functions/margin_ranking_loss.hpp Co-Authored-By: favre49 <40389657+favre49@users.noreply.github.com> --- src/mlpack/methods/ann/loss_functions/margin_ranking_loss.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mlpack/methods/ann/loss_functions/margin_ranking_loss.hpp b/src/mlpack/methods/ann/loss_functions/margin_ranking_loss.hpp index cdd8d8e5f1..8eac9f4e12 100644 --- a/src/mlpack/methods/ann/loss_functions/margin_ranking_loss.hpp +++ b/src/mlpack/methods/ann/loss_functions/margin_ranking_loss.hpp @@ -40,7 +40,7 @@ class MarginRankingLoss /** * Computes the Margin Ranking Loss function. * Measures the loss between two inputs and a label with -1 and 1 values. - * a value of 1 in the label means the first input should be ranked higher + * A value of 1 in the label means the first input should be ranked higher * and a value of -1 means the second input should be ranked higher. * * @param input Concatenation of the two inputs for evaluating the specified From 13585beb0e5b7b2bdeeae2cb2376db8641ffe5a6 Mon Sep 17 00:00:00 2001 From: Andrei Mihalea Date: Tue, 31 Mar 2020 18:16:56 +0300 Subject: [PATCH 074/111] Update src/mlpack/methods/ann/loss_functions/margin_ranking_loss.hpp Co-Authored-By: favre49 <40389657+favre49@users.noreply.github.com> --- src/mlpack/methods/ann/loss_functions/margin_ranking_loss.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mlpack/methods/ann/loss_functions/margin_ranking_loss.hpp b/src/mlpack/methods/ann/loss_functions/margin_ranking_loss.hpp index 8eac9f4e12..116bcf90fc 100644 --- a/src/mlpack/methods/ann/loss_functions/margin_ranking_loss.hpp +++ b/src/mlpack/methods/ann/loss_functions/margin_ranking_loss.hpp @@ -45,7 +45,7 @@ class MarginRankingLoss * * @param input Concatenation of the two inputs for evaluating the specified * function. - * @param target The label vector which contains -1 or 1 values. + * @param target The label vector which contains values of -1 or 1. */ template < typename InputType, From fd5e1b9eedada110d464938a8f3b3a900dc50e97 Mon Sep 17 00:00:00 2001 From: AndreiMihalea Date: Tue, 31 Mar 2020 18:50:13 +0300 Subject: [PATCH 075/111] Moved the description and renamed variables --- HISTORY.md | 2 ++ .../ann/loss_functions/margin_ranking_loss.hpp | 8 +++++--- .../ann/loss_functions/margin_ranking_loss_impl.hpp | 12 ++++++------ 3 files changed, 13 insertions(+), 9 deletions(-) diff --git a/HISTORY.md b/HISTORY.md index 9dba6a517d..6d0d37a9d7 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -66,6 +66,8 @@ * Add Hinge Embedding Loss Function (#2229). + * Add Margin Ranking Loss Function (#2264). + ### mlpack 3.2.2 ###### 2019-11-26 * Add `valid` and `same` padding option in `Convolution` and `Atrous diff --git a/src/mlpack/methods/ann/loss_functions/margin_ranking_loss.hpp b/src/mlpack/methods/ann/loss_functions/margin_ranking_loss.hpp index 116bcf90fc..b5d64f0aae 100644 --- a/src/mlpack/methods/ann/loss_functions/margin_ranking_loss.hpp +++ b/src/mlpack/methods/ann/loss_functions/margin_ranking_loss.hpp @@ -18,6 +18,11 @@ namespace mlpack { namespace ann /** Artificial Neural Network. */ { /** + * Margin ranking loss measures the loss given inputs and a label vector with + * values of 1 or -1. If the label is 1 then the first input should be ranked + * higher than the second input at a distance larger than a margin, and vice- + * versa if the label is -1. + * * @tparam InputDataType Type of the input data (arma::colvec, arma::mat, * arma::sp_mat or arma::cube). * @tparam OutputDataType Type of the output data (arma::colvec, arma::mat, @@ -39,9 +44,6 @@ class MarginRankingLoss /** * Computes the Margin Ranking Loss function. - * Measures the loss between two inputs and a label with -1 and 1 values. - * A value of 1 in the label means the first input should be ranked higher - * and a value of -1 means the second input should be ranked higher. * * @param input Concatenation of the two inputs for evaluating the specified * function. diff --git a/src/mlpack/methods/ann/loss_functions/margin_ranking_loss_impl.hpp b/src/mlpack/methods/ann/loss_functions/margin_ranking_loss_impl.hpp index 9920b07085..79e4d879e9 100644 --- a/src/mlpack/methods/ann/loss_functions/margin_ranking_loss_impl.hpp +++ b/src/mlpack/methods/ann/loss_functions/margin_ranking_loss_impl.hpp @@ -34,9 +34,9 @@ double MarginRankingLoss::Forward( const InputType& input, const TargetType& target) { - int input_rows = input.n_rows; - const InputType& input1 = input.rows(0, input_rows / 2 - 1); - const InputType& input2 = input.rows(input_rows / 2, input_rows - 1); + const int inputRows = input.n_rows; + const InputType& input1 = input.rows(0, inputRows / 2 - 1); + const InputType& input2 = input.rows(inputRows / 2, inputRows - 1); return arma::accu(arma::max(arma::zeros(size(target)), -target % (input1 - input2) + margin)) / target.n_cols; } @@ -52,9 +52,9 @@ void MarginRankingLoss::Backward( const TargetType& target, OutputType& output) { - int input_rows = input.n_rows; - const InputType& input1 = input.rows(0, input_rows / 2 - 1); - const InputType& input2 = input.rows(input_rows / 2, input_rows - 1); + const int inputRows = input.n_rows; + const InputType& input1 = input.rows(0, inputRows / 2 - 1); + const InputType& input2 = input.rows(inputRows / 2, inputRows - 1); output = -target % (input1 - input2) + margin; output.elem(arma::find(output >= 0)).ones(); output.elem(arma::find(output < 0)).zeros(); From 96be8af21de48ea4b5f6aab51bf9bfd90de367ae Mon Sep 17 00:00:00 2001 From: Mikhail Lozhnikov Date: Tue, 31 Mar 2020 23:42:27 +0300 Subject: [PATCH 076/111] Updated documentation for various string encoding tests. --- .../bag_of_words_encoding_policy.hpp | 1 + .../tf_idf_encoding_policy.hpp | 1 + src/mlpack/tests/string_encoding_test.cpp | 644 ++++++++++++------ 3 files changed, 450 insertions(+), 196 deletions(-) diff --git a/src/mlpack/core/data/string_encoding_policies/bag_of_words_encoding_policy.hpp b/src/mlpack/core/data/string_encoding_policies/bag_of_words_encoding_policy.hpp index e3be0f5faa..44a8e72903 100644 --- a/src/mlpack/core/data/string_encoding_policies/bag_of_words_encoding_policy.hpp +++ b/src/mlpack/core/data/string_encoding_policies/bag_of_words_encoding_policy.hpp @@ -1,6 +1,7 @@ /** * @file bag_of_words_encoding_policy.hpp * @author Jeffin Sam + * @author Mikhail Lozhnikov * * Definition of the BagOfWordsEncodingPolicy class. * diff --git a/src/mlpack/core/data/string_encoding_policies/tf_idf_encoding_policy.hpp b/src/mlpack/core/data/string_encoding_policies/tf_idf_encoding_policy.hpp index 2ad8b15e9f..4acfc03c56 100644 --- a/src/mlpack/core/data/string_encoding_policies/tf_idf_encoding_policy.hpp +++ b/src/mlpack/core/data/string_encoding_policies/tf_idf_encoding_policy.hpp @@ -1,6 +1,7 @@ /** * @file tf_idf_encoding_policy.hpp * @author Jeffin Sam + * @author Mikhail Lozhnikov * * Definition of the TfIdfEncodingPolicy class. * diff --git a/src/mlpack/tests/string_encoding_test.cpp b/src/mlpack/tests/string_encoding_test.cpp index baec566bcb..23d1dc59dd 100644 --- a/src/mlpack/tests/string_encoding_test.cpp +++ b/src/mlpack/tests/string_encoding_test.cpp @@ -522,6 +522,7 @@ BOOST_AUTO_TEST_CASE(CharExtractDictionaryEncodingSerialization) EncoderType encoder; CharExtract tokenizer; arma::mat output; + encoder.Encode(stringEncodingInput, output, tokenizer); EncoderType xmlEncoder, textEncoder, binaryEncoder; @@ -541,7 +542,7 @@ BOOST_AUTO_TEST_CASE(CharExtractDictionaryEncodingSerialization) } /** - * Test for Bag of Words encoding algorithm. + * Test the Bag of Words encoding algorithm. */ BOOST_AUTO_TEST_CASE(BagOfWordsEncodingTest) { @@ -552,29 +553,71 @@ BOOST_AUTO_TEST_CASE(BagOfWordsEncodingTest) SplitByAnyOf tokenizer(" ,."); encoder.Encode(stringEncodingInput, output, tokenizer); + const DictionaryType& dictionary = encoder.Dictionary(); - // Checking that everything is mapped to different numbers. + // Checking that each token has a unique label. std::unordered_map keysCount; + for (auto& keyValue : dictionary.Mapping()) { keysCount[keyValue.second]++; - // Every token should be mapped only once. + BOOST_REQUIRE_EQUAL(keysCount[keyValue.second], 1); } + +/* The expected values were obtained by the following Python script: + + from sklearn.feature_extraction.text import CountVectorizer + from collections import OrderedDict + import re + + string_encoding_input = [ + "mlpack is an intuitive, fast, and flexible C++ machine learning library " + "with bindings to other languages. ", + "It is meant to be a machine learning analog to LAPACK, and aims to " + "implement a wide array of machine learning methods and functions " + "as a \"swiss army knife\" for machine learning researchers.", + "In addition to its powerful C++ interface, mlpack also provides " + "command-line programs and Python bindings." + ] + + dictionary = OrderedDict() + + count = 0 + for line in string_encoding_input: + for word in re.split(' |,|\.', line): + if word and (not (word in dictionary)): + dictionary[word] = count + count += 1 + + def tokenizer(line): + return re.split(' |,|\.', line) + + vectorizer = CountVectorizer(strip_accents=False, lowercase=False, + preprocessor=None, tokenizer=tokenizer, stop_words=None, + vocabulary=dictionary, binary=True) + + X = vectorizer.fit_transform(string_encoding_input) + + for row in X.toarray(): + print("{ " + ", ".join(map(str, row)) + " },") +*/ + arma::mat expected = { - { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, - { 0, 1, 0, 0, 0, 1, 0, 0, 1, 1, 0, 0, 0, 1, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, - { 1, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 } + { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, + { 0, 1, 0, 0, 0, 1, 0, 0, 1, 1, 0, 0, 0, 1, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, + { 1, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 } }; + CheckMatrices(output, expected.t()); } /** - * Bag of Words encoding algorithm output saved in a vector. + * Test the Bag of Words encoding algorithm. The output is saved into a vector. */ BOOST_AUTO_TEST_CASE(VectorBagOfWordsEncodingTest) { @@ -589,29 +632,32 @@ BOOST_AUTO_TEST_CASE(VectorBagOfWordsEncodingTest) const DictionaryType& dictionary = encoder.Dictionary(); - // Checking that everything is mapped to different numbers. + // Checking that each token has a unique label. std::unordered_map keysCount; + for (auto& keyValue : dictionary.Mapping()) { keysCount[keyValue.second]++; - // Every token should be mapped only once. + BOOST_REQUIRE_EQUAL(keysCount[keyValue.second], 1); } + /* The expected values were obtained by the same script as in + BagOfWordsEncodingTest. */ vector> expected = { - { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, - { 0, 1, 0, 0, 0, 1, 0, 0, 1, 1, 0, 0, 0, 1, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, - { 1, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 } + { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, + { 0, 1, 0, 0, 0, 1, 0, 0, 1, 1, 0, 0, 0, 1, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, + { 1, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 } }; BOOST_REQUIRE(output == expected); } /** - * Test Bag of Words encoding for characters. + * Test the Bag of Words algorithm for individual characters. */ BOOST_AUTO_TEST_CASE(BagOfWordsEncodingIndividualCharactersTest) { @@ -624,8 +670,8 @@ BOOST_AUTO_TEST_CASE(BagOfWordsEncodingIndividualCharactersTest) arma::mat output; BagOfWordsEncoding encoder; - // Passing a empty string to encode characters. encoder.Encode(input, output, CharExtract()); + arma::mat target = { { 1, 1, 1, 0, 0 }, { 0, 1, 1, 1, 1 }, @@ -637,7 +683,7 @@ BOOST_AUTO_TEST_CASE(BagOfWordsEncodingIndividualCharactersTest) /** * Test the Bag of Words encoding algorithm in case of individual - * character encoding, storing resulting in vector>. + * characters encoding. The output type is vector>. */ BOOST_AUTO_TEST_CASE(VectorBagOfWordsEncodingIndividualCharactersTest) { @@ -650,7 +696,6 @@ BOOST_AUTO_TEST_CASE(VectorBagOfWordsEncodingIndividualCharactersTest) vector> output; BagOfWordsEncoding encoder; - // Passing a empty string to encode characters. encoder.Encode(input, output, CharExtract()); vector> expected = { @@ -663,8 +708,9 @@ BOOST_AUTO_TEST_CASE(VectorBagOfWordsEncodingIndividualCharactersTest) } /** - * Test the Tf-Idf Encoding using rawcount type and smoothidf as true, - * which is the default value used for algorithm. + * Test the Tf-Idf encoding algorithm with the raw count term frequency type + * and the smooth inverse document frequency type. These parameters are + * the default ones. */ BOOST_AUTO_TEST_CASE(RawCountSmoothIdfEncodingTest) { @@ -677,40 +723,96 @@ BOOST_AUTO_TEST_CASE(RawCountSmoothIdfEncodingTest) encoder.Encode(stringEncodingInput, output, tokenizer); const DictionaryType& dictionary = encoder.Dictionary(); - // Checking that everything is mapped to different numbers. + // Checking that each token has a unique label. std::unordered_map keysCount; + for (auto& keyValue : dictionary.Mapping()) { keysCount[keyValue.second]++; - // Every token should be mapped only once. + BOOST_REQUIRE_EQUAL(keysCount[keyValue.second], 1); } + + /* The expected values were obtained by the following Python script: + + from sklearn.feature_extraction.text import TfidfVectorizer + from collections import OrderedDict + import re + + string_encoding_input = [ + "mlpack is an intuitive, fast, and flexible C++ machine learning library " + "with bindings to other languages. ", + "It is meant to be a machine learning analog to LAPACK, and aims to " + "implement a wide array of machine learning methods and functions " + "as a \"swiss army knife\" for machine learning researchers.", + "In addition to its powerful C++ interface, mlpack also provides " + "command-line programs and Python bindings." + ] + + smooth_idf = True + tf_type = 'raw_count' + + dictionary = OrderedDict() + + count = 0 + for line in string_encoding_input: + for word in re.split(' |,|\.', line): + if word and (not (word in dictionary)): + dictionary[word] = count + count += 1 + + def tokenizer(line): + return re.split(' |,|\.', line) + + if tf_type == 'raw_count': + binary = False + sublinear_tf = False + elif tf_type == 'binary': + binary = True + sublinear_tf = False + elif tf_type == 'sublinear_tf': + binary = False + sublinear_tf = True + + vectorizer = TfidfVectorizer(strip_accents=False, lowercase=False, + preprocessor=None, tokenizer=tokenizer, stop_words=None, + vocabulary=dictionary, binary=binary, norm=None, smooth_idf=smooth_idf, + sublinear_tf=sublinear_tf) + + X = vectorizer.fit_transform(string_encoding_input) + + def format_result(value): + if value == int(value): + return str(int(value)) + else: + return "{0:.8f}".format(value) + + for row in X.toarray(): + print("{ " + ", ".join(map(format_result, row)) + " },") + */ arma::mat expected = { - { 1.28768207245178, 1.28768207245178, 1.69314718055995, 1.69314718055995, - 1.69314718055995, 1, 1.69314718055995, 1.28768207245178, - 1.28768207245178, 1.28768207245178, 1.69314718055995, 1.69314718055995, - 1.28768207245178, 1, 1.69314718055995, 1.69314718055995, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0 }, - { 0, 1.28768207245178, 0, 0, 0, 2, 0, 0, 3.86304621735534, - 3.86304621735534, 0, 0, 0, 3, 0, 0, 1.69314718055995, 1.69314718055995, - 1.69314718055995, 5.07944154167984, 1.69314718055995, 1.69314718055995, - 1.69314718055995, 1.69314718055995, 1.69314718055995, 1.69314718055995, - 1.69314718055995, 1.69314718055995, 1.69314718055995, 1.69314718055995, - 1.69314718055995, 1.69314718055995, 1.69314718055995, 1.69314718055995, - 1.69314718055995, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, - { 1.28768207245178, 0, 0, 0, 0, 1, 0, 1.28768207245178, 0, 0, 0, 0, - 1.28768207245178, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 1.69314718055995, 1.69314718055995, 1.69314718055995, - 1.69314718055995, 1.69314718055995, 1.69314718055995, 1.69314718055995, - 1.69314718055995, 1.69314718055995, 1.69314718055995 } + { 1.28768207, 1.28768207, 1.69314718, 1.69314718, 1.69314718, 1, 1.69314718, + 1.28768207, 1.28768207, 1.28768207, 1.69314718, 1.69314718, 1.28768207, 1, + 1.69314718, 1.69314718, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, + { 0, 1.28768207, 0, 0, 0, 2, 0, 0, 3.86304622, 3.86304622, 0, 0, 0, 3, 0, + 0, 1.69314718, 1.69314718, 1.69314718, 5.07944154, 1.69314718, 1.69314718, + 1.69314718, 1.69314718, 1.69314718, 1.69314718, 1.69314718, 1.69314718, + 1.69314718, 1.69314718, 1.69314718, 1.69314718, 1.69314718, 1.69314718, + 1.69314718, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, + { 1.28768207, 0, 0, 0, 0, 1, 0, 1.28768207, 0, 0, 0, 0, 1.28768207, 1, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1.69314718, + 1.69314718, 1.69314718, 1.69314718, 1.69314718, 1.69314718, 1.69314718, + 1.69314718, 1.69314718, 1.69314718 } }; - CheckMatrices(output, expected.t(), 1e-12); + + CheckMatrices(output, expected.t(), 1e-6); } /** - * Test the TfIdf encoding algorithm, using rawcount as type of tf and smoothIdf - * as true, storing the result in a vector>. + * Test the Tf-Idf encoding algorithm with the raw count term frequency type + * and the smooth inverse document frequency type. These parameters are + * the default ones. The output type is vector>. */ BOOST_AUTO_TEST_CASE(VectorRawCountSmoothIdfEncodingTest) { @@ -725,41 +827,40 @@ BOOST_AUTO_TEST_CASE(VectorRawCountSmoothIdfEncodingTest) const DictionaryType& dictionary = encoder.Dictionary(); - // Checking that everything is mapped to different numbers. + // Checking that each token has a unique label. std::unordered_map keysCount; + for (auto& keyValue : dictionary.Mapping()) { keysCount[keyValue.second]++; - // Every token should be mapped only once. + BOOST_REQUIRE_EQUAL(keysCount[keyValue.second], 1); } + /* The expected values were obtained by the same script as in + RawCountSmoothIdfEncodingTest. */ vector> expected = { - { 1.28768207245178, 1.28768207245178, 1.69314718055995, 1.69314718055995, - 1.69314718055995, 1, 1.69314718055995, 1.28768207245178, - 1.28768207245178, 1.28768207245178, 1.69314718055995, 1.69314718055995, - 1.28768207245178, 1, 1.69314718055995, 1.69314718055995, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0 }, - { 0, 1.28768207245178, 0, 0, 0, 2, 0, 0, 3.86304621735534, - 3.86304621735534, 0, 0, 0, 3, 0, 0, 1.69314718055995, 1.69314718055995, - 1.69314718055995, 5.07944154167984, 1.69314718055995, 1.69314718055995, - 1.69314718055995, 1.69314718055995, 1.69314718055995, 1.69314718055995, - 1.69314718055995, 1.69314718055995, 1.69314718055995, 1.69314718055995, - 1.69314718055995, 1.69314718055995, 1.69314718055995, 1.69314718055995, - 1.69314718055995, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, - { 1.28768207245178, 0, 0, 0, 0, 1, 0, 1.28768207245178, 0, 0, 0, 0, - 1.28768207245178, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 1.69314718055995, 1.69314718055995, 1.69314718055995, - 1.69314718055995, 1.69314718055995, 1.69314718055995, 1.69314718055995, - 1.69314718055995, 1.69314718055995, 1.69314718055995 } + { 1.28768207, 1.28768207, 1.69314718, 1.69314718, 1.69314718, 1, 1.69314718, + 1.28768207, 1.28768207, 1.28768207, 1.69314718, 1.69314718, 1.28768207, 1, + 1.69314718, 1.69314718, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, + { 0, 1.28768207, 0, 0, 0, 2, 0, 0, 3.86304622, 3.86304622, 0, 0, 0, 3, 0, + 0, 1.69314718, 1.69314718, 1.69314718, 5.07944154, 1.69314718, 1.69314718, + 1.69314718, 1.69314718, 1.69314718, 1.69314718, 1.69314718, 1.69314718, + 1.69314718, 1.69314718, 1.69314718, 1.69314718, 1.69314718, 1.69314718, + 1.69314718, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, + { 1.28768207, 0, 0, 0, 0, 1, 0, 1.28768207, 0, 0, 0, 0, 1.28768207, 1, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1.69314718, + 1.69314718, 1.69314718, 1.69314718, 1.69314718, 1.69314718, 1.69314718, + 1.69314718, 1.69314718, 1.69314718 } }; - CheckVectors(output, expected, 1e-12); + CheckVectors(output, expected, 1e-6); } /** - * Test TFIDF encoding for characters, using rawcount as tf type and - * smoothidf as true. + * Test the Tf-Idf encoding algorithm for individual characters with the + * raw count term frequency type and the smooth inverse document frequency type. + * These parameters are the default ones. */ BOOST_AUTO_TEST_CASE(RawCountSmoothIdfEncodingIndividualCharactersTest) { @@ -772,19 +873,75 @@ BOOST_AUTO_TEST_CASE(RawCountSmoothIdfEncodingIndividualCharactersTest) arma::mat output; TfIdfEncoding encoder; - // Passing a empty string to encode characters. encoder.Encode(input, output, CharExtract()); + + /* The expected values were obtained by the following Python script: + + from sklearn.feature_extraction.text import TfidfVectorizer + from collections import OrderedDict + import re + + input_string = [ + "GACCA", + "ABCABCD", + "GAB" + ] + + smooth_idf = True + tf_type = 'raw_count' + + dictionary = OrderedDict() + + count = 0 + for line in input_string: + for word in list(line): + if word and (not (word in dictionary)): + dictionary[word] = count + count += 1 + + def tokenizer(line): + return list(line) + + if tf_type == 'raw_count': + binary = False + sublinear_tf = False + elif tf_type == 'binary': + binary = True + sublinear_tf = False + elif tf_type == 'sublinear_tf': + binary = False + sublinear_tf = True + + vectorizer = TfidfVectorizer(strip_accents=False, lowercase=False, + preprocessor=None, tokenizer=tokenizer, stop_words=None, + vocabulary=dictionary, binary=binary, norm=None, smooth_idf=smooth_idf, + sublinear_tf=sublinear_tf) + + X = vectorizer.fit_transform(input_string) + + def format_result(value): + if value == int(value): + return str(int(value)) + else: + return "{0:.14f}".format(value) + + for row in X.toarray(): + print("{ " + ", ".join(map(format_result, row)) + " },") + */ arma::mat target = { - { 1.2876820724517808, 2, 2.5753641449035616, 0, 0 }, - { 0, 2, 2.5753641449035616, 2.5753641449035616, 1.6931471805599454 }, - { 1.2876820724517808, 1, 0, 1.2876820724517808, 0 } + { 1.28768207245178, 2, 2.57536414490356, 0, 0 }, + { 0, 2, 2.57536414490356, 2.57536414490356, 1.69314718055995 }, + { 1.28768207245178, 1, 0, 1.28768207245178, 0 } }; + CheckMatrices(output, target.t(), 1e-12); } /** - * Test the Tf-Idf encoding algorithm to store result in vector - * in case of individual character encoding using default values. + * Test the Tf-Idf encoding algorithm for individual characters with the + * raw count term frequency type and the smooth inverse document frequency type. + * These parameters are the default ones. The output type is + * vector>. */ BOOST_AUTO_TEST_CASE(VectorRawCountSmoothIdfEncodingIndividualCharactersTest) { @@ -797,18 +954,22 @@ BOOST_AUTO_TEST_CASE(VectorRawCountSmoothIdfEncodingIndividualCharactersTest) vector> output; TfIdfEncoding encoder; - // Passing a empty string to encode characters. encoder.Encode(input, output, CharExtract()); + + /* The expected values were obtained by the same script as in + RawCountSmoothIdfEncodingIndividualCharactersTest. */ vector> expected = { - { 1.2876820724517808, 2, 2.5753641449035616, 0, 0 }, - { 0, 2, 2.5753641449035616, 2.5753641449035616, 1.6931471805599454 }, - { 1.2876820724517808, 1, 0, 1.2876820724517808, 0 } + { 1.28768207245178, 2, 2.57536414490356, 0, 0 }, + { 0, 2, 2.57536414490356, 2.57536414490356, 1.69314718055995 }, + { 1.28768207245178, 1, 0, 1.28768207245178, 0 } }; + CheckVectors(output, expected, 1e-12); } /** - * Test the Tf-Idf Encoding using rawcount type and smoothidf as false. + * Test the Tf-Idf encoding algorithm with the raw count term frequency type + * and the non-smooth inverse document frequency type. */ BOOST_AUTO_TEST_CASE(TfIdfRawCountEncodingTest) { @@ -816,49 +977,50 @@ BOOST_AUTO_TEST_CASE(TfIdfRawCountEncodingTest) arma::mat output; TfIdfEncoding encoder( - (TfIdfEncodingPolicy(TfIdfEncodingPolicy::TfTypes::RAW_COUNT, false))); + TfIdfEncodingPolicy(TfIdfEncodingPolicy::TfTypes::RAW_COUNT, false)); SplitByAnyOf tokenizer(" ,."); encoder.Encode(stringEncodingInput, output, tokenizer); const DictionaryType& dictionary = encoder.Dictionary(); - // Checking that everything is mapped to different numbers. + // Checking that each token has a unique label. std::unordered_map keysCount; + for (auto& keyValue : dictionary.Mapping()) { keysCount[keyValue.second]++; - // Every token should be mapped only once. + BOOST_REQUIRE_EQUAL(keysCount[keyValue.second], 1); } + /* The expected values were obtained by almost the same script as in + RawCountSmoothIdfEncodingTest. The only difference is smooth_idf equals + False. */ arma::mat expected = { - { 1.40546510810816, 1.40546510810816, 2.09861228866811, 2.09861228866811, - 2.09861228866811, 1, 2.09861228866811, 1.40546510810816, - 1.40546510810816, 1.40546510810816, 2.09861228866811, 2.09861228866811, - 1.40546510810816, 1, 2.09861228866811, 2.09861228866811, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0 }, - { 0, 1.40546510810816, 0, 0, 0, 2, 0, 0, 4.21639532432449, - 4.21639532432449, 0, 0, 0, 3, 0, 0, 2.09861228866811, 2.09861228866811, - 2.09861228866811, 6.29583686600433, 2.09861228866811, 2.09861228866811, - 2.09861228866811, 2.09861228866811, 2.09861228866811, 2.09861228866811, - 2.09861228866811, 2.09861228866811, 2.09861228866811, 2.09861228866811, - 2.09861228866811, 2.09861228866811, 2.09861228866811, 2.09861228866811, - 2.09861228866811, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, - { 1.40546510810816, 0, 0, 0, 0, 1, 0, 1.40546510810816, 0, 0, 0, 0, - 1.40546510810816, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 2.09861228866811, 2.09861228866811, 2.09861228866811, - 2.09861228866811, 2.09861228866811, 2.09861228866811, 2.09861228866811, - 2.09861228866811, 2.09861228866811, 2.09861228866811 } + { 1.40546511, 1.40546511, 2.09861229, 2.09861229, 2.09861229, 1, 2.09861229, + 1.40546511, 1.40546511, 1.40546511, 2.09861229, 2.09861229, 1.40546511, 1, + 2.09861229, 2.09861229, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, + { 0, 1.40546511, 0, 0, 0, 2, 0, 0, 4.21639532, 4.21639532, 0, 0, 0, 3, 0, 0, + 2.09861229, 2.09861229, 2.09861229, 6.29583687, 2.09861229, 2.09861229, + 2.09861229, 2.09861229, 2.09861229, 2.09861229, 2.09861229, 2.09861229, + 2.09861229, 2.09861229, 2.09861229, 2.09861229, 2.09861229, 2.09861229, + 2.09861229, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, + { 1.40546511, 0, 0, 0, 0, 1, 0, 1.40546511, 0, 0, 0, 0, 1.40546511, 1, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2.09861229, + 2.09861229, 2.09861229, 2.09861229, 2.09861229, 2.09861229, 2.09861229, + 2.09861229, 2.09861229, 2.09861229 } }; - CheckMatrices(output, expected.t(), 1e-12); + + CheckMatrices(output, expected.t(), 1e-6); } /** - * Test the TfIdf encoding algorithm for output type as vector, with rawcount - * as type, but with smoothidf as false. - */ + * Test the Tf-Idf encoding algorithm with the raw count term frequency type + * and the non-smooth inverse document frequency type. The output type is + * vector>. + */ BOOST_AUTO_TEST_CASE(VectorTfIdfRawCountEncodingTest) { using DictionaryType = StringEncodingDictionary; @@ -872,43 +1034,42 @@ BOOST_AUTO_TEST_CASE(VectorTfIdfRawCountEncodingTest) const DictionaryType& dictionary = encoder.Dictionary(); - // Checking that everything is mapped to different numbers. + // Checking that each token has a unique label. std::unordered_map keysCount; for (auto& keyValue : dictionary.Mapping()) { keysCount[keyValue.second]++; - // Every token should be mapped only once. + BOOST_REQUIRE_EQUAL(keysCount[keyValue.second], 1); } + /* The expected values were obtained by almost the same script as in + RawCountSmoothIdfEncodingTest. The only difference is smooth_idf equals + False. */ vector> expected = { - { 1.40546510810816, 1.40546510810816, 2.09861228866811, 2.09861228866811, - 2.09861228866811, 1, 2.09861228866811, 1.40546510810816, - 1.40546510810816, 1.40546510810816, 2.09861228866811, 2.09861228866811, - 1.40546510810816, 1, 2.09861228866811, 2.09861228866811, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0 }, - { 0, 1.40546510810816, 0, 0, 0, 2, 0, 0, 4.21639532432449, - 4.21639532432449, 0, 0, 0, 3, 0, 0, 2.09861228866811, 2.09861228866811, - 2.09861228866811, 6.29583686600433, 2.09861228866811, 2.09861228866811, - 2.09861228866811, 2.09861228866811, 2.09861228866811, 2.09861228866811, - 2.09861228866811, 2.09861228866811, 2.09861228866811, 2.09861228866811, - 2.09861228866811, 2.09861228866811, 2.09861228866811, 2.09861228866811, - 2.09861228866811, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, - { 1.40546510810816, 0, 0, 0, 0, 1, 0, 1.40546510810816, 0, 0, 0, 0, - 1.40546510810816, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 2.09861228866811, 2.09861228866811, 2.09861228866811, - 2.09861228866811, 2.09861228866811, 2.09861228866811, 2.09861228866811, - 2.09861228866811, 2.09861228866811, 2.09861228866811 } + { 1.40546511, 1.40546511, 2.09861229, 2.09861229, 2.09861229, 1, 2.09861229, + 1.40546511, 1.40546511, 1.40546511, 2.09861229, 2.09861229, 1.40546511, 1, + 2.09861229, 2.09861229, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, + { 0, 1.40546511, 0, 0, 0, 2, 0, 0, 4.21639532, 4.21639532, 0, 0, 0, 3, 0, 0, + 2.09861229, 2.09861229, 2.09861229, 6.29583687, 2.09861229, 2.09861229, + 2.09861229, 2.09861229, 2.09861229, 2.09861229, 2.09861229, 2.09861229, + 2.09861229, 2.09861229, 2.09861229, 2.09861229, 2.09861229, 2.09861229, + 2.09861229, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, + { 1.40546511, 0, 0, 0, 0, 1, 0, 1.40546511, 0, 0, 0, 0, 1.40546511, 1, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2.09861229, + 2.09861229, 2.09861229, 2.09861229, 2.09861229, 2.09861229, 2.09861229, + 2.09861229, 2.09861229, 2.09861229 } }; - CheckVectors(output, expected, 1e-12); + CheckVectors(output, expected, 1e-6); } /** - * Test TFIDF encoding for characters, using rawcount as - * tf type and smoothidf as false. + * Test the Tf-Idf encoding algorithm for individual characters with the + * raw count term frequency type and the non-smooth inverse document frequency + * type. */ -BOOST_AUTO_TEST_CASE(RawcountTfIdfEncodingIndividualCharactersTest) +BOOST_AUTO_TEST_CASE(RawCountTfIdfEncodingIndividualCharactersTest) { vector input = { "GACCA", @@ -917,25 +1078,29 @@ BOOST_AUTO_TEST_CASE(RawcountTfIdfEncodingIndividualCharactersTest) }; arma::mat output; - TfIdfEncoding - encoder(TfIdfEncodingPolicy::TfTypes::RAW_COUNT, false); + TfIdfEncoding encoder( + TfIdfEncodingPolicy::TfTypes::RAW_COUNT, false); - // Passing a empty string to encode charactersrawcountsmoothidftrue. encoder.Encode(input, output, CharExtract()); + + /* The expected values were obtained by almost the same script as in + RawCountSmoothIdfEncodingIndividualCharactersTest. The only difference is + smooth_idf equals False. */ arma::mat target = { - { 1.4054651081081644, 2, 2.8109302162163288, 0, 0 }, - { 0, 2, 2.8109302162163288, 2.8109302162163288, 2.0986122886681100 }, - { 1.4054651081081644, 1, 0, 1.4054651081081644, 0 } + { 1.40546510810816, 2, 2.81093021621633, 0, 0 }, + { 0, 2, 2.81093021621633, 2.81093021621633, 2.09861228866811 }, + { 1.40546510810816, 1, 0, 1.40546510810816, 0 } }; + CheckMatrices(output, target.t(), 1e-12); } /** - * Test the Tf Idf encoding algorithm to store result in vector - * in case of individual character encoding, using raw count as type, - * and smoothidf as false. + * Test the Tf-Idf encoding algorithm for individual characters with the + * raw count term frequency type and the non-smooth inverse document frequency + * type. The output type is vector>. */ -BOOST_AUTO_TEST_CASE(VectorRawcountEncodingIndividualCharactersTest) +BOOST_AUTO_TEST_CASE(VectorRawCountTfIdfEncodingIndividualCharactersTest) { std::vector input = { "GACCA", @@ -944,22 +1109,26 @@ BOOST_AUTO_TEST_CASE(VectorRawcountEncodingIndividualCharactersTest) }; vector> output; - TfIdfEncoding - encoder(TfIdfEncodingPolicy::TfTypes::RAW_COUNT, false); + TfIdfEncoding encoder( + TfIdfEncodingPolicy::TfTypes::RAW_COUNT, false); - // Passing a empty string to encode characters. encoder.Encode(input, output, CharExtract()); + + /* The expected values were obtained by almost the same script as in + RawCountSmoothIdfEncodingIndividualCharactersTest. The only difference is + smooth_idf equals False. */ vector> expected = { - { 1.4054651081081644, 2, 2.8109302162163288, 0, 0 }, - { 0, 2, 2.8109302162163288, 2.8109302162163288, 2.0986122886681100 }, - { 1.4054651081081644, 1, 0, 1.4054651081081644, 0 } + { 1.40546510810816, 2, 2.81093021621633, 0, 0 }, + { 0, 2, 2.81093021621633, 2.81093021621633, 2.09861228866811 }, + { 1.40546510810816, 1, 0, 1.40546510810816, 0 } }; + CheckVectors(output, expected, 1e-12); } /** - * Test TFIDF encoding for characters, using binary weighting scheme - * for tf and smoothidf as true. + * Test the Tf-Idf encoding algorithm for individual characters with the + * binary term frequency type and the smooth inverse document frequency type. */ BOOST_AUTO_TEST_CASE(BinarySmoothIdfEncodingIndividualCharactersTest) { @@ -970,24 +1139,29 @@ BOOST_AUTO_TEST_CASE(BinarySmoothIdfEncodingIndividualCharactersTest) }; arma::mat output; - TfIdfEncoding - encoder(TfIdfEncodingPolicy::TfTypes::BINARY, true); + TfIdfEncoding encoder( + TfIdfEncodingPolicy::TfTypes::BINARY, true); - // Passing a empty string to encode charactersrawcountsmoothidftrue. encoder.Encode(input, output, CharExtract()); + + /* The expected values were obtained by almost the same script as in + RawCountSmoothIdfEncodingIndividualCharactersTest. The only difference is + tf_type equals 'binary'. */ arma::mat target = { - { 1.2876820724517808, 1, 1.2876820724517808, 0, 0 }, - { 0, 1, 1.2876820724517808, 1.2876820724517808, 1.6931471805599454 }, - { 1.2876820724517808, 1, 0, 1.2876820724517808, 0 } + { 1.28768207245178, 1, 1.28768207245178, 0, 0 }, + { 0, 1, 1.28768207245178, 1.28768207245178, 1.69314718055995 }, + { 1.28768207245178, 1, 0, 1.28768207245178, 0 } }; + CheckMatrices(output, target.t(), 1e-12); } /** - * Test TFIDF encoding for characters to store results in vector, using binary - * weighting scheme for tf and smoothidf as true. + * Test the Tf-Idf encoding algorithm for individual characters with the + * binary term frequency type and the smooth inverse document frequency type. + * The output type is vector>. */ -BOOST_AUTO_TEST_CASE(VectorBnarySmoothIdfEncodingIndividualCharactersTest) +BOOST_AUTO_TEST_CASE(VectorBinarySmoothIdfEncodingIndividualCharactersTest) { std::vector input = { "GACCA", @@ -999,19 +1173,24 @@ BOOST_AUTO_TEST_CASE(VectorBnarySmoothIdfEncodingIndividualCharactersTest) TfIdfEncoding encoder(TfIdfEncodingPolicy::TfTypes::BINARY, true); - // Passing a empty string to encode characters. encoder.Encode(input, output, CharExtract()); + + /* The expected values were obtained by almost the same script as in + RawCountSmoothIdfEncodingIndividualCharactersTest. The only difference is + tf_type equals 'binary'. */ vector> expected = { - { 1.2876820724517808, 1, 1.2876820724517808, 0, 0 }, - { 0, 1, 1.2876820724517808, 1.2876820724517808, 1.6931471805599454 }, - { 1.2876820724517808, 1, 0, 1.2876820724517808, 0 } + { 1.28768207245178, 1, 1.28768207245178, 0, 0 }, + { 0, 1, 1.28768207245178, 1.28768207245178, 1.69314718055995 }, + { 1.28768207245178, 1, 0, 1.28768207245178, 0 } }; + CheckVectors(output, expected, 1e-12); } /** - * Test TFIDF encoding for characters, using binary - * as weighting scheme and smoothidf as false. + * Test the Tf-Idf encoding algorithm for individual characters with the + * binary term frequency type and the non-smooth inverse document frequency + * type. */ BOOST_AUTO_TEST_CASE(BinaryTfIdfEncodingIndividualCharactersTest) { @@ -1022,22 +1201,27 @@ BOOST_AUTO_TEST_CASE(BinaryTfIdfEncodingIndividualCharactersTest) }; arma::mat output; - TfIdfEncoding - encoder(TfIdfEncodingPolicy::TfTypes::BINARY, false); + TfIdfEncoding encoder( + TfIdfEncodingPolicy::TfTypes::BINARY, false); - // Passing a empty string to encode charactersrawcountsmoothidftrue. encoder.Encode(input, output, CharExtract()); + + /* The expected values were obtained by almost the same script as in + RawCountSmoothIdfEncodingIndividualCharactersTest. The only difference is + tf_type equals 'binary' and smooth_idf equals False. */ arma::mat target = { - { 1.4054651081081644, 1, 1.4054651081081644, 0, 0 }, - { 0, 1, 1.4054651081081644, 1.4054651081081644, 2.0986122886681100 }, - { 1.4054651081081644, 1, 0, 1.4054651081081644, 0 } + { 1.40546510810816, 1, 1.40546510810816, 0, 0 }, + { 0, 1, 1.40546510810816, 1.40546510810816, 2.09861228866811 }, + { 1.40546510810816, 1, 0, 1.40546510810816, 0 } }; + CheckMatrices(output, target.t(), 1e-12); } /** - * Test TFIDF encoding for characters, using sublinear - * as weighting scheme and smoothidf as true. + * Test the Tf-Idf encoding algorithm for individual characters with the + * sublinear term frequency type and the smooth inverse document frequency + * type. */ BOOST_AUTO_TEST_CASE(SublinearSmoothIdfEncodingIndividualCharactersTest) { @@ -1048,23 +1232,28 @@ BOOST_AUTO_TEST_CASE(SublinearSmoothIdfEncodingIndividualCharactersTest) }; arma::mat output; - TfIdfEncoding - encoder(TfIdfEncodingPolicy::TfTypes::SUBLINEAR_TF, true); + TfIdfEncoding encoder( + TfIdfEncodingPolicy::TfTypes::SUBLINEAR_TF, true); - // Passing a empty string to encode charactersrawcountsmoothidftrue. encoder.Encode(input, output, CharExtract()); + + /* The expected values were obtained by almost the same script as in + RawCountSmoothIdfEncodingIndividualCharactersTest. The only difference is + tf_type equals 'sublinear_tf'. */ arma::mat target = { - { 1.2876820724517808, 1.6931471805599454, 2.1802352704293200, 0, 0 }, - { 0, 1.6931471805599454, 2.1802352704293200, 2.1802352704293200, - 1.6931471805599454 }, - { 1.2876820724517808, 1, 0, 1.2876820724517808, 0 } + { 1.28768207245178, 1.69314718055995, 2.18023527042932, 0, 0 }, + { 0, 1.69314718055995, 2.18023527042932, 2.18023527042932, + 1.69314718055995 }, + { 1.28768207245178, 1, 0, 1.28768207245178, 0 } }; + CheckMatrices(output, target.t(), 1e-12); } /** - * Test TFIDF encoding for characters, using sublinear - * as weighting scheme and smoothidf as false. + * Test the Tf-Idf encoding algorithm for individual characters with the + * sublinear term frequency type and the non-smooth inverse document frequency + * type. */ BOOST_AUTO_TEST_CASE(SublinearTfIdfEncodingIndividualCharactersTest) { @@ -1078,20 +1267,25 @@ BOOST_AUTO_TEST_CASE(SublinearTfIdfEncodingIndividualCharactersTest) TfIdfEncoding encoder(TfIdfEncodingPolicy::TfTypes::SUBLINEAR_TF, false); - // Passing a empty string to encode charactersrawcountsmoothidftrue. + /* The expected values were obtained by almost the same script as in + RawCountSmoothIdfEncodingIndividualCharactersTest. The only difference is + tf_type equals 'sublinear_tf' and smooth_idf equals False. */ encoder.Encode(input, output, CharExtract()); + arma::mat target = { - { 1.4054651081081644, 1.6931471805599454, 2.3796592851687173, 0, 0 }, - { 0, 1.6931471805599454, 2.3796592851687173, 2.3796592851687173, - 2.0986122886681100 }, - { 1.4054651081081644, 1, 0, 1.4054651081081644, 0 } + { 1.40546510810816, 1.69314718055995, 2.37965928516872, 0, 0 }, + { 0, 1.69314718055995, 2.37965928516872, 2.37965928516872, + 2.09861228866811 }, + { 1.40546510810816, 1, 0, 1.40546510810816, 0 } }; + CheckMatrices(output, target.t(), 1e-12); } /** - * Test TFIDF encoding for characters, using term - * Frequency as weighting scheme and smoothidf as true. + * Test the Tf-Idf encoding algorithm for individual characters with the + * standard term frequency type and the smooth inverse document frequency + * type. */ BOOST_AUTO_TEST_CASE(TermFrequencySmoothIdfEncodingIndividualCharactersTest) { @@ -1102,23 +1296,77 @@ BOOST_AUTO_TEST_CASE(TermFrequencySmoothIdfEncodingIndividualCharactersTest) }; arma::mat output; - TfIdfEncoding - encoder(TfIdfEncodingPolicy::TfTypes::TERM_FREQUENCY, true); + TfIdfEncoding encoder( + TfIdfEncodingPolicy::TfTypes::TERM_FREQUENCY, true); - // Passing a empty string to encode charactersrawcountsmoothidftrue. encoder.Encode(input, output, CharExtract()); + + /* The expected values were obtained by the following Python script: + + from sklearn.feature_extraction.text import CountVectorizer + from sklearn.feature_extraction.text import TfidfTransformer + from collections import OrderedDict + import numpy as np + import re + + input_string = [ + "GACCA", + "ABCABCD", + "GAB" + ] + + smooth_idf = True + + dictionary = OrderedDict() + + count = 0 + for line in input_string: + for word in list(line): + if word and (not (word in dictionary)): + dictionary[word] = count + count += 1 + + def tokenizer(line): + return list(line) + + vectorizer = CountVectorizer(strip_accents=False, lowercase=False, + preprocessor=None, tokenizer=tokenizer, stop_words=None, + vocabulary=dictionary, binary=False) + + count = vectorizer.fit_transform(input_string) + + lens = np.array(list(map(len, input_string))).reshape(len(input_string), 1) + + tf = count.toarray() / lens + + transformer = TfidfTransformer(norm=None, smooth_idf=smooth_idf, + sublinear_tf=False) + + X = transformer.fit_transform(tf) + + def format_result(value): + if value == int(value): + return str(int(value)) + else: + return "{0:.16}".format(value) + + for row in X.toarray(): + print("{ " + ", ".join(map(format_result, row)) + " },") + */ arma::mat target = { { 0.2575364144903562, 0.4, 0.5150728289807124, 0, 0 }, { 0, 0.2857142857142857, 0.3679091635576516, 0.3679091635576516, 0.2418781686514208 }, { 0.4292273574839269, 0.3333333333333333, 0, 0.4292273574839269, 0 } }; + CheckMatrices(output, target.t(), 1e-12); } /** - * Test TFIDF encoding for characters, using Term - * Frequency as weighting scheme and smoothidf as false. + * Test the Tf-Idf encoding algorithm for individual characters with the + * standard term frequency type and the non-smooth inverse document frequency + * type. */ BOOST_AUTO_TEST_CASE(TermFrequencyTfIdfEncodingIndividualCharactersTest) { @@ -1129,22 +1377,26 @@ BOOST_AUTO_TEST_CASE(TermFrequencyTfIdfEncodingIndividualCharactersTest) }; arma::mat output; - TfIdfEncoding - encoder(TfIdfEncodingPolicy::TfTypes::TERM_FREQUENCY, false); + TfIdfEncoding encoder( + TfIdfEncodingPolicy::TfTypes::TERM_FREQUENCY, false); - // Passing a empty string to encode charactersrawcountsmoothidftrue. encoder.Encode(input, output, CharExtract()); + + /* The expected values were obtained by almost the same script as in + TermFrequencySmoothIdfEncodingIndividualCharactersTest. The only difference + is smooth_idf equals False. */ arma::mat target = { { 0.2810930216216329, 0.4, 0.5621860432432658, 0, 0 }, { 0, 0.2857142857142857, 0.4015614594594755, 0.4015614594594755, 0.2998017555240157 }, { 0.4684883693693881, 0.3333333333333333, 0, 0.4684883693693881, 0 } }; + CheckMatrices(output, target.t(), 1e-12); } /** - * Serialization test for the TF-IDF encoding algorithm with + * Serialization test for the Tf-Idf encoding algorithm with * the SplitByAnyOf tokenizer. */ BOOST_AUTO_TEST_CASE(SplitByAnyOfTfIdfEncodingSerialization) From 4ef37bd6958952f993e09c3f11a22c551121a421 Mon Sep 17 00:00:00 2001 From: jeffin143 Date: Wed, 1 Apr 2020 12:22:28 +0530 Subject: [PATCH 077/111] probable typo that caused the CI build failed --- src/mlpack/methods/preprocess/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mlpack/methods/preprocess/CMakeLists.txt b/src/mlpack/methods/preprocess/CMakeLists.txt index 2f9f4b0725..5b5a7bbdb6 100644 --- a/src/mlpack/methods/preprocess/CMakeLists.txt +++ b/src/mlpack/methods/preprocess/CMakeLists.txt @@ -44,6 +44,6 @@ add_markdown_docs(preprocess_scale "cli;python" "preprocessing") if (STB_AVAILABLE) add_cli_executable(image_converter) - add_python_binding(limage_converter) + add_python_binding(image_converter) add_markdown_docs(image_converter "cli;python" "preprocessing") endif () \ No newline at end of file From 26fedd0e62e44fafd5c88c9e245544771a6deee9 Mon Sep 17 00:00:00 2001 From: jeffin143 Date: Wed, 1 Apr 2020 18:58:20 +0530 Subject: [PATCH 078/111] Docs update --- src/mlpack/core/data/save_image.cpp | 5 ++++ .../preprocess/image_converter_main.cpp | 23 ++++++++----------- src/mlpack/tests/image_load_test.cpp | 18 ++++++++++++++- .../tests/main_tests/image_converter_test.cpp | 2 +- 4 files changed, 32 insertions(+), 16 deletions(-) diff --git a/src/mlpack/core/data/save_image.cpp b/src/mlpack/core/data/save_image.cpp index c1802073e4..944087e6d3 100644 --- a/src/mlpack/core/data/save_image.cpp +++ b/src/mlpack/core/data/save_image.cpp @@ -54,6 +54,11 @@ bool SaveImage(const std::string& filename, Log::Warn << "Only the first image will be saved!" << std::endl; } + if (info.Width() * info.Height() * info.Channels() != image.n_elem) + { + Log::Fatal << "The dimension given does not match the image dimesion. \n"; + } + bool status = false; unsigned char* imageMem = image.memptr(); diff --git a/src/mlpack/methods/preprocess/image_converter_main.cpp b/src/mlpack/methods/preprocess/image_converter_main.cpp index af05853f7a..8cc73359fc 100644 --- a/src/mlpack/methods/preprocess/image_converter_main.cpp +++ b/src/mlpack/methods/preprocess/image_converter_main.cpp @@ -23,16 +23,16 @@ using namespace mlpack::data; PROGRAM_INFO("Image Converter", // Short description. "A utility to load an image or set of images into a single dataset that" - "can then be used by other mlpack methods and utilities. This can also" - "unpack an image dataset into individual files, for instance after mlpack" - "methods have been used.", + " can then be used by other mlpack methods and utilities. This can also" + " unpack an image dataset into individual files, for instance after mlpack" + " methods have been used.", // Long description. - "This utility takes a image or an array of images and loads them to a" + "This utility takes an image or an array of images and loads them to a" " matrix. You can optionally specify the height " + PRINT_PARAM_STRING("height") + " width " + PRINT_PARAM_STRING("width") + " and channel " + PRINT_PARAM_STRING("channels") + " of the images that" - "needs to be loaded; otherwise, these parameters will be automatically" - "detected from the image." + " needs to be loaded; otherwise, these parameters will be automatically" + " detected from the image." "\n" "There are other options too, that can be specified such as " + PRINT_PARAM_STRING("quality") @@ -75,7 +75,7 @@ static void mlpackMain() // Parse command line options. const vector fileNames = CLI::GetParam >("input"); arma::mat out; - + if (!CLI::HasParam("save")) { ReportIgnoredParam("width", "Width of image is determined from file."); @@ -88,8 +88,8 @@ static void mlpackMain() } else { - RequireNoneOrAllPassed({ "save", "width", "height", "channels" }, true, - "image size information is needed when 'save' is specified!"); + RequireNoneOrAllPassed({ "save", "width", "height", "channels", "dataset" } + , true, "Image size information is needed when 'save' is specified!"); // Positive value for width. RequireParamValue("width", [](int x) { return x >= 0;}, true, "width must be positive"); @@ -108,11 +108,6 @@ static void mlpackMain() const size_t channels = CLI::GetParam("channels"); const size_t quality = CLI::GetParam("quality"); data::ImageInfo info(width, height, channels, quality); - if (!CLI::HasParam("dataset")) - { - throw std::runtime_error("Please provide a input matrix to save " - "images from"); - } Save(fileNames, CLI::GetParam("dataset"), info, true); } } diff --git a/src/mlpack/tests/image_load_test.cpp b/src/mlpack/tests/image_load_test.cpp index 7d2108264c..b5172dbeb0 100644 --- a/src/mlpack/tests/image_load_test.cpp +++ b/src/mlpack/tests/image_load_test.cpp @@ -12,7 +12,6 @@ #include #include -#include #include "test_tools.hpp" #include "serialization.hpp" @@ -76,6 +75,23 @@ BOOST_AUTO_TEST_CASE(SaveImageAPITest) remove("APITest.bmp"); } +/** + * Test if an image with a wrong dimesion throws an expected + * exception while saving. + */ +BOOST_AUTO_TEST_CASE(SaveImageWrongInfo) +{ + data::ImageInfo info(5, 5, 3, 90); + + arma::Mat im1; + size_t dimension = info.Width() * info.Height() * info.Channels(); + im1 = arma::randi>(24 * 25 * 7, 1); + Log::Fatal.ignoreInput = true; + BOOST_REQUIRE_THROW(data::Save("APITest.bmp", im1, info, false), + std::runtime_error); + Log::Fatal.ignoreInput = false; +} + /** * Test that the image is loaded correctly into the matrix using the API * for vectors. diff --git a/src/mlpack/tests/main_tests/image_converter_test.cpp b/src/mlpack/tests/main_tests/image_converter_test.cpp index 35e140c45e..cd77cbaf66 100644 --- a/src/mlpack/tests/main_tests/image_converter_test.cpp +++ b/src/mlpack/tests/main_tests/image_converter_test.cpp @@ -1,5 +1,5 @@ /** - * @file load_save_image_test.cpp + * @file image_converter_test.cpp * @author Jeffin Sam * * Test mlpackMain() of load_save_image_main.cpp. From 3c2c3ce5f0717cd0426eafca0a4a82593bcb536c Mon Sep 17 00:00:00 2001 From: AndreiMihalea Date: Thu, 2 Apr 2020 15:34:56 +0300 Subject: [PATCH 079/111] Changed return type accordingly to #2339 --- .../methods/ann/loss_functions/margin_ranking_loss.hpp | 9 +++------ .../ann/loss_functions/margin_ranking_loss_impl.hpp | 8 +++----- 2 files changed, 6 insertions(+), 11 deletions(-) diff --git a/src/mlpack/methods/ann/loss_functions/margin_ranking_loss.hpp b/src/mlpack/methods/ann/loss_functions/margin_ranking_loss.hpp index b5d64f0aae..e8798dd62b 100644 --- a/src/mlpack/methods/ann/loss_functions/margin_ranking_loss.hpp +++ b/src/mlpack/methods/ann/loss_functions/margin_ranking_loss.hpp @@ -49,12 +49,9 @@ class MarginRankingLoss * function. * @param target The label vector which contains values of -1 or 1. */ - template < - typename InputType, - typename TargetType - > - double Forward(const InputType& input, - const TargetType& target); + template + typename InputType::elem_type Forward(const InputType& input, + const TargetType& target); /** * Ordinary feed backward pass of a neural network. diff --git a/src/mlpack/methods/ann/loss_functions/margin_ranking_loss_impl.hpp b/src/mlpack/methods/ann/loss_functions/margin_ranking_loss_impl.hpp index 79e4d879e9..59649cbb3a 100644 --- a/src/mlpack/methods/ann/loss_functions/margin_ranking_loss_impl.hpp +++ b/src/mlpack/methods/ann/loss_functions/margin_ranking_loss_impl.hpp @@ -26,11 +26,9 @@ MarginRankingLoss::MarginRankingLoss( } template -template < - typename InputType, - typename TargetType -> -double MarginRankingLoss::Forward( +template +typename InputType::elem_type +MarginRankingLoss::Forward( const InputType& input, const TargetType& target) { From b371bd45aa5b23f92fb5fc6b1ad341f306e8e03c Mon Sep 17 00:00:00 2001 From: jeffin sam Date: Thu, 2 Apr 2020 18:51:52 +0530 Subject: [PATCH 080/111] Apply suggestions from code review Co-Authored-By: Ryan Curtin --- HISTORY.md | 4 ++-- src/mlpack/core/data/save_image.cpp | 5 +++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/HISTORY.md b/HISTORY.md index 06067e1b4e..4f8443e9ae 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -31,8 +31,8 @@ * CMake fix for finding STB include directory (#2145). - * Add bindings for Loading and Saving image (#2019); `mlpack_image_converter` - from the command-line, `mlpack_image_converter()` from Python. + * Add bindings for loading and saving images (#2019); `mlpack_image_converter` + from the command-line, `mlpack.image_converter()` from Python. * Add normalization support for CF binding (#2136). diff --git a/src/mlpack/core/data/save_image.cpp b/src/mlpack/core/data/save_image.cpp index 944087e6d3..3235cb12b1 100644 --- a/src/mlpack/core/data/save_image.cpp +++ b/src/mlpack/core/data/save_image.cpp @@ -54,9 +54,10 @@ bool SaveImage(const std::string& filename, Log::Warn << "Only the first image will be saved!" << std::endl; } - if (info.Width() * info.Height() * info.Channels() != image.n_elem) + if (info.Width() * info.Height() * info.Channels() != image.n_elem) { - Log::Fatal << "The dimension given does not match the image dimesion. \n"; + Log::Fatal << "data::Save(): The given image dimensions do not match the " + << "dimensions of the matrix to be saved!" << std::endl; } bool status = false; From 7d26d2e5a563898fe7cacfa89e9233bd8435a1f3 Mon Sep 17 00:00:00 2001 From: jeffin143 Date: Thu, 2 Apr 2020 23:16:51 +0530 Subject: [PATCH 081/111] Add julia binding --- src/mlpack/core/data/save_image.cpp | 2 +- src/mlpack/methods/preprocess/CMakeLists.txt | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/mlpack/core/data/save_image.cpp b/src/mlpack/core/data/save_image.cpp index 3235cb12b1..2c4654b711 100644 --- a/src/mlpack/core/data/save_image.cpp +++ b/src/mlpack/core/data/save_image.cpp @@ -57,7 +57,7 @@ bool SaveImage(const std::string& filename, if (info.Width() * info.Height() * info.Channels() != image.n_elem) { Log::Fatal << "data::Save(): The given image dimensions do not match the " - << "dimensions of the matrix to be saved!" << std::endl; + << "dimensions of the matrix to be saved!" << std::endl; } bool status = false; diff --git a/src/mlpack/methods/preprocess/CMakeLists.txt b/src/mlpack/methods/preprocess/CMakeLists.txt index 5b5a7bbdb6..d1b1dd5816 100644 --- a/src/mlpack/methods/preprocess/CMakeLists.txt +++ b/src/mlpack/methods/preprocess/CMakeLists.txt @@ -40,10 +40,12 @@ add_markdown_docs(preprocess_imputer "cli" "preprocessing") add_cli_executable(preprocess_scale) add_python_binding(preprocess_scale) -add_markdown_docs(preprocess_scale "cli;python" "preprocessing") +add_julia_binding(preprocess_scale) +add_markdown_docs(preprocess_scale "cli;python;julia" "preprocessing") if (STB_AVAILABLE) add_cli_executable(image_converter) add_python_binding(image_converter) - add_markdown_docs(image_converter "cli;python" "preprocessing") + add_julia_binding(image_converter) + add_markdown_docs(image_converter "cli;python;julia" "preprocessing") endif () \ No newline at end of file From b84745ac5a3b80c56115362088b8109dd017faf5 Mon Sep 17 00:00:00 2001 From: AndreiMihalea Date: Thu, 2 Apr 2020 21:27:47 +0300 Subject: [PATCH 082/111] Rebuild From 0f31459896de2b6faad8f0aa5e535004c0e6837b Mon Sep 17 00:00:00 2001 From: AndreiMihalea Date: Fri, 3 Apr 2020 01:29:21 +0300 Subject: [PATCH 083/111] Rebuild From 16d2fd3e945383775aff3145d745d110cdae5e49 Mon Sep 17 00:00:00 2001 From: Mikhail Lozhnikov Date: Fri, 3 Apr 2020 01:56:16 +0300 Subject: [PATCH 084/111] Various documentation fixes for string encoding algorithms. --- src/mlpack/core/data/string_encoding.hpp | 9 +++--- src/mlpack/core/data/string_encoding_impl.hpp | 4 +-- .../bag_of_words_encoding_policy.hpp | 20 +++++++------ .../dictionary_encoding_policy.hpp | 8 +++--- .../tf_idf_encoding_policy.hpp | 28 +++++++++---------- 5 files changed, 36 insertions(+), 33 deletions(-) diff --git a/src/mlpack/core/data/string_encoding.hpp b/src/mlpack/core/data/string_encoding.hpp index db5896576c..9ed18cba6e 100644 --- a/src/mlpack/core/data/string_encoding.hpp +++ b/src/mlpack/core/data/string_encoding.hpp @@ -101,7 +101,7 @@ class StringEncoding * * @tparam OutputType Type of the output container. The function supports * the following types: arma::mat, arma::sp_mat, - * std::vector>. + * std::vector>. * @tparam TokenizerType Type of the tokenizer. * * @param input Corpus of text to encode. @@ -148,7 +148,7 @@ class StringEncoding * * @tparam OutputType Type of the output container. The function supports * the following types: arma::mat, arma::sp_mat, - * std::vector>. + * std::vector>. * @tparam TokenizerType Type of the tokenizer. * @tparam PolicyType The type of the encoding policy. It has to be * equal to EncodingPolicyType. @@ -180,6 +180,7 @@ class StringEncoding * @tparam TokenizerType Type of the tokenizer. * @tparam PolicyType The type of the encoding policy. It has to be * equal to EncodingPolicyType. + * @tparam ElemType Type of the output values. * * @param input Corpus of text to encode. * @param output Output container to store the result. @@ -193,9 +194,9 @@ class StringEncoding * 2. IsTokenEmpty() that accepts a token and returns true if the given * token is empty. */ - template + template void EncodeHelper(const std::vector& input, - std::vector>& output, + std::vector>& output, const TokenizerType& tokenizer, PolicyType& policy, typename std::enable_if& input, } template -template +template void StringEncoding:: EncodeHelper(const std::vector& input, - std::vector>& output, + std::vector>& output, const TokenizerType& tokenizer, PolicyType& policy, typename std::enable_if>. + * Overloaded function to save the result in vector>. * - * @tparam OutputType Type of the output vector. + * @tparam ElemType Type of the output values. * * @param output Output matrix to store the encoded results. * @param datasetSize The number of strings in the input dataset. @@ -70,13 +70,13 @@ class BagOfWordsEncodingPolicy input dataset (not used). * @param dictionarySize The size of the dictionary. */ - template - static void InitMatrix(std::vector>& output, + template + static void InitMatrix(std::vector>& output, const size_t datasetSize, const size_t /* maxNumTokens */, const size_t dictionarySize) { - output.resize(datasetSize, std::vector(dictionarySize, 0)); + output.resize(datasetSize, std::vector(dictionarySize)); } /** @@ -106,16 +106,18 @@ class BagOfWordsEncodingPolicy * the encoded token to the output. The encoder writes data in the * row-major order. * - * Overload function to accepted vector> as output type. + * Overloaded function to accept vector> as the output + * type. + * + * @tparam ElemType Type of the output values. * * @param output Output matrix to store the encoded results. * @param value The encoded token. * @param line The line number at which the encoding is performed. * @param index The line token number at which the encoding is performed. - * @tparam OutputType The type of output vector. */ - template - static void Encode(std::vector>& output, + template + static void Encode(std::vector>& output, const size_t value, const size_t line, const size_t /* index */) diff --git a/src/mlpack/core/data/string_encoding_policies/dictionary_encoding_policy.hpp b/src/mlpack/core/data/string_encoding_policies/dictionary_encoding_policy.hpp index 4b26860abd..d0b49775e0 100644 --- a/src/mlpack/core/data/string_encoding_policies/dictionary_encoding_policy.hpp +++ b/src/mlpack/core/data/string_encoding_policies/dictionary_encoding_policy.hpp @@ -80,13 +80,13 @@ class DictionaryEncodingPolicy * the result into the given vector to avoid padding. The encoder writes data * in the row-major order. * - * @tparam OutputType Type of the output vector. + * @tparam ElemType Type of the output values. * - * @param output Output vector to store the encoded results. + * @param output Output vector to store the encoded line. * @param value The encoded token. */ - template - static void Encode(std::vector& output, size_t value) + template + static void Encode(std::vector& output, size_t value) { output.push_back(value); } diff --git a/src/mlpack/core/data/string_encoding_policies/tf_idf_encoding_policy.hpp b/src/mlpack/core/data/string_encoding_policies/tf_idf_encoding_policy.hpp index 4acfc03c56..141bb8ea1f 100644 --- a/src/mlpack/core/data/string_encoding_policies/tf_idf_encoding_policy.hpp +++ b/src/mlpack/core/data/string_encoding_policies/tf_idf_encoding_policy.hpp @@ -36,7 +36,7 @@ class TfIdfEncodingPolicy { public: /** - * Enum class used to identify the type of term frequency encoding. + * Enum class used to identify the type of the term frequency statistics. * * The present implementation supports the following types: * BINARY Term frequency equals 1 if the row contains the encoded @@ -58,8 +58,8 @@ class TfIdfEncodingPolicy }; /** - * A constructor for the class which is use to set the type of term frequency - * and also the value for smoothIdf. + * Construct this using the term frequency type and the inverse document + * frequency type. * * @param tfType Type of the term frequency statistics. * @param smoothIdf Used to indicate whether to use smooth idf or not. @@ -102,9 +102,9 @@ class TfIdfEncodingPolicy * The function initializes the output matrix. The encoder writes data * in the row-major order. * - * Overloaded function to store result in vector>. + * Overloaded function to save the result in vector>. * - * @tparam OutputType Type of the output vector. + * @tparam ElemType Type of the output values. * * @param output Output matrix to store the encoded results. * @param datasetSize The number of strings in the input dataset. @@ -112,13 +112,13 @@ class TfIdfEncodingPolicy input dataset (not used). * @param dictionarySize The size of the dictionary. */ - template - static void InitMatrix(std::vector>& output, + template + static void InitMatrix(std::vector>& output, const size_t datasetSize, const size_t /* maxNumTokens */, const size_t dictionarySize) { - output.resize(datasetSize, std::vector(dictionarySize, 0)); + output.resize(datasetSize, std::vector(dictionarySize)); } /** @@ -155,26 +155,26 @@ class TfIdfEncodingPolicy * the encoded token to the output. The encoder writes data in the * row-major order. * - * Overloaded function to accept vector> as the output + * Overloaded function to accept vector> as the output * type. * - * @tparam OutputType Type of the output vector. + * @tparam ElemType Type of the output values. * * @param output Output matrix to store the encoded results. * @param value The encoded token. * @param line The line number at which the encoding is performed. * @param index The token index in the line. */ - template - void Encode(std::vector >& output, + template + void Encode(std::vector>& output, const size_t value, const size_t line, const size_t /* index */) { - const OutputType tf = TermFrequency( + const ElemType tf = TermFrequency( tokensFrequences[line][value], linesSizes[line]); - const OutputType idf = InverseDocumentFrequency( + const ElemType idf = InverseDocumentFrequency( output.size(), numContainingStrings[value]); output[line][value - 1] = tf * idf; From 4bf8321ef16185ba905903f1e9f52eb52b5a3dc8 Mon Sep 17 00:00:00 2001 From: kartikdutt18 Date: Fri, 14 Feb 2020 19:23:15 +0530 Subject: [PATCH 085/111] Add Cosine Embeddings --- .../methods/ann/loss_functions/CMakeLists.txt | 2 + .../loss_functions/cosine_embedding_loss.hpp | 145 ++++++++++++++++++ .../cosine_embedding_loss_impl.hpp | 88 +++++++++++ src/mlpack/tests/loss_functions_test.cpp | 26 ++++ 4 files changed, 261 insertions(+) create mode 100644 src/mlpack/methods/ann/loss_functions/cosine_embedding_loss.hpp create mode 100644 src/mlpack/methods/ann/loss_functions/cosine_embedding_loss_impl.hpp diff --git a/src/mlpack/methods/ann/loss_functions/CMakeLists.txt b/src/mlpack/methods/ann/loss_functions/CMakeLists.txt index 8a2111091b..d807d6ecf9 100644 --- a/src/mlpack/methods/ann/loss_functions/CMakeLists.txt +++ b/src/mlpack/methods/ann/loss_functions/CMakeLists.txt @@ -3,6 +3,8 @@ set(SOURCES cross_entropy_error.hpp cross_entropy_error_impl.hpp + cosine_embedding_loss.hpp + cosine_embedding_loss_impl.hpp dice_loss.hpp dice_loss_impl.hpp earth_mover_distance.hpp diff --git a/src/mlpack/methods/ann/loss_functions/cosine_embedding_loss.hpp b/src/mlpack/methods/ann/loss_functions/cosine_embedding_loss.hpp new file mode 100644 index 0000000000..926a74fe4a --- /dev/null +++ b/src/mlpack/methods/ann/loss_functions/cosine_embedding_loss.hpp @@ -0,0 +1,145 @@ +/** + * @file cosine_embedding_loss.hpp + * @author Kartik Dutt + * + * Definition of the Cosine Embedding loss function. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_METHODS_ANN_LOSS_FUNCTION_COSINE_EMBEDDING_HPP +#define MLPACK_METHODS_ANN_LOSS_FUNCTION_COSINE_EMBEDDING_HPP + +#include + +namespace mlpack { +namespace ann /** Artificial Neural Network. */ { + +/** + * Cosine Embeddings Loss function is used for measuring whether two inputs are + * similar or dissimilar, using using the cosine distance, and is typically used + * for learning nonlinear embeddings or semi-supervised learning. + * + * @f{eqnarray*}{ + * f(x) = 1 - cos(x1, x2) , for y = 1 + * f(x) = max(0, cos(x1, x2) - margin) , for y = -1 + * @f} + * + * @tparam InputDataType Type of the input data (arma::colvec, arma::mat, + * arma::sp_mat or arma::cube). + * @tparam OutputDataType Type of the output data (arma::colvec, arma::mat, + * arma::sp_mat or arma::cube). + */ +template < + typename InputDataType = arma::mat, + typename OutputDataType = arma::mat +> +class CosineEmbeddingLoss +{ + public: + /** + * Create the CosineEmbeddingLoss object. + * + * @param margin Increases cosine distance in case of dissimilarity. + * Refer definition of cosine-embedding-loss above. + * @param takeMean Boolean variable to specify whether to take mean or not. + * Specifies reduction method i.e. sum or mean corresponding + * to 0 and 1 respectively. Default value = 0. + */ + CosineEmbeddingLoss(const double margin = 0.0, const bool takeMean = false); + + /** + * Ordinary feed forward pass of a neural network. + * + * @param x1 Input data used for evaluating the given function. + * @param x2 Input data used for evaluating the given function. + * @param y Input data used to determine whether to calculate + * cosine similarity or dissimilarity. It should only + * contain values equal to 1 or -1. + */ + template + double Forward(const InputType&& x1, const InputType x2, + const TargetType&& y); + + /** + * Ordinary feed backward pass of a neural network. The negative log + * likelihood layer expects that the input contains log-probabilities for + * each class. The layer also expects a class index, in the range between 1 + * and the number of classes, as target when calling the Forward function. + * + * @param input The propagated input activation. + * @param target The target vector, that contains the class index in the range + * between 1 and the number of classes. + * @param output The calculated error. + */ + template + void Backward(const InputType&& input, + const TargetType&& target, + OutputType&& output); + + //! Get the input parameter. + InputDataType& InputParameter() const { return inputParameter; } + //! Modify the input parameter. + InputDataType& InputParameter() { return inputParameter; } + + //! Get the output parameter. + OutputDataType& OutputParameter() const { return outputParameter; } + //! Modify the output parameter. + OutputDataType& OutputParameter() { return outputParameter; } + + //! Get the delta. + OutputDataType& Delta() const { return delta; } + //! Modify the delta. + OutputDataType& Delta() { return delta; } + + //! Get the value of takeMean. + bool TakeMean() const { return takeMean; } + //! Modify the value of takeMean. + bool& TakeMean() { return takeMean; } + + //! Get the value of margin. + bool Margin() const { return margin; } + //! Modify the value of takeMean. + bool& Margin() { return margin; } + + + /** + * Serialize the layer. + */ + template + void serialize(Archive& /* ar */, const unsigned int /* version */); + + private: + // Returns Cosine-Distance between two vectors. + template + double CosineDistance(InputType x1, InputType x2) + { + return 1.0 - arma::accu(x1 % x2) / + std::sqrt(arma::accu(arma::pow(x1, 2)) * arma::accu(arma::pow(x2, 2))); + } + + //! Locally-stored delta object. + OutputDataType delta; + + //! Locally-stored input parameter object. + InputDataType inputParameter; + + //! Locally-stored output parameter object. + OutputDataType outputParameter; + + //! Locally-stored value of margin parameter. + double margin; + + //! Locally-stored value of takeMean parameter. + bool takeMean; +}; // class CosineEmbeddingLoss + +} // namespace ann +} // namespace mlpack + +// Include implementation. +#include "cosine_embedding_loss_impl.hpp" + +#endif \ No newline at end of file diff --git a/src/mlpack/methods/ann/loss_functions/cosine_embedding_loss_impl.hpp b/src/mlpack/methods/ann/loss_functions/cosine_embedding_loss_impl.hpp new file mode 100644 index 0000000000..e837397459 --- /dev/null +++ b/src/mlpack/methods/ann/loss_functions/cosine_embedding_loss_impl.hpp @@ -0,0 +1,88 @@ +/** + * @file cosine_embedding_loss_impl.hpp + * @author Kartik Dutt + * + * Implementation of the Cosine Embedding loss function. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_METHODS_ANN_LOSS_FUNCTION_COSINE_EMBEDDING_IMPL_HPP +#define MLPACK_METHODS_ANN_LOSS_FUNCTION_COSINE_EMBEDDING_IMPL_HPP + +// In case it hasn't yet been included. +#include "cosine_embedding_loss.hpp" + +namespace mlpack { +namespace ann /** Artificial Neural Network. */ { + +template +CosineEmbeddingLoss::CosineEmbeddingLoss( + const double margin, const bool takeMean): + margin(margin), takeMean(takeMean) +{ + // Nothing to do here. +} + +template +template +double CosineEmbeddingLoss::Forward( + const InputType &&x1, const InputType &&x2, const TargetType &&y) +{ + if (x1.n_rows != x2.n_rows || x1.n_cols != x2.n_cols || + x1.n_slices != x2.n_slices) + { + Log::Fatal << "Input Dimensions must be same." << std::endl; + } + + if(y.n_elem != x1.n_rows * x1.n_slices) + { + Log::Fatal << "Number of rows mismatch." << std::endl; + } + + arma::colvec inputTemp1 = arma::vectorise(x1); + arma::colvec inputTemp2 = arma::vectorise(x2); + double loss = 0.0; + const size_t cols = x1.n_cols; + const size_t batchSize = x1.n_rows * x1.n_slices; + + for(size_t i = 0; i < batchSize; i += cols) + { + if (y(i / cols) == 1) + { + loss += 1 - CosineDistance(inputTemp1(arma::span(i, i + cols - 1)), + inputTemp2(arma::span(i, i + cols -1))); + } + else if (y(i / cols) == -1) + { + loss += std::max(CosineDistance(inputTemp1(arma::span(i, i + cols - 1)), + inputTemp2(arma::span(i, i + cols -1))) - margin, 0); + } + else + { + Log::Fatal << "y should only contain 1 and -1." << std::endl; + } + } + + if(takeMean) + { + loss = (double)loss / y.n_elem; + } + return loss; +} + +template +template +void CosineEmbeddingLoss::serialize( + Archive& /* ar */, + const unsigned int /* version */) +{ + // Nothing to do here. +} + +} // namespace ann +} // namespace mlpack + +#endif \ No newline at end of file diff --git a/src/mlpack/tests/loss_functions_test.cpp b/src/mlpack/tests/loss_functions_test.cpp index e361c398d0..f65d72143a 100644 --- a/src/mlpack/tests/loss_functions_test.cpp +++ b/src/mlpack/tests/loss_functions_test.cpp @@ -25,8 +25,15 @@ #include #include #include +<<<<<<< HEAD #include +<<<<<<< HEAD #include +======= +======= +#include +>>>>>>> Add Cosine Embeddings +>>>>>>> Add Cosine Embeddings #include #include @@ -578,4 +585,23 @@ BOOST_AUTO_TEST_CASE(HingeEmbeddingLossTest) BOOST_REQUIRE_EQUAL(output.n_rows, input.n_rows); BOOST_REQUIRE_EQUAL(output.n_cols, input.n_cols); } + +/** + * Simple test for the Cosine Embedding loss function. + */ +BOOST_AUTO_TEST_CASE(CosineEmbeddingLossTest) +{ + arma::mat input1, input2, y, target, output; + double loss; + CosineEmbeddingLoss<> module; + + // Test the Forward function. Loss should be 0 if input1 = input2 and y = 1. + input1 = arma::ones(10, 1); + input2 = arma::ones(10, 1); + y = arma::ones(1); + loss = module.Forward(std::move(input1),std::move(input1), + std::move(target)); + BOOST_REQUIRE_CLOSE(loss, 0.0, 1e-4); +} + BOOST_AUTO_TEST_SUITE_END(); From 834f602118d64681e8694bdc6c655b416df86602 Mon Sep 17 00:00:00 2001 From: kartikdutt18 Date: Sat, 15 Feb 2020 20:26:40 +0530 Subject: [PATCH 086/111] Completed Cosine Embeddings --- .../loss_functions/cosine_embedding_loss.hpp | 37 +++++---- .../cosine_embedding_loss_impl.hpp | 78 ++++++++++++++++--- src/mlpack/tests/loss_functions_test.cpp | 68 ++++++++++++++-- 3 files changed, 153 insertions(+), 30 deletions(-) diff --git a/src/mlpack/methods/ann/loss_functions/cosine_embedding_loss.hpp b/src/mlpack/methods/ann/loss_functions/cosine_embedding_loss.hpp index 926a74fe4a..dca30b8891 100644 --- a/src/mlpack/methods/ann/loss_functions/cosine_embedding_loss.hpp +++ b/src/mlpack/methods/ann/loss_functions/cosine_embedding_loss.hpp @@ -59,9 +59,13 @@ class CosineEmbeddingLoss * cosine similarity or dissimilarity. It should only * contain values equal to 1 or -1. */ - template - double Forward(const InputType&& x1, const InputType x2, - const TargetType&& y); + template< + typename FirstTensor, + typename SecondTensor, + typename ThirdTensor + > + double Forward(const FirstTensor&& x1, const SecondTensor&& x2, + const ThirdTensor&& y); /** * Ordinary feed backward pass of a neural network. The negative log @@ -69,15 +73,22 @@ class CosineEmbeddingLoss * each class. The layer also expects a class index, in the range between 1 * and the number of classes, as target when calling the Forward function. * - * @param input The propagated input activation. - * @param target The target vector, that contains the class index in the range + * @param x1 The propagated input activation. + * @param x2 The propagated input activation. + * @param y The target vector, that contains the class index in the range * between 1 and the number of classes. * @param output The calculated error. */ - template - void Backward(const InputType&& input, - const TargetType&& target, - OutputType&& output); + template< + typename FirstTensor, + typename SecondTensor, + typename ThirdTensor, + typename OutputTensor + > + void Backward(const FirstTensor&& x1, + const SecondTensor&& x2, + const ThirdTensor&& y, + const OutputTensor&& output); //! Get the input parameter. InputDataType& InputParameter() const { return inputParameter; } @@ -113,11 +124,11 @@ class CosineEmbeddingLoss private: // Returns Cosine-Distance between two vectors. - template - double CosineDistance(InputType x1, InputType x2) + template + double CosineDistance(const FirstTensor&& x1, + const SecondTensor&& x2) { - return 1.0 - arma::accu(x1 % x2) / - std::sqrt(arma::accu(arma::pow(x1, 2)) * arma::accu(arma::pow(x2, 2))); + return arma::dot(arma::normalise(x1), arma::normalise(x2)); } //! Locally-stored delta object. diff --git a/src/mlpack/methods/ann/loss_functions/cosine_embedding_loss_impl.hpp b/src/mlpack/methods/ann/loss_functions/cosine_embedding_loss_impl.hpp index e837397459..a463742b49 100644 --- a/src/mlpack/methods/ann/loss_functions/cosine_embedding_loss_impl.hpp +++ b/src/mlpack/methods/ann/loss_functions/cosine_embedding_loss_impl.hpp @@ -26,18 +26,26 @@ CosineEmbeddingLoss::CosineEmbeddingLoss( // Nothing to do here. } -template -template +template +template< + typename FirstTensor, + typename SecondTensor, + typename ThirdTensor +> double CosineEmbeddingLoss::Forward( - const InputType &&x1, const InputType &&x2, const TargetType &&y) + const FirstTensor&& x1, + const SecondTensor&& x2, + const ThirdTensor&& y) { + const size_t cols = x1.n_cols; + const size_t batchSize = x1.n_elem / cols; if (x1.n_rows != x2.n_rows || x1.n_cols != x2.n_cols || - x1.n_slices != x2.n_slices) + x1.n_elem != x2.n_elem) { Log::Fatal << "Input Dimensions must be same." << std::endl; } - if(y.n_elem != x1.n_rows * x1.n_slices) + if (y.n_elem < batchSize) { Log::Fatal << "Number of rows mismatch." << std::endl; } @@ -45,20 +53,19 @@ double CosineEmbeddingLoss::Forward( arma::colvec inputTemp1 = arma::vectorise(x1); arma::colvec inputTemp2 = arma::vectorise(x2); double loss = 0.0; - const size_t cols = x1.n_cols; - const size_t batchSize = x1.n_rows * x1.n_slices; - for(size_t i = 0; i < batchSize; i += cols) + for(size_t i = 0; i < inputTemp1.n_elem; i+=cols) { if (y(i / cols) == 1) { loss += 1 - CosineDistance(inputTemp1(arma::span(i, i + cols - 1)), - inputTemp2(arma::span(i, i + cols -1))); + inputTemp2(arma::span(i, i + cols - 1))); } else if (y(i / cols) == -1) { - loss += std::max(CosineDistance(inputTemp1(arma::span(i, i + cols - 1)), - inputTemp2(arma::span(i, i + cols -1))) - margin, 0); + double currentLoss = CosineDistance(inputTemp1(arma::span(i, i + cols - 1)), + inputTemp2(arma::span(i, i + cols - 1))) - margin; + loss += currentLoss > 0 ? currentLoss : 0; } else { @@ -66,13 +73,60 @@ double CosineEmbeddingLoss::Forward( } } - if(takeMean) + if (takeMean) { loss = (double)loss / y.n_elem; } return loss; } +template +template< + typename FirstTensor, + typename SecondTensor, + typename ThirdTensor, + typename OutputTensor +> +void CosineEmbeddingLoss::Backward( + const FirstTensor&& x1, + const SecondTensor&& x2, + const ThirdTensor&& y, + const OutputTensor&& output) +{ + const size_t cols = x1.n_cols; + const size_t batchSize = x1.n_elem / cols; + if (x1.n_rows != x2.n_rows || x1.n_cols != x2.n_cols || + x1.n_elem != x2.n_elem) + { + Log::Fatal << "Input Dimensions must be same." << std::endl; + } + + if (y.n_elem < batchSize) + { + Log::Fatal << "Number of rows mismatch." << std::endl; + } + + arma::colvec inputTemp1 = arma::vectorise(x1); + arma::colvec inputTemp2 = arma::vectorise(x2); + arma::colvec outputTemp(inputTemp1.n_elem, 1); + + for(size_t i = 0; i < inputTemp1.n_elem; i+=cols) + { + if (y(i / cols) != 1 && y(i / cols) != -1) + { + Log::Fatal << "y should only contain 1 and -1." << std::endl; + } + + outputTemp(arma::span(i, i + cols -1)) = arma::sign(y(i / cols)) * + (arma::normalise(inputTemp2(arma::span(i, i + cols - 1))) - + arma::normalise(inputTemp1(arma::span(i, i + cols - 1))) * + CosineDistance(inputTemp1(arma::span(i, i + cols - 1)), + inputTemp2(arma::span(i, i + cols - 1)))) / + std::sqrt(arma::accu(arma::pow(inputTemp1(arma::span(i, i + cols - 1)), + 2))); + } +} + template template void CosineEmbeddingLoss::serialize( diff --git a/src/mlpack/tests/loss_functions_test.cpp b/src/mlpack/tests/loss_functions_test.cpp index f65d72143a..8a5819afe9 100644 --- a/src/mlpack/tests/loss_functions_test.cpp +++ b/src/mlpack/tests/loss_functions_test.cpp @@ -591,17 +591,75 @@ BOOST_AUTO_TEST_CASE(HingeEmbeddingLossTest) */ BOOST_AUTO_TEST_CASE(CosineEmbeddingLossTest) { - arma::mat input1, input2, y, target, output; + arma::mat input1, input2, y, output; double loss; CosineEmbeddingLoss<> module; // Test the Forward function. Loss should be 0 if input1 = input2 and y = 1. - input1 = arma::ones(10, 1); - input2 = arma::ones(10, 1); - y = arma::ones(1); + input1 = arma::mat(1, 10); + input2 = arma::mat(1, 10); + input1.ones(); + input2.ones(); + y = arma::mat(1, 1); + y.ones(); loss = module.Forward(std::move(input1),std::move(input1), - std::move(target)); + std::move(y)); BOOST_REQUIRE_CLOSE(loss, 0.0, 1e-4); + // Check for dissimilarity. + y.fill(-1); + loss = module.Forward(std::move(input1),std::move(input1), + std::move(y)); + BOOST_REQUIRE_CLOSE(loss, 1.0, 1e-4); + + // Test the Backward function. + module.Backward(std::move(input1), std::move(input1), std::move(y), + std::move(output)); + + input1 = arma::mat(3, 2); + input2 = arma::mat(3, 2); + input1.fill(1); + input1(4) = 2; + input2.fill(1); + input2(0) = 2; + input2(1) = 2; + input2(2) = 2; + y = arma::mat(3, 1); + y.fill(-1); + loss = module.Forward(std::move(input1),std::move(input2), + std::move(y)); + // Caclulated using torch.nn.CosineEmbeddingLoss(). + BOOST_REQUIRE_CLOSE(loss, 2.8973665961010275, 1e-3); + + // Check for correctness for cube. + CosineEmbeddingLoss<> module2(0.5); + + arma::cube input3(3, 2, 2); + arma::cube input4(3, 2, 2); + input3.fill(1); + input4.fill(1); + input3(0) = 2; + input3(1) = 2; + input3(4) = 2; + input3(6) = 2; + input3(8) = 2; + input3(10) = 2; + input4(2) = 2; + input4(9) = 2; + input4(11) = 2; + y = arma::mat(6, 1); + y.fill(1); + y(5) = -1; + y(2) = -1; + loss = module2.Forward(std::move(input3),std::move(input4), + std::move(y)); + // Caclulated using torch.nn.CosineEmbeddingLoss(). + BOOST_REQUIRE_CLOSE(loss, 1.0513167019494862, 1e-3); + + // Check Output for mean type of reduction. + CosineEmbeddingLoss<> module3(0.0, true); + loss = module3.Forward(std::move(input3),std::move(input4), + std::move(y)); + BOOST_REQUIRE_CLOSE(loss, 0.34188611699158106, 1e-3); } BOOST_AUTO_TEST_SUITE_END(); From 531358172356636c5e7531e390c28999da70c885 Mon Sep 17 00:00:00 2001 From: kartikdutt18 Date: Tue, 18 Feb 2020 21:36:51 +0530 Subject: [PATCH 087/111] Added more tests (all input types), some style fix, fixed backward Remaining style fix Added more descriptive logs Spacing issue fixed in the new description Renamed Input Variables Change to BOOST_SMALL for precision error Style Fix' Rebased Small typo fix Design Acc. to ANN API Style Fix Style Fix --- .../loss_functions/cosine_embedding_loss.hpp | 94 ++++++------- .../cosine_embedding_loss_impl.hpp | 123 ++++++++---------- src/mlpack/tests/loss_functions_test.cpp | 69 +++++----- 3 files changed, 128 insertions(+), 158 deletions(-) diff --git a/src/mlpack/methods/ann/loss_functions/cosine_embedding_loss.hpp b/src/mlpack/methods/ann/loss_functions/cosine_embedding_loss.hpp index dca30b8891..f792c19675 100644 --- a/src/mlpack/methods/ann/loss_functions/cosine_embedding_loss.hpp +++ b/src/mlpack/methods/ann/loss_functions/cosine_embedding_loss.hpp @@ -19,7 +19,7 @@ namespace ann /** Artificial Neural Network. */ { /** * Cosine Embeddings Loss function is used for measuring whether two inputs are - * similar or dissimilar, using using the cosine distance, and is typically used + * similar or dissimilar, using the cosine distance, and is typically used * for learning nonlinear embeddings or semi-supervised learning. * * @f{eqnarray*}{ @@ -41,96 +41,77 @@ class CosineEmbeddingLoss public: /** * Create the CosineEmbeddingLoss object. - * + * * @param margin Increases cosine distance in case of dissimilarity. * Refer definition of cosine-embedding-loss above. + * @param similarity Determines whether to use similarity or dissimilarity for + * comparision. * @param takeMean Boolean variable to specify whether to take mean or not. * Specifies reduction method i.e. sum or mean corresponding * to 0 and 1 respectively. Default value = 0. */ - CosineEmbeddingLoss(const double margin = 0.0, const bool takeMean = false); + CosineEmbeddingLoss(const double margin = 0.0, + const bool similarity = true, + const bool takeMean = false); /** * Ordinary feed forward pass of a neural network. * - * @param x1 Input data used for evaluating the given function. - * @param x2 Input data used for evaluating the given function. - * @param y Input data used to determine whether to calculate - * cosine similarity or dissimilarity. It should only - * contain values equal to 1 or -1. + * @param input Input data used for evaluating the specified function. + * @param target The target vector. */ - template< - typename FirstTensor, - typename SecondTensor, - typename ThirdTensor - > - double Forward(const FirstTensor&& x1, const SecondTensor&& x2, - const ThirdTensor&& y); + template + double Forward(const InputType& input, const TargetType& target); /** - * Ordinary feed backward pass of a neural network. The negative log - * likelihood layer expects that the input contains log-probabilities for - * each class. The layer also expects a class index, in the range between 1 - * and the number of classes, as target when calling the Forward function. + * Ordinary feed backward pass of a neural network. * - * @param x1 The propagated input activation. - * @param x2 The propagated input activation. - * @param y The target vector, that contains the class index in the range - * between 1 and the number of classes. + * @param input The propagated input activation. + * @param target The target vector. * @param output The calculated error. */ - template< - typename FirstTensor, - typename SecondTensor, - typename ThirdTensor, - typename OutputTensor - > - void Backward(const FirstTensor&& x1, - const SecondTensor&& x2, - const ThirdTensor&& y, - const OutputTensor&& output); + template + void Backward(const InputType& input, + const TargetType& target, + OutputType& output); //! Get the input parameter. - InputDataType& InputParameter() const { return inputParameter; } + InputDataType &InputParameter() const { return inputParameter; } //! Modify the input parameter. - InputDataType& InputParameter() { return inputParameter; } + InputDataType &InputParameter() { return inputParameter; } //! Get the output parameter. - OutputDataType& OutputParameter() const { return outputParameter; } + OutputDataType &OutputParameter() const { return outputParameter; } //! Modify the output parameter. - OutputDataType& OutputParameter() { return outputParameter; } + OutputDataType &OutputParameter() { return outputParameter; } //! Get the delta. - OutputDataType& Delta() const { return delta; } + OutputDataType &Delta() const { return delta; } //! Modify the delta. - OutputDataType& Delta() { return delta; } + OutputDataType &Delta() { return delta; } //! Get the value of takeMean. bool TakeMean() const { return takeMean; } //! Modify the value of takeMean. - bool& TakeMean() { return takeMean; } + bool &TakeMean() { return takeMean; } //! Get the value of margin. - bool Margin() const { return margin; } + double Margin() const { return margin; } //! Modify the value of takeMean. - bool& Margin() { return margin; } - + double &Margin() { return margin; } + + //! Get the value of similarity hyperparameter. + bool Similarity() const { return similarity; } + //! Modify the value of takeMean. + bool &Similarity() { return similarity; } /** * Serialize the layer. */ template - void serialize(Archive& /* ar */, const unsigned int /* version */); + void serialize(Archive& ar, const unsigned int /* version */); private: - // Returns Cosine-Distance between two vectors. - template - double CosineDistance(const FirstTensor&& x1, - const SecondTensor&& x2) - { - return arma::dot(arma::normalise(x1), arma::normalise(x2)); - } - //! Locally-stored delta object. OutputDataType delta; @@ -140,10 +121,13 @@ class CosineEmbeddingLoss //! Locally-stored output parameter object. OutputDataType outputParameter; - //! Locally-stored value of margin parameter. + //! Locally-stored value of similarity hyper-parameter. + bool similarity; + + //! Locally-stored value of margin hyper-parameter. double margin; - //! Locally-stored value of takeMean parameter. + //! Locally-stored value of takeMean hyper-parameter. bool takeMean; }; // class CosineEmbeddingLoss @@ -153,4 +137,4 @@ class CosineEmbeddingLoss // Include implementation. #include "cosine_embedding_loss_impl.hpp" -#endif \ No newline at end of file +#endif diff --git a/src/mlpack/methods/ann/loss_functions/cosine_embedding_loss_impl.hpp b/src/mlpack/methods/ann/loss_functions/cosine_embedding_loss_impl.hpp index a463742b49..5e0bbccbe8 100644 --- a/src/mlpack/methods/ann/loss_functions/cosine_embedding_loss_impl.hpp +++ b/src/mlpack/methods/ann/loss_functions/cosine_embedding_loss_impl.hpp @@ -20,123 +20,104 @@ namespace ann /** Artificial Neural Network. */ { template CosineEmbeddingLoss::CosineEmbeddingLoss( - const double margin, const bool takeMean): - margin(margin), takeMean(takeMean) + const double margin, const bool similarity, const bool takeMean): + margin(margin), similarity(similarity), takeMean(takeMean) { // Nothing to do here. } template -template< - typename FirstTensor, - typename SecondTensor, - typename ThirdTensor -> +template double CosineEmbeddingLoss::Forward( - const FirstTensor&& x1, - const SecondTensor&& x2, - const ThirdTensor&& y) + const InputType& input, const TargetType& target) { - const size_t cols = x1.n_cols; - const size_t batchSize = x1.n_elem / cols; - if (x1.n_rows != x2.n_rows || x1.n_cols != x2.n_cols || - x1.n_elem != x2.n_elem) + const size_t cols = input.n_cols; + const size_t batchSize = input.n_elem / cols; + if (arma::size(input) != arma::size(target)) { - Log::Fatal << "Input Dimensions must be same." << std::endl; + Log::Fatal << "Input Tensors must have same dimensions." << std::endl; } - if (y.n_elem < batchSize) - { - Log::Fatal << "Number of rows mismatch." << std::endl; - } - - arma::colvec inputTemp1 = arma::vectorise(x1); - arma::colvec inputTemp2 = arma::vectorise(x2); + arma::colvec inputTemp1 = arma::vectorise(input); + arma::colvec inputTemp2 = arma::vectorise(target); double loss = 0.0; - for(size_t i = 0; i < inputTemp1.n_elem; i+=cols) + for (size_t i = 0; i < inputTemp1.n_elem; i += cols) { - if (y(i / cols) == 1) + double cosDist = kernel::CosineDistance::Evaluate(inputTemp1(arma::span(i, + i + cols - 1)), inputTemp2(arma::span(i, i + cols - 1))); + if (similarity) { - loss += 1 - CosineDistance(inputTemp1(arma::span(i, i + cols - 1)), - inputTemp2(arma::span(i, i + cols - 1))); - } - else if (y(i / cols) == -1) - { - double currentLoss = CosineDistance(inputTemp1(arma::span(i, i + cols - 1)), - inputTemp2(arma::span(i, i + cols - 1))) - margin; - loss += currentLoss > 0 ? currentLoss : 0; + loss += 1 - cosDist; } else { - Log::Fatal << "y should only contain 1 and -1." << std::endl; + double currentLoss = cosDist - margin; + loss += currentLoss > 0 ? currentLoss : 0; } } if (takeMean) { - loss = (double)loss / y.n_elem; + loss = (double)loss / batchSize; } + return loss; } template -template< - typename FirstTensor, - typename SecondTensor, - typename ThirdTensor, - typename OutputTensor -> +template void CosineEmbeddingLoss::Backward( - const FirstTensor&& x1, - const SecondTensor&& x2, - const ThirdTensor&& y, - const OutputTensor&& output) + const InputType& input, + const TargetType& target, + OutputType& output) { - const size_t cols = x1.n_cols; - const size_t batchSize = x1.n_elem / cols; - if (x1.n_rows != x2.n_rows || x1.n_cols != x2.n_cols || - x1.n_elem != x2.n_elem) + const size_t cols = input.n_cols; + const size_t batchSize = input.n_elem / cols; + if (arma::size(input) != arma::size(target)) { - Log::Fatal << "Input Dimensions must be same." << std::endl; + Log::Fatal << "Input Tensors must have same dimensions." << std::endl; } - if (y.n_elem < batchSize) - { - Log::Fatal << "Number of rows mismatch." << std::endl; - } - - arma::colvec inputTemp1 = arma::vectorise(x1); - arma::colvec inputTemp2 = arma::vectorise(x2); - arma::colvec outputTemp(inputTemp1.n_elem, 1); + arma::colvec inputTemp1 = arma::vectorise(input); + arma::colvec inputTemp2 = arma::vectorise(target); + output.set_size(arma::size(inputTemp1)); - for(size_t i = 0; i < inputTemp1.n_elem; i+=cols) + arma::colvec outputTemp(output.memptr(), inputTemp1.n_elem, + false, false); + for (size_t i = 0; i < inputTemp1.n_elem; i += cols) { - if (y(i / cols) != 1 && y(i / cols) != -1) + double cosDist = kernel::CosineDistance::Evaluate(inputTemp1(arma::span(i, + i + cols -1)), inputTemp2(arma::span(i, i + cols -1))); + + if (cosDist < margin && !similarity) { - Log::Fatal << "y should only contain 1 and -1." << std::endl; + outputTemp(arma::span(i, i + cols - 1)).zeros(); + } + else + { + int multiplier = similarity ? 1 : -1; + outputTemp(arma::span(i, i + cols -1)) = -1 * multiplier * + (arma::normalise(inputTemp2(arma::span(i, i + cols - 1))) - + cosDist * arma::normalise(inputTemp1(arma::span(i, i + cols - + 1)))) / std::sqrt(arma::accu(arma::pow(inputTemp1(arma::span(i, i + + cols - 1)), 2))); } - - outputTemp(arma::span(i, i + cols -1)) = arma::sign(y(i / cols)) * - (arma::normalise(inputTemp2(arma::span(i, i + cols - 1))) - - arma::normalise(inputTemp1(arma::span(i, i + cols - 1))) * - CosineDistance(inputTemp1(arma::span(i, i + cols - 1)), - inputTemp2(arma::span(i, i + cols - 1)))) / - std::sqrt(arma::accu(arma::pow(inputTemp1(arma::span(i, i + cols - 1)), - 2))); } } template template void CosineEmbeddingLoss::serialize( - Archive& /* ar */, + Archive& ar , const unsigned int /* version */) { - // Nothing to do here. + ar & BOOST_SERIALIZATION_NVP(margin); + ar & BOOST_SERIALIZATION_NVP(similarity); + ar & BOOST_SERIALIZATION_NVP(takeMean); } } // namespace ann } // namespace mlpack -#endif \ No newline at end of file +#endif diff --git a/src/mlpack/tests/loss_functions_test.cpp b/src/mlpack/tests/loss_functions_test.cpp index 8a5819afe9..a09cf77b2a 100644 --- a/src/mlpack/tests/loss_functions_test.cpp +++ b/src/mlpack/tests/loss_functions_test.cpp @@ -25,15 +25,9 @@ #include #include #include -<<<<<<< HEAD #include -<<<<<<< HEAD #include -======= -======= #include ->>>>>>> Add Cosine Embeddings ->>>>>>> Add Cosine Embeddings #include #include @@ -602,19 +596,22 @@ BOOST_AUTO_TEST_CASE(CosineEmbeddingLossTest) input2.ones(); y = arma::mat(1, 1); y.ones(); - loss = module.Forward(std::move(input1),std::move(input1), - std::move(y)); - BOOST_REQUIRE_CLOSE(loss, 0.0, 1e-4); + loss = module.Forward(input1, input1); + BOOST_REQUIRE_SMALL(loss, 1e-6); + + // Test the Backward function. + module.Backward(input1, input1, output); + BOOST_REQUIRE_SMALL(arma::accu(output), 1e-6); + // Check for dissimilarity. - y.fill(-1); - loss = module.Forward(std::move(input1),std::move(input1), - std::move(y)); + module.Similarity() = false; + loss = module.Forward(input1, input1); BOOST_REQUIRE_CLOSE(loss, 1.0, 1e-4); // Test the Backward function. - module.Backward(std::move(input1), std::move(input1), std::move(y), - std::move(output)); - + module.Backward(input1, input1, output); + BOOST_REQUIRE_SMALL(arma::accu(output), 1e-6); + input1 = arma::mat(3, 2); input2 = arma::mat(3, 2); input1.fill(1); @@ -623,15 +620,16 @@ BOOST_AUTO_TEST_CASE(CosineEmbeddingLossTest) input2(0) = 2; input2(1) = 2; input2(2) = 2; - y = arma::mat(3, 1); - y.fill(-1); - loss = module.Forward(std::move(input1),std::move(input2), - std::move(y)); + loss = module.Forward(input1, input2); // Caclulated using torch.nn.CosineEmbeddingLoss(). - BOOST_REQUIRE_CLOSE(loss, 2.8973665961010275, 1e-3); + BOOST_REQUIRE_CLOSE(loss, 2.897367, 1e-3); + + // Test the Backward function. + module.Backward(input1, input2, output); + BOOST_REQUIRE_CLOSE(arma::accu(output), 0.06324556, 1e-3); // Check for correctness for cube. - CosineEmbeddingLoss<> module2(0.5); + CosineEmbeddingLoss<> module2(0.5, true); arma::cube input3(3, 2, 2); arma::cube input4(3, 2, 2); @@ -646,20 +644,27 @@ BOOST_AUTO_TEST_CASE(CosineEmbeddingLossTest) input4(2) = 2; input4(9) = 2; input4(11) = 2; - y = arma::mat(6, 1); - y.fill(1); - y(5) = -1; - y(2) = -1; - loss = module2.Forward(std::move(input3),std::move(input4), - std::move(y)); + loss = module2.Forward(input3, input4); // Caclulated using torch.nn.CosineEmbeddingLoss(). - BOOST_REQUIRE_CLOSE(loss, 1.0513167019494862, 1e-3); + BOOST_REQUIRE_CLOSE(loss, 0.55395, 1e-3); + + // Test the Backward function. + module2.Backward(input3, input4, output); + BOOST_REQUIRE_CLOSE(arma::accu(output), -0.36649111, 1e-3); // Check Output for mean type of reduction. - CosineEmbeddingLoss<> module3(0.0, true); - loss = module3.Forward(std::move(input3),std::move(input4), - std::move(y)); - BOOST_REQUIRE_CLOSE(loss, 0.34188611699158106, 1e-3); + CosineEmbeddingLoss<> module3(0.0, true, true); + loss = module3.Forward(input3, input4); + BOOST_REQUIRE_CLOSE(loss, 0.092325, 1e-3); + + // Check correctness for cube. + module3.Similarity() = false; + loss = module3.Forward(input3, input4); + BOOST_REQUIRE_CLOSE(loss, 0.90767498236, 1e-3); + + // Test the Backward function. + module3.Backward(input3, input4, output); + BOOST_REQUIRE_CLOSE(arma::accu(output), 0.36649111, 1e-4); } BOOST_AUTO_TEST_SUITE_END(); From fe5d4bd1b01f55134a81e58e448c1dad20c34fc2 Mon Sep 17 00:00:00 2001 From: kartikdutt18 Date: Fri, 3 Apr 2020 13:12:01 +0530 Subject: [PATCH 088/111] Added to History.md, Changed to templated return, remobed unnecc brackets --- HISTORY.md | 2 + .../loss_functions/cosine_embedding_loss.hpp | 27 +++++++------- .../cosine_embedding_loss_impl.hpp | 37 +++++++++---------- src/mlpack/tests/loss_functions_test.cpp | 4 +- 4 files changed, 35 insertions(+), 35 deletions(-) diff --git a/HISTORY.md b/HISTORY.md index bcd3e5fac6..0241e86681 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -72,6 +72,8 @@ * Add Hinge Embedding Loss Function (#2229). + * Add Cosine Embedding Loss Function (#2209). + ### mlpack 3.2.2 ###### 2019-11-26 * Add `valid` and `same` padding option in `Convolution` and `Atrous diff --git a/src/mlpack/methods/ann/loss_functions/cosine_embedding_loss.hpp b/src/mlpack/methods/ann/loss_functions/cosine_embedding_loss.hpp index f792c19675..044fbbdd0e 100644 --- a/src/mlpack/methods/ann/loss_functions/cosine_embedding_loss.hpp +++ b/src/mlpack/methods/ann/loss_functions/cosine_embedding_loss.hpp @@ -18,7 +18,7 @@ namespace mlpack { namespace ann /** Artificial Neural Network. */ { /** - * Cosine Embeddings Loss function is used for measuring whether two inputs are + * Cosine Embedding Loss function is used for measuring whether two inputs are * similar or dissimilar, using the cosine distance, and is typically used * for learning nonlinear embeddings or semi-supervised learning. * @@ -51,8 +51,8 @@ class CosineEmbeddingLoss * to 0 and 1 respectively. Default value = 0. */ CosineEmbeddingLoss(const double margin = 0.0, - const bool similarity = true, - const bool takeMean = false); + const bool similarity = true, + const bool takeMean = false); /** * Ordinary feed forward pass of a neural network. @@ -61,7 +61,8 @@ class CosineEmbeddingLoss * @param target The target vector. */ template - double Forward(const InputType& input, const TargetType& target); + typename InputType::elem_type Forward(const InputType& input, + const TargetType& target); /** * Ordinary feed backward pass of a neural network. @@ -76,34 +77,34 @@ class CosineEmbeddingLoss OutputType& output); //! Get the input parameter. - InputDataType &InputParameter() const { return inputParameter; } + InputDataType& InputParameter() const { return inputParameter; } //! Modify the input parameter. - InputDataType &InputParameter() { return inputParameter; } + InputDataType& InputParameter() { return inputParameter; } //! Get the output parameter. - OutputDataType &OutputParameter() const { return outputParameter; } + OutputDataType& OutputParameter() const { return outputParameter; } //! Modify the output parameter. - OutputDataType &OutputParameter() { return outputParameter; } + OutputDataType& OutputParameter() { return outputParameter; } //! Get the delta. - OutputDataType &Delta() const { return delta; } + OutputDataType& Delta() const { return delta; } //! Modify the delta. - OutputDataType &Delta() { return delta; } + OutputDataType& Delta() { return delta; } //! Get the value of takeMean. bool TakeMean() const { return takeMean; } //! Modify the value of takeMean. - bool &TakeMean() { return takeMean; } + bool& TakeMean() { return takeMean; } //! Get the value of margin. double Margin() const { return margin; } //! Modify the value of takeMean. - double &Margin() { return margin; } + double& Margin() { return margin; } //! Get the value of similarity hyperparameter. bool Similarity() const { return similarity; } //! Modify the value of takeMean. - bool &Similarity() { return similarity; } + bool& Similarity() { return similarity; } /** * Serialize the layer. diff --git a/src/mlpack/methods/ann/loss_functions/cosine_embedding_loss_impl.hpp b/src/mlpack/methods/ann/loss_functions/cosine_embedding_loss_impl.hpp index 5e0bbccbe8..b708567b28 100644 --- a/src/mlpack/methods/ann/loss_functions/cosine_embedding_loss_impl.hpp +++ b/src/mlpack/methods/ann/loss_functions/cosine_embedding_loss_impl.hpp @@ -28,39 +28,38 @@ CosineEmbeddingLoss::CosineEmbeddingLoss( template template -double CosineEmbeddingLoss::Forward( - const InputType& input, const TargetType& target) +typename InputType::elem_type +CosineEmbeddingLoss::Forward( + const InputType& input, + const TargetType& target) { + typedef typename InputType::elem_type ElemType; + const size_t cols = input.n_cols; const size_t batchSize = input.n_elem / cols; if (arma::size(input) != arma::size(target)) - { Log::Fatal << "Input Tensors must have same dimensions." << std::endl; - } arma::colvec inputTemp1 = arma::vectorise(input); arma::colvec inputTemp2 = arma::vectorise(target); - double loss = 0.0; + ElemType loss = 0.0; for (size_t i = 0; i < inputTemp1.n_elem; i += cols) { - double cosDist = kernel::CosineDistance::Evaluate(inputTemp1(arma::span(i, - i + cols - 1)), inputTemp2(arma::span(i, i + cols - 1))); + const ElemType cosDist = kernel::CosineDistance::Evaluate( + inputTemp1(arma::span(i, i + cols - 1)), inputTemp2(arma::span(i, + i + cols - 1))); if (similarity) - { loss += 1 - cosDist; - } else { - double currentLoss = cosDist - margin; + const ElemType currentLoss = cosDist - margin; loss += currentLoss > 0 ? currentLoss : 0; } } if (takeMean) - { - loss = (double)loss / batchSize; - } + loss = (ElemType) loss / batchSize; return loss; } @@ -72,12 +71,12 @@ void CosineEmbeddingLoss::Backward( const TargetType& target, OutputType& output) { + typedef typename InputType::elem_type ElemType; + const size_t cols = input.n_cols; const size_t batchSize = input.n_elem / cols; if (arma::size(input) != arma::size(target)) - { Log::Fatal << "Input Tensors must have same dimensions." << std::endl; - } arma::colvec inputTemp1 = arma::vectorise(input); arma::colvec inputTemp2 = arma::vectorise(target); @@ -87,16 +86,14 @@ void CosineEmbeddingLoss::Backward( false, false); for (size_t i = 0; i < inputTemp1.n_elem; i += cols) { - double cosDist = kernel::CosineDistance::Evaluate(inputTemp1(arma::span(i, - i + cols -1)), inputTemp2(arma::span(i, i + cols -1))); + const ElemType cosDist = kernel::CosineDistance::Evaluate(inputTemp1( + arma::span(i, i + cols -1)), inputTemp2(arma::span(i, i + cols -1))); if (cosDist < margin && !similarity) - { outputTemp(arma::span(i, i + cols - 1)).zeros(); - } else { - int multiplier = similarity ? 1 : -1; + const int multiplier = similarity ? 1 : -1; outputTemp(arma::span(i, i + cols -1)) = -1 * multiplier * (arma::normalise(inputTemp2(arma::span(i, i + cols - 1))) - cosDist * arma::normalise(inputTemp1(arma::span(i, i + cols - diff --git a/src/mlpack/tests/loss_functions_test.cpp b/src/mlpack/tests/loss_functions_test.cpp index a09cf77b2a..9182f0da88 100644 --- a/src/mlpack/tests/loss_functions_test.cpp +++ b/src/mlpack/tests/loss_functions_test.cpp @@ -621,7 +621,7 @@ BOOST_AUTO_TEST_CASE(CosineEmbeddingLossTest) input2(1) = 2; input2(2) = 2; loss = module.Forward(input1, input2); - // Caclulated using torch.nn.CosineEmbeddingLoss(). + // Calculated using torch.nn.CosineEmbeddingLoss(). BOOST_REQUIRE_CLOSE(loss, 2.897367, 1e-3); // Test the Backward function. @@ -645,7 +645,7 @@ BOOST_AUTO_TEST_CASE(CosineEmbeddingLossTest) input4(9) = 2; input4(11) = 2; loss = module2.Forward(input3, input4); - // Caclulated using torch.nn.CosineEmbeddingLoss(). + // Calculated using torch.nn.CosineEmbeddingLoss(). BOOST_REQUIRE_CLOSE(loss, 0.55395, 1e-3); // Test the Backward function. From 8ca0543a0261017a621c0b1f21f9cdf3cc298f56 Mon Sep 17 00:00:00 2001 From: AndreiMihalea Date: Fri, 3 Apr 2020 11:11:20 +0300 Subject: [PATCH 089/111] Retrigger tests From 51c05e6ef58ae7b74d216630114025d86a458f04 Mon Sep 17 00:00:00 2001 From: AndreiMihalea Date: Fri, 3 Apr 2020 11:11:30 +0300 Subject: [PATCH 090/111] Rebuild From 9deace019341918e9a18318e36087e562e8acaf8 Mon Sep 17 00:00:00 2001 From: Mikhail Lozhnikov Date: Fri, 3 Apr 2020 13:05:35 +0300 Subject: [PATCH 091/111] - Fixed tiny style issues. - The Bag of Words algorithm calculates the number of occurrences. --- .../bag_of_words_encoding_policy.hpp | 16 ++++++++-------- .../dictionary_encoding_policy.hpp | 6 +++--- .../tf_idf_encoding_policy.hpp | 6 +++--- src/mlpack/tests/string_encoding_test.cpp | 14 +++++++------- 4 files changed, 21 insertions(+), 21 deletions(-) diff --git a/src/mlpack/core/data/string_encoding_policies/bag_of_words_encoding_policy.hpp b/src/mlpack/core/data/string_encoding_policies/bag_of_words_encoding_policy.hpp index 3b3333c380..ff8097c1ac 100644 --- a/src/mlpack/core/data/string_encoding_policies/bag_of_words_encoding_policy.hpp +++ b/src/mlpack/core/data/string_encoding_policies/bag_of_words_encoding_policy.hpp @@ -24,10 +24,10 @@ namespace data { * Definition of the BagOfWordsEncodingPolicy class. * * BagOfWords is used as a helper class for StringEncoding. The encoder maps - * each dataset item to a vector of size N, where N is equal to the total number - * of tokens. If an item of the dataset has the i-th token, then the i-th - * coordinate of the corresponding vector is equal to 1, otherwise it's equal to - * zero. The order in which the tokens are labeled is defined by the dictionary + * each dataset item to a vector of size N, where N is equal to the total unique + * number of tokens. The i-th coordinate of the output vector is equal to + * the number of times when the i-th token occurs in the corresponding dataset + * item. The order in which the tokens are labeled is defined by the dictionary * used by the StringEncoding class. The encoder writes data either in the * column-major order or in the row-major order depending on the output data * type. @@ -44,7 +44,7 @@ class BagOfWordsEncodingPolicy * @param output Output matrix to store the encoded results (sp_mat or mat). * @param datasetSize The number of strings in the input dataset. * @param maxNumTokens The maximum number of tokens in the strings of the - input dataset (not used). + * input dataset (not used). * @param dictionarySize The size of the dictionary. */ template @@ -67,7 +67,7 @@ class BagOfWordsEncodingPolicy * @param output Output matrix to store the encoded results. * @param datasetSize The number of strings in the input dataset. * @param maxNumTokens The maximum number of tokens in the strings of the - input dataset (not used). + * input dataset (not used). * @param dictionarySize The size of the dictionary. */ template @@ -98,7 +98,7 @@ class BagOfWordsEncodingPolicy const size_t /* index */) { // The labels are assigned sequentially starting from one. - output(value - 1, line) = 1; + output(value - 1, line) += 1; } /** @@ -123,7 +123,7 @@ class BagOfWordsEncodingPolicy const size_t /* index */) { // The labels are assigned sequentially starting from one. - output[line][value - 1] = 1; + output[line][value - 1] += 1; } /** diff --git a/src/mlpack/core/data/string_encoding_policies/dictionary_encoding_policy.hpp b/src/mlpack/core/data/string_encoding_policies/dictionary_encoding_policy.hpp index d0b49775e0..24a5fa21b9 100644 --- a/src/mlpack/core/data/string_encoding_policies/dictionary_encoding_policy.hpp +++ b/src/mlpack/core/data/string_encoding_policies/dictionary_encoding_policy.hpp @@ -41,7 +41,7 @@ class DictionaryEncodingPolicy * @param output Output matrix to store the encoded results (sp_mat or mat). * @param datasetSize The number of strings in the input dataset. * @param maxNumTokens The maximum number of tokens in the strings of the - input dataset. + * input dataset. * @param dictionarySize The size of the dictionary (not used). */ template @@ -55,7 +55,7 @@ class DictionaryEncodingPolicy /** * The function performs the dictionary encoding algorithm i.e. it writes - * the encoded token to the ouput. The encoder writes data in the + * the encoded token to the output. The encoder writes data in the * column-major order. * * @tparam MatType The output matrix type. @@ -76,7 +76,7 @@ class DictionaryEncodingPolicy /** * The function performs the dictionary encoding algorithm i.e. it writes - * the encoded token to the ouput. This is an overload function which saves + * the encoded token to the output. This is an overload function which saves * the result into the given vector to avoid padding. The encoder writes data * in the row-major order. * diff --git a/src/mlpack/core/data/string_encoding_policies/tf_idf_encoding_policy.hpp b/src/mlpack/core/data/string_encoding_policies/tf_idf_encoding_policy.hpp index 141bb8ea1f..c9aae8f5e3 100644 --- a/src/mlpack/core/data/string_encoding_policies/tf_idf_encoding_policy.hpp +++ b/src/mlpack/core/data/string_encoding_policies/tf_idf_encoding_policy.hpp @@ -46,7 +46,7 @@ class TfIdfEncodingPolicy * TERM_FREQUENCY Term frequency equals the number of times when the encoded * token occurs in the row divided by the row size. * SUBLINEAR_TF Term frequency equals \f$ 1 + log(rawCount), \f$ where - rawCount is equal to the number of times when the encoded + * rawCount is equal to the number of times when the encoded * token occurs in the row. */ enum class TfTypes @@ -86,7 +86,7 @@ class TfIdfEncodingPolicy * @param output Output matrix to store the encoded results (sp_mat or mat). * @param datasetSize The number of strings in the input dataset. * @param maxNumTokens The maximum number of tokens in the strings of the - input dataset (not used). + * input dataset (not used). * @param dictionarySize The size of the dictionary. */ template @@ -109,7 +109,7 @@ class TfIdfEncodingPolicy * @param output Output matrix to store the encoded results. * @param datasetSize The number of strings in the input dataset. * @param maxNumTokens The maximum number of tokens in the strings of the - input dataset (not used). + * input dataset (not used). * @param dictionarySize The size of the dictionary. */ template diff --git a/src/mlpack/tests/string_encoding_test.cpp b/src/mlpack/tests/string_encoding_test.cpp index 23d1dc59dd..92befdfd77 100644 --- a/src/mlpack/tests/string_encoding_test.cpp +++ b/src/mlpack/tests/string_encoding_test.cpp @@ -596,7 +596,7 @@ BOOST_AUTO_TEST_CASE(BagOfWordsEncodingTest) vectorizer = CountVectorizer(strip_accents=False, lowercase=False, preprocessor=None, tokenizer=tokenizer, stop_words=None, - vocabulary=dictionary, binary=True) + vocabulary=dictionary, binary=False) X = vectorizer.fit_transform(string_encoding_input) @@ -607,7 +607,7 @@ BOOST_AUTO_TEST_CASE(BagOfWordsEncodingTest) arma::mat expected = { { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, - { 0, 1, 0, 0, 0, 1, 0, 0, 1, 1, 0, 0, 0, 1, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, + { 0, 1, 0, 0, 0, 2, 0, 0, 3, 3, 0, 0, 0, 3, 0, 0, 1, 1, 1, 3, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, { 1, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 } @@ -647,7 +647,7 @@ BOOST_AUTO_TEST_CASE(VectorBagOfWordsEncodingTest) vector> expected = { { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, - { 0, 1, 0, 0, 0, 1, 0, 0, 1, 1, 0, 0, 0, 1, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, + { 0, 1, 0, 0, 0, 2, 0, 0, 3, 3, 0, 0, 0, 3, 0, 0, 1, 1, 1, 3, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, { 1, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 } @@ -673,8 +673,8 @@ BOOST_AUTO_TEST_CASE(BagOfWordsEncodingIndividualCharactersTest) encoder.Encode(input, output, CharExtract()); arma::mat target = { - { 1, 1, 1, 0, 0 }, - { 0, 1, 1, 1, 1 }, + { 1, 2, 2, 0, 0 }, + { 0, 2, 2, 2, 1 }, { 1, 1, 0, 1, 0 } }; @@ -699,8 +699,8 @@ BOOST_AUTO_TEST_CASE(VectorBagOfWordsEncodingIndividualCharactersTest) encoder.Encode(input, output, CharExtract()); vector> expected = { - { 1, 1, 1, 0, 0 }, - { 0, 1, 1, 1, 1 }, + { 1, 2, 2, 0, 0 }, + { 0, 2, 2, 2, 1 }, { 1, 1, 0, 1, 0 } }; From 5ff26b029f88453c910f7784c868c866cb847a3c Mon Sep 17 00:00:00 2001 From: Bisakh Mondal Date: Fri, 3 Apr 2020 15:41:11 +0530 Subject: [PATCH 092/111] Windows build updated --- .ci/ci.yaml | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/.ci/ci.yaml b/.ci/ci.yaml index 739ba9906c..e0654b49e1 100644 --- a/.ci/ci.yaml +++ b/.ci/ci.yaml @@ -57,23 +57,6 @@ jobs: steps: - template: macos-steps.yaml -- job: WindowsVS14 - timeoutInMinutes: 360 - displayName: Windows VS14 - pool: - vmImage: vs2015-win2012r2 - strategy: - matrix: - Plain: - CMakeArgs: '-DDEBUG=ON -DPROFILE=OFF -DBUILD_PYTHON_BINDINGS=OFF' - CMakeGenerator: '-G "Visual Studio 14 2015 Win64"' - MSBuildVersion: '14.0' - ArchiveNoLibs: 'mlpack-windows-vs14-no-libs.zip' - ArchiveLibs: 'mlpack-windows-vs14.zip' - ArchiveTests: 'mlpack_test-vs14.xml' - steps: - - template: windows-steps.yaml - - job: WindowsVS15 timeoutInMinutes: 360 displayName: Windows VS15 From c57eeeb1798996021fed7e2f48269e5ff8271e78 Mon Sep 17 00:00:00 2001 From: AndreiMihalea Date: Fri, 3 Apr 2020 13:19:21 +0300 Subject: [PATCH 093/111] Retrigger tests From 69004bab0749069f55299205360f370cca3049cc Mon Sep 17 00:00:00 2001 From: Mikhail Lozhnikov Date: Fri, 3 Apr 2020 17:38:43 +0300 Subject: [PATCH 094/111] Force the string encoder to reset the necessary policy internal variables. --- src/mlpack/core/data/string_encoding_impl.hpp | 5 +++++ .../bag_of_words_encoding_policy.hpp | 8 ++++++++ .../dictionary_encoding_policy.hpp | 8 ++++++++ .../tf_idf_encoding_policy.hpp | 10 ++++++++++ src/mlpack/tests/string_encoding_test.cpp | 4 ++-- 5 files changed, 33 insertions(+), 2 deletions(-) diff --git a/src/mlpack/core/data/string_encoding_impl.hpp b/src/mlpack/core/data/string_encoding_impl.hpp index 2dac65563f..eea72b8fc6 100644 --- a/src/mlpack/core/data/string_encoding_impl.hpp +++ b/src/mlpack/core/data/string_encoding_impl.hpp @@ -107,6 +107,8 @@ EncodeHelper(const std::vector& input, { size_t numColumns = 0; + policy.Reset(); + // The first pass adds the extracted tokens to the dictionary. for (size_t i = 0; i < input.size(); i++) { @@ -132,6 +134,7 @@ EncodeHelper(const std::vector& input, token = tokenizer(strView); numTokens++; } + numColumns = std::max(numColumns, numTokens); } policy.InitMatrix(output, input.size(), numColumns, dictionary.Size()); @@ -162,6 +165,8 @@ EncodeHelper(const std::vector& input, typename std::enable_if::onePassEncoding>::type*) { + policy.Reset(); + // The loop below extracts the tokens and writes the encoded values // at once. for (size_t i = 0; i < input.size(); i++) diff --git a/src/mlpack/core/data/string_encoding_policies/bag_of_words_encoding_policy.hpp b/src/mlpack/core/data/string_encoding_policies/bag_of_words_encoding_policy.hpp index ff8097c1ac..d71a82ac6e 100644 --- a/src/mlpack/core/data/string_encoding_policies/bag_of_words_encoding_policy.hpp +++ b/src/mlpack/core/data/string_encoding_policies/bag_of_words_encoding_policy.hpp @@ -35,6 +35,14 @@ namespace data { class BagOfWordsEncodingPolicy { public: + /** + * Clear the necessary internal variables. + */ + static void Reset() + { + // Nothing to do. + } + /** * The function initializes the output matrix. The encoder writes data * in the column-major order. diff --git a/src/mlpack/core/data/string_encoding_policies/dictionary_encoding_policy.hpp b/src/mlpack/core/data/string_encoding_policies/dictionary_encoding_policy.hpp index 24a5fa21b9..100c8394a1 100644 --- a/src/mlpack/core/data/string_encoding_policies/dictionary_encoding_policy.hpp +++ b/src/mlpack/core/data/string_encoding_policies/dictionary_encoding_policy.hpp @@ -32,6 +32,14 @@ namespace data { class DictionaryEncodingPolicy { public: + /** + * Clear the necessary internal variables. + */ + static void Reset() + { + // Nothing to do. + } + /** * The function initializes the output matrix. The encoder writes data * in the column-major order. diff --git a/src/mlpack/core/data/string_encoding_policies/tf_idf_encoding_policy.hpp b/src/mlpack/core/data/string_encoding_policies/tf_idf_encoding_policy.hpp index c9aae8f5e3..e93dfd573f 100644 --- a/src/mlpack/core/data/string_encoding_policies/tf_idf_encoding_policy.hpp +++ b/src/mlpack/core/data/string_encoding_policies/tf_idf_encoding_policy.hpp @@ -77,6 +77,16 @@ class TfIdfEncodingPolicy smoothIdf(smoothIdf) { } + /** + * Clear the necessary internal variables. + */ + void Reset() + { + tokensFrequences.clear(); + numContainingStrings.clear(); + linesSizes.clear(); + } + /** * The function initializes the output matrix. The encoder writes data * in the row-major order. diff --git a/src/mlpack/tests/string_encoding_test.cpp b/src/mlpack/tests/string_encoding_test.cpp index 92befdfd77..31a0f22713 100644 --- a/src/mlpack/tests/string_encoding_test.cpp +++ b/src/mlpack/tests/string_encoding_test.cpp @@ -1267,11 +1267,11 @@ BOOST_AUTO_TEST_CASE(SublinearTfIdfEncodingIndividualCharactersTest) TfIdfEncoding encoder(TfIdfEncodingPolicy::TfTypes::SUBLINEAR_TF, false); + encoder.Encode(input, output, CharExtract()); + /* The expected values were obtained by almost the same script as in RawCountSmoothIdfEncodingIndividualCharactersTest. The only difference is tf_type equals 'sublinear_tf' and smooth_idf equals False. */ - encoder.Encode(input, output, CharExtract()); - arma::mat target = { { 1.40546510810816, 1.69314718055995, 2.37965928516872, 0, 0 }, { 0, 1.69314718055995, 2.37965928516872, 2.37965928516872, From 6d4dee6383de65fa7827acf2dab1a02a5b6fee0f Mon Sep 17 00:00:00 2001 From: jeffin143 Date: Fri, 3 Apr 2020 23:15:54 +0530 Subject: [PATCH 095/111] Reording include to pass CI build --- src/mlpack/methods/preprocess/image_converter_main.cpp | 2 +- src/mlpack/methods/preprocess/preprocess_scale_main.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/mlpack/methods/preprocess/image_converter_main.cpp b/src/mlpack/methods/preprocess/image_converter_main.cpp index 8cc73359fc..0d4beecf4b 100644 --- a/src/mlpack/methods/preprocess/image_converter_main.cpp +++ b/src/mlpack/methods/preprocess/image_converter_main.cpp @@ -10,8 +10,8 @@ * http://www.opensource.org/licenses/BSD-3-Clause for more information. */ #include -#include #include +#include #include using namespace mlpack; diff --git a/src/mlpack/methods/preprocess/preprocess_scale_main.cpp b/src/mlpack/methods/preprocess/preprocess_scale_main.cpp index bcbdc46f08..dadd2554a7 100644 --- a/src/mlpack/methods/preprocess/preprocess_scale_main.cpp +++ b/src/mlpack/methods/preprocess/preprocess_scale_main.cpp @@ -10,9 +10,9 @@ * http://www.opensource.org/licenses/BSD-3-Clause for more information. */ #include +#include #include #include -#include #include #include #include From f369ae57e92d89b7c8fd520f0284bd0f6d14476a Mon Sep 17 00:00:00 2001 From: jeffin143 Date: Fri, 3 Apr 2020 23:35:59 +0530 Subject: [PATCH 096/111] add ccov.hpp to fix julia binding failures --- src/mlpack/methods/preprocess/preprocess_scale_main.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/mlpack/methods/preprocess/preprocess_scale_main.cpp b/src/mlpack/methods/preprocess/preprocess_scale_main.cpp index dadd2554a7..cf68b8408c 100644 --- a/src/mlpack/methods/preprocess/preprocess_scale_main.cpp +++ b/src/mlpack/methods/preprocess/preprocess_scale_main.cpp @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include From 8c5fe9dc84c6e926c5dea7475446abdaf9e070df Mon Sep 17 00:00:00 2001 From: Mikhail Lozhnikov Date: Sat, 4 Apr 2020 15:28:14 +0300 Subject: [PATCH 097/111] Tiny documentation fixes for some string encoding algorithms. --- src/mlpack/core/data/string_encoding_impl.hpp | 1 + .../string_encoding_policies/dictionary_encoding_policy.hpp | 2 +- .../data/string_encoding_policies/tf_idf_encoding_policy.hpp | 3 ++- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/mlpack/core/data/string_encoding_impl.hpp b/src/mlpack/core/data/string_encoding_impl.hpp index eea72b8fc6..0e50f6c1d8 100644 --- a/src/mlpack/core/data/string_encoding_impl.hpp +++ b/src/mlpack/core/data/string_encoding_impl.hpp @@ -137,6 +137,7 @@ EncodeHelper(const std::vector& input, numColumns = std::max(numColumns, numTokens); } + policy.InitMatrix(output, input.size(), numColumns, dictionary.Size()); // The second pass writes the encoded values to the output. diff --git a/src/mlpack/core/data/string_encoding_policies/dictionary_encoding_policy.hpp b/src/mlpack/core/data/string_encoding_policies/dictionary_encoding_policy.hpp index 100c8394a1..d9a37cacac 100644 --- a/src/mlpack/core/data/string_encoding_policies/dictionary_encoding_policy.hpp +++ b/src/mlpack/core/data/string_encoding_policies/dictionary_encoding_policy.hpp @@ -84,7 +84,7 @@ class DictionaryEncodingPolicy /** * The function performs the dictionary encoding algorithm i.e. it writes - * the encoded token to the output. This is an overload function which saves + * the encoded token to the output. This is an overloaded function which saves * the result into the given vector to avoid padding. The encoder writes data * in the row-major order. * diff --git a/src/mlpack/core/data/string_encoding_policies/tf_idf_encoding_policy.hpp b/src/mlpack/core/data/string_encoding_policies/tf_idf_encoding_policy.hpp index e93dfd573f..853493e285 100644 --- a/src/mlpack/core/data/string_encoding_policies/tf_idf_encoding_policy.hpp +++ b/src/mlpack/core/data/string_encoding_policies/tf_idf_encoding_policy.hpp @@ -44,7 +44,8 @@ class TfIdfEncodingPolicy * RAW_COUNT Term frequency equals the number of times when the encoded * token occurs in the row. * TERM_FREQUENCY Term frequency equals the number of times when the encoded - * token occurs in the row divided by the row size. + * token occurs in the row divided by the total number of + * tokens in the row. * SUBLINEAR_TF Term frequency equals \f$ 1 + log(rawCount), \f$ where * rawCount is equal to the number of times when the encoded * token occurs in the row. From dcb926d2f6ea13ddb1cac826f51fdedc70ed3698 Mon Sep 17 00:00:00 2001 From: Conrad Sanderson Date: Mon, 6 Apr 2020 10:53:12 +0200 Subject: [PATCH 098/111] corrections to handle vector types in Armadillo 9.870 This patch corrects handling of Armadillo vector types by `IsVector`. Without this patch, mlpack will not compile with Armadillo 9.870 (and development version 9.869). Armadillo 9.870 has extended handling of sparse submatrix views. In addition to `arma::SpSubview`, it now has `arma::SpSubview_col` and `arma::SpSubview_row`. The latter two types are vector types, so `IsVector` needs to be adjusted accordingly. --- src/mlpack/core/util/arma_traits.hpp | 35 +++++++++++++++++++++------- 1 file changed, 27 insertions(+), 8 deletions(-) diff --git a/src/mlpack/core/util/arma_traits.hpp b/src/mlpack/core/util/arma_traits.hpp index 45e5dac125..efa59c4fd2 100644 --- a/src/mlpack/core/util/arma_traits.hpp +++ b/src/mlpack/core/util/arma_traits.hpp @@ -81,14 +81,33 @@ struct IsVector > const static bool value = true; }; -// I'm not so sure about this one. An SpSubview object can be a row or column, -// but it can also be a matrix subview. -// template<> -template -struct IsVector > -{ - const static bool value = true; -}; +#if ( (ARMA_VERSION_MAJOR >= 10) || ((ARMA_VERSION_MAJOR == 9) && (ARMA_VERSION_MINOR >= 869)) ) + + // Armadillo 9.869+ has SpSubview_col and SpSubview_row + + template + struct IsVector > + { + const static bool value = true; + }; + + template + struct IsVector > + { + const static bool value = true; + }; + +#else + + // fallback for older Armadillo versions + + template + struct IsVector > + { + const static bool value = true; + }; + +#endif #endif From 674737eb190a01adae7e000364a3b0d0224a1adf Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Sat, 4 Apr 2020 23:14:47 -0400 Subject: [PATCH 099/111] Workaround Debian buster STB include bug. --- src/mlpack/core/data/load_image.cpp | 7 +++---- src/mlpack/core/data/save_image.cpp | 16 +++++++++++----- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/src/mlpack/core/data/load_image.cpp b/src/mlpack/core/data/load_image.cpp index 2796e9a1f4..649ebde9cb 100644 --- a/src/mlpack/core/data/load_image.cpp +++ b/src/mlpack/core/data/load_image.cpp @@ -9,13 +9,12 @@ #ifdef HAS_STB +// The definition of STB_IMAGE_IMPLEMENTATION means that the implementation will +// be included here directly. #define STB_IMAGE_STATIC #define STB_IMAGE_IMPLEMENTATION -#include -#define STB_IMAGE_WRITE_STATIC -#define STB_IMAGE_WRITE_IMPLEMENTATION -#include +#include namespace mlpack { namespace data { diff --git a/src/mlpack/core/data/save_image.cpp b/src/mlpack/core/data/save_image.cpp index 2c4654b711..b03063c944 100644 --- a/src/mlpack/core/data/save_image.cpp +++ b/src/mlpack/core/data/save_image.cpp @@ -8,13 +8,19 @@ #ifdef HAS_STB -#define STB_IMAGE_STATIC -#define STB_IMAGE_IMPLEMENTATION -#include - +// The implementation of the functions is included directly, so we need to make +// sure it doesn't get included twice. This is to work around a bug in old +// versions of STB where not all functions were correctly marked static. #define STB_IMAGE_WRITE_STATIC -#define STB_IMAGE_WRITE_IMPLEMENTATION +#ifndef STB_IMAGE_WRITE_IMPLEMENTATION + #define STB_IMAGE_WRITE_IMPLEMENTATION +#else + #undef STB_IMAGE_WRITE_IMPLEMENTATION +#endif #include +#ifndef STB_IMAGE_WRITE_IMPLEMENTATION + #define STB_IMAGE_WRITE_IMPLEMENTATION +#endif namespace mlpack { namespace data { From eebece7c5407a8100ad060b0683cf539957b4126 Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Mon, 6 Apr 2020 16:16:27 -0400 Subject: [PATCH 100/111] Set size of parameters in all Train() calls. --- .../logistic_regression/logistic_regression_impl.hpp | 10 +++++----- .../softmax_regression/softmax_regression_impl.hpp | 4 ++-- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/mlpack/methods/logistic_regression/logistic_regression_impl.hpp b/src/mlpack/methods/logistic_regression/logistic_regression_impl.hpp index 760af72acd..1f5b81eb33 100644 --- a/src/mlpack/methods/logistic_regression/logistic_regression_impl.hpp +++ b/src/mlpack/methods/logistic_regression/logistic_regression_impl.hpp @@ -25,7 +25,6 @@ LogisticRegression::LogisticRegression( const MatType& predictors, const arma::Row& responses, const double lambda) : - parameters(arma::rowvec(predictors.n_rows + 1, arma::fill::zeros)), lambda(lambda) { Train(predictors, responses); @@ -60,7 +59,6 @@ LogisticRegression::LogisticRegression( const arma::Row& responses, OptimizerType& optimizer, const double lambda) : - parameters(arma::rowvec(predictors.n_rows + 1, arma::fill::zeros)), lambda(lambda) { Train(predictors, responses, optimizer); @@ -85,9 +83,11 @@ double LogisticRegression::Train( OptimizerType& optimizer, CallbackTypes&&... callbacks) { - LogisticRegressionFunction errorFunction(predictors, - responses, - lambda); + LogisticRegressionFunction errorFunction(predictors, responses, + lambda); + + // Set size of parameters vector according to the input data received. + parameters = arma::rowvec(predictors.n_rows + 1, arma::fill::zeros); errorFunction.InitialPoint() = parameters; Timer::Start("logistic_regression_optimization"); diff --git a/src/mlpack/methods/softmax_regression/softmax_regression_impl.hpp b/src/mlpack/methods/softmax_regression/softmax_regression_impl.hpp index 30fc341cf1..a38831d657 100644 --- a/src/mlpack/methods/softmax_regression/softmax_regression_impl.hpp +++ b/src/mlpack/methods/softmax_regression/softmax_regression_impl.hpp @@ -65,7 +65,7 @@ double SoftmaxRegression::Train(const arma::mat& data, { SoftmaxRegressionFunction regressor(data, labels, numClasses, lambda, fitIntercept); - if (parameters.is_empty()) + if (parameters.n_elem != regressor.GetInitialPoint().n_elem) parameters = regressor.GetInitialPoint(); // Train the model. @@ -88,7 +88,7 @@ double SoftmaxRegression::Train(const arma::mat& data, { SoftmaxRegressionFunction regressor(data, labels, numClasses, lambda, fitIntercept); - if (parameters.is_empty()) + if (parameters.n_elem != regressor.GetInitialPoint().n_elem) parameters = regressor.GetInitialPoint(); // Train the model. From 7dfdae66a5cf9442a1b73078b29d1b7e183ff32d Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Mon, 6 Apr 2020 16:17:15 -0400 Subject: [PATCH 101/111] Update history. --- HISTORY.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/HISTORY.md b/HISTORY.md index 6d086bf76f..fc4a190b55 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -56,7 +56,7 @@ * Add LiSHT activation function (#2182). * Add Valid and Same Padding for Transposed Convolution layer (#2163). - + * Add CELU activation function (#2191) * Add Log-Hyperbolic-Cosine Loss function (#2207) @@ -79,6 +79,9 @@ * Add Margin Ranking Loss Function (#2264). + * Bugfix for incorrect parameter vector sizes in logistic regression and + softmax regression (#2359). + ### mlpack 3.2.2 ###### 2019-11-26 * Add `valid` and `same` padding option in `Convolution` and `Atrous From fad498cae06cebbe5a21e2ea768a2fd0d38ce722 Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Mon, 6 Apr 2020 16:34:20 -0400 Subject: [PATCH 102/111] Add a test case. --- src/mlpack/tests/logistic_regression_test.cpp | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/src/mlpack/tests/logistic_regression_test.cpp b/src/mlpack/tests/logistic_regression_test.cpp index d603004a28..f3bbe6b32d 100644 --- a/src/mlpack/tests/logistic_regression_test.cpp +++ b/src/mlpack/tests/logistic_regression_test.cpp @@ -1002,4 +1002,25 @@ BOOST_AUTO_TEST_CASE(LogisticRegressionTrainReturnObjective) BOOST_REQUIRE_EQUAL(std::isfinite(objVal), true); } +/** + * Test that construction *then* training works fine. Thanks @Trento89 for the + * test case (see #2358). + */ +BOOST_AUTO_TEST_CASE(ConstructionThenTraining) +{ + arma::mat myMatrix; + + // Four points, three dimensions. + myMatrix << 0.555950 << 0.274690 << 0.540605 << 0.798938 << arma::endr + << 0.948014 << 0.973234 << 0.216504 << 0.883152 << arma::endr + << 0.023787 << 0.675382 << 0.231751 << 0.450332 << arma::endr; + + arma::Row myTargets("1 0 1 0"); + + regression::LogisticRegression<> lr; + + // Make sure that training doesn't crash with invalid parameter sizes. + BOOST_REQUIRE_NO_THROW(lr.Train(myMatrix, myTargets)); +} + BOOST_AUTO_TEST_SUITE_END(); From 77a0513d1df916c9e0f9bbcd91a998a039fcdaa5 Mon Sep 17 00:00:00 2001 From: Conrad Sanderson Date: Tue, 7 Apr 2020 02:56:12 +0200 Subject: [PATCH 103/111] layout changes Co-Authored-By: Ryan Curtin --- src/mlpack/core/util/arma_traits.hpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/mlpack/core/util/arma_traits.hpp b/src/mlpack/core/util/arma_traits.hpp index efa59c4fd2..155a6a0d82 100644 --- a/src/mlpack/core/util/arma_traits.hpp +++ b/src/mlpack/core/util/arma_traits.hpp @@ -82,7 +82,8 @@ struct IsVector > }; -#if ( (ARMA_VERSION_MAJOR >= 10) || ((ARMA_VERSION_MAJOR == 9) && (ARMA_VERSION_MINOR >= 869)) ) +#if ((ARMA_VERSION_MAJOR >= 10) || \ + ((ARMA_VERSION_MAJOR == 9) && (ARMA_VERSION_MINOR >= 869))) // Armadillo 9.869+ has SpSubview_col and SpSubview_row From ec96ed17a12831e9dc5c87b0dafb048f06404254 Mon Sep 17 00:00:00 2001 From: Mrityunjay Tripathi <35535378+mrityunjay-tripathi@users.noreply.github.com> Date: Tue, 7 Apr 2020 10:41:21 +0530 Subject: [PATCH 104/111] inclusion of all layers in layer.hpp (#2353) * added remaining layers * slight change * including linear_no_bias.hpp in layer_types.hpp --- src/mlpack/methods/ann/layer/layer.hpp | 35 +++++++++++++++++--- src/mlpack/methods/ann/layer/layer_types.hpp | 1 + 2 files changed, 31 insertions(+), 5 deletions(-) diff --git a/src/mlpack/methods/ann/layer/layer.hpp b/src/mlpack/methods/ann/layer/layer.hpp index 6006073a99..44cc8b667f 100644 --- a/src/mlpack/methods/ann/layer/layer.hpp +++ b/src/mlpack/methods/ann/layer/layer.hpp @@ -12,32 +12,57 @@ #ifndef MLPACK_METHODS_ANN_LAYER_LAYER_HPP #define MLPACK_METHODS_ANN_LAYER_LAYER_HPP +#include "add.hpp" #include "add_merge.hpp" +#include "alpha_dropout.hpp" #include "atrous_convolution.hpp" +#include "base_layer.hpp" #include "batch_norm.hpp" +#include "bilinear_interpolation.hpp" +#include "c_relu.hpp" +#include "celu.hpp" #include "concat_performance.hpp" +#include "concat.hpp" +#include "concatenate.hpp" +#include "constant.hpp" #include "convolution.hpp" #include "dropconnect.hpp" +#include "dropout.hpp" +#include "elu.hpp" +#include "fast_lstm.hpp" +#include "flexible_relu.hpp" #include "glimpse.hpp" +#include "gru.hpp" +#include "hard_tanh.hpp" +#include "hardshrink.hpp" #include "highway.hpp" +#include "join.hpp" #include "layer_norm.hpp" #include "layer_types.hpp" +#include "leaky_relu.hpp" #include "linear.hpp" #include "linear_no_bias.hpp" +#include "log_softmax.hpp" +#include "lookup.hpp" #include "lstm.hpp" +#include "max_pooling.hpp" +#include "mean_pooling.hpp" #include "minibatch_discrimination.hpp" +#include "multiply_constant.hpp" #include "multiply_merge.hpp" #include "padding.hpp" -#include "gru.hpp" -#include "fast_lstm.hpp" -#include "recurrent.hpp" +#include "parametric_relu.hpp" #include "recurrent_attention.hpp" +#include "recurrent.hpp" +#include "reinforce_normal.hpp" #include "reparametrization.hpp" +#include "select.hpp" #include "sequential.hpp" +#include "softshrink.hpp" #include "subview.hpp" -#include "concat.hpp" -#include "vr_class_reward.hpp" #include "transposed_convolution.hpp" +#include "virtual_batch_norm.hpp" +#include "vr_class_reward.hpp" #include "weight_norm.hpp" #endif diff --git a/src/mlpack/methods/ann/layer/layer_types.hpp b/src/mlpack/methods/ann/layer/layer_types.hpp index 3c17021e45..5a07c59016 100644 --- a/src/mlpack/methods/ann/layer/layer_types.hpp +++ b/src/mlpack/methods/ann/layer/layer_types.hpp @@ -30,6 +30,7 @@ #include #include #include +#include #include #include #include From 1e2d3f22868fbd8d9393a897ec6de14bec61a210 Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Tue, 7 Apr 2020 09:14:22 -0400 Subject: [PATCH 105/111] Update history for release. --- HISTORY.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/HISTORY.md b/HISTORY.md index fc4a190b55..9279dc9fe7 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,7 +1,7 @@ -### mlpack ?.?.? -###### ????-??-?? +### mlpack 3.3.0 +###### 2020-04-07 * Templated return type of `Forward function` of loss functions (#2339). - + * Added `R2 Score` regression metric (#2323). * Added `mean squared logarithmic error` loss function for neural networks @@ -66,7 +66,7 @@ * Bump minimum Boost version to 1.58 (#2305). - * Refactor STB support so HAS_STB macro is not needed when compiling against + * Refactor STB support so `HAS_STB` macro is not needed when compiling against mlpack (#2312). * Add Hard Shrink Activation Function (#2186). From 02dbca40d2bea1f8b464bc87df6116cc83a007b7 Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Tue, 7 Apr 2020 09:17:04 -0400 Subject: [PATCH 106/111] Add missing licenses. --- src/mlpack/bindings/julia/get_julia_type.hpp | 5 +++++ src/mlpack/bindings/julia/get_printable_type.hpp | 5 +++++ src/mlpack/bindings/julia/get_printable_type_impl.hpp | 5 +++++ src/mlpack/bindings/julia/julia_util.cpp | 5 +++++ src/mlpack/bindings/julia/print_doc.hpp | 5 +++++ src/mlpack/bindings/julia/print_input_param.hpp | 5 +++++ src/mlpack/bindings/julia/print_input_processing.hpp | 5 +++++ src/mlpack/bindings/julia/print_input_processing_impl.hpp | 5 +++++ src/mlpack/bindings/julia/print_jl.cpp | 5 +++++ src/mlpack/bindings/julia/print_jl.hpp | 5 +++++ src/mlpack/bindings/julia/print_output_processing.hpp | 5 +++++ src/mlpack/bindings/julia/print_output_processing_impl.hpp | 5 +++++ src/mlpack/bindings/julia/print_param_defn.hpp | 5 +++++ src/mlpack/bindings/julia/strip_type.hpp | 5 +++++ src/mlpack/core/data/load_image.cpp | 5 +++++ src/mlpack/core/data/save_image.cpp | 5 +++++ 16 files changed, 80 insertions(+) diff --git a/src/mlpack/bindings/julia/get_julia_type.hpp b/src/mlpack/bindings/julia/get_julia_type.hpp index 096c420ab9..d320494ba2 100644 --- a/src/mlpack/bindings/julia/get_julia_type.hpp +++ b/src/mlpack/bindings/julia/get_julia_type.hpp @@ -3,6 +3,11 @@ * @author Ryan Curtin * * Get the Julia-named type of an mlpack C++ type. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. */ #ifndef MLPACK_BINDINGS_JULIA_GET_JULIA_TYPE_HPP #define MLPACK_BINDINGS_JULIA_GET_JULIA_TYPE_HPP diff --git a/src/mlpack/bindings/julia/get_printable_type.hpp b/src/mlpack/bindings/julia/get_printable_type.hpp index 53747acdf1..c353497349 100644 --- a/src/mlpack/bindings/julia/get_printable_type.hpp +++ b/src/mlpack/bindings/julia/get_printable_type.hpp @@ -4,6 +4,11 @@ * * Get the printable type of a parameter. This type is not the C++ type but * instead the Julia type that a user would use. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. */ #ifndef MLPACK_BINDINGS_JULIA_GET_PRINTABLE_TYPE_HPP #define MLPACK_BINDINGS_JULIA_GET_PRINTABLE_TYPE_HPP diff --git a/src/mlpack/bindings/julia/get_printable_type_impl.hpp b/src/mlpack/bindings/julia/get_printable_type_impl.hpp index 524fe62a0b..2df1a753ec 100644 --- a/src/mlpack/bindings/julia/get_printable_type_impl.hpp +++ b/src/mlpack/bindings/julia/get_printable_type_impl.hpp @@ -4,6 +4,11 @@ * * Get the printable type of a parameter. This type is not the C++ type but * instead the Julia type that a user would use. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. */ #ifndef MLPACK_BINDINGS_JULIA_GET_PRINTABLE_TYPE_IMPL_HPP #define MLPACK_BINDINGS_JULIA_GET_PRINTABLE_TYPE_IMPL_HPP diff --git a/src/mlpack/bindings/julia/julia_util.cpp b/src/mlpack/bindings/julia/julia_util.cpp index faa639956f..ee3bf70681 100644 --- a/src/mlpack/bindings/julia/julia_util.cpp +++ b/src/mlpack/bindings/julia/julia_util.cpp @@ -3,6 +3,11 @@ * @author Ryan Curtin * * Implementations of Julia binding functionality. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. */ #include #include diff --git a/src/mlpack/bindings/julia/print_doc.hpp b/src/mlpack/bindings/julia/print_doc.hpp index 522ea3576b..62a8acdde8 100644 --- a/src/mlpack/bindings/julia/print_doc.hpp +++ b/src/mlpack/bindings/julia/print_doc.hpp @@ -3,6 +3,11 @@ * @author Ryan Curtin * * Print inline documentation for a single option. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. */ #ifndef MLPACK_BINDINGS_JULIA_PRINT_DOC_HPP #define MLPACK_BINDINGS_JULIA_PRINT_DOC_HPP diff --git a/src/mlpack/bindings/julia/print_input_param.hpp b/src/mlpack/bindings/julia/print_input_param.hpp index a76ba172a3..9f7ad35285 100644 --- a/src/mlpack/bindings/julia/print_input_param.hpp +++ b/src/mlpack/bindings/julia/print_input_param.hpp @@ -4,6 +4,11 @@ * * Print the declaration of an input parameter as part of a line in a Julia * function definition. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. */ #ifndef MLPACK_BINDINGS_JULIA_PRINT_INPUT_PARAM_HPP #define MLPACK_BINDINGS_JULIA_PRINT_INPUT_PARAM_HPP diff --git a/src/mlpack/bindings/julia/print_input_processing.hpp b/src/mlpack/bindings/julia/print_input_processing.hpp index a5ae2412bf..317c796d65 100644 --- a/src/mlpack/bindings/julia/print_input_processing.hpp +++ b/src/mlpack/bindings/julia/print_input_processing.hpp @@ -3,6 +3,11 @@ * @author Ryan Curtin * * Print Julia code to handle input arguments. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. */ #ifndef MLPACK_BINDINGS_JULIA_PRINT_INPUT_PROCESSING_HPP #define MLPACK_BINDINGS_JULIA_PRINT_INPUT_PROCESSING_HPP diff --git a/src/mlpack/bindings/julia/print_input_processing_impl.hpp b/src/mlpack/bindings/julia/print_input_processing_impl.hpp index 6839abf29d..cd71693f3e 100644 --- a/src/mlpack/bindings/julia/print_input_processing_impl.hpp +++ b/src/mlpack/bindings/julia/print_input_processing_impl.hpp @@ -3,6 +3,11 @@ * @author Ryan Curtin * * Print Julia code to handle input arguments. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. */ #ifndef MLPACK_BINDINGS_JULIA_PRINT_INPUT_PROCESSING_IMPL_HPP #define MLPACK_BINDINGS_JULIA_PRINT_INPUT_PROCESSING_IMPL_HPP diff --git a/src/mlpack/bindings/julia/print_jl.cpp b/src/mlpack/bindings/julia/print_jl.cpp index 7f4fb86a02..23d0b5ee0e 100644 --- a/src/mlpack/bindings/julia/print_jl.cpp +++ b/src/mlpack/bindings/julia/print_jl.cpp @@ -3,6 +3,11 @@ * @author Ryan Curtin * * Implementation of utility PrintJL() function. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. */ #include "print_jl.hpp" #include diff --git a/src/mlpack/bindings/julia/print_jl.hpp b/src/mlpack/bindings/julia/print_jl.hpp index 6b07dd7e3b..e712fc0a3c 100644 --- a/src/mlpack/bindings/julia/print_jl.hpp +++ b/src/mlpack/bindings/julia/print_jl.hpp @@ -3,6 +3,11 @@ * @author Ryan Curtin * * Definition of utility PrintJL() function. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. */ #ifndef MLPACK_BINDINGS_JULIA_PRINT_JL_HPP #define MLPACK_BINDINGS_JULIA_PRINT_JL_HPP diff --git a/src/mlpack/bindings/julia/print_output_processing.hpp b/src/mlpack/bindings/julia/print_output_processing.hpp index 069b344b6e..6426eb293d 100644 --- a/src/mlpack/bindings/julia/print_output_processing.hpp +++ b/src/mlpack/bindings/julia/print_output_processing.hpp @@ -3,6 +3,11 @@ * @author Ryan Curtin * * Print Julia code to handle output arguments. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. */ #ifndef MLPACK_BINDINGS_JULIA_PRINT_OUTPUT_PROCESSING_HPP #define MLPACK_BINDINGS_JULIA_PRINT_OUTPUT_PROCESSING_HPP diff --git a/src/mlpack/bindings/julia/print_output_processing_impl.hpp b/src/mlpack/bindings/julia/print_output_processing_impl.hpp index 407eb5b657..058bae5b87 100644 --- a/src/mlpack/bindings/julia/print_output_processing_impl.hpp +++ b/src/mlpack/bindings/julia/print_output_processing_impl.hpp @@ -3,6 +3,11 @@ * @author Ryan Curtin * * Print Julia code to handle output arguments. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. */ #ifndef MLPACK_BINDINGS_JULIA_PRINT_OUTPUT_PROCESSING_IMPL_HPP #define MLPACK_BINDINGS_JULIA_PRINT_OUTPUT_PROCESSING_IMPL_HPP diff --git a/src/mlpack/bindings/julia/print_param_defn.hpp b/src/mlpack/bindings/julia/print_param_defn.hpp index 46dc604a2c..84e482431a 100644 --- a/src/mlpack/bindings/julia/print_param_defn.hpp +++ b/src/mlpack/bindings/julia/print_param_defn.hpp @@ -4,6 +4,11 @@ * * If the type is serializable, we need to define a special utility function to * set a CLI parameter of that type. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. */ #ifndef MLPACK_BINDINGS_JULIA_PRINT_PARAM_DEFN_HPP #define MLPACK_BINDINGS_JULIA_PRINT_PARAM_DEFN_HPP diff --git a/src/mlpack/bindings/julia/strip_type.hpp b/src/mlpack/bindings/julia/strip_type.hpp index 3a2a179ac1..087e74e940 100644 --- a/src/mlpack/bindings/julia/strip_type.hpp +++ b/src/mlpack/bindings/julia/strip_type.hpp @@ -4,6 +4,11 @@ * * Given a C++ type name, turn it into something that has no special characters * that can simply be printed. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. */ #ifndef MLPACK_BINDINGS_JULIA_STRIP_TYPE_HPP #define MLPACK_BINDINGS_JULIA_STRIP_TYPE_HPP diff --git a/src/mlpack/core/data/load_image.cpp b/src/mlpack/core/data/load_image.cpp index 649ebde9cb..7801a6d07a 100644 --- a/src/mlpack/core/data/load_image.cpp +++ b/src/mlpack/core/data/load_image.cpp @@ -3,6 +3,11 @@ * @author Mehul Kumar Nirala * * Implementation of image loading functionality via STB. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. */ #include "load.hpp" #include "image_info.hpp" diff --git a/src/mlpack/core/data/save_image.cpp b/src/mlpack/core/data/save_image.cpp index b03063c944..abc8b701e9 100644 --- a/src/mlpack/core/data/save_image.cpp +++ b/src/mlpack/core/data/save_image.cpp @@ -3,6 +3,11 @@ * @author Mehul Kumar Nirala * * Implementation of image saving functionality via STB. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. */ #include "save.hpp" From 5677c2b5b91fdc62ca64195076da502da29a2398 Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Tue, 7 Apr 2020 09:17:16 -0400 Subject: [PATCH 107/111] Update version to 3.3.0. --- CMakeLists.txt | 8 ++++---- README.md | 2 +- .../sample-ml-app/sample-ml-app.vcxproj | 8 ++++---- doc/guide/build.hpp | 12 ++++++------ doc/guide/python_quickstart.hpp | 6 +++--- doc/guide/sample_ml_app.hpp | 8 ++++---- src/mlpack/CMakeLists.txt | 2 +- src/mlpack/core/util/version.hpp | 4 ++-- 8 files changed, 25 insertions(+), 25 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 4dc6d0beb8..4bb2eef1ba 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -359,14 +359,14 @@ endif () find_package(Ensmallen 2.10.0) if (NOT ENSMALLEN_FOUND) if (DOWNLOAD_ENSMALLEN) - file(DOWNLOAD http://www.ensmallen.org/files/ensmallen-latest.tar.gz - "${CMAKE_BINARY_DIR}/deps/ensmallen-latest.tar.gz" + file(DOWNLOAD http://www.ensmallen.org/files/ensmallen-2.12.0.tar.gz + "${CMAKE_BINARY_DIR}/deps/ensmallen-2.12.0.tar.gz" STATUS ENS_DOWNLOAD_STATUS_LIST LOG ENS_DOWNLOAD_LOG SHOW_PROGRESS) list(GET ENS_DOWNLOAD_STATUS_LIST 0 ENS_DOWNLOAD_STATUS) if (ENS_DOWNLOAD_STATUS EQUAL 0) execute_process(COMMAND ${CMAKE_COMMAND} -E - tar xzf "${CMAKE_BINARY_DIR}/deps/ensmallen-latest.tar.gz" + tar xzf "${CMAKE_BINARY_DIR}/deps/ensmallen-2.12.0.tar.gz" WORKING_DIRECTORY "${CMAKE_BINARY_DIR}/deps/") # Get the name of the directory. @@ -374,7 +374,7 @@ if (NOT ENSMALLEN_FOUND) "${CMAKE_BINARY_DIR}/deps/ensmallen-[0-9]*.[0-9]*.[0-9]*") # list(FILTER) is not available on 3.5 or older, but try to keep # configuring without filtering the list anyway (it might work if only - # the file ensmallen-latest.tar.gz is present. + # the file ensmallen-2.12.0.tar.gz is present. if (${CMAKE_VERSION} VERSION_GREATER_EQUAL "3.6.0") list(FILTER ENS_DIRECTORIES EXCLUDE REGEX "ensmallen-.*\.tar\.gz") endif () diff --git a/README.md b/README.md index d204bb4b3a..0220d4bc2b 100644 --- a/README.md +++ b/README.md @@ -23,7 +23,7 @@ src="https://cdn.rawgit.com/mlpack/mlpack.org/e7d36ed8/mlpack-black.svg" style="

Download: - current stable version (3.2.2) + current stable version (3.2.2)

diff --git a/doc/examples/sample-ml-app/sample-ml-app/sample-ml-app.vcxproj b/doc/examples/sample-ml-app/sample-ml-app/sample-ml-app.vcxproj index fdc41c5d2a..aadce63f17 100644 --- a/doc/examples/sample-ml-app/sample-ml-app/sample-ml-app.vcxproj +++ b/doc/examples/sample-ml-app/sample-ml-app/sample-ml-app.vcxproj @@ -104,16 +104,16 @@ true _DEBUG;_CONSOLE;%(PreprocessorDefinitions) false - C:\boost\boost_1_66_0;C:\mlpack\armadillo-8.500.1\include;C:\mlpack\mlpack-3.2.1\build\include;%(AdditionalIncludeDirectories) + C:\boost\boost_1_66_0;C:\mlpack\armadillo-8.500.1\include;C:\mlpack\mlpack-3.3.0\build\include;%(AdditionalIncludeDirectories) Console true - C:\mlpack\mlpack-3.2.1\build\Debug\mlpack.lib;C:\boost\boost_1_66_0\lib64-msvc-14.1\libboost_serialization-vc141-mt-gd-x64-1_66.lib;C:\boost\boost_1_66_0\lib64-msvc-14.1\libboost_program_options-vc141-mt-gd-x64-1_66.lib;%(AdditionalDependencies) + C:\mlpack\mlpack-3.3.0\build\Debug\mlpack.lib;C:\boost\boost_1_66_0\lib64-msvc-14.1\libboost_serialization-vc141-mt-gd-x64-1_66.lib;C:\boost\boost_1_66_0\lib64-msvc-14.1\libboost_program_options-vc141-mt-gd-x64-1_66.lib;%(AdditionalDependencies) - xcopy /y "C:\mlpack\mlpack-3.2.1\build\Debug\mlpack.dll" $(OutDir) -xcopy /y "C:\mlpack\mlpack-3.2.1\packages\OpenBLAS.0.2.14.1\lib\native\bin\x64\*.dll" $(OutDir) + xcopy /y "C:\mlpack\mlpack-3.3.0\build\Debug\mlpack.dll" $(OutDir) +xcopy /y "C:\mlpack\mlpack-3.3.0\packages\OpenBLAS.0.2.14.1\lib\native\bin\x64\*.dll" $(OutDir) xcopy /y "$(ProjectDir)..\..\..\..\src\mlpack\tests\data\german.csv" "$(ProjectDir)data\german.csv*" diff --git a/doc/guide/build.hpp b/doc/guide/build.hpp index df5e2edb82..d54dd3274e 100644 --- a/doc/guide/build.hpp +++ b/doc/guide/build.hpp @@ -30,7 +30,7 @@ to build mlpack on Windows, see \ref build_windows (alternatively, you can read is based on older versions). You can download the latest mlpack release from here: -mlpack-3.2.2 +mlpack-3.3.0 @section build_simple Simple Linux build instructions @@ -38,9 +38,9 @@ Assuming all dependencies are installed in the system, you can run the commands below directly to build and install mlpack. @code -$ wget https://www.mlpack.org/files/mlpack-3.2.2.tar.gz -$ tar -xvzpf mlpack-3.2.2.tar.gz -$ mkdir mlpack-3.2.2/build && cd mlpack-3.2.2/build +$ wget https://www.mlpack.org/files/mlpack-3.3.0.tar.gz +$ tar -xvzpf mlpack-3.3.0.tar.gz +$ mkdir mlpack-3.3.0/build && cd mlpack-3.3.0/build $ cmake ../ $ make -j4 # The -j is the number of cores you want to use for a build. $ sudo make install @@ -65,8 +65,8 @@ configure mlpack. First we should unpack the mlpack source and create a build directory. @code -$ tar -xvzpf mlpack-3.2.2.tar.gz -$ cd mlpack-3.2.2 +$ tar -xvzpf mlpack-3.3.0.tar.gz +$ cd mlpack-3.3.0 $ mkdir build @endcode diff --git a/doc/guide/python_quickstart.hpp b/doc/guide/python_quickstart.hpp index 115eecc107..4a1da9152c 100644 --- a/doc/guide/python_quickstart.hpp +++ b/doc/guide/python_quickstart.hpp @@ -31,9 +31,9 @@ build and install mlpack. You can copy-paste the commands into your shell. @code{.sh} sudo apt-get install libboost-all-dev g++ cmake libarmadillo-dev python-pip wget sudo pip install cython setuptools distutils numpy pandas -wget https://www.mlpack.org/files/mlpack-3.2.1.tar.gz -tar -xvzpf mlpack-3.2.1.tar.gz -mkdir -p mlpack-3.2.1/build/ && cd mlpack-3.2.1/build/ +wget https://www.mlpack.org/files/mlpack-3.3.0.tar.gz +tar -xvzpf mlpack-3.3.0.tar.gz +mkdir -p mlpack-3.3.0/build/ && cd mlpack-3.3.0/build/ cmake ../ && make -j4 && sudo make install @endcode diff --git a/doc/guide/sample_ml_app.hpp b/doc/guide/sample_ml_app.hpp index 6e42a19c07..1cb4877037 100644 --- a/doc/guide/sample_ml_app.hpp +++ b/doc/guide/sample_ml_app.hpp @@ -29,18 +29,18 @@ mlpack and dependencies in Release Mode). @code - C:\boost\boost_1_71_0\lib\native\include - C:\mlpack\armadillo-9.800.3\include - - C:\mlpack\mlpack-3.2.2\build\include + - C:\mlpack\mlpack-3.3.0\build\include @endcode - Under Linker > Input > Additional Dependencies add: @code - - C:\mlpack\mlpack-3.2.2\build\Debug\mlpack.lib + - C:\mlpack\mlpack-3.3.0\build\Debug\mlpack.lib - C:\boost\boost_1_71_0\lib64-msvc-14.2\libboost_serialization-vc142-mt-gd-x64-1_71.lib - C:\boost\boost_1_71_0\lib64-msvc-14.2\libboost_program_options-vc142-mt-gd-x64-1_71.lib @endcode - Under Build Events > Post-Build Event > Command Line add: @code - - xcopy /y "C:\mlpack\mlpack-3.2.2\build\Debug\mlpack.dll" $(OutDir) - - xcopy /y "C:\mlpack\mlpack-3.2.2\packages\OpenBLAS.0.2.14.1\lib\native\bin\x64\*.dll" $(OutDir) + - xcopy /y "C:\mlpack\mlpack-3.3.0\build\Debug\mlpack.dll" $(OutDir) + - xcopy /y "C:\mlpack\mlpack-3.3.0\packages\OpenBLAS.0.2.14.1\lib\native\bin\x64\*.dll" $(OutDir) @endcode @note Recent versions of Visual Studio set "Conformance Mode" enabled by default. This causes some issues with diff --git a/src/mlpack/CMakeLists.txt b/src/mlpack/CMakeLists.txt index 02d60b0c1c..2b2513daff 100644 --- a/src/mlpack/CMakeLists.txt +++ b/src/mlpack/CMakeLists.txt @@ -44,7 +44,7 @@ target_link_libraries(mlpack ${MLPACK_LIBRARIES}) set_target_properties(mlpack PROPERTIES - VERSION 3.2 + VERSION 3.3 SOVERSION 3 ) diff --git a/src/mlpack/core/util/version.hpp b/src/mlpack/core/util/version.hpp index 14d8daa822..2f1914e16d 100644 --- a/src/mlpack/core/util/version.hpp +++ b/src/mlpack/core/util/version.hpp @@ -17,8 +17,8 @@ // The version of mlpack. If this is a git repository, this will be a version // with higher number than the most recent release. #define MLPACK_VERSION_MAJOR 3 -#define MLPACK_VERSION_MINOR 2 -#define MLPACK_VERSION_PATCH 3 +#define MLPACK_VERSION_MINOR 3 +#define MLPACK_VERSION_PATCH 0 // The name of the version (for use by --version). namespace mlpack { From a5814d0c5cd802bcf5978377010de25e561febb4 Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Tue, 7 Apr 2020 09:17:16 -0400 Subject: [PATCH 108/111] Update version to next release version. --- CMakeLists.txt | 8 ++++---- src/mlpack/core/util/version.hpp | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 4bb2eef1ba..4dc6d0beb8 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -359,14 +359,14 @@ endif () find_package(Ensmallen 2.10.0) if (NOT ENSMALLEN_FOUND) if (DOWNLOAD_ENSMALLEN) - file(DOWNLOAD http://www.ensmallen.org/files/ensmallen-2.12.0.tar.gz - "${CMAKE_BINARY_DIR}/deps/ensmallen-2.12.0.tar.gz" + file(DOWNLOAD http://www.ensmallen.org/files/ensmallen-latest.tar.gz + "${CMAKE_BINARY_DIR}/deps/ensmallen-latest.tar.gz" STATUS ENS_DOWNLOAD_STATUS_LIST LOG ENS_DOWNLOAD_LOG SHOW_PROGRESS) list(GET ENS_DOWNLOAD_STATUS_LIST 0 ENS_DOWNLOAD_STATUS) if (ENS_DOWNLOAD_STATUS EQUAL 0) execute_process(COMMAND ${CMAKE_COMMAND} -E - tar xzf "${CMAKE_BINARY_DIR}/deps/ensmallen-2.12.0.tar.gz" + tar xzf "${CMAKE_BINARY_DIR}/deps/ensmallen-latest.tar.gz" WORKING_DIRECTORY "${CMAKE_BINARY_DIR}/deps/") # Get the name of the directory. @@ -374,7 +374,7 @@ if (NOT ENSMALLEN_FOUND) "${CMAKE_BINARY_DIR}/deps/ensmallen-[0-9]*.[0-9]*.[0-9]*") # list(FILTER) is not available on 3.5 or older, but try to keep # configuring without filtering the list anyway (it might work if only - # the file ensmallen-2.12.0.tar.gz is present. + # the file ensmallen-latest.tar.gz is present. if (${CMAKE_VERSION} VERSION_GREATER_EQUAL "3.6.0") list(FILTER ENS_DIRECTORIES EXCLUDE REGEX "ensmallen-.*\.tar\.gz") endif () diff --git a/src/mlpack/core/util/version.hpp b/src/mlpack/core/util/version.hpp index 2f1914e16d..a1c421d0fa 100644 --- a/src/mlpack/core/util/version.hpp +++ b/src/mlpack/core/util/version.hpp @@ -18,7 +18,7 @@ // with higher number than the most recent release. #define MLPACK_VERSION_MAJOR 3 #define MLPACK_VERSION_MINOR 3 -#define MLPACK_VERSION_PATCH 0 +#define MLPACK_VERSION_PATCH 1 // The name of the version (for use by --version). namespace mlpack { From 5e55b16ce5a00e704288bcfb4d06f822662d2987 Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Tue, 7 Apr 2020 09:17:16 -0400 Subject: [PATCH 109/111] Add new block to HISTORY.md for next version. --- HISTORY.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/HISTORY.md b/HISTORY.md index 9279dc9fe7..8412db4856 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,3 +1,6 @@ +### mlpack ?.?.? +###### ????-??-?? + ### mlpack 3.3.0 ###### 2020-04-07 * Templated return type of `Forward function` of loss functions (#2339). From 81c02c8e9cdd2a5bbf1cf59131480b668d5e75a3 Mon Sep 17 00:00:00 2001 From: Ryan Curtin Date: Tue, 7 Apr 2020 09:24:28 -0400 Subject: [PATCH 110/111] Update name of link. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 0220d4bc2b..d8f4c5c0ec 100644 --- a/README.md +++ b/README.md @@ -23,7 +23,7 @@ src="https://cdn.rawgit.com/mlpack/mlpack.org/e7d36ed8/mlpack-black.svg" style="

Download: - current stable version (3.2.2) + current stable version (3.3.0)

From 3382b7238b8bb3832830e8bc5f732349919b150a Mon Sep 17 00:00:00 2001 From: Nishant Kumar Date: Tue, 7 Apr 2020 23:57:33 +0530 Subject: [PATCH 111/111] Addition of q_networks (#2317) * added q_network * added vanilla dqn, along with tests * added comments * style errors removed * style error removed * Revert "style error removed" This reverts commit 578601546d47f7a0df3d3a2d05e13c21d8492c0b. * style errors removed * Update src/mlpack/methods/reinforcement_learning/q_networks/vanilla_dqn.hpp Co-Authored-By: Marcus Edel * made suggested changes in vanillaDQN * changed vanillaDQN to simpleDQN * backward compatibility for custom models Co-authored-by: Marcus Edel --- .../q_networks/CMakeLists.txt | 14 ++ .../q_networks/simple_dqn.hpp | 127 ++++++++++++++++++ src/mlpack/tests/q_learning_test.cpp | 44 ++---- src/mlpack/tests/reward_clipping_test.cpp | 9 +- 4 files changed, 151 insertions(+), 43 deletions(-) create mode 100644 src/mlpack/methods/reinforcement_learning/q_networks/CMakeLists.txt create mode 100644 src/mlpack/methods/reinforcement_learning/q_networks/simple_dqn.hpp diff --git a/src/mlpack/methods/reinforcement_learning/q_networks/CMakeLists.txt b/src/mlpack/methods/reinforcement_learning/q_networks/CMakeLists.txt new file mode 100644 index 0000000000..3a8a010bfe --- /dev/null +++ b/src/mlpack/methods/reinforcement_learning/q_networks/CMakeLists.txt @@ -0,0 +1,14 @@ +# Define the files we need to compile +# Anything not in this list will not be compiled into mlpack. +set(SOURCES + simple_dqn.hpp +) + +# Add directory name to sources. +set(DIR_SRCS) +foreach(file ${SOURCES}) + set(DIR_SRCS ${DIR_SRCS} ${CMAKE_CURRENT_SOURCE_DIR}/${file}) +endforeach() +# Append sources (with directory name) to list of all mlpack sources (used at +# the parent scope). +set(MLPACK_SRCS ${MLPACK_SRCS} ${DIR_SRCS} PARENT_SCOPE) \ No newline at end of file diff --git a/src/mlpack/methods/reinforcement_learning/q_networks/simple_dqn.hpp b/src/mlpack/methods/reinforcement_learning/q_networks/simple_dqn.hpp new file mode 100644 index 0000000000..30d13180a5 --- /dev/null +++ b/src/mlpack/methods/reinforcement_learning/q_networks/simple_dqn.hpp @@ -0,0 +1,127 @@ +/** + * @file simple_dqn.hpp + * @author Nishant Kumar + * + * This file contains the implementation of the simple deep q network. + * + * mlpack is free software; you may redistribute it and/or modify it under the + * terms of the 3-clause BSD license. You should have received a copy of the + * 3-clause BSD license along with mlpack. If not, see + * http://www.opensource.org/licenses/BSD-3-Clause for more information. + */ +#ifndef MLPACK_METHODS_RL_SIMPLE_DQN_HPP +#define MLPACK_METHODS_RL_SIMPLE_DQN_HPP + +#include +#include +#include +#include +#include + +namespace mlpack { +namespace rl { + +using namespace mlpack::ann; + +/** + * @tparam NetworkType The type of network used for simple dqn. + */ +template , + GaussianInitialization>> +class SimpleDQN +{ + public: + /** + * Default constructor. + */ + SimpleDQN() : network() + { /* Nothing to do here. */ } + + /** + * Construct an instance of SimpleDQN class. + * + * @param inputDim Number of inputs. + * @param h1 Number of neurons in hiddenlayer-1. + * @param h2 Number of neurons in hiddenlayer-2. + * @param outputDim Number of neurons in output layer. + */ + SimpleDQN(const int inputDim, + const int h1, + const int h2, + const int outputDim) : network() + { + FFN, GaussianInitialization> model(MeanSquaredError<>(), + GaussianInitialization(0, 0.001)); + model.Add>(inputDim, h1); + model.Add>(); + model.Add>(h1, h2); + model.Add>(); + model.Add>(h2, outputDim); + network = model; + } + + SimpleDQN(NetworkType network) : network(std::move(network)) + { /* Nothing to do here. */ } + + /** + * Predict the responses to a given set of predictors. The responses will + * reflect the output of the given output layer as returned by the + * output layer function. + * + * If you want to pass in a parameter and discard the original parameter + * object, be sure to use std::move to avoid unnecessary copy. + * + * @param state Input state. + * @param actionValue Matrix to put output action values of states input. + */ + void Predict(const arma::mat state, arma::mat& actionValue) + { + network.Predict(state, actionValue); + } + + /** + * Perform the forward pass of the states in real batch mode. + * + * @param state The input state. + * @param target The predicted target. + */ + void Forward(const arma::mat state, arma::mat& target) + { + network.Forward(state, target); + } + + /** + * Resets the parameters of the network. + */ + void ResetParameters() + { + network.ResetParameters(); + } + + //! Return the Parameters. + const arma::mat& Parameters() const { return network.Parameters(); } + //! Modify the Parameters. + arma::mat& Parameters() { return network.Parameters(); } + + /** + * Perform the backward pass of the state in real batch mode. + * + * @param state The input state. + * @param target The training target. + * @return gradient The gradient. + */ + void Backward(const arma::mat state, arma::mat& target, +arma::mat& gradient) + { + network.Backward(state, target, gradient); + } + + private: + //! Locally-stored network. + NetworkType network; +}; + +} // namespace rl +} // namespace mlpack + +#endif diff --git a/src/mlpack/tests/q_learning_test.cpp b/src/mlpack/tests/q_learning_test.cpp index 95a26f40ad..67a62d651d 100644 --- a/src/mlpack/tests/q_learning_test.cpp +++ b/src/mlpack/tests/q_learning_test.cpp @@ -18,6 +18,7 @@ #include #include #include +#include #include #include #include @@ -41,13 +42,7 @@ BOOST_AUTO_TEST_SUITE(QLearningTest); BOOST_AUTO_TEST_CASE(CartPoleWithDQN) { // Set up the network. - FFN, GaussianInitialization> model(MeanSquaredError<>(), - GaussianInitialization(0, 0.001)); - model.Add>(4, 128); - model.Add>(); - model.Add>(128, 128); - model.Add>(); - model.Add>(128, 2); + SimpleDQN<> model(4, 128, 128, 2); // Set up the policy and replay method. GreedyPolicy policy(1.0, 1000, 0.1, 0.99); @@ -107,13 +102,7 @@ BOOST_AUTO_TEST_CASE(CartPoleWithDQN) BOOST_AUTO_TEST_CASE(CartPoleWithDQNPrioritizedReplay) { // Set up the network. - FFN, GaussianInitialization> model(MeanSquaredError<>(), - GaussianInitialization(0, 0.001)); - model.Add>(4, 128); - model.Add>(); - model.Add>(128, 128); - model.Add>(); - model.Add>(128, 2); + SimpleDQN<> model(4, 128, 128, 2); // Set up the policy and replay method. GreedyPolicy policy(1.0, 1000, 0.1); @@ -182,13 +171,7 @@ BOOST_AUTO_TEST_CASE(CartPoleWithDoubleDQN) for (size_t trial = 0; trial < 4; ++trial) { // Set up the network. - FFN, GaussianInitialization> model(MeanSquaredError<>(), - GaussianInitialization(0, 0.001)); - model.Add>(4, 20); - model.Add>(); - model.Add>(20, 20); - model.Add>(); - model.Add>(20, 2); + SimpleDQN<> model(4, 20, 20, 2); // Set up the policy and replay method. GreedyPolicy policy(1.0, 1000, 0.1, 0.99); @@ -251,13 +234,7 @@ BOOST_AUTO_TEST_CASE(AcrobotWithDQN) for (size_t trial = 0; trial < 3; ++trial) { // Set up the network. - FFN, GaussianInitialization> model(MeanSquaredError<>(), - GaussianInitialization(0, 0.001)); - model.Add>(4, 64); - model.Add>(); - model.Add>(64, 32); - model.Add>(); - model.Add>(32, 3); + SimpleDQN<> model(4, 64, 32, 3); // Set up the policy and replay method. GreedyPolicy policy(1.0, 1000, 0.1, 0.99); @@ -328,13 +305,7 @@ BOOST_AUTO_TEST_CASE(MountainCarWithDQN) for (size_t trial = 0; trial < 3; trial++) { // Set up the network. - FFN, GaussianInitialization> model(MeanSquaredError<>(), - GaussianInitialization(0, 0.001)); - model.Add>(2, 64); - model.Add>(); - model.Add>(64, 32); - model.Add>(); - model.Add>(32, 3); + SimpleDQN<> model(2, 64, 32, 3); // Set up the policy and replay method. GreedyPolicy policy(1.0, 1000, 0.1, 0.99); @@ -404,7 +375,8 @@ BOOST_AUTO_TEST_CASE(DoublePoleCartWithDQN) bool success = false; for (size_t trial = 0; trial < 4; trial++) { - // Set up the network. + // Set up the network. Note that we use a custom model here, and + // pass it directly into the agent, without using SimpleDQN. FFN, GaussianInitialization> model(MeanSquaredError<>(), GaussianInitialization(0, 0.001)); model.Add>(6, 256); diff --git a/src/mlpack/tests/reward_clipping_test.cpp b/src/mlpack/tests/reward_clipping_test.cpp index 335b4b8200..e4b0c2b121 100644 --- a/src/mlpack/tests/reward_clipping_test.cpp +++ b/src/mlpack/tests/reward_clipping_test.cpp @@ -13,6 +13,7 @@ #include #include +#include #include #include #include @@ -63,13 +64,7 @@ BOOST_AUTO_TEST_CASE(RewardClippedAcrobotWithDQN) for (size_t trial = 0; trial < 3; ++trial) { // Set up the network. - FFN, GaussianInitialization> model(MeanSquaredError<>(), - GaussianInitialization(0, 0.001)); - model.Add>(4, 64); - model.Add>(); - model.Add>(64, 32); - model.Add>(); - model.Add>(32, 3); + SimpleDQN<> model(4, 64, 32, 3); // Set up the policy and replay method. GreedyPolicy> policy(1.0, 1000, 0.1, 0.99);